boolean skipRecord(GoogleURL urlObject, Reporter reporter) {

    if (_skipDomain) {
      reporter.incrCounter(Counters.SKIPPING_BAD_DOMAIN_URL, 1);
      return true;
    }

    if (!urlObject.isValid()) {
      reporter.incrCounter(Counters.SKIPPING_INVALID_URL, 1);
      return true;
    } else if (urlObject.has_query()) {
      reporter.incrCounter(Counters.HIT_QUERY_CHECK_CONDITION, 1);
      if ((_flags & (HAS_HOMEPAGE_URLDATA | HAS_FEED_URLDATA)) == 0) {
        reporter.incrCounter(Counters.SKIPPING_QUERY_URL, 1);
        return true;
      }
    } else {
      // if redirect ... skip
      if ((_flags & HAS_REDIRECT_DATA) != 0) {
        reporter.incrCounter(Counters.SKIPPING_REDIRECTED_URL, 1);
        return true;
      }

      if ((_flags & (HAS_HOMEPAGE_URLDATA | HAS_FEED_URLDATA)) != 0) {
        if (!_skipEverythingButHomepage || ((_flags & HAS_HOMEPAGE_URLDATA) != 0)) {
          reporter.incrCounter(Counters.ALLOWING_HOMEPAGE_OR_FEEDURL, 1);
          return false;
        }
      }

      if (_skipEverythingButHomepage) {
        reporter.incrCounter(Counters.SKIPPING_EVERYTHING_BUT_HOMEPAGE_URL, 1);
        return true;
      }

      if (_crawlStatus != null) {
        if (_crawlStatus.has("crawl_status")) {
          JsonObject realCrawlStatus = _crawlStatus.get("crawl_status").getAsJsonObject();
          if (realCrawlStatus.has("http_result")) {
            int httpResult = realCrawlStatus.get("http_result").getAsInt();
            if (httpResult == 200 || httpResult == 404) {
              if ((_flags & HAS_BLOGPROBE_URLDATA) != 0) {
                if (_blogURLSkipFlag.get()) {
                  reporter.incrCounter(Counters.SKIPPING_BLOGPROBE_URL, 1);
                  return true;
                } else {
                  reporter.incrCounter(Counters.RECRAWLING_BLOGPROBE_URL, 1);
                  return false;
                }
              } else {
                reporter.incrCounter(Counters.SKIPPING_ALREADY_FETCHED, 1);
                return true;
              }
            }
          }
        }
      }
    }
    return false;
  }
  void emitLastRecord(Reporter reporter) throws IOException {

    if (_flags != 0) {
      if (_domainStats == null) {
        reporter.incrCounter(Counters.EMITTING_URL_RECORD_WITH_NULL_DOMAINSTATS, 1);
      } else {
        reporter.incrCounter(Counters.EMITTING_URL_RECORD_WITH_DOMINSTATS, 1);
      }

      if (_crawlStatus != null) {
        reporter.incrCounter(Counters.EMITTED_RECORD_HAD_CRAWLSTATUS, 1);
      } else {
        reporter.incrCounter(Counters.EMITTED_RECORD_HAD_NULL_CRAWLSTATUS, 1);
      }
    }

    if (_contextURLBytes.getLength() >= 4097) {
      reporter.incrCounter(Counters.SKIPPING_INVALID_LENGTH_URL, 1);
    } else {
      GoogleURL urlObject = new GoogleURL(_contextURLBytes.toString());

      if (!skipRecord(urlObject, reporter)) {

        if (urlObject.has_query()) {
          reporter.incrCounter(Counters.LET_THROUGH_QUERY_URL, 1);
        }

        URLFPV2 fp = URLUtils.getURLFPV2FromURLObject(urlObject);
        if (fp != null) {
          if (_emittedURLSFilter.isPresent(fp)) {
            reporter.incrCounter(Counters.SKIPPING_ALREADY_EMITTED_URL, 1);
          } else {
            _emittedURLSFilter.add(fp);
            _emittedURLSInFilter++;

            SegmentGeneratorItem itemValue = new SegmentGeneratorItem();

            itemValue.setDomainFP(fp.getDomainHash());
            itemValue.setRootDomainFP(fp.getRootDomainHash());
            itemValue.setUrlFP(fp.getUrlHash());
            itemValue.setUrl(urlObject.getCanonicalURL());
            itemValue.setPageRank(0);
            itemValue.setModifiedStatus((byte) 0);

            items.add(itemValue);

            if (items.size() >= SPILL_THRESHOLD) spillItems(reporter);
          }
        } else {
          reporter.incrCounter(Counters.NULL_FP_FOR_URL, 1);
        }
      }
    }

    // reset stuff
    _flags = 0;
    _crawlStatus = null;
    _contextURLBytes.clear();
    _blogURLSkipFlag.set(true);
  }