void updateModelGivenUrlsSampleRecord(TextBytes inputData) {
      int curpos = inputData.getOffset();
      int endpos = inputData.getOffset() + inputData.getLength();

      byte lfPattern[] = {0xA};
      byte tabPattern[] = {0x9};

      TextBytes urlText = new TextBytes();

      while (curpos != endpos) {
        int tabIndex =
            ByteArrayUtils.indexOf(inputData.getBytes(), curpos, endpos - curpos, tabPattern);
        if (tabIndex == -1) {
          break;
        } else {
          int lfIndex =
              ByteArrayUtils.indexOf(
                  inputData.getBytes(), tabIndex + 1, endpos - (tabIndex + 1), lfPattern);
          if (lfIndex == -1) {
            break;
          } else {
            long sourceDomainHash =
                ByteArrayUtils.parseLong(inputData.getBytes(), curpos, tabIndex - curpos, 10);
            urlText.set(inputData.getBytes(), tabIndex + 1, lfIndex - (tabIndex + 1));
            incoming.put(sourceDomainHash, urlText.toString());
            curpos = lfIndex + 1;
          }
        }
      }
    }
 private static void rawValueToTextBytes(
     DataOutputBuffer dataBuffer, DataInputBuffer inputBuffer, TextBytes textOut)
     throws IOException {
   inputBuffer.reset(dataBuffer.getData(), dataBuffer.getLength());
   int newLength = WritableUtils.readVInt(inputBuffer);
   textOut.set(inputBuffer.getData(), inputBuffer.getPosition(), newLength);
 }
  @Test
  public void testSourceInputOutputWriters() throws IOException {
    _sourceInputsBuffer = new DataOutputBuffer(16348 * 4);
    _sourceInputsTrackingFilter = new URLFPBloomFilter(100000, NUM_HASH_FUNCTIONS, NUM_BITS);

    String sourceDomainURL = "http://sourcedomain.com/foo";
    URLFPV2 sourceFP = URLUtils.getURLFPV2FromCanonicalURL(sourceDomainURL);

    String urls[] = {"http://somedomain.com/foo", "http://someother.com/bar"};

    for (String url : urls) {
      URLFPV2 fp = URLUtils.getURLFPV2FromCanonicalURL(url);
      // double insert and validate actual single insertion
      trackPotentialLinkSource(fp, url, sourceFP);
      trackPotentialLinkSource(fp, url, sourceFP);
    }

    //  validate data ...
    TextBytes firstVersion = new TextBytes();
    firstVersion.set(_sourceInputsBuffer.getData(), 0, _sourceInputsBuffer.getLength());

    StringTokenizer tokenizer = new StringTokenizer(firstVersion.toString(), "\n");
    int itemIndex = 0;
    while (tokenizer.hasMoreElements()) {
      String nextLine = tokenizer.nextToken();
      String splits[] = nextLine.split("\t");
      // validate fp
      URLFPV2 fp = URLUtils.getURLFPV2FromCanonicalURL(urls[itemIndex]);
      Assert.assertEquals(fp.getDomainHash(), Long.parseLong(splits[0]));
      // validate actual url ...
      Assert.assertEquals(splits[1], urls[itemIndex]);
      itemIndex++;
    }

    // reset output buffer ...
    _sourceInputsBuffer = new DataOutputBuffer(16348 * 4);
    // and source bloom filter ...
    _sourceInputsTrackingFilter = new URLFPBloomFilter(10000000, NUM_HASH_FUNCTIONS, NUM_BITS);
    importLinkSourceData(sourceFP, firstVersion);
    // second text should match first ..
    TextBytes secondVersion = new TextBytes();
    secondVersion.set(_sourceInputsBuffer.getData(), 0, _sourceInputsBuffer.getLength());
    Assert.assertEquals(firstVersion, secondVersion);
  }
  void iterateItems(MultiFileInputReader<TextBytes> multiFileInputReader, Reporter reporter)
      throws IOException {

    Pair<KeyAndValueData<TextBytes>, Iterable<RawRecordValue>> nextItem = null;

    int iterationCount = 0;

    while ((nextItem = multiFileInputReader.getNextItemIterator()) != null) {

      reporter.incrCounter(Counters.GOT_RECORD, 1);

      int type = PartitionUtils.getTypeGivenPartitionKey(nextItem.e0._keyObject);
      PartitionUtils.getDomainGivenPartitionKey(nextItem.e0._keyObject, _newDomainBytes);
      PartitionUtils.getURLGivenPartitionKey(nextItem.e0._keyObject, _newURLBytes);

      if (_newURLBytes.compareTo(_contextURLBytes) != 0) {
        emitLastRecord(reporter);
      }

      long newDomainFP = SuperDomainList.domainFingerprintGivenName(_newDomainBytes.toString());

      if (newDomainFP != _currentDomainId) {
        reporter.incrCounter(Counters.TRANSITIONING_DOMAIN, 1);
        domainTransition(newDomainFP, _newDomainBytes.toString(), reporter);
      }

      RawRecordValue valueRaw = Iterables.getFirst(nextItem.e1, null);

      switch (type) {
        case CrawlListGeneratorTask.KEY_TYPE_CRAWLSTATS:
          {
            reporter.incrCounter(Counters.GOT_CRAWLSTATS, 1);
            setDomainStats(
                rawValueToJsonObject(valueRaw.data, tempBuffer, tempTextBuffer), reporter);
          }
          break;

        case CrawlListGeneratorTask.KEY_TYPE_HOMEPAGE_URL:
          {
            reporter.incrCounter(Counters.GOT_HOMEPAGE_DATA, 1);
            rawValueToTextBytes(valueRaw.data, tempBuffer, _contextURLBytes);
            _flags |= HAS_HOMEPAGE_URLDATA;
          }
          break;

        case CrawlListGeneratorTask.KEY_TYPE_BLOGPROBE_URL:
          {
            reporter.incrCounter(Counters.GOT_BLOGPROBE_DATA, 1);
            rawValueToWritable(valueRaw, tempBuffer, _blogURLSkipFlag);
            _contextURLBytes.set(_newURLBytes);
            _flags |= HAS_BLOGPROBE_URLDATA;
          }
          break;

        case CrawlListGeneratorTask.KEY_TYPE_FEED_URL:
          {
            reporter.incrCounter(Counters.GOT_FEEDURL_DATA, 1);
            rawValueToTextBytes(valueRaw.data, tempBuffer, _contextURLBytes);
            _flags |= HAS_FEED_URLDATA;
          }
          break;

        case CrawlListGeneratorTask.KEY_TYPE_REDIRECT_RECORD:
          {
            reporter.incrCounter(Counters.GOT_REDIRECT_DATA, 1);
            _contextURLBytes.set(_newURLBytes);
            _flags |= HAS_REDIRECT_DATA;
          }
          break;

        case CrawlListGeneratorTask.KEY_TYPE_CRAWLDATA:
          {
            reporter.incrCounter(Counters.GOT_CRAWLURL_DATA, 1);
            _contextURLBytes.set(_newURLBytes);
            _crawlStatus = rawValueToJsonObject(valueRaw.data, tempBuffer, tempTextBuffer);
            _flags |= HAS_CRAWL_STATUS;
          }
          break;
      }
    }
    // flush trailing record ...
    emitLastRecord(reporter);
    flushDomain(reporter);
  }