void updateModelGivenUrlsSampleRecord(TextBytes inputData) {
int curpos = inputData.getOffset();
int endpos = inputData.getOffset() + inputData.getLength();
byte lfPattern[] = {0xA};
byte tabPattern[] = {0x9};
TextBytes urlText = new TextBytes();
while (curpos != endpos) {
int tabIndex =
ByteArrayUtils.indexOf(inputData.getBytes(), curpos, endpos - curpos, tabPattern);
if (tabIndex == -1) {
break;
} else {
int lfIndex =
ByteArrayUtils.indexOf(
inputData.getBytes(), tabIndex + 1, endpos - (tabIndex + 1), lfPattern);
if (lfIndex == -1) {
break;
} else {
long sourceDomainHash =
ByteArrayUtils.parseLong(inputData.getBytes(), curpos, tabIndex - curpos, 10);
urlText.set(inputData.getBytes(), tabIndex + 1, lfIndex - (tabIndex + 1));
incoming.put(sourceDomainHash, urlText.toString());
curpos = lfIndex + 1;
}
}
}
}
void emitLastRecord(Reporter reporter) throws IOException {
if (_flags != 0) {
if (_domainStats == null) {
reporter.incrCounter(Counters.EMITTING_URL_RECORD_WITH_NULL_DOMAINSTATS, 1);
} else {
reporter.incrCounter(Counters.EMITTING_URL_RECORD_WITH_DOMINSTATS, 1);
}
if (_crawlStatus != null) {
reporter.incrCounter(Counters.EMITTED_RECORD_HAD_CRAWLSTATUS, 1);
} else {
reporter.incrCounter(Counters.EMITTED_RECORD_HAD_NULL_CRAWLSTATUS, 1);
}
}
if (_contextURLBytes.getLength() >= 4097) {
reporter.incrCounter(Counters.SKIPPING_INVALID_LENGTH_URL, 1);
} else {
GoogleURL urlObject = new GoogleURL(_contextURLBytes.toString());
if (!skipRecord(urlObject, reporter)) {
if (urlObject.has_query()) {
reporter.incrCounter(Counters.LET_THROUGH_QUERY_URL, 1);
}
URLFPV2 fp = URLUtils.getURLFPV2FromURLObject(urlObject);
if (fp != null) {
if (_emittedURLSFilter.isPresent(fp)) {
reporter.incrCounter(Counters.SKIPPING_ALREADY_EMITTED_URL, 1);
} else {
_emittedURLSFilter.add(fp);
_emittedURLSInFilter++;
SegmentGeneratorItem itemValue = new SegmentGeneratorItem();
itemValue.setDomainFP(fp.getDomainHash());
itemValue.setRootDomainFP(fp.getRootDomainHash());
itemValue.setUrlFP(fp.getUrlHash());
itemValue.setUrl(urlObject.getCanonicalURL());
itemValue.setPageRank(0);
itemValue.setModifiedStatus((byte) 0);
items.add(itemValue);
if (items.size() >= SPILL_THRESHOLD) spillItems(reporter);
}
} else {
reporter.incrCounter(Counters.NULL_FP_FOR_URL, 1);
}
}
}
// reset stuff
_flags = 0;
_crawlStatus = null;
_contextURLBytes.clear();
_blogURLSkipFlag.set(true);
}
