@Test
public void testSourceInputOutputWriters() throws IOException {
_sourceInputsBuffer = new DataOutputBuffer(16348 * 4);
_sourceInputsTrackingFilter = new URLFPBloomFilter(100000, NUM_HASH_FUNCTIONS, NUM_BITS);
String sourceDomainURL = "http://sourcedomain.com/foo";
URLFPV2 sourceFP = URLUtils.getURLFPV2FromCanonicalURL(sourceDomainURL);
String urls[] = {"http://somedomain.com/foo", "http://someother.com/bar"};
for (String url : urls) {
URLFPV2 fp = URLUtils.getURLFPV2FromCanonicalURL(url);
// double insert and validate actual single insertion
trackPotentialLinkSource(fp, url, sourceFP);
trackPotentialLinkSource(fp, url, sourceFP);
}
// validate data ...
TextBytes firstVersion = new TextBytes();
firstVersion.set(_sourceInputsBuffer.getData(), 0, _sourceInputsBuffer.getLength());
StringTokenizer tokenizer = new StringTokenizer(firstVersion.toString(), "\n");
int itemIndex = 0;
while (tokenizer.hasMoreElements()) {
String nextLine = tokenizer.nextToken();
String splits[] = nextLine.split("\t");
// validate fp
URLFPV2 fp = URLUtils.getURLFPV2FromCanonicalURL(urls[itemIndex]);
Assert.assertEquals(fp.getDomainHash(), Long.parseLong(splits[0]));
// validate actual url ...
Assert.assertEquals(splits[1], urls[itemIndex]);
itemIndex++;
}
// reset output buffer ...
_sourceInputsBuffer = new DataOutputBuffer(16348 * 4);
// and source bloom filter ...
_sourceInputsTrackingFilter = new URLFPBloomFilter(10000000, NUM_HASH_FUNCTIONS, NUM_BITS);
importLinkSourceData(sourceFP, firstVersion);
// second text should match first ..
TextBytes secondVersion = new TextBytes();
secondVersion.set(_sourceInputsBuffer.getData(), 0, _sourceInputsBuffer.getLength());
Assert.assertEquals(firstVersion, secondVersion);
}
