@Override
public void visit(Page page) {
String url = page.getWebURL().getURL();
// We are only interested in processing images
if (!(page.getParseData() instanceof BinaryParseData)) {
return;
}
if (!imgPatterns.matcher(url).matches()) {
return;
}
// Not interested in very small images
if (page.getContentData().length < 10 * 1024) {
return;
}
// get a unique name for storing this image
String extension = url.substring(url.lastIndexOf("."));
String hashedName = Cryptography.MD5(url) + extension;
// store image
IO.writeBytesToFile(page.getContentData(), storageFolder.getAbsolutePath() + "/" + hashedName);
System.out.println("Stored: " + url);
}
