@Override
public void objectToEntry(WebURL url, TupleOutput output) {
output.writeString(url.getPrimeKey());
output.writeString(url.getURL());
output.writeInt(url.getDocid());
output.writeInt(url.getParentDocid());
output.writeString(url.getParentUrl());
output.writeShort(url.getDepth());
output.writeString(url.getCookie());
output.writeString(url.getFormParams());
output.writeString(url.getMethod());
}
public void downLoadText(String pageText, WebURL webUrl) throws IOException {
int docid = webUrl.getDocid();
File file_output = new File("E:/crawlData/pageText/" + docid + ".txt");
if (!file_output.exists()) {
File parent = new File(file_output.getParent());
if (!parent.exists()) {
parent.mkdirs();
}
}
FileWriter fWriter = new FileWriter(file_output);
fWriter.write(pageText);
fWriter.close();
}
