public void getIndexInfo(String indexdir, int freqThreshold) {
IndexReader reader = null;
try {
Directory dir = FSDirectory.open(new File(indexdir));
System.out.println(dir);
reader = IndexReader.open(dir);
System.out.println("document num:" + reader.numDocs());
System.out.println("======================");
TermEnum terms = reader.terms();
sortedTermQueue.clear();
maxDocNum = reader.maxDoc();
linkMap.clear();
termList.clear();
while (terms.next()) {
// System.out.print(terms.term() + "\tDocFreq:" +
TermDocs termDocs = reader.termDocs(terms.term());
MyTerm temp = new MyTerm(terms.term(), termDocs, maxDocNum);
if (temp.totalFreq < freqThreshold) {
continue;
} /*
* if(temp.originTrem.text().length()==1){ continue; }
*/
linkMap.put(temp.originTrem.text(), temp);
sortedTermQueue.add(temp);
termList.add(temp);
}
System.out.println("total Size:" + sortedTermQueue.size());
System.out.println("mapsize:" + linkMap.keySet().size());
// System.exit(0);
int num = 0;
this.maxFreq = sortedTermQueue.peek().totalFreq;
while (!sortedTermQueue.isEmpty()) {
num++;
System.out.println(num + ":" + sortedTermQueue.poll());
}
System.out.println("read index info done");
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
reader.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
void clear() {
pending_entries.clear();
output_set.clear();
object_tasks.clear();
active_threads.clear();
time_marks.clear();
thread_entries = null;
next_time = 0;
end_time = 0;
current_thread = null;
thread_map.clear();
cpu_time = null;
thread_counter = 0;
task_counter = 0;
max_delta = 1;
}
// 清除所有优先队列的数据，设置优先种子
private static void SetSeeds(String filepath) throws Exception {
pq.clear();
BufferedReader br = new BufferedReader(new FileReader(filepath));
String line = null;
UrlValue cur = null;
visitedPrint();
while ((line = br.readLine()) != null) {
line = line.trim();
// System.out.println(line);
if (!line.equals("")) {
if (!visitedUrl.contains(line)) {
cur = new UrlValue();
cur.url = line;
cur.value = 1;
pq.offer(cur);
} else {
System.out.println("contain");
}
}
}
br.close();
}
public static void clear() {
pq.clear();
}
