public ArrayList<JsonObject> PaginateItems(ArrayList<JsonObject> items, PageInfo requiredPage) {
if (requiredPage == null) return items;
int hiIndex = requiredPage.GetHiIndex();
if (hiIndex == 0) return items;
int lowIndex = requiredPage.GetLowIndex();
hiIndex = Math.min(hiIndex, items.size());
lowIndex = Math.min(lowIndex, items.size());
return new ArrayList<JsonObject>(items.subList(lowIndex, hiIndex));
}
public void run(int topNum, int minEntrane) throws Exception {
lstHost = getHostInfos();
HostInjectTool tool = new HostInjectTool();
for (HostInfo info : lstHost) {
final File dir = ScanConfig.instance.getSaveDir(info);
System.out.println("checking " + dir);
if (!dir.exists()) {
System.out.println("skip empty " + dir);
continue;
}
try {
List<PageInfo> top = this.getTop(dir, info, topNum);
for (PageInfo pi : top) {
System.out.println(pi);
}
if (top.size() < minEntrane) {
System.out.println("skip " + info + ", too few entrance " + top.size());
} else {
// Consoler.readString("-----press any key to continue>");
for (PageInfo pi : top) {
try {
tool.injectAlwaysAllow(info, pi.getUrl());
} catch (Exception e) {
e.printStackTrace();
}
}
setted.add(info);
}
HostInitorTool initor = new HostInitorTool(info.getHostName(), 1024 * 32);
try {
initor.markAllUrlToCrawled();
} finally {
initor.close();
}
Connectioner.instance().setHostStatus(info.getHostName(), HostFilter.Status_Incr_watch);
if ("delOrg".equalsIgnoreCase(System.getProperty("isDelOrg"))) {
System.out.println("deleteing ................... " + dir);
Misc.del(dir);
}
} catch (Exception e) {
e.printStackTrace();
}
}
tool.close();
}
public HashMap<String, Integer> parse() {
String[] words = parseWordsFromPage(m_PageContent);
// System.out.println("before count words:" + words.length);
HashMap<String, Integer> map = CountWords(words);
// System.out.println("after count words:" + map.size());
// String[] idxseg = m_urlindex.split(" ");
PageInfo info = new PageInfo(m_urlindex);
info.setPageOffset(this.m_CurrentOffset);
info.setFileID(this.m_DataFileID);
info.setPageLength(this.m_PageContent.length());
int word_count = words.length;
info.setWordCount(word_count);
m_docIDList.add(info);
return map;
}
public List queryCriteria(ISession s, DetachedCriteria detachedCrit, PageInfo info) {
Session session = getSession(s);
List result = Collections.EMPTY_LIST;
try {
Profiler.start("QueryCriteria():");
Criteria criteria = detachedCrit.getExecutableCriteria(session);
criteria.addOrder(Order.asc("alias"));
if (info != null) {
if (info.getFirstResult() != null) {
criteria.setFirstResult(info.getFirstResult());
}
if (info.getMaxResult() != null) {
criteria.setMaxResults(info.getMaxResult());
}
}
result = criteria.list();
} finally {
Profiler.stop();
closeSession(session);
}
return (result);
}
/**
* Based on the gunzip1 implementation.
*
* <p>Takes 9ms for the 100kb test document on the reference machine
*
* @throws IOException
* @throws AlreadyGzippedException
* @throws InterruptedException
*/
public void testUsedGunzipImplementationPerformance()
throws IOException, AlreadyGzippedException, InterruptedException {
byte[] gzip = getGzipFileAsBytes();
Collection headers = new ArrayList();
String[] header = new String[] {"Content-Encoding", "gzip"};
headers.add(header);
PageInfo pageInfo = new PageInfo(200, "text/plain", headers, new ArrayList(), gzip, true);
long initialMemoryUsed = memoryUsed();
StopWatch stopWatch = new StopWatch();
int size = 0;
long timeTaken = 0;
long finalMemoryUsed = 0;
long incrementalMemoryUsed = 0;
byte[] ungzipped = null;
// warmup JVM
for (int i = 0; i < 5; i++) {
ungzipped = pageInfo.getUngzippedBody();
Thread.sleep(200);
}
stopWatch.getElapsedTime();
for (int i = 0; i < 50; i++) {
ungzipped = pageInfo.getUngzippedBody();
}
size = ungzipped.length;
timeTaken = stopWatch.getElapsedTime() / 50;
finalMemoryUsed = memoryUsed();
incrementalMemoryUsed = finalMemoryUsed - initialMemoryUsed;
LOG.info(
"Average gunzip time: "
+ timeTaken
+ ". Memory used: "
+ incrementalMemoryUsed
+ ". Size: "
+ size);
assertEquals(100000, size);
assertTrue(timeTaken < 30);
}
private String getPageJson(FeedItem item) {
PageInfo info = new PageInfo();
SimpleDateFormat dateFormat = new SimpleDateFormat("MMM d, yyyy h:mm a");
info.title = item.title();
info.date = dateFormat.format(item.pubDate());
info.creator = item.creator();
info.description = item.description();
info.url = item.link();
return GsonUtils.toJson(info);
}
/**
* Tests that we can reliably determine if a byte[] is gzipped. Note that this test demonstrates
* that byte[] can be gzipped multiple times and are still gzipped i.e. though gzip has been run
* multiple times a valid gzip file results, although ungzipping once won't be enough to get back
* to uncompressed source byte[]
*/
public void magicNumberTest() throws IOException {
byte[] gzip = getGzipFileAsBytes();
// A large generated gzip file
assertTrue(PageInfo.isGzipped(gzip));
// Short String
assertTrue(PageInfo.isGzipped(gzip("The rain in spain".getBytes())));
// Null
assertTrue(!PageInfo.isGzipped(null));
// Less than two bytes
assertTrue(!PageInfo.isGzipped(new byte[] {0x11}));
// Not Gzipped
assertTrue(!PageInfo.isGzipped("This is not gzipped".getBytes()));
// Double Gzipped
assertTrue(PageInfo.isGzipped(gzip(getGzipFileAsBytes())));
// Triple Gzipped
assertTrue(PageInfo.isGzipped(gzip(gzip(getGzipFileAsBytes()))));
}
PageInfoDialog(final PageActivity activity, PageInfo pageInfo, int height) {
super(activity, R.layout.dialog_page_info);
View parentView = getDialogLayout();
flipper = (ViewFlipper) parentView.findViewById(R.id.page_info_flipper);
final ListView disambigList = (ListView) parentView.findViewById(R.id.disambig_list);
ListView issuesList = (ListView) parentView.findViewById(R.id.page_issues_list);
disambigHeading = (Button) parentView.findViewById(R.id.page_info_similar_titles_heading);
issuesHeading = (Button) parentView.findViewById(R.id.page_info_page_issues_heading);
View separatorHeading = parentView.findViewById(R.id.page_info_heading_separator);
View closeButton = parentView.findViewById(R.id.page_info_close);
closeButton.setOnClickListener(
new View.OnClickListener() {
@Override
public void onClick(View v) {
dismiss();
}
});
parentView.setLayoutParams(
new FrameLayout.LayoutParams(FrameLayout.LayoutParams.MATCH_PARENT, height));
issuesList.setAdapter(new IssuesListAdapter(activity, pageInfo.getIssues()));
disambigList.setAdapter(new DisambigListAdapter(activity, pageInfo.getDisambigs()));
disambigList.setOnItemClickListener(
new AdapterView.OnItemClickListener() {
@Override
public void onItemClick(AdapterView<?> parent, View view, int position, long id) {
PageTitle title =
((DisambigResult) disambigList.getAdapter().getItem(position)).getTitle();
HistoryEntry historyEntry = new HistoryEntry(title, HistoryEntry.SOURCE_INTERNAL_LINK);
dismiss();
activity.displayNewPage(title, historyEntry);
}
});
if (pageInfo.getDisambigs().length > 0) {
disambigHeading.setOnClickListener(
new View.OnClickListener() {
@Override
public void onClick(View v) {
showDisambig();
}
});
} else {
disambigHeading.setVisibility(View.GONE);
separatorHeading.setVisibility(View.GONE);
}
if (pageInfo.getIssues().length > 0) {
issuesHeading.setOnClickListener(
new View.OnClickListener() {
@Override
public void onClick(View v) {
showIssues();
}
});
} else {
issuesHeading.setVisibility(View.GONE);
separatorHeading.setVisibility(View.GONE);
}
}
@Override
public CharSequence getPageTitle(int position) {
return CroutonDemo.this.getString(PageInfo.values()[position].titleResId);
}
@Override
public int getCount() {
return PageInfo.values().length;
}
