private void search(SearchEngine se) { try { initializeJdbc(); final WebClient webClient = new WebClient( getBrowserVersionFromName(se.getDefaultBrowser())); // BrowserVersion.FIREFOX_24); // webClient.getOptions().setThrowExceptionOnScriptError(false); String htmlSource; HtmlPage searchResultPage = null; // String baseUrl = url; int numberOfUrlsExtracted; searchResultPage = clickSearchButtonMainPage(webClient, se); htmlSource = searchResultPage.getWebResponse().getContentAsString(); numberOfUrlsExtracted = ExtractAndInsertSeedUrls(searchResultPage, htmlSource, 1, 0, se.getName()); target.dispatchEvent(new EventSearchWorker(target, 1)); try { for (int i = 1; i < maxPage; i++) { HtmlAnchor ha = (HtmlAnchor) getNextButtonOrLink(searchResultPage, se.getNextButtonIdentifier()); searchResultPage = ha.click(); htmlSource = searchResultPage.getWebResponse().getContentAsString(); numberOfUrlsExtracted = ExtractAndInsertSeedUrls( searchResultPage, htmlSource, i + 1, numberOfUrlsExtracted, se.getName()); // nextUrl = getNextUrl(b, se, baseUrl); target.dispatchEvent(new EventSearchWorker(target, 1)); Random r = new Random(); int randomSleepTime = r.nextInt(2000) + 200; // minimum 200 ms max 2200 ms Thread.sleep(randomSleepTime); } // fw.close(); } catch (Exception ex) { String x = ex.getMessage(); } webClient.closeAllWindows(); } catch (Exception ex) { String x = ex.getMessage(); } }
private SearchEngine getSearchEngineFromName(String enginename) { List<SearchEngine> selist = getSearchEnginesFromXml().getSearchEngines(); String[] seNames = new String[selist.size()]; int i = 0; for (SearchEngine se : selist) { if (se.getName().equals(enginename)) { return se; } } return null; }
private void setupCheckboxes(List<SearchEngine> searchEngines, JPanel parent) { final Map<JCheckBox, BooleanSetting> cBoxes = new HashMap<JCheckBox, BooleanSetting>(); ItemListener listener = new ItemListener() { public void itemStateChanged(ItemEvent e) { if (areAll(false)) { ((JCheckBox) e.getItemSelectable()).setSelected(true); } if (resultPanel != null) { resultPanel.filterChanged(new SearchEngineFilter(engineCheckboxes), 0); } } }; for (SearchEngine se : searchEngines) { JCheckBox cBox = new JCheckBox(se.getName()); cBox.setSelected(se.isEnabled()); cBox.setEnabled(se.isEnabled()); if (!cBox.isEnabled()) { cBox.setToolTipText( se.getName() + " " + I18n.tr( "has been disabled on your FrostWire Search Options. (Go to Tools > Options > Search to enable)")); } parent.add(cBox); cBoxes.put(cBox, se.getEnabledSetting()); cBox.addItemListener(listener); engineCheckboxes.put(se, cBox); } }