コード例 #1
0
  /** Entry point */
  public static void main(String[] args) {
    System.setProperty(
        "webdriver.firefox.bin", "C:\\Program Files (x86)\\Mozilla Firefox\\Firefox.exe");

    CrawljaxConfigurationBuilder builder = CrawljaxConfiguration.builderFor(URL);

    BrowserConfiguration browserConfig = new BrowserConfiguration(BrowserType.firefox);

    builder.setBrowserConfig(browserConfig);

    // PopUpCancel configuration  = new PopUpCancel();

    // limit the crawling scope
    builder.setMaximumStates(MAX_STATES);
    builder.setMaximumDepth(MAX_CRAWL_DEPTH);

    builder.crawlRules().setInputSpec(getInputSpecification());

    CrawljaxController crawljax = new CrawljaxController(builder.build());
    crawljax.run();
  }
コード例 #2
0
  private CrawljaxConfiguration getCrawljaxBuilder() {
    String url = "";
    if (options.getUrl().toLowerCase().startsWith("http://")) {
      url = options.getUrl();
    } else {
      server = new Server(8080);
      ResourceHandler handler = new ResourceHandler();
      try {
        File fileToCrawl = new File(options.getUrl());
        handler.setBaseResource(
            Resource.newResource(fileToCrawl.getParentFile().getAbsolutePath()));
        server.setHandler(handler);
        server.start();
        int port = ((ServerConnector) server.getConnectors()[0]).getLocalPort();
        url = "http://localhost:" + port + "/" + fileToCrawl.getName(); // URI.create(url);
      } catch (Exception e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      }
    }
    CrawljaxConfigurationBuilder builder = CrawljaxConfiguration.builderFor(url);
    // builder.addPlugin(new CrawlOverview());
    builder.addPlugin(
        new OnNewStatePlugin() {
          @Override
          public void onNewState(CrawlerContext arg0, StateVertex arg1) {
            if (cancelSupplier != null) {
              if (cancelSupplier.getAsBoolean()) {
                crawljax.stop();
              }
            }
            Document document;
            try {
              document = arg1.getDocument();
              documents.add(document);
              notifyObservers(document);
            } catch (IOException e) {
              e.printStackTrace();
            }
          }
        });
    builder.setBrowserConfig(new BrowserConfiguration(BrowserType.PHANTOMJS, 1));
    builder.setOutputDirectory(
        new File(options.getOutputDirectory().getAbsolutePath() + "/crawljax"));
    builder.setMaximumDepth(options.getMaxDepth());
    builder.setMaximumStates(options.getMaxStates());
    CrawlRulesBuilder crawlRules = builder.crawlRules();
    if (options.shouldClickDefaultElements()) {
      crawlRules.clickDefaultElements();
    }
    if (options.getDontClickElements().size() > 0) {
      for (String dontClick : options.getDontClickElements()) {
        crawlRules.dontClick(dontClick);
        // TODO: .withAttribute("value", "I don't recognize");
        // TODO: .underXPath("//*[@id='pageFooter']");
        //		.underXPath("//*[@id='content']/div/div[2]");
      }
    }
    if (options.getClickElements().size() > 0) {
      for (String click : options.getClickElements()) {
        crawlRules.click(click);
        // TODO: .withAttribute("type", "submit");
      }
    }
    if (options.getDontClickElementsChildrenOf().size() > 0) {
      for (String dontClick : options.getDontClickElementsChildrenOf()) {
        crawlRules.dontClickChildrenOf(dontClick);
      }
    }
    crawlRules.insertRandomDataInInputForms(options.shouldPutRandomDataInForms());
    crawlRules.clickElementsInRandomOrder(options.shouldClickElementsInRandomOrder());
    crawlRules.crawlFrames(options.shouldCrawlFrames());
    crawlRules.waitAfterReloadUrl(options.getWaitTimeAferReload(), TimeUnit.MILLISECONDS);
    crawlRules.waitAfterEvent(options.getWaitTimeAfterEvent(), TimeUnit.MILLISECONDS);
    crawlRules.clickOnce(options.shouldClickOnce());
    crawlRules.crawlHiddenAnchors(options.shouldCrawlHiddenAnchorsButton());

    return builder.build();
  }