예제 #1
0
 @Test
 public void singleNullValue() {
   when(cursor.getCount()).thenReturn(1);
   when(cursor.moveToFirst()).thenReturn(true);
   Fetcher fetcher = Fetcher.of(cursor);
   Optional<String> result = fetcher.toValue(Converter.STRING);
   assertFalse("We should not have a value", result.isPresent());
 }
예제 #2
0
 @Test
 public void emptyCursorToValue() {
   when(cursor.getCount()).thenReturn(0);
   Fetcher fetcher = Fetcher.of(cursor);
   Optional<String> result = fetcher.toValue(Converter.STRING);
   assertFalse(result.isPresent());
   verify(cursor).close();
 }
예제 #3
0
 @Test
 public void emptyCursorToList() {
   when(cursor.getCount()).thenReturn(0);
   Fetcher fetcher = Fetcher.of(cursor);
   List<String> result = fetcher.toList(Converter.STRING);
   assertThat(result).isNotNull().isEmpty();
   verify(cursor).close();
 }
예제 #4
0
 @Test
 public void singleNullValueToList() {
   when(cursor.getCount()).thenReturn(1);
   when(cursor.moveToNext()).thenReturn(true, false);
   when(cursor.getString(0)).thenReturn(null);
   Fetcher fetcher = Fetcher.of(cursor);
   List<String> result = fetcher.toList(Converter.STRING);
   assertThat(result).isNotNull().hasSize(1).contains((String) null);
 }
예제 #5
0
 @Test
 public void singleValueToValue() {
   when(cursor.getCount()).thenReturn(1);
   when(cursor.moveToFirst()).thenReturn(true);
   when(cursor.getInt(0)).thenReturn(42);
   Fetcher fetcher = Fetcher.of(cursor);
   Optional<Integer> result = fetcher.toValue(Converter.INT);
   assertTrue("We should have a value", result.isPresent());
   assertThat(result.get()).isEqualTo(42);
 }
    /**
     * Returns the data encapsulated in this {@link TransferableObject}. If the {@link Fetcher}
     * constructor was used, then this is when the {@link Fetcher#getObject getObject()} method will
     * be called. If the requested data flavor is not supported, then the {@link Fetcher#getObject
     * getObject()} method will not be called.
     *
     * @param flavor The data flavor for the data to return
     * @return The dropped data
     * @since 1.1
     */
    public Object getTransferData(java.awt.datatransfer.DataFlavor flavor)
        throws java.awt.datatransfer.UnsupportedFlavorException, java.io.IOException {
      // Native object
      if (flavor.equals(DATA_FLAVOR)) return fetcher == null ? data : fetcher.getObject();

      // String
      if (flavor.equals(java.awt.datatransfer.DataFlavor.stringFlavor))
        return fetcher == null ? data.toString() : fetcher.getObject().toString();

      // We can't do anything else
      throw new java.awt.datatransfer.UnsupportedFlavorException(flavor);
    } // end getTransferData
예제 #7
0
 @Test
 public void skipNulls() {
   when(cursor.getCount()).thenReturn(3);
   when(cursor.moveToNext()).thenReturn(true, true, true, false);
   when(cursor.getString(0)).thenReturn("First", null, "Third");
   Fetcher fetcher = Fetcher.of(cursor).skipNulls();
   List<String> result = fetcher.toList(Converter.STRING);
   assertThat(result)
       .isNotNull()
       .hasSize(2)
       .contains("First", "Third")
       .doesNotContain((String) null);
 }
  private static void testStart(String site, String page) {
    Fetcher fetcher =
        TestHelper.tryToGet(
            args -> Fetcher.builder().site((String) args[0]).page((String) args[1]).build(),
            site,
            page);

    assertNotNull(fetcher);
    assertNotNull(fetcher.getHeader());
    assertNotNull(fetcher.getLyrics());
    assertNotNull(fetcher.getSource());

    printFetcher(fetcher);
  }
예제 #9
0
  @Test
  public void rxBindingSingleValue() {
    when(cursor.getCount()).thenReturn(1);
    when(cursor.moveToNext()).thenReturn(true, false);
    when(cursor.getString(0)).thenReturn("Yo");

    TestSubscriber<String> testSubscriber = new TestSubscriber<>();

    Fetcher fetcher = Fetcher.of(cursor);
    fetcher.subscribe(Converter.STRING, testSubscriber);

    testSubscriber.requestMore(1);
    testSubscriber.assertValueCount(1);
    testSubscriber.assertValue("Yo");
  }
 public void close() throws IOException {
   if (isShuttingDown.compareAndSet(false, true)) {
     logger.info("ZkConsumerConnector shutting down");
     try {
       scheduler.shutdown();
       if (fetcher != null) {
         fetcher.stopConnectionsToAllBrokers();
       }
       sendShutdownToAllQueues();
       if (config.isAutoCommit()) {
         commitOffsets();
       }
       // waiting rebalance listener to closed and then shutdown the zkclient
       for (ZKRebalancerListener<?> listener : this.rebalancerListeners) {
         Closer.closeQuietly(listener);
       }
       if (this.zkClient != null) {
         this.zkClient.close();
         zkClient = null;
       }
     } catch (Exception e) {
       logger.error("error during consumer connector shutdown", e);
     }
     logger.info("ZkConsumerConnector shut down completed");
   }
 }
예제 #11
0
  public static void start(List<Fetcher> fetchers) {

    try {

      for (Fetcher fetcher : fetchers) {
        Fetch fetch = fetcher.getFetch();
        if (fetch == null) {
          continue;
        }
        Link link = new Link(fetch.getTitle(), fetch.getUrl());
        link.setRoot(fetch.getUrl());
        fetch(fetcher.createRequest(), fetcher.getRegexRule(), link, 10000);
        EventInst.getInstance().getAsyncEventBus().post(fetcher);
      }

    } catch (Exception ex) {
      Logger.getLogger(FetchUtils.class.getName()).log(Level.SEVERE, null, ex);
    }
  }
예제 #12
0
파일: main.java 프로젝트: NiravC/contig5
  /**
   * @param args
   * @throws SQLException
   * @throws ClassNotFoundException
   * @throws IOException
   */
  public static void main(String[] args) throws ClassNotFoundException, SQLException, IOException {
    // TODO Auto-generated method stub

    Fetcher f = new Fetcher();

    System.out.println("Running BFS...");

    long startTime = System.nanoTime();

    int end = f.BFS(0, 6321440);

    long stopTime = System.nanoTime();
    long duration = stopTime - startTime;

    System.out.println("BFS duration: " + duration);
    System.out.println("BFS layout overhead: " + f.dur + ". Misses: " + f.count);
    System.out.println("visited: " + end);
    int num = f.c.edgeList.capacity() / 28;
    System.out.println("Buffer size: " + f.c.edgeList.capacity() + " -> " + num + " nodes");
  }
  public void run() {
    try {
      fetcher.fetch();

      //			for(Tweet tw: ((RestApiTweetFetcher) fetcher).getCurFetch()){
      //				System.out.println(tw.getPublishedTimeGmtStr() + "\t" + tw.getContent().replaceAll("\n",
      // ""));
      //			}

      System.out.println(
          fetcher.getCurFetch().size()
              + " tweets retrieved in the last "
              + fetcher.getIntervalMinute()
              + " minutes ("
              + fetcher.getLagMinute()
              + " minutes lagged time).");
    } catch (Exception e) {
      e.printStackTrace();
    }
  }
예제 #14
0
  public ActionForward generate(
      ActionMapping mapping,
      ActionForm actionForm,
      HttpServletRequest request,
      HttpServletResponse response)
      throws Exception {
    ExecutionCourse executionCourse = getExecutionCourse(request);
    if (executionCourse == null) {
      return mapping.findForward("fallback");
    }

    ArchiveOptions options = getOptions(request);
    if (options == null) {
      return prepare(mapping, actionForm, request, response);
    }

    String name = getArchiveName(executionCourse);
    // NOTE: Using a DiskZipArchive instead of a ZipArchive because a
    // ZipArchive
    // writes directly to the response during the entire process and
    // prevents
    // the server from showing an error page when something goes wrong. This
    // leaves the user with a corrupt Zip file and no other information.
    Archive archive = new DiskZipArchive(response, name);
    Fetcher fetcher = new Fetcher(archive, request, response);

    queueResources(request, executionCourse, options, fetcher);

    List<Content> contents = new ArrayList<Content>();
    contents.add(MetaDomainObject.getMeta(ExecutionCourseSite.class).getAssociatedPortal());
    contents.add(executionCourse.getSite());
    FilterFunctionalityContext context = new FilterFunctionalityContext(request, contents);

    fetcher.process(context);
    archive.finish();

    return null;
  }
예제 #15
0
 @Test
 public void nullCursor() {
   Fetcher fetcher = Fetcher.of(null);
   assertThat(fetcher.toList(Converter.STRING)).isNotNull().isEmpty();
   assertThat(fetcher.toSet(Converter.STRING)).isNotNull().isEmpty();
   assertThat(fetcher.toCollection(Converter.STRING, Collections.<String>emptyList()))
       .isNotNull()
       .isEmpty();
   final Optional<String> result = fetcher.toValue(Converter.STRING);
   assertThat(result).isNotNull();
   assertFalse(result.isPresent());
 }
예제 #16
0
  @Override
  public void seekFetcherToInitialOffsets(
      List<KafkaTopicPartitionLeader> partitions, Fetcher fetcher) throws Exception {
    for (KafkaTopicPartitionLeader tp : partitions) {
      long offset =
          getOffsetFromZooKeeper(
              curatorClient,
              groupId,
              tp.getTopicPartition().getTopic(),
              tp.getTopicPartition().getPartition());

      if (offset != OFFSET_NOT_SET) {
        LOG.info(
            "Offset for partition {} was set to {} in ZooKeeper. Seeking fetcher to that position.",
            tp.getTopicPartition().getPartition(),
            offset);

        // the offset in Zookeeper was the last read offset, seek is accepting the
        // next-to-read-offset.
        fetcher.seek(tp.getTopicPartition(), offset + 1);
      }
    }
  }
예제 #17
0
 /**
  * This method corresponds to a projection in select clasuse Thus for query Select Object(d) from
  * ....., This method corresponds to expression Object(d) for query Select d.name from ....., This
  * method corresponds to expression d.name
  *
  * @param pathExpr Data about path expression.
  * @return ResultElement capable of fetching the entity from resultset
  */
 @Override
 public ResultElement leaveProjection(Object pathExpr) {
   Fetcher fetcher = (Fetcher) pathExpr;
   return fetcher.fetch(selectPlan);
 }
예제 #18
0
 @Test
 public void countEmpty() {
   assertThat(Fetcher.of(null).count()).isEqualTo(0);
   assertThat(Fetcher.of(cursor).count()).isEqualTo(0);
   verify(cursor).close();
 }
 private static void printFetcher(Fetcher fetcher) {
   System.out.println("Source = " + fetcher.getSource());
   fetcher.getHeader().forEach(System.out::println);
   System.out.println("------------------------------------");
   fetcher.getLyrics().forEach(System.out::println);
 }
예제 #20
0
 @Test
 public void countNonEmpty() {
   when(cursor.getCount()).thenReturn(42);
   assertThat(Fetcher.of(cursor).count()).isEqualTo(42);
   verify(cursor).close();
 }
예제 #21
0
  private void queueResources(
      HttpServletRequest request,
      ExecutionCourse executionCourse,
      ArchiveOptions options,
      Fetcher fetcher) {
    fetcher.queue(new Resource("CSS/main.css", "/CSS/main.css"));
    fetcher.queue(new Resource("images/icon_email.gif", "/images/icon_email.gif"));

    String announcementsName = options.isAnnouncements() ? "announcements.html" : "#";
    String planningName = options.isPlanning() ? "planning.html" : "#";
    String scheduleName = options.isSchedule() ? "schedule.html" : "#";
    String shiftsName = options.isShifts() ? "shifts.html" : "#";
    String groupingsName = options.isGroupings() ? "groupings.html" : "#";
    String evaluationName = options.isEvaluations() ? "evaluation.html" : "#";

    String contextPath = request.getContextPath();

    List<Rule> globalRules = new ArrayList<Rule>();
    globalRules.add(new ResourceRule(contextPath + "/CSS/(.*)", "CSS/$1"));
    globalRules.add(new ResourceRule(contextPath + "/images/(.*)", "images/$1"));
    globalRules.add(
        new SimpleTransformRule(
            contextPath + "/publico/showCourseSite.do\\?.*?method=showCurricularCourseSite.*",
            "#"));
    globalRules.add(
        new SimpleTransformRule(
            contextPath
                + "/publico/executionCourse.do\\?(executionCourseID=[0-9]+|&amp;|method=firstPage)+",
            "index.html"));
    globalRules.add(
        new SimpleTransformRule(
            contextPath
                + "/publico/announcementManagement.do\\?(executionCourseID=[0-9]+|&amp;|method=start)+",
            announcementsName));
    globalRules.add(
        new SimpleTransformRule(
            contextPath
                + "/publico/executionCourse.do\\?(executionCourseID=[0-9]+|&amp;|method=lessonPlannings)+",
            planningName));
    globalRules.add(
        new SimpleTransformRule(
            contextPath
                + "/publico/executionCourse.do\\?(executionCourseID=[0-9]+|&amp;|method=summaries)+",
            "summaries.html"));
    globalRules.add(
        new SimpleTransformRule(
            contextPath
                + "/publico/executionCourse.do\\?(executionCourseID=[0-9]+|&amp;|method=objectives)+",
            "objectives.html"));
    globalRules.add(
        new SimpleTransformRule(
            contextPath
                + "/publico/executionCourse.do\\?(executionCourseID=[0-9]+|&amp;|method=program)+",
            "program.html"));
    globalRules.add(
        new SimpleTransformRule(
            contextPath
                + "/publico/executionCourse.do\\?(executionCourseID=[0-9]+|&amp;|method=evaluationMethod)+",
            "evaluation-method.html"));
    globalRules.add(
        new SimpleTransformRule(
            contextPath
                + "/publico/executionCourse.do\\?(executionCourseID=[0-9]+|&amp;|method=bibliographicReference)+",
            "bibliography.html"));
    globalRules.add(
        new SimpleTransformRule(
            contextPath
                + "/publico/executionCourse.do\\?(executionCourseID=[0-9]+|&amp;|method=schedule)+",
            scheduleName));
    globalRules.add(
        new SimpleTransformRule(
            contextPath
                + "/publico/executionCourse.do\\?(executionCourseID=[0-9]+|&amp;|method=shifts)+",
            shiftsName));
    globalRules.add(
        new SimpleTransformRule(
            contextPath
                + "/publico/executionCourse.do\\?(executionCourseID=[0-9]+|&amp;|method=groupings)+",
            groupingsName));
    globalRules.add(
        new SimpleTransformRule(
            contextPath
                + "/publico/executionCourse.do\\?(executionCourseID=[0-9]+|&amp;|method=evaluations)+",
            evaluationName));
    globalRules.add(
        new SimpleTransformRule(
            contextPath
                + "/publico/searchScormContent.do\\?(executionCourseID=[0-9]+|&amp;|method=prepareSearchForExecutionCourse)+",
            "#"));
    globalRules.add(
        new SimpleTransformRule(
            contextPath + "/publico/executionCourse.do\\?.*?sectionID=([0-9]+).*",
            "section-$1.html"));
    globalRules.add(
        new SimpleTransformRule(
            contextPath + "/publico/executionCourse.do\\?.*?method=rss.*", "#"));

    Resource resource =
        new Resource(
            "index.html",
            "/publico/executionCourse.do?method=firstPage&executionCourseID="
                + executionCourse.getExternalId());
    resource.addAllRules(globalRules);
    resource.addRule(
        new SimpleTransformRule(
            contextPath + "/publico/announcementManagement.do\\?.*?announcementId=([0-9]+).*",
            announcementsName.equals("#") ? "#" : announcementsName + "#ID_$1"));
    fetcher.queue(resource);

    if (options.isAnnouncements()) {
      resource =
          new Resource(
              "announcements.html",
              "/publico/announcementManagement.do?method=start&ommitArchive=true&executionCourseID="
                  + executionCourse.getExternalId());
      resource.addAllRules(globalRules);
      resource.addRule(
          new SimpleTransformRule(
              contextPath + "/publico/announcementManagement.do.*?announcementId=([0-9]+).*",
              "announcements.html#ID_$1"));
      resource.addRule(
          new SimpleTransformRule(
              contextPath + "/publico/announcementManagement.do.*?announcementBoardId=([0-9]+).*",
              "#"));
      fetcher.queue(resource);
    }

    if (options.isPlanning()) {
      resource =
          new Resource(
              "planning.html",
              "/publico/executionCourse.do?method=lessonPlannings&executionCourseID="
                  + executionCourse.getExternalId());
      resource.addAllRules(globalRules);
      fetcher.queue(resource);
    }

    resource =
        new Resource(
            "summaries.html",
            "/publico/executionCourse.do?method=summaries&ommitFilter=true&executionCourseID="
                + executionCourse.getExternalId());
    resource.addAllRules(globalRules);
    fetcher.queue(resource);

    resource =
        new Resource(
            "objectives.html",
            "/publico/executionCourse.do?method=objectives&executionCourseID="
                + executionCourse.getExternalId());
    resource.addAllRules(globalRules);
    fetcher.queue(resource);

    resource =
        new Resource(
            "program.html",
            "/publico/executionCourse.do?method=program&executionCourseID="
                + executionCourse.getExternalId());
    resource.addAllRules(globalRules);
    fetcher.queue(resource);

    resource =
        new Resource(
            "evaluation-method.html",
            "/publico/executionCourse.do?method=evaluationMethod&executionCourseID="
                + executionCourse.getExternalId());
    resource.addAllRules(globalRules);
    fetcher.queue(resource);

    resource =
        new Resource(
            "bibliography.html",
            "/publico/executionCourse.do?method=bibliographicReference&executionCourseID="
                + executionCourse.getExternalId());
    resource.addAllRules(globalRules);
    fetcher.queue(resource);

    if (options.isSchedule()) {
      resource =
          new Resource(
              "schedule.html",
              "/publico/executionCourse.do?method=schedule&executionCourseID="
                  + executionCourse.getExternalId());
      resource.addAllRules(globalRules);
      resource.addRule(
          new SimpleTransformRule("(" + contextPath + "/publico/)?siteViewer.do.*", "#"));
      fetcher.queue(resource);
    }

    if (options.isShifts()) {
      resource =
          new Resource(
              "shifts.html",
              "/publico/executionCourse.do?method=shifts&executionCourseID="
                  + executionCourse.getExternalId());
      resource.addAllRules(globalRules);
      resource.addRule(
          new SimpleTransformRule("(" + contextPath + "/publico/)?siteViewer.do.*", "#"));
      resource.addRule(
          new SimpleTransformRule(
              "("
                  + contextPath
                  + "/publico/)?viewClassTimeTableWithClassNameAndDegreeInitialsAction.do.*",
              "#"));
      fetcher.queue(resource);
    }

    if (options.isGroupings()) {
      resource =
          new Resource(
              "groupings.html",
              "/publico/executionCourse.do?method=groupings&executionCourseID="
                  + executionCourse.getExternalId());
      resource.addAllRules(globalRules);
      ResourceRule groupRule =
          new ResourceRule(
              contextPath + "/publico/executionCourse.do\\?method=grouping.*?groupingID=([0-9]+).*",
              "group-$1.html");
      groupRule.addAllRules(globalRules);
      resource.addRule(groupRule);
      fetcher.queue(resource);
    }

    if (options.isEvaluations()) {
      resource =
          new Resource(
              "evaluation.html",
              "/publico/executionCourse.do?method=evaluations&executionCourseID="
                  + executionCourse.getExternalId());
      resource.addAllRules(globalRules);
      ResourceRule marksRule =
          new ResourceRule(
              contextPath + "/publico/executionCourse.do\\?.*method=marks.*", "marks.html");
      marksRule.addAllRules(globalRules);
      resource.addRule(marksRule);
      fetcher.queue(resource);
    }

    for (Section section : executionCourse.getSite().getAssociatedSections()) {
      String name = String.format("section-%s.html", section.getExternalId());
      String url =
          String.format(
              "/publico/executionCourse.do?method=section&executionCourseID=%s&sectionID=%s",
              executionCourse.getExternalId(), section.getExternalId());
      resource = new Resource(name, url);
      resource.addAllRules(globalRules);

      if (options.isFiles()) {
        ResourceRule fileRule =
            new ResourceRule(".*?/bitstream/([0-9]+/[0-9]+/[0-9]+)/(.*)", "files/$2");
        resource.addRule(fileRule);
      }

      fetcher.queue(resource);
    }
  }