@Test
public void singleNullValue() {
when(cursor.getCount()).thenReturn(1);
when(cursor.moveToFirst()).thenReturn(true);
Fetcher fetcher = Fetcher.of(cursor);
Optional<String> result = fetcher.toValue(Converter.STRING);
assertFalse("We should not have a value", result.isPresent());
}
@Test
public void emptyCursorToValue() {
when(cursor.getCount()).thenReturn(0);
Fetcher fetcher = Fetcher.of(cursor);
Optional<String> result = fetcher.toValue(Converter.STRING);
assertFalse(result.isPresent());
verify(cursor).close();
}
@Test
public void emptyCursorToList() {
when(cursor.getCount()).thenReturn(0);
Fetcher fetcher = Fetcher.of(cursor);
List<String> result = fetcher.toList(Converter.STRING);
assertThat(result).isNotNull().isEmpty();
verify(cursor).close();
}
@Test
public void singleNullValueToList() {
when(cursor.getCount()).thenReturn(1);
when(cursor.moveToNext()).thenReturn(true, false);
when(cursor.getString(0)).thenReturn(null);
Fetcher fetcher = Fetcher.of(cursor);
List<String> result = fetcher.toList(Converter.STRING);
assertThat(result).isNotNull().hasSize(1).contains((String) null);
}
@Test
public void singleValueToValue() {
when(cursor.getCount()).thenReturn(1);
when(cursor.moveToFirst()).thenReturn(true);
when(cursor.getInt(0)).thenReturn(42);
Fetcher fetcher = Fetcher.of(cursor);
Optional<Integer> result = fetcher.toValue(Converter.INT);
assertTrue("We should have a value", result.isPresent());
assertThat(result.get()).isEqualTo(42);
}
/**
* Returns the data encapsulated in this {@link TransferableObject}. If the {@link Fetcher}
* constructor was used, then this is when the {@link Fetcher#getObject getObject()} method will
* be called. If the requested data flavor is not supported, then the {@link Fetcher#getObject
* getObject()} method will not be called.
*
* @param flavor The data flavor for the data to return
* @return The dropped data
* @since 1.1
*/
public Object getTransferData(java.awt.datatransfer.DataFlavor flavor)
throws java.awt.datatransfer.UnsupportedFlavorException, java.io.IOException {
// Native object
if (flavor.equals(DATA_FLAVOR)) return fetcher == null ? data : fetcher.getObject();
// String
if (flavor.equals(java.awt.datatransfer.DataFlavor.stringFlavor))
return fetcher == null ? data.toString() : fetcher.getObject().toString();
// We can't do anything else
throw new java.awt.datatransfer.UnsupportedFlavorException(flavor);
} // end getTransferData
@Test
public void skipNulls() {
when(cursor.getCount()).thenReturn(3);
when(cursor.moveToNext()).thenReturn(true, true, true, false);
when(cursor.getString(0)).thenReturn("First", null, "Third");
Fetcher fetcher = Fetcher.of(cursor).skipNulls();
List<String> result = fetcher.toList(Converter.STRING);
assertThat(result)
.isNotNull()
.hasSize(2)
.contains("First", "Third")
.doesNotContain((String) null);
}
private static void testStart(String site, String page) {
Fetcher fetcher =
TestHelper.tryToGet(
args -> Fetcher.builder().site((String) args[0]).page((String) args[1]).build(),
site,
page);
assertNotNull(fetcher);
assertNotNull(fetcher.getHeader());
assertNotNull(fetcher.getLyrics());
assertNotNull(fetcher.getSource());
printFetcher(fetcher);
}
@Test
public void rxBindingSingleValue() {
when(cursor.getCount()).thenReturn(1);
when(cursor.moveToNext()).thenReturn(true, false);
when(cursor.getString(0)).thenReturn("Yo");
TestSubscriber<String> testSubscriber = new TestSubscriber<>();
Fetcher fetcher = Fetcher.of(cursor);
fetcher.subscribe(Converter.STRING, testSubscriber);
testSubscriber.requestMore(1);
testSubscriber.assertValueCount(1);
testSubscriber.assertValue("Yo");
}
public void close() throws IOException {
if (isShuttingDown.compareAndSet(false, true)) {
logger.info("ZkConsumerConnector shutting down");
try {
scheduler.shutdown();
if (fetcher != null) {
fetcher.stopConnectionsToAllBrokers();
}
sendShutdownToAllQueues();
if (config.isAutoCommit()) {
commitOffsets();
}
// waiting rebalance listener to closed and then shutdown the zkclient
for (ZKRebalancerListener<?> listener : this.rebalancerListeners) {
Closer.closeQuietly(listener);
}
if (this.zkClient != null) {
this.zkClient.close();
zkClient = null;
}
} catch (Exception e) {
logger.error("error during consumer connector shutdown", e);
}
logger.info("ZkConsumerConnector shut down completed");
}
}
public static void start(List<Fetcher> fetchers) {
try {
for (Fetcher fetcher : fetchers) {
Fetch fetch = fetcher.getFetch();
if (fetch == null) {
continue;
}
Link link = new Link(fetch.getTitle(), fetch.getUrl());
link.setRoot(fetch.getUrl());
fetch(fetcher.createRequest(), fetcher.getRegexRule(), link, 10000);
EventInst.getInstance().getAsyncEventBus().post(fetcher);
}
} catch (Exception ex) {
Logger.getLogger(FetchUtils.class.getName()).log(Level.SEVERE, null, ex);
}
}
/**
* @param args
* @throws SQLException
* @throws ClassNotFoundException
* @throws IOException
*/
public static void main(String[] args) throws ClassNotFoundException, SQLException, IOException {
// TODO Auto-generated method stub
Fetcher f = new Fetcher();
System.out.println("Running BFS...");
long startTime = System.nanoTime();
int end = f.BFS(0, 6321440);
long stopTime = System.nanoTime();
long duration = stopTime - startTime;
System.out.println("BFS duration: " + duration);
System.out.println("BFS layout overhead: " + f.dur + ". Misses: " + f.count);
System.out.println("visited: " + end);
int num = f.c.edgeList.capacity() / 28;
System.out.println("Buffer size: " + f.c.edgeList.capacity() + " -> " + num + " nodes");
}
public void run() {
try {
fetcher.fetch();
// for(Tweet tw: ((RestApiTweetFetcher) fetcher).getCurFetch()){
// System.out.println(tw.getPublishedTimeGmtStr() + "\t" + tw.getContent().replaceAll("\n",
// ""));
// }
System.out.println(
fetcher.getCurFetch().size()
+ " tweets retrieved in the last "
+ fetcher.getIntervalMinute()
+ " minutes ("
+ fetcher.getLagMinute()
+ " minutes lagged time).");
} catch (Exception e) {
e.printStackTrace();
}
}
public ActionForward generate(
ActionMapping mapping,
ActionForm actionForm,
HttpServletRequest request,
HttpServletResponse response)
throws Exception {
ExecutionCourse executionCourse = getExecutionCourse(request);
if (executionCourse == null) {
return mapping.findForward("fallback");
}
ArchiveOptions options = getOptions(request);
if (options == null) {
return prepare(mapping, actionForm, request, response);
}
String name = getArchiveName(executionCourse);
// NOTE: Using a DiskZipArchive instead of a ZipArchive because a
// ZipArchive
// writes directly to the response during the entire process and
// prevents
// the server from showing an error page when something goes wrong. This
// leaves the user with a corrupt Zip file and no other information.
Archive archive = new DiskZipArchive(response, name);
Fetcher fetcher = new Fetcher(archive, request, response);
queueResources(request, executionCourse, options, fetcher);
List<Content> contents = new ArrayList<Content>();
contents.add(MetaDomainObject.getMeta(ExecutionCourseSite.class).getAssociatedPortal());
contents.add(executionCourse.getSite());
FilterFunctionalityContext context = new FilterFunctionalityContext(request, contents);
fetcher.process(context);
archive.finish();
return null;
}
@Test
public void nullCursor() {
Fetcher fetcher = Fetcher.of(null);
assertThat(fetcher.toList(Converter.STRING)).isNotNull().isEmpty();
assertThat(fetcher.toSet(Converter.STRING)).isNotNull().isEmpty();
assertThat(fetcher.toCollection(Converter.STRING, Collections.<String>emptyList()))
.isNotNull()
.isEmpty();
final Optional<String> result = fetcher.toValue(Converter.STRING);
assertThat(result).isNotNull();
assertFalse(result.isPresent());
}
@Override
public void seekFetcherToInitialOffsets(
List<KafkaTopicPartitionLeader> partitions, Fetcher fetcher) throws Exception {
for (KafkaTopicPartitionLeader tp : partitions) {
long offset =
getOffsetFromZooKeeper(
curatorClient,
groupId,
tp.getTopicPartition().getTopic(),
tp.getTopicPartition().getPartition());
if (offset != OFFSET_NOT_SET) {
LOG.info(
"Offset for partition {} was set to {} in ZooKeeper. Seeking fetcher to that position.",
tp.getTopicPartition().getPartition(),
offset);
// the offset in Zookeeper was the last read offset, seek is accepting the
// next-to-read-offset.
fetcher.seek(tp.getTopicPartition(), offset + 1);
}
}
}
/**
* This method corresponds to a projection in select clasuse Thus for query Select Object(d) from
* ....., This method corresponds to expression Object(d) for query Select d.name from ....., This
* method corresponds to expression d.name
*
* @param pathExpr Data about path expression.
* @return ResultElement capable of fetching the entity from resultset
*/
@Override
public ResultElement leaveProjection(Object pathExpr) {
Fetcher fetcher = (Fetcher) pathExpr;
return fetcher.fetch(selectPlan);
}
@Test
public void countEmpty() {
assertThat(Fetcher.of(null).count()).isEqualTo(0);
assertThat(Fetcher.of(cursor).count()).isEqualTo(0);
verify(cursor).close();
}
private static void printFetcher(Fetcher fetcher) {
System.out.println("Source = " + fetcher.getSource());
fetcher.getHeader().forEach(System.out::println);
System.out.println("------------------------------------");
fetcher.getLyrics().forEach(System.out::println);
}
@Test
public void countNonEmpty() {
when(cursor.getCount()).thenReturn(42);
assertThat(Fetcher.of(cursor).count()).isEqualTo(42);
verify(cursor).close();
}
private void queueResources(
HttpServletRequest request,
ExecutionCourse executionCourse,
ArchiveOptions options,
Fetcher fetcher) {
fetcher.queue(new Resource("CSS/main.css", "/CSS/main.css"));
fetcher.queue(new Resource("images/icon_email.gif", "/images/icon_email.gif"));
String announcementsName = options.isAnnouncements() ? "announcements.html" : "#";
String planningName = options.isPlanning() ? "planning.html" : "#";
String scheduleName = options.isSchedule() ? "schedule.html" : "#";
String shiftsName = options.isShifts() ? "shifts.html" : "#";
String groupingsName = options.isGroupings() ? "groupings.html" : "#";
String evaluationName = options.isEvaluations() ? "evaluation.html" : "#";
String contextPath = request.getContextPath();
List<Rule> globalRules = new ArrayList<Rule>();
globalRules.add(new ResourceRule(contextPath + "/CSS/(.*)", "CSS/$1"));
globalRules.add(new ResourceRule(contextPath + "/images/(.*)", "images/$1"));
globalRules.add(
new SimpleTransformRule(
contextPath + "/publico/showCourseSite.do\\?.*?method=showCurricularCourseSite.*",
"#"));
globalRules.add(
new SimpleTransformRule(
contextPath
+ "/publico/executionCourse.do\\?(executionCourseID=[0-9]+|&|method=firstPage)+",
"index.html"));
globalRules.add(
new SimpleTransformRule(
contextPath
+ "/publico/announcementManagement.do\\?(executionCourseID=[0-9]+|&|method=start)+",
announcementsName));
globalRules.add(
new SimpleTransformRule(
contextPath
+ "/publico/executionCourse.do\\?(executionCourseID=[0-9]+|&|method=lessonPlannings)+",
planningName));
globalRules.add(
new SimpleTransformRule(
contextPath
+ "/publico/executionCourse.do\\?(executionCourseID=[0-9]+|&|method=summaries)+",
"summaries.html"));
globalRules.add(
new SimpleTransformRule(
contextPath
+ "/publico/executionCourse.do\\?(executionCourseID=[0-9]+|&|method=objectives)+",
"objectives.html"));
globalRules.add(
new SimpleTransformRule(
contextPath
+ "/publico/executionCourse.do\\?(executionCourseID=[0-9]+|&|method=program)+",
"program.html"));
globalRules.add(
new SimpleTransformRule(
contextPath
+ "/publico/executionCourse.do\\?(executionCourseID=[0-9]+|&|method=evaluationMethod)+",
"evaluation-method.html"));
globalRules.add(
new SimpleTransformRule(
contextPath
+ "/publico/executionCourse.do\\?(executionCourseID=[0-9]+|&|method=bibliographicReference)+",
"bibliography.html"));
globalRules.add(
new SimpleTransformRule(
contextPath
+ "/publico/executionCourse.do\\?(executionCourseID=[0-9]+|&|method=schedule)+",
scheduleName));
globalRules.add(
new SimpleTransformRule(
contextPath
+ "/publico/executionCourse.do\\?(executionCourseID=[0-9]+|&|method=shifts)+",
shiftsName));
globalRules.add(
new SimpleTransformRule(
contextPath
+ "/publico/executionCourse.do\\?(executionCourseID=[0-9]+|&|method=groupings)+",
groupingsName));
globalRules.add(
new SimpleTransformRule(
contextPath
+ "/publico/executionCourse.do\\?(executionCourseID=[0-9]+|&|method=evaluations)+",
evaluationName));
globalRules.add(
new SimpleTransformRule(
contextPath
+ "/publico/searchScormContent.do\\?(executionCourseID=[0-9]+|&|method=prepareSearchForExecutionCourse)+",
"#"));
globalRules.add(
new SimpleTransformRule(
contextPath + "/publico/executionCourse.do\\?.*?sectionID=([0-9]+).*",
"section-$1.html"));
globalRules.add(
new SimpleTransformRule(
contextPath + "/publico/executionCourse.do\\?.*?method=rss.*", "#"));
Resource resource =
new Resource(
"index.html",
"/publico/executionCourse.do?method=firstPage&executionCourseID="
+ executionCourse.getExternalId());
resource.addAllRules(globalRules);
resource.addRule(
new SimpleTransformRule(
contextPath + "/publico/announcementManagement.do\\?.*?announcementId=([0-9]+).*",
announcementsName.equals("#") ? "#" : announcementsName + "#ID_$1"));
fetcher.queue(resource);
if (options.isAnnouncements()) {
resource =
new Resource(
"announcements.html",
"/publico/announcementManagement.do?method=start&ommitArchive=true&executionCourseID="
+ executionCourse.getExternalId());
resource.addAllRules(globalRules);
resource.addRule(
new SimpleTransformRule(
contextPath + "/publico/announcementManagement.do.*?announcementId=([0-9]+).*",
"announcements.html#ID_$1"));
resource.addRule(
new SimpleTransformRule(
contextPath + "/publico/announcementManagement.do.*?announcementBoardId=([0-9]+).*",
"#"));
fetcher.queue(resource);
}
if (options.isPlanning()) {
resource =
new Resource(
"planning.html",
"/publico/executionCourse.do?method=lessonPlannings&executionCourseID="
+ executionCourse.getExternalId());
resource.addAllRules(globalRules);
fetcher.queue(resource);
}
resource =
new Resource(
"summaries.html",
"/publico/executionCourse.do?method=summaries&ommitFilter=true&executionCourseID="
+ executionCourse.getExternalId());
resource.addAllRules(globalRules);
fetcher.queue(resource);
resource =
new Resource(
"objectives.html",
"/publico/executionCourse.do?method=objectives&executionCourseID="
+ executionCourse.getExternalId());
resource.addAllRules(globalRules);
fetcher.queue(resource);
resource =
new Resource(
"program.html",
"/publico/executionCourse.do?method=program&executionCourseID="
+ executionCourse.getExternalId());
resource.addAllRules(globalRules);
fetcher.queue(resource);
resource =
new Resource(
"evaluation-method.html",
"/publico/executionCourse.do?method=evaluationMethod&executionCourseID="
+ executionCourse.getExternalId());
resource.addAllRules(globalRules);
fetcher.queue(resource);
resource =
new Resource(
"bibliography.html",
"/publico/executionCourse.do?method=bibliographicReference&executionCourseID="
+ executionCourse.getExternalId());
resource.addAllRules(globalRules);
fetcher.queue(resource);
if (options.isSchedule()) {
resource =
new Resource(
"schedule.html",
"/publico/executionCourse.do?method=schedule&executionCourseID="
+ executionCourse.getExternalId());
resource.addAllRules(globalRules);
resource.addRule(
new SimpleTransformRule("(" + contextPath + "/publico/)?siteViewer.do.*", "#"));
fetcher.queue(resource);
}
if (options.isShifts()) {
resource =
new Resource(
"shifts.html",
"/publico/executionCourse.do?method=shifts&executionCourseID="
+ executionCourse.getExternalId());
resource.addAllRules(globalRules);
resource.addRule(
new SimpleTransformRule("(" + contextPath + "/publico/)?siteViewer.do.*", "#"));
resource.addRule(
new SimpleTransformRule(
"("
+ contextPath
+ "/publico/)?viewClassTimeTableWithClassNameAndDegreeInitialsAction.do.*",
"#"));
fetcher.queue(resource);
}
if (options.isGroupings()) {
resource =
new Resource(
"groupings.html",
"/publico/executionCourse.do?method=groupings&executionCourseID="
+ executionCourse.getExternalId());
resource.addAllRules(globalRules);
ResourceRule groupRule =
new ResourceRule(
contextPath + "/publico/executionCourse.do\\?method=grouping.*?groupingID=([0-9]+).*",
"group-$1.html");
groupRule.addAllRules(globalRules);
resource.addRule(groupRule);
fetcher.queue(resource);
}
if (options.isEvaluations()) {
resource =
new Resource(
"evaluation.html",
"/publico/executionCourse.do?method=evaluations&executionCourseID="
+ executionCourse.getExternalId());
resource.addAllRules(globalRules);
ResourceRule marksRule =
new ResourceRule(
contextPath + "/publico/executionCourse.do\\?.*method=marks.*", "marks.html");
marksRule.addAllRules(globalRules);
resource.addRule(marksRule);
fetcher.queue(resource);
}
for (Section section : executionCourse.getSite().getAssociatedSections()) {
String name = String.format("section-%s.html", section.getExternalId());
String url =
String.format(
"/publico/executionCourse.do?method=section&executionCourseID=%s§ionID=%s",
executionCourse.getExternalId(), section.getExternalId());
resource = new Resource(name, url);
resource.addAllRules(globalRules);
if (options.isFiles()) {
ResourceRule fileRule =
new ResourceRule(".*?/bitstream/([0-9]+/[0-9]+/[0-9]+)/(.*)", "files/$2");
resource.addRule(fileRule);
}
fetcher.queue(resource);
}
}