private void process(Object entity) {
IndexFilterChecker checker = new IndexFilterChecker(entity);
if (checker.needIndex()) {
Document doc = new Document();
IndexCreator creator = new IndexCreator(entity, "");
creator.create(doc);
try {
this.writer.addDocument(doc);
} catch (CorruptIndexException ex) {
LOG.error("IndexWriter can not add a document to the lucene index", ex);
} catch (IOException ex) {
LOG.error("IndexWriter can not add a document to the lucene index", ex);
}
}
}
@Test
public void testObjectColumnSelectorOnVaryingColumnSchema() throws Exception {
IncrementalIndex index = indexCreator.createIndex();
index.add(
new MapBasedInputRow(
new DateTime("2014-09-01T00:00:00"),
Lists.newArrayList("billy"),
ImmutableMap.<String, Object>of("billy", "hi")));
index.add(
new MapBasedInputRow(
new DateTime("2014-09-01T01:00:00"),
Lists.newArrayList("billy", "sally"),
ImmutableMap.<String, Object>of(
"billy", "hip",
"sally", "hop")));
GroupByQueryEngine engine = makeGroupByQueryEngine();
final Sequence<Row> rows =
engine.process(
GroupByQuery.builder()
.setDataSource("test")
.setGranularity(QueryGranularity.ALL)
.setInterval(new Interval(0, new DateTime().getMillis()))
.addDimension("billy")
.addDimension("sally")
.addAggregator(new LongSumAggregatorFactory("cnt", "cnt"))
.addAggregator(
new JavaScriptAggregatorFactory(
"fieldLength",
Arrays.asList("sally", "billy"),
"function(current, s, b) { return current + (s == null ? 0 : s.length) + (b == null ? 0 : b.length); }",
"function() { return 0; }",
"function(a,b) { return a + b; }"))
.build(),
new IncrementalIndexStorageAdapter(index));
final ArrayList<Row> results = Sequences.toList(rows, Lists.<Row>newArrayList());
Assert.assertEquals(2, results.size());
MapBasedRow row = (MapBasedRow) results.get(0);
Assert.assertEquals(
ImmutableMap.of("billy", "hi", "cnt", 1L, "fieldLength", 2.0), row.getEvent());
row = (MapBasedRow) results.get(1);
Assert.assertEquals(
ImmutableMap.of("billy", "hip", "sally", "hop", "cnt", 1L, "fieldLength", 6.0),
row.getEvent());
}
@Test
public void testResetSanity() throws IOException {
IncrementalIndex index = indexCreator.createIndex();
DateTime t = DateTime.now();
Interval interval = new Interval(t.minusMinutes(1), t.plusMinutes(1));
index.add(
new MapBasedInputRow(
t.minus(1).getMillis(),
Lists.newArrayList("billy"),
ImmutableMap.<String, Object>of("billy", "hi")));
index.add(
new MapBasedInputRow(
t.minus(1).getMillis(),
Lists.newArrayList("sally"),
ImmutableMap.<String, Object>of("sally", "bo")));
IncrementalIndexStorageAdapter adapter = new IncrementalIndexStorageAdapter(index);
for (boolean descending : Arrays.asList(false, true)) {
Sequence<Cursor> cursorSequence =
adapter.makeCursors(
new SelectorFilter("sally", "bo"), interval, QueryGranularity.NONE, descending);
Cursor cursor =
Sequences.toList(Sequences.limit(cursorSequence, 1), Lists.<Cursor>newArrayList()).get(0);
DimensionSelector dimSelector;
dimSelector = cursor.makeDimensionSelector(new DefaultDimensionSpec("sally", "sally"));
Assert.assertEquals("bo", dimSelector.lookupName(dimSelector.getRow().get(0)));
index.add(
new MapBasedInputRow(
t.minus(1).getMillis(),
Lists.newArrayList("sally"),
ImmutableMap.<String, Object>of("sally", "ah")));
// Cursor reset should not be affected by out of order values
cursor.reset();
dimSelector = cursor.makeDimensionSelector(new DefaultDimensionSpec("sally", "sally"));
Assert.assertEquals("bo", dimSelector.lookupName(dimSelector.getRow().get(0)));
}
}
@Test
public void testSingleValueTopN() throws IOException {
IncrementalIndex index = indexCreator.createIndex();
DateTime t = DateTime.now();
index.add(
new MapBasedInputRow(
t.minus(1).getMillis(),
Lists.newArrayList("sally"),
ImmutableMap.<String, Object>of("sally", "bo")));
TopNQueryEngine engine =
new TopNQueryEngine(
new StupidPool<ByteBuffer>(
new Supplier<ByteBuffer>() {
@Override
public ByteBuffer get() {
return ByteBuffer.allocate(50000);
}
}));
final Iterable<Result<TopNResultValue>> results =
Sequences.toList(
engine.query(
new TopNQueryBuilder()
.dataSource("test")
.granularity(QueryGranularity.ALL)
.intervals(Lists.newArrayList(new Interval(0, new DateTime().getMillis())))
.dimension("sally")
.metric("cnt")
.threshold(10)
.aggregators(
Lists.<AggregatorFactory>newArrayList(
new LongSumAggregatorFactory("cnt", "cnt")))
.build(),
new IncrementalIndexStorageAdapter(index)),
Lists.<Result<TopNResultValue>>newLinkedList());
Assert.assertEquals(1, Iterables.size(results));
Assert.assertEquals(1, results.iterator().next().getValue().getValue().size());
}
@Test
public void testSanity() throws Exception {
IncrementalIndex index = indexCreator.createIndex();
index.add(
new MapBasedInputRow(
new DateTime().minus(1).getMillis(),
Lists.newArrayList("billy"),
ImmutableMap.<String, Object>of("billy", "hi")));
index.add(
new MapBasedInputRow(
new DateTime().minus(1).getMillis(),
Lists.newArrayList("sally"),
ImmutableMap.<String, Object>of("sally", "bo")));
GroupByQueryEngine engine = makeGroupByQueryEngine();
final Sequence<Row> rows =
engine.process(
GroupByQuery.builder()
.setDataSource("test")
.setGranularity(QueryGranularity.ALL)
.setInterval(new Interval(0, new DateTime().getMillis()))
.addDimension("billy")
.addDimension("sally")
.addAggregator(new LongSumAggregatorFactory("cnt", "cnt"))
.build(),
new IncrementalIndexStorageAdapter(index));
final ArrayList<Row> results = Sequences.toList(rows, Lists.<Row>newArrayList());
Assert.assertEquals(2, results.size());
MapBasedRow row = (MapBasedRow) results.get(0);
Assert.assertEquals(ImmutableMap.of("billy", "hi", "cnt", 1L), row.getEvent());
row = (MapBasedRow) results.get(1);
Assert.assertEquals(ImmutableMap.of("sally", "bo", "cnt", 1L), row.getEvent());
}
private void createObfFiles(File country, GroupFiles g) throws Exception {
File obf = g.getObfFileName(country);
if (!obf.exists() || g.getTimestamp() - obf.lastModified() > 1000) {
if (obf.exists()) {
log.info(
"The file "
+ obf.getName()
+ " was updated for "
+ (g.getTimestamp() - obf.lastModified()) / 1000
+ " seconds");
} else {
log.info("The file " + obf.getName() + " doesn't exist");
}
RTree.clearCache();
IndexCreator ic = new IndexCreator(country);
ic.setIndexAddress(false);
ic.setIndexPOI(true);
ic.setIndexRouting(true);
ic.setIndexMap(true);
ic.setGenerateLowLevelIndexes(false);
ic.setDialects(DBDialect.SQLITE_IN_MEMORY, DBDialect.SQLITE_IN_MEMORY);
ic.setLastModifiedDate(g.getTimestamp());
File tmpFile = new File(g.dayName + ".tmp.odb");
tmpFile.delete();
ic.setRegionName(Algorithms.capitalizeFirstLetterAndLowercase(g.dayName));
ic.setNodesDBFile(tmpFile);
log.info("Processing " + g.dayName + " " + g.osmGzFiles.size() + " files");
ic.generateIndexes(
g.getSortedFiles(),
new ConsoleProgressImplementation(),
null,
MapZooms.parseZooms("13-14;15-"),
new MapRenderingTypesEncoder(country.getName()),
log,
false);
File targetFile = new File(country, ic.getMapFileName());
targetFile.setLastModified(g.getTimestamp());
FileInputStream fis = new FileInputStream(targetFile);
GZIPOutputStream gzout = new GZIPOutputStream(new FileOutputStream(obf));
Algorithms.streamCopy(fis, gzout);
fis.close();
gzout.close();
obf.setLastModified(g.getTimestamp());
targetFile.delete();
}
}
