@Test
public void testPersistMergeCaseInsensitive() throws Exception {
final long timestamp = System.currentTimeMillis();
IncrementalIndex toPersist1 = IncrementalIndexTest.createCaseInsensitiveIndex(timestamp);
IncrementalIndex toPersist2 =
new IncrementalIndex(0L, QueryGranularity.NONE, new AggregatorFactory[] {});
toPersist2.add(
new MapBasedInputRow(
timestamp,
Arrays.asList("DIm1", "DIM2"),
ImmutableMap.<String, Object>of(
"dim1", "1", "dim2", "2", "DIm1", "10000", "DIM2", "100000000")));
toPersist2.add(
new MapBasedInputRow(
timestamp,
Arrays.asList("dIM1", "dIm2"),
ImmutableMap.<String, Object>of("DIm1", "1", "DIM2", "2", "dim1", "5", "dim2", "6")));
final File tempDir1 = Files.createTempDir();
final File tempDir2 = Files.createTempDir();
final File mergedDir = Files.createTempDir();
try {
QueryableIndex index1 = IndexIO.loadIndex(IndexMerger.persist(toPersist1, tempDir1));
Assert.assertEquals(2, index1.getTimeColumn().getLength());
Assert.assertEquals(
Arrays.asList("dim1", "dim2"), Lists.newArrayList(index1.getAvailableDimensions()));
Assert.assertEquals(2, index1.getColumnNames().size());
QueryableIndex index2 = IndexIO.loadIndex(IndexMerger.persist(toPersist2, tempDir2));
Assert.assertEquals(2, index2.getTimeColumn().getLength());
Assert.assertEquals(
Arrays.asList("dim1", "dim2"), Lists.newArrayList(index2.getAvailableDimensions()));
Assert.assertEquals(2, index2.getColumnNames().size());
QueryableIndex merged =
IndexIO.loadIndex(
IndexMerger.mergeQueryableIndex(
Arrays.asList(index1, index2), new AggregatorFactory[] {}, mergedDir));
Assert.assertEquals(3, merged.getTimeColumn().getLength());
Assert.assertEquals(
Arrays.asList("dim1", "dim2"), Lists.newArrayList(merged.getAvailableDimensions()));
Assert.assertEquals(2, merged.getColumnNames().size());
} finally {
FileUtils.deleteQuietly(tempDir1);
FileUtils.deleteQuietly(tempDir2);
FileUtils.deleteQuietly(mergedDir);
}
}
public QueryableIndex buildMMappedMergedIndex() {
Preconditions.checkNotNull(indexMerger, "indexMerger");
Preconditions.checkNotNull(tmpDir, "tmpDir");
final List<QueryableIndex> persisted = Lists.newArrayList();
try {
for (int i = 0; i < rows.size(); i += ROWS_PER_INDEX_FOR_MERGING) {
persisted.add(
TestHelper.getTestIndexIO()
.loadIndex(
indexMerger.persist(
buildIncrementalIndexWithRows(
schema,
maxRows,
rows.subList(i, Math.min(rows.size(), i + ROWS_PER_INDEX_FOR_MERGING))),
new File(
tmpDir, String.format("testIndex-%s", UUID.randomUUID().toString())),
indexSpec)));
}
final QueryableIndex merged =
TestHelper.getTestIndexIO()
.loadIndex(
indexMerger.merge(
Lists.transform(
persisted,
new Function<QueryableIndex, IndexableAdapter>() {
@Override
public IndexableAdapter apply(QueryableIndex input) {
return new QueryableIndexIndexableAdapter(input);
}
}),
true,
Iterables.toArray(
Iterables.transform(
Arrays.asList(schema.getMetrics()),
new Function<AggregatorFactory, AggregatorFactory>() {
@Override
public AggregatorFactory apply(AggregatorFactory input) {
return input.getCombiningFactory();
}
}),
AggregatorFactory.class),
new File(tmpDir, String.format("testIndex-%s", UUID.randomUUID())),
indexSpec));
for (QueryableIndex index : persisted) {
index.close();
}
return merged;
} catch (IOException e) {
throw Throwables.propagate(e);
}
}
@Test
public void testPersistEmptyColumn() throws Exception {
final IncrementalIndex toPersist1 =
new IncrementalIndex(0L, QueryGranularity.NONE, new AggregatorFactory[] {});
final IncrementalIndex toPersist2 =
new IncrementalIndex(0L, QueryGranularity.NONE, new AggregatorFactory[] {});
final File tmpDir1 = Files.createTempDir();
final File tmpDir2 = Files.createTempDir();
final File tmpDir3 = Files.createTempDir();
try {
toPersist1.add(
new MapBasedInputRow(
1L,
ImmutableList.of("dim1", "dim2"),
ImmutableMap.<String, Object>of("dim1", ImmutableList.of(), "dim2", "foo")));
toPersist2.add(
new MapBasedInputRow(
1L,
ImmutableList.of("dim1", "dim2"),
ImmutableMap.<String, Object>of("dim1", ImmutableList.of(), "dim2", "bar")));
final QueryableIndex index1 = IndexIO.loadIndex(IndexMerger.persist(toPersist1, tmpDir1));
final QueryableIndex index2 = IndexIO.loadIndex(IndexMerger.persist(toPersist1, tmpDir2));
final QueryableIndex merged =
IndexIO.loadIndex(
IndexMerger.mergeQueryableIndex(
Arrays.asList(index1, index2), new AggregatorFactory[] {}, tmpDir3));
Assert.assertEquals(1, index1.getTimeColumn().getLength());
Assert.assertEquals(
ImmutableList.of("dim2"), ImmutableList.copyOf(index1.getAvailableDimensions()));
Assert.assertEquals(1, index2.getTimeColumn().getLength());
Assert.assertEquals(
ImmutableList.of("dim2"), ImmutableList.copyOf(index2.getAvailableDimensions()));
Assert.assertEquals(1, merged.getTimeColumn().getLength());
Assert.assertEquals(
ImmutableList.of("dim2"), ImmutableList.copyOf(merged.getAvailableDimensions()));
} finally {
FileUtils.deleteQuietly(tmpDir1);
FileUtils.deleteQuietly(tmpDir2);
FileUtils.deleteQuietly(tmpDir3);
}
}
public QueryableIndex buildMMappedIndex() {
Preconditions.checkNotNull(indexMerger, "indexMerger");
Preconditions.checkNotNull(tmpDir, "tmpDir");
try (final IncrementalIndex incrementalIndex = buildIncrementalIndex()) {
return TestHelper.getTestIndexIO()
.loadIndex(
indexMerger.persist(
incrementalIndex,
new File(
tmpDir,
String.format("testIndex-%s", new Random().nextInt(Integer.MAX_VALUE))),
indexSpec));
} catch (IOException e) {
throw Throwables.propagate(e);
}
}
@Test
public void testPersistCaseInsensitive() throws Exception {
final long timestamp = System.currentTimeMillis();
IncrementalIndex toPersist = IncrementalIndexTest.createCaseInsensitiveIndex(timestamp);
final File tempDir = Files.createTempDir();
try {
QueryableIndex index = IndexIO.loadIndex(IndexMerger.persist(toPersist, tempDir));
Assert.assertEquals(2, index.getTimeColumn().getLength());
Assert.assertEquals(
Arrays.asList("dim1", "dim2"), Lists.newArrayList(index.getAvailableDimensions()));
Assert.assertEquals(2, index.getColumnNames().size());
} finally {
tempDir.delete();
}
}
