@Override
public void initialize(InputSplit split, TaskAttemptContext context)
throws IOException, InterruptedException {
ObjectMapper jsonMapper = DruidInitialization.getInstance().getObjectMapper();
SegmentLoadSpec spec = readSegmentJobSpec(context.getConfiguration(), jsonMapper);
final List<String> dimensions = spec.getDimensions();
final List<String> metrics = spec.getMetrics();
final DimFilter filter = spec.getFilter();
final Interval interval =
new Interval(context.getConfiguration().get(DruidInputFormat.CONF_DRUID_INTERVAL));
String hdfsPath = ((DruidInputSplit) split).getPath();
logger.info("Reading segment from " + hdfsPath);
segmentDir = Files.createTempDir();
logger.info("segment dir: " + segmentDir);
FileSystem fs = FileSystem.get(context.getConfiguration());
getSegmentFiles(hdfsPath, segmentDir, fs);
logger.info("finished getting segment files");
QueryableIndex index = IndexIO.loadIndex(segmentDir);
StorageAdapter adapter = new QueryableIndexStorageAdapter(index);
List<StorageAdapter> adapters = Lists.newArrayList(adapter);
rowYielder =
new IngestSegmentFirehose(
adapters, dimensions, metrics, filter, interval, QueryGranularity.NONE);
}
@Test
public void testSanity() throws Exception {
File segmentDir = tempFolder.newFolder();
createTestIndex(segmentDir);
QueryableIndex qi = null;
try {
qi = IndexIO.loadIndex(segmentDir);
StorageAdapter sa = new QueryableIndexStorageAdapter(qi);
WindowedStorageAdapter wsa = new WindowedStorageAdapter(sa, sa.getInterval());
IngestSegmentFirehose firehose =
new IngestSegmentFirehose(
ImmutableList.of(wsa, wsa),
ImmutableList.of("host"),
ImmutableList.of("visited_sum", "unique_hosts"),
null,
QueryGranularity.NONE);
int count = 0;
while (firehose.hasMore()) {
firehose.nextRow();
count++;
}
Assert.assertEquals(18, count);
} finally {
if (qi != null) {
qi.close();
}
}
}
@Setup
public void setup() throws IOException {
log.info("SETUP CALLED AT " + System.currentTimeMillis());
if (ComplexMetrics.getSerdeForType("hyperUnique") == null) {
ComplexMetrics.registerSerde("hyperUnique", new HyperUniquesSerde(Hashing.murmur3_128()));
}
executorService = Execs.multiThreaded(numSegments, "TimeseriesThreadPool");
setupQueries();
String[] schemaQuery = schemaAndQuery.split("\\.");
String schemaName = schemaQuery[0];
String queryName = schemaQuery[1];
schemaInfo = BenchmarkSchemas.SCHEMA_MAP.get(schemaName);
query = SCHEMA_QUERY_MAP.get(schemaName).get(queryName);
incIndexes = new ArrayList<>();
for (int i = 0; i < numSegments; i++) {
log.info("Generating rows for segment " + i);
BenchmarkDataGenerator gen =
new BenchmarkDataGenerator(
schemaInfo.getColumnSchemas(),
RNG_SEED + i,
schemaInfo.getDataInterval(),
rowsPerSegment);
IncrementalIndex incIndex = makeIncIndex();
for (int j = 0; j < rowsPerSegment; j++) {
InputRow row = gen.nextRow();
if (j % 10000 == 0) {
log.info(j + " rows generated.");
}
incIndex.add(row);
}
log.info(rowsPerSegment + " rows generated");
incIndexes.add(incIndex);
}
File tmpFile = Files.createTempDir();
log.info("Using temp dir: " + tmpFile.getAbsolutePath());
tmpFile.deleteOnExit();
qIndexes = new ArrayList<>();
for (int i = 0; i < numSegments; i++) {
File indexFile = INDEX_MERGER_V9.persist(incIndexes.get(i), tmpFile, new IndexSpec());
QueryableIndex qIndex = INDEX_IO.loadIndex(indexFile);
qIndexes.add(qIndex);
}
factory =
new TimeseriesQueryRunnerFactory(
new TimeseriesQueryQueryToolChest(
QueryBenchmarkUtil.NoopIntervalChunkingQueryRunnerDecorator()),
new TimeseriesQueryEngine(),
QueryBenchmarkUtil.NOOP_QUERYWATCHER);
}