@Override
public void addSegment(DataSegment segment) {
try {
log.info("Loading segment %s", segment.getIdentifier());
try {
serverManager.loadSegment(segment);
} catch (Exception e) {
removeSegment(segment);
throw new SegmentLoadingException(
e, "Exception loading segment[%s]", segment.getIdentifier());
}
File segmentInfoCacheFile = new File(config.getInfoDir(), segment.getIdentifier());
if (!segmentInfoCacheFile.exists()) {
try {
jsonMapper.writeValue(segmentInfoCacheFile, segment);
} catch (IOException e) {
removeSegment(segment);
throw new SegmentLoadingException(
e, "Failed to write to disk segment info cache file[%s]", segmentInfoCacheFile);
}
}
try {
announcer.announceSegment(segment);
} catch (IOException e) {
throw new SegmentLoadingException(
e, "Failed to announce segment[%s]", segment.getIdentifier());
}
} catch (SegmentLoadingException e) {
log.makeAlert(e, "Failed to load segment for dataSource").addData("segment", segment).emit();
}
}
private void loadCache() {
File baseDir = config.getInfoDir();
if (!baseDir.exists()) {
return;
}
List<DataSegment> cachedSegments = Lists.newArrayList();
for (File file : baseDir.listFiles()) {
log.info("Loading segment cache file [%s]", file);
try {
DataSegment segment = jsonMapper.readValue(file, DataSegment.class);
if (serverManager.isSegmentCached(segment)) {
cachedSegments.add(segment);
} else {
log.warn(
"Unable to find cache file for %s. Deleting lookup entry", segment.getIdentifier());
File segmentInfoCacheFile = new File(config.getInfoDir(), segment.getIdentifier());
if (!segmentInfoCacheFile.delete()) {
log.warn("Unable to delete segmentInfoCacheFile[%s]", segmentInfoCacheFile);
}
}
} catch (Exception e) {
log.makeAlert(e, "Failed to load segment from segmentInfo file")
.addData("file", file)
.emit();
}
}
addSegments(cachedSegments);
}
@DELETE
@Path("{id}")
@Produces(MediaType.APPLICATION_JSON)
public Response getServer(@PathParam("id") String queryId) {
if (log.isDebugEnabled()) {
log.debug("Received cancel request for query [%s]", queryId);
}
queryManager.cancelQuery(queryId);
return Response.status(Response.Status.ACCEPTED).build();
}
public void addSegments(Iterable<DataSegment> segments) {
try {
final List<String> segmentFailures = Lists.newArrayList();
final List<DataSegment> validSegments = Lists.newArrayList();
for (DataSegment segment : segments) {
log.info("Loading segment %s", segment.getIdentifier());
try {
serverManager.loadSegment(segment);
} catch (Exception e) {
log.error(e, "Exception loading segment[%s]", segment.getIdentifier());
removeSegment(segment);
segmentFailures.add(segment.getIdentifier());
continue;
}
File segmentInfoCacheFile = new File(config.getInfoDir(), segment.getIdentifier());
if (!segmentInfoCacheFile.exists()) {
try {
jsonMapper.writeValue(segmentInfoCacheFile, segment);
} catch (IOException e) {
log.error(
e, "Failed to write to disk segment info cache file[%s]", segmentInfoCacheFile);
removeSegment(segment);
segmentFailures.add(segment.getIdentifier());
continue;
}
}
validSegments.add(segment);
}
try {
announcer.announceSegments(validSegments);
} catch (IOException e) {
throw new SegmentLoadingException(e, "Failed to announce segments[%s]", segments);
}
if (!segmentFailures.isEmpty()) {
for (String segmentFailure : segmentFailures) {
log.error("%s failed to load", segmentFailure);
}
throw new SegmentLoadingException(
"%,d errors seen while loading segments", segmentFailures.size());
}
} catch (SegmentLoadingException e) {
log.makeAlert(e, "Failed to load segments for dataSource")
.addData("segments", segments)
.emit();
}
}
@Override
public void removeSegment(DataSegment segment) {
try {
serverManager.dropSegment(segment);
File segmentInfoCacheFile = new File(config.getInfoDir(), segment.getIdentifier());
if (!segmentInfoCacheFile.delete()) {
log.warn("Unable to delete segmentInfoCacheFile[%s]", segmentInfoCacheFile);
}
announcer.unannounceSegment(segment);
} catch (Exception e) {
log.makeAlert(e, "Failed to remove segment").addData("segment", segment).emit();
}
}
public void dropSegment(String from, String segmentName, final LoadPeonCallback callback) {
try {
final DruidServer fromServer = serverInventoryView.getInventoryValue(from);
if (fromServer == null) {
throw new IAE("Unable to find server [%s]", from);
}
final DataSegment segment = fromServer.getSegment(segmentName);
if (segment == null) {
throw new IAE("Unable to find segment [%s] on server [%s]", segmentName, from);
}
final LoadQueuePeon dropPeon = loadManagementPeons.get(from);
if (dropPeon == null) {
throw new IAE("LoadQueuePeon hasn't been created yet for path [%s]", from);
}
if (!dropPeon.getSegmentsToDrop().contains(segment)) {
dropPeon.dropSegment(segment, callback);
}
} catch (Exception e) {
log.makeAlert(e, "Exception dropping segment %s", segmentName).emit();
callback.execute();
}
}
@Before
public void setUp() throws IOException {
EmittingLogger.registerEmitter(new NoopServiceEmitter());
queryWaitLatch = new CountDownLatch(1);
queryWaitYieldLatch = new CountDownLatch(1);
queryNotifyLatch = new CountDownLatch(1);
factory = new MyQueryRunnerFactory(queryWaitLatch, queryWaitYieldLatch, queryNotifyLatch);
serverManagerExec = Executors.newFixedThreadPool(2);
serverManager =
new ServerManager(
new SegmentLoader() {
@Override
public boolean isSegmentLoaded(DataSegment segment) throws SegmentLoadingException {
return false;
}
@Override
public Segment getSegment(final DataSegment segment) {
return new SegmentForTesting(
MapUtils.getString(segment.getLoadSpec(), "version"),
(Interval) segment.getLoadSpec().get("interval"));
}
@Override
public File getSegmentFiles(DataSegment segment) throws SegmentLoadingException {
throw new UnsupportedOperationException();
}
@Override
public void cleanup(DataSegment segment) throws SegmentLoadingException {}
},
new QueryRunnerFactoryConglomerate() {
@Override
public <T, QueryType extends Query<T>> QueryRunnerFactory<T, QueryType> findFactory(
QueryType query) {
return (QueryRunnerFactory) factory;
}
},
new NoopServiceEmitter(),
serverManagerExec,
new DefaultObjectMapper(),
new LocalCacheProvider().get(),
new CacheConfig());
loadQueryable("test", "1", new Interval("P1d/2011-04-01"));
loadQueryable("test", "1", new Interval("P1d/2011-04-02"));
loadQueryable("test", "2", new Interval("P1d/2011-04-02"));
loadQueryable("test", "1", new Interval("P1d/2011-04-03"));
loadQueryable("test", "1", new Interval("P1d/2011-04-04"));
loadQueryable("test", "1", new Interval("P1d/2011-04-05"));
loadQueryable("test", "2", new Interval("PT1h/2011-04-04T01"));
loadQueryable("test", "2", new Interval("PT1h/2011-04-04T02"));
loadQueryable("test", "2", new Interval("PT1h/2011-04-04T03"));
loadQueryable("test", "2", new Interval("PT1h/2011-04-04T05"));
loadQueryable("test", "2", new Interval("PT1h/2011-04-04T06"));
loadQueryable("test2", "1", new Interval("P1d/2011-04-01"));
loadQueryable("test2", "1", new Interval("P1d/2011-04-02"));
}
@Provides
@ManageLifecycle
public ServiceEmitter getServiceEmitter(
@Self Supplier<DruidNode> configSupplier, Emitter emitter) {
final DruidNode config = configSupplier.get();
final ServiceEmitter retVal =
new ServiceEmitter(config.getServiceName(), config.getHost(), emitter);
EmittingLogger.registerEmitter(retVal);
return retVal;
}
private void stopBeingLeader() {
synchronized (lock) {
try {
log.info("I am no longer the leader...");
for (String server : loadManagementPeons.keySet()) {
LoadQueuePeon peon = loadManagementPeons.remove(server);
peon.stop();
}
loadManagementPeons.clear();
serviceAnnouncer.unannounce(self);
serverInventoryView.stop();
databaseRuleManager.stop();
databaseSegmentManager.stop();
leader = false;
} catch (Exception e) {
log.makeAlert(e, "Unable to stopBeingLeader").emit();
}
}
}
protected void addSingleInventory(final DruidServer container, final DataSegment inventory) {
log.info("Server[%s] added segment[%s]", container.getName(), inventory.getIdentifier());
if (container.getSegment(inventory.getIdentifier()) != null) {
log.warn(
"Not adding or running callbacks for existing segment[%s] on server[%s]",
inventory.getIdentifier(), container.getName());
return;
}
container.addDataSegment(inventory.getIdentifier(), inventory);
runSegmentCallbacks(
new Function<SegmentCallback, CallbackAction>() {
@Override
public CallbackAction apply(SegmentCallback input) {
return input.segmentAdded(container.getMetadata(), inventory);
}
});
}
protected void removeSingleInventory(final DruidServer container, String inventoryKey) {
log.info("Server[%s] removed segment[%s]", container.getName(), inventoryKey);
final DataSegment segment = container.getSegment(inventoryKey);
if (segment == null) {
log.warn(
"Not running cleanup or callbacks for non-existing segment[%s] on server[%s]",
inventoryKey, container.getName());
return;
}
container.removeDataSegment(inventoryKey);
runSegmentCallbacks(
new Function<SegmentCallback, CallbackAction>() {
@Override
public CallbackAction apply(SegmentCallback input) {
return input.segmentRemoved(container.getMetadata(), segment);
}
});
}
private void initializeEmitter() {
if (emitter == null) {
final HttpClient httpClient =
HttpClientInit.createClient(
HttpClientConfig.builder().withNumConnections(1).build(), lifecycle);
emitter =
new ServiceEmitter(
PropUtils.getProperty(props, "druid.service"),
PropUtils.getProperty(props, "druid.host"),
Emitters.create(props, httpClient, jsonMapper, lifecycle));
}
EmittingLogger.registerEmitter(emitter);
}
@Override
public DruidCoordinatorRuntimeParams run(DruidCoordinatorRuntimeParams params) {
DatasourceWhitelist whitelist = whitelistRef.get();
for (DataSegment dataSegment : params.getAvailableSegments()) {
if (whitelist == null || whitelist.contains(dataSegment.getDataSource())) {
final Integer binaryVersion = dataSegment.getBinaryVersion();
if (binaryVersion == null || binaryVersion < IndexIO.CURRENT_VERSION_ID) {
log.info("Upgrading version on segment[%s]", dataSegment.getIdentifier());
indexingServiceClient.upgradeSegment(dataSegment);
}
}
}
return params;
}
@LifecycleStop
public void stop() {
log.info("Stopping ZkCoordinator for [%s]", me);
synchronized (lock) {
if (!started) {
return;
}
try {
loadQueueCache.close();
} catch (Exception e) {
throw Throwables.propagate(e);
} finally {
loadQueueCache = null;
started = false;
}
}
}
@Before
public void setUp() throws IOException {
EmittingLogger.registerEmitter(new NoopServiceEmitter());
factory = new MyQueryRunnerFactory();
serverManager =
new ServerManager(
new SegmentLoader() {
@Override
public Segment getSegment(final DataSegment segment) {
return new SegmentForTesting(
MapUtils.getString(segment.getLoadSpec(), "version"),
(Interval) segment.getLoadSpec().get("interval"));
}
@Override
public void cleanup(DataSegment segment) throws SegmentLoadingException {}
},
new QueryRunnerFactoryConglomerate() {
@Override
public <T, QueryType extends Query<T>> QueryRunnerFactory<T, QueryType> findFactory(
QueryType query) {
return (QueryRunnerFactory) factory;
}
},
new NoopServiceEmitter(),
MoreExecutors.sameThreadExecutor());
loadQueryable("test", "1", new Interval("P1d/2011-04-01"));
loadQueryable("test", "1", new Interval("P1d/2011-04-02"));
loadQueryable("test", "2", new Interval("P1d/2011-04-02"));
loadQueryable("test", "1", new Interval("P1d/2011-04-03"));
loadQueryable("test", "1", new Interval("P1d/2011-04-04"));
loadQueryable("test", "1", new Interval("P1d/2011-04-05"));
loadQueryable("test", "2", new Interval("PT1h/2011-04-04T01"));
loadQueryable("test", "2", new Interval("PT1h/2011-04-04T02"));
loadQueryable("test", "2", new Interval("PT1h/2011-04-04T03"));
loadQueryable("test", "2", new Interval("PT1h/2011-04-04T05"));
loadQueryable("test", "2", new Interval("PT1h/2011-04-04T06"));
loadQueryable("test2", "1", new Interval("P1d/2011-04-01"));
loadQueryable("test2", "1", new Interval("P1d/2011-04-02"));
}
@LifecycleStop
public void stop() {
synchronized (lock) {
if (!started) {
return;
}
stopBeingLeader();
try {
leaderLatch.get().close();
} catch (IOException e) {
log.warn(e, "Unable to close leaderLatch, ignoring");
}
started = false;
exec.shutdownNow();
}
}
@Override
public TaskStatus call() {
final long startTime = System.currentTimeMillis();
final File taskDir = toolbox.getTaskWorkDir();
TaskStatus status;
try {
log.info("Running task: %s", task.getId());
status = task.run(toolbox);
} catch (InterruptedException e) {
log.error(e, "Interrupted while running task[%s]", task);
throw Throwables.propagate(e);
} catch (Exception e) {
log.error(e, "Exception while running task[%s]", task);
status = TaskStatus.failure(task.getId());
} catch (Throwable t) {
log.error(t, "Uncaught Throwable while running task[%s]", task);
throw Throwables.propagate(t);
}
try {
if (taskDir.exists()) {
log.info("Removing task directory: %s", taskDir);
FileUtils.deleteDirectory(taskDir);
}
} catch (Exception e) {
log.makeAlert(e, "Failed to delete task directory")
.addData("taskDir", taskDir.toString())
.addData("task", task.getId())
.emit();
}
try {
return status.withDuration(System.currentTimeMillis() - startTime);
} catch (Exception e) {
log.error(e, "Uncaught Exception during callback for task[%s]", task);
throw Throwables.propagate(e);
}
}
@Override
public Plumber findPlumber(final Schema schema, final FireDepartmentMetrics metrics) {
verifyState();
final RejectionPolicy rejectionPolicy = rejectionPolicyFactory.create(windowPeriod);
log.info("Creating plumber using rejectionPolicy[%s]", rejectionPolicy);
return new RealtimePlumber(
windowPeriod,
basePersistDirectory,
segmentGranularity,
schema,
metrics,
rejectionPolicy,
emitter,
conglomerate,
segmentAnnouncer,
queryExecutorService,
versioningPolicy,
dataSegmentPusher,
segmentPublisher,
serverView,
maxPendingPersists);
}
private <T> QueryRunner<T> getQueryRunnerImpl(Query<T> query) {
QueryRunner<T> queryRunner = null;
final String queryDataSource = Iterables.getOnlyElement(query.getDataSource().getNames());
for (final ThreadPoolTaskRunnerWorkItem taskRunnerWorkItem :
ImmutableList.copyOf(runningItems)) {
final Task task = taskRunnerWorkItem.getTask();
if (task.getDataSource().equals(queryDataSource)) {
final QueryRunner<T> taskQueryRunner = task.getQueryRunner(query);
if (taskQueryRunner != null) {
if (queryRunner == null) {
queryRunner = taskQueryRunner;
} else {
log.makeAlert("Found too many query runners for datasource")
.addData("dataSource", queryDataSource)
.emit();
}
}
}
}
return queryRunner == null ? new NoopQueryRunner<T>() : queryRunner;
}
public Set<DataSegment> getAvailableDataSegments() {
Set<DataSegment> availableSegments =
Sets.newTreeSet(Comparators.inverse(DataSegment.bucketMonthComparator()));
Iterable<DataSegment> dataSegments =
Iterables.concat(
Iterables.transform(
databaseSegmentManager.getInventory(),
new Function<DruidDataSource, Iterable<DataSegment>>() {
@Override
public Iterable<DataSegment> apply(DruidDataSource input) {
return input.getSegments();
}
}));
for (DataSegment dataSegment : dataSegments) {
if (dataSegment.getSize() < 0) {
log.makeAlert("No size on Segment, wtf?").addData("segment", dataSegment).emit();
}
availableSegments.add(dataSegment);
}
return availableSegments;
}
@Override
public Sequence<T> run(final Query<T> query, final Map<String, Object> responseContext) {
final QueryToolChest<T, Query<T>> toolChest = warehouse.getToolChest(query);
final CacheStrategy<T, Object, Query<T>> strategy = toolChest.getCacheStrategy(query);
final Map<DruidServer, List<SegmentDescriptor>> serverSegments = Maps.newTreeMap();
final List<Pair<Interval, byte[]>> cachedResults = Lists.newArrayList();
final Map<String, CachePopulator> cachePopulatorMap = Maps.newHashMap();
final boolean useCache =
BaseQuery.getContextUseCache(query, true)
&& strategy != null
&& cacheConfig.isUseCache()
&& cacheConfig.isQueryCacheable(query);
final boolean populateCache =
BaseQuery.getContextPopulateCache(query, true)
&& strategy != null
&& cacheConfig.isPopulateCache()
&& cacheConfig.isQueryCacheable(query);
final boolean isBySegment = BaseQuery.getContextBySegment(query, false);
final ImmutableMap.Builder<String, Object> contextBuilder = new ImmutableMap.Builder<>();
final int priority = BaseQuery.getContextPriority(query, 0);
contextBuilder.put("priority", priority);
if (populateCache) {
// prevent down-stream nodes from caching results as well if we are populating the cache
contextBuilder.put(CacheConfig.POPULATE_CACHE, false);
contextBuilder.put("bySegment", true);
}
TimelineLookup<String, ServerSelector> timeline = serverView.getTimeline(query.getDataSource());
if (timeline == null) {
return Sequences.empty();
}
// build set of segments to query
Set<Pair<ServerSelector, SegmentDescriptor>> segments = Sets.newLinkedHashSet();
List<TimelineObjectHolder<String, ServerSelector>> serversLookup = Lists.newLinkedList();
// Note that enabling this leads to putting uncovered intervals information in the response
// headers
// and might blow up in some cases https://github.com/druid-io/druid/issues/2108
int uncoveredIntervalsLimit = BaseQuery.getContextUncoveredIntervalsLimit(query, 0);
if (uncoveredIntervalsLimit > 0) {
List<Interval> uncoveredIntervals = Lists.newArrayListWithCapacity(uncoveredIntervalsLimit);
boolean uncoveredIntervalsOverflowed = false;
for (Interval interval : query.getIntervals()) {
Iterable<TimelineObjectHolder<String, ServerSelector>> lookup = timeline.lookup(interval);
long startMillis = interval.getStartMillis();
long endMillis = interval.getEndMillis();
for (TimelineObjectHolder<String, ServerSelector> holder : lookup) {
Interval holderInterval = holder.getInterval();
long intervalStart = holderInterval.getStartMillis();
if (!uncoveredIntervalsOverflowed && startMillis != intervalStart) {
if (uncoveredIntervalsLimit > uncoveredIntervals.size()) {
uncoveredIntervals.add(new Interval(startMillis, intervalStart));
} else {
uncoveredIntervalsOverflowed = true;
}
}
startMillis = holderInterval.getEndMillis();
serversLookup.add(holder);
}
if (!uncoveredIntervalsOverflowed && startMillis < endMillis) {
if (uncoveredIntervalsLimit > uncoveredIntervals.size()) {
uncoveredIntervals.add(new Interval(startMillis, endMillis));
} else {
uncoveredIntervalsOverflowed = true;
}
}
}
if (!uncoveredIntervals.isEmpty()) {
// This returns intervals for which NO segment is present.
// Which is not necessarily an indication that the data doesn't exist or is
// incomplete. The data could exist and just not be loaded yet. In either
// case, though, this query will not include any data from the identified intervals.
responseContext.put("uncoveredIntervals", uncoveredIntervals);
responseContext.put("uncoveredIntervalsOverflowed", uncoveredIntervalsOverflowed);
}
} else {
for (Interval interval : query.getIntervals()) {
Iterables.addAll(serversLookup, timeline.lookup(interval));
}
}
// Let tool chest filter out unneeded segments
final List<TimelineObjectHolder<String, ServerSelector>> filteredServersLookup =
toolChest.filterSegments(query, serversLookup);
Map<String, Optional<RangeSet<String>>> dimensionRangeCache = Maps.newHashMap();
// Filter unneeded chunks based on partition dimension
for (TimelineObjectHolder<String, ServerSelector> holder : filteredServersLookup) {
final Set<PartitionChunk<ServerSelector>> filteredChunks =
DimFilterUtils.filterShards(
query.getFilter(),
holder.getObject(),
new Function<PartitionChunk<ServerSelector>, ShardSpec>() {
@Override
public ShardSpec apply(PartitionChunk<ServerSelector> input) {
return input.getObject().getSegment().getShardSpec();
}
},
dimensionRangeCache);
for (PartitionChunk<ServerSelector> chunk : filteredChunks) {
ServerSelector selector = chunk.getObject();
final SegmentDescriptor descriptor =
new SegmentDescriptor(
holder.getInterval(), holder.getVersion(), chunk.getChunkNumber());
segments.add(Pair.of(selector, descriptor));
}
}
final byte[] queryCacheKey;
if ((populateCache || useCache) // implies strategy != null
&& !isBySegment) // explicit bySegment queries are never cached
{
queryCacheKey = strategy.computeCacheKey(query);
} else {
queryCacheKey = null;
}
if (queryCacheKey != null) {
// cachKeys map must preserve segment ordering, in order for shards to always be combined in
// the same order
Map<Pair<ServerSelector, SegmentDescriptor>, Cache.NamedKey> cacheKeys =
Maps.newLinkedHashMap();
for (Pair<ServerSelector, SegmentDescriptor> segment : segments) {
final Cache.NamedKey segmentCacheKey =
CacheUtil.computeSegmentCacheKey(
segment.lhs.getSegment().getIdentifier(), segment.rhs, queryCacheKey);
cacheKeys.put(segment, segmentCacheKey);
}
// Pull cached segments from cache and remove from set of segments to query
final Map<Cache.NamedKey, byte[]> cachedValues;
if (useCache) {
cachedValues =
cache.getBulk(
Iterables.limit(cacheKeys.values(), cacheConfig.getCacheBulkMergeLimit()));
} else {
cachedValues = ImmutableMap.of();
}
for (Map.Entry<Pair<ServerSelector, SegmentDescriptor>, Cache.NamedKey> entry :
cacheKeys.entrySet()) {
Pair<ServerSelector, SegmentDescriptor> segment = entry.getKey();
Cache.NamedKey segmentCacheKey = entry.getValue();
final Interval segmentQueryInterval = segment.rhs.getInterval();
final byte[] cachedValue = cachedValues.get(segmentCacheKey);
if (cachedValue != null) {
// remove cached segment from set of segments to query
segments.remove(segment);
cachedResults.add(Pair.of(segmentQueryInterval, cachedValue));
} else if (populateCache) {
// otherwise, if populating cache, add segment to list of segments to cache
final String segmentIdentifier = segment.lhs.getSegment().getIdentifier();
cachePopulatorMap.put(
String.format("%s_%s", segmentIdentifier, segmentQueryInterval),
new CachePopulator(cache, objectMapper, segmentCacheKey));
}
}
}
// Compile list of all segments not pulled from cache
for (Pair<ServerSelector, SegmentDescriptor> segment : segments) {
final QueryableDruidServer queryableDruidServer = segment.lhs.pick();
if (queryableDruidServer == null) {
log.makeAlert(
"No servers found for SegmentDescriptor[%s] for DataSource[%s]?! How can this be?!",
segment.rhs, query.getDataSource())
.emit();
} else {
final DruidServer server = queryableDruidServer.getServer();
List<SegmentDescriptor> descriptors = serverSegments.get(server);
if (descriptors == null) {
descriptors = Lists.newArrayList();
serverSegments.put(server, descriptors);
}
descriptors.add(segment.rhs);
}
}
return new LazySequence<>(
new Supplier<Sequence<T>>() {
@Override
public Sequence<T> get() {
ArrayList<Sequence<T>> sequencesByInterval = Lists.newArrayList();
addSequencesFromCache(sequencesByInterval);
addSequencesFromServer(sequencesByInterval);
return mergeCachedAndUncachedSequences(query, sequencesByInterval);
}
private void addSequencesFromCache(ArrayList<Sequence<T>> listOfSequences) {
if (strategy == null) {
return;
}
final Function<Object, T> pullFromCacheFunction = strategy.pullFromCache();
final TypeReference<Object> cacheObjectClazz = strategy.getCacheObjectClazz();
for (Pair<Interval, byte[]> cachedResultPair : cachedResults) {
final byte[] cachedResult = cachedResultPair.rhs;
Sequence<Object> cachedSequence =
new BaseSequence<>(
new BaseSequence.IteratorMaker<Object, Iterator<Object>>() {
@Override
public Iterator<Object> make() {
try {
if (cachedResult.length == 0) {
return Iterators.emptyIterator();
}
return objectMapper.readValues(
objectMapper.getFactory().createParser(cachedResult),
cacheObjectClazz);
} catch (IOException e) {
throw Throwables.propagate(e);
}
}
@Override
public void cleanup(Iterator<Object> iterFromMake) {}
});
listOfSequences.add(Sequences.map(cachedSequence, pullFromCacheFunction));
}
}
private void addSequencesFromServer(ArrayList<Sequence<T>> listOfSequences) {
listOfSequences.ensureCapacity(listOfSequences.size() + serverSegments.size());
final Query<T> rewrittenQuery = query.withOverriddenContext(contextBuilder.build());
// Loop through each server, setting up the query and initiating it.
// The data gets handled as a Future and parsed in the long Sequence chain in the
// resultSeqToAdd setter.
for (Map.Entry<DruidServer, List<SegmentDescriptor>> entry :
serverSegments.entrySet()) {
final DruidServer server = entry.getKey();
final List<SegmentDescriptor> descriptors = entry.getValue();
final QueryRunner clientQueryable = serverView.getQueryRunner(server);
if (clientQueryable == null) {
log.error("WTF!? server[%s] doesn't have a client Queryable?", server);
continue;
}
final MultipleSpecificSegmentSpec segmentSpec =
new MultipleSpecificSegmentSpec(descriptors);
final Sequence<T> resultSeqToAdd;
if (!server.isAssignable()
|| !populateCache
|| isBySegment) { // Direct server queryable
if (!isBySegment) {
resultSeqToAdd =
clientQueryable.run(query.withQuerySegmentSpec(segmentSpec), responseContext);
} else {
// bySegment queries need to be de-serialized, see DirectDruidClient.run()
@SuppressWarnings("unchecked")
final Query<Result<BySegmentResultValueClass<T>>> bySegmentQuery =
(Query<Result<BySegmentResultValueClass<T>>>) ((Query) query);
@SuppressWarnings("unchecked")
final Sequence<Result<BySegmentResultValueClass<T>>> resultSequence =
clientQueryable.run(
bySegmentQuery.withQuerySegmentSpec(segmentSpec), responseContext);
resultSeqToAdd =
(Sequence)
Sequences.map(
resultSequence,
new Function<
Result<BySegmentResultValueClass<T>>,
Result<BySegmentResultValueClass<T>>>() {
@Override
public Result<BySegmentResultValueClass<T>> apply(
Result<BySegmentResultValueClass<T>> input) {
final BySegmentResultValueClass<T> bySegmentValue =
input.getValue();
return new Result<>(
input.getTimestamp(),
new BySegmentResultValueClass<T>(
Lists.transform(
bySegmentValue.getResults(),
toolChest.makePreComputeManipulatorFn(
query, MetricManipulatorFns.deserializing())),
bySegmentValue.getSegmentId(),
bySegmentValue.getInterval()));
}
});
}
} else { // Requires some manipulation on broker side
@SuppressWarnings("unchecked")
final Sequence<Result<BySegmentResultValueClass<T>>> runningSequence =
clientQueryable.run(
rewrittenQuery.withQuerySegmentSpec(segmentSpec), responseContext);
resultSeqToAdd =
new MergeSequence(
query.getResultOrdering(),
Sequences.<Result<BySegmentResultValueClass<T>>, Sequence<T>>map(
runningSequence,
new Function<Result<BySegmentResultValueClass<T>>, Sequence<T>>() {
private final Function<T, Object> cacheFn =
strategy.prepareForCache();
// Acctually do something with the results
@Override
public Sequence<T> apply(Result<BySegmentResultValueClass<T>> input) {
final BySegmentResultValueClass<T> value = input.getValue();
final CachePopulator cachePopulator =
cachePopulatorMap.get(
String.format(
"%s_%s", value.getSegmentId(), value.getInterval()));
final Queue<ListenableFuture<Object>> cacheFutures =
new ConcurrentLinkedQueue<>();
return Sequences.<T>withEffect(
Sequences.<T, T>map(
Sequences.<T, T>map(
Sequences.<T>simple(value.getResults()),
new Function<T, T>() {
@Override
public T apply(final T input) {
if (cachePopulator != null) {
// only compute cache data if populating cache
cacheFutures.add(
backgroundExecutorService.submit(
new Callable<Object>() {
@Override
public Object call() {
return cacheFn.apply(input);
}
}));
}
return input;
}
}),
toolChest.makePreComputeManipulatorFn(
// Ick... most makePreComputeManipulatorFn directly cast
// to their ToolChest query type of choice
// This casting is sub-optimal, but hasn't caused any
// major problems yet...
(Query) rewrittenQuery,
MetricManipulatorFns.deserializing())),
new Runnable() {
@Override
public void run() {
if (cachePopulator != null) {
Futures.addCallback(
Futures.allAsList(cacheFutures),
new FutureCallback<List<Object>>() {
@Override
public void onSuccess(List<Object> cacheData) {
cachePopulator.populate(cacheData);
// Help out GC by making sure all references are
// gone
cacheFutures.clear();
}
@Override
public void onFailure(Throwable throwable) {
log.error(throwable, "Background caching failed");
}
},
backgroundExecutorService);
}
}
},
MoreExecutors.sameThreadExecutor()); // End withEffect
}
}));
}
listOfSequences.add(resultSeqToAdd);
}
}
} // End of Supplier
);
}
public AppenderatorTester(final int maxRowsInMemory, final File basePersistDirectory) {
objectMapper = new DefaultObjectMapper();
objectMapper.registerSubtypes(LinearShardSpec.class);
final Map<String, Object> parserMap =
objectMapper.convertValue(
new MapInputRowParser(
new JSONParseSpec(
new TimestampSpec("ts", "auto", null), new DimensionsSpec(null, null, null))),
Map.class);
schema =
new DataSchema(
DATASOURCE,
parserMap,
new AggregatorFactory[] {
new CountAggregatorFactory("count"), new LongSumAggregatorFactory("met", "met")
},
new UniformGranularitySpec(Granularity.MINUTE, QueryGranularities.NONE, null),
objectMapper);
tuningConfig =
new RealtimeTuningConfig(
maxRowsInMemory,
null,
null,
basePersistDirectory,
null,
null,
null,
null,
null,
null,
0,
0,
null,
null);
metrics = new FireDepartmentMetrics();
queryExecutor = Execs.singleThreaded("queryExecutor(%d)");
indexIO =
new IndexIO(
objectMapper,
new ColumnConfig() {
@Override
public int columnCacheSizeBytes() {
return 0;
}
});
indexMerger = new IndexMerger(objectMapper, indexIO);
emitter =
new ServiceEmitter(
"test",
"test",
new LoggingEmitter(
new Logger(AppenderatorTester.class), LoggingEmitter.Level.INFO, objectMapper));
emitter.start();
EmittingLogger.registerEmitter(emitter);
dataSegmentPusher =
new DataSegmentPusher() {
@Deprecated
@Override
public String getPathForHadoop(String dataSource) {
return getPathForHadoop();
}
@Override
public String getPathForHadoop() {
throw new UnsupportedOperationException();
}
@Override
public DataSegment push(File file, DataSegment segment) throws IOException {
pushedSegments.add(segment);
return segment;
}
};
appenderator =
Appenderators.createRealtime(
schema,
tuningConfig,
metrics,
dataSegmentPusher,
objectMapper,
indexIO,
indexMerger,
new DefaultQueryRunnerFactoryConglomerate(
ImmutableMap.<Class<? extends Query>, QueryRunnerFactory>of(
TimeseriesQuery.class,
new TimeseriesQueryRunnerFactory(
new TimeseriesQueryQueryToolChest(
new IntervalChunkingQueryRunnerDecorator(
queryExecutor, QueryRunnerTestHelper.NOOP_QUERYWATCHER, emitter)),
new TimeseriesQueryEngine(),
QueryRunnerTestHelper.NOOP_QUERYWATCHER))),
new DataSegmentAnnouncer() {
@Override
public void announceSegment(DataSegment segment) throws IOException {}
@Override
public void unannounceSegment(DataSegment segment) throws IOException {}
@Override
public void announceSegments(Iterable<DataSegment> segments) throws IOException {}
@Override
public void unannounceSegments(Iterable<DataSegment> segments) throws IOException {}
@Override
public boolean isAnnounced(DataSegment segment) {
return false;
}
},
emitter,
queryExecutor,
MapCache.create(2048),
new CacheConfig());
}
@LifecycleStart
public void start() throws IOException {
synchronized (lock) {
if (started) {
return;
}
log.info("Starting zkCoordinator for server[%s]", me.getName());
final String loadQueueLocation = ZKPaths.makePath(zkPaths.getLoadQueuePath(), me.getName());
final String servedSegmentsLocation =
ZKPaths.makePath(zkPaths.getServedSegmentsPath(), me.getName());
final String liveSegmentsLocation =
ZKPaths.makePath(zkPaths.getLiveSegmentsPath(), me.getName());
loadQueueCache = new PathChildrenCache(curator, loadQueueLocation, true, true, loadingExec);
try {
curator.newNamespaceAwareEnsurePath(loadQueueLocation).ensure(curator.getZookeeperClient());
curator
.newNamespaceAwareEnsurePath(servedSegmentsLocation)
.ensure(curator.getZookeeperClient());
curator
.newNamespaceAwareEnsurePath(liveSegmentsLocation)
.ensure(curator.getZookeeperClient());
loadLocalCache();
loadQueueCache
.getListenable()
.addListener(
new PathChildrenCacheListener() {
@Override
public void childEvent(CuratorFramework client, PathChildrenCacheEvent event)
throws Exception {
final ChildData child = event.getData();
switch (event.getType()) {
case CHILD_ADDED:
final String path = child.getPath();
final DataSegmentChangeRequest request =
jsonMapper.readValue(child.getData(), DataSegmentChangeRequest.class);
log.info("New request[%s] with zNode[%s].", request.asString(), path);
try {
request.go(
getDataSegmentChangeHandler(),
new DataSegmentChangeCallback() {
boolean hasRun = false;
@Override
public void execute() {
try {
if (!hasRun) {
curator.delete().guaranteed().forPath(path);
log.info("Completed request [%s]", request.asString());
hasRun = true;
}
} catch (Exception e) {
try {
curator.delete().guaranteed().forPath(path);
} catch (Exception e1) {
log.error(
e1,
"Failed to delete zNode[%s], but ignoring exception.",
path);
}
log.error(e, "Exception while removing zNode[%s]", path);
throw Throwables.propagate(e);
}
}
});
} catch (Exception e) {
try {
curator.delete().guaranteed().forPath(path);
} catch (Exception e1) {
log.error(
e1, "Failed to delete zNode[%s], but ignoring exception.", path);
}
log.makeAlert(e, "Segment load/unload: uncaught exception.")
.addData("node", path)
.addData("nodeProperties", request)
.emit();
}
break;
case CHILD_REMOVED:
log.info("zNode[%s] was removed", event.getData().getPath());
break;
default:
log.info("Ignoring event[%s]", event);
}
}
});
loadQueueCache.start();
} catch (Exception e) {
Throwables.propagateIfPossible(e, IOException.class);
throw Throwables.propagate(e);
}
started = true;
}
}
@Override
public Sequence<T> run(final Query<T> query) {
final QueryToolChest<T, Query<T>> toolChest = warehouse.getToolChest(query);
final CacheStrategy<T, Object, Query<T>> strategy = toolChest.getCacheStrategy(query);
final Map<DruidServer, List<SegmentDescriptor>> serverSegments = Maps.newTreeMap();
final List<Pair<DateTime, byte[]>> cachedResults = Lists.newArrayList();
final Map<String, CachePopulator> cachePopulatorMap = Maps.newHashMap();
final boolean useCache =
Boolean.parseBoolean(query.getContextValue("useCache", "true")) && strategy != null;
final boolean populateCache =
Boolean.parseBoolean(query.getContextValue("populateCache", "true")) && strategy != null;
final boolean isBySegment = Boolean.parseBoolean(query.getContextValue("bySegment", "false"));
ImmutableMap.Builder<String, String> contextBuilder = new ImmutableMap.Builder<>();
final String priority = query.getContextValue("priority", "0");
contextBuilder.put("priority", priority);
if (populateCache) {
contextBuilder.put("bySegment", "true");
}
contextBuilder.put("intermediate", "true");
final Query<T> rewrittenQuery = query.withOverriddenContext(contextBuilder.build());
VersionedIntervalTimeline<String, ServerSelector> timeline =
serverView.getTimeline(query.getDataSource());
if (timeline == null) {
return Sequences.empty();
}
// build set of segments to query
Set<Pair<ServerSelector, SegmentDescriptor>> segments = Sets.newLinkedHashSet();
List<TimelineObjectHolder<String, ServerSelector>> serversLookup = Lists.newLinkedList();
for (Interval interval : rewrittenQuery.getIntervals()) {
serversLookup.addAll(timeline.lookup(interval));
}
// Let tool chest filter out unneeded segments
final List<TimelineObjectHolder<String, ServerSelector>> filteredServersLookup =
toolChest.filterSegments(query, serversLookup);
for (TimelineObjectHolder<String, ServerSelector> holder : filteredServersLookup) {
for (PartitionChunk<ServerSelector> chunk : holder.getObject()) {
ServerSelector selector = chunk.getObject();
final SegmentDescriptor descriptor =
new SegmentDescriptor(
holder.getInterval(), holder.getVersion(), chunk.getChunkNumber());
segments.add(Pair.of(selector, descriptor));
}
}
final byte[] queryCacheKey;
if (strategy != null) {
queryCacheKey = strategy.computeCacheKey(query);
} else {
queryCacheKey = null;
}
if (queryCacheKey != null) {
Map<Pair<ServerSelector, SegmentDescriptor>, Cache.NamedKey> cacheKeys = Maps.newHashMap();
for (Pair<ServerSelector, SegmentDescriptor> segment : segments) {
final Cache.NamedKey segmentCacheKey =
computeSegmentCacheKey(
segment.lhs.getSegment().getIdentifier(), segment.rhs, queryCacheKey);
cacheKeys.put(segment, segmentCacheKey);
}
// Pull cached segments from cache and remove from set of segments to query
final Map<Cache.NamedKey, byte[]> cachedValues;
if (useCache) {
cachedValues = cache.getBulk(cacheKeys.values());
} else {
cachedValues = ImmutableMap.of();
}
for (Map.Entry<Pair<ServerSelector, SegmentDescriptor>, Cache.NamedKey> entry :
cacheKeys.entrySet()) {
Pair<ServerSelector, SegmentDescriptor> segment = entry.getKey();
Cache.NamedKey segmentCacheKey = entry.getValue();
final Interval segmentQueryInterval = segment.rhs.getInterval();
final byte[] cachedValue = cachedValues.get(segmentCacheKey);
if (cachedValue != null) {
// remove cached segment from set of segments to query
segments.remove(segment);
cachedResults.add(Pair.of(segmentQueryInterval.getStart(), cachedValue));
} else if (populateCache) {
final String segmentIdentifier = segment.lhs.getSegment().getIdentifier();
cachePopulatorMap.put(
String.format("%s_%s", segmentIdentifier, segmentQueryInterval),
new CachePopulator(cache, objectMapper, segmentCacheKey));
}
}
}
// Compile list of all segments not pulled from cache
for (Pair<ServerSelector, SegmentDescriptor> segment : segments) {
final QueryableDruidServer queryableDruidServer = segment.lhs.pick();
if (queryableDruidServer == null) {
log.error("No servers found for %s?! How can this be?!", segment.rhs);
} else {
final DruidServer server = queryableDruidServer.getServer();
List<SegmentDescriptor> descriptors = serverSegments.get(server);
if (descriptors == null) {
descriptors = Lists.newArrayList();
serverSegments.put(server, descriptors);
}
descriptors.add(segment.rhs);
}
}
return new LazySequence<>(
new Supplier<Sequence<T>>() {
@Override
public Sequence<T> get() {
ArrayList<Pair<DateTime, Sequence<T>>> listOfSequences = Lists.newArrayList();
addSequencesFromServer(listOfSequences);
addSequencesFromCache(listOfSequences);
Collections.sort(
listOfSequences,
Ordering.natural().onResultOf(Pair.<DateTime, Sequence<T>>lhsFn()));
final Sequence<Sequence<T>> seq =
Sequences.simple(
Iterables.transform(listOfSequences, Pair.<DateTime, Sequence<T>>rhsFn()));
if (strategy == null) {
return toolChest.mergeSequences(seq);
} else {
return strategy.mergeSequences(seq);
}
}
private void addSequencesFromCache(
ArrayList<Pair<DateTime, Sequence<T>>> listOfSequences) {
if (strategy == null) {
return;
}
final Function<Object, T> pullFromCacheFunction = strategy.pullFromCache();
final TypeReference<Object> cacheObjectClazz = strategy.getCacheObjectClazz();
for (Pair<DateTime, byte[]> cachedResultPair : cachedResults) {
final byte[] cachedResult = cachedResultPair.rhs;
Sequence<Object> cachedSequence =
new BaseSequence<>(
new BaseSequence.IteratorMaker<Object, Iterator<Object>>() {
@Override
public Iterator<Object> make() {
try {
if (cachedResult.length == 0) {
return Iterators.emptyIterator();
}
return objectMapper.readValues(
objectMapper.getFactory().createParser(cachedResult),
cacheObjectClazz);
} catch (IOException e) {
throw Throwables.propagate(e);
}
}
@Override
public void cleanup(Iterator<Object> iterFromMake) {}
});
listOfSequences.add(
Pair.of(
cachedResultPair.lhs, Sequences.map(cachedSequence, pullFromCacheFunction)));
}
}
@SuppressWarnings("unchecked")
private void addSequencesFromServer(
ArrayList<Pair<DateTime, Sequence<T>>> listOfSequences) {
for (Map.Entry<DruidServer, List<SegmentDescriptor>> entry :
serverSegments.entrySet()) {
final DruidServer server = entry.getKey();
final List<SegmentDescriptor> descriptors = entry.getValue();
final QueryRunner clientQueryable = serverView.getQueryRunner(server);
if (clientQueryable == null) {
log.makeAlert("WTF!? server[%s] doesn't have a client Queryable?", server).emit();
continue;
}
final Sequence<T> resultSeqToAdd;
final MultipleSpecificSegmentSpec segmentSpec =
new MultipleSpecificSegmentSpec(descriptors);
List<Interval> intervals = segmentSpec.getIntervals();
if (!server.isAssignable() || !populateCache || isBySegment) {
resultSeqToAdd = clientQueryable.run(query.withQuerySegmentSpec(segmentSpec));
} else {
resultSeqToAdd =
toolChest.mergeSequences(
Sequences.map(
clientQueryable.run(rewrittenQuery.withQuerySegmentSpec(segmentSpec)),
new Function<Object, Sequence<T>>() {
private final Function<T, Object> prepareForCache =
strategy.prepareForCache();
@Override
public Sequence<T> apply(Object input) {
Result<Object> result = (Result<Object>) input;
final BySegmentResultValueClass<T> value =
(BySegmentResultValueClass<T>) result.getValue();
String segmentIdentifier = value.getSegmentId();
final Iterable<T> segmentResults = value.getResults();
CachePopulator cachePopulator =
cachePopulatorMap.get(
String.format(
"%s_%s", segmentIdentifier, value.getInterval()));
if (cachePopulator != null) {
cachePopulator.populate(
Iterables.transform(segmentResults, prepareForCache));
}
return Sequences.simple(
Iterables.transform(
segmentResults,
toolChest.makeMetricManipulatorFn(
rewrittenQuery,
new MetricManipulationFn() {
@Override
public Object manipulate(
AggregatorFactory factory, Object object) {
return factory.deserialize(object);
}
})));
}
}));
}
listOfSequences.add(Pair.of(intervals.get(0).getStart(), resultSeqToAdd));
}
}
});
}
@Override
public TaskStatus run(final TaskToolbox toolbox) throws Exception {
if (this.plumber != null) {
throw new IllegalStateException("WTF?!? run with non-null plumber??!");
}
// Shed any locks we might have (e.g. if we were uncleanly killed and restarted) since we'll
// reacquire
// them if we actually need them
for (final TaskLock taskLock : getTaskLocks(toolbox)) {
toolbox.getTaskActionClient().submit(new LockReleaseAction(taskLock.getInterval()));
}
boolean normalExit = true;
// Set up firehose
final Period intermediatePersistPeriod = fireDepartmentConfig.getIntermediatePersistPeriod();
final Firehose firehose = firehoseFactory.connect();
// It would be nice to get the PlumberSchool in the constructor. Although that will need
// jackson injectables for
// stuff like the ServerView, which seems kind of odd? Perhaps revisit this when Guice has been
// introduced.
final RealtimePlumberSchool realtimePlumberSchool =
new RealtimePlumberSchool(
windowPeriod, new File(toolbox.getTaskWorkDir(), "persist"), segmentGranularity);
realtimePlumberSchool.setDefaultMaxPendingPersists(maxPendingPersists);
final SegmentPublisher segmentPublisher = new TaskActionSegmentPublisher(this, toolbox);
// NOTE: We talk to the coordinator in various places in the plumber and we could be more robust
// to issues
// with the coordinator. Right now, we'll block/throw in whatever thread triggered the
// coordinator behavior,
// which will typically be either the main data processing loop or the persist thread.
// Wrap default DataSegmentAnnouncer such that we unlock intervals as we unannounce segments
final DataSegmentAnnouncer lockingSegmentAnnouncer =
new DataSegmentAnnouncer() {
@Override
public void announceSegment(final DataSegment segment) throws IOException {
// Side effect: Calling announceSegment causes a lock to be acquired
toolbox.getTaskActionClient().submit(new LockAcquireAction(segment.getInterval()));
toolbox.getSegmentAnnouncer().announceSegment(segment);
}
@Override
public void unannounceSegment(final DataSegment segment) throws IOException {
try {
toolbox.getSegmentAnnouncer().unannounceSegment(segment);
} finally {
toolbox.getTaskActionClient().submit(new LockReleaseAction(segment.getInterval()));
}
}
@Override
public void announceSegments(Iterable<DataSegment> segments) throws IOException {
// Side effect: Calling announceSegments causes locks to be acquired
for (DataSegment segment : segments) {
toolbox.getTaskActionClient().submit(new LockAcquireAction(segment.getInterval()));
}
toolbox.getSegmentAnnouncer().announceSegments(segments);
}
@Override
public void unannounceSegments(Iterable<DataSegment> segments) throws IOException {
try {
toolbox.getSegmentAnnouncer().unannounceSegments(segments);
} finally {
for (DataSegment segment : segments) {
toolbox.getTaskActionClient().submit(new LockReleaseAction(segment.getInterval()));
}
}
}
};
// NOTE: getVersion will block if there is lock contention, which will block plumber.getSink
// NOTE: (and thus the firehose)
// Shouldn't usually happen, since we don't expect people to submit tasks that intersect with
// the
// realtime window, but if they do it can be problematic. If we decide to care, we can use more
// threads in
// the plumber such that waiting for the coordinator doesn't block data processing.
final VersioningPolicy versioningPolicy =
new VersioningPolicy() {
@Override
public String getVersion(final Interval interval) {
try {
// Side effect: Calling getVersion causes a lock to be acquired
final TaskLock myLock =
toolbox.getTaskActionClient().submit(new LockAcquireAction(interval));
return myLock.getVersion();
} catch (IOException e) {
throw Throwables.propagate(e);
}
}
};
// NOTE: This pusher selects path based purely on global configuration and the DataSegment,
// which means
// NOTE: that redundant realtime tasks will upload to the same location. This can cause
// index.zip and
// NOTE: descriptor.json to mismatch, or it can cause historical nodes to load different
// instances of the
// NOTE: "same" segment.
realtimePlumberSchool.setDataSegmentPusher(toolbox.getSegmentPusher());
realtimePlumberSchool.setConglomerate(toolbox.getQueryRunnerFactoryConglomerate());
realtimePlumberSchool.setQueryExecutorService(toolbox.getQueryExecutorService());
realtimePlumberSchool.setVersioningPolicy(versioningPolicy);
realtimePlumberSchool.setSegmentAnnouncer(lockingSegmentAnnouncer);
realtimePlumberSchool.setSegmentPublisher(segmentPublisher);
realtimePlumberSchool.setServerView(toolbox.getNewSegmentServerView());
realtimePlumberSchool.setEmitter(toolbox.getEmitter());
if (this.rejectionPolicyFactory != null) {
realtimePlumberSchool.setRejectionPolicyFactory(rejectionPolicyFactory);
}
final FireDepartment fireDepartment =
new FireDepartment(schema, fireDepartmentConfig, null, null);
final RealtimeMetricsMonitor metricsMonitor =
new RealtimeMetricsMonitor(ImmutableList.of(fireDepartment));
this.queryRunnerFactoryConglomerate = toolbox.getQueryRunnerFactoryConglomerate();
this.plumber = realtimePlumberSchool.findPlumber(schema, fireDepartment.getMetrics());
try {
plumber.startJob();
// Set up metrics emission
toolbox.getMonitorScheduler().addMonitor(metricsMonitor);
// Time to read data!
long nextFlush = new DateTime().plus(intermediatePersistPeriod).getMillis();
while (firehose.hasMore()) {
final InputRow inputRow;
try {
inputRow = firehose.nextRow();
if (inputRow == null) {
continue;
}
final Sink sink = plumber.getSink(inputRow.getTimestampFromEpoch());
if (sink == null) {
fireDepartment.getMetrics().incrementThrownAway();
log.debug("Throwing away event[%s]", inputRow);
if (System.currentTimeMillis() > nextFlush) {
plumber.persist(firehose.commit());
nextFlush = new DateTime().plus(intermediatePersistPeriod).getMillis();
}
continue;
}
if (sink.isEmpty()) {
log.info("Task %s: New sink: %s", getId(), sink);
}
int currCount = sink.add(inputRow);
fireDepartment.getMetrics().incrementProcessed();
if (currCount >= fireDepartmentConfig.getMaxRowsInMemory()
|| System.currentTimeMillis() > nextFlush) {
plumber.persist(firehose.commit());
nextFlush = new DateTime().plus(intermediatePersistPeriod).getMillis();
}
} catch (FormattedException e) {
log.warn(e, "unparseable line");
fireDepartment.getMetrics().incrementUnparseable();
}
}
} catch (Throwable e) {
normalExit = false;
log.makeAlert(e, "Exception aborted realtime processing[%s]", schema.getDataSource()).emit();
throw e;
} finally {
if (normalExit) {
try {
plumber.persist(firehose.commit());
plumber.finishJob();
} catch (Exception e) {
log.makeAlert(e, "Failed to finish realtime task").emit();
} finally {
Closeables.closeQuietly(firehose);
toolbox.getMonitorScheduler().removeMonitor(metricsMonitor);
}
}
}
return TaskStatus.success(getId());
}
@Before
public void setUp() throws Exception {
emitter = EasyMock.createMock(ServiceEmitter.class);
EmittingLogger.registerEmitter(emitter);
queryRunnerFactoryConglomerate =
EasyMock.createStrictMock(QueryRunnerFactoryConglomerate.class);
monitorScheduler = EasyMock.createStrictMock(MonitorScheduler.class);
publishCountDown = new CountDownLatch(1);
announcedSinks = 0;
pushedSegments = 0;
tmpDir = temporaryFolder.newFolder();
TestUtils testUtils = new TestUtils();
mapper = testUtils.getTestObjectMapper();
tqc =
mapper.readValue(
"{\"startDelay\":\"PT0S\", \"restartDelay\":\"PT1S\", \"storageSyncRate\":\"PT0.5S\"}",
TaskQueueConfig.class);
indexSpec = new IndexSpec();
if (taskStorageType.equals("HeapMemoryTaskStorage")) {
ts = new HeapMemoryTaskStorage(new TaskStorageConfig(null) {});
} else if (taskStorageType.equals("MetadataTaskStorage")) {
testDerbyConnector = derbyConnectorRule.getConnector();
mapper.registerSubtypes(
new NamedType(MockExceptionalFirehoseFactory.class, "mockExcepFirehoseFactory"),
new NamedType(MockFirehoseFactory.class, "mockFirehoseFactory"));
testDerbyConnector.createTaskTables();
testDerbyConnector.createSegmentTable();
ts =
new MetadataTaskStorage(
testDerbyConnector,
new TaskStorageConfig(null),
new SQLMetadataStorageActionHandlerFactory(
testDerbyConnector,
derbyConnectorRule.metadataTablesConfigSupplier().get(),
mapper));
} else {
throw new RuntimeException(String.format("Unknown task storage type [%s]", taskStorageType));
}
serverView =
new FilteredServerView() {
@Override
public void registerSegmentCallback(
Executor exec, ServerView.SegmentCallback callback, Predicate<DataSegment> filter) {
segmentCallbacks.add(callback);
}
};
setUpAndStartTaskQueue(
new DataSegmentPusher() {
@Override
public String getPathForHadoop(String dataSource) {
throw new UnsupportedOperationException();
}
@Override
public DataSegment push(File file, DataSegment segment) throws IOException {
pushedSegments++;
return segment;
}
});
}
@Before
public void setUp() throws Exception {
req = EasyMock.createStrictMock(HttpServletRequest.class);
supervisorManager = EasyMock.createMock(SupervisorManager.class);
taskLockbox = EasyMock.createStrictMock(TaskLockbox.class);
taskLockbox.syncFromStorage();
EasyMock.expectLastCall().atLeastOnce();
taskLockbox.add(EasyMock.<Task>anyObject());
EasyMock.expectLastCall().atLeastOnce();
taskLockbox.remove(EasyMock.<Task>anyObject());
EasyMock.expectLastCall().atLeastOnce();
// for second Noop Task directly added to deep storage.
taskLockbox.add(EasyMock.<Task>anyObject());
EasyMock.expectLastCall().atLeastOnce();
taskLockbox.remove(EasyMock.<Task>anyObject());
EasyMock.expectLastCall().atLeastOnce();
taskActionClientFactory = EasyMock.createStrictMock(TaskActionClientFactory.class);
EasyMock.expect(taskActionClientFactory.create(EasyMock.<Task>anyObject()))
.andReturn(null)
.anyTimes();
EasyMock.replay(taskLockbox, taskActionClientFactory);
taskStorage = new HeapMemoryTaskStorage(new TaskStorageConfig(null));
runTaskCountDownLatches = new CountDownLatch[2];
runTaskCountDownLatches[0] = new CountDownLatch(1);
runTaskCountDownLatches[1] = new CountDownLatch(1);
taskCompletionCountDownLatches = new CountDownLatch[2];
taskCompletionCountDownLatches[0] = new CountDownLatch(1);
taskCompletionCountDownLatches[1] = new CountDownLatch(1);
announcementLatch = new CountDownLatch(1);
IndexerZkConfig indexerZkConfig =
new IndexerZkConfig(new ZkPathsConfig(), null, null, null, null, null);
setupServerAndCurator();
curator.start();
curator.blockUntilConnected();
curator.create().creatingParentsIfNeeded().forPath(indexerZkConfig.getLeaderLatchPath());
druidNode = new DruidNode("hey", "what", 1234);
ServiceEmitter serviceEmitter = new NoopServiceEmitter();
taskMaster =
new TaskMaster(
new TaskQueueConfig(null, new Period(1), null, new Period(10)),
taskLockbox,
taskStorage,
taskActionClientFactory,
druidNode,
indexerZkConfig,
new TaskRunnerFactory<MockTaskRunner>() {
@Override
public MockTaskRunner build() {
return new MockTaskRunner(runTaskCountDownLatches, taskCompletionCountDownLatches);
}
},
curator,
new NoopServiceAnnouncer() {
@Override
public void announce(DruidNode node) {
announcementLatch.countDown();
}
},
serviceEmitter,
supervisorManager);
EmittingLogger.registerEmitter(serviceEmitter);
}
public void removeSegment(DataSegment segment) {
log.info("Removing Segment[%s]", segment);
databaseSegmentManager.removeSegment(segment.getDataSource(), segment.getIdentifier());
}
public void moveSegment(
String from, String to, String segmentName, final LoadPeonCallback callback) {
try {
final DruidServer fromServer = serverInventoryView.getInventoryValue(from);
if (fromServer == null) {
throw new IAE("Unable to find server [%s]", from);
}
final DruidServer toServer = serverInventoryView.getInventoryValue(to);
if (toServer == null) {
throw new IAE("Unable to find server [%s]", to);
}
if (to.equalsIgnoreCase(from)) {
throw new IAE(
"Redundant command to move segment [%s] from [%s] to [%s]", segmentName, from, to);
}
final DataSegment segment = fromServer.getSegment(segmentName);
if (segment == null) {
throw new IAE("Unable to find segment [%s] on server [%s]", segmentName, from);
}
final LoadQueuePeon loadPeon = loadManagementPeons.get(to);
if (loadPeon == null) {
throw new IAE("LoadQueuePeon hasn't been created yet for path [%s]", to);
}
final LoadQueuePeon dropPeon = loadManagementPeons.get(from);
if (dropPeon == null) {
throw new IAE("LoadQueuePeon hasn't been created yet for path [%s]", from);
}
final ServerHolder toHolder = new ServerHolder(toServer, loadPeon);
if (toHolder.getAvailableSize() < segment.getSize()) {
throw new IAE(
"Not enough capacity on server [%s] for segment [%s]. Required: %,d, available: %,d.",
to, segment, segment.getSize(), toHolder.getAvailableSize());
}
final String toLoadQueueSegPath =
ZKPaths.makePath(ZKPaths.makePath(zkPaths.getLoadQueuePath(), to), segmentName);
final String toServedSegPath =
ZKPaths.makePath(
ZKPaths.makePath(
serverInventoryView.getInventoryManagerConfig().getInventoryPath(), to),
segmentName);
loadPeon.loadSegment(
segment,
new LoadPeonCallback() {
@Override
protected void execute() {
try {
if (curator.checkExists().forPath(toServedSegPath) != null
&& curator.checkExists().forPath(toLoadQueueSegPath) == null
&& !dropPeon.getSegmentsToDrop().contains(segment)) {
dropPeon.dropSegment(segment, callback);
} else if (callback != null) {
callback.execute();
}
} catch (Exception e) {
throw Throwables.propagate(e);
}
}
});
} catch (Exception e) {
log.makeAlert(e, "Exception moving segment %s", segmentName).emit();
callback.execute();
}
}
private void becomeLeader() {
synchronized (lock) {
if (!started) {
return;
}
log.info("I am the leader of the coordinators, all must bow!");
try {
leader = true;
databaseSegmentManager.start();
databaseRuleManager.start();
serverInventoryView.start();
serviceAnnouncer.announce(self);
final List<Pair<? extends CoordinatorRunnable, Duration>> coordinatorRunnables =
Lists.newArrayList();
dynamicConfigs =
configManager.watch(
CoordinatorDynamicConfig.CONFIG_KEY,
CoordinatorDynamicConfig.class,
new CoordinatorDynamicConfig.Builder().build());
coordinatorRunnables.add(
Pair.of(new CoordinatorHistoricalManagerRunnable(), config.getCoordinatorPeriod()));
if (indexingServiceClient != null) {
coordinatorRunnables.add(
Pair.of(
new CoordinatorIndexingServiceRunnable(
makeIndexingServiceHelpers(
configManager.watch(
DatasourceWhitelist.CONFIG_KEY, DatasourceWhitelist.class))),
config.getCoordinatorIndexingPeriod()));
}
for (final Pair<? extends CoordinatorRunnable, Duration> coordinatorRunnable :
coordinatorRunnables) {
ScheduledExecutors.scheduleWithFixedDelay(
exec,
config.getCoordinatorStartDelay(),
coordinatorRunnable.rhs,
new Callable<ScheduledExecutors.Signal>() {
private final CoordinatorRunnable theRunnable = coordinatorRunnable.lhs;
@Override
public ScheduledExecutors.Signal call() {
if (leader) {
theRunnable.run();
}
if (leader) { // (We might no longer be coordinator)
return ScheduledExecutors.Signal.REPEAT;
} else {
return ScheduledExecutors.Signal.STOP;
}
}
});
}
} catch (Exception e) {
log.makeAlert(e, "Unable to become leader").emit();
final LeaderLatch oldLatch = createNewLeaderLatch();
Closeables.closeQuietly(oldLatch);
try {
leaderLatch.get().start();
} catch (Exception e1) {
// If an exception gets thrown out here, then the coordinator will zombie out 'cause it
// won't be looking for
// the latch anymore. I don't believe it's actually possible for an Exception to throw
// out here, but
// Curator likes to have "throws Exception" on methods so it might happen...
log.makeAlert(e1, "I am a zombie").emit();
}
}
}
}