@Test(enabled = false)
public void testQuery() throws Exception {
URI location =
client.execute(
preparePost()
.setUri(uriFor("/v1/query"))
.setBodyGenerator(createStaticBodyGenerator("query", UTF_8))
.build(),
new CreatedResponseHandler());
assertQueryStatus(location, QueryState.RUNNING);
QueryInfo queryInfo =
client.execute(
prepareGet().setUri(location).build(),
createJsonResponseHandler(jsonCodec(QueryInfo.class)));
TaskInfo taskInfo = queryInfo.getOutputStage().getTasks().get(0);
URI outputLocation = uriFor("/v1/task/" + taskInfo.getTaskId() + "/results/out");
long sequenceId = 0;
PagesResponse response =
client.execute(
prepareGet()
.setUri(
uriBuilderFrom(outputLocation).appendPath(String.valueOf(sequenceId)).build())
.build(),
new PageResponseHandler());
List<Page> pages = response.getPages();
assertEquals(countPositions(pages), 220);
assertQueryStatus(location, QueryState.RUNNING);
sequenceId += pages.size();
response =
client.execute(
prepareGet()
.setUri(
uriBuilderFrom(outputLocation).appendPath(String.valueOf(sequenceId)).build())
.build(),
new PageResponseHandler());
pages = response.getPages();
assertEquals(countPositions(pages), 44 + 48);
sequenceId += pages.size();
response =
client.execute(
prepareGet()
.setUri(
uriBuilderFrom(outputLocation).appendPath(String.valueOf(sequenceId)).build())
.build(),
new PageResponseHandler());
pages = response.getPages();
assertEquals(countPositions(pages), 0);
assertQueryStatus(location, QueryState.FINISHED);
// cancel the query
StatusResponse cancelResponse =
client.execute(prepareDelete().setUri(location).build(), createStatusResponseHandler());
assertQueryStatus(location, QueryState.FINISHED);
assertEquals(cancelResponse.getStatusCode(), HttpStatus.NO_CONTENT.code());
}
private void assertQueryStatus(URI location, QueryState expectedQueryState) {
URI statusUri = uriBuilderFrom(location).build();
JsonResponse<QueryInfo> response =
client.execute(
prepareGet().setUri(statusUri).build(),
createFullJsonResponseHandler(jsonCodec(QueryInfo.class)));
if (expectedQueryState == QueryState.FINISHED
&& response.getStatusCode() == HttpStatus.GONE.code()) {
// when query finishes the server may delete it
return;
}
QueryInfo queryInfo = response.getValue();
assertEquals(queryInfo.getState(), expectedQueryState);
}
public void createdEvent(QueryInfo queryInfo) {
eventClient.post(
new QueryCreatedEvent(
queryInfo.getQueryId(),
queryInfo.getSession().getUser(),
queryInfo.getSession().getSource(),
environment,
queryInfo.getSession().getCatalog(),
queryInfo.getSession().getSchema(),
queryInfo.getSession().getRemoteUserAddress(),
queryInfo.getSession().getUserAgent(),
queryInfo.getSelf(),
queryInfo.getQuery(),
queryInfo.getQueryStats().getCreateTime()));
}
public void completionEvent(QueryInfo queryInfo) {
try {
QueryStats queryStats = queryInfo.getQueryStats();
FailureInfo failureInfo = queryInfo.getFailureInfo();
String failureType = failureInfo == null ? null : failureInfo.getType();
String failureMessage = failureInfo == null ? null : failureInfo.getMessage();
eventClient.post(
new QueryCompletionEvent(
queryInfo.getQueryId(),
queryInfo.getSession().getUser(),
queryInfo.getSession().getSource(),
environment,
queryInfo.getSession().getCatalog(),
queryInfo.getSession().getSchema(),
queryInfo.getSession().getRemoteUserAddress(),
queryInfo.getSession().getUserAgent(),
queryInfo.getState(),
queryInfo.getSelf(),
queryInfo.getFieldNames(),
queryInfo.getQuery(),
queryStats.getCreateTime(),
queryStats.getExecutionStartTime(),
queryStats.getEndTime(),
queryStats.getQueuedTime(),
queryStats.getAnalysisTime(),
queryStats.getDistributedPlanningTime(),
queryStats.getTotalScheduledTime(),
queryStats.getTotalCpuTime(),
queryStats.getRawInputDataSize(),
queryStats.getRawInputPositions(),
queryStats.getTotalDrivers(),
queryInfo.getErrorCode(),
failureType,
failureMessage,
objectMapper.writeValueAsString(queryInfo.getOutputStage()),
objectMapper.writeValueAsString(queryInfo.getFailureInfo()),
objectMapper.writeValueAsString(queryInfo.getInputs())));
logQueryTimeline(queryInfo);
} catch (JsonProcessingException e) {
throw Throwables.propagate(e);
}
}
private void logQueryTimeline(QueryInfo queryInfo) {
try {
QueryStats queryStats = queryInfo.getQueryStats();
DateTime queryStartTime = queryStats.getCreateTime();
DateTime queryEndTime = queryStats.getEndTime();
// query didn't finish cleanly
if (queryStartTime == null || queryEndTime == null) {
return;
}
// planning duration -- start to end of planning
Duration planning = queryStats.getTotalPlanningTime();
if (planning == null) {
planning = new Duration(0, MILLISECONDS);
}
List<StageInfo> stages = StageInfo.getAllStages(queryInfo.getOutputStage());
// long lastSchedulingCompletion = 0;
long firstTaskStartTime = queryEndTime.getMillis();
long lastTaskStartTime = queryStartTime.getMillis() + planning.toMillis();
long lastTaskEndTime = queryStartTime.getMillis() + planning.toMillis();
for (StageInfo stage : stages) {
// only consider leaf stages
if (!stage.getSubStages().isEmpty()) {
continue;
}
for (TaskInfo taskInfo : stage.getTasks()) {
TaskStats taskStats = taskInfo.getStats();
DateTime firstStartTime = taskStats.getFirstStartTime();
if (firstStartTime != null) {
firstTaskStartTime = Math.min(firstStartTime.getMillis(), firstTaskStartTime);
}
DateTime lastStartTime = taskStats.getLastStartTime();
if (lastStartTime != null) {
lastTaskStartTime = Math.max(lastStartTime.getMillis(), lastTaskStartTime);
}
DateTime endTime = taskStats.getEndTime();
if (endTime != null) {
lastTaskEndTime = Math.max(endTime.getMillis(), lastTaskEndTime);
}
}
}
Duration elapsed = millis(queryEndTime.getMillis() - queryStartTime.getMillis());
Duration scheduling =
millis(firstTaskStartTime - queryStartTime.getMillis() - planning.toMillis());
Duration running = millis(lastTaskEndTime - firstTaskStartTime);
Duration finishing = millis(queryEndTime.getMillis() - lastTaskEndTime);
log.info(
"TIMELINE: Query %s :: elapsed %s :: planning %s :: scheduling %s :: running %s :: finishing %s :: begin %s :: end %s",
queryInfo.getQueryId(),
elapsed,
planning,
scheduling,
running,
finishing,
queryStartTime,
queryEndTime);
} catch (Exception e) {
log.error(e, "Error logging query timeline");
}
}
