@Override
public void databases(final LockResult lr) {
if (qp == null) {
lr.writeAll = true;
} else {
qp.databases(lr);
info.readLocked = lr.readAll ? null : lr.read;
info.writeLocked = lr.writeAll ? null : lr.write;
}
}
/**
* Indicate to the tracker that a fragment is complete. This is from internal execution within the
* daemon
*
* @param fragmentInfo
*/
void fragmentComplete(QueryFragmentInfo fragmentInfo) {
QueryIdentifier qId = fragmentInfo.getQueryInfo().getQueryIdentifier();
QueryInfo queryInfo = queryInfoMap.get(qId);
if (queryInfo == null) {
// Possible because a queryComplete message from the AM can come in first - KILL / SUCCESSFUL,
// before the fragmentComplete is reported
LOG.info("Ignoring fragmentComplete message for unknown query: {}", qId);
} else {
queryInfo.unregisterFragment(fragmentInfo);
}
}
/**
* Register an update to a source within an executing dag
*
* @param queryIdentifier
* @param sourceName
* @param sourceState
*/
void registerSourceStateChange(
QueryIdentifier queryIdentifier, String sourceName, SourceStateProto sourceState) {
getSourceCompletionMap(queryIdentifier).put(sourceName, sourceState);
QueryInfo queryInfo = queryInfoMap.get(queryIdentifier);
if (queryInfo != null) {
queryInfo.sourceStateUpdated(sourceName);
} else {
// Could be null if there's a race between the threads processing requests, with a
// dag finish processed earlier.
}
}
/**
* Register a new fragment for a specific query
*
* @param queryIdentifier
* @param appIdString
* @param dagName
* @param dagIdentifier
* @param vertexName
* @param fragmentNumber
* @param attemptNumber
* @param user
* @throws IOException
*/
QueryFragmentInfo registerFragment(
QueryIdentifier queryIdentifier,
String appIdString,
String dagName,
int dagIdentifier,
String vertexName,
int fragmentNumber,
int attemptNumber,
String user,
FragmentSpecProto fragmentSpec)
throws IOException {
ReadWriteLock dagLock = getDagLock(queryIdentifier);
dagLock.readLock().lock();
try {
if (!completedDagMap.contains(queryIdentifier)) {
QueryInfo queryInfo = queryInfoMap.get(queryIdentifier);
if (queryInfo == null) {
queryInfo =
new QueryInfo(
queryIdentifier,
appIdString,
dagName,
dagIdentifier,
user,
getSourceCompletionMap(queryIdentifier),
localDirsBase,
localFs);
queryInfoMap.putIfAbsent(queryIdentifier, queryInfo);
}
return queryInfo.registerFragment(vertexName, fragmentNumber, attemptNumber, fragmentSpec);
} else {
// Cleanup the dag lock here, since it may have been created after the query completed
dagSpecificLocks.remove(queryIdentifier);
throw new RuntimeException(
"Dag "
+ dagName
+ " already complete. Rejecting fragment ["
+ vertexName
+ ", "
+ fragmentNumber
+ ", "
+ attemptNumber
+ "]");
}
} finally {
dagLock.readLock().unlock();
}
}
/**
* Parses the query.
*
* @param p performance
* @throws QueryException query exception
*/
private void parse(final Performance p) throws QueryException {
qp.http(http);
for (final Entry<String, String[]> entry : vars.entrySet()) {
final String name = entry.getKey();
final String[] value = entry.getValue();
if (name == null) qp.context(value[0], value[1]);
else qp.bind(name, value[0], value[1]);
}
qp.parse();
if (p != null) info.parsing += p.time();
}
/**
* Register completion for a query
*
* @param queryIdentifier
* @param deleteDelay
*/
List<QueryFragmentInfo> queryComplete(QueryIdentifier queryIdentifier, long deleteDelay) {
if (deleteDelay == -1) {
deleteDelay = defaultDeleteDelaySeconds;
}
ReadWriteLock dagLock = getDagLock(queryIdentifier);
dagLock.writeLock().lock();
try {
rememberCompletedDag(queryIdentifier);
LOG.info(
"Processing queryComplete for queryIdentifier={} with deleteDelay={} seconds",
queryIdentifier,
deleteDelay);
QueryInfo queryInfo = queryInfoMap.remove(queryIdentifier);
if (queryInfo == null) {
LOG.warn("Ignoring query complete for unknown dag: {}", queryIdentifier);
return Collections.emptyList();
}
String[] localDirs = queryInfo.getLocalDirsNoCreate();
if (localDirs != null) {
for (String localDir : localDirs) {
cleanupDir(localDir, deleteDelay);
ShuffleHandler.get()
.unregisterDag(localDir, queryInfo.getAppIdString(), queryInfo.getDagIdentifier());
}
}
// Clearing this before sending a kill is OK, since canFinish will change to false.
// Ideally this should be a state machine where kills are issued to the executor,
// and the structures are cleaned up once all tasks complete. New requests, however,
// should not be allowed after a query complete is received.
sourceCompletionMap.remove(queryIdentifier);
String savedQueryId = queryIdentifierToHiveQueryId.remove(queryIdentifier);
dagSpecificLocks.remove(queryIdentifier);
if (savedQueryId != null) {
ObjectCacheFactory.removeLlapQueryCache(savedQueryId);
}
return queryInfo.getRegisteredFragments();
} finally {
dagLock.writeLock().unlock();
}
}
/*
* public List<NodeInfo> queryIndex( String indexId, String queryString )
* throws ParseException, IOException { return queryIndex(indexId,
* this.queryConstructor.construct(getEnvironment(indexId), queryString)); }
*
*
* public List<NodeInfo> queryIndex( String indexId, Query query ) throws
* IOException { return new Querier(getEnvironment(indexId)).query(query); }
*/
public String highlightQuery(Integer queryId, String fieldName, String text) {
// return text;
if (null == this.queryHighlighter) {
// sort of lazy init if we forgot to specify more advanced
// highlighter
this.setQueryHighlighter(new QueryHighlighter());
}
QueryInfo queryInfo = this.queryPool.getQueryInfo(queryId);
try {
return queryHighlighter
.setEnvironment(getEnvironment(queryInfo.getIndexId()))
.highlightQuery(queryInfo, fieldName, text);
} catch (RuntimeException e) {
e.printStackTrace();
logger.debug("DEBUG1!!->" + queryId);
logger.error("ERROR!!->");
logger.error("ERROR12!!->" + queryInfo);
logger.error("ERROR2!!->" + queryInfo.getIndexId());
logger.error("ERROR3!!->" + getEnvironment(queryInfo.getIndexId()));
}
return null;
}
@Override
public void process(IQ packet) {
Paging<GroupInfo> paging;
try {
/* 分页查询圈子数据 */
RSMPacketExtension rsm = QueryInfo.getRSM(packet);
paging = groupManager.search(QueryInfo.getQuery(packet), rsm.getIndex(0), rsm.getMax(10));
} catch (Exception e) {
handleException(e, "搜索圈子失败");
replyError(packet, PacketError.Condition.internal_server_error);
return;
}
/* 构建分页数据包 */
PagingPacket pagingPacket =
new PagingPacket(
SEARCH_NAMESPACE,
paging,
new GroupSummaryProcessDelegate(groupService.getServiceDomain()));
IQ reply = IQ.createResultIQ(packet);
reply.setChildElement(pagingPacket.getElement());
routePacket(reply);
}
public void failAbandonedQueries() {
for (QueryExecution queryExecution : queries.values()) {
try {
QueryInfo queryInfo = queryExecution.getQueryInfo();
if (queryInfo.getState().isDone()) {
continue;
}
if (isAbandoned(queryExecution)) {
log.info("Failing abandoned query %s", queryExecution.getQueryInfo().getQueryId());
queryExecution.fail(
new AbandonedException(
"Query " + queryInfo.getQueryId(),
queryInfo.getQueryStats().getLastHeartbeat(),
DateTime.now()));
}
} catch (RuntimeException e) {
log.warn(
e,
"Error while inspecting age of query %s",
queryExecution.getQueryInfo().getQueryId());
}
}
}
/**
* Evaluates the specified query.
*
* @param query query
* @return success flag
*/
final boolean query(final String query) {
final Performance p = new Performance();
String error;
if (exception != null) {
error = Util.message(exception);
} else {
try {
long hits = 0;
final boolean run = options.get(MainOptions.RUNQUERY);
final boolean serial = options.get(MainOptions.SERIALIZE);
final int runs = Math.max(1, options.get(MainOptions.RUNS));
for (int r = 0; r < runs; ++r) {
// reuse existing processor instance
if (r != 0) qp = null;
qp(query, context);
parse(p);
if (r == 0) plan(false);
qp.compile();
info.compiling += p.time();
if (r == 0) plan(true);
if (!run) continue;
final PrintOutput po = r == 0 && serial ? out : new NullOutput();
try (final Serializer ser = qp.getSerializer(po)) {
if (maxResults >= 0) {
result = qp.cache(maxResults);
info.evaluating += p.time();
result.serialize(ser);
hits = result.size();
} else {
hits = 0;
final Iter ir = qp.iter();
info.evaluating += p.time();
for (Item it; (it = ir.next()) != null; ) {
ser.serialize(it);
++hits;
checkStop();
}
}
}
qp.close();
info.serializing += p.time();
}
// dump some query info
// out.flush();
// remove string list if global locking is used and if query is updating
if (soptions.get(StaticOptions.GLOBALLOCK) && qp.updating) {
info.readLocked = null;
info.writeLocked = null;
}
return info(info.toString(qp, out.size(), hits, options.get(MainOptions.QUERYINFO)));
} catch (final QueryException | IOException ex) {
exception = ex;
error = Util.message(ex);
} catch (final ProcException ex) {
error = INTERRUPTED;
} catch (final StackOverflowError ex) {
Util.debug(ex);
error = BASX_STACKOVERFLOW.desc;
} catch (final RuntimeException ex) {
extError("");
Util.debug(info());
throw ex;
} finally {
// close processor after exceptions
if (qp != null) qp.close();
}
}
return extError(error);
}
public String queryPaged(Integer queryId, QueryInfo query, HttpServletRequestParameterMap map)
throws IOException {
return new Querier(getEnvironment(query.getIndexId())).queryPaged(queryId, query, map);
}
public ScoreDoc[] queryAllDocs(
Integer queryId, QueryInfo query, HttpServletRequestParameterMap map) throws IOException {
return new Querier(getEnvironment(query.getIndexId())).queryAllDocs(queryId, query, map);
}
public String getQueryString(Integer queryId) {
QueryInfo info = this.queryPool.getQueryInfo(queryId);
return null != info ? info.getQueryString() : null;
}
/**
* Resolves a query, given a QueryInfo
*
* @param info The info encapsulating the query information
* @param con the db connection
* @param params the parameters passed to query
* @return the resolved query
* @throws SQLException if resolving the query failed
*/
PreparedStatement resolveQuery(QueryInfo info, Connection con, Map<String, Object> params)
throws SQLException, ResourceException {
String queryStr = info.getQueryString();
List<String> tokenNames = info.getTokenNames();
// replace ${list:variable} tokens with the correct number of bind variables
Map<String, Integer> listReplacements = new HashMap<String, Integer>();
for (String tokenName : tokenNames) {
String[] tokenParts = tokenName.split(":", 2);
if (PREFIX_LIST.equals(tokenParts[0]) && params.containsKey(tokenParts[1])) {
listReplacements.put(tokenName, ((String) params.get(tokenParts[1])).split(",").length);
}
}
if (listReplacements.size() > 0) {
TokenHandler tokenHandler = new TokenHandler();
queryStr = tokenHandler.replaceListTokens(queryStr, listReplacements, "?");
}
// now prepare the statement using the correct number of bind variables
PreparedStatement statement = getPreparedStatement(con, queryStr);
int count = 1; // DB column count starts at 1
for (String tokenName : tokenNames) {
String[] tokenParts = tokenName.split(":", 2);
if (tokenParts.length == 1) {
// handle single value - assume String
Object objValue = params.get(tokenName);
String value = null;
if (objValue != null) {
value = trimValue(objValue);
} else {
// fail with an exception if token not found
throw new BadRequestException(
"Missing entry in params passed to query for token " + tokenName);
}
statement.setString(count, value);
count++;
} else {
Object objValue = params.get(tokenParts[1]);
if (objValue == null) {
// fail with an exception if token not found
throw new BadRequestException(
"Missing entry in params passed to query for token " + tokenName);
}
if (PREFIX_INT.equals(tokenParts[0])) {
// handle single integer value
Integer int_value = null;
if (objValue != null) {
int_value = Integer.parseInt(objValue.toString());
}
statement.setInt(count, int_value);
count++;
} else if (PREFIX_LIST.equals(tokenParts[0])) {
// handle list of values - presently assumes Strings, TODO support integer lists
if (objValue != null) {
for (String list_value : objValue.toString().split(",")) {
// if list value is surrounded by single quotes remove them
if (list_value != null && list_value.startsWith("'") && list_value.endsWith("'")) {
list_value = list_value.substring(1, list_value.length() - 1);
}
statement.setString(count, trimValue(list_value));
count++;
}
} else {
statement.setString(count, null);
count++;
}
}
}
}
logger.debug("Prepared statement: {}", statement);
return statement;
}
