@Override public void databases(final LockResult lr) { if (qp == null) { lr.writeAll = true; } else { qp.databases(lr); info.readLocked = lr.readAll ? null : lr.read; info.writeLocked = lr.writeAll ? null : lr.write; } }
/** * Indicate to the tracker that a fragment is complete. This is from internal execution within the * daemon * * @param fragmentInfo */ void fragmentComplete(QueryFragmentInfo fragmentInfo) { QueryIdentifier qId = fragmentInfo.getQueryInfo().getQueryIdentifier(); QueryInfo queryInfo = queryInfoMap.get(qId); if (queryInfo == null) { // Possible because a queryComplete message from the AM can come in first - KILL / SUCCESSFUL, // before the fragmentComplete is reported LOG.info("Ignoring fragmentComplete message for unknown query: {}", qId); } else { queryInfo.unregisterFragment(fragmentInfo); } }
/** * Register an update to a source within an executing dag * * @param queryIdentifier * @param sourceName * @param sourceState */ void registerSourceStateChange( QueryIdentifier queryIdentifier, String sourceName, SourceStateProto sourceState) { getSourceCompletionMap(queryIdentifier).put(sourceName, sourceState); QueryInfo queryInfo = queryInfoMap.get(queryIdentifier); if (queryInfo != null) { queryInfo.sourceStateUpdated(sourceName); } else { // Could be null if there's a race between the threads processing requests, with a // dag finish processed earlier. } }
/** * Register a new fragment for a specific query * * @param queryIdentifier * @param appIdString * @param dagName * @param dagIdentifier * @param vertexName * @param fragmentNumber * @param attemptNumber * @param user * @throws IOException */ QueryFragmentInfo registerFragment( QueryIdentifier queryIdentifier, String appIdString, String dagName, int dagIdentifier, String vertexName, int fragmentNumber, int attemptNumber, String user, FragmentSpecProto fragmentSpec) throws IOException { ReadWriteLock dagLock = getDagLock(queryIdentifier); dagLock.readLock().lock(); try { if (!completedDagMap.contains(queryIdentifier)) { QueryInfo queryInfo = queryInfoMap.get(queryIdentifier); if (queryInfo == null) { queryInfo = new QueryInfo( queryIdentifier, appIdString, dagName, dagIdentifier, user, getSourceCompletionMap(queryIdentifier), localDirsBase, localFs); queryInfoMap.putIfAbsent(queryIdentifier, queryInfo); } return queryInfo.registerFragment(vertexName, fragmentNumber, attemptNumber, fragmentSpec); } else { // Cleanup the dag lock here, since it may have been created after the query completed dagSpecificLocks.remove(queryIdentifier); throw new RuntimeException( "Dag " + dagName + " already complete. Rejecting fragment [" + vertexName + ", " + fragmentNumber + ", " + attemptNumber + "]"); } } finally { dagLock.readLock().unlock(); } }
/** * Parses the query. * * @param p performance * @throws QueryException query exception */ private void parse(final Performance p) throws QueryException { qp.http(http); for (final Entry<String, String[]> entry : vars.entrySet()) { final String name = entry.getKey(); final String[] value = entry.getValue(); if (name == null) qp.context(value[0], value[1]); else qp.bind(name, value[0], value[1]); } qp.parse(); if (p != null) info.parsing += p.time(); }
/** * Register completion for a query * * @param queryIdentifier * @param deleteDelay */ List<QueryFragmentInfo> queryComplete(QueryIdentifier queryIdentifier, long deleteDelay) { if (deleteDelay == -1) { deleteDelay = defaultDeleteDelaySeconds; } ReadWriteLock dagLock = getDagLock(queryIdentifier); dagLock.writeLock().lock(); try { rememberCompletedDag(queryIdentifier); LOG.info( "Processing queryComplete for queryIdentifier={} with deleteDelay={} seconds", queryIdentifier, deleteDelay); QueryInfo queryInfo = queryInfoMap.remove(queryIdentifier); if (queryInfo == null) { LOG.warn("Ignoring query complete for unknown dag: {}", queryIdentifier); return Collections.emptyList(); } String[] localDirs = queryInfo.getLocalDirsNoCreate(); if (localDirs != null) { for (String localDir : localDirs) { cleanupDir(localDir, deleteDelay); ShuffleHandler.get() .unregisterDag(localDir, queryInfo.getAppIdString(), queryInfo.getDagIdentifier()); } } // Clearing this before sending a kill is OK, since canFinish will change to false. // Ideally this should be a state machine where kills are issued to the executor, // and the structures are cleaned up once all tasks complete. New requests, however, // should not be allowed after a query complete is received. sourceCompletionMap.remove(queryIdentifier); String savedQueryId = queryIdentifierToHiveQueryId.remove(queryIdentifier); dagSpecificLocks.remove(queryIdentifier); if (savedQueryId != null) { ObjectCacheFactory.removeLlapQueryCache(savedQueryId); } return queryInfo.getRegisteredFragments(); } finally { dagLock.writeLock().unlock(); } }
/* * public List<NodeInfo> queryIndex( String indexId, String queryString ) * throws ParseException, IOException { return queryIndex(indexId, * this.queryConstructor.construct(getEnvironment(indexId), queryString)); } * * * public List<NodeInfo> queryIndex( String indexId, Query query ) throws * IOException { return new Querier(getEnvironment(indexId)).query(query); } */ public String highlightQuery(Integer queryId, String fieldName, String text) { // return text; if (null == this.queryHighlighter) { // sort of lazy init if we forgot to specify more advanced // highlighter this.setQueryHighlighter(new QueryHighlighter()); } QueryInfo queryInfo = this.queryPool.getQueryInfo(queryId); try { return queryHighlighter .setEnvironment(getEnvironment(queryInfo.getIndexId())) .highlightQuery(queryInfo, fieldName, text); } catch (RuntimeException e) { e.printStackTrace(); logger.debug("DEBUG1!!->" + queryId); logger.error("ERROR!!->"); logger.error("ERROR12!!->" + queryInfo); logger.error("ERROR2!!->" + queryInfo.getIndexId()); logger.error("ERROR3!!->" + getEnvironment(queryInfo.getIndexId())); } return null; }
@Override public void process(IQ packet) { Paging<GroupInfo> paging; try { /* 分页查询圈子数据 */ RSMPacketExtension rsm = QueryInfo.getRSM(packet); paging = groupManager.search(QueryInfo.getQuery(packet), rsm.getIndex(0), rsm.getMax(10)); } catch (Exception e) { handleException(e, "搜索圈子失败"); replyError(packet, PacketError.Condition.internal_server_error); return; } /* 构建分页数据包 */ PagingPacket pagingPacket = new PagingPacket( SEARCH_NAMESPACE, paging, new GroupSummaryProcessDelegate(groupService.getServiceDomain())); IQ reply = IQ.createResultIQ(packet); reply.setChildElement(pagingPacket.getElement()); routePacket(reply); }
public void failAbandonedQueries() { for (QueryExecution queryExecution : queries.values()) { try { QueryInfo queryInfo = queryExecution.getQueryInfo(); if (queryInfo.getState().isDone()) { continue; } if (isAbandoned(queryExecution)) { log.info("Failing abandoned query %s", queryExecution.getQueryInfo().getQueryId()); queryExecution.fail( new AbandonedException( "Query " + queryInfo.getQueryId(), queryInfo.getQueryStats().getLastHeartbeat(), DateTime.now())); } } catch (RuntimeException e) { log.warn( e, "Error while inspecting age of query %s", queryExecution.getQueryInfo().getQueryId()); } } }
/** * Evaluates the specified query. * * @param query query * @return success flag */ final boolean query(final String query) { final Performance p = new Performance(); String error; if (exception != null) { error = Util.message(exception); } else { try { long hits = 0; final boolean run = options.get(MainOptions.RUNQUERY); final boolean serial = options.get(MainOptions.SERIALIZE); final int runs = Math.max(1, options.get(MainOptions.RUNS)); for (int r = 0; r < runs; ++r) { // reuse existing processor instance if (r != 0) qp = null; qp(query, context); parse(p); if (r == 0) plan(false); qp.compile(); info.compiling += p.time(); if (r == 0) plan(true); if (!run) continue; final PrintOutput po = r == 0 && serial ? out : new NullOutput(); try (final Serializer ser = qp.getSerializer(po)) { if (maxResults >= 0) { result = qp.cache(maxResults); info.evaluating += p.time(); result.serialize(ser); hits = result.size(); } else { hits = 0; final Iter ir = qp.iter(); info.evaluating += p.time(); for (Item it; (it = ir.next()) != null; ) { ser.serialize(it); ++hits; checkStop(); } } } qp.close(); info.serializing += p.time(); } // dump some query info // out.flush(); // remove string list if global locking is used and if query is updating if (soptions.get(StaticOptions.GLOBALLOCK) && qp.updating) { info.readLocked = null; info.writeLocked = null; } return info(info.toString(qp, out.size(), hits, options.get(MainOptions.QUERYINFO))); } catch (final QueryException | IOException ex) { exception = ex; error = Util.message(ex); } catch (final ProcException ex) { error = INTERRUPTED; } catch (final StackOverflowError ex) { Util.debug(ex); error = BASX_STACKOVERFLOW.desc; } catch (final RuntimeException ex) { extError(""); Util.debug(info()); throw ex; } finally { // close processor after exceptions if (qp != null) qp.close(); } } return extError(error); }
public String queryPaged(Integer queryId, QueryInfo query, HttpServletRequestParameterMap map) throws IOException { return new Querier(getEnvironment(query.getIndexId())).queryPaged(queryId, query, map); }
public ScoreDoc[] queryAllDocs( Integer queryId, QueryInfo query, HttpServletRequestParameterMap map) throws IOException { return new Querier(getEnvironment(query.getIndexId())).queryAllDocs(queryId, query, map); }
public String getQueryString(Integer queryId) { QueryInfo info = this.queryPool.getQueryInfo(queryId); return null != info ? info.getQueryString() : null; }
/** * Resolves a query, given a QueryInfo * * @param info The info encapsulating the query information * @param con the db connection * @param params the parameters passed to query * @return the resolved query * @throws SQLException if resolving the query failed */ PreparedStatement resolveQuery(QueryInfo info, Connection con, Map<String, Object> params) throws SQLException, ResourceException { String queryStr = info.getQueryString(); List<String> tokenNames = info.getTokenNames(); // replace ${list:variable} tokens with the correct number of bind variables Map<String, Integer> listReplacements = new HashMap<String, Integer>(); for (String tokenName : tokenNames) { String[] tokenParts = tokenName.split(":", 2); if (PREFIX_LIST.equals(tokenParts[0]) && params.containsKey(tokenParts[1])) { listReplacements.put(tokenName, ((String) params.get(tokenParts[1])).split(",").length); } } if (listReplacements.size() > 0) { TokenHandler tokenHandler = new TokenHandler(); queryStr = tokenHandler.replaceListTokens(queryStr, listReplacements, "?"); } // now prepare the statement using the correct number of bind variables PreparedStatement statement = getPreparedStatement(con, queryStr); int count = 1; // DB column count starts at 1 for (String tokenName : tokenNames) { String[] tokenParts = tokenName.split(":", 2); if (tokenParts.length == 1) { // handle single value - assume String Object objValue = params.get(tokenName); String value = null; if (objValue != null) { value = trimValue(objValue); } else { // fail with an exception if token not found throw new BadRequestException( "Missing entry in params passed to query for token " + tokenName); } statement.setString(count, value); count++; } else { Object objValue = params.get(tokenParts[1]); if (objValue == null) { // fail with an exception if token not found throw new BadRequestException( "Missing entry in params passed to query for token " + tokenName); } if (PREFIX_INT.equals(tokenParts[0])) { // handle single integer value Integer int_value = null; if (objValue != null) { int_value = Integer.parseInt(objValue.toString()); } statement.setInt(count, int_value); count++; } else if (PREFIX_LIST.equals(tokenParts[0])) { // handle list of values - presently assumes Strings, TODO support integer lists if (objValue != null) { for (String list_value : objValue.toString().split(",")) { // if list value is surrounded by single quotes remove them if (list_value != null && list_value.startsWith("'") && list_value.endsWith("'")) { list_value = list_value.substring(1, list_value.length() - 1); } statement.setString(count, trimValue(list_value)); count++; } } else { statement.setString(count, null); count++; } } } } logger.debug("Prepared statement: {}", statement); return statement; }