@Override public void execute( QueryMaster.QueryMasterContext context, QueryContext queryContext, Query query, ExecutionBlockId finalExecBlockId, Path finalOutputDir) throws Exception { CatalogService catalog = context.getWorkerContext().getCatalog(); SubQuery lastStage = query.getSubQuery(finalExecBlockId); TableMeta meta = lastStage.getTableMeta(); TableStats stats = lastStage.getTableStat(); CreateTableNode createTableNode = (CreateTableNode) lastStage.getBlock().getPlan(); TableDesc tableDescTobeCreated = new TableDesc( createTableNode.getTableName(), createTableNode.getTableSchema(), meta, finalOutputDir); if (createTableNode.hasPartition()) { tableDescTobeCreated.setPartitionMethod(createTableNode.getPartitionMethod()); } stats.setNumBytes(getTableVolume(query.systemConf, finalOutputDir)); tableDescTobeCreated.setStats(stats); query.setResultDesc(tableDescTobeCreated); catalog.addTable(tableDescTobeCreated); }
@Override public void transition(Query query, QueryEvent event) { synchronized (query.subqueries) { for (SubQuery subquery : query.subqueries.values()) { query.eventHandler.handle(new SubQueryEvent(subquery.getId(), SubQueryEventType.SQ_KILL)); } } }
@Override public boolean isEligible( QueryContext queryContext, Query query, ExecutionBlockId finalExecBlockId, Path finalOutputDir) { SubQuery lastStage = query.getSubQuery(finalExecBlockId); return lastStage.getBlock().getPlan().getType() == NodeType.INSERT; }
@Override public void transition(Query query, QueryEvent queryEvent) { query.setStartTime(); SubQuery subQuery = new SubQuery( query.context, query.getPlan(), query.getExecutionBlockCursor().nextBlock(), query.sm); subQuery.setPriority(query.priority--); query.addSubQuery(subQuery); subQuery.handle(new SubQueryEvent(subQuery.getId(), SubQueryEventType.SQ_INIT)); LOG.debug("Schedule unit plan: \n" + subQuery.getBlock().getPlan()); }
public void xqderiveMemberAddressList( String function, SubQuery<MemberAddressCB> subQuery, String operand, Object value, DerivedReferrerOption option) { assertObjectNotNull("subQuery<MemberAddressCB>", subQuery); MemberAddressCB cb = new MemberAddressCB(); cb.xsetupForDerivedReferrer(this); subQuery.query(cb); String subQueryPropertyName = keepRegionId_QueryDerivedReferrer_MemberAddressList(cb.query()); // for saving query-value. String parameterPropertyName = keepRegionId_QueryDerivedReferrer_MemberAddressListParameter(value); registerQueryDerivedReferrer( function, cb.query(), "REGION_ID", "REGION_ID", subQueryPropertyName, operand, value, parameterPropertyName, option); }
public void xqderiveMemberServiceList( String function, SubQuery<MemberServiceCB> subQuery, String operand, Object value, DerivedReferrerOption option) { assertObjectNotNull("subQuery<MemberServiceCB>", subQuery); MemberServiceCB cb = new MemberServiceCB(); cb.xsetupForDerivedReferrer(this); subQuery.query(cb); String subQueryPropertyName = keepServiceRankCode_QueryDerivedReferrer_MemberServiceList( cb.query()); // for saving query-value. String parameterPropertyName = keepServiceRankCode_QueryDerivedReferrer_MemberServiceListParameter(value); registerQueryDerivedReferrer( function, cb.query(), "SERVICE_RANK_CODE", "SERVICE_RANK_CODE", subQueryPropertyName, operand, value, parameterPropertyName, option); }
/** * Prepare for MyselfExists (correlated sub-query). * * @param subCBLambda The implementation of sub-query. (NotNull) */ public void myselfExists(SubQuery<TweetCB> subCBLambda) { assertObjectNotNull("subCBLambda", subCBLambda); TweetCB cb = new TweetCB(); cb.xsetupForMyselfExists(this); lockCall(() -> subCBLambda.query(cb)); String pp = keepMyselfExists(cb.query()); registerMyselfExists(cb.query(), pp); }
/** * Myself InScope (SubQuery). {mainly to avoid CLOB and Union problem or no master table} * * @param subQuery The implementation of sub query. (NotNull) */ public void myselfInScope(SubQuery<ServiceRankCB> subQuery) { assertObjectNotNull("subQuery<ServiceRankCB>", subQuery); ServiceRankCB cb = new ServiceRankCB(); cb.xsetupForMyselfInScope(this); subQuery.query(cb); String subQueryPropertyName = keepMyselfInScope(cb.query()); // for saving query-value. registerMyselfInScope(cb.query(), subQueryPropertyName); }
protected void xscalarCondition(String function, SubQuery<RegionCB> subQuery, String operand) { assertObjectNotNull("subQuery<RegionCB>", subQuery); RegionCB cb = new RegionCB(); cb.xsetupForScalarCondition(this); subQuery.query(cb); String subQueryPropertyName = keepScalarCondition(cb.query()); // for saving query-value. registerScalarCondition(function, cb.query(), subQueryPropertyName, operand); }
/** * Set up ExistsReferrer (co-related sub-query). <br> * {exists (select REGION_ID from MEMBER_ADDRESS where ...)} <br> * MEMBER_ADDRESS as 'memberAddressAsOne'. * * <pre> * cb.query().<span style="color: #FD4747">existsMemberAddressList</span>(new SubQuery<MemberAddressCB>() { * public void query(RegionCB subCB) { * subCB.query().setXxx... * } * }); * </pre> * * @param subQuery The sub-query of MemberAddressList for 'exists'. (NotNull) */ public void existsMemberAddressList(SubQuery<MemberAddressCB> subQuery) { assertObjectNotNull("subQuery<MemberAddressCB>", subQuery); MemberAddressCB cb = new MemberAddressCB(); cb.xsetupForExistsReferrer(this); subQuery.query(cb); String subQueryPropertyName = keepRegionId_ExistsReferrer_MemberAddressList(cb.query()); // for saving query-value. registerExistsReferrer(cb.query(), "REGION_ID", "REGION_ID", subQueryPropertyName); }
// =================================================================================== // MyselfDerived // ============= public void xsmyselfDerive(String fn, SubQuery<TweetCB> sq, String al, DerivedReferrerOption op) { assertObjectNotNull("subQuery", sq); TweetCB cb = new TweetCB(); cb.xsetupForDerivedReferrer(this); lockCall(() -> sq.query(cb)); String pp = keepSpecifyMyselfDerived(cb.query()); String pk = "TWEET_ID"; registerSpecifyMyselfDerived(fn, cb.query(), pk, pk, pp, "myselfDerived", al, op); }
/** * Set up NotInScopeRelation (sub-query). <br> * {not in (select REGION_ID from MEMBER_ADDRESS where ...)} <br> * MEMBER_ADDRESS as 'memberAddressAsOne'. * * @param subQuery The sub-query of MemberAddressList for 'not in-scope'. (NotNull) */ public void notInScopeMemberAddressList(SubQuery<MemberAddressCB> subQuery) { assertObjectNotNull("subQuery<MemberAddressCB>", subQuery); MemberAddressCB cb = new MemberAddressCB(); cb.xsetupForInScopeRelation(this); subQuery.query(cb); String subQueryPropertyName = keepRegionId_NotInScopeRelation_MemberAddressList(cb.query()); // for saving query-value. registerNotInScopeRelation(cb.query(), "REGION_ID", "REGION_ID", subQueryPropertyName); }
/** * A Constructor for handling MULTIJOINEXPRESSIONS * * @param leftItem - the JOIN's left ITEM, which is a subquery * @param t - the Tree where we get the join, right, onclause * @param treeBuilder * @param selectStatement * @throws TreeParsingException * @throws ColumnCallException */ public JoinExpression( SubQuery leftItem, Tree t, TreeBuilder treeBuilder, SelectStatement selectStatement) throws TreeParsingException, ColumnCallException { this.builder = treeBuilder; this.selectStatement = selectStatement; this.leftItem = leftItem; leftItem.setisPartOfJoin(); this.build(t, this.builder); }
@SuppressWarnings("unchecked") protected <CB extends ConditionBean> void xscalarCondition( String fn, SubQuery<CB> sq, String rd, HpSLCCustomized<CB> cs, ScalarConditionOption op) { assertObjectNotNull("subQuery", sq); TweetCB cb = xcreateScalarConditionCB(); sq.query((CB) cb); String pp = keepScalarCondition(cb.query()); // for saving query-value cs.setPartitionByCBean((CB) xcreateScalarConditionPartitionByCB()); // for using partition-by registerScalarCondition(fn, cb.query(), pp, rd, cs, op); }
/** * Set up ExistsReferrer (co-related sub-query). <br> * {exists (select SERVICE_RANK_CODE from MEMBER_SERVICE where ...)} <br> * MEMBER_SERVICE as 'memberServiceAsOne'. * * <pre> * cb.query().<span style="color: #FD4747">existsMemberServiceList</span>(new SubQuery<MemberServiceCB>() { * public void query(ServiceRankCB subCB) { * subCB.query().setXxx... * } * }); * </pre> * * @param subQuery The sub-query of MemberServiceList for 'exists'. (NotNull) */ public void existsMemberServiceList(SubQuery<MemberServiceCB> subQuery) { assertObjectNotNull("subQuery<MemberServiceCB>", subQuery); MemberServiceCB cb = new MemberServiceCB(); cb.xsetupForExistsReferrer(this); subQuery.query(cb); String subQueryPropertyName = keepServiceRankCode_ExistsReferrer_MemberServiceList(cb.query()); // for saving query-value. registerExistsReferrer( cb.query(), "SERVICE_RANK_CODE", "SERVICE_RANK_CODE", subQueryPropertyName); }
@Override public void execute( QueryMaster.QueryMasterContext context, QueryContext queryContext, Query query, ExecutionBlockId finalExecBlockId, Path finalOutputDir) throws Exception { SubQuery lastStage = query.getSubQuery(finalExecBlockId); TableMeta meta = lastStage.getTableMeta(); TableStats stats = lastStage.getTableStat(); TableDesc resultTableDesc = new TableDesc(query.getId().toString(), lastStage.getSchema(), meta, finalOutputDir); stats.setNumBytes(getTableVolume(query.systemConf, finalOutputDir)); resultTableDesc.setStats(stats); query.setResultDesc(resultTableDesc); }
/** * Set up NotInScopeRelation (sub-query). <br> * {not in (select SERVICE_RANK_CODE from MEMBER_SERVICE where ...)} <br> * MEMBER_SERVICE as 'memberServiceAsOne'. * * @param subQuery The sub-query of MemberServiceList for 'not in-scope'. (NotNull) */ public void notInScopeMemberServiceList(SubQuery<MemberServiceCB> subQuery) { assertObjectNotNull("subQuery<MemberServiceCB>", subQuery); MemberServiceCB cb = new MemberServiceCB(); cb.xsetupForInScopeRelation(this); subQuery.query(cb); String subQueryPropertyName = keepServiceRankCode_NotInScopeRelation_MemberServiceList( cb.query()); // for saving query-value. registerNotInScopeRelation( cb.query(), "SERVICE_RANK_CODE", "SERVICE_RANK_CODE", subQueryPropertyName); }
@SuppressWarnings("unchecked") protected <CB extends ConditionBean> void xqderiveMyselfDerived( String fn, SubQuery<CB> sq, String rd, Object vl, DerivedReferrerOption op) { assertObjectNotNull("subQuery", sq); TweetCB cb = new TweetCB(); cb.xsetupForDerivedReferrer(this); sq.query((CB) cb); String pk = "TWEET_ID"; String sqpp = keepQueryMyselfDerived(cb.query()); // for saving query-value. String prpp = keepQueryMyselfDerivedParameter(vl); registerQueryMyselfDerived(fn, cb.query(), pk, pk, sqpp, "myselfDerived", rd, vl, prpp, op); }
public float getProgress() { QueryState state = getStateMachine().getCurrentState(); if (state == QueryState.QUERY_SUCCEEDED) { return 1.0f; } else { int idx = 0; List<SubQuery> tempSubQueries = new ArrayList<SubQuery>(); synchronized (subqueries) { tempSubQueries.addAll(subqueries.values()); } float[] subProgresses = new float[tempSubQueries.size()]; boolean finished = true; for (SubQuery subquery : tempSubQueries) { if (subquery.getState() != SubQueryState.NEW) { subProgresses[idx] = subquery.getProgress(); if (finished && subquery.getState() != SubQueryState.SUCCEEDED) { finished = false; } } else { subProgresses[idx] = 0.0f; } idx++; } if (finished) { return 1.0f; } float totalProgress = 0; float proportion = 1.0f / (float) (getExecutionBlockCursor().size() - 1); // minus one is due to for (int i = 0; i < subProgresses.length; i++) { totalProgress += subProgresses[i] * proportion; } return totalProgress; } }
/** * A Constructor to build JoinExpressions from two Subqueries * * @param builder - the TreeBuilder for the helper functions * @param leftItem - Subquery for the left side of the join * @param rightItem - Subquery for the right side of the join * @param onClause - the onClause to link the 2 subqueries * @param selectStatement - the selectStatement which contains the join */ public JoinExpression( TreeBuilder builder, Node leftItem, Node rightItem, OnClause onClause, SelectStatement selectStatement) { this.selectStatement = selectStatement; this.builder = builder; this.leftItem = leftItem; this.rightItem = rightItem; if (leftItem.tokenType == JdbcGrammarParser.SUBQUERY) { SubQuery subQuery = SubQuery.class.cast(leftItem); subQuery.setisPartOfJoin(); } if (rightItem.tokenType == JdbcGrammarParser.SUBQUERY) { SubQuery subQuery = SubQuery.class.cast(rightItem); subQuery.setisPartOfJoin(); } this.onClause = onClause; this.tokenType = JdbcGrammarParser.JOINEXPRESSION; this.tokenName = JdbcGrammarParser.tokenNames[JdbcGrammarParser.JOINEXPRESSION]; }
@Override public void execute( QueryMaster.QueryMasterContext context, QueryContext queryContext, Query query, ExecutionBlockId finalExecBlockId, Path finalOutputDir) throws Exception { CatalogService catalog = context.getWorkerContext().getCatalog(); SubQuery lastStage = query.getSubQuery(finalExecBlockId); TableMeta meta = lastStage.getTableMeta(); TableStats stats = lastStage.getTableStat(); InsertNode insertNode = (InsertNode) lastStage.getBlock().getPlan(); TableDesc finalTable; if (insertNode.hasTargetTable()) { String tableName = insertNode.getTableName(); finalTable = catalog.getTableDesc(tableName); } else { String tableName = query.getId().toString(); finalTable = new TableDesc(tableName, lastStage.getSchema(), meta, finalOutputDir); } long volume = getTableVolume(query.systemConf, finalOutputDir); stats.setNumBytes(volume); finalTable.setStats(stats); if (insertNode.hasTargetTable()) { catalog.deleteTable(insertNode.getTableName()); catalog.addTable(finalTable); } query.setResultDesc(finalTable); }
public void xsderiveMemberAddressList( String function, SubQuery<MemberAddressCB> subQuery, String aliasName, DerivedReferrerOption option) { assertObjectNotNull("subQuery<MemberAddressCB>", subQuery); MemberAddressCB cb = new MemberAddressCB(); cb.xsetupForDerivedReferrer(this); subQuery.query(cb); String subQueryPropertyName = keepRegionId_SpecifyDerivedReferrer_MemberAddressList( cb.query()); // for saving query-value. registerSpecifyDerivedReferrer( function, cb.query(), "REGION_ID", "REGION_ID", subQueryPropertyName, aliasName, option); }
private void executeNextBlock(Query query) { ExecutionBlockCursor cursor = query.getExecutionBlockCursor(); ExecutionBlock nextBlock = cursor.nextBlock(); SubQuery nextSubQuery = new SubQuery(query.context, query.getPlan(), nextBlock, query.sm); nextSubQuery.setPriority(query.priority--); query.addSubQuery(nextSubQuery); nextSubQuery.handle(new SubQueryEvent(nextSubQuery.getId(), SubQueryEventType.SQ_INIT)); LOG.info("Scheduling SubQuery:" + nextSubQuery.getId()); if (LOG.isDebugEnabled()) { LOG.debug("Scheduling SubQuery's Priority: " + nextSubQuery.getPriority()); LOG.debug("Scheduling SubQuery's Plan: \n" + nextSubQuery.getBlock().getPlan()); } }
public void xsderiveMemberServiceList( String function, SubQuery<MemberServiceCB> subQuery, String aliasName, DerivedReferrerOption option) { assertObjectNotNull("subQuery<MemberServiceCB>", subQuery); MemberServiceCB cb = new MemberServiceCB(); cb.xsetupForDerivedReferrer(this); subQuery.query(cb); String subQueryPropertyName = keepServiceRankCode_SpecifyDerivedReferrer_MemberServiceList( cb.query()); // for saving query-value. registerSpecifyDerivedReferrer( function, cb.query(), "SERVICE_RANK_CODE", "SERVICE_RANK_CODE", subQueryPropertyName, aliasName, option); }
public void addSubQuery(SubQuery subquery) { subqueries.put(subquery.getId(), subquery); }
public void build(Tree t, TreeBuilder builder) throws TreeParsingException, ColumnCallException { if (t.getType() == JdbcGrammarParser.JOINEXPRESSION || t.getType() == JdbcGrammarParser.MULTIJOINEXPRESSION) { this.tokenName = JdbcGrammarParser.tokenNames[t.getType()]; this.tokenType = t.getType(); this.logger.debug("BUILDING " + this.tokenName); for (int i = 0; i < t.getChildCount(); i++) { Tree child = t.getChild(i); switch (child.getType()) { case JdbcGrammarParser.LEFTEXPR: switch (child.getChild(0).getType()) { case JdbcGrammarParser.SUBQUERY: SubQuery subQuery = new SubQuery(child.getChild(0), builder); subQuery.setisPartOfJoin(); this.leftItem = subQuery; break; case JdbcGrammarParser.SOURCETABLE: Resolver resolve = new Resolver(builder); // whats the sourcetable? SourceTable tablenode = new SourceTable(child.getChild(0), builder); String alias = tablenode.getAlias(); // getting the columns for it List<ColumnCall> columnlist = resolve.parseSrcTableForJokers(tablenode); // making a new Subquery with those columns + // fromexpression this.logger.debug("got columns"); tablenode.alias = null; // making a fromexpression with that sourcetable FromExpression fromexpression = new FromExpression(tablenode, builder); Expression expression = new Expression(columnlist, builder); SelectStatement mySelectStatement = new SelectStatement(expression, fromexpression, builder); expression.setSelectStatement(mySelectStatement); SubQuery mySubQuery = null; if (alias != null) { // putting the selectstatement into a subquery mySubQuery = new SubQuery(alias, builder, mySelectStatement); } else { String newalias = ""; if (tablenode.getDataset() != null) { newalias += tablenode.getDataset() + "."; } newalias += tablenode.getName(); mySubQuery = new SubQuery(newalias, builder, mySelectStatement); } // adding the subquery to the fromexpression mySubQuery.setisPartOfJoin(); this.leftItem = mySubQuery; break; default: break; } break; case JdbcGrammarParser.RIGHTEXPR: switch (child.getChild(0).getType()) { case JdbcGrammarParser.SUBQUERY: SubQuery subQuery = new SubQuery(child.getChild(0), builder); subQuery.setisPartOfJoin(); this.rightItem = subQuery; break; case JdbcGrammarParser.SOURCETABLE: Resolver resolve = new Resolver(builder); // whats the sourcetable? SourceTable tablenode = new SourceTable(child.getChild(0), builder); String alias = tablenode.getAlias(); // getting the columns for it List<ColumnCall> columnlist = resolve.parseSrcTableForJokers(tablenode); // making a new Subquery with those columns + // fromexpression this.logger.debug("got columns"); tablenode.alias = null; // making a fromexpression with that sourcetable FromExpression fromexpression = new FromExpression(tablenode, builder); Expression expression = new Expression(columnlist, builder); SelectStatement mySelectStatement = new SelectStatement(expression, fromexpression, builder); expression.setSelectStatement(mySelectStatement); SubQuery mySubQuery = null; if (alias != null) { // putting the selectstatement into a subquery mySubQuery = new SubQuery(alias, builder, mySelectStatement); } else { String newalias = ""; if (tablenode.getDataset() != null) { newalias += tablenode.getDataset() + "."; } newalias += tablenode.getName(); mySubQuery = new SubQuery(newalias, builder, mySelectStatement); } // adding the subquery to the fromexpression mySubQuery.setisPartOfJoin(); this.rightItem = mySubQuery; break; default: break; } break; case JdbcGrammarParser.JOINTYPE: switch (child.getChild(0).getType()) { case JdbcGrammarParser.LEFT_KEYWORD: this.type = JoinType.LEFT; break; case JdbcGrammarParser.RIGHT_KEYWORD: this.type = JoinType.RIGHT; break; case JdbcGrammarParser.FULl_KEYWORD: this.type = JoinType.FULL; break; case JdbcGrammarParser.INNERKEYWORD: this.type = JoinType.INNER; break; default: break; } if (child.getChild(1).getType() == JdbcGrammarParser.EACH) { each = true; } break; default: break; } } for (int i = 0; i < t.getChildCount(); i++) { Tree child = t.getChild(i); switch (child.getType()) { case JdbcGrammarParser.ONCLAUSE: this.onClause = new OnClause(child, builder, this.selectStatement); break; default: break; } } for (Node condition : onClause.children) { OnClauseCondition onClauseCondition = OnClauseCondition.class.cast(condition); ColumnReference left = onClauseCondition.left; ColumnReference right = onClauseCondition.right; left.setPointedNode(left.searchPointedNodeInSubQuery((SubQuery) leftItem)); if (left.getPointedNode() == null) { left.setPointedNode(left.searchPointedNodeInSubQuery((SubQuery) rightItem)); } right.setPointedNode(right.searchPointedNodeInSubQuery((SubQuery) leftItem)); if (right.getPointedNode() == null) { right.setPointedNode(right.searchPointedNodeInSubQuery((SubQuery) rightItem)); } } } else { throw new TreeParsingException("This Tree is not an JOINEXPRESSION"); } }