@Override
public void execute(
QueryMaster.QueryMasterContext context,
QueryContext queryContext,
Query query,
ExecutionBlockId finalExecBlockId,
Path finalOutputDir)
throws Exception {
CatalogService catalog = context.getWorkerContext().getCatalog();
SubQuery lastStage = query.getSubQuery(finalExecBlockId);
TableMeta meta = lastStage.getTableMeta();
TableStats stats = lastStage.getTableStat();
CreateTableNode createTableNode = (CreateTableNode) lastStage.getBlock().getPlan();
TableDesc tableDescTobeCreated =
new TableDesc(
createTableNode.getTableName(),
createTableNode.getTableSchema(),
meta,
finalOutputDir);
if (createTableNode.hasPartition()) {
tableDescTobeCreated.setPartitionMethod(createTableNode.getPartitionMethod());
}
stats.setNumBytes(getTableVolume(query.systemConf, finalOutputDir));
tableDescTobeCreated.setStats(stats);
query.setResultDesc(tableDescTobeCreated);
catalog.addTable(tableDescTobeCreated);
}
@Override
public void transition(Query query, QueryEvent event) {
synchronized (query.subqueries) {
for (SubQuery subquery : query.subqueries.values()) {
query.eventHandler.handle(new SubQueryEvent(subquery.getId(), SubQueryEventType.SQ_KILL));
}
}
}
@Override
public boolean isEligible(
QueryContext queryContext,
Query query,
ExecutionBlockId finalExecBlockId,
Path finalOutputDir) {
SubQuery lastStage = query.getSubQuery(finalExecBlockId);
return lastStage.getBlock().getPlan().getType() == NodeType.INSERT;
}
@Override
public void transition(Query query, QueryEvent queryEvent) {
query.setStartTime();
SubQuery subQuery =
new SubQuery(
query.context,
query.getPlan(),
query.getExecutionBlockCursor().nextBlock(),
query.sm);
subQuery.setPriority(query.priority--);
query.addSubQuery(subQuery);
subQuery.handle(new SubQueryEvent(subQuery.getId(), SubQueryEventType.SQ_INIT));
LOG.debug("Schedule unit plan: \n" + subQuery.getBlock().getPlan());
}
public void xqderiveMemberAddressList(
String function,
SubQuery<MemberAddressCB> subQuery,
String operand,
Object value,
DerivedReferrerOption option) {
assertObjectNotNull("subQuery<MemberAddressCB>", subQuery);
MemberAddressCB cb = new MemberAddressCB();
cb.xsetupForDerivedReferrer(this);
subQuery.query(cb);
String subQueryPropertyName =
keepRegionId_QueryDerivedReferrer_MemberAddressList(cb.query()); // for saving query-value.
String parameterPropertyName =
keepRegionId_QueryDerivedReferrer_MemberAddressListParameter(value);
registerQueryDerivedReferrer(
function,
cb.query(),
"REGION_ID",
"REGION_ID",
subQueryPropertyName,
operand,
value,
parameterPropertyName,
option);
}
public void xqderiveMemberServiceList(
String function,
SubQuery<MemberServiceCB> subQuery,
String operand,
Object value,
DerivedReferrerOption option) {
assertObjectNotNull("subQuery<MemberServiceCB>", subQuery);
MemberServiceCB cb = new MemberServiceCB();
cb.xsetupForDerivedReferrer(this);
subQuery.query(cb);
String subQueryPropertyName =
keepServiceRankCode_QueryDerivedReferrer_MemberServiceList(
cb.query()); // for saving query-value.
String parameterPropertyName =
keepServiceRankCode_QueryDerivedReferrer_MemberServiceListParameter(value);
registerQueryDerivedReferrer(
function,
cb.query(),
"SERVICE_RANK_CODE",
"SERVICE_RANK_CODE",
subQueryPropertyName,
operand,
value,
parameterPropertyName,
option);
}
/**
* Prepare for MyselfExists (correlated sub-query).
*
* @param subCBLambda The implementation of sub-query. (NotNull)
*/
public void myselfExists(SubQuery<TweetCB> subCBLambda) {
assertObjectNotNull("subCBLambda", subCBLambda);
TweetCB cb = new TweetCB();
cb.xsetupForMyselfExists(this);
lockCall(() -> subCBLambda.query(cb));
String pp = keepMyselfExists(cb.query());
registerMyselfExists(cb.query(), pp);
}
/**
* Myself InScope (SubQuery). {mainly to avoid CLOB and Union problem or no master table}
*
* @param subQuery The implementation of sub query. (NotNull)
*/
public void myselfInScope(SubQuery<ServiceRankCB> subQuery) {
assertObjectNotNull("subQuery<ServiceRankCB>", subQuery);
ServiceRankCB cb = new ServiceRankCB();
cb.xsetupForMyselfInScope(this);
subQuery.query(cb);
String subQueryPropertyName = keepMyselfInScope(cb.query()); // for saving query-value.
registerMyselfInScope(cb.query(), subQueryPropertyName);
}
protected void xscalarCondition(String function, SubQuery<RegionCB> subQuery, String operand) {
assertObjectNotNull("subQuery<RegionCB>", subQuery);
RegionCB cb = new RegionCB();
cb.xsetupForScalarCondition(this);
subQuery.query(cb);
String subQueryPropertyName = keepScalarCondition(cb.query()); // for saving query-value.
registerScalarCondition(function, cb.query(), subQueryPropertyName, operand);
}
/**
* Set up ExistsReferrer (co-related sub-query). <br>
* {exists (select REGION_ID from MEMBER_ADDRESS where ...)} <br>
* MEMBER_ADDRESS as 'memberAddressAsOne'.
*
* <pre>
* cb.query().<span style="color: #FD4747">existsMemberAddressList</span>(new SubQuery<MemberAddressCB>() {
* public void query(RegionCB subCB) {
* subCB.query().setXxx...
* }
* });
* </pre>
*
* @param subQuery The sub-query of MemberAddressList for 'exists'. (NotNull)
*/
public void existsMemberAddressList(SubQuery<MemberAddressCB> subQuery) {
assertObjectNotNull("subQuery<MemberAddressCB>", subQuery);
MemberAddressCB cb = new MemberAddressCB();
cb.xsetupForExistsReferrer(this);
subQuery.query(cb);
String subQueryPropertyName =
keepRegionId_ExistsReferrer_MemberAddressList(cb.query()); // for saving query-value.
registerExistsReferrer(cb.query(), "REGION_ID", "REGION_ID", subQueryPropertyName);
}
// ===================================================================================
// MyselfDerived
// =============
public void xsmyselfDerive(String fn, SubQuery<TweetCB> sq, String al, DerivedReferrerOption op) {
assertObjectNotNull("subQuery", sq);
TweetCB cb = new TweetCB();
cb.xsetupForDerivedReferrer(this);
lockCall(() -> sq.query(cb));
String pp = keepSpecifyMyselfDerived(cb.query());
String pk = "TWEET_ID";
registerSpecifyMyselfDerived(fn, cb.query(), pk, pk, pp, "myselfDerived", al, op);
}
/**
* Set up NotInScopeRelation (sub-query). <br>
* {not in (select REGION_ID from MEMBER_ADDRESS where ...)} <br>
* MEMBER_ADDRESS as 'memberAddressAsOne'.
*
* @param subQuery The sub-query of MemberAddressList for 'not in-scope'. (NotNull)
*/
public void notInScopeMemberAddressList(SubQuery<MemberAddressCB> subQuery) {
assertObjectNotNull("subQuery<MemberAddressCB>", subQuery);
MemberAddressCB cb = new MemberAddressCB();
cb.xsetupForInScopeRelation(this);
subQuery.query(cb);
String subQueryPropertyName =
keepRegionId_NotInScopeRelation_MemberAddressList(cb.query()); // for saving query-value.
registerNotInScopeRelation(cb.query(), "REGION_ID", "REGION_ID", subQueryPropertyName);
}
/**
* A Constructor for handling MULTIJOINEXPRESSIONS
*
* @param leftItem - the JOIN's left ITEM, which is a subquery
* @param t - the Tree where we get the join, right, onclause
* @param treeBuilder
* @param selectStatement
* @throws TreeParsingException
* @throws ColumnCallException
*/
public JoinExpression(
SubQuery leftItem, Tree t, TreeBuilder treeBuilder, SelectStatement selectStatement)
throws TreeParsingException, ColumnCallException {
this.builder = treeBuilder;
this.selectStatement = selectStatement;
this.leftItem = leftItem;
leftItem.setisPartOfJoin();
this.build(t, this.builder);
}
@SuppressWarnings("unchecked")
protected <CB extends ConditionBean> void xscalarCondition(
String fn, SubQuery<CB> sq, String rd, HpSLCCustomized<CB> cs, ScalarConditionOption op) {
assertObjectNotNull("subQuery", sq);
TweetCB cb = xcreateScalarConditionCB();
sq.query((CB) cb);
String pp = keepScalarCondition(cb.query()); // for saving query-value
cs.setPartitionByCBean((CB) xcreateScalarConditionPartitionByCB()); // for using partition-by
registerScalarCondition(fn, cb.query(), pp, rd, cs, op);
}
/**
* Set up ExistsReferrer (co-related sub-query). <br>
* {exists (select SERVICE_RANK_CODE from MEMBER_SERVICE where ...)} <br>
* MEMBER_SERVICE as 'memberServiceAsOne'.
*
* <pre>
* cb.query().<span style="color: #FD4747">existsMemberServiceList</span>(new SubQuery<MemberServiceCB>() {
* public void query(ServiceRankCB subCB) {
* subCB.query().setXxx...
* }
* });
* </pre>
*
* @param subQuery The sub-query of MemberServiceList for 'exists'. (NotNull)
*/
public void existsMemberServiceList(SubQuery<MemberServiceCB> subQuery) {
assertObjectNotNull("subQuery<MemberServiceCB>", subQuery);
MemberServiceCB cb = new MemberServiceCB();
cb.xsetupForExistsReferrer(this);
subQuery.query(cb);
String subQueryPropertyName =
keepServiceRankCode_ExistsReferrer_MemberServiceList(cb.query()); // for saving query-value.
registerExistsReferrer(
cb.query(), "SERVICE_RANK_CODE", "SERVICE_RANK_CODE", subQueryPropertyName);
}
@Override
public void execute(
QueryMaster.QueryMasterContext context,
QueryContext queryContext,
Query query,
ExecutionBlockId finalExecBlockId,
Path finalOutputDir)
throws Exception {
SubQuery lastStage = query.getSubQuery(finalExecBlockId);
TableMeta meta = lastStage.getTableMeta();
TableStats stats = lastStage.getTableStat();
TableDesc resultTableDesc =
new TableDesc(query.getId().toString(), lastStage.getSchema(), meta, finalOutputDir);
stats.setNumBytes(getTableVolume(query.systemConf, finalOutputDir));
resultTableDesc.setStats(stats);
query.setResultDesc(resultTableDesc);
}
/**
* Set up NotInScopeRelation (sub-query). <br>
* {not in (select SERVICE_RANK_CODE from MEMBER_SERVICE where ...)} <br>
* MEMBER_SERVICE as 'memberServiceAsOne'.
*
* @param subQuery The sub-query of MemberServiceList for 'not in-scope'. (NotNull)
*/
public void notInScopeMemberServiceList(SubQuery<MemberServiceCB> subQuery) {
assertObjectNotNull("subQuery<MemberServiceCB>", subQuery);
MemberServiceCB cb = new MemberServiceCB();
cb.xsetupForInScopeRelation(this);
subQuery.query(cb);
String subQueryPropertyName =
keepServiceRankCode_NotInScopeRelation_MemberServiceList(
cb.query()); // for saving query-value.
registerNotInScopeRelation(
cb.query(), "SERVICE_RANK_CODE", "SERVICE_RANK_CODE", subQueryPropertyName);
}
@SuppressWarnings("unchecked")
protected <CB extends ConditionBean> void xqderiveMyselfDerived(
String fn, SubQuery<CB> sq, String rd, Object vl, DerivedReferrerOption op) {
assertObjectNotNull("subQuery", sq);
TweetCB cb = new TweetCB();
cb.xsetupForDerivedReferrer(this);
sq.query((CB) cb);
String pk = "TWEET_ID";
String sqpp = keepQueryMyselfDerived(cb.query()); // for saving query-value.
String prpp = keepQueryMyselfDerivedParameter(vl);
registerQueryMyselfDerived(fn, cb.query(), pk, pk, sqpp, "myselfDerived", rd, vl, prpp, op);
}
public float getProgress() {
QueryState state = getStateMachine().getCurrentState();
if (state == QueryState.QUERY_SUCCEEDED) {
return 1.0f;
} else {
int idx = 0;
List<SubQuery> tempSubQueries = new ArrayList<SubQuery>();
synchronized (subqueries) {
tempSubQueries.addAll(subqueries.values());
}
float[] subProgresses = new float[tempSubQueries.size()];
boolean finished = true;
for (SubQuery subquery : tempSubQueries) {
if (subquery.getState() != SubQueryState.NEW) {
subProgresses[idx] = subquery.getProgress();
if (finished && subquery.getState() != SubQueryState.SUCCEEDED) {
finished = false;
}
} else {
subProgresses[idx] = 0.0f;
}
idx++;
}
if (finished) {
return 1.0f;
}
float totalProgress = 0;
float proportion =
1.0f / (float) (getExecutionBlockCursor().size() - 1); // minus one is due to
for (int i = 0; i < subProgresses.length; i++) {
totalProgress += subProgresses[i] * proportion;
}
return totalProgress;
}
}
/**
* A Constructor to build JoinExpressions from two Subqueries
*
* @param builder - the TreeBuilder for the helper functions
* @param leftItem - Subquery for the left side of the join
* @param rightItem - Subquery for the right side of the join
* @param onClause - the onClause to link the 2 subqueries
* @param selectStatement - the selectStatement which contains the join
*/
public JoinExpression(
TreeBuilder builder,
Node leftItem,
Node rightItem,
OnClause onClause,
SelectStatement selectStatement) {
this.selectStatement = selectStatement;
this.builder = builder;
this.leftItem = leftItem;
this.rightItem = rightItem;
if (leftItem.tokenType == JdbcGrammarParser.SUBQUERY) {
SubQuery subQuery = SubQuery.class.cast(leftItem);
subQuery.setisPartOfJoin();
}
if (rightItem.tokenType == JdbcGrammarParser.SUBQUERY) {
SubQuery subQuery = SubQuery.class.cast(rightItem);
subQuery.setisPartOfJoin();
}
this.onClause = onClause;
this.tokenType = JdbcGrammarParser.JOINEXPRESSION;
this.tokenName = JdbcGrammarParser.tokenNames[JdbcGrammarParser.JOINEXPRESSION];
}
@Override
public void execute(
QueryMaster.QueryMasterContext context,
QueryContext queryContext,
Query query,
ExecutionBlockId finalExecBlockId,
Path finalOutputDir)
throws Exception {
CatalogService catalog = context.getWorkerContext().getCatalog();
SubQuery lastStage = query.getSubQuery(finalExecBlockId);
TableMeta meta = lastStage.getTableMeta();
TableStats stats = lastStage.getTableStat();
InsertNode insertNode = (InsertNode) lastStage.getBlock().getPlan();
TableDesc finalTable;
if (insertNode.hasTargetTable()) {
String tableName = insertNode.getTableName();
finalTable = catalog.getTableDesc(tableName);
} else {
String tableName = query.getId().toString();
finalTable = new TableDesc(tableName, lastStage.getSchema(), meta, finalOutputDir);
}
long volume = getTableVolume(query.systemConf, finalOutputDir);
stats.setNumBytes(volume);
finalTable.setStats(stats);
if (insertNode.hasTargetTable()) {
catalog.deleteTable(insertNode.getTableName());
catalog.addTable(finalTable);
}
query.setResultDesc(finalTable);
}
public void xsderiveMemberAddressList(
String function,
SubQuery<MemberAddressCB> subQuery,
String aliasName,
DerivedReferrerOption option) {
assertObjectNotNull("subQuery<MemberAddressCB>", subQuery);
MemberAddressCB cb = new MemberAddressCB();
cb.xsetupForDerivedReferrer(this);
subQuery.query(cb);
String subQueryPropertyName =
keepRegionId_SpecifyDerivedReferrer_MemberAddressList(
cb.query()); // for saving query-value.
registerSpecifyDerivedReferrer(
function, cb.query(), "REGION_ID", "REGION_ID", subQueryPropertyName, aliasName, option);
}
private void executeNextBlock(Query query) {
ExecutionBlockCursor cursor = query.getExecutionBlockCursor();
ExecutionBlock nextBlock = cursor.nextBlock();
SubQuery nextSubQuery = new SubQuery(query.context, query.getPlan(), nextBlock, query.sm);
nextSubQuery.setPriority(query.priority--);
query.addSubQuery(nextSubQuery);
nextSubQuery.handle(new SubQueryEvent(nextSubQuery.getId(), SubQueryEventType.SQ_INIT));
LOG.info("Scheduling SubQuery:" + nextSubQuery.getId());
if (LOG.isDebugEnabled()) {
LOG.debug("Scheduling SubQuery's Priority: " + nextSubQuery.getPriority());
LOG.debug("Scheduling SubQuery's Plan: \n" + nextSubQuery.getBlock().getPlan());
}
}
public void xsderiveMemberServiceList(
String function,
SubQuery<MemberServiceCB> subQuery,
String aliasName,
DerivedReferrerOption option) {
assertObjectNotNull("subQuery<MemberServiceCB>", subQuery);
MemberServiceCB cb = new MemberServiceCB();
cb.xsetupForDerivedReferrer(this);
subQuery.query(cb);
String subQueryPropertyName =
keepServiceRankCode_SpecifyDerivedReferrer_MemberServiceList(
cb.query()); // for saving query-value.
registerSpecifyDerivedReferrer(
function,
cb.query(),
"SERVICE_RANK_CODE",
"SERVICE_RANK_CODE",
subQueryPropertyName,
aliasName,
option);
}
public void addSubQuery(SubQuery subquery) {
subqueries.put(subquery.getId(), subquery);
}
public void build(Tree t, TreeBuilder builder) throws TreeParsingException, ColumnCallException {
if (t.getType() == JdbcGrammarParser.JOINEXPRESSION
|| t.getType() == JdbcGrammarParser.MULTIJOINEXPRESSION) {
this.tokenName = JdbcGrammarParser.tokenNames[t.getType()];
this.tokenType = t.getType();
this.logger.debug("BUILDING " + this.tokenName);
for (int i = 0; i < t.getChildCount(); i++) {
Tree child = t.getChild(i);
switch (child.getType()) {
case JdbcGrammarParser.LEFTEXPR:
switch (child.getChild(0).getType()) {
case JdbcGrammarParser.SUBQUERY:
SubQuery subQuery = new SubQuery(child.getChild(0), builder);
subQuery.setisPartOfJoin();
this.leftItem = subQuery;
break;
case JdbcGrammarParser.SOURCETABLE:
Resolver resolve = new Resolver(builder);
// whats the sourcetable?
SourceTable tablenode = new SourceTable(child.getChild(0), builder);
String alias = tablenode.getAlias();
// getting the columns for it
List<ColumnCall> columnlist = resolve.parseSrcTableForJokers(tablenode);
// making a new Subquery with those columns +
// fromexpression
this.logger.debug("got columns");
tablenode.alias = null;
// making a fromexpression with that sourcetable
FromExpression fromexpression = new FromExpression(tablenode, builder);
Expression expression = new Expression(columnlist, builder);
SelectStatement mySelectStatement =
new SelectStatement(expression, fromexpression, builder);
expression.setSelectStatement(mySelectStatement);
SubQuery mySubQuery = null;
if (alias != null) {
// putting the selectstatement into a subquery
mySubQuery = new SubQuery(alias, builder, mySelectStatement);
} else {
String newalias = "";
if (tablenode.getDataset() != null) {
newalias += tablenode.getDataset() + ".";
}
newalias += tablenode.getName();
mySubQuery = new SubQuery(newalias, builder, mySelectStatement);
}
// adding the subquery to the fromexpression
mySubQuery.setisPartOfJoin();
this.leftItem = mySubQuery;
break;
default:
break;
}
break;
case JdbcGrammarParser.RIGHTEXPR:
switch (child.getChild(0).getType()) {
case JdbcGrammarParser.SUBQUERY:
SubQuery subQuery = new SubQuery(child.getChild(0), builder);
subQuery.setisPartOfJoin();
this.rightItem = subQuery;
break;
case JdbcGrammarParser.SOURCETABLE:
Resolver resolve = new Resolver(builder);
// whats the sourcetable?
SourceTable tablenode = new SourceTable(child.getChild(0), builder);
String alias = tablenode.getAlias();
// getting the columns for it
List<ColumnCall> columnlist = resolve.parseSrcTableForJokers(tablenode);
// making a new Subquery with those columns +
// fromexpression
this.logger.debug("got columns");
tablenode.alias = null;
// making a fromexpression with that sourcetable
FromExpression fromexpression = new FromExpression(tablenode, builder);
Expression expression = new Expression(columnlist, builder);
SelectStatement mySelectStatement =
new SelectStatement(expression, fromexpression, builder);
expression.setSelectStatement(mySelectStatement);
SubQuery mySubQuery = null;
if (alias != null) {
// putting the selectstatement into a subquery
mySubQuery = new SubQuery(alias, builder, mySelectStatement);
} else {
String newalias = "";
if (tablenode.getDataset() != null) {
newalias += tablenode.getDataset() + ".";
}
newalias += tablenode.getName();
mySubQuery = new SubQuery(newalias, builder, mySelectStatement);
}
// adding the subquery to the fromexpression
mySubQuery.setisPartOfJoin();
this.rightItem = mySubQuery;
break;
default:
break;
}
break;
case JdbcGrammarParser.JOINTYPE:
switch (child.getChild(0).getType()) {
case JdbcGrammarParser.LEFT_KEYWORD:
this.type = JoinType.LEFT;
break;
case JdbcGrammarParser.RIGHT_KEYWORD:
this.type = JoinType.RIGHT;
break;
case JdbcGrammarParser.FULl_KEYWORD:
this.type = JoinType.FULL;
break;
case JdbcGrammarParser.INNERKEYWORD:
this.type = JoinType.INNER;
break;
default:
break;
}
if (child.getChild(1).getType() == JdbcGrammarParser.EACH) {
each = true;
}
break;
default:
break;
}
}
for (int i = 0; i < t.getChildCount(); i++) {
Tree child = t.getChild(i);
switch (child.getType()) {
case JdbcGrammarParser.ONCLAUSE:
this.onClause = new OnClause(child, builder, this.selectStatement);
break;
default:
break;
}
}
for (Node condition : onClause.children) {
OnClauseCondition onClauseCondition = OnClauseCondition.class.cast(condition);
ColumnReference left = onClauseCondition.left;
ColumnReference right = onClauseCondition.right;
left.setPointedNode(left.searchPointedNodeInSubQuery((SubQuery) leftItem));
if (left.getPointedNode() == null) {
left.setPointedNode(left.searchPointedNodeInSubQuery((SubQuery) rightItem));
}
right.setPointedNode(right.searchPointedNodeInSubQuery((SubQuery) leftItem));
if (right.getPointedNode() == null) {
right.setPointedNode(right.searchPointedNodeInSubQuery((SubQuery) rightItem));
}
}
} else {
throw new TreeParsingException("This Tree is not an JOINEXPRESSION");
}
}
