/**
* Sets up projection that will transfer all of the columns in batch, and also populate the
* partition column based on which partition a record falls into in the partition table
*
* @param batch
* @throws SchemaChangeException
*/
protected void setupNewSchema(VectorAccessible batch) throws SchemaChangeException {
container.clear();
final ErrorCollector collector = new ErrorCollectorImpl();
final List<TransferPair> transfers = Lists.newArrayList();
final ClassGenerator<OrderedPartitionProjector> cg =
CodeGenerator.getRoot(
OrderedPartitionProjector.TEMPLATE_DEFINITION, context.getFunctionRegistry());
for (VectorWrapper<?> vw : batch) {
TransferPair tp = vw.getValueVector().getTransferPair();
transfers.add(tp);
container.add(tp.getTo());
}
cg.setMappingSet(mainMapping);
int count = 0;
for (Ordering od : popConfig.getOrderings()) {
final LogicalExpression expr =
ExpressionTreeMaterializer.materialize(
od.getExpr(), batch, collector, context.getFunctionRegistry());
if (collector.hasErrors())
throw new SchemaChangeException(
"Failure while materializing expression. " + collector.toErrorString());
cg.setMappingSet(incomingMapping);
ClassGenerator.HoldingContainer left = cg.addExpr(expr, false);
cg.setMappingSet(partitionMapping);
ClassGenerator.HoldingContainer right =
cg.addExpr(
new ValueVectorReadExpression(new TypedFieldId(expr.getMajorType(), count++)), false);
cg.setMappingSet(mainMapping);
LogicalExpression fh =
FunctionGenerationHelper.getComparator(left, right, context.getFunctionRegistry());
ClassGenerator.HoldingContainer out = cg.addExpr(fh, false);
JConditional jc = cg.getEvalBlock()._if(out.getValue().ne(JExpr.lit(0)));
if (od.getDirection() == Direction.ASCENDING) {
jc._then()._return(out.getValue());
} else {
jc._then()._return(out.getValue().minus());
}
}
cg.getEvalBlock()._return(JExpr.lit(0));
container.add(this.partitionKeyVector);
container.buildSchema(batch.getSchema().getSelectionVectorMode());
try {
this.projector = context.getImplementationClass(cg);
projector.setup(
context, batch, this, transfers, partitionVectors, partitions, popConfig.getRef());
} catch (ClassTransformationException | IOException e) {
throw new SchemaChangeException("Failure while attempting to load generated class", e);
}
}
private MSorter createNewMSorter(
FragmentContext context,
List<Ordering> orderings,
VectorAccessible batch,
MappingSet mainMapping,
MappingSet leftMapping,
MappingSet rightMapping)
throws ClassTransformationException, IOException, SchemaChangeException {
CodeGenerator<MSorter> cg =
CodeGenerator.get(
MSorter.TEMPLATE_DEFINITION, context.getFunctionRegistry(), context.getOptions());
ClassGenerator<MSorter> g = cg.getRoot();
g.setMappingSet(mainMapping);
for (Ordering od : orderings) {
// first, we rewrite the evaluation stack for each side of the comparison.
ErrorCollector collector = new ErrorCollectorImpl();
final LogicalExpression expr =
ExpressionTreeMaterializer.materialize(
od.getExpr(), batch, collector, context.getFunctionRegistry());
if (collector.hasErrors()) {
throw new SchemaChangeException(
"Failure while materializing expression. " + collector.toErrorString());
}
g.setMappingSet(leftMapping);
HoldingContainer left = g.addExpr(expr, ClassGenerator.BlkCreateMode.FALSE);
g.setMappingSet(rightMapping);
HoldingContainer right = g.addExpr(expr, ClassGenerator.BlkCreateMode.FALSE);
g.setMappingSet(mainMapping);
// next we wrap the two comparison sides and add the expression block for the comparison.
LogicalExpression fh =
FunctionGenerationHelper.getOrderingComparator(
od.nullsSortHigh(), left, right, context.getFunctionRegistry());
HoldingContainer out = g.addExpr(fh, ClassGenerator.BlkCreateMode.FALSE);
JConditional jc = g.getEvalBlock()._if(out.getValue().ne(JExpr.lit(0)));
if (od.getDirection() == Direction.ASCENDING) {
jc._then()._return(out.getValue());
} else {
jc._then()._return(out.getValue().minus());
}
g.rotateBlock();
}
g.rotateBlock();
g.getEvalBlock()._return(JExpr.lit(0));
return context.getImplementationClass(cg);
}
/**
* Creates a copier that does a project for every Nth record from a VectorContainer incoming into
* VectorContainer outgoing. Each Ordering in orderings generates a column, and evaluation of the
* expression associated with each Ordering determines the value of each column. These records
* will later be sorted based on the values in each column, in the same order as the orderings.
*
* @param sv4
* @param incoming
* @param outgoing
* @param orderings
* @return
* @throws SchemaChangeException
*/
private SampleCopier getCopier(
SelectionVector4 sv4,
VectorContainer incoming,
VectorContainer outgoing,
List<Ordering> orderings,
List<ValueVector> localAllocationVectors)
throws SchemaChangeException {
final ErrorCollector collector = new ErrorCollectorImpl();
final ClassGenerator<SampleCopier> cg =
CodeGenerator.getRoot(SampleCopier.TEMPLATE_DEFINITION, context.getFunctionRegistry());
int i = 0;
for (Ordering od : orderings) {
final LogicalExpression expr =
ExpressionTreeMaterializer.materialize(
od.getExpr(), incoming, collector, context.getFunctionRegistry());
SchemaPath schemaPath = SchemaPath.getSimplePath("f" + i++);
TypeProtos.MajorType.Builder builder =
TypeProtos.MajorType.newBuilder()
.mergeFrom(expr.getMajorType())
.clearMode()
.setMode(TypeProtos.DataMode.REQUIRED);
TypeProtos.MajorType newType = builder.build();
MaterializedField outputField = MaterializedField.create(schemaPath, newType);
if (collector.hasErrors()) {
throw new SchemaChangeException(
String.format(
"Failure while trying to materialize incoming schema. Errors:\n %s.",
collector.toErrorString()));
}
ValueVector vector = TypeHelper.getNewVector(outputField, oContext.getAllocator());
localAllocationVectors.add(vector);
TypedFieldId fid = outgoing.add(vector);
ValueVectorWriteExpression write = new ValueVectorWriteExpression(fid, expr, true);
HoldingContainer hc = cg.addExpr(write);
cg.getEvalBlock()._if(hc.getValue().eq(JExpr.lit(0)))._then()._return(JExpr.FALSE);
}
cg.rotateBlock();
cg.getEvalBlock()._return(JExpr.TRUE);
outgoing.buildSchema(BatchSchema.SelectionVectorMode.NONE);
try {
SampleCopier sampleCopier = context.getImplementationClass(cg);
sampleCopier.setupCopier(context, sv4, incoming, outgoing);
return sampleCopier;
} catch (ClassTransformationException | IOException e) {
throw new SchemaChangeException(e);
}
}
private void setupIsSame(ClassGenerator<StreamingAggregator> cg, LogicalExpression[] keyExprs) {
cg.setMappingSet(IS_SAME_I1);
for (final LogicalExpression expr : keyExprs) {
// first, we rewrite the evaluation stack for each side of the comparison.
cg.setMappingSet(IS_SAME_I1);
final HoldingContainer first = cg.addExpr(expr, false);
cg.setMappingSet(IS_SAME_I2);
final HoldingContainer second = cg.addExpr(expr, false);
final LogicalExpression fh =
FunctionGenerationHelper.getOrderingComparatorNullsHigh(
first, second, context.getFunctionRegistry());
final HoldingContainer out = cg.addExpr(fh, false);
cg.getEvalBlock()._if(out.getValue().ne(JExpr.lit(0)))._then()._return(JExpr.FALSE);
}
cg.getEvalBlock()._return(JExpr.TRUE);
}
protected HoldingContainer generateEvalBody(
ClassGenerator<?> g, HoldingContainer[] inputVariables, String body, JVar[] workspaceJVars) {
// g.getBlock().directStatement(String.format("//---- start of eval portion of %s function.
// ----//", functionName));
JBlock sub = new JBlock(true, true);
JBlock topSub = sub;
HoldingContainer out = null;
MajorType returnValueType = returnValue.type;
// add outside null handling if it is defined.
if (nullHandling == NullHandling.NULL_IF_NULL) {
JExpression e = null;
for (HoldingContainer v : inputVariables) {
if (v.isOptional()) {
if (e == null) {
e = v.getIsSet();
} else {
e = e.mul(v.getIsSet());
}
}
}
if (e != null) {
// if at least one expression must be checked, set up the conditional.
returnValueType = returnValue.type.toBuilder().setMode(DataMode.OPTIONAL).build();
out = g.declare(returnValueType);
e = e.eq(JExpr.lit(0));
JConditional jc = sub._if(e);
jc._then().assign(out.getIsSet(), JExpr.lit(0));
sub = jc._else();
}
}
if (out == null) out = g.declare(returnValueType);
// add the subblock after the out declaration.
g.getEvalBlock().add(topSub);
JVar internalOutput =
sub.decl(
JMod.FINAL,
g.getHolderType(returnValueType),
returnValue.name,
JExpr._new(g.getHolderType(returnValueType)));
addProtectedBlock(g, sub, body, inputVariables, workspaceJVars, false);
if (sub != topSub)
sub.assign(internalOutput.ref("isSet"), JExpr.lit(1)); // Assign null if NULL_IF_NULL mode
sub.assign(out.getHolder(), internalOutput);
if (sub != topSub)
sub.assign(internalOutput.ref("isSet"), JExpr.lit(1)); // Assign null if NULL_IF_NULL mode
return out;
}
private void generateComparisons(ClassGenerator<?> g, VectorAccessible batch)
throws SchemaChangeException {
g.setMappingSet(MAIN_MAPPING);
for (Ordering od : popConfig.getOrderings()) {
// first, we rewrite the evaluation stack for each side of the comparison.
ErrorCollector collector = new ErrorCollectorImpl();
final LogicalExpression expr =
ExpressionTreeMaterializer.materialize(
od.getExpr(), batch, collector, context.getFunctionRegistry());
if (collector.hasErrors()) {
throw new SchemaChangeException(
"Failure while materializing expression. " + collector.toErrorString());
}
g.setMappingSet(LEFT_MAPPING);
HoldingContainer left = g.addExpr(expr, ClassGenerator.BlkCreateMode.FALSE);
g.setMappingSet(RIGHT_MAPPING);
HoldingContainer right = g.addExpr(expr, ClassGenerator.BlkCreateMode.FALSE);
g.setMappingSet(MAIN_MAPPING);
// next we wrap the two comparison sides and add the expression block for the comparison.
LogicalExpression fh =
FunctionGenerationHelper.getOrderingComparator(
od.nullsSortHigh(), left, right, context.getFunctionRegistry());
HoldingContainer out = g.addExpr(fh, ClassGenerator.BlkCreateMode.FALSE);
JConditional jc = g.getEvalBlock()._if(out.getValue().ne(JExpr.lit(0)));
if (od.getDirection() == Direction.ASCENDING) {
jc._then()._return(out.getValue());
} else {
jc._then()._return(out.getValue().minus());
}
g.rotateBlock();
}
g.rotateBlock();
g.getEvalBlock()._return(JExpr.lit(0));
}
