private void setPartitionConstraintsDFS(
OperatorDescriptorId opId,
Map<IConnectorDescriptor, TargetConstraint> tgtConstraints,
IOperatorDescriptor parentOp) {
List<IConnectorDescriptor> opInputs = jobSpec.getOperatorInputMap().get(opId);
AlgebricksPartitionConstraint opConstraint = null;
IOperatorDescriptor opDesc = jobSpec.getOperatorMap().get(opId);
if (opInputs != null) {
for (IConnectorDescriptor conn : opInputs) {
ConnectorDescriptorId cid = conn.getConnectorId();
org.apache.commons.lang3.tuple.Pair<
org.apache.commons.lang3.tuple.Pair<IOperatorDescriptor, Integer>,
org.apache.commons.lang3.tuple.Pair<IOperatorDescriptor, Integer>>
p = jobSpec.getConnectorOperatorMap().get(cid);
IOperatorDescriptor src = p.getLeft().getLeft();
// DFS
setPartitionConstraintsDFS(src.getOperatorId(), tgtConstraints, opDesc);
TargetConstraint constraint = tgtConstraints.get(conn);
if (constraint != null) {
switch (constraint) {
case ONE:
{
opConstraint = new AlgebricksCountPartitionConstraint(1);
break;
}
case SAME_COUNT:
{
opConstraint = partitionConstraintMap.get(src);
break;
}
}
}
}
}
if (partitionConstraintMap.get(opDesc) == null) {
if (opConstraint == null) {
if (parentOp != null) {
AlgebricksPartitionConstraint pc = partitionConstraintMap.get(parentOp);
if (pc != null) {
opConstraint = pc;
} else if (opInputs == null || opInputs.size() == 0) {
opConstraint = new AlgebricksCountPartitionConstraint(1);
}
}
if (opConstraint == null) {
opConstraint = clusterLocations;
}
}
partitionConstraintMap.put(opDesc, opConstraint);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(
jobSpec, opDesc, opConstraint);
}
}
private void setSpecifiedPartitionConstraints() {
for (ILogicalOperator op : pcForMicroOps.keySet()) {
AlgebricksPartitionConstraint pc = pcForMicroOps.get(op);
Integer k = algebraicOpBelongingToMetaAsterixOp.get(op);
AlgebricksMetaOperatorDescriptor amod = metaAsterixOps.get(k);
partitionConstraintMap.put(amod, pc);
}
for (IOperatorDescriptor opDesc : partitionConstraintMap.keySet()) {
AlgebricksPartitionConstraint pc = partitionConstraintMap.get(opDesc);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(jobSpec, opDesc, pc);
}
}
@Override
public JobSpecification buildCreationJobSpec() throws AsterixException, AlgebricksException {
JobSpecification spec = new JobSpecification();
// prepare a LocalResourceMetadata which will be stored in NC's local resource repository
ILocalResourceMetadata localResourceMetadata =
new LSMRTreeLocalResourceMetadata(
secondaryRecDesc.getTypeTraits(),
secondaryComparatorFactories,
primaryComparatorFactories,
valueProviderFactories,
RTreePolicyType.RTREE,
AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES);
ILocalResourceFactoryProvider localResourceFactoryProvider =
new PersistentLocalResourceFactoryProvider(
localResourceMetadata, LocalResource.LSMRTreeResource);
TreeIndexCreateOperatorDescriptor secondaryIndexCreateOp =
new TreeIndexCreateOperatorDescriptor(
spec,
AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER,
secondaryFileSplitProvider,
secondaryRecDesc.getTypeTraits(),
secondaryComparatorFactories,
null,
new LSMRTreeDataflowHelperFactory(
valueProviderFactories,
RTreePolicyType.RTREE,
primaryComparatorFactories,
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES),
localResourceFactoryProvider,
NoOpOperationCallbackFactory.INSTANCE);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(
spec, secondaryIndexCreateOp, secondaryPartitionConstraint);
spec.addRoot(secondaryIndexCreateOp);
spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
return spec;
}
