private void setPartitionConstraintsDFS( OperatorDescriptorId opId, Map<IConnectorDescriptor, TargetConstraint> tgtConstraints, IOperatorDescriptor parentOp) { List<IConnectorDescriptor> opInputs = jobSpec.getOperatorInputMap().get(opId); AlgebricksPartitionConstraint opConstraint = null; IOperatorDescriptor opDesc = jobSpec.getOperatorMap().get(opId); if (opInputs != null) { for (IConnectorDescriptor conn : opInputs) { ConnectorDescriptorId cid = conn.getConnectorId(); org.apache.commons.lang3.tuple.Pair< org.apache.commons.lang3.tuple.Pair<IOperatorDescriptor, Integer>, org.apache.commons.lang3.tuple.Pair<IOperatorDescriptor, Integer>> p = jobSpec.getConnectorOperatorMap().get(cid); IOperatorDescriptor src = p.getLeft().getLeft(); // DFS setPartitionConstraintsDFS(src.getOperatorId(), tgtConstraints, opDesc); TargetConstraint constraint = tgtConstraints.get(conn); if (constraint != null) { switch (constraint) { case ONE: { opConstraint = new AlgebricksCountPartitionConstraint(1); break; } case SAME_COUNT: { opConstraint = partitionConstraintMap.get(src); break; } } } } } if (partitionConstraintMap.get(opDesc) == null) { if (opConstraint == null) { if (parentOp != null) { AlgebricksPartitionConstraint pc = partitionConstraintMap.get(parentOp); if (pc != null) { opConstraint = pc; } else if (opInputs == null || opInputs.size() == 0) { opConstraint = new AlgebricksCountPartitionConstraint(1); } } if (opConstraint == null) { opConstraint = clusterLocations; } } partitionConstraintMap.put(opDesc, opConstraint); AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec( jobSpec, opDesc, opConstraint); } }
private void setSpecifiedPartitionConstraints() { for (ILogicalOperator op : pcForMicroOps.keySet()) { AlgebricksPartitionConstraint pc = pcForMicroOps.get(op); Integer k = algebraicOpBelongingToMetaAsterixOp.get(op); AlgebricksMetaOperatorDescriptor amod = metaAsterixOps.get(k); partitionConstraintMap.put(amod, pc); } for (IOperatorDescriptor opDesc : partitionConstraintMap.keySet()) { AlgebricksPartitionConstraint pc = partitionConstraintMap.get(opDesc); AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(jobSpec, opDesc, pc); } }
@Override public JobSpecification buildCreationJobSpec() throws AsterixException, AlgebricksException { JobSpecification spec = new JobSpecification(); // prepare a LocalResourceMetadata which will be stored in NC's local resource repository ILocalResourceMetadata localResourceMetadata = new LSMRTreeLocalResourceMetadata( secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, primaryComparatorFactories, valueProviderFactories, RTreePolicyType.RTREE, AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length), GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES); ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider( localResourceMetadata, LocalResource.LSMRTreeResource); TreeIndexCreateOperatorDescriptor secondaryIndexCreateOp = new TreeIndexCreateOperatorDescriptor( spec, AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER, AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER, secondaryFileSplitProvider, secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, null, new LSMRTreeDataflowHelperFactory( valueProviderFactories, RTreePolicyType.RTREE, primaryComparatorFactories, AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER, AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER, AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER, AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER, AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length), GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES), localResourceFactoryProvider, NoOpOperationCallbackFactory.INSTANCE); AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec( spec, secondaryIndexCreateOp, secondaryPartitionConstraint); spec.addRoot(secondaryIndexCreateOp); spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy()); return spec; }