@Override
public JobSpecification buildCreationJobSpec() throws AsterixException, AlgebricksException {
JobSpecification spec = new JobSpecification();
// prepare a LocalResourceMetadata which will be stored in NC's local resource repository
ILocalResourceMetadata localResourceMetadata =
new LSMRTreeLocalResourceMetadata(
secondaryRecDesc.getTypeTraits(),
secondaryComparatorFactories,
primaryComparatorFactories,
valueProviderFactories,
RTreePolicyType.RTREE,
AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES);
ILocalResourceFactoryProvider localResourceFactoryProvider =
new PersistentLocalResourceFactoryProvider(
localResourceMetadata, LocalResource.LSMRTreeResource);
TreeIndexCreateOperatorDescriptor secondaryIndexCreateOp =
new TreeIndexCreateOperatorDescriptor(
spec,
AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER,
secondaryFileSplitProvider,
secondaryRecDesc.getTypeTraits(),
secondaryComparatorFactories,
null,
new LSMRTreeDataflowHelperFactory(
valueProviderFactories,
RTreePolicyType.RTREE,
primaryComparatorFactories,
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES),
localResourceFactoryProvider,
NoOpOperationCallbackFactory.INSTANCE);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(
spec, secondaryIndexCreateOp, secondaryPartitionConstraint);
spec.addRoot(secondaryIndexCreateOp);
spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
return spec;
}
@Override
public JobSpecification buildLoadingJobSpec() throws AsterixException, AlgebricksException {
JobSpecification spec = new JobSpecification();
// Create dummy key provider for feeding the primary index scan.
AbstractOperatorDescriptor keyProviderOp = createDummyKeyProviderOp(spec);
// Create primary index scan op.
BTreeSearchOperatorDescriptor primaryScanOp = createPrimaryIndexScanOp(spec);
// Assign op.
AlgebricksMetaOperatorDescriptor asterixAssignOp =
createAssignOp(spec, primaryScanOp, numNestedSecondaryKeyFields);
// If any of the secondary fields are nullable, then add a select op that filters nulls.
AlgebricksMetaOperatorDescriptor selectOp = null;
if (anySecondaryKeyIsNullable) {
selectOp = createFilterNullsSelectOp(spec, numNestedSecondaryKeyFields);
}
// Create secondary RTree bulk load op.
TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp =
createTreeIndexBulkLoadOp(
spec,
numNestedSecondaryKeyFields,
new LSMRTreeDataflowHelperFactory(
valueProviderFactories,
RTreePolicyType.RTREE,
primaryComparatorFactories,
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES),
BTree.DEFAULT_FILL_FACTOR);
// Connect the operators.
spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryScanOp, 0);
spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, asterixAssignOp, 0);
if (anySecondaryKeyIsNullable) {
spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, selectOp, 0);
spec.connect(new OneToOneConnectorDescriptor(spec), selectOp, 0, secondaryBulkLoadOp, 0);
} else {
spec.connect(
new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, secondaryBulkLoadOp, 0);
}
spec.addRoot(secondaryBulkLoadOp);
spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
return spec;
}