@Override public JobSpecification buildCreationJobSpec() throws AsterixException, AlgebricksException { JobSpecification spec = new JobSpecification(); // prepare a LocalResourceMetadata which will be stored in NC's local resource repository ILocalResourceMetadata localResourceMetadata = new LSMRTreeLocalResourceMetadata( secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, primaryComparatorFactories, valueProviderFactories, RTreePolicyType.RTREE, AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length), GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES); ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider( localResourceMetadata, LocalResource.LSMRTreeResource); TreeIndexCreateOperatorDescriptor secondaryIndexCreateOp = new TreeIndexCreateOperatorDescriptor( spec, AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER, AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER, secondaryFileSplitProvider, secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, null, new LSMRTreeDataflowHelperFactory( valueProviderFactories, RTreePolicyType.RTREE, primaryComparatorFactories, AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER, AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER, AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER, AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER, AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length), GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES), localResourceFactoryProvider, NoOpOperationCallbackFactory.INSTANCE); AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec( spec, secondaryIndexCreateOp, secondaryPartitionConstraint); spec.addRoot(secondaryIndexCreateOp); spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy()); return spec; }
@Override public JobSpecification buildLoadingJobSpec() throws AsterixException, AlgebricksException { JobSpecification spec = new JobSpecification(); // Create dummy key provider for feeding the primary index scan. AbstractOperatorDescriptor keyProviderOp = createDummyKeyProviderOp(spec); // Create primary index scan op. BTreeSearchOperatorDescriptor primaryScanOp = createPrimaryIndexScanOp(spec); // Assign op. AlgebricksMetaOperatorDescriptor asterixAssignOp = createAssignOp(spec, primaryScanOp, numNestedSecondaryKeyFields); // If any of the secondary fields are nullable, then add a select op that filters nulls. AlgebricksMetaOperatorDescriptor selectOp = null; if (anySecondaryKeyIsNullable) { selectOp = createFilterNullsSelectOp(spec, numNestedSecondaryKeyFields); } // Create secondary RTree bulk load op. TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp( spec, numNestedSecondaryKeyFields, new LSMRTreeDataflowHelperFactory( valueProviderFactories, RTreePolicyType.RTREE, primaryComparatorFactories, AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER, AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER, AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER, AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER, AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length), GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES), BTree.DEFAULT_FILL_FACTOR); // Connect the operators. spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryScanOp, 0); spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, asterixAssignOp, 0); if (anySecondaryKeyIsNullable) { spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, selectOp, 0); spec.connect(new OneToOneConnectorDescriptor(spec), selectOp, 0, secondaryBulkLoadOp, 0); } else { spec.connect( new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, secondaryBulkLoadOp, 0); } spec.addRoot(secondaryBulkLoadOp); spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy()); return spec; }
@Override protected void setSecondaryRecDescAndComparators( CompiledCreateIndexStatement createIndexStmt, AqlMetadataProvider metadata) throws AlgebricksException, AsterixException { List<String> secondaryKeyFields = createIndexStmt.getKeyFields(); int numSecondaryKeys = secondaryKeyFields.size(); if (numSecondaryKeys != 1) { throw new AsterixException( "Cannot use " + numSecondaryKeys + " fields as a key for the R-tree index. There can be only one field as a key for the R-tree index."); } Pair<IAType, Boolean> spatialTypePair = Index.getNonNullableKeyFieldType(secondaryKeyFields.get(0), itemType); IAType spatialType = spatialTypePair.first; anySecondaryKeyIsNullable = spatialTypePair.second; if (spatialType == null) { throw new AsterixException( "Could not find field " + secondaryKeyFields.get(0) + " in the schema."); } int numDimensions = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag()); numNestedSecondaryKeyFields = numDimensions * 2; secondaryFieldAccessEvalFactories = metadata .getFormat() .createMBRFactory(itemType, secondaryKeyFields.get(0), numPrimaryKeys, numDimensions); secondaryComparatorFactories = new IBinaryComparatorFactory[numNestedSecondaryKeyFields]; valueProviderFactories = new IPrimitiveValueProviderFactory[numNestedSecondaryKeyFields]; ISerializerDeserializer[] secondaryRecFields = new ISerializerDeserializer[numPrimaryKeys + numNestedSecondaryKeyFields]; ITypeTraits[] secondaryTypeTraits = new ITypeTraits[numNestedSecondaryKeyFields + numPrimaryKeys]; IAType nestedKeyType = NonTaggedFormatUtil.getNestedSpatialType(spatialType.getTypeTag()); keyType = nestedKeyType.getTypeTag(); for (int i = 0; i < numNestedSecondaryKeyFields; i++) { ISerializerDeserializer keySerde = AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(nestedKeyType); secondaryRecFields[i] = keySerde; secondaryComparatorFactories[i] = AqlBinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory( nestedKeyType, true); secondaryTypeTraits[i] = AqlTypeTraitProvider.INSTANCE.getTypeTrait(nestedKeyType); valueProviderFactories[i] = AqlPrimitiveValueProviderFactory.INSTANCE; } // Add serializers and comparators for primary index fields. for (int i = 0; i < numPrimaryKeys; i++) { secondaryRecFields[numNestedSecondaryKeyFields + i] = primaryRecDesc.getFields()[i]; secondaryTypeTraits[numNestedSecondaryKeyFields + i] = primaryRecDesc.getTypeTraits()[i]; } secondaryRecDesc = new RecordDescriptor(secondaryRecFields, secondaryTypeTraits); }