Пример #1
0
  @Override
  public JobSpecification buildCreationJobSpec() throws AsterixException, AlgebricksException {
    JobSpecification spec = new JobSpecification();

    // prepare a LocalResourceMetadata which will be stored in NC's local resource repository
    ILocalResourceMetadata localResourceMetadata =
        new LSMRTreeLocalResourceMetadata(
            secondaryRecDesc.getTypeTraits(),
            secondaryComparatorFactories,
            primaryComparatorFactories,
            valueProviderFactories,
            RTreePolicyType.RTREE,
            AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
            GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
            GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES);
    ILocalResourceFactoryProvider localResourceFactoryProvider =
        new PersistentLocalResourceFactoryProvider(
            localResourceMetadata, LocalResource.LSMRTreeResource);

    TreeIndexCreateOperatorDescriptor secondaryIndexCreateOp =
        new TreeIndexCreateOperatorDescriptor(
            spec,
            AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER,
            AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER,
            secondaryFileSplitProvider,
            secondaryRecDesc.getTypeTraits(),
            secondaryComparatorFactories,
            null,
            new LSMRTreeDataflowHelperFactory(
                valueProviderFactories,
                RTreePolicyType.RTREE,
                primaryComparatorFactories,
                AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
                AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
                AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
                AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
                AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
                GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
                GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES),
            localResourceFactoryProvider,
            NoOpOperationCallbackFactory.INSTANCE);
    AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(
        spec, secondaryIndexCreateOp, secondaryPartitionConstraint);
    spec.addRoot(secondaryIndexCreateOp);
    spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
    return spec;
  }
Пример #2
0
  @Override
  public JobSpecification buildLoadingJobSpec() throws AsterixException, AlgebricksException {
    JobSpecification spec = new JobSpecification();

    // Create dummy key provider for feeding the primary index scan.
    AbstractOperatorDescriptor keyProviderOp = createDummyKeyProviderOp(spec);

    // Create primary index scan op.
    BTreeSearchOperatorDescriptor primaryScanOp = createPrimaryIndexScanOp(spec);

    // Assign op.
    AlgebricksMetaOperatorDescriptor asterixAssignOp =
        createAssignOp(spec, primaryScanOp, numNestedSecondaryKeyFields);

    // If any of the secondary fields are nullable, then add a select op that filters nulls.
    AlgebricksMetaOperatorDescriptor selectOp = null;
    if (anySecondaryKeyIsNullable) {
      selectOp = createFilterNullsSelectOp(spec, numNestedSecondaryKeyFields);
    }

    // Create secondary RTree bulk load op.
    TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp =
        createTreeIndexBulkLoadOp(
            spec,
            numNestedSecondaryKeyFields,
            new LSMRTreeDataflowHelperFactory(
                valueProviderFactories,
                RTreePolicyType.RTREE,
                primaryComparatorFactories,
                AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
                AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
                AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
                AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
                AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
                GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
                GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES),
            BTree.DEFAULT_FILL_FACTOR);

    // Connect the operators.
    spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryScanOp, 0);
    spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, asterixAssignOp, 0);
    if (anySecondaryKeyIsNullable) {
      spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, selectOp, 0);
      spec.connect(new OneToOneConnectorDescriptor(spec), selectOp, 0, secondaryBulkLoadOp, 0);
    } else {
      spec.connect(
          new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, secondaryBulkLoadOp, 0);
    }
    spec.addRoot(secondaryBulkLoadOp);
    spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
    return spec;
  }
Пример #3
0
 @Override
 protected void setSecondaryRecDescAndComparators(
     CompiledCreateIndexStatement createIndexStmt, AqlMetadataProvider metadata)
     throws AlgebricksException, AsterixException {
   List<String> secondaryKeyFields = createIndexStmt.getKeyFields();
   int numSecondaryKeys = secondaryKeyFields.size();
   if (numSecondaryKeys != 1) {
     throw new AsterixException(
         "Cannot use "
             + numSecondaryKeys
             + " fields as a key for the R-tree index. There can be only one field as a key for the R-tree index.");
   }
   Pair<IAType, Boolean> spatialTypePair =
       Index.getNonNullableKeyFieldType(secondaryKeyFields.get(0), itemType);
   IAType spatialType = spatialTypePair.first;
   anySecondaryKeyIsNullable = spatialTypePair.second;
   if (spatialType == null) {
     throw new AsterixException(
         "Could not find field " + secondaryKeyFields.get(0) + " in the schema.");
   }
   int numDimensions = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag());
   numNestedSecondaryKeyFields = numDimensions * 2;
   secondaryFieldAccessEvalFactories =
       metadata
           .getFormat()
           .createMBRFactory(itemType, secondaryKeyFields.get(0), numPrimaryKeys, numDimensions);
   secondaryComparatorFactories = new IBinaryComparatorFactory[numNestedSecondaryKeyFields];
   valueProviderFactories = new IPrimitiveValueProviderFactory[numNestedSecondaryKeyFields];
   ISerializerDeserializer[] secondaryRecFields =
       new ISerializerDeserializer[numPrimaryKeys + numNestedSecondaryKeyFields];
   ITypeTraits[] secondaryTypeTraits =
       new ITypeTraits[numNestedSecondaryKeyFields + numPrimaryKeys];
   IAType nestedKeyType = NonTaggedFormatUtil.getNestedSpatialType(spatialType.getTypeTag());
   keyType = nestedKeyType.getTypeTag();
   for (int i = 0; i < numNestedSecondaryKeyFields; i++) {
     ISerializerDeserializer keySerde =
         AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(nestedKeyType);
     secondaryRecFields[i] = keySerde;
     secondaryComparatorFactories[i] =
         AqlBinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(
             nestedKeyType, true);
     secondaryTypeTraits[i] = AqlTypeTraitProvider.INSTANCE.getTypeTrait(nestedKeyType);
     valueProviderFactories[i] = AqlPrimitiveValueProviderFactory.INSTANCE;
   }
   // Add serializers and comparators for primary index fields.
   for (int i = 0; i < numPrimaryKeys; i++) {
     secondaryRecFields[numNestedSecondaryKeyFields + i] = primaryRecDesc.getFields()[i];
     secondaryTypeTraits[numNestedSecondaryKeyFields + i] = primaryRecDesc.getTypeTraits()[i];
   }
   secondaryRecDesc = new RecordDescriptor(secondaryRecFields, secondaryTypeTraits);
 }