예제 #1
0
  private void setPartitionConstraintsDFS(
      OperatorDescriptorId opId,
      Map<IConnectorDescriptor, TargetConstraint> tgtConstraints,
      IOperatorDescriptor parentOp) {
    List<IConnectorDescriptor> opInputs = jobSpec.getOperatorInputMap().get(opId);
    AlgebricksPartitionConstraint opConstraint = null;
    IOperatorDescriptor opDesc = jobSpec.getOperatorMap().get(opId);
    if (opInputs != null) {
      for (IConnectorDescriptor conn : opInputs) {
        ConnectorDescriptorId cid = conn.getConnectorId();
        org.apache.commons.lang3.tuple.Pair<
                org.apache.commons.lang3.tuple.Pair<IOperatorDescriptor, Integer>,
                org.apache.commons.lang3.tuple.Pair<IOperatorDescriptor, Integer>>
            p = jobSpec.getConnectorOperatorMap().get(cid);
        IOperatorDescriptor src = p.getLeft().getLeft();
        // DFS
        setPartitionConstraintsDFS(src.getOperatorId(), tgtConstraints, opDesc);

        TargetConstraint constraint = tgtConstraints.get(conn);
        if (constraint != null) {
          switch (constraint) {
            case ONE:
              {
                opConstraint = new AlgebricksCountPartitionConstraint(1);
                break;
              }
            case SAME_COUNT:
              {
                opConstraint = partitionConstraintMap.get(src);
                break;
              }
          }
        }
      }
    }
    if (partitionConstraintMap.get(opDesc) == null) {
      if (opConstraint == null) {
        if (parentOp != null) {
          AlgebricksPartitionConstraint pc = partitionConstraintMap.get(parentOp);
          if (pc != null) {
            opConstraint = pc;
          } else if (opInputs == null || opInputs.size() == 0) {
            opConstraint = new AlgebricksCountPartitionConstraint(1);
          }
        }
        if (opConstraint == null) {
          opConstraint = clusterLocations;
        }
      }
      partitionConstraintMap.put(opDesc, opConstraint);
      AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(
          jobSpec, opDesc, opConstraint);
    }
  }
예제 #2
0
 private void setSpecifiedPartitionConstraints() {
   for (ILogicalOperator op : pcForMicroOps.keySet()) {
     AlgebricksPartitionConstraint pc = pcForMicroOps.get(op);
     Integer k = algebraicOpBelongingToMetaAsterixOp.get(op);
     AlgebricksMetaOperatorDescriptor amod = metaAsterixOps.get(k);
     partitionConstraintMap.put(amod, pc);
   }
   for (IOperatorDescriptor opDesc : partitionConstraintMap.keySet()) {
     AlgebricksPartitionConstraint pc = partitionConstraintMap.get(opDesc);
     AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(jobSpec, opDesc, pc);
   }
 }
예제 #3
0
  @Override
  public JobSpecification buildCreationJobSpec() throws AsterixException, AlgebricksException {
    JobSpecification spec = new JobSpecification();

    // prepare a LocalResourceMetadata which will be stored in NC's local resource repository
    ILocalResourceMetadata localResourceMetadata =
        new LSMRTreeLocalResourceMetadata(
            secondaryRecDesc.getTypeTraits(),
            secondaryComparatorFactories,
            primaryComparatorFactories,
            valueProviderFactories,
            RTreePolicyType.RTREE,
            AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
            GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
            GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES);
    ILocalResourceFactoryProvider localResourceFactoryProvider =
        new PersistentLocalResourceFactoryProvider(
            localResourceMetadata, LocalResource.LSMRTreeResource);

    TreeIndexCreateOperatorDescriptor secondaryIndexCreateOp =
        new TreeIndexCreateOperatorDescriptor(
            spec,
            AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER,
            AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER,
            secondaryFileSplitProvider,
            secondaryRecDesc.getTypeTraits(),
            secondaryComparatorFactories,
            null,
            new LSMRTreeDataflowHelperFactory(
                valueProviderFactories,
                RTreePolicyType.RTREE,
                primaryComparatorFactories,
                AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
                AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
                AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
                AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
                AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
                GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
                GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES),
            localResourceFactoryProvider,
            NoOpOperationCallbackFactory.INSTANCE);
    AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(
        spec, secondaryIndexCreateOp, secondaryPartitionConstraint);
    spec.addRoot(secondaryIndexCreateOp);
    spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
    return spec;
  }