コード例 #1
0
  private synchronized void handleIntakeJobStartMessage(FeedIntakeInfo intakeJobInfo)
      throws Exception {
    List<OperatorDescriptorId> intakeOperatorIds = new ArrayList<OperatorDescriptorId>();
    Map<OperatorDescriptorId, IOperatorDescriptor> operators =
        intakeJobInfo.getSpec().getOperatorMap();
    for (Entry<OperatorDescriptorId, IOperatorDescriptor> entry : operators.entrySet()) {
      IOperatorDescriptor opDesc = entry.getValue();
      if (opDesc instanceof FeedIntakeOperatorDescriptor) {
        intakeOperatorIds.add(opDesc.getOperatorId());
      }
    }

    IHyracksClientConnection hcc = AsterixAppContextInfo.getInstance().getHcc();
    JobInfo info = hcc.getJobInfo(intakeJobInfo.getJobId());
    List<String> intakeLocations = new ArrayList<String>();
    for (OperatorDescriptorId intakeOperatorId : intakeOperatorIds) {
      Map<Integer, String> operatorLocations = info.getOperatorLocations().get(intakeOperatorId);
      int nOperatorInstances = operatorLocations.size();
      for (int i = 0; i < nOperatorInstances; i++) {
        intakeLocations.add(operatorLocations.get(i));
      }
    }
    // intakeLocations is an ordered list; element at position i corresponds to location of i'th
    // instance of operator
    intakeJobInfo.setIntakeLocation(intakeLocations);
    intakeJobInfo.getIntakeFeedJoint().setState(State.ACTIVE);
    intakeJobInfo.setState(FeedJobState.ACTIVE);

    // notify event listeners
    notifyFeedEventSubscribers(intakeJobInfo, FeedLifecycleEvent.FEED_INTAKE_STARTED);
  }
コード例 #2
0
  private void handleFeedIntakeJobFinishMessage(FeedIntakeInfo intakeInfo, Message message)
      throws Exception {
    IHyracksClientConnection hcc = AsterixAppContextInfo.getInstance().getHcc();
    JobInfo info = hcc.getJobInfo(message.jobId);
    JobStatus status = info.getStatus();
    FeedLifecycleEvent event;
    event =
        status.equals(JobStatus.FAILURE)
            ? FeedLifecycleEvent.FEED_INTAKE_FAILURE
            : FeedLifecycleEvent.FEED_ENDED;

    // remove feed joints
    deregisterFeedIntakeJob(message.jobId);

    // notify event listeners
    notifyFeedEventSubscribers(intakeInfo, event);
  }
コード例 #3
0
  private void handleFeedCollectJobFinishMessage(FeedConnectJobInfo cInfo) throws Exception {
    FeedConnectionId connectionId = cInfo.getConnectionId();

    IHyracksClientConnection hcc = AsterixAppContextInfo.getInstance().getHcc();
    JobInfo info = hcc.getJobInfo(cInfo.getJobId());
    JobStatus status = info.getStatus();
    boolean failure = status != null && status.equals(JobStatus.FAILURE);
    FeedPolicyAccessor fpa = new FeedPolicyAccessor(cInfo.getFeedPolicy());

    boolean removeJobHistory = !failure;
    boolean retainSubsription =
        cInfo.getState().equals(FeedJobState.UNDER_RECOVERY)
            || (failure && fpa.continueOnHardwareFailure());

    if (!retainSubsription) {
      IFeedJoint feedJoint = cInfo.getSourceFeedJoint();
      feedJoint.removeReceiver(connectionId);
      if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info(
            "Subscription "
                + cInfo.getConnectionId()
                + " completed successfully. Removed subscription");
      }
      removeFeedJointsPostPipelineTermination(cInfo.getConnectionId());
    }

    if (removeJobHistory) {
      connectJobInfos.remove(connectionId);
      jobInfos.remove(cInfo.getJobId());
      feedIntakeProgressTrackers.remove(cInfo.getConnectionId());
    }
    deregisterFeedActivity(cInfo);

    // notify event listeners
    FeedLifecycleEvent event =
        failure ? FeedLifecycleEvent.FEED_COLLECT_FAILURE : FeedLifecycleEvent.FEED_ENDED;
    notifyFeedEventSubscribers(cInfo, event);
  }
コード例 #4
0
  private void setLocations(FeedConnectJobInfo cInfo) {
    JobSpecification jobSpec = cInfo.getSpec();

    List<OperatorDescriptorId> collectOperatorIds = new ArrayList<OperatorDescriptorId>();
    List<OperatorDescriptorId> computeOperatorIds = new ArrayList<OperatorDescriptorId>();
    List<OperatorDescriptorId> storageOperatorIds = new ArrayList<OperatorDescriptorId>();

    Map<OperatorDescriptorId, IOperatorDescriptor> operators = jobSpec.getOperatorMap();
    for (Entry<OperatorDescriptorId, IOperatorDescriptor> entry : operators.entrySet()) {
      IOperatorDescriptor opDesc = entry.getValue();
      IOperatorDescriptor actualOp = null;
      if (opDesc instanceof FeedMetaOperatorDescriptor) {
        actualOp = ((FeedMetaOperatorDescriptor) opDesc).getCoreOperator();
      } else {
        actualOp = opDesc;
      }

      if (actualOp instanceof AlgebricksMetaOperatorDescriptor) {
        AlgebricksMetaOperatorDescriptor op = ((AlgebricksMetaOperatorDescriptor) actualOp);
        IPushRuntimeFactory[] runtimeFactories = op.getPipeline().getRuntimeFactories();
        boolean computeOp = false;
        for (IPushRuntimeFactory rf : runtimeFactories) {
          if (rf instanceof AssignRuntimeFactory) {
            IConnectorDescriptor connDesc =
                jobSpec.getOperatorInputMap().get(op.getOperatorId()).get(0);
            IOperatorDescriptor sourceOp =
                jobSpec
                    .getConnectorOperatorMap()
                    .get(connDesc.getConnectorId())
                    .getLeft()
                    .getLeft();
            if (sourceOp instanceof FeedCollectOperatorDescriptor) {
              computeOp = true;
              break;
            }
          }
        }
        if (computeOp) {
          computeOperatorIds.add(entry.getKey());
        }
      } else if (actualOp instanceof LSMTreeIndexInsertUpdateDeleteOperatorDescriptor) {
        storageOperatorIds.add(entry.getKey());
      } else if (actualOp instanceof FeedCollectOperatorDescriptor) {
        collectOperatorIds.add(entry.getKey());
      }
    }

    try {
      IHyracksClientConnection hcc = AsterixAppContextInfo.getInstance().getHcc();
      JobInfo info = hcc.getJobInfo(cInfo.getJobId());
      List<String> collectLocations = new ArrayList<String>();
      for (OperatorDescriptorId collectOpId : collectOperatorIds) {
        Map<Integer, String> operatorLocations = info.getOperatorLocations().get(collectOpId);
        int nOperatorInstances = operatorLocations.size();
        for (int i = 0; i < nOperatorInstances; i++) {
          collectLocations.add(operatorLocations.get(i));
        }
      }

      List<String> computeLocations = new ArrayList<String>();
      for (OperatorDescriptorId computeOpId : computeOperatorIds) {
        Map<Integer, String> operatorLocations = info.getOperatorLocations().get(computeOpId);
        if (operatorLocations != null) {
          int nOperatorInstances = operatorLocations.size();
          for (int i = 0; i < nOperatorInstances; i++) {
            computeLocations.add(operatorLocations.get(i));
          }
        } else {
          computeLocations.clear();
          computeLocations.addAll(collectLocations);
        }
      }

      List<String> storageLocations = new ArrayList<String>();
      for (OperatorDescriptorId storageOpId : storageOperatorIds) {
        Map<Integer, String> operatorLocations = info.getOperatorLocations().get(storageOpId);
        if (operatorLocations == null) {
          continue;
        }
        int nOperatorInstances = operatorLocations.size();
        for (int i = 0; i < nOperatorInstances; i++) {
          storageLocations.add(operatorLocations.get(i));
        }
      }
      cInfo.setCollectLocations(collectLocations);
      cInfo.setComputeLocations(computeLocations);
      cInfo.setStorageLocations(storageLocations);

    } catch (Exception e) {
      e.printStackTrace();
    }
  }