Пример #1
0
 public VertexParallelismUpdatedProto toProto() {
   VertexParallelismUpdatedProto.Builder builder = VertexParallelismUpdatedProto.newBuilder();
   builder.setVertexId(vertexID.toString()).setNumTasks(numTasks);
   if (vertexLocationHint != null) {
     builder.setVertexLocationHint(
         DagTypeConverters.convertVertexLocationHintToProto(this.vertexLocationHint));
   }
   if (sourceEdgeManagers != null) {
     for (Entry<String, EdgeManagerPluginDescriptor> entry : sourceEdgeManagers.entrySet()) {
       EdgeManagerDescriptorProto.Builder edgeMgrBuilder = EdgeManagerDescriptorProto.newBuilder();
       edgeMgrBuilder.setEdgeName(entry.getKey());
       edgeMgrBuilder.setEntityDescriptor(DagTypeConverters.convertToDAGPlan(entry.getValue()));
       builder.addEdgeManagerDescriptors(edgeMgrBuilder.build());
     }
   }
   if (rootInputSpecUpdates != null) {
     for (Entry<String, InputSpecUpdate> entry : rootInputSpecUpdates.entrySet()) {
       RootInputSpecUpdateProto.Builder rootInputSpecUpdateBuilder =
           RootInputSpecUpdateProto.newBuilder();
       rootInputSpecUpdateBuilder.setInputName(entry.getKey());
       rootInputSpecUpdateBuilder.setForAllWorkUnits(entry.getValue().isForAllWorkUnits());
       rootInputSpecUpdateBuilder.addAllNumPhysicalInputs(
           entry.getValue().getAllNumPhysicalInputs());
       builder.addRootInputSpecUpdates(rootInputSpecUpdateBuilder.build());
     }
   }
   return builder.build();
 }
Пример #2
0
 public void fromProto(VertexParallelismUpdatedProto proto) {
   this.vertexID = TezVertexID.fromString(proto.getVertexId());
   this.numTasks = proto.getNumTasks();
   if (proto.hasVertexLocationHint()) {
     this.vertexLocationHint =
         DagTypeConverters.convertVertexLocationHintFromProto(proto.getVertexLocationHint());
   }
   if (proto.getEdgeManagerDescriptorsCount() > 0) {
     this.sourceEdgeManagers =
         new HashMap<String, EdgeManagerPluginDescriptor>(proto.getEdgeManagerDescriptorsCount());
     for (EdgeManagerDescriptorProto edgeManagerProto : proto.getEdgeManagerDescriptorsList()) {
       EdgeManagerPluginDescriptor edgeManagerDescriptor =
           DagTypeConverters.convertEdgeManagerPluginDescriptorFromDAGPlan(
               edgeManagerProto.getEntityDescriptor());
       sourceEdgeManagers.put(edgeManagerProto.getEdgeName(), edgeManagerDescriptor);
     }
   }
   if (proto.getRootInputSpecUpdatesCount() > 0) {
     this.rootInputSpecUpdates = Maps.newHashMap();
     for (RootInputSpecUpdateProto rootInputSpecUpdateProto :
         proto.getRootInputSpecUpdatesList()) {
       InputSpecUpdate specUpdate;
       if (rootInputSpecUpdateProto.getForAllWorkUnits()) {
         specUpdate =
             InputSpecUpdate.createAllTaskInputSpecUpdate(
                 rootInputSpecUpdateProto.getNumPhysicalInputs(0));
       } else {
         specUpdate =
             InputSpecUpdate.createPerTaskInputSpecUpdate(
                 rootInputSpecUpdateProto.getNumPhysicalInputsList());
       }
       this.rootInputSpecUpdates.put(rootInputSpecUpdateProto.getInputName(), specUpdate);
     }
   }
 }
 public TezSessionStatus getSessionStatus() throws TezException, IOException {
   try {
     ApplicationReport appReport = yarnClient.getApplicationReport(applicationId);
     switch (appReport.getYarnApplicationState()) {
       case NEW:
       case NEW_SAVING:
       case ACCEPTED:
       case SUBMITTED:
         return TezSessionStatus.INITIALIZING;
       case FINISHED:
       case FAILED:
       case KILLED:
         return TezSessionStatus.SHUTDOWN;
       case RUNNING:
         try {
           DAGClientAMProtocolBlockingPB proxy =
               TezClientUtils.getSessionAMProxy(
                   yarnClient, sessionConfig.getYarnConfiguration(), applicationId);
           if (proxy == null) {
             return TezSessionStatus.INITIALIZING;
           }
           GetAMStatusResponseProto response =
               proxy.getAMStatus(null, GetAMStatusRequestProto.newBuilder().build());
           return DagTypeConverters.convertTezSessionStatusFromProto(response.getStatus());
         } catch (TezException e) {
           LOG.info("Failed to retrieve AM Status via proxy", e);
         } catch (ServiceException e) {
           LOG.info("Failed to retrieve AM Status via proxy", e);
         }
     }
   } catch (YarnException e) {
     throw new TezException(e);
   }
   return TezSessionStatus.INITIALIZING;
 }
Пример #4
0
 public void fromProto(DAGFinishedProto proto) {
   this.dagID = TezDAGID.fromString(proto.getDagId());
   this.finishTime = proto.getFinishTime();
   this.state = DAGState.values()[proto.getState()];
   if (proto.hasDiagnostics()) {
     this.diagnostics = proto.getDiagnostics();
   }
   if (proto.hasCounters()) {
     this.tezCounters = DagTypeConverters.convertTezCountersFromProto(proto.getCounters());
   }
 }
Пример #5
0
 public void fromProto(DAGSubmittedProto proto) {
   this.dagID = TezDAGID.fromString(proto.getDagId());
   this.dagPlan = proto.getDagPlan();
   this.dagName = this.dagPlan.getName();
   this.submitTime = proto.getSubmitTime();
   this.applicationAttemptId =
       ConverterUtils.toApplicationAttemptId(proto.getApplicationAttemptId());
   if (proto.hasCumulativeAdditionalAmResources()) {
     this.cumulativeAdditionalLocalResources =
         DagTypeConverters.convertFromPlanLocalResources(
             proto.getCumulativeAdditionalAmResources());
   }
 }
Пример #6
0
 public void fromProto(TaskFinishedProto proto) {
   this.taskID = TezTaskID.fromString(proto.getTaskId());
   this.finishTime = proto.getFinishTime();
   this.state = TaskState.values()[proto.getState()];
   if (proto.hasDiagnostics()) {
     this.diagnostics = proto.getDiagnostics();
   }
   if (proto.hasCounters()) {
     this.tezCounters = DagTypeConverters.convertTezCountersFromProto(proto.getCounters());
   }
   if (proto.hasSuccessfulTaskAttemptId()) {
     this.successfulAttemptID = TezTaskAttemptID.fromString(proto.getSuccessfulTaskAttemptId());
   }
 }
Пример #7
0
 public TaskFinishedProto toProto() {
   TaskFinishedProto.Builder builder = TaskFinishedProto.newBuilder();
   builder.setTaskId(taskID.toString()).setState(state.ordinal()).setFinishTime(finishTime);
   if (diagnostics != null) {
     builder.setDiagnostics(diagnostics);
   }
   if (tezCounters != null) {
     builder.setCounters(DagTypeConverters.convertTezCountersToProto(tezCounters));
   }
   if (successfulAttemptID != null) {
     builder.setSuccessfulTaskAttemptId(successfulAttemptID.toString());
   }
   return builder.build();
 }
Пример #8
0
 public DAGSubmittedProto toProto() {
   DAGSubmittedProto.Builder builder =
       DAGSubmittedProto.newBuilder()
           .setDagId(dagID.toString())
           .setApplicationAttemptId(applicationAttemptId.toString())
           .setDagPlan(dagPlan)
           .setSubmitTime(submitTime);
   if (cumulativeAdditionalLocalResources != null
       && !cumulativeAdditionalLocalResources.isEmpty()) {
     builder.setCumulativeAdditionalAmResources(
         DagTypeConverters.convertFromLocalResources(cumulativeAdditionalLocalResources));
   }
   return builder.build();
 }
Пример #9
0
  public DAGFinishedProto toProto() {
    DAGFinishedProto.Builder builder = DAGFinishedProto.newBuilder();

    builder.setDagId(dagID.toString()).setState(state.ordinal()).setFinishTime(finishTime);

    if (diagnostics != null) {
      builder.setDiagnostics(diagnostics);
    }
    if (tezCounters != null) {
      builder.setCounters(DagTypeConverters.convertTezCountersToProto(tezCounters));
    }

    return builder.build();
  }
Пример #10
0
  public static Map<String, Object> convertDAGPlanToATSMap(DAGPlan dagPlan) throws IOException {

    final String VERSION_KEY = "version";
    final int version = 1;
    Map<String, Object> dagMap = new LinkedHashMap<String, Object>();
    dagMap.put(DAG_NAME_KEY, dagPlan.getName());
    if (dagPlan.hasDagInfo()) {
      dagMap.put(DAG_INFO_KEY, dagPlan.getDagInfo());
    }
    dagMap.put(VERSION_KEY, version);
    ArrayList<Object> verticesList = new ArrayList<Object>();
    for (DAGProtos.VertexPlan vertexPlan : dagPlan.getVertexList()) {
      Map<String, Object> vertexMap = new LinkedHashMap<String, Object>();
      vertexMap.put(VERTEX_NAME_KEY, vertexPlan.getName());
      if (vertexPlan.hasProcessorDescriptor()) {
        vertexMap.put(PROCESSOR_CLASS_KEY, vertexPlan.getProcessorDescriptor().getClassName());
        if (vertexPlan.getProcessorDescriptor().hasHistoryText()) {
          vertexMap.put(
              USER_PAYLOAD_AS_TEXT,
              DagTypeConverters.getHistoryTextFromProto(vertexPlan.getProcessorDescriptor()));
        }
      }

      ArrayList<Object> inEdgeIdList = new ArrayList<Object>();
      inEdgeIdList.addAll(vertexPlan.getInEdgeIdList());
      putInto(vertexMap, IN_EDGE_IDS_KEY, inEdgeIdList);

      ArrayList<Object> outEdgeIdList = new ArrayList<Object>();
      outEdgeIdList.addAll(vertexPlan.getOutEdgeIdList());
      putInto(vertexMap, OUT_EDGE_IDS_KEY, outEdgeIdList);

      ArrayList<Object> inputsList = new ArrayList<Object>();
      for (DAGProtos.RootInputLeafOutputProto input : vertexPlan.getInputsList()) {
        Map<String, Object> inputMap = new LinkedHashMap<String, Object>();
        inputMap.put(NAME_KEY, input.getName());
        inputMap.put(CLASS_KEY, input.getIODescriptor().getClassName());
        if (input.hasControllerDescriptor()) {
          inputMap.put(INITIALIZER_KEY, input.getControllerDescriptor().getClassName());
        }
        if (input.getIODescriptor().hasHistoryText()) {
          inputMap.put(
              USER_PAYLOAD_AS_TEXT,
              DagTypeConverters.getHistoryTextFromProto(input.getIODescriptor()));
        }
        inputsList.add(inputMap);
      }
      putInto(vertexMap, ADDITIONAL_INPUTS_KEY, inputsList);

      ArrayList<Object> outputsList = new ArrayList<Object>();
      for (DAGProtos.RootInputLeafOutputProto output : vertexPlan.getOutputsList()) {
        Map<String, Object> outputMap = new LinkedHashMap<String, Object>();
        outputMap.put(NAME_KEY, output.getName());
        outputMap.put(CLASS_KEY, output.getIODescriptor().getClassName());
        if (output.hasControllerDescriptor()) {
          outputMap.put(INITIALIZER_KEY, output.getControllerDescriptor().getClassName());
        }
        if (output.getIODescriptor().hasHistoryText()) {
          outputMap.put(
              USER_PAYLOAD_AS_TEXT,
              DagTypeConverters.getHistoryTextFromProto(output.getIODescriptor()));
        }
        outputsList.add(outputMap);
      }
      putInto(vertexMap, ADDITIONAL_OUTPUTS_KEY, outputsList);

      if (vertexPlan.hasVertexManagerPlugin()) {
        vertexMap.put(
            VERTEX_MANAGER_PLUGIN_CLASS_KEY, vertexPlan.getVertexManagerPlugin().getClassName());
      }

      verticesList.add(vertexMap);
    }
    putInto(dagMap, VERTICES_KEY, verticesList);

    ArrayList<Object> edgesList = new ArrayList<Object>();
    for (DAGProtos.EdgePlan edgePlan : dagPlan.getEdgeList()) {
      Map<String, Object> edgeMap = new LinkedHashMap<String, Object>();
      edgeMap.put(EDGE_ID_KEY, edgePlan.getId());
      edgeMap.put(INPUT_VERTEX_NAME_KEY, edgePlan.getInputVertexName());
      edgeMap.put(OUTPUT_VERTEX_NAME_KEY, edgePlan.getOutputVertexName());
      edgeMap.put(DATA_MOVEMENT_TYPE_KEY, edgePlan.getDataMovementType().name());
      edgeMap.put(DATA_SOURCE_TYPE_KEY, edgePlan.getDataSourceType().name());
      edgeMap.put(SCHEDULING_TYPE_KEY, edgePlan.getSchedulingType().name());
      edgeMap.put(EDGE_SOURCE_CLASS_KEY, edgePlan.getEdgeSource().getClassName());
      edgeMap.put(EDGE_DESTINATION_CLASS_KEY, edgePlan.getEdgeDestination().getClassName());
      if (edgePlan.getEdgeSource().hasHistoryText()) {
        edgeMap.put(
            OUTPUT_USER_PAYLOAD_AS_TEXT,
            DagTypeConverters.getHistoryTextFromProto(edgePlan.getEdgeSource()));
      }
      if (edgePlan.getEdgeDestination().hasHistoryText()) {
        edgeMap.put(
            INPUT_USER_PAYLOAD_AS_TEXT,
            DagTypeConverters.getHistoryTextFromProto(edgePlan.getEdgeDestination()));
      } // TEZ-2286 this is missing edgemanager descriptor for custom edge
      edgesList.add(edgeMap);
    }
    putInto(dagMap, EDGES_KEY, edgesList);

    ArrayList<Object> vertexGroupsList = new ArrayList<Object>();
    for (DAGProtos.PlanVertexGroupInfo vertexGroupInfo : dagPlan.getVertexGroupsList()) {
      Map<String, Object> groupMap = new LinkedHashMap<String, Object>();
      groupMap.put(VERTEX_GROUP_NAME_KEY, vertexGroupInfo.getGroupName());
      if (vertexGroupInfo.getGroupMembersCount() > 0) {
        groupMap.put(VERTEX_GROUP_MEMBERS_KEY, vertexGroupInfo.getGroupMembersList());
      }
      if (vertexGroupInfo.getOutputsCount() > 0) {
        groupMap.put(VERTEX_GROUP_OUTPUTS_KEY, vertexGroupInfo.getOutputsList());
      }

      if (vertexGroupInfo.getEdgeMergedInputsCount() > 0) {
        ArrayList<Object> edgeMergedInputs = new ArrayList<Object>();
        for (PlanGroupInputEdgeInfo edgeMergedInputInfo :
            vertexGroupInfo.getEdgeMergedInputsList()) {
          Map<String, Object> edgeMergedInput = new LinkedHashMap<String, Object>();
          edgeMergedInput.put(
              VERTEX_GROUP_DESTINATION_VERTEX_NAME_KEY, edgeMergedInputInfo.getDestVertexName());
          if (edgeMergedInputInfo.hasMergedInput()
              && edgeMergedInputInfo.getMergedInput().hasClassName()) {
            edgeMergedInput.put(
                PROCESSOR_CLASS_KEY, edgeMergedInputInfo.getMergedInput().getClassName());
            if (edgeMergedInputInfo.getMergedInput().hasHistoryText()) {
              edgeMergedInput.put(
                  USER_PAYLOAD_AS_TEXT,
                  DagTypeConverters.getHistoryTextFromProto(edgeMergedInputInfo.getMergedInput()));
            }
          }
          edgeMergedInputs.add(edgeMergedInput);
        }
        groupMap.put(VERTEX_GROUP_EDGE_MERGED_INPUTS_KEY, edgeMergedInputs);
      }
      vertexGroupsList.add(groupMap);
    }
    putInto(dagMap, VERTEX_GROUPS_KEY, vertexGroupsList);

    return dagMap;
  }