public VertexParallelismUpdatedProto toProto() {
VertexParallelismUpdatedProto.Builder builder = VertexParallelismUpdatedProto.newBuilder();
builder.setVertexId(vertexID.toString()).setNumTasks(numTasks);
if (vertexLocationHint != null) {
builder.setVertexLocationHint(
DagTypeConverters.convertVertexLocationHintToProto(this.vertexLocationHint));
}
if (sourceEdgeManagers != null) {
for (Entry<String, EdgeManagerPluginDescriptor> entry : sourceEdgeManagers.entrySet()) {
EdgeManagerDescriptorProto.Builder edgeMgrBuilder = EdgeManagerDescriptorProto.newBuilder();
edgeMgrBuilder.setEdgeName(entry.getKey());
edgeMgrBuilder.setEntityDescriptor(DagTypeConverters.convertToDAGPlan(entry.getValue()));
builder.addEdgeManagerDescriptors(edgeMgrBuilder.build());
}
}
if (rootInputSpecUpdates != null) {
for (Entry<String, InputSpecUpdate> entry : rootInputSpecUpdates.entrySet()) {
RootInputSpecUpdateProto.Builder rootInputSpecUpdateBuilder =
RootInputSpecUpdateProto.newBuilder();
rootInputSpecUpdateBuilder.setInputName(entry.getKey());
rootInputSpecUpdateBuilder.setForAllWorkUnits(entry.getValue().isForAllWorkUnits());
rootInputSpecUpdateBuilder.addAllNumPhysicalInputs(
entry.getValue().getAllNumPhysicalInputs());
builder.addRootInputSpecUpdates(rootInputSpecUpdateBuilder.build());
}
}
return builder.build();
}
public void fromProto(VertexParallelismUpdatedProto proto) {
this.vertexID = TezVertexID.fromString(proto.getVertexId());
this.numTasks = proto.getNumTasks();
if (proto.hasVertexLocationHint()) {
this.vertexLocationHint =
DagTypeConverters.convertVertexLocationHintFromProto(proto.getVertexLocationHint());
}
if (proto.getEdgeManagerDescriptorsCount() > 0) {
this.sourceEdgeManagers =
new HashMap<String, EdgeManagerPluginDescriptor>(proto.getEdgeManagerDescriptorsCount());
for (EdgeManagerDescriptorProto edgeManagerProto : proto.getEdgeManagerDescriptorsList()) {
EdgeManagerPluginDescriptor edgeManagerDescriptor =
DagTypeConverters.convertEdgeManagerPluginDescriptorFromDAGPlan(
edgeManagerProto.getEntityDescriptor());
sourceEdgeManagers.put(edgeManagerProto.getEdgeName(), edgeManagerDescriptor);
}
}
if (proto.getRootInputSpecUpdatesCount() > 0) {
this.rootInputSpecUpdates = Maps.newHashMap();
for (RootInputSpecUpdateProto rootInputSpecUpdateProto :
proto.getRootInputSpecUpdatesList()) {
InputSpecUpdate specUpdate;
if (rootInputSpecUpdateProto.getForAllWorkUnits()) {
specUpdate =
InputSpecUpdate.createAllTaskInputSpecUpdate(
rootInputSpecUpdateProto.getNumPhysicalInputs(0));
} else {
specUpdate =
InputSpecUpdate.createPerTaskInputSpecUpdate(
rootInputSpecUpdateProto.getNumPhysicalInputsList());
}
this.rootInputSpecUpdates.put(rootInputSpecUpdateProto.getInputName(), specUpdate);
}
}
}
public TezSessionStatus getSessionStatus() throws TezException, IOException {
try {
ApplicationReport appReport = yarnClient.getApplicationReport(applicationId);
switch (appReport.getYarnApplicationState()) {
case NEW:
case NEW_SAVING:
case ACCEPTED:
case SUBMITTED:
return TezSessionStatus.INITIALIZING;
case FINISHED:
case FAILED:
case KILLED:
return TezSessionStatus.SHUTDOWN;
case RUNNING:
try {
DAGClientAMProtocolBlockingPB proxy =
TezClientUtils.getSessionAMProxy(
yarnClient, sessionConfig.getYarnConfiguration(), applicationId);
if (proxy == null) {
return TezSessionStatus.INITIALIZING;
}
GetAMStatusResponseProto response =
proxy.getAMStatus(null, GetAMStatusRequestProto.newBuilder().build());
return DagTypeConverters.convertTezSessionStatusFromProto(response.getStatus());
} catch (TezException e) {
LOG.info("Failed to retrieve AM Status via proxy", e);
} catch (ServiceException e) {
LOG.info("Failed to retrieve AM Status via proxy", e);
}
}
} catch (YarnException e) {
throw new TezException(e);
}
return TezSessionStatus.INITIALIZING;
}
public void fromProto(DAGFinishedProto proto) {
this.dagID = TezDAGID.fromString(proto.getDagId());
this.finishTime = proto.getFinishTime();
this.state = DAGState.values()[proto.getState()];
if (proto.hasDiagnostics()) {
this.diagnostics = proto.getDiagnostics();
}
if (proto.hasCounters()) {
this.tezCounters = DagTypeConverters.convertTezCountersFromProto(proto.getCounters());
}
}
public void fromProto(DAGSubmittedProto proto) {
this.dagID = TezDAGID.fromString(proto.getDagId());
this.dagPlan = proto.getDagPlan();
this.dagName = this.dagPlan.getName();
this.submitTime = proto.getSubmitTime();
this.applicationAttemptId =
ConverterUtils.toApplicationAttemptId(proto.getApplicationAttemptId());
if (proto.hasCumulativeAdditionalAmResources()) {
this.cumulativeAdditionalLocalResources =
DagTypeConverters.convertFromPlanLocalResources(
proto.getCumulativeAdditionalAmResources());
}
}
public void fromProto(TaskFinishedProto proto) {
this.taskID = TezTaskID.fromString(proto.getTaskId());
this.finishTime = proto.getFinishTime();
this.state = TaskState.values()[proto.getState()];
if (proto.hasDiagnostics()) {
this.diagnostics = proto.getDiagnostics();
}
if (proto.hasCounters()) {
this.tezCounters = DagTypeConverters.convertTezCountersFromProto(proto.getCounters());
}
if (proto.hasSuccessfulTaskAttemptId()) {
this.successfulAttemptID = TezTaskAttemptID.fromString(proto.getSuccessfulTaskAttemptId());
}
}
public TaskFinishedProto toProto() {
TaskFinishedProto.Builder builder = TaskFinishedProto.newBuilder();
builder.setTaskId(taskID.toString()).setState(state.ordinal()).setFinishTime(finishTime);
if (diagnostics != null) {
builder.setDiagnostics(diagnostics);
}
if (tezCounters != null) {
builder.setCounters(DagTypeConverters.convertTezCountersToProto(tezCounters));
}
if (successfulAttemptID != null) {
builder.setSuccessfulTaskAttemptId(successfulAttemptID.toString());
}
return builder.build();
}
public DAGSubmittedProto toProto() {
DAGSubmittedProto.Builder builder =
DAGSubmittedProto.newBuilder()
.setDagId(dagID.toString())
.setApplicationAttemptId(applicationAttemptId.toString())
.setDagPlan(dagPlan)
.setSubmitTime(submitTime);
if (cumulativeAdditionalLocalResources != null
&& !cumulativeAdditionalLocalResources.isEmpty()) {
builder.setCumulativeAdditionalAmResources(
DagTypeConverters.convertFromLocalResources(cumulativeAdditionalLocalResources));
}
return builder.build();
}
public DAGFinishedProto toProto() {
DAGFinishedProto.Builder builder = DAGFinishedProto.newBuilder();
builder.setDagId(dagID.toString()).setState(state.ordinal()).setFinishTime(finishTime);
if (diagnostics != null) {
builder.setDiagnostics(diagnostics);
}
if (tezCounters != null) {
builder.setCounters(DagTypeConverters.convertTezCountersToProto(tezCounters));
}
return builder.build();
}
public static Map<String, Object> convertDAGPlanToATSMap(DAGPlan dagPlan) throws IOException {
final String VERSION_KEY = "version";
final int version = 1;
Map<String, Object> dagMap = new LinkedHashMap<String, Object>();
dagMap.put(DAG_NAME_KEY, dagPlan.getName());
if (dagPlan.hasDagInfo()) {
dagMap.put(DAG_INFO_KEY, dagPlan.getDagInfo());
}
dagMap.put(VERSION_KEY, version);
ArrayList<Object> verticesList = new ArrayList<Object>();
for (DAGProtos.VertexPlan vertexPlan : dagPlan.getVertexList()) {
Map<String, Object> vertexMap = new LinkedHashMap<String, Object>();
vertexMap.put(VERTEX_NAME_KEY, vertexPlan.getName());
if (vertexPlan.hasProcessorDescriptor()) {
vertexMap.put(PROCESSOR_CLASS_KEY, vertexPlan.getProcessorDescriptor().getClassName());
if (vertexPlan.getProcessorDescriptor().hasHistoryText()) {
vertexMap.put(
USER_PAYLOAD_AS_TEXT,
DagTypeConverters.getHistoryTextFromProto(vertexPlan.getProcessorDescriptor()));
}
}
ArrayList<Object> inEdgeIdList = new ArrayList<Object>();
inEdgeIdList.addAll(vertexPlan.getInEdgeIdList());
putInto(vertexMap, IN_EDGE_IDS_KEY, inEdgeIdList);
ArrayList<Object> outEdgeIdList = new ArrayList<Object>();
outEdgeIdList.addAll(vertexPlan.getOutEdgeIdList());
putInto(vertexMap, OUT_EDGE_IDS_KEY, outEdgeIdList);
ArrayList<Object> inputsList = new ArrayList<Object>();
for (DAGProtos.RootInputLeafOutputProto input : vertexPlan.getInputsList()) {
Map<String, Object> inputMap = new LinkedHashMap<String, Object>();
inputMap.put(NAME_KEY, input.getName());
inputMap.put(CLASS_KEY, input.getIODescriptor().getClassName());
if (input.hasControllerDescriptor()) {
inputMap.put(INITIALIZER_KEY, input.getControllerDescriptor().getClassName());
}
if (input.getIODescriptor().hasHistoryText()) {
inputMap.put(
USER_PAYLOAD_AS_TEXT,
DagTypeConverters.getHistoryTextFromProto(input.getIODescriptor()));
}
inputsList.add(inputMap);
}
putInto(vertexMap, ADDITIONAL_INPUTS_KEY, inputsList);
ArrayList<Object> outputsList = new ArrayList<Object>();
for (DAGProtos.RootInputLeafOutputProto output : vertexPlan.getOutputsList()) {
Map<String, Object> outputMap = new LinkedHashMap<String, Object>();
outputMap.put(NAME_KEY, output.getName());
outputMap.put(CLASS_KEY, output.getIODescriptor().getClassName());
if (output.hasControllerDescriptor()) {
outputMap.put(INITIALIZER_KEY, output.getControllerDescriptor().getClassName());
}
if (output.getIODescriptor().hasHistoryText()) {
outputMap.put(
USER_PAYLOAD_AS_TEXT,
DagTypeConverters.getHistoryTextFromProto(output.getIODescriptor()));
}
outputsList.add(outputMap);
}
putInto(vertexMap, ADDITIONAL_OUTPUTS_KEY, outputsList);
if (vertexPlan.hasVertexManagerPlugin()) {
vertexMap.put(
VERTEX_MANAGER_PLUGIN_CLASS_KEY, vertexPlan.getVertexManagerPlugin().getClassName());
}
verticesList.add(vertexMap);
}
putInto(dagMap, VERTICES_KEY, verticesList);
ArrayList<Object> edgesList = new ArrayList<Object>();
for (DAGProtos.EdgePlan edgePlan : dagPlan.getEdgeList()) {
Map<String, Object> edgeMap = new LinkedHashMap<String, Object>();
edgeMap.put(EDGE_ID_KEY, edgePlan.getId());
edgeMap.put(INPUT_VERTEX_NAME_KEY, edgePlan.getInputVertexName());
edgeMap.put(OUTPUT_VERTEX_NAME_KEY, edgePlan.getOutputVertexName());
edgeMap.put(DATA_MOVEMENT_TYPE_KEY, edgePlan.getDataMovementType().name());
edgeMap.put(DATA_SOURCE_TYPE_KEY, edgePlan.getDataSourceType().name());
edgeMap.put(SCHEDULING_TYPE_KEY, edgePlan.getSchedulingType().name());
edgeMap.put(EDGE_SOURCE_CLASS_KEY, edgePlan.getEdgeSource().getClassName());
edgeMap.put(EDGE_DESTINATION_CLASS_KEY, edgePlan.getEdgeDestination().getClassName());
if (edgePlan.getEdgeSource().hasHistoryText()) {
edgeMap.put(
OUTPUT_USER_PAYLOAD_AS_TEXT,
DagTypeConverters.getHistoryTextFromProto(edgePlan.getEdgeSource()));
}
if (edgePlan.getEdgeDestination().hasHistoryText()) {
edgeMap.put(
INPUT_USER_PAYLOAD_AS_TEXT,
DagTypeConverters.getHistoryTextFromProto(edgePlan.getEdgeDestination()));
} // TEZ-2286 this is missing edgemanager descriptor for custom edge
edgesList.add(edgeMap);
}
putInto(dagMap, EDGES_KEY, edgesList);
ArrayList<Object> vertexGroupsList = new ArrayList<Object>();
for (DAGProtos.PlanVertexGroupInfo vertexGroupInfo : dagPlan.getVertexGroupsList()) {
Map<String, Object> groupMap = new LinkedHashMap<String, Object>();
groupMap.put(VERTEX_GROUP_NAME_KEY, vertexGroupInfo.getGroupName());
if (vertexGroupInfo.getGroupMembersCount() > 0) {
groupMap.put(VERTEX_GROUP_MEMBERS_KEY, vertexGroupInfo.getGroupMembersList());
}
if (vertexGroupInfo.getOutputsCount() > 0) {
groupMap.put(VERTEX_GROUP_OUTPUTS_KEY, vertexGroupInfo.getOutputsList());
}
if (vertexGroupInfo.getEdgeMergedInputsCount() > 0) {
ArrayList<Object> edgeMergedInputs = new ArrayList<Object>();
for (PlanGroupInputEdgeInfo edgeMergedInputInfo :
vertexGroupInfo.getEdgeMergedInputsList()) {
Map<String, Object> edgeMergedInput = new LinkedHashMap<String, Object>();
edgeMergedInput.put(
VERTEX_GROUP_DESTINATION_VERTEX_NAME_KEY, edgeMergedInputInfo.getDestVertexName());
if (edgeMergedInputInfo.hasMergedInput()
&& edgeMergedInputInfo.getMergedInput().hasClassName()) {
edgeMergedInput.put(
PROCESSOR_CLASS_KEY, edgeMergedInputInfo.getMergedInput().getClassName());
if (edgeMergedInputInfo.getMergedInput().hasHistoryText()) {
edgeMergedInput.put(
USER_PAYLOAD_AS_TEXT,
DagTypeConverters.getHistoryTextFromProto(edgeMergedInputInfo.getMergedInput()));
}
}
edgeMergedInputs.add(edgeMergedInput);
}
groupMap.put(VERTEX_GROUP_EDGE_MERGED_INPUTS_KEY, edgeMergedInputs);
}
vertexGroupsList.add(groupMap);
}
putInto(dagMap, VERTEX_GROUPS_KEY, vertexGroupsList);
return dagMap;
}