/**
* Collect byte-level metrics.
*
* <p>This method is only supposed to be called after the writer commits.
*
* @param bytesWritten number of bytes written by the writer
* @param branchIndex fork branch index
*/
public void updateByteMetrics(long bytesWritten, int branchIndex) {
TaskMetrics metrics = TaskMetrics.get(this);
String forkBranchId = ForkOperatorUtils.getForkId(this.taskId, branchIndex);
metrics.getCounter(MetricGroup.TASK.name(), forkBranchId, BYTES).inc(bytesWritten);
metrics.getMeter(MetricGroup.TASK.name(), forkBranchId, BYTES_PER_SECOND).mark(bytesWritten);
metrics.getCounter(MetricGroup.JOB.name(), this.jobId, BYTES).inc(bytesWritten);
metrics.getMeter(MetricGroup.JOB.name(), this.jobId, BYTES_PER_SECOND).mark(bytesWritten);
}
/**
* Update record-level metrics.
*
* @param recordsWritten number of records written by the writer
* @param branchIndex fork branch index
*/
public void updateRecordMetrics(long recordsWritten, int branchIndex) {
TaskMetrics metrics = TaskMetrics.get(this);
String forkBranchId = ForkOperatorUtils.getForkId(this.taskId, branchIndex);
Counter taskRecordCounter =
metrics.getCounter(gobblin.runtime.util.MetricGroup.TASK.name(), forkBranchId, RECORDS);
long inc = recordsWritten - taskRecordCounter.getCount();
taskRecordCounter.inc(inc);
metrics.getMeter(MetricGroup.TASK.name(), forkBranchId, RECORDS_PER_SECOND).mark(inc);
metrics.getCounter(MetricGroup.JOB.name(), this.jobId, RECORDS).inc(inc);
metrics.getMeter(MetricGroup.JOB.name(), this.jobId, RECORDS_PER_SECOND).mark(inc);
}
/**
* Adjust job-level metrics when the task gets retried.
*
* @param branches number of forked branches
*/
public void adjustJobMetricsOnRetry(int branches) {
TaskMetrics metrics = TaskMetrics.get(this);
for (int i = 0; i < branches; i++) {
String forkBranchId = ForkOperatorUtils.getForkId(this.taskId, i);
long recordsWritten =
metrics.getCounter(MetricGroup.TASK.name(), forkBranchId, RECORDS).getCount();
long bytesWritten =
metrics.getCounter(MetricGroup.TASK.name(), forkBranchId, BYTES).getCount();
metrics.getCounter(MetricGroup.JOB.name(), this.jobId, RECORDS).dec(recordsWritten);
metrics.getCounter(MetricGroup.JOB.name(), this.jobId, BYTES).dec(bytesWritten);
}
}
/**
* Convert this {@link TaskState} instance to a {@link TaskExecutionInfo} instance.
*
* @return a {@link TaskExecutionInfo} instance
*/
public TaskExecutionInfo toTaskExecutionInfo() {
TaskExecutionInfo taskExecutionInfo = new TaskExecutionInfo();
taskExecutionInfo.setJobId(this.jobId);
taskExecutionInfo.setTaskId(this.taskId);
if (this.startTime > 0) {
taskExecutionInfo.setStartTime(this.startTime);
}
if (this.endTime > 0) {
taskExecutionInfo.setEndTime(this.endTime);
}
taskExecutionInfo.setDuration(this.duration);
taskExecutionInfo.setState(TaskStateEnum.valueOf(getWorkingState().name()));
if (this.contains(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY)) {
taskExecutionInfo.setFailureException(
this.getProp(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY));
}
taskExecutionInfo.setHighWatermark(this.getHighWaterMark());
// Add extract/table information
Table table = new Table();
Extract extract = this.getExtract();
table.setNamespace(extract.getNamespace());
table.setName(extract.getTable());
if (extract.hasType()) {
table.setType(TableTypeEnum.valueOf(extract.getType().name()));
}
taskExecutionInfo.setTable(table);
// Add task metrics
TaskMetrics taskMetrics = TaskMetrics.get(this);
MetricArray metricArray = new MetricArray();
for (Map.Entry<String, ? extends com.codahale.metrics.Metric> entry :
taskMetrics.getMetricContext().getCounters().entrySet()) {
Metric counter = new Metric();
counter.setGroup(MetricGroup.TASK.name());
counter.setName(entry.getKey());
counter.setType(MetricTypeEnum.valueOf(GobblinMetrics.MetricType.COUNTER.name()));
counter.setValue(Long.toString(((Counter) entry.getValue()).getCount()));
metricArray.add(counter);
}
for (Map.Entry<String, ? extends com.codahale.metrics.Metric> entry :
taskMetrics.getMetricContext().getMeters().entrySet()) {
Metric meter = new Metric();
meter.setGroup(MetricGroup.TASK.name());
meter.setName(entry.getKey());
meter.setType(MetricTypeEnum.valueOf(GobblinMetrics.MetricType.METER.name()));
meter.setValue(Double.toString(((Meter) entry.getValue()).getMeanRate()));
metricArray.add(meter);
}
for (Map.Entry<String, ? extends com.codahale.metrics.Metric> entry :
taskMetrics.getMetricContext().getGauges().entrySet()) {
Metric gauge = new Metric();
gauge.setGroup(MetricGroup.TASK.name());
gauge.setName(entry.getKey());
gauge.setType(MetricTypeEnum.valueOf(GobblinMetrics.MetricType.GAUGE.name()));
gauge.setValue(((Gauge) entry.getValue()).getValue().toString());
metricArray.add(gauge);
}
taskExecutionInfo.setMetrics(metricArray);
// Add task properties
Map<String, String> taskProperties = Maps.newHashMap();
for (String name : this.getPropertyNames()) {
String value = this.getProp(name);
if (!Strings.isNullOrEmpty(value)) taskProperties.put(name, value);
}
taskExecutionInfo.setTaskProperties(new StringMap(taskProperties));
return taskExecutionInfo;
}
