public void addJobState(JobState jobState) {
if (jobStates.containsKey(jobState.getJob().getId().id)) {
throw new IllegalStateException(String.format("We already processing %s", jobState));
}
this.jobStates.put(jobState.getJob().getId().id, jobState);
}
@Test(dependsOnMethods = "testPersistJobState")
public void testGetJobState() throws IOException {
JobState jobState =
this.fsDatasetStateStore.get(
TEST_JOB_NAME,
FsDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX
+ FsDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX,
TEST_JOB_ID);
Assert.assertEquals(jobState.getJobName(), TEST_JOB_NAME);
Assert.assertEquals(jobState.getJobId(), TEST_JOB_ID);
Assert.assertEquals(jobState.getState(), JobState.RunningState.COMMITTED);
Assert.assertEquals(jobState.getProp("foo"), "bar");
Assert.assertEquals(jobState.getStartTime(), this.startTime);
Assert.assertEquals(jobState.getEndTime(), this.startTime + 1000);
Assert.assertEquals(jobState.getDuration(), 1000);
Assert.assertEquals(jobState.getCompletedTasks(), 3);
for (int i = 0; i < jobState.getCompletedTasks(); i++) {
TaskState taskState = jobState.getTaskStates().get(i);
Assert.assertEquals(taskState.getJobId(), TEST_JOB_ID);
Assert.assertEquals(taskState.getTaskId(), TEST_TASK_ID_PREFIX + i);
Assert.assertEquals(taskState.getId(), TEST_TASK_ID_PREFIX + i);
Assert.assertEquals(taskState.getWorkingState(), WorkUnitState.WorkingState.COMMITTED);
}
}
@Override
public WorkerState clone() {
WorkerState state = new WorkerState();
for (JobState jobState : jobStates.values()) {
state.addJobState(jobState.clone());
}
return state;
}
private void logIfJobIsCompleted(JobInstance jobInstance) {
JobState currentState = getCurrentState(jobInstance.getId());
if (currentState.isCompleted()) {
String message =
String.format(
"State change for a completed Job is not allowed. Job %s is currently State=%s, Result=%s",
jobInstance.getIdentifier(), jobInstance.getState(), jobInstance.getResult());
LOG.warn(message, new Exception().fillInStackTrace());
}
}
public synchronized void update(JobState prevState, QueueJobInfo info) {
if (LogMgr.getInstance().isLoggable(LogMgr.Kind.Ops, LogMgr.Level.Finest)) {
LogMgr.getInstance()
.log(
LogMgr.Kind.Ops,
LogMgr.Level.Finest,
"Job Count Update ["
+ info.getJobID()
+ "]: "
+ prevState
+ " -> "
+ info.getState());
}
// if(LogMgr.getInstance().isLoggable(LogMgr.Kind.Ops, LogMgr.Level.Finest)) {
// StringBuilder buf = new StringBuilder();
// buf.append("Job Pre-Counts [" + info.getJobID() + "]: " +
// prevState + " -> " + info.getState() + "\n ");
// for(JobState js : JobState.all())
// buf.append(js + "[" + pCounts[js.ordinal()] + "] ");
// LogMgr.getInstance().log(LogMgr.Kind.Ops, LogMgr.Level.Finest, buf.toString());
// }
if (prevState != null) {
if (pCounts[prevState.ordinal()] > 0) {
pCounts[prevState.ordinal()]--;
} else {
LogMgr.getInstance()
.logAndFlush(
LogMgr.Kind.Ops,
LogMgr.Level.Warning,
"Somehow the count of jobs with a "
+ prevState
+ " state was already "
+ "when attempting to decrement the count after a change to a "
+ info.getState()
+ " state for the job ("
+ info.getJobID()
+ ")!");
}
}
pCounts[info.getState().ordinal()]++;
// if(LogMgr.getInstance().isLoggable(LogMgr.Kind.Ops, LogMgr.Level.Finest)) {
// StringBuilder buf = new StringBuilder();
// buf.append("Job Post-Counts [" + info.getJobID() + "]: " +
// prevState + " -> " + info.getState() + "\n ");
// for(JobState js : JobState.all())
// buf.append(js + "[" + pCounts[js.ordinal()] + "] ");
// LogMgr.getInstance().log(LogMgr.Kind.Ops, LogMgr.Level.Finest, buf.toString());
// }
}
public String dumpJobStatesIds() {
StringBuilder sb = new StringBuilder();
ArrayList<Id> keys = new ArrayList<>(jobStates.keySet());
Collections.sort(keys);
sb.append(keys.size());
for (Id jobId : keys) {
sb.append(",");
JobState jobState = jobStates.get(jobId);
sb.append(jobState.getStatus().toString().charAt(0));
sb.append("-job-");
sb.append(jobState.getJob().getId().toShortString());
}
return sb.toString();
}
@Test
public void testPersistJobState() throws IOException {
JobState jobState = new JobState(TEST_JOB_NAME, TEST_JOB_ID);
jobState.setId(TEST_JOB_ID);
jobState.setProp("foo", "bar");
jobState.setState(JobState.RunningState.COMMITTED);
jobState.setStartTime(this.startTime);
jobState.setEndTime(this.startTime + 1000);
jobState.setDuration(1000);
for (int i = 0; i < 3; i++) {
TaskState taskState = new TaskState();
taskState.setJobId(TEST_JOB_ID);
taskState.setTaskId(TEST_TASK_ID_PREFIX + i);
taskState.setId(TEST_TASK_ID_PREFIX + i);
taskState.setWorkingState(WorkUnitState.WorkingState.COMMITTED);
jobState.addTaskState(taskState);
}
this.fsJobStateStore.put(
TEST_JOB_NAME,
FsDatasetStateStore.CURRENT_DATASET_STATE_FILE_SUFFIX
+ FsDatasetStateStore.DATASET_STATE_STORE_TABLE_SUFFIX,
jobState);
}
private JobState getCurrentState(long jobId) {
String state = (String) getSqlMapClientTemplate().queryForObject("currentJobState", jobId);
if (state == null) {
return JobState.Unknown;
}
return JobState.valueOf(state);
}
/** Get the distribution of job states for the jobs in the given group. */
public double[] getDistribution(TaskTimer timer, long groupID) {
Counters counters = null;
timer.acquire();
synchronized (pCountersByGroup) {
timer.resume();
counters = pCountersByGroup.get(groupID);
}
if (counters != null) {
double dist[] = counters.distribution();
// if(LogMgr.getInstance().isLoggable(LogMgr.Kind.Ops, LogMgr.Level.Finest)) {
// StringBuilder buf = new StringBuilder();
// buf.append("Job Group Distribution [" + groupID + "]:\n ");
// for(JobState js : JobState.all())
// buf.append(js + "[" + String.format("%1$.4f", dist[js.ordinal()]) + "] ");
// LogMgr.getInstance().log(LogMgr.Kind.Ops, LogMgr.Level.Finest, buf.toString());
// }
return dist;
} else {
LogMgr.getInstance()
.logAndFlush(
LogMgr.Kind.Ops,
LogMgr.Level.Warning,
"Somehow the job group (" + groupID + ") was not in the state counts table!");
return new double[JobState.all().size()];
}
}
public boolean setState(JobState state, boolean force) {
JobState curr = getState();
if (force
|| (isEnabled() && curr.canTransition(state))
|| (!isEnabled() && (state == JobState.IDLE))
|| (!isEnabled() && (state == JobState.ERROR))) {
// Note dependence on ordering!
this.state = state.ordinal();
return true;
} else if (state != curr) {
log.warn(
"[job.setstate] {}job {} cannot transition {} -> {}",
(disabled) ? "disabled " : "",
getId(),
curr,
state);
for (StackTraceElement elt : Thread.currentThread().getStackTrace()) {
log.warn(elt.toString());
}
return false;
}
return true;
}
protected void addMessage(String p_message) {
m_jobState.setCurrentOperation(p_message);
}
@Override
public JobState getState() {
JobState jobState = JobState.makeState(state);
return jobState == null ? JobState.UNKNOWN : jobState;
}
/** Returns last progress message. */
public String progress_msg() {
return isStopped() ? _state.toString() : progress_msg_impl();
}
public Counters(long total) {
pCounts = new long[JobState.all().size()];
pTotal = ((double) total);
}
