public static Result listBackupsFor_JSON(String username, Long jobId) {
Logger.info("Listing backups for job " + jobId);
Job job = Job.findById(jobId);
if (job == null) return notFound();
List<Backup> backups = Backup.findByJob(job);
return ok(Json.toJson(BackupFilter.map(backups)));
}
public static List<Job> findJobsByValidLocation() {
return Job.find("byLocationValid", true).fetch();
}
public static List<Job> findMissingCoordinatesByCompany(Company company) {
return Job.find("latitude < 0 and longitude < 0 and company=?1", company).fetch();
}
public static List<Job> findByCompany(Company company) {
return Job.find("byCompany", company).fetch();
}
public static List<Job> findByJobKey(List<String> jobKeyList) {
return Job.find("key in (:jobkeys)").bind("jobkeys", jobKeyList).fetch();
}
public static Job byId(long id) {
return Job.find("byId", id).first();
}
public static Job findByJobKey(String key) {
return Job.find("byKey", key).first();
}
public static Result createJob(String username) {
// TODO needs to come from the session
User user = getUser(username);
// Get source profile
String sourceProfileId = getFormParam("sourceProfileId");
if (sourceProfileId == null) {
Logger.warn(MISSING_PARAM + "sourceProfileId");
return badRequest(MISSING_PARAM + "sourceProfileId");
}
DatastoreProfile sourceProfile = null;
try {
sourceProfile = DatastoreProfile.find.byId(Long.parseLong(sourceProfileId));
} catch (Throwable t) {
Logger.error(t.getMessage());
}
if (sourceProfile == null) {
Logger.warn(PROFILE_NOT_FOUND + sourceProfileId);
return notFound(PROFILE_NOT_FOUND + sourceProfileId);
}
if (sourceProfile.type != Type.DATASOURCE) {
Logger.warn(NOT_A_SOURCE.replace("@@id@@", sourceProfileId));
return badRequest(NOT_A_SOURCE.replace("@@id@@", sourceProfileId));
}
Logger.info("New job - source " + sourceProfile.pluginClass);
String listOfActions = getFormParam("requiredActions");
if (listOfActions == null) listOfActions = "";
Logger.info("New job - actions " + listOfActions);
// Get sink profile
String sinkProfileId = getFormParam("sinkProfileId");
if (sinkProfileId == null) {
Logger.warn(MISSING_PARAM + "sinkProfileId");
return badRequest(MISSING_PARAM + "sinkProfileId");
}
DatastoreProfile sinkProfile = null;
try {
sinkProfile = DatastoreProfile.find.byId(Long.parseLong(sinkProfileId));
} catch (Throwable t) {
Logger.error(t.getMessage());
}
if (sinkProfile == null) {
Logger.warn(PROFILE_NOT_FOUND + sinkProfileId);
return notFound(PROFILE_NOT_FOUND + sinkProfileId);
}
if (sinkProfile.type != Type.DATASINK) {
Logger.warn(NOT_A_SINK.replace("@@id@@", sinkProfileId));
return badRequest(NOT_A_SINK.replace("@@id@@", sinkProfileId));
}
Logger.info("New job - sink " + sinkProfile.pluginClass);
// Time expression
String timeExpression = getFormParam("timeExpression");
Logger.info("New job - time expression: " + timeExpression);
if (timeExpression == null) return badRequest(MISSING_PARAM + "timeExpression");
// TODO - we start immediately for testing purposes ONLY
Date start = new Date();
// Job Title
String jobTitle = getFormParam("jobTitle");
if (jobTitle == null) return badRequest(MISSING_PARAM + "jobTitle");
Job job =
new Job(
user,
jobTitle,
sourceProfile,
listOfActions.trim(),
sinkProfile,
start,
getDelay(timeExpression));
job.save();
SimpleJobManager.getInstance().queueJob(job);
return ok();
}
/** Common implementations * */
private static List<Job> getJobsFor(String username) {
// TODO needs to come from the session
User user = getUser(username);
return Job.findByUser(user);
}
public static Result deleteJob(String username, Long jobId) {
Job job = Job.findById(jobId);
if (job != null) Job.deleteCascaded(job);
return noContent();
}
/**
* When a StartJobMessage is received, the corresponding Job is extracted and the unprocessed
* instances are sent and received one at a time to the solver. Every time a text annotation is
* processed, JobProcessingActor notifies its parent, the MasterActor, of its progress.
*/
@Override
public void onReceive(Object message) throws Exception {
if (message instanceof SetUpJobMessage) {
ActorRef master = getSender();
SetUpJobMessage jobInfo = (SetUpJobMessage) message;
this.conf_id = jobInfo.getConf_id();
this.record_id = jobInfo.getRecord_id();
this.url = jobInfo.getUrl();
LearnerSettings learnerSettings = jobInfo.getLearnerSettings();
Job job = Core.setUpJob(conf_id, url, record_id);
ClassificationTester eval = new ClassificationTester();
Evaluator evaluator;
String viewName;
try {
evaluator = Core.getEvaluator(conf_id);
viewName = Core.getEvaluatorView(conf_id);
} catch (Exception e) {
master.tell(
new StatusUpdate(0, 0, 0, record_id, eval, "Error receiving evaluator from database"),
getSelf());
Core.storeResultsOfRunInDatabase(eval, record_id, false);
return;
}
if (job.getError() != null) {
master.tell(new StatusUpdate(0, 0, 0, record_id, eval, job.getError()), getSelf());
Core.storeResultsOfRunInDatabase(eval, record_id, false);
return;
}
List<TextAnnotation> unprocessedInstances = job.getUnprocessedInstances();
List<TextAnnotation> goldInstances = job.getGoldInstances();
completed = 0;
skipped = 0;
total = unprocessedInstances.size();
master.tell(new StatusUpdate(completed, skipped, total, record_id, eval, null), getSelf());
System.out.println("Created Job Processor Worker");
System.out.println("Sending and recieving annotations:");
try {
int maxBatchSize = learnerSettings.maxNumInstancesAccepted;
for (int startIndex = 0;
startIndex < unprocessedInstances.size();
startIndex += maxBatchSize) {
int batchSize = Math.min(maxBatchSize, unprocessedInstances.size() - startIndex);
List<TextAnnotation> batch = makeBatch(unprocessedInstances, startIndex, batchSize);
Promise<LearnerInstancesResponse> response = job.sendAndReceiveRequestsFromSolver(batch);
int batchStartIndex = startIndex;
response.onRedeem(
new F.Callback<LearnerInstancesResponse>() {
@Override
public void invoke(LearnerInstancesResponse learnerInstancesResponse)
throws Throwable {
for (int batchIndex = 0; batchIndex < batchSize; batchIndex++) {
if (learnerInstancesResponse.textAnnotations[batchIndex] != null) {
TextAnnotation goldInstance = goldInstances.get(batchStartIndex + batchIndex);
try {
Core.evaluate(
evaluator,
eval,
goldInstance,
learnerInstancesResponse.textAnnotations[batchIndex],
viewName);
} catch (Exception e) {
Core.storeResultsOfRunInDatabase(eval, record_id, false);
master.tell(
new StatusUpdate(
completed,
skipped,
total,
record_id,
eval,
"Error in evaluator. Please check your returned View: " + viewName),
getSelf());
master.tell(new StopRunMessage(record_id), master);
return;
}
completed++;
} else {
skipped++;
}
}
if (completed + skipped < total)
Core.storeResultsOfRunInDatabase(eval, record_id, true);
else Core.storeResultsOfRunInDatabase(eval, record_id, false);
master.tell(
new StatusUpdate(completed, skipped, total, record_id, eval, null),
getSelf());
System.out.println(String.format("Completed batch of size %s", batchSize));
}
});
response.get(learnerTimeout);
if (killCommandHasBeenSent()) {
System.out.println("Exiting");
Core.storeResultsOfRunInDatabase(eval, record_id, false);
break;
}
}
} catch (Exception ex) {
System.out.println("Err sending and receiving text annotations" + ex.getMessage());
master.tell(
new StatusUpdate(
completed,
skipped,
total,
record_id,
eval,
"Error receiving and sending text annotations"),
getSelf());
Core.storeResultsOfRunInDatabase(eval, record_id, false);
ex.printStackTrace();
}
System.out.println("Done");
} else unhandled(message);
}
