@Override
public void run() {
try {
StopWatch stp2 = new StopWatch();
stp2.start();
JSONObject json = new JSONObject();
job.setStatus("running");
if (job.progress() == 100.0) {
finalizeJob(job);
return;
}
Vector<String> ids = new Vector<String>();
Vector<String> original_names = new Vector<String>();
String data = job.getNextDataBatch();
if (data == null || data.equals("")) return;
String[] lines = data.split("\n");
if (job.containsId()) {
for (int i = 0; i < lines.length; i++) {
if (lines[i].trim().equals("")) continue;
ids.add(NameUtil.getNameId(lines[i]));
}
}
for (int i = 0; i < lines.length; i++) {
original_names.add(NameUtil.processName(lines[i], job.containsId()));
}
String names = NameUtil.CleanNames(lines, job);
if (names.equals("")) return;
if (job.getType() == TnrsJob.NAME_MATCH_JOB) {
TaxamatchInterface taxa_match = new TaxamatchInterface(tnrsBaseUrl);
String result = taxa_match.queryTaxamatch(names, job);
json = (JSONObject) JSONSerializer.toJSON(result);
} else if (job.getType() == TnrsJob.PARSING_JOB) {
json = gni_interface.parseNames(names);
}
if (job.outstandingNames() == 0) {
JobHelper.persistJobInfo(baseFolder, job);
}
saveResults(job, json, ids, original_names, "");
job.setStatus("idle");
stp2.stop();
log.info("overall :" + stp2.toString());
} catch (Exception ex) {
log.error(ExceptionUtils.getFullStackTrace(ex));
job.setStatus("failed");
ex.printStackTrace();
}
}
public void sendSubmissionEmail(TnrsJob job) throws Exception {
String job_type = "";
if (job.getType() == TnrsJob.NAME_MATCH_JOB) {
job_type = " matching ";
} else {
job_type = " parsing ";
}
Email response = new HtmlEmail();
response.setHostName("localhost");
response.setSmtpPort(25);
response.setFrom("*****@*****.**");
response.setSubject("TNRS Job submission");
response.setMsg(
"Your TNRS "
+ job_type
+ " job ("
+ job.getRequest().getOriginalFilename()
+ ") was successfully submitted on "
+ dateFormat.format(new Date())
+ ". \n\n"
+ "When your list is done processing, you will receive an email notification that contains instructions regarding retrieval of your results from the TNRS website at http://tnrs.iplantcollaborative.org/.\n\n"
+ "Please contact us at [email protected] if you have any difficulty retrieving your results. \n"
+ "\n\n You can check the status of your job in the 'Retrieve results' tab of the application using your email and the following key: "
+ job.getRequest().getId()
+ ".\n\nTo update the status for your job in progress, please select the 'Retrieve results' button again. Your progress will update at this time.\n\n"
+ "Thank you, \n"
+ "iPlant Collaborative");
response.addTo(job.getRequest().getEmail());
response.send();
}
public void finalizeJob(TnrsJob job) throws Exception {
synchronized (jobs) {
jobs.remove(job);
}
job.setStatus("complete");
if (job.email() && !job.getRequest().getEmail().trim().equals("*****@*****.**")) {
job.setFinishedAt(new Date().toString());
job_ids.remove(job.getRequest().getId());
sendNormalCompletionEmail(job);
}
}
@Override
public void handle(HttpExchange arg0) throws IOException {
try {
JSONObject json =
(JSONObject) JSONSerializer.toJSON(IOUtils.toString(arg0.getRequestBody()));
JSONObject result = new JSONObject();
String email = json.getString("email");
String key = json.getString("key");
for (int i = 0; i < jobs.size(); i++) {
TnrsJob job = jobs.get(i);
if (job.getRequest().getEmail().equals(email) && job.getRequest().getId().equals(key)) {
if (job.status().equals("failed") || job.status().equals("error")) {
result.put("type", "failed");
} else {
result.put("type", "incomplete");
double progress = job.progress();
if (job.progress() == 100.0) {
progress = 99.0;
}
result.put("progress", progress);
}
HandlerHelper.writeResponseRequest(arg0, 200, result.toString(), "application/json");
return;
}
}
String filename = baseFolder + email.replace("@", "-").replace(".", "-") + "/result" + key;
File results = new File(filename);
if (!results.exists()) {
result.put("type", "non-existent");
} else {
result.put("type", "complete");
TnrsJob job = JobHelper.readJobInfo(baseFolder, email, key);
result.put("job_type", job.getTypeString());
}
HandlerHelper.writeResponseRequest(arg0, 200, result.toString(), "application/json");
return;
} catch (Exception ex) {
log.error(ExceptionUtils.getFullStackTrace(ex));
throw new IOException(ex);
}
}
/**
* This main loop iterates over the submitted jobs and creates the corresponding threads that will
* carry the execution of each job segment.
*/
@Override
public void run() {
try {
int jobno = 0;
while (true) {
sleep(20);
synchronized (jobs) {
if (jobs.size() == 0) {
continue;
}
}
TnrsJob job = jobs.get(jobno % jobs.size());
if (job.status().equalsIgnoreCase("idle")) {
ExecutionThread thread = new ExecutionThread(job);
threads.add(thread);
thread.start();
} else if (job.status().equalsIgnoreCase("stopped")) {
jobs.remove(job);
JobHelper.cleanJobData(baseFolder, job);
jobno = 0;
continue;
} else if (job.status().equals("failed")) {
sendFailedJobEmail(job);
job.setStatus("error");
}
jobno++;
if (jobs.size() == 200) {
for (int i = 0; i < threads.size(); i++) {
threads.elementAt(i).join();
}
jobno = 0;
threads.clear();
}
sleep(10);
}
} catch (Exception e) {
log.error(ExceptionUtils.getFullStackTrace(e));
e.printStackTrace();
}
}
@Override
public void handle(HttpExchange arg0) throws IOException {
try {
JSONObject request =
(JSONObject) JSONSerializer.toJSON(IOUtils.toString(arg0.getRequestBody()));
String email = request.getString("email");
String key = request.getString("key");
for (int i = 0; i < jobs.size(); i++) {
TnrsJob job = jobs.get(i);
if (job.getRequest().getId().equals(key) && job.getRequest().getEmail().equals(email)) {
JSONObject json = (JSONObject) JSONSerializer.toJSON(job.toJsonString());
json.put("status", "incomplete");
json.put("progress", job.progress());
HandlerHelper.writeResponseRequest(arg0, 200, json.toString(), "application/json");
return;
}
}
if (JobHelper.jobFileExists(baseFolder, email, key)) {
TnrsJob job = JobHelper.readJobInfo(baseFolder, email, key);
HandlerHelper.writeResponseRequest(arg0, 200, job.toJsonString(), "application/json");
} else {
HandlerHelper.writeResponseRequest(
arg0, 500, "No such job exists o it might have expired", "text/plain");
}
} catch (Exception ex) {
log.error(ExceptionUtils.getFullStackTrace(ex));
throw new IOException(ex);
}
}
@Override
public void handle(HttpExchange arg0) throws IOException {
try {
String message =
"<html><body><table border=\"1\"><tr><th>Email</th><th>Submitted at:</th><th>Progress:</th><th>Status:</th><th>Job id</th></tr>";
for (int i = 0; i < jobs.size(); i++) {
TnrsJob job = jobs.get(i);
message +=
"<tr> <td> "
+ job.getRequest().getEmail()
+ "</td><td>"
+ job.getSubmissionDate()
+ " </td><td> "
+ job.progress()
+ "% </td><td> "
+ job.status()
+ "</td><td>"
+ job.getRequest().getId()
+ "</td></tr>\n";
}
message += "</table>";
HandlerHelper.writeResponseRequest(arg0, 200, message, "text/html");
} catch (Exception ex) {
log.error(ExceptionUtils.getFullStackTrace(ex));
throw new IOException(ex);
}
}
private void sendNormalCompletionEmail(TnrsJob job) throws Exception {
EmailAttachment attachment = new EmailAttachment();
attachment.setPath(JobHelper.createJobInfoFile(job));
attachment.setDisposition(EmailAttachment.ATTACHMENT);
attachment.setDescription("Job information");
attachment.setName(job.getRequest().getFilename() + " " + job.getRequest().getId() + ".txt");
MultiPartEmail response = new MultiPartEmail();
response.setHostName(properties.getProperty("org.iplantc.tnrs.mail.host"));
response.setSmtpPort(Integer.parseInt(properties.getProperty("org.iplantc.tnrs.mail.port")));
response.setFrom("*****@*****.**");
response.setSubject("TNRS Job completion");
response.setMsg(
"Your TNRS "
+ job.getTypeString()
+ " job for the file "
+ job.getRequest().getOriginalFilename()
+ " completed on "
+ dateFormat.format(new Date())
+ ".Details describing the settings applied for this job are attached to this email. You may wish to retain this for your records. \n\n Your results will be available for 7 days, after which they will be deleted from our system. \n\n"
+ "To view your results, go to the TNRS website at http://tnrs.iplantcollaborative.org, select the \"Retrieve results\" tab, and enter your email address and the following submission key:\n\n"
+ job.getRequest().getId()
+ "\n\n"
+ "Please contact us at [email protected] if you have any difficulty retrieving your results.\n"
+ "\n"
+ "Thank you,\n"
+ "iPlant Collaborative");
response.addTo(job.getRequest().getEmail());
response.attach(attachment);
response.send();
}
@Override
public void handle(HttpExchange arg0) throws IOException {
try {
JSONObject json = new JSONObject();
JSONArray array = new JSONArray();
for (int i = 0; i < jobs.size(); i++) {
JSONObject job_info = new JSONObject();
TnrsJob job = jobs.get(i);
job_info.put("email", job.getRequest().getEmail());
job_info.put("progress", job.progress());
array.add(job_info);
}
json.put("jobs", array);
HandlerHelper.writeResponseRequest(arg0, 200, json.toString(), "application/json");
} catch (Exception ex) {
log.error(ExceptionUtils.getFullStackTrace(ex));
throw new IOException(ex);
}
}
@Override
public void handle(HttpExchange arg0) throws IOException {
try {
JSONObject json =
(JSONObject) JSONSerializer.toJSON(IOUtils.toString(arg0.getRequestBody()));
String email = json.getString("email");
String key = json.getString("key");
String session_id = json.getString("session_id");
TnrsJob job = JobHelper.readJobInfo(baseFolder, email, key);
if (job.getType() == TnrsJob.PARSING_JOB) {
ParsingResultsFile results = new ParsingResultsFile(job, baseFolder);
results.createFileForDownload(properties.getProperty("org.iplantc.folder.tmp"));
} else {
MatchingResultsFile results = new MatchingResultsFile(job, baseFolder, session_id, false);
results.createFileForDownload(
properties.getProperty("org.iplantc.tnrs.folder.tmp"), json);
results.close();
}
String url = servicesUrl + "getcsv?id=" + key;
HandlerHelper.writeResponseRequest(arg0, 200, url, null);
} catch (Exception ex) {
log.error(ExceptionUtils.getFullStackTrace(ex));
ex.printStackTrace();
}
}
private void sendFailedJobEmail(TnrsJob job) throws Exception {
if (!job.email()) return;
HtmlEmail response = new HtmlEmail();
response.setHostName("localhost");
response.setSmtpPort(25);
response.setFrom("*****@*****.**");
response.setSubject("TNRS Job failure");
response.setMsg(
"Your TNRS "
+ job.getTypeString()
+ " job for the file "
+ job.getRequest().getOriginalFilename()
+ " failed on "
+ dateFormat.format(new Date())
+ ". Please provide the job key and if possible the name list you were runing so we can better help you diagnose the issue. \n\n"
+ "The job key is:"
+ job.getRequest().getId()
+ "\n\n"
+ "Thank you,\n"
+ "iPlant Collaborative");
response.addTo(job.getRequest().getEmail());
response.send();
}
@Override
public void handle(HttpExchange arg0) throws IOException {
try {
JSONObject command =
(JSONObject) JSONSerializer.toJSON(IOUtils.toString(arg0.getRequestBody()));
String job_id = command.getString("job_id");
TnrsJob job = null;
for (TnrsJob jobc : jobs) {
if (jobc.getRequest().getId().equals(job_id)) {
job = jobc;
break;
}
}
if (job == null) {
HandlerHelper.writeResponseRequest(arg0, 500, "Invalid job identifier", "text/plain");
}
if (command.getString("command").equals("stop")) {
synchronized (job) {
job.disable();
}
} else if (command.getString("command").equals("pause")) {
synchronized (job) {
job.pause();
}
} else if (command.getString("command").equals("resume")) {
synchronized (job) {
job.resume();
}
} else {
HandlerHelper.writeResponseRequest(arg0, 500, "Wrong command", "text/plain");
return;
}
HandlerHelper.writeResponseRequest(arg0, 200, "", "text/plain");
} catch (Exception ex) {
log.error(ExceptionUtils.getFullStackTrace(ex));
}
}
private void saveResults(
TnrsJob job,
JSONObject results,
Vector<String> ids,
Vector<String> original_names,
String session_id)
throws Exception {
if (job.getType() == TnrsJob.PARSING_JOB) {
ParsingResultsFile csv = new ParsingResultsFile(job, baseFolder);
csv.writeJsonData(results.getJSONArray("parsedNames"), ids);
} else {
MatchingResultsFile csv = new MatchingResultsFile(job, baseFolder, session_id, false);
TNRSResultsTransformer transformer = new TNRSResultsTransformer();
JSONArray results_array = transformer.transform(results, job, ids, original_names);
csv.writeJsonData(results_array, ids);
csv.close();
}
}
public TnrsJob submitJob(String jsons) throws Exception {
JSONObject json = (JSONObject) JSONSerializer.toJSON(jsons);
boolean emailr = false;
TnrsJobRequest request =
new TnrsJobRequest(
json.getString("email"),
json.getString("file_name"),
json.getString("original"),
emailr);
TnrsJob job = new TnrsJob(request, new Date().toString(), json.getString("type"));
job.setSensitivity(Double.parseDouble(json.getString("sensitivity").toLowerCase().trim()));
job.setTnrs_version(properties.getProperty("org.iplantc.tnrs.version"));
synchronized (jobs) {
jobs.add(job);
}
job_ids.put(request.getId(), job);
job.setContainsId(json.getBoolean("has_id"));
job.setStatus("idle");
job.getRequest().setId(json.getString("id"));
return job;
}
@Override
public void handle(HttpExchange arg0) throws IOException {
try {
String request = IOUtils.toString(arg0.getRequestBody());
JSONObject datas = (JSONObject) JSONSerializer.toJSON(request);
UUID uid = UUID.randomUUID();
HashMap<String, String> info = new HashMap<String, String>();
info.put("email", datas.getString("email"));
info.put("institution", "");
info.put("name", "");
info.put("sensitivity", datas.getString("sensitivity"));
info.put("has_id", datas.getString("has_id"));
info.put("id", uid.toString().replace("-", ""));
info.put("type", datas.getString("type"));
info.put("sources", datas.getString("sources"));
info.put("classification", datas.getString("classification"));
info.put("match_to_rank", datas.getString("match_to_rank"));
/**
* *
*
* <p>To be replaced by better code
*/
byte[] data = null;
if (datas.has("upload")) {
data =
IOUtils.toByteArray(
new GZIPInputStream(
new Base64InputStream(
new ByteArrayInputStream(datas.getString("upload").getBytes()))));
} else if (datas.has("names")) {
data = datas.getString("names").getBytes();
} else {
HandlerHelper.writeResponseRequest(arg0, 500, "Invalid request!", null);
return;
}
UniversalDetector detector = new UniversalDetector(null);
detector.handleData(data, 0, data.length);
detector.dataEnd();
String content = "";
String encoding = detector.getDetectedCharset();
if (encoding != null) {
if (encoding.equals("WINDOWS-1252")) {
encoding = "ISO-8859-1";
ByteArrayOutputStream t = new ByteArrayOutputStream();
OutputStreamWriter wr = new OutputStreamWriter(t, "UTF-8");
wr.write(new String(data, encoding));
wr.close();
content = t.toString("UTF-8");
} else {
content = new String(data, "UTF-8");
}
} else {
content = new String(data, "UTF-8");
}
info.put("upload", content);
/** */
JSONObject json = createJSONJobRequest(info);
json.put("submitted_date", new Date().toString().replace(":", ""));
json.put("original", datas.getString("file_name"));
json.put("sensitivity", datas.getString("sensitivity"));
TnrsJob job = submitJob(json.toString());
job.setTaxonomic(datas.getBoolean("taxonomic"));
job.setEmail(!info.containsKey("noemail"));
job.setSources(info.get("sources"));
job.setClassification(info.get("classification"));
job.setAllowPartial(Boolean.parseBoolean(info.get("match_to_rank")));
log.info(job.getRequest().getId() + " " + job.getRequest().getEmail());
HandlerHelper.writeResponseRequest(arg0, 200, job.getRequest().getId(), null);
if (info.get("email").trim().equals("*****@*****.**") || !job.email()) return;
try {
sendSubmissionEmail(job);
} catch (Exception ex) {
log.error(ExceptionUtils.getFullStackTrace(ex));
ex.printStackTrace();
}
} catch (Exception ex) {
log.error(ExceptionUtils.getFullStackTrace(ex));
ex.printStackTrace();
throw new IOException(ex);
}
}