@Override
public String executeProcess(String sosEndpoint, RunMetadata metadata) {
if (StringUtils.isBlank(sosEndpoint)) {
throw new IllegalArgumentException("sosEndpoint parameter may not be null or blank");
}
WPSTask task = new WPSTask(sosEndpoint, metadata.toKeyValueMap());
task.start();
if (task.isAlive()) {
return "ok";
} else {
return "not started";
}
}
private void sendFailedEmail(Exception ex, String to) {
String subject = "WaterSMART processing failed";
StringBuilder content = new StringBuilder();
content.append("Your request unfortunately failed, we are looking into it.");
content.append("\n\tUpload: ").append((uploadSuccessful) ? "success" : "failure");
content.append("\n\tParse: ").append((netcdfSuccessful) ? "success" : "failure");
content.append("\n\tStatistics: ").append((rStatsSuccessful) ? "success" : "failure");
content.append("\n\tMetadata: ").append((cswTransSuccessful) ? "success" : "failure");
RunMetadata metaObj = RunMetadata.getInstance(metadata);
content.append("\n\n\tFile: ").append(metaObj.getFileName());
content.append("\n\tModeler: ").append(metaObj.getName());
content.append("\n\tComments: ").append(metaObj.getComments());
content.append("\n\tDate: ").append(metaObj.getCreationDate());
content.append("\n\nthe application failed with message: ").append(ex.getMessage());
content.append("\n\nhere is the stack trace for troubleshooting:\n\n");
for (StackTraceElement el : ex.getStackTrace()) {
content.append(el.toString()).append("\n");
}
List<String> bcc = new ArrayList<String>();
String from = props.getProperty("watersmart.email.from");
String bccAddr = props.getProperty("watersmart.email.tracker");
if (!"".equals(bccAddr)) {
bcc.add(bccAddr);
}
EmailMessage message = new EmailMessage(from, to, null, bcc, subject, content.toString());
try {
EmailHandler.sendMessage(message);
} catch (AddressException ex1) {
log.error(
"Unable to send failed e-mail:\n"
+ message
+ "\n\nOriginal Exception:\n"
+ ex.getMessage(),
ex1);
} catch (MessagingException ex1) {
log.error(
"Unable to send failed e-mail:\n"
+ message
+ "\n\nOriginal Exception:\n"
+ ex.getMessage(),
ex1);
}
}
private void sendCompleteEmail(Map<String, String> outputs, String to) {
String subject = "Processing Complete";
StringBuilder content = new StringBuilder();
content
.append("Your upload has finished conversion and processing,")
.append(" you may view the results of the processing by going to:\n");
for (String alg : outputs.keySet()) {
content.append("\t").append(alg).append(": ").append(outputs.get(alg)).append("\n");
}
content.append("\nor return to the application to view your upload.");
content.append("\nJust to double check, here is what happened");
content.append("\n\tUpload: ").append((uploadSuccessful) ? "success" : "failure");
content.append("\n\tParse: ").append((netcdfSuccessful) ? "success" : "failure");
content.append("\n\tStatistics: ").append((rStatsSuccessful) ? "success" : "failure");
content.append("\n\tMetadata: ").append((cswTransSuccessful) ? "success" : "failure");
RunMetadata metaObj = RunMetadata.getInstance(metadata);
content.append("\n\n\tFile: ").append(metaObj.getFileName());
content.append("\n\tModeler: ").append(metaObj.getName());
content.append("\n\tComments: ").append(metaObj.getComments());
content.append("\n\tDate: ").append(metaObj.getCreationDate());
content.append("\n\nHave a nice day!");
List<String> bcc = new ArrayList<String>();
String from = props.getProperty("watersmart.email.from");
String bccAddr = props.getProperty("watersmart.email.tracker");
if (!"".equals(bccAddr)) {
bcc.add(bccAddr);
}
EmailMessage message = new EmailMessage(from, to, null, bcc, subject, content.toString());
try {
EmailHandler.sendMessage(message);
} catch (AddressException ex) {
log.error("Unable to send completed e-mail:\n" + message, ex);
} catch (MessagingException ex) {
log.error("Unable to send completed e-mail:\n" + message, ex);
}
}
@Override
public String executeProcess(File zipLocation, RunMetadata metadata) {
return executeProcess(zipLocation, metadata.toKeyValueMap());
}
private void sendFailedEmail(Exception ex) {
sendFailedEmail(ex, RunMetadata.getInstance(metadata).getEmail());
}
private void sendCompleteEmail(Map<String, String> outputs) {
sendCompleteEmail(outputs, RunMetadata.getInstance(metadata).getEmail());
}
/**
* @param alg Algorithm to run
* @param wpsRequest XML representing WPS request
* @param uuid UUID for output
* @param metaObj RunMetadata associated with user
* @return Web Accessible File with process results
* @throws IOException
* @throws ParserConfigurationException
* @throws SAXException
* @throws XPathExpressionException
* @throws InterruptedException
*/
private String runNamedAlgorithm(String alg, String wpsRequest, UUID uuid, RunMetadata metaObj)
throws IOException, ParserConfigurationException, SAXException, XPathExpressionException,
InterruptedException {
// String wpsRequest = WPSImpl.createNahatStatsRequest(sosEndpoint, info.stations,
// info.properties);
String wpsResponse = postToWPS(props.getProperty("watersmart.wps.url"), wpsRequest);
log.debug(wpsResponse);
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
factory.setNamespaceAware(true);
Document wpsResponseDoc =
factory.newDocumentBuilder().parse(new ByteArrayInputStream(wpsResponse.getBytes()));
ProcessStatus procStat = new ProcessStatus(wpsResponseDoc);
String wpsCheckPoint = procStat.getStatusLocation();
log.debug(wpsCheckPoint);
String contextPath = props.getProperty("watersmart.external.mapping.url");
InputStream is = null;
InputStream resultIs = null;
try {
boolean completed = false;
Document document = null;
int checks = 0;
while (!completed) {
checks++;
// TODO-
// http://stackoverflow.com/questions/3535754/netbeans-java-new-hint-thread-sleep-called-in-loop
Thread.sleep(CHECK_WAIT);
log.debug("Checking: " + checks);
is = HTTPUtils.sendPacket(new URL(wpsCheckPoint), "GET");
document = CheckProcessCompletion.parseDocument(is);
completed = checkWPSProcess(document);
if (checks == CHECKS_UNTIL_NOTIFY) {
sendMaybeEmail(metaObj.getEmail());
}
if (checks > CHECKS_UNTIL_FAIL) {
throw new IOException("R Statistics never returned");
}
}
ProcessStatus resultStatus = new ProcessStatus(document);
String outputReference = resultStatus.getOutputReference();
resultIs = HTTPUtils.sendPacket(new URL(outputReference), "GET");
String resultStr = IOUtils.toString(resultIs, "UTF-8");
// copy results to persistant location // switch to completed document above
log.debug(resultStr);
File destinationDir =
new File(
props.getProperty("watersmart.file.location")
+ props.getProperty("watersmart.file.location.wps.repository")
+ File.separatorChar
+ uuid);
if (!destinationDir.exists()) {
FileUtils.forceMkdir(destinationDir);
}
String filename =
metaObj.getTypeString()
+ "-"
+ metaObj.getScenario()
+ "-"
+ metaObj.getModelVersion()
+ "."
+ metaObj.getRunIdent()
+ "-"
+ alg
+ ".txt";
File destinationFile =
new File(destinationDir.getCanonicalPath() + File.separatorChar + filename);
FileUtils.write(destinationFile, resultStr, "UTF-8");
String destinationFileName = destinationFile.getName();
String webAccessibleFile =
contextPath
+ props.getProperty("watersmart.file.location.wps.repository")
+ "/"
+ uuid
+ "/"
+ destinationFileName;
return webAccessibleFile;
} finally {
IOUtils.closeQuietly(is);
IOUtils.closeQuietly(resultIs);
}
}
@Override
public void run() {
CSWTransactionHelper helper;
Map<String, String> wpsOutputMap = Maps.newHashMap();
ReturnInfo info;
RunMetadata metaObj = RunMetadata.getInstance(metadata);
String compReq;
String repo = props.getProperty("watersmart.sos.model.repo");
String netCDFFailMessage = "NetCDF failed unexpectedly ";
String cswResponse;
UUID uuid = UUID.randomUUID();
// -- Start here if we have a file and no SOS endpoint...
// 1. Create NetCDF file
// 2. Add results from NetCDF creation to CSW record
// 3. Wait for THREDDS
// -- Start here if we have an SOS endpoint
// 4. Run the compare stats WPS process
// 5. Add results from WPS process to CSW record
if (StringUtils.isBlank(sosEndpoint)) {
// SOS is blank which means we have to create it. Otherwise, the typical use
// case is that the client is requesting a re-run
// 1. Create NetCDF file
try {
log.debug("Creating NetCDF file");
// CreateDSGFromZip.create() seems to cause a lot of grief. We keep getting:
// java.lang.UnsatisfiedLinkError: Native Library
// ${application_path}/loader/com/sun/jna/linux-amd64/libnetcdf.so already loaded in another
// classloader
// When developing and I see this, I have to restart the server and redeploy the project
// The fault happens at
// gov.usgs.cida.jna.NetCDFJNAInitializer.contextDestroyed(NetCDFJNAInitializer.java:21)
info = CreateDSGFromZip.create(zipLocation, metaObj);
if (info != null && info.properties != null) {
netcdfSuccessful = true;
} else {
log.error(netCDFFailMessage);
sendFailedEmail(new RuntimeException(netCDFFailMessage));
throw new IOException("Output from NetCDF creation process was null");
}
} catch (IOException ex) {
log.error(netCDFFailMessage, ex);
sendFailedEmail(new RuntimeException(netCDFFailMessage));
return;
} catch (XMLStreamException ex) {
log.error(netCDFFailMessage, ex);
sendFailedEmail(new RuntimeException(netCDFFailMessage));
return;
} catch (RuntimeException ex) {
log.error(netCDFFailMessage, ex);
sendFailedEmail(new RuntimeException(netCDFFailMessage));
return;
}
// The NetCDF process has passed so create a CSW record for the run and
// insert what we have so far.
// The WPS output will be updated once the process succeeds/fails. The UI
// will show "Process not yet completed" in the meantime.
sosEndpoint = repo + metaObj.getTypeString() + "/" + info.filename;
wpsOutputMap.put(WPSImpl.stats_compare, "");
helper = new CSWTransactionHelper(metaObj, sosEndpoint, wpsOutputMap);
// 2. Add results from NetCDF creation to CSW record
try {
log.debug("Adding results from NetCDF creation to CSW record");
cswResponse = helper.addServiceIdentification();
if (cswResponse != null) {
cswTransSuccessful = true;
} else {
cswTransSuccessful = false;
throw new IOException("Unable to update CSW Record");
}
} catch (Exception ex) {
log.error("Failed to perform CSW insert", ex);
sendFailedEmail(ex);
return;
}
// 3. Wait for THREDDS
try {
log.debug("Beginning THREDDS wait period");
Thread.sleep(SLEEP_FOR_THREDDS);
} catch (InterruptedException ex) {
// Typically we don't care about this, but we can log and move on.
log.warn("THREDDS wait period was interrupted.");
// If anything needs to be handled on an interruption, handle it here
}
log.trace("End of THREDDS wait period");
}
// 4. Run the compare stats using the R-WPS package
try {
log.debug("Sending request for compare stats");
compReq = WPSImpl.createCompareStatsRequest(sosEndpoint);
String algorithmOutput = runNamedAlgorithm("compare", compReq, uuid, metaObj);
wpsOutputMap.put(WPSImpl.stats_compare, algorithmOutput);
} catch (Exception ex) {
log.error("Failed to run WPS algorithm", ex);
sendFailedEmail(ex);
return;
}
// 5. Add results from WPS process to CSW record
if (wpsOutputMap.get(WPSImpl.stats_compare) != null) {
log.debug("Stats compare completed successfully");
rStatsSuccessful = true;
helper = new CSWTransactionHelper(metaObj, sosEndpoint, wpsOutputMap);
try {
cswResponse = helper.updateRunMetadata(metaObj);
cswTransSuccessful = cswResponse != null;
sendCompleteEmail(wpsOutputMap);
} catch (IOException ex) {
log.error("Failed to perform CSW update", ex);
sendFailedEmail(ex);
} catch (URISyntaxException ex) {
log.error("Failed to perform CSW update,", ex);
sendFailedEmail(ex);
}
} else {
log.error("Stats compare failed");
sendFailedEmail(new Exception("Failed to run WPS algorithm"));
}
}