/** This method simply lists the tools available */
public void listTools(HttpClient httpclient) {
Document cipresResponseDoc = cipresQuery(httpclient, baseURL + "/tool", "tools");
if (cipresResponseDoc != null) {
String elementName = "tool";
List tools = cipresResponseDoc.getRootElement().elements(elementName);
int count = 0;
for (Iterator iter = tools.iterator();
iter.hasNext(); ) { // let's get a count as to how many tools there are.
Element nextTool = (Element) iter.next();
count++;
}
String[] toolName = new String[count];
count = 0;
for (Iterator iter = tools.iterator(); iter.hasNext(); ) {
Element nextTool = (Element) iter.next();
String name = nextTool.elementText("toolId");
if (!StringUtil.blank(name) && count < toolName.length) {
ownerModule.logln(name);
}
name = nextTool.elementText("toolName");
if (!StringUtil.blank(name) && count < toolName.length) {
ownerModule.logln(" " + name);
}
count++;
}
}
}
/*.................................................................................................................*/
boolean checkUsernamePassword(boolean tellUserAboutCipres) {
if (StringUtil.blank(username) || StringUtil.blank(password)) {
MesquiteBoolean answer = new MesquiteBoolean(false);
MesquiteString usernameString = new MesquiteString();
if (username != null) usernameString.setValue(username);
MesquiteString passwordString = new MesquiteString();
if (password != null) passwordString.setValue(password);
String help =
"You will need an account on the CIPRes REST system to use this service. To register, go to https://www.phylo.org/restusers/register.action";
new UserNamePasswordDialog(
ownerModule.containerOfModule(),
"Sign in to CIPRes",
help,
"",
"Username",
"Password",
answer,
usernameString,
passwordString);
if (answer.getValue()) {
username = usernameString.getValue();
password = passwordString.getValue();
}
ownerModule.storePreferences();
}
boolean success = StringUtil.notEmpty(username) && StringUtil.notEmpty(password);
if (!success && tellUserAboutCipres) {
MesquiteMessage.discreetNotifyUser(
"Use of the CIPRes service requires an account with CIPRes's REST service. Go to https://www.phylo.org/restusers/register.action to register for an account");
}
return success;
}
/*.................................................................................................................*/
public void deleteAllJobs() {
if (checkUsernamePassword(
false)) { // TODO: check to see if a Zephyr run is ongoing, and warn user if so.
HttpClient httpclient = getHttpClient();
String[] jobURLs = getJobURLs(httpclient);
if (jobURLs != null) {
for (int job = 0; job < jobURLs.length; job++) {
deleteJob(httpclient, jobURLs[job]);
}
if (jobURLs.length == 0) ownerModule.logln("No CIPRes jobs to delete.");
else ownerModule.logln("All " + jobURLs.length + " CIPRes jobs deleted.");
}
}
}
/** Lists to the log the current jobs of the user */
public void listJobs(HttpClient httpclient) {
Document cipresResponseDoc = cipresQuery(httpclient, baseURL + "/job/" + username, "joblist");
if (cipresResponseDoc != null) {
String[] jobList = getJobURLs(cipresResponseDoc);
if (jobList != null)
for (int job = 0; job < jobList.length; job++) {
ownerModule.logln("\njob " + job + ": " + jobList[job]);
String status = getJobStatus(jobList[job]);
ownerModule.logln(" " + status);
String wd = getWorkingDirectory(jobList[job]);
ownerModule.logln(" " + wd);
}
}
}
/*.................................................................................................................*/
public boolean monitorAndCleanUpShell(String jobURL) {
boolean stillGoing = true;
if (!checkUsernamePassword(true)) {
return false;
}
lastModified = null;
if (outputFilePaths != null) {
lastModified = new long[outputFilePaths.length];
LongArray.deassignArray(lastModified);
}
String status = "";
while (!jobCompleted(jobURL) && stillGoing) {
if (StringUtil.blank(status))
ownerModule.logln(
"CIPRes Job Status: " + getJobStatus(jobURL) + " (" + StringUtil.getDateTime() + ")");
// if (jobSubmitted(jobURL))
// processOutputFiles();
try {
Thread.sleep(minPollIntervalSeconds * 1000);
} catch (InterruptedException e) {
MesquiteMessage.notifyProgrammer("InterruptedException in CIPRes monitoring");
return false;
}
stillGoing = watcher == null || watcher.continueShellProcess(null);
String newStatus = getJobStatus(jobURL);
if (newStatus != null && !newStatus.equalsIgnoreCase(status)) {
ownerModule.logln(
"CIPRes Job Status: " + newStatus + " (" + StringUtil.getDateTime() + ")");
status = newStatus;
} else ownerModule.log(".");
if (newStatus != null && newStatus.equalsIgnoreCase("SUBMITTED")) { // job is running
processOutputFiles(jobURL);
}
}
ownerModule.logln("CIPRes job completed.");
if (outputFileProcessor != null) {
if (rootDir != null) {
ownerModule.logln("About to download results from CIPRes.");
if (downloadResults(jobURL, rootDir, false))
outputFileProcessor.processCompletedOutputFiles(outputFilePaths);
else return false;
}
}
return true;
}
public static boolean checkNoContigAceFiles(CharacterData data, MesquiteModule ownerModule) {
DNAData editedData = ChromaseqUtil.getEditedData(data);
if (editedData == null) return false;
int count = 0;
boolean resave = false;
boolean warn = false;
for (int it = 0; it < editedData.getNumTaxa(); it++)
if (AceFile.hasAceFilePath(editedData, it)) {
if (!AceFile.hasAceFile(ownerModule, editedData, it)) {}
AceFile ace = AceFile.getAceFile(ownerModule, editedData, it);
if (ace != null) {
if (ace.getNumContigs() <= 0 || ace.getContig(0).getNumBases() == 0) {
if (!warn
&& !AlertDialog.query(
ownerModule.containerOfModule(),
"Reprocess and save file?",
"Some of the contigs need to be reprocessed, which will"
+ " alter the modified .ace files produced by Phrap and Chromaseq. To be compatible with these altered .ace files, the Mesquite file "
+ "would then need to be re-saved. If instead you choose not to reprocess contigs, they will not be fully editable in Chromaseq.",
"Reprocess and Save",
"Do not reprocess",
-1)) {
return false;
} else resave = true;
warn = true;
ChromaseqUtil.setReprocessContig(editedData, it);
}
}
count++;
}
return resave;
}
/*.................................................................................................................*/
public boolean downloadResults(
HttpClient httpclient, String jobURL, String rootDir, boolean onlyNewOrModified) {
String resultsUri = getResultsDirectory(jobURL);
if (StringUtil.notEmpty(resultsUri)) {
Document cipresResponseDoc = cipresQuery(httpclient, resultsUri, "results");
if (cipresResponseDoc != null) {
CipresJobFile[] cipresJobFiles = processFilesDocument(cipresResponseDoc);
if (cipresJobFiles == null || cipresJobFiles.length == 0) {
ownerModule.logln(cipresResponseDoc.toString());
return false;
}
for (int job = 0; job < cipresJobFiles.length; job++) {
if (!onlyNewOrModified || fileNewOrModified(previousCipresJobFiles, cipresJobFiles, job))
cipresDownload(
httpclient,
cipresJobFiles[job].getDownloadURL(),
rootDir + cipresJobFiles[job].getFileName());
}
previousCipresJobFiles = cipresJobFiles.clone();
return true;
}
}
return false;
}
/**
* this is the primary method that sends a query to the CIPRes REST service. It expects to receive
* an XML file, which it returns in Document if the root tag matc hes xmlRootTag
*/
public Document cipresQuery(HttpClient httpclient, String URL, String xmlRootTag) {
if (StringUtil.blank(URL)) return null;
try {
HttpGet httpget = new HttpGet(URL);
httpget.addHeader("cipres-appkey", CIPRESkey);
try {
HttpResponse response = httpclient.execute(httpget);
HttpEntity responseEntity = response.getEntity();
InputStream instream = responseEntity.getContent();
BufferedReader br = new BufferedReader(new InputStreamReader(instream));
String line = "";
StringBuffer sb = new StringBuffer();
while ((line = br.readLine()) != null) {
sb.append(line + StringUtil.lineEnding());
}
Document cipresResponseDoc = loadXMLFile(xmlRootTag, sb.toString());
if (cipresResponseDoc != null && verbose) {
ownerModule.logln(sb.toString());
}
if (cipresResponseDoc == null) {
Document errorDoc = loadXMLFile(sb.toString());
if (errorDoc != null) reportError(errorDoc, "Error in communicating with CIPRes", true);
}
EntityUtils.consume(response.getEntity());
return cipresResponseDoc;
} catch (IOException e) {
Debugg.printStackTrace(e);
} catch (Exception e) {
Debugg.printStackTrace(e);
}
} catch (Exception e) {
Debugg.printStackTrace(e);
}
return null;
}
/** Reports job status in the log. */
public void reportJobStatus(String jobURL) {
if (checkUsernamePassword(false)) {
HttpClient httpclient = getHttpClient();
Document cipresResponseDoc = cipresQuery(httpclient, jobURL, "jobstatus");
if (cipresResponseDoc != null) {
ownerModule.logln("CIPRes Job Status: " + jobStatusFromResponse(cipresResponseDoc));
}
}
}
/*..........................................MContinuousStates................*/
public int userQueryItem(String message, MesquiteModule module) {
int numItems = getNumItems();
String[] items = new String[numItems];
for (int i = 0; i < items.length; i++) {
if (StringUtil.blank(getItemName(i))) items[i] = "(unnamed)";
else items[i] = getItemName(i);
}
return ListDialog.queryList(
module.containerOfModule(), "Select item", message, MesquiteString.helpString, items, 0);
}
public PAdjustToolExtra(NodePositionAdjust ownerModule, TreeDisplay treeDisplay) {
super(ownerModule, treeDisplay);
selectModule = ownerModule;
adjustTool =
new TreeTool(
this,
"posadjustor",
ownerModule.getPath(),
"adjustPosition.gif",
7,
7,
"Adjust node position",
"This tool adjusts branch lengths in such a way as to keep the relative positions of nodes constant, except for the node moved.");
adjustTool.setTouchedCommand(MesquiteModule.makeCommand("touchedPositionAdjust", this));
adjustTool.setDroppedCommand(MesquiteModule.makeCommand("droppedPositionAdjust", this));
adjustTool.setDraggedCommand(MesquiteModule.makeCommand("draggedPositionAdjust", this));
if (ownerModule.containerOfModule() instanceof MesquiteWindow) {
((MesquiteWindow) ownerModule.containerOfModule()).addTool(adjustTool);
}
}
/*.................................................................................................................*/
public void processAceFileWithContig(
CharacterData data,
MesquiteModule ownerModule,
String processedAceFilePath,
String fragmentDirPath,
AceFile ace,
SequenceUploader uploader,
String geneName,
MesquiteString fullName,
String baseName,
MesquiteString voucherCode,
int it) {
DNAData editedData = ChromaseqUtil.getEditedData(data);
DNAData originalData = ChromaseqUtil.getOriginalData(data);
Taxa taxa = data.getTaxa();
ace.setNameTranslation(fileNameTranslation);
ownerModule.log(ace.contigListForLog() + StringUtil.lineEnding());
if (processPolymorphisms) ace.processPolys(); // creates an additional CO that has polys in it
if (renameContigsInAceFiles) ace.renameContigs(fullName.toString(), addFragName, geneName);
ace.setLowQualityToLowerCase(qualThresholdForLowerCase);
ace.writeToPropertiesFile(contigPropertiesFileBuffer, fullName.toString());
if (truncateMixedEnds)
ace.trimMixedEnds(mixedEndThreshold, mixedEndWindow, qualThresholdForTrim, addPhrapFailures);
/* if (uploadResultsToDatabase && StringUtil.notEmpty(databaseURL)) {
uploader.uploadAceFileToServer(MesquiteXMLToLUtilities.getTOLPageDatabaseURL(databaseURL), ace, processPolymorphisms, qualThresholdForTrim);
}
*/
System.out.println("\n\nfasta file name: " + baseName + " ace file: " + ace);
MesquiteFile.putFileContents(
fragmentDirPath
+ MesquiteFile.fileSeparator
+ ChromaseqUtil.processedFastaFolder
+ MesquiteFile.fileSeparator
+ baseName
+ ".fas",
ace.toFASTAString(processPolymorphisms, qualThresholdForTrim),
true);
MesquiteFile.putFileContents(processedAceFilePath, ace.toString(processPolymorphisms), true);
ace.importSequence(
taxa,
editedData,
it,
originalData,
ChromaseqUtil.getQualityData(data),
ChromaseqUtil.getRegistryData(data),
singleTaxaBlock,
processPolymorphisms,
maxChar,
" contig ",
false,
voucherCode);
}
public void reportError(Document doc, String noteToUser, boolean resetPassword) {
if (doc == null) return;
String displayMessage = doc.getRootElement().elementText("displayMessage");
String message = doc.getRootElement().elementText("message");
if (StringUtil.notEmpty(message)) {
if ("Authentication Error".equalsIgnoreCase(displayMessage)) {
if (resetPassword) password = "";
} else {
ownerModule.logln("\n******************");
ownerModule.logln(noteToUser);
ownerModule.logln(displayMessage);
ownerModule.logln(message);
List paramErrors = doc.getRootElement().elements("paramError");
if (paramErrors != null)
for (Iterator iter = paramErrors.iterator(); iter.hasNext(); ) {
Element nextEntry = (Element) iter.next();
String param = nextEntry.elementText("param");
String error = nextEntry.elementText("error");
ownerModule.logln(" " + param + ": " + error);
}
ownerModule.logln("\n******************\n");
}
}
}
/*.................................................................................................................*/
public boolean sendJobToCipres(
MultipartEntityBuilder builder, String tool, MesquiteString jobURL) {
if (checkUsernamePassword(true)) {
HttpClient httpclient = getHttpClient();
if (builder == null) builder = MultipartEntityBuilder.create();
prepareBuilder(builder, tool, "ZEPHYR.GI." + username + "." + jobNumber);
jobNumber++;
ownerModule.storePreferences();
return postJob(httpclient, builder, jobURL);
}
return false;
}
public void processJobSubmissionResponse(Document cipresResponseDoc, MesquiteString jobURL) {
Element element = cipresResponseDoc.getRootElement().element("selfUri");
Element subelement = null;
if (element != null) subelement = element.element("url");
if ("false".equals(cipresResponseDoc.getRootElement().elementText("failed"))) {
element = cipresResponseDoc.getRootElement().element("metadata");
List entries = element.elements("entry");
String reportedJobID = "";
for (Iterator iter = entries.iterator(); iter.hasNext(); ) {
Element nextEntry = (Element) iter.next();
if ("clientJobId".equals(nextEntry.elementText("key")))
reportedJobID = nextEntry.elementText("value");
}
ownerModule.logln("\nJob successfully submitted to CIPRes.");
ownerModule.logln(" Job URL: " + subelement.getText());
ownerModule.logln(" Job ID: " + reportedJobID + "\n");
if (jobURL != null) jobURL.setValue(subelement.getText());
}
}
/** The core method that initiates a job on CIPRes. */
public boolean postJob(
HttpClient httpclient, MultipartEntityBuilder builder, MesquiteString jobURL) {
if (builder == null) return false;
String URL = baseURL + "/job/" + username;
HttpPost httppost = new HttpPost(URL);
httppost.addHeader("cipres-appkey", CIPRESkey);
// some of this from
// http://stackoverflow.com/questions/18964288/upload-a-file-through-an-http-form-via-multipartentitybuilder-with-a-progress
HttpEntity cipresEntity = builder.build();
httppost.setEntity(cipresEntity);
try {
HttpResponse response = httpclient.execute(httppost);
HttpEntity responseEntity = response.getEntity();
InputStream instream = responseEntity.getContent();
BufferedReader br = new BufferedReader(new InputStreamReader(instream));
StringBuffer sb = new StringBuffer();
String line = "";
while ((line = br.readLine()) != null) {
sb.append(line + StringUtil.lineEnding());
}
Document cipresResponseDoc = loadXMLFile("jobstatus", sb.toString()); // let's see how it went
boolean success = false;
if (cipresResponseDoc != null) {
processJobSubmissionResponse(cipresResponseDoc, jobURL);
if (verbose) ownerModule.logln(sb.toString());
if (jobURL != null) success = StringUtil.notEmpty(jobURL.getValue());
else success = true;
} else {
cipresResponseDoc = loadXMLFile(sb.toString());
reportError(cipresResponseDoc, "Error with CIPRes run", false);
}
EntityUtils.consume(response.getEntity());
return success;
} catch (IOException e) {
Debugg.printStackTrace(e);
}
return false;
}
/*.................................................................................................................*/
public void setTableAndTaxa(MesquiteTable table, Taxa taxa) {
this.table = table;
this.taxa = taxa;
if (containerOfModule() instanceof ListWindow && tool == null) {
String sortExplanation = "Assigns colors to taxa. ";
tool =
new TableTool(
this,
"sort",
getPath(),
"color.gif",
0,
0,
"Color taxon",
sortExplanation,
MesquiteModule.makeCommand("paint", this),
null,
null);
tool.setWorksOnRowNames(true);
((ListWindow) containerOfModule()).addTool(tool);
tool.setPopUpOwner(this);
}
}
/*.................................................................................................................*/
public void reprocessAceFileDirectory(
MesquiteFile file, MesquiteModule ownerModule, DNAData data, int it) {
if (data == null || file == null) return;
String aceFileDirectoryPath =
ChromaseqUtil.getAceFileDirectory(file.getDirectoryName(), ownerModule, data, it);
File aceFileDirectory = new File(aceFileDirectoryPath);
boolean addFragName = false; // control of this?
int currentRead = -1;
String dataFilePath = MesquiteFile.composePath(data.getProject().getHomeDirectoryName(), "");
boolean addingPhrapFailures = false;
AceFile ace = null;
MesquiteProject project = data.getProject();
if (project == null) return;
String processedAceFilePath = "";
MesquiteString fullName = null;
MesquiteString voucherCode = null;
String geneName = ChromaseqUtil.getGeneName(data);
if (aceFileDirectory.isDirectory()) {
int numPhdFiles = getNumPhdFilesInDirectory(aceFileDirectory, aceFileDirectoryPath);
fileNameTranslation = new String[5][numPhdFiles];
fillNameTranslation(data, it, numPhdFiles);
String[] files = aceFileDirectory.list();
for (int i = 0;
i < files.length;
i++) { // going through the folders and finding the ace files
if (files[i] != null) {
String filePath = aceFileDirectoryPath + MesquiteFile.fileSeparator + files[i];
String infoFilePath =
aceFileDirectoryPath + MesquiteFile.fileSeparator + ChromaseqUtil.infoFileName;
File cFile = new File(filePath);
if (cFile.exists()) {
if (!cFile.isDirectory()) {
if (files[i].endsWith(ChromaseqUtil.processedACESuffix + ".ace")) {
// don't do anything
} else if (files[i].endsWith(".ace")
&& !files[i].startsWith(".")
&& !addingPhrapFailures) {
ownerModule.logln("Processing ACE file: " + files[i]);
String baseName =
files[i].substring(
0, files[i].length() - 4); // this is the name of the sequence
processedAceFilePath =
aceFileDirectoryPath
+ MesquiteFile.fileSeparator
+ baseName
+ ChromaseqUtil.processedACESuffix
+ ".ace";
ace =
new AceFile(
filePath,
processedAceFilePath,
dataFilePath,
dataFilePath,
ownerModule,
processPolymorphisms,
polyThreshold,
false);
if (ace == null) return;
ace.setBaseName(baseName);
fullName = new MesquiteString(baseName);
voucherCode = new MesquiteString();
ChromaseqInfoFile.processInfoFile(infoFilePath, fullName, voucherCode);
String fragmentDirPath =
StringUtil.getAllButLastItem(
StringUtil.getAllButLastItem(
aceFileDirectoryPath, MesquiteFile.fileSeparator),
MesquiteFile.fileSeparator);
ace.setLongSequenceName(fullName.toString());
if (ace.getNumContigs() >= 1) {
processAceFileWithContig(
data,
ownerModule,
processedAceFilePath,
fragmentDirPath,
ace,
null,
geneName,
fullName,
baseName,
voucherCode,
it);
} else {
ownerModule.logln(" ACE file contains no contigs!");
if (project != null) {
addingPhrapFailures = true;
i = 0;
ace.createEmptyContigs(
MesquiteFile.numFilesEndingWith(
aceFileDirectoryPath, files, ".phd.1")); // create an empty contig
ace.renameContigs(fullName.toString(), addFragName, geneName);
}
}
if (!addingPhrapFailures) ace.dispose();
} else if (files[i].endsWith(".phd.1") && addingPhrapFailures) {
ownerModule.logln(" Importing single-read Phred file " + files[i]);
currentRead++;
ace.addPhdFileAsSingleReadInContig(
currentRead,
aceFileDirectoryPath,
files[i],
processPolymorphisms,
polyThreshold);
}
}
}
}
}
}
if (addingPhrapFailures && ace != null) { // have to process AceFile that we have manually made
MesquiteFile.putFileContents(processedAceFilePath, ace.toString(processPolymorphisms), true);
if (project != null) {
processAceFileWithoutContig(
data, processedAceFilePath, ace, geneName, fullName, it, voucherCode);
}
ace.dispose();
}
}