/** This method simply lists the tools available */ public void listTools(HttpClient httpclient) { Document cipresResponseDoc = cipresQuery(httpclient, baseURL + "/tool", "tools"); if (cipresResponseDoc != null) { String elementName = "tool"; List tools = cipresResponseDoc.getRootElement().elements(elementName); int count = 0; for (Iterator iter = tools.iterator(); iter.hasNext(); ) { // let's get a count as to how many tools there are. Element nextTool = (Element) iter.next(); count++; } String[] toolName = new String[count]; count = 0; for (Iterator iter = tools.iterator(); iter.hasNext(); ) { Element nextTool = (Element) iter.next(); String name = nextTool.elementText("toolId"); if (!StringUtil.blank(name) && count < toolName.length) { ownerModule.logln(name); } name = nextTool.elementText("toolName"); if (!StringUtil.blank(name) && count < toolName.length) { ownerModule.logln(" " + name); } count++; } } }
/*.................................................................................................................*/ boolean checkUsernamePassword(boolean tellUserAboutCipres) { if (StringUtil.blank(username) || StringUtil.blank(password)) { MesquiteBoolean answer = new MesquiteBoolean(false); MesquiteString usernameString = new MesquiteString(); if (username != null) usernameString.setValue(username); MesquiteString passwordString = new MesquiteString(); if (password != null) passwordString.setValue(password); String help = "You will need an account on the CIPRes REST system to use this service. To register, go to https://www.phylo.org/restusers/register.action"; new UserNamePasswordDialog( ownerModule.containerOfModule(), "Sign in to CIPRes", help, "", "Username", "Password", answer, usernameString, passwordString); if (answer.getValue()) { username = usernameString.getValue(); password = passwordString.getValue(); } ownerModule.storePreferences(); } boolean success = StringUtil.notEmpty(username) && StringUtil.notEmpty(password); if (!success && tellUserAboutCipres) { MesquiteMessage.discreetNotifyUser( "Use of the CIPRes service requires an account with CIPRes's REST service. Go to https://www.phylo.org/restusers/register.action to register for an account"); } return success; }
/*.................................................................................................................*/ public void deleteAllJobs() { if (checkUsernamePassword( false)) { // TODO: check to see if a Zephyr run is ongoing, and warn user if so. HttpClient httpclient = getHttpClient(); String[] jobURLs = getJobURLs(httpclient); if (jobURLs != null) { for (int job = 0; job < jobURLs.length; job++) { deleteJob(httpclient, jobURLs[job]); } if (jobURLs.length == 0) ownerModule.logln("No CIPRes jobs to delete."); else ownerModule.logln("All " + jobURLs.length + " CIPRes jobs deleted."); } } }
/** Lists to the log the current jobs of the user */ public void listJobs(HttpClient httpclient) { Document cipresResponseDoc = cipresQuery(httpclient, baseURL + "/job/" + username, "joblist"); if (cipresResponseDoc != null) { String[] jobList = getJobURLs(cipresResponseDoc); if (jobList != null) for (int job = 0; job < jobList.length; job++) { ownerModule.logln("\njob " + job + ": " + jobList[job]); String status = getJobStatus(jobList[job]); ownerModule.logln(" " + status); String wd = getWorkingDirectory(jobList[job]); ownerModule.logln(" " + wd); } } }
/*.................................................................................................................*/ public boolean monitorAndCleanUpShell(String jobURL) { boolean stillGoing = true; if (!checkUsernamePassword(true)) { return false; } lastModified = null; if (outputFilePaths != null) { lastModified = new long[outputFilePaths.length]; LongArray.deassignArray(lastModified); } String status = ""; while (!jobCompleted(jobURL) && stillGoing) { if (StringUtil.blank(status)) ownerModule.logln( "CIPRes Job Status: " + getJobStatus(jobURL) + " (" + StringUtil.getDateTime() + ")"); // if (jobSubmitted(jobURL)) // processOutputFiles(); try { Thread.sleep(minPollIntervalSeconds * 1000); } catch (InterruptedException e) { MesquiteMessage.notifyProgrammer("InterruptedException in CIPRes monitoring"); return false; } stillGoing = watcher == null || watcher.continueShellProcess(null); String newStatus = getJobStatus(jobURL); if (newStatus != null && !newStatus.equalsIgnoreCase(status)) { ownerModule.logln( "CIPRes Job Status: " + newStatus + " (" + StringUtil.getDateTime() + ")"); status = newStatus; } else ownerModule.log("."); if (newStatus != null && newStatus.equalsIgnoreCase("SUBMITTED")) { // job is running processOutputFiles(jobURL); } } ownerModule.logln("CIPRes job completed."); if (outputFileProcessor != null) { if (rootDir != null) { ownerModule.logln("About to download results from CIPRes."); if (downloadResults(jobURL, rootDir, false)) outputFileProcessor.processCompletedOutputFiles(outputFilePaths); else return false; } } return true; }
/*.................................................................................................................*/ public boolean downloadResults( HttpClient httpclient, String jobURL, String rootDir, boolean onlyNewOrModified) { String resultsUri = getResultsDirectory(jobURL); if (StringUtil.notEmpty(resultsUri)) { Document cipresResponseDoc = cipresQuery(httpclient, resultsUri, "results"); if (cipresResponseDoc != null) { CipresJobFile[] cipresJobFiles = processFilesDocument(cipresResponseDoc); if (cipresJobFiles == null || cipresJobFiles.length == 0) { ownerModule.logln(cipresResponseDoc.toString()); return false; } for (int job = 0; job < cipresJobFiles.length; job++) { if (!onlyNewOrModified || fileNewOrModified(previousCipresJobFiles, cipresJobFiles, job)) cipresDownload( httpclient, cipresJobFiles[job].getDownloadURL(), rootDir + cipresJobFiles[job].getFileName()); } previousCipresJobFiles = cipresJobFiles.clone(); return true; } } return false; }
public static boolean checkNoContigAceFiles(CharacterData data, MesquiteModule ownerModule) { DNAData editedData = ChromaseqUtil.getEditedData(data); if (editedData == null) return false; int count = 0; boolean resave = false; boolean warn = false; for (int it = 0; it < editedData.getNumTaxa(); it++) if (AceFile.hasAceFilePath(editedData, it)) { if (!AceFile.hasAceFile(ownerModule, editedData, it)) {} AceFile ace = AceFile.getAceFile(ownerModule, editedData, it); if (ace != null) { if (ace.getNumContigs() <= 0 || ace.getContig(0).getNumBases() == 0) { if (!warn && !AlertDialog.query( ownerModule.containerOfModule(), "Reprocess and save file?", "Some of the contigs need to be reprocessed, which will" + " alter the modified .ace files produced by Phrap and Chromaseq. To be compatible with these altered .ace files, the Mesquite file " + "would then need to be re-saved. If instead you choose not to reprocess contigs, they will not be fully editable in Chromaseq.", "Reprocess and Save", "Do not reprocess", -1)) { return false; } else resave = true; warn = true; ChromaseqUtil.setReprocessContig(editedData, it); } } count++; } return resave; }
/** * this is the primary method that sends a query to the CIPRes REST service. It expects to receive * an XML file, which it returns in Document if the root tag matc hes xmlRootTag */ public Document cipresQuery(HttpClient httpclient, String URL, String xmlRootTag) { if (StringUtil.blank(URL)) return null; try { HttpGet httpget = new HttpGet(URL); httpget.addHeader("cipres-appkey", CIPRESkey); try { HttpResponse response = httpclient.execute(httpget); HttpEntity responseEntity = response.getEntity(); InputStream instream = responseEntity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(instream)); String line = ""; StringBuffer sb = new StringBuffer(); while ((line = br.readLine()) != null) { sb.append(line + StringUtil.lineEnding()); } Document cipresResponseDoc = loadXMLFile(xmlRootTag, sb.toString()); if (cipresResponseDoc != null && verbose) { ownerModule.logln(sb.toString()); } if (cipresResponseDoc == null) { Document errorDoc = loadXMLFile(sb.toString()); if (errorDoc != null) reportError(errorDoc, "Error in communicating with CIPRes", true); } EntityUtils.consume(response.getEntity()); return cipresResponseDoc; } catch (IOException e) { Debugg.printStackTrace(e); } catch (Exception e) { Debugg.printStackTrace(e); } } catch (Exception e) { Debugg.printStackTrace(e); } return null; }
/** Reports job status in the log. */ public void reportJobStatus(String jobURL) { if (checkUsernamePassword(false)) { HttpClient httpclient = getHttpClient(); Document cipresResponseDoc = cipresQuery(httpclient, jobURL, "jobstatus"); if (cipresResponseDoc != null) { ownerModule.logln("CIPRes Job Status: " + jobStatusFromResponse(cipresResponseDoc)); } } }
/*..........................................MContinuousStates................*/ public int userQueryItem(String message, MesquiteModule module) { int numItems = getNumItems(); String[] items = new String[numItems]; for (int i = 0; i < items.length; i++) { if (StringUtil.blank(getItemName(i))) items[i] = "(unnamed)"; else items[i] = getItemName(i); } return ListDialog.queryList( module.containerOfModule(), "Select item", message, MesquiteString.helpString, items, 0); }
public PAdjustToolExtra(NodePositionAdjust ownerModule, TreeDisplay treeDisplay) { super(ownerModule, treeDisplay); selectModule = ownerModule; adjustTool = new TreeTool( this, "posadjustor", ownerModule.getPath(), "adjustPosition.gif", 7, 7, "Adjust node position", "This tool adjusts branch lengths in such a way as to keep the relative positions of nodes constant, except for the node moved."); adjustTool.setTouchedCommand(MesquiteModule.makeCommand("touchedPositionAdjust", this)); adjustTool.setDroppedCommand(MesquiteModule.makeCommand("droppedPositionAdjust", this)); adjustTool.setDraggedCommand(MesquiteModule.makeCommand("draggedPositionAdjust", this)); if (ownerModule.containerOfModule() instanceof MesquiteWindow) { ((MesquiteWindow) ownerModule.containerOfModule()).addTool(adjustTool); } }
/*.................................................................................................................*/ public void processAceFileWithContig( CharacterData data, MesquiteModule ownerModule, String processedAceFilePath, String fragmentDirPath, AceFile ace, SequenceUploader uploader, String geneName, MesquiteString fullName, String baseName, MesquiteString voucherCode, int it) { DNAData editedData = ChromaseqUtil.getEditedData(data); DNAData originalData = ChromaseqUtil.getOriginalData(data); Taxa taxa = data.getTaxa(); ace.setNameTranslation(fileNameTranslation); ownerModule.log(ace.contigListForLog() + StringUtil.lineEnding()); if (processPolymorphisms) ace.processPolys(); // creates an additional CO that has polys in it if (renameContigsInAceFiles) ace.renameContigs(fullName.toString(), addFragName, geneName); ace.setLowQualityToLowerCase(qualThresholdForLowerCase); ace.writeToPropertiesFile(contigPropertiesFileBuffer, fullName.toString()); if (truncateMixedEnds) ace.trimMixedEnds(mixedEndThreshold, mixedEndWindow, qualThresholdForTrim, addPhrapFailures); /* if (uploadResultsToDatabase && StringUtil.notEmpty(databaseURL)) { uploader.uploadAceFileToServer(MesquiteXMLToLUtilities.getTOLPageDatabaseURL(databaseURL), ace, processPolymorphisms, qualThresholdForTrim); } */ System.out.println("\n\nfasta file name: " + baseName + " ace file: " + ace); MesquiteFile.putFileContents( fragmentDirPath + MesquiteFile.fileSeparator + ChromaseqUtil.processedFastaFolder + MesquiteFile.fileSeparator + baseName + ".fas", ace.toFASTAString(processPolymorphisms, qualThresholdForTrim), true); MesquiteFile.putFileContents(processedAceFilePath, ace.toString(processPolymorphisms), true); ace.importSequence( taxa, editedData, it, originalData, ChromaseqUtil.getQualityData(data), ChromaseqUtil.getRegistryData(data), singleTaxaBlock, processPolymorphisms, maxChar, " contig ", false, voucherCode); }
public void reportError(Document doc, String noteToUser, boolean resetPassword) { if (doc == null) return; String displayMessage = doc.getRootElement().elementText("displayMessage"); String message = doc.getRootElement().elementText("message"); if (StringUtil.notEmpty(message)) { if ("Authentication Error".equalsIgnoreCase(displayMessage)) { if (resetPassword) password = ""; } else { ownerModule.logln("\n******************"); ownerModule.logln(noteToUser); ownerModule.logln(displayMessage); ownerModule.logln(message); List paramErrors = doc.getRootElement().elements("paramError"); if (paramErrors != null) for (Iterator iter = paramErrors.iterator(); iter.hasNext(); ) { Element nextEntry = (Element) iter.next(); String param = nextEntry.elementText("param"); String error = nextEntry.elementText("error"); ownerModule.logln(" " + param + ": " + error); } ownerModule.logln("\n******************\n"); } } }
public void processJobSubmissionResponse(Document cipresResponseDoc, MesquiteString jobURL) { Element element = cipresResponseDoc.getRootElement().element("selfUri"); Element subelement = null; if (element != null) subelement = element.element("url"); if ("false".equals(cipresResponseDoc.getRootElement().elementText("failed"))) { element = cipresResponseDoc.getRootElement().element("metadata"); List entries = element.elements("entry"); String reportedJobID = ""; for (Iterator iter = entries.iterator(); iter.hasNext(); ) { Element nextEntry = (Element) iter.next(); if ("clientJobId".equals(nextEntry.elementText("key"))) reportedJobID = nextEntry.elementText("value"); } ownerModule.logln("\nJob successfully submitted to CIPRes."); ownerModule.logln(" Job URL: " + subelement.getText()); ownerModule.logln(" Job ID: " + reportedJobID + "\n"); if (jobURL != null) jobURL.setValue(subelement.getText()); } }
/*.................................................................................................................*/ public boolean sendJobToCipres( MultipartEntityBuilder builder, String tool, MesquiteString jobURL) { if (checkUsernamePassword(true)) { HttpClient httpclient = getHttpClient(); if (builder == null) builder = MultipartEntityBuilder.create(); prepareBuilder(builder, tool, "ZEPHYR.GI." + username + "." + jobNumber); jobNumber++; ownerModule.storePreferences(); return postJob(httpclient, builder, jobURL); } return false; }
/** The core method that initiates a job on CIPRes. */ public boolean postJob( HttpClient httpclient, MultipartEntityBuilder builder, MesquiteString jobURL) { if (builder == null) return false; String URL = baseURL + "/job/" + username; HttpPost httppost = new HttpPost(URL); httppost.addHeader("cipres-appkey", CIPRESkey); // some of this from // http://stackoverflow.com/questions/18964288/upload-a-file-through-an-http-form-via-multipartentitybuilder-with-a-progress HttpEntity cipresEntity = builder.build(); httppost.setEntity(cipresEntity); try { HttpResponse response = httpclient.execute(httppost); HttpEntity responseEntity = response.getEntity(); InputStream instream = responseEntity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(instream)); StringBuffer sb = new StringBuffer(); String line = ""; while ((line = br.readLine()) != null) { sb.append(line + StringUtil.lineEnding()); } Document cipresResponseDoc = loadXMLFile("jobstatus", sb.toString()); // let's see how it went boolean success = false; if (cipresResponseDoc != null) { processJobSubmissionResponse(cipresResponseDoc, jobURL); if (verbose) ownerModule.logln(sb.toString()); if (jobURL != null) success = StringUtil.notEmpty(jobURL.getValue()); else success = true; } else { cipresResponseDoc = loadXMLFile(sb.toString()); reportError(cipresResponseDoc, "Error with CIPRes run", false); } EntityUtils.consume(response.getEntity()); return success; } catch (IOException e) { Debugg.printStackTrace(e); } return false; }
/*.................................................................................................................*/ public void setTableAndTaxa(MesquiteTable table, Taxa taxa) { this.table = table; this.taxa = taxa; if (containerOfModule() instanceof ListWindow && tool == null) { String sortExplanation = "Assigns colors to taxa. "; tool = new TableTool( this, "sort", getPath(), "color.gif", 0, 0, "Color taxon", sortExplanation, MesquiteModule.makeCommand("paint", this), null, null); tool.setWorksOnRowNames(true); ((ListWindow) containerOfModule()).addTool(tool); tool.setPopUpOwner(this); } }
/*.................................................................................................................*/ public void reprocessAceFileDirectory( MesquiteFile file, MesquiteModule ownerModule, DNAData data, int it) { if (data == null || file == null) return; String aceFileDirectoryPath = ChromaseqUtil.getAceFileDirectory(file.getDirectoryName(), ownerModule, data, it); File aceFileDirectory = new File(aceFileDirectoryPath); boolean addFragName = false; // control of this? int currentRead = -1; String dataFilePath = MesquiteFile.composePath(data.getProject().getHomeDirectoryName(), ""); boolean addingPhrapFailures = false; AceFile ace = null; MesquiteProject project = data.getProject(); if (project == null) return; String processedAceFilePath = ""; MesquiteString fullName = null; MesquiteString voucherCode = null; String geneName = ChromaseqUtil.getGeneName(data); if (aceFileDirectory.isDirectory()) { int numPhdFiles = getNumPhdFilesInDirectory(aceFileDirectory, aceFileDirectoryPath); fileNameTranslation = new String[5][numPhdFiles]; fillNameTranslation(data, it, numPhdFiles); String[] files = aceFileDirectory.list(); for (int i = 0; i < files.length; i++) { // going through the folders and finding the ace files if (files[i] != null) { String filePath = aceFileDirectoryPath + MesquiteFile.fileSeparator + files[i]; String infoFilePath = aceFileDirectoryPath + MesquiteFile.fileSeparator + ChromaseqUtil.infoFileName; File cFile = new File(filePath); if (cFile.exists()) { if (!cFile.isDirectory()) { if (files[i].endsWith(ChromaseqUtil.processedACESuffix + ".ace")) { // don't do anything } else if (files[i].endsWith(".ace") && !files[i].startsWith(".") && !addingPhrapFailures) { ownerModule.logln("Processing ACE file: " + files[i]); String baseName = files[i].substring( 0, files[i].length() - 4); // this is the name of the sequence processedAceFilePath = aceFileDirectoryPath + MesquiteFile.fileSeparator + baseName + ChromaseqUtil.processedACESuffix + ".ace"; ace = new AceFile( filePath, processedAceFilePath, dataFilePath, dataFilePath, ownerModule, processPolymorphisms, polyThreshold, false); if (ace == null) return; ace.setBaseName(baseName); fullName = new MesquiteString(baseName); voucherCode = new MesquiteString(); ChromaseqInfoFile.processInfoFile(infoFilePath, fullName, voucherCode); String fragmentDirPath = StringUtil.getAllButLastItem( StringUtil.getAllButLastItem( aceFileDirectoryPath, MesquiteFile.fileSeparator), MesquiteFile.fileSeparator); ace.setLongSequenceName(fullName.toString()); if (ace.getNumContigs() >= 1) { processAceFileWithContig( data, ownerModule, processedAceFilePath, fragmentDirPath, ace, null, geneName, fullName, baseName, voucherCode, it); } else { ownerModule.logln(" ACE file contains no contigs!"); if (project != null) { addingPhrapFailures = true; i = 0; ace.createEmptyContigs( MesquiteFile.numFilesEndingWith( aceFileDirectoryPath, files, ".phd.1")); // create an empty contig ace.renameContigs(fullName.toString(), addFragName, geneName); } } if (!addingPhrapFailures) ace.dispose(); } else if (files[i].endsWith(".phd.1") && addingPhrapFailures) { ownerModule.logln(" Importing single-read Phred file " + files[i]); currentRead++; ace.addPhdFileAsSingleReadInContig( currentRead, aceFileDirectoryPath, files[i], processPolymorphisms, polyThreshold); } } } } } } if (addingPhrapFailures && ace != null) { // have to process AceFile that we have manually made MesquiteFile.putFileContents(processedAceFilePath, ace.toString(processPolymorphisms), true); if (project != null) { processAceFileWithoutContig( data, processedAceFilePath, ace, geneName, fullName, it, voucherCode); } ace.dispose(); } }
public String getImagePath() { // TODO: this should use path to standard image if (stacked) return MesquiteModule.getRootPath() + "images/miniscrollVert.gif"; else return MesquiteModule.getRootPath() + "images/miniscrollHoriz.gif"; }