/*.................................................................................................................*/ public void processAceFileWithoutContig( DNAData data, String processedAceFilePath, AceFile ace, String geneName, MesquiteString fullName, int it, MesquiteString voucherCode) { DNAData editedData = ChromaseqUtil.getEditedData(data); DNAData originalData = ChromaseqUtil.getOriginalData(data); Taxa taxa = data.getTaxa(); ace.processFailedContig(polyThreshold); ace.setNameTranslation(fileNameTranslation); ace.renameContigs(fullName.toString(), addFragName, geneName); ace.setLowQualityToLowerCase(qualThresholdForLowerCase); ace.writeToPropertiesFile(contigPropertiesFileBuffer, fullName.toString()); if (truncateMixedEnds) { ace.trimMixedEnds(mixedEndThreshold, mixedEndWindow, qualThresholdForTrim, addPhrapFailures); } MesquiteFile.putFileContents(processedAceFilePath, ace.toString(processPolymorphisms), true); ace.importSequence( taxa, editedData, it, originalData, ChromaseqUtil.getQualityData(data), ChromaseqUtil.getRegistryData(data), singleTaxaBlock, processPolymorphisms, maxChar, "", true, voucherCode); }
/*.................................................................................................................*/ public Object getProgramArguments( String dataFileName, String configFileName, boolean isPreflight) { MesquiteString arguments = new MesquiteString(); if (externalProcRunner.isWindows()) arguments.setValue(" --batch " + configFileName); else arguments.setValue( ""); // GARLI command is very simple as all of the arguments are in the config file return arguments; }
/*.................................................................................................................*/ public void processAceFileWithContig( CharacterData data, MesquiteModule ownerModule, String processedAceFilePath, String fragmentDirPath, AceFile ace, SequenceUploader uploader, String geneName, MesquiteString fullName, String baseName, MesquiteString voucherCode, int it) { DNAData editedData = ChromaseqUtil.getEditedData(data); DNAData originalData = ChromaseqUtil.getOriginalData(data); Taxa taxa = data.getTaxa(); ace.setNameTranslation(fileNameTranslation); ownerModule.log(ace.contigListForLog() + StringUtil.lineEnding()); if (processPolymorphisms) ace.processPolys(); // creates an additional CO that has polys in it if (renameContigsInAceFiles) ace.renameContigs(fullName.toString(), addFragName, geneName); ace.setLowQualityToLowerCase(qualThresholdForLowerCase); ace.writeToPropertiesFile(contigPropertiesFileBuffer, fullName.toString()); if (truncateMixedEnds) ace.trimMixedEnds(mixedEndThreshold, mixedEndWindow, qualThresholdForTrim, addPhrapFailures); /* if (uploadResultsToDatabase && StringUtil.notEmpty(databaseURL)) { uploader.uploadAceFileToServer(MesquiteXMLToLUtilities.getTOLPageDatabaseURL(databaseURL), ace, processPolymorphisms, qualThresholdForTrim); } */ System.out.println("\n\nfasta file name: " + baseName + " ace file: " + ace); MesquiteFile.putFileContents( fragmentDirPath + MesquiteFile.fileSeparator + ChromaseqUtil.processedFastaFolder + MesquiteFile.fileSeparator + baseName + ".fas", ace.toFASTAString(processPolymorphisms, qualThresholdForTrim), true); MesquiteFile.putFileContents(processedAceFilePath, ace.toString(processPolymorphisms), true); ace.importSequence( taxa, editedData, it, originalData, ChromaseqUtil.getQualityData(data), ChromaseqUtil.getRegistryData(data), singleTaxaBlock, processPolymorphisms, maxChar, " contig ", false, voucherCode); }
/** Called to alter data in all cells */ public boolean operateOnData(CharacterData data) { if (!(data instanceof ContinuousData)) return false; ContinuousData cData = (ContinuousData) data; int numItems = cData.getNumItems(); String[] items = new String[numItems]; for (int i = 0; i < items.length; i++) { if (StringUtil.blank(cData.getItemName(i))) items[i] = "(unnamed)"; else items[i] = cData.getItemName(i); } int d = ListDialog.queryList( containerOfModule(), "Rename item", "Rename item:", MesquiteString.helpString, items, 0); if (!MesquiteInteger.isCombinable(d) || d < 0 || d >= numItems) return false; else { String s = MesquiteString.queryString( containerOfModule(), "Rename Item", "New name for " + items[d], items[d]); if (StringUtil.blank(s)) return false; cData.setItemReference(d, NameReference.getNameReference(s)); return true; } }
/*.................................................................................................................*/ boolean checkUsernamePassword(boolean tellUserAboutCipres) { if (StringUtil.blank(username) || StringUtil.blank(password)) { MesquiteBoolean answer = new MesquiteBoolean(false); MesquiteString usernameString = new MesquiteString(); if (username != null) usernameString.setValue(username); MesquiteString passwordString = new MesquiteString(); if (password != null) passwordString.setValue(password); String help = "You will need an account on the CIPRes REST system to use this service. To register, go to https://www.phylo.org/restusers/register.action"; new UserNamePasswordDialog( ownerModule.containerOfModule(), "Sign in to CIPRes", help, "", "Username", "Password", answer, usernameString, passwordString); if (answer.getValue()) { username = usernameString.getValue(); password = passwordString.getValue(); } ownerModule.storePreferences(); } boolean success = StringUtil.notEmpty(username) && StringUtil.notEmpty(password); if (!success && tellUserAboutCipres) { MesquiteMessage.discreetNotifyUser( "Use of the CIPRes service requires an account with CIPRes's REST service. Go to https://www.phylo.org/restusers/register.action to register for an account"); } return success; }
/*.................................................................................................................*/ private void recordSourceProject() { MesquiteString s = new MesquiteString(); s.setValue(getProject().getHomeFileName()); if (!QueryDialogs.queryString( containerOfModule(), "Name to Stamp", "Indicate source file name to stamp on matrix rows", s)) return; int numMatrices = getProject().getNumberCharMatrices(); NameReference sourceRef = NameReference.getNameReference("SourceFile"); for (int im = 0; im < numMatrices; im++) { CharacterData data = getProject().getCharacterMatrix(im); Taxa taxa = data.getTaxa(); Associable tInfo = data.getTaxaInfo(true); boolean anySelected = taxa.anySelected(); for (int it = 0; it < taxa.getNumTaxa(); it++) { if (data.hasDataForTaxon(it) && (!anySelected || taxa.getSelected(it))) tInfo.setAssociatedObject(sourceRef, it, s.getValue()); } } }
/*.................................................................................................................*/ public void calculateNumber( Tree tree1, Tree tree2, MesquiteNumber result, MesquiteString resultString) { if (result == null) return; clearResultAndLastResult(result); if (tree1 == null) return; if (tree2 == null) return; int numTaxa = tree1.getTaxa().getNumTaxa(); double[][] patristic1 = null; patristic1 = p1.calculatePatristic( tree1, numTaxa, patristic1); // for this tree calculate patristic distances (number of nodes separating // terminals; no branch lengths) double[][] patristic2 = null; patristic2 = p2.calculatePatristic( tree2, numTaxa, patristic2); // for this tree calculate patristic distances (number of nodes separating // terminals; no branch lengths) double correl = offDiagonalPMCorrelationFILTERED(patristic1, patristic2); if (isDistance && (MesquiteDouble.isCombinable(correl))) correl = -correl + 1.0; // shifting 1 to -1 to be 0 to 2 to act as distance result.setValue(correl); if (resultString != null) { if (isDistance) resultString.setValue( "Patristic correlation (converted to distance): " + result.toString()); else resultString.setValue("Patristic correlation: " + result.toString()); } saveLastResult(result); saveLastResultString(resultString); }
/*.................................................................................................................*/ public Object doCommand(String commandName, String arguments, CommandChecker checker) { if (checker.compare( this.getClass(), "Sets module supplying characters", "[name of module]", commandName, "setCharacterSource")) { CharacterObedSource newCharacterSourceTask; if (hiringCondition != null) newCharacterSourceTask = (CharacterObedSource) replaceCompatibleEmployee( CharacterObedSource.class, arguments, characterSourceTask, hiringCondition); // , "Source of characters" else newCharacterSourceTask = (CharacterObedSource) replaceEmployee( CharacterObedSource.class, arguments, "Source of characters", characterSourceTask); if (newCharacterSourceTask != null) { characterSourceTask = newCharacterSourceTask; characterSourceTask.setHiringCommand(cstC); charSourceName.setValue(characterSourceTask.getName()); resetContainingMenuBar(); parametersChanged(); return characterSourceTask; } else { discreetAlert( "Unable to activate character source \"" + arguments + "\" for use by " + employer.getName()); } } else if (characterSourceTask != null) { // todo: temporary, for snapshot conversions return characterSourceTask.doCommand(commandName, arguments, checker); } else return super.doCommand(commandName, arguments, checker); return null; }
public void calculateNumber(Taxon taxon, MesquiteNumber result, MesquiteString resultString) { if (result == null) return; clearResultAndLastResult(result); Taxa taxa = taxon.getTaxa(); int it = taxa.whichTaxonNumber(taxon); if (taxa != currentTaxa || observedStates == null) { observedStates = matrixSourceTask.getCurrentMatrix(taxa); currentTaxa = taxa; } if (observedStates == null || !(observedStates.getParentData() instanceof DNAData)) return; DNAData data = (DNAData) observedStates.getParentData(); int count = data.getAminoAcidNumbers(it, ProteinData.TER, countEvenIfOthers.getValue()); if (result != null) result.setValue(count); if (resultString != null) resultString.setValue( "Number of stop codons in taxon " + observedStates.getName() + ": " + count); saveLastResult(result); saveLastResultString(resultString); }
/** The core method that initiates a job on CIPRes. */ public boolean postJob( HttpClient httpclient, MultipartEntityBuilder builder, MesquiteString jobURL) { if (builder == null) return false; String URL = baseURL + "/job/" + username; HttpPost httppost = new HttpPost(URL); httppost.addHeader("cipres-appkey", CIPRESkey); // some of this from // http://stackoverflow.com/questions/18964288/upload-a-file-through-an-http-form-via-multipartentitybuilder-with-a-progress HttpEntity cipresEntity = builder.build(); httppost.setEntity(cipresEntity); try { HttpResponse response = httpclient.execute(httppost); HttpEntity responseEntity = response.getEntity(); InputStream instream = responseEntity.getContent(); BufferedReader br = new BufferedReader(new InputStreamReader(instream)); StringBuffer sb = new StringBuffer(); String line = ""; while ((line = br.readLine()) != null) { sb.append(line + StringUtil.lineEnding()); } Document cipresResponseDoc = loadXMLFile("jobstatus", sb.toString()); // let's see how it went boolean success = false; if (cipresResponseDoc != null) { processJobSubmissionResponse(cipresResponseDoc, jobURL); if (verbose) ownerModule.logln(sb.toString()); if (jobURL != null) success = StringUtil.notEmpty(jobURL.getValue()); else success = true; } else { cipresResponseDoc = loadXMLFile(sb.toString()); reportError(cipresResponseDoc, "Error with CIPRes run", false); } EntityUtils.consume(response.getEntity()); return success; } catch (IOException e) { Debugg.printStackTrace(e); } return false; }
public void processJobSubmissionResponse(Document cipresResponseDoc, MesquiteString jobURL) { Element element = cipresResponseDoc.getRootElement().element("selfUri"); Element subelement = null; if (element != null) subelement = element.element("url"); if ("false".equals(cipresResponseDoc.getRootElement().elementText("failed"))) { element = cipresResponseDoc.getRootElement().element("metadata"); List entries = element.elements("entry"); String reportedJobID = ""; for (Iterator iter = entries.iterator(); iter.hasNext(); ) { Element nextEntry = (Element) iter.next(); if ("clientJobId".equals(nextEntry.elementText("key"))) reportedJobID = nextEntry.elementText("value"); } ownerModule.logln("\nJob successfully submitted to CIPRes."); ownerModule.logln(" Job URL: " + subelement.getText()); ownerModule.logln(" Job ID: " + reportedJobID + "\n"); if (jobURL != null) jobURL.setValue(subelement.getText()); } }
/*.................................................................................................................*/ public void reprocessAceFileDirectory( MesquiteFile file, MesquiteModule ownerModule, DNAData data, int it) { if (data == null || file == null) return; String aceFileDirectoryPath = ChromaseqUtil.getAceFileDirectory(file.getDirectoryName(), ownerModule, data, it); File aceFileDirectory = new File(aceFileDirectoryPath); boolean addFragName = false; // control of this? int currentRead = -1; String dataFilePath = MesquiteFile.composePath(data.getProject().getHomeDirectoryName(), ""); boolean addingPhrapFailures = false; AceFile ace = null; MesquiteProject project = data.getProject(); if (project == null) return; String processedAceFilePath = ""; MesquiteString fullName = null; MesquiteString voucherCode = null; String geneName = ChromaseqUtil.getGeneName(data); if (aceFileDirectory.isDirectory()) { int numPhdFiles = getNumPhdFilesInDirectory(aceFileDirectory, aceFileDirectoryPath); fileNameTranslation = new String[5][numPhdFiles]; fillNameTranslation(data, it, numPhdFiles); String[] files = aceFileDirectory.list(); for (int i = 0; i < files.length; i++) { // going through the folders and finding the ace files if (files[i] != null) { String filePath = aceFileDirectoryPath + MesquiteFile.fileSeparator + files[i]; String infoFilePath = aceFileDirectoryPath + MesquiteFile.fileSeparator + ChromaseqUtil.infoFileName; File cFile = new File(filePath); if (cFile.exists()) { if (!cFile.isDirectory()) { if (files[i].endsWith(ChromaseqUtil.processedACESuffix + ".ace")) { // don't do anything } else if (files[i].endsWith(".ace") && !files[i].startsWith(".") && !addingPhrapFailures) { ownerModule.logln("Processing ACE file: " + files[i]); String baseName = files[i].substring( 0, files[i].length() - 4); // this is the name of the sequence processedAceFilePath = aceFileDirectoryPath + MesquiteFile.fileSeparator + baseName + ChromaseqUtil.processedACESuffix + ".ace"; ace = new AceFile( filePath, processedAceFilePath, dataFilePath, dataFilePath, ownerModule, processPolymorphisms, polyThreshold, false); if (ace == null) return; ace.setBaseName(baseName); fullName = new MesquiteString(baseName); voucherCode = new MesquiteString(); ChromaseqInfoFile.processInfoFile(infoFilePath, fullName, voucherCode); String fragmentDirPath = StringUtil.getAllButLastItem( StringUtil.getAllButLastItem( aceFileDirectoryPath, MesquiteFile.fileSeparator), MesquiteFile.fileSeparator); ace.setLongSequenceName(fullName.toString()); if (ace.getNumContigs() >= 1) { processAceFileWithContig( data, ownerModule, processedAceFilePath, fragmentDirPath, ace, null, geneName, fullName, baseName, voucherCode, it); } else { ownerModule.logln(" ACE file contains no contigs!"); if (project != null) { addingPhrapFailures = true; i = 0; ace.createEmptyContigs( MesquiteFile.numFilesEndingWith( aceFileDirectoryPath, files, ".phd.1")); // create an empty contig ace.renameContigs(fullName.toString(), addFragName, geneName); } } if (!addingPhrapFailures) ace.dispose(); } else if (files[i].endsWith(".phd.1") && addingPhrapFailures) { ownerModule.logln(" Importing single-read Phred file " + files[i]); currentRead++; ace.addPhdFileAsSingleReadInContig( currentRead, aceFileDirectoryPath, files[i], processPolymorphisms, polyThreshold); } } } } } } if (addingPhrapFailures && ace != null) { // have to process AceFile that we have manually made MesquiteFile.putFileContents(processedAceFilePath, ace.toString(processPolymorphisms), true); if (project != null) { processAceFileWithoutContig( data, processedAceFilePath, ace, geneName, fullName, it, voucherCode); } ace.dispose(); } }
/*.................................................................................................................*/ public Tree getTrees( TreeVector trees, Taxa taxa, MCharactersDistribution matrix, long seed, MesquiteDouble finalScore) { if (!initializeGetTrees(CategoricalData.class, taxa, matrix)) return null; setTNTSeed(seed); isProtein = data instanceof ProteinData; // David: if isDoomed() then module is closing down; abort somehow // write data file String tempDir = MesquiteFileUtil.createDirectoryForFiles( this, MesquiteFileUtil.IN_SUPPORT_DIR, "TNT", "-Run."); if (tempDir == null) return null; String dataFileName = "data.ss"; // replace this with actual file name? String dataFilePath = tempDir + dataFileName; FileInterpreterI exporter = ZephyrUtil.getFileInterpreter(this, "#InterpretTNT"); if (exporter == null) return null; boolean fileSaved = false; String translationTable = namer.getTranslationTable(taxa); ((InterpretHennig86Base) exporter).setTaxonNamer(namer); fileSaved = ZephyrUtil.saveExportFile(this, exporter, dataFilePath, data, selectedTaxaOnly); if (!fileSaved) return null; String translationFileName = IOUtil.translationTableFileName; setTaxonTranslation(taxa); taxonNumberTranslation = getTaxonNumberTranslation(taxa); namer.setNumberTranslationTable(taxonNumberTranslation); setFileNames(); TaxaSelectionSet outgroupSet = (TaxaSelectionSet) taxa.getSpecsSet(outgroupTaxSetString, TaxaSelectionSet.class); int firstOutgroup = MesquiteInteger.unassigned; if (outgroupSet != null) firstOutgroup = outgroupSet.firstBitOn(); formCommandFile(dataFileName, firstOutgroup); logln("\n\nCommands given to TNT:"); logln(commands); logln(""); MesquiteString arguments = new MesquiteString(); arguments.setValue(" proc " + commandsFileName); String programCommand = externalProcRunner.getExecutableCommand(); int numInputFiles = 3; String[] fileContents = new String[numInputFiles]; String[] fileNames = new String[numInputFiles]; for (int i = 0; i < numInputFiles; i++) { fileContents[i] = ""; fileNames[i] = ""; } fileContents[0] = MesquiteFile.getFileContentsAsString(dataFilePath); fileNames[0] = dataFileName; fileContents[1] = commands; fileNames[1] = commandsFileName; fileContents[2] = translationTable; fileNames[2] = translationFileName; // ----------// boolean success = runProgramOnExternalProcess( programCommand, arguments, fileContents, fileNames, ownerModule.getName()); if (!isDoomed()) { if (success) { desuppressProjectPanelReset(); return retrieveTreeBlock(trees, finalScore); // here's where we actually process everything. } else { if (!beanWritten) postBean("unsuccessful [1]", false); beanWritten = true; } } desuppressProjectPanelReset(); if (data == null) data.decrementEditInhibition(); externalProcRunner.finalCleanup(); return null; }