/**
* Creates a staging file with contents from the given DataMatrix.
*
* @param portalMetadata PortalMetadata
* @param cancerStudyMetadata CancerStudyMetadata
* @param datatypeMetadata DatatypeMetadata
* @param dataMatrix DataMatrix
* @throws Exception
*/
@Override
public void writeStagingFile(
PortalMetadata portalMetadata,
CancerStudyMetadata cancerStudyMetadata,
DatatypeMetadata datatypeMetadata,
DataMatrix dataMatrix)
throws Exception {
// staging file
String stagingFilename = datatypeMetadata.getStagingFilename();
stagingFilename =
stagingFilename.replaceAll(
DatatypeMetadata.CANCER_STUDY_TAG, cancerStudyMetadata.toString());
File stagingFile =
org.apache.commons.io.FileUtils.getFile(
portalMetadata.getStagingDirectory(),
cancerStudyMetadata.getStudyPath(),
stagingFilename);
if (LOG.isInfoEnabled()) {
LOG.info("writingStagingFile(), staging file: " + stagingFile);
}
FileOutputStream out = org.apache.commons.io.FileUtils.openOutputStream(stagingFile, false);
dataMatrix.write(out);
IOUtils.closeQuietly(out);
// meta file
if (datatypeMetadata.requiresMetafile()) {
if (LOG.isInfoEnabled()) {
LOG.info("writingStagingFile(), creating metadata file for staging file: " + stagingFile);
}
writeMetadataFile(portalMetadata, cancerStudyMetadata, datatypeMetadata, dataMatrix);
}
}
/**
* Returns an override file (if it exists) for the given portal & cancer study. The override in
* this case is the override file that a DataMatrix is created from.
*
* <p>Null is returned if an override file is not found.
*
* @param portalMetadata PortalMetadata
* @param cancerStudyMetadata CancerStudyMetadata
* @param filename String
* @return File
* @throws Exception
*/
@Override
public File getOverrideFile(
PortalMetadata portalMetadata, CancerStudyMetadata cancerStudyMetadata, String filename)
throws Exception {
File overrideFile =
org.apache.commons.io.FileUtils.getFile(
portalMetadata.getOverrideDirectory(), cancerStudyMetadata.getStudyPath(), filename);
return (overrideFile.exists()) ? overrideFile : null;
}
/**
* If it exists, moves an override file into the proper location in the given portals staging area
*
* @param portalMetadata PortalMetadata
* @param cancerStudyMetadata CancerStudyMetadata
* @param overrideFilename String
* @param stagingFilename String
* @throws Exception
*/
@Override
public void applyOverride(
PortalMetadata portalMetadata,
CancerStudyMetadata cancerStudyMetadata,
String overrideFilename,
String stagingFilename)
throws Exception {
// check for override file
File overrideFile =
org.apache.commons.io.FileUtils.getFile(
portalMetadata.getOverrideDirectory(),
cancerStudyMetadata.getStudyPath(),
overrideFilename);
if (overrideFile.exists()) {
File stagingFile =
org.apache.commons.io.FileUtils.getFile(
portalMetadata.getStagingDirectory(),
cancerStudyMetadata.getStudyPath(),
stagingFilename);
if (LOG.isInfoEnabled()) {
LOG.info(
"applyOverride(), override file exists for "
+ stagingFile.getCanonicalPath()
+ ": "
+ overrideFile.getCanonicalPath());
}
// copy override file to staging area
if (overrideFile.isFile()) {
org.apache.commons.io.FileUtils.copyFile(overrideFile, stagingFile);
} else {
org.apache.commons.io.FileUtils.copyDirectory(overrideFile, stagingFile);
}
}
}
/**
* Method which writes the cancer study metadata file.
*
* @param portalMetadata PortalMetadata
* @param cancerStudyMetadata CancerStudyMetadata
* @param numCases int
* @throws Exception
*/
@Override
public void writeCancerStudyMetadataFile(
PortalMetadata portalMetadata, CancerStudyMetadata cancerStudyMetadata, int numCases)
throws Exception {
File metaFile =
org.apache.commons.io.FileUtils.getFile(
portalMetadata.getStagingDirectory(),
cancerStudyMetadata.getStudyPath(),
cancerStudyMetadata.getCancerStudyMetadataFilename());
if (LOG.isInfoEnabled()) {
LOG.info("writeMetadataFile(), meta file: " + metaFile);
}
PrintWriter writer =
new PrintWriter(org.apache.commons.io.FileUtils.openOutputStream(metaFile, false));
writer.print("type_of_cancer: " + cancerStudyMetadata.getTumorType() + "\n");
writer.print("cancer_study_identifier: " + cancerStudyMetadata + "\n");
String name =
(cancerStudyMetadata.getName().length() > 0)
? cancerStudyMetadata.getName()
: cancerStudyMetadata.getTumorTypeMetadata().getName();
name =
name.replaceAll(
CancerStudyMetadata.TUMOR_TYPE_NAME_TAG,
cancerStudyMetadata.getTumorTypeMetadata().getName());
writer.print("name: " + name + "\n");
String description = cancerStudyMetadata.getDescription();
description =
description.replaceAll(CancerStudyMetadata.NUM_CASES_TAG, Integer.toString(numCases));
description =
description.replaceAll(
CancerStudyMetadata.TUMOR_TYPE_TAG,
cancerStudyMetadata.getTumorTypeMetadata().getType());
description =
description.replaceAll(
CancerStudyMetadata.TUMOR_TYPE_NAME_TAG,
cancerStudyMetadata.getTumorTypeMetadata().getName());
writer.print("description: " + description + "\n");
if (cancerStudyMetadata.getCitation().length() > 0) {
writer.print("citation: " + cancerStudyMetadata.getCitation() + "\n");
}
if (cancerStudyMetadata.getPMID().length() > 0) {
writer.print("pmid: " + cancerStudyMetadata.getPMID() + "\n");
}
writer.flush();
writer.close();
}
/**
* Creates a staging file for mutation data (and meta file) with contents from the given
* DataMatrix. This is called when the mutation file needs to be run through the Oncotator and
* Mutation Assessor Tools.
*
* @param portalMetadata PortalMetadata
* @param cancerStudy CancerStudyMetadata
* @param datatypeMetadata DatatypeMetadata
* @param dataMatrix DataMatrix
* @throws Exception
*/
@Override
public void writeMutationStagingFile(
PortalMetadata portalMetadata,
CancerStudyMetadata cancerStudyMetadata,
DatatypeMetadata datatypeMetadata,
DataMatrix dataMatrix)
throws Exception {
// we only have data matrix at this point, we need to create a temp with its contents
File oncotatorInputFile =
org.apache.commons.io.FileUtils.getFile(
org.apache.commons.io.FileUtils.getTempDirectory(), "oncotatorInputFile");
FileOutputStream out = org.apache.commons.io.FileUtils.openOutputStream(oncotatorInputFile);
dataMatrix.write(out);
IOUtils.closeQuietly(out);
// output should be the path/name of staging file
String stagingFilename = datatypeMetadata.getStagingFilename();
stagingFilename =
stagingFilename.replaceAll(
DatatypeMetadata.CANCER_STUDY_TAG, cancerStudyMetadata.toString());
File stagingFile =
org.apache.commons.io.FileUtils.getFile(
portalMetadata.getStagingDirectory(),
cancerStudyMetadata.getStudyPath(),
stagingFilename);
// call oncotateAF
oncotateMAF(
FileUtils.FILE_URL_PREFIX + oncotatorInputFile.getCanonicalPath(),
FileUtils.FILE_URL_PREFIX + stagingFile.getCanonicalPath());
// clean up
if (oncotatorInputFile.exists()) {
org.apache.commons.io.FileUtils.forceDelete(oncotatorInputFile);
}
// meta file
if (datatypeMetadata.requiresMetafile()) {
if (LOG.isInfoEnabled()) {
LOG.info(
"writingMutationStagingFile(), creating metadata file for staging file: "
+ stagingFile);
}
writeMetadataFile(portalMetadata, cancerStudyMetadata, datatypeMetadata, dataMatrix);
}
}
/**
* Method which writes a metadata file for the given DatatypeMetadata. DataMatrix may be null.
*
* @param portalMetadata PortalMetadata
* @param cancerStudyMetadata CancerStudyMetadata
* @param datatypeMetadata DatatypeMetadata
* @param dataMatrix DataMatrix
* @throws Exception
*/
@Override
public void writeMetadataFile(
PortalMetadata portalMetadata,
CancerStudyMetadata cancerStudyMetadata,
DatatypeMetadata datatypeMetadata,
DataMatrix dataMatrix)
throws Exception {
File metaFile =
org.apache.commons.io.FileUtils.getFile(
portalMetadata.getStagingDirectory(),
cancerStudyMetadata.getStudyPath(),
datatypeMetadata.getMetaFilename());
if (LOG.isInfoEnabled()) {
LOG.info("writeMetadataFile(), meta file: " + metaFile);
}
PrintWriter writer =
new PrintWriter(org.apache.commons.io.FileUtils.openOutputStream(metaFile, false));
writer.print("cancer_study_identifier: " + cancerStudyMetadata + "\n");
writer.print(
"genetic_alteration_type: " + datatypeMetadata.getMetaGeneticAlterationType() + "\n");
String stableID = datatypeMetadata.getMetaStableID();
stableID =
stableID.replaceAll(DatatypeMetadata.CANCER_STUDY_TAG, cancerStudyMetadata.toString());
writer.print("stable_id: " + stableID + "\n");
writer.print(
"show_profile_in_analysis_tab: "
+ datatypeMetadata.getMetaShowProfileInAnalysisTab()
+ "\n");
String profileDescription = datatypeMetadata.getMetaProfileDescription();
if (dataMatrix != null) {
profileDescription =
profileDescription.replaceAll(
DatatypeMetadata.NUM_GENES_TAG, Integer.toString(dataMatrix.getGeneIDs().size()));
profileDescription =
profileDescription.replaceAll(
DatatypeMetadata.NUM_CASES_TAG, Integer.toString(dataMatrix.getCaseIDs().size()));
}
profileDescription =
profileDescription.replaceAll(
DatatypeMetadata.TUMOR_TYPE_TAG, cancerStudyMetadata.getTumorType());
writer.print("profile_description: " + profileDescription + "\n");
writer.print("profile_name: " + datatypeMetadata.getMetaProfileName() + "\n");
writer.flush();
writer.close();
}
/**
* Create a case list file from the given case list metadata file.
*
* @param portalMetadata PortalMetadata
* @param cancerStudyMetadata CancerStudyMetadata
* @param caseListMetadata CaseListMetadata
* @param caseList String[]
* @throws Exception
*/
@Override
public void writeCaseListFile(
PortalMetadata portalMetadata,
CancerStudyMetadata cancerStudyMetadata,
CaseListMetadata caseListMetadata,
String[] caseList)
throws Exception {
File caseListFile =
org.apache.commons.io.FileUtils.getFile(
portalMetadata.getStagingDirectory(),
cancerStudyMetadata.getStudyPath(),
"case_lists",
caseListMetadata.getCaseListFilename());
if (LOG.isInfoEnabled()) {
LOG.info("writeCaseListFile(), case list file: " + caseListFile.getCanonicalPath());
}
PrintWriter writer =
new PrintWriter(org.apache.commons.io.FileUtils.openOutputStream(caseListFile, false));
writer.print("cancer_study_identifier: " + cancerStudyMetadata + "\n");
String stableID = caseListMetadata.getMetaStableID();
stableID =
stableID.replaceAll(DatatypeMetadata.CANCER_STUDY_TAG, cancerStudyMetadata.toString());
writer.print("stable_id: " + stableID + "\n");
writer.print("case_list_name: " + caseListMetadata.getMetaCaseListName() + "\n");
String caseListDescription = caseListMetadata.getMetaCaseListDescription();
caseListDescription =
caseListDescription.replaceAll(
DatatypeMetadata.NUM_CASES_TAG, Integer.toString(caseList.length));
writer.print("case_list_description: " + caseListDescription + "\n");
writer.print("case_list_category: " + caseListMetadata.getMetaCaseListCategory() + "\n");
writer.print("case_list_ids: ");
for (String caseID : caseList) {
writer.print(caseID + Converter.VALUE_DELIMITER);
}
writer.println();
writer.flush();
writer.close();
}
/**
* Creates a z-score staging file from the given dependencies. It assumes that the dependency -
* staging files have already been created. This code also assumes that the dependencies are
* ordered by cna, then expression.
*
* @param portalMetadata PortalMetadata
* @param cancerStudy CancerStudyMetadata
* @param datatypeMetadata DatatypeMetadata
* @param dependencies DatatypeMetadata[]
* @throws Exception
*/
@Override
public void writeZScoresStagingFile(
PortalMetadata portalMetadata,
CancerStudyMetadata cancerStudyMetadata,
DatatypeMetadata datatypeMetadata,
DatatypeMetadata[] dependencies)
throws Exception {
// sanity check
if (dependencies.length != 2) {
throw new IllegalArgumentException(
"writeZScoresStagingFile(), datatypeMetadatas.length != 2, aborting...");
}
// check for existence of dependencies
if (LOG.isInfoEnabled()) {
LOG.info(
"writeZScoresStagingFile(), checking for existence of dependencies: "
+ Arrays.asList(dependencies));
}
File cnaFile =
org.apache.commons.io.FileUtils.getFile(
portalMetadata.getStagingDirectory(),
cancerStudyMetadata.getStudyPath(),
dependencies[0].getStagingFilename());
if (!cnaFile.exists()) {
if (LOG.isInfoEnabled()) {
LOG.info(
"writeZScoresStagingFile(), cannot find cna file dependency: "
+ cnaFile.getCanonicalPath());
}
return;
}
File expressionFile =
org.apache.commons.io.FileUtils.getFile(
portalMetadata.getStagingDirectory(),
cancerStudyMetadata.getStudyPath(),
dependencies[1].getStagingFilename());
if (!expressionFile.exists()) {
if (LOG.isInfoEnabled()) {
LOG.info(
"writeZScoresStagingFile(), cannot find expression file dependency: "
+ expressionFile.getCanonicalPath());
}
return;
}
// we need a zscore file
File zScoresFile =
org.apache.commons.io.FileUtils.getFile(
portalMetadata.getStagingDirectory(),
cancerStudyMetadata.getStudyPath(),
datatypeMetadata.getStagingFilename());
// call NormalizeExpressionLevels
String[] args = {
cnaFile.getCanonicalPath(), expressionFile.getCanonicalPath(), zScoresFile.getCanonicalPath()
};
if (LOG.isInfoEnabled()) {
LOG.info(
"writingZScoresStagingFlie(), calling NormalizeExpressionLevels: "
+ Arrays.toString(args));
}
try {
NormalizeExpressionLevels.driver(args);
} catch (RuntimeException e) {
// houston we have a problem...
if (LOG.isInfoEnabled()) {
LOG.info(
"writingZScoresStagingFlie(), exception thrown by NormalizeExpressionLevels: "
+ e.getMessage()
+ ", aborting...");
}
if (zScoresFile.exists()) {
org.apache.commons.io.FileUtils.forceDelete(zScoresFile);
}
return;
}
// meta file
if (datatypeMetadata.requiresMetafile()) {
if (LOG.isInfoEnabled()) {
LOG.info(
"writingZScoresStagingFile(), creating metadata file for staging file: "
+ zScoresFile.getCanonicalPath());
}
writeMetadataFile(portalMetadata, cancerStudyMetadata, datatypeMetadata, null);
}
}
/**
* Get the case list from the staging file.
*
* @param caseIDs CaseIDs;
* @param portalMetadata PortalMetadata
* @param cancerStudyMetadata CancerStudyMetadata
* @param stagingFilename String
* @return List<String>
* @throws Exception
*/
@Override
public List<String> getCaseListFromStagingFile(
CaseIDs caseIDs,
PortalMetadata portalMetadata,
CancerStudyMetadata cancerStudyMetadata,
String stagingFilename)
throws Exception {
if (LOG.isInfoEnabled()) {
LOG.info("getCaseListFromStagingFile(): " + stagingFilename);
}
// we use set here
HashSet<String> caseSet = new HashSet<String>();
// staging file
File stagingFile =
org.apache.commons.io.FileUtils.getFile(
portalMetadata.getStagingDirectory(),
cancerStudyMetadata.getStudyPath(),
stagingFilename);
// sanity check
if (!stagingFile.exists()) {
return new ArrayList<String>();
}
// iterate over all rows in file
org.apache.commons.io.LineIterator it =
org.apache.commons.io.FileUtils.lineIterator(stagingFile);
try {
int mafCaseIDColumnIndex = 0;
boolean processHeader = true;
while (it.hasNext()) {
// create a string list from row in file
List<String> thisRow = Arrays.asList(it.nextLine().split(Converter.VALUE_DELIMITER));
// is this the header file?
if (processHeader) {
// look for MAF file case id column header
mafCaseIDColumnIndex = thisRow.indexOf(Converter.MUTATION_CASE_ID_COLUMN_HEADER);
// this is not a MAF file, header contains the case ids, return here
if (mafCaseIDColumnIndex == -1) {
for (String potentialCaseID : thisRow) {
if (caseIDs.isTumorCaseID(potentialCaseID)) {
caseSet.add(caseIDs.convertCaseID(potentialCaseID));
}
}
break;
}
processHeader = false;
continue;
}
// we want to add the value at mafCaseIDColumnIndex into return set - this is a case ID
String potentialCaseID = thisRow.get(mafCaseIDColumnIndex);
if (caseIDs.isTumorCaseID(potentialCaseID)) {
caseSet.add(caseIDs.convertCaseID(potentialCaseID));
}
}
} finally {
it.close();
}
// outta here
return new ArrayList<String>(caseSet);
}
/**
* Copy's the given portal's seg files to location used for linking to IGV from cBio Portal web
* site.
*
* @param portalMetadata PortalMetadata
* @param datatypeMetadata DatataypeMetadata
* @param remoteUserName String
* @throws Exception
*/
@Override
public void copySegFiles(
PortalMetadata portalMetadata, DatatypeMetadata datatypeMetadata, String remoteUserName)
throws Exception {
if (LOG.isInfoEnabled()) {
LOG.info("copySegFiles()");
}
// check args
if (portalMetadata == null || remoteUserName == null) {
throw new IllegalArgumentException("portal or remoteUserName must not be null");
}
// seg file location
URL segFileLocation = portalMetadata.getIGVSegFileLinkingLocation();
// we need this to determine location
Collection<DataSourcesMetadata> dataSourcesMetadata = config.getDataSourcesMetadata(Config.ALL);
// iterate over all cancer studies
for (CancerStudyMetadata cancerStudyMetadata :
config.getCancerStudyMetadata(portalMetadata.getName())) {
// lets determine if cancer study is in staging directory or studies directory
String rootDirectory =
MetadataUtils.getCancerStudyRootDirectory(
portalMetadata, dataSourcesMetadata, cancerStudyMetadata);
if (rootDirectory == null) {
if (LOG.isInfoEnabled()) {
LOG.info(
"loadStagingFiles(), cannot find root directory for study: "
+ cancerStudyMetadata
+ " skipping...");
}
continue;
}
// construct staging filename for seg
String sourceFilename =
(rootDirectory
+ File.separator
+ cancerStudyMetadata.getStudyPath()
+ File.separator
+ datatypeMetadata.getStagingFilename());
sourceFilename =
sourceFilename.replaceAll(
DatatypeMetadata.CANCER_STUDY_TAG, cancerStudyMetadata.toString());
String destinationFilename =
datatypeMetadata
.getStagingFilename()
.replaceAll(DatatypeMetadata.CANCER_STUDY_TAG, cancerStudyMetadata.toString());
String[] command =
new String[] {
"scp",
sourceFilename,
remoteUserName
+ "@"
+ segFileLocation.getHost()
+ ":"
+ segFileLocation.getFile()
+ destinationFilename
};
if (LOG.isInfoEnabled()) {
LOG.info("executing: " + Arrays.asList(command));
}
if (Shell.exec(Arrays.asList(command), ".")) {
if (LOG.isInfoEnabled()) {
LOG.info("copy successful.");
}
} else if (LOG.isInfoEnabled()) {
LOG.info("copy unsucessful.");
}
}
}