/** {@inheritDoc} */
@Override
public void process(Reader reader) throws Exception {
// Data has format:
// id | description
@SuppressWarnings("rawtypes")
Iterator lineIter = FormattedTextParser.parseTabDelimitedReader(reader);
int count = 0;
while (lineIter.hasNext()) {
String[] line = (String[]) lineIter.next();
try {
String entrez = line[0];
String description = line[1];
LOG.error("description " + count++ + " " + description);
if (!StringUtils.isBlank(description)) {
Item gene = createItem("Gene");
gene.setAttribute("primaryIdentifier", entrez);
gene.setAttribute("description", description);
gene.setReference("organism", getOrganism(HUMAN_TAXON_ID));
store(gene);
}
} catch (IndexOutOfBoundsException e) {
LOG.info("Failed to read line: " + Arrays.asList(line));
}
}
}
protected void createFromFile(File f) throws IOException {
// data is in format:
// ZDBID ID1,ID2,ID3
Iterator<?> lineIter =
FormattedTextParser.parseTabDelimitedReader(new BufferedReader(new FileReader(f)));
while (lineIter.hasNext()) {
String[] line = (String[]) lineIter.next();
if (line.length < 2 || line[0].startsWith("#") || !line[0].startsWith(GENE_PATTERN)) {
continue;
}
String zfinId = line[0];
String[] synonyms = StringUtil.split(line[1].trim(), ",");
resolver.addMainIds(taxonId, zfinId, Collections.singleton(zfinId));
resolver.addSynonyms(taxonId, zfinId, new HashSet<String>(Arrays.asList(synonyms)));
}
}
/**
* Process Topo files and create two maps.
*
* @param file a topo file.
* @return Map<key:primaryId/fullName, value:Map<key:symbol/symbol, value:level/position>>
*/
private Map<String, Map<String, String>> processTopoFile(File file) {
Map<String, Map<String, String>> topoMap = new HashMap<String, Map<String, String>>();
try {
Reader reader = new FileReader(file);
Iterator<?> tsvIter;
try {
tsvIter = FormattedTextParser.parseTabDelimitedReader(reader);
} catch (IOException e) {
e.printStackTrace();
throw new RuntimeException(e);
}
while (tsvIter.hasNext()) {
String[] line = (String[]) tsvIter.next();
if (line.length > 1) { // the file could end with an empty line
if ("topos_tf.tsv".equals(file.getName())) {
String symbol = line[0];
String primaryId = line[1];
String level = line[2];
Map<String, String> aMap = new HashMap<String, String>();
aMap.put(symbol, level);
topoMap.put(primaryId, aMap);
} else if ("topos_mirna.tsv".equals(file.getName())) {
String symbol = line[0];
String fullName = line[1];
String position = line[2];
Map<String, String> aMap = new HashMap<String, String>();
aMap.put(symbol, position);
topoMap.put(fullName, aMap);
}
}
}
} catch (FileNotFoundException e) {
e.printStackTrace();
throw new RuntimeException(e);
}
return topoMap;
}
public void processJournals(Reader reader) throws Exception {
Iterator lineIter = FormattedTextParser.parseDelimitedReader(reader, '|');
while (lineIter.hasNext()) {
String[] line = (String[]) lineIter.next();
if (line.length < 2) {
throw new RuntimeException(
"Journal line does not have enough elements: " + line.length + line[0]);
}
String primaryIdentifier = line[0];
String name = line[1];
String abbrev = line[2];
String publisher = line[3];
Item journal = getJournal(primaryIdentifier);
if (!StringUtils.isEmpty(name)) {
journal.setAttribute("name", name);
}
if (!StringUtils.isEmpty(abbrev)) {
journal.setAttribute("abbrev", abbrev);
}
if (!StringUtils.isEmpty(publisher)) {
journal.setAttribute("publisher", publisher);
}
if (journal.getAttribute("primaryIdentifier").getValue().equals("ZDB-JRNL-050621-1000")) {
System.out.println("storing: ZDB-JRNL-050621-1000");
}
try {
store(journal);
} catch (ObjectStoreException e) {
throw new SAXException(e);
}
}
}
/**
* Process all rows of the map_title.tab file
*
* @param reader a reader for the map_title.tab file
* @throws IOException
* @throws ObjectStoreException
*/
private void processMapTitleFile(Reader reader) throws IOException, ObjectStoreException {
Iterator lineIter = FormattedTextParser.parseTabDelimitedReader(reader);
// this file has data of the format:
// pathway id | pathway name
while (lineIter.hasNext()) {
// line is a string array with the one element for each tab separated value
// on the next line of the file
String[] line = (String[]) lineIter.next();
String pathwayId = line[0];
String pathwayName = line[1];
// getPathway will create an Item or fetch it from a map if seen before
Item pathway = getPathway(pathwayId);
pathway.setAttribute("name", pathwayName);
// once we have set the pathway name that is all the information needed so we can store
store(pathway);
}
}
/**
* @param reader
* @throws Exception
* @throws ObjectStoreException
*/
private void processHalfLifeDataFile(Reader preader) throws Exception, ObjectStoreException {
/*
* Sample line
* protein half-life YMR028W 8.2 min 0.1 hr 25466257
* protein half-life YLR110C 10.0 min 0.2 hr 25466257
*/
System.out.println("Processing Protien HalfLife Data file....");
Iterator<?> tsvIter;
try {
tsvIter = FormattedTextParser.parseTabDelimitedReader(preader);
} catch (Exception e) {
throw new BuildException("cannot parse file: " + getCurrentFile(), e);
}
while (tsvIter.hasNext()) {
String[] line = (String[]) tsvIter.next();
if (line.length < 5) {
LOG.error("Couldn't process line. Expected 6 cols, but was " + line.length);
continue;
}
String experiment = line[0].trim();
String protein = line[1].trim();
String value = line[2].trim();
String units = line[3].trim();
// String valueMinutes = line[2].trim();
// String stringMinutes = line[3].trim();
// String valueHours = line[4].trim();
// String stringHours = line[5].trim();
String pmid = line[4].trim();
newProduct(experiment, protein, value, units, pmid);
}
preader.close();
}
/** {@inheritDoc} */
public void process(Reader reader) throws Exception {
// Create a chromosome
Item chromosome = createItem("Chromosome");
chromosome.setAttribute("primaryIdentifier", CHROMOSOME_PID);
store(chromosome);
@SuppressWarnings("rawtypes")
Iterator lineIter = FormattedTextParser.parseTabDelimitedReader(reader);
while (lineIter.hasNext()) {
String[] line = (String[]) lineIter.next();
// remove header line
if (!line[0].equals(HEADER_LINE)) {
String ecogeneId = line[0];
String geneName = line[1];
String eCK = line[2];
String swissProtId = line[3];
String wisconsinGenBankId = line[4];
String genBankProteinId = line[5];
String genoBaseId = line[6];
String type = line[7];
String strand = line[8];
String start = line[9];
String end = line[10];
String synonym = line[11];
Set<String> symSet = new TreeSet<String>();
if (!eCK.equals(NULL_STRING)) {
symSet.add(eCK);
}
if (!genoBaseId.equals(NULL_STRING)) {
symSet.addAll(Arrays.asList(StringUtil.split(genoBaseId, "; ")));
}
if (!synonym.equals(NONE_STRING)) {
symSet.addAll(Arrays.asList(synonym.split(", ")));
}
if (type.equals(TYPE_GENE)) {
Item gene = createItem("Gene");
gene.setReference("chromosome", chromosome);
gene.setReference("organism", getOrganism(ECOLI_TAXON));
gene.setAttribute("primaryIdentifier", ecogeneId);
gene.setAttribute("secondaryIdentifier", wisconsinGenBankId);
gene.setAttribute("name", geneName);
gene.setAttribute("symbol", geneName);
if (symSet.size() > 0) {
for (String sym : symSet) {
createSynonym(gene, sym, true);
}
}
if (!swissProtId.equals(NULL_STRING)) {
if (proteinMap.containsKey(swissProtId)) {
// Reference a protein to a gene (a gene has proteins
// collection)
gene.addToCollection("proteins", proteinMap.get(swissProtId));
} else {
Item protein = createItem("Protein");
protein.setAttribute("primaryAccession", swissProtId);
// NCBI Protein id, remove "g"
protein.setAttribute("secondaryIdentifier", genBankProteinId.substring(1));
gene.addToCollection("proteins", protein);
store(protein);
proteinMap.put(swissProtId, protein);
}
}
// Create chromosome location
if (start.matches(DIGIT_REGEX) && end.matches(DIGIT_REGEX)) {
Item location = createItem("Location");
location.setAttribute("start", start);
location.setAttribute("end", end);
location.setReference("feature", gene);
location.setReference("locatedOn", chromosome);
if (strand.equals(CLOCKWISE)) {
location.setAttribute("strand", "+1");
} else if (strand.equals(COUNTER_CLOCKWISE)) {
location.setAttribute("strand", "-1");
} else {
location.setAttribute("strand", "0");
}
gene.setReference("chromosomeLocation", location);
store(location);
}
store(gene);
} else if (type.equals(TYPE_RNA)) { // TODO code refactory
Item rna = createItem("NcRNA");
rna.setReference("chromosome", chromosome);
rna.setReference("organism", getOrganism(ECOLI_TAXON));
rna.setAttribute("primaryIdentifier", ecogeneId);
rna.setAttribute("secondaryIdentifier", wisconsinGenBankId);
rna.setAttribute("name", geneName);
rna.setAttribute("symbol", geneName);
if (symSet.size() > 0) {
for (String sym : symSet) {
createSynonym(rna, sym, true);
}
}
// Create chromosome location
if (start.matches(DIGIT_REGEX) && end.matches(DIGIT_REGEX)) {
Item location = createItem("Location");
location.setAttribute("start", start);
location.setAttribute("end", end);
location.setReference("feature", rna);
location.setReference("locatedOn", chromosome);
if (strand.equals(CLOCKWISE)) {
location.setAttribute("strand", "+1");
} else if (strand.equals(COUNTER_CLOCKWISE)) {
location.setAttribute("strand", "-1");
} else {
location.setAttribute("strand", "0");
}
rna.setReference("chromosomeLocation", location);
store(location);
}
store(rna);
}
}
}
}
/**
* Process the edge data file.
*
* @param file the edge data file
* @param tfMap a customized map with TF information
* @param miRNAMap a customized map with miRNA information
*/
private void processEdgeFile(
File file,
Map<String, Map<String, String>> tfMap,
Map<String, Map<String, String>> miRNAMap) {
try {
Reader reader = new FileReader(file);
Iterator<?> tsvIter;
try {
tsvIter = FormattedTextParser.parseTabDelimitedReader(reader);
} catch (IOException e) {
e.printStackTrace();
throw new RuntimeException(e);
}
while (tsvIter.hasNext()) {
String[] line = (String[]) tsvIter.next();
if (line.length > 1) {
String sourceIdentifier = line[0];
String targetIdentifier = line[1];
try {
if (tfMap.containsKey(sourceIdentifier)) {
// Create source gene
String sourceSymbol = tfMap.get(sourceIdentifier).keySet().iterator().next();
String sourceLevel = tfMap.get(sourceIdentifier).get(sourceSymbol);
String sourceGenePid = createGene(sourceIdentifier, sourceSymbol);
// Create networkProperty for source gene
Item sourceNetworkProperty = createNetworkProperty(TOPO_TYPE_LEVEL, sourceLevel);
sourceNetworkProperty.setReference("node", geneItems.get(sourceGenePid));
store(sourceNetworkProperty);
if (tfMap.containsKey(targetIdentifier)) {
// Create regulation for both genes
Item regulation = createRegulation(INTERACTION_TYPE_TF_TF);
// Create target gene
String targetSymbol = tfMap.get(targetIdentifier).keySet().iterator().next();
String targetLevel = tfMap.get(targetIdentifier).get(targetSymbol);
String targetGenePid = createGene(targetIdentifier, targetSymbol);
// Create networkProperty for target gene
Item targetNetworkProperty = createNetworkProperty(TOPO_TYPE_LEVEL, targetLevel);
targetNetworkProperty.setReference("node", geneItems.get(targetGenePid));
store(targetNetworkProperty);
regulation.setReference("source", geneItems.get(sourceGenePid));
regulation.setReference("target", geneItems.get(targetGenePid));
store(regulation);
} else if (miRNAMap.containsKey(targetIdentifier)) {
// Create regulation for both genes
Item regulation = createRegulation(INTERACTION_TYPE_TF_MIRNA);
// Create target gene
String targetSymbol = miRNAMap.get(targetIdentifier).keySet().iterator().next();
String targetPosition = miRNAMap.get(targetIdentifier).get(targetSymbol);
String targetGenePid = createGene(null, targetSymbol);
// Create networkProperty for target gene
Item targetNetworkProperty =
createNetworkProperty(TOPO_TYPE_POSITION, targetPosition);
targetNetworkProperty.setReference("node", geneItems.get(targetGenePid));
store(targetNetworkProperty);
regulation.setReference("source", geneItems.get(sourceGenePid));
regulation.setReference("target", geneItems.get(targetGenePid));
store(regulation);
} else {
continue;
}
} else if (miRNAMap.containsKey(sourceIdentifier)) {
// Create source gene
String sourceSymbol = miRNAMap.get(sourceIdentifier).keySet().iterator().next();
String sourcePosition = miRNAMap.get(sourceIdentifier).get(sourceSymbol);
String sourceGenePid = createGene(null, sourceSymbol);
// Create networkProperty for source gene
Item sourceNetworkProperty =
createNetworkProperty(TOPO_TYPE_POSITION, sourcePosition);
sourceNetworkProperty.setReference("node", geneItems.get(sourceGenePid));
store(sourceNetworkProperty);
if (tfMap.containsKey(targetIdentifier)) {
// Create regulation for both genes
Item regulation = createRegulation(INTERACTION_TYPE_MIRNA_TF);
// Create target gene
String targetSymbol = tfMap.get(targetIdentifier).keySet().iterator().next();
String targetLevel = tfMap.get(targetIdentifier).get(targetSymbol);
String targetGenePid = createGene(targetIdentifier, targetSymbol);
// Create networkProperty for target gene
Item targetNetworkProperty = createNetworkProperty(TOPO_TYPE_LEVEL, targetLevel);
targetNetworkProperty.setReference("node", geneItems.get(targetGenePid));
store(targetNetworkProperty);
regulation.setReference("source", geneItems.get(sourceGenePid));
regulation.setReference("target", geneItems.get(targetGenePid));
store(regulation);
} else if (miRNAMap.containsKey(targetIdentifier)) {
// Create regulation for both genes
Item regulation = createRegulation(INTERACTION_TYPE_MIRNA_MIRNA);
// Create target gene
String targetSymbol = miRNAMap.get(targetIdentifier).keySet().iterator().next();
String targetPosition = miRNAMap.get(targetIdentifier).get(targetSymbol);
String targetGenePid = createGene(null, targetSymbol);
// Create networkProperty for target gene
Item targetNetworkProperty =
createNetworkProperty(TOPO_TYPE_POSITION, targetPosition);
targetNetworkProperty.setReference("node", geneItems.get(targetGenePid));
store(targetNetworkProperty);
regulation.setReference("source", geneItems.get(sourceGenePid));
regulation.setReference("target", geneItems.get(targetGenePid));
store(regulation);
} else {
continue;
}
} else {
continue;
}
} catch (ObjectStoreException e) {
e.printStackTrace();
throw new RuntimeException(e);
}
}
}
} catch (FileNotFoundException e) {
e.printStackTrace();
throw new RuntimeException(e);
}
}
/**
* Process the csv file
*
* @param reader the Reader
* @see DataConverter#process
* @throws Exception if something goes wrong
*/
@Override
public void process(Reader reader) throws Exception {
if (rslv == null) {
rslv = IdResolverService.getFlyIdResolver();
}
Iterator<String[]> it = FormattedTextParser.parseTabDelimitedReader(reader);
while (it.hasNext()) {
String[] lineBits = it.next();
String geneCG = lineBits[0];
if (!geneCG.startsWith("CG")) {
// ignore clones for now
continue;
}
// Try to create/fetch gene, if null the IdResolver failed so do nothing for this row
Item gene = getGene(geneCG);
if (gene == null) {
continue;
}
String stage = lineBits[1];
String resultKey = geneCG + stage;
Item result = getResult(resultKey, gene.getIdentifier(), pub.getIdentifier(), stage);
Integer stageNumber = null;
try {
stageNumber = new Integer(stage);
} catch (NumberFormatException e) {
// bad line in file, just keep going
continue;
}
result.setAttribute("stageRange", STAGE_LABELS[stageNumber.intValue()] + " (BDGP in situ)");
if (lineBits.length > 2) {
String image = lineBits[2];
if (StringUtils.isNotEmpty(image)) {
setImage(result, URL + image);
}
}
if (lineBits.length > 3) {
String term = lineBits[3];
Item termItem = getTerm(term);
if (termItem != null) {
result.addToCollection("mRNAExpressionTerms", termItem);
}
if ("no staining".equals(term)) {
result.setAttribute("expressed", "false");
}
}
}
for (Item result : results.values()) {
if (!result.hasCollection("mRNAExpressionTerms")
|| result.getCollection("mRNAExpressionTerms").getRefIds().isEmpty()) {
result.setAttribute("expressed", "false");
}
}
storeAll(imgs);
storeAll(results);
}