private String getEvidence() throws ObjectStoreException {
if (evidenceRefId == null) {
Item item = createItem("OrthologueEvidenceCode");
item.setAttribute("abbreviation", EVIDENCE_CODE_ABBR);
item.setAttribute("name", EVIDENCE_CODE_NAME);
try {
store(item);
} catch (ObjectStoreException e) {
throw new ObjectStoreException(e);
}
String refId = item.getIdentifier();
item = createItem("OrthologueEvidence");
item.setReference("evidenceCode", refId);
try {
store(item);
} catch (ObjectStoreException e) {
throw new ObjectStoreException(e);
}
evidenceRefId = item.getIdentifier();
}
return evidenceRefId;
}
/**
* Create and store a NetworkProperty item on the first time called.
*
* @param type e.g. "level", "position"
* @param value the value of level or position
* @return an Item representing the NetworkProperty
*/
private Item createNetworkProperty(String type, String value) throws ObjectStoreException {
Item networkproperty = createItem("NetworkProperty");
networkproperty.setAttribute("type", type);
networkproperty.setAttribute("value", value);
return networkproperty;
}
/** {@inheritDoc} */
@Override
public void process(Reader reader) throws Exception {
// Data has format:
// id | description
@SuppressWarnings("rawtypes")
Iterator lineIter = FormattedTextParser.parseTabDelimitedReader(reader);
int count = 0;
while (lineIter.hasNext()) {
String[] line = (String[]) lineIter.next();
try {
String entrez = line[0];
String description = line[1];
LOG.error("description " + count++ + " " + description);
if (!StringUtils.isBlank(description)) {
Item gene = createItem("Gene");
gene.setAttribute("primaryIdentifier", entrez);
gene.setAttribute("description", description);
gene.setReference("organism", getOrganism(HUMAN_TAXON_ID));
store(gene);
}
} catch (IndexOutOfBoundsException e) {
LOG.info("Failed to read line: " + Arrays.asList(line));
}
}
}
private Integer createGoAnnotation(
String productIdentifier,
String productType,
String termIdentifier,
Item organism,
String qualifier,
String withText,
String dataSourceCode)
throws ObjectStoreException {
Item goAnnotation = createItem(annotationClassName);
goAnnotation.setReference("subject", productIdentifier);
goAnnotation.setReference("ontologyTerm", termIdentifier);
if (!StringUtils.isEmpty(qualifier)) {
goAnnotation.setAttribute("qualifier", qualifier);
}
// with objects
if (!StringUtils.isEmpty(withText)) {
goAnnotation.setAttribute("withText", withText);
List<String> with = createWithObjects(withText, organism, dataSourceCode);
if (!with.isEmpty()) {
goAnnotation.addCollection(new ReferenceList("with", with));
}
}
goAnnotation.addToCollection("dataSets", getDataset(dataSourceCode));
if ("gene".equals(productType)) {
addProductCollection(productIdentifier, goAnnotation.getIdentifier());
}
Integer storedAnnotationId = store(goAnnotation);
return storedAnnotationId;
}
/**
* Create and store a BioEntity item on the first time called.
*
* @param type gene
* @param primaryId the gene primaryIdentifier
* @param symbol the gene symbol
* @throws ObjectStoreException
*/
private void createBioEntity(String type, String primaryId, String symbol)
throws ObjectStoreException {
Item bioentity = null;
bioentity = createItem("Gene");
bioentity.setAttribute("primaryIdentifier", primaryId);
bioentity.setAttribute("symbol", symbol);
store(bioentity);
geneItems.put(primaryId, bioentity.getIdentifier());
}
private void setStages() throws ObjectStoreException {
Item item = createItem("Ontology");
item.setAttribute("name", "Fly Development");
store(item);
stages = new String[17];
for (int i = 1; i <= 16; i++) {
Item stage = createItem("DevelopmentTerm");
stage.setAttribute("name", "embryonic stage " + i);
stage.setReference("ontology", item);
stages[i] = stage.getIdentifier();
store(stage);
}
}
private void setSynonym(String subjectRefId, String type, String value) throws SAXException {
String key = subjectRefId + type + value;
if (!synonyms.contains(key)) {
Item synonym = createItem("Synonym");
synonym.setAttribute("type", type);
synonym.setAttribute("value", value);
synonym.setReference("subject", subjectRefId);
synonyms.add(key);
try {
store(synonym);
} catch (ObjectStoreException e) {
throw new SAXException(e);
}
}
}
private void setNames(
List<?> names,
String symbol,
List<String> synonyms,
Set<Item> synonymsToAdd,
String primaryIdentifier,
Item feature,
ClassDescriptor cd) {
if (cd.getFieldDescriptorByName("symbol") == null) { // if symbol is not in the model
String name = (String) names.get(0);
feature.setAttribute("name", name);
for (Iterator<?> i = names.iterator(); i.hasNext(); ) {
String recordName = (String) i.next();
if (!recordName.equals(primaryIdentifier) && !recordName.equals(name)) {
synonymsToAdd.add(getSynonym(feature, recordName));
}
}
if (synonyms != null) {
for (Iterator<?> i = synonyms.iterator(); i.hasNext(); ) {
String recordName = (String) i.next();
if (!recordName.equals(primaryIdentifier) && !recordName.equals(name)) {
synonymsToAdd.add(getSynonym(feature, recordName));
}
}
}
} else {
if (symbol == null) {
symbol = (String) names.get(0);
}
feature.setAttribute("symbol", symbol);
for (Iterator<?> i = names.iterator(); i.hasNext(); ) {
String recordName = (String) i.next();
if (!recordName.equals(primaryIdentifier) && !recordName.equals(symbol)) {
synonymsToAdd.add(getSynonym(feature, recordName));
}
}
if (synonyms != null) {
for (Iterator<?> i = synonyms.iterator(); i.hasNext(); ) {
String recordName = (String) i.next();
if (!recordName.equals(primaryIdentifier) && !recordName.equals(symbol)) {
synonymsToAdd.add(getSynonym(feature, recordName));
}
}
}
}
}
/**
* Produce random data.
*
* <p>{@inheritDoc}
*/
public void process(Reader inputFile) throws Exception {
Random random = new Random();
Set doneValues = new HashSet();
long time = System.currentTimeMillis();
for (int i = 0; i < count / 2; i++) {
Item itemTo = createItem("ReferenceTo2");
Item itemFrom = createItem("ReferenceFrom2");
Integer firstInt;
do {
firstInt = new Integer(random.nextInt());
} while (doneValues.contains(firstInt));
doneValues.add(firstInt);
itemTo.setAttribute("att", "" + firstInt);
itemFrom.setReference("ref", itemTo.getIdentifier());
getItemWriter().store(ItemHelper.convert(itemFrom));
getItemWriter().store(ItemHelper.convert(itemTo));
}
long now = System.currentTimeMillis();
LOG.info(
"Finished generating "
+ count
+ " objects at "
+ ((60000L * count) / (now - time))
+ " objects per minute ("
+ (now - time)
+ " ms total)");
}
private String getGene(String identifierType, String id, String taxonId)
throws ObjectStoreException {
String identifier = id;
if (rslv != null && rslv.hasTaxon(taxonId)) {
identifier = resolveGene(identifier, taxonId);
if (identifier == null) {
return null;
}
}
String refId = identifiersToGenes.get(identifier);
if (refId == null) {
Item gene = createItem("Gene");
refId = gene.getIdentifier();
gene.setAttribute(identifierType, identifier);
gene.setReference("organism", getOrganism(taxonId));
identifiersToGenes.put(identifier, refId);
try {
store(gene);
} catch (ObjectStoreException e) {
throw new ObjectStoreException(e);
}
}
return refId;
}
private Item getGene(String geneCG) throws ObjectStoreException {
if (rslv == null || !rslv.hasTaxon(TAXON_FLY)) {
return null;
}
int resCount = rslv.countResolutions(TAXON_FLY, geneCG);
if (resCount != 1) {
LOG.info(
"RESOLVER: failed to resolve gene to one identifier, ignoring gene: "
+ geneCG
+ " count: "
+ resCount
+ " FBgn: "
+ rslv.resolveId(TAXON_FLY, geneCG));
return null;
}
String primaryIdentifier = rslv.resolveId(TAXON_FLY, geneCG).iterator().next();
if (genes.containsKey(primaryIdentifier)) {
return genes.get(primaryIdentifier);
}
Item gene = createItem("Gene");
gene.setAttribute("primaryIdentifier", primaryIdentifier);
gene.setReference("organism", orgDrosophila);
genes.put(primaryIdentifier, gene);
store(gene);
return gene;
}
/**
* Return the organism Item created for this GFF3Converter from the organism abbreviation passed
* to the constructor.
*
* @return the organism item
* @throws ObjectStoreException if the Organism item can't be stored
*/
public Item getOrganism() throws ObjectStoreException {
if (organism == null) {
organism = createItem("Organism");
organism.setAttribute("taxonId", orgTaxonId);
store(organism);
}
return organism;
}
/**
* Construct a new instance of BDGPInsituConverter.
*
* @param model the Model
* @param writer the ItemWriter used to handle the resultant items
* @throws ObjectStoreException if an error occurs in storing
*/
public BDGPInsituConverter(ItemWriter writer, Model model) throws ObjectStoreException {
super(writer, model, "BDGP", "BDGP in situ data set");
orgDrosophila = createItem("Organism");
orgDrosophila.setAttribute("taxonId", TAXON_FLY);
store(orgDrosophila);
pub = createItem("Publication");
pub.setAttribute("pubMedId", "17645804");
store(pub);
setStages();
ontology = createItem("Ontology");
ontology.setAttribute("name", "ImaGO");
store(ontology);
}
private void setImage(Item result, String img) {
if (!imgs.containsKey(img)) {
Item item = createItem("Image");
item.setAttribute("url", img);
imgs.put(img, item);
result.addToCollection("images", item.getIdentifier());
}
}
private void storeEvidenceCode(String code) throws ObjectStoreException {
if (evidenceCodes.get(code) == null) {
Item item = createItem("DOEvidenceCode");
item.setAttribute("code", code);
evidenceCodes.put(code, item.getIdentifier());
store(item);
}
}
/**
* Create a new pathway Item or fetch from a map if it has been seen before
*
* @param pathwayId the id of a KEGG pathway to look up
* @return an Item representing the pathway
*/
private Item getPathway(String pathwayId) {
Item pathway = pathwayMap.get(pathwayId);
if (pathway == null) {
pathway = createItem("Pathway");
pathway.setAttribute("identifier", pathwayId);
pathwayMap.put(pathwayId, pathway);
}
return pathway;
}
private Item newOrganism(String taxonId) throws ObjectStoreException {
Item item = organisms.get(taxonId);
if (item == null) {
item = createItem("Organism");
item.setAttribute("taxonId", taxonId);
organisms.put(taxonId, item);
store(item);
}
return item;
}
/**
* @param modSite
* @param modType
* @param source
* @param pmid
* @return
*/
private Item getProteinHalfLife(String experiment, String value, String units, String pmid)
throws ObjectStoreException {
Item item = createItem("ProteinHalfLife");
if (StringUtils.isNotEmpty(experiment)) {
item.setAttribute("experiment", experiment);
}
if (StringUtils.isNotEmpty(value)) {
item.setAttribute("value", value);
}
if (StringUtils.isNotEmpty(units)) {
item.setAttribute("units", units);
}
item.setAttribute("source", "SGD");
Item publication = pubmedIdMap.get(pmid);
if (publication == null) {
publication = createItem("Publication");
publication.setAttribute("pubMedId", pmid);
pubmedIdMap.put(pmid, publication);
item.setReference("publication", publication);
try {
store(publication);
} catch (ObjectStoreException e) {
throw new ObjectStoreException(e);
}
} else {
item.setReference("publication", publication);
}
try {
store(item);
} catch (ObjectStoreException e) {
throw new ObjectStoreException(e);
}
return item;
}
/* @Override necessary */
public void process(GFF3Record record) {
Item feature = getFeature();
String clsName = feature.getClassName();
if ("Gene".equals(clsName)) {
// move Gene.primaryIdentifier to Gene.secondaryIdentifier
// move Gene.symbol to Gene.primaryIdentifier
if (feature.getAttribute("primaryIdentifier") != null) {
String secondary = feature.getAttribute("primaryIdentifier").getValue();
feature.setAttribute("secondaryIdentifier", secondary);
}
if (feature.getAttribute("symbol") != null) {
String primary = feature.getAttribute("symbol").getValue();
feature.setAttribute("primaryIdentifier", primary);
feature.removeAttribute("symbol");
}
}
}
private void processMeshTerms(Item publication, List<String> newTerms) {
for (String name : newTerms) {
Item item = meshTerms.get(name);
if (item == null) {
item = itemFactory.makeItemForClass("MeshTerm");
item.setAttribute("name", name);
meshTerms.put(name, item);
}
publication.addToCollection("meshTerms", item);
}
}
public void processJournals(Reader reader) throws Exception {
Iterator lineIter = FormattedTextParser.parseDelimitedReader(reader, '|');
while (lineIter.hasNext()) {
String[] line = (String[]) lineIter.next();
if (line.length < 2) {
throw new RuntimeException(
"Journal line does not have enough elements: " + line.length + line[0]);
}
String primaryIdentifier = line[0];
String name = line[1];
String abbrev = line[2];
String publisher = line[3];
Item journal = getJournal(primaryIdentifier);
if (!StringUtils.isEmpty(name)) {
journal.setAttribute("name", name);
}
if (!StringUtils.isEmpty(abbrev)) {
journal.setAttribute("abbrev", abbrev);
}
if (!StringUtils.isEmpty(publisher)) {
journal.setAttribute("publisher", publisher);
}
if (journal.getAttribute("primaryIdentifier").getValue().equals("ZDB-JRNL-050621-1000")) {
System.out.println("storing: ZDB-JRNL-050621-1000");
}
try {
store(journal);
} catch (ObjectStoreException e) {
throw new SAXException(e);
}
}
}
/**
* @param geneId
* @return
* @throws ObjectStoreException
*/
private Item getProteinItem(String proteinId) throws ObjectStoreException {
Item protein = proteinIdMap.get(proteinId);
if (protein == null) {
protein = createItem("Protein");
proteinIdMap.put(proteinId, protein);
protein.setAttribute("secondaryIdentifier", proteinId);
}
return protein;
}
// save homologue pair
private void processHomologue(String gene1, String gene2) throws ObjectStoreException {
Item homologue = createItem("Homologue");
homologue.setReference("gene", gene1);
homologue.setReference("homologue", gene2);
homologue.addToCollection("evidence", getEvidence());
homologue.setAttribute("type", "homologue");
try {
store(homologue);
} catch (ObjectStoreException e) {
throw new ObjectStoreException(e);
}
}
/**
* Return a DataSource item with the given details.
*
* @param title the DataSet title
* @param url the new url field, or null if the url shouldn't be set
* @param description the new description field, or null if the field shouldn't be set
* @param dataSourceItem the DataSource referenced by the the DataSet
* @return the DataSet Item
*/
public Item getDataSetItem(String title, String url, String description, Item dataSourceItem) {
Item item = dataSets.get(title);
if (item == null) {
item = createItem("DataSet");
item.setAttribute("name", title);
item.setReference("dataSource", dataSourceItem);
if (url != null) {
item.setAttribute("url", url);
}
if (description != null) {
item.setAttribute("description", description);
}
try {
store(item);
} catch (ObjectStoreException e) {
throw new RuntimeException("failed to store DataSet with title: " + title, e);
}
dataSets.put(title, item);
}
return item;
}
private Item getJournal(String primaryIdentifier) throws SAXException {
Item item = journals.get(primaryIdentifier);
if (item == null) {
item = createItem("Journal");
item.setAttribute("primaryIdentifier", primaryIdentifier);
// item.setReference("organism", organismRefId);
journals.put(primaryIdentifier, item);
// setSynonym(item.getIdentifier(), "identifier", primaryIdentifier);
// setSynonym(item.getIdentifier(), "accession", primaryIdentifier);
}
return item;
}
private Item getTerm(String name) throws ObjectStoreException {
if (!isValidTerm(name)) {
return null;
} else if (terms.containsKey(name)) {
return terms.get(name);
}
Item termItem = createItem("OntologyTerm");
termItem.setAttribute("name", name);
termItem.setReference("ontology", ontology);
store(termItem);
terms.put(name, termItem);
return termItem;
}
private void setNames(
List<?> names, Set<Item> synonymsToAdd, String recordId, Item feature, ClassDescriptor cd) {
if (cd.getFieldDescriptorByName("symbol") == null) {
String name = (String) names.get(0);
feature.setAttribute("name", name);
for (Iterator<?> i = names.iterator(); i.hasNext(); ) {
String recordName = (String) i.next();
if (!recordName.equals(recordId) && !recordName.equals(name)) {
synonymsToAdd.add(getSynonym(feature, recordName));
}
}
} else {
String symbol = (String) names.get(0);
feature.setAttribute("symbol", (String) names.get(0));
for (Iterator<?> i = names.iterator(); i.hasNext(); ) {
String recordName = (String) i.next();
if (!recordName.equals(recordId) && !recordName.equals(symbol)) {
synonymsToAdd.add(getSynonym(feature, recordName));
}
}
}
}
/**
* Return a DataSet item for the given title
*
* @param name the DataSet name
* @return the DataSet Item
*/
public Item getDataSourceItem(String name) {
Item item = dataSources.get(name);
if (item == null) {
item = createItem("DataSource");
item.setAttribute("name", name);
try {
store(item);
} catch (ObjectStoreException e) {
throw new RuntimeException("failed to store DataSource with name: " + name, e);
}
dataSources.put(name, item);
}
return item;
}
private String getDataset(String code) throws ObjectStoreException {
String dataSetIdentifier = dataSets.get(code);
if (dataSetIdentifier == null) {
String dataSourceName = getDataSourceName(code);
String title = "DO Annotation from " + dataSourceName;
Item item = createItem("DataSet");
item.setAttribute("name", title);
item.setReference("dataSource", getDataSource(dataSourceName));
dataSetIdentifier = item.getIdentifier();
dataSets.put(code, dataSetIdentifier);
store(item);
}
return dataSetIdentifier;
}
private Item getResult(String key, String geneId, String pubId, String stage) {
if (results.containsKey(key)) {
return results.get(key);
}
Item result = createItem("MRNAExpressionResult");
result.setAttribute("expressed", "true");
result.setReference("gene", geneId);
result.setReference("publication", pubId);
result.setCollection("stages", getStages(stage));
// result.setCollection("images", new ArrayList<String>());
// result.setCollection("mRNAExpressionTerms", new ArrayList<String>());
results.put(key, result);
return result;
}