/**
* Produce random data.
*
* <p>{@inheritDoc}
*/
public void process(Reader inputFile) throws Exception {
Random random = new Random();
Set doneValues = new HashSet();
long time = System.currentTimeMillis();
for (int i = 0; i < count / 2; i++) {
Item itemTo = createItem("ReferenceTo2");
Item itemFrom = createItem("ReferenceFrom2");
Integer firstInt;
do {
firstInt = new Integer(random.nextInt());
} while (doneValues.contains(firstInt));
doneValues.add(firstInt);
itemTo.setAttribute("att", "" + firstInt);
itemFrom.setReference("ref", itemTo.getIdentifier());
getItemWriter().store(ItemHelper.convert(itemFrom));
getItemWriter().store(ItemHelper.convert(itemTo));
}
long now = System.currentTimeMillis();
LOG.info(
"Finished generating "
+ count
+ " objects at "
+ ((60000L * count) / (now - time))
+ " objects per minute ("
+ (now - time)
+ " ms total)");
}
private String getGene(String identifierType, String id, String taxonId)
throws ObjectStoreException {
String identifier = id;
if (rslv != null && rslv.hasTaxon(taxonId)) {
identifier = resolveGene(identifier, taxonId);
if (identifier == null) {
return null;
}
}
String refId = identifiersToGenes.get(identifier);
if (refId == null) {
Item gene = createItem("Gene");
refId = gene.getIdentifier();
gene.setAttribute(identifierType, identifier);
gene.setReference("organism", getOrganism(taxonId));
identifiersToGenes.put(identifier, refId);
try {
store(gene);
} catch (ObjectStoreException e) {
throw new ObjectStoreException(e);
}
}
return refId;
}
/**
* @return return/create item of class seqClsName for given identifier
* @throws ObjectStoreException if the Item can't be stored
*/
private Item getSeq(String id) throws ObjectStoreException {
// the seqHandler may have changed the id used, e.g. if using an IdResolver
String identifier = sequenceHandler.getSeqIdentifier(id);
if (identifier == null) {
return null;
}
// if (identifier.startsWith("chr")) {
// identifier = identifier.substring(3);
// }
Item seq = seqs.get(identifier);
if (seq == null) {
seq = sequenceHandler.makeSequenceItem(this, identifier);
// sequence handler may choose not to create sequence
if (seq != null) {
seq.addReference(getOrgRef());
store(seq);
seqs.put(identifier, seq);
}
}
handler.setSequence(seq);
return seq;
}
/** {@inheritDoc} */
@Override
public void process(Reader reader) throws Exception {
// Data has format:
// id | description
@SuppressWarnings("rawtypes")
Iterator lineIter = FormattedTextParser.parseTabDelimitedReader(reader);
int count = 0;
while (lineIter.hasNext()) {
String[] line = (String[]) lineIter.next();
try {
String entrez = line[0];
String description = line[1];
LOG.error("description " + count++ + " " + description);
if (!StringUtils.isBlank(description)) {
Item gene = createItem("Gene");
gene.setAttribute("primaryIdentifier", entrez);
gene.setAttribute("description", description);
gene.setReference("organism", getOrganism(HUMAN_TAXON_ID));
store(gene);
}
} catch (IndexOutOfBoundsException e) {
LOG.info("Failed to read line: " + Arrays.asList(line));
}
}
}
private Item getGene(String geneCG) throws ObjectStoreException {
if (rslv == null || !rslv.hasTaxon(TAXON_FLY)) {
return null;
}
int resCount = rslv.countResolutions(TAXON_FLY, geneCG);
if (resCount != 1) {
LOG.info(
"RESOLVER: failed to resolve gene to one identifier, ignoring gene: "
+ geneCG
+ " count: "
+ resCount
+ " FBgn: "
+ rslv.resolveId(TAXON_FLY, geneCG));
return null;
}
String primaryIdentifier = rslv.resolveId(TAXON_FLY, geneCG).iterator().next();
if (genes.containsKey(primaryIdentifier)) {
return genes.get(primaryIdentifier);
}
Item gene = createItem("Gene");
gene.setAttribute("primaryIdentifier", primaryIdentifier);
gene.setReference("organism", orgDrosophila);
genes.put(primaryIdentifier, gene);
store(gene);
return gene;
}
/**
* Create and store a NetworkProperty item on the first time called.
*
* @param type e.g. "level", "position"
* @param value the value of level or position
* @return an Item representing the NetworkProperty
*/
private Item createNetworkProperty(String type, String value) throws ObjectStoreException {
Item networkproperty = createItem("NetworkProperty");
networkproperty.setAttribute("type", type);
networkproperty.setAttribute("value", value);
return networkproperty;
}
private void setImage(Item result, String img) {
if (!imgs.containsKey(img)) {
Item item = createItem("Image");
item.setAttribute("url", img);
imgs.put(img, item);
result.addToCollection("images", item.getIdentifier());
}
}
private void storeEvidenceCode(String code) throws ObjectStoreException {
if (evidenceCodes.get(code) == null) {
Item item = createItem("DOEvidenceCode");
item.setAttribute("code", code);
evidenceCodes.put(code, item.getIdentifier());
store(item);
}
}
/**
* Create a new pathway Item or fetch from a map if it has been seen before
*
* @param pathwayId the id of a KEGG pathway to look up
* @return an Item representing the pathway
*/
private Item getPathway(String pathwayId) {
Item pathway = pathwayMap.get(pathwayId);
if (pathway == null) {
pathway = createItem("Pathway");
pathway.setAttribute("identifier", pathwayId);
pathwayMap.put(pathwayId, pathway);
}
return pathway;
}
private Item newOrganism(String taxonId) throws ObjectStoreException {
Item item = organisms.get(taxonId);
if (item == null) {
item = createItem("Organism");
item.setAttribute("taxonId", taxonId);
organisms.put(taxonId, item);
store(item);
}
return item;
}
/**
* Create and store a BioEntity item on the first time called.
*
* @param type gene
* @param primaryId the gene primaryIdentifier
* @param symbol the gene symbol
* @throws ObjectStoreException
*/
private void createBioEntity(String type, String primaryId, String symbol)
throws ObjectStoreException {
Item bioentity = null;
bioentity = createItem("Gene");
bioentity.setAttribute("primaryIdentifier", primaryId);
bioentity.setAttribute("symbol", symbol);
store(bioentity);
geneItems.put(primaryId, bioentity.getIdentifier());
}
private void processMeshTerms(Item publication, List<String> newTerms) {
for (String name : newTerms) {
Item item = meshTerms.get(name);
if (item == null) {
item = itemFactory.makeItemForClass("MeshTerm");
item.setAttribute("name", name);
meshTerms.put(name, item);
}
publication.addToCollection("meshTerms", item);
}
}
private Item getJournal(String primaryIdentifier) throws SAXException {
Item item = journals.get(primaryIdentifier);
if (item == null) {
item = createItem("Journal");
item.setAttribute("primaryIdentifier", primaryIdentifier);
// item.setReference("organism", organismRefId);
journals.put(primaryIdentifier, item);
// setSynonym(item.getIdentifier(), "identifier", primaryIdentifier);
// setSynonym(item.getIdentifier(), "accession", primaryIdentifier);
}
return item;
}
// save homologue pair
private void processHomologue(String gene1, String gene2) throws ObjectStoreException {
Item homologue = createItem("Homologue");
homologue.setReference("gene", gene1);
homologue.setReference("homologue", gene2);
homologue.addToCollection("evidence", getEvidence());
homologue.setAttribute("type", "homologue");
try {
store(homologue);
} catch (ObjectStoreException e) {
throw new ObjectStoreException(e);
}
}
/**
* @param geneId
* @return
* @throws ObjectStoreException
*/
private Item getProteinItem(String proteinId) throws ObjectStoreException {
Item protein = proteinIdMap.get(proteinId);
if (protein == null) {
protein = createItem("Protein");
proteinIdMap.put(proteinId, protein);
protein.setAttribute("secondaryIdentifier", proteinId);
}
return protein;
}
/**
* @param proteinId
* @param modSite
* @param modType
* @param source
* @param pmid
* @throws ObjectStoreException
* @throws Exception
*/
private void newProduct(
String experiment, String proteinId, String value, String units, String pmid)
throws ObjectStoreException, Exception {
Item protein = getProteinItem(proteinId);
Item pmods = getProteinHalfLife(experiment, value, units, pmid);
protein.addToCollection("proteinHalfLife", pmods.getIdentifier());
// Item pmods2 = getProteinHalfLife(experiment, valueMins, stringMinutes, pmid);
// protein.addToCollection("proteinHalfLife", pmods2.getIdentifier());
}
private Item getTerm(String name) throws ObjectStoreException {
if (!isValidTerm(name)) {
return null;
} else if (terms.containsKey(name)) {
return terms.get(name);
}
Item termItem = createItem("OntologyTerm");
termItem.setAttribute("name", name);
termItem.setReference("ontology", ontology);
store(termItem);
terms.put(name, termItem);
return termItem;
}
/**
* For a collection of items return the count of given class name.
*
* @param items the Item collection to count
* @return the count for a given classname in the collection
*/
public static int countItemByClass(Collection<Item> items, String className) {
Map<String, List> counts = new TreeMap<String, List>();
for (Item item : items) {
List<String> clsItems = counts.get(item.getClassName());
if (clsItems == null) {
clsItems = new ArrayList<String>();
counts.put(item.getClassName(), clsItems);
}
clsItems.add(item.getIdentifier());
}
return counts.get(className).size();
}
private String getDataset(String code) throws ObjectStoreException {
String dataSetIdentifier = dataSets.get(code);
if (dataSetIdentifier == null) {
String dataSourceName = getDataSourceName(code);
String title = "DO Annotation from " + dataSourceName;
Item item = createItem("DataSet");
item.setAttribute("name", title);
item.setReference("dataSource", getDataSource(dataSourceName));
dataSetIdentifier = item.getIdentifier();
dataSets.put(code, dataSetIdentifier);
store(item);
}
return dataSetIdentifier;
}
private Item getResult(String key, String geneId, String pubId, String stage) {
if (results.containsKey(key)) {
return results.get(key);
}
Item result = createItem("MRNAExpressionResult");
result.setAttribute("expressed", "true");
result.setReference("gene", geneId);
result.setReference("publication", pubId);
result.setCollection("stages", getStages(stage));
// result.setCollection("images", new ArrayList<String>());
// result.setCollection("mRNAExpressionTerms", new ArrayList<String>());
results.put(key, result);
return result;
}
private Integer createGoAnnotation(
String productIdentifier,
String productType,
String termIdentifier,
Item organism,
String qualifier,
String withText,
String dataSourceCode)
throws ObjectStoreException {
Item goAnnotation = createItem(annotationClassName);
goAnnotation.setReference("subject", productIdentifier);
goAnnotation.setReference("ontologyTerm", termIdentifier);
if (!StringUtils.isEmpty(qualifier)) {
goAnnotation.setAttribute("qualifier", qualifier);
}
// with objects
if (!StringUtils.isEmpty(withText)) {
goAnnotation.setAttribute("withText", withText);
List<String> with = createWithObjects(withText, organism, dataSourceCode);
if (!with.isEmpty()) {
goAnnotation.addCollection(new ReferenceList("with", with));
}
}
goAnnotation.addToCollection("dataSets", getDataset(dataSourceCode));
if ("gene".equals(productType)) {
addProductCollection(productIdentifier, goAnnotation.getIdentifier());
}
Integer storedAnnotationId = store(goAnnotation);
return storedAnnotationId;
}
/**
* Return a DataSet item for the given title
*
* @param name the DataSet name
* @return the DataSet Item
*/
public Item getDataSourceItem(String name) {
Item item = dataSources.get(name);
if (item == null) {
item = createItem("DataSource");
item.setAttribute("name", name);
try {
store(item);
} catch (ObjectStoreException e) {
throw new RuntimeException("failed to store DataSource with name: " + name, e);
}
dataSources.put(name, item);
}
return item;
}
private void setSynonym(String subjectRefId, String type, String value) throws SAXException {
String key = subjectRefId + type + value;
if (!synonyms.contains(key)) {
Item synonym = createItem("Synonym");
synonym.setAttribute("type", type);
synonym.setAttribute("value", value);
synonym.setReference("subject", subjectRefId);
synonyms.add(key);
try {
store(synonym);
} catch (ObjectStoreException e) {
throw new SAXException(e);
}
}
}
/**
* Given two sets of Items (a and b) return a set of Items that are present in a but not b.
*
* @param a a set of Items
* @param b a set of Items
* @return the set of Items in a but not in b
*/
public static Set<Item> diffItemSets(Set<Item> a, Set<Item> b) {
Set<Item> diff = new HashSet<Item>(a);
Iterator<Item> i = a.iterator();
while (i.hasNext()) {
Item itemA = i.next();
Iterator<Item> j = b.iterator();
while (j.hasNext()) {
Item itemB = j.next();
if (itemA.equals(itemB)) {
diff.remove(itemA);
}
}
}
return diff;
}
private void setNames(
List<?> names,
String symbol,
List<String> synonyms,
Set<Item> synonymsToAdd,
String primaryIdentifier,
Item feature,
ClassDescriptor cd) {
if (cd.getFieldDescriptorByName("symbol") == null) { // if symbol is not in the model
String name = (String) names.get(0);
feature.setAttribute("name", name);
for (Iterator<?> i = names.iterator(); i.hasNext(); ) {
String recordName = (String) i.next();
if (!recordName.equals(primaryIdentifier) && !recordName.equals(name)) {
synonymsToAdd.add(getSynonym(feature, recordName));
}
}
if (synonyms != null) {
for (Iterator<?> i = synonyms.iterator(); i.hasNext(); ) {
String recordName = (String) i.next();
if (!recordName.equals(primaryIdentifier) && !recordName.equals(name)) {
synonymsToAdd.add(getSynonym(feature, recordName));
}
}
}
} else {
if (symbol == null) {
symbol = (String) names.get(0);
}
feature.setAttribute("symbol", symbol);
for (Iterator<?> i = names.iterator(); i.hasNext(); ) {
String recordName = (String) i.next();
if (!recordName.equals(primaryIdentifier) && !recordName.equals(symbol)) {
synonymsToAdd.add(getSynonym(feature, recordName));
}
}
if (synonyms != null) {
for (Iterator<?> i = synonyms.iterator(); i.hasNext(); ) {
String recordName = (String) i.next();
if (!recordName.equals(primaryIdentifier) && !recordName.equals(symbol)) {
synonymsToAdd.add(getSynonym(feature, recordName));
}
}
}
}
}
/**
* Construct a new instance of BDGPInsituConverter.
*
* @param model the Model
* @param writer the ItemWriter used to handle the resultant items
* @throws ObjectStoreException if an error occurs in storing
*/
public BDGPInsituConverter(ItemWriter writer, Model model) throws ObjectStoreException {
super(writer, model, "BDGP", "BDGP in situ data set");
orgDrosophila = createItem("Organism");
orgDrosophila.setAttribute("taxonId", TAXON_FLY);
store(orgDrosophila);
pub = createItem("Publication");
pub.setAttribute("pubMedId", "17645804");
store(pub);
setStages();
ontology = createItem("Ontology");
ontology.setAttribute("name", "ImaGO");
store(ontology);
}
/**
* For a collection of items return a string containing counts of each item classname - useful for
* degugging tests.
*
* @param items the Item collection to count
* @return a formatted string with counts for each classname in the collection
*/
public static String countItemClasses(Collection<Item> items) {
Map<String, List> counts = new TreeMap<String, List>();
for (Item item : items) {
List<String> clsItems = counts.get(item.getClassName());
if (clsItems == null) {
clsItems = new ArrayList<String>();
counts.put(item.getClassName(), clsItems);
}
clsItems.add(item.getIdentifier());
}
StringBuffer sb = new StringBuffer();
for (Map.Entry<String, List> entry : counts.entrySet()) {
sb.append(entry.getKey() + " - " + entry.getValue().size() + " " + entry.getValue() + ENDL);
}
return sb.toString();
}
/**
* Return the organism Item created for this GFF3Converter from the organism abbreviation passed
* to the constructor.
*
* @return the organism item
* @throws ObjectStoreException if the Organism item can't be stored
*/
public Item getOrganism() throws ObjectStoreException {
if (organism == null) {
organism = createItem("Organism");
organism.setAttribute("taxonId", orgTaxonId);
store(organism);
}
return organism;
}
private void storeEvidence() throws ObjectStoreException {
for (Set<Evidence> annotationEvidence : goTermGeneToEvidence.values()) {
List<String> evidenceRefIds = new ArrayList<String>();
Integer goAnnotationRefId = null;
for (Evidence evidence : annotationEvidence) {
Item goevidence = createItem("DOEvidence");
goevidence.setReference("code", evidenceCodes.get(evidence.getEvidenceCode()));
List<String> publicationEvidence = evidence.getPublications();
if (!publicationEvidence.isEmpty()) {
goevidence.setCollection("publications", publicationEvidence);
}
store(goevidence);
evidenceRefIds.add(goevidence.getIdentifier());
goAnnotationRefId = evidence.getStoredAnnotationId();
}
ReferenceList refIds = new ReferenceList("evidence", new ArrayList<String>(evidenceRefIds));
store(refIds, goAnnotationRefId);
}
}
private String newGoTerm(String identifier, String dataSourceCode) throws ObjectStoreException {
String goId = resolveTerm(identifier);
if (goId == null) {
return null;
}
String goTermIdentifier = goTerms.get(goId);
if (goTermIdentifier == null) {
Item item = createItem(termClassName);
item.setAttribute("identifier", goId);
item.addToCollection("dataSets", getDataset(dataSourceCode));
store(item);
goTermIdentifier = item.getIdentifier();
goTerms.put(goId, goTermIdentifier);
}
return goTermIdentifier;
}