public void testConstructQueries() throws Exception {
PrecomputeTask pt = new PrecomputeTask();
List actual = pt.constructQueries(true, os, "Employee department Department", "flibble");
QueryClass qcEmpl = new QueryClass(Employee.class);
QueryClass qcDept = new QueryClass(Department.class);
QueryObjectReference ref = new QueryObjectReference(qcEmpl, "department");
ContainsConstraint cc = new ContainsConstraint(ref, ConstraintOp.CONTAINS, qcDept);
Query q1 = new Query();
q1.setDistinct(false);
q1.addToSelect(qcEmpl);
q1.addFrom(qcEmpl);
q1.addToSelect(qcDept);
q1.addFrom(qcDept);
q1.setConstraint(cc);
q1.addToOrderBy(qcEmpl);
q1.addToOrderBy(qcDept);
Query q2 = new Query();
q2.setDistinct(false);
q2.addToSelect(qcEmpl);
q2.addFrom(qcEmpl);
q2.addToSelect(qcDept);
q2.addFrom(qcDept);
q2.setConstraint(cc);
q2.addToOrderBy(qcDept);
q2.addToOrderBy(qcEmpl);
// List of queries in both possible orders
List expected = new ArrayList(Arrays.asList(new Query[] {q2, q1}));
compareQueryLists(expected, actual);
}
/** Test that transactions can be aborted */
public void testAbortTransactions() throws Exception {
Address address1 = new Address();
address1.setAddress("Address 3");
Address address2 = new Address();
address2.setAddress("Address 4");
Query q = new Query();
QueryClass qcAddress = new QueryClass(Address.class);
QueryField qf = new QueryField(qcAddress, "address");
ConstraintSet cs1 = new ConstraintSet(ConstraintOp.OR);
cs1.addConstraint(new SimpleConstraint(qf, ConstraintOp.MATCHES, new QueryValue("Address%")));
q.addToSelect(qcAddress);
q.addFrom(qcAddress);
q.addToOrderBy(qf);
q.setConstraint(cs1);
Results res = writer.execute(q);
assertEquals(res.toString(), 0, res.size());
res = realOs.execute(q);
assertEquals(res.toString(), 0, res.size());
writer.beginTransaction();
assertTrue(writer.isInTransaction());
writer.store(address1);
writer.store(address2);
// TODO: These lines now fail, because we do not allow querying on writers with uncommitted
// data. The writer should relax this restriction.
res = writer.execute(q);
assertEquals(2, res.size());
res = realOs.execute(q);
assertEquals(res.toString(), 0, res.size());
writer.abortTransaction();
assertFalse(writer.isInTransaction());
// Should be nothing there unless we commit
res = writer.execute(q);
assertEquals(res.toString(), 0, res.size());
res = realOs.execute(q);
assertEquals(res.toString(), 0, res.size());
}
/** Test that transactions do actually commit and that isInTransaction() works. */
public void testCommitTransactions() throws Exception {
Address address1 = new Address();
address1.setAddress("Address 1");
Address address2 = new Address();
address2.setAddress("Address 2");
Query q = new Query();
QueryClass qcAddress = new QueryClass(Address.class);
QueryField qf = new QueryField(qcAddress, "address");
ConstraintSet cs1 = new ConstraintSet(ConstraintOp.OR);
cs1.addConstraint(new SimpleConstraint(qf, ConstraintOp.MATCHES, new QueryValue("Address%")));
q.addToSelect(qcAddress);
q.addFrom(qcAddress);
q.addToOrderBy(qf);
q.setConstraint(cs1);
try {
writer.beginTransaction();
assertTrue(writer.isInTransaction());
writer.store(address1);
writer.store(address2);
// Should be nothing in OS until we commit
Results res = realOs.execute(q);
assertEquals(0, res.size());
// However, they should be in the WRITER.
// TODO: These lines now fail, because we do not allow querying on writers with uncommitted
// data. The writer should relax this restriction.
res = writer.execute(q);
assertEquals(2, res.size());
writer.commitTransaction();
assertFalse(writer.isInTransaction());
res = realOs.execute(q);
assertEquals(2, res.size());
assertEquals(address1, (Address) ((ResultsRow) res.get(0)).get(0));
assertEquals(address2, (Address) ((ResultsRow) res.get(1)).get(0));
} finally {
writer.delete(address1);
writer.delete(address2);
}
}
/**
* @param os object store
* @return map of name to so term
* @throws ObjectStoreException if something goes wrong
*/
protected Map<String, SOTerm> populateSOTermMap(ObjectStore os) throws ObjectStoreException {
Map<String, SOTerm> soTerms = new HashMap<String, SOTerm>();
Query q = new Query();
q.setDistinct(false);
QueryClass qcSOTerm = new QueryClass(SOTerm.class);
q.addToSelect(qcSOTerm);
q.addFrom(qcSOTerm);
q.addToOrderBy(qcSOTerm);
Results res = os.execute(q);
Iterator it = res.iterator();
while (it.hasNext()) {
ResultsRow<InterMineObject> rr = (ResultsRow<InterMineObject>) it.next();
SOTerm soTerm = (SOTerm) rr.get(0);
soTerms.put(soTerm.getName(), soTerm);
}
return soTerms;
}
/**
* Create a queryField starting from the path given in input and add the contain constraints
* between all the queryclass composing the path. If addToSelect is true, add the queryFiled as
* select, group by and order by element.
*
* @param path the path used to create the queryField. The path doesn't containt the startClass
* @param query the query to modify
* @param addToSelect if true add the queryFiled as select, group by and order by element
* @return the queryField created
*/
protected QueryField createQueryFieldByPath(String path, Query query, boolean addToSelect) {
QueryField queryField = null;
String[] splittedPath = path.split("\\.");
QueryClass qc = startClass;
String attribute;
String attributePath = "";
for (int i = 0; i < splittedPath.length; i++) {
attribute = splittedPath[i];
if (i == splittedPath.length - 1) {
queryField = new QueryField(qc, attribute);
if (addToSelect) {
query.addToSelect(queryField);
query.addToGroupBy(queryField);
query.addToOrderBy(queryField);
}
} else {
attributePath = createAttributePath(splittedPath, i);
qc = addReference(query, qc, attribute, attributePath);
}
}
return queryField;
}
/** @return query to get all parent so terms */
protected Query getAllParents() {
Query q = new Query();
q.setDistinct(false);
QueryClass qcFeature =
new QueryClass(model.getClassDescriptorByName("SequenceFeature").getType());
q.addToSelect(qcFeature);
q.addFrom(qcFeature);
QueryClass qcSOTerm = new QueryClass(OntologyTerm.class);
q.addToSelect(qcSOTerm);
q.addFrom(qcSOTerm);
q.addToOrderBy(qcSOTerm);
ConstraintSet cs = new ConstraintSet(ConstraintOp.AND);
QueryObjectReference ref1 = new QueryObjectReference(qcFeature, "sequenceOntologyTerm");
cs.addConstraint(new ContainsConstraint(ref1, ConstraintOp.CONTAINS, qcSOTerm));
// Set the constraint of the query
q.setConstraint(cs);
return q;
}
/** The method to run all the queries. */
@SuppressWarnings("rawtypes")
private void queryExecutor() {
// Use spanOverlapFullResultMap to store the data in the session
@SuppressWarnings("unchecked")
Map<String, Map<GenomicRegion, List<SpanQueryResultRow>>> spanOverlapFullResultMap =
(Map<String, Map<GenomicRegion, List<SpanQueryResultRow>>>)
request.getSession().getAttribute("spanOverlapFullResultMap");
if (spanOverlapFullResultMap == null) {
spanOverlapFullResultMap =
new HashMap<String, Map<GenomicRegion, List<SpanQueryResultRow>>>();
}
Map<GenomicRegion, List<SpanQueryResultRow>> spanOverlapResultDisplayMap =
Collections.synchronizedMap(new LinkedHashMap<GenomicRegion, List<SpanQueryResultRow>>());
// GBrowse track
@SuppressWarnings("unchecked")
Map<String, Map<GenomicRegion, LinkedHashMap<String, LinkedHashSet<GBrowseTrackInfo>>>>
gbrowseFullTrackMap =
(HashMap<
String,
Map<GenomicRegion, LinkedHashMap<String, LinkedHashSet<GBrowseTrackInfo>>>>)
request.getSession().getAttribute("gbrowseFullTrackMap");
if (gbrowseFullTrackMap == null) {
gbrowseFullTrackMap =
new HashMap<
String, Map<GenomicRegion, LinkedHashMap<String, LinkedHashSet<GBrowseTrackInfo>>>>();
}
Map<GenomicRegion, LinkedHashMap<String, LinkedHashSet<GBrowseTrackInfo>>> gbrowseTrackMap =
Collections.synchronizedMap(
new LinkedHashMap<
GenomicRegion, LinkedHashMap<String, LinkedHashSet<GBrowseTrackInfo>>>());
if (!spanOverlapFullResultMap.containsKey(spanUUIDString)) {
spanOverlapFullResultMap.put(spanUUIDString, spanOverlapResultDisplayMap);
request.getSession().setAttribute("spanOverlapFullResultMap", spanOverlapFullResultMap);
gbrowseFullTrackMap.put(spanUUIDString, gbrowseTrackMap);
request.getSession().setAttribute("gbrowseFullTrackMap", gbrowseFullTrackMap);
try {
Query q;
for (GenomicRegion aSpan : spanList) {
q = new Query();
q.setDistinct(true);
String chrPID = aSpan.getChr();
Integer start = aSpan.getStart();
Integer end = aSpan.getEnd();
/*
>>>>> TEST CODE <<<<<
LOG.info("OrgName: " + orgName);
LOG.info("chrPID: " + chrPID);
LOG.info("start: " + start);
LOG.info("end: " + end);
LOG.info("FeatureTypes: " + ftKeys);
LOG.info("Submissions: " + subKeys);
>>>>> TEST CODE <<<<<
*/
// DB tables
QueryClass qcOrg = new QueryClass(Organism.class);
QueryClass qcChr = new QueryClass(Chromosome.class);
QueryClass qcFeature = new QueryClass(SequenceFeature.class);
QueryClass qcLoc = new QueryClass(Location.class);
QueryClass qcSubmission = new QueryClass(Submission.class);
QueryField qfOrgName = new QueryField(qcOrg, "shortName");
QueryField qfChrPID = new QueryField(qcChr, "primaryIdentifier");
QueryField qfFeaturePID = new QueryField(qcFeature, "primaryIdentifier");
QueryField qfFeatureId = new QueryField(qcFeature, "id");
QueryField qfFeatureClass = new QueryField(qcFeature, "class");
QueryField qfSubmissionTitle = new QueryField(qcSubmission, "title");
QueryField qfSubmissionDCCid = new QueryField(qcSubmission, "DCCid");
QueryField qfChr = new QueryField(qcChr, "primaryIdentifier");
QueryField qfLocStart = new QueryField(qcLoc, "start");
QueryField qfLocEnd = new QueryField(qcLoc, "end");
q.addToSelect(qfFeatureId);
q.addToSelect(qfFeaturePID);
q.addToSelect(qfFeatureClass);
q.addToSelect(qfChr);
q.addToSelect(qfLocStart);
q.addToSelect(qfLocEnd);
q.addToSelect(qfSubmissionDCCid);
q.addToSelect(qfSubmissionTitle);
q.addFrom(qcChr);
q.addFrom(qcOrg);
q.addFrom(qcFeature);
q.addFrom(qcLoc);
q.addFrom(qcSubmission);
q.addToOrderBy(qfLocStart, "ascending");
ConstraintSet constraints = new ConstraintSet(ConstraintOp.AND);
q.setConstraint(constraints);
// SequenceFeature.organism = Organism
QueryObjectReference organism = new QueryObjectReference(qcFeature, "organism");
ContainsConstraint ccOrg = new ContainsConstraint(organism, ConstraintOp.CONTAINS, qcOrg);
constraints.addConstraint(ccOrg);
// Organism.name = orgName
SimpleConstraint scOrg =
new SimpleConstraint(qfOrgName, ConstraintOp.EQUALS, new QueryValue(orgName));
constraints.addConstraint(scOrg);
// Location.feature = SequenceFeature
QueryObjectReference locSubject = new QueryObjectReference(qcLoc, "feature");
ContainsConstraint ccLocSubject =
new ContainsConstraint(locSubject, ConstraintOp.CONTAINS, qcFeature);
constraints.addConstraint(ccLocSubject);
// Location.locatedOn = Chromosome
QueryObjectReference locObject = new QueryObjectReference(qcLoc, "locatedOn");
ContainsConstraint ccLocObject =
new ContainsConstraint(locObject, ConstraintOp.CONTAINS, qcChr);
constraints.addConstraint(ccLocObject);
// Chromosome.primaryIdentifier = chrPID
SimpleConstraint scChr =
new SimpleConstraint(qfChrPID, ConstraintOp.EQUALS, new QueryValue(chrPID));
constraints.addConstraint(scChr);
// SequenceFeature.submissions = Submission
QueryCollectionReference submission =
new QueryCollectionReference(qcFeature, "submissions");
ContainsConstraint ccSubmission =
new ContainsConstraint(submission, ConstraintOp.CONTAINS, qcSubmission);
constraints.addConstraint(ccSubmission);
// SequenceFeature.class in a list
constraints.addConstraint(new BagConstraint(qfFeatureClass, ConstraintOp.IN, ftKeys));
// Submission.CCDid in a list
constraints.addConstraint(new BagConstraint(qfSubmissionDCCid, ConstraintOp.IN, subKeys));
OverlapRange overlapInput =
new OverlapRange(new QueryValue(start), new QueryValue(end), locObject);
OverlapRange overlapFeature =
new OverlapRange(
new QueryField(qcLoc, "start"), new QueryField(qcLoc, "end"), locObject);
OverlapConstraint oc =
new OverlapConstraint(overlapInput, ConstraintOp.OVERLAPS, overlapFeature);
constraints.addConstraint(oc);
Results results = im.getObjectStore().execute(q);
/*
>>>>> TEST CODE <<<<<
LOG.info("Query: " + q.toString());
LOG.info("Result Size: " + results.size());
LOG.info("Result >>>>> " + results);
>>>>> TEST CODE <<<<<
*/
List<SpanQueryResultRow> spanResults = new ArrayList<SpanQueryResultRow>();
if (results == null || results.isEmpty()) {
spanOverlapResultDisplayMap.put(aSpan, null);
gbrowseTrackMap.put(aSpan, null);
} else {
for (Iterator<?> iter = results.iterator(); iter.hasNext(); ) {
ResultsRow<?> row = (ResultsRow<?>) iter.next();
SpanQueryResultRow aRow = new SpanQueryResultRow();
aRow.setFeatureId((Integer) row.get(0));
aRow.setFeaturePID((String) row.get(1));
aRow.setFeatureClass(((Class) row.get(2)).getSimpleName());
aRow.setChr((String) row.get(3));
aRow.setStart((Integer) row.get(4));
aRow.setEnd((Integer) row.get(5));
aRow.setSubDCCid((String) row.get(6));
aRow.setSubTitle((String) row.get(7));
spanResults.add(aRow);
}
spanOverlapResultDisplayMap.put(aSpan, spanResults);
gbrowseTrackMap.put(aSpan, getSubGbrowseTrack(spanResults)); // Gbrowse
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
public void testLoad() throws Exception {
TSVFileReaderTask tsvTask = new TSVFileReaderTask();
tsvTask.setIgnoreDuplicates(true);
tsvTask.setIntegrationWriterAlias("integration.unittestmulti");
tsvTask.setSourceName("testsource");
cleanObjects(tsvTask.getDirectDataLoader().getIntegrationWriter());
File tempFile = File.createTempFile("TSVFileReaderTaskTest", "tmp");
FileWriter fw = new FileWriter(tempFile);
InputStream is = getClass().getClassLoader().getResourceAsStream("TSVFileReaderTaskTest.tsv");
BufferedReader br = new BufferedReader(new InputStreamReader(is));
String line = null;
while ((line = br.readLine()) != null) {
fw.write(line + "\n");
}
fw.close();
FileSet fileSet = new FileSet();
fileSet.setFile(tempFile);
tsvTask.addFileSet(fileSet);
InputStream confInputStream =
getClass().getClassLoader().getResourceAsStream("TSVFileReaderTaskTest.properties");
DelimitedFileConfiguration dfc = new DelimitedFileConfiguration(model, confInputStream);
tsvTask.executeInternal(dfc, tempFile);
// Check the results to see if we have some data...
ObjectStore os = tsvTask.getDirectDataLoader().getIntegrationWriter().getObjectStore();
Query q = new Query();
QueryClass empQueryClass = new QueryClass(Employee.class);
QueryField qf0 = new QueryField(empQueryClass, "age");
QueryField qf1 = new QueryField(empQueryClass, "name");
QueryField qf2 = new QueryField(empQueryClass, "fullTime");
q.addToSelect(qf0);
q.addToSelect(qf1);
q.addToSelect(qf2);
q.addFrom(empQueryClass);
q.addToOrderBy(qf1);
Results r = os.execute(q);
if (r.size() != 3) {
for (List<Object> rr : (List<List<Object>>) ((List) r)) {
System.err.print("row: ");
for (Object obj : rr) {
System.err.print("{" + obj + "} ");
}
System.err.println();
}
}
assertEquals(3, r.size());
List expectedRow0 = Arrays.asList(new Object[] {new Integer(10), "EmployeeA1", Boolean.FALSE});
assertEquals(expectedRow0, r.get(0));
List expectedRow1 = Arrays.asList(new Object[] {new Integer(20), "EmployeeA2", Boolean.TRUE});
assertEquals(expectedRow1, r.get(1));
List expectedRow2 = Arrays.asList(new Object[] {new Integer(0), "EmployeeA3", Boolean.FALSE});
assertEquals(expectedRow2, r.get(2));
}
/** {@inheritDoc} */
public Query getQuery(String action, List<String> keys) {
// classes for FROM clause
QueryClass qcDisease = new QueryClass(Disease.class);
QueryClass qcGene = new QueryClass(Gene.class);
QueryClass qcPub = new QueryClass(Publication.class);
// fields for SELECT clause
QueryField qfDiseaseName = new QueryField(qcDisease, "diseaseId");
QueryField qfGeneId = new QueryField(qcGene, "id");
QueryField qfGeneName = new QueryField(qcGene, "primaryIdentifier");
QueryField qfId = new QueryField(qcPub, "pubMedId");
QueryField qfPubTitle = new QueryField(qcPub, "title");
// constraints
ConstraintSet cs = new ConstraintSet(ConstraintOp.AND);
// constrain genes to be in list
if (!action.startsWith("population")) {
cs.addConstraint(new BagConstraint(qfGeneId, ConstraintOp.IN, bag.getOsb()));
}
// disease.publication = publication
QueryCollectionReference qcr = new QueryCollectionReference(qcGene, "diseases");
cs.addConstraint(new ContainsConstraint(qcr, ConstraintOp.CONTAINS, qcDisease));
QueryCollectionReference fin =
new QueryCollectionReference(qcDisease, "associatedPublications");
cs.addConstraint(new ContainsConstraint(fin, ConstraintOp.CONTAINS, qcPub));
Query q = new Query();
q.setDistinct(true);
// from statement
q.addFrom(qcGene);
q.addFrom(qcDisease);
q.addFrom(qcPub);
// add constraints to query
q.setConstraint(cs);
// needed for the 'not analysed' number
if ("analysed".equals(action)) {
q.addToSelect(qfGeneId);
// export query
// needed for export button on widget
} else if ("export".equals(action)) {
q.addToSelect(qfId);
q.addToSelect(qfGeneName);
q.addToOrderBy(qfId);
// total queries
// needed for enrichment calculations
} else if (action.endsWith("Total")) {
q.addToSelect(qfGeneId);
Query subQ = q;
q = new Query();
q.addFrom(subQ);
q.addToSelect(new QueryFunction()); // disease count
// needed for enrichment calculations
} else {
q.addToSelect(qfId);
q.addToGroupBy(qfId);
q.addToSelect(new QueryFunction()); // disease count
if ("sample".equals(action)) {
q.addToSelect(qfPubTitle);
q.addToGroupBy(qfPubTitle);
}
}
return q;
}
/**
* Read the UTRs collection of MRNA then set the fivePrimeUTR and threePrimeUTR fields with the
* corresponding UTRs.
*
* @throws Exception if anything goes wrong
*/
public void createUtrRefs() throws Exception {
long startTime = System.currentTimeMillis();
Query q = new Query();
q.setDistinct(false);
QueryClass qcMRNA = new QueryClass(model.getClassDescriptorByName("MRNA").getType());
q.addFrom(qcMRNA);
q.addToSelect(qcMRNA);
q.addToOrderBy(qcMRNA);
QueryClass qcUTR = new QueryClass(model.getClassDescriptorByName("UTR").getType());
q.addFrom(qcUTR);
q.addToSelect(qcUTR);
q.addToOrderBy(qcUTR);
QueryCollectionReference mrnaUtrsRef = new QueryCollectionReference(qcMRNA, "UTRs");
ContainsConstraint mrnaUtrsConstraint =
new ContainsConstraint(mrnaUtrsRef, ConstraintOp.CONTAINS, qcUTR);
QueryObjectReference fivePrimeRef = new QueryObjectReference(qcMRNA, "fivePrimeUTR");
ContainsConstraint fivePrimeNullConstraint =
new ContainsConstraint(fivePrimeRef, ConstraintOp.IS_NULL);
QueryObjectReference threePrimeRef = new QueryObjectReference(qcMRNA, "threePrimeUTR");
ContainsConstraint threePrimeNullConstraint =
new ContainsConstraint(threePrimeRef, ConstraintOp.IS_NULL);
ConstraintSet cs = new ConstraintSet(ConstraintOp.AND);
cs.addConstraint(mrnaUtrsConstraint);
cs.addConstraint(fivePrimeNullConstraint);
cs.addConstraint(threePrimeNullConstraint);
q.setConstraint(cs);
ObjectStore os = osw.getObjectStore();
((ObjectStoreInterMineImpl) os).precompute(q, Constants.PRECOMPUTE_CATEGORY);
Results res = os.execute(q, 500, true, true, true);
int count = 0;
InterMineObject lastMRNA = null;
InterMineObject fivePrimeUTR = null;
InterMineObject threePrimeUTR = null;
osw.beginTransaction();
Class<? extends FastPathObject> fivePrimeUTRCls =
model.getClassDescriptorByName("FivePrimeUTR").getType();
Iterator<?> resIter = res.iterator();
while (resIter.hasNext()) {
ResultsRow<?> rr = (ResultsRow<?>) resIter.next();
InterMineObject mrna = (InterMineObject) rr.get(0);
InterMineObject utr = (InterMineObject) rr.get(1);
if (lastMRNA != null && !mrna.getId().equals(lastMRNA.getId())) {
// clone so we don't change the ObjectStore cache
InterMineObject tempMRNA = PostProcessUtil.cloneInterMineObject(lastMRNA);
if (fivePrimeUTR != null) {
tempMRNA.setFieldValue("fivePrimeUTR", fivePrimeUTR);
fivePrimeUTR = null;
}
if (threePrimeUTR != null) {
tempMRNA.setFieldValue("threePrimeUTR", threePrimeUTR);
threePrimeUTR = null;
}
osw.store(tempMRNA);
count++;
}
if (DynamicUtil.isInstance(utr, fivePrimeUTRCls)) {
fivePrimeUTR = utr;
} else {
threePrimeUTR = utr;
}
lastMRNA = mrna;
}
if (lastMRNA != null) {
// clone so we don't change the ObjectStore cache
InterMineObject tempMRNA = PostProcessUtil.cloneInterMineObject(lastMRNA);
tempMRNA.setFieldValue("fivePrimeUTR", fivePrimeUTR);
tempMRNA.setFieldValue("threePrimeUTR", threePrimeUTR);
osw.store(tempMRNA);
count++;
}
LOG.info(
"Stored MRNA "
+ count
+ " times ("
+ count * 2
+ " fields set)"
+ " - took "
+ (System.currentTimeMillis() - startTime)
+ " ms.");
osw.commitTransaction();
// now ANALYSE tables relating to class that has been altered - may be rows added
// to indirection tables
if (osw instanceof ObjectStoreWriterInterMineImpl) {
ClassDescriptor cld = model.getClassDescriptorByName("MRNA");
DatabaseUtil.analyse(((ObjectStoreWriterInterMineImpl) osw).getDatabase(), cld, false);
}
}