private String selectBooks(
String schema, String collection, String title, String author, String isbn)
throws UnknownHostException {
System.out.println(schema);
System.out.println(collection);
System.out.println(title);
System.out.println(author);
List<String> l = new ArrayList<String>();
MongoDBConnectionManager mgr = MongoDBConnectionManager.getInstance();
Mongo mongo = mgr.getMongo();
DB db = mongo.getDB(schema);
try {
DBCollection c = db.getCollection(collection);
BasicDBObject titleQuery = new BasicDBObject();
titleQuery.put("title", new BasicDBObject("$regex", title));
BasicDBObject authorQuery = new BasicDBObject();
authorQuery.put("author", new BasicDBObject("$regex", author));
BasicDBList or = new BasicDBList();
or.add(titleQuery);
or.add(titleQuery);
DBObject query = new BasicDBObject("$and", or);
DBCursor cursor = c.find(titleQuery);
while (cursor.hasNext()) l.add(cursor.next().toString());
} catch (Exception e) {
System.out.println(e.getMessage());
}
System.out.println(l);
if (l.size() > 0) return l.toString();
else return null;
}
@Override
public QueryResponse getAllByPositionList(List<Position> positionList, QueryOptions options) {
// db.regulatory_region.find({"chunkIds": {$in:["1_200", "1_300"]}, "start": 601156})
String featureType = options.getString("featureType", null);
String featureClass = options.getString("featureClass", null);
List<DBObject> queries = new ArrayList<>();
for (Position position : positionList) {
String chunkId =
position.getChromosome() + "_" + getChunkId(position.getPosition(), CHUNKSIZE);
BasicDBList chunksId = new BasicDBList();
chunksId.add(chunkId);
QueryBuilder builder =
QueryBuilder.start("chunkIds").in(chunksId).and("start").is(position.getPosition());
if (featureType != null) {
builder.and("featureType").is(featureType);
}
if (featureClass != null) {
builder.and("featureClass").is(featureClass);
}
// System.out.println("Query: " + builder.get());
queries.add(builder.get());
}
System.out.println("Query: " + queries);
options = addExcludeReturnFields("chunkIds", options);
return executeQueryList(positionList, queries, options);
}
@Override
public QueryResult next(String chromosome, int position, QueryOptions options) {
String featureType = options.getString("featureType", null);
String featureClass = options.getString("featureClass", null);
BasicDBList chunksId = new BasicDBList();
String chunkId =
chromosome
+ "_"
+ getChunkId(position, regulatoryRegionChunkSize)
+ "_"
+ regulatoryRegionChunkSize / 1000
+ "k";
chunksId.add(chunkId);
// TODO: Add query to find next item considering next chunk
// db.regulatory_region.find({ "chromosome" : "19" , "start" : { "$gt" : 62005} , "featureType"
// : "TF_binding_site_motif"}).sort({start:1}).limit(1)
QueryBuilder builder;
if (options.getString("strand") == null
|| (options.getString("strand").equals("1") || options.getString("strand").equals("+"))) {
// db.core.find({chromosome: "1", start: {$gt: 1000000}}).sort({start: 1}).limit(1)
builder =
QueryBuilder.start("_chunkIds")
.in(chunksId)
.and("chromosome")
.is(chromosome)
.and("start")
.greaterThan(position);
options.put("sort", new BasicDBObject("start", 1));
options.put("limit", 1);
} else {
builder =
QueryBuilder.start("_chunkIds")
.in(chunksId)
.and("chromosome")
.is(chromosome)
.and("end")
.lessThan(position);
options.put("sort", new BasicDBObject("end", -1));
options.put("limit", 1);
}
if (featureType != null) {
builder.and("featureType").is(featureType);
}
if (featureClass != null) {
builder.and("featureClass").is(featureClass);
}
System.out.println(builder.get());
return executeQuery("result", builder.get(), options);
}
@Override
public List<QueryResult> getAllByRegionList(List<Region> regionList, QueryOptions options) {
// db.regulatory_region.find({"chunkIds": {$in:["1_200", "1_300"]}, "start": 601156})
QueryBuilder builder = new QueryBuilder();
List<Object> featureType = options.getList("featureType", null);
List<Object> featureClass = options.getList("featureClass", null);
// options = addExcludeReturnFields("chunkIds", options);
List<DBObject> queries = new ArrayList<>();
for (Region region : regionList) {
int firstChunkId = getChunkId(region.getStart(), regulatoryRegionChunkSize);
int lastChunkId = getChunkId(region.getEnd(), regulatoryRegionChunkSize);
BasicDBList chunksId = new BasicDBList();
for (int j = firstChunkId; j <= lastChunkId; j++) {
String chunkId =
region.getChromosome() + "_" + j + "_" + regulatoryRegionChunkSize / 1000 + "k";
chunksId.add(chunkId);
}
// logger.info(chunksId.toString());
builder =
builder
.start("_chunkIds")
.in(chunksId)
.and("start")
.lessThanEquals(region.getEnd())
.and("end")
.greaterThanEquals(region.getStart());
if (featureType != null && featureType.size() > 0) {
BasicDBList featureTypeDBList = new BasicDBList();
featureTypeDBList.addAll(featureType);
builder = builder.and("featureType").in(featureTypeDBList);
}
if (featureClass != null && featureClass.size() > 0) {
BasicDBList featureClassDBList = new BasicDBList();
featureClassDBList.addAll(featureClass);
builder = builder.and("featureClass").in(featureClassDBList);
}
queries.add(builder.get());
}
// System.out.println(">>"+regionList);
// System.out.println(">>"+builder.get().toString());
return executeQueryList2(regionList, queries, options);
}
/**
* Returns the current CommitEntry as a mongo DBObject.
*
* @return DBObject representing the CommitEntry
*/
public DBObject asDBObject() {
final BasicDBList events = new BasicDBList();
BasicDBObjectBuilder commitBuilder =
BasicDBObjectBuilder.start()
.add(AGGREGATE_IDENTIFIER_PROPERTY, aggregateIdentifier)
.add(SEQUENCE_NUMBER_PROPERTY, firstSequenceNumber)
.add(LAST_SEQUENCE_NUMBER_PROPERTY, lastSequenceNumber)
.add(FIRST_SEQUENCE_NUMBER_PROPERTY, firstSequenceNumber)
.add(TIME_STAMP_PROPERTY, firstTimestamp)
.add(FIRST_TIME_STAMP_PROPERTY, firstTimestamp)
.add(LAST_TIME_STAMP_PROPERTY, lastTimestamp)
.add(EVENTS_PROPERTY, events);
for (EventEntry eventEntry : eventEntries) {
events.add(eventEntry.asDBObject());
}
return commitBuilder.get();
}
private BasicDBList processGeneList(String genes) {
BasicDBList list = new BasicDBList();
Client wsRestClient = Client.create();
WebResource webResource =
wsRestClient.resource("http://ws.bioinfo.cipf.es/cellbase/rest/latest/hsa/feature/gene/");
ObjectMapper mapper = new ObjectMapper();
String response =
webResource.path(genes).path("info").queryParam("of", "json").get(String.class);
try {
JsonNode actualObj = mapper.readTree(response);
Iterator<JsonNode> it = actualObj.iterator();
Iterator<JsonNode> aux;
while (it.hasNext()) {
JsonNode node = it.next();
if (node.isArray()) {
aux = node.iterator();
while (aux.hasNext()) {
JsonNode auxNode = aux.next();
DBObject regionClause = new BasicDBObject("chr", auxNode.get("chromosome").asText());
regionClause.put(
"pos",
new BasicDBObject("$gte", auxNode.get("start").asInt())
.append("$lte", auxNode.get("end").asInt()));
list.add(regionClause);
}
}
}
} catch (IOException e) {
e.printStackTrace();
}
return list;
}
@Override
public QueryResponse getAllByRegionList(List<Region> regionList, QueryOptions options) {
// db.regulatory_region.find({"chunkIds": {$in:["1_200", "1_300"]}, "start": 601156})
String featureType = options.getString("featureType", null);
String featureClass = options.getString("featureClass", null);
List<DBObject> queries = new ArrayList<>();
List<String> ids = new ArrayList<>(regionList.size());
for (Region region : regionList) {
int firstChunkId = getChunkId(region.getStart(), CHUNKSIZE);
int lastChunkId = getChunkId(region.getEnd(), CHUNKSIZE);
BasicDBList chunksId = new BasicDBList();
for (int j = firstChunkId; j <= lastChunkId; j++) {
String chunkId = region.getChromosome() + "_" + j;
chunksId.add(chunkId);
}
QueryBuilder builder =
QueryBuilder.start("chunkIds")
.in(chunksId)
.and("start")
.lessThanEquals(region.getEnd())
.and("end")
.greaterThanEquals(region.getStart());
if (featureType != null) {
builder.and("featureType").is(featureType);
}
if (featureClass != null) {
builder.and("featureClass").is(featureClass);
}
System.out.println("Query: " + builder.get());
queries.add(builder.get());
ids.add(region.toString());
}
options = addExcludeReturnFields("chunkIds", options);
return executeQueryList(ids, queries, options);
}
public QueryResult<VariantInfo> getRecordsMongo(
int page, int start, int limit, MutableInt count, Map<String, String> options) {
long startTime = System.currentTimeMillis();
QueryResult<VariantInfo> queryResult = new QueryResult<>();
List<VariantInfo> res = new ArrayList<>();
String sourceId = options.get("studyId");
DBCollection coll = db.getCollection("variants");
BasicDBObject elemMatch = new BasicDBObject("sourceId", sourceId);
DBObject query = new BasicDBObject();
BasicDBList orList = new BasicDBList();
Map<String, List<String>> sampleGenotypes = processSamplesGT(options);
System.out.println("map = " + options);
if (options.containsKey("region") && !options.get("region").equals("")) {
String[] regions = options.get("region").split(",");
Pattern pattern = Pattern.compile("(\\w+):(\\d+)-(\\d+)");
Matcher matcher, matcherChr;
for (int i = 0; i < regions.length; i++) {
String region = regions[i];
matcher = pattern.matcher(region);
if (matcher.find()) {
String chr = matcher.group(1);
int s = Integer.valueOf(matcher.group(2));
int e = Integer.valueOf(matcher.group(3));
DBObject regionClause = new BasicDBObject("chr", chr);
regionClause.put("pos", new BasicDBObject("$gte", s).append("$lte", e));
orList.add(regionClause);
} else {
Pattern patternChr = Pattern.compile("(\\w+)");
matcherChr = patternChr.matcher(region);
if (matcherChr.find()) {
String chr = matcherChr.group();
DBObject regionClause = new BasicDBObject("chr", chr);
orList.add(regionClause);
}
}
}
query.put("$or", orList);
} else if (options.containsKey("genes") && !options.get("genes").equals("")) {
orList = processGeneList(options.get("genes"));
if (orList.size() > 0) {
query.put("$or", orList);
} else {
queryResult.setWarningMsg("Wrong gene name");
queryResult.setResult(res);
queryResult.setNumResults(res.size());
return queryResult;
}
}
if (options.containsKey("conseq_type") && !options.get("conseq_type").equals("")) {
String[] cts = options.get("conseq_type").split(",");
BasicDBList ctList = new BasicDBList();
for (String ct : cts) {
ctList.add(ct);
}
elemMatch.put("effects", new BasicDBObject("$in", ctList));
}
if (sampleGenotypes.size() > 0) {
for (Map.Entry<String, List<String>> entry : sampleGenotypes.entrySet()) {
BasicDBList gtList = new BasicDBList();
for (String gt : entry.getValue()) {
gtList.add(gt);
}
elemMatch.put("samples." + entry.getKey() + ".GT", new BasicDBObject("$in", gtList));
}
}
if (options.containsKey("miss_gt") && !options.get("miss_gt").equalsIgnoreCase("")) {
Integer val = Integer.valueOf(options.get("miss_gt"));
Object missGt = getMongoOption(options.get("option_miss_gt"), val);
elemMatch.put("stats.missGenotypes", missGt);
}
BasicDBList andControls = new BasicDBList();
if (options.containsKey("maf_1000g_controls")
&& !options.get("maf_1000g_controls").equalsIgnoreCase("")) {
BasicDBList or = new BasicDBList();
or.add(new BasicDBObject("attributes.1000G_maf", new BasicDBObject("$exists", false)));
or.add(
new BasicDBObject(
"attributes.1000G_maf",
new BasicDBObject("$lte", options.get("maf_1000g_controls"))));
andControls.add(new BasicDBObject("$or", or));
}
if (options.containsKey("maf_1000g_afr_controls")
&& !options.get("maf_1000g_afr_controls").equalsIgnoreCase("")) {
BasicDBList or = new BasicDBList();
or.add(new BasicDBObject("attributes.1000G_AFR_maf", new BasicDBObject("$exists", false)));
or.add(
new BasicDBObject(
"attributes.1000G_AFR_maf",
new BasicDBObject("$lte", options.get("maf_1000g_afr_controls"))));
andControls.add(new BasicDBObject("$or", or));
}
if (options.containsKey("maf_1000g_asi_controls")
&& !options.get("maf_1000g_asi_controls").equalsIgnoreCase("")) {
BasicDBList or = new BasicDBList();
or.add(new BasicDBObject("attributes.1000G_ASI_maf", new BasicDBObject("$exists", false)));
or.add(
new BasicDBObject(
"attributes.1000G_ASI_maf",
new BasicDBObject("$lte", options.get("maf_1000g_asi_controls"))));
andControls.add(new BasicDBObject("$or", or));
}
if (options.containsKey("maf_1000g_eur_controls")
&& !options.get("maf_1000g_eur_controls").equalsIgnoreCase("")) {
System.out.print("EUR");
BasicDBList or = new BasicDBList();
or.add(new BasicDBObject("attributes.1000G_EUR_maf", new BasicDBObject("$exists", false)));
or.add(
new BasicDBObject(
"attributes.1000G_EUR_maf",
new BasicDBObject("$lte", options.get("maf_1000g_eur_controls"))));
andControls.add(new BasicDBObject("$or", or));
}
if (options.containsKey("maf_1000g_ame_controls")
&& !options.get("maf_1000g_ame_controls").equalsIgnoreCase("")) {
System.out.print("AME");
BasicDBList or = new BasicDBList();
or.add(new BasicDBObject("attributes.1000G_AME_maf", new BasicDBObject("$exists", false)));
or.add(
new BasicDBObject(
"attributes.1000G_AME_maf",
new BasicDBObject("$lte", options.get("maf_1000g_ame_controls"))));
andControls.add(new BasicDBObject("$or", or));
}
if (options.containsKey("maf_evs_controls")
&& !options.get("maf_evs_controls").equalsIgnoreCase("")) {
BasicDBList or = new BasicDBList();
or.add(new BasicDBObject("attributes.EVS_maf", new BasicDBObject("$exists", false)));
or.add(
new BasicDBObject(
"attributes.EVS_maf", new BasicDBObject("$lte", options.get("maf_evs_controls"))));
andControls.add(new BasicDBObject("$or", or));
}
if (options.containsKey("maf_bier_controls")
&& !options.get("maf_bier_controls").equalsIgnoreCase("")) {
BasicDBList or = new BasicDBList();
or.add(new BasicDBObject("attributes.BIER_maf", new BasicDBObject("$exists", false)));
or.add(
new BasicDBObject(
"attributes.BIER_maf", new BasicDBObject("$lte", options.get("maf_bier_controls"))));
andControls.add(new BasicDBObject("$or", or));
}
if (andControls.size() > 0) {
elemMatch.append("$and", andControls);
}
query.put("sources", new BasicDBObject("$elemMatch", elemMatch));
System.out.println("#############################");
System.out.println(query);
System.out.println("#############################");
long dbStart = System.currentTimeMillis();
DBObject sort = null;
DBCursor cursor;
if (options.containsKey("sort")) {
sort = getQuerySort(options.get("sort"));
cursor = coll.find(query).sort(sort).skip(start).limit(limit);
} else {
cursor = coll.find(query).skip(start).limit(limit);
}
count.setValue(cursor.count());
queryResult.setDbTime(dbStart - System.currentTimeMillis());
for (DBObject obj : cursor) {
BasicDBObject elem = (BasicDBObject) obj;
VariantInfo vi = new VariantInfo();
VariantStats vs = new VariantStats();
String chr = elem.getString("chr");
int pos = elem.getInt("pos");
vi.setChromosome(chr);
vi.setPosition(pos);
BasicDBList studies = (BasicDBList) elem.get("sources");
Iterator<Object> it = studies.iterator();
while (it.hasNext()) {
BasicDBObject study = (BasicDBObject) it.next();
if (study.getString("sourceId").equalsIgnoreCase(sourceId)) {
BasicDBObject stats = (BasicDBObject) study.get("stats");
String ref = study.getString("ref");
BasicDBList alt = (BasicDBList) study.get("alt");
vi.setRef(ref);
vi.setAlt(Joiner.on(",").join(alt.toArray()));
vs.setMaf((float) stats.getDouble("maf"));
vs.setMgf((float) stats.getDouble("mgf"));
vs.setMafAllele(stats.getString("alleleMaf"));
vs.setMgfAllele(stats.getString("genotypeMaf"));
vs.setMissingAlleles(stats.getInt("missAllele"));
vs.setMissingGenotypes(stats.getInt("missGenotypes"));
vs.setMendelinanErrors(stats.getInt("mendelErr"));
vs.setCasesPercentDominant((float) stats.getDouble("casesPercentDominant"));
vs.setControlsPercentDominant((float) stats.getDouble("controlsPercentDominant"));
vs.setCasesPercentRecessive((float) stats.getDouble("casesPercentRecessive"));
vs.setControlsPercentRecessive((float) stats.getDouble("controlsPercentRecessive"));
BasicDBObject samples = (BasicDBObject) study.get("samples");
for (String sampleName : samples.keySet()) {
DBObject sample = (DBObject) samples.get(sampleName);
if (sample.containsField("GT")) {
String sampleGT = (String) sample.get("GT");
vi.addSammpleGenotype(sampleName, sampleGT);
}
}
vi.setSnpid((String) study.get("snpId"));
if (study.containsField("effects")) {
BasicDBList conseqTypes = (BasicDBList) study.get("effects");
conseqTypes.remove("");
String cts = Joiner.on(",").join(conseqTypes.iterator());
vi.addConsequenceTypes(cts);
}
if (study.containsField("genes")) {
BasicDBList genesList = (BasicDBList) study.get("genes");
String genes = Joiner.on(",").join(genesList.iterator());
vi.addGenes(genes);
}
if (study.containsField("attributes")) {
BasicDBObject attr = (BasicDBObject) study.get("attributes");
if (attr.containsField("1000G_maf")) {
vi.addControl("1000G_maf", (String) attr.get("1000G_maf"));
vi.addControl("1000G_amaf", (String) attr.get("1000G_amaf"));
vi.addControl("1000G_gt", (String) attr.get("1000G_gt"));
}
if (attr.containsField("1000G_ASI_maf")) {
vi.addControl("1000G-ASI_maf", (String) attr.get("1000G_ASI_maf"));
vi.addControl("1000G-ASI_amaf", (String) attr.get("1000G_ASI_amaf"));
vi.addControl("1000G-ASI_gt", (String) attr.get("1000G_ASI_gt"));
}
if (attr.containsField("1000G_AFR_maf")) {
vi.addControl("1000G-AFR_maf", (String) attr.get("1000G_AFR_maf"));
vi.addControl("1000G-AFR_amaf", (String) attr.get("1000G_AFR_amaf"));
vi.addControl("1000G-AFR_gt", (String) attr.get("1000G_AFR_gt"));
}
if (attr.containsField("1000G_AME_maf")) {
vi.addControl("1000G-AME_maf", (String) attr.get("1000G_AME_maf"));
vi.addControl("1000G-AME_amaf", (String) attr.get("1000G_AME_amaf"));
vi.addControl("1000G-AME_gt", (String) attr.get("1000G_AME_gt"));
}
if (attr.containsField("1000G_EUR_maf")) {
vi.addControl("1000G-EUR_maf", (String) attr.get("1000G_EUR_maf"));
vi.addControl("1000G-EUR_amaf", (String) attr.get("1000G_EUR_amaf"));
vi.addControl("1000G-EUR_gt", (String) attr.get("1000G_EUR_gt"));
}
if (attr.containsField("EVS_maf")) {
vi.addControl("EVS_maf", (String) attr.get("EVS_maf"));
vi.addControl("EVS_amaf", (String) attr.get("EVS_amaf"));
vi.addControl("EVS_gt", (String) attr.get("EVS_gt"));
}
if (attr.containsField("BIER_maf")) {
vi.addControl("BIER_maf", (String) attr.get("BIER_maf"));
vi.addControl("BIER_amaf", (String) attr.get("BIER_amaf"));
vi.addControl("BIER_gt", (String) attr.get("BIER_gt"));
}
if (attr.containsField("PolyphenScore")) {
vi.setPolyphen_score(Double.parseDouble(attr.getString("PolyphenScore")));
vi.setPolyphen_effect(Integer.parseInt(attr.getString("PolyphenEffect")));
}
if (attr.containsField("SIFTScore")) {
vi.setSift_score(Double.parseDouble(attr.getString("SIFTScore")));
vi.setSift_effect(Integer.parseInt(attr.getString("SIFTEffect")));
}
}
continue;
}
}
vi.addStats(vs);
res.add(vi);
}
queryResult.setResult(res);
queryResult.setTime(startTime - System.currentTimeMillis());
return queryResult;
}
@Test
public void testMetadata() throws TranslatorException {
MongoDBMetadataProcessor mp = new MongoDBMetadataProcessor();
MetadataFactory mf =
new MetadataFactory(
"vdb",
1,
"mongodb",
SystemMetadata.getInstance().getRuntimeTypeMap(),
new Properties(),
null);
MongoDBConnection conn = Mockito.mock(MongoDBConnection.class);
DBCollection dbCollection = Mockito.mock(DBCollection.class);
HashSet<String> tables = new HashSet<String>();
tables.add("table");
DB db = Mockito.mock(DB.class);
BasicDBList array = new BasicDBList();
array.add("one");
array.add("two");
BasicDBObject row = new BasicDBObject();
row.append("_id", new Integer(1));
row.append("col2", new Double(2.0));
row.append("col3", new Long(3L));
row.append("col5", Boolean.TRUE);
row.append("col6", new Date(0L));
row.append("col6", new DBRef(db, "ns", "one"));
row.append("col7", array);
BasicDBObject child = new BasicDBObject();
child.append("col1", "one");
child.append("col2", "two");
row.append("child", child);
Mockito.stub(db.getCollectionNames()).toReturn(tables);
Mockito.stub(db.getCollection(Mockito.anyString())).toReturn(dbCollection);
Mockito.stub(dbCollection.findOne()).toReturn(row);
Mockito.stub(conn.getDatabase()).toReturn(db);
mp.process(mf, conn);
String metadataDDL = DDLStringVisitor.getDDLString(mf.getSchema(), null, null);
String expected =
"CREATE FOREIGN TABLE child (\n"
+ "\tcol1 string,\n"
+ "\tcol2 string\n"
+ ") OPTIONS (UPDATABLE TRUE, \"teiid_mongo:MERGE\" 'table');\n"
+ "\n"
+ "CREATE FOREIGN TABLE \"table\" (\n"
+ "\t\"_id\" integer,\n"
+ "\tcol2 double,\n"
+ "\tcol3 long,\n"
+ "\tcol5 boolean,\n"
+ "\tcol6 string,\n"
+ "\tcol7 string[] OPTIONS (SEARCHABLE 'Unsearchable'),\n"
+ "\tCONSTRAINT PK0 PRIMARY KEY(\"_id\"),\n"
+ "\tCONSTRAINT FK_col6 FOREIGN KEY(col6) REFERENCES ns \n"
+ ") OPTIONS (UPDATABLE TRUE);";
assertEquals(expected, metadataDDL);
}