@Override
public QueryResult next(String chromosome, int position, QueryOptions options) {
String featureType = options.getString("featureType", null);
String featureClass = options.getString("featureClass", null);
BasicDBList chunksId = new BasicDBList();
String chunkId =
chromosome
+ "_"
+ getChunkId(position, regulatoryRegionChunkSize)
+ "_"
+ regulatoryRegionChunkSize / 1000
+ "k";
chunksId.add(chunkId);
// TODO: Add query to find next item considering next chunk
// db.regulatory_region.find({ "chromosome" : "19" , "start" : { "$gt" : 62005} , "featureType"
// : "TF_binding_site_motif"}).sort({start:1}).limit(1)
QueryBuilder builder;
if (options.getString("strand") == null
|| (options.getString("strand").equals("1") || options.getString("strand").equals("+"))) {
// db.core.find({chromosome: "1", start: {$gt: 1000000}}).sort({start: 1}).limit(1)
builder =
QueryBuilder.start("_chunkIds")
.in(chunksId)
.and("chromosome")
.is(chromosome)
.and("start")
.greaterThan(position);
options.put("sort", new BasicDBObject("start", 1));
options.put("limit", 1);
} else {
builder =
QueryBuilder.start("_chunkIds")
.in(chunksId)
.and("chromosome")
.is(chromosome)
.and("end")
.lessThan(position);
options.put("sort", new BasicDBObject("end", -1));
options.put("limit", 1);
}
if (featureType != null) {
builder.and("featureType").is(featureType);
}
if (featureClass != null) {
builder.and("featureClass").is(featureClass);
}
System.out.println(builder.get());
return executeQuery("result", builder.get(), options);
}
@Override
public List<QueryResult> getAllByPositionList(List<Position> positionList, QueryOptions options) {
// db.regulatory_region.find({"chunkIds": {$in:["1_200", "1_300"]}, "start": 601156})
String featureType = options.getString("featureType", null);
String featureClass = options.getString("featureClass", null);
List<DBObject> queries = new ArrayList<>();
for (Position position : positionList) {
String chunkId =
position.getChromosome()
+ "_"
+ getChunkId(position.getPosition(), regulatoryRegionChunkSize)
+ "_"
+ regulatoryRegionChunkSize / 1000
+ "k";
BasicDBList chunksId = new BasicDBList();
chunksId.add(chunkId);
QueryBuilder builder =
QueryBuilder.start("_chunkIds").in(chunksId).and("start").is(position.getPosition());
if (featureType != null) {
builder.and("featureType").is(featureType);
}
if (featureClass != null) {
builder.and("featureClass").is(featureClass);
}
// System.out.println("Query: " + builder.get());
queries.add(builder.get());
}
System.out.println("Query: " + queries);
// options = addExcludeReturnFields("chunkIds", options);
return executeQueryList2(positionList, queries, options);
}
public QueryResult<File> index(
int fileId, int outDirId, String storageEngine, String sessionId, QueryOptions options)
throws IOException, CatalogException, AnalysisExecutionException {
if (options == null) {
options = new QueryOptions();
}
final boolean execute = options.getBoolean(AnalysisJobExecuter.EXECUTE);
final boolean simulate = options.getBoolean(AnalysisJobExecuter.SIMULATE);
final boolean recordOutput = options.getBoolean(AnalysisJobExecuter.RECORD_OUTPUT);
final long start = System.currentTimeMillis();
/** Query catalog for user data. * */
String userId = catalogManager.getUserIdBySessionId(sessionId);
File file = catalogManager.getFile(fileId, sessionId).first();
File outDir = catalogManager.getFile(outDirId, sessionId).first();
int studyIdByOutDirId = catalogManager.getStudyIdByFileId(outDirId);
Study study = catalogManager.getStudy(studyIdByOutDirId, sessionId).getResult().get(0);
if (file.getType() != File.Type.FILE) {
throw new CatalogException(
"Expected file type = " + File.Type.FILE + " instead of " + file.getType());
}
final String dbName;
if (options.containsKey(DB_NAME)) {
dbName = options.getString(DB_NAME);
} else {
if (study.getAttributes().containsKey(DB_NAME)
&& study.getAttributes().get(DB_NAME) != null) {
dbName = study.getAttributes().get(DB_NAME).toString();
} else {
int projectId = catalogManager.getProjectIdByStudyId(study.getId());
String alias =
catalogManager
.getProject(projectId, new QueryOptions("include", "alias"), sessionId)
.first()
.getAlias();
dbName =
Config.getAnalysisProperties()
.getProperty(OPENCGA_ANALYSIS_STORAGE_DATABASE_PREFIX, "opencga_")
+ userId
+ "_"
+ alias;
}
}
// TODO: Check if file can be indexed
// ObjectMap to fill with modifications over the indexed file (like new attributes or jobId)
ObjectMap indexFileModifyParams = new ObjectMap("attributes", new ObjectMap());
/** Create temporal Job Outdir * */
final URI temporalOutDirUri;
final String randomString = "I_" + StringUtils.randomString(10);
if (simulate) {
temporalOutDirUri = createSimulatedOutDirUri(randomString);
} else {
temporalOutDirUri =
catalogManager.createJobOutDir(studyIdByOutDirId, randomString, sessionId);
}
List<Sample> sampleList;
/** Create index file* */
final File index;
if (options.containsKey(INDEX_FILE_ID)) {
logger.debug("Using an existing indexedFile.");
int indexFileId = options.getInt(INDEX_FILE_ID);
index = catalogManager.getFile(indexFileId, sessionId).first();
if (index.getType() != File.Type.INDEX) {
throw new CatalogException("Expected {type: INDEX} in IndexedFile " + indexFileId);
}
if (index.getStatus() != File.Status.READY) {
throw new CatalogException("Expected {status: READY} in IndexedFile " + indexFileId);
}
if (simulate) {
index.setStatus(File.Status.INDEXING);
} else {
ObjectMap parameters = new ObjectMap("status", File.Status.INDEXING);
catalogManager.modifyFile(index.getId(), parameters, sessionId);
}
/** Get file samples * */
sampleList =
catalogManager
.getAllSamples(study.getId(), new QueryOptions("id", index.getSampleIds()), sessionId)
.getResult();
} else {
/** Get file samples * */
sampleList = getFileSamples(study, file, indexFileModifyParams, simulate, options, sessionId);
String indexedFileDescription = "Indexation of " + file.getName() + " (" + fileId + ")";
String indexedFileName = file.getName() + "." + storageEngine;
String indexedFilePath = Paths.get(outDir.getPath(), indexedFileName).toString();
if (simulate) {
index =
new File(
-10,
indexedFileName,
File.Type.INDEX,
file.getFormat(),
file.getBioformat(),
indexedFilePath,
userId,
TimeUtils.getTime(),
indexedFileDescription,
File.Status.INDEXING,
-1,
-1,
null,
-1,
null,
null,
new HashMap<String, Object>());
} else {
index =
catalogManager
.createFile(
studyIdByOutDirId,
File.Type.INDEX,
file.getFormat(),
file.getBioformat(),
indexedFilePath,
null,
null,
indexedFileDescription,
File.Status.INDEXING,
0,
-1,
null,
-1,
null,
null,
false,
null,
sessionId)
.first();
}
}
/** Create commandLine * */
String commandLine =
createCommandLine(
study,
file,
index,
sampleList,
storageEngine,
temporalOutDirUri,
indexFileModifyParams,
dbName,
options);
if (options.containsKey(PARAMETERS)) {
List<String> extraParams = options.getAsStringList(PARAMETERS);
for (String extraParam : extraParams) {
commandLine += " " + extraParam;
}
}
/** Create job * */
ObjectMap jobResourceManagerAttributes = new ObjectMap();
jobResourceManagerAttributes.put(Job.TYPE, Job.Type.INDEX);
jobResourceManagerAttributes.put(Job.INDEXED_FILE_ID, index.getId());
String jobName = "index";
String jobDescription = "Indexing file " + file.getName() + " (" + fileId + ")";
final Job job =
AnalysisJobExecuter.createJob(
catalogManager,
studyIdByOutDirId,
jobName,
OPENCGA_STORAGE_BIN_NAME,
jobDescription,
outDir,
Collections.<Integer>emptyList(),
sessionId,
randomString,
temporalOutDirUri,
commandLine,
execute,
simulate,
recordOutput,
jobResourceManagerAttributes)
.first();
if (simulate) {
index.getAttributes().put("job", job);
// index.getAttributes().putAll(indexFileModifyParams.getMap("attributes"));
index.setSampleIds(indexFileModifyParams.getAsIntegerList("sampleIds"));
// VariantSource variantSource = (VariantSource)
// index.getAttributes().get("variantSource");
// for (Map.Entry<String, Integer> entry :
// variantSource.getSamplesPosition().entrySet()) {
// System.out.println("entry.getKey() = " + entry.getKey());
// System.out.println("entry.getValue() = " + entry.getValue());
// }
// for (String s : variantSource.getSamples()) {
// System.out.println("sample = " + s);
// }
// variantSource.setSamplesPosition(new HashMap<String, Integer>());
return new QueryResult<>(
"indexFile",
(int) (System.currentTimeMillis() - start),
1,
1,
"",
"",
Collections.singletonList(index));
} else {
/** Update IndexFile to add extra information (jobId, sampleIds, attributes, ...) * */
indexFileModifyParams.put("jobId", job.getId());
Set<Integer> jobIds;
try {
jobIds = new HashSet<>(new ObjectMap(index.getAttributes()).getAsIntegerList("jobIds"));
} catch (Exception ignore) {
jobIds = new HashSet<>(1);
}
if (index.getJobId() > 0) {
jobIds.add(index.getJobId());
}
jobIds.add(job.getId());
indexFileModifyParams.getMap("attributes").put("jobIds", jobIds);
catalogManager.modifyFile(index.getId(), indexFileModifyParams, sessionId).getResult();
return new QueryResult<>(
"indexFile",
(int) (System.currentTimeMillis() - start),
1,
1,
"",
"",
catalogManager.getFile(index.getId(), sessionId).getResult());
}
}