/**
* Adds the configurations for data stores whose entities will be mapped using the engine created
* by this class.
*
* @param dataStores configurations of data stores that will have their entities mapped through
* this engine.
*/
public final void addDataStoreConfigurations(DataStoreConfiguration... dataStores) {
Args.notEmpty(dataStores, "Data stores");
for (DataStoreConfiguration config : dataStores) {
Args.notNull(config, "Data store configuration");
dataStoreConfigurations.add(config);
}
}
/**
* Adds instances of {@link CustomDataStoreFactory} that should be used by uniVocity to read any
* custom {@link DataStoreConfiguration}, provided by the user in the constructor of this class,
* to properly create instances of {@link CustomDataStore}.
*
* @param customFactories the factories that process user-provided data store configurations and
* generate custom data store instances.
*/
public final void addCustomDataStoreFactories(CustomDataStoreFactory<?>... customFactories) {
Args.notEmpty(customFactories, "Custom data store factories");
for (CustomDataStoreFactory<?> customFactory : customFactories) {
Args.notNull(customFactory, "Custom data store factory");
customDataStoreFactories.add(customFactory);
}
}
/**
* Creates a new engine configuration with the essential configuration required by uniVocity for
* enabling the definition and execution of data mappings.
*
* @param engineName the name of the new engine. The engine name is used to obtain instances of
* {@link DataIntegrationEngine} and manage them using {@link Univocity}
* @param dataStores optional parameter for the configurations of data stores that will have their
* entities mapped through this engine. More dataStores can be added later using {@link
* #addDataStoreConfigurations(DataStoreConfiguration...)}
*/
public EngineConfiguration(String engineName, DataStoreConfiguration... dataStores) {
Args.notBlank(engineName, "Engine name");
this.engineName = engineName;
if (dataStores != null && dataStores.length > 0) {
addDataStoreConfigurations(dataStores);
}
}
protected JobOutput executeJellyfishMerger(
Args args, String ecqName, Set<File> fileSet, File outputDir)
throws InterruptedException, ProcessExecutionException, ConanParameterException, IOException {
String suffix = "jellyfish_" + ecqName + "_all.jf31_0";
String jobName = args.getJobPrefix() + "-merge-" + suffix;
List<File> files = new ArrayList<>();
files.addAll(fileSet);
File outputFile = new File(outputDir, suffix);
JellyfishMergeV11 jellyfishMerge =
this.makeJellyfishMerge(files, outputFile, args.getOrganism());
ExecutionResult id =
this.conanExecutorService.executeProcess(
jellyfishMerge,
args.getOutputDir(),
jobName,
args.getThreadsPerProcess(),
args.getMemoryPerProcess(),
args.isRunParallel());
id.setName("merge-" + suffix);
return new JobOutput(id, outputFile);
}
protected JobOutput executeJellyfishCount(Args args, String ecqName, File outputDir, Library lib)
throws ProcessExecutionException, InterruptedException, ConanParameterException, IOException {
String suffix = "jellyfish_" + ecqName + "_" + lib.getName() + ".jf31";
// Create the process
JellyfishCountV11 jellyfishProcess =
this.makeJellyfishCount(
this.makeInputStringFromLib(lib),
new File(new File(outputDir, ecqName), suffix).getAbsolutePath(),
args.getOrganism(),
args.getThreadsPerProcess());
File outputFile = jellyfishProcess.getArgs().getOutputFile();
// Create a job name
String jobName = args.getJobPrefix() + "-count-" + suffix;
// Start jellyfish
final ExecutionResult id =
this.conanExecutorService.executeProcess(
jellyfishProcess,
new File(outputDir, ecqName),
jobName,
args.getThreadsPerProcess(),
args.getMemoryPerProcess(),
args.isRunParallel());
id.setName("count-" + suffix);
return new JobOutput(id, outputFile);
}
@Override
public List<Shape<V>> caseBuiltin(Builtin builtin, Args<V> arg) {
if (Debug) System.out.println("inside ShapePropgator, builtin fn is " + builtin);
if (Debug) System.out.println("the number of output variables is " + arg.getNargout());
if (builtin instanceof HasShapePropagationInfo) {
// call shape prop tool
ShapePropTool<V> shapePropTool = new ShapePropTool<V>();
@SuppressWarnings({"unchecked"})
List<Shape<V>> result =
shapePropTool.matchByValues(
((HasShapePropagationInfo<V>) builtin).getShapePropagationInfo(), arg);
return result;
}
throw new UnsupportedOperationException();
}
public CryptoMapper(final IRequestMapper wrappedMapper, final IProvider<ICrypt> cryptProvider) {
super();
this.wrappedMapper = (IRequestMapper) Args.notNull((Object) wrappedMapper, "wrappedMapper");
this.cryptProvider = (IProvider<ICrypt>) Args.notNull((Object) cryptProvider, "cryptProvider");
}
void doTest(String name) throws Exception {
Method m = tests.get(name);
Method m_check = tests.get(name + "_check");
Class[] paramTypes = m.getParameterTypes();
Object[] params = new Object[paramTypes.length];
Class retType = m.getReturnType();
boolean isIntArray =
(retType.isPrimitive() && !retType.equals(Void.TYPE))
|| (retType.equals(Void.TYPE) && paramTypes[0].getComponentType().isPrimitive())
|| (retType.isArray() && retType.getComponentType().isPrimitive());
Args args = m.getAnnotation(Args.class);
Object src = null;
switch (args.src()) {
case SMALL:
{
if (isIntArray) {
src = small_int_src;
} else {
src = small_a_src;
}
break;
}
case LARGE:
{
if (isIntArray) {
src = large_int_src;
} else {
src = large_a_src;
}
break;
}
case ZERO:
{
if (isIntArray) {
src = zero_int_src;
} else {
src = zero_a_src;
}
break;
}
}
for (int i = 0; i < 20000; i++) {
boolean failure = false;
int p = 0;
if (params.length > 0) {
if (isIntArray) {
params[0] = ((int[]) src).clone();
} else {
params[0] = ((A[]) src).clone();
}
p++;
}
if (params.length > 1) {
switch (args.dst()) {
case NEW:
{
if (isIntArray) {
params[1] = new int[((int[]) params[0]).length];
} else {
params[1] = new A[((A[]) params[0]).length];
}
p++;
break;
}
case SRC:
{
params[1] = params[0];
p++;
break;
}
case NONE:
break;
}
}
for (int j = 0; j < args.extra_args().length; j++) {
params[p + j] = args.extra_args()[j];
}
Object res = m.invoke(null, params);
if (retType.isPrimitive() && !retType.equals(Void.TYPE)) {
int s = (int) res;
int sum = 0;
int[] int_res = (int[]) src;
for (int j = 0; j < int_res.length; j++) {
sum += int_res[j];
}
failure = (s != sum);
if (failure) {
System.out.println("Test " + name + " failed: result = " + s + " != " + sum);
}
} else {
Object dest = null;
if (!retType.equals(Void.TYPE)) {
dest = res;
} else {
dest = params[1];
}
if (m_check != null) {
failure = (boolean) m_check.invoke(null, new Object[] {src, dest});
} else {
if (isIntArray) {
int[] int_res = (int[]) src;
int[] int_dest = (int[]) dest;
for (int j = 0; j < int_res.length; j++) {
if (int_res[j] != int_dest[j]) {
System.out.println(
"Test "
+ name
+ " failed for "
+ j
+ " src["
+ j
+ "]="
+ int_res[j]
+ ", dest["
+ j
+ "]="
+ int_dest[j]);
failure = true;
}
}
} else {
Object[] object_res = (Object[]) src;
Object[] object_dest = (Object[]) dest;
for (int j = 0; j < object_res.length; j++) {
if (object_res[j] != object_dest[j]) {
System.out.println(
"Test "
+ name
+ " failed for "
+ j
+ " src["
+ j
+ "]="
+ object_res[j]
+ ", dest["
+ j
+ "]="
+ object_dest[j]);
failure = true;
}
}
}
}
}
if (failure) {
success = false;
break;
}
}
}
@Override
public ExecutionResult execute(ExecutionContext executionContext)
throws ProcessExecutionException, InterruptedException {
try {
StopWatch stopWatch = new StopWatch();
stopWatch.start();
log.info("Starting Kmer Counting on all Reads");
// Create shortcut to args for convienience
Args args = this.getArgs();
// Force run parallel to false if not using a scheduler
if (!executionContext.usingScheduler() && args.isRunParallel()) {
log.warn("Forcing linear execution due to lack of job scheduler");
args.setRunParallel(false);
}
// Create the output directory
args.getOutputDir().mkdirs();
JobOutputMap jfCountOutputs = new JobOutputMap();
List<ExecutionResult> jobResults = new ArrayList<>();
List<ExecutionResult> allJobResults = new ArrayList<>();
// Create the output directory for the RAW datasets
File rawOutputDir = new File(args.getOutputDir(), "raw");
if (!rawOutputDir.exists()) {
rawOutputDir.mkdirs();
}
// Start jellyfish on all RAW datasets
for (Library lib : args.getAllLibraries()) {
// Execute jellyfish and add id to list of job ids
JobOutput jfOut = this.executeJellyfishCount(args, "raw", args.getOutputDir(), lib);
jobResults.add(jfOut.getResult());
allJobResults.add(jfOut.getResult());
jfCountOutputs.updateTracker("raw", jfOut.getOutputFile());
}
// Also start jellyfish on all the prep-processed libraries from MECQ
if (args.getAllMecqs() != null) {
for (Mecq.EcqArgs ecqArgs : args.getAllMecqs()) {
// Create the output directory for the RAW datasets
File ecqOutputDir = new File(args.getOutputDir(), ecqArgs.getName());
if (!ecqOutputDir.exists()) {
ecqOutputDir.mkdirs();
}
for (Library lib : ecqArgs.getOutputLibraries()) {
// Add jellyfish id to list of job ids
JobOutput jfOut =
this.executeJellyfishCount(args, ecqArgs.getName(), args.getOutputDir(), lib);
jobResults.add(jfOut.getResult());
allJobResults.add(jfOut.getResult());
jfCountOutputs.updateTracker(ecqArgs.getName(), jfOut.getOutputFile());
}
}
}
// If we're using a scheduler and we have been asked to run each job
// in parallel, then we should wait for all those to complete before continueing.
if (executionContext.usingScheduler() && args.isRunParallel()) {
log.info("Kmer counting all ECQ groups in parallel, waiting for completion");
this.conanExecutorService.executeScheduledWait(
jobResults,
args.getJobPrefix() + "-count-*",
ExitStatus.Type.COMPLETED_ANY,
args.getJobPrefix() + "-kmer-count-wait",
args.getOutputDir());
}
// Waiting point... clear job ids.
jobResults.clear();
JobOutputMap mergedOutputs = new JobOutputMap();
// Now execute merge jobs if required
for (Map.Entry<String, Set<File>> entry : jfCountOutputs.entrySet()) {
String ecqName = entry.getKey();
Set<File> fileSet = entry.getValue();
// Only merge if there's more than one library
if (fileSet.size() > 1) {
JobOutput jfOut =
this.executeJellyfishMerger(
args, ecqName, fileSet, new File(args.getOutputDir(), ecqName));
jobResults.add(jfOut.getResult());
allJobResults.add(jfOut.getResult());
mergedOutputs.updateTracker(ecqName, jfOut.getOutputFile());
}
}
// If we're using a scheduler and we have been asked to run each job
// in parallel, then we should wait for all those to complete before continueing.
if (executionContext.usingScheduler() && args.isRunParallel()) {
log.info(
"Creating merged kmer counts for all ECQ groups in parallel, waiting for completion");
this.conanExecutorService.executeScheduledWait(
jobResults,
args.getJobPrefix() + "-merge-*",
ExitStatus.Type.COMPLETED_ANY,
args.getJobPrefix() + "-kmer-merge-wait",
args.getOutputDir());
}
// Waiting point... clear job ids.
jobResults.clear();
// Combine all jellyfish out maps
jfCountOutputs.combine(mergedOutputs);
String katGcpJobPrefix = args.getJobPrefix() + "-kat-gcp";
// Run KAT GCP on everything
List<ExecutionResult> katGcpResults =
this.executeKatGcp(
jfCountOutputs,
katGcpJobPrefix,
args.getThreadsPerProcess(),
args.getMemoryPerProcess(),
args.isRunParallel());
for (ExecutionResult result : katGcpResults) {
result.setName(result.getName().substring(args.getJobPrefix().length() + 1));
jobResults.add(result);
allJobResults.add(result);
}
// If we're using a scheduler and we have been asked to run each job
// in parallel, then we should wait for all those to complete before continueing.
if (executionContext.usingScheduler() && args.isRunParallel()) {
log.info("Running \"kat gcp\" for all ECQ groups in parallel, waiting for completion");
this.conanExecutorService.executeScheduledWait(
jobResults,
katGcpJobPrefix + "*",
ExitStatus.Type.COMPLETED_ANY,
args.getJobPrefix() + "-kat-gcp-wait",
args.getOutputDir());
}
// Waiting point... clear job ids.
jobResults.clear();
log.info("Kmer counting of all reads finished.");
stopWatch.stop();
TaskResult taskResult =
new DefaultTaskResult(
"rampart-read_analysis-kmer", true, allJobResults, stopWatch.getTime() / 1000L);
// Output the resource usage to file
FileUtils.writeLines(
new File(args.getOutputDir(), args.getJobPrefix() + ".summary"), taskResult.getOutput());
return new DefaultExecutionResult(
taskResult.getTaskName(),
0,
new String[] {},
null,
-1,
new ResourceUsage(
taskResult.getMaxMemUsage(),
taskResult.getActualTotalRuntime(),
taskResult.getTotalExternalCputime()));
} catch (ConanParameterException | IOException e) {
throw new ProcessExecutionException(-1, e);
}
}