private void queueAll(List<IngestModuleAbstract> modules, final List<Content> inputs) {
int processed = 0;
for (Content input : inputs) {
final String inputName = input.getName();
final List<IngestModuleDataSource> dataSourceMods = new ArrayList<IngestModuleDataSource>();
final List<IngestModuleAbstractFile> fileMods = new ArrayList<IngestModuleAbstractFile>();
for (IngestModuleAbstract module : modules) {
if (isCancelled()) {
logger.log(Level.INFO, "Terminating ingest queueing due to cancellation.");
return;
}
final String moduleName = module.getName();
progress.progress(moduleName + " " + inputName, processed);
switch (module.getType()) {
case DataSource:
final IngestModuleDataSource newModuleInstance =
(IngestModuleDataSource) moduleLoader.getNewIngestModuleInstance(module);
if (newModuleInstance != null) {
dataSourceMods.add(newModuleInstance);
} else {
logger.log(
Level.INFO,
"Error loading module and adding input "
+ inputName
+ " with module "
+ module.getName());
}
break;
case AbstractFile:
// enqueue the same singleton AbstractFile module
logger.log(
Level.INFO,
"Adding input " + inputName + " for AbstractFileModule " + module.getName());
fileMods.add((IngestModuleAbstractFile) module);
break;
default:
logger.log(Level.SEVERE, "Unexpected module type: " + module.getType().name());
}
} // for modules
// queue to schedulers
// queue to datasource-level ingest pipeline(s)
final boolean processUnalloc = getProcessUnallocSpace();
final ScheduledTask<IngestModuleDataSource> dataSourceTask =
new ScheduledTask<IngestModuleDataSource>(input, dataSourceMods);
final PipelineContext<IngestModuleDataSource> dataSourcePipelineContext =
new PipelineContext<IngestModuleDataSource>(dataSourceTask, processUnalloc);
logger.log(Level.INFO, "Queing data source ingest task: " + dataSourceTask);
progress.progress("DataSource Ingest" + " " + inputName, processed);
final IngestScheduler.DataSourceScheduler dataSourceScheduler =
scheduler.getDataSourceScheduler();
dataSourceScheduler.schedule(dataSourcePipelineContext);
progress.progress("DataSource Ingest" + " " + inputName, ++processed);
// queue to file-level ingest pipeline
final ScheduledTask<IngestModuleAbstractFile> fTask = new ScheduledTask(input, fileMods);
final PipelineContext<IngestModuleAbstractFile> filepipelineContext =
new PipelineContext<IngestModuleAbstractFile>(fTask, processUnalloc);
logger.log(Level.INFO, "Queing file ingest task: " + fTask);
progress.progress("File Ingest" + " " + inputName, processed);
final IngestScheduler.FileScheduler fileScheduler = scheduler.getFileScheduler();
fileScheduler.schedule(filepipelineContext);
progress.progress("File Ingest" + " " + inputName, ++processed);
} // for data sources
// logger.log(Level.INFO, AbstractFileQueue.printQueue());
}
