/** Writes a detailed informational status message to the log, at INFO level */
private final void logStatusInfo() {
if (info(log)) {
try {
final int batchesInProgress =
importThreadPool.getQueueSize() + importThreadPool.getActiveCount();
final Float batchesPerSecond =
importStatus.getTargetCounterRate(
BulkImportStatus.TARGET_COUNTER_BATCHES_COMPLETE, SECONDS);
final Long estimatedCompletionTimeInNs = importStatus.getEstimatedRemainingDurationInNs();
String message = null;
if (batchesPerSecond != null && estimatedCompletionTimeInNs != null) {
message =
String.format(
"Multithreaded import in progress - %d batch%s yet to be imported. "
+ "At current rate (%.3f batch%s per second), estimated completion in %s.",
batchesInProgress,
pluralise(batchesInProgress, "es"),
batchesPerSecond,
pluralise(batchesPerSecond, "es"),
getHumanReadableDuration(estimatedCompletionTimeInNs, false));
} else {
message =
String.format(
"Multithreaded import in progress - %d batch%s yet to be imported.",
batchesInProgress, pluralise(batchesInProgress, "es"));
}
info(log, message);
} catch (final IllegalFormatException ife) {
// To help troubleshoot bugs in the String.format calls above
error(log, ife);
}
}
}
/**
* Used to submit a batch to the import thread pool. Note that this method can block (due to the
* use of a blocking queue in the thread pool).
*
* @param batch The batch to submit <i>(may be null or empty, although that will result in a
* no-op)</i>.
*/
private void submitBatch(final Batch batch) {
if (batch != null && batch.size() > 0) {
if (importStatus.inProgress() && !importStatus.isStopping()) {
importThreadPool.execute(new BatchImportJob(batch));
} else {
if (warn(log)) warn(log, "New batch submitted during shutdown - ignoring new work.");
}
}
}
private synchronized void submitCurrentBatch() throws InterruptedException {
// Implement pauses at batch boundaries only
pauser.blockIfPaused();
if (currentBatch != null && currentBatch.size() > 0) {
final Batch batch = new Batch(currentBatchNumber, currentBatch);
// Prepare for the next batch
currentBatch = null;
importStatus.incrementTargetCounter(BulkImportStatus.TARGET_COUNTER_BATCHES_SUBMITTED);
if (multiThreadedImport) {
// Submit the batch to the thread pool
submitBatch(batch);
} else {
// Import the batch directly on this thread
batchImporter.importBatch(userId, target, batch, replaceExisting, dryRun);
// Check if the multi-threading threshold has been reached
multiThreadedImport = filePhase && currentBatchNumber >= MULTITHREADING_THRESHOLD;
if (multiThreadedImport && debug(log))
debug(
log,
"Multi-threading threshold ("
+ MULTITHREADING_THRESHOLD
+ " batch"
+ pluralise(MULTITHREADING_THRESHOLD, "es")
+ ") reached - switching to multi-threaded import.");
}
}
}
/**
* @see
* org.alfresco.extension.bulkimport.BulkImportCallback#submit(org.alfresco.extension.bulkimport.source.BulkImportItem)
*/
@Override
@SuppressWarnings({"rawtypes", "unchecked"})
public synchronized void submit(final BulkImportItem item) throws InterruptedException {
// PRECONDITIONS
if (item == null) {
throw new IllegalArgumentException(
"Import source '"
+ source.getName()
+ "' has logic errors - a null import item was submitted.");
}
if (item.getVersions() == null || item.getVersions().size() <= 0) {
throw new IllegalArgumentException(
"Import source '"
+ source.getName()
+ "' has logic errors - an empty import item was submitted.");
}
// Body
if (importStatus.isStopping() || Thread.currentThread().isInterrupted())
throw new InterruptedException(
Thread.currentThread().getName() + " was interrupted. Terminating early.");
// If the weight of the new item would blow out the current batch, submit the batch as-is (i.e.
// *before* adding the newly submitted item).
// This ensures that heavy items start a new batch (and possibly end up in a batch by
// themselves).
int weight = weight(item);
if (weightOfCurrentBatch + weight > batchWeight) {
submitCurrentBatch();
}
// Create a new batch, if necessary
if (currentBatch == null) {
currentBatchNumber++;
currentBatch = new ArrayList<>(batchWeight);
weightOfCurrentBatch = 0;
}
// Finally, add the item to the current batch
currentBatch.add(item);
weightOfCurrentBatch += weight;
}
/** @see java.lang.Runnable#run() */
@Override
public void run() {
boolean inPlacePossible = false;
try {
source.init(importStatus, parameters);
inPlacePossible = source.inPlaceImportPossible();
if (info(log))
info(
log,
"Import ("
+ (inPlacePossible ? "in-place" : "streaming")
+ ") started from "
+ source.getName()
+ ".");
importStatus.importStarted(
userId, source, targetAsPath, importThreadPool, batchWeight, inPlacePossible, dryRun);
// ------------------------------------------------------------------
// Phase 1 - Folder scanning (single threaded)
// ------------------------------------------------------------------
source.scanFolders(importStatus, this);
if (debug(log))
debug(
log,
"Folder import complete in "
+ getHumanReadableDuration(importStatus.getDurationInNs())
+ ".");
// ------------------------------------------------------------------
// Phase 2 - File scanning
// ------------------------------------------------------------------
filePhase = true;
// Maximise level of concurrency, since there's no longer any risk of out-of-order batches
source.scanFiles(importStatus, this);
if (debug(log))
debug(
log,
"File scan complete in "
+ getHumanReadableDuration(importStatus.getDurationInNs())
+ ".");
importStatus.scanningComplete();
// ------------------------------------------------------------------
// Phase 3 - Wait for multi-threaded import to complete and shutdown
// ------------------------------------------------------------------
submitCurrentBatch(); // Submit whatever is left in the final (partial) batch...
awaitCompletion();
if (debug(log))
debug(log, "Import complete" + (multiThreadedImport ? ", thread pool shutdown" : "") + ".");
} catch (final Throwable t) {
Throwable rootCause = getRootCause(t);
String rootCauseClassName = rootCause.getClass().getName();
if (importStatus.isStopping()
&& (rootCause instanceof InterruptedException
|| rootCause instanceof ClosedByInterruptException
|| "com.hazelcast.core.RuntimeInterruptedException"
.equals(rootCauseClassName))) // For compatibility across 4.x *sigh*
{
// A stop import was requested
if (debug(log))
debug(log, Thread.currentThread().getName() + " was interrupted by a stop request.", t);
} else {
// An unexpected exception occurred during scanning - log it and kill the import
error(log, "Bulk import from '" + source.getName() + "' failed.", t);
importStatus.unexpectedError(t);
}
if (debug(log))
debug(log, "Forcibly shutting down import thread pool and awaiting shutdown...");
importThreadPool.shutdownNow();
try {
importThreadPool.awaitTermination(
Long.MAX_VALUE,
TimeUnit.DAYS); // Wait forever (technically merely a very long time, but whatevs...)
} catch (final InterruptedException ie) {
// Not much we can do here but log it and keep on truckin'
if (warn(log))
warn(
log,
Thread.currentThread().getName()
+ " was interrupted while awaiting shutdown of import thread pool.",
ie);
}
} finally {
// Reset the thread factory
if (importThreadPool.getThreadFactory() instanceof BulkImportThreadFactory) {
((BulkImportThreadFactory) importThreadPool.getThreadFactory()).reset();
}
// Mark the import complete
importStatus.importComplete();
// Invoke the completion handlers (if any)
if (completionHandlers != null) {
for (final BulkImportCompletionHandler handler : completionHandlers) {
try {
handler.importComplete(importStatus);
} catch (final Exception e) {
if (error(log))
error(
log, "Completion handler threw an unexpected exception. It will be ignored.", e);
}
}
}
// Always invoke the logging completion handler last
loggingBulkImportCompletionHandler.importComplete(importStatus);
}
}