Пример #1
0
  /** Writes a detailed informational status message to the log, at INFO level */
  private final void logStatusInfo() {
    if (info(log)) {
      try {
        final int batchesInProgress =
            importThreadPool.getQueueSize() + importThreadPool.getActiveCount();
        final Float batchesPerSecond =
            importStatus.getTargetCounterRate(
                BulkImportStatus.TARGET_COUNTER_BATCHES_COMPLETE, SECONDS);
        final Long estimatedCompletionTimeInNs = importStatus.getEstimatedRemainingDurationInNs();
        String message = null;

        if (batchesPerSecond != null && estimatedCompletionTimeInNs != null) {
          message =
              String.format(
                  "Multithreaded import in progress - %d batch%s yet to be imported. "
                      + "At current rate (%.3f batch%s per second), estimated completion in %s.",
                  batchesInProgress,
                  pluralise(batchesInProgress, "es"),
                  batchesPerSecond,
                  pluralise(batchesPerSecond, "es"),
                  getHumanReadableDuration(estimatedCompletionTimeInNs, false));
        } else {
          message =
              String.format(
                  "Multithreaded import in progress - %d batch%s yet to be imported.",
                  batchesInProgress, pluralise(batchesInProgress, "es"));
        }

        info(log, message);
      } catch (final IllegalFormatException ife) {
        // To help troubleshoot bugs in the String.format calls above
        error(log, ife);
      }
    }
  }
Пример #2
0
 /**
  * Used to submit a batch to the import thread pool. Note that this method can block (due to the
  * use of a blocking queue in the thread pool).
  *
  * @param batch The batch to submit <i>(may be null or empty, although that will result in a
  *     no-op)</i>.
  */
 private void submitBatch(final Batch batch) {
   if (batch != null && batch.size() > 0) {
     if (importStatus.inProgress() && !importStatus.isStopping()) {
       importThreadPool.execute(new BatchImportJob(batch));
     } else {
       if (warn(log)) warn(log, "New batch submitted during shutdown - ignoring new work.");
     }
   }
 }
Пример #3
0
  private synchronized void submitCurrentBatch() throws InterruptedException {
    // Implement pauses at batch boundaries only
    pauser.blockIfPaused();

    if (currentBatch != null && currentBatch.size() > 0) {
      final Batch batch = new Batch(currentBatchNumber, currentBatch);

      // Prepare for the next batch
      currentBatch = null;
      importStatus.incrementTargetCounter(BulkImportStatus.TARGET_COUNTER_BATCHES_SUBMITTED);

      if (multiThreadedImport) {
        // Submit the batch to the thread pool
        submitBatch(batch);
      } else {
        // Import the batch directly on this thread
        batchImporter.importBatch(userId, target, batch, replaceExisting, dryRun);

        // Check if the multi-threading threshold has been reached
        multiThreadedImport = filePhase && currentBatchNumber >= MULTITHREADING_THRESHOLD;

        if (multiThreadedImport && debug(log))
          debug(
              log,
              "Multi-threading threshold ("
                  + MULTITHREADING_THRESHOLD
                  + " batch"
                  + pluralise(MULTITHREADING_THRESHOLD, "es")
                  + ") reached - switching to multi-threaded import.");
      }
    }
  }
Пример #4
0
  /**
   * @see
   *     org.alfresco.extension.bulkimport.BulkImportCallback#submit(org.alfresco.extension.bulkimport.source.BulkImportItem)
   */
  @Override
  @SuppressWarnings({"rawtypes", "unchecked"})
  public synchronized void submit(final BulkImportItem item) throws InterruptedException {
    // PRECONDITIONS
    if (item == null) {
      throw new IllegalArgumentException(
          "Import source '"
              + source.getName()
              + "' has logic errors - a null import item was submitted.");
    }

    if (item.getVersions() == null || item.getVersions().size() <= 0) {
      throw new IllegalArgumentException(
          "Import source '"
              + source.getName()
              + "' has logic errors - an empty import item was submitted.");
    }

    // Body
    if (importStatus.isStopping() || Thread.currentThread().isInterrupted())
      throw new InterruptedException(
          Thread.currentThread().getName() + " was interrupted. Terminating early.");

    // If the weight of the new item would blow out the current batch, submit the batch as-is (i.e.
    // *before* adding the newly submitted item).
    // This ensures that heavy items start a new batch (and possibly end up in a batch by
    // themselves).
    int weight = weight(item);

    if (weightOfCurrentBatch + weight > batchWeight) {
      submitCurrentBatch();
    }

    // Create a new batch, if necessary
    if (currentBatch == null) {
      currentBatchNumber++;
      currentBatch = new ArrayList<>(batchWeight);
      weightOfCurrentBatch = 0;
    }

    // Finally, add the item to the current batch
    currentBatch.add(item);
    weightOfCurrentBatch += weight;
  }
Пример #5
0
  /** @see java.lang.Runnable#run() */
  @Override
  public void run() {
    boolean inPlacePossible = false;

    try {
      source.init(importStatus, parameters);
      inPlacePossible = source.inPlaceImportPossible();

      if (info(log))
        info(
            log,
            "Import ("
                + (inPlacePossible ? "in-place" : "streaming")
                + ") started from "
                + source.getName()
                + ".");

      importStatus.importStarted(
          userId, source, targetAsPath, importThreadPool, batchWeight, inPlacePossible, dryRun);

      // ------------------------------------------------------------------
      // Phase 1 - Folder scanning (single threaded)
      // ------------------------------------------------------------------

      source.scanFolders(importStatus, this);

      if (debug(log))
        debug(
            log,
            "Folder import complete in "
                + getHumanReadableDuration(importStatus.getDurationInNs())
                + ".");

      // ------------------------------------------------------------------
      // Phase 2 - File scanning
      // ------------------------------------------------------------------

      filePhase = true;

      // Maximise level of concurrency, since there's no longer any risk of out-of-order batches
      source.scanFiles(importStatus, this);

      if (debug(log))
        debug(
            log,
            "File scan complete in "
                + getHumanReadableDuration(importStatus.getDurationInNs())
                + ".");

      importStatus.scanningComplete();

      // ------------------------------------------------------------------
      // Phase 3 - Wait for multi-threaded import to complete and shutdown
      // ------------------------------------------------------------------

      submitCurrentBatch(); // Submit whatever is left in the final (partial) batch...
      awaitCompletion();

      if (debug(log))
        debug(log, "Import complete" + (multiThreadedImport ? ", thread pool shutdown" : "") + ".");
    } catch (final Throwable t) {
      Throwable rootCause = getRootCause(t);
      String rootCauseClassName = rootCause.getClass().getName();

      if (importStatus.isStopping()
          && (rootCause instanceof InterruptedException
              || rootCause instanceof ClosedByInterruptException
              || "com.hazelcast.core.RuntimeInterruptedException"
                  .equals(rootCauseClassName))) // For compatibility across 4.x *sigh*
      {
        // A stop import was requested
        if (debug(log))
          debug(log, Thread.currentThread().getName() + " was interrupted by a stop request.", t);
      } else {
        // An unexpected exception occurred during scanning - log it and kill the import
        error(log, "Bulk import from '" + source.getName() + "' failed.", t);
        importStatus.unexpectedError(t);
      }

      if (debug(log))
        debug(log, "Forcibly shutting down import thread pool and awaiting shutdown...");
      importThreadPool.shutdownNow();

      try {
        importThreadPool.awaitTermination(
            Long.MAX_VALUE,
            TimeUnit.DAYS); // Wait forever (technically merely a very long time, but whatevs...)
      } catch (final InterruptedException ie) {
        // Not much we can do here but log it and keep on truckin'
        if (warn(log))
          warn(
              log,
              Thread.currentThread().getName()
                  + " was interrupted while awaiting shutdown of import thread pool.",
              ie);
      }
    } finally {
      // Reset the thread factory
      if (importThreadPool.getThreadFactory() instanceof BulkImportThreadFactory) {
        ((BulkImportThreadFactory) importThreadPool.getThreadFactory()).reset();
      }

      // Mark the import complete
      importStatus.importComplete();

      // Invoke the completion handlers (if any)
      if (completionHandlers != null) {
        for (final BulkImportCompletionHandler handler : completionHandlers) {
          try {
            handler.importComplete(importStatus);
          } catch (final Exception e) {
            if (error(log))
              error(
                  log, "Completion handler threw an unexpected exception. It will be ignored.", e);
          }
        }
      }

      // Always invoke the logging completion handler last
      loggingBulkImportCompletionHandler.importComplete(importStatus);
    }
  }