コード例 #1
0
  private synchronized void submitCurrentBatch() throws InterruptedException {
    // Implement pauses at batch boundaries only
    pauser.blockIfPaused();

    if (currentBatch != null && currentBatch.size() > 0) {
      final Batch batch = new Batch(currentBatchNumber, currentBatch);

      // Prepare for the next batch
      currentBatch = null;
      importStatus.incrementTargetCounter(BulkImportStatus.TARGET_COUNTER_BATCHES_SUBMITTED);

      if (multiThreadedImport) {
        // Submit the batch to the thread pool
        submitBatch(batch);
      } else {
        // Import the batch directly on this thread
        batchImporter.importBatch(userId, target, batch, replaceExisting, dryRun);

        // Check if the multi-threading threshold has been reached
        multiThreadedImport = filePhase && currentBatchNumber >= MULTITHREADING_THRESHOLD;

        if (multiThreadedImport && debug(log))
          debug(
              log,
              "Multi-threading threshold ("
                  + MULTITHREADING_THRESHOLD
                  + " batch"
                  + pluralise(MULTITHREADING_THRESHOLD, "es")
                  + ") reached - switching to multi-threaded import.");
      }
    }
  }
コード例 #2
0
  private void shouldImportCsvData0(BatchingPageCache.WriterFactory delegateWriterFactory)
      throws Exception {
    // GIVEN
    Configuration config =
        new Configuration.Default() {
          @Override
          public int denseNodeThreshold() {
            return 30;
          }
        };
    BatchImporter inserter =
        new ParallelBatchImporter(
            directory.getAbsolutePath(),
            new DefaultFileSystemAbstraction(),
            config,
            new DetailedExecutionMonitor(),
            new IoQueue(config.numberOfIoThreads(), delegateWriterFactory));

    // WHEN
    int nodeCount = 100_000;
    int relationshipCount = nodeCount * 10;
    inserter.doImport(
        nodes(nodeCount), relationships(relationshipCount, nodeCount), IdMappers.actualIds());
    inserter.shutdown();

    // THEN
    GraphDatabaseService db =
        new GraphDatabaseFactory().newEmbeddedDatabase(directory.getAbsolutePath());
    try (Transaction tx = db.beginTx()) {
      verifyData(nodeCount, db);

      tx.success();
    } finally {
      db.shutdown();
    }
  }