public void processDocuments(
      int harvestType,
      List<DocumentPojo> toAdd,
      List<DocumentPojo> toUpdate_subsetOfAdd,
      List<DocumentPojo> toDelete) {
    PropertiesManager props = new PropertiesManager();

    // Note: toAdd = toAdd(old) + toUpdate
    // Need to treat updates as follows:
    // - Delete (inc children, eg events) but get fields to keep (currently _id, created; in the
    // future comments etc)

    // Delete toUpdate and toAdd (also overwriting "created" for updated docs, well all actually...)
    toDelete.addAll(toUpdate_subsetOfAdd);
    StoreAndIndexManager storageManager = new StoreAndIndexManager();
    storageManager.removeFromDatastore_byURL(toDelete, (harvestType != InfiniteEnums.DATABASE));
    // (note: expands toDelete if any sourceUrl "docs" are present, see FileHarvester)

    // (Storing docs messes up the doc/event/entity objects, so don't do that just yet...)

    // Aggregation:
    // 1+2. Create aggregate entities/events ("features") and write them to the DB
    // (then can store feeds - doesn't matter that the event/entities have been modified by the
    // aggregation)
    // 3. (Scheduled for efficiency) Update all documents' frequencies based on new entities and
    // events
    // 4. (Scheduled for efficiency) Synchronize with index [after this, queries can find them - so
    // (2) must have happened]
    // (Syncronization currently "corrupts" the entities so needs to be run last)

    AggregationManager perSourceAggregation = null;

    if (!props.getAggregationDisabled()) {
      perSourceAggregation = new AggregationManager();
    }

    // 1+2]
    if (null != perSourceAggregation) {
      perSourceAggregation.doAggregation(toAdd, toDelete);
      perSourceAggregation.createOrUpdateFeatureEntries();
    }

    // Save feeds to feeds collection in MongoDb
    // (second field determines if content gets saved)
    if (null != perSourceAggregation) {
      perSourceAggregation.applyAggregationToDocs(toAdd);
      // (First save aggregated statistics back to the docs' entity/event instances)
    }
    storeFeeds(toAdd, (harvestType != InfiniteEnums.DATABASE));

    // Then finish aggregation:

    if (null != perSourceAggregation) {
      // 3]
      perSourceAggregation.runScheduledDocumentUpdates();

      // 4] This needs to happen last because it "corrupts" the entities and events
      perSourceAggregation.runScheduledSynchronization();
    }
  } // TESTED (by eye - logic is v simple)
  /**
   * Used to start the sync service
   *
   * @throws IOException
   * @throws InterruptedException
   */
  public void startService(LinkedList<SourcePojo> sources)
      throws IOException, InterruptedException {
    // Let the client know the server is starting
    System.out.println("[SERVER] Harvest server is coming online");

    // Intialize/update generic process controller (do this here so that it blocks before threading
    // fun starts)
    new GenericProcessingController().Initialize();

    // Start the background aggregation thread (will do nothing if disabled)
    EntityBackgroundAggregationManager.startThread();
    AssociationBackgroundAggregationManager.startThread();

    _mainThread = Thread.currentThread();

    String hostname = "unknown.host";
    try {
      hostname = java.net.InetAddress.getLocalHost().getHostName();
    } catch (Exception e) {
    }

    // Add the shutdown hook
    ShutdownHook shutdownHook = new ShutdownHook();
    Runtime.getRuntime().addShutdownHook(shutdownHook);

    Date startDate = new Date();
    _logger.info("Starting harvest process at: " + startDate + ", host=" + hostname);

    // Perform processing

    PropertiesManager threadConfig = new PropertiesManager();
    String sThreadConfig = threadConfig.getHarvestThreadConfig();

    HashSet<String> types = new HashSet<String>();
    try {
      String harvestTypes =
          new com.ikanow.infinit.e.harvest.utils.PropertiesManager().getHarvesterTypes();
      for (String s : harvestTypes.split("\\s*,\\s*")) {
        types.add(s.toLowerCase());
      }
    } catch (Exception e) {
      _logger.error(
          Globals.populateStackTrace(new StringBuffer("Failed to register all harvest types"), e));
    } // TESTED (by hand)

    // Max time for harvester (defaults to 25 mins)

    long maxTime_secs = threadConfig.getMaximumHarvestTime();
    if (maxTime_secs > 0) {
      new Timer().schedule(new InternalShutdown(), maxTime_secs * 1000); // (arg in ms)
    } // TOTEST

    try {
      // All source types in a single thread

      int nThreads = Integer.parseInt(sThreadConfig);
      SourceTypeHarvesterRunnable allTypes = new SourceTypeHarvesterRunnable(sources, nThreads);
      _logger.info("(Launching " + nThreads + " threads for all source types)");
      allTypes.run();
    } catch (NumberFormatException e) {

      // The thread config must be comma-separated list of type:threads

      // (step over each type and launch that number of threads for that type)

      String[] sConfigBlocks = sThreadConfig.split("\\s*,\\s*");
      ExecutorService exec = Executors.newFixedThreadPool(sConfigBlocks.length);
      for (String sConfigBlock : sConfigBlocks) {
        String[] sTypeOrNumThreads = sConfigBlock.split("\\s*:\\s*");
        if (2 == sTypeOrNumThreads.length) {
          try {
            int nThreads = Integer.parseInt(sTypeOrNumThreads[1]);
            types.remove(sTypeOrNumThreads[0].toLowerCase());
            SourceTypeHarvesterRunnable typeRunner =
                new SourceTypeHarvesterRunnable(sources, nThreads, sTypeOrNumThreads[0]);
            _logger.info(
                "(Launching "
                    + nThreads
                    + " threads for "
                    + sTypeOrNumThreads[0]
                    + " source types)");
            exec.submit(typeRunner);
          } catch (NumberFormatException e2) {
            _logger.error("Error in harvester thread configuration: " + sThreadConfig);
          }
        } else {
          _logger.error("Error in harvester thread configuration: " + sThreadConfig);
        }
      } // (end loop over different file types)

      // (generate one thread for everything else)
      for (String unusedType : types) { // (note case unimportant)
        SourceTypeHarvesterRunnable typeRunner =
            new SourceTypeHarvesterRunnable(sources, 1, unusedType);
        _logger.info("(Launching 1 thread for " + unusedType + " source types)");
        exec.submit(typeRunner);
      } // TESTED (by hand)

      exec.shutdown();
      int i = 0;
      while (!exec.isTerminated()) {
        try {
          Thread.sleep(1000);
        } catch (InterruptedException e3) {
        }
        if (_bStopHarvest) i++;
        if (i > 14400) { // emergency shutdown time...
          _logger.error("Emergency shutdown after 4 hours of waiting for manual shutdown");
          System.exit(0);
        }
      }
    }
    com.ikanow.infinit.e.processing.generic.utils.PropertiesManager aggProps =
        new com.ikanow.infinit.e.processing.generic.utils.PropertiesManager();
    boolean bAggDisabled = aggProps.getAggregationDisabled();
    StoreAndIndexManager dataStore = new StoreAndIndexManager();
    boolean bResizedDB = dataStore.resizeDB();
    boolean deletedDocs = true;
    if (!bAggDisabled) {
      deletedDocs = AggregationManager.updateEntitiesFromDeletedDocuments(dataStore.getUUID());
    }
    if (deletedDocs) { // (or if agg disabled, in which case we don't know)
      dataStore.removeSoftDeletedDocuments();
      if (!bAggDisabled) {
        AggregationManager.updateDocEntitiesFromDeletedDocuments(dataStore.getUUID());
      }
    }
    if (bResizedDB) {
      _logger.info("(resized DB, now " + dataStore.getDatabaseSize() + " documents)");
    }

    HarvestController.logHarvesterStats();
    _logger.info("Completed harvest process at: " + new Date().toString());

    Date endDate = new Date();
    // Not allowed to cycle harvester runs too quickly
    // Sleep for some period:
    long nDiff = endDate.getTime() - startDate.getTime();
    long nToSleep = threadConfig.getMinimumHarvestTimeMs() - nDiff;
    if ((nToSleep > 0) && !_bCurrentlySleepingBeforeExit) {
      try {
        _bCurrentlySleepingBeforeExit =
            true; // (don't really care there's a minor race condition here)
        Thread.sleep(nToSleep);
      } catch (InterruptedException e) {
        // Do nothing, probably got a signal
      }
    } // TESTED (cut and paste from tested Beta code)

    // Stop background aggregation
    EntityBackgroundAggregationManager.stopThreadAndWait();
    AssociationBackgroundAggregationManager.stopThreadAndWait();

    _logger.info("Harvest server is going offline");
    _bStopHarvest = true;
    _bReadyToTerminate =
        true; // (if we were terminated manually tell the shutdown hook it can stop)
    System.exit(0);
  } // TESTED