/** * Used to start the sync service * * @throws IOException * @throws InterruptedException */ public void startService(LinkedList<SourcePojo> sources) throws IOException, InterruptedException { // Let the client know the server is starting System.out.println("[SERVER] Harvest server is coming online"); // Intialize/update generic process controller (do this here so that it blocks before threading // fun starts) new GenericProcessingController().Initialize(); // Start the background aggregation thread (will do nothing if disabled) EntityBackgroundAggregationManager.startThread(); AssociationBackgroundAggregationManager.startThread(); _mainThread = Thread.currentThread(); String hostname = "unknown.host"; try { hostname = java.net.InetAddress.getLocalHost().getHostName(); } catch (Exception e) { } // Add the shutdown hook ShutdownHook shutdownHook = new ShutdownHook(); Runtime.getRuntime().addShutdownHook(shutdownHook); Date startDate = new Date(); _logger.info("Starting harvest process at: " + startDate + ", host=" + hostname); // Perform processing PropertiesManager threadConfig = new PropertiesManager(); String sThreadConfig = threadConfig.getHarvestThreadConfig(); HashSet<String> types = new HashSet<String>(); try { String harvestTypes = new com.ikanow.infinit.e.harvest.utils.PropertiesManager().getHarvesterTypes(); for (String s : harvestTypes.split("\\s*,\\s*")) { types.add(s.toLowerCase()); } } catch (Exception e) { _logger.error( Globals.populateStackTrace(new StringBuffer("Failed to register all harvest types"), e)); } // TESTED (by hand) // Max time for harvester (defaults to 25 mins) long maxTime_secs = threadConfig.getMaximumHarvestTime(); if (maxTime_secs > 0) { new Timer().schedule(new InternalShutdown(), maxTime_secs * 1000); // (arg in ms) } // TOTEST try { // All source types in a single thread int nThreads = Integer.parseInt(sThreadConfig); SourceTypeHarvesterRunnable allTypes = new SourceTypeHarvesterRunnable(sources, nThreads); _logger.info("(Launching " + nThreads + " threads for all source types)"); allTypes.run(); } catch (NumberFormatException e) { // The thread config must be comma-separated list of type:threads // (step over each type and launch that number of threads for that type) String[] sConfigBlocks = sThreadConfig.split("\\s*,\\s*"); ExecutorService exec = Executors.newFixedThreadPool(sConfigBlocks.length); for (String sConfigBlock : sConfigBlocks) { String[] sTypeOrNumThreads = sConfigBlock.split("\\s*:\\s*"); if (2 == sTypeOrNumThreads.length) { try { int nThreads = Integer.parseInt(sTypeOrNumThreads[1]); types.remove(sTypeOrNumThreads[0].toLowerCase()); SourceTypeHarvesterRunnable typeRunner = new SourceTypeHarvesterRunnable(sources, nThreads, sTypeOrNumThreads[0]); _logger.info( "(Launching " + nThreads + " threads for " + sTypeOrNumThreads[0] + " source types)"); exec.submit(typeRunner); } catch (NumberFormatException e2) { _logger.error("Error in harvester thread configuration: " + sThreadConfig); } } else { _logger.error("Error in harvester thread configuration: " + sThreadConfig); } } // (end loop over different file types) // (generate one thread for everything else) for (String unusedType : types) { // (note case unimportant) SourceTypeHarvesterRunnable typeRunner = new SourceTypeHarvesterRunnable(sources, 1, unusedType); _logger.info("(Launching 1 thread for " + unusedType + " source types)"); exec.submit(typeRunner); } // TESTED (by hand) exec.shutdown(); int i = 0; while (!exec.isTerminated()) { try { Thread.sleep(1000); } catch (InterruptedException e3) { } if (_bStopHarvest) i++; if (i > 14400) { // emergency shutdown time... _logger.error("Emergency shutdown after 4 hours of waiting for manual shutdown"); System.exit(0); } } } com.ikanow.infinit.e.processing.generic.utils.PropertiesManager aggProps = new com.ikanow.infinit.e.processing.generic.utils.PropertiesManager(); boolean bAggDisabled = aggProps.getAggregationDisabled(); StoreAndIndexManager dataStore = new StoreAndIndexManager(); boolean bResizedDB = dataStore.resizeDB(); boolean deletedDocs = true; if (!bAggDisabled) { deletedDocs = AggregationManager.updateEntitiesFromDeletedDocuments(dataStore.getUUID()); } if (deletedDocs) { // (or if agg disabled, in which case we don't know) dataStore.removeSoftDeletedDocuments(); if (!bAggDisabled) { AggregationManager.updateDocEntitiesFromDeletedDocuments(dataStore.getUUID()); } } if (bResizedDB) { _logger.info("(resized DB, now " + dataStore.getDatabaseSize() + " documents)"); } HarvestController.logHarvesterStats(); _logger.info("Completed harvest process at: " + new Date().toString()); Date endDate = new Date(); // Not allowed to cycle harvester runs too quickly // Sleep for some period: long nDiff = endDate.getTime() - startDate.getTime(); long nToSleep = threadConfig.getMinimumHarvestTimeMs() - nDiff; if ((nToSleep > 0) && !_bCurrentlySleepingBeforeExit) { try { _bCurrentlySleepingBeforeExit = true; // (don't really care there's a minor race condition here) Thread.sleep(nToSleep); } catch (InterruptedException e) { // Do nothing, probably got a signal } } // TESTED (cut and paste from tested Beta code) // Stop background aggregation EntityBackgroundAggregationManager.stopThreadAndWait(); AssociationBackgroundAggregationManager.stopThreadAndWait(); _logger.info("Harvest server is going offline"); _bStopHarvest = true; _bReadyToTerminate = true; // (if we were terminated manually tell the shutdown hook it can stop) System.exit(0); } // TESTED