/** * Drives the actual test on an Executor and verifies the result * * @param maps the caches to be tested * @throws IOException * @throws InterruptedException */ private void testConcurrentLocking(List<ConcurrentMap<String, String>> maps) throws IOException, InterruptedException { SharedStats stats = new SharedStats(); ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(NODES_NUM); List<StressingThread> threads = new ArrayList<StressingThread>(); for (ConcurrentMap<String, String> map : maps) { StressingThread thread = new StressingThread(stats, map); threads.add(thread); executor.execute(thread); } executor.shutdown(); Thread.sleep(5000); int putsAfter5Seconds = stats.succesfullPutsCounter.get(); System.out.println("\nSituation after 5 seconds:"); System.out.println(stats.toString()); executor.awaitTermination(STRESS_TIME_MINUTES, TimeUnit.MINUTES); stats.globalQuit = true; executor.awaitTermination(10, TimeUnit.SECONDS); // give some time to awake and quit executor.shutdownNow(); System.out.println("\nFinal situation:"); System.out.println(stats.toString()); assert !stats.seenFailures : "at least one thread has seen unexpected state"; assert stats.succesfullPutsCounter.get() > 0 : "the lock should have been taken at least once"; assert stats.succesfullPutsCounter.get() > putsAfter5Seconds : "the lock count didn't improve since the first 5 seconds. Deadlock?"; assert stats.succesfullPutsCounter.get() == stats.lockReleasedCounter.get() : "there's a mismatch in acquires and releases count"; assert stats.lockOwnersCounter.get() == 0 : "the lock is still held at test finish"; }
private static void realMain(String[] args) throws Throwable { final int n = 4; final CyclicBarrier barrier = new CyclicBarrier(2 * n + 1); final ThreadPoolExecutor pool = new ThreadPoolExecutor( n, 2 * n, KEEPALIVE_MS, MILLISECONDS, new SynchronousQueue<Runnable>()); final Runnable r = new Runnable() { public void run() { try { barrier.await(); barrier.await(); } catch (Throwable t) { unexpected(t); } } }; for (int i = 0; i < 2 * n; i++) pool.execute(r); barrier.await(); checkPoolSizes(pool, 2 * n, n, 2 * n); barrier.await(); long nap = KEEPALIVE_MS + (KEEPALIVE_MS >> 2); for (long sleepyTime = 0L; pool.getPoolSize() > n; ) { check((sleepyTime += nap) <= LONG_DELAY_MS); Thread.sleep(nap); } checkPoolSizes(pool, n, n, 2 * n); Thread.sleep(nap); checkPoolSizes(pool, n, n, 2 * n); pool.shutdown(); check(pool.awaitTermination(LONG_DELAY_MS, MILLISECONDS)); }
@Test public void testConcurrentPutGet() throws NetInfCheckedException, InterruptedException { List<InformationObject> insertedIOs = new ArrayList<InformationObject>(); ThreadPoolExecutor executor = new ThreadPoolExecutor( 5, 10, 10, TimeUnit.SECONDS, new ArrayBlockingQueue<Runnable>(10000)); for (int i = 0; i < NODE_NUMBER; i++) { for (int j = 0; j < IOS_PER_NODE; j++) { InformationObject io = createUniqueIO(); executor.execute(new NodePutCommand(resolutionServices.get(i), io)); insertedIOs.add(io); } } for (InformationObject io : insertedIOs) { NodeGetCommand getter = new NodeGetCommand(resolutionServices.get(0), io); executor.execute(getter); getterCommands.add(getter); } executor.shutdown(); executor.awaitTermination(30, TimeUnit.SECONDS); for (NodeGetCommand getter : getterCommands) { Assert.assertTrue(getter.isCorrect()); } }
public static void execute( final String fromDirectoryPath, final String toFirstPartDirectoryPath, final String toSecondPartDirectoryPath) { ThreadPoolExecutor executor = new ThreadPoolExecutor( 100, 500, Long.MAX_VALUE, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>()); long start = System.currentTimeMillis(); List<String> fileNames = produceInput(fromDirectoryPath); long mid = System.currentTimeMillis(); System.out.println( "Time Taken for producing input: " + Double.valueOf((mid - start) / 1000) + " seconds"); for (String filePath : fileNames) { EtlTask readerTask = new FileSystemEtlTask(filePath, toFirstPartDirectoryPath, toSecondPartDirectoryPath); executor.submit(readerTask); } long end = System.currentTimeMillis(); System.out.println( "Time Taken for creating tasks: " + Double.valueOf((end - start) / 1000) + " seconds"); executor.shutdown(); try { executor.awaitTermination(Long.MAX_VALUE, TimeUnit.SECONDS); long end2 = System.currentTimeMillis(); System.out.println( "Time Taken till termination: " + Double.valueOf((end2 - start) / 1000) + " seconds"); } catch (InterruptedException e) { System.err.println("ThreadPoolExecutor was interrupted"); e.printStackTrace(); } }
@Test @Ignore("NXP-20582: timeout waiting termination") public void testConcurrency() throws Exception { final String seqName = "mt"; int nbCalls = 5000; final UIDSequencer seq = uidGeneratorService.getSequencer(); ThreadPoolExecutor tpe = new ThreadPoolExecutor( 5, 5, 500L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>(nbCalls + 1)); for (int i = 0; i < nbCalls; i++) { tpe.submit( new Runnable() { @Override public void run() { seq.getNext(seqName); } }); } tpe.shutdown(); boolean finish = tpe.awaitTermination(20, TimeUnit.SECONDS); assertTrue("timeout", finish); assertEquals(nbCalls + 1, seq.getNext(seqName)); }
public void close() { _executorService.shutdown(); try { _executorService.awaitTermination(60, TimeUnit.SECONDS); } catch (final InterruptedException e) { log.warn("waiting for termination of executor service interrupted", e); } }
public void shutDown() { threadPool.shutdownNow(); try { threadPool.awaitTermination(10, TimeUnit.SECONDS); } catch (InterruptedException e) { throw new RuntimeException("Workers-pool termination failed", e); } }
public synchronized void exit() { mStopped = true; try { mExecutor.shutdown(); mExecutor.awaitTermination(30, TimeUnit.SECONDS); mExecutor.shutdownNow(); } catch (Exception ignored) { } }
public void shutdown() { _shutdown = true; try { _effectsScheduledThreadPool.awaitTermination(1, TimeUnit.SECONDS); _generalScheduledThreadPool.awaitTermination(1, TimeUnit.SECONDS); _generalPacketsThreadPool.awaitTermination(1, TimeUnit.SECONDS); _ioPacketsThreadPool.awaitTermination(1, TimeUnit.SECONDS); _generalThreadPool.awaitTermination(1, TimeUnit.SECONDS); _effectsScheduledThreadPool.shutdown(); _generalScheduledThreadPool.shutdown(); _generalPacketsThreadPool.shutdown(); _ioPacketsThreadPool.shutdown(); _generalThreadPool.shutdown(); _log.info("All ThreadPools are now stopped"); } catch (InterruptedException e) { _log.log(Level.WARNING, "", e); } }
public void destroy() { queryExecutor.shutdown(); // wait for all submitted task to finish. // Block until shutdown() complete, or the timeout occurs, or // the current thread is interrupted, whichever happens first. try { queryExecutor.awaitTermination(2L, TimeUnit.MINUTES); } catch (InterruptedException e) { throw new IllegalStateException(QUERY_POOL_NAME + " graceful shutdown interrupted.", e); } }
public void shutdown() { _shutdown = true; try { _effectsScheduledThreadPool.awaitTermination(1, TimeUnit.SECONDS); _generalScheduledThreadPool.awaitTermination(1, TimeUnit.SECONDS); _generalPacketsThreadPool.awaitTermination(1, TimeUnit.SECONDS); _ioPacketsThreadPool.awaitTermination(1, TimeUnit.SECONDS); _generalThreadPool.awaitTermination(1, TimeUnit.SECONDS); _aiThreadPool.awaitTermination(1, TimeUnit.SECONDS); _effectsScheduledThreadPool.shutdown(); _generalScheduledThreadPool.shutdown(); _generalPacketsThreadPool.shutdown(); _ioPacketsThreadPool.shutdown(); _generalThreadPool.shutdown(); _aiThreadPool.shutdown(); System.out.println("All ThreadPools are now stoped"); } catch (InterruptedException e) { _log.log(Level.SEVERE, e.getLocalizedMessage(), e); } }
public void abortConnection() throws SQLException { Connection connection = DriverManager.getConnection("jdbc:derby://localhost/java7book"); ThreadPoolExecutor executor = new DebugExecutorService(2, 10, 60, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>()); connection.abort(executor); executor.shutdown(); try { executor.awaitTermination(5, TimeUnit.MINUTES); System.out.println(executor.getCompletedTaskCount()); } catch (InterruptedException e) { e.printStackTrace(); } }
private void waitFor(ThreadPoolExecutor t, String name) { boolean done = false; while (!done) { try { done = t.awaitTermination(60, TimeUnit.SECONDS); LOG.info("Waiting for " + name + " to finish..."); if (!done) { t.shutdownNow(); } } catch (InterruptedException ie) { LOG.warn("Interrupted waiting for " + name + " to finish..."); } } }
/** * Stops the timer, cancelling all tasks * * @throws InterruptedException if interrupted while waiting for thread to return */ public void stop() { stopRunner(); List<Runnable> remaining_tasks = pool.shutdownNow(); for (Runnable task : remaining_tasks) { if (task instanceof Future) { Future future = (Future) task; future.cancel(true); } } pool.getQueue().clear(); try { pool.awaitTermination(Global.THREADPOOL_SHUTDOWN_WAIT_TIME, TimeUnit.MILLISECONDS); } catch (InterruptedException e) { } }
/** 主函数 */ public static void main(String[] args) { final ThreadPoolExecutor pool = ApplyThreadPool.getThreadPoolExector(); Runtime.getRuntime() .addShutdownHook( new Thread( new Runnable() { @Override public void run() { pool.shutdown(); } })); /** 从150机器上获取皮皮用户uid和token */ logger.info("从150机器上获取皮皮用户uid: "); TokenService tokenService = new TokenService(); /** 连接48数据库 */ logger.info("连接48数据库: "); WeiboJDBC weiboJDBC = new WeiboJDBC("192.168.1.48", "pp_fenxi", "q#tmuYzC@sqB6!ok@sHd", "pp_fenxi"); // 创建粉丝分析结果数据表 try { weiboJDBC.createFansAnalysisTable(FansAnalysisInfoUtils.PP_SINA_FANS_ANALYSIS); } catch (SQLException e) { throw new RuntimeException(e); } weiboJDBC.dbClose(); /** 循环计算 */ logger.info("循环计算: "); for (Long uid : tokenService.getSinaUids()) { if (!pool.isShutdown()) { pool.execute(new FansAnalysisRun(uid)); } } pool.shutdown(); try { pool.awaitTermination(100, TimeUnit.SECONDS); } catch (InterruptedException e) { e.printStackTrace(); } }
@Test public void testShutdownWithReferencesDoesNotStopExecutor() throws Exception { Map<String, Object> cache = new HashMap<String, Object>(); ThreadPoolBuilder builder = new ThreadPoolBuilder(name.getTableNameString(), new Configuration(false)); ThreadPoolExecutor exec = ThreadPoolManager.getExecutor(builder, cache); assertNotNull("Got a null exector from the pool!", exec); ThreadPoolExecutor exec2 = ThreadPoolManager.getExecutor(builder, cache); assertTrue("Should have gotten the same executor", exec2 == exec); exec.shutdown(); assertFalse( "Executor is shutting down, even though we have a live reference!", exec.isShutdown() || exec.isTerminating()); exec2.shutdown(); // wait 5 minutes for thread pool to shutdown assertTrue( "Executor is NOT shutting down, after releasing live reference!", exec.awaitTermination(300, TimeUnit.SECONDS)); }
private void assertConcurrentUpdates(Function<Integer, String> keyGenerator) throws Exception { final AtomicInteger safeIndex = new AtomicInteger(-1); List<String> keys = Lists.newArrayListWithCapacity(ATTEMPTS); ThreadPoolExecutor executor = new ThreadPoolExecutor( 3, 3, 5, TimeUnit.SECONDS, new ArrayBlockingQueue<Runnable>(ATTEMPTS)); synchronized (indexer) { for (int i = 0; i < ATTEMPTS; i++) { final String key = keyGenerator.apply(i); keys.add(key); executor.execute( new Runnable() { @Override public void run() { int index = indexer.getOrCreateIndex(key); if (safeIndex.get() < index) { safeIndex.set(index); } indexer.addString(key); } }); } } try { while (!executor.getQueue().isEmpty()) { // Validate that we can execute concurrent queries too. if (safeIndex.get() >= 0) { int index = safeIndex.get(); // Retrieve string using random existing index and validate reverse mapping. String key = indexer.getStringForIndex(index); assertNotNull(key); assertEquals(index, indexer.getIndex(key)); } } } finally { executor.shutdown(); executor.awaitTermination(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS); } for (String key : keys) { // Validate mapping between keys and indices. assertEquals(key, indexer.getStringForIndex(indexer.getIndex(key))); } }
@Override public synchronized void stop() { super.stop(); shutdown = true; if (backgroundOperationPool != null) { backgroundOperationPool.shutdown(); long timeout = hiveConf.getTimeVar(ConfVars.HIVE_SERVER2_ASYNC_EXEC_SHUTDOWN_TIMEOUT, TimeUnit.SECONDS); try { backgroundOperationPool.awaitTermination(timeout, TimeUnit.SECONDS); } catch (InterruptedException e) { LOG.warn( "HIVE_SERVER2_ASYNC_EXEC_SHUTDOWN_TIMEOUT = " + timeout + " seconds has been exceeded. RUNNING background operations will be shut down", e); } backgroundOperationPool = null; } cleanupLoggingRootDir(); }
public void processMTRoot(String path) { long t0 = System.currentTimeMillis(); DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss"); Date date = new Date(); try { ts.addIndexPath(new File(path).getCanonicalPath()); System.out.println("MediaIndexer.processMTRoot()" + path); System.out.println("MediaIndexer.processMTRoot() started at time " + dateFormat.format(date)); System.out.println("MediaIndexer.processMTRoot() computing number of files..."); totalNumberOfFiles = this.countFiles(path); lastProgressTime = System.currentTimeMillis(); System.out.println("Number of files to explore " + totalNumberOfFiles); if (executorService.isShutdown()) { executorService = new ThreadPoolExecutor( maxThreads, maxThreads, 0L, TimeUnit.MILLISECONDS, new LimitedQueue<Runnable>(50)); } this.processedFiles = 0; // this.processMT(new File(path)); TreeWalker t = new TreeWalker(this); t.walk(path); } catch (IOException e) { e.printStackTrace(); } executorService.shutdown(); try { executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS); } catch (InterruptedException e) { e.printStackTrace(); } long t1 = System.currentTimeMillis(); date = new Date(); System.out.println("MediaIndexer.processMTRoot() finished at time " + dateFormat.format(date)); System.out.println("MediaIndexer.processMTRoot() found " + newFiles + " new files"); System.out.println("MediaIndexer.processMTRoot() updated " + updatedFiles + " files"); System.out.println("MediaIndexer.processMTRoot() total " + ts.size() + " files"); System.out.println("MediaIndexer.processMTRoot took " + (t1 - t0) / 1000 + " s"); }
public void testRehash() throws IOException, InterruptedException { EmbeddedCacheManager cacheManager = buildCacheManager(); cacheManagers.addLast(cacheManager); cacheManager.getCache("serviceGroup"); new AddNodeTask().run(); new AddNodeTask().run(); new AddNodeTask().run(); Thread.sleep(3000); log.info("Start testing"); ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(TEST_THREADS); executor.prestartAllCoreThreads(); for (int i = 0; i < TEST_LOOPS; i++) { executor.submit(new SimulateTask()); } for (int i = 0; i < 10; i++) { try { Thread.sleep(3000); if (i != 1) { new AddNodeTask().run(); // 2 } else { new RemoveNodeTask().run(); } } catch (RuntimeException e) { log.warn("Error during add/remove node", e); } } log.info("Rehash phase is completed..."); executor.shutdown(); executor.awaitTermination(1, TimeUnit.DAYS); }
public void loadSavedMetrics() { List<RetentionLevel> accessLevels = environment.retentions().getAllAccessLevels(); int numberOfAccessLevels = accessLevels.size(); log.info("trying to load Metrics for " + numberOfAccessLevels + " accessLevels"); List<Future<?>> futures = new ArrayList<>(); try { for (RetentionLevel rlevel : accessLevels) { futures.addAll(loadSavedMetricsToCaches(rlevel.name())); } } catch (Exception e) { loadMetricsThreadPool.shutdownNow(); try { loadMetricsThreadPool.awaitTermination(Long.MAX_VALUE, TimeUnit.DAYS); } catch (InterruptedException e1) { log.error("Waiting for loadMetrics thread pool shutdown was interrupted", e1); } for (LevelCache cache : caches.values()) { cache.shutdown(); } throw e; } log.info("Submitted a total of {} futures to pre-load metrics", futures.size()); int logEvery = Math.max(1, futures.size() / 10); int futuresDone = 0; for (Future<?> f : futures) { try { f.get(); } catch (InterruptedException | ExecutionException e) { log.warn("Exception while waiting for load metrics threads", e); } futuresDone++; if (futuresDone % logEvery == 0) { log.info("Preloaded {} of {} metrics", futuresDone, futures.size()); } } log.info("all metrics preloaded!"); }
@Override public synchronized void close() throws SecurityException { if (pool.isShutdown()) { return; } try { // first, anything in the retry queue should be tried one last time and then we give up on it allowRetry = false; for (LogglySample sample : retryQueue) { pool.submit(sample); } retryQueue.clear(); System.out.println( "Shutting down Loggly handler - waiting 90 seconds for " + pool.getQueue().size() + " logs to finish"); pool.shutdown(); try { boolean result = pool.awaitTermination(90, TimeUnit.SECONDS); if (!result) { System.out.println( "Not all Loggly messages sent out - still had " + pool.getQueue().size() + " left :("); pool.shutdownNow(); } } catch (InterruptedException e) { // ignore } } finally { httpClient.getConnectionManager().shutdown(); System.out.println("Loggly handler shut down"); } }
/** * if no surveyId is passed in, this will check for new surveys and, if there are some new ones, * downloads them to the DATA_DIR. If a surveyId is passed in, then that specific survey will be * downloaded. If it's already on the device, the survey will be replaced with the new one. */ private void checkAndDownload(String surveyId) { if (isAbleToRun()) { try { lock.acquire(); databaseAdaptor = new SurveyDbAdapter(this); databaseAdaptor.open(); int precacheOption = Integer.parseInt(databaseAdaptor.findPreference(ConstantUtil.PRECACHE_SETTING_KEY)); String serverBase = databaseAdaptor.findPreference(ConstantUtil.SERVER_SETTING_KEY); if (serverBase != null && serverBase.trim().length() > 0) { serverBase = getResources().getStringArray(R.array.servers)[Integer.parseInt(serverBase)]; } else { serverBase = props.getProperty(ConstantUtil.SERVER_BASE); } int surveyCheckOption = Integer.parseInt(databaseAdaptor.findPreference(ConstantUtil.CHECK_FOR_SURVEYS)); String deviceId = databaseAdaptor.findPreference(ConstantUtil.DEVICE_IDENT_KEY); ArrayList<Survey> surveys = null; if (surveyId != null && surveyId.trim().length() > 0) { surveys = getSurveyHeader(serverBase, surveyId, deviceId); if (surveys != null && surveys.size() > 0) { // if we already have the survey, delete it first databaseAdaptor.deleteSurvey(surveyId.trim(), true); } } else { if (canDownload(surveyCheckOption)) { surveys = checkForSurveys(serverBase, deviceId); } } if (surveys != null && surveys.size() > 0) { // if there are surveys for this device, see if we need // them surveys = databaseAdaptor.checkSurveyVersions(surveys); int updateCount = 0; if (surveys != null && surveys.size() > 0) { for (int i = 0; i < surveys.size(); i++) { Survey survey = surveys.get(i); try { if (downloadSurvey(serverBase, survey)) { databaseAdaptor.saveSurvey(survey); downloadHelp(survey, precacheOption); updateCount++; } } catch (Exception e) { Log.e(TAG, "Could not download survey", e); PersistentUncaughtExceptionHandler.recordException(e); } } if (updateCount > 0) { fireNotification(updateCount); } } } // now check if any previously downloaded surveys still need // don't have their help media pre-cached if (canDownload(precacheOption)) { surveys = databaseAdaptor.listSurveys(null); if (surveys != null) { for (int i = 0; i < surveys.size(); i++) { if (!surveys.get(i).isHelpDownloaded()) { downloadHelp(surveys.get(i), precacheOption); } } } } } catch (Exception e) { Log.e(TAG, "Could not update surveys", e); PersistentUncaughtExceptionHandler.recordException(e); } finally { databaseAdaptor.close(); lock.release(); } } try { downloadExecutor.shutdown(); // wait up to 30 minutes to download the media downloadExecutor.awaitTermination(1800, TimeUnit.SECONDS); } catch (InterruptedException e) { Log.e(TAG, "Error while waiting for download executor to terminate", e); } stopSelf(); }
private void start() { BimServerConfig config = new BimServerConfig(); Path homeDir = Paths.get("home"); try { if (Files.isDirectory(homeDir)) { PathUtils.removeDirectoryWithContent(homeDir); } } catch (IOException e) { e.printStackTrace(); } config.setClassPath(System.getProperty("java.class.path")); config.setHomeDir(homeDir); config.setPort(8080); config.setStartEmbeddedWebServer(true); config.setResourceFetcher(new LocalDevelopmentResourceFetcher(Paths.get("../"))); final BimServer bimServer = new BimServer(config); try { LocalDevPluginLoader.loadPlugins(bimServer.getPluginManager(), null); bimServer.start(); if (bimServer.getServerInfo().getServerState() == ServerState.NOT_SETUP) { bimServer .getService(AdminInterface.class) .setup( "http://localhost", "localhost", "*****@*****.**", "Administrator", "*****@*****.**", "admin"); } } catch (PluginException e2) { e2.printStackTrace(); } catch (ServerException e) { e.printStackTrace(); } catch (DatabaseInitException e) { e.printStackTrace(); } catch (BimserverDatabaseException e) { e.printStackTrace(); } catch (DatabaseRestartRequiredException e) { e.printStackTrace(); } catch (UserException e) { e.printStackTrace(); } try { final ServiceMap serviceMap = bimServer.getServiceFactory().get(AccessMethod.INTERNAL); ServiceInterface serviceInterface = serviceMap.get(ServiceInterface.class); SettingsInterface settingsInterface = serviceMap.get(SettingsInterface.class); final Bimsie1AuthInterface authInterface = serviceMap.get(Bimsie1AuthInterface.class); serviceInterface = bimServer .getServiceFactory() .get(authInterface.login("*****@*****.**", "admin"), AccessMethod.INTERNAL) .get(ServiceInterface.class); settingsInterface.setCacheOutputFiles(true); settingsInterface.setGenerateGeometryOnCheckin(false); final SProject project = serviceMap.getBimsie1ServiceInterface().addProject("test", "ifc2x3tc1"); SDeserializerPluginConfiguration deserializerByName = serviceMap.getBimsie1ServiceInterface().getDeserializerByName("IfcStepDeserializer"); Path file = Paths.get("../TestData/data/AC11-Institute-Var-2-IFC.ifc"); serviceInterface.checkin( project.getOid(), "test", deserializerByName.getOid(), file.toFile().length(), file.getFileName().toString(), new DataHandler(new FileDataSource(file.toFile())), false, true); final SProject projectUpdate = serviceMap.getBimsie1ServiceInterface().getProjectByPoid(project.getOid()); ThreadPoolExecutor executor = new ThreadPoolExecutor(20, 20, 1, TimeUnit.HOURS, new ArrayBlockingQueue<Runnable>(1000)); for (int i = 0; i < 20; i++) { executor.execute( new Runnable() { @Override public void run() { try { ServiceMap serviceMap2 = bimServer .getServiceFactory() .get( authInterface.login("*****@*****.**", "admin"), AccessMethod.INTERNAL); SSerializerPluginConfiguration serializerPluginConfiguration = serviceMap.getBimsie1ServiceInterface().getSerializerByName("Ifc2x3"); Long download = serviceMap2 .getBimsie1ServiceInterface() .download( projectUpdate.getLastRevisionId(), serializerPluginConfiguration.getOid(), true, true); SDownloadResult downloadData = serviceMap2.getBimsie1ServiceInterface().getDownloadData(download); if (downloadData.getFile().getDataSource() instanceof CacheStoringEmfSerializerDataSource) { CacheStoringEmfSerializerDataSource c = (CacheStoringEmfSerializerDataSource) downloadData.getFile().getDataSource(); try { ByteArrayOutputStream baos = new ByteArrayOutputStream(); c.writeToOutputStream(baos, null); System.out.println(baos.size()); } catch (SerializerException e) { e.printStackTrace(); } } else { ByteArrayOutputStream baos = new ByteArrayOutputStream(); IOUtils.copy(downloadData.getFile().getInputStream(), baos); System.out.println(baos.size()); } serviceMap2.getServiceInterface().cleanupLongAction(download); } catch (ServerException e) { e.printStackTrace(); } catch (UserException e) { e.printStackTrace(); } catch (FileNotFoundException e) { e.printStackTrace(); } catch (IOException e) { e.printStackTrace(); } catch (PublicInterfaceNotFoundException e1) { e1.printStackTrace(); } } }); } executor.shutdown(); executor.awaitTermination(1, TimeUnit.HOURS); bimServer.stop(); } catch (ServerException e1) { e1.printStackTrace(); } catch (UserException e1) { e1.printStackTrace(); } catch (InterruptedException e) { e.printStackTrace(); } }
/** Start scheduling jobs for processing. */ public void terminate() throws Exception { if (controller.getException() != null) { throw controller.getException(); } service.awaitTermination(0, TimeUnit.MILLISECONDS); }
@Override public void stop() { if (sharedHttpServers != null) { for (HttpServer server : sharedHttpServers.values()) { server.close(); } sharedHttpServers = null; } if (sharedNetServers != null) { for (NetServer server : sharedNetServers.values()) { server.close(); } sharedNetServers = null; } if (timer != null) { timer.stop(); timer = null; } if (eventBus != null) { eventBus.close(null); } if (backgroundPool != null) { backgroundPool.shutdown(); } if (acceptorPool != null) { acceptorPool.shutdown(); } try { if (backgroundPool != null) { backgroundPool.awaitTermination(20, TimeUnit.SECONDS); backgroundPool = null; } } catch (InterruptedException ex) { // ignore } try { if (acceptorPool != null) { acceptorPool.awaitTermination(20, TimeUnit.SECONDS); acceptorPool = null; } } catch (InterruptedException ex) { // ignore } // log.info("Release external resources from worker pool"); if (corePool != null) { corePool.releaseExternalResources(); corePool = null; } // log.info("Release external resources: done"); setContext(null); }
@Override public synchronized long claimRecords(SDFSEvent evt, LargeBloomFilter bf) throws IOException { if (this.isClosed()) throw new IOException("Hashtable " + this.fileName + " is close"); executor = new ThreadPoolExecutor( Main.writeThreads + 1, Main.writeThreads + 1, 10, TimeUnit.SECONDS, worksQueue, new ProcessPriorityThreadFactory(Thread.MIN_PRIORITY), executionHandler); csz = new AtomicLong(0); Lock l = this.gcLock.writeLock(); l.lock(); this.runningGC = true; lbf = null; lbf = new LargeBloomFilter(maxSz, .01); l.unlock(); try { SDFSLogger.getLog() .info("Claiming Records [" + this.getSize() + "] from [" + this.fileName + "]"); SDFSEvent tEvt = SDFSEvent.claimInfoEvent( "Claiming Records [" + this.getSize() + "] from [" + this.fileName + "]", evt); tEvt.maxCt = this.maps.size(); Iterator<ProgressiveFileByteArrayLongMap> iter = maps.iterator(); while (iter.hasNext()) { tEvt.curCt++; ProgressiveFileByteArrayLongMap m = null; try { m = iter.next(); executor.execute(new ClaimShard(m, bf, lbf, csz)); } catch (Exception e) { tEvt.endEvent( "Unable to claim records for " + m + " because : [" + e.toString() + "]", SDFSEvent.ERROR); SDFSLogger.getLog().error("Unable to claim records for " + m, e); throw new IOException(e); } } executor.shutdown(); try { while (!executor.awaitTermination(10, TimeUnit.SECONDS)) { SDFSLogger.getLog().debug("Awaiting fdisk completion of threads."); } } catch (InterruptedException e) { throw new IOException(e); } this.kSz.getAndAdd(-1 * csz.get()); tEvt.endEvent("removed [" + csz.get() + "] records"); SDFSLogger.getLog().info("removed [" + csz.get() + "] records"); iter = maps.iterator(); while (iter.hasNext()) { ProgressiveFileByteArrayLongMap m = null; try { m = iter.next(); if (m.isFull() && !m.isActive()) { double pf = (double) m.size() / (double) m.maxSize(); // SDFSLogger.getLog().info("pfull=" + pf); if (pf < .4 || pf == Double.NaN) { // SDFSLogger.getLog().info("deleting " + // m.toString()); m.iterInit(); KVPair p = m.nextKeyValue(); while (p != null) { ProgressiveFileByteArrayLongMap _m = this.getWriteMap(); try { _m.put(p.key, p.value); } catch (HashtableFullException e) { _m.setActive(false); _m = this.createWriteMap(); _m.put(p.key, p.value); } finally { this.activeWriteMaps.offer(_m); } p = m.nextKeyValue(); } int mapsz = maps.size(); maps.remove(m); mapsz = mapsz - maps.size(); // SDFSLogger.getLog().info( // "removing map " + m.toString() + " sz=" // + maps.size() + " rm=" + mapsz); m.vanish(); m = null; } } } catch (Exception e) { tEvt.endEvent( "Unable to compact " + m + " because : [" + e.toString() + "]", SDFSEvent.ERROR); SDFSLogger.getLog().error("to compact " + m, e); throw new IOException(e); } } return csz.get(); } finally { l.lock(); this.runningGC = false; l.unlock(); executor = null; } }
protected void doStop() throws Exception { executor.shutdown(); if (!executor.awaitTermination(60, TimeUnit.SECONDS)) executor.shutdownNow(); }
public void join() throws InterruptedException { executor.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS); }
/** * initializes the Object set of this hash table. * * @param initialCapacity an <code>int</code> value * @return an <code>int</code> value * @throws HashtableFullException * @throws FileNotFoundException */ public long setUp() throws Exception { File _fs = new File(fileName); if (!_fs.getParentFile().exists()) { _fs.getParentFile().mkdirs(); } SDFSLogger.getLog().info("Folder = " + _fs.getPath()); SDFSLogger.getLog().info("Loading freebits bitset"); long rsz = 0; long _tbs = maxSz / (256); int max = Integer.MAX_VALUE / ProgressiveFileByteArrayLongMap.EL; if (_tbs > max) { this.hashTblSz = max; } else if (_tbs > this.hashTblSz) { this.hashTblSz = (int) _tbs; } SDFSLogger.getLog() .info( "table setup max=" + max + " maxsz=" + this.maxSz + " _tbs=" + _tbs + " hashTblSz=" + this.hashTblSz); this.hashTblSz = NextPrime.getNextPrimeI((int) (this.hashTblSz)); File[] files = _fs.getParentFile().listFiles(new DBFileFilter()); if (files.length > 0) { CommandLineProgressBar bar = new CommandLineProgressBar("Loading Existing Hash Tables", files.length, System.out); this.loadEvent.maxCt = files.length + 128; for (int i = 0; i < files.length; i++) { this.loadEvent.curCt = this.loadEvent.curCt + 1; int sz = NextPrime.getNextPrimeI((int) (this.hashTblSz)); // SDFSLogger.getLog().debug("will create byte array of size " // + sz + " propsize was " + propsize); ProgressiveFileByteArrayLongMap m = null; String pth = files[i].getPath(); String pfx = pth.substring(0, pth.length() - 5); m = new ProgressiveFileByteArrayLongMap(pfx, sz); long mep = m.setUp(); if (mep > endPos) endPos = mep; maps.add(m); rsz = rsz + m.size(); bar.update(i); if (!m.isFull() && this.activeWriteMaps.remainingCapacity() > 0) { m.setActive(true); this.activeWriteMaps.add(m); this.loadCacheExecutor.execute(m); } else { m.setActive(false); m.full = true; } } bar.finish(); } this.loadEvent.shortMsg = "Loading BloomFilters"; if (maps.size() == 0) lbf = new LargeBloomFilter(maxSz, .01); else { try { lbf = new LargeBloomFilter(_fs.getParentFile(), maxSz, true); } catch (Exception e) { SDFSLogger.getLog().warn("Recreating BloomFilters..."); this.loadEvent.shortMsg = "Recreating BloomFilters"; lbf = new LargeBloomFilter(maxSz, .01); executor = new ThreadPoolExecutor( Main.writeThreads, Main.writeThreads, 10, TimeUnit.SECONDS, worksQueue, new ProcessPriorityThreadFactory(Thread.MIN_PRIORITY), executionHandler); CommandLineProgressBar bar = new CommandLineProgressBar("ReCreating BloomFilters", maps.size(), System.out); Iterator<ProgressiveFileByteArrayLongMap> iter = maps.iterator(); int i = 0; ArrayList<LBFReconstructThread> al = new ArrayList<LBFReconstructThread>(); while (iter.hasNext()) { ProgressiveFileByteArrayLongMap m = iter.next(); LBFReconstructThread th = new LBFReconstructThread(lbf, m); executor.execute(th); al.add(th); i++; bar.update(i); } executor.shutdown(); bar.finish(); try { System.out.print("Waiting for all BloomFilters creation threads to finish"); while (!executor.awaitTermination(10, TimeUnit.SECONDS)) { SDFSLogger.getLog().debug("Awaiting fdisk completion of threads."); System.out.print("."); } for (LBFReconstructThread th : al) { if (th.ex != null) throw th.ex; } System.out.println(" done"); } catch (Exception e1) { throw new IOException(e1); } } } while (this.activeWriteMaps.remainingCapacity() > 0) { String guid = null; boolean written = false; while (!written) { guid = RandomGUID.getGuid(); File f = new File(fileName + "-" + guid + ".keys"); if (!f.exists()) { ProgressiveFileByteArrayLongMap activeWMap = new ProgressiveFileByteArrayLongMap(fileName + "-" + guid, this.hashTblSz); activeWMap.setUp(); this.maps.add(activeWMap); written = true; activeWMap.setActive(true); this.activeWriteMaps.offer(activeWMap); } } } if (SDFSLogger.isDebug()) { long mem = MemoryMeasurer.measureBytes(lbf); long mmem = MemoryMeasurer.measureBytes(maps); SDFSLogger.getLog().debug("Large BloomFilter Size=" + StorageUnit.of(mem).format(mem)); SDFSLogger.getLog().debug("Maps Size=" + StorageUnit.of(mmem).format(mmem)); } this.loadEvent.endEvent("Loaded entries " + rsz); System.out.println("Loaded entries " + rsz); SDFSLogger.getLog().info("Active Maps " + this.activeWriteMaps.size()); SDFSLogger.getLog().info("Loaded entries " + rsz); SDFSLogger.getLog().info("Loading BloomFilters " + rsz); this.kSz.set(rsz); this.closed = false; return size; }