@GwtIncompatible("SerializableTester") public void testNewSetFromMapSerialization() { Set<Integer> set = Sets.newSetFromMap(new LinkedHashMap<Integer, Boolean>()); set.addAll(SOME_COLLECTION); Set<Integer> copy = SerializableTester.reserializeAndAssert(set); assertThat(copy).containsExactly(0, 1).inOrder(); }
public void testNewSetFromMapIllegal() { Map<Integer, Boolean> map = new LinkedHashMap<Integer, Boolean>(); map.put(2, true); try { Sets.newSetFromMap(map); fail(); } catch (IllegalArgumentException expected) { } }
/** * Creates a new threadsafe BeanDeployerEnvironment instance. These instances are used by {@link * ConcurrentBeanDeployer} during bootstrap. */ public static BeanDeployerEnvironment newConcurrentEnvironment( EjbDescriptors ejbDescriptors, BeanManagerImpl manager) { return new BeanDeployerEnvironment( Sets.newSetFromMap(new ConcurrentHashMap<SlimAnnotatedTypeContext<?>, Boolean>()), Sets.newSetFromMap(new ConcurrentHashMap<Class<?>, Boolean>()), Multimaps.<Class<?>, AbstractClassBean<?>>newConcurrentSetMultimap(), Sets.newSetFromMap(new ConcurrentHashMap<ProducerField<?, ?>, Boolean>()), Multimaps.<WeldMethodKey, ProducerMethod<?, ?>>newConcurrentSetMultimap(), Sets.newSetFromMap(new ConcurrentHashMap<RIBean<?>, Boolean>()), Sets.newSetFromMap(new ConcurrentHashMap<ObserverInitializationContext<?, ?>, Boolean>()), Sets.newSetFromMap(new ConcurrentHashMap<DisposalMethod<?, ?>, Boolean>()), Sets.newSetFromMap(new ConcurrentHashMap<DisposalMethod<?, ?>, Boolean>()), Sets.newSetFromMap(new ConcurrentHashMap<DecoratorImpl<?>, Boolean>()), Sets.newSetFromMap(new ConcurrentHashMap<InterceptorImpl<?>, Boolean>()), ejbDescriptors, Sets.newSetFromMap(new ConcurrentHashMap<EnhancedAnnotatedType<?>, Boolean>()), new ConcurrentHashMap<InternalEjbDescriptor<?>, EnhancedAnnotatedType<?>>(), manager); }
public void handleStreamEvent(StreamEvent event) { if (event.eventType == StreamEvent.Type.STREAM_PREPARED) { SessionInfo session = ((StreamEvent.SessionPreparedEvent) event).session; sessionsByHost.put(session.peer, session); } else if (event.eventType == StreamEvent.Type.FILE_PROGRESS) { ProgressInfo progressInfo = ((StreamEvent.ProgressEvent) event).progress; // update progress Set<ProgressInfo> progresses = progressByHost.get(progressInfo.peer); if (progresses == null) { progresses = Sets.newSetFromMap(new ConcurrentHashMap<ProgressInfo, Boolean>()); progressByHost.put(progressInfo.peer, progresses); } if (progresses.contains(progressInfo)) progresses.remove(progressInfo); progresses.add(progressInfo); StringBuilder sb = new StringBuilder(); sb.append("\rprogress: "); long totalProgress = 0; long totalSize = 0; for (Map.Entry<InetAddress, Set<ProgressInfo>> entry : progressByHost.entrySet()) { SessionInfo session = sessionsByHost.get(entry.getKey()); long size = session.getTotalSizeToSend(); long current = 0; int completed = 0; for (ProgressInfo progress : entry.getValue()) { if (progress.currentBytes == progress.totalBytes) completed++; current += progress.currentBytes; } totalProgress += current; totalSize += size; sb.append("[").append(entry.getKey()); sb.append(" ").append(completed).append("/").append(session.getTotalFilesToSend()); sb.append(" (").append(size == 0 ? 100L : current * 100L / size).append("%)] "); } long time = System.nanoTime(); long deltaTime = Math.max(1L, TimeUnit.NANOSECONDS.toMillis(time - lastTime)); lastTime = time; long deltaProgress = totalProgress - lastProgress; lastProgress = totalProgress; sb.append("[total: ") .append(totalSize == 0 ? 100L : totalProgress * 100L / totalSize) .append("% - "); sb.append(mbPerSec(deltaProgress, deltaTime)).append("MB/s"); sb.append(" (avg: ") .append(mbPerSec(totalProgress, TimeUnit.NANOSECONDS.toMillis(time - start))) .append("MB/s)]"); System.out.print(sb.toString()); } }
/** A mechanism to register and query service instances using ZooKeeper */ @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter") public class ServiceDiscoveryImpl<T> implements ServiceDiscovery<T> { private final Logger log = LoggerFactory.getLogger(getClass()); private final CuratorFramework client; private final String basePath; private final InstanceSerializer<T> serializer; private final ConcurrentMap<String, Entry<T>> services = Maps.newConcurrentMap(); private final Collection<ServiceCache<T>> caches = Sets.newSetFromMap(Maps.<ServiceCache<T>, Boolean>newConcurrentMap()); private final Collection<ServiceProvider<T>> providers = Sets.newSetFromMap(Maps.<ServiceProvider<T>, Boolean>newConcurrentMap()); private final boolean watchInstances; private final ConnectionStateListener connectionStateListener = new ConnectionStateListener() { @Override public void stateChanged(CuratorFramework client, ConnectionState newState) { if ((newState == ConnectionState.RECONNECTED) || (newState == ConnectionState.CONNECTED)) { try { log.debug("Re-registering due to reconnection"); reRegisterServices(); } catch (Exception e) { log.error("Could not re-register instances after reconnection", e); } } } }; private static class Entry<T> { private volatile ServiceInstance<T> service; private volatile NodeCache cache; private Entry(ServiceInstance<T> service) { this.service = service; } } /** * @param client the client * @param basePath base path to store data * @param serializer serializer for instances (e.g. {@link JsonInstanceSerializer}) * @param thisInstance instance that represents the service that is running. The instance will get * auto-registered * @param watchInstances if true, watches for changes to locally registered instances */ public ServiceDiscoveryImpl( CuratorFramework client, String basePath, InstanceSerializer<T> serializer, ServiceInstance<T> thisInstance, boolean watchInstances) { this.watchInstances = watchInstances; this.client = Preconditions.checkNotNull(client, "client cannot be null"); this.basePath = Preconditions.checkNotNull(basePath, "basePath cannot be null"); this.serializer = Preconditions.checkNotNull(serializer, "serializer cannot be null"); if (thisInstance != null) { Entry<T> entry = new Entry<T>(thisInstance); entry.cache = makeNodeCache(thisInstance); services.put(thisInstance.getId(), entry); } } /** * The discovery must be started before use * * @throws Exception errors */ @Override public void start() throws Exception { try { reRegisterServices(); } catch (KeeperException e) { log.error("Could not register instances - will try again later", e); } client.getConnectionStateListenable().addListener(connectionStateListener); } @Override public void close() throws IOException { for (ServiceCache<T> cache : Lists.newArrayList(caches)) { CloseableUtils.closeQuietly(cache); } for (ServiceProvider<T> provider : Lists.newArrayList(providers)) { CloseableUtils.closeQuietly(provider); } for (Entry<T> entry : services.values()) { try { internalUnregisterService(entry); } catch (KeeperException.NoNodeException ignore) { // ignore } catch (Exception e) { log.error("Could not unregister instance: " + entry.service.getName(), e); } } client.getConnectionStateListenable().removeListener(connectionStateListener); } /** * Register/re-register/update a service instance * * @param service service to add * @throws Exception errors */ @Override public void registerService(ServiceInstance<T> service) throws Exception { Entry<T> newEntry = new Entry<T>(service); Entry<T> oldEntry = services.putIfAbsent(service.getId(), newEntry); Entry<T> useEntry = (oldEntry != null) ? oldEntry : newEntry; synchronized (useEntry) { if (useEntry == newEntry) // i.e. is new { useEntry.cache = makeNodeCache(service); } internalRegisterService(service); } } @Override public void updateService(final ServiceInstance<T> service) throws Exception { Entry<T> entry = services.get(service.getId()); if (entry == null) { throw new Exception("Service not registered: " + service); } synchronized (entry) { entry.service = service; byte[] bytes = serializer.serialize(service); String path = pathForInstance(service.getName(), service.getId()); client.setData().forPath(path, bytes); } } @VisibleForTesting protected void internalRegisterService(ServiceInstance<T> service) throws Exception { byte[] bytes = serializer.serialize(service); String path = pathForInstance(service.getName(), service.getId()); final int MAX_TRIES = 2; boolean isDone = false; for (int i = 0; !isDone && (i < MAX_TRIES); ++i) { try { CreateMode mode = (service.getServiceType() == ServiceType.DYNAMIC) ? CreateMode.EPHEMERAL : CreateMode.PERSISTENT; client.create().creatingParentsIfNeeded().withMode(mode).forPath(path, bytes); isDone = true; } catch (KeeperException.NodeExistsException e) { client.delete().forPath(path); // must delete then re-create so that watchers fire } } } /** * Unregister/remove a service instance * * @param service the service * @throws Exception errors */ @Override public void unregisterService(ServiceInstance<T> service) throws Exception { Entry<T> entry = services.remove(service.getId()); internalUnregisterService(entry); } /** * Allocate a new builder. {@link ServiceProviderBuilder#providerStrategy} is set to {@link * RoundRobinStrategy} * * @return the builder */ @Override public ServiceProviderBuilder<T> serviceProviderBuilder() { return new ServiceProviderBuilderImpl<T>(this) .providerStrategy(new RoundRobinStrategy<T>()) .threadFactory(ThreadUtils.newThreadFactory("ServiceProvider")); } /** * Allocate a new service cache builder. The refresh padding is defaulted to 1 second. * * @return new cache builder */ @Override public ServiceCacheBuilder<T> serviceCacheBuilder() { return new ServiceCacheBuilderImpl<T>(this) .threadFactory(ThreadUtils.newThreadFactory("ServiceCache")); } /** * Return the names of all known services * * @return list of service names * @throws Exception errors */ @Override public Collection<String> queryForNames() throws Exception { List<String> names = client.getChildren().forPath(basePath); return ImmutableList.copyOf(names); } /** * Return all known instances for the given service * * @param name name of the service * @return list of instances (or an empty list) * @throws Exception errors */ @Override public Collection<ServiceInstance<T>> queryForInstances(String name) throws Exception { return queryForInstances(name, null); } /** * Return a service instance POJO * * @param name name of the service * @param id ID of the instance * @return the instance or <code>null</code> if not found * @throws Exception errors */ @Override public ServiceInstance<T> queryForInstance(String name, String id) throws Exception { String path = pathForInstance(name, id); try { byte[] bytes = client.getData().forPath(path); return serializer.deserialize(bytes); } catch (KeeperException.NoNodeException ignore) { // ignore } return null; } void cacheOpened(ServiceCache<T> cache) { caches.add(cache); } void cacheClosed(ServiceCache<T> cache) { caches.remove(cache); } void providerOpened(ServiceProvider<T> provider) { providers.add(provider); } void providerClosed(ServiceProvider<T> cache) { providers.remove(cache); } CuratorFramework getClient() { return client; } String pathForName(String name) { return ZKPaths.makePath(basePath, name); } InstanceSerializer<T> getSerializer() { return serializer; } List<ServiceInstance<T>> queryForInstances(String name, Watcher watcher) throws Exception { ImmutableList.Builder<ServiceInstance<T>> builder = ImmutableList.builder(); String path = pathForName(name); List<String> instanceIds; if (watcher != null) { instanceIds = getChildrenWatched(path, watcher, true); } else { try { instanceIds = client.getChildren().forPath(path); } catch (KeeperException.NoNodeException e) { instanceIds = Lists.newArrayList(); } } for (String id : instanceIds) { ServiceInstance<T> instance = queryForInstance(name, id); if (instance != null) { builder.add(instance); } } return builder.build(); } @VisibleForTesting int debugServicesQty() { return services.size(); } private List<String> getChildrenWatched(String path, Watcher watcher, boolean recurse) throws Exception { List<String> instanceIds; try { instanceIds = client.getChildren().usingWatcher(watcher).forPath(path); } catch (KeeperException.NoNodeException e) { if (recurse) { try { client.create().creatingParentsIfNeeded().forPath(path); } catch (KeeperException.NodeExistsException ignore) { // ignore } instanceIds = getChildrenWatched(path, watcher, false); } else { throw e; } } return instanceIds; } @VisibleForTesting String pathForInstance(String name, String id) { return ZKPaths.makePath(pathForName(name), id); } @VisibleForTesting ServiceInstance<T> getRegisteredService(String id) { Entry<T> entry = services.get(id); return (entry != null) ? entry.service : null; } private void reRegisterServices() throws Exception { for (final Entry<T> entry : services.values()) { synchronized (entry) { internalRegisterService(entry.service); } } } private NodeCache makeNodeCache(final ServiceInstance<T> instance) { if (!watchInstances) { return null; } final NodeCache nodeCache = new NodeCache(client, pathForInstance(instance.getName(), instance.getId())); try { nodeCache.start(true); } catch (Exception e) { log.error("Could not start node cache for: " + instance, e); } NodeCacheListener listener = new NodeCacheListener() { @Override public void nodeChanged() throws Exception { if (nodeCache.getCurrentData() != null) { ServiceInstance<T> newInstance = serializer.deserialize(nodeCache.getCurrentData().getData()); Entry<T> entry = services.get(newInstance.getId()); if (entry != null) { synchronized (entry) { entry.service = newInstance; } } } else { log.warn("Instance data has been deleted for: " + instance); } } }; nodeCache.getListenable().addListener(listener); return nodeCache; } private void internalUnregisterService(final Entry<T> entry) throws Exception { if (entry != null) { synchronized (entry) { if (entry.cache != null) { CloseableUtils.closeQuietly(entry.cache); entry.cache = null; } String path = pathForInstance(entry.service.getName(), entry.service.getId()); try { client.delete().guaranteed().forPath(path); } catch (KeeperException.NoNodeException ignore) { // ignore } } } } }
/** * Provides the mechanism for updating and generating chunks. * * @author Benjamin Glatzel <*****@*****.**> */ public final class ChunkUpdateManager { public enum UpdateType { DEFAULT, PLAYER_TRIGGERED } /* CONST */ private static final int MAX_THREADS = CoreRegistry.get(Config.class).getSystem().getMaxThreads(); /* CHUNK UPDATES */ private final Set<ChunkImpl> currentlyProcessedChunks = Sets.newSetFromMap(new ConcurrentHashMap<ChunkImpl, Boolean>()); private final ChunkTessellator tessellator; private final WorldProvider worldProvider; public ChunkUpdateManager(ChunkTessellator tessellator, WorldProvider worldProvider) { this.tessellator = tessellator; this.worldProvider = worldProvider; } /** * Updates the given chunk using a new thread from the thread pool. If the maximum amount of chunk * updates is reached, the chunk update is ignored. Chunk updates can be forced though. * * @param chunk The chunk to update * @param type The chunk update type * @return True if a chunk update was executed */ // TODO: Review this system public boolean queueChunkUpdate(ChunkImpl chunk, final UpdateType type) { if (!currentlyProcessedChunks.contains(chunk) && (currentlyProcessedChunks.size() < MAX_THREADS || type != UpdateType.DEFAULT)) { executeChunkUpdate(chunk); return true; } return false; } private void executeChunkUpdate(final ChunkImpl c) { currentlyProcessedChunks.add(c); CoreRegistry.get(GameEngine.class) .submitTask("Chunk Update", new ChunkUpdater(c, tessellator, worldProvider, this)); } private void finishedProcessing(ChunkImpl c) { currentlyProcessedChunks.remove(c); } private static class ChunkUpdater implements Runnable { private ChunkImpl c; private ChunkTessellator tessellator; private WorldProvider worldProvider; private ChunkUpdateManager chunkUpdateManager; public ChunkUpdater( ChunkImpl chunk, ChunkTessellator tessellator, WorldProvider worldProvider, ChunkUpdateManager chunkUpdateManager) { this.chunkUpdateManager = chunkUpdateManager; this.c = chunk; this.tessellator = tessellator; this.worldProvider = worldProvider; } @Override public void run() { ChunkMesh[] newMeshes = new ChunkMesh[WorldRendererLwjgl.VERTICAL_SEGMENTS]; ChunkView chunkView = worldProvider.getLocalView(c.getPos()); if (chunkView != null) { c.setDirty(false); for (int seg = 0; seg < WorldRendererLwjgl.VERTICAL_SEGMENTS; seg++) { int meshHeight = ChunkConstants.SIZE_Y / WorldRendererLwjgl.VERTICAL_SEGMENTS; newMeshes[seg] = tessellator.generateMesh( chunkView, c.getPos(), meshHeight, seg * (ChunkConstants.SIZE_Y / WorldRendererLwjgl.VERTICAL_SEGMENTS)); } c.setPendingMesh(newMeshes); ChunkMonitor.fireChunkTessellated(c.getPos(), newMeshes); } chunkUpdateManager.finishedProcessing(c); // Clean these up because the task executor holds the object in memory. c = null; tessellator = null; worldProvider = null; } } }
@ThreadSafe public class TaskExecutor { private static final Logger log = Logger.get(TaskExecutor.class); // each task is guaranteed a minimum number of tasks private static final int GUARANTEED_SPLITS_PER_TASK = 3; // each time we run a split, run it for this length before returning to the pool private static final Duration SPLIT_RUN_QUANTA = new Duration(1, TimeUnit.SECONDS); private static final AtomicLong NEXT_RUNNER_ID = new AtomicLong(); private static final AtomicLong NEXT_WORKER_ID = new AtomicLong(); private final ExecutorService executor; private final ThreadPoolExecutorMBean executorMBean; private final int runnerThreads; private final int minimumNumberOfTasks; private final Ticker ticker; @GuardedBy("this") private final List<TaskHandle> tasks; private final Set<PrioritizedSplitRunner> allSplits = new HashSet<>(); private final PriorityBlockingQueue<PrioritizedSplitRunner> pendingSplits; private final Set<PrioritizedSplitRunner> runningSplits = Sets.newSetFromMap(new ConcurrentHashMap<PrioritizedSplitRunner, Boolean>()); private final Set<PrioritizedSplitRunner> blockedSplits = Sets.newSetFromMap(new ConcurrentHashMap<PrioritizedSplitRunner, Boolean>()); private final AtomicLongArray completedTasksPerLevel = new AtomicLongArray(5); private final DistributionStat queuedTime = new DistributionStat(); private final DistributionStat wallTime = new DistributionStat(); private volatile boolean closed; @Inject public TaskExecutor(TaskManagerConfig config) { this(checkNotNull(config, "config is null").getMaxShardProcessorThreads()); } public TaskExecutor(int runnerThreads) { this(runnerThreads, Ticker.systemTicker()); } @VisibleForTesting public TaskExecutor(int runnerThreads, Ticker ticker) { checkArgument(runnerThreads > 0, "runnerThreads must be at least 1"); // we manages thread pool size directly, so create an unlimited pool this.executor = Executors.newCachedThreadPool(threadsNamed("task-processor-%d")); this.executorMBean = new ThreadPoolExecutorMBean((ThreadPoolExecutor) executor); this.runnerThreads = runnerThreads; this.ticker = checkNotNull(ticker, "ticker is null"); // we assume we need at least two tasks per runner thread to keep the system busy this.minimumNumberOfTasks = 2 * this.runnerThreads; this.pendingSplits = new PriorityBlockingQueue<>(Runtime.getRuntime().availableProcessors() * 10); this.tasks = new LinkedList<>(); } @PostConstruct public synchronized void start() { checkState(!closed, "TaskExecutor is closed"); for (int i = 0; i < runnerThreads; i++) { addRunnerThread(); } } @PreDestroy public synchronized void stop() { closed = true; executor.shutdownNow(); } @Override public synchronized String toString() { return Objects.toStringHelper(this) .add("runnerThreads", runnerThreads) .add("allSplits", allSplits.size()) .add("pendingSplits", pendingSplits.size()) .add("runningSplits", runningSplits.size()) .add("blockedSplits", blockedSplits.size()) .toString(); } private synchronized void addRunnerThread() { try { executor.execute(new Runner()); } catch (RejectedExecutionException ignored) { } } public synchronized TaskHandle addTask(TaskId taskId) { TaskHandle taskHandle = new TaskHandle(checkNotNull(taskId, "taskId is null")); tasks.add(taskHandle); return taskHandle; } public synchronized void removeTask(TaskHandle taskHandle) { taskHandle.destroy(); tasks.remove(taskHandle); // record completed stats long threadUsageNanos = taskHandle.getThreadUsageNanos(); int priorityLevel = calculatePriorityLevel(threadUsageNanos); completedTasksPerLevel.incrementAndGet(priorityLevel); } public synchronized ListenableFuture<?> enqueueSplit( TaskHandle taskHandle, SplitRunner taskSplit) { PrioritizedSplitRunner prioritizedSplitRunner = new PrioritizedSplitRunner(taskHandle, taskSplit, ticker); taskHandle.addSplit(prioritizedSplitRunner); scheduleTaskIfNecessary(taskHandle); addNewEntrants(); return prioritizedSplitRunner.getFinishedFuture(); } public synchronized ListenableFuture<?> forceRunSplit( TaskHandle taskHandle, SplitRunner taskSplit) { PrioritizedSplitRunner prioritizedSplitRunner = new PrioritizedSplitRunner(taskHandle, taskSplit, ticker); // Note: we do not record queued time for forced splits startSplit(prioritizedSplitRunner); return prioritizedSplitRunner.getFinishedFuture(); } private synchronized void splitFinished(PrioritizedSplitRunner split) { allSplits.remove(split); pendingSplits.remove(split); TaskHandle taskHandle = split.getTaskHandle(); taskHandle.splitComplete(split); wallTime.add(System.nanoTime() - split.createdNanos); scheduleTaskIfNecessary(taskHandle); addNewEntrants(); } private synchronized void scheduleTaskIfNecessary(TaskHandle taskHandle) { // if task has less than the minimum guaranteed splits running, // immediately schedule a new split for this task. This assures // that a task gets its fair amount of consideration (you have to // have splits to be considered for running on a thread). if (taskHandle.getRunningSplits() < GUARANTEED_SPLITS_PER_TASK) { PrioritizedSplitRunner split = taskHandle.pollNextSplit(); if (split != null) { startSplit(split); queuedTime.add(System.nanoTime() - split.createdNanos); } } } private synchronized void addNewEntrants() { int running = allSplits.size(); for (int i = 0; i < minimumNumberOfTasks - running; i++) { PrioritizedSplitRunner split = pollNextSplitWorker(); if (split == null) { break; } queuedTime.add(System.nanoTime() - split.createdNanos); startSplit(split); } } private synchronized void startSplit(PrioritizedSplitRunner split) { allSplits.add(split); pendingSplits.put(split); } private synchronized PrioritizedSplitRunner pollNextSplitWorker() { // todo find a better algorithm for this // find the first task that produces a split, then move that task to the // end of the task list, so we get round robin for (Iterator<TaskHandle> iterator = tasks.iterator(); iterator.hasNext(); ) { TaskHandle task = iterator.next(); PrioritizedSplitRunner split = task.pollNextSplit(); if (split != null) { // move task to end of list iterator.remove(); // CAUTION: we are modifying the list in the loop which would normally // cause a ConcurrentModificationException but we exit immediately tasks.add(task); return split; } } return null; } @NotThreadSafe public static class TaskHandle { private final TaskId taskId; private final Queue<PrioritizedSplitRunner> queuedSplits = new ArrayDeque<>(10); private final List<PrioritizedSplitRunner> runningSplits = new ArrayList<>(10); private final AtomicLong taskThreadUsageNanos = new AtomicLong(); private TaskHandle(TaskId taskId) { this.taskId = taskId; } private long addThreadUsageNanos(long durationNanos) { return taskThreadUsageNanos.addAndGet(durationNanos); } private TaskId getTaskId() { return taskId; } private void destroy() { for (PrioritizedSplitRunner runningSplit : runningSplits) { runningSplit.destroy(); } runningSplits.clear(); for (PrioritizedSplitRunner queuedSplit : queuedSplits) { queuedSplit.destroy(); } queuedSplits.clear(); } private void addSplit(PrioritizedSplitRunner split) { queuedSplits.add(split); } private int getRunningSplits() { return runningSplits.size(); } private long getThreadUsageNanos() { return taskThreadUsageNanos.get(); } private PrioritizedSplitRunner pollNextSplit() { PrioritizedSplitRunner split = queuedSplits.poll(); if (split != null) { runningSplits.add(split); } return split; } private void splitComplete(PrioritizedSplitRunner split) { runningSplits.remove(split); split.destroy(); } @Override public String toString() { return Objects.toStringHelper(this).add("taskId", taskId).toString(); } } private static class PrioritizedSplitRunner implements Comparable<PrioritizedSplitRunner> { private final long createdNanos = System.nanoTime(); private final TaskHandle taskHandle; private final long workerId; private final SplitRunner split; private final Ticker ticker; private final SettableFuture<?> finishedFuture = SettableFuture.create(); private final AtomicBoolean initialized = new AtomicBoolean(); private final AtomicBoolean destroyed = new AtomicBoolean(); private final AtomicInteger priorityLevel = new AtomicInteger(); private final AtomicLong threadUsageNanos = new AtomicLong(); private final AtomicLong lastRun = new AtomicLong(); private PrioritizedSplitRunner(TaskHandle taskHandle, SplitRunner split, Ticker ticker) { this.taskHandle = taskHandle; this.split = split; this.ticker = ticker; this.workerId = NEXT_WORKER_ID.getAndIncrement(); } private TaskHandle getTaskHandle() { return taskHandle; } private SettableFuture<?> getFinishedFuture() { return finishedFuture; } public void initializeIfNecessary() { if (initialized.compareAndSet(false, true)) { split.initialize(); } } public void destroy() { try { split.close(); } catch (RuntimeException e) { log.error(e, "Error closing split for task %s", taskHandle.getTaskId()); } destroyed.set(true); } public boolean isFinished() { boolean finished = split.isFinished(); if (finished) { finishedFuture.set(null); } return finished || destroyed.get(); } public ListenableFuture<?> process() throws Exception { try { long start = ticker.read(); ListenableFuture<?> blocked = split.processFor(SPLIT_RUN_QUANTA); long endTime = ticker.read(); // update priority level base on total thread usage of task long durationNanos = endTime - start; long threadUsageNanos = taskHandle.addThreadUsageNanos(durationNanos); this.threadUsageNanos.set(threadUsageNanos); priorityLevel.set(calculatePriorityLevel(threadUsageNanos)); // record last run for prioritization within a level lastRun.set(endTime); return blocked; } catch (Throwable e) { finishedFuture.setException(e); throw e; } } public boolean updatePriorityLevel() { int newPriority = calculatePriorityLevel(taskHandle.getThreadUsageNanos()); if (newPriority == priorityLevel.getAndSet(newPriority)) { return false; } // update thread usage while if level changed threadUsageNanos.set(taskHandle.getThreadUsageNanos()); return true; } @Override public int compareTo(PrioritizedSplitRunner o) { int level = priorityLevel.get(); int result = Ints.compare(level, o.priorityLevel.get()); if (result != 0) { return result; } if (level < 4) { result = Long.compare(threadUsageNanos.get(), threadUsageNanos.get()); } else { result = Long.compare(lastRun.get(), o.lastRun.get()); } if (result != 0) { return result; } return Longs.compare(workerId, o.workerId); } @Override public String toString() { return String.format( "Split %-15s %s %s", taskHandle.getTaskId(), priorityLevel, new Duration(threadUsageNanos.get(), TimeUnit.NANOSECONDS) .convertToMostSuccinctTimeUnit()); } } private static int calculatePriorityLevel(long threadUsageNanos) { long millis = TimeUnit.NANOSECONDS.toMillis(threadUsageNanos); int priorityLevel; if (millis < 1000) { priorityLevel = 0; } else if (millis < 10_000) { priorityLevel = 1; } else if (millis < 60_000) { priorityLevel = 2; } else if (millis < 300_000) { priorityLevel = 3; } else { priorityLevel = 4; } return priorityLevel; } private class Runner implements Runnable { private final long runnerId = NEXT_RUNNER_ID.getAndIncrement(); @Override public void run() { try (SetThreadName runnerName = new SetThreadName("SplitRunner-%s", runnerId)) { while (!closed && !Thread.currentThread().isInterrupted()) { // select next worker final PrioritizedSplitRunner split; try { split = pendingSplits.take(); if (split.updatePriorityLevel()) { // priority level changed, return split to queue for re-prioritization pendingSplits.put(split); continue; } } catch (InterruptedException e) { Thread.currentThread().interrupt(); return; } try (SetThreadName splitName = new SetThreadName(split.toString())) { runningSplits.add(split); boolean finished; ListenableFuture<?> blocked; try { split.initializeIfNecessary(); blocked = split.process(); finished = split.isFinished(); } finally { runningSplits.remove(split); } if (finished) { log.debug("%s is finished", split); splitFinished(split); } else { if (blocked.isDone()) { pendingSplits.put(split); } else { blockedSplits.add(split); blocked.addListener( new Runnable() { @Override public void run() { blockedSplits.remove(split); split.updatePriorityLevel(); pendingSplits.put(split); } }, executor); } } } catch (Throwable t) { log.error(t, "Error processing %s", split); splitFinished(split); } } } finally { // unless we have been closed, we need to replace this thread if (!closed) { addRunnerThread(); } } } } // // STATS // @Managed public int getTasks() { return tasks.size(); } @Managed public int getRunnerThreads() { return runnerThreads; } @Managed public int getMinimumNumberOfTasks() { return minimumNumberOfTasks; } @Managed public int getTotalSplits() { return allSplits.size(); } @Managed public int getPendingSplits() { return pendingSplits.size(); } @Managed public int getRunningSplits() { return runningSplits.size(); } @Managed public int getBlockedSplits() { return blockedSplits.size(); } @Managed public long getCompletedTasksLevel0() { return completedTasksPerLevel.get(0); } @Managed public long getCompletedTasksLevel1() { return completedTasksPerLevel.get(1); } @Managed public long getCompletedTasksLevel2() { return completedTasksPerLevel.get(2); } @Managed public long getCompletedTasksLevel3() { return completedTasksPerLevel.get(3); } @Managed public long getCompletedTasksLevel4() { return completedTasksPerLevel.get(4); } @Managed public long getRunningTasksLevel0() { return calculateRunningTasksForLevel(0); } @Managed public long getRunningTasksLevel1() { return calculateRunningTasksForLevel(1); } @Managed public long getRunningTasksLevel2() { return calculateRunningTasksForLevel(2); } @Managed public long getRunningTasksLevel3() { return calculateRunningTasksForLevel(3); } @Managed public long getRunningTasksLevel4() { return calculateRunningTasksForLevel(4); } @Managed @Nested public DistributionStat getQueuedTime() { return queuedTime; } @Managed @Nested public DistributionStat getWallTime() { return wallTime; } private synchronized int calculateRunningTasksForLevel(int level) { int count = 0; for (TaskHandle task : tasks) { if (calculatePriorityLevel(task.getThreadUsageNanos()) == level) { count++; } } return count; } @Managed(description = "Task processor executor") @Nested public ThreadPoolExecutorMBean getProcessorExecutor() { return executorMBean; } }
/** * Sets up Shiro lifecycles within Guice, enables the injecting of Shiro objects, and binds a * default {@link org.apache.shiro.mgt.SecurityManager} and {@link * org.apache.shiro.session.mgt.SessionManager}. At least one realm must be added by using {@link * #bindRealm() bindRealm}. */ public abstract class ShiroModule extends PrivateModule implements Destroyable { private Set<Destroyable> destroyables = Sets.newSetFromMap(new WeakHashMap<Destroyable, Boolean>()); public void configure() { this.binder().requireExplicitBindings(); // setup security manager bindSecurityManager(bind(SecurityManager.class)); bindSessionManager(bind(SessionManager.class)); bindEnvironment(bind(Environment.class)); bindListener(BeanTypeListener.MATCHER, new BeanTypeListener()); final DestroyableInjectionListener.DestroyableRegistry registry = new DestroyableInjectionListener.DestroyableRegistry() { public void add(Destroyable destroyable) { ShiroModule.this.add(destroyable); } @PreDestroy public void destroy() throws Exception { ShiroModule.this.destroy(); } }; bindListener(LifecycleTypeListener.MATCHER, new LifecycleTypeListener(registry)); expose(SecurityManager.class); configureShiro(); bind(realmCollectionKey()).to(realmSetKey()); bind(DestroyableInjectionListener.DestroyableRegistry.class).toInstance(registry); BeanTypeListener.ensureBeanTypeMapExists(binder()); } @SuppressWarnings({"unchecked"}) private Key<Set<Realm>> realmSetKey() { return (Key<Set<Realm>>) Key.get(TypeLiteral.get(Types.setOf(Realm.class))); } @SuppressWarnings({"unchecked"}) private Key<Collection<Realm>> realmCollectionKey() { return (Key<Collection<Realm>>) Key.get(Types.newParameterizedType(Collection.class, Realm.class)); } /** * Implement this method in order to configure your realms and any other Shiro customization you * may need. */ protected abstract void configureShiro(); /** * This is the preferred manner to bind a realm. The {@link org.apache.shiro.mgt.SecurityManager} * will be injected with any Realm bound with this method. * * @return a binding builder for a realm */ protected final LinkedBindingBuilder<Realm> bindRealm() { Multibinder<Realm> multibinder = Multibinder.newSetBinder(binder(), Realm.class); return multibinder.addBinding(); } /** * Binds the security manager. Override this method in order to provide your own security manager * binding. * * <p>By default, a {@link org.apache.shiro.mgt.DefaultSecurityManager} is bound as an eager * singleton. * * @param bind */ protected void bindSecurityManager(AnnotatedBindingBuilder<? super SecurityManager> bind) { try { bind.toConstructor(DefaultSecurityManager.class.getConstructor(Collection.class)) .asEagerSingleton(); } catch (NoSuchMethodException e) { throw new ConfigurationException( "This really shouldn't happen. Either something has changed in Shiro, or there's a bug in " + ShiroModule.class.getSimpleName(), e); } } /** * Binds the session manager. Override this method in order to provide your own session manager * binding. * * <p>By default, a {@link org.apache.shiro.session.mgt.DefaultSessionManager} is bound as an * eager singleton. * * @param bind */ protected void bindSessionManager(AnnotatedBindingBuilder<SessionManager> bind) { bind.to(DefaultSessionManager.class).asEagerSingleton(); } /** * Binds the environment. Override this method in order to provide your own environment binding. * * <p>By default, a {@link GuiceEnvironment} is bound as an eager singleton. * * @param bind */ protected void bindEnvironment(AnnotatedBindingBuilder<Environment> bind) { bind.to(GuiceEnvironment.class).asEagerSingleton(); } /** * Binds a key to use for injecting setters in shiro classes. * * @param typeLiteral the bean property type * @param key the key to use to satisfy the bean property dependency * @param <T> */ protected final <T> void bindBeanType(TypeLiteral<T> typeLiteral, Key<? extends T> key) { BeanTypeListener.bindBeanType(binder(), typeLiteral, key); } /** * Destroys all beans created within this module that implement {@link * org.apache.shiro.util.Destroyable}. Should be called when this module will no longer be used. * * @throws Exception */ public final void destroy() throws Exception { for (Destroyable destroyable : destroyables) { destroyable.destroy(); } } public void add(Destroyable destroyable) { this.destroyables.add(destroyable); } }
/** * Decoration on an ExecutorService that tracks created futures and provides a method to close * futures created via this class */ public class CloseableExecutorService implements Closeable { protected final AtomicBoolean isOpen = new AtomicBoolean(true); private final Logger log = LoggerFactory.getLogger(CloseableExecutorService.class); private final Set<Future<?>> futures = Sets.newSetFromMap(Maps.<Future<?>, Boolean>newConcurrentMap()); private final ExecutorService executorService; private final boolean shutdownOnClose; /** @param executorService the service to decorate */ public CloseableExecutorService(ExecutorService executorService) { this(executorService, false); } /** * @param executorService the service to decorate * @param shutdownOnClose if true, shutdown the executor service when this is closed */ public CloseableExecutorService(ExecutorService executorService, boolean shutdownOnClose) { this.executorService = Preconditions.checkNotNull(executorService, "executorService cannot be null"); this.shutdownOnClose = shutdownOnClose; } /** * Returns <tt>true</tt> if this executor has been shut down. * * @return <tt>true</tt> if this executor has been shut down */ public boolean isShutdown() { return !isOpen.get(); } @VisibleForTesting int size() { return futures.size(); } /** Closes any tasks currently in progress */ @Override public void close() { isOpen.set(false); Iterator<Future<?>> iterator = futures.iterator(); while (iterator.hasNext()) { Future<?> future = iterator.next(); iterator.remove(); if (!future.isDone() && !future.isCancelled() && !future.cancel(true)) { log.warn("Could not cancel " + future); } } if (shutdownOnClose) { executorService.shutdownNow(); } } /** * Submits a value-returning task for execution and returns a Future representing the pending * results of the task. Upon completion, this task may be taken or polled. * * @param task the task to submit * @return a future to watch the task */ public <V> Future<V> submit(Callable<V> task) { Preconditions.checkState(isOpen.get(), "CloseableExecutorService is closed"); InternalFutureTask<V> futureTask = new InternalFutureTask<V>(new FutureTask<V>(task)); executorService.execute(futureTask); return futureTask; } /** * Submits a Runnable task for execution and returns a Future representing that task. Upon * completion, this task may be taken or polled. * * @param task the task to submit * @return a future to watch the task */ public Future<?> submit(Runnable task) { Preconditions.checkState(isOpen.get(), "CloseableExecutorService is closed"); InternalFutureTask<Void> futureTask = new InternalFutureTask<Void>(new FutureTask<Void>(task, null)); executorService.execute(futureTask); return futureTask; } protected class InternalScheduledFutureTask implements Future<Void> { private final ScheduledFuture<?> scheduledFuture; public InternalScheduledFutureTask(ScheduledFuture<?> scheduledFuture) { this.scheduledFuture = scheduledFuture; futures.add(scheduledFuture); } @Override public boolean cancel(boolean mayInterruptIfRunning) { futures.remove(scheduledFuture); return scheduledFuture.cancel(mayInterruptIfRunning); } @Override public boolean isCancelled() { return scheduledFuture.isCancelled(); } @Override public boolean isDone() { return scheduledFuture.isDone(); } @Override public Void get() throws InterruptedException, ExecutionException { return null; } @Override public Void get(long timeout, TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { return null; } } protected class InternalFutureTask<T> extends FutureTask<T> { private final RunnableFuture<T> task; InternalFutureTask(RunnableFuture<T> task) { super(task, null); this.task = task; futures.add(task); } protected void done() { futures.remove(task); } } }
public FormatterSmokeTester() throws NoSuchAlgorithmException { messageDigest = MessageDigest.getInstance("MD5"); seen = Sets.newSetFromMap(new ConcurrentHashMap<BigInteger, Boolean>()); }
public void testNewSetFromMap() { Set<Integer> set = Sets.newSetFromMap(new HashMap<Integer, Boolean>()); set.addAll(SOME_COLLECTION); verifySetContents(set, SOME_COLLECTION); }
/** * The main class for non-obfuscated hook handling code * * <p>Anything that doesn't require obfuscated or client/server specific code should go in this * handler * * <p>It also contains a reference to the sided handler instance that is valid allowing for common * code to access specific properties from the obfuscated world without a direct dependency * * @author cpw */ public class FMLCommonHandler { /** The singleton */ private static final FMLCommonHandler INSTANCE = new FMLCommonHandler(); /** The delegate for side specific data and functions */ private IFMLSidedHandler sidedDelegate; private List<IScheduledTickHandler> scheduledClientTicks = Lists.newArrayList(); private List<IScheduledTickHandler> scheduledServerTicks = Lists.newArrayList(); private Class<?> forge; private boolean noForge; private List<String> brandings; private List<ICrashCallable> crashCallables = Lists.newArrayList(Loader.instance().getCallableCrashInformation()); private Set<SaveHandler> handlerSet = Sets.newSetFromMap(new MapMaker().weakKeys().<SaveHandler, Boolean>makeMap()); public void beginLoading(IFMLSidedHandler handler) { sidedDelegate = handler; FMLLog.info("Attempting early MinecraftForge initialization"); callForgeMethod("initialize"); callForgeMethod("registerCrashCallable"); FMLLog.info("Completed early MinecraftForge initialization"); } public void rescheduleTicks(Side side) { TickRegistry.updateTickQueue( side.isClient() ? scheduledClientTicks : scheduledServerTicks, side); } public void tickStart(EnumSet<TickType> ticks, Side side, Object... data) { List<IScheduledTickHandler> scheduledTicks = side.isClient() ? scheduledClientTicks : scheduledServerTicks; if (scheduledTicks.size() == 0) { return; } for (IScheduledTickHandler ticker : scheduledTicks) { EnumSet<TickType> ticksToRun = EnumSet.copyOf(Objects.firstNonNull(ticker.ticks(), EnumSet.noneOf(TickType.class))); ticksToRun.removeAll(EnumSet.complementOf(ticks)); if (!ticksToRun.isEmpty()) { ticker.tickStart(ticksToRun, data); } } } public void tickEnd(EnumSet<TickType> ticks, Side side, Object... data) { List<IScheduledTickHandler> scheduledTicks = side.isClient() ? scheduledClientTicks : scheduledServerTicks; if (scheduledTicks.size() == 0) { return; } for (IScheduledTickHandler ticker : scheduledTicks) { EnumSet<TickType> ticksToRun = EnumSet.copyOf(Objects.firstNonNull(ticker.ticks(), EnumSet.noneOf(TickType.class))); ticksToRun.removeAll(EnumSet.complementOf(ticks)); if (!ticksToRun.isEmpty()) { ticker.tickEnd(ticksToRun, data); } } } /** @return the instance */ public static FMLCommonHandler instance() { return INSTANCE; } /** * Find the container that associates with the supplied mod object * * @param mod */ public ModContainer findContainerFor(Object mod) { return Loader.instance().getReversedModObjectList().get(mod); } /** * Get the forge mod loader logging instance (goes to the forgemodloader log file) * * @return The log instance for the FML log file */ public Logger getFMLLogger() { return FMLLog.getLogger(); } public Side getSide() { return sidedDelegate.getSide(); } /** * Return the effective side for the context in the game. This is dependent on thread analysis to * try and determine whether the code is running in the server or not. Use at your own risk */ public Side getEffectiveSide() { Thread thr = Thread.currentThread(); if ((thr instanceof ThreadMinecraftServer) || (thr instanceof ServerListenThread)) { return Side.SERVER; } return Side.CLIENT; } /** Raise an exception */ public void raiseException(Throwable exception, String message, boolean stopGame) { FMLCommonHandler.instance().getFMLLogger().throwing("FMLHandler", "raiseException", exception); if (stopGame) { getSidedDelegate().haltGame(message, exception); } } private Class<?> findMinecraftForge() { if (forge == null && !noForge) { try { forge = Class.forName("net.minecraftforge.common.MinecraftForge"); } catch (Exception ex) { noForge = true; } } return forge; } private Object callForgeMethod(String method) { if (noForge) return null; try { return findMinecraftForge().getMethod(method).invoke(null); } catch (Exception e) { // No Forge installation return null; } } public void computeBranding() { if (brandings == null) { Builder brd = ImmutableList.<String>builder(); brd.add(Loader.instance().getMCVersionString()); brd.add(Loader.instance().getMCPVersionString()); brd.add("FML v" + Loader.instance().getFMLVersionString()); String forgeBranding = (String) callForgeMethod("getBrandingVersion"); if (!Strings.isNullOrEmpty(forgeBranding)) { brd.add(forgeBranding); } if (sidedDelegate != null) { brd.addAll(sidedDelegate.getAdditionalBrandingInformation()); } try { Properties props = new Properties(); props.load(getClass().getClassLoader().getResourceAsStream("fmlbranding.properties")); brd.add(props.getProperty("fmlbranding")); } catch (Exception ex) { // Ignore - no branding file found } int tModCount = Loader.instance().getModList().size(); int aModCount = Loader.instance().getActiveModList().size(); brd.add( String.format( "%d mod%s loaded, %d mod%s active", tModCount, tModCount != 1 ? "s" : "", aModCount, aModCount != 1 ? "s" : "")); brandings = brd.build(); } } public List<String> getBrandings() { if (brandings == null) { computeBranding(); } return ImmutableList.copyOf(brandings); } public IFMLSidedHandler getSidedDelegate() { return sidedDelegate; } public void onPostServerTick() { tickEnd(EnumSet.of(TickType.SERVER), Side.SERVER); } /** Every tick just after world and other ticks occur */ public void onPostWorldTick(Object world) { tickEnd(EnumSet.of(TickType.WORLD), Side.SERVER, world); } public void onPreServerTick() { tickStart(EnumSet.of(TickType.SERVER), Side.SERVER); } /** Every tick just before world and other ticks occur */ public void onPreWorldTick(Object world) { tickStart(EnumSet.of(TickType.WORLD), Side.SERVER, world); } public void onWorldLoadTick(World[] worlds) { rescheduleTicks(Side.SERVER); for (World w : worlds) { tickStart(EnumSet.of(TickType.WORLDLOAD), Side.SERVER, w); } } public void handleServerStarting(MinecraftServer server) { Loader.instance().serverStarting(server); } public void handleServerStarted() { Loader.instance().serverStarted(); } public void handleServerStopping() { Loader.instance().serverStopping(); } public MinecraftServer getMinecraftServerInstance() { return sidedDelegate.getServer(); } public void showGuiScreen(Object clientGuiElement) { sidedDelegate.showGuiScreen(clientGuiElement); } public Entity spawnEntityIntoClientWorld( EntityRegistration registration, EntitySpawnPacket entitySpawnPacket) { return sidedDelegate.spawnEntityIntoClientWorld(registration, entitySpawnPacket); } public void adjustEntityLocationOnClient( EntitySpawnAdjustmentPacket entitySpawnAdjustmentPacket) { sidedDelegate.adjustEntityLocationOnClient(entitySpawnAdjustmentPacket); } public void onServerStart(DedicatedServer dedicatedServer) { FMLServerHandler.instance(); sidedDelegate.beginServerLoading(dedicatedServer); } public void onServerStarted() { sidedDelegate.finishServerLoading(); } public void onPreClientTick() { tickStart(EnumSet.of(TickType.CLIENT), Side.CLIENT); } public void onPostClientTick() { tickEnd(EnumSet.of(TickType.CLIENT), Side.CLIENT); } public void onRenderTickStart(float timer) { tickStart(EnumSet.of(TickType.RENDER), Side.CLIENT, timer); } public void onRenderTickEnd(float timer) { tickEnd(EnumSet.of(TickType.RENDER), Side.CLIENT, timer); } public void onPlayerPreTick(EntityPlayer player) { Side side = player instanceof EntityPlayerMP ? Side.SERVER : Side.CLIENT; tickStart(EnumSet.of(TickType.PLAYER), side, player); } public void onPlayerPostTick(EntityPlayer player) { Side side = player instanceof EntityPlayerMP ? Side.SERVER : Side.CLIENT; tickEnd(EnumSet.of(TickType.PLAYER), side, player); } public void registerCrashCallable(ICrashCallable callable) { crashCallables.add(callable); } public void enhanceCrashReport(CrashReport crashReport, CrashReportCategory category) { for (ICrashCallable call : crashCallables) { category.func_71500_a(call.getLabel(), call); } } public void handleTinyPacket(NetHandler handler, Packet131MapData mapData) { sidedDelegate.handleTinyPacket(handler, mapData); } public void handleWorldDataSave( SaveHandler handler, WorldInfo worldInfo, NBTTagCompound tagCompound) { for (ModContainer mc : Loader.instance().getModList()) { if (mc instanceof InjectedModContainer) { WorldAccessContainer wac = ((InjectedModContainer) mc).getWrappedWorldAccessContainer(); if (wac != null) { NBTTagCompound dataForWriting = wac.getDataForWriting(handler, worldInfo); tagCompound.func_74766_a(mc.getModId(), dataForWriting); } } } } public void handleWorldDataLoad( SaveHandler handler, WorldInfo worldInfo, NBTTagCompound tagCompound) { if (getEffectiveSide() != Side.SERVER) { return; } if (handlerSet.contains(handler)) { return; } handlerSet.add(handler); Map<String, NBTBase> additionalProperties = Maps.newHashMap(); worldInfo.setAdditionalProperties(additionalProperties); for (ModContainer mc : Loader.instance().getModList()) { if (mc instanceof InjectedModContainer) { WorldAccessContainer wac = ((InjectedModContainer) mc).getWrappedWorldAccessContainer(); if (wac != null) { wac.readData( handler, worldInfo, additionalProperties, tagCompound.func_74775_l(mc.getModId())); } } } } public boolean shouldServerBeKilledQuietly() { return sidedDelegate.shouldServerShouldBeKilledQuietly(); } public void disconnectIDMismatch( MapDifference<Integer, ItemData> serverDifference, NetHandler toKill, INetworkManager network) { sidedDelegate.disconnectIDMismatch(serverDifference, toKill, network); } }