private TaskStatus runTask(final Task task) throws Exception { final Task dummyTask = new DefaultObjectMapper() .readValue("{\"type\":\"noop\", \"isReadyResult\":\"exception\"}\"", Task.class); final long startTime = System.currentTimeMillis(); Preconditions.checkArgument(!task.getId().equals(dummyTask.getId())); tq.add(dummyTask); tq.add(task); TaskStatus retVal = null; for (final String taskId : ImmutableList.of(dummyTask.getId(), task.getId())) { try { TaskStatus status; while ((status = tsqa.getStatus(taskId).get()).isRunnable()) { if (System.currentTimeMillis() > startTime + 10 * 1000) { throw new ISE("Where did the task go?!: %s", task.getId()); } Thread.sleep(100); } if (taskId.equals(task.getId())) { retVal = status; } } catch (Exception e) { throw Throwables.propagate(e); } } return retVal; }
/** * Performs the current task in the queue and checks to see if the task is complete. If the task * is complete, this removes the task from the queue and performs any necessary clean-up. */ public void DoCurrentTask() { try { // If we have something to do, try to get it done if (currentTask != null) { currentTask.PerformTaskActions(); // Check to see if the current task is complete if (currentTask.IsComplete()) { // Perform any cleanup actions associated with the task with // which we're finished currentTask.CleanUpTask(); // If we have another task to do, make it the current task, // otherwise we make the current task null if (taskQueue.Size() > 0) { currentTask = taskQueue.Pop(); } else { currentTask = null; } } } else if (taskQueue.Size() > 0) { currentTask = taskQueue.Pop(); } } catch (Exception ex) { currentTask = null; System.err.println(ex.toString()); } }
// ---------------------------------------------- Public Methods public void start() throws LifecycleException { lifecycle.fireLifecycleEvent(BEFORE_START_EVENT, null); TaskQueue taskqueue = new TaskQueue(maxQueueSize); TaskThreadFactory tf = new TaskThreadFactory(namePrefix); lifecycle.fireLifecycleEvent(START_EVENT, null); executor = new ThreadPoolExecutor( getMinSpareThreads(), getMaxThreads(), maxIdleTime, TimeUnit.MILLISECONDS, taskqueue, tf) { @Override protected void afterExecute(Runnable r, Throwable t) { AtomicInteger atomic = submittedTasksCount; if (atomic != null) { atomic.decrementAndGet(); } } }; taskqueue.setParent((ThreadPoolExecutor) executor); submittedTasksCount = new AtomicInteger(); lifecycle.fireLifecycleEvent(AFTER_START_EVENT, null); }
void closeAllQueues() { synchronized (queues) { for (Iterator iter = queues.values().iterator(); iter.hasNext(); ) { TaskQueue queue = (TaskQueue) iter.next(); queue.shutdown(); } } queues.clear(); }
@Test public void testSize() { TaskQueue<Object> taskQueue = new TaskQueue<Object>(10); Assert.assertEquals(0, taskQueue.size()); for (int i = 1; i <= 10; i++) { Assert.assertTrue(taskQueue.offer(new Object(), new boolean[1])); Assert.assertEquals(i, taskQueue.size()); } }
@Test public void testRemainingCapacity() { TaskQueue<Object> taskQueue = new TaskQueue<Object>(10); Assert.assertEquals(10, taskQueue.remainingCapacity()); for (int i = 1; i <= 10; i++) { Assert.assertTrue(taskQueue.offer(new Object(), new boolean[1])); Assert.assertEquals(10 - i, taskQueue.remainingCapacity()); } }
/** * Create the queue, if it doesn't exist already. If the queue has been created on a previous call * to createQueue(), then that instance is returned instead. * * @param name queue name * @param driverClass class to use as the queue driver * @param loggerIn queue logger * @return queue instance * @throws Exception error occurred during queue creation */ public TaskQueue createQueue(String name, Class driverClass, Logger loggerIn) throws Exception { TaskQueue retval = null; synchronized (queues) { retval = (TaskQueue) queues.get(name); if (retval == null) { retval = new TaskQueue(); QueueDriver driver = (QueueDriver) driverClass.newInstance(); driver.setLogger(loggerIn); driver.initialize(); retval.setQueueDriver(driver); queues.put(name, retval); } } return retval; }
@Test(timeout = 4000L) public void testRealtimeIndexTaskFailure() throws Exception { setUpAndStartTaskQueue( new DataSegmentPusher() { @Override public String getPathForHadoop(String s) { throw new UnsupportedOperationException(); } @Override public DataSegment push(File file, DataSegment dataSegment) throws IOException { throw new RuntimeException("FAILURE"); } }); monitorScheduler.addMonitor(EasyMock.anyObject(Monitor.class)); EasyMock.expectLastCall().atLeastOnce(); monitorScheduler.removeMonitor(EasyMock.anyObject(Monitor.class)); EasyMock.expectLastCall().anyTimes(); EasyMock.replay(monitorScheduler, queryRunnerFactoryConglomerate); RealtimeIndexTask realtimeIndexTask = giveMeARealtimeIndexTask(); final String taskId = realtimeIndexTask.getId(); tq.add(realtimeIndexTask); // Wait for realtime index task to fail while (tsqa.getStatus(taskId).get().isRunnable()) { Thread.sleep(10); } Assert.assertTrue("Task should be in Failure state", tsqa.getStatus(taskId).get().isFailure()); EasyMock.verify(monitorScheduler, queryRunnerFactoryConglomerate); }
/** Removes all tasks from the queue, performing clean-up actions as necessary. */ public void ClearAllTasks() { TaskBase popedTask; while (taskQueue.Size() > 0) { // Remove the task from the queue and perform necessary clean-up // actions try { popedTask = taskQueue.Pop(); popedTask.CleanUpTask(); } catch (Exception ex) { System.err.println(ex.toString()); } } // In addition to clearing the queue, set the current task to null currentTask = null; }
/** * Submits new harvesting task for partial harvest. * * @param context request context * @param resource resource to harvest * @param maxRecs maximum number of records to harvest (<code>null</code> for no maximum limit) * @param fromDate to harvest only from the specific date (<code>null</code> for no from date) * @return <code>true</code> if task has been submitted */ public boolean submit(RequestContext context, HrRecord resource, Integer maxRecs, Date fromDate) { if (resource == null) throw new IllegalArgumentException("No resource to harvest provided."); // create instance of the task // add only if no similar task currently executing boolean submitted = false; if (ApprovalStatus.isPubliclyVisible(resource.getApprovalStatus().name()) && resource.getSynchronizable()) { CommonCriteria criteria = new CommonCriteria(); criteria.setMaxRecords(maxRecs); if (fromDate != null) { Calendar cal = Calendar.getInstance(); cal.setTime(fromDate); cal.set(Calendar.HOUR_OF_DAY, 0); cal.set(Calendar.MINUTE, 0); cal.set(Calendar.SECOND, 0); cal.set(Calendar.MILLISECOND, 0); fromDate = cal.getTime(); } criteria.setFromDate(fromDate); submitted = !isExecutingLocally(resource.getUuid()) && (taskQueue != null ? taskQueue.add(context, resource, criteria) : false); LOGGER.log( Level.FINER, "[SYNCHRONIZER] Submitted resource: {0} ({1})", new Object[] {resource.getUuid(), resource.getName()}); } return submitted; }
/** * Returns true if a current task exists. * * @return True if a current task exists, false otherwise */ public boolean PerformingTask() { // If we are doing a task or are about to do a task, // return true if (taskQueue.Size() > 0 || currentTask != null) { return true; } return false; }
/** * Cancels pending tasks for given repository. * * @param context request context * @param uuid repository uuid * @return <code>true</code> if has been canceled properly */ public boolean cancel(RequestContext context, String uuid) { LOGGER.log(Level.FINER, "[SYNCHRONIZER] Canceled resource: {0}", uuid); // drop resource already being harvested boolean dropped = pool != null ? pool.drop(uuid) : false; // withdraw from the queue boolean canceled = taskQueue != null ? taskQueue.cancel(context, uuid) : false; // exit with status return canceled || dropped; }
static void clean() { TaskQueue.getInstance().cancelAll(); if (s_tasks != null) { for (int i = 0; i < s_tasks.length; i++) { if (s_tasks[i] != null) { s_tasks[i] = null; } } } }
@Test public void testTake() throws InterruptedException { final TaskQueue<Object> taskQueue = new TaskQueue<Object>(); final Object object = new Object(); Assert.assertTrue(taskQueue.offer(object, new boolean[1])); Assert.assertSame(object, taskQueue.take()); Thread thread = new Thread() { @Override public void run() { try { for (int i = 0; i < 10; i++) { Assert.assertEquals(i, taskQueue.take()); } } catch (InterruptedException ie) { Assert.fail(); } try { taskQueue.take(); Assert.fail(); } catch (InterruptedException ie) { } } }; thread.start(); for (int i = 0; i < 10; i++) { Assert.assertTrue(taskQueue.offer(i, new boolean[1])); } Thread.sleep(TestUtil.SHORT_WAIT); thread.interrupt(); thread.join(); }
public void run() { while (true) { synchronized (this.queue) { if (this.queue.isEmpty()) { try { System.out.println(name + " waiting for task"); queue.wait(); } catch (InterruptedException e) { e.printStackTrace(); } } // method should be synchronized this.task = queue.getNextTask(); this.length = task.length(); } doTask(); } }
@Test public void testOffer() { TaskQueue<Object> taskQueue = new TaskQueue<Object>(10); try { taskQueue.offer(null, new boolean[1]); Assert.fail(); } catch (NullPointerException npe) { } taskQueue = new TaskQueue<Object>(10); try { taskQueue.offer(new Object(), null); Assert.fail(); } catch (NullPointerException npe) { } taskQueue = new TaskQueue<Object>(10); try { taskQueue.offer(new Object(), new boolean[0]); Assert.fail(); } catch (IllegalArgumentException iae) { } taskQueue = new TaskQueue<Object>(10); boolean[] hasWaiterMarker = new boolean[1]; boolean result = taskQueue.offer(new Object(), hasWaiterMarker); Assert.assertTrue(result); Assert.assertFalse(hasWaiterMarker[0]); }
@Test public void testConstructor() { try { new TaskQueue<Object>(0); Assert.fail(); } catch (IllegalArgumentException iae) { } try { new TaskQueue<Object>(-1); Assert.fail(); } catch (IllegalArgumentException iae) { } TaskQueue<Object> taskQueue = new TaskQueue<Object>(10); Assert.assertEquals(10, taskQueue.remainingCapacity()); taskQueue = new TaskQueue<Object>(); Assert.assertEquals(Integer.MAX_VALUE, taskQueue.remainingCapacity()); }
/** Recovers records. */ private void resetRunning() { RequestContext context = RequestContext.extract(null); try { HjResetRunningRequest recover = new HjResetRunningRequest(context); recover.execute(); if (taskQueue != null) { taskQueue.notifyChange(); } } catch (SQLException ex) { LOGGER.log(Level.SEVERE, "[SYNCHRONIZER] Error recovering from the previous failout", ex); } finally { context.onExecutionPhaseCompleted(); } }
@Test public void testIsEmpty() { TaskQueue<Object> taskQueue = new TaskQueue<Object>(); Assert.assertTrue(taskQueue.isEmpty()); Assert.assertTrue(taskQueue.offer(new Object(), new boolean[1])); Assert.assertFalse(taskQueue.isEmpty()); Assert.assertNotNull(taskQueue.poll()); Assert.assertTrue(taskQueue.isEmpty()); }
@Test(timeout = 4000L) public void testRealtimeIndexTask() throws Exception { monitorScheduler.addMonitor(EasyMock.anyObject(Monitor.class)); EasyMock.expectLastCall().atLeastOnce(); monitorScheduler.removeMonitor(EasyMock.anyObject(Monitor.class)); EasyMock.expectLastCall().anyTimes(); EasyMock.replay(monitorScheduler, queryRunnerFactoryConglomerate); RealtimeIndexTask realtimeIndexTask = giveMeARealtimeIndexTask(); final String taskId = realtimeIndexTask.getId(); tq.add(realtimeIndexTask); // wait for task to process events and publish segment Assert.assertTrue(publishCountDown.await(1000, TimeUnit.MILLISECONDS)); // Realtime Task has published the segment, simulate loading of segment to a historical node so // that task finishes with SUCCESS status segmentCallbacks .get(0) .segmentAdded( new DruidServerMetadata("dummy", "dummy_host", 0, "historical", "dummy_tier", 0), mdc.getPublished().iterator().next()); // Wait for realtime index task to handle callback in plumber and succeed while (tsqa.getStatus(taskId).get().isRunnable()) { Thread.sleep(10); } Assert.assertTrue("Task should be in Success state", tsqa.getStatus(taskId).get().isSuccess()); Assert.assertEquals(1, announcedSinks); Assert.assertEquals(1, pushedSegments); Assert.assertEquals(1, mdc.getPublished().size()); DataSegment segment = mdc.getPublished().iterator().next(); Assert.assertEquals("test_ds", segment.getDataSource()); Assert.assertEquals(ImmutableList.of("dim1", "dim2"), segment.getDimensions()); Assert.assertEquals( new Interval(now.toString("YYYY-MM-dd") + "/" + now.plusDays(1).toString("YYYY-MM-dd")), segment.getInterval()); Assert.assertEquals(ImmutableList.of("count"), segment.getMetrics()); EasyMock.verify(monitorScheduler, queryRunnerFactoryConglomerate); }
/** * Spans new, separate thread exclusively for the resource. * * @param context request context * @param resource resource to harvest * @param maxRecs maximum number of records to harvest (<code>null</code> for no maximum limit) * @param fromDate to harvest only from the specific date (<code>null</code> for no from date) * @return <code>true</code> if task has been sumbited */ public boolean span(RequestContext context, HrRecord resource, Integer maxRecs, Date fromDate) { if (resource == null) throw new IllegalArgumentException("No resource to harvest provided."); // create instance of the task // add only if no similar task currently executing boolean submitted = false; if (resource.getApprovalStatus() == ApprovalStatus.approved && resource.getSynchronizable()) { CommonCriteria criteria = new CommonCriteria(); criteria.setMaxRecords(maxRecs); criteria.setFromDate(fromDate); submitted = pool != null && taskQueue != null ? !pool.isExecuting(resource.getUuid()) && taskQueue.register(context, resource, criteria) : false; if (submitted) pool.span(resource, criteria); LOGGER.log( Level.FINER, "[SYNCHRONIZER] Submitted resource: {0} ({1})", new Object[] {resource.getUuid(), resource.getName()}); } return submitted; }
public void post(Object event) { if (event == null) { throw new NullPointerException("Event must not be null"); } if (mMainThread == Thread.currentThread()) { // this is main thread Task task = Task.obtainTask(this, Task.CODE_POST, event); mTaskQueue.offer(task); if (!mProcessing) processQueue(); } else { // this is a background thread if (mMainThread.isAlive()) { Task task = Task.obtainTask(this, Task.CODE_DISPATCH_FROM_BACKGROUND, event) .setTaskCallbacks(mImpl); getMainHandlerNotNull().post(task); } } }
@After public void tearDown() { tq.stop(); }
/** * Adds the specified task to the queue. * * @param newTask Task to be added to the queue */ public void AddTask(TaskBase newTask) { taskQueue.Push(newTask); }
private void setUpAndStartTaskQueue(DataSegmentPusher dataSegmentPusher) { final TaskConfig taskConfig = new TaskConfig(tmpDir.toString(), null, null, 50000, null, null, null); tsqa = new TaskStorageQueryAdapter(ts); tl = new TaskLockbox(ts); mdc = newMockMDC(); tac = new LocalTaskActionClientFactory(ts, new TaskActionToolbox(tl, mdc, newMockEmitter())); tb = new TaskToolboxFactory( taskConfig, tac, newMockEmitter(), dataSegmentPusher, new LocalDataSegmentKiller(), new DataSegmentMover() { @Override public DataSegment move(DataSegment dataSegment, Map<String, Object> targetLoadSpec) throws SegmentLoadingException { return dataSegment; } }, new DataSegmentArchiver() { @Override public DataSegment archive(DataSegment segment) throws SegmentLoadingException { return segment; } @Override public DataSegment restore(DataSegment segment) throws SegmentLoadingException { return segment; } }, new DataSegmentAnnouncer() { @Override public void announceSegment(DataSegment segment) throws IOException { announcedSinks++; } @Override public void unannounceSegment(DataSegment segment) throws IOException {} @Override public void announceSegments(Iterable<DataSegment> segments) throws IOException {} @Override public void unannounceSegments(Iterable<DataSegment> segments) throws IOException {} }, // segment announcer serverView, // new segment server view queryRunnerFactoryConglomerate, // query runner factory conglomerate corporation // unionized collective null, // query executor service monitorScheduler, // monitor scheduler new SegmentLoaderFactory( new SegmentLoaderLocalCacheManager( null, new SegmentLoaderConfig() { @Override public List<StorageLocationConfig> getLocations() { return Lists.newArrayList(); } }, new DefaultObjectMapper())), MAPPER, INDEX_MERGER, INDEX_IO, MapCache.create(0), FireDepartmentTest.NO_CACHE_CONFIG); tr = new ThreadPoolTaskRunner(tb, taskConfig, emitter); tq = new TaskQueue(tqc, ts, tr, tac, tl, emitter); tq.start(); }
void processQueue() { Task task; ObjectsMeta meta; Object obj; Class<?> objClass; mProcessing = true; try { while ((task = mTaskQueue.poll()) != null) { obj = task.obj; objClass = obj.getClass(); switch (task.code) { case Task.CODE_REGISTER: { meta = OBJECTS_METAS.get(objClass); if (meta == null) { meta = new ObjectsMeta(obj); OBJECTS_METAS.put(objClass, meta); } meta.registerAtReceivers(obj, mEventSubscribers); meta.registerAtProducers(obj, mEventProducers); try { meta.dispatchEvents(obj, mEventSubscribers, OBJECTS_METAS, mImpl); meta.dispatchEvents(mEventProducers, obj, OBJECTS_METAS, mImpl); } catch (Exception e) { throw handleExceptionOnEventDispatch(e); } break; } case Task.CODE_UNREGISTER: { meta = OBJECTS_METAS.get(objClass); meta.unregisterFromReceivers(obj, mEventSubscribers); meta.unregisterFromProducers(obj, mEventProducers); break; } case Task.CODE_POST: { final HashSet<Object> receivers = mEventSubscribers.get(objClass); if (receivers != null) { Set<SubscriberCallback> subscriberCallback; try { for (Object receiver : receivers) { meta = OBJECTS_METAS.get(receiver.getClass()); subscriberCallback = meta.getEventCallback(objClass); mImpl.dispatchEvent(subscriberCallback, receiver, obj); } } catch (Exception e) { throw handleExceptionOnEventDispatch(e); } } break; } default: throw new IllegalStateException("unexpected task code: " + task.code); } task.recycle(); } } finally { mProcessing = false; } }
@Override public void unregister(Object obj) { assertObjectAndWorkerThread(obj); mTaskQueue.offer(Task.obtainTask(this, Task.CODE_UNREGISTER, obj)); if (!mProcessing) processQueue(); }
@Test public void testPoll() throws InterruptedException { TaskQueue<Object> taskQueue = new TaskQueue<Object>(); Assert.assertNull(taskQueue.poll()); taskQueue = new TaskQueue<Object>(); Object object1 = new Object(); Assert.assertTrue(taskQueue.offer(object1, new boolean[1])); Assert.assertSame(object1, taskQueue.poll()); taskQueue = new TaskQueue<Object>(); object1 = new Object(); Object object2 = new Object(); Assert.assertTrue(taskQueue.offer(object1, new boolean[1])); Assert.assertTrue(taskQueue.offer(object2, new boolean[1])); Assert.assertSame(object1, taskQueue.poll()); taskQueue = new TaskQueue<Object>(); Assert.assertNull(taskQueue.poll(0, TimeUnit.MILLISECONDS)); taskQueue = new TaskQueue<Object>(); Assert.assertNull(taskQueue.poll(-1, TimeUnit.MILLISECONDS)); taskQueue = new TaskQueue<Object>(); Assert.assertNull(taskQueue.poll(100, TimeUnit.MILLISECONDS)); taskQueue = new TaskQueue<Object>(); object1 = new Object(); Assert.assertTrue(taskQueue.offer(object1, new boolean[1])); Assert.assertSame(object1, taskQueue.poll(100, TimeUnit.MILLISECONDS)); taskQueue = new TaskQueue<Object>(); object1 = new Object(); object2 = new Object(); Assert.assertTrue(taskQueue.offer(object1, new boolean[1])); Assert.assertTrue(taskQueue.offer(object2, new boolean[1])); Assert.assertSame(object1, taskQueue.poll(100, TimeUnit.MILLISECONDS)); }
@Test public void testDrainTo() { TaskQueue<Object> taskQueue = new TaskQueue<Object>(); try { taskQueue.drainTo(null); Assert.fail(); } catch (NullPointerException npe) { } Object object1 = new Object(); Object object2 = new Object(); Object object3 = new Object(); Object object4 = new Object(); taskQueue = new TaskQueue<Object>(); Assert.assertTrue(taskQueue.offer(object1, new boolean[1])); Assert.assertTrue(taskQueue.offer(object2, new boolean[1])); Assert.assertTrue(taskQueue.offer(object3, new boolean[1])); Assert.assertTrue(taskQueue.offer(object4, new boolean[1])); Set<Object> set = new HashSet<Object>(); taskQueue.drainTo(set); Assert.assertEquals(4, set.size()); Assert.assertTrue(set.contains(object1)); Assert.assertTrue(set.contains(object2)); Assert.assertTrue(set.contains(object3)); Assert.assertTrue(set.contains(object4)); object1 = new Object(); object2 = new Object(); object3 = new Object(); object4 = new Object(); taskQueue = new TaskQueue<Object>(); Assert.assertTrue(taskQueue.offer(object1, new boolean[1])); Assert.assertTrue(taskQueue.offer(object2, new boolean[1])); Assert.assertTrue(taskQueue.offer(object3, new boolean[1])); Assert.assertTrue(taskQueue.offer(object4, new boolean[1])); List<Object> list = new ArrayList<Object>() { @Override public boolean add(Object e) { if (size() >= 2) { throw new IllegalStateException(); } return super.add(e); } }; try { taskQueue.drainTo(list); Assert.fail(); } catch (IllegalStateException ise) { } Assert.assertEquals(2, list.size()); Assert.assertSame(object1, list.get(0)); Assert.assertSame(object2, list.get(1)); Assert.assertEquals(2, taskQueue.size()); Assert.assertSame(object3, taskQueue.poll()); Assert.assertSame(object4, taskQueue.poll()); }
@Test public void testRemove() { TaskQueue<Object> taskQueue = new TaskQueue<Object>(10); Assert.assertFalse(taskQueue.remove(null)); Assert.assertFalse(taskQueue.remove(new Object())); Object object1 = new Object(); Object object2 = new Object(); Object object3 = new Object(); Assert.assertTrue(taskQueue.offer(object1, new boolean[1])); Assert.assertTrue(taskQueue.offer(object2, new boolean[1])); Assert.assertTrue(taskQueue.offer(object3, new boolean[1])); Assert.assertEquals(3, taskQueue.size()); Assert.assertTrue(taskQueue.remove(object2)); Assert.assertEquals(2, taskQueue.size()); Assert.assertTrue(taskQueue.remove(object1)); Assert.assertEquals(1, taskQueue.size()); Assert.assertTrue(taskQueue.remove(object3)); Assert.assertEquals(0, taskQueue.size()); }