@Override
  public T next() {
    T t;
    boolean hasResult;

    if (!resultsQueue.isEmpty()) {
      t = resultsQueue.poll();
      hasResult = true;
    } else {
      t = null;
      hasResult = false;
    }

    // Always restart when we call next.
    while (!deferQueue.isEmpty()) {
      executorService.submit(deferQueue.poll());
    }

    // If we don't have a result...
    while (!hasResult) {

      // recheck that there is an element to be had and then wait.
      if (!hasNext()) {
        throw new NoSuchElementException();
      }

      try {
        return resultsQueue.take();
      } catch (final InterruptedException e) {
        e.printStackTrace();
      }
    }

    return t;
  }
Beispiel #2
0
    @Override
    public void run() {

      try {
        for (int i = 0; i < otherWorkerIPPort.size(); i++) {

          BlockingQueue<URL> queue = getUrlsForOtherCrawlers().get(i);

          if (queue.isEmpty()) {
            continue;
          }

          StringBuilder contentBuilder = new StringBuilder();
          contentBuilder.append(DispatcherConstants.NEW_URLS_PARAM + "=");

          int items = DispatcherConstants.URLS_TO_SEND;

          while (!queue.isEmpty() || items < 0) {
            URL url = queue.take();
            String cleansedURL = URLEncoder.encode(url.toString(), CrawlerConstants.CHARSET);
            contentBuilder.append(cleansedURL + ";");
          }

          String urlString =
              "http://" + otherWorkerIPPort.get(i) + "/worker/" + DispatcherConstants.ADD_URLS_URL;
          URL url = new URL(urlString);
          DispatcherUtils.sendHttpRequest(
              url, contentBuilder.toString(), DispatcherUtils.Method.POST, true);
        }

      } catch (Exception e) {
        Utils.logStackTrace(e);
      }
    }
 @Override
 public void run() {
   isAppenderThread.set(Boolean.TRUE); // LOG4J2-485
   while (!shutdown) {
     Serializable s;
     try {
       s = queue.take();
       if (s != null && s instanceof String && SHUTDOWN.equals(s.toString())) {
         shutdown = true;
         continue;
       }
     } catch (final InterruptedException ex) {
       break; // LOG4J2-830
     }
     final Log4jLogEvent event = Log4jLogEvent.deserialize(s);
     event.setEndOfBatch(queue.isEmpty());
     final boolean success = callAppenders(event);
     if (!success && errorAppender != null) {
       try {
         errorAppender.callAppender(event);
       } catch (final Exception ex) {
         // Silently accept the error.
       }
     }
   }
   // Process any remaining items in the queue.
   LOGGER.trace(
       "AsyncAppender.AsyncThread shutting down. Processing remaining {} queue events.",
       queue.size());
   int count = 0;
   int ignored = 0;
   while (!queue.isEmpty()) {
     try {
       final Serializable s = queue.take();
       if (Log4jLogEvent.canDeserialize(s)) {
         final Log4jLogEvent event = Log4jLogEvent.deserialize(s);
         event.setEndOfBatch(queue.isEmpty());
         callAppenders(event);
         count++;
       } else {
         ignored++;
         LOGGER.trace("Ignoring event of class {}", s.getClass().getName());
       }
     } catch (final InterruptedException ex) {
       // May have been interrupted to shut down.
       // Here we ignore interrupts and try to process all remaining events.
     }
   }
   LOGGER.trace(
       "AsyncAppender.AsyncThread stopped. Queue has {} events remaining. "
           + "Processed {} and ignored {} events since shutdown started.",
       queue.size(),
       count,
       ignored);
 }
 @Override
 public void run() {
   while (!shutdown) {
     LogEvent event;
     try {
       event = queue.take();
       if (event == SHUTDOWN) {
         shutdown = true;
         continue;
       }
     } catch (final InterruptedException ex) {
       break; // LOG4J2-830
     }
     event.setEndOfBatch(queue.isEmpty());
     final boolean success = callAppenders(event);
     if (!success && errorAppender != null) {
       try {
         errorAppender.callAppender(event);
       } catch (final Exception ex) {
         // Silently accept the error.
       }
     }
   }
   // Process any remaining items in the queue.
   LOGGER.trace(
       "AsyncAppender.AsyncThread shutting down. Processing remaining {} queue events.",
       queue.size());
   int count = 0;
   int ignored = 0;
   while (!queue.isEmpty()) {
     try {
       final LogEvent event = queue.take();
       if (event instanceof Log4jLogEvent) {
         final Log4jLogEvent logEvent = (Log4jLogEvent) event;
         logEvent.setEndOfBatch(queue.isEmpty());
         callAppenders(logEvent);
         count++;
       } else {
         ignored++;
         LOGGER.trace("Ignoring event of class {}", event.getClass().getName());
       }
     } catch (final InterruptedException ex) {
       // May have been interrupted to shut down.
       // Here we ignore interrupts and try to process all remaining events.
     }
   }
   LOGGER.trace(
       "AsyncAppender.AsyncThread stopped. Queue has {} events remaining. "
           + "Processed {} and ignored {} events since shutdown started.",
       queue.size(),
       count,
       ignored);
 }
Beispiel #5
0
 public void WaitQueueFinished() {
   while (true) {
     if (queue.isEmpty() && queueResponse.isEmpty()) {
       break;
     }
     try {
       Thread.sleep(10);
     } catch (InterruptedException ex) {
       Logger.getLogger(QCmdProcessor.class.getName()).log(Level.SEVERE, null, ex);
     }
   }
 }
 void checkEmpty(BlockingQueue q) {
   try {
     assertTrue(q.isEmpty());
     assertEquals(0, q.size());
     assertNull(q.peek());
     assertNull(q.poll());
     assertNull(q.poll(0, MILLISECONDS));
     assertEquals("[]", q.toString());
     assertTrue(Arrays.equals(q.toArray(), new Object[0]));
     assertFalse(q.iterator().hasNext());
     try {
       q.element();
       shouldThrow();
     } catch (NoSuchElementException success) {
     }
     try {
       q.iterator().next();
       shouldThrow();
     } catch (NoSuchElementException success) {
     }
     try {
       q.remove();
       shouldThrow();
     } catch (NoSuchElementException success) {
     }
   } catch (InterruptedException ie) {
     threadUnexpectedException(ie);
   }
 }
 private void checkProblemFactChanges() {
   BlockingQueue<ProblemFactChange> problemFactChangeQueue =
       basicPlumbingTermination.getProblemFactChangeQueue();
   if (!problemFactChangeQueue.isEmpty()) {
     solverScope.setRestartSolver(true);
     solverScope.setWorkingSolutionFromBestSolution();
     Score score = null;
     int count = 0;
     ProblemFactChange problemFactChange = problemFactChangeQueue.poll();
     while (problemFactChange != null) {
       score = doProblemFactChange(problemFactChange);
       count++;
       problemFactChange = problemFactChangeQueue.poll();
     }
     Solution newBestSolution = solverScope.getScoreDirector().cloneWorkingSolution();
     // TODO BestSolutionRecaller.solverStarted() already calls countUninitializedVariables()
     int newBestUninitializedVariableCount =
         solverScope.getSolutionDescriptor().countUninitializedVariables(newBestSolution);
     bestSolutionRecaller.updateBestSolution(
         solverScope, newBestSolution, newBestUninitializedVariableCount);
     logger.info(
         "Done {} ProblemFactChange(s): new score ({}) possibly uninitialized. Restarting solver.",
         count,
         score);
   }
 }
 /*
  * (non-Javadoc)
  *
  * @see java.lang.Runnable#run()
  */
 public void run() {
   _run:
   while (!stop) {
     Object in = inQueue.poll();
     /* find a transition with this label */
     super.input(in);
     Object out = super.output();
     if (out == null) // nothing to do
     synchronized (this) {
         do {
           try {
             wait(200);
           } catch (InterruptedException e) {
             e.printStackTrace();
           }
           out = super.output();
         } while (inQueue.isEmpty() && out == null);
       }
     if (out != null) {
       try {
         outQueue.put(out);
       } catch (InterruptedException e) {
         e.printStackTrace();
       }
     }
   }
 }
  @Test(timeout = 60000)
  public void testAppRejectionWithCancelledDelegationToken() throws Exception {
    MyFS dfs = (MyFS) FileSystem.get(conf);
    LOG.info("dfs=" + (Object) dfs.hashCode() + ";conf=" + conf.hashCode());

    MyToken token = dfs.getDelegationToken(new Text("user1"));
    token.cancelToken();

    Credentials ts = new Credentials();
    ts.addToken(token.getKind(), token);

    // register the tokens for renewal
    ApplicationId appId = BuilderUtils.newApplicationId(0, 0);
    delegationTokenRenewer.addApplication(appId, ts, true, false);
    int waitCnt = 20;
    while (waitCnt-- > 0) {
      if (!eventQueue.isEmpty()) {
        Event evt = eventQueue.take();
        if (evt.getType() == RMAppEventType.APP_REJECTED) {
          Assert.assertTrue(((RMAppEvent) evt).getApplicationId().equals(appId));
          return;
        }
      } else {
        Thread.sleep(500);
      }
    }
    fail("App submission with a cancelled token should have failed");
  }
Beispiel #10
0
  private void handleScheduleAttemptedRejection(TaskWrapper taskWrapper) {
    if (enablePreemption
        && taskWrapper.getTaskRunnerCallable().canFinish()
        && !preemptionQueue.isEmpty()) {

      if (isDebugEnabled) {
        LOG.debug("Preemption Queue: " + preemptionQueue);
      }

      TaskWrapper pRequest = removeAndGetFromPreemptionQueue();

      // Avoid preempting tasks which are finishable - callback still to be processed.
      if (pRequest != null) {
        if (pRequest.getTaskRunnerCallable().canFinish()) {
          LOG.info(
              "Removed {} from preemption queue, but not preempting since it's now finishable",
              pRequest.getRequestId());
        } else {
          if (isInfoEnabled) {
            LOG.info(
                "Invoking kill task for {} due to pre-emption to run {}",
                pRequest.getRequestId(),
                taskWrapper.getRequestId());
          }
          // The task will either be killed or is already in the process of completing, which will
          // trigger the next scheduling run, or result in available slots being higher than 0,
          // which will cause the scheduler loop to continue.
          pRequest.getTaskRunnerCallable().killTask();
        }
      }
    }
  }
Beispiel #11
0
    public void run() {
      Throwable t = null;
      while (canRun()) {
        Buffer data = null;
        try {
          data = send_queue.take();
          if (data.hashCode() == termination.hashCode()) break;
        } catch (InterruptedException e) {
          t = e;
          break;
        }

        if (data != null) {
          try {
            _send(data.getBuf(), 0, data.getLength(), false, send_queue.isEmpty());
          } catch (Throwable ignored) {
            t = ignored;
          }
        }
      }
      server.notifyConnectionClosed(
          TcpConnection.this,
          String.format(
              "%s: %s", getClass().getSimpleName(), t != null ? t.toString() : "normal stop"));
    }
 public void writeToDB() throws InterruptedException, SQLException {
   while (!reactants.isEmpty() || FileParserObserver.INSTANCE.getFileParserStatus()) {
     Reactant reactant = getRecord();
     logManager.writeLog("DBWriter", reactant.getUniqueID());
     executePreparedStatement(reactant);
   }
 }
  @Listener
  public void onGamePreInitializationEvent(GamePreInitializationEvent event) {
    if (game.getPlatform().getType() == Platform.Type.SERVER) {

      try {
        game.getServiceManager()
            .setProvider(this, IRegistrationService.class, new SimpleRegistrationService());
      } catch (IllegalArgumentException e) {
        e.printStackTrace();
      }

      game.getScheduler()
          .createTaskBuilder()
          .execute(
              (task) -> {
                try {
                  if (!requestQueue.isEmpty()) requestQueue.take().run();
                } catch (InterruptedException e) {
                  e.printStackTrace();
                }
              })
          .name("FDS-WSAPI - CheckRequestsTask")
          .delayTicks(50)
          .interval(1, TimeUnit.MILLISECONDS)
          .submit(this);
    }
  }
  /**
   * Basic idea of the test: 0. Setup token KEEP_ALIVE 1. create tokens. 2. register them for
   * renewal - to be cancelled on app complete 3. Complete app. 4. Verify token is alive within the
   * KEEP_ALIVE time 5. Verify token has been cancelled after the KEEP_ALIVE_TIME
   *
   * @throws IOException
   * @throws URISyntaxException
   */
  @Test(timeout = 60000)
  public void testDTKeepAlive1() throws Exception {
    Configuration lconf = new Configuration(conf);
    lconf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
    // Keep tokens alive for 6 seconds.
    lconf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 6000l);
    // Try removing tokens every second.
    lconf.setLong(YarnConfiguration.RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS, 1000l);
    DelegationTokenRenewer localDtr = createNewDelegationTokenRenewer(lconf, counter);
    localDtr.init(lconf);
    RMContext mockContext = mock(RMContext.class);
    ClientRMService mockClientRMService = mock(ClientRMService.class);
    when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
    when(mockContext.getDelegationTokenRenewer()).thenReturn(localDtr);
    when(mockContext.getDispatcher()).thenReturn(dispatcher);
    InetSocketAddress sockAddr = InetSocketAddress.createUnresolved("localhost", 1234);
    when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
    localDtr.setRMContext(mockContext);
    localDtr.start();

    MyFS dfs = (MyFS) FileSystem.get(lconf);
    LOG.info("dfs=" + (Object) dfs.hashCode() + ";conf=" + lconf.hashCode());

    Credentials ts = new Credentials();
    // get the delegation tokens
    MyToken token1 = dfs.getDelegationToken(new Text("user1"));

    String nn1 = DelegationTokenRenewer.SCHEME + "://host1:0";
    ts.addToken(new Text(nn1), token1);

    // register the tokens for renewal
    ApplicationId applicationId_0 = BuilderUtils.newApplicationId(0, 0);
    localDtr.addApplication(applicationId_0, ts, true, false);
    waitForEventsToGetProcessed(localDtr);
    if (!eventQueue.isEmpty()) {
      Event evt = eventQueue.take();
      if (evt instanceof RMAppEvent) {
        Assert.assertEquals(((RMAppEvent) evt).getType(), RMAppEventType.START);
      } else {
        fail("RMAppEvent.START was expected!!");
      }
    }

    localDtr.applicationFinished(applicationId_0);
    waitForEventsToGetProcessed(localDtr);

    // Token should still be around. Renewal should not fail.
    token1.renew(lconf);

    // Allow the keepalive time to run out
    Thread.sleep(10000l);

    // The token should have been cancelled at this point. Renewal will fail.
    try {
      token1.renew(lconf);
      fail("Renewal of cancelled token should have failed");
    } catch (InvalidToken ite) {
    }
  }
  /**
   * Check if a new result should be expected and fetch with a call to {@link #next()}.
   *
   * <p>This first checks if the internal queue has a cached result, and returns true if it does.
   *
   * <p>Otherwise, each actor is checked if it still has results to fetch. If any returns true, this
   * returns true.
   *
   * <p>Finally, the results queue is re-checked in case an actor published a result between the
   * time we first checked the results queue and then checked each actor individually.
   *
   * @return True if there are more elements to fetch or retrieve out of the cache queue. False
   *     otherwise.
   */
  @Override
  public boolean hasNext() {
    // This is fast, so we opt to try this first.
    if (!resultsQueue.isEmpty()) {
      return true;
    }

    // If there are no results in the queue, check the actors. Are there any pending results?
    for (final Actor a : actors) {
      if (a.hasNext()) {
        return true;
      }
    }

    // If the producers (actors) are empty, recheck the results queue.
    return !resultsQueue.isEmpty();
  }
 @Override
 String getQueueStatus() {
   BlockingQueue<Runnable> q = queue;
   if (q.isEmpty()) {
     return "empty";
   } else {
     return "size: " + q.size();
   }
 }
Beispiel #17
0
 @Override
 public void execute() throws Throwable {
   long start = System.currentTimeMillis();
   List<TestBatch> testBatches = Lists.newArrayList();
   for (TestBatch batch : testBatchSupplier.get()) {
     testBatches.add(batch);
     if (batch.isParallel()) {
       parallelWorkQueue.add(batch);
     } else {
       isolatedWorkQueue.add(batch);
     }
   }
   try {
     int expectedNumHosts = hostExecutors.size();
     initalizeHosts();
     do {
       replaceBadHosts(expectedNumHosts);
       List<ListenableFuture<Void>> results = Lists.newArrayList();
       for (HostExecutor hostExecutor : ImmutableList.copyOf(hostExecutors)) {
         results.add(
             hostExecutor.submitTests(parallelWorkQueue, isolatedWorkQueue, failedTestResults));
       }
       Futures.allAsList(results).get();
     } while (!(parallelWorkQueue.isEmpty() && isolatedWorkQueue.isEmpty()));
     for (TestBatch batch : testBatches) {
       File batchLogDir;
       if (failedTestResults.contains(batch)) {
         batchLogDir = new File(failedLogDir, batch.getName());
       } else {
         batchLogDir = new File(succeededLogDir, batch.getName());
       }
       JUnitReportParser parser = new JUnitReportParser(logger, batchLogDir);
       executedTests.addAll(parser.getExecutedTests());
       failedTests.addAll(parser.getFailedTests());
     }
   } finally {
     long elapsed = System.currentTimeMillis() - start;
     logger.info(
         "PERF: exec phase "
             + TimeUnit.MINUTES.convert(elapsed, TimeUnit.MILLISECONDS)
             + " minutes");
   }
 }
  public void testInputStreamWithNoFIXMessages() throws Exception {

    String noFIXMessages = "No\nMessages Are\nIn This\nStream";
    InputStream is = new ByteArrayInputStream(noFIXMessages.getBytes());
    BlockingQueue<String> queue = new LinkedBlockingQueue<String>();

    parseAndWait(is, queue);
    assertFalse(queue.isEmpty());
    assertEquals(1, queue.size());
    assertEquals("DONE", queue.take());
  }
  public void testIncompleteFIXMessage() throws Exception {
    String message = "8=FIX.";

    InputStream is = new ByteArrayInputStream(message.getBytes());
    BlockingQueue<String> queue = new LinkedBlockingQueue<String>();

    parseAndWait(is, queue);
    assertFalse(queue.isEmpty());
    assertEquals(1, queue.size());
    assertEquals("DONE", queue.take());
  }
  public void testFIXMessageWithNonStandardOneCharacterDelimeter() throws Exception {
    String oneMessage = "8=FIX.4.2|9=456|35=D|10=123";
    InputStream is = new ByteArrayInputStream(oneMessage.getBytes());
    BlockingQueue<String> queue = new LinkedBlockingQueue<String>();

    parseAndWait(is, queue);
    assertFalse(queue.isEmpty());
    assertEquals(2, queue.size());
    assertEquals(oneMessage.replaceAll("\\|", "\u0001") + "\u0001", queue.take());
    assertEquals("DONE", queue.take());
  }
 public void shutdown() {
   shutdown = true;
   if (queue.isEmpty()) {
     queue.offer(SHUTDOWN);
   }
   if (getState() == State.TIMED_WAITING || getState() == State.WAITING) {
     this
         .interrupt(); // LOG4J2-1422: if underlying appender is stuck in wait/sleep/join/park
                       // call
   }
 }
 public int updateClientFileList() {
   if (client.isUpdateFileList() && !fileListQueue.isEmpty()) {
     try {
       FileInfo file = fileListQueue.take();
       return nextFile(file);
     } catch (InterruptedException ex) {
       Logger.getLogger(SocketChannelHandler.class.getName()).log(Level.SEVERE, null, ex);
     }
   }
   return 0;
 }
  public void testInputStreamWithOneFIXMessage() throws Exception {
    String message = "8=FIX.4.2\u00019=456\u000135=D\u000110=123\u0001";
    InputStream is = new ByteArrayInputStream(message.getBytes());
    BlockingQueue<String> queue = new LinkedBlockingQueue<String>();

    parseAndWait(is, queue);
    assertFalse(queue.isEmpty());
    assertEquals(2, queue.size());
    assertEquals(message, queue.take());
    assertEquals("DONE", queue.take());
  }
  /**
   * The parser should attempt to "fix" the FIX message. In this case, the message is missing the
   * last SOH character.
   */
  public void testFIXMessageThatDoesNotEndWithSOH() throws Exception {
    String oneMessage = "8=FIX.4.2\u00019=456\u000135=D\u000110=123";
    InputStream is = new ByteArrayInputStream(oneMessage.getBytes());
    BlockingQueue<String> queue = new LinkedBlockingQueue<String>();

    parseAndWait(is, queue);
    assertFalse(queue.isEmpty());
    assertEquals(2, queue.size());
    assertEquals(oneMessage + "\u0001", queue.take());
    assertEquals("DONE", queue.take());
  }
  @Override
  public void run() {
    try {
      while (!s1.isEmpty()) {
        s2.put(fib(s1.poll()));
      }
    } catch (InterruptedException ex) {
      Logger.getLogger(Producer.class.getName()).log(Level.SEVERE, null, ex);
    }

    System.out.println("Denne producer tråd er done. ID: " + getName());
  }
 @Override
 public boolean hasNext() throws IOException, CollectionException {
   try {
     if (documentQueue.isEmpty()) logger.info("Waiting for a new document.");
     currentDoc = documentQueue.take();
     if (currentDoc == CollectionDocument.LAST_DOCUMENT) return false;
     else return true;
   } catch (InterruptedException e) {
     logger.info("Stream {} interrupted", this.streamName);
     return false;
   }
 }
Beispiel #27
0
 public void shutdownPool() {
   while (!jobQueue.isEmpty()) {
     try {
       Thread.sleep(1000);
     } catch (InterruptedException e) {
       e.printStackTrace();
     }
   }
   shutdown = true;
   for (Thread workerThread : jobThreads) {
     workerThread.interrupt();
   }
 }
  /** Returns a response to satisfy {@code request}. */
  private MockResponse dispatch(RecordedRequest request) throws InterruptedException {
    if (responseQueue.isEmpty()) {
      throw new IllegalStateException("Unexpected request: " + request);
    }

    if (singleResponse) {
      return responseQueue.peek();
    } else {
      requestCount.incrementAndGet();
      requestQueue.add(request);
      return responseQueue.take();
    }
  }
    public void run() {
      try {
        while (!s1.isEmpty()) {
          int n;
          n = s1.poll();
          long result = fib(n);
          s2.put(result);
        }

      } catch (InterruptedException ex) {
        Logger.getLogger(Fibonacci.class.getName()).log(Level.SEVERE, null, ex);
      }
    }
  public void testFIXMessageWithJunkAtTheBeginningOfTheString() throws Exception {
    String junkInTheTrunkMessage = "JunkInTheTrunk";
    String message = "8=FIX.4.2|9=456|35=D|10=123|";

    InputStream is = new ByteArrayInputStream((junkInTheTrunkMessage + message).getBytes());
    BlockingQueue<String> queue = new LinkedBlockingQueue<String>();

    parseAndWait(is, queue);
    assertFalse(queue.isEmpty());
    assertEquals(2, queue.size());
    assertEquals(message.replaceAll("\\|", "\u0001"), queue.take());
    assertEquals("DONE", queue.take());
  }