@Override
public T next() {
T t;
boolean hasResult;
if (!resultsQueue.isEmpty()) {
t = resultsQueue.poll();
hasResult = true;
} else {
t = null;
hasResult = false;
}
// Always restart when we call next.
while (!deferQueue.isEmpty()) {
executorService.submit(deferQueue.poll());
}
// If we don't have a result...
while (!hasResult) {
// recheck that there is an element to be had and then wait.
if (!hasNext()) {
throw new NoSuchElementException();
}
try {
return resultsQueue.take();
} catch (final InterruptedException e) {
e.printStackTrace();
}
}
return t;
}
@Override
public void run() {
try {
for (int i = 0; i < otherWorkerIPPort.size(); i++) {
BlockingQueue<URL> queue = getUrlsForOtherCrawlers().get(i);
if (queue.isEmpty()) {
continue;
}
StringBuilder contentBuilder = new StringBuilder();
contentBuilder.append(DispatcherConstants.NEW_URLS_PARAM + "=");
int items = DispatcherConstants.URLS_TO_SEND;
while (!queue.isEmpty() || items < 0) {
URL url = queue.take();
String cleansedURL = URLEncoder.encode(url.toString(), CrawlerConstants.CHARSET);
contentBuilder.append(cleansedURL + ";");
}
String urlString =
"http://" + otherWorkerIPPort.get(i) + "/worker/" + DispatcherConstants.ADD_URLS_URL;
URL url = new URL(urlString);
DispatcherUtils.sendHttpRequest(
url, contentBuilder.toString(), DispatcherUtils.Method.POST, true);
}
} catch (Exception e) {
Utils.logStackTrace(e);
}
}
@Override
public void run() {
isAppenderThread.set(Boolean.TRUE); // LOG4J2-485
while (!shutdown) {
Serializable s;
try {
s = queue.take();
if (s != null && s instanceof String && SHUTDOWN.equals(s.toString())) {
shutdown = true;
continue;
}
} catch (final InterruptedException ex) {
break; // LOG4J2-830
}
final Log4jLogEvent event = Log4jLogEvent.deserialize(s);
event.setEndOfBatch(queue.isEmpty());
final boolean success = callAppenders(event);
if (!success && errorAppender != null) {
try {
errorAppender.callAppender(event);
} catch (final Exception ex) {
// Silently accept the error.
}
}
}
// Process any remaining items in the queue.
LOGGER.trace(
"AsyncAppender.AsyncThread shutting down. Processing remaining {} queue events.",
queue.size());
int count = 0;
int ignored = 0;
while (!queue.isEmpty()) {
try {
final Serializable s = queue.take();
if (Log4jLogEvent.canDeserialize(s)) {
final Log4jLogEvent event = Log4jLogEvent.deserialize(s);
event.setEndOfBatch(queue.isEmpty());
callAppenders(event);
count++;
} else {
ignored++;
LOGGER.trace("Ignoring event of class {}", s.getClass().getName());
}
} catch (final InterruptedException ex) {
// May have been interrupted to shut down.
// Here we ignore interrupts and try to process all remaining events.
}
}
LOGGER.trace(
"AsyncAppender.AsyncThread stopped. Queue has {} events remaining. "
+ "Processed {} and ignored {} events since shutdown started.",
queue.size(),
count,
ignored);
}
@Override
public void run() {
while (!shutdown) {
LogEvent event;
try {
event = queue.take();
if (event == SHUTDOWN) {
shutdown = true;
continue;
}
} catch (final InterruptedException ex) {
break; // LOG4J2-830
}
event.setEndOfBatch(queue.isEmpty());
final boolean success = callAppenders(event);
if (!success && errorAppender != null) {
try {
errorAppender.callAppender(event);
} catch (final Exception ex) {
// Silently accept the error.
}
}
}
// Process any remaining items in the queue.
LOGGER.trace(
"AsyncAppender.AsyncThread shutting down. Processing remaining {} queue events.",
queue.size());
int count = 0;
int ignored = 0;
while (!queue.isEmpty()) {
try {
final LogEvent event = queue.take();
if (event instanceof Log4jLogEvent) {
final Log4jLogEvent logEvent = (Log4jLogEvent) event;
logEvent.setEndOfBatch(queue.isEmpty());
callAppenders(logEvent);
count++;
} else {
ignored++;
LOGGER.trace("Ignoring event of class {}", event.getClass().getName());
}
} catch (final InterruptedException ex) {
// May have been interrupted to shut down.
// Here we ignore interrupts and try to process all remaining events.
}
}
LOGGER.trace(
"AsyncAppender.AsyncThread stopped. Queue has {} events remaining. "
+ "Processed {} and ignored {} events since shutdown started.",
queue.size(),
count,
ignored);
}
public void WaitQueueFinished() {
while (true) {
if (queue.isEmpty() && queueResponse.isEmpty()) {
break;
}
try {
Thread.sleep(10);
} catch (InterruptedException ex) {
Logger.getLogger(QCmdProcessor.class.getName()).log(Level.SEVERE, null, ex);
}
}
}
void checkEmpty(BlockingQueue q) {
try {
assertTrue(q.isEmpty());
assertEquals(0, q.size());
assertNull(q.peek());
assertNull(q.poll());
assertNull(q.poll(0, MILLISECONDS));
assertEquals("[]", q.toString());
assertTrue(Arrays.equals(q.toArray(), new Object[0]));
assertFalse(q.iterator().hasNext());
try {
q.element();
shouldThrow();
} catch (NoSuchElementException success) {
}
try {
q.iterator().next();
shouldThrow();
} catch (NoSuchElementException success) {
}
try {
q.remove();
shouldThrow();
} catch (NoSuchElementException success) {
}
} catch (InterruptedException ie) {
threadUnexpectedException(ie);
}
}
private void checkProblemFactChanges() {
BlockingQueue<ProblemFactChange> problemFactChangeQueue =
basicPlumbingTermination.getProblemFactChangeQueue();
if (!problemFactChangeQueue.isEmpty()) {
solverScope.setRestartSolver(true);
solverScope.setWorkingSolutionFromBestSolution();
Score score = null;
int count = 0;
ProblemFactChange problemFactChange = problemFactChangeQueue.poll();
while (problemFactChange != null) {
score = doProblemFactChange(problemFactChange);
count++;
problemFactChange = problemFactChangeQueue.poll();
}
Solution newBestSolution = solverScope.getScoreDirector().cloneWorkingSolution();
// TODO BestSolutionRecaller.solverStarted() already calls countUninitializedVariables()
int newBestUninitializedVariableCount =
solverScope.getSolutionDescriptor().countUninitializedVariables(newBestSolution);
bestSolutionRecaller.updateBestSolution(
solverScope, newBestSolution, newBestUninitializedVariableCount);
logger.info(
"Done {} ProblemFactChange(s): new score ({}) possibly uninitialized. Restarting solver.",
count,
score);
}
}
/*
* (non-Javadoc)
*
* @see java.lang.Runnable#run()
*/
public void run() {
_run:
while (!stop) {
Object in = inQueue.poll();
/* find a transition with this label */
super.input(in);
Object out = super.output();
if (out == null) // nothing to do
synchronized (this) {
do {
try {
wait(200);
} catch (InterruptedException e) {
e.printStackTrace();
}
out = super.output();
} while (inQueue.isEmpty() && out == null);
}
if (out != null) {
try {
outQueue.put(out);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
}
@Test(timeout = 60000)
public void testAppRejectionWithCancelledDelegationToken() throws Exception {
MyFS dfs = (MyFS) FileSystem.get(conf);
LOG.info("dfs=" + (Object) dfs.hashCode() + ";conf=" + conf.hashCode());
MyToken token = dfs.getDelegationToken(new Text("user1"));
token.cancelToken();
Credentials ts = new Credentials();
ts.addToken(token.getKind(), token);
// register the tokens for renewal
ApplicationId appId = BuilderUtils.newApplicationId(0, 0);
delegationTokenRenewer.addApplication(appId, ts, true, false);
int waitCnt = 20;
while (waitCnt-- > 0) {
if (!eventQueue.isEmpty()) {
Event evt = eventQueue.take();
if (evt.getType() == RMAppEventType.APP_REJECTED) {
Assert.assertTrue(((RMAppEvent) evt).getApplicationId().equals(appId));
return;
}
} else {
Thread.sleep(500);
}
}
fail("App submission with a cancelled token should have failed");
}
private void handleScheduleAttemptedRejection(TaskWrapper taskWrapper) {
if (enablePreemption
&& taskWrapper.getTaskRunnerCallable().canFinish()
&& !preemptionQueue.isEmpty()) {
if (isDebugEnabled) {
LOG.debug("Preemption Queue: " + preemptionQueue);
}
TaskWrapper pRequest = removeAndGetFromPreemptionQueue();
// Avoid preempting tasks which are finishable - callback still to be processed.
if (pRequest != null) {
if (pRequest.getTaskRunnerCallable().canFinish()) {
LOG.info(
"Removed {} from preemption queue, but not preempting since it's now finishable",
pRequest.getRequestId());
} else {
if (isInfoEnabled) {
LOG.info(
"Invoking kill task for {} due to pre-emption to run {}",
pRequest.getRequestId(),
taskWrapper.getRequestId());
}
// The task will either be killed or is already in the process of completing, which will
// trigger the next scheduling run, or result in available slots being higher than 0,
// which will cause the scheduler loop to continue.
pRequest.getTaskRunnerCallable().killTask();
}
}
}
}
public void run() {
Throwable t = null;
while (canRun()) {
Buffer data = null;
try {
data = send_queue.take();
if (data.hashCode() == termination.hashCode()) break;
} catch (InterruptedException e) {
t = e;
break;
}
if (data != null) {
try {
_send(data.getBuf(), 0, data.getLength(), false, send_queue.isEmpty());
} catch (Throwable ignored) {
t = ignored;
}
}
}
server.notifyConnectionClosed(
TcpConnection.this,
String.format(
"%s: %s", getClass().getSimpleName(), t != null ? t.toString() : "normal stop"));
}
public void writeToDB() throws InterruptedException, SQLException {
while (!reactants.isEmpty() || FileParserObserver.INSTANCE.getFileParserStatus()) {
Reactant reactant = getRecord();
logManager.writeLog("DBWriter", reactant.getUniqueID());
executePreparedStatement(reactant);
}
}
@Listener
public void onGamePreInitializationEvent(GamePreInitializationEvent event) {
if (game.getPlatform().getType() == Platform.Type.SERVER) {
try {
game.getServiceManager()
.setProvider(this, IRegistrationService.class, new SimpleRegistrationService());
} catch (IllegalArgumentException e) {
e.printStackTrace();
}
game.getScheduler()
.createTaskBuilder()
.execute(
(task) -> {
try {
if (!requestQueue.isEmpty()) requestQueue.take().run();
} catch (InterruptedException e) {
e.printStackTrace();
}
})
.name("FDS-WSAPI - CheckRequestsTask")
.delayTicks(50)
.interval(1, TimeUnit.MILLISECONDS)
.submit(this);
}
}
/**
* Basic idea of the test: 0. Setup token KEEP_ALIVE 1. create tokens. 2. register them for
* renewal - to be cancelled on app complete 3. Complete app. 4. Verify token is alive within the
* KEEP_ALIVE time 5. Verify token has been cancelled after the KEEP_ALIVE_TIME
*
* @throws IOException
* @throws URISyntaxException
*/
@Test(timeout = 60000)
public void testDTKeepAlive1() throws Exception {
Configuration lconf = new Configuration(conf);
lconf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, true);
// Keep tokens alive for 6 seconds.
lconf.setLong(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 6000l);
// Try removing tokens every second.
lconf.setLong(YarnConfiguration.RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS, 1000l);
DelegationTokenRenewer localDtr = createNewDelegationTokenRenewer(lconf, counter);
localDtr.init(lconf);
RMContext mockContext = mock(RMContext.class);
ClientRMService mockClientRMService = mock(ClientRMService.class);
when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
when(mockContext.getDelegationTokenRenewer()).thenReturn(localDtr);
when(mockContext.getDispatcher()).thenReturn(dispatcher);
InetSocketAddress sockAddr = InetSocketAddress.createUnresolved("localhost", 1234);
when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
localDtr.setRMContext(mockContext);
localDtr.start();
MyFS dfs = (MyFS) FileSystem.get(lconf);
LOG.info("dfs=" + (Object) dfs.hashCode() + ";conf=" + lconf.hashCode());
Credentials ts = new Credentials();
// get the delegation tokens
MyToken token1 = dfs.getDelegationToken(new Text("user1"));
String nn1 = DelegationTokenRenewer.SCHEME + "://host1:0";
ts.addToken(new Text(nn1), token1);
// register the tokens for renewal
ApplicationId applicationId_0 = BuilderUtils.newApplicationId(0, 0);
localDtr.addApplication(applicationId_0, ts, true, false);
waitForEventsToGetProcessed(localDtr);
if (!eventQueue.isEmpty()) {
Event evt = eventQueue.take();
if (evt instanceof RMAppEvent) {
Assert.assertEquals(((RMAppEvent) evt).getType(), RMAppEventType.START);
} else {
fail("RMAppEvent.START was expected!!");
}
}
localDtr.applicationFinished(applicationId_0);
waitForEventsToGetProcessed(localDtr);
// Token should still be around. Renewal should not fail.
token1.renew(lconf);
// Allow the keepalive time to run out
Thread.sleep(10000l);
// The token should have been cancelled at this point. Renewal will fail.
try {
token1.renew(lconf);
fail("Renewal of cancelled token should have failed");
} catch (InvalidToken ite) {
}
}
/**
* Check if a new result should be expected and fetch with a call to {@link #next()}.
*
* <p>This first checks if the internal queue has a cached result, and returns true if it does.
*
* <p>Otherwise, each actor is checked if it still has results to fetch. If any returns true, this
* returns true.
*
* <p>Finally, the results queue is re-checked in case an actor published a result between the
* time we first checked the results queue and then checked each actor individually.
*
* @return True if there are more elements to fetch or retrieve out of the cache queue. False
* otherwise.
*/
@Override
public boolean hasNext() {
// This is fast, so we opt to try this first.
if (!resultsQueue.isEmpty()) {
return true;
}
// If there are no results in the queue, check the actors. Are there any pending results?
for (final Actor a : actors) {
if (a.hasNext()) {
return true;
}
}
// If the producers (actors) are empty, recheck the results queue.
return !resultsQueue.isEmpty();
}
@Override
String getQueueStatus() {
BlockingQueue<Runnable> q = queue;
if (q.isEmpty()) {
return "empty";
} else {
return "size: " + q.size();
}
}
@Override
public void execute() throws Throwable {
long start = System.currentTimeMillis();
List<TestBatch> testBatches = Lists.newArrayList();
for (TestBatch batch : testBatchSupplier.get()) {
testBatches.add(batch);
if (batch.isParallel()) {
parallelWorkQueue.add(batch);
} else {
isolatedWorkQueue.add(batch);
}
}
try {
int expectedNumHosts = hostExecutors.size();
initalizeHosts();
do {
replaceBadHosts(expectedNumHosts);
List<ListenableFuture<Void>> results = Lists.newArrayList();
for (HostExecutor hostExecutor : ImmutableList.copyOf(hostExecutors)) {
results.add(
hostExecutor.submitTests(parallelWorkQueue, isolatedWorkQueue, failedTestResults));
}
Futures.allAsList(results).get();
} while (!(parallelWorkQueue.isEmpty() && isolatedWorkQueue.isEmpty()));
for (TestBatch batch : testBatches) {
File batchLogDir;
if (failedTestResults.contains(batch)) {
batchLogDir = new File(failedLogDir, batch.getName());
} else {
batchLogDir = new File(succeededLogDir, batch.getName());
}
JUnitReportParser parser = new JUnitReportParser(logger, batchLogDir);
executedTests.addAll(parser.getExecutedTests());
failedTests.addAll(parser.getFailedTests());
}
} finally {
long elapsed = System.currentTimeMillis() - start;
logger.info(
"PERF: exec phase "
+ TimeUnit.MINUTES.convert(elapsed, TimeUnit.MILLISECONDS)
+ " minutes");
}
}
public void testInputStreamWithNoFIXMessages() throws Exception {
String noFIXMessages = "No\nMessages Are\nIn This\nStream";
InputStream is = new ByteArrayInputStream(noFIXMessages.getBytes());
BlockingQueue<String> queue = new LinkedBlockingQueue<String>();
parseAndWait(is, queue);
assertFalse(queue.isEmpty());
assertEquals(1, queue.size());
assertEquals("DONE", queue.take());
}
public void testIncompleteFIXMessage() throws Exception {
String message = "8=FIX.";
InputStream is = new ByteArrayInputStream(message.getBytes());
BlockingQueue<String> queue = new LinkedBlockingQueue<String>();
parseAndWait(is, queue);
assertFalse(queue.isEmpty());
assertEquals(1, queue.size());
assertEquals("DONE", queue.take());
}
public void testFIXMessageWithNonStandardOneCharacterDelimeter() throws Exception {
String oneMessage = "8=FIX.4.2|9=456|35=D|10=123";
InputStream is = new ByteArrayInputStream(oneMessage.getBytes());
BlockingQueue<String> queue = new LinkedBlockingQueue<String>();
parseAndWait(is, queue);
assertFalse(queue.isEmpty());
assertEquals(2, queue.size());
assertEquals(oneMessage.replaceAll("\\|", "\u0001") + "\u0001", queue.take());
assertEquals("DONE", queue.take());
}
public void shutdown() {
shutdown = true;
if (queue.isEmpty()) {
queue.offer(SHUTDOWN);
}
if (getState() == State.TIMED_WAITING || getState() == State.WAITING) {
this
.interrupt(); // LOG4J2-1422: if underlying appender is stuck in wait/sleep/join/park
// call
}
}
public int updateClientFileList() {
if (client.isUpdateFileList() && !fileListQueue.isEmpty()) {
try {
FileInfo file = fileListQueue.take();
return nextFile(file);
} catch (InterruptedException ex) {
Logger.getLogger(SocketChannelHandler.class.getName()).log(Level.SEVERE, null, ex);
}
}
return 0;
}
public void testInputStreamWithOneFIXMessage() throws Exception {
String message = "8=FIX.4.2\u00019=456\u000135=D\u000110=123\u0001";
InputStream is = new ByteArrayInputStream(message.getBytes());
BlockingQueue<String> queue = new LinkedBlockingQueue<String>();
parseAndWait(is, queue);
assertFalse(queue.isEmpty());
assertEquals(2, queue.size());
assertEquals(message, queue.take());
assertEquals("DONE", queue.take());
}
/**
* The parser should attempt to "fix" the FIX message. In this case, the message is missing the
* last SOH character.
*/
public void testFIXMessageThatDoesNotEndWithSOH() throws Exception {
String oneMessage = "8=FIX.4.2\u00019=456\u000135=D\u000110=123";
InputStream is = new ByteArrayInputStream(oneMessage.getBytes());
BlockingQueue<String> queue = new LinkedBlockingQueue<String>();
parseAndWait(is, queue);
assertFalse(queue.isEmpty());
assertEquals(2, queue.size());
assertEquals(oneMessage + "\u0001", queue.take());
assertEquals("DONE", queue.take());
}
@Override
public void run() {
try {
while (!s1.isEmpty()) {
s2.put(fib(s1.poll()));
}
} catch (InterruptedException ex) {
Logger.getLogger(Producer.class.getName()).log(Level.SEVERE, null, ex);
}
System.out.println("Denne producer tråd er done. ID: " + getName());
}
@Override
public boolean hasNext() throws IOException, CollectionException {
try {
if (documentQueue.isEmpty()) logger.info("Waiting for a new document.");
currentDoc = documentQueue.take();
if (currentDoc == CollectionDocument.LAST_DOCUMENT) return false;
else return true;
} catch (InterruptedException e) {
logger.info("Stream {} interrupted", this.streamName);
return false;
}
}
public void shutdownPool() {
while (!jobQueue.isEmpty()) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
shutdown = true;
for (Thread workerThread : jobThreads) {
workerThread.interrupt();
}
}
/** Returns a response to satisfy {@code request}. */
private MockResponse dispatch(RecordedRequest request) throws InterruptedException {
if (responseQueue.isEmpty()) {
throw new IllegalStateException("Unexpected request: " + request);
}
if (singleResponse) {
return responseQueue.peek();
} else {
requestCount.incrementAndGet();
requestQueue.add(request);
return responseQueue.take();
}
}
public void run() {
try {
while (!s1.isEmpty()) {
int n;
n = s1.poll();
long result = fib(n);
s2.put(result);
}
} catch (InterruptedException ex) {
Logger.getLogger(Fibonacci.class.getName()).log(Level.SEVERE, null, ex);
}
}
public void testFIXMessageWithJunkAtTheBeginningOfTheString() throws Exception {
String junkInTheTrunkMessage = "JunkInTheTrunk";
String message = "8=FIX.4.2|9=456|35=D|10=123|";
InputStream is = new ByteArrayInputStream((junkInTheTrunkMessage + message).getBytes());
BlockingQueue<String> queue = new LinkedBlockingQueue<String>();
parseAndWait(is, queue);
assertFalse(queue.isEmpty());
assertEquals(2, queue.size());
assertEquals(message.replaceAll("\\|", "\u0001"), queue.take());
assertEquals("DONE", queue.take());
}