/**
* Test concurrent reader and writer (GH-458).
*
* <p><b>Test case:</b>
*
* <ol>
* <li/>start a long running reader;
* <li/>try to start a writer: it should time out;
* <li/>stop the reader;
* <li/>start the writer again: it should succeed.
* </ol>
*
* @throws Exception error during request execution
*/
@Test
@Ignore("There is no way to stop a query on the server!")
public void testReaderWriter() throws Exception {
final String readerQuery = "?query=(1%20to%20100000000000000)%5b.=1%5d";
final String writerQuery = "/test.xml";
final byte[] content = Token.token("<a/>");
final Get readerAction = new Get(readerQuery);
final Put writerAction = new Put(writerQuery, content);
final ExecutorService exec = Executors.newFixedThreadPool(2);
// start reader
exec.submit(readerAction);
Performance.sleep(TIMEOUT); // delay in order to be sure that the reader has started
// start writer
Future<HTTPResponse> writer = exec.submit(writerAction);
try {
final HTTPResponse result = writer.get(TIMEOUT, TimeUnit.MILLISECONDS);
if (result.status.isSuccess()) fail("Database modified while a reader is running");
throw new Exception(result.toString());
} catch (final TimeoutException e) {
// writer is blocked by the reader: stop it
writerAction.stop = true;
}
// stop reader
readerAction.stop = true;
// start the writer again
writer = exec.submit(writerAction);
assertEquals(HTTPCode.CREATED, writer.get().status);
}
public static void main(String[] args) throws InterruptedException, ExecutionException {
List<Future<String>> futures = new ArrayList<>();
ExecutorService executor = Executors.newFixedThreadPool(5);
for (int i = 1000; i <= 1010; i++) {
futures.add(executor.submit(getTask(i)));
}
futures.add(executor.submit(getTask(10000000)));
for (Future<String> future : futures) {
System.out.println(future.get());
}
executor.shutdown();
executor.awaitTermination(10, TimeUnit.SECONDS);
/* output
500500
501501
502503
503506
504510
505515
506521
507528
508536
509545
510555
50000005000000
*/
}
private int[] computeClusterNumberRange(int min, int max, int hop, Optimizer optimizer)
throws ExecutionException, InterruptedException {
ExecutorService executor;
if (parallelWorkers > 1) {
executor = Executors.newFixedThreadPool(2);
} else {
executor = Executors.newFixedThreadPool(1);
}
ConcurrentMap<Integer, Double> sharedScores = new ConcurrentHashMap<>();
SearchSpaceBoundaryFinder_old upperBoundFinder =
new SearchSpaceUpperBoundaryFinder_old(min, max, hop, optimizer, sharedScores);
Future<Integer> futureUpperBound = executor.submit(upperBoundFinder);
SearchSpaceBoundaryFinder_old lowerBoundFinder =
new SearchSpaceLowerBoundaryFinder_old(min, max, hop, optimizer, sharedScores);
Future<Integer> futureLowerBound = executor.submit(lowerBoundFinder);
executor.shutdown();
int[] r = new int[2];
Integer realMin = futureLowerBound.get();
Integer realMax = futureUpperBound.get();
r[0] = realMin == null ? -1 : realMin;
r[1] = realMax == null ? -1 : realMax;
scores.putAll(lowerBoundFinder.getSplitsAndScores());
// scores.putAll(upperBoundFinder.getSplitsAndScores());
return r;
}
public MessageSender(UniqueValue unique, int port) throws IOException {
this.unique = unique;
executor.submit(tcpListener = new TcpListener(port, this::acceptMessage));
executor.submit(udpListener = new UdpListener(port, this::acceptMessage));
executor.submit(tcpDispatcher);
executor.submit(udpDispatcher);
executor.submit(new IncomeMessagesProcessor());
}
private void executeConcurrentRandomReadAndWriteOperations()
throws InterruptedException, ExecutionException {
for (int i = 0; i < THREAD_COUNT; i++) {
futures.add(executorService.submit(new Writer()));
}
for (int i = 0; i < THREAD_COUNT; i++) {
futures.add(executorService.submit(new Reader()));
}
for (Future<Void> future : futures) future.get();
}
public void testManyRequests() throws Exception {
final ExecutorService es = Executors.newFixedThreadPool(4);
final Future f1 = es.submit(new ManyRequests(client, true));
final Future f2 = es.submit(new ManyRequests(client, true));
final Future f3 = es.submit(new ManyRequests(secondClient, true));
final Future f4 = es.submit(new ManyRequests(secondClient, true));
f1.get(2, TimeUnit.MINUTES);
f2.get(2, TimeUnit.MINUTES);
f3.get(2, TimeUnit.MINUTES);
f4.get(2, TimeUnit.MINUTES);
es.shutdownNow();
}
public void testParallelRequests()
throws ExecutionException, InterruptedException, TimeoutException {
final ExecutorService es = Executors.newFixedThreadPool(4);
final Future f1 = es.submit(new ParallelRequests(client));
final Future f2 = es.submit(new ParallelRequests(client));
final Future f3 = es.submit(new ParallelRequests(secondClient));
final Future f4 = es.submit(new ParallelRequests(secondClient));
f1.get(2, TimeUnit.MINUTES);
f2.get(2, TimeUnit.MINUTES);
f3.get(2, TimeUnit.MINUTES);
f4.get(2, TimeUnit.MINUTES);
es.shutdownNow();
}
/** {@inheritDoc} */
@Override
public void loadCache(GridBiInClosure<K, V> c, @Nullable Object... args) throws GridException {
ExecutorService exec =
new ThreadPoolExecutor(
threadsCnt,
threadsCnt,
0L,
MILLISECONDS,
new ArrayBlockingQueue<Runnable>(batchQueueSize),
new BlockingRejectedExecutionHandler());
Iterator<I> iter = inputIterator(args);
Collection<I> buf = new ArrayList<>(batchSize);
try {
while (iter.hasNext()) {
if (Thread.currentThread().isInterrupted()) {
U.warn(log, "Working thread was interrupted while loading data.");
break;
}
buf.add(iter.next());
if (buf.size() == batchSize) {
exec.submit(new Worker(c, buf, args));
buf = new ArrayList<>(batchSize);
}
}
if (!buf.isEmpty()) exec.submit(new Worker(c, buf, args));
} catch (RejectedExecutionException ignored) {
// Because of custom RejectedExecutionHandler.
assert false : "RejectedExecutionException was thrown while it shouldn't.";
} finally {
exec.shutdown();
try {
exec.awaitTermination(Long.MAX_VALUE, MILLISECONDS);
} catch (InterruptedException ignored) {
U.warn(log, "Working thread was interrupted while waiting for put operations to complete.");
Thread.currentThread().interrupt();
}
}
}
/** Run a basic task */
@Test
public void testBasicTask() throws Exception {
Callable<String> task = new BasicTestTask();
ExecutorService executor = createSingleNodeExecutorService("testBasicTask");
Future future = executor.submit(task);
assertEquals(future.get(), BasicTestTask.RESULT);
}
private void handleResetRequest(
ResetRequestTuple request, final AbstractLengthPrependerClient ctx) throws IOException {
DataList dl;
dl = publisherBuffers.remove(request.getIdentifier());
byte[] message;
if (dl == null) {
message = ("Invalid identifier '" + request.getIdentifier() + "'").getBytes();
} else {
AbstractLengthPrependerClient channel = publisherChannels.remove(request.getIdentifier());
if (channel != null) {
eventloop.disconnect(channel);
}
dl.reset();
message = ("Request sent for processing: " + request).getBytes();
}
final byte[] tuple = PayloadTuple.getSerializedTuple(0, message.length);
System.arraycopy(message, 0, tuple, tuple.length - message.length, message.length);
serverHelperExecutor.submit(
new Runnable() {
@Override
public void run() {
ctx.write(tuple);
eventloop.disconnect(ctx);
}
});
}
private void singleRequestCall(
ExecutorService jobService, final InstanceRequest request, int jobsPerRequest)
throws Exception {
TraceContext.register(request);
Future[] tasks = new Future[jobsPerRequest];
// fan out multiple threads for this request
for (int i = 0; i < jobsPerRequest; i++) {
final int taskId = i;
tasks[i] =
jobService.submit(
new TraceRunnable() {
@Override
public void runJob() {
String tid = Thread.currentThread().getId() + "";
TraceContext.log(tid, request.getRequestId() + SEP + taskId);
}
});
}
// wait for all threads to finish the job
for (int i = 0; i < jobsPerRequest; i++) {
// block waiting
tasks[i].get();
}
TraceContext.unregister(request);
}
@Test(timeout = 1500l)
public void testOnlyOneLockAllowedTwoThreads() throws Exception {
final CountDownLatch latch = new CountDownLatch(1);
Lock firstLock = new ReentrantZkLock(baseLockPath, zkSessionManager);
firstLock.lock();
try {
testService.submit(
new Runnable() {
@Override
public void run() {
Lock secondLock = new ReentrantZkLock(baseLockPath, zkSessionManager);
secondLock.lock();
try {
latch.countDown();
} finally {
secondLock.unlock();
}
}
});
boolean nowAcquired = latch.await(500, TimeUnit.MILLISECONDS);
assertTrue("The Second lock was acquired before the first lock was released!", !nowAcquired);
} finally {
firstLock.unlock();
}
}
/* Regression test for DB-3614 */
@Test
@Category(SlowTest.class)
@Ignore("-sf- takes way too long to fail and interferes rest of build")
public void testTenTableJoinExplainDuration() throws Exception {
int size = 10;
List<String> tables = new ArrayList<String>(size);
List<String> joins = new ArrayList<String>(size - 1);
for (int i = 0; i < size; i++) {
methodWatcher.executeUpdate(format("create table tentab%s (c1 int primary key)", i));
methodWatcher.executeUpdate(format("insert into tentab%s values (1)", i));
tables.add(format("tentab%s", i));
if (i > 0) {
joins.add(format("tentab%s.c1 = tentab%s.c1", i, i - 1));
}
}
System.out.println("Tables created");
final String fromClause = Joiner.on(", ").join(tables);
final String joinCriteria = Joiner.on(" AND ").join(joins);
ExecutorService es =
Executors.newSingleThreadExecutor(
new ThreadFactory() {
@Override
public Thread newThread(Runnable r) {
Thread t = new Thread(r);
t.setDaemon(true);
return t;
}
});
try {
final CyclicBarrier startLatch = new CyclicBarrier(2);
final CountDownLatch finishLatch = new CountDownLatch(1);
Future<Void> f =
es.submit(
new Callable<Void>() {
@Override
public Void call() throws Exception {
String query =
format("EXPLAIN SELECT * FROM %s WHERE %s ", fromClause, joinCriteria);
startLatch.await();
try {
ResultSet rs = methodWatcher.executeQuery(query);
// Loose check that explain statement took a few seconds or less,
// because we're just making sure the short circuit logic in
// OptimizerImpl.checkTimeout() blocks this from taking several minutes.
Assert.assertTrue("Explain did not return result!", rs.next());
} finally {
finishLatch.countDown();
}
return null;
}
});
System.out.println("Starting wait");
startLatch.await();
f.get(1, TimeUnit.SECONDS);
System.out.println("Finished waiting");
} finally {
System.out.println("shutting down");
}
}
@AfterClass
public static void tearDown() throws InterruptedException, ExecutionException, IOException {
DirDataImpl dirData = new DirDataImpl();
DirDataImpl.setNumber(3334);
List<String> paths = dirData.dividePath(rootPath, 30, THREADS);
List<Future<String>> list = new ArrayList<>();
try {
for (String p : paths) {
Future<String> submit =
executor.submit(
() -> {
dirData.delDir(p, 6);
return "It done!";
});
list.add(submit);
}
for (Future<String> future : list) {
future.get();
}
} finally {
executor.shutdown();
while (!executor.awaitTermination(10, TimeUnit.SECONDS)) {
LOG.info("Awaiting completion of threads.");
}
}
FileUtils.deleteDirectory(new File(rootPath));
}
public float generate(CA b1, CA b2, Updater u) {
if (_frw == null) {
_rw = new RuleWorker[WORKER_SIZE];
_frw = new Future[_rw.length];
}
int workers = Math.max(1, Math.min(_rw.length, b1.getHeight() / 200));
// workers = 1;
int x1 = 0, y1 = 0, step = b1.getHeight() / workers;
for (int i = 0; i < workers; i++) {
int bot = y1 + step;
if (i == workers - 1) {
bot = b1.getHeight();
}
_rw[i] = new RuleWorker(b1, b2, x1, y1, b1.getWidth(), bot);
_frw[i] = _pool.submit(_rw[i]);
y1 = y1 + step;
}
try {
for (int i = 0; i < workers; i++) {
_frw[i].get();
}
} catch (ExecutionException e) {
throw new Error(e);
} catch (InterruptedException e) {
}
return 0f;
}
public static void main(String[] args) {
ExecutorService pool = Executors.newFixedThreadPool(2);
final Vector[] vecs = {
new Vector(), new Vector(),
};
vecs[0].add(vecs[1]);
vecs[1].add(vecs[0]);
for (int i = 0; i < 2; i++) {
final int threadNumber = i;
pool.submit(
new Callable() {
public Object call() throws Exception {
for (int i = 0; i < 1000 * 1000; i++) {
ObjectOutputStream out = new ObjectOutputStream(new NullOutputStream());
out.writeObject(vecs[threadNumber]);
out.close();
}
System.out.println("done");
return null;
}
});
}
}
@Override
protected long runQueuePass() throws InterruptedException {
resetCounters();
Future<?>[] futures = new Future[NUM_WORKERS];
for (int i = 0; i < NUM_WORKERS; i++) {
futures[i] = EXECUTOR.submit(queueWorkers[i]);
}
long start = System.currentTimeMillis();
for (long i = 0; i < ITERATIONS; i++) {
blockingQueue.put(Long.valueOf(i));
}
while (blockingQueue.size() > 0) {
// spin while queue drains
}
for (int i = 0; i < NUM_WORKERS; i++) {
queueWorkers[i].halt();
futures[i].cancel(true);
}
long opsPerSecond = (ITERATIONS * 1000L) / (System.currentTimeMillis() - start);
assertEquals(ITERATIONS, sumCounters());
return opsPerSecond;
}
@Override
public Boolean call() throws Exception {
long timeoutMillis = 5000;
try {
getServersFile();
getZkRunning();
while (true) {
while (!restartQueue.isEmpty()) {
LOG.debug("Restart queue size [" + restartQueue.size() + "]");
RestartHandler handler = restartQueue.poll();
Future<ScriptContext> runner = pool.submit(handler);
ScriptContext scriptContext = runner.get(); // blocking call
if (scriptContext.getExitCode() != 0) restartQueue.add(handler);
}
try {
Thread.sleep(timeoutMillis);
} catch (InterruptedException e) {
}
}
} catch (Exception e) {
e.printStackTrace();
LOG.error(e);
pool.shutdown();
throw e;
}
}
@Test(timeout = 1000l)
public void testNoWritesWithASingleRead() throws Exception {
ReadWriteLock lock = new ReentrantZkReadWriteLock(testPath, zkSessionManager);
Lock readLock = lock.readLock();
final Lock writeLock = lock.writeLock();
readLock.lock();
final CountDownLatch latch = new CountDownLatch(1);
testService.submit(
new Runnable() {
@Override
public void run() {
writeLock.lock();
try {
latch.countDown();
} finally {
writeLock.unlock();
}
}
});
boolean acquired = latch.await(250, TimeUnit.MILLISECONDS);
assertTrue("The Write lock was improperly acquired!", !acquired);
// unlock the read lock, and make sure that the write lock is acquired
readLock.unlock();
acquired = latch.await(250, TimeUnit.MILLISECONDS);
assertTrue("The Write lock was never acquired!", acquired);
}
@Test(timeout = 1000l)
public void testOneWriteAllowed() throws Exception {
ReadWriteLock firstLock = new ReentrantZkReadWriteLock(testPath, zkSessionManager);
ReadWriteLock secondLock = new ReentrantZkReadWriteLock(testPath, zkSessionManager);
final Lock firstWriteLock = firstLock.writeLock();
final Lock secondWriteLock = secondLock.writeLock();
firstWriteLock.lock();
final CountDownLatch correctnessLatch = new CountDownLatch(1);
testService.submit(
new Runnable() {
@Override
public void run() {
secondWriteLock.lock();
try {
correctnessLatch.countDown();
} finally {
secondWriteLock.unlock();
}
}
});
boolean acquired = correctnessLatch.await(250, TimeUnit.MILLISECONDS);
assertTrue("Two Write locks were acquired concurrently!", !acquired);
// unlock and check
firstWriteLock.unlock();
acquired = correctnessLatch.await(250, TimeUnit.MILLISECONDS);
assertTrue("The second write lock was never acquired!", acquired);
}
final void test() throws Exception {
Future[] futures = new Future[nthreads];
for (int i = 0; i < nthreads; ++i) futures[i] = pool.submit(this);
barrier.await();
Thread.sleep(TIMEOUT);
boolean tooLate = false;
for (int i = 1; i < nthreads; ++i) {
if (!futures[i].cancel(true)) tooLate = true;
// Unbunch some of the cancels
if ((i & 3) == 0) Thread.sleep(1 + rng.next() % 10);
}
Object f0 = futures[0].get();
if (!tooLate) {
for (int i = 1; i < nthreads; ++i) {
if (!futures[i].isDone() || !futures[i].isCancelled())
throw new Error("Only one thread should complete");
}
} else System.out.print("(cancelled too late) ");
long endTime = System.nanoTime();
long time = endTime - timer.startTime;
if (print) {
double secs = (double) (time) / 1000000000.0;
System.out.println("\t " + secs + "s run time");
}
}
/** Execute a task that is executing something else inside. Nested Execution. */
@Test(timeout = 10000)
public void testNestedExecution() throws Exception {
Callable<String> task = new NestedExecutorTask();
ExecutorService executor = createSingleNodeExecutorService("testNestedExecution");
Future future = executor.submit(task);
future.get();
}
public static void main(String[] args) {
Callable<Integer> task =
() -> {
try {
TimeUnit.SECONDS.sleep(1);
return 123;
} catch (InterruptedException e) {
throw new IllegalStateException("task interrupted", e);
}
};
ExecutorService executor = Executors.newFixedThreadPool(1);
Future<Integer> future = executor.submit(task);
System.out.println("future done? " + future.isDone());
Integer result = -1;
try {
result = future.get();
} catch (Exception e) {
e.printStackTrace();
}
System.out.println("future done? " + future.isDone());
System.out.println("result: " + result);
stop(executor);
}
@Test
public void testPartitionedReferencePileThroughput() throws InterruptedException {
int backlog = 64 * 1024;
long start = System.nanoTime();
PartitionedReferencePile<Slot> pile = new PartitionedReferencePile<Slot>(backlog, Slot::new);
CountDownLatch latch = new CountDownLatch(threadCnt);
for (int i = 0; i < threadCnt; i++) {
int idx = i;
pool.submit(
() -> {
for (int j = 0; j < backlog; j++) {
Slot s = pile.get();
s.var0 = idx;
s.var1 = j;
}
for (Slot s : pile) {
int slotIdx = s.var0;
}
latch.countDown();
});
}
latch.await(5, TimeUnit.SECONDS);
long end = System.nanoTime();
double elapsed = TimeUnit.MILLISECONDS.convert(end - start, TimeUnit.NANOSECONDS);
System.out.println(
String.format("throughput: %s ops/ms", (long) ((threadCnt * backlog) / elapsed)));
}
/**
* Somewhat multi-threaded depth-first search. Performs a DFS of the subtrees from the current
* node in parallel.
*
* @param s The tile sequence
* @param b The board to search
* @param pool The thread pool in which to submit jobs to.
* @return The board with the highest evaluation or null if no board can continue.
*/
private Board solve_pdfs(Board b, ExecutorService pool) {
List<Future<Board>> rets = new ArrayList<>(BOARD_WIDTH);
Board best = null;
int best_score = -1;
for (Direction d : directions) {
Board n = new Board(b);
if (n.move(tileSequence, d)) {
rets.add(pool.submit(new ParallelDFS(n)));
}
}
for (Future<Board> ret : rets) {
try {
Board c = ret.get();
if (c != null) {
int score = evaluate(c);
if (score > best_score) {
best = c;
best_score = score;
}
}
} catch (InterruptedException | ExecutionException e) {
System.err.println("Error: " + e.getMessage());
}
}
return best;
}
@Test(timeout = 1000l)
public void testMultipleClientsCannotAccessSameLock() throws Exception {
final CountDownLatch latch = new CountDownLatch(1);
final Lock firstLock = new ReentrantZkLock(baseLockPath, zkSessionManager);
final Lock sameLock =
new ReentrantZkLock(baseLockPath, new BaseZkSessionManager(newZooKeeper()));
firstLock.lock();
testService.submit(
new Runnable() {
@Override
public void run() {
sameLock.lock();
try {
latch.countDown();
} finally {
sameLock.unlock();
}
}
});
boolean acquired = latch.await(500, TimeUnit.MILLISECONDS);
assertTrue("The Lock was acquired twice by two separate threads!", !acquired);
firstLock.unlock();
acquired = latch.await(500, TimeUnit.MILLISECONDS);
assertTrue("The Lock was never acquired by another thread!", acquired);
}
@Test(timeout = 1500l)
public void testConditionWaitsForSignalOtherThread() throws Exception {
final Lock firstLock = new ReentrantZkLock(baseLockPath, zkSessionManager);
final Condition firstCondition = firstLock.newCondition();
firstLock.lock();
// fire off a thread that will signal the main process thread
testService.submit(
new Runnable() {
@Override
public void run() {
firstLock.lock();
System.out.println("Lock acquired on second thread");
try {
firstCondition.signal();
System.out.println("Lock signalled on second thread");
} finally {
System.out.println("Lock released on second thread");
firstLock.unlock();
}
}
});
// wait for signal notification
System.out.println("First thread waiting for notification");
firstCondition.await();
System.out.println("First thread has been notified");
}
void renderPage(CharSequence source) {
final List<ImageInfo> imageInfos = scanForImageInfo(source);
Callable<List<ImageData>> task =
new Callable<List<ImageData>>() {
public List<ImageData> call() {
List<ImageData> result = new ArrayList<ImageData>();
for (ImageInfo imageInfo : imageInfos) result.add(imageInfo.downloadImage());
return result;
}
};
Future<List<ImageData>> future = executor.submit(task);
renderText(source);
try {
List<ImageData> imageData = future.get();
for (ImageData data : imageData) renderImage(data);
} catch (InterruptedException e) {
// Re-assert the thread's interrupted status
Thread.currentThread().interrupt();
// We don't need the result, so cancel the task too
future.cancel(true);
} catch (ExecutionException e) {
throw launderThrowable(e.getCause());
}
}
public void decodePage(
Object decodeKey,
int pageNum,
final DecodeCallback decodeCallback,
float zoom,
RectF pageSliceBounds) {
final DecodeTask decodeTask =
new DecodeTask(pageNum, decodeCallback, zoom, decodeKey, pageSliceBounds);
synchronized (decodingFutures) {
if (isRecycled) {
return;
}
final Future<?> future =
executorService.submit(
new Runnable() {
public void run() {
try {
Thread.currentThread().setPriority(Thread.NORM_PRIORITY - 1);
performDecode(decodeTask);
} catch (IOException e) {
Log.e(DECODE_SERVICE, "Decode fail", e);
}
}
});
final Future<?> removed = decodingFutures.put(decodeKey, future);
if (removed != null) {
removed.cancel(false);
}
}
}
@Test(timeout = 1500l)
public void testReentrancy() throws Exception {
final CountDownLatch latch = new CountDownLatch(1);
Lock firstLock = new ReentrantZkLock(baseLockPath, zkSessionManager);
firstLock.lock();
try {
testService.submit(
new Runnable() {
@Override
public void run() {
Lock secondLock = new ReentrantZkLock(baseLockPath, zkSessionManager);
secondLock.lock();
try {
latch.countDown();
} finally {
secondLock.unlock();
}
}
});
boolean nowAcquired = latch.await(500, TimeUnit.MILLISECONDS);
assertTrue("The Second lock was acquired before the first lock was released!", !nowAcquired);
// this should be fine
firstLock.lock();
System.out.println("Lock acquired twice!");
firstLock.unlock();
// should still be locked
nowAcquired = latch.await(500, TimeUnit.MILLISECONDS);
assertTrue(
"The Second lock was acquired before the first lock was released twice!", !nowAcquired);
} finally {
firstLock.unlock();
}
}