/**
* Test concurrent reader and writer (GH-458).
*
* <p><b>Test case:</b>
*
* <ol>
* <li/>start a long running reader;
* <li/>try to start a writer: it should time out;
* <li/>stop the reader;
* <li/>start the writer again: it should succeed.
* </ol>
*
* @throws Exception error during request execution
*/
@Test
@Ignore("There is no way to stop a query on the server!")
public void testReaderWriter() throws Exception {
final String readerQuery = "?query=(1%20to%20100000000000000)%5b.=1%5d";
final String writerQuery = "/test.xml";
final byte[] content = Token.token("<a/>");
final Get readerAction = new Get(readerQuery);
final Put writerAction = new Put(writerQuery, content);
final ExecutorService exec = Executors.newFixedThreadPool(2);
// start reader
exec.submit(readerAction);
Performance.sleep(TIMEOUT); // delay in order to be sure that the reader has started
// start writer
Future<HTTPResponse> writer = exec.submit(writerAction);
try {
final HTTPResponse result = writer.get(TIMEOUT, TimeUnit.MILLISECONDS);
if (result.status.isSuccess()) fail("Database modified while a reader is running");
throw new Exception(result.toString());
} catch (final TimeoutException e) {
// writer is blocked by the reader: stop it
writerAction.stop = true;
}
// stop reader
readerAction.stop = true;
// start the writer again
writer = exec.submit(writerAction);
assertEquals(HTTPCode.CREATED, writer.get().status);
}
public static void main(String[] args) {
ExecutorService pool = Executors.newFixedThreadPool(2);
final Vector[] vecs = {
new Vector(), new Vector(),
};
vecs[0].add(vecs[1]);
vecs[1].add(vecs[0]);
for (int i = 0; i < 2; i++) {
final int threadNumber = i;
pool.submit(
new Callable() {
public Object call() throws Exception {
for (int i = 0; i < 1000 * 1000; i++) {
ObjectOutputStream out = new ObjectOutputStream(new NullOutputStream());
out.writeObject(vecs[threadNumber]);
out.close();
}
System.out.println("done");
return null;
}
});
}
}
@Override
public Boolean call() throws Exception {
long timeoutMillis = 5000;
try {
getServersFile();
getZkRunning();
while (true) {
while (!restartQueue.isEmpty()) {
LOG.debug("Restart queue size [" + restartQueue.size() + "]");
RestartHandler handler = restartQueue.poll();
Future<ScriptContext> runner = pool.submit(handler);
ScriptContext scriptContext = runner.get(); // blocking call
if (scriptContext.getExitCode() != 0) restartQueue.add(handler);
}
try {
Thread.sleep(timeoutMillis);
} catch (InterruptedException e) {
}
}
} catch (Exception e) {
e.printStackTrace();
LOG.error(e);
pool.shutdown();
throw e;
}
}
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
if (attrs.isRegularFile()) {
if (m != null) m.reset(root.relativize(file).toString());
if (m == null || m.matches())
futures.add(exec.submit(new FileLoader(root, file, blocSize)));
}
return FileVisitResult.CONTINUE;
}
static Future<String> futureOutputOf(final InputStream is) {
return drainers.submit(
new Callable<String>() {
public String call() throws IOException {
return outputOf(is);
}
});
}
/**
* Test 2 concurrent readers (GH-458).
*
* <p><b>Test case:</b>
*
* <ol>
* <li/>start a long running reader;
* <li/>start a fast reader: it should succeed.
* </ol>
*
* @throws Exception error during request execution
*/
@Test
public void testMultipleReaders() throws Exception {
final String number = "63177";
final String slowQuery = "?query=(1%20to%20100000000000000)%5b.=1%5d";
final String fastQuery = "?query=" + number;
final Get slowAction = new Get(slowQuery);
final Get fastAction = new Get(fastQuery);
final ExecutorService exec = Executors.newFixedThreadPool(2);
exec.submit(slowAction);
Performance.sleep(TIMEOUT); // delay in order to be sure that the reader has started
final Future<HTTPResponse> fast = exec.submit(fastAction);
try {
final HTTPResponse result = fast.get(TIMEOUT, TimeUnit.MILLISECONDS);
assertEquals(HTTPCode.OK, result.status);
assertEquals(number, result.data);
} finally {
slowAction.stop = true;
}
}
private static void build(Iterable<String> in) throws IOException, InterruptedException {
final List<String> args = newArrayList(in);
Collections.sort(args);
final int threads = Runtime.getRuntime().availableProcessors();
final ExecutorService ex = Executors.newFixedThreadPool(threads);
final String base = "base";
new WithVm("base").createIfNotPresent();
for (String pkg : args)
ex.submit(
() -> {
final WithVm newVm = new WithVm("fbuild-" + pkg, TimeUnit.MINUTES.toMillis(30));
final File rbuild = new File("wip-" + pkg + ".rbuild");
try {
newVm.cloneFrom(base);
newVm.start();
newVm.inTee(rbuild, "apt-get", "-oAPT::Get::Only-Source=true", "source", pkg);
newVm.inTee(rbuild, "apt-get", "build-dep", "-y", "--force-yes", pkg);
newVm.inTee(rbuild, "ifdown", "eth0");
final boolean success =
0
== newVm.inTee(
rbuild, "sh", "-c", "cd " + pkg + "-* && dpkg-buildpackage -us -uc");
newVm.stopNow();
if (success) {
rbuild.renameTo(new File("success-" + pkg + ".rbuild"));
newVm.destroy();
System.out.println("success: " + pkg);
} else {
rbuild.renameTo(new File("failure-" + pkg + ".rbuild"));
System.out.println("failure: " + pkg);
}
} catch (Exception e) {
rbuild.renameTo(new File("error-" + pkg + ".rbuild"));
System.err.println("build error: " + pkg);
e.printStackTrace();
newVm.stopNow();
}
return null;
});
ex.shutdown();
ex.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
}
public static void main(final String[] args) throws Exception {
final int length = (1 << 14);
final ICircularLongsBuffer buffer = new UnsafeCircularLongsBuffer(length);
final File file = new File("output.log");
log.info("File " + file);
file.delete();
// file.deleteOnExit();
final FastLoggerImpl logger =
new FastLoggerImpl(
new ThreadFactory() {
@Override
public Thread newThread(final Runnable r) {
return new Thread(r);
}
},
buffer,
WaitingStrategy.SPINNING,
new RawFileWriter(file));
logger.startDraining();
final int workers = Runtime.getRuntime().availableProcessors() - 1;
final ExecutorService workersPool = Executors.newFixedThreadPool(workers);
for (int i = 0; i < workers; i++) {
workersPool.submit(
new Runnable() {
@Override
public void run() {
try {
for (int i = 0; i < 100000000; i++) {
logger.log("Message %f -- %d ").with(25.98 + i).with(100 - i).submit();
}
} catch (Exception e) {
log.error("Error", e);
}
}
});
}
workersPool.shutdown();
workersPool.awaitTermination(1000, TimeUnit.SECONDS);
log.info("Finished");
System.exit(1);
}
/**
* Test concurrent writers (GH-458).
*
* <p><b>Test case:</b>
*
* <ol>
* <li/>start several writers one after another;
* <li/>all writers should succeed.
* </ol>
*
* @throws Exception error during request execution
*/
@Test
public void testMultipleWriters() throws Exception {
final int count = 10;
final String template =
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
+ "<command xmlns=\"http://basex.org/rest\"><text><![CDATA["
+ "ADD TO %1$d <node id=\"%1$d\"/>"
+ "]]></text></command>";
@SuppressWarnings("unchecked")
final Future<HTTPResponse>[] tasks = new Future[count];
final ExecutorService exec = Executors.newFixedThreadPool(count);
// start all writers (not at the same time, but still in parallel)
for (int i = 0; i < count; i++) {
final String command = String.format(template, i);
tasks[i] = exec.submit(new Post("", Token.token(command)));
}
// check if all have finished successfully
for (final Future<HTTPResponse> task : tasks) {
assertEquals(HTTPCode.OK, task.get(TIMEOUT, TimeUnit.MILLISECONDS).status);
}
}
/**
* Discover WS device on the local network
*
* @return list of unique devices access strings which might be URLs in most cases
*/
public static Collection<String> discoverWsDevices() {
final Collection<String> addresses = new ConcurrentSkipListSet<>();
final CountDownLatch serverStarted = new CountDownLatch(1);
final CountDownLatch serverFinished = new CountDownLatch(1);
final Collection<InetAddress> addressList = new ArrayList<>();
try {
final Enumeration<NetworkInterface> interfaces = NetworkInterface.getNetworkInterfaces();
if (interfaces != null) {
while (interfaces.hasMoreElements()) {
NetworkInterface anInterface = interfaces.nextElement();
if (!anInterface.isLoopback()) {
final List<InterfaceAddress> interfaceAddresses = anInterface.getInterfaceAddresses();
for (InterfaceAddress address : interfaceAddresses) {
addressList.add(address.getAddress());
}
}
}
}
} catch (SocketException e) {
e.printStackTrace();
}
ExecutorService executorService = Executors.newCachedThreadPool();
for (final InetAddress address : addressList) {
Runnable runnable =
new Runnable() {
public void run() {
try {
final String uuid = UUID.randomUUID().toString();
final String probe =
WS_DISCOVERY_PROBE_MESSAGE.replaceAll(
"<wsa:MessageID>urn:uuid:.*</wsa:MessageID>",
"<wsa:MessageID>urn:uuid:" + uuid + "</wsa:MessageID>");
final int port = random.nextInt(20000) + 40000;
@SuppressWarnings("SocketOpenedButNotSafelyClosed")
final DatagramSocket server = new DatagramSocket(port, address);
new Thread() {
public void run() {
try {
final DatagramPacket packet = new DatagramPacket(new byte[4096], 4096);
server.setSoTimeout(WS_DISCOVERY_TIMEOUT);
long timerStarted = System.currentTimeMillis();
while (System.currentTimeMillis() - timerStarted < (WS_DISCOVERY_TIMEOUT)) {
serverStarted.countDown();
server.receive(packet);
final Collection<String> collection =
parseSoapResponseForUrls(
Arrays.copyOf(packet.getData(), packet.getLength()));
for (String key : collection) {
addresses.add(key);
}
}
} catch (SocketTimeoutException ignored) {
} catch (Exception e) {
e.printStackTrace();
} finally {
serverFinished.countDown();
server.close();
}
}
}.start();
try {
serverStarted.await(1000, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
e.printStackTrace();
}
if (address instanceof Inet4Address) {
server.send(
new DatagramPacket(
probe.getBytes(),
probe.length(),
InetAddress.getByName(WS_DISCOVERY_ADDRESS_IPv4),
WS_DISCOVERY_PORT));
} else {
server.send(
new DatagramPacket(
probe.getBytes(),
probe.length(),
InetAddress.getByName(WS_DISCOVERY_ADDRESS_IPv6),
WS_DISCOVERY_PORT));
}
} catch (Exception e) {
e.printStackTrace();
}
try {
serverFinished.await((WS_DISCOVERY_TIMEOUT), TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
};
executorService.submit(runnable);
}
try {
executorService.shutdown();
executorService.awaitTermination(WS_DISCOVERY_TIMEOUT + 2000, TimeUnit.MILLISECONDS);
} catch (InterruptedException ignored) {
}
return addresses;
}
protected void handleCommand(String command) {
if (command.contains("__")) {
namespace = command.split("__")[0];
command = command.substring(command.indexOf("__") + 2);
}
if (echo) {
if (Thread.currentThread().getName().toLowerCase().indexOf("main") < 0)
println(" [" + Thread.currentThread().getName() + "] " + command);
else println(command);
}
if (command == null || command.startsWith("//")) return;
command = command.trim();
if (command == null || command.length() == 0) {
return;
}
String first = command;
int spaceIndex = command.indexOf(' ');
String[] argsSplit = command.split(" ");
String[] args = new String[argsSplit.length];
for (int i = 0; i < argsSplit.length; i++) {
args[i] = argsSplit[i].trim();
}
if (spaceIndex != -1) {
first = args[0];
}
if (command.startsWith("help")) {
handleHelp(command);
} else if (first.startsWith("#") && first.length() > 1) {
int repeat = Integer.parseInt(first.substring(1));
long t0 = Clock.currentTimeMillis();
for (int i = 0; i < repeat; i++) {
handleCommand(command.substring(first.length()).replaceAll("\\$i", "" + i));
}
println("ops/s = " + repeat * 1000 / (Clock.currentTimeMillis() - t0));
return;
} else if (first.startsWith("&") && first.length() > 1) {
final int fork = Integer.parseInt(first.substring(1));
ExecutorService pool = Executors.newFixedThreadPool(fork);
final String threadCommand = command.substring(first.length());
for (int i = 0; i < fork; i++) {
final int threadID = i;
pool.submit(
new Runnable() {
public void run() {
String command = threadCommand;
String[] threadArgs = command.replaceAll("\\$t", "" + threadID).trim().split(" ");
// TODO &t #4 m.putmany x k
if ("m.putmany".equals(threadArgs[0]) || "m.removemany".equals(threadArgs[0])) {
if (threadArgs.length < 4) {
command += " " + Integer.parseInt(threadArgs[1]) * threadID;
}
}
handleCommand(command);
}
});
}
pool.shutdown();
try {
pool.awaitTermination(60 * 60, TimeUnit.SECONDS); // wait 1h
} catch (Exception e) {
e.printStackTrace();
}
} else if (first.startsWith("@")) {
if (first.length() == 1) {
println("usage: @<file-name>");
return;
}
File f = new File(first.substring(1));
println("Executing script file " + f.getAbsolutePath());
if (f.exists()) {
try {
BufferedReader br = new BufferedReader(new FileReader(f));
String l = br.readLine();
while (l != null) {
handleCommand(l);
l = br.readLine();
}
br.close();
} catch (IOException e) {
e.printStackTrace();
}
} else {
println("File not found! " + f.getAbsolutePath());
}
} else if (command.indexOf(';') != -1) {
StringTokenizer st = new StringTokenizer(command, ";");
while (st.hasMoreTokens()) {
handleCommand(st.nextToken());
}
return;
} else if ("silent".equals(first)) {
silent = Boolean.parseBoolean(args[1]);
} else if ("shutdown".equals(first)) {
hazelcast.getLifecycleService().shutdown();
} else if ("echo".equals(first)) {
echo = Boolean.parseBoolean(args[1]);
println("echo: " + echo);
} else if ("ns".equals(first)) {
if (args.length > 1) {
namespace = args[1];
println("namespace: " + namespace);
// init();
}
} else if ("whoami".equals(first)) {
println(hazelcast.getCluster().getLocalMember());
} else if ("who".equals(first)) {
println(hazelcast.getCluster());
} else if ("jvm".equals(first)) {
System.gc();
println("Memory max: " + Runtime.getRuntime().maxMemory() / 1024 / 1024 + "M");
println(
"Memory free: "
+ Runtime.getRuntime().freeMemory() / 1024 / 1024
+ "M "
+ (int) (Runtime.getRuntime().freeMemory() * 100 / Runtime.getRuntime().maxMemory())
+ "%");
long total = Runtime.getRuntime().totalMemory();
long free = Runtime.getRuntime().freeMemory();
println("Used Memory:" + ((total - free) / 1024 / 1024) + "MB");
println("# procs: " + Runtime.getRuntime().availableProcessors());
println(
"OS info: "
+ ManagementFactory.getOperatingSystemMXBean().getArch()
+ " "
+ ManagementFactory.getOperatingSystemMXBean().getName()
+ " "
+ ManagementFactory.getOperatingSystemMXBean().getVersion());
println(
"JVM: "
+ ManagementFactory.getRuntimeMXBean().getVmVendor()
+ " "
+ ManagementFactory.getRuntimeMXBean().getVmName()
+ " "
+ ManagementFactory.getRuntimeMXBean().getVmVersion());
} else if (first.indexOf("ock") != -1 && first.indexOf(".") == -1) {
handleLock(args);
} else if (first.indexOf(".size") != -1) {
handleSize(args);
} else if (first.indexOf(".clear") != -1) {
handleClear(args);
} else if (first.indexOf(".destroy") != -1) {
handleDestroy(args);
} else if (first.indexOf(".iterator") != -1) {
handleIterator(args);
} else if (first.indexOf(".contains") != -1) {
handleContains(args);
} else if (first.indexOf(".stats") != -1) {
handStats(args);
} else if ("t.publish".equals(first)) {
handleTopicPublish(args);
} else if ("q.offer".equals(first)) {
handleQOffer(args);
} else if ("q.take".equals(first)) {
handleQTake(args);
} else if ("q.poll".equals(first)) {
handleQPoll(args);
} else if ("q.peek".equals(first)) {
handleQPeek(args);
} else if ("q.capacity".equals(first)) {
handleQCapacity(args);
} else if ("q.offermany".equals(first)) {
handleQOfferMany(args);
} else if ("q.pollmany".equals(first)) {
handleQPollMany(args);
} else if ("s.add".equals(first)) {
handleSetAdd(args);
} else if ("s.remove".equals(first)) {
handleSetRemove(args);
} else if ("s.addmany".equals(first)) {
handleSetAddMany(args);
} else if ("s.removemany".equals(first)) {
handleSetRemoveMany(args);
} else if (first.equals("m.replace")) {
handleMapReplace(args);
} else if (first.equalsIgnoreCase("m.putIfAbsent")) {
handleMapPutIfAbsent(args);
} else if (first.equals("m.putAsync")) {
handleMapPutAsync(args);
} else if (first.equals("m.getAsync")) {
handleMapGetAsync(args);
} else if (first.equals("m.put")) {
handleMapPut(args);
} else if (first.equals("m.get")) {
handleMapGet(args);
} else if (first.equalsIgnoreCase("m.getMapEntry")) {
handleMapGetMapEntry(args);
} else if (first.equals("m.remove")) {
handleMapRemove(args);
} else if (first.equals("m.evict")) {
handleMapEvict(args);
} else if (first.equals("m.putmany") || first.equalsIgnoreCase("m.putAll")) {
handleMapPutMany(args);
} else if (first.equals("m.getmany")) {
handleMapGetMany(args);
} else if (first.equals("m.removemany")) {
handleMapRemoveMany(args);
} else if (command.equalsIgnoreCase("m.localKeys")) {
handleMapLocalKeys();
} else if (command.equals("m.keys")) {
handleMapKeys();
} else if (command.equals("m.values")) {
handleMapValues();
} else if (command.equals("m.entries")) {
handleMapEntries();
} else if (first.equals("m.lock")) {
handleMapLock(args);
} else if (first.equalsIgnoreCase("m.tryLock")) {
handleMapTryLock(args);
} else if (first.equals("m.unlock")) {
handleMapUnlock(args);
} else if (first.indexOf(".addListener") != -1) {
handleAddListener(args);
} else if (first.equals("m.removeMapListener")) {
handleRemoveListener(args);
} else if (first.equals("m.unlock")) {
handleMapUnlock(args);
} else if (first.equals("l.add")) {
handleListAdd(args);
} else if (first.equals("l.set")) {
handleListSet(args);
} else if ("l.addmany".equals(first)) {
handleListAddMany(args);
} else if (first.equals("l.remove")) {
handleListRemove(args);
} else if (first.equals("l.contains")) {
handleListContains(args);
} else if ("a.get".equals(first)) {
handleAtomicNumberGet(args);
} else if ("a.set".equals(first)) {
handleAtomicNumberSet(args);
} else if ("a.inc".equals(first)) {
handleAtomicNumberInc(args);
} else if ("a.dec".equals(first)) {
handleAtomicNumberDec(args);
// } else if (first.equals("execute")) {
// execute(args);
} else if (first.equals("partitions")) {
handlePartitions(args);
// } else if (first.equals("txn")) {
// hazelcast.getTransaction().begin();
// } else if (first.equals("commit")) {
// hazelcast.getTransaction().commit();
// } else if (first.equals("rollback")) {
// hazelcast.getTransaction().rollback();
// } else if (first.equalsIgnoreCase("executeOnKey")) {
// executeOnKey(args);
// } else if (first.equalsIgnoreCase("executeOnMember")) {
// executeOnMember(args);
// } else if (first.equalsIgnoreCase("executeOnMembers")) {
// executeOnMembers(args);
// } else if (first.equalsIgnoreCase("longOther") ||
// first.equalsIgnoreCase("executeLongOther")) {
// executeLongTaskOnOtherMember(args);
// } else if (first.equalsIgnoreCase("long") || first.equalsIgnoreCase("executeLong"))
// {
// executeLong(args);
} else if (first.equalsIgnoreCase("instances")) {
handleInstances(args);
} else if (first.equalsIgnoreCase("quit") || first.equalsIgnoreCase("exit")) {
System.exit(0);
} else {
println("type 'help' for help");
}
}
