@Test(enabled = false)
public void weCanCancelTasks(NodeMetadata node) throws InterruptedException, ExecutionException {
ListenableFuture<ExecResponse> future =
client.submitScriptOnNode(node.getId(), "sleep 300", nameTask("sleeper").runAsRoot(false));
ExecResponse response = null;
try {
response = future.get(1, TimeUnit.MILLISECONDS);
fail(node.getId() + ": " + response);
} catch (TimeoutException e) {
assert !future.isDone();
response =
client.runScriptOnNode(
node.getId(), "/tmp/init-sleeper status", wrapInInitScript(false).runAsRoot(false));
assert !response.getOutput().trim().equals("") : node.getId() + ": " + response;
future.cancel(true);
response =
client.runScriptOnNode(
node.getId(), "/tmp/init-sleeper status", wrapInInitScript(false).runAsRoot(false));
assert response.getOutput().trim().equals("") : node.getId() + ": " + response;
try {
future.get();
fail(future.toString());
} catch (CancellationException e1) {
}
}
}
@Test
public void testBufferCloseOnFinish() throws Exception {
SqlTask sqlTask = createInitialTask();
OutputBuffers outputBuffers =
INITIAL_EMPTY_OUTPUT_BUFFERS
.withBuffer("out", new UnpartitionedPagePartitionFunction())
.withNoMoreBufferIds();
updateTask(sqlTask, EMPTY_SOURCES, outputBuffers);
ListenableFuture<BufferResult> bufferResult =
sqlTask.getTaskResults("out", 0, new DataSize(1, MEGABYTE));
assertFalse(bufferResult.isDone());
// finish the task by closing the sources (no splits will ever be added)
updateTask(
sqlTask,
ImmutableList.of(
new TaskSource(TABLE_SCAN_NODE_ID, ImmutableSet.<ScheduledSplit>of(), true)),
outputBuffers);
assertEquals(sqlTask.getTaskInfo().getState(), TaskState.FINISHED);
// buffer will be closed by cancel event (wait for 500 MS for event to fire)
assertTrue(bufferResult.get(200, MILLISECONDS).isBufferClosed());
assertEquals(sqlTask.getTaskInfo().getOutputBuffers().getState(), BufferState.FINISHED);
// verify the buffer is closed
bufferResult = sqlTask.getTaskResults("out", 0, new DataSize(1, MEGABYTE));
assertTrue(bufferResult.isDone());
assertTrue(bufferResult.get().isBufferClosed());
}
@Test
public void pingPong() throws Exception {
connect();
Utils.rollMockClock(0);
// No ping pong happened yet.
assertEquals(Long.MAX_VALUE, peer.getLastPingTime());
assertEquals(Long.MAX_VALUE, peer.getPingTime());
ListenableFuture<Long> future = peer.ping();
assertEquals(Long.MAX_VALUE, peer.getLastPingTime());
assertEquals(Long.MAX_VALUE, peer.getPingTime());
assertFalse(future.isDone());
Ping pingMsg = (Ping) outbound(writeTarget);
Utils.rollMockClock(5);
// The pong is returned.
inbound(writeTarget, new Pong(pingMsg.getNonce()));
pingAndWait(writeTarget);
assertTrue(future.isDone());
long elapsed = future.get();
assertTrue("" + elapsed, elapsed > 1000);
assertEquals(elapsed, peer.getLastPingTime());
assertEquals(elapsed, peer.getPingTime());
// Do it again and make sure it affects the average.
future = peer.ping();
pingMsg = (Ping) outbound(writeTarget);
Utils.rollMockClock(50);
inbound(writeTarget, new Pong(pingMsg.getNonce()));
elapsed = future.get();
assertEquals(elapsed, peer.getLastPingTime());
assertEquals(7250, peer.getPingTime());
}
private void copyToS3(String fileName) {
String bucketName = (String) properties.get(BUCKET_PROPNAME);
String accessId = (String) properties.get(ACCESS_ID_PROPNAME);
String secretKey = (String) properties.get(SECRET_KEY_PROPNAME);
Properties overrides = new Properties();
overrides.setProperty("s3" + ".identity", accessId);
overrides.setProperty("s3" + ".credential", secretKey);
final Iterable<? extends Module> MODULES =
ImmutableSet.of(
new JavaUrlHttpCommandExecutorServiceModule(),
new Log4JLoggingModule(),
new NettyPayloadModule());
BlobStoreContext context =
ContextBuilder.newBuilder("s3")
.credentials(accessId, secretKey)
.modules(MODULES)
.overrides(overrides)
.buildView(BlobStoreContext.class);
// Create Container (the bucket in s3)
try {
AsyncBlobStore blobStore = context.getAsyncBlobStore(); // it can be changed to sync
// BlobStore (returns false if it already exists)
ListenableFuture<Boolean> container = blobStore.createContainerInLocation(null, bucketName);
if (container.get()) {
LOG.info("Created bucket " + bucketName);
}
} catch (Exception ex) {
logger.error("Could not start binary service: {}", ex.getMessage());
throw new RuntimeException(ex);
}
try {
File file = new File(fileName);
AsyncBlobStore blobStore = context.getAsyncBlobStore();
BlobBuilder blobBuilder =
blobStore
.blobBuilder(file.getName())
.payload(file)
.calculateMD5()
.contentType("text/plain")
.contentLength(file.length());
Blob blob = blobBuilder.build();
ListenableFuture<String> futureETag =
blobStore.putBlob(bucketName, blob, PutOptions.Builder.multipart());
LOG.info("Uploaded file etag=" + futureETag.get());
} catch (Exception e) {
LOG.error("Error uploading to blob store", e);
}
}
@Test
public void testMasterAwareExecution() throws Exception {
Settings settings = settingsBuilder().put("discovery.type", "local").build();
ListenableFuture<String> master = internalCluster().startNodeAsync(settings);
ListenableFuture<String> nonMaster =
internalCluster()
.startNodeAsync(settingsBuilder().put(settings).put("node.master", false).build());
master.get();
ensureGreen(); // make sure we have a cluster
ClusterService clusterService =
internalCluster().getInstance(ClusterService.class, nonMaster.get());
final boolean[] taskFailed = {false};
final CountDownLatch latch1 = new CountDownLatch(1);
clusterService.submitStateUpdateTask(
"test",
new ClusterStateUpdateTask() {
@Override
public ClusterState execute(ClusterState currentState) throws Exception {
latch1.countDown();
return currentState;
}
@Override
public void onFailure(String source, Throwable t) {
taskFailed[0] = true;
latch1.countDown();
}
});
latch1.await();
assertTrue("cluster state update task was executed on a non-master", taskFailed[0]);
taskFailed[0] = true;
final CountDownLatch latch2 = new CountDownLatch(1);
clusterService.submitStateUpdateTask(
"test",
new ClusterStateNonMasterUpdateTask() {
@Override
public ClusterState execute(ClusterState currentState) throws Exception {
taskFailed[0] = false;
latch2.countDown();
return currentState;
}
@Override
public void onFailure(String source, Throwable t) {
taskFailed[0] = true;
latch2.countDown();
}
});
latch2.await();
assertFalse("non-master cluster state update task was not executed", taskFailed[0]);
}
public void checkAsyncCallReturnValues() throws ExecutionException, InterruptedException {
// All these async calls include a delay, so none she be finished at first
assertFalse(getBeforeFuture.isDone());
assertFalse(getAfterFuture.isDone());
assertFalse(putFuture.isDone());
// Calls are timed to complete in order, but Verify that we can still wait on the
// futures out of order
assertEquals(getBeforeFuture.get(), "default");
assertEquals(getAfterFuture.get(), "testValue");
putFuture.get();
}
@Test
public void testAsyncOutOfOrder() throws Exception {
ListenableFuture<String> getFuture;
ListenableFuture<Void> putFuture;
try (DelayedMap.AsyncClient client =
createClient(DelayedMap.AsyncClient.class, syncServer).get()) {
getFuture = client.getValueSlowly(500, TimeUnit.MILLISECONDS, "testKey");
putFuture = client.putValueSlowly(250, TimeUnit.MILLISECONDS, "testKey", "testValue");
assertEquals(getFuture.get(1, TimeUnit.SECONDS), "testValue");
putFuture.get(1, TimeUnit.SECONDS);
}
}
private static void send(PaymentSession session) {
try {
System.out.println("Payment Request");
System.out.println("Amount: " + session.getValue().doubleValue() / 100000 + "mDOGE");
System.out.println("Date: " + session.getDate());
System.out.println("Memo: " + session.getMemo());
if (session.pkiVerificationData != null) {
System.out.println("Pki-Verified Name: " + session.pkiVerificationData.name);
if (session.pkiVerificationData.orgName != null)
System.out.println("Pki-Verified Org: " + session.pkiVerificationData.orgName);
System.out.println(
"PKI data verified by: " + session.pkiVerificationData.rootAuthorityName);
}
final Wallet.SendRequest req = session.getSendRequest();
if (password != null) {
if (!wallet.checkPassword(password)) {
System.err.println("Password is incorrect.");
return;
}
req.aesKey = wallet.getKeyCrypter().deriveKey(password);
}
wallet.completeTx(req); // may throw InsufficientMoneyException.
if (options.has("offline")) {
wallet.commitTx(req.tx);
return;
}
setup();
// No refund address specified, no user-specified memo field.
ListenableFuture<PaymentSession.Ack> future =
session.sendPayment(ImmutableList.of(req.tx), null, null);
if (future == null) {
// No payment_url for submission so, broadcast and wait.
peers.startAndWait();
peers.broadcastTransaction(req.tx).get();
} else {
PaymentSession.Ack ack = future.get();
wallet.commitTx(req.tx);
System.out.println("Memo from server: " + ack.getMemo());
}
} catch (PaymentRequestException e) {
System.err.println("Failed to send payment " + e.getMessage());
System.exit(1);
} catch (VerificationException e) {
System.err.println("Failed to send payment " + e.getMessage());
System.exit(1);
} catch (ExecutionException e) {
System.err.println("Failed to send payment " + e.getMessage());
System.exit(1);
} catch (IOException e) {
System.err.println("Invalid payment " + e.getMessage());
System.exit(1);
} catch (InterruptedException e1) {
// Ignore.
} catch (InsufficientMoneyException e) {
System.err.println(
"Insufficient funds: have " + Utils.bitcoinValueToFriendlyString(wallet.getBalance()));
} catch (BlockStoreException e) {
throw new RuntimeException(e);
}
}
@Test
public void testBufferNotCloseOnFail() throws Exception {
SqlTask sqlTask = createInitialTask();
updateTask(
sqlTask,
EMPTY_SOURCES,
INITIAL_EMPTY_OUTPUT_BUFFERS.withBuffer("out", new UnpartitionedPagePartitionFunction()));
ListenableFuture<BufferResult> bufferResult =
sqlTask.getTaskResults("out", 0, new DataSize(1, MEGABYTE));
assertFalse(bufferResult.isDone());
TaskState taskState = sqlTask.getTaskInfo().getState();
sqlTask.failed(new Exception("test"));
assertEquals(
sqlTask.getTaskInfo(taskState).get(200, MILLISECONDS).getState(), TaskState.FAILED);
// buffer will not be closed by fail event. event is async so wait a bit for event to fire
try {
assertTrue(bufferResult.get(200, MILLISECONDS).isBufferClosed());
fail("expected TimeoutException");
} catch (TimeoutException expected) {
}
assertFalse(sqlTask.getTaskResults("out", 0, new DataSize(1, MEGABYTE)).isDone());
}
private static void sendPaymentRequest(String location, boolean verifyPki) {
if (location.startsWith("http") || location.startsWith("defcoin")) {
try {
ListenableFuture<PaymentSession> future;
if (location.startsWith("http")) {
future = PaymentSession.createFromUrl(location, verifyPki);
} else {
BitcoinURI paymentRequestURI = new BitcoinURI(location);
future = PaymentSession.createFromBitcoinUri(paymentRequestURI, verifyPki);
}
PaymentSession session = future.get();
if (session != null) {
send(session);
} else {
System.err.println("Server returned null session");
System.exit(1);
}
} catch (PaymentRequestException e) {
System.err.println("Error creating payment session " + e.getMessage());
System.exit(1);
} catch (BitcoinURIParseException e) {
System.err.println("Invalid defcoin uri: " + e.getMessage());
System.exit(1);
} catch (InterruptedException e) {
// Ignore.
} catch (ExecutionException e) {
throw new RuntimeException(e);
}
} else {
// Try to open the payment request as a file.
FileInputStream stream = null;
try {
File paymentRequestFile = new File(location);
stream = new FileInputStream(paymentRequestFile);
} catch (Exception e) {
System.err.println("Failed to open file: " + e.getMessage());
System.exit(1);
}
try {
paymentRequest =
org.bitcoin.protocols.payments.Protos.PaymentRequest.newBuilder()
.mergeFrom(stream)
.build();
} catch (IOException e) {
System.err.println("Failed to parse payment request from file " + e.getMessage());
System.exit(1);
}
PaymentSession session = null;
try {
session = new PaymentSession(paymentRequest, verifyPki);
} catch (PaymentRequestException e) {
System.err.println("Error creating payment session " + e.getMessage());
System.exit(1);
}
send(session);
}
}
@Nullable
@Override
public BuildResult getBuildRuleResult(BuildTarget buildTarget)
throws ExecutionException, InterruptedException {
ListenableFuture<BuildResult> result = results.get(buildTarget);
if (result == null) {
return null;
}
return result.get();
}
@Override
public ListenableFuture<ServiceFilterResponse> handleRequest(
final ServiceFilterRequest request,
final NextServiceFilterCallback nextServiceFilterCallback) {
// In this example, if authentication is already in progress we block the request
// until authentication is complete to avoid unnecessary authentications as
// a result of HTTP status code 401.
// If authentication was detected, add the token to the request.
waitAndUpdateRequestToken(request);
// Send the request down the filter chain
// retrying up to 5 times on 401 response codes.
ListenableFuture<ServiceFilterResponse> future = null;
ServiceFilterResponse response = null;
int responseCode = 401;
for (int i = 0; (i < 5) && (responseCode == 401); i++) {
future = nextServiceFilterCallback.onNext(request);
try {
response = future.get();
responseCode = response.getStatus().getStatusCode();
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
if (e.getCause().getClass() == MobileServiceException.class) {
MobileServiceException mEx = (MobileServiceException) e.getCause();
responseCode = mEx.getResponse().getStatus().getStatusCode();
if (responseCode == 401) {
// Two simultaneous requests from independent threads could get HTTP status 401.
// Protecting against that right here so multiple authentication requests are
// not setup to run on the UI thread.
// We only want to authenticate once. Requests should just wait and retry
// with the new token.
if (mAtomicAuthenticatingFlag.compareAndSet(false, true)) {
// Authenticate on UI thread
runOnUiThread(
new Runnable() {
@Override
public void run() {
// Force a token refresh during authentication.
authenticate(true);
}
});
}
// Wait for authentication to complete then update the token in the request.
waitAndUpdateRequestToken(request);
mAtomicAuthenticatingFlag.set(false);
}
}
}
}
return future;
}
@Override
public JsonObject visit(UpdateOperation operation) throws Throwable {
MobileServiceJsonTable table = this.getRemoteTable(operation.getTableName());
table.setSystemProperties(getSystemProperties(this.mItem));
ListenableFuture<JsonObject> future = table.update(this.mItem);
try {
return future.get();
} catch (ExecutionException ex) {
throw ex.getCause();
}
}
// TODO use NeedModelBuilder here instead
@RequestMapping(value = "/create", method = RequestMethod.POST)
public String createNeedPost(@ModelAttribute("SpringWeb") NeedPojo needPojo, Model model)
throws Exception {
URI needURI;
try {
URI ownerURI = this.uriService.getOwnerProtocolOwnerServiceEndpointURI();
NeedPojoNeedModelBuilder needPojoNeedModelBuilder = new NeedPojoNeedModelBuilder(needPojo);
needPojoNeedModelBuilder.setUri("no:uri");
com.hp.hpl.jena.rdf.model.Model needModel = needPojoNeedModelBuilder.build();
needModel.setNsPrefix("", "no:uri");
if (needPojo.getWonNode().equals("")) {
ListenableFuture<URI> futureResult =
ownerService.createNeed(needModel, needPojo.getState() == NeedState.ACTIVE, null);
needURI = futureResult.get();
} else {
ListenableFuture<URI> futureResult =
ownerService.createNeed(
needModel,
needPojo.getState() == NeedState.ACTIVE,
URI.create(needPojo.getWonNode()),
null);
needURI = futureResult.get();
}
List<Need> needs = needRepository.findByNeedURI(needURI);
// TODO: race condition between need saving logic and redirect. adapt interface.
if (needs.size() == 1) return "redirect:/need/" + needs.get(0).getId().toString();
// return viewNeed(need.getId().toString(), model);
} catch (IllegalNeedContentException e) {
logger.warn("caught IllegalNeedContentException:", e);
}
model.addAttribute("command", new NeedPojo());
return "createNeed";
}
public void shutdown(int exitStatusCode) {
try {
if (_systemConfig != null) {
ListenableFuture<Void> closeResult = _systemConfig.closeAsync();
closeResult.get(30000l, TimeUnit.MILLISECONDS);
}
} catch (TimeoutException | InterruptedException | ExecutionException e) {
LOGGER.warn("Attempting to cleanly shutdown took too long, exiting immediately");
} finally {
cleanUp(exitStatusCode);
}
}
@Override
public JsonObject visit(InsertOperation operation) throws Throwable {
MobileServiceJsonTable table = this.getRemoteTable(operation.getTableName());
table.setSystemProperties(EnumSet.allOf(MobileServiceSystemProperty.class));
JsonObject item = removeSystemProperties(this.mItem);
ListenableFuture<JsonObject> future = table.insert(item);
try {
return future.get();
} catch (ExecutionException ex) {
throw ex.getCause();
}
}
@Test
public void testBufferCloseOnCancel() throws Exception {
SqlTask sqlTask = createInitialTask();
updateTask(
sqlTask,
EMPTY_SOURCES,
INITIAL_EMPTY_OUTPUT_BUFFERS.withBuffer("out", new UnpartitionedPagePartitionFunction()));
ListenableFuture<BufferResult> bufferResult =
sqlTask.getTaskResults("out", 0, new DataSize(1, MEGABYTE));
assertFalse(bufferResult.isDone());
sqlTask.cancel();
assertEquals(sqlTask.getTaskInfo().getState(), TaskState.CANCELED);
// buffer will be closed by cancel event.. the event is async so wait a bit for event to
// propagate
assertTrue(bufferResult.get(200, MILLISECONDS).isBufferClosed());
bufferResult = sqlTask.getTaskResults("out", 0, new DataSize(1, MEGABYTE));
assertTrue(bufferResult.isDone());
assertTrue(bufferResult.get().isBufferClosed());
}
// Call a method that will sleep for longer than the channel timeout, and expect a
// TimeoutException (wrapped in a TTransportException)
@Test
public void testAsyncTimeout() throws Exception {
try (DelayedMap.AsyncClient client =
createClient(DelayedMap.AsyncClient.class, syncServer).get()) {
ListenableFuture<String> getFuture =
client.getValueSlowly(1500, TimeUnit.MILLISECONDS, "testKey");
try {
getFuture.get(2000, TimeUnit.MILLISECONDS);
fail("Call did not timeout as expected");
} catch (java.util.concurrent.TimeoutException e) {
fail("Waited too long for channel timeout");
} catch (ExecutionException e) {
checkTransportException(e.getCause(), ReadTimeoutException.class);
}
}
}
private ClientResponse extractWebSocketUriFromRpc(final String methodName)
throws ExecutionException, InterruptedException, UnsupportedEncodingException {
ListenableFuture<ClientResponse> clientFuture =
restconfClient.get(
ResourceUri.STREAM.getPath() + "/" + encodeUri(this.streamInfo.getIdentifier()),
MediaType.APPLICATION_XML,
new Function<ClientResponse, ClientResponse>() {
@Override
public ClientResponse apply(final ClientResponse clientResponse) {
return clientResponse;
}
});
return clientFuture.get();
}
@Test
public void requestQueuedBuilds_callback_registers_exception_on_ack_future() throws Exception {
// Setup
when(_mockRequestController.sendRequest(getApiVersion(), "buildQueue", QueuedBuildList.class))
.thenReturn(
Futures.immediateFailedFuture(new RuntimeException("Unexpected test exception")));
// Exercise
final ListenableFuture<Void> ackFuture = _apiController.requestQueuedBuilds();
// Verify
try {
ackFuture.get();
} catch (ExecutionException e) {
if (e.getCause().getClass() == RuntimeException.class
&& e.getCause().getMessage().equals("Unexpected test exception")) return;
}
TestCase.fail();
}
@Override
public JsonObject visit(DeleteOperation operation) throws Throwable {
MobileServiceJsonTable table = this.getRemoteTable(operation.getTableName());
ListenableFuture<Void> future = table.delete(this.mItem);
try {
future.get();
return null;
} catch (ExecutionException ex) {
if (!ExceptionIs404NotFound(ex)) {
throw ex.getCause();
}
return null;
}
}
private void checkItemDoesNotExists(
final DOMDataReadWriteTransaction rWTransaction,
final LogicalDatastoreType store,
final YangInstanceIdentifier path) {
final ListenableFuture<Boolean> futureDatastoreData = rWTransaction.exists(store, path);
try {
if (futureDatastoreData.get()) {
final String errMsg =
"Post Configuration via Restconf was not executed because data already exists";
LOG.trace("{}:{}", errMsg, path);
rWTransaction.cancel();
throw new RestconfDocumentedException(
"Data already exists for path: " + path, ErrorType.PROTOCOL, ErrorTag.DATA_EXISTS);
}
} catch (InterruptedException | ExecutionException e) {
LOG.warn("It wasn't possible to get data loaded from datastore at path {}", path, e);
}
}
@Test
public void testEmptyPath() throws Exception {
ImmutableMap<String, List<FileStatus>> paths =
ImmutableMap.<String, List<FileStatus>>builder()
.put("/", ImmutableList.<FileStatus>of())
.build();
AsyncRecursiveWalker walker =
new AsyncRecursiveWalker(createMockFileSystem(paths), MoreExecutors.sameThreadExecutor());
MockFileStatusCallback callback = new MockFileStatusCallback();
ListenableFuture<Void> listenableFuture = walker.beginWalk(new Path("/"), callback);
Assert.assertTrue(listenableFuture.isDone());
Assert.assertTrue(callback.getProcessedFiles().isEmpty());
// Should not have an exception
listenableFuture.get();
}
@Test
public void testHiddenFiles() throws Exception {
ImmutableMap<String, List<FileStatus>> paths =
ImmutableMap.<String, List<FileStatus>>builder()
.put(
"/",
ImmutableList.of(
fileStatus("/.a", true),
fileStatus("/_b", true),
fileStatus("/c", true),
fileStatus("/file1", false),
fileStatus("/_file2", false),
fileStatus("/.file3", false)))
.put(
"/.a",
ImmutableList.of(fileStatus("/.a/file4", false), fileStatus("/.a/file5", false)))
.put(
"/_b",
ImmutableList.of(fileStatus("/_b/file6", false), fileStatus("/_b/file7", false)))
.put(
"/c",
ImmutableList.of(
fileStatus("/c/file8", false),
fileStatus("/c/.file9", false),
fileStatus("/c/_file10", false)))
.build();
AsyncRecursiveWalker walker =
new AsyncRecursiveWalker(createMockFileSystem(paths), MoreExecutors.sameThreadExecutor());
MockFileStatusCallback callback = new MockFileStatusCallback();
ListenableFuture<Void> listenableFuture = walker.beginWalk(new Path("/"), callback);
Assert.assertTrue(listenableFuture.isDone());
Assert.assertEquals(
ImmutableSet.copyOf(callback.getProcessedFiles()), ImmutableSet.of("/file1", "/c/file8"));
// Should not have an exception
listenableFuture.get();
}
private List<Bucket> getBuckets(CollectPhase collectNode)
throws InterruptedException, java.util.concurrent.ExecutionException {
List<Bucket> results = new ArrayList<>();
for (String nodeName : internalCluster().getNodeNames()) {
ContextPreparer contextPreparer =
internalCluster().getInstance(ContextPreparer.class, nodeName);
JobContextService contextService =
internalCluster().getInstance(JobContextService.class, nodeName);
JobExecutionContext.Builder builder = contextService.newBuilder(collectNode.jobId());
ListenableFuture<Bucket> future =
contextPreparer.prepare(
collectNode.jobId(),
NodeOperation.withDownstream(collectNode, mock(ExecutionPhase.class)),
builder);
assert future != null;
JobExecutionContext context = contextService.createContext(builder);
context.start();
results.add(future.get());
}
return results;
}
@Test
public void requestLastBuildStatus_callback_registers_exception_on_ack_future() throws Exception {
// Setup
final BuildTypeData bt1 = new BuildTypeData("bt1", "btName", "pname", "pId");
when(_mockRequestController.sendRequest(
getApiVersion(),
"builds/?locator=buildType:bt1,running:any,branch:(default:any),count:"
+ ApiController.MAX_BUILDS_TO_CONSIDER,
BuildList.class))
.thenReturn(
Futures.immediateFailedFuture(new RuntimeException("Unexpected test exception")));
// Exercise
final ListenableFuture<Void> ackFuture = _apiController.requestLastBuildStatus(bt1);
// Verify
try {
ackFuture.get();
} catch (ExecutionException e) {
if (e.getCause().getClass() == RuntimeException.class
&& e.getCause().getMessage().equals("Unexpected test exception")) return;
}
TestCase.fail();
}
private void createAndRegisterAckMailboxes(
final Set<Integer> localPartitions, HostMessenger messenger) {
m_zk = messenger.getZK();
m_mailboxesZKPath = VoltZK.exportGenerations + "/" + m_timestamp + "/" + "mailboxes";
m_mbox =
new LocalMailbox(messenger) {
@Override
public void deliver(VoltMessage message) {
if (message instanceof BinaryPayloadMessage) {
BinaryPayloadMessage bpm = (BinaryPayloadMessage) message;
ByteBuffer buf = ByteBuffer.wrap(bpm.m_payload);
final int partition = buf.getInt();
final int length = buf.getInt();
byte stringBytes[] = new byte[length];
buf.get(stringBytes);
String signature = new String(stringBytes, Constants.UTF8ENCODING);
final long ackUSO = buf.getLong();
final HashMap<String, ExportDataSource> partitionSources =
m_dataSourcesByPartition.get(partition);
if (partitionSources == null) {
exportLog.error(
"Received an export ack for partition "
+ partition
+ " which does not exist on this node");
return;
}
final ExportDataSource eds = partitionSources.get(signature);
if (eds == null) {
exportLog.error(
"Received an export ack for partition "
+ partition
+ " source signature "
+ signature
+ " which does not exist on this node");
return;
}
try {
eds.ack(ackUSO);
} catch (RejectedExecutionException ignoreIt) {
// ignore it: as it is already shutdown
}
} else {
exportLog.error("Receive unexpected message " + message + " in export subsystem");
}
}
};
messenger.createMailbox(null, m_mbox);
for (Integer partition : localPartitions) {
final String partitionDN = m_mailboxesZKPath + "/" + partition;
ZKUtil.asyncMkdirs(m_zk, partitionDN);
ZKUtil.StringCallback cb = new ZKUtil.StringCallback();
m_zk.create(
partitionDN + "/" + m_mbox.getHSId(),
null,
Ids.OPEN_ACL_UNSAFE,
CreateMode.EPHEMERAL,
cb,
null);
}
ListenableFuture<?> fut =
m_childUpdatingThread.submit(
new Runnable() {
@Override
public void run() {
List<Pair<Integer, ZKUtil.ChildrenCallback>> callbacks =
new ArrayList<Pair<Integer, ZKUtil.ChildrenCallback>>();
for (Integer partition : localPartitions) {
ZKUtil.ChildrenCallback callback = new ZKUtil.ChildrenCallback();
m_zk.getChildren(
m_mailboxesZKPath + "/" + partition,
constructMailboxChildWatcher(),
callback,
null);
callbacks.add(Pair.of(partition, callback));
}
for (Pair<Integer, ZKUtil.ChildrenCallback> p : callbacks) {
final Integer partition = p.getFirst();
List<String> children = null;
try {
children = p.getSecond().getChildren();
} catch (InterruptedException e) {
Throwables.propagate(e);
} catch (KeeperException e) {
Throwables.propagate(e);
}
ImmutableList.Builder<Long> mailboxes = ImmutableList.builder();
for (String child : children) {
if (child.equals(Long.toString(m_mbox.getHSId()))) continue;
mailboxes.add(Long.valueOf(child));
}
ImmutableList<Long> mailboxHsids = mailboxes.build();
for (ExportDataSource eds : m_dataSourcesByPartition.get(partition).values()) {
eds.updateAckMailboxes(Pair.of(m_mbox, mailboxHsids));
}
}
}
});
try {
fut.get();
} catch (Throwable t) {
Throwables.propagate(t);
}
}
// since surefire and eclipse don't otherwise guarantee the order, we are
// starting this one alphabetically before create2nodes..
@Test(
enabled = true,
dependsOnMethods = {"testCompareSizes"})
public void testAScriptExecutionAfterBootWithBasicTemplate() throws Exception {
String group = this.group + "r";
try {
client.destroyNodesMatching(inGroup(group));
} catch (Exception e) {
}
template = buildTemplate(client.templateBuilder());
template.getOptions().blockOnPort(22, 120);
try {
Set<? extends NodeMetadata> nodes = client.createNodesInGroup(group, 1, template);
NodeMetadata node = get(nodes, 0);
LoginCredentials good = node.getCredentials();
assert good.identity != null : nodes;
for (Entry<? extends NodeMetadata, ExecResponse> response :
client
.runScriptOnNodesMatching(
runningInGroup(group),
"hostname",
wrapInInitScript(false).runAsRoot(false).overrideLoginCredentials(good))
.entrySet()) {
checkResponseEqualsHostname(response.getValue(), response.getKey());
}
// test single-node execution
ExecResponse response =
client.runScriptOnNode(
node.getId(), "hostname", wrapInInitScript(false).runAsRoot(false));
checkResponseEqualsHostname(response, node);
OperatingSystem os = node.getOperatingSystem();
// test bad password
tryBadPassword(group, good);
runScriptWithCreds(group, os, good);
checkNodes(nodes, group, "runScriptWithCreds");
// test adding AdminAccess later changes the default boot user, in this
// case to foo, with home dir /over/ridden/foo
ListenableFuture<ExecResponse> future =
client.submitScriptOnNode(
node.getId(),
AdminAccess.builder().adminUsername("foo").adminHome("/over/ridden/foo").build(),
nameTask("adminUpdate"));
response = future.get(3, TimeUnit.MINUTES);
assert response.getExitStatus() == 0 : node.getId() + ": " + response;
node = client.getNodeMetadata(node.getId());
// test that the node updated to the correct admin user!
assertEquals(node.getCredentials().identity, "foo");
assert node.getCredentials().credential != null : nodes;
weCanCancelTasks(node);
assert response.getExitStatus() == 0 : node.getId() + ": " + response;
response =
client.runScriptOnNode(
node.getId(), "echo $USER", wrapInInitScript(false).runAsRoot(false));
assert response.getOutput().trim().equals("foo") : node.getId() + ": " + response;
} finally {
client.destroyNodesMatching(inGroup(group));
}
}
private void eq(int i, ListenableFuture<IntegerReply> zadd)
throws ExecutionException, InterruptedException {
eq(i, zadd.get());
}
@SuppressWarnings("PMD.EmptyCatchBlock")
public static int runTests(
final CommandRunnerParams params,
Iterable<TestRule> tests,
BuildContext buildContext,
ExecutionContext executionContext,
final TestRunningOptions options,
ListeningExecutorService service,
BuildEngine buildEngine,
final StepRunner stepRunner)
throws IOException, ExecutionException, InterruptedException {
if (options.isUsingOneTimeOutputDirectories()) {
BuckConstant.setOneTimeTestSubdirectory(UUID.randomUUID().toString());
}
ImmutableSet<JavaLibrary> rulesUnderTest;
// If needed, we first run instrumentation on the class files.
if (options.isCodeCoverageEnabled()) {
rulesUnderTest = getRulesUnderTest(tests);
if (!rulesUnderTest.isEmpty()) {
try {
stepRunner.runStepForBuildTarget(
new MakeCleanDirectoryStep(JUnitStep.JACOCO_OUTPUT_DIR),
Optional.<BuildTarget>absent());
} catch (StepFailedException e) {
params.getConsole().printBuildFailureWithoutStacktrace(e);
return 1;
}
}
} else {
rulesUnderTest = ImmutableSet.of();
}
final ImmutableSet<String> testTargets =
FluentIterable.from(tests)
.transform(HasBuildTarget.TO_TARGET)
.transform(Functions.toStringFunction())
.toSet();
final int totalNumberOfTests = Iterables.size(tests);
params
.getBuckEventBus()
.post(
TestRunEvent.started(
options.isRunAllTests(),
options.getTestSelectorList(),
options.shouldExplainTestSelectorList(),
testTargets));
// Start running all of the tests. The result of each java_test() rule is represented as a
// ListenableFuture.
List<ListenableFuture<TestResults>> results = Lists.newArrayList();
// Unless `--verbose 0` is specified, print out test results as they become available.
// Failures with the ListenableFuture should always be printed, as they indicate an error with
// Buck, not the test being run.
Verbosity verbosity = params.getConsole().getVerbosity();
final boolean printTestResults = (verbosity != Verbosity.SILENT);
// For grouping results!
final TestResultsGrouper grouper;
if (options.isIgnoreFailingDependencies()) {
grouper = new TestResultsGrouper(tests);
} else {
grouper = null;
}
TestRuleKeyFileHelper testRuleKeyFileHelper =
new TestRuleKeyFileHelper(executionContext.getProjectFilesystem(), buildEngine);
final AtomicInteger lastReportedTestSequenceNumber = new AtomicInteger();
final List<TestRun> separateTestRuns = Lists.newArrayList();
List<TestRun> parallelTestRuns = Lists.newArrayList();
for (final TestRule test : tests) {
// Determine whether the test needs to be executed.
boolean isTestRunRequired;
isTestRunRequired =
isTestRunRequiredForTest(
test,
buildEngine,
executionContext,
testRuleKeyFileHelper,
options.isResultsCacheEnabled(),
!options.getTestSelectorList().isEmpty());
List<Step> steps;
if (isTestRunRequired) {
params.getBuckEventBus().post(IndividualTestEvent.started(testTargets));
ImmutableList.Builder<Step> stepsBuilder = ImmutableList.builder();
Preconditions.checkState(buildEngine.isRuleBuilt(test.getBuildTarget()));
final Map<String, UUID> testUUIDMap = new HashMap<>();
List<Step> testSteps =
test.runTests(
buildContext,
executionContext,
options.isDryRun(),
options.isShufflingTests(),
options.getTestSelectorList(),
new TestRule.TestReportingCallback() {
@Override
public void testsDidBegin() {
LOG.debug("Tests for rule %s began", test.getBuildTarget());
}
@Override
public void testDidBegin(String testCaseName, String testName) {
LOG.debug(
"Test rule %s test case %s test name %s began",
test.getBuildTarget(), testCaseName, testName);
UUID testUUID = UUID.randomUUID();
// UUID is immutable and thread-safe as of Java 7, so it's
// safe to stash in a map and use later:
//
// http://bugs.java.com/view_bug.do?bug_id=6611830
testUUIDMap.put(testCaseName + ":" + testName, testUUID);
params
.getBuckEventBus()
.post(TestSummaryEvent.started(testUUID, testCaseName, testName));
}
@Override
public void testDidEnd(TestResultSummary testResultSummary) {
LOG.debug(
"Test rule %s test did end: %s", test.getBuildTarget(), testResultSummary);
UUID testUUID =
testUUIDMap.get(
testResultSummary.getTestCaseName()
+ ":"
+ testResultSummary.getTestName());
Preconditions.checkNotNull(testUUID);
params
.getBuckEventBus()
.post(TestSummaryEvent.finished(testUUID, testResultSummary));
}
@Override
public void testsDidEnd(List<TestCaseSummary> testCaseSummaries) {
LOG.debug(
"Test rule %s tests did end: %s", test.getBuildTarget(), testCaseSummaries);
}
});
if (!testSteps.isEmpty()) {
stepsBuilder.addAll(testSteps);
stepsBuilder.add(testRuleKeyFileHelper.createRuleKeyInDirStep(test));
}
steps = stepsBuilder.build();
} else {
steps = ImmutableList.of();
}
TestRun testRun =
TestRun.of(
test,
steps,
getCachingStatusTransformingCallable(
isTestRunRequired,
test.interpretTestResults(
executionContext,
/*isUsingTestSelectors*/ !options.getTestSelectorList().isEmpty(),
/*isDryRun*/ options.isDryRun())));
// Always run the commands, even if the list of commands as empty. There may be zero
// commands because the rule is cached, but its results must still be processed.
if (test.runTestSeparately()) {
LOG.debug("Running test %s in serial", test);
separateTestRuns.add(testRun);
} else {
LOG.debug("Running test %s in parallel", test);
parallelTestRuns.add(testRun);
}
}
final StepRunner.StepRunningCallback testStepRunningCallback =
new StepRunner.StepRunningCallback() {
@Override
public void stepsWillRun(Optional<BuildTarget> buildTarget) {
Preconditions.checkState(buildTarget.isPresent());
LOG.debug("Test steps will run for %s", buildTarget);
params.getBuckEventBus().post(TestRuleEvent.started(buildTarget.get()));
}
@Override
public void stepsDidRun(Optional<BuildTarget> buildTarget) {
Preconditions.checkState(buildTarget.isPresent());
LOG.debug("Test steps did run for %s", buildTarget);
params.getBuckEventBus().post(TestRuleEvent.finished(buildTarget.get()));
}
};
for (TestRun testRun : parallelTestRuns) {
ListenableFuture<TestResults> testResults =
stepRunner.runStepsAndYieldResult(
testRun.getSteps(),
testRun.getTestResultsCallable(),
Optional.of(testRun.getTest().getBuildTarget()),
service,
testStepRunningCallback);
results.add(
transformTestResults(
params,
testResults,
grouper,
testRun.getTest(),
testTargets,
printTestResults,
lastReportedTestSequenceNumber,
totalNumberOfTests));
}
ListenableFuture<List<TestResults>> parallelTestStepsFuture = Futures.allAsList(results);
final List<TestResults> completedResults = Lists.newArrayList();
final ListeningExecutorService directExecutorService = MoreExecutors.newDirectExecutorService();
ListenableFuture<Void> uberFuture =
stepRunner.addCallback(
parallelTestStepsFuture,
new FutureCallback<List<TestResults>>() {
@Override
public void onSuccess(List<TestResults> parallelTestResults) {
LOG.debug("Parallel tests completed, running separate tests...");
completedResults.addAll(parallelTestResults);
List<ListenableFuture<TestResults>> separateResultsList = Lists.newArrayList();
for (TestRun testRun : separateTestRuns) {
separateResultsList.add(
transformTestResults(
params,
stepRunner.runStepsAndYieldResult(
testRun.getSteps(),
testRun.getTestResultsCallable(),
Optional.of(testRun.getTest().getBuildTarget()),
directExecutorService,
testStepRunningCallback),
grouper,
testRun.getTest(),
testTargets,
printTestResults,
lastReportedTestSequenceNumber,
totalNumberOfTests));
}
ListenableFuture<List<TestResults>> serialResults =
Futures.allAsList(separateResultsList);
try {
completedResults.addAll(serialResults.get());
} catch (ExecutionException e) {
LOG.error(e, "Error fetching serial test results");
throw new HumanReadableException(e, "Error fetching serial test results");
} catch (InterruptedException e) {
LOG.error(e, "Interrupted fetching serial test results");
try {
serialResults.cancel(true);
} catch (CancellationException ignored) {
// Rethrow original InterruptedException instead.
}
Thread.currentThread().interrupt();
throw new HumanReadableException(e, "Test cancelled");
}
LOG.debug("Done running serial tests.");
}
@Override
public void onFailure(Throwable e) {
LOG.error(e, "Parallel tests failed, not running serial tests");
throw new HumanReadableException(e, "Parallel tests failed");
}
},
directExecutorService);
try {
// Block until all the tests have finished running.
uberFuture.get();
} catch (ExecutionException e) {
e.printStackTrace(params.getConsole().getStdErr());
return 1;
} catch (InterruptedException e) {
try {
uberFuture.cancel(true);
} catch (CancellationException ignored) {
// Rethrow original InterruptedException instead.
}
Thread.currentThread().interrupt();
throw e;
}
params.getBuckEventBus().post(TestRunEvent.finished(testTargets, completedResults));
// Write out the results as XML, if requested.
Optional<String> path = options.getPathToXmlTestOutput();
if (path.isPresent()) {
try (Writer writer = Files.newWriter(new File(path.get()), Charsets.UTF_8)) {
writeXmlOutput(completedResults, writer);
}
}
// Generate the code coverage report.
if (options.isCodeCoverageEnabled() && !rulesUnderTest.isEmpty()) {
try {
Optional<DefaultJavaPackageFinder> defaultJavaPackageFinderOptional =
Optional.fromNullable(params.getBuckConfig().createDefaultJavaPackageFinder());
stepRunner.runStepForBuildTarget(
getReportCommand(
rulesUnderTest,
defaultJavaPackageFinderOptional,
params.getRepository().getFilesystem(),
JUnitStep.JACOCO_OUTPUT_DIR,
options.getCoverageReportFormat()),
Optional.<BuildTarget>absent());
} catch (StepFailedException e) {
params.getConsole().printBuildFailureWithoutStacktrace(e);
return 1;
}
}
boolean failures =
Iterables.any(
completedResults,
new Predicate<TestResults>() {
@Override
public boolean apply(TestResults results) {
LOG.debug("Checking result %s for failure", results);
return !results.isSuccess();
}
});
return failures ? TEST_FAILURES_EXIT_CODE : 0;
}
