public static <T> Map<T, Exception> awaitCompletion(
Map<T, ? extends Future<?>> responses,
ExecutorService exec,
@Nullable Long maxTime,
final Logger logger,
final String logPrefix) {
if (responses.size() == 0) return ImmutableMap.of();
final int total = responses.size();
final CountDownLatch doneSignal = new CountDownLatch(total);
final AtomicInteger complete = new AtomicInteger(0);
final AtomicInteger errors = new AtomicInteger(0);
final long start = System.currentTimeMillis();
final Map<T, Exception> errorMap = Maps.newHashMap();
for (final java.util.Map.Entry<T, ? extends Future<?>> future : responses.entrySet()) {
Futures.makeListenable(future.getValue(), exec)
.addListener(
new Runnable() {
@Override
public void run() {
try {
future.getValue().get();
complete.incrementAndGet();
} catch (Exception e) {
errors.incrementAndGet();
logException(logger, logPrefix, total, complete.get(), errors.get(), start, e);
errorMap.put(future.getKey(), e);
}
doneSignal.countDown();
}
@Override
public String toString() {
return "callGetOnFuture(" + future.getKey() + "," + future.getValue() + ")";
}
},
exec);
}
try {
if (maxTime != null) doneSignal.await(maxTime, TimeUnit.MILLISECONDS);
else doneSignal.await();
if (errors.get() > 0) {
String message = message(logPrefix, total, complete.get(), errors.get(), start);
RuntimeException exception = new RuntimeException(message);
logger.error(exception, message);
}
if (logger.isTraceEnabled()) {
String message = message(logPrefix, total, complete.get(), errors.get(), start);
logger.trace(message);
}
} catch (InterruptedException e) {
String message = message(logPrefix, total, complete.get(), errors.get(), start);
TimeoutException exception = new TimeoutException(message);
logger.error(exception, message);
Throwables.propagate(exception);
}
return errorMap;
}
@Override
public Server apply(@Nullable String baseResource) {
if (!jetty.getState().equals(Server.STARTED)
// TODO code smell = hard coding addresses or ports!!
&& !new InetSocketAddressConnect().apply(new IPSocket("localhost", port))) {
ResourceHandler resource_handler = new ResourceHandler();
resource_handler.setDirectoriesListed(true);
resource_handler.setWelcomeFiles(new String[] {"index.html"});
resource_handler.setResourceBase(baseResource);
logger.info("serving " + resource_handler.getBaseResource());
HandlerList handlers = new HandlerList();
handlers.setHandlers(new Handler[] {resource_handler, new DefaultHandler()});
jetty.setHandler(handlers);
try {
jetty.start();
} catch (Exception e) {
logger.error(e, "Server jetty could not be started at this %s", baseResource);
}
return jetty;
} else {
logger.debug("Server jetty serving %s already running. Skipping start", baseResource);
return jetty;
}
}
@Override
public Void apply(IMachine machine) {
List<IMedium> mediaToBeDeleted = ImmutableList.of();
try {
mediaToBeDeleted = machine.unregister(vmSpec.getCleanupMode());
} catch (VBoxException e) {
ErrorCode errorCode = ErrorCode.valueOf(e);
switch (errorCode) {
case VBOX_E_OBJECT_NOT_FOUND:
logger.debug("Machine %s does not exists, cannot unregister", vmSpec.getVmName());
break;
default:
throw e;
}
}
List<IMedium> filteredMediaToBeDeleted =
Lists.newArrayList(
transform(
filter(mediaToBeDeleted, new AutoDeleteHardDiskPredicate(vmSpec)),
new DeleteChildrenOfMedium()));
if (!filteredMediaToBeDeleted.isEmpty()) {
try {
IProgress deletion = machine.delete(filteredMediaToBeDeleted);
deletion.waitForCompletion(-1);
} catch (Exception e) {
logger.error(e, "Problem in deleting the media attached to %s", machine.getName());
Throwables.propagate(e);
}
}
return null;
}
public void deleteDirectory(String container, String directory) {
// create complete dir path
String fullDirPath = buildPathStartingFromBaseDir(container, directory);
try {
Utils.deleteRecursively(new File(fullDirPath));
} catch (IOException ex) {
logger.error("An error occurred removing directory %s.", fullDirPath);
Throwables.propagate(ex);
}
}
@VisibleForTesting
SshException propagate(Exception e, String message) {
message += ": " + e.getMessage();
logger.error(e, "<< " + message);
if (e instanceof UserAuthException)
throw new AuthorizationException("(" + toString() + ") " + message, e);
throw e instanceof SshException
? SshException.class.cast(e)
: new SshException("(" + toString() + ") " + message, e);
}
private static void logException(
Logger logger,
String logPrefix,
int total,
int complete,
int errors,
long start,
Exception e) {
String message = message(logPrefix, total, complete, errors, start);
logger.error(e, message);
}
@Override
public boolean blobExists(String container, String key) {
filesystemContainerNameValidator.validate(container);
filesystemBlobKeyValidator.validate(key);
try {
return buildPathAndChecksIfBlobExists(container, key);
} catch (IOException e) {
logger.error(e, "An error occurred while checking key %s in container %s", container, key);
throw Throwables.propagate(e);
}
}
/**
* Calculates the object MD5 and returns it as eTag
*
* @param object
* @return
*/
private String getEtag(Blob object) {
try {
Payloads.calculateMD5(object, crypto.md5());
} catch (IOException ex) {
logger.error(
ex,
"An error occurred calculating MD5 for object with name %s.",
object.getMetadata().getName());
Throwables.propagate(ex);
}
String eTag = CryptoStreams.hex(object.getPayload().getContentMetadata().getContentMD5());
return eTag;
}
protected <T, C extends Connection<T>> T acquire(C connection) {
String errorMessage = String.format("(%s) error acquiring %s", toString(), connection);
for (int i = 0; i < sshRetries; i++) {
try {
connection.clear();
logger.debug(">> (%s) acquiring %s", toString(), connection);
T returnVal = connection.create();
logger.debug("<< (%s) acquired %s", toString(), returnVal);
return returnVal;
} catch (Exception from) {
try {
connection.clear();
} catch (Exception e1) {
logger.warn(from, "<< (%s) error closing connection", toString());
}
if (i + 1 == sshRetries) {
logger.error(from, "<< " + errorMessage + ": out of retries %d", sshRetries);
throw propagate(from, errorMessage);
} else if (Throwables2.getFirstThrowableOfType(from, IllegalStateException.class) != null) {
logger.warn(from, "<< " + errorMessage + ": " + from.getMessage());
disconnect();
backoffForAttempt(i + 1, errorMessage + ": " + from.getMessage());
connect();
continue;
} else if (shouldRetry(from)) {
logger.warn(from, "<< " + errorMessage + ": " + from.getMessage());
backoffForAttempt(i + 1, errorMessage + ": " + from.getMessage());
continue;
} else {
logger.error(from, "<< " + errorMessage + ": exception not retryable");
throw propagate(from, errorMessage);
}
}
}
assert false : "should not reach here";
return null;
}
/**
* Load the blob with the given key belonging to the container with the given name. There must
* exist a resource on the file system whose complete name is given concatenating the container
* name and the key
*
* @param container it's the name of the container the blob belongs to
* @param key it's the key of the blob
* @return the blob belonging to the given container with the given key
*/
private Blob loadFileBlob(String container, String key) {
logger.debug("Opening blob in container: %s - %s", container, key);
BlobBuilder builder = blobUtils.blobBuilder();
builder.name(key);
File file = storageStrategy.getFileForBlobKey(container, key);
try {
builder.payload(file).calculateMD5();
} catch (IOException e) {
logger.error("An error occurred calculating MD5 for blob %s from container ", key, container);
Throwables.propagateIfPossible(e);
}
Blob blob = builder.build();
if (blob.getPayload().getContentMetadata().getContentMD5() != null)
blob.getMetadata()
.setETag(CryptoStreams.hex(blob.getPayload().getContentMetadata().getContentMD5()));
return blob;
}
@Override
public void clearContainer(String container, ListContainerOptions options) {
filesystemContainerNameValidator.validate(container);
if (options.getDir() != null) {
container += denormalize("/" + options.getDir());
}
try {
File containerFile = openFolder(container);
File[] children = containerFile.listFiles();
if (null != children) {
for (File child : children)
if (options.isRecursive() || child.isFile()) {
Utils.deleteRecursively(child);
}
}
} catch (IOException e) {
logger.error(e, "An error occurred while clearing container %s", container);
Throwables.propagate(e);
}
}
/** {@inheritDoc} */
@Override
public ListenableFuture<String> putBlob(String containerName, Blob object) {
String blobKey = object.getMetadata().getName();
logger.debug("Put object with key [%s] to container [%s]", blobKey, containerName);
String eTag = getEtag(object);
try {
// TODO
// must override existing file?
storageStrategy.writePayloadOnFile(containerName, blobKey, object.getPayload());
} catch (IOException e) {
logger.error(
e,
"An error occurred storing the new object with name [%s] to container [%s].",
blobKey,
containerName);
Throwables.propagate(e);
}
return immediateFuture(eTag);
}
/** default maxResults is 1000 */
@Override
public ListenableFuture<PageSet<? extends StorageMetadata>> list(
final String container, ListContainerOptions options) {
// Check if the container exists
if (!containerExistsSyncImpl(container)) {
return immediateFailedFuture(cnfe(container));
}
// Loading blobs from container
Iterable<String> blobBelongingToContainer = null;
try {
blobBelongingToContainer = storageStrategy.getBlobKeysInsideContainer(container);
} catch (IOException e) {
logger.error(e, "An error occurred loading blobs contained into container %s", container);
Throwables.propagate(e);
}
SortedSet<StorageMetadata> contents =
newTreeSet(
transform(
blobBelongingToContainer,
new Function<String, StorageMetadata>() {
public StorageMetadata apply(String key) {
Blob oldBlob = loadFileBlob(container, key);
checkState(
oldBlob != null,
"blob "
+ key
+ " is not present although it was in the list of "
+ container);
checkState(
oldBlob.getMetadata() != null,
"blob " + container + "/" + key + " has no metadata");
MutableBlobMetadata md = copy(oldBlob.getMetadata());
String directoryName = ifDirectoryReturnName.execute(md);
if (directoryName != null) {
md.setName(directoryName);
md.setType(StorageType.RELATIVE_PATH);
}
return md;
}
}));
String marker = null;
if (options != null) {
if (options.getMarker() != null) {
final String finalMarker = options.getMarker();
StorageMetadata lastMarkerMetadata =
find(
contents,
new Predicate<StorageMetadata>() {
public boolean apply(StorageMetadata metadata) {
return metadata.getName().equals(finalMarker);
}
});
contents = contents.tailSet(lastMarkerMetadata);
contents.remove(lastMarkerMetadata);
}
final String prefix = options.getDir();
if (prefix != null) {
contents =
newTreeSet(
filter(
contents,
new Predicate<StorageMetadata>() {
public boolean apply(StorageMetadata o) {
return (o != null
&& o.getName().startsWith(prefix)
&& !o.getName().equals(prefix));
}
}));
}
Integer maxResults = options.getMaxResults() != null ? options.getMaxResults() : 1000;
if (contents.size() > 0) {
SortedSet<StorageMetadata> contentsSlice = firstSliceOfSize(contents, maxResults);
if (!contentsSlice.contains(contents.last())) {
// Partial listing
marker = contentsSlice.last().getName();
} else {
marker = null;
}
contents = contentsSlice;
}
final String delimiter = options.isRecursive() ? null : File.separator;
if (delimiter != null) {
SortedSet<String> commonPrefixes = null;
Iterable<String> iterable =
transform(contents, new CommonPrefixes(prefix != null ? prefix : null, delimiter));
commonPrefixes = iterable != null ? newTreeSet(iterable) : new TreeSet<String>();
commonPrefixes.remove(CommonPrefixes.NO_PREFIX);
contents =
newTreeSet(
filter(contents, new DelimiterFilter(prefix != null ? prefix : null, delimiter)));
Iterables.<StorageMetadata>addAll(
contents,
transform(
commonPrefixes,
new Function<String, StorageMetadata>() {
public StorageMetadata apply(String o) {
MutableStorageMetadata md = new MutableStorageMetadataImpl();
md.setType(StorageType.RELATIVE_PATH);
md.setName(o);
return md;
}
}));
}
// trim metadata, if the response isn't supposed to be detailed.
if (!options.isDetailed()) {
for (StorageMetadata md : contents) {
md.getUserMetadata().clear();
}
}
}
return Futures.<PageSet<? extends StorageMetadata>>immediateFuture(
new PageSetImpl<StorageMetadata>(contents, marker));
}
@Override
public ISnapshot apply(IMachine machine) {
// Snapshot a machine
ISession session = null;
ISnapshot snap = machine.getCurrentSnapshot();
if (snap == null) {
try {
session = manager.get().openMachineSession(machine);
int retries = 10;
while (true) {
try {
// running machines need to be pause before a snapshot can be taken
// due to a vbox bug see https://www.virtualbox.org/ticket/9255
boolean paused = false;
if (machine.getState() == MachineState.Running) {
session.getConsole().pause();
paused = true;
}
IProgress progress = session.getConsole().takeSnapshot(snapshotName, snapshotDesc);
progress.waitForCompletion(-1);
if (paused) {
session.getConsole().resume();
}
snap = machine.getCurrentSnapshot();
logger.debug(
"<< snapshot(%s) with description(%s) taken from master(%s)",
snapshotName, snapshotDesc, machine.getName());
break;
} catch (Exception e) {
if (e.getMessage().contains("VirtualBox error: The object is not ready")
|| e.getMessage().contains("This machine does not have any snapshots")) {
retries--;
if (retries == 0) {
logger.error(
e,
"Problem creating snapshot (too many retries) %s (description: %s) from machine %s",
snapshotName,
snapshotDesc,
machine.getName());
throw Throwables.propagate(e);
}
Uninterruptibles.sleepUninterruptibly(1, TimeUnit.SECONDS);
continue;
}
logger.error(
e,
"Problem creating snapshot %s (description: %s) from machine %s",
snapshotName,
snapshotDesc,
machine.getName());
throw Throwables.propagate(e);
}
}
} catch (Exception e) {
Throwables.propagate(e);
} finally {
if (session != null) {
manager.get().closeMachineSession(session);
}
}
}
return snap;
}
protected NodeAndInitialCredentials<ServerInDataCenter> createNodeWithGroupEncodedIntoName(
String group, String name, TemplateWithDataCenter template) {
checkArgument(
template.getLocation().getScope() == LocationScope.ZONE,
"Template must use a ZONE-scoped location");
final String dataCenterId = template.getDataCenter().id();
Hardware hardware = template.getHardware();
TemplateOptions options = template.getOptions();
final String loginUser =
isNullOrEmpty(options.getLoginUser()) ? "root" : options.getLoginUser();
final String password =
options.hasLoginPassword() ? options.getLoginPassword() : Passwords.generate();
final org.jclouds.compute.domain.Image image = template.getImage();
// provision all volumes based on hardware
List<? extends Volume> volumes = hardware.getVolumes();
List<String> volumeIds = Lists.newArrayListWithExpectedSize(volumes.size());
int i = 1;
for (final Volume volume : volumes) {
try {
logger.trace("<< provisioning volume '%s'", volume);
final org.apache.jclouds.profitbricks.rest.domain.Volume.Request.CreatePayload.Builder
request = org.apache.jclouds.profitbricks.rest.domain.Volume.Request.creatingBuilder();
if (i == 1) {
request.image(image.getId());
// we don't need to pass password to the API if we're using a snapshot
Provisionable.Type provisionableType =
Provisionable.Type.fromValue(
image.getUserMetadata().get(ProvisionableToImage.KEY_PROVISIONABLE_TYPE));
if (provisionableType == Provisionable.Type.IMAGE) {
request.imagePassword(password);
}
}
request
.dataCenterId(dataCenterId)
.name(format("%s-disk-%d", name, i++))
.size(volume.getSize().intValue())
.type(VolumeType.HDD);
org.apache.jclouds.profitbricks.rest.domain.Volume vol =
(org.apache.jclouds.profitbricks.rest.domain.Volume)
provisioningManager.provision(
jobFactory.create(
dataCenterId,
new Supplier<Object>() {
@Override
public Object get() {
return api.volumeApi().createVolume(request.build());
}
}));
volumeIds.add(vol.id());
logger.trace(">> provisioning complete for volume. returned id='%s'", vol.id());
} catch (Exception ex) {
if (i - 1 == 1) // if first volume (one with image) provisioning fails; stop method
{
throw Throwables.propagate(ex);
}
logger.warn(ex, ">> failed to provision volume. skipping..");
}
}
String volumeBootDeviceId = Iterables.get(volumeIds, 0); // must have atleast 1
waitVolumeUntilAvailable.apply(VolumeRef.create(dataCenterId, volumeBootDeviceId));
// provision server
final Server server;
Double cores = ComputeServiceUtils.getCores(hardware);
Server.BootVolume bootVolume = Server.BootVolume.create(volumeBootDeviceId);
try {
final Server.Request.CreatePayload serverRequest =
Server.Request.creatingBuilder()
.dataCenterId(dataCenterId)
.name(name)
.bootVolume(bootVolume)
.cores(cores.intValue())
.ram(hardware.getRam())
.build();
logger.trace("<< provisioning server '%s'", serverRequest);
server =
(Server)
provisioningManager.provision(
jobFactory.create(
dataCenterId,
new Supplier<Object>() {
@Override
public Object get() {
return api.serverApi().createServer(serverRequest);
}
}));
logger.trace(">> provisioning complete for server. returned id='%s'", server.id());
} catch (Exception ex) {
logger.error(ex, ">> failed to provision server. rollbacking..");
destroyVolumes(volumeIds, dataCenterId);
throw Throwables.propagate(ex);
}
waitServerUntilAvailable.apply(ServerRef.create(dataCenterId, server.id()));
waitDcUntilAvailable.apply(dataCenterId);
// attach bootVolume to Server
org.apache.jclouds.profitbricks.rest.domain.Volume volume =
api.serverApi()
.attachVolume(
Server.Request.attachVolumeBuilder()
.dataCenterId(dataCenterId)
.serverId(server.id())
.volumeId(bootVolume.id())
.build());
trackables.waitUntilRequestCompleted(volume);
waitServerUntilAvailable.apply(ServerRef.create(dataCenterId, server.id()));
waitDcUntilAvailable.apply(dataCenterId);
// fetch an existing lan and creat if non was found
Lan lan = null;
List<Lan> lans = api.lanApi().list(dataCenterId);
if (lans != null && !lans.isEmpty()) {
lan =
FluentIterable.from(lans)
.firstMatch(
new Predicate<Lan>() {
@Override
public boolean apply(Lan input) {
input =
api.lanApi().get(dataCenterId, input.id(), new DepthOptions().depth(3));
return input.properties().isPublic();
}
})
.orNull();
}
if (lan == null) {
logger.warn("Could not find an existing lan Creating one....");
lan =
api.lanApi()
.create(
Lan.Request.creatingBuilder()
.dataCenterId(dataCenterId)
.isPublic(Boolean.TRUE)
.name("lan " + name)
.build());
trackables.waitUntilRequestCompleted(lan);
}
// add a NIC to the server
int lanId = DEFAULT_LAN_ID;
if (options.getNetworks() != null) {
try {
String networkId = Iterables.get(options.getNetworks(), 0);
lanId = Integer.valueOf(networkId);
} catch (Exception ex) {
logger.warn(
"no valid network id found from options. using default id='%d'", DEFAULT_LAN_ID);
}
}
Nic nic =
api.nicApi()
.create(
Nic.Request.creatingBuilder()
.dataCenterId(dataCenterId)
.name("jclouds" + name)
.dhcp(Boolean.TRUE)
.lan(lanId)
.firewallActive(Boolean.FALSE)
.serverId(server.id())
.build());
trackables.waitUntilRequestCompleted(nic);
waitNICUntilAvailable.apply(NicRef.create(dataCenterId, server.id(), nic.id()));
waitDcUntilAvailable.apply(dataCenterId);
waitServerUntilAvailable.apply(ServerRef.create(dataCenterId, server.id()));
// connect the rest of volumes to server;delete if fails
final int volumeCount = volumeIds.size();
for (int j = 1; j < volumeCount; j++) { // skip first; already connected
final String volumeId = volumeIds.get(j);
try {
logger.trace("<< connecting volume '%s' to server '%s'", volumeId, server.id());
provisioningManager.provision(
jobFactory.create(
group,
new Supplier<Object>() {
@Override
public Object get() {
return api.serverApi()
.attachVolume(
Server.Request.attachVolumeBuilder()
.dataCenterId(dataCenterId)
.serverId(server.id())
.volumeId(volumeId)
.build());
}
}));
logger.trace(">> volume connected.");
} catch (Exception ex) {
try {
// delete unconnected volume
logger.warn(ex, ">> failed to connect volume '%s'. deleting..", volumeId);
destroyVolume(volumeId, dataCenterId);
logger.warn(ex, ">> rolling back server..", server.id());
destroyServer(server.id(), dataCenterId);
throw ex;
} catch (Exception ex1) {
logger.error(ex, ">> failed to rollback");
}
}
}
waitDcUntilAvailable.apply(dataCenterId);
waitServerUntilAvailable.apply(ServerRef.create(dataCenterId, server.id()));
LoginCredentials serverCredentials =
LoginCredentials.builder().user(loginUser).password(password).build();
String serverInDataCenterId =
DataCenterAndId.fromDataCenterAndId(dataCenterId, server.id()).slashEncode();
ServerInDataCenter serverInDatacenter = getNode(serverInDataCenterId);
return new NodeAndInitialCredentials<ServerInDataCenter>(
serverInDatacenter, serverInDataCenterId, serverCredentials);
}
