@Override
protected void connectToRepository(
Repository repository,
AuthenticationInfo authenticationInfo,
ProxyInfoProvider proxyInfoProvider)
throws AuthenticationException {
if (this.amazonS3 == null) {
AWSCredentials awsCredentials;
if (authenticationInfo == null) {
awsCredentials = null;
} else if (authenticationInfo.getPrivateKey() == null) {
awsCredentials = createAWSCredentials(authenticationInfo);
} else {
awsCredentials = createAWSSessionCredentials(authenticationInfo);
}
ClientConfiguration clientConfiguration = S3Utils.getClientConfiguration(proxyInfoProvider);
this.bucketName = S3Utils.getBucketName(repository);
this.baseDirectory = S3Utils.getBaseDirectory(repository);
this.amazonS3 = new AmazonS3Client(awsCredentials, clientConfiguration);
Region region =
Region.fromLocationConstraint(this.amazonS3.getBucketLocation(this.bucketName));
this.amazonS3.setEndpoint(region.getEndpoint());
}
}
public DelegateStore() {
s3Client =
new AmazonS3Client(S3Utils.getAWSCredentialsProvider(), S3Utils.buildClientConfiguration());
executorService =
Executors.newFixedThreadPool(
4,
new ThreadFactoryBuilder().setDaemon(true).setNameFormat("DelegateStore-%d").build());
}
@Override
public LoadedInstanceConfig storeConfig(ConfigCollection config, long compareVersion)
throws Exception {
{
ObjectMetadata metadata = getConfigMetadata();
if (metadata != null) {
Date lastModified = metadata.getLastModified();
if (lastModified.getTime() != compareVersion) {
return null; // apparently there's no atomic way to do this with S3 so this will have to
// do
}
}
}
PropertyBasedInstanceConfig propertyBasedInstanceConfig =
new PropertyBasedInstanceConfig(config);
ByteArrayOutputStream out = new ByteArrayOutputStream();
propertyBasedInstanceConfig
.getProperties()
.store(out, "Auto-generated by Exhibitor " + hostname);
byte[] bytes = out.toByteArray();
ObjectMetadata metadata =
S3Utils.simpleUploadFile(s3Client, bytes, arguments.getBucket(), arguments.getKey());
return new LoadedInstanceConfig(
propertyBasedInstanceConfig, metadata.getLastModified().getTime());
}
private boolean isObjectInBucket(final S3Coords coords) throws SegmentLoadingException {
try {
return S3Utils.retryS3Operation(
new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
return S3Utils.isObjectInBucket(s3Client, coords.bucket, coords.path);
}
});
} catch (S3ServiceException | IOException e) {
throw new SegmentLoadingException(e, "S3 fail! Key[%s]", coords);
} catch (Exception e) {
throw Throwables.propagate(e);
}
}
@Override
public long getLastModified(DataSegment segment) throws SegmentLoadingException {
final S3Coords coords = new S3Coords(segment);
try {
final S3Object objDetails =
S3Utils.retryS3Operation(
new Callable<S3Object>() {
@Override
public S3Object call() throws Exception {
return s3Client.getObjectDetails(new S3Bucket(coords.bucket), coords.path);
}
});
return objDetails.getLastModifiedDate().getTime();
} catch (S3ServiceException | IOException e) {
throw new SegmentLoadingException(e, e.getMessage());
} catch (Exception e) {
throw Throwables.propagate(e);
}
}
@Override
public void getSegmentFiles(final DataSegment segment, final File outDir)
throws SegmentLoadingException {
final S3Coords s3Coords = new S3Coords(segment);
log.info("Pulling index at path[%s] to outDir[%s]", s3Coords, outDir);
if (!isObjectInBucket(s3Coords)) {
throw new SegmentLoadingException("IndexFile[%s] does not exist.", s3Coords);
}
if (!outDir.exists()) {
outDir.mkdirs();
}
if (!outDir.isDirectory()) {
throw new ISE("outDir[%s] must be a directory.", outDir);
}
try {
S3Utils.retryS3Operation(
new Callable<Void>() {
@Override
public Void call() throws Exception {
long startTime = System.currentTimeMillis();
S3Object s3Obj = null;
try {
s3Obj = s3Client.getObject(s3Coords.bucket, s3Coords.path);
try (InputStream in = s3Obj.getDataInputStream()) {
final String key = s3Obj.getKey();
if (key.endsWith(".zip")) {
CompressionUtils.unzip(in, outDir);
} else if (key.endsWith(".gz")) {
final File outFile = new File(outDir, toFilename(key, ".gz"));
ByteStreams.copy(
new GZIPInputStream(in), Files.newOutputStreamSupplier(outFile));
} else {
ByteStreams.copy(
in, Files.newOutputStreamSupplier(new File(outDir, toFilename(key, ""))));
}
log.info(
"Pull of file[%s] completed in %,d millis",
s3Obj, System.currentTimeMillis() - startTime);
return null;
} catch (IOException e) {
throw new IOException(
String.format("Problem decompressing object[%s]", s3Obj), e);
}
} finally {
S3Utils.closeStreamsQuietly(s3Obj);
}
}
});
} catch (Exception e) {
try {
FileUtils.deleteDirectory(outDir);
} catch (IOException ioe) {
log.warn(
ioe,
"Failed to remove output directory for segment[%s] after exception: %s",
segment.getIdentifier(),
outDir);
}
throw new SegmentLoadingException(e, e.getMessage());
}
}
