public S3UnderFileSystem(String bucketName, TachyonConf tachyonConf) throws ServiceException {
super(tachyonConf);
Preconditions.checkArgument(
tachyonConf.containsKey(Constants.S3_ACCESS_KEY),
"Property " + Constants.S3_ACCESS_KEY + " is required to connect to S3");
Preconditions.checkArgument(
tachyonConf.containsKey(Constants.S3_SECRET_KEY),
"Property " + Constants.S3_SECRET_KEY + " is required to connect to S3");
AWSCredentials awsCredentials =
new AWSCredentials(
tachyonConf.get(Constants.S3_ACCESS_KEY), tachyonConf.get(Constants.S3_SECRET_KEY));
mBucketName = bucketName;
Jets3tProperties props = new Jets3tProperties();
if (tachyonConf.containsKey(Constants.UNDERFS_S3_PROXY_HOST)) {
props.setProperty("httpclient.proxy-autodetect", "false");
props.setProperty("httpclient.proxy-host", tachyonConf.get(Constants.UNDERFS_S3_PROXY_HOST));
props.setProperty("httpclient.proxy-port", tachyonConf.get(Constants.UNDERFS_S3_PROXY_PORT));
}
if (tachyonConf.containsKey(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY)) {
props.setProperty(
"s3service.https-only",
Boolean.toString(tachyonConf.getBoolean(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY)));
}
LOG.debug("Initializing S3 underFs with properties: " + props.getProperties());
mClient = new RestS3Service(awsCredentials, null, null, props);
mBucketPrefix = Constants.HEADER_S3N + mBucketName + PATH_SEPARATOR;
}
/** Sets the volume and the mount directory before a test runs. */
@Before
public final void before() {
mTachyonConf = new TachyonConf();
if (mTachyonConf.containsKey(Constants.UNDERFS_GLUSTERFS_MR_DIR)) {
mMount = mTachyonConf.get(Constants.UNDERFS_GLUSTERFS_MR_DIR);
}
if (mTachyonConf.containsKey(Constants.UNDERFS_GLUSTERFS_VOLUMES)) {
mVolume = mTachyonConf.get(Constants.UNDERFS_GLUSTERFS_VOLUMES);
}
}
public S3UnderFileSystem(String bucketName, TachyonConf tachyonConf) throws ServiceException {
super(tachyonConf);
Preconditions.checkArgument(
tachyonConf.containsKey(Constants.S3_ACCESS_KEY),
"Property " + Constants.S3_ACCESS_KEY + " is required to connect to S3");
Preconditions.checkArgument(
tachyonConf.containsKey(Constants.S3_SECRET_KEY),
"Property " + Constants.S3_SECRET_KEY + " is required to connect to S3");
AWSCredentials awsCredentials =
new AWSCredentials(
tachyonConf.get(Constants.S3_ACCESS_KEY), tachyonConf.get(Constants.S3_SECRET_KEY));
mBucketName = bucketName;
mClient = new RestS3Service(awsCredentials);
mBucketPrefix = Constants.HEADER_S3N + mBucketName + PATH_SEPARATOR;
}
/**
* Creates a <code>MetricsSystem</code> using the default metrics config.
*
* @param instance the instance name
* @param tachyonConf the {@link TachyonConf} instance for configuration properties.
*/
public MetricsSystem(String instance, TachyonConf tachyonConf) {
mInstance = instance;
mTachyonConf = tachyonConf;
String metricsConfFile = null;
if (mTachyonConf.containsKey(Constants.METRICS_CONF_FILE)) {
metricsConfFile = mTachyonConf.get(Constants.METRICS_CONF_FILE);
}
mMetricsConfig = new MetricsConfig(metricsConfFile);
}
private ServerBootstrap createBootstrap() {
final ServerBootstrap boot =
createBootstrapOfType(
mTachyonConf.getEnum(Constants.WORKER_NETWORK_NETTY_CHANNEL, ChannelType.class));
// use pooled buffers
boot.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);
boot.childOption(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);
// set write buffer
// this is the default, but its recommended to set it in case of change in future netty.
boot.childOption(
ChannelOption.WRITE_BUFFER_HIGH_WATER_MARK,
(int) mTachyonConf.getBytes(Constants.WORKER_NETWORK_NETTY_WATERMARK_HIGH));
boot.childOption(
ChannelOption.WRITE_BUFFER_LOW_WATER_MARK,
(int) mTachyonConf.getBytes(Constants.WORKER_NETWORK_NETTY_WATERMARK_LOW));
// more buffer settings on Netty socket option, one can tune them by specifying
// properties, e.g.:
// tachyon.worker.network.netty.backlog=50
// tachyon.worker.network.netty.buffer.send=64KB
// tachyon.worker.network.netty.buffer.receive=64KB
if (mTachyonConf.containsKey(Constants.WORKER_NETWORK_NETTY_BACKLOG)) {
boot.option(
ChannelOption.SO_BACKLOG, mTachyonConf.getInt(Constants.WORKER_NETWORK_NETTY_BACKLOG));
}
if (mTachyonConf.containsKey(Constants.WORKER_NETWORK_NETTY_BUFFER_SEND)) {
boot.option(
ChannelOption.SO_SNDBUF,
(int) mTachyonConf.getBytes(Constants.WORKER_NETWORK_NETTY_BUFFER_SEND));
}
if (mTachyonConf.containsKey(Constants.WORKER_NETWORK_NETTY_BUFFER_RECEIVE)) {
boot.option(
ChannelOption.SO_RCVBUF,
(int) mTachyonConf.getBytes(Constants.WORKER_NETWORK_NETTY_BUFFER_RECEIVE));
}
return boot;
}