public S3UnderFileSystem(String bucketName, TachyonConf tachyonConf) throws ServiceException {
super(tachyonConf);
Preconditions.checkArgument(
tachyonConf.containsKey(Constants.S3_ACCESS_KEY),
"Property " + Constants.S3_ACCESS_KEY + " is required to connect to S3");
Preconditions.checkArgument(
tachyonConf.containsKey(Constants.S3_SECRET_KEY),
"Property " + Constants.S3_SECRET_KEY + " is required to connect to S3");
AWSCredentials awsCredentials =
new AWSCredentials(
tachyonConf.get(Constants.S3_ACCESS_KEY), tachyonConf.get(Constants.S3_SECRET_KEY));
mBucketName = bucketName;
Jets3tProperties props = new Jets3tProperties();
if (tachyonConf.containsKey(Constants.UNDERFS_S3_PROXY_HOST)) {
props.setProperty("httpclient.proxy-autodetect", "false");
props.setProperty("httpclient.proxy-host", tachyonConf.get(Constants.UNDERFS_S3_PROXY_HOST));
props.setProperty("httpclient.proxy-port", tachyonConf.get(Constants.UNDERFS_S3_PROXY_PORT));
}
if (tachyonConf.containsKey(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY)) {
props.setProperty(
"s3service.https-only",
Boolean.toString(tachyonConf.getBoolean(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY)));
}
LOG.debug("Initializing S3 underFs with properties: " + props.getProperties());
mClient = new RestS3Service(awsCredentials, null, null, props);
mBucketPrefix = Constants.HEADER_S3N + mBucketName + PATH_SEPARATOR;
}
/** Sets the volume and the mount directory before a test runs. */
@Before
public final void before() {
mTachyonConf = new TachyonConf();
if (mTachyonConf.containsKey(Constants.UNDERFS_GLUSTERFS_MR_DIR)) {
mMount = mTachyonConf.get(Constants.UNDERFS_GLUSTERFS_MR_DIR);
}
if (mTachyonConf.containsKey(Constants.UNDERFS_GLUSTERFS_VOLUMES)) {
mVolume = mTachyonConf.get(Constants.UNDERFS_GLUSTERFS_VOLUMES);
}
}
@Override
public void connectFromWorker(TachyonConf conf, String host) throws IOException {
String workerKeytab = conf.get(Constants.WORKER_KEYTAB_KEY, null);
String workerPrincipal = conf.get(Constants.WORKER_PRINCIPAL_KEY, null);
if (workerKeytab == null || workerPrincipal == null) {
return;
}
login(
Constants.WORKER_KEYTAB_KEY,
workerKeytab,
Constants.WORKER_PRINCIPAL_KEY,
workerPrincipal,
host);
}
public S3UnderFileSystem(String bucketName, TachyonConf tachyonConf) throws ServiceException {
super(tachyonConf);
Preconditions.checkArgument(
tachyonConf.containsKey(Constants.S3_ACCESS_KEY),
"Property " + Constants.S3_ACCESS_KEY + " is required to connect to S3");
Preconditions.checkArgument(
tachyonConf.containsKey(Constants.S3_SECRET_KEY),
"Property " + Constants.S3_SECRET_KEY + " is required to connect to S3");
AWSCredentials awsCredentials =
new AWSCredentials(
tachyonConf.get(Constants.S3_ACCESS_KEY), tachyonConf.get(Constants.S3_SECRET_KEY));
mBucketName = bucketName;
mClient = new RestS3Service(awsCredentials);
mBucketPrefix = Constants.HEADER_S3N + mBucketName + PATH_SEPARATOR;
}
/**
* This method adds Aliyun credentials from system properties to the Tachyon Conf if they are not
* already present.
*
* @param tachyonConf the conf to check and add credentials to
* @return true if both access and secret key are present, false otherwise
*/
private boolean addAndCheckOSSCredentials(TachyonConf tachyonConf) {
String accessKeyConf = Constants.OSS_ACCESS_KEY;
if (System.getProperty(accessKeyConf) != null && tachyonConf.get(accessKeyConf) == null) {
tachyonConf.set(accessKeyConf, System.getProperty(accessKeyConf));
}
String secretKeyConf = Constants.OSS_SECRET_KEY;
if (System.getProperty(secretKeyConf) != null && tachyonConf.get(secretKeyConf) == null) {
tachyonConf.set(secretKeyConf, System.getProperty(secretKeyConf));
}
String endPointConf = Constants.OSS_ENDPOINT_KEY;
if (System.getProperty(endPointConf) != null && tachyonConf.get(endPointConf) == null) {
tachyonConf.set(endPointConf, System.getProperty(endPointConf));
}
return tachyonConf.get(accessKeyConf) != null
&& tachyonConf.get(secretKeyConf) != null
&& tachyonConf.get(endPointConf) != null;
}
/**
* Creates a <code>MetricsSystem</code> using the default metrics config.
*
* @param instance the instance name
* @param tachyonConf the {@link TachyonConf} instance for configuration properties.
*/
public MetricsSystem(String instance, TachyonConf tachyonConf) {
mInstance = instance;
mTachyonConf = tachyonConf;
String metricsConfFile = null;
if (mTachyonConf.containsKey(Constants.METRICS_CONF_FILE)) {
metricsConfFile = mTachyonConf.get(Constants.METRICS_CONF_FILE);
}
mMetricsConfig = new MetricsConfig(metricsConfFile);
}
public void start() throws IOException {
int maxLevel = 1;
mTachyonHome =
File.createTempFile("Tachyon", "U" + System.currentTimeMillis()).getAbsolutePath();
mWorkerDataFolder = "/datastore";
mHostname = NetworkAddressUtils.getLocalHostName(100);
mMasterConf = MasterContext.getConf();
mMasterConf.set(Constants.IN_TEST_MODE, "true");
mMasterConf.set(Constants.TACHYON_HOME, mTachyonHome);
mMasterConf.set(Constants.USE_ZOOKEEPER, "true");
mMasterConf.set(Constants.MASTER_HOSTNAME, mHostname);
mMasterConf.set(Constants.MASTER_BIND_HOST, mHostname);
mMasterConf.set(Constants.MASTER_PORT, "0");
mMasterConf.set(Constants.MASTER_WEB_BIND_HOST, mHostname);
mMasterConf.set(Constants.MASTER_WEB_PORT, "0");
mMasterConf.set(Constants.ZOOKEEPER_ADDRESS, mCuratorServer.getConnectString());
mMasterConf.set(Constants.ZOOKEEPER_ELECTION_PATH, "/election");
mMasterConf.set(Constants.ZOOKEEPER_LEADER_PATH, "/leader");
mMasterConf.set(Constants.USER_QUOTA_UNIT_BYTES, "10000");
mMasterConf.set(Constants.USER_DEFAULT_BLOCK_SIZE_BYTE, Integer.toString(mUserBlockSize));
// Since tests are always running on a single host keep the resolution timeout low as otherwise
// people running with strange network configurations will see very slow tests
mMasterConf.set(Constants.HOST_RESOLUTION_TIMEOUT_MS, "250");
// Disable hdfs client caching to avoid file system close() affecting other clients
System.setProperty("fs.hdfs.impl.disable.cache", "true");
// re-build the dir to set permission to 777
deleteDir(mTachyonHome);
mkdir(mTachyonHome);
for (int k = 0; k < mNumOfMasters; k++) {
final LocalTachyonMaster master = LocalTachyonMaster.create(mTachyonHome);
master.start();
LOG.info(
"master NO."
+ k
+ " started, isServing: "
+ master.isServing()
+ ", address: "
+ master.getAddress());
mMasters.add(master);
// Each master should generate a new port for binding
mMasterConf.set(Constants.MASTER_PORT, "0");
}
// Create the directories for the data and workers after LocalTachyonMaster construction,
// because LocalTachyonMaster sets the UNDERFS_DATA_FOLDER and UNDERFS_WORKERS_FOLDER.
mkdir(mMasterConf.get(Constants.UNDERFS_DATA_FOLDER));
mkdir(mMasterConf.get(Constants.UNDERFS_WORKERS_FOLDER));
LOG.info("all " + mNumOfMasters + " masters started.");
LOG.info("waiting for a leader.");
boolean hasLeader = false;
while (!hasLeader) {
for (int i = 0; i < mMasters.size(); i++) {
if (mMasters.get(i).isServing()) {
LOG.info(
"master NO."
+ i
+ " is selected as leader. address: "
+ mMasters.get(i).getAddress());
hasLeader = true;
break;
}
}
}
// Use first master port
mMasterConf.set(Constants.MASTER_PORT, getMasterPort() + "");
CommonUtils.sleepMs(10);
mWorkerConf = WorkerContext.getConf();
mWorkerConf.merge(mMasterConf);
mWorkerConf.set(Constants.WORKER_DATA_FOLDER, mWorkerDataFolder);
mWorkerConf.set(Constants.WORKER_MEMORY_SIZE, mWorkerCapacityBytes + "");
mWorkerConf.set(Constants.WORKER_TO_MASTER_HEARTBEAT_INTERVAL_MS, 15 + "");
// Setup conf for worker
mWorkerConf.set(Constants.WORKER_MAX_TIERED_STORAGE_LEVEL, Integer.toString(maxLevel));
mWorkerConf.set(String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_ALIAS_FORMAT, 0), "MEM");
mWorkerConf.set(
String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_PATH_FORMAT, 0),
mTachyonHome + "/ramdisk");
mWorkerConf.set(
String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_QUOTA_FORMAT, 0),
mWorkerCapacityBytes + "");
// Since tests are always running on a single host keep the resolution timeout low as otherwise
// people running with strange network configurations will see very slow tests
mWorkerConf.set(Constants.HOST_RESOLUTION_TIMEOUT_MS, "250");
for (int level = 1; level < maxLevel; level++) {
String tierLevelDirPath =
String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_PATH_FORMAT, level);
String[] dirPaths = mWorkerConf.get(tierLevelDirPath).split(",");
String newPath = "";
for (String dirPath : dirPaths) {
newPath += mTachyonHome + dirPath + ",";
}
mWorkerConf.set(
String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_PATH_FORMAT, level),
newPath.substring(0, newPath.length() - 1));
}
mWorkerConf.set(Constants.WORKER_BIND_HOST, mHostname);
mWorkerConf.set(Constants.WORKER_PORT, "0");
mWorkerConf.set(Constants.WORKER_DATA_BIND_HOST, mHostname);
mWorkerConf.set(Constants.WORKER_DATA_PORT, "0");
mWorkerConf.set(Constants.WORKER_WEB_BIND_HOST, mHostname);
mWorkerConf.set(Constants.WORKER_WEB_PORT, "0");
mWorkerConf.set(Constants.WORKER_MIN_WORKER_THREADS, "1");
mWorkerConf.set(Constants.WORKER_MAX_WORKER_THREADS, "100");
// Perform immediate shutdown of data server. Graceful shutdown is unnecessary and slow
mWorkerConf.set(Constants.WORKER_NETWORK_NETTY_SHUTDOWN_QUIET_PERIOD, Integer.toString(0));
mWorkerConf.set(Constants.WORKER_NETWORK_NETTY_SHUTDOWN_TIMEOUT, Integer.toString(0));
mWorker = new BlockWorker();
Runnable runWorker =
new Runnable() {
@Override
public void run() {
try {
mWorker.process();
} catch (Exception e) {
throw new RuntimeException(e + " \n Start Master Error \n" + e.getMessage(), e);
}
}
};
mWorkerThread = new Thread(runWorker);
mWorkerThread.start();
// The client context should reflect the updates to the conf.
if (sReinitializer == null) {
ClientContext.accessReinitializer(sReinitializerAccesser);
}
sReinitializer.reinitializeWithConf(mWorkerConf);
}
