/**
* Constructs a new instance of {@link S3UnderFileSystem}.
*
* @param uri the {@link AlluxioURI} for this UFS
* @param conf the configuration for Alluxio
* @param awsCredentials AWS Credentials configuration for S3 Access
* @throws ServiceException when a connection to S3 could not be created
*/
public S3UnderFileSystem(AlluxioURI uri, Configuration conf, AWSCredentials awsCredentials)
throws ServiceException {
super(uri, conf);
String bucketName = uri.getHost();
mBucketName = bucketName;
Jets3tProperties props = new Jets3tProperties();
if (conf.containsKey(Constants.UNDERFS_S3_PROXY_HOST)) {
props.setProperty("httpclient.proxy-autodetect", "false");
props.setProperty("httpclient.proxy-host", conf.get(Constants.UNDERFS_S3_PROXY_HOST));
props.setProperty("httpclient.proxy-port", conf.get(Constants.UNDERFS_S3_PROXY_PORT));
}
if (conf.containsKey(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY)) {
props.setProperty(
"s3service.https-only",
Boolean.toString(conf.getBoolean(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY)));
}
if (conf.containsKey(Constants.UNDERFS_S3_ENDPOINT)) {
props.setProperty("s3service.s3-endpoint", conf.get(Constants.UNDERFS_S3_ENDPOINT));
if (conf.getBoolean(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY)) {
props.setProperty(
"s3service.s3-endpoint-https-port", conf.get(Constants.UNDERFS_S3_ENDPOINT_HTTPS_PORT));
} else {
props.setProperty(
"s3service.s3-endpoint-http-port", conf.get(Constants.UNDERFS_S3_ENDPOINT_HTTP_PORT));
}
}
if (conf.containsKey(Constants.UNDERFS_S3_DISABLE_DNS_BUCKETS)) {
props.setProperty(
"s3service.disable-dns-buckets", conf.get(Constants.UNDERFS_S3_DISABLE_DNS_BUCKETS));
}
LOG.debug("Initializing S3 underFs with properties: {}", props.getProperties());
mClient = new RestS3Service(awsCredentials, null, null, props);
mBucketPrefix = PathUtils.normalizePath(Constants.HEADER_S3N + mBucketName, PATH_SEPARATOR);
}
/** Tests the default properties for the worker. */
@Test
public void workerDefaultTest() {
String value = sDefaultConfiguration.get(Constants.WORKER_DATA_FOLDER);
Assert.assertNotNull(value);
Assert.assertEquals("/alluxioworker/", value);
value = sDefaultConfiguration.get(Constants.WORKER_BIND_HOST);
Assert.assertNotNull(value);
Assert.assertEquals(NetworkAddressUtils.WILDCARD_ADDRESS, value);
int intValue = sDefaultConfiguration.getInt(Constants.WORKER_RPC_PORT);
Assert.assertEquals(29998, intValue);
value = sDefaultConfiguration.get(Constants.WORKER_DATA_BIND_HOST);
Assert.assertNotNull(value);
Assert.assertEquals(NetworkAddressUtils.WILDCARD_ADDRESS, value);
intValue = sDefaultConfiguration.getInt(Constants.WORKER_DATA_PORT);
Assert.assertEquals(29999, intValue);
value = sDefaultConfiguration.get(Constants.WORKER_WEB_BIND_HOST);
Assert.assertNotNull(value);
Assert.assertEquals(NetworkAddressUtils.WILDCARD_ADDRESS, value);
intValue = sDefaultConfiguration.getInt(Constants.WORKER_WEB_PORT);
Assert.assertEquals(30000, intValue);
intValue = sDefaultConfiguration.getInt(Constants.WORKER_BLOCK_HEARTBEAT_TIMEOUT_MS);
Assert.assertEquals(10 * Constants.SECOND_MS, intValue);
intValue = sDefaultConfiguration.getInt(Constants.WORKER_BLOCK_HEARTBEAT_INTERVAL_MS);
Assert.assertEquals(Constants.SECOND_MS, intValue);
intValue = sDefaultConfiguration.getInt(Constants.WORKER_WORKER_BLOCK_THREADS_MIN);
Assert.assertEquals(1, intValue);
intValue = sDefaultConfiguration.getInt(Constants.WORKER_SESSION_TIMEOUT_MS);
Assert.assertEquals(10 * Constants.SECOND_MS, intValue);
intValue = sDefaultConfiguration.getInt(Constants.WORKER_NETWORK_NETTY_BOSS_THREADS);
Assert.assertEquals(1, intValue);
intValue = sDefaultConfiguration.getInt(Constants.WORKER_NETWORK_NETTY_WORKER_THREADS);
Assert.assertEquals(0, intValue);
long longValue = sDefaultConfiguration.getBytes(Constants.WORKER_MEMORY_SIZE);
Assert.assertEquals(128 * Constants.MB, longValue);
}
/** @param conf the configuration for Alluxio */
public AppCallbackHandler(Configuration conf) {
if (conf.containsKey(Constants.SECURITY_LOGIN_USERNAME)) {
mUserName = conf.get(Constants.SECURITY_LOGIN_USERNAME);
} else {
mUserName = "";
}
}
private LocalAlluxioMaster() throws IOException {
mHostname = NetworkAddressUtils.getConnectHost(ServiceType.MASTER_RPC);
mJournalFolder = Configuration.get(Constants.MASTER_JOURNAL_FOLDER);
mAlluxioMaster = AlluxioMaster.Factory.create();
Whitebox.setInternalState(AlluxioMaster.class, "sAlluxioMaster", mAlluxioMaster);
// Reset the master port
Configuration.set(Constants.MASTER_RPC_PORT, Integer.toString(getRPCLocalPort()));
Runnable runMaster =
new Runnable() {
@Override
public void run() {
try {
mAlluxioMaster.start();
} catch (Exception e) {
throw new RuntimeException(e + " \n Start Master Error \n" + e.getMessage(), e);
}
}
};
mMasterThread = new Thread(runMaster);
}
/**
* Creates a new instance of {@link FileSystemWorker}.
*
* @param blockWorker the block worker handle
* @param workerId a reference to the id of this worker
* @throws IOException if an I/O error occurs
*/
public DefaultFileSystemWorker(BlockWorker blockWorker, AtomicReference<Long> workerId)
throws IOException {
super(
Executors.newFixedThreadPool(
3, ThreadFactoryUtils.build("file-system-worker-heartbeat-%d", true)));
mWorkerId = workerId;
mSessions = new Sessions();
UnderFileSystem ufs = UnderFileSystem.get(Configuration.get(PropertyKey.UNDERFS_ADDRESS));
mFileDataManager =
new FileDataManager(
Preconditions.checkNotNull(blockWorker),
ufs,
RateLimiter.create(Configuration.getBytes(PropertyKey.WORKER_FILE_PERSIST_RATE_LIMIT)));
mUnderFileSystemManager = new UnderFileSystemManager();
// Setup AbstractMasterClient
mFileSystemMasterWorkerClient =
new FileSystemMasterClient(NetworkAddressUtils.getConnectAddress(ServiceType.MASTER_RPC));
// Setup session cleaner
mSessionCleaner =
new SessionCleaner(
new SessionCleanupCallback() {
/** Cleans up after sessions, to prevent zombie sessions holding ufs resources. */
@Override
public void cleanupSessions() {
for (long session : mSessions.getTimedOutSessions()) {
mSessions.removeSession(session);
mUnderFileSystemManager.cleanupSession(session);
}
}
});
mServiceHandler = new FileSystemWorkerClientServiceHandler(this);
}
/**
* Starts sinks specified in the configuration. This is an no-op if the sinks have already been
* started. Note: This has to be called after Alluxio configuration is initialized.
*/
public static void startSinks() {
String metricsConfFile = Configuration.get(PropertyKey.METRICS_CONF_FILE);
if (metricsConfFile.isEmpty()) {
LOG.info("Metrics is not enabled.");
return;
}
MetricsConfig config = new MetricsConfig(metricsConfFile);
startSinksFromConfig(config);
}
private static List<CommandInfo.URI> getExecutorDependencyURIList() {
List<URI> dependencies = new ArrayList<>();
if (installJavaFromUrl()) {
dependencies.add(
CommandInfo.URI
.newBuilder()
.setValue(Configuration.get(PropertyKey.INTEGRATION_MESOS_JDK_URL))
.setExtract(true)
.build());
}
if (installAlluxioFromUrl()) {
dependencies.add(
CommandInfo.URI
.newBuilder()
.setValue(Configuration.get(PropertyKey.INTEGRATION_MESOS_ALLUXIO_JAR_URL))
.setExtract(true)
.build());
}
return dependencies;
}
private static String createStartAlluxioCommand(String command) {
List<String> commands = new ArrayList<>();
commands.add(String.format("echo 'Starting Alluxio with %s'", command));
if (installJavaFromUrl()) {
commands.add("export JAVA_HOME=" + Configuration.get(PropertyKey.INTEGRATION_MESOS_JDK_PATH));
commands.add("export PATH=$PATH:$JAVA_HOME/bin");
}
commands.add("mkdir conf");
commands.add("touch conf/alluxio-env.sh");
// If a jar is supplied, start Alluxio from the jar. Otherwise assume that Alluxio is already
// installed at PropertyKey.HOME.
if (installAlluxioFromUrl()) {
commands.add("rm *.tar.gz");
commands.add("mv alluxio-* alluxio");
}
String home = installAlluxioFromUrl() ? "alluxio" : Configuration.get(PropertyKey.HOME);
commands.add(
String.format("cp %s conf", PathUtils.concatPath(home, "conf", "log4j.properties")));
commands.add(PathUtils.concatPath(home, "integration", "bin", command));
return Joiner.on(" && ").join(commands);
}
/**
* Constructs a new Swift {@link UnderFileSystem}.
*
* @param uri the {@link AlluxioURI} for this UFS
*/
public SwiftUnderFileSystem(AlluxioURI uri) {
super(uri);
String containerName = uri.getHost();
LOG.debug("Constructor init: {}", containerName);
AccountConfig config = new AccountConfig();
if (Configuration.containsKey(Constants.SWIFT_API_KEY)) {
config.setPassword(Configuration.get(Constants.SWIFT_API_KEY));
} else if (Configuration.containsKey(Constants.SWIFT_PASSWORD_KEY)) {
config.setPassword(Configuration.get(Constants.SWIFT_PASSWORD_KEY));
}
config.setAuthUrl(Configuration.get(Constants.SWIFT_AUTH_URL_KEY));
String authMethod = Configuration.get(Constants.SWIFT_AUTH_METHOD_KEY);
if (authMethod != null) {
config.setUsername(Configuration.get(Constants.SWIFT_USER_KEY));
config.setTenantName(Configuration.get(Constants.SWIFT_TENANT_KEY));
switch (authMethod) {
case Constants.SWIFT_AUTH_KEYSTONE:
config.setAuthenticationMethod(AuthenticationMethod.KEYSTONE);
break;
case Constants.SWIFT_AUTH_SWIFTAUTH:
// swiftauth authenticates directly against swift
// note: this method is supported in swift object storage api v1
config.setAuthenticationMethod(AuthenticationMethod.BASIC);
break;
default:
config.setAuthenticationMethod(AuthenticationMethod.TEMPAUTH);
// tempauth requires authentication header to be of the form tenant:user.
// JOSS however generates header of the form user:tenant.
// To resolve this, we switch user with tenant
config.setTenantName(Configuration.get(Constants.SWIFT_USER_KEY));
config.setUsername(Configuration.get(Constants.SWIFT_TENANT_KEY));
}
}
ObjectMapper mapper = new ObjectMapper();
mapper.configure(SerializationConfig.Feature.WRAP_ROOT_VALUE, true);
mContainerName = containerName;
mAccount = new AccountFactory(config).createAccount();
// Do not allow container cache to avoid stale directory listings
mAccount.setAllowContainerCaching(false);
mAccess = mAccount.authenticate();
Container container = mAccount.getContainer(containerName);
if (!container.exists()) {
container.create();
}
mContainerPrefix = Constants.HEADER_SWIFT + mContainerName + PATH_SEPARATOR;
}
/**
* Gets the file / directory creation umask.
*
* @return the umask {@link Mode}
*/
public static Mode getUMask() {
int umask = Constants.DEFAULT_FILE_SYSTEM_UMASK;
String confUmask = Configuration.get(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_UMASK);
if (confUmask != null) {
if ((confUmask.length() > 4) || !tryParseInt(confUmask)) {
throw new IllegalArgumentException(
ExceptionMessage.INVALID_CONFIGURATION_VALUE.getMessage(
confUmask, PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_UMASK));
}
int newUmask = 0;
int lastIndex = confUmask.length() - 1;
for (int i = 0; i <= lastIndex; i++) {
newUmask += (confUmask.charAt(i) - '0') << 3 * (lastIndex - i);
}
umask = newUmask;
}
return new Mode((short) umask);
}
/** Tests the default properties for the master. */
@Test
public void masterDefaultTest() {
String alluxioHome = sDefaultConfiguration.get(Constants.HOME);
Assert.assertNotNull(alluxioHome);
Assert.assertEquals("/mnt/alluxio_default_home", alluxioHome);
String value = sDefaultConfiguration.get(Constants.MASTER_JOURNAL_FOLDER);
Assert.assertNotNull(value);
Assert.assertEquals(alluxioHome + "/journal/", value);
value = sDefaultConfiguration.get(Constants.MASTER_HOSTNAME);
Assert.assertNotNull(value);
Assert.assertEquals(NetworkAddressUtils.getLocalHostName(100), value);
value = sDefaultConfiguration.get(Constants.MASTER_FORMAT_FILE_PREFIX);
Assert.assertNotNull(value);
Assert.assertEquals(Constants.FORMAT_FILE_PREFIX, value);
value = sDefaultConfiguration.get(Constants.MASTER_ADDRESS);
Assert.assertNotNull(value);
value = sDefaultConfiguration.get(Constants.MASTER_BIND_HOST);
Assert.assertNotNull(value);
Assert.assertEquals(NetworkAddressUtils.WILDCARD_ADDRESS, value);
int intValue = sDefaultConfiguration.getInt(Constants.MASTER_RPC_PORT);
Assert.assertEquals(19998, intValue);
value = sDefaultConfiguration.get(Constants.MASTER_WEB_BIND_HOST);
Assert.assertNotNull(value);
Assert.assertEquals(NetworkAddressUtils.WILDCARD_ADDRESS, value);
intValue = sDefaultConfiguration.getInt(Constants.MASTER_WEB_PORT);
Assert.assertEquals(19999, intValue);
intValue = sDefaultConfiguration.getInt(Constants.WEB_THREAD_COUNT);
Assert.assertEquals(1, intValue);
intValue = sDefaultConfiguration.getInt(Constants.MASTER_HEARTBEAT_INTERVAL_MS);
Assert.assertEquals(Constants.SECOND_MS, intValue);
intValue = sDefaultConfiguration.getInt(Constants.MASTER_WORKER_THREADS_MIN);
Assert.assertEquals(Runtime.getRuntime().availableProcessors(), intValue);
intValue = sDefaultConfiguration.getInt(Constants.MASTER_WORKER_TIMEOUT_MS);
Assert.assertEquals(10 * Constants.SECOND_MS, intValue);
}
/** Tests the default common properties. */
@Test
public void commonDefaultTest() {
String alluxioHome = sDefaultConfiguration.get(Constants.HOME);
Assert.assertNotNull(alluxioHome);
Assert.assertEquals("/mnt/alluxio_default_home", alluxioHome);
String ufsAddress = sDefaultConfiguration.get(Constants.UNDERFS_ADDRESS);
Assert.assertNotNull(ufsAddress);
Assert.assertEquals(alluxioHome + "/underFSStorage", ufsAddress);
String value = sDefaultConfiguration.get(Constants.WEB_RESOURCES);
Assert.assertNotNull(value);
Assert.assertEquals(alluxioHome + "/core/server/src/main/webapp", value);
value = sDefaultConfiguration.get(Constants.UNDERFS_HDFS_IMPL);
Assert.assertNotNull(value);
Assert.assertEquals("org.apache.hadoop.hdfs.DistributedFileSystem", value);
value = sDefaultConfiguration.get(Constants.UNDERFS_HDFS_PREFIXS);
Assert.assertNotNull(value);
Assert.assertEquals(DEFAULT_HADOOP_UFS_PREFIX, value);
value = sDefaultConfiguration.get(Constants.UNDERFS_GLUSTERFS_IMPL);
Assert.assertNotNull(value);
Assert.assertEquals("org.apache.hadoop.fs.glusterfs.GlusterFileSystem", value);
boolean booleanValue = sDefaultConfiguration.getBoolean(Constants.ZOOKEEPER_ENABLED);
Assert.assertFalse(booleanValue);
booleanValue = sDefaultConfiguration.getBoolean(Constants.IN_TEST_MODE);
Assert.assertFalse(booleanValue);
int intValue = sDefaultConfiguration.getInt(Constants.NETWORK_HOST_RESOLUTION_TIMEOUT_MS);
Assert.assertEquals(Constants.DEFAULT_HOST_RESOLUTION_TIMEOUT_MS, intValue);
long longBytesValue =
sDefaultConfiguration.getBytes(Constants.USER_BLOCK_REMOTE_READ_BUFFER_SIZE_BYTES);
Assert.assertEquals(Constants.MB * 8, longBytesValue);
int maxTry = sDefaultConfiguration.getInt(Constants.ZOOKEEPER_LEADER_INQUIRY_RETRY_COUNT);
Assert.assertEquals(10, maxTry);
}
/** Tests the simple substitution of variables. */
@Test
public void variableSubstitutionSimpleTest() {
String home = mCustomPropsConfiguration.get("home");
Assert.assertEquals("hometest", home);
String homeAndPath = mCustomPropsConfiguration.get("homeandpath");
Assert.assertEquals(home + "/path1", homeAndPath);
String homeAndString = mCustomPropsConfiguration.get("homeandstring");
Assert.assertEquals(home + " string1", homeAndString);
String path2 = mCustomPropsConfiguration.get("path2");
Assert.assertEquals("path2", path2);
String multiplesubs = mCustomPropsConfiguration.get("multiplesubs");
Assert.assertEquals(home + "/path1/" + path2, multiplesubs);
String homePort = mCustomPropsConfiguration.get("home.port");
Assert.assertEquals("8080", homePort);
sTestProperties.put("complex.address", "alluxio://${home}:${home.port}");
String complexAddress = mCustomPropsConfiguration.get("complex.address");
Assert.assertEquals("alluxio://" + home + ":" + homePort, complexAddress);
}
@Override
public void resourceOffers(SchedulerDriver driver, List<Protos.Offer> offers) {
long masterCpu = Configuration.getInt(PropertyKey.INTEGRATION_MASTER_RESOURCE_CPU);
long masterMem =
Configuration.getBytes(PropertyKey.INTEGRATION_MASTER_RESOURCE_MEM) / Constants.MB;
long workerCpu = Configuration.getInt(PropertyKey.INTEGRATION_WORKER_RESOURCE_CPU);
long workerMem =
Configuration.getBytes(PropertyKey.INTEGRATION_WORKER_RESOURCE_MEM) / Constants.MB;
LOG.info(
"Master launched {}, master count {}, "
+ "requested master cpu {} mem {} MB and required master hostname {}",
mMasterLaunched,
mMasterCount,
masterCpu,
masterMem,
mRequiredMasterHostname);
for (Protos.Offer offer : offers) {
Protos.Offer.Operation.Launch.Builder launch = Protos.Offer.Operation.Launch.newBuilder();
double offerCpu = 0;
double offerMem = 0;
for (Protos.Resource resource : offer.getResourcesList()) {
if (resource.getName().equals(Constants.MESOS_RESOURCE_CPUS)) {
offerCpu += resource.getScalar().getValue();
} else if (resource.getName().equals(Constants.MESOS_RESOURCE_MEM)) {
offerMem += resource.getScalar().getValue();
} else {
// Other resources are currently ignored.
}
}
LOG.info(
"Received offer {} on host {} with cpus {} and mem {} MB and hasMasterPorts {}",
offer.getId().getValue(),
offer.getHostname(),
offerCpu,
offerMem,
OfferUtils.hasAvailableMasterPorts(offer));
Protos.ExecutorInfo.Builder executorBuilder = Protos.ExecutorInfo.newBuilder();
List<Protos.Resource> resources;
if (!mMasterLaunched
&& offerCpu >= masterCpu
&& offerMem >= masterMem
&& mMasterCount
< Configuration.getInt(PropertyKey.INTEGRATION_MESOS_ALLUXIO_MASTER_NODE_COUNT)
&& OfferUtils.hasAvailableMasterPorts(offer)
&& (mRequiredMasterHostname == null
|| mRequiredMasterHostname.equals(offer.getHostname()))) {
LOG.debug("Creating Alluxio Master executor");
executorBuilder
.setName("Alluxio Master Executor")
.setSource("master")
.setExecutorId(Protos.ExecutorID.newBuilder().setValue("master"))
.addAllResources(getExecutorResources())
.setCommand(
Protos.CommandInfo.newBuilder()
.setValue(createStartAlluxioCommand("alluxio-master-mesos.sh"))
.addAllUris(getExecutorDependencyURIList())
.setEnvironment(
Protos.Environment.newBuilder()
.addVariables(
Protos.Environment.Variable.newBuilder()
.setName("ALLUXIO_UNDERFS_ADDRESS")
.setValue(Configuration.get(PropertyKey.UNDERFS_ADDRESS))
.build())
.build()));
// pre-build resource list here, then use it to build Protos.Task later.
resources = getMasterRequiredResources(masterCpu, masterMem);
mMasterHostname = offer.getHostname();
mTaskName = Configuration.get(PropertyKey.INTEGRATION_MESOS_ALLUXIO_MASTER_NAME);
mMasterCount++;
mMasterTaskId = mLaunchedTasks;
} else if (mMasterLaunched
&& !mWorkers.contains(offer.getHostname())
&& offerCpu >= workerCpu
&& offerMem >= workerMem
&& OfferUtils.hasAvailableWorkerPorts(offer)) {
LOG.debug("Creating Alluxio Worker executor");
final String memSize = FormatUtils.getSizeFromBytes((long) workerMem * Constants.MB);
executorBuilder
.setName("Alluxio Worker Executor")
.setSource("worker")
.setExecutorId(Protos.ExecutorID.newBuilder().setValue("worker"))
.addAllResources(getExecutorResources())
.setCommand(
Protos.CommandInfo.newBuilder()
.setValue(createStartAlluxioCommand("alluxio-worker-mesos.sh"))
.addAllUris(getExecutorDependencyURIList())
.setEnvironment(
Protos.Environment.newBuilder()
.addVariables(
Protos.Environment.Variable.newBuilder()
.setName("ALLUXIO_MASTER_HOSTNAME")
.setValue(mMasterHostname)
.build())
.addVariables(
Protos.Environment.Variable.newBuilder()
.setName("ALLUXIO_WORKER_MEMORY_SIZE")
.setValue(memSize)
.build())
.addVariables(
Protos.Environment.Variable.newBuilder()
.setName("ALLUXIO_UNDERFS_ADDRESS")
.setValue(Configuration.get(PropertyKey.UNDERFS_ADDRESS))
.build())
.build()));
// pre-build resource list here, then use it to build Protos.Task later.
resources = getWorkerRequiredResources(workerCpu, workerMem);
mWorkers.add(offer.getHostname());
mTaskName = Configuration.get(PropertyKey.INTEGRATION_MESOS_ALLUXIO_WORKER_NAME);
} else {
// The resource offer cannot be used to start either master or a worker.
LOG.info("Declining offer {}", offer.getId().getValue());
driver.declineOffer(offer.getId());
continue;
}
Protos.TaskID taskId =
Protos.TaskID.newBuilder().setValue(String.valueOf(mLaunchedTasks)).build();
LOG.info("Launching task {} using offer {}", taskId.getValue(), offer.getId().getValue());
Protos.TaskInfo task =
Protos.TaskInfo.newBuilder()
.setName(mTaskName)
.setTaskId(taskId)
.setSlaveId(offer.getSlaveId())
.addAllResources(resources)
.setExecutor(executorBuilder)
.build();
launch.addTaskInfos(Protos.TaskInfo.newBuilder(task));
mLaunchedTasks++;
// NOTE: We use the new API `acceptOffers` here to launch tasks.
// The 'launchTasks' API will be deprecated.
List<Protos.OfferID> offerIds = new ArrayList<Protos.OfferID>();
offerIds.add(offer.getId());
List<Protos.Offer.Operation> operations = new ArrayList<Protos.Offer.Operation>();
Protos.Offer.Operation operation =
Protos.Offer.Operation.newBuilder()
.setType(Protos.Offer.Operation.Type.LAUNCH)
.setLaunch(launch)
.build();
operations.add(operation);
Protos.Filters filters = Protos.Filters.newBuilder().setRefuseSeconds(1).build();
driver.acceptOffers(offerIds, operations, filters);
}
}
/** Tests the substitution of system variables. */
@Test
public void systemVariableSubstitutionSampleTest() {
String masterAddress = mSystemPropsConfiguration.get(Constants.MASTER_ADDRESS);
Assert.assertNotNull(masterAddress);
Assert.assertEquals("alluxio-ft://master:20001", masterAddress);
}
/** Tests the recursive substitution of variables. */
@Test
public void variableSubstitutionRecursiveTest() {
String multiplesubs = mCustomPropsConfiguration.get("multiplesubs");
String recursive = mCustomPropsConfiguration.get("recursive");
Assert.assertEquals(multiplesubs, recursive);
}
private static boolean installAlluxioFromUrl() {
return Configuration.containsKey(PropertyKey.INTEGRATION_MESOS_ALLUXIO_JAR_URL)
&& !Configuration.get(PropertyKey.INTEGRATION_MESOS_ALLUXIO_JAR_URL)
.equalsIgnoreCase(Constants.MESOS_LOCAL_INSTALL);
}
@Before
public final void before() throws Exception {
Configuration.set(PropertyKey.UNDERFS_LISTING_LENGTH, 50);
mUnderfsAddress = Configuration.get(PropertyKey.UNDERFS_ADDRESS);
mUfs = UnderFileSystem.get(mUnderfsAddress + AlluxioURI.SEPARATOR);
}