/**
* Initializes the default contexts if the master address specified in the URI is different from
* the default one.
*
* @param uri the uri
* @param conf the hadoop conf
* @throws IOException if it fails to initialize
*/
void initializeInternal(URI uri, org.apache.hadoop.conf.Configuration conf) throws IOException {
// Load Alluxio configuration if any and merge to the one in Alluxio file system. These
// modifications to ClientContext are global, affecting all Alluxio clients in this JVM.
// We assume here that all clients use the same configuration.
ConfUtils.mergeHadoopConfiguration(conf);
Configuration.set(PropertyKey.MASTER_HOSTNAME, uri.getHost());
Configuration.set(PropertyKey.MASTER_RPC_PORT, uri.getPort());
Configuration.set(PropertyKey.ZOOKEEPER_ENABLED, isZookeeperMode());
// These must be reset to pick up the change to the master address.
// TODO(andrew): We should reset key value system in this situation - see ALLUXIO-1706.
LineageContext.INSTANCE.reset();
FileSystemContext.INSTANCE.reset();
// Try to connect to master, if it fails, the provided uri is invalid.
FileSystemMasterClient client = FileSystemContext.INSTANCE.acquireMasterClient();
try {
client.connect();
// Connected, initialize.
} catch (ConnectionFailedException | IOException e) {
LOG.error(
"Failed to connect to the provided master address {}: {}.", uri.toString(), e.toString());
throw new IOException(e);
} finally {
FileSystemContext.INSTANCE.releaseMasterClient(client);
}
}
/**
* Tests that allocation happens when the RAM, SSD and HDD size is lower than the default size.
*/
@Test
public void shouldAllocateTest() throws Exception {
Configuration conf = WorkerContext.getConf();
for (String strategyName : mStrategies) {
conf.set(Constants.WORKER_ALLOCATOR_CLASS, strategyName);
resetManagerView();
Allocator tierAllocator = Allocator.Factory.create(conf, getManagerView());
for (int i = 0; i < DEFAULT_RAM_NUM; i++) {
assertTempBlockMeta(tierAllocator, mAnyDirInTierLoc1, DEFAULT_RAM_SIZE - 1, true);
}
for (int i = 0; i < DEFAULT_SSD_NUM; i++) {
assertTempBlockMeta(tierAllocator, mAnyDirInTierLoc2, DEFAULT_SSD_SIZE - 1, true);
}
for (int i = 0; i < DEFAULT_HDD_NUM; i++) {
assertTempBlockMeta(tierAllocator, mAnyDirInTierLoc3, DEFAULT_HDD_SIZE - 1, true);
}
resetManagerView();
Allocator anyAllocator = Allocator.Factory.create(conf, getManagerView());
for (int i = 0; i < DEFAULT_RAM_NUM; i++) {
assertTempBlockMeta(anyAllocator, mAnyTierLoc, DEFAULT_RAM_SIZE - 1, true);
}
for (int i = 0; i < DEFAULT_SSD_NUM; i++) {
assertTempBlockMeta(anyAllocator, mAnyTierLoc, DEFAULT_SSD_SIZE - 1, true);
}
for (int i = 0; i < DEFAULT_HDD_NUM; i++) {
assertTempBlockMeta(anyAllocator, mAnyTierLoc, DEFAULT_HDD_SIZE - 1, true);
}
}
}
/**
* Resets the {@link LoginUser} and re-login with new user.
*
* @param user the new user
* @throws IOException if login fails
*/
public static void resetLoginUser(String user) throws IOException {
synchronized (LoginUser.class) {
resetLoginUser();
Configuration.set(Constants.SECURITY_LOGIN_USERNAME, user);
LoginUser.get();
}
}
private LocalAlluxioMaster() throws IOException {
mHostname = NetworkAddressUtils.getConnectHost(ServiceType.MASTER_RPC);
mJournalFolder = Configuration.get(Constants.MASTER_JOURNAL_FOLDER);
mAlluxioMaster = AlluxioMaster.Factory.create();
Whitebox.setInternalState(AlluxioMaster.class, "sAlluxioMaster", mAlluxioMaster);
// Reset the master port
Configuration.set(Constants.MASTER_RPC_PORT, Integer.toString(getRPCLocalPort()));
Runnable runMaster =
new Runnable() {
@Override
public void run() {
try {
mAlluxioMaster.start();
} catch (Exception e) {
throw new RuntimeException(e + " \n Start Master Error \n" + e.getMessage(), e);
}
}
};
mMasterThread = new Thread(runMaster);
}
/**
* Creates a new local alluxio master with a isolated home and port.
*
* @throws IOException when unable to do file operation or listen on port
* @return an instance of Alluxio master
*/
public static LocalAlluxioMaster create() throws IOException {
final String alluxioHome = uniquePath();
UnderFileSystemUtils.deleteDir(alluxioHome);
UnderFileSystemUtils.mkdirIfNotExists(alluxioHome);
// Update Alluxio home in the passed Alluxio configuration instance.
Configuration.set(Constants.HOME, alluxioHome);
return new LocalAlluxioMaster();
}
/** Tests that setting the {@link Constants#USER_FILE_BUFFER_BYTES} runs correctly. */
@Test
public void variableUserFileBufferBytesNormalCheckTest() {
Properties mProperties = new Properties();
mProperties.put(Constants.USER_FILE_BUFFER_BYTES, String.valueOf(Integer.MAX_VALUE) + "B");
mCustomPropsConfiguration = new Configuration(mProperties);
Assert.assertEquals(
Integer.MAX_VALUE,
(int) mCustomPropsConfiguration.getBytes(Constants.USER_FILE_BUFFER_BYTES));
mCustomPropsConfiguration.set(Constants.USER_FILE_BUFFER_BYTES, "1GB");
Assert.assertEquals(
1073741824, (int) mCustomPropsConfiguration.getBytes(Constants.USER_FILE_BUFFER_BYTES));
}
/** Tests that blocks are allocated in the first storage directory which has enough free space. */
@Test
public void allocateBlock() throws Exception {
Configuration.set(PropertyKey.WORKER_ALLOCATOR_CLASS, GreedyAllocator.class.getName());
mAllocator = Allocator.Factory.create(getManagerView());
//
// idx | tier1 | tier2 | tier3
// 0 1000
// 0 ├───── 2000
// 1 └───── 2000
// 0 ├─── 3000
// 1 ├─── 3000
// 2 └─── 3000
//
assertTempBlockMeta(mAllocator, mAnyTierLoc, 500, true, "MEM", 0);
//
// idx | tier1 | tier2 | tier3
// 0 500 <--- alloc
// 0 ├───── 2000
// 1 └───── 2000
// 0 ├─── 3000
// 1 ├─── 3000
// 2 └─── 3000
//
assertTempBlockMeta(mAllocator, mAnyDirInTierLoc2, 1000, true, "SSD", 0);
//
// idx | tier1 | tier2 | tier3
// 0 500
// 0 ├───── 1000 <--- alloc
// 1 └───── 2000
// 0 ├─── 3000
// 1 ├─── 3000
// 2 └─── 3000
//
assertTempBlockMeta(mAllocator, mAnyDirInTierLoc2, 1500, true, "SSD", 1);
//
// idx | tier1 | tier2 | tier3
// 0 500
// 0 ├───── 1000
// 1 └───── 500 <--- alloc
// 0 ├─── 3000
// 1 ├─── 3000
// 2 └─── 3000
//
assertTempBlockMeta(mAllocator, mAnyTierLoc, 1000, true, "SSD", 0);
//
// idx | tier1 | tier2 | tier3
// 0 500
// 0 ├───── 0 <--- alloc
// 1 └───── 500
// 0 ├─── 3000
// 1 ├─── 3000
// 2 └─── 3000
//
assertTempBlockMeta(mAllocator, mAnyTierLoc, 1000, true, "HDD", 0);
//
// idx | tier1 | tier2 | tier3
// 0 500
// 0 ├───── 0
// 1 └───── 500
// 0 ├─── 2000 <--- alloc
// 1 ├─── 3000
// 2 └─── 3000
//
assertTempBlockMeta(mAllocator, mAnyTierLoc, 2000, true, "HDD", 0);
//
// idx | tier1 | tier2 | tier3
// 0 500
// 0 ├───── 0
// 1 └───── 500
// 0 ├─── 0 <--- alloc
// 1 ├─── 3000
// 2 └─── 3000
//
assertTempBlockMeta(mAllocator, mAnyTierLoc, 500, true, "MEM", 0);
//
// idx | tier1 | tier2 | tier3
// 0 0 <--- alloc
// 0 ├───── 0
// 1 └───── 500
// 0 ├─── 0
// 1 ├─── 3000
// 2 └─── 3000
//
assertTempBlockMeta(mAllocator, mAnyTierLoc, 500, true, "SSD", 1);
//
// idx | tier1 | tier2 | tier3
// 0 0
// 0 ├───── 0
// 1 └───── 0 <--- alloc
// 0 ├─── 0
// 1 ├─── 3000
// 2 └─── 3000
//
assertTempBlockMeta(mAllocator, mAnyDirInTierLoc3, 1000, true, "HDD", 1);
//
// idx | tier1 | tier2 | tier3
// 0 0
// 0 ├───── 0
// 1 └───── 500
// 0 ├─── 0
// 1 ├─── 2000 <--- alloc
// 2 └─── 3000
//
assertTempBlockMeta(mAllocator, mAnyTierLoc, 700, true, "HDD", 1);
//
// idx | tier1 | tier2 | tier3
// 0 0
// 0 ├───── 0
// 1 └───── 500
// 0 ├─── 0
// 1 ├─── 1300 <--- alloc
// 2 └─── 3000
//
ConfigurationTestUtils.resetConfiguration();
}
@Before
public final void before() throws Exception {
Configuration.set(PropertyKey.UNDERFS_LISTING_LENGTH, 50);
mUnderfsAddress = Configuration.get(PropertyKey.UNDERFS_ADDRESS);
mUfs = UnderFileSystem.get(mUnderfsAddress + AlluxioURI.SEPARATOR);
}