/** Tests if list correctly returns file or folder names for a large directory. */
@Test
public void listLargeDirectory() throws IOException {
LargeDirectoryConfig config = prepareLargeDirectoryTest();
String[] children = config.getChildren();
// Retry for some time to allow list operation eventual consistency for S3 and GCS.
// See http://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html and
// https://cloud.google.com/storage/docs/consistency for more details.
// Note: not using CommonUtils.waitFor here because we intend to sleep with a longer interval.
String[] results = new String[] {};
for (int i = 0; i < 20; i++) {
results = mUfs.list(config.getTopLevelDirectory());
if (children.length == results.length) {
break;
}
CommonUtils.sleepMs(500);
}
Assert.assertEquals(children.length, results.length);
Arrays.sort(results);
for (int i = 0; i < children.length; ++i) {
Assert.assertTrue(
results[i].equals(
CommonUtils.stripPrefixIfPresent(
children[i], PathUtils.normalizePath(config.getTopLevelDirectory(), "/"))));
}
}
@Override
public String[] list(String path) throws IOException {
String prefix = addFolderSuffixIfNotPresent(stripContainerPrefixIfPresent(path));
prefix = prefix.equals(PATH_SEPARATOR) ? "" : prefix;
Collection<DirectoryOrObject> objects = listInternal(prefix);
Set<String> children = new HashSet<>();
final String self = stripFolderSuffixIfPresent(prefix);
boolean foundSelf = false;
for (DirectoryOrObject object : objects) {
String child = stripFolderSuffixIfPresent(object.getName());
String noPrefix = CommonUtils.stripPrefixIfPresent(child, prefix);
if (!noPrefix.equals(self)) {
children.add(noPrefix);
} else {
foundSelf = true;
}
}
if (!foundSelf) {
// Path does not exist
return null;
}
return children.toArray(new String[children.size()]);
}
/**
* @summary get the worker metrics
* @return the response object
*/
@GET
@Path(GET_METRICS)
@ReturnType("java.util.SortedMap<String, Long>")
public Response getMetrics() {
MetricRegistry metricRegistry = mWorker.getWorkerMetricsSystem().getMetricRegistry();
// Get all counters.
Map<String, Counter> counters = metricRegistry.getCounters();
// Only the gauge for cached blocks is retrieved here, other gauges are statistics of free/used
// spaces, those statistics can be gotten via other REST apis.
String blocksCachedProperty =
CommonUtils.argsToString(
".", WorkerContext.getWorkerSource().getName(), WorkerSource.BLOCKS_CACHED);
@SuppressWarnings("unchecked")
Gauge<Integer> blocksCached =
(Gauge<Integer>) metricRegistry.getGauges().get(blocksCachedProperty);
// Get values of the counters and gauges and put them into a metrics map.
SortedMap<String, Long> metrics = new TreeMap<>();
for (Map.Entry<String, Counter> counter : counters.entrySet()) {
metrics.put(counter.getKey(), counter.getValue().getCount());
}
metrics.put(blocksCachedProperty, blocksCached.getValue().longValue());
return RestUtils.createResponse(metrics);
}
@Override
public BlockLocation[] getFileBlockLocations(FileStatus file, long start, long len)
throws IOException {
if (file == null) {
return null;
}
if (mStatistics != null) {
mStatistics.incrementReadOps(1);
}
AlluxioURI path = new AlluxioURI(HadoopUtils.getPathWithoutScheme(file.getPath()));
List<FileBlockInfo> blocks = getFileBlocks(path);
List<BlockLocation> blockLocations = new ArrayList<>();
for (FileBlockInfo fileBlockInfo : blocks) {
long offset = fileBlockInfo.getOffset();
long end = offset + fileBlockInfo.getBlockInfo().getLength();
// Check if there is any overlapping between [start, start+len] and [offset, end]
if (end >= start && offset <= start + len) {
ArrayList<String> names = new ArrayList<>();
ArrayList<String> hosts = new ArrayList<>();
// add the existing in-memory block locations
for (alluxio.wire.BlockLocation location : fileBlockInfo.getBlockInfo().getLocations()) {
HostAndPort address =
HostAndPort.fromParts(
location.getWorkerAddress().getHost(), location.getWorkerAddress().getDataPort());
names.add(address.toString());
hosts.add(address.getHostText());
}
// add under file system locations
for (String location : fileBlockInfo.getUfsLocations()) {
names.add(location);
hosts.add(HostAndPort.fromString(location).getHostText());
}
blockLocations.add(
new BlockLocation(
CommonUtils.toStringArray(names),
CommonUtils.toStringArray(hosts),
offset,
fileBlockInfo.getBlockInfo().getLength()));
}
}
BlockLocation[] ret = new BlockLocation[blockLocations.size()];
blockLocations.toArray(ret);
return ret;
}
/**
* Factory for {@link Allocator}.
*
* @param view {@link BlockMetadataManagerView} to pass to {@link Allocator}
* @return the generated {@link Allocator}, it will be a {@link MaxFreeAllocator} by default
*/
public static Allocator create(BlockMetadataManagerView view) {
BlockMetadataManagerView managerView = Preconditions.checkNotNull(view);
try {
return CommonUtils.createNewClassInstance(
Configuration.<Allocator>getClass(PropertyKey.WORKER_ALLOCATOR_CLASS),
new Class[] {BlockMetadataManagerView.class},
new Object[] {managerView});
} catch (Exception e) {
throw Throwables.propagate(e);
}
}
/** Tests that deleting a file from a folder updates the folder's last modification time. */
@Test
public void lastModificationTimeDeleteTest() throws Exception {
mFsMaster.createDirectory(new AlluxioURI("/testFolder"), CreateDirectoryOptions.defaults());
mFsMaster.createFile(new AlluxioURI("/testFolder/testFile"), CreateFileOptions.defaults());
long folderId = mFsMaster.getFileId(new AlluxioURI("/testFolder"));
long modificationTimeBeforeDelete = mFsMaster.getFileInfo(folderId).getLastModificationTimeMs();
CommonUtils.sleepMs(2);
mFsMaster.delete(new AlluxioURI("/testFolder/testFile"), true);
long modificationTimeAfterDelete = mFsMaster.getFileInfo(folderId).getLastModificationTimeMs();
Assert.assertTrue(modificationTimeBeforeDelete < modificationTimeAfterDelete);
}
public static CommandLineJobInfo createRandom() {
CommandLineJobInfo result = new CommandLineJobInfo();
Random random = new Random();
String command = CommonUtils.randomString(random.nextInt(10));
JobConfInfo jobConfInfo = JobConfInfoTest.createRandom();
result.setCommand(command);
result.setConf(jobConfInfo);
return result;
}
@Override
public OutputStream create(String path, CreateOptions options) throws IOException {
LOG.debug("Create method: {}", path);
// create will attempt to create the parent directory if it does not already exist
if (!mkdirs(getParentPath(path), true)) {
// fail if the parent directory does not exist and creation was unsuccessful
LOG.error("Parent directory creation unsuccessful for {}", path);
return null;
}
final String strippedPath = CommonUtils.stripPrefixIfPresent(path, Constants.HEADER_SWIFT);
// TODO(adit): remove special handling of */_SUCCESS objects
if (strippedPath.endsWith("_SUCCESS")) {
// when path/_SUCCESS is created, there is need to create path as
// an empty object. This is required by Spark in case Spark
// accesses path directly, bypassing Alluxio
String plainName = CommonUtils.stripSuffixIfPresent(strippedPath, "_SUCCESS");
SwiftDirectClient.put(mAccess, plainName).close();
}
// We do not check if a folder with the same name exists
return SwiftDirectClient.put(mAccess, strippedPath);
}
/**
* Creates a directory flagged file with the folder suffix.
*
* @param path the path to create a folder
* @return true if the operation was successful, false otherwise
*/
private boolean mkdirsInternal(String path) {
try {
String keyAsFolder =
addFolderSuffixIfNotPresent(
CommonUtils.stripPrefixIfPresent(path, Constants.HEADER_SWIFT));
// We do not check if a file with same name exists, i.e. a file with name
// 'swift://swift-container/path' and a folder with name 'swift://swift-container/path/'
// may both exist simultaneously
SwiftDirectClient.put(mAccess, keyAsFolder).close();
return true;
} catch (IOException e) {
LOG.error("Failed to create directory: {}", path, e);
return false;
}
}
@Override
public void heartbeat() {
int masterWorkerTimeoutMs = Configuration.getInt(Constants.MASTER_WORKER_TIMEOUT_MS);
for (MasterWorkerInfo worker : mWorkers) {
synchronized (worker) {
final long lastUpdate = CommonUtils.getCurrentMs() - worker.getLastUpdatedTimeMs();
if (lastUpdate > masterWorkerTimeoutMs) {
LOG.error(
"The worker {} timed out after {}ms without a heartbeat!", worker, lastUpdate);
mLostWorkers.add(worker);
mWorkers.remove(worker);
processWorkerRemovedBlocks(worker, worker.getBlocks());
}
}
}
}
@Test
public void ttlExpiredCreateFileTest() throws Exception {
mFsMaster.createDirectory(new AlluxioURI("/testFolder"), CreateDirectoryOptions.defaults());
long ttl = 1;
CreateFileOptions options = CreateFileOptions.defaults().setTtl(ttl);
long fileId = mFsMaster.createFile(new AlluxioURI("/testFolder/testFile1"), options);
FileInfo folderInfo =
mFsMaster.getFileInfo(mFsMaster.getFileId(new AlluxioURI("/testFolder/testFile1")));
Assert.assertEquals(fileId, folderInfo.getFileId());
Assert.assertEquals(ttl, folderInfo.getTtl());
// Sleep for the ttl expiration.
CommonUtils.sleepMs(2 * TTL_CHECKER_INTERVAL_MS);
Assert.assertTrue(
HeartbeatScheduler.await(HeartbeatContext.MASTER_TTL_CHECK, 10, TimeUnit.SECONDS));
HeartbeatScheduler.schedule(HeartbeatContext.MASTER_TTL_CHECK);
Assert.assertTrue(
HeartbeatScheduler.await(HeartbeatContext.MASTER_TTL_CHECK, 10, TimeUnit.SECONDS));
mThrown.expect(FileDoesNotExistException.class);
mFsMaster.getFileInfo(fileId);
}
/** Tests if delete deletes all files or folders for a large directory. */
@Test
public void deleteLargeDirectory() throws IOException {
LargeDirectoryConfig config = prepareLargeDirectoryTest();
mUfs.delete(config.getTopLevelDirectory(), true);
String[] children = config.getChildren();
for (String child : children) {
// Retry for some time to allow list operation eventual consistency for S3 and GCS.
// See http://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html and
// https://cloud.google.com/storage/docs/consistency for more details.
// Note: not using CommonUtils.waitFor here because we intend to sleep with a longer interval.
boolean childDeleted = false;
for (int i = 0; i < 20; i++) {
childDeleted = !mUfs.exists(child);
if (childDeleted) {
break;
}
CommonUtils.sleepMs(500);
}
Assert.assertTrue(childDeleted);
}
}
@Test
public void readBlockTest() throws Exception {
Map<String, String> params = new HashMap<>();
params.put("blockId", "1");
params.put("sessionId", "1");
params.put("lockId", "1");
params.put("offset", "0");
params.put("length", "-1");
Random random = new Random();
byte[] bytes = CommonUtils.randomBytes(random.nextInt(64));
ByteBuffer byteBuffer = ByteBuffer.wrap(bytes);
BlockReader blockReader = PowerMockito.mock(BlockReader.class);
Mockito.doReturn(byteBuffer).when(blockReader).read(Mockito.anyLong(), Mockito.anyLong());
Mockito.doReturn((long) bytes.length).when(blockReader).getLength();
Mockito.doReturn(blockReader)
.when(mBlockWorker)
.readBlockRemote(Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong());
TestCase testCase =
TestCaseFactory.newWorkerTestCase(
getEndpoint(BlockWorkerClientRestServiceHandler.READ_BLOCK),
params,
"GET",
byteBuffer,
mResource);
HttpURLConnection connection = (HttpURLConnection) testCase.createURL().openConnection();
connection.setRequestMethod(testCase.getMethod());
connection.connect();
Assert.assertEquals(
testCase.getEndpoint(), connection.getResponseCode(), Response.Status.OK.getStatusCode());
Assert.assertEquals(new String(byteBuffer.array()), testCase.getResponse(connection));
Mockito.verify(mBlockWorker)
.readBlockRemote(Mockito.anyLong(), Mockito.anyLong(), Mockito.anyLong());
}
@Test
public void requestBlockLocationTest() throws Exception {
Map<String, String> params = new HashMap<>();
params.put("blockId", "1");
params.put("sessionId", "1");
params.put("initialBytes", "1");
String blockLocation = CommonUtils.randomString(10);
Mockito.doReturn(blockLocation)
.when(mBlockWorker)
.createBlock(Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString(), Mockito.anyLong());
TestCaseFactory.newWorkerTestCase(
getEndpoint(BlockWorkerClientRestServiceHandler.REQUEST_BLOCK_LOCATION),
params,
"POST",
blockLocation,
mResource)
.run();
Mockito.verify(mBlockWorker)
.createBlock(Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString(), Mockito.anyLong());
}
@Test
public void writeBlockTest() throws Exception {
Map<String, String> params = new HashMap<>();
params.put("blockId", "1");
params.put("sessionId", "1");
params.put("offset", "0");
params.put("length", "-1");
Random random = new Random();
byte[] bytes = CommonUtils.randomBytes(random.nextInt(64));
BlockWriter blockWriter = PowerMockito.mock(BlockWriter.class);
Mockito.doReturn(blockWriter)
.when(mBlockWorker)
.getTempBlockWriterRemote(Mockito.anyLong(), Mockito.anyLong());
TestCase testCase =
TestCaseFactory.newWorkerTestCase(
getEndpoint(BlockWorkerClientRestServiceHandler.WRITE_BLOCK),
params,
"POST",
null,
mResource);
HttpURLConnection connection = (HttpURLConnection) testCase.createURL().openConnection();
connection.setRequestProperty("Content-Type", MediaType.APPLICATION_OCTET_STREAM);
connection.setRequestMethod(testCase.getMethod());
connection.setDoOutput(true);
connection.connect();
connection.getOutputStream().write(bytes);
Assert.assertEquals(
testCase.getEndpoint(), Response.Status.OK.getStatusCode(), connection.getResponseCode());
Assert.assertEquals("", testCase.getResponse(connection));
Mockito.verify(mBlockWorker).getTempBlockWriterRemote(Mockito.anyLong(), Mockito.anyLong());
Mockito.verify(blockWriter).append(ByteBuffer.wrap(bytes));
}
private void setupShellMocks(String username, List<String> groups) throws IOException {
PowerMockito.mockStatic(CommonUtils.class);
PowerMockito.when(CommonUtils.getUnixGroups(Mockito.eq(username))).thenReturn(groups);
}
/**
* Strips the folder suffix if it exists. This is a string manipulation utility and does not
* guarantee the existence of the folder. This method will leave paths without a suffix unaltered.
*
* @param path the path to strip the suffix from
* @return the path with the suffix removed, or the path unaltered if the suffix is not present
*/
private String stripFolderSuffixIfPresent(final String path) {
return CommonUtils.stripSuffixIfPresent(path, FOLDER_SUFFIX);
}
/**
* Returns list of groups for a user.
*
* @param user get groups for this user
* @return list of groups for a given user
* @throws IOException when getting the UNIX groups
*/
@Override
public List<String> getGroups(String user) throws IOException {
List<String> groups = CommonUtils.getUnixGroups(user);
// remove duplicated primary group
return new ArrayList<String>(new LinkedHashSet<String>(groups));
}
/**
* Strips the Swift container prefix from the path if it is present. For example, for input path
* swift://my-container-name/my-path/file, the output would be my-path/file. This method will
* leave paths without a prefix unaltered, ie. my-path/file returns my-path/file.
*
* @param path the path to strip
* @return the path without the Swift container prefix
*/
private String stripContainerPrefixIfPresent(final String path) {
return CommonUtils.stripPrefixIfPresent(path, mContainerPrefix);
}