@Override
public boolean rename(Path src, Path dst) throws IOException {
LOG.info("rename({}, {})", src, dst);
if (mStatistics != null) {
mStatistics.incrementWriteOps(1);
}
AlluxioURI srcPath = new AlluxioURI(HadoopUtils.getPathWithoutScheme(src));
AlluxioURI dstPath = new AlluxioURI(HadoopUtils.getPathWithoutScheme(dst));
ensureExists(srcPath);
URIStatus dstStatus;
try {
dstStatus = mFileSystem.getStatus(dstPath);
} catch (IOException | AlluxioException e) {
dstStatus = null;
}
// If the destination is an existing folder, try to move the src into the folder
if (dstStatus != null && dstStatus.isFolder()) {
dstPath = dstPath.join(srcPath.getName());
}
try {
mFileSystem.rename(srcPath, dstPath);
return true;
} catch (IOException | AlluxioException e) {
LOG.error("Failed to rename {} to {}", src, dst, e);
return false;
}
}
/**
* {@inheritDoc}
*
* <p>If the file does not exist in Alluxio, query it from HDFS.
*/
@Override
public FileStatus getFileStatus(Path path) throws IOException {
LOG.info("getFileStatus({})", path);
if (mStatistics != null) {
mStatistics.incrementReadOps(1);
}
AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path));
URIStatus fileStatus;
try {
fileStatus = mFileSystem.getStatus(uri);
} catch (FileDoesNotExistException e) {
throw new FileNotFoundException(e.getMessage());
} catch (AlluxioException e) {
throw new IOException(e);
}
return new FileStatus(
fileStatus.getLength(),
fileStatus.isFolder(),
BLOCK_REPLICATION_CONSTANT,
fileStatus.getBlockSizeBytes(),
fileStatus.getLastModificationTimeMs(),
fileStatus.getCreationTimeMs(),
new FsPermission((short) fileStatus.getMode()),
fileStatus.getOwner(),
fileStatus.getGroup(),
new Path(mAlluxioHeader + uri));
}
/**
* {@inheritDoc}
*
* <p>Sets up a lazy connection to Alluxio through mFileSystem. This method will override and
* invalidate the current contexts. This must be called before client operations in order to
* guarantee the integrity of the contexts, meaning users should not alternate between using the
* Hadoop compatible API and native Alluxio API in the same process.
*
* <p>If hadoop file system cache is enabled, this method should only be called when switching
* user.
*/
@SuppressFBWarnings("ST_WRITE_TO_STATIC_FROM_INSTANCE_METHOD")
@Override
public void initialize(URI uri, org.apache.hadoop.conf.Configuration conf) throws IOException {
Preconditions.checkNotNull(uri.getHost(), PreconditionMessage.URI_HOST_NULL);
Preconditions.checkNotNull(uri.getPort(), PreconditionMessage.URI_PORT_NULL);
super.initialize(uri, conf);
LOG.info("initialize({}, {}). Connecting to Alluxio", uri, conf);
HadoopUtils.addS3Credentials(conf);
HadoopUtils.addSwiftCredentials(conf);
setConf(conf);
mAlluxioHeader = getScheme() + "://" + uri.getHost() + ":" + uri.getPort();
// Set the statistics member. Use mStatistics instead of the parent class's variable.
mStatistics = statistics;
mUri = URI.create(mAlluxioHeader);
boolean masterAddIsSameAsDefault = checkMasterAddress();
if (sInitialized) {
if (!masterAddIsSameAsDefault) {
throw new IOException(
ExceptionMessage.DIFFERENT_MASTER_ADDRESS.getMessage(
mUri.getHost() + ":" + mUri.getPort(),
FileSystemContext.INSTANCE.getMasterAddress()));
}
updateFileSystemAndContext();
return;
}
synchronized (INIT_LOCK) {
// If someone has initialized the object since the last check, return
if (sInitialized) {
if (!masterAddIsSameAsDefault) {
throw new IOException(
ExceptionMessage.DIFFERENT_MASTER_ADDRESS.getMessage(
mUri.getHost() + ":" + mUri.getPort(),
FileSystemContext.INSTANCE.getMasterAddress()));
}
updateFileSystemAndContext();
return;
}
initializeInternal(uri, conf);
sInitialized = true;
}
updateFileSystemAndContext();
}
public void reduce(
Text key, Iterator<Text> values, OutputCollector<Text, Text> output, Reporter reporter)
throws IOException {
if (m_reducer == null) {
ExecutionSource exeSrc = HadoopUtils.initFromJobAndCreateSource(m_job, true);
m_reducer = new MR4CReducer(exeSrc);
}
m_reducer.reduce(key, values, output, reporter);
}
/**
* Attempts to open the specified file for reading.
*
* @param path the file name to open
* @param bufferSize the size in bytes of the buffer to be used
* @return an {@link FSDataInputStream} at the indicated path of a file
* @throws IOException if the file cannot be opened (e.g., the path is a folder)
*/
@Override
public FSDataInputStream open(Path path, int bufferSize) throws IOException {
LOG.info("open({}, {})", path, bufferSize);
if (mStatistics != null) {
mStatistics.incrementReadOps(1);
}
AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path));
return new FSDataInputStream(
new HdfsFileInputStream(mContext, uri, getConf(), bufferSize, mStatistics));
}
/**
* Changes permission of a path.
*
* @param path path to set permission
* @param permission permission set to path
* @throws IOException if the path failed to be changed permission
*/
@Override
public void setPermission(Path path, FsPermission permission) throws IOException {
LOG.info("setMode({},{})", path, permission.toString());
AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path));
SetAttributeOptions options =
SetAttributeOptions.defaults().setMode(new Mode(permission.toShort())).setRecursive(false);
try {
mFileSystem.setAttribute(uri, options);
} catch (AlluxioException e) {
throw new IOException(e);
}
}
/**
* Attempts to create a file. Overwrite will not succeed if the path exists and is a folder.
*
* @param path path to create
* @param permission permissions of the created file/folder
* @param overwrite overwrite if file exists
* @param bufferSize the size in bytes of the buffer to be used
* @param replication under filesystem replication factor
* @param blockSize block size in bytes
* @param progress queryable progress
* @return an {@link FSDataOutputStream} created at the indicated path of a file
* @throws IOException if overwrite is not specified and the path already exists or if the path is
* a folder
*/
@Override
public FSDataOutputStream create(
Path path,
FsPermission permission,
boolean overwrite,
int bufferSize,
short replication,
long blockSize,
Progressable progress)
throws IOException {
LOG.info(
"create({}, {}, {}, {}, {}, {}, {})",
path,
permission,
overwrite,
bufferSize,
replication,
blockSize,
progress);
if (mStatistics != null) {
mStatistics.incrementWriteOps(1);
}
// Check whether the file already exists, and delete it if overwrite is true
AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path));
try {
if (mFileSystem.exists(uri)) {
if (!overwrite) {
throw new IOException(ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(uri));
}
if (mFileSystem.getStatus(uri).isFolder()) {
throw new IOException(ExceptionMessage.FILE_CREATE_IS_DIRECTORY.getMessage(uri));
}
mFileSystem.delete(uri);
}
} catch (AlluxioException e) {
throw new IOException(e);
}
// The file no longer exists at this point, so we can create it
CreateFileOptions options =
CreateFileOptions.defaults()
.setBlockSizeBytes(blockSize)
.setMode(new Mode(permission.toShort()));
try {
FileOutStream outStream = mFileSystem.createFile(uri, options);
return new FSDataOutputStream(outStream, mStatistics);
} catch (AlluxioException e) {
throw new IOException(e);
}
}
@Override
public FileStatus[] listStatus(Path path) throws IOException {
LOG.info("listStatus({})", path);
if (mStatistics != null) {
mStatistics.incrementReadOps(1);
}
AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path));
List<URIStatus> statuses;
try {
statuses = mFileSystem.listStatus(uri);
} catch (FileDoesNotExistException e) {
throw new FileNotFoundException(HadoopUtils.getPathWithoutScheme(path));
} catch (AlluxioException e) {
throw new IOException(e);
}
FileStatus[] ret = new FileStatus[statuses.size()];
for (int k = 0; k < statuses.size(); k++) {
URIStatus status = statuses.get(k);
ret[k] =
new FileStatus(
status.getLength(),
status.isFolder(),
BLOCK_REPLICATION_CONSTANT,
status.getBlockSizeBytes(),
status.getLastModificationTimeMs(),
status.getCreationTimeMs(),
new FsPermission((short) status.getMode()),
status.getOwner(),
status.getGroup(),
new Path(mAlluxioHeader + status.getPath()));
}
return ret;
}
/**
* Opens an {@link FSDataOutputStream} at the indicated Path with write-progress reporting. Same
* as {@link #create(Path, boolean, int, short, long, Progressable)}, except fails if parent
* directory doesn't already exist.
*
* <p>TODO(hy): We need to refactor this method after having a new internal API support
* (ALLUXIO-46).
*
* @param path the file name to open
* @param overwrite if a file with this name already exists, then if true, the file will be
* overwritten, and if false an error will be thrown.
* @param bufferSize the size of the buffer to be used
* @param replication required block replication for the file
* @param blockSize the size in bytes of the buffer to be used
* @param progress queryable progress
* @throws IOException if 1) overwrite is not specified and the path already exists, 2) if the
* path is a folder, or 3) the parent directory does not exist
* @see #setPermission(Path, FsPermission)
* @deprecated API only for 0.20-append
*/
@Override
@Deprecated
public FSDataOutputStream createNonRecursive(
Path path,
FsPermission permission,
boolean overwrite,
int bufferSize,
short replication,
long blockSize,
Progressable progress)
throws IOException {
AlluxioURI parentUri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path.getParent()));
ensureExists(parentUri);
return create(path, permission, overwrite, bufferSize, replication, blockSize, progress);
}
@Override
public BlockLocation[] getFileBlockLocations(FileStatus file, long start, long len)
throws IOException {
if (file == null) {
return null;
}
if (mStatistics != null) {
mStatistics.incrementReadOps(1);
}
AlluxioURI path = new AlluxioURI(HadoopUtils.getPathWithoutScheme(file.getPath()));
List<FileBlockInfo> blocks = getFileBlocks(path);
List<BlockLocation> blockLocations = new ArrayList<>();
for (FileBlockInfo fileBlockInfo : blocks) {
long offset = fileBlockInfo.getOffset();
long end = offset + fileBlockInfo.getBlockInfo().getLength();
// Check if there is any overlapping between [start, start+len] and [offset, end]
if (end >= start && offset <= start + len) {
ArrayList<String> names = new ArrayList<>();
ArrayList<String> hosts = new ArrayList<>();
// add the existing in-memory block locations
for (alluxio.wire.BlockLocation location : fileBlockInfo.getBlockInfo().getLocations()) {
HostAndPort address =
HostAndPort.fromParts(
location.getWorkerAddress().getHost(), location.getWorkerAddress().getDataPort());
names.add(address.toString());
hosts.add(address.getHostText());
}
// add under file system locations
for (String location : fileBlockInfo.getUfsLocations()) {
names.add(location);
hosts.add(HostAndPort.fromString(location).getHostText());
}
blockLocations.add(
new BlockLocation(
CommonUtils.toStringArray(names),
CommonUtils.toStringArray(hosts),
offset,
fileBlockInfo.getBlockInfo().getLength()));
}
}
BlockLocation[] ret = new BlockLocation[blockLocations.size()];
blockLocations.toArray(ret);
return ret;
}
@Override
public FSDataOutputStream append(Path path, int bufferSize, Progressable progress)
throws IOException {
LOG.info("append({}, {}, {})", path, bufferSize, progress);
if (mStatistics != null) {
mStatistics.incrementWriteOps(1);
}
AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path));
try {
if (mFileSystem.exists(uri)) {
throw new IOException(ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(uri));
}
return new FSDataOutputStream(mFileSystem.createFile(uri), mStatistics);
} catch (AlluxioException e) {
throw new IOException(e);
}
}
/**
* Attempts to delete the file or directory with the specified path.
*
* @param path path to delete
* @param recursive if true, will attempt to delete all children of the path
* @return true if one or more files/directories were deleted; false otherwise
* @throws IOException if the path failed to be deleted due to some constraint (ie. non empty
* directory with recursive flag disabled)
*/
@Override
public boolean delete(Path path, boolean recursive) throws IOException {
LOG.info("delete({}, {})", path, recursive);
if (mStatistics != null) {
mStatistics.incrementWriteOps(1);
}
AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path));
DeleteOptions options = DeleteOptions.defaults().setRecursive(recursive);
try {
mFileSystem.delete(uri, options);
return true;
} catch (InvalidPathException | FileDoesNotExistException e) {
LOG.error("delete failed: {}", e.getMessage());
return false;
} catch (AlluxioException e) {
throw new IOException(e);
}
}
/**
* Attempts to create a folder with the specified path. Parent directories will be created.
*
* @param path path to create
* @param permission permissions to grant the created folder
* @return true if the indicated folder is created successfully or already exists
* @throws IOException if the folder cannot be created
*/
@Override
public boolean mkdirs(Path path, FsPermission permission) throws IOException {
LOG.info("mkdirs({}, {})", path, permission);
if (mStatistics != null) {
mStatistics.incrementWriteOps(1);
}
AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path));
CreateDirectoryOptions options =
CreateDirectoryOptions.defaults()
.setRecursive(true)
.setAllowExists(true)
.setMode(new Mode(permission.toShort()));
try {
mFileSystem.createDirectory(uri, options);
return true;
} catch (AlluxioException e) {
throw new IOException(e);
}
}
/**
* Changes owner or group of a path (i.e. a file or a directory). If username is null, the
* original username remains unchanged. Same as groupname. If username and groupname are non-null,
* both of them will be changed.
*
* @param path path to set owner or group
* @param username username to be set
* @param groupname groupname to be set
* @throws IOException if changing owner or group of the path failed
*/
@Override
public void setOwner(Path path, final String username, final String groupname)
throws IOException {
LOG.info("setOwner({},{},{})", path, username, groupname);
AlluxioURI uri = new AlluxioURI(HadoopUtils.getPathWithoutScheme(path));
SetAttributeOptions options = SetAttributeOptions.defaults();
boolean ownerOrGroupChanged = false;
if (username != null && !username.isEmpty()) {
options.setOwner(username).setRecursive(false);
ownerOrGroupChanged = true;
}
if (groupname != null && !groupname.isEmpty()) {
options.setGroup(groupname).setRecursive(false);
ownerOrGroupChanged = true;
}
if (ownerOrGroupChanged) {
try {
mFileSystem.setAttribute(uri, options);
} catch (AlluxioException e) {
throw new IOException(e);
}
}
}
public static void setClassConf(
Configuration jobConf, Class<?> genericClass, Class<? extends Message> protoClass) {
HadoopUtils.setClassConf(jobConf, CLASS_CONF_PREFIX + genericClass.getName(), protoClass);
}
