@Override
public boolean rename(String src, String dst) throws IOException {
if (!exists(src)) {
LOG.error("Unable to rename " + src + " to " + dst + " because source does not exist.");
return false;
}
if (exists(dst)) {
LOG.error("Unable to rename " + src + " to " + dst + " because destination already exists.");
return false;
}
// Source exists and destination does not exist
if (isFolder(src)) {
// Rename the source folder first
if (!copy(convertToFolderName(src), convertToFolderName(dst))) {
return false;
}
// Rename each child in the src folder to destination/child
String[] children = list(src);
for (String child : children) {
if (!rename(PathUtils.concatPath(src, child), PathUtils.concatPath(dst, child))) {
return false;
}
}
// Delete src and everything under src
return delete(src, true);
}
// Source is a file and Destination does not exist
return copy(src, dst) && deleteInternal(src);
}
/** Test <code>long skip(long len)</code>. */
@Test
public void skipTest() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
TachyonFile f =
TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteUnderStore);
FileInStream is = mTfs.getInStream(f, mReadCache);
Assert.assertEquals(k / 2, is.skip(k / 2));
Assert.assertEquals(k / 2, is.read());
is.close();
Assert.assertFalse(mTfs.getInfo(f).getInMemoryPercentage() == 100);
if (k >= 3) {
is = mTfs.getInStream(f, mReadCache);
int t = k / 3;
Assert.assertEquals(t, is.skip(t));
Assert.assertEquals(t, is.read());
Assert.assertEquals(t, is.skip(t));
Assert.assertEquals(2 * t + 1, is.read());
is.close();
Assert.assertFalse(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
}
}
/**
* Copies a list of files or directories specified by srcFiles from the local filesystem to
* dstPath in the Tachyon filesystem space. This method is used when the input path contains
* wildcards.
*
* @param srcFiles The list of files in the local filesystem
* @param dstPath The {@link TachyonURI} of the destination
* @throws IOException if a non-Tachyon related exception occurs
*/
private void copyFromLocalWildcard(List<File> srcFiles, TachyonURI dstPath) throws IOException {
try {
mTfs.mkdir(dstPath);
} catch (FileAlreadyExistsException e) {
// it's fine if the directory already exists
} catch (TachyonException e) {
throw new IOException(e.getMessage());
}
FileInfo dstFileInfo;
try {
TachyonFile dstFd = mTfs.open(dstPath);
dstFileInfo = mTfs.getInfo(dstFd);
} catch (TachyonException e) {
throw new IOException(e.getMessage());
}
if (!dstFileInfo.isFolder) {
throw new IOException(
ExceptionMessage.DESTINATION_FILE_CANNOT_EXIST_WITH_WILDCARD_SOURCE.getMessage());
}
List<String> errorMessages = Lists.newArrayList();
for (File srcFile : srcFiles) {
try {
copyFromLocal(
srcFile, new TachyonURI(PathUtils.concatPath(dstPath.getPath(), srcFile.getName())));
} catch (IOException e) {
errorMessages.add(e.getMessage());
}
}
if (errorMessages.size() != 0) {
throw new IOException(Joiner.on('\n').join(errorMessages));
}
}
/**
* Creates a new file output stream.
*
* @param path the file path
* @param options the client options
* @throws IOException if an I/O error occurs
*/
public FileOutStream(TachyonURI path, OutStreamOptions options) throws IOException {
mUri = Preconditions.checkNotNull(path);
mNonce = ClientUtils.getRandomNonNegativeLong();
mBlockSize = options.getBlockSizeBytes();
mTachyonStorageType = options.getTachyonStorageType();
mUnderStorageType = options.getUnderStorageType();
mContext = FileSystemContext.INSTANCE;
mPreviousBlockOutStreams = new LinkedList<BufferedBlockOutStream>();
if (mUnderStorageType.isSyncPersist()) {
updateUfsPath();
String tmpPath = PathUtils.temporaryFileName(mNonce, mUfsPath);
UnderFileSystem ufs = UnderFileSystem.get(tmpPath, ClientContext.getConf());
// TODO(jiri): Implement collection of temporary files left behind by dead clients.
mUnderStorageOutputStream = ufs.create(tmpPath, (int) mBlockSize);
} else {
mUfsPath = null;
mUnderStorageOutputStream = null;
}
mClosed = false;
mCanceled = false;
mShouldCacheCurrentBlock = mTachyonStorageType.isStore();
mBytesWritten = 0;
mLocationPolicy =
Preconditions.checkNotNull(
options.getLocationPolicy(),
PreconditionMessage.FILE_WRITE_LOCATION_POLICY_UNSPECIFIED);
}
private void initStorageTier() throws AlreadyExistsException, IOException, OutOfSpaceException {
String workerDataFolder =
WorkerContext.getConf().get(Constants.WORKER_DATA_FOLDER, Constants.DEFAULT_DATA_FOLDER);
String tierDirPathConf =
String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_PATH_FORMAT, mTierLevel);
String[] dirPaths = WorkerContext.getConf().get(tierDirPathConf, "/mnt/ramdisk").split(",");
// Add the worker data folder path after each storage directory, the final path will be like
// /mnt/ramdisk/tachyonworker
for (int i = 0; i < dirPaths.length; i++) {
dirPaths[i] = PathUtils.concatPath(dirPaths[i].trim(), workerDataFolder);
}
String tierDirCapacityConf =
String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_QUOTA_FORMAT, mTierLevel);
String[] dirQuotas = WorkerContext.getConf().get(tierDirCapacityConf, "0").split(",");
mDirs = new ArrayList<StorageDir>(dirPaths.length);
long totalCapacity = 0;
for (int i = 0; i < dirPaths.length; i++) {
int index = i >= dirQuotas.length ? dirQuotas.length - 1 : i;
long capacity = FormatUtils.parseSpaceSize(dirQuotas[index]);
totalCapacity += capacity;
mDirs.add(StorageDir.newStorageDir(this, i, capacity, dirPaths[i]));
}
mCapacityBytes = totalCapacity;
}
/**
* Creates a store with the specified number of partitions.
*
* <p>NOTE: calling this method will set {@link Constants#KEY_VALUE_PARTITION_SIZE_BYTES_MAX} to
* {@link Constants#MB}.
*
* @param partitionNumber the number of partitions
* @param keyValuePairs the key-value pairs in the store, null if you don't want to know them
* @return the URI to the created store
*/
private TachyonURI createStoreOfMultiplePartitions(
int partitionNumber, List<KeyValuePair> keyValuePairs) throws Exception {
// These sizes are carefully selected, one partition holds only one key-value pair.
final long maxPartitionSize = Constants.MB; // Each partition is at most 1 MB
ClientContext.getConf()
.set(Constants.KEY_VALUE_PARTITION_SIZE_BYTES_MAX, String.valueOf(maxPartitionSize));
final int keyLength = 4; // 4Byte key
final int valueLength = 500 * Constants.KB; // 500KB value
TachyonURI storeUri = new TachyonURI(PathUtils.uniqPath());
mWriter = sKeyValueSystem.createStore(storeUri);
for (int i = 0; i < partitionNumber; i++) {
byte[] key = BufferUtils.getIncreasingByteArray(i, keyLength);
byte[] value = BufferUtils.getIncreasingByteArray(i, valueLength);
mWriter.put(key, value);
if (keyValuePairs != null) {
keyValuePairs.add(new KeyValuePair(key, value));
}
}
mWriter.close();
Assert.assertEquals(partitionNumber, getPartitionNumber(storeUri));
return storeUri;
}
/** Test <code>void read()</code>. Read from remote data server. */
@Test
public void readTest4() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
TachyonFile f =
TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteTachyon);
long blockId = mTfs.getInfo(f).getBlockIds().get(0);
BlockInfo info = TachyonBlockStore.get().getInfo(blockId);
RemoteBlockInStream is =
new RemoteBlockInStream(
info.getBlockId(), info.getLength(), info.getLocations().get(0).getWorkerAddress());
byte[] ret = new byte[k];
int value = is.read();
int cnt = 0;
while (value != -1) {
Assert.assertTrue(value >= 0);
Assert.assertTrue(value < 256);
ret[cnt++] = (byte) value;
value = is.read();
}
Assert.assertEquals(cnt, k);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
}
/** Test <code>void read(byte[] b, int off, int len)</code>. Read from underfs. */
@Test
public void readTest3() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
TachyonFile f =
TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteUnderStore);
FileInStream is = mTfs.getInStream(f, mReadNoCache);
byte[] ret = new byte[k / 2];
Assert.assertEquals(k / 2, is.read(ret, 0, k / 2));
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k / 2, ret));
is.close();
if (k == 0) {
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
} else {
Assert.assertFalse(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
is = mTfs.getInStream(f, mReadCache);
ret = new byte[k];
Assert.assertEquals(k, is.read(ret, 0, k));
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
is = mTfs.getInStream(f, mReadCache);
ret = new byte[k];
Assert.assertEquals(k, is.read(ret));
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
}
/** Test {@link tachyon.client.block.BufferedBlockInStream#read()}. */
@Test
public void readTest1() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
for (CreateFileOptions op : getOptionSet()) {
TachyonURI path = new TachyonURI(uniqPath + "/file_" + k + "_" + op.hashCode());
FileSystemTestUtils.createByteFile(sFileSystem, path, op, k);
for (int i = 0; i < 2; i++) {
FileInStream is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
byte[] ret = new byte[k];
int value = is.read();
int cnt = 0;
while (value != -1) {
Assert.assertTrue(value >= 0);
Assert.assertTrue(value < 256);
ret[cnt++] = (byte) value;
value = is.read();
}
Assert.assertEquals(cnt, k);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
}
}
}
}
/** Test <code>void read()</code>. Read from underfs. */
@Test
public void readTest1() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
TachyonFile f =
TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteUnderStore);
FileInStream is = mTfs.getInStream(f, mReadNoCache);
byte[] ret = new byte[k];
int value = is.read();
int cnt = 0;
while (value != -1) {
Assert.assertTrue(value >= 0);
Assert.assertTrue(value < 256);
ret[cnt++] = (byte) value;
value = is.read();
}
Assert.assertEquals(cnt, k);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
if (k == 0) {
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
} else {
Assert.assertFalse(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
is = mTfs.getInStream(f, mReadCache);
ret = new byte[k];
value = is.read();
cnt = 0;
while (value != -1) {
Assert.assertTrue(value >= 0);
Assert.assertTrue(value < 256);
ret[cnt++] = (byte) value;
value = is.read();
}
Assert.assertEquals(cnt, k);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
is = mTfs.getInStream(f, mReadCache);
ret = new byte[k];
value = is.read();
cnt = 0;
while (value != -1) {
Assert.assertTrue(value >= 0);
Assert.assertTrue(value < 256);
ret[cnt++] = (byte) value;
value = is.read();
}
Assert.assertEquals(cnt, k);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
}
/**
* Tests that not reading a file the whole way through then closing it will cause it to not
* recache
*/
@Test
public void incompleteFileReadCancelsRecache() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
TachyonFile f = TachyonFSTestUtils.createByteFile(mTfs, uniqPath, 2, mWriteUnderStore);
FileInStream is = mTfs.getInStream(f, mReadNoCache);
Assert.assertEquals(0, is.read());
is.close();
Assert.assertFalse(mTfs.getInfo(f).getInMemoryPercentage() == 100);
is = mTfs.getInStream(f, mReadNoCache);
is.close();
}
/** Tests that seeking around a file cached locally works. */
@Test
public void seekAroundLocalBlock() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
// The number of bytes per remote block read should be set to 100 in the before function
TachyonFile f = TachyonFSTestUtils.createByteFile(mTfs, uniqPath, 200, mWriteTachyon);
FileInStream is = mTfs.getInStream(f, mReadNoCache);
Assert.assertEquals(0, is.read());
is.seek(199);
Assert.assertEquals(199, is.read());
is.seek(99);
Assert.assertEquals(99, is.read());
is.close();
}
/** Tests that reading a file the whole way through with the STORE ReadType will recache it */
@Test
public void completeFileReadTriggersRecache() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
int len = 2;
TachyonFile f = TachyonFSTestUtils.createByteFile(mTfs, uniqPath, len, mWriteUnderStore);
FileInStream is = mTfs.getInStream(f, mReadCache);
for (int i = 0; i < len; ++i) {
Assert.assertEquals(i, is.read());
}
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
@Override
public boolean delete(String path, boolean recursive) throws IOException {
File file = new File(path);
boolean success = true;
if (recursive && file.isDirectory()) {
String[] files = file.list();
for (String child : files) {
success = success && delete(PathUtils.concatPath(path, child), true);
}
}
return success && file.delete();
}
/**
* Validates the path, verifying that it contains the {@link Constants#HEADER} or {@link
* Constants#HEADER_FT} and a hostname:port specified.
*
* @param path the path to be verified
* @param tachyonConf the instance of {@link tachyon.conf.TachyonConf} to be used
* @return the verified path in a form like tachyon://host:port/dir. If only the "/dir" or "dir"
* part is provided, the host and port are retrieved from property, tachyon.master.hostname
* and tachyon.master.port, respectively.
* @throws IOException if the given path is not valid
*/
public static String validatePath(String path, TachyonConf tachyonConf) throws IOException {
if (path.startsWith(Constants.HEADER) || path.startsWith(Constants.HEADER_FT)) {
if (!path.contains(":")) {
throw new IOException(
"Invalid Path: "
+ path
+ ". Use "
+ Constants.HEADER
+ "host:port/ ,"
+ Constants.HEADER_FT
+ "host:port/"
+ " , or /file");
} else {
return path;
}
} else {
String hostname = NetworkAddressUtils.getConnectHost(ServiceType.MASTER_RPC, tachyonConf);
int port = tachyonConf.getInt(Constants.MASTER_PORT);
if (tachyonConf.getBoolean(Constants.ZOOKEEPER_ENABLED)) {
return PathUtils.concatPath(Constants.HEADER_FT + hostname + ":" + port, path);
}
return PathUtils.concatPath(Constants.HEADER + hostname + ":" + port, path);
}
}
/**
* Copies a list of files or directories specified by srcFiles from the local filesystem to
* dstPath in the Tachyon filesystem space. This method is used when the input path contains
* wildcards.
*
* @param srcFiles The list of files in the local filesystem
* @param dstPath The TachyonURI of the destination
* @return 0 if command is successful, -1 if an error occurred.
* @throws IOException
*/
public int copyFromLocalWildcard(List<File> srcFiles, TachyonURI dstPath) throws IOException {
try {
mTfs.mkdir(dstPath);
} catch (TachyonException e) {
switch (e.getType()) {
case INVALID_PATH:
System.out.print("Fail to create directory (Invalid path): " + dstPath);
return -1;
case FILE_DOES_NOT_EXIST:
// this is fine
break;
default:
throw new IOException(e.getMessage());
}
}
FileInfo dstFileInfo;
try {
TachyonFile dstFd = mTfs.open(dstPath);
if (dstFd == null) {
System.out.print(ExceptionMessage.PATH_DOES_NOT_EXIST.getMessage(dstPath));
return -1;
}
dstFileInfo = mTfs.getInfo(dstFd);
} catch (TachyonException e) {
throw new IOException(e);
}
if (!dstFileInfo.isFolder) {
System.out.println(
"The destination cannot be an existent file when the src contains " + "wildcards.");
return -1;
}
int exitCode = 0;
for (File srcFile : srcFiles) {
try {
exitCode |=
copyFromLocal(
srcFile,
new TachyonURI(PathUtils.concatPath(dstPath.getPath(), srcFile.getName())));
} catch (IOException ioe) {
System.out.println(ioe.getMessage());
exitCode |= -1;
}
}
return exitCode;
}
/**
* Test <code>void seek(long pos)</code>. Validate the expected exception for seeking a position
* that is past block size.
*
* @throws IOException
* @throws TachyonException
*/
@Test
public void seekExceptionTest2() throws IOException, TachyonException {
mThrown.expect(IllegalArgumentException.class);
mThrown.expectMessage("Seek position is past EOF: 1, fileSize = 0");
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
TachyonFile f =
TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteUnderStore);
FileInStream is = mTfs.getInStream(f, mReadNoCache);
try {
is.seek(k + 1);
} finally {
is.close();
}
}
}
@BeforeClass
public static final void beforeClass() throws Exception {
sFileSystem = sLocalTachyonClusterResource.get().getClient();
sTachyonConf = sLocalTachyonClusterResource.get().getMasterTachyonConf();
sWriteBoth = StreamOptionUtils.getCreateFileOptionsCacheThrough(sTachyonConf);
sWriteTachyon = StreamOptionUtils.getCreateFileOptionsMustCache(sTachyonConf);
sWriteUnderStore = StreamOptionUtils.getCreateFileOptionsThrough(sTachyonConf);
sTestPath = PathUtils.uniqPath();
// Create files of varying size and write type to later read from
for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
for (CreateFileOptions op : getOptionSet()) {
TachyonURI path = new TachyonURI(sTestPath + "/file_" + k + "_" + op.hashCode());
FileSystemTestUtils.createByteFile(sFileSystem, path, op, k);
}
}
}
/**
* Creates a store with the specified number of key-value pairs. The key-value pairs are in the
* format specified in {@link #genBaseKey(int)} and {@link #genBaseValue(int)} with id starts from
* 0.
*
* <p>The created store's size is {@link Assert}ed before return.
*
* @param size the number of key-value pairs
* @param pairs the key-value pairs in the store, null if you don't want to know them
* @return the URI to the store
* @throws Exception if any error happens
*/
private TachyonURI createStoreOfSize(int size, List<KeyValuePair> pairs) throws Exception {
TachyonURI path = new TachyonURI(PathUtils.uniqPath());
KeyValueStoreWriter writer = sKeyValueSystem.createStore(path);
for (int i = 0; i < size; i++) {
byte[] key = genBaseKey(i).getBytes();
byte[] value = genBaseValue(i).getBytes();
writer.put(key, value);
if (pairs != null) {
pairs.add(new KeyValuePair(key, value));
}
}
writer.close();
Assert.assertEquals(size, sKeyValueSystem.openStore(path).size());
return path;
}
/**
* Test <code>void seek(long pos)</code>.
*
* @throws IOException
* @throws TachyonException
*/
@Test
public void seekTest() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
TachyonFile f =
TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteUnderStore);
FileInStream is = mTfs.getInStream(f, mReadNoCache);
Assert.assertEquals(0, is.read());
is.seek(k / 3);
Assert.assertEquals(k / 3, is.read());
is.seek(k / 2);
Assert.assertEquals(k / 2, is.read());
is.seek(k / 4);
Assert.assertEquals(k / 4, is.read());
is.close();
}
}
/** Tests that reading a file consisting of more than one block from the underfs works */
@Test
public void readMultiBlockFile() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
int blockSizeByte = 10;
int numBlocks = 10;
FileOutStream os = mTfs.getOutStream(new TachyonURI(uniqPath), mWriteUnderStore);
for (int i = 0; i < numBlocks; i++) {
for (int j = 0; j < blockSizeByte; j++) {
os.write((byte) (i * blockSizeByte + j));
}
}
os.close();
TachyonFile f = mTfs.open(new TachyonURI(uniqPath));
FileInStream is = mTfs.getInStream(f, mReadCache);
for (int i = 0; i < blockSizeByte * numBlocks; i++) {
Assert.assertEquals((byte) i, is.read());
}
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
/** Test {@link tachyon.client.block.BufferedBlockInStream#read(byte[])}. */
@Test
public void readTest2() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN; k <= MAX_LEN; k += DELTA) {
for (CreateFileOptions op : getOptionSet()) {
TachyonURI path = new TachyonURI(uniqPath + "/file_" + k + "_" + op.hashCode());
FileSystemTestUtils.createByteFile(sFileSystem, path, op, k);
FileInStream is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
byte[] ret = new byte[k];
Assert.assertEquals(k, is.read(ret));
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
ret = new byte[k];
Assert.assertEquals(k, is.read(ret));
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(k, ret));
is.close();
}
}
}
/** Test <code>void read(byte[] b, int off, int len)</code>. Read from remote data server. */
@Test
public void readTest6() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
TachyonFile f =
TachyonFSTestUtils.createByteFile(mTfs, uniqPath + "/file_" + k, k, mWriteTachyon);
long blockId = mTfs.getInfo(f).getBlockIds().get(0);
BlockInfo info = TachyonBlockStore.get().getInfo(blockId);
RemoteBlockInStream is =
new RemoteBlockInStream(
info.getBlockId(), info.getLength(), info.getLocations().get(0).getWorkerAddress());
byte[] ret = new byte[k / 2];
int start = 0;
while (start < k / 2) {
int read = is.read(ret, 0, (k / 2) - start);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(start, read, ret));
start += read;
}
is.close();
Assert.assertTrue(mTfs.getInfo(f).getInMemoryPercentage() == 100);
}
}
@Override
public boolean delete(String path, boolean recursive) throws IOException {
if (!recursive) {
if (isFolder(path) && listInternal(path, false).length != 0) {
LOG.error(
"Unable to delete "
+ path
+ " because it is a non empty directory. Specify "
+ "recursive as true in order to delete non empty directories.");
return false;
}
return deleteInternal(path);
}
// Get all relevant files
String[] pathsToDelete = listInternal(path, true);
for (String pathToDelete : pathsToDelete) {
// If we fail to deleteInternal one file, stop
if (!deleteInternal(PathUtils.concatPath(path, pathToDelete))) {
LOG.error("Failed to delete path " + pathToDelete + ", aborting delete.");
return false;
}
}
return deleteInternal(path);
}
/** Test {@link tachyon.client.block.BufferedBlockInStream#skip(long)}. */
@Test
public void skipTest() throws IOException, TachyonException {
String uniqPath = PathUtils.uniqPath();
for (int k = MIN_LEN + DELTA; k <= MAX_LEN; k += DELTA) {
for (CreateFileOptions op : getOptionSet()) {
TachyonURI path = new TachyonURI(uniqPath + "/file_" + k + "_" + op.hashCode());
FileSystemTestUtils.createByteFile(sFileSystem, path, op, k);
FileInStream is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
Assert.assertEquals(k / 2, is.skip(k / 2));
Assert.assertEquals(k / 2, is.read());
is.close();
is = sFileSystem.openFile(path, FileSystemTestUtils.toOpenFileOptions(op));
int t = k / 3;
Assert.assertEquals(t, is.skip(t));
Assert.assertEquals(t, is.read());
Assert.assertEquals(t, is.skip(t));
Assert.assertEquals(2 * t + 1, is.read());
is.close();
}
}
}
private LocalTachyonMaster(final String tachyonHome) throws IOException {
mTachyonHome = tachyonHome;
TachyonConf tachyonConf = MasterContext.getConf();
mHostname = NetworkAddressUtils.getConnectHost(ServiceType.MASTER_RPC, tachyonConf);
// To start the UFS either for integration or unit test. If it targets the unit test, UFS is
// setup over the local file system (see also {@link LocalFilesystemCluster} - under folder of
// "mTachyonHome/tachyon*". Otherwise, it starts some distributed file system cluster e.g.,
// miniDFSCluster (see also {@link tachyon.LocalMiniDFScluster} and setup the folder like
// "hdfs://xxx:xxx/tachyon*".
mUnderFSCluster = UnderFileSystemCluster.get(mTachyonHome + "/dfs", tachyonConf);
mUnderFSFolder = mUnderFSCluster.getUnderFilesystemAddress() + "/tachyon_underfs_folder";
// To setup the journalFolder under either local file system or distributed ufs like
// miniDFSCluster
mJournalFolder = mUnderFSCluster.getUnderFilesystemAddress() + "/journal";
UnderFileSystemUtils.mkdirIfNotExists(mJournalFolder, tachyonConf);
String[] masterServiceNames =
new String[] {
Constants.BLOCK_MASTER_SERVICE_NAME,
Constants.FILE_SYSTEM_MASTER_SERVICE_NAME,
Constants.RAW_TABLE_MASTER_SERVICE_NAME,
};
for (String masterServiceName : masterServiceNames) {
UnderFileSystemUtils.mkdirIfNotExists(
PathUtils.concatPath(mJournalFolder, masterServiceName), tachyonConf);
}
UnderFileSystemUtils.touch(
mJournalFolder + "/_format_" + System.currentTimeMillis(), tachyonConf);
tachyonConf.set(Constants.MASTER_JOURNAL_FOLDER, mJournalFolder);
tachyonConf.set(Constants.UNDERFS_ADDRESS, mUnderFSFolder);
tachyonConf.set(Constants.MASTER_MIN_WORKER_THREADS, "1");
tachyonConf.set(Constants.MASTER_MAX_WORKER_THREADS, "100");
// If tests fail to connect they should fail early rather than using the default ridiculously
// high retries
tachyonConf.set(Constants.MASTER_RETRY_COUNT, "3");
// Since tests are always running on a single host keep the resolution timeout low as otherwise
// people running with strange network configurations will see very slow tests
tachyonConf.set(Constants.HOST_RESOLUTION_TIMEOUT_MS, "250");
tachyonConf.set(Constants.WEB_THREAD_COUNT, "1");
tachyonConf.set(
Constants.WEB_RESOURCES,
PathUtils.concatPath(System.getProperty("user.dir"), "../servers/src/main/webapp"));
mTachyonMaster = TachyonMaster.Factory.createMaster();
// Reset the master port
tachyonConf.set(Constants.MASTER_PORT, Integer.toString(getRPCLocalPort()));
Runnable runMaster =
new Runnable() {
@Override
public void run() {
try {
mTachyonMaster.start();
} catch (Exception e) {
throw new RuntimeException(e + " \n Start Master Error \n" + e.getMessage(), e);
}
}
};
mMasterThread = new Thread(runMaster);
}
@Before
public void before() throws Exception {
mStoreUri = new TachyonURI(PathUtils.uniqPath());
}
/**
* @param baseDirectory the base journal directory
* @return the journal directory for this master
*/
public static String getJournalDirectory(String baseDirectory) {
return PathUtils.concatPath(baseDirectory, Constants.BLOCK_MASTER_NAME);
}
@Override
public void close() throws IOException {
if (mClosed) {
return;
}
if (mCurrentBlockOutStream != null) {
mPreviousBlockOutStreams.add(mCurrentBlockOutStream);
}
Boolean canComplete = false;
CompleteFileOptions options = CompleteFileOptions.defaults();
if (mUnderStorageType.isSyncPersist()) {
String tmpPath = PathUtils.temporaryFileName(mNonce, mUfsPath);
UnderFileSystem ufs = UnderFileSystem.get(tmpPath, ClientContext.getConf());
if (mCanceled) {
// TODO(yupeng): Handle this special case in under storage integrations.
mUnderStorageOutputStream.close();
if (!ufs.exists(tmpPath)) {
// Location of the temporary file has changed, recompute it.
updateUfsPath();
tmpPath = PathUtils.temporaryFileName(mNonce, mUfsPath);
}
ufs.delete(tmpPath, false);
} else {
mUnderStorageOutputStream.flush();
mUnderStorageOutputStream.close();
if (!ufs.exists(tmpPath)) {
// Location of the temporary file has changed, recompute it.
updateUfsPath();
tmpPath = PathUtils.temporaryFileName(mNonce, mUfsPath);
}
if (!ufs.rename(tmpPath, mUfsPath)) {
throw new IOException("Failed to rename " + tmpPath + " to " + mUfsPath);
}
options.setUfsLength(ufs.getFileSize(mUfsPath));
canComplete = true;
}
}
if (mTachyonStorageType.isStore()) {
try {
if (mCanceled) {
for (BufferedBlockOutStream bos : mPreviousBlockOutStreams) {
bos.cancel();
}
} else {
for (BufferedBlockOutStream bos : mPreviousBlockOutStreams) {
bos.close();
}
canComplete = true;
}
} catch (IOException ioe) {
handleCacheWriteException(ioe);
}
}
if (canComplete) {
FileSystemMasterClient masterClient = mContext.acquireMasterClient();
try {
masterClient.completeFile(mUri, options);
} catch (TachyonException e) {
throw new IOException(e);
} finally {
mContext.releaseMasterClient(masterClient);
}
}
if (mUnderStorageType.isAsyncPersist()) {
scheduleAsyncPersist();
}
mClosed = true;
}
