/*
* @Test public void locationTest() throws IOException { TachyonFile file =
* TachyonFSTestUtils.createByteFile(mTfs, "/testFile", CacheType.STORE,
* UnderStorageType.NO_PERSIST, 10); mFsShell.run(new String[] {"location", "/testFile"});
*
* FileInfo fileInfo = mTfs.getInfo(file); Assert.assertNotNull(fileInfo); List<String>
* locationsList = tFile.getLocationHosts(); String[] commandParameters = new String[3 +
* locationsList.size()]; commandParameters[0] = "location"; commandParameters[1] = "/testFile";
* commandParameters[2] = String.valueOf(file.getFileId()); Iterator<String> iter =
* locationsList.iterator(); int i = 3; while (iter.hasNext()) { commandParameters[i ++] =
* iter.next(); } Assert.assertEquals(getCommandOutput(commandParameters), mOutput.toString()); }
*/
@Test
public void lsrTest() throws IOException, TException {
FileInfo[] files = new FileInfo[4];
TachyonFile fileA =
TachyonFSTestUtils.createByteFile(
mTfs, "/testRoot/testFileA", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, 10);
files[0] = mTfs.getInfo(fileA);
TachyonFSTestUtils.createByteFile(
mTfs,
"/testRoot/testDir/testFileB",
TachyonStorageType.STORE,
UnderStorageType.NO_PERSIST,
20);
files[1] = mTfs.getInfo(mTfs.open(new TachyonURI("/testRoot/testDir")));
files[2] = mTfs.getInfo(mTfs.open(new TachyonURI("/testRoot/testDir/testFileB")));
TachyonFile fileC =
TachyonFSTestUtils.createByteFile(
mTfs, "/testRoot/testFileC", TachyonStorageType.NO_STORE, UnderStorageType.PERSIST, 30);
files[3] = mTfs.getInfo(fileC);
mFsShell.run(new String[] {"lsr", "/testRoot"});
String expected = "";
String format = "%-10s%-25s%-15s%-5s\n";
expected +=
String.format(
format,
FormatUtils.getSizeFromBytes(10),
TFsShell.convertMsToDate(files[0].getCreationTimeMs()),
"In Memory",
"/testRoot/testFileA");
expected +=
String.format(
format,
FormatUtils.getSizeFromBytes(0),
TFsShell.convertMsToDate(files[1].getCreationTimeMs()),
"",
"/testRoot/testDir");
expected +=
String.format(
format,
FormatUtils.getSizeFromBytes(20),
TFsShell.convertMsToDate(files[2].getCreationTimeMs()),
"In Memory",
"/testRoot/testDir/testFileB");
expected +=
String.format(
format,
FormatUtils.getSizeFromBytes(30),
TFsShell.convertMsToDate(files[3].getCreationTimeMs()),
"Not In Memory",
"/testRoot/testFileC");
Assert.assertEquals(expected, mOutput.toString());
}
private void initStorageTier() throws AlreadyExistsException, IOException, OutOfSpaceException {
String workerDataFolder =
WorkerContext.getConf().get(Constants.WORKER_DATA_FOLDER, Constants.DEFAULT_DATA_FOLDER);
String tierDirPathConf =
String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_PATH_FORMAT, mTierLevel);
String[] dirPaths = WorkerContext.getConf().get(tierDirPathConf, "/mnt/ramdisk").split(",");
// Add the worker data folder path after each storage directory, the final path will be like
// /mnt/ramdisk/tachyonworker
for (int i = 0; i < dirPaths.length; i++) {
dirPaths[i] = PathUtils.concatPath(dirPaths[i].trim(), workerDataFolder);
}
String tierDirCapacityConf =
String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_QUOTA_FORMAT, mTierLevel);
String[] dirQuotas = WorkerContext.getConf().get(tierDirCapacityConf, "0").split(",");
mDirs = new ArrayList<StorageDir>(dirPaths.length);
long totalCapacity = 0;
for (int i = 0; i < dirPaths.length; i++) {
int index = i >= dirQuotas.length ? dirQuotas.length - 1 : i;
long capacity = FormatUtils.parseSpaceSize(dirQuotas[index]);
totalCapacity += capacity;
mDirs.add(StorageDir.newStorageDir(this, i, capacity, dirPaths[i]));
}
mCapacityBytes = totalCapacity;
}
/**
* Marks a block as committed on a specific worker.
*
* @param workerId the worker id committing the block
* @param usedBytesOnTier the updated used bytes on the tier of the worker
* @param tierAlias the alias of the storage tier where the worker is committing the block to
* @param blockId the committing block id
* @param length the length of the block
*/
public void commitBlock(
long workerId, long usedBytesOnTier, String tierAlias, long blockId, long length) {
LOG.debug(
"Commit block from worker: {}",
FormatUtils.parametersToString(workerId, usedBytesOnTier, blockId, length));
synchronized (mBlocks) {
synchronized (mWorkers) {
MasterWorkerInfo workerInfo = mWorkers.getFirstByField(mIdIndex, workerId);
workerInfo.addBlock(blockId);
workerInfo.updateUsedBytes(tierAlias, usedBytesOnTier);
workerInfo.updateLastUpdatedTimeMs();
MasterBlockInfo masterBlockInfo = mBlocks.get(blockId);
if (masterBlockInfo == null) {
masterBlockInfo = new MasterBlockInfo(blockId, length);
mBlocks.put(blockId, masterBlockInfo);
BlockInfoEntry blockInfo =
BlockInfoEntry.newBuilder()
.setBlockId(masterBlockInfo.getBlockId())
.setLength(masterBlockInfo.getLength())
.build();
writeJournalEntry(JournalEntry.newBuilder().setBlockInfo(blockInfo).build());
flushJournal();
}
masterBlockInfo.addWorker(workerId, tierAlias);
mLostBlocks.remove(blockId);
}
}
}
public long getBytes(String key) {
if (mProperties.containsKey(key)) {
String rawValue = get(key);
try {
return FormatUtils.parseSpaceSize(rawValue);
} catch (Exception ex) {
throw new RuntimeException("Configuration cannot evaluate key " + key + " as bytes.");
}
}
throw new RuntimeException("Invalid configuration key " + key + ".");
}
@Test
public void lsrTest() throws IOException {
int fileIdA =
TachyonFSTestUtils.createByteFile(mTfs, "/testRoot/testFileA", WriteType.MUST_CACHE, 10);
TachyonFile[] files = new TachyonFile[4];
files[0] = mTfs.getFile(fileIdA);
TachyonFSTestUtils.createByteFile(
mTfs, "/testRoot/testDir/testFileB", WriteType.MUST_CACHE, 20);
files[1] = mTfs.getFile(new TachyonURI("/testRoot/testDir"));
files[2] = mTfs.getFile(new TachyonURI("/testRoot/testDir/testFileB"));
int fileIdC =
TachyonFSTestUtils.createByteFile(mTfs, "/testRoot/testFileC", WriteType.THROUGH, 30);
files[3] = mTfs.getFile(fileIdC);
mFsShell.ls(new String[] {"count", "/testRoot"});
String expected = "";
String format = "%-10s%-25s%-15s%-5s\n";
expected +=
String.format(
format,
FormatUtils.getSizeFromBytes(10),
TFsShell.convertMsToDate(files[0].getCreationTimeMs()),
"In Memory",
"/testRoot/testFileA");
expected +=
String.format(
format,
FormatUtils.getSizeFromBytes(0),
TFsShell.convertMsToDate(files[1].getCreationTimeMs()),
"",
"/testRoot/testDir");
expected +=
String.format(
format,
FormatUtils.getSizeFromBytes(30),
TFsShell.convertMsToDate(files[3].getCreationTimeMs()),
"Not In Memory",
"/testRoot/testFileC");
Assert.assertEquals(expected, mOutput.toString());
}
/**
* Marks a block as committed, but without a worker location. This means the block is only in ufs.
*
* @param blockId the id of the block to commit
* @param length the length of the block
*/
public void commitBlockInUFS(long blockId, long length) {
LOG.debug("Commit block to ufs: {}", FormatUtils.parametersToString(blockId, length));
synchronized (mBlocks) {
MasterBlockInfo masterBlockInfo = mBlocks.get(blockId);
if (masterBlockInfo == null) {
// The block has not been committed previously, so add the metadata to commit the block.
masterBlockInfo = new MasterBlockInfo(blockId, length);
mBlocks.put(blockId, masterBlockInfo);
BlockInfoEntry blockInfo =
BlockInfoEntry.newBuilder()
.setBlockId(masterBlockInfo.getBlockId())
.setLength(masterBlockInfo.getLength())
.build();
writeJournalEntry(JournalEntry.newBuilder().setBlockInfo(blockInfo).build());
flushJournal();
}
}
}
/**
* Displays information for all directories and files directly under the path specified in argv.
*
* @param path The TachyonURI path as the input of the command
* @return 0 if command is successful, -1 if an error occurred.
* @throws IOException
*/
public int ls(TachyonURI path) throws IOException {
List<FileInfo> files = listStatusSortedByIncreasingCreationTime(path);
String format = "%-10s%-25s%-15s%-5s%n";
for (FileInfo file : files) {
String inMemory = "";
if (!file.isFolder) {
if (100 == file.inMemoryPercentage) {
inMemory = "In Memory";
} else {
inMemory = "Not In Memory";
}
}
System.out.format(
format,
FormatUtils.getSizeFromBytes(file.getLength()),
convertMsToDate(file.getCreationTimeMs()),
inMemory,
file.getPath());
}
return 0;
}