private void initializeWorkerStorage()
throws IOException, FileDoesNotExistException, SuspectedFileSizeException, BlockInfoException,
TException {
LOG.info("Initializing the worker storage.");
if (!mLocalDataFolder.exists()) {
LOG.info("Local folder " + mLocalDataFolder + " does not exist. Creating a new one.");
mLocalDataFolder.mkdirs();
mLocalUserFolder.mkdirs();
CommonUtils.changeLocalFilePermission(mLocalDataFolder.getPath(), "775");
CommonUtils.changeLocalFilePermission(mLocalUserFolder.getPath(), "775");
return;
}
if (!mLocalDataFolder.isDirectory()) {
String tmp = "Data folder " + mLocalDataFolder + " is not a folder!";
LOG.error(tmp);
throw new IllegalArgumentException(tmp);
}
if (mLocalUserFolder.exists()) {
try {
FileUtils.deleteDirectory(mLocalUserFolder);
} catch (IOException e) {
LOG.error(e.getMessage(), e);
}
}
mLocalUserFolder.mkdir();
CommonUtils.changeLocalFilePermission(mLocalUserFolder.getPath(), "775");
mUnderfsOrphansFolder = mUnderfsWorkerFolder + "/orphans";
if (!mUnderFs.exists(mUnderfsOrphansFolder)) {
mUnderFs.mkdirs(mUnderfsOrphansFolder, true);
}
int cnt = 0;
for (File tFile : mLocalDataFolder.listFiles()) {
if (tFile.isFile()) {
cnt++;
LOG.info("File " + cnt + ": " + tFile.getPath() + " with size " + tFile.length() + " Bs.");
long blockId = CommonUtils.getBlockIdFromFileName(tFile.getName());
boolean success = mWorkerSpaceCounter.requestSpaceBytes(tFile.length());
try {
addFoundBlock(blockId, tFile.length());
} catch (FileDoesNotExistException e) {
LOG.error("BlockId: " + blockId + " becomes orphan for: \"" + e.message + "\"");
LOG.info(
"Swapout File " + cnt + ": blockId: " + blockId + " to " + mUnderfsOrphansFolder);
swapoutOrphanBlocks(blockId, tFile);
freeBlock(blockId);
continue;
}
mAddedBlockList.add(blockId);
if (!success) {
throw new RuntimeException("Pre-existing files exceed the local memory capacity.");
}
}
}
}
@Test
public void reserveTest() throws Exception {
// Reserve on top tier
long blockId = 100;
BlockStoreLocation tier0 = BlockStoreLocation.anyDirInTier(StorageLevelAlias.MEM.getValue());
for (int i = 0; i < 3; i++) {
TieredBlockStoreTestUtils.cache(SESSION_ID, blockId++, BLOCK_SIZE, mBlockStore, tier0);
}
CommonUtils.sleepMs(
WorkerContext.getConf().getLong(Constants.WORKER_SPACE_RESERVER_INTERVAL_MS));
BlockStoreMeta storeMeta = mBlockStore.getBlockStoreMeta();
Assert.assertEquals(3 * BLOCK_SIZE, storeMeta.getUsedBytes());
List<Long> usedBytesOnTiers = storeMeta.getUsedBytesOnTiers();
Assert.assertEquals(
2 * BLOCK_SIZE, (long) usedBytesOnTiers.get(StorageLevelAlias.MEM.getValue() - 1));
Assert.assertEquals(
BLOCK_SIZE, (long) usedBytesOnTiers.get(StorageLevelAlias.HDD.getValue() - 1));
// Reserve on under tier
for (int i = 0; i < 7; i++) {
TieredBlockStoreTestUtils.cache(SESSION_ID, blockId++, BLOCK_SIZE, mBlockStore, tier0);
}
CommonUtils.sleepMs(
WorkerContext.getConf().getLong(Constants.WORKER_SPACE_RESERVER_INTERVAL_MS));
storeMeta = mBlockStore.getBlockStoreMeta();
Assert.assertEquals(9 * BLOCK_SIZE, storeMeta.getUsedBytes());
usedBytesOnTiers = storeMeta.getUsedBytesOnTiers();
Assert.assertEquals(
2 * BLOCK_SIZE, (long) usedBytesOnTiers.get(StorageLevelAlias.MEM.getValue() - 1));
Assert.assertEquals(
7 * BLOCK_SIZE, (long) usedBytesOnTiers.get(StorageLevelAlias.HDD.getValue() - 1));
}
@Test
public void freeTest() throws IOException {
TachyonFSTestUtils.createByteFile(mTfs, "/testFile", WriteType.MUST_CACHE, 10);
mFsShell.free(new String[] {"free", "/testFile"});
TachyonConf tachyonConf = mLocalTachyonCluster.getMasterTachyonConf();
CommonUtils.sleepMs(null, CommonUtils.getToMasterHeartBeatIntervalMs(tachyonConf) * 2 + 10);
Assert.assertFalse(mTfs.getFile(new TachyonURI("/testFile")).isInMemory());
}
@Override
public void run() {
long lastHeartbeatMs = System.currentTimeMillis();
Command cmd = null;
while (!mStop) {
long diff = System.currentTimeMillis() - lastHeartbeatMs;
if (diff < WorkerConf.get().TO_MASTER_HEARTBEAT_INTERVAL_MS) {
LOG.debug("Heartbeat process takes {} ms.", diff);
CommonUtils.sleepMs(LOG, WorkerConf.get().TO_MASTER_HEARTBEAT_INTERVAL_MS - diff);
} else {
LOG.error("Heartbeat process takes " + diff + " ms.");
}
try {
cmd = mWorkerStorage.heartbeat();
lastHeartbeatMs = System.currentTimeMillis();
} catch (IOException e) {
LOG.error(e.getMessage(), e);
mWorkerStorage.resetMasterClient();
CommonUtils.sleepMs(LOG, Constants.SECOND_MS);
cmd = null;
if (System.currentTimeMillis() - lastHeartbeatMs >= WorkerConf.get().HEARTBEAT_TIMEOUT_MS) {
throw new RuntimeException(
"Timebeat timeout " + (System.currentTimeMillis() - lastHeartbeatMs) + "ms");
}
}
if (cmd != null) {
switch (cmd.mCommandType) {
case Unknown:
LOG.error("Unknown command: " + cmd);
break;
case Nothing:
LOG.debug("Nothing command: {}", cmd);
break;
case Register:
LOG.info("Register command: " + cmd);
mWorkerStorage.register();
break;
case Free:
mWorkerStorage.freeBlocks(cmd.mData);
LOG.info("Free command: " + cmd);
break;
case Delete:
LOG.info("Delete command: " + cmd);
break;
default:
throw new RuntimeException("Un-recognized command from master " + cmd.toString());
}
}
mWorkerStorage.checkStatus();
}
}
public static void main(String[] args) throws Exception {
LocalTachyonCluster cluster = new LocalTachyonCluster(100, 8 * Constants.MB, Constants.GB);
cluster.start();
CommonUtils.sleepMs(Constants.SECOND_MS);
cluster.stop();
CommonUtils.sleepMs(Constants.SECOND_MS);
cluster = new LocalTachyonCluster(100, 8 * Constants.MB, Constants.GB);
cluster.start();
CommonUtils.sleepMs(Constants.SECOND_MS);
cluster.stop();
CommonUtils.sleepMs(Constants.SECOND_MS);
}
@Test
public void singleMasterJournalCrashIntegrationTest() throws Exception {
LocalTachyonCluster cluster = setupSingleMasterCluster();
CommonUtils.sleepMs(TEST_TIME_MS);
// Shutdown the cluster
cluster.stopTFS();
CommonUtils.sleepMs(TEST_TIME_MS);
// Ensure the client threads are stopped.
mExecutorsForClient.shutdown();
mExecutorsForClient.awaitTermination(TEST_TIME_MS, TimeUnit.MILLISECONDS);
reproduceAndCheckState(mCreateFileThread.getSuccessNum());
// clean up
cluster.stopUFS();
}
@Test
public void createPathTest() throws Exception {
// save the last mod time of the root
long lastModTime = mTree.getRoot().getLastModificationTimeMs();
// sleep to ensure a different last modification time
CommonUtils.sleepMs(10);
// create nested directory
InodeTree.CreatePathResult createResult = mTree.createPath(NESTED_URI, sNestedDirectoryOptions);
List<Inode> modified = createResult.getModified();
List<Inode> created = createResult.getCreated();
// 1 modified directory
Assert.assertEquals(1, modified.size());
Assert.assertEquals("", modified.get(0).getName());
Assert.assertNotEquals(lastModTime, modified.get(0).getLastModificationTimeMs());
// 2 created directories
Assert.assertEquals(2, created.size());
Assert.assertEquals("nested", created.get(0).getName());
Assert.assertEquals("test", created.get(1).getName());
// save the last mod time of 'test'
lastModTime = created.get(1).getLastModificationTimeMs();
// sleep to ensure a different last modification time
CommonUtils.sleepMs(10);
// creating the directory path again results in no new inodes.
try {
createResult = mTree.createPath(NESTED_URI, sNestedDirectoryOptions);
Assert.assertTrue("createPath should throw FileAlreadyExistsException", false);
} catch (FileAlreadyExistsException faee) {
Assert.assertEquals(
faee.getMessage(), ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(NESTED_URI));
}
// create a file
CreatePathOptions options =
new CreatePathOptions.Builder(MasterContext.getConf())
.setBlockSizeBytes(Constants.KB)
.setRecursive(true)
.build();
createResult = mTree.createPath(NESTED_FILE_URI, options);
modified = createResult.getModified();
created = createResult.getCreated();
// test directory was modified
Assert.assertEquals(1, modified.size());
Assert.assertEquals("test", modified.get(0).getName());
Assert.assertNotEquals(lastModTime, modified.get(0).getLastModificationTimeMs());
// file was created
Assert.assertEquals(1, created.size());
Assert.assertEquals("file", created.get(0).getName());
}
// Tests that deletes go through despite failing initially due to concurrent read
@Test
public void deleteWhileReadTest() throws Exception {
TachyonFile file =
TachyonFSTestUtils.createByteFile(
mTFS,
"/test1",
TachyonStorageType.STORE,
UnderStorageType.NO_PERSIST,
MEM_CAPACITY_BYTES);
CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);
Assert.assertTrue(mTFS.getInfo(file).getInMemoryPercentage() == 100);
// Open the file
InStreamOptions options =
new InStreamOptions.Builder(new TachyonConf())
.setTachyonStorageType(TachyonStorageType.STORE)
.build();
FileInStream in = mTFS.getInStream(file, options);
Assert.assertEquals(0, in.read());
// Delete the file
mTFS.delete(file);
CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);
// After the delete, the master should no longer serve the file
Assert.assertNull(mTFS.open(new TachyonURI("/test1")));
// However, the previous read should still be able to read it as the data still exists
byte[] res = new byte[MEM_CAPACITY_BYTES];
Assert.assertEquals(MEM_CAPACITY_BYTES - 1, in.read(res, 1, MEM_CAPACITY_BYTES - 1));
res[0] = 0;
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(MEM_CAPACITY_BYTES, res));
in.close();
CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);
// After the file is closed, the master's delete should go through and new files can be created
TachyonFile newFile =
TachyonFSTestUtils.createByteFile(
mTFS,
"/test2",
TachyonStorageType.STORE,
UnderStorageType.NO_PERSIST,
MEM_CAPACITY_BYTES);
CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);
Assert.assertTrue(mTFS.getInfo(newFile).getInMemoryPercentage() == 100);
}
public String getBlockSizeBytes() {
if (mIsDirectory) {
return " ";
} else {
return CommonUtils.getSizeFromBytes(mBlockSizeBytes);
}
}
public String getSize() {
if (mIsDirectory) {
return " ";
} else {
return CommonUtils.getSizeFromBytes(mSize);
}
}
/**
* Keep creating files until something crashes or fail to create. Record how many files are
* created successfully.
*/
@Override
public void run() {
try {
// This infinity loop will be broken if something crashes or fail to create. This is
// expected since the master will shutdown at a certain time.
while (true) {
if (mOpType == 0) {
try {
mFileSystem.createFile(new TachyonURI(TEST_FILE_DIR + mSuccessNum)).close();
} catch (IOException e) {
break;
}
} else if (mOpType == 1) {
// TODO(gene): Add this back when there is new RawTable client API.
// if (mFileSystem.createRawTable(new TachyonURI(TEST_TABLE_DIR + mSuccessNum), 1) ==
// -1) {
// break;
// }
}
// The create operation may succeed at the master side but still returns false due to the
// shutdown. So the mSuccessNum may be less than the actual success number.
mSuccessNum++;
CommonUtils.sleepMs(100);
}
} catch (Exception e) {
// Something crashed. Stop the thread.
}
}
// TODO: Rethink the approach of this test and what it should be testing
// @Test
public void deleteDuringEvictionTest() throws IOException {
// This test may not trigger eviction each time, repeat it 20 times.
for (int i = 0; i < 20; i++) {
deleteDuringEviction(i);
CommonUtils.sleepMs(2 * HEARTBEAT_INTERVAL_MS); // ensure second delete completes
}
}
/**
* Stops the block worker. This method should only be called to terminate the worker.
*
* @throws IOException if the data server fails to close
*/
public void stop() throws IOException {
mDataServer.close();
mThriftServer.stop();
mThriftServerSocket.close();
mSessionCleanerThread.stop();
mBlockMasterClient.close();
if (mSpaceReserver != null) {
mSpaceReserver.stop();
}
mFileSystemMasterClient.close();
// Use shutdownNow because HeartbeatThreads never finish until they are interrupted
getExecutorService().shutdownNow();
mWorkerMetricsSystem.stop();
try {
mWebServer.shutdownWebServer();
} catch (Exception e) {
LOG.error("Failed to stop web server", e);
}
mBlockDataManager.stop();
while (!mDataServer.isClosed() || mThriftServer.isServing()) {
// The reason to stop and close again is due to some issues in Thrift.
mDataServer.close();
mThriftServer.stop();
mThriftServerSocket.close();
CommonUtils.sleepMs(100);
}
}
/**
* Creates the local block path and all the parent directories. Also, sets the appropriate
* permissions.
*
* @param path The path of the block.
* @throws IOException
*/
public static void createBlockPath(String path) throws IOException {
File localFolder;
try {
localFolder = new File(CommonUtils.getParent(path));
} catch (InvalidPathException e) {
throw new IOException(e);
}
if (!localFolder.exists()) {
if (localFolder.mkdirs()) {
CommonUtils.changeLocalFileToFullPermission(localFolder.getAbsolutePath());
LOG.info("Folder {} was created!", localFolder);
} else {
throw new IOException("Failed to create folder " + localFolder);
}
}
}
/**
* Notify the worker the block is cached.
*
* <p>This call is called remotely from {@link tachyon.client.TachyonFS#cacheBlock(long)} which is
* only ever called from {@link tachyon.client.BlockOutStream#close()} (though its a public api so
* anyone could call it). There are a few interesting preconditions for this to work.
*
* <p>1) Client process writes to files locally under a tachyon defined temp directory. 2) Worker
* process is on the same node as the client 3) Client is talking to the local worker directly
*
* <p>If all conditions are true, then and only then can this method ever be called; all
* operations work on local files.
*
* @param userId The user id of the client who send the notification
* @param blockId The id of the block
* @throws FileDoesNotExistException
* @throws SuspectedFileSizeException
* @throws BlockInfoException
* @throws TException
*/
public void cacheBlock(long userId, long blockId)
throws FileDoesNotExistException, SuspectedFileSizeException, BlockInfoException, TException {
File srcFile = new File(CommonUtils.concat(getUserTempFolder(userId), blockId));
File dstFile = new File(CommonUtils.concat(mLocalDataFolder, blockId));
long fileSizeBytes = srcFile.length();
if (!srcFile.exists()) {
throw new FileDoesNotExistException("File " + srcFile + " does not exist.");
}
if (!srcFile.renameTo(dstFile)) {
throw new FileDoesNotExistException(
"Failed to rename file from " + srcFile.getPath() + " to " + dstFile.getPath());
}
addBlockId(blockId, fileSizeBytes);
mUsers.addOwnBytes(userId, -fileSizeBytes);
mMasterClient.worker_cacheBlock(
mWorkerId, mWorkerSpaceCounter.getUsedBytes(), blockId, fileSizeBytes);
LOG.info(userId + " " + dstFile);
}
@Ignore
@Test
public void multiMasterJournalCrashIntegrationTest() throws Exception {
LocalTachyonClusterMultiMaster cluster = setupMultiMasterCluster();
// Kill the leader one by one.
for (int kills = 0; kills < TEST_NUM_MASTERS; kills++) {
CommonUtils.sleepMs(TEST_TIME_MS);
Assert.assertTrue(cluster.killLeader());
}
cluster.stopTFS();
CommonUtils.sleepMs(TEST_TIME_MS);
// Ensure the client threads are stopped.
mExecutorsForClient.shutdown();
while (!mExecutorsForClient.awaitTermination(TEST_TIME_MS, TimeUnit.MILLISECONDS)) {}
reproduceAndCheckState(mCreateFileThread.getSuccessNum());
// clean up
cluster.stopUFS();
}
@Test
public void freeTest() throws IOException, TException {
TachyonFile file =
TachyonFSTestUtils.createByteFile(
mTfs, "/testFile", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, 10);
mFsShell.run(new String[] {"free", "/testFile"});
TachyonConf tachyonConf = mLocalTachyonCluster.getMasterTachyonConf();
CommonUtils.sleepMs(tachyonConf.getInt(Constants.WORKER_TO_MASTER_HEARTBEAT_INTERVAL_MS));
Assert.assertFalse(mTfs.getInfo(file).getInMemoryPercentage() == 100);
}
/**
* Stop the current Tachyon cluster. This is used for preparation and clean up. To crash the
* Master, use <code>killMaster</code>.
*/
private static void stopCluster() {
String stopClusterCommand =
new TachyonConf().get(Constants.TACHYON_HOME) + "/bin/tachyon-stop.sh";
try {
Runtime.getRuntime().exec(stopClusterCommand).waitFor();
CommonUtils.sleepMs(LOG, 1000);
} catch (Exception e) {
LOG.error("Error when stop Tachyon cluster", e);
}
}
/**
* Factory for {@link Evictor}.
*
* @param conf {@link TachyonConf} to determine the {@link Evictor} type
* @param view {@link BlockMetadataManagerView} to pass to {@link Evictor}
* @param allocator an allocation policy
* @return the generated {@link Evictor}
*/
public static Evictor create(
TachyonConf conf, BlockMetadataManagerView view, Allocator allocator) {
try {
return CommonUtils.createNewClassInstance(
conf.<Evictor>getClass(Constants.WORKER_EVICTOR_CLASS),
new Class[] {BlockMetadataManagerView.class, Allocator.class},
new Object[] {view, allocator});
} catch (Exception e) {
throw Throwables.propagate(e);
}
}
/**
* Create a data server with direct access to worker storage.
*
* @param address The address of the data server.
* @param workerStorage The handler of the directly accessed worker storage.
*/
public DataServer(InetSocketAddress address, WorkerStorage workerStorage) {
LOG.info("Starting DataServer @ " + address);
mAddress = address;
mBlocksLocker = new BlocksLocker(workerStorage, Users.sDATASERVER_USER_ID);
try {
mSelector = initSelector();
} catch (IOException e) {
LOG.error(e.getMessage() + mAddress, e);
CommonUtils.runtimeException(e);
}
}
/**
* Add the checkpoint information of a file. The information is from the user <code>userId</code>.
*
* <p>This method is normally triggered from {@link tachyon.client.FileOutStream#close()} if and
* only if {@link tachyon.client.WriteType#isThrough()} is true. The current implementation of
* checkpointing is that through {@link tachyon.client.WriteType} operations write to {@link
* tachyon.UnderFileSystem} on the client's write path, but under a user temp directory (temp
* directory is defined in the worker as {@link #getUserUnderfsTempFolder(long)}).
*
* @param userId The user id of the client who send the notification
* @param fileId The id of the checkpointed file
* @throws FileDoesNotExistException
* @throws SuspectedFileSizeException
* @throws FailedToCheckpointException
* @throws BlockInfoException
* @throws TException
*/
public void addCheckpoint(long userId, int fileId)
throws FileDoesNotExistException, SuspectedFileSizeException, FailedToCheckpointException,
BlockInfoException, TException {
// TODO This part need to be changed.
String srcPath = CommonUtils.concat(getUserUnderfsTempFolder(userId), fileId);
String dstPath = CommonUtils.concat(COMMON_CONF.UNDERFS_DATA_FOLDER, fileId);
try {
if (!mUnderFs.rename(srcPath, dstPath)) {
throw new FailedToCheckpointException("Failed to rename " + srcPath + " to " + dstPath);
}
} catch (IOException e) {
throw new FailedToCheckpointException("Failed to rename " + srcPath + " to " + dstPath);
}
long fileSize;
try {
fileSize = mUnderFs.getFileSize(dstPath);
} catch (IOException e) {
throw new FailedToCheckpointException("Failed to getFileSize " + dstPath);
}
mMasterClient.addCheckpoint(mWorkerId, fileId, fileSize, dstPath);
}
/** @return Generated {@link WorkerInfo} for this worker */
public synchronized WorkerInfo generateClientWorkerInfo() {
WorkerInfo ret = new WorkerInfo();
ret.id = mId;
ret.address = mWorkerAddress;
ret.lastContactSec =
(int) ((CommonUtils.getCurrentMs() - mLastUpdatedTimeMs) / Constants.SECOND_MS);
ret.state = "In Service";
ret.capacityBytes = mCapacityBytes;
ret.usedBytes = mUsedBytes;
ret.startTimeMs = mStartTimeMs;
return ret;
}
// Tests that pinning a file and then unpinning
@Test
public void unpinFileTest() throws Exception {
// Create a file that fills the entire Tachyon store
TachyonFile file1 =
TachyonFSTestUtils.createByteFile(
mTFS,
"/test1",
TachyonStorageType.STORE,
UnderStorageType.NO_PERSIST,
MEM_CAPACITY_BYTES);
// Pin the file
mTFS.setState(file1, mSetPinned);
CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);
// Confirm the pin with master
Assert.assertTrue(mTFS.getInfo(file1).isIsPinned());
// Unpin the file
mTFS.setState(file1, mSetUnpinned);
CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);
// Confirm the unpin
Assert.assertFalse(mTFS.getInfo(file1).isIsPinned());
// Try to create a file that cannot be stored unless the previous file is evicted, this
// should succeed
TachyonFile file2 =
TachyonFSTestUtils.createByteFile(
mTFS,
"/test2",
TachyonStorageType.STORE,
UnderStorageType.NO_PERSIST,
MEM_CAPACITY_BYTES);
// File 2 should be in memory and File 1 should be evicted
CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);
Assert.assertFalse(mTFS.getInfo(file1).getInMemoryPercentage() == 100);
Assert.assertTrue(mTFS.getInfo(file2).getInMemoryPercentage() == 100);
}
BlockOutStream(TachyonFile file, WriteType opType, int blockIndex) throws IOException {
super(file, opType);
if (!opType.isCache()) {
throw new IOException("BlockOutStream only support WriteType.CACHE");
}
BLOCK_INDEX = blockIndex;
BLOCK_CAPACITY_BYTE = FILE.getBlockSizeByte();
BLOCK_ID = FILE.getBlockId(BLOCK_INDEX);
BLOCK_OFFSET = BLOCK_CAPACITY_BYTE * blockIndex;
PIN = FILE.needPin();
mCanWrite = true;
if (!TFS.hasLocalWorker()) {
mCanWrite = false;
String msg = "The machine does not have any local worker.";
throw new IOException(msg);
}
File localFolder = TFS.createAndGetUserTempFolder();
if (localFolder == null) {
mCanWrite = false;
String msg = "Failed to create temp user folder for tachyon client.";
throw new IOException(msg);
}
mLocalFilePath = CommonUtils.concat(localFolder.getPath(), BLOCK_ID);
mLocalFile = new RandomAccessFile(mLocalFilePath, "rw");
mLocalFileChannel = mLocalFile.getChannel();
// change the permission of the temporary file in order that the worker can move it.
CommonUtils.changeLocalFileToFullPermission(mLocalFilePath);
// use the sticky bit, only the client and the worker can write to the block
CommonUtils.setLocalFileStickyBit(mLocalFilePath);
LOG.info(mLocalFilePath + " was created!");
mBuffer = ByteBuffer.allocate(USER_CONF.FILE_BUFFER_BYTES + 4);
}
/** Register this TachyonWorker to the TachyonMaster */
public void register() {
long id = 0;
while (id == 0) {
try {
id =
mMasterClient.worker_register(
mWorkerAddress,
mSpaceCounter.getCapacityBytes(),
mSpaceCounter.getUsedBytes(),
new ArrayList<Long>(mMemoryData));
} catch (BlockInfoException e) {
LOG.error(e.getMessage(), e);
id = 0;
CommonUtils.sleepMs(LOG, Constants.SECOND_MS);
} catch (IOException e) {
LOG.error(e.getMessage(), e);
id = 0;
CommonUtils.sleepMs(LOG, Constants.SECOND_MS);
}
}
mWorkerId = id;
}
public static void createDependency() throws IOException {
long startTimeMs = CommonUtils.getCurrentMs();
List<String> children = new ArrayList<String>();
for (int k = 0; k < sFiles; k++) {
children.add(sFileFolder + "/part-" + k);
}
List<ByteBuffer> data = new ArrayList<ByteBuffer>();
data.add(ByteBuffer.allocate(10));
int depId =
sTachyonClient.createDependency(
new ArrayList<String>(),
children,
"fake command",
data,
"BasicCheckpoint Dependency",
"Tachyon Examples",
"0.3",
DependencyType.Narrow.getValue(),
512 * Constants.MB);
CommonUtils.printTimeTakenMs(startTimeMs, LOG, "createDependency with depId " + depId);
}
/**
* Keep requesting to Master until something crashes or fail to create. Record how many
* operations are performed successfully.
*/
@Override
public void run() {
// This infinity loop will be broken when the master is crashed and the client needs to stop.
while (true) {
synchronized (this) {
if (mIsStopped) {
break;
}
}
try {
TachyonURI testURI = new TachyonURI(mWorkDir + mSuccessNum);
if (ClientOpType.CREATE_FILE == mOpType) {
sTfs.getOutStream(testURI, sOutStreamOptions).close();
} else if (ClientOpType.CREATE_DELETE_FILE == mOpType) {
try {
sTfs.getOutStream(testURI, sOutStreamOptions).close();
} catch (TachyonException e) {
// If file already exists, ignore it.
if (e.getType() != TachyonExceptionType.FILE_ALREADY_EXISTS) {
throw e;
}
} catch (Exception e) {
throw e;
}
sTfs.delete(sTfs.open(testURI));
} else if (ClientOpType.CREATE_RENAME_FILE == mOpType) {
try {
sTfs.getOutStream(testURI, sOutStreamOptions).close();
} catch (TachyonException e) {
// If file already exists, ignore it.
if (e.getType() != TachyonExceptionType.FILE_ALREADY_EXISTS) {
throw e;
}
} catch (Exception e) {
throw e;
}
sTfs.rename(sTfs.open(testURI), new TachyonURI(testURI + "-rename"));
} else if (ClientOpType.CREATE_TABLE == mOpType) {
if (sOldTfs.createRawTable(new TachyonURI(mWorkDir + mSuccessNum), 1) == -1) {
break;
}
}
} catch (Exception e) {
// Since master may crash/restart for several times, so this exception is expected.
// Ignore the exception and still keep requesting to master.
continue;
}
mSuccessNum++;
CommonUtils.sleepMs(100);
}
}
@Override
public void run() {
long lastCheckMs = System.currentTimeMillis();
while (mRunning) {
// Check the time since last check, and wait until it is within check interval
long lastIntervalMs = System.currentTimeMillis() - lastCheckMs;
long toSleepMs = mCheckIntervalMs - lastIntervalMs;
if (toSleepMs > 0) {
CommonUtils.sleepMs(LOG, toSleepMs);
} else {
LOG.warn("Space reserver took: {}, expected: {}", lastIntervalMs, mCheckIntervalMs);
}
reserveSpace();
}
}
/**
* Stop this TachyonWorker. Stop all the threads belong to this TachyonWorker.
*
* @throws IOException
* @throws InterruptedException
*/
public void stop() throws IOException, InterruptedException {
mStop = true;
mWorkerStorage.stop();
mDataServer.close();
mServer.stop();
mServerTNonblockingServerSocket.close();
mExecutorService.shutdown();
while (!mDataServer.isClosed() || mServer.isServing() || mHeartbeatThread.isAlive()) {
// TODO The reason to stop and close again is due to some issues in Thrift.
mServer.stop();
mServerTNonblockingServerSocket.close();
CommonUtils.sleepMs(null, 100);
}
mHeartbeatThread.join();
}
private ByteBuffer readRemoteByteBuffer(ClientBlockInfo blockInfo, long offset, long len) {
ByteBuffer buf = null;
try {
List<NetAddress> blockLocations = blockInfo.getLocations();
LOG.info("Block locations:" + blockLocations);
for (int k = 0; k < blockLocations.size(); k++) {
String host = blockLocations.get(k).mHost;
int port = blockLocations.get(k).mSecondaryPort;
// The data is not in remote machine's memory if port == -1.
if (port == -1) {
continue;
}
if (host.equals(InetAddress.getLocalHost().getHostName())
|| host.equals(InetAddress.getLocalHost().getHostAddress())) {
String localFileName = CommonUtils.concat(TFS.getRootFolder(), blockInfo.blockId);
LOG.warn("Master thinks the local machine has data " + localFileName + "! But not!");
}
LOG.info(
host
+ ":"
+ port
+ " current host is "
+ InetAddress.getLocalHost().getHostName()
+ " "
+ InetAddress.getLocalHost().getHostAddress());
try {
buf =
retrieveByteBufferFromRemoteMachine(
new InetSocketAddress(host, port), blockInfo.blockId, offset, len);
if (buf != null) {
break;
}
} catch (IOException e) {
LOG.error(e.getMessage());
buf = null;
}
}
} catch (IOException e) {
LOG.error("Failed to get read data from remote " + e.getMessage());
buf = null;
}
return buf;
}