@BeforeClass
public static void beforeClass() {
sFileOptions =
new CreatePathOptions.Builder(MasterContext.getConf())
.setBlockSizeBytes(Constants.KB)
.setPermissionStatus(TEST_PERMISSION_STATUS)
.build();
sDirectoryOptions =
new CreatePathOptions.Builder(MasterContext.getConf())
.setBlockSizeBytes(Constants.KB)
.setDirectory(true)
.setPermissionStatus(TEST_PERMISSION_STATUS)
.build();
sNestedFileOptions =
new CreatePathOptions.Builder(MasterContext.getConf())
.setBlockSizeBytes(Constants.KB)
.setPermissionStatus(TEST_PERMISSION_STATUS)
.setRecursive(true)
.build();
sNestedDirectoryOptions =
new CreatePathOptions.Builder(MasterContext.getConf())
.setBlockSizeBytes(Constants.KB)
.setPermissionStatus(TEST_PERMISSION_STATUS)
.setDirectory(true)
.setRecursive(true)
.build();
}
@Override
public void start(boolean isLeader) throws IOException {
super.start(isLeader);
mGlobalStorageTierAssoc = new MasterStorageTierAssoc(MasterContext.getConf());
if (isLeader) {
mLostWorkerDetectionService =
getExecutorService()
.submit(
new HeartbeatThread(
HeartbeatContext.MASTER_LOST_WORKER_DETECTION,
new LostWorkerDetectionHeartbeatExecutor(),
MasterContext.getConf().getInt(Constants.MASTER_HEARTBEAT_INTERVAL_MS)));
}
}
/**
* @param master the master to apply the journal entries to
* @param journal the journal to tail
*/
public JournalTailerThread(Master master, Journal journal) {
mMaster = Preconditions.checkNotNull(master);
mJournal = Preconditions.checkNotNull(journal);
TachyonConf conf = MasterContext.getConf();
mShutdownQuietWaitTimeMs =
conf.getInt(Constants.MASTER_JOURNAL_TAILER_SHUTDOWN_QUIET_WAIT_TIME_MS);
mJournalTailerSleepTimeMs = conf.getInt(Constants.MASTER_JOURNAL_TAILER_SLEEP_TIME_MS);
}
@Test
public void createFileWithNegativeBlockSizeTest() throws Exception {
mThrown.expect(BlockInfoException.class);
mThrown.expectMessage("Invalid block size -1");
CreatePathOptions options =
new CreatePathOptions.Builder(MasterContext.getConf()).setBlockSizeBytes(-1).build();
mTree.createPath(TEST_URI, options);
}
@Test
public void createPathTest() throws Exception {
// save the last mod time of the root
long lastModTime = mTree.getRoot().getLastModificationTimeMs();
// sleep to ensure a different last modification time
CommonUtils.sleepMs(10);
// create nested directory
InodeTree.CreatePathResult createResult = mTree.createPath(NESTED_URI, sNestedDirectoryOptions);
List<Inode> modified = createResult.getModified();
List<Inode> created = createResult.getCreated();
// 1 modified directory
Assert.assertEquals(1, modified.size());
Assert.assertEquals("", modified.get(0).getName());
Assert.assertNotEquals(lastModTime, modified.get(0).getLastModificationTimeMs());
// 2 created directories
Assert.assertEquals(2, created.size());
Assert.assertEquals("nested", created.get(0).getName());
Assert.assertEquals("test", created.get(1).getName());
// save the last mod time of 'test'
lastModTime = created.get(1).getLastModificationTimeMs();
// sleep to ensure a different last modification time
CommonUtils.sleepMs(10);
// creating the directory path again results in no new inodes.
try {
createResult = mTree.createPath(NESTED_URI, sNestedDirectoryOptions);
Assert.assertTrue("createPath should throw FileAlreadyExistsException", false);
} catch (FileAlreadyExistsException faee) {
Assert.assertEquals(
faee.getMessage(), ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(NESTED_URI));
}
// create a file
CreatePathOptions options =
new CreatePathOptions.Builder(MasterContext.getConf())
.setBlockSizeBytes(Constants.KB)
.setRecursive(true)
.build();
createResult = mTree.createPath(NESTED_FILE_URI, options);
modified = createResult.getModified();
created = createResult.getCreated();
// test directory was modified
Assert.assertEquals(1, modified.size());
Assert.assertEquals("test", modified.get(0).getName());
Assert.assertNotEquals(lastModTime, modified.get(0).getLastModificationTimeMs());
// file was created
Assert.assertEquals(1, created.size());
Assert.assertEquals("file", created.get(0).getName());
}
@Before
public final void before() throws Exception {
TachyonConf tachyonConf = MasterContext.getConf();
tachyonConf.set(Constants.USER_FILE_BUFFER_BYTES, String.valueOf(100));
mLocalTachyonCluster =
new LocalTachyonCluster(MEM_CAPACITY_BYTES, USER_QUOTA_UNIT_BYTES, Constants.GB);
mLocalTachyonCluster.start();
mTFS = mLocalTachyonCluster.getClient();
mWorkerConf = mLocalTachyonCluster.getWorkerTachyonConf();
mWorkerToMasterHeartbeatIntervalMs =
mWorkerConf.getInt(Constants.WORKER_BLOCK_HEARTBEAT_INTERVAL_MS);
mSetPinned = new SetStateOptions.Builder(mWorkerConf).setPinned(true).build();
mSetUnpinned = new SetStateOptions.Builder(mWorkerConf).setPinned(false).build();
}
/**
* Creates a new key-value store.
*
* @param path URI of the key-value store
* @throws FileAlreadyExistsException if a key-value store URI exists
*/
public synchronized void createStore(TachyonURI path)
throws FileAlreadyExistsException, InvalidPathException {
try {
// Create this dir
mFileSystemMaster.mkdir(
path,
new CreateDirectoryOptions.Builder(MasterContext.getConf()).setRecursive(true).build());
} catch (IOException e) {
// TODO(binfan): Investigate why mFileSystemMaster.mkdir throws IOException
throw new InvalidPathException(
String.format("Failed to createStore: can not create path %s", path), e);
}
final long fileId = mFileSystemMaster.getFileId(path);
Preconditions.checkState(fileId != IdUtils.INVALID_FILE_ID);
createStoreInternal(fileId);
writeJournalEntry(newCreateStoreEntry(fileId));
flushJournal();
}
@Override
public void heartbeat() {
LOG.debug("System status checking.");
TachyonConf conf = MasterContext.getConf();
int masterWorkerTimeoutMs = conf.getInt(Constants.MASTER_WORKER_TIMEOUT_MS);
synchronized (mBlocks) {
synchronized (mWorkers) {
Iterator<MasterWorkerInfo> iter = mWorkers.iterator();
while (iter.hasNext()) {
MasterWorkerInfo worker = iter.next();
final long lastUpdate = CommonUtils.getCurrentMs() - worker.getLastUpdatedTimeMs();
if (lastUpdate > masterWorkerTimeoutMs) {
LOG.error("The worker {} got timed out!", worker);
mLostWorkers.add(worker);
iter.remove();
processLostWorker(worker);
}
}
}
}
}
/**
* A bucket with all files whose ttl value lies in the bucket's time interval. The bucket's time
* interval starts at a specific time and lasts for {@link Constants#MASTER_TTLCHECKER_INTERVAL_MS}.
*
* <p>Not thread-safe. Only for use related to {@link TTLBucketList}.
*/
public final class TTLBucket implements Comparable<TTLBucket> {
/** The time interval of this bucket is the same as ttl checker's interval. */
private static long sTTLIntervalMs =
MasterContext.getConf().getInt(Constants.MASTER_TTLCHECKER_INTERVAL_MS);
/**
* Each bucket has a time to live interval, this value is the start of the interval, interval
* value is the same as the configuration of {@link Constants#MASTER_TTLCHECKER_INTERVAL_MS}.
*/
private long mTTLIntervalStartTimeMs;
/** A set of InodeFiles whose ttl value is in the range of this bucket's interval. */
private Set<InodeFile> mFiles;
public TTLBucket(long startTimeMs) {
mTTLIntervalStartTimeMs = startTimeMs;
mFiles = new HashSet<InodeFile>();
}
public long getTTLIntervalStartTimeMs() {
return mTTLIntervalStartTimeMs;
}
public long getTTLIntervalEndTimeMs() {
return mTTLIntervalStartTimeMs + sTTLIntervalMs;
}
public static long getTTLIntervalMs() {
return sTTLIntervalMs;
}
public static void setTTlIntervalMs(long intervalMs) {
sTTLIntervalMs = intervalMs;
}
/**
* @return the set of all files in the bucket backed by the internal set, changes made to the
* returned set will be shown in the internal set, and vice versa
*/
public Set<InodeFile> getFiles() {
return mFiles;
}
/**
* Adds a file to the bucket.
*
* @param file the file to be added
*/
public void addFile(InodeFile file) {
mFiles.add(file);
}
/**
* Removes a file from the bucket.
*
* @param file the file to be removed
*/
public void removeFile(InodeFile file) {
mFiles.remove(file);
}
/**
* Compares this bucket's TTL interval start time to that of another bucket.
*
* @param ttlBucket the bucket to be compared to
* @return 0 when return values of {@link #getTTLIntervalStartTimeMs()} from the two buckets are
* the same, -1 when that value of current instance is smaller, otherwise, 1
*/
@Override
public int compareTo(TTLBucket ttlBucket) {
long startTime1 = getTTLIntervalStartTimeMs();
long startTime2 = ttlBucket.getTTLIntervalStartTimeMs();
if (startTime1 < startTime2) {
return -1;
}
if (startTime1 == startTime2) {
return 0;
}
return 1;
}
/**
* Compares to a specific object.
*
* @param o the object to compare
* @return true if object is also {@link TTLBucket} and represents the same TTLIntervalStartTime
*/
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (!(o instanceof TTLBucket)) {
return false;
}
TTLBucket that = (TTLBucket) o;
return mTTLIntervalStartTimeMs == that.mTTLIntervalStartTimeMs;
}
/**
* Returns the hash code for the {@link TTLBucket}.
*
* @return The hash code value for this {@link TTLBucket}
*/
@Override
public int hashCode() {
return Objects.hashCode(mTTLIntervalStartTimeMs);
}
}
/** @return the default {@link MkdirOptions} */
public static MkdirOptions defaults() {
return new Builder(MasterContext.getConf()).build();
}