@BeforeClass public static void beforeClass() { sFileOptions = new CreatePathOptions.Builder(MasterContext.getConf()) .setBlockSizeBytes(Constants.KB) .setPermissionStatus(TEST_PERMISSION_STATUS) .build(); sDirectoryOptions = new CreatePathOptions.Builder(MasterContext.getConf()) .setBlockSizeBytes(Constants.KB) .setDirectory(true) .setPermissionStatus(TEST_PERMISSION_STATUS) .build(); sNestedFileOptions = new CreatePathOptions.Builder(MasterContext.getConf()) .setBlockSizeBytes(Constants.KB) .setPermissionStatus(TEST_PERMISSION_STATUS) .setRecursive(true) .build(); sNestedDirectoryOptions = new CreatePathOptions.Builder(MasterContext.getConf()) .setBlockSizeBytes(Constants.KB) .setPermissionStatus(TEST_PERMISSION_STATUS) .setDirectory(true) .setRecursive(true) .build(); }
@Override public void start(boolean isLeader) throws IOException { super.start(isLeader); mGlobalStorageTierAssoc = new MasterStorageTierAssoc(MasterContext.getConf()); if (isLeader) { mLostWorkerDetectionService = getExecutorService() .submit( new HeartbeatThread( HeartbeatContext.MASTER_LOST_WORKER_DETECTION, new LostWorkerDetectionHeartbeatExecutor(), MasterContext.getConf().getInt(Constants.MASTER_HEARTBEAT_INTERVAL_MS))); } }
/** * @param master the master to apply the journal entries to * @param journal the journal to tail */ public JournalTailerThread(Master master, Journal journal) { mMaster = Preconditions.checkNotNull(master); mJournal = Preconditions.checkNotNull(journal); TachyonConf conf = MasterContext.getConf(); mShutdownQuietWaitTimeMs = conf.getInt(Constants.MASTER_JOURNAL_TAILER_SHUTDOWN_QUIET_WAIT_TIME_MS); mJournalTailerSleepTimeMs = conf.getInt(Constants.MASTER_JOURNAL_TAILER_SLEEP_TIME_MS); }
@Test public void createFileWithNegativeBlockSizeTest() throws Exception { mThrown.expect(BlockInfoException.class); mThrown.expectMessage("Invalid block size -1"); CreatePathOptions options = new CreatePathOptions.Builder(MasterContext.getConf()).setBlockSizeBytes(-1).build(); mTree.createPath(TEST_URI, options); }
@Test public void createPathTest() throws Exception { // save the last mod time of the root long lastModTime = mTree.getRoot().getLastModificationTimeMs(); // sleep to ensure a different last modification time CommonUtils.sleepMs(10); // create nested directory InodeTree.CreatePathResult createResult = mTree.createPath(NESTED_URI, sNestedDirectoryOptions); List<Inode> modified = createResult.getModified(); List<Inode> created = createResult.getCreated(); // 1 modified directory Assert.assertEquals(1, modified.size()); Assert.assertEquals("", modified.get(0).getName()); Assert.assertNotEquals(lastModTime, modified.get(0).getLastModificationTimeMs()); // 2 created directories Assert.assertEquals(2, created.size()); Assert.assertEquals("nested", created.get(0).getName()); Assert.assertEquals("test", created.get(1).getName()); // save the last mod time of 'test' lastModTime = created.get(1).getLastModificationTimeMs(); // sleep to ensure a different last modification time CommonUtils.sleepMs(10); // creating the directory path again results in no new inodes. try { createResult = mTree.createPath(NESTED_URI, sNestedDirectoryOptions); Assert.assertTrue("createPath should throw FileAlreadyExistsException", false); } catch (FileAlreadyExistsException faee) { Assert.assertEquals( faee.getMessage(), ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(NESTED_URI)); } // create a file CreatePathOptions options = new CreatePathOptions.Builder(MasterContext.getConf()) .setBlockSizeBytes(Constants.KB) .setRecursive(true) .build(); createResult = mTree.createPath(NESTED_FILE_URI, options); modified = createResult.getModified(); created = createResult.getCreated(); // test directory was modified Assert.assertEquals(1, modified.size()); Assert.assertEquals("test", modified.get(0).getName()); Assert.assertNotEquals(lastModTime, modified.get(0).getLastModificationTimeMs()); // file was created Assert.assertEquals(1, created.size()); Assert.assertEquals("file", created.get(0).getName()); }
@Before public final void before() throws Exception { TachyonConf tachyonConf = MasterContext.getConf(); tachyonConf.set(Constants.USER_FILE_BUFFER_BYTES, String.valueOf(100)); mLocalTachyonCluster = new LocalTachyonCluster(MEM_CAPACITY_BYTES, USER_QUOTA_UNIT_BYTES, Constants.GB); mLocalTachyonCluster.start(); mTFS = mLocalTachyonCluster.getClient(); mWorkerConf = mLocalTachyonCluster.getWorkerTachyonConf(); mWorkerToMasterHeartbeatIntervalMs = mWorkerConf.getInt(Constants.WORKER_BLOCK_HEARTBEAT_INTERVAL_MS); mSetPinned = new SetStateOptions.Builder(mWorkerConf).setPinned(true).build(); mSetUnpinned = new SetStateOptions.Builder(mWorkerConf).setPinned(false).build(); }
/** * Creates a new key-value store. * * @param path URI of the key-value store * @throws FileAlreadyExistsException if a key-value store URI exists */ public synchronized void createStore(TachyonURI path) throws FileAlreadyExistsException, InvalidPathException { try { // Create this dir mFileSystemMaster.mkdir( path, new CreateDirectoryOptions.Builder(MasterContext.getConf()).setRecursive(true).build()); } catch (IOException e) { // TODO(binfan): Investigate why mFileSystemMaster.mkdir throws IOException throw new InvalidPathException( String.format("Failed to createStore: can not create path %s", path), e); } final long fileId = mFileSystemMaster.getFileId(path); Preconditions.checkState(fileId != IdUtils.INVALID_FILE_ID); createStoreInternal(fileId); writeJournalEntry(newCreateStoreEntry(fileId)); flushJournal(); }
@Override public void heartbeat() { LOG.debug("System status checking."); TachyonConf conf = MasterContext.getConf(); int masterWorkerTimeoutMs = conf.getInt(Constants.MASTER_WORKER_TIMEOUT_MS); synchronized (mBlocks) { synchronized (mWorkers) { Iterator<MasterWorkerInfo> iter = mWorkers.iterator(); while (iter.hasNext()) { MasterWorkerInfo worker = iter.next(); final long lastUpdate = CommonUtils.getCurrentMs() - worker.getLastUpdatedTimeMs(); if (lastUpdate > masterWorkerTimeoutMs) { LOG.error("The worker {} got timed out!", worker); mLostWorkers.add(worker); iter.remove(); processLostWorker(worker); } } } } }
/** * A bucket with all files whose ttl value lies in the bucket's time interval. The bucket's time * interval starts at a specific time and lasts for {@link Constants#MASTER_TTLCHECKER_INTERVAL_MS}. * * <p>Not thread-safe. Only for use related to {@link TTLBucketList}. */ public final class TTLBucket implements Comparable<TTLBucket> { /** The time interval of this bucket is the same as ttl checker's interval. */ private static long sTTLIntervalMs = MasterContext.getConf().getInt(Constants.MASTER_TTLCHECKER_INTERVAL_MS); /** * Each bucket has a time to live interval, this value is the start of the interval, interval * value is the same as the configuration of {@link Constants#MASTER_TTLCHECKER_INTERVAL_MS}. */ private long mTTLIntervalStartTimeMs; /** A set of InodeFiles whose ttl value is in the range of this bucket's interval. */ private Set<InodeFile> mFiles; public TTLBucket(long startTimeMs) { mTTLIntervalStartTimeMs = startTimeMs; mFiles = new HashSet<InodeFile>(); } public long getTTLIntervalStartTimeMs() { return mTTLIntervalStartTimeMs; } public long getTTLIntervalEndTimeMs() { return mTTLIntervalStartTimeMs + sTTLIntervalMs; } public static long getTTLIntervalMs() { return sTTLIntervalMs; } public static void setTTlIntervalMs(long intervalMs) { sTTLIntervalMs = intervalMs; } /** * @return the set of all files in the bucket backed by the internal set, changes made to the * returned set will be shown in the internal set, and vice versa */ public Set<InodeFile> getFiles() { return mFiles; } /** * Adds a file to the bucket. * * @param file the file to be added */ public void addFile(InodeFile file) { mFiles.add(file); } /** * Removes a file from the bucket. * * @param file the file to be removed */ public void removeFile(InodeFile file) { mFiles.remove(file); } /** * Compares this bucket's TTL interval start time to that of another bucket. * * @param ttlBucket the bucket to be compared to * @return 0 when return values of {@link #getTTLIntervalStartTimeMs()} from the two buckets are * the same, -1 when that value of current instance is smaller, otherwise, 1 */ @Override public int compareTo(TTLBucket ttlBucket) { long startTime1 = getTTLIntervalStartTimeMs(); long startTime2 = ttlBucket.getTTLIntervalStartTimeMs(); if (startTime1 < startTime2) { return -1; } if (startTime1 == startTime2) { return 0; } return 1; } /** * Compares to a specific object. * * @param o the object to compare * @return true if object is also {@link TTLBucket} and represents the same TTLIntervalStartTime */ @Override public boolean equals(Object o) { if (this == o) { return true; } if (!(o instanceof TTLBucket)) { return false; } TTLBucket that = (TTLBucket) o; return mTTLIntervalStartTimeMs == that.mTTLIntervalStartTimeMs; } /** * Returns the hash code for the {@link TTLBucket}. * * @return The hash code value for this {@link TTLBucket} */ @Override public int hashCode() { return Objects.hashCode(mTTLIntervalStartTimeMs); } }
/** @return the default {@link MkdirOptions} */ public static MkdirOptions defaults() { return new Builder(MasterContext.getConf()).build(); }