Пример #1
0
 @BeforeClass
 public static void beforeClass() {
   sFileOptions =
       new CreatePathOptions.Builder(MasterContext.getConf())
           .setBlockSizeBytes(Constants.KB)
           .setPermissionStatus(TEST_PERMISSION_STATUS)
           .build();
   sDirectoryOptions =
       new CreatePathOptions.Builder(MasterContext.getConf())
           .setBlockSizeBytes(Constants.KB)
           .setDirectory(true)
           .setPermissionStatus(TEST_PERMISSION_STATUS)
           .build();
   sNestedFileOptions =
       new CreatePathOptions.Builder(MasterContext.getConf())
           .setBlockSizeBytes(Constants.KB)
           .setPermissionStatus(TEST_PERMISSION_STATUS)
           .setRecursive(true)
           .build();
   sNestedDirectoryOptions =
       new CreatePathOptions.Builder(MasterContext.getConf())
           .setBlockSizeBytes(Constants.KB)
           .setPermissionStatus(TEST_PERMISSION_STATUS)
           .setDirectory(true)
           .setRecursive(true)
           .build();
 }
Пример #2
0
 @Override
 public void start(boolean isLeader) throws IOException {
   super.start(isLeader);
   mGlobalStorageTierAssoc = new MasterStorageTierAssoc(MasterContext.getConf());
   if (isLeader) {
     mLostWorkerDetectionService =
         getExecutorService()
             .submit(
                 new HeartbeatThread(
                     HeartbeatContext.MASTER_LOST_WORKER_DETECTION,
                     new LostWorkerDetectionHeartbeatExecutor(),
                     MasterContext.getConf().getInt(Constants.MASTER_HEARTBEAT_INTERVAL_MS)));
   }
 }
Пример #3
0
 /**
  * @param master the master to apply the journal entries to
  * @param journal the journal to tail
  */
 public JournalTailerThread(Master master, Journal journal) {
   mMaster = Preconditions.checkNotNull(master);
   mJournal = Preconditions.checkNotNull(journal);
   TachyonConf conf = MasterContext.getConf();
   mShutdownQuietWaitTimeMs =
       conf.getInt(Constants.MASTER_JOURNAL_TAILER_SHUTDOWN_QUIET_WAIT_TIME_MS);
   mJournalTailerSleepTimeMs = conf.getInt(Constants.MASTER_JOURNAL_TAILER_SLEEP_TIME_MS);
 }
Пример #4
0
  @Test
  public void createFileWithNegativeBlockSizeTest() throws Exception {
    mThrown.expect(BlockInfoException.class);
    mThrown.expectMessage("Invalid block size -1");

    CreatePathOptions options =
        new CreatePathOptions.Builder(MasterContext.getConf()).setBlockSizeBytes(-1).build();
    mTree.createPath(TEST_URI, options);
  }
Пример #5
0
  @Test
  public void createPathTest() throws Exception {
    // save the last mod time of the root
    long lastModTime = mTree.getRoot().getLastModificationTimeMs();
    // sleep to ensure a different last modification time
    CommonUtils.sleepMs(10);

    // create nested directory
    InodeTree.CreatePathResult createResult = mTree.createPath(NESTED_URI, sNestedDirectoryOptions);
    List<Inode> modified = createResult.getModified();
    List<Inode> created = createResult.getCreated();
    // 1 modified directory
    Assert.assertEquals(1, modified.size());
    Assert.assertEquals("", modified.get(0).getName());
    Assert.assertNotEquals(lastModTime, modified.get(0).getLastModificationTimeMs());
    // 2 created directories
    Assert.assertEquals(2, created.size());
    Assert.assertEquals("nested", created.get(0).getName());
    Assert.assertEquals("test", created.get(1).getName());
    // save the last mod time of 'test'
    lastModTime = created.get(1).getLastModificationTimeMs();
    // sleep to ensure a different last modification time
    CommonUtils.sleepMs(10);

    // creating the directory path again results in no new inodes.
    try {
      createResult = mTree.createPath(NESTED_URI, sNestedDirectoryOptions);
      Assert.assertTrue("createPath should throw FileAlreadyExistsException", false);
    } catch (FileAlreadyExistsException faee) {
      Assert.assertEquals(
          faee.getMessage(), ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(NESTED_URI));
    }

    // create a file
    CreatePathOptions options =
        new CreatePathOptions.Builder(MasterContext.getConf())
            .setBlockSizeBytes(Constants.KB)
            .setRecursive(true)
            .build();
    createResult = mTree.createPath(NESTED_FILE_URI, options);
    modified = createResult.getModified();
    created = createResult.getCreated();
    // test directory was modified
    Assert.assertEquals(1, modified.size());
    Assert.assertEquals("test", modified.get(0).getName());
    Assert.assertNotEquals(lastModTime, modified.get(0).getLastModificationTimeMs());
    // file was created
    Assert.assertEquals(1, created.size());
    Assert.assertEquals("file", created.get(0).getName());
  }
  @Before
  public final void before() throws Exception {
    TachyonConf tachyonConf = MasterContext.getConf();
    tachyonConf.set(Constants.USER_FILE_BUFFER_BYTES, String.valueOf(100));

    mLocalTachyonCluster =
        new LocalTachyonCluster(MEM_CAPACITY_BYTES, USER_QUOTA_UNIT_BYTES, Constants.GB);
    mLocalTachyonCluster.start();
    mTFS = mLocalTachyonCluster.getClient();
    mWorkerConf = mLocalTachyonCluster.getWorkerTachyonConf();
    mWorkerToMasterHeartbeatIntervalMs =
        mWorkerConf.getInt(Constants.WORKER_BLOCK_HEARTBEAT_INTERVAL_MS);
    mSetPinned = new SetStateOptions.Builder(mWorkerConf).setPinned(true).build();
    mSetUnpinned = new SetStateOptions.Builder(mWorkerConf).setPinned(false).build();
  }
Пример #7
0
  /**
   * Creates a new key-value store.
   *
   * @param path URI of the key-value store
   * @throws FileAlreadyExistsException if a key-value store URI exists
   */
  public synchronized void createStore(TachyonURI path)
      throws FileAlreadyExistsException, InvalidPathException {
    try {
      // Create this dir
      mFileSystemMaster.mkdir(
          path,
          new CreateDirectoryOptions.Builder(MasterContext.getConf()).setRecursive(true).build());
    } catch (IOException e) {
      // TODO(binfan): Investigate why mFileSystemMaster.mkdir throws IOException
      throw new InvalidPathException(
          String.format("Failed to createStore: can not create path %s", path), e);
    }
    final long fileId = mFileSystemMaster.getFileId(path);
    Preconditions.checkState(fileId != IdUtils.INVALID_FILE_ID);

    createStoreInternal(fileId);
    writeJournalEntry(newCreateStoreEntry(fileId));
    flushJournal();
  }
Пример #8
0
    @Override
    public void heartbeat() {
      LOG.debug("System status checking.");
      TachyonConf conf = MasterContext.getConf();

      int masterWorkerTimeoutMs = conf.getInt(Constants.MASTER_WORKER_TIMEOUT_MS);
      synchronized (mBlocks) {
        synchronized (mWorkers) {
          Iterator<MasterWorkerInfo> iter = mWorkers.iterator();
          while (iter.hasNext()) {
            MasterWorkerInfo worker = iter.next();
            final long lastUpdate = CommonUtils.getCurrentMs() - worker.getLastUpdatedTimeMs();
            if (lastUpdate > masterWorkerTimeoutMs) {
              LOG.error("The worker {} got timed out!", worker);
              mLostWorkers.add(worker);
              iter.remove();
              processLostWorker(worker);
            }
          }
        }
      }
    }
Пример #9
0
/**
 * A bucket with all files whose ttl value lies in the bucket's time interval. The bucket's time
 * interval starts at a specific time and lasts for {@link Constants#MASTER_TTLCHECKER_INTERVAL_MS}.
 *
 * <p>Not thread-safe. Only for use related to {@link TTLBucketList}.
 */
public final class TTLBucket implements Comparable<TTLBucket> {
  /** The time interval of this bucket is the same as ttl checker's interval. */
  private static long sTTLIntervalMs =
      MasterContext.getConf().getInt(Constants.MASTER_TTLCHECKER_INTERVAL_MS);
  /**
   * Each bucket has a time to live interval, this value is the start of the interval, interval
   * value is the same as the configuration of {@link Constants#MASTER_TTLCHECKER_INTERVAL_MS}.
   */
  private long mTTLIntervalStartTimeMs;
  /** A set of InodeFiles whose ttl value is in the range of this bucket's interval. */
  private Set<InodeFile> mFiles;

  public TTLBucket(long startTimeMs) {
    mTTLIntervalStartTimeMs = startTimeMs;
    mFiles = new HashSet<InodeFile>();
  }

  public long getTTLIntervalStartTimeMs() {
    return mTTLIntervalStartTimeMs;
  }

  public long getTTLIntervalEndTimeMs() {
    return mTTLIntervalStartTimeMs + sTTLIntervalMs;
  }

  public static long getTTLIntervalMs() {
    return sTTLIntervalMs;
  }

  public static void setTTlIntervalMs(long intervalMs) {
    sTTLIntervalMs = intervalMs;
  }

  /**
   * @return the set of all files in the bucket backed by the internal set, changes made to the
   *     returned set will be shown in the internal set, and vice versa
   */
  public Set<InodeFile> getFiles() {
    return mFiles;
  }

  /**
   * Adds a file to the bucket.
   *
   * @param file the file to be added
   */
  public void addFile(InodeFile file) {
    mFiles.add(file);
  }

  /**
   * Removes a file from the bucket.
   *
   * @param file the file to be removed
   */
  public void removeFile(InodeFile file) {
    mFiles.remove(file);
  }

  /**
   * Compares this bucket's TTL interval start time to that of another bucket.
   *
   * @param ttlBucket the bucket to be compared to
   * @return 0 when return values of {@link #getTTLIntervalStartTimeMs()} from the two buckets are
   *     the same, -1 when that value of current instance is smaller, otherwise, 1
   */
  @Override
  public int compareTo(TTLBucket ttlBucket) {
    long startTime1 = getTTLIntervalStartTimeMs();
    long startTime2 = ttlBucket.getTTLIntervalStartTimeMs();

    if (startTime1 < startTime2) {
      return -1;
    }
    if (startTime1 == startTime2) {
      return 0;
    }
    return 1;
  }

  /**
   * Compares to a specific object.
   *
   * @param o the object to compare
   * @return true if object is also {@link TTLBucket} and represents the same TTLIntervalStartTime
   */
  @Override
  public boolean equals(Object o) {
    if (this == o) {
      return true;
    }
    if (!(o instanceof TTLBucket)) {
      return false;
    }
    TTLBucket that = (TTLBucket) o;
    return mTTLIntervalStartTimeMs == that.mTTLIntervalStartTimeMs;
  }

  /**
   * Returns the hash code for the {@link TTLBucket}.
   *
   * @return The hash code value for this {@link TTLBucket}
   */
  @Override
  public int hashCode() {
    return Objects.hashCode(mTTLIntervalStartTimeMs);
  }
}
Пример #10
0
 /** @return the default {@link MkdirOptions} */
 public static MkdirOptions defaults() {
   return new Builder(MasterContext.getConf()).build();
 }