Example #1
0
  private void initializeWorkerStorage()
      throws IOException, FileDoesNotExistException, SuspectedFileSizeException, BlockInfoException,
          TException {
    LOG.info("Initializing the worker storage.");
    if (!mLocalDataFolder.exists()) {
      LOG.info("Local folder " + mLocalDataFolder + " does not exist. Creating a new one.");
      mLocalDataFolder.mkdirs();
      mLocalUserFolder.mkdirs();

      CommonUtils.changeLocalFilePermission(mLocalDataFolder.getPath(), "775");
      CommonUtils.changeLocalFilePermission(mLocalUserFolder.getPath(), "775");
      return;
    }

    if (!mLocalDataFolder.isDirectory()) {
      String tmp = "Data folder " + mLocalDataFolder + " is not a folder!";
      LOG.error(tmp);
      throw new IllegalArgumentException(tmp);
    }

    if (mLocalUserFolder.exists()) {
      try {
        FileUtils.deleteDirectory(mLocalUserFolder);
      } catch (IOException e) {
        LOG.error(e.getMessage(), e);
      }
    }
    mLocalUserFolder.mkdir();
    CommonUtils.changeLocalFilePermission(mLocalUserFolder.getPath(), "775");

    mUnderfsOrphansFolder = mUnderfsWorkerFolder + "/orphans";
    if (!mUnderFs.exists(mUnderfsOrphansFolder)) {
      mUnderFs.mkdirs(mUnderfsOrphansFolder, true);
    }

    int cnt = 0;
    for (File tFile : mLocalDataFolder.listFiles()) {
      if (tFile.isFile()) {
        cnt++;
        LOG.info("File " + cnt + ": " + tFile.getPath() + " with size " + tFile.length() + " Bs.");

        long blockId = CommonUtils.getBlockIdFromFileName(tFile.getName());
        boolean success = mWorkerSpaceCounter.requestSpaceBytes(tFile.length());
        try {
          addFoundBlock(blockId, tFile.length());
        } catch (FileDoesNotExistException e) {
          LOG.error("BlockId: " + blockId + " becomes orphan for: \"" + e.message + "\"");
          LOG.info(
              "Swapout File " + cnt + ": blockId: " + blockId + " to " + mUnderfsOrphansFolder);
          swapoutOrphanBlocks(blockId, tFile);
          freeBlock(blockId);
          continue;
        }
        mAddedBlockList.add(blockId);
        if (!success) {
          throw new RuntimeException("Pre-existing files exceed the local memory capacity.");
        }
      }
    }
  }
Example #2
0
  @Test
  public void reserveTest() throws Exception {
    // Reserve on top tier
    long blockId = 100;
    BlockStoreLocation tier0 = BlockStoreLocation.anyDirInTier(StorageLevelAlias.MEM.getValue());
    for (int i = 0; i < 3; i++) {
      TieredBlockStoreTestUtils.cache(SESSION_ID, blockId++, BLOCK_SIZE, mBlockStore, tier0);
    }
    CommonUtils.sleepMs(
        WorkerContext.getConf().getLong(Constants.WORKER_SPACE_RESERVER_INTERVAL_MS));
    BlockStoreMeta storeMeta = mBlockStore.getBlockStoreMeta();
    Assert.assertEquals(3 * BLOCK_SIZE, storeMeta.getUsedBytes());
    List<Long> usedBytesOnTiers = storeMeta.getUsedBytesOnTiers();
    Assert.assertEquals(
        2 * BLOCK_SIZE, (long) usedBytesOnTiers.get(StorageLevelAlias.MEM.getValue() - 1));
    Assert.assertEquals(
        BLOCK_SIZE, (long) usedBytesOnTiers.get(StorageLevelAlias.HDD.getValue() - 1));

    // Reserve on under tier
    for (int i = 0; i < 7; i++) {
      TieredBlockStoreTestUtils.cache(SESSION_ID, blockId++, BLOCK_SIZE, mBlockStore, tier0);
    }
    CommonUtils.sleepMs(
        WorkerContext.getConf().getLong(Constants.WORKER_SPACE_RESERVER_INTERVAL_MS));
    storeMeta = mBlockStore.getBlockStoreMeta();
    Assert.assertEquals(9 * BLOCK_SIZE, storeMeta.getUsedBytes());
    usedBytesOnTiers = storeMeta.getUsedBytesOnTiers();
    Assert.assertEquals(
        2 * BLOCK_SIZE, (long) usedBytesOnTiers.get(StorageLevelAlias.MEM.getValue() - 1));
    Assert.assertEquals(
        7 * BLOCK_SIZE, (long) usedBytesOnTiers.get(StorageLevelAlias.HDD.getValue() - 1));
  }
Example #3
0
 @Test
 public void freeTest() throws IOException {
   TachyonFSTestUtils.createByteFile(mTfs, "/testFile", WriteType.MUST_CACHE, 10);
   mFsShell.free(new String[] {"free", "/testFile"});
   TachyonConf tachyonConf = mLocalTachyonCluster.getMasterTachyonConf();
   CommonUtils.sleepMs(null, CommonUtils.getToMasterHeartBeatIntervalMs(tachyonConf) * 2 + 10);
   Assert.assertFalse(mTfs.getFile(new TachyonURI("/testFile")).isInMemory());
 }
Example #4
0
  @Override
  public void run() {
    long lastHeartbeatMs = System.currentTimeMillis();
    Command cmd = null;
    while (!mStop) {
      long diff = System.currentTimeMillis() - lastHeartbeatMs;
      if (diff < WorkerConf.get().TO_MASTER_HEARTBEAT_INTERVAL_MS) {
        LOG.debug("Heartbeat process takes {} ms.", diff);
        CommonUtils.sleepMs(LOG, WorkerConf.get().TO_MASTER_HEARTBEAT_INTERVAL_MS - diff);
      } else {
        LOG.error("Heartbeat process takes " + diff + " ms.");
      }

      try {
        cmd = mWorkerStorage.heartbeat();

        lastHeartbeatMs = System.currentTimeMillis();
      } catch (IOException e) {
        LOG.error(e.getMessage(), e);
        mWorkerStorage.resetMasterClient();
        CommonUtils.sleepMs(LOG, Constants.SECOND_MS);
        cmd = null;
        if (System.currentTimeMillis() - lastHeartbeatMs >= WorkerConf.get().HEARTBEAT_TIMEOUT_MS) {
          throw new RuntimeException(
              "Timebeat timeout " + (System.currentTimeMillis() - lastHeartbeatMs) + "ms");
        }
      }

      if (cmd != null) {
        switch (cmd.mCommandType) {
          case Unknown:
            LOG.error("Unknown command: " + cmd);
            break;
          case Nothing:
            LOG.debug("Nothing command: {}", cmd);
            break;
          case Register:
            LOG.info("Register command: " + cmd);
            mWorkerStorage.register();
            break;
          case Free:
            mWorkerStorage.freeBlocks(cmd.mData);
            LOG.info("Free command: " + cmd);
            break;
          case Delete:
            LOG.info("Delete command: " + cmd);
            break;
          default:
            throw new RuntimeException("Un-recognized command from master " + cmd.toString());
        }
      }

      mWorkerStorage.checkStatus();
    }
  }
  public static void main(String[] args) throws Exception {
    LocalTachyonCluster cluster = new LocalTachyonCluster(100, 8 * Constants.MB, Constants.GB);
    cluster.start();
    CommonUtils.sleepMs(Constants.SECOND_MS);
    cluster.stop();
    CommonUtils.sleepMs(Constants.SECOND_MS);

    cluster = new LocalTachyonCluster(100, 8 * Constants.MB, Constants.GB);
    cluster.start();
    CommonUtils.sleepMs(Constants.SECOND_MS);
    cluster.stop();
    CommonUtils.sleepMs(Constants.SECOND_MS);
  }
 @Test
 public void singleMasterJournalCrashIntegrationTest() throws Exception {
   LocalTachyonCluster cluster = setupSingleMasterCluster();
   CommonUtils.sleepMs(TEST_TIME_MS);
   // Shutdown the cluster
   cluster.stopTFS();
   CommonUtils.sleepMs(TEST_TIME_MS);
   // Ensure the client threads are stopped.
   mExecutorsForClient.shutdown();
   mExecutorsForClient.awaitTermination(TEST_TIME_MS, TimeUnit.MILLISECONDS);
   reproduceAndCheckState(mCreateFileThread.getSuccessNum());
   // clean up
   cluster.stopUFS();
 }
Example #7
0
  @Test
  public void createPathTest() throws Exception {
    // save the last mod time of the root
    long lastModTime = mTree.getRoot().getLastModificationTimeMs();
    // sleep to ensure a different last modification time
    CommonUtils.sleepMs(10);

    // create nested directory
    InodeTree.CreatePathResult createResult = mTree.createPath(NESTED_URI, sNestedDirectoryOptions);
    List<Inode> modified = createResult.getModified();
    List<Inode> created = createResult.getCreated();
    // 1 modified directory
    Assert.assertEquals(1, modified.size());
    Assert.assertEquals("", modified.get(0).getName());
    Assert.assertNotEquals(lastModTime, modified.get(0).getLastModificationTimeMs());
    // 2 created directories
    Assert.assertEquals(2, created.size());
    Assert.assertEquals("nested", created.get(0).getName());
    Assert.assertEquals("test", created.get(1).getName());
    // save the last mod time of 'test'
    lastModTime = created.get(1).getLastModificationTimeMs();
    // sleep to ensure a different last modification time
    CommonUtils.sleepMs(10);

    // creating the directory path again results in no new inodes.
    try {
      createResult = mTree.createPath(NESTED_URI, sNestedDirectoryOptions);
      Assert.assertTrue("createPath should throw FileAlreadyExistsException", false);
    } catch (FileAlreadyExistsException faee) {
      Assert.assertEquals(
          faee.getMessage(), ExceptionMessage.FILE_ALREADY_EXISTS.getMessage(NESTED_URI));
    }

    // create a file
    CreatePathOptions options =
        new CreatePathOptions.Builder(MasterContext.getConf())
            .setBlockSizeBytes(Constants.KB)
            .setRecursive(true)
            .build();
    createResult = mTree.createPath(NESTED_FILE_URI, options);
    modified = createResult.getModified();
    created = createResult.getCreated();
    // test directory was modified
    Assert.assertEquals(1, modified.size());
    Assert.assertEquals("test", modified.get(0).getName());
    Assert.assertNotEquals(lastModTime, modified.get(0).getLastModificationTimeMs());
    // file was created
    Assert.assertEquals(1, created.size());
    Assert.assertEquals("file", created.get(0).getName());
  }
  // Tests that deletes go through despite failing initially due to concurrent read
  @Test
  public void deleteWhileReadTest() throws Exception {
    TachyonFile file =
        TachyonFSTestUtils.createByteFile(
            mTFS,
            "/test1",
            TachyonStorageType.STORE,
            UnderStorageType.NO_PERSIST,
            MEM_CAPACITY_BYTES);

    CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);

    Assert.assertTrue(mTFS.getInfo(file).getInMemoryPercentage() == 100);
    // Open the file
    InStreamOptions options =
        new InStreamOptions.Builder(new TachyonConf())
            .setTachyonStorageType(TachyonStorageType.STORE)
            .build();
    FileInStream in = mTFS.getInStream(file, options);
    Assert.assertEquals(0, in.read());

    // Delete the file
    mTFS.delete(file);

    CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);

    // After the delete, the master should no longer serve the file
    Assert.assertNull(mTFS.open(new TachyonURI("/test1")));

    // However, the previous read should still be able to read it as the data still exists
    byte[] res = new byte[MEM_CAPACITY_BYTES];
    Assert.assertEquals(MEM_CAPACITY_BYTES - 1, in.read(res, 1, MEM_CAPACITY_BYTES - 1));
    res[0] = 0;
    Assert.assertTrue(BufferUtils.equalIncreasingByteArray(MEM_CAPACITY_BYTES, res));
    in.close();

    CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);

    // After the file is closed, the master's delete should go through and new files can be created
    TachyonFile newFile =
        TachyonFSTestUtils.createByteFile(
            mTFS,
            "/test2",
            TachyonStorageType.STORE,
            UnderStorageType.NO_PERSIST,
            MEM_CAPACITY_BYTES);
    CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);
    Assert.assertTrue(mTFS.getInfo(newFile).getInMemoryPercentage() == 100);
  }
Example #9
0
 public String getBlockSizeBytes() {
   if (mIsDirectory) {
     return " ";
   } else {
     return CommonUtils.getSizeFromBytes(mBlockSizeBytes);
   }
 }
Example #10
0
 public String getSize() {
   if (mIsDirectory) {
     return " ";
   } else {
     return CommonUtils.getSizeFromBytes(mSize);
   }
 }
 /**
  * Keep creating files until something crashes or fail to create. Record how many files are
  * created successfully.
  */
 @Override
 public void run() {
   try {
     // This infinity loop will be broken if something crashes or fail to create. This is
     // expected since the master will shutdown at a certain time.
     while (true) {
       if (mOpType == 0) {
         try {
           mFileSystem.createFile(new TachyonURI(TEST_FILE_DIR + mSuccessNum)).close();
         } catch (IOException e) {
           break;
         }
       } else if (mOpType == 1) {
         // TODO(gene): Add this back when there is new RawTable client API.
         // if (mFileSystem.createRawTable(new TachyonURI(TEST_TABLE_DIR + mSuccessNum), 1) ==
         // -1) {
         // break;
         // }
       }
       // The create operation may succeed at the master side but still returns false due to the
       // shutdown. So the mSuccessNum may be less than the actual success number.
       mSuccessNum++;
       CommonUtils.sleepMs(100);
     }
   } catch (Exception e) {
     // Something crashed. Stop the thread.
   }
 }
 // TODO: Rethink the approach of this test and what it should be testing
 // @Test
 public void deleteDuringEvictionTest() throws IOException {
   // This test may not trigger eviction each time, repeat it 20 times.
   for (int i = 0; i < 20; i++) {
     deleteDuringEviction(i);
     CommonUtils.sleepMs(2 * HEARTBEAT_INTERVAL_MS); // ensure second delete completes
   }
 }
Example #13
0
 /**
  * Stops the block worker. This method should only be called to terminate the worker.
  *
  * @throws IOException if the data server fails to close
  */
 public void stop() throws IOException {
   mDataServer.close();
   mThriftServer.stop();
   mThriftServerSocket.close();
   mSessionCleanerThread.stop();
   mBlockMasterClient.close();
   if (mSpaceReserver != null) {
     mSpaceReserver.stop();
   }
   mFileSystemMasterClient.close();
   // Use shutdownNow because HeartbeatThreads never finish until they are interrupted
   getExecutorService().shutdownNow();
   mWorkerMetricsSystem.stop();
   try {
     mWebServer.shutdownWebServer();
   } catch (Exception e) {
     LOG.error("Failed to stop web server", e);
   }
   mBlockDataManager.stop();
   while (!mDataServer.isClosed() || mThriftServer.isServing()) {
     // The reason to stop and close again is due to some issues in Thrift.
     mDataServer.close();
     mThriftServer.stop();
     mThriftServerSocket.close();
     CommonUtils.sleepMs(100);
   }
 }
Example #14
0
  /**
   * Creates the local block path and all the parent directories. Also, sets the appropriate
   * permissions.
   *
   * @param path The path of the block.
   * @throws IOException
   */
  public static void createBlockPath(String path) throws IOException {
    File localFolder;
    try {
      localFolder = new File(CommonUtils.getParent(path));
    } catch (InvalidPathException e) {
      throw new IOException(e);
    }

    if (!localFolder.exists()) {
      if (localFolder.mkdirs()) {
        CommonUtils.changeLocalFileToFullPermission(localFolder.getAbsolutePath());
        LOG.info("Folder {} was created!", localFolder);
      } else {
        throw new IOException("Failed to create folder " + localFolder);
      }
    }
  }
Example #15
0
 /**
  * Notify the worker the block is cached.
  *
  * <p>This call is called remotely from {@link tachyon.client.TachyonFS#cacheBlock(long)} which is
  * only ever called from {@link tachyon.client.BlockOutStream#close()} (though its a public api so
  * anyone could call it). There are a few interesting preconditions for this to work.
  *
  * <p>1) Client process writes to files locally under a tachyon defined temp directory. 2) Worker
  * process is on the same node as the client 3) Client is talking to the local worker directly
  *
  * <p>If all conditions are true, then and only then can this method ever be called; all
  * operations work on local files.
  *
  * @param userId The user id of the client who send the notification
  * @param blockId The id of the block
  * @throws FileDoesNotExistException
  * @throws SuspectedFileSizeException
  * @throws BlockInfoException
  * @throws TException
  */
 public void cacheBlock(long userId, long blockId)
     throws FileDoesNotExistException, SuspectedFileSizeException, BlockInfoException, TException {
   File srcFile = new File(CommonUtils.concat(getUserTempFolder(userId), blockId));
   File dstFile = new File(CommonUtils.concat(mLocalDataFolder, blockId));
   long fileSizeBytes = srcFile.length();
   if (!srcFile.exists()) {
     throw new FileDoesNotExistException("File " + srcFile + " does not exist.");
   }
   if (!srcFile.renameTo(dstFile)) {
     throw new FileDoesNotExistException(
         "Failed to rename file from " + srcFile.getPath() + " to " + dstFile.getPath());
   }
   addBlockId(blockId, fileSizeBytes);
   mUsers.addOwnBytes(userId, -fileSizeBytes);
   mMasterClient.worker_cacheBlock(
       mWorkerId, mWorkerSpaceCounter.getUsedBytes(), blockId, fileSizeBytes);
   LOG.info(userId + " " + dstFile);
 }
 @Ignore
 @Test
 public void multiMasterJournalCrashIntegrationTest() throws Exception {
   LocalTachyonClusterMultiMaster cluster = setupMultiMasterCluster();
   // Kill the leader one by one.
   for (int kills = 0; kills < TEST_NUM_MASTERS; kills++) {
     CommonUtils.sleepMs(TEST_TIME_MS);
     Assert.assertTrue(cluster.killLeader());
   }
   cluster.stopTFS();
   CommonUtils.sleepMs(TEST_TIME_MS);
   // Ensure the client threads are stopped.
   mExecutorsForClient.shutdown();
   while (!mExecutorsForClient.awaitTermination(TEST_TIME_MS, TimeUnit.MILLISECONDS)) {}
   reproduceAndCheckState(mCreateFileThread.getSuccessNum());
   // clean up
   cluster.stopUFS();
 }
Example #17
0
 @Test
 public void freeTest() throws IOException, TException {
   TachyonFile file =
       TachyonFSTestUtils.createByteFile(
           mTfs, "/testFile", TachyonStorageType.STORE, UnderStorageType.NO_PERSIST, 10);
   mFsShell.run(new String[] {"free", "/testFile"});
   TachyonConf tachyonConf = mLocalTachyonCluster.getMasterTachyonConf();
   CommonUtils.sleepMs(tachyonConf.getInt(Constants.WORKER_TO_MASTER_HEARTBEAT_INTERVAL_MS));
   Assert.assertFalse(mTfs.getInfo(file).getInMemoryPercentage() == 100);
 }
Example #18
0
 /**
  * Stop the current Tachyon cluster. This is used for preparation and clean up. To crash the
  * Master, use <code>killMaster</code>.
  */
 private static void stopCluster() {
   String stopClusterCommand =
       new TachyonConf().get(Constants.TACHYON_HOME) + "/bin/tachyon-stop.sh";
   try {
     Runtime.getRuntime().exec(stopClusterCommand).waitFor();
     CommonUtils.sleepMs(LOG, 1000);
   } catch (Exception e) {
     LOG.error("Error when stop Tachyon cluster", e);
   }
 }
Example #19
0
 /**
  * Factory for {@link Evictor}.
  *
  * @param conf {@link TachyonConf} to determine the {@link Evictor} type
  * @param view {@link BlockMetadataManagerView} to pass to {@link Evictor}
  * @param allocator an allocation policy
  * @return the generated {@link Evictor}
  */
 public static Evictor create(
     TachyonConf conf, BlockMetadataManagerView view, Allocator allocator) {
   try {
     return CommonUtils.createNewClassInstance(
         conf.<Evictor>getClass(Constants.WORKER_EVICTOR_CLASS),
         new Class[] {BlockMetadataManagerView.class, Allocator.class},
         new Object[] {view, allocator});
   } catch (Exception e) {
     throw Throwables.propagate(e);
   }
 }
Example #20
0
 /**
  * Create a data server with direct access to worker storage.
  *
  * @param address The address of the data server.
  * @param workerStorage The handler of the directly accessed worker storage.
  */
 public DataServer(InetSocketAddress address, WorkerStorage workerStorage) {
   LOG.info("Starting DataServer @ " + address);
   mAddress = address;
   mBlocksLocker = new BlocksLocker(workerStorage, Users.sDATASERVER_USER_ID);
   try {
     mSelector = initSelector();
   } catch (IOException e) {
     LOG.error(e.getMessage() + mAddress, e);
     CommonUtils.runtimeException(e);
   }
 }
Example #21
0
 /**
  * Add the checkpoint information of a file. The information is from the user <code>userId</code>.
  *
  * <p>This method is normally triggered from {@link tachyon.client.FileOutStream#close()} if and
  * only if {@link tachyon.client.WriteType#isThrough()} is true. The current implementation of
  * checkpointing is that through {@link tachyon.client.WriteType} operations write to {@link
  * tachyon.UnderFileSystem} on the client's write path, but under a user temp directory (temp
  * directory is defined in the worker as {@link #getUserUnderfsTempFolder(long)}).
  *
  * @param userId The user id of the client who send the notification
  * @param fileId The id of the checkpointed file
  * @throws FileDoesNotExistException
  * @throws SuspectedFileSizeException
  * @throws FailedToCheckpointException
  * @throws BlockInfoException
  * @throws TException
  */
 public void addCheckpoint(long userId, int fileId)
     throws FileDoesNotExistException, SuspectedFileSizeException, FailedToCheckpointException,
         BlockInfoException, TException {
   // TODO This part need to be changed.
   String srcPath = CommonUtils.concat(getUserUnderfsTempFolder(userId), fileId);
   String dstPath = CommonUtils.concat(COMMON_CONF.UNDERFS_DATA_FOLDER, fileId);
   try {
     if (!mUnderFs.rename(srcPath, dstPath)) {
       throw new FailedToCheckpointException("Failed to rename " + srcPath + " to " + dstPath);
     }
   } catch (IOException e) {
     throw new FailedToCheckpointException("Failed to rename " + srcPath + " to " + dstPath);
   }
   long fileSize;
   try {
     fileSize = mUnderFs.getFileSize(dstPath);
   } catch (IOException e) {
     throw new FailedToCheckpointException("Failed to getFileSize " + dstPath);
   }
   mMasterClient.addCheckpoint(mWorkerId, fileId, fileSize, dstPath);
 }
Example #22
0
 /** @return Generated {@link WorkerInfo} for this worker */
 public synchronized WorkerInfo generateClientWorkerInfo() {
   WorkerInfo ret = new WorkerInfo();
   ret.id = mId;
   ret.address = mWorkerAddress;
   ret.lastContactSec =
       (int) ((CommonUtils.getCurrentMs() - mLastUpdatedTimeMs) / Constants.SECOND_MS);
   ret.state = "In Service";
   ret.capacityBytes = mCapacityBytes;
   ret.usedBytes = mUsedBytes;
   ret.startTimeMs = mStartTimeMs;
   return ret;
 }
  // Tests that pinning a file and then unpinning
  @Test
  public void unpinFileTest() throws Exception {
    // Create a file that fills the entire Tachyon store
    TachyonFile file1 =
        TachyonFSTestUtils.createByteFile(
            mTFS,
            "/test1",
            TachyonStorageType.STORE,
            UnderStorageType.NO_PERSIST,
            MEM_CAPACITY_BYTES);

    // Pin the file
    mTFS.setState(file1, mSetPinned);
    CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);

    // Confirm the pin with master
    Assert.assertTrue(mTFS.getInfo(file1).isIsPinned());

    // Unpin the file
    mTFS.setState(file1, mSetUnpinned);
    CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);

    // Confirm the unpin
    Assert.assertFalse(mTFS.getInfo(file1).isIsPinned());

    // Try to create a file that cannot be stored unless the previous file is evicted, this
    // should succeed
    TachyonFile file2 =
        TachyonFSTestUtils.createByteFile(
            mTFS,
            "/test2",
            TachyonStorageType.STORE,
            UnderStorageType.NO_PERSIST,
            MEM_CAPACITY_BYTES);

    // File 2 should be in memory and File 1 should be evicted
    CommonUtils.sleepMs(LOG, mWorkerToMasterHeartbeatIntervalMs * 3);
    Assert.assertFalse(mTFS.getInfo(file1).getInMemoryPercentage() == 100);
    Assert.assertTrue(mTFS.getInfo(file2).getInMemoryPercentage() == 100);
  }
Example #24
0
  BlockOutStream(TachyonFile file, WriteType opType, int blockIndex) throws IOException {
    super(file, opType);

    if (!opType.isCache()) {
      throw new IOException("BlockOutStream only support WriteType.CACHE");
    }

    BLOCK_INDEX = blockIndex;
    BLOCK_CAPACITY_BYTE = FILE.getBlockSizeByte();
    BLOCK_ID = FILE.getBlockId(BLOCK_INDEX);
    BLOCK_OFFSET = BLOCK_CAPACITY_BYTE * blockIndex;
    PIN = FILE.needPin();

    mCanWrite = true;

    if (!TFS.hasLocalWorker()) {
      mCanWrite = false;
      String msg = "The machine does not have any local worker.";
      throw new IOException(msg);
    }

    File localFolder = TFS.createAndGetUserTempFolder();
    if (localFolder == null) {
      mCanWrite = false;
      String msg = "Failed to create temp user folder for tachyon client.";
      throw new IOException(msg);
    }

    mLocalFilePath = CommonUtils.concat(localFolder.getPath(), BLOCK_ID);
    mLocalFile = new RandomAccessFile(mLocalFilePath, "rw");
    mLocalFileChannel = mLocalFile.getChannel();
    // change the permission of the temporary file in order that the worker can move it.
    CommonUtils.changeLocalFileToFullPermission(mLocalFilePath);
    // use the sticky bit, only the client and the worker can write to the block
    CommonUtils.setLocalFileStickyBit(mLocalFilePath);
    LOG.info(mLocalFilePath + " was created!");

    mBuffer = ByteBuffer.allocate(USER_CONF.FILE_BUFFER_BYTES + 4);
  }
Example #25
0
 /** Register this TachyonWorker to the TachyonMaster */
 public void register() {
   long id = 0;
   while (id == 0) {
     try {
       id =
           mMasterClient.worker_register(
               mWorkerAddress,
               mSpaceCounter.getCapacityBytes(),
               mSpaceCounter.getUsedBytes(),
               new ArrayList<Long>(mMemoryData));
     } catch (BlockInfoException e) {
       LOG.error(e.getMessage(), e);
       id = 0;
       CommonUtils.sleepMs(LOG, Constants.SECOND_MS);
     } catch (IOException e) {
       LOG.error(e.getMessage(), e);
       id = 0;
       CommonUtils.sleepMs(LOG, Constants.SECOND_MS);
     }
   }
   mWorkerId = id;
 }
Example #26
0
  public static void createDependency() throws IOException {
    long startTimeMs = CommonUtils.getCurrentMs();
    List<String> children = new ArrayList<String>();
    for (int k = 0; k < sFiles; k++) {
      children.add(sFileFolder + "/part-" + k);
    }
    List<ByteBuffer> data = new ArrayList<ByteBuffer>();
    data.add(ByteBuffer.allocate(10));
    int depId =
        sTachyonClient.createDependency(
            new ArrayList<String>(),
            children,
            "fake command",
            data,
            "BasicCheckpoint Dependency",
            "Tachyon Examples",
            "0.3",
            DependencyType.Narrow.getValue(),
            512 * Constants.MB);

    CommonUtils.printTimeTakenMs(startTimeMs, LOG, "createDependency with depId " + depId);
  }
Example #27
0
 /**
  * Keep requesting to Master until something crashes or fail to create. Record how many
  * operations are performed successfully.
  */
 @Override
 public void run() {
   // This infinity loop will be broken when the master is crashed and the client needs to stop.
   while (true) {
     synchronized (this) {
       if (mIsStopped) {
         break;
       }
     }
     try {
       TachyonURI testURI = new TachyonURI(mWorkDir + mSuccessNum);
       if (ClientOpType.CREATE_FILE == mOpType) {
         sTfs.getOutStream(testURI, sOutStreamOptions).close();
       } else if (ClientOpType.CREATE_DELETE_FILE == mOpType) {
         try {
           sTfs.getOutStream(testURI, sOutStreamOptions).close();
         } catch (TachyonException e) {
           // If file already exists, ignore it.
           if (e.getType() != TachyonExceptionType.FILE_ALREADY_EXISTS) {
             throw e;
           }
         } catch (Exception e) {
           throw e;
         }
         sTfs.delete(sTfs.open(testURI));
       } else if (ClientOpType.CREATE_RENAME_FILE == mOpType) {
         try {
           sTfs.getOutStream(testURI, sOutStreamOptions).close();
         } catch (TachyonException e) {
           // If file already exists, ignore it.
           if (e.getType() != TachyonExceptionType.FILE_ALREADY_EXISTS) {
             throw e;
           }
         } catch (Exception e) {
           throw e;
         }
         sTfs.rename(sTfs.open(testURI), new TachyonURI(testURI + "-rename"));
       } else if (ClientOpType.CREATE_TABLE == mOpType) {
         if (sOldTfs.createRawTable(new TachyonURI(mWorkDir + mSuccessNum), 1) == -1) {
           break;
         }
       }
     } catch (Exception e) {
       // Since master may crash/restart for several times, so this exception is expected.
       // Ignore the exception and still keep requesting to master.
       continue;
     }
     mSuccessNum++;
     CommonUtils.sleepMs(100);
   }
 }
Example #28
0
 @Override
 public void run() {
   long lastCheckMs = System.currentTimeMillis();
   while (mRunning) {
     // Check the time since last check, and wait until it is within check interval
     long lastIntervalMs = System.currentTimeMillis() - lastCheckMs;
     long toSleepMs = mCheckIntervalMs - lastIntervalMs;
     if (toSleepMs > 0) {
       CommonUtils.sleepMs(LOG, toSleepMs);
     } else {
       LOG.warn("Space reserver took: {}, expected: {}", lastIntervalMs, mCheckIntervalMs);
     }
     reserveSpace();
   }
 }
Example #29
0
 /**
  * Stop this TachyonWorker. Stop all the threads belong to this TachyonWorker.
  *
  * @throws IOException
  * @throws InterruptedException
  */
 public void stop() throws IOException, InterruptedException {
   mStop = true;
   mWorkerStorage.stop();
   mDataServer.close();
   mServer.stop();
   mServerTNonblockingServerSocket.close();
   mExecutorService.shutdown();
   while (!mDataServer.isClosed() || mServer.isServing() || mHeartbeatThread.isAlive()) {
     // TODO The reason to stop and close again is due to some issues in Thrift.
     mServer.stop();
     mServerTNonblockingServerSocket.close();
     CommonUtils.sleepMs(null, 100);
   }
   mHeartbeatThread.join();
 }
Example #30
0
  private ByteBuffer readRemoteByteBuffer(ClientBlockInfo blockInfo, long offset, long len) {
    ByteBuffer buf = null;

    try {
      List<NetAddress> blockLocations = blockInfo.getLocations();
      LOG.info("Block locations:" + blockLocations);

      for (int k = 0; k < blockLocations.size(); k++) {
        String host = blockLocations.get(k).mHost;
        int port = blockLocations.get(k).mSecondaryPort;

        // The data is not in remote machine's memory if port == -1.
        if (port == -1) {
          continue;
        }
        if (host.equals(InetAddress.getLocalHost().getHostName())
            || host.equals(InetAddress.getLocalHost().getHostAddress())) {
          String localFileName = CommonUtils.concat(TFS.getRootFolder(), blockInfo.blockId);
          LOG.warn("Master thinks the local machine has data " + localFileName + "! But not!");
        }
        LOG.info(
            host
                + ":"
                + port
                + " current host is "
                + InetAddress.getLocalHost().getHostName()
                + " "
                + InetAddress.getLocalHost().getHostAddress());

        try {
          buf =
              retrieveByteBufferFromRemoteMachine(
                  new InetSocketAddress(host, port), blockInfo.blockId, offset, len);
          if (buf != null) {
            break;
          }
        } catch (IOException e) {
          LOG.error(e.getMessage());
          buf = null;
        }
      }
    } catch (IOException e) {
      LOG.error("Failed to get read data from remote " + e.getMessage());
      buf = null;
    }

    return buf;
  }