Exemplo n.º 1
0
  /**
   * Creates a new local tachyon master with a isolated home and port.
   *
   * @throws IOException when unable to do file operation or listen on port
   * @return an instance of Tachyon master
   */
  public static LocalTachyonMaster create() throws IOException {
    final String tachyonHome = uniquePath();
    TachyonConf tachyonConf = MasterContext.getConf();
    UnderFileSystemUtils.deleteDir(tachyonHome, tachyonConf);
    UnderFileSystemUtils.mkdirIfNotExists(tachyonHome, tachyonConf);

    // Update Tachyon home in the passed TachyonConf instance.
    tachyonConf.set(Constants.TACHYON_HOME, tachyonHome);

    return new LocalTachyonMaster(tachyonHome);
  }
Exemplo n.º 2
0
  private LocalTachyonMaster(final String tachyonHome) throws IOException {
    mTachyonHome = tachyonHome;

    TachyonConf tachyonConf = MasterContext.getConf();
    mHostname = NetworkAddressUtils.getConnectHost(ServiceType.MASTER_RPC, tachyonConf);

    // To start the UFS either for integration or unit test. If it targets the unit test, UFS is
    // setup over the local file system (see also {@link LocalFilesystemCluster} - under folder of
    // "mTachyonHome/tachyon*". Otherwise, it starts some distributed file system cluster e.g.,
    // miniDFSCluster (see also {@link tachyon.LocalMiniDFScluster} and setup the folder like
    // "hdfs://xxx:xxx/tachyon*".
    mUnderFSCluster = UnderFileSystemCluster.get(mTachyonHome + "/dfs", tachyonConf);
    mUnderFSFolder = mUnderFSCluster.getUnderFilesystemAddress() + "/tachyon_underfs_folder";
    // To setup the journalFolder under either local file system or distributed ufs like
    // miniDFSCluster
    mJournalFolder = mUnderFSCluster.getUnderFilesystemAddress() + "/journal";

    UnderFileSystemUtils.mkdirIfNotExists(mJournalFolder, tachyonConf);
    String[] masterServiceNames =
        new String[] {
          Constants.BLOCK_MASTER_SERVICE_NAME,
          Constants.FILE_SYSTEM_MASTER_SERVICE_NAME,
          Constants.RAW_TABLE_MASTER_SERVICE_NAME,
        };
    for (String masterServiceName : masterServiceNames) {
      UnderFileSystemUtils.mkdirIfNotExists(
          PathUtils.concatPath(mJournalFolder, masterServiceName), tachyonConf);
    }
    UnderFileSystemUtils.touch(
        mJournalFolder + "/_format_" + System.currentTimeMillis(), tachyonConf);

    tachyonConf.set(Constants.MASTER_JOURNAL_FOLDER, mJournalFolder);
    tachyonConf.set(Constants.UNDERFS_ADDRESS, mUnderFSFolder);

    tachyonConf.set(Constants.MASTER_MIN_WORKER_THREADS, "1");
    tachyonConf.set(Constants.MASTER_MAX_WORKER_THREADS, "100");

    // If tests fail to connect they should fail early rather than using the default ridiculously
    // high retries
    tachyonConf.set(Constants.MASTER_RETRY_COUNT, "3");

    // Since tests are always running on a single host keep the resolution timeout low as otherwise
    // people running with strange network configurations will see very slow tests
    tachyonConf.set(Constants.HOST_RESOLUTION_TIMEOUT_MS, "250");

    tachyonConf.set(Constants.WEB_THREAD_COUNT, "1");
    tachyonConf.set(
        Constants.WEB_RESOURCES,
        PathUtils.concatPath(System.getProperty("user.dir"), "../servers/src/main/webapp"));

    mTachyonMaster = TachyonMaster.Factory.createMaster();

    // Reset the master port
    tachyonConf.set(Constants.MASTER_PORT, Integer.toString(getRPCLocalPort()));

    Runnable runMaster =
        new Runnable() {
          @Override
          public void run() {
            try {
              mTachyonMaster.start();
            } catch (Exception e) {
              throw new RuntimeException(e + " \n Start Master Error \n" + e.getMessage(), e);
            }
          }
        };

    mMasterThread = new Thread(runMaster);
  }
Exemplo n.º 3
0
  /**
   * Creates a new local tachyon master with a isolated port.
   *
   * @param tachyonHome Tachyon home directory, if the directory already exists, this method will
   *     reuse any directory/file if possible, no deletion will be made
   * @return an instance of Tachyon master
   * @throws IOException when unable to do file operation or listen on port
   */
  public static LocalTachyonMaster create(final String tachyonHome) throws IOException {
    TachyonConf tachyonConf = MasterContext.getConf();
    UnderFileSystemUtils.mkdirIfNotExists(tachyonHome, tachyonConf);

    return new LocalTachyonMaster(Preconditions.checkNotNull(tachyonHome));
  }