Ejemplo n.º 1
0
  public S3UnderFileSystem(String bucketName, TachyonConf tachyonConf) throws ServiceException {
    super(tachyonConf);
    Preconditions.checkArgument(
        tachyonConf.containsKey(Constants.S3_ACCESS_KEY),
        "Property " + Constants.S3_ACCESS_KEY + " is required to connect to S3");
    Preconditions.checkArgument(
        tachyonConf.containsKey(Constants.S3_SECRET_KEY),
        "Property " + Constants.S3_SECRET_KEY + " is required to connect to S3");
    AWSCredentials awsCredentials =
        new AWSCredentials(
            tachyonConf.get(Constants.S3_ACCESS_KEY), tachyonConf.get(Constants.S3_SECRET_KEY));
    mBucketName = bucketName;

    Jets3tProperties props = new Jets3tProperties();
    if (tachyonConf.containsKey(Constants.UNDERFS_S3_PROXY_HOST)) {
      props.setProperty("httpclient.proxy-autodetect", "false");
      props.setProperty("httpclient.proxy-host", tachyonConf.get(Constants.UNDERFS_S3_PROXY_HOST));
      props.setProperty("httpclient.proxy-port", tachyonConf.get(Constants.UNDERFS_S3_PROXY_PORT));
    }
    if (tachyonConf.containsKey(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY)) {
      props.setProperty(
          "s3service.https-only",
          Boolean.toString(tachyonConf.getBoolean(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY)));
    }
    LOG.debug("Initializing S3 underFs with properties: " + props.getProperties());
    mClient = new RestS3Service(awsCredentials, null, null, props);
    mBucketPrefix = Constants.HEADER_S3N + mBucketName + PATH_SEPARATOR;
  }
 /** Sets the volume and the mount directory before a test runs. */
 @Before
 public final void before() {
   mTachyonConf = new TachyonConf();
   if (mTachyonConf.containsKey(Constants.UNDERFS_GLUSTERFS_MR_DIR)) {
     mMount = mTachyonConf.get(Constants.UNDERFS_GLUSTERFS_MR_DIR);
   }
   if (mTachyonConf.containsKey(Constants.UNDERFS_GLUSTERFS_VOLUMES)) {
     mVolume = mTachyonConf.get(Constants.UNDERFS_GLUSTERFS_VOLUMES);
   }
 }
Ejemplo n.º 3
0
  @Override
  public void connectFromWorker(TachyonConf conf, String host) throws IOException {
    String workerKeytab = conf.get(Constants.WORKER_KEYTAB_KEY, null);
    String workerPrincipal = conf.get(Constants.WORKER_PRINCIPAL_KEY, null);
    if (workerKeytab == null || workerPrincipal == null) {
      return;
    }

    login(
        Constants.WORKER_KEYTAB_KEY,
        workerKeytab,
        Constants.WORKER_PRINCIPAL_KEY,
        workerPrincipal,
        host);
  }
Ejemplo n.º 4
0
 public S3UnderFileSystem(String bucketName, TachyonConf tachyonConf) throws ServiceException {
   super(tachyonConf);
   Preconditions.checkArgument(
       tachyonConf.containsKey(Constants.S3_ACCESS_KEY),
       "Property " + Constants.S3_ACCESS_KEY + " is required to connect to S3");
   Preconditions.checkArgument(
       tachyonConf.containsKey(Constants.S3_SECRET_KEY),
       "Property " + Constants.S3_SECRET_KEY + " is required to connect to S3");
   AWSCredentials awsCredentials =
       new AWSCredentials(
           tachyonConf.get(Constants.S3_ACCESS_KEY), tachyonConf.get(Constants.S3_SECRET_KEY));
   mBucketName = bucketName;
   mClient = new RestS3Service(awsCredentials);
   mBucketPrefix = Constants.HEADER_S3N + mBucketName + PATH_SEPARATOR;
 }
 /**
  * This method adds Aliyun credentials from system properties to the Tachyon Conf if they are not
  * already present.
  *
  * @param tachyonConf the conf to check and add credentials to
  * @return true if both access and secret key are present, false otherwise
  */
 private boolean addAndCheckOSSCredentials(TachyonConf tachyonConf) {
   String accessKeyConf = Constants.OSS_ACCESS_KEY;
   if (System.getProperty(accessKeyConf) != null && tachyonConf.get(accessKeyConf) == null) {
     tachyonConf.set(accessKeyConf, System.getProperty(accessKeyConf));
   }
   String secretKeyConf = Constants.OSS_SECRET_KEY;
   if (System.getProperty(secretKeyConf) != null && tachyonConf.get(secretKeyConf) == null) {
     tachyonConf.set(secretKeyConf, System.getProperty(secretKeyConf));
   }
   String endPointConf = Constants.OSS_ENDPOINT_KEY;
   if (System.getProperty(endPointConf) != null && tachyonConf.get(endPointConf) == null) {
     tachyonConf.set(endPointConf, System.getProperty(endPointConf));
   }
   return tachyonConf.get(accessKeyConf) != null
       && tachyonConf.get(secretKeyConf) != null
       && tachyonConf.get(endPointConf) != null;
 }
Ejemplo n.º 6
0
 /**
  * Creates a <code>MetricsSystem</code> using the default metrics config.
  *
  * @param instance the instance name
  * @param tachyonConf the {@link TachyonConf} instance for configuration properties.
  */
 public MetricsSystem(String instance, TachyonConf tachyonConf) {
   mInstance = instance;
   mTachyonConf = tachyonConf;
   String metricsConfFile = null;
   if (mTachyonConf.containsKey(Constants.METRICS_CONF_FILE)) {
     metricsConfFile = mTachyonConf.get(Constants.METRICS_CONF_FILE);
   }
   mMetricsConfig = new MetricsConfig(metricsConfFile);
 }
  public void start() throws IOException {
    int maxLevel = 1;
    mTachyonHome =
        File.createTempFile("Tachyon", "U" + System.currentTimeMillis()).getAbsolutePath();
    mWorkerDataFolder = "/datastore";

    mHostname = NetworkAddressUtils.getLocalHostName(100);

    mMasterConf = MasterContext.getConf();
    mMasterConf.set(Constants.IN_TEST_MODE, "true");
    mMasterConf.set(Constants.TACHYON_HOME, mTachyonHome);
    mMasterConf.set(Constants.USE_ZOOKEEPER, "true");
    mMasterConf.set(Constants.MASTER_HOSTNAME, mHostname);
    mMasterConf.set(Constants.MASTER_BIND_HOST, mHostname);
    mMasterConf.set(Constants.MASTER_PORT, "0");
    mMasterConf.set(Constants.MASTER_WEB_BIND_HOST, mHostname);
    mMasterConf.set(Constants.MASTER_WEB_PORT, "0");
    mMasterConf.set(Constants.ZOOKEEPER_ADDRESS, mCuratorServer.getConnectString());
    mMasterConf.set(Constants.ZOOKEEPER_ELECTION_PATH, "/election");
    mMasterConf.set(Constants.ZOOKEEPER_LEADER_PATH, "/leader");
    mMasterConf.set(Constants.USER_QUOTA_UNIT_BYTES, "10000");
    mMasterConf.set(Constants.USER_DEFAULT_BLOCK_SIZE_BYTE, Integer.toString(mUserBlockSize));

    // Since tests are always running on a single host keep the resolution timeout low as otherwise
    // people running with strange network configurations will see very slow tests
    mMasterConf.set(Constants.HOST_RESOLUTION_TIMEOUT_MS, "250");

    // Disable hdfs client caching to avoid file system close() affecting other clients
    System.setProperty("fs.hdfs.impl.disable.cache", "true");

    // re-build the dir to set permission to 777
    deleteDir(mTachyonHome);
    mkdir(mTachyonHome);

    for (int k = 0; k < mNumOfMasters; k++) {
      final LocalTachyonMaster master = LocalTachyonMaster.create(mTachyonHome);
      master.start();
      LOG.info(
          "master NO."
              + k
              + " started, isServing: "
              + master.isServing()
              + ", address: "
              + master.getAddress());
      mMasters.add(master);
      // Each master should generate a new port for binding
      mMasterConf.set(Constants.MASTER_PORT, "0");
    }

    // Create the directories for the data and workers after LocalTachyonMaster construction,
    // because LocalTachyonMaster sets the UNDERFS_DATA_FOLDER and UNDERFS_WORKERS_FOLDER.
    mkdir(mMasterConf.get(Constants.UNDERFS_DATA_FOLDER));
    mkdir(mMasterConf.get(Constants.UNDERFS_WORKERS_FOLDER));

    LOG.info("all " + mNumOfMasters + " masters started.");
    LOG.info("waiting for a leader.");
    boolean hasLeader = false;
    while (!hasLeader) {
      for (int i = 0; i < mMasters.size(); i++) {
        if (mMasters.get(i).isServing()) {
          LOG.info(
              "master NO."
                  + i
                  + " is selected as leader. address: "
                  + mMasters.get(i).getAddress());
          hasLeader = true;
          break;
        }
      }
    }
    // Use first master port
    mMasterConf.set(Constants.MASTER_PORT, getMasterPort() + "");

    CommonUtils.sleepMs(10);

    mWorkerConf = WorkerContext.getConf();
    mWorkerConf.merge(mMasterConf);
    mWorkerConf.set(Constants.WORKER_DATA_FOLDER, mWorkerDataFolder);
    mWorkerConf.set(Constants.WORKER_MEMORY_SIZE, mWorkerCapacityBytes + "");
    mWorkerConf.set(Constants.WORKER_TO_MASTER_HEARTBEAT_INTERVAL_MS, 15 + "");

    // Setup conf for worker
    mWorkerConf.set(Constants.WORKER_MAX_TIERED_STORAGE_LEVEL, Integer.toString(maxLevel));
    mWorkerConf.set(String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_ALIAS_FORMAT, 0), "MEM");
    mWorkerConf.set(
        String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_PATH_FORMAT, 0),
        mTachyonHome + "/ramdisk");
    mWorkerConf.set(
        String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_QUOTA_FORMAT, 0),
        mWorkerCapacityBytes + "");

    // Since tests are always running on a single host keep the resolution timeout low as otherwise
    // people running with strange network configurations will see very slow tests
    mWorkerConf.set(Constants.HOST_RESOLUTION_TIMEOUT_MS, "250");

    for (int level = 1; level < maxLevel; level++) {
      String tierLevelDirPath =
          String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_PATH_FORMAT, level);
      String[] dirPaths = mWorkerConf.get(tierLevelDirPath).split(",");
      String newPath = "";
      for (String dirPath : dirPaths) {
        newPath += mTachyonHome + dirPath + ",";
      }
      mWorkerConf.set(
          String.format(Constants.WORKER_TIERED_STORAGE_LEVEL_DIRS_PATH_FORMAT, level),
          newPath.substring(0, newPath.length() - 1));
    }

    mWorkerConf.set(Constants.WORKER_BIND_HOST, mHostname);
    mWorkerConf.set(Constants.WORKER_PORT, "0");
    mWorkerConf.set(Constants.WORKER_DATA_BIND_HOST, mHostname);
    mWorkerConf.set(Constants.WORKER_DATA_PORT, "0");
    mWorkerConf.set(Constants.WORKER_WEB_BIND_HOST, mHostname);
    mWorkerConf.set(Constants.WORKER_WEB_PORT, "0");
    mWorkerConf.set(Constants.WORKER_MIN_WORKER_THREADS, "1");
    mWorkerConf.set(Constants.WORKER_MAX_WORKER_THREADS, "100");

    // Perform immediate shutdown of data server. Graceful shutdown is unnecessary and slow
    mWorkerConf.set(Constants.WORKER_NETWORK_NETTY_SHUTDOWN_QUIET_PERIOD, Integer.toString(0));
    mWorkerConf.set(Constants.WORKER_NETWORK_NETTY_SHUTDOWN_TIMEOUT, Integer.toString(0));

    mWorker = new BlockWorker();
    Runnable runWorker =
        new Runnable() {
          @Override
          public void run() {
            try {
              mWorker.process();
            } catch (Exception e) {
              throw new RuntimeException(e + " \n Start Master Error \n" + e.getMessage(), e);
            }
          }
        };
    mWorkerThread = new Thread(runWorker);
    mWorkerThread.start();
    // The client context should reflect the updates to the conf.
    if (sReinitializer == null) {
      ClientContext.accessReinitializer(sReinitializerAccesser);
    }
    sReinitializer.reinitializeWithConf(mWorkerConf);
  }