コード例 #1
0
  /**
   * Constructs a new instance of {@link S3UnderFileSystem}.
   *
   * @param uri the {@link AlluxioURI} for this UFS
   * @param conf the configuration for Alluxio
   * @param awsCredentials AWS Credentials configuration for S3 Access
   * @throws ServiceException when a connection to S3 could not be created
   */
  public S3UnderFileSystem(AlluxioURI uri, Configuration conf, AWSCredentials awsCredentials)
      throws ServiceException {
    super(uri, conf);
    String bucketName = uri.getHost();
    mBucketName = bucketName;

    Jets3tProperties props = new Jets3tProperties();
    if (conf.containsKey(Constants.UNDERFS_S3_PROXY_HOST)) {
      props.setProperty("httpclient.proxy-autodetect", "false");
      props.setProperty("httpclient.proxy-host", conf.get(Constants.UNDERFS_S3_PROXY_HOST));
      props.setProperty("httpclient.proxy-port", conf.get(Constants.UNDERFS_S3_PROXY_PORT));
    }
    if (conf.containsKey(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY)) {
      props.setProperty(
          "s3service.https-only",
          Boolean.toString(conf.getBoolean(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY)));
    }
    if (conf.containsKey(Constants.UNDERFS_S3_ENDPOINT)) {
      props.setProperty("s3service.s3-endpoint", conf.get(Constants.UNDERFS_S3_ENDPOINT));
      if (conf.getBoolean(Constants.UNDERFS_S3_PROXY_HTTPS_ONLY)) {
        props.setProperty(
            "s3service.s3-endpoint-https-port", conf.get(Constants.UNDERFS_S3_ENDPOINT_HTTPS_PORT));
      } else {
        props.setProperty(
            "s3service.s3-endpoint-http-port", conf.get(Constants.UNDERFS_S3_ENDPOINT_HTTP_PORT));
      }
    }
    if (conf.containsKey(Constants.UNDERFS_S3_DISABLE_DNS_BUCKETS)) {
      props.setProperty(
          "s3service.disable-dns-buckets", conf.get(Constants.UNDERFS_S3_DISABLE_DNS_BUCKETS));
    }
    LOG.debug("Initializing S3 underFs with properties: {}", props.getProperties());
    mClient = new RestS3Service(awsCredentials, null, null, props);
    mBucketPrefix = PathUtils.normalizePath(Constants.HEADER_S3N + mBucketName, PATH_SEPARATOR);
  }
コード例 #2
0
  /** Tests the default properties for the worker. */
  @Test
  public void workerDefaultTest() {
    String value = sDefaultConfiguration.get(Constants.WORKER_DATA_FOLDER);
    Assert.assertNotNull(value);
    Assert.assertEquals("/alluxioworker/", value);

    value = sDefaultConfiguration.get(Constants.WORKER_BIND_HOST);
    Assert.assertNotNull(value);
    Assert.assertEquals(NetworkAddressUtils.WILDCARD_ADDRESS, value);

    int intValue = sDefaultConfiguration.getInt(Constants.WORKER_RPC_PORT);
    Assert.assertEquals(29998, intValue);

    value = sDefaultConfiguration.get(Constants.WORKER_DATA_BIND_HOST);
    Assert.assertNotNull(value);
    Assert.assertEquals(NetworkAddressUtils.WILDCARD_ADDRESS, value);

    intValue = sDefaultConfiguration.getInt(Constants.WORKER_DATA_PORT);
    Assert.assertEquals(29999, intValue);

    value = sDefaultConfiguration.get(Constants.WORKER_WEB_BIND_HOST);
    Assert.assertNotNull(value);
    Assert.assertEquals(NetworkAddressUtils.WILDCARD_ADDRESS, value);

    intValue = sDefaultConfiguration.getInt(Constants.WORKER_WEB_PORT);
    Assert.assertEquals(30000, intValue);

    intValue = sDefaultConfiguration.getInt(Constants.WORKER_BLOCK_HEARTBEAT_TIMEOUT_MS);
    Assert.assertEquals(10 * Constants.SECOND_MS, intValue);

    intValue = sDefaultConfiguration.getInt(Constants.WORKER_BLOCK_HEARTBEAT_INTERVAL_MS);
    Assert.assertEquals(Constants.SECOND_MS, intValue);

    intValue = sDefaultConfiguration.getInt(Constants.WORKER_WORKER_BLOCK_THREADS_MIN);
    Assert.assertEquals(1, intValue);

    intValue = sDefaultConfiguration.getInt(Constants.WORKER_SESSION_TIMEOUT_MS);
    Assert.assertEquals(10 * Constants.SECOND_MS, intValue);

    intValue = sDefaultConfiguration.getInt(Constants.WORKER_NETWORK_NETTY_BOSS_THREADS);
    Assert.assertEquals(1, intValue);

    intValue = sDefaultConfiguration.getInt(Constants.WORKER_NETWORK_NETTY_WORKER_THREADS);
    Assert.assertEquals(0, intValue);

    long longValue = sDefaultConfiguration.getBytes(Constants.WORKER_MEMORY_SIZE);
    Assert.assertEquals(128 * Constants.MB, longValue);
  }
コード例 #3
0
ファイル: AppLoginModule.java プロジェクト: yhua/alluxio
 /** @param conf the configuration for Alluxio */
 public AppCallbackHandler(Configuration conf) {
   if (conf.containsKey(Constants.SECURITY_LOGIN_USERNAME)) {
     mUserName = conf.get(Constants.SECURITY_LOGIN_USERNAME);
   } else {
     mUserName = "";
   }
 }
コード例 #4
0
  private LocalAlluxioMaster() throws IOException {
    mHostname = NetworkAddressUtils.getConnectHost(ServiceType.MASTER_RPC);

    mJournalFolder = Configuration.get(Constants.MASTER_JOURNAL_FOLDER);

    mAlluxioMaster = AlluxioMaster.Factory.create();
    Whitebox.setInternalState(AlluxioMaster.class, "sAlluxioMaster", mAlluxioMaster);

    // Reset the master port
    Configuration.set(Constants.MASTER_RPC_PORT, Integer.toString(getRPCLocalPort()));

    Runnable runMaster =
        new Runnable() {
          @Override
          public void run() {
            try {
              mAlluxioMaster.start();
            } catch (Exception e) {
              throw new RuntimeException(e + " \n Start Master Error \n" + e.getMessage(), e);
            }
          }
        };

    mMasterThread = new Thread(runMaster);
  }
コード例 #5
0
  /**
   * Creates a new instance of {@link FileSystemWorker}.
   *
   * @param blockWorker the block worker handle
   * @param workerId a reference to the id of this worker
   * @throws IOException if an I/O error occurs
   */
  public DefaultFileSystemWorker(BlockWorker blockWorker, AtomicReference<Long> workerId)
      throws IOException {
    super(
        Executors.newFixedThreadPool(
            3, ThreadFactoryUtils.build("file-system-worker-heartbeat-%d", true)));
    mWorkerId = workerId;
    mSessions = new Sessions();
    UnderFileSystem ufs = UnderFileSystem.get(Configuration.get(PropertyKey.UNDERFS_ADDRESS));
    mFileDataManager =
        new FileDataManager(
            Preconditions.checkNotNull(blockWorker),
            ufs,
            RateLimiter.create(Configuration.getBytes(PropertyKey.WORKER_FILE_PERSIST_RATE_LIMIT)));
    mUnderFileSystemManager = new UnderFileSystemManager();

    // Setup AbstractMasterClient
    mFileSystemMasterWorkerClient =
        new FileSystemMasterClient(NetworkAddressUtils.getConnectAddress(ServiceType.MASTER_RPC));

    // Setup session cleaner
    mSessionCleaner =
        new SessionCleaner(
            new SessionCleanupCallback() {
              /** Cleans up after sessions, to prevent zombie sessions holding ufs resources. */
              @Override
              public void cleanupSessions() {
                for (long session : mSessions.getTimedOutSessions()) {
                  mSessions.removeSession(session);
                  mUnderFileSystemManager.cleanupSession(session);
                }
              }
            });

    mServiceHandler = new FileSystemWorkerClientServiceHandler(this);
  }
コード例 #6
0
ファイル: MetricsSystem.java プロジェクト: sundapeng/tachyon
 /**
  * Starts sinks specified in the configuration. This is an no-op if the sinks have already been
  * started. Note: This has to be called after Alluxio configuration is initialized.
  */
 public static void startSinks() {
   String metricsConfFile = Configuration.get(PropertyKey.METRICS_CONF_FILE);
   if (metricsConfFile.isEmpty()) {
     LOG.info("Metrics is not enabled.");
     return;
   }
   MetricsConfig config = new MetricsConfig(metricsConfFile);
   startSinksFromConfig(config);
 }
コード例 #7
0
 private static List<CommandInfo.URI> getExecutorDependencyURIList() {
   List<URI> dependencies = new ArrayList<>();
   if (installJavaFromUrl()) {
     dependencies.add(
         CommandInfo.URI
             .newBuilder()
             .setValue(Configuration.get(PropertyKey.INTEGRATION_MESOS_JDK_URL))
             .setExtract(true)
             .build());
   }
   if (installAlluxioFromUrl()) {
     dependencies.add(
         CommandInfo.URI
             .newBuilder()
             .setValue(Configuration.get(PropertyKey.INTEGRATION_MESOS_ALLUXIO_JAR_URL))
             .setExtract(true)
             .build());
   }
   return dependencies;
 }
コード例 #8
0
  private static String createStartAlluxioCommand(String command) {
    List<String> commands = new ArrayList<>();
    commands.add(String.format("echo 'Starting Alluxio with %s'", command));
    if (installJavaFromUrl()) {
      commands.add("export JAVA_HOME=" + Configuration.get(PropertyKey.INTEGRATION_MESOS_JDK_PATH));
      commands.add("export PATH=$PATH:$JAVA_HOME/bin");
    }

    commands.add("mkdir conf");
    commands.add("touch conf/alluxio-env.sh");

    // If a jar is supplied, start Alluxio from the jar. Otherwise assume that Alluxio is already
    // installed at PropertyKey.HOME.
    if (installAlluxioFromUrl()) {
      commands.add("rm *.tar.gz");
      commands.add("mv alluxio-* alluxio");
    }
    String home = installAlluxioFromUrl() ? "alluxio" : Configuration.get(PropertyKey.HOME);
    commands.add(
        String.format("cp %s conf", PathUtils.concatPath(home, "conf", "log4j.properties")));
    commands.add(PathUtils.concatPath(home, "integration", "bin", command));
    return Joiner.on(" && ").join(commands);
  }
コード例 #9
0
  /**
   * Constructs a new Swift {@link UnderFileSystem}.
   *
   * @param uri the {@link AlluxioURI} for this UFS
   */
  public SwiftUnderFileSystem(AlluxioURI uri) {
    super(uri);
    String containerName = uri.getHost();
    LOG.debug("Constructor init: {}", containerName);
    AccountConfig config = new AccountConfig();
    if (Configuration.containsKey(Constants.SWIFT_API_KEY)) {
      config.setPassword(Configuration.get(Constants.SWIFT_API_KEY));
    } else if (Configuration.containsKey(Constants.SWIFT_PASSWORD_KEY)) {
      config.setPassword(Configuration.get(Constants.SWIFT_PASSWORD_KEY));
    }
    config.setAuthUrl(Configuration.get(Constants.SWIFT_AUTH_URL_KEY));
    String authMethod = Configuration.get(Constants.SWIFT_AUTH_METHOD_KEY);
    if (authMethod != null) {
      config.setUsername(Configuration.get(Constants.SWIFT_USER_KEY));
      config.setTenantName(Configuration.get(Constants.SWIFT_TENANT_KEY));
      switch (authMethod) {
        case Constants.SWIFT_AUTH_KEYSTONE:
          config.setAuthenticationMethod(AuthenticationMethod.KEYSTONE);
          break;
        case Constants.SWIFT_AUTH_SWIFTAUTH:
          // swiftauth authenticates directly against swift
          // note: this method is supported in swift object storage api v1
          config.setAuthenticationMethod(AuthenticationMethod.BASIC);
          break;
        default:
          config.setAuthenticationMethod(AuthenticationMethod.TEMPAUTH);
          // tempauth requires authentication header to be of the form tenant:user.
          // JOSS however generates header of the form user:tenant.
          // To resolve this, we switch user with tenant
          config.setTenantName(Configuration.get(Constants.SWIFT_USER_KEY));
          config.setUsername(Configuration.get(Constants.SWIFT_TENANT_KEY));
      }
    }

    ObjectMapper mapper = new ObjectMapper();
    mapper.configure(SerializationConfig.Feature.WRAP_ROOT_VALUE, true);
    mContainerName = containerName;
    mAccount = new AccountFactory(config).createAccount();
    // Do not allow container cache to avoid stale directory listings
    mAccount.setAllowContainerCaching(false);
    mAccess = mAccount.authenticate();
    Container container = mAccount.getContainer(containerName);
    if (!container.exists()) {
      container.create();
    }
    mContainerPrefix = Constants.HEADER_SWIFT + mContainerName + PATH_SEPARATOR;
  }
コード例 #10
0
ファイル: Mode.java プロジェクト: luoli523/tachyon
 /**
  * Gets the file / directory creation umask.
  *
  * @return the umask {@link Mode}
  */
 public static Mode getUMask() {
   int umask = Constants.DEFAULT_FILE_SYSTEM_UMASK;
   String confUmask = Configuration.get(PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_UMASK);
   if (confUmask != null) {
     if ((confUmask.length() > 4) || !tryParseInt(confUmask)) {
       throw new IllegalArgumentException(
           ExceptionMessage.INVALID_CONFIGURATION_VALUE.getMessage(
               confUmask, PropertyKey.SECURITY_AUTHORIZATION_PERMISSION_UMASK));
     }
     int newUmask = 0;
     int lastIndex = confUmask.length() - 1;
     for (int i = 0; i <= lastIndex; i++) {
       newUmask += (confUmask.charAt(i) - '0') << 3 * (lastIndex - i);
     }
     umask = newUmask;
   }
   return new Mode((short) umask);
 }
コード例 #11
0
  /** Tests the default properties for the master. */
  @Test
  public void masterDefaultTest() {
    String alluxioHome = sDefaultConfiguration.get(Constants.HOME);
    Assert.assertNotNull(alluxioHome);
    Assert.assertEquals("/mnt/alluxio_default_home", alluxioHome);

    String value = sDefaultConfiguration.get(Constants.MASTER_JOURNAL_FOLDER);
    Assert.assertNotNull(value);
    Assert.assertEquals(alluxioHome + "/journal/", value);

    value = sDefaultConfiguration.get(Constants.MASTER_HOSTNAME);
    Assert.assertNotNull(value);
    Assert.assertEquals(NetworkAddressUtils.getLocalHostName(100), value);

    value = sDefaultConfiguration.get(Constants.MASTER_FORMAT_FILE_PREFIX);
    Assert.assertNotNull(value);
    Assert.assertEquals(Constants.FORMAT_FILE_PREFIX, value);

    value = sDefaultConfiguration.get(Constants.MASTER_ADDRESS);
    Assert.assertNotNull(value);

    value = sDefaultConfiguration.get(Constants.MASTER_BIND_HOST);
    Assert.assertNotNull(value);
    Assert.assertEquals(NetworkAddressUtils.WILDCARD_ADDRESS, value);

    int intValue = sDefaultConfiguration.getInt(Constants.MASTER_RPC_PORT);
    Assert.assertEquals(19998, intValue);

    value = sDefaultConfiguration.get(Constants.MASTER_WEB_BIND_HOST);
    Assert.assertNotNull(value);
    Assert.assertEquals(NetworkAddressUtils.WILDCARD_ADDRESS, value);

    intValue = sDefaultConfiguration.getInt(Constants.MASTER_WEB_PORT);
    Assert.assertEquals(19999, intValue);

    intValue = sDefaultConfiguration.getInt(Constants.WEB_THREAD_COUNT);
    Assert.assertEquals(1, intValue);

    intValue = sDefaultConfiguration.getInt(Constants.MASTER_HEARTBEAT_INTERVAL_MS);
    Assert.assertEquals(Constants.SECOND_MS, intValue);

    intValue = sDefaultConfiguration.getInt(Constants.MASTER_WORKER_THREADS_MIN);
    Assert.assertEquals(Runtime.getRuntime().availableProcessors(), intValue);

    intValue = sDefaultConfiguration.getInt(Constants.MASTER_WORKER_TIMEOUT_MS);
    Assert.assertEquals(10 * Constants.SECOND_MS, intValue);
  }
コード例 #12
0
  /** Tests the default common properties. */
  @Test
  public void commonDefaultTest() {
    String alluxioHome = sDefaultConfiguration.get(Constants.HOME);
    Assert.assertNotNull(alluxioHome);
    Assert.assertEquals("/mnt/alluxio_default_home", alluxioHome);

    String ufsAddress = sDefaultConfiguration.get(Constants.UNDERFS_ADDRESS);
    Assert.assertNotNull(ufsAddress);
    Assert.assertEquals(alluxioHome + "/underFSStorage", ufsAddress);

    String value = sDefaultConfiguration.get(Constants.WEB_RESOURCES);
    Assert.assertNotNull(value);
    Assert.assertEquals(alluxioHome + "/core/server/src/main/webapp", value);

    value = sDefaultConfiguration.get(Constants.UNDERFS_HDFS_IMPL);
    Assert.assertNotNull(value);
    Assert.assertEquals("org.apache.hadoop.hdfs.DistributedFileSystem", value);

    value = sDefaultConfiguration.get(Constants.UNDERFS_HDFS_PREFIXS);
    Assert.assertNotNull(value);
    Assert.assertEquals(DEFAULT_HADOOP_UFS_PREFIX, value);

    value = sDefaultConfiguration.get(Constants.UNDERFS_GLUSTERFS_IMPL);
    Assert.assertNotNull(value);
    Assert.assertEquals("org.apache.hadoop.fs.glusterfs.GlusterFileSystem", value);

    boolean booleanValue = sDefaultConfiguration.getBoolean(Constants.ZOOKEEPER_ENABLED);
    Assert.assertFalse(booleanValue);

    booleanValue = sDefaultConfiguration.getBoolean(Constants.IN_TEST_MODE);
    Assert.assertFalse(booleanValue);

    int intValue = sDefaultConfiguration.getInt(Constants.NETWORK_HOST_RESOLUTION_TIMEOUT_MS);
    Assert.assertEquals(Constants.DEFAULT_HOST_RESOLUTION_TIMEOUT_MS, intValue);

    long longBytesValue =
        sDefaultConfiguration.getBytes(Constants.USER_BLOCK_REMOTE_READ_BUFFER_SIZE_BYTES);
    Assert.assertEquals(Constants.MB * 8, longBytesValue);

    int maxTry = sDefaultConfiguration.getInt(Constants.ZOOKEEPER_LEADER_INQUIRY_RETRY_COUNT);
    Assert.assertEquals(10, maxTry);
  }
コード例 #13
0
  /** Tests the simple substitution of variables. */
  @Test
  public void variableSubstitutionSimpleTest() {
    String home = mCustomPropsConfiguration.get("home");
    Assert.assertEquals("hometest", home);

    String homeAndPath = mCustomPropsConfiguration.get("homeandpath");
    Assert.assertEquals(home + "/path1", homeAndPath);

    String homeAndString = mCustomPropsConfiguration.get("homeandstring");
    Assert.assertEquals(home + " string1", homeAndString);

    String path2 = mCustomPropsConfiguration.get("path2");
    Assert.assertEquals("path2", path2);

    String multiplesubs = mCustomPropsConfiguration.get("multiplesubs");
    Assert.assertEquals(home + "/path1/" + path2, multiplesubs);

    String homePort = mCustomPropsConfiguration.get("home.port");
    Assert.assertEquals("8080", homePort);

    sTestProperties.put("complex.address", "alluxio://${home}:${home.port}");
    String complexAddress = mCustomPropsConfiguration.get("complex.address");
    Assert.assertEquals("alluxio://" + home + ":" + homePort, complexAddress);
  }
コード例 #14
0
  @Override
  public void resourceOffers(SchedulerDriver driver, List<Protos.Offer> offers) {
    long masterCpu = Configuration.getInt(PropertyKey.INTEGRATION_MASTER_RESOURCE_CPU);
    long masterMem =
        Configuration.getBytes(PropertyKey.INTEGRATION_MASTER_RESOURCE_MEM) / Constants.MB;
    long workerCpu = Configuration.getInt(PropertyKey.INTEGRATION_WORKER_RESOURCE_CPU);
    long workerMem =
        Configuration.getBytes(PropertyKey.INTEGRATION_WORKER_RESOURCE_MEM) / Constants.MB;

    LOG.info(
        "Master launched {}, master count {}, "
            + "requested master cpu {} mem {} MB and required master hostname {}",
        mMasterLaunched,
        mMasterCount,
        masterCpu,
        masterMem,
        mRequiredMasterHostname);

    for (Protos.Offer offer : offers) {
      Protos.Offer.Operation.Launch.Builder launch = Protos.Offer.Operation.Launch.newBuilder();
      double offerCpu = 0;
      double offerMem = 0;
      for (Protos.Resource resource : offer.getResourcesList()) {
        if (resource.getName().equals(Constants.MESOS_RESOURCE_CPUS)) {
          offerCpu += resource.getScalar().getValue();
        } else if (resource.getName().equals(Constants.MESOS_RESOURCE_MEM)) {
          offerMem += resource.getScalar().getValue();
        } else {
          // Other resources are currently ignored.
        }
      }

      LOG.info(
          "Received offer {} on host {} with cpus {} and mem {} MB and hasMasterPorts {}",
          offer.getId().getValue(),
          offer.getHostname(),
          offerCpu,
          offerMem,
          OfferUtils.hasAvailableMasterPorts(offer));

      Protos.ExecutorInfo.Builder executorBuilder = Protos.ExecutorInfo.newBuilder();
      List<Protos.Resource> resources;
      if (!mMasterLaunched
          && offerCpu >= masterCpu
          && offerMem >= masterMem
          && mMasterCount
              < Configuration.getInt(PropertyKey.INTEGRATION_MESOS_ALLUXIO_MASTER_NODE_COUNT)
          && OfferUtils.hasAvailableMasterPorts(offer)
          && (mRequiredMasterHostname == null
              || mRequiredMasterHostname.equals(offer.getHostname()))) {
        LOG.debug("Creating Alluxio Master executor");
        executorBuilder
            .setName("Alluxio Master Executor")
            .setSource("master")
            .setExecutorId(Protos.ExecutorID.newBuilder().setValue("master"))
            .addAllResources(getExecutorResources())
            .setCommand(
                Protos.CommandInfo.newBuilder()
                    .setValue(createStartAlluxioCommand("alluxio-master-mesos.sh"))
                    .addAllUris(getExecutorDependencyURIList())
                    .setEnvironment(
                        Protos.Environment.newBuilder()
                            .addVariables(
                                Protos.Environment.Variable.newBuilder()
                                    .setName("ALLUXIO_UNDERFS_ADDRESS")
                                    .setValue(Configuration.get(PropertyKey.UNDERFS_ADDRESS))
                                    .build())
                            .build()));
        // pre-build resource list here, then use it to build Protos.Task later.
        resources = getMasterRequiredResources(masterCpu, masterMem);
        mMasterHostname = offer.getHostname();
        mTaskName = Configuration.get(PropertyKey.INTEGRATION_MESOS_ALLUXIO_MASTER_NAME);
        mMasterCount++;
        mMasterTaskId = mLaunchedTasks;
      } else if (mMasterLaunched
          && !mWorkers.contains(offer.getHostname())
          && offerCpu >= workerCpu
          && offerMem >= workerMem
          && OfferUtils.hasAvailableWorkerPorts(offer)) {
        LOG.debug("Creating Alluxio Worker executor");
        final String memSize = FormatUtils.getSizeFromBytes((long) workerMem * Constants.MB);
        executorBuilder
            .setName("Alluxio Worker Executor")
            .setSource("worker")
            .setExecutorId(Protos.ExecutorID.newBuilder().setValue("worker"))
            .addAllResources(getExecutorResources())
            .setCommand(
                Protos.CommandInfo.newBuilder()
                    .setValue(createStartAlluxioCommand("alluxio-worker-mesos.sh"))
                    .addAllUris(getExecutorDependencyURIList())
                    .setEnvironment(
                        Protos.Environment.newBuilder()
                            .addVariables(
                                Protos.Environment.Variable.newBuilder()
                                    .setName("ALLUXIO_MASTER_HOSTNAME")
                                    .setValue(mMasterHostname)
                                    .build())
                            .addVariables(
                                Protos.Environment.Variable.newBuilder()
                                    .setName("ALLUXIO_WORKER_MEMORY_SIZE")
                                    .setValue(memSize)
                                    .build())
                            .addVariables(
                                Protos.Environment.Variable.newBuilder()
                                    .setName("ALLUXIO_UNDERFS_ADDRESS")
                                    .setValue(Configuration.get(PropertyKey.UNDERFS_ADDRESS))
                                    .build())
                            .build()));
        // pre-build resource list here, then use it to build Protos.Task later.
        resources = getWorkerRequiredResources(workerCpu, workerMem);
        mWorkers.add(offer.getHostname());
        mTaskName = Configuration.get(PropertyKey.INTEGRATION_MESOS_ALLUXIO_WORKER_NAME);
      } else {
        // The resource offer cannot be used to start either master or a worker.
        LOG.info("Declining offer {}", offer.getId().getValue());
        driver.declineOffer(offer.getId());
        continue;
      }

      Protos.TaskID taskId =
          Protos.TaskID.newBuilder().setValue(String.valueOf(mLaunchedTasks)).build();

      LOG.info("Launching task {} using offer {}", taskId.getValue(), offer.getId().getValue());

      Protos.TaskInfo task =
          Protos.TaskInfo.newBuilder()
              .setName(mTaskName)
              .setTaskId(taskId)
              .setSlaveId(offer.getSlaveId())
              .addAllResources(resources)
              .setExecutor(executorBuilder)
              .build();

      launch.addTaskInfos(Protos.TaskInfo.newBuilder(task));
      mLaunchedTasks++;

      // NOTE: We use the new API `acceptOffers` here to launch tasks.
      // The 'launchTasks' API will be deprecated.
      List<Protos.OfferID> offerIds = new ArrayList<Protos.OfferID>();
      offerIds.add(offer.getId());
      List<Protos.Offer.Operation> operations = new ArrayList<Protos.Offer.Operation>();
      Protos.Offer.Operation operation =
          Protos.Offer.Operation.newBuilder()
              .setType(Protos.Offer.Operation.Type.LAUNCH)
              .setLaunch(launch)
              .build();
      operations.add(operation);
      Protos.Filters filters = Protos.Filters.newBuilder().setRefuseSeconds(1).build();
      driver.acceptOffers(offerIds, operations, filters);
    }
  }
コード例 #15
0
 /** Tests the substitution of system variables. */
 @Test
 public void systemVariableSubstitutionSampleTest() {
   String masterAddress = mSystemPropsConfiguration.get(Constants.MASTER_ADDRESS);
   Assert.assertNotNull(masterAddress);
   Assert.assertEquals("alluxio-ft://master:20001", masterAddress);
 }
コード例 #16
0
 /** Tests the recursive substitution of variables. */
 @Test
 public void variableSubstitutionRecursiveTest() {
   String multiplesubs = mCustomPropsConfiguration.get("multiplesubs");
   String recursive = mCustomPropsConfiguration.get("recursive");
   Assert.assertEquals(multiplesubs, recursive);
 }
コード例 #17
0
 private static boolean installAlluxioFromUrl() {
   return Configuration.containsKey(PropertyKey.INTEGRATION_MESOS_ALLUXIO_JAR_URL)
       && !Configuration.get(PropertyKey.INTEGRATION_MESOS_ALLUXIO_JAR_URL)
           .equalsIgnoreCase(Constants.MESOS_LOCAL_INSTALL);
 }
コード例 #18
0
 @Before
 public final void before() throws Exception {
   Configuration.set(PropertyKey.UNDERFS_LISTING_LENGTH, 50);
   mUnderfsAddress = Configuration.get(PropertyKey.UNDERFS_ADDRESS);
   mUfs = UnderFileSystem.get(mUnderfsAddress + AlluxioURI.SEPARATOR);
 }