示例#1
0
  /**
   * Returns a list of metric names (including the passed in name) that are a submetric of the
   * specified metric. The metrics are topologically sorted by dependency, so the parent metric will
   * appear last.
   *
   * @param parentName
   * @return
   * @throws ConfigurationException
   */
  public List<String> getSubmetrics(String parentName) throws ConfigurationException {
    String type = getMetricType(parentName);
    Config config = getMetricConfig(parentName);
    List<String> toAdd = new ArrayList<String>();
    if (type.equals("ensemble") || type.equals("simple-ensemble")) {
      for (String child : config.getStringList("metrics")) {
        toAdd.addAll(getSubmetrics(child));
        toAdd.add(child);
      }
    } else if (type.equals("sparsevector.mostsimilarconcepts")) {
      toAdd.addAll(getSubmetrics(config.getString("generator.basemetric")));
    } else if (type.equals("milnewitten")) {
      toAdd.add(config.getString("inlink"));
      toAdd.add(config.getString("outlink"));
    } else if (config.hasPath("reliesOn")) {
      toAdd.addAll(config.getStringList("reliesOn"));
    }
    toAdd.add(parentName);
    List<String> results = new ArrayList<String>();

    // Make sure things only appear once. We save the FIRST time they appear to preserve
    // dependencies.
    for (String name : toAdd) {
      if (!results.contains(name)) {
        results.add(name);
      }
    }
    return results;
  }
    private ConversionConfig(Config config, Table table, String destinationFormat) {

      Preconditions.checkArgument(
          config.hasPath(DESTINATION_TABLE_KEY),
          String.format("Key %s.%s is not specified", destinationFormat, DESTINATION_TABLE_KEY));
      Preconditions.checkArgument(
          config.hasPath(DESTINATION_DB_KEY),
          String.format("Key %s.%s is not specified", destinationFormat, DESTINATION_DB_KEY));
      Preconditions.checkArgument(
          config.hasPath(DESTINATION_DATA_PATH_KEY),
          String.format(
              "Key %s.%s is not specified", destinationFormat, DESTINATION_DATA_PATH_KEY));

      // Required
      this.destinationFormat = destinationFormat;
      this.destinationTableName = resolveTemplate(config.getString(DESTINATION_TABLE_KEY), table);
      this.destinationStagingTableName =
          String.format(
              "%s_%s", this.destinationTableName, "staging"); // Fixed and non-configurable
      this.destinationDbName = resolveTemplate(config.getString(DESTINATION_DB_KEY), table);
      this.destinationDataPath =
          resolveTemplate(config.getString(DESTINATION_DATA_PATH_KEY), table);

      // Optional
      this.clusterBy = ConfigUtils.getStringList(config, CLUSTER_BY_KEY);
      this.numBuckets = Optional.fromNullable(ConfigUtils.getInt(config, NUM_BUCKETS_KEY, null));
      this.hiveRuntimeProperties =
          ConfigUtils.configToProperties(
              ConfigUtils.getConfig(
                  config, HIVE_RUNTIME_PROPERTIES_KEY_PREFIX, ConfigFactory.empty()));
      this.evolutionEnabled = ConfigUtils.getBoolean(config, EVOLUTION_ENABLED, false);
      this.rowLimit = Optional.fromNullable(ConfigUtils.getInt(config, ROW_LIMIT_KEY, null));
      this.sourceDataPathIdentifier =
          ConfigUtils.getStringList(config, SOURCE_DATA_PATH_IDENTIFIER_KEY);
    }
  public DateTimeDatasetVersionFinder(FileSystem fs, Config config) {
    super(fs);
    Preconditions.checkArgument(
        config.hasPath(DATE_TIME_PATTERN_KEY),
        "Missing required property " + DATE_TIME_PATTERN_KEY);
    String pattern = config.getString(DATE_TIME_PATTERN_KEY);

    if (config.hasPath(OPTIONAL_GLOB_PATTERN_TIMEZONE_KEY)) {
      this.globPattern = new Path(config.getString(OPTIONAL_GLOB_PATTERN_TIMEZONE_KEY));
    } else {
      this.globPattern = new Path(pattern.replaceAll("[^/]+", "*"));
    }

    LOGGER.debug(
        String.format(
            "Setting timezone for patthern: %s. By default it is %s",
            pattern, DEFAULT_DATE_TIME_PATTERN_TIMEZONE));

    if (config.hasPath(DATE_TIME_PATTERN_TIMEZONE_KEY)) {
      this.formatter =
          DateTimeFormat.forPattern(pattern)
              .withZone(DateTimeZone.forID(config.getString(DATE_TIME_PATTERN_TIMEZONE_KEY)));
    } else {
      this.formatter =
          DateTimeFormat.forPattern(pattern)
              .withZone(DateTimeZone.forID(DEFAULT_DATE_TIME_PATTERN_TIMEZONE));
    }

    this.datePartitionPattern = pattern;
  }
示例#4
0
 public String getMetricType(String name) throws ConfigurationException {
   Config config = getMetricConfig(name);
   String type = config.getString("type");
   if (type.equals("densevector") || type.equals("sparsevector")) {
     type += "." + config.getString("generator.type");
   }
   return type;
 }
示例#5
0
 private Conf() {
   Config config = ConfigFactory.load("taskCluster");
   // cluster list
   ConfigList list = config.getList("taskCluster.clusterList");
   for (ConfigValue configValue : list) {
     Config config1 = configValue.atPath("cluster").getConfig("cluster");
     clusterMap.put(
         config1.getString("name"),
         config1.getString("ip") + ":" + config1.getString("port") + config1.getString("url"));
   }
 }
示例#6
0
 @Override
 public void configure(Config config, String key) {
   fieldName1 = config.getString(key + ".field1");
   fieldName2 = config.getString(key + ".field2");
   if (config.hasPath(key + ".keys")) {
     keys = config.getStringList(key + ".keys");
   }
   key2 = config.getString(key + ".key2");
   constant = config.getDouble((key + ".constant"));
   outputName = config.getString(key + ".output");
 }
示例#7
0
  @Test
  public void testPropertiesToConfig() {

    Properties properties = new Properties();
    properties.setProperty("k1.kk1", "v1");
    properties.setProperty("k1.kk2", "v2");
    properties.setProperty("k2.kk", "v3");

    Config conf = ConfigUtils.propertiesToConfig(properties);
    Assert.assertEquals(conf.getString("k1.kk1"), "v1");
    Assert.assertEquals(conf.getString("k1.kk2"), "v2");
    Assert.assertEquals(conf.getString("k2.kk"), "v3");
  }
 private Connection getJdbcConnection() throws Exception {
   Connection connection;
   try {
     connection =
         DriverManager.getConnection(
             config.getString("metadata.jdbc.url"),
             config.getString("metadata.jdbc.username"),
             config.getString("metadata.jdbc.password"));
   } catch (Exception e) {
     LOG.error("error get connection for {}", config.getString("metadata.jdbc.url"), e);
     throw e;
   }
   return connection;
 }
 static Realm getRealm() {
   return new Realm.RealmBuilder()
       .setPrincipal(conf.getString("capsulecrm.token"))
       .setUsePreemptiveAuth(true)
       .setScheme(Realm.AuthScheme.BASIC)
       .build();
 }
  @Override
  public void configure(Config config, String key) {
    fieldName1 = config.getString(key + ".field1");
    outputName = config.getString(key + ".output");
    sb = new StringBuilder();

    limits = new ArrayList<>();
    stepSizes = new ArrayList<>();

    limits.add(1.0);
    stepSizes.add(1.0 / 32.0);

    limits.add(10.0);
    stepSizes.add(0.125);

    limits.add(25.0);
    stepSizes.add(0.25);

    limits.add(50.0);
    stepSizes.add(5.0);

    limits.add(100.0);
    stepSizes.add(10.0);

    limits.add(400.0);
    stepSizes.add(25.0);

    limits.add(2000.0);
    stepSizes.add(100.0);

    limits.add(10000.0);
    stepSizes.add(250.0);

    integerRoundingLimit = 25.0;
  }
示例#11
0
  @Override
  protected void loadRecentModel(long mostRecentModelGeneration) throws IOException {
    if (mostRecentModelGeneration <= modelGeneration) {
      return;
    }
    if (modelGeneration == NO_GENERATION) {
      log.info("Most recent generation {} is the first available one", mostRecentModelGeneration);
    } else {
      log.info(
          "Most recent generation {} is newer than current {}",
          mostRecentModelGeneration,
          modelGeneration);
    }

    File modelPMMLFile = File.createTempFile("model-", ".pmml.gz");
    modelPMMLFile.deleteOnExit();
    IOUtils.delete(modelPMMLFile);

    Config config = ConfigUtils.getDefaultConfig();
    String instanceDir = config.getString("model.instance-dir");

    String generationPrefix =
        Namespaces.getInstanceGenerationPrefix(instanceDir, mostRecentModelGeneration);
    String modelPMMLKey = generationPrefix + "model.pmml.gz";
    Store.get().download(modelPMMLKey, modelPMMLFile);
    log.info("Loading model description from {}", modelPMMLKey);

    Pair<DecisionForest, Map<Integer, BiMap<String, Integer>>> forestAndCatalog =
        DecisionForestPMML.read(modelPMMLFile);
    IOUtils.delete(modelPMMLFile);
    log.info("Loaded model description");

    modelGeneration = mostRecentModelGeneration;
    currentModel = new Generation(forestAndCatalog.getFirst(), forestAndCatalog.getSecond());
  }
示例#12
0
  /**
   * Get a classification model trainer corresponding to a given outcome type and a given algorithm.
   */
  public ClassificationModelTrainer constructTrainer(Config config) {
    config.checkValid(ConfigFactory.defaultReference(), "talismane.machineLearning");
    MachineLearningAlgorithm algorithm =
        MachineLearningAlgorithm.valueOf(config.getString("talismane.machineLearning.algorithm"));
    ClassificationModelTrainer modelTrainer = null;
    switch (algorithm) {
      case MaxEnt:
        MaxentModelTrainer maxentModelTrainer = new MaxentModelTrainer();
        modelTrainer = maxentModelTrainer;
        break;
      case LinearSVM:
      case LinearSVMOneVsRest:
        LinearSVMModelTrainer linearSVMModelTrainer = new LinearSVMModelTrainer();
        modelTrainer = linearSVMModelTrainer;
        break;
      case Perceptron:
        PerceptronClassificationModelTrainer perceptronModelTrainer =
            new PerceptronClassificationModelTrainer();
        modelTrainer = perceptronModelTrainer;
        break;
      default:
        throw new JolicielException("Machine learning algorithm not yet supported: " + algorithm);
    }

    modelTrainer.setParameters(config);
    return modelTrainer;
  }
 public SettingsImpl(Config config) {
   DB_URI = config.getString("myapp.db.uri");
   CIRCUIT_BREAKER_TIMEOUT =
       Duration.create(
           config.getDuration("myapp.circuit-breaker.timeout", TimeUnit.MILLISECONDS),
           TimeUnit.MILLISECONDS);
 }
  /**
   * Send a member installment to member when a member installment is created
   *
   * @param memberInstallment Member installment with information for mail content
   */
  private void sendMemberInstallmentNotice(MemberInstallment memberInstallment) {
    Config conf = ConfigFactory.load();
    String sendFromEmail = conf.getString("play.mailer.user");
    String sendFromName = Messages.get("app.global.title");
    String subject = sendFromName + ": " + Messages.get("adminUser.mail.subject.newAccount");

    Email email = new Email();
    email.setSubject(subject);
    email.setFrom(sendFromName + "<" + sendFromEmail + ">");
    email.addTo(
        memberInstallment.getMember().toString()
            + "<"
            + memberInstallment.getMember().getEmail()
            + ">");
    String body =
        views
            .html
            .subscription
            .memberInstallment
            .memberInstallmentNotice
            .render(subject, memberInstallment)
            .body();
    email.setBodyHtml(body);

    mailer.send(email);
  }
  /**
   * Use resource key(Optional) and rest json entry as a template and fill in template using Avro as
   * a reference. e.g: Rest JSON entry HOCON template:
   * AccountId=${sf_account_id},Member_Id__c=${member_id} Avro:
   * {"sf_account_id":{"string":"0016000000UiCYHAA3"},"member_id":{"long":296458833}}
   *
   * <p>Converted Json: {"AccountId":"0016000000UiCYHAA3","Member_Id__c":296458833}
   *
   * <p>As it's template based approach, it can produce nested JSON structure even Avro is flat (or
   * vice versa).
   *
   * <p>e.g: Rest resource template: /sobject/account/memberId/${member_id} Avro:
   * {"sf_account_id":{"string":"0016000000UiCYHAA3"},"member_id":{"long":296458833}} Converted
   * resource: /sobject/account/memberId/296458833
   *
   * <p>Converted resource will be used to form end point.
   * http://www.server.com:9090/sobject/account/memberId/296458833
   *
   * <p>{@inheritDoc}
   *
   * @see gobblin.converter.Converter#convertRecord(java.lang.Object, java.lang.Object,
   *     gobblin.configuration.WorkUnitState)
   */
  @Override
  public Iterable<RestEntry<JsonObject>> convertRecord(
      Void outputSchema, GenericRecord inputRecord, WorkUnitState workUnit)
      throws DataConversionException {

    Config srcConfig =
        ConfigFactory.parseString(
            inputRecord.toString(), ConfigParseOptions.defaults().setSyntax(ConfigSyntax.JSON));

    String resourceKey = workUnit.getProp(CONVERTER_AVRO_REST_ENTRY_RESOURCE_KEY, "");
    if (!StringUtils.isEmpty(resourceKey)) {
      final String dummyKey = "DUMMY";
      Config tmpConfig =
          ConfigFactory.parseString(dummyKey + "=" + resourceKey).resolveWith(srcConfig);
      resourceKey = tmpConfig.getString(dummyKey);
    }

    String hoconInput = workUnit.getProp(CONVERTER_AVRO_REST_JSON_ENTRY_TEMPLATE);
    if (StringUtils.isEmpty(hoconInput)) {
      return new SingleRecordIterable<>(
          new RestEntry<>(resourceKey, parser.parse(inputRecord.toString()).getAsJsonObject()));
    }

    Config destConfig = ConfigFactory.parseString(hoconInput).resolveWith(srcConfig);
    JsonObject json =
        parser.parse(destConfig.root().render(ConfigRenderOptions.concise())).getAsJsonObject();
    return new SingleRecordIterable<>(new RestEntry<>(resourceKey, json));
  }
示例#16
0
  @Override
  public void configure(final Env env, final Config config, final Binder binder) {
    super.configure(env, config, binder);

    if (scan) {
      pkgs.add(config.getString("application.ns"));
    }

    HbmUnitDescriptor descriptor =
        new HbmUnitDescriptor(getClass().getClassLoader(), dataSource(), config, pkgs);

    Map<Object, Object> integration = config(env, config, classes);
    emf = new HbmProvider(descriptor, integration);
    keys(EntityManagerFactory.class, key -> binder.bind(key).toProvider(emf).asEagerSingleton());

    List<Key<EntityManager>> emkeys = new ArrayList<>();

    keys(
        EntityManager.class,
        key -> {
          binder.bind(key).toProvider(Providers.outOfScope(key)).in(RequestScoped.class);
          emkeys.add(key);
        });

    Multibinder<Route.Definition> routes = Multibinder.newSetBinder(binder, Route.Definition.class);
    routes
        .addBinding()
        .toInstance(new Route.Definition("*", "*", new OpenSessionInView(emf, emkeys)).name("hbm"));
  }
 public GlobModTimeDatasetVersionFinder(FileSystem fs, Config config) {
   this(
       fs,
       config.hasPath(VERSION_FINDER_GLOB_PATTERN_KEY)
           ? new Path(config.getString(VERSION_FINDER_GLOB_PATTERN_KEY))
           : new Path("*"));
 }
示例#18
0
  public YarnService(
      Config config,
      String applicationName,
      String applicationId,
      YarnConfiguration yarnConfiguration,
      FileSystem fs,
      EventBus eventBus)
      throws Exception {
    this.applicationName = applicationName;
    this.applicationId = applicationId;

    this.config = config;

    this.eventBus = eventBus;

    this.gobblinMetrics =
        config.getBoolean(ConfigurationKeys.METRICS_ENABLED_KEY)
            ? Optional.of(buildGobblinMetrics())
            : Optional.<GobblinMetrics>absent();

    this.eventSubmitter =
        config.getBoolean(ConfigurationKeys.METRICS_ENABLED_KEY)
            ? Optional.of(buildEventSubmitter())
            : Optional.<EventSubmitter>absent();

    this.yarnConfiguration = yarnConfiguration;
    this.fs = fs;

    this.amrmClientAsync =
        closer.register(
            AMRMClientAsync.createAMRMClientAsync(1000, new AMRMClientCallbackHandler()));
    this.amrmClientAsync.init(this.yarnConfiguration);
    this.nmClientAsync =
        closer.register(NMClientAsync.createNMClientAsync(new NMClientCallbackHandler()));
    this.nmClientAsync.init(this.yarnConfiguration);

    this.initialContainers = config.getInt(GobblinYarnConfigurationKeys.INITIAL_CONTAINERS_KEY);
    this.requestedContainerMemoryMbs =
        config.getInt(GobblinYarnConfigurationKeys.CONTAINER_MEMORY_MBS_KEY);
    this.requestedContainerCores = config.getInt(GobblinYarnConfigurationKeys.CONTAINER_CORES_KEY);
    this.containerHostAffinityEnabled =
        config.getBoolean(GobblinYarnConfigurationKeys.CONTAINER_HOST_AFFINITY_ENABLED);

    this.helixInstanceMaxRetries =
        config.getInt(GobblinYarnConfigurationKeys.HELIX_INSTANCE_MAX_RETRIES);

    this.containerJvmArgs =
        config.hasPath(GobblinYarnConfigurationKeys.CONTAINER_JVM_ARGS_KEY)
            ? Optional.of(config.getString(GobblinYarnConfigurationKeys.CONTAINER_JVM_ARGS_KEY))
            : Optional.<String>absent();

    this.containerLaunchExecutor =
        Executors.newFixedThreadPool(
            10,
            ExecutorsUtils.newThreadFactory(
                Optional.of(LOGGER), Optional.of("ContainerLaunchExecutor")));

    this.tokens = getSecurityTokens();
  }
 @Inject
 private RobotRegistrationServlet(
     @Named(CoreSettingsNames.WAVE_SERVER_DOMAIN) String domain,
     RobotRegistrar robotRegistrar,
     Config config) {
   this.robotRegistrar = robotRegistrar;
   this.domain = domain;
   this.analyticsAccount = config.getString("administration.analytics_account");
 }
 public static void main(String[] args) throws Exception {
   Queue queue = new Queue();
   HostRouteHttpServer server =
       new HostRouteHttpServer(
           new HttpServerConfigurator(queue)
               .withAddress(
                   new Address(
                       CONFIG.getString("http.route.bind.host"),
                       CONFIG.getInt("http.route.bind.port"))),
           new HttpClientConfigurator(queue).withTrust(new Trust()));
   for (Config c : CONFIG.getConfigList("http.route.map")) {
     server.route(
         c.getStringList("hosts"),
         new Address(c.getString("to.host"), c.getInt("to.port")),
         c.getString("to.path"));
   }
   server.start();
 }
示例#21
0
 public CandidateFilterFactory() {
   Config config = ConfigUtils.getDefaultConfig();
   lshSampleRatio = config.getDouble("model.lsh.sample-ratio");
   numHashes = config.getInt("model.lsh.num-hashes");
   candidateFilterClassName =
       config.hasPath("serving-layer.candidate-filter-class")
           ? config.getString("serving-layer.candidate-filter-class")
           : null;
 }
 @Nonnull
 private Order<Rating> getRatingOrder(Config cfg) throws SpecificationException {
   Order<Rating> order = new RandomOrder<>();
   if (cfg.hasPath("order")) {
     switch (cfg.getString("order").toLowerCase()) {
       case "random":
         order = new RandomOrder<>();
         break;
       case "timestamp":
         order = new TimestampOrder<>();
         break;
       default:
         throw new SpecificationException(
             "invalid order " + cfg.getString("order") + " for crossfold");
     }
   }
   return order;
 }
示例#23
0
  public void buildMetric(String name) throws ConfigurationException, DaoException, IOException {

    LOG.info("building component metric " + name);
    String type = getMetricType(name);
    if (type.equals("densevector.word2vec")) {
      initWord2Vec(name);
    }

    SRMetric metric = getMetric(name);
    if (type.equals("ensemble")) {
      ((EnsembleMetric) metric).setTrainSubmetrics(false); // Do it by hand
    } else if (type.equals("sparsevector.mostsimilarconcepts")) {
      if (mode == Mode.SIMILARITY) {
        LOG.warn("metric " + name + " of type " + type + " requires mostSimilar... training BOTH");
        mode = Mode.BOTH;
      }
      throw new UnsupportedOperationException("This block needs to occur earlier.");
    } else if (type.equals("milnewitten")) {
      ((MilneWittenMetric) metric).setTrainSubmetrics(false);
    }

    if (metric instanceof BaseSRMetric) {
      ((BaseSRMetric) metric).setBuildMostSimilarCache(buildCosimilarity);
    }

    Dataset ds = getDataset();
    if (mode == Mode.SIMILARITY || mode == Mode.BOTH) {
      if (skipBuiltMetrics && metric.similarityIsTrained()) {
        LOG.info("metric " + name + " similarity() is already trained... skipping");
      } else {
        metric.trainSimilarity(ds);
      }
    }

    if (mode == Mode.MOSTSIMILAR || mode == Mode.BOTH) {
      if (skipBuiltMetrics && metric.mostSimilarIsTrained()) {
        LOG.info("metric " + name + " mostSimilar() is already trained... skipping");
      } else {
        Config config = getMetricConfig(name);
        int n = maxResults * EnsembleMetric.SEARCH_MULTIPLIER;
        TIntSet validIds = validMostSimilarIds;
        if (config.hasPath("maxResults")) {
          n = config.getInt("maxResults");
        }
        if (config.hasPath("mostSimilarConcepts")) {
          String path =
              String.format(
                  "%s/%s.txt",
                  config.getString("mostSimilarConcepts"), metric.getLanguage().getLangCode());
          validIds = readIds(path);
        }
        metric.trainMostSimilar(ds, n, validIds);
      }
    }
    metric.write();
  }
  /**
   * 初始化配置
   *
   * @throws TaskContainerConfigException
   */
  public TaskContainerConf() {
    Config config = ConfigFactory.load("taskContainer.conf");
    config.getString("taskContainer.version");
    if (StringUtils.isNotEmpty(config.getString("taskContainer.router"))) {
      router = config.getInt("taskContainer.router");
    }
    if (StringUtils.isNotEmpty(config.getString("taskContainer.worker"))) {
      worker = config.getInt("taskContainer.worker");
    }
    if (StringUtils.isNotEmpty(config.getString("taskContainer.alertPhone"))) {
      alertPhone = config.getString("taskContainer.alertPhone");
    }

    if (StringUtils.isNotEmpty(config.getString("taskContainer.projectName"))) {
      projectName = config.getString("taskContainer.projectName");
    }
    if (StringUtils.isNotEmpty(config.getString("taskContainer.processTaskClass"))) {
      processTaskClass = config.getString("taskContainer.processTaskClass");
    } else {
      LOAD_STATE = FAIL_LOAD;
      throw new TaskContainerConfigException("processTaskClass is empty");
    }
    try {
      aClass = Class.forName(processTaskClass);
      LOAD_STATE = FAIL_LOAD;
      processQueueInstance = aClass.newInstance();
    } catch (ClassNotFoundException e) {
      LOAD_STATE = FAIL_LOAD;
      throw new TaskContainerConfigException(e);
    } catch (InstantiationException e) {
      LOAD_STATE = FAIL_LOAD;
      throw new TaskContainerConfigException(e);
    } catch (IllegalAccessException e) {
      LOAD_STATE = FAIL_LOAD;
      throw new TaskContainerConfigException(e);
    }
    maxParallel = router * worker;
    queues = config.getStringList("taskContainer.taskList");
    logger.info("------------task container suc load conf---------------");
    logger.info("project.name:{}", projectName);
    logger.info("router:{}", router);
    logger.info("worker:{}", worker);
    logger.info("max.parallel:{}", maxParallel);
    logger.info("task.list:{}", queues);
    logger.info("process.task.class:{}", processTaskClass);
    logger.info("-------------------------------------------------------");
    LOAD_STATE = SUC_LOAD;
  }
  public String getString(String key) {
    String value = null;
    try {
      value = config.getString(key);

    } catch (Exception e) {
      log.warn("key {} is not found in config", key);
    }
    return value;
  }
  public List<String> getWorkload() {
    Config config = vampires.getConfig("workload");
    String task = config.getString("task");
    int startCount = config.getInt("start");
    int stopCount = config.getInt("stop");

    return IntStream.rangeClosed(startCount, stopCount)
        .mapToObj(i -> String.format(task, i))
        .collect(Collectors.toList());
  }
示例#27
0
 private void assertNameValueEquals2(String key, String value, Config config) {
   for (Map.Entry<String, ConfigValue> entry : config.entrySet()) {
     // assertEquals(key, entry.getKey()); // would fail
     assertEquals(key, trimQuote(entry.getKey()));
     String actualValue = entry.getValue().unwrapped().toString();
     assertEquals(value, actualValue);
     actualValue = config.getString(entry.getKey());
     assertEquals(value, actualValue);
   }
 }
  @Override
  public ApplicationEntity.Status status(
      Application<StormEnvironment, StormTopology> executor, com.typesafe.config.Config config) {
    String appId = config.getString("appId");
    LOG.info("Fetching {} status", appId);
    List<TopologySummary> topologySummaries;
    ApplicationEntity.Status status = null;
    try {
      if (Objects.equals(config.getString("mode"), ApplicationEntity.Mode.CLUSTER.name())) {
        Nimbus.Client stormClient =
            NimbusClient.getConfiguredClient(getStormConfig(config)).getClient();
        topologySummaries = stormClient.getClusterInfo().get_topologies();
      } else {
        topologySummaries = getLocalCluster().getClusterInfo().get_topologies();
      }

      for (TopologySummary topologySummary : topologySummaries) {
        if (topologySummary.get_name().equalsIgnoreCase(appId)) {
          if (topologySummary.get_status().equalsIgnoreCase("ACTIVE")) {
            status = ApplicationEntity.Status.RUNNING;
          } else if (topologySummary.get_status().equalsIgnoreCase("INACTIVE")) {
            status = ApplicationEntity.Status.STOPPED;
          } else if (topologySummary.get_status().equalsIgnoreCase("KILLED")) {
            status = ApplicationEntity.Status.STOPPED;
          } else {
            LOG.error(
                "Unknown storm topology ({}) status: {}",
                topologySummary.get_status(),
                topologySummary.get_status());
          }
        }
      }
      // If not exist, return removed
      if (status == null) {
        status = ApplicationEntity.Status.REMOVED;
      }
    } catch (TException e) {
      LOG.error("Got error to fetch status of {}", appId, e);
      status = ApplicationEntity.Status.UNKNOWN;
    }
    LOG.info("{} status is {}", appId, status);
    return status;
  }
示例#29
0
  @Test
  public void test() throws Exception {

    {
      final boolean isReady =
          true //
              && configService.updateIdentity() //
              && configService.updateVersion() //
              && configService.updateMaster() //
          ;

      assertTrue(isReady);
    }

    {
      final URL confURL = new URL("config:/instance/application.conf");

      log.info("### confURL : " + confURL);
    }

    {
      assertTrue(configService.isIdentityValid());
      assertTrue(configService.isVersionValid());
      assertTrue(configService.isMasterValid());
    }

    final Config config = configService.getMasterConfig();

    log.info("### config : " + config);

    log.info("### identity : {}", configService.getIdentity());

    log.info("### version root     : {}", configService.getVersionRoot());
    log.info("### version instance : {}", configService.getVersionInstance());

    log.info("### master root      : {}", configService.getMasterRoot());
    log.info("### master instance  : {}", configService.getMasterInstance());

    assertEquals(config.getString("main.name"), "app name");
    assertEquals(config.getString("main.version"), "1.0.6");
    assertEquals(config.getInt("main.size"), 123);
  }
 @Override
 public void stop(
     Application<StormEnvironment, StormTopology> executor, com.typesafe.config.Config config) {
   String appId = config.getString("appId");
   LOG.info("Stopping topology {} ...", appId);
   if (Objects.equals(config.getString("mode"), ApplicationEntity.Mode.CLUSTER.name())) {
     Nimbus.Client stormClient =
         NimbusClient.getConfiguredClient(getStormConfig(config)).getClient();
     try {
       stormClient.killTopologyWithOpts(appId, this.killOptions);
     } catch (NotAliveException | TException e) {
       LOG.error(
           "Failed to kill topology named {}, due to: {}", appId, e.getMessage(), e.getCause());
       throw new RuntimeException(e.getMessage(), e);
     }
   } else {
     getLocalCluster().killTopologyWithOpts(appId, this.killOptions);
   }
   LOG.info("Stopped topology {}", appId);
 }