private backtype.storm.Config getStormConfig(com.typesafe.config.Config config) {
    backtype.storm.Config conf = new backtype.storm.Config();
    conf.put(RichSpoutBatchExecutor.MAX_BATCH_SIZE_CONF, Int.box(64 * 1024));
    conf.put(backtype.storm.Config.TOPOLOGY_RECEIVER_BUFFER_SIZE, Int.box(8));
    conf.put(backtype.storm.Config.TOPOLOGY_TRANSFER_BUFFER_SIZE, Int.box(32));
    conf.put(backtype.storm.Config.TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE, Int.box(16384));
    conf.put(backtype.storm.Config.TOPOLOGY_EXECUTOR_SEND_BUFFER_SIZE, Int.box(16384));
    conf.put(backtype.storm.Config.NIMBUS_THRIFT_MAX_BUFFER_SIZE, Int.box(20480000));
    String nimbusHost = STORM_NIMBUS_HOST_DEFAULT;
    if (environment.config().hasPath(STORM_NIMBUS_HOST_CONF_PATH)) {
      nimbusHost = environment.config().getString(STORM_NIMBUS_HOST_CONF_PATH);
      LOG.info("Overriding {} = {}", STORM_NIMBUS_HOST_CONF_PATH, nimbusHost);
    } else {
      LOG.info("Using default {} = {}", STORM_NIMBUS_HOST_CONF_PATH, STORM_NIMBUS_HOST_DEFAULT);
    }
    Integer nimbusThriftPort = STORM_NIMBUS_THRIFT_DEFAULT;
    if (environment.config().hasPath(STORM_NIMBUS_THRIFT_CONF_PATH)) {
      nimbusThriftPort = environment.config().getInt(STORM_NIMBUS_THRIFT_CONF_PATH);
      LOG.info("Overriding {} = {}", STORM_NIMBUS_THRIFT_CONF_PATH, nimbusThriftPort);
    } else {
      LOG.info("Using default {} = {}", STORM_NIMBUS_THRIFT_CONF_PATH, STORM_NIMBUS_THRIFT_DEFAULT);
    }
    conf.put(backtype.storm.Config.NIMBUS_HOST, nimbusHost);
    conf.put(backtype.storm.Config.NIMBUS_THRIFT_PORT, nimbusThriftPort);
    conf.put(
        Config.STORM_THRIFT_TRANSPORT_PLUGIN, "backtype.storm.security.auth.SimpleTransportPlugin");
    if (config.hasPath(WORKERS)) {
      conf.setNumWorkers(config.getInt(WORKERS));
    }

    if (config.hasPath(TOPOLOGY_MESSAGE_TIMEOUT_SECS)) {
      conf.put(TOPOLOGY_MESSAGE_TIMEOUT_SECS, config.getInt(TOPOLOGY_MESSAGE_TIMEOUT_SECS));
    }
    return conf;
  }
Exemplo n.º 2
0
  public YarnService(
      Config config,
      String applicationName,
      String applicationId,
      YarnConfiguration yarnConfiguration,
      FileSystem fs,
      EventBus eventBus)
      throws Exception {
    this.applicationName = applicationName;
    this.applicationId = applicationId;

    this.config = config;

    this.eventBus = eventBus;

    this.gobblinMetrics =
        config.getBoolean(ConfigurationKeys.METRICS_ENABLED_KEY)
            ? Optional.of(buildGobblinMetrics())
            : Optional.<GobblinMetrics>absent();

    this.eventSubmitter =
        config.getBoolean(ConfigurationKeys.METRICS_ENABLED_KEY)
            ? Optional.of(buildEventSubmitter())
            : Optional.<EventSubmitter>absent();

    this.yarnConfiguration = yarnConfiguration;
    this.fs = fs;

    this.amrmClientAsync =
        closer.register(
            AMRMClientAsync.createAMRMClientAsync(1000, new AMRMClientCallbackHandler()));
    this.amrmClientAsync.init(this.yarnConfiguration);
    this.nmClientAsync =
        closer.register(NMClientAsync.createNMClientAsync(new NMClientCallbackHandler()));
    this.nmClientAsync.init(this.yarnConfiguration);

    this.initialContainers = config.getInt(GobblinYarnConfigurationKeys.INITIAL_CONTAINERS_KEY);
    this.requestedContainerMemoryMbs =
        config.getInt(GobblinYarnConfigurationKeys.CONTAINER_MEMORY_MBS_KEY);
    this.requestedContainerCores = config.getInt(GobblinYarnConfigurationKeys.CONTAINER_CORES_KEY);
    this.containerHostAffinityEnabled =
        config.getBoolean(GobblinYarnConfigurationKeys.CONTAINER_HOST_AFFINITY_ENABLED);

    this.helixInstanceMaxRetries =
        config.getInt(GobblinYarnConfigurationKeys.HELIX_INSTANCE_MAX_RETRIES);

    this.containerJvmArgs =
        config.hasPath(GobblinYarnConfigurationKeys.CONTAINER_JVM_ARGS_KEY)
            ? Optional.of(config.getString(GobblinYarnConfigurationKeys.CONTAINER_JVM_ARGS_KEY))
            : Optional.<String>absent();

    this.containerLaunchExecutor =
        Executors.newFixedThreadPool(
            10,
            ExecutorsUtils.newThreadFactory(
                Optional.of(LOGGER), Optional.of("ContainerLaunchExecutor")));

    this.tokens = getSecurityTokens();
  }
Exemplo n.º 3
0
  public List<String> getWorkload() {
    Config config = vampires.getConfig("workload");
    String task = config.getString("task");
    int startCount = config.getInt("start");
    int stopCount = config.getInt("stop");

    return IntStream.rangeClosed(startCount, stopCount)
        .mapToObj(i -> String.format(task, i))
        .collect(Collectors.toList());
  }
  /**
   * 初始化配置
   *
   * @throws TaskContainerConfigException
   */
  public TaskContainerConf() {
    Config config = ConfigFactory.load("taskContainer.conf");
    config.getString("taskContainer.version");
    if (StringUtils.isNotEmpty(config.getString("taskContainer.router"))) {
      router = config.getInt("taskContainer.router");
    }
    if (StringUtils.isNotEmpty(config.getString("taskContainer.worker"))) {
      worker = config.getInt("taskContainer.worker");
    }
    if (StringUtils.isNotEmpty(config.getString("taskContainer.alertPhone"))) {
      alertPhone = config.getString("taskContainer.alertPhone");
    }

    if (StringUtils.isNotEmpty(config.getString("taskContainer.projectName"))) {
      projectName = config.getString("taskContainer.projectName");
    }
    if (StringUtils.isNotEmpty(config.getString("taskContainer.processTaskClass"))) {
      processTaskClass = config.getString("taskContainer.processTaskClass");
    } else {
      LOAD_STATE = FAIL_LOAD;
      throw new TaskContainerConfigException("processTaskClass is empty");
    }
    try {
      aClass = Class.forName(processTaskClass);
      LOAD_STATE = FAIL_LOAD;
      processQueueInstance = aClass.newInstance();
    } catch (ClassNotFoundException e) {
      LOAD_STATE = FAIL_LOAD;
      throw new TaskContainerConfigException(e);
    } catch (InstantiationException e) {
      LOAD_STATE = FAIL_LOAD;
      throw new TaskContainerConfigException(e);
    } catch (IllegalAccessException e) {
      LOAD_STATE = FAIL_LOAD;
      throw new TaskContainerConfigException(e);
    }
    maxParallel = router * worker;
    queues = config.getStringList("taskContainer.taskList");
    logger.info("------------task container suc load conf---------------");
    logger.info("project.name:{}", projectName);
    logger.info("router:{}", router);
    logger.info("worker:{}", worker);
    logger.info("max.parallel:{}", maxParallel);
    logger.info("task.list:{}", queues);
    logger.info("process.task.class:{}", processTaskClass);
    logger.info("-------------------------------------------------------");
    LOAD_STATE = SUC_LOAD;
  }
 private void setWorkerThreads(final Builder builder, final Config config) {
   val workerThreads = config.getInt("worker-threads");
   if (workerThreads > 0) {
     log.info("  > worker-threads: " + workerThreads);
     builder.setWorkerThreads(workerThreads);
   }
 }
Exemplo n.º 6
0
 public int getCpuSetSize() {
   if (vampires.hasPath("cpuSetSize")) {
     return vampires.getInt("cpuSetSize");
   } else {
     LOG.error("missing executor cpuSetSize");
   }
   return 1;
 }
Exemplo n.º 7
0
 public static void main(String[] args) throws Exception {
   Queue queue = new Queue();
   HostRouteHttpServer server =
       new HostRouteHttpServer(
           new HttpServerConfigurator(queue)
               .withAddress(
                   new Address(
                       CONFIG.getString("http.route.bind.host"),
                       CONFIG.getInt("http.route.bind.port"))),
           new HttpClientConfigurator(queue).withTrust(new Trust()));
   for (Config c : CONFIG.getConfigList("http.route.map")) {
     server.route(
         c.getStringList("hosts"),
         new Address(c.getString("to.host"), c.getInt("to.port")),
         c.getString("to.path"));
   }
   server.start();
 }
Exemplo n.º 8
0
 public CandidateFilterFactory() {
   Config config = ConfigUtils.getDefaultConfig();
   lshSampleRatio = config.getDouble("model.lsh.sample-ratio");
   numHashes = config.getInt("model.lsh.num-hashes");
   candidateFilterClassName =
       config.hasPath("serving-layer.candidate-filter-class")
           ? config.getString("serving-layer.candidate-filter-class")
           : null;
 }
Exemplo n.º 9
0
  public void buildMetric(String name) throws ConfigurationException, DaoException, IOException {

    LOG.info("building component metric " + name);
    String type = getMetricType(name);
    if (type.equals("densevector.word2vec")) {
      initWord2Vec(name);
    }

    SRMetric metric = getMetric(name);
    if (type.equals("ensemble")) {
      ((EnsembleMetric) metric).setTrainSubmetrics(false); // Do it by hand
    } else if (type.equals("sparsevector.mostsimilarconcepts")) {
      if (mode == Mode.SIMILARITY) {
        LOG.warn("metric " + name + " of type " + type + " requires mostSimilar... training BOTH");
        mode = Mode.BOTH;
      }
      throw new UnsupportedOperationException("This block needs to occur earlier.");
    } else if (type.equals("milnewitten")) {
      ((MilneWittenMetric) metric).setTrainSubmetrics(false);
    }

    if (metric instanceof BaseSRMetric) {
      ((BaseSRMetric) metric).setBuildMostSimilarCache(buildCosimilarity);
    }

    Dataset ds = getDataset();
    if (mode == Mode.SIMILARITY || mode == Mode.BOTH) {
      if (skipBuiltMetrics && metric.similarityIsTrained()) {
        LOG.info("metric " + name + " similarity() is already trained... skipping");
      } else {
        metric.trainSimilarity(ds);
      }
    }

    if (mode == Mode.MOSTSIMILAR || mode == Mode.BOTH) {
      if (skipBuiltMetrics && metric.mostSimilarIsTrained()) {
        LOG.info("metric " + name + " mostSimilar() is already trained... skipping");
      } else {
        Config config = getMetricConfig(name);
        int n = maxResults * EnsembleMetric.SEARCH_MULTIPLIER;
        TIntSet validIds = validMostSimilarIds;
        if (config.hasPath("maxResults")) {
          n = config.getInt("maxResults");
        }
        if (config.hasPath("mostSimilarConcepts")) {
          String path =
              String.format(
                  "%s/%s.txt",
                  config.getString("mostSimilarConcepts"), metric.getLanguage().getLangCode());
          validIds = readIds(path);
        }
        metric.trainMostSimilar(ds, n, validIds);
      }
    }
    metric.write();
  }
Exemplo n.º 10
0
 /**
  * Builds a standard connection string (list of host:port) from the zoo.cfg file
  *
  * @param zookeeper_config
  * @return the connection string
  */
 public static String buildConnectionString(final Config zookeeper_config) {
   final int port = zookeeper_config.getInt("clientPort");
   final Config servers = zookeeper_config.getConfig("server");
   return servers
       .root()
       .entrySet()
       .stream()
       .map(kv -> kv.getValue().unwrapped().toString().split(":")[0] + ":" + port)
       .collect(Collectors.joining(","));
 }
  public int getInt(String key, int defaultValue) {
    int value = defaultValue;
    try {
      value = config.getInt(key);

    } catch (Exception e) {
      log.warn("key {} is not found in config", key);
    }
    return value;
  }
Exemplo n.º 12
0
 public ALSUpdate(Config config) {
   super(config);
   iterations = config.getInt("als.hyperparams.iterations");
   implicit = config.getBoolean("als.hyperparams.implicit");
   Preconditions.checkArgument(iterations > 0);
   hyperParamRanges =
       Arrays.asList(
           HyperParamRanges.fromConfig(config, "als.hyperparams.features"),
           HyperParamRanges.fromConfig(config, "als.hyperparams.lambda"),
           HyperParamRanges.fromConfig(config, "als.hyperparams.alpha"));
   noKnownItems = config.getBoolean("als.no-known-items");
 }
Exemplo n.º 13
0
 @Nonnull
 private PartitionAlgorithm<Rating> getRatingPartitionAlgorithm(Config cfg) {
   PartitionAlgorithm<Rating> partition = new HoldoutNPartition<>(10);
   if (cfg.hasPath("holdout")) {
     partition = new HoldoutNPartition<>(cfg.getInt("holdout"));
     if (cfg.hasPath("holdoutFraction")) {
       logger.warn("holdout and holdoutFraction specified, using holdout");
     }
     if (cfg.hasPath("retain")) {
       logger.warn("holdout and retain specified, using holdout");
     }
   } else if (cfg.hasPath("holdoutFraction")) {
     partition = new FractionPartition<>(cfg.getDouble("holdoutFraction"));
     if (cfg.hasPath("retain")) {
       logger.warn("holdoutFraction and retain specified, using holdout");
     }
   } else if (cfg.hasPath("retain")) {
     partition = new RetainNPartition<>(cfg.getInt("retain"));
   }
   return partition;
 }
Exemplo n.º 14
0
  @Test
  public void test() throws Exception {

    {
      final boolean isReady =
          true //
              && configService.updateIdentity() //
              && configService.updateVersion() //
              && configService.updateMaster() //
          ;

      assertTrue(isReady);
    }

    {
      final URL confURL = new URL("config:/instance/application.conf");

      log.info("### confURL : " + confURL);
    }

    {
      assertTrue(configService.isIdentityValid());
      assertTrue(configService.isVersionValid());
      assertTrue(configService.isMasterValid());
    }

    final Config config = configService.getMasterConfig();

    log.info("### config : " + config);

    log.info("### identity : {}", configService.getIdentity());

    log.info("### version root     : {}", configService.getVersionRoot());
    log.info("### version instance : {}", configService.getVersionInstance());

    log.info("### master root      : {}", configService.getMasterRoot());
    log.info("### master instance  : {}", configService.getMasterInstance());

    assertEquals(config.getString("main.name"), "app name");
    assertEquals(config.getString("main.version"), "1.0.6");
    assertEquals(config.getInt("main.size"), 123);
  }
 private void setIOThreads(final Builder builder, final Config config) {
   val ioThreads = config.getInt("io-threads");
   if (ioThreads > 0) builder.setIoThreads(ioThreads);
 }
Exemplo n.º 16
0
  @SuppressWarnings("OverlyLongMethod")
  private void run(Config config) {
    System.err.println(
        "      _\n"
            + " _ __| |___  __ _\n"
            + "| '_ \\ / _ \\/ _` |\n"
            + "| .__/_\\___/\\__, |\n"
            + "|_|         |___/ stress");

    final Config stressConfig = config.getConfig("plog.stress");

    final int threadCount = stressConfig.getInt("threads");
    log.info("Using {} threads", threadCount);

    final int rate = stressConfig.getInt("rate");
    final RateLimiter rateLimiter = RateLimiter.create(rate);

    final int socketRenewRate = stressConfig.getInt("renew_rate");
    final int minSize = stressConfig.getInt("min_size");
    final int maxSize = stressConfig.getInt("max_size");
    final int sizeIncrements = stressConfig.getInt("size_increments");
    final double sizeExponent = stressConfig.getDouble("size_exponent");

    final int sizeDelta = maxSize - minSize;
    final int differentSizes = sizeDelta / sizeIncrements;
    if (differentSizes == 0) {
      throw new RuntimeException("No sizes! Decrease plog.stress.size_increments");
    }

    final int stopAfter = stressConfig.getInt("stop_after");

    final int packetSize = stressConfig.getInt("udp.size");
    final int bufferSize = stressConfig.getInt("udp.SO_SNDBUF");

    final Fragmenter fragmenter = new Fragmenter(packetSize);

    final Random random = new Random(stressConfig.getLong("seed"));
    final byte[] randomBytes = new byte[maxSize];
    random.nextBytes(randomBytes);
    final ByteBuf randomMessage = Unpooled.wrappedBuffer(randomBytes);

    log.info("Generating {} different hashes", differentSizes);
    final int[] precomputedHashes = new int[differentSizes];
    for (int i = 0; i < differentSizes; i++) {
      precomputedHashes[i] = Murmur3.hash32(randomMessage, 0, minSize + sizeIncrements * i, 0);
    }

    final ByteBufAllocator allocator = new PooledByteBufAllocator();

    final double packetLoss = stressConfig.getDouble("udp.loss");

    final Meter socketMeter = registry.meter("Sockets used");
    final Meter messageMeter = registry.meter("Messages sent");
    final Meter packetMeter = registry.meter("Packets sent");
    final Meter sendFailureMeter = registry.meter("Send failures");
    final Meter lossMeter = registry.meter("Packets dropped");
    final Histogram messageSizeHistogram = registry.histogram("Message size");
    final Histogram packetSizeHistogram = registry.histogram("Packet size");

    final InetSocketAddress target =
        new InetSocketAddress(stressConfig.getString("host"), stressConfig.getInt("port"));

    log.info("Starting with config {}", config);

    final long consoleRate = stressConfig.getDuration("console.interval", TimeUnit.MILLISECONDS);
    ConsoleReporter.forRegistry(registry).build().start(consoleRate, TimeUnit.MILLISECONDS);

    for (int i = 0; i < threadCount; i++) {
      new Thread("stress_" + i) {
        private DatagramChannel channel = null;

        @Override
        public void run() {
          try {
            for (int sent = 0; sent < stopAfter; sent++, messageMeter.mark()) {
              if (sent % socketRenewRate == 0) {
                if (channel != null) {
                  channel.close();
                }
                channel = DatagramChannel.open();
                channel.socket().setSendBufferSize(bufferSize);
                socketMeter.mark();
              }

              // global rate limiting
              rateLimiter.acquire();

              final int sizeIndex =
                  (int) (Math.pow(random.nextDouble(), sizeExponent) * differentSizes);
              final int messageSize = minSize + sizeIncrements * sizeIndex;
              final int hash = precomputedHashes[sizeIndex];

              messageSizeHistogram.update(messageSize);

              final ByteBuf[] fragments =
                  fragmenter.fragment(allocator, randomMessage, null, sent, messageSize, hash);

              for (ByteBuf fragment : fragments) {
                if (random.nextDouble() < packetLoss) {
                  lossMeter.mark();
                } else {
                  final int packetSize = fragment.readableBytes();
                  final ByteBuffer buffer = fragment.nioBuffer();

                  try {
                    channel.send(buffer, target);
                    packetSizeHistogram.update(packetSize);
                    packetMeter.mark();
                  } catch (SocketException e) {
                    sendFailureMeter.mark();
                  }
                }
                fragment.release();
              }
            }
          } catch (Throwable t) {
            t.printStackTrace();
            System.exit(1);
          }
        }
      }.start();
    }
  }
Exemplo n.º 17
0
  private void initWord2Vec(String name) throws ConfigurationException, IOException, DaoException {
    Config config = getMetricConfig(name).getConfig("generator");
    File model = Word2VecGenerator.getModelFile(config.getString("modelDir"), language);
    if (skipBuiltMetrics && model.isFile()) {
      return;
    }

    if (config.hasPath("prebuilt") && config.getBoolean("prebuilt")) {
      if (model.isFile()) {
        return;
      }
      File downloadPath = new File(config.getString("binfile"));
      if (!downloadPath.isFile()) {
        throw new ConfigurationException(
            "word2vec model "
                + downloadPath.getAbsolutePath()
                + " cannot be found."
                + " You must download it from "
                + config.getString("url")
                + " into to the wikibrain download directory.");
      }
      if (!config.getStringList("languages").contains(language.getLangCode())) {
        throw new ConfigurationException(
            "word2vec model " + downloadPath + " does not support language" + language);
      }
      if (downloadPath.toString().toLowerCase().endsWith("gz")) {
        LOG.info("decompressing " + downloadPath + " to " + model);
        File tmp = File.createTempFile("word2vec", "bin");
        try {
          FileUtils.deleteQuietly(tmp);
          GZIPInputStream gz = new GZIPInputStream(new FileInputStream(downloadPath));
          FileUtils.copyInputStreamToFile(gz, tmp);
          gz.close();
          model.getParentFile().mkdirs();
          FileUtils.moveFile(tmp, model);
        } finally {
          FileUtils.deleteQuietly(tmp);
        }
      } else {
        FileUtils.copyFile(downloadPath, model);
      }
      return;
    }

    LinkProbabilityDao lpd = env.getConfigurator().get(LinkProbabilityDao.class);
    lpd.useCache(true);
    if (!lpd.isBuilt()) {
      lpd.build();
    }

    String corpusName = config.getString("corpus");
    Corpus corpus = null;
    if (!corpusName.equals("NONE")) {
      corpus =
          env.getConfigurator()
              .get(Corpus.class, config.getString("corpus"), "language", language.getLangCode());
      if (!corpus.exists()) {
        corpus.create();
      }
    }

    if (model.isFile()
        && (corpus == null || model.lastModified() > corpus.getCorpusFile().lastModified())) {
      return;
    }
    if (corpus == null) {
      throw new ConfigurationException(
          "word2vec metric "
              + name
              + " cannot build or find model!"
              + "configuration has no corpus, but model not found at "
              + model
              + ".");
    }
    Word2VecTrainer trainer =
        new Word2VecTrainer(env.getConfigurator().get(LocalPageDao.class), language);
    if (config.hasPath("dimensions")) {
      LOG.info("set number of dimensions to " + config.getInt("dimensions"));
      trainer.setLayer1Size(config.getInt("dimensions"));
    }
    if (config.hasPath("maxWords")) {
      LOG.info("set maxWords to " + config.getInt("maxWords"));
      trainer.setMaxWords(config.getInt("maxWords"));
    }
    if (config.hasPath("window")) {
      LOG.info("set window to " + config.getInt("maxWords"));
      trainer.setWindow(config.getInt("window"));
    }
    trainer.setKeepAllArticles(true);
    trainer.train(corpus.getDirectory());
    trainer.save(model);
  }
Exemplo n.º 18
0
  @Override
  public Crossfolder buildFromSpec(SpecificationContext context, Config cfg)
      throws SpecificationException {
    Crossfolder cf = new Crossfolder();
    if (cfg.hasPath("name")) {
      cf.setName(cfg.getString("name"));
    }
    cf.setSource(context.build(DataSource.class, cfg.getConfig("input")));
    if (cfg.hasPath("partitions")) {
      cf.setPartitionCount(cfg.getInt("partitions"));
    }

    String method = cfg.hasPath("method") ? cfg.getString("method") : "partition-users";
    switch (method) {
      case "partition-users":
        {
          PartitionAlgorithm<Rating> partition = getRatingPartitionAlgorithm(cfg);
          Order<Rating> order = getRatingOrder(cfg);
          cf.setMethod(CrossfoldMethods.partitionUsers(order, partition));
          break;
        }
      case "sample-users":
        {
          PartitionAlgorithm<Rating> partition = getRatingPartitionAlgorithm(cfg);
          Order<Rating> order = getRatingOrder(cfg);
          int sampleSize = cfg.hasPath("sampleSize") ? cfg.getInt("sampleSize") : 1000;
          cf.setMethod(CrossfoldMethods.sampleUsers(order, partition, sampleSize));
          break;
        }
      case "partition-ratings":
        cf.setMethod(CrossfoldMethods.partitionRatings());
        break;
      default:
        throw new SpecificationException("invalid crossfold method " + method);
    }

    if (cfg.hasPath("includeTimestamps")) {
      cf.setWriteTimestamps(cfg.getBoolean("includeTimestamps"));
    }

    if (cfg.hasPath("outputDir")) {
      cf.setOutputDir(cfg.getString("outputDir"));
    } else {
      logger.warn("no output directory specified for crossfold {}", cf.getName());
    }

    if (cfg.hasPath("outputFormat")) {
      switch (cfg.getString("outputFormat")) {
        case "pack":
          cf.setOutputFormat(OutputFormat.PACK);
          break;
        case "gzip":
          cf.setOutputFormat(OutputFormat.CSV_GZIP);
          break;
        case "xz":
          cf.setOutputFormat(OutputFormat.CSV_GZIP);
          break;
        default:
          throw new SpecificationException(
              "invalid output format " + cfg.getString("outputFormat"));
      }
    }

    if (cfg.hasPath("isolate")) {
      cf.setIsolate(cfg.getBoolean("isolate"));
    }

    return cf;
  }
Exemplo n.º 19
0
  @Override
  public ActorSystem buildActorSystem(Config config) throws Exception {

    // start redis
    final int redisPort =
        (config.hasPath("components.redis.port"))
            ? config.getInt("components.redis.port")
            : RedisURI.DEFAULT_REDIS_PORT;
    final String redisLogLevel =
        config.hasPath("components.redis.log-level")
            ? config.getString("components.redis.log-level")
            : "verbose";
    String logBase = System.getenv("LOG_BASE");
    if (StringUtils.isBlank(logBase)) logBase = System.getenv("TEMP");
    final String redisLogFile =
        config.hasPath("components.redis.log-file")
            ? config.getString("components.redis.log-file")
            : logBase + "\\redis.log";
    final String redisPidFile =
        config.hasPath("components.redis.pid-file")
            ? config.getString("components.redis.pid-file")
            : logBase + "\\redis.pid";

    try {
      this.redis =
          RedisServer.builder()
              .redisExecProvider(RedisExecProvider.defaultProvider())
              .port(redisPort)
              .setting("loglevel " + redisLogLevel)
              .setting("logfile " + redisLogFile)
              .setting("pidfile " + redisPidFile)
              .build();
    } catch (Exception ex) {
      this.logger.error("Fail to build redis server.", ex);
      throw new IllegalStateException("Fail to build redis server.", ex);
    }
    new Thread() {
      @Override
      public void run() {
        try {
          redis.start();
          logger.info("Started redis server on {} port", redisPort);
        } catch (Exception ex) {
          logger.error("Fail to start redis server.", ex);
          // @TODO Use future to stop the actor system at this point.
        }
      }
    }.start();

    // create redis client
    String redisUri = "redis://" + this.getAddress().getHostAddress() + ":" + redisPort + "/0";
    this.redisClient = new RedisClient(RedisURI.create(redisUri));

    ActorSystem system = ActorSystem.create(this.getClusterName(), config);
    Camel camel = CamelExtension.get(system);

    this.baseUrl =
        "http://"
            + this.getAddress().getHostAddress()
            + ":"
            + this.getHttpPort()
            + "/"
            + this.getApplicationName();
    String uri = "jetty:" + this.baseUrl;

    String recorderKeyBase = this.getClusterName() + ":" + "words";
    ActorRef recordingService =
        system.actorOf(
            Props.create(RecordingService.class, recorderKeyBase, this.redisClient),
            "recorderService");

    String tracerKey = this.getClusterName() + ":trace:node:1";
    ActorRef traceLogService =
        system.actorOf(
            Props.create(TraceLogService.class, tracerKey, this.redisClient, this.jacksonMapper),
            "traceLogService");

    ActorRef analysisService =
        system.actorOf(
            Props.create(AnalysisService.class, recordingService, traceLogService),
            "analysisService");

    String pathBase =
        "akka.tcp://" + this.getClusterName() + "@" + this.getAddress().getHostAddress() + ":";
    SimpleRoutingMap<String> routingMap = new SimpleRoutingMap<String>();
    routingMap.putPath(new Key<String>("2551"), pathBase + "2551/user/analysisService");
    routingMap.putPath(new Key<String>("2552"), pathBase + "2552/user/analysisService");

    ActorRef httpClerk =
        system.actorOf(Props.create(WebService.class, uri, routingMap), "httpClerk");

    Future<ActorRef> activationFuture =
        camel.activationFutureFor(
            httpClerk, new Timeout(Duration.create(10, TimeUnit.SECONDS)), system.dispatcher());

    return system;
  }
 ForestConfig(Config config) {
   this(
       config.getString("wood-block"),
       config.getString("leaves-block"),
       config.getString("thin-leaves-block"),
       config.getString("sapling-block"),
       config.getString("grows-on"),
       config.getString("seeds-on"),
       config.getInt("max-seed-height-difference"),
       config.getInt("max-generations"),
       config.getInt("min-generations"),
       config.getInt("trunk-radi"),
       config.getInt("trunk-height"),
       config.getInt("crown-radi"),
       config.getInt("crown-height"),
       config.getString("initial-state"),
       config.getInt("min-growth-delay"),
       config.getInt("random-growth-delay"),
       config.getInt("max-seeds-per-generation"),
       config.getInt("seed-every-generation"),
       config.getInt("random-growth"),
       config.getInt("leaf-decay-delay"),
       config.getInt("leave-sapling-probability"));
 }
Exemplo n.º 21
0
  public static InboundSettings create(Config config) {
    Config inbound = config.getConfig("inbound");

    List<String> columnNames;
    if (inbound.hasPath("column-names")) {
      columnNames = inbound.getStringList("column-names");
    } else {
      int numColumns = inbound.getInt("num-columns");
      columnNames = new ArrayList<>(numColumns);
      for (int i = 0; i < numColumns; i++) {
        columnNames.add(String.valueOf(i));
      }
    }

    Function<Object, Integer> lookup = new LookupFunction(columnNames);

    Collection<Integer> allColumns = Collections2.transform(columnNames, lookup);

    Collection<Integer> idColumns;
    if (inbound.hasPath("id-columns")) {
      idColumns =
          ImmutableSet.copyOf(Collections2.transform(inbound.getAnyRefList("id-columns"), lookup));
    } else {
      idColumns = ImmutableSet.of();
    }

    Collection<Integer> ignoredColumns;
    if (inbound.hasPath("ignored-columns")) {
      ignoredColumns =
          ImmutableSet.copyOf(
              Collections2.transform(inbound.getAnyRefList("ignored-columns"), lookup));
    } else {
      ignoredColumns = ImmutableSet.of();
    }

    Collection<Integer> categoricalColumns;
    Collection<Integer> numericColumns;
    if (inbound.hasPath("categorical-columns")) {
      Preconditions.checkState(!inbound.hasPath("numeric-columns"));
      categoricalColumns =
          new HashSet<>(
              Collections2.transform(inbound.getAnyRefList("categorical-columns"), lookup));
      numericColumns = new HashSet<>(allColumns);
      numericColumns.removeAll(categoricalColumns);
    } else if (inbound.hasPath("numeric-columns")) {
      Preconditions.checkState(!inbound.hasPath("categorical-columns"));
      numericColumns =
          new HashSet<>(Collections2.transform(inbound.getAnyRefList("numeric-columns"), lookup));
      categoricalColumns = new HashSet<>(allColumns);
      categoricalColumns.removeAll(numericColumns);
    } else {
      throw new IllegalArgumentException("No categorical-columns or numeric-columns set");
    }
    numericColumns.removeAll(idColumns);
    numericColumns.removeAll(ignoredColumns);
    categoricalColumns.removeAll(idColumns);
    categoricalColumns.removeAll(ignoredColumns);

    Integer targetColumn = null;
    if (inbound.hasPath("target-column")) {
      targetColumn = lookup.apply(inbound.getAnyRef("target-column"));
      Preconditions.checkState(
          categoricalColumns.contains(targetColumn) || numericColumns.contains(targetColumn),
          "Target column not specified as numeric or categorical");
    }

    return new InboundSettings(
        columnNames, idColumns, categoricalColumns, numericColumns, ignoredColumns, targetColumn);
  }
 private void setBufferSize(final Builder builder, final Config config) {
   val bufferSize = config.getInt("buffer-size");
   log.info("  > buffer-size: " + bufferSize);
   builder.setBufferSize(bufferSize);
 }
 public JobProcessingActor() {
   conf = ConfigFactory.load();
   learnerTimeout = conf.getInt("learner.default.timeout");
 }