public SettingsImpl(Config config) {
DB_URI = config.getString("myapp.db.uri");
CIRCUIT_BREAKER_TIMEOUT =
Duration.create(
config.getDuration("myapp.circuit-breaker.timeout", TimeUnit.MILLISECONDS),
TimeUnit.MILLISECONDS);
}
private backtype.storm.Config getStormConfig(com.typesafe.config.Config config) {
backtype.storm.Config conf = new backtype.storm.Config();
conf.put(RichSpoutBatchExecutor.MAX_BATCH_SIZE_CONF, Int.box(64 * 1024));
conf.put(backtype.storm.Config.TOPOLOGY_RECEIVER_BUFFER_SIZE, Int.box(8));
conf.put(backtype.storm.Config.TOPOLOGY_TRANSFER_BUFFER_SIZE, Int.box(32));
conf.put(backtype.storm.Config.TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE, Int.box(16384));
conf.put(backtype.storm.Config.TOPOLOGY_EXECUTOR_SEND_BUFFER_SIZE, Int.box(16384));
conf.put(backtype.storm.Config.NIMBUS_THRIFT_MAX_BUFFER_SIZE, Int.box(20480000));
String nimbusHost = STORM_NIMBUS_HOST_DEFAULT;
if (environment.config().hasPath(STORM_NIMBUS_HOST_CONF_PATH)) {
nimbusHost = environment.config().getString(STORM_NIMBUS_HOST_CONF_PATH);
LOG.info("Overriding {} = {}", STORM_NIMBUS_HOST_CONF_PATH, nimbusHost);
} else {
LOG.info("Using default {} = {}", STORM_NIMBUS_HOST_CONF_PATH, STORM_NIMBUS_HOST_DEFAULT);
}
Integer nimbusThriftPort = STORM_NIMBUS_THRIFT_DEFAULT;
if (environment.config().hasPath(STORM_NIMBUS_THRIFT_CONF_PATH)) {
nimbusThriftPort = environment.config().getInt(STORM_NIMBUS_THRIFT_CONF_PATH);
LOG.info("Overriding {} = {}", STORM_NIMBUS_THRIFT_CONF_PATH, nimbusThriftPort);
} else {
LOG.info("Using default {} = {}", STORM_NIMBUS_THRIFT_CONF_PATH, STORM_NIMBUS_THRIFT_DEFAULT);
}
conf.put(backtype.storm.Config.NIMBUS_HOST, nimbusHost);
conf.put(backtype.storm.Config.NIMBUS_THRIFT_PORT, nimbusThriftPort);
conf.put(
Config.STORM_THRIFT_TRANSPORT_PLUGIN, "backtype.storm.security.auth.SimpleTransportPlugin");
if (config.hasPath(WORKERS)) {
conf.setNumWorkers(config.getInt(WORKERS));
}
if (config.hasPath(TOPOLOGY_MESSAGE_TIMEOUT_SECS)) {
conf.put(TOPOLOGY_MESSAGE_TIMEOUT_SECS, config.getInt(TOPOLOGY_MESSAGE_TIMEOUT_SECS));
}
return conf;
}
public GlobModTimeDatasetVersionFinder(FileSystem fs, Config config) {
this(
fs,
config.hasPath(VERSION_FINDER_GLOB_PATTERN_KEY)
? new Path(config.getString(VERSION_FINDER_GLOB_PATTERN_KEY))
: new Path("*"));
}
/**
* Get a classification model trainer corresponding to a given outcome type and a given algorithm.
*/
public ClassificationModelTrainer constructTrainer(Config config) {
config.checkValid(ConfigFactory.defaultReference(), "talismane.machineLearning");
MachineLearningAlgorithm algorithm =
MachineLearningAlgorithm.valueOf(config.getString("talismane.machineLearning.algorithm"));
ClassificationModelTrainer modelTrainer = null;
switch (algorithm) {
case MaxEnt:
MaxentModelTrainer maxentModelTrainer = new MaxentModelTrainer();
modelTrainer = maxentModelTrainer;
break;
case LinearSVM:
case LinearSVMOneVsRest:
LinearSVMModelTrainer linearSVMModelTrainer = new LinearSVMModelTrainer();
modelTrainer = linearSVMModelTrainer;
break;
case Perceptron:
PerceptronClassificationModelTrainer perceptronModelTrainer =
new PerceptronClassificationModelTrainer();
modelTrainer = perceptronModelTrainer;
break;
default:
throw new JolicielException("Machine learning algorithm not yet supported: " + algorithm);
}
modelTrainer.setParameters(config);
return modelTrainer;
}
/**
* Use resource key(Optional) and rest json entry as a template and fill in template using Avro as
* a reference. e.g: Rest JSON entry HOCON template:
* AccountId=${sf_account_id},Member_Id__c=${member_id} Avro:
* {"sf_account_id":{"string":"0016000000UiCYHAA3"},"member_id":{"long":296458833}}
*
* <p>Converted Json: {"AccountId":"0016000000UiCYHAA3","Member_Id__c":296458833}
*
* <p>As it's template based approach, it can produce nested JSON structure even Avro is flat (or
* vice versa).
*
* <p>e.g: Rest resource template: /sobject/account/memberId/${member_id} Avro:
* {"sf_account_id":{"string":"0016000000UiCYHAA3"},"member_id":{"long":296458833}} Converted
* resource: /sobject/account/memberId/296458833
*
* <p>Converted resource will be used to form end point.
* http://www.server.com:9090/sobject/account/memberId/296458833
*
* <p>{@inheritDoc}
*
* @see gobblin.converter.Converter#convertRecord(java.lang.Object, java.lang.Object,
* gobblin.configuration.WorkUnitState)
*/
@Override
public Iterable<RestEntry<JsonObject>> convertRecord(
Void outputSchema, GenericRecord inputRecord, WorkUnitState workUnit)
throws DataConversionException {
Config srcConfig =
ConfigFactory.parseString(
inputRecord.toString(), ConfigParseOptions.defaults().setSyntax(ConfigSyntax.JSON));
String resourceKey = workUnit.getProp(CONVERTER_AVRO_REST_ENTRY_RESOURCE_KEY, "");
if (!StringUtils.isEmpty(resourceKey)) {
final String dummyKey = "DUMMY";
Config tmpConfig =
ConfigFactory.parseString(dummyKey + "=" + resourceKey).resolveWith(srcConfig);
resourceKey = tmpConfig.getString(dummyKey);
}
String hoconInput = workUnit.getProp(CONVERTER_AVRO_REST_JSON_ENTRY_TEMPLATE);
if (StringUtils.isEmpty(hoconInput)) {
return new SingleRecordIterable<>(
new RestEntry<>(resourceKey, parser.parse(inputRecord.toString()).getAsJsonObject()));
}
Config destConfig = ConfigFactory.parseString(hoconInput).resolveWith(srcConfig);
JsonObject json =
parser.parse(destConfig.root().render(ConfigRenderOptions.concise())).getAsJsonObject();
return new SingleRecordIterable<>(new RestEntry<>(resourceKey, json));
}
/**
* Send a member installment to member when a member installment is created
*
* @param memberInstallment Member installment with information for mail content
*/
private void sendMemberInstallmentNotice(MemberInstallment memberInstallment) {
Config conf = ConfigFactory.load();
String sendFromEmail = conf.getString("play.mailer.user");
String sendFromName = Messages.get("app.global.title");
String subject = sendFromName + ": " + Messages.get("adminUser.mail.subject.newAccount");
Email email = new Email();
email.setSubject(subject);
email.setFrom(sendFromName + "<" + sendFromEmail + ">");
email.addTo(
memberInstallment.getMember().toString()
+ "<"
+ memberInstallment.getMember().getEmail()
+ ">");
String body =
views
.html
.subscription
.memberInstallment
.memberInstallmentNotice
.render(subject, memberInstallment)
.body();
email.setBodyHtml(body);
mailer.send(email);
}
@Override
protected void loadRecentModel(long mostRecentModelGeneration) throws IOException {
if (mostRecentModelGeneration <= modelGeneration) {
return;
}
if (modelGeneration == NO_GENERATION) {
log.info("Most recent generation {} is the first available one", mostRecentModelGeneration);
} else {
log.info(
"Most recent generation {} is newer than current {}",
mostRecentModelGeneration,
modelGeneration);
}
File modelPMMLFile = File.createTempFile("model-", ".pmml.gz");
modelPMMLFile.deleteOnExit();
IOUtils.delete(modelPMMLFile);
Config config = ConfigUtils.getDefaultConfig();
String instanceDir = config.getString("model.instance-dir");
String generationPrefix =
Namespaces.getInstanceGenerationPrefix(instanceDir, mostRecentModelGeneration);
String modelPMMLKey = generationPrefix + "model.pmml.gz";
Store.get().download(modelPMMLKey, modelPMMLFile);
log.info("Loading model description from {}", modelPMMLKey);
Pair<DecisionForest, Map<Integer, BiMap<String, Integer>>> forestAndCatalog =
DecisionForestPMML.read(modelPMMLFile);
IOUtils.delete(modelPMMLFile);
log.info("Loaded model description");
modelGeneration = mostRecentModelGeneration;
currentModel = new Generation(forestAndCatalog.getFirst(), forestAndCatalog.getSecond());
}
/**
*
*
* <ul>
* <li>The constructor takes in a dataset {@link Config} which MUST have a comma separated list
* of destination formats at key, {@value #DESTINATION_CONVERSION_FORMATS_KEY}
* <li>Conversion configuration for a format can be set by using destination format as prefix.
* <li>E.g. If {@value #DESTINATION_CONVERSION_FORMATS_KEY}=flattenedOrc,nestedOrc.<br>
* The destination table name for flattened ORC is set at flattenedOrc.tableName<br>
* And the destination table name for nested ORC is set at nestedOrc.tableName
* </ul>
*
* @param fs
* @param clientPool
* @param table
* @param config
*/
public ConvertibleHiveDataset(
FileSystem fs, HiveMetastoreClientPool clientPool, Table table, Config config) {
super(fs, clientPool, table, config);
Preconditions.checkArgument(
config.hasPath(DESTINATION_CONVERSION_FORMATS_KEY),
String.format(
"Atleast one destination format should be specified at %s.%s. If you do not intend to convert this dataset set %s.%s to true",
super.properties.getProperty(HiveDatasetFinder.HIVE_DATASET_CONFIG_PREFIX_KEY, ""),
DESTINATION_CONVERSION_FORMATS_KEY,
super.properties.getProperty(HiveDatasetFinder.HIVE_DATASET_CONFIG_PREFIX_KEY, ""),
HiveDatasetFinder.HIVE_DATASET_IS_BLACKLISTED_KEY));
// value for DESTINATION_CONVERSION_FORMATS_KEY can be a TypeSafe list or a comma separated list
// of string
this.destFormats =
Sets.newHashSet(ConfigUtils.getStringList(config, DESTINATION_CONVERSION_FORMATS_KEY));
// For each format create ConversionConfig and store it in a Map<format,conversionConfig>
this.destConversionConfigs = Maps.newHashMap();
for (String format : this.destFormats) {
if (config.hasPath(format)) {
this.destConversionConfigs.put(
format, new ConversionConfig(config.getConfig(format), table, format));
}
}
}
public String getMetricType(String name) throws ConfigurationException {
Config config = getMetricConfig(name);
String type = config.getString("type");
if (type.equals("densevector") || type.equals("sparsevector")) {
type += "." + config.getString("generator.type");
}
return type;
}
public int getCpuSetSize() {
if (vampires.hasPath("cpuSetSize")) {
return vampires.getInt("cpuSetSize");
} else {
LOG.error("missing executor cpuSetSize");
}
return 1;
}
protected Config parse(String file, Config... overrides) throws IOException {
File tmpFile = File.createTempFile("morphlines_", ".conf");
IOUtils.copy(getClass().getResourceAsStream(file), new FileOutputStream(tmpFile));
Config config = new Compiler().parse(tmpFile, overrides);
config = config.getConfigList("morphlines").get(0);
Preconditions.checkNotNull(config);
return config;
}
public CandidateFilterFactory() {
Config config = ConfigUtils.getDefaultConfig();
lshSampleRatio = config.getDouble("model.lsh.sample-ratio");
numHashes = config.getInt("model.lsh.num-hashes");
candidateFilterClassName =
config.hasPath("serving-layer.candidate-filter-class")
? config.getString("serving-layer.candidate-filter-class")
: null;
}
public void buildMetric(String name) throws ConfigurationException, DaoException, IOException {
LOG.info("building component metric " + name);
String type = getMetricType(name);
if (type.equals("densevector.word2vec")) {
initWord2Vec(name);
}
SRMetric metric = getMetric(name);
if (type.equals("ensemble")) {
((EnsembleMetric) metric).setTrainSubmetrics(false); // Do it by hand
} else if (type.equals("sparsevector.mostsimilarconcepts")) {
if (mode == Mode.SIMILARITY) {
LOG.warn("metric " + name + " of type " + type + " requires mostSimilar... training BOTH");
mode = Mode.BOTH;
}
throw new UnsupportedOperationException("This block needs to occur earlier.");
} else if (type.equals("milnewitten")) {
((MilneWittenMetric) metric).setTrainSubmetrics(false);
}
if (metric instanceof BaseSRMetric) {
((BaseSRMetric) metric).setBuildMostSimilarCache(buildCosimilarity);
}
Dataset ds = getDataset();
if (mode == Mode.SIMILARITY || mode == Mode.BOTH) {
if (skipBuiltMetrics && metric.similarityIsTrained()) {
LOG.info("metric " + name + " similarity() is already trained... skipping");
} else {
metric.trainSimilarity(ds);
}
}
if (mode == Mode.MOSTSIMILAR || mode == Mode.BOTH) {
if (skipBuiltMetrics && metric.mostSimilarIsTrained()) {
LOG.info("metric " + name + " mostSimilar() is already trained... skipping");
} else {
Config config = getMetricConfig(name);
int n = maxResults * EnsembleMetric.SEARCH_MULTIPLIER;
TIntSet validIds = validMostSimilarIds;
if (config.hasPath("maxResults")) {
n = config.getInt("maxResults");
}
if (config.hasPath("mostSimilarConcepts")) {
String path =
String.format(
"%s/%s.txt",
config.getString("mostSimilarConcepts"), metric.getLanguage().getLangCode());
validIds = readIds(path);
}
metric.trainMostSimilar(ds, n, validIds);
}
}
metric.write();
}
public List<String> getWorkload() {
Config config = vampires.getConfig("workload");
String task = config.getString("task");
int startCount = config.getInt("start");
int stopCount = config.getInt("stop");
return IntStream.rangeClosed(startCount, stopCount)
.mapToObj(i -> String.format(task, i))
.collect(Collectors.toList());
}
private void assertNameValueEquals2(String key, String value, Config config) {
for (Map.Entry<String, ConfigValue> entry : config.entrySet()) {
// assertEquals(key, entry.getKey()); // would fail
assertEquals(key, trimQuote(entry.getKey()));
String actualValue = entry.getValue().unwrapped().toString();
assertEquals(value, actualValue);
actualValue = config.getString(entry.getKey());
assertEquals(value, actualValue);
}
}
/**
* Builds a standard connection string (list of host:port) from the zoo.cfg file
*
* @param zookeeper_config
* @return the connection string
*/
public static String buildConnectionString(final Config zookeeper_config) {
final int port = zookeeper_config.getInt("clientPort");
final Config servers = zookeeper_config.getConfig("server");
return servers
.root()
.entrySet()
.stream()
.map(kv -> kv.getValue().unwrapped().toString().split(":")[0] + ":" + port)
.collect(Collectors.joining(","));
}
public YarnService(
Config config,
String applicationName,
String applicationId,
YarnConfiguration yarnConfiguration,
FileSystem fs,
EventBus eventBus)
throws Exception {
this.applicationName = applicationName;
this.applicationId = applicationId;
this.config = config;
this.eventBus = eventBus;
this.gobblinMetrics =
config.getBoolean(ConfigurationKeys.METRICS_ENABLED_KEY)
? Optional.of(buildGobblinMetrics())
: Optional.<GobblinMetrics>absent();
this.eventSubmitter =
config.getBoolean(ConfigurationKeys.METRICS_ENABLED_KEY)
? Optional.of(buildEventSubmitter())
: Optional.<EventSubmitter>absent();
this.yarnConfiguration = yarnConfiguration;
this.fs = fs;
this.amrmClientAsync =
closer.register(
AMRMClientAsync.createAMRMClientAsync(1000, new AMRMClientCallbackHandler()));
this.amrmClientAsync.init(this.yarnConfiguration);
this.nmClientAsync =
closer.register(NMClientAsync.createNMClientAsync(new NMClientCallbackHandler()));
this.nmClientAsync.init(this.yarnConfiguration);
this.initialContainers = config.getInt(GobblinYarnConfigurationKeys.INITIAL_CONTAINERS_KEY);
this.requestedContainerMemoryMbs =
config.getInt(GobblinYarnConfigurationKeys.CONTAINER_MEMORY_MBS_KEY);
this.requestedContainerCores = config.getInt(GobblinYarnConfigurationKeys.CONTAINER_CORES_KEY);
this.containerHostAffinityEnabled =
config.getBoolean(GobblinYarnConfigurationKeys.CONTAINER_HOST_AFFINITY_ENABLED);
this.helixInstanceMaxRetries =
config.getInt(GobblinYarnConfigurationKeys.HELIX_INSTANCE_MAX_RETRIES);
this.containerJvmArgs =
config.hasPath(GobblinYarnConfigurationKeys.CONTAINER_JVM_ARGS_KEY)
? Optional.of(config.getString(GobblinYarnConfigurationKeys.CONTAINER_JVM_ARGS_KEY))
: Optional.<String>absent();
this.containerLaunchExecutor =
Executors.newFixedThreadPool(
10,
ExecutorsUtils.newThreadFactory(
Optional.of(LOGGER), Optional.of("ContainerLaunchExecutor")));
this.tokens = getSecurityTokens();
}
public List<String> getExecutors() {
if (vampires.hasPath("executors")) {
return vampires
.getStringList("executors")
.stream()
.map(String::toUpperCase)
.collect(Collectors.toList());
} else {
LOG.error("missing executors config value");
throw new IllegalArgumentException("missing executors config value");
}
}
public ALSUpdate(Config config) {
super(config);
iterations = config.getInt("als.hyperparams.iterations");
implicit = config.getBoolean("als.hyperparams.implicit");
Preconditions.checkArgument(iterations > 0);
hyperParamRanges =
Arrays.asList(
HyperParamRanges.fromConfig(config, "als.hyperparams.features"),
HyperParamRanges.fromConfig(config, "als.hyperparams.lambda"),
HyperParamRanges.fromConfig(config, "als.hyperparams.alpha"));
noKnownItems = config.getBoolean("als.no-known-items");
}
@Test
public void testPropertiesToConfig() {
Properties properties = new Properties();
properties.setProperty("k1.kk1", "v1");
properties.setProperty("k1.kk2", "v2");
properties.setProperty("k2.kk", "v3");
Config conf = ConfigUtils.propertiesToConfig(properties);
Assert.assertEquals(conf.getString("k1.kk1"), "v1");
Assert.assertEquals(conf.getString("k1.kk2"), "v2");
Assert.assertEquals(conf.getString("k2.kk"), "v3");
}
@Test
public void testPropertiesToConfigWithPrefix() {
Properties properties = new Properties();
properties.setProperty("k1.kk1", "v1");
properties.setProperty("k1.kk2", "v2");
properties.setProperty("k2.kk", "v3");
Config conf = ConfigUtils.propertiesToConfig(properties, Optional.of("k1"));
Assert.assertEquals(conf.getString("k1.kk1"), "v1");
Assert.assertEquals(conf.getString("k1.kk2"), "v2");
Assert.assertFalse(conf.hasPath("k2.kk"), "Should not contain key k2.kk");
}
@Inject
public UserRegistrationServlet(
AccountStore accountStore,
@Named(CoreSettingsNames.WAVE_SERVER_DOMAIN) String domain,
Config config,
WelcomeRobot welcomeBot) {
this.accountStore = accountStore;
this.domain = domain;
this.welcomeBot = welcomeBot;
this.registrationDisabled = config.getBoolean("administration.disable_registration");
this.analyticsAccount = config.getString("administration.analytics_account");
}
public KafkaSourcedSpoutScheme(String deserClsName, Config context) {
try {
Properties prop = new Properties();
if (context.getObject("eagleProps") != null) {
prop.putAll(context.getObject("eagleProps"));
}
Constructor<?> constructor = Class.forName(deserClsName).getConstructor(Properties.class);
deserializer = (SpoutKafkaMessageDeserializer) constructor.newInstance(prop);
} catch (Exception ex) {
throw new RuntimeException(
"Failed to create new instance for decoder class " + deserClsName, ex);
}
}
public FacebookUserstreamProvider() {
Config config = StreamsConfigurator.config.getConfig("facebook");
FacebookUserInformationConfiguration facebookUserInformationConfiguration;
try {
facebookUserInformationConfiguration =
mapper.readValue(
config.root().render(ConfigRenderOptions.concise()),
FacebookUserInformationConfiguration.class);
} catch (IOException e) {
e.printStackTrace();
return;
}
}
/** @return the piece of config (if any), associated with a given relationship. */
@SuppressWarnings("unchecked")
public Config getConfig() {
Config conf = ConfigFactory.load();
List<Config> rels = (List<Config>) conf.getConfigList("relationships");
for (Config rel : rels) {
String name = rel.getString("relationship");
// System.out.println(this.getClass().getName());
if (name.equalsIgnoreCase(this.getClass().getName())) {
return rel.getConfig("config");
}
}
return null;
}
@Override
public void initialize() {
super.initialize();
Config config = ConfigUtils.getDefaultConfig();
decayFactor = (float) config.getDouble("model.decay.factor");
Preconditions.checkArgument(
decayFactor > 0.0f && decayFactor <= 1.0f,
"Decay factor must be in (0,1]: %s",
zeroThreshold);
zeroThreshold = (float) config.getDouble("model.decay.zeroThreshold");
Preconditions.checkArgument(
zeroThreshold >= 0.0f, "Zero threshold must be nonnegative: %s", zeroThreshold);
doDecay = decayFactor < 1.0f;
}
private Connection getJdbcConnection() throws Exception {
Connection connection;
try {
connection =
DriverManager.getConnection(
config.getString("metadata.jdbc.url"),
config.getString("metadata.jdbc.username"),
config.getString("metadata.jdbc.password"));
} catch (Exception e) {
LOG.error("error get connection for {}", config.getString("metadata.jdbc.url"), e);
throw e;
}
return connection;
}
public Sample(
CommandBuilder builder,
Config config,
Command parent,
Command child,
MorphlineContext context) {
super(builder, config, parent, child, context);
this.probability = getConfigs().getDouble(config, "probability", 1.0);
if (probability < 0.0) {
throw new MorphlineCompilationException(
"Probability must not be negative: " + probability, config);
}
if (probability >= 1.0) {
this.prng = null;
} else {
if (config.hasPath("seed")) {
long seed = getConfigs().getLong(config, "seed");
this.prng = new Well19937c(seed); // non-secure & fast
} else {
Random rand = new SecureRandom();
int[] seed = new int[624];
for (int i = 0; i < seed.length; i++) {
seed[i] = rand.nextInt();
}
this.prng = new Well19937c(seed); // non-secure & fast
}
}
validateArguments();
}
@Override
public void configure(final Env env, final Config config, final Binder binder) {
super.configure(env, config, binder);
if (scan) {
pkgs.add(config.getString("application.ns"));
}
HbmUnitDescriptor descriptor =
new HbmUnitDescriptor(getClass().getClassLoader(), dataSource(), config, pkgs);
Map<Object, Object> integration = config(env, config, classes);
emf = new HbmProvider(descriptor, integration);
keys(EntityManagerFactory.class, key -> binder.bind(key).toProvider(emf).asEagerSingleton());
List<Key<EntityManager>> emkeys = new ArrayList<>();
keys(
EntityManager.class,
key -> {
binder.bind(key).toProvider(Providers.outOfScope(key)).in(RequestScoped.class);
emkeys.add(key);
});
Multibinder<Route.Definition> routes = Multibinder.newSetBinder(binder, Route.Definition.class);
routes
.addBinding()
.toInstance(new Route.Definition("*", "*", new OpenSessionInView(emf, emkeys)).name("hbm"));
}
private void setWorkerThreads(final Builder builder, final Config config) {
val workerThreads = config.getInt("worker-threads");
if (workerThreads > 0) {
log.info(" > worker-threads: " + workerThreads);
builder.setWorkerThreads(workerThreads);
}
}