/**
* Use resource key(Optional) and rest json entry as a template and fill in template using Avro as
* a reference. e.g: Rest JSON entry HOCON template:
* AccountId=${sf_account_id},Member_Id__c=${member_id} Avro:
* {"sf_account_id":{"string":"0016000000UiCYHAA3"},"member_id":{"long":296458833}}
*
* <p>Converted Json: {"AccountId":"0016000000UiCYHAA3","Member_Id__c":296458833}
*
* <p>As it's template based approach, it can produce nested JSON structure even Avro is flat (or
* vice versa).
*
* <p>e.g: Rest resource template: /sobject/account/memberId/${member_id} Avro:
* {"sf_account_id":{"string":"0016000000UiCYHAA3"},"member_id":{"long":296458833}} Converted
* resource: /sobject/account/memberId/296458833
*
* <p>Converted resource will be used to form end point.
* http://www.server.com:9090/sobject/account/memberId/296458833
*
* <p>{@inheritDoc}
*
* @see gobblin.converter.Converter#convertRecord(java.lang.Object, java.lang.Object,
* gobblin.configuration.WorkUnitState)
*/
@Override
public Iterable<RestEntry<JsonObject>> convertRecord(
Void outputSchema, GenericRecord inputRecord, WorkUnitState workUnit)
throws DataConversionException {
Config srcConfig =
ConfigFactory.parseString(
inputRecord.toString(), ConfigParseOptions.defaults().setSyntax(ConfigSyntax.JSON));
String resourceKey = workUnit.getProp(CONVERTER_AVRO_REST_ENTRY_RESOURCE_KEY, "");
if (!StringUtils.isEmpty(resourceKey)) {
final String dummyKey = "DUMMY";
Config tmpConfig =
ConfigFactory.parseString(dummyKey + "=" + resourceKey).resolveWith(srcConfig);
resourceKey = tmpConfig.getString(dummyKey);
}
String hoconInput = workUnit.getProp(CONVERTER_AVRO_REST_JSON_ENTRY_TEMPLATE);
if (StringUtils.isEmpty(hoconInput)) {
return new SingleRecordIterable<>(
new RestEntry<>(resourceKey, parser.parse(inputRecord.toString()).getAsJsonObject()));
}
Config destConfig = ConfigFactory.parseString(hoconInput).resolveWith(srcConfig);
JsonObject json =
parser.parse(destConfig.root().render(ConfigRenderOptions.concise())).getAsJsonObject();
return new SingleRecordIterable<>(new RestEntry<>(resourceKey, json));
}
public FacebookUserstreamProvider() {
Config config = StreamsConfigurator.config.getConfig("facebook");
FacebookUserInformationConfiguration facebookUserInformationConfiguration;
try {
facebookUserInformationConfiguration =
mapper.readValue(
config.root().render(ConfigRenderOptions.concise()),
FacebookUserInformationConfiguration.class);
} catch (IOException e) {
e.printStackTrace();
return;
}
}
private backtype.storm.Config getStormConfig(com.typesafe.config.Config config) {
backtype.storm.Config conf = new backtype.storm.Config();
conf.put(RichSpoutBatchExecutor.MAX_BATCH_SIZE_CONF, Int.box(64 * 1024));
conf.put(backtype.storm.Config.TOPOLOGY_RECEIVER_BUFFER_SIZE, Int.box(8));
conf.put(backtype.storm.Config.TOPOLOGY_TRANSFER_BUFFER_SIZE, Int.box(32));
conf.put(backtype.storm.Config.TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE, Int.box(16384));
conf.put(backtype.storm.Config.TOPOLOGY_EXECUTOR_SEND_BUFFER_SIZE, Int.box(16384));
conf.put(backtype.storm.Config.NIMBUS_THRIFT_MAX_BUFFER_SIZE, Int.box(20480000));
String nimbusHost = STORM_NIMBUS_HOST_DEFAULT;
if (environment.config().hasPath(STORM_NIMBUS_HOST_CONF_PATH)) {
nimbusHost = environment.config().getString(STORM_NIMBUS_HOST_CONF_PATH);
LOG.info("Overriding {} = {}", STORM_NIMBUS_HOST_CONF_PATH, nimbusHost);
} else {
LOG.info("Using default {} = {}", STORM_NIMBUS_HOST_CONF_PATH, STORM_NIMBUS_HOST_DEFAULT);
}
Integer nimbusThriftPort = STORM_NIMBUS_THRIFT_DEFAULT;
if (environment.config().hasPath(STORM_NIMBUS_THRIFT_CONF_PATH)) {
nimbusThriftPort = environment.config().getInt(STORM_NIMBUS_THRIFT_CONF_PATH);
LOG.info("Overriding {} = {}", STORM_NIMBUS_THRIFT_CONF_PATH, nimbusThriftPort);
} else {
LOG.info("Using default {} = {}", STORM_NIMBUS_THRIFT_CONF_PATH, STORM_NIMBUS_THRIFT_DEFAULT);
}
conf.put(backtype.storm.Config.NIMBUS_HOST, nimbusHost);
conf.put(backtype.storm.Config.NIMBUS_THRIFT_PORT, nimbusThriftPort);
conf.put(
Config.STORM_THRIFT_TRANSPORT_PLUGIN, "backtype.storm.security.auth.SimpleTransportPlugin");
if (config.hasPath(WORKERS)) {
conf.setNumWorkers(config.getInt(WORKERS));
}
if (config.hasPath(TOPOLOGY_MESSAGE_TIMEOUT_SECS)) {
conf.put(TOPOLOGY_MESSAGE_TIMEOUT_SECS, config.getInt(TOPOLOGY_MESSAGE_TIMEOUT_SECS));
}
if (config.hasPath(MetricConfigs.METRIC_SINK_CONF)) {
conf.registerMetricsConsumer(
StormMetricTaggedConsumer.class, config.root().render(ConfigRenderOptions.concise()), 1);
}
return conf;
}
