@Override
public void configure(Context context) {
if (morphlineContext == null) {
FaultTolerance faultTolerance =
new FaultTolerance(
context.getBoolean(FaultTolerance.IS_PRODUCTION_MODE, false),
context.getBoolean(FaultTolerance.IS_IGNORING_RECOVERABLE_EXCEPTIONS, false),
context.getString(FaultTolerance.RECOVERABLE_EXCEPTION_CLASSES));
morphlineContext =
new MorphlineContext.Builder()
.setExceptionHandler(faultTolerance)
.setMetricRegistry(new MetricRegistry())
.build();
}
String morphlineFile = context.getString(MORPHLINE_FILE_PARAM);
String morphlineId = context.getString(MORPHLINE_ID_PARAM);
if (morphlineFile == null || morphlineFile.trim().length() == 0) {
throw new MorphlineCompilationException("Missing parameter: " + MORPHLINE_FILE_PARAM, null);
}
Config override =
ConfigFactory.parseMap(context.getSubProperties(MORPHLINE_VARIABLE_PARAM + "."));
morphline =
new Compiler()
.compile(new File(morphlineFile), morphlineId, morphlineContext, finalChild, override);
morphlineFileAndId = morphlineFile + "@" + morphlineId;
}
/**
* Constructor to initialize hibernate configuration parameters
*
* @param sqlSourceHelper Contains the configuration parameters from flume config file
*/
public HibernateHelper(SQLSourceHelper sqlSourceHelper) {
this.sqlSourceHelper = sqlSourceHelper;
Context context = sqlSourceHelper.getContext();
Map<String, String> hibernateProperties = context.getSubProperties("hibernate.");
Iterator<Map.Entry<String, String>> it = hibernateProperties.entrySet().iterator();
config = new Configuration();
Map.Entry<String, String> e;
while (it.hasNext()) {
e = it.next();
config.setProperty("hibernate." + e.getKey(), e.getValue());
}
}
@Override
public synchronized void configure(Context context) {
spoolDirectory = context.getString(SPOOL_DIRECTORY);
Preconditions.checkState(
spoolDirectory != null, "Configuration must specify a spooling directory");
completedSuffix = context.getString(SPOOLED_FILE_SUFFIX, DEFAULT_SPOOLED_FILE_SUFFIX);
deletePolicy = context.getString(DELETE_POLICY, DEFAULT_DELETE_POLICY);
fileHeader = context.getBoolean(FILENAME_HEADER, DEFAULT_FILE_HEADER);
fileHeaderKey = context.getString(FILENAME_HEADER_KEY, DEFAULT_FILENAME_HEADER_KEY);
basenameHeader = context.getBoolean(BASENAME_HEADER, DEFAULT_BASENAME_HEADER);
basenameHeaderKey = context.getString(BASENAME_HEADER_KEY, DEFAULT_BASENAME_HEADER_KEY);
batchSize = context.getInteger(BATCH_SIZE, DEFAULT_BATCH_SIZE);
inputCharset = context.getString(INPUT_CHARSET, DEFAULT_INPUT_CHARSET);
decodeErrorPolicy =
DecodeErrorPolicy.valueOf(
context
.getString(DECODE_ERROR_POLICY, DEFAULT_DECODE_ERROR_POLICY)
.toUpperCase(Locale.ENGLISH));
ignorePattern = context.getString(IGNORE_PAT, DEFAULT_IGNORE_PAT);
trackerDirPath = context.getString(TRACKER_DIR, DEFAULT_TRACKER_DIR);
deserializerType = context.getString(DESERIALIZER, "ZipDeserializer");
deserializerContext = new Context(context.getSubProperties(DESERIALIZER + "."));
consumeOrder =
ConsumeOrder.valueOf(
context
.getString(CONSUME_ORDER, DEFAULT_CONSUME_ORDER.toString())
.toUpperCase(Locale.ENGLISH));
// "Hack" to support backwards compatibility with previous generation of
// spooling directory source, which did not support deserializers
Integer bufferMaxLineLength = context.getInteger(BUFFER_MAX_LINE_LENGTH);
if (bufferMaxLineLength != null
&& deserializerType != null
&& deserializerType.equalsIgnoreCase(DEFAULT_DESERIALIZER)) {
deserializerContext.put(LineDeserializer.MAXLINE_KEY, bufferMaxLineLength.toString());
}
maxBackoff = context.getInteger(MAX_BACKOFF, DEFAULT_MAX_BACKOFF);
if (sourceCounter == null) {
sourceCounter = new SourceCounter(getName());
}
}
public void configure(Context context) throws ConfigurationException {
super.configure(context);
try {
String channelList = context.getString(BasicConfigurationConstants.CONFIG_CHANNELS);
if (channelList != null) {
this.channels = new HashSet<String>(Arrays.asList(channelList.split("\\s+")));
}
if (channels.isEmpty()) {
errors.add(
new FlumeConfigurationError(
componentName,
ComponentType.CHANNEL.getComponentType(),
FlumeConfigurationErrorType.PROPERTY_VALUE_NULL,
ErrorOrWarning.ERROR));
throw new ConfigurationException("No channels set for " + this.getComponentName());
}
Map<String, String> selectorParams =
context.getSubProperties(
BasicConfigurationConstants.CONFIG_SOURCE_CHANNELSELECTOR_PREFIX);
String selType;
if (selectorParams != null && !selectorParams.isEmpty()) {
selType = selectorParams.get(BasicConfigurationConstants.CONFIG_TYPE);
System.out.println("Loading selector: " + selType);
} else {
selType = ChannelSelectorConfigurationType.REPLICATING.toString();
}
if (selType == null || selType.isEmpty()) {
selType = ChannelSelectorConfigurationType.REPLICATING.toString();
}
ChannelSelectorType selectorType = this.getKnownChannelSelector(selType);
Context selectorContext = new Context();
selectorContext.putAll(selectorParams);
String config = null;
if (selectorType == null) {
config = selectorContext.getString(BasicConfigurationConstants.CONFIG_CONFIG);
if (config == null || config.isEmpty()) {
config = "OTHER";
}
} else {
config = selectorType.toString().toUpperCase();
}
this.selectorConf =
(ChannelSelectorConfiguration)
ComponentConfigurationFactory.create(
ComponentType.CHANNELSELECTOR.getComponentType(),
config,
ComponentType.CHANNELSELECTOR);
selectorConf.setChannels(channels);
selectorConf.configure(selectorContext);
} catch (Exception e) {
errors.add(
new FlumeConfigurationError(
componentName,
ComponentType.CHANNELSELECTOR.getComponentType(),
FlumeConfigurationErrorType.CONFIG_ERROR,
ErrorOrWarning.ERROR));
throw new ConfigurationException("Failed to configure component!", e);
}
}
@Override
public void configure(Context context) {
if (!isLocal) {
if (StringUtils.isNotBlank(context.getString(HOSTNAMES))) {
serverAddresses = StringUtils.deleteWhitespace(context.getString(HOSTNAMES)).split(",");
}
Preconditions.checkState(
serverAddresses != null && serverAddresses.length > 0, "Missing Param:" + HOSTNAMES);
}
if (StringUtils.isNotBlank(context.getString(INDEX_NAME))) {
this.indexName = context.getString(INDEX_NAME);
}
if (StringUtils.isNotBlank(context.getString(INDEX_TYPE))) {
this.indexType = context.getString(INDEX_TYPE);
}
if (StringUtils.isNotBlank(context.getString(CLUSTER_NAME))) {
this.clusterName = context.getString(CLUSTER_NAME);
}
if (StringUtils.isNotBlank(context.getString(BATCH_SIZE))) {
this.batchSize = Integer.parseInt(context.getString(BATCH_SIZE));
}
if (StringUtils.isNotBlank(context.getString(TTL))) {
this.ttlMs = parseTTL(context.getString(TTL));
Preconditions.checkState(ttlMs > 0, TTL + " must be greater than 0 or not set.");
}
if (StringUtils.isNotBlank(context.getString(CLIENT_TYPE))) {
clientType = context.getString(CLIENT_TYPE);
}
elasticSearchClientContext = new Context();
elasticSearchClientContext.putAll(context.getSubProperties(CLIENT_PREFIX));
String serializerClazz = DEFAULT_SERIALIZER_CLASS;
if (StringUtils.isNotBlank(context.getString(SERIALIZER))) {
serializerClazz = context.getString(SERIALIZER);
}
Context serializerContext = new Context();
serializerContext.putAll(context.getSubProperties(SERIALIZER_PREFIX));
try {
@SuppressWarnings("unchecked")
Class<? extends Configurable> clazz =
(Class<? extends Configurable>) Class.forName(serializerClazz);
Configurable serializer = clazz.newInstance();
if (serializer instanceof ElasticSearchIndexRequestBuilderFactory) {
indexRequestFactory = (ElasticSearchIndexRequestBuilderFactory) serializer;
indexRequestFactory.configure(serializerContext);
} else if (serializer instanceof ElasticSearchEventSerializer) {
eventSerializer = (ElasticSearchEventSerializer) serializer;
eventSerializer.configure(serializerContext);
} else {
throw new IllegalArgumentException(
serializerClazz + " is not an ElasticSearchEventSerializer");
}
} catch (Exception e) {
logger.error("Could not instantiate event serializer.", e);
Throwables.propagate(e);
}
if (sinkCounter == null) {
sinkCounter = new SinkCounter(getName());
}
String indexNameBuilderClass = DEFAULT_INDEX_NAME_BUILDER_CLASS;
if (StringUtils.isNotBlank(context.getString(INDEX_NAME_BUILDER))) {
indexNameBuilderClass = context.getString(INDEX_NAME_BUILDER);
}
Context indexnameBuilderContext = new Context();
indexnameBuilderContext.putAll(context.getSubProperties(INDEX_NAME_BUILDER_PREFIX));
try {
@SuppressWarnings("unchecked")
Class<? extends IndexNameBuilder> clazz =
(Class<? extends IndexNameBuilder>) Class.forName(indexNameBuilderClass);
indexNameBuilder = clazz.newInstance();
indexnameBuilderContext.put(INDEX_NAME, indexName);
indexNameBuilder.configure(indexnameBuilderContext);
} catch (Exception e) {
logger.error("Could not instantiate index name builder.", e);
Throwables.propagate(e);
}
if (sinkCounter == null) {
sinkCounter = new SinkCounter(getName());
}
Preconditions.checkState(StringUtils.isNotBlank(indexName), "Missing Param:" + INDEX_NAME);
Preconditions.checkState(StringUtils.isNotBlank(indexType), "Missing Param:" + INDEX_TYPE);
Preconditions.checkState(StringUtils.isNotBlank(clusterName), "Missing Param:" + CLUSTER_NAME);
Preconditions.checkState(batchSize >= 1, BATCH_SIZE + " must be greater than 0");
}