@Test(groups = "dev")
public void autoWrapTest() throws EventDeliveryException {
ctx.put(MongoSink.AUTO_WRAP, Boolean.toString(true));
ctx.put(MongoSink.DB_NAME, "test_wrap");
MongoSink sink = new MongoSink();
Configurables.configure(sink, ctx);
sink.setChannel(channel);
sink.start();
Transaction tx = channel.getTransaction();
tx.begin();
String msg =
"2012/10/26 11:23:08 [error] 7289#0: *6430831 open() \"/usr/local/nginx/html/50x.html\" failed (2: No such file or directory), client: 10.160.105.161, server: sg15.redatoms.com, request: \"POST /mojo/ajax/embed HTTP/1.0\", upstream: \"fastcgi://unix:/tmp/php-fpm.sock:\", host: \"sg15.redatoms.com\", referrer: \"http://sg15.redatoms.com/mojo/mobile/package\"";
Event e = EventBuilder.withBody(msg.getBytes());
channel.put(e);
tx.commit();
tx.close();
sink.process();
sink.stop();
DB db = mongo.getDB("test_wrap");
DBCollection collection = db.getCollection("test_log");
DBCursor cursor = collection.find(new BasicDBObject(MongoSink.DEFAULT_WRAP_FIELD, msg));
assertTrue(cursor.hasNext());
DBObject dbObject = cursor.next();
assertNotNull(dbObject);
assertEquals(dbObject.get(MongoSink.DEFAULT_WRAP_FIELD), msg);
mongo.dropDatabase("test_wrap");
}
/** Verify that setting a timeout in the configuration does not impact the database setting */
@Test
public void timeoutConfiguredTest() {
String host = "localhost";
int timeout = 10;
JedisPoolFactory mockJedisPoolFactory = mock(JedisPoolFactory.class);
RedisSink redisSink = new RedisSink(mockJedisPoolFactory);
Channel channel = mock(Channel.class);
redisSink.setChannel(channel);
Context context = new Context();
context.put(RedisSinkConfigurationConstant.HOST, host);
context.put(RedisSinkConfigurationConstant.BATCH_SIZE, "10");
context.put(RedisSinkConfigurationConstant.TIMEOUT, Integer.toString(timeout));
redisSink.configure(context);
redisSink.start();
verify(mockJedisPoolFactory)
.create(
any(JedisPoolConfig.class),
eq(host),
eq(Protocol.DEFAULT_PORT),
eq(timeout),
isNull(String.class),
eq(Protocol.DEFAULT_DATABASE));
}
public void SequenceFileRenameRetryCoreTest(int numberOfRetriesRequired, boolean closeSucceed)
throws Exception {
String hdfsPath =
"file:///tmp/flume-test."
+ Calendar.getInstance().getTimeInMillis()
+ "."
+ Thread.currentThread().getId();
Context context = new Context();
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
Path dirPath = new Path(hdfsPath);
fs.delete(dirPath, true);
fs.mkdirs(dirPath);
context.put("hdfs.path", hdfsPath);
context.put("hdfs.closeTries", String.valueOf(numberOfRetriesRequired));
context.put("hdfs.rollCount", "1");
context.put("hdfs.retryInterval", "1");
context.put("hdfs.callTimeout", Long.toString(1000));
MockFileSystem mockFs = new MockFileSystem(fs, numberOfRetriesRequired, closeSucceed);
BucketWriter bucketWriter =
new BucketWriter(
0,
0,
1,
1,
ctx,
hdfsPath,
hdfsPath,
"singleBucket",
".tmp",
null,
null,
null,
new MockDataStream(mockFs),
timedRollerPool,
proxy,
new SinkCounter("test-bucket-writer-" + System.currentTimeMillis()),
0,
null,
null,
30000,
Executors.newSingleThreadExecutor(),
1,
numberOfRetriesRequired);
bucketWriter.setFileSystem(mockFs);
// At this point, we checked if isFileClosed is available in
// this JVM, so lets make it check again.
Event event = EventBuilder.withBody("test", Charsets.UTF_8);
bucketWriter.append(event);
// This is what triggers the close, so a 2nd append is required :/
bucketWriter.append(event);
TimeUnit.SECONDS.sleep(numberOfRetriesRequired + 2);
Assert.assertTrue(
"Expected " + numberOfRetriesRequired + " " + "but got " + bucketWriter.renameTries.get(),
bucketWriter.renameTries.get() == numberOfRetriesRequired);
}
@Test
public void testBatchEvents() throws InterruptedException, EventDeliveryException {
StressSource source = new StressSource();
source.setChannelProcessor(mockProcessor);
Context context = new Context();
context.put("maxTotalEvents", "35");
context.put("batchSize", "10");
source.configure(context);
source.start();
for (int i = 0; i < 50; i++) {
if (source.process() == Status.BACKOFF) {
TestCase.assertTrue("Source should have sent all events in 4 batches", i == 4);
break;
}
if (i < 3) {
verify(mockProcessor, times(i + 1)).processEventBatch(getLastProcessedEventList(source));
} else {
verify(mockProcessor, times(1)).processEventBatch(getLastProcessedEventList(source));
}
}
long successfulEvents = getCounterGroup(source).get("events.successful");
TestCase.assertTrue(
"Number of successful events should be 35 but was " + successfulEvents,
successfulEvents == 35);
long failedEvents = getCounterGroup(source).get("events.failed");
TestCase.assertTrue(
"Number of failure events should be 0 but was " + failedEvents, failedEvents == 0);
}
// @Test
public void testLifecycle() throws InterruptedException, LifecycleException {
LOG.debug("Starting...");
Context context = new Context();
context.put("hdfs.path", testPath);
context.put("hdfs.filePrefix", "pageview");
Configurables.configure(sink, context);
sink.setChannel(new MemoryChannel());
sink.start();
sink.stop();
}
@Test(groups = "dev")
public void timestampExistingFieldTest() throws EventDeliveryException, ParseException {
ctx.put(MongoSink.MODEL, MongoSink.CollectionModel.dynamic.name());
String tsField = "createdOn";
ctx.put(MongoSink.TIMESTAMP_FIELD, tsField);
MongoSink sink = new MongoSink();
Configurables.configure(sink, ctx);
sink.setChannel(channel);
sink.start();
JSONObject msg = new JSONObject();
msg.put("age", 11);
msg.put("birthday", new Date().getTime());
String dateText = "2013-02-19T14:20:53+08:00";
msg.put(tsField, dateText);
Transaction tx;
for (int i = 0; i < 10; i++) {
tx = channel.getTransaction();
tx.begin();
msg.put("name", "test" + i);
JSONObject header = new JSONObject();
header.put(MongoSink.COLLECTION, "my_events");
header.put(MongoSink.DB_NAME, "dynamic_db");
Event e = EventBuilder.withBody(msg.toJSONString().getBytes(), header);
channel.put(e);
tx.commit();
tx.close();
}
sink.process();
sink.stop();
msg.put(tsField, MongoSink.dateTimeFormatter.parseDateTime(dateText).toDate());
for (int i = 0; i < 10; i++) {
msg.put("name", "test" + i);
System.out.println("i = " + i);
DB db = mongo.getDB("dynamic_db");
DBCollection collection = db.getCollection("my_events");
DBCursor cursor = collection.find(new BasicDBObject(msg));
assertTrue(cursor.hasNext());
DBObject dbObject = cursor.next();
assertNotNull(dbObject);
assertEquals(dbObject.get("name"), msg.get("name"));
assertEquals(dbObject.get("age"), msg.get("age"));
assertEquals(dbObject.get("birthday"), msg.get("birthday"));
assertTrue(dbObject.get(tsField) instanceof Date);
System.out.println("ts = " + dbObject.get(tsField));
}
}
@Test(groups = "dev")
public void sinkSingleModelTest() throws EventDeliveryException {
ctx.put(MongoSink.MODEL, MongoSink.CollectionModel.single.name());
MongoSink sink = new MongoSink();
Configurables.configure(sink, ctx);
sink.setChannel(channel);
sink.start();
Transaction tx = channel.getTransaction();
tx.begin();
JSONObject msg = new JSONObject();
msg.put("name", "test");
msg.put("age", 11);
msg.put("birthday", new Date().getTime());
Event e = EventBuilder.withBody(msg.toJSONString().getBytes());
channel.put(e);
tx.commit();
tx.close();
sink.process();
sink.stop();
DB db = mongo.getDB("test_events");
DBCollection collection = db.getCollection("test_log");
DBCursor cursor = collection.find(new BasicDBObject(msg));
assertTrue(cursor.hasNext());
DBObject dbObject = cursor.next();
assertNotNull(dbObject);
assertEquals(dbObject.get("name"), msg.get("name"));
assertEquals(dbObject.get("age"), msg.get("age"));
assertEquals(dbObject.get("birthday"), msg.get("birthday"));
}
@Test
public void testMaxSuccessfulEvents() throws InterruptedException, EventDeliveryException {
StressSource source = new StressSource();
source.setChannelProcessor(mockProcessor);
Context context = new Context();
context.put("maxSuccessfulEvents", "35");
source.configure(context);
source.start();
for (int i = 0; i < 10; i++) {
source.process();
}
// 1 failed call, 10 successful
doThrow(new ChannelException("stub")).when(mockProcessor).processEvent(getEvent(source));
source.process();
doNothing().when(mockProcessor).processEvent(getEvent(source));
for (int i = 0; i < 10; i++) {
source.process();
}
// 1 failed call, 50 succesful
doThrow(new ChannelException("stub")).when(mockProcessor).processEvent(getEvent(source));
source.process();
doNothing().when(mockProcessor).processEvent(getEvent(source));
for (int i = 0; i < 50; i++) {
source.process();
}
// We should have called processEvent(evt) 37 times, twice for failures
// and twice for successful events.
verify(mockProcessor, times(37)).processEvent(getEvent(source));
}
@Test
public void testBatchEventsWithoutMatTotalEvents()
throws InterruptedException, EventDeliveryException {
StressSource source = new StressSource();
source.setChannelProcessor(mockProcessor);
Context context = new Context();
context.put("batchSize", "10");
source.configure(context);
source.start();
for (int i = 0; i < 10; i++) {
Assert.assertFalse(
"StressSource with no maxTotalEvents should not return " + Status.BACKOFF,
source.process() == Status.BACKOFF);
}
verify(mockProcessor, times(10)).processEventBatch(getLastProcessedEventList(source));
long successfulEvents = getCounterGroup(source).get("events.successful");
TestCase.assertTrue(
"Number of successful events should be 100 but was " + successfulEvents,
successfulEvents == 100);
long failedEvents = getCounterGroup(source).get("events.failed");
TestCase.assertTrue(
"Number of failure events should be 0 but was " + failedEvents, failedEvents == 0);
}
private RecoverableMemoryChannel createFileChannel() {
RecoverableMemoryChannel channel = new RecoverableMemoryChannel();
context = new Context();
context.put(RecoverableMemoryChannel.WAL_DATA_DIR, dataDir.getAbsolutePath());
Configurables.configure(channel, context);
channel.start();
return channel;
}
@Test
public void shouldUseSpecifiedSerializer() throws Exception {
Context context = new Context();
context.put(SERIALIZER, "org.apache.flume.sink.elasticsearch.FakeEventSerializer");
assertNull(fixture.getEventSerializer());
fixture.configure(context);
assertTrue(fixture.getEventSerializer() instanceof FakeEventSerializer);
}
@Test
public void testRollbackWithSink() throws Exception {
final NullSink nullSink = new NullSink();
Context ctx = new Context();
ctx.put("batchSize", "1");
nullSink.configure(ctx);
nullSink.setChannel(channel);
final int numItems = 99;
Thread t =
new Thread() {
@Override
public void run() {
int count = 0;
while (count++ < numItems) {
try {
nullSink.process();
Thread.sleep(1);
} catch (EventDeliveryException e) {
break;
} catch (Exception e) {
Throwables.propagate(e);
}
}
}
};
t.setDaemon(true);
t.setName("NullSink");
t.start();
putEvents(channel, "rollback", 10, 100);
Transaction transaction;
// put an item we will rollback
transaction = channel.getTransaction();
transaction.begin();
channel.put(EventBuilder.withBody("this is going to be rolledback".getBytes(Charsets.UTF_8)));
transaction.rollback();
transaction.close();
while (t.isAlive()) {
Thread.sleep(1);
}
// simulate crash
channel.stop();
channel = createFileChannel();
// get the item which was not rolled back
transaction = channel.getTransaction();
transaction.begin();
Event event = channel.take();
transaction.commit();
transaction.close();
Assert.assertNotNull(event);
Assert.assertEquals("rollback-90-9", new String(event.getBody(), Charsets.UTF_8));
}
@BeforeMethod(groups = {"dev"})
public static void setup() throws UnknownHostException {
mongo = new Mongo("localhost", 27017);
Map<String, String> ctxMap = new HashMap<String, String>();
ctxMap.put(MongoSink.HOST, "localhost");
ctxMap.put(MongoSink.PORT, "27017");
ctxMap.put(MongoSink.DB_NAME, "test_events");
ctxMap.put(MongoSink.COLLECTION, "test_log");
ctxMap.put(MongoSink.BATCH_SIZE, "100");
ctx.putAll(ctxMap);
Context channelCtx = new Context();
channelCtx.put("capacity", "1000000");
channelCtx.put("transactionCapacity", "1000000");
channel = new MemoryChannel();
Configurables.configure(channel, channelCtx);
}
@Test
public void shouldUseSpecifiedIndexNameBuilder() throws Exception {
Context context = new Context();
context.put(
ElasticSearchSinkConstants.INDEX_NAME_BUILDER,
"org.apache.flume.sink.elasticsearch.FakeIndexNameBuilder");
assertNull(fixture.getIndexNameBuilder());
fixture.configure(context);
assertTrue(fixture.getIndexNameBuilder() instanceof FakeIndexNameBuilder);
}
@Test(expected = RestSourceException.class)
public void testEmptyJson() throws Exception {
try {
context.put("urlJson", EMPTY_URL_JSON);
dynamicUrlHandler = new DynamicUrlHandler();
dynamicUrlHandler.configure(context);
} catch (RestSourceException e) {
}
}
@Test(expected = NullPointerException.class)
public void testNoJson() throws Exception {
try {
context.put("urlJson", "");
dynamicUrlHandler = new DynamicUrlHandler();
dynamicUrlHandler.configure(context);
} catch (NullPointerException e) {
}
}
@Test(expected = RestSourceException.class)
public void testMethod() throws Exception {
try {
context.put("urlJson", URL_JSON);
dynamicUrlHandler = new DynamicUrlHandler();
dynamicUrlHandler.configure(context);
} catch (RestSourceException e) {
} finally {
dynamicUrlHandler.buildUrl(properties);
}
}
@Test
public void testMaxTotalEvents() throws InterruptedException, EventDeliveryException {
StressSource source = new StressSource();
source.setChannelProcessor(mockProcessor);
Context context = new Context();
context.put("maxTotalEvents", "35");
source.configure(context);
source.start();
for (int i = 0; i < 50; i++) {
source.process();
}
verify(mockProcessor, times(35)).processEvent(getEvent(source));
}
@Test(groups = "dev")
public void sinkDynamicDbTest() throws EventDeliveryException {
ctx.put(MongoSink.MODEL, MongoSink.CollectionModel.dynamic.name());
MongoSink sink = new MongoSink();
Configurables.configure(sink, ctx);
sink.setChannel(channel);
sink.start();
JSONObject msg = new JSONObject();
msg.put("age", 11);
msg.put("birthday", new Date().getTime());
Transaction tx;
for (int i = 0; i < 10; i++) {
tx = channel.getTransaction();
tx.begin();
msg.put("name", "test" + i);
JSONObject header = new JSONObject();
header.put(MongoSink.COLLECTION, "my_events");
header.put(MongoSink.DB_NAME, "dynamic_db");
Event e = EventBuilder.withBody(msg.toJSONString().getBytes(), header);
channel.put(e);
tx.commit();
tx.close();
}
sink.process();
sink.stop();
for (int i = 0; i < 10; i++) {
msg.put("name", "test" + i);
System.out.println("i = " + i);
DB db = mongo.getDB("dynamic_db");
DBCollection collection = db.getCollection("my_events");
DBCursor cursor = collection.find(new BasicDBObject(msg));
assertTrue(cursor.hasNext());
DBObject dbObject = cursor.next();
assertNotNull(dbObject);
assertEquals(dbObject.get("name"), msg.get("name"));
assertEquals(dbObject.get("age"), msg.get("age"));
assertEquals(dbObject.get("birthday"), msg.get("birthday"));
}
}
@Override
public synchronized void configure(Context context) {
spoolDirectory = context.getString(SPOOL_DIRECTORY);
Preconditions.checkState(
spoolDirectory != null, "Configuration must specify a spooling directory");
completedSuffix = context.getString(SPOOLED_FILE_SUFFIX, DEFAULT_SPOOLED_FILE_SUFFIX);
deletePolicy = context.getString(DELETE_POLICY, DEFAULT_DELETE_POLICY);
fileHeader = context.getBoolean(FILENAME_HEADER, DEFAULT_FILE_HEADER);
fileHeaderKey = context.getString(FILENAME_HEADER_KEY, DEFAULT_FILENAME_HEADER_KEY);
basenameHeader = context.getBoolean(BASENAME_HEADER, DEFAULT_BASENAME_HEADER);
basenameHeaderKey = context.getString(BASENAME_HEADER_KEY, DEFAULT_BASENAME_HEADER_KEY);
batchSize = context.getInteger(BATCH_SIZE, DEFAULT_BATCH_SIZE);
inputCharset = context.getString(INPUT_CHARSET, DEFAULT_INPUT_CHARSET);
decodeErrorPolicy =
DecodeErrorPolicy.valueOf(
context
.getString(DECODE_ERROR_POLICY, DEFAULT_DECODE_ERROR_POLICY)
.toUpperCase(Locale.ENGLISH));
ignorePattern = context.getString(IGNORE_PAT, DEFAULT_IGNORE_PAT);
trackerDirPath = context.getString(TRACKER_DIR, DEFAULT_TRACKER_DIR);
deserializerType = context.getString(DESERIALIZER, "ZipDeserializer");
deserializerContext = new Context(context.getSubProperties(DESERIALIZER + "."));
consumeOrder =
ConsumeOrder.valueOf(
context
.getString(CONSUME_ORDER, DEFAULT_CONSUME_ORDER.toString())
.toUpperCase(Locale.ENGLISH));
// "Hack" to support backwards compatibility with previous generation of
// spooling directory source, which did not support deserializers
Integer bufferMaxLineLength = context.getInteger(BUFFER_MAX_LINE_LENGTH);
if (bufferMaxLineLength != null
&& deserializerType != null
&& deserializerType.equalsIgnoreCase(DEFAULT_DESERIALIZER)) {
deserializerContext.put(LineDeserializer.MAXLINE_KEY, bufferMaxLineLength.toString());
}
maxBackoff = context.getInteger(MAX_BACKOFF, DEFAULT_MAX_BACKOFF);
if (sourceCounter == null) {
sourceCounter = new SourceCounter(getName());
}
}
public void createAvroFile(File file, String codec, boolean useSchemaUrl) throws IOException {
// serialize a few events using the reflection-based avro serializer
OutputStream out = new FileOutputStream(file);
Context ctx = new Context();
if (codec != null) {
ctx.put("compressionCodec", codec);
}
Schema schema = Schema.createRecord("myrecord", null, null, false);
schema.setFields(
Arrays.asList(
new Schema.Field[] {
new Schema.Field("message", Schema.create(Schema.Type.STRING), null, null)
}));
GenericRecordBuilder recordBuilder = new GenericRecordBuilder(schema);
File schemaFile = null;
if (useSchemaUrl) {
schemaFile = File.createTempFile(getClass().getSimpleName(), ".avsc");
Files.write(schema.toString(), schemaFile, Charsets.UTF_8);
}
EventSerializer.Builder builder = new AvroEventSerializer.Builder();
EventSerializer serializer = builder.build(ctx, out);
serializer.afterCreate();
for (int i = 0; i < 3; i++) {
GenericRecord record = recordBuilder.set("message", "Hello " + i).build();
Event event = EventBuilder.withBody(serializeAvro(record, schema));
if (schemaFile == null) {
event.getHeaders().put(AvroEventSerializer.AVRO_SCHEMA_LITERAL_HEADER, schema.toString());
} else {
event
.getHeaders()
.put(
AvroEventSerializer.AVRO_SCHEMA_URL_HEADER,
schemaFile.toURI().toURL().toExternalForm());
}
serializer.write(event);
}
serializer.flush();
serializer.beforeClose();
out.flush();
out.close();
}
@SuppressWarnings("unchecked")
@Test
public void ThrowableThrownExceptionTest() throws EventDeliveryException {
Jedis jedis = mock(Jedis.class);
// Not really true, but fits the requirement
when(jedis.lpush(any(byte[].class), any(byte[].class))).thenThrow(clazz);
JedisPool jedisPool = mock(JedisPool.class);
MockJedisPoolFactory mockJedisPoolFactory = new MockJedisPoolFactory(jedisPool, jedis);
Channel channel = mock(Channel.class);
Transaction transactionMock = mock(Transaction.class);
when(channel.getTransaction()).thenReturn(transactionMock);
Event testEvent = new SimpleEvent();
byte[] testBody = new byte[] {'b', 'o', 'd', 'y'};
testEvent.setBody(testBody);
when(channel.take()).thenReturn(testEvent);
RedisSink redisSink = new RedisSink(mockJedisPoolFactory);
redisSink.setChannel(channel);
Context context = new Context();
context.put(RedisSinkConfigurationConstant.HOST, "localhost");
redisSink.configure(context);
redisSink.start();
redisSink.process();
verify(channel, times(1)).getTransaction();
verify(channel, times(1)).take();
verify(transactionMock, times(1)).begin();
verify(transactionMock, times(1)).close();
verify(transactionMock, times(0)).commit();
verify(transactionMock, times(1)).rollback();
verify(jedisPool, times(1)).getResource();
verify(jedisPool, times(1)).returnResource(jedis);
verify(jedis, times(1))
.lpush(eq(RedisSinkConfigurationConstant.DEFAULT_KEY.getBytes()), any(byte[][].class));
}
public void doTestTextBatchAppend() throws Exception {
LOG.debug("Starting...");
final long rollCount = 10;
final long batchSize = 2;
final String fileName = "PageView";
String newPath = testPath + "/singleTextBucket";
int totalEvents = 0;
int i = 1, j = 1;
// clear the test directory
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
Path dirPath = new Path(newPath);
fs.delete(dirPath, true);
fs.mkdirs(dirPath);
Context context = new Context();
context.put("hdfs.path", newPath);
context.put("hdfs.rollCount", String.valueOf(rollCount));
context.put("hdfs.batchSize", String.valueOf(batchSize));
context.put("hdfs.filePrefix", "pageview");
Channel channel = new MemoryChannel();
Configurables.configure(channel, context);
sink.setChannel(channel);
sink.start();
Calendar eventDate = Calendar.getInstance();
Date currentDate = new Date();
Map<String, String> header = new HashMap<String, String>();
header.put("topic", "PageView");
List<String> bodies = Lists.newArrayList();
// 将测试的事件推入到通道中
for (i = 1; i <= (rollCount * 10) / batchSize; i++) {
Transaction txn = channel.getTransaction();
txn.begin();
for (j = 1; j <= batchSize; j++) {
header.put("timestamp", String.valueOf(currentDate.getTime()));
Event event = new SimpleEvent();
eventDate.clear();
eventDate.set(2014, i, i, i, 0);
String body = "Test." + i + "." + j;
event.setHeaders(header);
event.setBody(body.getBytes());
bodies.add(body);
channel.put(event);
totalEvents++;
}
txn.commit();
txn.close();
// execute sink to process the events
sink.process();
}
sink.stop();
FileStatus[] dirStat = fs.listStatus(dirPath);
Path fList[] = FileUtil.stat2Paths(dirStat);
long expectedFiles = totalEvents / rollCount;
if (totalEvents % rollCount > 0) {
expectedFiles++;
}
Assert.assertEquals(
"num files wrong, found: " + Lists.newArrayList(fList), expectedFiles, fList.length);
// 检查所有写入文件的内容
verifyOutputTextFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);
}
@Override
public void configure(Context context) {
if (!isLocal) {
if (StringUtils.isNotBlank(context.getString(HOSTNAMES))) {
serverAddresses = StringUtils.deleteWhitespace(context.getString(HOSTNAMES)).split(",");
}
Preconditions.checkState(
serverAddresses != null && serverAddresses.length > 0, "Missing Param:" + HOSTNAMES);
}
if (StringUtils.isNotBlank(context.getString(INDEX_NAME))) {
this.indexName = context.getString(INDEX_NAME);
}
if (StringUtils.isNotBlank(context.getString(INDEX_TYPE))) {
this.indexType = context.getString(INDEX_TYPE);
}
if (StringUtils.isNotBlank(context.getString(CLUSTER_NAME))) {
this.clusterName = context.getString(CLUSTER_NAME);
}
if (StringUtils.isNotBlank(context.getString(BATCH_SIZE))) {
this.batchSize = Integer.parseInt(context.getString(BATCH_SIZE));
}
if (StringUtils.isNotBlank(context.getString(TTL))) {
this.ttlMs = parseTTL(context.getString(TTL));
Preconditions.checkState(ttlMs > 0, TTL + " must be greater than 0 or not set.");
}
if (StringUtils.isNotBlank(context.getString(CLIENT_TYPE))) {
clientType = context.getString(CLIENT_TYPE);
}
elasticSearchClientContext = new Context();
elasticSearchClientContext.putAll(context.getSubProperties(CLIENT_PREFIX));
String serializerClazz = DEFAULT_SERIALIZER_CLASS;
if (StringUtils.isNotBlank(context.getString(SERIALIZER))) {
serializerClazz = context.getString(SERIALIZER);
}
Context serializerContext = new Context();
serializerContext.putAll(context.getSubProperties(SERIALIZER_PREFIX));
try {
@SuppressWarnings("unchecked")
Class<? extends Configurable> clazz =
(Class<? extends Configurable>) Class.forName(serializerClazz);
Configurable serializer = clazz.newInstance();
if (serializer instanceof ElasticSearchIndexRequestBuilderFactory) {
indexRequestFactory = (ElasticSearchIndexRequestBuilderFactory) serializer;
indexRequestFactory.configure(serializerContext);
} else if (serializer instanceof ElasticSearchEventSerializer) {
eventSerializer = (ElasticSearchEventSerializer) serializer;
eventSerializer.configure(serializerContext);
} else {
throw new IllegalArgumentException(
serializerClazz + " is not an ElasticSearchEventSerializer");
}
} catch (Exception e) {
logger.error("Could not instantiate event serializer.", e);
Throwables.propagate(e);
}
if (sinkCounter == null) {
sinkCounter = new SinkCounter(getName());
}
String indexNameBuilderClass = DEFAULT_INDEX_NAME_BUILDER_CLASS;
if (StringUtils.isNotBlank(context.getString(INDEX_NAME_BUILDER))) {
indexNameBuilderClass = context.getString(INDEX_NAME_BUILDER);
}
Context indexnameBuilderContext = new Context();
indexnameBuilderContext.putAll(context.getSubProperties(INDEX_NAME_BUILDER_PREFIX));
try {
@SuppressWarnings("unchecked")
Class<? extends IndexNameBuilder> clazz =
(Class<? extends IndexNameBuilder>) Class.forName(indexNameBuilderClass);
indexNameBuilder = clazz.newInstance();
indexnameBuilderContext.put(INDEX_NAME, indexName);
indexNameBuilder.configure(indexnameBuilderContext);
} catch (Exception e) {
logger.error("Could not instantiate index name builder.", e);
Throwables.propagate(e);
}
if (sinkCounter == null) {
sinkCounter = new SinkCounter(getName());
}
Preconditions.checkState(StringUtils.isNotBlank(indexName), "Missing Param:" + INDEX_NAME);
Preconditions.checkState(StringUtils.isNotBlank(indexType), "Missing Param:" + INDEX_TYPE);
Preconditions.checkState(StringUtils.isNotBlank(clusterName), "Missing Param:" + CLUSTER_NAME);
Preconditions.checkState(batchSize >= 1, BATCH_SIZE + " must be greater than 0");
}
/**
* Creates a Flume context for Mongo/STH sinks.
*
* @param collectionPrefix
* @param dbPrefix
* @param dataModel
* @param enableEncoding
* @return A Flume context for Mongo/STH sinks.
*/
public static Context createContextForMongoSTH(
String collectionPrefix, String dbPrefix, String dataModel, String enableEncoding) {
Context context = new Context();
context.put("attr_persistence", "row");
context.put("batch_size", "100");
context.put("batch_timeout", "30");
context.put("batch_ttl", "10");
context.put("collection_prefix", collectionPrefix);
context.put("collection_size", "0");
context.put("data_expiration", "0");
context.put("data_model", dataModel);
context.put("db_prefix", dbPrefix);
context.put("enable_encoding", enableEncoding);
context.put("enable_grouping", "false");
context.put("enable_lowercase", "false");
context.put("max_documents", "0");
context.put("mongo_hosts", "localhost:27017");
context.put("mongo_password", "");
context.put("mongo_username", "");
return context;
} // createContextForMongoSTH
// @Ignore
@Before
public void setUp() {
conf = ConfigFactory.load();
List<String> zkHosts = conf.getStringList("zookeeper.hosts");
for (String host : zkHosts) {
ZOOKEEPER_HOSTS += host + ",";
}
ZOOKEEPER_HOSTS = ZOOKEEPER_HOSTS.substring(0, ZOOKEEPER_HOSTS.length() - 1);
List<String> kafkaHosts = conf.getStringList("kafka.hosts");
for (String host : kafkaHosts) {
KAFKA_HOSTS += host + ",";
}
KAFKA_HOSTS = KAFKA_HOSTS.substring(0, KAFKA_HOSTS.length() - 1);
LOGGER.debug("Using Zookeeper hosts: " + ZOOKEEPER_HOSTS);
LOGGER.debug("Using Zookeeper hosts: " + KAFKA_HOSTS);
// try {
// zookeeperServer = new ZookeeperServer();
// zookeeperServer.start();
// } catch (Exception e) {
// e.printStackTrace();
// }
// try {
// kafkaServer = new KafkaServer();
// kafkaServer.start();
// } catch (Exception e) {
// e.printStackTrace();
// }
String[] connection = KAFKA_HOSTS.split(":");
// simpleConsumer = new SimpleConsumer("localhost", 9092, 60000, 1024, CLIENT_ID);
simpleConsumer =
new SimpleConsumer(connection[0], Integer.parseInt(connection[1]), 60000, 1024, CLIENT_ID);
kafkaSink = new KafkaSink();
Context kafkaContext = new Context();
kafkaContext.put("topic", "test");
kafkaContext.put("writeBody", "false");
kafkaContext.put("kafka.metadata.broker.list", KAFKA_HOSTS);
kafkaContext.put("kafka.serializer.class", "kafka.serializer.StringEncoder");
Configurables.configure(kafkaSink, kafkaContext);
Context channelContext = new Context();
channelContext.put("capacity", "10000");
channelContext.put("transactionCapacity", "200");
channel = new MemoryChannel();
channel.setName("junitChannel");
Configurables.configure(channel, channelContext);
kafkaSink.setChannel(channel);
channel.start();
kafkaSink.start();
}
