@Test(groups = "dev")
  public void autoWrapTest() throws EventDeliveryException {
    ctx.put(MongoSink.AUTO_WRAP, Boolean.toString(true));
    ctx.put(MongoSink.DB_NAME, "test_wrap");

    MongoSink sink = new MongoSink();
    Configurables.configure(sink, ctx);

    sink.setChannel(channel);
    sink.start();

    Transaction tx = channel.getTransaction();
    tx.begin();
    String msg =
        "2012/10/26 11:23:08 [error] 7289#0: *6430831 open() \"/usr/local/nginx/html/50x.html\" failed (2: No such file or directory), client: 10.160.105.161, server: sg15.redatoms.com, request: \"POST /mojo/ajax/embed HTTP/1.0\", upstream: \"fastcgi://unix:/tmp/php-fpm.sock:\", host: \"sg15.redatoms.com\", referrer: \"http://sg15.redatoms.com/mojo/mobile/package\"";

    Event e = EventBuilder.withBody(msg.getBytes());
    channel.put(e);
    tx.commit();
    tx.close();

    sink.process();
    sink.stop();

    DB db = mongo.getDB("test_wrap");
    DBCollection collection = db.getCollection("test_log");
    DBCursor cursor = collection.find(new BasicDBObject(MongoSink.DEFAULT_WRAP_FIELD, msg));
    assertTrue(cursor.hasNext());
    DBObject dbObject = cursor.next();
    assertNotNull(dbObject);
    assertEquals(dbObject.get(MongoSink.DEFAULT_WRAP_FIELD), msg);
    mongo.dropDatabase("test_wrap");
  }
  /** Verify that setting a timeout in the configuration does not impact the database setting */
  @Test
  public void timeoutConfiguredTest() {

    String host = "localhost";
    int timeout = 10;

    JedisPoolFactory mockJedisPoolFactory = mock(JedisPoolFactory.class);

    RedisSink redisSink = new RedisSink(mockJedisPoolFactory);

    Channel channel = mock(Channel.class);
    redisSink.setChannel(channel);

    Context context = new Context();
    context.put(RedisSinkConfigurationConstant.HOST, host);
    context.put(RedisSinkConfigurationConstant.BATCH_SIZE, "10");
    context.put(RedisSinkConfigurationConstant.TIMEOUT, Integer.toString(timeout));
    redisSink.configure(context);

    redisSink.start();

    verify(mockJedisPoolFactory)
        .create(
            any(JedisPoolConfig.class),
            eq(host),
            eq(Protocol.DEFAULT_PORT),
            eq(timeout),
            isNull(String.class),
            eq(Protocol.DEFAULT_DATABASE));
  }
Beispiel #3
0
  public void SequenceFileRenameRetryCoreTest(int numberOfRetriesRequired, boolean closeSucceed)
      throws Exception {
    String hdfsPath =
        "file:///tmp/flume-test."
            + Calendar.getInstance().getTimeInMillis()
            + "."
            + Thread.currentThread().getId();

    Context context = new Context();
    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(conf);
    Path dirPath = new Path(hdfsPath);
    fs.delete(dirPath, true);
    fs.mkdirs(dirPath);
    context.put("hdfs.path", hdfsPath);
    context.put("hdfs.closeTries", String.valueOf(numberOfRetriesRequired));
    context.put("hdfs.rollCount", "1");
    context.put("hdfs.retryInterval", "1");
    context.put("hdfs.callTimeout", Long.toString(1000));
    MockFileSystem mockFs = new MockFileSystem(fs, numberOfRetriesRequired, closeSucceed);
    BucketWriter bucketWriter =
        new BucketWriter(
            0,
            0,
            1,
            1,
            ctx,
            hdfsPath,
            hdfsPath,
            "singleBucket",
            ".tmp",
            null,
            null,
            null,
            new MockDataStream(mockFs),
            timedRollerPool,
            proxy,
            new SinkCounter("test-bucket-writer-" + System.currentTimeMillis()),
            0,
            null,
            null,
            30000,
            Executors.newSingleThreadExecutor(),
            1,
            numberOfRetriesRequired);

    bucketWriter.setFileSystem(mockFs);
    // At this point, we checked if isFileClosed is available in
    // this JVM, so lets make it check again.
    Event event = EventBuilder.withBody("test", Charsets.UTF_8);
    bucketWriter.append(event);
    // This is what triggers the close, so a 2nd append is required :/
    bucketWriter.append(event);

    TimeUnit.SECONDS.sleep(numberOfRetriesRequired + 2);

    Assert.assertTrue(
        "Expected " + numberOfRetriesRequired + " " + "but got " + bucketWriter.renameTries.get(),
        bucketWriter.renameTries.get() == numberOfRetriesRequired);
  }
Beispiel #4
0
  @Test
  public void testBatchEvents() throws InterruptedException, EventDeliveryException {
    StressSource source = new StressSource();
    source.setChannelProcessor(mockProcessor);
    Context context = new Context();
    context.put("maxTotalEvents", "35");
    context.put("batchSize", "10");
    source.configure(context);
    source.start();

    for (int i = 0; i < 50; i++) {
      if (source.process() == Status.BACKOFF) {
        TestCase.assertTrue("Source should have sent all events in 4 batches", i == 4);
        break;
      }
      if (i < 3) {
        verify(mockProcessor, times(i + 1)).processEventBatch(getLastProcessedEventList(source));
      } else {
        verify(mockProcessor, times(1)).processEventBatch(getLastProcessedEventList(source));
      }
    }
    long successfulEvents = getCounterGroup(source).get("events.successful");
    TestCase.assertTrue(
        "Number of successful events should be 35 but was " + successfulEvents,
        successfulEvents == 35);
    long failedEvents = getCounterGroup(source).get("events.failed");
    TestCase.assertTrue(
        "Number of failure events should be 0 but was " + failedEvents, failedEvents == 0);
  }
Beispiel #5
0
  // @Test
  public void testLifecycle() throws InterruptedException, LifecycleException {
    LOG.debug("Starting...");
    Context context = new Context();
    context.put("hdfs.path", testPath);
    context.put("hdfs.filePrefix", "pageview");

    Configurables.configure(sink, context);
    sink.setChannel(new MemoryChannel());
    sink.start();
    sink.stop();
  }
  @Test(groups = "dev")
  public void timestampExistingFieldTest() throws EventDeliveryException, ParseException {
    ctx.put(MongoSink.MODEL, MongoSink.CollectionModel.dynamic.name());
    String tsField = "createdOn";
    ctx.put(MongoSink.TIMESTAMP_FIELD, tsField);
    MongoSink sink = new MongoSink();
    Configurables.configure(sink, ctx);

    sink.setChannel(channel);
    sink.start();

    JSONObject msg = new JSONObject();
    msg.put("age", 11);
    msg.put("birthday", new Date().getTime());
    String dateText = "2013-02-19T14:20:53+08:00";
    msg.put(tsField, dateText);

    Transaction tx;

    for (int i = 0; i < 10; i++) {
      tx = channel.getTransaction();
      tx.begin();
      msg.put("name", "test" + i);
      JSONObject header = new JSONObject();
      header.put(MongoSink.COLLECTION, "my_events");
      header.put(MongoSink.DB_NAME, "dynamic_db");

      Event e = EventBuilder.withBody(msg.toJSONString().getBytes(), header);
      channel.put(e);
      tx.commit();
      tx.close();
    }
    sink.process();
    sink.stop();

    msg.put(tsField, MongoSink.dateTimeFormatter.parseDateTime(dateText).toDate());
    for (int i = 0; i < 10; i++) {
      msg.put("name", "test" + i);

      System.out.println("i = " + i);

      DB db = mongo.getDB("dynamic_db");
      DBCollection collection = db.getCollection("my_events");
      DBCursor cursor = collection.find(new BasicDBObject(msg));
      assertTrue(cursor.hasNext());
      DBObject dbObject = cursor.next();
      assertNotNull(dbObject);
      assertEquals(dbObject.get("name"), msg.get("name"));
      assertEquals(dbObject.get("age"), msg.get("age"));
      assertEquals(dbObject.get("birthday"), msg.get("birthday"));
      assertTrue(dbObject.get(tsField) instanceof Date);
      System.out.println("ts = " + dbObject.get(tsField));
    }
  }
  @Test(groups = "dev")
  public void sinkSingleModelTest() throws EventDeliveryException {
    ctx.put(MongoSink.MODEL, MongoSink.CollectionModel.single.name());

    MongoSink sink = new MongoSink();
    Configurables.configure(sink, ctx);

    sink.setChannel(channel);
    sink.start();

    Transaction tx = channel.getTransaction();
    tx.begin();
    JSONObject msg = new JSONObject();
    msg.put("name", "test");
    msg.put("age", 11);
    msg.put("birthday", new Date().getTime());

    Event e = EventBuilder.withBody(msg.toJSONString().getBytes());
    channel.put(e);
    tx.commit();
    tx.close();

    sink.process();
    sink.stop();

    DB db = mongo.getDB("test_events");
    DBCollection collection = db.getCollection("test_log");
    DBCursor cursor = collection.find(new BasicDBObject(msg));
    assertTrue(cursor.hasNext());
    DBObject dbObject = cursor.next();
    assertNotNull(dbObject);
    assertEquals(dbObject.get("name"), msg.get("name"));
    assertEquals(dbObject.get("age"), msg.get("age"));
    assertEquals(dbObject.get("birthday"), msg.get("birthday"));
  }
Beispiel #8
0
  @Test
  public void testMaxSuccessfulEvents() throws InterruptedException, EventDeliveryException {
    StressSource source = new StressSource();
    source.setChannelProcessor(mockProcessor);
    Context context = new Context();
    context.put("maxSuccessfulEvents", "35");
    source.configure(context);
    source.start();

    for (int i = 0; i < 10; i++) {
      source.process();
    }

    // 1 failed call, 10 successful
    doThrow(new ChannelException("stub")).when(mockProcessor).processEvent(getEvent(source));
    source.process();
    doNothing().when(mockProcessor).processEvent(getEvent(source));
    for (int i = 0; i < 10; i++) {
      source.process();
    }

    // 1 failed call, 50 succesful
    doThrow(new ChannelException("stub")).when(mockProcessor).processEvent(getEvent(source));
    source.process();
    doNothing().when(mockProcessor).processEvent(getEvent(source));
    for (int i = 0; i < 50; i++) {
      source.process();
    }

    // We should have called processEvent(evt) 37 times, twice for failures
    // and twice for successful events.
    verify(mockProcessor, times(37)).processEvent(getEvent(source));
  }
Beispiel #9
0
  @Test
  public void testBatchEventsWithoutMatTotalEvents()
      throws InterruptedException, EventDeliveryException {
    StressSource source = new StressSource();
    source.setChannelProcessor(mockProcessor);
    Context context = new Context();
    context.put("batchSize", "10");
    source.configure(context);
    source.start();

    for (int i = 0; i < 10; i++) {
      Assert.assertFalse(
          "StressSource with no maxTotalEvents should not return " + Status.BACKOFF,
          source.process() == Status.BACKOFF);
    }
    verify(mockProcessor, times(10)).processEventBatch(getLastProcessedEventList(source));

    long successfulEvents = getCounterGroup(source).get("events.successful");
    TestCase.assertTrue(
        "Number of successful events should be 100 but was " + successfulEvents,
        successfulEvents == 100);

    long failedEvents = getCounterGroup(source).get("events.failed");
    TestCase.assertTrue(
        "Number of failure events should be 0 but was " + failedEvents, failedEvents == 0);
  }
 private RecoverableMemoryChannel createFileChannel() {
   RecoverableMemoryChannel channel = new RecoverableMemoryChannel();
   context = new Context();
   context.put(RecoverableMemoryChannel.WAL_DATA_DIR, dataDir.getAbsolutePath());
   Configurables.configure(channel, context);
   channel.start();
   return channel;
 }
Beispiel #11
0
  @Test
  public void shouldUseSpecifiedSerializer() throws Exception {
    Context context = new Context();
    context.put(SERIALIZER, "org.apache.flume.sink.elasticsearch.FakeEventSerializer");

    assertNull(fixture.getEventSerializer());
    fixture.configure(context);
    assertTrue(fixture.getEventSerializer() instanceof FakeEventSerializer);
  }
  @Test
  public void testRollbackWithSink() throws Exception {
    final NullSink nullSink = new NullSink();
    Context ctx = new Context();
    ctx.put("batchSize", "1");
    nullSink.configure(ctx);
    nullSink.setChannel(channel);
    final int numItems = 99;
    Thread t =
        new Thread() {
          @Override
          public void run() {
            int count = 0;
            while (count++ < numItems) {
              try {
                nullSink.process();
                Thread.sleep(1);
              } catch (EventDeliveryException e) {
                break;
              } catch (Exception e) {
                Throwables.propagate(e);
              }
            }
          }
        };
    t.setDaemon(true);
    t.setName("NullSink");
    t.start();

    putEvents(channel, "rollback", 10, 100);

    Transaction transaction;
    // put an item we will rollback
    transaction = channel.getTransaction();
    transaction.begin();
    channel.put(EventBuilder.withBody("this is going to be rolledback".getBytes(Charsets.UTF_8)));
    transaction.rollback();
    transaction.close();

    while (t.isAlive()) {
      Thread.sleep(1);
    }

    // simulate crash
    channel.stop();
    channel = createFileChannel();

    // get the item which was not rolled back
    transaction = channel.getTransaction();
    transaction.begin();
    Event event = channel.take();
    transaction.commit();
    transaction.close();
    Assert.assertNotNull(event);
    Assert.assertEquals("rollback-90-9", new String(event.getBody(), Charsets.UTF_8));
  }
  @BeforeMethod(groups = {"dev"})
  public static void setup() throws UnknownHostException {
    mongo = new Mongo("localhost", 27017);

    Map<String, String> ctxMap = new HashMap<String, String>();
    ctxMap.put(MongoSink.HOST, "localhost");
    ctxMap.put(MongoSink.PORT, "27017");
    ctxMap.put(MongoSink.DB_NAME, "test_events");
    ctxMap.put(MongoSink.COLLECTION, "test_log");
    ctxMap.put(MongoSink.BATCH_SIZE, "100");

    ctx.putAll(ctxMap);

    Context channelCtx = new Context();
    channelCtx.put("capacity", "1000000");
    channelCtx.put("transactionCapacity", "1000000");
    channel = new MemoryChannel();
    Configurables.configure(channel, channelCtx);
  }
Beispiel #14
0
  @Test
  public void shouldUseSpecifiedIndexNameBuilder() throws Exception {
    Context context = new Context();
    context.put(
        ElasticSearchSinkConstants.INDEX_NAME_BUILDER,
        "org.apache.flume.sink.elasticsearch.FakeIndexNameBuilder");

    assertNull(fixture.getIndexNameBuilder());
    fixture.configure(context);
    assertTrue(fixture.getIndexNameBuilder() instanceof FakeIndexNameBuilder);
  }
  @Test(expected = RestSourceException.class)
  public void testEmptyJson() throws Exception {
    try {
      context.put("urlJson", EMPTY_URL_JSON);

      dynamicUrlHandler = new DynamicUrlHandler();
      dynamicUrlHandler.configure(context);

    } catch (RestSourceException e) {
    }
  }
  @Test(expected = NullPointerException.class)
  public void testNoJson() throws Exception {
    try {
      context.put("urlJson", "");

      dynamicUrlHandler = new DynamicUrlHandler();
      dynamicUrlHandler.configure(context);

    } catch (NullPointerException e) {
    }
  }
  @Test(expected = RestSourceException.class)
  public void testMethod() throws Exception {
    try {
      context.put("urlJson", URL_JSON);

      dynamicUrlHandler = new DynamicUrlHandler();
      dynamicUrlHandler.configure(context);

    } catch (RestSourceException e) {
    } finally {
      dynamicUrlHandler.buildUrl(properties);
    }
  }
Beispiel #18
0
  @Test
  public void testMaxTotalEvents() throws InterruptedException, EventDeliveryException {
    StressSource source = new StressSource();
    source.setChannelProcessor(mockProcessor);
    Context context = new Context();
    context.put("maxTotalEvents", "35");
    source.configure(context);
    source.start();

    for (int i = 0; i < 50; i++) {
      source.process();
    }
    verify(mockProcessor, times(35)).processEvent(getEvent(source));
  }
  @Test(groups = "dev")
  public void sinkDynamicDbTest() throws EventDeliveryException {
    ctx.put(MongoSink.MODEL, MongoSink.CollectionModel.dynamic.name());
    MongoSink sink = new MongoSink();
    Configurables.configure(sink, ctx);

    sink.setChannel(channel);
    sink.start();

    JSONObject msg = new JSONObject();
    msg.put("age", 11);
    msg.put("birthday", new Date().getTime());

    Transaction tx;

    for (int i = 0; i < 10; i++) {
      tx = channel.getTransaction();
      tx.begin();
      msg.put("name", "test" + i);
      JSONObject header = new JSONObject();
      header.put(MongoSink.COLLECTION, "my_events");
      header.put(MongoSink.DB_NAME, "dynamic_db");

      Event e = EventBuilder.withBody(msg.toJSONString().getBytes(), header);
      channel.put(e);
      tx.commit();
      tx.close();
    }
    sink.process();
    sink.stop();

    for (int i = 0; i < 10; i++) {
      msg.put("name", "test" + i);

      System.out.println("i = " + i);

      DB db = mongo.getDB("dynamic_db");
      DBCollection collection = db.getCollection("my_events");
      DBCursor cursor = collection.find(new BasicDBObject(msg));
      assertTrue(cursor.hasNext());
      DBObject dbObject = cursor.next();
      assertNotNull(dbObject);
      assertEquals(dbObject.get("name"), msg.get("name"));
      assertEquals(dbObject.get("age"), msg.get("age"));
      assertEquals(dbObject.get("birthday"), msg.get("birthday"));
    }
  }
  @Override
  public synchronized void configure(Context context) {
    spoolDirectory = context.getString(SPOOL_DIRECTORY);
    Preconditions.checkState(
        spoolDirectory != null, "Configuration must specify a spooling directory");

    completedSuffix = context.getString(SPOOLED_FILE_SUFFIX, DEFAULT_SPOOLED_FILE_SUFFIX);
    deletePolicy = context.getString(DELETE_POLICY, DEFAULT_DELETE_POLICY);
    fileHeader = context.getBoolean(FILENAME_HEADER, DEFAULT_FILE_HEADER);
    fileHeaderKey = context.getString(FILENAME_HEADER_KEY, DEFAULT_FILENAME_HEADER_KEY);
    basenameHeader = context.getBoolean(BASENAME_HEADER, DEFAULT_BASENAME_HEADER);
    basenameHeaderKey = context.getString(BASENAME_HEADER_KEY, DEFAULT_BASENAME_HEADER_KEY);
    batchSize = context.getInteger(BATCH_SIZE, DEFAULT_BATCH_SIZE);
    inputCharset = context.getString(INPUT_CHARSET, DEFAULT_INPUT_CHARSET);
    decodeErrorPolicy =
        DecodeErrorPolicy.valueOf(
            context
                .getString(DECODE_ERROR_POLICY, DEFAULT_DECODE_ERROR_POLICY)
                .toUpperCase(Locale.ENGLISH));

    ignorePattern = context.getString(IGNORE_PAT, DEFAULT_IGNORE_PAT);
    trackerDirPath = context.getString(TRACKER_DIR, DEFAULT_TRACKER_DIR);

    deserializerType = context.getString(DESERIALIZER, "ZipDeserializer");
    deserializerContext = new Context(context.getSubProperties(DESERIALIZER + "."));

    consumeOrder =
        ConsumeOrder.valueOf(
            context
                .getString(CONSUME_ORDER, DEFAULT_CONSUME_ORDER.toString())
                .toUpperCase(Locale.ENGLISH));

    // "Hack" to support backwards compatibility with previous generation of
    // spooling directory source, which did not support deserializers
    Integer bufferMaxLineLength = context.getInteger(BUFFER_MAX_LINE_LENGTH);
    if (bufferMaxLineLength != null
        && deserializerType != null
        && deserializerType.equalsIgnoreCase(DEFAULT_DESERIALIZER)) {
      deserializerContext.put(LineDeserializer.MAXLINE_KEY, bufferMaxLineLength.toString());
    }

    maxBackoff = context.getInteger(MAX_BACKOFF, DEFAULT_MAX_BACKOFF);
    if (sourceCounter == null) {
      sourceCounter = new SourceCounter(getName());
    }
  }
  public void createAvroFile(File file, String codec, boolean useSchemaUrl) throws IOException {

    // serialize a few events using the reflection-based avro serializer
    OutputStream out = new FileOutputStream(file);

    Context ctx = new Context();
    if (codec != null) {
      ctx.put("compressionCodec", codec);
    }

    Schema schema = Schema.createRecord("myrecord", null, null, false);
    schema.setFields(
        Arrays.asList(
            new Schema.Field[] {
              new Schema.Field("message", Schema.create(Schema.Type.STRING), null, null)
            }));
    GenericRecordBuilder recordBuilder = new GenericRecordBuilder(schema);
    File schemaFile = null;
    if (useSchemaUrl) {
      schemaFile = File.createTempFile(getClass().getSimpleName(), ".avsc");
      Files.write(schema.toString(), schemaFile, Charsets.UTF_8);
    }

    EventSerializer.Builder builder = new AvroEventSerializer.Builder();
    EventSerializer serializer = builder.build(ctx, out);

    serializer.afterCreate();
    for (int i = 0; i < 3; i++) {
      GenericRecord record = recordBuilder.set("message", "Hello " + i).build();
      Event event = EventBuilder.withBody(serializeAvro(record, schema));
      if (schemaFile == null) {
        event.getHeaders().put(AvroEventSerializer.AVRO_SCHEMA_LITERAL_HEADER, schema.toString());
      } else {
        event
            .getHeaders()
            .put(
                AvroEventSerializer.AVRO_SCHEMA_URL_HEADER,
                schemaFile.toURI().toURL().toExternalForm());
      }
      serializer.write(event);
    }
    serializer.flush();
    serializer.beforeClose();
    out.flush();
    out.close();
  }
  @SuppressWarnings("unchecked")
  @Test
  public void ThrowableThrownExceptionTest() throws EventDeliveryException {
    Jedis jedis = mock(Jedis.class);
    // Not really true, but fits the requirement
    when(jedis.lpush(any(byte[].class), any(byte[].class))).thenThrow(clazz);

    JedisPool jedisPool = mock(JedisPool.class);
    MockJedisPoolFactory mockJedisPoolFactory = new MockJedisPoolFactory(jedisPool, jedis);

    Channel channel = mock(Channel.class);
    Transaction transactionMock = mock(Transaction.class);
    when(channel.getTransaction()).thenReturn(transactionMock);

    Event testEvent = new SimpleEvent();
    byte[] testBody = new byte[] {'b', 'o', 'd', 'y'};
    testEvent.setBody(testBody);
    when(channel.take()).thenReturn(testEvent);

    RedisSink redisSink = new RedisSink(mockJedisPoolFactory);
    redisSink.setChannel(channel);

    Context context = new Context();
    context.put(RedisSinkConfigurationConstant.HOST, "localhost");
    redisSink.configure(context);

    redisSink.start();

    redisSink.process();

    verify(channel, times(1)).getTransaction();
    verify(channel, times(1)).take();

    verify(transactionMock, times(1)).begin();
    verify(transactionMock, times(1)).close();
    verify(transactionMock, times(0)).commit();
    verify(transactionMock, times(1)).rollback();

    verify(jedisPool, times(1)).getResource();
    verify(jedisPool, times(1)).returnResource(jedis);

    verify(jedis, times(1))
        .lpush(eq(RedisSinkConfigurationConstant.DEFAULT_KEY.getBytes()), any(byte[][].class));
  }
Beispiel #23
0
  public void doTestTextBatchAppend() throws Exception {
    LOG.debug("Starting...");

    final long rollCount = 10;
    final long batchSize = 2;
    final String fileName = "PageView";

    String newPath = testPath + "/singleTextBucket";
    int totalEvents = 0;
    int i = 1, j = 1;

    // clear the test directory
    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(conf);
    Path dirPath = new Path(newPath);
    fs.delete(dirPath, true);
    fs.mkdirs(dirPath);

    Context context = new Context();
    context.put("hdfs.path", newPath);
    context.put("hdfs.rollCount", String.valueOf(rollCount));
    context.put("hdfs.batchSize", String.valueOf(batchSize));
    context.put("hdfs.filePrefix", "pageview");

    Channel channel = new MemoryChannel();
    Configurables.configure(channel, context);
    sink.setChannel(channel);
    sink.start();

    Calendar eventDate = Calendar.getInstance();
    Date currentDate = new Date();
    Map<String, String> header = new HashMap<String, String>();
    header.put("topic", "PageView");

    List<String> bodies = Lists.newArrayList();

    // 将测试的事件推入到通道中
    for (i = 1; i <= (rollCount * 10) / batchSize; i++) {
      Transaction txn = channel.getTransaction();
      txn.begin();
      for (j = 1; j <= batchSize; j++) {
        header.put("timestamp", String.valueOf(currentDate.getTime()));
        Event event = new SimpleEvent();
        eventDate.clear();
        eventDate.set(2014, i, i, i, 0);
        String body = "Test." + i + "." + j;

        event.setHeaders(header);
        event.setBody(body.getBytes());
        bodies.add(body);
        channel.put(event);
        totalEvents++;
      }
      txn.commit();
      txn.close();

      // execute sink to process the events
      sink.process();
    }
    sink.stop();

    FileStatus[] dirStat = fs.listStatus(dirPath);
    Path fList[] = FileUtil.stat2Paths(dirStat);

    long expectedFiles = totalEvents / rollCount;
    if (totalEvents % rollCount > 0) {
      expectedFiles++;
    }

    Assert.assertEquals(
        "num files wrong, found: " + Lists.newArrayList(fList), expectedFiles, fList.length);
    // 检查所有写入文件的内容
    verifyOutputTextFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);
  }
  @Override
  public void configure(Context context) {
    if (!isLocal) {
      if (StringUtils.isNotBlank(context.getString(HOSTNAMES))) {
        serverAddresses = StringUtils.deleteWhitespace(context.getString(HOSTNAMES)).split(",");
      }
      Preconditions.checkState(
          serverAddresses != null && serverAddresses.length > 0, "Missing Param:" + HOSTNAMES);
    }

    if (StringUtils.isNotBlank(context.getString(INDEX_NAME))) {
      this.indexName = context.getString(INDEX_NAME);
    }

    if (StringUtils.isNotBlank(context.getString(INDEX_TYPE))) {
      this.indexType = context.getString(INDEX_TYPE);
    }

    if (StringUtils.isNotBlank(context.getString(CLUSTER_NAME))) {
      this.clusterName = context.getString(CLUSTER_NAME);
    }

    if (StringUtils.isNotBlank(context.getString(BATCH_SIZE))) {
      this.batchSize = Integer.parseInt(context.getString(BATCH_SIZE));
    }

    if (StringUtils.isNotBlank(context.getString(TTL))) {
      this.ttlMs = parseTTL(context.getString(TTL));
      Preconditions.checkState(ttlMs > 0, TTL + " must be greater than 0 or not set.");
    }

    if (StringUtils.isNotBlank(context.getString(CLIENT_TYPE))) {
      clientType = context.getString(CLIENT_TYPE);
    }

    elasticSearchClientContext = new Context();
    elasticSearchClientContext.putAll(context.getSubProperties(CLIENT_PREFIX));

    String serializerClazz = DEFAULT_SERIALIZER_CLASS;
    if (StringUtils.isNotBlank(context.getString(SERIALIZER))) {
      serializerClazz = context.getString(SERIALIZER);
    }

    Context serializerContext = new Context();
    serializerContext.putAll(context.getSubProperties(SERIALIZER_PREFIX));

    try {
      @SuppressWarnings("unchecked")
      Class<? extends Configurable> clazz =
          (Class<? extends Configurable>) Class.forName(serializerClazz);
      Configurable serializer = clazz.newInstance();

      if (serializer instanceof ElasticSearchIndexRequestBuilderFactory) {
        indexRequestFactory = (ElasticSearchIndexRequestBuilderFactory) serializer;
        indexRequestFactory.configure(serializerContext);
      } else if (serializer instanceof ElasticSearchEventSerializer) {
        eventSerializer = (ElasticSearchEventSerializer) serializer;
        eventSerializer.configure(serializerContext);
      } else {
        throw new IllegalArgumentException(
            serializerClazz + " is not an ElasticSearchEventSerializer");
      }
    } catch (Exception e) {
      logger.error("Could not instantiate event serializer.", e);
      Throwables.propagate(e);
    }

    if (sinkCounter == null) {
      sinkCounter = new SinkCounter(getName());
    }

    String indexNameBuilderClass = DEFAULT_INDEX_NAME_BUILDER_CLASS;
    if (StringUtils.isNotBlank(context.getString(INDEX_NAME_BUILDER))) {
      indexNameBuilderClass = context.getString(INDEX_NAME_BUILDER);
    }

    Context indexnameBuilderContext = new Context();
    indexnameBuilderContext.putAll(context.getSubProperties(INDEX_NAME_BUILDER_PREFIX));

    try {
      @SuppressWarnings("unchecked")
      Class<? extends IndexNameBuilder> clazz =
          (Class<? extends IndexNameBuilder>) Class.forName(indexNameBuilderClass);
      indexNameBuilder = clazz.newInstance();
      indexnameBuilderContext.put(INDEX_NAME, indexName);
      indexNameBuilder.configure(indexnameBuilderContext);
    } catch (Exception e) {
      logger.error("Could not instantiate index name builder.", e);
      Throwables.propagate(e);
    }

    if (sinkCounter == null) {
      sinkCounter = new SinkCounter(getName());
    }

    Preconditions.checkState(StringUtils.isNotBlank(indexName), "Missing Param:" + INDEX_NAME);
    Preconditions.checkState(StringUtils.isNotBlank(indexType), "Missing Param:" + INDEX_TYPE);
    Preconditions.checkState(StringUtils.isNotBlank(clusterName), "Missing Param:" + CLUSTER_NAME);
    Preconditions.checkState(batchSize >= 1, BATCH_SIZE + " must be greater than 0");
  }
 /**
  * Creates a Flume context for Mongo/STH sinks.
  *
  * @param collectionPrefix
  * @param dbPrefix
  * @param dataModel
  * @param enableEncoding
  * @return A Flume context for Mongo/STH sinks.
  */
 public static Context createContextForMongoSTH(
     String collectionPrefix, String dbPrefix, String dataModel, String enableEncoding) {
   Context context = new Context();
   context.put("attr_persistence", "row");
   context.put("batch_size", "100");
   context.put("batch_timeout", "30");
   context.put("batch_ttl", "10");
   context.put("collection_prefix", collectionPrefix);
   context.put("collection_size", "0");
   context.put("data_expiration", "0");
   context.put("data_model", dataModel);
   context.put("db_prefix", dbPrefix);
   context.put("enable_encoding", enableEncoding);
   context.put("enable_grouping", "false");
   context.put("enable_lowercase", "false");
   context.put("max_documents", "0");
   context.put("mongo_hosts", "localhost:27017");
   context.put("mongo_password", "");
   context.put("mongo_username", "");
   return context;
 } // createContextForMongoSTH
Beispiel #26
0
  //    @Ignore
  @Before
  public void setUp() {

    conf = ConfigFactory.load();
    List<String> zkHosts = conf.getStringList("zookeeper.hosts");

    for (String host : zkHosts) {
      ZOOKEEPER_HOSTS += host + ",";
    }
    ZOOKEEPER_HOSTS = ZOOKEEPER_HOSTS.substring(0, ZOOKEEPER_HOSTS.length() - 1);

    List<String> kafkaHosts = conf.getStringList("kafka.hosts");

    for (String host : kafkaHosts) {
      KAFKA_HOSTS += host + ",";
    }
    KAFKA_HOSTS = KAFKA_HOSTS.substring(0, KAFKA_HOSTS.length() - 1);

    LOGGER.debug("Using Zookeeper hosts: " + ZOOKEEPER_HOSTS);
    LOGGER.debug("Using Zookeeper hosts: " + KAFKA_HOSTS);
    //        try {
    //            zookeeperServer = new ZookeeperServer();
    //            zookeeperServer.start();
    //        } catch (Exception e) {
    //            e.printStackTrace();
    //        }
    //        try {
    //            kafkaServer = new KafkaServer();
    //            kafkaServer.start();
    //        } catch (Exception e) {
    //            e.printStackTrace();
    //        }

    String[] connection = KAFKA_HOSTS.split(":");

    //        simpleConsumer = new SimpleConsumer("localhost", 9092, 60000, 1024, CLIENT_ID);
    simpleConsumer =
        new SimpleConsumer(connection[0], Integer.parseInt(connection[1]), 60000, 1024, CLIENT_ID);

    kafkaSink = new KafkaSink();

    Context kafkaContext = new Context();
    kafkaContext.put("topic", "test");
    kafkaContext.put("writeBody", "false");
    kafkaContext.put("kafka.metadata.broker.list", KAFKA_HOSTS);
    kafkaContext.put("kafka.serializer.class", "kafka.serializer.StringEncoder");

    Configurables.configure(kafkaSink, kafkaContext);

    Context channelContext = new Context();
    channelContext.put("capacity", "10000");
    channelContext.put("transactionCapacity", "200");

    channel = new MemoryChannel();
    channel.setName("junitChannel");
    Configurables.configure(channel, channelContext);

    kafkaSink.setChannel(channel);

    channel.start();
    kafkaSink.start();
  }