Beispiel #1
0
  @Test
  public void testBatchEvents() throws InterruptedException, EventDeliveryException {
    StressSource source = new StressSource();
    source.setChannelProcessor(mockProcessor);
    Context context = new Context();
    context.put("maxTotalEvents", "35");
    context.put("batchSize", "10");
    source.configure(context);
    source.start();

    for (int i = 0; i < 50; i++) {
      if (source.process() == Status.BACKOFF) {
        TestCase.assertTrue("Source should have sent all events in 4 batches", i == 4);
        break;
      }
      if (i < 3) {
        verify(mockProcessor, times(i + 1)).processEventBatch(getLastProcessedEventList(source));
      } else {
        verify(mockProcessor, times(1)).processEventBatch(getLastProcessedEventList(source));
      }
    }
    long successfulEvents = getCounterGroup(source).get("events.successful");
    TestCase.assertTrue(
        "Number of successful events should be 35 but was " + successfulEvents,
        successfulEvents == 35);
    long failedEvents = getCounterGroup(source).get("events.failed");
    TestCase.assertTrue(
        "Number of failure events should be 0 but was " + failedEvents, failedEvents == 0);
  }
  /** Verify that setting a timeout in the configuration does not impact the database setting */
  @Test
  public void timeoutConfiguredTest() {

    String host = "localhost";
    int timeout = 10;

    JedisPoolFactory mockJedisPoolFactory = mock(JedisPoolFactory.class);

    RedisSink redisSink = new RedisSink(mockJedisPoolFactory);

    Channel channel = mock(Channel.class);
    redisSink.setChannel(channel);

    Context context = new Context();
    context.put(RedisSinkConfigurationConstant.HOST, host);
    context.put(RedisSinkConfigurationConstant.BATCH_SIZE, "10");
    context.put(RedisSinkConfigurationConstant.TIMEOUT, Integer.toString(timeout));
    redisSink.configure(context);

    redisSink.start();

    verify(mockJedisPoolFactory)
        .create(
            any(JedisPoolConfig.class),
            eq(host),
            eq(Protocol.DEFAULT_PORT),
            eq(timeout),
            isNull(String.class),
            eq(Protocol.DEFAULT_DATABASE));
  }
  @Test(groups = "dev")
  public void autoWrapTest() throws EventDeliveryException {
    ctx.put(MongoSink.AUTO_WRAP, Boolean.toString(true));
    ctx.put(MongoSink.DB_NAME, "test_wrap");

    MongoSink sink = new MongoSink();
    Configurables.configure(sink, ctx);

    sink.setChannel(channel);
    sink.start();

    Transaction tx = channel.getTransaction();
    tx.begin();
    String msg =
        "2012/10/26 11:23:08 [error] 7289#0: *6430831 open() \"/usr/local/nginx/html/50x.html\" failed (2: No such file or directory), client: 10.160.105.161, server: sg15.redatoms.com, request: \"POST /mojo/ajax/embed HTTP/1.0\", upstream: \"fastcgi://unix:/tmp/php-fpm.sock:\", host: \"sg15.redatoms.com\", referrer: \"http://sg15.redatoms.com/mojo/mobile/package\"";

    Event e = EventBuilder.withBody(msg.getBytes());
    channel.put(e);
    tx.commit();
    tx.close();

    sink.process();
    sink.stop();

    DB db = mongo.getDB("test_wrap");
    DBCollection collection = db.getCollection("test_log");
    DBCursor cursor = collection.find(new BasicDBObject(MongoSink.DEFAULT_WRAP_FIELD, msg));
    assertTrue(cursor.hasNext());
    DBObject dbObject = cursor.next();
    assertNotNull(dbObject);
    assertEquals(dbObject.get(MongoSink.DEFAULT_WRAP_FIELD), msg);
    mongo.dropDatabase("test_wrap");
  }
Beispiel #4
0
  @Test
  public void testMaxSuccessfulEvents() throws InterruptedException, EventDeliveryException {
    StressSource source = new StressSource();
    source.setChannelProcessor(mockProcessor);
    Context context = new Context();
    context.put("maxSuccessfulEvents", "35");
    source.configure(context);
    source.start();

    for (int i = 0; i < 10; i++) {
      source.process();
    }

    // 1 failed call, 10 successful
    doThrow(new ChannelException("stub")).when(mockProcessor).processEvent(getEvent(source));
    source.process();
    doNothing().when(mockProcessor).processEvent(getEvent(source));
    for (int i = 0; i < 10; i++) {
      source.process();
    }

    // 1 failed call, 50 succesful
    doThrow(new ChannelException("stub")).when(mockProcessor).processEvent(getEvent(source));
    source.process();
    doNothing().when(mockProcessor).processEvent(getEvent(source));
    for (int i = 0; i < 50; i++) {
      source.process();
    }

    // We should have called processEvent(evt) 37 times, twice for failures
    // and twice for successful events.
    verify(mockProcessor, times(37)).processEvent(getEvent(source));
  }
Beispiel #5
0
  @Test
  public void testBatchEventsWithoutMatTotalEvents()
      throws InterruptedException, EventDeliveryException {
    StressSource source = new StressSource();
    source.setChannelProcessor(mockProcessor);
    Context context = new Context();
    context.put("batchSize", "10");
    source.configure(context);
    source.start();

    for (int i = 0; i < 10; i++) {
      Assert.assertFalse(
          "StressSource with no maxTotalEvents should not return " + Status.BACKOFF,
          source.process() == Status.BACKOFF);
    }
    verify(mockProcessor, times(10)).processEventBatch(getLastProcessedEventList(source));

    long successfulEvents = getCounterGroup(source).get("events.successful");
    TestCase.assertTrue(
        "Number of successful events should be 100 but was " + successfulEvents,
        successfulEvents == 100);

    long failedEvents = getCounterGroup(source).get("events.failed");
    TestCase.assertTrue(
        "Number of failure events should be 0 but was " + failedEvents, failedEvents == 0);
  }
 @Override
 public void doConfigure(Context context) {
   final String regex = context.getString(CONFIG_REGULAR_EXPRESSION, REGEX_DEFAULT);
   final boolean regexIgnoreCase = context.getBoolean(IGNORE_CASE_CONFIG, IGNORE_CASE_DEFAULT);
   inputPattern =
       Pattern.compile(regex, Pattern.DOTALL + (regexIgnoreCase ? Pattern.CASE_INSENSITIVE : 0));
 }
 @Override
 public void configure(Context context) {
   hours = context.getString("hours");
   Preconditions.checkArgument(StringUtils.isNotEmpty(hours), "Must configure with a valid hours");
   outputPattern = context.getString("outputPattern");
   Preconditions.checkArgument(
       StringUtils.isNotEmpty(outputPattern), "Must configure with a valid outputPattern");
 }
  @Test
  public void shouldUseSpecifiedSerializer() throws Exception {
    Context context = new Context();
    context.put(SERIALIZER, "org.apache.flume.sink.elasticsearch.FakeEventSerializer");

    assertNull(fixture.getEventSerializer());
    fixture.configure(context);
    assertTrue(fixture.getEventSerializer() instanceof FakeEventSerializer);
  }
  /**
   * Verify that the required configuration is set
   *
   * @param context
   */
  private void ensureConfigCompleteness(Context context) {

    if (StringUtils.isEmpty(context.getString(RabbitMQConstants.CONFIG_EXCHANGENAME))
        && StringUtils.isEmpty(context.getString(RabbitMQConstants.CONFIG_QUEUENAME))) {

      throw new IllegalArgumentException(
          "You must configure at least one of queue name or exchange name parameters");
    }
  }
  @Test
  public void testRollbackWithSink() throws Exception {
    final NullSink nullSink = new NullSink();
    Context ctx = new Context();
    ctx.put("batchSize", "1");
    nullSink.configure(ctx);
    nullSink.setChannel(channel);
    final int numItems = 99;
    Thread t =
        new Thread() {
          @Override
          public void run() {
            int count = 0;
            while (count++ < numItems) {
              try {
                nullSink.process();
                Thread.sleep(1);
              } catch (EventDeliveryException e) {
                break;
              } catch (Exception e) {
                Throwables.propagate(e);
              }
            }
          }
        };
    t.setDaemon(true);
    t.setName("NullSink");
    t.start();

    putEvents(channel, "rollback", 10, 100);

    Transaction transaction;
    // put an item we will rollback
    transaction = channel.getTransaction();
    transaction.begin();
    channel.put(EventBuilder.withBody("this is going to be rolledback".getBytes(Charsets.UTF_8)));
    transaction.rollback();
    transaction.close();

    while (t.isAlive()) {
      Thread.sleep(1);
    }

    // simulate crash
    channel.stop();
    channel = createFileChannel();

    // get the item which was not rolled back
    transaction = channel.getTransaction();
    transaction.begin();
    Event event = channel.take();
    transaction.commit();
    transaction.close();
    Assert.assertNotNull(event);
    Assert.assertEquals("rollback-90-9", new String(event.getBody(), Charsets.UTF_8));
  }
 @Override
 public void configure(Context context) {
   /*
    * Default is to listen on UDP port 162 on all IPv4 interfaces.
    * Since 162 is a privileged port, snmptrapd must typically be run as root.
    * Or change to non-privileged port > 1024.
    */
   bindAddress = context.getString("bind", DEFAULT_BIND);
   bindPort = context.getInteger("port", DEFAULT_PORT);
 }
  @Test(groups = "dev")
  public void timestampExistingFieldTest() throws EventDeliveryException, ParseException {
    ctx.put(MongoSink.MODEL, MongoSink.CollectionModel.dynamic.name());
    String tsField = "createdOn";
    ctx.put(MongoSink.TIMESTAMP_FIELD, tsField);
    MongoSink sink = new MongoSink();
    Configurables.configure(sink, ctx);

    sink.setChannel(channel);
    sink.start();

    JSONObject msg = new JSONObject();
    msg.put("age", 11);
    msg.put("birthday", new Date().getTime());
    String dateText = "2013-02-19T14:20:53+08:00";
    msg.put(tsField, dateText);

    Transaction tx;

    for (int i = 0; i < 10; i++) {
      tx = channel.getTransaction();
      tx.begin();
      msg.put("name", "test" + i);
      JSONObject header = new JSONObject();
      header.put(MongoSink.COLLECTION, "my_events");
      header.put(MongoSink.DB_NAME, "dynamic_db");

      Event e = EventBuilder.withBody(msg.toJSONString().getBytes(), header);
      channel.put(e);
      tx.commit();
      tx.close();
    }
    sink.process();
    sink.stop();

    msg.put(tsField, MongoSink.dateTimeFormatter.parseDateTime(dateText).toDate());
    for (int i = 0; i < 10; i++) {
      msg.put("name", "test" + i);

      System.out.println("i = " + i);

      DB db = mongo.getDB("dynamic_db");
      DBCollection collection = db.getCollection("my_events");
      DBCursor cursor = collection.find(new BasicDBObject(msg));
      assertTrue(cursor.hasNext());
      DBObject dbObject = cursor.next();
      assertNotNull(dbObject);
      assertEquals(dbObject.get("name"), msg.get("name"));
      assertEquals(dbObject.get("age"), msg.get("age"));
      assertEquals(dbObject.get("birthday"), msg.get("birthday"));
      assertTrue(dbObject.get(tsField) instanceof Date);
      System.out.println("ts = " + dbObject.get(tsField));
    }
  }
Beispiel #13
0
  @Test
  public void shouldUseSpecifiedIndexNameBuilder() throws Exception {
    Context context = new Context();
    context.put(
        ElasticSearchSinkConstants.INDEX_NAME_BUILDER,
        "org.apache.flume.sink.elasticsearch.FakeIndexNameBuilder");

    assertNull(fixture.getIndexNameBuilder());
    fixture.configure(context);
    assertTrue(fixture.getIndexNameBuilder() instanceof FakeIndexNameBuilder);
  }
Beispiel #14
0
  // @Test
  public void testLifecycle() throws InterruptedException, LifecycleException {
    LOG.debug("Starting...");
    Context context = new Context();
    context.put("hdfs.path", testPath);
    context.put("hdfs.filePrefix", "pageview");

    Configurables.configure(sink, context);
    sink.setChannel(new MemoryChannel());
    sink.start();
    sink.stop();
  }
  /**
   * The initialization method for the Source. The context contains all the Flume configuration
   * info, and can be used to retrieve any configuration values necessary to set up the Source.
   */
  @Override
  public void configure(Context context) {
    consumerKey = context.getString(TwitterSourceConstants.CONSUMER_KEY_KEY);
    consumerSecret = context.getString(TwitterSourceConstants.CONSUMER_SECRET_KEY);
    accessToken = context.getString(TwitterSourceConstants.ACCESS_TOKEN_KEY);
    accessTokenSecret = context.getString(TwitterSourceConstants.ACCESS_TOKEN_SECRET_KEY);

    String keywordString = context.getString(TwitterSourceConstants.KEYWORDS_KEY, "");
    keywords = keywordString.split(",");
    for (int i = 0; i < keywords.length; i++) {
      keywords[i] = keywords[i].trim();
    }
  }
 @Override
 public void configure(Context context) throws FlumeException {
   preserveExisting = context.getBoolean(PRESERVE, PRESERVE_DEFAULT);
   key = context.getString(KEY, KEY_DEFAULT);
   file = context.getString(FILE);
   period = context.getInteger(PERIOD, new Integer(PERIOD_DEFAULT));
   if (file != null) {
     value = readHeader(file);
   } else {
     logger.error("CSVHeaderInterceptor - file not specified");
     throw new FlumeException("CSVHeaderInterceptor - file not specified");
   }
 }
  /** Configures the keys required for accessing Twitter. Also gets the productKeys */
  @Override
  public void configure(Context context) {
    consumerKey = context.getString("consumerKey");
    consumerSecret = context.getString("consumerSecret");
    accessToken = context.getString("accessToken");
    accessTokenSecret = context.getString("accessTokenSecret");

    String prodctKeys = context.getString("Product", "");
    searchFor = prodctKeys.split(",");
    for (int i = 0; i < searchFor.length; i++) {
      searchFor[i] = searchFor[i].trim();
    }
  }
Beispiel #18
0
  @Test
  public void testMaxTotalEvents() throws InterruptedException, EventDeliveryException {
    StressSource source = new StressSource();
    source.setChannelProcessor(mockProcessor);
    Context context = new Context();
    context.put("maxTotalEvents", "35");
    source.configure(context);
    source.start();

    for (int i = 0; i < 50; i++) {
      source.process();
    }
    verify(mockProcessor, times(35)).processEvent(getEvent(source));
  }
  /** {@inheritDoc} */
  @Override
  public void configure(final Context context) {
    hostName = context.getString(HOSTNAME_CONFIG_PROP_NAME);
    port = context.getInteger(PORT_CONFIG_PROP_NAME);
    batchSize = context.getInteger(BATCH_SIZE_PROP_NAME, DEFAULT_BATCH_SIZE);

    if (sinkCounter == null) {
      sinkCounter = new SinkCounter(getName());
    }

    LOGGER.info(
        "Configuring ZipkinSpanCollectorSink. hostname: {}, port: {}, batchsize: {}",
        hostName,
        port,
        batchSize);
  }
  @Test
  public void checkStatusFileCorrectlyUpdated() throws Exception {

    File file = File.createTempFile("statusFileName", ".txt");

    when(context.getString("status.file.path", "/var/lib/flume")).thenReturn(file.getParent());
    when(context.getString("status.file.name")).thenReturn(file.getName());

    SQLSourceHelper sqlSourceHelper = new SQLSourceHelper(context, "Source Name");
    sqlSourceHelper.setCurrentIndex(10);

    sqlSourceHelper.updateStatusFile();

    SQLSourceHelper sqlSourceHelper2 = new SQLSourceHelper(context, "Source Name");
    assertEquals(10L, sqlSourceHelper2.getCurrentIndex());
  }
Beispiel #21
0
  @Override
  public void configure(Context context) {
    port = Integer.parseInt(context.getString("port"));
    bindAddress = context.getString("bind");
    try {
      maxThreads = context.getInteger(THREADS, 0);
    } catch (NumberFormatException e) {
      logger.warn(
          "AVRO source\'s \"threads\" property must specify an integer value.",
          context.getString(THREADS));
    }

    if (sourceCounter == null) {
      sourceCounter = new SourceCounter(getName());
    }
  }
  /**
   * Constructor to initialize hibernate configuration parameters
   *
   * @param sqlSourceHelper Contains the configuration parameters from flume config file
   */
  public HibernateHelper(SQLSourceHelper sqlSourceHelper) {

    this.sqlSourceHelper = sqlSourceHelper;
    Context context = sqlSourceHelper.getContext();

    Map<String, String> hibernateProperties = context.getSubProperties("hibernate.");
    Iterator<Map.Entry<String, String>> it = hibernateProperties.entrySet().iterator();

    config = new Configuration();
    Map.Entry<String, String> e;

    while (it.hasNext()) {
      e = it.next();
      config.setProperty("hibernate." + e.getKey(), e.getValue());
    }
  }
Beispiel #23
0
  @Override
  public void configure(Context context) {
    String resolutionsStr = context.getString("resolutions", "month,day,hour,minute,second");
    String[] resolutionsArray = resolutionsStr.split(",");

    for (String resolution : resolutionsArray) {
      if (resolution.trim().equals("month")) {
        resolutions[4] = true;
      } else if (resolution.trim().equals("day")) {
        resolutions[3] = true;
      } else if (resolution.trim().equals("hour")) {
        resolutions[2] = true;
      } else if (resolution.trim().equals("minute")) {
        resolutions[1] = true;
      } else if (resolution.trim().equals("second")) {
        resolutions[0] = true;
      } else {
        LOGGER.warn("[" + this.getName() + "] Unknown resolution " + resolution);
      } // if else
    } // for

    LOGGER.debug(
        "[" + this.getName() + "] Reading configuration (resolutions=" + resolutionsStr + ")");
    super.configure(context);
  } // configure
  @Test(groups = "dev")
  public void sinkSingleModelTest() throws EventDeliveryException {
    ctx.put(MongoSink.MODEL, MongoSink.CollectionModel.single.name());

    MongoSink sink = new MongoSink();
    Configurables.configure(sink, ctx);

    sink.setChannel(channel);
    sink.start();

    Transaction tx = channel.getTransaction();
    tx.begin();
    JSONObject msg = new JSONObject();
    msg.put("name", "test");
    msg.put("age", 11);
    msg.put("birthday", new Date().getTime());

    Event e = EventBuilder.withBody(msg.toJSONString().getBytes());
    channel.put(e);
    tx.commit();
    tx.close();

    sink.process();
    sink.stop();

    DB db = mongo.getDB("test_events");
    DBCollection collection = db.getCollection("test_log");
    DBCursor cursor = collection.find(new BasicDBObject(msg));
    assertTrue(cursor.hasNext());
    DBObject dbObject = cursor.next();
    assertNotNull(dbObject);
    assertEquals(dbObject.get("name"), msg.get("name"));
    assertEquals(dbObject.get("age"), msg.get("age"));
    assertEquals(dbObject.get("birthday"), msg.get("birthday"));
  }
 private RecoverableMemoryChannel createFileChannel() {
   RecoverableMemoryChannel channel = new RecoverableMemoryChannel();
   context = new Context();
   context.put(RecoverableMemoryChannel.WAL_DATA_DIR, dataDir.getAbsolutePath());
   Configurables.configure(channel, context);
   channel.start();
   return channel;
 }
  public void createAvroFile(File file, String codec, boolean useSchemaUrl) throws IOException {

    // serialize a few events using the reflection-based avro serializer
    OutputStream out = new FileOutputStream(file);

    Context ctx = new Context();
    if (codec != null) {
      ctx.put("compressionCodec", codec);
    }

    Schema schema = Schema.createRecord("myrecord", null, null, false);
    schema.setFields(
        Arrays.asList(
            new Schema.Field[] {
              new Schema.Field("message", Schema.create(Schema.Type.STRING), null, null)
            }));
    GenericRecordBuilder recordBuilder = new GenericRecordBuilder(schema);
    File schemaFile = null;
    if (useSchemaUrl) {
      schemaFile = File.createTempFile(getClass().getSimpleName(), ".avsc");
      Files.write(schema.toString(), schemaFile, Charsets.UTF_8);
    }

    EventSerializer.Builder builder = new AvroEventSerializer.Builder();
    EventSerializer serializer = builder.build(ctx, out);

    serializer.afterCreate();
    for (int i = 0; i < 3; i++) {
      GenericRecord record = recordBuilder.set("message", "Hello " + i).build();
      Event event = EventBuilder.withBody(serializeAvro(record, schema));
      if (schemaFile == null) {
        event.getHeaders().put(AvroEventSerializer.AVRO_SCHEMA_LITERAL_HEADER, schema.toString());
      } else {
        event
            .getHeaders()
            .put(
                AvroEventSerializer.AVRO_SCHEMA_URL_HEADER,
                schemaFile.toURI().toURL().toExternalForm());
      }
      serializer.write(event);
    }
    serializer.flush();
    serializer.beforeClose();
    out.flush();
    out.close();
  }
  @SuppressWarnings("unchecked")
  @Test
  public void ThrowableThrownExceptionTest() throws EventDeliveryException {
    Jedis jedis = mock(Jedis.class);
    // Not really true, but fits the requirement
    when(jedis.lpush(any(byte[].class), any(byte[].class))).thenThrow(clazz);

    JedisPool jedisPool = mock(JedisPool.class);
    MockJedisPoolFactory mockJedisPoolFactory = new MockJedisPoolFactory(jedisPool, jedis);

    Channel channel = mock(Channel.class);
    Transaction transactionMock = mock(Transaction.class);
    when(channel.getTransaction()).thenReturn(transactionMock);

    Event testEvent = new SimpleEvent();
    byte[] testBody = new byte[] {'b', 'o', 'd', 'y'};
    testEvent.setBody(testBody);
    when(channel.take()).thenReturn(testEvent);

    RedisSink redisSink = new RedisSink(mockJedisPoolFactory);
    redisSink.setChannel(channel);

    Context context = new Context();
    context.put(RedisSinkConfigurationConstant.HOST, "localhost");
    redisSink.configure(context);

    redisSink.start();

    redisSink.process();

    verify(channel, times(1)).getTransaction();
    verify(channel, times(1)).take();

    verify(transactionMock, times(1)).begin();
    verify(transactionMock, times(1)).close();
    verify(transactionMock, times(0)).commit();
    verify(transactionMock, times(1)).rollback();

    verify(jedisPool, times(1)).getResource();
    verify(jedisPool, times(1)).returnResource(jedis);

    verify(jedis, times(1))
        .lpush(eq(RedisSinkConfigurationConstant.DEFAULT_KEY.getBytes()), any(byte[][].class));
  }
  @BeforeMethod(groups = {"dev"})
  public static void setup() throws UnknownHostException {
    mongo = new Mongo("localhost", 27017);

    Map<String, String> ctxMap = new HashMap<String, String>();
    ctxMap.put(MongoSink.HOST, "localhost");
    ctxMap.put(MongoSink.PORT, "27017");
    ctxMap.put(MongoSink.DB_NAME, "test_events");
    ctxMap.put(MongoSink.COLLECTION, "test_log");
    ctxMap.put(MongoSink.BATCH_SIZE, "100");

    ctx.putAll(ctxMap);

    Context channelCtx = new Context();
    channelCtx.put("capacity", "1000000");
    channelCtx.put("transactionCapacity", "1000000");
    channel = new MemoryChannel();
    Configurables.configure(channel, channelCtx);
  }
  @Test(expected = NullPointerException.class)
  public void testNoJson() throws Exception {
    try {
      context.put("urlJson", "");

      dynamicUrlHandler = new DynamicUrlHandler();
      dynamicUrlHandler.configure(context);

    } catch (NullPointerException e) {
    }
  }
  @Test(expected = RestSourceException.class)
  public void testEmptyJson() throws Exception {
    try {
      context.put("urlJson", EMPTY_URL_JSON);

      dynamicUrlHandler = new DynamicUrlHandler();
      dynamicUrlHandler.configure(context);

    } catch (RestSourceException e) {
    }
  }