Exemplo n.º 1
0
  @Test
  public void shouldIndexFiveEvents() throws Exception {
    // Make it so we only need to call process once
    parameters.put(BATCH_SIZE, "5");
    Configurables.configure(fixture, new Context(parameters));
    Channel channel = bindAndStartChannel(fixture);

    int numberOfEvents = 5;
    Event[] events = new Event[numberOfEvents];

    Transaction tx = channel.getTransaction();
    tx.begin();
    for (int i = 0; i < numberOfEvents; i++) {
      String body = "event #" + i + " of " + numberOfEvents;
      Event event = EventBuilder.withBody(body.getBytes());
      events[i] = event;
      channel.put(event);
    }
    tx.commit();
    tx.close();

    fixture.process();
    fixture.stop();
    client.admin().indices().refresh(Requests.refreshRequest(timestampedIndexName)).actionGet();

    assertMatchAllQuery(numberOfEvents, events);
    assertBodyQuery(5, events);
  }
Exemplo n.º 2
0
  @Test
  public void shouldIndexFiveEventsOverThreeBatches() throws Exception {
    parameters.put(BATCH_SIZE, "2");
    Configurables.configure(fixture, new Context(parameters));
    Channel channel = bindAndStartChannel(fixture);

    int numberOfEvents = 5;
    Event[] events = new Event[numberOfEvents];

    Transaction tx = channel.getTransaction();
    tx.begin();
    for (int i = 0; i < numberOfEvents; i++) {
      String body = "event #" + i + " of " + numberOfEvents;
      Event event = EventBuilder.withBody(body.getBytes());
      events[i] = event;
      channel.put(event);
    }
    tx.commit();
    tx.close();

    int count = 0;
    Status status = Status.READY;
    while (status != Status.BACKOFF) {
      count++;
      status = fixture.process();
    }
    fixture.stop();

    assertEquals(3, count);

    client.admin().indices().refresh(Requests.refreshRequest(timestampedIndexName)).actionGet();
    assertMatchAllQuery(numberOfEvents, events);
    assertBodyQuery(5, events);
  }
  @Test(groups = "dev")
  public void autoWrapTest() throws EventDeliveryException {
    ctx.put(MongoSink.AUTO_WRAP, Boolean.toString(true));
    ctx.put(MongoSink.DB_NAME, "test_wrap");

    MongoSink sink = new MongoSink();
    Configurables.configure(sink, ctx);

    sink.setChannel(channel);
    sink.start();

    Transaction tx = channel.getTransaction();
    tx.begin();
    String msg =
        "2012/10/26 11:23:08 [error] 7289#0: *6430831 open() \"/usr/local/nginx/html/50x.html\" failed (2: No such file or directory), client: 10.160.105.161, server: sg15.redatoms.com, request: \"POST /mojo/ajax/embed HTTP/1.0\", upstream: \"fastcgi://unix:/tmp/php-fpm.sock:\", host: \"sg15.redatoms.com\", referrer: \"http://sg15.redatoms.com/mojo/mobile/package\"";

    Event e = EventBuilder.withBody(msg.getBytes());
    channel.put(e);
    tx.commit();
    tx.close();

    sink.process();
    sink.stop();

    DB db = mongo.getDB("test_wrap");
    DBCollection collection = db.getCollection("test_log");
    DBCursor cursor = collection.find(new BasicDBObject(MongoSink.DEFAULT_WRAP_FIELD, msg));
    assertTrue(cursor.hasNext());
    DBObject dbObject = cursor.next();
    assertNotNull(dbObject);
    assertEquals(dbObject.get(MongoSink.DEFAULT_WRAP_FIELD), msg);
    mongo.dropDatabase("test_wrap");
  }
  @Test(groups = "dev")
  public void sinkSingleModelTest() throws EventDeliveryException {
    ctx.put(MongoSink.MODEL, MongoSink.CollectionModel.single.name());

    MongoSink sink = new MongoSink();
    Configurables.configure(sink, ctx);

    sink.setChannel(channel);
    sink.start();

    Transaction tx = channel.getTransaction();
    tx.begin();
    JSONObject msg = new JSONObject();
    msg.put("name", "test");
    msg.put("age", 11);
    msg.put("birthday", new Date().getTime());

    Event e = EventBuilder.withBody(msg.toJSONString().getBytes());
    channel.put(e);
    tx.commit();
    tx.close();

    sink.process();
    sink.stop();

    DB db = mongo.getDB("test_events");
    DBCollection collection = db.getCollection("test_log");
    DBCursor cursor = collection.find(new BasicDBObject(msg));
    assertTrue(cursor.hasNext());
    DBObject dbObject = cursor.next();
    assertNotNull(dbObject);
    assertEquals(dbObject.get("name"), msg.get("name"));
    assertEquals(dbObject.get("age"), msg.get("age"));
    assertEquals(dbObject.get("birthday"), msg.get("birthday"));
  }
Exemplo n.º 5
0
  @Test
  public void shouldFailToConfigureWithInvalidSerializerClass() throws Exception {

    parameters.put(SERIALIZER, "java.lang.String");
    try {
      Configurables.configure(fixture, new Context(parameters));
    } catch (ClassCastException e) {
      // expected
    }

    parameters.put(SERIALIZER, FakeConfigurable.class.getName());
    try {
      Configurables.configure(fixture, new Context(parameters));
    } catch (IllegalArgumentException e) {
      // expected
    }
  }
 private RecoverableMemoryChannel createFileChannel() {
   RecoverableMemoryChannel channel = new RecoverableMemoryChannel();
   context = new Context();
   context.put(RecoverableMemoryChannel.WAL_DATA_DIR, dataDir.getAbsolutePath());
   Configurables.configure(channel, context);
   channel.start();
   return channel;
 }
  @Test(groups = "dev")
  public void timestampExistingFieldTest() throws EventDeliveryException, ParseException {
    ctx.put(MongoSink.MODEL, MongoSink.CollectionModel.dynamic.name());
    String tsField = "createdOn";
    ctx.put(MongoSink.TIMESTAMP_FIELD, tsField);
    MongoSink sink = new MongoSink();
    Configurables.configure(sink, ctx);

    sink.setChannel(channel);
    sink.start();

    JSONObject msg = new JSONObject();
    msg.put("age", 11);
    msg.put("birthday", new Date().getTime());
    String dateText = "2013-02-19T14:20:53+08:00";
    msg.put(tsField, dateText);

    Transaction tx;

    for (int i = 0; i < 10; i++) {
      tx = channel.getTransaction();
      tx.begin();
      msg.put("name", "test" + i);
      JSONObject header = new JSONObject();
      header.put(MongoSink.COLLECTION, "my_events");
      header.put(MongoSink.DB_NAME, "dynamic_db");

      Event e = EventBuilder.withBody(msg.toJSONString().getBytes(), header);
      channel.put(e);
      tx.commit();
      tx.close();
    }
    sink.process();
    sink.stop();

    msg.put(tsField, MongoSink.dateTimeFormatter.parseDateTime(dateText).toDate());
    for (int i = 0; i < 10; i++) {
      msg.put("name", "test" + i);

      System.out.println("i = " + i);

      DB db = mongo.getDB("dynamic_db");
      DBCollection collection = db.getCollection("my_events");
      DBCursor cursor = collection.find(new BasicDBObject(msg));
      assertTrue(cursor.hasNext());
      DBObject dbObject = cursor.next();
      assertNotNull(dbObject);
      assertEquals(dbObject.get("name"), msg.get("name"));
      assertEquals(dbObject.get("age"), msg.get("age"));
      assertEquals(dbObject.get("birthday"), msg.get("birthday"));
      assertTrue(dbObject.get(tsField) instanceof Date);
      System.out.println("ts = " + dbObject.get(tsField));
    }
  }
Exemplo n.º 8
0
  // @Test
  public void testLifecycle() throws InterruptedException, LifecycleException {
    LOG.debug("Starting...");
    Context context = new Context();
    context.put("hdfs.path", testPath);
    context.put("hdfs.filePrefix", "pageview");

    Configurables.configure(sink, context);
    sink.setChannel(new MemoryChannel());
    sink.start();
    sink.stop();
  }
 @Test
 public void testReconfigure() throws Exception {
   List<String> in = Lists.newArrayList();
   try {
     while (true) {
       in.addAll(putEvents(channel, "restart", 1, 1));
     }
   } catch (ChannelException e) {
     Assert.assertEquals("Cannot acquire capacity", e.getMessage());
   }
   Configurables.configure(channel, context);
   List<String> out = takeEvents(channel, 1, Integer.MAX_VALUE);
   Collections.sort(in);
   Collections.sort(out);
   Assert.assertEquals(in, out);
 }
  @Test(groups = "dev")
  public void sinkDynamicDbTest() throws EventDeliveryException {
    ctx.put(MongoSink.MODEL, MongoSink.CollectionModel.dynamic.name());
    MongoSink sink = new MongoSink();
    Configurables.configure(sink, ctx);

    sink.setChannel(channel);
    sink.start();

    JSONObject msg = new JSONObject();
    msg.put("age", 11);
    msg.put("birthday", new Date().getTime());

    Transaction tx;

    for (int i = 0; i < 10; i++) {
      tx = channel.getTransaction();
      tx.begin();
      msg.put("name", "test" + i);
      JSONObject header = new JSONObject();
      header.put(MongoSink.COLLECTION, "my_events");
      header.put(MongoSink.DB_NAME, "dynamic_db");

      Event e = EventBuilder.withBody(msg.toJSONString().getBytes(), header);
      channel.put(e);
      tx.commit();
      tx.close();
    }
    sink.process();
    sink.stop();

    for (int i = 0; i < 10; i++) {
      msg.put("name", "test" + i);

      System.out.println("i = " + i);

      DB db = mongo.getDB("dynamic_db");
      DBCollection collection = db.getCollection("my_events");
      DBCursor cursor = collection.find(new BasicDBObject(msg));
      assertTrue(cursor.hasNext());
      DBObject dbObject = cursor.next();
      assertNotNull(dbObject);
      assertEquals(dbObject.get("name"), msg.get("name"));
      assertEquals(dbObject.get("age"), msg.get("age"));
      assertEquals(dbObject.get("birthday"), msg.get("birthday"));
    }
  }
Exemplo n.º 11
0
  @Test
  public void shouldIndexOneEvent() throws Exception {
    Configurables.configure(fixture, new Context(parameters));
    Channel channel = bindAndStartChannel(fixture);

    Transaction tx = channel.getTransaction();
    tx.begin();
    Event event = EventBuilder.withBody("event #1 or 1".getBytes());
    channel.put(event);
    tx.commit();
    tx.close();

    fixture.process();
    fixture.stop();
    client.admin().indices().refresh(Requests.refreshRequest(timestampedIndexName)).actionGet();

    assertMatchAllQuery(1, event);
    assertBodyQuery(1, event);
  }
  @BeforeMethod(groups = {"dev"})
  public static void setup() throws UnknownHostException {
    mongo = new Mongo("localhost", 27017);

    Map<String, String> ctxMap = new HashMap<String, String>();
    ctxMap.put(MongoSink.HOST, "localhost");
    ctxMap.put(MongoSink.PORT, "27017");
    ctxMap.put(MongoSink.DB_NAME, "test_events");
    ctxMap.put(MongoSink.COLLECTION, "test_log");
    ctxMap.put(MongoSink.BATCH_SIZE, "100");

    ctx.putAll(ctxMap);

    Context channelCtx = new Context();
    channelCtx.put("capacity", "1000000");
    channelCtx.put("transactionCapacity", "1000000");
    channel = new MemoryChannel();
    Configurables.configure(channel, channelCtx);
  }
Exemplo n.º 13
0
  @OnScheduled
  public void onScheduled(final SchedulingContext context) {
    try {
      channel = new NifiSinkSessionChannel(SUCCESS, FAILURE);
      channel.start();

      sink =
          SINK_FACTORY.create(
              context.getProperty(SOURCE_NAME).getValue(),
              context.getProperty(SINK_TYPE).getValue());
      sink.setChannel(channel);

      String flumeConfig = context.getProperty(FLUME_CONFIG).getValue();
      String agentName = context.getProperty(AGENT_NAME).getValue();
      String sinkName = context.getProperty(SOURCE_NAME).getValue();
      Configurables.configure(sink, getFlumeSinkContext(flumeConfig, agentName, sinkName));

      sink.start();
    } catch (Throwable th) {
      getLogger().error("Error creating sink", th);
      throw Throwables.propagate(th);
    }
  }
Exemplo n.º 14
0
  @Test
  public void shouldIndexComplexJsonEvent() throws Exception {
    Configurables.configure(fixture, new Context(parameters));
    Channel channel = bindAndStartChannel(fixture);

    Transaction tx = channel.getTransaction();
    tx.begin();
    Event event = EventBuilder.withBody("{\"event\":\"json content\",\"num\":1}".getBytes());
    channel.put(event);
    tx.commit();
    tx.close();

    fixture.process();
    fixture.stop();
    client.admin().indices().refresh(Requests.refreshRequest(timestampedIndexName)).actionGet();

    Map<String, Object> expectedBody = new HashMap<String, Object>();
    expectedBody.put("event", "json content");
    expectedBody.put("num", 1);

    assertSearch(1, performSearch(QueryBuilders.matchAllQuery()), expectedBody, event);
    assertSearch(
        1, performSearch(QueryBuilders.fieldQuery("@message.event", "json")), expectedBody, event);
  }
Exemplo n.º 15
0
  public void doTestTextBatchAppend() throws Exception {
    LOG.debug("Starting...");

    final long rollCount = 10;
    final long batchSize = 2;
    final String fileName = "PageView";

    String newPath = testPath + "/singleTextBucket";
    int totalEvents = 0;
    int i = 1, j = 1;

    // clear the test directory
    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(conf);
    Path dirPath = new Path(newPath);
    fs.delete(dirPath, true);
    fs.mkdirs(dirPath);

    Context context = new Context();
    context.put("hdfs.path", newPath);
    context.put("hdfs.rollCount", String.valueOf(rollCount));
    context.put("hdfs.batchSize", String.valueOf(batchSize));
    context.put("hdfs.filePrefix", "pageview");

    Channel channel = new MemoryChannel();
    Configurables.configure(channel, context);
    sink.setChannel(channel);
    sink.start();

    Calendar eventDate = Calendar.getInstance();
    Date currentDate = new Date();
    Map<String, String> header = new HashMap<String, String>();
    header.put("topic", "PageView");

    List<String> bodies = Lists.newArrayList();

    // 将测试的事件推入到通道中
    for (i = 1; i <= (rollCount * 10) / batchSize; i++) {
      Transaction txn = channel.getTransaction();
      txn.begin();
      for (j = 1; j <= batchSize; j++) {
        header.put("timestamp", String.valueOf(currentDate.getTime()));
        Event event = new SimpleEvent();
        eventDate.clear();
        eventDate.set(2014, i, i, i, 0);
        String body = "Test." + i + "." + j;

        event.setHeaders(header);
        event.setBody(body.getBytes());
        bodies.add(body);
        channel.put(event);
        totalEvents++;
      }
      txn.commit();
      txn.close();

      // execute sink to process the events
      sink.process();
    }
    sink.stop();

    FileStatus[] dirStat = fs.listStatus(dirPath);
    Path fList[] = FileUtil.stat2Paths(dirStat);

    long expectedFiles = totalEvents / rollCount;
    if (totalEvents % rollCount > 0) {
      expectedFiles++;
    }

    Assert.assertEquals(
        "num files wrong, found: " + Lists.newArrayList(fList), expectedFiles, fList.length);
    // 检查所有写入文件的内容
    verifyOutputTextFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);
  }
Exemplo n.º 16
0
  //    @Ignore
  @Before
  public void setUp() {

    conf = ConfigFactory.load();
    List<String> zkHosts = conf.getStringList("zookeeper.hosts");

    for (String host : zkHosts) {
      ZOOKEEPER_HOSTS += host + ",";
    }
    ZOOKEEPER_HOSTS = ZOOKEEPER_HOSTS.substring(0, ZOOKEEPER_HOSTS.length() - 1);

    List<String> kafkaHosts = conf.getStringList("kafka.hosts");

    for (String host : kafkaHosts) {
      KAFKA_HOSTS += host + ",";
    }
    KAFKA_HOSTS = KAFKA_HOSTS.substring(0, KAFKA_HOSTS.length() - 1);

    LOGGER.debug("Using Zookeeper hosts: " + ZOOKEEPER_HOSTS);
    LOGGER.debug("Using Zookeeper hosts: " + KAFKA_HOSTS);
    //        try {
    //            zookeeperServer = new ZookeeperServer();
    //            zookeeperServer.start();
    //        } catch (Exception e) {
    //            e.printStackTrace();
    //        }
    //        try {
    //            kafkaServer = new KafkaServer();
    //            kafkaServer.start();
    //        } catch (Exception e) {
    //            e.printStackTrace();
    //        }

    String[] connection = KAFKA_HOSTS.split(":");

    //        simpleConsumer = new SimpleConsumer("localhost", 9092, 60000, 1024, CLIENT_ID);
    simpleConsumer =
        new SimpleConsumer(connection[0], Integer.parseInt(connection[1]), 60000, 1024, CLIENT_ID);

    kafkaSink = new KafkaSink();

    Context kafkaContext = new Context();
    kafkaContext.put("topic", "test");
    kafkaContext.put("writeBody", "false");
    kafkaContext.put("kafka.metadata.broker.list", KAFKA_HOSTS);
    kafkaContext.put("kafka.serializer.class", "kafka.serializer.StringEncoder");

    Configurables.configure(kafkaSink, kafkaContext);

    Context channelContext = new Context();
    channelContext.put("capacity", "10000");
    channelContext.put("transactionCapacity", "200");

    channel = new MemoryChannel();
    channel.setName("junitChannel");
    Configurables.configure(channel, channelContext);

    kafkaSink.setChannel(channel);

    channel.start();
    kafkaSink.start();
  }