Exemple #1
0
  /**
   * Tests to make sure the report sink receives data.
   *
   * @throws InterruptedException
   */
  @Test
  public void testReportSink() throws FlumeSpecException, IOException, InterruptedException {
    String spec =
        "{benchinject(\"foo\") => {benchreport(\"report\", \"[ console , counter(\\\"test\\\") ]\")  => null } }";
    EventSink snk = FlumeBuilder.buildSink(new ReportTestingContext(), spec);
    snk.open();
    snk.append(new EventImpl(new byte[0]));
    snk.append(new EventImpl(new byte[0]));
    snk.close();

    CounterSink ctr = (CounterSink) ReportManager.get().getReportable("test");
    Assert.assertEquals(1, ctr.getCount());
  }
  @Test
  public void testAutoRoll() throws IOException, InterruptedException {
    RollSink snk = new RollSink(new ReportTestingContext(), "counter(\"foo\")", 2000, 10000); // two
    // second sleeper, but check period is really long

    Event e = new EventImpl("this is a test message".getBytes());
    snk.open();
    snk.append(e);
    CounterSink cnt = (CounterSink) ReportManager.get().getReportable("foo");
    Clock.sleep(3000); // sleep 3s

    // the roller automatically flushed!
    assertEquals(1, cnt.getCount());
    snk.close();
  }
Exemple #3
0
  @Test
  public void testCountSink() throws IOException, InterruptedException {
    Benchmark b = new Benchmark("nullsink");
    b.mark("begin");
    TextFileSource txt = new TextFileSource(HADOOP_DATA[0]);
    txt.open();
    MemorySinkSource mem = new MemorySinkSource();
    mem.open();
    EventUtil.dumpAll(txt, mem);

    b.mark("disk_loaded");

    CounterSink snk = new CounterSink("counter");
    EventUtil.dumpAll(mem, snk);
    b.mark(snk.getName() + " done", snk.getCount());

    b.done();
  }
  public void doTestLogicalNodesConcurrentDFOMans(final int threads, final int events, int timeout)
      throws IOException, InterruptedException, FlumeSpecException {
    FlumeTestHarness.setupLocalWriteDir();
    FlumeMaster master = new FlumeMaster();
    FlumeNode node = new FlumeNode(new DirectMasterRPC(master), false, false);
    final Reportable[] dfos = new Reportable[threads];

    for (int i = 0; i < threads; i++) {
      String name = "test." + i;
      String report = "report." + i;
      int count = events + i;
      String src = "asciisynth(" + count + ",100)";
      String snk = "{ diskFailover => counter(\"" + report + "\") } ";
      node.getLogicalNodeManager().testingSpawn(name, src, snk);
      dfos[i] = node.getLogicalNodeManager().get(name);
    }

    // TODO (jon) using sleep is cheating to give all threads a chance to start.
    // Test seems flakey without this due to a race condition.
    Thread.sleep(500);

    // wait for all to be done.
    waitForEmptyDFOs(node, timeout);

    // check to make sure everyone got the right number of events
    boolean success = true;

    for (int i = 0; i < threads; i++) {
      LOG.info(dfos[i].getMetrics());
    }

    for (int i = 0; i < threads; i++) {

      CounterSink cnt = (CounterSink) ReportManager.get().getReportable("report." + i);
      LOG.info(i + " expected " + (events + i) + " and got " + cnt.getCount());
      success &= ((events + i) == cnt.getCount());
      assertEquals(events + i, cnt.getCount());
    }
    assertTrue("Counts did not line up", success);

    FlumeTestHarness.cleanupLocalWriteDir();
  }
  @Test
  public void testSimpleIntervalSamplerSink() throws IOException {
    System.out.println("Simple interval sampler sink");
    CounterSink count = new CounterSink("count");
    IntervalSampler<CounterSink> sink = new IntervalSampler<CounterSink>(count, 10);
    sink.open();
    for (int i = 0; i < 30; i++) {
      sink.append(new EventImpl(("test " + i).getBytes()));
    }
    Assert.assertEquals(3, count.getCount());
    sink.close();

    // do the boundary condition.
    CounterSink count2 = new CounterSink("count");
    IntervalSampler<CounterSink> sink2 = new IntervalSampler<CounterSink>(count2, 10);
    sink2.open();
    for (int i = 0; i < 31; i++) {
      sink2.append(new EventImpl(("test " + i).getBytes()));
    }
    Assert.assertEquals(4, count2.getCount());
    sink2.close();
  }
  /**
   * This tests the new failover builder that uses specs strings as arguments and instantiates them!
   *
   * @throws InterruptedException
   */
  @Test
  public void testBackoffFailoverBuilder() throws IOException, InterruptedException {
    SinkBuilder bld = BackOffFailOverSink.builder();
    EventSink snk =
        bld.build(
            new ReportTestingContext(),
            "{intervalFlakeyAppend(2) => counter(\"pri\") } ",
            "counter(\"sec\")");
    snk.open();

    Event e = new EventImpl("foo".getBytes());
    snk.append(e);
    snk.append(e);
    snk.append(e);
    snk.append(e);
    snk.append(e);

    snk.close();
    CounterSink priCnt = (CounterSink) ReportManager.get().getReportable("pri");
    CounterSink secCnt = (CounterSink) ReportManager.get().getReportable("sec");
    // these are timing based, may fail.
    Assert.assertEquals(1, priCnt.getCount());
    Assert.assertEquals(4, secCnt.getCount());
  }
  /**
   * tests a series of messages being sent when append of the primary will fail succeed or fail
   * based on its twiddle state.
   *
   * @throws InterruptedException
   */
  @Test
  public void testFailOverSink() throws IOException, InterruptedException {
    MockClock mock = new MockClock(0);
    Clock.setClock(mock);

    CounterSink primary = new CounterSink("primary");
    CounterSink secondary = new CounterSink("backup");
    ExceptionTwiddleDecorator<CounterSink> twiddle =
        new ExceptionTwiddleDecorator<CounterSink>(primary);
    BackOffFailOverSink failsink =
        new BackOffFailOverSink(twiddle, secondary, 100, 10000); // 100 ms
    // initial
    // backoff,
    // 10000ms max
    // backoff
    failsink.open();

    Event e = new EventImpl("event".getBytes());
    // two successful appends to primary.
    failsink.append(e);
    failsink.append(e);
    System.out.println(mock);
    System.out.printf(
        "pri: %4d sec: %4d fail: %4d\n",
        primary.getCount(), secondary.getCount(), failsink.getFails());
    Assert.assertEquals(2, primary.getCount());
    Assert.assertEquals(0, secondary.getCount());

    mock.forward(100);
    twiddle.setAppendOk(false); // go to fail over.
    failsink.append(e); // primary fails and automatically go to 2ndary
    System.out.println(mock);
    System.out.printf(
        "pri: %4d sec: %4d fail: %4d\n",
        primary.getCount(), secondary.getCount(), failsink.getFails());
    Assert.assertEquals(1, failsink.getFails()); // one attempt on primary
    // failed.
    Assert.assertEquals(2, primary.getCount()); // same as before,
    Assert.assertEquals(1, secondary.getCount()); // message went to the
    // secondary

    mock.forward(50);
    failsink.append(e); // skip primary and just go to 2ndary
    System.out.println(mock);
    System.out.printf(
        "pri: %4d sec: %4d fail: %4d\n",
        primary.getCount(), secondary.getCount(), failsink.getFails());
    Assert.assertEquals(1, failsink.getFails()); // still only one attempt on
    // primary
    Assert.assertEquals(2, primary.getCount()); // same as before,
    Assert.assertEquals(2, secondary.getCount()); // message went to the
    // secondary

    mock.forward(50);
    failsink.append(e); // after this fails backoff is now 200
    System.out.println(mock);
    System.out.printf(
        "pri: %4d sec: %4d fail: %4d\n",
        primary.getCount(), secondary.getCount(), failsink.getFails());
    Assert.assertEquals(2, failsink.getFails()); // try primary
    Assert.assertEquals(0, primary.getCount()); // resets because primary
    // restarted
    // (and still fails)
    Assert.assertEquals(3, secondary.getCount()); // but failover to secondary

    mock.forward(200);
    failsink.append(e); // should go to 2ndary, after this fails backoff is now
    // 400
    System.out.println(mock);
    System.out.printf(
        "pri: %4d sec: %4d fail: %4d\n",
        primary.getCount(), secondary.getCount(), failsink.getFails());
    Assert.assertEquals(3, failsink.getFails());
    Assert.assertEquals(0, primary.getCount());
    Assert.assertEquals(4, secondary.getCount());

    twiddle.setAppendOk(true);
    failsink.append(e); // even through primary is ok, we are backing off
    System.out.println(mock);
    System.out.printf(
        "pri: %4d sec: %4d fail: %4d\n",
        primary.getCount(), secondary.getCount(), failsink.getFails());
    Assert.assertEquals(3, failsink.getFails());
    Assert.assertEquals(0, primary.getCount());
    Assert.assertEquals(5, secondary.getCount());

    mock.forward(400);
    failsink.append(e); // now that the backoff has expired, we retry the
    // primary and succeed
    System.out.println(mock);
    System.out.printf(
        "pri: %4d sec: %4d fail: %4d\n",
        primary.getCount(), secondary.getCount(), failsink.getFails());
    Assert.assertEquals(3, failsink.getFails());
    Assert.assertEquals(1, primary.getCount());
    Assert.assertEquals(5, secondary.getCount());

    // this should succeed, with the counts being equal in primary and
    // secondary.
    failsink.close();
  }
  /**
   * Purposely tests backoff timeout.
   *
   * @throws InterruptedException
   */
  @Test
  public void testFailTimeout() throws IOException, InterruptedException {
    System.out.println("===========================");
    MockClock mock = new MockClock(0);
    Clock.setClock(mock);

    CounterSink primary = new CounterSink("primary");
    CounterSink secondary = new CounterSink("backup");
    ExceptionTwiddleDecorator<CounterSink> twiddle =
        new ExceptionTwiddleDecorator<CounterSink>(primary);
    BackOffFailOverSink failsink = new BackOffFailOverSink(twiddle, secondary, 100, 1000); // 100 ms
    // initial
    // backoff,
    // 10000ms max
    // backoff
    failsink.open();

    Event e = new EventImpl("event".getBytes());

    mock.forward(100);
    twiddle.setAppendOk(false); // go to fail over.
    failsink.append(e);
    failsink.append(e);
    System.out.println(mock);
    System.out.printf(
        "pri: %4d sec: %4d fail: %4d\n",
        primary.getCount(), secondary.getCount(), failsink.getFails());

    mock.forward(100);
    failsink.append(e);
    failsink.append(e);
    System.out.println(mock);
    System.out.printf(
        "pri: %4d sec: %4d fail: %4d\n",
        primary.getCount(), secondary.getCount(), failsink.getFails());

    mock.forward(200);
    failsink.append(e);
    failsink.append(e);
    System.out.println(mock);
    System.out.printf(
        "pri: %4d sec: %4d fail: %4d\n",
        primary.getCount(), secondary.getCount(), failsink.getFails());

    mock.forward(400);
    failsink.append(e);
    failsink.append(e);
    System.out.println(mock);
    System.out.printf(
        "pri: %4d sec: %4d fail: %4d\n",
        primary.getCount(), secondary.getCount(), failsink.getFails());
    Assert.assertEquals(4, failsink.getFails());
    Assert.assertEquals(0, primary.getCount());
    Assert.assertEquals(8, secondary.getCount());

    mock.forward(800);
    failsink.append(e);
    failsink.append(e);
    System.out.println(mock);
    System.out.printf(
        "pri: %4d sec: %4d fail: %4d\n",
        primary.getCount(), secondary.getCount(), failsink.getFails());
    Assert.assertEquals(5, failsink.getFails());
    Assert.assertEquals(0, primary.getCount());
    Assert.assertEquals(10, secondary.getCount());

    // without capping there would be no new fail here bug still the
    // twelve on the secondary count.
    mock.forward(1000);
    failsink.append(e);
    failsink.append(e);
    System.out.println(mock);
    System.out.printf(
        "pri: %4d sec: %4d fail: %4d\n",
        primary.getCount(), secondary.getCount(), failsink.getFails());
    Assert.assertEquals(6, failsink.getFails());
    Assert.assertEquals(0, primary.getCount());
    Assert.assertEquals(12, secondary.getCount());
  }