Ejemplo n.º 1
0
  /**
   * Test metrics
   *
   * @throws InterruptedException
   * @throws IOException
   */
  @Test
  public void testGetRollMetrics()
      throws JSONException, FlumeSpecException, IOException, InterruptedException {
    ReportTestUtils.setupSinkFactory();

    EventSink snk = FlumeBuilder.buildSink(new ReportTestingContext(), "roll(100) { one } ");
    ReportEvent rpt = ReportUtil.getFlattenedReport(snk);
    LOG.info(ReportUtil.toJSONObject(rpt).toString());
    assertNotNull(rpt.getLongMetric(RollSink.A_ROLLFAILS));
    assertNotNull(rpt.getLongMetric(RollSink.A_ROLLS));
    assertEquals("one", rpt.getStringMetric(RollSink.A_ROLLSPEC));
    assertNull(rpt.getStringMetric("One.name"));

    // need to open to have sub sink show up
    snk.open();

    ReportEvent all = ReportUtil.getFlattenedReport(snk);
    LOG.info(ReportUtil.toJSONObject(all).toString());
    assertNotNull(rpt.getLongMetric(RollSink.A_ROLLFAILS));
    assertNotNull(rpt.getLongMetric(RollSink.A_ROLLS));
    assertEquals("one", rpt.getStringMetric(RollSink.A_ROLLSPEC));
    assertEquals("One", all.getStringMetric("One.name"));

    snk.close();
  }
Ejemplo n.º 2
0
  public void testSyslogOutputFormat() throws IOException {
    // set the output format.
    FlumeConfiguration conf = FlumeConfiguration.get();
    conf.set(FlumeConfiguration.COLLECTOR_OUTPUT_FORMAT, "syslog");

    // build a sink that outputs to that format.
    File f = FileUtil.mktempdir();
    SinkBuilder builder = EscapedCustomDfsSink.builder();
    EventSink snk = builder.build(new Context(), "file:///" + f.getPath() + "/sub-%{service}");
    Event e = new EventImpl("this is a test message".getBytes());
    Attributes.setString(e, "service", "foo");
    snk.open();
    snk.append(e);
    snk.close();

    ByteArrayOutputStream exWriter = new ByteArrayOutputStream();
    SyslogEntryFormat fmt = new SyslogEntryFormat();
    fmt.format(exWriter, e);
    exWriter.close();
    String expected = new String(exWriter.toByteArray());

    // check the output to make sure it is what we expected.
    File fo = new File(f.getPath() + "/sub-foo");

    FileReader fr = new FileReader(fo);
    BufferedReader br = new BufferedReader(fr);
    String read = br.readLine() + "\n";
    assertEquals(expected, read);
  }
Ejemplo n.º 3
0
  /**
   * Tests to make sure the report sink receives data.
   *
   * @throws InterruptedException
   */
  @Test
  public void testReportSink() throws FlumeSpecException, IOException, InterruptedException {
    String spec =
        "{benchinject(\"foo\") => {benchreport(\"report\", \"[ console , counter(\\\"test\\\") ]\")  => null } }";
    EventSink snk = FlumeBuilder.buildSink(new ReportTestingContext(), spec);
    snk.open();
    snk.append(new EventImpl(new byte[0]));
    snk.append(new EventImpl(new byte[0]));
    snk.close();

    CounterSink ctr = (CounterSink) ReportManager.get().getReportable("test");
    Assert.assertEquals(1, ctr.getCount());
  }
  /*
   * This tests to see if the seqfile event sink releases file handles.
   *
   * Default handle limit per process is around 1000 so this should be ample to
   * cause problems.
   */
  @Test
  public void testSeqfileEventSinkHandleExhaust() throws IOException {
    BenchmarkHarness.setupLocalWriteDir();
    File tmp = BenchmarkHarness.tmpdir;

    for (int i = 0; i < 3000; i++) {
      File path = new File(tmp, "" + i);
      EventSink snk = new SeqfileEventSink(path);
      snk.open();
      Event e = new EventImpl(("foo " + i).getBytes());
      snk.append(e);
      snk.close();
    }
  }
Ejemplo n.º 5
0
  @Test
  public void testSimple() throws IOException, InterruptedException {
    EventSource src = new NoNlASCIISynthSource(25, 100, 1);
    EventSink snk = new ConsoleEventSink();
    EventSink snk2 = new BenchmarkReportDecorator<EventSink>("report", snk);
    EventSink snk3 = new BenchmarkInjectDecorator<EventSink>(snk2);

    DirectDriver connect = new DirectDriver(src, snk3);
    src.open();
    snk3.open();
    connect.start();
    connect.join(Long.MAX_VALUE);
    src.close();
    snk3.close();
    snk2.getReport().toText(new OutputStreamWriter(System.err));
  }
  /**
   * Checks for Benchmark Tags. If there are not tags events are passed through. If there are, there
   * are three kinds - 'start' which instantiates a benchmark; 'first' which starts a benchmark; and
   * 'stop' which ends a benchmark. These are consumed by this decorator.
   */
  @Override
  public void append(Event e) throws IOException, InterruptedException {
    byte[] bench = e.get(BenchmarkInjectDecorator.ATTR_BENCHMARK);
    if (bench == null) {
      // This is the normal case -- a regular message
      getSink().append(e);
      return;
    }

    // All of these messages are silently consumed and not forwarded
    byte[] tagbytes = e.get(BenchmarkInjectDecorator.ATTR_BENCHMARK_TAG);
    Preconditions.checkNotNull(tagbytes);
    String tag = new String(tagbytes, CharEncUtils.RAW);

    if (Arrays.equals(bench, BenchmarkInjectDecorator.BENCH_START)) {
      StringWriter out = new StringWriter();
      PrintWriter pw = new PrintWriter(out);
      Benchmark b = new Benchmark(tag, pw, pw);
      b.mark("benchmarkStart");
      benchmarks.put(tag, new Pair<Benchmark, StringWriter>(b, out));
    } else if (Arrays.equals(bench, BenchmarkInjectDecorator.BENCH_FIRST)) {
      Benchmark b = benchmarks.get(tag).getLeft();
      b.mark("benchmarkFirst");
    } else if (Arrays.equals(bench, BenchmarkInjectDecorator.BENCH_STOP)) {
      Benchmark b = benchmarks.get(tag).getLeft();
      b.mark("benchmarkDone");
      b.done();

      ReportEvent rpt = getMetrics();
      LOG.info(rpt.toText());
      reportSink.append(rpt);

    } else if (Arrays.equals(bench, BenchmarkInjectDecorator.BENCH_ERROR)) {
      Benchmark b = benchmarks.get(tag).getLeft();
      b.mark("benchmarkError");
      b.done();
      LOG.info(getMetrics().toText());

      ReportEvent rpt = getMetrics();
      LOG.info(rpt.toText());
      reportSink.append(rpt);
    } else {
      String msg = "Unexpected Benchmark event type: " + tag;
      LOG.error(msg);
      throw new IllegalArgumentException(msg);
    }
  }
  /**
   * This test builds a disk failover and then attempts to roll the output of it. The diskFailover
   * is set to retry every 1s (1000ms). We then check to see if the number of elements has gone up
   * for at most 3s.
   */
  @Test
  public void testAgentDFOCollector() throws IOException, FlumeSpecException, InterruptedException {
    String agentCollector = "{diskFailover(1000) => roll (100000) { null } }";
    Event e = new EventImpl("foo".getBytes());
    EventSink agent = FlumeBuilder.buildSink(new Context(), agentCollector);
    agent.open();
    agent.append(e);

    for (int i = 0; i < 30; i++) {
      Clock.sleep(100);
      ReportEvent r = mem.getReport();
      LOG.info(r);
      if (r.getLongMetric("number of events") > 0) {
        return;
      }
    }
    fail("Test timed out, event didn't make it");
  }
Ejemplo n.º 8
0
 @Override
 public Map<String, Reportable> getSubMetrics() {
   Map<String, Reportable> map = new HashMap<String, Reportable>();
   if (src != null) {
     map.put("source." + src.getName(), src);
   }
   if (snk != null) {
     map.put("sink." + snk.getName(), snk);
   }
   return map;
 }
Ejemplo n.º 9
0
  @Test
  public void testSizeTriggerFunctional()
      throws FlumeSpecException, IOException, InterruptedException {
    // a ridiculous amount of time trigger time to forces size trigger
    EventSink snk =
        FlumeBuilder.buildSink(
            LogicalNodeContext.testingContext(), "roll(1000000, trigger=size(10)) { console }");
    snk.open();

    // Events from this loop:
    // 0, 1, trigger, 2, 3, trigger, 4, 5, trigger, 6, 7, trigger, 8 ,9,
    for (int i = 0; i < 10; i++) {
      snk.append(new EventImpl("6chars".getBytes()));
    }
    snk.close();

    ReportEvent rpt = snk.getMetrics();
    // See above for why there are 4 triggers:
    assertEquals(4, (long) rpt.getLongMetric(RollSink.A_ROLLS));
  }
Ejemplo n.º 10
0
  @Deprecated
  public synchronized void getReports(Map<String, ReportEvent> reports) {
    String phyName = FlumeNode.getInstance().getPhysicalNodeName();
    String rprefix = phyName + "." + getName() + ".";

    if (snk != null) {
      snk.getReports(rprefix, reports);
    }
    if (src != null) {
      src.getReports(rprefix, reports);
    }
  }
Ejemplo n.º 11
0
  /**
   * Test to write few log lines, compress using gzip, write to disk, read back the compressed file
   * and verify the written lines.
   *
   * @throws IOException
   */
  public void testGzipOutputFormat() throws IOException {
    // set the output format.
    FlumeConfiguration conf = FlumeConfiguration.get();
    conf.set(FlumeConfiguration.COLLECTOR_OUTPUT_FORMAT, "syslog");
    conf.set(FlumeConfiguration.COLLECTOR_DFS_COMPRESS_GZIP, "true");

    // build a sink that outputs to that format.
    File f = FileUtil.mktempdir();
    SinkBuilder builder = EscapedCustomDfsSink.builder();
    EventSink snk = builder.build(new Context(), "file:///" + f.getPath() + "/sub-%{service}");
    Event e = new EventImpl("this is a test message".getBytes());
    Attributes.setString(e, "service", "foo");
    snk.open();
    snk.append(e);
    snk.close();

    ByteArrayOutputStream exWriter = new ByteArrayOutputStream();
    SyslogEntryFormat fmt = new SyslogEntryFormat();
    fmt.format(exWriter, e);
    exWriter.close();
    String expected = new String(exWriter.toByteArray());

    // check the output to make sure it is what we expected.
    // read the gzip file and verify the contents

    GZIPInputStream gzin = new GZIPInputStream(new FileInputStream(f.getPath() + "/sub-foo.gz"));
    byte[] buf = new byte[1];
    StringBuilder output = new StringBuilder();

    while ((gzin.read(buf)) > 0) {
      output.append(new String(buf));
    }
    assertEquals(expected, output.toString());

    assertTrue("temp folder successfully deleted", FileUtil.rmr(f));
  }
Ejemplo n.º 12
0
  @Deprecated
  public synchronized ReportEvent getReport() {
    ReportEvent rpt = new ReportEvent(nodeName);
    rpt.setStringMetric("nodename", nodeName);
    rpt.setStringMetric("version", new Date(lastGoodCfg.timestamp).toString());
    rpt.setStringMetric("state", state.state.toString());
    rpt.setStringMetric("hostname", state.host);
    rpt.setStringMetric(
        "sourceConfig", (lastGoodCfg.sourceConfig == null) ? "" : lastGoodCfg.sourceConfig);
    rpt.setStringMetric(
        "sinkConfig", (lastGoodCfg.sinkConfig == null) ? "" : lastGoodCfg.sinkConfig);
    rpt.setStringMetric("message", nodeMsg);
    rpt.setLongMetric(A_RECONFIGURES, reconfigures.get());
    rpt.setStringMetric("physicalnode", state.physicalNode);

    if (snk != null) {
      rpt.hierarchicalMerge("sink", snk.getReport());
    }

    return rpt;
  }
 @Override
 public void close() throws IOException, InterruptedException {
   super.close();
   reportSink.close();
   LOG.info(new String(getMetrics().getBody()));
 }
 @Override
 public void open() throws IOException, InterruptedException {
   super.open();
   reportSink.open();
 }
Ejemplo n.º 15
0
 @Override
 public String toString() {
   return source.getClass().getSimpleName() + " | " + sink.getName();
 }
Ejemplo n.º 16
0
    public void run() {
      EventSink sink = null;
      EventSource source = null;
      synchronized (DirectDriver.this) {
        sink = DirectDriver.this.sink;
        source = DirectDriver.this.source;
      }
      try {
        synchronized (stateSignal) {
          state = DriverState.OPENING;
          stateSignal.notifyAll();
        }
        source.open();
        sink.open();
      } catch (Exception e) {
        // if open is interrupted or has an exception there was a problem.
        LOG.error("Closing down due to exception on open calls");
        errorCleanup(PumperThread.this.getName(), e);
        return;
      }

      synchronized (stateSignal) {
        lastExn = null;
        state = DriverState.ACTIVE;
        stateSignal.notifyAll();
      }

      LOG.debug("Starting driver " + DirectDriver.this);
      try {
        Event e = null;

        while (!stopped) {
          try {
            e = source.next();
          } catch (InterruptedException eIn) {
            // If we are interrupted then its time to go down. re-throw the exception.
            // Details are logged by the outer catch block
            throw eIn;
          } catch (Exception eI) {
            // If this is a chained or converted Interrupt then throw it back
            if (eI.getCause() instanceof InterruptedException) throw eI;

            // If there's an exception, try to reopen the source
            // if the open or close still raises an exception, then bail out
            LOG.warn("Exception in source: " + source.getName(), eI);
            LOG.warn("Retrying after Error in source: " + source.getName());
            source.close();
            source.open();
            LOG.info(" Source Retry successful");
            e = source.next(); // try to get the next event again
          }

          if (e == null) {
            LOG.warn("{}: Event is null or empty()", source.getName());
            if ("NullSource".equals(source.getName())) {
              break;
            } else {
              continue;
            }
          }

          if (e.getBody().length == 0) {
            LOG.warn("Event is empty; continue");
            continue;
          }
          nextCount++;

          try {
            sink.append(e);
          } catch (InterruptedException eIn) {
            // If we are interrupted then its time to go down. re-throw the exception.
            // Details are logged by the outer catch block
            throw eIn;
          } catch (Exception eI) {
            // If this is a chained or converted Interrupt then throw it back
            if (eI.getCause() instanceof InterruptedException) throw eI;

            // If there's an exception, try to reopen the source
            // if the open or close still raises an exception, then bail out
            LOG.warn("Exception in sink: " + sink.getName(), eI);
            LOG.warn("Retrying after Error in source: " + sink.getName());
            sink.close();
            sink.open();
            LOG.info("Sink Retry successful");
            sink.append(e); // try to sink the event again
          }
          appendCount++;
          appendBytes += e.getBody().length;
        }
      } catch (Exception e1) {
        // Catches all exceptions or throwables. This is a separate thread
        LOG.error("Closing down due to exception during append calls");
        errorCleanup(PumperThread.this.getName(), e1);
        return;
      }

      try {
        synchronized (stateSignal) {
          state = DriverState.CLOSING;
          stateSignal.notifyAll();
        }
        source.close();
        sink.close();
      } catch (Exception e) {
        LOG.error("Closing down due to exception during close calls");
        errorCleanup(PumperThread.this.getName(), e);
        return;
      }
      synchronized (stateSignal) {
        LOG.debug("Driver completed: " + DirectDriver.this);
        stopped = true;
        state = DriverState.IDLE;
        stateSignal.notifyAll();
      }
    }