コード例 #1
0
  public void testSyslogOutputFormat() throws IOException {
    // set the output format.
    FlumeConfiguration conf = FlumeConfiguration.get();
    conf.set(FlumeConfiguration.COLLECTOR_OUTPUT_FORMAT, "syslog");

    // build a sink that outputs to that format.
    File f = FileUtil.mktempdir();
    SinkBuilder builder = EscapedCustomDfsSink.builder();
    EventSink snk = builder.build(new Context(), "file:///" + f.getPath() + "/sub-%{service}");
    Event e = new EventImpl("this is a test message".getBytes());
    Attributes.setString(e, "service", "foo");
    snk.open();
    snk.append(e);
    snk.close();

    ByteArrayOutputStream exWriter = new ByteArrayOutputStream();
    SyslogEntryFormat fmt = new SyslogEntryFormat();
    fmt.format(exWriter, e);
    exWriter.close();
    String expected = new String(exWriter.toByteArray());

    // check the output to make sure it is what we expected.
    File fo = new File(f.getPath() + "/sub-foo");

    FileReader fr = new FileReader(fo);
    BufferedReader br = new BufferedReader(fr);
    String read = br.readLine() + "\n";
    assertEquals(expected, read);
  }
コード例 #2
0
  /**
   * Tests that the rolling event sink correctly tags the output filename.
   *
   * @throws InterruptedException
   */
  @Test
  public void testEscapedFilenameCloseFlushes() throws IOException, InterruptedException {
    Tagger tagger =
        new ProcessTagger() {
          @Override
          public String getTag() {
            return "-testtag";
          }

          @Override
          public String newTag() {
            return "-testtag";
          }
        };
    final File f = FileUtil.mktempdir();
    RollSink snk =
        new RollSink(new Context(), "test", new TimeTrigger(tagger, 10000), 250) {
          @Override
          protected EventSink newSink(Context ctx) throws IOException {
            return new EscapedCustomDfsSink("file:///" + f.getPath(), "sub-%{service}%{rolltag}");
          }
        };

    Event e = new EventImpl("this is a test message".getBytes());
    Attributes.setString(e, "service", "foo");
    snk.open();
    snk.append(e);
    snk.close();
    File fo = new File(f.getPath() + "/sub-foo-testtag");
    assertTrue(fo.exists());
    FileUtil.rmr(f);
  }
コード例 #3
0
  @Override
  public ReportEvent getMetrics() {
    ReportEvent rpt = new ReportEvent(getName());

    StringWriter sw = new StringWriter();
    PrintWriter out = new PrintWriter(sw);
    StringWriter csvsw = new StringWriter();
    PrintWriter csvpw = new PrintWriter(csvsw);
    for (Pair<Benchmark, StringWriter> e : benchmarks.values()) {
      out.print(e.getRight().getBuffer().toString());
      e.getLeft().printCsvLog(csvpw);
    }
    out.close();
    csvpw.close();

    Attributes.setString(rpt, A_BENCHMARK_RPT, sw.toString());
    Attributes.setString(rpt, A_BENCHMARK_CSV, csvsw.toString());
    return rpt;
  }
コード例 #4
0
ファイル: WALAckManager.java プロジェクト: angushe/flume
 @Override
 public synchronized ReportEvent getReport() {
   ReportEvent rpt = new ReportEvent(getName());
   Attributes.setLong(rpt, A_RETRANSMIT_TIMEOUT, retransmitTime);
   StringBuilder pendingAcks = new StringBuilder();
   for (Map.Entry<String, Long> e : pending.entrySet()) {
     pendingAcks.append(e.getKey());
     pendingAcks.append(":");
     pendingAcks.append(new Date(e.getValue()).toString());
     pendingAcks.append(", ");
   }
   Attributes.setString(rpt, A_PENDING_ACK_INFO, pendingAcks.toString());
   return rpt;
 }
コード例 #5
0
  /**
   * Test to write few log lines, compress using gzip, write to disk, read back the compressed file
   * and verify the written lines.
   *
   * @throws IOException
   */
  public void testGzipOutputFormat() throws IOException {
    // set the output format.
    FlumeConfiguration conf = FlumeConfiguration.get();
    conf.set(FlumeConfiguration.COLLECTOR_OUTPUT_FORMAT, "syslog");
    conf.set(FlumeConfiguration.COLLECTOR_DFS_COMPRESS_GZIP, "true");

    // build a sink that outputs to that format.
    File f = FileUtil.mktempdir();
    SinkBuilder builder = EscapedCustomDfsSink.builder();
    EventSink snk = builder.build(new Context(), "file:///" + f.getPath() + "/sub-%{service}");
    Event e = new EventImpl("this is a test message".getBytes());
    Attributes.setString(e, "service", "foo");
    snk.open();
    snk.append(e);
    snk.close();

    ByteArrayOutputStream exWriter = new ByteArrayOutputStream();
    SyslogEntryFormat fmt = new SyslogEntryFormat();
    fmt.format(exWriter, e);
    exWriter.close();
    String expected = new String(exWriter.toByteArray());

    // check the output to make sure it is what we expected.
    // read the gzip file and verify the contents

    GZIPInputStream gzin = new GZIPInputStream(new FileInputStream(f.getPath() + "/sub-foo.gz"));
    byte[] buf = new byte[1];
    StringBuilder output = new StringBuilder();

    while ((gzin.read(buf)) > 0) {
      output.append(new String(buf));
    }
    assertEquals(expected, output.toString());

    assertTrue("temp folder successfully deleted", FileUtil.rmr(f));
  }