コード例 #1
0
  /**
   * Tests that the rolling event sink correctly tags the output filename.
   *
   * @throws InterruptedException
   */
  @Test
  public void testEscapedFilenameCloseFlushes() throws IOException, InterruptedException {
    Tagger tagger =
        new ProcessTagger() {
          @Override
          public String getTag() {
            return "-testtag";
          }

          @Override
          public String newTag() {
            return "-testtag";
          }
        };
    final File f = FileUtil.mktempdir();
    RollSink snk =
        new RollSink(new Context(), "test", new TimeTrigger(tagger, 10000), 250) {
          @Override
          protected EventSink newSink(Context ctx) throws IOException {
            return new EscapedCustomDfsSink("file:///" + f.getPath(), "sub-%{service}%{rolltag}");
          }
        };

    Event e = new EventImpl("this is a test message".getBytes());
    Attributes.setString(e, "service", "foo");
    snk.open();
    snk.append(e);
    snk.close();
    File fo = new File(f.getPath() + "/sub-foo-testtag");
    assertTrue(fo.exists());
    FileUtil.rmr(f);
  }
コード例 #2
0
  public synchronized void open() throws IOException {
    // TODO (jon) be less strict. ?? need to return on and figure out why thisis
    // wrong, add
    // latches.

    // Preconditions.checkState(state == ManagerState.CLOSED,
    // "Must be in CLOSED state to open, currently " + state);

    // make the dirs if they do not exist
    if (!FileUtil.makeDirs(importDir)) {
      throw new IOException("Unable to create import dir: " + importDir);
    }
    if (!FileUtil.makeDirs(writingDir)) {
      throw new IOException("Unable to create writing dir: " + writingDir);
    }
    if (!FileUtil.makeDirs(loggedDir)) {
      throw new IOException("Unable to create logged dir: " + loggedDir);
    }
    if (!FileUtil.makeDirs(sendingDir)) {
      throw new IOException("Unable to create sending dir: " + sendingDir);
    }
    if (!FileUtil.makeDirs(errorDir)) {
      throw new IOException("Unable to create error dir: " + sendingDir);
    }

    state = ManagerState.OPEN;
  }
コード例 #3
0
  public void testSyslogOutputFormat() throws IOException {
    // set the output format.
    FlumeConfiguration conf = FlumeConfiguration.get();
    conf.set(FlumeConfiguration.COLLECTOR_OUTPUT_FORMAT, "syslog");

    // build a sink that outputs to that format.
    File f = FileUtil.mktempdir();
    SinkBuilder builder = EscapedCustomDfsSink.builder();
    EventSink snk = builder.build(new Context(), "file:///" + f.getPath() + "/sub-%{service}");
    Event e = new EventImpl("this is a test message".getBytes());
    Attributes.setString(e, "service", "foo");
    snk.open();
    snk.append(e);
    snk.close();

    ByteArrayOutputStream exWriter = new ByteArrayOutputStream();
    SyslogEntryFormat fmt = new SyslogEntryFormat();
    fmt.format(exWriter, e);
    exWriter.close();
    String expected = new String(exWriter.toByteArray());

    // check the output to make sure it is what we expected.
    File fo = new File(f.getPath() + "/sub-foo");

    FileReader fr = new FileReader(fo);
    BufferedReader br = new BufferedReader(fr);
    String read = br.readLine() + "\n";
    assertEquals(expected, read);
  }
コード例 #4
0
  /**
   * This creates an environment where we have configurations set and then serving starts. This
   * simulates a zk configstore load and then the serve call being run.
   *
   * <p>Ideally we'd create a SetupTranslatingZKMasterTestEnv, but there is an issue when trying to
   * start/shutdown and start a new master in the same process/jvm.
   */
  @Before
  public void setCfgAndStartMaster() throws TTransportException, IOException, FlumeSpecException {
    // Give ZK a temporary directory, otherwise it's possible we'll reload some
    // old configs
    tmpdir = FileUtil.mktempdir();
    FlumeConfiguration.createTestableConfiguration();
    FlumeConfiguration.get().set(FlumeConfiguration.MASTER_STORE, "memory");

    buildMaster();

    // Instead of loading from a ZK Store, we just see the config in the "deep"
    // config manager. Any translations will not occur.
    ConfigurationManager loaded = cfgMan;
    loaded.setConfig("node1", "flow", "autoCollectorSource", "null");
    loaded.setConfig("node2", "flow", "autoCollectorSource", "null");
    loaded.setConfig("node3", "flow", "autoCollectorSource", "null");
    loaded.setConfig("node4", "flow", "autoCollectorSource", "null");
    loaded.setConfig("agent", "flow", "null", "autoBEChain");

    // this is the outer configman, should have no translation.
    ConfigurationManager cfgman1 = flumeMaster.getSpecMan();
    Map<String, FlumeConfigData> cfgs1 = cfgman1.getTranslatedConfigs();
    assertEquals(0, cfgs1.size()); // no translations happened

    // start the master (which should trigger an update and translation
    flumeMaster.serve();
  }
コード例 #5
0
  @After
  public void stopMaster() throws IOException {
    if (flumeMaster != null) {
      flumeMaster.shutdown();
      flumeMaster = null;
    }

    if (tmpdir != null) {
      FileUtil.rmr(tmpdir);
      tmpdir = null;
    }
  }
コード例 #6
0
  /**
   * Test to write few log lines, compress using gzip, write to disk, read back the compressed file
   * and verify the written lines.
   *
   * @throws IOException
   */
  public void testGzipOutputFormat() throws IOException {
    // set the output format.
    FlumeConfiguration conf = FlumeConfiguration.get();
    conf.set(FlumeConfiguration.COLLECTOR_OUTPUT_FORMAT, "syslog");
    conf.set(FlumeConfiguration.COLLECTOR_DFS_COMPRESS_GZIP, "true");

    // build a sink that outputs to that format.
    File f = FileUtil.mktempdir();
    SinkBuilder builder = EscapedCustomDfsSink.builder();
    EventSink snk = builder.build(new Context(), "file:///" + f.getPath() + "/sub-%{service}");
    Event e = new EventImpl("this is a test message".getBytes());
    Attributes.setString(e, "service", "foo");
    snk.open();
    snk.append(e);
    snk.close();

    ByteArrayOutputStream exWriter = new ByteArrayOutputStream();
    SyslogEntryFormat fmt = new SyslogEntryFormat();
    fmt.format(exWriter, e);
    exWriter.close();
    String expected = new String(exWriter.toByteArray());

    // check the output to make sure it is what we expected.
    // read the gzip file and verify the contents

    GZIPInputStream gzin = new GZIPInputStream(new FileInputStream(f.getPath() + "/sub-foo.gz"));
    byte[] buf = new byte[1];
    StringBuilder output = new StringBuilder();

    while ((gzin.read(buf)) > 0) {
      output.append(new String(buf));
    }
    assertEquals(expected, output.toString());

    assertTrue("temp folder successfully deleted", FileUtil.rmr(f));
  }
コード例 #7
0
ファイル: FlumeNode.java プロジェクト: jmhsieh/flume
  /**
   * This function checks the agent logs dir to make sure that the process has the ability to the
   * directory if necessary, that the path if it does exist is a directory, and that it can in fact
   * create files inside of the directory. If it fails any of these, it throws an exception.
   *
   * <p>Finally, it checks to see if the path is in /tmp and warns the user that this may not be the
   * best idea.
   */
  public static void nodeConfigChecksOk() throws IOException {
    // TODO (jon) if we add more checks in here, make the different managers
    // responsible for throwing an Exception on construction instead.

    FlumeConfiguration conf = FlumeConfiguration.get();

    String s = conf.getAgentLogsDir();
    File f = new File(s);

    if (!FileUtil.makeDirs(f)) {
      throw new IOException("Path to Log dir cannot be created: '" + s + "'.  Check permissions?");
    }

    if (!f.isDirectory()) {
      throw new IOException("Log dir '" + s + "' already exists as a file.  Check log dir path.");
    }

    File f2 = null;
    try {
      f2 = File.createTempFile("initcheck", ".test", f);
    } catch (IOException e) {
      throw new IOException("Failure to write in log directory: '" + s + "'.  Check permissions?");
    }
    if (!f2.delete()) {
      throw new IOException(
          "Unable to delete "
              + f2
              + " from log directory "
              + "(but writing succeeded) - something is strange here");
    }

    File tmp = new File("/tmp");
    File cur = f;
    while (cur != null) {
      if (cur.equals(tmp)) {
        LOG.warn("Log directory is writing inside of /tmp.  This data may not survive reboot!");
        break;
      }
      cur = cur.getParentFile();
    }
  }