private String format(Event e) {
   Date d = new Date(e.getTimestamp());
   String data =
       String.format(
           "%s %s %s: %s\n",
           DateUtils.asISO8601(d),
           e.getPriority(),
           "log4j",
           StringEscapeUtils.escapeJava(new String(e.getBody())));
   return data;
 }
  @Test
  public void testAvroNativeJson() throws IOException {
    AvroNativeFileOutputFormat format = new AvroNativeFileOutputFormat();
    ByteArrayOutputStream sos = new ByteArrayOutputStream();
    format.format(sos, e);
    format.close();
    byte[] bytes = sos.toByteArray();

    ReflectData reflectData = ReflectData.get();
    Schema schema = reflectData.getSchema(EventImpl.class);
    ReflectDatumReader<EventImpl> dr = new ReflectDatumReader<EventImpl>(schema);
    ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
    DataFileStream<EventImpl> dec = new DataFileStream<EventImpl>(bais, dr);

    Event er = dec.next();
    assertEquals(e.getHost(), er.getHost());
    assertEquals(e.getNanos(), er.getNanos());
    assertEquals(e.getPriority(), er.getPriority());
    assertTrue(Arrays.equals(e.getBody(), er.getBody()));
  }
  /** Calculate the crc based on the body of the message and xor it into the checksum. */
  public void append(Event e) throws IOException, InterruptedException {
    chk.reset();
    chk.update(e.getBody());
    long curchk = chk.getValue();
    checksum ^= curchk; // update but do not send.

    e.set(ATTR_ACK_TYPE, CHECKSUM_MSG);
    e.set(ATTR_ACK_TAG, tag);
    e.set(ATTR_ACK_HASH, ByteBuffer.allocate(8).putLong(curchk).array());
    super.append(e);
  }
  @Override
  public void append(Event e) throws IOException, InterruptedException {
    String helloWorldBody = "Hello World! -- " + new String(e.getBody());

    // make a copy of the event, but with the new body string.
    EventImpl e2 =
        new EventImpl(
            helloWorldBody.getBytes(),
            e.getTimestamp(),
            e.getPriority(),
            e.getNanos(),
            e.getHost(),
            e.getAttrs());

    super.append(e2);
  }
  private void assertCorrectResponse(int count, Event event, SearchResponse response) {
    SearchHits hits = response.getHits();

    assertEquals(count, hits.getTotalHits());

    SearchHit hit = hits.getAt(0);

    Map<String, Object> source = hit.getSource();

    assertEquals(event.getHost(), source.get("host"));
    assertEquals("1970-01-01T00:00:00.000Z", source.get("timestamp"));
    assertEquals(event.getPriority().name(), source.get("priority"));

    @SuppressWarnings("unchecked")
    Map<String, Object> message = (Map<String, Object>) source.get("message");
    assertEquals(new String(event.getBody()), message.get("text"));

    @SuppressWarnings("unchecked")
    Map<String, Object> fields = (Map<String, Object>) source.get("fields");

    assertEquals(new String(event.getAttrs().get("attr1")), fields.get("attr1"));
    assertEquals(new String(event.getAttrs().get("attr2")), fields.get("attr2"));
  }
示例#6
0
    public void run() {
      EventSink sink = null;
      EventSource source = null;
      synchronized (DirectDriver.this) {
        sink = DirectDriver.this.sink;
        source = DirectDriver.this.source;
      }
      try {
        synchronized (stateSignal) {
          state = DriverState.OPENING;
          stateSignal.notifyAll();
        }
        source.open();
        sink.open();
      } catch (Exception e) {
        // if open is interrupted or has an exception there was a problem.
        LOG.error("Closing down due to exception on open calls");
        errorCleanup(PumperThread.this.getName(), e);
        return;
      }

      synchronized (stateSignal) {
        lastExn = null;
        state = DriverState.ACTIVE;
        stateSignal.notifyAll();
      }

      LOG.debug("Starting driver " + DirectDriver.this);
      try {
        Event e = null;

        while (!stopped) {
          try {
            e = source.next();
          } catch (InterruptedException eIn) {
            // If we are interrupted then its time to go down. re-throw the exception.
            // Details are logged by the outer catch block
            throw eIn;
          } catch (Exception eI) {
            // If this is a chained or converted Interrupt then throw it back
            if (eI.getCause() instanceof InterruptedException) throw eI;

            // If there's an exception, try to reopen the source
            // if the open or close still raises an exception, then bail out
            LOG.warn("Exception in source: " + source.getName(), eI);
            LOG.warn("Retrying after Error in source: " + source.getName());
            source.close();
            source.open();
            LOG.info(" Source Retry successful");
            e = source.next(); // try to get the next event again
          }

          if (e == null) {
            LOG.warn("{}: Event is null or empty()", source.getName());
            if ("NullSource".equals(source.getName())) {
              break;
            } else {
              continue;
            }
          }

          if (e.getBody().length == 0) {
            LOG.warn("Event is empty; continue");
            continue;
          }
          nextCount++;

          try {
            sink.append(e);
          } catch (InterruptedException eIn) {
            // If we are interrupted then its time to go down. re-throw the exception.
            // Details are logged by the outer catch block
            throw eIn;
          } catch (Exception eI) {
            // If this is a chained or converted Interrupt then throw it back
            if (eI.getCause() instanceof InterruptedException) throw eI;

            // If there's an exception, try to reopen the source
            // if the open or close still raises an exception, then bail out
            LOG.warn("Exception in sink: " + sink.getName(), eI);
            LOG.warn("Retrying after Error in source: " + sink.getName());
            sink.close();
            sink.open();
            LOG.info("Sink Retry successful");
            sink.append(e); // try to sink the event again
          }
          appendCount++;
          appendBytes += e.getBody().length;
        }
      } catch (Exception e1) {
        // Catches all exceptions or throwables. This is a separate thread
        LOG.error("Closing down due to exception during append calls");
        errorCleanup(PumperThread.this.getName(), e1);
        return;
      }

      try {
        synchronized (stateSignal) {
          state = DriverState.CLOSING;
          stateSignal.notifyAll();
        }
        source.close();
        sink.close();
      } catch (Exception e) {
        LOG.error("Closing down due to exception during close calls");
        errorCleanup(PumperThread.this.getName(), e);
        return;
      }
      synchronized (stateSignal) {
        LOG.debug("Driver completed: " + DirectDriver.this);
        stopped = true;
        state = DriverState.IDLE;
        stateSignal.notifyAll();
      }
    }