protected void loadBuffer(int bufferNum) {
   CloseQuietly.close(holder);
   holder = singleThreadedBuffers.get(bufferNum);
   buffer = holder.get();
   currIndex = bufferNum;
   bigEndian = buffer.order().equals(ByteOrder.BIG_ENDIAN);
 }
Exemple #2
0
 @LifecycleStop
 public void stop() {
   for (Iterable<FireChief> chiefs : this.chiefs.values()) {
     for (FireChief chief : chiefs) {
       CloseQuietly.close(chief);
     }
   }
 }
 protected void loadBuffer(int bufferNum) {
   CloseQuietly.close(holder);
   holder = singleThreadedLongBuffers.get(bufferNum);
   buffer = holder.get();
   currIndex = bufferNum;
 }
Exemple #4
0
    @Override
    public void run() {
      plumber = initPlumber();
      final Period intermediatePersistPeriod = config.getIntermediatePersistPeriod();

      try {
        plumber.startJob();

        // Delay firehose connection to avoid claiming input resources while the plumber is starting
        // up.
        firehose = initFirehose();

        long nextFlush = new DateTime().plus(intermediatePersistPeriod).getMillis();
        while (firehose.hasMore()) {
          InputRow inputRow = null;
          try {
            try {
              inputRow = firehose.nextRow();
            } catch (Exception e) {
              log.debug(e, "thrown away line due to exception, considering unparseable");
              metrics.incrementUnparseable();
              continue;
            }

            boolean lateEvent = false;
            boolean indexLimitExceeded = false;
            try {
              lateEvent = plumber.add(inputRow) == -1;
            } catch (IndexSizeExceededException e) {
              log.info("Index limit exceeded: %s", e.getMessage());
              indexLimitExceeded = true;
            }
            if (indexLimitExceeded || lateEvent) {
              metrics.incrementThrownAway();
              log.debug("Throwing away event[%s]", inputRow);

              if (indexLimitExceeded || System.currentTimeMillis() > nextFlush) {
                plumber.persist(firehose.commit());
                nextFlush = new DateTime().plus(intermediatePersistPeriod).getMillis();
              }

              continue;
            }
            final Sink sink = plumber.getSink(inputRow.getTimestampFromEpoch());
            if ((sink != null && !sink.canAppendRow()) || System.currentTimeMillis() > nextFlush) {
              plumber.persist(firehose.commit());
              nextFlush = new DateTime().plus(intermediatePersistPeriod).getMillis();
            }
            metrics.incrementProcessed();
          } catch (ParseException e) {
            if (inputRow != null) {
              log.error(e, "unparseable line: %s", inputRow);
            }
            metrics.incrementUnparseable();
          }
        }
      } catch (RuntimeException e) {
        log.makeAlert(
                e,
                "RuntimeException aborted realtime processing[%s]",
                fireDepartment.getDataSchema().getDataSource())
            .emit();
        normalExit = false;
        throw e;
      } catch (Error e) {
        log.makeAlert(
                e,
                "Exception aborted realtime processing[%s]",
                fireDepartment.getDataSchema().getDataSource())
            .emit();
        normalExit = false;
        throw e;
      } finally {
        CloseQuietly.close(firehose);
        if (normalExit) {
          plumber.finishJob();
          plumber = null;
          firehose = null;
        }
      }
    }
 private void tearDownServerAndCurator() {
   CloseQuietly.close(curator);
   CloseQuietly.close(server);
 }