@Test
  public void testSerdeWithNonDefaults() throws Exception {
    String jsonStr =
        "{\n"
            + "  \"type\": \"realtime\",\n"
            + "  \"maxRowsInMemory\": 100,\n"
            + "  \"intermediatePersistPeriod\": \"PT1H\",\n"
            + "  \"windowPeriod\": \"PT1H\",\n"
            + "  \"basePersistDirectory\": \"/tmp/xxx\",\n"
            + "  \"maxPendingPersists\": 100,\n"
            + "  \"buildV9Directly\": false,\n"
            + "  \"persistThreadPriority\": 100,\n"
            + "  \"mergeThreadPriority\": 100,\n"
            + "  \"reportParseExceptions\": true,\n"
            + "  \"handoffConditionTimeout\": 100\n"
            + "}";

    ObjectMapper mapper = TestHelper.getObjectMapper();
    RealtimeTuningConfig config =
        (RealtimeTuningConfig)
            mapper.readValue(
                mapper.writeValueAsString(mapper.readValue(jsonStr, TuningConfig.class)),
                TuningConfig.class);

    Assert.assertEquals("/tmp/xxx", config.getBasePersistDirectory().toString());
    Assert.assertEquals(false, config.getBuildV9Directly());
    Assert.assertEquals(100, config.getHandoffConditionTimeout());
    Assert.assertEquals(new IndexSpec(), config.getIndexSpec());
    Assert.assertEquals(new Period("PT1H"), config.getIntermediatePersistPeriod());
    Assert.assertEquals(NoneShardSpec.instance(), config.getShardSpec());
    Assert.assertEquals(100, config.getMaxPendingPersists());
    Assert.assertEquals(100, config.getMaxRowsInMemory());
    Assert.assertEquals(100, config.getMergeThreadPriority());
    Assert.assertEquals(100, config.getPersistThreadPriority());
    Assert.assertEquals(new Period("PT1H"), config.getWindowPeriod());
    Assert.assertEquals(true, config.isReportParseExceptions());
  }
  @Test
  public void testSerdeWithDefaults() throws Exception {
    String jsonStr = "{\"type\":\"realtime\"}";

    ObjectMapper mapper = TestHelper.getObjectMapper();
    RealtimeTuningConfig config =
        (RealtimeTuningConfig)
            mapper.readValue(
                mapper.writeValueAsString(mapper.readValue(jsonStr, TuningConfig.class)),
                TuningConfig.class);

    Assert.assertNotNull(config.getBasePersistDirectory());
    Assert.assertEquals(true, config.getBuildV9Directly());
    Assert.assertEquals(0, config.getHandoffConditionTimeout());
    Assert.assertEquals(new IndexSpec(), config.getIndexSpec());
    Assert.assertEquals(new Period("PT10M"), config.getIntermediatePersistPeriod());
    Assert.assertEquals(NoneShardSpec.instance(), config.getShardSpec());
    Assert.assertEquals(0, config.getMaxPendingPersists());
    Assert.assertEquals(75000, config.getMaxRowsInMemory());
    Assert.assertEquals(0, config.getMergeThreadPriority());
    Assert.assertEquals(0, config.getPersistThreadPriority());
    Assert.assertEquals(new Period("PT10M"), config.getWindowPeriod());
    Assert.assertEquals(false, config.isReportParseExceptions());
  }
Beispiel #3
0
    @Override
    public void run() {
      plumber = initPlumber();
      final Period intermediatePersistPeriod = config.getIntermediatePersistPeriod();

      try {
        plumber.startJob();

        // Delay firehose connection to avoid claiming input resources while the plumber is starting
        // up.
        firehose = initFirehose();

        long nextFlush = new DateTime().plus(intermediatePersistPeriod).getMillis();
        while (firehose.hasMore()) {
          InputRow inputRow = null;
          try {
            try {
              inputRow = firehose.nextRow();
            } catch (Exception e) {
              log.debug(e, "thrown away line due to exception, considering unparseable");
              metrics.incrementUnparseable();
              continue;
            }

            boolean lateEvent = false;
            boolean indexLimitExceeded = false;
            try {
              lateEvent = plumber.add(inputRow) == -1;
            } catch (IndexSizeExceededException e) {
              log.info("Index limit exceeded: %s", e.getMessage());
              indexLimitExceeded = true;
            }
            if (indexLimitExceeded || lateEvent) {
              metrics.incrementThrownAway();
              log.debug("Throwing away event[%s]", inputRow);

              if (indexLimitExceeded || System.currentTimeMillis() > nextFlush) {
                plumber.persist(firehose.commit());
                nextFlush = new DateTime().plus(intermediatePersistPeriod).getMillis();
              }

              continue;
            }
            final Sink sink = plumber.getSink(inputRow.getTimestampFromEpoch());
            if ((sink != null && !sink.canAppendRow()) || System.currentTimeMillis() > nextFlush) {
              plumber.persist(firehose.commit());
              nextFlush = new DateTime().plus(intermediatePersistPeriod).getMillis();
            }
            metrics.incrementProcessed();
          } catch (ParseException e) {
            if (inputRow != null) {
              log.error(e, "unparseable line: %s", inputRow);
            }
            metrics.incrementUnparseable();
          }
        }
      } catch (RuntimeException e) {
        log.makeAlert(
                e,
                "RuntimeException aborted realtime processing[%s]",
                fireDepartment.getDataSchema().getDataSource())
            .emit();
        normalExit = false;
        throw e;
      } catch (Error e) {
        log.makeAlert(
                e,
                "Exception aborted realtime processing[%s]",
                fireDepartment.getDataSchema().getDataSource())
            .emit();
        normalExit = false;
        throw e;
      } finally {
        CloseQuietly.close(firehose);
        if (normalExit) {
          plumber.finishJob();
          plumber = null;
          firehose = null;
        }
      }
    }