@Test(timeout = 4000L)
  public void testRealtimeIndexTaskFailure() throws Exception {
    setUpAndStartTaskQueue(
        new DataSegmentPusher() {
          @Override
          public String getPathForHadoop(String s) {
            throw new UnsupportedOperationException();
          }

          @Override
          public DataSegment push(File file, DataSegment dataSegment) throws IOException {
            throw new RuntimeException("FAILURE");
          }
        });
    monitorScheduler.addMonitor(EasyMock.anyObject(Monitor.class));
    EasyMock.expectLastCall().atLeastOnce();
    monitorScheduler.removeMonitor(EasyMock.anyObject(Monitor.class));
    EasyMock.expectLastCall().anyTimes();
    EasyMock.replay(monitorScheduler, queryRunnerFactoryConglomerate);

    RealtimeIndexTask realtimeIndexTask = giveMeARealtimeIndexTask();
    final String taskId = realtimeIndexTask.getId();
    tq.add(realtimeIndexTask);

    // Wait for realtime index task to fail
    while (tsqa.getStatus(taskId).get().isRunnable()) {
      Thread.sleep(10);
    }

    Assert.assertTrue("Task should be in Failure state", tsqa.getStatus(taskId).get().isFailure());

    EasyMock.verify(monitorScheduler, queryRunnerFactoryConglomerate);
  }
  @Test(timeout = 4000L)
  public void testRealtimeIndexTask() throws Exception {
    monitorScheduler.addMonitor(EasyMock.anyObject(Monitor.class));
    EasyMock.expectLastCall().atLeastOnce();
    monitorScheduler.removeMonitor(EasyMock.anyObject(Monitor.class));
    EasyMock.expectLastCall().anyTimes();
    EasyMock.replay(monitorScheduler, queryRunnerFactoryConglomerate);

    RealtimeIndexTask realtimeIndexTask = giveMeARealtimeIndexTask();
    final String taskId = realtimeIndexTask.getId();

    tq.add(realtimeIndexTask);
    // wait for task to process events and publish segment
    Assert.assertTrue(publishCountDown.await(1000, TimeUnit.MILLISECONDS));

    // Realtime Task has published the segment, simulate loading of segment to a historical node so
    // that task finishes with SUCCESS status
    segmentCallbacks
        .get(0)
        .segmentAdded(
            new DruidServerMetadata("dummy", "dummy_host", 0, "historical", "dummy_tier", 0),
            mdc.getPublished().iterator().next());

    // Wait for realtime index task to handle callback in plumber and succeed
    while (tsqa.getStatus(taskId).get().isRunnable()) {
      Thread.sleep(10);
    }

    Assert.assertTrue("Task should be in Success state", tsqa.getStatus(taskId).get().isSuccess());

    Assert.assertEquals(1, announcedSinks);
    Assert.assertEquals(1, pushedSegments);
    Assert.assertEquals(1, mdc.getPublished().size());
    DataSegment segment = mdc.getPublished().iterator().next();
    Assert.assertEquals("test_ds", segment.getDataSource());
    Assert.assertEquals(ImmutableList.of("dim1", "dim2"), segment.getDimensions());
    Assert.assertEquals(
        new Interval(now.toString("YYYY-MM-dd") + "/" + now.plusDays(1).toString("YYYY-MM-dd")),
        segment.getInterval());
    Assert.assertEquals(ImmutableList.of("count"), segment.getMetrics());
    EasyMock.verify(monitorScheduler, queryRunnerFactoryConglomerate);
  }
示例#3
0
  @Test
  public void testRealtimeIndexTaskSerde() throws Exception {

    final RealtimeIndexTask task =
        new RealtimeIndexTask(
            null,
            new TaskResource("rofl", 2),
            new FireDepartment(
                new DataSchema(
                    "foo",
                    null,
                    new AggregatorFactory[0],
                    new UniformGranularitySpec(Granularity.HOUR, QueryGranularity.NONE, null),
                    jsonMapper),
                new RealtimeIOConfig(
                    new LocalFirehoseFactory(new File("lol"), "rofl", null),
                    new PlumberSchool() {
                      @Override
                      public Plumber findPlumber(
                          DataSchema schema,
                          RealtimeTuningConfig config,
                          FireDepartmentMetrics metrics) {
                        return null;
                      }
                    },
                    null),
                new RealtimeTuningConfig(
                    1,
                    new Period("PT10M"),
                    null,
                    null,
                    null,
                    null,
                    1,
                    new NoneShardSpec(),
                    indexSpec,
                    null,
                    0,
                    0)),
            null);

    final String json = jsonMapper.writeValueAsString(task);

    Thread.sleep(100); // Just want to run the clock a bit to make sure the task id doesn't change
    final RealtimeIndexTask task2 = (RealtimeIndexTask) jsonMapper.readValue(json, Task.class);

    Assert.assertEquals("foo", task.getDataSource());
    Assert.assertEquals(2, task.getTaskResource().getRequiredCapacity());
    Assert.assertEquals("rofl", task.getTaskResource().getAvailabilityGroup());
    Assert.assertEquals(
        new Period("PT10M"), task.getRealtimeIngestionSchema().getTuningConfig().getWindowPeriod());
    Assert.assertEquals(
        Granularity.HOUR,
        task.getRealtimeIngestionSchema()
            .getDataSchema()
            .getGranularitySpec()
            .getSegmentGranularity());

    Assert.assertEquals(task.getId(), task2.getId());
    Assert.assertEquals(task.getGroupId(), task2.getGroupId());
    Assert.assertEquals(task.getDataSource(), task2.getDataSource());
    Assert.assertEquals(
        task.getTaskResource().getRequiredCapacity(),
        task2.getTaskResource().getRequiredCapacity());
    Assert.assertEquals(
        task.getTaskResource().getAvailabilityGroup(),
        task2.getTaskResource().getAvailabilityGroup());
    Assert.assertEquals(
        task.getRealtimeIngestionSchema().getTuningConfig().getWindowPeriod(),
        task2.getRealtimeIngestionSchema().getTuningConfig().getWindowPeriod());
    Assert.assertEquals(
        task.getRealtimeIngestionSchema()
            .getDataSchema()
            .getGranularitySpec()
            .getSegmentGranularity(),
        task2
            .getRealtimeIngestionSchema()
            .getDataSchema()
            .getGranularitySpec()
            .getSegmentGranularity());
  }