Ejemplo n.º 1
0
  @Override
  public Set<DataSegment> perform(Task task, TaskActionToolbox toolbox) throws IOException {
    toolbox.verifyTaskLocksAndSinglePartitionSettitude(task, segments, true);

    final Set<DataSegment> retVal =
        toolbox.getIndexerDBCoordinator().announceHistoricalSegments(segments);

    // Emit metrics
    final ServiceMetricEvent.Builder metricBuilder =
        new ServiceMetricEvent.Builder().setUser2(task.getDataSource()).setUser4(task.getType());

    for (DataSegment segment : segments) {
      metricBuilder.setUser5(segment.getInterval().toString());
      toolbox.getEmitter().emit(metricBuilder.build("indexer/segment/bytes", segment.getSize()));
    }

    return retVal;
  }
  @Override
  public Optional<ImmutableWorkerInfo> findWorkerForTask(
      final WorkerTaskRunnerConfig config,
      final ImmutableMap<String, ImmutableWorkerInfo> zkWorkers,
      final Task task) {
    // don't run other datasources on affinity workers; we only want our configured datasources to
    // run on them
    ImmutableMap.Builder<String, ImmutableWorkerInfo> builder = new ImmutableMap.Builder<>();
    for (String workerHost : zkWorkers.keySet()) {
      if (!affinityWorkerHosts.contains(workerHost)) {
        builder.put(workerHost, zkWorkers.get(workerHost));
      }
    }
    ImmutableMap<String, ImmutableWorkerInfo> eligibleWorkers = builder.build();

    List<String> workerHosts = affinityConfig.getAffinity().get(task.getDataSource());
    if (workerHosts == null) {
      return super.findWorkerForTask(config, eligibleWorkers, task);
    }

    ImmutableMap.Builder<String, ImmutableWorkerInfo> affinityBuilder =
        new ImmutableMap.Builder<>();
    for (String workerHost : workerHosts) {
      ImmutableWorkerInfo zkWorker = zkWorkers.get(workerHost);
      if (zkWorker != null) {
        affinityBuilder.put(workerHost, zkWorker);
      }
    }
    ImmutableMap<String, ImmutableWorkerInfo> affinityWorkers = affinityBuilder.build();

    if (!affinityWorkers.isEmpty()) {
      Optional<ImmutableWorkerInfo> retVal = super.findWorkerForTask(config, affinityWorkers, task);
      if (retVal.isPresent()) {
        return retVal;
      }
    }

    return super.findWorkerForTask(config, eligibleWorkers, task);
  }
Ejemplo n.º 3
0
  private <T> QueryRunner<T> getQueryRunnerImpl(Query<T> query) {
    QueryRunner<T> queryRunner = null;
    final String queryDataSource = Iterables.getOnlyElement(query.getDataSource().getNames());

    for (final ThreadPoolTaskRunnerWorkItem taskRunnerWorkItem :
        ImmutableList.copyOf(runningItems)) {
      final Task task = taskRunnerWorkItem.getTask();
      if (task.getDataSource().equals(queryDataSource)) {
        final QueryRunner<T> taskQueryRunner = task.getQueryRunner(query);

        if (taskQueryRunner != null) {
          if (queryRunner == null) {
            queryRunner = taskQueryRunner;
          } else {
            log.makeAlert("Found too many query runners for datasource")
                .addData("dataSource", queryDataSource)
                .emit();
          }
        }
      }
    }

    return queryRunner == null ? new NoopQueryRunner<T>() : queryRunner;
  }