Ejemplo n.º 1
0
 public void addKnownInput(
     String hostName,
     int port,
     InputAttemptIdentifier srcAttemptIdentifier,
     int srcPhysicalIndex) {
   String identifier = InputHost.createIdentifier(hostName, port);
   InputHost host = knownSrcHosts.get(identifier);
   if (host == null) {
     host = new InputHost(hostName, port, inputContext.getApplicationId(), srcPhysicalIndex);
     assert identifier.equals(host.getIdentifier());
     InputHost old = knownSrcHosts.putIfAbsent(identifier, host);
     if (old != null) {
       host = old;
     }
   }
   if (LOG.isDebugEnabled()) {
     LOG.debug("Adding input: " + srcAttemptIdentifier + ", to host: " + host);
   }
   host.addKnownInput(srcAttemptIdentifier);
   lock.lock();
   try {
     boolean added = pendingHosts.offer(host);
     if (!added) {
       String errorMessage = "Unable to add host: " + host.getIdentifier() + " to pending queue";
       LOG.error(errorMessage);
       throw new TezUncheckedException(errorMessage);
     }
     wakeLoop.signal();
   } finally {
     lock.unlock();
   }
 }
Ejemplo n.º 2
0
  private Fetcher constructFetcherForHost(InputHost inputHost) {
    FetcherBuilder fetcherBuilder =
        new FetcherBuilder(
            ShuffleManager.this,
            httpConnectionParams,
            inputManager,
            inputContext.getApplicationId(),
            shuffleSecret,
            srcNameTrimmed);
    if (codec != null) {
      fetcherBuilder.setCompressionParameters(codec);
    }
    fetcherBuilder.setIFileParams(ifileReadAhead, ifileReadAheadLength);

    // Remove obsolete inputs from the list being given to the fetcher. Also
    // remove from the obsolete list.
    List<InputAttemptIdentifier> pendingInputsForHost = inputHost.clearAndGetPendingInputs();
    for (Iterator<InputAttemptIdentifier> inputIter = pendingInputsForHost.iterator();
        inputIter.hasNext(); ) {
      InputAttemptIdentifier input = inputIter.next();
      // Avoid adding attempts which have already completed.
      if (completedInputSet.contains(input.getInputIdentifier())) {
        inputIter.remove();
        continue;
      }
      // Avoid adding attempts which have been marked as OBSOLETE
      if (obsoletedInputs.contains(input)) {
        inputIter.remove();
      }
    }
    // TODO NEWTEZ Maybe limit the number of inputs being given to a single
    // fetcher, especially in the case where #hosts < #fetchers
    fetcherBuilder.assignWork(
        inputHost.getHost(),
        inputHost.getPort(),
        inputHost.getSrcPhysicalIndex(),
        pendingInputsForHost);
    LOG.info(
        "Created Fetcher for host: "
            + inputHost.getHost()
            + ", with inputs: "
            + pendingInputsForHost);
    return fetcherBuilder.build();
  }
Ejemplo n.º 3
0
 @Override
 public void onSuccess(FetchResult result) {
   fetcher.shutdown();
   if (isShutdown.get()) {
     LOG.info("Already shutdown. Ignoring event from fetcher");
   } else {
     Iterable<InputAttemptIdentifier> pendingInputs = result.getPendingInputs();
     if (pendingInputs != null && pendingInputs.iterator().hasNext()) {
       InputHost inputHost =
           knownSrcHosts.get(InputHost.createIdentifier(result.getHost(), result.getPort()));
       assert inputHost != null;
       for (InputAttemptIdentifier input : pendingInputs) {
         inputHost.addKnownInput(input);
       }
       pendingHosts.add(inputHost);
     }
     doBookKeepingForFetcherComplete();
   }
 }
Ejemplo n.º 4
0
    @Override
    public Void call() throws Exception {
      while (!isShutdown.get() && numCompletedInputs.get() < numInputs) {
        lock.lock();
        try {
          if (runningFetchers.size() >= numFetchers || pendingHosts.isEmpty()) {
            if (numCompletedInputs.get() < numInputs) {
              wakeLoop.await();
            }
          }
        } finally {
          lock.unlock();
        }

        if (shuffleError != null) {
          // InputContext has already been informed of a fatal error. Relying on
          // tez to kill the task.
          break;
        }

        if (LOG.isDebugEnabled()) {
          LOG.debug("NumCompletedInputs: " + numCompletedInputs);
        }
        if (numCompletedInputs.get() < numInputs && !isShutdown.get()) {
          lock.lock();
          try {
            int maxFetchersToRun = numFetchers - runningFetchers.size();
            int count = 0;
            while (pendingHosts.peek() != null && !isShutdown.get()) {
              InputHost inputHost = null;
              try {
                inputHost = pendingHosts.take();
              } catch (InterruptedException e) {
                if (isShutdown.get()) {
                  LOG.info(
                      "Interrupted and hasBeenShutdown, Breaking out of ShuffleScheduler Loop");
                  break;
                } else {
                  throw e;
                }
              }
              if (LOG.isDebugEnabled()) {
                LOG.debug("Processing pending host: " + inputHost.toDetailedString());
              }
              if (inputHost.getNumPendingInputs() > 0 && !isShutdown.get()) {
                LOG.info("Scheduling fetch for inputHost: " + inputHost.getIdentifier());
                Fetcher fetcher = constructFetcherForHost(inputHost);
                runningFetchers.add(fetcher);
                if (isShutdown.get()) {
                  LOG.info("hasBeenShutdown, Breaking out of ShuffleScheduler Loop");
                }
                ListenableFuture<FetchResult> future = fetcherExecutor.submit(fetcher);
                Futures.addCallback(future, new FetchFutureCallback(fetcher));
                if (++count >= maxFetchersToRun) {
                  break;
                }
              } else {
                if (LOG.isDebugEnabled()) {
                  LOG.debug(
                      "Skipping host: "
                          + inputHost.getIdentifier()
                          + " since it has no inputs to process");
                }
              }
            }
          } finally {
            lock.unlock();
          }
        }
      }
      LOG.info(
          "Shutting down FetchScheduler, Was Interrupted: "
              + Thread.currentThread().isInterrupted());
      // TODO NEWTEZ Maybe clean up inputs.
      if (!fetcherExecutor.isShutdown()) {
        fetcherExecutor.shutdownNow();
      }
      return null;
    }