Пример #1
0
 // TODO : make wait time configurable ?
 public static void shutdownExecutor(ExecutorService executor, final String name) {
   executor.shutdown();
   try {
     log.info("Waiting for %s to shutdown", name);
     if (!executor.awaitTermination(5, TimeUnit.SECONDS)) {
       log.warn("%s did not shutdown properly", name);
     }
   } catch (InterruptedException e) {
     log.warn("Interrupted while waiting for %s to shutdown", name);
     Thread.currentThread().interrupt();
   }
 }
Пример #2
0
 public static void closeChannels(ChannelGroup allChannels) {
   if (allChannels.size() > 0) {
     // TODO : allow an option here to control if we need to drain connections and wait instead of
     // killing them all
     try {
       log.info("Closing %s open client connections", allChannels.size());
       if (!allChannels.close().await(5, TimeUnit.SECONDS)) {
         log.warn("Failed to close all open client connections");
       }
     } catch (InterruptedException e) {
       log.warn("Interrupted while closing client connections");
       Thread.currentThread().interrupt();
     }
   }
 }
  public HiveMetadata(
      HiveConnectorId connectorId,
      HiveMetastore metastore,
      HdfsEnvironment hdfsEnvironment,
      DateTimeZone timeZone,
      boolean allowDropTable,
      boolean allowRenameTable,
      boolean allowCorruptWritesForTesting,
      HiveStorageFormat hiveStorageFormat,
      TypeManager typeManager) {
    this.connectorId = checkNotNull(connectorId, "connectorId is null").toString();

    this.allowDropTable = allowDropTable;
    this.allowRenameTable = allowRenameTable;
    this.allowCorruptWritesForTesting = allowCorruptWritesForTesting;

    this.metastore = checkNotNull(metastore, "metastore is null");
    this.hdfsEnvironment = checkNotNull(hdfsEnvironment, "hdfsEnvironment is null");
    this.timeZone = checkNotNull(timeZone, "timeZone is null");
    this.hiveStorageFormat = hiveStorageFormat;
    this.typeManager = checkNotNull(typeManager, "typeManager is null");

    if (!allowCorruptWritesForTesting && !timeZone.equals(DateTimeZone.getDefault())) {
      log.warn(
          "Hive writes are disabled. "
              + "To write data to Hive, your JVM timezone must match the Hive storage timezone. "
              + "Add -Duser.timezone=%s to your JVM arguments",
          timeZone.getID());
    }
  }
Пример #4
0
  /** Remove completed queries after a waiting period */
  public void removeExpiredQueries() {
    List<QueryExecution> sortedQueries =
        IterableTransformer.on(queries.values())
            .select(compose(not(isNull()), endTimeGetter()))
            .orderBy(Ordering.natural().onResultOf(endTimeGetter()))
            .list();

    int toRemove = Math.max(sortedQueries.size() - maxQueryHistory, 0);
    DateTime oldestAllowedQuery = DateTime.now().minus(maxQueryAge.toMillis());

    for (QueryExecution queryExecution : sortedQueries) {
      try {
        DateTime endTime = queryExecution.getQueryInfo().getQueryStats().getEndTime();
        if ((endTime.isBefore(oldestAllowedQuery) || toRemove > 0) && isAbandoned(queryExecution)) {
          removeQuery(queryExecution.getQueryInfo().getQueryId());
          --toRemove;
        }
      } catch (RuntimeException e) {
        log.warn(
            e,
            "Error while inspecting age of query %s",
            queryExecution.getQueryInfo().getQueryId());
      }
    }
  }
Пример #5
0
 private static ResumableTask.TaskStatus safeProcessTask(ResumableTask task) {
   try {
     return task.process();
   } catch (Throwable t) {
     log.warn(t, "ResumableTask completed exceptionally");
     return ResumableTask.TaskStatus.finished();
   }
 }
Пример #6
0
  public void requestFailed(Throwable reason) throws PrestoException {
    // cancellation is not a failure
    if (reason instanceof CancellationException) {
      return;
    }

    if (reason instanceof RejectedExecutionException) {
      throw new PrestoException(REMOTE_TASK_ERROR, reason);
    }

    // log failure message
    if (isExpectedError(reason)) {
      // don't print a stack for a known errors
      log.warn("Error " + jobDescription + " %s: %s: %s", taskId, reason.getMessage(), taskUri);
    } else {
      log.warn(reason, "Error " + jobDescription + " %s: %s", taskId, taskUri);
    }

    // remember the first 10 errors
    if (errorsSinceLastSuccess.size() < 10) {
      errorsSinceLastSuccess.add(reason);
    }

    // fail the task, if we have more than X failures in a row and more than Y seconds have passed
    // since the last request
    if (backoff.failure()) {
      // it is weird to mark the task failed locally and then cancel the remote task, but there is
      // no way to tell a remote task that it is failed
      PrestoException exception =
          new PrestoException(
              TOO_MANY_REQUESTS_FAILED,
              format(
                  "%s (%s %s - %s failures, time since last success %s)",
                  WORKER_NODE_ERROR,
                  jobDescription,
                  taskUri,
                  backoff.getFailureCount(),
                  backoff.getTimeSinceLastSuccess().convertTo(SECONDS)));
      errorsSinceLastSuccess.forEach(exception::addSuppressed);
      throw exception;
    }
  }
Пример #7
0
 public static void createTablesWithRetry(AliasDao dao) throws InterruptedException {
   Duration delay = new Duration(10, TimeUnit.SECONDS);
   while (true) {
     try {
       createTables(dao);
       return;
     } catch (UnableToObtainConnectionException e) {
       log.warn(
           "Failed to connect to database. Will retry again in %s. Exception: %s",
           delay, e.getMessage());
       Thread.sleep(delay.toMillis());
     }
   }
 }
Пример #8
0
    @Override
    public void close() throws IOException {
      if (closed) {
        return;
      }
      closed = true;

      try {
        super.close();
        uploadObject();
      } finally {
        if (!tempFile.delete()) {
          log.warn("Could not delete temporary file: %s", tempFile);
        }
        // close transfer manager but keep underlying S3 client open
        transferManager.shutdownNow(false);
      }
    }
Пример #9
0
  public void failAbandonedQueries() {
    for (QueryExecution queryExecution : queries.values()) {
      try {
        QueryInfo queryInfo = queryExecution.getQueryInfo();
        if (queryInfo.getState().isDone()) {
          continue;
        }

        if (isAbandoned(queryExecution)) {
          log.info("Failing abandoned query %s", queryExecution.getQueryInfo().getQueryId());
          queryExecution.fail(
              new AbandonedException(
                  "Query " + queryInfo.getQueryId(),
                  queryInfo.getQueryStats().getLastHeartbeat(),
                  DateTime.now()));
        }
      } catch (RuntimeException e) {
        log.warn(
            e,
            "Error while inspecting age of query %s",
            queryExecution.getQueryInfo().getQueryId());
      }
    }
  }
  public DirectoryDeploymentManager(File baseDir, String location, Duration tarTimeout) {
    Preconditions.checkNotNull(location, "location is null");
    Preconditions.checkArgument(location.startsWith("/"), "location must start with /");
    this.location = location;
    this.tarTimeout = tarTimeout;

    Preconditions.checkNotNull(baseDir, "baseDir is null");
    baseDir.mkdirs();
    Preconditions.checkArgument(
        baseDir.isDirectory(), "baseDir is not a directory: " + baseDir.getAbsolutePath());
    this.baseDir = baseDir;

    // verify deployment file is readable and writable
    deploymentFile = new File(baseDir, "airship-deployment.json");
    if (deploymentFile.exists()) {
      Preconditions.checkArgument(
          deploymentFile.canRead(),
          "Can not read slot-id file %s",
          deploymentFile.getAbsolutePath());
      Preconditions.checkArgument(
          deploymentFile.canWrite(),
          "Can not write slot-id file %s",
          deploymentFile.getAbsolutePath());
    }

    // load deployments
    if (deploymentFile.exists()) {
      try {
        Deployment deployment = load(deploymentFile);
        Preconditions.checkArgument(
            deployment.getDeploymentDir().isDirectory(),
            "Deployment directory is not a directory: %s",
            deployment.getDeploymentDir());
        this.deployment = deployment;
      } catch (IOException e) {
        throw new IllegalArgumentException(
            "Invalid deployment file: " + deploymentFile.getAbsolutePath(), e);
      }
    }

    // load slot-id
    File slotIdFile = new File(baseDir, "airship-slot-id.txt");
    UUID uuid = null;
    if (slotIdFile.exists()) {
      Preconditions.checkArgument(
          slotIdFile.canRead(), "can not read " + slotIdFile.getAbsolutePath());
      try {
        String slotIdString = Files.toString(slotIdFile, UTF_8).trim();
        try {
          uuid = UUID.fromString(slotIdString);
        } catch (IllegalArgumentException e) {

        }
        if (uuid == null) {
          log.warn(
              "Invalid slot id ["
                  + slotIdString
                  + "]: attempting to delete airship-slot-id.txt file and recreating a new one");
          slotIdFile.delete();
        }
      } catch (IOException e) {
        Preconditions.checkArgument(
            slotIdFile.canRead(), "can not read " + slotIdFile.getAbsolutePath());
      }
    }

    if (uuid == null) {
      uuid = UUID.randomUUID();
      try {
        Files.write(uuid.toString(), slotIdFile, UTF_8);
      } catch (IOException e) {
        Preconditions.checkArgument(
            slotIdFile.canRead(), "can not write " + slotIdFile.getAbsolutePath());
      }
    }
    slotId = uuid;
  }
  @Override
  public Deployment install(Installation installation) {
    Preconditions.checkNotNull(installation, "installation is null");

    File deploymentDir = new File(baseDir, "installation");

    Assignment assignment = installation.getAssignment();

    Deployment newDeployment =
        new Deployment(
            slotId, location, deploymentDir, getDataDir(), assignment, installation.getResources());
    File tempDir = createTempDir(baseDir, "tmp-install");
    try {
      // download the binary
      File binary = new File(tempDir, "airship-binary.tar.gz");
      try {
        Files.copy(Resources.newInputStreamSupplier(installation.getBinaryFile().toURL()), binary);
      } catch (IOException e) {
        throw new RuntimeException(
            "Unable to download binary "
                + assignment.getBinary()
                + " from "
                + installation.getBinaryFile(),
            e);
      }

      // unpack the binary into a temp unpack dir
      File unpackDir = new File(tempDir, "unpack");
      unpackDir.mkdirs();
      try {
        extractTar(binary, unpackDir, tarTimeout);
      } catch (CommandFailedException e) {
        throw new RuntimeException(
            "Unable to extract tar file " + assignment.getBinary() + ": " + e.getMessage());
      }

      // find the archive root dir (it should be the only file in the temp unpack dir)
      List<File> files = listFiles(unpackDir);
      if (files.size() != 1) {
        throw new RuntimeException(
            "Invalid tar file: file does not have a root directory " + assignment.getBinary());
      }
      File binaryRootDir = files.get(0);

      // unpack config bundle
      try {
        URL url = installation.getConfigFile().toURL();
        ConfigUtils.unpackConfig(Resources.newInputStreamSupplier(url), binaryRootDir);
      } catch (Exception e) {
        throw new RuntimeException(
            "Unable to extract config bundle " + assignment.getConfig() + ": " + e.getMessage());
      }

      // installation is good, clear the current deployment
      if (this.deployment != null) {
        this.deploymentFile.delete();
        deleteRecursively(this.deployment.getDeploymentDir());
        this.deployment = null;
      }

      // save deployment versions file
      try {
        save(newDeployment);
      } catch (IOException e) {
        throw new RuntimeException("Unable to save deployment file", e);
      }

      // move the binary root directory to the final target
      try {
        Files.move(binaryRootDir, deploymentDir);
      } catch (IOException e) {
        throw new RuntimeException("Unable to move deployment to final location", e);
      }
    } finally {
      if (!deleteRecursively(tempDir)) {
        log.warn("Unable to delete temp directory: %s", tempDir.getAbsolutePath());
      }
    }

    this.deployment = newDeployment;
    return newDeployment;
  }