// TODO : make wait time configurable ?
public static void shutdownExecutor(ExecutorService executor, final String name) {
executor.shutdown();
try {
log.info("Waiting for %s to shutdown", name);
if (!executor.awaitTermination(5, TimeUnit.SECONDS)) {
log.warn("%s did not shutdown properly", name);
}
} catch (InterruptedException e) {
log.warn("Interrupted while waiting for %s to shutdown", name);
Thread.currentThread().interrupt();
}
}
public static void closeChannels(ChannelGroup allChannels) {
if (allChannels.size() > 0) {
// TODO : allow an option here to control if we need to drain connections and wait instead of
// killing them all
try {
log.info("Closing %s open client connections", allChannels.size());
if (!allChannels.close().await(5, TimeUnit.SECONDS)) {
log.warn("Failed to close all open client connections");
}
} catch (InterruptedException e) {
log.warn("Interrupted while closing client connections");
Thread.currentThread().interrupt();
}
}
}
public HiveMetadata(
HiveConnectorId connectorId,
HiveMetastore metastore,
HdfsEnvironment hdfsEnvironment,
DateTimeZone timeZone,
boolean allowDropTable,
boolean allowRenameTable,
boolean allowCorruptWritesForTesting,
HiveStorageFormat hiveStorageFormat,
TypeManager typeManager) {
this.connectorId = checkNotNull(connectorId, "connectorId is null").toString();
this.allowDropTable = allowDropTable;
this.allowRenameTable = allowRenameTable;
this.allowCorruptWritesForTesting = allowCorruptWritesForTesting;
this.metastore = checkNotNull(metastore, "metastore is null");
this.hdfsEnvironment = checkNotNull(hdfsEnvironment, "hdfsEnvironment is null");
this.timeZone = checkNotNull(timeZone, "timeZone is null");
this.hiveStorageFormat = hiveStorageFormat;
this.typeManager = checkNotNull(typeManager, "typeManager is null");
if (!allowCorruptWritesForTesting && !timeZone.equals(DateTimeZone.getDefault())) {
log.warn(
"Hive writes are disabled. "
+ "To write data to Hive, your JVM timezone must match the Hive storage timezone. "
+ "Add -Duser.timezone=%s to your JVM arguments",
timeZone.getID());
}
}
/** Remove completed queries after a waiting period */
public void removeExpiredQueries() {
List<QueryExecution> sortedQueries =
IterableTransformer.on(queries.values())
.select(compose(not(isNull()), endTimeGetter()))
.orderBy(Ordering.natural().onResultOf(endTimeGetter()))
.list();
int toRemove = Math.max(sortedQueries.size() - maxQueryHistory, 0);
DateTime oldestAllowedQuery = DateTime.now().minus(maxQueryAge.toMillis());
for (QueryExecution queryExecution : sortedQueries) {
try {
DateTime endTime = queryExecution.getQueryInfo().getQueryStats().getEndTime();
if ((endTime.isBefore(oldestAllowedQuery) || toRemove > 0) && isAbandoned(queryExecution)) {
removeQuery(queryExecution.getQueryInfo().getQueryId());
--toRemove;
}
} catch (RuntimeException e) {
log.warn(
e,
"Error while inspecting age of query %s",
queryExecution.getQueryInfo().getQueryId());
}
}
}
private static ResumableTask.TaskStatus safeProcessTask(ResumableTask task) {
try {
return task.process();
} catch (Throwable t) {
log.warn(t, "ResumableTask completed exceptionally");
return ResumableTask.TaskStatus.finished();
}
}
public void requestFailed(Throwable reason) throws PrestoException {
// cancellation is not a failure
if (reason instanceof CancellationException) {
return;
}
if (reason instanceof RejectedExecutionException) {
throw new PrestoException(REMOTE_TASK_ERROR, reason);
}
// log failure message
if (isExpectedError(reason)) {
// don't print a stack for a known errors
log.warn("Error " + jobDescription + " %s: %s: %s", taskId, reason.getMessage(), taskUri);
} else {
log.warn(reason, "Error " + jobDescription + " %s: %s", taskId, taskUri);
}
// remember the first 10 errors
if (errorsSinceLastSuccess.size() < 10) {
errorsSinceLastSuccess.add(reason);
}
// fail the task, if we have more than X failures in a row and more than Y seconds have passed
// since the last request
if (backoff.failure()) {
// it is weird to mark the task failed locally and then cancel the remote task, but there is
// no way to tell a remote task that it is failed
PrestoException exception =
new PrestoException(
TOO_MANY_REQUESTS_FAILED,
format(
"%s (%s %s - %s failures, time since last success %s)",
WORKER_NODE_ERROR,
jobDescription,
taskUri,
backoff.getFailureCount(),
backoff.getTimeSinceLastSuccess().convertTo(SECONDS)));
errorsSinceLastSuccess.forEach(exception::addSuppressed);
throw exception;
}
}
public static void createTablesWithRetry(AliasDao dao) throws InterruptedException {
Duration delay = new Duration(10, TimeUnit.SECONDS);
while (true) {
try {
createTables(dao);
return;
} catch (UnableToObtainConnectionException e) {
log.warn(
"Failed to connect to database. Will retry again in %s. Exception: %s",
delay, e.getMessage());
Thread.sleep(delay.toMillis());
}
}
}
@Override
public void close() throws IOException {
if (closed) {
return;
}
closed = true;
try {
super.close();
uploadObject();
} finally {
if (!tempFile.delete()) {
log.warn("Could not delete temporary file: %s", tempFile);
}
// close transfer manager but keep underlying S3 client open
transferManager.shutdownNow(false);
}
}
public void failAbandonedQueries() {
for (QueryExecution queryExecution : queries.values()) {
try {
QueryInfo queryInfo = queryExecution.getQueryInfo();
if (queryInfo.getState().isDone()) {
continue;
}
if (isAbandoned(queryExecution)) {
log.info("Failing abandoned query %s", queryExecution.getQueryInfo().getQueryId());
queryExecution.fail(
new AbandonedException(
"Query " + queryInfo.getQueryId(),
queryInfo.getQueryStats().getLastHeartbeat(),
DateTime.now()));
}
} catch (RuntimeException e) {
log.warn(
e,
"Error while inspecting age of query %s",
queryExecution.getQueryInfo().getQueryId());
}
}
}
public DirectoryDeploymentManager(File baseDir, String location, Duration tarTimeout) {
Preconditions.checkNotNull(location, "location is null");
Preconditions.checkArgument(location.startsWith("/"), "location must start with /");
this.location = location;
this.tarTimeout = tarTimeout;
Preconditions.checkNotNull(baseDir, "baseDir is null");
baseDir.mkdirs();
Preconditions.checkArgument(
baseDir.isDirectory(), "baseDir is not a directory: " + baseDir.getAbsolutePath());
this.baseDir = baseDir;
// verify deployment file is readable and writable
deploymentFile = new File(baseDir, "airship-deployment.json");
if (deploymentFile.exists()) {
Preconditions.checkArgument(
deploymentFile.canRead(),
"Can not read slot-id file %s",
deploymentFile.getAbsolutePath());
Preconditions.checkArgument(
deploymentFile.canWrite(),
"Can not write slot-id file %s",
deploymentFile.getAbsolutePath());
}
// load deployments
if (deploymentFile.exists()) {
try {
Deployment deployment = load(deploymentFile);
Preconditions.checkArgument(
deployment.getDeploymentDir().isDirectory(),
"Deployment directory is not a directory: %s",
deployment.getDeploymentDir());
this.deployment = deployment;
} catch (IOException e) {
throw new IllegalArgumentException(
"Invalid deployment file: " + deploymentFile.getAbsolutePath(), e);
}
}
// load slot-id
File slotIdFile = new File(baseDir, "airship-slot-id.txt");
UUID uuid = null;
if (slotIdFile.exists()) {
Preconditions.checkArgument(
slotIdFile.canRead(), "can not read " + slotIdFile.getAbsolutePath());
try {
String slotIdString = Files.toString(slotIdFile, UTF_8).trim();
try {
uuid = UUID.fromString(slotIdString);
} catch (IllegalArgumentException e) {
}
if (uuid == null) {
log.warn(
"Invalid slot id ["
+ slotIdString
+ "]: attempting to delete airship-slot-id.txt file and recreating a new one");
slotIdFile.delete();
}
} catch (IOException e) {
Preconditions.checkArgument(
slotIdFile.canRead(), "can not read " + slotIdFile.getAbsolutePath());
}
}
if (uuid == null) {
uuid = UUID.randomUUID();
try {
Files.write(uuid.toString(), slotIdFile, UTF_8);
} catch (IOException e) {
Preconditions.checkArgument(
slotIdFile.canRead(), "can not write " + slotIdFile.getAbsolutePath());
}
}
slotId = uuid;
}
@Override
public Deployment install(Installation installation) {
Preconditions.checkNotNull(installation, "installation is null");
File deploymentDir = new File(baseDir, "installation");
Assignment assignment = installation.getAssignment();
Deployment newDeployment =
new Deployment(
slotId, location, deploymentDir, getDataDir(), assignment, installation.getResources());
File tempDir = createTempDir(baseDir, "tmp-install");
try {
// download the binary
File binary = new File(tempDir, "airship-binary.tar.gz");
try {
Files.copy(Resources.newInputStreamSupplier(installation.getBinaryFile().toURL()), binary);
} catch (IOException e) {
throw new RuntimeException(
"Unable to download binary "
+ assignment.getBinary()
+ " from "
+ installation.getBinaryFile(),
e);
}
// unpack the binary into a temp unpack dir
File unpackDir = new File(tempDir, "unpack");
unpackDir.mkdirs();
try {
extractTar(binary, unpackDir, tarTimeout);
} catch (CommandFailedException e) {
throw new RuntimeException(
"Unable to extract tar file " + assignment.getBinary() + ": " + e.getMessage());
}
// find the archive root dir (it should be the only file in the temp unpack dir)
List<File> files = listFiles(unpackDir);
if (files.size() != 1) {
throw new RuntimeException(
"Invalid tar file: file does not have a root directory " + assignment.getBinary());
}
File binaryRootDir = files.get(0);
// unpack config bundle
try {
URL url = installation.getConfigFile().toURL();
ConfigUtils.unpackConfig(Resources.newInputStreamSupplier(url), binaryRootDir);
} catch (Exception e) {
throw new RuntimeException(
"Unable to extract config bundle " + assignment.getConfig() + ": " + e.getMessage());
}
// installation is good, clear the current deployment
if (this.deployment != null) {
this.deploymentFile.delete();
deleteRecursively(this.deployment.getDeploymentDir());
this.deployment = null;
}
// save deployment versions file
try {
save(newDeployment);
} catch (IOException e) {
throw new RuntimeException("Unable to save deployment file", e);
}
// move the binary root directory to the final target
try {
Files.move(binaryRootDir, deploymentDir);
} catch (IOException e) {
throw new RuntimeException("Unable to move deployment to final location", e);
}
} finally {
if (!deleteRecursively(tempDir)) {
log.warn("Unable to delete temp directory: %s", tempDir.getAbsolutePath());
}
}
this.deployment = newDeployment;
return newDeployment;
}