private void copyLogTail() {
if (configuration.getTailLogLinesToSave() <= 0) {
return;
}
final Path tailOfLogPath =
taskDefinition.getTaskDirectoryPath().resolve(configuration.getServiceFinishedTailLog());
if (Files.exists(tailOfLogPath)) {
log.debug("{} already existed, skipping tail", tailOfLogPath);
return;
}
final List<String> cmd =
ImmutableList.of(
"tail",
"-n",
Integer.toString(configuration.getTailLogLinesToSave()),
taskDefinition.getServiceLogOut());
try {
new SimpleProcessManager(log).runCommand(cmd, Redirect.to(tailOfLogPath.toFile()));
} catch (Throwable t) {
log.error(
"Failed saving tail of log {} to {}",
taskDefinition.getServiceLogOut(),
configuration.getServiceFinishedTailLog(),
t);
}
}
private void ensureServiceOutExists() {
try {
if (!Files.exists(taskDefinition.getServiceLogOutPath())) {
Files.createFile(taskDefinition.getServiceLogOutPath());
}
} catch (FileAlreadyExistsException faee) {
log.debug("Executor out {} already existed", taskDefinition.getServiceLogOut());
} catch (Throwable t) {
log.error("Failed creating executor out {}", taskDefinition.getServiceLogOut(), t);
}
}
private void checkArtifactSignature(S3ArtifactSignature s3ArtifactSignature) {
final Path artifactPath =
Paths.get(
s3Configuration.getArtifactCacheDirectory(), s3ArtifactSignature.getArtifactFilename());
final Path artifactSignaturePath =
Paths.get(s3Configuration.getArtifactCacheDirectory(), s3ArtifactSignature.getFilename());
if (!Files.exists(artifactPath)) {
log.warn("Artifact {} not found for signature {}", artifactPath, s3ArtifactSignature);
return;
}
final List<String> verifyCommand =
new ArrayList<>(executorConfiguration.getArtifactSignatureVerificationCommand().size());
for (String arg : executorConfiguration.getArtifactSignatureVerificationCommand()) {
verifyCommand.add(
arg.replace("{artifactPath}", artifactPath.toString())
.replace("{artifactSignaturePath}", artifactSignaturePath.toString()));
}
try {
final ProcessBuilder processBuilder = new ProcessBuilder(verifyCommand);
processBuilder.directory(taskDefinition.getTaskDirectoryPath().toFile());
processBuilder.redirectError(taskDefinition.getSignatureVerifyOutPath().toFile());
processBuilder.redirectOutput(taskDefinition.getSignatureVerifyOutPath().toFile());
final Process p = processBuilder.start();
p.waitFor(); // TODO: add some sort of timeout?
if (p.exitValue() != 0) {
log.error(
"Failed to validate signature {} for artifact {}",
s3ArtifactSignature.getFilename(),
s3ArtifactSignature.getArtifactFilename());
if (executorConfiguration.isFailTaskOnInvalidArtifactSignature()) {
throw new RuntimeException(
String.format("Failed to validate signature for artifact %s", artifactPath));
}
} else {
log.info(
"Signature {} for artifact {} is valid!",
s3ArtifactSignature.getFilename(),
s3ArtifactSignature.getArtifactFilename());
}
} catch (InterruptedException | IOException e) {
throw Throwables.propagate(e);
}
}
public void checkSignatures() {
if (!taskDefinition.getExecutorData().getS3ArtifactSignatures().isPresent()
|| taskDefinition.getExecutorData().getS3ArtifactSignatures().get().isEmpty()) {
log.info("No s3 artifact signatures, skipping verification.");
return;
}
for (S3ArtifactSignature s3ArtifactSignature :
taskDefinition.getExecutorData().getS3ArtifactSignatures().get()) {
checkArtifactSignature(s3ArtifactSignature);
}
}
private String getS3KeyPattern() {
String s3KeyPattern = configuration.getS3KeyPattern();
final SingularityTaskId singularityTaskId = getSingularityTaskId();
return SingularityS3FormatHelper.getS3KeyFormat(
s3KeyPattern, singularityTaskId, taskDefinition.getExecutorData().getLoggingTag());
}
private boolean writeTailMetadata(boolean finished) {
if (!taskDefinition.getExecutorData().getLoggingTag().isPresent()) {
if (!finished) {
log.warn("Not writing logging metadata because logging tag is absent");
}
return true;
}
final TailMetadata tailMetadata =
new TailMetadata(
taskDefinition.getServiceLogOut(),
taskDefinition.getExecutorData().getLoggingTag().get(),
taskDefinition.getExecutorData().getLoggingExtraFields(),
finished);
final Path path =
TailMetadata.getTailMetadataPath(
configuration.getLogMetadataDirectory(),
configuration.getLogMetadataSuffix(),
tailMetadata);
return jsonObjectFileHelper.writeObject(tailMetadata, path, log);
}
private boolean writeS3MetadataFile(boolean finished) {
Path logrotateDirectory =
taskDefinition
.getServiceLogOutPath()
.getParent()
.resolve(configuration.getLogrotateToDirectory());
S3UploadMetadata s3UploadMetadata =
new S3UploadMetadata(
logrotateDirectory.toString(),
getS3Glob(),
configuration.getS3Bucket(),
getS3KeyPattern(),
finished);
String s3UploadMetadatafilename =
String.format("%s%s", taskDefinition.getTaskId(), configuration.getS3MetadataSuffix());
Path s3UploadMetadataPath =
configuration.getS3MetadataDirectory().resolve(s3UploadMetadatafilename);
return jsonObjectFileHelper.writeObject(s3UploadMetadata, s3UploadMetadataPath, log);
}
public boolean manualLogrotate() {
if (!Files.exists(getLogrotateConfPath())) {
log.info("{} did not exist, skipping manual logrotation", getLogrotateConfPath());
return true;
}
final List<String> command =
ImmutableList.of(
configuration.getLogrotateCommand(),
"-f",
"-s",
taskDefinition.getLogrotateStateFilePath().toString(),
getLogrotateConfPath().toString());
try {
new SimpleProcessManager(log).runCommand(command);
return true;
} catch (Throwable t) {
log.warn("Tried to manually logrotate using {}, but caught", getLogrotateConfPath(), t);
return false;
}
}
public Path getLogrotateConfPath() {
return configuration.getLogrotateConfDirectory().resolve(taskDefinition.getTaskId());
}
private SingularityTaskId getSingularityTaskId() {
return SingularityTaskId.valueOf(taskDefinition.getTaskId());
}
private String getS3Glob() {
return String.format("%s*.gz*", taskDefinition.getServiceLogOutPath().getFileName());
}