/**
* Creates a zip file of the metadata and recorded artifacts and stores it in the artifact cache.
*/
public void performUploadToArtifactCache(
ImmutableSet<RuleKey> ruleKeys, ArtifactCache artifactCache, BuckEventBus eventBus)
throws InterruptedException {
// Skip all of this if caching is disabled. Although artifactCache.store() will be a noop,
// building up the zip is wasted I/O.
if (!artifactCache.isStoreSupported()) {
return;
}
ArtifactCompressionEvent.Started started =
ArtifactCompressionEvent.started(ArtifactCompressionEvent.Operation.COMPRESS, ruleKeys);
eventBus.post(started);
final Path zip;
ImmutableSet<Path> pathsToIncludeInZip = ImmutableSet.of();
ImmutableMap<String, String> buildMetadata;
try {
pathsToIncludeInZip = getRecordedDirsAndFiles();
zip =
Files.createTempFile(
"buck_artifact_" + MoreFiles.sanitize(buildTarget.getShortName()), ".zip");
buildMetadata = getBuildMetadata();
projectFilesystem.createZip(pathsToIncludeInZip, zip, ImmutableMap.<Path, String>of());
} catch (IOException e) {
eventBus.post(
ConsoleEvent.info(
"Failed to create zip for %s containing:\n%s",
buildTarget, Joiner.on('\n').join(ImmutableSortedSet.copyOf(pathsToIncludeInZip))));
e.printStackTrace();
return;
} finally {
eventBus.post(ArtifactCompressionEvent.finished(started));
}
// Store the artifact, including any additional metadata.
ListenableFuture<Void> storeFuture =
artifactCache.store(ruleKeys, buildMetadata, BorrowablePath.notBorrowablePath(zip));
storeFuture.addListener(
new Runnable() {
@Override
public void run() {
try {
Files.deleteIfExists(zip);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
},
directExecutor());
}
@Override
public int execute(ExecutionContext context) throws IOException, InterruptedException {
MoreFiles.makeExecutable(filesystem.resolve(file));
return 0;
}
public void copyRecursively(Path source, Path pathRelativeToProjectRoot) throws IOException {
MoreFiles.copyRecursively(source, destPath.resolve(pathRelativeToProjectRoot));
}
/**
* This will copy the template directory, renaming files named {@code foo.fixture} to {@code foo}
* in the process. Files whose names end in {@code .expected} will not be copied.
*/
@SuppressWarnings("PMD.EmptyCatchBlock")
public ProjectWorkspace setUp() throws IOException {
MoreFiles.copyRecursively(templatePath, destPath, BUILD_FILE_RENAME);
// Stamp the buck-out directory if it exists and isn't stamped already
try (OutputStream outputStream =
new BufferedOutputStream(
Channels.newOutputStream(
Files.newByteChannel(
destPath.resolve(BuckConstant.getCurrentVersionFile()),
ImmutableSet.<OpenOption>of(
StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE))))) {
outputStream.write(BuckVersion.getVersion().getBytes(Charsets.UTF_8));
} catch (FileAlreadyExistsException | NoSuchFileException e) {
// If the current version file already exists we don't need to create it
// If buck-out doesn't exist we don't need to stamp it
}
if (Platform.detect() == Platform.WINDOWS) {
// Hack for symlinks on Windows.
SimpleFileVisitor<Path> copyDirVisitor =
new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path path, BasicFileAttributes attrs)
throws IOException {
// On Windows, symbolic links from git repository are checked out as normal files
// containing a one-line path. In order to distinguish them, paths are read and
// pointed
// files are trued to locate. Once the pointed file is found, it will be copied to
// target.
// On NTFS length of path must be greater than 0 and less than 4096.
if (attrs.size() > 0 && attrs.size() <= 4096) {
String linkTo = new String(Files.readAllBytes(path), UTF_8);
Path linkToFile;
try {
linkToFile = templatePath.resolve(linkTo);
} catch (InvalidPathException e) {
// Let's assume we were reading a normal text file, and not something meant to be
// a
// link.
return FileVisitResult.CONTINUE;
}
if (Files.isRegularFile(linkToFile)) {
Files.copy(linkToFile, path, StandardCopyOption.REPLACE_EXISTING);
} else if (Files.isDirectory(linkToFile)) {
Files.delete(path);
MoreFiles.copyRecursively(linkToFile, path);
}
}
return FileVisitResult.CONTINUE;
}
};
Files.walkFileTree(destPath, copyDirVisitor);
}
// Disable the directory cache by default. Tests that want to enable it can call
// `enableDirCache` on this object. Only do this if a .buckconfig.local file does not already
// exist, however (we assume the test knows what it is doing at that point).
if (!Files.exists(getPath(".buckconfig.local"))) {
writeContentsToPath("[cache]\n mode =", ".buckconfig.local");
}
isSetUp = true;
return this;
}
private CacheResult tryToFetchArtifactFromBuildCacheAndOverlayOnTopOfProjectFilesystem(
BuildRule rule,
RuleKey ruleKey,
BuildInfoRecorder buildInfoRecorder,
ArtifactCache artifactCache,
ProjectFilesystem filesystem,
BuildContext buildContext)
throws InterruptedException {
// Create a temp file whose extension must be ".zip" for Filesystems.newFileSystem() to infer
// that we are creating a zip-based FileSystem.
Path zipFile;
try {
zipFile =
Files.createTempFile(
"buck_artifact_" + MoreFiles.sanitize(rule.getBuildTarget().getShortName()), ".zip");
} catch (IOException e) {
throw new RuntimeException(e);
}
// TODO(mbolin): Change ArtifactCache.fetch() so that it returns a File instead of takes one.
// Then we could download directly from the remote cache into the on-disk cache and unzip it
// from there.
CacheResult cacheResult =
buildInfoRecorder.fetchArtifactForBuildable(ruleKey, zipFile, artifactCache);
if (!cacheResult.getType().isSuccess()) {
try {
Files.delete(zipFile);
} catch (IOException e) {
LOG.warn(e, "failed to delete %s", zipFile);
}
return cacheResult;
}
// We unzip the file in the root of the project directory.
// Ideally, the following would work:
//
// Path pathToZip = Paths.get(zipFile.getAbsolutePath());
// FileSystem fs = FileSystems.newFileSystem(pathToZip, /* loader */ null);
// Path root = Iterables.getOnlyElement(fs.getRootDirectories());
// MoreFiles.copyRecursively(root, projectRoot);
//
// Unfortunately, this does not appear to work, in practice, because MoreFiles fails when trying
// to resolve a Path for a zip entry against a file Path on disk.
ArtifactCacheEvent.Started started =
ArtifactCacheEvent.started(
ArtifactCacheEvent.Operation.DECOMPRESS, ImmutableSet.of(ruleKey));
buildContext.getEventBus().post(started);
try {
Unzip.extractZipFile(
zipFile.toAbsolutePath(),
filesystem,
Unzip.ExistingFileMode.OVERWRITE_AND_CLEAN_DIRECTORIES);
// We only delete the ZIP file when it has been unzipped successfully. Otherwise, we leave it
// around for debugging purposes.
Files.delete(zipFile);
if (cacheResult.getType() == CacheResult.Type.HIT) {
// If we have a hit, also write out the build metadata.
Path metadataDir = BuildInfo.getPathToMetadataDirectory(rule.getBuildTarget());
for (Map.Entry<String, String> ent : cacheResult.getMetadata().entrySet()) {
Path dest = metadataDir.resolve(ent.getKey());
filesystem.createParentDirs(dest);
filesystem.writeContentsToPath(ent.getValue(), dest);
}
}
} catch (IOException e) {
// In the wild, we have seen some inexplicable failures during this step. For now, we try to
// give the user as much information as we can to debug the issue, but return CacheResult.MISS
// so that Buck will fall back on doing a local build.
buildContext
.getEventBus()
.post(
ConsoleEvent.warning(
"Failed to unzip the artifact for %s at %s.\n"
+ "The rule will be built locally, "
+ "but here is the stacktrace of the failed unzip call:\n"
+ rule.getBuildTarget(),
zipFile.toAbsolutePath(),
Throwables.getStackTraceAsString(e)));
return CacheResult.miss();
} finally {
buildContext.getEventBus().post(ArtifactCacheEvent.finished(started));
}
return cacheResult;
}
/**
* This will copy the template directory, renaming files named {@code foo.fixture} to {@code foo}
* in the process. Files whose names end in {@code .expected} will not be copied.
*/
@SuppressWarnings("PMD.EmptyCatchBlock")
public ProjectWorkspace setUp() throws IOException {
MoreFiles.copyRecursively(templatePath, destPath, BUILD_FILE_RENAME);
// Stamp the buck-out directory if it exists and isn't stamped already
try (OutputStream outputStream =
new BufferedOutputStream(
Channels.newOutputStream(
Files.newByteChannel(
destPath.resolve(BuckConstant.CURRENT_VERSION_FILE),
ImmutableSet.<OpenOption>of(
StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE))))) {
outputStream.write(BuckVersion.getVersion().getBytes(Charsets.UTF_8));
} catch (FileAlreadyExistsException | NoSuchFileException e) {
// If the current version file already exists we don't need to create it
// If buck-out doesn't exist we don't need to stamp it
}
// If there's a local.properties in the host project but not in the destination, make a copy.
Path localProperties = FileSystems.getDefault().getPath("local.properties");
Path destLocalProperties = destPath.resolve(localProperties.getFileName());
if (localProperties.toFile().exists() && !destLocalProperties.toFile().exists()) {
Files.copy(localProperties, destLocalProperties);
}
if (Platform.detect() == Platform.WINDOWS) {
// Hack for symlinks on Windows.
SimpleFileVisitor<Path> copyDirVisitor =
new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path path, BasicFileAttributes attrs)
throws IOException {
// On Windows, symbolic links from git repository are checked out as normal files
// containing a one-line path. In order to distinguish them, paths are read and
// pointed
// files are trued to locate. Once the pointed file is found, it will be copied to
// target.
// On NTFS length of path must be greater than 0 and less than 4096.
if (attrs.size() > 0 && attrs.size() <= 4096) {
String linkTo = new String(Files.readAllBytes(path), UTF_8);
Path linkToFile = null;
try {
linkToFile = templatePath.resolve(linkTo);
} catch (InvalidPathException e) {
// Let's assume we were reading a normal text file, and not something meant to be
// a
// link.
return FileVisitResult.CONTINUE;
}
if (Files.isRegularFile(linkToFile)) {
Files.copy(linkToFile, path, StandardCopyOption.REPLACE_EXISTING);
} else if (Files.isDirectory(linkToFile)) {
Files.delete(path);
MoreFiles.copyRecursively(linkToFile, path);
}
}
return FileVisitResult.CONTINUE;
}
};
Files.walkFileTree(destPath, copyDirVisitor);
}
isSetUp = true;
return this;
}