@Test
public void ruleKeyDoesNotChangeWhenOnlyDependencyRuleKeyChanges() throws Exception {
ProjectFilesystem filesystem = new FakeProjectFilesystem();
BuildRuleResolver resolver =
new BuildRuleResolver(TargetGraph.EMPTY, new DefaultTargetNodeToBuildRuleTransformer());
SourcePathResolver pathResolver = new SourcePathResolver(resolver);
Path depOutput = Paths.get("output");
FakeBuildRule dep =
resolver.addToIndex(
new FakeBuildRule(BuildTargetFactory.newInstance("//:dep"), filesystem, pathResolver));
dep.setOutputFile(depOutput.toString());
filesystem.writeContentsToPath("hello", dep.getPathToOutput());
FakeFileHashCache hashCache =
new FakeFileHashCache(ImmutableMap.of(filesystem.resolve(depOutput), HashCode.fromInt(0)));
BuildRule rule =
GenruleBuilder.newGenruleBuilder(BuildTargetFactory.newInstance("//:rule"))
.setOut("out")
.setSrcs(ImmutableList.<SourcePath>of(new BuildTargetSourcePath(dep.getBuildTarget())))
.build(resolver, filesystem);
RuleKey inputKey1 = new InputBasedRuleKeyBuilderFactory(0, hashCache, pathResolver).build(rule);
RuleKey inputKey2 = new InputBasedRuleKeyBuilderFactory(0, hashCache, pathResolver).build(rule);
assertThat(inputKey1, Matchers.equalTo(inputKey2));
}
private void assertCompDir(Path compDir, Optional<String> failure) throws Exception {
ProjectFilesystem filesystem = new ProjectFilesystem(tmp.getRoot().toPath());
CxxPlatform platform = DefaultCxxPlatforms.build(new CxxBuckConfig(new FakeBuckConfig()));
// Build up the paths to various files the archive step will use.
ImmutableList<String> compiler =
platform.getCc().getCommandPrefix(new SourcePathResolver(new BuildRuleResolver()));
Path output = filesystem.resolve(Paths.get("output.o"));
Path relativeInput = Paths.get("input.c");
Path input = filesystem.resolve(relativeInput);
filesystem.writeContentsToPath("int main() {}", relativeInput);
ImmutableList.Builder<String> preprocessorCommand = ImmutableList.builder();
preprocessorCommand.addAll(compiler);
ImmutableList.Builder<String> compilerCommand = ImmutableList.builder();
compilerCommand.addAll(compiler);
compilerCommand.add("-g");
DebugPathSanitizer sanitizer =
new DebugPathSanitizer(200, File.separatorChar, compDir, ImmutableBiMap.<Path, Path>of());
// Build an archive step.
CxxPreprocessAndCompileStep step =
new CxxPreprocessAndCompileStep(
CxxPreprocessAndCompileStep.Operation.COMPILE_MUNGE_DEBUGINFO,
output,
relativeInput,
CxxSource.Type.C,
Optional.of(preprocessorCommand.build()),
Optional.of(compilerCommand.build()),
ImmutableMap.<Path, Path>of(),
sanitizer);
// Execute the archive step and verify it ran successfully.
ExecutionContext executionContext =
TestExecutionContext.newBuilder()
.setProjectFilesystem(new ProjectFilesystem(tmp.getRoot().toPath()))
.build();
TestConsole console = (TestConsole) executionContext.getConsole();
int exitCode = step.execute(executionContext);
if (failure.isPresent()) {
assertNotEquals("compile step succeeded", 0, exitCode);
assertThat(
console.getTextWrittenToStdErr(),
console.getTextWrittenToStdErr(),
Matchers.containsString(failure.get()));
} else {
assertEquals("compile step failed: " + console.getTextWrittenToStdErr(), 0, exitCode);
// Verify that we find the expected compilation dir embedded in the file.
String contents = new String(Files.readAllBytes(output));
assertThat(contents, Matchers.containsString(sanitizer.getCompilationDirectory()));
}
// Cleanup.
Files.delete(input);
Files.deleteIfExists(output);
}
@Test
public void recursiveIgnorePaths() throws IOException, InterruptedException {
Path ignoredBuildFile = Paths.get("a", "b", "BUCK");
ImmutableSet<Path> ignore = ImmutableSet.of(ignoredBuildFile.getParent());
ProjectFilesystem filesystem = new ProjectFilesystem(tmp.getRoot().toPath(), ignore);
Path buildFile = Paths.get("a", "BUCK");
filesystem.mkdirs(buildFile.getParent());
filesystem.writeContentsToPath("", buildFile);
filesystem.mkdirs(ignoredBuildFile.getParent());
filesystem.writeContentsToPath("", ignoredBuildFile);
// Test a recursive spec with an ignored dir.
BuildFileSpec recursiveSpec = BuildFileSpec.fromRecursivePath(buildFile.getParent());
ImmutableSet<Path> expectedBuildFiles = ImmutableSet.of(filesystem.resolve(buildFile));
Cell cell = new TestCellBuilder().setFilesystem(filesystem).build();
ImmutableSet<Path> actualBuildFiles = recursiveSpec.findBuildFiles(cell);
assertEquals(expectedBuildFiles, actualBuildFiles);
}
/**
* Writes the metadata currently stored in memory to the directory returned by {@link
* BuildInfo#getPathToMetadataDirectory(BuildTarget)}.
*/
public void writeMetadataToDisk(boolean clearExistingMetadata) throws IOException {
if (clearExistingMetadata) {
projectFilesystem.deleteRecursivelyIfExists(pathToMetadataDirectory);
}
projectFilesystem.mkdirs(pathToMetadataDirectory);
for (Map.Entry<String, String> entry :
Iterables.concat(metadataToWrite.entrySet(), getBuildMetadata().entrySet())) {
projectFilesystem.writeContentsToPath(
entry.getValue(), pathToMetadataDirectory.resolve(entry.getKey()));
}
}
private CacheResult tryToFetchArtifactFromBuildCacheAndOverlayOnTopOfProjectFilesystem(
BuildRule rule,
RuleKey ruleKey,
BuildInfoRecorder buildInfoRecorder,
ArtifactCache artifactCache,
ProjectFilesystem filesystem,
BuildContext buildContext)
throws InterruptedException {
// Create a temp file whose extension must be ".zip" for Filesystems.newFileSystem() to infer
// that we are creating a zip-based FileSystem.
Path zipFile;
try {
zipFile =
Files.createTempFile(
"buck_artifact_" + MoreFiles.sanitize(rule.getBuildTarget().getShortName()), ".zip");
} catch (IOException e) {
throw new RuntimeException(e);
}
// TODO(mbolin): Change ArtifactCache.fetch() so that it returns a File instead of takes one.
// Then we could download directly from the remote cache into the on-disk cache and unzip it
// from there.
CacheResult cacheResult =
buildInfoRecorder.fetchArtifactForBuildable(ruleKey, zipFile, artifactCache);
if (!cacheResult.getType().isSuccess()) {
try {
Files.delete(zipFile);
} catch (IOException e) {
LOG.warn(e, "failed to delete %s", zipFile);
}
return cacheResult;
}
// We unzip the file in the root of the project directory.
// Ideally, the following would work:
//
// Path pathToZip = Paths.get(zipFile.getAbsolutePath());
// FileSystem fs = FileSystems.newFileSystem(pathToZip, /* loader */ null);
// Path root = Iterables.getOnlyElement(fs.getRootDirectories());
// MoreFiles.copyRecursively(root, projectRoot);
//
// Unfortunately, this does not appear to work, in practice, because MoreFiles fails when trying
// to resolve a Path for a zip entry against a file Path on disk.
ArtifactCacheEvent.Started started =
ArtifactCacheEvent.started(
ArtifactCacheEvent.Operation.DECOMPRESS, ImmutableSet.of(ruleKey));
buildContext.getEventBus().post(started);
try {
Unzip.extractZipFile(
zipFile.toAbsolutePath(),
filesystem,
Unzip.ExistingFileMode.OVERWRITE_AND_CLEAN_DIRECTORIES);
// We only delete the ZIP file when it has been unzipped successfully. Otherwise, we leave it
// around for debugging purposes.
Files.delete(zipFile);
if (cacheResult.getType() == CacheResult.Type.HIT) {
// If we have a hit, also write out the build metadata.
Path metadataDir = BuildInfo.getPathToMetadataDirectory(rule.getBuildTarget());
for (Map.Entry<String, String> ent : cacheResult.getMetadata().entrySet()) {
Path dest = metadataDir.resolve(ent.getKey());
filesystem.createParentDirs(dest);
filesystem.writeContentsToPath(ent.getValue(), dest);
}
}
} catch (IOException e) {
// In the wild, we have seen some inexplicable failures during this step. For now, we try to
// give the user as much information as we can to debug the issue, but return CacheResult.MISS
// so that Buck will fall back on doing a local build.
buildContext
.getEventBus()
.post(
ConsoleEvent.warning(
"Failed to unzip the artifact for %s at %s.\n"
+ "The rule will be built locally, "
+ "but here is the stacktrace of the failed unzip call:\n"
+ rule.getBuildTarget(),
zipFile.toAbsolutePath(),
Throwables.getStackTraceAsString(e)));
return CacheResult.miss();
} finally {
buildContext.getEventBus().post(ArtifactCacheEvent.finished(started));
}
return cacheResult;
}
