@Test public void ruleKeyDoesNotChangeWhenOnlyDependencyRuleKeyChanges() throws Exception { ProjectFilesystem filesystem = new FakeProjectFilesystem(); BuildRuleResolver resolver = new BuildRuleResolver(TargetGraph.EMPTY, new DefaultTargetNodeToBuildRuleTransformer()); SourcePathResolver pathResolver = new SourcePathResolver(resolver); Path depOutput = Paths.get("output"); FakeBuildRule dep = resolver.addToIndex( new FakeBuildRule(BuildTargetFactory.newInstance("//:dep"), filesystem, pathResolver)); dep.setOutputFile(depOutput.toString()); filesystem.writeContentsToPath("hello", dep.getPathToOutput()); FakeFileHashCache hashCache = new FakeFileHashCache(ImmutableMap.of(filesystem.resolve(depOutput), HashCode.fromInt(0))); BuildRule rule = GenruleBuilder.newGenruleBuilder(BuildTargetFactory.newInstance("//:rule")) .setOut("out") .setSrcs(ImmutableList.<SourcePath>of(new BuildTargetSourcePath(dep.getBuildTarget()))) .build(resolver, filesystem); RuleKey inputKey1 = new InputBasedRuleKeyBuilderFactory(0, hashCache, pathResolver).build(rule); RuleKey inputKey2 = new InputBasedRuleKeyBuilderFactory(0, hashCache, pathResolver).build(rule); assertThat(inputKey1, Matchers.equalTo(inputKey2)); }
private void assertCompDir(Path compDir, Optional<String> failure) throws Exception { ProjectFilesystem filesystem = new ProjectFilesystem(tmp.getRoot().toPath()); CxxPlatform platform = DefaultCxxPlatforms.build(new CxxBuckConfig(new FakeBuckConfig())); // Build up the paths to various files the archive step will use. ImmutableList<String> compiler = platform.getCc().getCommandPrefix(new SourcePathResolver(new BuildRuleResolver())); Path output = filesystem.resolve(Paths.get("output.o")); Path relativeInput = Paths.get("input.c"); Path input = filesystem.resolve(relativeInput); filesystem.writeContentsToPath("int main() {}", relativeInput); ImmutableList.Builder<String> preprocessorCommand = ImmutableList.builder(); preprocessorCommand.addAll(compiler); ImmutableList.Builder<String> compilerCommand = ImmutableList.builder(); compilerCommand.addAll(compiler); compilerCommand.add("-g"); DebugPathSanitizer sanitizer = new DebugPathSanitizer(200, File.separatorChar, compDir, ImmutableBiMap.<Path, Path>of()); // Build an archive step. CxxPreprocessAndCompileStep step = new CxxPreprocessAndCompileStep( CxxPreprocessAndCompileStep.Operation.COMPILE_MUNGE_DEBUGINFO, output, relativeInput, CxxSource.Type.C, Optional.of(preprocessorCommand.build()), Optional.of(compilerCommand.build()), ImmutableMap.<Path, Path>of(), sanitizer); // Execute the archive step and verify it ran successfully. ExecutionContext executionContext = TestExecutionContext.newBuilder() .setProjectFilesystem(new ProjectFilesystem(tmp.getRoot().toPath())) .build(); TestConsole console = (TestConsole) executionContext.getConsole(); int exitCode = step.execute(executionContext); if (failure.isPresent()) { assertNotEquals("compile step succeeded", 0, exitCode); assertThat( console.getTextWrittenToStdErr(), console.getTextWrittenToStdErr(), Matchers.containsString(failure.get())); } else { assertEquals("compile step failed: " + console.getTextWrittenToStdErr(), 0, exitCode); // Verify that we find the expected compilation dir embedded in the file. String contents = new String(Files.readAllBytes(output)); assertThat(contents, Matchers.containsString(sanitizer.getCompilationDirectory())); } // Cleanup. Files.delete(input); Files.deleteIfExists(output); }
@Test public void recursiveIgnorePaths() throws IOException, InterruptedException { Path ignoredBuildFile = Paths.get("a", "b", "BUCK"); ImmutableSet<Path> ignore = ImmutableSet.of(ignoredBuildFile.getParent()); ProjectFilesystem filesystem = new ProjectFilesystem(tmp.getRoot().toPath(), ignore); Path buildFile = Paths.get("a", "BUCK"); filesystem.mkdirs(buildFile.getParent()); filesystem.writeContentsToPath("", buildFile); filesystem.mkdirs(ignoredBuildFile.getParent()); filesystem.writeContentsToPath("", ignoredBuildFile); // Test a recursive spec with an ignored dir. BuildFileSpec recursiveSpec = BuildFileSpec.fromRecursivePath(buildFile.getParent()); ImmutableSet<Path> expectedBuildFiles = ImmutableSet.of(filesystem.resolve(buildFile)); Cell cell = new TestCellBuilder().setFilesystem(filesystem).build(); ImmutableSet<Path> actualBuildFiles = recursiveSpec.findBuildFiles(cell); assertEquals(expectedBuildFiles, actualBuildFiles); }
/** * Writes the metadata currently stored in memory to the directory returned by {@link * BuildInfo#getPathToMetadataDirectory(BuildTarget)}. */ public void writeMetadataToDisk(boolean clearExistingMetadata) throws IOException { if (clearExistingMetadata) { projectFilesystem.deleteRecursivelyIfExists(pathToMetadataDirectory); } projectFilesystem.mkdirs(pathToMetadataDirectory); for (Map.Entry<String, String> entry : Iterables.concat(metadataToWrite.entrySet(), getBuildMetadata().entrySet())) { projectFilesystem.writeContentsToPath( entry.getValue(), pathToMetadataDirectory.resolve(entry.getKey())); } }
private CacheResult tryToFetchArtifactFromBuildCacheAndOverlayOnTopOfProjectFilesystem( BuildRule rule, RuleKey ruleKey, BuildInfoRecorder buildInfoRecorder, ArtifactCache artifactCache, ProjectFilesystem filesystem, BuildContext buildContext) throws InterruptedException { // Create a temp file whose extension must be ".zip" for Filesystems.newFileSystem() to infer // that we are creating a zip-based FileSystem. Path zipFile; try { zipFile = Files.createTempFile( "buck_artifact_" + MoreFiles.sanitize(rule.getBuildTarget().getShortName()), ".zip"); } catch (IOException e) { throw new RuntimeException(e); } // TODO(mbolin): Change ArtifactCache.fetch() so that it returns a File instead of takes one. // Then we could download directly from the remote cache into the on-disk cache and unzip it // from there. CacheResult cacheResult = buildInfoRecorder.fetchArtifactForBuildable(ruleKey, zipFile, artifactCache); if (!cacheResult.getType().isSuccess()) { try { Files.delete(zipFile); } catch (IOException e) { LOG.warn(e, "failed to delete %s", zipFile); } return cacheResult; } // We unzip the file in the root of the project directory. // Ideally, the following would work: // // Path pathToZip = Paths.get(zipFile.getAbsolutePath()); // FileSystem fs = FileSystems.newFileSystem(pathToZip, /* loader */ null); // Path root = Iterables.getOnlyElement(fs.getRootDirectories()); // MoreFiles.copyRecursively(root, projectRoot); // // Unfortunately, this does not appear to work, in practice, because MoreFiles fails when trying // to resolve a Path for a zip entry against a file Path on disk. ArtifactCacheEvent.Started started = ArtifactCacheEvent.started( ArtifactCacheEvent.Operation.DECOMPRESS, ImmutableSet.of(ruleKey)); buildContext.getEventBus().post(started); try { Unzip.extractZipFile( zipFile.toAbsolutePath(), filesystem, Unzip.ExistingFileMode.OVERWRITE_AND_CLEAN_DIRECTORIES); // We only delete the ZIP file when it has been unzipped successfully. Otherwise, we leave it // around for debugging purposes. Files.delete(zipFile); if (cacheResult.getType() == CacheResult.Type.HIT) { // If we have a hit, also write out the build metadata. Path metadataDir = BuildInfo.getPathToMetadataDirectory(rule.getBuildTarget()); for (Map.Entry<String, String> ent : cacheResult.getMetadata().entrySet()) { Path dest = metadataDir.resolve(ent.getKey()); filesystem.createParentDirs(dest); filesystem.writeContentsToPath(ent.getValue(), dest); } } } catch (IOException e) { // In the wild, we have seen some inexplicable failures during this step. For now, we try to // give the user as much information as we can to debug the issue, but return CacheResult.MISS // so that Buck will fall back on doing a local build. buildContext .getEventBus() .post( ConsoleEvent.warning( "Failed to unzip the artifact for %s at %s.\n" + "The rule will be built locally, " + "but here is the stacktrace of the failed unzip call:\n" + rule.getBuildTarget(), zipFile.toAbsolutePath(), Throwables.getStackTraceAsString(e))); return CacheResult.miss(); } finally { buildContext.getEventBus().post(ArtifactCacheEvent.finished(started)); } return cacheResult; }