Beispiel #1
0
  /**
   * Creates a zip file of the metadata and recorded artifacts and stores it in the artifact cache.
   */
  public void performUploadToArtifactCache(
      ImmutableSet<RuleKey> ruleKeys, ArtifactCache artifactCache, BuckEventBus eventBus)
      throws InterruptedException {

    // Skip all of this if caching is disabled. Although artifactCache.store() will be a noop,
    // building up the zip is wasted I/O.
    if (!artifactCache.isStoreSupported()) {
      return;
    }

    ArtifactCompressionEvent.Started started =
        ArtifactCompressionEvent.started(ArtifactCompressionEvent.Operation.COMPRESS, ruleKeys);
    eventBus.post(started);

    final Path zip;
    ImmutableSet<Path> pathsToIncludeInZip = ImmutableSet.of();
    ImmutableMap<String, String> buildMetadata;
    try {
      pathsToIncludeInZip = getRecordedDirsAndFiles();
      zip =
          Files.createTempFile(
              "buck_artifact_" + MoreFiles.sanitize(buildTarget.getShortName()), ".zip");
      buildMetadata = getBuildMetadata();
      projectFilesystem.createZip(pathsToIncludeInZip, zip, ImmutableMap.<Path, String>of());
    } catch (IOException e) {
      eventBus.post(
          ConsoleEvent.info(
              "Failed to create zip for %s containing:\n%s",
              buildTarget, Joiner.on('\n').join(ImmutableSortedSet.copyOf(pathsToIncludeInZip))));
      e.printStackTrace();
      return;
    } finally {
      eventBus.post(ArtifactCompressionEvent.finished(started));
    }

    // Store the artifact, including any additional metadata.
    ListenableFuture<Void> storeFuture =
        artifactCache.store(ruleKeys, buildMetadata, BorrowablePath.notBorrowablePath(zip));
    storeFuture.addListener(
        new Runnable() {
          @Override
          public void run() {
            try {
              Files.deleteIfExists(zip);
            } catch (IOException e) {
              throw new RuntimeException(e);
            }
          }
        },
        directExecutor());
  }
Beispiel #2
0
  private CacheResult tryToFetchArtifactFromBuildCacheAndOverlayOnTopOfProjectFilesystem(
      BuildRule rule,
      RuleKey ruleKey,
      BuildInfoRecorder buildInfoRecorder,
      ArtifactCache artifactCache,
      ProjectFilesystem filesystem,
      BuildContext buildContext)
      throws InterruptedException {

    // Create a temp file whose extension must be ".zip" for Filesystems.newFileSystem() to infer
    // that we are creating a zip-based FileSystem.
    Path zipFile;
    try {
      zipFile =
          Files.createTempFile(
              "buck_artifact_" + MoreFiles.sanitize(rule.getBuildTarget().getShortName()), ".zip");
    } catch (IOException e) {
      throw new RuntimeException(e);
    }

    // TODO(mbolin): Change ArtifactCache.fetch() so that it returns a File instead of takes one.
    // Then we could download directly from the remote cache into the on-disk cache and unzip it
    // from there.
    CacheResult cacheResult =
        buildInfoRecorder.fetchArtifactForBuildable(ruleKey, zipFile, artifactCache);
    if (!cacheResult.getType().isSuccess()) {
      try {
        Files.delete(zipFile);
      } catch (IOException e) {
        LOG.warn(e, "failed to delete %s", zipFile);
      }
      return cacheResult;
    }

    // We unzip the file in the root of the project directory.
    // Ideally, the following would work:
    //
    // Path pathToZip = Paths.get(zipFile.getAbsolutePath());
    // FileSystem fs = FileSystems.newFileSystem(pathToZip, /* loader */ null);
    // Path root = Iterables.getOnlyElement(fs.getRootDirectories());
    // MoreFiles.copyRecursively(root, projectRoot);
    //
    // Unfortunately, this does not appear to work, in practice, because MoreFiles fails when trying
    // to resolve a Path for a zip entry against a file Path on disk.
    ArtifactCacheEvent.Started started =
        ArtifactCacheEvent.started(
            ArtifactCacheEvent.Operation.DECOMPRESS, ImmutableSet.of(ruleKey));
    buildContext.getEventBus().post(started);
    try {
      Unzip.extractZipFile(
          zipFile.toAbsolutePath(),
          filesystem,
          Unzip.ExistingFileMode.OVERWRITE_AND_CLEAN_DIRECTORIES);

      // We only delete the ZIP file when it has been unzipped successfully. Otherwise, we leave it
      // around for debugging purposes.
      Files.delete(zipFile);

      if (cacheResult.getType() == CacheResult.Type.HIT) {

        // If we have a hit, also write out the build metadata.
        Path metadataDir = BuildInfo.getPathToMetadataDirectory(rule.getBuildTarget());
        for (Map.Entry<String, String> ent : cacheResult.getMetadata().entrySet()) {
          Path dest = metadataDir.resolve(ent.getKey());
          filesystem.createParentDirs(dest);
          filesystem.writeContentsToPath(ent.getValue(), dest);
        }
      }

    } catch (IOException e) {
      // In the wild, we have seen some inexplicable failures during this step. For now, we try to
      // give the user as much information as we can to debug the issue, but return CacheResult.MISS
      // so that Buck will fall back on doing a local build.
      buildContext
          .getEventBus()
          .post(
              ConsoleEvent.warning(
                  "Failed to unzip the artifact for %s at %s.\n"
                      + "The rule will be built locally, "
                      + "but here is the stacktrace of the failed unzip call:\n"
                      + rule.getBuildTarget(),
                  zipFile.toAbsolutePath(),
                  Throwables.getStackTraceAsString(e)));
      return CacheResult.miss();
    } finally {
      buildContext.getEventBus().post(ArtifactCacheEvent.finished(started));
    }

    return cacheResult;
  }