private void addDirectoryToZipEntryList(
      File directory, String currentPath, ImmutableMap.Builder<File, ZipEntry> zipEntriesBuilder)
      throws IOException {
    Preconditions.checkNotNull(currentPath);

    for (File inputFile : directory.listFiles()) {
      String childPath = currentPath + (currentPath.isEmpty() ? "" : "/") + inputFile.getName();

      if (inputFile.isDirectory()) {
        addDirectoryToZipEntryList(inputFile, childPath, zipEntriesBuilder);
      } else {
        ZipEntry nextEntry = new ZipEntry(childPath);
        long fileLength = inputFile.length();
        if (fileLength > maxDeflatedBytes
            || EXTENSIONS_NOT_TO_DEFLATE.contains(Files.getFileExtension(inputFile.getName()))) {
          nextEntry.setMethod(ZipEntry.STORED);
          nextEntry.setCompressedSize(inputFile.length());
          nextEntry.setSize(inputFile.length());
          HashCode crc = ByteStreams.hash(Files.newInputStreamSupplier(inputFile), Hashing.crc32());
          nextEntry.setCrc(crc.padToLong());
        }

        zipEntriesBuilder.put(inputFile, nextEntry);
      }
    }
  }
Ejemplo n.º 2
0
  private ArtifactCache createHttpArtifactCache(BuckEventBus buckEventBus) {
    URL url;
    try {
      url = new URL(getValue("cache", "http_url").or(DEFAULT_HTTP_URL));
    } catch (MalformedURLException e) {
      throw new HumanReadableException(e, "Malformed [cache]http_url: %s", e.getMessage());
    }

    int timeoutSeconds =
        Integer.parseInt(
            getValue("cache", "http_timeout_seconds").or(DEFAULT_HTTP_CACHE_TIMEOUT_SECONDS));

    boolean doStore = readCacheMode("http_mode", DEFAULT_HTTP_CACHE_MODE);

    // Setup the defaut client to use.
    OkHttpClient client = new OkHttpClient();
    final String localhost = getLocalhost();
    client
        .networkInterceptors()
        .add(
            new Interceptor() {
              @Override
              public Response intercept(Chain chain) throws IOException {
                return chain.proceed(
                    chain
                        .request()
                        .newBuilder()
                        .addHeader("X-BuckCache-User", System.getProperty("user.name", "<unknown>"))
                        .addHeader("X-BuckCache-Host", localhost)
                        .build());
              }
            });
    client.setConnectTimeout(timeoutSeconds, TimeUnit.SECONDS);
    client.setConnectionPool(
        new ConnectionPool(
            // It's important that this number is greater than the `-j` parallelism,
            // as if it's too small, we'll overflow the reusable connection pool and
            // start spamming new connections.  While this isn't the best location,
            // the other current option is setting this wherever we construct a `Build`
            // object and have access to the `-j` argument.  However, since that is
            // created in several places leave it here for now.
            /* maxIdleConnections */ 200, /* keepAliveDurationMs */ TimeUnit.MINUTES.toMillis(5)));

    // For fetches, use a client with a read timeout.
    OkHttpClient fetchClient = client.clone();
    fetchClient.setReadTimeout(timeoutSeconds, TimeUnit.SECONDS);

    return new HttpArtifactCache(
        "http",
        fetchClient,
        client,
        url,
        doStore,
        projectFilesystem,
        buckEventBus,
        Hashing.crc32());
  }
Ejemplo n.º 3
0
  @Test
  public void compressionCanBeSetOnAPerFileBasisAndIsHonoured() throws IOException {
    // Create some input that can be compressed.
    String packageName = getClass().getPackage().getName().replace(".", "/");
    URL sample = Resources.getResource(packageName + "/sample-bytes.properties");
    byte[] input = Resources.toByteArray(sample);

    try (CustomZipOutputStream out = ZipOutputStreams.newOutputStream(output)) {
      CustomZipEntry entry = new CustomZipEntry("default");
      // Don't set the compression level. Should be the default.
      out.putNextEntry(entry);
      out.write(input);

      entry = new CustomZipEntry("stored");
      entry.setCompressionLevel(NO_COMPRESSION);
      byte[] bytes = "stored".getBytes();
      entry.setSize(bytes.length);
      entry.setCrc(Hashing.crc32().hashBytes(bytes).padToLong());
      out.putNextEntry(entry);

      out.write(bytes);

      entry = new CustomZipEntry("best");
      entry.setCompressionLevel(BEST_COMPRESSION);
      out.putNextEntry(entry);
      out.write(input);
    }

    try (ZipInputStream in = new ZipInputStream(Files.newInputStream(output))) {
      ZipEntry entry = in.getNextEntry();
      assertEquals("default", entry.getName());
      ByteStreams.copy(in, ByteStreams.nullOutputStream());
      long defaultCompressedSize = entry.getCompressedSize();
      assertNotEquals(entry.getCompressedSize(), entry.getSize());

      entry = in.getNextEntry();
      ByteStreams.copy(in, ByteStreams.nullOutputStream());
      assertEquals("stored", entry.getName());
      assertEquals(entry.getCompressedSize(), entry.getSize());

      entry = in.getNextEntry();
      ByteStreams.copy(in, ByteStreams.nullOutputStream());
      assertEquals("best", entry.getName());
      ByteStreams.copy(in, ByteStreams.nullOutputStream());
      assertThat(entry.getCompressedSize(), lessThan(defaultCompressedSize));
    }
  }
Ejemplo n.º 4
0
  @Test
  public void shouldBeAbleToSimplyStoreInputFilesWithoutCompressing() throws IOException {
    File reference = File.createTempFile("reference", ".zip");

    try (CustomZipOutputStream out = ZipOutputStreams.newOutputStream(output);
        ZipOutputStream ref = new ZipOutputStream(new FileOutputStream(reference))) {
      byte[] bytes = "cheese".getBytes();
      ZipEntry entry = new ZipEntry("example.txt");
      entry.setMethod(ZipEntry.STORED);
      entry.setTime(System.currentTimeMillis());
      entry.setSize(bytes.length);
      entry.setCrc(Hashing.crc32().hashBytes(bytes).padToLong());
      out.putNextEntry(entry);
      ref.putNextEntry(entry);
      out.write(bytes);
      ref.write(bytes);
    }

    byte[] seen = Files.readAllBytes(output);
    byte[] expected = Files.readAllBytes(reference.toPath());

    assertArrayEquals(expected, seen);
  }