private void addDirectoryToZipEntryList( File directory, String currentPath, ImmutableMap.Builder<File, ZipEntry> zipEntriesBuilder) throws IOException { Preconditions.checkNotNull(currentPath); for (File inputFile : directory.listFiles()) { String childPath = currentPath + (currentPath.isEmpty() ? "" : "/") + inputFile.getName(); if (inputFile.isDirectory()) { addDirectoryToZipEntryList(inputFile, childPath, zipEntriesBuilder); } else { ZipEntry nextEntry = new ZipEntry(childPath); long fileLength = inputFile.length(); if (fileLength > maxDeflatedBytes || EXTENSIONS_NOT_TO_DEFLATE.contains(Files.getFileExtension(inputFile.getName()))) { nextEntry.setMethod(ZipEntry.STORED); nextEntry.setCompressedSize(inputFile.length()); nextEntry.setSize(inputFile.length()); HashCode crc = ByteStreams.hash(Files.newInputStreamSupplier(inputFile), Hashing.crc32()); nextEntry.setCrc(crc.padToLong()); } zipEntriesBuilder.put(inputFile, nextEntry); } } }
private ArtifactCache createHttpArtifactCache(BuckEventBus buckEventBus) { URL url; try { url = new URL(getValue("cache", "http_url").or(DEFAULT_HTTP_URL)); } catch (MalformedURLException e) { throw new HumanReadableException(e, "Malformed [cache]http_url: %s", e.getMessage()); } int timeoutSeconds = Integer.parseInt( getValue("cache", "http_timeout_seconds").or(DEFAULT_HTTP_CACHE_TIMEOUT_SECONDS)); boolean doStore = readCacheMode("http_mode", DEFAULT_HTTP_CACHE_MODE); // Setup the defaut client to use. OkHttpClient client = new OkHttpClient(); final String localhost = getLocalhost(); client .networkInterceptors() .add( new Interceptor() { @Override public Response intercept(Chain chain) throws IOException { return chain.proceed( chain .request() .newBuilder() .addHeader("X-BuckCache-User", System.getProperty("user.name", "<unknown>")) .addHeader("X-BuckCache-Host", localhost) .build()); } }); client.setConnectTimeout(timeoutSeconds, TimeUnit.SECONDS); client.setConnectionPool( new ConnectionPool( // It's important that this number is greater than the `-j` parallelism, // as if it's too small, we'll overflow the reusable connection pool and // start spamming new connections. While this isn't the best location, // the other current option is setting this wherever we construct a `Build` // object and have access to the `-j` argument. However, since that is // created in several places leave it here for now. /* maxIdleConnections */ 200, /* keepAliveDurationMs */ TimeUnit.MINUTES.toMillis(5))); // For fetches, use a client with a read timeout. OkHttpClient fetchClient = client.clone(); fetchClient.setReadTimeout(timeoutSeconds, TimeUnit.SECONDS); return new HttpArtifactCache( "http", fetchClient, client, url, doStore, projectFilesystem, buckEventBus, Hashing.crc32()); }
@Test public void compressionCanBeSetOnAPerFileBasisAndIsHonoured() throws IOException { // Create some input that can be compressed. String packageName = getClass().getPackage().getName().replace(".", "/"); URL sample = Resources.getResource(packageName + "/sample-bytes.properties"); byte[] input = Resources.toByteArray(sample); try (CustomZipOutputStream out = ZipOutputStreams.newOutputStream(output)) { CustomZipEntry entry = new CustomZipEntry("default"); // Don't set the compression level. Should be the default. out.putNextEntry(entry); out.write(input); entry = new CustomZipEntry("stored"); entry.setCompressionLevel(NO_COMPRESSION); byte[] bytes = "stored".getBytes(); entry.setSize(bytes.length); entry.setCrc(Hashing.crc32().hashBytes(bytes).padToLong()); out.putNextEntry(entry); out.write(bytes); entry = new CustomZipEntry("best"); entry.setCompressionLevel(BEST_COMPRESSION); out.putNextEntry(entry); out.write(input); } try (ZipInputStream in = new ZipInputStream(Files.newInputStream(output))) { ZipEntry entry = in.getNextEntry(); assertEquals("default", entry.getName()); ByteStreams.copy(in, ByteStreams.nullOutputStream()); long defaultCompressedSize = entry.getCompressedSize(); assertNotEquals(entry.getCompressedSize(), entry.getSize()); entry = in.getNextEntry(); ByteStreams.copy(in, ByteStreams.nullOutputStream()); assertEquals("stored", entry.getName()); assertEquals(entry.getCompressedSize(), entry.getSize()); entry = in.getNextEntry(); ByteStreams.copy(in, ByteStreams.nullOutputStream()); assertEquals("best", entry.getName()); ByteStreams.copy(in, ByteStreams.nullOutputStream()); assertThat(entry.getCompressedSize(), lessThan(defaultCompressedSize)); } }
@Test public void shouldBeAbleToSimplyStoreInputFilesWithoutCompressing() throws IOException { File reference = File.createTempFile("reference", ".zip"); try (CustomZipOutputStream out = ZipOutputStreams.newOutputStream(output); ZipOutputStream ref = new ZipOutputStream(new FileOutputStream(reference))) { byte[] bytes = "cheese".getBytes(); ZipEntry entry = new ZipEntry("example.txt"); entry.setMethod(ZipEntry.STORED); entry.setTime(System.currentTimeMillis()); entry.setSize(bytes.length); entry.setCrc(Hashing.crc32().hashBytes(bytes).padToLong()); out.putNextEntry(entry); ref.putNextEntry(entry); out.write(bytes); ref.write(bytes); } byte[] seen = Files.readAllBytes(output); byte[] expected = Files.readAllBytes(reference.toPath()); assertArrayEquals(expected, seen); }