private void addDirectoryToZipEntryList(
File directory, String currentPath, ImmutableMap.Builder<File, ZipEntry> zipEntriesBuilder)
throws IOException {
Preconditions.checkNotNull(currentPath);
for (File inputFile : directory.listFiles()) {
String childPath = currentPath + (currentPath.isEmpty() ? "" : "/") + inputFile.getName();
if (inputFile.isDirectory()) {
addDirectoryToZipEntryList(inputFile, childPath, zipEntriesBuilder);
} else {
ZipEntry nextEntry = new ZipEntry(childPath);
long fileLength = inputFile.length();
if (fileLength > maxDeflatedBytes
|| EXTENSIONS_NOT_TO_DEFLATE.contains(Files.getFileExtension(inputFile.getName()))) {
nextEntry.setMethod(ZipEntry.STORED);
nextEntry.setCompressedSize(inputFile.length());
nextEntry.setSize(inputFile.length());
HashCode crc = ByteStreams.hash(Files.newInputStreamSupplier(inputFile), Hashing.crc32());
nextEntry.setCrc(crc.padToLong());
}
zipEntriesBuilder.put(inputFile, nextEntry);
}
}
}
private ArtifactCache createHttpArtifactCache(BuckEventBus buckEventBus) {
URL url;
try {
url = new URL(getValue("cache", "http_url").or(DEFAULT_HTTP_URL));
} catch (MalformedURLException e) {
throw new HumanReadableException(e, "Malformed [cache]http_url: %s", e.getMessage());
}
int timeoutSeconds =
Integer.parseInt(
getValue("cache", "http_timeout_seconds").or(DEFAULT_HTTP_CACHE_TIMEOUT_SECONDS));
boolean doStore = readCacheMode("http_mode", DEFAULT_HTTP_CACHE_MODE);
// Setup the defaut client to use.
OkHttpClient client = new OkHttpClient();
final String localhost = getLocalhost();
client
.networkInterceptors()
.add(
new Interceptor() {
@Override
public Response intercept(Chain chain) throws IOException {
return chain.proceed(
chain
.request()
.newBuilder()
.addHeader("X-BuckCache-User", System.getProperty("user.name", "<unknown>"))
.addHeader("X-BuckCache-Host", localhost)
.build());
}
});
client.setConnectTimeout(timeoutSeconds, TimeUnit.SECONDS);
client.setConnectionPool(
new ConnectionPool(
// It's important that this number is greater than the `-j` parallelism,
// as if it's too small, we'll overflow the reusable connection pool and
// start spamming new connections. While this isn't the best location,
// the other current option is setting this wherever we construct a `Build`
// object and have access to the `-j` argument. However, since that is
// created in several places leave it here for now.
/* maxIdleConnections */ 200, /* keepAliveDurationMs */ TimeUnit.MINUTES.toMillis(5)));
// For fetches, use a client with a read timeout.
OkHttpClient fetchClient = client.clone();
fetchClient.setReadTimeout(timeoutSeconds, TimeUnit.SECONDS);
return new HttpArtifactCache(
"http",
fetchClient,
client,
url,
doStore,
projectFilesystem,
buckEventBus,
Hashing.crc32());
}
@Test
public void compressionCanBeSetOnAPerFileBasisAndIsHonoured() throws IOException {
// Create some input that can be compressed.
String packageName = getClass().getPackage().getName().replace(".", "/");
URL sample = Resources.getResource(packageName + "/sample-bytes.properties");
byte[] input = Resources.toByteArray(sample);
try (CustomZipOutputStream out = ZipOutputStreams.newOutputStream(output)) {
CustomZipEntry entry = new CustomZipEntry("default");
// Don't set the compression level. Should be the default.
out.putNextEntry(entry);
out.write(input);
entry = new CustomZipEntry("stored");
entry.setCompressionLevel(NO_COMPRESSION);
byte[] bytes = "stored".getBytes();
entry.setSize(bytes.length);
entry.setCrc(Hashing.crc32().hashBytes(bytes).padToLong());
out.putNextEntry(entry);
out.write(bytes);
entry = new CustomZipEntry("best");
entry.setCompressionLevel(BEST_COMPRESSION);
out.putNextEntry(entry);
out.write(input);
}
try (ZipInputStream in = new ZipInputStream(Files.newInputStream(output))) {
ZipEntry entry = in.getNextEntry();
assertEquals("default", entry.getName());
ByteStreams.copy(in, ByteStreams.nullOutputStream());
long defaultCompressedSize = entry.getCompressedSize();
assertNotEquals(entry.getCompressedSize(), entry.getSize());
entry = in.getNextEntry();
ByteStreams.copy(in, ByteStreams.nullOutputStream());
assertEquals("stored", entry.getName());
assertEquals(entry.getCompressedSize(), entry.getSize());
entry = in.getNextEntry();
ByteStreams.copy(in, ByteStreams.nullOutputStream());
assertEquals("best", entry.getName());
ByteStreams.copy(in, ByteStreams.nullOutputStream());
assertThat(entry.getCompressedSize(), lessThan(defaultCompressedSize));
}
}
@Test
public void shouldBeAbleToSimplyStoreInputFilesWithoutCompressing() throws IOException {
File reference = File.createTempFile("reference", ".zip");
try (CustomZipOutputStream out = ZipOutputStreams.newOutputStream(output);
ZipOutputStream ref = new ZipOutputStream(new FileOutputStream(reference))) {
byte[] bytes = "cheese".getBytes();
ZipEntry entry = new ZipEntry("example.txt");
entry.setMethod(ZipEntry.STORED);
entry.setTime(System.currentTimeMillis());
entry.setSize(bytes.length);
entry.setCrc(Hashing.crc32().hashBytes(bytes).padToLong());
out.putNextEntry(entry);
ref.putNextEntry(entry);
out.write(bytes);
ref.write(bytes);
}
byte[] seen = Files.readAllBytes(output);
byte[] expected = Files.readAllBytes(reference.toPath());
assertArrayEquals(expected, seen);
}