private void addDirectoryToZipEntryList(
      File directory, String currentPath, ImmutableMap.Builder<File, ZipEntry> zipEntriesBuilder)
      throws IOException {
    Preconditions.checkNotNull(currentPath);

    for (File inputFile : directory.listFiles()) {
      String childPath = currentPath + (currentPath.isEmpty() ? "" : "/") + inputFile.getName();

      if (inputFile.isDirectory()) {
        addDirectoryToZipEntryList(inputFile, childPath, zipEntriesBuilder);
      } else {
        ZipEntry nextEntry = new ZipEntry(childPath);
        long fileLength = inputFile.length();
        if (fileLength > maxDeflatedBytes
            || EXTENSIONS_NOT_TO_DEFLATE.contains(Files.getFileExtension(inputFile.getName()))) {
          nextEntry.setMethod(ZipEntry.STORED);
          nextEntry.setCompressedSize(inputFile.length());
          nextEntry.setSize(inputFile.length());
          HashCode crc = ByteStreams.hash(Files.newInputStreamSupplier(inputFile), Hashing.crc32());
          nextEntry.setCrc(crc.padToLong());
        }

        zipEntriesBuilder.put(inputFile, nextEntry);
      }
    }
  }
  /** Test of getBlob method, of class FilesystemAsyncBlobStore. */
  public void testGetBlob() throws IOException {
    String blobKey = TestUtils.createRandomBlobKey();
    GetOptions options = null;
    Blob resultBlob;

    blobStore.createContainerInLocation(null, CONTAINER_NAME);

    resultBlob = blobStore.getBlob(CONTAINER_NAME, blobKey, options);
    assertNull(resultBlob, "Blob exists");

    // create blob
    TestUtils.createBlobsInContainer(CONTAINER_NAME, blobKey);

    resultBlob = blobStore.getBlob(CONTAINER_NAME, blobKey, options);

    assertNotNull(resultBlob, "Blob exists");
    // checks file content
    InputSupplier<FileInputStream> expectedFile =
        Files.newInputStreamSupplier(new File(TARGET_CONTAINER_NAME, blobKey));
    assertTrue(
        ByteStreams.equal(expectedFile, resultBlob.getPayload()),
        "Blob payload differs from file content");
    // metadata are verified in the test for blobMetadata, so no need to
    // perform a complete test here
    assertNotNull(resultBlob.getMetadata(), "Metadata null");
    MutableBlobMetadata metadata = resultBlob.getMetadata();
    assertEquals(blobKey, metadata.getName(), "Wrong blob metadata");
  }
  @Override
  public int execute(ExecutionContext context) {
    File inputDirectory = inputDirectoryPath.toFile();
    Preconditions.checkState(
        inputDirectory.exists() && inputDirectory.isDirectory(),
        "%s must be a directory.",
        inputDirectoryPath);

    try {
      ImmutableMap.Builder<File, ZipEntry> zipEntriesBuilder = ImmutableMap.builder();
      addDirectoryToZipEntryList(inputDirectory, "", zipEntriesBuilder);
      ImmutableMap<File, ZipEntry> zipEntries = zipEntriesBuilder.build();

      if (!zipEntries.isEmpty()) {
        try (CustomZipOutputStream outputStream =
            ZipOutputStreams.newOutputStream(outputZipPath.toFile())) {
          for (Map.Entry<File, ZipEntry> zipEntry : zipEntries.entrySet()) {
            outputStream.putNextEntry(zipEntry.getValue());
            ByteStreams.copy(Files.newInputStreamSupplier(zipEntry.getKey()), outputStream);
            outputStream.closeEntry();
          }
        }
      }
    } catch (IOException e) {
      e.printStackTrace(context.getStdErr());
      return 1;
    }
    return 0;
  }
  public void testWritePayloadOnFile() throws IOException {
    String blobKey;
    File sourceFile;
    FilePayload filePayload;

    blobKey = TestUtils.createRandomBlobKey("writePayload-", ".img");
    sourceFile = TestUtils.getImageForBlobPayload();
    filePayload = new FilePayload(sourceFile);
    Blob blob = storageStrategy.newBlob(blobKey);
    blob.setPayload(filePayload);
    // write files
    storageStrategy.putBlob(CONTAINER_NAME, blob);
    // verify that the files is equal
    File blobFullPath = new File(TARGET_CONTAINER_NAME, blobKey);
    InputSupplier<FileInputStream> expectedInput = Files.newInputStreamSupplier(sourceFile);
    InputSupplier<FileInputStream> actualInput = Files.newInputStreamSupplier(blobFullPath);
    assertTrue(ByteStreams.equal(expectedInput, actualInput), "Files are not equal");
  }
 public static File readResourceToTempFile(String resourceName) throws IOException {
   InputSupplier<? extends InputStream> inSupplier;
   try {
     URL resourceURL = Resources.getResource(GroupLensRecommender.class, resourceName);
     inSupplier = Resources.newInputStreamSupplier(resourceURL);
   } catch (IllegalArgumentException iae) {
     File resourceFile = new File("src/main/java" + resourceName);
     inSupplier = Files.newInputStreamSupplier(resourceFile);
   }
   File tempFile = File.createTempFile("taste", null);
   tempFile.deleteOnExit();
   Files.copy(inSupplier, tempFile);
   return tempFile;
 }
    @Override
    public void run() {
      Asset asset;
      while ((asset = filesLeft.poll()) != null) {
        for (int i = 1; i < MAX_TRIES + 1; i++) {
          try {
            File file = new File(getAssetsDir(), "objects/" + asset.path);

            // does exist? create
            if (!file.exists()) {
              file.getParentFile().mkdirs();
              file.createNewFile();
            }

            File localMc = new File(minecraftDir, asset.path);
            BufferedInputStream stream;

            // check for local copy
            if (localMc.exists() && Constants.hash(localMc, "SHA1").equals(asset.hash))
              // if so, copy
              stream = new BufferedInputStream(Files.newInputStreamSupplier(localMc).getInput());
            else
              // otherwise download
              stream =
                  new BufferedInputStream(
                      new URL(Constants.ASSETS_URL + "/" + asset.path).openStream());

            Files.write(ByteStreams.toByteArray(stream), file);
            stream.close();

            // check hash...
            String hash = Constants.hash(file, "SHA1");
            if (asset.hash.equals(hash)) break; // hashes are fine;
            else {
              file.delete();
              getLogger()
                  .error("download attempt " + i + " failed! : " + asset.hash + " != " + hash);
            }
          } catch (Exception e) {
            getLogger().error("Error downloading asset: " + asset.path);
            e.printStackTrace();
            if (!errored) errored = true;
          }
        }
      }
    }
Exemplo n.º 7
0
  public static void setupClasspath(Path distributedClassPath, Job job) throws IOException {
    String classpathProperty = System.getProperty("druid.hadoop.internal.classpath");
    if (classpathProperty == null) {
      classpathProperty = System.getProperty("java.class.path");
    }

    String[] jarFiles = classpathProperty.split(File.pathSeparator);

    final Configuration conf = job.getConfiguration();
    final FileSystem fs = distributedClassPath.getFileSystem(conf);

    if (fs instanceof LocalFileSystem) {
      return;
    }

    for (String jarFilePath : jarFiles) {
      File jarFile = new File(jarFilePath);
      if (jarFile.getName().endsWith(".jar")) {
        final Path hdfsPath = new Path(distributedClassPath, jarFile.getName());

        if (!existing.contains(hdfsPath)) {
          if (jarFile.getName().matches(".*SNAPSHOT(-selfcontained)?\\.jar$")
              || !fs.exists(hdfsPath)) {
            log.info("Uploading jar to path[%s]", hdfsPath);
            ByteStreams.copy(
                Files.newInputStreamSupplier(jarFile),
                new OutputSupplier<OutputStream>() {
                  @Override
                  public OutputStream getOutput() throws IOException {
                    return fs.create(hdfsPath);
                  }
                });
          }

          existing.add(hdfsPath);
        }

        DistributedCache.addFileToClassPath(hdfsPath, conf, fs);
      }
    }
  }
Exemplo n.º 8
0
 @Override
 public InputSupplier<? extends InputStream> getContentSupplier() {
   return Files.newInputStreamSupplier(file);
 }