示例#1
0
  /**
   * Returns longest common path for the given list of files.
   *
   * @param files list of files.
   * @return longest common path
   */
  private static String commonPathForFiles(final List<String> files) {
    if (files == null || files.isEmpty()) {
      return "";
    }

    final int total = files.size();
    final String[][] folders = new String[total][];
    int shortest = Integer.MAX_VALUE;
    for (int i = 0; i < total; i++) {
      final Path path = new Path(files.get(i));
      folders[i] = Path.getPathWithoutSchemeAndAuthority(path).toString().split(PATH_SEPARATOR);
      shortest = Math.min(shortest, folders[i].length);
    }

    int latest;
    out:
    for (latest = 0; latest < shortest; latest++) {
      final String current = folders[0][latest];
      for (int i = 1; i < folders.length; i++) {
        if (!current.equals(folders[i][latest])) {
          break out;
        }
      }
    }
    final Path path = new Path(files.get(0));
    final URI uri = path.toUri();
    final String pathString = buildPath(folders[0], latest);
    return new Path(uri.getScheme(), uri.getAuthority(), pathString).toString();
  }
    private FileSelection expandSelection(DrillFileSystem fs, FileSelection selection)
        throws IOException {
      if (metaDataFileExists(fs, selection.getFirstPath(fs))) {
        FileStatus metaRootDir = selection.getFirstPath(fs);
        Path metaFilePath = getMetadataPath(metaRootDir);

        // get the metadata for the directory by reading the metadata file
        ParquetTableMetadata_v1 metadata = Metadata.readBlockMeta(fs, metaFilePath.toString());
        List<String> fileNames = Lists.newArrayList();
        for (ParquetFileMetadata file : metadata.files) {
          fileNames.add(file.path);
        }
        // when creating the file selection, set the selection root in the form /a/b instead of
        // file:/a/b.  The reason is that the file names above have been created in the form
        // /a/b/c.parquet and the format of the selection root must match that of the file names
        // otherwise downstream operations such as partition pruning can break.
        Path metaRootPath = Path.getPathWithoutSchemeAndAuthority(metaRootDir.getPath());
        return new FileSelection(
            fileNames, metaRootPath.toString(), metadata /* save metadata for future use */);
      } else {
        // don't expand yet; ParquetGroupScan's metadata gathering operation
        // does that.
        return selection;
      }
    }
示例#3
0
 @Override
 public Path getPathWithoutSchemeAndAuthority(Path path) {
   return Path.getPathWithoutSchemeAndAuthority(path);
 }