private List<String> getKeyFromSequenceFile(FileSystem fs, Path path, Configuration conf) throws Exception { List<String> list = new ArrayList<String>(); SequenceFile.Reader reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(path)); String next = (String) reader.next((String) null); while (next != null) { list.add(next); next = (String) reader.next((String) null); } reader.close(); return list; }
private Map<Text, CopyListingFileStatus> getListing(Path listingPath) throws Exception { SequenceFile.Reader reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(listingPath)); Text key = new Text(); CopyListingFileStatus value = new CopyListingFileStatus(); Map<Text, CopyListingFileStatus> values = new HashMap<>(); while (reader.next(key, value)) { values.put(key, value); key = new Text(); value = new CopyListingFileStatus(); } return values; }
public static void main(String[] args) throws Exception { // TODO Auto-generated method stub String mapUri = args[0]; Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(URI.create(mapUri), conf); Path map = new Path(mapUri); Path mapData = new Path(mapUri, MapFile.DATA_FILE_NAME); SequenceFile.Reader reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(mapData)); Class keyClass = reader.getKeyClass(); Class valueClass = reader.getValueClass(); reader.close(); long entries = MapFile.fix(fs, map, keyClass, valueClass, false, conf); System.out.printf("Created MapFile %s with %d entries\n", map, entries); }
private SequenceFile.Reader getListingFileReader(Configuration configuration) { final Path listingFilePath = getListingFilePath(configuration); try { final FileSystem fileSystem = listingFilePath.getFileSystem(configuration); if (!fileSystem.exists(listingFilePath)) throw new IllegalArgumentException("Listing file doesn't exist at: " + listingFilePath); return new SequenceFile.Reader(configuration, SequenceFile.Reader.file(listingFilePath)); } catch (IOException exception) { LOG.error("Couldn't find listing file at: " + listingFilePath, exception); throw new IllegalArgumentException( "Couldn't find listing-file at: " + listingFilePath, exception); } }
private List<byte[]> readResults(Path outputPath, Configuration config, FileSystem fs) throws IOException { List<byte[]> ret = new ArrayList<>(); for (RemoteIterator<LocatedFileStatus> it = fs.listFiles(outputPath, false); it.hasNext(); ) { Path p = it.next().getPath(); if (p.getName().equals("_SUCCESS")) { fs.delete(p, false); continue; } SequenceFile.Reader reader = new SequenceFile.Reader(config, SequenceFile.Reader.file(p)); LongWritable key = new LongWritable(); BytesWritable value = new BytesWritable(); while (reader.next(key, value)) { ret.add(value.copyBytes()); } reader.close(); fs.delete(p, false); } fs.delete(outputPath, false); if (LOG.isDebugEnabled()) { LOG.debug(outputPath + ": Returning " + ret.size()); } return ret; }