private List<String> getKeyFromSequenceFile(FileSystem fs, Path path, Configuration conf)
throws Exception {
List<String> list = new ArrayList<String>();
SequenceFile.Reader reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(path));
String next = (String) reader.next((String) null);
while (next != null) {
list.add(next);
next = (String) reader.next((String) null);
}
reader.close();
return list;
}
private Map<Text, CopyListingFileStatus> getListing(Path listingPath) throws Exception {
SequenceFile.Reader reader =
new SequenceFile.Reader(conf, SequenceFile.Reader.file(listingPath));
Text key = new Text();
CopyListingFileStatus value = new CopyListingFileStatus();
Map<Text, CopyListingFileStatus> values = new HashMap<>();
while (reader.next(key, value)) {
values.put(key, value);
key = new Text();
value = new CopyListingFileStatus();
}
return values;
}
public static void main(String[] args) throws Exception {
// TODO Auto-generated method stub
String mapUri = args[0];
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(URI.create(mapUri), conf);
Path map = new Path(mapUri);
Path mapData = new Path(mapUri, MapFile.DATA_FILE_NAME);
SequenceFile.Reader reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(mapData));
Class keyClass = reader.getKeyClass();
Class valueClass = reader.getValueClass();
reader.close();
long entries = MapFile.fix(fs, map, keyClass, valueClass, false, conf);
System.out.printf("Created MapFile %s with %d entries\n", map, entries);
}
private SequenceFile.Reader getListingFileReader(Configuration configuration) {
final Path listingFilePath = getListingFilePath(configuration);
try {
final FileSystem fileSystem = listingFilePath.getFileSystem(configuration);
if (!fileSystem.exists(listingFilePath))
throw new IllegalArgumentException("Listing file doesn't exist at: " + listingFilePath);
return new SequenceFile.Reader(configuration, SequenceFile.Reader.file(listingFilePath));
} catch (IOException exception) {
LOG.error("Couldn't find listing file at: " + listingFilePath, exception);
throw new IllegalArgumentException(
"Couldn't find listing-file at: " + listingFilePath, exception);
}
}
private List<byte[]> readResults(Path outputPath, Configuration config, FileSystem fs)
throws IOException {
List<byte[]> ret = new ArrayList<>();
for (RemoteIterator<LocatedFileStatus> it = fs.listFiles(outputPath, false); it.hasNext(); ) {
Path p = it.next().getPath();
if (p.getName().equals("_SUCCESS")) {
fs.delete(p, false);
continue;
}
SequenceFile.Reader reader = new SequenceFile.Reader(config, SequenceFile.Reader.file(p));
LongWritable key = new LongWritable();
BytesWritable value = new BytesWritable();
while (reader.next(key, value)) {
ret.add(value.copyBytes());
}
reader.close();
fs.delete(p, false);
}
fs.delete(outputPath, false);
if (LOG.isDebugEnabled()) {
LOG.debug(outputPath + ": Returning " + ret.size());
}
return ret;
}
