private File getHfdsFileToTmpFile(String hdfsPath, HdfsConfiguration configuration) {
try {
String fname = hdfsPath.substring(hdfsPath.lastIndexOf('/'));
File outputDest = File.createTempFile(fname, ".hdfs");
if (outputDest.exists()) {
outputDest.delete();
}
HdfsInfo hdfsInfo = HdfsInfoFactory.newHdfsInfo(hdfsPath);
FileSystem fileSystem = hdfsInfo.getFileSystem();
FileUtil.copy(fileSystem, new Path(hdfsPath), outputDest, false, fileSystem.getConf());
try {
FileUtil.copyMerge(
fileSystem, // src
new Path(hdfsPath),
FileSystem.getLocal(new Configuration()), // dest
new Path(outputDest.toURI()),
false,
fileSystem.getConf(),
null);
} catch (IOException e) {
return outputDest;
}
return new File(outputDest, fname);
} catch (IOException ex) {
throw new RuntimeCamelException(ex);
}
}
@SuppressWarnings("rawtypes")
@Override
public Closeable createOutputStream(String hdfsPath, HdfsConfiguration configuration) {
try {
Closeable rout;
HdfsInfo hdfsInfo = HdfsInfoFactory.newHdfsInfo(hdfsPath);
Class<? extends WritableComparable> keyWritableClass =
configuration.getKeyType().getWritableClass();
Class<? extends WritableComparable> valueWritableClass =
configuration.getValueType().getWritableClass();
rout =
new BloomMapFile.Writer(
hdfsInfo.getConf(),
new Path(hdfsPath),
org.apache.hadoop.io.MapFile.Writer.keyClass(keyWritableClass),
org.apache.hadoop.io.MapFile.Writer.valueClass(valueWritableClass),
org.apache.hadoop.io.MapFile.Writer.compression(
configuration.getCompressionType(),
configuration.getCompressionCodec().getCodec()),
org.apache.hadoop.io.MapFile.Writer.progressable(
new Progressable() {
@Override
public void progress() {}
}));
return rout;
} catch (IOException ex) {
throw new RuntimeCamelException(ex);
}
}
@Override
public Closeable createOutputStream(String hdfsPath, HdfsConfiguration configuration) {
try {
Closeable rout;
HdfsInfo hdfsInfo = HdfsInfoFactory.newHdfsInfo(hdfsPath);
Class<?> keyWritableClass = configuration.getKeyType().getWritableClass();
Class<?> valueWritableClass = configuration.getValueType().getWritableClass();
rout =
SequenceFile.createWriter(
hdfsInfo.getConf(),
Writer.file(hdfsInfo.getPath()),
Writer.keyClass(keyWritableClass),
Writer.valueClass(valueWritableClass),
Writer.bufferSize(configuration.getBufferSize()),
Writer.replication(configuration.getReplication()),
Writer.blockSize(configuration.getBlockSize()),
Writer.compression(
configuration.getCompressionType(),
configuration.getCompressionCodec().getCodec()),
Writer.progressable(
new Progressable() {
@Override
public void progress() {}
}),
Writer.metadata(new SequenceFile.Metadata()));
return rout;
} catch (IOException ex) {
throw new RuntimeCamelException(ex);
}
}
@Override
public Closeable createInputStream(String hdfsPath, HdfsConfiguration configuration) {
try {
Closeable rin;
HdfsInfo hdfsInfo = HdfsInfoFactory.newHdfsInfo(hdfsPath);
rin = new ArrayFile.Reader(hdfsInfo.getFileSystem(), hdfsPath, hdfsInfo.getConf());
return rin;
} catch (IOException ex) {
throw new RuntimeCamelException(ex);
}
}
@Override
public Closeable createInputStream(String hdfsPath, HdfsConfiguration configuration) {
try {
Closeable rin;
if (configuration.getFileSystemType().equals(HdfsFileSystemType.LOCAL)) {
HdfsInfo hdfsInfo = HdfsInfoFactory.newHdfsInfo(hdfsPath);
rin = hdfsInfo.getFileSystem().open(hdfsInfo.getPath());
} else {
rin = new FileInputStream(getHfdsFileToTmpFile(hdfsPath, configuration));
}
return rin;
} catch (IOException ex) {
throw new RuntimeCamelException(ex);
}
}
@Override
public Closeable createOutputStream(String hdfsPath, HdfsConfiguration configuration) {
try {
Closeable rout;
HdfsInfo hdfsInfo = HdfsInfoFactory.newHdfsInfo(hdfsPath);
if (!configuration.isAppend()) {
rout =
hdfsInfo
.getFileSystem()
.create(
hdfsInfo.getPath(),
configuration.isOverwrite(),
configuration.getBufferSize(),
configuration.getReplication(),
configuration.getBlockSize(),
new Progressable() {
@Override
public void progress() {}
});
} else {
rout =
hdfsInfo
.getFileSystem()
.append(
hdfsInfo.getPath(),
configuration.getBufferSize(),
new Progressable() {
@Override
public void progress() {}
});
}
return rout;
} catch (IOException ex) {
throw new RuntimeCamelException(ex);
}
}
@SuppressWarnings("rawtypes")
@Override
public Closeable createOutputStream(String hdfsPath, HdfsConfiguration configuration) {
try {
Closeable rout;
HdfsInfo hdfsInfo = HdfsInfoFactory.newHdfsInfo(hdfsPath);
Class<? extends WritableComparable> valueWritableClass =
configuration.getValueType().getWritableClass();
rout =
new ArrayFile.Writer(
hdfsInfo.getConf(),
hdfsInfo.getFileSystem(),
hdfsPath,
valueWritableClass,
configuration.getCompressionType(),
new Progressable() {
@Override
public void progress() {}
});
return rout;
} catch (IOException ex) {
throw new RuntimeCamelException(ex);
}
}