@Override
public Closeable createOutputStream(String hdfsPath, HdfsConfiguration configuration) {
try {
Closeable rout;
HdfsInfo hdfsInfo = HdfsInfoFactory.newHdfsInfo(hdfsPath);
Class<?> keyWritableClass = configuration.getKeyType().getWritableClass();
Class<?> valueWritableClass = configuration.getValueType().getWritableClass();
rout =
SequenceFile.createWriter(
hdfsInfo.getConf(),
Writer.file(hdfsInfo.getPath()),
Writer.keyClass(keyWritableClass),
Writer.valueClass(valueWritableClass),
Writer.bufferSize(configuration.getBufferSize()),
Writer.replication(configuration.getReplication()),
Writer.blockSize(configuration.getBlockSize()),
Writer.compression(
configuration.getCompressionType(),
configuration.getCompressionCodec().getCodec()),
Writer.progressable(
new Progressable() {
@Override
public void progress() {}
}),
Writer.metadata(new SequenceFile.Metadata()));
return rout;
} catch (IOException ex) {
throw new RuntimeCamelException(ex);
}
}
@Override
public Closeable createInputStream(String hdfsPath, HdfsConfiguration configuration) {
try {
Closeable rin;
HdfsInfo hdfsInfo = HdfsInfoFactory.newHdfsInfo(hdfsPath);
rin = new SequenceFile.Reader(hdfsInfo.getConf(), Reader.file(hdfsInfo.getPath()));
return rin;
} catch (IOException ex) {
throw new RuntimeCamelException(ex);
}
}
@Override
public Closeable createOutputStream(String hdfsPath, HdfsConfiguration configuration) {
try {
Closeable rout;
HdfsInfo hdfsInfo = HdfsInfoFactory.newHdfsInfo(hdfsPath);
if (!configuration.isAppend()) {
rout =
hdfsInfo
.getFileSystem()
.create(
hdfsInfo.getPath(),
configuration.isOverwrite(),
configuration.getBufferSize(),
configuration.getReplication(),
configuration.getBlockSize(),
new Progressable() {
@Override
public void progress() {}
});
} else {
rout =
hdfsInfo
.getFileSystem()
.append(
hdfsInfo.getPath(),
configuration.getBufferSize(),
new Progressable() {
@Override
public void progress() {}
});
}
return rout;
} catch (IOException ex) {
throw new RuntimeCamelException(ex);
}
}
@Override
public Closeable createInputStream(String hdfsPath, HdfsConfiguration configuration) {
try {
Closeable rin;
if (configuration.getFileSystemType().equals(HdfsFileSystemType.LOCAL)) {
HdfsInfo hdfsInfo = HdfsInfoFactory.newHdfsInfo(hdfsPath);
rin = hdfsInfo.getFileSystem().open(hdfsInfo.getPath());
} else {
rin = new FileInputStream(getHfdsFileToTmpFile(hdfsPath, configuration));
}
return rin;
} catch (IOException ex) {
throw new RuntimeCamelException(ex);
}
}