private File getHfdsFileToTmpFile(String hdfsPath, HdfsConfiguration configuration) {
try {
String fname = hdfsPath.substring(hdfsPath.lastIndexOf('/'));
File outputDest = File.createTempFile(fname, ".hdfs");
if (outputDest.exists()) {
outputDest.delete();
}
HdfsInfo hdfsInfo = HdfsInfoFactory.newHdfsInfo(hdfsPath);
FileSystem fileSystem = hdfsInfo.getFileSystem();
FileUtil.copy(fileSystem, new Path(hdfsPath), outputDest, false, fileSystem.getConf());
try {
FileUtil.copyMerge(
fileSystem, // src
new Path(hdfsPath),
FileSystem.getLocal(new Configuration()), // dest
new Path(outputDest.toURI()),
false,
fileSystem.getConf(),
null);
} catch (IOException e) {
return outputDest;
}
return new File(outputDest, fname);
} catch (IOException ex) {
throw new RuntimeCamelException(ex);
}
}
@Override
public Closeable createInputStream(String hdfsPath, HdfsConfiguration configuration) {
try {
Closeable rin;
HdfsInfo hdfsInfo = HdfsInfoFactory.newHdfsInfo(hdfsPath);
rin = new ArrayFile.Reader(hdfsInfo.getFileSystem(), hdfsPath, hdfsInfo.getConf());
return rin;
} catch (IOException ex) {
throw new RuntimeCamelException(ex);
}
}
@Override
public Closeable createOutputStream(String hdfsPath, HdfsConfiguration configuration) {
try {
Closeable rout;
HdfsInfo hdfsInfo = HdfsInfoFactory.newHdfsInfo(hdfsPath);
if (!configuration.isAppend()) {
rout =
hdfsInfo
.getFileSystem()
.create(
hdfsInfo.getPath(),
configuration.isOverwrite(),
configuration.getBufferSize(),
configuration.getReplication(),
configuration.getBlockSize(),
new Progressable() {
@Override
public void progress() {}
});
} else {
rout =
hdfsInfo
.getFileSystem()
.append(
hdfsInfo.getPath(),
configuration.getBufferSize(),
new Progressable() {
@Override
public void progress() {}
});
}
return rout;
} catch (IOException ex) {
throw new RuntimeCamelException(ex);
}
}
@Override
public Closeable createInputStream(String hdfsPath, HdfsConfiguration configuration) {
try {
Closeable rin;
if (configuration.getFileSystemType().equals(HdfsFileSystemType.LOCAL)) {
HdfsInfo hdfsInfo = HdfsInfoFactory.newHdfsInfo(hdfsPath);
rin = hdfsInfo.getFileSystem().open(hdfsInfo.getPath());
} else {
rin = new FileInputStream(getHfdsFileToTmpFile(hdfsPath, configuration));
}
return rin;
} catch (IOException ex) {
throw new RuntimeCamelException(ex);
}
}
@SuppressWarnings("rawtypes")
@Override
public Closeable createOutputStream(String hdfsPath, HdfsConfiguration configuration) {
try {
Closeable rout;
HdfsInfo hdfsInfo = HdfsInfoFactory.newHdfsInfo(hdfsPath);
Class<? extends WritableComparable> valueWritableClass =
configuration.getValueType().getWritableClass();
rout =
new ArrayFile.Writer(
hdfsInfo.getConf(),
hdfsInfo.getFileSystem(),
hdfsPath,
valueWritableClass,
configuration.getCompressionType(),
new Progressable() {
@Override
public void progress() {}
});
return rout;
} catch (IOException ex) {
throw new RuntimeCamelException(ex);
}
}