void doProcess(Exchange exchange) throws Exception {
Object body = exchange.getIn().getBody();
Object key = exchange.getIn().getHeader(HdfsHeader.KEY.name());
// must have ostream
if (ostream == null) {
ostream = setupHdfs(false);
}
boolean split = false;
List<SplitStrategy> strategies = config.getSplitStrategies();
for (SplitStrategy splitStrategy : strategies) {
split |= splitStrategy.getType().split(ostream, splitStrategy.value, this);
}
if (split) {
if (ostream != null) {
IOHelper.close(ostream, "output stream", log);
}
StringBuilder actualPath = newFileName();
ostream = HdfsOutputStream.createOutputStream(actualPath.toString(), config);
}
String path = ostream.getActualPath();
log.trace("Writing body to hdfs-file {}", path);
ostream.append(key, body, exchange.getContext().getTypeConverter());
idle.set(false);
// close if we do not have idle checker task to do this for us
boolean close = scheduler == null;
// but user may have a header to explict control the close
Boolean closeHeader = exchange.getIn().getHeader(HdfsConstants.HDFS_CLOSE, Boolean.class);
if (closeHeader != null) {
close = closeHeader;
}
// if no idle checker then we need to explicit close the stream after usage
if (close) {
try {
HdfsProducer.this.log.trace("Closing stream");
ostream.close();
ostream = null;
} catch (IOException e) {
// ignore
}
}
log.debug("Wrote body to hdfs-file {}", path);
}
@Override
public long append(
HdfsOutputStream hdfsostr, Object key, Object value, TypeConverter typeConverter) {
try {
Holder<Integer> valueSize = new Holder<Integer>();
Writable valueWritable = getWritable(value, typeConverter, valueSize);
((ArrayFile.Writer) hdfsostr.getOut()).append(valueWritable);
return valueSize.value;
} catch (Exception ex) {
throw new RuntimeCamelException(ex);
}
}
@Override
public long append(
HdfsOutputStream hdfsostr, Object key, Object value, TypeConverter typeConverter) {
InputStream is = null;
try {
is = typeConverter.convertTo(InputStream.class, value);
return copyBytes(
is, (FSDataOutputStream) hdfsostr.getOut(), HdfsConstants.DEFAULT_BUFFERSIZE, false);
} catch (IOException ex) {
throw new RuntimeCamelException(ex);
} finally {
IOHelper.close(is);
}
}
@Override
public long append(
HdfsOutputStream hdfsostr, Object key, Object value, TypeConverter typeConverter) {
try {
Holder<Integer> keySize = new Holder<Integer>();
Writable keyWritable = getWritable(key, typeConverter, keySize);
Holder<Integer> valueSize = new Holder<Integer>();
Writable valueWritable = getWritable(value, typeConverter, valueSize);
((BloomMapFile.Writer) hdfsostr.getOut())
.append((WritableComparable<?>) keyWritable, valueWritable);
return keySize.value + valueSize.value;
} catch (Exception ex) {
throw new RuntimeCamelException(ex);
}
}
private synchronized HdfsOutputStream setupHdfs(boolean onStartup) throws Exception {
if (ostream != null) {
return ostream;
}
StringBuilder actualPath = new StringBuilder(hdfsPath);
if (config.getSplitStrategies().size() > 0) {
actualPath = newFileName();
}
// if we are starting up then log at info level, and if runtime then log at debug level to not
// flood the log
if (onStartup) {
log.info(
"Connecting to hdfs file-system {}:{}/{} (may take a while if connection is not available)",
new Object[] {config.getHostName(), config.getPort(), actualPath.toString()});
} else {
if (log.isDebugEnabled()) {
log.debug(
"Connecting to hdfs file-system {}:{}/{} (may take a while if connection is not available)",
new Object[] {config.getHostName(), config.getPort(), actualPath.toString()});
}
}
HdfsOutputStream answer = HdfsOutputStream.createOutputStream(actualPath.toString(), config);
if (onStartup) {
log.info(
"Connected to hdfs file-system {}:{}/{}",
new Object[] {config.getHostName(), config.getPort(), actualPath.toString()});
} else {
if (log.isDebugEnabled()) {
log.debug(
"Connected to hdfs file-system {}:{}/{}",
new Object[] {config.getHostName(), config.getPort(), actualPath.toString()});
}
}
return answer;
}
@Override
public boolean split(HdfsOutputStream oldOstream, long value, HdfsProducer producer) {
return oldOstream.getNumOfWrittenMessages() >= value;
}
