void doProcess(Exchange exchange) throws Exception { Object body = exchange.getIn().getBody(); Object key = exchange.getIn().getHeader(HdfsHeader.KEY.name()); // must have ostream if (ostream == null) { ostream = setupHdfs(false); } boolean split = false; List<SplitStrategy> strategies = config.getSplitStrategies(); for (SplitStrategy splitStrategy : strategies) { split |= splitStrategy.getType().split(ostream, splitStrategy.value, this); } if (split) { if (ostream != null) { IOHelper.close(ostream, "output stream", log); } StringBuilder actualPath = newFileName(); ostream = HdfsOutputStream.createOutputStream(actualPath.toString(), config); } String path = ostream.getActualPath(); log.trace("Writing body to hdfs-file {}", path); ostream.append(key, body, exchange.getContext().getTypeConverter()); idle.set(false); // close if we do not have idle checker task to do this for us boolean close = scheduler == null; // but user may have a header to explict control the close Boolean closeHeader = exchange.getIn().getHeader(HdfsConstants.HDFS_CLOSE, Boolean.class); if (closeHeader != null) { close = closeHeader; } // if no idle checker then we need to explicit close the stream after usage if (close) { try { HdfsProducer.this.log.trace("Closing stream"); ostream.close(); ostream = null; } catch (IOException e) { // ignore } } log.debug("Wrote body to hdfs-file {}", path); }
@Override public long append( HdfsOutputStream hdfsostr, Object key, Object value, TypeConverter typeConverter) { try { Holder<Integer> valueSize = new Holder<Integer>(); Writable valueWritable = getWritable(value, typeConverter, valueSize); ((ArrayFile.Writer) hdfsostr.getOut()).append(valueWritable); return valueSize.value; } catch (Exception ex) { throw new RuntimeCamelException(ex); } }
@Override public long append( HdfsOutputStream hdfsostr, Object key, Object value, TypeConverter typeConverter) { InputStream is = null; try { is = typeConverter.convertTo(InputStream.class, value); return copyBytes( is, (FSDataOutputStream) hdfsostr.getOut(), HdfsConstants.DEFAULT_BUFFERSIZE, false); } catch (IOException ex) { throw new RuntimeCamelException(ex); } finally { IOHelper.close(is); } }
@Override public long append( HdfsOutputStream hdfsostr, Object key, Object value, TypeConverter typeConverter) { try { Holder<Integer> keySize = new Holder<Integer>(); Writable keyWritable = getWritable(key, typeConverter, keySize); Holder<Integer> valueSize = new Holder<Integer>(); Writable valueWritable = getWritable(value, typeConverter, valueSize); ((BloomMapFile.Writer) hdfsostr.getOut()) .append((WritableComparable<?>) keyWritable, valueWritable); return keySize.value + valueSize.value; } catch (Exception ex) { throw new RuntimeCamelException(ex); } }
private synchronized HdfsOutputStream setupHdfs(boolean onStartup) throws Exception { if (ostream != null) { return ostream; } StringBuilder actualPath = new StringBuilder(hdfsPath); if (config.getSplitStrategies().size() > 0) { actualPath = newFileName(); } // if we are starting up then log at info level, and if runtime then log at debug level to not // flood the log if (onStartup) { log.info( "Connecting to hdfs file-system {}:{}/{} (may take a while if connection is not available)", new Object[] {config.getHostName(), config.getPort(), actualPath.toString()}); } else { if (log.isDebugEnabled()) { log.debug( "Connecting to hdfs file-system {}:{}/{} (may take a while if connection is not available)", new Object[] {config.getHostName(), config.getPort(), actualPath.toString()}); } } HdfsOutputStream answer = HdfsOutputStream.createOutputStream(actualPath.toString(), config); if (onStartup) { log.info( "Connected to hdfs file-system {}:{}/{}", new Object[] {config.getHostName(), config.getPort(), actualPath.toString()}); } else { if (log.isDebugEnabled()) { log.debug( "Connected to hdfs file-system {}:{}/{}", new Object[] {config.getHostName(), config.getPort(), actualPath.toString()}); } } return answer; }
@Override public boolean split(HdfsOutputStream oldOstream, long value, HdfsProducer producer) { return oldOstream.getNumOfWrittenMessages() >= value; }