/** * Sends upto maxChunks chunks of data. * * <p>When blockInPosition is >= 0, assumes 'out' is a {@link SocketOutputStream} and tries {@link * SocketOutputStream#transferToFully(FileChannel, long, int)} to send data (and updates * blockInPosition). */ private int sendChunks(ByteBuffer pkt, int maxChunks, OutputStream out) throws IOException { // Sends multiple chunks in one packet with a single write(). int len = (int) Math.min(endOffset - offset, (((long) bytesPerChecksum) * ((long) maxChunks))); int numChunks = (len + bytesPerChecksum - 1) / bytesPerChecksum; int packetLen = len + numChunks * checksumSize + 4; boolean lastDataPacket = offset + len == endOffset && len > 0; pkt.clear(); PacketHeader header = new PacketHeader(packetLen, offset, seqno, (len == 0), len); header.putInBuffer(pkt); int checksumOff = pkt.position(); int checksumLen = numChunks * checksumSize; byte[] buf = pkt.array(); if (checksumSize > 0 && checksumIn != null) { try { checksumIn.readFully(buf, checksumOff, checksumLen); } catch (IOException e) { LOG.warn( " Could not read or failed to veirfy checksum for data" + " at offset " + offset + " for block " + block + " got : " + StringUtils.stringifyException(e)); IOUtils.closeStream(checksumIn); checksumIn = null; if (corruptChecksumOk) { if (checksumOff < checksumLen) { // Just fill the array with zeros. Arrays.fill(buf, checksumOff, checksumLen, (byte) 0); } } else { throw e; } } // write in progress that we need to use to get last checksum if (lastDataPacket && lastChunkChecksum != null) { int start = checksumOff + checksumLen - checksumSize; byte[] updatedChecksum = lastChunkChecksum.getChecksum(); if (updatedChecksum != null) { System.arraycopy(updatedChecksum, 0, buf, start, checksumSize); } } } int dataOff = checksumOff + checksumLen; if (blockInPosition < 0) { // normal transfer IOUtils.readFully(blockIn, buf, dataOff, len); if (verifyChecksum) { int dOff = dataOff; int cOff = checksumOff; int dLeft = len; for (int i = 0; i < numChunks; i++) { checksum.reset(); int dLen = Math.min(dLeft, bytesPerChecksum); checksum.update(buf, dOff, dLen); if (!checksum.compare(buf, cOff)) { long failedPos = offset + len - dLeft; throw new ChecksumException("Checksum failed at " + failedPos, failedPos); } dLeft -= dLen; dOff += dLen; cOff += checksumSize; } } // writing is done below (mainly to handle IOException) } try { if (blockInPosition >= 0) { // use transferTo(). Checks on out and blockIn are already done. SocketOutputStream sockOut = (SocketOutputStream) out; // first write the packet sockOut.write(buf, 0, dataOff); // no need to flush. since we know out is not a buffered stream. sockOut.transferToFully(((FileInputStream) blockIn).getChannel(), blockInPosition, len); blockInPosition += len; } else { // normal transfer out.write(buf, 0, dataOff + len); } } catch (IOException e) { /* Exception while writing to the client. Connection closure from * the other end is mostly the case and we do not care much about * it. But other things can go wrong, especially in transferTo(), * which we do not want to ignore. * * The message parsing below should not be considered as a good * coding example. NEVER do it to drive a program logic. NEVER. * It was done here because the NIO throws an IOException for EPIPE. */ String ioem = e.getMessage(); if (!ioem.startsWith("Broken pipe") && !ioem.startsWith("Connection reset")) { LOG.error("BlockSender.sendChunks() exception: ", e); } throw ioeToSocketException(e); } if (throttler != null) { // rebalancing so throttle throttler.throttle(packetLen); } return len; }
BlockSender( Block block, long startOffset, long length, boolean corruptChecksumOk, boolean chunkOffsetOK, boolean verifyChecksum, DataNode datanode, String clientTraceFmt) throws IOException { try { this.block = block; synchronized (datanode.data) { this.replica = datanode.data.getReplica(block.getBlockId()); if (replica == null) { throw new ReplicaNotFoundException(block); } this.replicaVisibleLength = replica.getVisibleLength(); } long minEndOffset = startOffset + length; // if this is a write in progress ChunkChecksum chunkChecksum = null; if (replica instanceof ReplicaBeingWritten) { for (int i = 0; i < 30 && replica.getBytesOnDisk() < minEndOffset; i++) { try { Thread.sleep(100); } catch (InterruptedException ie) { throw new IOException(ie); } } long currentBytesOnDisk = replica.getBytesOnDisk(); if (currentBytesOnDisk < minEndOffset) { throw new IOException( String.format( "need %d bytes, but only %d bytes available", minEndOffset, currentBytesOnDisk)); } ReplicaInPipeline rip = (ReplicaInPipeline) replica; chunkChecksum = rip.getLastChecksumAndDataLen(); } if (replica.getGenerationStamp() < block.getGenerationStamp()) { throw new IOException( "replica.getGenerationStamp() < block.getGenerationStamp(), block=" + block + ", replica=" + replica); } if (replicaVisibleLength < 0) { throw new IOException( "The replica is not readable, block=" + block + ", replica=" + replica); } if (DataNode.LOG.isDebugEnabled()) { DataNode.LOG.debug("block=" + block + ", replica=" + replica); } this.chunkOffsetOK = chunkOffsetOK; this.corruptChecksumOk = corruptChecksumOk; this.verifyChecksum = verifyChecksum; // transferToFully() fails on 32 bit platforms for block sizes >= 2GB, // use normal transfer in those cases this.transferToAllowed = datanode.transferToAllowed && (!is32Bit || length < (long) Integer.MAX_VALUE); this.clientTraceFmt = clientTraceFmt; if (!corruptChecksumOk || datanode.data.metaFileExists(block)) { checksumIn = new DataInputStream( new BufferedInputStream(datanode.data.getMetaDataInputStream(block), BUFFER_SIZE)); // read and handle the common header here. For now just a version BlockMetadataHeader header = BlockMetadataHeader.readHeader(checksumIn); short version = header.getVersion(); if (version != FSDataset.METADATA_VERSION) { LOG.warn( "Wrong version (" + version + ") for metadata file for " + block + " ignoring ..."); } checksum = header.getChecksum(); } else { LOG.warn("Could not find metadata file for " + block); // This only decides the buffer size. Use BUFFER_SIZE? checksum = DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_NULL, 16 * 1024); } /* If bytesPerChecksum is very large, then the metadata file * is mostly corrupted. For now just truncate bytesPerchecksum to * blockLength. */ bytesPerChecksum = checksum.getBytesPerChecksum(); if (bytesPerChecksum > 10 * 1024 * 1024 && bytesPerChecksum > replicaVisibleLength) { checksum = DataChecksum.newDataChecksum( checksum.getChecksumType(), Math.max((int) replicaVisibleLength, 10 * 1024 * 1024)); bytesPerChecksum = checksum.getBytesPerChecksum(); } checksumSize = checksum.getChecksumSize(); if (length < 0) { length = replicaVisibleLength; } // end is either last byte on disk or the length for which we have a // checksum if (chunkChecksum != null) { endOffset = chunkChecksum.getDataLength(); } else { endOffset = replica.getBytesOnDisk(); } if (startOffset < 0 || startOffset > endOffset || (length + startOffset) > endOffset) { String msg = " Offset " + startOffset + " and length " + length + " don't match block " + block + " ( blockLen " + endOffset + " )"; LOG.warn(datanode.dnRegistration + ":sendBlock() : " + msg); throw new IOException(msg); } offset = (startOffset - (startOffset % bytesPerChecksum)); if (length >= 0) { // Make sure endOffset points to end of a checksumed chunk. long tmpLen = startOffset + length; if (tmpLen % bytesPerChecksum != 0) { tmpLen += (bytesPerChecksum - tmpLen % bytesPerChecksum); } if (tmpLen < endOffset) { // will use on-disk checksum here since the end is a stable chunk endOffset = tmpLen; } else if (chunkChecksum != null) { // in last chunk which is changing. flag that we need to use in-memory // checksum this.lastChunkChecksum = chunkChecksum; } } // seek to the right offsets if (offset > 0) { long checksumSkip = (offset / bytesPerChecksum) * checksumSize; // note blockInStream is seeked when created below if (checksumSkip > 0) { // Should we use seek() for checksum file as well? IOUtils.skipFully(checksumIn, checksumSkip); } } seqno = 0; if (DataNode.LOG.isDebugEnabled()) { DataNode.LOG.debug("replica=" + replica); } blockIn = datanode.data.getBlockInputStream(block, offset); // seek to offset } catch (IOException ioe) { IOUtils.closeStream(this); IOUtils.closeStream(blockIn); throw ioe; } }