public void testGetMetaData() throws IOException {
   final SimulatedFSDataset fsdataset = getSimulatedFSDataset();
   ExtendedBlock b = new ExtendedBlock(bpid, 1, 5, 0);
   try {
     assertTrue(fsdataset.getMetaDataInputStream(b) == null);
     assertTrue("Expected an IO exception", false);
   } catch (IOException e) {
     // ok - as expected
   }
   addSomeBlocks(fsdataset); // Only need to add one but ....
   b = new ExtendedBlock(bpid, 1, 0, 0);
   InputStream metaInput = fsdataset.getMetaDataInputStream(b);
   DataInputStream metaDataInput = new DataInputStream(metaInput);
   short version = metaDataInput.readShort();
   assertEquals(BlockMetadataHeader.VERSION, version);
   DataChecksum checksum = DataChecksum.newDataChecksum(metaDataInput);
   assertEquals(DataChecksum.CHECKSUM_NULL, checksum.getChecksumType());
   assertEquals(0, checksum.getChecksumSize());
 }
예제 #2
0
  BlockSender(
      Block block,
      long startOffset,
      long length,
      boolean corruptChecksumOk,
      boolean chunkOffsetOK,
      boolean verifyChecksum,
      DataNode datanode,
      String clientTraceFmt)
      throws IOException {
    try {
      this.block = block;
      synchronized (datanode.data) {
        this.replica = datanode.data.getReplica(block.getBlockId());
        if (replica == null) {
          throw new ReplicaNotFoundException(block);
        }
        this.replicaVisibleLength = replica.getVisibleLength();
      }
      long minEndOffset = startOffset + length;
      // if this is a write in progress
      ChunkChecksum chunkChecksum = null;
      if (replica instanceof ReplicaBeingWritten) {
        for (int i = 0; i < 30 && replica.getBytesOnDisk() < minEndOffset; i++) {
          try {
            Thread.sleep(100);
          } catch (InterruptedException ie) {
            throw new IOException(ie);
          }
        }

        long currentBytesOnDisk = replica.getBytesOnDisk();

        if (currentBytesOnDisk < minEndOffset) {
          throw new IOException(
              String.format(
                  "need %d bytes, but only %d bytes available", minEndOffset, currentBytesOnDisk));
        }

        ReplicaInPipeline rip = (ReplicaInPipeline) replica;
        chunkChecksum = rip.getLastChecksumAndDataLen();
      }

      if (replica.getGenerationStamp() < block.getGenerationStamp()) {
        throw new IOException(
            "replica.getGenerationStamp() < block.getGenerationStamp(), block="
                + block
                + ", replica="
                + replica);
      }
      if (replicaVisibleLength < 0) {
        throw new IOException(
            "The replica is not readable, block=" + block + ", replica=" + replica);
      }
      if (DataNode.LOG.isDebugEnabled()) {
        DataNode.LOG.debug("block=" + block + ", replica=" + replica);
      }

      this.chunkOffsetOK = chunkOffsetOK;
      this.corruptChecksumOk = corruptChecksumOk;
      this.verifyChecksum = verifyChecksum;

      // transferToFully() fails on 32 bit platforms for block sizes >= 2GB,
      // use normal transfer in those cases
      this.transferToAllowed =
          datanode.transferToAllowed && (!is32Bit || length < (long) Integer.MAX_VALUE);
      this.clientTraceFmt = clientTraceFmt;

      if (!corruptChecksumOk || datanode.data.metaFileExists(block)) {
        checksumIn =
            new DataInputStream(
                new BufferedInputStream(datanode.data.getMetaDataInputStream(block), BUFFER_SIZE));

        // read and handle the common header here. For now just a version
        BlockMetadataHeader header = BlockMetadataHeader.readHeader(checksumIn);
        short version = header.getVersion();

        if (version != FSDataset.METADATA_VERSION) {
          LOG.warn(
              "Wrong version (" + version + ") for metadata file for " + block + " ignoring ...");
        }
        checksum = header.getChecksum();
      } else {
        LOG.warn("Could not find metadata file for " + block);
        // This only decides the buffer size. Use BUFFER_SIZE?
        checksum = DataChecksum.newDataChecksum(DataChecksum.CHECKSUM_NULL, 16 * 1024);
      }

      /* If bytesPerChecksum is very large, then the metadata file
       * is mostly corrupted. For now just truncate bytesPerchecksum to
       * blockLength.
       */
      bytesPerChecksum = checksum.getBytesPerChecksum();
      if (bytesPerChecksum > 10 * 1024 * 1024 && bytesPerChecksum > replicaVisibleLength) {
        checksum =
            DataChecksum.newDataChecksum(
                checksum.getChecksumType(), Math.max((int) replicaVisibleLength, 10 * 1024 * 1024));
        bytesPerChecksum = checksum.getBytesPerChecksum();
      }
      checksumSize = checksum.getChecksumSize();

      if (length < 0) {
        length = replicaVisibleLength;
      }

      // end is either last byte on disk or the length for which we have a
      // checksum
      if (chunkChecksum != null) {
        endOffset = chunkChecksum.getDataLength();
      } else {
        endOffset = replica.getBytesOnDisk();
      }

      if (startOffset < 0 || startOffset > endOffset || (length + startOffset) > endOffset) {
        String msg =
            " Offset "
                + startOffset
                + " and length "
                + length
                + " don't match block "
                + block
                + " ( blockLen "
                + endOffset
                + " )";
        LOG.warn(datanode.dnRegistration + ":sendBlock() : " + msg);
        throw new IOException(msg);
      }

      offset = (startOffset - (startOffset % bytesPerChecksum));
      if (length >= 0) {
        // Make sure endOffset points to end of a checksumed chunk.
        long tmpLen = startOffset + length;
        if (tmpLen % bytesPerChecksum != 0) {
          tmpLen += (bytesPerChecksum - tmpLen % bytesPerChecksum);
        }
        if (tmpLen < endOffset) {
          // will use on-disk checksum here since the end is a stable chunk
          endOffset = tmpLen;
        } else if (chunkChecksum != null) {
          // in last chunk which is changing. flag that we need to use in-memory
          // checksum
          this.lastChunkChecksum = chunkChecksum;
        }
      }

      // seek to the right offsets
      if (offset > 0) {
        long checksumSkip = (offset / bytesPerChecksum) * checksumSize;
        // note blockInStream is seeked when created below
        if (checksumSkip > 0) {
          // Should we use seek() for checksum file as well?
          IOUtils.skipFully(checksumIn, checksumSkip);
        }
      }
      seqno = 0;

      if (DataNode.LOG.isDebugEnabled()) {
        DataNode.LOG.debug("replica=" + replica);
      }

      blockIn = datanode.data.getBlockInputStream(block, offset); // seek to offset
    } catch (IOException ioe) {
      IOUtils.closeStream(this);
      IOUtils.closeStream(blockIn);
      throw ioe;
    }
  }