/**
* The following test first creates a file. It verifies the block information from a datanode.
* Then, it updates the block with new information and verifies again.
*/
@Test
public void testBlockMetaDataInfo() throws Exception {
MiniDFSCluster cluster = null;
try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
cluster.waitActive();
// create a file
DistributedFileSystem dfs = (DistributedFileSystem) cluster.getFileSystem();
String filestr = "/foo";
Path filepath = new Path(filestr);
DFSTestUtil.createFile(dfs, filepath, 1024L, (short) 3, 0L);
assertTrue(dfs.exists(filepath));
// get block info
LocatedBlock locatedblock =
getLastLocatedBlock(DFSClientAdapter.getDFSClient(dfs).getNamenode(), filestr);
DatanodeInfo[] datanodeinfo = locatedblock.getLocations();
assertTrue(datanodeinfo.length > 0);
// connect to a data node
DataNode datanode = cluster.getDataNode(datanodeinfo[0].getIpcPort());
InterDatanodeProtocol idp =
DataNodeTestUtils.createInterDatanodeProtocolProxy(datanode, datanodeinfo[0], conf);
// stop block scanner, so we could compare lastScanTime
DataNodeTestUtils.shutdownBlockScanner(datanode);
// verify BlockMetaDataInfo
ExtendedBlock b = locatedblock.getBlock();
InterDatanodeProtocol.LOG.info("b=" + b + ", " + b.getClass());
checkMetaInfo(b, datanode);
long recoveryId = b.getGenerationStamp() + 1;
idp.initReplicaRecovery(new RecoveringBlock(b, locatedblock.getLocations(), recoveryId));
// verify updateBlock
ExtendedBlock newblock =
new ExtendedBlock(
b.getBlockPoolId(), b.getBlockId(), b.getNumBytes() / 2, b.getGenerationStamp() + 1);
idp.updateReplicaUnderRecovery(b, recoveryId, newblock.getNumBytes());
checkMetaInfo(newblock, datanode);
// Verify correct null response trying to init recovery for a missing block
ExtendedBlock badBlock = new ExtendedBlock("fake-pool", b.getBlockId(), 0, 0);
assertNull(
idp.initReplicaRecovery(
new RecoveringBlock(badBlock, locatedblock.getLocations(), recoveryId)));
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
private int getTrueReplication(MiniDFSCluster cluster, ExtendedBlock block) throws IOException {
int count = 0;
for (DataNode dn : cluster.getDataNodes()) {
if (DataNodeTestUtils.getFSDataset(dn)
.getStoredBlock(block.getBlockPoolId(), block.getBlockId())
!= null) {
count++;
}
}
return count;
}
/** Convert an ExtendedBlock to a Json map. */
private static Map<String, Object> toJsonMap(final ExtendedBlock extendedblock) {
if (extendedblock == null) {
return null;
}
final Map<String, Object> m = new TreeMap<String, Object>();
m.put("blockPoolId", extendedblock.getBlockPoolId());
m.put("blockId", extendedblock.getBlockId());
m.put("numBytes", extendedblock.getNumBytes());
m.put("generationStamp", extendedblock.getGenerationStamp());
return m;
}
/**
* try to access a block on a data node. If fails - throws exception
*
* @param datanode
* @param lblock
* @throws IOException
*/
private void accessBlock(DatanodeInfo datanode, LocatedBlock lblock) throws IOException {
InetSocketAddress targetAddr = null;
ExtendedBlock block = lblock.getBlock();
targetAddr = NetUtils.createSocketAddr(datanode.getXferAddr());
BlockReader blockReader =
new BlockReaderFactory(new DfsClientConf(conf))
.setInetSocketAddress(targetAddr)
.setBlock(block)
.setFileName(
BlockReaderFactory.getFileName(targetAddr, "test-blockpoolid", block.getBlockId()))
.setBlockToken(lblock.getBlockToken())
.setStartOffset(0)
.setLength(-1)
.setVerifyChecksum(true)
.setClientName("TestDataNodeVolumeFailure")
.setDatanodeInfo(datanode)
.setCachingStrategy(CachingStrategy.newDefaultStrategy())
.setClientCacheContext(ClientContext.getFromConf(conf))
.setConfiguration(conf)
.setTracer(FsTracer.get(conf))
.setRemotePeerFactory(
new RemotePeerFactory() {
@Override
public Peer newConnectedPeer(
InetSocketAddress addr,
Token<BlockTokenIdentifier> blockToken,
DatanodeID datanodeId)
throws IOException {
Peer peer = null;
Socket sock = NetUtils.getDefaultSocketFactory(conf).createSocket();
try {
sock.connect(addr, HdfsConstants.READ_TIMEOUT);
sock.setSoTimeout(HdfsConstants.READ_TIMEOUT);
peer = DFSUtilClient.peerFromSocket(sock);
} finally {
if (peer == null) {
IOUtils.closeSocket(sock);
}
}
return peer;
}
})
.build();
blockReader.close();
}
/**
* go to each block on the 2nd DataNode until it fails...
*
* @param path
* @param size
* @throws IOException
*/
private void triggerFailure(String path, long size) throws IOException {
NamenodeProtocols nn = cluster.getNameNodeRpc();
List<LocatedBlock> locatedBlocks = nn.getBlockLocations(path, 0, size).getLocatedBlocks();
for (LocatedBlock lb : locatedBlocks) {
DatanodeInfo dinfo = lb.getLocations()[1];
ExtendedBlock b = lb.getBlock();
try {
accessBlock(dinfo, lb);
} catch (IOException e) {
System.out.println(
"Failure triggered, on block: "
+ b.getBlockId()
+ "; corresponding volume should be removed by now");
break;
}
}
}
/** Get a BlockReader for the given block. */
public BlockReader getBlockReader(LocatedBlock testBlock, int offset, int lenToRead)
throws IOException {
InetSocketAddress targetAddr = null;
Socket sock = null;
ExtendedBlock block = testBlock.getBlock();
DatanodeInfo[] nodes = testBlock.getLocations();
targetAddr = NetUtils.createSocketAddr(nodes[0].getName());
sock = NetUtils.getDefaultSocketFactory(conf).createSocket();
sock.connect(targetAddr, HdfsServerConstants.READ_TIMEOUT);
sock.setSoTimeout(HdfsServerConstants.READ_TIMEOUT);
return BlockReaderFactory.newBlockReader(
new DFSClient.Conf(conf),
sock,
targetAddr.toString() + ":" + block.getBlockId(),
block,
testBlock.getBlockToken(),
offset,
lenToRead,
conf.getInt("io.file.buffer.size", 4096),
true,
"");
}
/**
* The following test first creates a file with a few blocks. It randomly truncates the replica of
* the last block stored in each datanode. Finally, it triggers block synchronization to
* synchronize all stored block.
*/
public void testBlockSynchronization() throws Exception {
final int ORG_FILE_SIZE = 3000;
Configuration conf = new HdfsConfiguration();
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
MiniDFSCluster cluster = null;
try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(5).build();
cluster.waitActive();
// create a file
DistributedFileSystem dfs = (DistributedFileSystem) cluster.getFileSystem();
String filestr = "/foo";
Path filepath = new Path(filestr);
DFSTestUtil.createFile(dfs, filepath, ORG_FILE_SIZE, REPLICATION_NUM, 0L);
assertTrue(dfs.exists(filepath));
DFSTestUtil.waitReplication(dfs, filepath, REPLICATION_NUM);
// get block info for the last block
LocatedBlock locatedblock =
TestInterDatanodeProtocol.getLastLocatedBlock(dfs.dfs.getNamenode(), filestr);
DatanodeInfo[] datanodeinfos = locatedblock.getLocations();
assertEquals(REPLICATION_NUM, datanodeinfos.length);
// connect to data nodes
DataNode[] datanodes = new DataNode[REPLICATION_NUM];
for (int i = 0; i < REPLICATION_NUM; i++) {
datanodes[i] = cluster.getDataNode(datanodeinfos[i].getIpcPort());
assertTrue(datanodes[i] != null);
}
// verify Block Info
ExtendedBlock lastblock = locatedblock.getBlock();
DataNode.LOG.info("newblocks=" + lastblock);
for (int i = 0; i < REPLICATION_NUM; i++) {
checkMetaInfo(lastblock, datanodes[i]);
}
DataNode.LOG.info("dfs.dfs.clientName=" + dfs.dfs.clientName);
cluster.getNameNodeRpc().append(filestr, dfs.dfs.clientName);
// expire lease to trigger block recovery.
waitLeaseRecovery(cluster);
Block[] updatedmetainfo = new Block[REPLICATION_NUM];
long oldSize = lastblock.getNumBytes();
lastblock =
TestInterDatanodeProtocol.getLastLocatedBlock(dfs.dfs.getNamenode(), filestr).getBlock();
long currentGS = lastblock.getGenerationStamp();
for (int i = 0; i < REPLICATION_NUM; i++) {
updatedmetainfo[i] =
DataNodeTestUtils.getFSDataset(datanodes[i])
.getStoredBlock(lastblock.getBlockPoolId(), lastblock.getBlockId());
assertEquals(lastblock.getBlockId(), updatedmetainfo[i].getBlockId());
assertEquals(oldSize, updatedmetainfo[i].getNumBytes());
assertEquals(currentGS, updatedmetainfo[i].getGenerationStamp());
}
// verify that lease recovery does not occur when namenode is in safemode
System.out.println("Testing that lease recovery cannot happen during safemode.");
filestr = "/foo.safemode";
filepath = new Path(filestr);
dfs.create(filepath, (short) 1);
cluster.getNameNodeRpc().setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER);
assertTrue(dfs.dfs.exists(filestr));
DFSTestUtil.waitReplication(dfs, filepath, (short) 1);
waitLeaseRecovery(cluster);
// verify that we still cannot recover the lease
LeaseManager lm = NameNodeAdapter.getLeaseManager(cluster.getNamesystem());
assertTrue("Found " + lm.countLease() + " lease, expected 1", lm.countLease() == 1);
cluster.getNameNodeRpc().setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
// try reading a block using a BlockReader directly
protected void tryRead(final Configuration conf, LocatedBlock lblock, boolean shouldSucceed) {
InetSocketAddress targetAddr = null;
IOException ioe = null;
BlockReader blockReader = null;
ExtendedBlock block = lblock.getBlock();
try {
DatanodeInfo[] nodes = lblock.getLocations();
targetAddr = NetUtils.createSocketAddr(nodes[0].getXferAddr());
blockReader =
new BlockReaderFactory(new DfsClientConf(conf))
.setFileName(
BlockReaderFactory.getFileName(
targetAddr, "test-blockpoolid", block.getBlockId()))
.setBlock(block)
.setBlockToken(lblock.getBlockToken())
.setInetSocketAddress(targetAddr)
.setStartOffset(0)
.setLength(-1)
.setVerifyChecksum(true)
.setClientName("TestBlockTokenWithDFS")
.setDatanodeInfo(nodes[0])
.setCachingStrategy(CachingStrategy.newDefaultStrategy())
.setClientCacheContext(ClientContext.getFromConf(conf))
.setConfiguration(conf)
.setTracer(FsTracer.get(conf))
.setRemotePeerFactory(
new RemotePeerFactory() {
@Override
public Peer newConnectedPeer(
InetSocketAddress addr,
Token<BlockTokenIdentifier> blockToken,
DatanodeID datanodeId)
throws IOException {
Peer peer = null;
Socket sock = NetUtils.getDefaultSocketFactory(conf).createSocket();
try {
sock.connect(addr, HdfsConstants.READ_TIMEOUT);
sock.setSoTimeout(HdfsConstants.READ_TIMEOUT);
peer = DFSUtilClient.peerFromSocket(sock);
} finally {
if (peer == null) {
IOUtils.closeSocket(sock);
}
}
return peer;
}
})
.build();
} catch (IOException ex) {
ioe = ex;
} finally {
if (blockReader != null) {
try {
blockReader.close();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}
if (shouldSucceed) {
Assert.assertNotNull(
"OP_READ_BLOCK: access token is invalid, " + "when it is expected to be valid",
blockReader);
} else {
Assert.assertNotNull(
"OP_READ_BLOCK: access token is valid, " + "when it is expected to be invalid", ioe);
Assert.assertTrue(
"OP_READ_BLOCK failed due to reasons other than access token: ",
ioe instanceof InvalidBlockTokenException);
}
}
/*
* XXX (ab) Bulk of this method is copied verbatim from {@link DFSClient}, which is
* bad. Both places should be refactored to provide a method to copy blocks
* around.
*/
private void copyBlock(DFSClient dfs, LocatedBlock lblock, OutputStream fos) throws Exception {
int failures = 0;
InetSocketAddress targetAddr = null;
TreeSet<DatanodeInfo> deadNodes = new TreeSet<DatanodeInfo>();
BlockReader blockReader = null;
ExtendedBlock block = lblock.getBlock();
while (blockReader == null) {
DatanodeInfo chosenNode;
try {
chosenNode = bestNode(dfs, lblock.getLocations(), deadNodes);
targetAddr = NetUtils.createSocketAddr(chosenNode.getXferAddr());
} catch (IOException ie) {
if (failures >= DFSConfigKeys.DFS_CLIENT_MAX_BLOCK_ACQUIRE_FAILURES_DEFAULT) {
throw new IOException("Could not obtain block " + lblock, ie);
}
LOG.info("Could not obtain block from any node: " + ie);
try {
Thread.sleep(10000);
} catch (InterruptedException iex) {
}
deadNodes.clear();
failures++;
continue;
}
try {
String file =
BlockReaderFactory.getFileName(targetAddr, block.getBlockPoolId(), block.getBlockId());
blockReader =
new BlockReaderFactory(dfs.getConf())
.setFileName(file)
.setBlock(block)
.setBlockToken(lblock.getBlockToken())
.setStartOffset(0)
.setLength(-1)
.setVerifyChecksum(true)
.setClientName("fsck")
.setDatanodeInfo(chosenNode)
.setInetSocketAddress(targetAddr)
.setCachingStrategy(CachingStrategy.newDropBehind())
.setClientCacheContext(dfs.getClientContext())
.setConfiguration(namenode.conf)
.setRemotePeerFactory(
new RemotePeerFactory() {
@Override
public Peer newConnectedPeer(InetSocketAddress addr) throws IOException {
Peer peer = null;
Socket s = NetUtils.getDefaultSocketFactory(conf).createSocket();
try {
s.connect(addr, HdfsServerConstants.READ_TIMEOUT);
s.setSoTimeout(HdfsServerConstants.READ_TIMEOUT);
peer =
TcpPeerServer.peerFromSocketAndKey(
s, namenode.getRpcServer().getDataEncryptionKey());
} finally {
if (peer == null) {
IOUtils.closeQuietly(s);
}
}
return peer;
}
})
.build();
} catch (IOException ex) {
// Put chosen node into dead list, continue
LOG.info("Failed to connect to " + targetAddr + ":" + ex);
deadNodes.add(chosenNode);
}
}
byte[] buf = new byte[1024];
int cnt = 0;
boolean success = true;
long bytesRead = 0;
try {
while ((cnt = blockReader.read(buf, 0, buf.length)) > 0) {
fos.write(buf, 0, cnt);
bytesRead += cnt;
}
if (bytesRead != block.getNumBytes()) {
throw new IOException(
"Recorded block size is "
+ block.getNumBytes()
+ ", but datanode returned "
+ bytesRead
+ " bytes");
}
} catch (Exception e) {
LOG.error("Error reading block", e);
success = false;
} finally {
blockReader.close();
}
if (!success) {
throw new Exception("Could not copy block data for " + lblock.getBlock());
}
}
/**
* Open a DataInputStream to a DataNode so that it can be read from. We get block ID and the IDs
* of the destinations at startup, from the namenode.
*/
private synchronized DatanodeInfo blockSeekTo(long target) throws IOException {
if (target >= getFileLength()) {
throw new IOException("Attempted to read past end of file");
}
// Will be getting a new BlockReader.
if (blockReader != null) {
closeBlockReader(blockReader);
blockReader = null;
}
//
// Connect to best DataNode for desired Block, with potential offset
//
DatanodeInfo chosenNode = null;
int refetchToken = 1; // only need to get a new access token once
boolean connectFailedOnce = false;
while (true) {
//
// Compute desired block
//
LocatedBlock targetBlock = getBlockAt(target, true);
assert (target == pos) : "Wrong postion " + pos + " expect " + target;
long offsetIntoBlock = target - targetBlock.getStartOffset();
DNAddrPair retval = chooseDataNode(targetBlock);
chosenNode = retval.info;
InetSocketAddress targetAddr = retval.addr;
try {
ExtendedBlock blk = targetBlock.getBlock();
Token<BlockTokenIdentifier> accessToken = targetBlock.getBlockToken();
blockReader =
getBlockReader(
targetAddr,
chosenNode,
src,
blk,
accessToken,
offsetIntoBlock,
blk.getNumBytes() - offsetIntoBlock,
buffersize,
verifyChecksum,
dfsClient.clientName);
if (connectFailedOnce) {
DFSClient.LOG.info(
"Successfully connected to " + targetAddr + " for block " + blk.getBlockId());
}
return chosenNode;
} catch (IOException ex) {
if (ex instanceof InvalidBlockTokenException && refetchToken > 0) {
DFSClient.LOG.info(
"Will fetch a new access token and retry, "
+ "access token was invalid when connecting to "
+ targetAddr
+ " : "
+ ex);
/*
* Get a new access token and retry. Retry is needed in 2 cases. 1)
* When both NN and DN re-started while DFSClient holding a cached
* access token. 2) In the case that NN fails to update its
* access key at pre-set interval (by a wide margin) and
* subsequently restarts. In this case, DN re-registers itself with
* NN and receives a new access key, but DN will delete the old
* access key from its memory since it's considered expired based on
* the estimated expiration date.
*/
refetchToken--;
fetchBlockAt(target);
} else {
connectFailedOnce = true;
DFSClient.LOG.warn(
"Failed to connect to "
+ targetAddr
+ " for block"
+ ", add to deadNodes and continue. "
+ ex,
ex);
// Put chosen node into dead list, continue
addToDeadNodes(chosenNode);
}
}
}
}
/** Test for {@link FsDatasetImpl#updateReplicaUnderRecovery(ExtendedBlock, long, long)} */
@Test
public void testUpdateReplicaUnderRecovery() throws IOException {
MiniDFSCluster cluster = null;
try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
cluster.waitActive();
String bpid = cluster.getNamesystem().getBlockPoolId();
// create a file
DistributedFileSystem dfs = (DistributedFileSystem) cluster.getFileSystem();
String filestr = "/foo";
Path filepath = new Path(filestr);
DFSTestUtil.createFile(dfs, filepath, 1024L, (short) 3, 0L);
// get block info
final LocatedBlock locatedblock =
getLastLocatedBlock(DFSClientAdapter.getDFSClient(dfs).getNamenode(), filestr);
final DatanodeInfo[] datanodeinfo = locatedblock.getLocations();
Assert.assertTrue(datanodeinfo.length > 0);
// get DataNode and FSDataset objects
final DataNode datanode = cluster.getDataNode(datanodeinfo[0].getIpcPort());
Assert.assertTrue(datanode != null);
// initReplicaRecovery
final ExtendedBlock b = locatedblock.getBlock();
final long recoveryid = b.getGenerationStamp() + 1;
final long newlength = b.getNumBytes() - 1;
final FsDatasetSpi<?> fsdataset = DataNodeTestUtils.getFSDataset(datanode);
final ReplicaRecoveryInfo rri =
fsdataset.initReplicaRecovery(new RecoveringBlock(b, null, recoveryid));
// check replica
final ReplicaInfo replica =
FsDatasetTestUtil.fetchReplicaInfo(fsdataset, bpid, b.getBlockId());
Assert.assertEquals(ReplicaState.RUR, replica.getState());
// check meta data before update
FsDatasetImpl.checkReplicaFiles(replica);
// case "THIS IS NOT SUPPOSED TO HAPPEN"
// with (block length) != (stored replica's on disk length).
{
// create a block with same id and gs but different length.
final ExtendedBlock tmp =
new ExtendedBlock(
b.getBlockPoolId(),
rri.getBlockId(),
rri.getNumBytes() - 1,
rri.getGenerationStamp());
try {
// update should fail
fsdataset.updateReplicaUnderRecovery(tmp, recoveryid, newlength);
Assert.fail();
} catch (IOException ioe) {
System.out.println("GOOD: getting " + ioe);
}
}
// update
final String storageID =
fsdataset.updateReplicaUnderRecovery(
new ExtendedBlock(b.getBlockPoolId(), rri), recoveryid, newlength);
assertTrue(storageID != null);
} finally {
if (cluster != null) cluster.shutdown();
}
}
public static void checkMetaInfo(ExtendedBlock b, DataNode dn) throws IOException {
Block metainfo =
DataNodeTestUtils.getFSDataset(dn).getStoredBlock(b.getBlockPoolId(), b.getBlockId());
Assert.assertEquals(b.getBlockId(), metainfo.getBlockId());
Assert.assertEquals(b.getNumBytes(), metainfo.getNumBytes());
}