/** Verify the first block of the file is corrupted (for all its replica). */
private void verifyFirstBlockCorrupted(Path filePath, boolean isCorrupted)
throws AccessControlException, FileNotFoundException, UnresolvedLinkException, IOException {
final LocatedBlocks locatedBlocks =
dfs.dfs.getNamenode().getBlockLocations(filePath.toUri().getPath(), 0, Long.MAX_VALUE);
final LocatedBlock firstLocatedBlock = locatedBlocks.get(0);
Assert.assertEquals(isCorrupted, firstLocatedBlock.isCorrupt());
}
public void reorderBlocks(Configuration conf, LocatedBlocks lbs, String src)
throws IOException {
ServerName sn = AbstractFSWALProvider.getServerNameFromWALDirectoryName(conf, src);
if (sn == null) {
// It's not an WAL
return;
}
// Ok, so it's an WAL
String hostName = sn.getHostname();
if (LOG.isTraceEnabled()) {
LOG.trace(src + " is an WAL file, so reordering blocks, last hostname will be:" + hostName);
}
// Just check for all blocks
for (LocatedBlock lb : lbs.getLocatedBlocks()) {
DatanodeInfo[] dnis = lb.getLocations();
if (dnis != null && dnis.length > 1) {
boolean found = false;
for (int i = 0; i < dnis.length - 1 && !found; i++) {
if (hostName.equals(dnis[i].getHostName())) {
// advance the other locations by one and put this one at the last place.
DatanodeInfo toLast = dnis[i];
System.arraycopy(dnis, i + 1, dnis, i, dnis.length - i - 1);
dnis[dnis.length - 1] = toLast;
found = true;
}
}
}
}
}
private void checkForCorruptOpenFiles(FileStatus file, List<FileStatus> corruptFiles)
throws IOException {
String filePath = file.getPath().toUri().getPath();
if (file.isDir()) {
for (FileStatus fileStatus : nn.namesystem.dir.getListing(filePath)) {
checkForCorruptOpenFiles(fileStatus, corruptFiles);
}
} else {
LeaseManager.Lease lease = nn.getNamesystem().leaseManager.getLeaseByPath(filePath);
// Condition:
// 1. lease has expired hard limit
// 2. the file is open for write
// 3. the last block has 0 locations
if (lease != null && lease.expiredHardLimit()) {
LocatedBlocks blocks = nn.getNamesystem().getBlockLocations(filePath, 0, file.getLen());
List<LocatedBlock> locatedBlockList = blocks.getLocatedBlocks();
LocatedBlock lastBlock = locatedBlockList.get(locatedBlockList.size() - 1);
if (blocks.isUnderConstruction() && lastBlock.getLocations().length == 0) {
corruptFiles.add(file);
}
}
}
}
/**
* File partially fit in RamDisk after eviction. RamDisk can fit 2 blocks. Write a file with 5
* blocks. Expect 2 or less blocks are on RamDisk and 3 or more on disk.
*
* @throws IOException
*/
@Test
public void testFallbackToDiskPartial() throws IOException, InterruptedException {
startUpCluster(true, 2);
final String METHOD_NAME = GenericTestUtils.getMethodName();
Path path = new Path("/" + METHOD_NAME + ".dat");
makeTestFile(path, BLOCK_SIZE * 5, true);
// Sleep for a short time to allow the lazy writer thread to do its job
Thread.sleep(6 * LAZY_WRITER_INTERVAL_SEC * 1000);
triggerBlockReport();
int numBlocksOnRamDisk = 0;
int numBlocksOnDisk = 0;
long fileLength = client.getFileInfo(path.toString()).getLen();
LocatedBlocks locatedBlocks = client.getLocatedBlocks(path.toString(), 0, fileLength);
for (LocatedBlock locatedBlock : locatedBlocks.getLocatedBlocks()) {
if (locatedBlock.getStorageTypes()[0] == RAM_DISK) {
numBlocksOnRamDisk++;
} else if (locatedBlock.getStorageTypes()[0] == DEFAULT) {
numBlocksOnDisk++;
}
}
// Since eviction is asynchronous, depending on the timing of eviction
// wrt writes, we may get 2 or less blocks on RAM disk.
assert (numBlocksOnRamDisk <= 2);
assert (numBlocksOnDisk >= 3);
}
/**
* Make sure at least one non-transient volume has a saved copy of the replica. An infinite loop
* is used to ensure the async lazy persist tasks are completely done before verification. Caller
* of ensureLazyPersistBlocksAreSaved expects either a successful pass or timeout failure.
*/
protected final void ensureLazyPersistBlocksAreSaved(LocatedBlocks locatedBlocks)
throws IOException, InterruptedException {
final String bpid = cluster.getNamesystem().getBlockPoolId();
List<? extends FsVolumeSpi> volumes = cluster.getDataNodes().get(0).getFSDataset().getVolumes();
final Set<Long> persistedBlockIds = new HashSet<Long>();
while (persistedBlockIds.size() < locatedBlocks.getLocatedBlocks().size()) {
// Take 1 second sleep before each verification iteration
Thread.sleep(1000);
for (LocatedBlock lb : locatedBlocks.getLocatedBlocks()) {
for (FsVolumeSpi v : volumes) {
if (v.isTransientStorage()) {
continue;
}
FsVolumeImpl volume = (FsVolumeImpl) v;
File lazyPersistDir = volume.getBlockPoolSlice(bpid).getLazypersistDir();
long blockId = lb.getBlock().getBlockId();
File targetDir = DatanodeUtil.idToBlockDir(lazyPersistDir, blockId);
File blockFile = new File(targetDir, lb.getBlock().getBlockName());
if (blockFile.exists()) {
// Found a persisted copy for this block and added to the Set
persistedBlockIds.add(blockId);
}
}
}
}
// We should have found a persisted copy for each located block.
assertThat(persistedBlockIds.size(), is(locatedBlocks.getLocatedBlocks().size()));
}
/**
* Get blocks in the specified range. Includes only the complete blocks. Fetch them from the
* namenode if not cached.
*/
private synchronized List<LocatedBlock> getFinalizedBlockRange(long offset, long length)
throws IOException {
assert (locatedBlocks != null) : "locatedBlocks is null";
List<LocatedBlock> blockRange = new ArrayList<LocatedBlock>();
// search cached blocks first
int blockIdx = locatedBlocks.findBlock(offset);
if (blockIdx < 0) { // block is not cached
blockIdx = LocatedBlocks.getInsertIndex(blockIdx);
}
long remaining = length;
long curOff = offset;
while (remaining > 0) {
LocatedBlock blk = null;
if (blockIdx < locatedBlocks.locatedBlockCount()) blk = locatedBlocks.get(blockIdx);
if (blk == null || curOff < blk.getStartOffset()) {
LocatedBlocks newBlocks;
newBlocks = DFSClient.callGetBlockLocations(dfsClient.namenode, src, curOff, remaining);
locatedBlocks.insertRange(blockIdx, newBlocks.getLocatedBlocks());
continue;
}
assert curOff >= blk.getStartOffset() : "Block not found";
blockRange.add(blk);
long bytesRead = blk.getStartOffset() + blk.getBlockSize() - curOff;
remaining -= bytesRead;
curOff += bytesRead;
blockIdx++;
}
return blockRange;
}
static ClientDatanodeProtocolPB createClientDatanodeProtocolProxy(
DatanodeID datanodeid,
Configuration conf,
int socketTimeout,
boolean connectToDnViaHostname,
LocatedBlock locatedBlock)
throws IOException {
final String dnAddr = datanodeid.getIpcAddr(connectToDnViaHostname);
InetSocketAddress addr = NetUtils.createSocketAddr(dnAddr);
if (LOG.isDebugEnabled()) {
LOG.debug("Connecting to datanode " + dnAddr + " addr=" + addr);
}
// Since we're creating a new UserGroupInformation here, we know that no
// future RPC proxies will be able to re-use the same connection. And
// usages of this proxy tend to be one-off calls.
//
// This is a temporary fix: callers should really achieve this by using
// RPC.stopProxy() on the resulting object, but this is currently not
// working in trunk. See the discussion on HDFS-1965.
Configuration confWithNoIpcIdle = new Configuration(conf);
confWithNoIpcIdle.setInt(
CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY, 0);
UserGroupInformation ticket =
UserGroupInformation.createRemoteUser(locatedBlock.getBlock().getLocalBlock().toString());
ticket.addToken(locatedBlock.getBlockToken());
return createClientDatanodeProtocolProxy(
addr, ticket, confWithNoIpcIdle, NetUtils.getDefaultSocketFactory(conf), socketTimeout);
}
/**
* Verify the number of corrupted block replicas by fetching the block location from name node.
*/
private void verifyCorruptedBlockCount(Path filePath, int expectedReplicas)
throws AccessControlException, FileNotFoundException, UnresolvedLinkException, IOException {
final LocatedBlocks lBlocks =
dfs.dfs.getNamenode().getBlockLocations(filePath.toUri().getPath(), 0, Long.MAX_VALUE);
// we expect only the first block of the file is used for this test
LocatedBlock firstLocatedBlock = lBlocks.get(0);
Assert.assertEquals(expectedReplicas, firstLocatedBlock.getLocations().length);
}
/**
* Setup a {@link MiniDFSCluster}. Create a block with both {@link State#NORMAL} and {@link
* State#READ_ONLY_SHARED} replicas.
*/
@Before
public void setup() throws IOException, InterruptedException {
conf = new HdfsConfiguration();
SimulatedFSDataset.setFactory(conf);
Configuration[] overlays = new Configuration[NUM_DATANODES];
for (int i = 0; i < overlays.length; i++) {
overlays[i] = new Configuration();
if (i == RO_NODE_INDEX) {
overlays[i].setEnum(
SimulatedFSDataset.CONFIG_PROPERTY_STATE,
i == RO_NODE_INDEX ? READ_ONLY_SHARED : NORMAL);
}
}
cluster =
new MiniDFSCluster.Builder(conf)
.numDataNodes(NUM_DATANODES)
.dataNodeConfOverlays(overlays)
.build();
fs = cluster.getFileSystem();
blockManager = cluster.getNameNode().getNamesystem().getBlockManager();
datanodeManager = blockManager.getDatanodeManager();
client =
new DFSClient(
new InetSocketAddress("localhost", cluster.getNameNodePort()),
cluster.getConfiguration(0));
for (int i = 0; i < NUM_DATANODES; i++) {
DataNode dataNode = cluster.getDataNodes().get(i);
validateStorageState(
BlockManagerTestUtil.getStorageReportsForDatanode(
datanodeManager.getDatanode(dataNode.getDatanodeId())),
i == RO_NODE_INDEX ? READ_ONLY_SHARED : NORMAL);
}
// Create a 1 block file
DFSTestUtil.createFile(fs, PATH, BLOCK_SIZE, BLOCK_SIZE, BLOCK_SIZE, (short) 1, seed);
LocatedBlock locatedBlock = getLocatedBlock();
extendedBlock = locatedBlock.getBlock();
block = extendedBlock.getLocalBlock();
assertThat(locatedBlock.getLocations().length, is(1));
normalDataNode = locatedBlock.getLocations()[0];
readOnlyDataNode =
datanodeManager.getDatanode(cluster.getDataNodes().get(RO_NODE_INDEX).getDatanodeId());
assertThat(normalDataNode, is(not(readOnlyDataNode)));
validateNumberReplicas(1);
// Inject the block into the datanode with READ_ONLY_SHARED storage
cluster.injectBlocks(0, RO_NODE_INDEX, Collections.singleton(block));
// There should now be 2 *locations* for the block
// Must wait until the NameNode has processed the block report for the injected blocks
waitForLocations(2);
}
private DNAddrPair chooseDataNode(LocatedBlock block) throws IOException {
while (true) {
DatanodeInfo[] nodes = block.getLocations();
try {
DatanodeInfo chosenNode = bestNode(nodes, deadNodes);
InetSocketAddress targetAddr = NetUtils.createSocketAddr(chosenNode.getXferAddr());
return new DNAddrPair(chosenNode, targetAddr);
} catch (IOException ie) {
String blockInfo = block.getBlock() + " file=" + src;
if (failures >= dfsClient.getMaxBlockAcquireFailures()) {
throw new BlockMissingException(
src, "Could not obtain block: " + blockInfo, block.getStartOffset());
}
if (nodes == null || nodes.length == 0) {
DFSClient.LOG.info("No node available for block: " + blockInfo);
}
DFSClient.LOG.info(
"Could not obtain block "
+ block.getBlock()
+ " from any node: "
+ ie
+ ". Will get new block locations from namenode and retry...");
try {
// Introducing a random factor to the wait time before another retry.
// The wait time is dependent on # of failures and a random factor.
// At the first time of getting a BlockMissingException, the wait time
// is a random number between 0..3000 ms. If the first retry
// still fails, we will wait 3000 ms grace period before the 2nd retry.
// Also at the second retry, the waiting window is expanded to 6000 ms
// alleviating the request rate from the server. Similarly the 3rd retry
// will wait 6000ms grace period before retry and the waiting window is
// expanded to 9000ms.
double waitTime =
timeWindow * failures
+ // grace period for the last round of attempt
timeWindow
* (failures + 1)
* DFSUtil.getRandom().nextDouble(); // expanding time window for each failure
DFSClient.LOG.warn(
"DFS chooseDataNode: got # "
+ (failures + 1)
+ " IOException, will wait for "
+ waitTime
+ " msec.");
Thread.sleep((long) waitTime);
} catch (InterruptedException iex) {
}
deadNodes.clear(); // 2nd option is to remove only nodes[blockId]
openInfo();
block = getBlockAt(block.getStartOffset(), false);
failures++;
continue;
}
}
}
/**
* The following test first creates a file. It verifies the block information from a datanode.
* Then, it updates the block with new information and verifies again.
*/
@Test
public void testBlockMetaDataInfo() throws Exception {
MiniDFSCluster cluster = null;
try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
cluster.waitActive();
// create a file
DistributedFileSystem dfs = (DistributedFileSystem) cluster.getFileSystem();
String filestr = "/foo";
Path filepath = new Path(filestr);
DFSTestUtil.createFile(dfs, filepath, 1024L, (short) 3, 0L);
assertTrue(dfs.exists(filepath));
// get block info
LocatedBlock locatedblock =
getLastLocatedBlock(DFSClientAdapter.getDFSClient(dfs).getNamenode(), filestr);
DatanodeInfo[] datanodeinfo = locatedblock.getLocations();
assertTrue(datanodeinfo.length > 0);
// connect to a data node
DataNode datanode = cluster.getDataNode(datanodeinfo[0].getIpcPort());
InterDatanodeProtocol idp =
DataNodeTestUtils.createInterDatanodeProtocolProxy(datanode, datanodeinfo[0], conf);
// stop block scanner, so we could compare lastScanTime
DataNodeTestUtils.shutdownBlockScanner(datanode);
// verify BlockMetaDataInfo
ExtendedBlock b = locatedblock.getBlock();
InterDatanodeProtocol.LOG.info("b=" + b + ", " + b.getClass());
checkMetaInfo(b, datanode);
long recoveryId = b.getGenerationStamp() + 1;
idp.initReplicaRecovery(new RecoveringBlock(b, locatedblock.getLocations(), recoveryId));
// verify updateBlock
ExtendedBlock newblock =
new ExtendedBlock(
b.getBlockPoolId(), b.getBlockId(), b.getNumBytes() / 2, b.getGenerationStamp() + 1);
idp.updateReplicaUnderRecovery(b, recoveryId, newblock.getNumBytes());
checkMetaInfo(newblock, datanode);
// Verify correct null response trying to init recovery for a missing block
ExtendedBlock badBlock = new ExtendedBlock("fake-pool", b.getBlockId(), 0, 0);
assertNull(
idp.initReplicaRecovery(
new RecoveringBlock(badBlock, locatedblock.getLocations(), recoveryId)));
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
private void addBlocks(String fileName, String clientName) throws IOException {
for (int jdx = 0; jdx < blocksPerFile; jdx++) {
LocatedBlock loc = nameNode.addBlock(fileName, clientName);
for (DatanodeInfo dnInfo : loc.getLocations()) {
int dnIdx = Arrays.binarySearch(datanodes, dnInfo.getName());
datanodes[dnIdx].addBlock(loc.getBlock());
nameNode.blockReceived(
datanodes[dnIdx].dnRegistration, new Block[] {loc.getBlock()}, new String[] {""});
}
}
}
/** Test constructing LocatedBlock with null cachedLocs */
@Test
public void testLocatedBlockConstructorWithNullCachedLocs() {
DatanodeInfo d = DFSTestUtil.getLocalDatanodeInfo();
DatanodeInfo[] ds = new DatanodeInfo[1];
ds[0] = d;
ExtendedBlock b1 = new ExtendedBlock("bpid", 1, 1, 1);
LocatedBlock l1 = new LocatedBlock(b1, ds, null, null, 0, false, null);
final DatanodeInfo[] cachedLocs = l1.getCachedLocations();
assertTrue(cachedLocs.length == 0);
}
/**
* TC11: Racing rename
*
* @throws IOException an exception might be thrown
*/
public void testTC11() throws Exception {
final Path p = new Path("/TC11/foo");
System.out.println("p=" + p);
// a. Create file and write one block of data. Close file.
final int len1 = (int) BLOCK_SIZE;
{
FSDataOutputStream out = fs.create(p, false, buffersize, REPLICATION, BLOCK_SIZE);
AppendTestUtil.write(out, 0, len1);
out.close();
}
// b. Reopen file in "append" mode. Append half block of data.
FSDataOutputStream out = fs.append(p);
final int len2 = (int) BLOCK_SIZE / 2;
AppendTestUtil.write(out, len1, len2);
out.hflush();
// c. Rename file to file.new.
final Path pnew = new Path(p + ".new");
assertTrue(fs.rename(p, pnew));
// d. Close file handle that was opened in (b).
try {
out.close();
fail("close() should throw an exception");
} catch (Exception e) {
AppendTestUtil.LOG.info("GOOD!", e);
}
// wait for the lease recovery
cluster.setLeasePeriod(1000, 1000);
AppendTestUtil.sleep(5000);
// check block sizes
final long len = fs.getFileStatus(pnew).getLen();
final LocatedBlocks locatedblocks =
fs.dfs.getNamenode().getBlockLocations(pnew.toString(), 0L, len);
final int numblock = locatedblocks.locatedBlockCount();
for (int i = 0; i < numblock; i++) {
final LocatedBlock lb = locatedblocks.get(i);
final Block blk = lb.getBlock();
final long size = lb.getBlockSize();
if (i < numblock - 1) {
assertEquals(BLOCK_SIZE, size);
}
for (DatanodeInfo datanodeinfo : lb.getLocations()) {
final DataNode dn = cluster.getDataNode(datanodeinfo.getIpcPort());
final Block metainfo = dn.data.getStoredBlock(blk.getBlockId());
assertEquals(size, metainfo.getNumBytes());
}
}
}
protected final LocatedBlocks ensureFileReplicasOnStorageType(Path path, StorageType storageType)
throws IOException {
// Ensure that returned block locations returned are correct!
LOG.info("Ensure path: " + path + " is on StorageType: " + storageType);
assertThat(fs.exists(path), is(true));
long fileLength = client.getFileInfo(path.toString()).getLen();
LocatedBlocks locatedBlocks = client.getLocatedBlocks(path.toString(), 0, fileLength);
for (LocatedBlock locatedBlock : locatedBlocks.getLocatedBlocks()) {
assertThat(locatedBlock.getStorageTypes()[0], is(storageType));
}
return locatedBlocks;
}
private BlockInfo createBlockInfo(Path file, LocatedBlock b) {
DatanodeInfo[] locations = b.getLocations();
String[] hosts = new String[locations.length];
String[] names = new String[locations.length];
for (int i = 0; i < locations.length; ++i) {
DatanodeInfo d = locations[i];
hosts[i] = d.getHost();
names[i] = d.getName();
}
BlockLocation loc = new BlockLocation(names, hosts, b.getStartOffset(), b.getBlockSize());
return new BlockInfo(loc, file);
}
/** Read the block length from one of the datanodes. */
private long readBlockLength(LocatedBlock locatedblock) throws IOException {
assert locatedblock != null : "LocatedBlock cannot be null";
int replicaNotFoundCount = locatedblock.getLocations().length;
for (DatanodeInfo datanode : locatedblock.getLocations()) {
ClientDatanodeProtocol cdp = null;
try {
cdp =
DFSUtil.createClientDatanodeProtocolProxy(
datanode, dfsClient.conf, dfsClient.getConf().socketTimeout, locatedblock);
final long n = cdp.getReplicaVisibleLength(locatedblock.getBlock());
if (n >= 0) {
return n;
}
} catch (IOException ioe) {
if (ioe instanceof RemoteException
&& (((RemoteException) ioe).unwrapRemoteException()
instanceof ReplicaNotFoundException)) {
// special case : replica might not be on the DN, treat as 0 length
replicaNotFoundCount--;
}
if (DFSClient.LOG.isDebugEnabled()) {
DFSClient.LOG.debug(
"Failed to getReplicaVisibleLength from datanode "
+ datanode
+ " for block "
+ locatedblock.getBlock(),
ioe);
}
} finally {
if (cdp != null) {
RPC.stopProxy(cdp);
}
}
}
// Namenode told us about these locations, but none know about the replica
// means that we hit the race between pipeline creation start and end.
// we require all 3 because some other exception could have happened
// on a DN that has it. we want to report that error
if (replicaNotFoundCount == 0) {
return 0;
}
throw new IOException("Cannot obtain block length for " + locatedblock);
}
/** Convert a Json map to LocatedBlock. */
private static LocatedBlock toLocatedBlock(final Map<?, ?> m) throws IOException {
if (m == null) {
return null;
}
final Block b = toBlock((Map<?, ?>) m.get("block"));
final DatanodeInfo[] locations = toDatanodeInfoArray((Object[]) m.get("locations"));
final long startOffset = (Long) m.get("startOffset");
final boolean isCorrupt = (Boolean) m.get("isCorrupt");
final LocatedBlock locatedblock = new LocatedBlock(b, locations, startOffset, isCorrupt);
locatedblock.setBlockToken(toBlockToken((Map<?, ?>) m.get("blockToken")));
return locatedblock;
}
/** Convert a LocatedBlock to a Json map. */
private static Map<String, Object> toJsonMap(final LocatedBlock locatedblock) throws IOException {
if (locatedblock == null) {
return null;
}
final Map<String, Object> m = new TreeMap<String, Object>();
m.put("blockToken", toJsonMap(locatedblock.getBlockToken()));
m.put("isCorrupt", locatedblock.isCorrupt());
m.put("startOffset", locatedblock.getStartOffset());
m.put("block", toJsonMap(locatedblock.getBlock()));
m.put("locations", toJsonArray(locatedblock.getLocations()));
return m;
}
private void waitForLocations(int locations) throws IOException, InterruptedException {
for (int tries = 0; tries < RETRIES; )
try {
LocatedBlock locatedBlock = getLocatedBlock();
assertThat(locatedBlock.getLocations().length, is(locations));
break;
} catch (AssertionError e) {
if (++tries < RETRIES) {
Thread.sleep(1000);
} else {
throw e;
}
}
}
/** Test that the {@link PlacementMonitor.BlockAndDatanodeResolver} works correctly. */
@Test
public void testBlockAndDatanodeResolver() throws Exception {
setupCluster();
try {
Path src = new Path("/dir/file");
Path parity = new Path("/raid/dir/file");
DFSTestUtil.createFile(fs, src, 20, (short) 2, 0L);
DFSTestUtil.createFile(fs, parity, 11, (short) 2, 0L);
DFSTestUtil.waitReplication(fs, src, (short) 2);
DFSTestUtil.waitReplication(fs, parity, (short) 2);
LocatedBlocks srcLbs, parityLbs;
List<BlockInfo> srcInfos, parityInfos;
srcLbs = namenode.getBlockLocations(src.toString(), 4, 10);
srcInfos = placementMonitor.getBlockInfos(fs, src, 4, 10);
parityLbs = namenode.getBlockLocations(parity.toString(), 3, 7);
parityInfos = placementMonitor.getBlockInfos(fs, parity, 3, 7);
Assert.assertEquals(10, srcLbs.getLocatedBlocks().size());
Assert.assertEquals(7, parityLbs.getLocatedBlocks().size());
Assert.assertEquals(10, srcInfos.size());
Assert.assertEquals(7, parityInfos.size());
BlockAndDatanodeResolver resolver = new BlockAndDatanodeResolver(src, fs, parity, fs);
for (int i = 0; i < srcInfos.size(); ++i) {
LocatedBlock lb = resolver.getLocatedBlock(srcInfos.get(i));
Assert.assertEquals(srcLbs.get(i).getBlock(), lb.getBlock());
for (String nodeName : srcInfos.get(i).getNames()) {
DatanodeInfo node = resolver.getDatanodeInfo(nodeName);
Assert.assertEquals(node.getName(), nodeName);
}
}
for (int i = 0; i < parityInfos.size(); ++i) {
LocatedBlock lb = resolver.getLocatedBlock(parityInfos.get(i));
Assert.assertEquals(parityLbs.get(i).getBlock(), lb.getBlock());
for (String nodeName : parityInfos.get(i).getNames()) {
DatanodeInfo node = resolver.getDatanodeInfo(nodeName);
Assert.assertEquals(node.getName(), nodeName);
}
}
} finally {
if (cluster != null) {
cluster.shutdown();
}
if (placementMonitor != null) {
placementMonitor.stop();
}
}
}
/**
* try to access a block on a data node. If fails - throws exception
*
* @param datanode
* @param lblock
* @throws IOException
*/
private void accessBlock(DatanodeInfo datanode, LocatedBlock lblock) throws IOException {
InetSocketAddress targetAddr = null;
ExtendedBlock block = lblock.getBlock();
targetAddr = NetUtils.createSocketAddr(datanode.getXferAddr());
BlockReader blockReader =
new BlockReaderFactory(new DfsClientConf(conf))
.setInetSocketAddress(targetAddr)
.setBlock(block)
.setFileName(
BlockReaderFactory.getFileName(targetAddr, "test-blockpoolid", block.getBlockId()))
.setBlockToken(lblock.getBlockToken())
.setStartOffset(0)
.setLength(-1)
.setVerifyChecksum(true)
.setClientName("TestDataNodeVolumeFailure")
.setDatanodeInfo(datanode)
.setCachingStrategy(CachingStrategy.newDefaultStrategy())
.setClientCacheContext(ClientContext.getFromConf(conf))
.setConfiguration(conf)
.setTracer(FsTracer.get(conf))
.setRemotePeerFactory(
new RemotePeerFactory() {
@Override
public Peer newConnectedPeer(
InetSocketAddress addr,
Token<BlockTokenIdentifier> blockToken,
DatanodeID datanodeId)
throws IOException {
Peer peer = null;
Socket sock = NetUtils.getDefaultSocketFactory(conf).createSocket();
try {
sock.connect(addr, HdfsConstants.READ_TIMEOUT);
sock.setSoTimeout(HdfsConstants.READ_TIMEOUT);
peer = DFSUtilClient.peerFromSocket(sock);
} finally {
if (peer == null) {
IOUtils.closeSocket(sock);
}
}
return peer;
}
})
.build();
blockReader.close();
}
/**
* Test that {@link PlacementMonitor} moves block correctly
*
* @throws Exception
*/
@Test
public void testMoveBlock() throws Exception {
setupCluster();
try {
Path path = new Path("/dir/file");
DFSTestUtil.createFile(fs, path, 1, (short) 1, 0L);
DFSTestUtil.waitReplication(fs, path, (short) 1);
FileStatus status = fs.getFileStatus(path);
LocatedBlocks blocks = namenode.getBlockLocations(path.toString(), 0, status.getLen());
Assert.assertEquals(1, blocks.getLocatedBlocks().size());
LocatedBlock block = blocks.getLocatedBlocks().get(0);
Assert.assertEquals(1, block.getLocations().length);
DatanodeInfo source = block.getLocations()[0];
Set<DatanodeInfo> excluded = new HashSet<DatanodeInfo>();
for (DatanodeInfo d : datanodes) {
excluded.add(d);
}
excluded.remove(source);
DatanodeInfo target = excluded.iterator().next();
excluded.add(source);
excluded.remove(target);
BlockMover.BlockMoveAction action =
blockMover.new BlockMoveAction(block, source, excluded, 1);
LOG.info("Start moving block from " + source + " to " + target);
action.run();
LOG.info("Done moving block");
boolean blockMoved = false;
for (int i = 0; i < 100; ++i) {
blocks = namenode.getBlockLocations(path.toString(), 0, status.getLen());
block = blocks.getLocatedBlocks().get(0);
if (block.getLocations().length == 1 && block.getLocations()[0].equals((target))) {
blockMoved = true;
break;
}
Thread.sleep(100L);
}
Assert.assertTrue(blockMoved);
} finally {
if (cluster != null) {
cluster.shutdown();
}
if (placementMonitor != null) {
placementMonitor.stop();
}
}
}
/** removes a specified block from MiniDFS storage and reports it as corrupt */
private void removeAndReportBlock(
DistributedFileSystem blockDfs, Path filePath, LocatedBlock block) throws IOException {
TestRaidDfs.corruptBlock(filePath, block.getBlock(), NUM_DATANODES, true);
// report deleted block to the name node
LocatedBlock[] toReport = {block};
blockDfs.getClient().namenode.reportBadBlocks(toReport);
}
/**
* Read bytes starting from the specified position.
*
* @param position start read from this position
* @param buffer read buffer
* @param offset offset into buffer
* @param length number of bytes to read
* @return actual number of bytes read
*/
@Override
public int read(long position, byte[] buffer, int offset, int length) throws IOException {
// sanity checks
dfsClient.checkOpen();
if (closed) {
throw new IOException("Stream closed");
}
failures = 0;
long filelen = getFileLength();
if ((position < 0) || (position >= filelen)) {
return -1;
}
int realLen = length;
if ((position + length) > filelen) {
realLen = (int) (filelen - position);
}
// determine the block and byte range within the block
// corresponding to position and realLen
List<LocatedBlock> blockRange = getBlockRange(position, realLen);
int remaining = realLen;
Map<ExtendedBlock, Set<DatanodeInfo>> corruptedBlockMap =
new HashMap<ExtendedBlock, Set<DatanodeInfo>>();
for (LocatedBlock blk : blockRange) {
long targetStart = position - blk.getStartOffset();
long bytesToRead = Math.min(remaining, blk.getBlockSize() - targetStart);
try {
fetchBlockByteRange(
blk, targetStart, targetStart + bytesToRead - 1, buffer, offset, corruptedBlockMap);
} finally {
// Check and report if any block replicas are corrupted.
// BlockMissingException may be caught if all block replicas are
// corrupted.
reportCheckSumFailure(corruptedBlockMap, blk.getLocations().length);
}
remaining -= bytesToRead;
position += bytesToRead;
offset += bytesToRead;
}
assert remaining == 0 : "Wrong number of bytes read.";
if (dfsClient.stats != null) {
dfsClient.stats.incrementBytesRead(realLen);
}
return realLen;
}
/**
* go to each block on the 2nd DataNode until it fails...
*
* @param path
* @param size
* @throws IOException
*/
private void triggerFailure(String path, long size) throws IOException {
NamenodeProtocols nn = cluster.getNameNodeRpc();
List<LocatedBlock> locatedBlocks = nn.getBlockLocations(path, 0, size).getLocatedBlocks();
for (LocatedBlock lb : locatedBlocks) {
DatanodeInfo dinfo = lb.getLocations()[1];
ExtendedBlock b = lb.getBlock();
try {
accessBlock(dinfo, lb);
} catch (IOException e) {
System.out.println(
"Failure triggered, on block: "
+ b.getBlockId()
+ "; corresponding volume should be removed by now");
break;
}
}
}
/**
* TC7: Corrupted replicas are present.
*
* @throws IOException an exception might be thrown
*/
public void testTC7() throws Exception {
final short repl = 2;
final Path p = new Path("/TC7/foo");
System.out.println("p=" + p);
// a. Create file with replication factor of 2. Write half block of data. Close file.
final int len1 = (int) (BLOCK_SIZE / 2);
{
FSDataOutputStream out = fs.create(p, false, buffersize, repl, BLOCK_SIZE);
AppendTestUtil.write(out, 0, len1);
out.close();
}
DFSTestUtil.waitReplication(fs, p, repl);
// b. Log into one datanode that has one replica of this block.
// Find the block file on this datanode and truncate it to zero size.
final LocatedBlocks locatedblocks =
fs.dfs.getNamenode().getBlockLocations(p.toString(), 0L, len1);
assertEquals(1, locatedblocks.locatedBlockCount());
final LocatedBlock lb = locatedblocks.get(0);
final Block blk = lb.getBlock();
assertEquals(len1, lb.getBlockSize());
DatanodeInfo[] datanodeinfos = lb.getLocations();
assertEquals(repl, datanodeinfos.length);
final DataNode dn = cluster.getDataNode(datanodeinfos[0].getIpcPort());
final FSDataset data = (FSDataset) dn.getFSDataset();
final RandomAccessFile raf = new RandomAccessFile(data.getBlockFile(blk), "rw");
AppendTestUtil.LOG.info("dn=" + dn + ", blk=" + blk + " (length=" + blk.getNumBytes() + ")");
assertEquals(len1, raf.length());
raf.setLength(0);
raf.close();
// c. Open file in "append mode". Append a new block worth of data. Close file.
final int len2 = (int) BLOCK_SIZE;
{
FSDataOutputStream out = fs.append(p);
AppendTestUtil.write(out, len1, len2);
out.close();
}
// d. Reopen file and read two blocks worth of data.
AppendTestUtil.check(fs, p, len1 + len2);
}
private int printLocatedBlocks(Path filePath) throws Exception {
LocatedBlocks lbs = dfs.getLocatedBlocks(filePath, 0L, Integer.MAX_VALUE);
StringBuilder sb = new StringBuilder();
sb.append("Path " + filePath + ":");
int maxRepl = 0;
for (LocatedBlock lb : lbs.getLocatedBlocks()) {
sb.append(lb.getBlock());
sb.append(":");
for (DatanodeInfo loc : lb.getLocations()) {
sb.append(loc.getHostName());
sb.append(" ");
}
if (lb.getLocations().length > maxRepl) {
maxRepl = lb.getLocations().length;
}
}
LOG.info(sb.toString());
return maxRepl;
}
/**
* Get block at the specified position. Fetch it from the namenode if not cached.
*
* @param offset
* @param updatePosition whether to update current position
* @return located block
* @throws IOException
*/
private synchronized LocatedBlock getBlockAt(long offset, boolean updatePosition)
throws IOException {
assert (locatedBlocks != null) : "locatedBlocks is null";
final LocatedBlock blk;
// check offset
if (offset < 0 || offset >= getFileLength()) {
throw new IOException(
"offset < 0 || offset > getFileLength(), offset="
+ offset
+ ", updatePosition="
+ updatePosition
+ ", locatedBlocks="
+ locatedBlocks);
} else if (offset >= locatedBlocks.getFileLength()) {
// offset to the portion of the last block,
// which is not known to the name-node yet;
// getting the last block
blk = locatedBlocks.getLastLocatedBlock();
} else {
// search cached blocks first
int targetBlockIdx = locatedBlocks.findBlock(offset);
if (targetBlockIdx < 0) { // block is not cached
targetBlockIdx = LocatedBlocks.getInsertIndex(targetBlockIdx);
// fetch more blocks
LocatedBlocks newBlocks;
newBlocks = DFSClient.callGetBlockLocations(dfsClient.namenode, src, offset, prefetchSize);
assert (newBlocks != null) : "Could not find target position " + offset;
locatedBlocks.insertRange(targetBlockIdx, newBlocks.getLocatedBlocks());
}
blk = locatedBlocks.get(targetBlockIdx);
}
// update current position
if (updatePosition) {
pos = offset;
blockEnd = blk.getStartOffset() + blk.getBlockSize() - 1;
currentLocatedBlock = blk;
}
return blk;
}
/*
* Since NameNode will not persist any locations of the block, addBlock()
* retry call after restart NN should re-select the locations and return to
* client. refer HDFS-5257
*/
@Test
public void testAddBlockRetryShouldReturnBlockWithLocations() throws Exception {
final String src = "/testAddBlockRetryShouldReturnBlockWithLocations";
NamenodeProtocols nameNodeRpc = cluster.getNameNodeRpc();
// create file
nameNodeRpc.create(
src,
FsPermission.getFileDefault(),
"clientName",
new EnumSetWritable<CreateFlag>(EnumSet.of(CreateFlag.CREATE)),
true,
(short) 3,
1024,
null);
// start first addBlock()
LOG.info("Starting first addBlock for " + src);
LocatedBlock lb1 =
nameNodeRpc.addBlock(src, "clientName", null, null, INodeId.GRANDFATHER_INODE_ID, null);
assertTrue("Block locations should be present", lb1.getLocations().length > 0);
cluster.restartNameNode();
nameNodeRpc = cluster.getNameNodeRpc();
LocatedBlock lb2 =
nameNodeRpc.addBlock(src, "clientName", null, null, INodeId.GRANDFATHER_INODE_ID, null);
assertEquals("Blocks are not equal", lb1.getBlock(), lb2.getBlock());
assertTrue("Wrong locations with retry", lb2.getLocations().length > 0);
}