/** Asynchronously lazy persist the block from the RamDisk to Disk. */
void submitLazyPersistTask(
String bpId,
long blockId,
long genStamp,
long creationTime,
File metaFile,
File blockFile,
FsVolumeReference target)
throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug(
"LazyWriter schedule async task to persist RamDisk block pool id: "
+ bpId
+ " block id: "
+ blockId);
}
FsVolumeImpl volume = (FsVolumeImpl) target.getVolume();
File lazyPersistDir = volume.getLazyPersistDir(bpId);
if (!lazyPersistDir.exists() && !lazyPersistDir.mkdirs()) {
FsDatasetImpl.LOG.warn("LazyWriter failed to create " + lazyPersistDir);
throw new IOException(
"LazyWriter fail to find or create lazy persist dir: " + lazyPersistDir.toString());
}
ReplicaLazyPersistTask lazyPersistTask =
new ReplicaLazyPersistTask(
bpId, blockId, genStamp, creationTime, blockFile, metaFile, target, lazyPersistDir);
execute(volume.getCurrentDir(), lazyPersistTask);
}
protected final boolean verifyDeletedBlocks(LocatedBlocks locatedBlocks)
throws IOException, InterruptedException {
LOG.info("Verifying replica has no saved copy after deletion.");
triggerBlockReport();
while (DataNodeTestUtils.getPendingAsyncDeletions(cluster.getDataNodes().get(0)) > 0L) {
Thread.sleep(1000);
}
final String bpid = cluster.getNamesystem().getBlockPoolId();
List<? extends FsVolumeSpi> volumes = cluster.getDataNodes().get(0).getFSDataset().getVolumes();
// Make sure deleted replica does not have a copy on either finalized dir of
// transient volume or finalized dir of non-transient volume
for (FsVolumeSpi v : volumes) {
FsVolumeImpl volume = (FsVolumeImpl) v;
File targetDir =
(v.isTransientStorage())
? volume.getBlockPoolSlice(bpid).getFinalizedDir()
: volume.getBlockPoolSlice(bpid).getLazypersistDir();
if (verifyBlockDeletedFromDir(targetDir, locatedBlocks) == false) {
return false;
}
}
return true;
}
/**
* Make sure at least one non-transient volume has a saved copy of the replica. An infinite loop
* is used to ensure the async lazy persist tasks are completely done before verification. Caller
* of ensureLazyPersistBlocksAreSaved expects either a successful pass or timeout failure.
*/
protected final void ensureLazyPersistBlocksAreSaved(LocatedBlocks locatedBlocks)
throws IOException, InterruptedException {
final String bpid = cluster.getNamesystem().getBlockPoolId();
List<? extends FsVolumeSpi> volumes = cluster.getDataNodes().get(0).getFSDataset().getVolumes();
final Set<Long> persistedBlockIds = new HashSet<Long>();
while (persistedBlockIds.size() < locatedBlocks.getLocatedBlocks().size()) {
// Take 1 second sleep before each verification iteration
Thread.sleep(1000);
for (LocatedBlock lb : locatedBlocks.getLocatedBlocks()) {
for (FsVolumeSpi v : volumes) {
if (v.isTransientStorage()) {
continue;
}
FsVolumeImpl volume = (FsVolumeImpl) v;
File lazyPersistDir = volume.getBlockPoolSlice(bpid).getLazypersistDir();
long blockId = lb.getBlock().getBlockId();
File targetDir = DatanodeUtil.idToBlockDir(lazyPersistDir, blockId);
File blockFile = new File(targetDir, lb.getBlock().getBlockName());
if (blockFile.exists()) {
// Found a persisted copy for this block and added to the Set
persistedBlockIds.add(blockId);
}
}
}
}
// We should have found a persisted copy for each located block.
assertThat(persistedBlockIds.size(), is(locatedBlocks.getLocatedBlocks().size()));
}
/**
* If ramDiskStorageLimit is >=0, then RAM_DISK capacity is artificially capped. If
* ramDiskStorageLimit < 0 then it is ignored.
*/
protected final void startUpCluster(
final int numDataNodes,
final StorageType[] storageTypes,
final long ramDiskStorageLimit,
final boolean useSCR)
throws IOException {
Configuration conf = new Configuration();
conf.setLong(DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
conf.setInt(
DFS_NAMENODE_LAZY_PERSIST_FILE_SCRUB_INTERVAL_SEC, LAZY_WRITE_FILE_SCRUBBER_INTERVAL_SEC);
conf.setLong(DFS_HEARTBEAT_INTERVAL_KEY, HEARTBEAT_INTERVAL_SEC);
conf.setInt(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, HEARTBEAT_RECHECK_INTERVAL_MSEC);
conf.setInt(DFS_DATANODE_LAZY_WRITER_INTERVAL_SEC, LAZY_WRITER_INTERVAL_SEC);
if (useSCR) {
conf.setBoolean(DFS_CLIENT_READ_SHORTCIRCUIT_KEY, useSCR);
conf.set(DFS_CLIENT_CONTEXT, UUID.randomUUID().toString());
sockDir = new TemporarySocketDirectory();
conf.set(
DFS_DOMAIN_SOCKET_PATH_KEY,
new File(sockDir.getDir(), this.getClass().getSimpleName() + "._PORT.sock")
.getAbsolutePath());
conf.set(
DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY,
UserGroupInformation.getCurrentUser().getShortUserName());
}
cluster =
new MiniDFSCluster.Builder(conf)
.numDataNodes(numDataNodes)
.storageTypes(
storageTypes != null ? storageTypes : new StorageType[] {DEFAULT, DEFAULT})
.build();
fs = cluster.getFileSystem();
client = fs.getClient();
// Artificially cap the storage capacity of the RAM_DISK volume.
if (ramDiskStorageLimit >= 0) {
List<? extends FsVolumeSpi> volumes =
cluster.getDataNodes().get(0).getFSDataset().getVolumes();
for (FsVolumeSpi volume : volumes) {
if (volume.getStorageType() == RAM_DISK) {
((FsVolumeImpl) volume).setCapacityForTesting(ramDiskStorageLimit);
}
}
}
LOG.info("Cluster startup complete");
}
/**
* Generate testing environment and return a collection of blocks on which to run the tests.
*
* @param bpid Block pool ID to generate blocks for
* @param dataSet Namespace in which to insert blocks
* @return Contrived blocks for further testing.
* @throws IOException
*/
private ExtendedBlock[] setup(String bpid, FsDatasetImpl dataSet) throws IOException {
// setup replicas map
ExtendedBlock[] blocks =
new ExtendedBlock[] {
new ExtendedBlock(bpid, 1, 1, 2001), new ExtendedBlock(bpid, 2, 1, 2002),
new ExtendedBlock(bpid, 3, 1, 2003), new ExtendedBlock(bpid, 4, 1, 2004),
new ExtendedBlock(bpid, 5, 1, 2005), new ExtendedBlock(bpid, 6, 1, 2006)
};
ReplicaMap replicasMap = dataSet.volumeMap;
FsVolumeImpl vol =
(FsVolumeImpl) dataSet.volumes.getNextVolume(StorageType.DEFAULT, 0).getVolume();
ReplicaInfo replicaInfo =
new FinalizedReplica(
blocks[FINALIZED].getLocalBlock(), vol, vol.getCurrentDir().getParentFile());
replicasMap.add(bpid, replicaInfo);
replicaInfo.getBlockFile().createNewFile();
replicaInfo.getMetaFile().createNewFile();
replicasMap.add(
bpid,
new ReplicaInPipeline(
blocks[TEMPORARY].getBlockId(),
blocks[TEMPORARY].getGenerationStamp(),
vol,
vol.createTmpFile(bpid, blocks[TEMPORARY].getLocalBlock()).getParentFile(),
0));
replicaInfo =
new ReplicaBeingWritten(
blocks[RBW].getLocalBlock(),
vol,
vol.createRbwFile(bpid, blocks[RBW].getLocalBlock()).getParentFile(),
null);
replicasMap.add(bpid, replicaInfo);
replicaInfo.getBlockFile().createNewFile();
replicaInfo.getMetaFile().createNewFile();
replicasMap.add(
bpid,
new ReplicaWaitingToBeRecovered(
blocks[RWR].getLocalBlock(),
vol,
vol.createRbwFile(bpid, blocks[RWR].getLocalBlock()).getParentFile()));
replicasMap.add(
bpid,
new ReplicaUnderRecovery(
new FinalizedReplica(
blocks[RUR].getLocalBlock(), vol, vol.getCurrentDir().getParentFile()),
2007));
return blocks;
}
private void testAppend(String bpid, FsDatasetImpl dataSet, ExtendedBlock[] blocks)
throws IOException {
long newGS = blocks[FINALIZED].getGenerationStamp() + 1;
final FsVolumeImpl v =
(FsVolumeImpl) dataSet.volumeMap.get(bpid, blocks[FINALIZED].getLocalBlock()).getVolume();
long available = v.getCapacity() - v.getDfsUsed();
long expectedLen = blocks[FINALIZED].getNumBytes();
try {
v.decDfsUsed(bpid, -available);
blocks[FINALIZED].setNumBytes(expectedLen + 100);
dataSet.append(blocks[FINALIZED], newGS, expectedLen);
Assert.fail("Should not have space to append to an RWR replica" + blocks[RWR]);
} catch (DiskOutOfSpaceException e) {
Assert.assertTrue(e.getMessage().startsWith("Insufficient space for appending to "));
}
v.decDfsUsed(bpid, available);
blocks[FINALIZED].setNumBytes(expectedLen);
newGS = blocks[RBW].getGenerationStamp() + 1;
dataSet.append(blocks[FINALIZED], newGS, blocks[FINALIZED].getNumBytes()); // successful
blocks[FINALIZED].setGenerationStamp(newGS);
try {
dataSet.append(
blocks[TEMPORARY],
blocks[TEMPORARY].getGenerationStamp() + 1,
blocks[TEMPORARY].getNumBytes());
Assert.fail("Should not have appended to a temporary replica " + blocks[TEMPORARY]);
} catch (ReplicaNotFoundException e) {
Assert.assertEquals(
ReplicaNotFoundException.UNFINALIZED_REPLICA + blocks[TEMPORARY], e.getMessage());
}
try {
dataSet.append(blocks[RBW], blocks[RBW].getGenerationStamp() + 1, blocks[RBW].getNumBytes());
Assert.fail("Should not have appended to an RBW replica" + blocks[RBW]);
} catch (ReplicaNotFoundException e) {
Assert.assertEquals(
ReplicaNotFoundException.UNFINALIZED_REPLICA + blocks[RBW], e.getMessage());
}
try {
dataSet.append(blocks[RWR], blocks[RWR].getGenerationStamp() + 1, blocks[RBW].getNumBytes());
Assert.fail("Should not have appended to an RWR replica" + blocks[RWR]);
} catch (ReplicaNotFoundException e) {
Assert.assertEquals(
ReplicaNotFoundException.UNFINALIZED_REPLICA + blocks[RWR], e.getMessage());
}
try {
dataSet.append(blocks[RUR], blocks[RUR].getGenerationStamp() + 1, blocks[RUR].getNumBytes());
Assert.fail("Should not have appended to an RUR replica" + blocks[RUR]);
} catch (ReplicaNotFoundException e) {
Assert.assertEquals(
ReplicaNotFoundException.UNFINALIZED_REPLICA + blocks[RUR], e.getMessage());
}
try {
dataSet.append(
blocks[NON_EXISTENT],
blocks[NON_EXISTENT].getGenerationStamp(),
blocks[NON_EXISTENT].getNumBytes());
Assert.fail("Should not have appended to a non-existent replica " + blocks[NON_EXISTENT]);
} catch (ReplicaNotFoundException e) {
Assert.assertEquals(
ReplicaNotFoundException.NON_EXISTENT_REPLICA + blocks[NON_EXISTENT], e.getMessage());
}
newGS = blocks[FINALIZED].getGenerationStamp() + 1;
dataSet.recoverAppend(blocks[FINALIZED], newGS, blocks[FINALIZED].getNumBytes()); // successful
blocks[FINALIZED].setGenerationStamp(newGS);
try {
dataSet.recoverAppend(
blocks[TEMPORARY],
blocks[TEMPORARY].getGenerationStamp() + 1,
blocks[TEMPORARY].getNumBytes());
Assert.fail("Should not have appended to a temporary replica " + blocks[TEMPORARY]);
} catch (ReplicaNotFoundException e) {
Assert.assertTrue(
e.getMessage().startsWith(ReplicaNotFoundException.UNFINALIZED_AND_NONRBW_REPLICA));
}
newGS = blocks[RBW].getGenerationStamp() + 1;
dataSet.recoverAppend(blocks[RBW], newGS, blocks[RBW].getNumBytes());
blocks[RBW].setGenerationStamp(newGS);
try {
dataSet.recoverAppend(
blocks[RWR], blocks[RWR].getGenerationStamp() + 1, blocks[RBW].getNumBytes());
Assert.fail("Should not have appended to an RWR replica" + blocks[RWR]);
} catch (ReplicaNotFoundException e) {
Assert.assertTrue(
e.getMessage().startsWith(ReplicaNotFoundException.UNFINALIZED_AND_NONRBW_REPLICA));
}
try {
dataSet.recoverAppend(
blocks[RUR], blocks[RUR].getGenerationStamp() + 1, blocks[RUR].getNumBytes());
Assert.fail("Should not have appended to an RUR replica" + blocks[RUR]);
} catch (ReplicaNotFoundException e) {
Assert.assertTrue(
e.getMessage().startsWith(ReplicaNotFoundException.UNFINALIZED_AND_NONRBW_REPLICA));
}
try {
dataSet.recoverAppend(
blocks[NON_EXISTENT],
blocks[NON_EXISTENT].getGenerationStamp(),
blocks[NON_EXISTENT].getNumBytes());
Assert.fail("Should not have appended to a non-existent replica " + blocks[NON_EXISTENT]);
} catch (ReplicaNotFoundException e) {
Assert.assertTrue(e.getMessage().startsWith(ReplicaNotFoundException.NON_EXISTENT_REPLICA));
}
}