/** Test truncate over quota does not mark file as UC or create a lease */
@Test(timeout = 60000)
public void testTruncateOverQuota() throws Exception {
final Path dir = new Path("/TestTruncateOverquota");
final Path file = new Path(dir, "file");
// create partial block file
dfs.mkdirs(dir);
DFSTestUtil.createFile(dfs, file, BLOCKSIZE / 2, REPLICATION, seed);
// lower quota to cause exception when appending to partial block
dfs.setQuota(dir, Long.MAX_VALUE - 1, 1);
final INodeDirectory dirNode = fsdir.getINode4Write(dir.toString()).asDirectory();
final long spaceUsed =
dirNode.getDirectoryWithQuotaFeature().getSpaceConsumed().getStorageSpace();
try {
dfs.truncate(file, BLOCKSIZE / 2 - 1);
Assert.fail("truncate didn't fail");
} catch (RemoteException e) {
assertTrue(e.getClassName().contains("DSQuotaExceededException"));
}
// check that the file exists, isn't UC, and has no dangling lease
LeaseManager lm = cluster.getNamesystem().getLeaseManager();
INodeFile inode = fsdir.getINode(file.toString()).asFile();
Assert.assertNotNull(inode);
Assert.assertFalse("should not be UC", inode.isUnderConstruction());
Assert.assertNull("should not have a lease", lm.getLease(inode));
// make sure the quota usage is unchanged
final long newSpaceUsed =
dirNode.getDirectoryWithQuotaFeature().getSpaceConsumed().getStorageSpace();
assertEquals(spaceUsed, newSpaceUsed);
// make sure edits aren't corrupted
dfs.recoverLease(file);
cluster.restartNameNodes();
}
@Override
void recordModification(int latestSnapshotId) {
if (isInLatestSnapshot(latestSnapshotId)) {
INodeDirectory parent = getParent();
parent.saveChild2Snapshot(this, latestSnapshotId, new INodeSymlink(this));
}
}
@Override
INode recordModification(Snapshot latest, final INodeMap inodeMap) throws QuotaExceededException {
if (isInLatestSnapshot(latest)) {
INodeDirectory parent = getParent();
parent.saveChild2Snapshot(this, latest, new INodeSymlink(this), inodeMap);
}
return this;
}
/** Test if the quota can be correctly updated for append */
@Test(timeout = 60000)
public void testUpdateQuotaForAppend() throws Exception {
final Path foo = new Path(dir, "foo");
final Path bar = new Path(foo, "bar");
long currentFileLen = BLOCKSIZE;
DFSTestUtil.createFile(dfs, bar, currentFileLen, REPLICATION, seed);
dfs.setQuota(foo, Long.MAX_VALUE - 1, Long.MAX_VALUE - 1);
// append half of the block data, the previous file length is at block
// boundary
DFSTestUtil.appendFile(dfs, bar, BLOCKSIZE / 2);
currentFileLen += (BLOCKSIZE / 2);
INodeDirectory fooNode = fsdir.getINode4Write(foo.toString()).asDirectory();
assertTrue(fooNode.isQuotaSet());
QuotaCounts quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
long ns = quota.getNameSpace();
long ds = quota.getStorageSpace();
assertEquals(2, ns); // foo and bar
assertEquals(currentFileLen * REPLICATION, ds);
ContentSummary c = dfs.getContentSummary(foo);
assertEquals(c.getSpaceConsumed(), ds);
// append another block, the previous file length is not at block boundary
DFSTestUtil.appendFile(dfs, bar, BLOCKSIZE);
currentFileLen += BLOCKSIZE;
quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
ns = quota.getNameSpace();
ds = quota.getStorageSpace();
assertEquals(2, ns); // foo and bar
assertEquals(currentFileLen * REPLICATION, ds);
c = dfs.getContentSummary(foo);
assertEquals(c.getSpaceConsumed(), ds);
// append several blocks
DFSTestUtil.appendFile(dfs, bar, BLOCKSIZE * 3 + BLOCKSIZE / 8);
currentFileLen += (BLOCKSIZE * 3 + BLOCKSIZE / 8);
quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
ns = quota.getNameSpace();
ds = quota.getStorageSpace();
assertEquals(2, ns); // foo and bar
assertEquals(currentFileLen * REPLICATION, ds);
c = dfs.getContentSummary(foo);
assertEquals(c.getSpaceConsumed(), ds);
}
/**
* Convert an existing directory inode to one with the given quota
*
* @param nsQuota Namespace quota to be assigned to this inode
* @param dsQuota Diskspace quota to be assigned to this indoe
* @param other The other inode from which all other properties are copied
*/
INodeDirectoryWithQuota(long nsQuota, long dsQuota, INodeDirectory other)
throws QuotaExceededException {
super(other);
INode.DirCounts counts = new INode.DirCounts();
other.spaceConsumedInTree(counts);
this.nsCount = counts.getNsCount();
this.diskspace = counts.getDsCount();
setQuota(nsQuota, dsQuota);
}
private static INodeDirectory createINodeDirectory(
INodeDirectory parent, String name, String owner, String group, short perm)
throws IOException {
PermissionStatus permStatus =
PermissionStatus.createImmutable(owner, group, FsPermission.createImmutable(perm));
INodeDirectory inodeDirectory =
new INodeDirectory(INodeId.GRANDFATHER_INODE_ID, name.getBytes("UTF-8"), permStatus, 0L);
parent.addChild(inodeDirectory);
return inodeDirectory;
}
/** Test if the quota can be correctly updated when file length is updated through fsync */
@Test(timeout = 60000)
public void testUpdateQuotaForFSync() throws Exception {
final Path foo = new Path("/foo");
final Path bar = new Path(foo, "bar");
DFSTestUtil.createFile(dfs, bar, BLOCKSIZE, REPLICATION, 0L);
dfs.setQuota(foo, Long.MAX_VALUE - 1, Long.MAX_VALUE - 1);
FSDataOutputStream out = dfs.append(bar);
out.write(new byte[BLOCKSIZE / 4]);
((DFSOutputStream) out.getWrappedStream())
.hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH));
INodeDirectory fooNode = fsdir.getINode4Write(foo.toString()).asDirectory();
QuotaCounts quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
long ns = quota.getNameSpace();
long ds = quota.getStorageSpace();
assertEquals(2, ns); // foo and bar
assertEquals(BLOCKSIZE * 2 * REPLICATION, ds); // file is under construction
out.write(new byte[BLOCKSIZE / 4]);
out.close();
fooNode = fsdir.getINode4Write(foo.toString()).asDirectory();
quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
ns = quota.getNameSpace();
ds = quota.getStorageSpace();
assertEquals(2, ns);
assertEquals((BLOCKSIZE + BLOCKSIZE / 2) * REPLICATION, ds);
// append another block
DFSTestUtil.appendFile(dfs, bar, BLOCKSIZE);
quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
ns = quota.getNameSpace();
ds = quota.getStorageSpace();
assertEquals(2, ns); // foo and bar
assertEquals((BLOCKSIZE * 2 + BLOCKSIZE / 2) * REPLICATION, ds);
}
/**
* Test append over a specific type of storage quota does not mark file as UC or create a lease
*/
@Test(timeout = 60000)
public void testAppendOverTypeQuota() throws Exception {
final Path dir = new Path("/TestAppendOverTypeQuota");
final Path file = new Path(dir, "file");
// create partial block file
dfs.mkdirs(dir);
// set the storage policy on dir
dfs.setStoragePolicy(dir, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
DFSTestUtil.createFile(dfs, file, BLOCKSIZE / 2, REPLICATION, seed);
// set quota of SSD to 1L
dfs.setQuotaByStorageType(dir, StorageType.SSD, 1L);
final INodeDirectory dirNode = fsdir.getINode4Write(dir.toString()).asDirectory();
final long spaceUsed =
dirNode.getDirectoryWithQuotaFeature().getSpaceConsumed().getStorageSpace();
try {
DFSTestUtil.appendFile(dfs, file, BLOCKSIZE);
Assert.fail("append didn't fail");
} catch (QuotaByStorageTypeExceededException e) {
// ignore
}
// check that the file exists, isn't UC, and has no dangling lease
LeaseManager lm = cluster.getNamesystem().getLeaseManager();
INodeFile inode = fsdir.getINode(file.toString()).asFile();
Assert.assertNotNull(inode);
Assert.assertFalse("should not be UC", inode.isUnderConstruction());
Assert.assertNull("should not have a lease", lm.getLease(inode));
// make sure the quota usage is unchanged
final long newSpaceUsed =
dirNode.getDirectoryWithQuotaFeature().getSpaceConsumed().getStorageSpace();
assertEquals(spaceUsed, newSpaceUsed);
// make sure edits aren't corrupted
dfs.recoverLease(file);
cluster.restartNameNodes();
}
private void scanDirsWithQuota(
INodeDirectory dir,
HashMap<String, Long> nsMap,
HashMap<String, Long> dsMap,
boolean verify) {
if (dir.isQuotaSet()) {
// get the current consumption
QuotaCounts q = dir.getDirectoryWithQuotaFeature().getSpaceConsumed();
String name = dir.getFullPathName();
if (verify) {
assertEquals(nsMap.get(name).longValue(), q.getNameSpace());
assertEquals(dsMap.get(name).longValue(), q.getStorageSpace());
} else {
nsMap.put(name, Long.valueOf(q.getNameSpace()));
dsMap.put(name, Long.valueOf(q.getStorageSpace()));
}
}
for (INode child : dir.getChildrenList(Snapshot.CURRENT_STATE_ID)) {
if (child instanceof INodeDirectory) {
scanDirsWithQuota((INodeDirectory) child, nsMap, dsMap, verify);
}
}
}
private static INodeFile createINodeFile(
INodeDirectory parent, String name, String owner, String group, short perm)
throws IOException {
PermissionStatus permStatus =
PermissionStatus.createImmutable(owner, group, FsPermission.createImmutable(perm));
INodeFile inodeFile =
new INodeFile(
INodeId.GRANDFATHER_INODE_ID,
name.getBytes("UTF-8"),
permStatus,
0L,
0L,
null,
REPLICATION,
PREFERRED_BLOCK_SIZE);
parent.addChild(inodeFile);
return inodeFile;
}
