/** Test truncate over quota does not mark file as UC or create a lease */ @Test(timeout = 60000) public void testTruncateOverQuota() throws Exception { final Path dir = new Path("/TestTruncateOverquota"); final Path file = new Path(dir, "file"); // create partial block file dfs.mkdirs(dir); DFSTestUtil.createFile(dfs, file, BLOCKSIZE / 2, REPLICATION, seed); // lower quota to cause exception when appending to partial block dfs.setQuota(dir, Long.MAX_VALUE - 1, 1); final INodeDirectory dirNode = fsdir.getINode4Write(dir.toString()).asDirectory(); final long spaceUsed = dirNode.getDirectoryWithQuotaFeature().getSpaceConsumed().getStorageSpace(); try { dfs.truncate(file, BLOCKSIZE / 2 - 1); Assert.fail("truncate didn't fail"); } catch (RemoteException e) { assertTrue(e.getClassName().contains("DSQuotaExceededException")); } // check that the file exists, isn't UC, and has no dangling lease LeaseManager lm = cluster.getNamesystem().getLeaseManager(); INodeFile inode = fsdir.getINode(file.toString()).asFile(); Assert.assertNotNull(inode); Assert.assertFalse("should not be UC", inode.isUnderConstruction()); Assert.assertNull("should not have a lease", lm.getLease(inode)); // make sure the quota usage is unchanged final long newSpaceUsed = dirNode.getDirectoryWithQuotaFeature().getSpaceConsumed().getStorageSpace(); assertEquals(spaceUsed, newSpaceUsed); // make sure edits aren't corrupted dfs.recoverLease(file); cluster.restartNameNodes(); }
/** Test if the quota can be correctly updated when file length is updated through fsync */ @Test(timeout = 60000) public void testUpdateQuotaForFSync() throws Exception { final Path foo = new Path("/foo"); final Path bar = new Path(foo, "bar"); DFSTestUtil.createFile(dfs, bar, BLOCKSIZE, REPLICATION, 0L); dfs.setQuota(foo, Long.MAX_VALUE - 1, Long.MAX_VALUE - 1); FSDataOutputStream out = dfs.append(bar); out.write(new byte[BLOCKSIZE / 4]); ((DFSOutputStream) out.getWrappedStream()) .hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH)); INodeDirectory fooNode = fsdir.getINode4Write(foo.toString()).asDirectory(); QuotaCounts quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed(); long ns = quota.getNameSpace(); long ds = quota.getStorageSpace(); assertEquals(2, ns); // foo and bar assertEquals(BLOCKSIZE * 2 * REPLICATION, ds); // file is under construction out.write(new byte[BLOCKSIZE / 4]); out.close(); fooNode = fsdir.getINode4Write(foo.toString()).asDirectory(); quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed(); ns = quota.getNameSpace(); ds = quota.getStorageSpace(); assertEquals(2, ns); assertEquals((BLOCKSIZE + BLOCKSIZE / 2) * REPLICATION, ds); // append another block DFSTestUtil.appendFile(dfs, bar, BLOCKSIZE); quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed(); ns = quota.getNameSpace(); ds = quota.getStorageSpace(); assertEquals(2, ns); // foo and bar assertEquals((BLOCKSIZE * 2 + BLOCKSIZE / 2) * REPLICATION, ds); }
/** Test if the quota can be correctly updated for create file */ @Test(timeout = 60000) public void testQuotaUpdateWithFileCreate() throws Exception { final Path foo = new Path(dir, "foo"); Path createdFile = new Path(foo, "created_file.data"); dfs.mkdirs(foo); dfs.setQuota(foo, Long.MAX_VALUE - 1, Long.MAX_VALUE - 1); long fileLen = BLOCKSIZE * 2 + BLOCKSIZE / 2; DFSTestUtil.createFile(dfs, createdFile, BLOCKSIZE / 16, fileLen, BLOCKSIZE, REPLICATION, seed); INode fnode = fsdir.getINode4Write(foo.toString()); assertTrue(fnode.isDirectory()); assertTrue(fnode.isQuotaSet()); QuotaCounts cnt = fnode.asDirectory().getDirectoryWithQuotaFeature().getSpaceConsumed(); assertEquals(2, cnt.getNameSpace()); assertEquals(fileLen * REPLICATION, cnt.getStorageSpace()); }
/** Test if the quota can be correctly updated for append */ @Test(timeout = 60000) public void testUpdateQuotaForAppend() throws Exception { final Path foo = new Path(dir, "foo"); final Path bar = new Path(foo, "bar"); long currentFileLen = BLOCKSIZE; DFSTestUtil.createFile(dfs, bar, currentFileLen, REPLICATION, seed); dfs.setQuota(foo, Long.MAX_VALUE - 1, Long.MAX_VALUE - 1); // append half of the block data, the previous file length is at block // boundary DFSTestUtil.appendFile(dfs, bar, BLOCKSIZE / 2); currentFileLen += (BLOCKSIZE / 2); INodeDirectory fooNode = fsdir.getINode4Write(foo.toString()).asDirectory(); assertTrue(fooNode.isQuotaSet()); QuotaCounts quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed(); long ns = quota.getNameSpace(); long ds = quota.getStorageSpace(); assertEquals(2, ns); // foo and bar assertEquals(currentFileLen * REPLICATION, ds); ContentSummary c = dfs.getContentSummary(foo); assertEquals(c.getSpaceConsumed(), ds); // append another block, the previous file length is not at block boundary DFSTestUtil.appendFile(dfs, bar, BLOCKSIZE); currentFileLen += BLOCKSIZE; quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed(); ns = quota.getNameSpace(); ds = quota.getStorageSpace(); assertEquals(2, ns); // foo and bar assertEquals(currentFileLen * REPLICATION, ds); c = dfs.getContentSummary(foo); assertEquals(c.getSpaceConsumed(), ds); // append several blocks DFSTestUtil.appendFile(dfs, bar, BLOCKSIZE * 3 + BLOCKSIZE / 8); currentFileLen += (BLOCKSIZE * 3 + BLOCKSIZE / 8); quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed(); ns = quota.getNameSpace(); ds = quota.getStorageSpace(); assertEquals(2, ns); // foo and bar assertEquals(currentFileLen * REPLICATION, ds); c = dfs.getContentSummary(foo); assertEquals(c.getSpaceConsumed(), ds); }
/** * Test append over a specific type of storage quota does not mark file as UC or create a lease */ @Test(timeout = 60000) public void testAppendOverTypeQuota() throws Exception { final Path dir = new Path("/TestAppendOverTypeQuota"); final Path file = new Path(dir, "file"); // create partial block file dfs.mkdirs(dir); // set the storage policy on dir dfs.setStoragePolicy(dir, HdfsConstants.ONESSD_STORAGE_POLICY_NAME); DFSTestUtil.createFile(dfs, file, BLOCKSIZE / 2, REPLICATION, seed); // set quota of SSD to 1L dfs.setQuotaByStorageType(dir, StorageType.SSD, 1L); final INodeDirectory dirNode = fsdir.getINode4Write(dir.toString()).asDirectory(); final long spaceUsed = dirNode.getDirectoryWithQuotaFeature().getSpaceConsumed().getStorageSpace(); try { DFSTestUtil.appendFile(dfs, file, BLOCKSIZE); Assert.fail("append didn't fail"); } catch (QuotaByStorageTypeExceededException e) { // ignore } // check that the file exists, isn't UC, and has no dangling lease LeaseManager lm = cluster.getNamesystem().getLeaseManager(); INodeFile inode = fsdir.getINode(file.toString()).asFile(); Assert.assertNotNull(inode); Assert.assertFalse("should not be UC", inode.isUnderConstruction()); Assert.assertNull("should not have a lease", lm.getLease(inode)); // make sure the quota usage is unchanged final long newSpaceUsed = dirNode.getDirectoryWithQuotaFeature().getSpaceConsumed().getStorageSpace(); assertEquals(spaceUsed, newSpaceUsed); // make sure edits aren't corrupted dfs.recoverLease(file); cluster.restartNameNodes(); }