/** @throws Exception If failed. */ public void testCreateFileColocated() throws Exception { GridGgfsPath path = new GridGgfsPath("/colocated"); UUID uuid = UUID.randomUUID(); GridUuid affKey; long idx = 0; while (true) { affKey = new GridUuid(uuid, idx); if (grid(0).mapKeyToNode(DATA_CACHE_NAME, affKey).id().equals(grid(0).localNode().id())) break; idx++; } try (GridGgfsOutputStream out = fs.create(path, 1024, true, affKey, 0, 1024, null)) { // Write 5M, should be enough to test distribution. for (int i = 0; i < 15; i++) out.write(new byte[1024 * 1024]); } GridGgfsFile info = fs.info(path); Collection<GridGgfsBlockLocation> affNodes = fs.affinity(path, 0, info.length()); assertEquals(1, affNodes.size()); Collection<UUID> nodeIds = F.first(affNodes).nodeIds(); assertEquals(1, nodeIds.size()); assertEquals(grid(0).localNode().id(), F.first(nodeIds)); }
/** * Test file creation. * * @param path Path to file to store. * @param size Size of file to store. * @param salt Salt for file content generation. * @throws Exception In case of any exception. */ private void testCreateFile(final GridGgfsPath path, final long size, final int salt) throws Exception { info("Create file [path=" + path + ", size=" + size + ", salt=" + salt + ']'); final AtomicInteger cnt = new AtomicInteger(0); final Collection<GridGgfsPath> cleanUp = new ConcurrentLinkedQueue<>(); long time = runMultiThreaded( new Callable<Object>() { @Override public Object call() throws Exception { int id = cnt.incrementAndGet(); GridGgfsPath f = new GridGgfsPath(path.parent(), "asdf" + (id > 1 ? "-" + id : "")); try (GridGgfsOutputStream out = fs.create(f, 0, true, null, 0, 1024, null)) { assertNotNull(out); cleanUp.add(f); // Add all created into cleanup list. U.copy(new GridGgfsTestInputStream(size, salt), out); } return null; } }, WRITING_THREADS_CNT, "perform-multi-thread-writing"); if (time > 0) { double rate = size * 1000. / time / 1024 / 1024; info( String.format( "Write file [path=%s, size=%d kB, rate=%2.1f MB/s]", path, WRITING_THREADS_CNT * size / 1024, WRITING_THREADS_CNT * rate)); } info("Read and validate saved file: " + path); final InputStream expIn = new GridGgfsTestInputStream(size, salt); final GridGgfsInputStream actIn = fs.open(path, CFG_BLOCK_SIZE * READING_THREADS_CNT * 11 / 10); // Validate continuous reading of whole file. assertEqualStreams(expIn, actIn, size, null); // Validate random seek and reading. final Random rnd = new Random(); runMultiThreaded( new Callable<Object>() { @Override public Object call() throws Exception { long skip = Math.abs(rnd.nextLong() % (size + 1)); long range = Math.min(size - skip, rnd.nextInt(CFG_BLOCK_SIZE * 400)); assertEqualStreams(new GridGgfsTestInputStream(size, salt), actIn, range, skip); return null; } }, READING_THREADS_CNT, "validate-multi-thread-reading"); expIn.close(); actIn.close(); info("Get stored file info: " + path); GridGgfsFile desc = fs.info(path); info("Validate stored file info: " + desc); assertNotNull(desc); if (log.isDebugEnabled()) log.debug("File descriptor: " + desc); Collection<GridGgfsBlockLocation> aff = fs.affinity(path, 0, desc.length()); assertFalse("Affinity: " + aff, desc.length() != 0 && aff.isEmpty()); int blockSize = desc.blockSize(); assertEquals("File size", size, desc.length()); assertEquals("Binary block size", CFG_BLOCK_SIZE, blockSize); // assertEquals("Permission", "rwxr-xr-x", desc.getPermission().toString()); // assertEquals("Permission sticky bit marks this is file", false, // desc.getPermission().getStickyBit()); assertEquals("Type", true, desc.isFile()); assertEquals("Type", false, desc.isDirectory()); info("Cleanup files: " + cleanUp); for (GridGgfsPath f : cleanUp) { fs.delete(f, true); assertNull(fs.info(f)); } }
/** @throws Exception If failed. */ public void testCreateFileFragmented() throws Exception { GridGgfsEx impl = (GridGgfsEx) grid(0).ggfs("ggfs"); GridGgfsFragmentizerManager fragmentizer = impl.context().fragmentizer(); GridTestUtils.setFieldValue(fragmentizer, "fragmentizerEnabled", false); GridGgfsPath path = new GridGgfsPath("/file"); try { GridGgfs fs0 = grid(0).ggfs("ggfs"); GridGgfs fs1 = grid(1).ggfs("ggfs"); GridGgfs fs2 = grid(2).ggfs("ggfs"); try (GridGgfsOutputStream out = fs0.create( path, 128, false, 1, CFG_GRP_SIZE, F.asMap(GridGgfs.PROP_PREFER_LOCAL_WRITES, "true"))) { // 1.5 blocks byte[] data = new byte[CFG_BLOCK_SIZE * 3 / 2]; Arrays.fill(data, (byte) 1); out.write(data); } try (GridGgfsOutputStream out = fs1.append(path, false)) { // 1.5 blocks. byte[] data = new byte[CFG_BLOCK_SIZE * 3 / 2]; Arrays.fill(data, (byte) 2); out.write(data); } // After this we should have first two block colocated with grid 0 and last block colocated // with grid 1. GridGgfsFileImpl fileImpl = (GridGgfsFileImpl) fs.info(path); GridCache<Object, Object> metaCache = grid(0).cachex(META_CACHE_NAME); GridGgfsFileInfo fileInfo = (GridGgfsFileInfo) metaCache.get(fileImpl.fileId()); GridGgfsFileMap map = fileInfo.fileMap(); List<GridGgfsFileAffinityRange> ranges = map.ranges(); assertEquals(2, ranges.size()); assertTrue(ranges.get(0).startOffset() == 0); assertTrue(ranges.get(0).endOffset() == 2 * CFG_BLOCK_SIZE - 1); assertTrue(ranges.get(1).startOffset() == 2 * CFG_BLOCK_SIZE); assertTrue(ranges.get(1).endOffset() == 3 * CFG_BLOCK_SIZE - 1); // Validate data read after colocated writes. try (GridGgfsInputStream in = fs2.open(path)) { // Validate first part of file. for (int i = 0; i < CFG_BLOCK_SIZE * 3 / 2; i++) assertEquals((byte) 1, in.read()); // Validate second part of file. for (int i = 0; i < CFG_BLOCK_SIZE * 3 / 2; i++) assertEquals((byte) 2, in.read()); assertEquals(-1, in.read()); } } finally { GridTestUtils.setFieldValue(fragmentizer, "fragmentizerEnabled", true); boolean hasData = false; for (int i = 0; i < NODES_CNT; i++) hasData |= !grid(i).cachex(DATA_CACHE_NAME).isEmpty(); assertTrue(hasData); fs.delete(path, true); } GridTestUtils.retryAssert( log, ASSERT_RETRIES, ASSERT_RETRY_INTERVAL, new CAX() { @Override public void applyx() { for (int i = 0; i < NODES_CNT; i++) assertTrue(grid(i).cachex(DATA_CACHE_NAME).isEmpty()); } }); }