/** {@inheritDoc} */ @Override public Serializable reduce(List<GridComputeJobResult> results) throws GridException { if (log.isInfoEnabled()) log.info("Reducing job [job=" + this + ", results=" + results + ']'); if (results.size() > 1) fail(); return results.get(0).getData(); }
/** * Performs flush. * * @throws GridException If failed. */ private void doFlush() throws GridException { lastFlushTime = U.currentTimeMillis(); List<GridFuture> activeFuts0 = null; int doneCnt = 0; for (GridFuture<?> f : activeFuts) { if (!f.isDone()) { if (activeFuts0 == null) activeFuts0 = new ArrayList<>((int) (activeFuts.size() * 1.2)); activeFuts0.add(f); } else { f.get(); doneCnt++; } } if (activeFuts0 == null || activeFuts0.isEmpty()) return; while (true) { Queue<GridFuture<?>> q = null; for (Buffer buf : bufMappings.values()) { GridFuture<?> flushFut = buf.flush(); if (flushFut != null) { if (q == null) q = new ArrayDeque<>(bufMappings.size() * 2); q.add(flushFut); } } if (q != null) { assert !q.isEmpty(); boolean err = false; for (GridFuture fut = q.poll(); fut != null; fut = q.poll()) { try { fut.get(); } catch (GridException e) { if (log.isDebugEnabled()) log.debug("Failed to flush buffer: " + e); err = true; } } if (err) // Remaps needed - flush buffers. continue; } doneCnt = 0; for (int i = 0; i < activeFuts0.size(); i++) { GridFuture f = activeFuts0.get(i); if (f == null) doneCnt++; else if (f.isDone()) { f.get(); doneCnt++; activeFuts0.set(i, null); } else break; } if (doneCnt == activeFuts0.size()) return; } }
/** @throws Exception If failed. */ public void testCreateFileFragmented() throws Exception { GridGgfsEx impl = (GridGgfsEx) grid(0).ggfs("ggfs"); GridGgfsFragmentizerManager fragmentizer = impl.context().fragmentizer(); GridTestUtils.setFieldValue(fragmentizer, "fragmentizerEnabled", false); GridGgfsPath path = new GridGgfsPath("/file"); try { GridGgfs fs0 = grid(0).ggfs("ggfs"); GridGgfs fs1 = grid(1).ggfs("ggfs"); GridGgfs fs2 = grid(2).ggfs("ggfs"); try (GridGgfsOutputStream out = fs0.create( path, 128, false, 1, CFG_GRP_SIZE, F.asMap(GridGgfs.PROP_PREFER_LOCAL_WRITES, "true"))) { // 1.5 blocks byte[] data = new byte[CFG_BLOCK_SIZE * 3 / 2]; Arrays.fill(data, (byte) 1); out.write(data); } try (GridGgfsOutputStream out = fs1.append(path, false)) { // 1.5 blocks. byte[] data = new byte[CFG_BLOCK_SIZE * 3 / 2]; Arrays.fill(data, (byte) 2); out.write(data); } // After this we should have first two block colocated with grid 0 and last block colocated // with grid 1. GridGgfsFileImpl fileImpl = (GridGgfsFileImpl) fs.info(path); GridCache<Object, Object> metaCache = grid(0).cachex(META_CACHE_NAME); GridGgfsFileInfo fileInfo = (GridGgfsFileInfo) metaCache.get(fileImpl.fileId()); GridGgfsFileMap map = fileInfo.fileMap(); List<GridGgfsFileAffinityRange> ranges = map.ranges(); assertEquals(2, ranges.size()); assertTrue(ranges.get(0).startOffset() == 0); assertTrue(ranges.get(0).endOffset() == 2 * CFG_BLOCK_SIZE - 1); assertTrue(ranges.get(1).startOffset() == 2 * CFG_BLOCK_SIZE); assertTrue(ranges.get(1).endOffset() == 3 * CFG_BLOCK_SIZE - 1); // Validate data read after colocated writes. try (GridGgfsInputStream in = fs2.open(path)) { // Validate first part of file. for (int i = 0; i < CFG_BLOCK_SIZE * 3 / 2; i++) assertEquals((byte) 1, in.read()); // Validate second part of file. for (int i = 0; i < CFG_BLOCK_SIZE * 3 / 2; i++) assertEquals((byte) 2, in.read()); assertEquals(-1, in.read()); } } finally { GridTestUtils.setFieldValue(fragmentizer, "fragmentizerEnabled", true); boolean hasData = false; for (int i = 0; i < NODES_CNT; i++) hasData |= !grid(i).cachex(DATA_CACHE_NAME).isEmpty(); assertTrue(hasData); fs.delete(path, true); } GridTestUtils.retryAssert( log, ASSERT_RETRIES, ASSERT_RETRY_INTERVAL, new CAX() { @Override public void applyx() { for (int i = 0; i < NODES_CNT; i++) assertTrue(grid(i).cachex(DATA_CACHE_NAME).isEmpty()); } }); }
/** * This method is called to map or split grid task into multiple grid jobs. This is the first * method that gets called when task execution starts. * * @param data Task execution argument. Can be {@code null}. This is the same argument as the one * passed into {@code Grid#execute(...)} methods. * @param subgrid Nodes available for this task execution. Note that order of nodes is guaranteed * to be randomized by container. This ensures that every time you simply iterate through grid * nodes, the order of nodes will be random which over time should result into all nodes being * used equally. * @return Map of grid jobs assigned to subgrid node. Unless {@link * GridComputeTaskContinuousMapper} is injected into task, if {@code null} or empty map is * returned, exception will be thrown. * @throws GridException If mapping could not complete successfully. This exception will be thrown * out of {@link GridComputeTaskFuture#get()} method. */ @Override public Map<? extends GridComputeJob, GridNode> map( List<GridNode> subgrid, @Nullable final Collection<Integer> data) throws GridException { assert !subgrid.isEmpty(); // Give preference to wanted node. Otherwise, take the first one. GridNode targetNode = F.find( subgrid, subgrid.get(0), new GridPredicate<GridNode>() { @Override public boolean apply(GridNode e) { return preferredNode.equals(e.id()); } }); return Collections.singletonMap( new GridComputeJobAdapter() { @GridLoggerResource private GridLogger log; @GridInstanceResource private Grid grid; @Override public Object execute() throws GridException { log.info("Going to put data: " + data.size()); GridCache<Object, Object> cache = grid.cache(cacheName); assert cache != null; Map<Integer, T2<Integer, Collection<Integer>>> putMap = groupData(data); for (Map.Entry<Integer, T2<Integer, Collection<Integer>>> entry : putMap.entrySet()) { T2<Integer, Collection<Integer>> pair = entry.getValue(); Object affKey = pair.get1(); // Group lock partition. try (GridCacheTx tx = cache.txStartPartition( cache.affinity().partition(affKey), optimistic ? OPTIMISTIC : PESSIMISTIC, REPEATABLE_READ, 0, pair.get2().size())) { for (Integer val : pair.get2()) cache.put(val, val); tx.commit(); } } log.info("Finished put data: " + data.size()); return data; } /** * Groups values by partitions. * * @param data Data to put. * @return Grouped map. */ private Map<Integer, T2<Integer, Collection<Integer>>> groupData(Iterable<Integer> data) { GridCache<Object, Object> cache = grid.cache(cacheName); Map<Integer, T2<Integer, Collection<Integer>>> res = new HashMap<>(); for (Integer val : data) { int part = cache.affinity().partition(val); T2<Integer, Collection<Integer>> tup = res.get(part); if (tup == null) { tup = new T2<Integer, Collection<Integer>>(val, new LinkedList<Integer>()); res.put(part, tup); } tup.get2().add(val); } return res; } }, targetNode); }