/** {@inheritDoc} */ @SuppressWarnings("TypeMayBeWeakened") @Nullable private Collection<Object> unmarshalFieldsCollection( @Nullable Collection<byte[]> byteCol, GridCacheContext<K, V> ctx, ClassLoader ldr) throws GridException { assert ctx != null; assert ldr != null; Collection<Object> col = unmarshalCollection(byteCol, ctx, ldr); Collection<Object> col0 = null; if (col != null) { col0 = new ArrayList<>(col.size()); for (Object o : col) { List<Object> list = (List<Object>) o; List<Object> list0 = new ArrayList<>(list.size()); for (Object obj : list) list0.add(obj != null ? ctx.marshaller().unmarshal((byte[]) obj, ldr) : null); col0.add(list0); } } return col0; }
/** {@inheritDoc} */ @SuppressWarnings("TypeMayBeWeakened") @Nullable private Collection<byte[]> marshalFieldsCollection( @Nullable Collection<Object> col, GridCacheContext<K, V> ctx) throws GridException { assert ctx != null; if (col == null) return null; Collection<List<Object>> col0 = new ArrayList<>(col.size()); for (Object o : col) { List<GridIndexingEntity<?>> list = (List<GridIndexingEntity<?>>) o; List<Object> list0 = new ArrayList<>(list.size()); for (GridIndexingEntity<?> ent : list) { if (ent.bytes() != null) list0.add(ent.bytes()); else { if (ctx.deploymentEnabled()) prepareObject(ent.value(), ctx); list0.add(CU.marshal(ctx, ent.value())); } } col0.add(list0); } return marshalCollection(col0, ctx); }
/** {@inheritDoc} */ @Override public Serializable reduce(List<GridComputeJobResult> results) throws GridException { if (log.isInfoEnabled()) log.info("Reducing job [job=" + this + ", results=" + results + ']'); if (results.size() > 1) fail(); return results.get(0).getData(); }
/** * Gets values referenced by sequential keys, e.g. {@code key1...keyN}. * * @param keyPrefix Key prefix, e.g. {@code key} for {@code key1...keyN}. * @param params Parameters map. * @return Values. */ @Nullable protected List<Object> values(String keyPrefix, Map<String, Object> params) { assert keyPrefix != null; List<Object> vals = new LinkedList<>(); for (int i = 1; ; i++) { String key = keyPrefix + i; if (params.containsKey(key)) vals.add(params.get(key)); else break; } return vals; }
/** {@inheritDoc} */ @Override public void run() { List<Grid> clientGrids = runGrid(); assert clientGrids.size() == clientNodes; int threadsCnt = clientNodes * threadsPerClient; Executor e = Executors.newFixedThreadPool(threadsCnt); for (Grid grid : clientGrids) { for (int j = 0; j < threadsPerClient; j++) e.execute(new GridJobLoadTestSubmitter(grid, taskParams, cancelRate, submitDelay)); } }
/** * Run all grid nodes as defined in test configuration. * * @return list of run nodes. */ private List<Grid> runGrid() { List<Grid> clientGrids = new ArrayList<>(clientNodes); try { loadTestConfiguration(); for (int i = 0; i < srvNodes; i++) startNode("server", SERVER_NODE_CONFIGURATION); // Start clients in the second order to cache a client node in GridGain. for (int i = 0; i < clientNodes; i++) clientGrids.add(startNode("client", CLIENT_NODE_CONFIGURATION)); } catch (Exception e) { throw new RuntimeException(e); } return clientGrids; }
/** * Creates REST request. * * @param cmd Command. * @param params Parameters. * @return REST request. * @throws GridException If creation failed. */ @Nullable private GridRestRequest createRequest( GridRestCommand cmd, Map<String, Object> params, ServletRequest req) throws GridException { GridRestRequest restReq; switch (cmd) { case CACHE_GET: case CACHE_GET_ALL: case CACHE_PUT: case CACHE_PUT_ALL: case CACHE_REMOVE: case CACHE_REMOVE_ALL: case CACHE_ADD: case CACHE_CAS: case CACHE_METRICS: case CACHE_REPLACE: case CACHE_DECREMENT: case CACHE_INCREMENT: case CACHE_APPEND: case CACHE_PREPEND: { GridRestCacheRequest restReq0 = new GridRestCacheRequest(); restReq0.cacheName((String) params.get("cacheName")); restReq0.key(params.get("key")); restReq0.value(params.get("val")); restReq0.value2(params.get("val2")); Object val1 = params.get("val1"); if (val1 != null) restReq0.value(val1); restReq0.cacheFlags(intValue("cacheFlags", params, 0)); restReq0.ttl(longValue("exp", params, null)); restReq0.initial(longValue("init", params, null)); restReq0.delta(longValue("delta", params, null)); if (cmd == CACHE_GET_ALL || cmd == CACHE_PUT_ALL || cmd == CACHE_REMOVE_ALL) { List<Object> keys = values("k", params); List<Object> vals = values("v", params); if (keys.size() < vals.size()) throw new GridException( "Number of keys must be greater or equals to number of values."); Map<Object, Object> map = U.newHashMap(keys.size()); Iterator<Object> keyIt = keys.iterator(); Iterator<Object> valIt = vals.iterator(); while (keyIt.hasNext()) map.put(keyIt.next(), valIt.hasNext() ? valIt.next() : null); restReq0.values(map); } restReq = restReq0; break; } case TOPOLOGY: case NODE: { GridRestTopologyRequest restReq0 = new GridRestTopologyRequest(); restReq0.includeMetrics(Boolean.parseBoolean((String) params.get("mtr"))); restReq0.includeAttributes(Boolean.parseBoolean((String) params.get("attr"))); restReq0.nodeIp((String) params.get("ip")); restReq0.nodeId(uuidValue("id", params)); restReq = restReq0; break; } case EXE: case RESULT: case NOOP: { GridRestTaskRequest restReq0 = new GridRestTaskRequest(); restReq0.taskId((String) params.get("id")); restReq0.taskName((String) params.get("name")); restReq0.params(values("p", params)); restReq0.async(Boolean.parseBoolean((String) params.get("async"))); restReq0.timeout(longValue("timeout", params, 0L)); restReq = restReq0; break; } case LOG: { GridRestLogRequest restReq0 = new GridRestLogRequest(); restReq0.path((String) params.get("path")); restReq0.from(intValue("from", params, -1)); restReq0.to(intValue("to", params, -1)); restReq = restReq0; break; } case VERSION: { restReq = new GridRestRequest(); break; } default: throw new GridException("Invalid command: " + cmd); } restReq.address(new InetSocketAddress(req.getRemoteAddr(), req.getRemotePort())); restReq.command(cmd); if (params.containsKey("gridgain.login") || params.containsKey("gridgain.password")) { GridSecurityCredentials cred = new GridSecurityCredentials( (String) params.get("gridgain.login"), (String) params.get("gridgain.password")); restReq.credentials(cred); } String clientId = (String) params.get("clientId"); try { if (clientId != null) restReq.clientId(UUID.fromString(clientId)); } catch (Exception ignored) { // Ignore invalid client id. Rest handler will process this logic. } String destId = (String) params.get("destId"); try { if (destId != null) restReq.destinationId(UUID.fromString(destId)); } catch (IllegalArgumentException ignored) { // Don't fail - try to execute locally. } String sesTokStr = (String) params.get("sessionToken"); try { if (sesTokStr != null) restReq.sessionToken(U.hexString2ByteArray(sesTokStr)); } catch (IllegalArgumentException ignored) { // Ignore invalid session token. } return restReq; }
/** * Performs flush. * * @throws GridException If failed. */ private void doFlush() throws GridException { lastFlushTime = U.currentTimeMillis(); List<GridFuture> activeFuts0 = null; int doneCnt = 0; for (GridFuture<?> f : activeFuts) { if (!f.isDone()) { if (activeFuts0 == null) activeFuts0 = new ArrayList<>((int) (activeFuts.size() * 1.2)); activeFuts0.add(f); } else { f.get(); doneCnt++; } } if (activeFuts0 == null || activeFuts0.isEmpty()) return; while (true) { Queue<GridFuture<?>> q = null; for (Buffer buf : bufMappings.values()) { GridFuture<?> flushFut = buf.flush(); if (flushFut != null) { if (q == null) q = new ArrayDeque<>(bufMappings.size() * 2); q.add(flushFut); } } if (q != null) { assert !q.isEmpty(); boolean err = false; for (GridFuture fut = q.poll(); fut != null; fut = q.poll()) { try { fut.get(); } catch (GridException e) { if (log.isDebugEnabled()) log.debug("Failed to flush buffer: " + e); err = true; } } if (err) // Remaps needed - flush buffers. continue; } doneCnt = 0; for (int i = 0; i < activeFuts0.size(); i++) { GridFuture f = activeFuts0.get(i); if (f == null) doneCnt++; else if (f.isDone()) { f.get(); doneCnt++; activeFuts0.set(i, null); } else break; } if (doneCnt == activeFuts0.size()) return; } }
/** @throws Exception If failed. */ public void testCreateFileFragmented() throws Exception { GridGgfsEx impl = (GridGgfsEx) grid(0).ggfs("ggfs"); GridGgfsFragmentizerManager fragmentizer = impl.context().fragmentizer(); GridTestUtils.setFieldValue(fragmentizer, "fragmentizerEnabled", false); GridGgfsPath path = new GridGgfsPath("/file"); try { GridGgfs fs0 = grid(0).ggfs("ggfs"); GridGgfs fs1 = grid(1).ggfs("ggfs"); GridGgfs fs2 = grid(2).ggfs("ggfs"); try (GridGgfsOutputStream out = fs0.create( path, 128, false, 1, CFG_GRP_SIZE, F.asMap(GridGgfs.PROP_PREFER_LOCAL_WRITES, "true"))) { // 1.5 blocks byte[] data = new byte[CFG_BLOCK_SIZE * 3 / 2]; Arrays.fill(data, (byte) 1); out.write(data); } try (GridGgfsOutputStream out = fs1.append(path, false)) { // 1.5 blocks. byte[] data = new byte[CFG_BLOCK_SIZE * 3 / 2]; Arrays.fill(data, (byte) 2); out.write(data); } // After this we should have first two block colocated with grid 0 and last block colocated // with grid 1. GridGgfsFileImpl fileImpl = (GridGgfsFileImpl) fs.info(path); GridCache<Object, Object> metaCache = grid(0).cachex(META_CACHE_NAME); GridGgfsFileInfo fileInfo = (GridGgfsFileInfo) metaCache.get(fileImpl.fileId()); GridGgfsFileMap map = fileInfo.fileMap(); List<GridGgfsFileAffinityRange> ranges = map.ranges(); assertEquals(2, ranges.size()); assertTrue(ranges.get(0).startOffset() == 0); assertTrue(ranges.get(0).endOffset() == 2 * CFG_BLOCK_SIZE - 1); assertTrue(ranges.get(1).startOffset() == 2 * CFG_BLOCK_SIZE); assertTrue(ranges.get(1).endOffset() == 3 * CFG_BLOCK_SIZE - 1); // Validate data read after colocated writes. try (GridGgfsInputStream in = fs2.open(path)) { // Validate first part of file. for (int i = 0; i < CFG_BLOCK_SIZE * 3 / 2; i++) assertEquals((byte) 1, in.read()); // Validate second part of file. for (int i = 0; i < CFG_BLOCK_SIZE * 3 / 2; i++) assertEquals((byte) 2, in.read()); assertEquals(-1, in.read()); } } finally { GridTestUtils.setFieldValue(fragmentizer, "fragmentizerEnabled", true); boolean hasData = false; for (int i = 0; i < NODES_CNT; i++) hasData |= !grid(i).cachex(DATA_CACHE_NAME).isEmpty(); assertTrue(hasData); fs.delete(path, true); } GridTestUtils.retryAssert( log, ASSERT_RETRIES, ASSERT_RETRY_INTERVAL, new CAX() { @Override public void applyx() { for (int i = 0; i < NODES_CNT; i++) assertTrue(grid(i).cachex(DATA_CACHE_NAME).isEmpty()); } }); }
/** * This method is called to map or split grid task into multiple grid jobs. This is the first * method that gets called when task execution starts. * * @param data Task execution argument. Can be {@code null}. This is the same argument as the one * passed into {@code Grid#execute(...)} methods. * @param subgrid Nodes available for this task execution. Note that order of nodes is guaranteed * to be randomized by container. This ensures that every time you simply iterate through grid * nodes, the order of nodes will be random which over time should result into all nodes being * used equally. * @return Map of grid jobs assigned to subgrid node. Unless {@link * GridComputeTaskContinuousMapper} is injected into task, if {@code null} or empty map is * returned, exception will be thrown. * @throws GridException If mapping could not complete successfully. This exception will be thrown * out of {@link GridComputeTaskFuture#get()} method. */ @Override public Map<? extends GridComputeJob, GridNode> map( List<GridNode> subgrid, @Nullable final Collection<Integer> data) throws GridException { assert !subgrid.isEmpty(); // Give preference to wanted node. Otherwise, take the first one. GridNode targetNode = F.find( subgrid, subgrid.get(0), new GridPredicate<GridNode>() { @Override public boolean apply(GridNode e) { return preferredNode.equals(e.id()); } }); return Collections.singletonMap( new GridComputeJobAdapter() { @GridLoggerResource private GridLogger log; @GridInstanceResource private Grid grid; @Override public Object execute() throws GridException { log.info("Going to put data: " + data.size()); GridCache<Object, Object> cache = grid.cache(cacheName); assert cache != null; Map<Integer, T2<Integer, Collection<Integer>>> putMap = groupData(data); for (Map.Entry<Integer, T2<Integer, Collection<Integer>>> entry : putMap.entrySet()) { T2<Integer, Collection<Integer>> pair = entry.getValue(); Object affKey = pair.get1(); // Group lock partition. try (GridCacheTx tx = cache.txStartPartition( cache.affinity().partition(affKey), optimistic ? OPTIMISTIC : PESSIMISTIC, REPEATABLE_READ, 0, pair.get2().size())) { for (Integer val : pair.get2()) cache.put(val, val); tx.commit(); } } log.info("Finished put data: " + data.size()); return data; } /** * Groups values by partitions. * * @param data Data to put. * @return Grouped map. */ private Map<Integer, T2<Integer, Collection<Integer>>> groupData(Iterable<Integer> data) { GridCache<Object, Object> cache = grid.cache(cacheName); Map<Integer, T2<Integer, Collection<Integer>>> res = new HashMap<>(); for (Integer val : data) { int part = cache.affinity().partition(val); T2<Integer, Collection<Integer>> tup = res.get(part); if (tup == null) { tup = new T2<Integer, Collection<Integer>>(val, new LinkedList<Integer>()); res.put(part, tup); } tup.get2().add(val); } return res; } }, targetNode); }