/** {@inheritDoc} */ @SuppressWarnings("TypeMayBeWeakened") @Nullable private Collection<byte[]> marshalFieldsCollection( @Nullable Collection<Object> col, GridCacheContext<K, V> ctx) throws GridException { assert ctx != null; if (col == null) return null; Collection<List<Object>> col0 = new ArrayList<>(col.size()); for (Object o : col) { List<GridIndexingEntity<?>> list = (List<GridIndexingEntity<?>>) o; List<Object> list0 = new ArrayList<>(list.size()); for (GridIndexingEntity<?> ent : list) { if (ent.bytes() != null) list0.add(ent.bytes()); else { if (ctx.deploymentEnabled()) prepareObject(ent.value(), ctx); list0.add(CU.marshal(ctx, ent.value())); } } col0.add(list0); } return marshalCollection(col0, ctx); }
/** {@inheritDoc} */ @SuppressWarnings("TypeMayBeWeakened") @Nullable private Collection<Object> unmarshalFieldsCollection( @Nullable Collection<byte[]> byteCol, GridCacheContext<K, V> ctx, ClassLoader ldr) throws GridException { assert ctx != null; assert ldr != null; Collection<Object> col = unmarshalCollection(byteCol, ctx, ldr); Collection<Object> col0 = null; if (col != null) { col0 = new ArrayList<>(col.size()); for (Object o : col) { List<Object> list = (List<Object>) o; List<Object> list0 = new ArrayList<>(list.size()); for (Object obj : list) list0.add(obj != null ? ctx.marshaller().unmarshal((byte[]) obj, ldr) : null); col0.add(list0); } } return col0; }
/** {@inheritDoc} */ @Override public Serializable reduce(List<GridComputeJobResult> results) throws GridException { if (log.isInfoEnabled()) log.info("Reducing job [job=" + this + ", results=" + results + ']'); if (results.size() > 1) fail(); return results.get(0).getData(); }
/** * Gets values referenced by sequential keys, e.g. {@code key1...keyN}. * * @param keyPrefix Key prefix, e.g. {@code key} for {@code key1...keyN}. * @param params Parameters map. * @return Values. */ @Nullable protected List<Object> values(String keyPrefix, Map<String, Object> params) { assert keyPrefix != null; List<Object> vals = new LinkedList<>(); for (int i = 1; ; i++) { String key = keyPrefix + i; if (params.containsKey(key)) vals.add(params.get(key)); else break; } return vals; }
/** {@inheritDoc} */ @Override public void run() { List<Grid> clientGrids = runGrid(); assert clientGrids.size() == clientNodes; int threadsCnt = clientNodes * threadsPerClient; Executor e = Executors.newFixedThreadPool(threadsCnt); for (Grid grid : clientGrids) { for (int j = 0; j < threadsPerClient; j++) e.execute(new GridJobLoadTestSubmitter(grid, taskParams, cancelRate, submitDelay)); } }
/** * Adds a Near key. * * @param key Key. * @param keyBytes Key bytes. * @param ctx Context. * @throws GridException If failed. */ public void addNearKey(K key, byte[] keyBytes, GridCacheContext<K, V> ctx) throws GridException { if (ctx.deploymentEnabled()) prepareObject(key, ctx); if (nearKeyBytes == null) nearKeyBytes = new ArrayList<>(); nearKeyBytes.add(keyBytes); }
/** * Run all grid nodes as defined in test configuration. * * @return list of run nodes. */ private List<Grid> runGrid() { List<Grid> clientGrids = new ArrayList<>(clientNodes); try { loadTestConfiguration(); for (int i = 0; i < srvNodes; i++) startNode("server", SERVER_NODE_CONFIGURATION); // Start clients in the second order to cache a client node in GridGain. for (int i = 0; i < clientNodes; i++) clientGrids.add(startNode("client", CLIENT_NODE_CONFIGURATION)); } catch (Exception e) { throw new RuntimeException(e); } return clientGrids; }
/** {@inheritDoc} */ @SuppressWarnings("all") @Override public boolean writeTo(ByteBuffer buf) { commState.setBuffer(buf); if (!super.writeTo(buf)) return false; if (!commState.typeWritten) { if (!commState.putByte(directType())) return false; commState.typeWritten = true; } switch (commState.idx) { case 8: if (nearKeyBytes != null) { if (commState.it == null) { if (!commState.putInt(nearKeyBytes.size())) return false; commState.it = nearKeyBytes.iterator(); } while (commState.it.hasNext() || commState.cur != NULL) { if (commState.cur == NULL) commState.cur = commState.it.next(); if (!commState.putByteArray((byte[]) commState.cur)) return false; commState.cur = NULL; } commState.it = null; } else { if (!commState.putInt(-1)) return false; } commState.idx++; } return true; }
/** {@inheritDoc} */ @SuppressWarnings("all") @Override public boolean readFrom(ByteBuffer buf) { commState.setBuffer(buf); if (!super.readFrom(buf)) return false; switch (commState.idx) { case 8: if (commState.readSize == -1) { if (buf.remaining() < 4) return false; commState.readSize = commState.getInt(); } if (commState.readSize >= 0) { if (nearKeyBytes == null) nearKeyBytes = new ArrayList<>(commState.readSize); for (int i = commState.readItems; i < commState.readSize; i++) { byte[] _val = commState.getByteArray(); if (_val == BYTE_ARR_NOT_READ) return false; nearKeyBytes.add((byte[]) _val); commState.readItems++; } } commState.readSize = -1; commState.readItems = 0; commState.idx++; } return true; }
/** * Creates REST request. * * @param cmd Command. * @param params Parameters. * @return REST request. * @throws GridException If creation failed. */ @Nullable private GridRestRequest createRequest( GridRestCommand cmd, Map<String, Object> params, ServletRequest req) throws GridException { GridRestRequest restReq; switch (cmd) { case CACHE_GET: case CACHE_GET_ALL: case CACHE_PUT: case CACHE_PUT_ALL: case CACHE_REMOVE: case CACHE_REMOVE_ALL: case CACHE_ADD: case CACHE_CAS: case CACHE_METRICS: case CACHE_REPLACE: case CACHE_DECREMENT: case CACHE_INCREMENT: case CACHE_APPEND: case CACHE_PREPEND: { GridRestCacheRequest restReq0 = new GridRestCacheRequest(); restReq0.cacheName((String) params.get("cacheName")); restReq0.key(params.get("key")); restReq0.value(params.get("val")); restReq0.value2(params.get("val2")); Object val1 = params.get("val1"); if (val1 != null) restReq0.value(val1); restReq0.cacheFlags(intValue("cacheFlags", params, 0)); restReq0.ttl(longValue("exp", params, null)); restReq0.initial(longValue("init", params, null)); restReq0.delta(longValue("delta", params, null)); if (cmd == CACHE_GET_ALL || cmd == CACHE_PUT_ALL || cmd == CACHE_REMOVE_ALL) { List<Object> keys = values("k", params); List<Object> vals = values("v", params); if (keys.size() < vals.size()) throw new GridException( "Number of keys must be greater or equals to number of values."); Map<Object, Object> map = U.newHashMap(keys.size()); Iterator<Object> keyIt = keys.iterator(); Iterator<Object> valIt = vals.iterator(); while (keyIt.hasNext()) map.put(keyIt.next(), valIt.hasNext() ? valIt.next() : null); restReq0.values(map); } restReq = restReq0; break; } case TOPOLOGY: case NODE: { GridRestTopologyRequest restReq0 = new GridRestTopologyRequest(); restReq0.includeMetrics(Boolean.parseBoolean((String) params.get("mtr"))); restReq0.includeAttributes(Boolean.parseBoolean((String) params.get("attr"))); restReq0.nodeIp((String) params.get("ip")); restReq0.nodeId(uuidValue("id", params)); restReq = restReq0; break; } case EXE: case RESULT: case NOOP: { GridRestTaskRequest restReq0 = new GridRestTaskRequest(); restReq0.taskId((String) params.get("id")); restReq0.taskName((String) params.get("name")); restReq0.params(values("p", params)); restReq0.async(Boolean.parseBoolean((String) params.get("async"))); restReq0.timeout(longValue("timeout", params, 0L)); restReq = restReq0; break; } case LOG: { GridRestLogRequest restReq0 = new GridRestLogRequest(); restReq0.path((String) params.get("path")); restReq0.from(intValue("from", params, -1)); restReq0.to(intValue("to", params, -1)); restReq = restReq0; break; } case VERSION: { restReq = new GridRestRequest(); break; } default: throw new GridException("Invalid command: " + cmd); } restReq.address(new InetSocketAddress(req.getRemoteAddr(), req.getRemotePort())); restReq.command(cmd); if (params.containsKey("gridgain.login") || params.containsKey("gridgain.password")) { GridSecurityCredentials cred = new GridSecurityCredentials( (String) params.get("gridgain.login"), (String) params.get("gridgain.password")); restReq.credentials(cred); } String clientId = (String) params.get("clientId"); try { if (clientId != null) restReq.clientId(UUID.fromString(clientId)); } catch (Exception ignored) { // Ignore invalid client id. Rest handler will process this logic. } String destId = (String) params.get("destId"); try { if (destId != null) restReq.destinationId(UUID.fromString(destId)); } catch (IllegalArgumentException ignored) { // Don't fail - try to execute locally. } String sesTokStr = (String) params.get("sessionToken"); try { if (sesTokStr != null) restReq.sessionToken(U.hexString2ByteArray(sesTokStr)); } catch (IllegalArgumentException ignored) { // Ignore invalid session token. } return restReq; }
/** * Performs flush. * * @throws GridException If failed. */ private void doFlush() throws GridException { lastFlushTime = U.currentTimeMillis(); List<GridFuture> activeFuts0 = null; int doneCnt = 0; for (GridFuture<?> f : activeFuts) { if (!f.isDone()) { if (activeFuts0 == null) activeFuts0 = new ArrayList<>((int) (activeFuts.size() * 1.2)); activeFuts0.add(f); } else { f.get(); doneCnt++; } } if (activeFuts0 == null || activeFuts0.isEmpty()) return; while (true) { Queue<GridFuture<?>> q = null; for (Buffer buf : bufMappings.values()) { GridFuture<?> flushFut = buf.flush(); if (flushFut != null) { if (q == null) q = new ArrayDeque<>(bufMappings.size() * 2); q.add(flushFut); } } if (q != null) { assert !q.isEmpty(); boolean err = false; for (GridFuture fut = q.poll(); fut != null; fut = q.poll()) { try { fut.get(); } catch (GridException e) { if (log.isDebugEnabled()) log.debug("Failed to flush buffer: " + e); err = true; } } if (err) // Remaps needed - flush buffers. continue; } doneCnt = 0; for (int i = 0; i < activeFuts0.size(); i++) { GridFuture f = activeFuts0.get(i); if (f == null) doneCnt++; else if (f.isDone()) { f.get(); doneCnt++; activeFuts0.set(i, null); } else break; } if (doneCnt == activeFuts0.size()) return; } }
/** @throws Exception If failed. */ public void testCreateFileFragmented() throws Exception { GridGgfsEx impl = (GridGgfsEx) grid(0).ggfs("ggfs"); GridGgfsFragmentizerManager fragmentizer = impl.context().fragmentizer(); GridTestUtils.setFieldValue(fragmentizer, "fragmentizerEnabled", false); GridGgfsPath path = new GridGgfsPath("/file"); try { GridGgfs fs0 = grid(0).ggfs("ggfs"); GridGgfs fs1 = grid(1).ggfs("ggfs"); GridGgfs fs2 = grid(2).ggfs("ggfs"); try (GridGgfsOutputStream out = fs0.create( path, 128, false, 1, CFG_GRP_SIZE, F.asMap(GridGgfs.PROP_PREFER_LOCAL_WRITES, "true"))) { // 1.5 blocks byte[] data = new byte[CFG_BLOCK_SIZE * 3 / 2]; Arrays.fill(data, (byte) 1); out.write(data); } try (GridGgfsOutputStream out = fs1.append(path, false)) { // 1.5 blocks. byte[] data = new byte[CFG_BLOCK_SIZE * 3 / 2]; Arrays.fill(data, (byte) 2); out.write(data); } // After this we should have first two block colocated with grid 0 and last block colocated // with grid 1. GridGgfsFileImpl fileImpl = (GridGgfsFileImpl) fs.info(path); GridCache<Object, Object> metaCache = grid(0).cachex(META_CACHE_NAME); GridGgfsFileInfo fileInfo = (GridGgfsFileInfo) metaCache.get(fileImpl.fileId()); GridGgfsFileMap map = fileInfo.fileMap(); List<GridGgfsFileAffinityRange> ranges = map.ranges(); assertEquals(2, ranges.size()); assertTrue(ranges.get(0).startOffset() == 0); assertTrue(ranges.get(0).endOffset() == 2 * CFG_BLOCK_SIZE - 1); assertTrue(ranges.get(1).startOffset() == 2 * CFG_BLOCK_SIZE); assertTrue(ranges.get(1).endOffset() == 3 * CFG_BLOCK_SIZE - 1); // Validate data read after colocated writes. try (GridGgfsInputStream in = fs2.open(path)) { // Validate first part of file. for (int i = 0; i < CFG_BLOCK_SIZE * 3 / 2; i++) assertEquals((byte) 1, in.read()); // Validate second part of file. for (int i = 0; i < CFG_BLOCK_SIZE * 3 / 2; i++) assertEquals((byte) 2, in.read()); assertEquals(-1, in.read()); } } finally { GridTestUtils.setFieldValue(fragmentizer, "fragmentizerEnabled", true); boolean hasData = false; for (int i = 0; i < NODES_CNT; i++) hasData |= !grid(i).cachex(DATA_CACHE_NAME).isEmpty(); assertTrue(hasData); fs.delete(path, true); } GridTestUtils.retryAssert( log, ASSERT_RETRIES, ASSERT_RETRY_INTERVAL, new CAX() { @Override public void applyx() { for (int i = 0; i < NODES_CNT; i++) assertTrue(grid(i).cachex(DATA_CACHE_NAME).isEmpty()); } }); }