/** * Increases priority if job has bumped down. * * @param waitJobs Ordered collection of collision contexts for jobs that are currently waiting * for execution. * @param passiveJobs Reordered collection of collision contexts for waiting jobs. */ private void bumpPriority( Collection<GridCollisionJobContext> waitJobs, List<GridCollisionJobContext> passiveJobs) { assert waitJobs != null; assert passiveJobs != null; assert waitJobs.size() == passiveJobs.size(); for (int i = 0; i < passiveJobs.size(); i++) { GridCollisionJobContext ctx = passiveJobs.get(i); if (i > indexOf(waitJobs, ctx)) ctx.getJobContext().setAttribute(jobAttrKey, getJobPriority(ctx) + starvationInc); } }
/** {@inheritDoc} */ @SuppressWarnings("TypeMayBeWeakened") @Nullable private Collection<Object> unmarshalFieldsCollection( @Nullable Collection<byte[]> byteCol, GridCacheContext<K, V> ctx, ClassLoader ldr) throws GridException { assert ctx != null; assert ldr != null; Collection<Object> col = unmarshalCollection(byteCol, ctx, ldr); Collection<Object> col0 = null; if (col != null) { col0 = new ArrayList<>(col.size()); for (Object o : col) { List<Object> list = (List<Object>) o; List<Object> list0 = new ArrayList<>(list.size()); for (Object obj : list) list0.add(obj != null ? ctx.marshaller().unmarshal((byte[]) obj, ldr) : null); col0.add(list0); } } return col0; }
/** {@inheritDoc} */ @SuppressWarnings({"unchecked"}) @Override public T reduce(List<GridJobResult> results) throws GridException { assert results != null; assert results.size() == 1; return (T) results.get(0).getData(); }
/** {@inheritDoc} */ @Override public Serializable reduce(List<GridComputeJobResult> results) throws GridException { if (log.isInfoEnabled()) log.info("Reducing job [job=" + this + ", results=" + results + ']'); if (results.size() > 1) fail(); return results.get(0).getData(); }
/** {@inheritDoc} */ @Override public Map<? extends GridComputeJob, GridNode> map(List<GridNode> subgrid, Integer arg) throws GridException { assert taskSes != null; assert arg != null; assert arg > 0; Map<GridSessionLoadTestJob, GridNode> map = new HashMap<>(subgrid.size()); Iterator<GridNode> iter = subgrid.iterator(); Random rnd = new Random(); params = new HashMap<>(arg); Collection<UUID> assigned = new ArrayList<>(subgrid.size()); for (int i = 0; i < arg; i++) { // Recycle iterator. if (!iter.hasNext()) iter = subgrid.iterator(); String paramName = UUID.randomUUID().toString(); int paramVal = rnd.nextInt(); taskSes.setAttribute(paramName, paramVal); GridNode node = iter.next(); assigned.add(node.id()); map.put(new GridSessionLoadTestJob(paramName), node); params.put(paramName, paramVal); if (log.isDebugEnabled()) log.debug("Set session attribute [name=" + paramName + ", value=" + paramVal + ']'); } taskSes.setAttribute("nodes", assigned); return map; }
/** {@inheritDoc} */ @SuppressWarnings("BusyWait") @Override public Boolean reduce(List<GridComputeJobResult> results) throws GridException { assert taskSes != null; assert results != null; assert params != null; assert !params.isEmpty(); assert results.size() == params.size(); Map<String, Integer> receivedParams = new HashMap<>(); boolean allAttrReceived = false; int cnt = 0; while (!allAttrReceived && cnt++ < 3) { allAttrReceived = true; for (Map.Entry<String, Integer> entry : params.entrySet()) { assert taskSes.getAttribute(entry.getKey()) != null; Integer newVal = (Integer) taskSes.getAttribute(entry.getKey()); assert newVal != null; receivedParams.put(entry.getKey(), newVal); if (newVal != entry.getValue() + 1) allAttrReceived = false; } if (!allAttrReceived) { try { Thread.sleep(100); } catch (InterruptedException e) { throw new GridException("Thread interrupted.", e); } } } if (log.isDebugEnabled()) { for (Map.Entry<String, Integer> entry : receivedParams.entrySet()) { log.debug( "Received session attr value [name=" + entry.getKey() + ", val=" + entry.getValue() + ", expected=" + (params.get(entry.getKey()) + 1) + ']'); } } return allAttrReceived; }
/** {@inheritDoc} */ @Override public void run() { List<Grid> clientGrids = runGrid(); assert clientGrids.size() == clientNodes; int threadsCnt = clientNodes * threadsPerClient; Executor e = Executors.newFixedThreadPool(threadsCnt); for (Grid grid : clientGrids) { for (int j = 0; j < threadsPerClient; j++) e.execute(new GridJobLoadTestSubmitter(grid, taskParams, cancelRate, submitDelay)); } }
/** {@inheritDoc} */ @SuppressWarnings("all") @Override public boolean writeTo(ByteBuffer buf) { commState.setBuffer(buf); if (!super.writeTo(buf)) return false; if (!commState.typeWritten) { if (!commState.putByte(directType())) return false; commState.typeWritten = true; } switch (commState.idx) { case 8: if (nearKeyBytes != null) { if (commState.it == null) { if (!commState.putInt(nearKeyBytes.size())) return false; commState.it = nearKeyBytes.iterator(); } while (commState.it.hasNext() || commState.cur != NULL) { if (commState.cur == NULL) commState.cur = commState.it.next(); if (!commState.putByteArray((byte[]) commState.cur)) return false; commState.cur = NULL; } commState.it = null; } else { if (!commState.putInt(-1)) return false; } commState.idx++; } return true; }
/** * Creates REST request. * * @param cmd Command. * @param params Parameters. * @return REST request. * @throws GridException If creation failed. */ @Nullable private GridRestRequest createRequest( GridRestCommand cmd, Map<String, Object> params, ServletRequest req) throws GridException { GridRestRequest restReq; switch (cmd) { case CACHE_GET: case CACHE_GET_ALL: case CACHE_PUT: case CACHE_PUT_ALL: case CACHE_REMOVE: case CACHE_REMOVE_ALL: case CACHE_ADD: case CACHE_CAS: case CACHE_METRICS: case CACHE_REPLACE: case CACHE_DECREMENT: case CACHE_INCREMENT: case CACHE_APPEND: case CACHE_PREPEND: { GridRestCacheRequest restReq0 = new GridRestCacheRequest(); restReq0.cacheName((String) params.get("cacheName")); restReq0.key(params.get("key")); restReq0.value(params.get("val")); restReq0.value2(params.get("val2")); Object val1 = params.get("val1"); if (val1 != null) restReq0.value(val1); restReq0.cacheFlags(intValue("cacheFlags", params, 0)); restReq0.ttl(longValue("exp", params, null)); restReq0.initial(longValue("init", params, null)); restReq0.delta(longValue("delta", params, null)); if (cmd == CACHE_GET_ALL || cmd == CACHE_PUT_ALL || cmd == CACHE_REMOVE_ALL) { List<Object> keys = values("k", params); List<Object> vals = values("v", params); if (keys.size() < vals.size()) throw new GridException( "Number of keys must be greater or equals to number of values."); Map<Object, Object> map = U.newHashMap(keys.size()); Iterator<Object> keyIt = keys.iterator(); Iterator<Object> valIt = vals.iterator(); while (keyIt.hasNext()) map.put(keyIt.next(), valIt.hasNext() ? valIt.next() : null); restReq0.values(map); } restReq = restReq0; break; } case TOPOLOGY: case NODE: { GridRestTopologyRequest restReq0 = new GridRestTopologyRequest(); restReq0.includeMetrics(Boolean.parseBoolean((String) params.get("mtr"))); restReq0.includeAttributes(Boolean.parseBoolean((String) params.get("attr"))); restReq0.nodeIp((String) params.get("ip")); restReq0.nodeId(uuidValue("id", params)); restReq = restReq0; break; } case EXE: case RESULT: case NOOP: { GridRestTaskRequest restReq0 = new GridRestTaskRequest(); restReq0.taskId((String) params.get("id")); restReq0.taskName((String) params.get("name")); restReq0.params(values("p", params)); restReq0.async(Boolean.parseBoolean((String) params.get("async"))); restReq0.timeout(longValue("timeout", params, 0L)); restReq = restReq0; break; } case LOG: { GridRestLogRequest restReq0 = new GridRestLogRequest(); restReq0.path((String) params.get("path")); restReq0.from(intValue("from", params, -1)); restReq0.to(intValue("to", params, -1)); restReq = restReq0; break; } case VERSION: { restReq = new GridRestRequest(); break; } default: throw new GridException("Invalid command: " + cmd); } restReq.address(new InetSocketAddress(req.getRemoteAddr(), req.getRemotePort())); restReq.command(cmd); if (params.containsKey("gridgain.login") || params.containsKey("gridgain.password")) { GridSecurityCredentials cred = new GridSecurityCredentials( (String) params.get("gridgain.login"), (String) params.get("gridgain.password")); restReq.credentials(cred); } String clientId = (String) params.get("clientId"); try { if (clientId != null) restReq.clientId(UUID.fromString(clientId)); } catch (Exception ignored) { // Ignore invalid client id. Rest handler will process this logic. } String destId = (String) params.get("destId"); try { if (destId != null) restReq.destinationId(UUID.fromString(destId)); } catch (IllegalArgumentException ignored) { // Don't fail - try to execute locally. } String sesTokStr = (String) params.get("sessionToken"); try { if (sesTokStr != null) restReq.sessionToken(U.hexString2ByteArray(sesTokStr)); } catch (IllegalArgumentException ignored) { // Ignore invalid session token. } return restReq; }
/** * @param nodeId Primary node ID. * @param req Request. * @return Remote transaction. * @throws GridException If failed. * @throws GridDistributedLockCancelledException If lock has been cancelled. */ @SuppressWarnings({"RedundantTypeArguments"}) @Nullable public GridNearTxRemote<K, V> startRemoteTx(UUID nodeId, GridDhtLockRequest<K, V> req) throws GridException, GridDistributedLockCancelledException { List<byte[]> nearKeyBytes = req.nearKeyBytes(); GridNearTxRemote<K, V> tx = null; ClassLoader ldr = ctx.deploy().globalLoader(); if (ldr != null) { for (int i = 0; i < nearKeyBytes.size(); i++) { byte[] bytes = nearKeyBytes.get(i); if (bytes == null) continue; K key = req.nearKeys().get(i); Collection<GridCacheMvccCandidate<K>> cands = req.candidatesByIndex(i); if (log.isDebugEnabled()) log.debug("Unmarshalled key: " + key); GridNearCacheEntry<K, V> entry = null; while (true) { try { entry = peekExx(key); if (entry != null) { entry.keyBytes(bytes); // Handle implicit locks for pessimistic transactions. if (req.inTx()) { tx = ctx.tm().tx(req.version()); if (tx != null) tx.addWrite(key, bytes, null /*Value.*/, null /*Value bytes.*/); else { tx = new GridNearTxRemote<K, V>( nodeId, req.nearNodeId(), req.threadId(), req.version(), null, PESSIMISTIC, req.isolation(), req.isInvalidate(), req.timeout(), key, bytes, null, // Value. null, // Value bytes. ctx); if (tx.empty()) return tx; tx = ctx.tm().onCreated(tx); if (tx == null || !ctx.tm().onStarted(tx)) throw new GridCacheTxRollbackException( "Failed to acquire lock " + "(transaction has been completed): " + req.version()); } } // Add remote candidate before reordering. entry.addRemote( req.nodeId(), nodeId, req.threadId(), req.version(), req.timeout(), tx != null && tx.ec(), tx != null, tx != null && tx.implicitSingle()); // Remote candidates for ordered lock queuing. entry.addRemoteCandidates( cands, req.version(), req.committedVersions(), req.rolledbackVersions()); entry.orderOwned(req.version(), req.owned(entry.key())); } // Double-check in case if sender node left the grid. if (ctx.discovery().node(req.nodeId()) == null) { if (log.isDebugEnabled()) log.debug("Node requesting lock left grid (lock request will be ignored): " + req); if (tx != null) tx.rollback(); return null; } // Entry is legit. break; } catch (GridCacheEntryRemovedException ignored) { assert entry.obsoleteVersion() != null : "Obsolete flag not set on removed entry: " + entry; if (log.isDebugEnabled()) log.debug("Received entry removed exception (will retry on renewed entry): " + entry); if (tx != null) { tx.clearEntry(entry.key()); if (log.isDebugEnabled()) log.debug( "Cleared removed entry from remote transaction (will retry) [entry=" + entry + ", tx=" + tx + ']'); } } } } } else { String err = "Failed to acquire deployment class loader for message: " + req; U.warn(log, err); throw new GridException(err); } return tx; }
/** * Performs flush. * * @throws GridException If failed. */ private void doFlush() throws GridException { lastFlushTime = U.currentTimeMillis(); List<GridFuture> activeFuts0 = null; int doneCnt = 0; for (GridFuture<?> f : activeFuts) { if (!f.isDone()) { if (activeFuts0 == null) activeFuts0 = new ArrayList<>((int) (activeFuts.size() * 1.2)); activeFuts0.add(f); } else { f.get(); doneCnt++; } } if (activeFuts0 == null || activeFuts0.isEmpty()) return; while (true) { Queue<GridFuture<?>> q = null; for (Buffer buf : bufMappings.values()) { GridFuture<?> flushFut = buf.flush(); if (flushFut != null) { if (q == null) q = new ArrayDeque<>(bufMappings.size() * 2); q.add(flushFut); } } if (q != null) { assert !q.isEmpty(); boolean err = false; for (GridFuture fut = q.poll(); fut != null; fut = q.poll()) { try { fut.get(); } catch (GridException e) { if (log.isDebugEnabled()) log.debug("Failed to flush buffer: " + e); err = true; } } if (err) // Remaps needed - flush buffers. continue; } doneCnt = 0; for (int i = 0; i < activeFuts0.size(); i++) { GridFuture f = activeFuts0.get(i); if (f == null) doneCnt++; else if (f.isDone()) { f.get(); doneCnt++; activeFuts0.set(i, null); } else break; } if (doneCnt == activeFuts0.size()) return; } }
/** @throws Exception If failed. */ public void testCreateFileFragmented() throws Exception { GridGgfsEx impl = (GridGgfsEx) grid(0).ggfs("ggfs"); GridGgfsFragmentizerManager fragmentizer = impl.context().fragmentizer(); GridTestUtils.setFieldValue(fragmentizer, "fragmentizerEnabled", false); GridGgfsPath path = new GridGgfsPath("/file"); try { GridGgfs fs0 = grid(0).ggfs("ggfs"); GridGgfs fs1 = grid(1).ggfs("ggfs"); GridGgfs fs2 = grid(2).ggfs("ggfs"); try (GridGgfsOutputStream out = fs0.create( path, 128, false, 1, CFG_GRP_SIZE, F.asMap(GridGgfs.PROP_PREFER_LOCAL_WRITES, "true"))) { // 1.5 blocks byte[] data = new byte[CFG_BLOCK_SIZE * 3 / 2]; Arrays.fill(data, (byte) 1); out.write(data); } try (GridGgfsOutputStream out = fs1.append(path, false)) { // 1.5 blocks. byte[] data = new byte[CFG_BLOCK_SIZE * 3 / 2]; Arrays.fill(data, (byte) 2); out.write(data); } // After this we should have first two block colocated with grid 0 and last block colocated // with grid 1. GridGgfsFileImpl fileImpl = (GridGgfsFileImpl) fs.info(path); GridCache<Object, Object> metaCache = grid(0).cachex(META_CACHE_NAME); GridGgfsFileInfo fileInfo = (GridGgfsFileInfo) metaCache.get(fileImpl.fileId()); GridGgfsFileMap map = fileInfo.fileMap(); List<GridGgfsFileAffinityRange> ranges = map.ranges(); assertEquals(2, ranges.size()); assertTrue(ranges.get(0).startOffset() == 0); assertTrue(ranges.get(0).endOffset() == 2 * CFG_BLOCK_SIZE - 1); assertTrue(ranges.get(1).startOffset() == 2 * CFG_BLOCK_SIZE); assertTrue(ranges.get(1).endOffset() == 3 * CFG_BLOCK_SIZE - 1); // Validate data read after colocated writes. try (GridGgfsInputStream in = fs2.open(path)) { // Validate first part of file. for (int i = 0; i < CFG_BLOCK_SIZE * 3 / 2; i++) assertEquals((byte) 1, in.read()); // Validate second part of file. for (int i = 0; i < CFG_BLOCK_SIZE * 3 / 2; i++) assertEquals((byte) 2, in.read()); assertEquals(-1, in.read()); } } finally { GridTestUtils.setFieldValue(fragmentizer, "fragmentizerEnabled", true); boolean hasData = false; for (int i = 0; i < NODES_CNT; i++) hasData |= !grid(i).cachex(DATA_CACHE_NAME).isEmpty(); assertTrue(hasData); fs.delete(path, true); } GridTestUtils.retryAssert( log, ASSERT_RETRIES, ASSERT_RETRY_INTERVAL, new CAX() { @Override public void applyx() { for (int i = 0; i < NODES_CNT; i++) assertTrue(grid(i).cachex(DATA_CACHE_NAME).isEmpty()); } }); }
/** * Processes lock request. * * @param nodeId Sender node ID. * @param msg Lock request. */ @SuppressWarnings({"unchecked", "ThrowableInstanceNeverThrown"}) private void processLockRequest(UUID nodeId, GridDistributedLockRequest<K, V> msg) { assert !nodeId.equals(locNodeId); List<byte[]> keys = msg.keyBytes(); int cnt = keys.size(); GridReplicatedTxRemote<K, V> tx = null; GridDistributedLockResponse res; ClassLoader ldr = null; try { ldr = ctx.deploy().globalLoader(); if (ldr != null) { res = new GridDistributedLockResponse(msg.version(), msg.futureId(), cnt); for (int i = 0; i < keys.size(); i++) { byte[] bytes = keys.get(i); K key = msg.keys().get(i); Collection<GridCacheMvccCandidate<K>> cands = msg.candidatesByIndex(i); if (bytes == null) continue; if (log.isDebugEnabled()) log.debug("Unmarshalled key: " + key); GridDistributedCacheEntry<K, V> entry = null; while (true) { try { entry = entryexx(key); // Handle implicit locks for pessimistic transactions. if (msg.inTx()) { tx = ctx.tm().tx(msg.version()); if (tx != null) { if (msg.txRead()) tx.addRead(key, bytes); else tx.addWrite(key, bytes); } else { tx = new GridReplicatedTxRemote<K, V>( nodeId, msg.threadId(), msg.version(), null, PESSIMISTIC, msg.isolation(), msg.isInvalidate(), msg.timeout(), key, bytes, msg.txRead(), ctx); tx = ctx.tm().onCreated(tx); if (tx == null || !ctx.tm().onStarted(tx)) throw new GridCacheTxRollbackException( "Failed to acquire lock " + "(transaction has been completed): " + msg.version()); } } // Add remote candidate before reordering. entry.addRemote( msg.nodeId(), null, msg.threadId(), msg.version(), msg.timeout(), tx != null && tx.ec(), tx != null, tx != null && tx.implicitSingle()); // Remote candidates for ordered lock queuing. entry.addRemoteCandidates( cands, msg.version(), msg.committedVersions(), msg.rolledbackVersions()); // Double-check in case if sender node left the grid. if (ctx.discovery().node(msg.nodeId()) == null) { if (log.isDebugEnabled()) log.debug( "Node requesting lock left grid (lock request will be ignored): " + msg); if (tx != null) tx.rollback(); return; } res.setCandidates( i, entry.localCandidates(), ctx.tm().committedVersions(msg.version()), ctx.tm().rolledbackVersions(msg.version())); res.addValueBytes( entry.rawGet(), msg.returnValue(i) ? entry.valueBytes(null) : null, ctx); // Entry is legit. break; } catch (GridCacheEntryRemovedException ignored) { assert entry.obsoleteVersion() != null : "Obsolete flag not set on removed entry: " + entry; if (log.isDebugEnabled()) log.debug( "Received entry removed exception (will retry on renewed entry): " + entry); if (tx != null) { tx.clearEntry(entry.key()); if (log.isDebugEnabled()) log.debug( "Cleared removed entry from remote transaction (will retry) [entry=" + entry + ", tx=" + tx + ']'); } } } } } else { String err = "Failed to acquire deployment class for message: " + msg; U.warn(log, err); res = new GridDistributedLockResponse(msg.version(), msg.futureId(), new GridException(err)); } } catch (GridCacheTxRollbackException e) { if (log.isDebugEnabled()) log.debug("Received lock request for completed transaction (will ignore): " + e); res = new GridDistributedLockResponse(msg.version(), msg.futureId(), e); } catch (GridException e) { String err = "Failed to unmarshal at least one of the keys for lock request message: " + msg; log.error(err, e); res = new GridDistributedLockResponse(msg.version(), msg.futureId(), new GridException(err, e)); if (tx != null) tx.rollback(); } catch (GridDistributedLockCancelledException ignored) { // Received lock request for cancelled lock. if (log.isDebugEnabled()) log.debug("Received lock request for canceled lock (will ignore): " + msg); if (tx != null) tx.rollback(); // Don't send response back. return; } GridNode node = ctx.discovery().node(msg.nodeId()); boolean releaseAll = false; if (node != null) { try { // Reply back to sender. ctx.io().send(node, res); } catch (GridException e) { U.error(log, "Failed to send message to node (did the node leave grid?): " + node.id(), e); releaseAll = ldr != null; } } // If sender left grid, release all locks acquired so far. else releaseAll = ldr != null; // Release all locks because sender node left grid. if (releaseAll) { for (K key : msg.keys()) { while (true) { GridDistributedCacheEntry<K, V> entry = peekexx(key); try { if (entry != null) entry.removeExplicitNodeLocks(msg.nodeId()); break; } catch (GridCacheEntryRemovedException ignore) { if (log.isDebugEnabled()) log.debug( "Attempted to remove lock on removed entity during failure " + "of replicated lock request handling (will retry): " + entry); } } } U.warn( log, "Sender node left grid in the midst of lock acquisition (locks will be released)."); } }
/** {@inheritDoc} */ @Override public String reduce(List<GridComputeJobResult> results) throws GridException { assert results.size() == 1; return results.get(0).getData(); }
/** * @param seq Start/stop sequence. * @throws Exception If failed. */ private void checkSequence0(boolean[] seq) throws Exception { try { startGrid(0); TreeSet<Integer> started = new TreeSet<>(); started.add(0); int topVer = 1; for (boolean start : seq) { if (start) { int nextIdx = nextIndex(started); startGrid(nextIdx); started.add(nextIdx); } else { int idx = started.last(); stopGrid(idx); started.remove(idx); } topVer++; info("Grid 0: " + grid(0).localNode().id()); ((GridKernal) grid(0)) .internalCache() .context() .affinity() .affinityReadyFuture(topVer) .get(); for (int i : started) { if (i != 0) { GridEx grid = grid(i); ((GridKernal) grid) .internalCache() .context() .affinity() .affinityReadyFuture(topVer) .get(); info("Grid " + i + ": " + grid.localNode().id()); for (int part = 0; part < parts; part++) { List<GridNode> firstNodes = (List<GridNode>) grid(0).cache(null).affinity().mapPartitionToPrimaryAndBackups(part); List<GridNode> secondNodes = (List<GridNode>) grid.cache(null).affinity().mapPartitionToPrimaryAndBackups(part); assertEquals(firstNodes.size(), secondNodes.size()); for (int n = 0; n < firstNodes.size(); n++) assertEquals(firstNodes.get(n), secondNodes.get(n)); } } } } } finally { stopAllGrids(); } }