/** {@inheritDoc} */
@SuppressWarnings("TypeMayBeWeakened")
@Nullable
private Collection<byte[]> marshalFieldsCollection(
@Nullable Collection<Object> col, GridCacheContext<K, V> ctx) throws GridException {
assert ctx != null;
if (col == null) return null;
Collection<List<Object>> col0 = new ArrayList<>(col.size());
for (Object o : col) {
List<GridIndexingEntity<?>> list = (List<GridIndexingEntity<?>>) o;
List<Object> list0 = new ArrayList<>(list.size());
for (GridIndexingEntity<?> ent : list) {
if (ent.bytes() != null) list0.add(ent.bytes());
else {
if (ctx.deploymentEnabled()) prepareObject(ent.value(), ctx);
list0.add(CU.marshal(ctx, ent.value()));
}
}
col0.add(list0);
}
return marshalCollection(col0, ctx);
}
/** {@inheritDoc} */
@SuppressWarnings("TypeMayBeWeakened")
@Nullable
private Collection<Object> unmarshalFieldsCollection(
@Nullable Collection<byte[]> byteCol, GridCacheContext<K, V> ctx, ClassLoader ldr)
throws GridException {
assert ctx != null;
assert ldr != null;
Collection<Object> col = unmarshalCollection(byteCol, ctx, ldr);
Collection<Object> col0 = null;
if (col != null) {
col0 = new ArrayList<>(col.size());
for (Object o : col) {
List<Object> list = (List<Object>) o;
List<Object> list0 = new ArrayList<>(list.size());
for (Object obj : list)
list0.add(obj != null ? ctx.marshaller().unmarshal((byte[]) obj, ldr) : null);
col0.add(list0);
}
}
return col0;
}
/** {@inheritDoc} */
@Override
public void addAttributeListener(GridTaskSessionAttributeListener lsnr, boolean rewind) {
A.notNull(lsnr, "lsnr");
Map<Object, Object> attrs = null;
List<GridTaskSessionAttributeListener> lsnrs;
synchronized (mux) {
lsnrs = new ArrayList<GridTaskSessionAttributeListener>(this.lsnrs.size());
lsnrs.addAll(this.lsnrs);
lsnrs.add(lsnr);
lsnrs = Collections.unmodifiableList(lsnrs);
this.lsnrs = lsnrs;
if (rewind) attrs = new HashMap<Object, Object>(this.attrs);
}
if (rewind) {
for (Map.Entry<Object, Object> entry : attrs.entrySet()) {
for (GridTaskSessionAttributeListener l : lsnrs) {
l.onAttributeSet(entry.getKey(), entry.getValue());
}
}
}
}
/**
* @return Collection of readers after check.
* @throws GridCacheEntryRemovedException If removed.
*/
public Collection<ReaderId> checkReaders() throws GridCacheEntryRemovedException {
synchronized (mux) {
checkObsolete();
if (!readers.isEmpty()) {
List<ReaderId> rmv = null;
for (ReaderId reader : readers) {
if (!cctx.discovery().alive(reader.nodeId())) {
if (rmv == null) rmv = new LinkedList<ReaderId>();
rmv.add(reader);
}
}
if (rmv != null) {
readers = new LinkedList<ReaderId>(readers);
for (ReaderId rdr : rmv) readers.remove(rdr);
readers = Collections.unmodifiableList(readers);
}
}
return readers;
}
}
/** {@inheritDoc} */
@Override
public void explicitUndeploy(UUID nodeId, String rsrcName) {
Collection<SharedDeployment> undeployed = new LinkedList<SharedDeployment>();
synchronized (mux) {
for (Iterator<List<SharedDeployment>> i1 = cache.values().iterator(); i1.hasNext(); ) {
List<SharedDeployment> deps = i1.next();
for (Iterator<SharedDeployment> i2 = deps.iterator(); i2.hasNext(); ) {
SharedDeployment dep = i2.next();
if (dep.hasName(rsrcName)) {
if (!dep.isUndeployed()) {
dep.undeploy();
dep.onRemoved();
// Undeploy.
i2.remove();
undeployed.add(dep);
if (log.isInfoEnabled()) log.info("Undeployed per-version class loader: " + dep);
}
break;
}
}
if (deps.isEmpty()) i1.remove();
}
}
recordUndeployed(null, undeployed);
}
/** {@inheritDoc} */
@Override
public Serializable reduce(List<GridComputeJobResult> results) throws GridException {
if (log.isInfoEnabled()) log.info("Reducing job [job=" + this + ", results=" + results + ']');
if (results.size() > 1) fail();
return results.get(0).getData();
}
/** {@inheritDoc} */
@SuppressWarnings({"unchecked"})
@Override
public T reduce(List<GridJobResult> results) throws GridException {
assert results != null;
assert results.size() == 1;
return (T) results.get(0).getData();
}
/**
* Increases priority if job has bumped down.
*
* @param waitJobs Ordered collection of collision contexts for jobs that are currently waiting
* for execution.
* @param passiveJobs Reordered collection of collision contexts for waiting jobs.
*/
private void bumpPriority(
Collection<GridCollisionJobContext> waitJobs, List<GridCollisionJobContext> passiveJobs) {
assert waitJobs != null;
assert passiveJobs != null;
assert waitJobs.size() == passiveJobs.size();
for (int i = 0; i < passiveJobs.size(); i++) {
GridCollisionJobContext ctx = passiveJobs.get(i);
if (i > indexOf(waitJobs, ctx))
ctx.getJobContext().setAttribute(jobAttrKey, getJobPriority(ctx) + starvationInc);
}
}
/** {@inheritDoc} */
@Override
public boolean removeAttributeListener(GridTaskSessionAttributeListener lsnr) {
A.notNull(lsnr, "lsnr");
synchronized (mux) {
List<GridTaskSessionAttributeListener> lsnrs =
new ArrayList<GridTaskSessionAttributeListener>(this.lsnrs);
boolean rmv = lsnrs.remove(lsnr);
this.lsnrs = Collections.unmodifiableList(lsnrs);
return rmv;
}
}
/**
* Gets values referenced by sequential keys, e.g. {@code key1...keyN}.
*
* @param keyPrefix Key prefix, e.g. {@code key} for {@code key1...keyN}.
* @param params Parameters map.
* @return Values.
*/
@Nullable
protected List<Object> values(String keyPrefix, Map<String, Object> params) {
assert keyPrefix != null;
List<Object> vals = new LinkedList<>();
for (int i = 1; ; i++) {
String key = keyPrefix + i;
if (params.containsKey(key)) vals.add(params.get(key));
else break;
}
return vals;
}
/** {@inheritDoc} */
@Override
public void run() {
List<Grid> clientGrids = runGrid();
assert clientGrids.size() == clientNodes;
int threadsCnt = clientNodes * threadsPerClient;
Executor e = Executors.newFixedThreadPool(threadsCnt);
for (Grid grid : clientGrids) {
for (int j = 0; j < threadsPerClient; j++)
e.execute(new GridJobLoadTestSubmitter(grid, taskParams, cancelRate, submitDelay));
}
}
/**
* Adds a Near key.
*
* @param key Key.
* @param keyBytes Key bytes.
* @param ctx Context.
* @throws GridException If failed.
*/
public void addNearKey(K key, byte[] keyBytes, GridCacheContext<K, V> ctx) throws GridException {
if (ctx.deploymentEnabled()) prepareObject(key, ctx);
if (nearKeyBytes == null) nearKeyBytes = new ArrayList<>();
nearKeyBytes.add(keyBytes);
}
/**
* Run all grid nodes as defined in test configuration.
*
* @return list of run nodes.
*/
private List<Grid> runGrid() {
List<Grid> clientGrids = new ArrayList<>(clientNodes);
try {
loadTestConfiguration();
for (int i = 0; i < srvNodes; i++) startNode("server", SERVER_NODE_CONFIGURATION);
// Start clients in the second order to cache a client node in GridGain.
for (int i = 0; i < clientNodes; i++)
clientGrids.add(startNode("client", CLIENT_NODE_CONFIGURATION));
} catch (Exception e) {
throw new RuntimeException(e);
}
return clientGrids;
}
/** {@inheritDoc} */
@Override
protected boolean hasReaders() throws GridCacheEntryRemovedException {
synchronized (mux) {
checkReaders();
return !readers.isEmpty();
}
}
/** {@inheritDoc} */
@Override
public final Map<? extends GridJob, GridNode> map(List<GridNode> subgrid, Callable<T> arg)
throws GridException {
assert subgrid != null;
assert !subgrid.isEmpty();
GridJob job = createJob(arg);
return Collections.singletonMap(job, balancer.getBalancedNode(job, null));
}
/** {@inheritDoc} */
@Override
public Map<? extends GridComputeJob, GridNode> map(List<GridNode> subgrid, Integer arg)
throws GridException {
assert taskSes != null;
assert arg != null;
assert arg > 0;
Map<GridSessionLoadTestJob, GridNode> map = new HashMap<>(subgrid.size());
Iterator<GridNode> iter = subgrid.iterator();
Random rnd = new Random();
params = new HashMap<>(arg);
Collection<UUID> assigned = new ArrayList<>(subgrid.size());
for (int i = 0; i < arg; i++) {
// Recycle iterator.
if (!iter.hasNext()) iter = subgrid.iterator();
String paramName = UUID.randomUUID().toString();
int paramVal = rnd.nextInt();
taskSes.setAttribute(paramName, paramVal);
GridNode node = iter.next();
assigned.add(node.id());
map.put(new GridSessionLoadTestJob(paramName), node);
params.put(paramName, paramVal);
if (log.isDebugEnabled())
log.debug("Set session attribute [name=" + paramName + ", value=" + paramVal + ']');
}
taskSes.setAttribute("nodes", assigned);
return map;
}
/** {@inheritDoc} */
@SuppressWarnings("BusyWait")
@Override
public Boolean reduce(List<GridComputeJobResult> results) throws GridException {
assert taskSes != null;
assert results != null;
assert params != null;
assert !params.isEmpty();
assert results.size() == params.size();
Map<String, Integer> receivedParams = new HashMap<>();
boolean allAttrReceived = false;
int cnt = 0;
while (!allAttrReceived && cnt++ < 3) {
allAttrReceived = true;
for (Map.Entry<String, Integer> entry : params.entrySet()) {
assert taskSes.getAttribute(entry.getKey()) != null;
Integer newVal = (Integer) taskSes.getAttribute(entry.getKey());
assert newVal != null;
receivedParams.put(entry.getKey(), newVal);
if (newVal != entry.getValue() + 1) allAttrReceived = false;
}
if (!allAttrReceived) {
try {
Thread.sleep(100);
} catch (InterruptedException e) {
throw new GridException("Thread interrupted.", e);
}
}
}
if (log.isDebugEnabled()) {
for (Map.Entry<String, Integer> entry : receivedParams.entrySet()) {
log.debug(
"Received session attr value [name="
+ entry.getKey()
+ ", val="
+ entry.getValue()
+ ", expected="
+ (params.get(entry.getKey()) + 1)
+ ']');
}
}
return allAttrReceived;
}
/** {@inheritDoc} */
@SuppressWarnings("all")
@Override
public boolean writeTo(ByteBuffer buf) {
commState.setBuffer(buf);
if (!super.writeTo(buf)) return false;
if (!commState.typeWritten) {
if (!commState.putByte(directType())) return false;
commState.typeWritten = true;
}
switch (commState.idx) {
case 8:
if (nearKeyBytes != null) {
if (commState.it == null) {
if (!commState.putInt(nearKeyBytes.size())) return false;
commState.it = nearKeyBytes.iterator();
}
while (commState.it.hasNext() || commState.cur != NULL) {
if (commState.cur == NULL) commState.cur = commState.it.next();
if (!commState.putByteArray((byte[]) commState.cur)) return false;
commState.cur = NULL;
}
commState.it = null;
} else {
if (!commState.putInt(-1)) return false;
}
commState.idx++;
}
return true;
}
/**
* @param nodeId Reader to remove.
* @param msgId Message ID.
* @return {@code True} if reader was removed as a result of this operation.
* @throws GridCacheEntryRemovedException If entry was removed.
*/
public boolean removeReader(UUID nodeId, long msgId) throws GridCacheEntryRemovedException {
synchronized (mux) {
checkObsolete();
ReaderId reader = readerId(nodeId);
if (reader == null || reader.messageId() > msgId) return false;
readers = new LinkedList<ReaderId>(readers);
readers.remove(reader);
// Seal.
readers = Collections.unmodifiableList(readers);
return true;
}
}
/** {@inheritDoc} */
@SuppressWarnings("all")
@Override
public boolean readFrom(ByteBuffer buf) {
commState.setBuffer(buf);
if (!super.readFrom(buf)) return false;
switch (commState.idx) {
case 8:
if (commState.readSize == -1) {
if (buf.remaining() < 4) return false;
commState.readSize = commState.getInt();
}
if (commState.readSize >= 0) {
if (nearKeyBytes == null) nearKeyBytes = new ArrayList<>(commState.readSize);
for (int i = commState.readItems; i < commState.readSize; i++) {
byte[] _val = commState.getByteArray();
if (_val == BYTE_ARR_NOT_READ) return false;
nearKeyBytes.add((byte[]) _val);
commState.readItems++;
}
}
commState.readSize = -1;
commState.readItems = 0;
commState.idx++;
}
return true;
}
/**
* @param seq Start/stop sequence.
* @throws Exception If failed.
*/
private void checkSequence0(boolean[] seq) throws Exception {
try {
startGrid(0);
TreeSet<Integer> started = new TreeSet<>();
started.add(0);
int topVer = 1;
for (boolean start : seq) {
if (start) {
int nextIdx = nextIndex(started);
startGrid(nextIdx);
started.add(nextIdx);
} else {
int idx = started.last();
stopGrid(idx);
started.remove(idx);
}
topVer++;
info("Grid 0: " + grid(0).localNode().id());
((GridKernal) grid(0))
.internalCache()
.context()
.affinity()
.affinityReadyFuture(topVer)
.get();
for (int i : started) {
if (i != 0) {
GridEx grid = grid(i);
((GridKernal) grid)
.internalCache()
.context()
.affinity()
.affinityReadyFuture(topVer)
.get();
info("Grid " + i + ": " + grid.localNode().id());
for (int part = 0; part < parts; part++) {
List<GridNode> firstNodes =
(List<GridNode>)
grid(0).cache(null).affinity().mapPartitionToPrimaryAndBackups(part);
List<GridNode> secondNodes =
(List<GridNode>)
grid.cache(null).affinity().mapPartitionToPrimaryAndBackups(part);
assertEquals(firstNodes.size(), secondNodes.size());
for (int n = 0; n < firstNodes.size(); n++)
assertEquals(firstNodes.get(n), secondNodes.get(n));
}
}
}
}
} finally {
stopAllGrids();
}
}
/**
* Creates REST request.
*
* @param cmd Command.
* @param params Parameters.
* @return REST request.
* @throws GridException If creation failed.
*/
@Nullable
private GridRestRequest createRequest(
GridRestCommand cmd, Map<String, Object> params, ServletRequest req) throws GridException {
GridRestRequest restReq;
switch (cmd) {
case CACHE_GET:
case CACHE_GET_ALL:
case CACHE_PUT:
case CACHE_PUT_ALL:
case CACHE_REMOVE:
case CACHE_REMOVE_ALL:
case CACHE_ADD:
case CACHE_CAS:
case CACHE_METRICS:
case CACHE_REPLACE:
case CACHE_DECREMENT:
case CACHE_INCREMENT:
case CACHE_APPEND:
case CACHE_PREPEND:
{
GridRestCacheRequest restReq0 = new GridRestCacheRequest();
restReq0.cacheName((String) params.get("cacheName"));
restReq0.key(params.get("key"));
restReq0.value(params.get("val"));
restReq0.value2(params.get("val2"));
Object val1 = params.get("val1");
if (val1 != null) restReq0.value(val1);
restReq0.cacheFlags(intValue("cacheFlags", params, 0));
restReq0.ttl(longValue("exp", params, null));
restReq0.initial(longValue("init", params, null));
restReq0.delta(longValue("delta", params, null));
if (cmd == CACHE_GET_ALL || cmd == CACHE_PUT_ALL || cmd == CACHE_REMOVE_ALL) {
List<Object> keys = values("k", params);
List<Object> vals = values("v", params);
if (keys.size() < vals.size())
throw new GridException(
"Number of keys must be greater or equals to number of values.");
Map<Object, Object> map = U.newHashMap(keys.size());
Iterator<Object> keyIt = keys.iterator();
Iterator<Object> valIt = vals.iterator();
while (keyIt.hasNext()) map.put(keyIt.next(), valIt.hasNext() ? valIt.next() : null);
restReq0.values(map);
}
restReq = restReq0;
break;
}
case TOPOLOGY:
case NODE:
{
GridRestTopologyRequest restReq0 = new GridRestTopologyRequest();
restReq0.includeMetrics(Boolean.parseBoolean((String) params.get("mtr")));
restReq0.includeAttributes(Boolean.parseBoolean((String) params.get("attr")));
restReq0.nodeIp((String) params.get("ip"));
restReq0.nodeId(uuidValue("id", params));
restReq = restReq0;
break;
}
case EXE:
case RESULT:
case NOOP:
{
GridRestTaskRequest restReq0 = new GridRestTaskRequest();
restReq0.taskId((String) params.get("id"));
restReq0.taskName((String) params.get("name"));
restReq0.params(values("p", params));
restReq0.async(Boolean.parseBoolean((String) params.get("async")));
restReq0.timeout(longValue("timeout", params, 0L));
restReq = restReq0;
break;
}
case LOG:
{
GridRestLogRequest restReq0 = new GridRestLogRequest();
restReq0.path((String) params.get("path"));
restReq0.from(intValue("from", params, -1));
restReq0.to(intValue("to", params, -1));
restReq = restReq0;
break;
}
case VERSION:
{
restReq = new GridRestRequest();
break;
}
default:
throw new GridException("Invalid command: " + cmd);
}
restReq.address(new InetSocketAddress(req.getRemoteAddr(), req.getRemotePort()));
restReq.command(cmd);
if (params.containsKey("gridgain.login") || params.containsKey("gridgain.password")) {
GridSecurityCredentials cred =
new GridSecurityCredentials(
(String) params.get("gridgain.login"), (String) params.get("gridgain.password"));
restReq.credentials(cred);
}
String clientId = (String) params.get("clientId");
try {
if (clientId != null) restReq.clientId(UUID.fromString(clientId));
} catch (Exception ignored) {
// Ignore invalid client id. Rest handler will process this logic.
}
String destId = (String) params.get("destId");
try {
if (destId != null) restReq.destinationId(UUID.fromString(destId));
} catch (IllegalArgumentException ignored) {
// Don't fail - try to execute locally.
}
String sesTokStr = (String) params.get("sessionToken");
try {
if (sesTokStr != null) restReq.sessionToken(U.hexString2ByteArray(sesTokStr));
} catch (IllegalArgumentException ignored) {
// Ignore invalid session token.
}
return restReq;
}
/**
* @param nodeId Primary node ID.
* @param req Request.
* @return Remote transaction.
* @throws GridException If failed.
* @throws GridDistributedLockCancelledException If lock has been cancelled.
*/
@SuppressWarnings({"RedundantTypeArguments"})
@Nullable
public GridNearTxRemote<K, V> startRemoteTx(UUID nodeId, GridDhtLockRequest<K, V> req)
throws GridException, GridDistributedLockCancelledException {
List<byte[]> nearKeyBytes = req.nearKeyBytes();
GridNearTxRemote<K, V> tx = null;
ClassLoader ldr = ctx.deploy().globalLoader();
if (ldr != null) {
for (int i = 0; i < nearKeyBytes.size(); i++) {
byte[] bytes = nearKeyBytes.get(i);
if (bytes == null) continue;
K key = req.nearKeys().get(i);
Collection<GridCacheMvccCandidate<K>> cands = req.candidatesByIndex(i);
if (log.isDebugEnabled()) log.debug("Unmarshalled key: " + key);
GridNearCacheEntry<K, V> entry = null;
while (true) {
try {
entry = peekExx(key);
if (entry != null) {
entry.keyBytes(bytes);
// Handle implicit locks for pessimistic transactions.
if (req.inTx()) {
tx = ctx.tm().tx(req.version());
if (tx != null) tx.addWrite(key, bytes, null /*Value.*/, null /*Value bytes.*/);
else {
tx =
new GridNearTxRemote<K, V>(
nodeId,
req.nearNodeId(),
req.threadId(),
req.version(),
null,
PESSIMISTIC,
req.isolation(),
req.isInvalidate(),
req.timeout(),
key,
bytes,
null, // Value.
null, // Value bytes.
ctx);
if (tx.empty()) return tx;
tx = ctx.tm().onCreated(tx);
if (tx == null || !ctx.tm().onStarted(tx))
throw new GridCacheTxRollbackException(
"Failed to acquire lock "
+ "(transaction has been completed): "
+ req.version());
}
}
// Add remote candidate before reordering.
entry.addRemote(
req.nodeId(),
nodeId,
req.threadId(),
req.version(),
req.timeout(),
tx != null && tx.ec(),
tx != null,
tx != null && tx.implicitSingle());
// Remote candidates for ordered lock queuing.
entry.addRemoteCandidates(
cands, req.version(), req.committedVersions(), req.rolledbackVersions());
entry.orderOwned(req.version(), req.owned(entry.key()));
}
// Double-check in case if sender node left the grid.
if (ctx.discovery().node(req.nodeId()) == null) {
if (log.isDebugEnabled())
log.debug("Node requesting lock left grid (lock request will be ignored): " + req);
if (tx != null) tx.rollback();
return null;
}
// Entry is legit.
break;
} catch (GridCacheEntryRemovedException ignored) {
assert entry.obsoleteVersion() != null
: "Obsolete flag not set on removed entry: " + entry;
if (log.isDebugEnabled())
log.debug("Received entry removed exception (will retry on renewed entry): " + entry);
if (tx != null) {
tx.clearEntry(entry.key());
if (log.isDebugEnabled())
log.debug(
"Cleared removed entry from remote transaction (will retry) [entry="
+ entry
+ ", tx="
+ tx
+ ']');
}
}
}
}
} else {
String err = "Failed to acquire deployment class loader for message: " + req;
U.warn(log, err);
throw new GridException(err);
}
return tx;
}
/**
* Processes lock request.
*
* @param nodeId Sender node ID.
* @param msg Lock request.
*/
@SuppressWarnings({"unchecked", "ThrowableInstanceNeverThrown"})
private void processLockRequest(UUID nodeId, GridDistributedLockRequest<K, V> msg) {
assert !nodeId.equals(locNodeId);
List<byte[]> keys = msg.keyBytes();
int cnt = keys.size();
GridReplicatedTxRemote<K, V> tx = null;
GridDistributedLockResponse res;
ClassLoader ldr = null;
try {
ldr = ctx.deploy().globalLoader();
if (ldr != null) {
res = new GridDistributedLockResponse(msg.version(), msg.futureId(), cnt);
for (int i = 0; i < keys.size(); i++) {
byte[] bytes = keys.get(i);
K key = msg.keys().get(i);
Collection<GridCacheMvccCandidate<K>> cands = msg.candidatesByIndex(i);
if (bytes == null) continue;
if (log.isDebugEnabled()) log.debug("Unmarshalled key: " + key);
GridDistributedCacheEntry<K, V> entry = null;
while (true) {
try {
entry = entryexx(key);
// Handle implicit locks for pessimistic transactions.
if (msg.inTx()) {
tx = ctx.tm().tx(msg.version());
if (tx != null) {
if (msg.txRead()) tx.addRead(key, bytes);
else tx.addWrite(key, bytes);
} else {
tx =
new GridReplicatedTxRemote<K, V>(
nodeId,
msg.threadId(),
msg.version(),
null,
PESSIMISTIC,
msg.isolation(),
msg.isInvalidate(),
msg.timeout(),
key,
bytes,
msg.txRead(),
ctx);
tx = ctx.tm().onCreated(tx);
if (tx == null || !ctx.tm().onStarted(tx))
throw new GridCacheTxRollbackException(
"Failed to acquire lock "
+ "(transaction has been completed): "
+ msg.version());
}
}
// Add remote candidate before reordering.
entry.addRemote(
msg.nodeId(),
null,
msg.threadId(),
msg.version(),
msg.timeout(),
tx != null && tx.ec(),
tx != null,
tx != null && tx.implicitSingle());
// Remote candidates for ordered lock queuing.
entry.addRemoteCandidates(
cands, msg.version(), msg.committedVersions(), msg.rolledbackVersions());
// Double-check in case if sender node left the grid.
if (ctx.discovery().node(msg.nodeId()) == null) {
if (log.isDebugEnabled())
log.debug(
"Node requesting lock left grid (lock request will be ignored): " + msg);
if (tx != null) tx.rollback();
return;
}
res.setCandidates(
i,
entry.localCandidates(),
ctx.tm().committedVersions(msg.version()),
ctx.tm().rolledbackVersions(msg.version()));
res.addValueBytes(
entry.rawGet(), msg.returnValue(i) ? entry.valueBytes(null) : null, ctx);
// Entry is legit.
break;
} catch (GridCacheEntryRemovedException ignored) {
assert entry.obsoleteVersion() != null
: "Obsolete flag not set on removed entry: " + entry;
if (log.isDebugEnabled())
log.debug(
"Received entry removed exception (will retry on renewed entry): " + entry);
if (tx != null) {
tx.clearEntry(entry.key());
if (log.isDebugEnabled())
log.debug(
"Cleared removed entry from remote transaction (will retry) [entry="
+ entry
+ ", tx="
+ tx
+ ']');
}
}
}
}
} else {
String err = "Failed to acquire deployment class for message: " + msg;
U.warn(log, err);
res =
new GridDistributedLockResponse(msg.version(), msg.futureId(), new GridException(err));
}
} catch (GridCacheTxRollbackException e) {
if (log.isDebugEnabled())
log.debug("Received lock request for completed transaction (will ignore): " + e);
res = new GridDistributedLockResponse(msg.version(), msg.futureId(), e);
} catch (GridException e) {
String err = "Failed to unmarshal at least one of the keys for lock request message: " + msg;
log.error(err, e);
res =
new GridDistributedLockResponse(msg.version(), msg.futureId(), new GridException(err, e));
if (tx != null) tx.rollback();
} catch (GridDistributedLockCancelledException ignored) {
// Received lock request for cancelled lock.
if (log.isDebugEnabled())
log.debug("Received lock request for canceled lock (will ignore): " + msg);
if (tx != null) tx.rollback();
// Don't send response back.
return;
}
GridNode node = ctx.discovery().node(msg.nodeId());
boolean releaseAll = false;
if (node != null) {
try {
// Reply back to sender.
ctx.io().send(node, res);
} catch (GridException e) {
U.error(log, "Failed to send message to node (did the node leave grid?): " + node.id(), e);
releaseAll = ldr != null;
}
}
// If sender left grid, release all locks acquired so far.
else releaseAll = ldr != null;
// Release all locks because sender node left grid.
if (releaseAll) {
for (K key : msg.keys()) {
while (true) {
GridDistributedCacheEntry<K, V> entry = peekexx(key);
try {
if (entry != null) entry.removeExplicitNodeLocks(msg.nodeId());
break;
} catch (GridCacheEntryRemovedException ignore) {
if (log.isDebugEnabled())
log.debug(
"Attempted to remove lock on removed entity during failure "
+ "of replicated lock request handling (will retry): "
+ entry);
}
}
}
U.warn(
log, "Sender node left grid in the midst of lock acquisition (locks will be released).");
}
}
/** {@inheritDoc} */
@Override
public void onCollision(
Collection<GridCollisionJobContext> waitJobs,
Collection<GridCollisionJobContext> activeJobs) {
assert waitJobs != null;
assert activeJobs != null;
int activeSize = F.size(activeJobs, RUNNING_JOBS);
waitingCnt.set(waitJobs.size());
runningCnt.set(activeSize);
heldCnt.set(activeJobs.size() - activeSize);
int waitSize = waitJobs.size();
int activateCnt = parallelJobsNum - activeSize;
if (activateCnt > 0 && !waitJobs.isEmpty()) {
if (waitJobs.size() <= activateCnt) {
for (GridCollisionJobContext waitJob : waitJobs) {
waitJob.activate();
waitSize--;
}
} else {
List<GridCollisionJobContext> passiveList =
new ArrayList<GridCollisionJobContext>(waitJobs);
Collections.sort(
passiveList,
new Comparator<GridCollisionJobContext>() {
/** {@inheritDoc} */
@Override
public int compare(GridCollisionJobContext o1, GridCollisionJobContext o2) {
int p1 = getJobPriority(o1);
int p2 = getJobPriority(o2);
return p1 < p2 ? 1 : p1 == p2 ? 0 : -1;
}
});
if (preventStarvation) bumpPriority(waitJobs, passiveList);
for (int i = 0; i < activateCnt; i++) {
passiveList.get(i).activate();
waitSize--;
}
}
}
if (waitSize > waitJobsNum) {
List<GridCollisionJobContext> waitList = new ArrayList<GridCollisionJobContext>(waitJobs);
// Put jobs with highest priority first.
Collections.sort(
waitList,
new Comparator<GridCollisionJobContext>() {
/** {@inheritDoc} */
@Override
public int compare(GridCollisionJobContext o1, GridCollisionJobContext o2) {
int p1 = getJobPriority(o1);
int p2 = getJobPriority(o2);
return p1 < p2 ? 1 : p1 == p2 ? 0 : -1;
}
});
int skip = waitJobs.size() - waitSize;
int i = 0;
for (GridCollisionJobContext waitCtx : waitList) {
if (++i >= skip) {
waitCtx.cancel();
if (--waitSize <= waitJobsNum) break;
}
}
}
}
/**
* Performs flush.
*
* @throws GridException If failed.
*/
private void doFlush() throws GridException {
lastFlushTime = U.currentTimeMillis();
List<GridFuture> activeFuts0 = null;
int doneCnt = 0;
for (GridFuture<?> f : activeFuts) {
if (!f.isDone()) {
if (activeFuts0 == null) activeFuts0 = new ArrayList<>((int) (activeFuts.size() * 1.2));
activeFuts0.add(f);
} else {
f.get();
doneCnt++;
}
}
if (activeFuts0 == null || activeFuts0.isEmpty()) return;
while (true) {
Queue<GridFuture<?>> q = null;
for (Buffer buf : bufMappings.values()) {
GridFuture<?> flushFut = buf.flush();
if (flushFut != null) {
if (q == null) q = new ArrayDeque<>(bufMappings.size() * 2);
q.add(flushFut);
}
}
if (q != null) {
assert !q.isEmpty();
boolean err = false;
for (GridFuture fut = q.poll(); fut != null; fut = q.poll()) {
try {
fut.get();
} catch (GridException e) {
if (log.isDebugEnabled()) log.debug("Failed to flush buffer: " + e);
err = true;
}
}
if (err)
// Remaps needed - flush buffers.
continue;
}
doneCnt = 0;
for (int i = 0; i < activeFuts0.size(); i++) {
GridFuture f = activeFuts0.get(i);
if (f == null) doneCnt++;
else if (f.isDone()) {
f.get();
doneCnt++;
activeFuts0.set(i, null);
} else break;
}
if (doneCnt == activeFuts0.size()) return;
}
}
/**
* This method is called to map or split grid task into multiple grid jobs. This is the first
* method that gets called when task execution starts.
*
* @param data Task execution argument. Can be {@code null}. This is the same argument as the one
* passed into {@code Grid#execute(...)} methods.
* @param subgrid Nodes available for this task execution. Note that order of nodes is guaranteed
* to be randomized by container. This ensures that every time you simply iterate through grid
* nodes, the order of nodes will be random which over time should result into all nodes being
* used equally.
* @return Map of grid jobs assigned to subgrid node. Unless {@link
* GridComputeTaskContinuousMapper} is injected into task, if {@code null} or empty map is
* returned, exception will be thrown.
* @throws GridException If mapping could not complete successfully. This exception will be thrown
* out of {@link GridComputeTaskFuture#get()} method.
*/
@Override
public Map<? extends GridComputeJob, GridNode> map(
List<GridNode> subgrid, @Nullable final Collection<Integer> data) throws GridException {
assert !subgrid.isEmpty();
// Give preference to wanted node. Otherwise, take the first one.
GridNode targetNode =
F.find(
subgrid,
subgrid.get(0),
new GridPredicate<GridNode>() {
@Override
public boolean apply(GridNode e) {
return preferredNode.equals(e.id());
}
});
return Collections.singletonMap(
new GridComputeJobAdapter() {
@GridLoggerResource private GridLogger log;
@GridInstanceResource private Grid grid;
@Override
public Object execute() throws GridException {
log.info("Going to put data: " + data.size());
GridCache<Object, Object> cache = grid.cache(cacheName);
assert cache != null;
Map<Integer, T2<Integer, Collection<Integer>>> putMap = groupData(data);
for (Map.Entry<Integer, T2<Integer, Collection<Integer>>> entry : putMap.entrySet()) {
T2<Integer, Collection<Integer>> pair = entry.getValue();
Object affKey = pair.get1();
// Group lock partition.
try (GridCacheTx tx =
cache.txStartPartition(
cache.affinity().partition(affKey),
optimistic ? OPTIMISTIC : PESSIMISTIC,
REPEATABLE_READ,
0,
pair.get2().size())) {
for (Integer val : pair.get2()) cache.put(val, val);
tx.commit();
}
}
log.info("Finished put data: " + data.size());
return data;
}
/**
* Groups values by partitions.
*
* @param data Data to put.
* @return Grouped map.
*/
private Map<Integer, T2<Integer, Collection<Integer>>> groupData(Iterable<Integer> data) {
GridCache<Object, Object> cache = grid.cache(cacheName);
Map<Integer, T2<Integer, Collection<Integer>>> res = new HashMap<>();
for (Integer val : data) {
int part = cache.affinity().partition(val);
T2<Integer, Collection<Integer>> tup = res.get(part);
if (tup == null) {
tup = new T2<Integer, Collection<Integer>>(val, new LinkedList<Integer>());
res.put(part, tup);
}
tup.get2().add(val);
}
return res;
}
},
targetNode);
}
/** @throws Exception If failed. */
public void testCreateFileFragmented() throws Exception {
GridGgfsEx impl = (GridGgfsEx) grid(0).ggfs("ggfs");
GridGgfsFragmentizerManager fragmentizer = impl.context().fragmentizer();
GridTestUtils.setFieldValue(fragmentizer, "fragmentizerEnabled", false);
GridGgfsPath path = new GridGgfsPath("/file");
try {
GridGgfs fs0 = grid(0).ggfs("ggfs");
GridGgfs fs1 = grid(1).ggfs("ggfs");
GridGgfs fs2 = grid(2).ggfs("ggfs");
try (GridGgfsOutputStream out =
fs0.create(
path,
128,
false,
1,
CFG_GRP_SIZE,
F.asMap(GridGgfs.PROP_PREFER_LOCAL_WRITES, "true"))) {
// 1.5 blocks
byte[] data = new byte[CFG_BLOCK_SIZE * 3 / 2];
Arrays.fill(data, (byte) 1);
out.write(data);
}
try (GridGgfsOutputStream out = fs1.append(path, false)) {
// 1.5 blocks.
byte[] data = new byte[CFG_BLOCK_SIZE * 3 / 2];
Arrays.fill(data, (byte) 2);
out.write(data);
}
// After this we should have first two block colocated with grid 0 and last block colocated
// with grid 1.
GridGgfsFileImpl fileImpl = (GridGgfsFileImpl) fs.info(path);
GridCache<Object, Object> metaCache = grid(0).cachex(META_CACHE_NAME);
GridGgfsFileInfo fileInfo = (GridGgfsFileInfo) metaCache.get(fileImpl.fileId());
GridGgfsFileMap map = fileInfo.fileMap();
List<GridGgfsFileAffinityRange> ranges = map.ranges();
assertEquals(2, ranges.size());
assertTrue(ranges.get(0).startOffset() == 0);
assertTrue(ranges.get(0).endOffset() == 2 * CFG_BLOCK_SIZE - 1);
assertTrue(ranges.get(1).startOffset() == 2 * CFG_BLOCK_SIZE);
assertTrue(ranges.get(1).endOffset() == 3 * CFG_BLOCK_SIZE - 1);
// Validate data read after colocated writes.
try (GridGgfsInputStream in = fs2.open(path)) {
// Validate first part of file.
for (int i = 0; i < CFG_BLOCK_SIZE * 3 / 2; i++) assertEquals((byte) 1, in.read());
// Validate second part of file.
for (int i = 0; i < CFG_BLOCK_SIZE * 3 / 2; i++) assertEquals((byte) 2, in.read());
assertEquals(-1, in.read());
}
} finally {
GridTestUtils.setFieldValue(fragmentizer, "fragmentizerEnabled", true);
boolean hasData = false;
for (int i = 0; i < NODES_CNT; i++) hasData |= !grid(i).cachex(DATA_CACHE_NAME).isEmpty();
assertTrue(hasData);
fs.delete(path, true);
}
GridTestUtils.retryAssert(
log,
ASSERT_RETRIES,
ASSERT_RETRY_INTERVAL,
new CAX() {
@Override
public void applyx() {
for (int i = 0; i < NODES_CNT; i++)
assertTrue(grid(i).cachex(DATA_CACHE_NAME).isEmpty());
}
});
}
/**
* Runs all tests belonging to this test suite on the grid.
*
* @param result Test result collector.
*/
@Override
public void run(TestResult result) {
if (isDisabled) {
copy.run(result);
} else {
GridTestRouter router = createRouter();
Grid grid = startGrid();
try {
List<GridTaskFuture<?>> futs = new ArrayList<GridTaskFuture<?>>(testCount());
List<GridJunit3SerializableTest> tests =
new ArrayList<GridJunit3SerializableTest>(testCount());
for (int i = 0; i < testCount(); i++) {
Test junit = testAt(i);
GridJunit3SerializableTest test;
if (junit instanceof TestSuite) {
test = new GridJunit3SerializableTestSuite((TestSuite) junit);
} else {
assert junit instanceof TestCase
: "Test must be either TestSuite or TestCase: " + junit;
test = new GridJunit3SerializableTestCase((TestCase) junit);
}
tests.add(test);
if (clsLdr == null) {
clsLdr = U.detectClassLoader(junit.getClass());
}
futs.add(
grid.execute(
new GridJunit3Task(junit.getClass(), clsLdr),
new GridJunit3Argument(router, test, locTests.contains(test.getName())),
timeout));
}
for (int i = 0; i < testCount(); i++) {
GridTaskFuture<?> fut = futs.get(i);
GridJunit3SerializableTest origTest = tests.get(i);
try {
GridJunit3SerializableTest resTest = (GridJunit3SerializableTest) fut.get();
origTest.setResult(resTest);
origTest.getTest().run(result);
} catch (GridException e) {
handleFail(result, origTest, e);
}
}
} finally {
stopGrid();
}
}
}
/** {@inheritDoc} */
@Override
public String reduce(List<GridComputeJobResult> results) throws GridException {
assert results.size() == 1;
return results.get(0).getData();
}
