/**
* @param c Connection.
* @param space Space.
* @param qry Query.
* @return Cursor for plans.
* @throws IgniteCheckedException if failed.
*/
private Iterator<List<?>> explainPlan(JdbcConnection c, String space, GridCacheTwoStepQuery qry)
throws IgniteCheckedException {
List<List<?>> lists = new ArrayList<>();
for (int i = 0, mapQrys = qry.mapQueries().size(); i < mapQrys; i++) {
ResultSet rs =
h2.executeSqlQueryWithTimer(space, c, "SELECT PLAN FROM " + table(i), null, false);
lists.add(F.asList(getPlan(rs)));
}
int tblIdx = 0;
for (GridCacheSqlQuery mapQry : qry.mapQueries()) {
GridMergeTable tbl = createMergeTable(c, mapQry, false);
fakeTable(c, tblIdx++).setInnerTable(tbl);
}
GridCacheSqlQuery rdc = qry.reduceQuery();
ResultSet rs =
h2.executeSqlQueryWithTimer(
space, c, "EXPLAIN " + rdc.query(), F.asList(rdc.parameters()), false);
lists.add(F.asList(getPlan(rs)));
return lists.iterator();
}
/**
* @param ctx Context.
* @param e entry.
* @return Entry collection.
*/
private Collection<CacheContinuousQueryEntry> handleEvent(
GridKernalContext ctx, CacheContinuousQueryEntry e) {
assert e != null;
if (internal) {
if (e.isFiltered()) return Collections.emptyList();
else return F.asList(e);
}
// Initial query entry or evicted entry. These events should be fired immediately.
if (e.updateCounter() == -1L) return F.asList(e);
PartitionRecovery rec = rcvs.get(e.partition());
if (rec == null) {
rec =
new PartitionRecovery(
ctx.log(getClass()),
initTopVer,
initUpdCntrs == null ? null : initUpdCntrs.get(e.partition()));
PartitionRecovery oldRec = rcvs.putIfAbsent(e.partition(), rec);
if (oldRec != null) rec = oldRec;
}
return rec.collectEntries(e);
}
/**
* @param nodeId Node ID.
* @param retryCnt Number of retries.
*/
private void sendAllPartitions(final UUID nodeId, final int retryCnt) {
ClusterNode n = cctx.node(nodeId);
try {
if (n != null) sendAllPartitions(F.asList(n), exchId);
} catch (IgniteCheckedException e) {
if (e instanceof ClusterTopologyCheckedException || !cctx.discovery().alive(n)) {
log.debug(
"Failed to send full partition map to node, node left grid "
+ "[rmtNode="
+ nodeId
+ ", exchangeId="
+ exchId
+ ']');
return;
}
if (retryCnt > 0) {
long timeout = cctx.gridConfig().getNetworkSendRetryDelay();
LT.error(
log,
e,
"Failed to send full partition map to node (will retry after timeout) "
+ "[node="
+ nodeId
+ ", exchangeId="
+ exchId
+ ", timeout="
+ timeout
+ ']');
cctx.time()
.addTimeoutObject(
new GridTimeoutObjectAdapter(timeout) {
@Override
public void onTimeout() {
sendAllPartitions(nodeId, retryCnt - 1);
}
});
} else
U.error(
log,
"Failed to send full partition map [node=" + n + ", exchangeId=" + exchId + ']',
e);
}
}
/**
* Finds all files in folder and in it's sub-tree of specified depth.
*
* @param file Starting folder
* @param maxDepth Depth of the tree. If 1 - just look in the folder, no sub-folders.
* @param filter file filter.
* @return List of found files.
*/
public static List<VisorLogFile> fileTree(File file, int maxDepth, @Nullable FileFilter filter) {
if (file.isDirectory()) {
File[] files = (filter == null) ? file.listFiles() : file.listFiles(filter);
if (files == null) return Collections.emptyList();
List<VisorLogFile> res = new ArrayList<>(files.length);
for (File f : files) {
if (f.isFile() && f.length() > 0) res.add(new VisorLogFile(f));
else if (maxDepth > 1) res.addAll(fileTree(f, maxDepth - 1, filter));
}
return res;
}
return F.asList(new VisorLogFile(file));
}
/** @throws Exception If failed. */
public void testGroupIndexOperations() throws Exception {
IgniteCache<Integer, GroupIndexTestValue> c =
ignite(0)
.getOrCreateCache(cacheConfig("grp", false, Integer.class, GroupIndexTestValue.class));
try {
// Check group index usage.
String qry = "select 1 from GroupIndexTestValue ";
String plan = columnQuery(c, "explain " + qry + "where a = 1 and b > 0").get(0).toString();
info("Plan: " + plan);
assertTrue(plan.contains("grpIdx"));
// Sorted list
List<GroupIndexTestValue> list =
F.asList(
new GroupIndexTestValue(0, 0),
new GroupIndexTestValue(0, 5),
new GroupIndexTestValue(1, 1),
new GroupIndexTestValue(1, 3),
new GroupIndexTestValue(2, -1),
new GroupIndexTestValue(2, 2));
// Fill cache.
for (int i = 0; i < list.size(); i++) c.put(i, list.get(i));
// Check results.
assertEquals(1, columnQuery(c, qry + "where a = 1 and b = 1").size());
assertEquals(2, columnQuery(c, qry + "where a = 1 and b < 4").size());
assertEquals(2, columnQuery(c, qry + "where a = 1 and b <= 3").size());
assertEquals(1, columnQuery(c, qry + "where a = 1 and b < 3").size());
assertEquals(2, columnQuery(c, qry + "where a = 1 and b > 0").size());
assertEquals(1, columnQuery(c, qry + "where a = 1 and b > 1").size());
assertEquals(2, columnQuery(c, qry + "where a = 1 and b >= 1").size());
assertEquals(4, columnQuery(c, qry + "where a > 0 and b > 0").size());
assertEquals(4, columnQuery(c, qry + "where a > 0 and b >= 1").size());
assertEquals(3, columnQuery(c, qry + "where a > 0 and b > 1").size());
} finally {
c.destroy();
}
}
/**
* Add continuous entry.
*
* @param entry Cache continuous query entry.
* @return Collection entries which will be fired.
*/
public Collection<CacheContinuousQueryEntry> collectEntries(CacheContinuousQueryEntry entry) {
assert entry != null;
List<CacheContinuousQueryEntry> entries;
synchronized (pendingEvts) {
// Received first event.
if (curTop == AffinityTopologyVersion.NONE) {
lastFiredEvt = entry.updateCounter();
curTop = entry.topologyVersion();
return F.asList(entry);
}
if (curTop.compareTo(entry.topologyVersion()) < 0) {
if (entry.updateCounter() == 1L && !entry.isBackup()) {
entries = new ArrayList<>(pendingEvts.size());
for (CacheContinuousQueryEntry evt : pendingEvts.values()) {
if (evt != HOLE && !evt.isFiltered()) entries.add(evt);
}
pendingEvts.clear();
curTop = entry.topologyVersion();
lastFiredEvt = entry.updateCounter();
entries.add(entry);
return entries;
}
curTop = entry.topologyVersion();
}
// Check duplicate.
if (entry.updateCounter() > lastFiredEvt) {
pendingEvts.put(entry.updateCounter(), entry);
// Put filtered events.
if (entry.filteredEvents() != null) {
for (long cnrt : entry.filteredEvents()) {
if (cnrt > lastFiredEvt) pendingEvts.put(cnrt, HOLE);
}
}
} else {
if (log.isDebugEnabled()) log.debug("Skip duplicate continuous query message: " + entry);
return Collections.emptyList();
}
if (pendingEvts.isEmpty()) return Collections.emptyList();
Iterator<Map.Entry<Long, CacheContinuousQueryEntry>> iter =
pendingEvts.entrySet().iterator();
entries = new ArrayList<>();
if (pendingEvts.size() >= MAX_BUFF_SIZE) {
for (int i = 0; i < MAX_BUFF_SIZE - (MAX_BUFF_SIZE / 10); i++) {
Map.Entry<Long, CacheContinuousQueryEntry> e = iter.next();
if (e.getValue() != HOLE && !e.getValue().isFiltered()) entries.add(e.getValue());
lastFiredEvt = e.getKey();
iter.remove();
}
} else {
// Elements are consistently.
while (iter.hasNext()) {
Map.Entry<Long, CacheContinuousQueryEntry> e = iter.next();
if (e.getKey() == lastFiredEvt + 1) {
++lastFiredEvt;
if (e.getValue() != HOLE && !e.getValue().isFiltered()) entries.add(e.getValue());
iter.remove();
} else break;
}
}
}
return entries;
}
/** @throws Exception If failed. */
public void testFindCycle3() throws Exception {
Map<GridCacheVersion, Set<GridCacheVersion>> wfg =
new HashMap<GridCacheVersion, Set<GridCacheVersion>>() {
{
put(T1, Collections.singleton(T2));
put(T2, Collections.singleton(T3));
put(T3, Collections.singleton(T1));
}
};
assertEquals(F.asList(T3, T2, T1, T3), findCycle(wfg, T1));
assertEquals(F.asList(T1, T3, T2, T1), findCycle(wfg, T2));
assertEquals(F.asList(T2, T1, T3, T2), findCycle(wfg, T3));
assertAllNull(wfg, T1, T2, T3);
wfg =
new HashMap<GridCacheVersion, Set<GridCacheVersion>>() {
{
put(T1, Collections.singleton(T2));
put(T2, Collections.singleton(T3));
put(T3, Collections.singleton(T4));
put(T4, asLinkedHashSet(T2, T5));
}
};
assertEquals(F.asList(T4, T3, T2, T4), findCycle(wfg, T1));
assertEquals(F.asList(T4, T3, T2, T4), findCycle(wfg, T2));
assertEquals(F.asList(T2, T4, T3, T2), findCycle(wfg, T3));
assertEquals(F.asList(T3, T2, T4, T3), findCycle(wfg, T4));
assertAllNull(wfg, T1, T2, T3, T4);
wfg =
new HashMap<GridCacheVersion, Set<GridCacheVersion>>() {
{
put(T1, Collections.singleton(T2));
put(T2, asLinkedHashSet(T3, T4));
put(T3, Collections.singleton(T1));
put(T4, Collections.singleton(T5));
put(T5, Collections.singleton(T6));
put(T6, Collections.singleton(T4));
}
};
assertEquals(F.asList(T6, T5, T4, T6), findCycle(wfg, T1));
assertEquals(F.asList(T6, T5, T4, T6), findCycle(wfg, T2));
assertEquals(F.asList(T2, T1, T3, T2), findCycle(wfg, T3));
wfg =
new HashMap<GridCacheVersion, Set<GridCacheVersion>>() {
{
put(T1, Collections.singleton(T2));
put(T2, Collections.singleton(T3));
put(T3, Collections.singleton(T4));
put(T4, Collections.singleton(T5));
put(T5, Collections.singleton(T6));
put(T6, Collections.singleton(T4));
}
};
assertEquals(F.asList(T6, T5, T4, T6), findCycle(wfg, T1));
assertEquals(F.asList(T6, T5, T4, T6), findCycle(wfg, T2));
assertEquals(F.asList(T6, T5, T4, T6), findCycle(wfg, T3));
assertEquals(F.asList(T6, T5, T4, T6), findCycle(wfg, T4));
assertEquals(F.asList(T4, T6, T5, T4), findCycle(wfg, T5));
assertEquals(F.asList(T5, T4, T6, T5), findCycle(wfg, T6));
}
/**
* @param cctx Cache context.
* @param qry Query.
* @param keepPortable Keep portable.
* @return Cursor.
*/
public Iterator<List<?>> query(
GridCacheContext<?, ?> cctx, GridCacheTwoStepQuery qry, boolean keepPortable) {
for (int attempt = 0; ; attempt++) {
if (attempt != 0) {
try {
Thread.sleep(attempt * 10); // Wait for exchange.
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
throw new CacheException("Query was interrupted.", e);
}
}
long qryReqId = reqIdGen.incrementAndGet();
QueryRun r = new QueryRun();
r.pageSize = qry.pageSize() <= 0 ? GridCacheTwoStepQuery.DFLT_PAGE_SIZE : qry.pageSize();
r.idxs = new ArrayList<>(qry.mapQueries().size());
String space = cctx.name();
r.conn = (JdbcConnection) h2.connectionForSpace(space);
AffinityTopologyVersion topVer = h2.readyTopologyVersion();
List<String> extraSpaces = extraSpaces(space, qry.spaces());
Collection<ClusterNode> nodes;
// Explicit partition mapping for unstable topology.
Map<ClusterNode, IntArray> partsMap = null;
if (isPreloadingActive(cctx, extraSpaces)) {
if (cctx.isReplicated()) nodes = replicatedUnstableDataNodes(cctx, extraSpaces);
else {
partsMap = partitionedUnstableDataNodes(cctx, extraSpaces);
nodes = partsMap == null ? null : partsMap.keySet();
}
} else nodes = stableDataNodes(topVer, cctx, extraSpaces);
if (nodes == null) continue; // Retry.
assert !nodes.isEmpty();
if (cctx.isReplicated() || qry.explain()) {
assert qry.explain() || !nodes.contains(ctx.discovery().localNode())
: "We must be on a client node.";
// Select random data node to run query on a replicated data or get EXPLAIN PLAN from a
// single node.
nodes = Collections.singleton(F.rand(nodes));
}
int tblIdx = 0;
final boolean skipMergeTbl = !qry.explain() && qry.skipMergeTable();
for (GridCacheSqlQuery mapQry : qry.mapQueries()) {
GridMergeIndex idx;
if (!skipMergeTbl) {
GridMergeTable tbl;
try {
tbl = createMergeTable(r.conn, mapQry, qry.explain());
} catch (IgniteCheckedException e) {
throw new IgniteException(e);
}
idx = tbl.getScanIndex(null);
fakeTable(r.conn, tblIdx++).setInnerTable(tbl);
} else idx = GridMergeIndexUnsorted.createDummy();
for (ClusterNode node : nodes) idx.addSource(node.id());
r.idxs.add(idx);
}
r.latch = new CountDownLatch(r.idxs.size() * nodes.size());
runs.put(qryReqId, r);
try {
if (ctx.clientDisconnected()) {
throw new CacheException(
"Query was cancelled, client node disconnected.",
new IgniteClientDisconnectedException(
ctx.cluster().clientReconnectFuture(), "Client node disconnected."));
}
Collection<GridCacheSqlQuery> mapQrys = qry.mapQueries();
if (qry.explain()) {
mapQrys = new ArrayList<>(qry.mapQueries().size());
for (GridCacheSqlQuery mapQry : qry.mapQueries())
mapQrys.add(new GridCacheSqlQuery("EXPLAIN " + mapQry.query(), mapQry.parameters()));
}
if (nodes.size() != 1 || !F.first(nodes).isLocal()) { // Marshall params for remotes.
Marshaller m = ctx.config().getMarshaller();
for (GridCacheSqlQuery mapQry : mapQrys) mapQry.marshallParams(m);
}
boolean retry = false;
if (send(
nodes,
new GridQueryRequest(qryReqId, r.pageSize, space, mapQrys, topVer, extraSpaces, null),
partsMap)) {
awaitAllReplies(r, nodes);
Object state = r.state.get();
if (state != null) {
if (state instanceof CacheException) {
CacheException err = (CacheException) state;
if (err.getCause() instanceof IgniteClientDisconnectedException) throw err;
throw new CacheException("Failed to run map query remotely.", err);
}
if (state instanceof AffinityTopologyVersion) {
retry = true;
// If remote node asks us to retry then we have outdated full partition map.
h2.awaitForReadyTopologyVersion((AffinityTopologyVersion) state);
}
}
} else // Send failed.
retry = true;
Iterator<List<?>> resIter = null;
if (!retry) {
if (qry.explain()) return explainPlan(r.conn, space, qry);
if (skipMergeTbl) {
List<List<?>> res = new ArrayList<>();
assert r.idxs.size() == 1 : r.idxs;
GridMergeIndex idx = r.idxs.get(0);
Cursor cur = idx.findInStream(null, null);
while (cur.next()) {
Row row = cur.get();
int cols = row.getColumnCount();
List<Object> resRow = new ArrayList<>(cols);
for (int c = 0; c < cols; c++) resRow.add(row.getValue(c).getObject());
res.add(resRow);
}
resIter = res.iterator();
} else {
GridCacheSqlQuery rdc = qry.reduceQuery();
// Statement caching is prohibited here because we can't guarantee correct merge index
// reuse.
ResultSet res =
h2.executeSqlQueryWithTimer(
space, r.conn, rdc.query(), F.asList(rdc.parameters()), false);
resIter = new Iter(res);
}
}
for (GridMergeIndex idx : r.idxs) {
if (!idx.fetchedAll()) // We have to explicitly cancel queries on remote nodes.
send(nodes, new GridQueryCancelRequest(qryReqId), null);
}
if (retry) {
if (Thread.currentThread().isInterrupted())
throw new IgniteInterruptedCheckedException("Query was interrupted.");
continue;
}
return new GridQueryCacheObjectsIterator(resIter, cctx, keepPortable);
} catch (IgniteCheckedException | RuntimeException e) {
U.closeQuiet(r.conn);
if (e instanceof CacheException) throw (CacheException) e;
Throwable cause = e;
if (e instanceof IgniteCheckedException) {
Throwable disconnectedErr =
((IgniteCheckedException) e).getCause(IgniteClientDisconnectedException.class);
if (disconnectedErr != null) cause = disconnectedErr;
}
throw new CacheException("Failed to run reduce query locally.", cause);
} finally {
if (!runs.remove(qryReqId, r)) U.warn(log, "Query run was already removed: " + qryReqId);
if (!skipMergeTbl) {
for (int i = 0, mapQrys = qry.mapQueries().size(); i < mapQrys; i++)
fakeTable(null, i).setInnerTable(null); // Drop all merge tables.
}
}
}
}
/** @throws Exception Thrown if test failed. */
public void testA() throws Exception {
Collection<Integer> set = new GridConcurrentWeakHashSet<>();
Integer i = 1;
assert set.add(i);
assert !set.add(i);
assert set.contains(i);
assert set.size() == 1;
Collection<Integer> c = F.asList(2, 3, 4, 5);
assert set.addAll(c);
assert !set.addAll(c);
assert set.containsAll(c);
assert set.size() == 1 + c.size();
assert set.remove(i);
assert !set.remove(i);
assert !set.contains(i);
assert set.size() == c.size();
assert set.removeAll(c);
assert !set.removeAll(c);
assert !set.containsAll(c);
assert set.isEmpty();
Collection<Integer> c1 = Arrays.asList(1, 3, 5, 7, 9);
int cnt = 0;
for (Iterator<Integer> iter = set.iterator(); iter.hasNext(); cnt++) c1.contains(iter.next());
assert set.size() == cnt;
assert set.size() == set.toArray().length;
assert set.addAll(c1);
assert set.retainAll(c);
assert !set.retainAll(c);
Collection<Integer> c2 = F.retain(c1, true, c);
assert set.containsAll(c2);
assert !set.containsAll(c1);
assert !set.containsAll(c);
assert set.size() == c2.size();
set.clear();
assert set.isEmpty();
try {
set.iterator().next();
assert false;
} catch (NoSuchElementException ignored) {
assert true;
}
try {
set.add(null);
assert false;
} catch (NullPointerException ignored) {
assert true;
}
}
/** @throws Exception Thrown if test failed. */
@SuppressWarnings({"UnusedAssignment"})
public void testB() throws Exception {
Collection<SampleBean> set = new GridConcurrentWeakHashSet<>();
SampleBean bean1 = new SampleBean(1);
assert set.add(bean1);
assert !set.add(bean1);
assert set.size() == 1;
assert set.contains(bean1);
bean1 = null;
gc();
assert set.isEmpty();
Collection<SampleBean> c =
F.asList(new SampleBean(1), new SampleBean(2), new SampleBean(3), new SampleBean(4));
assert set.addAll(c);
assert !set.addAll(c);
assert set.size() == c.size();
assert set.containsAll(c);
c = null;
gc();
assert set.isEmpty();
SampleBean b1 = new SampleBean(1);
SampleBean b2 = new SampleBean(2);
SampleBean b3 = new SampleBean(3);
SampleBean b4 = new SampleBean(4);
SampleBean b5 = new SampleBean(5);
set.add(b1);
set.add(b2);
set.add(b3);
set.add(b4);
set.add(b5);
Iterator iter = set.iterator();
assert iter.hasNext();
b2 = null;
b3 = null;
b4 = null;
gc();
int cnt = 0;
while (iter.hasNext()) {
info(iter.next().toString());
cnt++;
}
assert set.size() == cnt;
}
