/**
* @param cancel {@code True} to close with cancellation.
* @throws GridException If failed.
*/
@Override
public void close(boolean cancel) throws GridException {
if (!closed.compareAndSet(false, true)) return;
busyLock.block();
if (log.isDebugEnabled())
log.debug("Closing data loader [ldr=" + this + ", cancel=" + cancel + ']');
GridException e = null;
try {
// Assuming that no methods are called on this loader after this method is called.
if (cancel) {
cancelled = true;
for (Buffer buf : bufMappings.values()) buf.cancelAll();
} else doFlush();
ctx.event().removeLocalEventListener(discoLsnr);
ctx.io().removeMessageListener(topic);
} catch (GridException e0) {
e = e0;
}
fut.onDone(null, e);
if (e != null) throw e;
}
/** @throws GridException If operation failed. */
private void initializeLatch() throws GridException {
if (initGuard.compareAndSet(false, true)) {
try {
internalLatch =
CU.outTx(
new Callable<CountDownLatch>() {
@Override
public CountDownLatch call() throws Exception {
GridCacheTx tx =
CU.txStartInternal(ctx, latchView, PESSIMISTIC, REPEATABLE_READ);
try {
GridCacheCountDownLatchValue val = latchView.get(key);
if (val == null) {
if (log.isDebugEnabled())
log.debug("Failed to find count down latch with given name: " + name);
assert cnt == 0;
return new CountDownLatch(cnt);
}
tx.commit();
return new CountDownLatch(val.get());
} finally {
tx.end();
}
}
},
ctx);
if (log.isDebugEnabled()) log.debug("Initialized internal latch: " + internalLatch);
} finally {
initLatch.countDown();
}
} else {
try {
initLatch.await();
} catch (InterruptedException ignored) {
throw new GridException("Thread has been interrupted.");
}
if (internalLatch == null)
throw new GridException("Internal latch has not been properly initialized.");
}
}
public EmReactor() {
Timers = new TreeMap<Long, ArrayList<Long>>();
Connections = new HashMap<Long, EventableChannel>();
Acceptors = new HashMap<Long, ServerSocketChannel>();
NewConnections = new ArrayList<Long>();
UnboundConnections = new ArrayList<Long>();
DetachedConnections = new ArrayList<EventableSocketChannel>();
BindingIndex = 0;
loopBreaker = new AtomicBoolean();
loopBreaker.set(false);
myReadBuffer =
ByteBuffer.allocate(
32 * 1024); // don't use a direct buffer. Ruby doesn't seem to like them.
timerQuantum = 98;
}
/** @param e Node left exception. */
void onResult(GridTopologyException e) {
if (isDone()) return;
if (rcvRes.compareAndSet(false, true)) {
if (log.isDebugEnabled())
log.debug(
"Remote node left grid while sending or waiting for reply (will fail): " + this);
if (tx != null) tx.removeMapping(node.id());
// Primary node left the grid, so fail the future.
GridNearLockFuture.this.onDone(newTopologyException(e, node.id()));
onDone(true);
}
}
private <E> Predicate<E> createJoinPredicate(final AtomicBoolean more) {
if (column == null && this.nodes == null) return null;
final EntityCache<E> joinCache = this.joinEntity.getCache();
Predicate<E> filter = createElementPredicate(joinCache, true);
if (this.nodes != null) {
for (FilterNode node : this.nodes) {
if (((FilterJoinNode) node).joinClass != this.joinClass) {
more.set(true);
continue;
}
Predicate<E> f = ((FilterJoinNode) node).createJoinPredicate(more);
if (f == null) continue;
final Predicate<E> one = filter;
final Predicate<E> two = f;
filter =
(filter == null)
? f
: (or
? new Predicate<E>() {
@Override
public boolean test(E t) {
return one.test(t) || two.test(t);
}
@Override
public String toString() {
return "(" + one + " OR " + two + ")";
}
}
: new Predicate<E>() {
@Override
public boolean test(E t) {
return one.test(t) && two.test(t);
}
@Override
public String toString() {
return "(" + one + " AND " + two + ")";
}
});
}
}
return filter;
}
/** @param e Error. */
void onResult(Throwable e) {
if (rcvRes.compareAndSet(false, true)) {
if (log.isDebugEnabled())
log.debug("Failed to get future result [fut=" + this + ", err=" + e + ']');
// Fail.
onDone(e);
} else
U.warn(
log,
"Received error after another result has been processed [fut="
+ GridNearLockFuture.this
+ ", mini="
+ this
+ ']',
e);
}
/**
* Initializes store.
*
* @throws GridException If failed to initialize.
*/
private void init() throws GridException {
if (initGuard.compareAndSet(false, true)) {
if (log.isDebugEnabled()) log.debug("Initializing cache store.");
try {
if (sesFactory != null)
// Session factory has been provided - nothing to do.
return;
if (!F.isEmpty(hibernateCfgPath)) {
try {
URL url = new URL(hibernateCfgPath);
sesFactory = new Configuration().configure(url).buildSessionFactory();
if (log.isDebugEnabled()) log.debug("Configured session factory using URL: " + url);
// Session factory has been successfully initialized.
return;
} catch (MalformedURLException e) {
if (log.isDebugEnabled())
log.debug("Caught malformed URL exception: " + e.getMessage());
}
// Provided path is not a valid URL. File?
File cfgFile = new File(hibernateCfgPath);
if (cfgFile.exists()) {
sesFactory = new Configuration().configure(cfgFile).buildSessionFactory();
if (log.isDebugEnabled())
log.debug("Configured session factory using file: " + hibernateCfgPath);
// Session factory has been successfully initialized.
return;
}
// Provided path is not a file. Classpath resource?
sesFactory = new Configuration().configure(hibernateCfgPath).buildSessionFactory();
if (log.isDebugEnabled())
log.debug("Configured session factory using classpath resource: " + hibernateCfgPath);
} else {
if (hibernateProps == null) {
U.warn(
log, "No Hibernate configuration has been provided for store (will use default).");
hibernateProps = new Properties();
hibernateProps.setProperty("hibernate.connection.url", DFLT_CONN_URL);
hibernateProps.setProperty("hibernate.show_sql", DFLT_SHOW_SQL);
hibernateProps.setProperty("hibernate.hbm2ddl.auto", DFLT_HBM2DDL_AUTO);
}
Configuration cfg = new Configuration();
cfg.setProperties(hibernateProps);
assert resourceAvailable(MAPPING_RESOURCE);
cfg.addResource(MAPPING_RESOURCE);
sesFactory = cfg.buildSessionFactory();
if (log.isDebugEnabled())
log.debug("Configured session factory using properties: " + hibernateProps);
}
} catch (HibernateException e) {
throw new GridException("Failed to initialize store.", e);
} finally {
initLatch.countDown();
}
} else if (initLatch.getCount() > 0) U.await(initLatch);
if (sesFactory == null) throw new GridException("Cache store was not properly initialized.");
}
/**
* Asynchronous sequence update operation. Will add given amount to the sequence value.
*
* @param l Increment amount.
* @param updateCall Cache call that will update sequence reservation count in accordance with l.
* @param updated If {@code true}, will return sequence value after update, otherwise will return
* sequence value prior to update.
* @return Future indicating sequence value.
* @throws GridException If update failed.
*/
private GridFuture<Long> internalUpdateAsync(
long l, @Nullable Callable<Long> updateCall, boolean updated) throws GridException {
checkRemoved();
A.ensure(l > 0, " Parameter mustn't be less then 1: " + l);
lock.lock();
try {
// If reserved range isn't exhausted.
if (locVal + l <= upBound) {
long curVal = locVal;
locVal += l;
return new GridFinishedFuture<Long>(ctx.kernalContext(), updated ? locVal : curVal);
}
} finally {
lock.unlock();
}
if (updateCall == null) updateCall = internalUpdate(l, updated);
while (true) {
if (updateGuard.compareAndSet(false, true)) {
try {
// This call must be outside lock.
return ctx.closures().callLocalSafe(updateCall, true);
} finally {
lock.lock();
try {
updateGuard.set(false);
cond.signalAll();
} finally {
lock.unlock();
}
}
} else {
lock.lock();
try {
while (locVal >= upBound && updateGuard.get()) {
try {
cond.await(500, MILLISECONDS);
} catch (InterruptedException e) {
throw new GridInterruptedException(e);
}
}
checkRemoved();
// If reserved range isn't exhausted.
if (locVal + l <= upBound) {
long curVal = locVal;
locVal += l;
return new GridFinishedFuture<Long>(ctx.kernalContext(), updated ? locVal : curVal);
}
} finally {
lock.unlock();
}
}
}
}
/**
* Synchronous sequence update operation. Will add given amount to the sequence value.
*
* @param l Increment amount.
* @param updateCall Cache call that will update sequence reservation count in accordance with l.
* @param updated If {@code true}, will return sequence value after update, otherwise will return
* sequence value prior to update.
* @return Sequence value.
* @throws GridException If update failed.
*/
private long internalUpdate(long l, @Nullable Callable<Long> updateCall, boolean updated)
throws GridException {
checkRemoved();
assert l > 0;
lock.lock();
try {
// If reserved range isn't exhausted.
if (locVal + l <= upBound) {
long curVal = locVal;
locVal += l;
return updated ? locVal : curVal;
}
} finally {
lock.unlock();
}
if (updateCall == null) updateCall = internalUpdate(l, updated);
while (true) {
if (updateGuard.compareAndSet(false, true)) {
try {
// This call must be outside lock.
return CU.outTx(updateCall, ctx);
} finally {
lock.lock();
try {
updateGuard.set(false);
cond.signalAll();
} finally {
lock.unlock();
}
}
} else {
lock.lock();
try {
while (locVal >= upBound && updateGuard.get()) {
try {
cond.await(500, MILLISECONDS);
} catch (InterruptedException e) {
throw new GridInterruptedException(e);
}
}
checkRemoved();
// If reserved range isn't exhausted.
if (locVal + l <= upBound) {
long curVal = locVal;
locVal += l;
return updated ? locVal : curVal;
}
} finally {
lock.unlock();
}
}
}
}
@Override
protected <T, E> Predicate<T> createPredicate(final EntityCache<T> cache) {
if (column == null && this.nodes == null) return null;
final EntityCache<E> joinCache = this.joinEntity.getCache();
final AtomicBoolean more = new AtomicBoolean();
Predicate<E> filter = createJoinPredicate(more);
Predicate<T> rs = null;
if (filter == null && !more.get()) return rs;
if (filter != null) {
final Predicate<E> inner = filter;
rs =
new Predicate<T>() {
@Override
public boolean test(final T t) {
Predicate<E> joinPredicate = null;
for (String joinColumn : joinColumns) {
final Serializable key = cache.getAttribute(joinColumn).get(t);
final Attribute<E, Serializable> joinAttr = joinCache.getAttribute(joinColumn);
Predicate<E> p = (E e) -> key.equals(joinAttr.get(e));
joinPredicate = joinPredicate == null ? p : joinPredicate.and(p);
}
return joinCache.exists(inner.and(joinPredicate));
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append(" #-- ON ")
.append(joinColumns[0])
.append("=")
.append(joinClass == null ? "null" : joinClass.getSimpleName())
.append(".")
.append(joinColumns[0]);
for (int i = 1; i < joinColumns.length; i++) {
sb.append(" AND ")
.append(joinColumns[i])
.append("=")
.append(joinClass == null ? "null" : joinClass.getSimpleName())
.append(".")
.append(joinColumns[i]);
}
sb.append(" --# ").append(inner.toString());
return sb.toString();
}
};
}
if (more.get()) { // 存在不同Class的关联表
if (this.nodes != null) {
for (FilterNode node : this.nodes) {
if (((FilterJoinNode) node).joinClass == this.joinClass) continue;
Predicate<T> f = node.createPredicate(cache);
if (f == null) continue;
final Predicate<T> one = rs;
final Predicate<T> two = f;
rs =
(rs == null)
? f
: (or
? new Predicate<T>() {
@Override
public boolean test(T t) {
return one.test(t) || two.test(t);
}
@Override
public String toString() {
return "(" + one + " OR " + two + ")";
}
}
: new Predicate<T>() {
@Override
public boolean test(T t) {
return one.test(t) && two.test(t);
}
@Override
public String toString() {
return "(" + one + " AND " + two + ")";
}
});
}
}
}
return rs;
}
public void signalLoopbreak() {
loopBreaker.set(true);
if (mySelector != null) mySelector.wakeup();
}
void runLoopbreaks() {
if (loopBreaker.getAndSet(false)) {
eventCallback(0, EM_LOOPBREAK_SIGNAL, null);
}
}
/** @param res Result callback. */
void onResult(GridNearLockResponse<K, V> res) {
if (rcvRes.compareAndSet(false, true)) {
if (res.error() != null) {
if (log.isDebugEnabled())
log.debug(
"Finishing mini future with an error due to error in response [miniFut="
+ this
+ ", res="
+ res
+ ']');
// Fail.
if (res.error() instanceof GridCacheLockTimeoutException) onDone(false);
else onDone(res.error());
return;
}
int i = 0;
long topVer = topSnapshot.get().topologyVersion();
for (K k : keys) {
while (true) {
GridNearCacheEntry<K, V> entry = cctx.near().entryExx(k, topVer);
try {
if (res.dhtVersion(i) == null) {
onDone(
new GridException(
"Failed to receive DHT version from remote node "
+ "(will fail the lock): "
+ res));
return;
}
GridTuple3<GridCacheVersion, V, byte[]> oldValTup = valMap.get(entry.key());
V oldVal = entry.rawGet();
boolean hasOldVal = false;
V newVal = res.value(i);
byte[] newBytes = res.valueBytes(i);
boolean readRecordable = false;
if (retval) {
readRecordable = cctx.events().isRecordable(EVT_CACHE_OBJECT_READ);
if (readRecordable) hasOldVal = entry.hasValue();
}
GridCacheVersion dhtVer = res.dhtVersion(i);
GridCacheVersion mappedVer = res.mappedVersion(i);
if (newVal == null) {
if (oldValTup != null) {
if (oldValTup.get1().equals(dhtVer)) {
newVal = oldValTup.get2();
newBytes = oldValTup.get3();
}
oldVal = oldValTup.get2();
}
}
// Lock is held at this point, so we can set the
// returned value if any.
entry.resetFromPrimary(newVal, newBytes, lockVer, dhtVer, node.id());
if (inTx() && implicitTx() && tx.onePhaseCommit()) {
boolean pass = res.filterResult(i);
tx.entry(k).filters(pass ? CU.<K, V>empty() : CU.<K, V>alwaysFalse());
}
entry.readyNearLock(
lockVer,
mappedVer,
res.committedVersions(),
res.rolledbackVersions(),
res.pending());
if (retval) {
if (readRecordable)
cctx.events()
.addEvent(
entry.partition(),
entry.key(),
tx,
null,
EVT_CACHE_OBJECT_READ,
newVal,
newVal != null || newBytes != null,
oldVal,
hasOldVal,
CU.subjectId(tx, cctx));
cctx.cache().metrics0().onRead(false);
}
if (log.isDebugEnabled())
log.debug("Processed response for entry [res=" + res + ", entry=" + entry + ']');
break; // Inner while loop.
} catch (GridCacheEntryRemovedException ignored) {
if (log.isDebugEnabled())
log.debug("Failed to add candidates because entry was removed (will renew).");
// Replace old entry with new one.
entries.set(i, (GridDistributedCacheEntry<K, V>) cctx.cache().entryEx(entry.key()));
} catch (GridException e) {
onDone(e);
return;
}
}
i++;
}
try {
proceedMapping(mappings);
} catch (GridException e) {
onDone(e);
}
onDone(true);
}
}
/**
* Grabs local events and detects if events was lost since last poll.
*
* @param ignite Target grid.
* @param evtOrderKey Unique key to take last order key from node local map.
* @param evtThrottleCntrKey Unique key to take throttle count from node local map.
* @param evtTypes Event types to collect.
* @param evtMapper Closure to map grid events to Visor data transfer objects.
* @return Collections of node events
*/
public static Collection<VisorGridEvent> collectEvents(
Ignite ignite,
String evtOrderKey,
String evtThrottleCntrKey,
final int[] evtTypes,
IgniteClosure<Event, VisorGridEvent> evtMapper) {
assert ignite != null;
assert evtTypes != null && evtTypes.length > 0;
ConcurrentMap<String, Long> nl = ignite.cluster().nodeLocalMap();
final long lastOrder = getOrElse(nl, evtOrderKey, -1L);
final long throttle = getOrElse(nl, evtThrottleCntrKey, 0L);
// When we first time arrive onto a node to get its local events,
// we'll grab only last those events that not older than given period to make sure we are
// not grabbing GBs of data accidentally.
final long notOlderThan = System.currentTimeMillis() - EVENTS_COLLECT_TIME_WINDOW;
// Flag for detecting gaps between events.
final AtomicBoolean lastFound = new AtomicBoolean(lastOrder < 0);
IgnitePredicate<Event> p =
new IgnitePredicate<Event>() {
/** */
private static final long serialVersionUID = 0L;
@Override
public boolean apply(Event e) {
// Detects that events were lost.
if (!lastFound.get() && (lastOrder == e.localOrder())) lastFound.set(true);
// Retains events by lastOrder, period and type.
return e.localOrder() > lastOrder
&& e.timestamp() > notOlderThan
&& F.contains(evtTypes, e.type());
}
};
Collection<Event> evts = ignite.events().localQuery(p);
// Update latest order in node local, if not empty.
if (!evts.isEmpty()) {
Event maxEvt = Collections.max(evts, EVTS_ORDER_COMPARATOR);
nl.put(evtOrderKey, maxEvt.localOrder());
}
// Update throttle counter.
if (!lastFound.get())
nl.put(evtThrottleCntrKey, throttle == 0 ? EVENTS_LOST_THROTTLE : throttle - 1);
boolean lost = !lastFound.get() && throttle == 0;
Collection<VisorGridEvent> res = new ArrayList<>(evts.size() + (lost ? 1 : 0));
if (lost) res.add(new VisorGridEventsLost(ignite.cluster().localNode().id()));
for (Event e : evts) {
VisorGridEvent visorEvt = evtMapper.apply(e);
if (visorEvt != null) res.add(visorEvt);
}
return res;
}
/**
* @param cacheMode Cache mode.
* @param sameAff If {@code false} uses different number of partitions for caches.
* @param concurrency Transaction concurrency.
* @param isolation Transaction isolation.
* @throws Exception If failed.
*/
private void crossCacheTxFailover(
CacheMode cacheMode,
boolean sameAff,
final TransactionConcurrency concurrency,
final TransactionIsolation isolation)
throws Exception {
IgniteKernal ignite0 = (IgniteKernal) ignite(0);
final AtomicBoolean stop = new AtomicBoolean();
try {
ignite0.createCache(cacheConfiguration(CACHE1, cacheMode, 256));
ignite0.createCache(cacheConfiguration(CACHE2, cacheMode, sameAff ? 256 : 128));
final AtomicInteger threadIdx = new AtomicInteger();
IgniteInternalFuture<?> fut =
GridTestUtils.runMultiThreadedAsync(
new Callable<Void>() {
@Override
public Void call() throws Exception {
int idx = threadIdx.getAndIncrement();
Ignite ignite = ignite(idx % GRID_CNT);
log.info(
"Started update thread [node="
+ ignite.name()
+ ", client="
+ ignite.configuration().isClientMode()
+ ']');
IgniteCache<TestKey, TestValue> cache1 = ignite.cache(CACHE1);
IgniteCache<TestKey, TestValue> cache2 = ignite.cache(CACHE2);
assertNotSame(cache1, cache2);
IgniteTransactions txs = ignite.transactions();
ThreadLocalRandom rnd = ThreadLocalRandom.current();
long iter = 0;
while (!stop.get()) {
boolean sameKey = rnd.nextBoolean();
try {
try (Transaction tx = txs.txStart(concurrency, isolation)) {
if (sameKey) {
TestKey key = new TestKey(rnd.nextLong(KEY_RANGE));
cacheOperation(rnd, cache1, key);
cacheOperation(rnd, cache2, key);
} else {
TestKey key1 = new TestKey(rnd.nextLong(KEY_RANGE));
TestKey key2 = new TestKey(key1.key() + 1);
cacheOperation(rnd, cache1, key1);
cacheOperation(rnd, cache2, key2);
}
tx.commit();
}
} catch (CacheException | IgniteException e) {
log.info("Update error: " + e);
}
if (iter++ % 500 == 0) log.info("Iteration: " + iter);
}
return null;
}
/**
* @param rnd Random.
* @param cache Cache.
* @param key Key.
*/
private void cacheOperation(
ThreadLocalRandom rnd, IgniteCache<TestKey, TestValue> cache, TestKey key) {
switch (rnd.nextInt(4)) {
case 0:
cache.put(key, new TestValue(rnd.nextLong()));
break;
case 1:
cache.remove(key);
break;
case 2:
cache.invoke(key, new TestEntryProcessor(rnd.nextBoolean() ? 1L : null));
break;
case 3:
cache.get(key);
break;
default:
assert false;
}
}
},
10,
"tx-thread");
long stopTime = System.currentTimeMillis() + 3 * 60_000;
long topVer = ignite0.cluster().topologyVersion();
boolean failed = false;
while (System.currentTimeMillis() < stopTime) {
log.info("Start node.");
IgniteKernal ignite = (IgniteKernal) startGrid(GRID_CNT);
assertFalse(ignite.configuration().isClientMode());
topVer++;
IgniteInternalFuture<?> affFut =
ignite
.context()
.cache()
.context()
.exchange()
.affinityReadyFuture(new AffinityTopologyVersion(topVer));
try {
if (affFut != null) affFut.get(30_000);
} catch (IgniteFutureTimeoutCheckedException e) {
log.error("Failed to wait for affinity future after start: " + topVer);
failed = true;
break;
}
Thread.sleep(500);
log.info("Stop node.");
stopGrid(GRID_CNT);
topVer++;
affFut =
ignite0
.context()
.cache()
.context()
.exchange()
.affinityReadyFuture(new AffinityTopologyVersion(topVer));
try {
if (affFut != null) affFut.get(30_000);
} catch (IgniteFutureTimeoutCheckedException e) {
log.error("Failed to wait for affinity future after stop: " + topVer);
failed = true;
break;
}
}
stop.set(true);
fut.get();
assertFalse("Test failed, see log for details.", failed);
} finally {
stop.set(true);
ignite0.destroyCache(CACHE1);
ignite0.destroyCache(CACHE2);
awaitPartitionMapExchange();
}
}