public void newCharacter(CharacterEvent ce) {
int oldChar2type;
// Previous character not typed correctly - 1 point penalty
if (ce.source == generator.get()) {
oldChar2type = char2type.getAndSet(ce.character); // 接收随机字母
if (oldChar2type != -1) {
score.decrementAndGet();
setScore();
}
}
// If character is extraneous - 1 point penalty
// If character does not match - 1 point penalty
else if (ce.source == typist.get()) {
// 在此有个假设：即正在使用的该变量值不会被变更且程序代码完成时也是如此
// 所有已经被我们设定的具有特定值的变量就应当是那个值。
while (true) {
oldChar2type = char2type.get(); // 获取上次已接收到的随机字母
if (oldChar2type != ce.character) { // ce.character是用户输入的字母，现在作比较
score.decrementAndGet();
break;
} else if (char2type.compareAndSet(oldChar2type, -1)) {
score.incrementAndGet();
break;
}
}
setScore();
}
}
/**
* The error from the user provided Observable is handled by the subscribe try/catch because this
* is synchronous
*
* <p>Result: Passes
*/
@Test
public void testCustomObservableWithErrorInObservableSynchronous() {
final AtomicInteger count = new AtomicInteger();
final AtomicReference<Throwable> error = new AtomicReference<>();
// FIXME custom built???
Observable.just("1", "2")
.concatWith(Observable.error(() -> new NumberFormatException()))
.subscribe(
new Observer<String>() {
@Override
public void onComplete() {
System.out.println("completed");
}
@Override
public void onError(Throwable e) {
error.set(e);
System.out.println("error");
e.printStackTrace();
}
@Override
public void onNext(String v) {
System.out.println(v);
count.incrementAndGet();
}
});
assertEquals(2, count.get());
assertNotNull(error.get());
if (!(error.get() instanceof NumberFormatException)) {
fail("It should be a NumberFormatException");
}
}
/**
* https://github.com/ReactiveX/RxJava/issues/198
*
* <p>Rx Design Guidelines 5.2
*
* <p>"when calling the Subscribe method that only has an onNext argument, the OnError behavior
* will be to rethrow the exception on the thread that the message comes out from the Observable.
* The OnCompleted behavior in this case is to do nothing."
*
* @throws InterruptedException
*/
@Test
@Ignore("Subscribers can't throw")
public void testErrorThrownWithoutErrorHandlerAsynchronous() throws InterruptedException {
final CountDownLatch latch = new CountDownLatch(1);
final AtomicReference<Throwable> exception = new AtomicReference<>();
Observable.create(
observer -> {
new Thread(
() -> {
try {
observer.onError(new Error("failure"));
} catch (Throwable e) {
// without an onError handler it has to just throw on whatever thread
// invokes it
exception.set(e);
}
latch.countDown();
})
.start();
})
.subscribe();
// wait for exception
latch.await(3000, TimeUnit.MILLISECONDS);
assertNotNull(exception.get());
assertEquals("failure", exception.get().getMessage());
}
@Test
public void testTakeWithErrorInObserver() {
final AtomicInteger count = new AtomicInteger();
final AtomicReference<Throwable> error = new AtomicReference<>();
Observable.just("1", "2", "three", "4")
.take(3)
.safeSubscribe(
new Observer<String>() {
@Override
public void onComplete() {
System.out.println("completed");
}
@Override
public void onError(Throwable e) {
error.set(e);
System.out.println("error");
e.printStackTrace();
}
@Override
public void onNext(String v) {
int num = Integer.parseInt(v);
System.out.println(num);
// doSomething(num);
count.incrementAndGet();
}
});
assertEquals(2, count.get());
assertNotNull(error.get());
if (!(error.get() instanceof NumberFormatException)) {
fail("It should be a NumberFormatException");
}
}
/** {@inheritDoc} */
@Override
public GridDiscoveryTopologySnapshot topologySnapshot() throws IgniteCheckedException {
get();
if (topSnapshot.get() == null)
topSnapshot.compareAndSet(
null,
new GridDiscoveryTopologySnapshot(discoEvt.topologyVersion(), discoEvt.topologyNodes()));
return topSnapshot.get();
}
@Ignore("Non-positive requests are relayed to the plugin and is a no-op otherwise")
@Test
public void testNegativeRequestThrowsIllegalArgumentException() throws InterruptedException {
final CountDownLatch latch = new CountDownLatch(1);
final AtomicReference<Throwable> exception = new AtomicReference<>();
Observable.just(1, 2, 3, 4)
.subscribe(
new Observer<Integer>() {
@Override
public void onStart() {
request(1);
}
@Override
public void onComplete() {}
@Override
public void onError(Throwable e) {
exception.set(e);
latch.countDown();
}
@Override
public void onNext(Integer t) {
request(-1);
request(1);
}
});
Assert.assertTrue(latch.await(10, TimeUnit.SECONDS));
Assert.assertTrue(exception.get() instanceof IllegalArgumentException);
}
/**
* Completeness callback.
*
* @param success {@code True} if lock was acquired.
* @param distribute {@code True} if need to distribute lock removal in case of failure.
* @return {@code True} if complete by this operation.
*/
private boolean onComplete(boolean success, boolean distribute) {
if (log.isDebugEnabled())
log.debug(
"Received onComplete(..) callback [success="
+ success
+ ", distribute="
+ distribute
+ ", fut="
+ this
+ ']');
if (!success) undoLocks(distribute);
if (tx != null) cctx.tm().txContext(tx);
if (super.onDone(success, err.get())) {
if (log.isDebugEnabled()) log.debug("Completing future: " + this);
// Clean up.
cctx.mvcc().removeFuture(this);
if (timeoutObj != null) cctx.time().removeTimeoutObject(timeoutObj);
return true;
}
return false;
}
/** @param evts Events to add. */
private void addAll(Collection<E> evts) {
WindowHolder tup = ref.get();
int cnt = addAllInternal(evts, tup.collection(), tup.set());
tup.size().addAndGet(cnt);
}
private void recheck() {
// If this is the oldest node.
if (oldestNode.get().id().equals(cctx.localNodeId())) {
Collection<UUID> remaining = remaining();
if (!remaining.isEmpty()) {
try {
cctx.io()
.safeSend(
cctx.discovery().nodes(remaining),
new GridDhtPartitionsSingleRequest(exchId),
SYSTEM_POOL,
null);
} catch (IgniteCheckedException e) {
U.error(
log,
"Failed to request partitions from nodes [exchangeId="
+ exchId
+ ", nodes="
+ remaining
+ ']',
e);
}
}
// Resend full partition map because last attempt failed.
else {
if (spreadPartitions()) onDone(exchId.topologyVersion());
}
} else sendPartitions();
// Schedule another send.
scheduleRecheck();
}
/** {@inheritDoc} */
@Override
protected Collection<E> dequeue0(int cnt) {
WindowHolder tup = ref.get();
AtomicInteger size = tup.size();
Collection<T> evts = tup.collection();
Collection<E> resCol = new ArrayList<>(cnt);
while (true) {
int curSize = size.get();
if (curSize > 0) {
if (size.compareAndSet(curSize, curSize - 1)) {
E res = pollInternal(evts, tup.set());
if (res != null) {
resCol.add(res);
if (resCol.size() >= cnt) return resCol;
} else {
size.incrementAndGet();
return resCol;
}
}
} else return resCol;
}
}
/**
* Poll evicted internal implementation.
*
* @return Evicted element.
*/
@Nullable
private E pollEvictedInternal() {
WindowHolder tup = ref.get();
AtomicInteger size = tup.size();
while (true) {
int curSize = size.get();
if (curSize > maxSize) {
if (size.compareAndSet(curSize, curSize - 1)) {
E evt = pollInternal(tup.collection(), tup.set());
if (evt != null) return evt;
else {
// No actual events in queue, it means that other thread is just adding event.
// return null as it is a concurrent add call.
size.incrementAndGet();
return null;
}
}
} else return null;
}
}
/**
* @param nodes Nodes.
* @param id ID.
* @throws IgniteCheckedException If failed.
*/
private void sendAllPartitions(
Collection<? extends ClusterNode> nodes, GridDhtPartitionExchangeId id)
throws IgniteCheckedException {
GridDhtPartitionsFullMessage m =
new GridDhtPartitionsFullMessage(id, lastVer.get(), id.topologyVersion());
for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
if (!cacheCtx.isLocal()) {
AffinityTopologyVersion startTopVer = cacheCtx.startTopologyVersion();
boolean ready = startTopVer == null || startTopVer.compareTo(id.topologyVersion()) <= 0;
if (ready)
m.addFullPartitionsMap(cacheCtx.cacheId(), cacheCtx.topology().partitionMap(true));
}
}
// It is important that client topologies be added after contexts.
for (GridClientPartitionTopology top : cctx.exchange().clientTopologies())
m.addFullPartitionsMap(top.cacheId(), top.partitionMap(true));
if (log.isDebugEnabled())
log.debug(
"Sending full partition map [nodeIds="
+ F.viewReadOnly(nodes, F.node2id())
+ ", exchId="
+ exchId
+ ", msg="
+ m
+ ']');
cctx.io().safeSend(nodes, m, SYSTEM_POOL, null);
}
public void setUpper(int i) {
while (true) {
IntPair oldv = values.get();
if (i < oldv.lower)
throw new IllegalArgumentException("Can't set upper to " + i + " < lower");
IntPair newv = new IntPair(oldv.lower, i);
if (values.compareAndSet(oldv, newv)) return;
}
}
/**
* The error from the user provided Observer is not handled by the subscribe method try/catch.
*
* <p>It is handled by the AtomicObserver that wraps the provided Observer.
*
* <p>Result: Passes (if AtomicObserver functionality exists)
*/
@Test
public void testCustomObservableWithErrorInObserverAsynchronous() throws InterruptedException {
final CountDownLatch latch = new CountDownLatch(1);
final AtomicInteger count = new AtomicInteger();
final AtomicReference<Throwable> error = new AtomicReference<>();
// FIXME custom built???
Observable.just("1", "2", "three", "4")
.subscribeOn(Schedulers.newThread())
.safeSubscribe(
new Observer<String>() {
@Override
public void onComplete() {
System.out.println("completed");
latch.countDown();
}
@Override
public void onError(Throwable e) {
error.set(e);
System.out.println("error");
e.printStackTrace();
latch.countDown();
}
@Override
public void onNext(String v) {
int num = Integer.parseInt(v);
System.out.println(num);
// doSomething(num);
count.incrementAndGet();
}
});
// wait for async sequence to complete
latch.await();
assertEquals(2, count.get());
assertNotNull(error.get());
if (!(error.get() instanceof NumberFormatException)) {
fail("It should be a NumberFormatException");
}
}
@Test
public void testScanShouldNotRequestZero() {
final AtomicReference<Subscription> producer = new AtomicReference<>();
Observable<Integer> o =
Observable.create(
new Publisher<Integer>() {
@Override
public void subscribe(final Subscriber<? super Integer> subscriber) {
Subscription p =
spy(
new Subscription() {
private AtomicBoolean requested = new AtomicBoolean(false);
@Override
public void request(long n) {
if (requested.compareAndSet(false, true)) {
subscriber.onNext(1);
subscriber.onComplete();
}
}
@Override
public void cancel() {}
});
producer.set(p);
subscriber.onSubscribe(p);
}
})
.scan(
100,
new BiFunction<Integer, Integer, Integer>() {
@Override
public Integer apply(Integer t1, Integer t2) {
return t1 + t2;
}
});
o.subscribe(
new TestSubscriber<Integer>(1L) {
@Override
public void onNext(Integer integer) {
request(1);
}
});
verify(producer.get(), never()).request(0);
verify(producer.get(), times(3)).request(1); // FIXME this was 2 in 1.x
}
/** {@inheritDoc} */
@Override
public int size() {
int size = ref.get().size().get();
return size > 0 ? size : 0;
}
/**
* Starts activity.
*
* @throws IgniteInterruptedCheckedException If interrupted.
*/
public void init() throws IgniteInterruptedCheckedException {
if (isDone()) return;
if (init.compareAndSet(false, true)) {
if (isDone()) return;
try {
// Wait for event to occur to make sure that discovery
// will return corresponding nodes.
U.await(evtLatch);
assert discoEvt != null : this;
assert !dummy && !forcePreload : this;
ClusterNode oldest = CU.oldestAliveCacheServerNode(cctx, exchId.topologyVersion());
oldestNode.set(oldest);
startCaches();
// True if client node joined or failed.
boolean clientNodeEvt;
if (F.isEmpty(reqs)) {
int type = discoEvt.type();
assert type == EVT_NODE_JOINED || type == EVT_NODE_LEFT || type == EVT_NODE_FAILED
: discoEvt;
clientNodeEvt = CU.clientNode(discoEvt.eventNode());
} else {
assert discoEvt.type() == EVT_DISCOVERY_CUSTOM_EVT : discoEvt;
boolean clientOnlyStart = true;
for (DynamicCacheChangeRequest req : reqs) {
if (!req.clientStartOnly()) {
clientOnlyStart = false;
break;
}
}
clientNodeEvt = clientOnlyStart;
}
if (clientNodeEvt) {
ClusterNode node = discoEvt.eventNode();
// Client need to initialize affinity for local join event or for stated client caches.
if (!node.isLocal()) {
for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
if (cacheCtx.isLocal()) continue;
GridDhtPartitionTopology top = cacheCtx.topology();
top.updateTopologyVersion(exchId, this, -1, stopping(cacheCtx.cacheId()));
if (cacheCtx.affinity().affinityTopologyVersion() == AffinityTopologyVersion.NONE) {
initTopology(cacheCtx);
top.beforeExchange(this);
} else
cacheCtx.affinity().clientEventTopologyChange(discoEvt, exchId.topologyVersion());
}
if (exchId.isLeft())
cctx.mvcc().removeExplicitNodeLocks(exchId.nodeId(), exchId.topologyVersion());
onDone(exchId.topologyVersion());
skipPreload = cctx.kernalContext().clientNode();
return;
}
}
if (cctx.kernalContext().clientNode()) {
skipPreload = true;
for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
if (cacheCtx.isLocal()) continue;
GridDhtPartitionTopology top = cacheCtx.topology();
top.updateTopologyVersion(exchId, this, -1, stopping(cacheCtx.cacheId()));
}
for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
if (cacheCtx.isLocal()) continue;
initTopology(cacheCtx);
}
if (oldestNode.get() != null) {
rmtNodes =
new ConcurrentLinkedQueue<>(
CU.aliveRemoteServerNodesWithCaches(cctx, exchId.topologyVersion()));
rmtIds = Collections.unmodifiableSet(new HashSet<>(F.nodeIds(rmtNodes)));
ready.set(true);
initFut.onDone(true);
if (log.isDebugEnabled()) log.debug("Initialized future: " + this);
sendPartitions();
} else onDone(exchId.topologyVersion());
return;
}
assert oldestNode.get() != null;
for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
if (isCacheAdded(cacheCtx.cacheId(), exchId.topologyVersion())) {
if (cacheCtx
.discovery()
.cacheAffinityNodes(cacheCtx.name(), topologyVersion())
.isEmpty())
U.quietAndWarn(log, "No server nodes found for cache client: " + cacheCtx.namex());
}
cacheCtx.preloader().onExchangeFutureAdded();
}
List<String> cachesWithoutNodes = null;
if (exchId.isLeft()) {
for (String name : cctx.cache().cacheNames()) {
if (cctx.discovery().cacheAffinityNodes(name, topologyVersion()).isEmpty()) {
if (cachesWithoutNodes == null) cachesWithoutNodes = new ArrayList<>();
cachesWithoutNodes.add(name);
// Fire event even if there is no client cache started.
if (cctx.gridEvents().isRecordable(EventType.EVT_CACHE_NODES_LEFT)) {
Event evt =
new CacheEvent(
name,
cctx.localNode(),
cctx.localNode(),
"All server nodes have left the cluster.",
EventType.EVT_CACHE_NODES_LEFT,
0,
false,
null,
null,
null,
null,
false,
null,
false,
null,
null,
null);
cctx.gridEvents().record(evt);
}
}
}
}
if (cachesWithoutNodes != null) {
StringBuilder sb =
new StringBuilder(
"All server nodes for the following caches have left the cluster: ");
for (int i = 0; i < cachesWithoutNodes.size(); i++) {
String cache = cachesWithoutNodes.get(i);
sb.append('\'').append(cache).append('\'');
if (i != cachesWithoutNodes.size() - 1) sb.append(", ");
}
U.quietAndWarn(log, sb.toString());
U.quietAndWarn(log, "Must have server nodes for caches to operate.");
}
assert discoEvt != null;
assert exchId.nodeId().equals(discoEvt.eventNode().id());
for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
GridClientPartitionTopology clientTop =
cctx.exchange().clearClientTopology(cacheCtx.cacheId());
long updSeq = clientTop == null ? -1 : clientTop.lastUpdateSequence();
// Update before waiting for locks.
if (!cacheCtx.isLocal())
cacheCtx
.topology()
.updateTopologyVersion(exchId, this, updSeq, stopping(cacheCtx.cacheId()));
}
// Grab all alive remote nodes with order of equal or less than last joined node.
rmtNodes =
new ConcurrentLinkedQueue<>(
CU.aliveRemoteServerNodesWithCaches(cctx, exchId.topologyVersion()));
rmtIds = Collections.unmodifiableSet(new HashSet<>(F.nodeIds(rmtNodes)));
for (Map.Entry<UUID, GridDhtPartitionsSingleMessage> m : singleMsgs.entrySet())
// If received any messages, process them.
onReceive(m.getKey(), m.getValue());
for (Map.Entry<UUID, GridDhtPartitionsFullMessage> m : fullMsgs.entrySet())
// If received any messages, process them.
onReceive(m.getKey(), m.getValue());
AffinityTopologyVersion topVer = exchId.topologyVersion();
for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
if (cacheCtx.isLocal()) continue;
// Must initialize topology after we get discovery event.
initTopology(cacheCtx);
cacheCtx.preloader().updateLastExchangeFuture(this);
}
IgniteInternalFuture<?> partReleaseFut = cctx.partitionReleaseFuture(topVer);
// Assign to class variable so it will be included into toString() method.
this.partReleaseFut = partReleaseFut;
if (log.isDebugEnabled()) log.debug("Before waiting for partition release future: " + this);
while (true) {
try {
partReleaseFut.get(2 * cctx.gridConfig().getNetworkTimeout(), TimeUnit.MILLISECONDS);
break;
} catch (IgniteFutureTimeoutCheckedException ignored) {
// Print pending transactions and locks that might have led to hang.
dumpPendingObjects();
}
}
if (log.isDebugEnabled()) log.debug("After waiting for partition release future: " + this);
if (!F.isEmpty(reqs)) blockGateways();
if (exchId.isLeft())
cctx.mvcc().removeExplicitNodeLocks(exchId.nodeId(), exchId.topologyVersion());
IgniteInternalFuture<?> locksFut = cctx.mvcc().finishLocks(exchId.topologyVersion());
while (true) {
try {
locksFut.get(2 * cctx.gridConfig().getNetworkTimeout(), TimeUnit.MILLISECONDS);
break;
} catch (IgniteFutureTimeoutCheckedException ignored) {
U.warn(
log,
"Failed to wait for locks release future. "
+ "Dumping pending objects that might be the cause: "
+ cctx.localNodeId());
U.warn(log, "Locked entries:");
Map<IgniteTxKey, Collection<GridCacheMvccCandidate>> locks =
cctx.mvcc().unfinishedLocks(exchId.topologyVersion());
for (Map.Entry<IgniteTxKey, Collection<GridCacheMvccCandidate>> e : locks.entrySet())
U.warn(log, "Locked entry [key=" + e.getKey() + ", mvcc=" + e.getValue() + ']');
}
}
for (GridCacheContext cacheCtx : cctx.cacheContexts()) {
if (cacheCtx.isLocal()) continue;
// Notify replication manager.
GridCacheContext drCacheCtx =
cacheCtx.isNear() ? cacheCtx.near().dht().context() : cacheCtx;
if (drCacheCtx.isDrEnabled()) drCacheCtx.dr().beforeExchange(topVer, exchId.isLeft());
// Partition release future is done so we can flush the write-behind store.
cacheCtx.store().forceFlush();
// Process queued undeploys prior to sending/spreading map.
cacheCtx.preloader().unwindUndeploys();
GridDhtPartitionTopology top = cacheCtx.topology();
assert topVer.equals(top.topologyVersion())
: "Topology version is updated only in this class instances inside single ExchangeWorker thread.";
top.beforeExchange(this);
}
for (GridClientPartitionTopology top : cctx.exchange().clientTopologies()) {
top.updateTopologyVersion(exchId, this, -1, stopping(top.cacheId()));
top.beforeExchange(this);
}
} catch (IgniteInterruptedCheckedException e) {
onDone(e);
throw e;
} catch (Throwable e) {
U.error(
log,
"Failed to reinitialize local partitions (preloading will be stopped): " + exchId,
e);
onDone(e);
if (e instanceof Error) throw (Error) e;
return;
}
if (F.isEmpty(rmtIds)) {
onDone(exchId.topologyVersion());
return;
}
ready.set(true);
initFut.onDone(true);
if (log.isDebugEnabled()) log.debug("Initialized future: " + this);
// If this node is not oldest.
if (!oldestNode.get().id().equals(cctx.localNodeId())) sendPartitions();
else {
boolean allReceived = allReceived();
if (allReceived && replied.compareAndSet(false, true)) {
if (spreadPartitions()) onDone(exchId.topologyVersion());
}
}
scheduleRecheck();
} else assert false : "Skipped init future: " + this;
}
/** {@inheritDoc} */
@Override
protected GridStreamerWindowIterator<E> iterator0() {
WindowHolder win = ref.get();
return iteratorInternal(win.collection(), win.set(), win.size());
}
/** @param evt Event to add. */
private void add(E evt) {
WindowHolder tup = ref.get();
if (addInternal(evt, tup.collection(), tup.set())) tup.size().incrementAndGet();
}
/**
* @param nodeId Sender node id.
* @param msg Single partition info.
*/
public void onReceive(final UUID nodeId, final GridDhtPartitionsSingleMessage msg) {
assert msg != null;
assert msg.exchangeId().equals(exchId);
// Update last seen version.
while (true) {
GridCacheVersion old = lastVer.get();
if (old == null || old.compareTo(msg.lastVersion()) < 0) {
if (lastVer.compareAndSet(old, msg.lastVersion())) break;
} else break;
}
if (isDone()) {
if (log.isDebugEnabled())
log.debug(
"Received message for finished future (will reply only to sender) [msg="
+ msg
+ ", fut="
+ this
+ ']');
sendAllPartitions(nodeId, cctx.gridConfig().getNetworkSendRetryCount());
} else {
initFut.listen(
new CI1<IgniteInternalFuture<Boolean>>() {
@Override
public void apply(IgniteInternalFuture<Boolean> t) {
try {
if (!t.get()) // Just to check if there was an error.
return;
ClusterNode loc = cctx.localNode();
singleMsgs.put(nodeId, msg);
boolean match = true;
// Check if oldest node has changed.
if (!oldestNode.get().equals(loc)) {
match = false;
synchronized (mux) {
// Double check.
if (oldestNode.get().equals(loc)) match = true;
}
}
if (match) {
boolean allReceived;
synchronized (rcvdIds) {
if (rcvdIds.add(nodeId)) updatePartitionSingleMap(msg);
allReceived = allReceived();
}
// If got all replies, and initialization finished, and reply has not been sent
// yet.
if (allReceived && ready.get() && replied.compareAndSet(false, true)) {
spreadPartitions();
onDone(exchId.topologyVersion());
} else if (log.isDebugEnabled())
log.debug(
"Exchange future full map is not sent [allReceived="
+ allReceived()
+ ", ready="
+ ready
+ ", replied="
+ replied.get()
+ ", init="
+ init.get()
+ ", fut="
+ this
+ ']');
}
} catch (IgniteCheckedException e) {
U.error(log, "Failed to initialize exchange future: " + this, e);
}
}
});
}
}
/**
* Get underlying collection.
*
* @return Collection.
*/
@SuppressWarnings("ConstantConditions")
protected Collection<T> collection() {
return ref.get().get1();
}
/** Checks window consistency. Used for testing. */
void consistencyCheck() {
WindowHolder win = ref.get();
consistencyCheck(win.collection(), win.set(), win.size());
}
public int getUpper() {
return values.get().upper;
}
public int getLower() {
return values.get().lower;
}
// use a small block size and a large write so that DN is busy creating
// new blocks. This makes it almost 100% sure we can reproduce
// case of client getting a DN that hasn't yet created the blocks
public void testImmediateReadOfNewFile() throws IOException {
final int blockSize = 64 * 1024;
final int writeSize = 10 * blockSize;
Configuration conf = new Configuration();
conf.setLong("dfs.block.size", blockSize);
init(conf);
final int requiredSuccessfulOpens = 100;
final Path file = new Path("/file1");
final AtomicBoolean openerDone = new AtomicBoolean(false);
final AtomicReference<String> errorMessage = new AtomicReference<String>();
final FSDataOutputStream out = fileSystem.create(file);
final Thread writer =
new Thread(
new Runnable() {
@Override
public void run() {
try {
while (!openerDone.get()) {
out.write(DFSTestUtil.generateSequentialBytes(0, writeSize));
out.hflush();
}
} catch (IOException e) {
LOG.warn("error in writer", e);
} finally {
try {
out.close();
} catch (IOException e) {
LOG.error("unable to close file");
}
}
}
});
Thread opener =
new Thread(
new Runnable() {
@Override
public void run() {
try {
for (int i = 0; i < requiredSuccessfulOpens; i++) {
fileSystem.open(file).close();
}
openerDone.set(true);
} catch (IOException e) {
openerDone.set(true);
errorMessage.set(
String.format("got exception : %s", StringUtils.stringifyException(e)));
} catch (Exception e) {
openerDone.set(true);
errorMessage.set(
String.format("got exception : %s", StringUtils.stringifyException(e)));
writer.interrupt();
fail("here");
}
}
});
writer.start();
opener.start();
try {
writer.join();
opener.join();
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
assertNull(errorMessage.get(), errorMessage.get());
}
/**
* Maps keys to nodes. Note that we can not simply group keys by nodes and send lock request as
* such approach does not preserve order of lock acquisition. Instead, keys are split in
* continuous groups belonging to one primary node and locks for these groups are acquired
* sequentially.
*
* @param keys Keys.
*/
private void map(Iterable<? extends K> keys) {
try {
GridDiscoveryTopologySnapshot snapshot = topSnapshot.get();
assert snapshot != null;
long topVer = snapshot.topologyVersion();
assert topVer > 0;
if (CU.affinityNodes(cctx, topVer).isEmpty()) {
onDone(
new GridTopologyException(
"Failed to map keys for near-only cache (all "
+ "partition nodes left the grid)."));
return;
}
ConcurrentLinkedDeque8<GridNearLockMapping<K, V>> mappings = new ConcurrentLinkedDeque8<>();
// Assign keys to primary nodes.
GridNearLockMapping<K, V> map = null;
for (K key : keys) {
GridNearLockMapping<K, V> updated = map(key, map, topVer);
// If new mapping was created, add to collection.
if (updated != map) mappings.add(updated);
map = updated;
}
if (isDone()) {
if (log.isDebugEnabled()) log.debug("Abandoning (re)map because future is done: " + this);
return;
}
if (log.isDebugEnabled())
log.debug("Starting (re)map for mappings [mappings=" + mappings + ", fut=" + this + ']');
// Create mini futures.
for (Iterator<GridNearLockMapping<K, V>> iter = mappings.iterator(); iter.hasNext(); ) {
GridNearLockMapping<K, V> mapping = iter.next();
GridNode node = mapping.node();
Collection<K> mappedKeys = mapping.mappedKeys();
assert !mappedKeys.isEmpty();
GridNearLockRequest<K, V> req = null;
Collection<K> distributedKeys = new ArrayList<>(mappedKeys.size());
boolean explicit = false;
for (K key : mappedKeys) {
while (true) {
GridNearCacheEntry<K, V> entry = null;
try {
entry = cctx.near().entryExx(key, topVer);
if (!cctx.isAll(entry.wrap(false), filter)) {
if (log.isDebugEnabled())
log.debug("Entry being locked did not pass filter (will not lock): " + entry);
onComplete(false, false);
return;
}
// Removed exception may be thrown here.
GridCacheMvccCandidate<K> cand = addEntry(topVer, entry, node.id());
if (isDone()) {
if (log.isDebugEnabled())
log.debug(
"Abandoning (re)map because future is done after addEntry attempt "
+ "[fut="
+ this
+ ", entry="
+ entry
+ ']');
return;
}
if (cand != null) {
if (tx == null && !cand.reentry())
cctx.mvcc().addExplicitLock(threadId, cand, snapshot);
GridTuple3<GridCacheVersion, V, byte[]> val = entry.versionedValue();
if (val == null) {
GridDhtCacheEntry<K, V> dhtEntry = dht().peekExx(key);
try {
if (dhtEntry != null) val = dhtEntry.versionedValue(topVer);
} catch (GridCacheEntryRemovedException ignored) {
assert dhtEntry.obsolete()
: " Got removed exception for non-obsolete entry: " + dhtEntry;
if (log.isDebugEnabled())
log.debug(
"Got removed exception for DHT entry in map (will ignore): " + dhtEntry);
}
}
GridCacheVersion dhtVer = null;
if (val != null) {
dhtVer = val.get1();
valMap.put(key, val);
}
if (!cand.reentry()) {
if (req == null) {
req =
new GridNearLockRequest<>(
topVer,
cctx.nodeId(),
threadId,
futId,
lockVer,
inTx(),
implicitTx(),
implicitSingleTx(),
read,
isolation(),
isInvalidate(),
timeout,
syncCommit(),
syncRollback(),
mappedKeys.size(),
inTx() ? tx.size() : mappedKeys.size(),
inTx() ? tx.groupLockKey() : null,
inTx() && tx.partitionLock(),
inTx() ? tx.subjectId() : null);
mapping.request(req);
}
distributedKeys.add(key);
GridCacheTxEntry<K, V> writeEntry = tx != null ? tx.writeMap().get(key) : null;
if (tx != null) tx.addKeyMapping(key, mapping.node());
req.addKeyBytes(
key,
node.isLocal() ? null : entry.getOrMarshalKeyBytes(),
retval && dhtVer == null,
dhtVer, // Include DHT version to match remote DHT entry.
writeEntry,
inTx() ? tx.entry(key).drVersion() : null,
cctx);
// Clear transfer required flag since we are sending message.
if (writeEntry != null) writeEntry.transferRequired(false);
}
if (cand.reentry())
explicit = tx != null && !entry.hasLockCandidate(tx.xidVersion());
} else
// Ignore reentries within transactions.
explicit = tx != null && !entry.hasLockCandidate(tx.xidVersion());
if (explicit) tx.addKeyMapping(key, mapping.node());
break;
} catch (GridCacheEntryRemovedException ignored) {
assert entry.obsolete() : "Got removed exception on non-obsolete entry: " + entry;
if (log.isDebugEnabled())
log.debug("Got removed entry in lockAsync(..) method (will retry): " + entry);
}
}
// Mark mapping explicit lock flag.
if (explicit) {
boolean marked = tx != null && tx.markExplicit(node.id());
assert tx == null || marked;
}
}
if (!distributedKeys.isEmpty()) mapping.distributedKeys(distributedKeys);
else {
assert mapping.request() == null;
iter.remove();
}
}
cctx.mvcc().recheckPendingLocks();
proceedMapping(mappings);
} catch (GridException ex) {
onError(ex);
}
}
/** Completeness callback. */
private void onComplete() {
onDone(tx, err.get());
}