@Test
public void testMapPutTtlWithListener() throws InterruptedException {
Config cfg = new Config();
TestHazelcastInstanceFactory factory = createHazelcastInstanceFactory(2);
final HazelcastInstance[] instances = factory.newInstances(cfg);
warmUpPartitions(instances);
final int k = 10;
final int putCount = 10000;
final CountDownLatch latch = new CountDownLatch(k * putCount);
final IMap map = instances[0].getMap("testMapEvictionTtlWithListener");
final AtomicBoolean error = new AtomicBoolean(false);
final Set<Long> times = Collections.newSetFromMap(new ConcurrentHashMap<Long, Boolean>());
map.addEntryListener(
new EntryAdapter() {
public void entryEvicted(final EntryEvent event) {
final Long expectedEvictionTime = (Long) (event.getOldValue());
long timeDifference = System.currentTimeMillis() - expectedEvictionTime;
if (timeDifference > 5000) {
error.set(true);
times.add(timeDifference);
}
latch.countDown();
}
},
true);
for (int i = 0; i < k; i++) {
final int threadId = i;
int ttl = (int) (Math.random() * 5000 + 3000);
for (int j = 0; j < putCount; j++) {
final long expectedEvictionTime = ttl + System.currentTimeMillis();
map.put(j + putCount * threadId, expectedEvictionTime, ttl, TimeUnit.MILLISECONDS);
}
}
assertTrue(latch.await(1, TimeUnit.MINUTES));
assertFalse("Some evictions took more than 3 seconds! -> " + times, error.get());
}
/*
* (non-Javadoc)
*
* @see
* org.mpilone.hazelcastmq.core.DefaultHazelcastMQConsumer.ReceiveStrategy
* #receive(com.hazelcast.core.IQueue)
*/
@Override
public byte[] receive(IQueue<byte[]> queue) {
if (closed) {
return null;
}
long t = Math.min(timeout, 500L);
timeout -= 500L;
try {
if (!active) {
receiveCondition.await(t, TimeUnit.MILLISECONDS);
} else {
return queue.poll(t, TimeUnit.MILLISECONDS);
}
} catch (InterruptedException ex) {
// Ignore for now
}
return null;
}
public BaseHazelCastStateTracker(String connectionString, String type, int stateTrackerPort)
throws Exception {
log.info(
"Setting up hazelcast with type "
+ type
+ " connection string "
+ connectionString
+ " and port "
+ stateTrackerPort);
if (type.equals("master") && !PortTaken.portTaken(stateTrackerPort)) {
// sets up a proper connection string for reference wrt external actors needing a reference
if (connectionString.equals("master")) {
String host = InetAddress.getLocalHost().getHostName();
this.connectionString = host + ":" + stateTrackerPort;
}
this.hazelCastPort = stateTrackerPort;
config = hazelcast();
h = Hazelcast.newHazelcastInstance(config);
h.getCluster()
.addMembershipListener(
new MembershipListener() {
@Override
public void memberAdded(MembershipEvent membershipEvent) {
log.info("Member added " + membershipEvent.toString());
}
@Override
public void memberRemoved(MembershipEvent membershipEvent) {
log.info("Member removed " + membershipEvent.toString());
}
@Override
public void memberAttributeChanged(MemberAttributeEvent memberAttributeEvent) {
log.info("Member changed " + memberAttributeEvent.toString());
}
});
} else if (type.equals("master") && PortTaken.portTaken(stateTrackerPort))
throw new IllegalStateException(
"Specified type was master and the port specified was taken, please specify a different port");
else {
setConnectionString(connectionString);
log.info("Connecting to hazelcast on " + connectionString);
ClientConfig client = new ClientConfig();
client.getNetworkConfig().addAddress(connectionString);
h = HazelcastClient.newHazelcastClient(client);
}
this.type = type;
jobs = h.getList(JOBS);
workers = h.getList(WORKERS);
// we can make the assumption workers isn't empty because
// the master node by default comes with a applyTransformToDestination of workers
if (!this.type.equals("master")) {
while (workers.isEmpty()) {
log.warn("Waiting for data sync...");
Thread.sleep(1000);
}
log.info("Workers is " + workers.size());
}
begunTraining = h.getAtomicReference(BEGUN);
miniBatchSize = h.getAtomicReference(INPUT_SPLIT);
workerEnabled = h.getMap(WORKER_ENABLED);
replicate = h.getList(REPLICATE_WEIGHTS);
topics = h.getList(TOPICS);
updates = h.getList(UPDATES);
heartbeat = h.getMap(HEART_BEAT);
master = h.getAtomicReference(RESULT);
isPretrain = h.getAtomicReference(IS_PRETRAIN);
numTimesPretrain = h.getAtomicReference(NUM_TIMES_RUN_PRETRAIN);
numTimesPretrainRan = h.getAtomicReference(NUM_TIMES_PRETRAIN_RAN);
done = h.getAtomicReference(DONE);
validationEpochs = h.getAtomicReference(VALIDATION_EPOCHS);
improvementThreshold = h.getAtomicReference(IMPROVEMENT_THRESHOLD);
bestLoss = h.getAtomicReference(BEST_LOSS);
earlyStop = h.getAtomicReference(EARLY_STOP);
patience = h.getAtomicReference(PATIENCE);
patienceIncrease = h.getAtomicReference(PATIENCE_INCREASE);
numBatches = h.getAtomicReference(NUM_BATCHES_SO_FAR_RAN);
// applyTransformToDestination defaults only when master, otherwise, overrides previous values
if (type.equals("master")) {
begunTraining.set(false);
saver = createUpdateSaver();
numTimesPretrainRan.set(0);
numTimesPretrain.set(1);
isPretrain.set(true);
done.set(false);
resource = new StateTrackerDropWizardResource(this);
bestLoss.set(Double.POSITIVE_INFINITY);
earlyStop.set(true);
patience.set(40.0);
patienceIncrease.set(2.0);
improvementThreshold.set(0.995);
validationEpochs.set((int) Math.min(10, patience() / 2));
numBatches.set(0);
}
}
public long findDelayMillis(ScheduledEntry entry) {
long diffMillis = (System.nanoTime() - entry.getScheduleTimeNanos()) / 1000000;
return Math.max(0, entry.getScheduledDelayMillis() - diffMillis);
}
