@Test
public void addIndex() {
HazelcastClient hClient = getHazelcastClient();
IMap map = hClient.getMap("addIndex");
int size = 1000;
for (int i = 0; i < size; i++) {
map.put(String.valueOf(i), new Employee("name" + i, i, true, 0));
}
EntryObject e = new PredicateBuilder().getEntryObject();
Predicate predicate = e.get("age").equal(23);
long begin = Clock.currentTimeMillis();
Set<Entry<Object, Object>> set = map.entrySet(predicate);
long timeWithoutIndex = Clock.currentTimeMillis() - begin;
assertEquals(1, set.size());
assertEquals(size, map.size());
map.destroy();
map = hClient.getMap("addIndex");
map.addIndex("age", true);
for (int i = 0; i < size; i++) {
map.put(String.valueOf(i), new Employee("name" + i, i, true, 0));
}
begin = Clock.currentTimeMillis();
set = map.entrySet(predicate);
long timeWithIndex = Clock.currentTimeMillis() - begin;
assertEquals(1, set.size());
assertEquals(size, map.size());
assertTrue(timeWithoutIndex > 2 * timeWithIndex);
// map.addIndex("age", true);
}
@Test
public void testMessageFieldSetCorrectly() {
final ReliableMessageListenerMock listener = new ReliableMessageListenerMock();
topic.addMessageListener(listener);
final long beforePublishTime = Clock.currentTimeMillis();
topic.publish("foo");
final long afterPublishTime = Clock.currentTimeMillis();
assertTrueEventually(
new AssertTask() {
@Override
public void run() throws Exception {
assertEquals(1, listener.messages.size());
Message<String> message = listener.messages.get(0);
assertEquals("foo", message.getMessageObject());
Member localMember = local.getCluster().getLocalMember();
assertEquals(localMember, message.getPublishingMember());
long actualPublishTime = message.getPublishTime();
assertTrue(actualPublishTime >= beforePublishTime);
assertTrue(actualPublishTime <= afterPublishTime);
}
});
}
private void doShutdown(boolean force) {
long start = Clock.currentTimeMillis();
logger.finest("** we are being asked to shutdown when active = " + String.valueOf(active));
if (!force && isActive()) {
final int maxWaitSeconds = groupProperties.GRACEFUL_SHUTDOWN_MAX_WAIT.getInteger();
if (!partitionService.prepareToSafeShutdown(maxWaitSeconds, TimeUnit.SECONDS)) {
logger.warning(
"Graceful shutdown could not be completed in " + maxWaitSeconds + " seconds!");
}
}
if (isActive()) {
if (!force) {
clusterService.sendShutdownMessage();
}
// set the joined=false first so that
// threads do not process unnecessary
// events, such as remove address
joined.set(false);
setActive(false);
setMasterAddress(null);
try {
Runtime.getRuntime().removeShutdownHook(shutdownHookThread);
} catch (Throwable ignored) {
}
if (managementCenterService != null) {
managementCenterService.shutdown();
}
logger.finest("Shutting down client command service");
clientEngine.shutdown();
logger.finest("Shutting down node engine");
nodeEngine.shutdown();
if (multicastService != null) {
logger.finest("Shutting down multicast service");
multicastService.stop();
}
logger.finest("Shutting down connection manager");
connectionManager.shutdown();
textCommandService.stop();
masterAddress = null;
if (securityContext != null) {
securityContext.destroy();
}
initializer.destroy();
serializationService.destroy();
int numThreads = threadGroup.activeCount();
Thread[] threads = new Thread[numThreads * 2];
numThreads = threadGroup.enumerate(threads, false);
for (int i = 0; i < numThreads; i++) {
Thread thread = threads[i];
if (thread.isAlive()) {
logger.finest("Shutting down thread " + thread.getName());
thread.interrupt();
}
}
failedConnections.clear();
systemLogService.shutdown();
logger.info(
"Hazelcast Shutdown is completed in " + (Clock.currentTimeMillis() - start) + " ms.");
}
}
private void joinViaTargetMember(Address targetAddress, long maxJoinMillis) {
try {
if (targetAddress == null) {
throw new IllegalArgumentException("Invalid target address -> NULL");
}
if (logger.isFinestEnabled()) {
logger.finest("Joining over target member " + targetAddress);
}
if (targetAddress.equals(node.getThisAddress()) || isLocalAddress(targetAddress)) {
node.setAsMaster();
return;
}
long joinStartTime = Clock.currentTimeMillis();
Connection connection;
while (node.isRunning()
&& !node.joined()
&& (Clock.currentTimeMillis() - joinStartTime < maxJoinMillis)) {
connection = node.connectionManager.getOrConnect(targetAddress);
if (connection == null) {
//noinspection BusyWait
Thread.sleep(JOIN_RETRY_WAIT_TIME);
continue;
}
if (logger.isFinestEnabled()) {
logger.finest("Sending joinRequest " + targetAddress);
}
clusterJoinManager.sendJoinRequest(targetAddress, true);
//noinspection BusyWait
Thread.sleep(JOIN_RETRY_WAIT_TIME);
}
} catch (final Exception e) {
logger.warning(e);
}
}
protected void handleMapPutMany(String[] args) {
int count = 1;
if (args.length > 1) count = Integer.parseInt(args[1]);
int b = 100;
byte[] value = new byte[b];
if (args.length > 2) {
b = Integer.parseInt(args[2]);
value = new byte[b];
}
int start = getMap().size();
if (args.length > 3) {
start = Integer.parseInt(args[3]);
}
Map theMap = new HashMap(count);
for (int i = 0; i < count; i++) {
theMap.put("key" + (start + i), value);
}
long t0 = Clock.currentTimeMillis();
getMap().putAll(theMap);
long t1 = Clock.currentTimeMillis();
if (t1 - t0 > 1) {
println(
"size = "
+ getMap().size()
+ ", "
+ count * 1000 / (t1 - t0)
+ " evt/s, "
+ (count * 1000 / (t1 - t0)) * (b * 8) / 1024
+ " Kbit/s, "
+ count * b / 1024
+ " KB added");
}
}
private boolean prepareClusterState(ClusterServiceImpl clusterService) {
if (!preCheckClusterState(clusterService)) {
return false;
}
long until = Clock.currentTimeMillis() + mergeNextRunDelayMs;
while (clusterService.getClusterState() == ClusterState.ACTIVE) {
try {
clusterService.changeClusterState(ClusterState.FROZEN);
return true;
} catch (Exception e) {
String error = e.getClass().getName() + ": " + e.getMessage();
logger.warning("While freezing cluster state! " + error);
}
if (Clock.currentTimeMillis() >= until) {
logger.warning(
"Could not change cluster state to FROZEN in time. "
+ "Postponing merge process until next attempt.");
return false;
}
try {
TimeUnit.SECONDS.sleep(1);
} catch (InterruptedException e) {
logger.warning("Interrupted while preparing cluster for merge!");
// restore interrupt flag
Thread.currentThread().interrupt();
return false;
}
}
return false;
}
private void tryToJoinPossibleAddresses(Collection<Address> possibleAddresses)
throws InterruptedException {
long connectionTimeoutMillis = TimeUnit.SECONDS.toMillis(getConnTimeoutSeconds());
long start = Clock.currentTimeMillis();
while (!node.joined() && Clock.currentTimeMillis() - start < connectionTimeoutMillis) {
Address masterAddress = node.getMasterAddress();
if (isAllBlacklisted(possibleAddresses) && masterAddress == null) {
return;
}
if (masterAddress != null) {
if (logger.isFinestEnabled()) {
logger.finest("Sending join request to " + masterAddress);
}
clusterJoinManager.sendJoinRequest(masterAddress, true);
} else {
sendMasterQuestion(possibleAddresses);
}
if (!node.joined()) {
Thread.sleep(JOIN_RETRY_WAIT_TIME);
}
}
}
@SuppressWarnings({"checkstyle:npathcomplexity", "checkstyle:cyclomaticcomplexity"})
private void lookForMaster(Collection<Address> possibleAddresses) throws InterruptedException {
int tryCount = 0;
while (node.getMasterAddress() == null && tryCount++ < LOOK_FOR_MASTER_MAX_TRY_COUNT) {
sendMasterQuestion(possibleAddresses);
//noinspection BusyWait
Thread.sleep(JOIN_RETRY_WAIT_TIME);
if (isAllBlacklisted(possibleAddresses)) {
break;
}
}
if (node.joined()) {
return;
}
if (isAllBlacklisted(possibleAddresses) && node.getMasterAddress() == null) {
if (logger.isFinestEnabled()) {
logger.finest("Setting myself as master! No possible addresses remaining to connect...");
}
node.setAsMaster();
return;
}
long maxMasterJoinTime = getMaxJoinTimeToMasterNode();
long start = Clock.currentTimeMillis();
while (node.isRunning()
&& !node.joined()
&& Clock.currentTimeMillis() - start < maxMasterJoinTime) {
Address master = node.getMasterAddress();
if (master != null) {
if (logger.isFinestEnabled()) {
logger.finest("Joining to master " + master);
}
clusterJoinManager.sendJoinRequest(master, true);
} else {
break;
}
//noinspection BusyWait
Thread.sleep(JOIN_RETRY_WAIT_TIME);
}
if (!node.joined()) {
Address master = node.getMasterAddress();
if (master != null) {
logger.warning("Couldn't join to the master : " + master);
} else {
if (logger.isFinestEnabled()) {
logger.finest(
"Couldn't find a master! But there was connections available: " + possibleAddresses);
}
}
}
}
protected void handleMapRemoveMany(String[] args) {
int count = 1;
if (args.length > 1) count = Integer.parseInt(args[1]);
int start = 0;
if (args.length > 2) start = Integer.parseInt(args[2]);
long t0 = Clock.currentTimeMillis();
for (int i = 0; i < count; i++) {
getMap().remove("key" + (start + i));
}
long t1 = Clock.currentTimeMillis();
println("size = " + getMap().size() + ", " + count * 1000 / (t1 - t0) + " evt/s");
}
protected void handleSetRemoveMany(String[] args) {
int count = 1;
if (args.length > 1) count = Integer.parseInt(args[1]);
int successCount = 0;
long t0 = Clock.currentTimeMillis();
for (int i = 0; i < count; i++) {
boolean success = getSet().remove("obj" + i);
if (success) successCount++;
}
long t1 = Clock.currentTimeMillis();
println("Removed " + successCount + " objects.");
println("size = " + getSet().size() + ", " + successCount * 1000 / (t1 - t0) + " evt/s");
}
@SuppressWarnings("checkstyle:npathcomplexity")
private boolean claimMastership(Collection<Address> possibleAddresses) {
if (logger.isFinestEnabled()) {
Set<Address> votingEndpoints = new HashSet<Address>(possibleAddresses);
votingEndpoints.removeAll(blacklistedAddresses.keySet());
logger.finest("Claiming myself as master node! Asking to endpoints: " + votingEndpoints);
}
claimingMaster = true;
Collection<Future<Boolean>> responses = new LinkedList<Future<Boolean>>();
for (Address address : possibleAddresses) {
if (isBlacklisted(address)) {
continue;
}
if (node.getConnectionManager().getConnection(address) != null) {
Future<Boolean> future =
node.nodeEngine
.getOperationService()
.createInvocationBuilder(
ClusterServiceImpl.SERVICE_NAME, new MasterClaimOperation(), address)
.setTryCount(1)
.invoke();
responses.add(future);
}
}
final long maxWait = TimeUnit.SECONDS.toMillis(10);
long waitTime = 0L;
boolean consensus = true;
for (Future<Boolean> response : responses) {
long t = Clock.currentTimeMillis();
try {
consensus = response.get(1, TimeUnit.SECONDS);
} catch (Exception e) {
logger.finest(e);
consensus = false;
} finally {
waitTime += (Clock.currentTimeMillis() - t);
}
if (!consensus) {
break;
}
if (waitTime > maxWait) {
consensus = false;
break;
}
}
return consensus;
}
public void doJoin(AtomicBoolean joined) {
int tryCount = 0;
long joinStartTime = Clock.currentTimeMillis();
long maxJoinMillis = node.getGroupProperties().MAX_JOIN_SECONDS.getInteger() * 1000;
while (node.isActive()
&& !joined.get()
&& (Clock.currentTimeMillis() - joinStartTime < maxJoinMillis)) {
String msg = "Joining to master node: " + node.getMasterAddress();
logger.log(Level.FINEST, msg);
systemLogService.logJoin(msg);
final Address masterAddressNow;
if (targetAddress == null) {
masterAddressNow = findMasterWithMulticast();
} else {
// if target address is set explicitly, try to join target address first.
masterAddressNow = targetAddress;
targetAddress = null;
}
node.setMasterAddress(masterAddressNow);
if (masterAddressNow != null) {
systemLogService.logJoin("Setting master address to " + masterAddressNow);
}
if (node.getMasterAddress() == null || node.address.equals(node.getMasterAddress())) {
TcpIpConfig tcpIpConfig = config.getNetworkConfig().getJoin().getTcpIpConfig();
if (tcpIpConfig != null && tcpIpConfig.isEnabled()) {
doTCP(joined);
} else {
node.setAsMaster();
}
return;
}
if (++tryCount > 49) {
failedJoiningToMaster(true, tryCount);
}
if (!node.getMasterAddress().equals(node.address)) {
connectAndSendJoinRequest(node.getMasterAddress());
} else {
node.setMasterAddress(null);
tryCount = 0;
}
try {
//noinspection BusyWait
Thread.sleep(500L);
} catch (InterruptedException ignored) {
}
}
}
@Override
protected void writeInternal(ObjectDataOutput out) throws IOException {
int confSize = configs.size();
out.writeInt(confSize);
for (CacheConfig config : configs) {
out.writeObject(config);
}
int count = data.size();
out.writeInt(count);
long now = Clock.currentTimeMillis();
for (Map.Entry<String, Map<Data, CacheRecord>> entry : data.entrySet()) {
Map<Data, CacheRecord> cacheMap = entry.getValue();
int subCount = cacheMap.size();
out.writeInt(subCount);
out.writeUTF(entry.getKey());
for (Map.Entry<Data, CacheRecord> e : cacheMap.entrySet()) {
final Data key = e.getKey();
final CacheRecord record = e.getValue();
final long expirationTime = record.getExpirationTime();
// If entry is already expired we skip it
if (expirationTime > now) {
out.writeData(key);
out.writeObject(record);
}
}
// Empty data will terminate the iteration for read in case
// expired entries were found while serializing, since the
// real subCount will then be different from the one written
// before
out.writeData(new DefaultData());
}
}
private void putAndGet(Map<String, String> map, int counter) {
long beginTime = Clock.currentTimeMillis();
for (int i = 1; i <= counter; i++) {
if (i % (counter / 10) == 0) {
System.out.println(i + ": " + (Clock.currentTimeMillis() - beginTime) + " ms");
}
map.put("key_" + i, String.valueOf(i));
}
beginTime = Clock.currentTimeMillis();
for (int i = 1; i <= counter; i++) {
if (i % (counter / 10) == 0) {
System.out.println(i + ": " + (Clock.currentTimeMillis() - beginTime) + " ms");
}
assertEquals(String.valueOf(i), map.get("key_" + i));
}
}
public StoreWorker(MapContainer mapContainer, WriteBehindProcessor writeBehindProcessor) {
this.mapName = mapContainer.getName();
this.mapServiceContext = mapContainer.getMapServiceContext();
this.writeBehindProcessor = writeBehindProcessor;
this.backupRunIntervalTime = getReplicaWaitTime();
this.lastRunTime = Clock.currentTimeMillis();
}
public void setAsMaster() {
logger.finest("This node is being set as the master");
masterAddress = address;
setJoined();
this.getClusterService().getClusterClock().setClusterStartTime(Clock.currentTimeMillis());
this.getClusterService().setClusterId(UuidUtil.createClusterUuid());
}
public void process() {
final Address endPoint = conn.getEndPoint();
if (endPoint == null) {
return;
}
final ILogger logger = node.getLogger(MasterConfirmation.class.getName());
final ClusterManager clusterManager = node.clusterManager;
final MemberImpl member = clusterManager.getMember(endPoint);
if (member != null) {
if (getNode().isMaster()) {
if (logger.isLoggable(Level.FINEST)) {
logger.log(Level.FINEST, "MasterConfirmation has been received from " + member);
}
clusterManager.memberMasterConfirmationTimes.put(member, Clock.currentTimeMillis());
} else {
logger.log(
Level.WARNING,
endPoint + " has sent MasterConfirmation, but this node is not master!");
}
} else {
logger.log(
Level.WARNING,
"MasterConfirmation has been received from "
+ endPoint
+ ", but it is not a member of this cluster!");
clusterManager.sendProcessableTo(new MemberRemover(clusterManager.thisAddress), conn);
}
}
public int getAdjustedTTLSeconds(int ttl) {
if (ttl <= MONTH_SECONDS) {
return ttl;
} else {
return ttl - (int) (Clock.currentTimeMillis() / 1000);
}
}
public void addPersistenceFailure(Throwable t) {
PERSISTENCE_COUNT.incrementAndGet(this);
lastPersistenceTime = Clock.currentTimeMillis();
lastPersistenceDuration = 0;
lastPersistenceWrittenBytes = 0;
lastPersistenceKeyCount = 0;
lastPersistenceFailure = t.getClass().getSimpleName() + ": " + t.getMessage();
}
public void addPersistence(long duration, int writtenBytes, int keyCount) {
PERSISTENCE_COUNT.incrementAndGet(this);
lastPersistenceTime = Clock.currentTimeMillis();
lastPersistenceDuration = duration;
lastPersistenceWrittenBytes = writtenBytes;
lastPersistenceKeyCount = keyCount;
lastPersistenceFailure = "";
}
private Connection connectToOne(final Collection<InetSocketAddress> socketAddresses)
throws Exception {
final int connectionAttemptLimit = getClientConfig().getConnectionAttemptLimit();
final ManagerAuthenticator authenticator = new ManagerAuthenticator();
int attempt = 0;
Throwable lastError = null;
while (true) {
final long nextTry =
Clock.currentTimeMillis() + getClientConfig().getConnectionAttemptPeriod();
for (InetSocketAddress isa : socketAddresses) {
Address address = new Address(isa);
try {
final Connection connection =
getConnectionManager().firstConnection(address, authenticator);
active = true;
fireConnectionEvent(false);
return connection;
} catch (IOException e) {
active = false;
lastError = e;
logger.finest("IO error during initial connection...", e);
} catch (AuthenticationException e) {
active = false;
lastError = e;
logger.warning("Authentication error on " + address, e);
}
}
if (attempt++ >= connectionAttemptLimit) {
break;
}
final long remainingTime = nextTry - Clock.currentTimeMillis();
logger.warning(
String.format(
"Unable to get alive cluster connection," + " try in %d ms later, attempt %d of %d.",
Math.max(0, remainingTime), attempt, connectionAttemptLimit));
if (remainingTime > 0) {
try {
Thread.sleep(remainingTime);
} catch (InterruptedException e) {
break;
}
}
}
throw new IllegalStateException("Unable to connect to any address in the config!", lastError);
}
public void scheduleTtlEviction(String mapName, Record record, long delay) {
if (record.getStatistics() != null) {
record.getStatistics().setExpirationTime(Clock.currentTimeMillis() + delay);
}
getMapContainer(mapName)
.getTtlEvictionScheduler()
.schedule(delay, toData(record.getKey()), null);
}
public void afterRun() throws Exception {
long elapsed = Math.max(0, Clock.currentTimeMillis() - begin);
if (coll != null) {
getOrCreateContainer().update();
for (CollectionRecord record : coll) {
publishEvent(EntryEventType.REMOVED, dataKey, record.getObject());
}
}
}
@Test
public void test_lockClusterState_getLockExpiryTime() throws Exception {
final Address initiator = newAddress();
clusterStateManager.lockClusterState(FROZEN, initiator, TXN, TimeUnit.DAYS.toMillis(1), 0);
final ClusterStateLock stateLock = clusterStateManager.getStateLock();
assertTrue(
Clock.currentTimeMillis() + TimeUnit.HOURS.toMillis(12) < stateLock.getLockExpiryTime());
}
@Override
public void afterRun() throws Exception {
long elapsed = Math.max(0, Clock.currentTimeMillis() - begin);
final MultiMapService service = getService();
service.getLocalMultiMapStatsImpl(name).incrementPuts(elapsed);
if (Boolean.TRUE.equals(response)) {
publishEvent(EntryEventType.ADDED, dataKey, value, null);
}
}
private boolean tryInitialConnection(Collection<Address> possibleAddresses)
throws InterruptedException {
long connectionTimeoutMillis = TimeUnit.SECONDS.toMillis(getConnTimeoutSeconds());
long start = Clock.currentTimeMillis();
while (Clock.currentTimeMillis() - start < connectionTimeoutMillis) {
if (isAllBlacklisted(possibleAddresses)) {
return false;
}
if (logger.isFinestEnabled()) {
logger.finest("Will send master question to each address in: " + possibleAddresses);
}
if (sendMasterQuestion(possibleAddresses)) {
return true;
}
Thread.sleep(JOIN_RETRY_WAIT_TIME);
}
return false;
}
@Override
public void afterRun() throws Exception {
long elapsed = Math.max(0, Clock.currentTimeMillis() - begin);
final MultiMapService service = getService();
service.getLocalMultiMapStatsImpl(name).incrementRemoves(elapsed);
if (Boolean.TRUE.equals(response)) {
getOrCreateContainer().update();
publishEvent(EntryEventType.REMOVED, dataKey, null, value);
}
}
@Override
public void run0() {
heartbeatPacketsReceived.inc();
long timeMillis = Clock.currentTimeMillis();
updateMemberHeartbeat(timeMillis);
for (long callId : (long[]) serializationService.toObject(callIds)) {
updateHeartbeat(callId, timeMillis);
}
}
public WanSyncStateImpl(
WanSyncStatus status,
int syncedPartitionCount,
String activeWanConfigName,
String activePublisherName) {
creationTime = Clock.currentTimeMillis();
this.status = status;
this.syncedPartitionCount = syncedPartitionCount;
this.activeWanConfigName = activeWanConfigName;
this.activePublisherName = activePublisherName;
}
public void join() {
final long joinStartTime = joiner != null ? joiner.getStartTime() : Clock.currentTimeMillis();
final long maxJoinMillis = getGroupProperties().MAX_JOIN_SECONDS.getInteger() * 1000;
try {
if (joiner == null) {
logger.warning("No join method is enabled! Starting standalone.");
setAsMaster();
} else {
joiner.join(joined);
}
} catch (Exception e) {
if (Clock.currentTimeMillis() - joinStartTime < maxJoinMillis) {
logger.warning("Trying to rejoin: " + e.getMessage());
rejoin();
} else {
logger.severe("Could not join cluster, shutting down!", e);
shutdown(false, true);
}
}
}
