private int makeBackups(
BackupAwareOperation backupAwareOp,
int partitionId,
long[] replicaVersions,
int syncBackups,
int asyncBackups) {
int sendSyncBackups = 0;
InternalPartitionService partitionService = node.getPartitionService();
InternalPartition partition = partitionService.getPartition(partitionId);
Data backupOpData = getBackupOperationData(backupAwareOp);
for (int replicaIndex = 1; replicaIndex <= syncBackups + asyncBackups; replicaIndex++) {
Address target = partition.getReplicaAddress(replicaIndex);
if (target == null) {
continue;
}
assertNoBackupOnPrimaryMember(partition, target);
boolean isSyncBackup = replicaIndex <= syncBackups;
Backup backup =
newBackup(backupAwareOp, backupOpData, replicaVersions, replicaIndex, isSyncBackup);
operationService.send(backup, target);
if (isSyncBackup) {
sendSyncBackups++;
}
}
return sendSyncBackups;
}
public LocalMapStats createStats(String name) {
LocalMultiMapStatsImpl stats = getLocalMultiMapStatsImpl(name);
long ownedEntryCount = 0;
long backupEntryCount = 0;
long hits = 0;
long lockedEntryCount = 0;
ClusterServiceImpl clusterService = (ClusterServiceImpl) nodeEngine.getClusterService();
Address thisAddress = clusterService.getThisAddress();
for (int i = 0; i < nodeEngine.getPartitionService().getPartitionCount(); i++) {
InternalPartition partition = nodeEngine.getPartitionService().getPartition(i);
MultiMapPartitionContainer partitionContainer = getPartitionContainer(i);
MultiMapContainer multiMapContainer = partitionContainer.getCollectionContainer(name);
if (multiMapContainer == null) {
continue;
}
Address owner = partition.getOwnerOrNull();
if (owner != null) {
if (owner.equals(thisAddress)) {
lockedEntryCount += multiMapContainer.getLockedCount();
for (MultiMapWrapper wrapper : multiMapContainer.multiMapWrappers.values()) {
hits += wrapper.getHits();
ownedEntryCount += wrapper.getCollection(false).size();
}
} else {
int backupCount = multiMapContainer.config.getTotalBackupCount();
for (int j = 1; j <= backupCount; j++) {
Address replicaAddress = partition.getReplicaAddress(j);
int memberSize = nodeEngine.getClusterService().getMembers().size();
int tryCount = REPLICA_ADDRESS_TRY_COUNT;
// wait if the partition table is not updated yet
while (memberSize > backupCount && replicaAddress == null && tryCount-- > 0) {
try {
Thread.sleep(REPLICA_ADDRESS_SLEEP_WAIT_MILLIS);
} catch (InterruptedException e) {
throw ExceptionUtil.rethrow(e);
}
replicaAddress = partition.getReplicaAddress(j);
}
if (replicaAddress != null && replicaAddress.equals(thisAddress)) {
for (MultiMapWrapper wrapper : multiMapContainer.multiMapWrappers.values()) {
backupEntryCount += wrapper.getCollection(false).size();
}
}
}
}
}
}
stats.setOwnedEntryCount(ownedEntryCount);
stats.setBackupEntryCount(backupEntryCount);
stats.setHits(hits);
stats.setLockedEntryCount(lockedEntryCount);
return stats;
}
@Override
public void run() {
final long now = Clock.currentTimeMillis();
final String mapName = this.mapName;
final MapServiceContext mapServiceContext = this.mapServiceContext;
final NodeEngine nodeEngine = mapServiceContext.getNodeEngine();
final ClusterService clusterService = nodeEngine.getClusterService();
final InternalPartitionService partitionService = nodeEngine.getPartitionService();
final Address thisAddress = clusterService.getThisAddress();
final int partitionCount = partitionService.getPartitionCount();
Map<Integer, Integer> partitionToEntryCountHolder = Collections.emptyMap();
List<DelayedEntry> entries = Collections.emptyList();
boolean createLazy = true;
for (int partitionId = 0; partitionId < partitionCount; partitionId++) {
final InternalPartition partition = partitionService.getPartition(partitionId, false);
final Address owner = partition.getOwnerOrNull();
final RecordStore recordStore = getRecordStoreOrNull(mapName, partitionId);
if (owner == null || recordStore == null) {
// no-op because no owner is set yet.
// Therefore we don't know anything about the map
continue;
}
final WriteBehindQueue<DelayedEntry> queue = getWriteBehindQueue(recordStore);
final List<DelayedEntry> delayedEntries = filterItemsLessThanOrEqualToTime(queue, now);
if (delayedEntries.isEmpty()) {
continue;
}
if (!owner.equals(thisAddress)) {
if (now > lastRunTime + backupRunIntervalTime) {
doInBackup(queue, delayedEntries, partitionId);
}
continue;
}
// initialize when needed, we do not want
// to create these on backups for every second.
if (createLazy) {
partitionToEntryCountHolder = new HashMap<Integer, Integer>();
entries = new ArrayList<DelayedEntry>();
createLazy = false;
}
partitionToEntryCountHolder.put(partitionId, delayedEntries.size());
entries.addAll(delayedEntries);
}
if (!entries.isEmpty()) {
final Map<Integer, List<DelayedEntry>> failsPerPartition =
writeBehindProcessor.process(entries);
removeProcessed(mapName, getEntryPerPartitionMap(entries));
addFailsToQueue(mapName, failsPerPartition);
lastRunTime = now;
}
}
@Override
public void run() throws Exception {
InternalPartitionServiceImpl partitionService = getService();
int partitionId = getPartitionId();
int replicaIndex = syncReplicaIndex;
InternalPartition partition = partitionService.getPartition(partitionId);
Address target = partition.getReplicaAddress(replicaIndex);
if (target == null) {
notifyCallback(false);
return;
}
invokeCheckReplicaVersion(partitionId, replicaIndex, target);
}
@Override
public void beforeRun() throws Exception {
final NodeEngine nodeEngine = getNodeEngine();
final int partitionId = getPartitionId();
final InternalPartition partition = nodeEngine.getPartitionService().getPartition(partitionId);
final Address owner = partition.getReplicaAddress(getReplicaIndex());
if (!nodeEngine.getThisAddress().equals(owner)) {
valid = false;
final ILogger logger = getLogger();
if (logger.isFinestEnabled()) {
logger.finest(
"Wrong target! " + toString() + " cannot be processed! Target should be: " + owner);
}
}
}
/**
* @param queue write behind queue.
* @param delayedEntries entries to be processed.
* @param partitionId corresponding partition id.
*/
private void doInBackup(
final WriteBehindQueue queue,
final List<DelayedEntry> delayedEntries,
final int partitionId) {
final NodeEngine nodeEngine = mapServiceContext.getNodeEngine();
final ClusterService clusterService = nodeEngine.getClusterService();
final InternalPartitionService partitionService = nodeEngine.getPartitionService();
final Address thisAddress = clusterService.getThisAddress();
final InternalPartition partition = partitionService.getPartition(partitionId, false);
final Address owner = partition.getOwnerOrNull();
if (owner != null && !owner.equals(thisAddress)) {
writeBehindProcessor.callBeforeStoreListeners(delayedEntries);
removeProcessed(mapName, getEntryPerPartitionMap(delayedEntries));
writeBehindProcessor.callAfterStoreListeners(delayedEntries);
}
}
public Set<Data> localKeySet(String name) {
Set<Data> keySet = new HashSet<Data>();
for (int i = 0; i < nodeEngine.getPartitionService().getPartitionCount(); i++) {
InternalPartition partition = nodeEngine.getPartitionService().getPartition(i);
MultiMapPartitionContainer partitionContainer = getPartitionContainer(i);
MultiMapContainer multiMapContainer = partitionContainer.getCollectionContainer(name);
if (multiMapContainer == null) {
continue;
}
if (partition.isLocal()) {
keySet.addAll(multiMapContainer.keySet());
}
}
getLocalMultiMapStatsImpl(name).incrementOtherOperations();
return keySet;
}
public Set<Data> localKeySet(String name) {
Set<Data> keySet = new HashSet<Data>();
ClusterServiceImpl clusterService = (ClusterServiceImpl) nodeEngine.getClusterService();
Address thisAddress = clusterService.getThisAddress();
for (int i = 0; i < nodeEngine.getPartitionService().getPartitionCount(); i++) {
InternalPartition partition = nodeEngine.getPartitionService().getPartition(i);
MultiMapPartitionContainer partitionContainer = getPartitionContainer(i);
MultiMapContainer multiMapContainer = partitionContainer.getCollectionContainer(name);
if (multiMapContainer == null) {
continue;
}
if (thisAddress.equals(partition.getOwnerOrNull())) {
keySet.addAll(multiMapContainer.keySet());
}
}
getLocalMultiMapStatsImpl(name).incrementOtherOperations();
return keySet;
}
/**
* Gets replica address. Waits if necessary.
*
* @see #waitForReplicaAddress
*/
protected Address getReplicaAddress(
int replica,
InternalPartition partition,
InternalPartitionService partitionService,
int backupCount) {
Address replicaAddress = partition.getReplicaAddress(replica);
if (replicaAddress == null) {
replicaAddress = waitForReplicaAddress(replica, partition, partitionService, backupCount);
}
return replicaAddress;
}
protected Object call() {
InternalPartitionService service = getService(InternalPartitionService.SERVICE_NAME);
service.firstArrangement();
Map<Address, List<Integer>> partitionsMap = new HashMap<Address, List<Integer>>();
for (InternalPartition partition : service.getPartitions()) {
Address owner = partition.getOwnerOrNull();
if (owner == null) {
partitionsMap.clear();
return ClientGetPartitionsCodec.encodeResponse(partitionsMap.entrySet());
}
List<Integer> indexes = partitionsMap.get(owner);
if (indexes == null) {
indexes = new LinkedList<Integer>();
partitionsMap.put(owner, indexes);
}
indexes.add(partition.getPartitionId());
}
return ClientGetPartitionsCodec.encodeResponse(partitionsMap.entrySet());
}
public LocalMapStatsImpl createLocalMapStats(String mapName) {
MapContainer mapContainer = mapServiceContext.getMapContainer(mapName);
LocalMapStatsImpl stats = getLocalMapStatsImpl(mapName);
if (!mapContainer.getMapConfig().isStatisticsEnabled()) {
return stats;
}
int backupCount = mapContainer.getTotalBackupCount();
Address thisAddress = clusterService.getThisAddress();
LocalMapOnDemandCalculatedStats onDemandStats = new LocalMapOnDemandCalculatedStats();
onDemandStats.setBackupCount(backupCount);
addNearCacheStats(stats, onDemandStats, mapContainer);
for (int partitionId = 0; partitionId < partitionService.getPartitionCount(); partitionId++) {
InternalPartition partition = partitionService.getPartition(partitionId);
Address owner = partition.getOwnerOrNull();
if (owner == null) {
// no-op because no owner is set yet. Therefore we don't know anything about the map
continue;
}
if (owner.equals(thisAddress)) {
addOwnerPartitionStats(stats, onDemandStats, mapName, partitionId);
} else {
addReplicaPartitionStats(
onDemandStats,
mapName,
partitionId,
partition,
partitionService,
backupCount,
thisAddress);
}
}
onDemandStats.copyValuesTo(stats);
return stats;
}
public void run() {
final long now = Clock.currentTimeMillis();
final MapService mapService = ExpirationManager.this.mapService;
final NodeEngine nodeEngine = mapService.getNodeEngine();
final int partitionCount = nodeEngine.getPartitionService().getPartitionCount();
List<PartitionContainer> partitionContainers = Collections.emptyList();
boolean createLazy = true;
int currentlyRunningCleanupOperationsCount = 0;
for (int partitionId = 0; partitionId < partitionCount; partitionId++) {
InternalPartition partition = nodeEngine.getPartitionService().getPartition(partitionId);
if (partition.isOwnerOrBackup(nodeEngine.getThisAddress())) {
final PartitionContainer partitionContainer =
mapService.getPartitionContainer(partitionId);
if (isContainerEmpty(partitionContainer)) {
continue;
}
if (hasRunningCleanup(partitionContainer)) {
currentlyRunningCleanupOperationsCount++;
continue;
}
if (currentlyRunningCleanupOperationsCount > getMaxCleanupOperationCountInOneRound()
|| notInProcessableTimeWindow(partitionContainer, now)
|| notAnyExpirableRecord(partitionContainer)) {
continue;
}
if (createLazy) {
partitionContainers = new ArrayList<PartitionContainer>();
createLazy = false;
}
partitionContainers.add(partitionContainer);
}
}
if (partitionContainers.isEmpty()) {
return;
}
Collections.sort(partitionContainers, partitionContainerComparator);
sendCleanupOperations(partitionContainers);
}
/** Waits partition table update to get replica address if current replica address is null. */
protected Address waitForReplicaAddress(
int replica,
InternalPartition partition,
InternalPartitionService partitionService,
int backupCount) {
int tryCount = RETRY_COUNT;
Address replicaAddress = null;
while (replicaAddress == null
&& partitionService.getMemberGroupsSize() > backupCount
&& tryCount-- > 0) {
sleep();
replicaAddress = partition.getReplicaAddress(replica);
}
return replicaAddress;
}
public LocalMapStatsImpl createLocalMapStats(String mapName) {
MapContainer mapContainer = getMapContainer(mapName);
LocalMapStatsImpl localMapStats = getLocalMapStatsImpl(mapName);
if (!mapContainer.getMapConfig().isStatisticsEnabled()) {
return localMapStats;
}
long ownedEntryCount = 0;
long backupEntryCount = 0;
long dirtyCount = 0;
long ownedEntryMemoryCost = 0;
long backupEntryMemoryCost = 0;
long hits = 0;
long lockedEntryCount = 0;
long heapCost = 0;
int backupCount = mapContainer.getTotalBackupCount();
ClusterService clusterService = nodeEngine.getClusterService();
final InternalPartitionService partitionService = nodeEngine.getPartitionService();
Address thisAddress = clusterService.getThisAddress();
for (int partitionId = 0; partitionId < partitionService.getPartitionCount(); partitionId++) {
InternalPartition partition = partitionService.getPartition(partitionId);
Address owner = partition.getOwner();
if (owner == null) {
// no-op because no owner is set yet. Therefor we don't know anything about the map
continue;
}
if (owner.equals(thisAddress)) {
PartitionContainer partitionContainer = getPartitionContainer(partitionId);
RecordStore recordStore = partitionContainer.getExistingRecordStore(mapName);
// we don't want to force loading the record store because we are loading statistics. So
// that is why
// we ask for 'getExistingRecordStore' instead of 'getRecordStore' which does the load.
if (recordStore != null) {
heapCost += recordStore.getHeapCost();
Map<Data, Record> records = recordStore.getReadonlyRecordMap();
for (Record record : records.values()) {
RecordStatistics stats = record.getStatistics();
// there is map store and the record is dirty (waits to be stored)
ownedEntryCount++;
ownedEntryMemoryCost += record.getCost();
localMapStats.setLastAccessTime(stats.getLastAccessTime());
localMapStats.setLastUpdateTime(stats.getLastUpdateTime());
hits += stats.getHits();
if (recordStore.isLocked(record.getKey())) {
lockedEntryCount++;
}
}
}
} else {
for (int replica = 1; replica <= backupCount; replica++) {
Address replicaAddress = partition.getReplicaAddress(replica);
int tryCount = 30;
// wait if the partition table is not updated yet
while (replicaAddress == null
&& clusterService.getSize() > backupCount
&& tryCount-- > 0) {
try {
Thread.sleep(100);
} catch (InterruptedException e) {
throw ExceptionUtil.rethrow(e);
}
replicaAddress = partition.getReplicaAddress(replica);
}
if (replicaAddress != null && replicaAddress.equals(thisAddress)) {
PartitionContainer partitionContainer = getPartitionContainer(partitionId);
RecordStore recordStore = partitionContainer.getRecordStore(mapName);
heapCost += recordStore.getHeapCost();
Map<Data, Record> records = recordStore.getReadonlyRecordMap();
for (Record record : records.values()) {
backupEntryCount++;
backupEntryMemoryCost += record.getCost();
}
} else if (replicaAddress == null && clusterService.getSize() > backupCount) {
logger.warning("Partition: " + partition + ", replica: " + replica + " has no owner!");
}
}
}
}
if (mapContainer.getMapStoreScheduler() != null) {
dirtyCount = mapContainer.getMapStoreScheduler().size();
}
localMapStats.setBackupCount(backupCount);
localMapStats.setDirtyEntryCount(zeroOrPositive(dirtyCount));
localMapStats.setLockedEntryCount(zeroOrPositive(lockedEntryCount));
localMapStats.setHits(zeroOrPositive(hits));
localMapStats.setOwnedEntryCount(zeroOrPositive(ownedEntryCount));
localMapStats.setBackupEntryCount(zeroOrPositive(backupEntryCount));
localMapStats.setOwnedEntryMemoryCost(zeroOrPositive(ownedEntryMemoryCost));
localMapStats.setBackupEntryMemoryCost(zeroOrPositive(backupEntryMemoryCost));
// add near cache heap cost.
heapCost += mapContainer.getNearCacheSizeEstimator().getSize();
localMapStats.setHeapCost(heapCost);
if (mapContainer.getMapConfig().isNearCacheEnabled()) {
NearCacheStatsImpl nearCacheStats = getNearCache(mapName).getNearCacheStats();
localMapStats.setNearCacheStats(nearCacheStats);
}
return localMapStats;
}