public Runnable prepareMergeRunnable() {
Map<MapContainer, Collection<Record>> recordMap =
new HashMap<MapContainer, Collection<Record>>(mapContainers.size());
InternalPartitionService partitionService = nodeEngine.getPartitionService();
int partitionCount = partitionService.getPartitionCount();
Address thisAddress = nodeEngine.getClusterService().getThisAddress();
for (MapContainer mapContainer : mapContainers.values()) {
for (int i = 0; i < partitionCount; i++) {
RecordStore recordStore = getPartitionContainer(i).getRecordStore(mapContainer.getName());
// add your owned entries to the map so they will be merged
if (thisAddress.equals(partitionService.getPartitionOwner(i))) {
Collection<Record> records = recordMap.get(mapContainer);
if (records == null) {
records = new ArrayList<Record>();
recordMap.put(mapContainer, records);
}
records.addAll(recordStore.getReadonlyRecordMap().values());
}
// clear all records either owned or backup
recordStore.reset();
}
}
return new Merger(recordMap);
}
private List<Object>[] buildMapping(InternalPartitionService partitionService) {
List<Object>[] mapping = new List[partitionService.getPartitionCount()];
for (Object key : keys) {
int pid = partitionService.getPartitionId(key);
List<Object> list = mapping[pid];
if (list == null) {
list = new ArrayList<Object>();
mapping[pid] = list;
}
list.add(key);
}
return mapping;
}
@Override
public void run() {
final long now = Clock.currentTimeMillis();
final String mapName = this.mapName;
final MapServiceContext mapServiceContext = this.mapServiceContext;
final NodeEngine nodeEngine = mapServiceContext.getNodeEngine();
final ClusterService clusterService = nodeEngine.getClusterService();
final InternalPartitionService partitionService = nodeEngine.getPartitionService();
final Address thisAddress = clusterService.getThisAddress();
final int partitionCount = partitionService.getPartitionCount();
Map<Integer, Integer> partitionToEntryCountHolder = Collections.emptyMap();
List<DelayedEntry> entries = Collections.emptyList();
boolean createLazy = true;
for (int partitionId = 0; partitionId < partitionCount; partitionId++) {
final InternalPartition partition = partitionService.getPartition(partitionId, false);
final Address owner = partition.getOwnerOrNull();
final RecordStore recordStore = getRecordStoreOrNull(mapName, partitionId);
if (owner == null || recordStore == null) {
// no-op because no owner is set yet.
// Therefore we don't know anything about the map
continue;
}
final WriteBehindQueue<DelayedEntry> queue = getWriteBehindQueue(recordStore);
final List<DelayedEntry> delayedEntries = filterItemsLessThanOrEqualToTime(queue, now);
if (delayedEntries.isEmpty()) {
continue;
}
if (!owner.equals(thisAddress)) {
if (now > lastRunTime + backupRunIntervalTime) {
doInBackup(queue, delayedEntries, partitionId);
}
continue;
}
// initialize when needed, we do not want
// to create these on backups for every second.
if (createLazy) {
partitionToEntryCountHolder = new HashMap<Integer, Integer>();
entries = new ArrayList<DelayedEntry>();
createLazy = false;
}
partitionToEntryCountHolder.put(partitionId, delayedEntries.size());
entries.addAll(delayedEntries);
}
if (!entries.isEmpty()) {
final Map<Integer, List<DelayedEntry>> failsPerPartition =
writeBehindProcessor.process(entries);
removeProcessed(mapName, getEntryPerPartitionMap(entries));
addFailsToQueue(mapName, failsPerPartition);
lastRunTime = now;
}
}
@Override
public Collection<Integer> getPartitions() {
InternalPartitionService partitionService = getClientEngine().getPartitionService();
int partitions = partitionService.getPartitionCount();
int capacity = Math.min(partitions, keys.size()); // todo: is there better way to estimate size?
Set<Integer> partitionIds = new HashSet<Integer>(capacity);
Iterator<Data> iterator = keys.iterator();
while (iterator.hasNext() && partitionIds.size() < partitions) {
Data key = iterator.next();
partitionIds.add(partitionService.getPartitionId(key));
}
return partitionIds;
}
public LocalMapStatsImpl createLocalMapStats(String mapName) {
MapContainer mapContainer = mapServiceContext.getMapContainer(mapName);
LocalMapStatsImpl stats = getLocalMapStatsImpl(mapName);
if (!mapContainer.getMapConfig().isStatisticsEnabled()) {
return stats;
}
int backupCount = mapContainer.getTotalBackupCount();
Address thisAddress = clusterService.getThisAddress();
LocalMapOnDemandCalculatedStats onDemandStats = new LocalMapOnDemandCalculatedStats();
onDemandStats.setBackupCount(backupCount);
addNearCacheStats(stats, onDemandStats, mapContainer);
for (int partitionId = 0; partitionId < partitionService.getPartitionCount(); partitionId++) {
InternalPartition partition = partitionService.getPartition(partitionId);
Address owner = partition.getOwnerOrNull();
if (owner == null) {
// no-op because no owner is set yet. Therefore we don't know anything about the map
continue;
}
if (owner.equals(thisAddress)) {
addOwnerPartitionStats(stats, onDemandStats, mapName, partitionId);
} else {
addReplicaPartitionStats(
onDemandStats,
mapName,
partitionId,
partition,
partitionService,
backupCount,
thisAddress);
}
}
onDemandStats.copyValuesTo(stats);
return stats;
}
private static boolean isEvictablePerPartition(final MapContainer mapContainer) {
final MapService mapService = mapContainer.getMapService();
final MaxSizeConfig maxSizeConfig = mapContainer.getMapConfig().getMaxSizeConfig();
final int maxSize = getApproximateMaxSize(maxSizeConfig.getSize());
final String mapName = mapContainer.getName();
final NodeEngine nodeEngine = mapService.getNodeEngine();
final InternalPartitionService partitionService = nodeEngine.getPartitionService();
for (int i = 0; i < partitionService.getPartitionCount(); i++) {
final Address owner = partitionService.getPartitionOwner(i);
if (nodeEngine.getThisAddress().equals(owner)) {
final PartitionContainer container = mapService.getPartitionContainer(i);
if (container == null) {
return false;
}
final int size = container.getRecordStore(mapName).size();
if (size >= maxSize) {
return true;
}
}
}
return false;
}
public LocalMapStatsImpl createLocalMapStats(String mapName) {
MapContainer mapContainer = getMapContainer(mapName);
LocalMapStatsImpl localMapStats = getLocalMapStatsImpl(mapName);
if (!mapContainer.getMapConfig().isStatisticsEnabled()) {
return localMapStats;
}
long ownedEntryCount = 0;
long backupEntryCount = 0;
long dirtyCount = 0;
long ownedEntryMemoryCost = 0;
long backupEntryMemoryCost = 0;
long hits = 0;
long lockedEntryCount = 0;
long heapCost = 0;
int backupCount = mapContainer.getTotalBackupCount();
ClusterService clusterService = nodeEngine.getClusterService();
final InternalPartitionService partitionService = nodeEngine.getPartitionService();
Address thisAddress = clusterService.getThisAddress();
for (int partitionId = 0; partitionId < partitionService.getPartitionCount(); partitionId++) {
InternalPartition partition = partitionService.getPartition(partitionId);
Address owner = partition.getOwner();
if (owner == null) {
// no-op because no owner is set yet. Therefor we don't know anything about the map
continue;
}
if (owner.equals(thisAddress)) {
PartitionContainer partitionContainer = getPartitionContainer(partitionId);
RecordStore recordStore = partitionContainer.getExistingRecordStore(mapName);
// we don't want to force loading the record store because we are loading statistics. So
// that is why
// we ask for 'getExistingRecordStore' instead of 'getRecordStore' which does the load.
if (recordStore != null) {
heapCost += recordStore.getHeapCost();
Map<Data, Record> records = recordStore.getReadonlyRecordMap();
for (Record record : records.values()) {
RecordStatistics stats = record.getStatistics();
// there is map store and the record is dirty (waits to be stored)
ownedEntryCount++;
ownedEntryMemoryCost += record.getCost();
localMapStats.setLastAccessTime(stats.getLastAccessTime());
localMapStats.setLastUpdateTime(stats.getLastUpdateTime());
hits += stats.getHits();
if (recordStore.isLocked(record.getKey())) {
lockedEntryCount++;
}
}
}
} else {
for (int replica = 1; replica <= backupCount; replica++) {
Address replicaAddress = partition.getReplicaAddress(replica);
int tryCount = 30;
// wait if the partition table is not updated yet
while (replicaAddress == null
&& clusterService.getSize() > backupCount
&& tryCount-- > 0) {
try {
Thread.sleep(100);
} catch (InterruptedException e) {
throw ExceptionUtil.rethrow(e);
}
replicaAddress = partition.getReplicaAddress(replica);
}
if (replicaAddress != null && replicaAddress.equals(thisAddress)) {
PartitionContainer partitionContainer = getPartitionContainer(partitionId);
RecordStore recordStore = partitionContainer.getRecordStore(mapName);
heapCost += recordStore.getHeapCost();
Map<Data, Record> records = recordStore.getReadonlyRecordMap();
for (Record record : records.values()) {
backupEntryCount++;
backupEntryMemoryCost += record.getCost();
}
} else if (replicaAddress == null && clusterService.getSize() > backupCount) {
logger.warning("Partition: " + partition + ", replica: " + replica + " has no owner!");
}
}
}
}
if (mapContainer.getMapStoreScheduler() != null) {
dirtyCount = mapContainer.getMapStoreScheduler().size();
}
localMapStats.setBackupCount(backupCount);
localMapStats.setDirtyEntryCount(zeroOrPositive(dirtyCount));
localMapStats.setLockedEntryCount(zeroOrPositive(lockedEntryCount));
localMapStats.setHits(zeroOrPositive(hits));
localMapStats.setOwnedEntryCount(zeroOrPositive(ownedEntryCount));
localMapStats.setBackupEntryCount(zeroOrPositive(backupEntryCount));
localMapStats.setOwnedEntryMemoryCost(zeroOrPositive(ownedEntryMemoryCost));
localMapStats.setBackupEntryMemoryCost(zeroOrPositive(backupEntryMemoryCost));
// add near cache heap cost.
heapCost += mapContainer.getNearCacheSizeEstimator().getSize();
localMapStats.setHeapCost(heapCost);
if (mapContainer.getMapConfig().isNearCacheEnabled()) {
NearCacheStatsImpl nearCacheStats = getNearCache(mapName).getNearCacheStats();
localMapStats.setNearCacheStats(nearCacheStats);
}
return localMapStats;
}
