Beispiel #1
0
  public void init(final NodeEngine nodeEngine, Properties properties) {
    int partitionCount = nodeEngine.getPartitionService().getPartitionCount();
    for (int i = 0; i < partitionCount; i++) {
      partitionContainers[i] = new PartitionContainer(this, i);
    }
    final LockService lockService = nodeEngine.getSharedService(LockService.SERVICE_NAME);
    if (lockService != null) {
      lockService.registerLockStoreConstructor(
          SERVICE_NAME,
          new ConstructorFunction<ObjectNamespace, LockStoreInfo>() {
            public LockStoreInfo createNew(final ObjectNamespace key) {
              final MapContainer mapContainer = getMapContainer(key.getObjectName());
              return new LockStoreInfo() {
                public int getBackupCount() {
                  return mapContainer.getBackupCount();
                }

                public int getAsyncBackupCount() {
                  return mapContainer.getAsyncBackupCount();
                }
              };
            }
          });
    }
    mapEvictionManager.init();
  }
Beispiel #2
0
  public Runnable prepareMergeRunnable() {
    Map<MapContainer, Collection<Record>> recordMap =
        new HashMap<MapContainer, Collection<Record>>(mapContainers.size());
    InternalPartitionService partitionService = nodeEngine.getPartitionService();
    int partitionCount = partitionService.getPartitionCount();
    Address thisAddress = nodeEngine.getClusterService().getThisAddress();

    for (MapContainer mapContainer : mapContainers.values()) {
      for (int i = 0; i < partitionCount; i++) {
        RecordStore recordStore = getPartitionContainer(i).getRecordStore(mapContainer.getName());
        // add your owned entries to the map so they will be merged
        if (thisAddress.equals(partitionService.getPartitionOwner(i))) {
          Collection<Record> records = recordMap.get(mapContainer);
          if (records == null) {
            records = new ArrayList<Record>();
            recordMap.put(mapContainer, records);
          }
          records.addAll(recordStore.getReadonlyRecordMap().values());
        }
        // clear all records either owned or backup
        recordStore.reset();
      }
    }
    return new Merger(recordMap);
  }
Beispiel #3
0
 public void invalidateAllNearCaches(String mapName, Set<Data> keys) {
   if (!isNearCacheEnabled(mapName)) {
     return;
   }
   if (keys == null || keys.isEmpty()) {
     return;
   }
   // send operation.
   Operation operation =
       new NearCacheKeySetInvalidationOperation(mapName, keys).setServiceName(SERVICE_NAME);
   Collection<MemberImpl> members = nodeEngine.getClusterService().getMemberList();
   for (MemberImpl member : members) {
     try {
       if (member.localMember()) continue;
       nodeEngine.getOperationService().send(operation, member.getAddress());
     } catch (Throwable throwable) {
       logger.warning(throwable);
     }
   }
   // below local invalidation is for the case the data is cached before partition is
   // owned/migrated
   for (final Data key : keys) {
     invalidateNearCache(mapName, key);
   }
 }
Beispiel #4
0
 @SuppressWarnings("unchecked")
 public void dispatchEvent(EventData eventData, EntryListener listener) {
   Member member = nodeEngine.getClusterService().getMember(eventData.getCaller());
   EntryEvent event =
       new DataAwareEntryEvent(
           member,
           eventData.getEventType(),
           eventData.getMapName(),
           eventData.getDataKey(),
           eventData.getDataNewValue(),
           eventData.getDataOldValue(),
           getSerializationService());
   switch (event.getEventType()) {
     case ADDED:
       listener.entryAdded(event);
       break;
     case EVICTED:
       listener.entryEvicted(event);
       break;
     case UPDATED:
       listener.entryUpdated(event);
       break;
     case REMOVED:
       listener.entryRemoved(event);
       break;
     default:
       throw new IllegalArgumentException("Invalid event type: " + event.getEventType());
   }
   MapContainer mapContainer = getMapContainer(eventData.getMapName());
   if (mapContainer.getMapConfig().isStatisticsEnabled()) {
     getLocalMapStatsImpl(eventData.getMapName()).incrementReceivedEvents();
   }
 }
Beispiel #5
0
 public Data toData(Object object) {
   if (object == null) return null;
   if (object instanceof Data) {
     return (Data) object;
   } else {
     return nodeEngine.getSerializationService().toData(object);
   }
 }
Beispiel #6
0
 public Object toObject(Object data) {
   if (data == null) return null;
   if (data instanceof Data) {
     return nodeEngine.toObject(data);
   } else {
     return data;
   }
 }
Beispiel #7
0
 public Data toData(Object object, PartitioningStrategy partitionStrategy) {
   if (object == null) return null;
   if (object instanceof Data) {
     return (Data) object;
   } else {
     return nodeEngine.getSerializationService().toData(object, partitionStrategy);
   }
 }
Beispiel #8
0
 public String addEventListener(
     EntryListener entryListener, EventFilter eventFilter, String mapName) {
   EventRegistration registration =
       nodeEngine
           .getEventService()
           .registerListener(SERVICE_NAME, mapName, eventFilter, entryListener);
   return registration.getId();
 }
Beispiel #9
0
 public void invalidateAllNearCaches(String mapName, Data key) {
   if (!isNearCacheEnabled(mapName)) {
     return;
   }
   Collection<MemberImpl> members = nodeEngine.getClusterService().getMemberList();
   for (MemberImpl member : members) {
     try {
       if (member.localMember()) continue;
       Operation operation =
           new InvalidateNearCacheOperation(mapName, key).setServiceName(SERVICE_NAME);
       nodeEngine.getOperationService().send(operation, member.getAddress());
     } catch (Throwable throwable) {
       throw new HazelcastException(throwable);
     }
   }
   // below local invalidation is for the case the data is cached before partition is
   // owned/migrated
   invalidateNearCache(mapName, key);
 }
Beispiel #10
0
 @Override
 public void onReplicationEvent(WanReplicationEvent replicationEvent) {
   Object eventObject = replicationEvent.getEventObject();
   if (eventObject instanceof MapReplicationUpdate) {
     MapReplicationUpdate replicationUpdate = (MapReplicationUpdate) eventObject;
     EntryView entryView = replicationUpdate.getEntryView();
     MapMergePolicy mergePolicy = replicationUpdate.getMergePolicy();
     String mapName = replicationUpdate.getMapName();
     MapContainer mapContainer = getMapContainer(mapName);
     MergeOperation operation =
         new MergeOperation(
             mapName,
             toData(entryView.getKey(), mapContainer.getPartitioningStrategy()),
             entryView,
             mergePolicy);
     try {
       int partitionId = nodeEngine.getPartitionService().getPartitionId(entryView.getKey());
       Future f =
           nodeEngine
               .getOperationService()
               .invokeOnPartition(SERVICE_NAME, operation, partitionId);
       f.get();
     } catch (Throwable t) {
       throw ExceptionUtil.rethrow(t);
     }
   } else if (eventObject instanceof MapReplicationRemove) {
     MapReplicationRemove replicationRemove = (MapReplicationRemove) eventObject;
     WanOriginatedDeleteOperation operation =
         new WanOriginatedDeleteOperation(
             replicationRemove.getMapName(), replicationRemove.getKey());
     try {
       int partitionId =
           nodeEngine.getPartitionService().getPartitionId(replicationRemove.getKey());
       Future f =
           nodeEngine
               .getOperationService()
               .invokeOnPartition(SERVICE_NAME, operation, partitionId);
       f.get();
     } catch (Throwable t) {
       throw ExceptionUtil.rethrow(t);
     }
   }
 }
Beispiel #11
0
 public MapService(NodeEngine nodeEngine) {
   this.nodeEngine = nodeEngine;
   this.mapEvictionManager = new MapEvictionManager(this);
   this.replicaWaitSecondsForScheduledTasks =
       getNodeEngine()
               .getGroupProperties()
               .MAP_REPLICA_WAIT_SECONDS_FOR_SCHEDULED_TASKS
               .getInteger()
           * 1000;
   logger = nodeEngine.getLogger(MapService.class.getName());
   partitionContainers =
       new PartitionContainer[nodeEngine.getPartitionService().getPartitionCount()];
   ownedPartitions = new AtomicReference<List<Integer>>();
   mergePolicyMap = new ConcurrentHashMap<String, MapMergePolicy>();
   mergePolicyMap.put(PutIfAbsentMapMergePolicy.class.getName(), new PutIfAbsentMapMergePolicy());
   mergePolicyMap.put(HigherHitsMapMergePolicy.class.getName(), new HigherHitsMapMergePolicy());
   mergePolicyMap.put(PassThroughMergePolicy.class.getName(), new PassThroughMergePolicy());
   mergePolicyMap.put(
       LatestUpdateMapMergePolicy.class.getName(), new LatestUpdateMapMergePolicy());
 }
Beispiel #12
0
 public QueryResult queryOnPartition(String mapName, Predicate predicate, int partitionId) {
   final QueryResult result = new QueryResult();
   List<QueryEntry> list = new LinkedList<QueryEntry>();
   PartitionContainer container = getPartitionContainer(partitionId);
   RecordStore recordStore = container.getRecordStore(mapName);
   Map<Data, Record> records = recordStore.getReadonlyRecordMap();
   SerializationService serializationService = nodeEngine.getSerializationService();
   final PagingPredicate pagingPredicate =
       predicate instanceof PagingPredicate ? (PagingPredicate) predicate : null;
   Comparator<Map.Entry> wrapperComparator = SortingUtil.newComparator(pagingPredicate);
   for (Record record : records.values()) {
     Data key = record.getKey();
     Object value = record.getValue();
     if (value == null) {
       continue;
     }
     QueryEntry queryEntry = new QueryEntry(serializationService, key, key, value);
     if (predicate.apply(queryEntry)) {
       if (pagingPredicate != null) {
         Map.Entry anchor = pagingPredicate.getAnchor();
         if (anchor != null
             && SortingUtil.compare(
                     pagingPredicate.getComparator(),
                     pagingPredicate.getIterationType(),
                     anchor,
                     queryEntry)
                 >= 0) {
           continue;
         }
       }
       list.add(queryEntry);
     }
   }
   if (pagingPredicate != null) {
     Collections.sort(list, wrapperComparator);
     if (list.size() > pagingPredicate.getPageSize()) {
       list = list.subList(0, pagingPredicate.getPageSize());
     }
   }
   for (QueryEntry entry : list) {
     result.add(
         new QueryResultEntryImpl(entry.getKeyData(), entry.getKeyData(), entry.getValueData()));
   }
   return result;
 }
Beispiel #13
0
 public void destroyDistributedObject(String name) {
   MapContainer mapContainer = mapContainers.remove(name);
   if (mapContainer != null) {
     if (mapContainer.isNearCacheEnabled()) {
       NearCache nearCache = nearCacheMap.remove(name);
       if (nearCache != null) {
         nearCache.clear();
       }
     }
     mapContainer.shutDownMapStoreScheduledExecutor();
   }
   final PartitionContainer[] containers = partitionContainers;
   for (PartitionContainer container : containers) {
     if (container != null) {
       container.destroyMap(name);
     }
   }
   nodeEngine.getEventService().deregisterAllListeners(SERVICE_NAME, name);
 }
Beispiel #14
0
  public MapMergePolicy getMergePolicy(String mergePolicyName) {
    MapMergePolicy mergePolicy = mergePolicyMap.get(mergePolicyName);
    if (mergePolicy == null && mergePolicyName != null) {
      try {

        // check if user has entered custom class name instead of policy name
        mergePolicy =
            ClassLoaderUtil.newInstance(nodeEngine.getConfigClassLoader(), mergePolicyName);
        mergePolicyMap.put(mergePolicyName, mergePolicy);
      } catch (Exception e) {
        logger.severe(e);
        throw ExceptionUtil.rethrow(e);
      }
    }
    if (mergePolicy == null) {
      return mergePolicyMap.get(MapConfig.DEFAULT_MAP_MERGE_POLICY);
    }
    return mergePolicy;
  }
Beispiel #15
0
 public SerializationService getSerializationService() {
   return nodeEngine.getSerializationService();
 }
Beispiel #16
0
 public boolean removeEventListener(String mapName, String registrationId) {
   return nodeEngine.getEventService().deregisterListener(SERVICE_NAME, mapName, registrationId);
 }
Beispiel #17
0
 public void publishEvent(
     Address caller,
     String mapName,
     EntryEventType eventType,
     Data dataKey,
     Data dataOldValue,
     Data dataValue) {
   Collection<EventRegistration> candidates =
       nodeEngine.getEventService().getRegistrations(SERVICE_NAME, mapName);
   Set<EventRegistration> registrationsWithValue = new HashSet<EventRegistration>();
   Set<EventRegistration> registrationsWithoutValue = new HashSet<EventRegistration>();
   if (candidates.isEmpty()) return;
   Object key = null;
   Object value = null;
   Object oldValue = null;
   for (EventRegistration candidate : candidates) {
     EventFilter filter = candidate.getFilter();
     if (filter instanceof EventServiceImpl.EmptyFilter) {
       registrationsWithValue.add(candidate);
     } else if (filter instanceof QueryEventFilter) {
       Object testValue;
       if (eventType == EntryEventType.REMOVED || eventType == EntryEventType.EVICTED) {
         oldValue = oldValue != null ? oldValue : toObject(dataOldValue);
         testValue = oldValue;
       } else {
         value = value != null ? value : toObject(dataValue);
         testValue = value;
       }
       key = key != null ? key : toObject(dataKey);
       QueryEventFilter queryEventFilter = (QueryEventFilter) filter;
       QueryEntry entry = new QueryEntry(getSerializationService(), dataKey, key, testValue);
       if (queryEventFilter.eval(entry)) {
         if (queryEventFilter.isIncludeValue()) {
           registrationsWithValue.add(candidate);
         } else {
           registrationsWithoutValue.add(candidate);
         }
       }
     } else if (filter.eval(dataKey)) {
       EntryEventFilter eventFilter = (EntryEventFilter) filter;
       if (eventFilter.isIncludeValue()) {
         registrationsWithValue.add(candidate);
       } else {
         registrationsWithoutValue.add(candidate);
       }
     }
   }
   if (registrationsWithValue.isEmpty() && registrationsWithoutValue.isEmpty()) return;
   String source = nodeEngine.getThisAddress().toString();
   if (eventType == EntryEventType.REMOVED || eventType == EntryEventType.EVICTED) {
     dataValue = dataValue != null ? dataValue : dataOldValue;
   }
   EventData event =
       new EventData(
           source, mapName, caller, dataKey, dataValue, dataOldValue, eventType.getType());
   int orderKey = dataKey.hashCode();
   nodeEngine
       .getEventService()
       .publishEvent(SERVICE_NAME, registrationsWithValue, event, orderKey);
   nodeEngine
       .getEventService()
       .publishEvent(
           SERVICE_NAME, registrationsWithoutValue, event.cloneWithoutValues(), orderKey);
 }
Beispiel #18
0
 private List<Integer> getMemberPartitions() {
   InternalPartitionService partitionService = nodeEngine.getPartitionService();
   List<Integer> partitions = partitionService.getMemberPartitions(nodeEngine.getThisAddress());
   return Collections.unmodifiableList(partitions);
 }
Beispiel #19
0
  public LocalMapStatsImpl createLocalMapStats(String mapName) {
    MapContainer mapContainer = getMapContainer(mapName);
    LocalMapStatsImpl localMapStats = getLocalMapStatsImpl(mapName);
    if (!mapContainer.getMapConfig().isStatisticsEnabled()) {
      return localMapStats;
    }

    long ownedEntryCount = 0;
    long backupEntryCount = 0;
    long dirtyCount = 0;
    long ownedEntryMemoryCost = 0;
    long backupEntryMemoryCost = 0;
    long hits = 0;
    long lockedEntryCount = 0;
    long heapCost = 0;

    int backupCount = mapContainer.getTotalBackupCount();
    ClusterService clusterService = nodeEngine.getClusterService();
    final InternalPartitionService partitionService = nodeEngine.getPartitionService();

    Address thisAddress = clusterService.getThisAddress();
    for (int partitionId = 0; partitionId < partitionService.getPartitionCount(); partitionId++) {
      InternalPartition partition = partitionService.getPartition(partitionId);
      Address owner = partition.getOwner();
      if (owner == null) {
        // no-op because no owner is set yet. Therefor we don't know anything about the map
        continue;
      }
      if (owner.equals(thisAddress)) {
        PartitionContainer partitionContainer = getPartitionContainer(partitionId);
        RecordStore recordStore = partitionContainer.getExistingRecordStore(mapName);

        // we don't want to force loading the record store because we are loading statistics. So
        // that is why
        // we ask for 'getExistingRecordStore' instead of 'getRecordStore' which does the load.
        if (recordStore != null) {
          heapCost += recordStore.getHeapCost();
          Map<Data, Record> records = recordStore.getReadonlyRecordMap();
          for (Record record : records.values()) {
            RecordStatistics stats = record.getStatistics();
            // there is map store and the record is dirty (waits to be stored)
            ownedEntryCount++;
            ownedEntryMemoryCost += record.getCost();
            localMapStats.setLastAccessTime(stats.getLastAccessTime());
            localMapStats.setLastUpdateTime(stats.getLastUpdateTime());
            hits += stats.getHits();
            if (recordStore.isLocked(record.getKey())) {
              lockedEntryCount++;
            }
          }
        }
      } else {
        for (int replica = 1; replica <= backupCount; replica++) {
          Address replicaAddress = partition.getReplicaAddress(replica);
          int tryCount = 30;
          // wait if the partition table is not updated yet
          while (replicaAddress == null
              && clusterService.getSize() > backupCount
              && tryCount-- > 0) {
            try {
              Thread.sleep(100);
            } catch (InterruptedException e) {
              throw ExceptionUtil.rethrow(e);
            }
            replicaAddress = partition.getReplicaAddress(replica);
          }

          if (replicaAddress != null && replicaAddress.equals(thisAddress)) {
            PartitionContainer partitionContainer = getPartitionContainer(partitionId);
            RecordStore recordStore = partitionContainer.getRecordStore(mapName);
            heapCost += recordStore.getHeapCost();

            Map<Data, Record> records = recordStore.getReadonlyRecordMap();
            for (Record record : records.values()) {
              backupEntryCount++;
              backupEntryMemoryCost += record.getCost();
            }
          } else if (replicaAddress == null && clusterService.getSize() > backupCount) {
            logger.warning("Partition: " + partition + ", replica: " + replica + " has no owner!");
          }
        }
      }
    }

    if (mapContainer.getMapStoreScheduler() != null) {
      dirtyCount = mapContainer.getMapStoreScheduler().size();
    }
    localMapStats.setBackupCount(backupCount);
    localMapStats.setDirtyEntryCount(zeroOrPositive(dirtyCount));
    localMapStats.setLockedEntryCount(zeroOrPositive(lockedEntryCount));
    localMapStats.setHits(zeroOrPositive(hits));
    localMapStats.setOwnedEntryCount(zeroOrPositive(ownedEntryCount));
    localMapStats.setBackupEntryCount(zeroOrPositive(backupEntryCount));
    localMapStats.setOwnedEntryMemoryCost(zeroOrPositive(ownedEntryMemoryCost));
    localMapStats.setBackupEntryMemoryCost(zeroOrPositive(backupEntryMemoryCost));
    // add near cache heap cost.
    heapCost += mapContainer.getNearCacheSizeEstimator().getSize();
    localMapStats.setHeapCost(heapCost);
    if (mapContainer.getMapConfig().isNearCacheEnabled()) {
      NearCacheStatsImpl nearCacheStats = getNearCache(mapName).getNearCacheStats();
      localMapStats.setNearCacheStats(nearCacheStats);
    }

    return localMapStats;
  }
Beispiel #20
0
 public MapContainer createNew(String mapName) {
   return new MapContainer(
       mapName, nodeEngine.getConfig().findMapConfig(mapName), MapService.this);
 }