Exemple #1
0
  public Runnable prepareMergeRunnable() {
    Map<MapContainer, Collection<Record>> recordMap =
        new HashMap<MapContainer, Collection<Record>>(mapContainers.size());
    InternalPartitionService partitionService = nodeEngine.getPartitionService();
    int partitionCount = partitionService.getPartitionCount();
    Address thisAddress = nodeEngine.getClusterService().getThisAddress();

    for (MapContainer mapContainer : mapContainers.values()) {
      for (int i = 0; i < partitionCount; i++) {
        RecordStore recordStore = getPartitionContainer(i).getRecordStore(mapContainer.getName());
        // add your owned entries to the map so they will be merged
        if (thisAddress.equals(partitionService.getPartitionOwner(i))) {
          Collection<Record> records = recordMap.get(mapContainer);
          if (records == null) {
            records = new ArrayList<Record>();
            recordMap.put(mapContainer, records);
          }
          records.addAll(recordStore.getReadonlyRecordMap().values());
        }
        // clear all records either owned or backup
        recordStore.reset();
      }
    }
    return new Merger(recordMap);
  }
 public boolean remove(Data dataKey, Object testValue) {
   checkIfLoaded();
   Record record = records.get(dataKey);
   Object oldValue = null;
   boolean removed = false;
   if (record == null) {
     if (mapContainer.getStore() != null) {
       oldValue = mapContainer.getStore().load(mapService.toObject(dataKey));
     }
     if (oldValue == null) return false;
   } else {
     oldValue = record.getValue();
   }
   if (mapService.compare(name, testValue, oldValue)) {
     mapService.interceptRemove(name, oldValue);
     removeIndex(dataKey);
     mapStoreDelete(record, dataKey);
     // reduce size
     updateSizeEstimator(-calculateRecordSize(record));
     deleteRecord(dataKey);
     cancelAssociatedSchedulers(dataKey);
     removed = true;
   }
   return removed;
 }
Exemple #3
0
 public void publishWanReplicationRemove(String mapName, Data key, long removeTime) {
   MapContainer mapContainer = getMapContainer(mapName);
   MapReplicationRemove replicationEvent = new MapReplicationRemove(mapName, key, removeTime);
   mapContainer
       .getWanReplicationPublisher()
       .publishReplicationEvent(SERVICE_NAME, replicationEvent);
 }
 public Object remove(Data dataKey) {
   checkIfLoaded();
   Record record = records.get(dataKey);
   Object oldValue = null;
   if (record == null) {
     if (mapContainer.getStore() != null) {
       oldValue = mapContainer.getStore().load(mapService.toObject(dataKey));
       if (oldValue != null) {
         removeIndex(dataKey);
         mapStoreDelete(null, dataKey);
       }
     }
   } else {
     oldValue = record.getValue();
     oldValue = mapService.interceptRemove(name, oldValue);
     if (oldValue != null) {
       removeIndex(dataKey);
       mapStoreDelete(record, dataKey);
     }
     // reduce size
     updateSizeEstimator(-calculateRecordSize(record));
     deleteRecord(dataKey);
     cancelAssociatedSchedulers(dataKey);
   }
   return oldValue;
 }
 public Object put(Data dataKey, Object value, long ttl) {
   checkIfLoaded();
   Record record = records.get(dataKey);
   Object oldValue = null;
   if (record == null) {
     if (mapContainer.getStore() != null) {
       oldValue = mapContainer.getStore().load(mapService.toObject(dataKey));
     }
     value = mapService.interceptPut(name, null, value);
     value = writeMapStore(dataKey, value, null);
     record = mapService.createRecord(name, dataKey, value, ttl);
     records.put(dataKey, record);
     updateSizeEstimator(calculateRecordSize(record));
     saveIndex(record);
   } else {
     oldValue = record.getValue();
     value = mapService.interceptPut(name, oldValue, value);
     value = writeMapStore(dataKey, value, record);
     // if key exists before, first reduce size
     updateSizeEstimator(-calculateRecordSize(record));
     setRecordValue(record, value);
     // then increase size.
     updateSizeEstimator(calculateRecordSize(record));
     updateTtl(record, ttl);
     saveIndex(record);
   }
   return oldValue;
 }
  public Object putIfAbsent(Data dataKey, Object value, long ttl) {
    checkIfLoaded();
    Record record = records.get(dataKey);
    Object oldValue = null;
    if (record == null) {
      if (mapContainer.getStore() != null) {
        oldValue = mapContainer.getStore().load(mapService.toObject(dataKey));
        if (oldValue != null) {
          record = mapService.createRecord(name, dataKey, oldValue, DEFAULT_TTL);
          records.put(dataKey, record);
          updateSizeEstimator(calculateRecordSize(record));
        }
      }
    } else {
      accessRecord(record);
      oldValue = record.getValue();
    }
    if (oldValue == null) {
      value = mapService.interceptPut(name, null, value);
      value = writeMapStore(dataKey, value, record);
      record = mapService.createRecord(name, dataKey, value, ttl);
      records.put(dataKey, record);
      updateSizeEstimator(calculateRecordSize(record));
      updateTtl(record, ttl);
    }
    saveIndex(record);

    return oldValue;
  }
Exemple #7
0
 @SuppressWarnings("unchecked")
 public void dispatchEvent(EventData eventData, EntryListener listener) {
   Member member = nodeEngine.getClusterService().getMember(eventData.getCaller());
   EntryEvent event =
       new DataAwareEntryEvent(
           member,
           eventData.getEventType(),
           eventData.getMapName(),
           eventData.getDataKey(),
           eventData.getDataNewValue(),
           eventData.getDataOldValue(),
           getSerializationService());
   switch (event.getEventType()) {
     case ADDED:
       listener.entryAdded(event);
       break;
     case EVICTED:
       listener.entryEvicted(event);
       break;
     case UPDATED:
       listener.entryUpdated(event);
       break;
     case REMOVED:
       listener.entryRemoved(event);
       break;
     default:
       throw new IllegalArgumentException("Invalid event type: " + event.getEventType());
   }
   MapContainer mapContainer = getMapContainer(eventData.getMapName());
   if (mapContainer.getMapConfig().isStatisticsEnabled()) {
     getLocalMapStatsImpl(eventData.getMapName()).incrementReceivedEvents();
   }
 }
Exemple #8
0
 public void publishWanReplicationUpdate(String mapName, EntryView entryView) {
   MapContainer mapContainer = getMapContainer(mapName);
   MapReplicationUpdate replicationEvent =
       new MapReplicationUpdate(mapName, mapContainer.getWanMergePolicy(), entryView);
   mapContainer
       .getWanReplicationPublisher()
       .publishReplicationEvent(SERVICE_NAME, replicationEvent);
 }
Exemple #9
0
 private void destroyMapStores() {
   for (MapContainer mapContainer : mapContainers.values()) {
     MapStoreWrapper store = mapContainer.getStore();
     if (store != null) {
       store.destroy();
     }
   }
 }
 private void accessRecord(Record record) {
   increaseRecordEvictionCounter(record, mapContainer.getMapConfig().getEvictionPolicy());
   record.onAccess();
   final int maxIdleSeconds = mapContainer.getMapConfig().getMaxIdleSeconds();
   if (maxIdleSeconds > 0) {
     mapService.scheduleIdleEviction(
         name, record.getKey(), TimeUnit.SECONDS.toMillis(maxIdleSeconds));
   }
 }
Exemple #11
0
  public boolean compare(String mapName, Object value1, Object value2) {
    if (value1 == null && value2 == null) {
      return true;
    }
    if (value1 == null) {
      return false;
    }
    if (value2 == null) {
      return false;
    }

    MapContainer mapContainer = getMapContainer(mapName);
    return mapContainer.getRecordFactory().isEquals(value1, value2);
  }
 private Record getRecordInternal(Data dataKey, boolean enableIndex) {
   Record record = null;
   if (mapContainer.getStore() != null) {
     final Object value = mapContainer.getStore().load(mapService.toObject(dataKey));
     if (value != null) {
       record = mapService.createRecord(name, dataKey, value, DEFAULT_TTL);
       records.put(dataKey, record);
       if (enableIndex) {
         saveIndex(record);
       }
       updateSizeEstimator(calculateRecordSize(record));
     }
   }
   return record;
 }
  public void clear() {
    checkIfLoaded();
    resetSizeEstimator();
    final Collection<Data> lockedKeys =
        lockStore != null ? lockStore.getLockedKeys() : Collections.<Data>emptySet();
    final Map<Data, Record> lockedRecords = new HashMap<Data, Record>(lockedKeys.size());
    // Locked records should not be removed!
    for (Data key : lockedKeys) {
      Record record = records.get(key);
      if (record != null) {
        lockedRecords.put(key, record);
        updateSizeEstimator(calculateRecordSize(record));
      }
    }
    Set<Data> keysToDelete = records.keySet();
    keysToDelete.removeAll(lockedRecords.keySet());

    final MapStoreWrapper store = mapContainer.getStore();
    // Use an ArrayList so that we don't trigger calls to equals or hashCode on the key objects
    Collection<Object> keysObject = new ArrayList<Object>(keysToDelete.size());
    for (Data key : keysToDelete) {
      // todo ea have a clear(Keys) method for optimizations
      removeIndex(key);
      keysObject.add(mapService.toObject(key));
    }

    if (store != null) {
      store.deleteAll(keysObject);
      toBeRemovedKeys.removeAll(keysToDelete);
    }

    clearRecordsMap(lockedRecords);
    cancelAssociatedSchedulers(keysToDelete);
    resetAccessSequenceNumber();
  }
 private void mapStoreDelete(Record record, Data key) {
   final MapStoreWrapper store = mapContainer.getStore();
   if (store != null) {
     long writeDelayMillis = mapContainer.getWriteDelayMillis();
     if (writeDelayMillis == 0) {
       store.delete(mapService.toObject(key));
       // todo ea record will be deleted then why calling onStore
       if (record != null) {
         record.onStore();
       }
     } else {
       mapService.scheduleMapStoreDelete(name, key, writeDelayMillis);
       toBeRemovedKeys.add(key);
     }
   }
 }
 private void updateTtl(Record record, long ttl) {
   if (ttl > 0) {
     mapService.scheduleTtlEviction(name, record, ttl);
   } else if (ttl == 0) {
     mapContainer.getTtlEvictionScheduler().cancel(record.getKey());
   }
 }
 private void saveIndex(Record record) {
   Data dataKey = record.getKey();
   final IndexService indexService = mapContainer.getIndexService();
   if (indexService.hasIndex()) {
     SerializationService ss = mapService.getSerializationService();
     QueryableEntry queryableEntry = new QueryEntry(ss, dataKey, dataKey, record.getValue());
     indexService.saveEntryIndex(queryableEntry);
   }
 }
 private Object writeMapStore(Data dataKey, Object recordValue, Record record) {
   final MapStoreWrapper store = mapContainer.getStore();
   if (store != null) {
     if (mapContainer.getWriteDelayMillis() <= 0) {
       Object objectValue = mapService.toObject(recordValue);
       store.store(mapService.toObject(dataKey), objectValue);
       if (record != null) {
         record.onStore();
       }
       // if store is not a post-processing map-store, then avoid extra de-serialization phase.
       return store.isPostProcessingMapStore() ? objectValue : recordValue;
     } else {
       mapService.scheduleMapStoreWrite(
           name, dataKey, recordValue, mapContainer.getWriteDelayMillis());
     }
   }
   return recordValue;
 }
 public MapEntrySet getAll(Set<Data> keySet) {
   checkIfLoaded();
   final MapEntrySet mapEntrySet = new MapEntrySet();
   Map<Object, Data> keyMapForLoader = null;
   if (mapContainer.getStore() != null) {
     keyMapForLoader = new HashMap<Object, Data>();
   }
   for (Data dataKey : keySet) {
     Record record = records.get(dataKey);
     if (record == null) {
       if (mapContainer.getStore() != null) {
         keyMapForLoader.put(mapService.toObject(dataKey), dataKey);
       }
     } else {
       accessRecord(record);
       Object value = record.getValue();
       value = mapService.interceptGet(name, value);
       if (value != null) {
         mapEntrySet.add(new AbstractMap.SimpleImmutableEntry(dataKey, mapService.toData(value)));
       }
     }
   }
   if (mapContainer.getStore() == null || keyMapForLoader.size() == 0) {
     return mapEntrySet;
   }
   final Map<Object, Object> loadedKeys =
       mapContainer.getStore().loadAll(keyMapForLoader.keySet());
   for (Map.Entry entry : loadedKeys.entrySet()) {
     final Object objectKey = entry.getKey();
     Object value = entry.getValue();
     Data dataKey = keyMapForLoader.get(objectKey);
     if (value != null) {
       Record record = mapService.createRecord(name, dataKey, value, DEFAULT_TTL);
       records.put(dataKey, record);
       saveIndex(record);
       updateSizeEstimator(calculateRecordSize(record));
     }
     value = mapService.interceptGet(name, value);
     if (value != null) {
       mapEntrySet.add(new AbstractMap.SimpleImmutableEntry(dataKey, mapService.toData(value)));
     }
   }
   return mapEntrySet;
 }
Exemple #19
0
    public void run() {
      for (final MapContainer mapContainer : recordMap.keySet()) {
        Collection<Record> recordList = recordMap.get(mapContainer);
        String mergePolicyName = mapContainer.getMapConfig().getMergePolicy();
        MapMergePolicy mergePolicy = getMergePolicy(mergePolicyName);

        // todo number of records may be high. below can be optimized a many records can be send in
        // single invocation
        final MapMergePolicy finalMergePolicy = mergePolicy;
        for (final Record record : recordList) {
          // todo too many submission. should submit them in subgroups
          nodeEngine
              .getExecutionService()
              .submit(
                  "hz:map-merge",
                  new Runnable() {
                    public void run() {
                      SimpleEntryView entryView =
                          new SimpleEntryView(
                              record.getKey(),
                              toData(record.getValue()),
                              record.getStatistics(),
                              record.getCost(),
                              record.getVersion());
                      MergeOperation operation =
                          new MergeOperation(
                              mapContainer.getName(), record.getKey(), entryView, finalMergePolicy);
                      try {
                        int partitionId =
                            nodeEngine.getPartitionService().getPartitionId(record.getKey());
                        Future f =
                            nodeEngine
                                .getOperationService()
                                .invokeOnPartition(SERVICE_NAME, operation, partitionId);
                        f.get();
                      } catch (Throwable t) {
                        throw ExceptionUtil.rethrow(t);
                      }
                    }
                  });
        }
      }
    }
 private void flush(Data key) {
   checkIfLoaded();
   EntryTaskScheduler writeScheduler = mapContainer.getMapStoreScheduler();
   Set<Data> keys = new HashSet<Data>(1);
   keys.add(key);
   if (writeScheduler != null) {
     Set<Data> processedKeys = writeScheduler.flush(keys);
     for (Data pkey : processedKeys) {
       records.get(pkey).onStore();
     }
   }
   EntryTaskScheduler deleteScheduler = mapContainer.getMapStoreScheduler();
   if (deleteScheduler != null) {
     if (toBeRemovedKeys.contains(key)) {
       deleteScheduler.flush(keys);
       toBeRemovedKeys.remove(key);
     }
   }
 }
Exemple #21
0
 @Override
 public void onReplicationEvent(WanReplicationEvent replicationEvent) {
   Object eventObject = replicationEvent.getEventObject();
   if (eventObject instanceof MapReplicationUpdate) {
     MapReplicationUpdate replicationUpdate = (MapReplicationUpdate) eventObject;
     EntryView entryView = replicationUpdate.getEntryView();
     MapMergePolicy mergePolicy = replicationUpdate.getMergePolicy();
     String mapName = replicationUpdate.getMapName();
     MapContainer mapContainer = getMapContainer(mapName);
     MergeOperation operation =
         new MergeOperation(
             mapName,
             toData(entryView.getKey(), mapContainer.getPartitioningStrategy()),
             entryView,
             mergePolicy);
     try {
       int partitionId = nodeEngine.getPartitionService().getPartitionId(entryView.getKey());
       Future f =
           nodeEngine
               .getOperationService()
               .invokeOnPartition(SERVICE_NAME, operation, partitionId);
       f.get();
     } catch (Throwable t) {
       throw ExceptionUtil.rethrow(t);
     }
   } else if (eventObject instanceof MapReplicationRemove) {
     MapReplicationRemove replicationRemove = (MapReplicationRemove) eventObject;
     WanOriginatedDeleteOperation operation =
         new WanOriginatedDeleteOperation(
             replicationRemove.getMapName(), replicationRemove.getKey());
     try {
       int partitionId =
           nodeEngine.getPartitionService().getPartitionId(replicationRemove.getKey());
       Future f =
           nodeEngine
               .getOperationService()
               .invokeOnPartition(SERVICE_NAME, operation, partitionId);
       f.get();
     } catch (Throwable t) {
       throw ExceptionUtil.rethrow(t);
     }
   }
 }
Exemple #22
0
 public void destroyDistributedObject(String name) {
   MapContainer mapContainer = mapContainers.remove(name);
   if (mapContainer != null) {
     if (mapContainer.isNearCacheEnabled()) {
       NearCache nearCache = nearCacheMap.remove(name);
       if (nearCache != null) {
         nearCache.clear();
       }
     }
     mapContainer.shutDownMapStoreScheduledExecutor();
   }
   final PartitionContainer[] containers = partitionContainers;
   for (PartitionContainer container : containers) {
     if (container != null) {
       container.destroyMap(name);
     }
   }
   nodeEngine.getEventService().deregisterAllListeners(SERVICE_NAME, name);
 }
 public void flush() {
   checkIfLoaded();
   Set<Data> keys = new HashSet<Data>();
   for (Record record : records.values()) {
     keys.add(record.getKey());
   }
   EntryTaskScheduler writeScheduler = mapContainer.getMapStoreScheduler();
   if (writeScheduler != null) {
     Set<Data> processedKeys = writeScheduler.flush(keys);
     for (Data key : processedKeys) {
       records.get(key).onStore();
     }
   }
   EntryTaskScheduler deleteScheduler = mapContainer.getMapStoreScheduler();
   if (deleteScheduler != null) {
     deleteScheduler.flush(toBeRemovedKeys);
     toBeRemovedKeys.clear();
   }
 }
  public boolean containsKey(Data dataKey) {
    checkIfLoaded();
    Record record = records.get(dataKey);
    if (record == null) {
      if (mapContainer.getStore() != null) {
        Object value = mapContainer.getStore().load(mapService.toObject(dataKey));
        if (value != null) {
          record = mapService.createRecord(name, dataKey, value, DEFAULT_TTL);
          records.put(dataKey, record);
          updateSizeEstimator(calculateRecordSize(record));
        }
      }
    }

    boolean contains = record != null;
    if (contains) {
      accessRecord(record);
    }
    return contains;
  }
Exemple #25
0
 private void migrateIndex(PartitionMigrationEvent event) {
   final PartitionContainer container = partitionContainers[event.getPartitionId()];
   for (RecordStore recordStore : container.getMaps().values()) {
     final MapContainer mapContainer = getMapContainer(recordStore.getName());
     final IndexService indexService = mapContainer.getIndexService();
     if (indexService.hasIndex()) {
       for (Record record : recordStore.getReadonlyRecordMap().values()) {
         if (event.getMigrationEndpoint() == MigrationEndpoint.SOURCE) {
           indexService.removeEntryIndex(record.getKey());
         } else {
           Object value = record.getValue();
           if (value != null) {
             indexService.saveEntryIndex(
                 new QueryEntry(
                     getSerializationService(), record.getKey(), record.getKey(), value));
           }
         }
       }
     }
   }
 }
Exemple #26
0
 private void setDelays(MapContainer mapContainer, RecordInfo info, Data key, int extraDelay) {
   long deleteDelay = -1;
   long writeDelay = -1;
   long idleDelay;
   long ttlDelay;
   if (mapContainer.getMapStoreScheduler() != null) {
     final ScheduledEntry scheduledEntry = mapContainer.getMapStoreScheduler().get(key);
     if (scheduledEntry != null) {
       if (scheduledEntry.getValue() == null) {
         deleteDelay = extraDelay + findDelayMillis(scheduledEntry);
       } else {
         writeDelay = extraDelay + findDelayMillis(scheduledEntry);
       }
     }
   }
   idleDelay = getDelay(mapContainer.getIdleEvictionScheduler(), key, extraDelay);
   ttlDelay = getDelay(mapContainer.getTtlEvictionScheduler(), key, extraDelay);
   // set delays.
   info.setMapStoreDeleteDelayMillis(deleteDelay);
   info.setMapStoreWriteDelayMillis(writeDelay);
   info.setIdleDelayMillis(idleDelay);
   info.setTtlDelayMillis(ttlDelay);
 }
  public Object get(Data dataKey) {
    checkIfLoaded();
    Record record = records.get(dataKey);
    Object value = null;
    if (record == null) {
      if (mapContainer.getStore() != null) {
        value = mapContainer.getStore().load(mapService.toObject(dataKey));
        if (value != null) {
          record = mapService.createRecord(name, dataKey, value, DEFAULT_TTL);
          records.put(dataKey, record);
          saveIndex(record);
          updateSizeEstimator(calculateRecordSize(record));
        }
      }

    } else {
      accessRecord(record);
      value = record.getValue();
    }
    value = mapService.interceptGet(name, value);

    return value;
  }
 public void clearPartition() {
   final LockService lockService =
       mapService.getNodeEngine().getSharedService(LockService.SERVICE_NAME);
   if (lockService != null) {
     lockService.clearLockStore(
         partitionId, new DefaultObjectNamespace(MapService.SERVICE_NAME, name));
   }
   final IndexService indexService = mapContainer.getIndexService();
   if (indexService.hasIndex()) {
     for (Data key : records.keySet()) {
       indexService.removeEntryIndex(key);
     }
   }
   cancelAssociatedSchedulers(records.keySet());
   clearRecordsMap(Collections.<Data, Record>emptyMap());
   resetSizeEstimator();
   resetAccessSequenceNumber();
 }
Exemple #29
0
  public Record createRecord(
      String name, Data dataKey, Object value, long ttl, boolean shouldSchedule) {
    MapContainer mapContainer = getMapContainer(name);
    Record record = mapContainer.getRecordFactory().newRecord(dataKey, value);

    if (shouldSchedule) {
      // if ttl is 0 then no eviction. if ttl is -1 then default configured eviction is applied
      if (ttl < 0 && mapContainer.getMapConfig().getTimeToLiveSeconds() > 0) {
        scheduleTtlEviction(
            name, record, mapContainer.getMapConfig().getTimeToLiveSeconds() * 1000);
      } else if (ttl > 0) {
        scheduleTtlEviction(name, record, ttl);
      }
      if (mapContainer.getMapConfig().getMaxIdleSeconds() > 0) {
        scheduleIdleEviction(name, dataKey, mapContainer.getMapConfig().getMaxIdleSeconds() * 1000);
      }
    }
    return record;
  }
 private void cancelAssociatedSchedulers(Data key) {
   mapContainer.getIdleEvictionScheduler().cancel(key);
   mapContainer.getTtlEvictionScheduler().cancel(key);
 }