@SuppressWarnings({"rawtypes", "unchecked"}) protected Function<? super Collection<?>, ?> lookupTransformation(String t1) { if ("average".equalsIgnoreCase(t1)) return new Enrichers.ComputingAverage(null, null, targetSensor.getTypeToken()); if ("sum".equalsIgnoreCase(t1)) return new Enrichers.ComputingSum(null, null, targetSensor.getTypeToken()); if ("isQuorate".equalsIgnoreCase(t1)) return new Enrichers.ComputingIsQuorate( targetSensor.getTypeToken(), QuorumChecks.of(config().get(QUORUM_CHECK_TYPE)), config().get(QUORUM_TOTAL_SIZE)); if ("list".equalsIgnoreCase(t1)) return new ComputingList(); return null; }
/** * Manages the persistence/rebind process. * * <p>Lifecycle is to create an instance of this, set it up (e.g. {@link * #setPeriodicPersistPeriod(Duration)}, {@link #setPersister(BrooklynMementoPersister)}; however * noting that persist period must be set before the persister). * * <p>Usually done for you by the conveniences (such as the launcher). */ public class RebindManagerImpl implements RebindManager { // TODO Use ImmediateDeltaChangeListener if the period is set to 0? public static final ConfigKey<RebindFailureMode> DANGLING_REFERENCE_FAILURE_MODE = ConfigKeys.newConfigKey( RebindFailureMode.class, "rebind.failureMode.danglingRef", "Action to take if a dangling reference is discovered during rebind", RebindFailureMode.CONTINUE); public static final ConfigKey<RebindFailureMode> REBIND_FAILURE_MODE = ConfigKeys.newConfigKey( RebindFailureMode.class, "rebind.failureMode.rebind", "Action to take if a failure occurs during rebind", RebindFailureMode.FAIL_AT_END); public static final ConfigKey<RebindFailureMode> ADD_CONFIG_FAILURE_MODE = ConfigKeys.newConfigKey( RebindFailureMode.class, "rebind.failureMode.addConfig", "Action to take if a failure occurs when setting a config value. It could happen coercion of the value type to fail.", RebindFailureMode.FAIL_AT_END); public static final ConfigKey<RebindFailureMode> ADD_POLICY_FAILURE_MODE = ConfigKeys.newConfigKey( RebindFailureMode.class, "rebind.failureMode.addPolicy", "Action to take if a failure occurs when adding a policy or enricher", RebindFailureMode.CONTINUE); public static final ConfigKey<RebindFailureMode> LOAD_POLICY_FAILURE_MODE = ConfigKeys.newConfigKey( RebindFailureMode.class, "rebind.failureMode.loadPolicy", "Action to take if a failure occurs when loading a policy or enricher", RebindFailureMode.CONTINUE); public static final ConfigKey<QuorumCheck> DANGLING_REFERENCES_MIN_REQUIRED_HEALTHY = ConfigKeys.newConfigKey( QuorumCheck.class, "rebind.failureMode.danglingRefs.minRequiredHealthy", "Number of items which must be rebinded at various sizes; " + "a small number of dangling references is possible if items are in the process of being created or deleted, " + "and that should be resolved on retry; the default set here allows max 2 dangling up to 10 items, " + "then linear regression to allow max 5% at 100 items and above", QuorumChecks.newLinearRange("[[0,-2],[10,8],[100,95],[200,190]]")); public static final Logger LOG = LoggerFactory.getLogger(RebindManagerImpl.class); private final ManagementContextInternal managementContext; private volatile Duration periodicPersistPeriod = Duration.ONE_SECOND; private volatile boolean persistenceRunning = false; private volatile PeriodicDeltaChangeListener persistenceRealChangeListener; private volatile ChangeListener persistencePublicChangeListener; private volatile boolean readOnlyRunning = false; private volatile ScheduledTask readOnlyTask = null; private transient Semaphore rebindActive = new Semaphore(1); private transient AtomicInteger readOnlyRebindCount = new AtomicInteger(Integer.MIN_VALUE); private volatile BrooklynMementoPersister persistenceStoreAccess; final boolean persistPoliciesEnabled; final boolean persistEnrichersEnabled; final boolean persistFeedsEnabled; final boolean persistCatalogItemsEnabled; private RebindFailureMode danglingRefFailureMode; private RebindFailureMode rebindFailureMode; private RebindFailureMode addConfigFailureMode; private RebindFailureMode addPolicyFailureMode; private RebindFailureMode loadPolicyFailureMode; private QuorumCheck danglingRefsQuorumRequiredHealthy; private boolean isAwaitingInitialRebind; private PersistenceActivityMetrics rebindMetrics = new PersistenceActivityMetrics(); private PersistenceActivityMetrics persistMetrics = new PersistenceActivityMetrics(); Integer firstRebindAppCount, firstRebindEntityCount, firstRebindItemCount; /** * For tracking if rebinding, for {@link AbstractEnricher#isRebinding()} etc. * * <p>TODO What is a better way to do this?! * * @author aled */ @Beta public static class RebindTracker { private static ThreadLocal<Boolean> rebinding = new ThreadLocal<Boolean>(); public static boolean isRebinding() { return (rebinding.get() == Boolean.TRUE); } static void reset() { rebinding.set(Boolean.FALSE); } static void setRebinding() { rebinding.set(Boolean.TRUE); } } public RebindManagerImpl(ManagementContextInternal managementContext) { this.managementContext = managementContext; this.persistencePublicChangeListener = ChangeListener.NOOP; this.persistPoliciesEnabled = BrooklynFeatureEnablement.isEnabled( BrooklynFeatureEnablement.FEATURE_POLICY_PERSISTENCE_PROPERTY); this.persistEnrichersEnabled = BrooklynFeatureEnablement.isEnabled( BrooklynFeatureEnablement.FEATURE_ENRICHER_PERSISTENCE_PROPERTY); this.persistFeedsEnabled = BrooklynFeatureEnablement.isEnabled( BrooklynFeatureEnablement.FEATURE_FEED_PERSISTENCE_PROPERTY); this.persistCatalogItemsEnabled = BrooklynFeatureEnablement.isEnabled( BrooklynFeatureEnablement.FEATURE_CATALOG_PERSISTENCE_PROPERTY); danglingRefFailureMode = managementContext.getConfig().getConfig(DANGLING_REFERENCE_FAILURE_MODE); rebindFailureMode = managementContext.getConfig().getConfig(REBIND_FAILURE_MODE); addConfigFailureMode = managementContext.getConfig().getConfig(ADD_CONFIG_FAILURE_MODE); addPolicyFailureMode = managementContext.getConfig().getConfig(ADD_POLICY_FAILURE_MODE); loadPolicyFailureMode = managementContext.getConfig().getConfig(LOAD_POLICY_FAILURE_MODE); danglingRefsQuorumRequiredHealthy = managementContext.getConfig().getConfig(DANGLING_REFERENCES_MIN_REQUIRED_HEALTHY); LOG.debug( "{} initialized, settings: policies={}, enrichers={}, feeds={}, catalog={}", new Object[] { this, persistPoliciesEnabled, persistEnrichersEnabled, persistFeedsEnabled, persistCatalogItemsEnabled }); } public ManagementContextInternal getManagementContext() { return managementContext; } /** Must be called before setPerister() */ public void setPeriodicPersistPeriod(Duration period) { if (persistenceStoreAccess != null) throw new IllegalStateException("Cannot set period after persister is generated."); this.periodicPersistPeriod = period; } /** @deprecated since 0.7.0; use {@link #setPeriodicPersistPeriod(Duration)} */ public void setPeriodicPersistPeriod(long periodMillis) { setPeriodicPersistPeriod(Duration.of(periodMillis, TimeUnit.MILLISECONDS)); } public boolean isPersistenceRunning() { return persistenceRunning; } public boolean isReadOnlyRunning() { return readOnlyRunning; } @Override public void setPersister(BrooklynMementoPersister val) { PersistenceExceptionHandler exceptionHandler = PersistenceExceptionHandlerImpl.builder().build(); setPersister(val, exceptionHandler); } @Override public void setPersister( BrooklynMementoPersister val, PersistenceExceptionHandler exceptionHandler) { if (persistenceStoreAccess != null && persistenceStoreAccess != val) { throw new IllegalStateException( "Dynamically changing persister is not supported: old=" + persistenceStoreAccess + "; new=" + val); } if (persistenceRealChangeListener != null) { // TODO should probably throw here, but previously we have not -- so let's log for now to be // sure it's not happening LOG.warn( "Persister reset after listeners have been set", new Throwable("Source of persister reset")); } this.persistenceStoreAccess = checkNotNull(val, "persister"); this.persistenceRealChangeListener = new PeriodicDeltaChangeListener( managementContext.getServerExecutionContext(), persistenceStoreAccess, exceptionHandler, persistMetrics, periodicPersistPeriod); this.persistencePublicChangeListener = new SafeChangeListener(persistenceRealChangeListener); if (persistenceRunning) { persistenceRealChangeListener.start(); } } @Override @VisibleForTesting public BrooklynMementoPersister getPersister() { return persistenceStoreAccess; } @Override public void startPersistence() { if (readOnlyRunning) { throw new IllegalStateException( "Cannot start read-only when already running with persistence"); } LOG.debug( "Starting persistence (" + this + "), mgmt " + managementContext.getManagementNodeId()); if (!persistenceRunning) { if (managementContext .getBrooklynProperties() .getConfig(BrooklynServerConfig.PERSISTENCE_BACKUPS_REQUIRED_ON_PROMOTION)) { BrooklynPersistenceUtils.createBackup( managementContext, CreateBackupMode.PROMOTION, MementoCopyMode.REMOTE); } } persistenceRunning = true; readOnlyRebindCount.set(Integer.MIN_VALUE); persistenceStoreAccess.enableWriteAccess(); if (persistenceRealChangeListener != null) persistenceRealChangeListener.start(); } @Override public void stopPersistence() { LOG.debug( "Stopping persistence (" + this + "), mgmt " + managementContext.getManagementNodeId()); persistenceRunning = false; if (persistenceRealChangeListener != null) persistenceRealChangeListener.stop(); if (persistenceStoreAccess != null) persistenceStoreAccess.disableWriteAccess(true); LOG.debug("Stopped rebind (persistence), mgmt " + managementContext.getManagementNodeId()); } @SuppressWarnings("unchecked") @Override public void startReadOnly(final ManagementNodeState mode) { if (!ManagementNodeState.isHotProxy(mode)) { throw new IllegalStateException( "Read-only rebind thread only permitted for hot proxy modes; not " + mode); } if (persistenceRunning) { throw new IllegalStateException( "Cannot start read-only when already running with persistence"); } if (readOnlyRunning || readOnlyTask != null) { LOG.warn( "Cannot request read-only mode for " + this + " when already running - " + readOnlyTask + "; ignoring"); return; } LOG.debug( "Starting read-only rebinding (" + this + "), mgmt " + managementContext.getManagementNodeId()); if (persistenceRealChangeListener != null) persistenceRealChangeListener.stop(); if (persistenceStoreAccess != null) persistenceStoreAccess.disableWriteAccess(true); readOnlyRunning = true; readOnlyRebindCount.set(0); try { rebind(null, null, mode); } catch (Exception e) { throw Exceptions.propagate(e); } Callable<Task<?>> taskFactory = new Callable<Task<?>>() { @Override public Task<Void> call() { return Tasks.<Void>builder() .dynamic(false) .displayName("rebind (periodic run") .body( new Callable<Void>() { public Void call() { try { rebind(null, null, mode); return null; } catch (RuntimeInterruptedException e) { LOG.debug("Interrupted rebinding (re-interrupting): " + e); if (LOG.isTraceEnabled()) LOG.trace("Interrupted rebinding (re-interrupting), details: " + e, e); Thread.currentThread().interrupt(); return null; } catch (Exception e) { // Don't rethrow: the behaviour of executionManager is different from a // scheduledExecutorService, // if we throw an exception, then our task will never get executed again if (!readOnlyRunning) { LOG.debug( "Problem rebinding (read-only running has probably just been turned off): " + e); if (LOG.isTraceEnabled()) { LOG.trace( "Problem rebinding (read-only running has probably just been turned off), details: " + e, e); } } else { LOG.error("Problem rebinding: " + Exceptions.collapseText(e), e); } return null; } catch (Throwable t) { LOG.warn("Problem rebinding (rethrowing)", t); throw Exceptions.propagate(t); } } }) .build(); } }; readOnlyTask = (ScheduledTask) managementContext .getServerExecutionContext() .submit( new ScheduledTask( MutableMap.of("displayName", "Periodic read-only rebind"), taskFactory) .period(periodicPersistPeriod)); } @Override public void stopReadOnly() { readOnlyRunning = false; if (readOnlyTask != null) { LOG.debug( "Stopping read-only rebinding (" + this + "), mgmt " + managementContext.getManagementNodeId()); readOnlyTask.cancel(true); readOnlyTask.blockUntilEnded(); boolean reallyEnded = Tasks.blockUntilInternalTasksEnded(readOnlyTask, Duration.TEN_SECONDS); if (!reallyEnded) { LOG.warn( "Rebind (read-only) tasks took too long to die after interrupt (ignoring): " + readOnlyTask); } readOnlyTask = null; LOG.debug( "Stopped read-only rebinding (" + this + "), mgmt " + managementContext.getManagementNodeId()); } } @Override public void start() { ManagementNodeState target = getRebindMode(); if (target == ManagementNodeState.HOT_STANDBY || target == ManagementNodeState.HOT_BACKUP) { startReadOnly(target); } else if (target == ManagementNodeState.MASTER) { startPersistence(); } else { LOG.warn("Nothing to start in " + this + " when HA mode is " + target); } } @Override public void stop() { stopReadOnly(); stopPersistence(); if (persistenceStoreAccess != null) persistenceStoreAccess.stop(true); } public void rebindPartialActive( CompoundTransformer transformer, Iterator<BrooklynObject> objectsToRebind) { final ClassLoader classLoader = managementContext.getCatalogClassLoader(); // TODO we might want different exception handling for partials; // failure at various points should leave proxies in a sensible state, // either pointing at old or at new, though this is relatively untested, // and some things e.g. policies might not be properly started final RebindExceptionHandler exceptionHandler = RebindExceptionHandlerImpl.builder() .danglingRefFailureMode(danglingRefFailureMode) .danglingRefQuorumRequiredHealthy(danglingRefsQuorumRequiredHealthy) .rebindFailureMode(rebindFailureMode) .addConfigFailureMode(addConfigFailureMode) .addPolicyFailureMode(addPolicyFailureMode) .loadPolicyFailureMode(loadPolicyFailureMode) .build(); final ManagementNodeState mode = getRebindMode(); ActivePartialRebindIteration iteration = new ActivePartialRebindIteration( this, mode, classLoader, exceptionHandler, rebindActive, readOnlyRebindCount, rebindMetrics, persistenceStoreAccess); iteration.setObjectIterator( Iterators.transform( objectsToRebind, new Function<BrooklynObject, BrooklynObject>() { @Override public BrooklynObject apply(BrooklynObject obj) { // entities must be deproxied if (obj instanceof Entity) obj = Entities.deproxy((Entity) obj); return obj; } })); if (transformer != null) iteration.applyTransformer(transformer); iteration.run(); } public void rebindPartialActive(CompoundTransformer transformer, String... objectsToRebindIds) { List<BrooklynObject> objectsToRebind = MutableList.of(); for (String objectId : objectsToRebindIds) { BrooklynObject obj = managementContext.lookup(objectId); objectsToRebind.add(obj); } rebindPartialActive(transformer, objectsToRebind.iterator()); } protected ManagementNodeState getRebindMode() { if (managementContext == null) throw new IllegalStateException("Invalid " + this + ": no management context"); if (!(managementContext.getHighAvailabilityManager() instanceof HighAvailabilityManagerImpl)) throw new IllegalStateException( "Invalid " + this + ": unknown HA manager type " + managementContext.getHighAvailabilityManager()); ManagementNodeState target = ((HighAvailabilityManagerImpl) managementContext.getHighAvailabilityManager()) .getTransitionTargetNodeState(); return target; } @Override @VisibleForTesting public void waitForPendingComplete(Duration timeout, boolean canTrigger) throws InterruptedException, TimeoutException { if (persistenceStoreAccess == null || !persistenceRunning) return; persistenceRealChangeListener.waitForPendingComplete(timeout, canTrigger); persistenceStoreAccess.waitForWritesCompleted(timeout); } @Override @VisibleForTesting public void forcePersistNow() { forcePersistNow(false, null); } @Override @VisibleForTesting public void forcePersistNow(boolean full, PersistenceExceptionHandler exceptionHandler) { if (full) { BrooklynMementoRawData memento = BrooklynPersistenceUtils.newStateMemento(managementContext, MementoCopyMode.LOCAL); if (exceptionHandler == null) { exceptionHandler = persistenceRealChangeListener.getExceptionHandler(); } persistenceStoreAccess.checkpoint(memento, exceptionHandler); } else { if (!persistenceRealChangeListener.persistNowSafely()) { throw new IllegalStateException("Forced persistence failed; see logs fore more detail"); } } } @Override public ChangeListener getChangeListener() { return persistencePublicChangeListener; } @Override public List<Application> rebind() { return rebind(null, null, null); } @Override public List<Application> rebind(final ClassLoader classLoader) { return rebind(classLoader, null, null); } @Override public List<Application> rebind( final ClassLoader classLoader, final RebindExceptionHandler exceptionHandler) { return rebind(classLoader, exceptionHandler, null); } @Override public List<Application> rebind( ClassLoader classLoaderO, RebindExceptionHandler exceptionHandlerO, ManagementNodeState modeO) { final ClassLoader classLoader = classLoaderO != null ? classLoaderO : managementContext.getCatalogClassLoader(); final RebindExceptionHandler exceptionHandler = exceptionHandlerO != null ? exceptionHandlerO : RebindExceptionHandlerImpl.builder() .danglingRefFailureMode(danglingRefFailureMode) .danglingRefQuorumRequiredHealthy(danglingRefsQuorumRequiredHealthy) .rebindFailureMode(rebindFailureMode) .addConfigFailureMode(addConfigFailureMode) .addPolicyFailureMode(addPolicyFailureMode) .loadPolicyFailureMode(loadPolicyFailureMode) .build(); final ManagementNodeState mode = modeO != null ? modeO : getRebindMode(); if (mode != ManagementNodeState.MASTER && mode != ManagementNodeState.HOT_STANDBY && mode != ManagementNodeState.HOT_BACKUP) throw new IllegalStateException( "Must be either master or hot standby/backup to rebind (mode " + mode + ")"); ExecutionContext ec = BasicExecutionContext.getCurrentExecutionContext(); if (ec == null) { ec = managementContext.getServerExecutionContext(); Task<List<Application>> task = ec.submit( new Callable<List<Application>>() { @Override public List<Application> call() throws Exception { return rebindImpl(classLoader, exceptionHandler, mode); } }); try { return task.get(); } catch (Exception e) { throw Exceptions.propagate(e); } } else { return rebindImpl(classLoader, exceptionHandler, mode); } } @Override public BrooklynMementoRawData retrieveMementoRawData() { RebindExceptionHandler exceptionHandler = RebindExceptionHandlerImpl.builder() .danglingRefFailureMode(danglingRefFailureMode) .rebindFailureMode(rebindFailureMode) .addConfigFailureMode(addConfigFailureMode) .addPolicyFailureMode(addPolicyFailureMode) .loadPolicyFailureMode(loadPolicyFailureMode) .build(); return loadMementoRawData(exceptionHandler); } /** * Uses the persister to retrieve (and thus deserialize) the memento. * * <p>In so doing, it instantiates the entities + locations, registering them with the * rebindContext. */ protected BrooklynMementoRawData loadMementoRawData( final RebindExceptionHandler exceptionHandler) { try { if (persistenceStoreAccess == null) { throw new IllegalStateException( "Persistence not configured; cannot load memento data from persistent backing store"); } if (!(persistenceStoreAccess instanceof BrooklynMementoPersisterToObjectStore)) { throw new IllegalStateException( "Cannot load raw memento with persister " + persistenceStoreAccess); } return ((BrooklynMementoPersisterToObjectStore) persistenceStoreAccess) .loadMementoRawData(exceptionHandler); } catch (RuntimeException e) { throw exceptionHandler.onFailed(e); } } protected List<Application> rebindImpl( final ClassLoader classLoader, final RebindExceptionHandler exceptionHandler, ManagementNodeState mode) { RebindIteration iteration = new InitialFullRebindIteration( this, mode, classLoader, exceptionHandler, rebindActive, readOnlyRebindCount, rebindMetrics, persistenceStoreAccess); iteration.run(); if (firstRebindAppCount == null) { firstRebindAppCount = iteration.getApplications().size(); firstRebindEntityCount = iteration.getRebindContext().getEntities().size(); firstRebindItemCount = iteration.getRebindContext().getAllBrooklynObjects().size(); } isAwaitingInitialRebind = false; return iteration.getApplications(); } /** * Sorts the map of nodes, so that a node's parent is guaranteed to come before that node (unless * the parent is missing). * * <p>Relies on ordering guarantees of returned map (i.e. LinkedHashMap, which guarantees * insertion order even if a key is re-inserted into the map). * * <p>TODO Inefficient implementation! */ @VisibleForTesting static <T extends TreeNode> Map<String, T> sortParentFirst(Map<String, T> nodes) { Map<String, T> result = Maps.newLinkedHashMap(); for (T node : nodes.values()) { List<T> tempchain = Lists.newLinkedList(); T nodeinchain = node; while (nodeinchain != null) { tempchain.add(0, nodeinchain); nodeinchain = (nodeinchain.getParent() == null) ? null : nodes.get(nodeinchain.getParent()); } for (T n : tempchain) { result.put(n.getId(), n); } } return result; } public boolean isAwaitingInitialRebind() { return isAwaitingInitialRebind; } public void setAwaitingInitialRebind(boolean isAwaitingInitialRebind) { this.isAwaitingInitialRebind = isAwaitingInitialRebind; } /** * Wraps a ChangeListener, to log and never propagate any exceptions that it throws. * * <p>Catches Throwable, because really don't want a problem to propagate up to user code, to * cause business-level operations to fail. For example, if there is a linkage error due to some * problem in the serialization dependencies then just log it. For things more severe (e.g. * OutOfMemoryError) then the catch+log means we'll report that we failed to persist, and we'd * expect other threads to throw the OutOfMemoryError so we shouldn't lose anything. */ private static class SafeChangeListener implements ChangeListener { private final ChangeListener delegate; public SafeChangeListener(ChangeListener delegate) { this.delegate = delegate; } @Override public void onManaged(BrooklynObject instance) { try { delegate.onManaged(instance); } catch (Throwable t) { LOG.error("Error persisting mememento onManaged(" + instance + "); continuing.", t); } } @Override public void onChanged(BrooklynObject instance) { try { delegate.onChanged(instance); } catch (Throwable t) { LOG.error("Error persisting mememento onChanged(" + instance + "); continuing.", t); } } @Override public void onUnmanaged(BrooklynObject instance) { try { delegate.onUnmanaged(instance); } catch (Throwable t) { LOG.error("Error persisting mememento onUnmanaged(" + instance + "); continuing.", t); } } } public int getReadOnlyRebindCount() { return readOnlyRebindCount.get(); } @Override public Map<String, Object> getMetrics() { Map<String, Object> result = MutableMap.of(); result.put("rebind", rebindMetrics.asMap()); result.put("persist", persistMetrics.asMap()); if (readOnlyRebindCount.get() >= 0) result.put("rebindReadOnlyCount", readOnlyRebindCount); // include first rebind counts, so we know whether we rebinded or not result.put( "firstRebindCounts", MutableMap.of( "applications", firstRebindAppCount, "entities", firstRebindEntityCount, "allItems", firstRebindItemCount)); return result; } @Override public String toString() { return super.toString() + "[mgmt=" + managementContext.getManagementNodeId() + "]"; } }