private void doAggregateEvent(PortalEvent item) { checkShutdown(); eventCounter.increment(); for (final ApplicationEventFilter<PortalEvent> applicationEventFilter : applicationEventFilters) { if (!applicationEventFilter.supports(item)) { logger.trace( "Skipping event {} - {} excluded by filter {}", new Object[] {eventCounter, item, applicationEventFilter}); return; } } logger.trace("Aggregating event {} - {}", eventCounter, item); // Load or create the event session EventSession eventSession = getEventSession(item); // Give each interval aware aggregator a chance at the event for (final IntervalAwarePortalEventAggregator<PortalEvent> portalEventAggregator : intervalAwarePortalEventAggregators) { if (checkSupports(portalEventAggregator, item)) { final Class<? extends IPortalEventAggregator<?>> aggregatorType = PortalRawEventsAggregatorImpl.this.getClass(portalEventAggregator); // Get aggregator specific interval info map final Map<AggregationInterval, AggregationIntervalInfo> aggregatorIntervalInfo = this.getAggregatorIntervalInfo(aggregatorType); // If there is an event session get the aggregator specific version of it if (eventSession != null) { final AggregatedGroupConfig aggregatorGroupConfig = getAggregatorGroupConfig(aggregatorType); final CacheKey key = CacheKey.build(EVENT_SESSION_CACHE_KEY_SOURCE, eventSession, aggregatorGroupConfig); EventSession filteredEventSession = this.eventAggregationContext.getAttribute(key); if (filteredEventSession == null) { filteredEventSession = new FilteredEventSession(eventSession, aggregatorGroupConfig); this.eventAggregationContext.setAttribute(key, filteredEventSession); } eventSession = filteredEventSession; } // Aggregation magic happens here! portalEventAggregator.aggregateEvent( item, eventSession, eventAggregationContext, aggregatorIntervalInfo); } } // Give each simple aggregator a chance at the event for (final SimplePortalEventAggregator<PortalEvent> portalEventAggregator : simplePortalEventAggregators) { if (checkSupports(portalEventAggregator, item)) { portalEventAggregator.aggregateEvent(item, eventSession); } } }
private void doHandleIntervalBoundary( AggregationInterval interval, Map<AggregationInterval, AggregationIntervalInfo> intervals) { for (final IntervalAwarePortalEventAggregator<PortalEvent> portalEventAggregator : intervalAwarePortalEventAggregators) { final Class<? extends IPortalEventAggregator<?>> aggregatorType = PortalRawEventsAggregatorImpl.this.getClass(portalEventAggregator); final AggregatedIntervalConfig aggregatorIntervalConfig = this.intervalsForAggregatorHelper.getAggregatorIntervalConfig(aggregatorType); // If the aggreagator is configured to use the interval notify it of the interval boundary if (aggregatorIntervalConfig.isIncluded(interval)) { final Map<AggregationInterval, AggregationIntervalInfo> aggregatorIntervalInfo = this.getAggregatorIntervalInfo(aggregatorType); portalEventAggregator.handleIntervalBoundary( interval, eventAggregationContext, aggregatorIntervalInfo); } } }
@Override @AggrEventsTransactional public EventProcessingResult doCloseAggregations() { if (!this.clusterLockService.isLockOwner(AGGREGATION_LOCK_NAME)) { throw new IllegalStateException( "The cluster lock " + AGGREGATION_LOCK_NAME + " must be owned by the current thread and server"); } final IEventAggregatorStatus cleanUnclosedStatus = eventAggregationManagementDao.getEventAggregatorStatus(ProcessingType.CLEAN_UNCLOSED, true); // Update status with current server name final String serverName = this.portalInfoProvider.getUniqueServerName(); cleanUnclosedStatus.setServerName(serverName); cleanUnclosedStatus.setLastStart(new DateTime()); // Determine date of most recently aggregated data final IEventAggregatorStatus eventAggregatorStatus = eventAggregationManagementDao.getEventAggregatorStatus(ProcessingType.AGGREGATION, false); if (eventAggregatorStatus == null || eventAggregatorStatus.getLastEventDate() == null) { // Nothing has been aggregated, skip unclosed cleanup cleanUnclosedStatus.setLastEnd(new DateTime()); eventAggregationManagementDao.updateEventAggregatorStatus(cleanUnclosedStatus); return new EventProcessingResult(0, null, null, true); } final DateTime lastAggregatedDate = eventAggregatorStatus.getLastEventDate(); // If lastCleanUnclosedDate is null use the oldest date dimension as there can be // no aggregations that exist before it final DateTime lastCleanUnclosedDate; if (cleanUnclosedStatus.getLastEventDate() == null) { final DateDimension oldestDateDimension = this.dateDimensionDao.getOldestDateDimension(); lastCleanUnclosedDate = oldestDateDimension.getDate().toDateTime(); } else { lastCleanUnclosedDate = cleanUnclosedStatus.getLastEventDate(); } if (!(lastCleanUnclosedDate.isBefore(lastAggregatedDate))) { logger.debug( "No events aggregated since last unclosed aggregation cleaning, skipping clean: {}", lastAggregatedDate); return new EventProcessingResult(0, lastCleanUnclosedDate, lastAggregatedDate, true); } // Switch to flush on commit to avoid flushes during queries final EntityManager entityManager = this.getEntityManager(); entityManager.flush(); entityManager.setFlushMode(FlushModeType.COMMIT); // Track the number of closed aggregations and the last date of a cleaned interval int closedAggregations = 0; int cleanedIntervals = 0; DateTime cleanUnclosedEnd; final Thread currentThread = Thread.currentThread(); final String currentName = currentThread.getName(); try { currentThread.setName(currentName + "-" + lastCleanUnclosedDate + "-" + lastAggregatedDate); // Local caches used to reduce db io final IntervalsForAggregatorHelper intervalsForAggregatorHelper = new IntervalsForAggregatorHelper(); final Map<AggregationInterval, AggregationIntervalInfo> previousIntervals = new HashMap<AggregationInterval, AggregationIntervalInfo>(); // A DateTime within the next interval to close aggregations in DateTime nextIntervalDate = lastCleanUnclosedDate; do { // Reset our goal of catching up to the last aggregated event on every iteration cleanUnclosedEnd = lastAggregatedDate; // For each interval the aggregator supports, cleanup the unclosed aggregations for (final AggregationInterval interval : intervalsForAggregatorHelper.getHandledIntervals()) { final AggregationIntervalInfo previousInterval = previousIntervals.get(interval); if (previousInterval != null && nextIntervalDate.isBefore(previousInterval.getEnd())) { logger.debug( "{} interval before {} has already been cleaned during this execution, ignoring", interval, previousInterval.getEnd()); continue; } // The END date of the last clean session will find us the next interval to clean final AggregationIntervalInfo nextIntervalToClean = intervalHelper.getIntervalInfo(interval, nextIntervalDate); previousIntervals.put(interval, nextIntervalToClean); if (nextIntervalToClean == null) { continue; } final DateTime start = nextIntervalToClean.getStart(); final DateTime end = nextIntervalToClean.getEnd(); if (!end.isBefore(lastAggregatedDate)) { logger.debug( "{} interval between {} and {} is still active, ignoring", new Object[] {interval, start, end}); continue; } // Track the oldest interval end, this ensures that nothing is missed if (end.isBefore(cleanUnclosedEnd)) { cleanUnclosedEnd = end; } logger.debug( "Cleaning unclosed {} aggregations between {} and {}", new Object[] {interval, start, end}); for (final IntervalAwarePortalEventAggregator<PortalEvent> portalEventAggregator : intervalAwarePortalEventAggregators) { checkShutdown(); final Class<? extends IPortalEventAggregator<?>> aggregatorType = getClass(portalEventAggregator); // Get aggregator specific interval info config final AggregatedIntervalConfig aggregatorIntervalConfig = intervalsForAggregatorHelper.getAggregatorIntervalConfig(aggregatorType); // If the aggregator is being used for the specified interval call // cleanUnclosedAggregations if (aggregatorIntervalConfig.isIncluded(interval)) { closedAggregations += portalEventAggregator.cleanUnclosedAggregations(start, end, interval); } } cleanedIntervals++; } // Set the next interval to the end date from the last aggregation run nextIntervalDate = cleanUnclosedEnd; logger.debug( "Closed {} aggregations across {} interval before {} with goal of {}", new Object[] { closedAggregations, cleanedIntervals, cleanUnclosedEnd, lastAggregatedDate }); // Loop until either the batchSize of cleaned aggregations has been reached or no // aggregation work is done } while (closedAggregations <= cleanUnclosedAggregationsBatchSize && cleanedIntervals <= cleanUnclosedIntervalsBatchSize && cleanUnclosedEnd.isBefore(lastAggregatedDate)); } finally { currentThread.setName(currentName); } // Update the status object and store it cleanUnclosedStatus.setLastEventDate(cleanUnclosedEnd); cleanUnclosedStatus.setLastEnd(new DateTime()); eventAggregationManagementDao.updateEventAggregatorStatus(cleanUnclosedStatus); return new EventProcessingResult( closedAggregations, lastCleanUnclosedDate, lastAggregatedDate, !cleanUnclosedEnd.isBefore(lastAggregatedDate)); }