/** @return The interval info map for the aggregator */
protected Map<AggregationInterval, AggregationIntervalInfo> getAggregatorIntervalInfo(
final Class<? extends IPortalEventAggregator<?>> aggregatorType) {
final AggregatedIntervalConfig aggregatorIntervalConfig =
this.intervalsForAggregatorHelper.getAggregatorIntervalConfig(aggregatorType);
Map<AggregationInterval, AggregationIntervalInfo> intervalInfo =
this.aggregatorReadOnlyIntervalInfo.get(aggregatorType);
if (intervalInfo == null) {
final Builder<AggregationInterval, AggregationIntervalInfo> intervalInfoBuilder =
ImmutableMap.builder();
for (Map.Entry<AggregationInterval, AggregationIntervalInfo> intervalInfoEntry :
this.currentIntervalInfo.entrySet()) {
final AggregationInterval key = intervalInfoEntry.getKey();
if (aggregatorIntervalConfig.isIncluded(key)) {
intervalInfoBuilder.put(key, intervalInfoEntry.getValue());
}
}
intervalInfo = intervalInfoBuilder.build();
aggregatorReadOnlyIntervalInfo.put(aggregatorType, intervalInfo);
}
return intervalInfo;
}
public IntervalsForAggregatorHelper() {
this.defaultAggregatedIntervalConfig =
eventAggregationManagementDao.getDefaultAggregatedIntervalConfig();
// Create the set of intervals that are actually being aggregated
final Set<AggregationInterval> handledIntervalsNotIncluded =
EnumSet.allOf(AggregationInterval.class);
final Set<AggregationInterval> handledIntervalsBuilder =
EnumSet.noneOf(AggregationInterval.class);
for (final IntervalAwarePortalEventAggregator<PortalEvent> portalEventAggregator :
intervalAwarePortalEventAggregators) {
final Class<? extends IPortalEventAggregator<?>> aggregatorType =
PortalRawEventsAggregatorImpl.this.getClass(portalEventAggregator);
// Get aggregator specific interval info config
final AggregatedIntervalConfig aggregatorIntervalConfig =
this.getAggregatorIntervalConfig(aggregatorType);
for (final Iterator<AggregationInterval> intervalsIterator =
handledIntervalsNotIncluded.iterator();
intervalsIterator.hasNext(); ) {
final AggregationInterval interval = intervalsIterator.next();
if (aggregatorIntervalConfig.isIncluded(interval)) {
handledIntervalsBuilder.add(interval);
intervalsIterator.remove();
}
}
}
handledIntervals = Sets.immutableEnumSet(handledIntervalsBuilder);
}
private void doHandleIntervalBoundary(
AggregationInterval interval, Map<AggregationInterval, AggregationIntervalInfo> intervals) {
for (final IntervalAwarePortalEventAggregator<PortalEvent> portalEventAggregator :
intervalAwarePortalEventAggregators) {
final Class<? extends IPortalEventAggregator<?>> aggregatorType =
PortalRawEventsAggregatorImpl.this.getClass(portalEventAggregator);
final AggregatedIntervalConfig aggregatorIntervalConfig =
this.intervalsForAggregatorHelper.getAggregatorIntervalConfig(aggregatorType);
// If the aggreagator is configured to use the interval notify it of the interval boundary
if (aggregatorIntervalConfig.isIncluded(interval)) {
final Map<AggregationInterval, AggregationIntervalInfo> aggregatorIntervalInfo =
this.getAggregatorIntervalInfo(aggregatorType);
portalEventAggregator.handleIntervalBoundary(
interval, eventAggregationContext, aggregatorIntervalInfo);
}
}
}
@Override
@AggrEventsTransactional
public EventProcessingResult doCloseAggregations() {
if (!this.clusterLockService.isLockOwner(AGGREGATION_LOCK_NAME)) {
throw new IllegalStateException(
"The cluster lock "
+ AGGREGATION_LOCK_NAME
+ " must be owned by the current thread and server");
}
final IEventAggregatorStatus cleanUnclosedStatus =
eventAggregationManagementDao.getEventAggregatorStatus(ProcessingType.CLEAN_UNCLOSED, true);
// Update status with current server name
final String serverName = this.portalInfoProvider.getUniqueServerName();
cleanUnclosedStatus.setServerName(serverName);
cleanUnclosedStatus.setLastStart(new DateTime());
// Determine date of most recently aggregated data
final IEventAggregatorStatus eventAggregatorStatus =
eventAggregationManagementDao.getEventAggregatorStatus(ProcessingType.AGGREGATION, false);
if (eventAggregatorStatus == null || eventAggregatorStatus.getLastEventDate() == null) {
// Nothing has been aggregated, skip unclosed cleanup
cleanUnclosedStatus.setLastEnd(new DateTime());
eventAggregationManagementDao.updateEventAggregatorStatus(cleanUnclosedStatus);
return new EventProcessingResult(0, null, null, true);
}
final DateTime lastAggregatedDate = eventAggregatorStatus.getLastEventDate();
// If lastCleanUnclosedDate is null use the oldest date dimension as there can be
// no aggregations that exist before it
final DateTime lastCleanUnclosedDate;
if (cleanUnclosedStatus.getLastEventDate() == null) {
final DateDimension oldestDateDimension = this.dateDimensionDao.getOldestDateDimension();
lastCleanUnclosedDate = oldestDateDimension.getDate().toDateTime();
} else {
lastCleanUnclosedDate = cleanUnclosedStatus.getLastEventDate();
}
if (!(lastCleanUnclosedDate.isBefore(lastAggregatedDate))) {
logger.debug(
"No events aggregated since last unclosed aggregation cleaning, skipping clean: {}",
lastAggregatedDate);
return new EventProcessingResult(0, lastCleanUnclosedDate, lastAggregatedDate, true);
}
// Switch to flush on commit to avoid flushes during queries
final EntityManager entityManager = this.getEntityManager();
entityManager.flush();
entityManager.setFlushMode(FlushModeType.COMMIT);
// Track the number of closed aggregations and the last date of a cleaned interval
int closedAggregations = 0;
int cleanedIntervals = 0;
DateTime cleanUnclosedEnd;
final Thread currentThread = Thread.currentThread();
final String currentName = currentThread.getName();
try {
currentThread.setName(currentName + "-" + lastCleanUnclosedDate + "-" + lastAggregatedDate);
// Local caches used to reduce db io
final IntervalsForAggregatorHelper intervalsForAggregatorHelper =
new IntervalsForAggregatorHelper();
final Map<AggregationInterval, AggregationIntervalInfo> previousIntervals =
new HashMap<AggregationInterval, AggregationIntervalInfo>();
// A DateTime within the next interval to close aggregations in
DateTime nextIntervalDate = lastCleanUnclosedDate;
do {
// Reset our goal of catching up to the last aggregated event on every iteration
cleanUnclosedEnd = lastAggregatedDate;
// For each interval the aggregator supports, cleanup the unclosed aggregations
for (final AggregationInterval interval :
intervalsForAggregatorHelper.getHandledIntervals()) {
final AggregationIntervalInfo previousInterval = previousIntervals.get(interval);
if (previousInterval != null && nextIntervalDate.isBefore(previousInterval.getEnd())) {
logger.debug(
"{} interval before {} has already been cleaned during this execution, ignoring",
interval,
previousInterval.getEnd());
continue;
}
// The END date of the last clean session will find us the next interval to clean
final AggregationIntervalInfo nextIntervalToClean =
intervalHelper.getIntervalInfo(interval, nextIntervalDate);
previousIntervals.put(interval, nextIntervalToClean);
if (nextIntervalToClean == null) {
continue;
}
final DateTime start = nextIntervalToClean.getStart();
final DateTime end = nextIntervalToClean.getEnd();
if (!end.isBefore(lastAggregatedDate)) {
logger.debug(
"{} interval between {} and {} is still active, ignoring",
new Object[] {interval, start, end});
continue;
}
// Track the oldest interval end, this ensures that nothing is missed
if (end.isBefore(cleanUnclosedEnd)) {
cleanUnclosedEnd = end;
}
logger.debug(
"Cleaning unclosed {} aggregations between {} and {}",
new Object[] {interval, start, end});
for (final IntervalAwarePortalEventAggregator<PortalEvent> portalEventAggregator :
intervalAwarePortalEventAggregators) {
checkShutdown();
final Class<? extends IPortalEventAggregator<?>> aggregatorType =
getClass(portalEventAggregator);
// Get aggregator specific interval info config
final AggregatedIntervalConfig aggregatorIntervalConfig =
intervalsForAggregatorHelper.getAggregatorIntervalConfig(aggregatorType);
// If the aggregator is being used for the specified interval call
// cleanUnclosedAggregations
if (aggregatorIntervalConfig.isIncluded(interval)) {
closedAggregations +=
portalEventAggregator.cleanUnclosedAggregations(start, end, interval);
}
}
cleanedIntervals++;
}
// Set the next interval to the end date from the last aggregation run
nextIntervalDate = cleanUnclosedEnd;
logger.debug(
"Closed {} aggregations across {} interval before {} with goal of {}",
new Object[] {
closedAggregations, cleanedIntervals, cleanUnclosedEnd, lastAggregatedDate
});
// Loop until either the batchSize of cleaned aggregations has been reached or no
// aggregation work is done
} while (closedAggregations <= cleanUnclosedAggregationsBatchSize
&& cleanedIntervals <= cleanUnclosedIntervalsBatchSize
&& cleanUnclosedEnd.isBefore(lastAggregatedDate));
} finally {
currentThread.setName(currentName);
}
// Update the status object and store it
cleanUnclosedStatus.setLastEventDate(cleanUnclosedEnd);
cleanUnclosedStatus.setLastEnd(new DateTime());
eventAggregationManagementDao.updateEventAggregatorStatus(cleanUnclosedStatus);
return new EventProcessingResult(
closedAggregations,
lastCleanUnclosedDate,
lastAggregatedDate,
!cleanUnclosedEnd.isBefore(lastAggregatedDate));
}
