@Test
public void aggregateRawEventsIncompleteByProcessCount() throws Exception {
when(transactionOperations.execute(any(TransactionCallback.class)))
.then(
new Answer<EventProcessingResult>() {
@Override
public EventProcessingResult answer(InvocationOnMock invocation) throws Throwable {
final TransactionStatus status = mock(TransactionStatus.class);
return ((TransactionCallback<EventProcessingResult>) invocation.getArguments()[0])
.doInTransaction(status);
}
});
when(clusterLockService.isLockOwner(PortalRawEventsAggregator.AGGREGATION_LOCK_NAME))
.thenReturn(true);
when(portalEventDimensionPopulator.isCheckedDimensions()).thenReturn(true);
when(eventAggregationManagementDao.getEventAggregatorStatus(ProcessingType.AGGREGATION, true))
.thenReturn(eventAggregatorStatus);
when(portalInfoProvider.getUniqueServerName()).thenReturn("serverName_abcd");
when(eventAggregatorStatus.getLastEventDate()).thenReturn(new DateTime(1325881376117l));
when(portalEventDao.aggregatePortalEvents(
any(DateTime.class),
any(DateTime.class),
(int) any(Integer.TYPE),
(Function<PortalEvent, Boolean>) any(Function.class)))
.then(
new Answer<Boolean>() {
@Override
public Boolean answer(InvocationOnMock invocation) throws Throwable {
((Function<PortalEvent, Boolean>) invocation.getArguments()[3])
.apply(new MockPortalEvent(this, "serverName", "eventSessionId", person));
return false;
}
});
when(eventSessionDao.getEventSession(any(PortalEvent.class))).thenReturn(eventSession);
this.portalEventAggregator.setEventAggregationBatchSize(1);
final EventProcessingResult result = portalEventAggregator.doAggregateRawEvents();
assertNotNull(result);
assertEquals(1, result.getProcessed());
assertEquals(false, result.isComplete());
this.portalEventAggregator.setEventAggregationBatchSize(1000);
}
private EventProcessingResult doAggregateRawEventsInternal() {
if (!this.clusterLockService.isLockOwner(AGGREGATION_LOCK_NAME)) {
throw new IllegalStateException(
"The cluster lock "
+ AGGREGATION_LOCK_NAME
+ " must be owned by the current thread and server");
}
if (!this.portalEventDimensionPopulator.isCheckedDimensions()) {
// First time aggregation has happened, run populateDimensions to ensure enough dimension data
// exists
final boolean populatedDimensions = this.portalEventAggregationManager.populateDimensions();
if (!populatedDimensions) {
this.logger.warn(
"Aborting raw event aggregation, populateDimensions returned false so the state of date/time dimensions is unknown");
return null;
}
}
// Flush any dimension creation before aggregation
final EntityManager entityManager = this.getEntityManager();
entityManager.flush();
entityManager.setFlushMode(FlushModeType.COMMIT);
final IEventAggregatorStatus eventAggregatorStatus =
eventAggregationManagementDao.getEventAggregatorStatus(ProcessingType.AGGREGATION, true);
// Update status with current server name
final String serverName = this.portalInfoProvider.getUniqueServerName();
final String previousServerName = eventAggregatorStatus.getServerName();
if (previousServerName != null && !serverName.equals(previousServerName)) {
this.logger.debug(
"Last aggregation run on {} clearing all aggregation caches", previousServerName);
final Session session = getEntityManager().unwrap(Session.class);
final Cache cache = session.getSessionFactory().getCache();
cache.evictEntityRegions();
}
eventAggregatorStatus.setServerName(serverName);
// Calculate date range for aggregation
DateTime lastAggregated = eventAggregatorStatus.getLastEventDate();
if (lastAggregated == null) {
lastAggregated = portalEventDao.getOldestPortalEventTimestamp();
// No portal events to aggregate, skip aggregation
if (lastAggregated == null) {
return new EventProcessingResult(0, null, null, true);
}
// First time aggregation has run, initialize the CLEAN_UNCLOSED status to save catch-up time
final IEventAggregatorStatus cleanUnclosedStatus =
eventAggregationManagementDao.getEventAggregatorStatus(
ProcessingType.CLEAN_UNCLOSED, true);
AggregationIntervalInfo oldestMinuteInterval =
this.intervalHelper.getIntervalInfo(AggregationInterval.MINUTE, lastAggregated);
cleanUnclosedStatus.setLastEventDate(oldestMinuteInterval.getStart().minusMinutes(1));
eventAggregationManagementDao.updateEventAggregatorStatus(cleanUnclosedStatus);
}
final DateTime newestEventTime =
DateTime.now().minus(this.aggregationDelay).secondOfMinute().roundFloorCopy();
final Thread currentThread = Thread.currentThread();
final String currentName = currentThread.getName();
final MutableInt events = new MutableInt();
final MutableObject lastEventDate = new MutableObject(newestEventTime);
boolean complete;
try {
currentThread.setName(currentName + "-" + lastAggregated + "_" + newestEventTime);
logger.debug(
"Starting aggregation of events between {} (inc) and {} (exc)",
lastAggregated,
newestEventTime);
// Do aggregation, capturing the start and end dates
eventAggregatorStatus.setLastStart(DateTime.now());
complete =
portalEventDao.aggregatePortalEvents(
lastAggregated,
newestEventTime,
this.eventAggregationBatchSize,
new AggregateEventsHandler(events, lastEventDate, eventAggregatorStatus));
eventAggregatorStatus.setLastEventDate((DateTime) lastEventDate.getValue());
eventAggregatorStatus.setLastEnd(DateTime.now());
} finally {
currentThread.setName(currentName);
}
// Store the results of the aggregation
eventAggregationManagementDao.updateEventAggregatorStatus(eventAggregatorStatus);
complete =
complete
&& (this.eventAggregationBatchSize <= 0
|| events.intValue() < this.eventAggregationBatchSize);
return new EventProcessingResult(
events.intValue(), lastAggregated, eventAggregatorStatus.getLastEventDate(), complete);
}
@Override
@AggrEventsTransactional
public EventProcessingResult doCloseAggregations() {
if (!this.clusterLockService.isLockOwner(AGGREGATION_LOCK_NAME)) {
throw new IllegalStateException(
"The cluster lock "
+ AGGREGATION_LOCK_NAME
+ " must be owned by the current thread and server");
}
final IEventAggregatorStatus cleanUnclosedStatus =
eventAggregationManagementDao.getEventAggregatorStatus(ProcessingType.CLEAN_UNCLOSED, true);
// Update status with current server name
final String serverName = this.portalInfoProvider.getUniqueServerName();
cleanUnclosedStatus.setServerName(serverName);
cleanUnclosedStatus.setLastStart(new DateTime());
// Determine date of most recently aggregated data
final IEventAggregatorStatus eventAggregatorStatus =
eventAggregationManagementDao.getEventAggregatorStatus(ProcessingType.AGGREGATION, false);
if (eventAggregatorStatus == null || eventAggregatorStatus.getLastEventDate() == null) {
// Nothing has been aggregated, skip unclosed cleanup
cleanUnclosedStatus.setLastEnd(new DateTime());
eventAggregationManagementDao.updateEventAggregatorStatus(cleanUnclosedStatus);
return new EventProcessingResult(0, null, null, true);
}
final DateTime lastAggregatedDate = eventAggregatorStatus.getLastEventDate();
// If lastCleanUnclosedDate is null use the oldest date dimension as there can be
// no aggregations that exist before it
final DateTime lastCleanUnclosedDate;
if (cleanUnclosedStatus.getLastEventDate() == null) {
final DateDimension oldestDateDimension = this.dateDimensionDao.getOldestDateDimension();
lastCleanUnclosedDate = oldestDateDimension.getDate().toDateTime();
} else {
lastCleanUnclosedDate = cleanUnclosedStatus.getLastEventDate();
}
if (!(lastCleanUnclosedDate.isBefore(lastAggregatedDate))) {
logger.debug(
"No events aggregated since last unclosed aggregation cleaning, skipping clean: {}",
lastAggregatedDate);
return new EventProcessingResult(0, lastCleanUnclosedDate, lastAggregatedDate, true);
}
// Switch to flush on commit to avoid flushes during queries
final EntityManager entityManager = this.getEntityManager();
entityManager.flush();
entityManager.setFlushMode(FlushModeType.COMMIT);
// Track the number of closed aggregations and the last date of a cleaned interval
int closedAggregations = 0;
int cleanedIntervals = 0;
DateTime cleanUnclosedEnd;
final Thread currentThread = Thread.currentThread();
final String currentName = currentThread.getName();
try {
currentThread.setName(currentName + "-" + lastCleanUnclosedDate + "-" + lastAggregatedDate);
// Local caches used to reduce db io
final IntervalsForAggregatorHelper intervalsForAggregatorHelper =
new IntervalsForAggregatorHelper();
final Map<AggregationInterval, AggregationIntervalInfo> previousIntervals =
new HashMap<AggregationInterval, AggregationIntervalInfo>();
// A DateTime within the next interval to close aggregations in
DateTime nextIntervalDate = lastCleanUnclosedDate;
do {
// Reset our goal of catching up to the last aggregated event on every iteration
cleanUnclosedEnd = lastAggregatedDate;
// For each interval the aggregator supports, cleanup the unclosed aggregations
for (final AggregationInterval interval :
intervalsForAggregatorHelper.getHandledIntervals()) {
final AggregationIntervalInfo previousInterval = previousIntervals.get(interval);
if (previousInterval != null && nextIntervalDate.isBefore(previousInterval.getEnd())) {
logger.debug(
"{} interval before {} has already been cleaned during this execution, ignoring",
interval,
previousInterval.getEnd());
continue;
}
// The END date of the last clean session will find us the next interval to clean
final AggregationIntervalInfo nextIntervalToClean =
intervalHelper.getIntervalInfo(interval, nextIntervalDate);
previousIntervals.put(interval, nextIntervalToClean);
if (nextIntervalToClean == null) {
continue;
}
final DateTime start = nextIntervalToClean.getStart();
final DateTime end = nextIntervalToClean.getEnd();
if (!end.isBefore(lastAggregatedDate)) {
logger.debug(
"{} interval between {} and {} is still active, ignoring",
new Object[] {interval, start, end});
continue;
}
// Track the oldest interval end, this ensures that nothing is missed
if (end.isBefore(cleanUnclosedEnd)) {
cleanUnclosedEnd = end;
}
logger.debug(
"Cleaning unclosed {} aggregations between {} and {}",
new Object[] {interval, start, end});
for (final IntervalAwarePortalEventAggregator<PortalEvent> portalEventAggregator :
intervalAwarePortalEventAggregators) {
checkShutdown();
final Class<? extends IPortalEventAggregator<?>> aggregatorType =
getClass(portalEventAggregator);
// Get aggregator specific interval info config
final AggregatedIntervalConfig aggregatorIntervalConfig =
intervalsForAggregatorHelper.getAggregatorIntervalConfig(aggregatorType);
// If the aggregator is being used for the specified interval call
// cleanUnclosedAggregations
if (aggregatorIntervalConfig.isIncluded(interval)) {
closedAggregations +=
portalEventAggregator.cleanUnclosedAggregations(start, end, interval);
}
}
cleanedIntervals++;
}
// Set the next interval to the end date from the last aggregation run
nextIntervalDate = cleanUnclosedEnd;
logger.debug(
"Closed {} aggregations across {} interval before {} with goal of {}",
new Object[] {
closedAggregations, cleanedIntervals, cleanUnclosedEnd, lastAggregatedDate
});
// Loop until either the batchSize of cleaned aggregations has been reached or no
// aggregation work is done
} while (closedAggregations <= cleanUnclosedAggregationsBatchSize
&& cleanedIntervals <= cleanUnclosedIntervalsBatchSize
&& cleanUnclosedEnd.isBefore(lastAggregatedDate));
} finally {
currentThread.setName(currentName);
}
// Update the status object and store it
cleanUnclosedStatus.setLastEventDate(cleanUnclosedEnd);
cleanUnclosedStatus.setLastEnd(new DateTime());
eventAggregationManagementDao.updateEventAggregatorStatus(cleanUnclosedStatus);
return new EventProcessingResult(
closedAggregations,
lastCleanUnclosedDate,
lastAggregatedDate,
!cleanUnclosedEnd.isBefore(lastAggregatedDate));
}
