private EventProcessingResult doAggregateRawEventsInternal() {
if (!this.clusterLockService.isLockOwner(AGGREGATION_LOCK_NAME)) {
throw new IllegalStateException(
"The cluster lock "
+ AGGREGATION_LOCK_NAME
+ " must be owned by the current thread and server");
}
if (!this.portalEventDimensionPopulator.isCheckedDimensions()) {
// First time aggregation has happened, run populateDimensions to ensure enough dimension data
// exists
final boolean populatedDimensions = this.portalEventAggregationManager.populateDimensions();
if (!populatedDimensions) {
this.logger.warn(
"Aborting raw event aggregation, populateDimensions returned false so the state of date/time dimensions is unknown");
return null;
}
}
// Flush any dimension creation before aggregation
final EntityManager entityManager = this.getEntityManager();
entityManager.flush();
entityManager.setFlushMode(FlushModeType.COMMIT);
final IEventAggregatorStatus eventAggregatorStatus =
eventAggregationManagementDao.getEventAggregatorStatus(ProcessingType.AGGREGATION, true);
// Update status with current server name
final String serverName = this.portalInfoProvider.getUniqueServerName();
final String previousServerName = eventAggregatorStatus.getServerName();
if (previousServerName != null && !serverName.equals(previousServerName)) {
this.logger.debug(
"Last aggregation run on {} clearing all aggregation caches", previousServerName);
final Session session = getEntityManager().unwrap(Session.class);
final Cache cache = session.getSessionFactory().getCache();
cache.evictEntityRegions();
}
eventAggregatorStatus.setServerName(serverName);
// Calculate date range for aggregation
DateTime lastAggregated = eventAggregatorStatus.getLastEventDate();
if (lastAggregated == null) {
lastAggregated = portalEventDao.getOldestPortalEventTimestamp();
// No portal events to aggregate, skip aggregation
if (lastAggregated == null) {
return new EventProcessingResult(0, null, null, true);
}
// First time aggregation has run, initialize the CLEAN_UNCLOSED status to save catch-up time
final IEventAggregatorStatus cleanUnclosedStatus =
eventAggregationManagementDao.getEventAggregatorStatus(
ProcessingType.CLEAN_UNCLOSED, true);
AggregationIntervalInfo oldestMinuteInterval =
this.intervalHelper.getIntervalInfo(AggregationInterval.MINUTE, lastAggregated);
cleanUnclosedStatus.setLastEventDate(oldestMinuteInterval.getStart().minusMinutes(1));
eventAggregationManagementDao.updateEventAggregatorStatus(cleanUnclosedStatus);
}
final DateTime newestEventTime =
DateTime.now().minus(this.aggregationDelay).secondOfMinute().roundFloorCopy();
final Thread currentThread = Thread.currentThread();
final String currentName = currentThread.getName();
final MutableInt events = new MutableInt();
final MutableObject lastEventDate = new MutableObject(newestEventTime);
boolean complete;
try {
currentThread.setName(currentName + "-" + lastAggregated + "_" + newestEventTime);
logger.debug(
"Starting aggregation of events between {} (inc) and {} (exc)",
lastAggregated,
newestEventTime);
// Do aggregation, capturing the start and end dates
eventAggregatorStatus.setLastStart(DateTime.now());
complete =
portalEventDao.aggregatePortalEvents(
lastAggregated,
newestEventTime,
this.eventAggregationBatchSize,
new AggregateEventsHandler(events, lastEventDate, eventAggregatorStatus));
eventAggregatorStatus.setLastEventDate((DateTime) lastEventDate.getValue());
eventAggregatorStatus.setLastEnd(DateTime.now());
} finally {
currentThread.setName(currentName);
}
// Store the results of the aggregation
eventAggregationManagementDao.updateEventAggregatorStatus(eventAggregatorStatus);
complete =
complete
&& (this.eventAggregationBatchSize <= 0
|| events.intValue() < this.eventAggregationBatchSize);
return new EventProcessingResult(
events.intValue(), lastAggregated, eventAggregatorStatus.getLastEventDate(), complete);
}
/**
* Creates a storage form the uploaded local storage directory.
*
* @param localStorageData Local storage information.
* @throws IOException If {@link IOException} occurs.
* @throws BusinessException If there is not enough space for the unpacking the storage.
* @throws SerializationException If serialization fails.
*/
public void createStorageFromUploadedDir(final IStorageData localStorageData)
throws IOException, BusinessException, SerializationException {
long storageBytesLeft = getBytesHardDriveOccupancyLeft();
if (localStorageData.getDiskSize() > storageBytesLeft) {
throw new BusinessException(
"Create the uploaded storage " + localStorageData + ".",
StorageErrorCodeEnum.LOW_DISK_SPACE);
}
Path uploadPath = Paths.get(this.getStorageUploadsFolder());
if (Files.notExists(uploadPath)) {
throw new IOException(
"Can not perform storage unpacking. The main upload path "
+ uploadPath.toString()
+ " does not exist.");
} else {
final MutableObject storageUploadPath = new MutableObject();
final MutableObject uploadedStorageData = new MutableObject();
final ISerializer serializer = getSerializationManagerProvider().createSerializer();
Files.walkFileTree(
uploadPath,
new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
throws IOException {
// skip all other files, search for the local data
if (!file.toString()
.endsWith(
localStorageData.getId()
+ StorageFileType.LOCAL_STORAGE_FILE.getExtension())) {
return FileVisitResult.CONTINUE;
}
// when found confirm it is the one we wanted to upload
InputStream inputStream = null;
Input input = null;
try {
inputStream = Files.newInputStream(file, StandardOpenOption.READ);
input = new Input(inputStream);
Object deserialized = serializer.deserialize(input);
if (Objects.equals(deserialized, localStorageData)) {
uploadedStorageData.setValue(new StorageData(localStorageData));
storageUploadPath.setValue(file.toAbsolutePath().getParent());
return FileVisitResult.TERMINATE;
}
} catch (SerializationException e) {
log.warn("Error de-serializing local storage file.", e);
} finally {
if (null != input) {
input.close();
}
}
return FileVisitResult.CONTINUE;
}
});
// do the rest out of the file walk
Path parentDir = (Path) storageUploadPath.getValue();
StorageData storageData = (StorageData) uploadedStorageData.getValue();
if (null != storageData && null != parentDir) {
Path storageDir = getStoragePath(storageData);
if (existingStoragesSet.add(storageData)) {
if (Files.notExists(storageDir)) {
printStorageCmrVersionWarn(storageData);
Files.walkFileTree(parentDir, new CopyMoveFileVisitor(parentDir, storageDir, true));
Path localInformation =
getStoragePath(storageData)
.resolve(
storageData.getId() + StorageFileType.LOCAL_STORAGE_FILE.getExtension());
Files.deleteIfExists(localInformation);
writeStorageDataToDisk(storageData);
} else {
throw new IOException("Directory to place uploaded storage already exists.");
}
} else {
log.info(
"Uploaded storage on path "
+ parentDir.toString()
+ " contains the storage that is already available on the CMR. Dir will be deleted.");
Files.walkFileTree(parentDir, new DeleteFileVisitor());
}
}
}
}
