private EventProcessingResult doAggregateRawEventsInternal() {
    if (!this.clusterLockService.isLockOwner(AGGREGATION_LOCK_NAME)) {
      throw new IllegalStateException(
          "The cluster lock "
              + AGGREGATION_LOCK_NAME
              + " must be owned by the current thread and server");
    }

    if (!this.portalEventDimensionPopulator.isCheckedDimensions()) {
      // First time aggregation has happened, run populateDimensions to ensure enough dimension data
      // exists
      final boolean populatedDimensions = this.portalEventAggregationManager.populateDimensions();
      if (!populatedDimensions) {
        this.logger.warn(
            "Aborting raw event aggregation, populateDimensions returned false so the state of date/time dimensions is unknown");
        return null;
      }
    }

    // Flush any dimension creation before aggregation
    final EntityManager entityManager = this.getEntityManager();
    entityManager.flush();
    entityManager.setFlushMode(FlushModeType.COMMIT);

    final IEventAggregatorStatus eventAggregatorStatus =
        eventAggregationManagementDao.getEventAggregatorStatus(ProcessingType.AGGREGATION, true);

    // Update status with current server name
    final String serverName = this.portalInfoProvider.getUniqueServerName();
    final String previousServerName = eventAggregatorStatus.getServerName();
    if (previousServerName != null && !serverName.equals(previousServerName)) {
      this.logger.debug(
          "Last aggregation run on {} clearing all aggregation caches", previousServerName);
      final Session session = getEntityManager().unwrap(Session.class);
      final Cache cache = session.getSessionFactory().getCache();
      cache.evictEntityRegions();
    }

    eventAggregatorStatus.setServerName(serverName);

    // Calculate date range for aggregation
    DateTime lastAggregated = eventAggregatorStatus.getLastEventDate();
    if (lastAggregated == null) {
      lastAggregated = portalEventDao.getOldestPortalEventTimestamp();

      // No portal events to aggregate, skip aggregation
      if (lastAggregated == null) {
        return new EventProcessingResult(0, null, null, true);
      }

      // First time aggregation has run, initialize the CLEAN_UNCLOSED status to save catch-up time
      final IEventAggregatorStatus cleanUnclosedStatus =
          eventAggregationManagementDao.getEventAggregatorStatus(
              ProcessingType.CLEAN_UNCLOSED, true);
      AggregationIntervalInfo oldestMinuteInterval =
          this.intervalHelper.getIntervalInfo(AggregationInterval.MINUTE, lastAggregated);
      cleanUnclosedStatus.setLastEventDate(oldestMinuteInterval.getStart().minusMinutes(1));
      eventAggregationManagementDao.updateEventAggregatorStatus(cleanUnclosedStatus);
    }

    final DateTime newestEventTime =
        DateTime.now().minus(this.aggregationDelay).secondOfMinute().roundFloorCopy();

    final Thread currentThread = Thread.currentThread();
    final String currentName = currentThread.getName();
    final MutableInt events = new MutableInt();
    final MutableObject lastEventDate = new MutableObject(newestEventTime);

    boolean complete;
    try {
      currentThread.setName(currentName + "-" + lastAggregated + "_" + newestEventTime);

      logger.debug(
          "Starting aggregation of events between {} (inc) and {} (exc)",
          lastAggregated,
          newestEventTime);

      // Do aggregation, capturing the start and end dates
      eventAggregatorStatus.setLastStart(DateTime.now());

      complete =
          portalEventDao.aggregatePortalEvents(
              lastAggregated,
              newestEventTime,
              this.eventAggregationBatchSize,
              new AggregateEventsHandler(events, lastEventDate, eventAggregatorStatus));

      eventAggregatorStatus.setLastEventDate((DateTime) lastEventDate.getValue());
      eventAggregatorStatus.setLastEnd(DateTime.now());
    } finally {
      currentThread.setName(currentName);
    }

    // Store the results of the aggregation
    eventAggregationManagementDao.updateEventAggregatorStatus(eventAggregatorStatus);

    complete =
        complete
            && (this.eventAggregationBatchSize <= 0
                || events.intValue() < this.eventAggregationBatchSize);
    return new EventProcessingResult(
        events.intValue(), lastAggregated, eventAggregatorStatus.getLastEventDate(), complete);
  }
Exemplo n.º 2
0
  /**
   * Creates a storage form the uploaded local storage directory.
   *
   * @param localStorageData Local storage information.
   * @throws IOException If {@link IOException} occurs.
   * @throws BusinessException If there is not enough space for the unpacking the storage.
   * @throws SerializationException If serialization fails.
   */
  public void createStorageFromUploadedDir(final IStorageData localStorageData)
      throws IOException, BusinessException, SerializationException {
    long storageBytesLeft = getBytesHardDriveOccupancyLeft();
    if (localStorageData.getDiskSize() > storageBytesLeft) {
      throw new BusinessException(
          "Create the uploaded storage " + localStorageData + ".",
          StorageErrorCodeEnum.LOW_DISK_SPACE);
    }

    Path uploadPath = Paths.get(this.getStorageUploadsFolder());
    if (Files.notExists(uploadPath)) {
      throw new IOException(
          "Can not perform storage unpacking. The main upload path "
              + uploadPath.toString()
              + " does not exist.");
    } else {
      final MutableObject storageUploadPath = new MutableObject();
      final MutableObject uploadedStorageData = new MutableObject();
      final ISerializer serializer = getSerializationManagerProvider().createSerializer();
      Files.walkFileTree(
          uploadPath,
          new SimpleFileVisitor<Path>() {
            @Override
            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
                throws IOException {
              // skip all other files, search for the local data
              if (!file.toString()
                  .endsWith(
                      localStorageData.getId()
                          + StorageFileType.LOCAL_STORAGE_FILE.getExtension())) {
                return FileVisitResult.CONTINUE;
              }

              // when found confirm it is the one we wanted to upload
              InputStream inputStream = null;
              Input input = null;
              try {
                inputStream = Files.newInputStream(file, StandardOpenOption.READ);
                input = new Input(inputStream);
                Object deserialized = serializer.deserialize(input);
                if (Objects.equals(deserialized, localStorageData)) {
                  uploadedStorageData.setValue(new StorageData(localStorageData));
                  storageUploadPath.setValue(file.toAbsolutePath().getParent());
                  return FileVisitResult.TERMINATE;
                }
              } catch (SerializationException e) {
                log.warn("Error de-serializing local storage file.", e);
              } finally {
                if (null != input) {
                  input.close();
                }
              }
              return FileVisitResult.CONTINUE;
            }
          });

      // do the rest out of the file walk
      Path parentDir = (Path) storageUploadPath.getValue();
      StorageData storageData = (StorageData) uploadedStorageData.getValue();
      if (null != storageData && null != parentDir) {
        Path storageDir = getStoragePath(storageData);
        if (existingStoragesSet.add(storageData)) {
          if (Files.notExists(storageDir)) {
            printStorageCmrVersionWarn(storageData);

            Files.walkFileTree(parentDir, new CopyMoveFileVisitor(parentDir, storageDir, true));
            Path localInformation =
                getStoragePath(storageData)
                    .resolve(
                        storageData.getId() + StorageFileType.LOCAL_STORAGE_FILE.getExtension());
            Files.deleteIfExists(localInformation);
            writeStorageDataToDisk(storageData);
          } else {
            throw new IOException("Directory to place uploaded storage already exists.");
          }
        } else {
          log.info(
              "Uploaded storage on path "
                  + parentDir.toString()
                  + " contains the storage that is already available on the CMR. Dir will be deleted.");
          Files.walkFileTree(parentDir, new DeleteFileVisitor());
        }
      }
    }
  }