コード例 #1
0
  private int aggregateTimelineCandidates(
      final List<TimelineChunk> timelineChunkCandidates,
      final int aggregationLevel,
      final int chunksToAggregate) {
    final TimelineChunk firstCandidate = timelineChunkCandidates.get(0);
    final int sourceId = firstCandidate.getSourceId();
    final int metricId = firstCandidate.getMetricId();
    log.debug(
        "For sourceId {}, metricId {}, looking to aggregate {} candidates in {} chunks",
        new Object[] {sourceId, metricId, timelineChunkCandidates.size(), chunksToAggregate});
    int aggregatesCreated = 0;
    int chunkIndex = 0;
    while (timelineChunkCandidates.size() >= chunkIndex + chunksToAggregate) {
      final List<TimelineChunk> chunkCandidates =
          timelineChunkCandidates.subList(chunkIndex, chunkIndex + chunksToAggregate);
      chunkIndex += chunksToAggregate;
      timelineChunksCombined.addAndGet(chunksToAggregate);
      try {
        aggregateHostSampleChunks(chunkCandidates, aggregationLevel);
      } catch (IOException e) {
        log.error(
            String.format(
                "IOException aggregating {} chunks, sourceId %s, metricId %s, looking to aggregate %s candidates in %s chunks",
                new Object[] {
                  firstCandidate.getSourceId(),
                  firstCandidate.getMetricId(),
                  timelineChunkCandidates.size(),
                  chunksToAggregate
                }),
            e);
      }
      aggregatesCreated++;
    }

    return aggregatesCreated;
  }
コード例 #2
0
  /**
   * The sequence of events is:
   *
   * <ul>
   *   <li>Build the aggregated TimelineChunk object, and save it, setting not_valid to true, and
   *       aggregation_level to 1. This means that it won't be noticed by any of the dashboard
   *       queries. The save operation returns the new timeline_times_id
   *   <li>Then, in a single transaction, update the aggregated TimelineChunk object to have
   *       not_valid = 0, and also delete the TimelineChunk objects that were the basis of the
   *       aggregation, and flush any TimelineChunks that happen to be in the cache.
   *       <p>
   *
   * @param timelineChunks the TimelineChunks to be aggregated
   */
  private void aggregateHostSampleChunks(
      final List<TimelineChunk> timelineChunks, final int aggregationLevel) throws IOException {
    final TimelineChunk firstTimesChunk = timelineChunks.get(0);
    final TimelineChunk lastTimesChunk = timelineChunks.get(timelineChunks.size() - 1);
    final int chunkCount = timelineChunks.size();
    final int sourceId = firstTimesChunk.getSourceId();
    final DateTime startTime = firstTimesChunk.getStartTime();
    final DateTime endTime = lastTimesChunk.getEndTime();
    final List<byte[]> timeParts = new ArrayList<byte[]>(chunkCount);
    try {
      final List<byte[]> sampleParts = new ArrayList<byte[]>(chunkCount);
      final List<Long> timelineChunkIds = new ArrayList<Long>(chunkCount);
      int sampleCount = 0;
      for (final TimelineChunk timelineChunk : timelineChunks) {
        timeParts.add(timelineChunk.getTimeBytesAndSampleBytes().getTimeBytes());
        sampleParts.add(timelineChunk.getTimeBytesAndSampleBytes().getSampleBytes());
        sampleCount += timelineChunk.getSampleCount();
        timelineChunkIds.add(timelineChunk.getChunkId());
      }
      final byte[] combinedTimeBytes = timelineCoder.combineTimelines(timeParts, sampleCount);
      final byte[] combinedSampleBytes = sampleCoder.combineSampleBytes(sampleParts);
      final int timeBytesLength = combinedTimeBytes.length;
      final int totalSize = 4 + timeBytesLength + combinedSampleBytes.length;
      log.debug(
          "For sourceId {}, aggregationLevel {}, aggregating {} timelines ({} bytes, {} samples): {}",
          new Object[] {
            firstTimesChunk.getSourceId(),
            firstTimesChunk.getAggregationLevel(),
            timelineChunks.size(),
            totalSize,
            sampleCount
          });
      timelineChunksBytesCreated.addAndGet(totalSize);
      final int totalSampleCount = sampleCount;
      final TimelineChunk chunk =
          new TimelineChunk(
              0,
              sourceId,
              firstTimesChunk.getMetricId(),
              startTime,
              endTime,
              combinedTimeBytes,
              combinedSampleBytes,
              totalSampleCount,
              aggregationLevel + 1,
              false,
              false);
      chunksToWrite.add(chunk);
      chunkIdsToInvalidateOrDelete.addAll(timelineChunkIds);
      timelineChunksQueuedForCreation.incrementAndGet();

      if (chunkIdsToInvalidateOrDelete.size() >= config.getMaxChunkIdsToInvalidateOrDelete()) {
        performWrites();
      }
    } catch (Exception e) {
      log.error(
          String.format(
              "Exception aggregating level %d, sourceId %d, metricId %d, startTime %s, endTime %s",
              aggregationLevel, sourceId, firstTimesChunk.getMetricId(), startTime, endTime),
          e);
    }
  }