コード例 #1
0
 /**
  * get meta, that is correct target of timestamp if present
  *
  * @param timestamp
  */
 private ChunkMeta getChunkMeta(BTreeMap<Integer, ChunkMeta> chunkMetaMap, int timestamp) {
   int timestamp_year = TimeUtil.roundLowerYear(timestamp);
   int timestamp_next_year = TimeUtil.roundNextYear(timestamp);
   Integer key = chunkMetaMap.ceilingKey(timestamp_year);
   if (key == null) {
     return null;
   }
   if (timestamp_next_year <= key) {
     return null;
   }
   ChunkMeta chunkMeta = chunkMetaMap.get(key);
   throwNull(chunkMeta);
   return chunkMeta;
 }
コード例 #2
0
  private void insertIntoOneChunk(
      BTreeMap<Integer, ChunkMeta> chunkMetaMap,
      BTreeMap<Integer, Chunk> chunkMap,
      ArrayList<DataEntry> entryList) {
    // int timestamp_chunk = TimeConverter.roundLowerYear(entryList.get(0).timestamp);
    int timestamp_next_year = TimeUtil.roundNextYear(entryList.get(0).timestamp);
    if (timestamp_next_year <= entryList.get(entryList.size() - 1).timestamp) {
      throw new RuntimeException("data of more than one chunk");
    }
    // ChunkMeta oldChunkMeta = chunkMetaMap.get(timestamp_chunk);
    ChunkMeta oldChunkMeta = getChunkMeta(chunkMetaMap, entryList.get(0).timestamp);
    if (oldChunkMeta == null) {
      insertChunk(chunkMetaMap, chunkMap, Chunk.of(entryList));
    } else {
      Chunk oldChunk = chunkMap.get(oldChunkMeta.firstTimestamp);
      Iterator<DataEntry> oldIt = Arrays.stream(oldChunk.data).iterator();
      Iterator<DataEntry> newIt = entryList.iterator();
      ArrayList<DataEntry> resultList = new ArrayList<DataEntry>();

      DataEntry old_curr = oldIt.hasNext() ? oldIt.next() : null;
      DataEntry new_curr = newIt.hasNext() ? newIt.next() : null;

      while (old_curr != null || new_curr != null) {
        if (old_curr != null) {
          if (new_curr != null) {
            if (old_curr.timestamp == new_curr.timestamp) { // overwrite old data with new data
              resultList.add(new_curr);
              old_curr = oldIt.hasNext() ? oldIt.next() : null;
              new_curr = newIt.hasNext() ? newIt.next() : null;
            } else if (old_curr.timestamp < new_curr.timestamp) {
              resultList.add(old_curr);
              old_curr = oldIt.hasNext() ? oldIt.next() : null;
            } else {
              resultList.add(new_curr);
              new_curr = newIt.hasNext() ? newIt.next() : null;
            }
          } else {
            resultList.add(old_curr);
            old_curr = oldIt.hasNext() ? oldIt.next() : null;
          }
        } else {
          resultList.add(new_curr);
          new_curr = newIt.hasNext() ? newIt.next() : null;
        }
      }

      removeChunk(chunkMetaMap, chunkMap, oldChunkMeta);
      insertChunk(chunkMetaMap, chunkMap, Chunk.of(resultList));
    }
  }
コード例 #3
0
  public void insertSensorData(String stationName, String sensorName, DataEntry[] data) {
    // log.info("streamDB insert data "+stationName+" "+sensorName+" "+data.length);
    throwNull(stationName);
    throwNull(sensorName);
    throwNull(data);
    if (data.length == 0) {
      log.warn("no data to insert");
      return;
    }
    SensorMeta sensorMeta = getSensorMeta(stationName, sensorName, true);
    BTreeMap<Integer, ChunkMeta> chunkMetaMap = getSensorChunkMetaMap(sensorMeta);
    BTreeMap<Integer, Chunk> chunkMap = getSensorChunkMap(sensorMeta);

    int timestamp_next_year = Integer.MIN_VALUE;
    ArrayList<DataEntry> entryList = new ArrayList<DataEntry>(data.length);
    int prevTimestamp = -1;
    for (DataEntry entry : data) {
      if (entry.timestamp <= prevTimestamp) {
        throw new RuntimeException(
            "not ordered timestamps "
                + TimeUtil.oleMinutesToText(prevTimestamp)
                + "  "
                + TimeUtil.oleMinutesToText(entry.timestamp)
                + "   "
                + entry.value
                + "  "
                + stationName
                + "/"
                + sensorName);
      }
      if (entry.timestamp < timestamp_next_year) {
        entryList.add(entry);
      } else {
        if (!entryList.isEmpty()) {
          insertIntoOneChunk(chunkMetaMap, chunkMap, entryList);
        }
        timestamp_next_year = TimeUtil.roundNextYear(entry.timestamp);
        entryList.clear();
        entryList.add(entry);
      }
      prevTimestamp = entry.timestamp;
    }
    if (!entryList.isEmpty()) {
      insertIntoOneChunk(chunkMetaMap, chunkMap, entryList);
    }
  }