public void removeSensorData(String stationName, String sensorName, int start, int end) { SensorMeta sensorMeta = getSensorMeta(stationName, sensorName, false); if (sensorMeta == null) { log.info("no sensor: " + stationName + " " + sensorName + " -> nothing removed"); return; } BTreeMap<Integer, ChunkMeta> chunkMetaMap = getSensorChunkMetaMap(sensorMeta); BTreeMap<Integer, Chunk> chunkMap = getSensorChunkMap(sensorMeta); ChunkMeta[] allChunkMetas = chunkMetaMap.values().toArray(new ChunkMeta[0]); // proxy for (ChunkMeta chunkMeta : allChunkMetas) { if (start <= chunkMeta.firstTimestamp && chunkMeta.lastTimestamp <= end) { // remove full chunk removeChunk(chunkMetaMap, chunkMap, chunkMeta); } else if (start <= chunkMeta.lastTimestamp && chunkMeta.firstTimestamp <= end) { // partial data chunk remove Chunk oldChunk = chunkMap.get(chunkMeta.firstTimestamp); Chunk newChunk = removeIntervalInChunk(oldChunk, start, end); if (newChunk != null) { removeChunk(chunkMetaMap, chunkMap, chunkMeta); insertChunk(chunkMetaMap, chunkMap, newChunk); log.trace("chunk part reinserted"); } else { log.error("chunk not removed (internal error): " + chunkMeta); } } } }
private SensorMeta getSensorMeta( StationMeta stationMeta, String sensorName, boolean createIfNotExists) { throwNull(stationMeta); throwNull(sensorName); BTreeMap<String, SensorMeta> sensorMap = db.getTreeMap(stationMeta.db_name_sensor_map); SensorMeta sensorMeta = sensorMap.get(sensorName); if (sensorMeta == null && createIfNotExists) { sensorMeta = new SensorMeta(stationMeta.stationName, sensorName); db.checkNameNotExists(sensorMeta.db_name_sensor_chunk_map); db.createTreeMap(sensorMeta.db_name_sensor_chunk_map) .keySerializer(BTreeKeySerializer.ZERO_OR_POSITIVE_INT) // .valueSerializer(Chunk.DELTA_TIME_DELTA_DELTA_VALUE_INT_QUANTIZED_SERIALIZER) // .valueSerializer(Chunk.SNAPPY_DELTA_TIME_DELTA_DELTA_VALUE_INT_QUANTIZED_SERIALIZER) .valueSerializer(ChunkSerializer.DEFAULT) // .valuesOutsideNodesEnable() // !!! does not work: growing database // . .makeOrGet(); db.checkNameNotExists(sensorMeta.db_name_sensor_chunkmeta_map); db.createTreeMap(sensorMeta.db_name_sensor_chunkmeta_map) .keySerializer(BTreeKeySerializer.ZERO_OR_POSITIVE_INT) .valueSerializer(ChunkMeta.SERIALIZER) .makeOrGet(); sensorMap.put(sensorName, sensorMeta); } if (sensorMeta == null) { // new Throwable().printStackTrace(); log.warn("no sensor: " + sensorName + " in station: " + stationMeta.stationName); } return sensorMeta; }
private StationMeta getStationMeta(String stationName, boolean createIfNotExists) { throwNull(stationName); StationMeta stationMeta = stationMetaMap.get(stationName); if (stationMeta == null && createIfNotExists) { stationMeta = new StationMeta(stationName); db.checkNameNotExists(stationMeta.db_name_sensor_map); db.createTreeMap(stationMeta.db_name_sensor_map) .keySerializer(BTreeKeySerializer.STRING) .valueSerializer(SensorMeta.SERIALIZER) .makeOrGet(); db.checkNameNotExists(stationMeta.db_name_sensor_time_series_mask_map); db.createTreeMap(stationMeta.db_name_sensor_time_series_mask_map) .keySerializer(BTreeKeySerializer.STRING) .valueSerializer(TimeSeriesMask.SERIALIZER) .makeOrGet(); stationMetaMap.put(stationName, stationMeta); } if (stationMeta == null) { // new Throwable().printStackTrace(); log.warn("no station: " + stationName); } return stationMeta; }
/** * Check if data of sensor exits in station. If station does not exist return false. * * @param stationID * @param sensorName * @return */ public boolean existSensor(String stationID, String sensorName) { StationMeta stationMeta = stationMetaMap.get(stationID); if (stationMeta == null) { return false; } return getSensorMap(stationMeta).containsKey(sensorName); }
public NavigableSet<String> getSensorNames(String stationName) { throwNull(stationName); StationMeta stationMeta = stationMetaMap.get(stationName); if (stationMeta == null) { log.warn("no station: " + stationName); return new TreeSet<String>(); } BTreeMap<String, SensorMeta> sensorMap = db.getTreeMap(stationMeta.db_name_sensor_map); return sensorMap.keySet(); }
private void insertIntoOneChunk( BTreeMap<Integer, ChunkMeta> chunkMetaMap, BTreeMap<Integer, Chunk> chunkMap, ArrayList<DataEntry> entryList) { // int timestamp_chunk = TimeConverter.roundLowerYear(entryList.get(0).timestamp); int timestamp_next_year = TimeUtil.roundNextYear(entryList.get(0).timestamp); if (timestamp_next_year <= entryList.get(entryList.size() - 1).timestamp) { throw new RuntimeException("data of more than one chunk"); } // ChunkMeta oldChunkMeta = chunkMetaMap.get(timestamp_chunk); ChunkMeta oldChunkMeta = getChunkMeta(chunkMetaMap, entryList.get(0).timestamp); if (oldChunkMeta == null) { insertChunk(chunkMetaMap, chunkMap, Chunk.of(entryList)); } else { Chunk oldChunk = chunkMap.get(oldChunkMeta.firstTimestamp); Iterator<DataEntry> oldIt = Arrays.stream(oldChunk.data).iterator(); Iterator<DataEntry> newIt = entryList.iterator(); ArrayList<DataEntry> resultList = new ArrayList<DataEntry>(); DataEntry old_curr = oldIt.hasNext() ? oldIt.next() : null; DataEntry new_curr = newIt.hasNext() ? newIt.next() : null; while (old_curr != null || new_curr != null) { if (old_curr != null) { if (new_curr != null) { if (old_curr.timestamp == new_curr.timestamp) { // overwrite old data with new data resultList.add(new_curr); old_curr = oldIt.hasNext() ? oldIt.next() : null; new_curr = newIt.hasNext() ? newIt.next() : null; } else if (old_curr.timestamp < new_curr.timestamp) { resultList.add(old_curr); old_curr = oldIt.hasNext() ? oldIt.next() : null; } else { resultList.add(new_curr); new_curr = newIt.hasNext() ? newIt.next() : null; } } else { resultList.add(old_curr); old_curr = oldIt.hasNext() ? oldIt.next() : null; } } else { resultList.add(new_curr); new_curr = newIt.hasNext() ? newIt.next() : null; } } removeChunk(chunkMetaMap, chunkMap, oldChunkMeta); insertChunk(chunkMetaMap, chunkMap, Chunk.of(resultList)); } }
/** * get meta, that is correct target of timestamp if present * * @param timestamp */ private ChunkMeta getChunkMeta(BTreeMap<Integer, ChunkMeta> chunkMetaMap, int timestamp) { int timestamp_year = TimeUtil.roundLowerYear(timestamp); int timestamp_next_year = TimeUtil.roundNextYear(timestamp); Integer key = chunkMetaMap.ceilingKey(timestamp_year); if (key == null) { return null; } if (timestamp_next_year <= key) { return null; } ChunkMeta chunkMeta = chunkMetaMap.get(key); throwNull(chunkMeta); return chunkMeta; }
public TimeSeriesMask getSensorTimeSeriesMask( StationMeta stationMeta, String sensorName, boolean createIfNotExists) { throwNull(stationMeta); throwNull(sensorName); BTreeMap<String, TimeSeriesMask> maskMap = db.getTreeMap(stationMeta.db_name_sensor_time_series_mask_map); TimeSeriesMask mask = maskMap.get(sensorName); if (mask == null && createIfNotExists) { mask = new TimeSeriesMask(); maskMap.put(sensorName, mask); } if (mask == null) { // log.info("no time series mask: "+sensorName+" in station: "+stationMeta.stationName); } return mask; }