public void removeSensorData(String stationName, String sensorName, int start, int end) {
SensorMeta sensorMeta = getSensorMeta(stationName, sensorName, false);
if (sensorMeta == null) {
log.info("no sensor: " + stationName + " " + sensorName + " -> nothing removed");
return;
}
BTreeMap<Integer, ChunkMeta> chunkMetaMap = getSensorChunkMetaMap(sensorMeta);
BTreeMap<Integer, Chunk> chunkMap = getSensorChunkMap(sensorMeta);
ChunkMeta[] allChunkMetas = chunkMetaMap.values().toArray(new ChunkMeta[0]); // proxy
for (ChunkMeta chunkMeta : allChunkMetas) {
if (start <= chunkMeta.firstTimestamp
&& chunkMeta.lastTimestamp <= end) { // remove full chunk
removeChunk(chunkMetaMap, chunkMap, chunkMeta);
} else if (start <= chunkMeta.lastTimestamp
&& chunkMeta.firstTimestamp <= end) { // partial data chunk remove
Chunk oldChunk = chunkMap.get(chunkMeta.firstTimestamp);
Chunk newChunk = removeIntervalInChunk(oldChunk, start, end);
if (newChunk != null) {
removeChunk(chunkMetaMap, chunkMap, chunkMeta);
insertChunk(chunkMetaMap, chunkMap, newChunk);
log.trace("chunk part reinserted");
} else {
log.error("chunk not removed (internal error): " + chunkMeta);
}
}
}
}
private SensorMeta getSensorMeta(
StationMeta stationMeta, String sensorName, boolean createIfNotExists) {
throwNull(stationMeta);
throwNull(sensorName);
BTreeMap<String, SensorMeta> sensorMap = db.getTreeMap(stationMeta.db_name_sensor_map);
SensorMeta sensorMeta = sensorMap.get(sensorName);
if (sensorMeta == null && createIfNotExists) {
sensorMeta = new SensorMeta(stationMeta.stationName, sensorName);
db.checkNameNotExists(sensorMeta.db_name_sensor_chunk_map);
db.createTreeMap(sensorMeta.db_name_sensor_chunk_map)
.keySerializer(BTreeKeySerializer.ZERO_OR_POSITIVE_INT)
// .valueSerializer(Chunk.DELTA_TIME_DELTA_DELTA_VALUE_INT_QUANTIZED_SERIALIZER)
// .valueSerializer(Chunk.SNAPPY_DELTA_TIME_DELTA_DELTA_VALUE_INT_QUANTIZED_SERIALIZER)
.valueSerializer(ChunkSerializer.DEFAULT)
// .valuesOutsideNodesEnable() // !!! does not work: growing database
// .
.makeOrGet();
db.checkNameNotExists(sensorMeta.db_name_sensor_chunkmeta_map);
db.createTreeMap(sensorMeta.db_name_sensor_chunkmeta_map)
.keySerializer(BTreeKeySerializer.ZERO_OR_POSITIVE_INT)
.valueSerializer(ChunkMeta.SERIALIZER)
.makeOrGet();
sensorMap.put(sensorName, sensorMeta);
}
if (sensorMeta == null) {
// new Throwable().printStackTrace();
log.warn("no sensor: " + sensorName + " in station: " + stationMeta.stationName);
}
return sensorMeta;
}
private StationMeta getStationMeta(String stationName, boolean createIfNotExists) {
throwNull(stationName);
StationMeta stationMeta = stationMetaMap.get(stationName);
if (stationMeta == null && createIfNotExists) {
stationMeta = new StationMeta(stationName);
db.checkNameNotExists(stationMeta.db_name_sensor_map);
db.createTreeMap(stationMeta.db_name_sensor_map)
.keySerializer(BTreeKeySerializer.STRING)
.valueSerializer(SensorMeta.SERIALIZER)
.makeOrGet();
db.checkNameNotExists(stationMeta.db_name_sensor_time_series_mask_map);
db.createTreeMap(stationMeta.db_name_sensor_time_series_mask_map)
.keySerializer(BTreeKeySerializer.STRING)
.valueSerializer(TimeSeriesMask.SERIALIZER)
.makeOrGet();
stationMetaMap.put(stationName, stationMeta);
}
if (stationMeta == null) {
// new Throwable().printStackTrace();
log.warn("no station: " + stationName);
}
return stationMeta;
}
/**
* Check if data of sensor exits in station. If station does not exist return false.
*
* @param stationID
* @param sensorName
* @return
*/
public boolean existSensor(String stationID, String sensorName) {
StationMeta stationMeta = stationMetaMap.get(stationID);
if (stationMeta == null) {
return false;
}
return getSensorMap(stationMeta).containsKey(sensorName);
}
public NavigableSet<String> getSensorNames(String stationName) {
throwNull(stationName);
StationMeta stationMeta = stationMetaMap.get(stationName);
if (stationMeta == null) {
log.warn("no station: " + stationName);
return new TreeSet<String>();
}
BTreeMap<String, SensorMeta> sensorMap = db.getTreeMap(stationMeta.db_name_sensor_map);
return sensorMap.keySet();
}
private void insertIntoOneChunk(
BTreeMap<Integer, ChunkMeta> chunkMetaMap,
BTreeMap<Integer, Chunk> chunkMap,
ArrayList<DataEntry> entryList) {
// int timestamp_chunk = TimeConverter.roundLowerYear(entryList.get(0).timestamp);
int timestamp_next_year = TimeUtil.roundNextYear(entryList.get(0).timestamp);
if (timestamp_next_year <= entryList.get(entryList.size() - 1).timestamp) {
throw new RuntimeException("data of more than one chunk");
}
// ChunkMeta oldChunkMeta = chunkMetaMap.get(timestamp_chunk);
ChunkMeta oldChunkMeta = getChunkMeta(chunkMetaMap, entryList.get(0).timestamp);
if (oldChunkMeta == null) {
insertChunk(chunkMetaMap, chunkMap, Chunk.of(entryList));
} else {
Chunk oldChunk = chunkMap.get(oldChunkMeta.firstTimestamp);
Iterator<DataEntry> oldIt = Arrays.stream(oldChunk.data).iterator();
Iterator<DataEntry> newIt = entryList.iterator();
ArrayList<DataEntry> resultList = new ArrayList<DataEntry>();
DataEntry old_curr = oldIt.hasNext() ? oldIt.next() : null;
DataEntry new_curr = newIt.hasNext() ? newIt.next() : null;
while (old_curr != null || new_curr != null) {
if (old_curr != null) {
if (new_curr != null) {
if (old_curr.timestamp == new_curr.timestamp) { // overwrite old data with new data
resultList.add(new_curr);
old_curr = oldIt.hasNext() ? oldIt.next() : null;
new_curr = newIt.hasNext() ? newIt.next() : null;
} else if (old_curr.timestamp < new_curr.timestamp) {
resultList.add(old_curr);
old_curr = oldIt.hasNext() ? oldIt.next() : null;
} else {
resultList.add(new_curr);
new_curr = newIt.hasNext() ? newIt.next() : null;
}
} else {
resultList.add(old_curr);
old_curr = oldIt.hasNext() ? oldIt.next() : null;
}
} else {
resultList.add(new_curr);
new_curr = newIt.hasNext() ? newIt.next() : null;
}
}
removeChunk(chunkMetaMap, chunkMap, oldChunkMeta);
insertChunk(chunkMetaMap, chunkMap, Chunk.of(resultList));
}
}
/**
* get meta, that is correct target of timestamp if present
*
* @param timestamp
*/
private ChunkMeta getChunkMeta(BTreeMap<Integer, ChunkMeta> chunkMetaMap, int timestamp) {
int timestamp_year = TimeUtil.roundLowerYear(timestamp);
int timestamp_next_year = TimeUtil.roundNextYear(timestamp);
Integer key = chunkMetaMap.ceilingKey(timestamp_year);
if (key == null) {
return null;
}
if (timestamp_next_year <= key) {
return null;
}
ChunkMeta chunkMeta = chunkMetaMap.get(key);
throwNull(chunkMeta);
return chunkMeta;
}
public TimeSeriesMask getSensorTimeSeriesMask(
StationMeta stationMeta, String sensorName, boolean createIfNotExists) {
throwNull(stationMeta);
throwNull(sensorName);
BTreeMap<String, TimeSeriesMask> maskMap =
db.getTreeMap(stationMeta.db_name_sensor_time_series_mask_map);
TimeSeriesMask mask = maskMap.get(sensorName);
if (mask == null && createIfNotExists) {
mask = new TimeSeriesMask();
maskMap.put(sensorName, mask);
}
if (mask == null) {
// log.info("no time series mask: "+sensorName+" in station: "+stationMeta.stationName);
}
return mask;
}