private SensorMeta getSensorMeta(
StationMeta stationMeta, String sensorName, boolean createIfNotExists) {
throwNull(stationMeta);
throwNull(sensorName);
BTreeMap<String, SensorMeta> sensorMap = db.getTreeMap(stationMeta.db_name_sensor_map);
SensorMeta sensorMeta = sensorMap.get(sensorName);
if (sensorMeta == null && createIfNotExists) {
sensorMeta = new SensorMeta(stationMeta.stationName, sensorName);
db.checkNameNotExists(sensorMeta.db_name_sensor_chunk_map);
db.createTreeMap(sensorMeta.db_name_sensor_chunk_map)
.keySerializer(BTreeKeySerializer.ZERO_OR_POSITIVE_INT)
// .valueSerializer(Chunk.DELTA_TIME_DELTA_DELTA_VALUE_INT_QUANTIZED_SERIALIZER)
// .valueSerializer(Chunk.SNAPPY_DELTA_TIME_DELTA_DELTA_VALUE_INT_QUANTIZED_SERIALIZER)
.valueSerializer(ChunkSerializer.DEFAULT)
// .valuesOutsideNodesEnable() // !!! does not work: growing database
// .
.makeOrGet();
db.checkNameNotExists(sensorMeta.db_name_sensor_chunkmeta_map);
db.createTreeMap(sensorMeta.db_name_sensor_chunkmeta_map)
.keySerializer(BTreeKeySerializer.ZERO_OR_POSITIVE_INT)
.valueSerializer(ChunkMeta.SERIALIZER)
.makeOrGet();
sensorMap.put(sensorName, sensorMeta);
}
if (sensorMeta == null) {
// new Throwable().printStackTrace();
log.warn("no sensor: " + sensorName + " in station: " + stationMeta.stationName);
}
return sensorMeta;
}
public void removeSensorData(String stationName, String sensorName, int start, int end) {
SensorMeta sensorMeta = getSensorMeta(stationName, sensorName, false);
if (sensorMeta == null) {
log.info("no sensor: " + stationName + " " + sensorName + " -> nothing removed");
return;
}
BTreeMap<Integer, ChunkMeta> chunkMetaMap = getSensorChunkMetaMap(sensorMeta);
BTreeMap<Integer, Chunk> chunkMap = getSensorChunkMap(sensorMeta);
ChunkMeta[] allChunkMetas = chunkMetaMap.values().toArray(new ChunkMeta[0]); // proxy
for (ChunkMeta chunkMeta : allChunkMetas) {
if (start <= chunkMeta.firstTimestamp
&& chunkMeta.lastTimestamp <= end) { // remove full chunk
removeChunk(chunkMetaMap, chunkMap, chunkMeta);
} else if (start <= chunkMeta.lastTimestamp
&& chunkMeta.firstTimestamp <= end) { // partial data chunk remove
Chunk oldChunk = chunkMap.get(chunkMeta.firstTimestamp);
Chunk newChunk = removeIntervalInChunk(oldChunk, start, end);
if (newChunk != null) {
removeChunk(chunkMetaMap, chunkMap, chunkMeta);
insertChunk(chunkMetaMap, chunkMap, newChunk);
log.trace("chunk part reinserted");
} else {
log.error("chunk not removed (internal error): " + chunkMeta);
}
}
}
}
private StationMeta getStationMeta(String stationName, boolean createIfNotExists) {
throwNull(stationName);
StationMeta stationMeta = stationMetaMap.get(stationName);
if (stationMeta == null && createIfNotExists) {
stationMeta = new StationMeta(stationName);
db.checkNameNotExists(stationMeta.db_name_sensor_map);
db.createTreeMap(stationMeta.db_name_sensor_map)
.keySerializer(BTreeKeySerializer.STRING)
.valueSerializer(SensorMeta.SERIALIZER)
.makeOrGet();
db.checkNameNotExists(stationMeta.db_name_sensor_time_series_mask_map);
db.createTreeMap(stationMeta.db_name_sensor_time_series_mask_map)
.keySerializer(BTreeKeySerializer.STRING)
.valueSerializer(TimeSeriesMask.SERIALIZER)
.makeOrGet();
stationMetaMap.put(stationName, stationMeta);
}
if (stationMeta == null) {
// new Throwable().printStackTrace();
log.warn("no station: " + stationName);
}
return stationMeta;
}
public int[] getSensorTimeInterval(SensorMeta sensorMeta) {
throwNull(sensorMeta);
BTreeMap<Integer, ChunkMeta> chunkMetaMap = getSensorChunkMetaMap(sensorMeta);
if (chunkMetaMap.isEmpty()) {
return null;
}
return new int[] {chunkMetaMap.firstKey(), chunkMetaMap.lastEntry().getValue().lastTimestamp};
}
@Test
public void BTreeMap_snapshot() {
BTreeMap map =
DBMaker.newMemoryDB().transactionDisable().snapshotEnable().make().getTreeMap("aaa");
map.put("aa", "aa");
Map map2 = map.snapshot();
map.put("aa", "bb");
assertEquals("aa", map2.get("aa"));
}
public void setSensorTimeSeriesMask(
String stationName, String sensorName, TimeSeriesMask timeSeriesMask) {
throwNull(stationName);
throwNull(sensorName);
throwNull(timeSeriesMask);
StationMeta stationMeta = getStationMeta(stationName, true);
BTreeMap<String, TimeSeriesMask> maskMap =
db.getTreeMap(stationMeta.db_name_sensor_time_series_mask_map);
maskMap.put(sensorName, timeSeriesMask);
}
public void clearMaskOfStation(String stationName) {
StationMeta stationMeta = getStationMeta(stationName, false);
if (stationMeta == null) {
// log.warn("station not found "+stationName);
return;
}
BTreeMap<String, TimeSeriesMask> maskMap =
db.getTreeMap(stationMeta.db_name_sensor_time_series_mask_map);
maskMap.clear();
}
public NavigableSet<String> getSensorNames(String stationName) {
throwNull(stationName);
StationMeta stationMeta = stationMetaMap.get(stationName);
if (stationMeta == null) {
log.warn("no station: " + stationName);
return new TreeSet<String>();
}
BTreeMap<String, SensorMeta> sensorMap = db.getTreeMap(stationMeta.db_name_sensor_map);
return sensorMap.keySet();
}
private void insertChunk(
BTreeMap<Integer, ChunkMeta> chunkMetaMap, BTreeMap<Integer, Chunk> chunkMap, Chunk chunk) {
throwNull(chunkMetaMap);
throwNull(chunkMap);
throwNull(chunk);
chunkMap.put(chunk.data[0].timestamp, chunk);
chunkMetaMap.put(
chunk.data[0].timestamp,
new ChunkMeta(
chunk.data[0].timestamp,
chunk.data[chunk.data.length - 1].timestamp,
chunk.data.length));
}
/**
* get meta, that is correct target of timestamp if present
*
* @param timestamp
*/
private ChunkMeta getChunkMeta(BTreeMap<Integer, ChunkMeta> chunkMetaMap, int timestamp) {
int timestamp_year = TimeUtil.roundLowerYear(timestamp);
int timestamp_next_year = TimeUtil.roundNextYear(timestamp);
Integer key = chunkMetaMap.ceilingKey(timestamp_year);
if (key == null) {
return null;
}
if (timestamp_next_year <= key) {
return null;
}
ChunkMeta chunkMeta = chunkMetaMap.get(key);
throwNull(chunkMeta);
return chunkMeta;
}
private void removeChunk(
BTreeMap<Integer, ChunkMeta> chunkMetaMap,
BTreeMap<Integer, Chunk> chunkMap,
ChunkMeta oldChunkMeta) {
throwNull(chunkMetaMap);
throwNull(chunkMap);
throwNull(oldChunkMeta);
// log.info("remove chunk "+oldChunkMeta);
if (chunkMetaMap.remove(oldChunkMeta.firstTimestamp) == null) {
log.error("could not remove oldChunkMeta");
}
if (chunkMap.remove(oldChunkMeta.firstTimestamp) == null) {
log.error("could not remove old Chunk");
}
}
public TimeSeriesMask getSensorTimeSeriesMask(
StationMeta stationMeta, String sensorName, boolean createIfNotExists) {
throwNull(stationMeta);
throwNull(sensorName);
BTreeMap<String, TimeSeriesMask> maskMap =
db.getTreeMap(stationMeta.db_name_sensor_time_series_mask_map);
TimeSeriesMask mask = maskMap.get(sensorName);
if (mask == null && createIfNotExists) {
mask = new TimeSeriesMask();
maskMap.put(sensorName, mask);
}
if (mask == null) {
// log.info("no time series mask: "+sensorName+" in station: "+stationMeta.stationName);
}
return mask;
}
/**
* Check if data of sensor exits in station. If station does not exist return false.
*
* @param stationID
* @param sensorName
* @return
*/
public boolean existSensor(String stationID, String sensorName) {
StationMeta stationMeta = stationMetaMap.get(stationID);
if (stationMeta == null) {
return false;
}
return getSensorMap(stationMeta).containsKey(sensorName);
}
/** delete all content in db */
public void clear() {
for (StationMeta stationMeta : stationMetaMap.values()) {
BTreeMap<String, SensorMeta> sensorMap = getSensorMap(stationMeta);
for (SensorMeta sensorMeta : sensorMap.values()) {
/*BTreeMap<Integer, ChunkMeta> chunkMetaMap = getSensorChunkMetaMap(sensorMeta);
chunkMetaMap.clear();
BTreeMap<Integer, Chunk> chunkMap = getSensorChunkMap(sensorMeta);
chunkMap.clear();*/
db.delete(sensorMeta.db_name_sensor_chunkmeta_map);
db.delete(sensorMeta.db_name_sensor_chunk_map);
}
// sensorMap.clear();
db.delete(stationMeta.db_name_sensor_map);
db.delete(stationMeta.db_name_sensor_time_series_mask_map);
}
stationMetaMap.clear();
commit();
compact();
}
public void printStatistics() {
for (StationMeta stationMeta : stationMetaMap.values()) {
System.out.println(stationMeta.stationName);
for (SensorMeta sensorMeta : getSensorMap(stationMeta).values()) {
BTreeMap<Integer, ChunkMeta> sensorChunkMetaMap = getSensorChunkMetaMap(sensorMeta);
int entryCount = 0;
for (ChunkMeta chunkMeta : sensorChunkMetaMap.values()) {
entryCount += chunkMeta.entryCount;
}
BTreeMap<Integer, Chunk> sensorChunkMap = getSensorChunkMap(sensorMeta);
System.out.print(
sensorMeta.sensorName
+ " "
+ sensorChunkMetaMap.size()
+ ";"
+ sensorChunkMap.size()
+ ":"
+ entryCount
+ " ");
}
System.out.println();
}
for (String key : db.getAll().keySet()) {
System.out.println(key);
}
}
private void insertIntoOneChunk(
BTreeMap<Integer, ChunkMeta> chunkMetaMap,
BTreeMap<Integer, Chunk> chunkMap,
ArrayList<DataEntry> entryList) {
// int timestamp_chunk = TimeConverter.roundLowerYear(entryList.get(0).timestamp);
int timestamp_next_year = TimeUtil.roundNextYear(entryList.get(0).timestamp);
if (timestamp_next_year <= entryList.get(entryList.size() - 1).timestamp) {
throw new RuntimeException("data of more than one chunk");
}
// ChunkMeta oldChunkMeta = chunkMetaMap.get(timestamp_chunk);
ChunkMeta oldChunkMeta = getChunkMeta(chunkMetaMap, entryList.get(0).timestamp);
if (oldChunkMeta == null) {
insertChunk(chunkMetaMap, chunkMap, Chunk.of(entryList));
} else {
Chunk oldChunk = chunkMap.get(oldChunkMeta.firstTimestamp);
Iterator<DataEntry> oldIt = Arrays.stream(oldChunk.data).iterator();
Iterator<DataEntry> newIt = entryList.iterator();
ArrayList<DataEntry> resultList = new ArrayList<DataEntry>();
DataEntry old_curr = oldIt.hasNext() ? oldIt.next() : null;
DataEntry new_curr = newIt.hasNext() ? newIt.next() : null;
while (old_curr != null || new_curr != null) {
if (old_curr != null) {
if (new_curr != null) {
if (old_curr.timestamp == new_curr.timestamp) { // overwrite old data with new data
resultList.add(new_curr);
old_curr = oldIt.hasNext() ? oldIt.next() : null;
new_curr = newIt.hasNext() ? newIt.next() : null;
} else if (old_curr.timestamp < new_curr.timestamp) {
resultList.add(old_curr);
old_curr = oldIt.hasNext() ? oldIt.next() : null;
} else {
resultList.add(new_curr);
new_curr = newIt.hasNext() ? newIt.next() : null;
}
} else {
resultList.add(old_curr);
old_curr = oldIt.hasNext() ? oldIt.next() : null;
}
} else {
resultList.add(new_curr);
new_curr = newIt.hasNext() ? newIt.next() : null;
}
}
removeChunk(chunkMetaMap, chunkMap, oldChunkMeta);
insertChunk(chunkMetaMap, chunkMap, Chunk.of(resultList));
}
}
public int[] getStationTimeInterval(String stationName) {
throwNull(stationName);
BTreeMap<String, SensorMeta> sensorMap = getSensorMap(stationName);
if (sensorMap == null || sensorMap.isEmpty()) {
return null;
}
int minTimestamp = Integer.MAX_VALUE;
int maxTimestamp = Integer.MIN_VALUE;
for (SensorMeta sensorMeta : sensorMap.values()) {
int[] interval = getSensorTimeInterval(sensorMeta);
if (interval != null) {
if (interval[0] < minTimestamp) {
minTimestamp = interval[0];
}
if (maxTimestamp < interval[1]) {
maxTimestamp = interval[1];
}
}
}
if (minTimestamp == Integer.MAX_VALUE || maxTimestamp == Integer.MIN_VALUE) {
return null;
}
return new int[] {minTimestamp, maxTimestamp};
}
@Override
public Records recordsNewerThan(
DateTime changeSetTime, boolean inclusive, boolean descendingOrder) {
if (stopped) {
return Records.EMPTY;
}
long changeSetMillisUTC = -1;
long searchBound = -1;
if (changeSetTime != null) {
changeSetMillisUTC = changeSetTime.getMillis();
// adjust the millis using a delta so that we are sure we catch everything in a cluster which
// may have differences in
// clock time
searchBound = TIME_BASED_KEYS.getCounterStartingAt(changeSetMillisUTC - searchTimeDelta);
}
NavigableMap<Long, JournalRecord> subMap = records.tailMap(searchBound, true);
// process each of the records from the result and look at the timestamp of the changeset, so
// that we're sure we only include
// the correct ones (we used a delta to make sure we get everything)
long startKeyInSubMap = -1;
for (Long timeBasedKey : subMap.keySet()) {
JournalRecord record = subMap.get(timeBasedKey);
long recordChangeTimeMillisUTC = record.getChangeTimeMillis();
if (((recordChangeTimeMillisUTC == changeSetMillisUTC) && inclusive)
|| recordChangeTimeMillisUTC > changeSetMillisUTC) {
startKeyInSubMap = timeBasedKey;
break;
}
}
return startKeyInSubMap != -1
? recordsFrom(subMap.tailMap(startKeyInSubMap, true), descendingOrder)
: Records.EMPTY;
}
/**
* Check if station exists in StreamDB. Station exists only if it contains time series data.
*
* @param stationID
* @return
*/
public boolean existStation(String stationID) {
return stationMetaMap.containsKey(stationID);
}
public NavigableSet<String> getStationNames() {
return stationMetaMap.keySet();
}