@Test
public void testGetTreeMap() throws Exception {
Map m1 = db.getTreeMap("test");
m1.put(1, 2);
m1.put(3, 4);
assertTrue(m1 == db.getTreeMap("test"));
assertEquals(m1, new DB(engine).getTreeMap("test"));
}
private SensorMeta getSensorMeta(
StationMeta stationMeta, String sensorName, boolean createIfNotExists) {
throwNull(stationMeta);
throwNull(sensorName);
BTreeMap<String, SensorMeta> sensorMap = db.getTreeMap(stationMeta.db_name_sensor_map);
SensorMeta sensorMeta = sensorMap.get(sensorName);
if (sensorMeta == null && createIfNotExists) {
sensorMeta = new SensorMeta(stationMeta.stationName, sensorName);
db.checkNameNotExists(sensorMeta.db_name_sensor_chunk_map);
db.createTreeMap(sensorMeta.db_name_sensor_chunk_map)
.keySerializer(BTreeKeySerializer.ZERO_OR_POSITIVE_INT)
// .valueSerializer(Chunk.DELTA_TIME_DELTA_DELTA_VALUE_INT_QUANTIZED_SERIALIZER)
// .valueSerializer(Chunk.SNAPPY_DELTA_TIME_DELTA_DELTA_VALUE_INT_QUANTIZED_SERIALIZER)
.valueSerializer(ChunkSerializer.DEFAULT)
// .valuesOutsideNodesEnable() // !!! does not work: growing database
// .
.makeOrGet();
db.checkNameNotExists(sensorMeta.db_name_sensor_chunkmeta_map);
db.createTreeMap(sensorMeta.db_name_sensor_chunkmeta_map)
.keySerializer(BTreeKeySerializer.ZERO_OR_POSITIVE_INT)
.valueSerializer(ChunkMeta.SERIALIZER)
.makeOrGet();
sensorMap.put(sensorName, sensorMeta);
}
if (sensorMeta == null) {
// new Throwable().printStackTrace();
log.warn("no sensor: " + sensorName + " in station: " + stationMeta.stationName);
}
return sensorMeta;
}
public static void main(String[] args) throws IOException {
// Open db in temp directory
File f = File.createTempFile("mapdb", "temp");
DB db = DBMaker.newFileDB(f).make();
// Open or create table
Map<String, Person> dbMap = db.getTreeMap("personAndCity");
// Add data
Person bilbo = new Person("Bilbo", "The Shire");
Person sauron = new Person("Sauron", "Mordor");
Person radagast = new Person("Radagast", "Crazy Farm");
dbMap.put("west", bilbo);
dbMap.put("south", sauron);
dbMap.put("mid", radagast);
// Commit and close
db.commit();
db.close();
//
// Second option for using cystom values is to use your own serializer.
// This usually leads to better performance as MapDB does not have to
// analyze the class structure.
//
class CustomSerializer implements ValueSerializer<Person>, Serializable {
@Override
public void serialize(DataOutput out, Person value) throws IOException {
out.writeUTF(value.getName());
out.writeUTF(value.getCity());
}
@Override
public Person deserialize(DataInput in, int available) throws IOException {
return new Person(in.readUTF(), in.readUTF());
}
@Override
public int fixedSize() {
return -1;
}
}
ValueSerializer<Person> serializer = new CustomSerializer();
DB db2 = DBMaker.newTempFileDB().make();
Map<String, Person> map2 = db2.createHashMap("map").valueSerializer(serializer).make();
map2.put("North", new Person("Yet another dwarf", "Somewhere"));
db2.commit();
db2.close();
}
public void clearMaskOfStation(String stationName) {
StationMeta stationMeta = getStationMeta(stationName, false);
if (stationMeta == null) {
// log.warn("station not found "+stationName);
return;
}
BTreeMap<String, TimeSeriesMask> maskMap =
db.getTreeMap(stationMeta.db_name_sensor_time_series_mask_map);
maskMap.clear();
}
public NavigableSet<String> getSensorNames(String stationName) {
throwNull(stationName);
StationMeta stationMeta = stationMetaMap.get(stationName);
if (stationMeta == null) {
log.warn("no station: " + stationName);
return new TreeSet<String>();
}
BTreeMap<String, SensorMeta> sensorMap = db.getTreeMap(stationMeta.db_name_sensor_map);
return sensorMap.keySet();
}
public void setSensorTimeSeriesMask(
String stationName, String sensorName, TimeSeriesMask timeSeriesMask) {
throwNull(stationName);
throwNull(sensorName);
throwNull(timeSeriesMask);
StationMeta stationMeta = getStationMeta(stationName, true);
BTreeMap<String, TimeSeriesMask> maskMap =
db.getTreeMap(stationMeta.db_name_sensor_time_series_mask_map);
maskMap.put(sensorName, timeSeriesMask);
}
@Override
public void open() throws IOException {
if (memoryMode) {
this.index = new RAMDirectory();
this.store = new HashMap<String, AdDefinition>();
} else {
if (!addb.manager.getContext().getConfiguration().containsKey(CONFIG_DATADIR)) {
throw new IOException("data directory can not be empty");
}
String dir = (String) addb.manager.getContext().getConfiguration().get(CONFIG_DATADIR);
if (!dir.endsWith("/") || !dir.endsWith("\\")) {
dir += "/";
}
File temp = new File(dir + "store");
if (!temp.exists()) {
temp.mkdirs();
}
temp = new File(dir + "index");
if (!temp.exists()) {
temp.mkdirs();
}
// create lucene index directory
index = FSDirectory.open(temp);
nrt_index = new NRTCachingDirectory(index, 5.0, 60.0);
// create database
db = DBMaker.newAppendFileDB(new File(dir + "store/ads")).closeOnJvmShutdown().make();
store = db.getTreeMap("collectionName");
}
IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_44, new KeywordAnalyzer());
// CREATE_OR_APPEND
config.setOpenMode(OpenMode.CREATE_OR_APPEND);
writer = new IndexWriter(nrt_index != null ? nrt_index : index, config);
// nrt_manager = new NRTManager(new NRTManager.TrackingIndexWriter(writer), null);
nrt_manager = new SearcherManager(writer, true, null);
// trackingWriter = new TrackingIndexWriter(writer);
// nrtThread = new ControlledRealTimeReopenThread<IndexSearcher>(trackingWriter, nrt_manager,
// (double)5, (double)5);
// nrtThread.setName("NRTManager Reopen Thread");
// nrtThread.setPriority(Math.min(Thread.currentThread().getPriority()+2,
// Thread.MAX_PRIORITY));
// nrtThread.setDaemon(true);
// nrtThread.start();
}
public TimeSeriesMask getSensorTimeSeriesMask(
StationMeta stationMeta, String sensorName, boolean createIfNotExists) {
throwNull(stationMeta);
throwNull(sensorName);
BTreeMap<String, TimeSeriesMask> maskMap =
db.getTreeMap(stationMeta.db_name_sensor_time_series_mask_map);
TimeSeriesMask mask = maskMap.get(sensorName);
if (mask == null && createIfNotExists) {
mask = new TimeSeriesMask();
maskMap.put(sensorName, mask);
}
if (mask == null) {
// log.info("no time series mask: "+sensorName+" in station: "+stationMeta.stationName);
}
return mask;
}
public static void main(String[] args) {
if (1 == 1) return;
// create inMemory store which does not use serialization,
// and has speed comparable to `java.util` collections
DB inMemory = new DB(new StoreHeap());
Map m = inMemory.getTreeMap("test");
Random r = new Random();
// insert random stuff, keep on mind it needs to fit into memory
for (int i = 0; i < 10000; i++) {
m.put(r.nextInt(), "dwqas" + i);
}
// now create on-disk store, it needs to be completely empty
File targetFile = UtilsTest.tempDbFile();
DB target = DBMaker.newFileDB(targetFile).make();
Pump.copy(inMemory, target);
inMemory.close();
target.close();
}
@Override
protected Map<String, byte[]> getMap(DB database) {
// OPEN MAP
return database.getTreeMap("references");
}
public void read(DB mapDb, String adminLevel) throws IOException, ClassNotFoundException {
List<Relation> relations = new ArrayList<Relation>();
Map<Long, Relation> relationStore = mapDb.getTreeMap("relationStore");
int totalRelCounter = 0;
for (Long id : relationStore.keySet()) {
totalRelCounter++;
if (totalRelCounter % 10000 == 0)
log.info(
"Rel counter: "
+ totalRelCounter
+ ", adminLevels added="
+ relations.size() /*+relations.size()*/);
Relation rel = relationStore.get(id);
if (adminLevel.equals(rel.tags.get("admin_level"))) relations.add(rel);
}
log.info("Found " + relations.size() + " relations. Processing...");
// log.info();
FileWriter out = new FileWriter("states.txt");
Map<String, FileWriter> countryPolyFiles = new HashMap<String, FileWriter>();
for (Relation rel : relations) {
String name = rel.tags.get("name");
int totalNodes = 0;
for (Member m : rel.members) {
if (m.member instanceof Way) {
totalNodes += ((Way) m.member).numNodes();
}
}
int ways = 0, nodes = 0;
Set<String> roles = new HashSet<String>();
List<List<Node>> multipoly = new ArrayList<List<Node>>();
List<String> rolesList = new ArrayList<String>();
List<GraphNode> graphNodes = new ArrayList<GraphNode>();
List<GraphEdge> graphEdges = new ArrayList<GraphEdge>();
for (Member m : rel.members) {
if (m.member instanceof Way) {
ways++;
roles.add(m.role);
Way way = (Way) m.member;
// log.info(way.nodes.get(0)+"->"+way.nodes.get(way.nodes.size()-1));
GraphNode src = new GraphNode(Arrays.toString(way.getNode(0)));
GraphNode dest = new GraphNode(Arrays.toString(way.getNode(way.numNodes() - 1)));
log.debug(src + "->" + dest + ", size=" + way.numNodes());
/*for (int i=0; i<way.numNodes(); i++) {
log.info("Way point "+i+": "+Arrays.toString(way.getNode(i)));
}*/
if (graphNodes.contains(src)) src = graphNodes.get(graphNodes.indexOf(src));
else graphNodes.add(src);
if (graphNodes.contains(dest)) dest = graphNodes.get(graphNodes.indexOf(dest));
else graphNodes.add(dest);
float[] reversedWayNodes = way.getPointsReversed();
float[] wayNodes = way.getPoints();
if (src.equals(dest)) {
src.selfPath = wayNodes;
dest.selfPath = wayNodes;
}
GraphEdge to = new GraphEdge(src, dest, wayNodes, "" + way.id);
GraphEdge from = new GraphEdge(dest, src, reversedWayNodes, "" + way.id);
// log.info("ALL: "+to+", id="+to.id);
// if (way.id.equals("219576421"))
// log.info("219576421: "+to.path);
// if(src.edges.contains(to)==false)
src.edges.add(to);
// if(dest.edges.contains(from)==false)
dest.edges.add(from);
graphEdges.add(to);
graphEdges.add(from);
} else if (m.member instanceof Node) {
nodes++;
}
}
List<List<GraphNode>> loops = GraphProcessor.traverse(graphNodes);
for (List<GraphNode> loop : loops) {
List<Node> poly = new ArrayList<Node>();
if (loop.size() == 1) {
log.debug("Self loop came: " + loop);
poly.addAll(Utils.nodesFromPoints(loop.get(0).selfPath));
} else
for (int i = 1; i < loop.size(); i++) {
GraphNode prev = loop.get(i - 1);
GraphNode cur = loop.get(i);
if (prev.equals(cur)) {
// poly.addAll(prev.selfPath);
continue;
} else {
/*GraphEdge tmpEdge = new GraphEdge(prev, cur, null);
GraphEdge actualEdge = graphEdges.get(loops.indexOf(tmpEdge));*/
GraphEdge actualEdge = null;
for (GraphEdge e : prev.edges) if (e.dest.equals(cur)) actualEdge = e;
if (actualEdge == null) {
System.err.println(
"ERROR: Prev: " + prev + ", cur=" + cur + ", prev.edges: " + prev.edges);
} else {
poly.addAll(Utils.nodesFromPoints(actualEdge.path));
}
}
}
if (poly.size() > 0) multipoly.add(poly);
}
// log.info("Max size poly = "+maxSizePoly);
int processedNodes = 0;
for (List<Node> poly : multipoly) processedNodes += poly.size();
if (totalNodes > 0)
log.info(
name
+ ": "
+ processedNodes
+ "/"
+ totalNodes
+ " ("
+ (processedNodes * 100 / totalNodes)
+ "%), polygons="
+ multipoly.size());
if (multipoly.size() > 0 && rel.tags.containsKey("name")) {
places.add(new AdminPolygon(rel.id, rel.tags, rolesList, multipoly));
// log.info(rel.tags.get("name")+": "+ways+", "+nodes+", roles="+roles+",
// Multipoly="+multipoly.size()+", RolesList="+rolesList.size());
// log.info("isin: "+rel.tags.get("is_in:country_code"));
// log.info(rel.tags);
}
// }
// }
}
out.close();
for (String c : countryPolyFiles.keySet()) countryPolyFiles.get(c).close();
}
public BTreeMap<String, SensorMeta> getSensorMap(StationMeta stationMeta) {
throwNull(stationMeta);
return db.getTreeMap(stationMeta.db_name_sensor_map);
}
public BTreeMap<Integer, ChunkMeta> getSensorChunkMetaMap(SensorMeta sensorMeta) {
throwNull(sensorMeta);
return db.getTreeMap(sensorMeta.db_name_sensor_chunkmeta_map);
}
private BTreeMap<Integer, Chunk> getSensorChunkMap(SensorMeta sensorMeta) {
throwNull(sensorMeta);
return db.getTreeMap(sensorMeta.db_name_sensor_chunk_map);
}