Esempio n. 1
0
 @Test
 public void testGetTreeMap() throws Exception {
   Map m1 = db.getTreeMap("test");
   m1.put(1, 2);
   m1.put(3, 4);
   assertTrue(m1 == db.getTreeMap("test"));
   assertEquals(m1, new DB(engine).getTreeMap("test"));
 }
 private SensorMeta getSensorMeta(
     StationMeta stationMeta, String sensorName, boolean createIfNotExists) {
   throwNull(stationMeta);
   throwNull(sensorName);
   BTreeMap<String, SensorMeta> sensorMap = db.getTreeMap(stationMeta.db_name_sensor_map);
   SensorMeta sensorMeta = sensorMap.get(sensorName);
   if (sensorMeta == null && createIfNotExists) {
     sensorMeta = new SensorMeta(stationMeta.stationName, sensorName);
     db.checkNameNotExists(sensorMeta.db_name_sensor_chunk_map);
     db.createTreeMap(sensorMeta.db_name_sensor_chunk_map)
         .keySerializer(BTreeKeySerializer.ZERO_OR_POSITIVE_INT)
         // .valueSerializer(Chunk.DELTA_TIME_DELTA_DELTA_VALUE_INT_QUANTIZED_SERIALIZER)
         // .valueSerializer(Chunk.SNAPPY_DELTA_TIME_DELTA_DELTA_VALUE_INT_QUANTIZED_SERIALIZER)
         .valueSerializer(ChunkSerializer.DEFAULT)
         // .valuesOutsideNodesEnable() // !!! does not work: growing database
         // .
         .makeOrGet();
     db.checkNameNotExists(sensorMeta.db_name_sensor_chunkmeta_map);
     db.createTreeMap(sensorMeta.db_name_sensor_chunkmeta_map)
         .keySerializer(BTreeKeySerializer.ZERO_OR_POSITIVE_INT)
         .valueSerializer(ChunkMeta.SERIALIZER)
         .makeOrGet();
     sensorMap.put(sensorName, sensorMeta);
   }
   if (sensorMeta == null) {
     // new Throwable().printStackTrace();
     log.warn("no sensor: " + sensorName + "  in station: " + stationMeta.stationName);
   }
   return sensorMeta;
 }
Esempio n. 3
0
  public static void main(String[] args) throws IOException {

    // Open db in temp directory
    File f = File.createTempFile("mapdb", "temp");
    DB db = DBMaker.newFileDB(f).make();

    // Open or create table
    Map<String, Person> dbMap = db.getTreeMap("personAndCity");

    // Add data
    Person bilbo = new Person("Bilbo", "The Shire");
    Person sauron = new Person("Sauron", "Mordor");
    Person radagast = new Person("Radagast", "Crazy Farm");

    dbMap.put("west", bilbo);
    dbMap.put("south", sauron);
    dbMap.put("mid", radagast);

    // Commit and close
    db.commit();
    db.close();

    //
    // Second option for using cystom values is to use your own serializer.
    // This usually leads to better performance as MapDB does not have to
    // analyze the class structure.
    //

    class CustomSerializer implements ValueSerializer<Person>, Serializable {

      @Override
      public void serialize(DataOutput out, Person value) throws IOException {
        out.writeUTF(value.getName());
        out.writeUTF(value.getCity());
      }

      @Override
      public Person deserialize(DataInput in, int available) throws IOException {
        return new Person(in.readUTF(), in.readUTF());
      }

      @Override
      public int fixedSize() {
        return -1;
      }
    }

    ValueSerializer<Person> serializer = new CustomSerializer();

    DB db2 = DBMaker.newTempFileDB().make();

    Map<String, Person> map2 = db2.createHashMap("map").valueSerializer(serializer).make();

    map2.put("North", new Person("Yet another dwarf", "Somewhere"));

    db2.commit();
    db2.close();
  }
 public void clearMaskOfStation(String stationName) {
   StationMeta stationMeta = getStationMeta(stationName, false);
   if (stationMeta == null) {
     // log.warn("station not found "+stationName);
     return;
   }
   BTreeMap<String, TimeSeriesMask> maskMap =
       db.getTreeMap(stationMeta.db_name_sensor_time_series_mask_map);
   maskMap.clear();
 }
 public NavigableSet<String> getSensorNames(String stationName) {
   throwNull(stationName);
   StationMeta stationMeta = stationMetaMap.get(stationName);
   if (stationMeta == null) {
     log.warn("no station: " + stationName);
     return new TreeSet<String>();
   }
   BTreeMap<String, SensorMeta> sensorMap = db.getTreeMap(stationMeta.db_name_sensor_map);
   return sensorMap.keySet();
 }
 public void setSensorTimeSeriesMask(
     String stationName, String sensorName, TimeSeriesMask timeSeriesMask) {
   throwNull(stationName);
   throwNull(sensorName);
   throwNull(timeSeriesMask);
   StationMeta stationMeta = getStationMeta(stationName, true);
   BTreeMap<String, TimeSeriesMask> maskMap =
       db.getTreeMap(stationMeta.db_name_sensor_time_series_mask_map);
   maskMap.put(sensorName, timeSeriesMask);
 }
Esempio n. 7
0
  @Override
  public void open() throws IOException {

    if (memoryMode) {
      this.index = new RAMDirectory();
      this.store = new HashMap<String, AdDefinition>();
    } else {
      if (!addb.manager.getContext().getConfiguration().containsKey(CONFIG_DATADIR)) {
        throw new IOException("data directory can not be empty");
      }

      String dir = (String) addb.manager.getContext().getConfiguration().get(CONFIG_DATADIR);
      if (!dir.endsWith("/") || !dir.endsWith("\\")) {
        dir += "/";
      }
      File temp = new File(dir + "store");
      if (!temp.exists()) {
        temp.mkdirs();
      }
      temp = new File(dir + "index");
      if (!temp.exists()) {
        temp.mkdirs();
      }
      // create lucene index directory
      index = FSDirectory.open(temp);
      nrt_index = new NRTCachingDirectory(index, 5.0, 60.0);

      // create database
      db = DBMaker.newAppendFileDB(new File(dir + "store/ads")).closeOnJvmShutdown().make();

      store = db.getTreeMap("collectionName");
    }

    IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_44, new KeywordAnalyzer());
    // CREATE_OR_APPEND
    config.setOpenMode(OpenMode.CREATE_OR_APPEND);
    writer = new IndexWriter(nrt_index != null ? nrt_index : index, config);

    //		nrt_manager = new NRTManager(new NRTManager.TrackingIndexWriter(writer), null);
    nrt_manager = new SearcherManager(writer, true, null);
    //		trackingWriter = new TrackingIndexWriter(writer);
    //		nrtThread = new ControlledRealTimeReopenThread<IndexSearcher>(trackingWriter, nrt_manager,
    // (double)5, (double)5);
    //		nrtThread.setName("NRTManager Reopen Thread");
    //		nrtThread.setPriority(Math.min(Thread.currentThread().getPriority()+2,
    // Thread.MAX_PRIORITY));
    //		nrtThread.setDaemon(true);
    //		nrtThread.start();
  }
 public TimeSeriesMask getSensorTimeSeriesMask(
     StationMeta stationMeta, String sensorName, boolean createIfNotExists) {
   throwNull(stationMeta);
   throwNull(sensorName);
   BTreeMap<String, TimeSeriesMask> maskMap =
       db.getTreeMap(stationMeta.db_name_sensor_time_series_mask_map);
   TimeSeriesMask mask = maskMap.get(sensorName);
   if (mask == null && createIfNotExists) {
     mask = new TimeSeriesMask();
     maskMap.put(sensorName, mask);
   }
   if (mask == null) {
     // log.info("no time series mask: "+sensorName+"  in station: "+stationMeta.stationName);
   }
   return mask;
 }
  public static void main(String[] args) {
    if (1 == 1) return;

    // create inMemory store which does not use serialization,
    // and has speed comparable to `java.util` collections
    DB inMemory = new DB(new StoreHeap());
    Map m = inMemory.getTreeMap("test");

    Random r = new Random();
    // insert random stuff, keep on mind it needs to fit into memory
    for (int i = 0; i < 10000; i++) {
      m.put(r.nextInt(), "dwqas" + i);
    }

    // now create on-disk store, it needs to be completely empty
    File targetFile = UtilsTest.tempDbFile();
    DB target = DBMaker.newFileDB(targetFile).make();

    Pump.copy(inMemory, target);

    inMemory.close();
    target.close();
  }
Esempio n. 10
0
 @Override
 protected Map<String, byte[]> getMap(DB database) {
   // OPEN MAP
   return database.getTreeMap("references");
 }
Esempio n. 11
0
  public void read(DB mapDb, String adminLevel) throws IOException, ClassNotFoundException {
    List<Relation> relations = new ArrayList<Relation>();

    Map<Long, Relation> relationStore = mapDb.getTreeMap("relationStore");
    int totalRelCounter = 0;

    for (Long id : relationStore.keySet()) {
      totalRelCounter++;
      if (totalRelCounter % 10000 == 0)
        log.info(
            "Rel counter: "
                + totalRelCounter
                + ", adminLevels added="
                + relations.size() /*+relations.size()*/);

      Relation rel = relationStore.get(id);
      if (adminLevel.equals(rel.tags.get("admin_level"))) relations.add(rel);
    }

    log.info("Found " + relations.size() + " relations. Processing...");

    //	log.info();
    FileWriter out = new FileWriter("states.txt");
    Map<String, FileWriter> countryPolyFiles = new HashMap<String, FileWriter>();
    for (Relation rel : relations) {

      String name = rel.tags.get("name");

      int totalNodes = 0;

      for (Member m : rel.members) {
        if (m.member instanceof Way) {
          totalNodes += ((Way) m.member).numNodes();
        }
      }

      int ways = 0, nodes = 0;
      Set<String> roles = new HashSet<String>();

      List<List<Node>> multipoly = new ArrayList<List<Node>>();
      List<String> rolesList = new ArrayList<String>();

      List<GraphNode> graphNodes = new ArrayList<GraphNode>();
      List<GraphEdge> graphEdges = new ArrayList<GraphEdge>();

      for (Member m : rel.members) {
        if (m.member instanceof Way) {
          ways++;
          roles.add(m.role);

          Way way = (Way) m.member;

          // log.info(way.nodes.get(0)+"->"+way.nodes.get(way.nodes.size()-1));
          GraphNode src = new GraphNode(Arrays.toString(way.getNode(0)));
          GraphNode dest = new GraphNode(Arrays.toString(way.getNode(way.numNodes() - 1)));

          log.debug(src + "->" + dest + ", size=" + way.numNodes());
          /*for (int i=0; i<way.numNodes(); i++) {
            log.info("Way point "+i+": "+Arrays.toString(way.getNode(i)));
          }*/

          if (graphNodes.contains(src)) src = graphNodes.get(graphNodes.indexOf(src));
          else graphNodes.add(src);
          if (graphNodes.contains(dest)) dest = graphNodes.get(graphNodes.indexOf(dest));
          else graphNodes.add(dest);

          float[] reversedWayNodes = way.getPointsReversed();
          float[] wayNodes = way.getPoints();

          if (src.equals(dest)) {
            src.selfPath = wayNodes;
            dest.selfPath = wayNodes;
          }

          GraphEdge to = new GraphEdge(src, dest, wayNodes, "" + way.id);
          GraphEdge from = new GraphEdge(dest, src, reversedWayNodes, "" + way.id);

          // log.info("ALL: "+to+", id="+to.id);
          //          if (way.id.equals("219576421"))
          //            log.info("219576421: "+to.path);

          // if(src.edges.contains(to)==false)
          src.edges.add(to);
          // if(dest.edges.contains(from)==false)
          dest.edges.add(from);

          graphEdges.add(to);
          graphEdges.add(from);

        } else if (m.member instanceof Node) {
          nodes++;
        }
      }

      List<List<GraphNode>> loops = GraphProcessor.traverse(graphNodes);

      for (List<GraphNode> loop : loops) {
        List<Node> poly = new ArrayList<Node>();
        if (loop.size() == 1) {
          log.debug("Self loop came: " + loop);
          poly.addAll(Utils.nodesFromPoints(loop.get(0).selfPath));
        } else
          for (int i = 1; i < loop.size(); i++) {
            GraphNode prev = loop.get(i - 1);
            GraphNode cur = loop.get(i);
            if (prev.equals(cur)) {
              // poly.addAll(prev.selfPath);
              continue;
            } else {
              /*GraphEdge tmpEdge = new GraphEdge(prev, cur, null);
              GraphEdge actualEdge = graphEdges.get(loops.indexOf(tmpEdge));*/
              GraphEdge actualEdge = null;
              for (GraphEdge e : prev.edges) if (e.dest.equals(cur)) actualEdge = e;
              if (actualEdge == null) {
                System.err.println(
                    "ERROR: Prev: " + prev + ", cur=" + cur + ", prev.edges: " + prev.edges);
              } else {
                poly.addAll(Utils.nodesFromPoints(actualEdge.path));
              }
            }
          }
        if (poly.size() > 0) multipoly.add(poly);
      }

      // log.info("Max size poly = "+maxSizePoly);
      int processedNodes = 0;
      for (List<Node> poly : multipoly) processedNodes += poly.size();
      if (totalNodes > 0)
        log.info(
            name
                + ": "
                + processedNodes
                + "/"
                + totalNodes
                + " ("
                + (processedNodes * 100 / totalNodes)
                + "%), polygons="
                + multipoly.size());

      if (multipoly.size() > 0 && rel.tags.containsKey("name")) {
        places.add(new AdminPolygon(rel.id, rel.tags, rolesList, multipoly));

        // log.info(rel.tags.get("name")+": "+ways+", "+nodes+", roles="+roles+",
        // Multipoly="+multipoly.size()+", RolesList="+rolesList.size());
        // log.info("isin: "+rel.tags.get("is_in:country_code"));
        // log.info(rel.tags);
      }
      // }
      // }
    }
    out.close();
    for (String c : countryPolyFiles.keySet()) countryPolyFiles.get(c).close();
  }
 public BTreeMap<String, SensorMeta> getSensorMap(StationMeta stationMeta) {
   throwNull(stationMeta);
   return db.getTreeMap(stationMeta.db_name_sensor_map);
 }
 public BTreeMap<Integer, ChunkMeta> getSensorChunkMetaMap(SensorMeta sensorMeta) {
   throwNull(sensorMeta);
   return db.getTreeMap(sensorMeta.db_name_sensor_chunkmeta_map);
 }
 private BTreeMap<Integer, Chunk> getSensorChunkMap(SensorMeta sensorMeta) {
   throwNull(sensorMeta);
   return db.getTreeMap(sensorMeta.db_name_sensor_chunk_map);
 }