Esempio n. 1
0
  /**
   * Import all of the ways from the parsed XML into the HBase table as a segment. Ways are OSM
   * values which consist of a list of nodes. Segments are custom values we use which represent a
   * single node and its neighbors.
   */
  private static void import_ways() {
    System.out.println("Importing ways (segments)...");
    Table segmentTable = Util.get_table("segment");
    if (segmentTable == null) {
      System.err.println("Segment table failed to load.");
      return;
    }

    int counter = 0;
    int batch = 100;
    List<Put> puts = new ArrayList<>();
    for (Way way : ways) {
      Node previousNode = null;
      for (Node node : way.getNodes()) {
        if (previousNode == null) {
          previousNode = node;
          continue;
        }
        Put p = new Put(Bytes.toBytes(previousNode.getGeohash()));
        p.addColumn(
            NODE,
            Bytes.toBytes(node.getGeohash()),
            Bytes.toBytes(String.valueOf(way.getTagsAsSerializedJSON())));
        puts.add(p);
        p = new Put(Bytes.toBytes(node.getGeohash()));
        p.addColumn(
            NODE,
            Bytes.toBytes(previousNode.getGeohash()),
            Bytes.toBytes(String.valueOf(way.getTagsAsSerializedJSON())));
        puts.add(p);
      }
      counter += 1;
      if (counter % batch == 0) {
        try {
          System.out.print("\rBatch " + counter + " / " + ways.size());
          segmentTable.put(puts);
          puts.clear();
        } catch (IOException e) {
          System.out.println("Segment put failed");
          e.printStackTrace();
        }
      }
    }
    try {
      segmentTable.put(puts);
    } catch (IOException e) {
      System.out.println("Segment put failed");
      e.printStackTrace();
    }
    System.out.println("Added all segments!");
  }
Esempio n. 2
0
  /** Import all of the nodes from the parsed XML into the HBase table as a node. */
  private static void import_nodes() {
    System.out.println("Importing nodes...");
    Table nodeTable = Util.get_table("node");
    if (nodeTable == null) {
      System.err.println("Node table failed to load.");
      return;
    }

    int counter = 0;
    int batch = 500;
    List<Put> puts = new ArrayList<>();
    for (Map.Entry<Long, Node> pair : nodes.entrySet()) {
      Node node = pair.getValue();
      Put p = new Put(Bytes.toBytes(node.getGeohash()));
      p.addColumn(DATA, TAGS, Bytes.toBytes(node.getTagsWithIDAsSerializedJSON()));
      puts.add(p);
      counter += 1;
      if (counter % batch == 0) {
        try {
          System.out.print("\rBatch " + counter + " / " + nodes.size());
          nodeTable.put(puts);
          puts.clear();
        } catch (IOException e) {
          System.err.println("Node put failed");
          e.printStackTrace();
        }
      }
    }
    try {
      nodeTable.put(puts);
      puts.clear();
    } catch (IOException e) {
      System.out.println("Node put failed");
      e.printStackTrace();
    }
    System.out.println("Added all nodes!");
  }