/** * Import all of the ways from the parsed XML into the HBase table as a segment. Ways are OSM * values which consist of a list of nodes. Segments are custom values we use which represent a * single node and its neighbors. */ private static void import_ways() { System.out.println("Importing ways (segments)..."); Table segmentTable = Util.get_table("segment"); if (segmentTable == null) { System.err.println("Segment table failed to load."); return; } int counter = 0; int batch = 100; List<Put> puts = new ArrayList<>(); for (Way way : ways) { Node previousNode = null; for (Node node : way.getNodes()) { if (previousNode == null) { previousNode = node; continue; } Put p = new Put(Bytes.toBytes(previousNode.getGeohash())); p.addColumn( NODE, Bytes.toBytes(node.getGeohash()), Bytes.toBytes(String.valueOf(way.getTagsAsSerializedJSON()))); puts.add(p); p = new Put(Bytes.toBytes(node.getGeohash())); p.addColumn( NODE, Bytes.toBytes(previousNode.getGeohash()), Bytes.toBytes(String.valueOf(way.getTagsAsSerializedJSON()))); puts.add(p); } counter += 1; if (counter % batch == 0) { try { System.out.print("\rBatch " + counter + " / " + ways.size()); segmentTable.put(puts); puts.clear(); } catch (IOException e) { System.out.println("Segment put failed"); e.printStackTrace(); } } } try { segmentTable.put(puts); } catch (IOException e) { System.out.println("Segment put failed"); e.printStackTrace(); } System.out.println("Added all segments!"); }
/** Import all of the nodes from the parsed XML into the HBase table as a node. */ private static void import_nodes() { System.out.println("Importing nodes..."); Table nodeTable = Util.get_table("node"); if (nodeTable == null) { System.err.println("Node table failed to load."); return; } int counter = 0; int batch = 500; List<Put> puts = new ArrayList<>(); for (Map.Entry<Long, Node> pair : nodes.entrySet()) { Node node = pair.getValue(); Put p = new Put(Bytes.toBytes(node.getGeohash())); p.addColumn(DATA, TAGS, Bytes.toBytes(node.getTagsWithIDAsSerializedJSON())); puts.add(p); counter += 1; if (counter % batch == 0) { try { System.out.print("\rBatch " + counter + " / " + nodes.size()); nodeTable.put(puts); puts.clear(); } catch (IOException e) { System.err.println("Node put failed"); e.printStackTrace(); } } } try { nodeTable.put(puts); puts.clear(); } catch (IOException e) { System.out.println("Node put failed"); e.printStackTrace(); } System.out.println("Added all nodes!"); }