/** Import all of the nodes from the parsed XML into the HBase table as a node. */
private static void import_nodes() {
System.out.println("Importing nodes...");
Table nodeTable = Util.get_table("node");
if (nodeTable == null) {
System.err.println("Node table failed to load.");
return;
}
int counter = 0;
int batch = 500;
List<Put> puts = new ArrayList<>();
for (Map.Entry<Long, Node> pair : nodes.entrySet()) {
Node node = pair.getValue();
Put p = new Put(Bytes.toBytes(node.getGeohash()));
p.addColumn(DATA, TAGS, Bytes.toBytes(node.getTagsWithIDAsSerializedJSON()));
puts.add(p);
counter += 1;
if (counter % batch == 0) {
try {
System.out.print("\rBatch " + counter + " / " + nodes.size());
nodeTable.put(puts);
puts.clear();
} catch (IOException e) {
System.err.println("Node put failed");
e.printStackTrace();
}
}
}
try {
nodeTable.put(puts);
puts.clear();
} catch (IOException e) {
System.out.println("Node put failed");
e.printStackTrace();
}
System.out.println("Added all nodes!");
}
/**
* Import all of the ways from the parsed XML into the HBase table as a segment. Ways are OSM
* values which consist of a list of nodes. Segments are custom values we use which represent a
* single node and its neighbors.
*/
private static void import_ways() {
System.out.println("Importing ways (segments)...");
Table segmentTable = Util.get_table("segment");
if (segmentTable == null) {
System.err.println("Segment table failed to load.");
return;
}
int counter = 0;
int batch = 100;
List<Put> puts = new ArrayList<>();
for (Way way : ways) {
Node previousNode = null;
for (Node node : way.getNodes()) {
if (previousNode == null) {
previousNode = node;
continue;
}
Put p = new Put(Bytes.toBytes(previousNode.getGeohash()));
p.addColumn(
NODE,
Bytes.toBytes(node.getGeohash()),
Bytes.toBytes(String.valueOf(way.getTagsAsSerializedJSON())));
puts.add(p);
p = new Put(Bytes.toBytes(node.getGeohash()));
p.addColumn(
NODE,
Bytes.toBytes(previousNode.getGeohash()),
Bytes.toBytes(String.valueOf(way.getTagsAsSerializedJSON())));
puts.add(p);
}
counter += 1;
if (counter % batch == 0) {
try {
System.out.print("\rBatch " + counter + " / " + ways.size());
segmentTable.put(puts);
puts.clear();
} catch (IOException e) {
System.out.println("Segment put failed");
e.printStackTrace();
}
}
}
try {
segmentTable.put(puts);
} catch (IOException e) {
System.out.println("Segment put failed");
e.printStackTrace();
}
System.out.println("Added all segments!");
}
/**
* Given the OSM XML file, parse out all of the nodes and ways and then run the import methods to
* put the values into HBase.
*
* @param filename path to the OSM XML file
* @throws XMLStreamException XML file is invalid
*/
public static void import_from_file(String filename) throws XMLStreamException {
System.out.println("Starting...");
System.out.println(configuration.toString());
XMLInputFactory xmlInputFactory = XMLInputFactory.newFactory();
XMLStreamReader xmlStreamReader;
try {
xmlStreamReader = xmlInputFactory.createXMLStreamReader(new FileInputStream(filename));
} catch (FileNotFoundException e) {
throw new XMLStreamException("OSM XML file not found!");
}
// Begin reading the XML
Node node = null;
Way way = null;
while (xmlStreamReader.hasNext()) {
if (xmlStreamReader.isStartElement()) {
// Start of Element, consider only the things we care about
String s1 = xmlStreamReader.getLocalName();
switch (s1) {
case "node": // Start new node
node = new Node();
// Get the node attributes we care about
for (int i = 0; i < xmlStreamReader.getAttributeCount(); i++) {
String value = xmlStreamReader.getAttributeValue(i);
String s = xmlStreamReader.getAttributeLocalName(i);
switch (s) {
case "id":
node.setOsmId(value);
break;
case "lat":
node.setLat(value);
break;
case "lon":
node.setLon(value);
break;
}
if (node.isComplete()) break;
}
break;
case "way": // Start new way
way = new Way();
// Get the way attributes we care about
for (int i = 0; i < xmlStreamReader.getAttributeCount(); i++) {
String value = xmlStreamReader.getAttributeValue(i);
String s = xmlStreamReader.getAttributeLocalName(i);
if (s.equals("id")) {
way.setId(value);
}
}
break;
case "nd": // Start the way's node reference
if (way == null) {
System.err.println("Parser Error, Turns out we needed our way object.");
return;
}
// Get the id of the node
long node_id = Long.MIN_VALUE;
for (int i = 0; i < xmlStreamReader.getAttributeCount(); i++) {
String s = xmlStreamReader.getAttributeLocalName(i);
if (s.equals("ref")) {
String value = xmlStreamReader.getAttributeValue(i);
node_id = Long.parseLong(value);
}
}
// Add node to way
if (node_id != Long.MIN_VALUE) {
way.addNode(nodes.get(node_id));
}
break;
case "tag": // Start the tag parsing
String key = "";
String value = "";
for (int i = 0; i < xmlStreamReader.getAttributeCount(); i++) {
String s = xmlStreamReader.getAttributeLocalName(i);
if (s.equals("k")) {
key = xmlStreamReader.getAttributeValue(i);
}
if (s.equals("v")) {
value = xmlStreamReader.getAttributeValue(i);
}
}
if (!key.equals("") && !value.equals("")) {
if (node != null) {
// This is a node tag
node.addTag(key, value);
System.out.println(
"Added Node Tag " + key + ": " + value + "for " + node.getOsmId());
} else if (way != null) {
// This is a way tag
way.addTag(key, value);
System.out.println("Added Way Tag " + key + ": " + value + "for " + way.getId());
}
/*
else {
// This will barf because we don't care about 'relation' data types
// System.err.println(key + ": " + value + " not associated to node or way!");
}
*/
}
break;
}
} else if (xmlStreamReader.isEndElement()) {
String s = xmlStreamReader.getLocalName();
if (s.equals("node") && node != null) {
nodes.put(node.getOsmId(), node);
System.out.print("\rAdded Node " + String.valueOf(node.getOsmId()));
node = null;
} else if (s.equals("way") && way != null) {
ways.add(way);
System.out.print("\rAdded Way " + String.valueOf(way.getId()) + " ");
way = null;
}
}
xmlStreamReader.next();
}
System.out.println("Number of nodes read: " + nodes.size());
System.out.println("Number of ways read: " + ways.size());
Main.print_heap_usage();
import_nodes();
Main.print_heap_usage();
import_ways();
System.out.println("Finished!");
}