private void writeSubfileMetaDataToContainerHeader(
int i, long startIndexOfSubfile, long subfileSize) {
// HEADER META DATA FOR SUB FILE
// write zoom interval configuration to header
byte minZoomCurrentInterval = dataStore.getZoomIntervalConfiguration().getMinZoom(i);
byte maxZoomCurrentInterval = dataStore.getZoomIntervalConfiguration().getMaxZoom(i);
byte baseZoomCurrentInterval = dataStore.getZoomIntervalConfiguration().getBaseZoom(i);
bufferZoomIntervalConfig.put(baseZoomCurrentInterval);
bufferZoomIntervalConfig.put(minZoomCurrentInterval);
bufferZoomIntervalConfig.put(maxZoomCurrentInterval);
bufferZoomIntervalConfig.put(Serializer.getFiveBytes(startIndexOfSubfile));
bufferZoomIntervalConfig.put(Serializer.getFiveBytes(subfileSize));
}
MapFileWriter(TileBasedDataStore dataStore, RandomAccessFile file, int threadpoolSize) {
super();
this.dataStore = dataStore;
this.randomAccessFile = file;
fivePercentOfTilesToProcess = dataStore.cumulatedNumberOfTiles() / 20;
if (fivePercentOfTilesToProcess == 0) fivePercentOfTilesToProcess = 1;
executorService = Executors.newFixedThreadPool(threadpoolSize);
}
final void writeFile(
long date,
int version,
short tilePixel,
String comment,
boolean debugStrings,
boolean waynodeCompression,
boolean polygonClipping,
boolean pixelCompression,
GeoCoordinate mapStartPosition)
throws IOException {
// CONTAINER HEADER
long totalHeaderSize =
writeContainerHeader(
date,
version,
tilePixel,
comment,
debugStrings,
waynodeCompression,
polygonClipping,
pixelCompression,
mapStartPosition);
int n_zoom_intervals = dataStore.getZoomIntervalConfiguration().getNumberOfZoomIntervals();
// SUB FILES
// for each zoom interval write a sub file
long currentFileSize = totalHeaderSize;
for (int i = 0; i < n_zoom_intervals; i++) {
// SUB FILE INDEX AND DATA
long subfileSize =
writeSubfile(
currentFileSize,
i,
debugStrings,
waynodeCompression,
polygonClipping,
pixelCompression);
// SUB FILE META DATA IN CONTAINER HEADER
writeSubfileMetaDataToContainerHeader(i, currentFileSize, subfileSize);
currentFileSize += subfileSize;
}
randomAccessFile.setLength(currentFileSize);
randomAccessFile.close();
}
private long writeSubfile(
long startPositionSubfile,
int zoomIntervalIndex,
boolean debugStrings,
boolean waynodeCompression,
boolean polygonClipping,
boolean pixelCompression)
throws IOException {
logger.fine(
"writing data for zoom interval "
+ zoomIntervalIndex
+ ", number of tiles: "
+ dataStore.numberOfHorizontalTiles(zoomIntervalIndex)
* dataStore.numberOfVerticalTiles(zoomIntervalIndex));
TileCoordinate upperLeft = dataStore.getUpperLeft(zoomIntervalIndex);
int lengthX = dataStore.numberOfHorizontalTiles(zoomIntervalIndex);
int lengthY = dataStore.numberOfVerticalTiles(zoomIntervalIndex);
byte minZoomCurrentInterval =
dataStore.getZoomIntervalConfiguration().getMinZoom(zoomIntervalIndex);
byte maxZoomCurrentInterval =
dataStore.getZoomIntervalConfiguration().getMaxZoom(zoomIntervalIndex);
byte baseZoomCurrentInterval =
dataStore.getZoomIntervalConfiguration().getBaseZoom(zoomIntervalIndex);
byte maxMaxZoomlevel = dataStore.getZoomIntervalConfiguration().getMaxMaxZoom();
int tileAmountInBytes =
dataStore.numberOfHorizontalTiles(zoomIntervalIndex)
* dataStore.numberOfVerticalTiles(zoomIntervalIndex)
* BYTE_AMOUNT_SUBFILE_INDEX_PER_TILE;
int indexBufferSize =
tileAmountInBytes + (debugStrings ? DEBUG_INDEX_START_STRING.getBytes().length : 0);
MappedByteBuffer indexBuffer =
randomAccessFile
.getChannel()
.map(MapMode.READ_WRITE, startPositionSubfile, indexBufferSize);
MappedByteBuffer tileBuffer =
randomAccessFile
.getChannel()
.map(MapMode.READ_WRITE, startPositionSubfile + indexBufferSize, TILE_BUFFER_SIZE);
long currentSubfileOffset = indexBufferSize;
for (int tileY = upperLeft.getY(); tileY < upperLeft.getY() + lengthY; tileY++) {
for (int tileX = upperLeft.getX(); tileX < upperLeft.getX() + lengthX; tileX++) {
// logger.info("writing data for tile (" + tileX + ", " + tileY + ")");
long currentTileOffsetInBuffer = tileBuffer.position();
TileCoordinate currentTileCoordinate =
new TileCoordinate(tileX, tileY, baseZoomCurrentInterval);
// seek to index frame of this tile and write relative offset of this
// tile as five bytes to the index
indexBuffer.put(Serializer.getFiveBytes(currentSubfileOffset));
// get statistics for tile
TileData currentTile = dataStore.getTile(zoomIntervalIndex, tileX, tileY);
// ************* POI ************
// write amount of POIs and ways for each zoom level
// TODO is this computation correct? Ways that have an associated zoom level of
// e.g. 9
// are lifted to zoom level 12 for an interval 12,14,17
Map<Byte, List<TDNode>> poisByZoomlevel =
currentTile.poisByZoomlevel(minZoomCurrentInterval, maxMaxZoomlevel);
Map<Byte, List<TDWay>> waysByZoomlevel =
currentTile.waysByZoomlevel(minZoomCurrentInterval, maxMaxZoomlevel);
if (poisByZoomlevel.size() > 0 || waysByZoomlevel.size() > 0) {
int tileContainerStart = tileBuffer.position();
if (debugStrings) {
// write tile header
StringBuilder sb = new StringBuilder();
sb.append(DEBUG_STRING_TILE_HEAD)
.append(tileX)
.append(",")
.append(tileY)
.append(DEBUG_STRING_TILE_TAIL);
tileBuffer.put(sb.toString().getBytes());
// append withespaces so that block has 32 bytes
appendWhitespace(32 - sb.toString().getBytes().length, tileBuffer);
}
short cumulatedPOIs = 0;
short cumulatedWays = 0;
for (byte zoomlevel = minZoomCurrentInterval;
zoomlevel <= maxZoomCurrentInterval;
zoomlevel++) {
if (poisByZoomlevel.get(zoomlevel) != null)
cumulatedPOIs += poisByZoomlevel.get(zoomlevel).size();
if (waysByZoomlevel.get(zoomlevel) != null)
cumulatedWays += waysByZoomlevel.get(zoomlevel).size();
tileBuffer.putShort(cumulatedPOIs);
tileBuffer.putShort(cumulatedWays);
}
// skip 4 bytes, later these 4 bytes will contain the start
// position of the ways in this tile
int fileIndexStartWayContainer = tileBuffer.position();
tileBuffer.position(fileIndexStartWayContainer + 4);
// write POIs for each zoom level beginning with lowest zoom level
for (byte zoomlevel = minZoomCurrentInterval;
zoomlevel <= maxZoomCurrentInterval;
zoomlevel++) {
List<TDNode> pois = poisByZoomlevel.get(zoomlevel);
if (pois == null) continue;
for (TDNode poi : pois) {
if (debugStrings) {
StringBuilder sb = new StringBuilder();
sb.append(DEBUG_STRING_POI_HEAD).append(poi.getId()).append(DEBUG_STRING_POI_TAIL);
tileBuffer.put(sb.toString().getBytes());
// append withespaces so that block has 32 bytes
appendWhitespace(32 - sb.toString().getBytes().length, tileBuffer);
}
// write poi features to the file
tileBuffer.putInt(poi.getLatitude());
tileBuffer.putInt(poi.getLongitude());
// write byte with layer and tag amount info
tileBuffer.put(
buildLayerTagAmountByte(
poi.getLayer(), poi.getTags() == null ? 0 : (short) poi.getTags().size()));
// write tag ids to the file
if (poi.getTags() != null) {
for (PoiEnum poiEnum : poi.getTags()) {
tileBuffer.putShort((short) poiEnum.ordinal());
}
}
// write byte with bits set to 1 if the poi has a name, an elevation
// or a housenumber
tileBuffer.put(
buildInfoByteForPOI(poi.getName(), poi.getElevation(), poi.getHouseNumber()));
if (poi.getName() != null && poi.getName().length() > 0) {
writeUTF8(poi.getName(), tileBuffer);
}
if (poi.getElevation() != 0) {
tileBuffer.putShort(poi.getElevation());
}
if (poi.getHouseNumber() != null && poi.getHouseNumber().length() > 0) {
writeUTF8(poi.getHouseNumber(), tileBuffer);
}
}
} // end for loop over POIs
// write offset to first way in the tile header
tileBuffer.putInt(fileIndexStartWayContainer, tileBuffer.position() - tileContainerStart);
// ************* WAYS ************
// write ways
for (byte zoomlevel = minZoomCurrentInterval;
zoomlevel <= maxZoomCurrentInterval;
zoomlevel++) {
List<TDWay> ways = waysByZoomlevel.get(zoomlevel);
if (ways == null) continue;
// use executor service to parallelize computation of subtile bitmasks
// for all
// ways in the current tile
short[] bitmaskComputationResults = computeSubtileBitmasks(ways, currentTileCoordinate);
assert bitmaskComputationResults.length == ways.size();
// needed to access bitmask computation results in the foreach loop
int i = 0;
for (TDWay way : ways) {
// // INNER WAY
// // inner ways will be written as part of the outer way
// if (way.isInnerWay())
// continue;
int startIndexWay = tileBuffer.position();
WayNodePreprocessingResult wayNodePreprocessingResult =
preprocessWayNodes(
way,
waynodeCompression,
pixelCompression,
polygonClipping,
maxZoomCurrentInterval,
minZoomCurrentInterval,
currentTileCoordinate);
if (wayNodePreprocessingResult == null) {
continue;
}
if (debugStrings) {
StringBuilder sb = new StringBuilder();
sb.append(DEBUG_STRING_WAY_HEAD).append(way.getId()).append(DEBUG_STRING_WAY_TAIL);
tileBuffer.put(sb.toString().getBytes());
// append withespaces so that block has 32 bytes
appendWhitespace(32 - sb.toString().getBytes().length, tileBuffer);
}
// skip 4 bytes to reserve space for way size
int startIndexWaySize = tileBuffer.position();
tileBuffer.position(startIndexWaySize + 4);
// write way features
// short bitmask = GeoUtils.computeBitmask(way,
// currentTileCoordinate);
// short bitmask = (short) 0xffff;
tileBuffer.putShort(bitmaskComputationResults[i++]);
// write byte with layer and tag amount
tileBuffer.put(
buildLayerTagAmountByte(
way.getLayer(), way.getTags() == null ? 0 : (short) way.getTags().size()));
// set type of the way node compression
int compressionType = wayNodePreprocessingResult.getCompressionType();
// write byte with amount of tags which are rendered
tileBuffer.put(buildRenderTagWayNodeCompressionByte(way.getTags(), compressionType));
// write tag bitmap
tileBuffer.put(buildTagBitmapByte(way.getTags()));
// file.writeByte((byte) 0xff);
// write tag ids
if (way.getTags() != null) {
for (WayEnum wayEnum : way.getTags()) {
tileBuffer.putShort((short) wayEnum.ordinal());
}
}
// write the amount of way nodes to the file
tileBuffer.putShort(
(short) (wayNodePreprocessingResult.getWaynodesAsList().size() / 2));
// write the way nodes:
// the first node is always stored with four bytes
// the remaining way node differences are stored according to the
// compression type
writeWayNodes(
wayNodePreprocessingResult.getWaynodesAsList(),
wayNodePreprocessingResult.getCompressionType(),
tileBuffer);
// write a byte with name, label and way type information
tileBuffer.put(buildInfoByteForWay(way.getName(), way.getWaytype(), way.getRef()));
// // if the way has a name, write it to the file
if (way.getName() != null && way.getName().length() > 0) {
writeUTF8(way.getName(), tileBuffer);
}
// if the way has a ref, write it to the file
if (way.getRef() != null && way.getRef().length() > 0) {
writeUTF8(way.getRef(), tileBuffer);
}
//
// // // if the way has a label position write it to the file
// // if (labelPositionLatitude != 0 && labelPositionLongitude != 0)
// {
// // raf.writeInt(labelPositionLatitude);
// // raf.writeInt(labelPositionLongitude);
// // }
//
// *********MULTIPOLYGON PROCESSING***********
if (way.getWaytype() == 3
&& dataStore.getInnerWaysOfMultipolygon(way.getId()) != null) {
List<TDWay> innerways = dataStore.getInnerWaysOfMultipolygon(way.getId());
if (innerways == null) {
tileBuffer.put((byte) 0);
} else {
tileBuffer.put((byte) innerways.size());
for (TDWay innerway : innerways) {
WayNodePreprocessingResult innerWayNodePreprocessingResult =
preprocessWayNodes(
innerway,
waynodeCompression,
pixelCompression,
false,
maxZoomCurrentInterval,
minZoomCurrentInterval,
currentTileCoordinate);
// write the amount of way nodes to the file
tileBuffer.putShort(
(short) (innerWayNodePreprocessingResult.getWaynodesAsList().size() / 2));
writeWayNodes(
innerWayNodePreprocessingResult.getWaynodesAsList(),
wayNodePreprocessingResult.getCompressionType(),
tileBuffer);
}
}
}
// write the size of the way to the file
tileBuffer.putInt(startIndexWaySize, tileBuffer.position() - startIndexWay);
}
} // end for loop over ways
} // end if clause checking if tile is empty or not
long tileSize = tileBuffer.position() - currentTileOffsetInBuffer;
currentSubfileOffset += tileSize;
// if necessary, allocate new buffer
if (tileBuffer.remaining() < MIN_TILE_BUFFER_SIZE)
tileBuffer =
randomAccessFile
.getChannel()
.map(
MapMode.READ_WRITE,
startPositionSubfile + currentSubfileOffset,
TILE_BUFFER_SIZE);
tilesProcessed++;
if (tilesProcessed % fivePercentOfTilesToProcess == 0) {
logger.info(
"written " + (tilesProcessed / fivePercentOfTilesToProcess) * 5 + "% of file");
}
} // end for loop over tile columns
} // /end for loop over tile rows
// return size of sub file in bytes
return currentSubfileOffset;
}
private long writeContainerHeader(
long date,
int version,
short tilePixel,
String comment,
boolean debugStrings,
boolean waynodeCompression,
boolean polygonClipping,
boolean pixelCompression,
GeoCoordinate mapStartPosition)
throws IOException {
// get metadata for the map file
int numberOfZoomIntervals = dataStore.getZoomIntervalConfiguration().getNumberOfZoomIntervals();
logger.fine("writing header");
MappedByteBuffer containerHeaderBuffer =
randomAccessFile.getChannel().map(MapMode.READ_WRITE, 0, HEADER_BUFFER_SIZE);
// write file header
// magic byte
byte[] magicBytes = MAGIC_BYTE.getBytes();
containerHeaderBuffer.put(magicBytes);
// write container header size
int headerSizePosition = containerHeaderBuffer.position();
containerHeaderBuffer.position(headerSizePosition + 4);
// version number of the binary file format
containerHeaderBuffer.putInt(version);
// meta info byte
containerHeaderBuffer.put(
buildMetaInfoByte(
debugStrings,
mapStartPosition != null,
pixelCompression,
polygonClipping,
waynodeCompression));
// amount of map files inside this file
containerHeaderBuffer.put((byte) numberOfZoomIntervals);
// projection type
writeUTF8(PROJECTION, containerHeaderBuffer);
// width and height of a tile in pixel
containerHeaderBuffer.putShort(tilePixel);
logger.fine(
"Bounding box for file: "
+ dataStore.getBoundingBox().maxLatitudeE6
+ ", "
+ dataStore.getBoundingBox().minLongitudeE6
+ ", "
+ dataStore.getBoundingBox().minLatitudeE6
+ ", "
+ dataStore.getBoundingBox().maxLongitudeE6);
// upper left corner of the bounding box
containerHeaderBuffer.putInt(dataStore.getBoundingBox().maxLatitudeE6);
containerHeaderBuffer.putInt(dataStore.getBoundingBox().minLongitudeE6);
containerHeaderBuffer.putInt(dataStore.getBoundingBox().minLatitudeE6);
containerHeaderBuffer.putInt(dataStore.getBoundingBox().maxLongitudeE6);
if (mapStartPosition != null) {
containerHeaderBuffer.putInt(mapStartPosition.getLatitudeE6());
containerHeaderBuffer.putInt(mapStartPosition.getLongitudeE6());
}
// date of the map data
containerHeaderBuffer.putLong(date);
// store the mapping of tags to tag ids
containerHeaderBuffer.putShort((short) PoiEnum.values().length);
for (PoiEnum poiEnum : PoiEnum.values()) {
writeUTF8(poiEnum.toString(), containerHeaderBuffer);
containerHeaderBuffer.putShort((short) poiEnum.ordinal());
}
containerHeaderBuffer.putShort((short) WayEnum.values().length);
for (WayEnum wayEnum : WayEnum.values()) {
writeUTF8(wayEnum.toString(), containerHeaderBuffer);
containerHeaderBuffer.putShort((short) wayEnum.ordinal());
}
// comment
if (comment != null && !comment.equals("")) {
writeUTF8(comment, containerHeaderBuffer);
} else {
writeUTF8("", containerHeaderBuffer);
}
// initialize buffer for writing zoom interval configurations
bufferZoomIntervalConfig =
randomAccessFile
.getChannel()
.map(
MapMode.READ_WRITE,
containerHeaderBuffer.position(),
SIZE_ZOOMINTERVAL_CONFIGURATION * numberOfZoomIntervals);
containerHeaderBuffer.position(
containerHeaderBuffer.position() + SIZE_ZOOMINTERVAL_CONFIGURATION * numberOfZoomIntervals);
// -4 bytes of header size variable itself
int headerSize = containerHeaderBuffer.position() - headerSizePosition - 4;
containerHeaderBuffer.putInt(headerSizePosition, headerSize);
return containerHeaderBuffer.position();
}