Exemplo n.º 1
0
  private void writeSubfileMetaDataToContainerHeader(
      int i, long startIndexOfSubfile, long subfileSize) {

    // HEADER META DATA FOR SUB FILE
    // write zoom interval configuration to header
    byte minZoomCurrentInterval = dataStore.getZoomIntervalConfiguration().getMinZoom(i);
    byte maxZoomCurrentInterval = dataStore.getZoomIntervalConfiguration().getMaxZoom(i);
    byte baseZoomCurrentInterval = dataStore.getZoomIntervalConfiguration().getBaseZoom(i);

    bufferZoomIntervalConfig.put(baseZoomCurrentInterval);
    bufferZoomIntervalConfig.put(minZoomCurrentInterval);
    bufferZoomIntervalConfig.put(maxZoomCurrentInterval);
    bufferZoomIntervalConfig.put(Serializer.getFiveBytes(startIndexOfSubfile));
    bufferZoomIntervalConfig.put(Serializer.getFiveBytes(subfileSize));
  }
Exemplo n.º 2
0
 MapFileWriter(TileBasedDataStore dataStore, RandomAccessFile file, int threadpoolSize) {
   super();
   this.dataStore = dataStore;
   this.randomAccessFile = file;
   fivePercentOfTilesToProcess = dataStore.cumulatedNumberOfTiles() / 20;
   if (fivePercentOfTilesToProcess == 0) fivePercentOfTilesToProcess = 1;
   executorService = Executors.newFixedThreadPool(threadpoolSize);
 }
Exemplo n.º 3
0
  final void writeFile(
      long date,
      int version,
      short tilePixel,
      String comment,
      boolean debugStrings,
      boolean waynodeCompression,
      boolean polygonClipping,
      boolean pixelCompression,
      GeoCoordinate mapStartPosition)
      throws IOException {

    // CONTAINER HEADER
    long totalHeaderSize =
        writeContainerHeader(
            date,
            version,
            tilePixel,
            comment,
            debugStrings,
            waynodeCompression,
            polygonClipping,
            pixelCompression,
            mapStartPosition);

    int n_zoom_intervals = dataStore.getZoomIntervalConfiguration().getNumberOfZoomIntervals();

    // SUB FILES
    // for each zoom interval write a sub file
    long currentFileSize = totalHeaderSize;
    for (int i = 0; i < n_zoom_intervals; i++) {
      // SUB FILE INDEX AND DATA
      long subfileSize =
          writeSubfile(
              currentFileSize,
              i,
              debugStrings,
              waynodeCompression,
              polygonClipping,
              pixelCompression);
      // SUB FILE META DATA IN CONTAINER HEADER
      writeSubfileMetaDataToContainerHeader(i, currentFileSize, subfileSize);
      currentFileSize += subfileSize;
    }

    randomAccessFile.setLength(currentFileSize);
    randomAccessFile.close();
  }
Exemplo n.º 4
0
  private long writeSubfile(
      long startPositionSubfile,
      int zoomIntervalIndex,
      boolean debugStrings,
      boolean waynodeCompression,
      boolean polygonClipping,
      boolean pixelCompression)
      throws IOException {

    logger.fine(
        "writing data for zoom interval "
            + zoomIntervalIndex
            + ", number of tiles: "
            + dataStore.numberOfHorizontalTiles(zoomIntervalIndex)
                * dataStore.numberOfVerticalTiles(zoomIntervalIndex));

    TileCoordinate upperLeft = dataStore.getUpperLeft(zoomIntervalIndex);
    int lengthX = dataStore.numberOfHorizontalTiles(zoomIntervalIndex);
    int lengthY = dataStore.numberOfVerticalTiles(zoomIntervalIndex);

    byte minZoomCurrentInterval =
        dataStore.getZoomIntervalConfiguration().getMinZoom(zoomIntervalIndex);
    byte maxZoomCurrentInterval =
        dataStore.getZoomIntervalConfiguration().getMaxZoom(zoomIntervalIndex);
    byte baseZoomCurrentInterval =
        dataStore.getZoomIntervalConfiguration().getBaseZoom(zoomIntervalIndex);
    byte maxMaxZoomlevel = dataStore.getZoomIntervalConfiguration().getMaxMaxZoom();

    int tileAmountInBytes =
        dataStore.numberOfHorizontalTiles(zoomIntervalIndex)
            * dataStore.numberOfVerticalTiles(zoomIntervalIndex)
            * BYTE_AMOUNT_SUBFILE_INDEX_PER_TILE;
    int indexBufferSize =
        tileAmountInBytes + (debugStrings ? DEBUG_INDEX_START_STRING.getBytes().length : 0);
    MappedByteBuffer indexBuffer =
        randomAccessFile
            .getChannel()
            .map(MapMode.READ_WRITE, startPositionSubfile, indexBufferSize);
    MappedByteBuffer tileBuffer =
        randomAccessFile
            .getChannel()
            .map(MapMode.READ_WRITE, startPositionSubfile + indexBufferSize, TILE_BUFFER_SIZE);

    long currentSubfileOffset = indexBufferSize;

    for (int tileY = upperLeft.getY(); tileY < upperLeft.getY() + lengthY; tileY++) {
      for (int tileX = upperLeft.getX(); tileX < upperLeft.getX() + lengthX; tileX++) {
        // logger.info("writing data for tile (" + tileX + ", " + tileY + ")");

        long currentTileOffsetInBuffer = tileBuffer.position();
        TileCoordinate currentTileCoordinate =
            new TileCoordinate(tileX, tileY, baseZoomCurrentInterval);

        // seek to index frame of this tile and write relative offset of this
        // tile as five bytes to the index
        indexBuffer.put(Serializer.getFiveBytes(currentSubfileOffset));

        // get statistics for tile
        TileData currentTile = dataStore.getTile(zoomIntervalIndex, tileX, tileY);

        // ************* POI ************
        // write amount of POIs and ways for each zoom level
        // TODO is this computation correct? Ways that have an associated zoom level of
        // e.g. 9
        // are lifted to zoom level 12 for an interval 12,14,17
        Map<Byte, List<TDNode>> poisByZoomlevel =
            currentTile.poisByZoomlevel(minZoomCurrentInterval, maxMaxZoomlevel);
        Map<Byte, List<TDWay>> waysByZoomlevel =
            currentTile.waysByZoomlevel(minZoomCurrentInterval, maxMaxZoomlevel);

        if (poisByZoomlevel.size() > 0 || waysByZoomlevel.size() > 0) {
          int tileContainerStart = tileBuffer.position();
          if (debugStrings) {
            // write tile header
            StringBuilder sb = new StringBuilder();
            sb.append(DEBUG_STRING_TILE_HEAD)
                .append(tileX)
                .append(",")
                .append(tileY)
                .append(DEBUG_STRING_TILE_TAIL);
            tileBuffer.put(sb.toString().getBytes());
            // append withespaces so that block has 32 bytes
            appendWhitespace(32 - sb.toString().getBytes().length, tileBuffer);
          }

          short cumulatedPOIs = 0;
          short cumulatedWays = 0;
          for (byte zoomlevel = minZoomCurrentInterval;
              zoomlevel <= maxZoomCurrentInterval;
              zoomlevel++) {
            if (poisByZoomlevel.get(zoomlevel) != null)
              cumulatedPOIs += poisByZoomlevel.get(zoomlevel).size();
            if (waysByZoomlevel.get(zoomlevel) != null)
              cumulatedWays += waysByZoomlevel.get(zoomlevel).size();
            tileBuffer.putShort(cumulatedPOIs);
            tileBuffer.putShort(cumulatedWays);
          }

          // skip 4 bytes, later these 4 bytes will contain the start
          // position of the ways in this tile
          int fileIndexStartWayContainer = tileBuffer.position();
          tileBuffer.position(fileIndexStartWayContainer + 4);

          // write POIs for each zoom level beginning with lowest zoom level
          for (byte zoomlevel = minZoomCurrentInterval;
              zoomlevel <= maxZoomCurrentInterval;
              zoomlevel++) {
            List<TDNode> pois = poisByZoomlevel.get(zoomlevel);
            if (pois == null) continue;
            for (TDNode poi : pois) {
              if (debugStrings) {
                StringBuilder sb = new StringBuilder();
                sb.append(DEBUG_STRING_POI_HEAD).append(poi.getId()).append(DEBUG_STRING_POI_TAIL);
                tileBuffer.put(sb.toString().getBytes());
                // append withespaces so that block has 32 bytes
                appendWhitespace(32 - sb.toString().getBytes().length, tileBuffer);
              }

              // write poi features to the file
              tileBuffer.putInt(poi.getLatitude());
              tileBuffer.putInt(poi.getLongitude());

              // write byte with layer and tag amount info
              tileBuffer.put(
                  buildLayerTagAmountByte(
                      poi.getLayer(), poi.getTags() == null ? 0 : (short) poi.getTags().size()));

              // write tag ids to the file
              if (poi.getTags() != null) {
                for (PoiEnum poiEnum : poi.getTags()) {
                  tileBuffer.putShort((short) poiEnum.ordinal());
                }
              }

              // write byte with bits set to 1 if the poi has a name, an elevation
              // or a housenumber
              tileBuffer.put(
                  buildInfoByteForPOI(poi.getName(), poi.getElevation(), poi.getHouseNumber()));

              if (poi.getName() != null && poi.getName().length() > 0) {
                writeUTF8(poi.getName(), tileBuffer);
              }
              if (poi.getElevation() != 0) {
                tileBuffer.putShort(poi.getElevation());
              }
              if (poi.getHouseNumber() != null && poi.getHouseNumber().length() > 0) {
                writeUTF8(poi.getHouseNumber(), tileBuffer);
              }
            }
          } // end for loop over POIs

          // write offset to first way in the tile header
          tileBuffer.putInt(fileIndexStartWayContainer, tileBuffer.position() - tileContainerStart);

          // ************* WAYS ************
          // write ways
          for (byte zoomlevel = minZoomCurrentInterval;
              zoomlevel <= maxZoomCurrentInterval;
              zoomlevel++) {
            List<TDWay> ways = waysByZoomlevel.get(zoomlevel);
            if (ways == null) continue;

            // use executor service to parallelize computation of subtile bitmasks
            // for all
            // ways in the current tile
            short[] bitmaskComputationResults = computeSubtileBitmasks(ways, currentTileCoordinate);
            assert bitmaskComputationResults.length == ways.size();
            // needed to access bitmask computation results in the foreach loop
            int i = 0;
            for (TDWay way : ways) {
              // // INNER WAY
              // // inner ways will be written as part of the outer way
              // if (way.isInnerWay())
              // continue;
              int startIndexWay = tileBuffer.position();

              WayNodePreprocessingResult wayNodePreprocessingResult =
                  preprocessWayNodes(
                      way,
                      waynodeCompression,
                      pixelCompression,
                      polygonClipping,
                      maxZoomCurrentInterval,
                      minZoomCurrentInterval,
                      currentTileCoordinate);

              if (wayNodePreprocessingResult == null) {
                continue;
              }
              if (debugStrings) {
                StringBuilder sb = new StringBuilder();
                sb.append(DEBUG_STRING_WAY_HEAD).append(way.getId()).append(DEBUG_STRING_WAY_TAIL);
                tileBuffer.put(sb.toString().getBytes());
                // append withespaces so that block has 32 bytes
                appendWhitespace(32 - sb.toString().getBytes().length, tileBuffer);
              }

              // skip 4 bytes to reserve space for way size
              int startIndexWaySize = tileBuffer.position();
              tileBuffer.position(startIndexWaySize + 4);

              // write way features
              // short bitmask = GeoUtils.computeBitmask(way,
              // currentTileCoordinate);
              // short bitmask = (short) 0xffff;
              tileBuffer.putShort(bitmaskComputationResults[i++]);

              // write byte with layer and tag amount
              tileBuffer.put(
                  buildLayerTagAmountByte(
                      way.getLayer(), way.getTags() == null ? 0 : (short) way.getTags().size()));

              // set type of the way node compression
              int compressionType = wayNodePreprocessingResult.getCompressionType();

              // write byte with amount of tags which are rendered
              tileBuffer.put(buildRenderTagWayNodeCompressionByte(way.getTags(), compressionType));

              // write tag bitmap
              tileBuffer.put(buildTagBitmapByte(way.getTags()));
              // file.writeByte((byte) 0xff);

              // write tag ids
              if (way.getTags() != null) {
                for (WayEnum wayEnum : way.getTags()) {
                  tileBuffer.putShort((short) wayEnum.ordinal());
                }
              }
              // write the amount of way nodes to the file
              tileBuffer.putShort(
                  (short) (wayNodePreprocessingResult.getWaynodesAsList().size() / 2));

              // write the way nodes:
              // the first node is always stored with four bytes
              // the remaining way node differences are stored according to the
              // compression type
              writeWayNodes(
                  wayNodePreprocessingResult.getWaynodesAsList(),
                  wayNodePreprocessingResult.getCompressionType(),
                  tileBuffer);

              // write a byte with name, label and way type information
              tileBuffer.put(buildInfoByteForWay(way.getName(), way.getWaytype(), way.getRef()));

              // // if the way has a name, write it to the file
              if (way.getName() != null && way.getName().length() > 0) {
                writeUTF8(way.getName(), tileBuffer);
              }

              // if the way has a ref, write it to the file
              if (way.getRef() != null && way.getRef().length() > 0) {
                writeUTF8(way.getRef(), tileBuffer);
              }
              //
              // // // if the way has a label position write it to the file
              // // if (labelPositionLatitude != 0 && labelPositionLongitude != 0)
              // {
              // // raf.writeInt(labelPositionLatitude);
              // // raf.writeInt(labelPositionLongitude);
              // // }
              //
              // *********MULTIPOLYGON PROCESSING***********
              if (way.getWaytype() == 3
                  && dataStore.getInnerWaysOfMultipolygon(way.getId()) != null) {
                List<TDWay> innerways = dataStore.getInnerWaysOfMultipolygon(way.getId());

                if (innerways == null) {
                  tileBuffer.put((byte) 0);
                } else {
                  tileBuffer.put((byte) innerways.size());
                  for (TDWay innerway : innerways) {
                    WayNodePreprocessingResult innerWayNodePreprocessingResult =
                        preprocessWayNodes(
                            innerway,
                            waynodeCompression,
                            pixelCompression,
                            false,
                            maxZoomCurrentInterval,
                            minZoomCurrentInterval,
                            currentTileCoordinate);
                    // write the amount of way nodes to the file
                    tileBuffer.putShort(
                        (short) (innerWayNodePreprocessingResult.getWaynodesAsList().size() / 2));
                    writeWayNodes(
                        innerWayNodePreprocessingResult.getWaynodesAsList(),
                        wayNodePreprocessingResult.getCompressionType(),
                        tileBuffer);
                  }
                }
              }
              // write the size of the way to the file
              tileBuffer.putInt(startIndexWaySize, tileBuffer.position() - startIndexWay);
            }
          } // end for loop over ways
        } // end if clause checking if tile is empty or not
        long tileSize = tileBuffer.position() - currentTileOffsetInBuffer;
        currentSubfileOffset += tileSize;

        // if necessary, allocate new buffer
        if (tileBuffer.remaining() < MIN_TILE_BUFFER_SIZE)
          tileBuffer =
              randomAccessFile
                  .getChannel()
                  .map(
                      MapMode.READ_WRITE,
                      startPositionSubfile + currentSubfileOffset,
                      TILE_BUFFER_SIZE);

        tilesProcessed++;
        if (tilesProcessed % fivePercentOfTilesToProcess == 0) {
          logger.info(
              "written " + (tilesProcessed / fivePercentOfTilesToProcess) * 5 + "% of file");
        }
      } // end for loop over tile columns
    } // /end for loop over tile rows

    // return size of sub file in bytes
    return currentSubfileOffset;
  }
Exemplo n.º 5
0
  private long writeContainerHeader(
      long date,
      int version,
      short tilePixel,
      String comment,
      boolean debugStrings,
      boolean waynodeCompression,
      boolean polygonClipping,
      boolean pixelCompression,
      GeoCoordinate mapStartPosition)
      throws IOException {

    // get metadata for the map file
    int numberOfZoomIntervals = dataStore.getZoomIntervalConfiguration().getNumberOfZoomIntervals();

    logger.fine("writing header");

    MappedByteBuffer containerHeaderBuffer =
        randomAccessFile.getChannel().map(MapMode.READ_WRITE, 0, HEADER_BUFFER_SIZE);

    // write file header
    // magic byte
    byte[] magicBytes = MAGIC_BYTE.getBytes();
    containerHeaderBuffer.put(magicBytes);

    // write container header size
    int headerSizePosition = containerHeaderBuffer.position();
    containerHeaderBuffer.position(headerSizePosition + 4);

    // version number of the binary file format
    containerHeaderBuffer.putInt(version);

    // meta info byte
    containerHeaderBuffer.put(
        buildMetaInfoByte(
            debugStrings,
            mapStartPosition != null,
            pixelCompression,
            polygonClipping,
            waynodeCompression));

    // amount of map files inside this file
    containerHeaderBuffer.put((byte) numberOfZoomIntervals);

    // projection type
    writeUTF8(PROJECTION, containerHeaderBuffer);

    // width and height of a tile in pixel
    containerHeaderBuffer.putShort(tilePixel);

    logger.fine(
        "Bounding box for file: "
            + dataStore.getBoundingBox().maxLatitudeE6
            + ", "
            + dataStore.getBoundingBox().minLongitudeE6
            + ", "
            + dataStore.getBoundingBox().minLatitudeE6
            + ", "
            + dataStore.getBoundingBox().maxLongitudeE6);
    // upper left corner of the bounding box
    containerHeaderBuffer.putInt(dataStore.getBoundingBox().maxLatitudeE6);
    containerHeaderBuffer.putInt(dataStore.getBoundingBox().minLongitudeE6);
    containerHeaderBuffer.putInt(dataStore.getBoundingBox().minLatitudeE6);
    containerHeaderBuffer.putInt(dataStore.getBoundingBox().maxLongitudeE6);

    if (mapStartPosition != null) {
      containerHeaderBuffer.putInt(mapStartPosition.getLatitudeE6());
      containerHeaderBuffer.putInt(mapStartPosition.getLongitudeE6());
    }

    // date of the map data
    containerHeaderBuffer.putLong(date);

    // store the mapping of tags to tag ids
    containerHeaderBuffer.putShort((short) PoiEnum.values().length);
    for (PoiEnum poiEnum : PoiEnum.values()) {
      writeUTF8(poiEnum.toString(), containerHeaderBuffer);
      containerHeaderBuffer.putShort((short) poiEnum.ordinal());
    }
    containerHeaderBuffer.putShort((short) WayEnum.values().length);
    for (WayEnum wayEnum : WayEnum.values()) {
      writeUTF8(wayEnum.toString(), containerHeaderBuffer);
      containerHeaderBuffer.putShort((short) wayEnum.ordinal());
    }

    // comment
    if (comment != null && !comment.equals("")) {
      writeUTF8(comment, containerHeaderBuffer);
    } else {
      writeUTF8("", containerHeaderBuffer);
    }

    // initialize buffer for writing zoom interval configurations
    bufferZoomIntervalConfig =
        randomAccessFile
            .getChannel()
            .map(
                MapMode.READ_WRITE,
                containerHeaderBuffer.position(),
                SIZE_ZOOMINTERVAL_CONFIGURATION * numberOfZoomIntervals);

    containerHeaderBuffer.position(
        containerHeaderBuffer.position() + SIZE_ZOOMINTERVAL_CONFIGURATION * numberOfZoomIntervals);

    // -4 bytes of header size variable itself
    int headerSize = containerHeaderBuffer.position() - headerSizePosition - 4;
    containerHeaderBuffer.putInt(headerSizePosition, headerSize);

    return containerHeaderBuffer.position();
  }