Ejemplo n.º 1
0
  private void writeWayNodes(List<Integer> waynodes, int compressionType, MappedByteBuffer buffer) {
    if (!waynodes.isEmpty() && waynodes.size() % 2 == 0) {
      Iterator<Integer> waynodeIterator = waynodes.iterator();
      buffer.putInt(waynodeIterator.next());
      buffer.putInt(waynodeIterator.next());

      while (waynodeIterator.hasNext()) {
        switch (compressionType) {
          case 0:
            buffer.putInt(waynodeIterator.next().intValue());
            buffer.putInt(waynodeIterator.next().intValue());
            break;
          case 1:
            buffer.put(Serializer.getSignedThreeBytes(waynodeIterator.next().intValue()));
            buffer.put(Serializer.getSignedThreeBytes(waynodeIterator.next().intValue()));
            break;
          case 2:
            buffer.putShort(waynodeIterator.next().shortValue());
            buffer.putShort(waynodeIterator.next().shortValue());
            break;
          case 3:
            buffer.put(waynodeIterator.next().byteValue());
            buffer.put(waynodeIterator.next().byteValue());
            break;
        }
      }
    }
  }
Ejemplo n.º 2
0
 private void writeRecordHeaderToMappedBuffer(RecordHeader header) {
   mappedDataBuffer.position(header.getPosition());
   mappedDataBuffer.putInt(header.getDataSize());
   mappedDataBuffer.put(header.getFragmented());
   if (header.isFragmented()) {
     mappedDataBuffer.putInt(header.getNextPos());
   }
 }
Ejemplo n.º 3
0
 @SuppressWarnings("deprecation")
 @Override
 protected void writeToBuffer(MappedByteBuffer buffer, Tuple e) {
   KeyValue kv = KeyValueUtil.ensureKeyValue(e.getValue(0));
   buffer.putInt(kv.getLength() + Bytes.SIZEOF_INT);
   buffer.putInt(kv.getLength());
   buffer.put(kv.getBuffer(), kv.getOffset(), kv.getLength());
 }
Ejemplo n.º 4
0
  /* (non-Javadoc)
   * @see com.ongraphdb.store.DiskSore1#start()
   */
  public void start() throws IOException {
    File file = new File(dataFileName);
    boolean newStore = file.exists() ? false : true;

    RandomAccessFile dataFile = new RandomAccessFile(dataFileName, "rw");
    dataFile.setLength(initialFileSize);
    dataChannel = dataFile.getChannel();
    dataLock = dataChannel.lock();
    mappedDataBuffer = dataChannel.map(MapMode.READ_WRITE, 0, dataMappedMemorySize);

    if (newStore) {
      nextPosition = NEXT_BYTES;
      mappedDataBuffer.putInt(nextPosition);
    } else {
      nextPosition = mappedDataBuffer.getInt();
    }
    shutdownHookThread =
        new Thread() {
          public void run() {
            try {
              mappedDataBuffer.force();
              dataLock.release();
              dataChannel.close();
              unmap(mappedDataBuffer);
            } catch (Exception e) {
              e.printStackTrace();
            }
          }
        };
    Runtime.getRuntime().addShutdownHook(shutdownHookThread);
  }
Ejemplo n.º 5
0
 private int getNextPosition(int increment) {
   int result = nextPosition;
   nextPosition = nextPosition + increment;
   mappedDataBuffer.position(0);
   mappedDataBuffer.putInt(nextPosition);
   return result;
 }
Ejemplo n.º 6
0
    // Flush the current page to the memory mapped byte buffer
    private void flushBuffer() throws BufferOverflowException {
      Collection<byte[]> values = ImmutableMap.copyOf(pageMap).values();
      buffer.clear();
      // number of elements
      buffer.putInt(values.size());
      for (byte[] value : values) {
        // element length
        buffer.putInt(value.length);
        // element
        buffer.put(value, 0, value.length);
      }

      // Reset page stats
      pageMap.clear();
      totalResultSize = 0;
    }
Ejemplo n.º 7
0
  public void test_position() throws IOException {
    File tmp = File.createTempFile("hmy", "tmp");
    tmp.deleteOnExit();
    RandomAccessFile f = new RandomAccessFile(tmp, "rw");
    FileChannel ch = f.getChannel();
    MappedByteBuffer mbb = ch.map(MapMode.READ_WRITE, 0L, 100L);
    ch.close();

    mbb.putInt(1, 1);
    mbb.position(50);
    mbb.putInt(50);

    mbb.flip();
    mbb.get();
    assertEquals(1, mbb.getInt());

    mbb.position(50);
    assertEquals(50, mbb.getInt());
  }
Ejemplo n.º 8
0
    public static void main(String[] args) throws Exception {
      MappedByteBuffer map = openMemoryMappedFile();

      for (int value = 0; value < MESSAGES; value++) {

        while (map.get(TAG_INDEX) != CAN_WRITE) {
          loopWait();
        }
        map.putInt(DATA_INDEX, value);
        map.put(TAG_INDEX, CAN_READ);
      }
    }
Ejemplo n.º 9
0
  private long writeSubfile(
      long startPositionSubfile,
      int zoomIntervalIndex,
      boolean debugStrings,
      boolean waynodeCompression,
      boolean polygonClipping,
      boolean pixelCompression)
      throws IOException {

    logger.fine(
        "writing data for zoom interval "
            + zoomIntervalIndex
            + ", number of tiles: "
            + dataStore.numberOfHorizontalTiles(zoomIntervalIndex)
                * dataStore.numberOfVerticalTiles(zoomIntervalIndex));

    TileCoordinate upperLeft = dataStore.getUpperLeft(zoomIntervalIndex);
    int lengthX = dataStore.numberOfHorizontalTiles(zoomIntervalIndex);
    int lengthY = dataStore.numberOfVerticalTiles(zoomIntervalIndex);

    byte minZoomCurrentInterval =
        dataStore.getZoomIntervalConfiguration().getMinZoom(zoomIntervalIndex);
    byte maxZoomCurrentInterval =
        dataStore.getZoomIntervalConfiguration().getMaxZoom(zoomIntervalIndex);
    byte baseZoomCurrentInterval =
        dataStore.getZoomIntervalConfiguration().getBaseZoom(zoomIntervalIndex);
    byte maxMaxZoomlevel = dataStore.getZoomIntervalConfiguration().getMaxMaxZoom();

    int tileAmountInBytes =
        dataStore.numberOfHorizontalTiles(zoomIntervalIndex)
            * dataStore.numberOfVerticalTiles(zoomIntervalIndex)
            * BYTE_AMOUNT_SUBFILE_INDEX_PER_TILE;
    int indexBufferSize =
        tileAmountInBytes + (debugStrings ? DEBUG_INDEX_START_STRING.getBytes().length : 0);
    MappedByteBuffer indexBuffer =
        randomAccessFile
            .getChannel()
            .map(MapMode.READ_WRITE, startPositionSubfile, indexBufferSize);
    MappedByteBuffer tileBuffer =
        randomAccessFile
            .getChannel()
            .map(MapMode.READ_WRITE, startPositionSubfile + indexBufferSize, TILE_BUFFER_SIZE);

    long currentSubfileOffset = indexBufferSize;

    for (int tileY = upperLeft.getY(); tileY < upperLeft.getY() + lengthY; tileY++) {
      for (int tileX = upperLeft.getX(); tileX < upperLeft.getX() + lengthX; tileX++) {
        // logger.info("writing data for tile (" + tileX + ", " + tileY + ")");

        long currentTileOffsetInBuffer = tileBuffer.position();
        TileCoordinate currentTileCoordinate =
            new TileCoordinate(tileX, tileY, baseZoomCurrentInterval);

        // seek to index frame of this tile and write relative offset of this
        // tile as five bytes to the index
        indexBuffer.put(Serializer.getFiveBytes(currentSubfileOffset));

        // get statistics for tile
        TileData currentTile = dataStore.getTile(zoomIntervalIndex, tileX, tileY);

        // ************* POI ************
        // write amount of POIs and ways for each zoom level
        // TODO is this computation correct? Ways that have an associated zoom level of
        // e.g. 9
        // are lifted to zoom level 12 for an interval 12,14,17
        Map<Byte, List<TDNode>> poisByZoomlevel =
            currentTile.poisByZoomlevel(minZoomCurrentInterval, maxMaxZoomlevel);
        Map<Byte, List<TDWay>> waysByZoomlevel =
            currentTile.waysByZoomlevel(minZoomCurrentInterval, maxMaxZoomlevel);

        if (poisByZoomlevel.size() > 0 || waysByZoomlevel.size() > 0) {
          int tileContainerStart = tileBuffer.position();
          if (debugStrings) {
            // write tile header
            StringBuilder sb = new StringBuilder();
            sb.append(DEBUG_STRING_TILE_HEAD)
                .append(tileX)
                .append(",")
                .append(tileY)
                .append(DEBUG_STRING_TILE_TAIL);
            tileBuffer.put(sb.toString().getBytes());
            // append withespaces so that block has 32 bytes
            appendWhitespace(32 - sb.toString().getBytes().length, tileBuffer);
          }

          short cumulatedPOIs = 0;
          short cumulatedWays = 0;
          for (byte zoomlevel = minZoomCurrentInterval;
              zoomlevel <= maxZoomCurrentInterval;
              zoomlevel++) {
            if (poisByZoomlevel.get(zoomlevel) != null)
              cumulatedPOIs += poisByZoomlevel.get(zoomlevel).size();
            if (waysByZoomlevel.get(zoomlevel) != null)
              cumulatedWays += waysByZoomlevel.get(zoomlevel).size();
            tileBuffer.putShort(cumulatedPOIs);
            tileBuffer.putShort(cumulatedWays);
          }

          // skip 4 bytes, later these 4 bytes will contain the start
          // position of the ways in this tile
          int fileIndexStartWayContainer = tileBuffer.position();
          tileBuffer.position(fileIndexStartWayContainer + 4);

          // write POIs for each zoom level beginning with lowest zoom level
          for (byte zoomlevel = minZoomCurrentInterval;
              zoomlevel <= maxZoomCurrentInterval;
              zoomlevel++) {
            List<TDNode> pois = poisByZoomlevel.get(zoomlevel);
            if (pois == null) continue;
            for (TDNode poi : pois) {
              if (debugStrings) {
                StringBuilder sb = new StringBuilder();
                sb.append(DEBUG_STRING_POI_HEAD).append(poi.getId()).append(DEBUG_STRING_POI_TAIL);
                tileBuffer.put(sb.toString().getBytes());
                // append withespaces so that block has 32 bytes
                appendWhitespace(32 - sb.toString().getBytes().length, tileBuffer);
              }

              // write poi features to the file
              tileBuffer.putInt(poi.getLatitude());
              tileBuffer.putInt(poi.getLongitude());

              // write byte with layer and tag amount info
              tileBuffer.put(
                  buildLayerTagAmountByte(
                      poi.getLayer(), poi.getTags() == null ? 0 : (short) poi.getTags().size()));

              // write tag ids to the file
              if (poi.getTags() != null) {
                for (PoiEnum poiEnum : poi.getTags()) {
                  tileBuffer.putShort((short) poiEnum.ordinal());
                }
              }

              // write byte with bits set to 1 if the poi has a name, an elevation
              // or a housenumber
              tileBuffer.put(
                  buildInfoByteForPOI(poi.getName(), poi.getElevation(), poi.getHouseNumber()));

              if (poi.getName() != null && poi.getName().length() > 0) {
                writeUTF8(poi.getName(), tileBuffer);
              }
              if (poi.getElevation() != 0) {
                tileBuffer.putShort(poi.getElevation());
              }
              if (poi.getHouseNumber() != null && poi.getHouseNumber().length() > 0) {
                writeUTF8(poi.getHouseNumber(), tileBuffer);
              }
            }
          } // end for loop over POIs

          // write offset to first way in the tile header
          tileBuffer.putInt(fileIndexStartWayContainer, tileBuffer.position() - tileContainerStart);

          // ************* WAYS ************
          // write ways
          for (byte zoomlevel = minZoomCurrentInterval;
              zoomlevel <= maxZoomCurrentInterval;
              zoomlevel++) {
            List<TDWay> ways = waysByZoomlevel.get(zoomlevel);
            if (ways == null) continue;

            // use executor service to parallelize computation of subtile bitmasks
            // for all
            // ways in the current tile
            short[] bitmaskComputationResults = computeSubtileBitmasks(ways, currentTileCoordinate);
            assert bitmaskComputationResults.length == ways.size();
            // needed to access bitmask computation results in the foreach loop
            int i = 0;
            for (TDWay way : ways) {
              // // INNER WAY
              // // inner ways will be written as part of the outer way
              // if (way.isInnerWay())
              // continue;
              int startIndexWay = tileBuffer.position();

              WayNodePreprocessingResult wayNodePreprocessingResult =
                  preprocessWayNodes(
                      way,
                      waynodeCompression,
                      pixelCompression,
                      polygonClipping,
                      maxZoomCurrentInterval,
                      minZoomCurrentInterval,
                      currentTileCoordinate);

              if (wayNodePreprocessingResult == null) {
                continue;
              }
              if (debugStrings) {
                StringBuilder sb = new StringBuilder();
                sb.append(DEBUG_STRING_WAY_HEAD).append(way.getId()).append(DEBUG_STRING_WAY_TAIL);
                tileBuffer.put(sb.toString().getBytes());
                // append withespaces so that block has 32 bytes
                appendWhitespace(32 - sb.toString().getBytes().length, tileBuffer);
              }

              // skip 4 bytes to reserve space for way size
              int startIndexWaySize = tileBuffer.position();
              tileBuffer.position(startIndexWaySize + 4);

              // write way features
              // short bitmask = GeoUtils.computeBitmask(way,
              // currentTileCoordinate);
              // short bitmask = (short) 0xffff;
              tileBuffer.putShort(bitmaskComputationResults[i++]);

              // write byte with layer and tag amount
              tileBuffer.put(
                  buildLayerTagAmountByte(
                      way.getLayer(), way.getTags() == null ? 0 : (short) way.getTags().size()));

              // set type of the way node compression
              int compressionType = wayNodePreprocessingResult.getCompressionType();

              // write byte with amount of tags which are rendered
              tileBuffer.put(buildRenderTagWayNodeCompressionByte(way.getTags(), compressionType));

              // write tag bitmap
              tileBuffer.put(buildTagBitmapByte(way.getTags()));
              // file.writeByte((byte) 0xff);

              // write tag ids
              if (way.getTags() != null) {
                for (WayEnum wayEnum : way.getTags()) {
                  tileBuffer.putShort((short) wayEnum.ordinal());
                }
              }
              // write the amount of way nodes to the file
              tileBuffer.putShort(
                  (short) (wayNodePreprocessingResult.getWaynodesAsList().size() / 2));

              // write the way nodes:
              // the first node is always stored with four bytes
              // the remaining way node differences are stored according to the
              // compression type
              writeWayNodes(
                  wayNodePreprocessingResult.getWaynodesAsList(),
                  wayNodePreprocessingResult.getCompressionType(),
                  tileBuffer);

              // write a byte with name, label and way type information
              tileBuffer.put(buildInfoByteForWay(way.getName(), way.getWaytype(), way.getRef()));

              // // if the way has a name, write it to the file
              if (way.getName() != null && way.getName().length() > 0) {
                writeUTF8(way.getName(), tileBuffer);
              }

              // if the way has a ref, write it to the file
              if (way.getRef() != null && way.getRef().length() > 0) {
                writeUTF8(way.getRef(), tileBuffer);
              }
              //
              // // // if the way has a label position write it to the file
              // // if (labelPositionLatitude != 0 && labelPositionLongitude != 0)
              // {
              // // raf.writeInt(labelPositionLatitude);
              // // raf.writeInt(labelPositionLongitude);
              // // }
              //
              // *********MULTIPOLYGON PROCESSING***********
              if (way.getWaytype() == 3
                  && dataStore.getInnerWaysOfMultipolygon(way.getId()) != null) {
                List<TDWay> innerways = dataStore.getInnerWaysOfMultipolygon(way.getId());

                if (innerways == null) {
                  tileBuffer.put((byte) 0);
                } else {
                  tileBuffer.put((byte) innerways.size());
                  for (TDWay innerway : innerways) {
                    WayNodePreprocessingResult innerWayNodePreprocessingResult =
                        preprocessWayNodes(
                            innerway,
                            waynodeCompression,
                            pixelCompression,
                            false,
                            maxZoomCurrentInterval,
                            minZoomCurrentInterval,
                            currentTileCoordinate);
                    // write the amount of way nodes to the file
                    tileBuffer.putShort(
                        (short) (innerWayNodePreprocessingResult.getWaynodesAsList().size() / 2));
                    writeWayNodes(
                        innerWayNodePreprocessingResult.getWaynodesAsList(),
                        wayNodePreprocessingResult.getCompressionType(),
                        tileBuffer);
                  }
                }
              }
              // write the size of the way to the file
              tileBuffer.putInt(startIndexWaySize, tileBuffer.position() - startIndexWay);
            }
          } // end for loop over ways
        } // end if clause checking if tile is empty or not
        long tileSize = tileBuffer.position() - currentTileOffsetInBuffer;
        currentSubfileOffset += tileSize;

        // if necessary, allocate new buffer
        if (tileBuffer.remaining() < MIN_TILE_BUFFER_SIZE)
          tileBuffer =
              randomAccessFile
                  .getChannel()
                  .map(
                      MapMode.READ_WRITE,
                      startPositionSubfile + currentSubfileOffset,
                      TILE_BUFFER_SIZE);

        tilesProcessed++;
        if (tilesProcessed % fivePercentOfTilesToProcess == 0) {
          logger.info(
              "written " + (tilesProcessed / fivePercentOfTilesToProcess) * 5 + "% of file");
        }
      } // end for loop over tile columns
    } // /end for loop over tile rows

    // return size of sub file in bytes
    return currentSubfileOffset;
  }
Ejemplo n.º 10
0
  private long writeContainerHeader(
      long date,
      int version,
      short tilePixel,
      String comment,
      boolean debugStrings,
      boolean waynodeCompression,
      boolean polygonClipping,
      boolean pixelCompression,
      GeoCoordinate mapStartPosition)
      throws IOException {

    // get metadata for the map file
    int numberOfZoomIntervals = dataStore.getZoomIntervalConfiguration().getNumberOfZoomIntervals();

    logger.fine("writing header");

    MappedByteBuffer containerHeaderBuffer =
        randomAccessFile.getChannel().map(MapMode.READ_WRITE, 0, HEADER_BUFFER_SIZE);

    // write file header
    // magic byte
    byte[] magicBytes = MAGIC_BYTE.getBytes();
    containerHeaderBuffer.put(magicBytes);

    // write container header size
    int headerSizePosition = containerHeaderBuffer.position();
    containerHeaderBuffer.position(headerSizePosition + 4);

    // version number of the binary file format
    containerHeaderBuffer.putInt(version);

    // meta info byte
    containerHeaderBuffer.put(
        buildMetaInfoByte(
            debugStrings,
            mapStartPosition != null,
            pixelCompression,
            polygonClipping,
            waynodeCompression));

    // amount of map files inside this file
    containerHeaderBuffer.put((byte) numberOfZoomIntervals);

    // projection type
    writeUTF8(PROJECTION, containerHeaderBuffer);

    // width and height of a tile in pixel
    containerHeaderBuffer.putShort(tilePixel);

    logger.fine(
        "Bounding box for file: "
            + dataStore.getBoundingBox().maxLatitudeE6
            + ", "
            + dataStore.getBoundingBox().minLongitudeE6
            + ", "
            + dataStore.getBoundingBox().minLatitudeE6
            + ", "
            + dataStore.getBoundingBox().maxLongitudeE6);
    // upper left corner of the bounding box
    containerHeaderBuffer.putInt(dataStore.getBoundingBox().maxLatitudeE6);
    containerHeaderBuffer.putInt(dataStore.getBoundingBox().minLongitudeE6);
    containerHeaderBuffer.putInt(dataStore.getBoundingBox().minLatitudeE6);
    containerHeaderBuffer.putInt(dataStore.getBoundingBox().maxLongitudeE6);

    if (mapStartPosition != null) {
      containerHeaderBuffer.putInt(mapStartPosition.getLatitudeE6());
      containerHeaderBuffer.putInt(mapStartPosition.getLongitudeE6());
    }

    // date of the map data
    containerHeaderBuffer.putLong(date);

    // store the mapping of tags to tag ids
    containerHeaderBuffer.putShort((short) PoiEnum.values().length);
    for (PoiEnum poiEnum : PoiEnum.values()) {
      writeUTF8(poiEnum.toString(), containerHeaderBuffer);
      containerHeaderBuffer.putShort((short) poiEnum.ordinal());
    }
    containerHeaderBuffer.putShort((short) WayEnum.values().length);
    for (WayEnum wayEnum : WayEnum.values()) {
      writeUTF8(wayEnum.toString(), containerHeaderBuffer);
      containerHeaderBuffer.putShort((short) wayEnum.ordinal());
    }

    // comment
    if (comment != null && !comment.equals("")) {
      writeUTF8(comment, containerHeaderBuffer);
    } else {
      writeUTF8("", containerHeaderBuffer);
    }

    // initialize buffer for writing zoom interval configurations
    bufferZoomIntervalConfig =
        randomAccessFile
            .getChannel()
            .map(
                MapMode.READ_WRITE,
                containerHeaderBuffer.position(),
                SIZE_ZOOMINTERVAL_CONFIGURATION * numberOfZoomIntervals);

    containerHeaderBuffer.position(
        containerHeaderBuffer.position() + SIZE_ZOOMINTERVAL_CONFIGURATION * numberOfZoomIntervals);

    // -4 bytes of header size variable itself
    int headerSize = containerHeaderBuffer.position() - headerSizePosition - 4;
    containerHeaderBuffer.putInt(headerSizePosition, headerSize);

    return containerHeaderBuffer.position();
  }
Ejemplo n.º 11
0
 private void setIndex(int position, int value) {
   buffer.putInt(this.position + position * PHYSICAL_SIZE, value);
 }