コード例 #1
1
ファイル: MapTest.java プロジェクト: FauxFaux/jdk9-jdk
  /**
   * Maps blah file with a random offset and checks to see if data written out to the file can be
   * read back in
   */
  private static void testWrite() throws Exception {
    StringBuilder sb = new StringBuilder();
    sb.setLength(4);

    for (int x = 0; x < 1000; x++) {
      try (RandomAccessFile raf = new RandomAccessFile(blah, "rw")) {
        FileChannel fc = raf.getChannel();

        long offset = generator.nextInt(1000);
        MappedByteBuffer b = fc.map(MapMode.READ_WRITE, offset, 100);

        for (int i = 0; i < 4; i++) {
          b.put(i, (byte) ('0' + i));
        }

        for (int i = 0; i < 4; i++) {
          byte aByte = b.get(i);
          sb.setCharAt(i, (char) aByte);
        }
        if (!sb.toString().equals("0123")) throw new Exception("Write test failed");
      }
    }
  }
コード例 #2
0
ファイル: MapFileWriter.java プロジェクト: biafra23/mapsforge
  private void writeWayNodes(List<Integer> waynodes, int compressionType, MappedByteBuffer buffer) {
    if (!waynodes.isEmpty() && waynodes.size() % 2 == 0) {
      Iterator<Integer> waynodeIterator = waynodes.iterator();
      buffer.putInt(waynodeIterator.next());
      buffer.putInt(waynodeIterator.next());

      while (waynodeIterator.hasNext()) {
        switch (compressionType) {
          case 0:
            buffer.putInt(waynodeIterator.next().intValue());
            buffer.putInt(waynodeIterator.next().intValue());
            break;
          case 1:
            buffer.put(Serializer.getSignedThreeBytes(waynodeIterator.next().intValue()));
            buffer.put(Serializer.getSignedThreeBytes(waynodeIterator.next().intValue()));
            break;
          case 2:
            buffer.putShort(waynodeIterator.next().shortValue());
            buffer.putShort(waynodeIterator.next().shortValue());
            break;
          case 3:
            buffer.put(waynodeIterator.next().byteValue());
            buffer.put(waynodeIterator.next().byteValue());
            break;
        }
      }
    }
  }
コード例 #3
0
  /**
   * @Description: 将文件的一部分映射入内存中 byte =char(FG:a,b c,d ) (byte) 97:a (byte) 122):Z byte就是最基本的单位 了,
   * 如果是字符点 1byte , 如果 是int占2 byte,....
   *
   * @param filePath
   * @param start
   * @param size
   * @throws IOException
   */
  public static void mappedReadFile(String filePath, int start, int size) throws IOException {
    // start = 0;
    // size = 1024;

    RandomAccessFile raf = new RandomAccessFile("C:\\usemappedfile.txt", "rw");
    FileChannel fc = raf.getChannel();

    MappedByteBuffer mbb = fc.map(FileChannel.MapMode.READ_WRITE, start, size);
    // 可以用put方法, get方法设置相应的值
    mbb.put(0, (byte) 97); // 将位置 致为相应值
    mbb.put(1023, (byte) 122); // 将位置 致为相应值
    // mbb.flip()

    raf.close();
  }
コード例 #4
0
ファイル: MapFileWriter.java プロジェクト: biafra23/mapsforge
  private void writeSubfileMetaDataToContainerHeader(
      int i, long startIndexOfSubfile, long subfileSize) {

    // HEADER META DATA FOR SUB FILE
    // write zoom interval configuration to header
    byte minZoomCurrentInterval = dataStore.getZoomIntervalConfiguration().getMinZoom(i);
    byte maxZoomCurrentInterval = dataStore.getZoomIntervalConfiguration().getMaxZoom(i);
    byte baseZoomCurrentInterval = dataStore.getZoomIntervalConfiguration().getBaseZoom(i);

    bufferZoomIntervalConfig.put(baseZoomCurrentInterval);
    bufferZoomIntervalConfig.put(minZoomCurrentInterval);
    bufferZoomIntervalConfig.put(maxZoomCurrentInterval);
    bufferZoomIntervalConfig.put(Serializer.getFiveBytes(startIndexOfSubfile));
    bufferZoomIntervalConfig.put(Serializer.getFiveBytes(subfileSize));
  }
コード例 #5
0
  // single pass attempt to copy if any can not accept the data then they are skipped
  // and true will be returned instead of false.
  private static boolean doingCopy(
      TapeWriteStage ss,
      int byteTailPos,
      int primaryTailPos,
      int totalPrimaryCopy,
      int totalBytesCopy) {

    IntBuffer primaryInts = RingBuffer.wrappedStructuredLayoutRingBuffer(ss.source);
    ByteBuffer secondaryBytes = RingBuffer.wrappedUnstructuredLayoutRingBufferA(ss.source);

    primaryInts.position(primaryTailPos);
    primaryInts.limit(
        primaryTailPos
            + totalPrimaryCopy); // TODO: AA, this will not work on the wrap, we must mask and do
                                 // muliple copies

    secondaryBytes.position(byteTailPos);
    secondaryBytes.limit(byteTailPos + totalBytesCopy);

    ss.header.clear();
    ss.headerAsInts.put(totalBytesCopy + (totalPrimaryCopy << 2));
    ss.headerAsInts.put(totalPrimaryCopy << 2);

    // TODO: must return false if there is no room to write.

    // TODO: BB, this creates a bit of garbage for the map, perhaps we should map larger blocks
    // expecting to use them for multiple writes.
    MappedByteBuffer mapped;
    try {
      mapped =
          ss.fileChannel.map(
              MapMode.READ_WRITE,
              ss.fileChannel.position(),
              8 + totalBytesCopy + (totalPrimaryCopy << 2));
      mapped.put(ss.header);

      IntBuffer asIntBuffer = mapped.asIntBuffer();
      asIntBuffer.position(2);
      asIntBuffer.put(primaryInts);

      mapped.position(mapped.position() + (totalPrimaryCopy << 2));
      mapped.put(secondaryBytes);

    } catch (IOException e) {
      throw new RuntimeException(e);
    }
    return true;
  }
コード例 #6
0
 public static void main(String[] args) throws Exception {
   fc = new RandomAccessFile("test.dat", "rw").getChannel();
   MappedByteBuffer out = fc.map(FileChannel.MapMode.READ_WRITE, 0, LENGTH);
   for (int i = 0; i < LENGTH; i++) out.put((byte) 'x');
   new LockAndModify(out, 0, 0 + LENGTH / 3);
   new LockAndModify(out, LENGTH / 2, LENGTH / 2 + LENGTH / 4);
 }
コード例 #7
0
ファイル: BlockOutStream.java プロジェクト: haroldpku/tachyon
  private synchronized void appendCurrentBuffer(byte[] buf, int offset, int length)
      throws IOException {
    if (!TFS.requestSpace(length)) {
      mCanWrite = false;

      String msg =
          "Local tachyon worker does not have enough "
              + "space ("
              + length
              + ") or no worker for "
              + FILE.FID
              + " "
              + BLOCK_ID;
      if (PIN) {
        TFS.outOfMemoryForPinFile(FILE.FID);
        throw new IOException(msg);
      }

      throw new IOException(msg);
    }

    MappedByteBuffer out = mLocalFileChannel.map(MapMode.READ_WRITE, mInFileBytes, length);
    out.put(buf, 0, length);
    mInFileBytes += length;
  }
コード例 #8
0
 private void writeRecordHeaderToMappedBuffer(RecordHeader header) {
   mappedDataBuffer.position(header.getPosition());
   mappedDataBuffer.putInt(header.getDataSize());
   mappedDataBuffer.put(header.getFragmented());
   if (header.isFragmented()) {
     mappedDataBuffer.putInt(header.getNextPos());
   }
 }
コード例 #9
0
 @SuppressWarnings("deprecation")
 @Override
 protected void writeToBuffer(MappedByteBuffer buffer, Tuple e) {
   KeyValue kv = KeyValueUtil.ensureKeyValue(e.getValue(0));
   buffer.putInt(kv.getLength() + Bytes.SIZEOF_INT);
   buffer.putInt(kv.getLength());
   buffer.put(kv.getBuffer(), kv.getOffset(), kv.getLength());
 }
コード例 #10
0
  /**
   * 通过FileChannel.map()拿到MappedByteBuffer 使用内存文件映射,写2G大文件,速度会快很多
   *
   * @throws IOException
   */
  public static void writeBigFileByMappedByteBuffer() throws IOException {
    long start = System.currentTimeMillis();

    File mbFile = new File("mb.dat");
    if (mbFile.exists()) {
      mbFile.delete();
    }

    RandomAccessFile raf = new RandomAccessFile(mbFile, "rw");
    FileChannel fileChannel = raf.getChannel();

    int pos = 0;
    MappedByteBuffer mbb = null;
    byte[] data = null;
    long len = LEN;
    int dataChunk = DATA_CHUNK / (1024 * 1024);
    while (len >= DATA_CHUNK) {
      System.out.println("write a data chunk:" + dataChunk + "MB");

      mbb = fileChannel.map(MapMode.READ_WRITE, pos, DATA_CHUNK);
      data = new byte[DATA_CHUNK];
      mbb.put(data);

      data = null;

      len -= DATA_CHUNK;
      pos += DATA_CHUNK;
    }

    if (len > 0) {
      System.out.println("write rest data chunk:" + len + "B");

      mbb = fileChannel.map(MapMode.READ_WRITE, pos, len);
      data = new byte[(int) len];
      mbb.put(data);
      data = null;
    }

    // 关闭文件和通道流
    fileChannel.close();
    raf.close();

    long end = System.currentTimeMillis();
    System.out.println(String.format("===>写2G文件 " + mbFile.getName() + " 耗时:%s毫秒", end - start));
  }
コード例 #11
0
ファイル: Dict.java プロジェクト: Tomgogo/DicWS
 private void writeWordBlock(WordBlock block) throws IOException {
   MappedByteBuffer buffer = fileChannel.map(MapMode.READ_WRITE, block.position, block_size);
   byte[] data = block.encode();
   if (data.length > block_size) {
     throw new RuntimeException();
   }
   buffer.put(data);
   // buffer.force();
 }
コード例 #12
0
 public static void main(String[] args) throws Exception {
   MappedByteBuffer out =
       new RandomAccessFile("test.dat", "rw")
           .getChannel()
           .map(FileChannel.MapMode.READ_WRITE, 0, length);
   for (int i = 0; i < length; i++) out.put((byte) 'x');
   print("Finished writing");
   for (int i = length / 2; i < length / 2 + 6; i++) printnb((char) out.get(i));
 }
コード例 #13
0
  private static void corrupt(DataSegment segment) throws IOException {
    final Map<String, Object> localLoadSpec = segment.getLoadSpec();
    final Path segmentPath = Paths.get(localLoadSpec.get("path").toString());

    final MappedByteBuffer buffer = Files.map(segmentPath.toFile(), FileChannel.MapMode.READ_WRITE);
    while (buffer.hasRemaining()) {
      buffer.put((byte) 0xFF);
    }
  }
コード例 #14
0
ファイル: WriteAheadLog.java プロジェクト: imace/zava
  /**
   * Log a put operation.
   *
   * @param key The key of the value.
   * @param value The value to place into the database.
   * @param options Write options that define how the value is written to the memtable.
   */
  public void put(final TableKey key, final TableValue value, final WriteOptions options) {
    ByteBuffer data = serialize(key, value, options);
    data.flip();

    synchronized (this) {
      if (logBuffer.remaining() < data.capacity()) {
        logBuffer = mapWAL();
      }
      logBuffer.put(data);
    }
  }
コード例 #15
0
 public static void createFileForTest(Integer size, File file) throws IOException {
   RandomAccessFile out = new RandomAccessFile(file, "rw");
   FileChannel fc = out.getChannel();
   long length = size;
   MappedByteBuffer MappByteBuff = fc.map(FileChannel.MapMode.READ_WRITE, 0, length);
   for (int i = 0; i < length; i++) {
     MappByteBuff.put((byte) 'V');
   }
   fc.close();
   System.out.println("File successFully written");
 }
コード例 #16
0
ファイル: MmfIpcSpike.java プロジェクト: orfjackal/jumi
    public static void main(String[] args) throws Exception {
      MappedByteBuffer map = openMemoryMappedFile();

      for (int value = 0; value < MESSAGES; value++) {

        while (map.get(TAG_INDEX) != CAN_WRITE) {
          loopWait();
        }
        map.putInt(DATA_INDEX, value);
        map.put(TAG_INDEX, CAN_READ);
      }
    }
コード例 #17
0
  @Override
  public void setChainHead(StoredBlock chainHead) throws BlockStoreException {
    final MappedByteBuffer buffer = this.buffer;
    if (buffer == null) throw new BlockStoreException("Store closed");

    lock.lock();
    try {
      lastChainHead = chainHead;
      byte[] headHash = chainHead.getHeader().getHash().getBytes();
      buffer.position(8);
      buffer.put(headHash);
    } finally {
      lock.unlock();
    }
  }
コード例 #18
0
ファイル: SpillMap.java プロジェクト: JamesRTaylor/phoenix
    // Flush the current page to the memory mapped byte buffer
    private void flushBuffer() throws BufferOverflowException {
      Collection<byte[]> values = ImmutableMap.copyOf(pageMap).values();
      buffer.clear();
      // number of elements
      buffer.putInt(values.size());
      for (byte[] value : values) {
        // element length
        buffer.putInt(value.length);
        // element
        buffer.put(value, 0, value.length);
      }

      // Reset page stats
      pageMap.clear();
      totalResultSize = 0;
    }
コード例 #19
0
 private void initNewStore(NetworkParameters params) throws Exception {
   byte[] header;
   header = HEADER_MAGIC.getBytes("US-ASCII");
   buffer.put(header);
   // Insert the genesis block.
   lock.lock();
   try {
     setRingCursor(buffer, FILE_PROLOGUE_BYTES);
   } finally {
     lock.unlock();
   }
   Block genesis = params.getGenesisBlock().cloneAsHeader();
   StoredBlock storedGenesis = new StoredBlock(genesis, genesis.getWork(), 0);
   put(storedGenesis);
   setChainHead(storedGenesis);
 }
コード例 #20
0
ファイル: WriteAheadLog.java プロジェクト: imace/zava
  /**
   * Log a series of writes. These writes are committed "atomically". Since all writes in the log
   * are serialized, we just need to make sure that all these writes are contiguous.
   *
   * @param ops Set of write operations
   */
  public void put(final WriteBatch ops) {
    // Lock the log so that we can commit several at a time.
    synchronized (this) {
      Iterator<TableKey> iter = ops.iterator();
      while (iter.hasNext()) {
        TableKey key = iter.next();
        for (WriteBatch.TableWrite write : ops.getValues(key)) {
          ByteBuffer data = serialize(key, write.value, write.options);
          if (logBuffer.remaining() < data.position()) {
            logBuffer = mapWAL();
          }

          data.flip();
          logBuffer.put(data);
        }
      }
    }
  }
コード例 #21
0
      public int testWrite(final String fileName) throws Exception {
        FileChannel channel = new RandomAccessFile(fileName, "rw").getChannel();
        MappedByteBuffer buffer = channel.map(READ_WRITE, 0, Math.min(channel.size(), MAX_VALUE));
        int checkSum = 0;

        for (long i = 0; i < FILE_SIZE; i++) {
          if (!buffer.hasRemaining()) {
            buffer = channel.map(READ_WRITE, i, Math.min(channel.size() - i, MAX_VALUE));
          }

          byte b = (byte) i;
          checkSum += b;
          buffer.put(b);
        }

        channel.close();

        return checkSum;
      }
コード例 #22
0
ファイル: MmfIpcSpike.java プロジェクト: orfjackal/jumi
    public static void main(String[] args) throws Exception {
      MappedByteBuffer map = openMemoryMappedFile();

      long start = System.currentTimeMillis();
      for (int expected = 0; expected < MESSAGES; expected++) {

        while (map.get(TAG_INDEX) != CAN_READ) {
          loopWait();
        }
        int value = map.getInt(DATA_INDEX);
        assertThat(value, is(expected));
        map.put(TAG_INDEX, CAN_WRITE);
      }
      long end = System.currentTimeMillis();

      long durationMs = end - start;
      double nsPerMessage = durationMs * 1000000.0 / MESSAGES;
      System.out.println("read " + MESSAGES + " messages in " + durationMs + " ms");
      System.out.println(nsPerMessage + " ns/roundtrip, " + (nsPerMessage / 2) + " ns IPC latency");
    }
コード例 #23
0
  /** @tests {@link java.nio.MappedByteBuffer#force()} */
  public void test_force() throws IOException {
    // buffer was not mapped in read/write mode
    FileInputStream fileInputStream = new FileInputStream(tmpFile);
    FileChannel fileChannelRead = fileInputStream.getChannel();
    MappedByteBuffer mmbRead = fileChannelRead.map(MapMode.READ_ONLY, 0, fileChannelRead.size());

    mmbRead.force();

    FileInputStream inputStream = new FileInputStream(tmpFile);
    FileChannel fileChannelR = inputStream.getChannel();
    MappedByteBuffer resultRead = fileChannelR.map(MapMode.READ_ONLY, 0, fileChannelR.size());

    // If this buffer was not mapped in read/write mode, then invoking this method has no effect.
    assertEquals(
        "Invoking force() should have no effect when this buffer was not mapped in read/write mode",
        mmbRead,
        resultRead);

    // Buffer was mapped in read/write mode
    RandomAccessFile randomFile = new RandomAccessFile(tmpFile, "rw");
    FileChannel fileChannelReadWrite = randomFile.getChannel();
    MappedByteBuffer mmbReadWrite =
        fileChannelReadWrite.map(FileChannel.MapMode.READ_WRITE, 0, fileChannelReadWrite.size());

    mmbReadWrite.put((byte) 'o');
    mmbReadWrite.force();

    RandomAccessFile random = new RandomAccessFile(tmpFile, "rw");
    FileChannel fileChannelRW = random.getChannel();
    MappedByteBuffer resultReadWrite =
        fileChannelRW.map(FileChannel.MapMode.READ_WRITE, 0, fileChannelRW.size());

    // Invoking force() will change the buffer
    assertFalse(mmbReadWrite.equals(resultReadWrite));

    fileChannelRead.close();
    fileChannelR.close();
    fileChannelReadWrite.close();
    fileChannelRW.close();
  }
コード例 #24
0
  @Override
  public void put(StoredBlock block) throws BlockStoreException {
    final MappedByteBuffer buffer = this.buffer;
    if (buffer == null) throw new BlockStoreException("Store closed");

    lock.lock();
    try {
      int cursor = getRingCursor(buffer);
      if (cursor == getFileSize()) {
        // Wrapped around.
        cursor = FILE_PROLOGUE_BYTES;
      }
      buffer.position(cursor);
      Sha256Hash hash = block.getHeader().getHash();
      notFoundCache.remove(hash);
      buffer.put(hash.getBytes());
      block.serializeCompact(buffer);
      setRingCursor(buffer, buffer.position());
      blockCache.put(hash, block);
    } finally {
      lock.unlock();
    }
  }
コード例 #25
0
ファイル: MapFileWriter.java プロジェクト: biafra23/mapsforge
 private void writeUTF8(String string, MappedByteBuffer buffer) {
   buffer.putShort((short) string.getBytes().length);
   buffer.put(string.getBytes());
 }
コード例 #26
0
 private void writeRecordDataToMappedBuffer(RecordHeader header, byte[] data) {
   mappedDataBuffer.position(header.getPosition() + header.getSize());
   mappedDataBuffer.put(data);
 }
コード例 #27
0
ファイル: MapFile.java プロジェクト: bboyjing/sample
  public static void main(String[] argv) throws Exception {
    // Create a temp file and get a channel connected to it
    File tempFile = File.createTempFile("mmaptest", null);
    RandomAccessFile file = new RandomAccessFile(tempFile, "rw");
    FileChannel channel = file.getChannel();

    // write strings to file by channel
    ByteBuffer temp = ByteBuffer.allocate(100);
    temp.put("This is the file content".getBytes());
    temp.flip();
    channel.write(temp, 0);

    temp.clear();
    temp.put("This is more file content".getBytes());
    temp.flip();
    channel.write(temp, 8192);

    // readonly
    MappedByteBuffer ro = channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size());
    // read and write
    MappedByteBuffer rw = channel.map(FileChannel.MapMode.READ_WRITE, 0, channel.size());
    // copy on write
    MappedByteBuffer cow = channel.map(FileChannel.MapMode.PRIVATE, 0, channel.size());

    System.out.println("Begin");

    // ro、rw、cow 相同
    showBuffers(ro, rw, cow);

    // Modify the copy-on-write buffer,copy-on-write不会影响数据源
    cow.position(8);
    cow.put("COW".getBytes());
    System.out.println("Change to COW buffer");
    showBuffers(ro, rw, cow);

    // Modify the read/write buffer,read/write会影响数据源
    rw.position(9);
    rw.put(" R/W ".getBytes());
    rw.position(8194);
    rw.put(" R/W ".getBytes());
    rw.force();
    System.out.println("Change to R/W buffer");
    showBuffers(ro, rw, cow);

    // Write to the file through the channel; hit both pages
    temp.clear();
    temp.put("Channel write ".getBytes());
    temp.flip();
    channel.write(temp, 0);
    temp.rewind();
    channel.write(temp, 8202);
    System.out.println("Write on channel");
    showBuffers(ro, rw, cow);

    // Modify the copy-on-write buffer again
    cow.position(8207);
    cow.put(" COW2 ".getBytes());
    System.out.println("Second change to COW buffer");
    showBuffers(ro, rw, cow);

    // Modify the read/write buffer
    rw.position(0);
    rw.put(" R/W2 ".getBytes());
    rw.position(8210);
    rw.put(" R/W2 ".getBytes());
    rw.force();
    System.out.println("Second change to R/W buffer");
    showBuffers(ro, rw, cow);
  }
コード例 #28
0
ファイル: MapFileWriter.java プロジェクト: biafra23/mapsforge
 private void appendWhitespace(int amount, MappedByteBuffer buffer) {
   for (int i = 0; i < amount; i++) {
     buffer.put((byte) ' ');
   }
 }
コード例 #29
0
ファイル: MapFileWriter.java プロジェクト: biafra23/mapsforge
  private long writeSubfile(
      long startPositionSubfile,
      int zoomIntervalIndex,
      boolean debugStrings,
      boolean waynodeCompression,
      boolean polygonClipping,
      boolean pixelCompression)
      throws IOException {

    logger.fine(
        "writing data for zoom interval "
            + zoomIntervalIndex
            + ", number of tiles: "
            + dataStore.numberOfHorizontalTiles(zoomIntervalIndex)
                * dataStore.numberOfVerticalTiles(zoomIntervalIndex));

    TileCoordinate upperLeft = dataStore.getUpperLeft(zoomIntervalIndex);
    int lengthX = dataStore.numberOfHorizontalTiles(zoomIntervalIndex);
    int lengthY = dataStore.numberOfVerticalTiles(zoomIntervalIndex);

    byte minZoomCurrentInterval =
        dataStore.getZoomIntervalConfiguration().getMinZoom(zoomIntervalIndex);
    byte maxZoomCurrentInterval =
        dataStore.getZoomIntervalConfiguration().getMaxZoom(zoomIntervalIndex);
    byte baseZoomCurrentInterval =
        dataStore.getZoomIntervalConfiguration().getBaseZoom(zoomIntervalIndex);
    byte maxMaxZoomlevel = dataStore.getZoomIntervalConfiguration().getMaxMaxZoom();

    int tileAmountInBytes =
        dataStore.numberOfHorizontalTiles(zoomIntervalIndex)
            * dataStore.numberOfVerticalTiles(zoomIntervalIndex)
            * BYTE_AMOUNT_SUBFILE_INDEX_PER_TILE;
    int indexBufferSize =
        tileAmountInBytes + (debugStrings ? DEBUG_INDEX_START_STRING.getBytes().length : 0);
    MappedByteBuffer indexBuffer =
        randomAccessFile
            .getChannel()
            .map(MapMode.READ_WRITE, startPositionSubfile, indexBufferSize);
    MappedByteBuffer tileBuffer =
        randomAccessFile
            .getChannel()
            .map(MapMode.READ_WRITE, startPositionSubfile + indexBufferSize, TILE_BUFFER_SIZE);

    long currentSubfileOffset = indexBufferSize;

    for (int tileY = upperLeft.getY(); tileY < upperLeft.getY() + lengthY; tileY++) {
      for (int tileX = upperLeft.getX(); tileX < upperLeft.getX() + lengthX; tileX++) {
        // logger.info("writing data for tile (" + tileX + ", " + tileY + ")");

        long currentTileOffsetInBuffer = tileBuffer.position();
        TileCoordinate currentTileCoordinate =
            new TileCoordinate(tileX, tileY, baseZoomCurrentInterval);

        // seek to index frame of this tile and write relative offset of this
        // tile as five bytes to the index
        indexBuffer.put(Serializer.getFiveBytes(currentSubfileOffset));

        // get statistics for tile
        TileData currentTile = dataStore.getTile(zoomIntervalIndex, tileX, tileY);

        // ************* POI ************
        // write amount of POIs and ways for each zoom level
        // TODO is this computation correct? Ways that have an associated zoom level of
        // e.g. 9
        // are lifted to zoom level 12 for an interval 12,14,17
        Map<Byte, List<TDNode>> poisByZoomlevel =
            currentTile.poisByZoomlevel(minZoomCurrentInterval, maxMaxZoomlevel);
        Map<Byte, List<TDWay>> waysByZoomlevel =
            currentTile.waysByZoomlevel(minZoomCurrentInterval, maxMaxZoomlevel);

        if (poisByZoomlevel.size() > 0 || waysByZoomlevel.size() > 0) {
          int tileContainerStart = tileBuffer.position();
          if (debugStrings) {
            // write tile header
            StringBuilder sb = new StringBuilder();
            sb.append(DEBUG_STRING_TILE_HEAD)
                .append(tileX)
                .append(",")
                .append(tileY)
                .append(DEBUG_STRING_TILE_TAIL);
            tileBuffer.put(sb.toString().getBytes());
            // append withespaces so that block has 32 bytes
            appendWhitespace(32 - sb.toString().getBytes().length, tileBuffer);
          }

          short cumulatedPOIs = 0;
          short cumulatedWays = 0;
          for (byte zoomlevel = minZoomCurrentInterval;
              zoomlevel <= maxZoomCurrentInterval;
              zoomlevel++) {
            if (poisByZoomlevel.get(zoomlevel) != null)
              cumulatedPOIs += poisByZoomlevel.get(zoomlevel).size();
            if (waysByZoomlevel.get(zoomlevel) != null)
              cumulatedWays += waysByZoomlevel.get(zoomlevel).size();
            tileBuffer.putShort(cumulatedPOIs);
            tileBuffer.putShort(cumulatedWays);
          }

          // skip 4 bytes, later these 4 bytes will contain the start
          // position of the ways in this tile
          int fileIndexStartWayContainer = tileBuffer.position();
          tileBuffer.position(fileIndexStartWayContainer + 4);

          // write POIs for each zoom level beginning with lowest zoom level
          for (byte zoomlevel = minZoomCurrentInterval;
              zoomlevel <= maxZoomCurrentInterval;
              zoomlevel++) {
            List<TDNode> pois = poisByZoomlevel.get(zoomlevel);
            if (pois == null) continue;
            for (TDNode poi : pois) {
              if (debugStrings) {
                StringBuilder sb = new StringBuilder();
                sb.append(DEBUG_STRING_POI_HEAD).append(poi.getId()).append(DEBUG_STRING_POI_TAIL);
                tileBuffer.put(sb.toString().getBytes());
                // append withespaces so that block has 32 bytes
                appendWhitespace(32 - sb.toString().getBytes().length, tileBuffer);
              }

              // write poi features to the file
              tileBuffer.putInt(poi.getLatitude());
              tileBuffer.putInt(poi.getLongitude());

              // write byte with layer and tag amount info
              tileBuffer.put(
                  buildLayerTagAmountByte(
                      poi.getLayer(), poi.getTags() == null ? 0 : (short) poi.getTags().size()));

              // write tag ids to the file
              if (poi.getTags() != null) {
                for (PoiEnum poiEnum : poi.getTags()) {
                  tileBuffer.putShort((short) poiEnum.ordinal());
                }
              }

              // write byte with bits set to 1 if the poi has a name, an elevation
              // or a housenumber
              tileBuffer.put(
                  buildInfoByteForPOI(poi.getName(), poi.getElevation(), poi.getHouseNumber()));

              if (poi.getName() != null && poi.getName().length() > 0) {
                writeUTF8(poi.getName(), tileBuffer);
              }
              if (poi.getElevation() != 0) {
                tileBuffer.putShort(poi.getElevation());
              }
              if (poi.getHouseNumber() != null && poi.getHouseNumber().length() > 0) {
                writeUTF8(poi.getHouseNumber(), tileBuffer);
              }
            }
          } // end for loop over POIs

          // write offset to first way in the tile header
          tileBuffer.putInt(fileIndexStartWayContainer, tileBuffer.position() - tileContainerStart);

          // ************* WAYS ************
          // write ways
          for (byte zoomlevel = minZoomCurrentInterval;
              zoomlevel <= maxZoomCurrentInterval;
              zoomlevel++) {
            List<TDWay> ways = waysByZoomlevel.get(zoomlevel);
            if (ways == null) continue;

            // use executor service to parallelize computation of subtile bitmasks
            // for all
            // ways in the current tile
            short[] bitmaskComputationResults = computeSubtileBitmasks(ways, currentTileCoordinate);
            assert bitmaskComputationResults.length == ways.size();
            // needed to access bitmask computation results in the foreach loop
            int i = 0;
            for (TDWay way : ways) {
              // // INNER WAY
              // // inner ways will be written as part of the outer way
              // if (way.isInnerWay())
              // continue;
              int startIndexWay = tileBuffer.position();

              WayNodePreprocessingResult wayNodePreprocessingResult =
                  preprocessWayNodes(
                      way,
                      waynodeCompression,
                      pixelCompression,
                      polygonClipping,
                      maxZoomCurrentInterval,
                      minZoomCurrentInterval,
                      currentTileCoordinate);

              if (wayNodePreprocessingResult == null) {
                continue;
              }
              if (debugStrings) {
                StringBuilder sb = new StringBuilder();
                sb.append(DEBUG_STRING_WAY_HEAD).append(way.getId()).append(DEBUG_STRING_WAY_TAIL);
                tileBuffer.put(sb.toString().getBytes());
                // append withespaces so that block has 32 bytes
                appendWhitespace(32 - sb.toString().getBytes().length, tileBuffer);
              }

              // skip 4 bytes to reserve space for way size
              int startIndexWaySize = tileBuffer.position();
              tileBuffer.position(startIndexWaySize + 4);

              // write way features
              // short bitmask = GeoUtils.computeBitmask(way,
              // currentTileCoordinate);
              // short bitmask = (short) 0xffff;
              tileBuffer.putShort(bitmaskComputationResults[i++]);

              // write byte with layer and tag amount
              tileBuffer.put(
                  buildLayerTagAmountByte(
                      way.getLayer(), way.getTags() == null ? 0 : (short) way.getTags().size()));

              // set type of the way node compression
              int compressionType = wayNodePreprocessingResult.getCompressionType();

              // write byte with amount of tags which are rendered
              tileBuffer.put(buildRenderTagWayNodeCompressionByte(way.getTags(), compressionType));

              // write tag bitmap
              tileBuffer.put(buildTagBitmapByte(way.getTags()));
              // file.writeByte((byte) 0xff);

              // write tag ids
              if (way.getTags() != null) {
                for (WayEnum wayEnum : way.getTags()) {
                  tileBuffer.putShort((short) wayEnum.ordinal());
                }
              }
              // write the amount of way nodes to the file
              tileBuffer.putShort(
                  (short) (wayNodePreprocessingResult.getWaynodesAsList().size() / 2));

              // write the way nodes:
              // the first node is always stored with four bytes
              // the remaining way node differences are stored according to the
              // compression type
              writeWayNodes(
                  wayNodePreprocessingResult.getWaynodesAsList(),
                  wayNodePreprocessingResult.getCompressionType(),
                  tileBuffer);

              // write a byte with name, label and way type information
              tileBuffer.put(buildInfoByteForWay(way.getName(), way.getWaytype(), way.getRef()));

              // // if the way has a name, write it to the file
              if (way.getName() != null && way.getName().length() > 0) {
                writeUTF8(way.getName(), tileBuffer);
              }

              // if the way has a ref, write it to the file
              if (way.getRef() != null && way.getRef().length() > 0) {
                writeUTF8(way.getRef(), tileBuffer);
              }
              //
              // // // if the way has a label position write it to the file
              // // if (labelPositionLatitude != 0 && labelPositionLongitude != 0)
              // {
              // // raf.writeInt(labelPositionLatitude);
              // // raf.writeInt(labelPositionLongitude);
              // // }
              //
              // *********MULTIPOLYGON PROCESSING***********
              if (way.getWaytype() == 3
                  && dataStore.getInnerWaysOfMultipolygon(way.getId()) != null) {
                List<TDWay> innerways = dataStore.getInnerWaysOfMultipolygon(way.getId());

                if (innerways == null) {
                  tileBuffer.put((byte) 0);
                } else {
                  tileBuffer.put((byte) innerways.size());
                  for (TDWay innerway : innerways) {
                    WayNodePreprocessingResult innerWayNodePreprocessingResult =
                        preprocessWayNodes(
                            innerway,
                            waynodeCompression,
                            pixelCompression,
                            false,
                            maxZoomCurrentInterval,
                            minZoomCurrentInterval,
                            currentTileCoordinate);
                    // write the amount of way nodes to the file
                    tileBuffer.putShort(
                        (short) (innerWayNodePreprocessingResult.getWaynodesAsList().size() / 2));
                    writeWayNodes(
                        innerWayNodePreprocessingResult.getWaynodesAsList(),
                        wayNodePreprocessingResult.getCompressionType(),
                        tileBuffer);
                  }
                }
              }
              // write the size of the way to the file
              tileBuffer.putInt(startIndexWaySize, tileBuffer.position() - startIndexWay);
            }
          } // end for loop over ways
        } // end if clause checking if tile is empty or not
        long tileSize = tileBuffer.position() - currentTileOffsetInBuffer;
        currentSubfileOffset += tileSize;

        // if necessary, allocate new buffer
        if (tileBuffer.remaining() < MIN_TILE_BUFFER_SIZE)
          tileBuffer =
              randomAccessFile
                  .getChannel()
                  .map(
                      MapMode.READ_WRITE,
                      startPositionSubfile + currentSubfileOffset,
                      TILE_BUFFER_SIZE);

        tilesProcessed++;
        if (tilesProcessed % fivePercentOfTilesToProcess == 0) {
          logger.info(
              "written " + (tilesProcessed / fivePercentOfTilesToProcess) * 5 + "% of file");
        }
      } // end for loop over tile columns
    } // /end for loop over tile rows

    // return size of sub file in bytes
    return currentSubfileOffset;
  }
コード例 #30
0
ファイル: MapFileWriter.java プロジェクト: biafra23/mapsforge
  private long writeContainerHeader(
      long date,
      int version,
      short tilePixel,
      String comment,
      boolean debugStrings,
      boolean waynodeCompression,
      boolean polygonClipping,
      boolean pixelCompression,
      GeoCoordinate mapStartPosition)
      throws IOException {

    // get metadata for the map file
    int numberOfZoomIntervals = dataStore.getZoomIntervalConfiguration().getNumberOfZoomIntervals();

    logger.fine("writing header");

    MappedByteBuffer containerHeaderBuffer =
        randomAccessFile.getChannel().map(MapMode.READ_WRITE, 0, HEADER_BUFFER_SIZE);

    // write file header
    // magic byte
    byte[] magicBytes = MAGIC_BYTE.getBytes();
    containerHeaderBuffer.put(magicBytes);

    // write container header size
    int headerSizePosition = containerHeaderBuffer.position();
    containerHeaderBuffer.position(headerSizePosition + 4);

    // version number of the binary file format
    containerHeaderBuffer.putInt(version);

    // meta info byte
    containerHeaderBuffer.put(
        buildMetaInfoByte(
            debugStrings,
            mapStartPosition != null,
            pixelCompression,
            polygonClipping,
            waynodeCompression));

    // amount of map files inside this file
    containerHeaderBuffer.put((byte) numberOfZoomIntervals);

    // projection type
    writeUTF8(PROJECTION, containerHeaderBuffer);

    // width and height of a tile in pixel
    containerHeaderBuffer.putShort(tilePixel);

    logger.fine(
        "Bounding box for file: "
            + dataStore.getBoundingBox().maxLatitudeE6
            + ", "
            + dataStore.getBoundingBox().minLongitudeE6
            + ", "
            + dataStore.getBoundingBox().minLatitudeE6
            + ", "
            + dataStore.getBoundingBox().maxLongitudeE6);
    // upper left corner of the bounding box
    containerHeaderBuffer.putInt(dataStore.getBoundingBox().maxLatitudeE6);
    containerHeaderBuffer.putInt(dataStore.getBoundingBox().minLongitudeE6);
    containerHeaderBuffer.putInt(dataStore.getBoundingBox().minLatitudeE6);
    containerHeaderBuffer.putInt(dataStore.getBoundingBox().maxLongitudeE6);

    if (mapStartPosition != null) {
      containerHeaderBuffer.putInt(mapStartPosition.getLatitudeE6());
      containerHeaderBuffer.putInt(mapStartPosition.getLongitudeE6());
    }

    // date of the map data
    containerHeaderBuffer.putLong(date);

    // store the mapping of tags to tag ids
    containerHeaderBuffer.putShort((short) PoiEnum.values().length);
    for (PoiEnum poiEnum : PoiEnum.values()) {
      writeUTF8(poiEnum.toString(), containerHeaderBuffer);
      containerHeaderBuffer.putShort((short) poiEnum.ordinal());
    }
    containerHeaderBuffer.putShort((short) WayEnum.values().length);
    for (WayEnum wayEnum : WayEnum.values()) {
      writeUTF8(wayEnum.toString(), containerHeaderBuffer);
      containerHeaderBuffer.putShort((short) wayEnum.ordinal());
    }

    // comment
    if (comment != null && !comment.equals("")) {
      writeUTF8(comment, containerHeaderBuffer);
    } else {
      writeUTF8("", containerHeaderBuffer);
    }

    // initialize buffer for writing zoom interval configurations
    bufferZoomIntervalConfig =
        randomAccessFile
            .getChannel()
            .map(
                MapMode.READ_WRITE,
                containerHeaderBuffer.position(),
                SIZE_ZOOMINTERVAL_CONFIGURATION * numberOfZoomIntervals);

    containerHeaderBuffer.position(
        containerHeaderBuffer.position() + SIZE_ZOOMINTERVAL_CONFIGURATION * numberOfZoomIntervals);

    // -4 bytes of header size variable itself
    int headerSize = containerHeaderBuffer.position() - headerSizePosition - 4;
    containerHeaderBuffer.putInt(headerSizePosition, headerSize);

    return containerHeaderBuffer.position();
  }