/**
* Maps blah file with a random offset and checks to see if data written out to the file can be
* read back in
*/
private static void testWrite() throws Exception {
StringBuilder sb = new StringBuilder();
sb.setLength(4);
for (int x = 0; x < 1000; x++) {
try (RandomAccessFile raf = new RandomAccessFile(blah, "rw")) {
FileChannel fc = raf.getChannel();
long offset = generator.nextInt(1000);
MappedByteBuffer b = fc.map(MapMode.READ_WRITE, offset, 100);
for (int i = 0; i < 4; i++) {
b.put(i, (byte) ('0' + i));
}
for (int i = 0; i < 4; i++) {
byte aByte = b.get(i);
sb.setCharAt(i, (char) aByte);
}
if (!sb.toString().equals("0123")) throw new Exception("Write test failed");
}
}
}
private void writeWayNodes(List<Integer> waynodes, int compressionType, MappedByteBuffer buffer) {
if (!waynodes.isEmpty() && waynodes.size() % 2 == 0) {
Iterator<Integer> waynodeIterator = waynodes.iterator();
buffer.putInt(waynodeIterator.next());
buffer.putInt(waynodeIterator.next());
while (waynodeIterator.hasNext()) {
switch (compressionType) {
case 0:
buffer.putInt(waynodeIterator.next().intValue());
buffer.putInt(waynodeIterator.next().intValue());
break;
case 1:
buffer.put(Serializer.getSignedThreeBytes(waynodeIterator.next().intValue()));
buffer.put(Serializer.getSignedThreeBytes(waynodeIterator.next().intValue()));
break;
case 2:
buffer.putShort(waynodeIterator.next().shortValue());
buffer.putShort(waynodeIterator.next().shortValue());
break;
case 3:
buffer.put(waynodeIterator.next().byteValue());
buffer.put(waynodeIterator.next().byteValue());
break;
}
}
}
}
/**
* @Description: 将文件的一部分映射入内存中 byte =char(FG:a,b c,d ) (byte) 97:a (byte) 122):Z byte就是最基本的单位 了,
* 如果是字符点 1byte , 如果 是int占2 byte,....
*
* @param filePath
* @param start
* @param size
* @throws IOException
*/
public static void mappedReadFile(String filePath, int start, int size) throws IOException {
// start = 0;
// size = 1024;
RandomAccessFile raf = new RandomAccessFile("C:\\usemappedfile.txt", "rw");
FileChannel fc = raf.getChannel();
MappedByteBuffer mbb = fc.map(FileChannel.MapMode.READ_WRITE, start, size);
// 可以用put方法, get方法设置相应的值
mbb.put(0, (byte) 97); // 将位置 致为相应值
mbb.put(1023, (byte) 122); // 将位置 致为相应值
// mbb.flip()
raf.close();
}
private void writeSubfileMetaDataToContainerHeader(
int i, long startIndexOfSubfile, long subfileSize) {
// HEADER META DATA FOR SUB FILE
// write zoom interval configuration to header
byte minZoomCurrentInterval = dataStore.getZoomIntervalConfiguration().getMinZoom(i);
byte maxZoomCurrentInterval = dataStore.getZoomIntervalConfiguration().getMaxZoom(i);
byte baseZoomCurrentInterval = dataStore.getZoomIntervalConfiguration().getBaseZoom(i);
bufferZoomIntervalConfig.put(baseZoomCurrentInterval);
bufferZoomIntervalConfig.put(minZoomCurrentInterval);
bufferZoomIntervalConfig.put(maxZoomCurrentInterval);
bufferZoomIntervalConfig.put(Serializer.getFiveBytes(startIndexOfSubfile));
bufferZoomIntervalConfig.put(Serializer.getFiveBytes(subfileSize));
}
// single pass attempt to copy if any can not accept the data then they are skipped
// and true will be returned instead of false.
private static boolean doingCopy(
TapeWriteStage ss,
int byteTailPos,
int primaryTailPos,
int totalPrimaryCopy,
int totalBytesCopy) {
IntBuffer primaryInts = RingBuffer.wrappedStructuredLayoutRingBuffer(ss.source);
ByteBuffer secondaryBytes = RingBuffer.wrappedUnstructuredLayoutRingBufferA(ss.source);
primaryInts.position(primaryTailPos);
primaryInts.limit(
primaryTailPos
+ totalPrimaryCopy); // TODO: AA, this will not work on the wrap, we must mask and do
// muliple copies
secondaryBytes.position(byteTailPos);
secondaryBytes.limit(byteTailPos + totalBytesCopy);
ss.header.clear();
ss.headerAsInts.put(totalBytesCopy + (totalPrimaryCopy << 2));
ss.headerAsInts.put(totalPrimaryCopy << 2);
// TODO: must return false if there is no room to write.
// TODO: BB, this creates a bit of garbage for the map, perhaps we should map larger blocks
// expecting to use them for multiple writes.
MappedByteBuffer mapped;
try {
mapped =
ss.fileChannel.map(
MapMode.READ_WRITE,
ss.fileChannel.position(),
8 + totalBytesCopy + (totalPrimaryCopy << 2));
mapped.put(ss.header);
IntBuffer asIntBuffer = mapped.asIntBuffer();
asIntBuffer.position(2);
asIntBuffer.put(primaryInts);
mapped.position(mapped.position() + (totalPrimaryCopy << 2));
mapped.put(secondaryBytes);
} catch (IOException e) {
throw new RuntimeException(e);
}
return true;
}
public static void main(String[] args) throws Exception {
fc = new RandomAccessFile("test.dat", "rw").getChannel();
MappedByteBuffer out = fc.map(FileChannel.MapMode.READ_WRITE, 0, LENGTH);
for (int i = 0; i < LENGTH; i++) out.put((byte) 'x');
new LockAndModify(out, 0, 0 + LENGTH / 3);
new LockAndModify(out, LENGTH / 2, LENGTH / 2 + LENGTH / 4);
}
private synchronized void appendCurrentBuffer(byte[] buf, int offset, int length)
throws IOException {
if (!TFS.requestSpace(length)) {
mCanWrite = false;
String msg =
"Local tachyon worker does not have enough "
+ "space ("
+ length
+ ") or no worker for "
+ FILE.FID
+ " "
+ BLOCK_ID;
if (PIN) {
TFS.outOfMemoryForPinFile(FILE.FID);
throw new IOException(msg);
}
throw new IOException(msg);
}
MappedByteBuffer out = mLocalFileChannel.map(MapMode.READ_WRITE, mInFileBytes, length);
out.put(buf, 0, length);
mInFileBytes += length;
}
private void writeRecordHeaderToMappedBuffer(RecordHeader header) {
mappedDataBuffer.position(header.getPosition());
mappedDataBuffer.putInt(header.getDataSize());
mappedDataBuffer.put(header.getFragmented());
if (header.isFragmented()) {
mappedDataBuffer.putInt(header.getNextPos());
}
}
@SuppressWarnings("deprecation")
@Override
protected void writeToBuffer(MappedByteBuffer buffer, Tuple e) {
KeyValue kv = KeyValueUtil.ensureKeyValue(e.getValue(0));
buffer.putInt(kv.getLength() + Bytes.SIZEOF_INT);
buffer.putInt(kv.getLength());
buffer.put(kv.getBuffer(), kv.getOffset(), kv.getLength());
}
/**
* 通过FileChannel.map()拿到MappedByteBuffer 使用内存文件映射，写2G大文件，速度会快很多
*
* @throws IOException
*/
public static void writeBigFileByMappedByteBuffer() throws IOException {
long start = System.currentTimeMillis();
File mbFile = new File("mb.dat");
if (mbFile.exists()) {
mbFile.delete();
}
RandomAccessFile raf = new RandomAccessFile(mbFile, "rw");
FileChannel fileChannel = raf.getChannel();
int pos = 0;
MappedByteBuffer mbb = null;
byte[] data = null;
long len = LEN;
int dataChunk = DATA_CHUNK / (1024 * 1024);
while (len >= DATA_CHUNK) {
System.out.println("write a data chunk:" + dataChunk + "MB");
mbb = fileChannel.map(MapMode.READ_WRITE, pos, DATA_CHUNK);
data = new byte[DATA_CHUNK];
mbb.put(data);
data = null;
len -= DATA_CHUNK;
pos += DATA_CHUNK;
}
if (len > 0) {
System.out.println("write rest data chunk:" + len + "B");
mbb = fileChannel.map(MapMode.READ_WRITE, pos, len);
data = new byte[(int) len];
mbb.put(data);
data = null;
}
// 关闭文件和通道流
fileChannel.close();
raf.close();
long end = System.currentTimeMillis();
System.out.println(String.format("===>写2G文件 " + mbFile.getName() + " 耗时：%s毫秒", end - start));
}
private void writeWordBlock(WordBlock block) throws IOException {
MappedByteBuffer buffer = fileChannel.map(MapMode.READ_WRITE, block.position, block_size);
byte[] data = block.encode();
if (data.length > block_size) {
throw new RuntimeException();
}
buffer.put(data);
// buffer.force();
}
public static void main(String[] args) throws Exception {
MappedByteBuffer out =
new RandomAccessFile("test.dat", "rw")
.getChannel()
.map(FileChannel.MapMode.READ_WRITE, 0, length);
for (int i = 0; i < length; i++) out.put((byte) 'x');
print("Finished writing");
for (int i = length / 2; i < length / 2 + 6; i++) printnb((char) out.get(i));
}
private static void corrupt(DataSegment segment) throws IOException {
final Map<String, Object> localLoadSpec = segment.getLoadSpec();
final Path segmentPath = Paths.get(localLoadSpec.get("path").toString());
final MappedByteBuffer buffer = Files.map(segmentPath.toFile(), FileChannel.MapMode.READ_WRITE);
while (buffer.hasRemaining()) {
buffer.put((byte) 0xFF);
}
}
/**
* Log a put operation.
*
* @param key The key of the value.
* @param value The value to place into the database.
* @param options Write options that define how the value is written to the memtable.
*/
public void put(final TableKey key, final TableValue value, final WriteOptions options) {
ByteBuffer data = serialize(key, value, options);
data.flip();
synchronized (this) {
if (logBuffer.remaining() < data.capacity()) {
logBuffer = mapWAL();
}
logBuffer.put(data);
}
}
public static void createFileForTest(Integer size, File file) throws IOException {
RandomAccessFile out = new RandomAccessFile(file, "rw");
FileChannel fc = out.getChannel();
long length = size;
MappedByteBuffer MappByteBuff = fc.map(FileChannel.MapMode.READ_WRITE, 0, length);
for (int i = 0; i < length; i++) {
MappByteBuff.put((byte) 'V');
}
fc.close();
System.out.println("File successFully written");
}
public static void main(String[] args) throws Exception {
MappedByteBuffer map = openMemoryMappedFile();
for (int value = 0; value < MESSAGES; value++) {
while (map.get(TAG_INDEX) != CAN_WRITE) {
loopWait();
}
map.putInt(DATA_INDEX, value);
map.put(TAG_INDEX, CAN_READ);
}
}
@Override
public void setChainHead(StoredBlock chainHead) throws BlockStoreException {
final MappedByteBuffer buffer = this.buffer;
if (buffer == null) throw new BlockStoreException("Store closed");
lock.lock();
try {
lastChainHead = chainHead;
byte[] headHash = chainHead.getHeader().getHash().getBytes();
buffer.position(8);
buffer.put(headHash);
} finally {
lock.unlock();
}
}
// Flush the current page to the memory mapped byte buffer
private void flushBuffer() throws BufferOverflowException {
Collection<byte[]> values = ImmutableMap.copyOf(pageMap).values();
buffer.clear();
// number of elements
buffer.putInt(values.size());
for (byte[] value : values) {
// element length
buffer.putInt(value.length);
// element
buffer.put(value, 0, value.length);
}
// Reset page stats
pageMap.clear();
totalResultSize = 0;
}
private void initNewStore(NetworkParameters params) throws Exception {
byte[] header;
header = HEADER_MAGIC.getBytes("US-ASCII");
buffer.put(header);
// Insert the genesis block.
lock.lock();
try {
setRingCursor(buffer, FILE_PROLOGUE_BYTES);
} finally {
lock.unlock();
}
Block genesis = params.getGenesisBlock().cloneAsHeader();
StoredBlock storedGenesis = new StoredBlock(genesis, genesis.getWork(), 0);
put(storedGenesis);
setChainHead(storedGenesis);
}
/**
* Log a series of writes. These writes are committed "atomically". Since all writes in the log
* are serialized, we just need to make sure that all these writes are contiguous.
*
* @param ops Set of write operations
*/
public void put(final WriteBatch ops) {
// Lock the log so that we can commit several at a time.
synchronized (this) {
Iterator<TableKey> iter = ops.iterator();
while (iter.hasNext()) {
TableKey key = iter.next();
for (WriteBatch.TableWrite write : ops.getValues(key)) {
ByteBuffer data = serialize(key, write.value, write.options);
if (logBuffer.remaining() < data.position()) {
logBuffer = mapWAL();
}
data.flip();
logBuffer.put(data);
}
}
}
}
public int testWrite(final String fileName) throws Exception {
FileChannel channel = new RandomAccessFile(fileName, "rw").getChannel();
MappedByteBuffer buffer = channel.map(READ_WRITE, 0, Math.min(channel.size(), MAX_VALUE));
int checkSum = 0;
for (long i = 0; i < FILE_SIZE; i++) {
if (!buffer.hasRemaining()) {
buffer = channel.map(READ_WRITE, i, Math.min(channel.size() - i, MAX_VALUE));
}
byte b = (byte) i;
checkSum += b;
buffer.put(b);
}
channel.close();
return checkSum;
}
public static void main(String[] args) throws Exception {
MappedByteBuffer map = openMemoryMappedFile();
long start = System.currentTimeMillis();
for (int expected = 0; expected < MESSAGES; expected++) {
while (map.get(TAG_INDEX) != CAN_READ) {
loopWait();
}
int value = map.getInt(DATA_INDEX);
assertThat(value, is(expected));
map.put(TAG_INDEX, CAN_WRITE);
}
long end = System.currentTimeMillis();
long durationMs = end - start;
double nsPerMessage = durationMs * 1000000.0 / MESSAGES;
System.out.println("read " + MESSAGES + " messages in " + durationMs + " ms");
System.out.println(nsPerMessage + " ns/roundtrip, " + (nsPerMessage / 2) + " ns IPC latency");
}
/** @tests {@link java.nio.MappedByteBuffer#force()} */
public void test_force() throws IOException {
// buffer was not mapped in read/write mode
FileInputStream fileInputStream = new FileInputStream(tmpFile);
FileChannel fileChannelRead = fileInputStream.getChannel();
MappedByteBuffer mmbRead = fileChannelRead.map(MapMode.READ_ONLY, 0, fileChannelRead.size());
mmbRead.force();
FileInputStream inputStream = new FileInputStream(tmpFile);
FileChannel fileChannelR = inputStream.getChannel();
MappedByteBuffer resultRead = fileChannelR.map(MapMode.READ_ONLY, 0, fileChannelR.size());
// If this buffer was not mapped in read/write mode, then invoking this method has no effect.
assertEquals(
"Invoking force() should have no effect when this buffer was not mapped in read/write mode",
mmbRead,
resultRead);
// Buffer was mapped in read/write mode
RandomAccessFile randomFile = new RandomAccessFile(tmpFile, "rw");
FileChannel fileChannelReadWrite = randomFile.getChannel();
MappedByteBuffer mmbReadWrite =
fileChannelReadWrite.map(FileChannel.MapMode.READ_WRITE, 0, fileChannelReadWrite.size());
mmbReadWrite.put((byte) 'o');
mmbReadWrite.force();
RandomAccessFile random = new RandomAccessFile(tmpFile, "rw");
FileChannel fileChannelRW = random.getChannel();
MappedByteBuffer resultReadWrite =
fileChannelRW.map(FileChannel.MapMode.READ_WRITE, 0, fileChannelRW.size());
// Invoking force() will change the buffer
assertFalse(mmbReadWrite.equals(resultReadWrite));
fileChannelRead.close();
fileChannelR.close();
fileChannelReadWrite.close();
fileChannelRW.close();
}
@Override
public void put(StoredBlock block) throws BlockStoreException {
final MappedByteBuffer buffer = this.buffer;
if (buffer == null) throw new BlockStoreException("Store closed");
lock.lock();
try {
int cursor = getRingCursor(buffer);
if (cursor == getFileSize()) {
// Wrapped around.
cursor = FILE_PROLOGUE_BYTES;
}
buffer.position(cursor);
Sha256Hash hash = block.getHeader().getHash();
notFoundCache.remove(hash);
buffer.put(hash.getBytes());
block.serializeCompact(buffer);
setRingCursor(buffer, buffer.position());
blockCache.put(hash, block);
} finally {
lock.unlock();
}
}
private void writeUTF8(String string, MappedByteBuffer buffer) {
buffer.putShort((short) string.getBytes().length);
buffer.put(string.getBytes());
}
private void writeRecordDataToMappedBuffer(RecordHeader header, byte[] data) {
mappedDataBuffer.position(header.getPosition() + header.getSize());
mappedDataBuffer.put(data);
}
public static void main(String[] argv) throws Exception {
// Create a temp file and get a channel connected to it
File tempFile = File.createTempFile("mmaptest", null);
RandomAccessFile file = new RandomAccessFile(tempFile, "rw");
FileChannel channel = file.getChannel();
// write strings to file by channel
ByteBuffer temp = ByteBuffer.allocate(100);
temp.put("This is the file content".getBytes());
temp.flip();
channel.write(temp, 0);
temp.clear();
temp.put("This is more file content".getBytes());
temp.flip();
channel.write(temp, 8192);
// readonly
MappedByteBuffer ro = channel.map(FileChannel.MapMode.READ_ONLY, 0, channel.size());
// read and write
MappedByteBuffer rw = channel.map(FileChannel.MapMode.READ_WRITE, 0, channel.size());
// copy on write
MappedByteBuffer cow = channel.map(FileChannel.MapMode.PRIVATE, 0, channel.size());
System.out.println("Begin");
// ro、rw、cow 相同
showBuffers(ro, rw, cow);
// Modify the copy-on-write buffer，copy-on-write不会影响数据源
cow.position(8);
cow.put("COW".getBytes());
System.out.println("Change to COW buffer");
showBuffers(ro, rw, cow);
// Modify the read/write buffer，read/write会影响数据源
rw.position(9);
rw.put(" R/W ".getBytes());
rw.position(8194);
rw.put(" R/W ".getBytes());
rw.force();
System.out.println("Change to R/W buffer");
showBuffers(ro, rw, cow);
// Write to the file through the channel; hit both pages
temp.clear();
temp.put("Channel write ".getBytes());
temp.flip();
channel.write(temp, 0);
temp.rewind();
channel.write(temp, 8202);
System.out.println("Write on channel");
showBuffers(ro, rw, cow);
// Modify the copy-on-write buffer again
cow.position(8207);
cow.put(" COW2 ".getBytes());
System.out.println("Second change to COW buffer");
showBuffers(ro, rw, cow);
// Modify the read/write buffer
rw.position(0);
rw.put(" R/W2 ".getBytes());
rw.position(8210);
rw.put(" R/W2 ".getBytes());
rw.force();
System.out.println("Second change to R/W buffer");
showBuffers(ro, rw, cow);
}
private void appendWhitespace(int amount, MappedByteBuffer buffer) {
for (int i = 0; i < amount; i++) {
buffer.put((byte) ' ');
}
}
private long writeSubfile(
long startPositionSubfile,
int zoomIntervalIndex,
boolean debugStrings,
boolean waynodeCompression,
boolean polygonClipping,
boolean pixelCompression)
throws IOException {
logger.fine(
"writing data for zoom interval "
+ zoomIntervalIndex
+ ", number of tiles: "
+ dataStore.numberOfHorizontalTiles(zoomIntervalIndex)
* dataStore.numberOfVerticalTiles(zoomIntervalIndex));
TileCoordinate upperLeft = dataStore.getUpperLeft(zoomIntervalIndex);
int lengthX = dataStore.numberOfHorizontalTiles(zoomIntervalIndex);
int lengthY = dataStore.numberOfVerticalTiles(zoomIntervalIndex);
byte minZoomCurrentInterval =
dataStore.getZoomIntervalConfiguration().getMinZoom(zoomIntervalIndex);
byte maxZoomCurrentInterval =
dataStore.getZoomIntervalConfiguration().getMaxZoom(zoomIntervalIndex);
byte baseZoomCurrentInterval =
dataStore.getZoomIntervalConfiguration().getBaseZoom(zoomIntervalIndex);
byte maxMaxZoomlevel = dataStore.getZoomIntervalConfiguration().getMaxMaxZoom();
int tileAmountInBytes =
dataStore.numberOfHorizontalTiles(zoomIntervalIndex)
* dataStore.numberOfVerticalTiles(zoomIntervalIndex)
* BYTE_AMOUNT_SUBFILE_INDEX_PER_TILE;
int indexBufferSize =
tileAmountInBytes + (debugStrings ? DEBUG_INDEX_START_STRING.getBytes().length : 0);
MappedByteBuffer indexBuffer =
randomAccessFile
.getChannel()
.map(MapMode.READ_WRITE, startPositionSubfile, indexBufferSize);
MappedByteBuffer tileBuffer =
randomAccessFile
.getChannel()
.map(MapMode.READ_WRITE, startPositionSubfile + indexBufferSize, TILE_BUFFER_SIZE);
long currentSubfileOffset = indexBufferSize;
for (int tileY = upperLeft.getY(); tileY < upperLeft.getY() + lengthY; tileY++) {
for (int tileX = upperLeft.getX(); tileX < upperLeft.getX() + lengthX; tileX++) {
// logger.info("writing data for tile (" + tileX + ", " + tileY + ")");
long currentTileOffsetInBuffer = tileBuffer.position();
TileCoordinate currentTileCoordinate =
new TileCoordinate(tileX, tileY, baseZoomCurrentInterval);
// seek to index frame of this tile and write relative offset of this
// tile as five bytes to the index
indexBuffer.put(Serializer.getFiveBytes(currentSubfileOffset));
// get statistics for tile
TileData currentTile = dataStore.getTile(zoomIntervalIndex, tileX, tileY);
// ************* POI ************
// write amount of POIs and ways for each zoom level
// TODO is this computation correct? Ways that have an associated zoom level of
// e.g. 9
// are lifted to zoom level 12 for an interval 12,14,17
Map<Byte, List<TDNode>> poisByZoomlevel =
currentTile.poisByZoomlevel(minZoomCurrentInterval, maxMaxZoomlevel);
Map<Byte, List<TDWay>> waysByZoomlevel =
currentTile.waysByZoomlevel(minZoomCurrentInterval, maxMaxZoomlevel);
if (poisByZoomlevel.size() > 0 || waysByZoomlevel.size() > 0) {
int tileContainerStart = tileBuffer.position();
if (debugStrings) {
// write tile header
StringBuilder sb = new StringBuilder();
sb.append(DEBUG_STRING_TILE_HEAD)
.append(tileX)
.append(",")
.append(tileY)
.append(DEBUG_STRING_TILE_TAIL);
tileBuffer.put(sb.toString().getBytes());
// append withespaces so that block has 32 bytes
appendWhitespace(32 - sb.toString().getBytes().length, tileBuffer);
}
short cumulatedPOIs = 0;
short cumulatedWays = 0;
for (byte zoomlevel = minZoomCurrentInterval;
zoomlevel <= maxZoomCurrentInterval;
zoomlevel++) {
if (poisByZoomlevel.get(zoomlevel) != null)
cumulatedPOIs += poisByZoomlevel.get(zoomlevel).size();
if (waysByZoomlevel.get(zoomlevel) != null)
cumulatedWays += waysByZoomlevel.get(zoomlevel).size();
tileBuffer.putShort(cumulatedPOIs);
tileBuffer.putShort(cumulatedWays);
}
// skip 4 bytes, later these 4 bytes will contain the start
// position of the ways in this tile
int fileIndexStartWayContainer = tileBuffer.position();
tileBuffer.position(fileIndexStartWayContainer + 4);
// write POIs for each zoom level beginning with lowest zoom level
for (byte zoomlevel = minZoomCurrentInterval;
zoomlevel <= maxZoomCurrentInterval;
zoomlevel++) {
List<TDNode> pois = poisByZoomlevel.get(zoomlevel);
if (pois == null) continue;
for (TDNode poi : pois) {
if (debugStrings) {
StringBuilder sb = new StringBuilder();
sb.append(DEBUG_STRING_POI_HEAD).append(poi.getId()).append(DEBUG_STRING_POI_TAIL);
tileBuffer.put(sb.toString().getBytes());
// append withespaces so that block has 32 bytes
appendWhitespace(32 - sb.toString().getBytes().length, tileBuffer);
}
// write poi features to the file
tileBuffer.putInt(poi.getLatitude());
tileBuffer.putInt(poi.getLongitude());
// write byte with layer and tag amount info
tileBuffer.put(
buildLayerTagAmountByte(
poi.getLayer(), poi.getTags() == null ? 0 : (short) poi.getTags().size()));
// write tag ids to the file
if (poi.getTags() != null) {
for (PoiEnum poiEnum : poi.getTags()) {
tileBuffer.putShort((short) poiEnum.ordinal());
}
}
// write byte with bits set to 1 if the poi has a name, an elevation
// or a housenumber
tileBuffer.put(
buildInfoByteForPOI(poi.getName(), poi.getElevation(), poi.getHouseNumber()));
if (poi.getName() != null && poi.getName().length() > 0) {
writeUTF8(poi.getName(), tileBuffer);
}
if (poi.getElevation() != 0) {
tileBuffer.putShort(poi.getElevation());
}
if (poi.getHouseNumber() != null && poi.getHouseNumber().length() > 0) {
writeUTF8(poi.getHouseNumber(), tileBuffer);
}
}
} // end for loop over POIs
// write offset to first way in the tile header
tileBuffer.putInt(fileIndexStartWayContainer, tileBuffer.position() - tileContainerStart);
// ************* WAYS ************
// write ways
for (byte zoomlevel = minZoomCurrentInterval;
zoomlevel <= maxZoomCurrentInterval;
zoomlevel++) {
List<TDWay> ways = waysByZoomlevel.get(zoomlevel);
if (ways == null) continue;
// use executor service to parallelize computation of subtile bitmasks
// for all
// ways in the current tile
short[] bitmaskComputationResults = computeSubtileBitmasks(ways, currentTileCoordinate);
assert bitmaskComputationResults.length == ways.size();
// needed to access bitmask computation results in the foreach loop
int i = 0;
for (TDWay way : ways) {
// // INNER WAY
// // inner ways will be written as part of the outer way
// if (way.isInnerWay())
// continue;
int startIndexWay = tileBuffer.position();
WayNodePreprocessingResult wayNodePreprocessingResult =
preprocessWayNodes(
way,
waynodeCompression,
pixelCompression,
polygonClipping,
maxZoomCurrentInterval,
minZoomCurrentInterval,
currentTileCoordinate);
if (wayNodePreprocessingResult == null) {
continue;
}
if (debugStrings) {
StringBuilder sb = new StringBuilder();
sb.append(DEBUG_STRING_WAY_HEAD).append(way.getId()).append(DEBUG_STRING_WAY_TAIL);
tileBuffer.put(sb.toString().getBytes());
// append withespaces so that block has 32 bytes
appendWhitespace(32 - sb.toString().getBytes().length, tileBuffer);
}
// skip 4 bytes to reserve space for way size
int startIndexWaySize = tileBuffer.position();
tileBuffer.position(startIndexWaySize + 4);
// write way features
// short bitmask = GeoUtils.computeBitmask(way,
// currentTileCoordinate);
// short bitmask = (short) 0xffff;
tileBuffer.putShort(bitmaskComputationResults[i++]);
// write byte with layer and tag amount
tileBuffer.put(
buildLayerTagAmountByte(
way.getLayer(), way.getTags() == null ? 0 : (short) way.getTags().size()));
// set type of the way node compression
int compressionType = wayNodePreprocessingResult.getCompressionType();
// write byte with amount of tags which are rendered
tileBuffer.put(buildRenderTagWayNodeCompressionByte(way.getTags(), compressionType));
// write tag bitmap
tileBuffer.put(buildTagBitmapByte(way.getTags()));
// file.writeByte((byte) 0xff);
// write tag ids
if (way.getTags() != null) {
for (WayEnum wayEnum : way.getTags()) {
tileBuffer.putShort((short) wayEnum.ordinal());
}
}
// write the amount of way nodes to the file
tileBuffer.putShort(
(short) (wayNodePreprocessingResult.getWaynodesAsList().size() / 2));
// write the way nodes:
// the first node is always stored with four bytes
// the remaining way node differences are stored according to the
// compression type
writeWayNodes(
wayNodePreprocessingResult.getWaynodesAsList(),
wayNodePreprocessingResult.getCompressionType(),
tileBuffer);
// write a byte with name, label and way type information
tileBuffer.put(buildInfoByteForWay(way.getName(), way.getWaytype(), way.getRef()));
// // if the way has a name, write it to the file
if (way.getName() != null && way.getName().length() > 0) {
writeUTF8(way.getName(), tileBuffer);
}
// if the way has a ref, write it to the file
if (way.getRef() != null && way.getRef().length() > 0) {
writeUTF8(way.getRef(), tileBuffer);
}
//
// // // if the way has a label position write it to the file
// // if (labelPositionLatitude != 0 && labelPositionLongitude != 0)
// {
// // raf.writeInt(labelPositionLatitude);
// // raf.writeInt(labelPositionLongitude);
// // }
//
// *********MULTIPOLYGON PROCESSING***********
if (way.getWaytype() == 3
&& dataStore.getInnerWaysOfMultipolygon(way.getId()) != null) {
List<TDWay> innerways = dataStore.getInnerWaysOfMultipolygon(way.getId());
if (innerways == null) {
tileBuffer.put((byte) 0);
} else {
tileBuffer.put((byte) innerways.size());
for (TDWay innerway : innerways) {
WayNodePreprocessingResult innerWayNodePreprocessingResult =
preprocessWayNodes(
innerway,
waynodeCompression,
pixelCompression,
false,
maxZoomCurrentInterval,
minZoomCurrentInterval,
currentTileCoordinate);
// write the amount of way nodes to the file
tileBuffer.putShort(
(short) (innerWayNodePreprocessingResult.getWaynodesAsList().size() / 2));
writeWayNodes(
innerWayNodePreprocessingResult.getWaynodesAsList(),
wayNodePreprocessingResult.getCompressionType(),
tileBuffer);
}
}
}
// write the size of the way to the file
tileBuffer.putInt(startIndexWaySize, tileBuffer.position() - startIndexWay);
}
} // end for loop over ways
} // end if clause checking if tile is empty or not
long tileSize = tileBuffer.position() - currentTileOffsetInBuffer;
currentSubfileOffset += tileSize;
// if necessary, allocate new buffer
if (tileBuffer.remaining() < MIN_TILE_BUFFER_SIZE)
tileBuffer =
randomAccessFile
.getChannel()
.map(
MapMode.READ_WRITE,
startPositionSubfile + currentSubfileOffset,
TILE_BUFFER_SIZE);
tilesProcessed++;
if (tilesProcessed % fivePercentOfTilesToProcess == 0) {
logger.info(
"written " + (tilesProcessed / fivePercentOfTilesToProcess) * 5 + "% of file");
}
} // end for loop over tile columns
} // /end for loop over tile rows
// return size of sub file in bytes
return currentSubfileOffset;
}
private long writeContainerHeader(
long date,
int version,
short tilePixel,
String comment,
boolean debugStrings,
boolean waynodeCompression,
boolean polygonClipping,
boolean pixelCompression,
GeoCoordinate mapStartPosition)
throws IOException {
// get metadata for the map file
int numberOfZoomIntervals = dataStore.getZoomIntervalConfiguration().getNumberOfZoomIntervals();
logger.fine("writing header");
MappedByteBuffer containerHeaderBuffer =
randomAccessFile.getChannel().map(MapMode.READ_WRITE, 0, HEADER_BUFFER_SIZE);
// write file header
// magic byte
byte[] magicBytes = MAGIC_BYTE.getBytes();
containerHeaderBuffer.put(magicBytes);
// write container header size
int headerSizePosition = containerHeaderBuffer.position();
containerHeaderBuffer.position(headerSizePosition + 4);
// version number of the binary file format
containerHeaderBuffer.putInt(version);
// meta info byte
containerHeaderBuffer.put(
buildMetaInfoByte(
debugStrings,
mapStartPosition != null,
pixelCompression,
polygonClipping,
waynodeCompression));
// amount of map files inside this file
containerHeaderBuffer.put((byte) numberOfZoomIntervals);
// projection type
writeUTF8(PROJECTION, containerHeaderBuffer);
// width and height of a tile in pixel
containerHeaderBuffer.putShort(tilePixel);
logger.fine(
"Bounding box for file: "
+ dataStore.getBoundingBox().maxLatitudeE6
+ ", "
+ dataStore.getBoundingBox().minLongitudeE6
+ ", "
+ dataStore.getBoundingBox().minLatitudeE6
+ ", "
+ dataStore.getBoundingBox().maxLongitudeE6);
// upper left corner of the bounding box
containerHeaderBuffer.putInt(dataStore.getBoundingBox().maxLatitudeE6);
containerHeaderBuffer.putInt(dataStore.getBoundingBox().minLongitudeE6);
containerHeaderBuffer.putInt(dataStore.getBoundingBox().minLatitudeE6);
containerHeaderBuffer.putInt(dataStore.getBoundingBox().maxLongitudeE6);
if (mapStartPosition != null) {
containerHeaderBuffer.putInt(mapStartPosition.getLatitudeE6());
containerHeaderBuffer.putInt(mapStartPosition.getLongitudeE6());
}
// date of the map data
containerHeaderBuffer.putLong(date);
// store the mapping of tags to tag ids
containerHeaderBuffer.putShort((short) PoiEnum.values().length);
for (PoiEnum poiEnum : PoiEnum.values()) {
writeUTF8(poiEnum.toString(), containerHeaderBuffer);
containerHeaderBuffer.putShort((short) poiEnum.ordinal());
}
containerHeaderBuffer.putShort((short) WayEnum.values().length);
for (WayEnum wayEnum : WayEnum.values()) {
writeUTF8(wayEnum.toString(), containerHeaderBuffer);
containerHeaderBuffer.putShort((short) wayEnum.ordinal());
}
// comment
if (comment != null && !comment.equals("")) {
writeUTF8(comment, containerHeaderBuffer);
} else {
writeUTF8("", containerHeaderBuffer);
}
// initialize buffer for writing zoom interval configurations
bufferZoomIntervalConfig =
randomAccessFile
.getChannel()
.map(
MapMode.READ_WRITE,
containerHeaderBuffer.position(),
SIZE_ZOOMINTERVAL_CONFIGURATION * numberOfZoomIntervals);
containerHeaderBuffer.position(
containerHeaderBuffer.position() + SIZE_ZOOMINTERVAL_CONFIGURATION * numberOfZoomIntervals);
// -4 bytes of header size variable itself
int headerSize = containerHeaderBuffer.position() - headerSizePosition - 4;
containerHeaderBuffer.putInt(headerSizePosition, headerSize);
return containerHeaderBuffer.position();
}