/** * Create a roaring array based on a previously serialized ByteBuffer. As much as possible, the * ByteBuffer is used as the backend, however if you modify the content, the result is * unspecified. * * @param bb The source ByteBuffer */ protected RoaringArray(ByteBuffer bb) { bb.order(ByteOrder.LITTLE_ENDIAN); if (bb.getInt() != SERIAL_COOKIE) throw new RuntimeException("I failed to find the right cookie."); this.size = bb.getInt(); // we fully read the meta-data array to RAM, but the containers // themselves are memory-mapped this.array = new Element[this.size]; final short keys[] = new short[this.size]; final int cardinalities[] = new int[this.size]; final boolean isBitmap[] = new boolean[this.size]; for (int k = 0; k < this.size; ++k) { keys[k] = bb.getShort(); cardinalities[k] = Util.toIntUnsigned(bb.getShort()) + 1; isBitmap[k] = cardinalities[k] > ArrayContainer.DEFAULT_MAX_SIZE; } for (int k = 0; k < this.size; ++k) { if (cardinalities[k] == 0) throw new RuntimeException("no"); Container val; if (isBitmap[k]) { final LongBuffer bitmapArray = bb.asLongBuffer().slice(); bitmapArray.limit(BitmapContainer.MAX_CAPACITY / 64); bb.position(bb.position() + BitmapContainer.MAX_CAPACITY / 8); val = new BitmapContainer(bitmapArray, cardinalities[k]); } else { final ShortBuffer shortArray = bb.asShortBuffer().slice(); shortArray.limit(cardinalities[k]); bb.position(bb.position() + cardinalities[k] * 2); val = new ArrayContainer(shortArray, cardinalities[k]); } this.array[k] = new Element(keys[k], val); } }
/** * Receive as many items as possible from the given byte buffer to this buffer. * * <p>The <TT>receiveItems()</TT> method must not block the calling thread; if it does, all * message I/O in MP will be blocked. * * @param i Index of first item to receive, in the range 0 .. <TT>length</TT>-1. * @param num Maximum number of items to receive. * @param buffer Byte buffer. * @return Number of items received. */ protected int receiveItems(int i, int num, ByteBuffer buffer) { LongBuffer longbuffer = buffer.asLongBuffer(); int n = 0; int r = i / myColCount; int row = r * myRowStride + myLowerRow; int c = i % myColCount; int col = c * myColStride + myLowerCol; int ncols = Math.min(myColCount - c, longbuffer.remaining()); while (r < myRowCount && ncols > 0) { long[] myMatrix_row = myMatrix[row]; while (c < ncols) { myMatrix_row[col] = myOp.op(myMatrix_row[col], longbuffer.get()); ++c; col += myColStride; } n += ncols; ++r; row += myRowStride; c = 0; col = myLowerCol; ncols = Math.min(myColCount, longbuffer.remaining()); } buffer.position(buffer.position() + 8 * n); return n; }
@Override public void onNewData(ByteBuffer decompressed) { long timestamp = decompressed.getLong(); LongBuffer data = decompressed.asLongBuffer(); for (int i = 0; i < variables.size(); i++) { YoVariable<?> variable = variables.get(i); long previousValue = variable.getValueAsLongBits(); long newValue = data.get(); variable.setValueFromLongBits(newValue, false); if (previousValue != newValue) { ArrayList<VariableChangedListener> changedListeners = variable.getVariableChangedListeners(); if (changedListeners != null) { for (int listener = 0; listener < changedListeners.size(); listener++) { VariableChangedListener changedListener = changedListeners.get(listener); if (!(changedListener instanceof LogControlVariableChangeListener)) { changedListener.variableChanged(variable); } } } } } for (int i = 0; i < jointStates.size(); i++) { jointStates.get(i).update(data); } listener.receivedTimestampAndData(timestamp, decompressed); }
/** * Receive as many items as possible from the given byte buffer to this buffer. * * <p>The <TT>receiveItems()</TT> method must not block the calling thread; if it does, all * message I/O in MP will be blocked. * * @param i Index of first item to receive, in the range 0 .. <TT>length</TT>-1. * @param num Maximum number of items to receive. * @param buffer Byte buffer. * @return Number of items received. */ protected int receiveItems(int i, int num, ByteBuffer buffer) { LongBuffer longbuffer = buffer.asLongBuffer(); int n = Math.min(num, Math.min(myLength - i, longbuffer.remaining())); longbuffer.get(myArray, myArrayOffset + i, n); buffer.position(buffer.position() + 8 * n); return n; }
/** Creates a percentiler from its serialized representation. */ public Percentiler(byte[] data) { ByteBuffer in = ByteBuffer.wrap(data); // read our int data IntBuffer iin = in.asIntBuffer(); _max = iin.get(); iin.get(_counts); in.position(iin.position() * INT_SIZE); // read our long data LongBuffer lin = in.asLongBuffer(); _snapTotal = (_total = lin.get()); in.position(iin.position() * INT_SIZE + lin.position() * 2 * INT_SIZE); // read our min value (which was added afterwards and must do some jockeying to maintain // backwards compatibility) if (in.position() == in.limit()) { _min = 0; // legacy } else { _min = in.asIntBuffer().get(); } // Un-break percentilers that have been stored with bogus data if (_max < _min) { log.warning("Percentiler initialized with bogus range. Coping.", "min", _min, "max", _max); _max = _min + 1; } // compute our percentiles recomputePercentiles(); }
public SpillRecord(Path indexFileName, JobConf job, Checksum crc, String expectedIndexOwner) throws IOException { final FileSystem rfs = FileSystem.getLocal(job).getRaw(); final DataInputStream in = new DataInputStream( SecureIOUtils.openForRead( new File(indexFileName.toUri().getPath()), expectedIndexOwner, null)); try { final long length = rfs.getFileStatus(indexFileName).getLen(); final int partitions = (int) length / MAP_OUTPUT_INDEX_RECORD_LENGTH; final int size = partitions * MAP_OUTPUT_INDEX_RECORD_LENGTH; buf = ByteBuffer.allocate(size); if (crc != null) { crc.reset(); CheckedInputStream chk = new CheckedInputStream(in, crc); IOUtils.readFully(chk, buf.array(), 0, size); if (chk.getChecksum().getValue() != in.readLong()) { throw new ChecksumException("Checksum error reading spill index: " + indexFileName, -1); } } else { IOUtils.readFully(in, buf.array(), 0, size); } entries = buf.asLongBuffer(); } finally { in.close(); } }
public static void main(String[] args) { ByteBuffer bb = ByteBuffer.allocate(BSIZE); // Allocation automatically zeroes the ByteBuffer: int i = 0; while (i++ < bb.limit()) if (bb.get() != 0) print("nonzero"); print("i = " + i); bb.rewind(); // Store and read a char array: bb.asCharBuffer().put("Howdy!"); char c; while ((c = bb.getChar()) != 0) printnb(c + " "); print(); bb.rewind(); // Store and read a short: bb.asShortBuffer().put((short) 471142); print(bb.getShort()); bb.rewind(); // Store and read an int: bb.asIntBuffer().put(99471142); print(bb.getInt()); bb.rewind(); // Store and read a long: bb.asLongBuffer().put(99471142); print(bb.getLong()); bb.rewind(); // Store and read a float: bb.asFloatBuffer().put(99471142); print(bb.getFloat()); bb.rewind(); // Store and read a double: bb.asDoubleBuffer().put(99471142); print(bb.getDouble()); bb.rewind(); }
public static void main(String... args) { ByteBuffer bb = ByteBuffer.allocate(8); System.out.println("Native order = " + ByteOrder.nativeOrder()); System.out.println("Default order = " + bb.order()); // bb.order(ByteOrder.BIG_ENDIAN) ; // bb.order(ByteOrder.LITTLE_ENDIAN) ; System.out.println("Order = " + bb.order()); bb.asLongBuffer().put(0x0102030405060708L); for (int i = 0; i < bb.capacity(); i++) System.out.printf("0x%02X ", bb.get(i)); // Comes out hight to low : 0x01 0x02 0x03 0x04 0x05 0x06 0x07 0x08 }
public MappedDatabase( final ByteBuffer db, final DataOutput dupStream, final PrintWriter solverLog) { final LongBuffer ldb = db.asLongBuffer(); if ((ldb.limit() & 1) == 0) { this.db = ldb; this.solvers = new IDMap<>(1 << 12); this.dupStream = dupStream; this.solverLog = solverLog; return; } throw new IllegalArgumentException("Truncated Buffer"); }
public static void main(String[] args) throws Exception { DatagramChannel dc = DatagramChannel.open(); DatagramSocket socket = dc.socket(); socket.bind(new InetSocketAddress(22500)); ByteBuffer byteBuf = ByteBuffer.allocate(8); // byteBuf.order(ByteOrder.LITTLE_ENDIAN); LongBuffer longBuf = byteBuf.asLongBuffer(); while (true) { dc.receive(byteBuf); System.out.println(longBuf.get(0)); byteBuf.clear(); } }
public NioWrapLittleConversion() { // big/little endian difference one liner order = ByteOrder.LITTLE_ENDIAN; byteBuffer = ByteBuffer.allocateDirect(BUFFER_SIZE).order(order); // views on the bytebuffer to fill/drain it efficiently charBuffer = byteBuffer.asCharBuffer(); shortBuffer = byteBuffer.asShortBuffer(); intBuffer = byteBuffer.asIntBuffer(); longBuffer = byteBuffer.asLongBuffer(); floatBuffer = byteBuffer.asFloatBuffer(); doubleBuffer = byteBuffer.asDoubleBuffer(); }
@NotNull public static IndexedCollection<Long> ofLong(@NotNull ByteBuffer byteBuffer) { int magic = byteBuffer.getInt(); if (magic != MAGIC) { throw new IllegalArgumentException("bad magic number"); } int version = byteBuffer.getInt(); if (version != VERSION) { throw new IllegalArgumentException("bad version number"); } int size = byteBuffer.getInt(); LongBuffer values = byteBuffer.asLongBuffer(); values.limit(size); byteBuffer.position(byteBuffer.position() + size * Integer.BYTES); return new LongBufferCollection(values.slice()); }
/** Converts this percentiler to a byte array so that it may be stored into a database. */ public byte[] toBytes() { byte[] data = new byte[(BUCKET_COUNT + 4) * INT_SIZE]; ByteBuffer out = ByteBuffer.wrap(data); // write our int data IntBuffer iout = out.asIntBuffer(); iout.put(_max); iout.put(_counts); out.position(iout.position() * INT_SIZE); // write our long data LongBuffer lout = out.asLongBuffer(); lout.put(_total); out.position(iout.position() * INT_SIZE + lout.position() * 2 * INT_SIZE); // write our min value (added later so we can't write it above like we wish we could) out.asIntBuffer().put(_min); return data; }
/** Allocates a new packed image frame in native memory where rows are 8-byte aligned. */ public Frame(int width, int height, int depth, int channels) { int pixelSize = Math.abs(depth) / 8; this.imageWidth = width; this.imageHeight = height; this.imageDepth = depth; this.imageChannels = channels; this.imageStride = ((imageWidth * imageChannels * pixelSize + 7) & ~7) / pixelSize; // 8-byte aligned this.image = new Buffer[1]; ByteBuffer buffer = ByteBuffer.allocateDirect(imageHeight * imageStride * pixelSize) .order(ByteOrder.nativeOrder()); switch (imageDepth) { case DEPTH_BYTE: case DEPTH_UBYTE: image[0] = buffer; break; case DEPTH_SHORT: case DEPTH_USHORT: image[0] = buffer.asShortBuffer(); break; case DEPTH_INT: image[0] = buffer.asIntBuffer(); break; case DEPTH_LONG: image[0] = buffer.asLongBuffer(); break; case DEPTH_FLOAT: image[0] = buffer.asFloatBuffer(); break; case DEPTH_DOUBLE: image[0] = buffer.asDoubleBuffer(); break; default: throw new UnsupportedOperationException("Unsupported depth value: " + imageDepth); } }
@Override public Blob getBlob(final String container, final String key) { BlobBuilder builder = blobBuilders.get(); builder.name(key); File file = getFileForBlobKey(container, key); ByteSource byteSource; if (getDirectoryBlobSuffix(key) != null) { logger.debug("%s - %s is a directory", container, key); byteSource = ByteSource.empty(); } else { byteSource = Files.asByteSource(file); } try { String cacheControl = null; String contentDisposition = null; String contentEncoding = null; String contentLanguage = null; String contentType = null; HashCode hashCode = null; Date expires = null; ImmutableMap.Builder<String, String> userMetadata = ImmutableMap.builder(); UserDefinedFileAttributeView view = getUserDefinedFileAttributeView(file.toPath()); if (view != null) { Set<String> attributes = ImmutableSet.copyOf(view.list()); cacheControl = readStringAttributeIfPresent(view, attributes, XATTR_CACHE_CONTROL); contentDisposition = readStringAttributeIfPresent(view, attributes, XATTR_CONTENT_DISPOSITION); contentEncoding = readStringAttributeIfPresent(view, attributes, XATTR_CONTENT_ENCODING); contentLanguage = readStringAttributeIfPresent(view, attributes, XATTR_CONTENT_LANGUAGE); contentType = readStringAttributeIfPresent(view, attributes, XATTR_CONTENT_TYPE); if (contentType == null && autoDetectContentType) { contentType = probeContentType(file.toPath()); } if (attributes.contains(XATTR_CONTENT_MD5)) { ByteBuffer buf = ByteBuffer.allocate(view.size(XATTR_CONTENT_MD5)); view.read(XATTR_CONTENT_MD5, buf); hashCode = HashCode.fromBytes(buf.array()); } if (attributes.contains(XATTR_EXPIRES)) { ByteBuffer buf = ByteBuffer.allocate(view.size(XATTR_EXPIRES)); view.read(XATTR_EXPIRES, buf); buf.flip(); expires = new Date(buf.asLongBuffer().get()); } for (String attribute : attributes) { if (!attribute.startsWith(XATTR_USER_METADATA_PREFIX)) { continue; } String value = readStringAttributeIfPresent(view, attributes, attribute); userMetadata.put(attribute.substring(XATTR_USER_METADATA_PREFIX.length()), value); } builder .payload(byteSource) .cacheControl(cacheControl) .contentDisposition(contentDisposition) .contentEncoding(contentEncoding) .contentLanguage(contentLanguage) .contentLength(byteSource.size()) .contentMD5(hashCode) .contentType(contentType) .expires(expires) .userMetadata(userMetadata.build()); } else { builder .payload(byteSource) .contentLength(byteSource.size()) .contentMD5(byteSource.hash(Hashing.md5()).asBytes()); } } catch (IOException e) { throw Throwables.propagate(e); } Blob blob = builder.build(); blob.getMetadata().setContainer(container); blob.getMetadata().setLastModified(new Date(file.lastModified())); blob.getMetadata().setSize(file.length()); if (blob.getPayload().getContentMetadata().getContentMD5() != null) blob.getMetadata() .setETag( base16().lowerCase().encode(blob.getPayload().getContentMetadata().getContentMD5())); return blob; }
private void readImageFromFile(HashMap<String, String> headerMap, DataInputStream dis) { // Image Recovery try { boolean littleEndian = "LowByteFirst".equals(headerMap.get("ByteOrder")); int dimX = Integer.valueOf(headerMap.get("Dim_1")); int dimY = Integer.valueOf(headerMap.get("Dim_2")); String dataType = headerMap.get("DataType"); int y = 0, x = 0; Object imageValue = null; boolean unsigned = dataType.startsWith("Unsigned"); if ("SignedByte".equals(dataType)) { imageValue = new byte[dimY][dimX]; byte[][] arrayImageValue = (byte[][]) imageValue; while (y < dimY) { int read = dis.read(); if (read == -1) { imageValue = null; break; } else { arrayImageValue[y][x] = (byte) read; x++; if (x >= dimX) { x = 0; y++; } } } } else if ("UnsignedByte".equals(dataType)) { imageValue = new short[dimY][dimX]; short[][] arrayImageValue = (short[][]) imageValue; while (y < dimY) { int read = dis.read(); if (read == -1) { imageValue = null; break; } else { arrayImageValue[y][x] = (short) read; x++; if (x >= dimX) { x = 0; y++; } } } } else if ("UnsignedShort".equals(dataType) || "SignedInteger".equals(dataType)) { // Specific imageValue = new int[dimY][dimX]; // short = 2 bytes // int = 4 bytes int factor = (unsigned) ? 2 : 4; int sizeToRead = factor * dimY * dimX; // Global Object flatImageValue = null; ByteBuffer byteBuffer = EdfFileReader.readAsBytes(sizeToRead, littleEndian, dis); // Specific if ("UnsignedShort".equals(dataType)) { flatImageValue = new short[dimY * dimX]; ShortBuffer shortBuffer = byteBuffer.asShortBuffer(); shortBuffer.get((short[]) flatImageValue); } else { flatImageValue = new int[dimY * dimX]; IntBuffer integerBuffer = byteBuffer.asIntBuffer(); integerBuffer.get((int[]) flatImageValue); } int globalIndex = 0; for (int yIndex = 0; yIndex < dimY; yIndex++) { for (int xIndex = 0; xIndex < dimX; xIndex++) { int value = Array.getInt(flatImageValue, globalIndex); value = (unsigned) ? value & 0xffff : value; ((int[][]) imageValue)[yIndex][xIndex] = value; globalIndex++; } } } else if ("SignedLong".equals(dataType) || "UnsignedInteger".equals(dataType)) { // Specific imageValue = new long[dimY][dimX]; // long = 8 bytes // int = 4 bytes int factor = (unsigned) ? 4 : 8; int sizeToRead = factor * dimY * dimX; // Global Object flatImageValue = null; ByteBuffer byteBuffer = EdfFileReader.readAsBytes(sizeToRead, littleEndian, dis); // Specific if ("UnsignedInteger".equals(dataType)) { flatImageValue = new int[dimY * dimX]; IntBuffer integerBuffer = byteBuffer.asIntBuffer(); integerBuffer.get((int[]) flatImageValue); } else { flatImageValue = new long[dimY * dimX]; LongBuffer longBuffer = byteBuffer.asLongBuffer(); longBuffer.get((long[]) flatImageValue); } int globalIndex = 0; for (int yIndex = 0; yIndex < dimY; yIndex++) { for (int xIndex = 0; xIndex < dimX; xIndex++) { long value = Array.getLong(flatImageValue, globalIndex); value = (unsigned) ? value & 0xFFFFFFFFL : value; ((long[][]) imageValue)[yIndex][xIndex] = value; globalIndex++; } } } else if ("Signed64".equals(dataType)) { throw new NotImplementedException(); } else if ("Unsigned64".equals(dataType)) { unsigned = true; throw new NotImplementedException(); } else if ("FloatValue".equals(dataType)) { throw new NotImplementedException(); } else if ("DoubleValue".equals(dataType)) { throw new NotImplementedException(); } if (imageValue != null) { EdfDataItem imageDataItem = new EdfDataItem("Image", new DefaultArrayMatrix(EdfFactory.NAME, imageValue), unsigned); addDataItem(imageDataItem); } } catch (Exception e) { e.printStackTrace(); // ignore exceptions for Image Recovery } }
/** * Creates a buffer for the given number of elements and native byte ordering * * @param elements The number of elements in the buffer * @return The buffer */ public static LongBuffer createLongBuffer(int elements) { ByteBuffer byteBuffer = ByteBuffer.allocateDirect(elements * 8); byteBuffer.order(ByteOrder.nativeOrder()); return byteBuffer.asLongBuffer(); }
public static LongBuffer newLongBuffer(int numLongs) { ByteBuffer buffer = ByteBuffer.allocateDirect(numLongs * 8); buffer.order(ByteOrder.nativeOrder()); return buffer.asLongBuffer(); }
Int64BufferSE(ByteBuffer bb) { super(bb); this.pb = bb.asLongBuffer(); }
public SpillRecord(int numPartitions) { buf = ByteBuffer.allocate(numPartitions * MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH); entries = buf.asLongBuffer(); }