/**
* Create a roaring array based on a previously serialized ByteBuffer. As much as possible, the
* ByteBuffer is used as the backend, however if you modify the content, the result is
* unspecified.
*
* @param bb The source ByteBuffer
*/
protected RoaringArray(ByteBuffer bb) {
bb.order(ByteOrder.LITTLE_ENDIAN);
if (bb.getInt() != SERIAL_COOKIE)
throw new RuntimeException("I failed to find the right cookie.");
this.size = bb.getInt();
// we fully read the meta-data array to RAM, but the containers
// themselves are memory-mapped
this.array = new Element[this.size];
final short keys[] = new short[this.size];
final int cardinalities[] = new int[this.size];
final boolean isBitmap[] = new boolean[this.size];
for (int k = 0; k < this.size; ++k) {
keys[k] = bb.getShort();
cardinalities[k] = Util.toIntUnsigned(bb.getShort()) + 1;
isBitmap[k] = cardinalities[k] > ArrayContainer.DEFAULT_MAX_SIZE;
}
for (int k = 0; k < this.size; ++k) {
if (cardinalities[k] == 0) throw new RuntimeException("no");
Container val;
if (isBitmap[k]) {
final LongBuffer bitmapArray = bb.asLongBuffer().slice();
bitmapArray.limit(BitmapContainer.MAX_CAPACITY / 64);
bb.position(bb.position() + BitmapContainer.MAX_CAPACITY / 8);
val = new BitmapContainer(bitmapArray, cardinalities[k]);
} else {
final ShortBuffer shortArray = bb.asShortBuffer().slice();
shortArray.limit(cardinalities[k]);
bb.position(bb.position() + cardinalities[k] * 2);
val = new ArrayContainer(shortArray, cardinalities[k]);
}
this.array[k] = new Element(keys[k], val);
}
}
/**
* Receive as many items as possible from the given byte buffer to this buffer.
*
* <p>The <TT>receiveItems()</TT> method must not block the calling thread; if it does, all
* message I/O in MP will be blocked.
*
* @param i Index of first item to receive, in the range 0 .. <TT>length</TT>-1.
* @param num Maximum number of items to receive.
* @param buffer Byte buffer.
* @return Number of items received.
*/
protected int receiveItems(int i, int num, ByteBuffer buffer) {
LongBuffer longbuffer = buffer.asLongBuffer();
int n = 0;
int r = i / myColCount;
int row = r * myRowStride + myLowerRow;
int c = i % myColCount;
int col = c * myColStride + myLowerCol;
int ncols = Math.min(myColCount - c, longbuffer.remaining());
while (r < myRowCount && ncols > 0) {
long[] myMatrix_row = myMatrix[row];
while (c < ncols) {
myMatrix_row[col] = myOp.op(myMatrix_row[col], longbuffer.get());
++c;
col += myColStride;
}
n += ncols;
++r;
row += myRowStride;
c = 0;
col = myLowerCol;
ncols = Math.min(myColCount, longbuffer.remaining());
}
buffer.position(buffer.position() + 8 * n);
return n;
}
@Override
public void onNewData(ByteBuffer decompressed) {
long timestamp = decompressed.getLong();
LongBuffer data = decompressed.asLongBuffer();
for (int i = 0; i < variables.size(); i++) {
YoVariable<?> variable = variables.get(i);
long previousValue = variable.getValueAsLongBits();
long newValue = data.get();
variable.setValueFromLongBits(newValue, false);
if (previousValue != newValue) {
ArrayList<VariableChangedListener> changedListeners =
variable.getVariableChangedListeners();
if (changedListeners != null) {
for (int listener = 0; listener < changedListeners.size(); listener++) {
VariableChangedListener changedListener = changedListeners.get(listener);
if (!(changedListener instanceof LogControlVariableChangeListener)) {
changedListener.variableChanged(variable);
}
}
}
}
}
for (int i = 0; i < jointStates.size(); i++) {
jointStates.get(i).update(data);
}
listener.receivedTimestampAndData(timestamp, decompressed);
}
/**
* Receive as many items as possible from the given byte buffer to this buffer.
*
* <p>The <TT>receiveItems()</TT> method must not block the calling thread; if it does, all
* message I/O in MP will be blocked.
*
* @param i Index of first item to receive, in the range 0 .. <TT>length</TT>-1.
* @param num Maximum number of items to receive.
* @param buffer Byte buffer.
* @return Number of items received.
*/
protected int receiveItems(int i, int num, ByteBuffer buffer) {
LongBuffer longbuffer = buffer.asLongBuffer();
int n = Math.min(num, Math.min(myLength - i, longbuffer.remaining()));
longbuffer.get(myArray, myArrayOffset + i, n);
buffer.position(buffer.position() + 8 * n);
return n;
}
/** Creates a percentiler from its serialized representation. */
public Percentiler(byte[] data) {
ByteBuffer in = ByteBuffer.wrap(data);
// read our int data
IntBuffer iin = in.asIntBuffer();
_max = iin.get();
iin.get(_counts);
in.position(iin.position() * INT_SIZE);
// read our long data
LongBuffer lin = in.asLongBuffer();
_snapTotal = (_total = lin.get());
in.position(iin.position() * INT_SIZE + lin.position() * 2 * INT_SIZE);
// read our min value (which was added afterwards and must do some jockeying to maintain
// backwards compatibility)
if (in.position() == in.limit()) {
_min = 0; // legacy
} else {
_min = in.asIntBuffer().get();
}
// Un-break percentilers that have been stored with bogus data
if (_max < _min) {
log.warning("Percentiler initialized with bogus range. Coping.", "min", _min, "max", _max);
_max = _min + 1;
}
// compute our percentiles
recomputePercentiles();
}
public SpillRecord(Path indexFileName, JobConf job, Checksum crc, String expectedIndexOwner)
throws IOException {
final FileSystem rfs = FileSystem.getLocal(job).getRaw();
final DataInputStream in =
new DataInputStream(
SecureIOUtils.openForRead(
new File(indexFileName.toUri().getPath()), expectedIndexOwner, null));
try {
final long length = rfs.getFileStatus(indexFileName).getLen();
final int partitions = (int) length / MAP_OUTPUT_INDEX_RECORD_LENGTH;
final int size = partitions * MAP_OUTPUT_INDEX_RECORD_LENGTH;
buf = ByteBuffer.allocate(size);
if (crc != null) {
crc.reset();
CheckedInputStream chk = new CheckedInputStream(in, crc);
IOUtils.readFully(chk, buf.array(), 0, size);
if (chk.getChecksum().getValue() != in.readLong()) {
throw new ChecksumException("Checksum error reading spill index: " + indexFileName, -1);
}
} else {
IOUtils.readFully(in, buf.array(), 0, size);
}
entries = buf.asLongBuffer();
} finally {
in.close();
}
}
public static void main(String[] args) {
ByteBuffer bb = ByteBuffer.allocate(BSIZE);
// Allocation automatically zeroes the ByteBuffer:
int i = 0;
while (i++ < bb.limit()) if (bb.get() != 0) print("nonzero");
print("i = " + i);
bb.rewind();
// Store and read a char array:
bb.asCharBuffer().put("Howdy!");
char c;
while ((c = bb.getChar()) != 0) printnb(c + " ");
print();
bb.rewind();
// Store and read a short:
bb.asShortBuffer().put((short) 471142);
print(bb.getShort());
bb.rewind();
// Store and read an int:
bb.asIntBuffer().put(99471142);
print(bb.getInt());
bb.rewind();
// Store and read a long:
bb.asLongBuffer().put(99471142);
print(bb.getLong());
bb.rewind();
// Store and read a float:
bb.asFloatBuffer().put(99471142);
print(bb.getFloat());
bb.rewind();
// Store and read a double:
bb.asDoubleBuffer().put(99471142);
print(bb.getDouble());
bb.rewind();
}
public static void main(String... args) {
ByteBuffer bb = ByteBuffer.allocate(8);
System.out.println("Native order = " + ByteOrder.nativeOrder());
System.out.println("Default order = " + bb.order());
// bb.order(ByteOrder.BIG_ENDIAN) ;
// bb.order(ByteOrder.LITTLE_ENDIAN) ;
System.out.println("Order = " + bb.order());
bb.asLongBuffer().put(0x0102030405060708L);
for (int i = 0; i < bb.capacity(); i++) System.out.printf("0x%02X ", bb.get(i));
// Comes out hight to low : 0x01 0x02 0x03 0x04 0x05 0x06 0x07 0x08
}
public MappedDatabase(
final ByteBuffer db, final DataOutput dupStream, final PrintWriter solverLog) {
final LongBuffer ldb = db.asLongBuffer();
if ((ldb.limit() & 1) == 0) {
this.db = ldb;
this.solvers = new IDMap<>(1 << 12);
this.dupStream = dupStream;
this.solverLog = solverLog;
return;
}
throw new IllegalArgumentException("Truncated Buffer");
}
public static void main(String[] args) throws Exception {
DatagramChannel dc = DatagramChannel.open();
DatagramSocket socket = dc.socket();
socket.bind(new InetSocketAddress(22500));
ByteBuffer byteBuf = ByteBuffer.allocate(8);
// byteBuf.order(ByteOrder.LITTLE_ENDIAN);
LongBuffer longBuf = byteBuf.asLongBuffer();
while (true) {
dc.receive(byteBuf);
System.out.println(longBuf.get(0));
byteBuf.clear();
}
}
public NioWrapLittleConversion() {
// big/little endian difference one liner
order = ByteOrder.LITTLE_ENDIAN;
byteBuffer = ByteBuffer.allocateDirect(BUFFER_SIZE).order(order);
// views on the bytebuffer to fill/drain it efficiently
charBuffer = byteBuffer.asCharBuffer();
shortBuffer = byteBuffer.asShortBuffer();
intBuffer = byteBuffer.asIntBuffer();
longBuffer = byteBuffer.asLongBuffer();
floatBuffer = byteBuffer.asFloatBuffer();
doubleBuffer = byteBuffer.asDoubleBuffer();
}
@NotNull
public static IndexedCollection<Long> ofLong(@NotNull ByteBuffer byteBuffer) {
int magic = byteBuffer.getInt();
if (magic != MAGIC) {
throw new IllegalArgumentException("bad magic number");
}
int version = byteBuffer.getInt();
if (version != VERSION) {
throw new IllegalArgumentException("bad version number");
}
int size = byteBuffer.getInt();
LongBuffer values = byteBuffer.asLongBuffer();
values.limit(size);
byteBuffer.position(byteBuffer.position() + size * Integer.BYTES);
return new LongBufferCollection(values.slice());
}
/** Converts this percentiler to a byte array so that it may be stored into a database. */
public byte[] toBytes() {
byte[] data = new byte[(BUCKET_COUNT + 4) * INT_SIZE];
ByteBuffer out = ByteBuffer.wrap(data);
// write our int data
IntBuffer iout = out.asIntBuffer();
iout.put(_max);
iout.put(_counts);
out.position(iout.position() * INT_SIZE);
// write our long data
LongBuffer lout = out.asLongBuffer();
lout.put(_total);
out.position(iout.position() * INT_SIZE + lout.position() * 2 * INT_SIZE);
// write our min value (added later so we can't write it above like we wish we could)
out.asIntBuffer().put(_min);
return data;
}
/** Allocates a new packed image frame in native memory where rows are 8-byte aligned. */
public Frame(int width, int height, int depth, int channels) {
int pixelSize = Math.abs(depth) / 8;
this.imageWidth = width;
this.imageHeight = height;
this.imageDepth = depth;
this.imageChannels = channels;
this.imageStride =
((imageWidth * imageChannels * pixelSize + 7) & ~7) / pixelSize; // 8-byte aligned
this.image = new Buffer[1];
ByteBuffer buffer =
ByteBuffer.allocateDirect(imageHeight * imageStride * pixelSize)
.order(ByteOrder.nativeOrder());
switch (imageDepth) {
case DEPTH_BYTE:
case DEPTH_UBYTE:
image[0] = buffer;
break;
case DEPTH_SHORT:
case DEPTH_USHORT:
image[0] = buffer.asShortBuffer();
break;
case DEPTH_INT:
image[0] = buffer.asIntBuffer();
break;
case DEPTH_LONG:
image[0] = buffer.asLongBuffer();
break;
case DEPTH_FLOAT:
image[0] = buffer.asFloatBuffer();
break;
case DEPTH_DOUBLE:
image[0] = buffer.asDoubleBuffer();
break;
default:
throw new UnsupportedOperationException("Unsupported depth value: " + imageDepth);
}
}
@Override
public Blob getBlob(final String container, final String key) {
BlobBuilder builder = blobBuilders.get();
builder.name(key);
File file = getFileForBlobKey(container, key);
ByteSource byteSource;
if (getDirectoryBlobSuffix(key) != null) {
logger.debug("%s - %s is a directory", container, key);
byteSource = ByteSource.empty();
} else {
byteSource = Files.asByteSource(file);
}
try {
String cacheControl = null;
String contentDisposition = null;
String contentEncoding = null;
String contentLanguage = null;
String contentType = null;
HashCode hashCode = null;
Date expires = null;
ImmutableMap.Builder<String, String> userMetadata = ImmutableMap.builder();
UserDefinedFileAttributeView view = getUserDefinedFileAttributeView(file.toPath());
if (view != null) {
Set<String> attributes = ImmutableSet.copyOf(view.list());
cacheControl = readStringAttributeIfPresent(view, attributes, XATTR_CACHE_CONTROL);
contentDisposition =
readStringAttributeIfPresent(view, attributes, XATTR_CONTENT_DISPOSITION);
contentEncoding = readStringAttributeIfPresent(view, attributes, XATTR_CONTENT_ENCODING);
contentLanguage = readStringAttributeIfPresent(view, attributes, XATTR_CONTENT_LANGUAGE);
contentType = readStringAttributeIfPresent(view, attributes, XATTR_CONTENT_TYPE);
if (contentType == null && autoDetectContentType) {
contentType = probeContentType(file.toPath());
}
if (attributes.contains(XATTR_CONTENT_MD5)) {
ByteBuffer buf = ByteBuffer.allocate(view.size(XATTR_CONTENT_MD5));
view.read(XATTR_CONTENT_MD5, buf);
hashCode = HashCode.fromBytes(buf.array());
}
if (attributes.contains(XATTR_EXPIRES)) {
ByteBuffer buf = ByteBuffer.allocate(view.size(XATTR_EXPIRES));
view.read(XATTR_EXPIRES, buf);
buf.flip();
expires = new Date(buf.asLongBuffer().get());
}
for (String attribute : attributes) {
if (!attribute.startsWith(XATTR_USER_METADATA_PREFIX)) {
continue;
}
String value = readStringAttributeIfPresent(view, attributes, attribute);
userMetadata.put(attribute.substring(XATTR_USER_METADATA_PREFIX.length()), value);
}
builder
.payload(byteSource)
.cacheControl(cacheControl)
.contentDisposition(contentDisposition)
.contentEncoding(contentEncoding)
.contentLanguage(contentLanguage)
.contentLength(byteSource.size())
.contentMD5(hashCode)
.contentType(contentType)
.expires(expires)
.userMetadata(userMetadata.build());
} else {
builder
.payload(byteSource)
.contentLength(byteSource.size())
.contentMD5(byteSource.hash(Hashing.md5()).asBytes());
}
} catch (IOException e) {
throw Throwables.propagate(e);
}
Blob blob = builder.build();
blob.getMetadata().setContainer(container);
blob.getMetadata().setLastModified(new Date(file.lastModified()));
blob.getMetadata().setSize(file.length());
if (blob.getPayload().getContentMetadata().getContentMD5() != null)
blob.getMetadata()
.setETag(
base16().lowerCase().encode(blob.getPayload().getContentMetadata().getContentMD5()));
return blob;
}
private void readImageFromFile(HashMap<String, String> headerMap, DataInputStream dis) {
// Image Recovery
try {
boolean littleEndian = "LowByteFirst".equals(headerMap.get("ByteOrder"));
int dimX = Integer.valueOf(headerMap.get("Dim_1"));
int dimY = Integer.valueOf(headerMap.get("Dim_2"));
String dataType = headerMap.get("DataType");
int y = 0, x = 0;
Object imageValue = null;
boolean unsigned = dataType.startsWith("Unsigned");
if ("SignedByte".equals(dataType)) {
imageValue = new byte[dimY][dimX];
byte[][] arrayImageValue = (byte[][]) imageValue;
while (y < dimY) {
int read = dis.read();
if (read == -1) {
imageValue = null;
break;
} else {
arrayImageValue[y][x] = (byte) read;
x++;
if (x >= dimX) {
x = 0;
y++;
}
}
}
} else if ("UnsignedByte".equals(dataType)) {
imageValue = new short[dimY][dimX];
short[][] arrayImageValue = (short[][]) imageValue;
while (y < dimY) {
int read = dis.read();
if (read == -1) {
imageValue = null;
break;
} else {
arrayImageValue[y][x] = (short) read;
x++;
if (x >= dimX) {
x = 0;
y++;
}
}
}
} else if ("UnsignedShort".equals(dataType) || "SignedInteger".equals(dataType)) {
// Specific
imageValue = new int[dimY][dimX];
// short = 2 bytes
// int = 4 bytes
int factor = (unsigned) ? 2 : 4;
int sizeToRead = factor * dimY * dimX;
// Global
Object flatImageValue = null;
ByteBuffer byteBuffer = EdfFileReader.readAsBytes(sizeToRead, littleEndian, dis);
// Specific
if ("UnsignedShort".equals(dataType)) {
flatImageValue = new short[dimY * dimX];
ShortBuffer shortBuffer = byteBuffer.asShortBuffer();
shortBuffer.get((short[]) flatImageValue);
} else {
flatImageValue = new int[dimY * dimX];
IntBuffer integerBuffer = byteBuffer.asIntBuffer();
integerBuffer.get((int[]) flatImageValue);
}
int globalIndex = 0;
for (int yIndex = 0; yIndex < dimY; yIndex++) {
for (int xIndex = 0; xIndex < dimX; xIndex++) {
int value = Array.getInt(flatImageValue, globalIndex);
value = (unsigned) ? value & 0xffff : value;
((int[][]) imageValue)[yIndex][xIndex] = value;
globalIndex++;
}
}
} else if ("SignedLong".equals(dataType) || "UnsignedInteger".equals(dataType)) {
// Specific
imageValue = new long[dimY][dimX];
// long = 8 bytes
// int = 4 bytes
int factor = (unsigned) ? 4 : 8;
int sizeToRead = factor * dimY * dimX;
// Global
Object flatImageValue = null;
ByteBuffer byteBuffer = EdfFileReader.readAsBytes(sizeToRead, littleEndian, dis);
// Specific
if ("UnsignedInteger".equals(dataType)) {
flatImageValue = new int[dimY * dimX];
IntBuffer integerBuffer = byteBuffer.asIntBuffer();
integerBuffer.get((int[]) flatImageValue);
} else {
flatImageValue = new long[dimY * dimX];
LongBuffer longBuffer = byteBuffer.asLongBuffer();
longBuffer.get((long[]) flatImageValue);
}
int globalIndex = 0;
for (int yIndex = 0; yIndex < dimY; yIndex++) {
for (int xIndex = 0; xIndex < dimX; xIndex++) {
long value = Array.getLong(flatImageValue, globalIndex);
value = (unsigned) ? value & 0xFFFFFFFFL : value;
((long[][]) imageValue)[yIndex][xIndex] = value;
globalIndex++;
}
}
} else if ("Signed64".equals(dataType)) {
throw new NotImplementedException();
} else if ("Unsigned64".equals(dataType)) {
unsigned = true;
throw new NotImplementedException();
} else if ("FloatValue".equals(dataType)) {
throw new NotImplementedException();
} else if ("DoubleValue".equals(dataType)) {
throw new NotImplementedException();
}
if (imageValue != null) {
EdfDataItem imageDataItem =
new EdfDataItem("Image", new DefaultArrayMatrix(EdfFactory.NAME, imageValue), unsigned);
addDataItem(imageDataItem);
}
} catch (Exception e) {
e.printStackTrace();
// ignore exceptions for Image Recovery
}
}
/**
* Creates a buffer for the given number of elements and native byte ordering
*
* @param elements The number of elements in the buffer
* @return The buffer
*/
public static LongBuffer createLongBuffer(int elements) {
ByteBuffer byteBuffer = ByteBuffer.allocateDirect(elements * 8);
byteBuffer.order(ByteOrder.nativeOrder());
return byteBuffer.asLongBuffer();
}
public static LongBuffer newLongBuffer(int numLongs) {
ByteBuffer buffer = ByteBuffer.allocateDirect(numLongs * 8);
buffer.order(ByteOrder.nativeOrder());
return buffer.asLongBuffer();
}
Int64BufferSE(ByteBuffer bb) {
super(bb);
this.pb = bb.asLongBuffer();
}
public SpillRecord(int numPartitions) {
buf = ByteBuffer.allocate(numPartitions * MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH);
entries = buf.asLongBuffer();
}
