/**
  * Create a roaring array based on a previously serialized ByteBuffer. As much as possible, the
  * ByteBuffer is used as the backend, however if you modify the content, the result is
  * unspecified.
  *
  * @param bb The source ByteBuffer
  */
 protected RoaringArray(ByteBuffer bb) {
   bb.order(ByteOrder.LITTLE_ENDIAN);
   if (bb.getInt() != SERIAL_COOKIE)
     throw new RuntimeException("I failed to find the right cookie.");
   this.size = bb.getInt();
   // we fully read the meta-data array to RAM, but the containers
   // themselves are memory-mapped
   this.array = new Element[this.size];
   final short keys[] = new short[this.size];
   final int cardinalities[] = new int[this.size];
   final boolean isBitmap[] = new boolean[this.size];
   for (int k = 0; k < this.size; ++k) {
     keys[k] = bb.getShort();
     cardinalities[k] = Util.toIntUnsigned(bb.getShort()) + 1;
     isBitmap[k] = cardinalities[k] > ArrayContainer.DEFAULT_MAX_SIZE;
   }
   for (int k = 0; k < this.size; ++k) {
     if (cardinalities[k] == 0) throw new RuntimeException("no");
     Container val;
     if (isBitmap[k]) {
       final LongBuffer bitmapArray = bb.asLongBuffer().slice();
       bitmapArray.limit(BitmapContainer.MAX_CAPACITY / 64);
       bb.position(bb.position() + BitmapContainer.MAX_CAPACITY / 8);
       val = new BitmapContainer(bitmapArray, cardinalities[k]);
     } else {
       final ShortBuffer shortArray = bb.asShortBuffer().slice();
       shortArray.limit(cardinalities[k]);
       bb.position(bb.position() + cardinalities[k] * 2);
       val = new ArrayContainer(shortArray, cardinalities[k]);
     }
     this.array[k] = new Element(keys[k], val);
   }
 }
 /**
  * Receive as many items as possible from the given byte buffer to this buffer.
  *
  * <p>The <TT>receiveItems()</TT> method must not block the calling thread; if it does, all
  * message I/O in MP will be blocked.
  *
  * @param i Index of first item to receive, in the range 0 .. <TT>length</TT>-1.
  * @param num Maximum number of items to receive.
  * @param buffer Byte buffer.
  * @return Number of items received.
  */
 protected int receiveItems(int i, int num, ByteBuffer buffer) {
   LongBuffer longbuffer = buffer.asLongBuffer();
   int n = 0;
   int r = i / myColCount;
   int row = r * myRowStride + myLowerRow;
   int c = i % myColCount;
   int col = c * myColStride + myLowerCol;
   int ncols = Math.min(myColCount - c, longbuffer.remaining());
   while (r < myRowCount && ncols > 0) {
     long[] myMatrix_row = myMatrix[row];
     while (c < ncols) {
       myMatrix_row[col] = myOp.op(myMatrix_row[col], longbuffer.get());
       ++c;
       col += myColStride;
     }
     n += ncols;
     ++r;
     row += myRowStride;
     c = 0;
     col = myLowerCol;
     ncols = Math.min(myColCount, longbuffer.remaining());
   }
   buffer.position(buffer.position() + 8 * n);
   return n;
 }
  @Override
  public void onNewData(ByteBuffer decompressed) {
    long timestamp = decompressed.getLong();
    LongBuffer data = decompressed.asLongBuffer();

    for (int i = 0; i < variables.size(); i++) {
      YoVariable<?> variable = variables.get(i);
      long previousValue = variable.getValueAsLongBits();
      long newValue = data.get();
      variable.setValueFromLongBits(newValue, false);
      if (previousValue != newValue) {
        ArrayList<VariableChangedListener> changedListeners =
            variable.getVariableChangedListeners();
        if (changedListeners != null) {
          for (int listener = 0; listener < changedListeners.size(); listener++) {
            VariableChangedListener changedListener = changedListeners.get(listener);
            if (!(changedListener instanceof LogControlVariableChangeListener)) {
              changedListener.variableChanged(variable);
            }
          }
        }
      }
    }

    for (int i = 0; i < jointStates.size(); i++) {
      jointStates.get(i).update(data);
    }

    listener.receivedTimestampAndData(timestamp, decompressed);
  }
示例#4
0
 /**
  * Receive as many items as possible from the given byte buffer to this buffer.
  *
  * <p>The <TT>receiveItems()</TT> method must not block the calling thread; if it does, all
  * message I/O in MP will be blocked.
  *
  * @param i Index of first item to receive, in the range 0 .. <TT>length</TT>-1.
  * @param num Maximum number of items to receive.
  * @param buffer Byte buffer.
  * @return Number of items received.
  */
 protected int receiveItems(int i, int num, ByteBuffer buffer) {
   LongBuffer longbuffer = buffer.asLongBuffer();
   int n = Math.min(num, Math.min(myLength - i, longbuffer.remaining()));
   longbuffer.get(myArray, myArrayOffset + i, n);
   buffer.position(buffer.position() + 8 * n);
   return n;
 }
示例#5
0
  /** Creates a percentiler from its serialized representation. */
  public Percentiler(byte[] data) {
    ByteBuffer in = ByteBuffer.wrap(data);

    // read our int data
    IntBuffer iin = in.asIntBuffer();
    _max = iin.get();
    iin.get(_counts);
    in.position(iin.position() * INT_SIZE);

    // read our long data
    LongBuffer lin = in.asLongBuffer();
    _snapTotal = (_total = lin.get());
    in.position(iin.position() * INT_SIZE + lin.position() * 2 * INT_SIZE);

    // read our min value (which was added afterwards and must do some jockeying to maintain
    // backwards compatibility)
    if (in.position() == in.limit()) {
      _min = 0; // legacy
    } else {
      _min = in.asIntBuffer().get();
    }

    // Un-break percentilers that have been stored with bogus data
    if (_max < _min) {
      log.warning("Percentiler initialized with bogus range. Coping.", "min", _min, "max", _max);
      _max = _min + 1;
    }

    // compute our percentiles
    recomputePercentiles();
  }
示例#6
0
  public SpillRecord(Path indexFileName, JobConf job, Checksum crc, String expectedIndexOwner)
      throws IOException {

    final FileSystem rfs = FileSystem.getLocal(job).getRaw();
    final DataInputStream in =
        new DataInputStream(
            SecureIOUtils.openForRead(
                new File(indexFileName.toUri().getPath()), expectedIndexOwner, null));
    try {
      final long length = rfs.getFileStatus(indexFileName).getLen();
      final int partitions = (int) length / MAP_OUTPUT_INDEX_RECORD_LENGTH;
      final int size = partitions * MAP_OUTPUT_INDEX_RECORD_LENGTH;

      buf = ByteBuffer.allocate(size);
      if (crc != null) {
        crc.reset();
        CheckedInputStream chk = new CheckedInputStream(in, crc);
        IOUtils.readFully(chk, buf.array(), 0, size);
        if (chk.getChecksum().getValue() != in.readLong()) {
          throw new ChecksumException("Checksum error reading spill index: " + indexFileName, -1);
        }
      } else {
        IOUtils.readFully(in, buf.array(), 0, size);
      }
      entries = buf.asLongBuffer();
    } finally {
      in.close();
    }
  }
 public static void main(String[] args) {
   ByteBuffer bb = ByteBuffer.allocate(BSIZE);
   // Allocation automatically zeroes the ByteBuffer:
   int i = 0;
   while (i++ < bb.limit()) if (bb.get() != 0) print("nonzero");
   print("i = " + i);
   bb.rewind();
   // Store and read a char array:
   bb.asCharBuffer().put("Howdy!");
   char c;
   while ((c = bb.getChar()) != 0) printnb(c + " ");
   print();
   bb.rewind();
   // Store and read a short:
   bb.asShortBuffer().put((short) 471142);
   print(bb.getShort());
   bb.rewind();
   // Store and read an int:
   bb.asIntBuffer().put(99471142);
   print(bb.getInt());
   bb.rewind();
   // Store and read a long:
   bb.asLongBuffer().put(99471142);
   print(bb.getLong());
   bb.rewind();
   // Store and read a float:
   bb.asFloatBuffer().put(99471142);
   print(bb.getFloat());
   bb.rewind();
   // Store and read a double:
   bb.asDoubleBuffer().put(99471142);
   print(bb.getDouble());
   bb.rewind();
 }
示例#8
0
 public static void main(String... args) {
   ByteBuffer bb = ByteBuffer.allocate(8);
   System.out.println("Native order = " + ByteOrder.nativeOrder());
   System.out.println("Default order = " + bb.order());
   // bb.order(ByteOrder.BIG_ENDIAN) ;
   // bb.order(ByteOrder.LITTLE_ENDIAN) ;
   System.out.println("Order = " + bb.order());
   bb.asLongBuffer().put(0x0102030405060708L);
   for (int i = 0; i < bb.capacity(); i++) System.out.printf("0x%02X ", bb.get(i));
   // Comes out hight to low : 0x01 0x02 0x03 0x04 0x05 0x06 0x07 0x08
 }
示例#9
0
 public MappedDatabase(
     final ByteBuffer db, final DataOutput dupStream, final PrintWriter solverLog) {
   final LongBuffer ldb = db.asLongBuffer();
   if ((ldb.limit() & 1) == 0) {
     this.db = ldb;
     this.solvers = new IDMap<>(1 << 12);
     this.dupStream = dupStream;
     this.solverLog = solverLog;
     return;
   }
   throw new IllegalArgumentException("Truncated Buffer");
 }
  public static void main(String[] args) throws Exception {
    DatagramChannel dc = DatagramChannel.open();
    DatagramSocket socket = dc.socket();
    socket.bind(new InetSocketAddress(22500));
    ByteBuffer byteBuf = ByteBuffer.allocate(8);
    //        byteBuf.order(ByteOrder.LITTLE_ENDIAN);
    LongBuffer longBuf = byteBuf.asLongBuffer();

    while (true) {
      dc.receive(byteBuf);
      System.out.println(longBuf.get(0));
      byteBuf.clear();
    }
  }
  public NioWrapLittleConversion() {

    // big/little endian difference one liner
    order = ByteOrder.LITTLE_ENDIAN;

    byteBuffer = ByteBuffer.allocateDirect(BUFFER_SIZE).order(order);

    // views on the bytebuffer to fill/drain it efficiently
    charBuffer = byteBuffer.asCharBuffer();
    shortBuffer = byteBuffer.asShortBuffer();
    intBuffer = byteBuffer.asIntBuffer();
    longBuffer = byteBuffer.asLongBuffer();
    floatBuffer = byteBuffer.asFloatBuffer();
    doubleBuffer = byteBuffer.asDoubleBuffer();
  }
  @NotNull
  public static IndexedCollection<Long> ofLong(@NotNull ByteBuffer byteBuffer) {
    int magic = byteBuffer.getInt();
    if (magic != MAGIC) {
      throw new IllegalArgumentException("bad magic number");
    }
    int version = byteBuffer.getInt();
    if (version != VERSION) {
      throw new IllegalArgumentException("bad version number");
    }

    int size = byteBuffer.getInt();
    LongBuffer values = byteBuffer.asLongBuffer();
    values.limit(size);
    byteBuffer.position(byteBuffer.position() + size * Integer.BYTES);
    return new LongBufferCollection(values.slice());
  }
示例#13
0
  /** Converts this percentiler to a byte array so that it may be stored into a database. */
  public byte[] toBytes() {
    byte[] data = new byte[(BUCKET_COUNT + 4) * INT_SIZE];
    ByteBuffer out = ByteBuffer.wrap(data);

    // write our int data
    IntBuffer iout = out.asIntBuffer();
    iout.put(_max);
    iout.put(_counts);
    out.position(iout.position() * INT_SIZE);

    // write our long data
    LongBuffer lout = out.asLongBuffer();
    lout.put(_total);
    out.position(iout.position() * INT_SIZE + lout.position() * 2 * INT_SIZE);

    // write our min value (added later so we can't write it above like we wish we could)
    out.asIntBuffer().put(_min);

    return data;
  }
示例#14
0
  /** Allocates a new packed image frame in native memory where rows are 8-byte aligned. */
  public Frame(int width, int height, int depth, int channels) {
    int pixelSize = Math.abs(depth) / 8;
    this.imageWidth = width;
    this.imageHeight = height;
    this.imageDepth = depth;
    this.imageChannels = channels;
    this.imageStride =
        ((imageWidth * imageChannels * pixelSize + 7) & ~7) / pixelSize; // 8-byte aligned
    this.image = new Buffer[1];

    ByteBuffer buffer =
        ByteBuffer.allocateDirect(imageHeight * imageStride * pixelSize)
            .order(ByteOrder.nativeOrder());
    switch (imageDepth) {
      case DEPTH_BYTE:
      case DEPTH_UBYTE:
        image[0] = buffer;
        break;
      case DEPTH_SHORT:
      case DEPTH_USHORT:
        image[0] = buffer.asShortBuffer();
        break;
      case DEPTH_INT:
        image[0] = buffer.asIntBuffer();
        break;
      case DEPTH_LONG:
        image[0] = buffer.asLongBuffer();
        break;
      case DEPTH_FLOAT:
        image[0] = buffer.asFloatBuffer();
        break;
      case DEPTH_DOUBLE:
        image[0] = buffer.asDoubleBuffer();
        break;
      default:
        throw new UnsupportedOperationException("Unsupported depth value: " + imageDepth);
    }
  }
  @Override
  public Blob getBlob(final String container, final String key) {
    BlobBuilder builder = blobBuilders.get();
    builder.name(key);
    File file = getFileForBlobKey(container, key);
    ByteSource byteSource;

    if (getDirectoryBlobSuffix(key) != null) {
      logger.debug("%s - %s is a directory", container, key);
      byteSource = ByteSource.empty();
    } else {
      byteSource = Files.asByteSource(file);
    }
    try {
      String cacheControl = null;
      String contentDisposition = null;
      String contentEncoding = null;
      String contentLanguage = null;
      String contentType = null;
      HashCode hashCode = null;
      Date expires = null;
      ImmutableMap.Builder<String, String> userMetadata = ImmutableMap.builder();

      UserDefinedFileAttributeView view = getUserDefinedFileAttributeView(file.toPath());
      if (view != null) {
        Set<String> attributes = ImmutableSet.copyOf(view.list());

        cacheControl = readStringAttributeIfPresent(view, attributes, XATTR_CACHE_CONTROL);
        contentDisposition =
            readStringAttributeIfPresent(view, attributes, XATTR_CONTENT_DISPOSITION);
        contentEncoding = readStringAttributeIfPresent(view, attributes, XATTR_CONTENT_ENCODING);
        contentLanguage = readStringAttributeIfPresent(view, attributes, XATTR_CONTENT_LANGUAGE);
        contentType = readStringAttributeIfPresent(view, attributes, XATTR_CONTENT_TYPE);
        if (contentType == null && autoDetectContentType) {
          contentType = probeContentType(file.toPath());
        }
        if (attributes.contains(XATTR_CONTENT_MD5)) {
          ByteBuffer buf = ByteBuffer.allocate(view.size(XATTR_CONTENT_MD5));
          view.read(XATTR_CONTENT_MD5, buf);
          hashCode = HashCode.fromBytes(buf.array());
        }
        if (attributes.contains(XATTR_EXPIRES)) {
          ByteBuffer buf = ByteBuffer.allocate(view.size(XATTR_EXPIRES));
          view.read(XATTR_EXPIRES, buf);
          buf.flip();
          expires = new Date(buf.asLongBuffer().get());
        }
        for (String attribute : attributes) {
          if (!attribute.startsWith(XATTR_USER_METADATA_PREFIX)) {
            continue;
          }
          String value = readStringAttributeIfPresent(view, attributes, attribute);
          userMetadata.put(attribute.substring(XATTR_USER_METADATA_PREFIX.length()), value);
        }

        builder
            .payload(byteSource)
            .cacheControl(cacheControl)
            .contentDisposition(contentDisposition)
            .contentEncoding(contentEncoding)
            .contentLanguage(contentLanguage)
            .contentLength(byteSource.size())
            .contentMD5(hashCode)
            .contentType(contentType)
            .expires(expires)
            .userMetadata(userMetadata.build());
      } else {
        builder
            .payload(byteSource)
            .contentLength(byteSource.size())
            .contentMD5(byteSource.hash(Hashing.md5()).asBytes());
      }
    } catch (IOException e) {
      throw Throwables.propagate(e);
    }
    Blob blob = builder.build();
    blob.getMetadata().setContainer(container);
    blob.getMetadata().setLastModified(new Date(file.lastModified()));
    blob.getMetadata().setSize(file.length());
    if (blob.getPayload().getContentMetadata().getContentMD5() != null)
      blob.getMetadata()
          .setETag(
              base16().lowerCase().encode(blob.getPayload().getContentMetadata().getContentMD5()));
    return blob;
  }
示例#16
0
  private void readImageFromFile(HashMap<String, String> headerMap, DataInputStream dis) {
    // Image Recovery
    try {
      boolean littleEndian = "LowByteFirst".equals(headerMap.get("ByteOrder"));
      int dimX = Integer.valueOf(headerMap.get("Dim_1"));
      int dimY = Integer.valueOf(headerMap.get("Dim_2"));
      String dataType = headerMap.get("DataType");
      int y = 0, x = 0;
      Object imageValue = null;
      boolean unsigned = dataType.startsWith("Unsigned");
      if ("SignedByte".equals(dataType)) {
        imageValue = new byte[dimY][dimX];
        byte[][] arrayImageValue = (byte[][]) imageValue;
        while (y < dimY) {
          int read = dis.read();
          if (read == -1) {
            imageValue = null;
            break;
          } else {

            arrayImageValue[y][x] = (byte) read;
            x++;
            if (x >= dimX) {
              x = 0;
              y++;
            }
          }
        }
      } else if ("UnsignedByte".equals(dataType)) {

        imageValue = new short[dimY][dimX];
        short[][] arrayImageValue = (short[][]) imageValue;

        while (y < dimY) {
          int read = dis.read();
          if (read == -1) {
            imageValue = null;
            break;
          } else {
            arrayImageValue[y][x] = (short) read;
            x++;
            if (x >= dimX) {
              x = 0;
              y++;
            }
          }
        }
      } else if ("UnsignedShort".equals(dataType) || "SignedInteger".equals(dataType)) {

        // Specific
        imageValue = new int[dimY][dimX];

        // short = 2 bytes
        // int = 4 bytes
        int factor = (unsigned) ? 2 : 4;
        int sizeToRead = factor * dimY * dimX;

        // Global
        Object flatImageValue = null;
        ByteBuffer byteBuffer = EdfFileReader.readAsBytes(sizeToRead, littleEndian, dis);

        // Specific
        if ("UnsignedShort".equals(dataType)) {
          flatImageValue = new short[dimY * dimX];
          ShortBuffer shortBuffer = byteBuffer.asShortBuffer();
          shortBuffer.get((short[]) flatImageValue);
        } else {
          flatImageValue = new int[dimY * dimX];
          IntBuffer integerBuffer = byteBuffer.asIntBuffer();
          integerBuffer.get((int[]) flatImageValue);
        }

        int globalIndex = 0;
        for (int yIndex = 0; yIndex < dimY; yIndex++) {
          for (int xIndex = 0; xIndex < dimX; xIndex++) {
            int value = Array.getInt(flatImageValue, globalIndex);
            value = (unsigned) ? value & 0xffff : value;
            ((int[][]) imageValue)[yIndex][xIndex] = value;
            globalIndex++;
          }
        }

      } else if ("SignedLong".equals(dataType) || "UnsignedInteger".equals(dataType)) {

        // Specific
        imageValue = new long[dimY][dimX];

        // long = 8 bytes
        // int = 4 bytes
        int factor = (unsigned) ? 4 : 8;
        int sizeToRead = factor * dimY * dimX;

        // Global
        Object flatImageValue = null;
        ByteBuffer byteBuffer = EdfFileReader.readAsBytes(sizeToRead, littleEndian, dis);

        // Specific
        if ("UnsignedInteger".equals(dataType)) {
          flatImageValue = new int[dimY * dimX];
          IntBuffer integerBuffer = byteBuffer.asIntBuffer();
          integerBuffer.get((int[]) flatImageValue);
        } else {
          flatImageValue = new long[dimY * dimX];
          LongBuffer longBuffer = byteBuffer.asLongBuffer();
          longBuffer.get((long[]) flatImageValue);
        }

        int globalIndex = 0;
        for (int yIndex = 0; yIndex < dimY; yIndex++) {
          for (int xIndex = 0; xIndex < dimX; xIndex++) {
            long value = Array.getLong(flatImageValue, globalIndex);
            value = (unsigned) ? value & 0xFFFFFFFFL : value;
            ((long[][]) imageValue)[yIndex][xIndex] = value;
            globalIndex++;
          }
        }
      } else if ("Signed64".equals(dataType)) {
        throw new NotImplementedException();
      } else if ("Unsigned64".equals(dataType)) {
        unsigned = true;
        throw new NotImplementedException();
      } else if ("FloatValue".equals(dataType)) {
        throw new NotImplementedException();
      } else if ("DoubleValue".equals(dataType)) {
        throw new NotImplementedException();
      }
      if (imageValue != null) {
        EdfDataItem imageDataItem =
            new EdfDataItem("Image", new DefaultArrayMatrix(EdfFactory.NAME, imageValue), unsigned);
        addDataItem(imageDataItem);
      }
    } catch (Exception e) {
      e.printStackTrace();
      // ignore exceptions for Image Recovery
    }
  }
示例#17
0
 /**
  * Creates a buffer for the given number of elements and native byte ordering
  *
  * @param elements The number of elements in the buffer
  * @return The buffer
  */
 public static LongBuffer createLongBuffer(int elements) {
   ByteBuffer byteBuffer = ByteBuffer.allocateDirect(elements * 8);
   byteBuffer.order(ByteOrder.nativeOrder());
   return byteBuffer.asLongBuffer();
 }
示例#18
0
 public static LongBuffer newLongBuffer(int numLongs) {
   ByteBuffer buffer = ByteBuffer.allocateDirect(numLongs * 8);
   buffer.order(ByteOrder.nativeOrder());
   return buffer.asLongBuffer();
 }
示例#19
0
  Int64BufferSE(ByteBuffer bb) {
    super(bb);

    this.pb = bb.asLongBuffer();
  }
示例#20
0
 public SpillRecord(int numPartitions) {
   buf = ByteBuffer.allocate(numPartitions * MapTask.MAP_OUTPUT_INDEX_RECORD_LENGTH);
   entries = buf.asLongBuffer();
 }