protected void putUUID(String name, UUID val) {
   _put(BINARY, name);
   _buf.writeInt(16);
   _buf.write(B_UUID);
   _buf.writeLong(val.getMostSignificantBits());
   _buf.writeLong(val.getLeastSignificantBits());
 }
Beispiel #2
0
 /**
  * The struct's top level method for indexing 1 record. Each Enum field will call SubIndex
  *
  * @param src the ByteBuffer of the input file
  * @param register array holding values pointing to Stack offsets
  * @param stack A stack of 32-bit pointers only to src positions
  */
 static void index(ByteBuffer src, int[] register, IntBuffer stack) {
   for (UUID UUID_ : values()) {
     String hdr = UUID_.name();
     System.err.println("hdr:pos " + hdr + ':' + stack.position());
     UUID_.subIndex(src, register, stack);
   }
 }
Beispiel #3
0
/**
 * Parser for extended memcache protocol. Handles parsing and encoding activity.
 *
 * @author 2012 Copyright (C) GridGain Systems
 * @version 4.0.2c.12042012
 */
public class GridTcpRestParser implements GridNioParser<GridClientMessage> {
  /** Ping message. */
  public static final GridClientMessage PING_MESSAGE = new GridClientPingPacket();

  /** Ping packet. */
  private static final byte[] PING_PACKET = new byte[] {(byte) 0x90, 0x00, 0x00, 0x00, 0x00};

  /** Meta name for parser state. */
  private static final String PARSER_STATE_META_NAME = UUID.randomUUID().toString();

  /** JDK marshaller. */
  private final GridMarshaller jdkMarshaller = new GridJdkMarshaller();

  /** Hessian marshaller. */
  @GridToStringExclude
  private final GridClientMarshaller marshaller = new GridClientProtobufMarshaller();

  /** {@inheritDoc} */
  @Nullable
  @Override
  public GridClientMessage decode(GridNioSession ses, ByteBuffer buf)
      throws IOException, GridException {
    ParserState state = ses.removeMeta(PARSER_STATE_META_NAME);

    if (state == null) state = new ParserState();

    PacketType type = state.packetType();

    if (type == null) {
      byte hdr = buf.get(buf.position());

      switch (hdr) {
        case MEMCACHE_REQ_FLAG:
          state.packet(new GridTcpRestPacket());
          state.packetType(PacketType.MEMCACHE);

          break;
        case GRIDGAIN_REQ_FLAG:
          // Skip header.
          buf.get();

          state.packetType(PacketType.GRIDGAIN);

          break;

        default:
          throw new IOException(
              "Failed to parse incoming packet (invalid packet start) [ses="
                  + ses
                  + ", b="
                  + Integer.toHexString(hdr & 0xFF)
                  + ']');
      }
    }

    GridClientMessage result = null;

    switch (state.packetType()) {
      case MEMCACHE:
        result = parseMemcachePacket(ses, buf, state);

        break;
      case GRIDGAIN:
        result = parseCustomPacket(ses, buf, state);

        break;
    }

    if (result == null)
      // Packet was not fully parsed yet.
      ses.addMeta(PARSER_STATE_META_NAME, state);

    return result;
  }

  /** {@inheritDoc} */
  @Override
  public ByteBuffer encode(GridNioSession ses, GridClientMessage msg)
      throws IOException, GridException {
    assert msg != null;

    if (msg instanceof GridTcpRestPacket) return encodeMemcache((GridTcpRestPacket) msg);
    else if (msg == PING_MESSAGE) return ByteBuffer.wrap(PING_PACKET);
    else {
      byte[] data = marshaller.marshal(msg);

      assert data.length > 0;

      ByteBuffer res = ByteBuffer.allocate(data.length + 5);

      res.put(GRIDGAIN_REQ_FLAG);
      res.put(U.intToBytes(data.length));
      res.put(data);

      res.flip();

      return res;
    }
  }

  /**
   * Parses memcache protocol message.
   *
   * @param ses Session.
   * @param buf Buffer containing not parsed bytes.
   * @param state Current parser state.
   * @return Parsed packet.s
   * @throws IOException If packet cannot be parsed.
   * @throws GridException If deserialization error occurred.
   */
  @Nullable
  private GridClientMessage parseMemcachePacket(
      GridNioSession ses, ByteBuffer buf, ParserState state) throws IOException, GridException {
    assert state.packetType() == PacketType.MEMCACHE;
    assert state.packet() != null;
    assert state.packet() instanceof GridTcpRestPacket;

    GridTcpRestPacket req = (GridTcpRestPacket) state.packet();
    ByteArrayOutputStream tmp = state.buffer();
    int i = state.index();

    while (buf.remaining() > 0) {
      byte b = buf.get();

      if (i == 0) req.requestFlag(b);
      else if (i == 1) req.operationCode(b);
      else if (i == 2 || i == 3) {
        tmp.write(b);

        if (i == 3) {
          req.keyLength(U.bytesToShort(tmp.toByteArray(), 0));

          tmp.reset();
        }
      } else if (i == 4) req.extrasLength(b);
      else if (i >= 8 && i <= 11) {
        tmp.write(b);

        if (i == 11) {
          req.totalLength(U.bytesToInt(tmp.toByteArray(), 0));

          tmp.reset();
        }
      } else if (i >= 12 && i <= 15) {
        tmp.write(b);

        if (i == 15) {
          req.opaque(tmp.toByteArray());

          tmp.reset();
        }
      } else if (i >= HDR_LEN && i < HDR_LEN + req.extrasLength()) {
        tmp.write(b);

        if (i == HDR_LEN + req.extrasLength() - 1) {
          req.extras(tmp.toByteArray());

          tmp.reset();
        }
      } else if (i >= HDR_LEN + req.extrasLength()
          && i < HDR_LEN + req.extrasLength() + req.keyLength()) {
        tmp.write(b);

        if (i == HDR_LEN + req.extrasLength() + req.keyLength() - 1) {
          req.key(tmp.toByteArray());

          tmp.reset();
        }
      } else if (i >= HDR_LEN + req.extrasLength() + req.keyLength()
          && i < HDR_LEN + req.totalLength()) {
        tmp.write(b);

        if (i == HDR_LEN + req.totalLength() - 1) {
          req.value(tmp.toByteArray());

          tmp.reset();
        }
      }

      if (i == HDR_LEN + req.totalLength() - 1)
        // Assembled the packet.
        return assemble(ses, req);

      i++;
    }

    state.index(i);

    return null;
  }

  /**
   * Parses custom packet serialized by hessian marshaller.
   *
   * @param ses Session.
   * @param buf Buffer containing not parsed bytes.
   * @param state Parser state.
   * @return Parsed message.
   * @throws IOException If packet parsing or deserialization failed.
   */
  @Nullable
  private GridClientMessage parseCustomPacket(GridNioSession ses, ByteBuffer buf, ParserState state)
      throws IOException {
    assert state.packetType() == PacketType.GRIDGAIN;
    assert state.packet() == null;

    ByteArrayOutputStream tmp = state.buffer();

    int len = state.index();

    while (buf.remaining() > 0) {
      byte b = buf.get();

      if (len == 0) {
        tmp.write(b);

        if (tmp.size() == 4) {
          len = U.bytesToInt(tmp.toByteArray(), 0);

          tmp.reset();

          if (len == 0) return PING_MESSAGE;
          else if (len < 0)
            throw new IOException(
                "Failed to parse incoming packet (invalid packet length) [ses="
                    + ses
                    + ", len="
                    + len
                    + ']');

          state.index(len);
        }
      } else {
        tmp.write(b);

        if (tmp.size() == len) return marshaller.unmarshal(tmp.toByteArray());
      }
    }

    return null;
  }

  /**
   * Encodes memcache message to a raw byte array.
   *
   * @param msg Message being serialized.
   * @return Serialized message.
   * @throws GridException If serialization failed.
   */
  private ByteBuffer encodeMemcache(GridTcpRestPacket msg) throws GridException {
    GridByteArrayList res = new GridByteArrayList(HDR_LEN);

    int keyLength = 0;

    int keyFlags = 0;

    if (msg.key() != null) {
      ByteArrayOutputStream rawKey = new ByteArrayOutputStream();

      keyFlags = encodeObj(msg.key(), rawKey);

      msg.key(rawKey.toByteArray());

      keyLength = rawKey.size();
    }

    int dataLength = 0;

    int valFlags = 0;

    if (msg.value() != null) {
      ByteArrayOutputStream rawVal = new ByteArrayOutputStream();

      valFlags = encodeObj(msg.value(), rawVal);

      msg.value(rawVal.toByteArray());

      dataLength = rawVal.size();
    }

    int flagsLength = 0;

    if (msg.addFlags()) // || keyFlags > 0 || valFlags > 0)
    flagsLength = FLAGS_LENGTH;

    res.add(MEMCACHE_RES_FLAG);

    res.add(msg.operationCode());

    // Cast is required due to packet layout.
    res.add((short) keyLength);

    // Cast is required due to packet layout.
    res.add((byte) flagsLength);

    // Data type is always 0x00.
    res.add((byte) 0x00);

    res.add((short) msg.status());

    res.add(keyLength + flagsLength + dataLength);

    res.add(msg.opaque(), 0, msg.opaque().length);

    // CAS, unused.
    res.add(0L);

    assert res.size() == HDR_LEN;

    if (flagsLength > 0) {
      res.add((short) keyFlags);
      res.add((short) valFlags);
    }

    assert msg.key() == null || msg.key() instanceof byte[];
    assert msg.value() == null || msg.value() instanceof byte[];

    if (keyLength > 0) res.add((byte[]) msg.key(), 0, ((byte[]) msg.key()).length);

    if (dataLength > 0) res.add((byte[]) msg.value(), 0, ((byte[]) msg.value()).length);

    return ByteBuffer.wrap(res.entireArray());
  }

  /**
   * Validates incoming packet and deserializes all fields that need to be deserialized.
   *
   * @param ses Session on which packet is being parsed.
   * @param req Raw packet.
   * @return Same packet with fields deserialized.
   * @throws IOException If parsing failed.
   * @throws GridException If deserialization failed.
   */
  private GridClientMessage assemble(GridNioSession ses, GridTcpRestPacket req)
      throws IOException, GridException {
    byte[] extras = req.extras();

    // First, decode key and value, if any
    if (req.key() != null || req.value() != null) {
      short keyFlags = 0;
      short valFlags = 0;

      if (req.hasFlags()) {
        if (extras == null || extras.length < FLAGS_LENGTH)
          throw new IOException(
              "Failed to parse incoming packet (flags required for command) [ses="
                  + ses
                  + ", opCode="
                  + Integer.toHexString(req.operationCode() & 0xFF)
                  + ']');

        keyFlags = U.bytesToShort(extras, 0);
        valFlags = U.bytesToShort(extras, 2);
      }

      if (req.key() != null) {
        assert req.key() instanceof byte[];

        byte[] rawKey = (byte[]) req.key();

        // Only values can be hessian-encoded.
        req.key(decodeObj(keyFlags, rawKey));
      }

      if (req.value() != null) {
        assert req.value() instanceof byte[];

        byte[] rawVal = (byte[]) req.value();

        req.value(decodeObj(valFlags, rawVal));
      }
    }

    if (req.hasExpiration()) {
      if (extras == null || extras.length < 8)
        throw new IOException(
            "Failed to parse incoming packet (expiration value required for command) [ses="
                + ses
                + ", opCode="
                + Integer.toHexString(req.operationCode() & 0xFF)
                + ']');

      req.expiration(U.bytesToInt(extras, 4) & 0xFFFFFFFFL);
    }

    if (req.hasInitial()) {
      if (extras == null || extras.length < 16)
        throw new IOException(
            "Failed to parse incoming packet (initial value required for command) [ses="
                + ses
                + ", opCode="
                + Integer.toHexString(req.operationCode() & 0xFF)
                + ']');

      req.initial(U.bytesToLong(extras, 8));
    }

    if (req.hasDelta()) {
      if (extras == null || extras.length < 8)
        throw new IOException(
            "Failed to parse incoming packet (delta value required for command) [ses="
                + ses
                + ", opCode="
                + Integer.toHexString(req.operationCode() & 0xFF)
                + ']');

      req.delta(U.bytesToLong(extras, 0));
    }

    if (extras != null) {
      // Clients that include cache name must always include flags.
      int length = 4;

      if (req.hasExpiration()) length += 4;

      if (req.hasDelta()) length += 8;

      if (req.hasInitial()) length += 8;

      if (extras.length - length > 0) {
        byte[] cacheName = new byte[extras.length - length];

        System.arraycopy(extras, length, cacheName, 0, extras.length - length);

        req.cacheName(new String(cacheName));
      }
    }

    return req;
  }

  /**
   * Decodes value from a given byte array to the object according to the flags given.
   *
   * @param flags Flags.
   * @param bytes Byte array to decode.
   * @return Decoded value.
   * @throws GridException If deserialization failed.
   */
  private Object decodeObj(short flags, byte[] bytes) throws GridException {
    assert bytes != null;

    if ((flags & SERIALIZED_FLAG) != 0)
      return jdkMarshaller.unmarshal(new ByteArrayInputStream(bytes), null);

    int masked = flags & 0xff00;

    switch (masked) {
      case BOOLEAN_FLAG:
        return bytes[0] == '1';
      case INT_FLAG:
        return U.bytesToInt(bytes, 0);
      case LONG_FLAG:
        return U.bytesToLong(bytes, 0);
      case DATE_FLAG:
        return new Date(U.bytesToLong(bytes, 0));
      case BYTE_FLAG:
        return bytes[0];
      case FLOAT_FLAG:
        return Float.intBitsToFloat(U.bytesToInt(bytes, 0));
      case DOUBLE_FLAG:
        return Double.longBitsToDouble(U.bytesToLong(bytes, 0));
      case BYTE_ARR_FLAG:
        return bytes;
      default:
        return new String(bytes);
    }
  }

  /**
   * Encodes given object to a byte array and returns flags that describe the type of serialized
   * object.
   *
   * @param obj Object to serialize.
   * @param out Output stream to which object should be written.
   * @return Serialization flags.
   * @throws GridException If JDK serialization failed.
   */
  private int encodeObj(Object obj, ByteArrayOutputStream out) throws GridException {
    int flags = 0;

    byte[] data = null;

    if (obj instanceof String) {
      data = ((String) obj).getBytes();
    } else if (obj instanceof Boolean) {
      data = new byte[] {(byte) ((Boolean) obj ? '1' : '0')};

      flags |= BOOLEAN_FLAG;
    } else if (obj instanceof Integer) {
      data = U.intToBytes((Integer) obj);

      flags |= INT_FLAG;
    } else if (obj instanceof Long) {
      data = U.longToBytes((Long) obj);

      flags |= LONG_FLAG;
    } else if (obj instanceof Date) {
      data = U.longToBytes(((Date) obj).getTime());

      flags |= DATE_FLAG;
    } else if (obj instanceof Byte) {
      data = new byte[] {(Byte) obj};

      flags |= BYTE_FLAG;
    } else if (obj instanceof Float) {
      data = U.intToBytes(Float.floatToIntBits((Float) obj));

      flags |= FLOAT_FLAG;
    } else if (obj instanceof Double) {
      data = U.longToBytes(Double.doubleToLongBits((Double) obj));

      flags |= DOUBLE_FLAG;
    } else if (obj instanceof byte[]) {
      data = (byte[]) obj;

      flags |= BYTE_ARR_FLAG;
    } else {
      jdkMarshaller.marshal(obj, out);

      flags |= SERIALIZED_FLAG;
    }

    if (data != null) out.write(data, 0, data.length);

    return flags;
  }

  /** {@inheritDoc} */
  public String toString() {
    return S.toString(
        GridTcpRestParser.class, this, "clientMarshaller", marshaller.getClass().getSimpleName());
  }

  /** Type of message being parsed. */
  private enum PacketType {
    /** Memcache protocol message. */
    MEMCACHE,

    /** Custom hessian-serialized message. */
    GRIDGAIN
  }

  /** Holder for parser state and temporary buffer. */
  private static class ParserState {
    /** Parser index. */
    private int idx;

    /** Temporary data buffer. */
    private ByteArrayOutputStream buf = new ByteArrayOutputStream();

    /** Packet being assembled. */
    private GridClientMessage packet;

    /** Packet type. */
    private PacketType packetType;

    /** @return Stored parser index. */
    private int index() {
      return idx;
    }

    /** @param idx Index to store. */
    private void index(int idx) {
      this.idx = idx;
    }

    /** @return Temporary data buffer. */
    private ByteArrayOutputStream buffer() {
      return buf;
    }

    /** @return Pending packet. */
    private GridClientMessage packet() {
      return packet;
    }

    /** @param packet Pending packet. */
    private void packet(GridClientMessage packet) {
      assert this.packet == null;

      this.packet = packet;
    }

    /** @return Pending packet type. */
    private PacketType packetType() {
      return packetType;
    }

    /** @param packetType Pending packet type. */
    private void packetType(PacketType packetType) {
      this.packetType = packetType;
    }

    /** {@inheritDoc} */
    @Override
    public String toString() {
      return S.toString(ParserState.class, this);
    }
  }
}
  /** @throws Exception If failed. */
  public void testCompact() throws Exception {
    File file = new File(UUID.randomUUID().toString());

    X.println("file: " + file.getPath());

    FileSwapSpaceSpi.SwapFile f = new FileSwapSpaceSpi.SwapFile(file, 8);

    Random rnd = new Random();

    ArrayList<FileSwapSpaceSpi.SwapValue> arr = new ArrayList<>();

    int size = 0;

    for (int a = 0; a < 100; a++) {
      FileSwapSpaceSpi.SwapValue[] vals = new FileSwapSpaceSpi.SwapValue[1 + rnd.nextInt(10)];

      int size0 = 0;

      for (int i = 0; i < vals.length; i++) {
        byte[] bytes = new byte[1 + rnd.nextInt(49)];

        rnd.nextBytes(bytes);

        size0 += bytes.length;

        vals[i] = new FileSwapSpaceSpi.SwapValue(bytes);

        arr.add(vals[i]);
      }

      f.write(new FileSwapSpaceSpi.SwapValues(vals, size0), 1);

      size += size0;

      assertEquals(f.length(), size);
      assertEquals(file.length(), size);
    }

    int i = 0;

    for (FileSwapSpaceSpi.SwapValue val : arr) assertEquals(val.idx(), ++i);

    i = 0;

    for (int cnt = arr.size() / 2; i < cnt; i++) {

      FileSwapSpaceSpi.SwapValue v = arr.remove(rnd.nextInt(arr.size()));

      assertTrue(f.tryRemove(v.idx(), v));
    }

    int hash0 = 0;

    for (FileSwapSpaceSpi.SwapValue val : arr) hash0 += Arrays.hashCode(val.readValue(f.readCh));

    ArrayList<T2<ByteBuffer, ArrayDeque<FileSwapSpaceSpi.SwapValue>>> bufs = new ArrayList();

    for (; ; ) {
      ArrayDeque<FileSwapSpaceSpi.SwapValue> que = new ArrayDeque<>();

      ByteBuffer buf = f.compact(que, 1024);

      if (buf == null) break;

      bufs.add(new T2(buf, que));
    }

    f.delete();

    int hash1 = 0;

    for (FileSwapSpaceSpi.SwapValue val : arr) hash1 += Arrays.hashCode(val.value(null));

    assertEquals(hash0, hash1);

    File file0 = new File(UUID.randomUUID().toString());

    FileSwapSpaceSpi.SwapFile f0 = new FileSwapSpaceSpi.SwapFile(file0, 8);

    for (T2<ByteBuffer, ArrayDeque<FileSwapSpaceSpi.SwapValue>> t : bufs)
      f0.write(t.get2(), t.get1(), 1);

    int hash2 = 0;

    for (FileSwapSpaceSpi.SwapValue val : arr) hash2 += Arrays.hashCode(val.readValue(f0.readCh));

    assertEquals(hash2, hash1);
  }