protected void putUUID(String name, UUID val) { _put(BINARY, name); _buf.writeInt(16); _buf.write(B_UUID); _buf.writeLong(val.getMostSignificantBits()); _buf.writeLong(val.getLeastSignificantBits()); }
/** * The struct's top level method for indexing 1 record. Each Enum field will call SubIndex * * @param src the ByteBuffer of the input file * @param register array holding values pointing to Stack offsets * @param stack A stack of 32-bit pointers only to src positions */ static void index(ByteBuffer src, int[] register, IntBuffer stack) { for (UUID UUID_ : values()) { String hdr = UUID_.name(); System.err.println("hdr:pos " + hdr + ':' + stack.position()); UUID_.subIndex(src, register, stack); } }
/** * Parser for extended memcache protocol. Handles parsing and encoding activity. * * @author 2012 Copyright (C) GridGain Systems * @version 4.0.2c.12042012 */ public class GridTcpRestParser implements GridNioParser<GridClientMessage> { /** Ping message. */ public static final GridClientMessage PING_MESSAGE = new GridClientPingPacket(); /** Ping packet. */ private static final byte[] PING_PACKET = new byte[] {(byte) 0x90, 0x00, 0x00, 0x00, 0x00}; /** Meta name for parser state. */ private static final String PARSER_STATE_META_NAME = UUID.randomUUID().toString(); /** JDK marshaller. */ private final GridMarshaller jdkMarshaller = new GridJdkMarshaller(); /** Hessian marshaller. */ @GridToStringExclude private final GridClientMarshaller marshaller = new GridClientProtobufMarshaller(); /** {@inheritDoc} */ @Nullable @Override public GridClientMessage decode(GridNioSession ses, ByteBuffer buf) throws IOException, GridException { ParserState state = ses.removeMeta(PARSER_STATE_META_NAME); if (state == null) state = new ParserState(); PacketType type = state.packetType(); if (type == null) { byte hdr = buf.get(buf.position()); switch (hdr) { case MEMCACHE_REQ_FLAG: state.packet(new GridTcpRestPacket()); state.packetType(PacketType.MEMCACHE); break; case GRIDGAIN_REQ_FLAG: // Skip header. buf.get(); state.packetType(PacketType.GRIDGAIN); break; default: throw new IOException( "Failed to parse incoming packet (invalid packet start) [ses=" + ses + ", b=" + Integer.toHexString(hdr & 0xFF) + ']'); } } GridClientMessage result = null; switch (state.packetType()) { case MEMCACHE: result = parseMemcachePacket(ses, buf, state); break; case GRIDGAIN: result = parseCustomPacket(ses, buf, state); break; } if (result == null) // Packet was not fully parsed yet. ses.addMeta(PARSER_STATE_META_NAME, state); return result; } /** {@inheritDoc} */ @Override public ByteBuffer encode(GridNioSession ses, GridClientMessage msg) throws IOException, GridException { assert msg != null; if (msg instanceof GridTcpRestPacket) return encodeMemcache((GridTcpRestPacket) msg); else if (msg == PING_MESSAGE) return ByteBuffer.wrap(PING_PACKET); else { byte[] data = marshaller.marshal(msg); assert data.length > 0; ByteBuffer res = ByteBuffer.allocate(data.length + 5); res.put(GRIDGAIN_REQ_FLAG); res.put(U.intToBytes(data.length)); res.put(data); res.flip(); return res; } } /** * Parses memcache protocol message. * * @param ses Session. * @param buf Buffer containing not parsed bytes. * @param state Current parser state. * @return Parsed packet.s * @throws IOException If packet cannot be parsed. * @throws GridException If deserialization error occurred. */ @Nullable private GridClientMessage parseMemcachePacket( GridNioSession ses, ByteBuffer buf, ParserState state) throws IOException, GridException { assert state.packetType() == PacketType.MEMCACHE; assert state.packet() != null; assert state.packet() instanceof GridTcpRestPacket; GridTcpRestPacket req = (GridTcpRestPacket) state.packet(); ByteArrayOutputStream tmp = state.buffer(); int i = state.index(); while (buf.remaining() > 0) { byte b = buf.get(); if (i == 0) req.requestFlag(b); else if (i == 1) req.operationCode(b); else if (i == 2 || i == 3) { tmp.write(b); if (i == 3) { req.keyLength(U.bytesToShort(tmp.toByteArray(), 0)); tmp.reset(); } } else if (i == 4) req.extrasLength(b); else if (i >= 8 && i <= 11) { tmp.write(b); if (i == 11) { req.totalLength(U.bytesToInt(tmp.toByteArray(), 0)); tmp.reset(); } } else if (i >= 12 && i <= 15) { tmp.write(b); if (i == 15) { req.opaque(tmp.toByteArray()); tmp.reset(); } } else if (i >= HDR_LEN && i < HDR_LEN + req.extrasLength()) { tmp.write(b); if (i == HDR_LEN + req.extrasLength() - 1) { req.extras(tmp.toByteArray()); tmp.reset(); } } else if (i >= HDR_LEN + req.extrasLength() && i < HDR_LEN + req.extrasLength() + req.keyLength()) { tmp.write(b); if (i == HDR_LEN + req.extrasLength() + req.keyLength() - 1) { req.key(tmp.toByteArray()); tmp.reset(); } } else if (i >= HDR_LEN + req.extrasLength() + req.keyLength() && i < HDR_LEN + req.totalLength()) { tmp.write(b); if (i == HDR_LEN + req.totalLength() - 1) { req.value(tmp.toByteArray()); tmp.reset(); } } if (i == HDR_LEN + req.totalLength() - 1) // Assembled the packet. return assemble(ses, req); i++; } state.index(i); return null; } /** * Parses custom packet serialized by hessian marshaller. * * @param ses Session. * @param buf Buffer containing not parsed bytes. * @param state Parser state. * @return Parsed message. * @throws IOException If packet parsing or deserialization failed. */ @Nullable private GridClientMessage parseCustomPacket(GridNioSession ses, ByteBuffer buf, ParserState state) throws IOException { assert state.packetType() == PacketType.GRIDGAIN; assert state.packet() == null; ByteArrayOutputStream tmp = state.buffer(); int len = state.index(); while (buf.remaining() > 0) { byte b = buf.get(); if (len == 0) { tmp.write(b); if (tmp.size() == 4) { len = U.bytesToInt(tmp.toByteArray(), 0); tmp.reset(); if (len == 0) return PING_MESSAGE; else if (len < 0) throw new IOException( "Failed to parse incoming packet (invalid packet length) [ses=" + ses + ", len=" + len + ']'); state.index(len); } } else { tmp.write(b); if (tmp.size() == len) return marshaller.unmarshal(tmp.toByteArray()); } } return null; } /** * Encodes memcache message to a raw byte array. * * @param msg Message being serialized. * @return Serialized message. * @throws GridException If serialization failed. */ private ByteBuffer encodeMemcache(GridTcpRestPacket msg) throws GridException { GridByteArrayList res = new GridByteArrayList(HDR_LEN); int keyLength = 0; int keyFlags = 0; if (msg.key() != null) { ByteArrayOutputStream rawKey = new ByteArrayOutputStream(); keyFlags = encodeObj(msg.key(), rawKey); msg.key(rawKey.toByteArray()); keyLength = rawKey.size(); } int dataLength = 0; int valFlags = 0; if (msg.value() != null) { ByteArrayOutputStream rawVal = new ByteArrayOutputStream(); valFlags = encodeObj(msg.value(), rawVal); msg.value(rawVal.toByteArray()); dataLength = rawVal.size(); } int flagsLength = 0; if (msg.addFlags()) // || keyFlags > 0 || valFlags > 0) flagsLength = FLAGS_LENGTH; res.add(MEMCACHE_RES_FLAG); res.add(msg.operationCode()); // Cast is required due to packet layout. res.add((short) keyLength); // Cast is required due to packet layout. res.add((byte) flagsLength); // Data type is always 0x00. res.add((byte) 0x00); res.add((short) msg.status()); res.add(keyLength + flagsLength + dataLength); res.add(msg.opaque(), 0, msg.opaque().length); // CAS, unused. res.add(0L); assert res.size() == HDR_LEN; if (flagsLength > 0) { res.add((short) keyFlags); res.add((short) valFlags); } assert msg.key() == null || msg.key() instanceof byte[]; assert msg.value() == null || msg.value() instanceof byte[]; if (keyLength > 0) res.add((byte[]) msg.key(), 0, ((byte[]) msg.key()).length); if (dataLength > 0) res.add((byte[]) msg.value(), 0, ((byte[]) msg.value()).length); return ByteBuffer.wrap(res.entireArray()); } /** * Validates incoming packet and deserializes all fields that need to be deserialized. * * @param ses Session on which packet is being parsed. * @param req Raw packet. * @return Same packet with fields deserialized. * @throws IOException If parsing failed. * @throws GridException If deserialization failed. */ private GridClientMessage assemble(GridNioSession ses, GridTcpRestPacket req) throws IOException, GridException { byte[] extras = req.extras(); // First, decode key and value, if any if (req.key() != null || req.value() != null) { short keyFlags = 0; short valFlags = 0; if (req.hasFlags()) { if (extras == null || extras.length < FLAGS_LENGTH) throw new IOException( "Failed to parse incoming packet (flags required for command) [ses=" + ses + ", opCode=" + Integer.toHexString(req.operationCode() & 0xFF) + ']'); keyFlags = U.bytesToShort(extras, 0); valFlags = U.bytesToShort(extras, 2); } if (req.key() != null) { assert req.key() instanceof byte[]; byte[] rawKey = (byte[]) req.key(); // Only values can be hessian-encoded. req.key(decodeObj(keyFlags, rawKey)); } if (req.value() != null) { assert req.value() instanceof byte[]; byte[] rawVal = (byte[]) req.value(); req.value(decodeObj(valFlags, rawVal)); } } if (req.hasExpiration()) { if (extras == null || extras.length < 8) throw new IOException( "Failed to parse incoming packet (expiration value required for command) [ses=" + ses + ", opCode=" + Integer.toHexString(req.operationCode() & 0xFF) + ']'); req.expiration(U.bytesToInt(extras, 4) & 0xFFFFFFFFL); } if (req.hasInitial()) { if (extras == null || extras.length < 16) throw new IOException( "Failed to parse incoming packet (initial value required for command) [ses=" + ses + ", opCode=" + Integer.toHexString(req.operationCode() & 0xFF) + ']'); req.initial(U.bytesToLong(extras, 8)); } if (req.hasDelta()) { if (extras == null || extras.length < 8) throw new IOException( "Failed to parse incoming packet (delta value required for command) [ses=" + ses + ", opCode=" + Integer.toHexString(req.operationCode() & 0xFF) + ']'); req.delta(U.bytesToLong(extras, 0)); } if (extras != null) { // Clients that include cache name must always include flags. int length = 4; if (req.hasExpiration()) length += 4; if (req.hasDelta()) length += 8; if (req.hasInitial()) length += 8; if (extras.length - length > 0) { byte[] cacheName = new byte[extras.length - length]; System.arraycopy(extras, length, cacheName, 0, extras.length - length); req.cacheName(new String(cacheName)); } } return req; } /** * Decodes value from a given byte array to the object according to the flags given. * * @param flags Flags. * @param bytes Byte array to decode. * @return Decoded value. * @throws GridException If deserialization failed. */ private Object decodeObj(short flags, byte[] bytes) throws GridException { assert bytes != null; if ((flags & SERIALIZED_FLAG) != 0) return jdkMarshaller.unmarshal(new ByteArrayInputStream(bytes), null); int masked = flags & 0xff00; switch (masked) { case BOOLEAN_FLAG: return bytes[0] == '1'; case INT_FLAG: return U.bytesToInt(bytes, 0); case LONG_FLAG: return U.bytesToLong(bytes, 0); case DATE_FLAG: return new Date(U.bytesToLong(bytes, 0)); case BYTE_FLAG: return bytes[0]; case FLOAT_FLAG: return Float.intBitsToFloat(U.bytesToInt(bytes, 0)); case DOUBLE_FLAG: return Double.longBitsToDouble(U.bytesToLong(bytes, 0)); case BYTE_ARR_FLAG: return bytes; default: return new String(bytes); } } /** * Encodes given object to a byte array and returns flags that describe the type of serialized * object. * * @param obj Object to serialize. * @param out Output stream to which object should be written. * @return Serialization flags. * @throws GridException If JDK serialization failed. */ private int encodeObj(Object obj, ByteArrayOutputStream out) throws GridException { int flags = 0; byte[] data = null; if (obj instanceof String) { data = ((String) obj).getBytes(); } else if (obj instanceof Boolean) { data = new byte[] {(byte) ((Boolean) obj ? '1' : '0')}; flags |= BOOLEAN_FLAG; } else if (obj instanceof Integer) { data = U.intToBytes((Integer) obj); flags |= INT_FLAG; } else if (obj instanceof Long) { data = U.longToBytes((Long) obj); flags |= LONG_FLAG; } else if (obj instanceof Date) { data = U.longToBytes(((Date) obj).getTime()); flags |= DATE_FLAG; } else if (obj instanceof Byte) { data = new byte[] {(Byte) obj}; flags |= BYTE_FLAG; } else if (obj instanceof Float) { data = U.intToBytes(Float.floatToIntBits((Float) obj)); flags |= FLOAT_FLAG; } else if (obj instanceof Double) { data = U.longToBytes(Double.doubleToLongBits((Double) obj)); flags |= DOUBLE_FLAG; } else if (obj instanceof byte[]) { data = (byte[]) obj; flags |= BYTE_ARR_FLAG; } else { jdkMarshaller.marshal(obj, out); flags |= SERIALIZED_FLAG; } if (data != null) out.write(data, 0, data.length); return flags; } /** {@inheritDoc} */ public String toString() { return S.toString( GridTcpRestParser.class, this, "clientMarshaller", marshaller.getClass().getSimpleName()); } /** Type of message being parsed. */ private enum PacketType { /** Memcache protocol message. */ MEMCACHE, /** Custom hessian-serialized message. */ GRIDGAIN } /** Holder for parser state and temporary buffer. */ private static class ParserState { /** Parser index. */ private int idx; /** Temporary data buffer. */ private ByteArrayOutputStream buf = new ByteArrayOutputStream(); /** Packet being assembled. */ private GridClientMessage packet; /** Packet type. */ private PacketType packetType; /** @return Stored parser index. */ private int index() { return idx; } /** @param idx Index to store. */ private void index(int idx) { this.idx = idx; } /** @return Temporary data buffer. */ private ByteArrayOutputStream buffer() { return buf; } /** @return Pending packet. */ private GridClientMessage packet() { return packet; } /** @param packet Pending packet. */ private void packet(GridClientMessage packet) { assert this.packet == null; this.packet = packet; } /** @return Pending packet type. */ private PacketType packetType() { return packetType; } /** @param packetType Pending packet type. */ private void packetType(PacketType packetType) { this.packetType = packetType; } /** {@inheritDoc} */ @Override public String toString() { return S.toString(ParserState.class, this); } } }
/** @throws Exception If failed. */ public void testCompact() throws Exception { File file = new File(UUID.randomUUID().toString()); X.println("file: " + file.getPath()); FileSwapSpaceSpi.SwapFile f = new FileSwapSpaceSpi.SwapFile(file, 8); Random rnd = new Random(); ArrayList<FileSwapSpaceSpi.SwapValue> arr = new ArrayList<>(); int size = 0; for (int a = 0; a < 100; a++) { FileSwapSpaceSpi.SwapValue[] vals = new FileSwapSpaceSpi.SwapValue[1 + rnd.nextInt(10)]; int size0 = 0; for (int i = 0; i < vals.length; i++) { byte[] bytes = new byte[1 + rnd.nextInt(49)]; rnd.nextBytes(bytes); size0 += bytes.length; vals[i] = new FileSwapSpaceSpi.SwapValue(bytes); arr.add(vals[i]); } f.write(new FileSwapSpaceSpi.SwapValues(vals, size0), 1); size += size0; assertEquals(f.length(), size); assertEquals(file.length(), size); } int i = 0; for (FileSwapSpaceSpi.SwapValue val : arr) assertEquals(val.idx(), ++i); i = 0; for (int cnt = arr.size() / 2; i < cnt; i++) { FileSwapSpaceSpi.SwapValue v = arr.remove(rnd.nextInt(arr.size())); assertTrue(f.tryRemove(v.idx(), v)); } int hash0 = 0; for (FileSwapSpaceSpi.SwapValue val : arr) hash0 += Arrays.hashCode(val.readValue(f.readCh)); ArrayList<T2<ByteBuffer, ArrayDeque<FileSwapSpaceSpi.SwapValue>>> bufs = new ArrayList(); for (; ; ) { ArrayDeque<FileSwapSpaceSpi.SwapValue> que = new ArrayDeque<>(); ByteBuffer buf = f.compact(que, 1024); if (buf == null) break; bufs.add(new T2(buf, que)); } f.delete(); int hash1 = 0; for (FileSwapSpaceSpi.SwapValue val : arr) hash1 += Arrays.hashCode(val.value(null)); assertEquals(hash0, hash1); File file0 = new File(UUID.randomUUID().toString()); FileSwapSpaceSpi.SwapFile f0 = new FileSwapSpaceSpi.SwapFile(file0, 8); for (T2<ByteBuffer, ArrayDeque<FileSwapSpaceSpi.SwapValue>> t : bufs) f0.write(t.get2(), t.get1(), 1); int hash2 = 0; for (FileSwapSpaceSpi.SwapValue val : arr) hash2 += Arrays.hashCode(val.readValue(f0.readCh)); assertEquals(hash2, hash1); }