/** * Writes the value and value length for non-first record. * * @param kv Key-value writer. * @return The offset of the new record. */ private long writeValueAndLength(KvSource kv) throws SerDeException { long valueOffset = writeBuffers.getWritePoint(); kv.writeValue(writeBuffers); long tailOffset = writeBuffers.getWritePoint(); writeBuffers.writeVLong(tailOffset - valueOffset); // LOG.info("Writing value at " + valueOffset + " length " + (tailOffset - valueOffset)); return tailOffset; }
/** * @param ref The ref. * @return The offset to list record pointer; list record is created if it doesn't exist. */ private long createOrGetListRecord(long ref) { if (Ref.hasList(ref)) { // LOG.info("Found list record at " + writeBuffers.getReadPoint()); return writeBuffers.getReadPoint(); // Assumes we are here after key compare. } long firstTailOffset = Ref.getOffset(ref); // LOG.info("First tail offset to create list record is " + firstTailOffset); // Determine the length of storage for value and key lengths of the first record. writeBuffers.setReadPoint(firstTailOffset); writeBuffers.skipVLong(); writeBuffers.skipVLong(); int lengthsLength = (int) (writeBuffers.getReadPoint() - firstTailOffset); // Create the list record, copy first record value/key lengths there. writeBuffers.writeBytes(firstTailOffset, lengthsLength); long lrPtrOffset = writeBuffers.getWritePoint(); // LOG.info("Creating list record: copying " + lengthsLength + ", lrPtrOffset " + lrPtrOffset); // Reserve 5 bytes for writeValueRecord to fill. There might be junk there so null them. writeBuffers.write(FIVE_ZEROES); // Link the list record to the first element. writeBuffers.writeFiveByteULong(firstTailOffset, lrPtrOffset - lengthsLength - firstTailOffset); return lrPtrOffset; }
public void put(KvSource kv, int keyHashCode) throws SerDeException { if (resizeThreshold <= keysAssigned) { expandAndRehash(); } // Reserve 4 bytes for the hash (don't just reserve, there may be junk there) writeBuffers.write(FOUR_ZEROES); // Write key to buffer to compute hashcode and compare; if it's a new key, it will // become part of the record; otherwise, we will just write over it later. long keyOffset = writeBuffers.getWritePoint(); kv.writeKey(writeBuffers); int keyLength = (int) (writeBuffers.getWritePoint() - keyOffset); int hashCode = (keyHashCode == -1) ? writeBuffers.hashCode(keyOffset, keyLength) : keyHashCode; int slot = findKeySlotToWrite(keyOffset, keyLength, hashCode); // LOG.info("Write hash code is " + Integer.toBinaryString(hashCode) + " - " + slot); long ref = refs[slot]; if (ref == 0) { // This is a new key, keep writing the first record. long tailOffset = writeFirstValueRecord(kv, keyOffset, keyLength, hashCode); byte stateByte = kv.updateStateByte(null); refs[slot] = Ref.makeFirstRef(tailOffset, stateByte, hashCode, startingHashBitCount); ++keysAssigned; } else { // This is not a new key; we'll overwrite the key and hash bytes - not needed anymore. writeBuffers.setWritePoint(keyOffset - 4); long lrPtrOffset = createOrGetListRecord(ref); long tailOffset = writeValueAndLength(kv); addRecordToList(lrPtrOffset, tailOffset); byte oldStateByte = Ref.getStateByte(ref); byte stateByte = kv.updateStateByte(oldStateByte); if (oldStateByte != stateByte) { ref = Ref.setStateByte(ref, stateByte); } refs[slot] = Ref.setListFlag(ref); } ++numValues; }
/** * Writes first value and lengths to finish the first record after the key has been written. * * @param kv Key-value writer. * @param keyOffset * @param keyLength Key length (already written). * @param hashCode * @return The offset of the new record. */ private long writeFirstValueRecord(KvSource kv, long keyOffset, int keyLength, int hashCode) throws SerDeException { long valueOffset = writeBuffers.getWritePoint(); kv.writeValue(writeBuffers); long tailOffset = writeBuffers.getWritePoint(); int valueLength = (int) (tailOffset - valueOffset); // LOG.info("Writing value at " + valueOffset + " length " + valueLength); // In an unlikely case of 0-length key and value for the very first entry, we want to tell // this apart from an empty value. We'll just advance one byte; this byte will be lost. if (tailOffset == 0) { writeBuffers.reserve(1); ++tailOffset; } // LOG.info("First tail offset " + writeBuffers.getWritePoint()); writeBuffers.writeVLong(valueLength); writeBuffers.writeVLong(keyLength); long lengthsLength = writeBuffers.getWritePoint() - tailOffset; if (lengthsLength < 5) { // Reserve space for potential future list writeBuffers.reserve(5 - (int) lengthsLength); } // Finally write the hash code. writeBuffers.writeInt(keyOffset - 4, hashCode); return tailOffset; }