/** Lets go of any references to a hash map. */ public void forget() { hashMap = null; readPos = null; byteSegmentRef.reset(0, 0); hasRows = false; readIndex = 0; nextTailOffset = -1; }
private List<Object> unpack(WriteBuffers.ByteSegmentRef ref) throws HiveException { if (ref.getLength() == 0) { return EMPTY_LIST; // shortcut, 0 length means no fields } uselessIndirection.setData(ref.getBytes()); valueStruct.init(uselessIndirection, (int) ref.getOffset(), ref.getLength()); List<Object> result; if (!needsComplexObjectFixup) { // Good performance for common case where small table has no complex objects. result = valueStruct.getFieldsAsList(); } else { // Convert the complex LazyBinary objects to standard (Java) objects so downstream // operators like FileSinkOperator can serialize complex objects in the form they expect // (i.e. Java objects). result = getComplexFieldsAsList(valueStruct, complexObjectArrayBuffer, internalValueOi); } return result; }
/** Writes the debug dump of the table into logs. Not thread-safe. */ public void debugDumpTable() { StringBuilder dump = new StringBuilder(keysAssigned + " keys\n"); TreeMap<Long, Integer> byteIntervals = new TreeMap<Long, Integer>(); int examined = 0; for (int slot = 0; slot < refs.length; ++slot) { long ref = refs[slot]; if (ref == 0) { continue; } ++examined; long recOffset = getFirstRecordLengthsOffset(ref, null); long tailOffset = Ref.getOffset(ref); writeBuffers.setReadPoint(recOffset); int valueLength = (int) writeBuffers.readVLong(), keyLength = (int) writeBuffers.readVLong(); long ptrOffset = writeBuffers.getReadPoint(); if (Ref.hasList(ref)) { byteIntervals.put(recOffset, (int) (ptrOffset + 5 - recOffset)); } long keyOffset = tailOffset - valueLength - keyLength; byte[] key = new byte[keyLength]; WriteBuffers.ByteSegmentRef fakeRef = new WriteBuffers.ByteSegmentRef(keyOffset, keyLength); byteIntervals.put(keyOffset - 4, keyLength + 4); writeBuffers.populateValue(fakeRef); System.arraycopy(fakeRef.getBytes(), (int) fakeRef.getOffset(), key, 0, keyLength); dump.append(Utils.toStringBinary(key, 0, key.length)) .append(" ref [") .append(dumpRef(ref)) .append("]: "); Result hashMapResult = new Result(); getValueResult(key, 0, key.length, hashMapResult); List<WriteBuffers.ByteSegmentRef> results = new ArrayList<WriteBuffers.ByteSegmentRef>(); WriteBuffers.ByteSegmentRef byteSegmentRef = hashMapResult.first(); while (byteSegmentRef != null) { results.add(hashMapResult.byteSegmentRef); byteSegmentRef = hashMapResult.next(); } dump.append(results.size()).append(" rows\n"); for (int i = 0; i < results.size(); ++i) { WriteBuffers.ByteSegmentRef segment = results.get(i); byteIntervals.put( segment.getOffset(), segment.getLength() + ((i == 0) ? 1 : 0)); // state byte in the first record } } if (examined != keysAssigned) { dump.append("Found " + examined + " keys!\n"); } // Report suspicious gaps in writeBuffers long currentOffset = 0; for (Map.Entry<Long, Integer> e : byteIntervals.entrySet()) { long start = e.getKey(), len = e.getValue(); if (start - currentOffset > 4) { dump.append("Gap! [" + currentOffset + ", " + start + ")\n"); } currentOffset = start + len; } LOG.info("Hashtable dump:\n " + dump.toString()); }
private void debugDumpKeyProbe(long keyOffset, int keyLength, int hashCode, int finalSlot) { final int bucketMask = refs.length - 1; WriteBuffers.ByteSegmentRef fakeRef = new WriteBuffers.ByteSegmentRef(keyOffset, keyLength); writeBuffers.populateValue(fakeRef); int slot = hashCode & bucketMask; long probeSlot = slot; StringBuilder sb = new StringBuilder("Probe path debug for ["); sb.append( Utils.toStringBinary(fakeRef.getBytes(), (int) fakeRef.getOffset(), fakeRef.getLength())); sb.append("] hashCode ").append(Integer.toBinaryString(hashCode)).append(" is: "); int i = 0; while (slot != finalSlot) { probeSlot += (++i); slot = (int) (probeSlot & bucketMask); sb.append(slot) .append(" - ") .append(probeSlot) .append(" - ") .append(Long.toBinaryString(refs[slot])) .append("\n"); } LOG.info(sb.toString()); }
/** * Read the current value. * * @return The ByteSegmentRef to the current value read. */ private WriteBuffers.ByteSegmentRef internalRead() { if (!hasList) { /* * Single value. */ if (readIndex > 0) { return null; } // For a non-list (i.e. single value), the offset is for the variable length long (VLong) // holding the value length (followed by the key length). hashMap.writeBuffers.setReadPoint(firstOffset, readPos); int valueLength = (int) hashMap.writeBuffers.readVLong(readPos); // The value is before the offset. Make byte segment reference absolute. byteSegmentRef.reset(firstOffset - valueLength, valueLength); hashMap.writeBuffers.populateValue(byteSegmentRef); readIndex++; return byteSegmentRef; } /* * Multiple values. */ if (readIndex == 0) { // For a list, the value and key lengths of 1st record were overwritten with the // relative offset to a new list record. long relativeOffset = hashMap.writeBuffers.readNByteLong(firstOffset, 5, readPos); // At the beginning of the list record will be the value length. hashMap.writeBuffers.setReadPoint(firstOffset + relativeOffset, readPos); int valueLength = (int) hashMap.writeBuffers.readVLong(readPos); // The value is before the list record offset. Make byte segment reference absolute. byteSegmentRef.reset(firstOffset - valueLength, valueLength); hashMap.writeBuffers.populateValue(byteSegmentRef); readIndex++; return byteSegmentRef; } if (readIndex == 1) { // We remembered the offset of just after the key length in the list record. // Read the absolute offset to the 2nd value. nextTailOffset = hashMap.writeBuffers.readNByteLong(offsetAfterListRecordKeyLen, 5, readPos); if (nextTailOffset <= 0) { throw new Error("Expecting a second value"); } } else if (nextTailOffset <= 0) { return null; } hashMap.writeBuffers.setReadPoint(nextTailOffset, readPos); // Get the value length. int valueLength = (int) hashMap.writeBuffers.readVLong(readPos); // Now read the relative offset to next record. Next record is always before the // previous record in the write buffers (see writeBuffers javadoc). long delta = hashMap.writeBuffers.readVLong(readPos); long newTailOffset = delta == 0 ? 0 : (nextTailOffset - delta); // The value is before the value record offset. Make byte segment reference absolute. byteSegmentRef.reset(nextTailOffset - valueLength, valueLength); hashMap.writeBuffers.populateValue(byteSegmentRef); nextTailOffset = newTailOffset; readIndex++; return byteSegmentRef; }