/** Lets go of any references to a hash map. */
public void forget() {
hashMap = null;
readPos = null;
byteSegmentRef.reset(0, 0);
hasRows = false;
readIndex = 0;
nextTailOffset = -1;
}
private List<Object> unpack(WriteBuffers.ByteSegmentRef ref) throws HiveException {
if (ref.getLength() == 0) {
return EMPTY_LIST; // shortcut, 0 length means no fields
}
uselessIndirection.setData(ref.getBytes());
valueStruct.init(uselessIndirection, (int) ref.getOffset(), ref.getLength());
List<Object> result;
if (!needsComplexObjectFixup) {
// Good performance for common case where small table has no complex objects.
result = valueStruct.getFieldsAsList();
} else {
// Convert the complex LazyBinary objects to standard (Java) objects so downstream
// operators like FileSinkOperator can serialize complex objects in the form they expect
// (i.e. Java objects).
result = getComplexFieldsAsList(valueStruct, complexObjectArrayBuffer, internalValueOi);
}
return result;
}
/** Writes the debug dump of the table into logs. Not thread-safe. */
public void debugDumpTable() {
StringBuilder dump = new StringBuilder(keysAssigned + " keys\n");
TreeMap<Long, Integer> byteIntervals = new TreeMap<Long, Integer>();
int examined = 0;
for (int slot = 0; slot < refs.length; ++slot) {
long ref = refs[slot];
if (ref == 0) {
continue;
}
++examined;
long recOffset = getFirstRecordLengthsOffset(ref, null);
long tailOffset = Ref.getOffset(ref);
writeBuffers.setReadPoint(recOffset);
int valueLength = (int) writeBuffers.readVLong(), keyLength = (int) writeBuffers.readVLong();
long ptrOffset = writeBuffers.getReadPoint();
if (Ref.hasList(ref)) {
byteIntervals.put(recOffset, (int) (ptrOffset + 5 - recOffset));
}
long keyOffset = tailOffset - valueLength - keyLength;
byte[] key = new byte[keyLength];
WriteBuffers.ByteSegmentRef fakeRef = new WriteBuffers.ByteSegmentRef(keyOffset, keyLength);
byteIntervals.put(keyOffset - 4, keyLength + 4);
writeBuffers.populateValue(fakeRef);
System.arraycopy(fakeRef.getBytes(), (int) fakeRef.getOffset(), key, 0, keyLength);
dump.append(Utils.toStringBinary(key, 0, key.length))
.append(" ref [")
.append(dumpRef(ref))
.append("]: ");
Result hashMapResult = new Result();
getValueResult(key, 0, key.length, hashMapResult);
List<WriteBuffers.ByteSegmentRef> results = new ArrayList<WriteBuffers.ByteSegmentRef>();
WriteBuffers.ByteSegmentRef byteSegmentRef = hashMapResult.first();
while (byteSegmentRef != null) {
results.add(hashMapResult.byteSegmentRef);
byteSegmentRef = hashMapResult.next();
}
dump.append(results.size()).append(" rows\n");
for (int i = 0; i < results.size(); ++i) {
WriteBuffers.ByteSegmentRef segment = results.get(i);
byteIntervals.put(
segment.getOffset(),
segment.getLength() + ((i == 0) ? 1 : 0)); // state byte in the first record
}
}
if (examined != keysAssigned) {
dump.append("Found " + examined + " keys!\n");
}
// Report suspicious gaps in writeBuffers
long currentOffset = 0;
for (Map.Entry<Long, Integer> e : byteIntervals.entrySet()) {
long start = e.getKey(), len = e.getValue();
if (start - currentOffset > 4) {
dump.append("Gap! [" + currentOffset + ", " + start + ")\n");
}
currentOffset = start + len;
}
LOG.info("Hashtable dump:\n " + dump.toString());
}
private void debugDumpKeyProbe(long keyOffset, int keyLength, int hashCode, int finalSlot) {
final int bucketMask = refs.length - 1;
WriteBuffers.ByteSegmentRef fakeRef = new WriteBuffers.ByteSegmentRef(keyOffset, keyLength);
writeBuffers.populateValue(fakeRef);
int slot = hashCode & bucketMask;
long probeSlot = slot;
StringBuilder sb = new StringBuilder("Probe path debug for [");
sb.append(
Utils.toStringBinary(fakeRef.getBytes(), (int) fakeRef.getOffset(), fakeRef.getLength()));
sb.append("] hashCode ").append(Integer.toBinaryString(hashCode)).append(" is: ");
int i = 0;
while (slot != finalSlot) {
probeSlot += (++i);
slot = (int) (probeSlot & bucketMask);
sb.append(slot)
.append(" - ")
.append(probeSlot)
.append(" - ")
.append(Long.toBinaryString(refs[slot]))
.append("\n");
}
LOG.info(sb.toString());
}
/**
* Read the current value.
*
* @return The ByteSegmentRef to the current value read.
*/
private WriteBuffers.ByteSegmentRef internalRead() {
if (!hasList) {
/*
* Single value.
*/
if (readIndex > 0) {
return null;
}
// For a non-list (i.e. single value), the offset is for the variable length long (VLong)
// holding the value length (followed by the key length).
hashMap.writeBuffers.setReadPoint(firstOffset, readPos);
int valueLength = (int) hashMap.writeBuffers.readVLong(readPos);
// The value is before the offset. Make byte segment reference absolute.
byteSegmentRef.reset(firstOffset - valueLength, valueLength);
hashMap.writeBuffers.populateValue(byteSegmentRef);
readIndex++;
return byteSegmentRef;
}
/*
* Multiple values.
*/
if (readIndex == 0) {
// For a list, the value and key lengths of 1st record were overwritten with the
// relative offset to a new list record.
long relativeOffset = hashMap.writeBuffers.readNByteLong(firstOffset, 5, readPos);
// At the beginning of the list record will be the value length.
hashMap.writeBuffers.setReadPoint(firstOffset + relativeOffset, readPos);
int valueLength = (int) hashMap.writeBuffers.readVLong(readPos);
// The value is before the list record offset. Make byte segment reference absolute.
byteSegmentRef.reset(firstOffset - valueLength, valueLength);
hashMap.writeBuffers.populateValue(byteSegmentRef);
readIndex++;
return byteSegmentRef;
}
if (readIndex == 1) {
// We remembered the offset of just after the key length in the list record.
// Read the absolute offset to the 2nd value.
nextTailOffset =
hashMap.writeBuffers.readNByteLong(offsetAfterListRecordKeyLen, 5, readPos);
if (nextTailOffset <= 0) {
throw new Error("Expecting a second value");
}
} else if (nextTailOffset <= 0) {
return null;
}
hashMap.writeBuffers.setReadPoint(nextTailOffset, readPos);
// Get the value length.
int valueLength = (int) hashMap.writeBuffers.readVLong(readPos);
// Now read the relative offset to next record. Next record is always before the
// previous record in the write buffers (see writeBuffers javadoc).
long delta = hashMap.writeBuffers.readVLong(readPos);
long newTailOffset = delta == 0 ? 0 : (nextTailOffset - delta);
// The value is before the value record offset. Make byte segment reference absolute.
byteSegmentRef.reset(nextTailOffset - valueLength, valueLength);
hashMap.writeBuffers.populateValue(byteSegmentRef);
nextTailOffset = newTailOffset;
readIndex++;
return byteSegmentRef;
}