/** Lets go of any references to a hash map. */
 public void forget() {
   hashMap = null;
   readPos = null;
   byteSegmentRef.reset(0, 0);
   hasRows = false;
   readIndex = 0;
   nextTailOffset = -1;
 }
예제 #2
0
 private List<Object> unpack(WriteBuffers.ByteSegmentRef ref) throws HiveException {
   if (ref.getLength() == 0) {
     return EMPTY_LIST; // shortcut, 0 length means no fields
   }
   uselessIndirection.setData(ref.getBytes());
   valueStruct.init(uselessIndirection, (int) ref.getOffset(), ref.getLength());
   List<Object> result;
   if (!needsComplexObjectFixup) {
     // Good performance for common case where small table has no complex objects.
     result = valueStruct.getFieldsAsList();
   } else {
     // Convert the complex LazyBinary objects to standard (Java) objects so downstream
     // operators like FileSinkOperator can serialize complex objects in the form they expect
     // (i.e. Java objects).
     result = getComplexFieldsAsList(valueStruct, complexObjectArrayBuffer, internalValueOi);
   }
   return result;
 }
 /** Writes the debug dump of the table into logs. Not thread-safe. */
 public void debugDumpTable() {
   StringBuilder dump = new StringBuilder(keysAssigned + " keys\n");
   TreeMap<Long, Integer> byteIntervals = new TreeMap<Long, Integer>();
   int examined = 0;
   for (int slot = 0; slot < refs.length; ++slot) {
     long ref = refs[slot];
     if (ref == 0) {
       continue;
     }
     ++examined;
     long recOffset = getFirstRecordLengthsOffset(ref, null);
     long tailOffset = Ref.getOffset(ref);
     writeBuffers.setReadPoint(recOffset);
     int valueLength = (int) writeBuffers.readVLong(), keyLength = (int) writeBuffers.readVLong();
     long ptrOffset = writeBuffers.getReadPoint();
     if (Ref.hasList(ref)) {
       byteIntervals.put(recOffset, (int) (ptrOffset + 5 - recOffset));
     }
     long keyOffset = tailOffset - valueLength - keyLength;
     byte[] key = new byte[keyLength];
     WriteBuffers.ByteSegmentRef fakeRef = new WriteBuffers.ByteSegmentRef(keyOffset, keyLength);
     byteIntervals.put(keyOffset - 4, keyLength + 4);
     writeBuffers.populateValue(fakeRef);
     System.arraycopy(fakeRef.getBytes(), (int) fakeRef.getOffset(), key, 0, keyLength);
     dump.append(Utils.toStringBinary(key, 0, key.length))
         .append(" ref [")
         .append(dumpRef(ref))
         .append("]: ");
     Result hashMapResult = new Result();
     getValueResult(key, 0, key.length, hashMapResult);
     List<WriteBuffers.ByteSegmentRef> results = new ArrayList<WriteBuffers.ByteSegmentRef>();
     WriteBuffers.ByteSegmentRef byteSegmentRef = hashMapResult.first();
     while (byteSegmentRef != null) {
       results.add(hashMapResult.byteSegmentRef);
       byteSegmentRef = hashMapResult.next();
     }
     dump.append(results.size()).append(" rows\n");
     for (int i = 0; i < results.size(); ++i) {
       WriteBuffers.ByteSegmentRef segment = results.get(i);
       byteIntervals.put(
           segment.getOffset(),
           segment.getLength() + ((i == 0) ? 1 : 0)); // state byte in the first record
     }
   }
   if (examined != keysAssigned) {
     dump.append("Found " + examined + " keys!\n");
   }
   // Report suspicious gaps in writeBuffers
   long currentOffset = 0;
   for (Map.Entry<Long, Integer> e : byteIntervals.entrySet()) {
     long start = e.getKey(), len = e.getValue();
     if (start - currentOffset > 4) {
       dump.append("Gap! [" + currentOffset + ", " + start + ")\n");
     }
     currentOffset = start + len;
   }
   LOG.info("Hashtable dump:\n " + dump.toString());
 }
 private void debugDumpKeyProbe(long keyOffset, int keyLength, int hashCode, int finalSlot) {
   final int bucketMask = refs.length - 1;
   WriteBuffers.ByteSegmentRef fakeRef = new WriteBuffers.ByteSegmentRef(keyOffset, keyLength);
   writeBuffers.populateValue(fakeRef);
   int slot = hashCode & bucketMask;
   long probeSlot = slot;
   StringBuilder sb = new StringBuilder("Probe path debug for [");
   sb.append(
       Utils.toStringBinary(fakeRef.getBytes(), (int) fakeRef.getOffset(), fakeRef.getLength()));
   sb.append("] hashCode ").append(Integer.toBinaryString(hashCode)).append(" is: ");
   int i = 0;
   while (slot != finalSlot) {
     probeSlot += (++i);
     slot = (int) (probeSlot & bucketMask);
     sb.append(slot)
         .append(" - ")
         .append(probeSlot)
         .append(" - ")
         .append(Long.toBinaryString(refs[slot]))
         .append("\n");
   }
   LOG.info(sb.toString());
 }
    /**
     * Read the current value.
     *
     * @return The ByteSegmentRef to the current value read.
     */
    private WriteBuffers.ByteSegmentRef internalRead() {

      if (!hasList) {

        /*
         * Single value.
         */

        if (readIndex > 0) {
          return null;
        }

        // For a non-list (i.e. single value), the offset is for the variable length long (VLong)
        // holding the value length (followed by the key length).
        hashMap.writeBuffers.setReadPoint(firstOffset, readPos);
        int valueLength = (int) hashMap.writeBuffers.readVLong(readPos);

        // The value is before the offset.  Make byte segment reference absolute.
        byteSegmentRef.reset(firstOffset - valueLength, valueLength);
        hashMap.writeBuffers.populateValue(byteSegmentRef);

        readIndex++;
        return byteSegmentRef;
      }

      /*
       * Multiple values.
       */

      if (readIndex == 0) {
        // For a list, the value and key lengths of 1st record were overwritten with the
        // relative offset to a new list record.
        long relativeOffset = hashMap.writeBuffers.readNByteLong(firstOffset, 5, readPos);

        // At the beginning of the list record will be the value length.
        hashMap.writeBuffers.setReadPoint(firstOffset + relativeOffset, readPos);
        int valueLength = (int) hashMap.writeBuffers.readVLong(readPos);

        // The value is before the list record offset.  Make byte segment reference absolute.
        byteSegmentRef.reset(firstOffset - valueLength, valueLength);
        hashMap.writeBuffers.populateValue(byteSegmentRef);

        readIndex++;
        return byteSegmentRef;
      }

      if (readIndex == 1) {
        // We remembered the offset of just after the key length in the list record.
        // Read the absolute offset to the 2nd value.
        nextTailOffset =
            hashMap.writeBuffers.readNByteLong(offsetAfterListRecordKeyLen, 5, readPos);
        if (nextTailOffset <= 0) {
          throw new Error("Expecting a second value");
        }
      } else if (nextTailOffset <= 0) {
        return null;
      }

      hashMap.writeBuffers.setReadPoint(nextTailOffset, readPos);

      // Get the value length.
      int valueLength = (int) hashMap.writeBuffers.readVLong(readPos);

      // Now read the relative offset to next record. Next record is always before the
      // previous record in the write buffers (see writeBuffers javadoc).
      long delta = hashMap.writeBuffers.readVLong(readPos);
      long newTailOffset = delta == 0 ? 0 : (nextTailOffset - delta);

      // The value is before the value record offset.  Make byte segment reference absolute.
      byteSegmentRef.reset(nextTailOffset - valueLength, valueLength);
      hashMap.writeBuffers.populateValue(byteSegmentRef);

      nextTailOffset = newTailOffset;
      readIndex++;
      return byteSegmentRef;
    }