コード例 #1
0
 private DirectBytes getKeyAsBytes(K key) {
   DirectBytes bytes = acquireBytes();
   if (generatedKeyType) ((BytesMarshallable) key).writeMarshallable(bytes);
   else bytes.writeInstance(kClass, key);
   bytes.flip();
   return bytes;
 }
コード例 #2
0
 DirectBytes acquireBytes() {
   DirectBytes bytes = localBytes.get();
   if (bytes == null) {
     localBytes.set(
         bytes = new DirectStore(ms.bytesMarshallerFactory(), entrySize * 2, false).createSlice());
   } else {
     bytes.clear();
   }
   return bytes;
 }
コード例 #3
0
 private long longHashCode(DirectBytes bytes) {
   long h = 0;
   int i = 0;
   long limit = bytes.limit(); // clustering.
   for (; i < limit - 7; i += 8) h = 10191 * h + bytes.readLong(i);
   //        for (; i < bytes.limit() - 3; i += 2)
   //            h = 10191 * h + bytes.readInt(i);
   for (; i < limit; i++) h = 57 * h + bytes.readByte(i);
   h ^= (h >>> 31) + (h << 31);
   h += (h >>> 21) + (h >>> 11);
   return h;
 }
コード例 #4
0
 private void appendInstance(DirectBytes bytes, V value) {
   bytes.clear();
   if (builder.generatedValueType()) ((BytesMarshallable) value).writeMarshallable(bytes);
   else bytes.writeInstance(vClass, value);
   bytes.flip();
   if (bytes.remaining() > tmpBytes.remaining())
     throw new IllegalArgumentException(
         "Value too large for entry was "
             + bytes.remaining()
             + ", remaining: "
             + tmpBytes.remaining());
   tmpBytes.write(bytes);
 }
コード例 #5
0
    public V acquire(DirectBytes keyBytes, V value, int hash2, boolean create) {
      if (hash2 == hashLookup.unsetKey()) hash2 = ~hash2;
      lock();
      try {
        hashLookup.startSearch(hash2);
        while (true) {
          int pos = hashLookup.nextInt();
          if (pos == hashLookup.unsetValue()) {
            return create ? acquireEntry(keyBytes, value, hash2) : null;

          } else {
            long offset = entriesOffset + pos * builder.entrySize();
            tmpBytes.storePositionAndSize(bytes, offset, builder.entrySize());
            long start0 = System.nanoTime();
            boolean miss = !keyEquals(keyBytes, tmpBytes);
            long time0 = System.nanoTime() - start0;
            if (time0 > 1e6)
              System.out.println("startsWith took " + time0 / 100000 / 10.0 + " ms.");
            if (miss) continue;
            long keyLength =
                align(keyBytes.remaining() + tmpBytes.position()); // includes the stop bit length.
            tmpBytes.position(keyLength);
            return readObjectUsing(value, offset + keyLength);
          }
        }
      } finally {
        unlock();
      }
    }
コード例 #6
0
    public V remove(DirectBytes keyBytes, V value, int hash2) {
      if (hash2 == hashLookup.unsetKey()) hash2 = ~hash2;
      lock();
      try {
        hashLookup.startSearch(hash2);
        while (true) {
          int pos = hashLookup.nextInt();
          if (pos == hashLookup.unsetValue()) {
            return null;

          } else {
            long offset = entriesOffset + pos * builder.entrySize();
            tmpBytes.storePositionAndSize(bytes, offset, builder.entrySize());
            if (!keyEquals(keyBytes, tmpBytes)) continue;
            long keyLength = align(keyBytes.remaining());
            tmpBytes.skip(keyLength);
            V v =
                value == null && builder.removeReturnsNull()
                    ? null
                    : readObjectUsing(value, offset + keyLength);
            hashLookup.remove(hash2, pos);
            freeList.clear(pos);
            if (pos < nextSet) nextSet = pos;
            return v;
          }
        }
      } finally {
        unlock();
      }
    }
コード例 #7
0
    /**
     * @param keyBytes
     * @param value
     * @param hash2
     * @return
     */
    V acquireEntry(DirectBytes keyBytes, V value, int hash2) {
      value = createValueIfNull(value);

      final int pos = nextFree();
      final long offset = entriesOffset + pos * entrySize;
      tmpBytes.storePositionAndSize(bytes, offset, entrySize);
      final long keyLength = keyBytes.remaining();
      tmpBytes.writeStopBit(keyLength);
      tmpBytes.write(keyBytes);
      if (value instanceof Byteable) {
        Byteable byteable = (Byteable) value;
        int length = byteable.maxSize();
        tmpBytes.writeStopBit(length);
        tmpBytes.position(align(tmpBytes.position()));
        if (length > tmpBytes.remaining())
          throw new IllegalStateException(
              "Not enough space left in entry for value, needs "
                  + length
                  + " but only "
                  + tmpBytes.remaining()
                  + " left");
        tmpBytes.zeroOut(tmpBytes.position(), tmpBytes.position() + length);
        byteable.bytes(bytes, offset + tmpBytes.position());
      } else {
        appendInstance(keyBytes, value);
      }
      // add to index if successful.
      hashLookup.put(hash2, pos);
      incrementSize();
      return value;
    }
コード例 #8
0
 private void putEntry(DirectBytes keyBytes, V value, int hash2) {
   int pos = nextFree();
   long offset = entriesOffset + pos * builder.entrySize();
   tmpBytes.storePositionAndSize(bytes, offset, builder.entrySize());
   long keyLength = keyBytes.remaining();
   tmpBytes.writeStopBit(keyLength);
   tmpBytes.write(keyBytes);
   tmpBytes.position(align(tmpBytes.position()));
   appendInstance(keyBytes, value);
   // add to index if successful.
   hashLookup.put(hash2, pos);
 }
コード例 #9
0
 private V acquireEntry(DirectBytes keyBytes, V value, int hash2) {
   int pos = nextFree();
   long offset = entriesOffset + pos * builder.entrySize();
   tmpBytes.storePositionAndSize(bytes, offset, builder.entrySize());
   long keyLength = keyBytes.remaining();
   tmpBytes.writeStopBit(keyLength);
   tmpBytes.write(keyBytes);
   tmpBytes.position(align(tmpBytes.position()));
   tmpBytes.zeroOut(tmpBytes.position(), tmpBytes.limit());
   V v = readObjectUsing(value, offset + tmpBytes.position());
   // add to index if successful.
   hashLookup.put(hash2, pos);
   return v;
 }
コード例 #10
0
    /**
     * implementation for map.put(Key,Value)
     *
     * @param keyBytes
     * @param value
     * @param hash2 a hash code relating to the {@keyBytes} ( not the natural hash of {@keyBytes} )
     * @param replaceIfPresent
     * @return
     */
    V put(final DirectBytes keyBytes, final V value, int hash2, boolean replaceIfPresent) {
      lock();
      try {
        hash2 = hashLookup.startSearch(hash2);
        while (true) {
          final int pos = hashLookup.nextPos();
          if (pos < 0) {
            putEntry(keyBytes, value, hash2);

            return null;

          } else {
            final long offset = entriesOffset + pos * entrySize;
            tmpBytes.storePositionAndSize(bytes, offset, entrySize);
            if (!keyEquals(keyBytes, tmpBytes)) continue;
            final long keyLength = keyBytes.remaining();
            tmpBytes.skip(keyLength);
            if (replaceIfPresent) {
              if (putReturnsNull) {
                appendInstance(keyBytes, value);
                return null;
              }
              long valuePosition = tmpBytes.position();
              tmpBytes.readStopBit();
              final long alignPosition = align(tmpBytes.position());
              tmpBytes.position(alignPosition);
              final V v = readObjectUsing(null, offset + alignPosition);
              tmpBytes.position(valuePosition);
              appendInstance(keyBytes, value);
              return v;

            } else {
              if (putReturnsNull) {
                return null;
              }

              return readObjectUsing(null, offset + keyLength);
            }
          }
        }
      } finally {
        unlock();
      }
    }
コード例 #11
0
    /**
     * implementation for map.replace(Key,Value) and map.replace(Key,Old,New)
     *
     * @param keyBytes the key of the entry to be replaced
     * @param expectedValue the expected value to replaced
     * @param newValue the new value that will only be set if the existing value in the map equals
     *     the {@param expectedValue} or {@param expectedValue} is null
     * @param hash2 a hash code relating to the {@keyBytes} ( not the natural hash of {@keyBytes} )
     * @return null if the value was not replaced, else the value that is replaced is returned
     */
    V replace(
        final DirectBytes keyBytes, final V expectedValue, final V newValue, final int hash2) {
      lock();
      try {

        hashLookup.startSearch(hash2);
        while (true) {

          final int pos = hashLookup.nextPos();

          if (pos < 0) {
            return null;

          } else {

            final long offset = entriesOffset + pos * entrySize;
            tmpBytes.storePositionAndSize(bytes, offset, entrySize);

            if (!keyEquals(keyBytes, tmpBytes)) continue;

            final long keyLength = keyBytes.remaining();
            tmpBytes.skip(keyLength);
            long valuePosition = tmpBytes.position();
            tmpBytes.readStopBit();
            final long alignPosition = align(tmpBytes.position());
            tmpBytes.position(alignPosition);

            final V valueRead = readObjectUsing(null, offset + keyLength);

            if (valueRead == null) return null;

            if (expectedValue == null || expectedValue.equals(valueRead)) {
              tmpBytes.position(valuePosition);
              appendInstance(keyBytes, newValue);
            }

            return valueRead;
          }
        }
      } finally {
        unlock();
      }
    }
コード例 #12
0
 Segment(
     HugeConfig config,
     Hasher hasher,
     boolean csKey,
     boolean bytesMarshallable,
     Class<V> vClass) {
   this.csKey = csKey;
   this.hasher = hasher;
   this.bytesMarshallable = bytesMarshallable;
   this.vClass = vClass;
   smallEntrySize = (config.getSmallEntrySize() + 7) & ~7; // round to next multiple of 8.
   entriesPerSegment = config.getEntriesPerSegment();
   store = new DirectStore(bmf, smallEntrySize * entriesPerSegment, false);
   usedSet = new BitSet(config.getEntriesPerSegment());
   smallMap = new IntIntMultiMap(entriesPerSegment * 2);
   tmpBytes = new DirectStore(bmf, 64 * smallEntrySize, false).createSlice();
   offHeapUsed = tmpBytes.capacity() + store.size();
   sbKey = csKey ? new StringBuilder() : null;
 }
コード例 #13
0
    public V put(DirectBytes keyBytes, V value, int hash2, boolean replaceIfPresent) {
      if (hash2 == hashLookup.unsetKey()) hash2 = ~hash2;
      lock();
      try {
        hashLookup.startSearch(hash2);
        while (true) {
          int pos = hashLookup.nextInt();
          if (pos == hashLookup.unsetValue()) {
            putEntry(keyBytes, value, hash2);
            return null;

          } else {
            long offset = entriesOffset + pos * builder.entrySize();
            tmpBytes.storePositionAndSize(bytes, offset, builder.entrySize());
            if (!keyEquals(keyBytes, tmpBytes)) continue;
            long keyLength = keyBytes.remaining();
            tmpBytes.skip(keyLength);
            long alignPosition = align(tmpBytes.position());
            tmpBytes.position(alignPosition);
            if (replaceIfPresent) {
              if (builder.putReturnsNull()) {
                appendInstance(keyBytes, value);
                return null;
              }
              V v = readObjectUsing(null, offset + alignPosition);
              tmpBytes.position(alignPosition);
              appendInstance(keyBytes, value);
              return v;

            } else {
              if (builder.putReturnsNull()) {
                return null;
              }
              return readObjectUsing(null, offset + keyLength);
            }
          }
        }
      } finally {
        unlock();
      }
    }
コード例 #14
0
    /**
     * implementation for map.remove(Key,Value)
     *
     * @param keyBytes the key of the entry to remove
     * @param expectedValue the entry will only be removed if the {@param existingValue} equals null
     *     or the {@param existingValue} equals that of the entry.value
     * @param hash2 a hash code relating to the {@keyBytes} ( not the natural hash of {@keyBytes} )
     * @return
     */
    V remove(final DirectBytes keyBytes, final V expectedValue, int hash2) {
      lock();
      try {
        hash2 = hashLookup.startSearch(hash2);
        while (true) {

          final int pos = hashLookup.nextPos();
          if (pos < 0) {
            return null;

          } else {
            final long offset = entriesOffset + pos * entrySize;
            tmpBytes.storePositionAndSize(bytes, offset, entrySize);
            if (!keyEquals(keyBytes, tmpBytes)) continue;
            final long keyLength =
                align(keyBytes.remaining() + tmpBytes.position()); // includes the stop bit length.
            tmpBytes.position(keyLength);
            V valueRemoved =
                expectedValue == null && removeReturnsNull
                    ? null
                    : readObjectUsing(null, offset + keyLength);

            if (expectedValue != null && !expectedValue.equals(valueRemoved)) return null;

            hashLookup.remove(hash2, pos);
            decrementSize();

            freeList.clear(pos);
            if (pos < nextSet) nextSet = pos;

            return valueRemoved;
          }
        }
      } finally {
        unlock();
      }
    }
コード例 #15
0
    /**
     * used to acquire and object of type V from the map,
     *
     * <p>when {@param create }== true, this method is equivalent to :
     *
     * <pre>
     * Object value = map.get("Key");
     *
     * if ( counter == null ) {
     *    value = new Object();
     *    map.put("Key", value);
     * }
     *
     * return value;
     * </pre>
     *
     * @param keyBytes the key of the entry
     * @param value an object to be reused, null creates a new object.
     * @param hash2 a hash code relating to the {@keyBytes} ( not the natural hash of {@keyBytes} )
     * @param create false - if the {@keyBytes} can not be found null will be returned, true - if
     *     the {@keyBytes} can not be found an value will be acquired
     * @return an entry.value whose entry.key equals {@param keyBytes}
     */
    V acquire(DirectBytes keyBytes, V value, int hash2, boolean create) {
      lock();
      try {
        hash2 = hashLookup.startSearch(hash2);
        while (true) {
          int pos = hashLookup.nextPos();
          if (pos < 0) {
            return create ? acquireEntry(keyBytes, value, hash2) : null;

          } else {
            final long offset = entriesOffset + pos * entrySize;
            tmpBytes.storePositionAndSize(bytes, offset, entrySize);
            final boolean miss;
            if (LOGGER.isLoggable(Level.FINE)) {
              final long start0 = System.nanoTime();
              miss = !keyEquals(keyBytes, tmpBytes);
              final long time0 = System.nanoTime() - start0;
              if (time0 > 1e6) LOGGER.fine("startsWith took " + time0 / 100000 / 10.0 + " ms.");
            } else {
              miss = !keyEquals(keyBytes, tmpBytes);
            }
            if (miss) continue;
            long valueLengthOffset = keyBytes.remaining() + tmpBytes.position();
            tmpBytes.position(valueLengthOffset);
            // skip the value length
            // todo use the value length to limit reading below
            long valueLength = tmpBytes.readStopBit();
            final long valueOffset = align(tmpBytes.position()); // includes the stop bit length.
            tmpBytes.position(valueOffset);
            return readObjectUsing(value, offset + valueOffset);
          }
        }
      } finally {
        unlock();
      }
    }
コード例 #16
0
 private boolean keyEquals(DirectBytes keyBytes, MultiStoreBytes tmpBytes) {
   // check the length is the same.
   long keyLength = tmpBytes.readStopBit();
   return keyLength == keyBytes.remaining() && tmpBytes.startsWith(keyBytes);
 }
コード例 #17
0
    synchronized void put(long hash, K key, V value, boolean ifPresent, boolean ifAbsent) {
      // search for the previous entry
      int h = smallMap.startSearch(hash);
      boolean foundSmall = false, foundLarge = false;
      while (true) {
        int pos = smallMap.nextPos();
        if (pos < 0) {
          K key2 = key instanceof CharSequence ? (K) key.toString() : key;
          final DirectStore store = map.get(key2);
          if (store == null) {
            if (ifPresent && !ifAbsent) return;
            break;
          }
          if (ifAbsent) return;
          bytes.storePositionAndSize(store, 0, store.size());
          foundLarge = true;
          break;
        } else {
          bytes.storePositionAndSize(store, pos * smallEntrySize, smallEntrySize);
          K key2 = getKey();
          if (equals(key, key2)) {
            if (ifAbsent && !ifPresent) return;
            foundSmall = true;
            break;
          }
        }
      }

      tmpBytes.clear();
      if (csKey)
        //noinspection ConstantConditions
        tmpBytes.writeUTFΔ((CharSequence) key);
      else tmpBytes.writeObject(key);
      long startOfValuePos = tmpBytes.position();
      if (bytesMarshallable) ((BytesMarshallable) value).writeMarshallable(tmpBytes);
      else tmpBytes.writeObject(value);
      long size = tmpBytes.position();
      if (size <= smallEntrySize) {
        if (foundSmall) {
          bytes.position(0);
          bytes.write(tmpBytes, 0, size);
          return;
        } else if (foundLarge) {
          remove(hash, key);
        }
        // look for a free spot.
        int position = h & (entriesPerSegment - 1);
        int free = usedSet.nextClearBit(position);
        if (free >= entriesPerSegment) free = usedSet.nextClearBit(0);
        if (free < entriesPerSegment) {
          bytes.storePositionAndSize(store, free * smallEntrySize, smallEntrySize);
          bytes.write(tmpBytes, 0, size);
          smallMap.put(h, free);
          usedSet.set(free);
          this.size++;
          return;
        }
      }
      if (foundSmall) {
        remove(hash, key);
      } else if (foundLarge) {
        // can it be reused.
        if (bytes.capacity() <= size || bytes.capacity() - size < (size >> 3)) {
          bytes.write(tmpBytes, startOfValuePos, size);
          return;
        }
        remove(hash, key);
      }
      size = size - startOfValuePos;
      DirectStore store = new DirectStore(bmf, size);
      bytes.storePositionAndSize(store, 0, size);
      bytes.write(tmpBytes, startOfValuePos, size);
      K key2 = key instanceof CharSequence ? (K) key.toString() : key;
      map.put(key2, store);
      offHeapUsed += size;
      this.size++;
    }