public byte[] toStream() throws OSerializationException {
    final long timer = OProfiler.getInstance().startChrono();

    try {
      final OMemoryStream outStream = new OMemoryStream();
      outStream.jump(0);
      outStream.set(pageSize);

      outStream.setAsFixed(parentRid.toStream());
      outStream.setAsFixed(leftRid.toStream());
      outStream.setAsFixed(rightRid.toStream());

      outStream.set(color);
      outStream.set(size);

      for (int i = 0; i < size; ++i) serializedKeys[i] = outStream.set(serializeNewKey(i));

      for (int i = 0; i < size; ++i) serializedValues[i] = outStream.set(serializeNewValue(i));

      final byte[] buffer = outStream.toByteArray();

      stream.setSource(buffer);
      record.fromStream(buffer);
      return buffer;

    } catch (IOException e) {
      throw new OSerializationException("Cannot marshall RB+Tree node", e);
    } finally {
      OProfiler.getInstance().stopChrono("OMVRBTreeMapEntry.toStream", timer);
    }
  }
  protected static boolean commitBuffer(final OMMapBufferEntry iEntry) {
    final long timer = OProfiler.getInstance().startChrono();

    // FORCE THE WRITE OF THE BUFFER
    boolean forceSucceed = false;
    for (int i = 0; i < FORCE_RETRY; ++i) {
      try {
        iEntry.buffer.force();
        forceSucceed = true;
        break;
      } catch (Exception e) {
        OLogManager.instance()
            .debug(
                iEntry,
                "Can't write memory buffer to disk. Retrying ("
                    + (i + 1)
                    + "/"
                    + FORCE_RETRY
                    + ")...");
        OMemoryWatchDog.freeMemory(FORCE_DELAY);
      }
    }

    if (!forceSucceed)
      OLogManager.instance()
          .debug(iEntry, "Can't commit memory buffer to disk after %d retries", FORCE_RETRY);
    else OProfiler.getInstance().updateCounter("OMMapManager.pagesCommitted", 1);

    OProfiler.getInstance().stopChrono("OMMapManager.commitPages", timer);

    return forceSucceed;
  }
Exemple #3
0
  /**
   * Acquire a network channel from the pool. Don't lock the write stream since the connection usage
   * is exclusive.
   *
   * @param iCommand
   * @return
   * @throws IOException
   */
  protected OChannelBinaryClient beginRequest(final byte iCommand) throws IOException {
    OChannelBinaryClient network = null;

    if (debug) System.out.println("-> req: " + getSessionId());

    // FIND THE FIRST FREE CHANNEL AVAILABLE
    synchronized (networkPool) {
      final int beginCursor = networkPoolCursor;

      while (network == null) {
        if (networkPool.size() == 0) throw new ONetworkProtocolException("Connection pool closed");

        network = networkPool.get(networkPoolCursor);
        if (network.getLockWrite().tryLock()) break;

        network = null;

        networkPoolCursor++;

        if (networkPoolCursor >= networkPool.size())
          // RESTART FROM THE FIRST ONE
          networkPoolCursor = 0;

        if (networkPoolCursor == beginCursor) {
          // COMPLETE ROUND AND NOT FREE CONNECTIONS FOUND

          if (networkPool.size() < maxPool) {
            // CREATE NEW CONNECTION
            network = createNetworkConnection();
            network.getLockWrite().lock();
            networkPool.add(network);

            if (debug) System.out.println("Created new connection " + networkPool.size());
          } else {
            if (debug) System.out.println("-> req (waiting) : " + getSessionId());

            final long startToWait = System.currentTimeMillis();
            try {
              networkPool.wait(5000);
              OProfiler.getInstance().updateCounter("network.connectionPool.timeout", +1);
            } catch (InterruptedException e) {
              break;
            }

            final long elapsed =
                OProfiler.getInstance()
                    .stopChrono("network.connectionPool.waitingTime", startToWait);

            if (debug) System.out.println("Waiting for connection = elapsed: " + elapsed);
          }
        }
      }
    }

    network.writeByte(iCommand);
    network.writeInt(getSessionId());

    return network;
  }
 private static OMMapBufferEntry mapBuffer(
     final OFileMMap iFile, final long iBeginOffset, final int iSize) throws IOException {
   long timer = OProfiler.getInstance().startChrono();
   try {
     return new OMMapBufferEntry(iFile, iFile.map(iBeginOffset, iSize), iBeginOffset, iSize);
   } finally {
     OProfiler.getInstance().stopChrono("OMMapManager.loadPage", timer);
   }
 }
  private static boolean allocIfOverlaps(
      final long iBeginOffset,
      final int iSize,
      final List<OMMapBufferEntry> fileEntries,
      final int p) {
    if (overlapStrategy == OVERLAP_STRATEGY.OVERLAP) return true;

    boolean overlaps = false;
    OMMapBufferEntry entry = null;
    if (p > 0) {
      // CHECK LOWER OFFSET
      entry = fileEntries.get(p - 1);
      overlaps =
          entry.beginOffset <= iBeginOffset && entry.beginOffset + entry.size >= iBeginOffset;
    }

    if (!overlaps && p < fileEntries.size() - 1) {
      // CHECK HIGHER OFFSET
      entry = fileEntries.get(p);
      overlaps = iBeginOffset + iSize >= entry.beginOffset;
    }

    if (overlaps) {
      // READ NOT IN BUFFER POOL: RETURN NULL TO LET TO THE CALLER TO EXECUTE A DIRECT READ WITHOUT
      // MMAP
      OProfiler.getInstance().updateCounter("OMMapManager.overlappedPageUsingChannel", 1);
      if (overlapStrategy == OVERLAP_STRATEGY.NO_OVERLAP_FLUSH_AND_USE_CHANNEL) commitBuffer(entry);
      return false;
    }

    return true;
  }
  @SuppressWarnings("unchecked")
  public V getValueAt(final int iIndex) {
    V v = values[iIndex];
    if (v == null)
      try {
        OProfiler.getInstance().updateCounter("OMVRBTreeMapEntry.unserializeValue", 1);

        v = (V) valueFromStream(iIndex);

        if (((OMVRBTreeMapProvider<K, V>) treeDataProvider).keepValuesInMemory)
          // KEEP THE UNMARSHALLED VALUE IN MEMORY
          values[iIndex] = v;

      } catch (IOException e) {

        OLogManager.instance()
            .error(
                this,
                "Cannot lazy load the value #" + iIndex + " in tree node " + this,
                e,
                OSerializationException.class);
      }

    return v;
  }
  @SuppressWarnings("unchecked")
  public K getKeyAt(final int iIndex) {
    K k = keys[iIndex];
    if (k == null)
      try {
        OProfiler.getInstance().updateCounter("OMVRBTreeMapEntry.unserializeKey", 1);

        k = (K) keyFromStream(iIndex);

        if (iIndex == 0
            || iIndex == size
            || ((OMVRBTreeMapProvider<K, V>) treeDataProvider).keepKeysInMemory)
          // KEEP THE UNMARSHALLED KEY IN MEMORY. TO OPTIMIZE FIRST AND LAST ITEM ARE ALWAYS KEPT IN
          // MEMORY TO SPEEDUP FREQUENT
          // NODE CHECKING OF BOUNDS
          keys[iIndex] = k;

      } catch (IOException e) {
        OLogManager.instance()
            .error(
                this,
                "Cannot lazy load the key #" + iIndex + " in tree node " + this,
                e,
                OSerializationException.class);
      }

    return k;
  }
  @SuppressWarnings("unchecked")
  public OSerializableStream fromStream(final byte[] iStream) throws OSerializationException {
    final long timer = OProfiler.getInstance().startChrono();

    if (stream == null) stream = new OMemoryStream(iStream);
    else stream.setSource(iStream);

    try {
      pageSize = stream.getAsInteger();

      parentRid = new ORecordId().fromStream(stream.getAsByteArrayFixed(ORecordId.PERSISTENT_SIZE));
      leftRid = new ORecordId().fromStream(stream.getAsByteArrayFixed(ORecordId.PERSISTENT_SIZE));
      rightRid = new ORecordId().fromStream(stream.getAsByteArrayFixed(ORecordId.PERSISTENT_SIZE));

      color = stream.getAsBoolean();
      size = stream.getAsInteger();

      if (size > pageSize)
        throw new OConfigurationException(
            "Loaded index with page size setted to "
                + pageSize
                + " while the loaded was built with: "
                + size);

      // UNCOMPACT KEYS SEPARATELY
      serializedKeys = new int[pageSize];
      for (int i = 0; i < size; ++i) {
        serializedKeys[i] = stream.getAsByteArrayOffset();
      }

      // KEYS WILL BE LOADED LAZY
      keys = (K[]) new Object[pageSize];

      // UNCOMPACT VALUES SEPARATELY
      serializedValues = new int[pageSize];
      for (int i = 0; i < size; ++i) {
        serializedValues[i] = stream.getAsByteArrayOffset();
      }

      // VALUES WILL BE LOADED LAZY
      values = (V[]) new Object[pageSize];

      return this;
    } finally {
      OProfiler.getInstance().stopChrono("OMVRBTreeMapEntry.fromStream", timer);
    }
  }
 /** Serialize only the new keys or the changed. */
 protected byte[] serializeNewKey(final int iIndex) throws IOException {
   if (serializedKeys[iIndex] <= 0) {
     // NEW OR MODIFIED: MARSHALL CONTENT
     OProfiler.getInstance().updateCounter("OMVRBTreeMapEntry.serializeValue", 1);
     return ((OMVRBTreeMapProvider<K, V>) treeDataProvider).keySerializer.toStream(keys[iIndex]);
   }
   // RETURN ORIGINAL CONTENT
   return stream.getAsByteArray(serializedKeys[iIndex]);
 }
  @Override
  public void init() {
    OProfiler.getInstance().startRecording();

    database = new ODatabaseDocumentTx(System.getProperty("url")).open("admin", "admin");
    record = database.newInstance();

    database.declareIntent(new OIntentMassiveInsert());
    database.begin(TXTYPE.NOTX);
  }
Exemple #11
0
  /**
   * Search for a buffer in the ordered list.
   *
   * @param fileEntries
   * @param iBeginOffset
   * @param iSize
   * @return negative number means not found. The position to insert is the (return value +1)*-1.
   *     Zero or positive number is the found position.
   */
  private static int searchEntry(
      final List<OMMapBufferEntry> fileEntries, final long iBeginOffset, final int iSize) {
    if (fileEntries == null || fileEntries.size() == 0) return -1;

    int high = fileEntries.size() - 1;
    if (high < 0)
      // NOT FOUND
      return -1;

    int low = 0;
    int mid = -1;

    // BINARY SEARCH
    OMMapBufferEntry e;

    while (low <= high) {
      mid = (low + high) >>> 1;
      e = fileEntries.get(mid);

      if (iBeginOffset >= e.beginOffset && iBeginOffset + iSize <= e.beginOffset + e.size) {
        // FOUND: USE IT
        OProfiler.getInstance().updateCounter("OMMapManager.reusedPage", 1);
        e.counter++;
        return mid;
      }

      if (low == high) {
        if (iBeginOffset > e.beginOffset)
          // NEXT POSITION
          low++;

        // NOT FOUND
        return (low + 2) * -1;
      }

      if (iBeginOffset >= e.beginOffset) low = mid + 1;
      else high = mid;
    }

    // NOT FOUND
    return mid;
  }
Exemple #12
0
  private static OMMapBufferEntry searchBetweenLastBlocks(
      final OFileMMap iFile, final long iBeginOffset, final int iSize) {
    if (bufferPoolLRU.size() > 0) {
      // SEARCH IF IT'S BETWEEN THE LAST 5 BLOCK USED: THIS IS THE COMMON CASE ON MASSIVE INSERTION
      final int min = Math.max(bufferPoolLRU.size() - 5, -1);
      for (int i = bufferPoolLRU.size() - 1; i > min; --i) {
        final OMMapBufferEntry e = bufferPoolLRU.get(i);

        if (e.file == iFile
            && iBeginOffset >= e.beginOffset
            && iBeginOffset + iSize <= e.beginOffset + e.size) {
          // FOUND: USE IT
          OProfiler.getInstance().updateCounter("OMMapManager.reusedPageBetweenLast", 1);
          e.counter++;
          return e;
        }
      }
    }
    return null;
  }
  /**
   * Register statistic information about usage of index in {@link OProfilerStub}.
   *
   * @param index which usage is registering.
   */
  private void updateStatistic(OIndex<?> index) {

    final OProfiler profiler = Orient.instance().getProfiler();
    if (profiler.isRecording()) {
      Orient.instance()
          .getProfiler()
          .updateCounter(
              profiler.getDatabaseMetric(index.getDatabaseName(), "query.indexUsed"),
              "Used index in query",
              +1);

      final int paramCount = index.getDefinition().getParamCount();
      if (paramCount > 1) {
        final String profiler_prefix =
            profiler.getDatabaseMetric(index.getDatabaseName(), "query.compositeIndexUsed");
        profiler.updateCounter(profiler_prefix, "Used composite index in query", +1);
        profiler.updateCounter(
            profiler_prefix + "." + paramCount,
            "Used composite index in query with " + paramCount + " params",
            +1);
      }
    }
  }
Exemple #14
0
  static {
    blockSize = OGlobalConfiguration.FILE_MMAP_BLOCK_SIZE.getValueAsInteger();
    FORCE_DELAY = OGlobalConfiguration.FILE_MMAP_FORCE_DELAY.getValueAsInteger();
    FORCE_RETRY = OGlobalConfiguration.FILE_MMAP_FORCE_RETRY.getValueAsInteger();
    maxMemory = OGlobalConfiguration.FILE_MMAP_MAX_MEMORY.getValueAsLong();
    setOverlapStrategy(OGlobalConfiguration.FILE_MMAP_OVERLAP_STRATEGY.getValueAsInteger());

    OProfiler.getInstance()
        .registerHookValue(
            "mmap.totalMemory",
            new OProfilerHookValue() {
              public Object getValue() {
                return totalMemory;
              }
            });

    OProfiler.getInstance()
        .registerHookValue(
            "mmap.maxMemory",
            new OProfilerHookValue() {
              public Object getValue() {
                return maxMemory;
              }
            });

    OProfiler.getInstance()
        .registerHookValue(
            "mmap.blockSize",
            new OProfilerHookValue() {
              public Object getValue() {
                return blockSize;
              }
            });

    OProfiler.getInstance()
        .registerHookValue(
            "mmap.blocks",
            new OProfilerHookValue() {
              public synchronized Object getValue() {
                return bufferPoolLRU.size();
              }
            });

    OProfiler.getInstance()
        .registerHookValue(
            "mmap.alloc.strategy",
            new OProfilerHookValue() {
              public Object getValue() {
                return lastStrategy;
              }
            });

    OProfiler.getInstance()
        .registerHookValue(
            "mmap.overlap.strategy",
            new OProfilerHookValue() {
              public Object getValue() {
                return overlapStrategy;
              }
            });
  }
Exemple #15
0
  /**
   * Requests a mmap buffer to use.
   *
   * @param iFile MMap file
   * @param iBeginOffset Begin offset
   * @param iSize Portion size requested
   * @param iForce Tells if the size is mandatory or can be rounded to the next segment
   * @param iOperationType READ or WRITE
   * @param iStrategy
   * @return The mmap buffer entry if found, or null if the operation is READ and the buffer pool is
   *     full.
   */
  public static synchronized OMMapBufferEntry request(
      final OFileMMap iFile,
      final long iBeginOffset,
      final int iSize,
      final boolean iForce,
      final OPERATION_TYPE iOperationType,
      final ALLOC_STRATEGY iStrategy) {

    if (iStrategy == ALLOC_STRATEGY.MMAP_NEVER) return null;

    lastStrategy = iStrategy;

    OMMapBufferEntry entry = searchBetweenLastBlocks(iFile, iBeginOffset, iSize);
    if (entry != null) return entry;

    // SEARCH THE REQUESTED RANGE IN THE CACHED BUFFERS
    List<OMMapBufferEntry> fileEntries = bufferPoolPerFile.get(iFile);
    if (fileEntries == null) {
      fileEntries = new ArrayList<OMMapBufferEntry>();
      bufferPoolPerFile.put(iFile, fileEntries);
    }

    int position = searchEntry(fileEntries, iBeginOffset, iSize);
    if (position > -1)
      // FOUND !!!
      return fileEntries.get(position);

    int p = (position + 2) * -1;

    // CHECK IF THERE IS A BUFFER THAT OVERLAPS
    if (!allocIfOverlaps(iBeginOffset, iSize, fileEntries, p)) {
      OProfiler.getInstance().updateCounter("OMMapManager.usedChannel", 1);
      return null;
    }

    int bufferSize = computeBestEntrySize(iFile, iBeginOffset, iSize, iForce, fileEntries, p);

    if (totalMemory + bufferSize > maxMemory
        && (iStrategy == ALLOC_STRATEGY.MMAP_ONLY_AVAIL_POOL
            || iOperationType == OPERATION_TYPE.READ
                && iStrategy == ALLOC_STRATEGY.MMAP_WRITE_ALWAYS_READ_IF_AVAIL_POOL)) {
      OProfiler.getInstance().updateCounter("OMMapManager.usedChannel", 1);
      return null;
    }

    entry = null;
    // FREE LESS-USED BUFFERS UNTIL THE FREE-MEMORY IS DOWN THE CONFIGURED MAX LIMIT
    do {
      if (totalMemory + bufferSize > maxMemory) freeResources();

      // RECOMPUTE THE POSITION AFTER REMOVING
      fileEntries = bufferPoolPerFile.get(iFile);
      position = searchEntry(fileEntries, iBeginOffset, iSize);
      if (position > -1)
        // FOUND: THIS IS PRETTY STRANGE SINCE IT WASN'T FOUND!
        return fileEntries.get(position);

      // LOAD THE PAGE
      try {
        entry = mapBuffer(iFile, iBeginOffset, bufferSize);
      } catch (IllegalArgumentException e) {
        throw e;
      } catch (Exception e) {
        // REDUCE MAX MEMORY TO FORCE EMPTY BUFFERS
        maxMemory = maxMemory * 90 / 100;
        OLogManager.instance()
            .warn(
                OMMapManager.class,
                "Memory mapping error, try to reduce max memory to %d and retry...",
                e,
                maxMemory);
      }
    } while (entry == null && maxMemory > MIN_MEMORY);

    if (entry == null)
      throw new OIOException(
          "You can't access to the file portion "
              + iBeginOffset
              + "-"
              + iBeginOffset
              + iSize
              + " bytes");

    totalMemory += bufferSize;
    bufferPoolLRU.add(entry);

    p = (position + 2) * -1;
    if (p < 0) p = 0;

    if (fileEntries == null) {
      // IN CASE THE CLEAN HAS REMOVED THE LIST
      fileEntries = new ArrayList<OMMapBufferEntry>();
      bufferPoolPerFile.put(iFile, fileEntries);
    }

    fileEntries.add(p, entry);

    return entry;
  }