public byte[] toStream() throws OSerializationException { final long timer = OProfiler.getInstance().startChrono(); try { final OMemoryStream outStream = new OMemoryStream(); outStream.jump(0); outStream.set(pageSize); outStream.setAsFixed(parentRid.toStream()); outStream.setAsFixed(leftRid.toStream()); outStream.setAsFixed(rightRid.toStream()); outStream.set(color); outStream.set(size); for (int i = 0; i < size; ++i) serializedKeys[i] = outStream.set(serializeNewKey(i)); for (int i = 0; i < size; ++i) serializedValues[i] = outStream.set(serializeNewValue(i)); final byte[] buffer = outStream.toByteArray(); stream.setSource(buffer); record.fromStream(buffer); return buffer; } catch (IOException e) { throw new OSerializationException("Cannot marshall RB+Tree node", e); } finally { OProfiler.getInstance().stopChrono("OMVRBTreeMapEntry.toStream", timer); } }
protected static boolean commitBuffer(final OMMapBufferEntry iEntry) { final long timer = OProfiler.getInstance().startChrono(); // FORCE THE WRITE OF THE BUFFER boolean forceSucceed = false; for (int i = 0; i < FORCE_RETRY; ++i) { try { iEntry.buffer.force(); forceSucceed = true; break; } catch (Exception e) { OLogManager.instance() .debug( iEntry, "Can't write memory buffer to disk. Retrying (" + (i + 1) + "/" + FORCE_RETRY + ")..."); OMemoryWatchDog.freeMemory(FORCE_DELAY); } } if (!forceSucceed) OLogManager.instance() .debug(iEntry, "Can't commit memory buffer to disk after %d retries", FORCE_RETRY); else OProfiler.getInstance().updateCounter("OMMapManager.pagesCommitted", 1); OProfiler.getInstance().stopChrono("OMMapManager.commitPages", timer); return forceSucceed; }
/** * Acquire a network channel from the pool. Don't lock the write stream since the connection usage * is exclusive. * * @param iCommand * @return * @throws IOException */ protected OChannelBinaryClient beginRequest(final byte iCommand) throws IOException { OChannelBinaryClient network = null; if (debug) System.out.println("-> req: " + getSessionId()); // FIND THE FIRST FREE CHANNEL AVAILABLE synchronized (networkPool) { final int beginCursor = networkPoolCursor; while (network == null) { if (networkPool.size() == 0) throw new ONetworkProtocolException("Connection pool closed"); network = networkPool.get(networkPoolCursor); if (network.getLockWrite().tryLock()) break; network = null; networkPoolCursor++; if (networkPoolCursor >= networkPool.size()) // RESTART FROM THE FIRST ONE networkPoolCursor = 0; if (networkPoolCursor == beginCursor) { // COMPLETE ROUND AND NOT FREE CONNECTIONS FOUND if (networkPool.size() < maxPool) { // CREATE NEW CONNECTION network = createNetworkConnection(); network.getLockWrite().lock(); networkPool.add(network); if (debug) System.out.println("Created new connection " + networkPool.size()); } else { if (debug) System.out.println("-> req (waiting) : " + getSessionId()); final long startToWait = System.currentTimeMillis(); try { networkPool.wait(5000); OProfiler.getInstance().updateCounter("network.connectionPool.timeout", +1); } catch (InterruptedException e) { break; } final long elapsed = OProfiler.getInstance() .stopChrono("network.connectionPool.waitingTime", startToWait); if (debug) System.out.println("Waiting for connection = elapsed: " + elapsed); } } } } network.writeByte(iCommand); network.writeInt(getSessionId()); return network; }
private static OMMapBufferEntry mapBuffer( final OFileMMap iFile, final long iBeginOffset, final int iSize) throws IOException { long timer = OProfiler.getInstance().startChrono(); try { return new OMMapBufferEntry(iFile, iFile.map(iBeginOffset, iSize), iBeginOffset, iSize); } finally { OProfiler.getInstance().stopChrono("OMMapManager.loadPage", timer); } }
private static boolean allocIfOverlaps( final long iBeginOffset, final int iSize, final List<OMMapBufferEntry> fileEntries, final int p) { if (overlapStrategy == OVERLAP_STRATEGY.OVERLAP) return true; boolean overlaps = false; OMMapBufferEntry entry = null; if (p > 0) { // CHECK LOWER OFFSET entry = fileEntries.get(p - 1); overlaps = entry.beginOffset <= iBeginOffset && entry.beginOffset + entry.size >= iBeginOffset; } if (!overlaps && p < fileEntries.size() - 1) { // CHECK HIGHER OFFSET entry = fileEntries.get(p); overlaps = iBeginOffset + iSize >= entry.beginOffset; } if (overlaps) { // READ NOT IN BUFFER POOL: RETURN NULL TO LET TO THE CALLER TO EXECUTE A DIRECT READ WITHOUT // MMAP OProfiler.getInstance().updateCounter("OMMapManager.overlappedPageUsingChannel", 1); if (overlapStrategy == OVERLAP_STRATEGY.NO_OVERLAP_FLUSH_AND_USE_CHANNEL) commitBuffer(entry); return false; } return true; }
@SuppressWarnings("unchecked") public V getValueAt(final int iIndex) { V v = values[iIndex]; if (v == null) try { OProfiler.getInstance().updateCounter("OMVRBTreeMapEntry.unserializeValue", 1); v = (V) valueFromStream(iIndex); if (((OMVRBTreeMapProvider<K, V>) treeDataProvider).keepValuesInMemory) // KEEP THE UNMARSHALLED VALUE IN MEMORY values[iIndex] = v; } catch (IOException e) { OLogManager.instance() .error( this, "Cannot lazy load the value #" + iIndex + " in tree node " + this, e, OSerializationException.class); } return v; }
@SuppressWarnings("unchecked") public K getKeyAt(final int iIndex) { K k = keys[iIndex]; if (k == null) try { OProfiler.getInstance().updateCounter("OMVRBTreeMapEntry.unserializeKey", 1); k = (K) keyFromStream(iIndex); if (iIndex == 0 || iIndex == size || ((OMVRBTreeMapProvider<K, V>) treeDataProvider).keepKeysInMemory) // KEEP THE UNMARSHALLED KEY IN MEMORY. TO OPTIMIZE FIRST AND LAST ITEM ARE ALWAYS KEPT IN // MEMORY TO SPEEDUP FREQUENT // NODE CHECKING OF BOUNDS keys[iIndex] = k; } catch (IOException e) { OLogManager.instance() .error( this, "Cannot lazy load the key #" + iIndex + " in tree node " + this, e, OSerializationException.class); } return k; }
@SuppressWarnings("unchecked") public OSerializableStream fromStream(final byte[] iStream) throws OSerializationException { final long timer = OProfiler.getInstance().startChrono(); if (stream == null) stream = new OMemoryStream(iStream); else stream.setSource(iStream); try { pageSize = stream.getAsInteger(); parentRid = new ORecordId().fromStream(stream.getAsByteArrayFixed(ORecordId.PERSISTENT_SIZE)); leftRid = new ORecordId().fromStream(stream.getAsByteArrayFixed(ORecordId.PERSISTENT_SIZE)); rightRid = new ORecordId().fromStream(stream.getAsByteArrayFixed(ORecordId.PERSISTENT_SIZE)); color = stream.getAsBoolean(); size = stream.getAsInteger(); if (size > pageSize) throw new OConfigurationException( "Loaded index with page size setted to " + pageSize + " while the loaded was built with: " + size); // UNCOMPACT KEYS SEPARATELY serializedKeys = new int[pageSize]; for (int i = 0; i < size; ++i) { serializedKeys[i] = stream.getAsByteArrayOffset(); } // KEYS WILL BE LOADED LAZY keys = (K[]) new Object[pageSize]; // UNCOMPACT VALUES SEPARATELY serializedValues = new int[pageSize]; for (int i = 0; i < size; ++i) { serializedValues[i] = stream.getAsByteArrayOffset(); } // VALUES WILL BE LOADED LAZY values = (V[]) new Object[pageSize]; return this; } finally { OProfiler.getInstance().stopChrono("OMVRBTreeMapEntry.fromStream", timer); } }
/** Serialize only the new keys or the changed. */ protected byte[] serializeNewKey(final int iIndex) throws IOException { if (serializedKeys[iIndex] <= 0) { // NEW OR MODIFIED: MARSHALL CONTENT OProfiler.getInstance().updateCounter("OMVRBTreeMapEntry.serializeValue", 1); return ((OMVRBTreeMapProvider<K, V>) treeDataProvider).keySerializer.toStream(keys[iIndex]); } // RETURN ORIGINAL CONTENT return stream.getAsByteArray(serializedKeys[iIndex]); }
@Override public void init() { OProfiler.getInstance().startRecording(); database = new ODatabaseDocumentTx(System.getProperty("url")).open("admin", "admin"); record = database.newInstance(); database.declareIntent(new OIntentMassiveInsert()); database.begin(TXTYPE.NOTX); }
/** * Search for a buffer in the ordered list. * * @param fileEntries * @param iBeginOffset * @param iSize * @return negative number means not found. The position to insert is the (return value +1)*-1. * Zero or positive number is the found position. */ private static int searchEntry( final List<OMMapBufferEntry> fileEntries, final long iBeginOffset, final int iSize) { if (fileEntries == null || fileEntries.size() == 0) return -1; int high = fileEntries.size() - 1; if (high < 0) // NOT FOUND return -1; int low = 0; int mid = -1; // BINARY SEARCH OMMapBufferEntry e; while (low <= high) { mid = (low + high) >>> 1; e = fileEntries.get(mid); if (iBeginOffset >= e.beginOffset && iBeginOffset + iSize <= e.beginOffset + e.size) { // FOUND: USE IT OProfiler.getInstance().updateCounter("OMMapManager.reusedPage", 1); e.counter++; return mid; } if (low == high) { if (iBeginOffset > e.beginOffset) // NEXT POSITION low++; // NOT FOUND return (low + 2) * -1; } if (iBeginOffset >= e.beginOffset) low = mid + 1; else high = mid; } // NOT FOUND return mid; }
private static OMMapBufferEntry searchBetweenLastBlocks( final OFileMMap iFile, final long iBeginOffset, final int iSize) { if (bufferPoolLRU.size() > 0) { // SEARCH IF IT'S BETWEEN THE LAST 5 BLOCK USED: THIS IS THE COMMON CASE ON MASSIVE INSERTION final int min = Math.max(bufferPoolLRU.size() - 5, -1); for (int i = bufferPoolLRU.size() - 1; i > min; --i) { final OMMapBufferEntry e = bufferPoolLRU.get(i); if (e.file == iFile && iBeginOffset >= e.beginOffset && iBeginOffset + iSize <= e.beginOffset + e.size) { // FOUND: USE IT OProfiler.getInstance().updateCounter("OMMapManager.reusedPageBetweenLast", 1); e.counter++; return e; } } } return null; }
/** * Register statistic information about usage of index in {@link OProfilerStub}. * * @param index which usage is registering. */ private void updateStatistic(OIndex<?> index) { final OProfiler profiler = Orient.instance().getProfiler(); if (profiler.isRecording()) { Orient.instance() .getProfiler() .updateCounter( profiler.getDatabaseMetric(index.getDatabaseName(), "query.indexUsed"), "Used index in query", +1); final int paramCount = index.getDefinition().getParamCount(); if (paramCount > 1) { final String profiler_prefix = profiler.getDatabaseMetric(index.getDatabaseName(), "query.compositeIndexUsed"); profiler.updateCounter(profiler_prefix, "Used composite index in query", +1); profiler.updateCounter( profiler_prefix + "." + paramCount, "Used composite index in query with " + paramCount + " params", +1); } } }
static { blockSize = OGlobalConfiguration.FILE_MMAP_BLOCK_SIZE.getValueAsInteger(); FORCE_DELAY = OGlobalConfiguration.FILE_MMAP_FORCE_DELAY.getValueAsInteger(); FORCE_RETRY = OGlobalConfiguration.FILE_MMAP_FORCE_RETRY.getValueAsInteger(); maxMemory = OGlobalConfiguration.FILE_MMAP_MAX_MEMORY.getValueAsLong(); setOverlapStrategy(OGlobalConfiguration.FILE_MMAP_OVERLAP_STRATEGY.getValueAsInteger()); OProfiler.getInstance() .registerHookValue( "mmap.totalMemory", new OProfilerHookValue() { public Object getValue() { return totalMemory; } }); OProfiler.getInstance() .registerHookValue( "mmap.maxMemory", new OProfilerHookValue() { public Object getValue() { return maxMemory; } }); OProfiler.getInstance() .registerHookValue( "mmap.blockSize", new OProfilerHookValue() { public Object getValue() { return blockSize; } }); OProfiler.getInstance() .registerHookValue( "mmap.blocks", new OProfilerHookValue() { public synchronized Object getValue() { return bufferPoolLRU.size(); } }); OProfiler.getInstance() .registerHookValue( "mmap.alloc.strategy", new OProfilerHookValue() { public Object getValue() { return lastStrategy; } }); OProfiler.getInstance() .registerHookValue( "mmap.overlap.strategy", new OProfilerHookValue() { public Object getValue() { return overlapStrategy; } }); }
/** * Requests a mmap buffer to use. * * @param iFile MMap file * @param iBeginOffset Begin offset * @param iSize Portion size requested * @param iForce Tells if the size is mandatory or can be rounded to the next segment * @param iOperationType READ or WRITE * @param iStrategy * @return The mmap buffer entry if found, or null if the operation is READ and the buffer pool is * full. */ public static synchronized OMMapBufferEntry request( final OFileMMap iFile, final long iBeginOffset, final int iSize, final boolean iForce, final OPERATION_TYPE iOperationType, final ALLOC_STRATEGY iStrategy) { if (iStrategy == ALLOC_STRATEGY.MMAP_NEVER) return null; lastStrategy = iStrategy; OMMapBufferEntry entry = searchBetweenLastBlocks(iFile, iBeginOffset, iSize); if (entry != null) return entry; // SEARCH THE REQUESTED RANGE IN THE CACHED BUFFERS List<OMMapBufferEntry> fileEntries = bufferPoolPerFile.get(iFile); if (fileEntries == null) { fileEntries = new ArrayList<OMMapBufferEntry>(); bufferPoolPerFile.put(iFile, fileEntries); } int position = searchEntry(fileEntries, iBeginOffset, iSize); if (position > -1) // FOUND !!! return fileEntries.get(position); int p = (position + 2) * -1; // CHECK IF THERE IS A BUFFER THAT OVERLAPS if (!allocIfOverlaps(iBeginOffset, iSize, fileEntries, p)) { OProfiler.getInstance().updateCounter("OMMapManager.usedChannel", 1); return null; } int bufferSize = computeBestEntrySize(iFile, iBeginOffset, iSize, iForce, fileEntries, p); if (totalMemory + bufferSize > maxMemory && (iStrategy == ALLOC_STRATEGY.MMAP_ONLY_AVAIL_POOL || iOperationType == OPERATION_TYPE.READ && iStrategy == ALLOC_STRATEGY.MMAP_WRITE_ALWAYS_READ_IF_AVAIL_POOL)) { OProfiler.getInstance().updateCounter("OMMapManager.usedChannel", 1); return null; } entry = null; // FREE LESS-USED BUFFERS UNTIL THE FREE-MEMORY IS DOWN THE CONFIGURED MAX LIMIT do { if (totalMemory + bufferSize > maxMemory) freeResources(); // RECOMPUTE THE POSITION AFTER REMOVING fileEntries = bufferPoolPerFile.get(iFile); position = searchEntry(fileEntries, iBeginOffset, iSize); if (position > -1) // FOUND: THIS IS PRETTY STRANGE SINCE IT WASN'T FOUND! return fileEntries.get(position); // LOAD THE PAGE try { entry = mapBuffer(iFile, iBeginOffset, bufferSize); } catch (IllegalArgumentException e) { throw e; } catch (Exception e) { // REDUCE MAX MEMORY TO FORCE EMPTY BUFFERS maxMemory = maxMemory * 90 / 100; OLogManager.instance() .warn( OMMapManager.class, "Memory mapping error, try to reduce max memory to %d and retry...", e, maxMemory); } } while (entry == null && maxMemory > MIN_MEMORY); if (entry == null) throw new OIOException( "You can't access to the file portion " + iBeginOffset + "-" + iBeginOffset + iSize + " bytes"); totalMemory += bufferSize; bufferPoolLRU.add(entry); p = (position + 2) * -1; if (p < 0) p = 0; if (fileEntries == null) { // IN CASE THE CLEAN HAS REMOVED THE LIST fileEntries = new ArrayList<OMMapBufferEntry>(); bufferPoolPerFile.put(iFile, fileEntries); } fileEntries.add(p, entry); return entry; }