private void pruneDeletedTombstones() { long timeMSec = engineConfig.getThreadPool().estimatedTimeInMillis(); // TODO: not good that we reach into LiveVersionMap here; can we move this inside VersionMap // instead? problem is the dirtyLock... // we only need to prune the deletes map; the current/old version maps are cleared on refresh: for (Map.Entry<BytesRef, VersionValue> entry : versionMap.getAllTombstones()) { BytesRef uid = entry.getKey(); synchronized ( dirtyLock( uid)) { // can we do it without this lock on each value? maybe batch to a set and get // the lock once per set? // Must re-get it here, vs using entry.getValue(), in case the uid was indexed/deleted since // we pulled the iterator: VersionValue versionValue = versionMap.getTombstoneUnderLock(uid); if (versionValue != null) { if (timeMSec - versionValue.time() > getGcDeletesInMillis()) { versionMap.removeTombstoneUnderLock(uid); } } } } lastDeleteVersionPruneTimeMSec = timeMSec; }
private boolean innerIndex(Index index) throws IOException { synchronized (dirtyLock(index.uid())) { lastWriteNanos = index.startTime(); final long currentVersion; final boolean deleted; VersionValue versionValue = versionMap.getUnderLock(index.uid().bytes()); if (versionValue == null) { currentVersion = loadCurrentVersionFromIndex(index.uid()); deleted = currentVersion == Versions.NOT_FOUND; } else { deleted = versionValue.delete(); if (engineConfig.isEnableGcDeletes() && versionValue.delete() && (engineConfig.getThreadPool().estimatedTimeInMillis() - versionValue.time()) > getGcDeletesInMillis()) { currentVersion = Versions.NOT_FOUND; // deleted, and GC } else { currentVersion = versionValue.version(); } } long expectedVersion = index.version(); if (isVersionConflictForWrites(index, currentVersion, deleted, expectedVersion)) { if (index.origin() != Operation.Origin.RECOVERY) { throw new VersionConflictEngineException( shardId, index.type(), index.id(), index .versionType() .explainConflictForWrites(currentVersion, expectedVersion, deleted)); } return false; } long updatedVersion = index.versionType().updateVersion(currentVersion, expectedVersion); final boolean created; index.updateVersion(updatedVersion); if (currentVersion == Versions.NOT_FOUND) { // document does not exists, we can optimize for create created = true; index(index, indexWriter); } else { created = update(index, versionValue, indexWriter); } Translog.Location translogLocation = translog.add(new Translog.Index(index)); versionMap.putUnderLock( index.uid().bytes(), new VersionValue(updatedVersion, translogLocation)); index.setTranslogLocation(translogLocation); return created; } }
@Override protected final void writerSegmentStats(SegmentsStats stats) { stats.addVersionMapMemoryInBytes(versionMap.ramBytesUsed()); stats.addIndexWriterMemoryInBytes(indexWriter.ramBytesUsed()); stats.addIndexWriterMaxMemoryInBytes( (long) (indexWriter.getConfig().getRAMBufferSizeMB() * 1024 * 1024)); }
@Override public GetResult get(Get get, Function<String, Searcher> searcherFactory) throws EngineException { try (ReleasableLock lock = readLock.acquire()) { ensureOpen(); if (get.realtime()) { VersionValue versionValue = versionMap.getUnderLock(get.uid().bytes()); if (versionValue != null) { if (versionValue.delete()) { return GetResult.NOT_EXISTS; } if (get.versionType().isVersionConflictForReads(versionValue.version(), get.version())) { Uid uid = Uid.createUid(get.uid().text()); throw new VersionConflictEngineException( shardId, uid.type(), uid.id(), get.versionType().explainConflictForReads(versionValue.version(), get.version())); } Translog.Operation op = translog.read(versionValue.translogLocation()); if (op != null) { return new GetResult(true, versionValue.version(), op.getSource()); } } } // no version, get the version from the index, we know that we refresh on flush return getFromSearcher(get, searcherFactory); } }
@Override public void writeIndexingBuffer() throws EngineException { // we obtain a read lock here, since we don't want a flush to happen while we are writing // since it flushes the index as well (though, in terms of concurrency, we are allowed to do it) try (ReleasableLock lock = readLock.acquire()) { ensureOpen(); // TODO: it's not great that we secretly tie searcher visibility to "freeing up heap" here... // really we should keep two // searcher managers, one for searching which is only refreshed by the schedule the user // requested (refresh_interval, or invoking // refresh API), and another for version map interactions. See #15768. final long versionMapBytes = versionMap.ramBytesUsedForRefresh(); final long indexingBufferBytes = indexWriter.ramBytesUsed(); final boolean useRefresh = versionMapRefreshPending.get() || (indexingBufferBytes / 4 < versionMapBytes); if (useRefresh) { // The version map is using > 25% of the indexing buffer, so we do a refresh so the version // map also clears logger.debug( "use refresh to write indexing buffer (heap size=[{}]), to also clear version map (heap size=[{}])", new ByteSizeValue(indexingBufferBytes), new ByteSizeValue(versionMapBytes)); refresh("write indexing buffer"); } else { // Most of our heap is used by the indexing buffer, so we do a cheaper (just writes // segments, doesn't open a new searcher) IW.flush: logger.debug( "use IndexWriter.flush to write indexing buffer (heap size=[{}]) since version map is small (heap size=[{}])", new ByteSizeValue(indexingBufferBytes), new ByteSizeValue(versionMapBytes)); indexWriter.flush(); } } catch (AlreadyClosedException e) { ensureOpen(); maybeFailEngine("writeIndexingBuffer", e); } catch (EngineClosedException e) { throw e; } catch (Throwable t) { failEngine("writeIndexingBuffer failed", t); throw new RefreshFailedEngineException(shardId, t); } }
@Override public long getIndexBufferRAMBytesUsed() { return indexWriter.ramBytesUsed() + versionMap.ramBytesUsedForRefresh(); }
private void innerDelete(Delete delete) throws IOException { synchronized (dirtyLock(delete.uid())) { lastWriteNanos = delete.startTime(); final long currentVersion; final boolean deleted; VersionValue versionValue = versionMap.getUnderLock(delete.uid().bytes()); if (versionValue == null) { currentVersion = loadCurrentVersionFromIndex(delete.uid()); deleted = currentVersion == Versions.NOT_FOUND; } else { deleted = versionValue.delete(); if (engineConfig.isEnableGcDeletes() && versionValue.delete() && (engineConfig.getThreadPool().estimatedTimeInMillis() - versionValue.time()) > getGcDeletesInMillis()) { currentVersion = Versions.NOT_FOUND; // deleted, and GC } else { currentVersion = versionValue.version(); } } long updatedVersion; long expectedVersion = delete.version(); if (delete .versionType() .isVersionConflictForWrites(currentVersion, expectedVersion, deleted)) { if (delete.origin() == Operation.Origin.RECOVERY) { return; } else { throw new VersionConflictEngineException( shardId, delete.type(), delete.id(), delete .versionType() .explainConflictForWrites(currentVersion, expectedVersion, deleted)); } } updatedVersion = delete.versionType().updateVersion(currentVersion, expectedVersion); final boolean found; if (currentVersion == Versions.NOT_FOUND) { // doc does not exist and no prior deletes found = false; } else if (versionValue != null && versionValue.delete()) { // a "delete on delete", in this case, we still increment the version, log it, and return // that version found = false; } else { // we deleted a currently existing document indexWriter.deleteDocuments(delete.uid()); found = true; } delete.updateVersion(updatedVersion, found); Translog.Location translogLocation = translog.add(new Translog.Delete(delete)); versionMap.putUnderLock( delete.uid().bytes(), new DeleteVersionValue( updatedVersion, engineConfig.getThreadPool().estimatedTimeInMillis(), translogLocation)); delete.setTranslogLocation(translogLocation); } }
public InternalEngine(EngineConfig engineConfig) throws EngineException { super(engineConfig); openMode = engineConfig.getOpenMode(); this.versionMap = new LiveVersionMap(); store.incRef(); IndexWriter writer = null; Translog translog = null; SearcherManager manager = null; EngineMergeScheduler scheduler = null; boolean success = false; try { this.lastDeleteVersionPruneTimeMSec = engineConfig.getThreadPool().estimatedTimeInMillis(); mergeScheduler = scheduler = new EngineMergeScheduler(engineConfig.getShardId(), engineConfig.getIndexSettings()); this.dirtyLocks = new Object [Runtime.getRuntime().availableProcessors() * 10]; // we multiply it to have enough... for (int i = 0; i < dirtyLocks.length; i++) { dirtyLocks[i] = new Object(); } throttle = new IndexThrottle(); this.searcherFactory = new SearchFactory(logger, isClosed, engineConfig); try { writer = createWriter(openMode == EngineConfig.OpenMode.CREATE_INDEX_AND_TRANSLOG); indexWriter = writer; translog = openTranslog(engineConfig, writer); assert translog.getGeneration() != null; } catch (IOException | TranslogCorruptedException e) { throw new EngineCreationFailureException(shardId, "failed to create engine", e); } catch (AssertionError e) { // IndexWriter throws AssertionError on init, if asserts are enabled, if any files don't // exist, but tests that // randomly throw FNFE/NSFE can also hit this: if (ExceptionsHelper.stackTrace(e) .contains("org.apache.lucene.index.IndexWriter.filesExist")) { throw new EngineCreationFailureException(shardId, "failed to create engine", e); } else { throw e; } } this.translog = translog; manager = createSearcherManager(); this.searcherManager = manager; this.versionMap.setManager(searcherManager); // don't allow commits until we are done with recovering allowCommits.compareAndSet(true, openMode != EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG); success = true; } finally { if (success == false) { IOUtils.closeWhileHandlingException(writer, translog, manager, scheduler); versionMap.clear(); if (isClosed.get() == false) { // failure we need to dec the store reference store.decRef(); } } } logger.trace("created new InternalEngine"); }