@Override
public IndexCommit getIndexCommit() throws IOException {
Directory dir = _diskReader.directory();
SegmentInfos segmentInfos = new SegmentInfos();
segmentInfos.read(dir);
return new ZoieSolrIndexCommit(segmentInfos, dir);
}
@Override
public SyncedFlushResult syncFlush(String syncId, CommitId expectedCommitId)
throws EngineException {
// best effort attempt before we acquire locks
ensureOpen();
if (indexWriter.hasUncommittedChanges()) {
logger.trace("can't sync commit [{}]. have pending changes", syncId);
return SyncedFlushResult.PENDING_OPERATIONS;
}
if (expectedCommitId.idsEqual(lastCommittedSegmentInfos.getId()) == false) {
logger.trace("can't sync commit [{}]. current commit id is not equal to expected.", syncId);
return SyncedFlushResult.COMMIT_MISMATCH;
}
try (ReleasableLock lock = writeLock.acquire()) {
ensureOpen();
ensureCanFlush();
if (indexWriter.hasUncommittedChanges()) {
logger.trace("can't sync commit [{}]. have pending changes", syncId);
return SyncedFlushResult.PENDING_OPERATIONS;
}
if (expectedCommitId.idsEqual(lastCommittedSegmentInfos.getId()) == false) {
logger.trace("can't sync commit [{}]. current commit id is not equal to expected.", syncId);
return SyncedFlushResult.COMMIT_MISMATCH;
}
logger.trace("starting sync commit [{}]", syncId);
commitIndexWriter(indexWriter, translog, syncId);
logger.debug("successfully sync committed. sync id [{}].", syncId);
lastCommittedSegmentInfos = store.readLastCommittedSegmentsInfo();
return SyncedFlushResult.SUCCESS;
} catch (IOException ex) {
maybeFailEngine("sync commit", ex);
throw new EngineException(shardId, "failed to sync commit", ex);
}
}
public void testSizeInBytesCache() throws Exception {
Directory dir = newDirectory();
IndexWriterConfig conf =
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer())
.setMergePolicy(newLogMergePolicy());
IndexWriter writer = new IndexWriter(dir, conf);
writer.setInfoStream(VERBOSE ? System.out : null);
Document doc = new Document();
doc.add(new Field("a", "value", Store.YES, Index.ANALYZED));
writer.addDocument(doc);
writer.close();
SegmentInfos sis = new SegmentInfos();
sis.read(dir);
SegmentInfo si = sis.info(0);
long sizeInBytesNoStore = si.sizeInBytes(false);
long sizeInBytesWithStore = si.sizeInBytes(true);
assertTrue(
"sizeInBytesNoStore="
+ sizeInBytesNoStore
+ " sizeInBytesWithStore="
+ sizeInBytesWithStore,
sizeInBytesWithStore > sizeInBytesNoStore);
dir.close();
}
@Override
public List<LuceneSegmentInputSplit> getSplits(JobContext context)
throws IOException, InterruptedException {
Configuration configuration = context.getConfiguration();
LuceneStorageConfiguration lucene2SeqConfiguration =
new LuceneStorageConfiguration(configuration);
List<LuceneSegmentInputSplit> inputSplits = new ArrayList<>();
List<Path> indexPaths = lucene2SeqConfiguration.getIndexPaths();
for (Path indexPath : indexPaths) {
ReadOnlyFileSystemDirectory directory =
new ReadOnlyFileSystemDirectory(
FileSystem.get(configuration), indexPath, false, configuration);
SegmentInfos segmentInfos = new SegmentInfos();
segmentInfos.read(directory);
for (SegmentCommitInfo segmentInfo : segmentInfos) {
LuceneSegmentInputSplit inputSplit =
new LuceneSegmentInputSplit(
indexPath, segmentInfo.info.name, segmentInfo.sizeInBytes());
inputSplits.add(inputSplit);
LOG.info(
"Created {} byte input split for index '{}' segment {}",
segmentInfo.sizeInBytes(),
indexPath.toUri(),
segmentInfo.info.name);
}
}
return inputSplits;
}
public void listSegments() throws IOException {
DecimalFormat formatter = new DecimalFormat("###,###.###");
for (int x = 0; x < infos.size(); x++) {
SegmentInfo info = infos.info(x);
String sizeStr = formatter.format(info.sizeInBytes(true));
System.out.println(info.name + " " + sizeStr);
}
}
public void remove(String[] segs) throws IOException {
for (String n : segs) {
int idx = getIdx(n);
infos.remove(idx);
}
infos.changed();
infos.commit(fsDir);
}
@Override
public boolean isCurrent() throws CorruptIndexException, IOException {
ensureOpen();
if (writer == null || writer.isClosed()) {
// we loaded SegmentInfos from the directory
return SegmentInfos.readCurrentVersion(directory) == segmentInfos.getVersion();
} else {
return writer.nrtIsCurrent(segmentInfos);
}
}
@Override
public long getVersion() {
try {
SegmentInfos sinfos = new SegmentInfos();
sinfos.read(_dir);
return sinfos.getVersion();
} catch (Exception e) {
return 0L;
}
}
// Used by near real-time search
DirectoryReader(
IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor, boolean applyAllDeletes)
throws IOException {
this.directory = writer.getDirectory();
this.readOnly = true;
this.applyAllDeletes = applyAllDeletes; // saved for reopen
this.termInfosIndexDivisor = termInfosIndexDivisor;
readerFinishedListeners = writer.getReaderFinishedListeners();
// IndexWriter synchronizes externally before calling
// us, which ensures infos will not change; so there's
// no need to process segments in reverse order
final int numSegments = infos.size();
List<SegmentReader> readers = new ArrayList<SegmentReader>();
final Directory dir = writer.getDirectory();
segmentInfos = (SegmentInfos) infos.clone();
int infosUpto = 0;
for (int i = 0; i < numSegments; i++) {
boolean success = false;
try {
final SegmentInfo info = infos.info(i);
assert info.dir == dir;
final SegmentReader reader =
writer.readerPool.getReadOnlyClone(info, true, termInfosIndexDivisor);
if (reader.numDocs() > 0 || writer.getKeepFullyDeletedSegments()) {
reader.readerFinishedListeners = readerFinishedListeners;
readers.add(reader);
infosUpto++;
} else {
reader.close();
segmentInfos.remove(infosUpto);
}
success = true;
} finally {
if (!success) {
// Close all readers we had opened:
for (SegmentReader reader : readers) {
try {
reader.close();
} catch (Throwable ignore) {
// keep going - we want to clean up as much as possible
}
}
}
}
}
this.writer = writer;
initialize(readers.toArray(new SegmentReader[readers.size()]));
}
/**
* Current version number from segments file.
*
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
public static long readCurrentVersion(Directory directory)
throws CorruptIndexException, IOException {
// Fully read the segments file: this ensures that it's
// completely written so that if
// IndexWriter.prepareCommit has been called (but not
// yet commit), then the reader will still see itself as
// current:
SegmentInfos sis = new SegmentInfos();
sis.read(directory);
return sis.version;
}
public CommitStats(SegmentInfos segmentInfos) {
// clone the map to protect against concurrent changes
userData =
MapBuilder.<String, String>newMapBuilder()
.putAll(segmentInfos.getUserData())
.immutableMap();
// lucene calls the current generation, last generation.
generation = segmentInfos.getLastGeneration();
if (segmentInfos.getId() != null) { // id is only written starting with Lucene 5.0
id = Base64.encodeBytes(segmentInfos.getId());
}
numDocs = Lucene.getNumDocs(segmentInfos);
}
/** Construct reading the named set of readers. */
DirectoryReader(
Directory directory,
SegmentInfos sis,
IndexDeletionPolicy deletionPolicy,
boolean readOnly,
int termInfosIndexDivisor,
Collection<ReaderFinishedListener> readerFinishedListeners)
throws IOException {
this.directory = directory;
this.readOnly = readOnly;
this.segmentInfos = sis;
this.deletionPolicy = deletionPolicy;
this.termInfosIndexDivisor = termInfosIndexDivisor;
if (readerFinishedListeners == null) {
this.readerFinishedListeners =
new MapBackedSet<ReaderFinishedListener>(
new ConcurrentHashMap<ReaderFinishedListener, Boolean>());
} else {
this.readerFinishedListeners = readerFinishedListeners;
}
applyAllDeletes = false;
// To reduce the chance of hitting FileNotFound
// (and having to retry), we open segments in
// reverse because IndexWriter merges & deletes
// the newest segments first.
SegmentReader[] readers = new SegmentReader[sis.size()];
for (int i = sis.size() - 1; i >= 0; i--) {
boolean success = false;
try {
readers[i] = SegmentReader.get(readOnly, sis.info(i), termInfosIndexDivisor);
readers[i].readerFinishedListeners = this.readerFinishedListeners;
success = true;
} finally {
if (!success) {
// Close all readers we had opened:
for (i++; i < sis.size(); i++) {
try {
readers[i].close();
} catch (Throwable ignore) {
// keep going - we want to clean up as much as possible
}
}
}
}
}
initialize(readers);
}
private void verifyCommitOrder(List<? extends IndexCommit> commits) {
if (commits.isEmpty()) {
return;
}
final IndexCommit firstCommit = commits.get(0);
long last = SegmentInfos.generationFromSegmentsFileName(firstCommit.getSegmentsFileName());
assertEquals(last, firstCommit.getGeneration());
for (int i = 1; i < commits.size(); i++) {
final IndexCommit commit = commits.get(i);
long now = SegmentInfos.generationFromSegmentsFileName(commit.getSegmentsFileName());
assertTrue("SegmentInfos commits are out-of-order", now > last);
assertEquals(now, commit.getGeneration());
last = now;
}
}
@Test
public void testCloseNoEmptyCommits() throws Exception {
// LUCENE-4972: DTW used to create empty commits even if no changes were made
Directory dir = newDirectory();
DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir);
taxoWriter.addCategory(new CategoryPath("a"));
taxoWriter.commit();
long gen1 = SegmentInfos.getLastCommitGeneration(dir);
taxoWriter.close();
long gen2 = SegmentInfos.getLastCommitGeneration(dir);
assertEquals("empty commit should not have changed the index", gen1, gen2);
taxoWriter.close();
dir.close();
}
final boolean tryRenewSyncCommit() {
boolean renewed = false;
try (ReleasableLock lock = writeLock.acquire()) {
ensureOpen();
ensureCanFlush();
String syncId = lastCommittedSegmentInfos.getUserData().get(SYNC_COMMIT_ID);
if (syncId != null
&& translog.totalOperations() == 0
&& indexWriter.hasUncommittedChanges()) {
logger.trace("start renewing sync commit [{}]", syncId);
commitIndexWriter(indexWriter, translog, syncId);
logger.debug("successfully sync committed. sync id [{}].", syncId);
lastCommittedSegmentInfos = store.readLastCommittedSegmentsInfo();
renewed = true;
}
} catch (IOException ex) {
maybeFailEngine("renew sync commit", ex);
throw new EngineException(shardId, "failed to renew sync commit", ex);
}
if (renewed) { // refresh outside of the write lock
refresh("renew sync commit");
}
return renewed;
}
private void recoverFromTranslog(TranslogRecoveryPerformer handler) throws IOException {
Translog.TranslogGeneration translogGeneration = translog.getGeneration();
final int opsRecovered;
try {
Translog.Snapshot snapshot = translog.newSnapshot();
opsRecovered = handler.recoveryFromSnapshot(this, snapshot);
} catch (Throwable e) {
throw new EngineException(shardId, "failed to recover from translog", e);
}
// flush if we recovered something or if we have references to older translogs
// note: if opsRecovered == 0 and we have older translogs it means they are corrupted or 0
// length.
assert allowCommits.get() == false : "commits are allowed but shouldn't";
allowCommits.set(true); // we are good - now we can commit
if (opsRecovered > 0) {
logger.trace(
"flushing post recovery from translog. ops recovered [{}]. committed translog id [{}]. current id [{}]",
opsRecovered,
translogGeneration == null ? null : translogGeneration.translogFileGeneration,
translog.currentFileGeneration());
flush(true, true);
} else if (translog.isCurrent(translogGeneration) == false) {
commitIndexWriter(
indexWriter,
translog,
lastCommittedSegmentInfos.getUserData().get(Engine.SYNC_COMMIT_ID));
}
}
private final IndexReader doOpenFromWriter(boolean openReadOnly, IndexCommit commit)
throws CorruptIndexException, IOException {
assert readOnly;
if (!openReadOnly) {
throw new IllegalArgumentException(
"a reader obtained from IndexWriter.getReader() can only be reopened with openReadOnly=true (got false)");
}
if (commit != null) {
throw new IllegalArgumentException(
"a reader obtained from IndexWriter.getReader() cannot currently accept a commit");
}
if (writer.nrtIsCurrent(segmentInfos)) {
return null;
}
IndexReader reader = writer.getReader(applyAllDeletes);
// If in fact no changes took place, return null:
if (reader.getVersion() == segmentInfos.getVersion()) {
reader.decRef();
return null;
}
reader.readerFinishedListeners = readerFinishedListeners;
return reader;
}
@Override
public final synchronized IndexReader clone(boolean openReadOnly)
throws CorruptIndexException, IOException {
// doOpenIfChanged calls ensureOpen
DirectoryReader newReader =
doOpenIfChanged((SegmentInfos) segmentInfos.clone(), true, openReadOnly);
if (this != newReader) {
newReader.deletionPolicy = deletionPolicy;
}
newReader.writer = writer;
// If we're cloning a non-readOnly reader, move the
// writeLock (if there is one) to the new reader:
if (!openReadOnly && writeLock != null) {
// In near real-time search, reader is always readonly
assert writer == null;
newReader.writeLock = writeLock;
newReader.hasChanges = hasChanges;
newReader.hasDeletions = hasDeletions;
writeLock = null;
hasChanges = false;
}
assert newReader.readerFinishedListeners != null;
return newReader;
}
/*
* Test a deletion policy that keeps last N commits.
*/
public void testKeepLastNDeletionPolicy() throws IOException {
final int N = 5;
for (int pass = 0; pass < 2; pass++) {
boolean useCompoundFile = (pass % 2) != 0;
Directory dir = newDirectory();
if (dir instanceof MockDirectoryWrapper) {
// test manually deletes files
((MockDirectoryWrapper) dir).setEnableVirusScanner(false);
}
KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(N);
for (int j = 0; j < N + 1; j++) {
IndexWriterConfig conf =
newIndexWriterConfig(new MockAnalyzer(random()))
.setOpenMode(OpenMode.CREATE)
.setIndexDeletionPolicy(policy)
.setMaxBufferedDocs(10);
MergePolicy mp = conf.getMergePolicy();
mp.setNoCFSRatio(useCompoundFile ? 1.0 : 0.0);
IndexWriter writer = new IndexWriter(dir, conf);
policy = (KeepLastNDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
for (int i = 0; i < 17; i++) {
addDoc(writer);
}
writer.forceMerge(1);
writer.close();
}
assertTrue(policy.numDelete > 0);
assertEquals(N + 1, policy.numOnInit);
assertEquals(N + 1, policy.numOnCommit);
// Simplistic check: just verify only the past N segments_N's still
// exist, and, I can open a reader on each:
long gen = SegmentInfos.getLastCommitGeneration(dir);
for (int i = 0; i < N + 1; i++) {
try {
IndexReader reader = DirectoryReader.open(dir);
reader.close();
if (i == N) {
fail("should have failed on commits prior to last " + N);
}
} catch (IOException e) {
if (i != N) {
throw e;
}
}
if (i < N) {
dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
}
gen--;
}
dir.close();
}
}
/** Returns a copy of this instance, also copying each SegmentInfo. */
@Override
public SegmentInfos clone() {
try {
final SegmentInfos sis = (SegmentInfos) super.clone();
// deep clone, first recreate all collections:
sis.segments = new ArrayList<>(size());
for (final SegmentCommitInfo info : this) {
assert info.info.getCodec() != null;
// dont directly access segments, use add method!!!
sis.add(info.clone());
}
sis.userData = new HashMap<>(userData);
return sis;
} catch (CloneNotSupportedException e) {
throw new RuntimeException("should not happen", e);
}
}
@Override
public void prepare(Directory dir, String[] qqNames, String indexField) throws IOException {
this.dir = dir;
this.qqNames = qqNames;
this.indexField = indexField;
// load index, calc avdl
SegmentInfos segInfo = new SegmentInfos();
segInfo.read(dir);
try {
this.avdl = Utils.getAvgDocLen(segInfo);
} catch (Exception e) {
e.printStackTrace();
System.exit(1);
}
}
public void split(File destDir, String[] segs) throws IOException {
destDir.mkdirs();
FSDirectory destFSDir = FSDirectory.open(destDir);
SegmentInfos destInfos = new SegmentInfos();
destInfos.counter = infos.counter;
for (String n : segs) {
SegmentInfo info = getInfo(n);
destInfos.add(info);
// now copy files over
List<String> files = info.files();
for (final String srcName : files) {
File srcFile = new File(dir, srcName);
File destFile = new File(destDir, srcName);
copyFile(srcFile, destFile);
}
}
destInfos.changed();
destInfos.commit(destFSDir);
// System.out.println("destDir:"+destDir.getAbsolutePath());
}
private synchronized IndexReader doOpenNoWriter(final boolean openReadOnly, IndexCommit commit)
throws CorruptIndexException, IOException {
if (commit == null) {
if (hasChanges) {
// We have changes, which means we are not readOnly:
assert readOnly == false;
// and we hold the write lock:
assert writeLock != null;
// so no other writer holds the write lock, which
// means no changes could have been done to the index:
assert isCurrent();
if (openReadOnly) {
return clone(openReadOnly);
} else {
return null;
}
} else if (isCurrent()) {
if (openReadOnly != readOnly) {
// Just fallback to clone
return clone(openReadOnly);
} else {
return null;
}
}
} else {
if (directory != commit.getDirectory()) {
throw new IOException("the specified commit does not match the specified Directory");
}
if (segmentInfos != null
&& commit.getSegmentsFileName().equals(segmentInfos.getCurrentSegmentFileName())) {
if (readOnly != openReadOnly) {
// Just fallback to clone
return clone(openReadOnly);
} else {
return null;
}
}
}
return (IndexReader)
new SegmentInfos.FindSegmentsFile(directory) {
@Override
protected Object doBody(String segmentFileName)
throws CorruptIndexException, IOException {
SegmentInfos infos = new SegmentInfos();
infos.read(directory, segmentFileName);
return doOpenIfChanged(infos, false, openReadOnly);
}
}.run(commit);
}
@Override
public MergeSpecification findMerges(
MergeTrigger mergeTrigger, SegmentInfos segmentInfos, IndexWriter writer)
throws IOException {
MergeSpecification ms = new MergeSpecification();
if (doMerge) {
OneMerge om = new OneMerge(segmentInfos.asList().subList(start, start + length));
ms.add(om);
doMerge = false;
return ms;
}
return null;
}
ReaderCommit(SegmentInfos infos, Directory dir) throws IOException {
segmentsFileName = infos.getCurrentSegmentFileName();
this.dir = dir;
userData = infos.getUserData();
files = Collections.unmodifiableCollection(infos.files(dir, true));
version = infos.getVersion();
generation = infos.getGeneration();
segmentCount = infos.size();
}
private void initialize(SegmentReader[] subReaders) throws IOException {
this.subReaders = subReaders;
starts = new int[subReaders.length + 1]; // build starts array
for (int i = 0; i < subReaders.length; i++) {
starts[i] = maxDoc;
maxDoc += subReaders[i].maxDoc(); // compute maxDocs
if (subReaders[i].hasDeletions()) hasDeletions = true;
}
starts[subReaders.length] = maxDoc;
if (!readOnly) {
maxIndexVersion = SegmentInfos.readCurrentVersion(directory);
}
}
/** Returns a copy of this instance, also copying each SegmentInfo. */
@Override
public Object clone() {
try {
final SegmentInfos sis = (SegmentInfos) super.clone();
// deep clone, first recreate all collections:
sis.segments = new ArrayList<SegmentInfo>(size());
sis.segmentSet = new HashSet<SegmentInfo>(size());
sis.cachedUnmodifiableList = null;
sis.cachedUnmodifiableSet = null;
for (final SegmentInfo info : this) {
// dont directly access segments, use add method!!!
sis.add((SegmentInfo) info.clone());
}
sis.userData = new HashMap<String, String>(userData);
return sis;
} catch (CloneNotSupportedException e) {
throw new RuntimeException("should not happen", e);
}
}
/** {@inheritDoc} */
@Override
public String toString() {
final StringBuilder buffer = new StringBuilder();
if (hasChanges) {
buffer.append("*");
}
buffer.append(getClass().getSimpleName());
buffer.append('(');
final String segmentsFile = segmentInfos.getCurrentSegmentFileName();
if (segmentsFile != null) {
buffer.append(segmentsFile);
}
if (writer != null) {
buffer.append(":nrt");
}
for (int i = 0; i < subReaders.length; i++) {
buffer.append(' ');
buffer.append(subReaders[i]);
}
buffer.append(')');
return buffer.toString();
}
/** @see org.apache.lucene.index.IndexReader#listCommits */
public static Collection<IndexCommit> listCommits(Directory dir) throws IOException {
final String[] files = dir.listAll();
List<IndexCommit> commits = new ArrayList<IndexCommit>();
SegmentInfos latest = new SegmentInfos();
latest.read(dir);
final long currentGen = latest.getGeneration();
commits.add(new ReaderCommit(latest, dir));
for (int i = 0; i < files.length; i++) {
final String fileName = files[i];
if (fileName.startsWith(IndexFileNames.SEGMENTS)
&& !fileName.equals(IndexFileNames.SEGMENTS_GEN)
&& SegmentInfos.generationFromSegmentsFileName(fileName) < currentGen) {
SegmentInfos sis = new SegmentInfos();
try {
// IOException allowed to throw there, in case
// segments_N is corrupt
sis.read(dir, fileName);
} catch (FileNotFoundException fnfe) {
// LUCENE-948: on NFS (and maybe others), if
// you have writers switching back and forth
// between machines, it's very likely that the
// dir listing will be stale and will claim a
// file segments_X exists when in fact it
// doesn't. So, we catch this and handle it
// as if the file does not exist
sis = null;
}
if (sis != null) commits.add(new ReaderCommit(sis, dir));
}
}
// Ensure that the commit points are sorted in ascending order.
Collections.sort(commits);
return commits;
}
/**
* Tries to acquire the WriteLock on this directory. this method is only valid if this IndexReader
* is directory owner.
*
* @throws StaleReaderException if the index has changed since this reader was opened
* @throws CorruptIndexException if the index is corrupt
* @throws org.apache.lucene.store.LockObtainFailedException if another writer has this index open
* (<code>write.lock</code> could not be obtained)
* @throws IOException if there is a low-level IO error
*/
@Override
protected void acquireWriteLock()
throws StaleReaderException, CorruptIndexException, LockObtainFailedException, IOException {
if (readOnly) {
// NOTE: we should not reach this code w/ the core
// IndexReader classes; however, an external subclass
// of IndexReader could reach this.
ReadOnlySegmentReader.noWrite();
}
if (segmentInfos != null) {
ensureOpen();
if (stale)
throw new StaleReaderException(
"IndexReader out of date and no longer valid for delete, undelete, or setNorm operations");
if (writeLock == null) {
Lock writeLock = directory.makeLock(IndexWriter.WRITE_LOCK_NAME);
if (!writeLock.obtain(IndexWriterConfig.WRITE_LOCK_TIMEOUT)) // obtain write lock
throw new LockObtainFailedException("Index locked for write: " + writeLock);
this.writeLock = writeLock;
// we have to check whether index has changed since this reader was opened.
// if so, this reader is no longer valid for
// deletion
if (SegmentInfos.readCurrentVersion(directory) > maxIndexVersion) {
stale = true;
this.writeLock.release();
this.writeLock = null;
throw new StaleReaderException(
"IndexReader out of date and no longer valid for delete, undelete, or setNorm operations");
}
}
}
}