/**
* Wait for any running merge threads to finish. This call is not interruptible as used by {@link
* #close()}.
*/
public void sync() {
boolean interrupted = false;
try {
while (true) {
MergeThread toSync = null;
synchronized (this) {
for (MergeThread t : mergeThreads) {
if (t.isAlive()) {
toSync = t;
break;
}
}
}
if (toSync != null) {
try {
toSync.join();
} catch (InterruptedException ie) {
// ignore this Exception, we will retry until all threads are dead
interrupted = true;
}
} else {
break;
}
}
} finally {
// finally, restore interrupt status:
if (interrupted) Thread.currentThread().interrupt();
}
}
/**
* Returns the number of merge threads that are alive, ignoring the calling thread if it is a
* merge thread. Note that this number is ≤ {@link #mergeThreads} size.
*
* @lucene.internal
*/
public synchronized int mergeThreadCount() {
Thread currentThread = Thread.currentThread();
int count = 0;
for (MergeThread mergeThread : mergeThreads) {
if (currentThread != mergeThread
&& mergeThread.isAlive()
&& mergeThread.merge.rateLimiter.getAbort() == false) {
count++;
}
}
return count;
}
private boolean isBacklog(long now, OneMerge merge) {
double mergeMB = bytesToMB(merge.estimatedMergeBytes);
for (MergeThread mergeThread : mergeThreads) {
long mergeStartNS = mergeThread.merge.mergeStartNS;
if (mergeThread.isAlive()
&& mergeThread.merge != merge
&& mergeStartNS != -1
&& mergeThread.merge.estimatedMergeBytes >= MIN_BIG_MERGE_MB * 1024 * 1024
&& nsToSec(now - mergeStartNS) > 3.0) {
double otherMergeMB = bytesToMB(mergeThread.merge.estimatedMergeBytes);
double ratio = otherMergeMB / mergeMB;
if (ratio > 0.3 && ratio < 3.0) {
return true;
}
}
}
return false;
}
/**
* Called whenever the running merges have changed, to set merge IO limits. This method sorts the
* merge threads by their merge size in descending order and then pauses/unpauses threads from
* first to last -- that way, smaller merges are guaranteed to run before larger ones.
*/
protected synchronized void updateMergeThreads() {
// Only look at threads that are alive & not in the
// process of stopping (ie have an active merge):
final List<MergeThread> activeMerges = new ArrayList<>();
int threadIdx = 0;
while (threadIdx < mergeThreads.size()) {
final MergeThread mergeThread = mergeThreads.get(threadIdx);
if (!mergeThread.isAlive()) {
// Prune any dead threads
mergeThreads.remove(threadIdx);
continue;
}
activeMerges.add(mergeThread);
threadIdx++;
}
// Sort the merge threads, largest first:
CollectionUtil.timSort(activeMerges);
final int activeMergeCount = activeMerges.size();
int bigMergeCount = 0;
for (threadIdx = activeMergeCount - 1; threadIdx >= 0; threadIdx--) {
MergeThread mergeThread = activeMerges.get(threadIdx);
if (mergeThread.merge.estimatedMergeBytes > MIN_BIG_MERGE_MB * 1024 * 1024) {
bigMergeCount = 1 + threadIdx;
break;
}
}
long now = System.nanoTime();
StringBuilder message;
if (verbose()) {
message = new StringBuilder();
message.append(
String.format(
Locale.ROOT,
"updateMergeThreads ioThrottle=%s targetMBPerSec=%.1f MB/sec",
doAutoIOThrottle,
targetMBPerSec));
} else {
message = null;
}
for (threadIdx = 0; threadIdx < activeMergeCount; threadIdx++) {
MergeThread mergeThread = activeMerges.get(threadIdx);
OneMerge merge = mergeThread.merge;
// pause the thread if maxThreadCount is smaller than the number of merge threads.
final boolean doPause = threadIdx < bigMergeCount - maxThreadCount;
double newMBPerSec;
if (doPause) {
newMBPerSec = 0.0;
} else if (merge.maxNumSegments != -1) {
newMBPerSec = forceMergeMBPerSec;
} else if (doAutoIOThrottle == false) {
newMBPerSec = Double.POSITIVE_INFINITY;
} else if (merge.estimatedMergeBytes < MIN_BIG_MERGE_MB * 1024 * 1024) {
// Don't rate limit small merges:
newMBPerSec = Double.POSITIVE_INFINITY;
} else {
newMBPerSec = targetMBPerSec;
}
double curMBPerSec = merge.rateLimiter.getMBPerSec();
if (verbose()) {
long mergeStartNS = merge.mergeStartNS;
if (mergeStartNS == -1) {
// IndexWriter didn't start the merge yet:
mergeStartNS = now;
}
message.append('\n');
message.append(
String.format(
Locale.ROOT,
"merge thread %s estSize=%.1f MB (written=%.1f MB) runTime=%.1fs (stopped=%.1fs, paused=%.1fs) rate=%s\n",
mergeThread.getName(),
bytesToMB(merge.estimatedMergeBytes),
bytesToMB(merge.rateLimiter.totalBytesWritten),
nsToSec(now - mergeStartNS),
nsToSec(merge.rateLimiter.getTotalStoppedNS()),
nsToSec(merge.rateLimiter.getTotalPausedNS()),
rateToString(merge.rateLimiter.getMBPerSec())));
if (newMBPerSec != curMBPerSec) {
if (newMBPerSec == 0.0) {
message.append(" now stop");
} else if (curMBPerSec == 0.0) {
if (newMBPerSec == Double.POSITIVE_INFINITY) {
message.append(" now resume");
} else {
message.append(
String.format(Locale.ROOT, " now resume to %.1f MB/sec", newMBPerSec));
}
} else {
message.append(
String.format(
Locale.ROOT,
" now change from %.1f MB/sec to %.1f MB/sec",
curMBPerSec,
newMBPerSec));
}
} else if (curMBPerSec == 0.0) {
message.append(" leave stopped");
} else {
message.append(String.format(Locale.ROOT, " leave running at %.1f MB/sec", curMBPerSec));
}
}
merge.rateLimiter.setMBPerSec(newMBPerSec);
}
if (verbose()) {
message(message.toString());
}
}