@Override
public Facet facet() {
TShortIntHashMap facets = aggregator.facets();
if (facets.isEmpty()) {
pushFacets(facets);
return new InternalShortTermsFacet(
facetName,
comparatorType,
size,
ImmutableList.<InternalShortTermsFacet.ShortEntry>of(),
aggregator.missing());
} else {
// we need to fetch facets of "size * numberOfShards" because of problems in how they are
// distributed across shards
BoundedTreeSet<InternalShortTermsFacet.ShortEntry> ordered =
new BoundedTreeSet<InternalShortTermsFacet.ShortEntry>(
comparatorType.comparator(), size * numberOfShards);
for (TShortIntIterator it = facets.iterator(); it.hasNext(); ) {
it.advance();
ordered.add(new InternalShortTermsFacet.ShortEntry(it.key(), it.value()));
}
pushFacets(facets);
return new InternalShortTermsFacet(
facetName, comparatorType, size, ordered, aggregator.missing());
}
}
@Override
public Facet reduce(List<Facet> facets) {
if (facets.size() == 1) {
return facets.get(0);
}
InternalDoubleTermsFacet first = (InternalDoubleTermsFacet) facets.get(0);
TDoubleIntHashMap aggregated = CacheRecycler.popDoubleIntMap();
long missing = 0;
long total = 0;
for (Facet facet : facets) {
InternalDoubleTermsFacet mFacet = (InternalDoubleTermsFacet) facet;
missing += mFacet.getMissingCount();
total += mFacet.getTotalCount();
for (DoubleEntry entry : mFacet.entries) {
aggregated.adjustOrPutValue(entry.term, entry.getCount(), entry.getCount());
}
}
BoundedTreeSet<DoubleEntry> ordered =
new BoundedTreeSet<DoubleEntry>(first.comparatorType.comparator(), first.requiredSize);
for (TDoubleIntIterator it = aggregated.iterator(); it.hasNext(); ) {
it.advance();
ordered.add(new DoubleEntry(it.key(), it.value()));
}
first.entries = ordered;
first.missing = missing;
first.total = total;
CacheRecycler.pushDoubleIntMap(aggregated);
return first;
}
@Override
public Facet reduce(String name, List<Facet> facets) {
if (facets.size() == 1) {
return facets.get(0);
}
InternalIntTermsFacet first = (InternalIntTermsFacet) facets.get(0);
TIntIntHashMap aggregated = aggregateCache.get().get();
aggregated.clear();
long missing = 0;
for (Facet facet : facets) {
InternalIntTermsFacet mFacet = (InternalIntTermsFacet) facet;
missing += mFacet.missingCount();
for (IntEntry entry : mFacet.entries) {
aggregated.adjustOrPutValue(entry.term, entry.count(), entry.count());
}
}
BoundedTreeSet<IntEntry> ordered =
new BoundedTreeSet<IntEntry>(first.comparatorType.comparator(), first.requiredSize);
for (TIntIntIterator it = aggregated.iterator(); it.hasNext(); ) {
it.advance();
ordered.add(new IntEntry(it.key(), it.value()));
}
first.entries = ordered;
first.missing = missing;
return first;
}
@Override
public InternalFacet buildFacet(String facetName) {
if (facets.v().isEmpty()) {
facets.close();
return new InternalLongTermsFacet(
facetName,
comparatorType,
size,
ImmutableList.<InternalLongTermsFacet.LongEntry>of(),
missing,
total);
} else {
LongIntOpenHashMap facetEntries = facets.v();
final boolean[] states = facets.v().allocated;
final long[] keys = facets.v().keys;
final int[] values = facets.v().values;
if (size < EntryPriorityQueue.LIMIT) {
EntryPriorityQueue ordered = new EntryPriorityQueue(shardSize, comparatorType.comparator());
for (int i = 0; i < states.length; i++) {
if (states[i]) {
ordered.insertWithOverflow(new InternalLongTermsFacet.LongEntry(keys[i], values[i]));
}
}
InternalLongTermsFacet.LongEntry[] list =
new InternalLongTermsFacet.LongEntry[ordered.size()];
for (int i = ordered.size() - 1; i >= 0; i--) {
list[i] = (InternalLongTermsFacet.LongEntry) ordered.pop();
}
facets.close();
return new InternalLongTermsFacet(
facetName, comparatorType, size, Arrays.asList(list), missing, total);
} else {
BoundedTreeSet<InternalLongTermsFacet.LongEntry> ordered =
new BoundedTreeSet<>(comparatorType.comparator(), shardSize);
for (int i = 0; i < states.length; i++) {
if (states[i]) {
ordered.add(new InternalLongTermsFacet.LongEntry(keys[i], values[i]));
}
}
facets.close();
return new InternalLongTermsFacet(facetName, comparatorType, size, ordered, missing, total);
}
}
}
@Override
public Facet reduce(List<Facet> facets) {
if (facets.size() == 1) {
return facets.get(0);
}
InternalLongTermsFacet first = null;
TLongIntHashMap aggregated = CacheRecycler.popLongIntMap();
long missing = 0;
long total = 0;
for (Facet facet : facets) {
TermsFacet termsFacet = (TermsFacet) facet;
// termsFacet could be of type InternalStringTermsFacet representing unmapped fields
if (first == null && termsFacet instanceof InternalLongTermsFacet) {
first = (InternalLongTermsFacet) termsFacet;
}
missing += termsFacet.getMissingCount();
total += termsFacet.getTotalCount();
for (Entry entry : termsFacet.getEntries()) {
aggregated.adjustOrPutValue(((LongEntry) entry).term, entry.getCount(), entry.getCount());
}
}
BoundedTreeSet<LongEntry> ordered =
new BoundedTreeSet<LongEntry>(first.comparatorType.comparator(), first.requiredSize);
for (TLongIntIterator it = aggregated.iterator(); it.hasNext(); ) {
it.advance();
ordered.add(new LongEntry(it.key(), it.value()));
}
first.entries = ordered;
first.missing = missing;
first.total = total;
CacheRecycler.pushLongIntMap(aggregated);
return first;
}
@Override
public Facet facet() {
TIntIntHashMap facets = aggregator.facets();
if (facets.isEmpty()) {
CacheRecycler.pushIntIntMap(facets);
return new InternalIntTermsFacet(
facetName,
comparatorType,
size,
ImmutableList.<InternalIntTermsFacet.IntEntry>of(),
aggregator.missing());
} else {
if (size < EntryPriorityQueue.LIMIT) {
EntryPriorityQueue ordered = new EntryPriorityQueue(size, comparatorType.comparator());
for (TIntIntIterator it = facets.iterator(); it.hasNext(); ) {
it.advance();
ordered.insertWithOverflow(new InternalIntTermsFacet.IntEntry(it.key(), it.value()));
}
InternalIntTermsFacet.IntEntry[] list = new InternalIntTermsFacet.IntEntry[ordered.size()];
for (int i = ordered.size() - 1; i >= 0; i--) {
list[i] = (InternalIntTermsFacet.IntEntry) ordered.pop();
}
CacheRecycler.pushIntIntMap(facets);
return new InternalIntTermsFacet(
facetName, comparatorType, size, Arrays.asList(list), aggregator.missing());
} else {
BoundedTreeSet<InternalIntTermsFacet.IntEntry> ordered =
new BoundedTreeSet<InternalIntTermsFacet.IntEntry>(comparatorType.comparator(), size);
for (TIntIntIterator it = facets.iterator(); it.hasNext(); ) {
it.advance();
ordered.add(new InternalIntTermsFacet.IntEntry(it.key(), it.value()));
}
CacheRecycler.pushIntIntMap(facets);
return new InternalIntTermsFacet(
facetName, comparatorType, size, ordered, aggregator.missing());
}
}
}
@Override
public Facet facet() {
if (current != null) {
missing += current.counts[0];
total += current.total - current.counts[0];
// if we have values for this one, add it
if (current.values.ordinals().getNumOrds() > 1) {
aggregators.add(current);
}
}
AggregatorPriorityQueue queue = new AggregatorPriorityQueue(aggregators.size());
for (ReaderAggregator aggregator : aggregators) {
if (aggregator.nextPosition()) {
queue.add(aggregator);
}
}
// YACK, we repeat the same logic, but once with an optimizer priority queue for smaller sizes
if (size < EntryPriorityQueue.LIMIT) {
// optimize to use priority size
EntryPriorityQueue ordered = new EntryPriorityQueue(size, comparatorType.comparator());
while (queue.size() > 0) {
ReaderAggregator agg = queue.top();
BytesRef value =
agg.values.makeSafe(
agg.current); // we need to makeSafe it, since we end up pushing it... (can we get
// around this?)
int count = 0;
do {
count += agg.counts[agg.position];
if (agg.nextPosition()) {
agg = queue.updateTop();
} else {
// we are done with this reader
queue.pop();
agg = queue.top();
}
} while (agg != null && value.equals(agg.current));
if (count > minCount) {
if (excluded != null && excluded.contains(value)) {
continue;
}
// LUCENE 4 UPGRADE: use Lucene's RegexCapabilities
if (matcher != null && !matcher.reset(value.utf8ToString()).matches()) {
continue;
}
InternalStringTermsFacet.TermEntry entry =
new InternalStringTermsFacet.TermEntry(value, count);
ordered.insertWithOverflow(entry);
}
}
InternalStringTermsFacet.TermEntry[] list =
new InternalStringTermsFacet.TermEntry[ordered.size()];
for (int i = ordered.size() - 1; i >= 0; i--) {
list[i] = (InternalStringTermsFacet.TermEntry) ordered.pop();
}
for (ReaderAggregator aggregator : aggregators) {
CacheRecycler.pushIntArray(aggregator.counts);
}
return new InternalStringTermsFacet(
facetName, comparatorType, size, Arrays.asList(list), missing, total);
}
BoundedTreeSet<InternalStringTermsFacet.TermEntry> ordered =
new BoundedTreeSet<InternalStringTermsFacet.TermEntry>(comparatorType.comparator(), size);
while (queue.size() > 0) {
ReaderAggregator agg = queue.top();
BytesRef value =
agg.values.makeSafe(
agg.current); // we need to makeSafe it, since we end up pushing it... (can we work
// around that?)
int count = 0;
do {
count += agg.counts[agg.position];
if (agg.nextPosition()) {
agg = queue.updateTop();
} else {
// we are done with this reader
queue.pop();
agg = queue.top();
}
} while (agg != null && value.equals(agg.current));
if (count > minCount) {
if (excluded != null && excluded.contains(value)) {
continue;
}
// LUCENE 4 UPGRADE: use Lucene's RegexCapabilities
if (matcher != null && !matcher.reset(value.utf8ToString()).matches()) {
continue;
}
InternalStringTermsFacet.TermEntry entry =
new InternalStringTermsFacet.TermEntry(value, count);
ordered.add(entry);
}
}
for (ReaderAggregator aggregator : aggregators) {
CacheRecycler.pushIntArray(aggregator.counts);
}
return new InternalStringTermsFacet(facetName, comparatorType, size, ordered, missing, total);
}
@Override
public Facet facet() {
if (current != null) {
missing += current.counts[0];
total += current.total - current.counts[0];
// if we have values for this one, add it
if (current.values.length > 1) {
aggregators.add(current);
}
}
AggregatorPriorityQueue queue = new AggregatorPriorityQueue(aggregators.size());
for (ReaderAggregator aggregator : aggregators) {
if (aggregator.nextPosition()) {
queue.add(aggregator);
}
}
// YACK, we repeat the same logic, but once with an optimizer priority queue for smaller sizes
if (size < EntryPriorityQueue.LIMIT) {
// optimize to use priority size
EntryPriorityQueue ordered = new EntryPriorityQueue(size, comparatorType.comparator());
while (queue.size() > 0) {
ReaderAggregator agg = queue.top();
short value = agg.current;
int count = 0;
do {
count += agg.counts[agg.position];
if (agg.nextPosition()) {
agg = queue.updateTop();
} else {
// we are done with this reader
queue.pop();
agg = queue.top();
}
} while (agg != null && value == agg.current);
if (count > minCount) {
if (excluded == null || !excluded.contains(value)) {
InternalShortTermsFacet.ShortEntry entry =
new InternalShortTermsFacet.ShortEntry(value, count);
ordered.insertWithOverflow(entry);
}
}
}
InternalShortTermsFacet.ShortEntry[] list =
new InternalShortTermsFacet.ShortEntry[ordered.size()];
for (int i = ordered.size() - 1; i >= 0; i--) {
list[i] = (InternalShortTermsFacet.ShortEntry) ordered.pop();
}
for (ReaderAggregator aggregator : aggregators) {
CacheRecycler.pushIntArray(aggregator.counts);
}
return new InternalShortTermsFacet(
facetName, comparatorType, size, Arrays.asList(list), missing, total);
}
BoundedTreeSet<InternalShortTermsFacet.ShortEntry> ordered =
new BoundedTreeSet<InternalShortTermsFacet.ShortEntry>(comparatorType.comparator(), size);
while (queue.size() > 0) {
ReaderAggregator agg = queue.top();
short value = agg.current;
int count = 0;
do {
count += agg.counts[agg.position];
if (agg.nextPosition()) {
agg = queue.updateTop();
} else {
// we are done with this reader
queue.pop();
agg = queue.top();
}
} while (agg != null && value == agg.current);
if (count > minCount) {
if (excluded == null || !excluded.contains(value)) {
InternalShortTermsFacet.ShortEntry entry =
new InternalShortTermsFacet.ShortEntry(value, count);
ordered.add(entry);
}
}
}
for (ReaderAggregator aggregator : aggregators) {
CacheRecycler.pushIntArray(aggregator.counts);
}
return new InternalShortTermsFacet(facetName, comparatorType, size, ordered, missing, total);
}