@Override
public byte[] encode() {
byte[] bytes = new byte[encodedSize()];
bytes[0] = (byte) 0x80;
BitWriter bitWriter = new BitWriter(bytes, 0, bytes.length, 5, true);
int lastSetBit = -1;
for (int setPos = bitSet.nextSetBit(0); setPos >= 0; setPos = bitSet.nextSetBit(setPos + 1)) {
// skip the distance between setPos and lastSetBit
bitWriter.skip(setPos - lastSetBit - 1);
/*
* Because this field is present, we need to use 2 bits to indicate the
* type information necessary to parse. The format for the type bit is
*
* Untyped: 00
* Double: 01
* Float: 10
* Scalar: 11
*/
if (scalarFields != null && scalarFields.get(setPos)) {
bitWriter.set(3);
} else if (floatFields != null && floatFields.get(setPos)) {
bitWriter.set(2);
bitWriter.skipNext();
} else if (doubleFields != null && doubleFields.get(setPos)) {
bitWriter.setNext();
bitWriter.skipNext();
bitWriter.setNext();
} else {
bitWriter.setNext();
bitWriter.skip(2);
}
lastSetBit = setPos;
}
return bytes;
}
/**
* Computes stem indices of words that are one-word label candidates or are non-stop words from
* phrase label candidates.
*/
private int[] computeRequiredStemIndices(PreprocessingContext context) {
final int[] labelsFeatureIndex = context.allLabels.featureIndex;
final int[] wordsStemIndex = context.allWords.stemIndex;
final short[] wordsTypes = context.allWords.type;
final int[][] phrasesWordIndices = context.allPhrases.wordIndices;
final int wordCount = wordsStemIndex.length;
final int[][] stemsTfByDocument = context.allStems.tfByDocument;
int documentCount = context.documents.size();
final BitSet requiredStemIndices = new BitSet(labelsFeatureIndex.length);
for (int i = 0; i < labelsFeatureIndex.length; i++) {
final int featureIndex = labelsFeatureIndex[i];
if (featureIndex < wordCount) {
addStemIndex(
wordsStemIndex, documentCount, stemsTfByDocument, requiredStemIndices, featureIndex);
} else {
final int[] wordIndices = phrasesWordIndices[featureIndex - wordCount];
for (int j = 0; j < wordIndices.length; j++) {
final int wordIndex = wordIndices[j];
if (!TokenTypeUtils.isCommon(wordsTypes[wordIndex])) {
addStemIndex(
wordsStemIndex, documentCount, stemsTfByDocument, requiredStemIndices, wordIndex);
}
}
}
}
return requiredStemIndices.asIntLookupContainer().toArray();
}
@Override
public int cardinality(int position) {
int count = 0;
for (int i = bitSet.nextSetBit(0); i >= 0 && i < position; i = bitSet.nextSetBit(i + 1)) {
count++;
}
return count;
}
@Override
public int hashCode() {
int result = bitSet.hashCode();
result = 31 * result + (scalarFields != null ? scalarFields.hashCode() : 0);
result = 31 * result + (floatFields != null ? floatFields.hashCode() : 0);
result = 31 * result + (doubleFields != null ? doubleFields.hashCode() : 0);
return result;
}
public static void main(final String[] args) {
final com.carrotsearch.hppc.BitSet hppcBitSet =
new com.carrotsearch.hppc.BitSet(Long.MAX_VALUE);
hppcBitSet.set(Integer.MAX_VALUE);
final java.util.BitSet javaBitSet = new java.util.BitSet(Integer.MAX_VALUE);
javaBitSet.set(Integer.MAX_VALUE - 1);
System.out.println(ObjectSizeCalculator.getObjectSize(hppcBitSet));
System.out.println(ObjectSizeCalculator.getObjectSize(javaBitSet));
}
/** Collect documents from a bitset. */
private List<Document> collectDocuments(List<Document> l, BitSet bitset) {
if (l == null) {
l = Lists.newArrayListWithCapacity((int) bitset.cardinality());
}
final BitSetIterator i = bitset.iterator();
for (int d = i.nextSetBit(); d >= 0; d = i.nextSetBit()) {
l.add(documents.get(d));
}
return l;
}
@Override
public BitSet findMatchings(
T expectedElement, List<T> annotatorResult, BitSet alreadyUsedResults) {
BitSet matchings;
matchings =
matchingsCounter[0].findMatchings(expectedElement, annotatorResult, alreadyUsedResults);
for (int i = 1; (i < matchingsCounter.length) && (!matchings.isEmpty()); i++) {
matchings.intersect(
matchingsCounter[i].findMatchings(expectedElement, annotatorResult, alreadyUsedResults));
}
return matchings;
}
private ByteEntryAccumulator getKeyAccumulator() {
if (indexKeyAccumulator == null) {
BitSet keyFields = new BitSet();
for (int keyColumn : mainColToIndexPosMap) {
if (keyColumn >= 0) keyFields.set(keyColumn);
}
indexKeyAccumulator =
new ByteEntryAccumulator(EntryPredicateFilter.emptyPredicate(), keyFields);
}
return indexKeyAccumulator;
}
public static BitIndex wrap(byte[] data, int position, int limit) {
// create a BitSet underneath
BitSet bitSet = new BitSet();
BitSet scalarFields = new BitSet();
BitSet floatFields = new BitSet();
BitSet doubleFields = new BitSet();
BitReader bitReader = new BitReader(data, position, limit, 5, true);
int bitPos = 0;
while (bitReader.hasNext()) {
int zeros = bitReader.nextSetBit();
if (zeros < 0) break;
bitPos += zeros;
bitSet.set(bitPos);
if (bitReader.next() != 0) {
// either float or scalar
if (bitReader.next() != 0) {
scalarFields.set(bitPos);
} else floatFields.set(bitPos);
} else {
// either a double or untyped
if (bitReader.next() != 0) doubleFields.set(bitPos);
}
bitPos++;
}
return new UncompressedBitIndex(bitSet, scalarFields, floatFields, doubleFields);
}
@Test
public void testAlwaysAcceptEntryWorks() throws Exception {
BitSet fields = new BitSet();
fields.set(0);
fields.set(2);
EntryPredicateFilter predicateFilter = new EntryPredicateFilter(fields);
EntryAccumulator accumulator = new ByteEntryAccumulator(predicateFilter, false, null);
byte[] encodedOne = Encoding.encode(1);
accumulator.add(2, encodedOne, 0, encodedOne.length);
byte[] encodedTwo = Encoding.encode(2);
accumulator.add(0, encodedTwo, 0, encodedTwo.length);
byte[] bytes = accumulator.finish();
MultiFieldDecoder decoder = MultiFieldDecoder.wrap(bytes);
Assert.assertEquals(2, decoder.decodeNextInt());
Assert.assertEquals(1, decoder.decodeNextInt());
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (!(o instanceof UncompressedBitIndex)) return false;
UncompressedBitIndex that = (UncompressedBitIndex) o;
if (!bitSet.equals(that.bitSet)) return false;
if (doubleFields != null ? !doubleFields.equals(that.doubleFields) : that.doubleFields != null)
return false;
if (floatFields != null ? !floatFields.equals(that.floatFields) : that.floatFields != null)
return false;
if (scalarFields != null ? !scalarFields.equals(that.scalarFields) : that.scalarFields != null)
return false;
return true;
}
/**
* Do we need to update the index, i.e. did any of the values change?
*
* @param mutation
* @param indexedColumns
* @return
*/
public boolean areIndexKeysModified(KVPair mutation, BitSet indexedColumns) {
EntryDecoder newPutDecoder = new EntryDecoder();
newPutDecoder.set(mutation.getValue());
BitIndex updateIndex = newPutDecoder.getCurrentIndex();
for (int i = updateIndex.nextSetBit(0); i >= 0; i = updateIndex.nextSetBit(i + 1)) {
if (indexedColumns.get(i)) return true;
}
return false;
}
/**
* Create the junk (unassigned documents) cluster and create the final set of clusters in Carrot2
* format.
*/
private void postProcessing(ArrayList<ClusterCandidate> clusters) {
// Adapt to Carrot2 classes, counting used documents on the way.
final BitSet all = new BitSet(documents.size());
final ArrayList<Document> docs = Lists.newArrayListWithCapacity(documents.size());
final ArrayList<String> phrases = Lists.newArrayListWithCapacity(3);
for (ClusterCandidate c : clusters) {
final Cluster c2 = new Cluster();
c2.addPhrases(collectPhrases(phrases, c));
c2.addDocuments(collectDocuments(docs, c.documents));
c2.setScore((double) c.score);
this.clusters.add(c2);
all.or(c.documents);
docs.clear();
phrases.clear();
}
Cluster.appendOtherTopics(this.documents, this.clusters);
}
private EntryEncoder getRowEncoder() {
if (indexValueEncoder == null) {
BitSet nonNullFields = new BitSet();
int highestSetPosition = 0;
for (int keyColumn : mainColToIndexPosMap) {
if (keyColumn > highestSetPosition) highestSetPosition = keyColumn;
}
nonNullFields.set(highestSetPosition + 1);
indexValueEncoder =
EntryEncoder.create(
SpliceKryoRegistry.getInstance(),
1,
nonNullFields,
new BitSet(),
new BitSet(),
new BitSet());
}
return indexValueEncoder;
}
/** Adds stem index to the set with a check on the stem's document frequency. */
private void addStemIndex(
final int[] wordsStemIndex,
int documentCount,
int[][] stemsTfByDocument,
final BitSet requiredStemIndices,
final int featureIndex) {
final int stemIndex = wordsStemIndex[featureIndex];
final int df = stemsTfByDocument[stemIndex].length / 2;
if (((double) df / documentCount) <= maxWordDf) {
requiredStemIndices.set(stemIndex);
}
}
@Override
public int encodedSize() {
/*
* The number of bytes goes as follows:
*
* you need at least as many bits as the highest 1-bit in the bitSet(equivalent
* to bitSet.length()). Because each set bit will have an additional 2-bit "type delimiter"
* set afterwords, we need to have 3 bits for every set bit, but 1 for every non-set bit
*
* This is equivalent to length()+2*numSetBits().
*
* we have 4 available bits in the header, and 7 bits in each subsequent byte (we use a continuation
* bit).
*/
int numBits = (int) (bitSet.length() + 2 * bitSet.cardinality());
int numBytes = 1;
numBits -= 4;
if (numBits > 0) {
numBytes += numBits / 7;
if (numBits % 7 != 0) numBytes++;
}
return numBytes;
}
private DataResult fetchBaseRow(KVPair mutation, WriteContext ctx, BitSet indexedColumns)
throws IOException {
baseGet =
SIDriver.driver()
.getOperationFactory()
.newDataGet(ctx.getTxn(), mutation.getRowKey(), baseGet);
EntryPredicateFilter epf;
if (indexedColumns != null && indexedColumns.size() > 0) {
epf = new EntryPredicateFilter(indexedColumns);
} else epf = EntryPredicateFilter.emptyPredicate();
TransactionalRegion region = ctx.txnRegion();
TxnFilter txnFilter = region.packedFilter(ctx.getTxn(), epf, false);
baseGet.setFilter(txnFilter);
baseResult = ctx.getRegion().get(baseGet, baseResult);
return baseResult;
}
/**
* Translate the given base table record mutation into its associated, referencing index record.
* <br>
* Encapsulates the logic required to create an index record for a given base table record with
* all the required discriminating and encoding rules (column is part of a PK, value is null,
* etc).
*
* @param mutation KVPair containing the rowKey of the base table record for which we want to
* translate to the associated index. This mutation should already have its requred {@link
* KVPair.Type Type} set.
* @return A KVPair representing the index record of the given base table mutation. This KVPair is
* suitable for performing the required modification of the index record associated with this
* mutation.
* @throws IOException for encoding/decoding problems.
*/
public KVPair translate(KVPair mutation) throws IOException {
if (mutation == null) {
return null;
}
EntryAccumulator keyAccumulator = getKeyAccumulator();
keyAccumulator.reset();
boolean hasNullKeyFields = false;
/*
* Handle index columns from the source table's primary key.
*/
if (table.getColumnOrderingCount() > 0) {
// we have key columns to check
MultiFieldDecoder keyDecoder = getSrcKeyDecoder();
keyDecoder.set(mutation.getRowKey());
for (int i = 0; i < table.getColumnOrderingCount(); i++) {
int sourceKeyColumnPos = table.getColumnOrdering(i);
int indexKeyPos =
sourceKeyColumnPos < mainColToIndexPosMap.length
? mainColToIndexPosMap[sourceKeyColumnPos]
: -1;
int offset = keyDecoder.offset();
boolean isNull = skip(keyDecoder, table.getFormatIds(sourceKeyColumnPos));
if (!indexedCols.get(sourceKeyColumnPos)) continue;
if (indexKeyPos >= 0) {
/*
* since primary keys have an implicit NOT NULL constraint here, we don't need to check for it,
* and isNull==true would represent a programmer error, rather than an actual state the
* system can be in.
*/
assert !isNull : "Programmer error: Cannot update a primary key to a null value!";
int length = keyDecoder.offset() - offset - 1;
/*
* A note about sort order:
*
* We are in the primary key section, which means that the element is ordered in
* ASCENDING order. In an ideal world, that wouldn't matter because
*/
accumulate(
keyAccumulator,
indexKeyPos,
table.getFormatIds(sourceKeyColumnPos),
index.getDescColumns(indexKeyPos),
keyDecoder.array(),
offset,
length);
}
}
}
/*
* Handle non-null index columns from the source tables non-primary key columns.
*
* this will set indexed columns with values taken from the incoming mutation (rather than
* backfilling them with existing values, which would occur elsewhere).
*/
EntryDecoder rowDecoder = getSrcValueDecoder();
rowDecoder.set(mutation.getValue());
BitIndex bitIndex = rowDecoder.getCurrentIndex();
MultiFieldDecoder rowFieldDecoder = rowDecoder.getEntryDecoder();
for (int i = bitIndex.nextSetBit(0); i >= 0; i = bitIndex.nextSetBit(i + 1)) {
if (!indexedCols.get(i)) {
// skip non-indexed columns
rowDecoder.seekForward(rowFieldDecoder, i);
continue;
}
int keyColumnPos = i < mainColToIndexPosMap.length ? mainColToIndexPosMap[i] : -1;
if (keyColumnPos < 0) {
rowDecoder.seekForward(rowFieldDecoder, i);
} else {
int offset = rowFieldDecoder.offset();
boolean isNull = rowDecoder.seekForward(rowFieldDecoder, i);
hasNullKeyFields = isNull || hasNullKeyFields;
int length;
if (!isNull) {
length = rowFieldDecoder.offset() - offset - 1;
accumulate(
keyAccumulator,
keyColumnPos,
table.getFormatIds(i),
index.getDescColumns(keyColumnPos),
rowFieldDecoder.array(),
offset,
length);
} else {
/*
* because the field is NULL and it's source is the incoming mutation, we
* still need to accumulate it. We must be careful, however, to accumulate the
* proper null value.
*
* In theory, we could use a sparse encoding here--just accumulate a length 0 entry,
* which will allow us to use a very short row key to determine nullity. However, that
* doesn't work correctly, because doubles and floats at the end of the index might decode
* the row key as a double, resulting in goofball answers.
*
* Instead, we must use the dense encoding approach here. That means that we must
* select the proper dense type based on columnTypes[i]. For most data types, this is still
* a length-0 array, but for floats and doubles it will put the proper type into place.
*/
accumulateNull(keyAccumulator, keyColumnPos, table.getFormatIds(i));
}
}
}
/*
* Handle NULL index columns from the source tables non-primary key columns.
*/
for (int srcColIndex = 0; srcColIndex < mainColToIndexPosMap.length; srcColIndex++) {
/* position of the source column within the index encoding */
int indexColumnPosition = mainColToIndexPosMap[srcColIndex];
if (!isSourceColumnPrimaryKey(srcColIndex)
&& indexColumnPosition >= 0
&& !bitIndex.isSet(srcColIndex)) {
hasNullKeyFields = true;
keyAccumulator.add(indexColumnPosition, new byte[] {}, 0, 0);
}
}
// add the row key to the end of the index key
byte[] srcRowKey = Encoding.encodeBytesUnsorted(mutation.getRowKey());
EntryEncoder rowEncoder = getRowEncoder();
MultiFieldEncoder entryEncoder = rowEncoder.getEntryEncoder();
entryEncoder.reset();
entryEncoder.setRawBytes(srcRowKey);
byte[] indexValue = rowEncoder.encode();
byte[] indexRowKey;
if (index.getUnique()) {
boolean nonUnique =
index.getUniqueWithDuplicateNulls() && (hasNullKeyFields || !keyAccumulator.isFinished());
indexRowKey = getIndexRowKey(srcRowKey, nonUnique);
} else indexRowKey = getIndexRowKey(srcRowKey, true);
return new KVPair(indexRowKey, indexValue, mutation.getType());
}
/**
* Performs the actual clustering with an assumption that all documents are written in one <code>
* language</code>.
*/
private void cluster(LanguageCode language) {
// Preprocessing of documents
final PreprocessingContext context =
preprocessingPipeline.preprocess(documents, query, language);
// Further processing only if there are words to process
clusters = Lists.newArrayList();
if (context.hasLabels()) {
// Term-document matrix building and reduction
final VectorSpaceModelContext vsmContext = new VectorSpaceModelContext(context);
final ReducedVectorSpaceModelContext reducedVsmContext =
new ReducedVectorSpaceModelContext(vsmContext);
LingoProcessingContext lingoContext = new LingoProcessingContext(reducedVsmContext);
matrixBuilder.buildTermDocumentMatrix(vsmContext);
matrixBuilder.buildTermPhraseMatrix(vsmContext);
matrixReducer.reduce(
reducedVsmContext, computeClusterCount(desiredClusterCountBase, documents.size()));
// Cluster label building
clusterBuilder.buildLabels(lingoContext, matrixBuilder.termWeighting);
// Document assignment
clusterBuilder.assignDocuments(lingoContext);
// Cluster merging
clusterBuilder.merge(lingoContext);
// Format final clusters
final int[] clusterLabelIndex = lingoContext.clusterLabelFeatureIndex;
final BitSet[] clusterDocuments = lingoContext.clusterDocuments;
final double[] clusterLabelScore = lingoContext.clusterLabelScore;
for (int i = 0; i < clusterLabelIndex.length; i++) {
final Cluster cluster = new Cluster();
final int labelFeature = clusterLabelIndex[i];
if (labelFeature < 0) {
// Cluster removed during merging
continue;
}
// Add label and score
cluster.addPhrases(labelFormatter.format(context, labelFeature));
cluster.setAttribute(Cluster.SCORE, clusterLabelScore[i]);
// Add documents
final BitSet bs = clusterDocuments[i];
for (int bit = bs.nextSetBit(0); bit >= 0; bit = bs.nextSetBit(bit + 1)) {
cluster.addDocuments(documents.get(bit));
}
// Add cluster
clusters.add(cluster);
}
Collections.sort(clusters, Cluster.byReversedWeightedScoreAndSizeComparator(scoreWeight));
}
Cluster.appendOtherTopics(documents, clusters);
}
@Override
public int nextSetBit(int currentPosition) {
return bitSet.nextSetBit(currentPosition);
}
/**
* Create final clusters by merging base clusters and pruning their labels. Cluster merging is a
* greedy process of compacting clusters with document sets that overlap by a certain ratio. In
* other words, phrases that "cover" nearly identical document sets will be conflated.
*/
private ArrayList<ClusterCandidate> createMergedClusters(
ArrayList<ClusterCandidate> baseClusters) {
/*
* Calculate overlap between base clusters first, saving adjacency lists for
* each base cluster.
*/
// [i] - next neighbor or END, [i + 1] - neighbor cluster index.
final int END = -1;
final IntStack neighborList = new IntStack();
neighborList.push(END);
final int[] neighbors = new int[baseClusters.size()];
final float m = (float) mergeThreshold;
for (int i = 0; i < baseClusters.size(); i++) {
for (int j = i + 1; j < baseClusters.size(); j++) {
final ClusterCandidate c1 = baseClusters.get(i);
final ClusterCandidate c2 = baseClusters.get(j);
final float a = c1.cardinality;
final float b = c2.cardinality;
final float c = BitSet.intersectionCount(c1.documents, c2.documents);
if (c / a > m && c / b > m) {
neighborList.push(neighbors[i], j);
neighbors[i] = neighborList.size() - 2;
neighborList.push(neighbors[j], i);
neighbors[j] = neighborList.size() - 2;
}
}
}
/*
* Find connected components in the similarity graph using Tarjan's algorithm
* (flattened to use the stack instead of recursion).
*/
final int NO_INDEX = -1;
final int[] merged = new int[baseClusters.size()];
Arrays.fill(merged, NO_INDEX);
final ArrayList<ClusterCandidate> mergedClusters =
Lists.newArrayListWithCapacity(baseClusters.size());
final IntStack stack = new IntStack(baseClusters.size());
final IntStack mergeList = new IntStack(baseClusters.size());
int mergedIndex = 0;
for (int v = 0; v < baseClusters.size(); v++) {
if (merged[v] != NO_INDEX) continue;
// Recursively mark all connected components from an unmerged cluster.
stack.push(v);
while (stack.size() > 0) {
final int c = stack.pop();
assert merged[c] == NO_INDEX || merged[c] == mergedIndex;
if (merged[c] == mergedIndex) continue;
merged[c] = mergedIndex;
mergeList.push(c);
for (int i = neighbors[c]; neighborList.get(i) != END; ) {
final int neighbor = neighborList.get(i + 1);
if (merged[neighbor] == NO_INDEX) {
stack.push(neighbor);
} else {
assert merged[neighbor] == mergedIndex;
}
i = neighborList.get(i);
}
}
mergedIndex++;
/*
* Aggregate documents from each base cluster of the current merge, compute
* the score and labels.
*/
mergedClusters.add(merge(mergeList, baseClusters));
mergeList.clear();
}
/*
* Sort merged clusters.
*/
Collections.sort(
mergedClusters,
new Comparator<ClusterCandidate>() {
public int compare(ClusterCandidate c1, ClusterCandidate c2) {
if (c1.score < c2.score) return 1;
if (c1.score > c2.score) return -1;
if (c1.cardinality < c2.cardinality) return 1;
if (c1.cardinality > c2.cardinality) return -1;
return 0;
};
});
if (mergedClusters.size() > maxClusters) {
mergedClusters.subList(maxClusters, mergedClusters.size()).clear();
}
return mergedClusters;
}
@Override
public BitSet and(BitSet bitSet) {
final BitSet result = (BitSet) this.bitSet.clone();
result.and(bitSet);
return result;
}
@Override
public boolean isSet(int pos) {
return bitSet.get(pos);
}
@Override
public int length() {
return (int) bitSet.length();
}
@Override
public boolean isDoubleType(int position) {
return doubleFields != null && doubleFields.get(position);
}
@Override
public boolean isScalarType(int position) {
return scalarFields != null && scalarFields.get(position);
}
@Override
public boolean isFloatType(int position) {
return floatFields != null && floatFields.get(position);
}
@Override
public int cardinality() {
return (int) bitSet.cardinality();
}
@Override
public boolean isEmpty() {
return bitSet.isEmpty();
}