@Override
public void process() throws ProcessingException {
final PreprocessingContext preprocessingContext =
preprocessing.preprocess(documents, "", LanguageCode.ENGLISH);
clusters = Lists.newArrayListWithCapacity(preprocessingContext.allTokens.image.length);
for (char[] token : preprocessingContext.allTokens.image) {
if (token != null) {
clusters.add(new Cluster(new String(token)));
}
}
}
/**
* Performs the actual clustering with an assumption that all documents are written in one <code>
* language</code>.
*/
private void cluster(LanguageCode language) {
clusters = new ArrayList<Cluster>();
/*
* Step 1. Preprocessing: tokenization, stop word marking and stemming (if available).
*/
context = preprocessingPipeline.preprocess(documents, query, language);
/*
* Step 2: Create a generalized suffix tree from phrases in the input.
*/
sb = new GeneralizedSuffixTree.SequenceBuilder();
final int[] tokenIndex = context.allTokens.wordIndex;
final short[] tokenType = context.allTokens.type;
for (int i = 0; i < tokenIndex.length; i++) {
/* Advance until the first real token. */
if (tokenIndex[i] == -1) {
if ((tokenType[i] & (ITokenizer.TF_SEPARATOR_DOCUMENT | ITokenizer.TF_TERMINATOR)) != 0) {
sb.endDocument();
}
continue;
}
/* We have the first token. Advance until non-token. */
final int s = i;
while (tokenIndex[i + 1] != -1) i++;
final int phraseLenght = 1 + i - s;
if (phraseLenght >= 1) {
/* We have a phrase. */
sb.addPhrase(tokenIndex, s, phraseLenght);
}
}
sb.buildSuffixTree();
/*
* Step 3: Find "base" clusters by looking up frequently recurring phrases in the
* generalized suffix tree.
*/
final ArrayList<ClusterCandidate> baseClusters = createBaseClusters(sb);
/*
* Step 4: Merge base clusters that overlap too much to form final clusters.
*/
final ArrayList<ClusterCandidate> mergedClusters = createMergedClusters(baseClusters);
/*
* Step 5: Create the junk (unassigned documents) cluster and create the final
* set of clusters in Carrot2 format.
*/
postProcessing(mergedClusters);
}
@Override
public void process() throws ProcessingException {
final PreprocessingContext preprocessingContext =
preprocessing.preprocess(documents, "", LanguageCode.ENGLISH);
final AllTokens allTokens = preprocessingContext.allTokens;
final AllWords allWords = preprocessingContext.allWords;
final AllStems allStems = preprocessingContext.allStems;
clusters = Lists.newArrayListWithCapacity(allTokens.image.length);
for (int i = 0; i < allTokens.image.length; i++) {
if (allTokens.wordIndex[i] >= 0) {
clusters.add(
new Cluster(new String(allStems.image[allWords.stemIndex[allTokens.wordIndex[i]]])));
}
}
}
/**
* Performs the actual clustering with an assumption that all documents are written in one <code>
* language</code>.
*/
private void cluster(LanguageCode language) {
clusters = new ArrayList<Cluster>();
context = preprocessingPipeline.preprocess(documents, null, language);
}