/**
* Group files with similar min timestamp into buckets. Files with recent min timestamps are
* grouped together into buckets designated to short timespans while files with older timestamps
* are grouped into buckets representing longer timespans.
*
* @param files pairs consisting of a file and its min timestamp
* @param timeUnit
* @param base
* @param now
* @return a list of buckets of files. The list is ordered such that the files with newest
* timestamps come first. Each bucket is also a list of files ordered from newest to oldest.
*/
@VisibleForTesting
static <T> List<List<T>> getBuckets(
Collection<Pair<T, Long>> files, long timeUnit, int base, long now) {
// Sort files by age. Newest first.
final List<Pair<T, Long>> sortedFiles = Lists.newArrayList(files);
Collections.sort(
sortedFiles,
Collections.reverseOrder(
new Comparator<Pair<T, Long>>() {
public int compare(Pair<T, Long> p1, Pair<T, Long> p2) {
return p1.right.compareTo(p2.right);
}
}));
List<List<T>> buckets = Lists.newArrayList();
Target target = getInitialTarget(now, timeUnit);
PeekingIterator<Pair<T, Long>> it = Iterators.peekingIterator(sortedFiles.iterator());
outerLoop:
while (it.hasNext()) {
while (!target.onTarget(it.peek().right)) {
// If the file is too new for the target, skip it.
if (target.compareToTimestamp(it.peek().right) < 0) {
it.next();
if (!it.hasNext()) break outerLoop;
} else // If the file is too old for the target, switch targets.
target = target.nextTarget(base);
}
List<T> bucket = Lists.newArrayList();
while (target.onTarget(it.peek().right)) {
bucket.add(it.next().left);
if (!it.hasNext()) break;
}
buckets.add(bucket);
}
return buckets;
}
/**
* A cluster (group) of {@link Document}s. Each cluster has a human-readable label consisting of one
* or more phrases, a list of documents it contains and a list of its subclusters. Optionally,
* additional attributes can be associated with a cluster, e.g. {@link #OTHER_TOPICS}. This class is
* <strong>not</strong> thread-safe.
*/
@Root(name = "group", strict = false)
@JsonAutoDetect(JsonMethod.NONE)
@JsonWriteNullProperties(false)
public final class Cluster {
/**
* Indicates that the cluster is an <i>Other Topics</i> cluster. Such a cluster contains documents
* that remain unclustered at given level of cluster hierarchy.
*
* <p>Type of this attribute is {@link Boolean}.
*
* @see #setAttribute(String, Object)
* @see #getAttribute(String)
*/
public static final String OTHER_TOPICS = "other-topics";
/** Default label for the <i>Other Topics</i> cluster. */
static final String OTHER_TOPICS_LABEL = "Other Topics";
/**
* Score of this cluster that indicates the clustering algorithm's beliefs on the quality of this
* cluster. The exact semantics of the score varies across algorithms.
*
* <p>Type of this attribute is {@link Double}.
*
* @see #setAttribute(String, Object)
* @see #getAttribute(String)
*/
public static final String SCORE = "score";
/** @see #getId() */
@Attribute(required = false)
Integer id;
/** Phrases describing this cluster. */
@ElementList(required = false, name = "title", entry = "phrase")
private ArrayList<String> phrases = new ArrayList<String>();
/** A read-only list of phrases exposed in {@link #getPhrases()}. */
private List<String> phrasesView = Collections.unmodifiableList(phrases);
/** Subclusters of this cluster. */
@ElementList(required = false, inline = true)
private ArrayList<Cluster> subclusters = new ArrayList<Cluster>();
/** A read-only list of subclusters exposed in {@link #getSubclusters()}. */
private List<Cluster> subclustersView = Collections.unmodifiableList(subclusters);
/** Documents contained in this cluster. */
private final ArrayList<Document> documents = new ArrayList<Document>();
/** A read-only list of this cluster's document exposed in {@link #getDocuments()}. */
private final List<Document> documentsView = Collections.unmodifiableList(documents);
/** Attributes of this cluster. */
private Map<String, Object> attributes = new HashMap<String, Object>();
/** A Read-only view of the attributes of this cluster. */
private Map<String, Object> attributesView = Collections.unmodifiableMap(attributes);
/** Cached concatenated label */
private String labelCache = null;
/** Cached list of documents from this cluster and subclusters */
private List<Document> allDocuments;
/** Attributes of this cluster for serialization/ deserialization purposes. */
@ElementMap(entry = "attribute", key = "key", attribute = true, inline = true, required = false)
private HashMap<String, SimpleXmlWrapperValue> otherAttributesForSerialization;
/** List of document ids used for serialization/ deserialization purposes. */
@ElementList(required = false, inline = true)
List<DocumentRefid> documentIds;
/** A helper class for serialization/ deserialization of documents with refids. */
@Root(name = "document")
static class DocumentRefid {
@Attribute Integer refid;
DocumentRefid() {}
DocumentRefid(Integer refid) {
this.refid = refid;
}
}
/** Creates a {@link Cluster} with an empty label, no documents and no subclusters. */
public Cluster() {}
/**
* Creates a {@link Cluster} with the provided <code>phrase</code> to be used as the cluster's
* label and <code>documents</code> contained in the cluster.
*
* @param phrase the phrase to form the cluster's label
* @param documents documents contained in the cluster
*/
public Cluster(String phrase, Document... documents) {
addPhrases(phrase);
addDocuments(documents);
}
/**
* Formats this cluster's label. If there is more than one phrase describing this cluster, phrases
* will be separated by a comma followed by a space, e.g. "Phrase one, Phrase two". To format
* multi-phrase label in a different way, use {@link #getPhrases()}.
*
* @return formatted label of this cluster
*/
public String getLabel() {
if (labelCache == null) {
labelCache = StringUtils.toString(phrases, ", ");
}
return labelCache;
}
/**
* Returns all phrases describing this cluster. The returned list is unmodifiable.
*
* @return phrases describing this cluster
*/
@JsonProperty
public List<String> getPhrases() {
return phrasesView;
}
/**
* Returns all subclusters of this cluster. The returned list is unmodifiable.
*
* @return subclusters of this cluster
*/
public List<Cluster> getSubclusters() {
return subclustersView;
}
/** For JSON serialization only. */
@JsonProperty("clusters")
@SuppressWarnings("unused")
private List<Cluster> getSubclustersForSerialization() {
return subclustersView.isEmpty() ? null : subclustersView;
}
/**
* Returns all documents contained in this cluster. The returned list is unmodifiable.
*
* @return documents contained in this cluster
*/
public List<Document> getDocuments() {
return documentsView;
}
/**
* Returns all documents contained in this cluster and (recursively) all documents from this
* cluster's subclusters. The returned list contains unique documents, i.e. if a document is
* attached to multiple subclusters if this cluster, the document will appear only once on the
* list. The documents are enumerated in breadth first order, i.e. first come documents returned
* by {@link #getDocuments()} and then documents from subclusters.
*
* @return all documents from this cluster and its subclusters
*/
public List<Document> getAllDocuments() {
if (allDocuments == null) {
allDocuments =
new ArrayList<Document>(collectAllDocuments(this, new LinkedHashSet<Document>()));
}
return allDocuments;
}
/**
* Returns all documents in this cluster ordered according to the provided comparator. See {@link
* Document} for common comparators, e.g. {@link Document#BY_ID_COMPARATOR} .
*/
public List<Document> getAllDocuments(Comparator<Document> comparator) {
final List<Document> sortedDocuments = Lists.newArrayList(getAllDocuments());
Collections.sort(sortedDocuments, comparator);
return sortedDocuments;
}
/** A recursive routine for collecting unique documents from this cluster and subclusters. */
private static Set<Document> collectAllDocuments(Cluster cluster, Set<Document> docs) {
if (cluster == null) {
return docs;
}
docs.addAll(cluster.getDocuments());
final List<Cluster> subclusters = cluster.getSubclusters();
for (final Cluster subcluster : subclusters) {
collectAllDocuments(subcluster, docs);
}
return docs;
}
/**
* Adds phrases to the description of this cluster.
*
* @param phrases to be added to the description of this cluster
* @return this cluster for convenience
*/
public Cluster addPhrases(String... phrases) {
labelCache = null;
for (final String phrase : phrases) {
this.phrases.add(phrase);
}
return this;
}
/**
* Adds phrases to the description of this cluster.
*
* @param phrases to be added to the description of this cluster
* @return this cluster for convenience
*/
public Cluster addPhrases(Iterable<String> phrases) {
labelCache = null;
for (final String phrase : phrases) {
this.phrases.add(phrase);
}
return this;
}
/**
* Adds document to this cluster.
*
* @param documents to be added to this cluster
* @return this cluster for convenience
*/
public Cluster addDocuments(Document... documents) {
for (final Document document : documents) {
this.documents.add(document);
}
allDocuments = null;
return this;
}
/**
* Adds document to this cluster.
*
* @param documents to be added to this cluster
* @return this cluster for convenience
*/
public Cluster addDocuments(Iterable<Document> documents) {
for (final Document document : documents) {
this.documents.add(document);
}
allDocuments = null;
return this;
}
/**
* Adds subclusters to this cluster
*
* @param subclusters to be added to this cluster
* @return this cluster for convenience
*/
public Cluster addSubclusters(Cluster... subclusters) {
for (final Cluster cluster : subclusters) {
this.subclusters.add(cluster);
}
allDocuments = null;
return this;
}
/**
* Adds subclusters to this cluster
*
* @param clusters to be added to this cluster
* @return this cluster for convenience
*/
public Cluster addSubclusters(Iterable<Cluster> clusters) {
for (final Cluster cluster : clusters) {
this.subclusters.add(cluster);
}
allDocuments = null;
return this;
}
/** Returns this cluster's {@value #SCORE} field. */
@JsonProperty
@Attribute(required = false)
public Double getScore() {
return getAttribute(SCORE);
}
/**
* Sets this cluster's {@link #SCORE} field.
*
* @param score score to set
* @return this cluster for convenience
*/
@Attribute(required = false)
public Cluster setScore(Double score) {
return setAttribute(SCORE, score);
}
/**
* Returns the attribute associated with this cluster under the provided <code>key</code>. If
* there is no attribute under the provided <code>key</code>, <code>null</code> will be returned.
*
* @param key of the attribute
* @return attribute value of <code>null</code>
*/
@SuppressWarnings("unchecked")
public <T> T getAttribute(String key) {
return (T) attributes.get(key);
}
/**
* Associates an attribute with this cluster.
*
* @param key for the attribute
* @param value for the attribute
* @return this cluster for convenience
*/
public <T> Cluster setAttribute(String key, T value) {
synchronized (attributes) {
attributes.put(key, value);
}
return this;
}
/**
* Returns all attributes of this cluster. The returned map is unmodifiable.
*
* @return all attributes of this cluster
*/
public Map<String, Object> getAttributes() {
return attributesView;
}
/**
* Returns the size of the cluster calculated as the number of unique documents it contains,
* including its subclusters.
*
* @return size of the cluster
*/
public int size() {
return getAllDocuments().size();
}
/** For serialization only. */
@JsonProperty
@Attribute(required = false)
@SuppressWarnings("unused")
private int getSize() {
return size();
}
/** Empty implementation, SimpleXML requires both a getter and a setter. */
@Attribute(required = false)
@SuppressWarnings("unused")
private void setSize(int size) {
// We only serialize the size, hence empty implementation
}
/**
* Internal identifier of this cluster within the {@link ProcessingResult}. This identifier is
* assigned dynamically after clusters are passed to {@link ProcessingResult}.
*
* @see ProcessingResult
*/
@JsonProperty
public Integer getId() {
return id;
}
/** Returns <code>true</code> if this cluster is the {@link #OTHER_TOPICS} cluster. */
public boolean isOtherTopics() {
final Boolean otherTopics = getAttribute(OTHER_TOPICS);
return otherTopics != null && otherTopics.booleanValue();
}
/**
* Sets the {@link #OTHER_TOPICS} attribute of this cluster.
*
* @param isOtherTopics if <code>true</code>, this cluster will be marked as an <i>Other
* Topics</i> cluster.
* @return this cluster for convenience
*/
public Cluster setOtherTopics(boolean isOtherTopics) {
return setAttribute(OTHER_TOPICS, isOtherTopics).setScore(0.0);
}
/**
* Compares clusters by size as returned by {@link #size()}. Clusters with more documents are
* larger.
*/
public static final Comparator<Cluster> BY_SIZE_COMPARATOR =
Ordering.natural()
.nullsFirst()
.onResultOf(
new Function<Cluster, Integer>() {
public Integer apply(Cluster cluster) {
return cluster.size();
}
});
/**
* Compares clusters by score as returned by {@link #SCORE}. Clusters with larger score are
* larger.
*/
public static final Comparator<Cluster> BY_SCORE_COMPARATOR =
Ordering.natural()
.nullsFirst()
.onResultOf(
new Function<Cluster, Double>() {
public Double apply(Cluster cluster) {
return cluster.getAttribute(SCORE);
}
});
/** Compares clusters by the natural order of their labels as returned by {@link #getLabel()}. */
public static final Comparator<Cluster> BY_LABEL_COMPARATOR =
Ordering.natural()
.nullsFirst()
.onResultOf(
new Function<Cluster, String>() {
public String apply(Cluster cluster) {
return cluster.getLabel();
}
});
/**
* Compares clusters first by their size as returned by {@link #size()} and labels as returned by
* {@link #getLabel()}. In case of equal sizes, natural order of the labels decides.
*
* <p><b>Please note</b>: this is a reversed comparator, so "larger" clusters end up nearer the
* beginning of the list being sorted (which is usually the order in which the applications want
* to display clusters).
*/
public static final Comparator<Cluster> BY_REVERSED_SIZE_AND_LABEL_COMPARATOR =
Ordering.from(Collections.reverseOrder(BY_SIZE_COMPARATOR)).compound(BY_LABEL_COMPARATOR);
/**
* Compares clusters first by their size as returned by {@link #SCORE} and labels as returned by
* {@link #getLabel()}. In case of equal scores, natural order of the labels decides.
*
* <p><b>Please note</b>: this is a reversed comparator, so "larger" clusters end up nearer the
* beginning of the list being sorted (which is usually the order in which the applications want
* to display clusters).
*/
public static final Comparator<Cluster> BY_REVERSED_SCORE_AND_LABEL_COMPARATOR =
Ordering.from(Collections.reverseOrder(BY_SCORE_COMPARATOR)).compound(BY_LABEL_COMPARATOR);
/**
* Returns a comparator that compares clusters based on the aggregation of their size and score.
* If <code>scoreWeight</code> is 0.0, the order depends only on cluster sizes. If <code>
* scoreWeight</code> is 1.1, the order depends only on cluster scores. For <code>scoreWeight
* </code> values between 0.0 and 1.0, the higher the <code>scoreWeight</code>, the more
* contribution of cluster scores to the order. In case of a tie on the aggregated cluster size
* and score, clusters are compared by the natural order of their labels.
*
* <p><b>Please note</b>: this is a reversed comparator, so "larger" clusters end up nearer the
* beginning of the list being sorted (which is usually the order in which the applications want
* to display clusters).
*/
public static Comparator<Cluster> byReversedWeightedScoreAndSizeComparator(
final double scoreWeight) {
if (scoreWeight < 0 || scoreWeight > 1) {
throw new IllegalArgumentException(
"Score weight must be between 0.0 (inclusive) and 1.0 (inclusive) ");
}
return Ordering.natural()
.onResultOf(
new Function<Cluster, Double>() {
public Double apply(Cluster cluster) {
return -Math.pow(cluster.size(), (1 - scoreWeight))
* Math.pow((Double) cluster.getAttribute(SCORE), scoreWeight);
}
})
.compound(BY_LABEL_COMPARATOR);
}
/**
* A comparator that puts {@link #OTHER_TOPICS} clusters at the end of the list. In other words,
* to this comparator an {@link #OTHER_TOPICS} topics cluster is "bigger" than a non-{{@link
* #OTHER_TOPICS} cluster.
*
* <p><strong>Note:</strong> This comparator is designed for use in combination with other
* comparators, such as {@link #BY_REVERSED_SIZE_AND_LABEL_COMPARATOR}. If you only need to
* partition a list of clusters into regular and other topic ones, this is better done in linear
* time without resorting to {@link Collections#sort(List)}.
*/
public static final Comparator<Cluster> OTHER_TOPICS_AT_THE_END =
Ordering.natural()
.onResultOf(
new Function<Cluster, Double>() {
public Double apply(Cluster cluster) {
return cluster.isOtherTopics() ? 1.0 : -1.0;
}
});
/**
* Assigns sequential identifiers to the provided <code>clusters</code> (and their sub-clusters).
* If a cluster already has an identifier, the identifier will not be changed.
*
* @param clusters Clusters to assign identifiers to.
* @throws IllegalArgumentException if the provided clusters contain non-unique identifiers
*/
public static void assignClusterIds(Collection<Cluster> clusters) {
final ArrayList<Cluster> flattened = Lists.newArrayListWithExpectedSize(clusters.size());
flatten(flattened, clusters);
synchronized (clusters) {
final HashSet<Integer> ids = Sets.newHashSet();
// First, find the start value for the id and check uniqueness of the ids
// already provided.
int maxId = Integer.MIN_VALUE;
for (final Cluster cluster : flattened) {
if (cluster.id != null) {
if (!ids.add(cluster.id)) {
throw new IllegalArgumentException("Non-unique cluster id found: " + cluster.id);
}
maxId = Math.max(maxId, cluster.id);
}
}
// We'd rather start with 0
maxId = Math.max(maxId, -1);
// Assign missing ids
for (final Cluster c : flattened) {
if (c.id == null) {
c.id = ++maxId;
}
}
}
}
/*
* Recursive descent into subclusters.
*/
private static void flatten(ArrayList<Cluster> flattened, Collection<Cluster> clusters) {
for (Cluster c : clusters) {
flattened.add(c);
final List<Cluster> subclusters = c.getSubclusters();
if (!subclusters.isEmpty()) {
flatten(flattened, subclusters);
}
}
}
/**
* Locate the first cluster that has id equal to <code>id</code>. The search includes all the
* clusters in the input and their sub-clusters. The first cluster with matching identifier is
* returned or <code>null</code> if no such cluster could be found.
*/
public static Cluster find(int id, Collection<Cluster> clusters) {
for (Cluster c : clusters) {
if (c != null) {
if (c.id != null && c.id == id) {
return c;
}
if (!c.getSubclusters().isEmpty()) {
final Cluster sub = find(id, c.getSubclusters());
if (sub != null) {
return sub;
}
}
}
}
return null;
}
/**
* Builds an "Other Topics" cluster that groups those documents from <code>allDocument</code> that
* were not referenced in any cluster in <code>clusters</code>.
*
* @param allDocuments all documents to check against
* @param clusters list of clusters with assigned documents
* @return the "Other Topics" cluster
*/
public static Cluster buildOtherTopics(List<Document> allDocuments, List<Cluster> clusters) {
return buildOtherTopics(allDocuments, clusters, OTHER_TOPICS_LABEL);
}
/**
* Builds an "Other Topics" cluster that groups those documents from <code>allDocument</code> that
* were not referenced in any cluster in <code>clusters</code>.
*
* @param allDocuments all documents to check against
* @param clusters list of clusters with assigned documents
* @param label label for the "Other Topics" group
* @return the "Other Topics" cluster
*/
public static Cluster buildOtherTopics(
List<Document> allDocuments, List<Cluster> clusters, String label) {
final Set<Document> unclusteredDocuments = Sets.newLinkedHashSet(allDocuments);
final Set<Document> assignedDocuments = Sets.newHashSet();
for (Cluster cluster : clusters) {
collectAllDocuments(cluster, assignedDocuments);
}
unclusteredDocuments.removeAll(assignedDocuments);
final Cluster otherTopics = new Cluster(label);
otherTopics.addDocuments(unclusteredDocuments);
otherTopics.setOtherTopics(true);
return otherTopics;
}
/**
* If there are unclustered documents, appends the "Other Topics" group to the <code>clusters
* </code>.
*
* @see #buildOtherTopics(List, List)
*/
public static void appendOtherTopics(List<Document> allDocuments, List<Cluster> clusters) {
appendOtherTopics(allDocuments, clusters, OTHER_TOPICS_LABEL);
}
/**
* If there are unclustered documents, appends the "Other Topics" group to the <code>clusters
* </code>.
*
* @see #buildOtherTopics(List, List, String)
*/
public static void appendOtherTopics(
List<Document> allDocuments, List<Cluster> clusters, String label) {
final Cluster otherTopics = buildOtherTopics(allDocuments, clusters, label);
if (!otherTopics.getDocuments().isEmpty()) {
clusters.add(otherTopics);
}
}
@Persist
@SuppressWarnings("unused")
private void beforeSerialization() {
documentIds =
Lists.transform(
documents,
new Function<Document, DocumentRefid>() {
public DocumentRefid apply(Document document) {
return new DocumentRefid(document.getId());
}
});
// Remove score from attributes for serialization
otherAttributesForSerialization = MapUtils.asHashMap(SimpleXmlWrappers.wrap(attributes));
otherAttributesForSerialization.remove(SCORE);
if (otherAttributesForSerialization.isEmpty()) {
otherAttributesForSerialization = null;
}
}
@Commit
@SuppressWarnings("unused")
private void afterDeserialization() throws Exception {
if (otherAttributesForSerialization != null) {
attributes.putAll(SimpleXmlWrappers.unwrap(otherAttributesForSerialization));
}
phrasesView = Collections.unmodifiableList(phrases);
subclustersView = Collections.unmodifiableList(subclusters);
// Documents will be restored on the ProcessingResult level
}
/** For JSON serialization only. */
@JsonProperty("documents")
@SuppressWarnings("unused")
private List<Integer> getDocumentIds() {
return Lists.transform(documents, Document.DocumentToId.INSTANCE);
}
/** For JSON and XML serialization only. */
@JsonProperty("attributes")
@SuppressWarnings("unused")
private Map<String, Object> getOtherAttributes() {
final Map<String, Object> otherAttributes = Maps.newHashMap(attributesView);
return otherAttributes.isEmpty() ? null : otherAttributes;
}
}