private void rescale(long newLandmarkInSeconds) { // rescale the weights based on a new landmark to avoid numerical overflow issues final double factor = Math.exp(-alpha * (newLandmarkInSeconds - landmarkInSeconds)); weightedCount *= factor; postOrderTraversal( root, new Callback() { @Override public boolean process(Node node) { double oldWeight = node.weightedCount; node.weightedCount *= factor; if (oldWeight >= ZERO_WEIGHT_THRESHOLD && node.weightedCount < ZERO_WEIGHT_THRESHOLD) { --nonZeroNodeCount; } return true; } }); landmarkInSeconds = newLandmarkInSeconds; }
public void serialize(final DataOutput output) { try { output.writeDouble(maxError); output.writeDouble(alpha); output.writeLong(landmarkInSeconds); output.writeLong(min); output.writeLong(max); output.writeInt(totalNodeCount); postOrderTraversal( root, new Callback() { @Override public boolean process(Node node) { try { serializeNode(output, node); } catch (IOException e) { Throwables.propagate(e); } return true; } }); } catch (IOException e) { Throwables.propagate(e); } }
@VisibleForTesting void compress() { ++compressions; final int compressionFactor = calculateCompressionFactor(); postOrderTraversal( root, new Callback() { @Override public boolean process(Node node) { if (node.isLeaf()) { return true; } // if children's weights are ~0 remove them and shift the weight to their parent double leftWeight = 0; if (node.left != null) { leftWeight = node.left.weightedCount; } double rightWeight = 0; if (node.right != null) { rightWeight = node.right.weightedCount; } boolean shouldCompress = node.weightedCount + leftWeight + rightWeight < (int) (weightedCount / compressionFactor); double oldNodeWeight = node.weightedCount; if (shouldCompress || leftWeight < ZERO_WEIGHT_THRESHOLD) { node.left = tryRemove(node.left); weightedCount += leftWeight; node.weightedCount += leftWeight; } if (shouldCompress || rightWeight < ZERO_WEIGHT_THRESHOLD) { node.right = tryRemove(node.right); weightedCount += rightWeight; node.weightedCount += rightWeight; } if (oldNodeWeight < ZERO_WEIGHT_THRESHOLD && node.weightedCount >= ZERO_WEIGHT_THRESHOLD) { ++nonZeroNodeCount; } return true; } }); if (root != null && root.weightedCount < ZERO_WEIGHT_THRESHOLD) { root = tryRemove(root); } }
public String toGraphviz() { StringBuilder builder = new StringBuilder(); builder.append("digraph QuantileDigest {\n").append("\tgraph [ordering=\"out\"];"); final List<Node> nodes = new ArrayList<>(); postOrderTraversal( root, new Callback() { @Override public boolean process(Node node) { nodes.add(node); return true; } }); Multimap<Integer, Node> nodesByLevel = Multimaps.index( nodes, new Function<Node, Integer>() { @Override public Integer apply(Node input) { return input.level; } }); for (Map.Entry<Integer, Collection<Node>> entry : nodesByLevel.asMap().entrySet()) { builder.append("\tsubgraph level_" + entry.getKey() + " {\n").append("\t\trank = same;\n"); for (Node node : entry.getValue()) { builder.append( String.format( "\t\t%s [label=\"[%s..%s]@%s\\n%s\", shape=rect, style=filled,color=%s];\n", idFor(node), node.getLowerBound(), node.getUpperBound(), node.level, node.weightedCount, node.weightedCount > 0 ? "salmon2" : "white")); } builder.append("\t}\n"); } for (Node node : nodes) { if (node.left != null) { builder.append(format("\t%s -> %s;\n", idFor(node), idFor(node.left))); } if (node.right != null) { builder.append(format("\t%s -> %s;\n", idFor(node), idFor(node.right))); } } builder.append("}\n"); return builder.toString(); }
/* * Get the exponentially-decayed approximate counts of values in multiple buckets. The elements in * the provided list denote the upper bound each of the buckets and must be sorted in ascending * order. * * The approximate count in each bucket is guaranteed to be within 2 * totalCount * maxError of * the real count. */ public List<Bucket> getHistogram(List<Long> bucketUpperBounds) { checkArgument( Ordering.natural().isOrdered(bucketUpperBounds), "buckets must be sorted in increasing order"); final ImmutableList.Builder<Bucket> builder = ImmutableList.builder(); final PeekingIterator<Long> iterator = Iterators.peekingIterator(bucketUpperBounds.iterator()); final AtomicDouble sum = new AtomicDouble(); final AtomicDouble lastSum = new AtomicDouble(); // for computing weighed average of values in bucket final AtomicDouble bucketWeightedSum = new AtomicDouble(); final double normalizationFactor = weight(TimeUnit.NANOSECONDS.toSeconds(ticker.read())); postOrderTraversal( root, new Callback() { @Override public boolean process(Node node) { while (iterator.hasNext() && iterator.peek() <= node.getUpperBound()) { double bucketCount = sum.get() - lastSum.get(); Bucket bucket = new Bucket( bucketCount / normalizationFactor, bucketWeightedSum.get() / bucketCount); builder.add(bucket); lastSum.set(sum.get()); bucketWeightedSum.set(0); iterator.next(); } bucketWeightedSum.addAndGet(node.getMiddle() * node.weightedCount); sum.addAndGet(node.weightedCount); return iterator.hasNext(); } }); while (iterator.hasNext()) { double bucketCount = sum.get() - lastSum.get(); Bucket bucket = new Bucket(bucketCount / normalizationFactor, bucketWeightedSum.get() / bucketCount); builder.add(bucket); iterator.next(); } return builder.build(); }
public long getMax() { final AtomicLong chosen = new AtomicLong(max); postOrderTraversal( root, new Callback() { @Override public boolean process(Node node) { if (node.weightedCount >= ZERO_WEIGHT_THRESHOLD) { chosen.set(node.getUpperBound()); return false; } return true; } }, TraversalOrder.REVERSE); return Math.min(max, chosen.get()); }
@VisibleForTesting void validate() { final AtomicDouble sumOfWeights = new AtomicDouble(); final AtomicInteger actualNodeCount = new AtomicInteger(); final AtomicInteger actualNonZeroNodeCount = new AtomicInteger(); if (root != null) { validateStructure(root); postOrderTraversal( root, new Callback() { @Override public boolean process(Node node) { sumOfWeights.addAndGet(node.weightedCount); actualNodeCount.incrementAndGet(); if (node.weightedCount >= ZERO_WEIGHT_THRESHOLD) { actualNonZeroNodeCount.incrementAndGet(); } return true; } }); } checkState( Math.abs(sumOfWeights.get() - weightedCount) < ZERO_WEIGHT_THRESHOLD, "Computed weight (%s) doesn't match summary (%s)", sumOfWeights.get(), weightedCount); checkState( actualNodeCount.get() == totalNodeCount, "Actual node count (%s) doesn't match summary (%s)", actualNodeCount.get(), totalNodeCount); checkState( actualNonZeroNodeCount.get() == nonZeroNodeCount, "Actual non-zero node count (%s) doesn't match summary (%s)", actualNonZeroNodeCount.get(), nonZeroNodeCount); }
/** * Gets the values at the specified quantiles +/- maxError. The list of quantiles must be sorted * in increasing order, and each value must be in the range [0, 1] */ public List<Long> getQuantiles(List<Double> quantiles) { checkArgument( Ordering.natural().isOrdered(quantiles), "quantiles must be sorted in increasing order"); for (double quantile : quantiles) { checkArgument(quantile >= 0 && quantile <= 1, "quantile must be between [0,1]"); } final ImmutableList.Builder<Long> builder = ImmutableList.builder(); final PeekingIterator<Double> iterator = Iterators.peekingIterator(quantiles.iterator()); postOrderTraversal( root, new Callback() { private double sum = 0; @Override public boolean process(Node node) { sum += node.weightedCount; while (iterator.hasNext() && sum > iterator.peek() * weightedCount) { iterator.next(); // we know the max value ever seen, so cap the percentile to provide better error // bounds in this case long value = Math.min(node.getUpperBound(), max); builder.add(value); } return iterator.hasNext(); } }); // we finished the traversal without consuming all quantiles. This means the remaining quantiles // correspond to the max known value while (iterator.hasNext()) { builder.add(max); iterator.next(); } return builder.build(); }