/**
* Extracts all the tags we must use to group results.
*
* <ul>
* <li>If a tag has the form {@code name=*} then we'll create one group per value we find for
* that tag.
* <li>If a tag has the form {@code name={v1,v2,..,vN}} then we'll create {@code N} groups.
* </ul>
*
* In the both cases above, {@code name} will be stored in the {@code group_bys} attribute. In the
* second case specifically, the {@code N} values would be stored in {@code group_by_values}, the
* key in this map being {@code name}.
*
* @param tags The tags from which to extract the 'GROUP BY's. Each tag that represents a 'GROUP
* BY' will be removed from the map passed in argument.
*/
private void findGroupBys(final Map<String, String> tags) {
final Iterator<Map.Entry<String, String>> i = tags.entrySet().iterator();
while (i.hasNext()) {
final Map.Entry<String, String> tag = i.next();
final String tagvalue = tag.getValue();
if (tagvalue.equals("*") // 'GROUP BY' with any value.
|| tagvalue.indexOf('|', 1) >= 0) { // Multiple possible values.
if (group_bys == null) {
group_bys = new ArrayList<byte[]>();
}
group_bys.add(tsdb.tag_names.getId(tag.getKey()));
i.remove();
if (tagvalue.charAt(0) == '*') {
continue; // For a 'GROUP BY' with any value, we're done.
}
// 'GROUP BY' with specific values. Need to split the values
// to group on and store their IDs in group_by_values.
final String[] values = Tags.splitString(tagvalue, '|');
if (group_by_values == null) {
group_by_values = new ByteMap<byte[][]>();
}
final short value_width = tsdb.tag_values.width();
final byte[][] value_ids = new byte[values.length][value_width];
group_by_values.put(tsdb.tag_names.getId(tag.getKey()), value_ids);
for (int j = 0; j < values.length; j++) {
final byte[] value_id = tsdb.tag_values.getId(values[j]);
System.arraycopy(value_id, 0, value_ids[j], 0, value_width);
}
}
}
}
/**
* Sets the server-side regexp filter on the scanner. In order to find the rows with the relevant
* tags, we use a server-side filter that matches a regular expression on the row key.
*
* @param scanner The scanner on which to add the filter.
*/
void createAndSetFilter(final Scanner scanner) {
if (group_bys != null) {
Collections.sort(group_bys, Bytes.MEMCMP);
}
final short name_width = tsdb.tag_names.width();
final short value_width = tsdb.tag_values.width();
final short tagsize = (short) (name_width + value_width);
// Generate a regexp for our tags. Say we have 2 tags: { 0 0 1 0 0 2 }
// and { 4 5 6 9 8 7 }, the regexp will be:
// "^.{7}(?:.{6})*\\Q\000\000\001\000\000\002\\E(?:.{6})*\\Q\004\005\006\011\010\007\\E(?:.{6})*$"
final StringBuilder buf =
new StringBuilder(
15 // "^.{N}" + "(?:.{M})*" + "$"
+ ((13 + tagsize) // "(?:.{M})*\\Q" + tagsize bytes + "\\E"
* (tags.size() + (group_bys == null ? 0 : group_bys.size() * 3))));
// In order to avoid re-allocations, reserve a bit more w/ groups ^^^
// Alright, let's build this regexp. From the beginning...
buf.append(
"(?s)" // Ensure we use the DOTALL flag.
+ "^.{")
// ... start by skipping the metric ID and timestamp.
.append(tsdb.metrics.width() + Const.TIMESTAMP_BYTES)
.append("}");
final Iterator<byte[]> tags = this.tags.iterator();
final Iterator<byte[]> group_bys =
(this.group_bys == null ? new ArrayList<byte[]>(0).iterator() : this.group_bys.iterator());
byte[] tag = tags.hasNext() ? tags.next() : null;
byte[] group_by = group_bys.hasNext() ? group_bys.next() : null;
// Tags and group_bys are already sorted. We need to put them in the
// regexp in order by ID, which means we just merge two sorted lists.
do {
// Skip any number of tags.
buf.append("(?:.{").append(tagsize).append("})*\\Q");
if (isTagNext(name_width, tag, group_by)) {
addId(buf, tag);
tag = tags.hasNext() ? tags.next() : null;
} else { // Add a group_by.
addId(buf, group_by);
final byte[][] value_ids = (group_by_values == null ? null : group_by_values.get(group_by));
if (value_ids == null) { // We don't want any specific ID...
buf.append(".{").append(value_width).append('}'); // Any value ID.
} else { // We want specific IDs. List them: /(AAA|BBB|CCC|..)/
buf.append("(?:");
for (final byte[] value_id : value_ids) {
buf.append("\\Q");
addId(buf, value_id);
buf.append('|');
}
// Replace the pipe of the last iteration.
buf.setCharAt(buf.length() - 1, ')');
}
group_by = group_bys.hasNext() ? group_bys.next() : null;
}
} while (tag != group_by); // Stop when they both become null.
// Skip any number of tags before the end.
buf.append("(?:.{").append(tagsize).append("})*$");
scanner.setKeyRegexp(buf.toString(), CHARSET);
}
public String toString() {
final StringBuilder buf = new StringBuilder();
buf.append("TsdbQuery(start_time=")
.append(getStartTime())
.append(", end_time=")
.append(getEndTime())
.append(", metric=")
.append(Arrays.toString(metric));
try {
buf.append(" (").append(tsdb.metrics.getName(metric));
} catch (NoSuchUniqueId e) {
buf.append(" (<").append(e.getMessage()).append('>');
}
try {
buf.append("), tags=").append(Tags.resolveIds(tsdb, tags));
} catch (NoSuchUniqueId e) {
buf.append("), tags=<").append(e.getMessage()).append('>');
}
buf.append(", rate=")
.append(rate)
.append(", nointerpolation=")
.append(noInterpolation)
.append(", aggregator=")
.append(aggregator)
.append(", group_bys=(");
if (group_bys != null) {
for (final byte[] tag_id : group_bys) {
try {
buf.append(tsdb.tag_names.getName(tag_id));
} catch (NoSuchUniqueId e) {
buf.append('<').append(e.getMessage()).append('>');
}
buf.append(' ').append(Arrays.toString(tag_id));
if (group_by_values != null) {
final byte[][] value_ids = group_by_values.get(tag_id);
if (value_ids == null) {
continue;
}
buf.append("={");
for (final byte[] value_id : value_ids) {
try {
buf.append(tsdb.tag_values.getName(value_id));
} catch (NoSuchUniqueId e) {
buf.append('<').append(e.getMessage()).append('>');
}
buf.append(' ').append(Arrays.toString(value_id)).append(", ");
}
buf.append('}');
}
buf.append(", ");
}
}
buf.append("))");
return buf.toString();
}
/**
* Creates the {@link SpanGroup}s to form the final results of this query.
*
* @param spans The {@link Span}s found for this query ({@link #findSpans}). Can be {@code null},
* in which case the array returned will be empty.
* @return A possibly empty array of {@link SpanGroup}s built according to any 'GROUP BY'
* formulated in this query.
*/
private DataPoints[] groupByAndAggregate(final TreeMap<byte[], Span> spans) {
if (spans == null || spans.size() <= 0) {
return NO_RESULT;
}
if (group_bys == null) {
// We haven't been asked to find groups, so let's put all the spans
// together in the same group.
final SpanGroup group =
new SpanGroup(
tsdb,
getScanStartTime(),
getScanEndTime(),
spans.values(),
rate,
this.noInterpolation,
aggregator,
sample_interval,
downsampler);
return new SpanGroup[] {group};
}
// Maps group value IDs to the SpanGroup for those values. Say we've
// been asked to group by two things: foo=* bar=* Then the keys in this
// map will contain all the value IDs combinations we've seen. If the
// name IDs for `foo' and `bar' are respectively [0, 0, 7] and [0, 0, 2]
// then we'll have group_bys=[[0, 0, 2], [0, 0, 7]] (notice it's sorted
// by ID, so bar is first) and say we find foo=LOL bar=OMG as well as
// foo=LOL bar=WTF and that the IDs of the tag values are:
// LOL=[0, 0, 1] OMG=[0, 0, 4] WTF=[0, 0, 3]
// then the map will have two keys:
// - one for the LOL-OMG combination: [0, 0, 1, 0, 0, 4] and,
// - one for the LOL-WTF combination: [0, 0, 1, 0, 0, 3].
final ByteMap<SpanGroup> groups = new ByteMap<SpanGroup>();
final short value_width = tsdb.tag_values.width();
final byte[] group = new byte[group_bys.size() * value_width];
for (final Map.Entry<byte[], Span> entry : spans.entrySet()) {
final byte[] row = entry.getKey();
byte[] value_id = null;
int i = 0;
// TODO(tsuna): The following loop has a quadratic behavior. We can
// make it much better since both the row key and group_bys are sorted.
for (final byte[] tag_id : group_bys) {
value_id = Tags.getValueId(tsdb, row, tag_id);
if (value_id == null) {
break;
}
System.arraycopy(value_id, 0, group, i, value_width);
i += value_width;
}
if (value_id == null) {
LOG.error(
"WTF? Dropping span for row "
+ Arrays.toString(row)
+ " as it had no matching tag from the requested groups,"
+ " which is unexpected. Query="
+ this);
continue;
}
// LOG.info("Span belongs to group " + Arrays.toString(group) + ": " + Arrays.toString(row));
SpanGroup thegroup = groups.get(group);
if (thegroup == null) {
thegroup =
new SpanGroup(
tsdb,
getScanStartTime(),
getScanEndTime(),
null,
rate,
noInterpolation,
aggregator,
sample_interval,
downsampler);
// Copy the array because we're going to keep `group' and overwrite
// its contents. So we want the collection to have an immutable copy.
final byte[] group_copy = new byte[group.length];
System.arraycopy(group, 0, group_copy, 0, group.length);
groups.put(group_copy, thegroup);
}
thegroup.add(entry.getValue());
}
// for (final Map.Entry<byte[], SpanGroup> entry : groups) {
// LOG.info("group for " + Arrays.toString(entry.getKey()) + ": " + entry.getValue());
// }
return groups.values().toArray(new SpanGroup[groups.size()]);
}