@Test
public void testIndexTask() throws Exception {
final Task indexTask =
new IndexTask(
null,
null,
new IndexTask.IndexIngestionSpec(
new DataSchema(
"foo",
null,
new AggregatorFactory[] {new DoubleSumAggregatorFactory("met", "met")},
new UniformGranularitySpec(
Granularity.DAY, null, ImmutableList.of(new Interval("2010-01-01/P2D"))),
mapper),
new IndexTask.IndexIOConfig(new MockFirehoseFactory(false)),
new IndexTask.IndexTuningConfig(10000, 10, -1, indexSpec)),
mapper,
null);
final Optional<TaskStatus> preRunTaskStatus = tsqa.getStatus(indexTask.getId());
Assert.assertTrue("pre run task status not present", !preRunTaskStatus.isPresent());
final TaskStatus mergedStatus = runTask(indexTask);
final TaskStatus status = ts.getStatus(indexTask.getId()).get();
final List<DataSegment> publishedSegments = byIntervalOrdering.sortedCopy(mdc.getPublished());
final List<DataSegment> loggedSegments =
byIntervalOrdering.sortedCopy(tsqa.getInsertedSegments(indexTask.getId()));
Assert.assertEquals("statusCode", TaskStatus.Status.SUCCESS, status.getStatusCode());
Assert.assertEquals(
"merged statusCode", TaskStatus.Status.SUCCESS, mergedStatus.getStatusCode());
Assert.assertEquals("segments logged vs published", loggedSegments, publishedSegments);
Assert.assertEquals("num segments published", 2, mdc.getPublished().size());
Assert.assertEquals("num segments nuked", 0, mdc.getNuked().size());
Assert.assertEquals("segment1 datasource", "foo", publishedSegments.get(0).getDataSource());
Assert.assertEquals(
"segment1 interval",
new Interval("2010-01-01/P1D"),
publishedSegments.get(0).getInterval());
Assert.assertEquals(
"segment1 dimensions",
ImmutableList.of("dim1", "dim2"),
publishedSegments.get(0).getDimensions());
Assert.assertEquals(
"segment1 metrics", ImmutableList.of("met"), publishedSegments.get(0).getMetrics());
Assert.assertEquals("segment2 datasource", "foo", publishedSegments.get(1).getDataSource());
Assert.assertEquals(
"segment2 interval",
new Interval("2010-01-02/P1D"),
publishedSegments.get(1).getInterval());
Assert.assertEquals(
"segment2 dimensions",
ImmutableList.of("dim1", "dim2"),
publishedSegments.get(1).getDimensions());
Assert.assertEquals(
"segment2 metrics", ImmutableList.of("met"), publishedSegments.get(1).getMetrics());
}
@Override
public List<OProperty> listProperties(
OClass oClass, String tab, final DisplayMode mode, final Boolean extended) {
Collection<OProperty> properties = oClass.properties();
final String safeTab = tab != null ? tab : DEFAULT_TAB;
final UIComponentsRegistry registry = OrienteerWebApplication.get().getUIComponentsRegistry();
Collection<OProperty> filteredProperties =
Collections2.filter(
properties,
new Predicate<OProperty>() {
@Override
public boolean apply(OProperty input) {
String propertyTab = CustomAttributes.TAB.getValue(input);
boolean ret = safeTab.equals(propertyTab != null ? propertyTab : DEFAULT_TAB);
if (extended == null) return ret;
else {
CustomAttributes attr =
DisplayMode.EDIT.equals(mode)
? CustomAttributes.EDIT_COMPONENT
: CustomAttributes.VIEW_COMPONENT;
String component = attr.getValue(input);
if (component == null) return !extended;
return registry.getComponentFactory(mode, input.getType(), component).isExtended()
== extended;
}
}
});
if (filteredProperties == null || filteredProperties.isEmpty()) filteredProperties = properties;
return ORDER_PROPERTIES_BY_ORDER.sortedCopy(filteredProperties);
}
@Override
public Iterator<? extends T> iterator(long first, long count) {
Collection<T> data = dataModel.getObject();
if (data == null || data.size() == 0) return Iterators.emptyIterator();
Iterator<T> it;
final SortParam<S> sortParam = getSort();
if (sortParam != null && sortParam.getProperty() != null) {
Ordering<T> ordering =
Ordering.natural()
.nullsFirst()
.onResultOf(
new Function<T, Comparable<?>>() {
@Override
public Comparable<?> apply(T input) {
return comparableValue(input, sortParam.getProperty());
}
});
if (!sortParam.isAscending()) ordering = ordering.reverse();
it = ordering.sortedCopy(data).iterator();
} else {
it = data.iterator();
}
if (first > 0) Iterators.advance(it, (int) first);
return count >= 0 ? Iterators.limit(it, (int) count) : it;
}
public void testSortedCopy() {
List<Integer> unsortedInts = Collections.unmodifiableList(Arrays.asList(5, 0, 3, null, 0, 9));
List<Integer> sortedInts = numberOrdering.nullsLast().sortedCopy(unsortedInts);
assertEquals(Arrays.asList(0, 0, 3, 5, 9, null), sortedInts);
assertEquals(
Collections.emptyList(), numberOrdering.sortedCopy(Collections.<Integer>emptyList()));
}
@Override
public List<OProperty> getDisplayableProperties(OClass oClass) {
Collection<OProperty> properties = oClass.properties();
Collection<OProperty> filteredProperties =
Collections2.filter(properties, PropertyDisplayablePredicate.INSTANCE);
if (filteredProperties == null || filteredProperties.isEmpty()) filteredProperties = properties;
return ORDER_PROPERTIES_BY_ORDER.sortedCopy(filteredProperties);
}
void testSortedCopy() {
List<T> shuffledList = Lists.newArrayList(strictlyOrderedList);
shuffledList = shuffledCopy(shuffledList, new Random(5));
assertEquals(strictlyOrderedList, ordering.sortedCopy(shuffledList));
if (!strictlyOrderedList.contains(null)) {
assertEquals(strictlyOrderedList, ordering.immutableSortedCopy(shuffledList));
}
}
public BrokenDownSummaryConfusionMatrix<SignatureType> build() {
final ImmutableMap.Builder<SignatureType, SummaryConfusionMatrix> ret =
ImmutableMap.builder();
for (final SignatureType key : keyOrdering.sortedCopy(data.keySet())) {
ret.put(key, data.get(key).build());
}
return BrokenDownSummaryConfusionMatrix.fromMap(ret.build());
}
/**
* Get a flat list of validation messages
*
* <p>One message has the form:
*
* <pre>
* #/pointer/here: message here
* </pre>
*
* <p>The list is sorted by pointer.
*
* @return the list of messages
*/
public List<String> getMessages() {
final Iterable<JsonPointer> paths = Ordering.natural().sortedCopy(msgMap.keySet());
final ImmutableList.Builder<String> builder = ImmutableList.builder();
List<Message> messages;
for (final JsonPointer path : paths) {
messages = MESSAGE_ORDER.sortedCopy(msgMap.get(path));
for (final Message msg : messages) builder.add(path + ": " + msg);
}
return builder.build();
}
public void testAllEqual() {
Ordering<Object> comparator = Ordering.allEqual();
assertSame(comparator, comparator.reverse());
assertEquals(comparator.compare(null, null), 0);
assertEquals(comparator.compare(new Object(), new Object()), 0);
assertEquals(comparator.compare("apples", "oranges"), 0);
assertSame(comparator, reserialize(comparator));
assertEquals("Ordering.allEqual()", comparator.toString());
List<String> strings = ImmutableList.of("b", "a", "d", "c");
assertEquals(strings, comparator.sortedCopy(strings));
assertEquals(strings, comparator.immutableSortedCopy(strings));
}
/**
* Retrieve all messages as a JSON object
*
* <p>The retrieved JSON document is an object where:
*
* <ul>
* <li>keys are string representations of {@link JsonPointer}s,
* <li>values are arrays of objects where each individual object is the JSON representation of
* one message.
* </ul>
*
* <p>Note: the returned {@link JsonNode} is mutable.
*
* @see Message#toJsonNode()
* @return a JSON object with all validation messages
*/
public JsonNode asJsonObject() {
final ObjectNode ret = JsonNodeFactory.instance.objectNode();
ArrayNode node;
List<Message> messages;
for (final JsonPointer ptr : msgMap.keySet()) {
node = JsonNodeFactory.instance.arrayNode();
messages = MESSAGE_ORDER.sortedCopy(msgMap.get(ptr));
for (final Message message : messages) node.add(message.toJsonNode());
ret.put(ptr.toString(), node);
}
return ret;
}
/**
* Enumerates all the fields in this state interface.
*
* @param clazz a subclass of AccumulatorState
* @return list of state fields. Ordering is guaranteed to be stable, and have all primitive
* fields at the beginning.
*/
private static List<StateField> enumerateFields(Class<?> clazz) {
ImmutableList.Builder<StateField> builder = ImmutableList.builder();
final Set<Class<?>> primitiveClasses =
ImmutableSet.<Class<?>>of(byte.class, boolean.class, long.class, double.class);
Set<Class<?>> supportedClasses =
ImmutableSet.<Class<?>>of(byte.class, boolean.class, long.class, double.class, Slice.class);
for (Method method : clazz.getMethods()) {
if (method.getName().equals("getEstimatedSize")) {
continue;
}
if (method.getName().startsWith("get")) {
Class<?> type = method.getReturnType();
checkArgument(supportedClasses.contains(type), type.getName() + " is not supported");
String name = method.getName().substring(3);
builder.add(new StateField(name, type, getInitialValue(method)));
}
if (method.getName().startsWith("is")) {
Class<?> type = method.getReturnType();
checkArgument(type == boolean.class, "Only boolean is support for 'is' methods");
String name = method.getName().substring(2);
builder.add(new StateField(name, type, getInitialValue(method), method.getName()));
}
}
// We need this ordering because the serializer and deserializer are on different machines, and
// so the ordering of fields must be stable
Ordering<StateField> ordering =
new Ordering<StateField>() {
@Override
public int compare(StateField left, StateField right) {
if (primitiveClasses.contains(left.getType())
&& !primitiveClasses.contains(right.getType())) {
return -1;
}
if (primitiveClasses.contains(right.getType())
&& !primitiveClasses.contains(left.getType())) {
return 1;
}
// If they're the category, just sort by name
return left.getName().compareTo(right.getName());
}
};
List<StateField> fields = ordering.sortedCopy(builder.build());
checkInterface(clazz, fields);
return fields;
}
private static void runLeastOfComparison(int iterations, int elements, int seeds) {
Random random = new Random(42);
Ordering<Integer> ordering = Ordering.natural();
for (int i = 0; i < iterations; i++) {
List<Integer> list = Lists.newArrayList();
for (int j = 0; j < elements; j++) {
list.add(random.nextInt(10 * i + j + 1));
}
for (int seed = 1; seed < seeds; seed++) {
int k = random.nextInt(10 * seed);
assertEquals(ordering.sortedCopy(list).subList(0, k), ordering.leastOf(list, k));
}
}
}
private void appendChildComments(
List<Comment> target, ListMultimap<Long, Comment> byParent, long id, String op) {
Ordering<Comment> ordering = COMMENT_BY_CONFIDENCE;
if (op != null && prioritizeOpComments) {
ordering =
Ordering.natural()
.reverse()
.onResultOf((Comment c) -> op.equalsIgnoreCase(c.getName()))
.compound(ordering);
}
List<Comment> children = ordering.sortedCopy(byParent.get(id));
for (Comment child : children) {
target.add(child);
appendChildComments(target, byParent, (int) child.getId(), op);
}
}
/**
* Return the list of validation messages as a JSON array
*
* <p>This method makes its best to order validation messages correctly.
*
* <p>Each message in the resulting array is a JSON object, with the contents of the {@link
* Message} and with an added member named {@code path}, which contains the path into the instance
* where the error has occurred (as a {@link JsonPointer}).
*
* @see Message#toJsonNode()
* @return a JSON array with all validation messages
*/
public JsonNode asJsonArray() {
final ArrayNode ret = JsonNodeFactory.instance.arrayNode();
ObjectNode node;
final Iterable<JsonPointer> paths = Ordering.natural().sortedCopy(msgMap.keySet());
List<Message> messages;
for (final JsonPointer ptr : paths) {
messages = MESSAGE_ORDER.sortedCopy(msgMap.get(ptr));
for (final Message msg : messages) {
node = JsonNodeFactory.instance.objectNode().put("path", ptr.toString());
// I hate to do that...
node.putAll((ObjectNode) msg.toJsonNode());
ret.add(node);
}
}
return ret;
}
public void renderTo(CharSink sink) throws IOException {
checkNotNull(corefAnnotation);
final StringBuilder sb = new StringBuilder();
sb.append(htmlHeader());
sb.append(javascript());
sb.append(bodyHeader());
sb.append("<h1>");
sb.append(docID);
sb.append("</h1>\n");
for (final String type : Ordering.natural().sortedCopy(typeToResponse.keySet())) {
final Multimap<TypeRoleFillerRealis, Response> trfrToResponse =
Multimaps.index(
typeToResponse.get(type),
TypeRoleFillerRealis.extractFromSystemResponse(
corefAnnotation.laxCASNormalizerFunction()));
sb.append(href(type));
sb.append("<h2>");
sb.append(type);
sb.append("</h2>\n");
sb.append(closehref());
sb.append("<div id=\"");
sb.append(type);
sb.append("\" style=\"display:none\">\n");
sb.append("<ul>\n");
for (final TypeRoleFillerRealis trfr : trfrOrdering.sortedCopy(trfrToResponse.keySet())) {
final String trfrID = String.format("%s.%s", type, trfr.role().asString());
sb.append("<li>\n");
sb.append(String.format("<div id=\"%s\" style=\"display:inherit\" >", trfrID));
sb.append("<h3>");
sb.append(
String.format(
"%s-%s:%s - %s",
trfr.type().asString(),
trfr.role().asString(),
trfr.realis().name(),
trfr.argumentCanonicalString().string()));
sb.append("</h3>\n");
addSection(
sb,
trfrID,
Iterables.transform(
trfrToResponse.get(trfr),
new Function<Response, String>() {
@Override
public String apply(final Response r) {
return String.format(
"%s: %s", r.realis().name(), r.canonicalArgument().string());
}
}));
sb.append("</div>\n");
sb.append("</li>\n");
}
sb.append("</ul>\n");
sb.append("</div>\n");
sb.append("</li>\n");
}
sb.append(bodyFooter());
sb.append(htmlFooter());
sink.write(sb.toString());
}
private static List<AssociationRule> getRuleValues(
HasRuleValues hasRuleValues, final OutputField outputField) {
List<AssociationRule> associationRules =
hasRuleValues.getRuleValues(outputField.getAlgorithm());
Comparator<AssociationRule> comparator =
new Comparator<AssociationRule>() {
private OutputField.RankBasis rankBasis = outputField.getRankBasis();
private OutputField.RankOrder rankOrder = outputField.getRankOrder();
@Override
public int compare(AssociationRule left, AssociationRule right) {
int order;
switch (this.rankBasis) {
case CONFIDENCE:
order = (getConfidence(left)).compareTo(getConfidence(right));
break;
case SUPPORT:
order = (getSupport(left)).compareTo(getSupport(right));
break;
case LIFT:
order = (getLift(left)).compareTo(getLift(right));
break;
case LEVERAGE:
order = (getLeverage(left)).compareTo(getLeverage(right));
break;
case AFFINITY:
order = (getAffinity(left)).compareTo(getAffinity(right));
break;
default:
throw new UnsupportedFeatureException(outputField, this.rankBasis);
} // End switch
switch (this.rankOrder) {
case ASCENDING:
return order;
case DESCENDING:
return -order;
default:
throw new UnsupportedFeatureException(outputField, this.rankOrder);
}
}
private Double getConfidence(AssociationRule rule) {
return checkRuleFeature(rule, rule.getConfidence());
}
private Double getSupport(AssociationRule rule) {
return checkRuleFeature(rule, rule.getSupport());
}
private Double getLift(AssociationRule rule) {
return checkRuleFeature(rule, rule.getLift());
}
private Double getLeverage(AssociationRule rule) {
return checkRuleFeature(rule, rule.getLeverage());
}
private Double getAffinity(AssociationRule rule) {
return checkRuleFeature(rule, rule.getAffinity());
}
private <V> V checkRuleFeature(AssociationRule rule, V value) {
if (value == null) {
throw new InvalidFeatureException(rule);
}
return value;
}
};
Ordering<AssociationRule> ordering = Ordering.from(comparator);
return ordering.sortedCopy(associationRules);
}
public String printJobQueue(Ordering<R> ordering) {
return Joiner.on('\n').join(ordering.sortedCopy(requests.values()));
}
@Test
public void testResumeTasks() throws Exception {
final Task indexTask =
new IndexTask(
null,
null,
new IndexTask.IndexIngestionSpec(
new DataSchema(
"foo",
null,
new AggregatorFactory[] {new DoubleSumAggregatorFactory("met", "met")},
new UniformGranularitySpec(
Granularity.DAY, null, ImmutableList.of(new Interval("2010-01-01/P2D"))),
mapper),
new IndexTask.IndexIOConfig(new MockFirehoseFactory(false)),
new IndexTask.IndexTuningConfig(10000, 10, -1, indexSpec)),
mapper,
null);
final long startTime = System.currentTimeMillis();
// manually insert the task into TaskStorage, waiting for TaskQueue to sync from storage
ts.insert(indexTask, TaskStatus.running(indexTask.getId()));
while (tsqa.getStatus(indexTask.getId()).get().isRunnable()) {
if (System.currentTimeMillis() > startTime + 10 * 1000) {
throw new ISE("Where did the task go?!: %s", indexTask.getId());
}
Thread.sleep(100);
}
final TaskStatus status = ts.getStatus(indexTask.getId()).get();
final List<DataSegment> publishedSegments = byIntervalOrdering.sortedCopy(mdc.getPublished());
final List<DataSegment> loggedSegments =
byIntervalOrdering.sortedCopy(tsqa.getInsertedSegments(indexTask.getId()));
Assert.assertEquals("statusCode", TaskStatus.Status.SUCCESS, status.getStatusCode());
Assert.assertEquals("segments logged vs published", loggedSegments, publishedSegments);
Assert.assertEquals("num segments published", 2, mdc.getPublished().size());
Assert.assertEquals("num segments nuked", 0, mdc.getNuked().size());
Assert.assertEquals("segment1 datasource", "foo", publishedSegments.get(0).getDataSource());
Assert.assertEquals(
"segment1 interval",
new Interval("2010-01-01/P1D"),
publishedSegments.get(0).getInterval());
Assert.assertEquals(
"segment1 dimensions",
ImmutableList.of("dim1", "dim2"),
publishedSegments.get(0).getDimensions());
Assert.assertEquals(
"segment1 metrics", ImmutableList.of("met"), publishedSegments.get(0).getMetrics());
Assert.assertEquals("segment2 datasource", "foo", publishedSegments.get(1).getDataSource());
Assert.assertEquals(
"segment2 interval",
new Interval("2010-01-02/P1D"),
publishedSegments.get(1).getInterval());
Assert.assertEquals(
"segment2 dimensions",
ImmutableList.of("dim1", "dim2"),
publishedSegments.get(1).getDimensions());
Assert.assertEquals(
"segment2 metrics", ImmutableList.of("met"), publishedSegments.get(1).getMetrics());
}
@VisibleForTesting
LocalDate getOptimizedRawUsageStartDate(
final LocalDate firstEventStartDate,
final LocalDate targetDate,
final Iterable<InvoiceItem> existingUsageItems,
final Map<String, Usage> knownUsage) {
if (!existingUsageItems.iterator().hasNext()) {
return firstEventStartDate;
}
// Extract all usage billing period known in that catalog
final Set<BillingPeriod> knownUsageBillingPeriod =
ImmutableSet.copyOf(
Iterables.transform(
knownUsage.values(),
new Function<Usage, BillingPeriod>() {
@Nullable
@Override
public BillingPeriod apply(final Usage input) {
return input.getBillingPeriod();
}
}));
// Make sure all usage items are sorted by endDate
final List<InvoiceItem> sortedUsageItems = USAGE_ITEM_ORDERING.sortedCopy(existingUsageItems);
// Compute an array with one date per BillingPeriod:
// If BillingPeriod is never defined in the catalog (no need to look for items), we initialize
// its value
// such that it cannot be chosen
//
final LocalDate[] perBillingPeriodMostRecentConsumableInArrearItemEndDate =
new LocalDate[BillingPeriod.values().length - 1]; // Exclude the NO_BILLING_PERIOD
int idx = 0;
for (BillingPeriod bp : BillingPeriod.values()) {
if (bp != BillingPeriod.NO_BILLING_PERIOD) {
final LocalDate makerDateThanCannotBeChosenAsTheMinOfAllDates =
targetDate.plusMonths(config.getMaxRawUsagePreviousPeriod() * bp.getNumberOfMonths());
perBillingPeriodMostRecentConsumableInArrearItemEndDate[idx++] =
(knownUsageBillingPeriod.contains(bp))
? null
: makerDateThanCannotBeChosenAsTheMinOfAllDates;
}
}
final ListIterator<InvoiceItem> iterator =
sortedUsageItems.listIterator(sortedUsageItems.size());
while (iterator.hasPrevious()) {
final InvoiceItem previous = iterator.previous();
Preconditions.checkState(previous instanceof UsageInvoiceItem);
final UsageInvoiceItem item = (UsageInvoiceItem) previous;
final Usage usage = knownUsage.get(item.getUsageName());
if (perBillingPeriodMostRecentConsumableInArrearItemEndDate[
usage.getBillingPeriod().ordinal()]
== null) {
perBillingPeriodMostRecentConsumableInArrearItemEndDate[
usage.getBillingPeriod().ordinal()] =
item.getEndDate();
if (!containsNullEntries(perBillingPeriodMostRecentConsumableInArrearItemEndDate)) {
break;
}
}
}
// Extract the min from all the dates
LocalDate targetStartDate = null;
idx = 0;
for (BillingPeriod bp : BillingPeriod.values()) {
if (bp != BillingPeriod.NO_BILLING_PERIOD) {
final LocalDate tmp = perBillingPeriodMostRecentConsumableInArrearItemEndDate[idx];
final LocalDate targetBillingPeriodDate =
tmp != null
? tmp.minusMonths(config.getMaxRawUsagePreviousPeriod() * bp.getNumberOfMonths())
: null;
if (targetStartDate == null
|| (targetBillingPeriodDate != null
&& targetBillingPeriodDate.compareTo(targetStartDate) < 0)) {
targetStartDate = targetBillingPeriodDate;
}
idx++;
}
}
final LocalDate result =
targetStartDate.compareTo(firstEventStartDate) > 0 ? targetStartDate : firstEventStartDate;
return result;
}