@Override
public boolean isValid(List<RouteArc> value, ConstraintValidatorContext context) {
List<RouteArc> source =
value
.stream()
.sorted((o1, o2) -> Integer.compare(o1.getNumber(), o2.getNumber()))
.collect(Collectors.toList());
PeekingIterator<RouteArc> iterator = Iterators.peekingIterator(source.iterator());
List<RouteArc> result = new ArrayList<>();
RouteArc arc;
while (iterator.hasNext()) {
arc = iterator.next();
result.add(arc);
if (iterator.hasNext()) {
if (!arc.getEnd().equals(iterator.peek().getStart())) {
result.add(iterator.next());
context.disableDefaultConstraintViolation();
context
.buildConstraintViolationWithTemplate(context.getDefaultConstraintMessageTemplate())
.addPropertyNode("\n" + Joiner.on(",\n").join(result))
.addConstraintViolation();
return false;
}
}
}
return true;
}
public IterablePagination(
CommandSource src,
PaginationCalculator<CommandSource> calc,
Iterable<Map.Entry<Text, Integer>> counts,
Text title,
Text header,
Text footer,
String padding) {
super(src, calc, title, header, footer, padding);
this.countIterator = Iterators.peekingIterator(counts.iterator());
}
/*
* Get the exponentially-decayed approximate counts of values in multiple buckets. The elements in
* the provided list denote the upper bound each of the buckets and must be sorted in ascending
* order.
*
* The approximate count in each bucket is guaranteed to be within 2 * totalCount * maxError of
* the real count.
*/
public List<Bucket> getHistogram(List<Long> bucketUpperBounds) {
checkArgument(
Ordering.natural().isOrdered(bucketUpperBounds),
"buckets must be sorted in increasing order");
final ImmutableList.Builder<Bucket> builder = ImmutableList.builder();
final PeekingIterator<Long> iterator = Iterators.peekingIterator(bucketUpperBounds.iterator());
final AtomicDouble sum = new AtomicDouble();
final AtomicDouble lastSum = new AtomicDouble();
// for computing weighed average of values in bucket
final AtomicDouble bucketWeightedSum = new AtomicDouble();
final double normalizationFactor = weight(TimeUnit.NANOSECONDS.toSeconds(ticker.read()));
postOrderTraversal(
root,
new Callback() {
@Override
public boolean process(Node node) {
while (iterator.hasNext() && iterator.peek() <= node.getUpperBound()) {
double bucketCount = sum.get() - lastSum.get();
Bucket bucket =
new Bucket(
bucketCount / normalizationFactor, bucketWeightedSum.get() / bucketCount);
builder.add(bucket);
lastSum.set(sum.get());
bucketWeightedSum.set(0);
iterator.next();
}
bucketWeightedSum.addAndGet(node.getMiddle() * node.weightedCount);
sum.addAndGet(node.weightedCount);
return iterator.hasNext();
}
});
while (iterator.hasNext()) {
double bucketCount = sum.get() - lastSum.get();
Bucket bucket =
new Bucket(bucketCount / normalizationFactor, bucketWeightedSum.get() / bucketCount);
builder.add(bucket);
iterator.next();
}
return builder.build();
}
public void commitRows(PartitionName partitionName, List<Row> rows) throws Exception {
LOG.info("Received from partition:{} rows:{}", partitionName, rows.size());
PartitionClient client = partitionClientProvider.getPartition(partitionName);
Consistency consistency =
consistencyCache.computeIfAbsent(
partitionName,
partitionName1 -> {
try {
RingPartitionProperties properties =
partitionClientProvider.getProperties(partitionName1);
return properties != null ? properties.partitionProperties.consistency : null;
} catch (Exception e) {
LOG.error("Failed to get properties for partition:{}", partitionName1);
return null;
}
});
if (consistency == null) {
throw new RuntimeException("Missing consistency for partition: " + partitionName);
}
PeekingIterator<Row> iter = Iterators.peekingIterator(rows.iterator());
while (iter.hasNext()) {
byte[] prefix = iter.peek().prefix;
client.commit(
consistency,
prefix,
commitKeyValueStream -> {
while (iter.hasNext()) {
byte[] peek = iter.peek().prefix;
if ((prefix == null && peek == null)
|| (prefix != null && peek != null && Arrays.equals(prefix, peek))) {
Row row = iter.next();
commitKeyValueStream.commit(
row.key, row.value, row.valueTimestamp, row.valueTombstoned);
} else {
break;
}
}
return true;
},
additionalSolverAfterNMillis,
abandonSolutionAfterNMillis,
Optional.empty());
}
}
private void returnNotifications(ServiceRequest<PublishRequest, PublishResponse> service) {
LinkedHashSet<BaseMonitoredItem<?>> items = new LinkedHashSet<>();
lastIterator.forEachRemaining(items::add);
itemsById
.values()
.stream()
.filter(item -> item.hasNotifications() || item.isTriggered())
.forEach(items::add);
PeekingIterator<BaseMonitoredItem<?>> iterator = Iterators.peekingIterator(items.iterator());
gatherAndSend(iterator, Optional.of(service));
lastIterator = iterator.hasNext() ? iterator : Collections.emptyIterator();
}
/**
* Gets the values at the specified quantiles +/- maxError. The list of quantiles must be sorted
* in increasing order, and each value must be in the range [0, 1]
*/
public List<Long> getQuantiles(List<Double> quantiles) {
checkArgument(
Ordering.natural().isOrdered(quantiles), "quantiles must be sorted in increasing order");
for (double quantile : quantiles) {
checkArgument(quantile >= 0 && quantile <= 1, "quantile must be between [0,1]");
}
final ImmutableList.Builder<Long> builder = ImmutableList.builder();
final PeekingIterator<Double> iterator = Iterators.peekingIterator(quantiles.iterator());
postOrderTraversal(
root,
new Callback() {
private double sum = 0;
@Override
public boolean process(Node node) {
sum += node.weightedCount;
while (iterator.hasNext() && sum > iterator.peek() * weightedCount) {
iterator.next();
// we know the max value ever seen, so cap the percentile to provide better error
// bounds in this case
long value = Math.min(node.getUpperBound(), max);
builder.add(value);
}
return iterator.hasNext();
}
});
// we finished the traversal without consuming all quantiles. This means the remaining quantiles
// correspond to the max known value
while (iterator.hasNext()) {
builder.add(max);
iterator.next();
}
return builder.build();
}
/**
* Group files with similar min timestamp into buckets. Files with recent min timestamps are
* grouped together into buckets designated to short timespans while files with older timestamps
* are grouped into buckets representing longer timespans.
*
* @param files pairs consisting of a file and its min timestamp
* @param timeUnit
* @param base
* @param now
* @return a list of buckets of files. The list is ordered such that the files with newest
* timestamps come first. Each bucket is also a list of files ordered from newest to oldest.
*/
@VisibleForTesting
static <T> List<List<T>> getBuckets(
Collection<Pair<T, Long>> files, long timeUnit, int base, long now) {
// Sort files by age. Newest first.
final List<Pair<T, Long>> sortedFiles = Lists.newArrayList(files);
Collections.sort(
sortedFiles,
Collections.reverseOrder(
new Comparator<Pair<T, Long>>() {
public int compare(Pair<T, Long> p1, Pair<T, Long> p2) {
return p1.right.compareTo(p2.right);
}
}));
List<List<T>> buckets = Lists.newArrayList();
Target target = getInitialTarget(now, timeUnit);
PeekingIterator<Pair<T, Long>> it = Iterators.peekingIterator(sortedFiles.iterator());
outerLoop:
while (it.hasNext()) {
while (!target.onTarget(it.peek().right)) {
// If the file is too new for the target, skip it.
if (target.compareToTimestamp(it.peek().right) < 0) {
it.next();
if (!it.hasNext()) break outerLoop;
} else // If the file is too old for the target, switch targets.
target = target.nextTarget(base);
}
List<T> bucket = Lists.newArrayList();
while (target.onTarget(it.peek().right)) {
bucket.add(it.next().left);
if (!it.hasNext()) break;
}
buckets.add(bucket);
}
return buckets;
}
/**
* obtain an iterator of columns in this memtable in the specified order starting from a given
* column.
*/
public static IColumnIterator getSliceIterator(
final DecoratedKey key,
final ColumnFamily cf,
SliceQueryFilter filter,
AbstractType typeComparator) {
assert cf != null;
final boolean isSuper = cf.isSuper();
final Collection<IColumn> filteredColumns =
filter.reversed ? cf.getReverseSortedColumns() : cf.getSortedColumns();
// ok to not have subcolumnComparator since we won't be adding columns to this object
IColumn startColumn =
isSuper ? new SuperColumn(filter.start, (AbstractType) null) : new Column(filter.start);
Comparator<IColumn> comparator = filter.getColumnComparator(typeComparator);
final PeekingIterator<IColumn> filteredIter =
Iterators.peekingIterator(filteredColumns.iterator());
if (!filter.reversed || filter.start.remaining() != 0) {
while (filteredIter.hasNext() && comparator.compare(filteredIter.peek(), startColumn) < 0) {
filteredIter.next();
}
}
return new AbstractColumnIterator() {
public ColumnFamily getColumnFamily() {
return cf;
}
public DecoratedKey getKey() {
return key;
}
public boolean hasNext() {
return filteredIter.hasNext();
}
public IColumn next() {
return filteredIter.next();
}
};
}
public MergingIterator(
Iterable<? extends Iterator<? extends T>> iterators,
final Comparator<? super T> itemComparator) {
// A comparator that's used by the heap, allowing the heap
// to be sorted based on the top of each iterator.
Comparator<PeekingIterator<T>> heapComparator =
new Comparator<PeekingIterator<T>>() {
@Override
public int compare(PeekingIterator<T> o1, PeekingIterator<T> o2) {
return itemComparator.compare(o1.peek(), o2.peek());
}
};
queue = new PriorityQueue<PeekingIterator<T>>(2, heapComparator);
for (Iterator<? extends T> iterator : iterators) {
if (iterator.hasNext()) {
queue.add(Iterators.peekingIterator(iterator));
}
}
}
/**
* Retrieves non-overlapping bounds for the list of input bounds
*
* <p>Assume we have the following bounds (brackets representing left/right bound): [ ] [ ] [ ] [
* ] [ ] [ ] This method will return the following bounds: [ ] [ ]
*
* @param bounds unsorted bounds to find overlaps
* @return the non-overlapping bounds
*/
public static <T extends RingPosition<T>> Set<Bounds<T>> getNonOverlappingBounds(
Iterable<Bounds<T>> bounds) {
ArrayList<Bounds<T>> sortedBounds = Lists.newArrayList(bounds);
Collections.sort(
sortedBounds,
new Comparator<Bounds<T>>() {
public int compare(Bounds<T> o1, Bounds<T> o2) {
return o1.left.compareTo(o2.left);
}
});
Set<Bounds<T>> nonOverlappingBounds = Sets.newHashSet();
PeekingIterator<Bounds<T>> it = Iterators.peekingIterator(sortedBounds.iterator());
while (it.hasNext()) {
Bounds<T> beginBound = it.next();
Bounds<T> endBound = beginBound;
while (it.hasNext() && endBound.right.compareTo(it.peek().left) >= 0) endBound = it.next();
nonOverlappingBounds.add(new Bounds<>(beginBound.left, endBound.right));
}
return nonOverlappingBounds;
}
/**
* Executes the export operation using the parameters that have been specified.
*
* @return a FeatureCollection with the specified features
*/
@Override
protected SimpleFeatureStore _call() {
final ObjectDatabase database = objectDatabase();
if (filterFeatureTypeId != null) {
RevObject filterType = database.getIfPresent(filterFeatureTypeId);
checkArgument(
filterType instanceof RevFeatureType, "Provided filter feature type is does not exist");
}
final SimpleFeatureStore targetStore = getTargetStore();
final String refspec = resolveRefSpec();
final String treePath = refspec.substring(refspec.indexOf(':') + 1);
final RevTree rootTree = resolveRootTree(refspec);
final NodeRef typeTreeRef = resolTypeTreeRef(refspec, treePath, rootTree);
final ObjectId defaultMetadataId = typeTreeRef.getMetadataId();
final RevTree typeTree = database.getTree(typeTreeRef.getObjectId());
final ProgressListener progressListener = getProgressListener();
progressListener.started();
progressListener.setDescription(
"Exporting from " + path + " to " + targetStore.getName().getLocalPart() + "... ");
final Iterator<SimpleFeature> filtered;
{
final Iterator<SimpleFeature> plainFeatures =
getFeatures(typeTree, database, defaultMetadataId, progressListener);
Iterator<SimpleFeature> adaptedFeatures = adaptToArguments(plainFeatures, defaultMetadataId);
Iterator<Optional<Feature>> transformed =
Iterators.transform(adaptedFeatures, ExportOp.this.function);
Iterator<SimpleFeature> result =
Iterators.filter(
Iterators.transform(
transformed,
new Function<Optional<Feature>, SimpleFeature>() {
@Override
public SimpleFeature apply(Optional<Feature> input) {
return (SimpleFeature) input.orNull();
}
}),
Predicates.notNull());
// check the resulting schema has something to contribute
PeekingIterator<SimpleFeature> peekingIt = Iterators.peekingIterator(result);
if (peekingIt.hasNext()) {
Function<AttributeDescriptor, String> toString =
new Function<AttributeDescriptor, String>() {
@Override
public String apply(AttributeDescriptor input) {
return input.getLocalName();
}
};
SimpleFeature peek = peekingIt.peek();
Set<String> sourceAtts =
new HashSet<String>(
Lists.transform(peek.getFeatureType().getAttributeDescriptors(), toString));
Set<String> targetAtts =
new HashSet<String>(
Lists.transform(targetStore.getSchema().getAttributeDescriptors(), toString));
if (Sets.intersection(sourceAtts, targetAtts).isEmpty()) {
throw new GeoToolsOpException(
StatusCode.UNABLE_TO_ADD,
"No common attributes between source and target feature types");
}
}
filtered = peekingIt;
}
FeatureCollection<SimpleFeatureType, SimpleFeature> asFeatureCollection =
new BaseFeatureCollection<SimpleFeatureType, SimpleFeature>() {
@Override
public FeatureIterator<SimpleFeature> features() {
return new DelegateFeatureIterator<SimpleFeature>(filtered);
}
};
// add the feature collection to the feature store
final Transaction transaction;
if (transactional) {
transaction = new DefaultTransaction("create");
} else {
transaction = Transaction.AUTO_COMMIT;
}
try {
targetStore.setTransaction(transaction);
try {
targetStore.addFeatures(asFeatureCollection);
transaction.commit();
} catch (final Exception e) {
if (transactional) {
transaction.rollback();
}
Throwables.propagateIfInstanceOf(e, GeoToolsOpException.class);
throw new GeoToolsOpException(e, StatusCode.UNABLE_TO_ADD);
} finally {
transaction.close();
}
} catch (IOException e) {
throw new GeoToolsOpException(e, StatusCode.UNABLE_TO_ADD);
}
progressListener.complete();
return targetStore;
}
public SelectIterator(BindingSet bindings, Iterator<Map.Entry<Key, Value>> iter) {
this.bindings = bindings;
this.iter = Iterators.peekingIterator(iter);
converter.init(Collections.singletonMap(CBConverter.OPTION_VALUE_DELIMITER, VALUE_DELIMITER));
}
/**
* Supplies an arbitrary "default" instance for a wide range of types, often useful in testing
* utilities.
*
* <p>Covers arrays, enums and common types defined in {@code java.lang}, {@code java.lang.reflect},
* {@code java.io}, {@code java.nio}, {@code java.math}, {@code java.util}, {@code
* java.util.concurrent}, {@code java.util.regex}, {@code com.google.common.base}, {@code
* com.google.common.collect} and {@code com.google.common.primitives}. In addition, if the type
* exposes at least one public static final constant of the same type, one of the constants will be
* used; or if the class exposes a public parameter-less constructor then it will be "new"d and
* returned.
*
* <p>All default instances returned by {@link #get} are generics-safe. Clients won't get type
* errors for using {@code get(Comparator.class)} as a {@code Comparator<Foo>}, for example.
* Immutable empty instances are returned for collection types; {@code ""} for string; {@code 0} for
* number types; reasonable default instance for other stateless types. For mutable types, a fresh
* instance is created each time {@code get()} is called.
*
* @author Kevin Bourrillion
* @author Ben Yu
* @since 12.0
*/
@Beta
public final class ArbitraryInstances {
private static final Ordering<Field> BY_FIELD_NAME =
new Ordering<Field>() {
@Override
public int compare(Field left, Field right) {
return left.getName().compareTo(right.getName());
}
};
/**
* Returns a new {@code MatchResult} that corresponds to a successful match. Apache Harmony (used
* in Android) requires a successful match in order to generate a {@code MatchResult}:
* http://goo.gl/5VQFmC
*/
private static MatchResult newMatchResult() {
Matcher matcher = Pattern.compile(".").matcher("X");
matcher.find();
return matcher.toMatchResult();
}
private static final ClassToInstanceMap<Object> DEFAULTS =
ImmutableClassToInstanceMap.builder()
// primitives
.put(Object.class, "")
.put(Number.class, 0)
.put(UnsignedInteger.class, UnsignedInteger.ZERO)
.put(UnsignedLong.class, UnsignedLong.ZERO)
.put(BigInteger.class, BigInteger.ZERO)
.put(BigDecimal.class, BigDecimal.ZERO)
.put(CharSequence.class, "")
.put(String.class, "")
.put(Pattern.class, Pattern.compile(""))
.put(MatchResult.class, newMatchResult())
.put(TimeUnit.class, TimeUnit.SECONDS)
.put(Charset.class, Charsets.UTF_8)
.put(Currency.class, Currency.getInstance(Locale.US))
.put(Locale.class, Locale.US)
// common.base
.put(CharMatcher.class, CharMatcher.NONE)
.put(Joiner.class, Joiner.on(','))
.put(Splitter.class, Splitter.on(','))
.put(Optional.class, Optional.absent())
.put(Predicate.class, Predicates.alwaysTrue())
.put(Equivalence.class, Equivalence.equals())
.put(Ticker.class, Ticker.systemTicker())
.put(Stopwatch.class, Stopwatch.createUnstarted())
// io types
.put(InputStream.class, new ByteArrayInputStream(new byte[0]))
.put(ByteArrayInputStream.class, new ByteArrayInputStream(new byte[0]))
.put(Readable.class, new StringReader(""))
.put(Reader.class, new StringReader(""))
.put(StringReader.class, new StringReader(""))
.put(Buffer.class, ByteBuffer.allocate(0))
.put(CharBuffer.class, CharBuffer.allocate(0))
.put(ByteBuffer.class, ByteBuffer.allocate(0))
.put(ShortBuffer.class, ShortBuffer.allocate(0))
.put(IntBuffer.class, IntBuffer.allocate(0))
.put(LongBuffer.class, LongBuffer.allocate(0))
.put(FloatBuffer.class, FloatBuffer.allocate(0))
.put(DoubleBuffer.class, DoubleBuffer.allocate(0))
.put(File.class, new File(""))
.put(ByteSource.class, ByteSource.empty())
.put(CharSource.class, CharSource.empty())
.put(ByteSink.class, NullByteSink.INSTANCE)
.put(CharSink.class, NullByteSink.INSTANCE.asCharSink(Charsets.UTF_8))
// All collections are immutable empty. So safe for any type parameter.
.put(Iterator.class, ImmutableSet.of().iterator())
.put(PeekingIterator.class, Iterators.peekingIterator(ImmutableSet.of().iterator()))
.put(ListIterator.class, ImmutableList.of().listIterator())
.put(Iterable.class, ImmutableSet.of())
.put(Collection.class, ImmutableList.of())
.put(ImmutableCollection.class, ImmutableList.of())
.put(List.class, ImmutableList.of())
.put(ImmutableList.class, ImmutableList.of())
.put(Set.class, ImmutableSet.of())
.put(ImmutableSet.class, ImmutableSet.of())
.put(SortedSet.class, ImmutableSortedSet.of())
.put(ImmutableSortedSet.class, ImmutableSortedSet.of())
.put(NavigableSet.class, Sets.unmodifiableNavigableSet(Sets.newTreeSet()))
.put(Map.class, ImmutableMap.of())
.put(ImmutableMap.class, ImmutableMap.of())
.put(SortedMap.class, ImmutableSortedMap.of())
.put(ImmutableSortedMap.class, ImmutableSortedMap.of())
.put(NavigableMap.class, Maps.unmodifiableNavigableMap(Maps.newTreeMap()))
.put(Multimap.class, ImmutableMultimap.of())
.put(ImmutableMultimap.class, ImmutableMultimap.of())
.put(ListMultimap.class, ImmutableListMultimap.of())
.put(ImmutableListMultimap.class, ImmutableListMultimap.of())
.put(SetMultimap.class, ImmutableSetMultimap.of())
.put(ImmutableSetMultimap.class, ImmutableSetMultimap.of())
.put(
SortedSetMultimap.class,
Multimaps.unmodifiableSortedSetMultimap(TreeMultimap.create()))
.put(Multiset.class, ImmutableMultiset.of())
.put(ImmutableMultiset.class, ImmutableMultiset.of())
.put(SortedMultiset.class, ImmutableSortedMultiset.of())
.put(ImmutableSortedMultiset.class, ImmutableSortedMultiset.of())
.put(BiMap.class, ImmutableBiMap.of())
.put(ImmutableBiMap.class, ImmutableBiMap.of())
.put(Table.class, ImmutableTable.of())
.put(ImmutableTable.class, ImmutableTable.of())
.put(RowSortedTable.class, Tables.unmodifiableRowSortedTable(TreeBasedTable.create()))
.put(ClassToInstanceMap.class, ImmutableClassToInstanceMap.builder().build())
.put(ImmutableClassToInstanceMap.class, ImmutableClassToInstanceMap.builder().build())
.put(Comparable.class, ByToString.INSTANCE)
.put(Comparator.class, AlwaysEqual.INSTANCE)
.put(Ordering.class, AlwaysEqual.INSTANCE)
.put(Range.class, Range.all())
.put(MapConstraint.class, MapConstraints.notNull())
.put(MapDifference.class, Maps.difference(ImmutableMap.of(), ImmutableMap.of()))
.put(
SortedMapDifference.class,
Maps.difference(ImmutableSortedMap.of(), ImmutableSortedMap.of()))
// reflect
.put(AnnotatedElement.class, Object.class)
.put(GenericDeclaration.class, Object.class)
.put(Type.class, Object.class)
.build();
/**
* type -> implementation. Inherently mutable interfaces and abstract classes are mapped to their
* default implementations and are "new"d upon get().
*/
private static final ConcurrentMap<Class<?>, Class<?>> implementations = Maps.newConcurrentMap();
private static <T> void setImplementation(Class<T> type, Class<? extends T> implementation) {
checkArgument(type != implementation, "Don't register %s to itself!", type);
checkArgument(
!DEFAULTS.containsKey(type), "A default value was already registered for %s", type);
checkArgument(
implementations.put(type, implementation) == null,
"Implementation for %s was already registered",
type);
}
static {
setImplementation(Appendable.class, StringBuilder.class);
setImplementation(BlockingQueue.class, LinkedBlockingDeque.class);
setImplementation(BlockingDeque.class, LinkedBlockingDeque.class);
setImplementation(ConcurrentMap.class, ConcurrentHashMap.class);
setImplementation(ConcurrentNavigableMap.class, ConcurrentSkipListMap.class);
setImplementation(CountDownLatch.class, Dummies.DummyCountDownLatch.class);
setImplementation(Deque.class, ArrayDeque.class);
setImplementation(OutputStream.class, ByteArrayOutputStream.class);
setImplementation(PrintStream.class, Dummies.InMemoryPrintStream.class);
setImplementation(PrintWriter.class, Dummies.InMemoryPrintWriter.class);
setImplementation(Queue.class, ArrayDeque.class);
setImplementation(Random.class, Dummies.DeterministicRandom.class);
setImplementation(
ScheduledThreadPoolExecutor.class, Dummies.DummyScheduledThreadPoolExecutor.class);
setImplementation(ThreadPoolExecutor.class, Dummies.DummyScheduledThreadPoolExecutor.class);
setImplementation(Writer.class, StringWriter.class);
setImplementation(Runnable.class, Dummies.DummyRunnable.class);
setImplementation(ThreadFactory.class, Dummies.DummyThreadFactory.class);
setImplementation(Executor.class, Dummies.DummyExecutor.class);
}
@SuppressWarnings("unchecked") // it's a subtype map
@Nullable
private static <T> Class<? extends T> getImplementation(Class<T> type) {
return (Class<? extends T>) implementations.get(type);
}
private static final Logger logger = Logger.getLogger(ArbitraryInstances.class.getName());
/**
* Returns an arbitrary instance for {@code type}, or {@code null} if no arbitrary instance can be
* determined.
*/
@Nullable
public static <T> T get(Class<T> type) {
T defaultValue = DEFAULTS.getInstance(type);
if (defaultValue != null) {
return defaultValue;
}
Class<? extends T> implementation = getImplementation(type);
if (implementation != null) {
return get(implementation);
}
if (type.isEnum()) {
T[] enumConstants = type.getEnumConstants();
return (enumConstants.length == 0) ? null : enumConstants[0];
}
if (type.isArray()) {
return createEmptyArray(type);
}
T jvmDefault = Defaults.defaultValue(Primitives.unwrap(type));
if (jvmDefault != null) {
return jvmDefault;
}
if (Modifier.isAbstract(type.getModifiers()) || !Modifier.isPublic(type.getModifiers())) {
return arbitraryConstantInstanceOrNull(type);
}
final Constructor<T> constructor;
try {
constructor = type.getConstructor();
} catch (NoSuchMethodException e) {
return arbitraryConstantInstanceOrNull(type);
}
constructor.setAccessible(true); // accessibility check is too slow
try {
return constructor.newInstance();
} catch (InstantiationException impossible) {
throw new AssertionError(impossible);
} catch (IllegalAccessException impossible) {
throw new AssertionError(impossible);
} catch (InvocationTargetException e) {
logger.log(Level.WARNING, "Exception while invoking default constructor.", e.getCause());
return arbitraryConstantInstanceOrNull(type);
}
}
@Nullable
private static <T> T arbitraryConstantInstanceOrNull(Class<T> type) {
Field[] fields = type.getDeclaredFields();
Arrays.sort(fields, BY_FIELD_NAME);
for (Field field : fields) {
if (Modifier.isPublic(field.getModifiers())
&& Modifier.isStatic(field.getModifiers())
&& Modifier.isFinal(field.getModifiers())) {
if (field.getGenericType() == field.getType() && type.isAssignableFrom(field.getType())) {
field.setAccessible(true);
try {
T constant = type.cast(field.get(null));
if (constant != null) {
return constant;
}
} catch (IllegalAccessException impossible) {
throw new AssertionError(impossible);
}
}
}
}
return null;
}
private static <T> T createEmptyArray(Class<T> arrayType) {
return arrayType.cast(Array.newInstance(arrayType.getComponentType(), 0));
}
// Internal implementations of some classes, with public default constructor that get() needs.
private static final class Dummies {
public static final class InMemoryPrintStream extends PrintStream {
public InMemoryPrintStream() {
super(new ByteArrayOutputStream());
}
}
public static final class InMemoryPrintWriter extends PrintWriter {
public InMemoryPrintWriter() {
super(new StringWriter());
}
}
public static final class DeterministicRandom extends Random {
public DeterministicRandom() {
super(0);
}
}
public static final class DummyScheduledThreadPoolExecutor extends ScheduledThreadPoolExecutor {
public DummyScheduledThreadPoolExecutor() {
super(1);
}
}
public static final class DummyCountDownLatch extends CountDownLatch {
public DummyCountDownLatch() {
super(0);
}
}
public static final class DummyRunnable implements Runnable, Serializable {
@Override
public void run() {}
}
public static final class DummyThreadFactory implements ThreadFactory, Serializable {
@Override
public Thread newThread(Runnable r) {
return new Thread(r);
}
}
public static final class DummyExecutor implements Executor, Serializable {
@Override
public void execute(Runnable command) {}
}
}
private static final class NullByteSink extends ByteSink implements Serializable {
private static final NullByteSink INSTANCE = new NullByteSink();
@Override
public OutputStream openStream() {
return ByteStreams.nullOutputStream();
}
}
// Compare by toString() to satisfy 2 properties:
// 1. compareTo(null) should throw NullPointerException
// 2. the order is deterministic and easy to understand, for debugging purpose.
private static final class ByToString implements Comparable<Object>, Serializable {
private static final ByToString INSTANCE = new ByToString();
@Override
public int compareTo(Object o) {
return toString().compareTo(o.toString());
}
@Override
public String toString() {
return "BY_TO_STRING";
}
private Object readResolve() {
return INSTANCE;
}
}
// Always equal is a valid total ordering. And it works for any Object.
private static final class AlwaysEqual extends Ordering<Object> implements Serializable {
private static final AlwaysEqual INSTANCE = new AlwaysEqual();
@Override
public int compare(Object o1, Object o2) {
return 0;
}
@Override
public String toString() {
return "ALWAYS_EQUAL";
}
private Object readResolve() {
return INSTANCE;
}
}
private ArbitraryInstances() {}
}
@Override
public SweepResults run(String tableName, int batchSize, @Nullable byte[] startRow) {
Preconditions.checkNotNull(tableName);
Preconditions.checkState(!AtlasDbConstants.hiddenTables.contains(tableName));
if (tableName.startsWith(AtlasDbConstants.NAMESPACE_PREFIX)) {
// this happens sometimes; I think it's because some places in the code can
// start this sweeper without doing the full normally ordered KVSModule startup.
// I did check and sweep.stats did contain the FQ table name for all of the tables,
// so it is at least broken in some way that still allows namespaced tables to eventually be
// swept.
log.warn("The sweeper should not be run on tables passed through namespace mapping.");
return SweepResults.EMPTY_SWEEP;
}
// Earliest start timestamp of any currently open transaction, with two caveats:
// (1) unreadableTimestamps are calculated via wall-clock time, and so may not be correct
// under pathological clock conditions
// (2) immutableTimestamps do not account for locks have timed out after checking their locks;
// such a transaction may have a start timestamp less than the immutableTimestamp, and it
// could still get successfully committed (its commit timestamp may or may not be less than
// the immutableTimestamp
// Note that this is fine, because we'll either
// (1) force old readers to abort (if they read a garbage collection sentinel), or
// (2) force old writers to retry (note that we must roll back any uncommitted transactions that
// we encounter
SweepStrategy sweepStrategy = sweepStrategyManager.get().get(tableName);
if (sweepStrategy == null) {
sweepStrategy = SweepStrategy.CONSERVATIVE;
} else if (sweepStrategy == SweepStrategy.NOTHING) {
return SweepResults.EMPTY_SWEEP;
}
if (startRow == null) {
startRow = new byte[0];
}
RangeRequest rangeRequest =
RangeRequest.builder().startRowInclusive(startRow).batchHint(batchSize).build();
long sweepTimestamp = getSweepTimestamp(sweepStrategy);
ClosableIterator<RowResult<Value>> valueResults;
if (sweepStrategy == SweepStrategy.CONSERVATIVE) {
valueResults = ClosableIterators.wrap(ImmutableList.<RowResult<Value>>of().iterator());
} else {
valueResults = keyValueService.getRange(tableName, rangeRequest, sweepTimestamp);
}
ClosableIterator<RowResult<Set<Long>>> rowResults =
keyValueService.getRangeOfTimestamps(tableName, rangeRequest, sweepTimestamp);
try {
List<RowResult<Set<Long>>> rowResultTimestamps =
ImmutableList.copyOf(Iterators.limit(rowResults, batchSize));
PeekingIterator<RowResult<Value>> peekingValues = Iterators.peekingIterator(valueResults);
Set<Cell> sentinelsToAdd = Sets.newHashSet();
Multimap<Cell, Long> rowTimestamps =
getTimestampsFromRowResults(rowResultTimestamps, sweepStrategy);
Multimap<Cell, Long> cellTsPairsToSweep =
getCellTsPairsToSweep(
rowTimestamps, peekingValues, sweepTimestamp, sweepStrategy, sentinelsToAdd);
sweepCells(tableName, cellTsPairsToSweep, sentinelsToAdd);
byte[] nextRow =
rowResultTimestamps.size() < batchSize
? null
: RangeRequests.getNextStartRow(
false, Iterables.getLast(rowResultTimestamps).getRowName());
return new SweepResults(nextRow, rowResultTimestamps.size(), cellTsPairsToSweep.size());
} finally {
rowResults.close();
valueResults.close();
}
}
@Override
protected void innerReduce(
Context context, SortableBytes keyBytes, Iterable<DimValueCount> combinedIterable)
throws IOException, InterruptedException {
final ByteBuffer groupKey = ByteBuffer.wrap(keyBytes.getGroupKey());
groupKey.position(4); // Skip partition
final DateTime bucket = new DateTime(groupKey.getLong());
final PeekingIterator<DimValueCount> iterator =
Iterators.peekingIterator(combinedIterable.iterator());
log.info(
"Determining partitions for interval: %s",
config.getGranularitySpec().bucketInterval(bucket).orNull());
// First DVC should be the total row count indicator
final DimValueCount firstDvc = iterator.next();
final int totalRows = firstDvc.numRows;
if (!firstDvc.dim.equals("") || !firstDvc.value.equals("")) {
throw new IllegalStateException("WTF?! Expected total row indicator on first k/v pair!");
}
// "iterator" will now take us over many candidate dimensions
DimPartitions currentDimPartitions = null;
DimPartition currentDimPartition = null;
String currentDimPartitionStart = null;
boolean currentDimSkip = false;
// We'll store possible partitions in here
final Map<String, DimPartitions> dimPartitionss = Maps.newHashMap();
while (iterator.hasNext()) {
final DimValueCount dvc = iterator.next();
if (currentDimPartitions == null || !currentDimPartitions.dim.equals(dvc.dim)) {
// Starting a new dimension! Exciting!
currentDimPartitions = new DimPartitions(dvc.dim);
currentDimPartition = new DimPartition();
currentDimPartitionStart = null;
currentDimSkip = false;
}
// Respect poisoning
if (!currentDimSkip && dvc.numRows < 0) {
log.info("Cannot partition on multi-valued dimension: %s", dvc.dim);
currentDimSkip = true;
}
if (currentDimSkip) {
continue;
}
// See if we need to cut a new partition ending immediately before this dimension value
if (currentDimPartition.rows > 0
&& currentDimPartition.rows + dvc.numRows >= config.getTargetPartitionSize()) {
final ShardSpec shardSpec =
new SingleDimensionShardSpec(
currentDimPartitions.dim,
currentDimPartitionStart,
dvc.value,
currentDimPartitions.partitions.size());
log.info(
"Adding possible shard with %,d rows and %,d unique values: %s",
currentDimPartition.rows, currentDimPartition.cardinality, shardSpec);
currentDimPartition.shardSpec = shardSpec;
currentDimPartitions.partitions.add(currentDimPartition);
currentDimPartition = new DimPartition();
currentDimPartitionStart = dvc.value;
}
// Update counters
currentDimPartition.cardinality++;
currentDimPartition.rows += dvc.numRows;
if (!iterator.hasNext() || !currentDimPartitions.dim.equals(iterator.peek().dim)) {
// Finalize the current dimension
if (currentDimPartition.rows > 0) {
// One more shard to go
final ShardSpec shardSpec;
if (currentDimPartitions.partitions.isEmpty()) {
shardSpec = new NoneShardSpec();
} else {
if (currentDimPartition.rows
< config.getTargetPartitionSize() * SHARD_COMBINE_THRESHOLD) {
// Combine with previous shard
final DimPartition previousDimPartition =
currentDimPartitions.partitions.remove(
currentDimPartitions.partitions.size() - 1);
final SingleDimensionShardSpec previousShardSpec =
(SingleDimensionShardSpec) previousDimPartition.shardSpec;
shardSpec =
new SingleDimensionShardSpec(
currentDimPartitions.dim,
previousShardSpec.getStart(),
null,
previousShardSpec.getPartitionNum());
log.info("Removing possible shard: %s", previousShardSpec);
currentDimPartition.rows += previousDimPartition.rows;
currentDimPartition.cardinality += previousDimPartition.cardinality;
} else {
// Create new shard
shardSpec =
new SingleDimensionShardSpec(
currentDimPartitions.dim,
currentDimPartitionStart,
null,
currentDimPartitions.partitions.size());
}
}
log.info(
"Adding possible shard with %,d rows and %,d unique values: %s",
currentDimPartition.rows, currentDimPartition.cardinality, shardSpec);
currentDimPartition.shardSpec = shardSpec;
currentDimPartitions.partitions.add(currentDimPartition);
}
log.info(
"Completed dimension[%s]: %,d possible shards with %,d unique values",
currentDimPartitions.dim,
currentDimPartitions.partitions.size(),
currentDimPartitions.getCardinality());
// Add ourselves to the partitions map
dimPartitionss.put(currentDimPartitions.dim, currentDimPartitions);
}
}
// Choose best dimension
if (dimPartitionss.isEmpty()) {
throw new ISE("No suitable partitioning dimension found!");
}
int maxCardinality = Integer.MIN_VALUE;
long minDistance = Long.MAX_VALUE;
DimPartitions minDistancePartitions = null;
DimPartitions maxCardinalityPartitions = null;
for (final DimPartitions dimPartitions : dimPartitionss.values()) {
if (dimPartitions.getRows() != totalRows) {
log.info(
"Dimension[%s] is not present in all rows (row count %,d != expected row count %,d)",
dimPartitions.dim, dimPartitions.getRows(), totalRows);
continue;
}
// Make sure none of these shards are oversized
boolean oversized = false;
for (final DimPartition partition : dimPartitions.partitions) {
if (partition.rows > config.getMaxPartitionSize()) {
log.info(
"Dimension[%s] has an oversized shard: %s", dimPartitions.dim, partition.shardSpec);
oversized = true;
}
}
if (oversized) {
continue;
}
final int cardinality = dimPartitions.getCardinality();
final long distance =
dimPartitions.getDistanceSquaredFromTarget(config.getTargetPartitionSize());
if (cardinality > maxCardinality) {
maxCardinality = cardinality;
maxCardinalityPartitions = dimPartitions;
}
if (distance < minDistance) {
minDistance = distance;
minDistancePartitions = dimPartitions;
}
}
if (maxCardinalityPartitions == null) {
throw new ISE("No suitable partitioning dimension found!");
}
final OutputStream out =
Utils.makePathAndOutputStream(
context,
config.makeSegmentPartitionInfoPath(
config.getGranularitySpec().bucketInterval(bucket).get()),
config.isOverwriteFiles());
final DimPartitions chosenPartitions =
maxCardinality > HIGH_CARDINALITY_THRESHOLD
? maxCardinalityPartitions
: minDistancePartitions;
final List<ShardSpec> chosenShardSpecs =
Lists.transform(
chosenPartitions.partitions,
new Function<DimPartition, ShardSpec>() {
@Override
public ShardSpec apply(DimPartition dimPartition) {
return dimPartition.shardSpec;
}
});
log.info("Chosen partitions:");
for (ShardSpec shardSpec : chosenShardSpecs) {
log.info(" %s", HadoopDruidIndexerConfig.JSON_MAPPER.writeValueAsString(shardSpec));
}
try {
HadoopDruidIndexerConfig.JSON_MAPPER
.writerWithType(new TypeReference<List<ShardSpec>>() {})
.writeValue(out, chosenShardSpecs);
} finally {
Closeables.close(out, false);
}
}