@Override
public ConnectorSplitSource getSplits(
ConnectorTransactionHandle transactionHandle,
ConnectorSession session,
ConnectorTableLayoutHandle layoutHandle) {
AtopTableLayoutHandle handle =
checkType(layoutHandle, AtopTableLayoutHandle.class, "layoutHandle");
AtopTableHandle table = handle.getTableHandle();
List<ConnectorSplit> splits = new ArrayList<>();
DateTime end = DateTime.now().withZone(timeZone);
for (Node node : nodeManager.getActiveDatasourceNodes(connectorId.getId())) {
DateTime start = end.minusDays(maxHistoryDays - 1).withTimeAtStartOfDay();
while (start.isBefore(end)) {
DateTime splitEnd = start.withTime(23, 59, 59, 999);
Domain splitDomain =
Domain.create(
ValueSet.ofRanges(
Range.range(TIMESTAMP, start.getMillis(), true, splitEnd.getMillis(), true)),
false);
if (handle.getStartTimeConstraint().overlaps(splitDomain)
&& handle.getEndTimeConstraint().overlaps(splitDomain)) {
splits.add(new AtopSplit(table.getTable(), node.getHostAndPort(), start));
}
start = start.plusDays(1).withTimeAtStartOfDay();
}
}
return new FixedSplitSource(connectorId.getId(), splits);
}
@Test
public void testExtractValuesFromSingleDomain() throws Exception {
Map<Type, Object> testValues =
ImmutableMap.<Type, Object>builder()
.put(BOOLEAN, true)
.put(INTEGER, 1234L)
.put(BIGINT, 4321L)
.put(DOUBLE, 0.123)
.put(VARCHAR, utf8Slice(TEST_STRING))
.build();
for (Map.Entry<Type, Object> testValue : testValues.entrySet()) {
Domain predicateDomain = Domain.singleValue(testValue.getKey(), testValue.getValue());
Optional<Collection<Object>> discreteValues =
extractDiscreteValues(predicateDomain.getValues());
assertTrue(discreteValues.isPresent());
Collection<Object> objects = discreteValues.get();
assertEquals(objects.size(), 1);
assertEquals(objects.iterator().next(), testValue.getValue());
}
}
@Test
// simulate query on a 2 columns where 1 is used as part of the where, with and without bloom
// filter
public void testMatches() throws Exception {
// stripe column
Domain testingColumnHandleDomain = Domain.singleValue(BIGINT, 1234L);
TupleDomain.ColumnDomain<String> column0 =
new TupleDomain.ColumnDomain<>(COLUMN_0, testingColumnHandleDomain);
// predicate consist of the bigint_0 = 1234
TupleDomain<String> effectivePredicate =
TupleDomain.fromColumnDomains(Optional.of(ImmutableList.of(column0)));
TupleDomain<String> emptyEffectivePredicate = TupleDomain.all();
// predicate column references
List<ColumnReference<String>> columnReferences =
ImmutableList.<ColumnReference<String>>builder()
.add(new ColumnReference<>(COLUMN_0, 0, BIGINT))
.add(new ColumnReference<>(COLUMN_1, 1, BIGINT))
.build();
TupleDomainOrcPredicate<String> predicate =
new TupleDomainOrcPredicate<>(effectivePredicate, columnReferences, true);
TupleDomainOrcPredicate<String> emptyPredicate =
new TupleDomainOrcPredicate<>(emptyEffectivePredicate, columnReferences, true);
// assemble a matching and a non-matching bloom filter
HiveBloomFilter hiveBloomFilter = new HiveBloomFilter(new BloomFilter(1000, 0.01));
OrcProto.BloomFilter emptyOrcBloomFilter = toOrcBloomFilter(hiveBloomFilter);
hiveBloomFilter.addLong(1234);
OrcProto.BloomFilter orcBloomFilter = toOrcBloomFilter(hiveBloomFilter);
Map<Integer, ColumnStatistics> matchingStatisticsByColumnIndex =
ImmutableMap.of(
0,
new ColumnStatistics(
null,
null,
new IntegerStatistics(10L, 2000L),
null,
null,
null,
null,
toHiveBloomFilter(orcBloomFilter)));
Map<Integer, ColumnStatistics> nonMatchingStatisticsByColumnIndex =
ImmutableMap.of(
0,
new ColumnStatistics(
null,
null,
new IntegerStatistics(10L, 2000L),
null,
null,
null,
null,
toHiveBloomFilter(emptyOrcBloomFilter)));
Map<Integer, ColumnStatistics> withoutBloomFilterStatisticsByColumnIndex =
ImmutableMap.of(
0,
new ColumnStatistics(
null, null, new IntegerStatistics(10L, 2000L), null, null, null, null, null));
assertTrue(predicate.matches(1L, matchingStatisticsByColumnIndex));
assertTrue(predicate.matches(1L, withoutBloomFilterStatisticsByColumnIndex));
assertFalse(predicate.matches(1L, nonMatchingStatisticsByColumnIndex));
assertTrue(emptyPredicate.matches(1L, matchingStatisticsByColumnIndex));
}