@Test
public void testBucketedTemporalTableCompaction() throws Exception {
long day1 = 1;
long day2 = 2;
long day3 = 3;
long day4 = 4;
List<ShardIndexInfo> inputShards =
ImmutableList.of(
shardWithTemporalBucket(OptionalInt.of(1), DATE, day1, day1),
shardWithTemporalBucket(OptionalInt.of(2), DATE, day2, day2),
shardWithTemporalBucket(OptionalInt.of(1), DATE, day1, day1),
shardWithTemporalBucket(OptionalInt.of(2), DATE, day2, day2),
shardWithTemporalBucket(OptionalInt.of(1), DATE, day3, day3),
shardWithTemporalBucket(OptionalInt.of(2), DATE, day4, day4));
long tableId = bucketedTemporalTableInfo.getTableId();
Set<OrganizationSet> actual =
compactionSetCreator.createCompactionSets(bucketedTemporalTableInfo, inputShards);
assertEquals(actual.size(), 2);
Set<OrganizationSet> expected =
ImmutableSet.of(
new OrganizationSet(tableId, extractIndexes(inputShards, 0, 2), OptionalInt.of(1)),
new OrganizationSet(tableId, extractIndexes(inputShards, 1, 3), OptionalInt.of(2)));
assertEquals(actual, expected);
}
public synchronized PageBufferClientStatus getStatus() {
String state;
if (closed) {
state = "closed";
} else if (future != null) {
state = "running";
} else if (scheduled) {
state = "scheduled";
} else if (completed) {
state = "completed";
} else {
state = "queued";
}
String httpRequestState = "not scheduled";
if (future != null) {
httpRequestState = future.getState();
}
long rejectedRows = rowsRejected.get();
int rejectedPages = pagesRejected.get();
return new PageBufferClientStatus(
location,
state,
lastUpdate,
rowsReceived.get(),
pagesReceived.get(),
rejectedRows == 0 ? OptionalLong.empty() : OptionalLong.of(rejectedRows),
rejectedPages == 0 ? OptionalInt.empty() : OptionalInt.of(rejectedPages),
requestsScheduled.get(),
requestsCompleted.get(),
requestsFailed.get(),
httpRequestState);
}
public OptionalInt amount(String compound) {
Integer amount = compounds.get(compound);
if (amount != null) {
return OptionalInt.of(amount);
} else {
return OptionalInt.empty();
}
}
private static ShardIndexInfo shardWithBucket(int bucketNumber) {
return new ShardIndexInfo(
1,
OptionalInt.of(bucketNumber),
UUID.randomUUID(),
1,
1,
Optional.empty(),
Optional.empty());
}
@Test
public void testAndInt() {
List<Integer> ints = Arrays.asList(0b1100, 0b0110, 0b101110, 0b11110011);
Collector<Integer, ?, OptionalInt> collector = MoreCollectors.andingInt(Integer::intValue);
checkShortCircuitCollector("andInt", OptionalInt.of(0), 4, ints::stream, collector);
checkCollectorEmpty("andIntEmpty", OptionalInt.empty(), collector);
assertEquals(
OptionalInt.of(0),
IntStreamEx.iterate(16384, i -> i + 1).parallel().boxed().collect(collector));
assertEquals(
OptionalInt.of(16384),
IntStreamEx.iterate(16384, i -> i + 1).parallel().limit(16383).boxed().collect(collector));
Collector<Integer, ?, Integer> unwrapped =
MoreCollectors.collectingAndThen(
MoreCollectors.andingInt(Integer::intValue), OptionalInt::getAsInt);
assertTrue(unwrapped.characteristics().contains(Characteristics.UNORDERED));
checkShortCircuitCollector("andIntUnwrapped", 0, 4, ints::stream, unwrapped);
checkShortCircuitCollector(
"andIntUnwrapped", 0, 2, Arrays.asList(0x1, 0x10, 0x100)::stream, unwrapped);
}
@Test
public void testBucketedTableCompaction() throws Exception {
List<ShardIndexInfo> inputShards =
ImmutableList.of(
shardWithBucket(1),
shardWithBucket(2),
shardWithBucket(2),
shardWithBucket(1),
shardWithBucket(2),
shardWithBucket(1));
long tableId = bucketedTableInfo.getTableId();
Set<OrganizationSet> actual =
compactionSetCreator.createCompactionSets(bucketedTableInfo, inputShards);
assertEquals(actual.size(), 2);
Set<OrganizationSet> expected =
ImmutableSet.of(
new OrganizationSet(tableId, extractIndexes(inputShards, 0, 3, 5), OptionalInt.of(1)),
new OrganizationSet(tableId, extractIndexes(inputShards, 1, 2, 4), OptionalInt.of(2)));
assertEquals(actual, expected);
}
@Override
public OptionalInt findAny() {
Integer result =
performOperation(
TerminalFunctions.findAnyIntFunction(),
false,
(i1, i2) -> {
if (i1 != null) {
return i1;
} else {
return i2;
}
},
a -> a != null);
if (result != null) {
return OptionalInt.of(result);
} else {
return OptionalInt.empty();
}
}
public Stream<T> parse(InputStream is, CsvErrorSniffer context) {
ObjectMapper mapper = objectMapper.copy();
formatter.initMixIn(mapper);
ObjectReader reader = mapper.readerFor(formatter.getTargetClass());
CsvSchema schema = new CsvSchema(formatter);
return parseToCsvLine(is)
.map(
line -> {
line.getException()
.ifPresent(
e -> context.mark(new Location(line.getLineNumber(), OptionalInt.empty())));
Set<String> ignoreField = new HashSet<>();
while (true) {
try {
return reader.readValue(schema.toJson(line, ignoreField));
} catch (JsonMappingException e) {
String path = buildPath(e.getPath());
;
Location location =
new Location(
line.getLineNumber(), OptionalInt.of(schema.getColumnNumber(path)));
if (context.contains(location)) {
throw new IllegalStateException("invalid row state: " + e.getLocation());
}
context.mark(location);
ignoreField.add(path);
} catch (IOException e) {
context.mark(new Location(line.getLineNumber(), OptionalInt.empty()));
try {
return formatter.getTargetClass().newInstance();
} catch (ReflectiveOperationException e2) {
throw new ReflectiveOperationRuntimeException(e2);
}
}
}
});
}
/** Compute split-per-shard (separate split for each shard). */
private BucketShards compute() throws SQLException {
if (!resultSet.next()) {
return endOfData();
}
UUID shardUuid = uuidFromBytes(resultSet.getBytes("shard_uuid"));
Set<String> nodeIdentifiers;
OptionalInt bucketNumber = OptionalInt.empty();
if (bucketToNode != null) {
int bucket = resultSet.getInt("bucket_number");
bucketNumber = OptionalInt.of(bucket);
nodeIdentifiers = ImmutableSet.of(getBucketNode(bucket));
} else {
List<Integer> nodeIds = intArrayFromBytes(resultSet.getBytes("node_ids"));
nodeIdentifiers = getNodeIdentifiers(nodeIds, shardUuid);
}
ShardNodes shard = new ShardNodes(shardUuid, nodeIdentifiers);
return new BucketShards(bucketNumber, ImmutableSet.of(shard));
}
@Override
public OptionalInt max() {
Integer value =
performOperation(
TerminalFunctions.maxIntFunction(),
false,
(i1, i2) -> {
if (i1 != null) {
if (i2 != null) {
return i1 > i2 ? i1 : i2;
}
return i1;
}
return i2;
},
null);
if (value == null) {
return OptionalInt.empty();
} else {
return OptionalInt.of(value);
}
}
@Override
public OptionalInt reduce(IntBinaryOperator op) {
Integer result =
performOperation(
TerminalFunctions.reduceFunction(op),
true,
(i1, i2) -> {
if (i1 != null) {
if (i2 != null) {
return op.applyAsInt(i1, i2);
}
return i1;
}
return i2;
},
null);
if (result == null) {
return OptionalInt.empty();
} else {
return OptionalInt.of(result);
}
}
/** Compute split-per-bucket (single split for all shards in a bucket). */
private BucketShards computeMerged() throws SQLException {
if (resultSet.isAfterLast()) {
return endOfData();
}
if (first) {
first = false;
if (!resultSet.next()) {
return endOfData();
}
}
int bucketNumber = resultSet.getInt("bucket_number");
ImmutableSet.Builder<ShardNodes> shards = ImmutableSet.builder();
do {
UUID shardUuid = uuidFromBytes(resultSet.getBytes("shard_uuid"));
int bucket = resultSet.getInt("bucket_number");
Set<String> nodeIdentifiers = ImmutableSet.of(getBucketNode(bucket));
shards.add(new ShardNodes(shardUuid, nodeIdentifiers));
} while (resultSet.next() && resultSet.getInt("bucket_number") == bucketNumber);
return new BucketShards(OptionalInt.of(bucketNumber), shards.build());
}
@Override
public OptionalInt fromJson(String jsonValue, Type type) {
return NULL.equals(jsonValue)
? OptionalInt.empty()
: OptionalInt.of(Integer.parseInt(jsonValue));
}
public static OptionalInt getServerAddressColumn() {
return OptionalInt.of(-1);
}
public static OptionalInt getTimestampColumn() {
return OptionalInt.of(0);
}
@Test
public void should_be_able_to_catch_exceptions_thrown_by_all_proxied_methods() {
try (AutoCloseableBDDSoftAssertions softly = new AutoCloseableBDDSoftAssertions()) {
softly.then(BigDecimal.ZERO).isEqualTo(BigDecimal.ONE);
softly.then(Boolean.FALSE).isTrue();
softly.then(false).isTrue();
softly.then(new boolean[] {false}).isEqualTo(new boolean[] {true});
softly.then(new Byte((byte) 0)).isEqualTo((byte) 1);
softly.then((byte) 2).inHexadecimal().isEqualTo((byte) 3);
softly.then(new byte[] {4}).isEqualTo(new byte[] {5});
softly.then(new Character((char) 65)).isEqualTo(new Character((char) 66));
softly.then((char) 67).isEqualTo((char) 68);
softly.then(new char[] {69}).isEqualTo(new char[] {70});
softly.then(new StringBuilder("a")).isEqualTo(new StringBuilder("b"));
softly.then(Object.class).isEqualTo(String.class);
softly
.then(parseDatetime("1999-12-31T23:59:59"))
.isEqualTo(parseDatetime("2000-01-01T00:00:01"));
softly.then(new Double(6.0d)).isEqualTo(new Double(7.0d));
softly.then(8.0d).isEqualTo(9.0d);
softly.then(new double[] {10.0d}).isEqualTo(new double[] {11.0d});
softly
.then(new File("a"))
.overridingErrorMessage("expected:<File(b)> but was:<File(a)>")
.isEqualTo(new File("b"));
softly.then(new Float(12f)).isEqualTo(new Float(13f));
softly.then(14f).isEqualTo(15f);
softly.then(new float[] {16f}).isEqualTo(new float[] {17f});
softly
.then(new ByteArrayInputStream(new byte[] {(byte) 65}))
.hasSameContentAs(new ByteArrayInputStream(new byte[] {(byte) 66}));
softly.then(new Integer(20)).isEqualTo(new Integer(21));
softly.then(22).isEqualTo(23);
softly.then(new int[] {24}).isEqualTo(new int[] {25});
softly.then((Iterable<String>) Lists.newArrayList("26")).isEqualTo(Lists.newArrayList("27"));
softly.then(Lists.newArrayList("28").iterator()).contains("29");
softly.then(Lists.newArrayList("30")).isEqualTo(Lists.newArrayList("31"));
softly.then(new Long(32L)).isEqualTo(new Long(33L));
softly.then(34L).isEqualTo(35L);
softly.then(new long[] {36L}).isEqualTo(new long[] {37L});
softly
.then(Maps.mapOf(MapEntry.entry("38", "39")))
.isEqualTo(Maps.mapOf(MapEntry.entry("40", "41")));
softly.then(new Short((short) 42)).isEqualTo(new Short((short) 43));
softly.then((short) 44).isEqualTo((short) 45);
softly.then(new short[] {(short) 46}).isEqualTo(new short[] {(short) 47});
softly.then("48").isEqualTo("49");
softly
.then(
new Object() {
@Override
public String toString() {
return "50";
}
})
.isEqualTo(
new Object() {
@Override
public String toString() {
return "51";
}
});
softly
.then(
new Object[] {
new Object() {
@Override
public String toString() {
return "52";
}
}
})
.isEqualTo(
new Object[] {
new Object() {
@Override
public String toString() {
return "53";
}
}
});
final IllegalArgumentException illegalArgumentException =
new IllegalArgumentException("IllegalArgumentException message");
softly.then(illegalArgumentException).hasMessage("NullPointerException message");
softly.then(Optional.of("not empty")).isEqualTo("empty");
softly.then(OptionalInt.of(0)).isEqualTo(1);
softly.then(OptionalDouble.of(0.0)).isEqualTo(1.0);
softly.then(OptionalLong.of(0L)).isEqualTo(1L);
softly.then(LocalTime.of(12, 00)).isEqualTo(LocalTime.of(13, 00));
softly
.then(OffsetTime.of(12, 0, 0, 0, ZoneOffset.UTC))
.isEqualTo(OffsetTime.of(13, 0, 0, 0, ZoneOffset.UTC));
softly.then(OffsetDateTime.MIN).isEqualTo(OffsetDateTime.MAX);
} catch (SoftAssertionError e) {
List<String> errors = e.getErrors();
assertThat(errors).hasSize(45);
assertThat(errors.get(0)).isEqualTo("expected:<[1]> but was:<[0]>");
assertThat(errors.get(1)).isEqualTo("expected:<[tru]e> but was:<[fals]e>");
assertThat(errors.get(2)).isEqualTo("expected:<[tru]e> but was:<[fals]e>");
assertThat(errors.get(3)).isEqualTo("expected:<[[tru]e]> but was:<[[fals]e]>");
assertThat(errors.get(4)).isEqualTo("expected:<[1]> but was:<[0]>");
assertThat(errors.get(5)).isEqualTo("expected:<0x0[3]> but was:<0x0[2]>");
assertThat(errors.get(6)).isEqualTo("expected:<[[5]]> but was:<[[4]]>");
assertThat(errors.get(7)).isEqualTo("expected:<'[B]'> but was:<'[A]'>");
assertThat(errors.get(8)).isEqualTo("expected:<'[D]'> but was:<'[C]'>");
assertThat(errors.get(9)).isEqualTo("expected:<['[F]']> but was:<['[E]']>");
assertThat(errors.get(10)).isEqualTo("expected:<[b]> but was:<[a]>");
assertThat(errors.get(11))
.isEqualTo("expected:<java.lang.[String]> but was:<java.lang.[Object]>");
assertThat(errors.get(12))
.isEqualTo("expected:<[2000-01-01T00:00:01].000> but was:<[1999-12-31T23:59:59].000>");
assertThat(errors.get(13)).isEqualTo("expected:<[7].0> but was:<[6].0>");
assertThat(errors.get(14)).isEqualTo("expected:<[9].0> but was:<[8].0>");
assertThat(errors.get(15)).isEqualTo("expected:<[1[1].0]> but was:<[1[0].0]>");
assertThat(errors.get(16)).isEqualTo("expected:<File(b)> but was:<File(a)>");
assertThat(errors.get(17)).isEqualTo("expected:<1[3].0f> but was:<1[2].0f>");
assertThat(errors.get(18)).isEqualTo("expected:<1[5].0f> but was:<1[4].0f>");
assertThat(errors.get(19)).isEqualTo("expected:<[1[7].0f]> but was:<[1[6].0f]>");
assertThat(errors.get(20))
.isEqualTo(
String.format(
"%nInputStreams do not have same content:"
+ System.getProperty("line.separator")
+ "line:<1>, expected:<B> but was:<A>"));
assertThat(errors.get(21)).isEqualTo("expected:<2[1]> but was:<2[0]>");
assertThat(errors.get(22)).isEqualTo("expected:<2[3]> but was:<2[2]>");
assertThat(errors.get(23)).isEqualTo("expected:<[2[5]]> but was:<[2[4]]>");
assertThat(errors.get(24)).isEqualTo("expected:<[\"2[7]\"]> but was:<[\"2[6]\"]>");
assertThat(errors.get(25))
.isEqualTo(
String.format(
"%nExpecting:%n"
+ " <[\"28\"]>%n"
+ "to contain:%n"
+ " <[\"29\"]>%n"
+ "but could not find:%n"
+ " <[\"29\"]>%n"));
assertThat(errors.get(26)).isEqualTo("expected:<[\"3[1]\"]> but was:<[\"3[0]\"]>");
assertThat(errors.get(27)).isEqualTo("expected:<3[3]L> but was:<3[2]L>");
assertThat(errors.get(28)).isEqualTo("expected:<3[5]L> but was:<3[4]L>");
assertThat(errors.get(29)).isEqualTo("expected:<[3[7]L]> but was:<[3[6]L]>");
assertThat(errors.get(30))
.isEqualTo("expected:<{\"[40\"=\"41]\"}> but was:<{\"[38\"=\"39]\"}>");
assertThat(errors.get(31)).isEqualTo("expected:<4[3]> but was:<4[2]>");
assertThat(errors.get(32)).isEqualTo("expected:<4[5]> but was:<4[4]>");
assertThat(errors.get(33)).isEqualTo("expected:<[4[7]]> but was:<[4[6]]>");
assertThat(errors.get(34)).isEqualTo("expected:<\"4[9]\"> but was:<\"4[8]\">");
assertThat(errors.get(35)).isEqualTo("expected:<5[1]> but was:<5[0]>");
assertThat(errors.get(36)).isEqualTo("expected:<[5[3]]> but was:<[5[2]]>");
assertThat(errors.get(37))
.isEqualTo(
String.format(
"%nExpecting message:%n"
+ " <\"NullPointerException message\">%n"
+ "but was:%n"
+ " <\"IllegalArgumentException message\">"));
assertThat(errors.get(38))
.isEqualTo("expected:<[\"empty\"]> but was:<[Optional[not empty]]>");
assertThat(errors.get(39)).isEqualTo("expected:<[1]> but was:<[OptionalInt[0]]>");
assertThat(errors.get(40)).isEqualTo("expected:<[1.0]> but was:<[OptionalDouble[0.0]]>");
assertThat(errors.get(41)).isEqualTo("expected:<[1L]> but was:<[OptionalLong[0]]>");
assertThat(errors.get(42)).isEqualTo("expected:<1[3]:00> but was:<1[2]:00>");
assertThat(errors.get(43)).isEqualTo("expected:<1[3]:00Z> but was:<1[2]:00Z>");
assertThat(errors.get(44))
.isEqualTo(
"expected:<[+999999999-12-31T23:59:59.999999999-]18:00> but was:<[-999999999-01-01T00:00+]18:00>");
return;
}
fail("Should not reach here");
}
public void test_of_intAdjustment() {
ValueStep test = ValueStep.of(2, DELTA_MINUS_2000);
assertEquals(test.getDate(), Optional.empty());
assertEquals(test.getPeriodIndex(), OptionalInt.of(2));
assertEquals(test.getValue(), DELTA_MINUS_2000);
}
public class TestCompactionSetCreator {
private static final long MAX_SHARD_ROWS = 100;
private static final DataSize MAX_SHARD_SIZE = new DataSize(100, DataSize.Unit.BYTE);
private static final Table tableInfo =
new Table(1L, OptionalLong.empty(), OptionalInt.empty(), OptionalLong.empty());
private static final Table temporalTableInfo =
new Table(1L, OptionalLong.empty(), OptionalInt.empty(), OptionalLong.of(1));
private static final Table bucketedTableInfo =
new Table(1L, OptionalLong.empty(), OptionalInt.of(3), OptionalLong.empty());
private static final Table bucketedTemporalTableInfo =
new Table(1L, OptionalLong.empty(), OptionalInt.of(3), OptionalLong.of(1));
private final CompactionSetCreator compactionSetCreator =
new CompactionSetCreator(MAX_SHARD_SIZE, MAX_SHARD_ROWS);
@Test
public void testNonTemporalOrganizationSetSimple() throws Exception {
List<ShardIndexInfo> inputShards =
ImmutableList.of(shardWithSize(10, 10), shardWithSize(10, 10), shardWithSize(10, 10));
Set<OrganizationSet> compactionSets =
compactionSetCreator.createCompactionSets(tableInfo, inputShards);
assertEquals(compactionSets.size(), 1);
assertEquals(getOnlyElement(compactionSets).getShards(), extractIndexes(inputShards, 0, 1, 2));
}
@Test
public void testNonTemporalSizeBasedOrganizationSet() throws Exception {
List<ShardIndexInfo> inputShards =
ImmutableList.of(
shardWithSize(10, 70),
shardWithSize(10, 20),
shardWithSize(10, 30),
shardWithSize(10, 120));
Set<OrganizationSet> compactionSets =
compactionSetCreator.createCompactionSets(tableInfo, inputShards);
Set<UUID> actual = new HashSet<>();
for (OrganizationSet set : compactionSets) {
actual.addAll(set.getShards());
}
assertTrue(extractIndexes(inputShards, 0, 1, 2).containsAll(actual));
}
@Test
public void testNonTemporalRowCountBasedOrganizationSet() throws Exception {
List<ShardIndexInfo> inputShards =
ImmutableList.of(
shardWithSize(50, 10),
shardWithSize(100, 10),
shardWithSize(20, 10),
shardWithSize(30, 10));
Set<OrganizationSet> compactionSets =
compactionSetCreator.createCompactionSets(tableInfo, inputShards);
Set<UUID> actual = new HashSet<>();
for (OrganizationSet set : compactionSets) {
actual.addAll(set.getShards());
}
assertTrue(extractIndexes(inputShards, 0, 2, 3).containsAll(actual));
}
@Test
public void testTemporalCompactionNoCompactionAcrossDays() throws Exception {
long day1 = Duration.ofDays(Duration.ofNanos(System.nanoTime()).toDays()).toMillis();
long day2 = Duration.ofDays(Duration.ofMillis(day1).toDays() + 1).toMillis();
long day3 = Duration.ofDays(Duration.ofMillis(day1).toDays() + 2).toMillis();
List<ShardIndexInfo> inputShards =
ImmutableList.of(
shardWithTemporalRange(TIMESTAMP, day1, day1),
shardWithTemporalRange(TIMESTAMP, day2, day2),
shardWithTemporalRange(TIMESTAMP, day2, day2),
shardWithTemporalRange(TIMESTAMP, day1, day1),
shardWithTemporalRange(TIMESTAMP, day3, day3));
Set<OrganizationSet> actual =
compactionSetCreator.createCompactionSets(temporalTableInfo, inputShards);
assertEquals(actual.size(), 2);
Set<OrganizationSet> expected =
ImmutableSet.of(
new OrganizationSet(
temporalTableInfo.getTableId(),
extractIndexes(inputShards, 0, 3),
OptionalInt.empty()),
new OrganizationSet(
temporalTableInfo.getTableId(),
extractIndexes(inputShards, 1, 2),
OptionalInt.empty()));
assertEquals(actual, expected);
}
@Test
public void testTemporalCompactionSpanningDays() throws Exception {
long day1 = Duration.ofDays(Duration.ofNanos(System.nanoTime()).toDays()).toMillis();
long day2 = Duration.ofDays(Duration.ofMillis(day1).toDays() + 1).toMillis();
long day3 = Duration.ofDays(Duration.ofMillis(day1).toDays() + 2).toMillis();
long day4 = Duration.ofDays(Duration.ofMillis(day1).toDays() + 3).toMillis();
List<ShardIndexInfo> inputShards =
ImmutableList.of(
shardWithTemporalRange(TIMESTAMP, day1, day3), // day2
shardWithTemporalRange(TIMESTAMP, day2, day2), // day2
shardWithTemporalRange(TIMESTAMP, day1, day1), // day1
shardWithTemporalRange(TIMESTAMP, day1 + 100, day2 + 100), // day1
shardWithTemporalRange(TIMESTAMP, day1 - 100, day2 - 100), // day1
shardWithTemporalRange(TIMESTAMP, day2 - 100, day3 - 100), // day2
shardWithTemporalRange(TIMESTAMP, day1, day4) // day2
);
long tableId = temporalTableInfo.getTableId();
Set<OrganizationSet> compactionSets =
compactionSetCreator.createCompactionSets(temporalTableInfo, inputShards);
assertEquals(compactionSets.size(), 2);
Set<OrganizationSet> expected =
ImmutableSet.of(
new OrganizationSet(
tableId, extractIndexes(inputShards, 0, 1, 5, 6), OptionalInt.empty()),
new OrganizationSet(
tableId, extractIndexes(inputShards, 2, 3, 4), OptionalInt.empty()));
assertEquals(compactionSets, expected);
}
@Test
public void testTemporalCompactionDate() throws Exception {
long day1 = Duration.ofNanos(System.nanoTime()).toDays();
long day2 = day1 + 1;
long day3 = day1 + 2;
List<ShardIndexInfo> inputShards =
ImmutableList.of(
shardWithTemporalRange(DATE, day1, day1),
shardWithTemporalRange(DATE, day2, day2),
shardWithTemporalRange(DATE, day3, day3),
shardWithTemporalRange(DATE, day1, day3),
shardWithTemporalRange(DATE, day2, day3),
shardWithTemporalRange(DATE, day1, day2));
long tableId = temporalTableInfo.getTableId();
Set<OrganizationSet> actual =
compactionSetCreator.createCompactionSets(temporalTableInfo, inputShards);
assertEquals(actual.size(), 2);
Set<OrganizationSet> expected =
ImmutableSet.of(
new OrganizationSet(tableId, extractIndexes(inputShards, 0, 3, 5), OptionalInt.empty()),
new OrganizationSet(tableId, extractIndexes(inputShards, 1, 4), OptionalInt.empty()));
assertEquals(actual, expected);
}
@Test
public void testBucketedTableCompaction() throws Exception {
List<ShardIndexInfo> inputShards =
ImmutableList.of(
shardWithBucket(1),
shardWithBucket(2),
shardWithBucket(2),
shardWithBucket(1),
shardWithBucket(2),
shardWithBucket(1));
long tableId = bucketedTableInfo.getTableId();
Set<OrganizationSet> actual =
compactionSetCreator.createCompactionSets(bucketedTableInfo, inputShards);
assertEquals(actual.size(), 2);
Set<OrganizationSet> expected =
ImmutableSet.of(
new OrganizationSet(tableId, extractIndexes(inputShards, 0, 3, 5), OptionalInt.of(1)),
new OrganizationSet(tableId, extractIndexes(inputShards, 1, 2, 4), OptionalInt.of(2)));
assertEquals(actual, expected);
}
static Set<UUID> extractIndexes(List<ShardIndexInfo> inputShards, int... indexes) {
ImmutableSet.Builder<UUID> builder = ImmutableSet.builder();
for (int index : indexes) {
builder.add(inputShards.get(index).getShardUuid());
}
return builder.build();
}
@Test
public void testBucketedTemporalTableCompaction() throws Exception {
long day1 = 1;
long day2 = 2;
long day3 = 3;
long day4 = 4;
List<ShardIndexInfo> inputShards =
ImmutableList.of(
shardWithTemporalBucket(OptionalInt.of(1), DATE, day1, day1),
shardWithTemporalBucket(OptionalInt.of(2), DATE, day2, day2),
shardWithTemporalBucket(OptionalInt.of(1), DATE, day1, day1),
shardWithTemporalBucket(OptionalInt.of(2), DATE, day2, day2),
shardWithTemporalBucket(OptionalInt.of(1), DATE, day3, day3),
shardWithTemporalBucket(OptionalInt.of(2), DATE, day4, day4));
long tableId = bucketedTemporalTableInfo.getTableId();
Set<OrganizationSet> actual =
compactionSetCreator.createCompactionSets(bucketedTemporalTableInfo, inputShards);
assertEquals(actual.size(), 2);
Set<OrganizationSet> expected =
ImmutableSet.of(
new OrganizationSet(tableId, extractIndexes(inputShards, 0, 2), OptionalInt.of(1)),
new OrganizationSet(tableId, extractIndexes(inputShards, 1, 3), OptionalInt.of(2)));
assertEquals(actual, expected);
}
private static ShardIndexInfo shardWithSize(long rows, long size) {
return new ShardIndexInfo(
1, OptionalInt.empty(), UUID.randomUUID(), rows, size, Optional.empty(), Optional.empty());
}
private static ShardIndexInfo shardWithTemporalRange(Type type, Long start, Long end) {
return shardWithTemporalBucket(OptionalInt.empty(), type, start, end);
}
private static ShardIndexInfo shardWithBucket(int bucketNumber) {
return new ShardIndexInfo(
1,
OptionalInt.of(bucketNumber),
UUID.randomUUID(),
1,
1,
Optional.empty(),
Optional.empty());
}
private static ShardIndexInfo shardWithTemporalBucket(
OptionalInt bucketNumber, Type type, Long start, Long end) {
if (type.equals(DATE)) {
return new ShardIndexInfo(
1,
bucketNumber,
UUID.randomUUID(),
1,
1,
Optional.empty(),
Optional.of(
ShardRange.of(new Tuple(type, start.intValue()), new Tuple(type, end.intValue()))));
}
return new ShardIndexInfo(
1,
bucketNumber,
UUID.randomUUID(),
1,
1,
Optional.empty(),
Optional.of(ShardRange.of(new Tuple(type, start), new Tuple(type, end))));
}
}
@Test
public void should_create_error_message_for_optionalint() {
String errorMessage = shouldBeEmpty(OptionalInt.of(1)).create();
assertThat(errorMessage)
.isEqualTo(format("%nExpecting an empty OptionalInt but was containing value: <1>."));
}
public static void main(String[] args) throws Exception {
if (args.length < 2) {
System.err.println("Ahem, I need at least 2 arguments please.");
System.exit(1);
return;
}
Pattern scalePat = Pattern.compile("-(?:-scale|s)(?:-(w|h))?=(\\d+)");
@SuppressWarnings("unchecked")
Predicate<String> isVerbose =
(Predicate<String>) (Object) Predicate.isEqual("-v").or(Predicate.isEqual("--verbose"));
boolean verbose = Stream.of(args).anyMatch(isVerbose);
OptionalInt[] packed =
Stream.of(args)
.map(scalePat::matcher)
.filter(Matcher::matches)
.map(
matcher -> {
OptionalInt[] ints = new OptionalInt[] {OptionalInt.empty(), OptionalInt.empty()};
OptionalInt scale = OptionalInt.of(Integer.parseInt(matcher.group(2)));
if (matcher.group(1) == null) {
ints[0] = ints[1] = scale;
} else {
if (matcher.group(1).equals("w")) {
ints[0] = scale;
} else {
ints[1] = scale;
}
}
return ints;
})
.reduce(
new OptionalInt[] {OptionalInt.empty(), OptionalInt.empty()},
(id, next) -> {
OptionalInt[] ints = new OptionalInt[2];
OptionalInt aID = id[0];
OptionalInt bID = id[1];
OptionalInt aNx = next[0];
OptionalInt bNx = next[1];
ints[0] = aNx.isPresent() ? aNx : aID;
ints[1] = bNx.isPresent() ? bNx : bID;
return ints;
});
int scaleWidth = packed[0].orElse(1);
int scaleHeight = packed[1].orElse(1);
Pattern lightPat = Pattern.compile("-(?:-light|l)=([dDbB]+)");
List<Function<Color, Color>> lightChanges =
Stream.of(args)
.map(lightPat::matcher)
.filter(Matcher::matches)
.flatMap(
m ->
m.group(1)
.chars()
.mapToObj(
c ->
c == 'd' || c == 'D'
? (Function<Color, Color>) Color::darker
: (Function<Color, Color>) Color::brighter))
.collect(Collectors.toList());
args =
Stream.of(args)
.filter(isVerbose.or(scalePat.asPredicate()).or(lightPat.asPredicate()).negate())
.toArray(String[]::new);
Random r = new Random();
int width = Integer.parseInt(args[0]);
int height = Integer.parseInt(args[1]);
String file = args.length > 2 ? args[2] : ":-";
if (verbose) {
System.err.println("Generating an image...");
}
byte[] pixels = new byte[width * height * 4];
r.nextBytes(pixels);
BufferedImage created = new BufferedImage(width, height, BufferedImage.TYPE_INT_ARGB);
for (int i = 0, index = 0; i < pixels.length; i += 4, index++) {
int x = index % width;
int y = index / width;
Color color =
new Color(
((/* 0x0F */ 0xFF) << 24)
| ((pixels[i + 1] & 0xFF) << 16)
| ((pixels[i + 2] & 0xFF) << 8)
| ((pixels[i + 3] & 0xFF)));
for (Function<Color, Color> change : lightChanges) {
color = change.apply(color);
}
created.setRGB(x, y, color.getRGB());
}
int scaledWidth = width * scaleWidth;
int scaledHeight = height * scaleHeight;
BufferedImage tmp = new BufferedImage(scaledWidth, scaledHeight, BufferedImage.TYPE_INT_ARGB);
Graphics2D g = tmp.createGraphics();
AffineTransform at = AffineTransform.getScaleInstance(scaleWidth, scaleHeight);
g.drawRenderedImage(created, at);
created = tmp;
if (verbose) {
System.err.println("Writing to file...");
}
ImageIO.write(
created, "PNG", file.equals(":-") ? System.out : Files.newOutputStream(Paths.get(file)));
if (verbose) {
System.err.println("Complete...");
}
}
/**
* @param step The step being added
* @param index The index that the step should be added at.
*/
public StepAddedEvent(Step step, @Nonnegative int index) {
this.step = checkNotNull(step, "Step can not be null");
this.index = OptionalInt.of(index);
}
