@Test
public void testBasicOperation() throws ConnectionException, InterruptedException {
ApplicationScope scope = new ApplicationScopeImpl(new SimpleId("organization"));
IntegerField field = new IntegerField("count", 5);
Id entityId = new SimpleId(UUIDGenerator.newTimeUUID(), "entity");
UUID version = UUIDGenerator.newTimeUUID();
UniqueValue stored = new UniqueValueImpl(field, entityId, version);
strategy.write(scope, stored).execute();
UniqueValueSet fields =
strategy.load(scope, entityId.getType(), Collections.<Field>singleton(field));
UniqueValue retrieved = fields.getValue(field.getName());
Assert.assertNotNull(retrieved);
assertEquals(stored, retrieved);
Iterator<UniqueValue> allFieldsWritten = strategy.getAllUniqueFields(scope, entityId);
assertTrue(allFieldsWritten.hasNext());
// test this interface. In most cases, we won't know the field name, so we want them all
UniqueValue allFieldsValue = allFieldsWritten.next();
Assert.assertNotNull(allFieldsValue);
assertEquals(field, allFieldsValue.getField());
assertEquals(version, allFieldsValue.getEntityVersion());
assertFalse(allFieldsWritten.hasNext());
}
@Test
public void testCapitalizationFixes() throws ConnectionException {
ApplicationScope scope = new ApplicationScopeImpl(new SimpleId("organization"));
StringField field = new StringField("count", "MiXeD CaSe");
Id entityId = new SimpleId(UUIDGenerator.newTimeUUID(), "entity");
UUID version = UUIDGenerator.newTimeUUID();
UniqueValue stored = new UniqueValueImpl(field, entityId, version);
strategy.write(scope, stored).execute();
UniqueValueSet fields =
strategy.load(scope, entityId.getType(), Collections.<Field>singleton(field));
UniqueValue value = fields.getValue(field.getName());
assertEquals(field.getName(), value.getField().getName());
assertEquals(entityId, value.getEntityId());
// now test will all upper and all lower, we should get it all the same
fields =
strategy.load(
scope,
entityId.getType(),
Collections.<Field>singleton(new StringField(field.getName(), "MIXED CASE")));
value = fields.getValue(field.getName());
assertEquals(field.getName(), value.getField().getName());
assertEquals(entityId, value.getEntityId());
fields =
strategy.load(
scope,
entityId.getType(),
Collections.<Field>singleton(new StringField(field.getName(), "mixed case")));
value = fields.getValue(field.getName());
assertEquals(field.getName(), value.getField().getName());
assertEquals(entityId, value.getEntityId());
Iterator<UniqueValue> allFieldsWritten = strategy.getAllUniqueFields(scope, entityId);
assertTrue(allFieldsWritten.hasNext());
// test this interface. In most cases, we won't know the field name, so we want them all
UniqueValue writtenFieldEntry = allFieldsWritten.next();
Assert.assertNotNull(writtenFieldEntry);
assertEquals(field.getName(), writtenFieldEntry.getField().getName());
assertEquals(field.getValue().toLowerCase(), writtenFieldEntry.getField().getValue());
assertEquals(version, writtenFieldEntry.getEntityVersion());
assertFalse(allFieldsWritten.hasNext());
}
/**
* Queue up an indexOperationMessage for multi region execution
*
* @param indexOperationMessage
*/
public void queueIndexOperationMessage(final IndexOperationMessage indexOperationMessage) {
// don't try to produce something with nothing
if (indexOperationMessage == null || indexOperationMessage.isEmpty()) {
return;
}
final String jsonValue = ObjectJsonSerializer.INSTANCE.toString(indexOperationMessage);
final UUID newMessageId = UUIDGenerator.newTimeUUID();
final int expirationTimeInSeconds =
(int) TimeUnit.MILLISECONDS.toSeconds(indexProcessorFig.getIndexMessageTtl());
// write to the map in ES
esMapPersistence.putString(newMessageId.toString(), jsonValue, expirationTimeInSeconds);
// now queue up the index message
final ElasticsearchIndexEvent elasticsearchIndexEvent =
new ElasticsearchIndexEvent(queueFig.getPrimaryRegion(), newMessageId);
// send to the topic so all regions index the batch
offerTopic(elasticsearchIndexEvent);
}
@Test
public void testWriteWithTTL() throws InterruptedException, ConnectionException {
ApplicationScope scope = new ApplicationScopeImpl(new SimpleId("organization"));
// write object that lives 2 seconds
IntegerField field = new IntegerField("count", 5);
Id entityId = new SimpleId(UUIDGenerator.newTimeUUID(), "entity");
UUID version = UUIDGenerator.newTimeUUID();
UniqueValue stored = new UniqueValueImpl(field, entityId, version);
strategy.write(scope, stored, 5).execute();
Thread.sleep(1000);
// waited one sec, should be still here
UniqueValueSet fields =
strategy.load(scope, entityId.getType(), Collections.<Field>singleton(field));
UniqueValue retrieved = fields.getValue(field.getName());
Assert.assertNotNull(retrieved);
assertEquals(stored, retrieved);
Thread.sleep(5000);
// wait another second, should be gone now
fields = strategy.load(scope, entityId.getType(), Collections.<Field>singleton(field));
UniqueValue nullExpected = fields.getValue(field.getName());
Assert.assertNull(nullExpected);
// we still want to retain the log entry, even if we don't retain the unique value. Deleting
// something
// that doesn't exist is a tombstone, but so is the timeout.
Iterator<UniqueValue> allFieldsWritten = strategy.getAllUniqueFields(scope, entityId);
assertTrue(allFieldsWritten.hasNext());
// test this interface. In most cases, we won't know the field name, so we want them all
UniqueValue writtenFieldEntry = allFieldsWritten.next();
Assert.assertNotNull(writtenFieldEntry);
assertEquals(field, writtenFieldEntry.getField());
assertEquals(version, writtenFieldEntry.getEntityVersion());
assertFalse(allFieldsWritten.hasNext());
}
@Test(expected = NullPointerException.class)
public void loadListParamEntityId() throws ConnectionException {
logEntryStrategy.load(
new ApplicationScopeImpl(new SimpleId("organization")),
null,
UUIDGenerator.newTimeUUID(),
1);
}
@Test(expected = IllegalArgumentException.class)
public void loadListParamSize() throws ConnectionException {
logEntryStrategy.load(
new ApplicationScopeImpl(new SimpleId("organization")),
new SimpleId("test"),
UUIDGenerator.newTimeUUID(),
0);
}
@Test
public void getMultipleEntries() throws ConnectionException {
final Id applicationId = new SimpleId("application");
ApplicationScope context = new ApplicationScopeImpl(applicationId);
final Id id = new SimpleId("test");
int count = 10;
final UUID[] versions = new UUID[count];
final Stage COMPLETE = Stage.COMPLETE;
final MvccLogEntry[] entries = new MvccLogEntry[count];
for (int i = 0; i < count; i++) {
versions[i] = UUIDGenerator.newTimeUUID();
entries[i] = new MvccLogEntryImpl(id, versions[i], COMPLETE, MvccLogEntry.State.COMPLETE);
logEntryStrategy.write(context, entries[i]).execute();
// Read it back
MvccLogEntry returned =
logEntryStrategy.load(context, Collections.singleton(id), versions[i]).getMaxVersion(id);
assertNotNull("Returned value should not be null", returned);
assertEquals("Returned should equal the saved", entries[i], returned);
}
// now do a range scan from the end
List<MvccLogEntry> results = logEntryStrategy.load(context, id, versions[count - 1], count);
assertEquals(count, results.size());
for (int i = 0; i < count; i++) {
final MvccLogEntry saved = entries[count - i - 1];
final MvccLogEntry returned = results.get(i);
assertEquals("Entry was not equal to the saved value", saved, returned);
}
// now delete them all and ensure we get no results back
for (int i = 0; i < count; i++) {
logEntryStrategy.delete(context, id, versions[i]).execute();
}
results = logEntryStrategy.load(context, id, versions[versions.length - 1], versions.length);
assertEquals("All log entries were deleted", 0, results.size());
}
@Override
public ReIndexStatus rebuildIndex(final ReIndexRequestBuilder reIndexRequestBuilder) {
// load our last emitted Scope if a cursor is present
final Optional<EdgeScope> cursor = parseCursor(reIndexRequestBuilder.getCursor());
final CursorSeek<Edge> cursorSeek = getResumeEdge(cursor);
final Optional<ApplicationScope> appId = reIndexRequestBuilder.getApplicationScope();
Preconditions.checkArgument(
!(cursor.isPresent() && appId.isPresent()),
"You cannot specify an app id and a cursor. When resuming with cursor you must omit the appid");
final Observable<ApplicationScope> applicationScopes = getApplications(cursor, appId);
final String jobId = StringUtils.sanitizeUUID(UUIDGenerator.newTimeUUID());
final long modifiedSince = reIndexRequestBuilder.getUpdateTimestamp().or(Long.MIN_VALUE);
// create an observable that loads a batch to be indexed
final Observable<List<EdgeScope>> runningReIndex =
allEntityIdsObservable
.getEdgesToEntities(
applicationScopes,
reIndexRequestBuilder.getCollectionName(),
cursorSeek.getSeekValue())
.buffer(indexProcessorFig.getReindexBufferSize())
.doOnNext(
edges -> {
logger.info("Sending batch of {} to be indexed.", edges.size());
indexService.indexBatch(edges, modifiedSince);
});
// start our sampler and state persistence
// take a sample every sample interval to allow us to resume state with minimal loss
// create our flushing collector and flush the edge scopes to it
runningReIndex
.collect(
() -> new FlushingCollector(jobId),
((flushingCollector, edgeScopes) -> flushingCollector.flushBuffer(edgeScopes)))
.doOnNext(flushingCollector -> flushingCollector.complete())
// subscribe on our I/O scheduler and run the task
.subscribeOn(Schedulers.io())
.subscribe(); // want reindex to continually run so leave subscribe.
return new ReIndexStatus(jobId, Status.STARTED, 0, 0);
}
@Test
public void testDelete() throws ConnectionException {
ApplicationScope scope = new ApplicationScopeImpl(new SimpleId("organization"));
IntegerField field = new IntegerField("count", 5);
Id entityId = new SimpleId(UUIDGenerator.newTimeUUID(), "entity");
UUID version = UUIDGenerator.newTimeUUID();
UniqueValue stored = new UniqueValueImpl(field, entityId, version);
strategy.write(scope, stored).execute();
strategy.delete(scope, stored).execute();
UniqueValueSet fields =
strategy.load(scope, entityId.getType(), Collections.<Field>singleton(field));
UniqueValue nullExpected = fields.getValue(field.getName());
Assert.assertNull(nullExpected);
Iterator<UniqueValue> allFieldsWritten = strategy.getAllUniqueFields(scope, entityId);
assertFalse("No entries left", allFieldsWritten.hasNext());
}
@Test
public void loadNoData() throws ConnectionException {
final Id applicationId = new SimpleId("application");
ApplicationScope context = new ApplicationScopeImpl(applicationId);
final Id id = new SimpleId("test");
final UUID version = UUIDGenerator.newTimeUUID();
MvccLogEntry returned =
logEntryStrategy.load(context, Collections.singleton(id), version).getMaxVersion(id);
assertNull("Returned value should not exist", returned);
}
@Test
public void createAndDeleteEntries() throws ConnectionException {
final Id applicationId = new SimpleId("application");
ApplicationScope context = new ApplicationScopeImpl(applicationId);
final Id id = new SimpleId("test");
final int size = 10;
final List<MvccLogEntry> savedEntries = new ArrayList<>(size);
for (int i = 0; i < size; i++) {
final UUID version = UUIDGenerator.newTimeUUID();
MvccLogEntry saved =
new MvccLogEntryImpl(id, version, Stage.COMMITTED, MvccLogEntry.State.COMPLETE);
logEntryStrategy.write(context, saved).execute();
savedEntries.add(saved);
}
// now test we get them all back
final List<MvccLogEntry> results = logEntryStrategy.loadReversed(context, id, null, size);
assertEquals(size, results.size());
// assert they're the same
for (int i = 0; i < size; i++) {
assertEquals(savedEntries.get(i), results.get(i));
}
// now delete them all
for (final MvccLogEntry mvccLogEntry : savedEntries) {
logEntryStrategy.delete(context, id, mvccLogEntry.getVersion()).execute();
}
// now get them back, should be empty
final List<MvccLogEntry> emptyResults = logEntryStrategy.loadReversed(context, id, null, size);
assertEquals(0, emptyResults.size());
}
@Test
public void createAndDelete() throws ConnectionException {
final Id applicationId = new SimpleId("application");
ApplicationScope context = new ApplicationScopeImpl(applicationId);
final Id id = new SimpleId("test");
final UUID version = UUIDGenerator.newTimeUUID();
for (Stage stage : Stage.values()) {
MvccLogEntry saved = new MvccLogEntryImpl(id, version, stage, MvccLogEntry.State.COMPLETE);
logEntryStrategy.write(context, saved).execute();
// Read it back
MvccLogEntry returned =
logEntryStrategy.load(context, Collections.singleton(id), version).getMaxVersion(id);
assertNotNull("Returned value should not be null", returned);
assertEquals("Returned should equal the saved", saved, returned);
}
}
@Test
public void twoFieldsPerVersion() throws ConnectionException, InterruptedException {
ApplicationScope scope = new ApplicationScopeImpl(new SimpleId("organization"));
Id entityId = new SimpleId(UUIDGenerator.newTimeUUID(), "entity");
final UUID version1 = UUIDGenerator.newTimeUUID();
// write V1 of everything
IntegerField version1Field1 = new IntegerField("count", 1);
StringField version1Field2 = new StringField("field", "v1value");
UniqueValue version1Field1Value = new UniqueValueImpl(version1Field1, entityId, version1);
UniqueValue version1Field2Value = new UniqueValueImpl(version1Field2, entityId, version1);
final MutationBatch batch = strategy.write(scope, version1Field1Value);
batch.mergeShallow(strategy.write(scope, version1Field2Value));
// write V2 of everything
final UUID version2 = UUIDGenerator.newTimeUUID();
IntegerField version2Field1 = new IntegerField("count", 2);
StringField version2Field2 = new StringField("field", "v2value");
UniqueValue version2Field1Value = new UniqueValueImpl(version2Field1, entityId, version2);
UniqueValue version2Field2Value = new UniqueValueImpl(version2Field2, entityId, version2);
batch.mergeShallow(strategy.write(scope, version2Field1Value));
batch.mergeShallow(strategy.write(scope, version2Field2Value));
batch.execute();
UniqueValueSet fields =
strategy.load(
scope, entityId.getType(), Arrays.<Field>asList(version1Field1, version1Field2));
UniqueValue retrieved = fields.getValue(version1Field1.getName());
assertEquals(version1Field1Value, retrieved);
retrieved = fields.getValue(version1Field2.getName());
assertEquals(version1Field2Value, retrieved);
Iterator<UniqueValue> allFieldsWritten = strategy.getAllUniqueFields(scope, entityId);
assertTrue(allFieldsWritten.hasNext());
// test this interface. In most cases, we won't know the field name, so we want them all
UniqueValue allFieldsValue = allFieldsWritten.next();
// version 2 fields should come first, ordered by field name
assertEquals(version2Field1, allFieldsValue.getField());
assertEquals(version2, allFieldsValue.getEntityVersion());
allFieldsValue = allFieldsWritten.next();
assertEquals(version2Field2, allFieldsValue.getField());
assertEquals(version2, allFieldsValue.getEntityVersion());
// version 1 should come next ordered by field name
allFieldsValue = allFieldsWritten.next();
assertEquals(version1Field1, allFieldsValue.getField());
assertEquals(version1, allFieldsValue.getEntityVersion());
allFieldsValue = allFieldsWritten.next();
assertEquals(version1Field2, allFieldsValue.getField());
assertEquals(version1, allFieldsValue.getEntityVersion());
assertFalse(allFieldsWritten.hasNext());
}
@Test(expected = NullPointerException.class)
public void loadListParamContext() throws ConnectionException {
logEntryStrategy.load(null, new SimpleId("test"), UUIDGenerator.newTimeUUID(), 1);
}
@Test(expected = NullPointerException.class)
public void loadParamContext() throws ConnectionException {
logEntryStrategy.load(null, Collections.<Id>emptyList(), UUIDGenerator.newTimeUUID());
}
@Test
public void getReversedEntries() throws ConnectionException {
final Id applicationId = new SimpleId("application");
ApplicationScope context = new ApplicationScopeImpl(applicationId);
final Id id = new SimpleId("test");
int count = 10;
final UUID[] versions = new UUID[count];
final Stage COMPLETE = Stage.COMPLETE;
final MvccLogEntry[] entries = new MvccLogEntry[count];
for (int i = 0; i < count; i++) {
versions[i] = UUIDGenerator.newTimeUUID();
entries[i] = new MvccLogEntryImpl(id, versions[i], COMPLETE, MvccLogEntry.State.COMPLETE);
logEntryStrategy.write(context, entries[i]).execute();
// Read it back
MvccLogEntry returned =
logEntryStrategy.load(context, Collections.singleton(id), versions[i]).getMaxVersion(id);
assertNotNull("Returned value should not be null", returned);
assertEquals("Returned should equal the saved", entries[i], returned);
}
final UUID[] assertVersions = Arrays.copyOf(versions, versions.length);
Arrays.sort(assertVersions, (v1, v2) -> UUIDComparator.staticCompare(v1, v2) * -1);
// now do a range scan from the end
final int half = count / 2;
final List<MvccLogEntry> results =
logEntryStrategy.loadReversed(context, id, versions[0], half);
assertEquals(half, results.size());
for (int i = 0; i < count / 2; i++) {
final MvccLogEntry saved = entries[i];
final MvccLogEntry returned = results.get(i);
assertEquals("Entry was not equal to the saved value", saved, returned);
}
// now get the next batch
final List<MvccLogEntry> results2 =
logEntryStrategy.loadReversed(context, id, versions[half], count);
assertEquals(half, results2.size());
for (int i = 0; i < half; i++) {
final MvccLogEntry saved = entries[half + i];
final MvccLogEntry returned = results2.get(i);
assertEquals("Entry was not equal to the saved value", saved, returned);
}
// now delete them all and ensure we get no results back
for (int i = 0; i < count; i++) {
logEntryStrategy.delete(context, id, versions[i]).execute();
}
final List<MvccLogEntry> results3 =
logEntryStrategy.loadReversed(context, id, null, versions.length);
assertEquals("All log entries were deleted", 0, results3.size());
}