private IndexRule[] getIndexesNeedingPopulation() {
List<IndexRule> indexesNeedingPopulation = new ArrayList<>();
for (SchemaRule rule : schemaCache.schemaRules()) {
if (rule.getKind().isIndex()) {
IndexRule indexRule = (IndexRule) rule;
SchemaIndexProvider provider =
schemaIndexProviders.apply(indexRule.getProviderDescriptor());
if (provider.getInitialState(indexRule.getId()) != InternalIndexState.FAILED) {
indexesNeedingPopulation.add(indexRule);
}
}
}
return indexesNeedingPopulation.toArray(new IndexRule[indexesNeedingPopulation.size()]);
}
private void createConstraintRule(UniquenessConstraint constraint) {
// TODO: Do not create duplicate index
SchemaStore schemaStore = getSchemaStore();
long indexRuleId = schemaStore.nextId();
long constraintRuleId = schemaStore.nextId();
IndexRule indexRule =
IndexRule.constraintIndexRule(
indexRuleId,
constraint.label(),
constraint.propertyKeyId(),
this.schemaIndexProviders.getDefaultProvider().getProviderDescriptor(),
constraintRuleId);
UniquenessConstraintRule constraintRule =
UniquenessConstraintRule.uniquenessConstraintRule(
constraintRuleId, constraint.label(), constraint.propertyKeyId(), indexRuleId);
for (DynamicRecord record : schemaStore.allocateFrom(constraintRule)) {
schemaStore.updateRecord(record);
}
schemaCache.addSchemaRule(constraintRule);
for (DynamicRecord record : schemaStore.allocateFrom(indexRule)) {
schemaStore.updateRecord(record);
}
schemaCache.addSchemaRule(indexRule);
labelsTouched = true;
recordAccess.commit();
}
@Test
public void shouldUpdateHighIdsOnRecoveredTransaction() throws Exception {
// GIVEN
WriteTransaction tx = newWriteTransaction(NO_INDEXING);
int nodeId = 5, relId = 10, relationshipType = 3, propertyKeyId = 4, ruleId = 8;
// WHEN
tx.nodeCreate(nodeId);
tx.createRelationshipTypeToken(relationshipType, "type");
tx.relationshipCreate(relId, 0, nodeId, nodeId);
tx.relAddProperty(
relId,
propertyKeyId,
new long[] {
1l << 60, 1l << 60, 1l << 60, 1l << 60, 1l << 60, 1l << 60, 1l << 60, 1l << 60, 1l << 60,
1l << 60
});
tx.createPropertyKeyToken("key", propertyKeyId);
tx.nodeAddProperty(
nodeId,
propertyKeyId,
"something long and nasty that requires dynamic records for sure I would think and hope. Ok then åäö%!=");
for (int i = 0; i < 10; i++) {
tx.addLabelToNode(10000 + i, nodeId);
}
tx.createSchemaRule(IndexRule.indexRule(ruleId, 100, propertyKeyId, PROVIDER_DESCRIPTOR));
prepareAndCommitRecovered(tx);
// THEN
assertEquals("NodeStore", nodeId + 1, neoStore.getNodeStore().getHighId());
assertEquals(
"DynamicNodeLabelStore", 2, neoStore.getNodeStore().getDynamicLabelStore().getHighId());
assertEquals("RelationshipStore", relId + 1, neoStore.getRelationshipStore().getHighId());
assertEquals(
"RelationshipTypeStore",
relationshipType + 1,
neoStore.getRelationshipTypeStore().getHighId());
assertEquals(
"RelationshipType NameStore",
2,
neoStore.getRelationshipTypeStore().getNameStore().getHighId());
assertEquals("PropertyStore", 2, neoStore.getPropertyStore().getHighId());
assertEquals(
"PropertyStore DynamicStringStore",
2,
neoStore.getPropertyStore().getStringStore().getHighId());
assertEquals(
"PropertyStore DynamicArrayStore",
2,
neoStore.getPropertyStore().getArrayStore().getHighId());
assertEquals(
"PropertyIndexStore",
propertyKeyId + 1,
neoStore.getPropertyStore().getPropertyKeyTokenStore().getHighId());
assertEquals(
"PropertyKeyToken NameStore",
2,
neoStore.getPropertyStore().getPropertyKeyTokenStore().getNameStore().getHighId());
assertEquals("SchemaStore", ruleId + 1, neoStore.getSchemaStore().getHighId());
}
@Test
public void shouldStillReportInternalIndexStateAsPopulatingWhenConstraintIndexIsDonePopulating()
throws Exception {
// given
when(accessor.newUpdater(any(IndexUpdateMode.class))).thenReturn(updater);
IndexingService indexingService =
newIndexingServiceWithMockedDependencies(populator, accessor, withData());
life.start();
// when
indexingService.createIndex(
IndexRule.constraintIndexRule(0, labelId, propertyKeyId, PROVIDER_DESCRIPTOR, null));
IndexProxy proxy = indexingService.getProxyForRule(0);
verify(populator, timeout(1000)).close(true);
try (IndexUpdater updater = proxy.newUpdater(IndexUpdateMode.ONLINE)) {
updater.process(add(10, "foo"));
}
// then
assertEquals(InternalIndexState.POPULATING, proxy.getState());
InOrder order = inOrder(populator, accessor, updater);
order.verify(populator).create();
order.verify(populator).close(true);
order.verify(accessor).newUpdater(IndexUpdateMode.ONLINE);
order.verify(updater).process(add(10, "foo"));
order.verify(updater).close();
}
@Test
public void indexCreationShouldBeIdempotent() throws Exception {
// given
when(accessor.newUpdater(any(IndexUpdateMode.class))).thenReturn(updater);
IndexingService indexingService =
newIndexingServiceWithMockedDependencies(populator, accessor, withData());
life.start();
// when
indexingService.createIndex(
IndexRule.indexRule(0, labelId, propertyKeyId, PROVIDER_DESCRIPTOR));
indexingService.createIndex(
IndexRule.indexRule(0, labelId, propertyKeyId, PROVIDER_DESCRIPTOR));
// We are asserting that the second call to createIndex does not throw an exception.
}
@Test
public void shouldRemoveSchemaRuleWhenRollingBackTransaction() throws Exception {
// GIVEN
WriteTransaction writeTransaction = newWriteTransaction(NO_INDEXING);
// WHEN
final long ruleId = neoStore.getSchemaStore().nextId();
writeTransaction.createSchemaRule(IndexRule.indexRule(ruleId, 10, 7, PROVIDER_DESCRIPTOR));
writeTransaction.prepare();
writeTransaction.rollback();
// THEN
verifyNoMoreInteractions(cacheAccessBackDoor);
}
@Test
public void shouldAddSchemaRuleToCacheWhenApplyingTransactionThatCreatesOne() throws Exception {
// GIVEN
WriteTransaction writeTransaction = newWriteTransaction(NO_INDEXING);
// WHEN
final long ruleId = neoStore.getSchemaStore().nextId();
IndexRule schemaRule = IndexRule.indexRule(ruleId, 10, 8, PROVIDER_DESCRIPTOR);
writeTransaction.createSchemaRule(schemaRule);
writeTransaction.prepare();
writeTransaction.commit();
// THEN
verify(cacheAccessBackDoor).addSchemaRule(schemaRule);
}
@Test
public void createdSchemaRuleRecordMustBeWrittenHeavy() throws Exception {
// THEN
Visitor<XaCommand, RuntimeException> verifier = heavySchemaRuleVerifier();
// GIVEN
WriteTransaction tx = newWriteTransaction(NO_INDEXING, verifier);
long ruleId = 0;
int labelId = 5, propertyKeyId = 7;
SchemaRule rule = IndexRule.indexRule(ruleId, labelId, propertyKeyId, PROVIDER_DESCRIPTOR);
// WHEN
tx.createSchemaRule(rule);
prepareAndCommit(tx);
}
private void createIndexRule(Label label, String propertyKey) {
// TODO: Do not create duplicate index
SchemaStore schemaStore = getSchemaStore();
IndexRule schemaRule =
IndexRule.indexRule(
schemaStore.nextId(),
getOrCreateLabelId(label.name()),
getOrCreatePropertyKeyId(propertyKey),
this.schemaIndexProviders.getDefaultProvider().getProviderDescriptor());
for (DynamicRecord record : schemaStore.allocateFrom(schemaRule)) {
schemaStore.updateRecord(record);
}
schemaCache.addSchemaRule(schemaRule);
}
private void repopulateAllIndexes() throws IOException {
final IndexRule[] rules = getIndexesNeedingPopulation();
final IndexPopulator[] populators = new IndexPopulator[rules.length];
// the store is uncontended at this point, so creating a local LockService is safe.
LockService locks = new ReentrantLockService();
IndexStoreView storeView = new NeoStoreIndexStoreView(locks, neoStore);
final int[] labelIds = new int[rules.length];
final int[] propertyKeyIds = new int[rules.length];
for (int i = 0; i < labelIds.length; i++) {
IndexRule rule = rules[i];
labelIds[i] = rule.getLabel();
propertyKeyIds[i] = rule.getPropertyKey();
populators[i] =
schemaIndexProviders
.apply(rule.getProviderDescriptor())
.getPopulator(rule.getId(), new IndexConfiguration(rule.isConstraintIndex()));
populators[i].create();
}
Visitor<NodePropertyUpdate, IOException> propertyUpdateVisitor =
new Visitor<NodePropertyUpdate, IOException>() {
@Override
public boolean visit(NodePropertyUpdate update) throws IOException {
// Do a lookup from which property has changed to a list of indexes worried about that
// property.
int propertyKeyInQuestion = update.getPropertyKeyId();
for (int i = 0; i < propertyKeyIds.length; i++) {
if (propertyKeyIds[i] == propertyKeyInQuestion) {
if (update.forLabel(labelIds[i])) {
try {
populators[i].add(update.getNodeId(), update.getValueAfter());
} catch (IndexEntryConflictException conflict) {
throw conflict.notAllowed(rules[i].getLabel(), rules[i].getPropertyKey());
}
}
}
}
return true;
}
};
NodeLabelUpdateVisitor labelUpdateVisitor = new NodeLabelUpdateVisitor();
StoreScan<IOException> storeScan =
storeView.visitNodes(labelIds, propertyKeyIds, propertyUpdateVisitor, labelUpdateVisitor);
storeScan.run();
for (IndexPopulator populator : populators) {
populator.close(true);
}
labelUpdateVisitor.close();
}
@Test
public void shouldBringConstraintIndexOnlineWhenExplicitlyToldTo() throws Exception {
// given
IndexingService indexingService =
newIndexingServiceWithMockedDependencies(populator, accessor, withData());
life.start();
// when
indexingService.createIndex(
IndexRule.constraintIndexRule(0, labelId, propertyKeyId, PROVIDER_DESCRIPTOR, null));
IndexProxy proxy = indexingService.getProxyForRule(0);
indexingService.activateIndex(0);
// then
assertEquals(ONLINE, proxy.getState());
InOrder order = inOrder(populator, accessor);
order.verify(populator).create();
order.verify(populator).close(true);
}
@Test
public void shouldLogIndexStateOnStart() throws Exception {
// given
TestLogger logger = new TestLogger();
SchemaIndexProvider provider = mock(SchemaIndexProvider.class);
when(provider.getProviderDescriptor()).thenReturn(PROVIDER_DESCRIPTOR);
SchemaIndexProviderMap providerMap = new DefaultSchemaIndexProviderMap(provider);
TokenNameLookup mockLookup = mock(TokenNameLookup.class);
IndexingService indexingService =
new IndexingService(
mock(JobScheduler.class),
providerMap,
mock(IndexStoreView.class),
mockLookup,
mock(UpdateableSchemaState.class),
mockLogging(logger));
IndexRule onlineIndex = indexRule(1, 1, 1, PROVIDER_DESCRIPTOR);
IndexRule populatingIndex = indexRule(2, 1, 2, PROVIDER_DESCRIPTOR);
IndexRule failedIndex = indexRule(3, 2, 2, PROVIDER_DESCRIPTOR);
when(provider.getInitialState(onlineIndex.getId())).thenReturn(ONLINE);
when(provider.getInitialState(populatingIndex.getId()))
.thenReturn(InternalIndexState.POPULATING);
when(provider.getInitialState(failedIndex.getId())).thenReturn(InternalIndexState.FAILED);
indexingService.initIndexes(asList(onlineIndex, populatingIndex, failedIndex).iterator());
when(mockLookup.labelGetName(1)).thenReturn("LabelOne");
when(mockLookup.labelGetName(2)).thenReturn("LabelTwo");
when(mockLookup.propertyKeyGetName(1)).thenReturn("propertyOne");
when(mockLookup.propertyKeyGetName(2)).thenReturn("propertyTwo");
logger.clear();
// when
indexingService.start();
// then
verify(provider).getPopulationFailure(3);
logger.assertAtLeastOnce(
info("IndexingService.start: index on :LabelOne(propertyOne) is ONLINE"));
logger.assertAtLeastOnce(
info("IndexingService.start: index on :LabelOne(propertyTwo) is POPULATING"));
logger.assertAtLeastOnce(
info("IndexingService.start: index on :LabelTwo(propertyTwo) is FAILED"));
}
@Test
public void shouldRemoveSchemaRuleFromCacheWhenApplyingTransactionThatDeletesOne()
throws Exception {
// GIVEN
SchemaStore schemaStore = neoStore.getSchemaStore();
int labelId = 10, propertyKey = 10;
IndexRule rule =
IndexRule.indexRule(schemaStore.nextId(), labelId, propertyKey, PROVIDER_DESCRIPTOR);
Collection<DynamicRecord> records = schemaStore.allocateFrom(rule);
for (DynamicRecord record : records) {
schemaStore.updateRecord(record);
}
long ruleId = first(records).getId();
WriteTransaction writeTransaction = newWriteTransaction(NO_INDEXING);
// WHEN
writeTransaction.dropSchemaRule(ruleId);
writeTransaction.prepare();
writeTransaction.commit();
// THEN
verify(cacheAccessBackDoor).removeSchemaRuleFromCache(ruleId);
}
@Test
public void shouldCreateEqualNodePropertyUpdatesOnRecoveryOfCreatedNode() throws Exception {
/* There was an issue where recovering a tx where a node with a label and a property
* was created resulted in two exact copies of NodePropertyUpdates. */
// GIVEN
long nodeId = 1;
int labelId = 5, propertyKeyId = 7;
NodePropertyUpdate expectedUpdate =
NodePropertyUpdate.add(nodeId, propertyKeyId, "Neo", new long[] {labelId});
// -- an index
long ruleId = 0;
WriteTransaction tx = newWriteTransaction(NO_INDEXING);
SchemaRule rule = IndexRule.indexRule(ruleId, labelId, propertyKeyId, PROVIDER_DESCRIPTOR);
tx.createSchemaRule(rule);
prepareAndCommit(tx);
// -- and a tx creating a node with that label and property key
IndexingService index = mock(IndexingService.class);
CommandCapturingVisitor commandCapturingVisitor = new CommandCapturingVisitor();
tx = newWriteTransaction(index, commandCapturingVisitor);
tx.nodeCreate(nodeId);
tx.addLabelToNode(labelId, nodeId);
tx.nodeAddProperty(nodeId, propertyKeyId, "Neo");
prepareAndCommit(tx);
verify(index, times(1)).updateIndexes(argThat(matchesAll(expectedUpdate)));
reset(index);
// WHEN
// -- later recovering that tx, there should be only one update
tx = newWriteTransaction(index);
commandCapturingVisitor.injectInto(tx);
prepareAndCommitRecovered(tx);
verify(index, times(1)).updateIndexes(argThat(matchesAll(expectedUpdate)));
}
private IndexRule newIndexRule(long id, int label, int propertyKey) {
return IndexRule.indexRule(id, label, propertyKey, PROVIDER_DESCRIPTOR);
}
private static IndexDescriptor descriptor(IndexRule ruleRecord) {
return new IndexDescriptor(ruleRecord.getLabel(), ruleRecord.getPropertyKey());
}