private static Collection<OIndex<?>> prepareLastIndexVariants(
OClass iSchemaClass, OSQLFilterItemField.FieldChain fieldChain) {
OClass oClass = iSchemaClass;
final Collection<OIndex<?>> result = new ArrayList<OIndex<?>>();
for (int i = 0; i < fieldChain.getItemCount() - 1; i++) {
oClass = oClass.getProperty(fieldChain.getItemName(i)).getLinkedClass();
if (oClass == null) {
return result;
}
}
final Set<OIndex<?>> involvedIndexes =
new TreeSet<OIndex<?>>(
new Comparator<OIndex<?>>() {
public int compare(OIndex<?> o1, OIndex<?> o2) {
return o1.getDefinition().getParamCount() - o2.getDefinition().getParamCount();
}
});
involvedIndexes.addAll(
oClass.getInvolvedIndexes(fieldChain.getItemName(fieldChain.getItemCount() - 1)));
final Collection<Class<? extends OIndex>> indexTypes = new HashSet<Class<? extends OIndex>>(3);
for (OIndex<?> involvedIndex : involvedIndexes) {
if (!indexTypes.contains(involvedIndex.getInternal().getClass())) {
result.add(involvedIndex);
indexTypes.add(involvedIndex.getInternal().getClass());
}
}
return result;
}
public boolean existsCollection(String collectionName) throws SqlInjectionException {
if (Logger.isTraceEnabled()) Logger.trace("Method Start");
OIndex idx = db.getMetadata().getIndexManager().getIndex(COLLECTION_NAME_INDEX);
OIdentifiable record = (OIdentifiable) idx.get(collectionName);
if (Logger.isTraceEnabled()) Logger.trace("Method End");
return (record != null);
}
public ODocument getByName(String collectionName) throws SqlInjectionException {
if (Logger.isTraceEnabled()) Logger.trace("Method Start");
OIndex idx = db.getMetadata().getIndexManager().getIndex(COLLECTION_NAME_INDEX);
OIdentifiable record = (OIdentifiable) idx.get(collectionName);
if (record == null) return null;
return db.load(record.getIdentity());
}
public void testTransactionalUsageWorks() {
database.begin(OTransaction.TXTYPE.OPTIMISTIC);
// OIndex<?> index = getManualIndex();
ComparableBinary key3 =
new ComparableBinary(
new byte[] {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8,
9, 0, 3
});
ODocument doc1 = new ODocument().field("k", "key3");
final OIndex index = getIndex();
index.put(key3, doc1);
ComparableBinary key4 =
new ComparableBinary(
new byte[] {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8,
9, 0, 4
});
ODocument doc2 = new ODocument().field("k", "key4");
index.put(key4, doc2);
database.commit();
Assert.assertEquals(index.get(key3), doc1);
Assert.assertEquals(index.get(key4), doc2);
}
private List<OIdentifiable> applyTailIndexes(final Object lastIndexResult) {
final OIndex<?> beforeTheLastIndex = indexChain.get(indexChain.size() - 2);
Set<Comparable> currentKeys = prepareKeys(beforeTheLastIndex, lastIndexResult);
for (int j = indexChain.size() - 2; j > 0; j--) {
final OIndex<?> currentIndex = indexChain.get(j);
final OIndex<?> nextIndex = indexChain.get(j - 1);
final Set<Comparable> newKeys;
if (isComposite(currentIndex)) {
newKeys = new TreeSet<Comparable>();
for (Comparable currentKey : currentKeys) {
final List<OIdentifiable> currentResult = getFromCompositeIndex(currentKey, currentIndex);
newKeys.addAll(prepareKeys(nextIndex, currentResult));
}
} else {
final OIndexCursor cursor = currentIndex.iterateEntries(currentKeys, true);
final List<OIdentifiable> keys = cursorToList(cursor);
newKeys = prepareKeys(nextIndex, keys);
}
updateStatistic(currentIndex);
currentKeys = newKeys;
}
return applyFirstIndex(currentKeys);
}
@Test(dependsOnMethods = {"testTransactionalUsageWorks"})
public void testTransactionalUsageBreaks2() {
OIndex<?> index = getIndex();
database.begin(OTransaction.TXTYPE.OPTIMISTIC);
ComparableBinary key7 =
new ComparableBinary(
new byte[] {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8,
9, 0, 7
});
ODocument doc1 = new ODocument().field("k", "key7");
index.put(key7, doc1);
ComparableBinary key8 =
new ComparableBinary(
new byte[] {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8,
9, 0, 8
});
ODocument doc2 = new ODocument().field("k", "key8");
index.put(key8, doc2);
database.commit();
Assert.assertEquals(index.get(key7), doc1);
Assert.assertEquals(index.get(key8), doc2);
}
@Override
public boolean execute(final OHttpRequest iRequest, OHttpResponse iResponse) throws Exception {
final String[] urlParts =
checkSyntax(iRequest.url, 3, "Syntax error: index/<database>/<index-name>/<key>/[<value>]");
iRequest.data.commandInfo = "Index remove";
ODatabaseDocumentTx db = null;
try {
db = getProfiledDatabaseInstance(iRequest);
final OIndex<?> index = db.getMetadata().getIndexManager().getIndex(urlParts[2]);
if (index == null)
throw new IllegalArgumentException("Index name '" + urlParts[2] + "' not found");
final boolean found;
if (urlParts.length > 4) found = index.remove(urlParts[3], new ORecordId(urlParts[3]));
else found = index.remove(urlParts[3]);
if (found)
iResponse.send(OHttpUtils.STATUS_OK_CODE, "OK", OHttpUtils.CONTENT_TEXT_PLAIN, null, null);
else
iResponse.send(
OHttpUtils.STATUS_NOTFOUND_CODE,
OHttpUtils.STATUS_NOTFOUND_DESCRIPTION,
OHttpUtils.CONTENT_TEXT_PLAIN,
null,
null);
} finally {
if (db != null) db.close();
}
return false;
}
private void dropClassIndexes(final OClass cls) {
final ODatabaseDocument database = getDatabase();
final OIndexManager indexManager = database.getMetadata().getIndexManager();
for (final OIndex<?> index : indexManager.getClassIndexes(cls.getName()))
indexManager.dropIndex(index.getName());
}
@Test
public void vertexIndexLookupWithValue() {
OrientGraph graph = newGraph();
createVertexIndexLabel1(graph);
String value = "value1";
// verify index created
Assert.assertEquals(
graph.getIndexedKeys(Vertex.class, label1), new HashSet<>(Collections.singletonList(key)));
Assert.assertEquals(
graph.getIndexedKeys(Vertex.class, label2), new HashSet<>(Collections.emptyList()));
Assert.assertEquals(
graph.getIndexedKeys(Edge.class, label1), new HashSet<>(Collections.emptyList()));
Vertex v1 = graph.addVertex(T.label, label1, key, value);
Vertex v2 = graph.addVertex(T.label, label2, key, value);
// looking deep into the internals here - I can't find a nicer way to
// auto verify that an index is actually used
GraphTraversal<Vertex, Vertex> traversal =
graph.traversal().V().has(T.label, P.eq(label1)).has(key, P.eq(value));
OrientGraphStepStrategy.instance().apply(traversal.asAdmin());
OrientGraphStep orientGraphStep = (OrientGraphStep) traversal.asAdmin().getStartStep();
OrientIndexQuery orientIndexQuery = (OrientIndexQuery) orientGraphStep.findIndex().get();
OIndex index = orientIndexQuery.index;
Assert.assertEquals(1, index.getSize());
Assert.assertEquals(v1.id(), index.get(value));
}
@Test
public void testInsertUpdateTransactionWithIndex() throws Exception {
databaseDocumentTx.close();
databaseDocumentTx.open("admin", "admin");
OSchema schema = databaseDocumentTx.getMetadata().getSchema();
schema.reload();
databaseDocumentTx.begin();
ODocument doc = new ODocument("City");
doc.field("name", "");
ODocument doc1 = new ODocument("City");
doc1.field("name", "");
doc = databaseDocumentTx.save(doc);
doc1 = databaseDocumentTx.save(doc1);
databaseDocumentTx.commit();
databaseDocumentTx.begin();
doc = databaseDocumentTx.load(doc);
doc1 = databaseDocumentTx.load(doc1);
doc.field("name", "Rome");
doc1.field("name", "Rome");
databaseDocumentTx.save(doc);
databaseDocumentTx.save(doc1);
databaseDocumentTx.commit();
OIndex idx = schema.getClass("City").getClassIndex("City.name");
Collection<?> coll = (Collection<?>) idx.get("Rome");
Assert.assertEquals(coll.size(), 2);
Assert.assertEquals(idx.getSize(), 2);
}
public List<String> getIndexNames() {
final ArrayList<String> names = new ArrayList<String>(indexChain.size());
for (OIndex<?> oIndex : indexChain) {
names.add(oIndex.getName());
}
return names;
}
@Override
protected void handleObject(OIndex<?> object) {
indexName = object.getName();
OIndexDefinition indexDefinition = object.getDefinition();
if (indexDefinition != null) {
String className = indexDefinition.getClassName();
if (className != null) classModel = new OClassModel(className);
}
}
/** {@inheritDoc} */
@Override
public long getRebuildVersion() {
long rebuildVersion = 0;
for (OIndex<?> index : indexChain) {
rebuildVersion += index.getRebuildVersion();
}
return rebuildVersion;
}
@Override
public Object executeIndexQuery(
OCommandContext iContext,
OIndex<?> index,
final INDEX_OPERATION_TYPE iOperationType,
List<Object> keyParams,
int fetchLimit) {
final OIndexDefinition indexDefinition = index.getDefinition();
final OIndexInternal<?> internalIndex = index.getInternal();
if (!internalIndex.canBeUsedInEqualityOperators()) return null;
final Object result;
if (indexDefinition.getParamCount() == 1) {
final Object key;
if (indexDefinition instanceof OIndexDefinitionMultiValue)
key = ((OIndexDefinitionMultiValue) indexDefinition).createSingleValue(keyParams.get(0));
else key = indexDefinition.createValue(keyParams);
if (key == null) return null;
final Object indexResult;
if (iOperationType == INDEX_OPERATION_TYPE.GET) indexResult = index.get(key);
else indexResult = index.count(key);
result = convertIndexResult(indexResult);
} else {
// in case of composite keys several items can be returned in case of we perform search
// using part of composite key stored in index.
final OCompositeIndexDefinition compositeIndexDefinition =
(OCompositeIndexDefinition) indexDefinition;
final Object keyOne = compositeIndexDefinition.createSingleValue(keyParams);
if (keyOne == null) return null;
final Object keyTwo = compositeIndexDefinition.createSingleValue(keyParams);
if (internalIndex.hasRangeQuerySupport()) {
if (INDEX_OPERATION_TYPE.COUNT.equals(iOperationType)) {
result = index.count(keyOne, true, keyTwo, true, fetchLimit);
} else if (fetchLimit > -1)
result = index.getValuesBetween(keyOne, true, keyTwo, true, fetchLimit);
else result = index.getValuesBetween(keyOne, true, keyTwo, true);
} else {
if (indexDefinition.getParamCount() == keyParams.size()) {
final Object indexResult;
if (iOperationType == INDEX_OPERATION_TYPE.GET) indexResult = index.get(keyOne);
else indexResult = index.count(keyOne);
result = convertIndexResult(indexResult);
} else return null;
}
}
updateProfiler(iContext, index, keyParams, indexDefinition);
return result;
}
/** Execute the INSERT and return the ODocument object created. */
public Object execute(final Map<Object, Object> iArgs) {
if (newRecords == null && content == null)
throw new OCommandExecutionException(
"Cannot execute the command because it has not been parsed yet");
final OCommandParameters commandParameters = new OCommandParameters(iArgs);
if (indexName != null) {
if (newRecords == null) throw new OCommandExecutionException("No key/value found");
final OIndex<?> index = getDatabase().getMetadata().getIndexManager().getIndex(indexName);
if (index == null)
throw new OCommandExecutionException("Target index '" + indexName + "' not found");
// BIND VALUES
Map<String, Object> result = null;
for (Map<String, Object> candidate : newRecords) {
index.put(
getIndexKeyValue(commandParameters, candidate),
getIndexValue(commandParameters, candidate));
result = candidate;
}
// RETURN LAST ENTRY
return new ODocument(result);
} else {
// CREATE NEW DOCUMENTS
final List<ODocument> docs = new ArrayList<ODocument>();
if (newRecords != null) {
for (Map<String, Object> candidate : newRecords) {
final ODocument doc = className != null ? new ODocument(className) : new ODocument();
OSQLHelper.bindParameters(doc, candidate, commandParameters, context);
if (clusterName != null) {
doc.save(clusterName);
} else {
doc.save();
}
docs.add(doc);
}
if (docs.size() == 1) return docs.get(0);
else return docs;
} else if (content != null) {
final ODocument doc = className != null ? new ODocument(className) : new ODocument();
doc.merge(content, true, false);
doc.save();
return doc;
}
}
return null;
}
public void testWithIndex() {
databaseDocumentTx
.command(
new OCommandSQL(
"insert into Place set name = 'Test' , location = ST_GeomFromText('" + WKT + "')"))
.execute();
OIndex<?> index = databaseDocumentTx.getMetadata().getIndexManager().getIndex("Place.location");
Assert.assertEquals(index.getSize(), 1);
testQueryMultiLineString();
}
public static void drop(OSchema schema) {
OClass oClass = schema.getClass("Comment");
if (oClass != null) {
// Indexes
Set<OIndex<?>> idxs = oClass.getIndexes();
if (idxs != null && idxs.size() > 0) {
for (OIndex<?> idx : idxs) {
idx.delete();
}
}
// class
schema.dropClass("Comment");
}
}
private static boolean supportNullValues(OIndex<?> index) {
final ODocument metadata = index.getMetadata();
if (metadata == null) return false;
final Boolean ignoreNullValues = metadata.field("ignoreNullValues");
return Boolean.FALSE.equals(ignoreNullValues);
}
@Override
public OIndexCursor iterateEntriesBetween(
Object fromKey, boolean fromInclusive, Object toKey, boolean toInclusive, boolean ascOrder) {
final OIndexCursor internalCursor =
lastIndex.iterateEntriesBetween(fromKey, fromInclusive, toKey, toInclusive, ascOrder);
return new ExternalIndexCursor(internalCursor);
}
private String getAnalyzer(final OIndex classIndex) {
// analyzer is stored only in metadata and there is no way to get default analyzer.. just assume
// it
final ODocument metadata = classIndex.getMetadata();
final String analyzer = metadata != null ? metadata.<String>field(ANALYZER) : null;
return Objects.firstNonNull(analyzer, StandardAnalyzer.class.getName());
}
@Test
public void testDelete() {
create();
for (int i = 0; i < ITERATIONS; ++i) {
Integer deleted =
database.command(new OCommandSQL("delete from AutoShardingTest where id = ?")).execute(i);
Assert.assertEquals(deleted.intValue(), 2);
long totExpected = ITERATIONS - (i + 1);
Assert.assertEquals(idx.getSize(), totExpected * 2);
Assert.assertEquals(idx.getKeySize(), totExpected);
}
Assert.assertEquals(idx.getSize(), 0);
Assert.assertEquals(idx.getKeySize(), 0);
}
/** Execute the REMOVE INDEX. */
public Object execute(final Map<Object, Object> iArgs) {
if (name == null)
throw new OCommandExecutionException(
"Cannot execute the command because it has not been parsed yet");
if (name.equals("*")) {
long totalIndexed = 0;
for (OIndex<?> idx : getDatabase().getMetadata().getIndexManager().getIndexes()) {
getDatabase().getMetadata().getIndexManager().dropIndex(idx.getName());
totalIndexed++;
}
return totalIndexed;
} else getDatabase().getMetadata().getIndexManager().dropIndex(name);
return 1;
}
public <RET extends T> RET get(final String iKey, final String iFetchPlan) {
final OIdentifiable value = index.get(iKey);
if (value == null) return null;
if (value instanceof ORID)
return (RET) ODatabaseRecordThreadLocal.INSTANCE.get().load(((ORID) value), iFetchPlan);
return (RET) ((ODocument) value).load(iFetchPlan);
}
@Override
public void afterRegistration(
final OObjectDatabaseTx db,
final SchemeDescriptor descriptor,
final Field field,
final LuceneIndex annotation) {
db.getMetadata().getIndexManager().reload();
final String property = field.getName();
final String model = descriptor.schemeClass;
final String name =
Objects.firstNonNull(Strings.emptyToNull(annotation.name().trim()), model + '.' + property);
final OClass clazz = db.getMetadata().getSchema().getClass(model);
final OIndex<?> classIndex = clazz.getClassIndex(name);
final OClass.INDEX_TYPE type = OClass.INDEX_TYPE.FULLTEXT;
if (!descriptor.initialRegistration && classIndex != null) {
final IndexValidationSupport support = new IndexValidationSupport(classIndex, logger);
support.checkTypeCompatible(type);
support.checkFieldsCompatible(property);
final boolean correct =
support
.isIndexSigns(
classIndex.getConfiguration().field("algorithm"), getAnalyzer(classIndex))
.matchRequiredSigns(
type, OLuceneIndexFactory.LUCENE_ALGORITHM, annotation.value().getName());
if (!correct) {
support.dropIndex(db);
} else {
// index ok
return;
}
}
final ODocument metadata = createMetadata(annotation);
SchemeUtils.command(
db,
"create index %s on %s (%s) %s engine %s metadata %s",
name,
model,
property,
type.name(),
OLuceneIndexFactory.LUCENE_ALGORITHM,
metadata.toJSON());
logger.info("Lucene fulltext index '{}' ({} [{}]) created", name, model, property);
}
private void compareIndexes() {
ODatabaseRecordThreadLocal.INSTANCE.set(baseDocumentTx);
OIndexCursor cursor = baseDocumentTx.getMetadata().getIndexManager().getIndex("mi").cursor();
long lastTs = 0;
long minLostTs = Long.MAX_VALUE;
long restoredRecords = 0;
Map.Entry<Object, OIdentifiable> entry = cursor.nextEntry();
while (entry != null) {
ODatabaseRecordThreadLocal.INSTANCE.set(baseDocumentTx);
Integer key = (Integer) entry.getKey();
OIdentifiable identifiable = entry.getValue();
ODocument doc = identifiable.getRecord();
long ts = doc.<Long>field("ts");
if (ts > lastTs) lastTs = ts;
entry = cursor.nextEntry();
ODatabaseRecordThreadLocal.INSTANCE.set(testDocumentTx);
OIndex testIndex = testDocumentTx.getMetadata().getIndexManager().getIndex("mi");
Set<OIdentifiable> result = (Set<OIdentifiable>) testIndex.get(key);
if (result == null || result.isEmpty()) {
if (minLostTs > ts) minLostTs = ts;
} else restoredRecords++;
}
ODatabaseRecordThreadLocal.INSTANCE.set(baseDocumentTx);
System.out.println(
"Restored entries : "
+ restoredRecords
+ " out of : "
+ baseDocumentTx.getMetadata().getIndexManager().getIndex("mi").getSize());
long maxInterval = minLostTs == Long.MAX_VALUE ? 0 : lastTs - minLostTs;
System.out.println("Lost records max interval (ms) : " + maxInterval);
assertThat(maxInterval).isLessThan(2000);
}
/** {@inheritDoc} */
@Override
public T get(Object iKey) {
final Object lastIndexResult = lastIndex.get(iKey);
final Set<OIdentifiable> result = new HashSet<OIdentifiable>();
if (lastIndexResult != null) result.addAll(applyTailIndexes(lastIndexResult));
return (T) result;
}
@Test
public void testUpdate() {
create();
for (int i = 0; i < ITERATIONS; ++i) {
Integer updated =
database
.command(
new OCommandSQL(
"update AutoShardingTest INCREMENT id = " + ITERATIONS + " where id = ?"))
.execute(i);
Assert.assertEquals(updated.intValue(), 2);
Assert.assertEquals(idx.getSize(), ITERATIONS * 2);
Assert.assertEquals(idx.getKeySize(), ITERATIONS);
}
Assert.assertEquals(idx.getSize(), ITERATIONS * 2);
Assert.assertEquals(idx.getKeySize(), ITERATIONS);
}
public void addIndexEntry(
final OIndex<?> delegate,
final String iIndexName,
final OPERATION iStatus,
final Object iKey,
final OIdentifiable iValue) {
switch (iStatus) {
case CLEAR:
delegate.clear();
break;
case PUT:
delegate.put(iKey, iValue);
break;
case REMOVE:
if (iKey == null) delegate.remove(iValue);
else delegate.remove(iKey, iValue);
break;
}
}
/**
* Make type conversion of keys for specific index.
*
* @param index - index for which keys prepared for.
* @param keys - which should be prepared.
* @return keys converted to necessary type.
*/
private Set<Comparable> prepareKeys(OIndex<?> index, Object keys) {
final OIndexDefinition indexDefinition = index.getDefinition();
if (keys instanceof Collection) {
final Set<Comparable> newKeys = new TreeSet<Comparable>();
for (Object o : ((Collection) keys)) {
newKeys.add((Comparable) indexDefinition.createValue(o));
}
return newKeys;
} else {
return Collections.singleton((Comparable) indexDefinition.createValue(keys));
}
}
@Test
public void testKeyCursor() {
create();
final OIndexKeyCursor cursor = idx.keyCursor();
Assert.assertNotNull(cursor);
int count = 0;
for (Object entry = cursor.next(0); entry != null; entry = cursor.next(0)) {
count++;
}
Assert.assertEquals(count, ITERATIONS);
}
