@Override
public AbstractLongList getRawNeighborhood(AtomicQuery query, InternalTitanTransaction tx) {
Preconditions.checkArgument(
QueryUtil.queryCoveredByDiskIndexes(query),
"Raw retrieval is currently does not support in-memory filtering");
List<Entry> entries = queryForEntries(query, tx.getTxHandle());
InternalTitanVertex node = query.getNode();
TitanType titanType = null;
if (query.hasEdgeTypeCondition()) titanType = query.getTypeCondition();
AbstractLongList result = new LongArrayList();
for (Entry entry : entries) {
if (!query.hasEdgeTypeCondition()) {
long etid = IDHandler.readEdgeType(entry.getColumn(), idManager);
if (titanType == null || titanType.getID() != etid) {
titanType = getTypeFromID(etid, tx);
}
}
if (titanType.isPropertyKey() || (!titanType.isModifiable() && !query.queryUnmodifiable())) {
continue; // Skip since it does not match query
}
// Get neighboring node id
long iddiff = VariableLong.read(entry.getValue());
long nghid = iddiff + node.getID();
result.add(nghid);
if (result.size() >= query.getLimit()) break;
}
return result;
}
public void checkSlice(
String[][] values, Set<KeyColumn> removed, int key, int start, int end, int limit) {
List<Entry> entries;
if (limit <= 0)
entries =
store.getSlice(
KeyValueStoreUtil.getBuffer(key),
KeyValueStoreUtil.getBuffer(start),
KeyValueStoreUtil.getBuffer(end),
tx);
else
entries =
store.getSlice(
KeyValueStoreUtil.getBuffer(key),
KeyValueStoreUtil.getBuffer(start),
KeyValueStoreUtil.getBuffer(end),
limit,
tx);
int pos = 0;
for (int i = start; i < end; i++) {
if (removed.contains(new KeyColumn(key, i))) continue;
if (limit <= 0 || pos < limit) {
Entry entry = entries.get(pos);
int col = KeyValueStoreUtil.getID(entry.getColumn());
String str = KeyValueStoreUtil.getString(entry.getValue());
assertEquals(i, col);
assertEquals(values[key][i], str);
}
pos++;
}
assertNotNull(entries);
if (limit > 0 && pos > limit) assertEquals(limit, entries.size());
else assertEquals(pos, entries.size());
}
@Override
public void mutateMany(Map<ByteBuffer, Mutation> mutations, TransactionHandle txh)
throws StorageException {
// null txh means a Transaction is calling this method
if (null != txh) {
// non-null txh -> make sure locks are valid
AstyanaxTransaction atxh = (AstyanaxTransaction) txh;
if (!atxh.isMutationStarted()) {
// This is the first mutate call in the transaction
atxh.mutationStarted();
// Verify all blind lock claims now
atxh.verifyAllLockClaims(); // throws GSE and unlocks everything on any lock failure
}
}
MutationBatch m =
keyspace
.prepareMutationBatch()
.setConsistencyLevel(writeLevel)
.withRetryPolicy(retryPolicy.duplicate());
final long delTS = TimestampProvider.getApproxNSSinceEpoch(false);
final long addTS = TimestampProvider.getApproxNSSinceEpoch(true);
for (Map.Entry<ByteBuffer, Mutation> ent : mutations.entrySet()) {
// The CLMs for additions and deletions are separated because
// Astyanax's operation timestamp cannot be set on a per-delete
// or per-addition basis.
ColumnListMutation<ByteBuffer> dels = m.withRow(cf, ent.getKey());
dels.setTimestamp(delTS);
ColumnListMutation<ByteBuffer> adds = m.withRow(cf, ent.getKey());
adds.setTimestamp(addTS);
Mutation titanMutation = ent.getValue();
if (titanMutation.hasDeletions()) {
for (ByteBuffer b : titanMutation.getDeletions()) {
dels.deleteColumn(b);
}
}
if (titanMutation.hasAdditions()) {
for (Entry e : titanMutation.getAdditions()) {
adds.putColumn(e.getColumn(), e.getValue(), null);
}
}
}
try {
m.execute();
} catch (ConnectionException e) {
throw new TemporaryStorageException(e);
}
}
/**
* Convert Titan internal Mutation representation into HBase native commands.
*
* @param mutations Mutations to convert into HBase commands.
* @param putTimestamp The timestamp to use for Put commands.
* @param delTimestamp The timestamp to use for Delete commands.
* @return Commands sorted by key converted from Titan internal representation.
*/
private static Map<ByteBuffer, Pair<Put, Delete>> convertToCommands(
Map<String, Map<ByteBuffer, KCVMutation>> mutations,
final long putTimestamp,
final long delTimestamp) {
Map<ByteBuffer, Pair<Put, Delete>> commandsPerKey =
new HashMap<ByteBuffer, Pair<Put, Delete>>();
for (Map.Entry<String, Map<ByteBuffer, KCVMutation>> entry : mutations.entrySet()) {
byte[] cfName = entry.getKey().getBytes();
for (Map.Entry<ByteBuffer, KCVMutation> m : entry.getValue().entrySet()) {
ByteBuffer key = m.getKey();
KCVMutation mutation = m.getValue();
Pair<Put, Delete> commands = commandsPerKey.get(key);
if (commands == null) {
commands = new Pair<Put, Delete>();
commandsPerKey.put(key, commands);
}
if (mutation.hasDeletions()) {
if (commands.getSecond() == null)
commands.setSecond(new Delete(ByteBufferUtil.getArray(key), delTimestamp, null));
for (ByteBuffer b : mutation.getDeletions()) {
commands.getSecond().deleteColumns(cfName, ByteBufferUtil.getArray(b), delTimestamp);
}
}
if (mutation.hasAdditions()) {
if (commands.getFirst() == null)
commands.setFirst(new Put(ByteBufferUtil.getArray(key), putTimestamp));
for (Entry e : mutation.getAdditions()) {
commands
.getFirst()
.add(
cfName,
ByteBufferUtil.getArray(e.getColumn()),
putTimestamp,
ByteBufferUtil.getArray(e.getValue()));
}
}
}
}
return commandsPerKey;
}
@Override
public void save(
final Collection<InternalRelation> addedRelations,
final Collection<InternalRelation> deletedRelations,
final InternalTitanTransaction tx)
throws StorageException {
// Setup
log.debug(
"Saving transaction. Added {}, removed {}", addedRelations.size(), deletedRelations.size());
final Map<TitanType, TypeSignature> signatures = new HashMap<TitanType, TypeSignature>();
final TransactionHandle txh = tx.getTxHandle();
final StoreMutator mutator = getStoreMutator(txh);
final boolean acquireLocks = tx.getTxConfiguration().hasAcquireLocks();
// 1. Assign TitanVertex IDs
assignIDs(addedRelations, tx);
for (int saveAttempt = 0; saveAttempt < maxWriteRetryAttempts; saveAttempt++) {
// while (true) { //Indefinite loop, broken if no exception occurs, otherwise retried
// or failed immediately
try {
// 2. Collect deleted edges
ListMultimap<InternalTitanVertex, InternalRelation> mutations = ArrayListMultimap.create();
if (deletedRelations != null && !deletedRelations.isEmpty()) {
for (InternalRelation del : deletedRelations) {
assert del.isRemoved();
for (int pos = 0; pos < del.getArity(); pos++) {
InternalTitanVertex node = del.getVertex(pos);
if (pos == 0 || !del.isUnidirected()) {
mutations.put(node, del);
}
if (pos == 0
&& acquireLocks
&& del.getType().isFunctional()
&& ((InternalTitanType) del.getType()).isFunctionalLocking()) {
Entry entry = getEntry(tx, del, node, signatures);
mutator.acquireEdgeLock(
IDHandler.getKey(node.getID()), entry.getColumn(), entry.getValue());
}
}
if (acquireLocks && del.isProperty()) {
lockKeyedProperty((TitanProperty) del, mutator);
}
}
}
ListMultimap<InternalTitanType, InternalRelation> simpleEdgeTypes = null;
ListMultimap<InternalTitanType, InternalRelation> otherEdgeTypes = null;
// 3. Sort Added Edges
for (InternalRelation edge : addedRelations) {
if (edge.isRemoved()) continue;
assert edge.isNew();
TitanType et = edge.getType();
// Give special treatment to edge type definitions
if (SystemTypeManager.prepersistedSystemTypes.contains(et)) {
assert edge.getVertex(0) instanceof InternalTitanType;
InternalTitanType node = (InternalTitanType) edge.getVertex(0);
assert node.hasID();
if (node.isSimple()) {
if (simpleEdgeTypes == null) simpleEdgeTypes = ArrayListMultimap.create();
simpleEdgeTypes.put(node, edge);
} else {
if (otherEdgeTypes == null) otherEdgeTypes = ArrayListMultimap.create();
otherEdgeTypes.put(node, edge);
}
} else { // STANDARD TitanRelation
assert (edge.getArity() == 1 && edge.isProperty())
|| (edge.getArity() == 2 && edge.isEdge());
for (int pos = 0; pos < edge.getArity(); pos++) {
InternalTitanVertex node = edge.getVertex(pos);
assert node.hasID();
if (pos == 0 || !edge.isUnidirected()) {
mutations.put(node, edge);
}
if (pos == 0
&& acquireLocks
&& edge.getType().isFunctional()
&& !node.isNew()
&& ((InternalTitanType) edge.getType()).isFunctionalLocking()) {
Entry entry = getEntry(tx, edge, node, signatures, true);
mutator.acquireEdgeLock(IDHandler.getKey(node.getID()), entry.getColumn(), null);
}
}
}
if (acquireLocks && edge.isProperty()) {
lockKeyedProperty((TitanProperty) edge, mutator);
}
}
// 3. Persist
if (simpleEdgeTypes != null) persist(simpleEdgeTypes, signatures, tx, mutator);
if (otherEdgeTypes != null) persist(otherEdgeTypes, signatures, tx, mutator);
mutator.flush();
// Commit saved EdgeTypes to TypeManager
if (simpleEdgeTypes != null) commitEdgeTypes(simpleEdgeTypes.keySet());
if (otherEdgeTypes != null) commitEdgeTypes(otherEdgeTypes.keySet());
if (!mutations.isEmpty()) persist(mutations, signatures, tx, mutator);
mutator.flush();
// Successfully completed - return to break out of loop
break;
} catch (Throwable e) {
if (e instanceof TemporaryStorageException) {
if (saveAttempt < maxWriteRetryAttempts - 1) temporaryStorageException(e);
else
throw new PermanentStorageException(
"Tried committing "
+ maxWriteRetryAttempts
+ " times on temporary exception without success",
e);
} else if (e instanceof StorageException) {
throw (StorageException) e;
} else {
throw new PermanentStorageException(
"Unidentified exception occurred during persistence", e);
}
}
}
}
protected void loadRelations(
Iterable<Entry> entries, VertexRelationLoader loader, InternalTitanTransaction tx) {
Map<String, TitanType> etCache = new HashMap<String, TitanType>();
TitanType titanType = null;
for (Entry entry : entries) {
ByteBuffer column = entry.getColumn();
int dirID = IDHandler.getDirectionID(column.get(column.position()));
long etid = IDHandler.readEdgeType(column, idManager);
if (titanType == null || titanType.getID() != etid) {
titanType = getTypeFromID(etid, tx);
}
Object[] keys = null;
if (!titanType.isSimple()) {
TypeDefinition def = ((InternalTitanType) titanType).getDefinition();
String[] keysig = def.getKeySignature();
keys = new Object[keysig.length];
for (int i = 0; i < keysig.length; i++)
keys[i] = readInline(column, getEdgeType(keysig[i], etCache, tx));
}
long edgeid = 0;
if (!titanType.isFunctional()) {
edgeid = VariableLong.readPositive(column);
}
ByteBuffer value = entry.getValue();
if (titanType.isEdgeLabel()) {
long nodeIDDiff = VariableLong.read(value);
if (titanType.isFunctional()) edgeid = VariableLong.readPositive(value);
assert edgeid > 0;
long otherid = loader.getVertexId() + nodeIDDiff;
assert dirID == 3 || dirID == 2;
Direction dir = dirID == 3 ? Direction.IN : Direction.OUT;
if (!tx.isDeletedRelation(edgeid))
loader.loadEdge(edgeid, (TitanLabel) titanType, dir, otherid);
} else {
assert titanType.isPropertyKey();
assert dirID == 0;
TitanKey propType = ((TitanKey) titanType);
Object attribute = null;
if (hasGenericDataType(propType)) {
attribute = serializer.readClassAndObject(value);
} else {
attribute = serializer.readObjectNotNull(value, propType.getDataType());
}
assert attribute != null;
if (titanType.isFunctional()) edgeid = VariableLong.readPositive(value);
assert edgeid > 0;
if (!tx.isDeletedRelation(edgeid)) loader.loadProperty(edgeid, propType, attribute);
}
// Read value inline edges if any
if (!titanType.isSimple()) {
TypeDefinition def = ((InternalTitanType) titanType).getDefinition();
// First create all keys buffered above
String[] keysig = def.getKeySignature();
for (int i = 0; i < keysig.length; i++) {
createInlineEdge(loader, getEdgeType(keysig[i], etCache, tx), keys[i]);
}
// value signature
for (String str : def.getCompactSignature())
readLabel(loader, value, getEdgeType(str, etCache, tx));
// Third: read rest
while (value.hasRemaining()) {
TitanType type =
(TitanType) tx.getExistingVertex(IDHandler.readInlineEdgeType(value, idManager));
readLabel(loader, value, type);
}
}
}
}