@Override public AbstractLongList getRawNeighborhood(AtomicQuery query, InternalTitanTransaction tx) { Preconditions.checkArgument( QueryUtil.queryCoveredByDiskIndexes(query), "Raw retrieval is currently does not support in-memory filtering"); List<Entry> entries = queryForEntries(query, tx.getTxHandle()); InternalTitanVertex node = query.getNode(); TitanType titanType = null; if (query.hasEdgeTypeCondition()) titanType = query.getTypeCondition(); AbstractLongList result = new LongArrayList(); for (Entry entry : entries) { if (!query.hasEdgeTypeCondition()) { long etid = IDHandler.readEdgeType(entry.getColumn(), idManager); if (titanType == null || titanType.getID() != etid) { titanType = getTypeFromID(etid, tx); } } if (titanType.isPropertyKey() || (!titanType.isModifiable() && !query.queryUnmodifiable())) { continue; // Skip since it does not match query } // Get neighboring node id long iddiff = VariableLong.read(entry.getValue()); long nghid = iddiff + node.getID(); result.add(nghid); if (result.size() >= query.getLimit()) break; } return result; }
public void checkSlice( String[][] values, Set<KeyColumn> removed, int key, int start, int end, int limit) { List<Entry> entries; if (limit <= 0) entries = store.getSlice( KeyValueStoreUtil.getBuffer(key), KeyValueStoreUtil.getBuffer(start), KeyValueStoreUtil.getBuffer(end), tx); else entries = store.getSlice( KeyValueStoreUtil.getBuffer(key), KeyValueStoreUtil.getBuffer(start), KeyValueStoreUtil.getBuffer(end), limit, tx); int pos = 0; for (int i = start; i < end; i++) { if (removed.contains(new KeyColumn(key, i))) continue; if (limit <= 0 || pos < limit) { Entry entry = entries.get(pos); int col = KeyValueStoreUtil.getID(entry.getColumn()); String str = KeyValueStoreUtil.getString(entry.getValue()); assertEquals(i, col); assertEquals(values[key][i], str); } pos++; } assertNotNull(entries); if (limit > 0 && pos > limit) assertEquals(limit, entries.size()); else assertEquals(pos, entries.size()); }
@Override public void mutateMany(Map<ByteBuffer, Mutation> mutations, TransactionHandle txh) throws StorageException { // null txh means a Transaction is calling this method if (null != txh) { // non-null txh -> make sure locks are valid AstyanaxTransaction atxh = (AstyanaxTransaction) txh; if (!atxh.isMutationStarted()) { // This is the first mutate call in the transaction atxh.mutationStarted(); // Verify all blind lock claims now atxh.verifyAllLockClaims(); // throws GSE and unlocks everything on any lock failure } } MutationBatch m = keyspace .prepareMutationBatch() .setConsistencyLevel(writeLevel) .withRetryPolicy(retryPolicy.duplicate()); final long delTS = TimestampProvider.getApproxNSSinceEpoch(false); final long addTS = TimestampProvider.getApproxNSSinceEpoch(true); for (Map.Entry<ByteBuffer, Mutation> ent : mutations.entrySet()) { // The CLMs for additions and deletions are separated because // Astyanax's operation timestamp cannot be set on a per-delete // or per-addition basis. ColumnListMutation<ByteBuffer> dels = m.withRow(cf, ent.getKey()); dels.setTimestamp(delTS); ColumnListMutation<ByteBuffer> adds = m.withRow(cf, ent.getKey()); adds.setTimestamp(addTS); Mutation titanMutation = ent.getValue(); if (titanMutation.hasDeletions()) { for (ByteBuffer b : titanMutation.getDeletions()) { dels.deleteColumn(b); } } if (titanMutation.hasAdditions()) { for (Entry e : titanMutation.getAdditions()) { adds.putColumn(e.getColumn(), e.getValue(), null); } } } try { m.execute(); } catch (ConnectionException e) { throw new TemporaryStorageException(e); } }
/** * Convert Titan internal Mutation representation into HBase native commands. * * @param mutations Mutations to convert into HBase commands. * @param putTimestamp The timestamp to use for Put commands. * @param delTimestamp The timestamp to use for Delete commands. * @return Commands sorted by key converted from Titan internal representation. */ private static Map<ByteBuffer, Pair<Put, Delete>> convertToCommands( Map<String, Map<ByteBuffer, KCVMutation>> mutations, final long putTimestamp, final long delTimestamp) { Map<ByteBuffer, Pair<Put, Delete>> commandsPerKey = new HashMap<ByteBuffer, Pair<Put, Delete>>(); for (Map.Entry<String, Map<ByteBuffer, KCVMutation>> entry : mutations.entrySet()) { byte[] cfName = entry.getKey().getBytes(); for (Map.Entry<ByteBuffer, KCVMutation> m : entry.getValue().entrySet()) { ByteBuffer key = m.getKey(); KCVMutation mutation = m.getValue(); Pair<Put, Delete> commands = commandsPerKey.get(key); if (commands == null) { commands = new Pair<Put, Delete>(); commandsPerKey.put(key, commands); } if (mutation.hasDeletions()) { if (commands.getSecond() == null) commands.setSecond(new Delete(ByteBufferUtil.getArray(key), delTimestamp, null)); for (ByteBuffer b : mutation.getDeletions()) { commands.getSecond().deleteColumns(cfName, ByteBufferUtil.getArray(b), delTimestamp); } } if (mutation.hasAdditions()) { if (commands.getFirst() == null) commands.setFirst(new Put(ByteBufferUtil.getArray(key), putTimestamp)); for (Entry e : mutation.getAdditions()) { commands .getFirst() .add( cfName, ByteBufferUtil.getArray(e.getColumn()), putTimestamp, ByteBufferUtil.getArray(e.getValue())); } } } } return commandsPerKey; }
@Override public void save( final Collection<InternalRelation> addedRelations, final Collection<InternalRelation> deletedRelations, final InternalTitanTransaction tx) throws StorageException { // Setup log.debug( "Saving transaction. Added {}, removed {}", addedRelations.size(), deletedRelations.size()); final Map<TitanType, TypeSignature> signatures = new HashMap<TitanType, TypeSignature>(); final TransactionHandle txh = tx.getTxHandle(); final StoreMutator mutator = getStoreMutator(txh); final boolean acquireLocks = tx.getTxConfiguration().hasAcquireLocks(); // 1. Assign TitanVertex IDs assignIDs(addedRelations, tx); for (int saveAttempt = 0; saveAttempt < maxWriteRetryAttempts; saveAttempt++) { // while (true) { //Indefinite loop, broken if no exception occurs, otherwise retried // or failed immediately try { // 2. Collect deleted edges ListMultimap<InternalTitanVertex, InternalRelation> mutations = ArrayListMultimap.create(); if (deletedRelations != null && !deletedRelations.isEmpty()) { for (InternalRelation del : deletedRelations) { assert del.isRemoved(); for (int pos = 0; pos < del.getArity(); pos++) { InternalTitanVertex node = del.getVertex(pos); if (pos == 0 || !del.isUnidirected()) { mutations.put(node, del); } if (pos == 0 && acquireLocks && del.getType().isFunctional() && ((InternalTitanType) del.getType()).isFunctionalLocking()) { Entry entry = getEntry(tx, del, node, signatures); mutator.acquireEdgeLock( IDHandler.getKey(node.getID()), entry.getColumn(), entry.getValue()); } } if (acquireLocks && del.isProperty()) { lockKeyedProperty((TitanProperty) del, mutator); } } } ListMultimap<InternalTitanType, InternalRelation> simpleEdgeTypes = null; ListMultimap<InternalTitanType, InternalRelation> otherEdgeTypes = null; // 3. Sort Added Edges for (InternalRelation edge : addedRelations) { if (edge.isRemoved()) continue; assert edge.isNew(); TitanType et = edge.getType(); // Give special treatment to edge type definitions if (SystemTypeManager.prepersistedSystemTypes.contains(et)) { assert edge.getVertex(0) instanceof InternalTitanType; InternalTitanType node = (InternalTitanType) edge.getVertex(0); assert node.hasID(); if (node.isSimple()) { if (simpleEdgeTypes == null) simpleEdgeTypes = ArrayListMultimap.create(); simpleEdgeTypes.put(node, edge); } else { if (otherEdgeTypes == null) otherEdgeTypes = ArrayListMultimap.create(); otherEdgeTypes.put(node, edge); } } else { // STANDARD TitanRelation assert (edge.getArity() == 1 && edge.isProperty()) || (edge.getArity() == 2 && edge.isEdge()); for (int pos = 0; pos < edge.getArity(); pos++) { InternalTitanVertex node = edge.getVertex(pos); assert node.hasID(); if (pos == 0 || !edge.isUnidirected()) { mutations.put(node, edge); } if (pos == 0 && acquireLocks && edge.getType().isFunctional() && !node.isNew() && ((InternalTitanType) edge.getType()).isFunctionalLocking()) { Entry entry = getEntry(tx, edge, node, signatures, true); mutator.acquireEdgeLock(IDHandler.getKey(node.getID()), entry.getColumn(), null); } } } if (acquireLocks && edge.isProperty()) { lockKeyedProperty((TitanProperty) edge, mutator); } } // 3. Persist if (simpleEdgeTypes != null) persist(simpleEdgeTypes, signatures, tx, mutator); if (otherEdgeTypes != null) persist(otherEdgeTypes, signatures, tx, mutator); mutator.flush(); // Commit saved EdgeTypes to TypeManager if (simpleEdgeTypes != null) commitEdgeTypes(simpleEdgeTypes.keySet()); if (otherEdgeTypes != null) commitEdgeTypes(otherEdgeTypes.keySet()); if (!mutations.isEmpty()) persist(mutations, signatures, tx, mutator); mutator.flush(); // Successfully completed - return to break out of loop break; } catch (Throwable e) { if (e instanceof TemporaryStorageException) { if (saveAttempt < maxWriteRetryAttempts - 1) temporaryStorageException(e); else throw new PermanentStorageException( "Tried committing " + maxWriteRetryAttempts + " times on temporary exception without success", e); } else if (e instanceof StorageException) { throw (StorageException) e; } else { throw new PermanentStorageException( "Unidentified exception occurred during persistence", e); } } } }
protected void loadRelations( Iterable<Entry> entries, VertexRelationLoader loader, InternalTitanTransaction tx) { Map<String, TitanType> etCache = new HashMap<String, TitanType>(); TitanType titanType = null; for (Entry entry : entries) { ByteBuffer column = entry.getColumn(); int dirID = IDHandler.getDirectionID(column.get(column.position())); long etid = IDHandler.readEdgeType(column, idManager); if (titanType == null || titanType.getID() != etid) { titanType = getTypeFromID(etid, tx); } Object[] keys = null; if (!titanType.isSimple()) { TypeDefinition def = ((InternalTitanType) titanType).getDefinition(); String[] keysig = def.getKeySignature(); keys = new Object[keysig.length]; for (int i = 0; i < keysig.length; i++) keys[i] = readInline(column, getEdgeType(keysig[i], etCache, tx)); } long edgeid = 0; if (!titanType.isFunctional()) { edgeid = VariableLong.readPositive(column); } ByteBuffer value = entry.getValue(); if (titanType.isEdgeLabel()) { long nodeIDDiff = VariableLong.read(value); if (titanType.isFunctional()) edgeid = VariableLong.readPositive(value); assert edgeid > 0; long otherid = loader.getVertexId() + nodeIDDiff; assert dirID == 3 || dirID == 2; Direction dir = dirID == 3 ? Direction.IN : Direction.OUT; if (!tx.isDeletedRelation(edgeid)) loader.loadEdge(edgeid, (TitanLabel) titanType, dir, otherid); } else { assert titanType.isPropertyKey(); assert dirID == 0; TitanKey propType = ((TitanKey) titanType); Object attribute = null; if (hasGenericDataType(propType)) { attribute = serializer.readClassAndObject(value); } else { attribute = serializer.readObjectNotNull(value, propType.getDataType()); } assert attribute != null; if (titanType.isFunctional()) edgeid = VariableLong.readPositive(value); assert edgeid > 0; if (!tx.isDeletedRelation(edgeid)) loader.loadProperty(edgeid, propType, attribute); } // Read value inline edges if any if (!titanType.isSimple()) { TypeDefinition def = ((InternalTitanType) titanType).getDefinition(); // First create all keys buffered above String[] keysig = def.getKeySignature(); for (int i = 0; i < keysig.length; i++) { createInlineEdge(loader, getEdgeType(keysig[i], etCache, tx), keys[i]); } // value signature for (String str : def.getCompactSignature()) readLabel(loader, value, getEdgeType(str, etCache, tx)); // Third: read rest while (value.hasRemaining()) { TitanType type = (TitanType) tx.getExistingVertex(IDHandler.readInlineEdgeType(value, idManager)); readLabel(loader, value, type); } } } }