Exemplo n.º 1
0
  @Override
  public AbstractLongList getRawNeighborhood(AtomicQuery query, InternalTitanTransaction tx) {
    Preconditions.checkArgument(
        QueryUtil.queryCoveredByDiskIndexes(query),
        "Raw retrieval is currently does not support in-memory filtering");
    List<Entry> entries = queryForEntries(query, tx.getTxHandle());

    InternalTitanVertex node = query.getNode();
    TitanType titanType = null;
    if (query.hasEdgeTypeCondition()) titanType = query.getTypeCondition();

    AbstractLongList result = new LongArrayList();

    for (Entry entry : entries) {
      if (!query.hasEdgeTypeCondition()) {
        long etid = IDHandler.readEdgeType(entry.getColumn(), idManager);
        if (titanType == null || titanType.getID() != etid) {
          titanType = getTypeFromID(etid, tx);
        }
      }
      if (titanType.isPropertyKey() || (!titanType.isModifiable() && !query.queryUnmodifiable())) {
        continue; // Skip since it does not match query
      }
      // Get neighboring node id
      long iddiff = VariableLong.read(entry.getValue());
      long nghid = iddiff + node.getID();
      result.add(nghid);

      if (result.size() >= query.getLimit()) break;
    }
    return result;
  }
Exemplo n.º 2
0
  public void checkSlice(
      String[][] values, Set<KeyColumn> removed, int key, int start, int end, int limit) {
    List<Entry> entries;
    if (limit <= 0)
      entries =
          store.getSlice(
              KeyValueStoreUtil.getBuffer(key),
              KeyValueStoreUtil.getBuffer(start),
              KeyValueStoreUtil.getBuffer(end),
              tx);
    else
      entries =
          store.getSlice(
              KeyValueStoreUtil.getBuffer(key),
              KeyValueStoreUtil.getBuffer(start),
              KeyValueStoreUtil.getBuffer(end),
              limit,
              tx);

    int pos = 0;
    for (int i = start; i < end; i++) {
      if (removed.contains(new KeyColumn(key, i))) continue;
      if (limit <= 0 || pos < limit) {
        Entry entry = entries.get(pos);
        int col = KeyValueStoreUtil.getID(entry.getColumn());
        String str = KeyValueStoreUtil.getString(entry.getValue());
        assertEquals(i, col);
        assertEquals(values[key][i], str);
      }
      pos++;
    }
    assertNotNull(entries);
    if (limit > 0 && pos > limit) assertEquals(limit, entries.size());
    else assertEquals(pos, entries.size());
  }
  @Override
  public void mutateMany(Map<ByteBuffer, Mutation> mutations, TransactionHandle txh)
      throws StorageException {
    // null txh means a Transaction is calling this method
    if (null != txh) {
      // non-null txh -> make sure locks are valid
      AstyanaxTransaction atxh = (AstyanaxTransaction) txh;
      if (!atxh.isMutationStarted()) {
        // This is the first mutate call in the transaction
        atxh.mutationStarted();
        // Verify all blind lock claims now
        atxh.verifyAllLockClaims(); // throws GSE and unlocks everything on any lock failure
      }
    }

    MutationBatch m =
        keyspace
            .prepareMutationBatch()
            .setConsistencyLevel(writeLevel)
            .withRetryPolicy(retryPolicy.duplicate());

    final long delTS = TimestampProvider.getApproxNSSinceEpoch(false);
    final long addTS = TimestampProvider.getApproxNSSinceEpoch(true);

    for (Map.Entry<ByteBuffer, Mutation> ent : mutations.entrySet()) {
      // The CLMs for additions and deletions are separated because
      // Astyanax's operation timestamp cannot be set on a per-delete
      // or per-addition basis.
      ColumnListMutation<ByteBuffer> dels = m.withRow(cf, ent.getKey());
      dels.setTimestamp(delTS);
      ColumnListMutation<ByteBuffer> adds = m.withRow(cf, ent.getKey());
      adds.setTimestamp(addTS);

      Mutation titanMutation = ent.getValue();

      if (titanMutation.hasDeletions()) {
        for (ByteBuffer b : titanMutation.getDeletions()) {
          dels.deleteColumn(b);
        }
      }

      if (titanMutation.hasAdditions()) {
        for (Entry e : titanMutation.getAdditions()) {
          adds.putColumn(e.getColumn(), e.getValue(), null);
        }
      }
    }

    try {
      m.execute();
    } catch (ConnectionException e) {
      throw new TemporaryStorageException(e);
    }
  }
Exemplo n.º 4
0
  /**
   * Convert Titan internal Mutation representation into HBase native commands.
   *
   * @param mutations Mutations to convert into HBase commands.
   * @param putTimestamp The timestamp to use for Put commands.
   * @param delTimestamp The timestamp to use for Delete commands.
   * @return Commands sorted by key converted from Titan internal representation.
   */
  private static Map<ByteBuffer, Pair<Put, Delete>> convertToCommands(
      Map<String, Map<ByteBuffer, KCVMutation>> mutations,
      final long putTimestamp,
      final long delTimestamp) {
    Map<ByteBuffer, Pair<Put, Delete>> commandsPerKey =
        new HashMap<ByteBuffer, Pair<Put, Delete>>();

    for (Map.Entry<String, Map<ByteBuffer, KCVMutation>> entry : mutations.entrySet()) {
      byte[] cfName = entry.getKey().getBytes();

      for (Map.Entry<ByteBuffer, KCVMutation> m : entry.getValue().entrySet()) {
        ByteBuffer key = m.getKey();
        KCVMutation mutation = m.getValue();

        Pair<Put, Delete> commands = commandsPerKey.get(key);

        if (commands == null) {
          commands = new Pair<Put, Delete>();
          commandsPerKey.put(key, commands);
        }

        if (mutation.hasDeletions()) {
          if (commands.getSecond() == null)
            commands.setSecond(new Delete(ByteBufferUtil.getArray(key), delTimestamp, null));

          for (ByteBuffer b : mutation.getDeletions()) {
            commands.getSecond().deleteColumns(cfName, ByteBufferUtil.getArray(b), delTimestamp);
          }
        }

        if (mutation.hasAdditions()) {
          if (commands.getFirst() == null)
            commands.setFirst(new Put(ByteBufferUtil.getArray(key), putTimestamp));

          for (Entry e : mutation.getAdditions()) {
            commands
                .getFirst()
                .add(
                    cfName,
                    ByteBufferUtil.getArray(e.getColumn()),
                    putTimestamp,
                    ByteBufferUtil.getArray(e.getValue()));
          }
        }
      }
    }

    return commandsPerKey;
  }
Exemplo n.º 5
0
  @Override
  public void save(
      final Collection<InternalRelation> addedRelations,
      final Collection<InternalRelation> deletedRelations,
      final InternalTitanTransaction tx)
      throws StorageException {
    // Setup
    log.debug(
        "Saving transaction. Added {}, removed {}", addedRelations.size(), deletedRelations.size());
    final Map<TitanType, TypeSignature> signatures = new HashMap<TitanType, TypeSignature>();
    final TransactionHandle txh = tx.getTxHandle();

    final StoreMutator mutator = getStoreMutator(txh);
    final boolean acquireLocks = tx.getTxConfiguration().hasAcquireLocks();

    // 1. Assign TitanVertex IDs
    assignIDs(addedRelations, tx);

    for (int saveAttempt = 0; saveAttempt < maxWriteRetryAttempts; saveAttempt++) {
      //        while (true) { //Indefinite loop, broken if no exception occurs, otherwise retried
      // or failed immediately
      try {
        // 2. Collect deleted edges
        ListMultimap<InternalTitanVertex, InternalRelation> mutations = ArrayListMultimap.create();
        if (deletedRelations != null && !deletedRelations.isEmpty()) {
          for (InternalRelation del : deletedRelations) {
            assert del.isRemoved();
            for (int pos = 0; pos < del.getArity(); pos++) {
              InternalTitanVertex node = del.getVertex(pos);
              if (pos == 0 || !del.isUnidirected()) {
                mutations.put(node, del);
              }
              if (pos == 0
                  && acquireLocks
                  && del.getType().isFunctional()
                  && ((InternalTitanType) del.getType()).isFunctionalLocking()) {
                Entry entry = getEntry(tx, del, node, signatures);
                mutator.acquireEdgeLock(
                    IDHandler.getKey(node.getID()), entry.getColumn(), entry.getValue());
              }
            }
            if (acquireLocks && del.isProperty()) {
              lockKeyedProperty((TitanProperty) del, mutator);
            }
          }
        }

        ListMultimap<InternalTitanType, InternalRelation> simpleEdgeTypes = null;
        ListMultimap<InternalTitanType, InternalRelation> otherEdgeTypes = null;

        // 3. Sort Added Edges
        for (InternalRelation edge : addedRelations) {
          if (edge.isRemoved()) continue;
          assert edge.isNew();

          TitanType et = edge.getType();

          // Give special treatment to edge type definitions
          if (SystemTypeManager.prepersistedSystemTypes.contains(et)) {
            assert edge.getVertex(0) instanceof InternalTitanType;
            InternalTitanType node = (InternalTitanType) edge.getVertex(0);
            assert node.hasID();
            if (node.isSimple()) {
              if (simpleEdgeTypes == null) simpleEdgeTypes = ArrayListMultimap.create();
              simpleEdgeTypes.put(node, edge);
            } else {
              if (otherEdgeTypes == null) otherEdgeTypes = ArrayListMultimap.create();
              otherEdgeTypes.put(node, edge);
            }
          } else { // STANDARD TitanRelation
            assert (edge.getArity() == 1 && edge.isProperty())
                || (edge.getArity() == 2 && edge.isEdge());
            for (int pos = 0; pos < edge.getArity(); pos++) {
              InternalTitanVertex node = edge.getVertex(pos);
              assert node.hasID();
              if (pos == 0 || !edge.isUnidirected()) {
                mutations.put(node, edge);
              }
              if (pos == 0
                  && acquireLocks
                  && edge.getType().isFunctional()
                  && !node.isNew()
                  && ((InternalTitanType) edge.getType()).isFunctionalLocking()) {
                Entry entry = getEntry(tx, edge, node, signatures, true);
                mutator.acquireEdgeLock(IDHandler.getKey(node.getID()), entry.getColumn(), null);
              }
            }
          }
          if (acquireLocks && edge.isProperty()) {
            lockKeyedProperty((TitanProperty) edge, mutator);
          }
        }

        // 3. Persist
        if (simpleEdgeTypes != null) persist(simpleEdgeTypes, signatures, tx, mutator);
        if (otherEdgeTypes != null) persist(otherEdgeTypes, signatures, tx, mutator);
        mutator.flush();

        // Commit saved EdgeTypes to TypeManager
        if (simpleEdgeTypes != null) commitEdgeTypes(simpleEdgeTypes.keySet());
        if (otherEdgeTypes != null) commitEdgeTypes(otherEdgeTypes.keySet());

        if (!mutations.isEmpty()) persist(mutations, signatures, tx, mutator);
        mutator.flush();

        // Successfully completed - return to break out of loop
        break;
      } catch (Throwable e) {
        if (e instanceof TemporaryStorageException) {
          if (saveAttempt < maxWriteRetryAttempts - 1) temporaryStorageException(e);
          else
            throw new PermanentStorageException(
                "Tried committing "
                    + maxWriteRetryAttempts
                    + " times on temporary exception without success",
                e);
        } else if (e instanceof StorageException) {
          throw (StorageException) e;
        } else {
          throw new PermanentStorageException(
              "Unidentified exception occurred during persistence", e);
        }
      }
    }
  }
Exemplo n.º 6
0
  protected void loadRelations(
      Iterable<Entry> entries, VertexRelationLoader loader, InternalTitanTransaction tx) {
    Map<String, TitanType> etCache = new HashMap<String, TitanType>();
    TitanType titanType = null;

    for (Entry entry : entries) {
      ByteBuffer column = entry.getColumn();
      int dirID = IDHandler.getDirectionID(column.get(column.position()));
      long etid = IDHandler.readEdgeType(column, idManager);

      if (titanType == null || titanType.getID() != etid) {
        titanType = getTypeFromID(etid, tx);
      }

      Object[] keys = null;
      if (!titanType.isSimple()) {
        TypeDefinition def = ((InternalTitanType) titanType).getDefinition();
        String[] keysig = def.getKeySignature();
        keys = new Object[keysig.length];
        for (int i = 0; i < keysig.length; i++)
          keys[i] = readInline(column, getEdgeType(keysig[i], etCache, tx));
      }

      long edgeid = 0;
      if (!titanType.isFunctional()) {
        edgeid = VariableLong.readPositive(column);
      }

      ByteBuffer value = entry.getValue();
      if (titanType.isEdgeLabel()) {
        long nodeIDDiff = VariableLong.read(value);
        if (titanType.isFunctional()) edgeid = VariableLong.readPositive(value);
        assert edgeid > 0;
        long otherid = loader.getVertexId() + nodeIDDiff;
        assert dirID == 3 || dirID == 2;
        Direction dir = dirID == 3 ? Direction.IN : Direction.OUT;
        if (!tx.isDeletedRelation(edgeid))
          loader.loadEdge(edgeid, (TitanLabel) titanType, dir, otherid);
      } else {
        assert titanType.isPropertyKey();
        assert dirID == 0;
        TitanKey propType = ((TitanKey) titanType);
        Object attribute = null;

        if (hasGenericDataType(propType)) {
          attribute = serializer.readClassAndObject(value);
        } else {
          attribute = serializer.readObjectNotNull(value, propType.getDataType());
        }
        assert attribute != null;

        if (titanType.isFunctional()) edgeid = VariableLong.readPositive(value);
        assert edgeid > 0;
        if (!tx.isDeletedRelation(edgeid)) loader.loadProperty(edgeid, propType, attribute);
      }

      // Read value inline edges if any
      if (!titanType.isSimple()) {
        TypeDefinition def = ((InternalTitanType) titanType).getDefinition();
        // First create all keys buffered above
        String[] keysig = def.getKeySignature();
        for (int i = 0; i < keysig.length; i++) {
          createInlineEdge(loader, getEdgeType(keysig[i], etCache, tx), keys[i]);
        }

        // value signature
        for (String str : def.getCompactSignature())
          readLabel(loader, value, getEdgeType(str, etCache, tx));

        // Third: read rest
        while (value.hasRemaining()) {
          TitanType type =
              (TitanType) tx.getExistingVertex(IDHandler.readInlineEdgeType(value, idManager));
          readLabel(loader, value, type);
        }
      }
    }
  }