コード例 #1
0
  @Override
  public void update(Key k, Value v) {
    // SIGNAL from SingleTransposeIterator
    if (k.getTimestamp() % 2 != 0) return;

    String rStr;
    {
      ByteSequence rowData = k.getRowData();
      rStr = new String(rowData.toArray(), rowData.offset(), rowData.length());
    }
    int pos = rStr.indexOf(edgeSep);
    if (pos == -1) return; // this is a degree row, not an edge row.

    //    log.debug("edge row "+rStr+" : now "+setNodesReached.toString());

    String toNode = rStr.substring(pos + 1);
    setNodesReached.add(toNode);

    //    if (copyDeg) {
    //      Integer cnt = setNodesReachedCount.get(toNode);
    //      cnt = cnt == null ? new Integer(1) : new Integer(cnt+1);
    //      setNodesReachedCount.put(toNode, cnt);
    //    }

  }
コード例 #2
0
 @Override
 public boolean accept(Key k, Value v) {
   long ts = k.getTimestamp();
   if ((hasStart && (ts < start)) || (hasEnd && (ts > end))) return false;
   if (hasStart && !startInclusive && ts == start) return false;
   if (hasEnd && !endInclusive && ts == end) return false;
   return true;
 }
コード例 #3
0
  private static AccumuloBackedGraph setupGraph(
      Instance instance, Connector conn, String tableName, int numEntries) {
    long ageOffTimeInMilliseconds = (30 * 24 * 60 * 60 * 1000L); // 30 days in milliseconds

    try {
      // Create table
      // (this method creates the table, removes the versioning iterator, and adds the
      // SetOfStatisticsCombiner iterator,
      // and sets the age off iterator to age data off after it is more than
      // ageOffTimeInMilliseconds milliseconds old).
      TableUtils.createTable(conn, tableName, ageOffTimeInMilliseconds);

      // Create numEntries edges and add to Accumulo
      BatchWriter writer = conn.createBatchWriter(tableName, 1000000L, 1000L, 1);
      for (int i = 0; i < numEntries; i++) {
        Edge edge =
            new Edge(
                "customer",
                "" + i,
                "product",
                "B",
                "purchase",
                "instore",
                true,
                visibilityString,
                sevenDaysBefore,
                sixDaysBefore);
        SetOfStatistics statistics = new SetOfStatistics();
        statistics.addStatistic("count", new Count(i));
        Key key = ConversionUtils.getKeysFromEdge(edge).getFirst();
        Value value = ConversionUtils.getValueFromSetOfStatistics(statistics);
        Mutation m = new Mutation(key.getRow());
        m.put(
            key.getColumnFamily(),
            key.getColumnQualifier(),
            new ColumnVisibility(key.getColumnVisibility()),
            key.getTimestamp(),
            value);
        writer.addMutation(m);
      }
      writer.close();

      // Create Accumulo backed graph
      AccumuloBackedGraph graph = new AccumuloBackedGraph(conn, tableName);
      return graph;
    } catch (AccumuloException e) {
      fail("Failed to set up graph in Accumulo with exception: " + e);
    } catch (AccumuloSecurityException e) {
      fail("Failed to set up graph in Accumulo with exception: " + e);
    } catch (TableExistsException e) {
      fail("Failed to set up graph in Accumulo with exception: " + e);
    } catch (TableNotFoundException e) {
      fail("Failed to set up graph in Accumulo with exception: " + e);
    }
    return null;
  }
コード例 #4
0
 private static <M extends Map<Key, V>, V> M transposeMapHelp(Map<Key, V> orig, M neww) {
   for (Map.Entry<Key, V> entry : orig.entrySet()) {
     Key k0 = entry.getKey();
     Key k =
         new Key(
             k0.getColumnQualifier(),
             k0.getColumnFamily(),
             k0.getRow(),
             k0.getColumnVisibilityParsed(),
             k0.getTimestamp());
     neww.put(k, entry.getValue());
   }
   return neww;
 }
コード例 #5
0
 private static TreeSet<RowColumn> scanAll(
     ClientOnDefaultTable opts, ScannerOpts scanOpts, String tableName) throws Exception {
   TreeSet<RowColumn> result = new TreeSet<RowColumn>();
   Connector conn = opts.getConnector();
   Scanner scanner = conn.createScanner(tableName, auths);
   scanner.setBatchSize(scanOpts.scanBatchSize);
   for (Entry<Key, Value> entry : scanner) {
     Key key = entry.getKey();
     Column column =
         new Column(
             TextUtil.getBytes(key.getColumnFamily()),
             TextUtil.getBytes(key.getColumnQualifier()),
             TextUtil.getBytes(key.getColumnVisibility()));
     result.add(new RowColumn(key.getRow(), column, key.getTimestamp()));
   }
   return result;
 }
  private void testAggregatingMultiplePropertySetsAcrossRowIDRange(
      final AccumuloStore store, final AccumuloElementConverter elementConverter)
      throws StoreException, AccumuloElementConversionException, RangeFactoryException {
    String visibilityString = "public";
    try {
      // Create table
      // (this method creates the table, removes the versioning iterator, and adds the
      // SetOfStatisticsCombiner iterator).
      TableUtils.createTable(store);

      final Properties properties1 = new Properties();
      properties1.put(AccumuloPropertyNames.COUNT, 1);

      final Properties properties2 = new Properties();
      properties2.put(AccumuloPropertyNames.COUNT, 1);

      final Properties properties3 = new Properties();
      properties3.put(AccumuloPropertyNames.COUNT, 2);

      // Create edge
      final Edge edge = new Edge(TestGroups.EDGE);
      edge.setSource("2");
      edge.setDestination("1");
      edge.setDirected(true);
      edge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
      edge.putProperty(AccumuloPropertyNames.PROP_1, 0);
      edge.putProperty(AccumuloPropertyNames.PROP_2, 0);
      edge.putProperty(AccumuloPropertyNames.PROP_3, 0);
      edge.putProperty(AccumuloPropertyNames.PROP_4, 0);

      final Edge edge2 = new Edge(TestGroups.EDGE);
      edge2.setSource("B");
      edge2.setDestination("Z");
      edge2.setDirected(true);
      edge2.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
      edge2.putProperty(AccumuloPropertyNames.PROP_1, 1);
      edge2.putProperty(AccumuloPropertyNames.PROP_2, 1);
      edge2.putProperty(AccumuloPropertyNames.PROP_3, 1);
      edge2.putProperty(AccumuloPropertyNames.PROP_4, 1);

      final Edge edge3 = new Edge(TestGroups.EDGE);
      edge3.setSource("3");
      edge3.setDestination("8");
      edge3.setDirected(true);
      edge3.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
      edge3.putProperty(AccumuloPropertyNames.PROP_1, 0);
      edge3.putProperty(AccumuloPropertyNames.PROP_2, 0);
      edge3.putProperty(AccumuloPropertyNames.PROP_3, 0);
      edge3.putProperty(AccumuloPropertyNames.PROP_4, 0);

      final Edge edge6 = new Edge("BasicEdge2");
      edge6.setSource("1");
      edge6.setDestination("5");
      edge6.setDirected(true);
      edge6.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 2);
      edge6.putProperty(AccumuloPropertyNames.PROP_1, 0);
      edge6.putProperty(AccumuloPropertyNames.PROP_2, 0);
      edge6.putProperty(AccumuloPropertyNames.PROP_3, 0);
      edge6.putProperty(AccumuloPropertyNames.PROP_4, 0);

      final Edge edge7 = new Edge("BasicEdge2");
      edge7.setSource("2");
      edge7.setDestination("6");
      edge7.setDirected(true);
      edge7.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 1);
      edge7.putProperty(AccumuloPropertyNames.PROP_1, 0);
      edge7.putProperty(AccumuloPropertyNames.PROP_2, 0);
      edge7.putProperty(AccumuloPropertyNames.PROP_3, 0);
      edge7.putProperty(AccumuloPropertyNames.PROP_4, 0);

      final Edge edge8 = new Edge("BasicEdge2");
      edge8.setSource("4");
      edge8.setDestination("8");
      edge8.setDirected(true);
      edge8.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 2);
      edge8.putProperty(AccumuloPropertyNames.PROP_1, 0);
      edge8.putProperty(AccumuloPropertyNames.PROP_2, 0);
      edge8.putProperty(AccumuloPropertyNames.PROP_3, 0);
      edge8.putProperty(AccumuloPropertyNames.PROP_4, 0);

      final Edge edge9 = new Edge("BasicEdge2");
      edge9.setSource("5");
      edge9.setDestination("9");
      edge9.setDirected(true);
      edge9.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 2);
      edge9.putProperty(AccumuloPropertyNames.PROP_1, 0);
      edge9.putProperty(AccumuloPropertyNames.PROP_2, 0);
      edge9.putProperty(AccumuloPropertyNames.PROP_3, 0);
      edge9.putProperty(AccumuloPropertyNames.PROP_4, 0);

      // Accumulo key
      final Key key = elementConverter.getKeysFromEdge(edge).getFirst();
      final Key key2 = elementConverter.getKeysFromEdge(edge2).getFirst();
      final Key key3 = elementConverter.getKeysFromEdge(edge3).getFirst();
      final Key key4 = elementConverter.getKeysFromEdge(edge6).getFirst();
      final Key key5 = elementConverter.getKeysFromEdge(edge7).getFirst();
      final Key key6 = elementConverter.getKeysFromEdge(edge8).getFirst();
      final Key key7 = elementConverter.getKeysFromEdge(edge9).getFirst();

      // Accumulo values
      final Value value1 = elementConverter.getValueFromProperties(TestGroups.EDGE, properties1);
      final Value value2 = elementConverter.getValueFromProperties(TestGroups.EDGE, properties2);
      final Value value3 = elementConverter.getValueFromProperties(TestGroups.EDGE, properties3);
      final Value value4 = elementConverter.getValueFromProperties(TestGroups.EDGE_2, properties1);
      final Value value5 = elementConverter.getValueFromProperties(TestGroups.EDGE_2, properties2);

      // Create mutation
      final Mutation m1 = new Mutation(key.getRow());
      m1.put(
          key.getColumnFamily(),
          key.getColumnQualifier(),
          new ColumnVisibility(key.getColumnVisibility()),
          key.getTimestamp(),
          value1);
      final Mutation m2 = new Mutation(key.getRow());
      m2.put(
          key.getColumnFamily(),
          key.getColumnQualifier(),
          new ColumnVisibility(key.getColumnVisibility()),
          key.getTimestamp(),
          value2);
      final Mutation m3 = new Mutation(key.getRow());
      m3.put(
          key.getColumnFamily(),
          key.getColumnQualifier(),
          new ColumnVisibility(key.getColumnVisibility()),
          key.getTimestamp(),
          value3);
      final Mutation m4 = new Mutation(key2.getRow());
      m4.put(
          key2.getColumnFamily(),
          key2.getColumnQualifier(),
          new ColumnVisibility(key2.getColumnVisibility()),
          key2.getTimestamp(),
          value1);
      final Mutation m5 = new Mutation(key.getRow());
      m5.put(
          key3.getColumnFamily(),
          key3.getColumnQualifier(),
          new ColumnVisibility(key3.getColumnVisibility()),
          key3.getTimestamp(),
          value1);
      final Mutation m6 = new Mutation(key4.getRow());
      m6.put(
          key4.getColumnFamily(),
          key4.getColumnQualifier(),
          new ColumnVisibility(key4.getColumnVisibility()),
          key4.getTimestamp(),
          value4);
      final Mutation m7 = new Mutation(key5.getRow());
      m7.put(
          key5.getColumnFamily(),
          key5.getColumnQualifier(),
          new ColumnVisibility(key5.getColumnVisibility()),
          key5.getTimestamp(),
          value5);
      final Mutation m8 = new Mutation(key6.getRow());
      m8.put(
          key6.getColumnFamily(),
          key6.getColumnQualifier(),
          new ColumnVisibility(key6.getColumnVisibility()),
          key6.getTimestamp(),
          value5);
      final Mutation m9 = new Mutation(key7.getRow());
      m9.put(
          key7.getColumnFamily(),
          key7.getColumnQualifier(),
          new ColumnVisibility(key7.getColumnVisibility()),
          key7.getTimestamp(),
          value5);

      // Write mutation
      final BatchWriterConfig writerConfig = new BatchWriterConfig();
      writerConfig.setMaxMemory(1000000L);
      writerConfig.setMaxLatency(1000L, TimeUnit.MILLISECONDS);
      writerConfig.setMaxWriteThreads(1);
      final BatchWriter writer =
          store.getConnection().createBatchWriter(store.getProperties().getTable(), writerConfig);
      writer.addMutation(m1);
      writer.addMutation(m2);
      writer.addMutation(m3);
      writer.addMutation(m4);
      writer.addMutation(m5);
      writer.addMutation(m6);
      writer.addMutation(m7);
      writer.addMutation(m8);
      writer.addMutation(m9);
      writer.close();

      // Read data back and check we get one merged element
      final Authorizations authorizations = new Authorizations(visibilityString);
      final BatchScanner scanner =
          store
              .getConnection()
              .createBatchScanner(store.getProperties().getTable(), authorizations, 1000);
      try {
        scanner.addScanIterator(
            store
                .getKeyPackage()
                .getIteratorFactory()
                .getRowIDAggregatorIteratorSetting(store, "BasicEdge2"));
      } catch (IteratorSettingException e) {
        fail(e.getMessage());
      }
      final RangeFactory rangeF = store.getKeyPackage().getRangeFactory();
      final Range r =
          rangeF.getRangeFromPair(
              new Pair<ElementSeed>((new EntitySeed("1")), new EntitySeed("4")),
              new SummariseGroupOverRanges());
      final Range r2 =
          rangeF.getRangeFromPair(
              new Pair<ElementSeed>((new EntitySeed("5")), new EntitySeed("5")),
              new SummariseGroupOverRanges());
      scanner.setRanges(Arrays.asList(r, r2));
      final Iterator<Map.Entry<Key, Value>> it = scanner.iterator();
      Map.Entry<Key, Value> entry = it.next();
      Element readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());

      Edge expectedEdge = new Edge("BasicEdge2");
      expectedEdge.setSource("4");
      expectedEdge.setDestination("8");
      expectedEdge.setDirected(true);
      expectedEdge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 5);
      expectedEdge.putProperty(AccumuloPropertyNames.COUNT, 3);

      assertEquals(expectedEdge, readEdge);
      assertEquals(5, readEdge.getProperty(AccumuloPropertyNames.COLUMN_QUALIFIER));
      assertEquals(3, readEdge.getProperty(AccumuloPropertyNames.COUNT));
      // Check we get the Result of the second provided range
      assertTrue(it.hasNext());
      entry = it.next();
      readEdge = elementConverter.getFullElement(entry.getKey(), entry.getValue());
      expectedEdge = new Edge("BasicEdge2");
      expectedEdge.setSource("5");
      expectedEdge.setDestination("9");
      expectedEdge.setDirected(true);
      expectedEdge.putProperty(AccumuloPropertyNames.COLUMN_QUALIFIER, 2);
      expectedEdge.putProperty(AccumuloPropertyNames.COUNT, 1);
      assertEquals(expectedEdge, readEdge);
      // Check no additional rows are found. (For a table of this size we shouldn't see this)
      if (it.hasNext()) {
        fail("Additional row found.");
      }
    } catch (AccumuloException | TableExistsException | TableNotFoundException e) {
      fail(this.getClass().getSimpleName() + " failed with exception: " + e);
    }
  }
コード例 #7
0
 /**
  * Create a copy of key with all fields except the ones specified cleared.
  *
  * @param key The key to copy
  * @param pk What fields to retain from the key
  * @return A new Key object pointing to new copies of fields specified by pk; other fields are
  *     empty/default.
  */
 public static Key keyCopy(Key key, PartialKey pk) {
   if (key == null || pk == null) return null;
   switch (pk) {
     case ROW:
       return new Key(
           key.getRowData().getBackingArray(),
           EMPTY_BYTES,
           EMPTY_BYTES,
           EMPTY_BYTES,
           Long.MAX_VALUE,
           false,
           true);
     case ROW_COLFAM:
       return new Key(
           key.getRowData().getBackingArray(),
           key.getColumnFamilyData().getBackingArray(),
           EMPTY_BYTES,
           EMPTY_BYTES,
           Long.MAX_VALUE,
           false,
           true);
     case ROW_COLFAM_COLQUAL:
       return new Key(
           key.getRowData().getBackingArray(),
           key.getColumnFamilyData().getBackingArray(),
           key.getColumnQualifierData().getBackingArray(),
           EMPTY_BYTES,
           Long.MAX_VALUE,
           false,
           true);
     case ROW_COLFAM_COLQUAL_COLVIS:
       return new Key(
           key.getRowData().getBackingArray(),
           key.getColumnFamilyData().getBackingArray(),
           key.getColumnQualifierData().getBackingArray(),
           key.getColumnVisibilityData().getBackingArray(),
           Long.MAX_VALUE,
           false,
           true);
     case ROW_COLFAM_COLQUAL_COLVIS_TIME:
       return new Key(
           key.getRowData().getBackingArray(),
           key.getColumnFamilyData().getBackingArray(),
           key.getColumnQualifierData().getBackingArray(),
           key.getColumnVisibilityData().getBackingArray(),
           key.getTimestamp(),
           false,
           true);
     case ROW_COLFAM_COLQUAL_COLVIS_TIME_DEL:
       return new Key(
           key.getRowData().getBackingArray(),
           key.getColumnFamilyData().getBackingArray(),
           key.getColumnQualifierData().getBackingArray(),
           key.getColumnVisibilityData().getBackingArray(),
           key.getTimestamp(),
           key.isDeleted(),
           true);
     default:
       throw new AssertionError("unknown pk: " + pk);
   }
 }
コード例 #8
0
  static int checkClone(String srcTableId, String tableId, Connector conn, BatchWriter bw)
      throws TableNotFoundException, MutationsRejectedException {
    TabletIterator srcIter =
        new TabletIterator(
            createCloneScanner(srcTableId, conn),
            new KeyExtent(new Text(srcTableId), null, null).toMetadataRange(),
            true,
            true);
    TabletIterator cloneIter =
        new TabletIterator(
            createCloneScanner(tableId, conn),
            new KeyExtent(new Text(tableId), null, null).toMetadataRange(),
            true,
            true);

    if (!cloneIter.hasNext() || !srcIter.hasNext())
      throw new RuntimeException(
          " table deleted during clone?  srcTableId = " + srcTableId + " tableId=" + tableId);

    int rewrites = 0;

    while (cloneIter.hasNext()) {
      Map<Key, Value> cloneTablet = cloneIter.next();
      Text cloneEndRow =
          new KeyExtent(cloneTablet.keySet().iterator().next().getRow(), (Text) null).getEndRow();
      HashSet<String> cloneFiles = new HashSet<String>();

      boolean cloneSuccessful = false;
      for (Entry<Key, Value> entry : cloneTablet.entrySet()) {
        if (entry.getKey().getColumnFamily().equals(ClonedColumnFamily.NAME)) {
          cloneSuccessful = true;
          break;
        }
      }

      if (!cloneSuccessful) getFiles(cloneFiles, cloneTablet, null);

      List<Map<Key, Value>> srcTablets = new ArrayList<Map<Key, Value>>();
      Map<Key, Value> srcTablet = srcIter.next();
      srcTablets.add(srcTablet);

      Text srcEndRow =
          new KeyExtent(srcTablet.keySet().iterator().next().getRow(), (Text) null).getEndRow();

      int cmp = compareEndRows(cloneEndRow, srcEndRow);
      if (cmp < 0)
        throw new TabletIterator.TabletDeletedException(
            "Tablets deleted from src during clone : " + cloneEndRow + " " + srcEndRow);

      HashSet<String> srcFiles = new HashSet<String>();
      if (!cloneSuccessful) getFiles(srcFiles, srcTablet, srcTableId);

      while (cmp > 0) {
        srcTablet = srcIter.next();
        srcTablets.add(srcTablet);
        srcEndRow =
            new KeyExtent(srcTablet.keySet().iterator().next().getRow(), (Text) null).getEndRow();
        cmp = compareEndRows(cloneEndRow, srcEndRow);
        if (cmp < 0)
          throw new TabletIterator.TabletDeletedException(
              "Tablets deleted from src during clone : " + cloneEndRow + " " + srcEndRow);

        if (!cloneSuccessful) getFiles(srcFiles, srcTablet, srcTableId);
      }

      if (cloneSuccessful) continue;

      if (!srcFiles.containsAll(cloneFiles)) {
        // delete existing cloned tablet entry
        Mutation m = new Mutation(cloneTablet.keySet().iterator().next().getRow());

        for (Entry<Key, Value> entry : cloneTablet.entrySet()) {
          Key k = entry.getKey();
          m.putDelete(k.getColumnFamily(), k.getColumnQualifier(), k.getTimestamp());
        }

        bw.addMutation(m);

        for (Map<Key, Value> st : srcTablets)
          bw.addMutation(createCloneMutation(srcTableId, tableId, st));

        rewrites++;
      } else {
        // write out marker that this tablet was successfully cloned
        Mutation m = new Mutation(cloneTablet.keySet().iterator().next().getRow());
        m.put(ClonedColumnFamily.NAME, new Text(""), new Value("OK".getBytes()));
        bw.addMutation(m);
      }
    }

    bw.flush();
    return rewrites;
  }