Esempio n. 1
0
    @Override
    public void write(Cell cell) throws IOException {
      if (encryptor == null) {
        super.write(cell);
        return;
      }

      byte[] iv = nextIv();
      encryptor.setIv(iv);
      encryptor.reset();

      // TODO: Check if this is a cell for an encrypted CF. If not, we can
      // write a 0 here to signal an unwrapped cell and just dump the KV bytes
      // afterward

      StreamUtils.writeRawVInt32(out, iv.length);
      out.write(iv);

      // TODO: Add support for WAL compression

      ByteArrayOutputStream baos = new ByteArrayOutputStream();
      OutputStream cout = encryptor.createEncryptionStream(baos);

      int tlen = cell.getTagsLength();
      // Write the KeyValue infrastructure as VInts.
      StreamUtils.writeRawVInt32(cout, KeyValueUtil.keyLength(cell));
      StreamUtils.writeRawVInt32(cout, cell.getValueLength());
      // To support tags
      StreamUtils.writeRawVInt32(cout, tlen);

      // Write row, qualifier, and family
      StreamUtils.writeRawVInt32(cout, cell.getRowLength());
      cout.write(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
      StreamUtils.writeRawVInt32(cout, cell.getFamilyLength());
      cout.write(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength());
      StreamUtils.writeRawVInt32(cout, cell.getQualifierLength());
      cout.write(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength());
      // Write the rest ie. ts, type, value and tags parts
      StreamUtils.writeLong(cout, cell.getTimestamp());
      cout.write(cell.getTypeByte());
      cout.write(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
      if (tlen > 0) {
        cout.write(cell.getTagsArray(), cell.getTagsOffset(), tlen);
      }
      cout.close();

      StreamUtils.writeRawVInt32(out, baos.size());
      baos.writeTo(out);

      // Increment IV given the final payload length
      incrementIv(baos.size());
    }
  public static void main(String[] args) throws IOException {
    Configuration conf = HBaseConfiguration.create();

    HBaseHelper helper = HBaseHelper.getHelper(conf);
    helper.dropTable("testtable");
    helper.createTable("testtable", "colfam1", "colfam2");
    System.out.println("Adding rows to table...");
    helper.fillTable("testtable", 1, 10, 10, "colfam1", "colfam2");

    Connection connection = ConnectionFactory.createConnection(conf);
    Table table = connection.getTable(TableName.valueOf("testtable"));
    // vv SingleColumnValueFilterExample
    SingleColumnValueFilter filter =
        new SingleColumnValueFilter(
            Bytes.toBytes("colfam1"),
            Bytes.toBytes("col-5"),
            CompareFilter.CompareOp.NOT_EQUAL,
            new SubstringComparator("val-5"));
    filter.setFilterIfMissing(true);

    Scan scan = new Scan();
    scan.setFilter(filter);
    ResultScanner scanner = table.getScanner(scan);
    // ^^ SingleColumnValueFilterExample
    System.out.println("Results of scan:");
    // vv SingleColumnValueFilterExample
    for (Result result : scanner) {
      for (Cell cell : result.rawCells()) {
        System.out.println(
            "Cell: "
                + cell
                + ", Value: "
                + Bytes.toString(
                    cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
      }
    }
    scanner.close();

    Get get = new Get(Bytes.toBytes("row-6"));
    get.setFilter(filter);
    Result result = table.get(get);
    System.out.println("Result of get: ");
    for (Cell cell : result.rawCells()) {
      System.out.println(
          "Cell: "
              + cell
              + ", Value: "
              + Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
    }
    // ^^ SingleColumnValueFilterExample
  }
Esempio n. 3
0
 @Override
 public void write(Cell cell) throws IOException {
   checkFlushed();
   // Row
   write(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
   // Column family
   write(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength());
   // Qualifier
   write(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength());
   // Version
   this.out.write(Bytes.toBytes(cell.getTimestamp()));
   // Type
   this.out.write(cell.getTypeByte());
   // Value
   write(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());
   // Tags
   write(cell.getTagsArray(), cell.getTagsOffset(), cell.getTagsLength());
   // MvccVersion
   this.out.write(Bytes.toBytes(cell.getMvccVersion()));
 }
Esempio n. 4
0
  @Test(timeOut = 10_000)
  public void testCorrectMapingOfCellsToShadowCells() throws IOException {
    // Create the required data
    final byte[] validShadowCellQualifier =
        com.google.common.primitives.Bytes.concat(qualifier, SHADOW_CELL_SUFFIX);

    final byte[] qualifier2 = Bytes.toBytes("test-qual2");
    final byte[] validShadowCellQualifier2 =
        com.google.common.primitives.Bytes.concat(qualifier2, SHADOW_CELL_SUFFIX);

    final byte[] qualifier3 = Bytes.toBytes("test-qual3");

    Cell cell1 =
        new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value")); // Default type is Put
    Cell dupCell1 =
        new KeyValue(row, family, qualifier, 1, Bytes.toBytes("value")); // Default type is Put
    Cell dupCell1WithAnotherValue =
        new KeyValue(row, family, qualifier, 1, Bytes.toBytes("other-value"));
    Cell delCell1 = new KeyValue(row, family, qualifier, 1, Type.Delete, Bytes.toBytes("value"));
    Cell shadowCell1 =
        new KeyValue(row, family, validShadowCellQualifier, 1, Bytes.toBytes("sc-value"));

    Cell cell2 = new KeyValue(row, family, qualifier2, 1, Bytes.toBytes("value2"));
    Cell shadowCell2 =
        new KeyValue(row, family, validShadowCellQualifier2, 1, Bytes.toBytes("sc-value2"));

    Cell cell3 = new KeyValue(row, family, qualifier3, 1, Bytes.toBytes("value3"));

    // Check a list of cells with duplicate values
    List<Cell> badListWithDups = new ArrayList<>();
    badListWithDups.add(cell1);
    badListWithDups.add(dupCell1WithAnotherValue);

    // Check dup shadow cell with same MVCC is ignored
    SortedMap<Cell, Optional<Cell>> cellsToShadowCells =
        CellUtils.mapCellsToShadowCells(badListWithDups);
    assertEquals(cellsToShadowCells.size(), 1, "There should be only 1 key-value maps");
    assertTrue(cellsToShadowCells.containsKey(cell1));
    KeyValue firstKey = (KeyValue) cellsToShadowCells.firstKey();
    KeyValue lastKey = (KeyValue) cellsToShadowCells.lastKey();
    assertTrue(firstKey.equals(lastKey));
    assertTrue(
        0
            == Bytes.compareTo(
                firstKey.getValueArray(),
                firstKey.getValueOffset(),
                firstKey.getValueLength(),
                cell1.getValueArray(),
                cell1.getValueOffset(),
                cell1.getValueLength()),
        "Should be equal");

    // Modify dup shadow cell to have a greater MVCC and check that is replaced
    HBaseShims.setKeyValueSequenceId((KeyValue) dupCell1WithAnotherValue, 1);
    cellsToShadowCells = CellUtils.mapCellsToShadowCells(badListWithDups);
    assertEquals(cellsToShadowCells.size(), 1, "There should be only 1 key-value maps");
    assertTrue(cellsToShadowCells.containsKey(dupCell1WithAnotherValue));
    firstKey = (KeyValue) cellsToShadowCells.firstKey();
    lastKey = (KeyValue) cellsToShadowCells.lastKey();
    assertTrue(firstKey.equals(lastKey));
    assertTrue(
        0
            == Bytes.compareTo(
                firstKey.getValueArray(),
                firstKey.getValueOffset(),
                firstKey.getValueLength(),
                dupCell1WithAnotherValue.getValueArray(),
                dupCell1WithAnotherValue.getValueOffset(),
                dupCell1WithAnotherValue.getValueLength()),
        "Should be equal");
    // Check a list of cells with duplicate values
    List<Cell> cellListWithDups = new ArrayList<>();
    cellListWithDups.add(cell1);
    cellListWithDups.add(shadowCell1);
    cellListWithDups.add(dupCell1); // Dup cell
    cellListWithDups.add(delCell1); // Another Dup cell but with different type
    cellListWithDups.add(cell2);
    cellListWithDups.add(cell3);
    cellListWithDups.add(shadowCell2);

    cellsToShadowCells = CellUtils.mapCellsToShadowCells(cellListWithDups);
    assertEquals(cellsToShadowCells.size(), 3, "There should be only 3 key-value maps");
    assertTrue(cellsToShadowCells.get(cell1).get().equals(shadowCell1));
    assertTrue(cellsToShadowCells.get(dupCell1).get().equals(shadowCell1));
    assertFalse(
        cellsToShadowCells.containsKey(delCell1)); // TODO This is strange and needs to be solved.
    // The current algo avoids to put the delete cell
    // as key after the put cell with same value was added
    assertTrue(cellsToShadowCells.get(cell2).get().equals(shadowCell2));
    assertTrue(cellsToShadowCells.get(cell3).equals(Optional.absent()));
  }