@Test
  public void testV1CodecV2Compat() throws Exception {

    long now = System.currentTimeMillis();

    // NOTE: set visibilityUpperBound to 0 as this is expected default for decoding older version
    // that doesn't store it
    TreeMap<Long, TransactionManager.InProgressTx> inProgress =
        Maps.newTreeMap(
            ImmutableSortedMap.of(
                16L, new TransactionManager.InProgressTx(0L, now + 1000),
                17L, new TransactionManager.InProgressTx(0L, now + 1000)));

    TransactionSnapshot snapshot =
        new TransactionSnapshot(
            now,
            15,
            18,
            Lists.newArrayList(5L, 7L),
            inProgress,
            ImmutableMap.<Long, Set<ChangeId>>of(
                17L,
                Sets.newHashSet(
                    new ChangeId(Bytes.toBytes("ch1")), new ChangeId(Bytes.toBytes("ch2")))),
            ImmutableMap.<Long, Set<ChangeId>>of(
                16L,
                Sets.newHashSet(
                    new ChangeId(Bytes.toBytes("ch2")), new ChangeId(Bytes.toBytes("ch3")))));

    Configuration configV1 = HBaseConfiguration.create();
    configV1.setStrings(
        TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, SnapshotCodecV1.class.getName());

    SnapshotCodecProvider codecV1 = new SnapshotCodecProvider(configV1);

    // encoding with codec of v1
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    try {
      codecV1.encode(out, snapshot);
    } finally {
      out.close();
    }

    // decoding
    Configuration configV1V2 = HBaseConfiguration.create();
    configV1V2.setStrings(
        TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES,
        SnapshotCodecV1.class.getName(),
        SnapshotCodecV2.class.getName());
    SnapshotCodecProvider codecV1V2 = new SnapshotCodecProvider(configV1V2);
    TransactionSnapshot decoded = codecV1V2.decode(new ByteArrayInputStream(out.toByteArray()));

    assertEquals(snapshot, decoded);
  }
Beispiel #2
0
  /**
   * Returns the top words associated with the specified word and the number of times the words have
   * appeared together.
   *
   * @param word the word of interest
   * @param limit the number of associations to return, at most
   * @return a map of the top associated words to their co-occurrence count
   */
  public Map<String, Long> readWordAssocs(String word, int limit) {

    // Retrieve all columns of the word’s row
    Row result = this.table.get(new Get(word));
    TopKCollector collector = new TopKCollector(limit);
    if (!result.isEmpty()) {

      // Iterate over all columns
      for (Map.Entry<byte[], byte[]> entry : result.getColumns().entrySet()) {
        collector.add(Bytes.toLong(entry.getValue()), Bytes.toString(entry.getKey()));
      }
    }
    return collector.getTopK();
  }
  @Test
  public void testCLFLogWithEscapedDoubleQuotes()
      throws UnsupportedTypeException, UnexpectedFormatException {
    CombinedLogRecordFormat format = new CombinedLogRecordFormat();
    FormatSpecification spec =
        new FormatSpecification(
            CombinedLogRecordFormat.class.getCanonicalName(),
            null,
            ImmutableMap.<String, String>of());
    format.initialize(spec);
    String data =
        "10.10.10.10 - - [01/Feb/2015:06:38:58 +0000] \"GET /plugins/servlet/buildStatusImage/CDAP-DUT "
            + "HTTP/1.1\" 301 257 \"http://cdap.io/\" \"\\\"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, "
            + "like Gecko) Chrome/31.0.1650.57 Safari/537.36 OPR/18.0.1284.49\\\"\"";
    StructuredRecord output = format.read(new StreamEvent(ByteBuffer.wrap(Bytes.toBytes(data))));

    Assert.assertEquals("10.10.10.10", output.get("remote_host"));
    Assert.assertNull(output.get("remote_login"));
    Assert.assertNull(output.get("auth_user"));
    Assert.assertEquals("01/Feb/2015:06:38:58 +0000", output.get("date"));
    Assert.assertEquals(
        "GET /plugins/servlet/buildStatusImage/CDAP-DUT HTTP/1.1", output.get("request"));
    Assert.assertEquals(301, output.get("status"));
    Assert.assertEquals(257, output.get("content_length"));
    Assert.assertEquals("http://cdap.io/", output.get("referrer"));
    Assert.assertEquals(
        "\\\"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) "
            + "Chrome/31.0.1650.57 Safari/537.36 OPR/18.0.1284.49\\\"",
        output.get("user_agent"));
  }
  @Test
  public void testCLFLog() throws UnsupportedTypeException, UnexpectedFormatException {
    CombinedLogRecordFormat format = new CombinedLogRecordFormat();
    FormatSpecification spec =
        new FormatSpecification(
            CombinedLogRecordFormat.class.getCanonicalName(),
            null,
            ImmutableMap.<String, String>of());
    format.initialize(spec);
    String data =
        "10.10.10.10 - - [01/Feb/2015:06:47:10 +0000] \"GET /browse/COOP-DBT-JOB1-238/artifact HTTP/1.1\""
            + " 301 256 \"-\" \"Mozilla/5.0 (compatible; AhrefsBot/5.0; +http://ahrefs.com/robot/)\"";
    StructuredRecord output = format.read(new StreamEvent(ByteBuffer.wrap(Bytes.toBytes(data))));

    Assert.assertEquals("10.10.10.10", output.get("remote_host"));
    Assert.assertNull(output.get("remote_login"));
    Assert.assertNull(output.get("auth_user"));
    Assert.assertEquals("01/Feb/2015:06:47:10 +0000", output.get("date"));
    Assert.assertEquals("GET /browse/COOP-DBT-JOB1-238/artifact HTTP/1.1", output.get("request"));
    Assert.assertEquals(301, output.get("status"));
    Assert.assertEquals(256, output.get("content_length"));
    Assert.assertNull(output.get("referrer"));
    Assert.assertEquals(
        "Mozilla/5.0 (compatible; AhrefsBot/5.0; +http://ahrefs.com/robot/)",
        output.get("user_agent"));
  }
Beispiel #5
0
  /**
   * Stores associations between the specified set of words. That is, for every word in the set, an
   * association will be stored for each of the other words in the set.
   *
   * @param words words to store associations between
   */
  public void writeWordAssocs(Set<String> words) {

    // For sets of less than 2 words, there are no associations
    int n = words.size();

    if (n < 2) {
      return;
    }

    // Every word will get (n-1) increments (one for each of the other words)
    long[] values = new long[n - 1];
    Arrays.fill(values, 1);

    // Convert all words to bytes
    byte[][] wordBytes = new byte[n][];
    int i = 0;
    for (String word : words) {
      wordBytes[i++] = Bytes.toBytes(word);
    }

    // Generate an increment for each word
    for (int j = 0; j < n; j++) {
      byte[] row = wordBytes[j];
      byte[][] columns = new byte[n - 1][];
      System.arraycopy(wordBytes, 0, columns, 0, j);
      System.arraycopy(wordBytes, j + 1, columns, j, n - j - 1);
      this.table.increment(row, columns, values);
    }
  }
Beispiel #6
0
  @Test
  public void useTransactionTest() throws Exception {
    // Performing admin operations to create dataset instance
    // keyValueTable is a system dataset module
    Id.DatasetInstance myTableInstance = Id.DatasetInstance.from(namespace, "myTable");
    dsFramework.addInstance("keyValueTable", myTableInstance, DatasetProperties.EMPTY);

    Assert.assertTrue(feedManager.createFeed(FEED1));
    try {
      Cancellable cancellable =
          notificationService.subscribe(
              FEED1,
              new NotificationHandler<String>() {
                private int received = 0;

                @Override
                public Type getNotificationType() {
                  return String.class;
                }

                @Override
                public void received(
                    final String notification, NotificationContext notificationContext) {
                  notificationContext.execute(
                      new TxRunnable() {
                        @Override
                        public void run(DatasetContext context) throws Exception {
                          KeyValueTable table = context.getDataset("myTable");
                          table.write("foo", String.format("%s-%d", notification, received++));
                        }
                      },
                      TxRetryPolicy.maxRetries(5));
                }
              });
      TimeUnit.SECONDS.sleep(2);

      try {
        notificationService.publish(FEED1, "foobar");
        // Waiting for the subscriber to receive that notification
        TimeUnit.SECONDS.sleep(2);

        KeyValueTable table =
            dsFramework.getDataset(myTableInstance, DatasetDefinition.NO_ARGUMENTS, null);
        Assert.assertNotNull(table);
        Transaction tx1 = txManager.startShort(100);
        table.startTx(tx1);
        Assert.assertEquals("foobar-0", Bytes.toString(table.read("foo")));
        Assert.assertTrue(table.commitTx());
        txManager.canCommit(tx1, table.getTxChanges());
        txManager.commit(tx1);
        table.postTxCommit();
      } finally {
        cancellable.cancel();
      }
    } finally {
      dsFramework.deleteInstance(myTableInstance);
      feedManager.deleteFeed(FEED1);
    }
  }
 @Test(expected = UnexpectedFormatException.class)
 public void testInvalid() throws UnsupportedTypeException, UnexpectedFormatException {
   CombinedLogRecordFormat format = new CombinedLogRecordFormat();
   FormatSpecification spec =
       new FormatSpecification(
           CombinedLogRecordFormat.class.getCanonicalName(),
           null,
           ImmutableMap.<String, String>of());
   format.initialize(spec);
   String data = "10.10.10.10[01/Feb/2015:06:47:10 +0000";
   StructuredRecord output = format.read(new StreamEvent(ByteBuffer.wrap(Bytes.toBytes(data))));
 }
Beispiel #8
0
  @Test
  public void testColumnFamily() throws Exception {
    DatasetProperties props =
        DatasetProperties.builder().add(Table.PROPERTY_COLUMN_FAMILY, "t").build();
    HBaseTableDefinition tableDefinition = new HBaseTableDefinition("foo");
    String tableName = "testcf";
    DatasetSpecification spec = tableDefinition.configure(tableName, props);

    DatasetAdmin admin =
        new HBaseTableAdmin(
            CONTEXT1,
            spec,
            testHBase.getConfiguration(),
            hBaseTableUtil,
            CConfiguration.create(),
            new LocalLocationFactory(tmpFolder.newFolder()));
    admin.create();
    final HBaseTable table =
        new HBaseTable(CONTEXT1, spec, cConf, testHBase.getConfiguration(), hBaseTableUtil);

    TransactionSystemClient txClient = new DetachedTxSystemClient();
    TransactionExecutor executor = new DefaultTransactionExecutor(txClient, table);
    executor.execute(
        new TransactionExecutor.Subroutine() {
          @Override
          public void apply() throws Exception {
            table.put(new Put("row", "column", "testValue"));
          }
        });

    final HBaseTable table2 =
        new HBaseTable(CONTEXT1, spec, cConf, testHBase.getConfiguration(), hBaseTableUtil);
    executor = new DefaultTransactionExecutor(txClient, table2);
    executor.execute(
        new TransactionExecutor.Subroutine() {
          @Override
          public void apply() throws Exception {
            Assert.assertEquals(
                "testValue", table2.get(new Get("row", "column")).getString("column"));
          }
        });

    // Verify the column family name
    HTableDescriptor htd =
        hBaseTableUtil.getHTableDescriptor(
            testHBase.getHBaseAdmin(), TableId.from(CONTEXT1.getNamespaceId(), tableName));
    HColumnDescriptor hcd = htd.getFamily(Bytes.toBytes("t"));
    Assert.assertNotNull(hcd);
    Assert.assertEquals("t", hcd.getNameAsString());
  }
  @Test
  public void testCLFLogWithNull() throws UnsupportedTypeException, UnexpectedFormatException {
    CombinedLogRecordFormat format = new CombinedLogRecordFormat();
    FormatSpecification spec =
        new FormatSpecification(
            CombinedLogRecordFormat.class.getCanonicalName(),
            null,
            ImmutableMap.<String, String>of());
    format.initialize(spec);
    String data = "10.10.10.10 - - [01/Feb/2015:09:58:24 +0000] \"-\" 408 - \"-\" \"-\"";
    StructuredRecord output = format.read(new StreamEvent(ByteBuffer.wrap(Bytes.toBytes(data))));

    Assert.assertEquals("10.10.10.10", output.get("remote_host"));
    Assert.assertNull(output.get("remote_login"));
    Assert.assertNull(output.get("auth_user"));
    Assert.assertEquals("01/Feb/2015:09:58:24 +0000", output.get("date"));
    Assert.assertNull(output.get("request"));
    Assert.assertEquals(408, output.get("status"));
    Assert.assertNull(output.get("content_length"));
    Assert.assertNull(output.get("referrer"));
    Assert.assertNull(output.get("user_agent"));
  }
Beispiel #10
0
  @Test
  public void testPreSplit() throws Exception {
    byte[][] splits = new byte[][] {Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c")};
    DatasetProperties props =
        DatasetProperties.builder().add("hbase.splits", new Gson().toJson(splits)).build();
    String presplittedTable = "presplitted";
    getTableAdmin(CONTEXT1, presplittedTable, props).create();

    HBaseAdmin hBaseAdmin = testHBase.getHBaseAdmin();
    try {
      List<HRegionInfo> regions =
          hBaseTableUtil.getTableRegions(
              hBaseAdmin, TableId.from(NAMESPACE1.getId(), presplittedTable));
      // note: first region starts at very first row key, so we have one extra to the splits count
      Assert.assertEquals(4, regions.size());
      Assert.assertArrayEquals(Bytes.toBytes("a"), regions.get(1).getStartKey());
      Assert.assertArrayEquals(Bytes.toBytes("b"), regions.get(2).getStartKey());
      Assert.assertArrayEquals(Bytes.toBytes("c"), regions.get(3).getStartKey());
    } finally {
      hBaseAdmin.close();
    }
  }
 public void put(int key, String value) throws Exception {
   table.put(Bytes.toBytes(key), COL, Bytes.toBytes(value));
 }
Beispiel #12
0
  @Test
  public void testTTL() throws Exception {
    // for the purpose of this test it is fine not to configure ttl when creating table: we want to
    // see if it
    // applies on reading
    int ttl = 1000;
    String ttlTable = "ttl";
    String noTtlTable = "nottl";
    DatasetProperties props =
        DatasetProperties.builder().add(Table.PROPERTY_TTL, String.valueOf(ttl)).build();
    getTableAdmin(CONTEXT1, ttlTable, props).create();
    DatasetSpecification ttlTableSpec =
        DatasetSpecification.builder(ttlTable, HBaseTable.class.getName())
            .properties(props.getProperties())
            .build();
    HBaseTable table =
        new HBaseTable(CONTEXT1, ttlTableSpec, cConf, testHBase.getConfiguration(), hBaseTableUtil);

    DetachedTxSystemClient txSystemClient = new DetachedTxSystemClient();
    Transaction tx = txSystemClient.startShort();
    table.startTx(tx);
    table.put(b("row1"), b("col1"), b("val1"));
    table.commitTx();

    TimeUnit.SECONDS.sleep(2);

    tx = txSystemClient.startShort();
    table.startTx(tx);
    table.put(b("row2"), b("col2"), b("val2"));
    table.commitTx();

    // now, we should not see first as it should have expired, but see the last one
    tx = txSystemClient.startShort();
    table.startTx(tx);
    byte[] val = table.get(b("row1"), b("col1"));
    if (val != null) {
      LOG.info("Unexpected value " + Bytes.toStringBinary(val));
    }
    Assert.assertNull(val);
    Assert.assertArrayEquals(b("val2"), table.get(b("row2"), b("col2")));

    // test a table with no TTL
    DatasetProperties props2 =
        DatasetProperties.builder().add(Table.PROPERTY_TTL, String.valueOf(Tables.NO_TTL)).build();
    getTableAdmin(CONTEXT1, noTtlTable, props2).create();
    DatasetSpecification noTtlTableSpec =
        DatasetSpecification.builder(noTtlTable, HBaseTable.class.getName())
            .properties(props2.getProperties())
            .build();
    HBaseTable table2 =
        new HBaseTable(
            CONTEXT1, noTtlTableSpec, cConf, testHBase.getConfiguration(), hBaseTableUtil);

    tx = txSystemClient.startShort();
    table2.startTx(tx);
    table2.put(b("row1"), b("col1"), b("val1"));
    table2.commitTx();

    TimeUnit.SECONDS.sleep(2);

    tx = txSystemClient.startShort();
    table2.startTx(tx);
    table2.put(b("row2"), b("col2"), b("val2"));
    table2.commitTx();

    // if ttl is -1 (unlimited), it should see both
    tx = txSystemClient.startShort();
    table2.startTx(tx);
    Assert.assertArrayEquals(b("val1"), table2.get(b("row1"), b("col1")));
    Assert.assertArrayEquals(b("val2"), table2.get(b("row2"), b("col2")));
  }
Beispiel #13
0
 private static byte[] b(String s) {
   return Bytes.toBytes(s);
 }
Beispiel #14
0
  @Test
  public void testEnableIncrements() throws Exception {
    // setup a table with increments disabled and with it enabled
    String disableTableName = "incr-disable";
    String enabledTableName = "incr-enable";
    TableId disabledTableId = TableId.from(NAMESPACE1.getId(), disableTableName);
    TableId enabledTableId = TableId.from(NAMESPACE1.getId(), enabledTableName);
    HBaseTableAdmin disabledAdmin =
        getTableAdmin(CONTEXT1, disableTableName, DatasetProperties.EMPTY);
    disabledAdmin.create();
    HBaseAdmin admin = testHBase.getHBaseAdmin();

    DatasetProperties props =
        DatasetProperties.builder().add(Table.PROPERTY_READLESS_INCREMENT, "true").build();
    HBaseTableAdmin enabledAdmin = getTableAdmin(CONTEXT1, enabledTableName, props);
    enabledAdmin.create();

    try {

      try {
        HTableDescriptor htd = hBaseTableUtil.getHTableDescriptor(admin, disabledTableId);
        List<String> cps = htd.getCoprocessors();
        assertFalse(cps.contains(IncrementHandler.class.getName()));

        htd = hBaseTableUtil.getHTableDescriptor(admin, enabledTableId);
        cps = htd.getCoprocessors();
        assertTrue(cps.contains(IncrementHandler.class.getName()));
      } finally {
        admin.close();
      }

      BufferingTable table = getTable(CONTEXT1, enabledTableName, ConflictDetection.COLUMN);
      byte[] row = Bytes.toBytes("row1");
      byte[] col = Bytes.toBytes("col1");
      DetachedTxSystemClient txSystemClient = new DetachedTxSystemClient();
      Transaction tx = txSystemClient.startShort();
      table.startTx(tx);
      table.increment(row, col, 10);
      table.commitTx();
      // verify that value was written as a delta value
      final byte[] expectedValue =
          Bytes.add(IncrementHandlerState.DELTA_MAGIC_PREFIX, Bytes.toBytes(10L));
      final AtomicBoolean foundValue = new AtomicBoolean();
      byte[] enabledTableNameBytes =
          hBaseTableUtil.getHTableDescriptor(admin, enabledTableId).getName();
      testHBase.forEachRegion(
          enabledTableNameBytes,
          new Function<HRegion, Object>() {
            @Nullable
            @Override
            public Object apply(@Nullable HRegion hRegion) {
              Scan scan = new Scan();
              try {
                RegionScanner scanner = hRegion.getScanner(scan);
                List<Cell> results = Lists.newArrayList();
                boolean hasMore;
                do {
                  hasMore = scanner.next(results);
                  for (Cell cell : results) {
                    if (CellUtil.matchingValue(cell, expectedValue)) {
                      foundValue.set(true);
                    }
                  }
                } while (hasMore);
              } catch (IOException ioe) {
                fail("IOException scanning region: " + ioe.getMessage());
              }
              return null;
            }
          });
      assertTrue(
          "Should have seen the expected encoded delta value in the "
              + enabledTableName
              + " table region",
          foundValue.get());
    } finally {
      disabledAdmin.drop();
      enabledAdmin.drop();
    }
  }
 public void put(String key, KeyValue.Value value) throws Exception {
   table.put(Bytes.toBytes(key), COL, Bytes.toBytes(GSON.toJson(value)));
 }
 public KeyValue.Value get(String key) throws Exception {
   return GSON.fromJson(
       Bytes.toString(table.get(Bytes.toBytes(key), COL)), KeyValue.Value.class);
 }
 public String get(int key) throws Exception {
   return Bytes.toString(table.get(Bytes.toBytes(key), COL));
 }
 public void reduce(Text key, Iterable<Text> values, Context context)
     throws IOException, InterruptedException {
   for (Text value : values) {
     context.write(Bytes.toBytes(key.toString()), Bytes.toBytes(value.toString()));
   }
 }
 @Override
 public KeyValue makeRecord(byte[] key, Row row) {
   return new KeyValue(
       Bytes.toInt(key),
       GSON.fromJson(Bytes.toString(row.get(KeyValueTable.COL)), String.class));
 }