@Test
  public void testRegionObserverFlushTimeStacking() throws Exception {
    byte[] ROW = Bytes.toBytes("testRow");
    byte[] TABLE = Bytes.toBytes(getClass().getName());
    byte[] A = Bytes.toBytes("A");
    byte[][] FAMILIES = new byte[][] {A};

    Configuration conf = HBaseConfiguration.create();
    HRegion region = initHRegion(TABLE, getClass().getName(), conf, FAMILIES);
    RegionCoprocessorHost h = region.getCoprocessorHost();
    h.load(NoDataFromFlush.class, Coprocessor.PRIORITY_HIGHEST, conf);
    h.load(EmptyRegionObsever.class, Coprocessor.PRIORITY_USER, conf);

    // put a row and flush it to disk
    Put put = new Put(ROW);
    put.add(A, A, A);
    region.put(put);
    region.flushcache();
    Get get = new Get(ROW);
    Result r = region.get(get);
    assertNull(
        "Got an unexpected number of rows - no data should be returned with the NoDataFromScan coprocessor. Found: "
            + r,
        r.listCells());
  }
Пример #2
0
  /**
   * 遍历多行
   *
   * @param tableName 表名
   * @param start_rowkey 开始行键
   * @param stop_rowkey 结束行键
   * @return 行列表
   */
  public ArrayList<HbaseRow> scanRows(String tableName, String start_rowkey, String stop_rowkey) {
    ResultScanner rowstmp = null;
    ArrayList<HbaseRow> rows = null;

    try {
      Scan scan = new Scan();
      scan.setStartRow(Bytes.toBytes(start_rowkey));
      scan.setStopRow(Bytes.toBytes(stop_rowkey));
      HTable table = new HTable(conf, Bytes.toBytes(tableName));
      rowstmp = table.getScanner(scan);
      rows = new ArrayList<>();

      for (Result rowtmp : rowstmp) {
        HbaseRow row = new HbaseRow();
        row.rowkey = Bytes.toString(rowtmp.getRow());
        for (Cell cell : rowtmp.listCells()) {
          HbaseColumn col = new HbaseColumn(cell);
          row.cols.add(col);
        }
        rows.add(row);
      }
    } catch (Exception e) {
      logger.error("scanRows failed", e);
    } finally {
      rowstmp.close();
    }
    return rows;
  }
  @Override
  public void emitTuples() {
    try {
      Scan scan = nextScan();
      if (scan == null) return;

      ResultScanner resultScanner = store.getTable().getScanner(scan);

      while (true) {
        Result result = resultScanner.next();
        if (result == null) break;

        String readRow = Bytes.toString(result.getRow());
        if (readRow.equals(lastReadRow)) continue;

        Object instance = pojoType.newInstance();
        rowSetter.set(instance, readRow);

        List<Cell> cells = result.listCells();

        for (Cell cell : cells) {
          String columnName = Bytes.toString(CellUtil.cloneQualifier(cell));
          byte[] value = CellUtil.cloneValue(cell);
          fieldValueGenerator.setColumnValue(instance, columnName, value, valueConverter);
        }

        outputPort.emit(instance);
        lastReadRow = readRow;
      }

    } catch (Exception e) {
      throw new RuntimeException(e.getMessage());
    }
  }
Пример #4
0
  /** Test GroupingTableMapper class */
  @Test
  public void testGroupingTableMapper() throws Exception {

    GroupingTableMapper mapper = new GroupingTableMapper();
    Configuration configuration = new Configuration();
    configuration.set(GroupingTableMapper.GROUP_COLUMNS, "family1:clm family2:clm");
    mapper.setConf(configuration);

    Result result = mock(Result.class);
    @SuppressWarnings("unchecked")
    Mapper<ImmutableBytesWritable, Result, ImmutableBytesWritable, Result>.Context context =
        mock(Mapper.Context.class);
    context.write(any(ImmutableBytesWritable.class), any(Result.class));
    List<Cell> keyValue = new ArrayList<Cell>();
    byte[] row = {};
    keyValue.add(
        new KeyValue(row, Bytes.toBytes("family2"), Bytes.toBytes("clm"), Bytes.toBytes("value1")));
    keyValue.add(
        new KeyValue(row, Bytes.toBytes("family1"), Bytes.toBytes("clm"), Bytes.toBytes("value2")));
    when(result.listCells()).thenReturn(keyValue);
    mapper.map(null, result, context);
    // template data
    byte[][] data = {Bytes.toBytes("value1"), Bytes.toBytes("value2")};
    ImmutableBytesWritable ibw = mapper.createGroupKey(data);
    verify(context).write(ibw, result);
  }
 /**
  * Looks at every value of the mapreduce output and verifies that indeed the values have been
  * reversed.
  *
  * @param table Table to scan.
  * @throws IOException
  * @throws NullPointerException if we failed to find a cell value
  */
 private void verifyAttempt(final Table table) throws IOException, NullPointerException {
   Scan scan = new Scan();
   scan.addFamily(INPUT_FAMILY);
   scan.addFamily(OUTPUT_FAMILY);
   ResultScanner scanner = table.getScanner(scan);
   try {
     Iterator<Result> itr = scanner.iterator();
     assertTrue(itr.hasNext());
     while (itr.hasNext()) {
       Result r = itr.next();
       if (LOG.isDebugEnabled()) {
         if (r.size() > 2) {
           throw new IOException("Too many results, expected 2 got " + r.size());
         }
       }
       byte[] firstValue = null;
       byte[] secondValue = null;
       int count = 0;
       for (Cell kv : r.listCells()) {
         if (count == 0) {
           firstValue = CellUtil.cloneValue(kv);
         } else if (count == 1) {
           secondValue = CellUtil.cloneValue(kv);
         } else if (count == 2) {
           break;
         }
         count++;
       }
       String first = "";
       if (firstValue == null) {
         throw new NullPointerException(Bytes.toString(r.getRow()) + ": first value is null");
       }
       first = Bytes.toString(firstValue);
       String second = "";
       if (secondValue == null) {
         throw new NullPointerException(Bytes.toString(r.getRow()) + ": second value is null");
       }
       byte[] secondReversed = new byte[secondValue.length];
       for (int i = 0, j = secondValue.length - 1; j >= 0; j--, i++) {
         secondReversed[i] = secondValue[j];
       }
       second = Bytes.toString(secondReversed);
       if (first.compareTo(second) != 0) {
         if (LOG.isDebugEnabled()) {
           LOG.debug(
               "second key is not the reverse of first. row="
                   + Bytes.toStringBinary(r.getRow())
                   + ", first value="
                   + first
                   + ", second value="
                   + second);
         }
         fail();
       }
     }
   } finally {
     scanner.close();
   }
 }
  /**
   * Confirm ImportTsv via data in online table.
   *
   * @param dataAvailable
   */
  private static void validateTable(
      Configuration conf,
      TableName tableName,
      String family,
      int valueMultiplier,
      boolean dataAvailable)
      throws IOException {

    LOG.debug("Validating table.");
    Connection connection = ConnectionFactory.createConnection(conf);
    Table table = connection.getTable(tableName);
    boolean verified = false;
    long pause = conf.getLong("hbase.client.pause", 5 * 1000);
    int numRetries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
    for (int i = 0; i < numRetries; i++) {
      try {
        Scan scan = new Scan();
        // Scan entire family.
        scan.addFamily(Bytes.toBytes(family));
        if (dataAvailable) {
          ResultScanner resScanner = table.getScanner(scan);
          for (Result res : resScanner) {
            LOG.debug("Getting results " + res.size());
            assertTrue(res.size() == 2);
            List<Cell> kvs = res.listCells();
            assertTrue(CellUtil.matchingRow(kvs.get(0), Bytes.toBytes("KEY")));
            assertTrue(CellUtil.matchingRow(kvs.get(1), Bytes.toBytes("KEY")));
            assertTrue(
                CellUtil.matchingValue(kvs.get(0), Bytes.toBytes("VALUE" + valueMultiplier)));
            assertTrue(
                CellUtil.matchingValue(kvs.get(1), Bytes.toBytes("VALUE" + 2 * valueMultiplier)));
            // Only one result set is expected, so let it loop.
            verified = true;
          }
        } else {
          ResultScanner resScanner = table.getScanner(scan);
          Result[] next = resScanner.next(2);
          assertEquals(0, next.length);
          verified = true;
        }

        break;
      } catch (NullPointerException e) {
        // If here, a cell was empty. Presume its because updates came in
        // after the scanner had been opened. Wait a while and retry.
      }
      try {
        Thread.sleep(pause);
      } catch (InterruptedException e) {
        // continue
      }
    }
    table.close();
    connection.close();
    assertTrue(verified);
  }
Пример #7
0
 /** Return the number of rows in the given table. */
 public static int countMobRows(final Table table) throws IOException {
   Scan scan = new Scan();
   ResultScanner results = table.getScanner(scan);
   int count = 0;
   for (Result res : results) {
     count++;
     List<Cell> cells = res.listCells();
     for (Cell cell : cells) {
       // Verify the value
       Assert.assertTrue(CellUtil.cloneValue(cell).length > 0);
     }
   }
   results.close();
   return count;
 }
Пример #8
0
  /**
   * 查询列
   *
   * @param tableName 表名
   * @param rowKey 行键
   * @param familyName 列族名
   * @param columnName 列名
   * @return 列
   */
  public HbaseColumn getColumn(
      String tableName, String rowKey, String familyName, String columnName) {
    HbaseColumn col = null;

    try {
      HTable table = new HTable(conf, Bytes.toBytes(tableName));
      Get get = new Get(Bytes.toBytes(rowKey));
      get.addColumn(Bytes.toBytes(familyName), Bytes.toBytes(columnName)); // 获取指定列族和列修饰符对应的列
      Result rowtmp = table.get(get);

      col = new HbaseColumn();
      for (Cell cell : rowtmp.listCells()) {
        col.CopyFromCell(cell);
      }
    } catch (Exception e) {
      logger.error("getColumn failed", e);
    }
    return col;
  }
Пример #9
0
  /**
   * 查询单行
   *
   * @param tableName 表名
   * @param rowKey 行键名
   * @return 行
   */
  public HbaseRow getRow(String tableName, String rowKey) {
    HbaseRow row = null;

    try {
      Get get = new Get(Bytes.toBytes(rowKey));
      HTable table = new HTable(conf, Bytes.toBytes(tableName)); // 获取表
      Result rowtmp = table.get(get);
      row = new HbaseRow();

      row.rowkey = Bytes.toString(rowtmp.getRow());
      for (Cell cell : rowtmp.listCells()) {
        HbaseColumn col = new HbaseColumn(cell);
        row.cols.add(col);
      }
    } catch (Exception e) {
      logger.error("getRow failed", e);
    }
    return row;
  }
Пример #10
0
 private String resultToString(Result result) {
   StringBuilder sb = new StringBuilder();
   sb.append("cells=");
   if (result.isEmpty()) {
     sb.append("NONE");
     return sb.toString();
   }
   sb.append("{");
   boolean moreThanOne = false;
   for (Cell cell : result.listCells()) {
     if (moreThanOne) {
       sb.append(", ");
     } else {
       moreThanOne = true;
     }
     sb.append(CellUtil.toString(cell, true));
   }
   sb.append("}");
   return sb.toString();
 }
Пример #11
0
 /**
  * Extract columns values from the current record. This method returns null if any of the columns
  * are not found.
  *
  * <p>Override this method if you want to deal with nulls differently.
  *
  * @param r The current values.
  * @return Array of byte values.
  */
 protected byte[][] extractKeyValues(Result r) {
   byte[][] keyVals = null;
   ArrayList<byte[]> foundList = new ArrayList<byte[]>();
   int numCols = columns.length;
   if (numCols > 0) {
     for (Cell value : r.listCells()) {
       byte[] column =
           KeyValue.makeColumn(CellUtil.cloneFamily(value), CellUtil.cloneQualifier(value));
       for (int i = 0; i < numCols; i++) {
         if (Bytes.equals(column, columns[i])) {
           foundList.add(CellUtil.cloneValue(value));
           break;
         }
       }
     }
     if (foundList.size() == numCols) {
       keyVals = foundList.toArray(new byte[numCols][]);
     }
   }
   return keyVals;
 }
Пример #12
0
  /**
   * 查询列的多版本
   *
   * @param tableName 表名
   * @param rowKey 行键名
   * @param familyName 列族名
   * @param columnName 列名
   * @param num 版本数
   * @return 列列表
   */
  public ArrayList<HbaseColumn> getColumnVersions(
      String tableName, String rowKey, String familyName, String columnName, int num) {
    ArrayList<HbaseColumn> cols = null;

    try {
      HTable table = new HTable(conf, Bytes.toBytes(tableName));
      Get get = new Get(Bytes.toBytes(rowKey));
      get.addColumn(Bytes.toBytes(familyName), Bytes.toBytes(columnName));
      get.setMaxVersions(num);
      Result rowtmp = table.get(get);

      cols = new ArrayList<>();
      for (Cell cell : rowtmp.listCells()) {
        HbaseColumn col = new HbaseColumn(cell);
        cols.add(col);
      }
    } catch (Exception e) {
      logger.error("getColumnVersions failed", e);
    }
    return cols;
  }
  /**
   * Unfortunately, the easiest way to test this is to spin up a mini-cluster since we want to do
   * the usual compaction mechanism on the region, rather than going through the backdoor to the
   * region
   */
  @Test
  public void testRegionObserverCompactionTimeStacking() throws Exception {
    // setup a mini cluster so we can do a real compaction on a region
    Configuration conf = UTIL.getConfiguration();
    conf.setClass(HConstants.REGION_IMPL, CompactionCompletionNotifyingRegion.class, HRegion.class);
    conf.setInt("hbase.hstore.compaction.min", 2);
    UTIL.startMiniCluster();
    String tableName = "testRegionObserverCompactionTimeStacking";
    byte[] ROW = Bytes.toBytes("testRow");
    byte[] A = Bytes.toBytes("A");
    HTableDescriptor desc = new HTableDescriptor(TableName.valueOf(tableName));
    desc.addFamily(new HColumnDescriptor(A));
    desc.addCoprocessor(EmptyRegionObsever.class.getName(), null, Coprocessor.PRIORITY_USER, null);
    desc.addCoprocessor(
        NoDataFromCompaction.class.getName(), null, Coprocessor.PRIORITY_HIGHEST, null);

    Admin admin = UTIL.getHBaseAdmin();
    admin.createTable(desc);

    Table table = UTIL.getConnection().getTable(desc.getTableName());

    // put a row and flush it to disk
    Put put = new Put(ROW);
    put.add(A, A, A);
    table.put(put);

    HRegionServer rs = UTIL.getRSForFirstRegionInTable(desc.getTableName());
    List<HRegion> regions = rs.getOnlineRegions(desc.getTableName());
    assertEquals("More than 1 region serving test table with 1 row", 1, regions.size());
    HRegion region = regions.get(0);
    admin.flushRegion(region.getRegionName());
    CountDownLatch latch =
        ((CompactionCompletionNotifyingRegion) region).getCompactionStateChangeLatch();

    // put another row and flush that too
    put = new Put(Bytes.toBytes("anotherrow"));
    put.add(A, A, A);
    table.put(put);
    admin.flushRegion(region.getRegionName());

    // run a compaction, which normally would should get rid of the data
    // wait for the compaction checker to complete
    latch.await();
    // check both rows to ensure that they aren't there
    Get get = new Get(ROW);
    Result r = table.get(get);
    assertNull(
        "Got an unexpected number of rows - no data should be returned with the NoDataFromScan coprocessor. Found: "
            + r,
        r.listCells());

    get = new Get(Bytes.toBytes("anotherrow"));
    r = table.get(get);
    assertNull(
        "Got an unexpected number of rows - no data should be returned with the NoDataFromScan coprocessor Found: "
            + r,
        r.listCells());

    table.close();
    UTIL.shutdownMiniCluster();
  }