Пример #1
0
  public void saveRegisterModelList(List<RegisterModel> list) {

    try {
      List<Put> putList = new ArrayList<Put>();
      for (RegisterModel o : list) {
        String rowKey =
            System.currentTimeMillis()
                + ":"
                + o.getProvinceId()
                + ":"
                + o.getCityId()
                + ":"
                + UUID.randomUUID().toString();
        Put put = new Put(Bytes.toBytes(rowKey));
        put.add(
            Bytes.toBytes("info"), Bytes.toBytes(o.getTid()), Bytes.toBytes(JSON.toJSONString(o)));
        putList.add(put);
      }

      table = HBaseDataSource.getTable("TH_TERMINAL_REGISTER");
      if (table != null) {
        table.put(putList);
      } else {
        table.put(putList);
      }
    } catch (IOException e) {
      log.error("存储异常:" + e.getMessage(), e);
    }
  }
 private void loadData(Table table) throws IOException {
   for (int i = 0; i < ROWSIZE; i++) {
     Put put = new Put(ROWS[i]);
     put.addColumn(FAMILYNAME, QUALIFIER, Bytes.toBytes(i));
     table.put(put);
   }
 }
Пример #3
0
    @Override
    protected void updateMeta(final byte[] oldRegion1, final byte[] oldRegion2, HRegion newRegion)
        throws IOException {
      byte[][] regionsToDelete = {oldRegion1, oldRegion2};
      for (int r = 0; r < regionsToDelete.length; r++) {
        Delete delete = new Delete(regionsToDelete[r]);
        delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
        delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
        delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.STARTCODE_QUALIFIER);
        delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER);
        delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER);
        root.delete(delete, null, true);

        if (LOG.isDebugEnabled()) {
          LOG.debug("updated columns in row: " + Bytes.toStringBinary(regionsToDelete[r]));
        }
      }
      HRegionInfo newInfo = newRegion.getRegionInfo();
      newInfo.setOffline(true);
      Put put = new Put(newRegion.getRegionName());
      put.add(
          HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, Writables.getBytes(newInfo));
      root.put(put);
      if (LOG.isDebugEnabled()) {
        LOG.debug("updated columns in row: " + Bytes.toStringBinary(newRegion.getRegionName()));
      }
    }
 @Test
 public void testCheckAndDeleteHooks() throws IOException {
   TableName tableName =
       TableName.valueOf(TEST_TABLE.getNameAsString() + ".testCheckAndDeleteHooks");
   HTable table = util.createTable(tableName, new byte[][] {A, B, C});
   try {
     Put p = new Put(Bytes.toBytes(0));
     p.add(A, A, A);
     table.put(p);
     table.flushCommits();
     Delete d = new Delete(Bytes.toBytes(0));
     table.delete(d);
     verifyMethodResult(
         SimpleRegionObserver.class,
         new String[] {
           "hadPreCheckAndDelete", "hadPreCheckAndDeleteAfterRowLock", "hadPostCheckAndDelete"
         },
         tableName,
         new Boolean[] {false, false, false});
     table.checkAndDelete(Bytes.toBytes(0), A, A, A, d);
     verifyMethodResult(
         SimpleRegionObserver.class,
         new String[] {
           "hadPreCheckAndDelete", "hadPreCheckAndDeleteAfterRowLock", "hadPostCheckAndDelete"
         },
         tableName,
         new Boolean[] {true, true, true});
   } finally {
     util.deleteTable(tableName);
     table.close();
   }
 }
Пример #5
0
  /**
   * Writes TOTAL_ROWS number of distinct rows in to the table. Few rows have two columns, Few have
   * one.
   *
   * @param table
   * @throws IOException
   */
  private static void writeRows(Table table) throws IOException {
    final byte[] family = Bytes.toBytes(COL_FAM);
    final byte[] value = Bytes.toBytes("abcd");
    final byte[] col1 = Bytes.toBytes(COL1);
    final byte[] col2 = Bytes.toBytes(COL2);
    final byte[] col3 = Bytes.toBytes(COMPOSITE_COLUMN);
    ArrayList<Put> rowsUpdate = new ArrayList<Put>();
    // write few rows with two columns
    int i = 0;
    for (; i < TOTAL_ROWS - ROWS_WITH_ONE_COL; i++) {
      byte[] row = Bytes.toBytes("row" + i);
      Put put = new Put(row);
      put.add(family, col1, value);
      put.add(family, col2, value);
      put.add(family, col3, value);
      rowsUpdate.add(put);
    }

    // write few rows with only one column
    for (; i < TOTAL_ROWS; i++) {
      byte[] row = Bytes.toBytes("row" + i);
      Put put = new Put(row);
      put.add(family, col2, value);
      rowsUpdate.add(put);
    }
    table.put(rowsUpdate);
  }
Пример #6
0
 public static Put getSinglePut(
     String rowId, String columnFamily, String columnQualifier, String value) {
   Put put = new Put(Bytes.toBytes(rowId));
   put.addColumn(
       Bytes.toBytes(columnFamily), Bytes.toBytes(columnQualifier), Bytes.toBytes(value));
   return put;
 }
      @Override
      public void process(long now, HRegion region, List<Mutation> mutations, WALEdit walEdit)
          throws IOException {
        // Scan current counter
        List<Cell> kvs = new ArrayList<Cell>();
        Scan scan = new Scan(row, row);
        scan.addColumn(FAM, COUNTER);
        doScan(region, scan, kvs);
        counter = kvs.size() == 0 ? 0 : Bytes.toInt(CellUtil.cloneValue(kvs.iterator().next()));

        // Assert counter value
        assertEquals(expectedCounter, counter);

        // Increment counter and send it to both memstore and wal edit
        counter += 1;
        expectedCounter += 1;

        Put p = new Put(row);
        KeyValue kv = new KeyValue(row, FAM, COUNTER, now, Bytes.toBytes(counter));
        p.add(kv);
        mutations.add(p);
        walEdit.add(kv);

        // We can also inject some meta data to the walEdit
        KeyValue metaKv =
            new KeyValue(
                row,
                WALEdit.METAFAMILY,
                Bytes.toBytes("I just increment counter"),
                Bytes.toBytes(counter));
        walEdit.add(metaKv);
      }
  @Test
  public void testMultipleCellsInOneFamilyAreConverted() {
    byte[] row = dataHelper.randomData("rk-");
    byte[] family = dataHelper.randomData("f1");
    byte[] qualifier1 = dataHelper.randomData("qual1");
    byte[] qualifier2 = dataHelper.randomData("qual2");
    byte[] value1 = dataHelper.randomData("v1");
    byte[] value2 = dataHelper.randomData("v2");
    long timestamp1 = 1L;
    long timestamp2 = 2L;

    Put hbasePut = new Put(row);
    hbasePut.addColumn(family, qualifier1, timestamp1, value1);
    hbasePut.addColumn(family, qualifier2, timestamp2, value2);

    MutateRowRequest.Builder rowMutationBuilder = adapter.adapt(hbasePut);
    Assert.assertArrayEquals(row, rowMutationBuilder.getRowKey().toByteArray());

    Assert.assertEquals(2, rowMutationBuilder.getMutationsCount());
    Mutation mutation = rowMutationBuilder.getMutations(0);

    Assert.assertEquals(MutationCase.SET_CELL, mutation.getMutationCase());
    SetCell setCell = mutation.getSetCell();
    Assert.assertArrayEquals(family, setCell.getFamilyNameBytes().toByteArray());
    Assert.assertArrayEquals(qualifier1, setCell.getColumnQualifier().toByteArray());
    Assert.assertEquals(TimeUnit.MILLISECONDS.toMicros(timestamp1), setCell.getTimestampMicros());
    Assert.assertArrayEquals(value1, setCell.getValue().toByteArray());

    Mutation mod2 = rowMutationBuilder.getMutations(1);
    SetCell setCell2 = mod2.getSetCell();
    Assert.assertArrayEquals(family, setCell2.getFamilyNameBytes().toByteArray());
    Assert.assertArrayEquals(qualifier2, setCell2.getColumnQualifier().toByteArray());
    Assert.assertEquals(TimeUnit.MILLISECONDS.toMicros(timestamp2), setCell2.getTimestampMicros());
    Assert.assertArrayEquals(value2, setCell2.getValue().toByteArray());
  }
Пример #9
0
 /*
  * Add to each of the regions in .META. a value.  Key is the startrow of the
  * region (except its 'aaa' for first region).  Actual value is the row name.
  * @param expected
  * @return
  * @throws IOException
  */
 private static int addToEachStartKey(final int expected) throws IOException {
   HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
   HTable meta = new HTable(TEST_UTIL.getConfiguration(), HConstants.META_TABLE_NAME);
   int rows = 0;
   Scan scan = new Scan();
   scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
   ResultScanner s = meta.getScanner(scan);
   for (Result r = null; (r = s.next()) != null; ) {
     byte[] b = r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
     if (b == null || b.length <= 0) break;
     HRegionInfo hri = Writables.getHRegionInfo(b);
     // If start key, add 'aaa'.
     byte[] row = getStartKey(hri);
     Put p = new Put(row);
     p.setWriteToWAL(false);
     p.add(getTestFamily(), getTestQualifier(), row);
     t.put(p);
     rows++;
   }
   s.close();
   Assert.assertEquals(expected, rows);
   t.close();
   meta.close();
   return rows;
 }
  public void insertRowToDb(String[][] tableData, String colFamily) {

    String rowKey = tableData[0][0];

    Put resourcePut = new Put(Bytes.toBytes(rowKey));

    String[] userDataList = new String[tableData.length];
    String[] userColList = new String[tableData.length];

    String valueString = null;
    byte[] putValue = null;
    String action = null;

    for (int i = 0; i < tableData.length; i++) {
      userDataList[i] = tableData[i][0];
      userColList[i] = tableData[i][1];
    }

    for (int i = 1; i < userDataList.length; i++) {
      valueString = userDataList[i];
      if (HBaseManagerViewTable.coloumnTypeList.containsKey(userColList[i])) {
        action = HBaseManagerViewTable.coloumnTypeList.get(userColList[i]);
        putValue = HBaseTableManager.getConvertedValue(valueString, action, userColList[i]);
      } else {
        putValue = Bytes.toBytes(userDataList[i]);
      }
      resourcePut.add(Bytes.toBytes(colFamily), Bytes.toBytes(userColList[i]), putValue);
    }

    _insert(resourcePut);
  }
  @Test
  public void testSingleCellIsConverted() {
    byte[] row = dataHelper.randomData("rk-");
    byte[] family = dataHelper.randomData("f");
    byte[] qualifier = dataHelper.randomData("qual");
    byte[] value = dataHelper.randomData("v1");
    long timestamp = 2L;

    Put hbasePut = new Put(row);
    hbasePut.addColumn(family, qualifier, timestamp, value);

    MutateRowRequest.Builder rowMutationBuilder = adapter.adapt(hbasePut);
    Assert.assertArrayEquals(row, rowMutationBuilder.getRowKey().toByteArray());

    Assert.assertEquals(1, rowMutationBuilder.getMutationsCount());
    Mutation mutation = rowMutationBuilder.getMutations(0);

    Assert.assertEquals(MutationCase.SET_CELL, mutation.getMutationCase());
    SetCell setCell = mutation.getSetCell();

    Assert.assertArrayEquals(family, setCell.getFamilyNameBytes().toByteArray());
    Assert.assertArrayEquals(qualifier, setCell.getColumnQualifier().toByteArray());
    Assert.assertEquals(TimeUnit.MILLISECONDS.toMicros(timestamp), setCell.getTimestampMicros());
    Assert.assertArrayEquals(value, setCell.getValue().toByteArray());
  }
Пример #12
0
 public void fillTable(
     String table,
     int startRow,
     int endRow,
     int numCols,
     int pad,
     boolean setTimestamp,
     boolean random,
     String... colfams)
     throws IOException {
   HTable tbl = new HTable(conf, table);
   Random rnd = new Random();
   for (int row = startRow; row <= endRow; row++) {
     for (int col = 0; col < numCols; col++) {
       Put put = new Put(Bytes.toBytes("row-" + padNum(row, pad)));
       for (String cf : colfams) {
         String colName = "col-" + (padNum(col, pad));
         String val =
             "val-"
                 + (random
                     ? Integer.toString(rnd.nextInt(numCols))
                     : padNum(row, pad) + "." + padNum(col, pad));
         if (setTimestamp) {
           put.add(Bytes.toBytes(cf), Bytes.toBytes(colName), col, Bytes.toBytes(val));
         } else {
           put.add(Bytes.toBytes(cf), Bytes.toBytes(colName), Bytes.toBytes(val));
         }
       }
       tbl.put(put);
     }
   }
   tbl.close();
 }
Пример #13
0
 private void putRecord(MetaRecord rec, long ts) throws Exception {
   Put put = new Put(Bytes.toBytes(rec.getId()));
   put.add(FAMILY, OBJECT_TYPE, ts, Bytes.toBytes(rec.getObjectType()));
   put.add(FAMILY, SQL, ts, Bytes.toBytes(rec.getSQL()));
   // System.out.println("addRecord id: " + rec.getId() + ", sql=" + rec.getSQL());
   table.put(put);
 }
Пример #14
0
  public void saveAuthModelList(List<AuthModel> list) {

    try {
      List<Put> putList = new ArrayList<Put>();
      for (AuthModel o : list) {
        String rowKey =
            System.currentTimeMillis()
                + ":"
                + o.getResult()
                + ":"
                + o.getCommaddr()
                + ":"
                + UUID.randomUUID().toString();
        Put put = new Put(Bytes.toBytes(rowKey));
        put.add(
            Bytes.toBytes("info"), Bytes.toBytes(o.getAkey()), Bytes.toBytes(JSON.toJSONString(o)));
        putList.add(put);
      }
      table = HBaseDataSource.getTable("TH_VEHICLE_CHECKED");
      if (table != null) {
        table.put(putList);
      } else {

        table.put(putList);
      }
    } catch (IOException e) {
      log.error("存储异常:" + e.getMessage(), e);
    }
  }
Пример #15
0
  @Test
  public void insert_rowkey_prefix_date() throws IOException {

    System.out.println(errorTable);
    errorTable.setAutoFlushTo(false);
    List<Put> puts = new ArrayList<Put>();
    long t1 = System.currentTimeMillis();
    for (int i = 0; i < 10000000; i++) {
      String uuid = UUID.randomUUID().toString().replaceAll("-", "").substring(0, 8);
      Put put = new Put(Bytes.toBytes("20150705" + "_" + uuid));
      put.add(
          fBytes,
          Bytes.toBytes("stacktrace"),
          Bytes.toBytes("java.io.IOException:file not found" + UUID.randomUUID().toString()));
      //            puts.add(put);
      errorTable.put(put);
      if (i % 10000 == 0) {
        errorTable.flushCommits();
      }
    }
    errorTable.flushCommits();
    long t2 = System.currentTimeMillis();
    System.out.println("count=" + puts.size() + ",t2-t1=" + (t2 - t1));
    //        errorTable.close();
  }
Пример #16
0
  /*
   * 为表添加数据(适合知道有多少列族的固定表)
   *
   * @rowKey rowKey
   *
   * @tableName 表名
   *
   * @column1 第一个列族列表
   *
   * @value1 第一个列的值的列表
   *
   * @column2 第二个列族列表
   *
   * @value2 第二个列的值的列表
   */
  public static void addData(
      String rowKey,
      String tableName,
      String[] column1,
      String[] value1,
      String[] column2,
      String[] value2)
      throws IOException {

    Put put = new Put(Bytes.toBytes(rowKey)); // 设置rowkey
    HTableInterface htable = conn.getTable(tableName);
    HColumnDescriptor[] columnFamilies =
        htable
            .getTableDescriptor() // 获取所有的列族
            .getColumnFamilies();

    for (int i = 0; i < columnFamilies.length; i++) {
      String familyName = columnFamilies[i].getNameAsString(); // 获取列族名
      if (familyName.equals("article")) { // article列族put数据
        for (int j = 0; j < column1.length; j++) {
          put.add(Bytes.toBytes(familyName), Bytes.toBytes(column1[j]), Bytes.toBytes(value1[j]));
        }
      }
      if (familyName.equals("author")) { // author列族put数据
        for (int j = 0; j < column2.length; j++) {
          put.add(Bytes.toBytes(familyName), Bytes.toBytes(column2[j]), Bytes.toBytes(value2[j]));
        }
      }
    }
    htable.put(put);
    // System.out.println("add data Success!");
  }
Пример #17
0
 public void initData() {
   IndexHTable hTable = null;
   try {
     hTable = new IndexHTable(conf, tableName);
     hTable.setAutoFlushTo(false);
     List<Put> puts = new ArrayList<Put>();
     byte[] familyBytes = columnFamily.getBytes();
     byte[] qualifier = columnName.getBytes();
     for (int i = 0; i < 100; i++) {
       Put put = new Put(Bytes.toBytes("row" + i));
       put.add(familyBytes, qualifier, ("value" + i).getBytes());
       puts.add(put);
     }
     hTable.put(puts);
     hTable.flushCommits();
   } catch (Exception e) {
     LOG.error(e.getMessage(), e);
   } finally {
     try {
       hTable.close();
     } catch (IOException e) {
       LOG.error("htable close error");
     }
   }
 }
Пример #18
0
  @Test(timeout = 30000)
  public void testCreateDeleteTable() throws IOException {
    // Create table then get the single region for our new table.
    HTableDescriptor hdt = HTU.createTableDescriptor("testCreateDeleteTable");
    hdt.setRegionReplication(NB_SERVERS);
    hdt.addCoprocessor(SlowMeCopro.class.getName());
    Table table = HTU.createTable(hdt, new byte[][] {f}, HTU.getConfiguration());

    Put p = new Put(row);
    p.add(f, row, row);
    table.put(p);

    Get g = new Get(row);
    Result r = table.get(g);
    Assert.assertFalse(r.isStale());

    try {
      // But if we ask for stale we will get it
      SlowMeCopro.cdl.set(new CountDownLatch(1));
      g = new Get(row);
      g.setConsistency(Consistency.TIMELINE);
      r = table.get(g);
      Assert.assertTrue(r.isStale());
      SlowMeCopro.cdl.get().countDown();
    } finally {
      SlowMeCopro.cdl.get().countDown();
      SlowMeCopro.sleepTime.set(0);
    }

    HTU.getHBaseAdmin().disableTable(hdt.getTableName());
    HTU.deleteTable(hdt.getTableName());
  }
    public Put call(String v) throws Exception {
      String[] cells = v.split(",");
      Put put = new Put(Bytes.toBytes(cells[0]));

      put.add(Bytes.toBytes(cells[1]), Bytes.toBytes(cells[2]), Bytes.toBytes(cells[3]));
      return put;
    }
Пример #20
0
  private Put constructRow(String[] oneLine) {
    String rowKey = oneLine[0].trim();
    LOG.info("About to add row: " + rowKey);
    LOG.info("Number of lines : " + oneLine.length);
    LOG.info("First column value: " + oneLine[1]);

    Put put = new Put(Bytes.toBytes(rowKey));

    for (int i = 1; i < oneLine.length; i++) {
      String subLine[] = oneLine[i].split(":");
      LOG.info("column name: " + subLine[0]);
      LOG.info("column value: " + subLine[1]);

      if (subLine[0].trim().equals("city")) {
        LOG.info("About to add: " + subLine[1]);
        put.add(HBASE_CF, COL_CITY, Bytes.toBytes(subLine[1].trim()));
      }
      if (subLine[0].trim().equals("state")) {
        LOG.info("About to add: " + subLine[1]);
        put.add(HBASE_CF, COL_STATE, Bytes.toBytes(subLine[1].trim()));
      }
      if (subLine[0].trim().equals("country")) {
        LOG.info("About to add: " + subLine[1]);
        put.add(HBASE_CF, COL_COUNTRY, Bytes.toBytes(subLine[1].trim()));
      }
    }

    return put;
  }
  @Test
  public void testRowMutation() throws IOException {
    TableName tableName = TableName.valueOf(TEST_TABLE.getNameAsString() + ".testRowMutation");
    HTable table = util.createTable(tableName, new byte[][] {A, B, C});
    try {
      verifyMethodResult(
          SimpleRegionObserver.class,
          new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut", "hadDeleted"},
          tableName,
          new Boolean[] {false, false, false, false, false});
      Put put = new Put(ROW);
      put.add(A, A, A);
      put.add(B, B, B);
      put.add(C, C, C);

      Delete delete = new Delete(ROW);
      delete.deleteColumn(A, A);
      delete.deleteColumn(B, B);
      delete.deleteColumn(C, C);

      RowMutations arm = new RowMutations(ROW);
      arm.add(put);
      arm.add(delete);
      table.mutateRow(arm);

      verifyMethodResult(
          SimpleRegionObserver.class,
          new String[] {"hadPreGet", "hadPostGet", "hadPrePut", "hadPostPut", "hadDeleted"},
          tableName,
          new Boolean[] {false, false, true, true, true});
    } finally {
      util.deleteTable(tableName);
      table.close();
    }
  }
Пример #22
0
  @TimeDepend
  @Test
  public void testScan_ts_same() throws Exception {

    recreateTable();

    Date ts = parse("2000-01-01", "yyyy-MM-dd");

    Put put = new Put(rowKey_ForTest);

    put.add(ColumnFamilyNameBytes, QName1, ts.getTime(), Bytes.toBytes("a"));

    table.put(put);

    Set<String> resultRowKeys = new HashSet<String>();
    Scan scan = new Scan(rowKey_ForTest, rowKey_ForTest);
    scan.setTimeRange(ts.getTime(), ts.getTime());

    ResultScanner resultScanner = table.getScanner(scan);
    for (Result result = resultScanner.next(); result != null; result = resultScanner.next()) {
      resultRowKeys.add(Bytes.toString(result.getRow()));
    }

    close(resultScanner);

    Assert.assertTrue(resultRowKeys.size() == 0);

    recreateTable();
  }
  @Test
  public void testPreWALRestoreSkip() throws Exception {
    LOG.info(TestRegionObserverInterface.class.getName() + ".testPreWALRestoreSkip");
    TableName tableName = TableName.valueOf(SimpleRegionObserver.TABLE_SKIPPED);
    HTable table = util.createTable(tableName, new byte[][] {A, B, C});

    JVMClusterUtil.RegionServerThread rs1 = cluster.startRegionServer();
    ServerName sn2 = rs1.getRegionServer().getServerName();
    String regEN = table.getRegionLocations().firstEntry().getKey().getEncodedName();

    util.getHBaseAdmin().move(regEN.getBytes(), sn2.getServerName().getBytes());
    while (!sn2.equals(table.getRegionLocations().firstEntry().getValue())) {
      Thread.sleep(100);
    }

    Put put = new Put(ROW);
    put.add(A, A, A);
    put.add(B, B, B);
    put.add(C, C, C);
    table.put(put);
    table.flushCommits();

    cluster.killRegionServer(rs1.getRegionServer().getServerName());
    Threads.sleep(20000); // just to be sure that the kill has fully started.
    util.waitUntilAllRegionsAssigned(tableName);

    verifyMethodResult(
        SimpleRegionObserver.class,
        new String[] {"getCtPreWALRestore", "getCtPostWALRestore"},
        tableName,
        new Integer[] {0, 0});

    util.deleteTable(tableName);
    table.close();
  }
Пример #24
0
 private void doPuts(HRegion region) throws IOException {
   LoadTestKVGenerator dataGenerator = new LoadTestKVGenerator(MIN_VALUE_SIZE, MAX_VALUE_SIZE);
   for (int i = 0; i < NUM_ROWS; ++i) {
     byte[] key = LoadTestKVGenerator.md5PrefixedKey(i).getBytes();
     for (int j = 0; j < NUM_COLS_PER_ROW; ++j) {
       Put put = new Put(key);
       byte[] col = Bytes.toBytes(String.valueOf(j));
       byte[] value = dataGenerator.generateRandomSizeValue(key, col);
       if (includeTags) {
         Tag[] tag = new Tag[1];
         tag[0] = new Tag((byte) 1, "Visibility");
         KeyValue kv = new KeyValue(key, CF_BYTES, col, HConstants.LATEST_TIMESTAMP, value, tag);
         put.add(kv);
       } else {
         put.add(CF_BYTES, col, value);
       }
       if (VERBOSE) {
         KeyValue kvPut = new KeyValue(key, CF_BYTES, col, value);
         System.err.println(Strings.padFront(i + "", ' ', 4) + " " + kvPut);
       }
       region.put(put);
     }
     if (i % NUM_ROWS_PER_FLUSH == 0) {
       region.flushcache();
     }
   }
 }
  /**
   * Creates a HBase {@link Put} from a Storm {@link Tuple}
   *
   * @param tuple The {@link Tuple}
   * @return {@link Put}
   */
  public Put getPutFromTuple(final Tuple tuple) {
    byte[] rowKey = Bytes.toBytes(tuple.getStringByField(tupleRowKeyField));

    long ts = 0;
    if (!tupleTimestampField.equals("")) {
      ts = tuple.getLongByField(tupleTimestampField);
    }

    Put p = new Put(rowKey);
    p.setWriteToWAL(writeToWAL);

    if (columnFamilies.size() > 0) {
      for (String cf : columnFamilies.keySet()) {
        byte[] cfBytes = Bytes.toBytes(cf);
        for (String cq : columnFamilies.get(cf)) {
          byte[] cqBytes = Bytes.toBytes(cq);
          byte[] val = Bytes.toBytes(tuple.getStringByField(cq));

          if (ts > 0) {
            p.add(cfBytes, cqBytes, ts, val);
          } else {
            p.add(cfBytes, cqBytes, val);
          }
        }
      }
    }

    return p;
  }
Пример #26
0
 /*
  * (non-Javadoc)
  *
  * @see com.hazelcast.core.MapStore#store(java.lang.Object,
  * java.lang.Object)
  */
 @Override
 public void store(String key, String value) {
   HTableInterface table = null;
   try {
     table = pool.getTable(tableName);
     try {
       byte[] rowId = prefixDate ? IdUtil.bucketizeId(key) : Bytes.toBytes(key);
       Put p = new Put(rowId);
       if (outputFormatType == StoreFormatType.SMILE) {
         p.add(family, qualifier, jsonSmileConverter.convertToSmile(value));
       } else {
         p.add(family, qualifier, Bytes.toBytes(value));
       }
       table.put(p);
     } catch (NumberFormatException nfe) {
       LOG.error("Encountered bad key: " + key, nfe);
     }
   } catch (IOException e) {
     LOG.error("Error during put", e);
   } finally {
     if (table != null) {
       pool.putTable(table);
     }
   }
 }
Пример #27
0
    @Override
    protected void updateMeta(final byte[] oldRegion1, final byte[] oldRegion2, HRegion newRegion)
        throws IOException {
      byte[][] regionsToDelete = {oldRegion1, oldRegion2};
      for (int r = 0; r < regionsToDelete.length; r++) {
        if (Bytes.equals(regionsToDelete[r], latestRegion.getRegionName())) {
          latestRegion = null;
        }
        Delete delete = new Delete(regionsToDelete[r]);
        table.delete(delete);
        if (LOG.isDebugEnabled()) {
          LOG.debug("updated columns in row: " + Bytes.toStringBinary(regionsToDelete[r]));
        }
      }
      newRegion.getRegionInfo().setOffline(true);

      Put put = new Put(newRegion.getRegionName());
      put.add(
          HConstants.CATALOG_FAMILY,
          HConstants.REGIONINFO_QUALIFIER,
          Writables.getBytes(newRegion.getRegionInfo()));
      table.put(put);

      if (LOG.isDebugEnabled()) {
        LOG.debug("updated columns in row: " + Bytes.toStringBinary(newRegion.getRegionName()));
      }
    }
Пример #28
0
  /*
   * (non-Javadoc)
   *
   * @see com.hazelcast.core.MapStore#storeAll(java.util.Map)
   */
  @Override
  public void storeAll(Map<String, String> pairs) {
    HTable table = null;
    try {
      List<Put> puts = new ArrayList<Put>(pairs.size());
      for (Map.Entry<String, String> pair : pairs.entrySet()) {
        try {
          byte[] rowId =
              prefixDate ? IdUtil.bucketizeId(pair.getKey()) : Bytes.toBytes(pair.getKey());
          Put p = new Put(rowId);
          if (outputFormatType == StoreFormatType.SMILE) {
            p.add(family, qualifier, jsonSmileConverter.convertToSmile(pair.getValue()));
          } else {
            p.add(family, qualifier, Bytes.toBytes(pair.getValue()));
          }
          puts.add(p);
        } catch (NumberFormatException nfe) {
          LOG.error("Encountered bad key: " + pair.getKey(), nfe);
        }
      }

      table = (HTable) pool.getTable(tableName);
      table.setAutoFlush(false);
      table.put(puts);
      table.flushCommits();
    } catch (IOException e) {
      LOG.error("Error during puts", e);
    } finally {
      if (table != null) {
        pool.putTable(table);
      }
    }
  }
Пример #29
0
  @Override
  public Event intercept(Event event) {
    // TODO Auto-generated method stub
    Map<String, String> headers = event.getHeaders();
    String Filename = headers.get("file");
    String fileType = getFileType(new String(event.getBody()));
    Configuration conf = HBaseConfiguration.create();
    HTable table = null;
    try {
      table = new HTable(conf, "fs");
    } catch (IOException e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }
    Put put = new Put(Bytes.toBytes(fileType + "_" + Filename));
    put.add(Bytes.toBytes("fn"), Bytes.toBytes("ST"), Bytes.toBytes("PICKED"));

    try {
      table.put(put);
    } catch (RetriesExhaustedWithDetailsException e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    } catch (InterruptedIOException e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }
    try {

      table.close();
    } catch (IOException e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }
    return event;
  }
  private void putAndWait(byte[] row, byte[] fam, HTable source, HTable... targets)
      throws Exception {
    Put put = new Put(row);
    put.add(fam, row, row);
    source.put(put);

    Get get = new Get(row);
    for (int i = 0; i < NB_RETRIES; i++) {
      if (i == NB_RETRIES - 1) {
        fail("Waited too much time for put replication");
      }
      boolean replicatedToAll = true;
      for (HTable target : targets) {
        Result res = target.get(get);
        if (res.size() == 0) {
          LOG.info("Row not available");
          replicatedToAll = false;
          break;
        } else {
          assertArrayEquals(res.value(), row);
        }
      }
      if (replicatedToAll) {
        break;
      } else {
        Thread.sleep(SLEEP_TIME);
      }
    }
  }