Example #1
0
  public String[] getObjectIDs(String objectType, String... tags) throws IOException {
    List<String> ret = new ArrayList<String>();

    FilterList list = new FilterList(FilterList.Operator.MUST_PASS_ALL);
    SingleColumnValueFilter filter1 =
        new SingleColumnValueFilter(
            "tags".getBytes(), "OBJECTTYPE".getBytes(), CompareOp.EQUAL, Bytes.toBytes(objectType));
    list.addFilter(filter1);

    for (String tag : tags) {
      SingleColumnValueFilter filter2 =
          new SingleColumnValueFilter(
              "tags".getBytes(), tag.toUpperCase().getBytes(), CompareOp.EQUAL, Bytes.toBytes(1));
      filter2.setFilterIfMissing(true);
      list.addFilter(filter2);
    }
    Scan s = new Scan();
    s.setFilter(list);
    s.setMaxVersions(1);
    ResultScanner scanner = htable.getScanner(s);
    try {
      for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {

        String localObjectType =
            new String(rr.getValue("tags".getBytes(), "OBJECTTYPE".getBytes()));
        String localObjectId = new String(rr.getValue("tags".getBytes(), "OBJECTID".getBytes()));
        ret.add(localObjectId);
      }
    } finally {
      scanner.close();
    }

    return ret.toArray(new String[] {});
  }
Example #2
0
  @TimeDepend
  @Test
  public void testScan_ts_same() throws Exception {

    recreateTable();

    Date ts = parse("2000-01-01", "yyyy-MM-dd");

    Put put = new Put(rowKey_ForTest);

    put.add(ColumnFamilyNameBytes, QName1, ts.getTime(), Bytes.toBytes("a"));

    table.put(put);

    Set<String> resultRowKeys = new HashSet<String>();
    Scan scan = new Scan(rowKey_ForTest, rowKey_ForTest);
    scan.setTimeRange(ts.getTime(), ts.getTime());

    ResultScanner resultScanner = table.getScanner(scan);
    for (Result result = resultScanner.next(); result != null; result = resultScanner.next()) {
      resultRowKeys.add(Bytes.toString(result.getRow()));
    }

    close(resultScanner);

    Assert.assertTrue(resultRowKeys.size() == 0);

    recreateTable();
  }
  @Override
  public void emitTuples() {
    try {
      Scan scan = nextScan();
      if (scan == null) return;

      ResultScanner resultScanner = store.getTable().getScanner(scan);

      while (true) {
        Result result = resultScanner.next();
        if (result == null) break;

        String readRow = Bytes.toString(result.getRow());
        if (readRow.equals(lastReadRow)) continue;

        Object instance = pojoType.newInstance();
        rowSetter.set(instance, readRow);

        List<Cell> cells = result.listCells();

        for (Cell cell : cells) {
          String columnName = Bytes.toString(CellUtil.cloneQualifier(cell));
          byte[] value = CellUtil.cloneValue(cell);
          fieldValueGenerator.setColumnValue(instance, columnName, value, valueConverter);
        }

        outputPort.emit(instance);
        lastReadRow = readRow;
      }

    } catch (Exception e) {
      throw new RuntimeException(e.getMessage());
    }
  }
Example #4
0
 /* (non-Javadoc)
  * @see com.hazelcast.core.MapLoader#loadAllKeys()
  */
 @Override
 public Set<String> loadAllKeys() {
   Set<String> keySet = null;
   if (allowLoadAll) {
     keySet = new HashSet<String>();
     HTableInterface hti = null;
     try {
       hti = pool.getTable(tableName);
       Scan s = new Scan();
       s.addColumn(family, qualifier);
       ResultScanner rs = hti.getScanner(s);
       Result r = null;
       while ((r = rs.next()) != null) {
         String k = new String(r.getRow());
         keySet.add(k);
       }
     } catch (IOException e) {
       LOG.error("IOException while loading all keys", e);
     } finally {
       if (hti != null) {
         pool.putTable(hti);
       }
     }
   }
   return keySet;
 }
Example #5
0
 public boolean fetchReadAllFieldLine(List<Line> lines, LineSender sender) throws IOException {
   if (null == this.rs) {
     throw new IllegalStateException("HBase Client try to fetch data failed .");
   }
   for (Result result = rs.next(); result != null; result = rs.next()) {
     Get get = new Get(result.getRow());
     for (int i = 0; i < this.families.length; i++) {
       get.addColumn(this.families[i].getBytes(), this.columns[i].getBytes());
     }
     gets.add(get);
     if (gets.size() > this.BUFFER_LINE) {
       Result[] getResults = this.htable.get(gets);
       for (Result resu : getResults) {
         if (null != resu) {
           Line line = sender.createLine();
           for (int i = 0; i < this.families.length; i++) {
             byte[] value = resu.getValue(this.families[i].getBytes(), this.columns[i].getBytes());
             if (null == value) {
               line.addField(null);
             } else {
               line.addField(new String(value, encode));
             }
           }
           line.addField(new String(resu.getRow(), encode));
         }
       }
       return true;
     }
   }
   return false;
 }
 /*
  * Add to each of the regions in .META. a value.  Key is the startrow of the
  * region (except its 'aaa' for first region).  Actual value is the row name.
  * @param expected
  * @return
  * @throws IOException
  */
 private static int addToEachStartKey(final int expected) throws IOException {
   HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
   HTable meta = new HTable(TEST_UTIL.getConfiguration(), HConstants.META_TABLE_NAME);
   int rows = 0;
   Scan scan = new Scan();
   scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
   ResultScanner s = meta.getScanner(scan);
   for (Result r = null; (r = s.next()) != null; ) {
     byte[] b = r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
     if (b == null || b.length <= 0) break;
     HRegionInfo hri = Writables.getHRegionInfo(b);
     // If start key, add 'aaa'.
     byte[] row = getStartKey(hri);
     Put p = new Put(row);
     p.setWriteToWAL(false);
     p.add(getTestFamily(), getTestQualifier(), row);
     t.put(p);
     rows++;
   }
   s.close();
   Assert.assertEquals(expected, rows);
   t.close();
   meta.close();
   return rows;
 }
  /**
   * 遍历多行
   *
   * @param tableName 表名
   * @param start_rowkey 开始行键
   * @param stop_rowkey 结束行键
   * @return 行列表
   */
  public ArrayList<HbaseRow> scanRows(String tableName, String start_rowkey, String stop_rowkey) {
    ResultScanner rowstmp = null;
    ArrayList<HbaseRow> rows = null;

    try {
      Scan scan = new Scan();
      scan.setStartRow(Bytes.toBytes(start_rowkey));
      scan.setStopRow(Bytes.toBytes(stop_rowkey));
      HTable table = new HTable(conf, Bytes.toBytes(tableName));
      rowstmp = table.getScanner(scan);
      rows = new ArrayList<>();

      for (Result rowtmp : rowstmp) {
        HbaseRow row = new HbaseRow();
        row.rowkey = Bytes.toString(rowtmp.getRow());
        for (Cell cell : rowtmp.listCells()) {
          HbaseColumn col = new HbaseColumn(cell);
          row.cols.add(col);
        }
        rows.add(row);
      }
    } catch (Exception e) {
      logger.error("scanRows failed", e);
    } finally {
      rowstmp.close();
    }
    return rows;
  }
  public static void main(String[] args) throws Exception {
    if (args.length < 2) {
      throw new Exception("Table name not specified.");
    }
    Configuration conf = HBaseConfiguration.create();
    HTable table = new HTable(conf, args[0]);
    String startKey = args[1];

    TimeCounter executeTimer = new TimeCounter();
    executeTimer.begin();
    executeTimer.enter();

    Expression exp =
        ExpressionFactory.eq(
            ExpressionFactory.toLong(
                ExpressionFactory.toString(ExpressionFactory.columnValue("family", "longStr2"))),
            ExpressionFactory.constant(Long.parseLong("99")));
    ExpressionFilter expressionFilter = new ExpressionFilter(exp);
    Scan scan = new Scan(Bytes.toBytes(startKey), expressionFilter);
    int count = 0;
    ResultScanner scanner = table.getScanner(scan);
    Result r = scanner.next();
    while (r != null) {
      count++;
      r = scanner.next();
    }
    System.out.println("++ Scanning finished with count : " + count + " ++");
    scanner.close();

    executeTimer.leave();
    executeTimer.end();
    System.out.println("++ Time cost for scanning: " + executeTimer.getTimeString() + " ++");
  }
  @Test
  public void testCheckpointRollback() throws Exception {
    // start a transaction, using checkpoints between writes
    transactionContext.start();
    transactionAwareHTable.put(
        new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
    transactionContext.checkpoint();
    transactionAwareHTable.put(
        new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
    transactionContext.checkpoint();
    transactionAwareHTable.put(
        new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));

    transactionContext.abort();

    transactionContext.start();
    verifyRow(transactionAwareHTable, TestBytes.row, null);
    verifyRow(transactionAwareHTable, TestBytes.row2, null);
    verifyRow(transactionAwareHTable, TestBytes.row3, null);

    Scan scan = new Scan();
    ResultScanner scanner = transactionAwareHTable.getScanner(scan);
    assertNull(scanner.next());
    scanner.close();
    transactionContext.finish();
  }
    public static List<Delete> GetDeleteEventsBetween(
        Table VTEvent_Table, String imo_str, long first_timestamp, long last_timestamp)
        throws IOException {
      // scan
      // 'cdb_vessel:vessel_event',{FILTER=>"(PrefixFilter('0000003162')"}
      Scan GetEventsBetween = new Scan();
      GetEventsBetween.setStartRow(
              Bytes.toBytes(imo_str + LpadNum(Long.MAX_VALUE - last_timestamp, 19) + "0000000000"))
          .setStopRow(
              Bytes.toBytes(
                  imo_str + LpadNum(Long.MAX_VALUE - first_timestamp + 1, 19) + "9999999999"))
          .addColumn(details, exittime);
      GetEventsBetween.setCaching(100);

      Filter ExistTimeValuefilter =
          new ValueFilter(
              CompareFilter.CompareOp.LESS_OR_EQUAL,
              new BinaryComparator(
                  Bytes.toBytes(new DateTime(last_timestamp).toString(rawformatter))));
      GetEventsBetween.setFilter(ExistTimeValuefilter);

      ResultScanner Result_ExistingEvents = VTEvent_Table.getScanner(GetEventsBetween);
      List<Delete> deletes = new ArrayList<Delete>();

      for (Result res : Result_ExistingEvents) {
        deletes.add(new Delete(res.getRow()));
      }

      Result_ExistingEvents.close();
      return deletes;
    }
Example #11
0
  public String[] getTags(String objectType, String objectId) throws IOException {
    List<String> ret = new ArrayList<String>();
    String rowKey = objectType + "_" + objectId;
    Scan s = new Scan(rowKey.getBytes(), rowKey.getBytes());
    s.setMaxVersions(1);
    ResultScanner scanner = htable.getScanner(s);
    try {
      for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {

        String localObjectType =
            new String(rr.getValue("tags".getBytes(), "OBJECTTYPE".getBytes()));
        String localObjectId = new String(rr.getValue("tags".getBytes(), "OBJECTID".getBytes()));
        NavigableMap<byte[], byte[]> map = rr.getFamilyMap("tags".getBytes());
        Iterator<Entry<byte[], byte[]>> it = map.entrySet().iterator();
        while (it.hasNext()) {
          Entry<byte[], byte[]> entry = it.next();
          String key = new String(entry.getKey());
          if (!key.startsWith("OBJECT")) {
            int val = Bytes.toInt(entry.getValue());
            if (val > 0) ret.add(key);
          }
        }
      }
    } finally {
      scanner.close();
    }
    return ret.toArray(new String[] {});
  }
Example #12
0
 /**
  * Performs a full scan of a catalog table.
  *
  * @param catalogTracker
  * @param visitor Visitor invoked against each row.
  * @param startrow Where to start the scan. Pass null if want to begin scan at first row.
  * @param scanRoot True if we are to scan <code>-ROOT-</code> rather than <code>.META.</code>, the
  *     default (pass false to scan .META.)
  * @throws IOException
  */
 static void fullScan(
     CatalogTracker catalogTracker,
     final Visitor visitor,
     final byte[] startrow,
     final boolean scanRoot)
     throws IOException {
   Scan scan = new Scan();
   if (startrow != null) scan.setStartRow(startrow);
   if (startrow == null && !scanRoot) {
     int caching =
         catalogTracker
             .getConnection()
             .getConfiguration()
             .getInt(HConstants.HBASE_META_SCANNER_CACHING, 100);
     scan.setCaching(caching);
   }
   scan.addFamily(HConstants.CATALOG_FAMILY);
   HTable metaTable = scanRoot ? getRootHTable(catalogTracker) : getMetaHTable(catalogTracker);
   ResultScanner scanner = metaTable.getScanner(scan);
   try {
     Result data;
     while ((data = scanner.next()) != null) {
       if (data.isEmpty()) continue;
       // Break if visit returns false.
       if (!visitor.visit(data)) break;
     }
   } finally {
     scanner.close();
     metaTable.close();
   }
   return;
 }
Example #13
0
  public static void deleteTest(String tableStr) {
    try {
      Configuration conf = HBaseConfiguration.create();
      byte[] tableName = Bytes.toBytes(tableStr);

      HConnection hConnection = HConnectionManager.createConnection(conf);
      HTableInterface table = hConnection.getTable(tableName);

      byte[] startRow = Bytes.toBytes("rowKey_1");
      byte[] stopRow = Bytes.toBytes("rowKey_3");
      byte[] family = f0;

      Scan scan = new Scan();
      scan.addFamily(family);
      scan.setMaxVersions(1);

      //            scan.setStartRow(startRow);
      //            scan.setStopRow(stopRow);

      ResultScanner scanner = table.getScanner(scan);
      Result result = scanner.next();
      List<Delete> delete = new ArrayList<Delete>();
      while (result != null) {
        Delete del = new Delete(result.getRow());
        delete.add(del);
        result = scanner.next();
      }
      table.delete(delete);
      System.out.println("delete done");
      table.close(); // very important
    } catch (IOException e) {
      e.printStackTrace();
    }
  }
 /**
  * Looks at every value of the mapreduce output and verifies that indeed the values have been
  * reversed.
  *
  * @param table Table to scan.
  * @throws IOException
  * @throws NullPointerException if we failed to find a cell value
  */
 private void verifyAttempt(final Table table) throws IOException, NullPointerException {
   Scan scan = new Scan();
   scan.addFamily(INPUT_FAMILY);
   scan.addFamily(OUTPUT_FAMILY);
   ResultScanner scanner = table.getScanner(scan);
   try {
     Iterator<Result> itr = scanner.iterator();
     assertTrue(itr.hasNext());
     while (itr.hasNext()) {
       Result r = itr.next();
       if (LOG.isDebugEnabled()) {
         if (r.size() > 2) {
           throw new IOException("Too many results, expected 2 got " + r.size());
         }
       }
       byte[] firstValue = null;
       byte[] secondValue = null;
       int count = 0;
       for (Cell kv : r.listCells()) {
         if (count == 0) {
           firstValue = CellUtil.cloneValue(kv);
         } else if (count == 1) {
           secondValue = CellUtil.cloneValue(kv);
         } else if (count == 2) {
           break;
         }
         count++;
       }
       String first = "";
       if (firstValue == null) {
         throw new NullPointerException(Bytes.toString(r.getRow()) + ": first value is null");
       }
       first = Bytes.toString(firstValue);
       String second = "";
       if (secondValue == null) {
         throw new NullPointerException(Bytes.toString(r.getRow()) + ": second value is null");
       }
       byte[] secondReversed = new byte[secondValue.length];
       for (int i = 0, j = secondValue.length - 1; j >= 0; j--, i++) {
         secondReversed[i] = secondValue[j];
       }
       second = Bytes.toString(secondReversed);
       if (first.compareTo(second) != 0) {
         if (LOG.isDebugEnabled()) {
           LOG.debug(
               "second key is not the reverse of first. row="
                   + Bytes.toStringBinary(r.getRow())
                   + ", first value="
                   + first
                   + ", second value="
                   + second);
         }
         fail();
       }
     }
   } finally {
     scanner.close();
   }
 }
Example #15
0
  public static void main(String[] args) throws Exception {
    conf.set("hbase.zookeeper.quorum", "hadoop271.itversity.com");
    conf.set("hbase.zookeeper.property.clientPort", "2181");

    Connection connection = ConnectionFactory.createConnection(conf);
    Table table = connection.getTable(TableName.valueOf("demo"));

    Scan scan1 = new Scan();
    ResultScanner scanner1 = table.getScanner(scan1);

    for (Result res : scanner1) {
      System.out.println(Bytes.toString(res.getRow()));
      System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column1".getBytes())));
      System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column2".getBytes())));
    }

    scanner1.close();

    Put put = new Put("3".getBytes());

    put.addColumn("cf1".getBytes(), "column1".getBytes(), "value1".getBytes());
    put.addColumn("cf1".getBytes(), "column2".getBytes(), "value2".getBytes());

    table.put(put);

    Get get = new Get("3".getBytes());
    Result getResult = table.get(get);
    System.out.println("Printing colunns for rowkey 3");
    System.out.println(Bytes.toString(getResult.getValue("cf1".getBytes(), "column1".getBytes())));
    System.out.println(Bytes.toString(getResult.getValue("cf1".getBytes(), "column2".getBytes())));

    scanner1 = table.getScanner(scan1);
    System.out.println("Before Delete");
    for (Result res : scanner1) {
      System.out.println(Bytes.toString(res.getRow()));
      System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column1".getBytes())));
      System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column2".getBytes())));
    }

    scanner1.close();

    Delete del = new Delete("3".getBytes());
    table.delete(del);

    System.out.println("After Delete");

    scanner1 = table.getScanner(scan1);

    for (Result res : scanner1) {
      System.out.println(Bytes.toString(res.getRow()));
      System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column1".getBytes())));
      System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column2".getBytes())));
    }

    scanner1.close();
    table.close();
    connection.close();
  }
 private void verifyRowCount(Table table, int expectedRegionNum) throws IOException {
   ResultScanner scanner = table.getScanner(new Scan());
   int rowCount = 0;
   while (scanner.next() != null) {
     rowCount++;
   }
   assertEquals(expectedRegionNum, rowCount);
   scanner.close();
 }
  /**
   * Confirm ImportTsv via data in online table.
   *
   * @param dataAvailable
   */
  private static void validateTable(
      Configuration conf,
      TableName tableName,
      String family,
      int valueMultiplier,
      boolean dataAvailable)
      throws IOException {

    LOG.debug("Validating table.");
    Connection connection = ConnectionFactory.createConnection(conf);
    Table table = connection.getTable(tableName);
    boolean verified = false;
    long pause = conf.getLong("hbase.client.pause", 5 * 1000);
    int numRetries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
    for (int i = 0; i < numRetries; i++) {
      try {
        Scan scan = new Scan();
        // Scan entire family.
        scan.addFamily(Bytes.toBytes(family));
        if (dataAvailable) {
          ResultScanner resScanner = table.getScanner(scan);
          for (Result res : resScanner) {
            LOG.debug("Getting results " + res.size());
            assertTrue(res.size() == 2);
            List<Cell> kvs = res.listCells();
            assertTrue(CellUtil.matchingRow(kvs.get(0), Bytes.toBytes("KEY")));
            assertTrue(CellUtil.matchingRow(kvs.get(1), Bytes.toBytes("KEY")));
            assertTrue(
                CellUtil.matchingValue(kvs.get(0), Bytes.toBytes("VALUE" + valueMultiplier)));
            assertTrue(
                CellUtil.matchingValue(kvs.get(1), Bytes.toBytes("VALUE" + 2 * valueMultiplier)));
            // Only one result set is expected, so let it loop.
            verified = true;
          }
        } else {
          ResultScanner resScanner = table.getScanner(scan);
          Result[] next = resScanner.next(2);
          assertEquals(0, next.length);
          verified = true;
        }

        break;
      } catch (NullPointerException e) {
        // If here, a cell was empty. Presume its because updates came in
        // after the scanner had been opened. Wait a while and retry.
      }
      try {
        Thread.sleep(pause);
      } catch (InterruptedException e) {
        // continue
      }
    }
    table.close();
    connection.close();
    assertTrue(verified);
  }
    // Get all events with exit at last location
    public static Map<Integer, VesselEvent> getAllEventsStartBeforeEndAfterBeforeLocation(
        Table VTEvent_Table, String IMO_str, VesselLocation location) throws IOException {
      Scan getAllEventsWithExistAtLastLocation = new Scan();
      getAllEventsWithExistAtLastLocation
          .setStartRow(
              Bytes.toBytes(
                  IMO_str + LpadNum(Long.MAX_VALUE - location.recordtime, 19) + "0000000000"))
          .setStopRow(Bytes.toBytes(IMO_str + LpadNum(Long.MAX_VALUE, 19) + "9999999999"))
          .addColumn(details, exittime);
      getAllEventsWithExistAtLastLocation.setCaching(100);

      Filter ExistTimeValuefilter =
          new ValueFilter(
              CompareFilter.CompareOp.GREATER_OR_EQUAL,
              new BinaryComparator(
                  Bytes.toBytes(new DateTime(location.recordtime).toString(rawformatter))));
      getAllEventsWithExistAtLastLocation.setFilter(ExistTimeValuefilter);

      ResultScanner Result_event = VTEvent_Table.getScanner(getAllEventsWithExistAtLastLocation);

      Map<Integer, VesselEvent> events = new HashMap<Integer, VesselEvent>();

      for (Result res : Result_event) {

        Get get = new Get(res.getRow());
        get.addColumn(details, entrytime);
        get.addColumn(details, entrycoordinates);

        Result result = VTEvent_Table.get(get);
        String rowkey = Bytes.toString(result.getRow());
        String polygonid = rowkey.substring(26);

        VesselEvent VE = new VesselEvent();
        VE.exittime = location.recordtime;
        VE.exitcoordinates = location.coordinates;
        VE.destination = location.destination;
        VE.polygonid = Integer.parseInt(polygonid);

        for (Cell cell : result.rawCells()) {
          String Qualifier = Bytes.toString(CellUtil.cloneQualifier(cell));
          String Value = Bytes.toString(CellUtil.cloneValue(cell));

          if (Qualifier.equals("entertime")) {
            VE.entrytime = DateTime.parse(Value, rawformatter).getMillis();
          } else if (Qualifier.equals("entercoordinates")) {
            VE.entrycoordinates = Value;
          }
        }

        events.put(VE.polygonid, VE);
      }

      Result_event.close();
      return events;
    }
  public static void main(String[] args) throws IOException {
    Configuration conf = HBaseConfiguration.create();

    HBaseHelper helper = HBaseHelper.getHelper(conf);
    helper.dropTable("testtable");
    helper.createTable("testtable", "colfam1", "colfam2", "colfam3", "colfam4");
    System.out.println("Adding rows to table...");
    helper.fillTable("testtable", 1, 10, 2, "colfam1", "colfam2", "colfam3", "colfam4");

    HTable table = new HTable(conf, "testtable");

    // vv FamilyFilterExample
    Filter filter1 =
        new FamilyFilter(
            CompareFilter.CompareOp
                .LESS, // co FamilyFilterExample-1-Filter Create filter, while specifying the
                       // comparison operator and comparator.
            new BinaryComparator(Bytes.toBytes("colfam3")));

    Scan scan = new Scan();
    scan.setFilter(filter1);
    ResultScanner scanner =
        table.getScanner(
            scan); // co FamilyFilterExample-2-Scan Scan over table while applying the filter.
    // ^^ FamilyFilterExample
    System.out.println("Scanning table... ");
    // vv FamilyFilterExample
    for (Result result : scanner) {
      System.out.println(result);
    }
    scanner.close();

    Get get1 = new Get(Bytes.toBytes("row-5"));
    get1.setFilter(filter1);
    Result result1 =
        table.get(get1); // co FamilyFilterExample-3-Get Get a row while applying the same filter.
    System.out.println("Result of get(): " + result1);

    Filter filter2 =
        new FamilyFilter(
            CompareFilter.CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("colfam3")));
    Get get2 =
        new Get(
            Bytes.toBytes(
                "row-5")); // co FamilyFilterExample-4-Mismatch Create a filter on one column family
                           // while trying to retrieve another.
    get2.addFamily(Bytes.toBytes("colfam1"));
    get2.setFilter(filter2);
    Result result2 =
        table.get(
            get2); // co FamilyFilterExample-5-Get2 Get the same row while applying the new filter,
                   // this will return "NONE".
    System.out.println("Result of get(): " + result2);
    // ^^ FamilyFilterExample
  }
Example #20
0
 /**
  * Insert a whole batch of entries
  *
  * @throws Exception
  */
 @Test
 public void testBatchSink() throws Exception {
   List<WALEntry> entries = new ArrayList<WALEntry>(BATCH_SIZE);
   List<Cell> cells = new ArrayList<Cell>();
   for (int i = 0; i < BATCH_SIZE; i++) {
     entries.add(createEntry(TABLE_NAME1, i, KeyValue.Type.Put, cells));
   }
   SINK.replicateEntries(entries, CellUtil.createCellScanner(cells.iterator()));
   Scan scan = new Scan();
   ResultScanner scanRes = table1.getScanner(scan);
   assertEquals(BATCH_SIZE, scanRes.next(BATCH_SIZE).length);
 }
 /*
  * @return Count of rows in TABLENAME
  * @throws IOException
  */
 private static int count() throws IOException {
   HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
   int rows = 0;
   Scan scan = new Scan();
   ResultScanner s = t.getScanner(scan);
   for (Result r = null; (r = s.next()) != null; ) {
     rows++;
   }
   s.close();
   LOG.info("Counted=" + rows);
   return rows;
 }
Example #22
0
  @Override
  public void tweet(User user, String tweetText) throws IOException {
    final long epoch = System.currentTimeMillis();
    final long tranposeEpoch = Long.MAX_VALUE - epoch;
    final byte[] epochBytes = Bytes.toBytes(epoch);

    final byte[] tweetBytes = Bytes.toBytes(tweetText);
    byte[] nameBytes = Bytes.toBytes(user.getName());

    /** put tweet into tweets */
    Put tweetRowPut = new Put(generateTweetId(user));

    tweetRowPut.add(_DEFAULT, _NAME, nameBytes);
    tweetRowPut.add(_DEFAULT, _MAIL, Bytes.toBytes(user.getEmail()));
    tweetRowPut.add(_DEFAULT, _TWEET, tweetBytes);
    tweetRowPut.add(_DEFAULT, _TIME, epochBytes);

    tweetsTable.put(tweetRowPut);

    /** put tweets for followers */
    Scan followerScan = new Scan();
    followerScan.setStartRow(Bytes.toBytes(user.getUserId() + "-"));
    followerScan.setStopRow(Bytes.toBytes((user.getUserId() + 1) + "-"));

    ResultScanner followerRS = followersTable.getScanner(followerScan);

    /** put users on tweet to her own tweetline */
    Put put =
        new Put(Bytes.toBytes(user.getUserId() + "-" + tranposeEpoch + "-" + user.getUserId()));
    put.add(_DEFAULT, _NAME, nameBytes);
    put.add(_DEFAULT, _TWEET, tweetBytes);
    put.add(_DEFAULT, _TIME, epochBytes);

    List<Row> puts = new ArrayList<Row>();
    puts.add(put);
    for (Result result : followerRS) {

      Long followerid = Bytes.toLong(result.getColumnLatest(_DEFAULT, _USERID).getValue());

      put = new Put(Bytes.toBytes(followerid + "-" + tranposeEpoch + "-" + user.getUserId()));
      put.add(_DEFAULT, _NAME, nameBytes);
      put.add(_DEFAULT, _TWEET, tweetBytes);
      put.add(_DEFAULT, _TIME, epochBytes);

      puts.add(put);
    }
    followerRS.close();
    try {
      tweetlineTable.batch(puts);
    } catch (InterruptedException e) {
      e.printStackTrace(); // @TODO log and handle properly.
    }
  }
  public static void main(String[] args) throws IOException {
    Configuration conf = HBaseConfiguration.create();

    HBaseHelper helper = HBaseHelper.getHelper(conf);
    helper.dropTable("testtable");
    helper.createTable("testtable", "colfam1", "colfam2");
    System.out.println("Adding rows to table...");
    helper.fillTable("testtable", 1, 10, 10, "colfam1", "colfam2");

    Connection connection = ConnectionFactory.createConnection(conf);
    Table table = connection.getTable(TableName.valueOf("testtable"));
    // vv SingleColumnValueFilterExample
    SingleColumnValueFilter filter =
        new SingleColumnValueFilter(
            Bytes.toBytes("colfam1"),
            Bytes.toBytes("col-5"),
            CompareFilter.CompareOp.NOT_EQUAL,
            new SubstringComparator("val-5"));
    filter.setFilterIfMissing(true);

    Scan scan = new Scan();
    scan.setFilter(filter);
    ResultScanner scanner = table.getScanner(scan);
    // ^^ SingleColumnValueFilterExample
    System.out.println("Results of scan:");
    // vv SingleColumnValueFilterExample
    for (Result result : scanner) {
      for (Cell cell : result.rawCells()) {
        System.out.println(
            "Cell: "
                + cell
                + ", Value: "
                + Bytes.toString(
                    cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
      }
    }
    scanner.close();

    Get get = new Get(Bytes.toBytes("row-6"));
    get.setFilter(filter);
    Result result = table.get(get);
    System.out.println("Result of get: ");
    for (Cell cell : result.rawCells()) {
      System.out.println(
          "Cell: "
              + cell
              + ", Value: "
              + Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
    }
    // ^^ SingleColumnValueFilterExample
  }
  @BeforeClass
  public static void setUpBeforeClass() throws Exception {
    // Start up our mini cluster on top of an 0.92 root.dir that has data from
    // a 0.92 hbase run -- it has a table with 100 rows in it  -- and see if
    // we can migrate from 0.92
    TEST_UTIL.startMiniZKCluster();
    TEST_UTIL.startMiniDFSCluster(1);
    Path testdir = TEST_UTIL.getDataTestDir("TestMetaMigrationConvertToPB");
    // Untar our test dir.
    File untar = untar(new File(testdir.toString()));
    // Now copy the untar up into hdfs so when we start hbase, we'll run from it.
    Configuration conf = TEST_UTIL.getConfiguration();
    FsShell shell = new FsShell(conf);
    FileSystem fs = FileSystem.get(conf);
    // find where hbase will root itself, so we can copy filesystem there
    Path hbaseRootDir = TEST_UTIL.getDefaultRootDirPath();
    if (!fs.isDirectory(hbaseRootDir.getParent())) {
      // mkdir at first
      fs.mkdirs(hbaseRootDir.getParent());
    }
    doFsCommand(shell, new String[] {"-put", untar.toURI().toString(), hbaseRootDir.toString()});

    // windows fix: tgz file has .META. directory renamed as -META- since the original is an illegal
    // name under windows. So we rename it back. See
    // src/test/data//TestMetaMigrationConvertingToPB.README and
    // https://issues.apache.org/jira/browse/HBASE-6821
    doFsCommand(
        shell,
        new String[] {
          "-mv",
          new Path(hbaseRootDir, "-META-").toString(),
          new Path(hbaseRootDir, ".META.").toString()
        });
    // See whats in minihdfs.
    doFsCommand(shell, new String[] {"-lsr", "/"});
    TEST_UTIL.startMiniHBaseCluster(1, 1);
    // Assert we are running against the copied-up filesystem.  The copied-up
    // rootdir should have had a table named 'TestTable' in it.  Assert it
    // present.
    HTable t = new HTable(TEST_UTIL.getConfiguration(), TESTTABLE);
    ResultScanner scanner = t.getScanner(new Scan());
    int count = 0;
    while (scanner.next() != null) {
      count++;
    }
    // Assert that we find all 100 rows that are in the data we loaded.  If
    // so then we must have migrated it from 0.90 to 0.92.
    Assert.assertEquals(ROW_COUNT, count);
    scanner.close();
    t.close();
  }
Example #25
0
  public List<RowLogMessage> next(String subscription, Long minimalTimestamp, boolean problematic)
      throws RowLogException {
    byte[] rowPrefix;
    byte[] subscriptionBytes = Bytes.toBytes(subscription);
    if (problematic) {
      rowPrefix = PROBLEMATIC_MARKER;
      rowPrefix = Bytes.add(rowPrefix, subscriptionBytes);
    } else {
      rowPrefix = subscriptionBytes;
    }
    byte[] startRow = rowPrefix;
    if (minimalTimestamp != null) startRow = Bytes.add(startRow, Bytes.toBytes(minimalTimestamp));
    try {
      List<RowLogMessage> rowLogMessages = new ArrayList<RowLogMessage>();
      Scan scan = new Scan(startRow);
      if (minimalTimestamp != null) scan.setTimeRange(minimalTimestamp, Long.MAX_VALUE);
      scan.addColumn(MESSAGES_CF, MESSAGE_COLUMN);
      ResultScanner scanner = table.getScanner(scan);
      boolean keepScanning = problematic;
      do {
        Result[] results = scanner.next(batchSize);
        if (results.length == 0) {
          keepScanning = false;
        }
        for (Result next : results) {
          byte[] rowKey = next.getRow();
          if (!Bytes.startsWith(rowKey, rowPrefix)) {
            keepScanning = false;
            break; // There were no messages for this subscription
          }
          if (problematic) {
            rowKey = Bytes.tail(rowKey, rowKey.length - PROBLEMATIC_MARKER.length);
          }
          byte[] value = next.getValue(MESSAGES_CF, MESSAGE_COLUMN);
          byte[] messageId = Bytes.tail(rowKey, rowKey.length - subscriptionBytes.length);
          rowLogMessages.add(decodeMessage(messageId, value));
        }
      } while (keepScanning);

      // The scanner is not closed in a finally block, since when we get an IOException from
      // HBase, it is likely that closing the scanner will give problems too. Not closing
      // the scanner is not fatal since HBase will expire it after a while.
      Closer.close(scanner);

      return rowLogMessages;
    } catch (IOException e) {
      throw new RowLogException("Failed to fetch next message from RowLogShard", e);
    }
  }
  public static void main(String[] args) throws IOException {

    Configuration conf = HBaseClientHelper.loadDefaultConfiguration();

    Connection connection = ConnectionFactory.createConnection(conf);

    try {

      Table table = connection.getTable(TableName.valueOf("testtable"));
      try {
        // 1 Put
        Put p = new Put(Bytes.toBytes("row1"));
        p.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"), Bytes.toBytes("val1"));

        table.put(p);

        // 2 Get
        Get g = new Get(Bytes.toBytes("row1"));
        Result r = table.get(g);
        byte[] value = r.getValue(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"));
        String valueStr = Bytes.toString(value);
        System.out.println("GET: " + valueStr);

        // 3 Scan
        Scan s = new Scan();
        s.addColumn(Bytes.toBytes("colfam1"), Bytes.toBytes("qual1"));
        ResultScanner scanner = table.getScanner(s);
        try {
          for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
            System.out.println("Found row: " + rr);
          }

          // The other approach is to use a foreach loop. Scanners are
          // iterable!
          // for (Result rr : scanner) {
          // System.out.println("Found row: " + rr);
          // }
        } finally {
          scanner.close();
        }

        // Close your table and cluster connection.
      } finally {
        if (table != null) table.close();
      }
    } finally {
      connection.close();
    }
  }
 /** Return the number of rows in the given table. */
 public static int countMobRows(final Table table) throws IOException {
   Scan scan = new Scan();
   ResultScanner results = table.getScanner(scan);
   int count = 0;
   for (Result res : results) {
     count++;
     List<Cell> cells = res.listCells();
     for (Cell cell : cells) {
       // Verify the value
       Assert.assertTrue(CellUtil.cloneValue(cell).length > 0);
     }
   }
   results.close();
   return count;
 }
  public static void main(String[] args) throws IOException, SolrServerException {

    final Configuration conf;
    HttpSolrServer solrServer = new HttpSolrServer("http://c1master:8983/solr");
    conf = HBaseConfiguration.create();

    // Define Hbase Table Name
    HTable table = new HTable(conf, "test_global_shop");
    Scan scan = new Scan();

    // Define Hbase Column Family
    scan.addFamily(Bytes.toBytes("shop"));
    scan.setCaching(1000);
    scan.setCacheBlocks(false);
    ResultScanner ss = table.getScanner(scan);

    System.out.println("start Storing...");
    int i = 0;
    try {
      for (Result r : ss) {
        SolrInputDocument solrDoc = new SolrInputDocument();
        solrDoc.addField("key", new String(r.getRow()));
        for (KeyValue kv : r.raw()) {
          String fieldName = new String(kv.getQualifier());
          String fieldValue = new String(kv.getValue());
          if (fieldName.equalsIgnoreCase("address")
              || fieldName.equalsIgnoreCase("category")
              || fieldName.equalsIgnoreCase("name")
              || fieldName.equalsIgnoreCase("province")
              || fieldName.equalsIgnoreCase("tel")) {
            solrDoc.addField(fieldName, fieldValue);
          }
        }
        solrServer.add(solrDoc);
        solrServer.commit(true, true, true);
        i = i + 1;
        System.out.println("Already Succcess " + i + " number data");
      }
      ss.close();
      table.close();
      System.out.println("done !");
    } catch (IOException e) {
    } finally {
      ss.close();
      table.close();
      System.out.println("error !");
    }
  }
Example #29
0
  public Blog(String blogid) throws IOException {

    Configuration conf = HBaseConfiguration.create();
    table = new HTable(conf, "blogs");

    // 1. Get the row whose row key is blogid from above
    Get g = new Get(Bytes.toBytes(blogid));
    Result r = table.get(g);

    // 2. Extract the rowkey, blog text (column "body") and blog title
    // (column "meta:title")
    key = r.getRow();
    keyStr = Bytes.toString(key);
    blogText = Bytes.toString(r.getValue(Bytes.toBytes("body"), Bytes.toBytes("")));
    blogTitle = Bytes.toString(r.getValue(Bytes.toBytes("meta"), Bytes.toBytes("title")));
    Long reverseTimestamp = Long.parseLong(keyStr.substring(4));
    Long epoch = Math.abs(reverseTimestamp - Long.MAX_VALUE);
    dateOfPost = new Date(epoch);

    // Get an iterator for the comments
    Scan s = new Scan();
    s.addFamily(Bytes.toBytes("comment"));
    // Use a PrefixFilter
    PrefixFilter filter = new PrefixFilter(key);
    s.setFilter(filter);
    scanner = table.getScanner(s);
    resultIterator = scanner.iterator();
  }
Example #30
0
 /*
  * Check current row has a HRegionInfo.  Skip to next row if HRI is empty.
  * @return A Map of the row content else null if we are off the end.
  * @throws IOException
  */
 private Result getMetaRow() throws IOException {
   Result currentRow = metaScanner.next();
   boolean foundResult = false;
   while (currentRow != null) {
     LOG.info("Row: <" + Bytes.toStringBinary(currentRow.getRow()) + ">");
     byte[] regionInfoValue =
         currentRow.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
     if (regionInfoValue == null || regionInfoValue.length == 0) {
       currentRow = metaScanner.next();
       continue;
     }
     foundResult = true;
     break;
   }
   return foundResult ? currentRow : null;
 }