Ejemplo n.º 1
0
 /**
  * Creates the scanner for flushing snapshot. Also calls coprocessors.
  *
  * @param snapshotScanner
  * @param smallestReadPoint
  * @return The scanner; null if coprocessor is canceling the flush.
  */
 protected InternalScanner createScanner(KeyValueScanner snapshotScanner, long smallestReadPoint)
     throws IOException {
   InternalScanner scanner = null;
   if (store.getCoprocessorHost() != null) {
     scanner = store.getCoprocessorHost().preFlushScannerOpen(store, snapshotScanner);
   }
   if (scanner == null) {
     Scan scan = new Scan();
     scan.setMaxVersions(store.getScanInfo().getMaxVersions());
     scanner =
         new StoreScanner(
             store,
             store.getScanInfo(),
             scan,
             Collections.singletonList(snapshotScanner),
             ScanType.COMPACT_RETAIN_DELETES,
             smallestReadPoint,
             HConstants.OLDEST_TIMESTAMP);
   }
   assert scanner != null;
   if (store.getCoprocessorHost() != null) {
     try {
       return store.getCoprocessorHost().preFlush(store, scanner);
     } catch (IOException ioe) {
       scanner.close();
       throw ioe;
     }
   }
   return scanner;
 }
Ejemplo n.º 2
0
  /** Ensure that expired delete family markers don't override valid puts */
  public void testExpiredDeleteFamily() throws Exception {
    long now = System.currentTimeMillis();
    KeyValue[] kvs =
        new KeyValue[] {
          new KeyValue(
              Bytes.toBytes("R1"),
              Bytes.toBytes("cf"),
              null,
              now - 1000,
              KeyValue.Type.DeleteFamily),
          KeyValueTestUtil.create("R1", "cf", "a", now - 10, KeyValue.Type.Put, "dont-care"),
        };
    List<KeyValueScanner> scanners = scanFixture(kvs);
    Scan scan = new Scan();
    scan.setMaxVersions(1);
    // scanner with ttl equal to 500
    ScanInfo scanInfo = new ScanInfo(CF, 0, 1, 500, false, 0, KeyValue.COMPARATOR);
    ScanType scanType = ScanType.USER_SCAN;
    StoreScanner scanner = new StoreScanner(scan, scanInfo, scanType, null, scanners);

    List<KeyValue> results = new ArrayList<KeyValue>();
    assertEquals(true, scanner.next(results));
    assertEquals(1, results.size());
    assertEquals(kvs[1], results.get(0));
    results.clear();

    assertEquals(false, scanner.next(results));
  }
Ejemplo n.º 3
0
  public String[] getObjectIDs(String objectType, String... tags) throws IOException {
    List<String> ret = new ArrayList<String>();

    FilterList list = new FilterList(FilterList.Operator.MUST_PASS_ALL);
    SingleColumnValueFilter filter1 =
        new SingleColumnValueFilter(
            "tags".getBytes(), "OBJECTTYPE".getBytes(), CompareOp.EQUAL, Bytes.toBytes(objectType));
    list.addFilter(filter1);

    for (String tag : tags) {
      SingleColumnValueFilter filter2 =
          new SingleColumnValueFilter(
              "tags".getBytes(), tag.toUpperCase().getBytes(), CompareOp.EQUAL, Bytes.toBytes(1));
      filter2.setFilterIfMissing(true);
      list.addFilter(filter2);
    }
    Scan s = new Scan();
    s.setFilter(list);
    s.setMaxVersions(1);
    ResultScanner scanner = htable.getScanner(s);
    try {
      for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {

        String localObjectType =
            new String(rr.getValue("tags".getBytes(), "OBJECTTYPE".getBytes()));
        String localObjectId = new String(rr.getValue("tags".getBytes(), "OBJECTID".getBytes()));
        ret.add(localObjectId);
      }
    } finally {
      scanner.close();
    }

    return ret.toArray(new String[] {});
  }
Ejemplo n.º 4
0
 public List<String> getStationsNearPoint_Schema2(double lat, double lon) throws IOException {
   Scan scan = new Scan();
   scan.addFamily(BixiConstant.SCHEMA2_CLUSTER_FAMILY_NAME.getBytes());
   InternalScanner scanner =
       ((RegionCoprocessorEnvironment) getEnvironment()).getRegion().getScanner(scan);
   boolean hasMoreResult = false;
   List<KeyValue> res = new ArrayList<KeyValue>();
   List<String> result = new ArrayList<String>();
   try {
     do {
       hasMoreResult = scanner.next(res);
       for (KeyValue kv : res) {
         String clusterId = Bytes.toString(kv.getRow());
         String[] parts = clusterId.split(":");
         double cLat = Double.parseDouble(parts[0]);
         double cLon = Double.parseDouble(parts[1]);
         double dx = Double.parseDouble(parts[2]);
         double dy = Double.parseDouble(parts[3]);
         double distx = lat - cLat;
         double disty = lon - cLon;
         if (distx >= 0 && distx <= dx && disty >= 0 && disty <= dy) {
           // get stations in cluster
           result.add(Bytes.toString(kv.getQualifier()));
         }
       }
       res.clear();
     } while (hasMoreResult);
   } finally {
     scanner.close();
   }
   return result;
 }
  /*
   * (non-Javadoc)
   *
   * @see com.impetus.kunderahbase.dao.user.UserDao#findAllByUserName(int)
   */
  @Override
  public void findAllByUserName(int count) {
    Filter filter =
        new SingleColumnValueFilter(
            Bytes.toBytes("user_name"),
            Bytes.toBytes("user_name"),
            CompareOp.EQUAL,
            Bytes.toBytes("Amry"));
    Scan scan = new Scan();
    scan.setFilter(filter);
    scan.addColumn(Bytes.toBytes("user_name"), Bytes.toBytes("user_name"));
    scan.addColumn(Bytes.toBytes("user_nameCnt"), Bytes.toBytes("user_nameCnt"));
    scan.addColumn(Bytes.toBytes("password"), Bytes.toBytes("password"));
    scan.addColumn(Bytes.toBytes("relation"), Bytes.toBytes("relation"));

    ResultScanner scanner = null;
    int counter = 0;
    try {
      scanner = hTablePool.getTable("User").getScanner(scan);
    } catch (IOException e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }
    assert scanner != null;
    for (Result result : scanner) {
      counter++;
      assert result != null;
    }
    assert counter != 0;
    assert counter == count;
  }
Ejemplo n.º 6
0
  /**
   * Given a scan and a key range, return a new Scan whose range is truncated to only include keys
   * in that range. Returns null if the Scan does not overlap the given range.
   */
  private static final Scan truncateScan(Scan scan, byte[] rangeStart, byte[] rangeEnd) {
    byte[] scanStart = scan.getStartRow();
    byte[] scanEnd = scan.getStopRow();

    if (scanEnd.length > 0 && bytesCompare(scanEnd, rangeStart) <= 0) {
      // The entire scan range is before the entire cube key range
      return null;
    } else if (scanStart.length > 0 && bytesCompare(scanStart, rangeEnd) >= 0) {
      // The entire scan range is after the entire cube key range
      return null;
    } else {
      // Now we now that the scan range at least partially overlaps the cube key range.
      Scan truncated;
      try {
        truncated = new Scan(scan); // make a copy, don't modify input scan
      } catch (IOException e) {
        throw new RuntimeException(); // This is not plausible
      }

      if (scanStart.length == 0 || bytesCompare(rangeStart, scanStart) > 0) {
        // The scan includes extra keys at the beginning that are not part of the cube. Move
        // the scan start point so that it only touches keys belonging to the cube.
        truncated.setStartRow(rangeStart);
      }
      if (scanEnd.length == 0 || bytesCompare(rangeEnd, scanEnd) < 0) {
        // The scan includes extra keys at the end that are not part of the cube. Move the
        // scan end point so it only touches keys belonging to the cube.
        truncated.setStopRow(rangeEnd);
      }
      return truncated;
    }
  }
      @Override
      public void process(long now, HRegion region, List<Mutation> mutations, WALEdit walEdit)
          throws IOException {
        List<Cell> kvs = new ArrayList<Cell>();
        { // First scan to get friends of the person
          Scan scan = new Scan(row, row);
          scan.addColumn(FAM, person);
          doScan(region, scan, kvs);
        }

        // Second scan to get friends of friends
        Scan scan = new Scan(row, row);
        for (Cell kv : kvs) {
          byte[] friends = CellUtil.cloneValue(kv);
          for (byte f : friends) {
            scan.addColumn(FAM, new byte[] {f});
          }
        }
        doScan(region, scan, kvs);

        // Collect result
        result.clear();
        for (Cell kv : kvs) {
          for (byte b : CellUtil.cloneValue(kv)) {
            result.add((char) b + "");
          }
        }
      }
  public static ResultScanner getAllFamilyList(
      String startRowRange, String stopRowRange, byte[][] cfs, String ctableName) {

    Scan scan = new Scan();

    scan.setStartRow(Bytes.toBytes(startRowRange));

    for (byte[] family : cfs) {
      scan.addFamily(family);
    }

    if (stopRowRange != null) {
      scan.setStopRow(Bytes.toBytes(stopRowRange));
    }

    ResultScanner resultScanner = null;

    try {
      resultScanner = tblMngr.getTable(ctableName).getScanner(scan);

    } catch (Exception e) {

    }
    return resultScanner;
  }
  @Test
  public void testInclusiveStopFilter() throws Exception {
    // Grab rows from group one

    // If we just use start/stop row, we get total/2 - 1 rows
    long expectedRows = (numRows / 2) - 1;
    long expectedKeys = colsPerRow;
    Scan s = new Scan(Bytes.toBytes("testRowOne-0"), Bytes.toBytes("testRowOne-3"));
    verifyScan(s, expectedRows, expectedKeys);

    // Now use start row with inclusive stop filter
    expectedRows = numRows / 2;
    s = new Scan(Bytes.toBytes("testRowOne-0"));
    s.setFilter(new InclusiveStopFilter(Bytes.toBytes("testRowOne-3")));
    verifyScan(s, expectedRows, expectedKeys);

    // Grab rows from group two

    // If we just use start/stop row, we get total/2 - 1 rows
    expectedRows = (numRows / 2) - 1;
    expectedKeys = colsPerRow;
    s = new Scan(Bytes.toBytes("testRowTwo-0"), Bytes.toBytes("testRowTwo-3"));
    verifyScan(s, expectedRows, expectedKeys);

    // Now use start row with inclusive stop filter
    expectedRows = numRows / 2;
    s = new Scan(Bytes.toBytes("testRowTwo-0"));
    s.setFilter(new InclusiveStopFilter(Bytes.toBytes("testRowTwo-3")));
    verifyScan(s, expectedRows, expectedKeys);
  }
  /**
   * Sets up the actual job.
   *
   * @param conf The current configuration.
   * @param args The command line parameters.
   * @return The newly created job.
   * @throws IOException When setting up the job fails.
   */
  public static Job createSubmittableJob(Configuration conf, String[] args) throws IOException {
    String tableName = args[0];
    Job job = new Job(conf, NAME + "_" + tableName);
    job.setJarByClass(CachingRowCounter.class);
    // Columns are space delimited
    StringBuilder sb = new StringBuilder();
    final int columnoffset = 1;
    for (int i = columnoffset; i < args.length; i++) {
      if (i > columnoffset) {
        sb.append(" ");
      }
      sb.append(args[i]);
    }

    Scan scan = new Scan();
    scan.setFilter(new FirstKeyOnlyFilter());
    if (sb.length() > 0) {
      for (String columnName : sb.toString().split(" ")) {
        String[] fields = columnName.split(":");
        if (fields.length == 1) {
          scan.addFamily(Bytes.toBytes(fields[0]));
        } else {
          scan.addColumn(Bytes.toBytes(fields[0]), Bytes.toBytes(fields[1]));
        }
      }
    }
    scan.setCaching(100);

    // Second argument is the table name.
    job.setOutputFormatClass(NullOutputFormat.class);
    TableMapReduceUtil.initTableMapperJob(
        tableName, scan, RowCounterMapper.class, ImmutableBytesWritable.class, Result.class, job);
    job.setNumReduceTasks(0);
    return job;
  }
  public static ResultScanner getList(
      String startRowRange,
      String stopRowRange,
      byte[] cf1,
      byte[] cf2,
      long limit,
      FilterList filterList,
      String ctableName) {

    Scan scan = new Scan();
    scan.addFamily(cf1);
    scan.setStartRow(Bytes.toBytes(startRowRange));

    if (stopRowRange != null) {
      scan.setStopRow(Bytes.toBytes(stopRowRange));
    }
    if (limit != 0) {

      filterList.addFilter(new PageFilter(limit));
    } else {
      filterList.addFilter(new PageFilter(100));
    }
    scan.setFilter(filterList);
    ResultScanner resultScanner = null;

    try {
      resultScanner = tblMngr.getTable(ctableName).getScanner(scan);

    } catch (Exception e) {

    }
    return resultScanner;
  }
  @Test
  public void shouldRunMapReduce() throws Exception {
    // given
    Job job = new Job(configuration, "Average Rating");
    job.setJarByClass(AverageRatingMapper.class);

    Scan scan = new Scan();
    scan.setCaching(500);
    scan.setCacheBlocks(false);
    scan.addFamily(Bytes.toBytes(LoadMovieRatingData.FAMILY_NAME));

    TableMapReduceUtil.initTableMapperJob(
        LoadMovieRatingData.TABLE_NAME,
        scan,
        AverageRatingMapper.class,
        Text.class,
        DoubleWritable.class,
        job);
    job.setReducerClass(RatingExportReducer.class);
    job.setNumReduceTasks(1);
    FileOutputFormat.setOutputPath(
        job, new Path("/tmp/mr/mySummaryFile_" + System.currentTimeMillis()));

    // when
    boolean succeeded = job.waitForCompletion(true);

    // then
    assertThat(succeeded).isTrue();
  }
Ejemplo n.º 13
0
  @Override
  protected List<RawResource> getAllResources(String rangeStart, String rangeEnd)
      throws IOException {
    byte[] startRow = Bytes.toBytes(rangeStart);
    byte[] endRow = plusZero(Bytes.toBytes(rangeEnd));

    Scan scan = new Scan(startRow, endRow);
    scan.addColumn(B_FAMILY, B_COLUMN_TS);
    scan.addColumn(B_FAMILY, B_COLUMN);

    HTableInterface table = getConnection().getTable(getAllInOneTableName());
    List<RawResource> result = Lists.newArrayList();
    try {
      ResultScanner scanner = table.getScanner(scan);
      for (Result r : scanner) {
        result.add(new RawResource(getInputStream(Bytes.toString(r.getRow()), r), getTimestamp(r)));
      }
    } catch (IOException e) {
      for (RawResource rawResource : result) {
        IOUtils.closeQuietly(rawResource.resource);
      }
      throw e;
    } finally {
      IOUtils.closeQuietly(table);
    }
    return result;
  }
Ejemplo n.º 14
0
  @Override
  protected ArrayList<String> listResourcesImpl(String resPath) throws IOException {
    assert resPath.startsWith("/");
    String lookForPrefix = resPath.endsWith("/") ? resPath : resPath + "/";
    byte[] startRow = Bytes.toBytes(lookForPrefix);
    byte[] endRow = Bytes.toBytes(lookForPrefix);
    endRow[endRow.length - 1]++;

    ArrayList<String> result = new ArrayList<String>();

    HTableInterface table = getConnection().getTable(getAllInOneTableName());
    Scan scan = new Scan(startRow, endRow);
    scan.setFilter(new KeyOnlyFilter());
    try {
      ResultScanner scanner = table.getScanner(scan);
      for (Result r : scanner) {
        String path = Bytes.toString(r.getRow());
        assert path.startsWith(lookForPrefix);
        int cut = path.indexOf('/', lookForPrefix.length());
        String child = cut < 0 ? path : path.substring(0, cut);
        if (result.contains(child) == false) result.add(child);
      }
    } finally {
      IOUtils.closeQuietly(table);
    }
    // return null to indicate not a folder
    return result.isEmpty() ? null : result;
  }
Ejemplo n.º 15
0
  /**
   * 遍历多行
   *
   * @param tableName 表名
   * @param start_rowkey 开始行键
   * @param stop_rowkey 结束行键
   * @return 行列表
   */
  public ArrayList<HbaseRow> scanRows(String tableName, String start_rowkey, String stop_rowkey) {
    ResultScanner rowstmp = null;
    ArrayList<HbaseRow> rows = null;

    try {
      Scan scan = new Scan();
      scan.setStartRow(Bytes.toBytes(start_rowkey));
      scan.setStopRow(Bytes.toBytes(stop_rowkey));
      HTable table = new HTable(conf, Bytes.toBytes(tableName));
      rowstmp = table.getScanner(scan);
      rows = new ArrayList<>();

      for (Result rowtmp : rowstmp) {
        HbaseRow row = new HbaseRow();
        row.rowkey = Bytes.toString(rowtmp.getRow());
        for (Cell cell : rowtmp.listCells()) {
          HbaseColumn col = new HbaseColumn(cell);
          row.cols.add(col);
        }
        rows.add(row);
      }
    } catch (Exception e) {
      logger.error("scanRows failed", e);
    } finally {
      rowstmp.close();
    }
    return rows;
  }
  @Test
  public void testFilterList() throws Exception {
    // Test getting a single row, single key using Row, Qualifier, and Value
    // regular expression and substring filters
    // Use must pass all
    List<Filter> filters = new ArrayList<Filter>();
    filters.add(new RowFilter(CompareOp.EQUAL, new RegexStringComparator(".+-2")));
    filters.add(new QualifierFilter(CompareOp.EQUAL, new RegexStringComparator(".+-2")));
    filters.add(new ValueFilter(CompareOp.EQUAL, new SubstringComparator("One")));
    Filter f = new FilterList(Operator.MUST_PASS_ALL, filters);
    Scan s = new Scan();
    s.addFamily(FAMILIES[0]);
    s.setFilter(f);
    KeyValue[] kvs = {new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0])};
    verifyScanFull(s, kvs);

    // Test getting everything with a MUST_PASS_ONE filter including row, qf,
    // val, regular expression and substring filters
    filters.clear();
    filters.add(new RowFilter(CompareOp.EQUAL, new RegexStringComparator(".+Two.+")));
    filters.add(new QualifierFilter(CompareOp.EQUAL, new RegexStringComparator(".+-2")));
    filters.add(new ValueFilter(CompareOp.EQUAL, new SubstringComparator("One")));
    f = new FilterList(Operator.MUST_PASS_ONE, filters);
    s = new Scan();
    s.setFilter(f);
    verifyScanNoEarlyOut(s, numRows, colsPerRow);
  }
Ejemplo n.º 17
0
 /* (non-Javadoc)
  * @see com.hazelcast.core.MapLoader#loadAllKeys()
  */
 @Override
 public Set<String> loadAllKeys() {
   Set<String> keySet = null;
   if (allowLoadAll) {
     keySet = new HashSet<String>();
     HTableInterface hti = null;
     try {
       hti = pool.getTable(tableName);
       Scan s = new Scan();
       s.addColumn(family, qualifier);
       ResultScanner rs = hti.getScanner(s);
       Result r = null;
       while ((r = rs.next()) != null) {
         String k = new String(r.getRow());
         keySet.add(k);
       }
     } catch (IOException e) {
       LOG.error("IOException while loading all keys", e);
     } finally {
       if (hti != null) {
         pool.putTable(hti);
       }
     }
   }
   return keySet;
 }
Ejemplo n.º 18
0
 /*
  * Add to each of the regions in .META. a value.  Key is the startrow of the
  * region (except its 'aaa' for first region).  Actual value is the row name.
  * @param expected
  * @return
  * @throws IOException
  */
 private static int addToEachStartKey(final int expected) throws IOException {
   HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
   HTable meta = new HTable(TEST_UTIL.getConfiguration(), HConstants.META_TABLE_NAME);
   int rows = 0;
   Scan scan = new Scan();
   scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
   ResultScanner s = meta.getScanner(scan);
   for (Result r = null; (r = s.next()) != null; ) {
     byte[] b = r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
     if (b == null || b.length <= 0) break;
     HRegionInfo hri = Writables.getHRegionInfo(b);
     // If start key, add 'aaa'.
     byte[] row = getStartKey(hri);
     Put p = new Put(row);
     p.setWriteToWAL(false);
     p.add(getTestFamily(), getTestQualifier(), row);
     t.put(p);
     rows++;
   }
   s.close();
   Assert.assertEquals(expected, rows);
   t.close();
   meta.close();
   return rows;
 }
Ejemplo n.º 19
0
      @Override
      public void process(long now, HRegion region, List<Mutation> mutations, WALEdit walEdit)
          throws IOException {
        // Scan current counter
        List<Cell> kvs = new ArrayList<Cell>();
        Scan scan = new Scan(row, row);
        scan.addColumn(FAM, COUNTER);
        doScan(region, scan, kvs);
        counter = kvs.size() == 0 ? 0 : Bytes.toInt(CellUtil.cloneValue(kvs.iterator().next()));

        // Assert counter value
        assertEquals(expectedCounter, counter);

        // Increment counter and send it to both memstore and wal edit
        counter += 1;
        expectedCounter += 1;

        Put p = new Put(row);
        KeyValue kv = new KeyValue(row, FAM, COUNTER, now, Bytes.toBytes(counter));
        p.add(kv);
        mutations.add(p);
        walEdit.add(kv);

        // We can also inject some meta data to the walEdit
        KeyValue metaKv =
            new KeyValue(
                row,
                WALEdit.METAFAMILY,
                Bytes.toBytes("I just increment counter"),
                Bytes.toBytes(counter));
        walEdit.add(metaKv);
      }
 public static void main(String[] args) throws Exception {
   Configuration con = new Configuration();
   String[] otherArgs = new GenericOptionsParser(con, args).getRemainingArgs();
   HBaseConfiguration conf = new HBaseConfiguration();
   Job job = new Job(conf, "AverageCalc");
   job.setJarByClass(AverageCalculator.class);
   Scan scan = new Scan();
   scan.setCaching(500);
   scan.setCacheBlocks(false);
   scan.addFamily(Bytes.toBytes("Post"));
   FilterList li = new FilterList(FilterList.Operator.MUST_PASS_ALL);
   SingleColumnValueFilter filter =
       new SingleColumnValueFilter(
           Bytes.toBytes("Post"),
           Bytes.toBytes("PostTypeId"),
           CompareOp.EQUAL,
           Bytes.toBytes("1"));
   li.addFilter(filter);
   scan.setFilter(li);
   FileOutputFormat.setOutputPath(job, new Path(otherArgs[0]));
   job.setOutputKeyClass(Text.class);
   TableMapReduceUtil.initTableMapperJob(
       "bigd24-hbase-sample", scan, Mapper1.class, Text.class, IntWritable.class, job);
   job.setReducerClass(Reducer1.class);
   job.setOutputValueClass(FloatWritable.class);
   System.exit(job.waitForCompletion(true) ? 0 : 1);
 }
  /** Job configuration. */
  public static Job configureJob(Configuration conf, String[] args) throws IOException {
    String tableName = args[0];
    String columnFamily = args[1];
    String outputPath = args[2];
    String rowKeyType = args[3];
    conf.set("row.key.type", rowKeyType);
    conf.set("table.name", tableName);
    Scan scan = new Scan();
    scan.addFamily(Bytes.toBytes(columnFamily));
    scan.setBatch(ConstantsTruthy.TRUTHY_TABLE_SCAN_BATCH);

    conf.set("mapred.map.tasks.speculative.execution", "false");
    conf.set("mapred.reduce.tasks.speculative.execution", "false");
    Job job =
        Job.getInstance(
            conf, "Count the column count and indexRecordSize for each row in " + tableName);
    job.setJarByClass(TruthyIndexFeatureCounter.class);
    TableMapReduceUtil.initTableMapperJob(
        tableName, scan, TfcMapper.class, Text.class, Text.class, job, true);
    job.setNumReduceTasks(0);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);
    FileOutputFormat.setOutputPath(job, new Path(outputPath));
    TableMapReduceUtil.addDependencyJars(job);
    return job;
  }
Ejemplo n.º 22
0
  @TimeDepend
  @Test
  public void testScan_ts_same() throws Exception {

    recreateTable();

    Date ts = parse("2000-01-01", "yyyy-MM-dd");

    Put put = new Put(rowKey_ForTest);

    put.add(ColumnFamilyNameBytes, QName1, ts.getTime(), Bytes.toBytes("a"));

    table.put(put);

    Set<String> resultRowKeys = new HashSet<String>();
    Scan scan = new Scan(rowKey_ForTest, rowKey_ForTest);
    scan.setTimeRange(ts.getTime(), ts.getTime());

    ResultScanner resultScanner = table.getScanner(scan);
    for (Result result = resultScanner.next(); result != null; result = resultScanner.next()) {
      resultRowKeys.add(Bytes.toString(result.getRow()));
    }

    close(resultScanner);

    Assert.assertTrue(resultRowKeys.size() == 0);

    recreateTable();
  }
  /*
   * (non-Javadoc)
   *
   * @see
   * com.impetus.kunderahbase.dao.user.UserDao#findUserByUserName(java.lang
   * .String, boolean, java.util.List)
   */
  @Override
  public void findUserByUserName(String userName, boolean isBulk, List<UserHBaseDTO> users) {
    for (UserHBaseDTO user : users) {
      Filter filter =
          new SingleColumnValueFilter(
              Bytes.toBytes("user_name"),
              Bytes.toBytes("user_name"),
              CompareOp.EQUAL,
              Bytes.toBytes(user.getUserNameCounter()));
      Scan scan = new Scan();
      scan.setFilter(filter);
      scan.addColumn(Bytes.toBytes("user_name"), Bytes.toBytes("user_name"));
      scan.addColumn(Bytes.toBytes("user_nameCnt"), Bytes.toBytes("user_nameCnt"));
      scan.addColumn(Bytes.toBytes("password"), Bytes.toBytes("password"));
      scan.addColumn(Bytes.toBytes("relation"), Bytes.toBytes("relation"));

      ResultScanner scanner = null;
      int counter = 0;
      try {
        scanner = hTablePool.getTable("User").getScanner(scan);
      } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      }
      assert scanner != null;
      for (Result result : scanner) {
        counter++;
        assert result != null;
      }
      assert counter != 0;
    }
  }
 private void runTestOnTable(Table table)
     throws IOException, InterruptedException, ClassNotFoundException {
   Job job = null;
   try {
     LOG.info("Before map/reduce startup");
     job = new Job(table.getConfiguration(), "process column contents");
     job.setNumReduceTasks(1);
     Scan scan = new Scan();
     scan.addFamily(INPUT_FAMILY);
     TableMapReduceUtil.initTableMapperJob(
         table.getName(),
         scan,
         MultithreadedTableMapper.class,
         ImmutableBytesWritable.class,
         Put.class,
         job);
     MultithreadedTableMapper.setMapperClass(job, ProcessContentsMapper.class);
     MultithreadedTableMapper.setNumberOfThreads(job, NUMBER_OF_THREADS);
     TableMapReduceUtil.initTableReducerJob(
         table.getName().getNameAsString(), IdentityTableReducer.class, job);
     FileOutputFormat.setOutputPath(job, new Path("test"));
     LOG.info("Started " + table.getName());
     assertTrue(job.waitForCompletion(true));
     LOG.info("After map/reduce completion");
     // verify map-reduce results
     verify(table.getName());
   } finally {
     table.close();
     if (job != null) {
       FileUtil.fullyDelete(new File(job.getConfiguration().get("hadoop.tmp.dir")));
     }
   }
 }
    public static List<Delete> GetDeleteEventsBetween(
        Table VTEvent_Table, String imo_str, long first_timestamp, long last_timestamp)
        throws IOException {
      // scan
      // 'cdb_vessel:vessel_event',{FILTER=>"(PrefixFilter('0000003162')"}
      Scan GetEventsBetween = new Scan();
      GetEventsBetween.setStartRow(
              Bytes.toBytes(imo_str + LpadNum(Long.MAX_VALUE - last_timestamp, 19) + "0000000000"))
          .setStopRow(
              Bytes.toBytes(
                  imo_str + LpadNum(Long.MAX_VALUE - first_timestamp + 1, 19) + "9999999999"))
          .addColumn(details, exittime);
      GetEventsBetween.setCaching(100);

      Filter ExistTimeValuefilter =
          new ValueFilter(
              CompareFilter.CompareOp.LESS_OR_EQUAL,
              new BinaryComparator(
                  Bytes.toBytes(new DateTime(last_timestamp).toString(rawformatter))));
      GetEventsBetween.setFilter(ExistTimeValuefilter);

      ResultScanner Result_ExistingEvents = VTEvent_Table.getScanner(GetEventsBetween);
      List<Delete> deletes = new ArrayList<Delete>();

      for (Result res : Result_ExistingEvents) {
        deletes.add(new Delete(res.getRow()));
      }

      Result_ExistingEvents.close();
      return deletes;
    }
Ejemplo n.º 26
0
 private static void projectAllColumnFamilies(PTable table, Scan scan) {
   // Will project all known/declared column families
   scan.getFamilyMap().clear();
   for (PColumnFamily family : table.getColumnFamilies()) {
     scan.addFamily(family.getName().getBytes());
   }
 }
Ejemplo n.º 27
0
  public String[] getTags(String objectType, String objectId) throws IOException {
    List<String> ret = new ArrayList<String>();
    String rowKey = objectType + "_" + objectId;
    Scan s = new Scan(rowKey.getBytes(), rowKey.getBytes());
    s.setMaxVersions(1);
    ResultScanner scanner = htable.getScanner(s);
    try {
      for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {

        String localObjectType =
            new String(rr.getValue("tags".getBytes(), "OBJECTTYPE".getBytes()));
        String localObjectId = new String(rr.getValue("tags".getBytes(), "OBJECTID".getBytes()));
        NavigableMap<byte[], byte[]> map = rr.getFamilyMap("tags".getBytes());
        Iterator<Entry<byte[], byte[]>> it = map.entrySet().iterator();
        while (it.hasNext()) {
          Entry<byte[], byte[]> entry = it.next();
          String key = new String(entry.getKey());
          if (!key.startsWith("OBJECT")) {
            int val = Bytes.toInt(entry.getValue());
            if (val > 0) ret.add(key);
          }
        }
      }
    } finally {
      scanner.close();
    }
    return ret.toArray(new String[] {});
  }
  public static void main(String args[]) {
    if (args.length == 0) {
      System.out.println("JavaHBaseDistributedScan  {master} {tableName}");
    }

    String master = args[0];
    String tableName = args[1];

    JavaSparkContext jsc = new JavaSparkContext(master, "JavaHBaseDistributedScan");
    jsc.addJar("SparkHBase.jar");

    Configuration conf = HBaseConfiguration.create();
    conf.addResource(new Path("/etc/hbase/conf/core-site.xml"));
    conf.addResource(new Path("/etc/hbase/conf/hbase-site.xml"));

    JavaHBaseContext hbaseContext = new JavaHBaseContext(jsc, conf);

    Scan scan = new Scan();
    scan.setCaching(100);

    JavaRDD<Tuple2<byte[], List<Tuple3<byte[], byte[], byte[]>>>> javaRdd =
        hbaseContext.hbaseRDD(tableName, scan);

    List<Tuple2<byte[], List<Tuple3<byte[], byte[], byte[]>>>> results = javaRdd.collect();

    results.size();
  }
Ejemplo n.º 29
0
  public Blog(String blogid) throws IOException {

    Configuration conf = HBaseConfiguration.create();
    table = new HTable(conf, "blogs");

    // 1. Get the row whose row key is blogid from above
    Get g = new Get(Bytes.toBytes(blogid));
    Result r = table.get(g);

    // 2. Extract the rowkey, blog text (column "body") and blog title
    // (column "meta:title")
    key = r.getRow();
    keyStr = Bytes.toString(key);
    blogText = Bytes.toString(r.getValue(Bytes.toBytes("body"), Bytes.toBytes("")));
    blogTitle = Bytes.toString(r.getValue(Bytes.toBytes("meta"), Bytes.toBytes("title")));
    Long reverseTimestamp = Long.parseLong(keyStr.substring(4));
    Long epoch = Math.abs(reverseTimestamp - Long.MAX_VALUE);
    dateOfPost = new Date(epoch);

    // Get an iterator for the comments
    Scan s = new Scan();
    s.addFamily(Bytes.toBytes("comment"));
    // Use a PrefixFilter
    PrefixFilter filter = new PrefixFilter(key);
    s.setFilter(filter);
    scanner = table.getScanner(s);
    resultIterator = scanner.iterator();
  }
Ejemplo n.º 30
0
 private Scan createScan(final String columnFamily, final String[] headers) {
   Scan scan = new Scan();
   for (String header : headers) {
     scan.addColumn(Bytes.toBytes(columnFamily), Bytes.toBytes(header));
   }
   return scan;
 }