public String[] getObjectIDs(String objectType, String... tags) throws IOException { List<String> ret = new ArrayList<String>(); FilterList list = new FilterList(FilterList.Operator.MUST_PASS_ALL); SingleColumnValueFilter filter1 = new SingleColumnValueFilter( "tags".getBytes(), "OBJECTTYPE".getBytes(), CompareOp.EQUAL, Bytes.toBytes(objectType)); list.addFilter(filter1); for (String tag : tags) { SingleColumnValueFilter filter2 = new SingleColumnValueFilter( "tags".getBytes(), tag.toUpperCase().getBytes(), CompareOp.EQUAL, Bytes.toBytes(1)); filter2.setFilterIfMissing(true); list.addFilter(filter2); } Scan s = new Scan(); s.setFilter(list); s.setMaxVersions(1); ResultScanner scanner = htable.getScanner(s); try { for (Result rr = scanner.next(); rr != null; rr = scanner.next()) { String localObjectType = new String(rr.getValue("tags".getBytes(), "OBJECTTYPE".getBytes())); String localObjectId = new String(rr.getValue("tags".getBytes(), "OBJECTID".getBytes())); ret.add(localObjectId); } } finally { scanner.close(); } return ret.toArray(new String[] {}); }
public Where<? extends QueryOps<ROW_ID_TYPE>, ROW_ID_TYPE> match(String regexPattern) { RegexStringComparator regexStringComparator = new RegexStringComparator(regexPattern); SingleColumnValueFilter singleColumnValueFilter = new SingleColumnValueFilter(family, qualifier, CompareOp.EQUAL, regexStringComparator); singleColumnValueFilter.setFilterIfMissing(true); return whereScanner.addFilter(singleColumnValueFilter); }
private Filter getSingleValueEqualFilter( byte[] value, CompareOp compareOp, boolean filterIfMissing) { SingleColumnValueFilter singleColumnValueFilter = new SingleColumnValueFilter(family, qualifier, compareOp, value); singleColumnValueFilter.setFilterIfMissing(filterIfMissing); return singleColumnValueFilter; }
public <U> Where<? extends QueryOps<ROW_ID_TYPE>, ROW_ID_TYPE> contains(String pattern) { SubstringComparator comparator = new SubstringComparator(pattern); SingleColumnValueFilter singleColumnValueFilter = new SingleColumnValueFilter(family, qualifier, CompareOp.EQUAL, comparator); singleColumnValueFilter.setFilterIfMissing(true); return whereScanner.addFilter(singleColumnValueFilter); }
private static Filter getUserTableAndViewsFilter() { SingleColumnValueFilter tableFilter = new SingleColumnValueFilter( TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.TABLE_TYPE_BYTES, CompareOp.EQUAL, Bytes.toBytes(PTableType.TABLE.getSerializedValue())); tableFilter.setFilterIfMissing(true); SingleColumnValueFilter viewFilter = new SingleColumnValueFilter( TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.TABLE_TYPE_BYTES, CompareOp.EQUAL, Bytes.toBytes(PTableType.VIEW.getSerializedValue())); viewFilter.setFilterIfMissing(true); FilterList filter = new FilterList( FilterList.Operator.MUST_PASS_ONE, Arrays.asList(new Filter[] {tableFilter, viewFilter})); return filter; }
/** * Only include rows which are missing this field, this was the only possible way to do it. * * @param fieldName The field which should be missing * @return ScannerBuilder */ public EntityScannerBuilder<E> addIsMissingFilter(String fieldName) { SingleFieldEntityFilter singleFieldEntityFilter = new SingleFieldEntityFilter( entityMapper.getEntitySchema(), entityMapper.getEntitySerDe(), fieldName, "++++NON_SHALL_PASS++++", CompareFilter.CompareOp.EQUAL); SingleColumnValueFilter filter = (SingleColumnValueFilter) singleFieldEntityFilter.getFilter(); filter.setFilterIfMissing(false); filterList.add(filter); return this; }
static { // A record is a real row iff the deleted flag exists and is not true. // It is possible for the delete flag not to exist on a row: this is // in case a lock was taken on a not-yet-existing row and the record // creation failed. Therefore, the filterIfMissing is important. REAL_RECORDS_FILTER = new SingleColumnValueFilter( RecordCf.DATA.bytes, RecordColumn.DELETED.bytes, CompareFilter.CompareOp.NOT_EQUAL, Bytes.toBytes(true)); REAL_RECORDS_FILTER.setFilterIfMissing(true); }
public static void main(String[] args) throws IOException { Configuration conf = HBaseConfiguration.create(); HBaseHelper helper = HBaseHelper.getHelper(conf); helper.dropTable("testtable"); helper.createTable("testtable", "colfam1", "colfam2"); System.out.println("Adding rows to table..."); helper.fillTable("testtable", 1, 10, 10, "colfam1", "colfam2"); Connection connection = ConnectionFactory.createConnection(conf); Table table = connection.getTable(TableName.valueOf("testtable")); // vv SingleColumnValueFilterExample SingleColumnValueFilter filter = new SingleColumnValueFilter( Bytes.toBytes("colfam1"), Bytes.toBytes("col-5"), CompareFilter.CompareOp.NOT_EQUAL, new SubstringComparator("val-5")); filter.setFilterIfMissing(true); Scan scan = new Scan(); scan.setFilter(filter); ResultScanner scanner = table.getScanner(scan); // ^^ SingleColumnValueFilterExample System.out.println("Results of scan:"); // vv SingleColumnValueFilterExample for (Result result : scanner) { for (Cell cell : result.rawCells()) { System.out.println( "Cell: " + cell + ", Value: " + Bytes.toString( cell.getValueArray(), cell.getValueOffset(), cell.getValueLength())); } } scanner.close(); Get get = new Get(Bytes.toBytes("row-6")); get.setFilter(filter); Result result = table.get(get); System.out.println("Result of get: "); for (Cell cell : result.rawCells()) { System.out.println( "Cell: " + cell + ", Value: " + Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength())); } // ^^ SingleColumnValueFilterExample }
// select data // http://blog.csdn.net/cnweike/article/details/42920547 for more detail. public static String select( String tableName, String begin, String end, String colFamily, String column, String value) { String result = ""; TableName tn = TableName.valueOf(tableName); try { Table table = conn.getTable(tn); Scan scan = new Scan(Bytes.toBytes(begin), Bytes.toBytes(end)); SingleColumnValueFilter scvf = new SingleColumnValueFilter( Bytes.toBytes(colFamily), Bytes.toBytes(column), CompareFilter.CompareOp.EQUAL, new SubstringComparator(value)); scvf.setFilterIfMissing(false); scvf.setLatestVersionOnly(true); // OK scan.setFilter(scvf); ResultScanner scanner = table.getScanner(scan); List<Map<String, String>> total = new ArrayList<Map<String, String>>(); for (Result res : scanner) { Map<String, String> map = new HashMap<String, String>(); for (Cell cell : res.rawCells()) map.put( Bytes.toString(CellUtil.cloneQualifier(cell)), Bytes.toString(CellUtil.cloneValue(cell))); total.add(map); } ObjectMapper mapper = new ObjectMapper(); result = mapper.writeValueAsString(total); } catch (IOException e) { e.printStackTrace(); } return result; }
@Override public Scan getRawScan(ItemData d, Map<String, HCompareOp> ops) { int startShopID = 0; int startItemID = 0; int endShopID = MAX_SHOP_ID; int endItemID = MAX_ITEM_ID; // some performance improvement // shop_id指定 HCompareOp shopIDOp = ops.get("shop_id"); if (shopIDOp == HCompareOp.EQUAL) { startShopID = d.shopID; endShopID = startShopID; } // item idも指定 HCompareOp itemIDOp = ops.get("item_id"); if (itemIDOp == HCompareOp.EQUAL) { startItemID = d.itemID; endItemID = startItemID; } log.info(String.format("scan start row, shop_id=%d, item_id=%d", startShopID, startItemID)); log.info(String.format("scan stop row, shop_id=%d, item_id=%d", endShopID, endItemID)); byte[] startRow = encodeRowkey(startShopID, startItemID); byte[] endRow = encodeRowkey(endShopID, endItemID); Scan s = new Scan(startRow, endRow); s.addFamily(DATA_FAMILY); s.addFamily(META_FAMILY); s.setCacheBlocks(false); s.setMaxVersions(); s.setCaching(DEFAULT_SCAN_CACHE); FilterList fl = new FilterList(); for (String column : ops.keySet()) { byte[] value; byte[] family = DATA_FAMILY; if ("ctime".equals(column)) { value = Bytes.toBytes(d.ctime); family = META_FAMILY; } else if ("shop_id".equals(column)) { value = Bytes.toBytes(d.shopID); } else if ("item_id".equals(column)) { value = Bytes.toBytes(d.itemID); } else if ("genre_id".equals(column)) { value = Bytes.toBytes(d.genreID); } else if ("price".equals(column)) { value = Bytes.toBytes(d.price); } else if ("full_item_url".equals(column)) { value = Bytes.toBytes(d.fullItemUrl); } else if ("item_name".equals(column)) { value = Bytes.toBytes(d.itemName); } else { // ignore continue; } byte[] qualifier = Bytes.toBytes(column); HCompareOp hop = ops.get(column); CompareOp op = HClient.toCompareOp(hop); SingleColumnValueFilter filter = new SingleColumnValueFilter(family, qualifier, op, value); filter.setFilterIfMissing(true); fl.addFilter(filter); } s.setFilter(fl); return s; }
/* * Must be sure that column is in format like 'family: column' */ public void prepare(String[] columns) throws IOException { this.scan = new Scan(); if (this.startKey != null) { logger.info(String.format("HBaseReader set startkey to %s .", Bytes.toString(this.startKey))); scan.setStartRow(startKey); } if (this.endKey != null) { logger.info(String.format("HBaseReader set endkey to %s .", Bytes.toString(this.endKey))); scan.setStopRow(endKey); } if (this.minStamp > 0 && this.maxStamp > 0) { scan.setTimeRange(minStamp, maxStamp); if (StringUtils.isNotEmpty(primaryKey)) { String[] cs = StringUtils.split(primaryKey, ":"); scan.addColumn(cs[0].getBytes(), cs[1].getBytes()); } } else if (ETLStringUtils.isNotEmpty(startDate) && ETLStringUtils.isNotEmpty(familyName) && ETLStringUtils.isNotEmpty(fieldName)) { SingleColumnValueFilter scvf = new SingleColumnValueFilter( Bytes.toBytes(familyName), Bytes.toBytes(fieldName), CompareOp.GREATER_OR_EQUAL, new BinaryPrefixComparator(startDate.getBytes())); scvf.setFilterIfMissing(true); scvf.setLatestVersionOnly(true); if (ETLStringUtils.isNotEmpty(endDate)) { SingleColumnValueFilter scvf1 = new SingleColumnValueFilter( Bytes.toBytes(familyName), Bytes.toBytes(fieldName), CompareOp.LESS_OR_EQUAL, new BinaryPrefixComparator(endDate.getBytes())); scvf1.setFilterIfMissing(true); scvf1.setLatestVersionOnly(true); List<Filter> filters = new ArrayList<Filter>(); filters.add(scvf); filters.add(scvf1); FilterList fl = new FilterList(FilterList.Operator.MUST_PASS_ALL, filters); scan.setFilter(fl); } else { scan.setFilter(scvf); } } this.families = new String[columns.length]; this.columns = new String[columns.length]; int idx = 0; for (String column : columns) { this.families[idx] = column.split(":")[0].trim(); this.columns[idx] = column.split(":")[1].trim(); if (!(this.minStamp > 0 && this.maxStamp > 0)) { scan.addColumn(this.families[idx].getBytes(), this.columns[idx].getBytes()); } idx++; } if ("true".equalsIgnoreCase(this.isSaveOneObj)) { scan.addColumn(this.oneObjFamilyName.getBytes(), this.oneObjColName.getBytes()); } if (this.maxversion > 0) { scan.setMaxVersions(this.maxversion); } else { scan.setMaxVersions(); } htable.setScannerCaching(SCAN_CACHE); this.rs = htable.getScanner(this.scan); }
public Filter build() { Filter filter; switch (FilterType.valueOf(type)) { case ColumnCountGetFilter: filter = new ColumnCountGetFilter(limit); break; case ColumnPaginationFilter: filter = new ColumnPaginationFilter(limit, offset); break; case ColumnPrefixFilter: filter = new ColumnPrefixFilter(Base64.decode(value)); break; case ColumnRangeFilter: filter = new ColumnRangeFilter( Base64.decode(minColumn), minColumnInclusive, Base64.decode(maxColumn), maxColumnInclusive); break; case DependentColumnFilter: filter = new DependentColumnFilter( Base64.decode(family), qualifier != null ? Base64.decode(qualifier) : null, dropDependentColumn, CompareOp.valueOf(op), comparator.build()); break; case FamilyFilter: filter = new FamilyFilter(CompareOp.valueOf(op), comparator.build()); break; case FilterList: { List<Filter> list = new ArrayList<Filter>(); for (FilterModel model : filters) { list.add(model.build()); } filter = new FilterList(FilterList.Operator.valueOf(op), list); } break; case FirstKeyOnlyFilter: filter = new FirstKeyOnlyFilter(); break; case InclusiveStopFilter: filter = new InclusiveStopFilter(Base64.decode(value)); break; case KeyOnlyFilter: filter = new KeyOnlyFilter(); break; case MultipleColumnPrefixFilter: { byte[][] values = new byte[prefixes.size()][]; for (int i = 0; i < prefixes.size(); i++) { values[i] = Base64.decode(prefixes.get(i)); } filter = new MultipleColumnPrefixFilter(values); } break; case PageFilter: filter = new PageFilter(Long.valueOf(value)); break; case PrefixFilter: filter = new PrefixFilter(Base64.decode(value)); break; case QualifierFilter: filter = new QualifierFilter(CompareOp.valueOf(op), comparator.build()); break; case RandomRowFilter: filter = new RandomRowFilter(chance); break; case RowFilter: filter = new RowFilter(CompareOp.valueOf(op), comparator.build()); break; case SingleColumnValueFilter: filter = new SingleColumnValueFilter( Base64.decode(family), qualifier != null ? Base64.decode(qualifier) : null, CompareOp.valueOf(op), comparator.build()); if (ifMissing != null) { ((SingleColumnValueFilter) filter).setFilterIfMissing(ifMissing); } if (latestVersion != null) { ((SingleColumnValueFilter) filter).setLatestVersionOnly(latestVersion); } break; case SingleColumnValueExcludeFilter: filter = new SingleColumnValueExcludeFilter( Base64.decode(family), qualifier != null ? Base64.decode(qualifier) : null, CompareOp.valueOf(op), comparator.build()); if (ifMissing != null) { ((SingleColumnValueExcludeFilter) filter).setFilterIfMissing(ifMissing); } if (latestVersion != null) { ((SingleColumnValueExcludeFilter) filter).setLatestVersionOnly(latestVersion); } break; case SkipFilter: filter = new SkipFilter(filters.get(0).build()); break; case TimestampsFilter: filter = new TimestampsFilter(timestamps); break; case ValueFilter: filter = new ValueFilter(CompareOp.valueOf(op), comparator.build()); break; case WhileMatchFilter: filter = new WhileMatchFilter(filters.get(0).build()); break; default: throw new RuntimeException("unhandled filter type: " + type); } return filter; }