public <U> Where<? extends QueryOps<ROW_ID_TYPE>, ROW_ID_TYPE> contains(String pattern) {
SubstringComparator comparator = new SubstringComparator(pattern);
SingleColumnValueFilter singleColumnValueFilter =
new SingleColumnValueFilter(family, qualifier, CompareOp.EQUAL, comparator);
singleColumnValueFilter.setFilterIfMissing(true);
return whereScanner.addFilter(singleColumnValueFilter);
}
public String[] getObjectIDs(String objectType, String... tags) throws IOException {
List<String> ret = new ArrayList<String>();
FilterList list = new FilterList(FilterList.Operator.MUST_PASS_ALL);
SingleColumnValueFilter filter1 =
new SingleColumnValueFilter(
"tags".getBytes(), "OBJECTTYPE".getBytes(), CompareOp.EQUAL, Bytes.toBytes(objectType));
list.addFilter(filter1);
for (String tag : tags) {
SingleColumnValueFilter filter2 =
new SingleColumnValueFilter(
"tags".getBytes(), tag.toUpperCase().getBytes(), CompareOp.EQUAL, Bytes.toBytes(1));
filter2.setFilterIfMissing(true);
list.addFilter(filter2);
}
Scan s = new Scan();
s.setFilter(list);
s.setMaxVersions(1);
ResultScanner scanner = htable.getScanner(s);
try {
for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
String localObjectType =
new String(rr.getValue("tags".getBytes(), "OBJECTTYPE".getBytes()));
String localObjectId = new String(rr.getValue("tags".getBytes(), "OBJECTID".getBytes()));
ret.add(localObjectId);
}
} finally {
scanner.close();
}
return ret.toArray(new String[] {});
}
private Filter getSingleValueEqualFilter(
byte[] value, CompareOp compareOp, boolean filterIfMissing) {
SingleColumnValueFilter singleColumnValueFilter =
new SingleColumnValueFilter(family, qualifier, compareOp, value);
singleColumnValueFilter.setFilterIfMissing(filterIfMissing);
return singleColumnValueFilter;
}
public Where<? extends QueryOps<ROW_ID_TYPE>, ROW_ID_TYPE> match(String regexPattern) {
RegexStringComparator regexStringComparator = new RegexStringComparator(regexPattern);
SingleColumnValueFilter singleColumnValueFilter =
new SingleColumnValueFilter(family, qualifier, CompareOp.EQUAL, regexStringComparator);
singleColumnValueFilter.setFilterIfMissing(true);
return whereScanner.addFilter(singleColumnValueFilter);
}
/**
* Only include rows which are missing this field, this was the only possible way to do it.
*
* @param fieldName The field which should be missing
* @return ScannerBuilder
*/
public EntityScannerBuilder<E> addIsMissingFilter(String fieldName) {
SingleFieldEntityFilter singleFieldEntityFilter =
new SingleFieldEntityFilter(
entityMapper.getEntitySchema(),
entityMapper.getEntitySerDe(),
fieldName,
"++++NON_SHALL_PASS++++",
CompareFilter.CompareOp.EQUAL);
SingleColumnValueFilter filter = (SingleColumnValueFilter) singleFieldEntityFilter.getFilter();
filter.setFilterIfMissing(false);
filterList.add(filter);
return this;
}
public static void main(String[] args) throws IOException {
Configuration conf = HBaseConfiguration.create();
HBaseHelper helper = HBaseHelper.getHelper(conf);
helper.dropTable("testtable");
helper.createTable("testtable", "colfam1", "colfam2");
System.out.println("Adding rows to table...");
helper.fillTable("testtable", 1, 10, 10, "colfam1", "colfam2");
Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(TableName.valueOf("testtable"));
// vv SingleColumnValueFilterExample
SingleColumnValueFilter filter =
new SingleColumnValueFilter(
Bytes.toBytes("colfam1"),
Bytes.toBytes("col-5"),
CompareFilter.CompareOp.NOT_EQUAL,
new SubstringComparator("val-5"));
filter.setFilterIfMissing(true);
Scan scan = new Scan();
scan.setFilter(filter);
ResultScanner scanner = table.getScanner(scan);
// ^^ SingleColumnValueFilterExample
System.out.println("Results of scan:");
// vv SingleColumnValueFilterExample
for (Result result : scanner) {
for (Cell cell : result.rawCells()) {
System.out.println(
"Cell: "
+ cell
+ ", Value: "
+ Bytes.toString(
cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
}
}
scanner.close();
Get get = new Get(Bytes.toBytes("row-6"));
get.setFilter(filter);
Result result = table.get(get);
System.out.println("Result of get: ");
for (Cell cell : result.rawCells()) {
System.out.println(
"Cell: "
+ cell
+ ", Value: "
+ Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength()));
}
// ^^ SingleColumnValueFilterExample
}
/** Create a filter that will match a given family, qualifier, and cells per qualifier. */
private RowFilter createColumnSpecFilter(SingleColumnValueFilter filter) throws IOException {
return RowFilter.newBuilder()
.setChain(
Chain.newBuilder()
.addFilters(
RowFilter.newBuilder()
.setFamilyNameRegexFilter(
Bytes.toString(quoteRegularExpression(filter.getFamily()))))
.addFilters(
RowFilter.newBuilder()
.setColumnQualifierRegexFilter(
ByteStringer.wrap(quoteRegularExpression(filter.getQualifier()))))
.addFilters(createVersionLimitFilter(filter)))
.build();
}
private static Filter getUserTableAndViewsFilter() {
SingleColumnValueFilter tableFilter =
new SingleColumnValueFilter(
TABLE_FAMILY_BYTES,
PhoenixDatabaseMetaData.TABLE_TYPE_BYTES,
CompareOp.EQUAL,
Bytes.toBytes(PTableType.TABLE.getSerializedValue()));
tableFilter.setFilterIfMissing(true);
SingleColumnValueFilter viewFilter =
new SingleColumnValueFilter(
TABLE_FAMILY_BYTES,
PhoenixDatabaseMetaData.TABLE_TYPE_BYTES,
CompareOp.EQUAL,
Bytes.toBytes(PTableType.VIEW.getSerializedValue()));
viewFilter.setFilterIfMissing(true);
FilterList filter =
new FilterList(
FilterList.Operator.MUST_PASS_ONE,
Arrays.asList(new Filter[] {tableFilter, viewFilter}));
return filter;
}
static {
// A record is a real row iff the deleted flag exists and is not true.
// It is possible for the delete flag not to exist on a row: this is
// in case a lock was taken on a not-yet-existing row and the record
// creation failed. Therefore, the filterIfMissing is important.
REAL_RECORDS_FILTER =
new SingleColumnValueFilter(
RecordCf.DATA.bytes,
RecordColumn.DELETED.bytes,
CompareFilter.CompareOp.NOT_EQUAL,
Bytes.toBytes(true));
REAL_RECORDS_FILTER.setFilterIfMissing(true);
}
// select data
// http://blog.csdn.net/cnweike/article/details/42920547 for more detail.
public static String select(
String tableName, String begin, String end, String colFamily, String column, String value) {
String result = "";
TableName tn = TableName.valueOf(tableName);
try {
Table table = conn.getTable(tn);
Scan scan = new Scan(Bytes.toBytes(begin), Bytes.toBytes(end));
SingleColumnValueFilter scvf =
new SingleColumnValueFilter(
Bytes.toBytes(colFamily),
Bytes.toBytes(column),
CompareFilter.CompareOp.EQUAL,
new SubstringComparator(value));
scvf.setFilterIfMissing(false);
scvf.setLatestVersionOnly(true); // OK
scan.setFilter(scvf);
ResultScanner scanner = table.getScanner(scan);
List<Map<String, String>> total = new ArrayList<Map<String, String>>();
for (Result res : scanner) {
Map<String, String> map = new HashMap<String, String>();
for (Cell cell : res.rawCells())
map.put(
Bytes.toString(CellUtil.cloneQualifier(cell)),
Bytes.toString(CellUtil.cloneValue(cell)));
total.add(map);
}
ObjectMapper mapper = new ObjectMapper();
result = mapper.writeValueAsString(total);
} catch (IOException e) {
e.printStackTrace();
}
return result;
}
@Override
public RowFilter adapt(FilterAdapterContext context, SingleColumnValueFilter filter)
throws IOException {
if (filter.getFilterIfMissing()) {
return createEmitRowsWithValueFilter(context, filter);
} else {
return RowFilter.newBuilder()
.setCondition(
Condition.newBuilder()
.setPredicateFilter(createColumnSpecFilter(filter))
.setTrueFilter(createEmitRowsWithValueFilter(context, filter))
.setFalseFilter(ALL_VALUES_FILTER))
.build();
}
}
private void handleScvf(SingleColumnValueFilter scvf) {
ValuePartition vp = null;
if (scvf instanceof SingleColumnValuePartitionFilter) {
vp = ((SingleColumnValuePartitionFilter) scvf).getValuePartition();
}
Column column = new Column(scvf.getFamily(), scvf.getQualifier(), vp);
Pair<Value, Value> pair = colWithOperators.get(column);
if (pair == null) {
pair = new Pair<Value, Value>();
// The first operator should be set here
pair.setFirst(new Value(scvf.getOperator(), scvf.getComparator().getValue(), scvf));
colWithOperators.put(column, pair);
} else {
if (pair.getFirst() != null && pair.getSecond() == null) {
// TODO As Anoop said we may have to check the Value type also..
// We can not compare and validate this way. btw "a" and "K".
// Only in case of Numeric col type we can have this check.
byte[] curBoundValue = scvf.getComparator().getValue();
byte[] prevBoundValue = pair.getFirst().getValue();
int result = Bytes.compareTo(prevBoundValue, curBoundValue);
CompareOp curBoundOperator = scvf.getOperator();
CompareOp prevBoundOperator = pair.getFirst().getOperator();
switch (curBoundOperator) {
case GREATER:
case GREATER_OR_EQUAL:
if (prevBoundOperator == CompareOp.GREATER
|| prevBoundOperator == CompareOp.GREATER_OR_EQUAL) {
LOG.warn("Wrong usage. It should be < > || > <. Cannot be > >");
if (result > 1) {
pair.setFirst(new Value(curBoundOperator, curBoundValue, scvf));
}
pair.setSecond(null);
} else if (prevBoundOperator == CompareOp.LESS
|| prevBoundOperator == CompareOp.LESS_OR_EQUAL) {
if (result < 1) {
LOG.warn("Possible wrong usage as there cannot be a value < 10 and > 20");
pair.setFirst(null);
pair.setSecond(null);
} else {
pair.setSecond(new Value(curBoundOperator, curBoundValue, scvf));
}
} else if (prevBoundOperator == CompareOp.EQUAL) {
LOG.warn("Use the equal operator and ignore the current one");
pair.setSecond(null);
}
break;
case LESS:
case LESS_OR_EQUAL:
if (prevBoundOperator == CompareOp.LESS
|| prevBoundOperator == CompareOp.LESS_OR_EQUAL) {
LOG.warn("Wrong usage. It should be < > || > <. Cannot be > >");
if (result < 1) {
pair.setFirst(new Value(curBoundOperator, curBoundValue, scvf));
}
pair.setSecond(null);
} else if (prevBoundOperator == CompareOp.GREATER
|| prevBoundOperator == CompareOp.GREATER_OR_EQUAL) {
if (result > 1) {
LOG.warn("Possible wrong usage as there cannot be a value < 10 and > 20");
pair.setFirst(null);
pair.setSecond(null);
} else {
pair.setSecond(new Value(curBoundOperator, curBoundValue, scvf));
}
} else if (prevBoundOperator == CompareOp.EQUAL) {
LOG.warn("Use the EQUAL operator only and ignore the current one.");
pair.setSecond(null);
}
break;
case EQUAL:
// For equal condition give priority to equals only..
// If the prevOperator is also == and the current is also ==
// take the second one.(Currently)
if (prevBoundOperator == CompareOp.LESS
|| prevBoundOperator == CompareOp.LESS_OR_EQUAL
|| prevBoundOperator == CompareOp.EQUAL
|| prevBoundOperator == CompareOp.GREATER
|| prevBoundOperator == CompareOp.GREATER_OR_EQUAL) {
pair.setFirst(new Value(curBoundOperator, curBoundValue, scvf));
pair.setSecond(null);
}
break;
case NOT_EQUAL:
case NO_OP:
// Need to check this
break;
}
} else {
LOG.warn(
"Am getting an extra comparison coming for the same col family."
+ "I cannot have 3 conditions on the same column");
pair.setFirst(null);
pair.setSecond(null);
}
}
}
private void handleScvfOfOR(SingleColumnValueFilter scvf) {
ValuePartition vp = null;
if (scvf instanceof SingleColumnValuePartitionFilter) {
vp = ((SingleColumnValuePartitionFilter) scvf).getValuePartition();
}
Column key = new Column(scvf.getFamily(), scvf.getQualifier(), vp);
if (colWithOperatorsOfOR.get(key) == null) {
List<Value> valueList = new ArrayList<FilterGroupingWorker.Value>(1);
valueList.add(new Value(scvf.getOperator(), scvf.getComparator().getValue(), scvf));
colWithOperatorsOfOR.put(key, valueList);
} else {
List<Value> valueList = colWithOperatorsOfOR.get(key);
Iterator<Value> valueListItr = valueList.iterator();
CompareOp operator = scvf.getOperator();
byte[] value = scvf.getComparator().getValue();
Value prevValueObj = null;
while (valueListItr.hasNext()) {
prevValueObj = valueListItr.next();
// TODO As Anoop said we may have to check the Value type also..
// We can not compare and validate this way. btw "a" and "K".
// Only in case of Numeric col type we can have this check.
byte[] prevValue = prevValueObj.getValue();
int result = Bytes.compareTo(prevValue, value);
CompareOp prevOperator = prevValueObj.getOperator();
switch (operator) {
case GREATER:
if (prevOperator == CompareOp.GREATER || prevOperator == CompareOp.GREATER_OR_EQUAL) {
if (result > 0) {
valueListItr.remove();
} else {
// Already you found less or equal value than present value means present filter
// will
// return subset of previous filter. No need to add it to list.
return;
}
} else if (prevOperator == CompareOp.LESS || prevOperator == CompareOp.LESS_OR_EQUAL) {
// Need to handle conditions like previous is c1<5 and current c1>2. In these cases we
// can change this condition into three parts c1<2,c1>=2 AND C1=<5 ,c1>5 and add to
// list.
} else if (prevOperator == CompareOp.EQUAL) {
if (result > 0) {
valueListItr.remove();
} else if (result == 0) {
// remove this entry and convert GREATER to GREATER_OR_EQUAL
valueListItr.remove();
SingleColumnValueFilter newScvf = null;
if (vp == null) {
newScvf =
new SingleColumnValueFilter(
scvf.getFamily(),
scvf.getQualifier(),
CompareOp.GREATER_OR_EQUAL,
scvf.getComparator());
} else {
newScvf =
new SingleColumnValuePartitionFilter(
scvf.getFamily(),
scvf.getQualifier(),
CompareOp.GREATER_OR_EQUAL,
scvf.getComparator(),
vp);
}
Value newValue = new Value(CompareOp.GREATER_OR_EQUAL, prevValue, newScvf);
valueList.add(newValue);
return;
}
}
break;
case GREATER_OR_EQUAL:
if (prevOperator == CompareOp.GREATER || prevOperator == CompareOp.GREATER_OR_EQUAL) {
if (result >= 0) {
valueListItr.remove();
} else {
// Already you found less value than present value means present filter will
// return subset of previous filter. No need to add it to list.
return;
}
} else if (prevOperator == CompareOp.LESS || prevOperator == CompareOp.LESS_OR_EQUAL) {
// Need to handle conditions like previous is c1<5 and current c1>2. In these cases we
// can change this condition into three parts c1<2,c1>=2 AND C1=<5 ,c1>5 and add to
// list.
} else if (prevOperator == CompareOp.EQUAL) {
if (result >= 0) {
valueListItr.remove();
}
}
break;
case LESS:
if (prevOperator == CompareOp.LESS || prevOperator == CompareOp.LESS_OR_EQUAL) {
if (result < 0) {
valueListItr.remove();
} else {
// Already you found less or equal value than present value means present filter
// will
// return subset of previous filter. No need to add it to list.
return;
}
} else if (prevOperator == CompareOp.GREATER
|| prevOperator == CompareOp.GREATER_OR_EQUAL) {
// Need to handle conditions like previous is c1<5 and current c1>2. In these cases we
// can change this condition into three parts c1<2,c1>=2 AND C1=<5 ,c1>5 and add to
// list.
} else if (prevOperator == CompareOp.EQUAL) {
if (result < 0) {
valueListItr.remove();
} else if (result == 0) {
// remove this entry and convert LESS to LESS_OR_EQUAL
valueListItr.remove();
SingleColumnValueFilter newScvf = null;
if (vp == null) {
newScvf =
new SingleColumnValueFilter(
scvf.getFamily(),
scvf.getQualifier(),
CompareOp.LESS_OR_EQUAL,
scvf.getComparator());
} else {
newScvf =
new SingleColumnValuePartitionFilter(
scvf.getFamily(),
scvf.getQualifier(),
CompareOp.LESS_OR_EQUAL,
scvf.getComparator(),
vp);
}
Value newValue = new Value(CompareOp.LESS_OR_EQUAL, prevValue, newScvf);
valueList.add(newValue);
return;
}
}
break;
case LESS_OR_EQUAL:
if (prevOperator == CompareOp.LESS || prevOperator == CompareOp.LESS_OR_EQUAL) {
if (result <= 0) {
valueListItr.remove();
} else {
// Already you found less or equal value than present value means present filter
// will
// return subset of previous filter. No need to add it to list.
return;
}
} else if (prevOperator == CompareOp.GREATER
|| prevOperator == CompareOp.GREATER_OR_EQUAL) {
// Need to handle conditions like previous is c1<5 and current c1>2. In these cases we
// can change this condition into three parts c1<2,c1>=2 AND C1=<5 ,c1>5 and add to
// list.
} else if (prevOperator == CompareOp.EQUAL) {
// If we dont want to do conversion we can add into first if condition.
if (result <= 0) {
valueListItr.remove();
} else if (result == 0) {
// remove this entry and convert GREATER to GREATER_OR_EQUAL
}
}
break;
case EQUAL:
if (prevOperator == CompareOp.GREATER) {
if (result < 0) {
return;
} else if (result == 0) {
valueListItr.remove();
SingleColumnValueFilter newScvf = null;
if (vp == null) {
newScvf =
new SingleColumnValueFilter(
scvf.getFamily(),
scvf.getQualifier(),
CompareOp.GREATER_OR_EQUAL,
scvf.getComparator());
} else {
newScvf =
new SingleColumnValuePartitionFilter(
scvf.getFamily(),
scvf.getQualifier(),
CompareOp.GREATER_OR_EQUAL,
scvf.getComparator(),
vp);
}
Value newValue = new Value(CompareOp.GREATER_OR_EQUAL, prevValue, newScvf);
valueList.add(newValue);
return;
}
} else if (prevOperator == CompareOp.GREATER_OR_EQUAL) {
if (result <= 0) {
return;
}
} else if (prevOperator == CompareOp.LESS) {
if (result > 0) {
return;
} else if (result == 0) {
valueListItr.remove();
SingleColumnValueFilter newScvf = null;
if (vp == null) {
newScvf =
new SingleColumnValueFilter(
scvf.getFamily(),
scvf.getQualifier(),
CompareOp.LESS_OR_EQUAL,
scvf.getComparator());
} else {
newScvf =
new SingleColumnValuePartitionFilter(
scvf.getFamily(),
scvf.getQualifier(),
CompareOp.LESS_OR_EQUAL,
scvf.getComparator(),
vp);
}
Value newValue = new Value(CompareOp.LESS_OR_EQUAL, prevValue, newScvf);
valueList.add(newValue);
return;
}
} else if (prevOperator == CompareOp.LESS_OR_EQUAL) {
if (result >= 0) {
return;
}
} else if (prevOperator == CompareOp.EQUAL) {
if (result == 0) {
// Already same filter exists with same condiftion.
return;
}
}
break;
case NOT_EQUAL:
case NO_OP:
// Need to check this
break;
}
}
valueList.add(new Value(scvf.getOperator(), scvf.getComparator().getValue(), scvf));
}
}
public FilterModel(Filter filter) {
String typeName = filter.getClass().getSimpleName();
FilterType type = FilterType.valueOf(typeName);
this.type = typeName;
switch (type) {
case ColumnCountGetFilter:
this.limit = ((ColumnCountGetFilter) filter).getLimit();
break;
case ColumnPaginationFilter:
this.limit = ((ColumnPaginationFilter) filter).getLimit();
this.offset = ((ColumnPaginationFilter) filter).getOffset();
break;
case ColumnPrefixFilter:
this.value = Base64.encodeBytes(((ColumnPrefixFilter) filter).getPrefix());
break;
case ColumnRangeFilter:
this.minColumn = Base64.encodeBytes(((ColumnRangeFilter) filter).getMinColumn());
this.minColumnInclusive = ((ColumnRangeFilter) filter).getMinColumnInclusive();
this.maxColumn = Base64.encodeBytes(((ColumnRangeFilter) filter).getMaxColumn());
this.maxColumnInclusive = ((ColumnRangeFilter) filter).getMaxColumnInclusive();
break;
case DependentColumnFilter:
{
DependentColumnFilter dcf = (DependentColumnFilter) filter;
this.family = Base64.encodeBytes(dcf.getFamily());
byte[] qualifier = dcf.getQualifier();
if (qualifier != null) {
this.qualifier = Base64.encodeBytes(qualifier);
}
this.op = dcf.getOperator().toString();
this.comparator = new WritableByteArrayComparableModel(dcf.getComparator());
this.dropDependentColumn = dcf.dropDependentColumn();
}
break;
case FilterList:
this.op = ((FilterList) filter).getOperator().toString();
this.filters = new ArrayList<FilterModel>();
for (Filter child : ((FilterList) filter).getFilters()) {
this.filters.add(new FilterModel(child));
}
break;
case FirstKeyOnlyFilter:
case KeyOnlyFilter:
break;
case InclusiveStopFilter:
this.value = Base64.encodeBytes(((InclusiveStopFilter) filter).getStopRowKey());
break;
case MultipleColumnPrefixFilter:
this.prefixes = new ArrayList<String>();
for (byte[] prefix : ((MultipleColumnPrefixFilter) filter).getPrefix()) {
this.prefixes.add(Base64.encodeBytes(prefix));
}
break;
case PageFilter:
this.value = Long.toString(((PageFilter) filter).getPageSize());
break;
case PrefixFilter:
this.value = Base64.encodeBytes(((PrefixFilter) filter).getPrefix());
break;
case FamilyFilter:
case QualifierFilter:
case RowFilter:
case ValueFilter:
this.op = ((CompareFilter) filter).getOperator().toString();
this.comparator =
new WritableByteArrayComparableModel(((CompareFilter) filter).getComparator());
break;
case RandomRowFilter:
this.chance = ((RandomRowFilter) filter).getChance();
break;
case SingleColumnValueExcludeFilter:
case SingleColumnValueFilter:
{
SingleColumnValueFilter scvf = (SingleColumnValueFilter) filter;
this.family = Base64.encodeBytes(scvf.getFamily());
byte[] qualifier = scvf.getQualifier();
if (qualifier != null) {
this.qualifier = Base64.encodeBytes(qualifier);
}
this.op = scvf.getOperator().toString();
this.comparator = new WritableByteArrayComparableModel(scvf.getComparator());
if (scvf.getFilterIfMissing()) {
this.ifMissing = true;
}
if (scvf.getLatestVersionOnly()) {
this.latestVersion = true;
}
}
break;
case SkipFilter:
this.filters = new ArrayList<FilterModel>();
this.filters.add(new FilterModel(((SkipFilter) filter).getFilter()));
break;
case TimestampsFilter:
this.timestamps = ((TimestampsFilter) filter).getTimestamps();
break;
case WhileMatchFilter:
this.filters = new ArrayList<FilterModel>();
this.filters.add(new FilterModel(((WhileMatchFilter) filter).getFilter()));
break;
default:
throw new RuntimeException("unhandled filter type " + type);
}
}
/** Construct a ValueFilter for a SingleColumnValueFilter. */
private ValueFilter createValueFilter(SingleColumnValueFilter filter) {
return new ValueFilter(filter.getOperator(), filter.getComparator());
}
/** Emit a filter that will limit the number of cell versions that will be emitted. */
private RowFilter createVersionLimitFilter(SingleColumnValueFilter filter) {
return RowFilter.newBuilder()
.setCellsPerColumnLimitFilter(filter.getLatestVersionOnly() ? 1 : Integer.MAX_VALUE)
.build();
}
@Override
public Scan getRawScan(ItemData d, Map<String, HCompareOp> ops) {
int startShopID = 0;
int startItemID = 0;
int endShopID = MAX_SHOP_ID;
int endItemID = MAX_ITEM_ID;
// some performance improvement
// shop_id指定
HCompareOp shopIDOp = ops.get("shop_id");
if (shopIDOp == HCompareOp.EQUAL) {
startShopID = d.shopID;
endShopID = startShopID;
}
// item idも指定
HCompareOp itemIDOp = ops.get("item_id");
if (itemIDOp == HCompareOp.EQUAL) {
startItemID = d.itemID;
endItemID = startItemID;
}
log.info(String.format("scan start row, shop_id=%d, item_id=%d", startShopID, startItemID));
log.info(String.format("scan stop row, shop_id=%d, item_id=%d", endShopID, endItemID));
byte[] startRow = encodeRowkey(startShopID, startItemID);
byte[] endRow = encodeRowkey(endShopID, endItemID);
Scan s = new Scan(startRow, endRow);
s.addFamily(DATA_FAMILY);
s.addFamily(META_FAMILY);
s.setCacheBlocks(false);
s.setMaxVersions();
s.setCaching(DEFAULT_SCAN_CACHE);
FilterList fl = new FilterList();
for (String column : ops.keySet()) {
byte[] value;
byte[] family = DATA_FAMILY;
if ("ctime".equals(column)) {
value = Bytes.toBytes(d.ctime);
family = META_FAMILY;
} else if ("shop_id".equals(column)) {
value = Bytes.toBytes(d.shopID);
} else if ("item_id".equals(column)) {
value = Bytes.toBytes(d.itemID);
} else if ("genre_id".equals(column)) {
value = Bytes.toBytes(d.genreID);
} else if ("price".equals(column)) {
value = Bytes.toBytes(d.price);
} else if ("full_item_url".equals(column)) {
value = Bytes.toBytes(d.fullItemUrl);
} else if ("item_name".equals(column)) {
value = Bytes.toBytes(d.itemName);
} else {
// ignore
continue;
}
byte[] qualifier = Bytes.toBytes(column);
HCompareOp hop = ops.get(column);
CompareOp op = HClient.toCompareOp(hop);
SingleColumnValueFilter filter = new SingleColumnValueFilter(family, qualifier, op, value);
filter.setFilterIfMissing(true);
fl.addFilter(filter);
}
s.setFilter(fl);
return s;
}
public Filter build() {
Filter filter;
switch (FilterType.valueOf(type)) {
case ColumnCountGetFilter:
filter = new ColumnCountGetFilter(limit);
break;
case ColumnPaginationFilter:
filter = new ColumnPaginationFilter(limit, offset);
break;
case ColumnPrefixFilter:
filter = new ColumnPrefixFilter(Base64.decode(value));
break;
case ColumnRangeFilter:
filter =
new ColumnRangeFilter(
Base64.decode(minColumn),
minColumnInclusive,
Base64.decode(maxColumn),
maxColumnInclusive);
break;
case DependentColumnFilter:
filter =
new DependentColumnFilter(
Base64.decode(family),
qualifier != null ? Base64.decode(qualifier) : null,
dropDependentColumn,
CompareOp.valueOf(op),
comparator.build());
break;
case FamilyFilter:
filter = new FamilyFilter(CompareOp.valueOf(op), comparator.build());
break;
case FilterList:
{
List<Filter> list = new ArrayList<Filter>();
for (FilterModel model : filters) {
list.add(model.build());
}
filter = new FilterList(FilterList.Operator.valueOf(op), list);
}
break;
case FirstKeyOnlyFilter:
filter = new FirstKeyOnlyFilter();
break;
case InclusiveStopFilter:
filter = new InclusiveStopFilter(Base64.decode(value));
break;
case KeyOnlyFilter:
filter = new KeyOnlyFilter();
break;
case MultipleColumnPrefixFilter:
{
byte[][] values = new byte[prefixes.size()][];
for (int i = 0; i < prefixes.size(); i++) {
values[i] = Base64.decode(prefixes.get(i));
}
filter = new MultipleColumnPrefixFilter(values);
}
break;
case PageFilter:
filter = new PageFilter(Long.valueOf(value));
break;
case PrefixFilter:
filter = new PrefixFilter(Base64.decode(value));
break;
case QualifierFilter:
filter = new QualifierFilter(CompareOp.valueOf(op), comparator.build());
break;
case RandomRowFilter:
filter = new RandomRowFilter(chance);
break;
case RowFilter:
filter = new RowFilter(CompareOp.valueOf(op), comparator.build());
break;
case SingleColumnValueFilter:
filter =
new SingleColumnValueFilter(
Base64.decode(family),
qualifier != null ? Base64.decode(qualifier) : null,
CompareOp.valueOf(op),
comparator.build());
if (ifMissing != null) {
((SingleColumnValueFilter) filter).setFilterIfMissing(ifMissing);
}
if (latestVersion != null) {
((SingleColumnValueFilter) filter).setLatestVersionOnly(latestVersion);
}
break;
case SingleColumnValueExcludeFilter:
filter =
new SingleColumnValueExcludeFilter(
Base64.decode(family),
qualifier != null ? Base64.decode(qualifier) : null,
CompareOp.valueOf(op),
comparator.build());
if (ifMissing != null) {
((SingleColumnValueExcludeFilter) filter).setFilterIfMissing(ifMissing);
}
if (latestVersion != null) {
((SingleColumnValueExcludeFilter) filter).setLatestVersionOnly(latestVersion);
}
break;
case SkipFilter:
filter = new SkipFilter(filters.get(0).build());
break;
case TimestampsFilter:
filter = new TimestampsFilter(timestamps);
break;
case ValueFilter:
filter = new ValueFilter(CompareOp.valueOf(op), comparator.build());
break;
case WhileMatchFilter:
filter = new WhileMatchFilter(filters.get(0).build());
break;
default:
throw new RuntimeException("unhandled filter type: " + type);
}
return filter;
}
/*
* Must be sure that column is in format like 'family: column'
*/
public void prepare(String[] columns) throws IOException {
this.scan = new Scan();
if (this.startKey != null) {
logger.info(String.format("HBaseReader set startkey to %s .", Bytes.toString(this.startKey)));
scan.setStartRow(startKey);
}
if (this.endKey != null) {
logger.info(String.format("HBaseReader set endkey to %s .", Bytes.toString(this.endKey)));
scan.setStopRow(endKey);
}
if (this.minStamp > 0 && this.maxStamp > 0) {
scan.setTimeRange(minStamp, maxStamp);
if (StringUtils.isNotEmpty(primaryKey)) {
String[] cs = StringUtils.split(primaryKey, ":");
scan.addColumn(cs[0].getBytes(), cs[1].getBytes());
}
} else if (ETLStringUtils.isNotEmpty(startDate)
&& ETLStringUtils.isNotEmpty(familyName)
&& ETLStringUtils.isNotEmpty(fieldName)) {
SingleColumnValueFilter scvf =
new SingleColumnValueFilter(
Bytes.toBytes(familyName),
Bytes.toBytes(fieldName),
CompareOp.GREATER_OR_EQUAL,
new BinaryPrefixComparator(startDate.getBytes()));
scvf.setFilterIfMissing(true);
scvf.setLatestVersionOnly(true);
if (ETLStringUtils.isNotEmpty(endDate)) {
SingleColumnValueFilter scvf1 =
new SingleColumnValueFilter(
Bytes.toBytes(familyName),
Bytes.toBytes(fieldName),
CompareOp.LESS_OR_EQUAL,
new BinaryPrefixComparator(endDate.getBytes()));
scvf1.setFilterIfMissing(true);
scvf1.setLatestVersionOnly(true);
List<Filter> filters = new ArrayList<Filter>();
filters.add(scvf);
filters.add(scvf1);
FilterList fl = new FilterList(FilterList.Operator.MUST_PASS_ALL, filters);
scan.setFilter(fl);
} else {
scan.setFilter(scvf);
}
}
this.families = new String[columns.length];
this.columns = new String[columns.length];
int idx = 0;
for (String column : columns) {
this.families[idx] = column.split(":")[0].trim();
this.columns[idx] = column.split(":")[1].trim();
if (!(this.minStamp > 0 && this.maxStamp > 0)) {
scan.addColumn(this.families[idx].getBytes(), this.columns[idx].getBytes());
}
idx++;
}
if ("true".equalsIgnoreCase(this.isSaveOneObj)) {
scan.addColumn(this.oneObjFamilyName.getBytes(), this.oneObjColName.getBytes());
}
if (this.maxversion > 0) {
scan.setMaxVersions(this.maxversion);
} else {
scan.setMaxVersions();
}
htable.setScannerCaching(SCAN_CACHE);
this.rs = htable.getScanner(this.scan);
}