public Filter group(Filter filter) {
   if (filter instanceof FilterList) {
     FilterList fList = (FilterList) filter;
     // We need to create a new FL here taking up only the filters of our interest
     FilterList newFList = new FilterList(fList.getOperator());
     List<Filter> filters = fList.getFilters();
     if (fList.getOperator() == Operator.MUST_PASS_ONE) {
       for (Filter subFilter : filters) {
         Filter resultFilter = handleFilterWithinOR(subFilter);
         // If result filter is not SingleColumnValueFilter or filter list that means OR branch is
         // having different type of filter other than SCVF. In that case we should not consider
         // the OR branch for scanning.
         if (resultFilter instanceof FilterList) {
           newFList.addFilter(resultFilter);
         } else if (resultFilter != null) {
           // This means OR filter list have at least one filter other than SCVF(may be other
           // child OR branches).
           return null;
         }
       }
       addORColsToFinalList(newFList);
       if (newFList.getFilters().isEmpty()) {
         return null;
       }
       return newFList;
     } else {
       // AND condition as long as the condition is AND in one sub tree all those can be
       // grouped under one AND parent(new one).
       for (Filter subFilter : filters) {
         Filter group = handleFilterWithinAND(subFilter);
         // group is null means, all are AND conditions and will be handled at once with the
         // below createFinalFilter
         if (group != null) {
           newFList.addFilter(group);
         }
       }
       addANDColsToFinalList(newFList);
       if (newFList.getFilters().isEmpty()) {
         return null;
       }
       return newFList;
     }
   } else if (filter instanceof SingleColumnValueFilter
       || filter instanceof SingleColumnRangeFilter) {
     return filter;
   }
   return null;
 }
 /**
  * Since you can use Filter Lists as children of Filter Lists, you can create a hierarchy of
  * filters to be evaluated. In the hierarchy if OR branch having any filter type other than SCVF
  * as child then we should not consider the branch for scanning because we cannot fetch seek
  * points from other type of filters without column and value details. Ex: AND AND
  * __________|_______ | | | --> SCVF OR SCVF _______|______ | | ROWFILTER SVCF If the OR is root
  * then we should skip index table scanning for this filter. OR _______|______ --> null | |
  * ROWFILTER SVCF If the OR is child of another OR branch then parent OR branch will be excluded
  * for scanning. Ex: AND AND __________|_______ | | | --> SCVF OR SCVF _______|______ | | OR SVCF
  * _______|______ | | ROWFILTER SVCF
  *
  * @param filter
  * @return if filter is filter list with AND condition then we will return AND branch after
  *     grouping. if filter is filter list with OR condition return null if no children is of type
  *     other than SCVF or filter list else return different filter. if filter is SCVF then return
  *     null. returning null means we are combining the filter(s) with children of parent OR filter
  *     to perform optimizations.
  */
 private Filter handleFilterWithinOR(Filter filter) {
   if (filter instanceof FilterList) {
     FilterList fList = (FilterList) filter;
     if (fList.getOperator() == Operator.MUST_PASS_ONE) {
       List<Filter> filters = fList.getFilters();
       Filter resultFilter = null;
       for (Filter subFilter : filters) {
         // If this OR branch in the filter list have filter type other than SCVF we should report
         // it to parent by returning the other type of filter in such a way that the branch will
         // be skipped from index scan.
         resultFilter = handleFilterWithinOR(subFilter);
         if (resultFilter == null || (resultFilter instanceof FilterList)) {
           continue;
         } else {
           return resultFilter;
         }
       }
       return null;
     } else {
       return new FilterGroupingWorker().group(fList);
     }
   } else if (filter instanceof SingleColumnValueFilter) {
     handleScvfOfOR((SingleColumnValueFilter) filter);
     return null;
   } // TODO when we expose SingleColumnRangeFilter to handle that also here.
   // filter other than SingleColumnValueFilter.
   return filter;
 }
Example #3
0
 public static void andFilterAtEnd(Scan scan, Filter andWithFilter) {
   if (andWithFilter == null) {
     return;
   }
   Filter filter = scan.getFilter();
   if (filter == null) {
     scan.setFilter(andWithFilter);
   } else if (filter instanceof FilterList
       && ((FilterList) filter).getOperator() == FilterList.Operator.MUST_PASS_ALL) {
     FilterList filterList = (FilterList) filter;
     List<Filter> allFilters = new ArrayList<Filter>(filterList.getFilters().size() + 1);
     allFilters.addAll(filterList.getFilters());
     allFilters.add(andWithFilter);
     scan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, allFilters));
   } else {
     scan.setFilter(
         new FilterList(FilterList.Operator.MUST_PASS_ALL, Arrays.asList(filter, andWithFilter)));
   }
 }
  /**
   * Process a {@link org.plasma.query.model.LogicalOperator logical operator} query traversal start
   * event. If the {@link FilterList filter list} on the top of the filter stack is not an 'OR'
   * filter, since it's immutable and we cannot modify its operator, create an 'OR' filter and swaps
   * out the existing filters into the new 'OR' {@link FilterList filter list}.
   */
  public void start(LogicalOperator operator) {

    switch (operator.getValue()) {
      case AND:
        break; // default filter list oper is must-pass-all (AND)
      case OR:
        FilterList top = this.filterStack.peek();
        if (top.getOperator().ordinal() != FilterList.Operator.MUST_PASS_ONE.ordinal()) {
          FilterList orList = new FilterList(FilterList.Operator.MUST_PASS_ONE);
          for (Filter filter : top.getFilters()) orList.addFilter(filter);
          top.getFilters().clear();
          this.filterStack.pop();
          FilterList previous = this.filterStack.peek();
          if (!previous.getFilters().remove(top))
            throw new IllegalStateException("could not remove filter list");
          previous.addFilter(orList);
          this.filterStack.push(orList);
        }
        break;
    }
    super.start(operator);
  }
Example #5
0
  @Test
  public void testParsedFilter() {
    String q1 = "@cluster = \"cluster1\" and @datacenter = \"dc1\" and @field3 = 100000";
    try {
      FilterList filterList = (FilterList) buildFilter(q1);
      Assert.assertEquals(FilterList.Operator.MUST_PASS_ONE, filterList.getOperator());
      Assert.assertEquals(1, filterList.getFilters().size());
      Assert.assertEquals(2, ((FilterList) filterList.getFilters().get(0)).getFilters().size());
    } catch (EagleQueryParseException e) {
      Assert.fail(e.getMessage());
    }

    String q2 =
        "@cluster = \"cluster1\" and @datacenter = \"dc1\" and ( @field3 = 100000 or @field3 < 100000)";
    try {
      FilterList filterList = (FilterList) buildFilter(q2);
      Assert.assertEquals(FilterList.Operator.MUST_PASS_ONE, filterList.getOperator());
      Assert.assertEquals(2, filterList.getFilters().size());
      Assert.assertEquals(2, ((FilterList) filterList.getFilters().get(0)).getFilters().size());
    } catch (EagleQueryParseException e) {
      Assert.fail(e.getMessage());
    }

    // Test parse success but bad type of value
    String q3 =
        "@cluster = \"cluster1\" and @datacenter = \"dc1\" and ( @field3 = 100000 or @field3 < \"bad_int_100000\")";
    boolean q3Ex = false;
    try {
      Assert.assertNull(buildFilter(q3));
    } catch (EagleQueryParseException e) {
      Assert.fail(e.getMessage());
    } catch (IllegalArgumentException e) {
      LOG.debug("Expect: ", e);
      Assert.assertTrue(e.getCause() instanceof NumberFormatException);
      q3Ex = true;
    }
    Assert.assertTrue(q3Ex);
  }
Example #6
0
 public static void setRowKeyOffset(Scan scan, int offset) {
   Filter filter = scan.getFilter();
   if (filter == null) {
     return;
   }
   if (filter instanceof FilterList) {
     FilterList filterList = (FilterList) filter;
     for (Filter childFilter : filterList.getFilters()) {
       setRowKeyOffset(childFilter, offset);
     }
   } else {
     setRowKeyOffset(filter, offset);
   }
 }
 private Filter handleFilterWithinAND(Filter filter) {
   if (filter instanceof FilterList) {
     FilterList fList = (FilterList) filter;
     if (fList.getOperator() == Operator.MUST_PASS_ONE) {
       return new FilterGroupingWorker().group(fList);
     } else {
       List<Filter> filters = fList.getFilters();
       for (Filter subFilter : filters) {
         handleFilterWithinAND(subFilter);
       }
     }
   } else if (filter instanceof SingleColumnValueFilter) {
     handleScvf((SingleColumnValueFilter) filter);
   } // TODO when we expose SingleColumnRangeFilter to handle that also here.
   return null;
 }
 @Test
 /** FilterListAdapter should handle the fact that PageFilterAdapter returns null. */
 public void testPageFilter() throws IOException {
   byte[] qualA = Bytes.toBytes("qualA");
   PageFilter pageFilter = new PageFilter(20);
   FilterList filterList =
       new FilterList(
           Operator.MUST_PASS_ALL,
           new QualifierFilter(CompareOp.EQUAL, new BinaryComparator(qualA)),
           pageFilter);
   FilterAdapter adapter = FilterAdapter.buildAdapter();
   Optional<RowFilter> adapted =
       adapter.adaptFilter(
           new FilterAdapterContext(new Scan(), new DefaultReadHooks()), filterList);
   Assert.assertTrue(adapted.isPresent());
   Optional<RowFilter> qualifierAdapted =
       adapter.adaptFilter(
           new FilterAdapterContext(new Scan(), new DefaultReadHooks()),
           filterList.getFilters().get(0));
   Assert.assertEquals(qualifierAdapted.get(), adapted.get());
 }
Example #9
0
  public static boolean intersectScanRange(
      Scan scan, byte[] startKey, byte[] stopKey, boolean useSkipScan) {
    boolean mayHaveRows = false;
    int offset = 0;
    if (ScanUtil.isLocalIndex(scan)) {
      offset = startKey.length != 0 ? startKey.length : stopKey.length;
    }
    byte[] existingStartKey = scan.getStartRow();
    byte[] existingStopKey = scan.getStopRow();
    if (existingStartKey.length > 0) {
      if (startKey.length == 0 || Bytes.compareTo(existingStartKey, startKey) > 0) {
        startKey = existingStartKey;
      }
    } else {
      mayHaveRows = true;
    }
    if (existingStopKey.length > 0) {
      if (stopKey.length == 0 || Bytes.compareTo(existingStopKey, stopKey) < 0) {
        stopKey = existingStopKey;
      }
    } else {
      mayHaveRows = true;
    }
    scan.setStartRow(startKey);
    scan.setStopRow(stopKey);
    if (offset > 0 && useSkipScan) {
      byte[] temp = null;
      if (startKey.length != 0) {
        temp = new byte[startKey.length - offset];
        System.arraycopy(startKey, offset, temp, 0, startKey.length - offset);
        startKey = temp;
      }
      if (stopKey.length != 0) {
        temp = new byte[stopKey.length - offset];
        System.arraycopy(stopKey, offset, temp, 0, stopKey.length - offset);
        stopKey = temp;
      }
    }
    mayHaveRows = mayHaveRows || Bytes.compareTo(scan.getStartRow(), scan.getStopRow()) < 0;

    // If the scan is using skip scan filter, intersect and replace the filter.
    if (mayHaveRows && useSkipScan) {
      Filter filter = scan.getFilter();
      if (filter instanceof SkipScanFilter) {
        SkipScanFilter oldFilter = (SkipScanFilter) filter;
        SkipScanFilter newFilter = oldFilter.intersect(startKey, stopKey);
        if (newFilter == null) {
          return false;
        }
        // Intersect found: replace skip scan with intersected one
        scan.setFilter(newFilter);
      } else if (filter instanceof FilterList) {
        FilterList oldList = (FilterList) filter;
        FilterList newList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
        for (Filter f : oldList.getFilters()) {
          if (f instanceof SkipScanFilter) {
            SkipScanFilter newFilter = ((SkipScanFilter) f).intersect(startKey, stopKey);
            if (newFilter == null) {
              return false;
            }
            newList.addFilter(newFilter);
          } else {
            newList.addFilter(f);
          }
        }
        scan.setFilter(newList);
      }
    }
    return mayHaveRows;
  }
Example #10
0
  @Test
  public void testWithUnescapedString() {
    ///////////////////////////////////
    // Tag filter with IN or EQUAL
    // Should use RowKeyFilter only
    ///////////////////////////////////
    String query = "@cluster = \"cluster1\" and @datacenter = \"dc1\" and @jobID = \"job.1234\"";
    try {
      FilterList filter = (FilterList) buildFilter(query);
      Assert.assertEquals(
          RowFilter.class,
          ((FilterList) filter.getFilters().get(0)).getFilters().get(0).getClass());
      Assert.assertFalse(
          "Should use rowkey filter only", filter.toString().matches(".*job.1234.*"));
    } catch (EagleQueryParseException e) {
      Assert.fail(e.getMessage());
    } catch (Exception ex) {
      Assert.fail(ex.getMessage());
    }

    query = "@cluster = \"cluster1\" and @datacenter = \"dc1\" and @jobID in (\"job_1234\")";
    try {
      FilterList filter = (FilterList) buildFilter(query);
      Assert.assertEquals(
          RowFilter.class,
          ((FilterList) filter.getFilters().get(0)).getFilters().get(0).getClass());
      Assert.assertFalse(
          "Should use rowkey filter only", filter.toString().matches(".*job_1234.*"));
    } catch (EagleQueryParseException e) {
      Assert.fail(e.getMessage());
    } catch (Exception ex) {
      Assert.fail(ex.getMessage());
    }

    query = "@cluster = \"cluster1\" and @datacenter = \"dc1\" and @jobID in (\"job.1234\")";
    try {
      FilterList filter = (FilterList) buildFilter(query);
      Assert.assertEquals(
          RowFilter.class,
          ((FilterList) filter.getFilters().get(0)).getFilters().get(0).getClass());
      Assert.assertFalse(
          "Should use rowkey filter only", filter.toString().matches(".*job.*1234.*"));
    } catch (EagleQueryParseException e) {
      Assert.fail(e.getMessage());
    } catch (Exception ex) {
      Assert.fail(ex.getMessage());
    }

    ///////////////////////////////
    // Tag with other operators
    ///////////////////////////////
    query = "@cluster = \"cluster1\" and @datacenter = \"dc1\" and @jobID =~ \"job_1234\"";

    try {
      FilterList filter = (FilterList) buildFilter(query);
      Assert.assertEquals(
          RowFilter.class,
          ((FilterList) filter.getFilters().get(0)).getFilters().get(0).getClass());
      Assert.assertTrue(filter.toString().matches(".*job_1234.*"));
    } catch (EagleQueryParseException e) {
      Assert.fail(e.getMessage());
    } catch (Exception ex) {
      Assert.fail(ex.getMessage());
    }

    query = "@cluster = \"cluster1\" and @datacenter = \"dc1\" and @jobID =~ \"job.1234\"";

    try {
      FilterList filter = (FilterList) buildFilter(query);
      Assert.assertEquals(
          RowFilter.class,
          ((FilterList) filter.getFilters().get(0)).getFilters().get(0).getClass());
      Assert.assertTrue(filter.toString().matches(".*job.1234.*"));
    } catch (EagleQueryParseException e) {
      Assert.fail(e.getMessage());
    } catch (Exception ex) {
      Assert.fail(ex.getMessage());
    }

    ///////////////////////////////
    // Tag with IN
    // Should escape regexp chars
    ///////////////////////////////
    query = "@cluster = \"cluster1\" and @datacenter = \"dc1\" and @field7 = \"job_1234\"";

    try {
      FilterList filter = (FilterList) buildFilter(query);
      Assert.assertEquals(
          RowFilter.class,
          ((FilterList) filter.getFilters().get(0)).getFilters().get(0).getClass());
      Assert.assertTrue(filter.toString().matches(".*job_1234.*"));
    } catch (EagleQueryParseException e) {
      Assert.fail(e.getMessage());
    } catch (Exception ex) {
      ex.printStackTrace();
      Assert.fail(ex.getMessage());
    }

    query =
        "@cluster = \"cluster1\" and @datacenter = \"dc1\" and @field7 in (\"job.1234\",\"others\")";

    try {
      FilterList filter = (FilterList) buildFilter(query);
      Assert.assertEquals(
          RowFilter.class,
          ((FilterList) filter.getFilters().get(0)).getFilters().get(0).getClass());
      Assert.assertTrue(filter.toString().matches(".*job\\.1234.*"));
    } catch (EagleQueryParseException e) {
      Assert.fail(e.getMessage());
    } catch (Exception ex) {
      Assert.fail(ex.getMessage());
    }
  }
Example #11
0
  public Scan intersectScan(
      Scan scan,
      final byte[] originalStartKey,
      final byte[] originalStopKey,
      final int keyOffset,
      boolean crossesRegionBoundary) {
    byte[] startKey = originalStartKey;
    byte[] stopKey = originalStopKey;
    if (stopKey.length > 0 && Bytes.compareTo(startKey, stopKey) >= 0) {
      return null;
    }
    boolean mayHaveRows = false;
    // Keep the keys as they are if we have a point lookup, as we've already resolved the
    // salt bytes in that case.
    final int scanKeyOffset =
        this.isSalted && !this.isPointLookup ? SaltingUtil.NUM_SALTING_BYTES : 0;
    assert (scanKeyOffset == 0 || keyOffset == 0);
    // Total offset for startKey/stopKey. Either 1 for salted tables or the prefix length
    // of the current region for local indexes. We'll never have a case where a table is
    // both salted and local.
    final int totalKeyOffset = scanKeyOffset + keyOffset;
    byte[] prefixBytes = ByteUtil.EMPTY_BYTE_ARRAY;
    if (totalKeyOffset > 0) {
      prefixBytes = ScanUtil.getPrefix(startKey, totalKeyOffset);
      /*
       * If our startKey to stopKey crosses a region boundary consider everything after the startKey as our scan
       * is always done within a single region. This prevents us from having to prefix the key prior to knowing
       * whether or not there may be an intersection. We can't calculate whether or not we've crossed a region
       * boundary for local indexes, because we don't know the key offset of the next region, but only for the
       * current one (which is the one passed in). If the next prefix happened to be a subset of the previous
       * prefix, then this wouldn't detect that we crossed a region boundary.
       */
      if (crossesRegionBoundary) {
        stopKey = ByteUtil.EMPTY_BYTE_ARRAY;
      }
    }
    int scanStartKeyOffset = scanKeyOffset;
    byte[] scanStartKey = scan == null ? ByteUtil.EMPTY_BYTE_ARRAY : scan.getStartRow();
    // Compare ignoring key prefix and salt byte
    if (scanStartKey.length > 0) {
      if (startKey.length > 0
          && Bytes.compareTo(
                  scanStartKey,
                  scanKeyOffset,
                  scanStartKey.length - scanKeyOffset,
                  startKey,
                  totalKeyOffset,
                  startKey.length - totalKeyOffset)
              < 0) {
        scanStartKey = startKey;
        scanStartKeyOffset = totalKeyOffset;
      }
    } else {
      scanStartKey = startKey;
      scanStartKeyOffset = totalKeyOffset;
      mayHaveRows = true;
    }
    int scanStopKeyOffset = scanKeyOffset;
    byte[] scanStopKey = scan == null ? ByteUtil.EMPTY_BYTE_ARRAY : scan.getStopRow();
    if (scanStopKey.length > 0) {
      if (stopKey.length > 0
          && Bytes.compareTo(
                  scanStopKey,
                  scanKeyOffset,
                  scanStopKey.length - scanKeyOffset,
                  stopKey,
                  totalKeyOffset,
                  stopKey.length - totalKeyOffset)
              > 0) {
        scanStopKey = stopKey;
        scanStopKeyOffset = totalKeyOffset;
      }
    } else {
      scanStopKey = stopKey;
      scanStopKeyOffset = totalKeyOffset;
      mayHaveRows = true;
    }
    mayHaveRows =
        mayHaveRows
            || Bytes.compareTo(
                    scanStartKey,
                    scanStartKeyOffset,
                    scanStartKey.length - scanStartKeyOffset,
                    scanStopKey,
                    scanStopKeyOffset,
                    scanStopKey.length - scanStopKeyOffset)
                < 0;

    if (!mayHaveRows) {
      return null;
    }
    if (originalStopKey.length != 0 && scanStopKey.length == 0) {
      scanStopKey = originalStopKey;
    }
    Filter newFilter = null;
    // If the scan is using skip scan filter, intersect and replace the filter.
    if (scan == null || this.useSkipScanFilter()) {
      byte[] skipScanStartKey = scanStartKey;
      byte[] skipScanStopKey = scanStopKey;
      // If we have a keyOffset and we've used the startKey/stopKey that
      // were passed in (which have the prefix) for the above range check,
      // we need to remove the prefix before running our intersect method.
      // TODO: we could use skipScanFilter.setOffset(keyOffset) if both
      // the startKey and stopKey were used above *and* our intersect
      // method honored the skipScanFilter.offset variable.
      if (scanKeyOffset > 0) {
        if (skipScanStartKey != originalStartKey) { // original already has correct salt byte
          skipScanStartKey = replaceSaltByte(skipScanStartKey, prefixBytes);
        }
        if (skipScanStopKey != originalStopKey) {
          skipScanStopKey = replaceSaltByte(skipScanStopKey, prefixBytes);
        }
      } else if (keyOffset > 0) {
        if (skipScanStartKey == originalStartKey) {
          skipScanStartKey = stripPrefix(skipScanStartKey, keyOffset);
        }
        if (skipScanStopKey == originalStopKey) {
          skipScanStopKey = stripPrefix(skipScanStopKey, keyOffset);
        }
      }
      if (scan == null) {
        return filter.hasIntersect(skipScanStartKey, skipScanStopKey) ? HAS_INTERSECTION : null;
      }
      Filter filter = scan.getFilter();
      SkipScanFilter newSkipScanFilter = null;
      if (filter instanceof SkipScanFilter) {
        SkipScanFilter oldSkipScanFilter = (SkipScanFilter) filter;
        newFilter =
            newSkipScanFilter = oldSkipScanFilter.intersect(skipScanStartKey, skipScanStopKey);
        if (newFilter == null) {
          return null;
        }
      } else if (filter instanceof FilterList) {
        FilterList oldList = (FilterList) filter;
        FilterList newList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
        newFilter = newList;
        for (Filter f : oldList.getFilters()) {
          if (f instanceof SkipScanFilter) {
            newSkipScanFilter = ((SkipScanFilter) f).intersect(skipScanStartKey, skipScanStopKey);
            if (newSkipScanFilter == null) {
              return null;
            }
            newList.addFilter(newSkipScanFilter);
          } else {
            newList.addFilter(f);
          }
        }
      }
      // TODO: it seems that our SkipScanFilter or HBase runs into problems if we don't
      // have an enclosing range when we do a point lookup.
      if (isPointLookup) {
        scanStartKey = ScanUtil.getMinKey(schema, newSkipScanFilter.getSlots(), slotSpan);
        scanStopKey = ScanUtil.getMaxKey(schema, newSkipScanFilter.getSlots(), slotSpan);
      }
    }
    if (newFilter == null) {
      newFilter = scan.getFilter();
    }
    Scan newScan = ScanUtil.newScan(scan);
    newScan.setFilter(newFilter);
    // If we have an offset (salted table or local index), we need to make sure to
    // prefix our scan start/stop row by the prefix of the startKey or stopKey that
    // were passed in. Our scan either doesn't have the prefix or has a placeholder
    // for it.
    if (totalKeyOffset > 0) {
      if (scanStartKey != originalStartKey) {
        scanStartKey = prefixKey(scanStartKey, scanKeyOffset, prefixBytes, keyOffset);
      }
      if (scanStopKey != originalStopKey) {
        scanStopKey = prefixKey(scanStopKey, scanKeyOffset, prefixBytes, keyOffset);
      }
    }
    // Don't let the stopRow of the scan go beyond the originalStopKey
    if (originalStopKey.length > 0 && Bytes.compareTo(scanStopKey, originalStopKey) > 0) {
      scanStopKey = originalStopKey;
    }
    if (scanStopKey.length > 0 && Bytes.compareTo(scanStartKey, scanStopKey) >= 0) {
      return null;
    }
    newScan.setAttribute(SCAN_ACTUAL_START_ROW, scanStartKey);
    newScan.setStartRow(scanStartKey);
    newScan.setStopRow(scanStopKey);
    if (keyOffset > 0) {
      newScan.setAttribute(STARTKEY_OFFSET, Bytes.toBytes(keyOffset));
    }

    return newScan;
  }