private static void projectAllColumnFamilies(PTable table, Scan scan) {
// Will project all known/declared column families
scan.getFamilyMap().clear();
for (PColumnFamily family : table.getColumnFamilies()) {
scan.addFamily(family.getName().getBytes());
}
}
/**
* @param scan the scan specification
* @throws Exception
*/
public static ScannerModel fromScan(Scan scan) throws Exception {
ScannerModel model = new ScannerModel();
model.setStartRow(scan.getStartRow());
model.setEndRow(scan.getStopRow());
Map<byte[], NavigableSet<byte[]>> families = scan.getFamilyMap();
if (families != null) {
for (Map.Entry<byte[], NavigableSet<byte[]>> entry : families.entrySet()) {
if (entry.getValue() != null) {
for (byte[] qualifier : entry.getValue()) {
model.addColumn(Bytes.add(entry.getKey(), COLUMN_DIVIDER, qualifier));
}
} else {
model.addColumn(entry.getKey());
}
}
}
model.setStartTime(scan.getTimeRange().getMin());
model.setEndTime(scan.getTimeRange().getMax());
int caching = scan.getCaching();
if (caching > 0) {
model.setBatch(caching);
}
int maxVersions = scan.getMaxVersions();
if (maxVersions > 0) {
model.setMaxVersions(maxVersions);
}
Filter filter = scan.getFilter();
if (filter != null) {
model.setFilter(stringifyFilter(filter));
}
return model;
}
private RowFilter createColumnFamilyFilter(Scan scan) {
// Build a filter of the form:
// (fam1 | (qual1 + qual2 + qual3)) + (fam2 | qual1) + (fam3)
RowFilter.Interleave.Builder interleaveBuilder = RowFilter.Interleave.newBuilder();
Map<byte[], NavigableSet<byte[]>> familyMap = scan.getFamilyMap();
if (!scan.getFamilyMap().isEmpty()) {
for (Map.Entry<byte[], NavigableSet<byte[]>> entry : familyMap.entrySet()) {
if (entry.getValue() == null) {
// No qualifier, add the entire family:
interleaveBuilder.addFilters(createFamilyFilter(entry.getKey()));
} else if (entry.getValue().size() == 1) {
// Build filter of the form "family | qual"
Chain.Builder familyBuilder = interleaveBuilder.addFiltersBuilder().getChainBuilder();
familyBuilder.addFilters(createFamilyFilter(entry.getKey()));
familyBuilder.addFilters(createColumnQualifierFilter(entry.getValue().first()));
} else {
// Build filter of the form "family | (qual1 + qual2 + qual3)"
Chain.Builder familyBuilder = interleaveBuilder.addFiltersBuilder().getChainBuilder();
familyBuilder.addFilters(createFamilyFilter(entry.getKey()));
// Add a qualifier filter for each specified qualifier:
Interleave.Builder columnFilters =
familyBuilder.addFiltersBuilder().getInterleaveBuilder();
for (byte[] qualifier : entry.getValue()) {
columnFilters.addFilters(createColumnQualifierFilter(qualifier));
}
}
}
} else {
// Simplify processing a bit and add an explicit inclusion of all families:
interleaveBuilder.addFiltersBuilder().setFamilyNameRegexFilter(".*");
}
if (interleaveBuilder.getFiltersCount() > 1) {
return RowFilter.newBuilder().setInterleave(interleaveBuilder).build();
} else {
return interleaveBuilder.getFilters(0);
}
}
public static Scan newScan(Scan scan) {
try {
Scan newScan = new Scan(scan);
// Clone the underlying family map instead of sharing it between
// the existing and cloned Scan (which is the retarded default
// behavior).
TreeMap<byte[], NavigableSet<byte[]>> existingMap =
(TreeMap<byte[], NavigableSet<byte[]>>) scan.getFamilyMap();
Map<byte[], NavigableSet<byte[]>> clonedMap =
new TreeMap<byte[], NavigableSet<byte[]>>(existingMap);
newScan.setFamilyMap(clonedMap);
// Carry over the reversed attribute
newScan.setReversed(scan.isReversed());
newScan.setSmall(scan.isSmall());
return newScan;
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/**
* Attempts to run a Get on some queue. Will only return a non-null result if we currently own the
* queue.
*
* @param get The Get that we want to query
* @return The result of the Get if this server is the owner of the queue. Else it returns null.
* @throws IOException
*/
private Result getResultIfOwner(Get get) throws IOException {
Scan scan = new Scan(get);
// Check if the Get currently contains all columns or only specific columns
if (scan.getFamilyMap().size() > 0) {
// Add the OWNER column if the scan is already only over specific columns
scan.addColumn(CF_QUEUE, COL_QUEUE_OWNER);
}
scan.setMaxResultSize(1);
SingleColumnValueFilter checkOwner =
new SingleColumnValueFilter(
CF_QUEUE, COL_QUEUE_OWNER, CompareFilter.CompareOp.EQUAL, serverNameBytes);
scan.setFilter(checkOwner);
ResultScanner scanner = null;
try {
scanner = replicationTable.getScanner(scan);
Result result = scanner.next();
return (result == null || result.isEmpty()) ? null : result;
} finally {
if (scanner != null) {
scanner.close();
}
}
}
/**
* Builds the projection for the scan
*
* @param context query context kept between compilation of different query clauses
* @param statement TODO
* @param groupBy compiled GROUP BY clause
* @param targetColumns list of columns, parallel to aliasedNodes, that are being set for an
* UPSERT SELECT statement. Used to coerce expression types to the expected target type.
* @return projector used to access row values during scan
* @throws SQLException
*/
public static RowProjector compile(
StatementContext context,
SelectStatement statement,
GroupBy groupBy,
List<? extends PDatum> targetColumns)
throws SQLException {
List<AliasedNode> aliasedNodes = statement.getSelect();
// Setup projected columns in Scan
SelectClauseVisitor selectVisitor = new SelectClauseVisitor(context, groupBy);
List<ExpressionProjector> projectedColumns = new ArrayList<ExpressionProjector>();
TableRef tableRef = context.getResolver().getTables().get(0);
PTable table = tableRef.getTable();
boolean isWildcard = false;
Scan scan = context.getScan();
int index = 0;
List<Expression> projectedExpressions = Lists.newArrayListWithExpectedSize(aliasedNodes.size());
List<byte[]> projectedFamilies = Lists.newArrayListWithExpectedSize(aliasedNodes.size());
for (AliasedNode aliasedNode : aliasedNodes) {
ParseNode node = aliasedNode.getNode();
// TODO: visitor?
if (node instanceof WildcardParseNode) {
if (statement.isAggregate()) {
ExpressionCompiler.throwNonAggExpressionInAggException(node.toString());
}
isWildcard = true;
if (tableRef.getTable().getType() == PTableType.INDEX
&& ((WildcardParseNode) node).isRewrite()) {
projectAllIndexColumns(context, tableRef, projectedExpressions, projectedColumns);
} else {
projectAllTableColumns(context, tableRef, projectedExpressions, projectedColumns);
}
} else if (node instanceof FamilyWildcardParseNode) {
// Project everything for SELECT cf.*
// TODO: support cf.* expressions for multiple tables the same way with *.
String cfName = ((FamilyWildcardParseNode) node).getName();
// Delay projecting to scan, as when any other column in the column family gets
// added to the scan, it overwrites that we want to project the entire column
// family. Instead, we do the projection at the end.
// TODO: consider having a ScanUtil.addColumn and ScanUtil.addFamily to work
// around this, as this code depends on this function being the last place where
// columns are projected (which is currently true, but could change).
projectedFamilies.add(Bytes.toBytes(cfName));
if (tableRef.getTable().getType() == PTableType.INDEX
&& ((FamilyWildcardParseNode) node).isRewrite()) {
projectIndexColumnFamily(
context, cfName, tableRef, projectedExpressions, projectedColumns);
} else {
projectTableColumnFamily(
context, cfName, tableRef, projectedExpressions, projectedColumns);
}
} else {
Expression expression = node.accept(selectVisitor);
projectedExpressions.add(expression);
if (index < targetColumns.size()) {
PDatum targetColumn = targetColumns.get(index);
if (targetColumn.getDataType() != expression.getDataType()) {
PDataType targetType = targetColumn.getDataType();
// Check if coerce allowed using more relaxed isCastableTo check, since we promote
// INTEGER to LONG
// during expression evaluation and then convert back to INTEGER on UPSERT SELECT (and
// we don't have
// (an actual value we can specifically check against).
if (expression.getDataType() != null
&& !expression.getDataType().isCastableTo(targetType)) {
throw new ArgumentTypeMismatchException(
targetType, expression.getDataType(), "column: " + targetColumn);
}
expression = CoerceExpression.create(expression, targetType);
}
}
if (node instanceof BindParseNode) {
context.getBindManager().addParamMetaData((BindParseNode) node, expression);
}
if (!node.isStateless()) {
if (!selectVisitor.isAggregate() && statement.isAggregate()) {
ExpressionCompiler.throwNonAggExpressionInAggException(expression.toString());
}
}
String columnAlias =
aliasedNode.getAlias() != null
? aliasedNode.getAlias()
: SchemaUtil.normalizeIdentifier(aliasedNode.getNode().getAlias());
boolean isCaseSensitive =
(columnAlias != null
&& (aliasedNode.isCaseSensitve() || SchemaUtil.isCaseSensitive(columnAlias)))
|| selectVisitor.isCaseSensitive;
String name = columnAlias == null ? expression.toString() : columnAlias;
projectedColumns.add(
new ExpressionProjector(
name, table.getName().getString(), expression, isCaseSensitive));
}
selectVisitor.reset();
index++;
}
table = context.getCurrentTable().getTable(); // switch to current table for scan projection
// TODO make estimatedByteSize more accurate by counting the joined columns.
int estimatedKeySize = table.getRowKeySchema().getEstimatedValueLength();
int estimatedByteSize = 0;
for (Map.Entry<byte[], NavigableSet<byte[]>> entry : scan.getFamilyMap().entrySet()) {
PColumnFamily family = table.getColumnFamily(entry.getKey());
if (entry.getValue() == null) {
for (PColumn column : family.getColumns()) {
Integer byteSize = column.getByteSize();
estimatedByteSize +=
SizedUtil.KEY_VALUE_SIZE
+ estimatedKeySize
+ (byteSize == null ? RowKeySchema.ESTIMATED_VARIABLE_LENGTH_SIZE : byteSize);
}
} else {
for (byte[] cq : entry.getValue()) {
PColumn column = family.getColumn(cq);
Integer byteSize = column.getByteSize();
estimatedByteSize +=
SizedUtil.KEY_VALUE_SIZE
+ estimatedKeySize
+ (byteSize == null ? RowKeySchema.ESTIMATED_VARIABLE_LENGTH_SIZE : byteSize);
}
}
}
selectVisitor.compile();
// Since we don't have the empty key value in read-only tables,
// we must project everything.
boolean isProjectEmptyKeyValue =
table.getType() != PTableType.VIEW && table.getViewType() != ViewType.MAPPED && !isWildcard;
if (isProjectEmptyKeyValue) {
for (byte[] family : projectedFamilies) {
projectColumnFamily(table, scan, family);
}
} else {
/*
* TODO: this could be optimized by detecting:
* - if a column is projected that's not in the where clause
* - if a column is grouped by that's not in the where clause
* - if we're not using IS NULL or CASE WHEN expressions
*/
projectAllColumnFamilies(table, scan);
}
return new RowProjector(projectedColumns, estimatedByteSize, isProjectEmptyKeyValue);
}