public QueryPlan optimize(PhoenixStatement statement, SelectStatement select)
throws SQLException {
return optimize(
statement,
select,
FromCompiler.getResolverForQuery(select, statement.getConnection()),
Collections.<PColumn>emptyList(),
null);
}
public MutationPlan compile(final CreateFunctionStatement create) throws SQLException {
final PhoenixConnection connection = statement.getConnection();
PhoenixConnection connectionToBe = connection;
final StatementContext context = new StatementContext(statement);
final MetaDataClient client = new MetaDataClient(connectionToBe);
return new MutationPlan() {
@Override
public ParameterMetaData getParameterMetaData() {
return context.getBindManager().getParameterMetaData();
}
@Override
public MutationState execute() throws SQLException {
try {
return client.createFunction(create);
} finally {
if (client.getConnection() != connection) {
client.getConnection().close();
}
}
}
@Override
public ExplainPlan getExplainPlan() throws SQLException {
return new ExplainPlan(Collections.singletonList("CREATE FUNCTION"));
}
@Override
public PhoenixConnection getConnection() {
return connection;
}
@Override
public StatementContext getContext() {
return context;
}
};
}
public MutationPlan compile(UpsertStatement upsert) throws SQLException {
final PhoenixConnection connection = statement.getConnection();
ConnectionQueryServices services = connection.getQueryServices();
final int maxSize =
services
.getProps()
.getInt(
QueryServices.MAX_MUTATION_SIZE_ATTRIB,
QueryServicesOptions.DEFAULT_MAX_MUTATION_SIZE);
final ColumnResolver resolver = FromCompiler.getResolverForMutation(upsert, connection);
final TableRef tableRef = resolver.getTables().get(0);
final PTable table = tableRef.getTable();
if (table.getType() == PTableType.VIEW) {
if (table.getViewType().isReadOnly()) {
throw new ReadOnlyTableException(
table.getSchemaName().getString(), table.getTableName().getString());
}
}
boolean isSalted = table.getBucketNum() != null;
final boolean isTenantSpecific = table.isMultiTenant() && connection.getTenantId() != null;
final boolean isSharedViewIndex = table.getViewIndexId() != null;
String tenantId = isTenantSpecific ? connection.getTenantId().getString() : null;
int posOffset = isSalted ? 1 : 0;
// Setup array of column indexes parallel to values that are going to be set
List<ColumnName> columnNodes = upsert.getColumns();
final List<PColumn> allColumns = table.getColumns();
Set<PColumn> addViewColumnsToBe = Collections.emptySet();
Set<PColumn> overlapViewColumnsToBe = Collections.emptySet();
int[] columnIndexesToBe;
int nColumnsToSet = 0;
int[] pkSlotIndexesToBe;
List<PColumn> targetColumns;
if (table.getViewType() == ViewType.UPDATABLE) {
addViewColumnsToBe = Sets.newLinkedHashSetWithExpectedSize(allColumns.size());
for (PColumn column : allColumns) {
if (column.getViewConstant() != null) {
addViewColumnsToBe.add(column);
}
}
}
ImmutableBytesWritable ptr = new ImmutableBytesWritable();
// Allow full row upsert if no columns or only dynamic ones are specified and values count match
if (columnNodes.isEmpty()
|| columnNodes.size() == upsert.getTable().getDynamicColumns().size()) {
nColumnsToSet = allColumns.size() - posOffset;
columnIndexesToBe = new int[nColumnsToSet];
pkSlotIndexesToBe = new int[columnIndexesToBe.length];
targetColumns = Lists.newArrayListWithExpectedSize(columnIndexesToBe.length);
targetColumns.addAll(Collections.<PColumn>nCopies(columnIndexesToBe.length, null));
int minPKPos = 0;
if (isTenantSpecific) {
PColumn tenantColumn = table.getPKColumns().get(minPKPos);
columnIndexesToBe[minPKPos] = tenantColumn.getPosition();
targetColumns.set(minPKPos, tenantColumn);
minPKPos++;
}
if (isSharedViewIndex) {
PColumn indexIdColumn = table.getPKColumns().get(minPKPos);
columnIndexesToBe[minPKPos] = indexIdColumn.getPosition();
targetColumns.set(minPKPos, indexIdColumn);
minPKPos++;
}
for (int i = posOffset, j = 0; i < allColumns.size(); i++) {
PColumn column = allColumns.get(i);
if (SchemaUtil.isPKColumn(column)) {
pkSlotIndexesToBe[i - posOffset] = j + posOffset;
if (j++ < minPKPos) { // Skip, as it's already been set above
continue;
}
minPKPos = 0;
}
columnIndexesToBe[i - posOffset + minPKPos] = i;
targetColumns.set(i - posOffset + minPKPos, column);
}
if (!addViewColumnsToBe.isEmpty()) {
// All view columns overlap in this case
overlapViewColumnsToBe = addViewColumnsToBe;
addViewColumnsToBe = Collections.emptySet();
}
} else {
// Size for worse case
int numColsInUpsert = columnNodes.size();
nColumnsToSet =
numColsInUpsert
+ addViewColumnsToBe.size()
+ (isTenantSpecific ? 1 : 0)
+ +(isSharedViewIndex ? 1 : 0);
columnIndexesToBe = new int[nColumnsToSet];
pkSlotIndexesToBe = new int[columnIndexesToBe.length];
targetColumns = Lists.newArrayListWithExpectedSize(columnIndexesToBe.length);
targetColumns.addAll(Collections.<PColumn>nCopies(columnIndexesToBe.length, null));
Arrays.fill(
columnIndexesToBe,
-1); // TODO: necessary? So we'll get an AIOB exception if it's not replaced
Arrays.fill(
pkSlotIndexesToBe,
-1); // TODO: necessary? So we'll get an AIOB exception if it's not replaced
BitSet pkColumnsSet = new BitSet(table.getPKColumns().size());
int i = 0;
// Add tenant column directly, as we don't want to resolve it as this will fail
if (isTenantSpecific) {
PColumn tenantColumn = table.getPKColumns().get(i + posOffset);
columnIndexesToBe[i] = tenantColumn.getPosition();
pkColumnsSet.set(pkSlotIndexesToBe[i] = i + posOffset);
targetColumns.set(i, tenantColumn);
i++;
}
if (isSharedViewIndex) {
PColumn indexIdColumn = table.getPKColumns().get(i + posOffset);
columnIndexesToBe[i] = indexIdColumn.getPosition();
pkColumnsSet.set(pkSlotIndexesToBe[i] = i + posOffset);
targetColumns.set(i, indexIdColumn);
i++;
}
for (ColumnName colName : columnNodes) {
ColumnRef ref =
resolver.resolveColumn(null, colName.getFamilyName(), colName.getColumnName());
PColumn column = ref.getColumn();
if (IndexUtil.getViewConstantValue(column, ptr)) {
if (overlapViewColumnsToBe.isEmpty()) {
overlapViewColumnsToBe = Sets.newHashSetWithExpectedSize(addViewColumnsToBe.size());
}
nColumnsToSet--;
overlapViewColumnsToBe.add(column);
addViewColumnsToBe.remove(column);
}
columnIndexesToBe[i] = ref.getColumnPosition();
targetColumns.set(i, column);
if (SchemaUtil.isPKColumn(column)) {
pkColumnsSet.set(pkSlotIndexesToBe[i] = ref.getPKSlotPosition());
}
i++;
}
for (PColumn column : addViewColumnsToBe) {
columnIndexesToBe[i] = column.getPosition();
targetColumns.set(i, column);
if (SchemaUtil.isPKColumn(column)) {
pkColumnsSet.set(pkSlotIndexesToBe[i] = SchemaUtil.getPKPosition(table, column));
}
i++;
}
for (i = posOffset; i < table.getPKColumns().size(); i++) {
PColumn pkCol = table.getPKColumns().get(i);
if (!pkColumnsSet.get(i)) {
if (!pkCol.isNullable()) {
throw new ConstraintViolationException(
table.getName().getString()
+ "."
+ pkCol.getName().getString()
+ " may not be null");
}
}
}
}
List<ParseNode> valueNodes = upsert.getValues();
QueryPlan plan = null;
RowProjector rowProjectorToBe = null;
final int nValuesToSet;
boolean sameTable = false;
boolean runOnServer = false;
UpsertingParallelIteratorFactory upsertParallelIteratorFactoryToBe = null;
final boolean isAutoCommit = connection.getAutoCommit();
if (valueNodes == null) {
SelectStatement select = upsert.getSelect();
assert (select != null);
select = SubselectRewriter.flatten(select, connection);
ColumnResolver selectResolver = FromCompiler.getResolverForQuery(select, connection);
select = StatementNormalizer.normalize(select, selectResolver);
select = prependTenantAndViewConstants(table, select, tenantId, addViewColumnsToBe);
sameTable =
select.getFrom().size() == 1 && tableRef.equals(selectResolver.getTables().get(0));
/* We can run the upsert in a coprocessor if:
* 1) from has only 1 table and the into table matches from table
* 2) the select query isn't doing aggregation
* 3) autoCommit is on
* 4) the table is not immutable, as the client is the one that figures out the additional
* puts for index tables.
* 5) no limit clause
* Otherwise, run the query to pull the data from the server
* and populate the MutationState (upto a limit).
*/
runOnServer =
sameTable
&& isAutoCommit
&& !table.isImmutableRows()
&& !select.isAggregate()
&& !select.isDistinct()
&& select.getLimit() == null
&& table.getBucketNum() == null;
ParallelIteratorFactory parallelIteratorFactory;
if (select.isAggregate() || select.isDistinct() || select.getLimit() != null) {
parallelIteratorFactory = null;
} else {
// We can pipeline the upsert select instead of spooling everything to disk first,
// if we don't have any post processing that's required.
parallelIteratorFactory =
upsertParallelIteratorFactoryToBe =
new UpsertingParallelIteratorFactory(connection, tableRef);
}
// If we may be able to run on the server, add a hint that favors using the data table
// if all else is equal.
// TODO: it'd be nice if we could figure out in advance if the PK is potentially changing,
// as this would disallow running on the server. We currently use the row projector we
// get back to figure this out.
HintNode hint = upsert.getHint();
if (!upsert.getHint().hasHint(Hint.USE_INDEX_OVER_DATA_TABLE)) {
hint = HintNode.create(hint, Hint.USE_DATA_OVER_INDEX_TABLE);
}
select = SelectStatement.create(select, hint);
// Pass scan through if same table in upsert and select so that projection is computed
// correctly
// Use optimizer to choose the best plan
plan =
new QueryOptimizer(services)
.optimize(statement, select, selectResolver, targetColumns, parallelIteratorFactory);
runOnServer &= plan.getTableRef().equals(tableRef);
rowProjectorToBe = plan.getProjector();
nValuesToSet = rowProjectorToBe.getColumnCount();
// Cannot auto commit if doing aggregation or topN or salted
// Salted causes problems because the row may end up living on a different region
} else {
nValuesToSet =
valueNodes.size()
+ addViewColumnsToBe.size()
+ (isTenantSpecific ? 1 : 0)
+ (isSharedViewIndex ? 1 : 0);
}
final RowProjector projector = rowProjectorToBe;
final UpsertingParallelIteratorFactory upsertParallelIteratorFactory =
upsertParallelIteratorFactoryToBe;
final QueryPlan queryPlan = plan;
// Resize down to allow a subset of columns to be specifiable
if (columnNodes.isEmpty() && columnIndexesToBe.length >= nValuesToSet) {
nColumnsToSet = nValuesToSet;
columnIndexesToBe = Arrays.copyOf(columnIndexesToBe, nValuesToSet);
pkSlotIndexesToBe = Arrays.copyOf(pkSlotIndexesToBe, nValuesToSet);
}
if (nValuesToSet != nColumnsToSet) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.UPSERT_COLUMN_NUMBERS_MISMATCH)
.setMessage(
"Numbers of columns: " + nColumnsToSet + ". Number of values: " + nValuesToSet)
.build()
.buildException();
}
final int[] columnIndexes = columnIndexesToBe;
final int[] pkSlotIndexes = pkSlotIndexesToBe;
final Set<PColumn> addViewColumns = addViewColumnsToBe;
final Set<PColumn> overlapViewColumns = overlapViewColumnsToBe;
// TODO: break this up into multiple functions
////////////////////////////////////////////////////////////////////
// UPSERT SELECT
/////////////////////////////////////////////////////////////////////
if (valueNodes == null) {
// Before we re-order, check that for updatable view columns
// the projected expression either matches the column name or
// is a constant with the same required value.
throwIfNotUpdatable(tableRef, overlapViewColumnsToBe, targetColumns, projector, sameTable);
////////////////////////////////////////////////////////////////////
// UPSERT SELECT run server-side (maybe)
/////////////////////////////////////////////////////////////////////
if (runOnServer) {
// At most this array will grow bigger by the number of PK columns
int[] allColumnsIndexes = Arrays.copyOf(columnIndexes, columnIndexes.length + nValuesToSet);
int[] reverseColumnIndexes = new int[table.getColumns().size()];
List<Expression> projectedExpressions =
Lists.newArrayListWithExpectedSize(reverseColumnIndexes.length);
Arrays.fill(reverseColumnIndexes, -1);
for (int i = 0; i < nValuesToSet; i++) {
projectedExpressions.add(projector.getColumnProjector(i).getExpression());
reverseColumnIndexes[columnIndexes[i]] = i;
}
/*
* Order projected columns and projected expressions with PK columns
* leading order by slot position
*/
int offset = table.getBucketNum() == null ? 0 : 1;
for (int i = 0; i < table.getPKColumns().size() - offset; i++) {
PColumn column = table.getPKColumns().get(i + offset);
int pos = reverseColumnIndexes[column.getPosition()];
if (pos == -1) {
// Last PK column may be fixed width and nullable
// We don't want to insert a null expression b/c
// it's not valid to set a fixed width type to null.
if (column.getDataType().isFixedWidth()) {
continue;
}
// Add literal null for missing PK columns
pos = projectedExpressions.size();
Expression literalNull =
LiteralExpression.newConstant(null, column.getDataType(), true);
projectedExpressions.add(literalNull);
allColumnsIndexes[pos] = column.getPosition();
}
// Swap select expression at pos with i
Collections.swap(projectedExpressions, i, pos);
// Swap column indexes and reverse column indexes too
int tempPos = allColumnsIndexes[i];
allColumnsIndexes[i] = allColumnsIndexes[pos];
allColumnsIndexes[pos] = tempPos;
reverseColumnIndexes[tempPos] = reverseColumnIndexes[i];
reverseColumnIndexes[i] = i;
}
// If any pk slots are changing, be conservative and don't run this server side.
// If the row ends up living in a different region, we'll get an error otherwise.
for (int i = 0; i < table.getPKColumns().size(); i++) {
PColumn column = table.getPKColumns().get(i);
Expression source = projectedExpressions.get(i);
if (source == null
|| !source.equals(
new ColumnRef(tableRef, column.getPosition()).newColumnExpression())) {
// TODO: we could check the region boundaries to see if the pk will still be in it.
runOnServer = false; // bail on running server side, since PK may be changing
break;
}
}
////////////////////////////////////////////////////////////////////
// UPSERT SELECT run server-side
/////////////////////////////////////////////////////////////////////
if (runOnServer) {
// Iterate through columns being projected
List<PColumn> projectedColumns =
Lists.newArrayListWithExpectedSize(projectedExpressions.size());
for (int i = 0; i < projectedExpressions.size(); i++) {
// Must make new column if position has changed
PColumn column = allColumns.get(allColumnsIndexes[i]);
projectedColumns.add(column.getPosition() == i ? column : new PColumnImpl(column, i));
}
// Build table from projectedColumns
PTable projectedTable = PTableImpl.makePTable(table, projectedColumns);
SelectStatement select =
SelectStatement.create(SelectStatement.COUNT_ONE, upsert.getHint());
final RowProjector aggProjector =
ProjectionCompiler.compile(queryPlan.getContext(), select, GroupBy.EMPTY_GROUP_BY);
/*
* Transfer over PTable representing subset of columns selected, but all PK columns.
* Move columns setting PK first in pkSlot order, adding LiteralExpression of null for any missing ones.
* Transfer over List<Expression> for projection.
* In region scan, evaluate expressions in order, collecting first n columns for PK and collection non PK in mutation Map
* Create the PRow and get the mutations, adding them to the batch
*/
final StatementContext context = queryPlan.getContext();
final Scan scan = context.getScan();
scan.setAttribute(
BaseScannerRegionObserver.UPSERT_SELECT_TABLE,
UngroupedAggregateRegionObserver.serialize(projectedTable));
scan.setAttribute(
BaseScannerRegionObserver.UPSERT_SELECT_EXPRS,
UngroupedAggregateRegionObserver.serialize(projectedExpressions));
// Ignore order by - it has no impact
final QueryPlan aggPlan =
new AggregatePlan(
context,
select,
tableRef,
aggProjector,
null,
OrderBy.EMPTY_ORDER_BY,
null,
GroupBy.EMPTY_GROUP_BY,
null);
return new MutationPlan() {
@Override
public PhoenixConnection getConnection() {
return connection;
}
@Override
public ParameterMetaData getParameterMetaData() {
return queryPlan.getContext().getBindManager().getParameterMetaData();
}
@Override
public StatementContext getContext() {
return queryPlan.getContext();
}
@Override
public MutationState execute() throws SQLException {
ImmutableBytesWritable ptr = context.getTempPtr();
tableRef.getTable().getIndexMaintainers(ptr);
ServerCache cache = null;
try {
if (ptr.getLength() > 0) {
IndexMetaDataCacheClient client =
new IndexMetaDataCacheClient(connection, tableRef);
cache = client.addIndexMetadataCache(context.getScanRanges(), ptr);
byte[] uuidValue = cache.getId();
scan.setAttribute(PhoenixIndexCodec.INDEX_UUID, uuidValue);
}
ResultIterator iterator = aggPlan.iterator();
try {
Tuple row = iterator.next();
final long mutationCount =
(Long) aggProjector.getColumnProjector(0).getValue(row, PDataType.LONG, ptr);
return new MutationState(maxSize, connection) {
@Override
public long getUpdateCount() {
return mutationCount;
}
};
} finally {
iterator.close();
}
} finally {
if (cache != null) {
cache.close();
}
}
}
@Override
public ExplainPlan getExplainPlan() throws SQLException {
List<String> queryPlanSteps = aggPlan.getExplainPlan().getPlanSteps();
List<String> planSteps =
Lists.newArrayListWithExpectedSize(queryPlanSteps.size() + 1);
planSteps.add("UPSERT ROWS");
planSteps.addAll(queryPlanSteps);
return new ExplainPlan(planSteps);
}
};
}
}
////////////////////////////////////////////////////////////////////
// UPSERT SELECT run client-side
/////////////////////////////////////////////////////////////////////
return new MutationPlan() {
@Override
public PhoenixConnection getConnection() {
return connection;
}
@Override
public ParameterMetaData getParameterMetaData() {
return queryPlan.getContext().getBindManager().getParameterMetaData();
}
@Override
public StatementContext getContext() {
return queryPlan.getContext();
}
@Override
public MutationState execute() throws SQLException {
ResultIterator iterator = queryPlan.iterator();
if (upsertParallelIteratorFactory == null) {
return upsertSelect(
statement, tableRef, projector, iterator, columnIndexes, pkSlotIndexes);
}
upsertParallelIteratorFactory.setRowProjector(projector);
upsertParallelIteratorFactory.setColumnIndexes(columnIndexes);
upsertParallelIteratorFactory.setPkSlotIndexes(pkSlotIndexes);
Tuple tuple;
long totalRowCount = 0;
while ((tuple = iterator.next()) != null) { // Runs query
Cell kv = tuple.getValue(0);
totalRowCount +=
PDataType.LONG
.getCodec()
.decodeLong(kv.getValueArray(), kv.getValueOffset(), SortOrder.getDefault());
}
// Return total number of rows that have been updated. In the case of auto commit being
// off
// the mutations will all be in the mutation state of the current connection.
return new MutationState(maxSize, statement.getConnection(), totalRowCount);
}
@Override
public ExplainPlan getExplainPlan() throws SQLException {
List<String> queryPlanSteps = queryPlan.getExplainPlan().getPlanSteps();
List<String> planSteps = Lists.newArrayListWithExpectedSize(queryPlanSteps.size() + 1);
planSteps.add("UPSERT SELECT");
planSteps.addAll(queryPlanSteps);
return new ExplainPlan(planSteps);
}
};
}
////////////////////////////////////////////////////////////////////
// UPSERT VALUES
/////////////////////////////////////////////////////////////////////
int nodeIndex = 0;
// initialze values with constant byte values first
final byte[][] values = new byte[nValuesToSet][];
if (isTenantSpecific) {
values[nodeIndex++] = connection.getTenantId().getBytes();
}
if (isSharedViewIndex) {
values[nodeIndex++] = MetaDataUtil.getViewIndexIdDataType().toBytes(table.getViewIndexId());
}
final int nodeIndexOffset = nodeIndex;
// Allocate array based on size of all columns in table,
// since some values may not be set (if they're nullable).
final StatementContext context = new StatementContext(statement, resolver, new Scan());
UpsertValuesCompiler expressionBuilder = new UpsertValuesCompiler(context);
final List<Expression> constantExpressions =
Lists.newArrayListWithExpectedSize(valueNodes.size());
// First build all the expressions, as with sequences we want to collect them all first
// and initialize them in one batch
for (ParseNode valueNode : valueNodes) {
if (!valueNode.isStateless()) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.VALUE_IN_UPSERT_NOT_CONSTANT)
.build()
.buildException();
}
PColumn column = allColumns.get(columnIndexes[nodeIndex]);
expressionBuilder.setColumn(column);
Expression expression = valueNode.accept(expressionBuilder);
if (expression.getDataType() != null
&& !expression.getDataType().isCastableTo(column.getDataType())) {
throw TypeMismatchException.newException(
expression.getDataType(),
column.getDataType(),
"expression: " + expression.toString() + " in column " + column);
}
constantExpressions.add(expression);
nodeIndex++;
}
return new MutationPlan() {
@Override
public PhoenixConnection getConnection() {
return connection;
}
@Override
public ParameterMetaData getParameterMetaData() {
return context.getBindManager().getParameterMetaData();
}
@Override
public StatementContext getContext() {
return context;
}
@Override
public MutationState execute() throws SQLException {
ImmutableBytesWritable ptr = context.getTempPtr();
final SequenceManager sequenceManager = context.getSequenceManager();
// Next evaluate all the expressions
int nodeIndex = nodeIndexOffset;
Tuple tuple =
sequenceManager.getSequenceCount() == 0 ? null : sequenceManager.newSequenceTuple(null);
for (Expression constantExpression : constantExpressions) {
PColumn column = allColumns.get(columnIndexes[nodeIndex]);
constantExpression.evaluate(tuple, ptr);
Object value = null;
if (constantExpression.getDataType() != null) {
value =
constantExpression
.getDataType()
.toObject(
ptr,
constantExpression.getSortOrder(),
constantExpression.getMaxLength(),
constantExpression.getScale());
if (!constantExpression.getDataType().isCoercibleTo(column.getDataType(), value)) {
throw TypeMismatchException.newException(
constantExpression.getDataType(),
column.getDataType(),
"expression: " + constantExpression.toString() + " in column " + column);
}
if (!column
.getDataType()
.isSizeCompatible(
ptr,
value,
constantExpression.getDataType(),
constantExpression.getMaxLength(),
constantExpression.getScale(),
column.getMaxLength(),
column.getScale())) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.DATA_EXCEEDS_MAX_CAPACITY)
.setColumnName(column.getName().getString())
.setMessage("value=" + constantExpression.toString())
.build()
.buildException();
}
}
column
.getDataType()
.coerceBytes(
ptr,
value,
constantExpression.getDataType(),
constantExpression.getMaxLength(),
constantExpression.getScale(),
constantExpression.getSortOrder(),
column.getMaxLength(),
column.getScale(),
column.getSortOrder());
if (overlapViewColumns.contains(column)
&& Bytes.compareTo(
ptr.get(),
ptr.getOffset(),
ptr.getLength(),
column.getViewConstant(),
0,
column.getViewConstant().length - 1)
!= 0) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_UPDATE_VIEW_COLUMN)
.setColumnName(column.getName().getString())
.setMessage("value=" + constantExpression.toString())
.build()
.buildException();
}
values[nodeIndex] = ByteUtil.copyKeyBytesIfNecessary(ptr);
nodeIndex++;
}
// Add columns based on view
for (PColumn column : addViewColumns) {
if (IndexUtil.getViewConstantValue(column, ptr)) {
values[nodeIndex++] = ByteUtil.copyKeyBytesIfNecessary(ptr);
} else {
throw new IllegalStateException();
}
}
Map<ImmutableBytesPtr, Map<PColumn, byte[]>> mutation = Maps.newHashMapWithExpectedSize(1);
setValues(values, pkSlotIndexes, columnIndexes, tableRef.getTable(), mutation);
return new MutationState(tableRef, mutation, 0, maxSize, connection);
}
@Override
public ExplainPlan getExplainPlan() throws SQLException {
List<String> planSteps = Lists.newArrayListWithExpectedSize(2);
if (context.getSequenceManager().getSequenceCount() > 0) {
planSteps.add(
"CLIENT RESERVE " + context.getSequenceManager().getSequenceCount() + " SEQUENCES");
}
planSteps.add("PUT SINGLE ROW");
return new ExplainPlan(planSteps);
}
};
}
private static MutationState upsertSelect(
PhoenixStatement statement,
TableRef tableRef,
RowProjector projector,
ResultIterator iterator,
int[] columnIndexes,
int[] pkSlotIndexes)
throws SQLException {
try {
PhoenixConnection connection = statement.getConnection();
ConnectionQueryServices services = connection.getQueryServices();
int maxSize =
services
.getProps()
.getInt(
QueryServices.MAX_MUTATION_SIZE_ATTRIB,
QueryServicesOptions.DEFAULT_MAX_MUTATION_SIZE);
int batchSize = Math.min(connection.getMutateBatchSize(), maxSize);
boolean isAutoCommit = connection.getAutoCommit();
byte[][] values = new byte[columnIndexes.length][];
int rowCount = 0;
Map<ImmutableBytesPtr, Map<PColumn, byte[]>> mutation =
Maps.newHashMapWithExpectedSize(batchSize);
PTable table = tableRef.getTable();
ResultSet rs = new PhoenixResultSet(iterator, projector, statement);
ImmutableBytesWritable ptr = new ImmutableBytesWritable();
while (rs.next()) {
for (int i = 0; i < values.length; i++) {
PColumn column = table.getColumns().get(columnIndexes[i]);
byte[] bytes = rs.getBytes(i + 1);
ptr.set(bytes == null ? ByteUtil.EMPTY_BYTE_ARRAY : bytes);
Object value = rs.getObject(i + 1);
int rsPrecision = rs.getMetaData().getPrecision(i + 1);
Integer precision = rsPrecision == 0 ? null : rsPrecision;
int rsScale = rs.getMetaData().getScale(i + 1);
Integer scale = rsScale == 0 ? null : rsScale;
// We are guaranteed that the two column will have compatible types,
// as we checked that before.
if (!column
.getDataType()
.isSizeCompatible(
ptr,
value,
column.getDataType(),
precision,
scale,
column.getMaxLength(),
column.getScale())) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.DATA_EXCEEDS_MAX_CAPACITY)
.setColumnName(column.getName().getString())
.setMessage("value=" + column.getDataType().toStringLiteral(ptr, null))
.build()
.buildException();
}
column
.getDataType()
.coerceBytes(
ptr,
value,
column.getDataType(),
precision,
scale,
SortOrder.getDefault(),
column.getMaxLength(),
column.getScale(),
column.getSortOrder());
values[i] = ByteUtil.copyKeyBytesIfNecessary(ptr);
}
setValues(values, pkSlotIndexes, columnIndexes, table, mutation);
rowCount++;
// Commit a batch if auto commit is true and we're at our batch size
if (isAutoCommit && rowCount % batchSize == 0) {
MutationState state = new MutationState(tableRef, mutation, 0, maxSize, connection);
connection.getMutationState().join(state);
connection.commit();
mutation.clear();
}
}
// If auto commit is true, this last batch will be committed upon return
return new MutationState(
tableRef, mutation, rowCount / batchSize * batchSize, maxSize, connection);
} finally {
iterator.close();
}
}
private static QueryPlan addPlan(
PhoenixStatement statement,
SelectStatement select,
PTable index,
List<? extends PDatum> targetColumns,
ParallelIteratorFactory parallelIteratorFactory,
QueryPlan dataPlan,
boolean isHinted)
throws SQLException {
int nColumns = dataPlan.getProjector().getColumnCount();
String tableAlias = dataPlan.getTableRef().getTableAlias();
String alias =
tableAlias == null
? null
: '"' + tableAlias + '"'; // double quote in case it's case sensitive
String schemaName = index.getParentSchemaName().getString();
schemaName = schemaName.length() == 0 ? null : '"' + schemaName + '"';
String tableName = '"' + index.getTableName().getString() + '"';
TableNode table = FACTORY.namedTable(alias, FACTORY.table(schemaName, tableName));
SelectStatement indexSelect = FACTORY.select(select, table);
ColumnResolver resolver =
FromCompiler.getResolverForQuery(indexSelect, statement.getConnection());
// We will or will not do tuple projection according to the data plan.
boolean isProjected =
dataPlan.getContext().getResolver().getTables().get(0).getTable().getType()
== PTableType.PROJECTED;
// Check index state of now potentially updated index table to make sure it's active
if (PIndexState.ACTIVE.equals(resolver.getTables().get(0).getTable().getIndexState())) {
try {
// translate nodes that match expressions that are indexed to the associated column parse
// node
indexSelect =
ParseNodeRewriter.rewrite(
indexSelect,
new IndexExpressionParseNodeRewriter(
index, statement.getConnection(), indexSelect.getUdfParseNodes()));
QueryCompiler compiler =
new QueryCompiler(
statement,
indexSelect,
resolver,
targetColumns,
parallelIteratorFactory,
dataPlan.getContext().getSequenceManager(),
isProjected);
QueryPlan plan = compiler.compile();
// If query doesn't have where clause and some of columns to project are missing
// in the index then we need to get missing columns from main table for each row in
// local index. It's like full scan of both local index and data table which is inefficient.
// Then we don't use the index. If all the columns to project are present in the index
// then we can use the index even the query doesn't have where clause.
if (index.getIndexType() == IndexType.LOCAL
&& indexSelect.getWhere() == null
&& !plan.getContext().getDataColumns().isEmpty()) {
return null;
}
// Checking number of columns handles the wildcard cases correctly, as in that case the
// index
// must contain all columns from the data table to be able to be used.
if (plan.getTableRef().getTable().getIndexState() == PIndexState.ACTIVE) {
if (plan.getProjector().getColumnCount() == nColumns) {
return plan;
} else if (index.getIndexType() == IndexType.GLOBAL) {
throw new ColumnNotFoundException("*");
}
}
} catch (ColumnNotFoundException e) {
/* Means that a column is being used that's not in our index.
* Since we currently don't keep stats, we don't know the selectivity of the index.
* For now, if this is a hinted plan, we will try rewriting the query as a subquery;
* otherwise we just don't use this index (as opposed to trying to join back from
* the index table to the data table.
*/
SelectStatement dataSelect = (SelectStatement) dataPlan.getStatement();
ParseNode where = dataSelect.getWhere();
if (isHinted && where != null) {
StatementContext context = new StatementContext(statement, resolver);
WhereConditionRewriter whereRewriter =
new WhereConditionRewriter(FromCompiler.getResolver(dataPlan.getTableRef()), context);
where = where.accept(whereRewriter);
if (where != null) {
PTable dataTable = dataPlan.getTableRef().getTable();
List<PColumn> pkColumns = dataTable.getPKColumns();
List<AliasedNode> aliasedNodes =
Lists.<AliasedNode>newArrayListWithExpectedSize(pkColumns.size());
List<ParseNode> nodes = Lists.<ParseNode>newArrayListWithExpectedSize(pkColumns.size());
boolean isSalted = dataTable.getBucketNum() != null;
boolean isTenantSpecific =
dataTable.isMultiTenant() && statement.getConnection().getTenantId() != null;
int posOffset = (isSalted ? 1 : 0) + (isTenantSpecific ? 1 : 0);
for (int i = posOffset; i < pkColumns.size(); i++) {
PColumn column = pkColumns.get(i);
String indexColName = IndexUtil.getIndexColumnName(column);
ParseNode indexColNode =
new ColumnParseNode(null, '"' + indexColName + '"', indexColName);
PDataType indexColType = IndexUtil.getIndexColumnDataType(column);
PDataType dataColType = column.getDataType();
if (indexColType != dataColType) {
indexColNode = FACTORY.cast(indexColNode, dataColType, null, null);
}
aliasedNodes.add(FACTORY.aliasedNode(null, indexColNode));
nodes.add(new ColumnParseNode(null, '"' + column.getName().getString() + '"'));
}
SelectStatement innerSelect =
FACTORY.select(
indexSelect.getFrom(),
indexSelect.getHint(),
false,
aliasedNodes,
where,
null,
null,
null,
null,
null,
indexSelect.getBindCount(),
false,
indexSelect.hasSequence(),
Collections.<SelectStatement>emptyList(),
indexSelect.getUdfParseNodes());
ParseNode outerWhere =
FACTORY.in(
nodes.size() == 1 ? nodes.get(0) : FACTORY.rowValueConstructor(nodes),
FACTORY.subquery(innerSelect, false),
false,
true);
ParseNode extractedCondition = whereRewriter.getExtractedCondition();
if (extractedCondition != null) {
outerWhere = FACTORY.and(Lists.newArrayList(outerWhere, extractedCondition));
}
HintNode hint =
HintNode.combine(
HintNode.subtract(
indexSelect.getHint(),
new Hint[] {Hint.INDEX, Hint.NO_CHILD_PARENT_JOIN_OPTIMIZATION}),
FACTORY.hint("NO_INDEX"));
SelectStatement query = FACTORY.select(dataSelect, hint, outerWhere);
ColumnResolver queryResolver =
FromCompiler.getResolverForQuery(query, statement.getConnection());
query = SubqueryRewriter.transform(query, queryResolver, statement.getConnection());
queryResolver = FromCompiler.getResolverForQuery(query, statement.getConnection());
query = StatementNormalizer.normalize(query, queryResolver);
QueryPlan plan =
new QueryCompiler(
statement,
query,
queryResolver,
targetColumns,
parallelIteratorFactory,
dataPlan.getContext().getSequenceManager(),
isProjected)
.compile();
return plan;
}
}
}
}
return null;
}