/**
* Get the list of uncommitted KeyValues for the connection. Currently used to write an
* Phoenix-compliant HFile from a map/reduce job.
*
* @param conn an open JDBC connection
* @return the list of HBase mutations for uncommitted data
* @throws SQLException
*/
public static Iterator<Pair<byte[], List<KeyValue>>> getUncommittedDataIterator(
Connection conn, boolean includeMutableIndexes) throws SQLException {
final PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
final Iterator<Pair<byte[], List<Mutation>>> iterator =
pconn.getMutationState().toMutations(includeMutableIndexes);
return new Iterator<Pair<byte[], List<KeyValue>>>() {
@Override
public boolean hasNext() {
return iterator.hasNext();
}
@Override
public Pair<byte[], List<KeyValue>> next() {
Pair<byte[], List<Mutation>> pair = iterator.next();
List<KeyValue> keyValues =
Lists.newArrayListWithExpectedSize(
pair.getSecond().size() * 5); // Guess-timate 5 key values per row
for (Mutation mutation : pair.getSecond()) {
for (List<Cell> keyValueList : mutation.getFamilyCellMap().values()) {
for (Cell keyValue : keyValueList) {
keyValues.add(org.apache.hadoop.hbase.KeyValueUtil.ensureKeyValue(keyValue));
}
}
}
Collections.sort(keyValues, pconn.getKeyValueBuilder().getKeyValueComparator());
return new Pair<byte[], List<KeyValue>>(pair.getFirst(), keyValues);
}
@Override
public void remove() {
throw new UnsupportedOperationException();
}
};
}
@Override
protected void map(NullWritable key, PhoenixIndexDBWritable record, Context context)
throws IOException, InterruptedException {
context.getCounter(PhoenixJobCounters.INPUT_RECORDS).increment(1);
try {
final List<Object> values = record.getValues();
indxWritable.setValues(values);
indxWritable.write(this.pStatement);
this.pStatement.execute();
final PhoenixConnection pconn = connection.unwrap(PhoenixConnection.class);
MutationState currentMutationState = pconn.getMutationState();
if (mutationState == null) {
mutationState = currentMutationState;
return;
}
// Keep accumulating Mutations till batch size
mutationState.join(currentMutationState);
// Write Mutation Batch
if (context.getCounter(PhoenixJobCounters.INPUT_RECORDS).getValue() % batchSize == 0) {
writeBatch(mutationState, context);
mutationState = null;
}
// Make sure progress is reported to Application Master.
context.progress();
} catch (SQLException e) {
LOG.error(" Error {} while read/write of a record ", e.getMessage());
context.getCounter(PhoenixJobCounters.FAILED_RECORDS).increment(1);
throw new RuntimeException(e);
}
}
private static MutationState upsertSelect(
PhoenixStatement statement,
TableRef tableRef,
RowProjector projector,
ResultIterator iterator,
int[] columnIndexes,
int[] pkSlotIndexes)
throws SQLException {
try {
PhoenixConnection connection = statement.getConnection();
ConnectionQueryServices services = connection.getQueryServices();
int maxSize =
services
.getProps()
.getInt(
QueryServices.MAX_MUTATION_SIZE_ATTRIB,
QueryServicesOptions.DEFAULT_MAX_MUTATION_SIZE);
int batchSize = Math.min(connection.getMutateBatchSize(), maxSize);
boolean isAutoCommit = connection.getAutoCommit();
byte[][] values = new byte[columnIndexes.length][];
int rowCount = 0;
Map<ImmutableBytesPtr, Map<PColumn, byte[]>> mutation =
Maps.newHashMapWithExpectedSize(batchSize);
PTable table = tableRef.getTable();
ResultSet rs = new PhoenixResultSet(iterator, projector, statement);
ImmutableBytesWritable ptr = new ImmutableBytesWritable();
while (rs.next()) {
for (int i = 0; i < values.length; i++) {
PColumn column = table.getColumns().get(columnIndexes[i]);
byte[] bytes = rs.getBytes(i + 1);
ptr.set(bytes == null ? ByteUtil.EMPTY_BYTE_ARRAY : bytes);
Object value = rs.getObject(i + 1);
int rsPrecision = rs.getMetaData().getPrecision(i + 1);
Integer precision = rsPrecision == 0 ? null : rsPrecision;
int rsScale = rs.getMetaData().getScale(i + 1);
Integer scale = rsScale == 0 ? null : rsScale;
// We are guaranteed that the two column will have compatible types,
// as we checked that before.
if (!column
.getDataType()
.isSizeCompatible(
ptr,
value,
column.getDataType(),
precision,
scale,
column.getMaxLength(),
column.getScale())) {
throw new SQLExceptionInfo.Builder(SQLExceptionCode.DATA_EXCEEDS_MAX_CAPACITY)
.setColumnName(column.getName().getString())
.setMessage("value=" + column.getDataType().toStringLiteral(ptr, null))
.build()
.buildException();
}
column
.getDataType()
.coerceBytes(
ptr,
value,
column.getDataType(),
precision,
scale,
SortOrder.getDefault(),
column.getMaxLength(),
column.getScale(),
column.getSortOrder());
values[i] = ByteUtil.copyKeyBytesIfNecessary(ptr);
}
setValues(values, pkSlotIndexes, columnIndexes, table, mutation);
rowCount++;
// Commit a batch if auto commit is true and we're at our batch size
if (isAutoCommit && rowCount % batchSize == 0) {
MutationState state = new MutationState(tableRef, mutation, 0, maxSize, connection);
connection.getMutationState().join(state);
connection.commit();
mutation.clear();
}
}
// If auto commit is true, this last batch will be committed upon return
return new MutationState(
tableRef, mutation, rowCount / batchSize * batchSize, maxSize, connection);
} finally {
iterator.close();
}
}
@Override
public PeekingResultIterator newIterator(
final StatementContext parentContext,
ResultIterator iterator,
Scan scan,
String tableName,
QueryPlan plan)
throws SQLException {
final PhoenixConnection clonedConnection = new PhoenixConnection(this.connection);
MutationState state = mutate(parentContext, iterator, clonedConnection);
long totalRowCount = state.getUpdateCount();
if (clonedConnection.getAutoCommit()) {
clonedConnection.getMutationState().join(state);
state = clonedConnection.getMutationState();
}
final MutationState finalState = state;
byte[] value = PLong.INSTANCE.toBytes(totalRowCount);
KeyValue keyValue =
KeyValueUtil.newKeyValue(
UNGROUPED_AGG_ROW_KEY,
SINGLE_COLUMN_FAMILY,
SINGLE_COLUMN,
AGG_TIMESTAMP,
value,
0,
value.length);
final Tuple tuple = new SingleKeyValueTuple(keyValue);
return new PeekingResultIterator() {
private boolean done = false;
@Override
public Tuple next() throws SQLException {
if (done) {
return null;
}
done = true;
return tuple;
}
@Override
public void explain(List<String> planSteps) {}
@Override
public void close() throws SQLException {
try {
/*
* Join the child mutation states in close, since this is called in a single threaded manner
* after the parallel results have been processed.
* If auto-commit is on for the cloned child connection, then the finalState here is an empty mutation
* state (with no mutations). However, it still has the metrics for mutation work done by the
* mutating-iterator. Joining the mutation state makes sure those metrics are passed over
* to the parent connection.
*/
MutatingParallelIteratorFactory.this.connection.getMutationState().join(finalState);
} finally {
clonedConnection.close();
}
}
@Override
public Tuple peek() throws SQLException {
return done ? null : tuple;
}
};
}