@Override protected void map(NullWritable key, PhoenixIndexDBWritable record, Context context) throws IOException, InterruptedException { context.getCounter(PhoenixJobCounters.INPUT_RECORDS).increment(1); try { final List<Object> values = record.getValues(); indxWritable.setValues(values); indxWritable.write(this.pStatement); this.pStatement.execute(); final PhoenixConnection pconn = connection.unwrap(PhoenixConnection.class); MutationState currentMutationState = pconn.getMutationState(); if (mutationState == null) { mutationState = currentMutationState; return; } // Keep accumulating Mutations till batch size mutationState.join(currentMutationState); // Write Mutation Batch if (context.getCounter(PhoenixJobCounters.INPUT_RECORDS).getValue() % batchSize == 0) { writeBatch(mutationState, context); mutationState = null; } // Make sure progress is reported to Application Master. context.progress(); } catch (SQLException e) { LOG.error(" Error {} while read/write of a record ", e.getMessage()); context.getCounter(PhoenixJobCounters.FAILED_RECORDS).increment(1); throw new RuntimeException(e); } }
@Override protected void setup(final Context context) throws IOException, InterruptedException { super.setup(context); final Configuration configuration = context.getConfiguration(); writer = new DirectHTableWriter(configuration); try { indxTblColumnMetadata = PhoenixConfigurationUtil.getUpsertColumnMetadataList(configuration); indxWritable.setColumnMetadata(indxTblColumnMetadata); final Properties overrideProps = new Properties(); String scn = configuration.get(PhoenixConfigurationUtil.CURRENT_SCN_VALUE); String txScnValue = configuration.get(PhoenixConfigurationUtil.TX_SCN_VALUE); if (txScnValue == null) { overrideProps.put(PhoenixRuntime.CURRENT_SCN_ATTRIB, scn); } connection = ConnectionUtil.getOutputConnection(configuration, overrideProps); connection.setAutoCommit(false); // Get BatchSize ConnectionQueryServices services = ((PhoenixConnection) connection).getQueryServices(); int maxSize = services .getProps() .getInt( QueryServices.MAX_MUTATION_SIZE_ATTRIB, QueryServicesOptions.DEFAULT_MAX_MUTATION_SIZE); batchSize = Math.min(((PhoenixConnection) connection).getMutateBatchSize(), maxSize); LOG.info("Mutation Batch Size = " + batchSize); final String upsertQuery = PhoenixConfigurationUtil.getUpsertStatement(configuration); this.pStatement = connection.prepareStatement(upsertQuery); } catch (SQLException e) { throw new RuntimeException(e.getMessage()); } }