private List<Future> getBatches(DataLoadThreadTime dataLoadThreadTime, Scenario scenario) throws Exception { RowCalculator rowCalculator = new RowCalculator(getThreadPoolSize(), scenario.getRowCount()); List<Future> writeBatches = new ArrayList<>(); for (int i = 0; i < getThreadPoolSize(); i++) { List<Column> phxMetaCols = pUtil.getColumnsFromPhoenix( scenario.getSchemaName(), scenario.getTableNameWithoutSchemaName(), pUtil.getConnection()); int threadRowCount = rowCalculator.getNext(); logger.info("Kick off thread (#" + i + ")for upsert with (" + threadRowCount + ") rows."); Future<Info> write = upsertData( scenario, phxMetaCols, scenario.getTableName(), threadRowCount, dataLoadThreadTime); writeBatches.add(write); } if (writeBatches.isEmpty()) { throw new PherfException( "Holy shit snacks! Throwing up hands in disbelief and exiting. Could not write data for some unknown reason."); } return writeBatches; }
public ScenarioResult(Scenario scenario) { this.setDataOverride(scenario.getDataOverride()); this.setPhoenixProperties(scenario.getPhoenixProperties()); this.setRowCount(scenario.getRowCount()); this.setTableName(scenario.getTableName()); this.setName(scenario.getName()); }
private synchronized void exec( DataLoadTimeSummary dataLoadTimeSummary, DataLoadThreadTime dataLoadThreadTime, Scenario scenario) throws Exception { logger.info("\nLoading " + scenario.getRowCount() + " rows for " + scenario.getTableName()); long start = System.currentTimeMillis(); List<Future> writeBatches = getBatches(dataLoadThreadTime, scenario); waitForBatches(dataLoadTimeSummary, scenario, start, writeBatches); // always update stats for Phoenix base tables updatePhoenixStats(scenario.getTableName()); }