private List<Future> getBatches(DataLoadThreadTime dataLoadThreadTime, Scenario scenario)
throws Exception {
RowCalculator rowCalculator = new RowCalculator(getThreadPoolSize(), scenario.getRowCount());
List<Future> writeBatches = new ArrayList<>();
for (int i = 0; i < getThreadPoolSize(); i++) {
List<Column> phxMetaCols =
pUtil.getColumnsFromPhoenix(
scenario.getSchemaName(),
scenario.getTableNameWithoutSchemaName(),
pUtil.getConnection());
int threadRowCount = rowCalculator.getNext();
logger.info("Kick off thread (#" + i + ")for upsert with (" + threadRowCount + ") rows.");
Future<Info> write =
upsertData(
scenario, phxMetaCols, scenario.getTableName(), threadRowCount, dataLoadThreadTime);
writeBatches.add(write);
}
if (writeBatches.isEmpty()) {
throw new PherfException(
"Holy shit snacks! Throwing up hands in disbelief and exiting. Could not write data for some unknown reason.");
}
return writeBatches;
}
/**
* Timed query execution
*
* @throws Exception
*/
private void timedQuery() throws Exception {
boolean isSelectCountStatement =
query.getStatement().toUpperCase().trim().contains("COUNT(*)") ? true : false;
Connection conn = null;
PreparedStatement statement = null;
ResultSet rs = null;
Long start = System.currentTimeMillis();
Date startDate = Calendar.getInstance().getTime();
String exception = null;
long resultRowCount = 0;
try {
conn = pUtil.getConnection(query.getTenantId());
statement = conn.prepareStatement(query.getStatement());
boolean isQuery = statement.execute();
if (isQuery) {
rs = statement.getResultSet();
while (rs.next()) {
if (null != query.getExpectedAggregateRowCount()) {
if (rs.getLong(1) != query.getExpectedAggregateRowCount())
throw new RuntimeException(
"Aggregate count "
+ rs.getLong(1)
+ " does not match expected "
+ query.getExpectedAggregateRowCount());
}
if (isSelectCountStatement) {
resultRowCount = rs.getLong(1);
} else {
resultRowCount++;
}
}
} else {
conn.commit();
}
} catch (Exception e) {
e.printStackTrace();
exception = e.getMessage();
} finally {
getThreadTime()
.getRunTimesInMs()
.add(
new RunTime(
exception,
startDate,
resultRowCount,
(int) (System.currentTimeMillis() - start)));
if (rs != null) rs.close();
if (statement != null) statement.close();
if (conn != null) conn.close();
}
}
public WriteWorkload(XMLConfigParser parser) throws Exception {
this(PhoenixUtil.create(), parser);
}
/**
* TODO Move this method to PhoenixUtil Update Phoenix table stats
*
* @param tableName
* @throws Exception
*/
public void updatePhoenixStats(String tableName) throws Exception {
logger.info("Updating stats for " + tableName);
pUtil.executeStatement("UPDATE STATISTICS " + tableName);
}
class MultiThreadedRunner implements Runnable {
private static final Logger logger = LoggerFactory.getLogger(MultiThreadedRunner.class);
private Query query;
private ThreadTime threadTime;
private PhoenixUtil pUtil = PhoenixUtil.create();
private String threadName;
private DataModelResult dataModelResult;
private long numberOfExecutions;
private long executionDurationInMs;
private static long lastResultWritten = System.currentTimeMillis() - 1000;
private final ResultManager resultManager;
/**
* MultiThreadedRunner
*
* @param threadName
* @param query
* @param dataModelResult
* @param threadTime
* @param numberOfExecutions
* @param executionDurationInMs
*/
MultiThreadedRunner(
String threadName,
Query query,
DataModelResult dataModelResult,
ThreadTime threadTime,
long numberOfExecutions,
long executionDurationInMs,
boolean writeRuntimeResults) {
this.query = query;
this.threadName = threadName;
this.threadTime = threadTime;
this.dataModelResult = dataModelResult;
this.numberOfExecutions = numberOfExecutions;
this.executionDurationInMs = executionDurationInMs;
this.resultManager = new ResultManager(dataModelResult.getName(), writeRuntimeResults);
}
/** Executes run for a minimum of number of execution or execution duration */
@Override
public void run() {
logger.info(
"\n\nThread Starting "
+ threadName
+ " ; "
+ query.getStatement()
+ " for "
+ numberOfExecutions
+ "times\n\n");
Long start = System.currentTimeMillis();
for (long i = numberOfExecutions;
(i > 0 && ((System.currentTimeMillis() - start) < executionDurationInMs));
i--) {
try {
synchronized (resultManager) {
timedQuery();
if ((System.currentTimeMillis() - lastResultWritten) > 1000) {
resultManager.write(dataModelResult);
lastResultWritten = System.currentTimeMillis();
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
logger.info("\n\nThread exiting." + threadName + "\n\n");
}
private synchronized ThreadTime getThreadTime() {
return threadTime;
}
/**
* Timed query execution
*
* @throws Exception
*/
private void timedQuery() throws Exception {
boolean isSelectCountStatement =
query.getStatement().toUpperCase().trim().contains("COUNT(*)") ? true : false;
Connection conn = null;
PreparedStatement statement = null;
ResultSet rs = null;
Long start = System.currentTimeMillis();
Date startDate = Calendar.getInstance().getTime();
String exception = null;
long resultRowCount = 0;
try {
conn = pUtil.getConnection(query.getTenantId());
statement = conn.prepareStatement(query.getStatement());
boolean isQuery = statement.execute();
if (isQuery) {
rs = statement.getResultSet();
while (rs.next()) {
if (null != query.getExpectedAggregateRowCount()) {
if (rs.getLong(1) != query.getExpectedAggregateRowCount())
throw new RuntimeException(
"Aggregate count "
+ rs.getLong(1)
+ " does not match expected "
+ query.getExpectedAggregateRowCount());
}
if (isSelectCountStatement) {
resultRowCount = rs.getLong(1);
} else {
resultRowCount++;
}
}
} else {
conn.commit();
}
} catch (Exception e) {
e.printStackTrace();
exception = e.getMessage();
} finally {
getThreadTime()
.getRunTimesInMs()
.add(
new RunTime(
exception,
startDate,
resultRowCount,
(int) (System.currentTimeMillis() - start)));
if (rs != null) rs.close();
if (statement != null) statement.close();
if (conn != null) conn.close();
}
}
}