private boolean processBatchResults(
int currentQueryThreadIndex, QueryRunnerIntermediaryResult previousThreadResults)
throws TechnicalException, FunctionalException {
if (!first) {
outln("Merging results");
}
Timer intermediaryResultsProcessingTimer = null;
if (!this.first && !this.last) {
intermediaryResultsProcessingTimer = new Timer();
intermediaryResultsProcessingTimer.startTimer();
}
// Prepare results for join with the next query
this.batchResult = prepareResults(previousThreadResults, currentBatchResults);
if (!last) {
// wait till next thread is start processing the latest results
List<Status> nextReadyList = nextQueryThread.getReadyList();
List<Status> tmp = null;
Status firstStatus = null;
synchronized (nextReadyList) {
outln("wait for next thread to be ready");
while (nextReadyList.isEmpty()) {
try {
nextReadyList.wait();
} catch (InterruptedException e) {
throw new TechnicalException(e);
}
}
if (!nextReadyList.isEmpty()) {
firstStatus = nextReadyList.remove(0);
} else {
MyUtils.errorProgram("SPONTANEOUS NOTIFICATION!!!!!!!!"); // shouldn't happen really
}
tmp = new ArrayList<Status>(nextReadyList);
}
outln("next status to process: " + firstStatus);
if (checkCancelThread()) { // check here in case we sent a signal because of cancelling
return true;
}
check(tmp.isEmpty(), checklog("nextReadyList = " + nextReadyList));
check(
null
== this
.currentResult); // Must have been freed by next thread (call to
// resetCurrentResult())
// Add the newest results for the next thread to process as soon as it's ready
outln("Add input for next thread");
this.currentResult = batchResult;
synchronized (this.statusList) {
this.statusList.add(Status.RESULTS);
this.statusList.notify(); // To notify next thread
}
} else {
// Add final results
outln("Add final results");
List<Integer> ignoreList = ThreadCommunication.getIgnoreList();
boolean cancelationInProcess =
this.rows != null && ThreadCommunication.getTotalRows() >= this.rows;
if (!cancelationInProcess) {
try {
for (List<String> list : batchResult.getValues()) {
boolean first = true;
for (int i = 0; i < list.size(); i++) {
if (!ignoreList.contains(i)) {
ThreadCommunication.bufferedWriter.write(
(first ? "" : MyUtils.TAB_SEPARATOR) + list.get(i));
if (!QueryRunnerPrototypeConstants.LOG) {
System.out.print((first ? "" : MyUtils.TAB_SEPARATOR) + list.get(i));
}
first = false;
}
}
ThreadCommunication.bufferedWriter.write(MyUtils.LINE_SEPARATOR);
if (!QueryRunnerPrototypeConstants.LOG) {
System.out.println();
}
int totalRows = ThreadCommunication.addRowToTotalRows();
if (!this.unsynchronizedFirstFinalResults && totalRows == 1) { // if first row
outln("notify first results have arrived");
synchronized (this.firstFinalResults) {
this.firstFinalResults.setTrue();
this.firstFinalResults.notify(); // notify main thread
this.unsynchronizedFirstFinalResults = true;
}
}
if (!this.unsynchronizedSomeFinalResults
&& this.rows != null
&& totalRows == this.rows.intValue()) {
outln("notify some results have arrived");
synchronized (this.someFinalResults) {
this.unsynchronizedSomeFinalResults = true;
this.someFinalResults.setTrue();
this.someFinalResults.notify(); // notify main thread
}
break;
}
}
if (QueryRunnerPrototypeConstants.LOG) {
ThreadCommunication.bufferedWriter.flush();
}
} catch (IOException e) {
throw new TechnicalException(e);
}
}
}
this.batchResult = null;
if (!this.first && !this.last) {
intermediaryResultsProcessingTimer.stopTimer();
this.intermediaryResultsProcessingAverageTime.addToAverage(
intermediaryResultsProcessingTimer.getTimeEllapsedMs());
}
return false;
}
private void process() throws TechnicalException, FunctionalException {
try {
Connection connection = connect();
while (true) { // broken when done
this.checkCancelIndex = 0;
if (checkCancelThread()) { // regularily check for cancel request and exit when encountered
break;
}
QueryRunnerIntermediaryResult previousThreadResults = null;
if (!first) {
List<Status> tmp = fetchPreviousThreadNextStatus();
if (checkCancelThread()) { // see above similar line
break;
}
previousThreadResults = processPreviousThreadDoneStatus(tmp);
if (previousThreadResults == null) { // means previous thread is done
break;
}
}
if (checkCancelThread()) { // see above similar line
break;
}
StringBuffer currentQuery = new StringBuffer(this.queryTemplate);
boolean potentialResults = true;
if (first) {
previousThreadResults = new QueryRunnerIntermediaryResult(this.queryThreadIndex);
outln("first: no input needed");
} else {
outln("Processing input");
potentialResults = customizeQuery(previousThreadResults, currentQuery);
}
this.shortenedQuery =
currentQuery.length() < QueryRunnerPrototypeConstants.MAX_DISPLAY_QUERY_LENGTH
? currentQuery.toString()
: (currentQuery
.substring(0, QueryRunnerPrototypeConstants.MAX_DISPLAY_QUERY_LENGTH2)
.toString()
+ " ... "
+ currentQuery
.substring(
currentQuery.length()
- QueryRunnerPrototypeConstants.MAX_DISPLAY_QUERY_LENGTH2,
currentQuery.length())
.toString());
outln(potentialResults + ", " + "shortenedQuery = " + shortenedQuery);
writelnQueryFile(shortenedQuery);
if (checkCancelThread()) { // see above similar line
break;
}
outln("preparing statement");
synchronized (
this
.activeQuery) { // always get the lock on that object to handle the query
// related-object: 2 threads can use them (main one and current one)
lockln("lock acquired on activeQuery");
if (this.preparedStatement == null) {
check(!this.activeQuery.getValue());
check(this.resultSet == null); // not sure
prepareStreamingStatement(connection, currentQuery);
} else {
check(this.activeQuery.getValue());
check(this.resultSet != null); // not sure
this.resultSet.close();
this.preparedStatement.close();
prepareStreamingStatement(connection, currentQuery);
}
this.activeQuery
.setTrue(); // set it to true: meaning cancel would have to actually cancel the query
// and close as much as it can
lockln("lock released on activeQuery");
}
if (checkCancelThread()) { // see above similar line
break;
}
Timer queryTimer = new Timer();
queryTimer.startTimer();
Timer queryLaunchTimer = new Timer();
queryLaunchTimer.startTimer();
Integer columnCount = null;
if (potentialResults) {
outln("executing query");
synchronized (this.activeQuery) { // see above similar line
lockln("lock acquired on activeQuery");
this.resultSet =
this.preparedStatement.executeQuery(
// currentQuery.toString()); // for Statement
); // for PreparedStatement
outln("query prepared");
columnCount = this.resultSet.getMetaData().getColumnCount();
// activeQuery is already set to true here: it means we must always check whether
// resultSet is null or not before trying to close it
lockln("lock released on activeQuery");
}
}
queryLaunchTimer.stopTimer();
this.launchQueryAverageTime.addToAverage(queryLaunchTimer.getTimeEllapsedMs());
if (checkCancelThread()) { // see above similar line
break;
}
int batchNumber = 0; // For the current query
queryRow = 0; // For the current query
Boolean next = null;
boolean queryCancellation = false; // so we can break the 2 embedded query while loops
while ( // 1st query while loop: query one
(potentialResults && ((next = resultSetNext(resultSet)) || batchNumber == 0))
|| (!potentialResults
&& batchNumber == 0)) { // We want to go inside the first time even if no results
this.currentBatchResults = new ArrayList<List<String>>();
int batchRow = 0; // For the current batch for the current query
outln("Fetching batch");
Timer batchTimer = new Timer();
batchTimer.startTimer();
while ( // 2st query while loop: batch one
batchRow < this.batchSize
&& potentialResults
&& ((batchRow == 0 && next) || (batchRow > 0 && resultSetNext(resultSet)))) {
List<String> list = new ArrayList<String>();
// we also lock on that object when retrieving a row
synchronized (this.activeQuery) { // see above similar line
lockln("lock acquired on activeQuery");
if (this.activeQuery.getValue()) {
for (int i = 0; i < columnCount; i++) {
String fieldValue = resultSet.getString(i + 1);
list.add(fieldValue); // accepting null values
}
} else {
queryCancellation = true;
}
lockln("lock released on activeQuery");
if (queryCancellation) { // break 2nd query while loop: batch one (not the main one)
break;
}
}
currentBatchResults.add(list);
batchRow++;
queryRow++;
} // end of the 2nd query while loop (batch)
batchTimer.stopTimer();
this.fetchingBatchAverageTime.addToAverage(batchTimer.getTimeEllapsedMs());
writelnQueryFile(
MyUtils.TAB_SEPARATOR
+ MyUtils.TAB_SEPARATOR
+ batchRow
+ " rows in "
+ batchTimer.getTimeEllapsedMs()
+ " ms");
StringBuffer stringBuffer =
new StringBuffer(); /*stringBuffer.append("{");for (int i = 0; i < currentBatchResults.size(); i++) {stringBuffer.append(currentBatchResults.get(i).get(0) + ",");}stringBuffer.append("}");stringBuffer.toString();*/
outln("currentBatchResults.size() = " + currentBatchResults.size() + ", " + stringBuffer);
if (queryCancellation) { // break 1st query while loop: query one (not the main one)
break;
}
// if query produced results, process them
if (!currentBatchResults.isEmpty()) {
if (first) outln(MyUtils.DASH_LINE + currentBatchResults.get(0));
boolean threadCancellation =
processBatchResults(
queryThreadIndex, previousThreadResults); // also check for cancelation
if (threadCancellation) { // break 1st query while loop (not the main one)
break;
}
}
batchNumber++;
} // end of the 1st query while loop (query)s
queryTimer.stopTimer();
this.fetchingQueryAverageTime.addToAverage(queryTimer.getTimeEllapsedMs());
MyUtils.checkStatusProgram(potentialResults || queryRow == 0);
writelnQueryFile(
MyUtils.TAB_SEPARATOR
+ queryRow
+ " rows (sum) in "
+ queryTimer.getTimeEllapsedMs()
+ " ms");
// First thread only has 1 query and is finished
if (first) {
outln("Setting thread to done");
synchronized (this.done) {
this.done.setTrue();
}
synchronized (this.statusList) {
this.statusList.add(Status.FIRST_DONE);
this.statusList.notify(); // To notify second thread
}
break;
}
} // end of the main while loop
// close resources
outln("trying to acquire query lock");
synchronized (this.activeQuery) { // see above similar line
lockln("lock acquired on activeQuery");
if (this.activeQuery.getValue()) {
if (null != this.resultSet) {
this.resultSet.close();
}
check(this.preparedStatement != null);
this.preparedStatement.close();
this.activeQuery.setFalse();
}
lockln("lock released on activeQuery");
}
connection.close();
// Update average times
this.launchQueryAverageTime.update();
this.fetchingQueryAverageTime.update();
this.fetchingBatchAverageTime.update();
this.intermediaryResultsProcessingAverageTime.update();
} catch (SQLException e) {
throw new TechnicalException(e);
} catch (IOException e) {
throw new TechnicalException(e);
}
}