private QueryRunnerIntermediaryResult prepareResults(
QueryRunnerIntermediaryResult previousThreadResults, List<List<String>> currentBatchResults)
throws TechnicalException {
QueryRunnerIntermediaryResult batchResult =
new QueryRunnerIntermediaryResult(
previousThreadResults,
this.joinFieldIndexesWithinResults); // Tell next thread what field to use for the join
int totalResultRows = 0;
if (this.first) {
for (List<String> list : currentBatchResults) {
check(this.isLinkIndex || list.size() == this.totalFields);
List<String> row = new ArrayList<String>(list);
if (this.distinct && batchResult.containsRow(row)) {
continue;
}
batchResult.addValueRow(row);
totalResultRows++;
}
} else {
List<Integer> previousJoinFieldIndexesWithinResults =
previousThreadResults.getPreviousJoinFieldIndexesWithinResults();
boolean oneFieldJoin = previousJoinFieldIndexesWithinResults.size() == 1;
int max = -1;
if (QueryRunnerPrototypeConstants.CHECK)
max = Collections.max(previousJoinFieldIndexesWithinResults);
for (int rowNumber = 0; rowNumber < previousThreadResults.getTotalRows(); rowNumber++) {
List<String> previousRow = previousThreadResults.getValueRow(rowNumber);
check(previousRow.size() > max);
String previousJoinFieldValue = null;
List<String> previousJoinFieldValues = null;
if (oneFieldJoin) {
int index = previousJoinFieldIndexesWithinResults.get(0);
previousJoinFieldValue = previousRow.get(index);
} else {
previousJoinFieldValues = new ArrayList<String>();
for (int index : previousJoinFieldIndexesWithinResults) {
previousJoinFieldValues.add(previousRow.get(index));
}
}
for (List<String> list : currentBatchResults) {
String leftJoinFieldValue = null;
List<String> leftJoinFieldValues = null;
if (oneFieldJoin) {
leftJoinFieldValue = list.get(0);
} else {
leftJoinFieldValues = new ArrayList<String>();
for (int index = 0;
index < this.leftJoinFieldFullNames.size();
index++) { // Since these are the first ones
leftJoinFieldValues.add(list.get(index));
}
}
if ((oneFieldJoin
&& QueryRunnerPrototypeUtils.stringEquals(
previousJoinFieldValue, leftJoinFieldValue))
|| (!oneFieldJoin
&& QueryRunnerPrototypeUtils.stringListEquals(
previousJoinFieldValues, leftJoinFieldValues))) {
// nulls are considered (not a match if any of the string is null)
// same size for the list is a input requirement (as well as order)
List<String> row = new ArrayList<String>(previousRow);
row.addAll(list);
if (this.distinct && batchResult.containsRow(row)) {
continue;
}
batchResult.addValueRow(row);
totalResultRows++;
}
}
}
}
return batchResult;
}
private boolean customizeQuery(
QueryRunnerIntermediaryResult previousThreadResults, StringBuffer currentQuery) {
boolean potentialResults = true;
List<Integer> previousJoinFieldIndexesWithinResults =
previousThreadResults.getPreviousJoinFieldIndexesWithinResults();
check(previousJoinFieldIndexesWithinResults.size() == this.leftJoinFieldFullNames.size());
if (previousThreadResults.getTotalRows() > 0) {
// Use an IN list if only 1 field (more efficient)
StringBuffer stringBuffer = new StringBuffer();
boolean atLeastOne = false;
if (this.leftJoinFieldFullNames.size() == 1) {
// Just checked only 1
String column = this.aliasedLeftJoinFieldFullNames.get(0);
int index = previousJoinFieldIndexesWithinResults.get(0);
stringBuffer.append(" where " + column + " in (");
Set<String> setValues = new HashSet<String>();
for (int rowNumber = 0; rowNumber < previousThreadResults.getTotalRows(); rowNumber++) {
List<String> row = previousThreadResults.getValueRow(rowNumber);
String value = row.get(index);
if (null != value
&& !setValues.contains(
value)) { // Use of set to avoid doubles (useless in the IN list)
setValues.add(value);
stringBuffer.append((rowNumber == 0 ? "" : ",") + "'" + value + "'");
}
}
stringBuffer.append(")");
atLeastOne = !setValues.isEmpty(); // When only nulls
} else {
check(this.leftJoinFieldFullNames.size() == previousJoinFieldIndexesWithinResults.size());
for (int rowNumber = 0; rowNumber < previousThreadResults.getTotalRows(); rowNumber++) {
List<String> row = previousThreadResults.getValueRow(rowNumber);
StringBuffer stringBuffer2 = new StringBuffer();
for (int i = 0; i < this.aliasedLeftJoinFieldFullNames.size(); i++) {
int index =
previousJoinFieldIndexesWithinResults.get(
i); // because we know both list are the same size
String value = row.get(index);
if (null != value) {
stringBuffer2.append(
(i == 0 ? "" : " and ")
+ this.aliasedLeftJoinFieldFullNames.get(i)
+ "="
+ "'"
+ value
+ "'");
} else { // erase and break
stringBuffer2 = null;
break;
}
}
if (null != stringBuffer2) {
stringBuffer.append(
(!atLeastOne ? " where " : " or ")
+ // first one is where
"("
+ stringBuffer2
+ ")");
atLeastOne = true;
}
}
}
currentQuery.append(stringBuffer.toString());
if (!atLeastOne) {
potentialResults = false;
}
} else {
potentialResults = false; // if so, no results
}
return potentialResults;
}
private boolean processBatchResults(
int currentQueryThreadIndex, QueryRunnerIntermediaryResult previousThreadResults)
throws TechnicalException, FunctionalException {
if (!first) {
outln("Merging results");
}
Timer intermediaryResultsProcessingTimer = null;
if (!this.first && !this.last) {
intermediaryResultsProcessingTimer = new Timer();
intermediaryResultsProcessingTimer.startTimer();
}
// Prepare results for join with the next query
this.batchResult = prepareResults(previousThreadResults, currentBatchResults);
if (!last) {
// wait till next thread is start processing the latest results
List<Status> nextReadyList = nextQueryThread.getReadyList();
List<Status> tmp = null;
Status firstStatus = null;
synchronized (nextReadyList) {
outln("wait for next thread to be ready");
while (nextReadyList.isEmpty()) {
try {
nextReadyList.wait();
} catch (InterruptedException e) {
throw new TechnicalException(e);
}
}
if (!nextReadyList.isEmpty()) {
firstStatus = nextReadyList.remove(0);
} else {
MyUtils.errorProgram("SPONTANEOUS NOTIFICATION!!!!!!!!"); // shouldn't happen really
}
tmp = new ArrayList<Status>(nextReadyList);
}
outln("next status to process: " + firstStatus);
if (checkCancelThread()) { // check here in case we sent a signal because of cancelling
return true;
}
check(tmp.isEmpty(), checklog("nextReadyList = " + nextReadyList));
check(
null
== this
.currentResult); // Must have been freed by next thread (call to
// resetCurrentResult())
// Add the newest results for the next thread to process as soon as it's ready
outln("Add input for next thread");
this.currentResult = batchResult;
synchronized (this.statusList) {
this.statusList.add(Status.RESULTS);
this.statusList.notify(); // To notify next thread
}
} else {
// Add final results
outln("Add final results");
List<Integer> ignoreList = ThreadCommunication.getIgnoreList();
boolean cancelationInProcess =
this.rows != null && ThreadCommunication.getTotalRows() >= this.rows;
if (!cancelationInProcess) {
try {
for (List<String> list : batchResult.getValues()) {
boolean first = true;
for (int i = 0; i < list.size(); i++) {
if (!ignoreList.contains(i)) {
ThreadCommunication.bufferedWriter.write(
(first ? "" : MyUtils.TAB_SEPARATOR) + list.get(i));
if (!QueryRunnerPrototypeConstants.LOG) {
System.out.print((first ? "" : MyUtils.TAB_SEPARATOR) + list.get(i));
}
first = false;
}
}
ThreadCommunication.bufferedWriter.write(MyUtils.LINE_SEPARATOR);
if (!QueryRunnerPrototypeConstants.LOG) {
System.out.println();
}
int totalRows = ThreadCommunication.addRowToTotalRows();
if (!this.unsynchronizedFirstFinalResults && totalRows == 1) { // if first row
outln("notify first results have arrived");
synchronized (this.firstFinalResults) {
this.firstFinalResults.setTrue();
this.firstFinalResults.notify(); // notify main thread
this.unsynchronizedFirstFinalResults = true;
}
}
if (!this.unsynchronizedSomeFinalResults
&& this.rows != null
&& totalRows == this.rows.intValue()) {
outln("notify some results have arrived");
synchronized (this.someFinalResults) {
this.unsynchronizedSomeFinalResults = true;
this.someFinalResults.setTrue();
this.someFinalResults.notify(); // notify main thread
}
break;
}
}
if (QueryRunnerPrototypeConstants.LOG) {
ThreadCommunication.bufferedWriter.flush();
}
} catch (IOException e) {
throw new TechnicalException(e);
}
}
}
this.batchResult = null;
if (!this.first && !this.last) {
intermediaryResultsProcessingTimer.stopTimer();
this.intermediaryResultsProcessingAverageTime.addToAverage(
intermediaryResultsProcessingTimer.getTimeEllapsedMs());
}
return false;
}
private QueryRunnerIntermediaryResult processPreviousThreadDoneStatus(List<Status> tmp)
throws TechnicalException {
QueryRunnerIntermediaryResult previousThreadResults =
this.previousQueryThread.getCurrentResult();
WaitableBoolean previousDone = this.previousQueryThread.getDone();
check(
previousThreadResults != null || (previousThreadResults == null && previousDone.getValue()),
(previousThreadResults == null) + ", " + previousDone);
if (previousThreadResults == null) { // means done (the opposite isn't true)
check(
tmp == null
|| // can be null if CHECK is off
tmp.isEmpty(),
checklog(
"previousChangedStatusList = "
+ tmp)); // There could have only been one signal sent if done and no previous
// results to process
check(previousDone.getValue());
// If last thread and no results, notify that first results (=no results) have arrived
outln(
"previous thread ("
+ (queryThreadIndex - 1)
+ ") is done and did not provide data: "
+ "stopping current thread");
if (this.last) {
if (!this.unsynchronizedFirstFinalResults) {
outln("notify first results (=no results) have arrived");
synchronized (this.firstFinalResults) {
this.unsynchronizedFirstFinalResults = true;
this.firstFinalResults.setTrue();
this.firstFinalResults.notify(); // notify main thread
}
}
if (!this.unsynchronizedSomeFinalResults) {
outln("notify some results (=no results) have arrived");
synchronized (this.someFinalResults) {
this.unsynchronizedSomeFinalResults = true;
this.someFinalResults.setTrue();
this.someFinalResults.notify(); // notify main thread
}
}
}
// In any case set done to true and update status for next thread to be aware of
synchronized (this.done) {
this.done.setTrue();
}
synchronized (this.statusList) {
this.statusList.add(Status.DONE);
this.statusList.notify();
}
// Leave the main while loop
return null;
} else {
// Data has been provided, it will be processed
outln("data provided by previous thread");
check(
previousThreadResults != null && previousThreadResults.getTotalRows() > 0,
""
+ (previousThreadResults == null
? previousThreadResults
: previousThreadResults.getTotalRows()));
this.previousQueryThread
.resetCurrentResult(); // intermediary results have already been stored here,
// we can clear previous thread of it so it can go on with the next batch
// TODO move to previous thread? after the following notification: it would know it can do it
// update status for previous thread to know that its results are handled now and that it can
// fetch the next one
synchronized (this.readyList) { // TODO see above todo
this.readyList.add(Status.READY);
this.readyList.notify();
}
}
return previousThreadResults;
}
