Ejemplo n.º 1
0
  private boolean processBatchResults(
      int currentQueryThreadIndex, QueryRunnerIntermediaryResult previousThreadResults)
      throws TechnicalException, FunctionalException {

    if (!first) {
      outln("Merging results");
    }

    Timer intermediaryResultsProcessingTimer = null;
    if (!this.first && !this.last) {
      intermediaryResultsProcessingTimer = new Timer();
      intermediaryResultsProcessingTimer.startTimer();
    }

    // Prepare results for join with the next query
    this.batchResult = prepareResults(previousThreadResults, currentBatchResults);

    if (!last) {

      // wait till next thread is start processing the latest results
      List<Status> nextReadyList = nextQueryThread.getReadyList();
      List<Status> tmp = null;
      Status firstStatus = null;
      synchronized (nextReadyList) {
        outln("wait for next thread to be ready");
        while (nextReadyList.isEmpty()) {
          try {
            nextReadyList.wait();
          } catch (InterruptedException e) {
            throw new TechnicalException(e);
          }
        }
        if (!nextReadyList.isEmpty()) {
          firstStatus = nextReadyList.remove(0);
        } else {
          MyUtils.errorProgram("SPONTANEOUS NOTIFICATION!!!!!!!!"); // shouldn't happen really						
        }
        tmp = new ArrayList<Status>(nextReadyList);
      }
      outln("next status to process: " + firstStatus);
      if (checkCancelThread()) { // check here in case we sent a signal because of cancelling
        return true;
      }
      check(tmp.isEmpty(), checklog("nextReadyList = " + nextReadyList));
      check(
          null
              == this
                  .currentResult); // Must have been freed by next thread (call to
                                   // resetCurrentResult())

      // Add the newest results for the next thread to process as soon as it's ready
      outln("Add input for next thread");
      this.currentResult = batchResult;
      synchronized (this.statusList) {
        this.statusList.add(Status.RESULTS);
        this.statusList.notify(); // To notify next thread
      }
    } else {
      // Add final results
      outln("Add final results");

      List<Integer> ignoreList = ThreadCommunication.getIgnoreList();
      boolean cancelationInProcess =
          this.rows != null && ThreadCommunication.getTotalRows() >= this.rows;
      if (!cancelationInProcess) {
        try {
          for (List<String> list : batchResult.getValues()) {
            boolean first = true;
            for (int i = 0; i < list.size(); i++) {
              if (!ignoreList.contains(i)) {
                ThreadCommunication.bufferedWriter.write(
                    (first ? "" : MyUtils.TAB_SEPARATOR) + list.get(i));
                if (!QueryRunnerPrototypeConstants.LOG) {
                  System.out.print((first ? "" : MyUtils.TAB_SEPARATOR) + list.get(i));
                }
                first = false;
              }
            }
            ThreadCommunication.bufferedWriter.write(MyUtils.LINE_SEPARATOR);
            if (!QueryRunnerPrototypeConstants.LOG) {
              System.out.println();
            }
            int totalRows = ThreadCommunication.addRowToTotalRows();
            if (!this.unsynchronizedFirstFinalResults && totalRows == 1) { // if first row
              outln("notify first results have arrived");
              synchronized (this.firstFinalResults) {
                this.firstFinalResults.setTrue();
                this.firstFinalResults.notify(); // notify main thread
                this.unsynchronizedFirstFinalResults = true;
              }
            }
            if (!this.unsynchronizedSomeFinalResults
                && this.rows != null
                && totalRows == this.rows.intValue()) {
              outln("notify some results have arrived");
              synchronized (this.someFinalResults) {
                this.unsynchronizedSomeFinalResults = true;
                this.someFinalResults.setTrue();
                this.someFinalResults.notify(); // notify main thread
              }
              break;
            }
          }
          if (QueryRunnerPrototypeConstants.LOG) {
            ThreadCommunication.bufferedWriter.flush();
          }
        } catch (IOException e) {
          throw new TechnicalException(e);
        }
      }
    }
    this.batchResult = null;

    if (!this.first && !this.last) {
      intermediaryResultsProcessingTimer.stopTimer();
      this.intermediaryResultsProcessingAverageTime.addToAverage(
          intermediaryResultsProcessingTimer.getTimeEllapsedMs());
    }

    return false;
  }
Ejemplo n.º 2
0
  private void process() throws TechnicalException, FunctionalException {

    try {
      Connection connection = connect();

      while (true) { // broken when done

        this.checkCancelIndex = 0;

        if (checkCancelThread()) { // regularily check for cancel request and exit when encountered
          break;
        }

        QueryRunnerIntermediaryResult previousThreadResults = null;
        if (!first) {
          List<Status> tmp = fetchPreviousThreadNextStatus();

          if (checkCancelThread()) { // see above similar line
            break;
          }

          previousThreadResults = processPreviousThreadDoneStatus(tmp);
          if (previousThreadResults == null) { // means previous thread is done
            break;
          }
        }

        if (checkCancelThread()) { // see above similar line
          break;
        }

        StringBuffer currentQuery = new StringBuffer(this.queryTemplate);
        boolean potentialResults = true;
        if (first) {
          previousThreadResults = new QueryRunnerIntermediaryResult(this.queryThreadIndex);
          outln("first: no input needed");
        } else {
          outln("Processing input");
          potentialResults = customizeQuery(previousThreadResults, currentQuery);
        }

        this.shortenedQuery =
            currentQuery.length() < QueryRunnerPrototypeConstants.MAX_DISPLAY_QUERY_LENGTH
                ? currentQuery.toString()
                : (currentQuery
                        .substring(0, QueryRunnerPrototypeConstants.MAX_DISPLAY_QUERY_LENGTH2)
                        .toString()
                    + " ... "
                    + currentQuery
                        .substring(
                            currentQuery.length()
                                - QueryRunnerPrototypeConstants.MAX_DISPLAY_QUERY_LENGTH2,
                            currentQuery.length())
                        .toString());
        outln(potentialResults + ", " + "shortenedQuery = " + shortenedQuery);
        writelnQueryFile(shortenedQuery);

        if (checkCancelThread()) { // see above similar line
          break;
        }

        outln("preparing statement");
        synchronized (
            this
                .activeQuery) { // always get the lock on that object to handle the query
                                // related-object: 2 threads can use them (main one and current one)
          lockln("lock acquired on activeQuery");
          if (this.preparedStatement == null) {
            check(!this.activeQuery.getValue());
            check(this.resultSet == null); // not sure
            prepareStreamingStatement(connection, currentQuery);
          } else {
            check(this.activeQuery.getValue());
            check(this.resultSet != null); // not sure						
            this.resultSet.close();
            this.preparedStatement.close();
            prepareStreamingStatement(connection, currentQuery);
          }
          this.activeQuery
              .setTrue(); // set it to true: meaning cancel would have to actually cancel the query
                          // and close as much as it can
          lockln("lock released on activeQuery");
        }

        if (checkCancelThread()) { // see above similar line
          break;
        }

        Timer queryTimer = new Timer();
        queryTimer.startTimer();

        Timer queryLaunchTimer = new Timer();
        queryLaunchTimer.startTimer();

        Integer columnCount = null;
        if (potentialResults) {
          outln("executing query");
          synchronized (this.activeQuery) { // see above similar line
            lockln("lock acquired on activeQuery");
            this.resultSet =
                this.preparedStatement.executeQuery(
                    // currentQuery.toString());	// for Statement
                    ); // for PreparedStatement
            outln("query prepared");
            columnCount = this.resultSet.getMetaData().getColumnCount();
            // activeQuery is already set to true here: it means we must always check whether
            // resultSet is null or not before trying to close it
            lockln("lock released on activeQuery");
          }
        }

        queryLaunchTimer.stopTimer();
        this.launchQueryAverageTime.addToAverage(queryLaunchTimer.getTimeEllapsedMs());

        if (checkCancelThread()) { // see above similar line
          break;
        }

        int batchNumber = 0; // For the current query
        queryRow = 0; // For the current query
        Boolean next = null;
        boolean queryCancellation = false; // so we can break the 2 embedded query while loops
        while ( // 1st query while loop: query one
        (potentialResults && ((next = resultSetNext(resultSet)) || batchNumber == 0))
            || (!potentialResults
                && batchNumber == 0)) { // We want to go inside the first time even if no results

          this.currentBatchResults = new ArrayList<List<String>>();
          int batchRow = 0; // For the current batch for the current query
          outln("Fetching batch");

          Timer batchTimer = new Timer();
          batchTimer.startTimer();

          while ( // 2st query while loop: batch one
          batchRow < this.batchSize
              && potentialResults
              && ((batchRow == 0 && next) || (batchRow > 0 && resultSetNext(resultSet)))) {

            List<String> list = new ArrayList<String>();

            // we also lock on that object when retrieving a row
            synchronized (this.activeQuery) { // see above similar line
              lockln("lock acquired on activeQuery");
              if (this.activeQuery.getValue()) {
                for (int i = 0; i < columnCount; i++) {
                  String fieldValue = resultSet.getString(i + 1);
                  list.add(fieldValue); // accepting null values
                }
              } else {
                queryCancellation = true;
              }

              lockln("lock released on activeQuery");

              if (queryCancellation) { // break 2nd query while loop: batch one (not the main one)
                break;
              }
            }
            currentBatchResults.add(list);
            batchRow++;
            queryRow++;
          } // end of the 2nd query while loop (batch)

          batchTimer.stopTimer();
          this.fetchingBatchAverageTime.addToAverage(batchTimer.getTimeEllapsedMs());
          writelnQueryFile(
              MyUtils.TAB_SEPARATOR
                  + MyUtils.TAB_SEPARATOR
                  + batchRow
                  + " rows in "
                  + batchTimer.getTimeEllapsedMs()
                  + " ms");
          StringBuffer stringBuffer =
              new StringBuffer(); /*stringBuffer.append("{");for (int i = 0; i < currentBatchResults.size(); i++) {stringBuffer.append(currentBatchResults.get(i).get(0) + ",");}stringBuffer.append("}");stringBuffer.toString();*/
          outln("currentBatchResults.size() = " + currentBatchResults.size() + ", " + stringBuffer);

          if (queryCancellation) { // break 1st query while loop: query one (not the main one)
            break;
          }

          // if query produced results, process them
          if (!currentBatchResults.isEmpty()) {
            if (first) outln(MyUtils.DASH_LINE + currentBatchResults.get(0));
            boolean threadCancellation =
                processBatchResults(
                    queryThreadIndex, previousThreadResults); // also check for cancelation

            if (threadCancellation) { // break 1st query while loop (not the main one)
              break;
            }
          }

          batchNumber++;
        } // end of the 1st query while loop (query)s

        queryTimer.stopTimer();
        this.fetchingQueryAverageTime.addToAverage(queryTimer.getTimeEllapsedMs());
        MyUtils.checkStatusProgram(potentialResults || queryRow == 0);
        writelnQueryFile(
            MyUtils.TAB_SEPARATOR
                + queryRow
                + " rows (sum) in "
                + queryTimer.getTimeEllapsedMs()
                + " ms");

        // First thread only has 1 query and is finished
        if (first) {
          outln("Setting thread to done");
          synchronized (this.done) {
            this.done.setTrue();
          }
          synchronized (this.statusList) {
            this.statusList.add(Status.FIRST_DONE);
            this.statusList.notify(); // To notify second thread
          }
          break;
        }
      } // end of the main while loop

      // close resources
      outln("trying to acquire query lock");
      synchronized (this.activeQuery) { // see above similar line
        lockln("lock acquired on activeQuery");
        if (this.activeQuery.getValue()) {
          if (null != this.resultSet) {
            this.resultSet.close();
          }
          check(this.preparedStatement != null);
          this.preparedStatement.close();
          this.activeQuery.setFalse();
        }
        lockln("lock released on activeQuery");
      }
      connection.close();

      // Update average times
      this.launchQueryAverageTime.update();
      this.fetchingQueryAverageTime.update();
      this.fetchingBatchAverageTime.update();
      this.intermediaryResultsProcessingAverageTime.update();

    } catch (SQLException e) {
      throw new TechnicalException(e);
    } catch (IOException e) {
      throw new TechnicalException(e);
    }
  }