Exemple #1
1
  private List<Status> fetchPreviousThreadNextStatus() throws TechnicalException {
    outln("waiting for previous thread to either provide data and/or be done");
    List<Status> previousStatusList = this.previousQueryThread.getChangedStatusList();
    List<Status> tmp = null;
    Status firstStatus = null;

    // Obtain lock on previous status
    synchronized (previousStatusList) {
      while (previousStatusList.isEmpty()) {
        try {
          previousStatusList.wait();
        } catch (InterruptedException e) {
          throw new TechnicalException(e);
        }
      }
      if (!previousStatusList.isEmpty()) {
        firstStatus =
            previousStatusList.remove(0); // remove status from list, it will then be processed
      } else {
        MyUtils.errorProgram("SPONTANEOUS NOTIFICATION!!!!!!!!"); // shouldn't happen really						
      }
      if (QueryRunnerPrototypeConstants.CHECK) tmp = new ArrayList<Status>(previousStatusList);
    }
    outln("previous status to process: " + firstStatus);

    return tmp;
  }
Exemple #2
0
  public boolean checkDatabase(String templateName, DatabaseCheck databaseCheck, String rangeName)
      throws TechnicalException, FunctionalException {

    String datasetNameFromTableName = this.nctm.getDatasetName();
    if (null != datasetNameFromTableName
        && !MyUtils.isEmpty(datasetNameFromTableName)
        && !datasetNameFromTableName.equals(rangeName)) {
      throw new FunctionalException(
          "Mismatching dataset identifiers: datasetNameFromTableName = "
              + datasetNameFromTableName
              + ", rangeName = "
              + rangeName);
    }

    if (this.getTableConstraint() == null || this.getField() == null) {
      return false;
    }

    String tableName = null;
    if (this.main) {
      return true;
    } else {
      tableName =
          datasetNameFromTableName == null
              ? rangeName + "__" + this.getTableConstraint()
              : this.getTableConstraint();
      MyUtils.checkStatusProgram(
          tableName.split("__").length == 3, "tableName = " + tableName, true);
    }
    return databaseCheck.checkDatabase(templateName, tableName, this.getField())
        && databaseCheck.checkDatabase(templateName, tableName, this.getKey());
  }
  public void process() throws Exception {
    /*createFiles(0, 2, 1000);
    createFiles(10, 2, 1000);
    createFiles(50, 2, 1000);
    createFiles(100, 2, 1000);*/

    /*createFiles(0, 1, 1000000);
    createFiles(10, 1, 1000000);
    createFiles(50, 1, 1000000);
    createFiles(100, 1, 1000000);*/

    // createFiles(10, 1, 1000000, "bottom");
    // createFiles(50, 1, 1000000, "bottom");

    /*changeRdbs(Rdbs.MYSQL);
    createFiles(10, 1, 1000000, "bottom");
    createFiles(50, 1, 1000000, "bottom");
    changeRdbs(Rdbs.POSTGRESQL);
    createFiles(10, 1, 1000000, "bottom");
    createFiles(50, 1, 1000000, "bottom");*/
    changeRdbs(Rdbs.ORACLE);
    createFiles(10, 1, 1000000, BOTTOM2);
    createFiles(50, 1, 1000000, BOTTOM2);
    // createFiles(50, 1, 100000, "bottom");

    // createFiles(100, 1, 10000000);

    String scriptFileName = "script_" + this.rdbs + SCRIPT_EXTENSION;
    String scriptFilePathAndName =
        GENERAL_OUPUT_FOLDER_PATH_AND_NAME + MyUtils.FILE_SEPARATOR + scriptFileName;
    MyUtils.writeFile(scriptFilePathAndName, stringBuffer.toString());
    System.out.println("msl < " + scriptFilePathAndName);
  }
Exemple #4
0
 private boolean checkDatabaseForTable(String templateName, String tableName) {
   TemplateDatabaseDescription templateDatabaseDescription =
       templateNameToDatabaseDescriptionMap.get(templateName);
   MyUtils.checkStatusProgram(templateDatabaseDescription != null);
   List<String> tableList = templateDatabaseDescription.getTableList();
   return MartConfiguratorUtils.containsIgnoreCase(tableList, tableName);
 }
Exemple #5
0
  public void fetchTableList(String templateName) throws TechnicalException {

    String databaseName = getDatabaseName(templateName);
    MyUtils.checkStatusProgram(null != databaseName);

    try {
      List<String> tableList = null;
      String tableListSerialFilePathAndName =
          this.transformationsGeneralSerialFolderPathAndName
              + TABLE_LIST_SERIAL_FILE_PREFIX
              + databaseName;
      if (new File(tableListSerialFilePathAndName).exists()) {
        tableList = (List<String>) MyUtils.readSerializedObject(tableListSerialFilePathAndName);
        System.out.println("Using serial, quick check: " + tableList.size());
      } else {
        System.out.println("No serial at " + tableListSerialFilePathAndName + " , connecting...");
        connect(databaseName);

        tableList = new ArrayList<String>();

        // Prepare list of tables
        this.preparedStatementTable = this.connection.prepareStatement("show tables");

        ResultSet resultSet = this.preparedStatementTable.executeQuery();

        tableList = new ArrayList<String>();
        resultSet.beforeFirst();
        while (resultSet.next()) {
          tableList.add(resultSet.getString(1));
        }

        this.preparedStatementTable.close();
        resultSet.close();

        disconnect();

        MyUtils.writeSerializedObject(tableList, tableListSerialFilePathAndName);
      }

      MyUtils.checkStatusProgram(templateNameToDatabaseDescriptionMap.get(templateName) == null);
      templateNameToDatabaseDescriptionMap.put(
          templateName, new TemplateDatabaseDescription(tableList));

    } catch (SQLException e) {
      throw new TechnicalException(e);
    }
  }
Exemple #6
0
  public Document fetchTemplateXml(String virtualSchema, String templateName)
      throws TechnicalException {
    // Find what database the dataset/template belongs to
    String databaseName = getDatabaseName(templateName);
    MyUtils.checkStatusProgram(null != databaseName, templateName);

    return fetchTemplateXml(virtualSchema, templateName, databaseName);
  }
Exemple #7
0
  @Deprecated
  public boolean isTemplate(String name) throws FunctionalException {
    Boolean matchesDatasetName = false;
    Boolean matchesTemplateName = false;
    String databaseNameMatch = null;

    for (Iterator<String> it = databaseNameToDatasetInformationMap.keySet().iterator();
        it.hasNext(); ) {
      String databaseName = it.next();
      HashMap<String, List<String>> map = databaseNameToDatasetInformationMap.get(databaseName);

      for (Iterator<String> it2 = map.keySet().iterator(); it2.hasNext(); ) {
        String templateName = it2.next();
        List<String> datasetNames = map.get(templateName);

        for (String datasetName : datasetNames) {
          if (name.equals(datasetName)) {
            MyUtils.checkStatusProgram(
                !matchesDatasetName && !matchesTemplateName && null == databaseNameMatch);
            matchesDatasetName = true;
            databaseNameMatch = databaseName;
          }
          if (name.equals(templateName)) {
            MyUtils.checkStatusProgram(
                !matchesDatasetName
                    && (!matchesTemplateName
                        || (null != databaseNameMatch && databaseNameMatch.equals(databaseName))));
            matchesTemplateName = true;
            databaseNameMatch = databaseName;
          }
        }
      }
    }

    if (!matchesDatasetName && !matchesTemplateName) {
      throw new FunctionalException("Unknown dataset/template name: " + name);
    } else if (matchesDatasetName && matchesTemplateName) {
      throw new FunctionalException("Ambiguous dataset/template name: " + name);
    }
    return matchesTemplateName;
  }
Exemple #8
0
 private boolean checkDatabaseForField(String templateName, String tableName, String columnName)
     throws FunctionalException {
   TemplateDatabaseDescription templateDatabaseDescription =
       templateNameToDatabaseDescriptionMap.get(templateName);
   MyUtils.checkStatusProgram(templateDatabaseDescription != null);
   Map<String, List<String>> tableColumnMap = templateDatabaseDescription.getTableColumnMap();
   List<String> columnList = tableColumnMap.get(tableName);
   if (null == columnList) {
     throw new FunctionalException("No table called " + tableName + " found in map");
   }
   return MartConfiguratorUtils.containsIgnoreCase(columnList, columnName);
 }
 public void updateTransformedDatasetNameInPlainDatasetNameToTransformedDatasetName(
     String datasetOrTemplateName, String transformedDatasetName, boolean isWebService) {
   boolean atLeastOne = false;
   for (Iterator<String> it = this.plainDatasetNameToTransformedDatasetName.keySet().iterator();
       it.hasNext(); ) {
     String pointedDatasetPlainName = it.next();
     String datasetOrTemplateNameTmp =
         this.plainDatasetNameToTransformedDatasetName.get(pointedDatasetPlainName);
     if (datasetOrTemplateNameTmp.equals(datasetOrTemplateName)) {
       atLeastOne = true;
       /*MyUtils.checkStatusProgram(datasetOrTemplateNameTmp!=null &&
       (isWebService || (!datasetOrTemplateNameTmp.equals(transformedDatasetName))),
       (datasetOrTemplateNameTmp!=null) + ", " + isWebService + ", " + (!datasetOrTemplateNameTmp.equals(transformedDatasetName)) + ", " +
       datasetOrTemplateNameTmp + ", " + transformedDatasetName);*/
       this.plainDatasetNameToTransformedDatasetName.put(
           pointedDatasetPlainName, transformedDatasetName);
     }
   }
   MyUtils.checkStatusProgram(atLeastOne);
 }
Exemple #10
0
  private void process() throws TechnicalException, FunctionalException {

    try {
      Connection connection = connect();

      while (true) { // broken when done

        this.checkCancelIndex = 0;

        if (checkCancelThread()) { // regularily check for cancel request and exit when encountered
          break;
        }

        QueryRunnerIntermediaryResult previousThreadResults = null;
        if (!first) {
          List<Status> tmp = fetchPreviousThreadNextStatus();

          if (checkCancelThread()) { // see above similar line
            break;
          }

          previousThreadResults = processPreviousThreadDoneStatus(tmp);
          if (previousThreadResults == null) { // means previous thread is done
            break;
          }
        }

        if (checkCancelThread()) { // see above similar line
          break;
        }

        StringBuffer currentQuery = new StringBuffer(this.queryTemplate);
        boolean potentialResults = true;
        if (first) {
          previousThreadResults = new QueryRunnerIntermediaryResult(this.queryThreadIndex);
          outln("first: no input needed");
        } else {
          outln("Processing input");
          potentialResults = customizeQuery(previousThreadResults, currentQuery);
        }

        this.shortenedQuery =
            currentQuery.length() < QueryRunnerPrototypeConstants.MAX_DISPLAY_QUERY_LENGTH
                ? currentQuery.toString()
                : (currentQuery
                        .substring(0, QueryRunnerPrototypeConstants.MAX_DISPLAY_QUERY_LENGTH2)
                        .toString()
                    + " ... "
                    + currentQuery
                        .substring(
                            currentQuery.length()
                                - QueryRunnerPrototypeConstants.MAX_DISPLAY_QUERY_LENGTH2,
                            currentQuery.length())
                        .toString());
        outln(potentialResults + ", " + "shortenedQuery = " + shortenedQuery);
        writelnQueryFile(shortenedQuery);

        if (checkCancelThread()) { // see above similar line
          break;
        }

        outln("preparing statement");
        synchronized (
            this
                .activeQuery) { // always get the lock on that object to handle the query
                                // related-object: 2 threads can use them (main one and current one)
          lockln("lock acquired on activeQuery");
          if (this.preparedStatement == null) {
            check(!this.activeQuery.getValue());
            check(this.resultSet == null); // not sure
            prepareStreamingStatement(connection, currentQuery);
          } else {
            check(this.activeQuery.getValue());
            check(this.resultSet != null); // not sure						
            this.resultSet.close();
            this.preparedStatement.close();
            prepareStreamingStatement(connection, currentQuery);
          }
          this.activeQuery
              .setTrue(); // set it to true: meaning cancel would have to actually cancel the query
                          // and close as much as it can
          lockln("lock released on activeQuery");
        }

        if (checkCancelThread()) { // see above similar line
          break;
        }

        Timer queryTimer = new Timer();
        queryTimer.startTimer();

        Timer queryLaunchTimer = new Timer();
        queryLaunchTimer.startTimer();

        Integer columnCount = null;
        if (potentialResults) {
          outln("executing query");
          synchronized (this.activeQuery) { // see above similar line
            lockln("lock acquired on activeQuery");
            this.resultSet =
                this.preparedStatement.executeQuery(
                    // currentQuery.toString());	// for Statement
                    ); // for PreparedStatement
            outln("query prepared");
            columnCount = this.resultSet.getMetaData().getColumnCount();
            // activeQuery is already set to true here: it means we must always check whether
            // resultSet is null or not before trying to close it
            lockln("lock released on activeQuery");
          }
        }

        queryLaunchTimer.stopTimer();
        this.launchQueryAverageTime.addToAverage(queryLaunchTimer.getTimeEllapsedMs());

        if (checkCancelThread()) { // see above similar line
          break;
        }

        int batchNumber = 0; // For the current query
        queryRow = 0; // For the current query
        Boolean next = null;
        boolean queryCancellation = false; // so we can break the 2 embedded query while loops
        while ( // 1st query while loop: query one
        (potentialResults && ((next = resultSetNext(resultSet)) || batchNumber == 0))
            || (!potentialResults
                && batchNumber == 0)) { // We want to go inside the first time even if no results

          this.currentBatchResults = new ArrayList<List<String>>();
          int batchRow = 0; // For the current batch for the current query
          outln("Fetching batch");

          Timer batchTimer = new Timer();
          batchTimer.startTimer();

          while ( // 2st query while loop: batch one
          batchRow < this.batchSize
              && potentialResults
              && ((batchRow == 0 && next) || (batchRow > 0 && resultSetNext(resultSet)))) {

            List<String> list = new ArrayList<String>();

            // we also lock on that object when retrieving a row
            synchronized (this.activeQuery) { // see above similar line
              lockln("lock acquired on activeQuery");
              if (this.activeQuery.getValue()) {
                for (int i = 0; i < columnCount; i++) {
                  String fieldValue = resultSet.getString(i + 1);
                  list.add(fieldValue); // accepting null values
                }
              } else {
                queryCancellation = true;
              }

              lockln("lock released on activeQuery");

              if (queryCancellation) { // break 2nd query while loop: batch one (not the main one)
                break;
              }
            }
            currentBatchResults.add(list);
            batchRow++;
            queryRow++;
          } // end of the 2nd query while loop (batch)

          batchTimer.stopTimer();
          this.fetchingBatchAverageTime.addToAverage(batchTimer.getTimeEllapsedMs());
          writelnQueryFile(
              MyUtils.TAB_SEPARATOR
                  + MyUtils.TAB_SEPARATOR
                  + batchRow
                  + " rows in "
                  + batchTimer.getTimeEllapsedMs()
                  + " ms");
          StringBuffer stringBuffer =
              new StringBuffer(); /*stringBuffer.append("{");for (int i = 0; i < currentBatchResults.size(); i++) {stringBuffer.append(currentBatchResults.get(i).get(0) + ",");}stringBuffer.append("}");stringBuffer.toString();*/
          outln("currentBatchResults.size() = " + currentBatchResults.size() + ", " + stringBuffer);

          if (queryCancellation) { // break 1st query while loop: query one (not the main one)
            break;
          }

          // if query produced results, process them
          if (!currentBatchResults.isEmpty()) {
            if (first) outln(MyUtils.DASH_LINE + currentBatchResults.get(0));
            boolean threadCancellation =
                processBatchResults(
                    queryThreadIndex, previousThreadResults); // also check for cancelation

            if (threadCancellation) { // break 1st query while loop (not the main one)
              break;
            }
          }

          batchNumber++;
        } // end of the 1st query while loop (query)s

        queryTimer.stopTimer();
        this.fetchingQueryAverageTime.addToAverage(queryTimer.getTimeEllapsedMs());
        MyUtils.checkStatusProgram(potentialResults || queryRow == 0);
        writelnQueryFile(
            MyUtils.TAB_SEPARATOR
                + queryRow
                + " rows (sum) in "
                + queryTimer.getTimeEllapsedMs()
                + " ms");

        // First thread only has 1 query and is finished
        if (first) {
          outln("Setting thread to done");
          synchronized (this.done) {
            this.done.setTrue();
          }
          synchronized (this.statusList) {
            this.statusList.add(Status.FIRST_DONE);
            this.statusList.notify(); // To notify second thread
          }
          break;
        }
      } // end of the main while loop

      // close resources
      outln("trying to acquire query lock");
      synchronized (this.activeQuery) { // see above similar line
        lockln("lock acquired on activeQuery");
        if (this.activeQuery.getValue()) {
          if (null != this.resultSet) {
            this.resultSet.close();
          }
          check(this.preparedStatement != null);
          this.preparedStatement.close();
          this.activeQuery.setFalse();
        }
        lockln("lock released on activeQuery");
      }
      connection.close();

      // Update average times
      this.launchQueryAverageTime.update();
      this.fetchingQueryAverageTime.update();
      this.fetchingBatchAverageTime.update();
      this.intermediaryResultsProcessingAverageTime.update();

    } catch (SQLException e) {
      throw new TechnicalException(e);
    } catch (IOException e) {
      throw new TechnicalException(e);
    }
  }
Exemple #11
0
 private void check(boolean test, String message) {
   if (QueryRunnerPrototypeConstants.CHECK) MyUtils.checkStatusProgram(test, checklog(message));
 }
Exemple #12
0
  private OldElement(
      boolean isAttribute,
      Element jdomElement,
      String value,
      List<Element> childrenList,
      String internalName,
      String hidden,
      String hideDisplay,
      String description,
      String displayName,
      String tableConstraint,
      String key,
      String field,
      String pointerDataset,
      String pointerInterface,
      String pointerElement,
      String checkForNulls)
      throws FunctionalException {

    super(jdomElement, internalName);

    if (this.hidden) { // assigned in OldNode
      this.valid = false;
      return;
    }

    this.isAttribute = isAttribute;

    this.hideDisplay = TransformationUtils.getBooleanValueFromString(hideDisplay, "hideDisplay");
    this.description = description;
    this.displayName = displayName;
    this.tableConstraint = tableConstraint;
    this.key = key;
    this.field = field;
    this.pointerDataset = pointerDataset;
    this.pointerInterface = pointerInterface;
    this.pointerElement = pointerElement;
    this.checkForNulls =
        TransformationUtils.getBooleanValueFromString(checkForNulls, "checkForNulls");

    this.hasChildren = childrenList != null && childrenList.size() != 0;
    this.hasSubChildren = false;
    this.tree = false;
    if (this.hasChildren) {
      Element firstChild = childrenList.get(0);
      this.firstChildrenType = ElementChildrenType.getElementChildrenType(firstChild);

      // Will check for the 1st child that has subChildren (if any) and check the type of the
      // subchild
      for (Element child : childrenList) {
        List<Element> subChildrenList = child.getChildren();
        this.hasSubChildren = subChildrenList != null && subChildrenList.size() != 0;
        if (this.hasSubChildren) {
          ElementChildrenType subChildrenType =
              ElementChildrenType.getElementChildrenType(subChildrenList.get(0));
          this.tree =
              ElementChildrenType.OPTION_VALUE.equals(this.firstChildrenType)
                  && ElementChildrenType.OPTION_VALUE.equals(subChildrenType);
          break;
        }
      }
    }

    this.pointer =
        this.pointerElement != null && this.pointerInterface != null && this.pointerDataset != null;

    this.main =
        !this.pointer && tableConstraint != null && TransformationUtils.isMain(tableConstraint);
    this.dimension = !this.pointer && tableConstraint != null && !this.main;

    this.option = this instanceof OldOptionFilter;
    if (this.main || this.dimension) {
      this.nctm = new NamingConventionTableName(this.tableConstraint);
    } else {
      this.filterGroup = !pointer && !this.option;
      MyUtils.checkStatusProgram(
          !this.filterGroup || hasChildren, XmlUtils.displayJdomElement(jdomElement));
    }

    this.dimensionPartition = new DimensionPartition(this.tableConstraint, this.key);
    if (this.dimension) {
      this.dimensionPartition.searchForPotentialDimensionTablePartition();
    }

    if (!finalChecks()) {
      // System.out.println(this.internalName + "
      // ##########@@@@@@@@@@@@@######################*****************####################################");
      System.err.println(
          "######################### element has failed final checks" + this.internalName);
      this.valid = false;
    }
  }
Exemple #13
0
  public Document fetchTemplateXml(String virtualSchema, String templateName, String databaseName)
      throws TechnicalException {

    Document document = null;
    try {
      String templateXmlSerialFilePathAndName =
          this.transformationsGeneralSerialFolderPathAndName
              + TEMPLATE_XML_CONTENT_SERIAL_FILE_PREFIX
              + virtualSchema
              + MyUtils.INFO_SEPARATOR
              + MyUtils.INFO_SEPARATOR
              + templateName;
      String templateXmlFilePathAndName =
          this.transformationsGeneralSerialFolderPathAndName
              + virtualSchema
              + MyUtils.INFO_SEPARATOR
              + MyUtils.INFO_SEPARATOR
              + templateName;
      if (new File(templateXmlSerialFilePathAndName).exists()) {
        System.out.println("Using serial at " + templateXmlSerialFilePathAndName);
        document = (Document) MyUtils.readSerializedObject(templateXmlSerialFilePathAndName);
        System.out.println("quick check: " + document.getDocType());
      } else {
        System.out.println("No serial at " + templateXmlSerialFilePathAndName + " , connecting...");
        connect(databaseName);

        this.preparedStatementTemplateXml =
            this.connection.prepareStatement(
                "select meta_template__xml__dm.compressed_xml "
                    + "from meta_template__xml__dm "
                    + "where meta_template__xml__dm.template = '"
                    + templateName
                    + "';");

        ResultSet resultSet = this.preparedStatementTemplateXml.executeQuery();

        resultSet.first();
        byte[] bytes = resultSet.getBytes(1);
        MyUtils.checkStatusProgram(!resultSet.next()); // Check only 1 row

        this.preparedStatementTemplateXml.close();
        resultSet.close();

        disconnect();

        /* Transform into uncompressed XML document:
        	Compression of XML in MartEditor, see:
        		DatabaseDatasetConfigUtils.storeTemplateXML (compress)
        		MartRegistryXMLUtils.DataSourceToRegistryDocument (uncompress)
        */
        InputStream rstream = null;
        rstream = new GZIPInputStream(new ByteArrayInputStream(bytes));
        SAXBuilder builder = new SAXBuilder();
        InputSource is = new InputSource(rstream);
        document = builder.build(is);

        MyUtils.writeSerializedObject(document, templateXmlSerialFilePathAndName);
        XmlUtils.writeXmlFile(document, templateXmlFilePathAndName);
      }
    } catch (SQLException e) {
      throw new TechnicalException(e);
    } catch (JDOMException e) {
      throw new TechnicalException(e);
    } catch (IOException e) {
      throw new TechnicalException(e);
    }
    return document;
  }
Exemple #14
0
 public TemplateDatabaseDescription getTemplateDatabaseDescription(String templateName) {
   TemplateDatabaseDescription templateDatabaseDescription =
       this.templateNameToDatabaseDescriptionMap.get(templateName);
   MyUtils.checkStatusProgram(null != templateDatabaseDescription);
   return templateDatabaseDescription;
 }
Exemple #15
0
  public void fetchTableColumnMap(String templateName) throws TechnicalException {

    String databaseName = getDatabaseName(templateName);
    MyUtils.checkStatusProgram(null != databaseName);

    try {

      Map<String, List<String>> tableColumnMap = null;
      TemplateDatabaseDescription templateDatabaseDescription =
          templateNameToDatabaseDescriptionMap.get(templateName);
      MyUtils.checkStatusProgram(templateDatabaseDescription != null);

      String tableColumnMapContentSerialFilePathAndName =
          this.transformationsGeneralSerialFolderPathAndName
              + TABLE_COLUMN_LIST_SERIAL_FILE_PREFIX
              + databaseName;
      if (new File(tableColumnMapContentSerialFilePathAndName).exists()) {
        tableColumnMap =
            (Map<String, List<String>>)
                MyUtils.readSerializedObject(tableColumnMapContentSerialFilePathAndName);
        System.out.println("Using serial, quick check: " + tableColumnMap.size());
      } else {

        System.out.println(
            "No serial at " + tableColumnMapContentSerialFilePathAndName + " , connecting...");
        connect(databaseName);

        tableColumnMap = new HashMap<String, List<String>>();
        List<String> tableList = templateDatabaseDescription.getTableList();
        int table = 0;
        for (String tableName : tableList) {

          System.out.print(
              "tableName = " + tableName + ", " + (table + 1) + "/" + tableList.size());

          // Prepare list of descriptions
          this.preparedStatementField = this.connection.prepareStatement("desc " + tableName);

          ResultSet resultSet = this.preparedStatementField.executeQuery();

          List<String> columnList = new ArrayList<String>();
          resultSet.beforeFirst();
          while (resultSet.next()) {
            columnList.add(resultSet.getString(1));
          }

          tableColumnMap.put(tableName, columnList);

          this.preparedStatementField.close();
          resultSet.close();

          System.out.println(", " + columnList.size());

          table++;
        }

        disconnect();

        MyUtils.writeSerializedObject(tableColumnMap, tableColumnMapContentSerialFilePathAndName);
      }

      templateDatabaseDescription.setTableColumnMap(tableColumnMap);

    } catch (SQLException e) {
      throw new TechnicalException(e);
    }
  }
Exemple #16
0
  private boolean processBatchResults(
      int currentQueryThreadIndex, QueryRunnerIntermediaryResult previousThreadResults)
      throws TechnicalException, FunctionalException {

    if (!first) {
      outln("Merging results");
    }

    Timer intermediaryResultsProcessingTimer = null;
    if (!this.first && !this.last) {
      intermediaryResultsProcessingTimer = new Timer();
      intermediaryResultsProcessingTimer.startTimer();
    }

    // Prepare results for join with the next query
    this.batchResult = prepareResults(previousThreadResults, currentBatchResults);

    if (!last) {

      // wait till next thread is start processing the latest results
      List<Status> nextReadyList = nextQueryThread.getReadyList();
      List<Status> tmp = null;
      Status firstStatus = null;
      synchronized (nextReadyList) {
        outln("wait for next thread to be ready");
        while (nextReadyList.isEmpty()) {
          try {
            nextReadyList.wait();
          } catch (InterruptedException e) {
            throw new TechnicalException(e);
          }
        }
        if (!nextReadyList.isEmpty()) {
          firstStatus = nextReadyList.remove(0);
        } else {
          MyUtils.errorProgram("SPONTANEOUS NOTIFICATION!!!!!!!!"); // shouldn't happen really						
        }
        tmp = new ArrayList<Status>(nextReadyList);
      }
      outln("next status to process: " + firstStatus);
      if (checkCancelThread()) { // check here in case we sent a signal because of cancelling
        return true;
      }
      check(tmp.isEmpty(), checklog("nextReadyList = " + nextReadyList));
      check(
          null
              == this
                  .currentResult); // Must have been freed by next thread (call to
                                   // resetCurrentResult())

      // Add the newest results for the next thread to process as soon as it's ready
      outln("Add input for next thread");
      this.currentResult = batchResult;
      synchronized (this.statusList) {
        this.statusList.add(Status.RESULTS);
        this.statusList.notify(); // To notify next thread
      }
    } else {
      // Add final results
      outln("Add final results");

      List<Integer> ignoreList = ThreadCommunication.getIgnoreList();
      boolean cancelationInProcess =
          this.rows != null && ThreadCommunication.getTotalRows() >= this.rows;
      if (!cancelationInProcess) {
        try {
          for (List<String> list : batchResult.getValues()) {
            boolean first = true;
            for (int i = 0; i < list.size(); i++) {
              if (!ignoreList.contains(i)) {
                ThreadCommunication.bufferedWriter.write(
                    (first ? "" : MyUtils.TAB_SEPARATOR) + list.get(i));
                if (!QueryRunnerPrototypeConstants.LOG) {
                  System.out.print((first ? "" : MyUtils.TAB_SEPARATOR) + list.get(i));
                }
                first = false;
              }
            }
            ThreadCommunication.bufferedWriter.write(MyUtils.LINE_SEPARATOR);
            if (!QueryRunnerPrototypeConstants.LOG) {
              System.out.println();
            }
            int totalRows = ThreadCommunication.addRowToTotalRows();
            if (!this.unsynchronizedFirstFinalResults && totalRows == 1) { // if first row
              outln("notify first results have arrived");
              synchronized (this.firstFinalResults) {
                this.firstFinalResults.setTrue();
                this.firstFinalResults.notify(); // notify main thread
                this.unsynchronizedFirstFinalResults = true;
              }
            }
            if (!this.unsynchronizedSomeFinalResults
                && this.rows != null
                && totalRows == this.rows.intValue()) {
              outln("notify some results have arrived");
              synchronized (this.someFinalResults) {
                this.unsynchronizedSomeFinalResults = true;
                this.someFinalResults.setTrue();
                this.someFinalResults.notify(); // notify main thread
              }
              break;
            }
          }
          if (QueryRunnerPrototypeConstants.LOG) {
            ThreadCommunication.bufferedWriter.flush();
          }
        } catch (IOException e) {
          throw new TechnicalException(e);
        }
      }
    }
    this.batchResult = null;

    if (!this.first && !this.last) {
      intermediaryResultsProcessingTimer.stopTimer();
      this.intermediaryResultsProcessingAverageTime.addToAverage(
          intermediaryResultsProcessingTimer.getTimeEllapsedMs());
    }

    return false;
  }
Exemple #17
0
  public void fetchDatasetInformation() throws TechnicalException {

    try {
      String databaseMapSerialFilePathAndName =
          this.serialFolderPathAndName + DATABASE_MAP_SERIAL_FILE_NAME;
      String templateMapSerialFilePathAndName =
          this.serialFolderPathAndName + TEMPLATE_MAP_SERIAL_FILE_NAME;
      String datasetMapSerialFilePathAndName =
          this.serialFolderPathAndName + DATASET_MAP_SERIAL_FILE_NAME;
      String datasetMap2SerialFilePathAndName =
          this.serialFolderPathAndName + DATASET_MAP2_SERIAL_FILE_NAME;
      if (new File(databaseMapSerialFilePathAndName).exists()
          && new File(templateMapSerialFilePathAndName).exists()
          && new File(datasetMapSerialFilePathAndName).exists()
          && new File(datasetMap2SerialFilePathAndName).exists()) {
        databaseNameToDatasetInformationMap =
            (HashMap<String, HashMap<String, List<String>>>)
                MyUtils.readSerializedObject(databaseMapSerialFilePathAndName);
        System.out.println(
            "Using serial, quick check: " + this.databaseNameToDatasetInformationMap.size());
        /*datasetNameToDatabaseNameMap = (HashMap<String, String>)MyUtils.readSerializedObject(
        		datasetMapSerialFilePathAndName);
        System.out.println("Using serial, quick check: " + this.datasetNameToDatabaseNameMap.size());*/
        templateNameToDatabaseNameMap =
            (HashMap<String, String>)
                MyUtils.readSerializedObject(templateMapSerialFilePathAndName);
        System.out.println(
            "Using serial, quick check: " + this.templateNameToDatabaseNameMap.size());
        datasetNameToTemplateName =
            (HashMap<String, String>) MyUtils.readSerializedObject(datasetMapSerialFilePathAndName);
        System.out.println("Using serial, quick check: " + this.datasetNameToTemplateName.size());
        datasetNameToDatasetType =
            (HashMap<String, String>)
                MyUtils.readSerializedObject(datasetMap2SerialFilePathAndName);
        System.out.println("Using serial, quick check: " + this.datasetNameToDatasetType.size());
      } else {

        System.out.println("No serial at " + databaseMapSerialFilePathAndName + " , connecting...");

        this.databaseNameToDatasetInformationMap =
            new HashMap<String, HashMap<String, List<String>>>();

        // Check every database for the dataset or the template name
        for (String databaseName : this.databaseNames) {

          connect(databaseName);

          // Prepare list of tables
          this.preparedStatementDatabase =
              this.connection.prepareStatement(
                  "select meta_conf__dataset__main.dataset, meta_conf__dataset__main.type, meta_template__template__main.template "
                      + "from meta_conf__dataset__main, meta_template__template__main "
                      + "where meta_conf__dataset__main.dataset_id_key = meta_template__template__main.dataset_id_key;");

          ResultSet resultSet = this.preparedStatementDatabase.executeQuery();
          HashMap<String, List<String>> map = new HashMap<String, List<String>>();
          resultSet.beforeFirst();
          while (resultSet.next()) {
            String datasetName = resultSet.getString(1);
            String datasetType = resultSet.getString(2);
            String templateName = resultSet.getString(3);

            List<String> list = map.get(templateName);
            if (null == list) {
              list = new ArrayList<String>();
            }
            list.add(datasetName);
            map.put(templateName, list);

            MyUtils.checkStatusProgram(
                TransformationUtils.isGenomicSequence(datasetType)
                    || TransformationUtils.isTableSet(datasetType));

            /*MyUtils.checkStatusProgram(datasetNameToDatabaseNameMap.get(datasetName)==null);
            datasetNameToDatabaseNameMap.put(datasetName, databaseName);*/

            MyUtils.checkStatusProgram(datasetNameToTemplateName.get(datasetName) == null);
            datasetNameToTemplateName.put(datasetName, templateName);

            MyUtils.checkStatusProgram(datasetNameToDatasetType.get(datasetName) == null);
            datasetNameToDatasetType.put(datasetName, datasetType);

            String databaseNameTmp = templateNameToDatabaseNameMap.get(templateName);
            if (null == databaseNameTmp) {
              templateNameToDatabaseNameMap.put(templateName, databaseName);
            } else {
              MyUtils.checkStatusProgram(databaseNameTmp.equals(databaseName));
            }
          }

          this.preparedStatementDatabase.close();
          resultSet.close();

          disconnect();

          MyUtils.checkStatusProgram(
              this.databaseNameToDatasetInformationMap.get(databaseName) == null);
          this.databaseNameToDatasetInformationMap.put(databaseName, map);

          MyUtils.writeSerializedObject(
              this.databaseNameToDatasetInformationMap, databaseMapSerialFilePathAndName);
          // MyUtils.writeSerializedObject(this.datasetNameToDatabaseNameMap,
          // datasetMapSerialFilePathAndName);
          MyUtils.writeSerializedObject(
              this.templateNameToDatabaseNameMap, templateMapSerialFilePathAndName);
          MyUtils.writeSerializedObject(
              this.datasetNameToTemplateName, datasetMapSerialFilePathAndName);
          MyUtils.writeSerializedObject(
              this.datasetNameToDatasetType, datasetMap2SerialFilePathAndName);
        }
      }

      System.out.println(databaseNameToDatasetInformationMap);
    } catch (SQLException e) {
      throw new TechnicalException(e);
    }
  }