Пример #1
1
 /** Insert record in table with current data object. */
 public void insert(Connection con) throws SQLException {
   String sql = "select * from T_Parameters where F_Name=?";
   PreparedStatement stmt = null;
   ResultSet rs = null;
   try {
     stmt = con.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_UPDATABLE);
     stmt.setString(1, name_);
     stmt.setFetchSize(1);
     rs = stmt.executeQuery();
     rs.moveToInsertRow();
     if (setreg_[0]) rs.updateString(1, name_);
     if (setreg_[1]) rs.updateString(2, value_);
     if (setreg_[2]) rs.updateString(3, type_);
     rs.insertRow();
   } catch (SQLException e) {
     log_.severe(e.toString());
     throw e;
   } finally {
     try {
       if (rs != null) rs.close();
       if (stmt != null) stmt.close();
     } catch (SQLException x) {
     }
   }
 }
Пример #2
0
 private void prepareStreamingStatement(Connection connection, StringBuffer currentQuery)
     throws SQLException {
   this.preparedStatement =
       connection.prepareStatement(
           currentQuery.toString(),
           java.sql.ResultSet.TYPE_FORWARD_ONLY,
           java.sql.ResultSet.CONCUR_READ_ONLY);
   if (this.isMySql) {
     preparedStatement.setFetchSize(
         Integer.MIN_VALUE); // MySql's way of telling server to stream (only way)
   } else if (this.isPostgreSql) {
     preparedStatement.setFetchSize(
         1); // Postgres' way of telling server to stream (could be another value) + see autoCommit
   } // Nothing to do for Oracle: streaming is on by default
 }
Пример #3
0
  public ArrayList find(int theBegin, int theEnd) throws SQLException {
    ArrayList list = new ArrayList();
    String sql =
        "select NODE_LINK_ID, NAME, DESCRIPTION, LINK_TYPE, TEMPLATE_ID, CURRENT_NODE_ID, NEXT_NODE_ID, EXECUTOR_RELATION, EXECUTORS_METHOD, NUMBER_OR_PERCENT, PASS_VALUE, EXPRESSION, DEFAULT_PATH, ACTION_NAME from WF_LINK";
    PreparedStatement st = null;
    ResultSet rs = null;
    Connection conn = null;
    try {
      conn = ConnectionFactory.getConnection();
      st = conn.prepareStatement(sql);
      if (theEnd > 0) st.setFetchSize(theEnd);

      rs = st.executeQuery();
      if (theBegin > 1) rs.absolute(theBegin - 1);
      while (rs.next()) {
        list.add(parseResultSet(rs));
        if (rs.getRow() == theEnd) break;
      }
    } catch (SQLException e) {
      e.printStackTrace();
      throw new SQLException(e.getMessage());
    } finally {
      DBHelper.closeConnection(conn, st, rs);
    }

    return list;
  }
Пример #4
0
 public static PreparedStatement getResultStreamingPreparedStatemet(
     Connection conn, String sql, int fetchSize) throws SQLException {
   PreparedStatement pstmt =
       conn.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
   pstmt.setFetchSize(fetchSize);
   return pstmt;
 }
Пример #5
0
  public ArrayList getFollowedLinkList(int theBegin, int theEnd, int templateId, int currentNodeId)
      throws SQLException {
    ArrayList list = new ArrayList();
    String sql = "select * from wf_link where template_id = ? and current_node_id = ?";
    PreparedStatement st = null;
    ResultSet rs = null;
    Connection conn = null;
    try {
      conn = ConnectionFactory.getConnection();
      st = conn.prepareStatement(sql);
      if (theEnd > 0) st.setFetchSize(theEnd);

      st.setInt(1, templateId);
      st.setInt(2, currentNodeId);
      rs = st.executeQuery();
      if (theBegin > 1) rs.absolute(theBegin - 1);
      while (rs.next()) {
        list.add(parseResultSet(rs));
        if (rs.getRow() == theEnd) break;
      }
    } catch (SQLException e) {
      e.printStackTrace();
      throw new SQLException(e.getMessage());
    } finally {
      DBHelper.closeConnection(conn, st, rs);
    }

    return list;
  }
Пример #6
0
  public static void testStreamTest(String sql) {
    String url = "jdbc:mysql://localhost:3306/toy?user=cc&password=chenchi";

    try {
      Class.forName("com.mysql.jdbc.Driver");
    } catch (ClassNotFoundException e) {
      e.printStackTrace();
    }

    Connection connection;
    PreparedStatement preparedStatement;
    ResultSet resultSet;

    int counter = 0;

    try {
      connection = DriverManager.getConnection(url);
      preparedStatement =
          connection.prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
      preparedStatement.setFetchSize(Integer.MIN_VALUE);

      resultSet = preparedStatement.executeQuery();

      while (resultSet.next()) {
        System.out.println(resultSet.getString(2));
        System.out.println(++counter);
      }
    } catch (Exception e) {
      e.printStackTrace();
    } finally {
      System.out.println("finally");
      // 忽略
    }
  }
Пример #7
0
 public <T> T unique(Class<T> type) throws NoSuchElementException, SqlException {
   try {
     statement.setFetchSize(1);
   } catch (SQLException e) {
     throw new SqlException(e.getMessage(), e);
   }
   return execute().unique(type);
 }
Пример #8
0
  public void setFetchSize(int rows) throws SQLException {
    Profiler profiler = _profilerPoint.start();

    try {
      _preparedStatement.setFetchSize(rows);
    } finally {
      profiler.finish();
    }
  }
Пример #9
0
  /**
   * Method to apply any restrictions to the created ResultSet.
   *
   * @param ps The PreparedStatement
   * @param query The query
   * @param applyTimeout Whether to apply the query timeout (if any) direct to the PreparedStatement
   * @throws SQLException Thrown when an error occurs applying the constraints
   */
  public static void prepareStatementForExecution(
      PreparedStatement ps, Query query, boolean applyTimeout) throws SQLException {
    NucleusContext nucleusCtx = query.getExecutionContext().getNucleusContext();
    RDBMSStoreManager storeMgr = (RDBMSStoreManager) query.getStoreManager();
    PersistenceConfiguration conf = nucleusCtx.getPersistenceConfiguration();

    if (applyTimeout) {
      Integer timeout = query.getDatastoreReadTimeoutMillis();
      if (timeout != null && timeout > 0) {
        ps.setQueryTimeout(timeout / 1000);
      }
    }

    // Apply any fetch size
    int fetchSize = 0;
    if (query.getFetchPlan().getFetchSize() > 0) {
      // FetchPlan has a size set so use that
      fetchSize = query.getFetchPlan().getFetchSize();
    }
    if (storeMgr.getDatastoreAdapter().supportsQueryFetchSize(fetchSize)) {
      ps.setFetchSize(fetchSize);
    }

    // Apply any fetch direction
    String fetchDir =
        conf.getStringProperty(RDBMSPropertyNames.PROPERTY_RDBMS_QUERY_FETCH_DIRECTION);
    Object fetchDirExt =
        query.getExtension(RDBMSPropertyNames.PROPERTY_RDBMS_QUERY_FETCH_DIRECTION);
    if (fetchDirExt != null) {
      fetchDir = (String) fetchDirExt;
      if (!fetchDir.equals("forward")
          && !fetchDir.equals("reverse")
          && !fetchDir.equals("unknown")) {
        throw new NucleusUserException(LOCALISER.msg("052512"));
      }
    }

    if (fetchDir.equals("reverse")) {
      ps.setFetchDirection(ResultSet.FETCH_REVERSE);
    } else if (fetchDir.equals("unknown")) {
      ps.setFetchDirection(ResultSet.FETCH_UNKNOWN);
    }

    // Add a limit on the number of rows to include the maximum we may need
    long toExclNo = query.getRangeToExcl();
    if (toExclNo != 0 && toExclNo != Long.MAX_VALUE) {
      if (toExclNo > Integer.MAX_VALUE) {
        // setMaxRows takes an int as input so limit to the correct range
        ps.setMaxRows(Integer.MAX_VALUE);
      } else {
        ps.setMaxRows((int) toExclNo);
      }
    }
  }
Пример #10
0
 /**
  * @param fetchSize hint as to the number of rows that should be fetched from the database at a
  *     time. will be limited to 10,000
  */
 @Override
 public void runPreparedQuery(int fetchSize, String query, Object... data) throws JdbcException {
   try {
     allowReuse();
     preparedStatement = connection.prepareStatement(query, resultSetType, resultSetConcurrency);
     preparedStatement.setFetchSize(Math.min(fetchSize, JdbcConstants.JDBC__MAX_FETCH_SIZE));
     JdbcUtil.setInputParametersForStatement(preparedStatement, data);
     rSet = preparedStatement.executeQuery();
   } catch (SQLException ex) {
     throw newJdbcException(ex);
   }
 }
 @Test
 public void fetchSize() throws Exception {
   boolean ac = getConnection().getAutoCommit();
   getConnection().setAutoCommit(false);
   PreparedStatement p = newScan();
   ResultSet rs = p.executeQuery();
   List<List<Object>> rows = listRows(rs, 2);
   p.setFetchSize(2);
   rs = p.executeQuery();
   assertEquals("Rows with fetch size 2", rows, listRows(rs, 2));
   getConnection().setAutoCommit(ac);
 }
Пример #12
0
 private static PreparedStatement newScrollingSelectStatement(
     DbSession session, String sql, int fetchSize) {
   try {
     PreparedStatement stmt =
         session
             .getConnection()
             .prepareStatement(sql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
     stmt.setFetchSize(fetchSize);
     return stmt;
   } catch (SQLException e) {
     throw new IllegalStateException("Fail to create SQL statement: " + sql, e);
   }
 }
Пример #13
0
  public PreparedStatement createPreparedStatement(String sql, int fetchSize) {
    try {
      if ((connection_ == null) || connection_.isClosed()) {
        return null;
      }

      preparedStatement_ = connection_.prepareStatement(sql);
      preparedStatement_.setFetchSize(fetchSize);
      preparedStatementParameters_ = new ArrayList<>();
      return preparedStatement_;
    } catch (Exception e) {
      logger.error(e.toString() + System.lineSeparator() + StackTrace.getStringFromStackTrace(e));
      return null;
    }
  }
Пример #14
0
  /**
   * Long form of the method to execute a query
   *
   * @param statementScope - the request scope
   * @param conn - the database connection
   * @param sql - the SQL statement to execute
   * @param parameters - the parameters for the statement
   * @param skipResults - the number of results to skip
   * @param maxResults - the maximum number of results to return
   * @param callback - the row handler for the query
   * @throws SQLException - if the query fails
   */
  public void executeQuery(
      StatementScope statementScope,
      Connection conn,
      String sql,
      Object[] parameters,
      int skipResults,
      int maxResults,
      RowHandlerCallback callback)
      throws SQLException {
    logger.info("执行query:{}", sql);
    logger.info("参数:{}", parameters);
    ErrorContext errorContext = statementScope.getErrorContext();
    errorContext.setActivity("executing query");
    errorContext.setObjectId(sql);
    PreparedStatement ps = null;
    ResultSet rs = null;
    setupResultObjectFactory(statementScope);
    try {
      errorContext.setMoreInfo("Check the SQL Statement (preparation failed).");
      Integer rsType = statementScope.getStatement().getResultSetType();
      if (rsType != null) {
        ps = prepareStatement(statementScope.getSession(), conn, sql, rsType);
      } else {
        ps = prepareStatement(statementScope.getSession(), conn, sql);
      }
      setStatementTimeout(statementScope.getStatement(), ps);
      Integer fetchSize = statementScope.getStatement().getFetchSize();
      if (fetchSize != null) {
        ps.setFetchSize(fetchSize.intValue());
      }
      errorContext.setMoreInfo("Check the parameters (set parameters failed).");
      statementScope.getParameterMap().setParameters(statementScope, ps, parameters);
      errorContext.setMoreInfo("Check the statement (query failed).");
      ps.execute();
      errorContext.setMoreInfo("Check the results (failed to retrieve results).");

      // Begin ResultSet Handling
      rs = handleMultipleResults(ps, statementScope, skipResults, maxResults, callback);
      // End ResultSet Handling
    } finally {
      try {
        closeResultSet(rs);
      } finally {
        closeStatement(statementScope.getSession(), ps);
      }
    }
  }
Пример #15
0
 @Override
 public List<BlockImpl> getBlocksAfter(long blockId, int limit) {
   try (Connection con = Db.db.getConnection();
       PreparedStatement pstmt =
           con.prepareStatement(
               "SELECT * FROM block WHERE db_id > (SELECT db_id FROM block WHERE id = ?) ORDER BY db_id ASC LIMIT ?")) {
     List<BlockImpl> result = new ArrayList<>();
     pstmt.setLong(1, blockId);
     pstmt.setInt(2, limit);
     pstmt.setFetchSize(100);
     try (ResultSet rs = pstmt.executeQuery()) {
       while (rs.next()) {
         result.add(BlockDb.loadBlock(con, rs, true));
       }
     }
     return result;
   } catch (SQLException e) {
     throw new RuntimeException(e.toString(), e);
   }
 }
  public List<T> loadNotifications(String listenerName, Status status, int max, int maxFailCount) {
    String sql =
        "SELECT notification, failCount FROM \""
            + tablePrefix
            + listenerName
            + "\" WHERE status=? "
            + (maxFailCount <= 0 ? "" : " and failCount <= ?"); // conditional criterion

    Connection con = null;
    List<T> notifications = new ArrayList<T>();
    try {
      con = dataSource.getConnection();
      PreparedStatement pstmt = con.prepareStatement(sql);
      pstmt.setString(1, status.toString());
      if (maxFailCount > 0) pstmt.setInt(2, maxFailCount);
      pstmt.setFetchSize(max);
      ResultSet resultSet = pstmt.executeQuery();
      int count = 0;
      while (resultSet.next() && ++count <= max) {
        try {
          T notification = deserialise(resultSet.getBytes(1));
          notification.setFailCount(resultSet.getInt(2));
          notifications.add(notification);
          count++;
        } catch (Exception e) {
          LOGGER.warn("Failed deserialising notification", e);
        }
      }
    } catch (SQLException e) {
      LOGGER.error("Failed updating notification log", e);
    } finally {
      if (con != null) {
        try {
          con.close();
        } catch (Exception e) {
          LOGGER.error(e);
        }
      }
    }
    return notifications;
  }
Пример #17
0
 /**
  * 初始化真正的PreparedStatement,对当前对象的操作全部都设置到真正的PreparedStatement
  *
  * @throws SQLException
  */
 private void prepare() throws SQLException {
   DALFactory dalFactory = DALFactory.getDefault();
   List<Object> values = dalParameters.getValues();
   Map<String, Object> context = new HashMap<String, Object>();
   SQLStruct sqlStruct = dalFactory.getSqlAnalyzer().parse(sql, context);
   SQLInfo sqlInfo = null;
   if (sqlStruct.isCanParse()) {
     sqlInfo =
         dalFactory
             .getSqlAnalyzer()
             .analyse(sql, sqlStruct, values.toArray(new Object[values.size()]), context);
   }
   this.parsePartition(sqlStruct, sqlInfo);
   this.initRealPreparedStatement();
   if (this.maxFieldSize != 0) {
     ps.setMaxFieldSize(maxFieldSize);
   }
   if (this.maxRows != 0) {
     ps.setMaxRows(maxRows);
   }
   if (!this.escapeProcessing) {
     ps.setEscapeProcessing(escapeProcessing);
   }
   if (this.queryTimeout != 0) {
     ps.setQueryTimeout(queryTimeout);
   }
   if (this.cursorName != null) {
     ps.setCursorName(cursorName);
   }
   if (this.fetchDirection != 0) {
     ps.setFetchDirection(fetchDirection);
   }
   if (this.fetchSize != 0) {
     ps.setFetchSize(fetchSize);
   }
   if (!this.poolable) {
     ps.setPoolable(poolable);
   }
   this.dalParameters.initRealPreparedStatement(ps);
 }
Пример #18
0
 @Override
 public List<BlockImpl> getBlocksAfter(long blockId, List<Long> blockList) {
   List<BlockImpl> result = new ArrayList<>();
   if (blockList.isEmpty()) return result;
   try (Connection con = Db.db.getConnection();
       PreparedStatement pstmt =
           con.prepareStatement(
               "SELECT * FROM block WHERE db_id > (SELECT db_id FROM block WHERE id = ?) ORDER BY db_id ASC LIMIT ?")) {
     pstmt.setLong(1, blockId);
     pstmt.setInt(2, blockList.size());
     pstmt.setFetchSize(100);
     try (ResultSet rs = pstmt.executeQuery()) {
       int index = 0;
       while (rs.next()) {
         BlockImpl block = BlockDb.loadBlock(con, rs, true);
         if (block.getId() != blockList.get(index++)) break;
         result.add(block);
       }
     }
     return result;
   } catch (SQLException e) {
     throw new RuntimeException(e.toString(), e);
   }
 }
Пример #19
0
  public static void DbObdc() throws ClassNotFoundException {
    Connection conn = null;
    Statement stmt = null;
    PreparedStatement pstmt = null;
    ResultSet rs = null;
    // 1.装载数据库
    Class.forName(JDBC_DRIVER);
    try {
      // 2.建立数据库连接
      conn = DriverManager.getConnection(DB_URL, USER, PASSWORD);
      DB_URL = DB_URL + "useCursorFetch=true";
      // 3.执行SQL数据库
      pstmt = conn.prepareStatement("select ProductName,inventory from Product");
      pstmt.setFetchSize(1);
      rs = pstmt.executeQuery();
      // 4.获取执行结果
      while (rs.next()) {
        System.out.print("ProductName:" + rs.getString("ProductName"));
        System.out.print(" ");
        System.out.print("inventory:" + rs.getString("inventory"));
        System.out.println();
      }

    } catch (SQLException e) {
      // 异常处理
      e.printStackTrace();
    } finally {
      try {
        // 5.清理环境
        if (conn != null) conn.close();
        if (stmt != null) stmt.close();
        if (rs != null) rs.close();
      } catch (SQLException e) {
      }
    }
  }
 @Test
 public void testFetchSize() {
   long startNanos = System.nanoTime();
   doInJDBC(
       connection -> {
         try (PreparedStatement statement = connection.prepareStatement("select * from post")) {
           if (fetchSize != null) {
             statement.setFetchSize(fetchSize);
           }
           statement.execute();
           ResultSet resultSet = statement.getResultSet();
           while (resultSet.next()) {
             resultSet.getLong(1);
           }
         } catch (SQLException e) {
           fail(e.getMessage());
         }
       });
   LOGGER.info(
       "{} fetch size {} took {} millis",
       dataSourceProvider().database(),
       fetchSize != null ? fetchSize : "N/A",
       TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos));
 }
  private EventReaderSummary readEventsForSource(
      OracleTriggerMonitoredSourceInfo sourceInfo, long maxScn)
      throws DatabusException, EventCreationException, UnsupportedKeyException, SQLException,
          IOException {
    int retryMax = _numRetries;
    int numRetry = 0;
    Connection conn = null;
    PreparedStatement pstmt = null;
    ResultSet rs = null;
    KeyType keyType = _pKeyTypeMap.get(sourceInfo.getEventView());
    String keyName = _pKeyNameMap.get(sourceInfo.getEventView());
    String sql = _eventQueryMap.get(sourceInfo.getEventView());
    String beginSrcKey = _beginSrcKeyMap.get(sourceInfo.getEventView());
    String endSrcKey = _endSrcKeyMap.get(sourceInfo.getEventView());

    if (sql == null) {
      sql =
          generateEventQuery2(
              sourceInfo, keyName, keyType, getPKIndex(sourceInfo), getQueryHint(sourceInfo));
    }
    LOG.info("Chunked  Query for Source (" + sourceInfo + ") is :" + sql);
    LOG.info("EndSrcKey for source (" + sourceInfo + ") is :" + endSrcKey);

    PrimaryKeyTxn endKeyTxn = null;
    if ((null != endSrcKey) && (!endSrcKey.trim().isEmpty())) {
      if (KeyType.LONG == keyType) endKeyTxn = new PrimaryKeyTxn(new Long(endSrcKey));
      else endKeyTxn = new PrimaryKeyTxn(endSrcKey);
    }

    long timestamp = System.currentTimeMillis();
    int numRowsFetched = 0;
    long totalEventSize = 0;
    long timeStart = System.currentTimeMillis();
    long checkpointInterval = _commitInterval;
    boolean done = false;
    long lastTime = timeStart;
    long numRows = 0;
    PrimaryKeyTxn pKey = null;

    String minKeySQL = generateMinKeyQuery(sourceInfo, keyName);

    String srcName = sourceInfo.getEventView();
    LOG.info("Bootstrapping for Source :" + srcName);
    String lastKey = _lastKeys.get(sourceInfo.getEventView());
    File f = _keyTxnFilesMap.get(srcName);
    FileWriter oStream = new FileWriter(f, f.exists());
    BufferedWriter keyTxnWriter = new BufferedWriter(oStream, _keyTxnBufferSizeMap.get(srcName));

    _bootstrapSeedWriter.startEvents();
    RateMonitor seedingRate = new RateMonitor("Seeding Rate");
    RateMonitor queryRate = new RateMonitor("Query Rate");
    seedingRate.start();
    seedingRate.suspend();
    queryRate.start();
    queryRate.suspend();
    boolean isException = false;
    long totProcessTime = 0;
    try {
      conn = _dataSource.getConnection();
      pstmt = conn.prepareStatement(sql);

      if (_enableNumRowsQuery) numRows = getNumRows(conn, getTableName(sourceInfo));
      else numRows = -1;

      long currRowId = _lastRows.get(sourceInfo.getEventView());

      /**
       * First Key to be seeded will be decided in the following order: 1. Use
       * bootstrap_seeder_state's last srcKey as the key for the first chunk. 2. If (1) is empty,
       * use passed-in begin srcKey. 3. If (2) is also empty, use Oracle's minKey as the first Chunk
       * Key.
       */
      if (null == lastKey) {
        lastKey = _beginSrcKeyMap.get(sourceInfo.getEventView());
        LOG.info(
            "No last Src Key available in bootstrap_seeder_state for source ("
                + sourceInfo
                + ". Trying beginSrc Key from config :"
                + lastKey);
      }

      if ((null == lastKey) || (lastKey.trim().isEmpty())) {
        if (KeyType.LONG == keyType) pKey = new PrimaryKeyTxn(executeAndGetLong(minKeySQL));
        else pKey = new PrimaryKeyTxn(executeAndGetString(minKeySQL));
      } else {
        if (KeyType.LONG == keyType) pKey = new PrimaryKeyTxn(Long.parseLong(lastKey));
        else pKey = new PrimaryKeyTxn(lastKey);
      }

      PrimaryKeyTxn lastRoundKeyTxn = new PrimaryKeyTxn(pKey);
      PrimaryKeyTxn lastKeyTxn = new PrimaryKeyTxn(pKey);
      long numUniqueKeysThisRound = 0;

      boolean first = true;
      _rate.resume();
      while (!done) {
        LOG.info("MinKey being used for this round:" + pKey);
        numUniqueKeysThisRound = 0;
        try {
          lastRoundKeyTxn.copyFrom(pKey);

          if (KeyType.LONG == keyType) {
            pstmt.setLong(1, pKey.getKey());
          } else {
            String key = pKey.getKeyStr();
            pstmt.setString(1, key);
          }

          pstmt.setLong(2, _numRowsPerQuery);
          pstmt.setFetchSize(_numRowsPrefetch);

          if (_oraclePreparedStatementClass.isInstance(pstmt)) {
            try {
              _setLobPrefetchSizeMethod.invoke(pstmt, _LOBPrefetchSize);
            } catch (Exception e) {
              throw new EventCreationException("Unable to set Lob Prefetch size" + e.getMessage());
            }
          }

          LOG.info(
              "Executing Oracle Query :"
                  + sql
                  + ". Key: "
                  + pKey
                  + ",NumRows: "
                  + _numRowsPerQuery);
          queryRate.resume();
          rs = pstmt.executeQuery();
          queryRate.suspend();

          LOG.info("Total Query Latency :" + queryRate.getDuration() / 1000000000L);
          long totLatency = 0;
          long txnId = 0;
          int numRowsThisRound = 0;
          seedingRate.resume();
          while (rs.next()) {
            _rate.tick();
            seedingRate.tick();
            currRowId++;
            txnId = rs.getLong(2);
            if (KeyType.LONG == keyType) {
              pKey.setKeyTxn(rs.getLong(1), txnId);
            } else {
              String key = rs.getString(1);
              pKey.setKeyStrTxn(key, txnId);
            }

            // Write TXN to file
            pKey.writeTo(keyTxnWriter);

            // LOG.info("TXNId is :" + txnId + ",RowId is :" + currRowId);
            long start = System.nanoTime();
            long eventSize =
                sourceInfo
                    .getFactory()
                    .createAndAppendEvent(maxScn, timestamp, rs, _bootstrapSeedWriter, false, null);
            long latency = System.nanoTime() - start;
            totLatency += latency;
            totalEventSize += eventSize;
            totProcessTime += (totLatency / 1000 * 1000);
            numRowsFetched++;
            numRowsThisRound++;

            if (lastKeyTxn.compareKey(pKey) != 0) {
              numUniqueKeysThisRound++;
              lastKeyTxn.copyFrom(pKey);
            }

            if (numRowsFetched % checkpointInterval == 0) {
              // Commit this batch and reinit
              _bootstrapSeedWriter.endEvents(currRowId, timestamp, null);
              keyTxnWriter.flush();
              _bootstrapSeedWriter.startEvents();
              long procTime = totLatency / 1000000000;
              long currTime = System.currentTimeMillis();
              long diff = (currTime - lastTime) / 1000;
              long timeSinceStart = (currTime - timeStart) / 1000;
              double currRate = _rate.getRate();
              currRate = (currRate <= 0) ? 1 : currRate;

              if (_enableNumRowsQuery) {
                double remTime = (numRows - currRowId) / (currRate);

                LOG.info(
                    "Processed "
                        + checkpointInterval
                        + " rows in "
                        + diff
                        + " seconds, Processing Time (seconds) so far :"
                        + (procTime)
                        + ",Seconds elapsed since start :"
                        + (timeSinceStart)
                        + ",Approx Seconds remaining :"
                        + remTime
                        + ",Overall Row Rate:"
                        + _rate.getRate()
                        + "("
                        + seedingRate.getRate()
                        + ")"
                        + ",NumRows Fetched so far:"
                        + numRowsFetched
                        + ". TotalEventSize :"
                        + totalEventSize);
              } else {
                LOG.info(
                    "Processed "
                        + checkpointInterval
                        + " rows in "
                        + diff
                        + " seconds, Processing Time (seconds) so far :"
                        + (procTime)
                        + ",Seconds elapsed since start :"
                        + (timeSinceStart)
                        + ",Overall Row Rate:"
                        + _rate.getRate()
                        + "("
                        + seedingRate.getRate()
                        + ")"
                        + ",NumRows Fetched so far:"
                        + numRowsFetched
                        + ". TotalEventSize :"
                        + totalEventSize);
              }
              lastTime = currTime;
            }

            if ((null != endKeyTxn) && (endKeyTxn.compareKey(lastKeyTxn) < 0)) {
              LOG.info(
                  "Seeding to be stopped for current source as it has completed seeding upto endSrckey :"
                      + endKeyTxn
                      + ", Current SrcKey :"
                      + lastKeyTxn);
              break;
            }
          }
          seedingRate.suspend();

          if ((numRowsThisRound <= 1)
              || ((numRowsThisRound < _numRowsPerQuery) && (numUniqueKeysThisRound <= 1))) {
            LOG.info(
                "Seeding Done for source :"
                    + sourceInfo.getEventView()
                    + ", numRowsThisRound :"
                    + numRowsThisRound
                    + ", _numRowsPerQuery :"
                    + _numRowsPerQuery
                    + ", numUniqueKeys :"
                    + numUniqueKeysThisRound);
            done = true;
          } else if ((numRowsThisRound == _numRowsPerQuery) && (numUniqueKeysThisRound <= 1)) {
            String msg =
                "Seeding stuck at infinte loop for source : "
                    + sourceInfo.getEventView()
                    + ", numRowsThisRound :"
                    + numRowsThisRound
                    + ", _numRowsPerQuery :"
                    + _numRowsPerQuery
                    + ", numUniqueKeys :"
                    + numUniqueKeysThisRound
                    + ", lastChunkKey :"
                    + lastRoundKeyTxn;
            LOG.error(msg);
            throw new DatabusException(msg);
          } else if (null != endKeyTxn) {
            if (endKeyTxn.compareKey(lastKeyTxn) < 0) {
              LOG.info(
                  "Seeding stopped for source :"
                      + sourceInfo.getEventView()
                      + ", as it has completed seeding upto the endSrckey :"
                      + endKeyTxn
                      + ", numRowsThisRound :"
                      + numRowsThisRound
                      + ", _numRowsPerQuery :"
                      + _numRowsPerQuery
                      + ", numUniqueKeys :"
                      + numUniqueKeysThisRound
                      + " , Current SrcKey :"
                      + lastKeyTxn);
              done = true;
            }
          }

          if (currRowId > 0 && (!first || done)) {
            currRowId--; // Since next time, we will read the last seen record again
          }
          LOG.info("about to call end events with currRowId = " + currRowId);
          first = false;
          _bootstrapSeedWriter.endEvents(currRowId, timestamp, null);
          isException = false;
        } catch (SQLException ex) {
          LOG.error("Got SQLException for source (" + sourceInfo + ")", ex);

          _bootstrapSeedWriter.rollbackEvents();

          numRetry++;
          isException = true;

          if (numRetry >= retryMax) {
            throw new DatabusException(
                "Error: Reached max retries for reading/processing bootstrap", ex);
          }
        } finally {
          DBHelper.close(rs);
          rs = null;
        }
      }
    } catch (DatabusException ex) {

      isException = true;
      throw ex;

    } finally {

      DBHelper.close(rs, pstmt, conn);
      keyTxnWriter.close();
      rs = null;
      _rate.suspend();

      if (!isException) {
        dedupeKeyTxnFile(_keyTxnFilesMap.get(srcName), keyType);
      }
    }
    long timeEnd = System.currentTimeMillis();
    long elapsedMin = (timeEnd - timeStart) / (MILLISEC_TO_MIN);
    LOG.info(
        "Processed "
            + numRowsFetched
            + " rows of Source: "
            + sourceInfo.getSourceName()
            + " in "
            + elapsedMin
            + " minutes");
    return new EventReaderSummary(
        sourceInfo.getSourceId(),
        sourceInfo.getSourceName(),
        -1,
        numRowsFetched,
        totalEventSize,
        (timeEnd - timeStart),
        totProcessTime,
        0,
        0,
        0);
  }
  private void generateChunkingPreparedStatement(Table table, long blockSize) throws SQLException {
    String fqnTable =
        connection.getDatabaseObjectName(table.getSchema())
            + '.'
            + connection.getDatabaseObjectName(table.getName());

    StringBuffer sqlBuffer = new StringBuffer("SELECT ");
    StringBuffer colBuf = new StringBuffer();
    whereClause = new String();

    if (table.getPrimaryKey() != null) {
      // TODO
      // No dedicated chunking algorithm for this type of pk (either
      // composite or datatype not handled)
    } else {
      if (logger.isDebugEnabled()) logger.debug("Handling table " + table.toExtendedString());
      // This is a unique key that can be used
      Key key = table.getPKFromUniqueIndex();

      if (key == null) {
        logger.info("getPKFromUniqueIndex returned null key");
      }
      ArrayList<Column> colsList = key.getColumns();

      if (logger.isDebugEnabled()) logger.debug("colsList = " + colsList);

      Column[] columns = new Column[colsList.size()];
      int i = 0;
      for (Column column : colsList) {
        columns[i] = column;
        i++;
      }

      whereClause = buildWhereClause(columns, 0);

      for (int j = 0; j < columns.length; j++) {
        if (j > 0) {
          colBuf.append(", ");
        }
        colBuf.append(columns[j].getName());
      }
      sqlBuffer.append(colBuf);
    }

    sqlBuffer.append(" FROM ");
    sqlBuffer.append(fqnTable);
    if (eventId != null) {
      sqlBuffer.append(" AS OF SCN ");
      sqlBuffer.append(eventId);
    }

    sqlBuffer.append(" ORDER BY ");
    sqlBuffer.append(colBuf);

    String sql = sqlBuffer.toString();
    if (logger.isDebugEnabled()) logger.debug("Generated statement :" + sql);
    pStmt = connection.prepareStatement(sql);

    // TODO : have a setting ?
    pStmt.setFetchSize(100);
  }
 public void setFetchSize(int rows) throws SQLException {
   delegate.setFetchSize(rows);
 }
Пример #24
0
  @Override
  public void execute(Connection sourceConnection, Connection targetConnection, StrategyModel data)
      throws StrategyException, SQLException, ClassNotFoundException {

    int period = DEFAULT_PERIOD;
    if (data.getParam(PERIOD) != null) {
      period = Integer.parseInt(data.getParam(PERIOD));
    }

    Timestamp date = new Timestamp(Calendar.getInstance().getTimeInMillis() - period);

    int partEmail = DEFAULT_PART_EMAIL;
    if (data.getParam(PART_EMAIL) != null) {
      partEmail = Integer.parseInt(data.getParam(PART_EMAIL));
    }

    int rowCount = 0;

    try (PreparedStatement selectErrorsCount =
        sourceConnection.prepareStatement(
            "SELECT count(*) as count FROM rep2_superlog where c_date <= ?"); ) {

      selectErrorsCount.setTimestamp(1, date);
      try (ResultSet countResult = selectErrorsCount.executeQuery(); ) {
        if (countResult.next()) {
          rowCount = countResult.getInt(COUNT);
        }
      }
    }
    // Если нет ошибок то смысл в запуске данного кода бессмыслен
    if (rowCount != 0) {
      try (PreparedStatement selectPreparedStatement =
          sourceConnection.prepareStatement(
              "SELECT * FROM rep2_superlog where c_date <= ? ORDER BY id_superlog",
              ResultSet.TYPE_FORWARD_ONLY,
              ResultSet.CONCUR_READ_ONLY); ) {
        selectPreparedStatement.setTimestamp(1, date);
        selectPreparedStatement.setFetchSize(partEmail);

        try (ResultSet resultSet = selectPreparedStatement.executeQuery(); ) {
          List<String> cols = new ArrayList<String>(JdbcMetadata.getColumns(resultSet));
          int count = 0;
          StringBuffer rowDumpEmail =
              new StringBuffer(
                  String.format(
                      "\n\nВ %s превышен лимит таймаута в superlog в %s миллисекунд!\n\n",
                      data.getRunner().getSource().getPoolId(), period));
          while (resultSet.next() && (count < partEmail)) {
            count++;
            // при необходимости пишем ошибку в лог
            String rowDump =
                String.format(
                    "Ошибка настроек %s из %s \n[ tableName = REP2_SUPERLOG [ row = %s ] ]%s",
                    count,
                    rowCount,
                    Jdbc.resultSetToString(resultSet, cols),
                    "\n==========================================\n");
            rowDumpEmail.append(rowDump);
          }
          rowDumpEmail.append("Всего ");
          rowDumpEmail.append(rowCount);
          rowDumpEmail.append(
              " ошибочных записей. Полный список ошибок доступен в таблице REP2_SUPERLOG.");
          LOG.error(rowDumpEmail.toString());
        }
      }
    }
  }
Пример #25
0
 public void setFetchSize(int rows) throws SQLException {
   this.fetchSize = rows;
   if (ps != null) {
     ps.setFetchSize(rows);
   }
 }
Пример #26
0
 public void setFetchSize(int rows) throws SQLException {
   statement.setFetchSize(rows);
 }
Пример #27
0
  /**
   * Get the samples: <code>result_set</code> will have the samples, <code>value</code> will contain
   * the first sample
   *
   * @param start Start time
   * @param end End time
   * @throws Exception on error, including cancellation
   */
  private void determineInitialSample(final Timestamp start, final Timestamp end) throws Exception {
    java.sql.Timestamp start_stamp = TimestampHelper.toSQLTimestamp(start);
    final java.sql.Timestamp end_stamp = TimestampHelper.toSQLTimestamp(end);

    // Get time of initial sample
    final PreparedStatement statement =
        reader.getRDB().getConnection().prepareStatement(reader.getSQL().sample_sel_initial_time);
    reader.addForCancellation(statement);
    try {
      statement.setInt(1, channel_id);
      statement.setTimestamp(2, start_stamp);
      final ResultSet result = statement.executeQuery();
      if (result.next()) {
        // System.out.print("Start time corrected from " + start_stamp);
        start_stamp = result.getTimestamp(1);
        // Oracle has nanoseconds in TIMESTAMP, MySQL in separate column
        if (reader.getRDB().getDialect() == Dialect.MySQL
            || reader.getRDB().getDialect() == Dialect.PostgreSQL)
          start_stamp.setNanos(result.getInt(2));
        // System.out.println(" to " + start_stamp);
      }
    } finally {
      reader.removeFromCancellation(statement);
      statement.close();
    }

    // Fetch the samples
    if (reader.useArrayBlob())
      sel_samples =
          reader
              .getRDB()
              .getConnection()
              .prepareStatement(reader.getSQL().sample_sel_by_id_start_end_with_blob);
    else
      sel_samples =
          reader
              .getRDB()
              .getConnection()
              .prepareStatement(reader.getSQL().sample_sel_by_id_start_end);
    sel_samples.setFetchDirection(ResultSet.FETCH_FORWARD);

    // Test w/ ~170000 raw samples:
    //     10  17   seconds
    //    100   6   seconds
    //   1000   4.x seconds
    //  10000   4.x seconds
    // 100000   4.x seconds
    // So default is bad. 100 or 1000 are good.
    // Bigger numbers don't help much in repeated tests, but
    // just to be on the safe side, use a bigger number.
    sel_samples.setFetchSize(Preferences.getFetchSize());

    reader.addForCancellation(sel_samples);
    sel_samples.setInt(1, channel_id);
    sel_samples.setTimestamp(2, start_stamp);
    sel_samples.setTimestamp(3, end_stamp);
    result_set = sel_samples.executeQuery();
    // Get first sample
    if (result_set.next()) value = decodeSampleTableValue(result_set, true);
    // else leave value null to indicate end of samples
  }
  private void checkTableData(
      final String sourceConnectorId,
      final Connection sourceConnection,
      final SourceDatabaseConfiguration sourceConfiguration,
      final TableMetaData sourceTableMetaData,
      final String targetConnectorId,
      final Connection targetConnection,
      final SourceDatabaseConfiguration targetConfiguration,
      final TableMetaData targetTableMetaData,
      final int numberOfCheckData)
      throws SQLException {
    final String tableName1 =
        _connectorRepository
            .getConnectorHint(sourceConnectorId, TableNameMapper.class)
            .getValue()
            .mapTableName(sourceTableMetaData);
    final String tableName2 =
        _connectorRepository
            .getConnectorHint(targetConnectorId, TableNameMapper.class)
            .getValue()
            .mapTableName(targetTableMetaData);
    final CommonColumnTypeResolverTool commonColumnTypeResolver =
        new CommonColumnTypeResolverTool(_connectorRepository);
    final ColumnNameMapper sourceColumnNameMapper =
        _connectorRepository.getConnectorHint(sourceConnectorId, ColumnNameMapper.class).getValue();
    final ColumnNameMapper targetColumnNameMapper =
        _connectorRepository.getConnectorHint(targetConnectorId, ColumnNameMapper.class).getValue();

    if (sourceTableMetaData.getRowCount() != targetTableMetaData.getRowCount()) {
      throw new UnequalNumberOfRowsException(
          "Number of rows is not equal: "
              + tableName1
              + "="
              + sourceTableMetaData.getRowCount()
              + " vs. "
              + tableName2
              + "="
              + targetTableMetaData.getRowCount());
    }

    LOG.info("Checking data of " + tableName1 + " <--> " + tableName2 + " started");

    final PreparedStatement selectStatement1 =
        new SelectStatementCreator(_connectorRepository, sourceConnectorId)
            .createSelectStatement(sourceConnection, tableName1, sourceTableMetaData);
    selectStatement1.setFetchSize(numberOfCheckData);

    sourceConfiguration.beforeSelect(sourceConnection, sourceConnectorId, sourceTableMetaData);
    final ResultSet resultSet1 = selectStatement1.executeQuery();
    sourceConfiguration.afterSelect(sourceConnection, sourceConnectorId, sourceTableMetaData);

    final PreparedStatement selectStatement2 =
        new SelectStatementCreator(_connectorRepository, targetConnectorId)
            .createMappedSelectStatement(
                targetConnection,
                sourceTableMetaData,
                tableName2,
                targetTableMetaData,
                sourceConnectorId);
    selectStatement2.setFetchSize(numberOfCheckData);

    targetConfiguration.beforeSelect(targetConnection, targetConnectorId, targetTableMetaData);
    final ResultSet resultSet2 = selectStatement2.executeQuery();
    targetConfiguration.afterSelect(targetConnection, targetConnectorId, targetTableMetaData);

    final List<ColumnMetaData> orderedSourceColumns =
        ColumnOrderHint.getSortedColumns(
            _connectorRepository, sourceConnectorId, sourceTableMetaData);
    final ColumnMapper columnMapper =
        _connectorRepository.getConnectorHint(targetConnectorId, ColumnMapper.class).getValue();

    int rowIndex = 1;

    try {
      while (resultSet1.next() && resultSet2.next() && rowIndex <= numberOfCheckData) {
        int targetColumnIndex = 1;

        for (int sourceColumnIndex = 1;
            sourceColumnIndex <= orderedSourceColumns.size();
            sourceColumnIndex++) {
          final ColumnMetaData sourceColumn = orderedSourceColumns.get(sourceColumnIndex - 1);
          final ColumnMapperResult mapping = columnMapper.map(sourceColumn, targetTableMetaData);

          for (final ColumnMetaData columnMetaData2 : mapping.getColumns()) {
            final ColumnTypeMapping columnTypeMapping =
                commonColumnTypeResolver.getCommonColumnTypeMapping(
                    sourceConnectorId, sourceColumn, targetConnectorId, columnMetaData2);
            final String columnName1 = sourceColumnNameMapper.mapColumnName(sourceColumn);
            final String columnName2 = targetColumnNameMapper.mapColumnName(columnMetaData2);

            if (columnTypeMapping == null) {
              throw new IncompatibleColumnsException(
                  tableName1
                      + ": Columns have incompatible types: "
                      + columnName1
                      + "/"
                      + sourceColumn.getColumnTypeName()
                      + " vs. "
                      + columnName2
                      + "/"
                      + columnMetaData2.getColumnTypeName());
            }

            final ColumnType sourceColumnType = columnTypeMapping.getSourceColumnType();
            Object data1 = sourceColumnType.getValue(resultSet1, sourceColumnIndex);
            data1 =
                columnTypeMapping.getColumnDataMapper().map(sourceColumn, columnMetaData2, data1);
            Object data2 =
                columnTypeMapping.getTargetColumnType().getValue(resultSet2, targetColumnIndex);

            switch (sourceColumnType) {
              case CLASS_STRING:
                final ConnectorInfo connectionInfo1 =
                    _connectorRepository.getConnectionInfo(sourceConnectorId);
                final ConnectorInfo connectionInfo2 =
                    _connectorRepository.getConnectionInfo(targetConnectorId);

                // See http://www.postgresql.org/docs/8.3/static/datatype-character.html
                if (DatabaseType.POSTGRESQL.equals(connectionInfo1.getDatabaseType())
                    || DatabaseType.POSTGRESQL.equals(connectionInfo2.getDatabaseType())) {
                  data1 = trim((String) data1);
                  data2 = trim((String) data2);
                }
                break;
              case CLASS_BLOB:
                final Blob blob1 = (Blob) data1;
                final Blob blob2 = (Blob) data2;
                data1 = createStringFromBlob(blob1);
                data2 = createStringFromBlob(blob2);
                break;

              default:
                // No conversion needed
                break;
            }

            if ((data1 == null && data2 != null) || (data1 != null && data2 == null)) {
              throw createIncompatibleDataException(
                  tableName1, rowIndex, sourceColumnType, columnName1, data1, data2);
            } else if (data1 != null && data2 != null && !data1.equals(data2)) {
              throw createIncompatibleDataException(
                  tableName1, rowIndex, sourceColumnType, columnName1, data1, data2);
            }
          }

          targetColumnIndex += mapping.getColumns().size();
        }

        rowIndex++;
      }
    } finally {
      try {
        resultSet1.close();
        selectStatement1.close();
        resultSet2.close();
        selectStatement2.close();
      } catch (final Exception e) {
      }
    }

    LOG.info("Checking data of " + tableName1 + " <--> " + tableName2 + " finished");
  }
  private void classify(ExperimentConfiguration config)
      throws IOException, SQLException, ClassNotFoundException {

    // get trainingdata from file (and db)
    File trainingDataFile = new File(trainingDataFileName);
    List<ClassifyUnit> trainingData = new ArrayList<ClassifyUnit>();

    if (trainWithFile) {
      trainingData.addAll(
          jobs.getCategorizedParagraphsFromFile(
              trainingDataFile, config.getFeatureConfiguration().isTreatEncoding()));
      System.out.println(
          "added " + trainingData.size() + " training-paragraphs from training-file ");
    }

    if (trainWithDB) {
      List<ClassifyUnit> dbTrainingData =
          jobs.getCategorizedParagraphsFromDB(
              trainingDb, config.getFeatureConfiguration().isTreatEncoding());
      trainingData.addAll(dbTrainingData);
      System.out.println("added " + dbTrainingData.size() + " training-paragraphs from DB ");
    }
    if (trainingData.size() == 0) {
      System.out.println(
          "\nthere are no training paragraphs in the specified training-DB. \nPlease check configuration and try again");
      System.exit(0);
    }
    System.out.println("overall training paragraphs: " + trainingData.size() + "\n");
    System.out.println("\n...classifying...\n");

    trainingData = jobs.initializeClassifyUnits(trainingData);
    trainingData = jobs.setFeatures(trainingData, config.getFeatureConfiguration(), true);
    trainingData = jobs.setFeatureVectors(trainingData, config.getFeatureQuantifier(), null);

    // build model
    Model model = jobs.getNewModelForClassifier(trainingData, config);
    if (config.getModelFileName().contains("/myModels/")) {
      jobs.exportModel(config.getModelFile(), model);
    }
    // get data from db
    int done = 0;
    String query = null;
    int zeilenNr = 0, jahrgang = 0;
    ;
    int jobAdCount = 0;
    int paraCount = 0;
    query = "SELECT ZEILENNR, Jahrgang, STELLENBESCHREIBUNG FROM DL_ALL_Spinfo LIMIT ? OFFSET ?;";

    PreparedStatement prepStmt = inputDb.prepareStatement(query);
    prepStmt.setInt(1, queryLimit);
    prepStmt.setInt(2, currentId);
    prepStmt.setFetchSize(fetchSize);
    // execute
    ResultSet queryResult = prepStmt.executeQuery();

    // total entries to process:
    if (queryLimit < 0) {

      String countQuery = "SELECT COUNT(*) FROM DL_ALL_Spinfo;";
      Statement stmt = inputDb.createStatement();
      ResultSet countResult = stmt.executeQuery(countQuery);
      int tableSize = countResult.getInt(1);
      stmt.close();
      stmt = inputDb.createStatement();
      ResultSet rs = null;
      rs = stmt.executeQuery("SELECT COALESCE(" + tableSize + "+1, 0) FROM DL_ALL_Spinfo;");

      queryLimit = rs.getInt(1);
    }

    boolean goOn = true;
    boolean askAgain = true;
    long start = System.currentTimeMillis();

    while (queryResult.next() && goOn) {
      jobAdCount++;
      String jobAd = null;
      zeilenNr = queryResult.getInt("ZEILENNR");
      jahrgang = queryResult.getInt("Jahrgang");
      jobAd = queryResult.getString("STELLENBESCHREIBUNG");
      // if there is an empty job description, classifying is of no use,
      // so skip
      if (jobAd == null) {
        System.out.println("jobAd ist null");
        continue;
      }
      if (jobAd.isEmpty()) {
        System.out.println("Ist leer!");
        continue;
      }

      // 1. Split into paragraphs and create a ClassifyUnit per paragraph
      List<String> paragraphs = ClassifyUnitSplitter.splitIntoParagraphs(jobAd);
      // if treat enc
      if (config.getFeatureConfiguration().isTreatEncoding()) {
        paragraphs = EncodingProblemTreatment.normalizeEncoding(paragraphs);
      }
      List<ClassifyUnit> classifyUnits = new ArrayList<ClassifyUnit>();
      for (String string : paragraphs) {
        paraCount++;
        classifyUnits.add(new JASCClassifyUnit(string, jahrgang, zeilenNr));
      }
      // prepare ClassifyUnits
      classifyUnits = jobs.initializeClassifyUnits(classifyUnits);
      classifyUnits = jobs.setFeatures(classifyUnits, config.getFeatureConfiguration(), false);
      classifyUnits =
          jobs.setFeatureVectors(classifyUnits, config.getFeatureQuantifier(), model.getFUOrder());

      // 2. Classify
      RegexClassifier regexClassifier = new RegexClassifier("classification/data/regex.txt");
      Map<ClassifyUnit, boolean[]> preClassified = new HashMap<ClassifyUnit, boolean[]>();
      for (ClassifyUnit cu : classifyUnits) {
        boolean[] classes = regexClassifier.classify(cu, model);
        preClassified.put(cu, classes);
      }
      Map<ClassifyUnit, boolean[]> classified = jobs.classify(classifyUnits, config, model);
      classified = jobs.mergeResults(classified, preClassified);
      classified = jobs.translateClasses(classified);

      List<ClassifyUnit> results = new ArrayList<ClassifyUnit>();
      for (ClassifyUnit cu : classified.keySet()) {
        ((ZoneClassifyUnit) cu).setClassIDs(classified.get(cu));
        //				System.out.println();
        //				System.out.println(cu.getContent());
        //				System.out.print("----->  CLASS: ");
        boolean[] ids = ((ZoneClassifyUnit) cu).getClassIDs();
        boolean b = false;
        for (int i = 0; i < ids.length; i++) {
          if (ids[i]) {
            if (b) {
              // System.out.print("& " + (i + 1));
            } else {
              // System.out.println((i + 1));
            }
            b = true;
          }
        }
        results.add(cu);
      }
      DbConnector.insertClassifiedParagraphsinDB(corrConnection, results, jahrgang, zeilenNr, true);
      DbConnector.insertClassifiedParagraphsinDB(
          origConnection, results, jahrgang, zeilenNr, false);
      // progressbar
      done++;
      // ProgressBar.updateProgress((float) done/queryLimit);

      // time needed
      if (done % fetchSize == 0) {
        long end = System.currentTimeMillis();
        long time = (end - start) / 1000;

        // continue?
        if (askAgain) {

          System.out.println(
              "\n\n"
                  + "continue (c),\n"
                  + "don't interrupt again (d),\n"
                  + "or stop (s) classifying?");

          boolean answered = false;
          while (!answered) {
            System.out.println("\nnumber of classifed jobAds: " + jobAdCount);
            System.out.println("number of classified paragraphs: " + paraCount + "\n");
            BufferedReader in = new BufferedReader(new InputStreamReader(System.in));
            String answer = in.readLine();

            if (answer.toLowerCase().trim().equals("c")) {
              goOn = true;
              answered = true;
              System.out.println("\n...classifying...\n");
            } else if (answer.toLowerCase().trim().equals("d")) {
              goOn = true;
              askAgain = false;
              answered = true;
              System.out.println("\n...classifying...\n");
            } else if (answer.toLowerCase().trim().equals("s")) {
              goOn = false;
              answered = true;
            } else {
              System.out.println("C: invalid answer! please try again...");
              System.out.println();
            }
          }
        }
        start = System.currentTimeMillis();
      }
    }
    System.out.println("\nnumber of classifed jobAds: " + jobAdCount);
    System.out.println("number of classified paragraphs: " + paraCount + "\n");
    System.out.println("Classifying was fun! GoodBye!");
  }
Пример #30
-1
  private void prepareStatement() throws SQLException {

    // Get the SQL connection
    SqlConnector sql_connector = mDataset.getSqlConnector();
    if (sql_connector != null) {
      try {
        Connection connection = sql_connector.getConnection();
        // Check statements are still valid
        if (mStatQuery == null || mStatQuery.isClosed()) {
          // Create the query statement
          mStatQuery = connection.prepareStatement(mQuery);
          mStatQuery.setFetchSize(1000);
        }
        if (mStatCount == null || mStatCount.isClosed()) {
          // Create the count statement
          mStatCount = connection.prepareStatement("SELECT COUNT(*) FROM (" + mQuery + ")");
          mStatCount.setFetchSize(1000);
        }

      } catch (IOException e) {
        mNbRows = -1;
        Factory.getLogger().log(Level.SEVERE, e.getMessage(), e);
        close();
      }
    }

    mInitialized = true;
  }