コード例 #1
0
  public String getPKIndex(OracleTriggerMonitoredSourceInfo sourceInfo) {
    String index = _pKeyIndexMap.get(sourceInfo.getEventView());

    if (null == index) {
      index = sourceInfo.getEventView() + "_pk";
    }
    return index;
  }
コード例 #2
0
  @Override
  public ReadEventCycleSummary readEventsFromAllSources(long sinceSCN)
      throws DatabusException, EventCreationException, UnsupportedKeyException {
    List<EventReaderSummary> summaries = new ArrayList<EventReaderSummary>();
    long maxScn = EventReaderSummary.NO_EVENTS_SCN;
    long endScn = maxScn;
    boolean error = false;

    long startTS = System.currentTimeMillis();
    try {
      _rate.start();
      _rate.suspend();

      Connection conn = null;
      try {
        conn = _dataSource.getConnection();
        LOG.info("Oracle JDBC Version :" + conn.getMetaData().getDriverVersion());
      } finally {
        DBHelper.close(conn);
      }

      if (!_sources.isEmpty()) {
        // Script assumes seeding is done for one schema at a time
        // just use one source to get the schema name for sy$txlog
        maxScn = getMaxScn(_sources.get(0));
      }

      for (OracleTriggerMonitoredSourceInfo sourceInfo : _sources) {
        LOG.info("Bootstrapping " + sourceInfo.getEventView());
        _bootstrapSeedWriter.start(maxScn);
        EventReaderSummary summary = readEventsForSource(sourceInfo, maxScn);
        // Script assumes seeding is done for one schema at a time
        // just use one source to get the schema name for sy$txlog
        endScn = getMaxScn(_sources.get(0));
        _bootstrapSeedWriter.endEvents(BootstrapEventBuffer.END_OF_SOURCE, endScn, null);
        summaries.add(summary);
      }
    } catch (Exception ex) {
      error = true;
      throw new DatabusException(ex);
    } finally {
      // Notify writer that I am done
      if (error) {
        _bootstrapSeedWriter.endEvents(BootstrapEventBuffer.ERROR_CODE, endScn, null);
        LOG.error("Seeder stopping unexpectedly !!");
      } else {
        _bootstrapSeedWriter.endEvents(BootstrapEventBuffer.END_OF_FILE, endScn, null);
        LOG.info("Completed Seeding !!");
      }
      LOG.info("Start SCN :" + maxScn);
      LOG.info("End SCN :" + endScn);
    }
    long endTS = System.currentTimeMillis();

    ReadEventCycleSummary cycleSummary =
        new ReadEventCycleSummary("seeder", summaries, maxScn, (endTS - startTS));
    return cycleSummary;
  }
コード例 #3
0
  public void validate() throws Exception {

    if (null == _pKeyTypeMap) throw new Exception("_pKeyTypeMap cannot be null !!");

    if (null == _pKeyNameMap) throw new Exception("_pKeyNameMap cannot be null !!");

    boolean isNullIndex = (null == _pKeyIndexMap);
    boolean isNullQueryHint = (null == _queryHintMap);

    if (isNullIndex && isNullQueryHint)
      throw new Exception("Index and Query Hint both cannot be null !!");

    for (OracleTriggerMonitoredSourceInfo s : _sources) {
      if (null == _pKeyTypeMap.get(s.getEventView()))
        throw new Exception("pKey Type for Source (" + s.getEventView() + ") not provided !!");

      if (null == _pKeyNameMap.get(s.getEventView()))
        throw new Exception("pKey Name for Source (" + s.getEventView() + ") not provided !!");

      if ((isNullIndex || (null == _pKeyIndexMap.get(s.getEventView())))
          && (isNullQueryHint || (null == _queryHintMap.get(s.getEventView())))) {
        throw new Exception(
            "Both pkey Index and Query Hint for source (" + s.getEventView() + ") not provided !!");
      }
    }
  }
コード例 #4
0
  public String getQueryHint(OracleTriggerMonitoredSourceInfo sourceInfo) {
    String index = _queryHintMap.get(sourceInfo.getEventView());

    if (null == index) {
      return null;
    }

    return index;
  }
コード例 #5
0
 private long getMaxScn(OracleTriggerMonitoredSourceInfo sourceInfo) throws SQLException {
   String schema = (sourceInfo.getEventSchema() == null) ? "" : sourceInfo.getEventSchema() + ".";
   String table = schema + "sy$txlog";
   /*
   StringBuilder sql = new StringBuilder();
   sql.append("select max(scn) from ");
   sql.append(table);
   String query = sql.toString();
   */
   String query =
       "select "
           + "max("
           + schema
           + "sync_core.getScn(scn,ora_rowscn)) "
           + "from "
           + table
           + " where "
           + "scn >= (select max(scn) from "
           + table
           + ")";
   long maxScn = executeAndGetLong(query);
   return maxScn;
 }
コード例 #6
0
 public static String getTableName(OracleTriggerMonitoredSourceInfo sourceInfo) {
   String schema = (sourceInfo.getEventSchema() == null) ? "" : sourceInfo.getEventSchema() + ".";
   String table = schema + "sy$" + sourceInfo.getEventView();
   return table;
 }
コード例 #7
0
  private EventReaderSummary readEventsForSource(
      OracleTriggerMonitoredSourceInfo sourceInfo, long maxScn)
      throws DatabusException, EventCreationException, UnsupportedKeyException, SQLException,
          IOException {
    int retryMax = _numRetries;
    int numRetry = 0;
    Connection conn = null;
    PreparedStatement pstmt = null;
    ResultSet rs = null;
    KeyType keyType = _pKeyTypeMap.get(sourceInfo.getEventView());
    String keyName = _pKeyNameMap.get(sourceInfo.getEventView());
    String sql = _eventQueryMap.get(sourceInfo.getEventView());
    String beginSrcKey = _beginSrcKeyMap.get(sourceInfo.getEventView());
    String endSrcKey = _endSrcKeyMap.get(sourceInfo.getEventView());

    if (sql == null) {
      sql =
          generateEventQuery2(
              sourceInfo, keyName, keyType, getPKIndex(sourceInfo), getQueryHint(sourceInfo));
    }
    LOG.info("Chunked  Query for Source (" + sourceInfo + ") is :" + sql);
    LOG.info("EndSrcKey for source (" + sourceInfo + ") is :" + endSrcKey);

    PrimaryKeyTxn endKeyTxn = null;
    if ((null != endSrcKey) && (!endSrcKey.trim().isEmpty())) {
      if (KeyType.LONG == keyType) endKeyTxn = new PrimaryKeyTxn(new Long(endSrcKey));
      else endKeyTxn = new PrimaryKeyTxn(endSrcKey);
    }

    long timestamp = System.currentTimeMillis();
    int numRowsFetched = 0;
    long totalEventSize = 0;
    long timeStart = System.currentTimeMillis();
    long checkpointInterval = _commitInterval;
    boolean done = false;
    long lastTime = timeStart;
    long numRows = 0;
    PrimaryKeyTxn pKey = null;

    String minKeySQL = generateMinKeyQuery(sourceInfo, keyName);

    String srcName = sourceInfo.getEventView();
    LOG.info("Bootstrapping for Source :" + srcName);
    String lastKey = _lastKeys.get(sourceInfo.getEventView());
    File f = _keyTxnFilesMap.get(srcName);
    FileWriter oStream = new FileWriter(f, f.exists());
    BufferedWriter keyTxnWriter = new BufferedWriter(oStream, _keyTxnBufferSizeMap.get(srcName));

    _bootstrapSeedWriter.startEvents();
    RateMonitor seedingRate = new RateMonitor("Seeding Rate");
    RateMonitor queryRate = new RateMonitor("Query Rate");
    seedingRate.start();
    seedingRate.suspend();
    queryRate.start();
    queryRate.suspend();
    boolean isException = false;
    long totProcessTime = 0;
    try {
      conn = _dataSource.getConnection();
      pstmt = conn.prepareStatement(sql);

      if (_enableNumRowsQuery) numRows = getNumRows(conn, getTableName(sourceInfo));
      else numRows = -1;

      long currRowId = _lastRows.get(sourceInfo.getEventView());

      /**
       * First Key to be seeded will be decided in the following order: 1. Use
       * bootstrap_seeder_state's last srcKey as the key for the first chunk. 2. If (1) is empty,
       * use passed-in begin srcKey. 3. If (2) is also empty, use Oracle's minKey as the first Chunk
       * Key.
       */
      if (null == lastKey) {
        lastKey = _beginSrcKeyMap.get(sourceInfo.getEventView());
        LOG.info(
            "No last Src Key available in bootstrap_seeder_state for source ("
                + sourceInfo
                + ". Trying beginSrc Key from config :"
                + lastKey);
      }

      if ((null == lastKey) || (lastKey.trim().isEmpty())) {
        if (KeyType.LONG == keyType) pKey = new PrimaryKeyTxn(executeAndGetLong(minKeySQL));
        else pKey = new PrimaryKeyTxn(executeAndGetString(minKeySQL));
      } else {
        if (KeyType.LONG == keyType) pKey = new PrimaryKeyTxn(Long.parseLong(lastKey));
        else pKey = new PrimaryKeyTxn(lastKey);
      }

      PrimaryKeyTxn lastRoundKeyTxn = new PrimaryKeyTxn(pKey);
      PrimaryKeyTxn lastKeyTxn = new PrimaryKeyTxn(pKey);
      long numUniqueKeysThisRound = 0;

      boolean first = true;
      _rate.resume();
      while (!done) {
        LOG.info("MinKey being used for this round:" + pKey);
        numUniqueKeysThisRound = 0;
        try {
          lastRoundKeyTxn.copyFrom(pKey);

          if (KeyType.LONG == keyType) {
            pstmt.setLong(1, pKey.getKey());
          } else {
            String key = pKey.getKeyStr();
            pstmt.setString(1, key);
          }

          pstmt.setLong(2, _numRowsPerQuery);
          pstmt.setFetchSize(_numRowsPrefetch);

          if (_oraclePreparedStatementClass.isInstance(pstmt)) {
            try {
              _setLobPrefetchSizeMethod.invoke(pstmt, _LOBPrefetchSize);
            } catch (Exception e) {
              throw new EventCreationException("Unable to set Lob Prefetch size" + e.getMessage());
            }
          }

          LOG.info(
              "Executing Oracle Query :"
                  + sql
                  + ". Key: "
                  + pKey
                  + ",NumRows: "
                  + _numRowsPerQuery);
          queryRate.resume();
          rs = pstmt.executeQuery();
          queryRate.suspend();

          LOG.info("Total Query Latency :" + queryRate.getDuration() / 1000000000L);
          long totLatency = 0;
          long txnId = 0;
          int numRowsThisRound = 0;
          seedingRate.resume();
          while (rs.next()) {
            _rate.tick();
            seedingRate.tick();
            currRowId++;
            txnId = rs.getLong(2);
            if (KeyType.LONG == keyType) {
              pKey.setKeyTxn(rs.getLong(1), txnId);
            } else {
              String key = rs.getString(1);
              pKey.setKeyStrTxn(key, txnId);
            }

            // Write TXN to file
            pKey.writeTo(keyTxnWriter);

            // LOG.info("TXNId is :" + txnId + ",RowId is :" + currRowId);
            long start = System.nanoTime();
            long eventSize =
                sourceInfo
                    .getFactory()
                    .createAndAppendEvent(maxScn, timestamp, rs, _bootstrapSeedWriter, false, null);
            long latency = System.nanoTime() - start;
            totLatency += latency;
            totalEventSize += eventSize;
            totProcessTime += (totLatency / 1000 * 1000);
            numRowsFetched++;
            numRowsThisRound++;

            if (lastKeyTxn.compareKey(pKey) != 0) {
              numUniqueKeysThisRound++;
              lastKeyTxn.copyFrom(pKey);
            }

            if (numRowsFetched % checkpointInterval == 0) {
              // Commit this batch and reinit
              _bootstrapSeedWriter.endEvents(currRowId, timestamp, null);
              keyTxnWriter.flush();
              _bootstrapSeedWriter.startEvents();
              long procTime = totLatency / 1000000000;
              long currTime = System.currentTimeMillis();
              long diff = (currTime - lastTime) / 1000;
              long timeSinceStart = (currTime - timeStart) / 1000;
              double currRate = _rate.getRate();
              currRate = (currRate <= 0) ? 1 : currRate;

              if (_enableNumRowsQuery) {
                double remTime = (numRows - currRowId) / (currRate);

                LOG.info(
                    "Processed "
                        + checkpointInterval
                        + " rows in "
                        + diff
                        + " seconds, Processing Time (seconds) so far :"
                        + (procTime)
                        + ",Seconds elapsed since start :"
                        + (timeSinceStart)
                        + ",Approx Seconds remaining :"
                        + remTime
                        + ",Overall Row Rate:"
                        + _rate.getRate()
                        + "("
                        + seedingRate.getRate()
                        + ")"
                        + ",NumRows Fetched so far:"
                        + numRowsFetched
                        + ". TotalEventSize :"
                        + totalEventSize);
              } else {
                LOG.info(
                    "Processed "
                        + checkpointInterval
                        + " rows in "
                        + diff
                        + " seconds, Processing Time (seconds) so far :"
                        + (procTime)
                        + ",Seconds elapsed since start :"
                        + (timeSinceStart)
                        + ",Overall Row Rate:"
                        + _rate.getRate()
                        + "("
                        + seedingRate.getRate()
                        + ")"
                        + ",NumRows Fetched so far:"
                        + numRowsFetched
                        + ". TotalEventSize :"
                        + totalEventSize);
              }
              lastTime = currTime;
            }

            if ((null != endKeyTxn) && (endKeyTxn.compareKey(lastKeyTxn) < 0)) {
              LOG.info(
                  "Seeding to be stopped for current source as it has completed seeding upto endSrckey :"
                      + endKeyTxn
                      + ", Current SrcKey :"
                      + lastKeyTxn);
              break;
            }
          }
          seedingRate.suspend();

          if ((numRowsThisRound <= 1)
              || ((numRowsThisRound < _numRowsPerQuery) && (numUniqueKeysThisRound <= 1))) {
            LOG.info(
                "Seeding Done for source :"
                    + sourceInfo.getEventView()
                    + ", numRowsThisRound :"
                    + numRowsThisRound
                    + ", _numRowsPerQuery :"
                    + _numRowsPerQuery
                    + ", numUniqueKeys :"
                    + numUniqueKeysThisRound);
            done = true;
          } else if ((numRowsThisRound == _numRowsPerQuery) && (numUniqueKeysThisRound <= 1)) {
            String msg =
                "Seeding stuck at infinte loop for source : "
                    + sourceInfo.getEventView()
                    + ", numRowsThisRound :"
                    + numRowsThisRound
                    + ", _numRowsPerQuery :"
                    + _numRowsPerQuery
                    + ", numUniqueKeys :"
                    + numUniqueKeysThisRound
                    + ", lastChunkKey :"
                    + lastRoundKeyTxn;
            LOG.error(msg);
            throw new DatabusException(msg);
          } else if (null != endKeyTxn) {
            if (endKeyTxn.compareKey(lastKeyTxn) < 0) {
              LOG.info(
                  "Seeding stopped for source :"
                      + sourceInfo.getEventView()
                      + ", as it has completed seeding upto the endSrckey :"
                      + endKeyTxn
                      + ", numRowsThisRound :"
                      + numRowsThisRound
                      + ", _numRowsPerQuery :"
                      + _numRowsPerQuery
                      + ", numUniqueKeys :"
                      + numUniqueKeysThisRound
                      + " , Current SrcKey :"
                      + lastKeyTxn);
              done = true;
            }
          }

          if (currRowId > 0 && (!first || done)) {
            currRowId--; // Since next time, we will read the last seen record again
          }
          LOG.info("about to call end events with currRowId = " + currRowId);
          first = false;
          _bootstrapSeedWriter.endEvents(currRowId, timestamp, null);
          isException = false;
        } catch (SQLException ex) {
          LOG.error("Got SQLException for source (" + sourceInfo + ")", ex);

          _bootstrapSeedWriter.rollbackEvents();

          numRetry++;
          isException = true;

          if (numRetry >= retryMax) {
            throw new DatabusException(
                "Error: Reached max retries for reading/processing bootstrap", ex);
          }
        } finally {
          DBHelper.close(rs);
          rs = null;
        }
      }
    } catch (DatabusException ex) {

      isException = true;
      throw ex;

    } finally {

      DBHelper.close(rs, pstmt, conn);
      keyTxnWriter.close();
      rs = null;
      _rate.suspend();

      if (!isException) {
        dedupeKeyTxnFile(_keyTxnFilesMap.get(srcName), keyType);
      }
    }
    long timeEnd = System.currentTimeMillis();
    long elapsedMin = (timeEnd - timeStart) / (MILLISEC_TO_MIN);
    LOG.info(
        "Processed "
            + numRowsFetched
            + " rows of Source: "
            + sourceInfo.getSourceName()
            + " in "
            + elapsedMin
            + " minutes");
    return new EventReaderSummary(
        sourceInfo.getSourceId(),
        sourceInfo.getSourceName(),
        -1,
        numRowsFetched,
        totalEventSize,
        (timeEnd - timeStart),
        totProcessTime,
        0,
        0,
        0);
  }