public String getPKIndex(OracleTriggerMonitoredSourceInfo sourceInfo) {
String index = _pKeyIndexMap.get(sourceInfo.getEventView());
if (null == index) {
index = sourceInfo.getEventView() + "_pk";
}
return index;
}
@Override
public ReadEventCycleSummary readEventsFromAllSources(long sinceSCN)
throws DatabusException, EventCreationException, UnsupportedKeyException {
List<EventReaderSummary> summaries = new ArrayList<EventReaderSummary>();
long maxScn = EventReaderSummary.NO_EVENTS_SCN;
long endScn = maxScn;
boolean error = false;
long startTS = System.currentTimeMillis();
try {
_rate.start();
_rate.suspend();
Connection conn = null;
try {
conn = _dataSource.getConnection();
LOG.info("Oracle JDBC Version :" + conn.getMetaData().getDriverVersion());
} finally {
DBHelper.close(conn);
}
if (!_sources.isEmpty()) {
// Script assumes seeding is done for one schema at a time
// just use one source to get the schema name for sy$txlog
maxScn = getMaxScn(_sources.get(0));
}
for (OracleTriggerMonitoredSourceInfo sourceInfo : _sources) {
LOG.info("Bootstrapping " + sourceInfo.getEventView());
_bootstrapSeedWriter.start(maxScn);
EventReaderSummary summary = readEventsForSource(sourceInfo, maxScn);
// Script assumes seeding is done for one schema at a time
// just use one source to get the schema name for sy$txlog
endScn = getMaxScn(_sources.get(0));
_bootstrapSeedWriter.endEvents(BootstrapEventBuffer.END_OF_SOURCE, endScn, null);
summaries.add(summary);
}
} catch (Exception ex) {
error = true;
throw new DatabusException(ex);
} finally {
// Notify writer that I am done
if (error) {
_bootstrapSeedWriter.endEvents(BootstrapEventBuffer.ERROR_CODE, endScn, null);
LOG.error("Seeder stopping unexpectedly !!");
} else {
_bootstrapSeedWriter.endEvents(BootstrapEventBuffer.END_OF_FILE, endScn, null);
LOG.info("Completed Seeding !!");
}
LOG.info("Start SCN :" + maxScn);
LOG.info("End SCN :" + endScn);
}
long endTS = System.currentTimeMillis();
ReadEventCycleSummary cycleSummary =
new ReadEventCycleSummary("seeder", summaries, maxScn, (endTS - startTS));
return cycleSummary;
}
public void validate() throws Exception {
if (null == _pKeyTypeMap) throw new Exception("_pKeyTypeMap cannot be null !!");
if (null == _pKeyNameMap) throw new Exception("_pKeyNameMap cannot be null !!");
boolean isNullIndex = (null == _pKeyIndexMap);
boolean isNullQueryHint = (null == _queryHintMap);
if (isNullIndex && isNullQueryHint)
throw new Exception("Index and Query Hint both cannot be null !!");
for (OracleTriggerMonitoredSourceInfo s : _sources) {
if (null == _pKeyTypeMap.get(s.getEventView()))
throw new Exception("pKey Type for Source (" + s.getEventView() + ") not provided !!");
if (null == _pKeyNameMap.get(s.getEventView()))
throw new Exception("pKey Name for Source (" + s.getEventView() + ") not provided !!");
if ((isNullIndex || (null == _pKeyIndexMap.get(s.getEventView())))
&& (isNullQueryHint || (null == _queryHintMap.get(s.getEventView())))) {
throw new Exception(
"Both pkey Index and Query Hint for source (" + s.getEventView() + ") not provided !!");
}
}
}
public String getQueryHint(OracleTriggerMonitoredSourceInfo sourceInfo) {
String index = _queryHintMap.get(sourceInfo.getEventView());
if (null == index) {
return null;
}
return index;
}
private long getMaxScn(OracleTriggerMonitoredSourceInfo sourceInfo) throws SQLException {
String schema = (sourceInfo.getEventSchema() == null) ? "" : sourceInfo.getEventSchema() + ".";
String table = schema + "sy$txlog";
/*
StringBuilder sql = new StringBuilder();
sql.append("select max(scn) from ");
sql.append(table);
String query = sql.toString();
*/
String query =
"select "
+ "max("
+ schema
+ "sync_core.getScn(scn,ora_rowscn)) "
+ "from "
+ table
+ " where "
+ "scn >= (select max(scn) from "
+ table
+ ")";
long maxScn = executeAndGetLong(query);
return maxScn;
}
public static String getTableName(OracleTriggerMonitoredSourceInfo sourceInfo) {
String schema = (sourceInfo.getEventSchema() == null) ? "" : sourceInfo.getEventSchema() + ".";
String table = schema + "sy$" + sourceInfo.getEventView();
return table;
}
private EventReaderSummary readEventsForSource(
OracleTriggerMonitoredSourceInfo sourceInfo, long maxScn)
throws DatabusException, EventCreationException, UnsupportedKeyException, SQLException,
IOException {
int retryMax = _numRetries;
int numRetry = 0;
Connection conn = null;
PreparedStatement pstmt = null;
ResultSet rs = null;
KeyType keyType = _pKeyTypeMap.get(sourceInfo.getEventView());
String keyName = _pKeyNameMap.get(sourceInfo.getEventView());
String sql = _eventQueryMap.get(sourceInfo.getEventView());
String beginSrcKey = _beginSrcKeyMap.get(sourceInfo.getEventView());
String endSrcKey = _endSrcKeyMap.get(sourceInfo.getEventView());
if (sql == null) {
sql =
generateEventQuery2(
sourceInfo, keyName, keyType, getPKIndex(sourceInfo), getQueryHint(sourceInfo));
}
LOG.info("Chunked Query for Source (" + sourceInfo + ") is :" + sql);
LOG.info("EndSrcKey for source (" + sourceInfo + ") is :" + endSrcKey);
PrimaryKeyTxn endKeyTxn = null;
if ((null != endSrcKey) && (!endSrcKey.trim().isEmpty())) {
if (KeyType.LONG == keyType) endKeyTxn = new PrimaryKeyTxn(new Long(endSrcKey));
else endKeyTxn = new PrimaryKeyTxn(endSrcKey);
}
long timestamp = System.currentTimeMillis();
int numRowsFetched = 0;
long totalEventSize = 0;
long timeStart = System.currentTimeMillis();
long checkpointInterval = _commitInterval;
boolean done = false;
long lastTime = timeStart;
long numRows = 0;
PrimaryKeyTxn pKey = null;
String minKeySQL = generateMinKeyQuery(sourceInfo, keyName);
String srcName = sourceInfo.getEventView();
LOG.info("Bootstrapping for Source :" + srcName);
String lastKey = _lastKeys.get(sourceInfo.getEventView());
File f = _keyTxnFilesMap.get(srcName);
FileWriter oStream = new FileWriter(f, f.exists());
BufferedWriter keyTxnWriter = new BufferedWriter(oStream, _keyTxnBufferSizeMap.get(srcName));
_bootstrapSeedWriter.startEvents();
RateMonitor seedingRate = new RateMonitor("Seeding Rate");
RateMonitor queryRate = new RateMonitor("Query Rate");
seedingRate.start();
seedingRate.suspend();
queryRate.start();
queryRate.suspend();
boolean isException = false;
long totProcessTime = 0;
try {
conn = _dataSource.getConnection();
pstmt = conn.prepareStatement(sql);
if (_enableNumRowsQuery) numRows = getNumRows(conn, getTableName(sourceInfo));
else numRows = -1;
long currRowId = _lastRows.get(sourceInfo.getEventView());
/**
* First Key to be seeded will be decided in the following order: 1. Use
* bootstrap_seeder_state's last srcKey as the key for the first chunk. 2. If (1) is empty,
* use passed-in begin srcKey. 3. If (2) is also empty, use Oracle's minKey as the first Chunk
* Key.
*/
if (null == lastKey) {
lastKey = _beginSrcKeyMap.get(sourceInfo.getEventView());
LOG.info(
"No last Src Key available in bootstrap_seeder_state for source ("
+ sourceInfo
+ ". Trying beginSrc Key from config :"
+ lastKey);
}
if ((null == lastKey) || (lastKey.trim().isEmpty())) {
if (KeyType.LONG == keyType) pKey = new PrimaryKeyTxn(executeAndGetLong(minKeySQL));
else pKey = new PrimaryKeyTxn(executeAndGetString(minKeySQL));
} else {
if (KeyType.LONG == keyType) pKey = new PrimaryKeyTxn(Long.parseLong(lastKey));
else pKey = new PrimaryKeyTxn(lastKey);
}
PrimaryKeyTxn lastRoundKeyTxn = new PrimaryKeyTxn(pKey);
PrimaryKeyTxn lastKeyTxn = new PrimaryKeyTxn(pKey);
long numUniqueKeysThisRound = 0;
boolean first = true;
_rate.resume();
while (!done) {
LOG.info("MinKey being used for this round:" + pKey);
numUniqueKeysThisRound = 0;
try {
lastRoundKeyTxn.copyFrom(pKey);
if (KeyType.LONG == keyType) {
pstmt.setLong(1, pKey.getKey());
} else {
String key = pKey.getKeyStr();
pstmt.setString(1, key);
}
pstmt.setLong(2, _numRowsPerQuery);
pstmt.setFetchSize(_numRowsPrefetch);
if (_oraclePreparedStatementClass.isInstance(pstmt)) {
try {
_setLobPrefetchSizeMethod.invoke(pstmt, _LOBPrefetchSize);
} catch (Exception e) {
throw new EventCreationException("Unable to set Lob Prefetch size" + e.getMessage());
}
}
LOG.info(
"Executing Oracle Query :"
+ sql
+ ". Key: "
+ pKey
+ ",NumRows: "
+ _numRowsPerQuery);
queryRate.resume();
rs = pstmt.executeQuery();
queryRate.suspend();
LOG.info("Total Query Latency :" + queryRate.getDuration() / 1000000000L);
long totLatency = 0;
long txnId = 0;
int numRowsThisRound = 0;
seedingRate.resume();
while (rs.next()) {
_rate.tick();
seedingRate.tick();
currRowId++;
txnId = rs.getLong(2);
if (KeyType.LONG == keyType) {
pKey.setKeyTxn(rs.getLong(1), txnId);
} else {
String key = rs.getString(1);
pKey.setKeyStrTxn(key, txnId);
}
// Write TXN to file
pKey.writeTo(keyTxnWriter);
// LOG.info("TXNId is :" + txnId + ",RowId is :" + currRowId);
long start = System.nanoTime();
long eventSize =
sourceInfo
.getFactory()
.createAndAppendEvent(maxScn, timestamp, rs, _bootstrapSeedWriter, false, null);
long latency = System.nanoTime() - start;
totLatency += latency;
totalEventSize += eventSize;
totProcessTime += (totLatency / 1000 * 1000);
numRowsFetched++;
numRowsThisRound++;
if (lastKeyTxn.compareKey(pKey) != 0) {
numUniqueKeysThisRound++;
lastKeyTxn.copyFrom(pKey);
}
if (numRowsFetched % checkpointInterval == 0) {
// Commit this batch and reinit
_bootstrapSeedWriter.endEvents(currRowId, timestamp, null);
keyTxnWriter.flush();
_bootstrapSeedWriter.startEvents();
long procTime = totLatency / 1000000000;
long currTime = System.currentTimeMillis();
long diff = (currTime - lastTime) / 1000;
long timeSinceStart = (currTime - timeStart) / 1000;
double currRate = _rate.getRate();
currRate = (currRate <= 0) ? 1 : currRate;
if (_enableNumRowsQuery) {
double remTime = (numRows - currRowId) / (currRate);
LOG.info(
"Processed "
+ checkpointInterval
+ " rows in "
+ diff
+ " seconds, Processing Time (seconds) so far :"
+ (procTime)
+ ",Seconds elapsed since start :"
+ (timeSinceStart)
+ ",Approx Seconds remaining :"
+ remTime
+ ",Overall Row Rate:"
+ _rate.getRate()
+ "("
+ seedingRate.getRate()
+ ")"
+ ",NumRows Fetched so far:"
+ numRowsFetched
+ ". TotalEventSize :"
+ totalEventSize);
} else {
LOG.info(
"Processed "
+ checkpointInterval
+ " rows in "
+ diff
+ " seconds, Processing Time (seconds) so far :"
+ (procTime)
+ ",Seconds elapsed since start :"
+ (timeSinceStart)
+ ",Overall Row Rate:"
+ _rate.getRate()
+ "("
+ seedingRate.getRate()
+ ")"
+ ",NumRows Fetched so far:"
+ numRowsFetched
+ ". TotalEventSize :"
+ totalEventSize);
}
lastTime = currTime;
}
if ((null != endKeyTxn) && (endKeyTxn.compareKey(lastKeyTxn) < 0)) {
LOG.info(
"Seeding to be stopped for current source as it has completed seeding upto endSrckey :"
+ endKeyTxn
+ ", Current SrcKey :"
+ lastKeyTxn);
break;
}
}
seedingRate.suspend();
if ((numRowsThisRound <= 1)
|| ((numRowsThisRound < _numRowsPerQuery) && (numUniqueKeysThisRound <= 1))) {
LOG.info(
"Seeding Done for source :"
+ sourceInfo.getEventView()
+ ", numRowsThisRound :"
+ numRowsThisRound
+ ", _numRowsPerQuery :"
+ _numRowsPerQuery
+ ", numUniqueKeys :"
+ numUniqueKeysThisRound);
done = true;
} else if ((numRowsThisRound == _numRowsPerQuery) && (numUniqueKeysThisRound <= 1)) {
String msg =
"Seeding stuck at infinte loop for source : "
+ sourceInfo.getEventView()
+ ", numRowsThisRound :"
+ numRowsThisRound
+ ", _numRowsPerQuery :"
+ _numRowsPerQuery
+ ", numUniqueKeys :"
+ numUniqueKeysThisRound
+ ", lastChunkKey :"
+ lastRoundKeyTxn;
LOG.error(msg);
throw new DatabusException(msg);
} else if (null != endKeyTxn) {
if (endKeyTxn.compareKey(lastKeyTxn) < 0) {
LOG.info(
"Seeding stopped for source :"
+ sourceInfo.getEventView()
+ ", as it has completed seeding upto the endSrckey :"
+ endKeyTxn
+ ", numRowsThisRound :"
+ numRowsThisRound
+ ", _numRowsPerQuery :"
+ _numRowsPerQuery
+ ", numUniqueKeys :"
+ numUniqueKeysThisRound
+ " , Current SrcKey :"
+ lastKeyTxn);
done = true;
}
}
if (currRowId > 0 && (!first || done)) {
currRowId--; // Since next time, we will read the last seen record again
}
LOG.info("about to call end events with currRowId = " + currRowId);
first = false;
_bootstrapSeedWriter.endEvents(currRowId, timestamp, null);
isException = false;
} catch (SQLException ex) {
LOG.error("Got SQLException for source (" + sourceInfo + ")", ex);
_bootstrapSeedWriter.rollbackEvents();
numRetry++;
isException = true;
if (numRetry >= retryMax) {
throw new DatabusException(
"Error: Reached max retries for reading/processing bootstrap", ex);
}
} finally {
DBHelper.close(rs);
rs = null;
}
}
} catch (DatabusException ex) {
isException = true;
throw ex;
} finally {
DBHelper.close(rs, pstmt, conn);
keyTxnWriter.close();
rs = null;
_rate.suspend();
if (!isException) {
dedupeKeyTxnFile(_keyTxnFilesMap.get(srcName), keyType);
}
}
long timeEnd = System.currentTimeMillis();
long elapsedMin = (timeEnd - timeStart) / (MILLISEC_TO_MIN);
LOG.info(
"Processed "
+ numRowsFetched
+ " rows of Source: "
+ sourceInfo.getSourceName()
+ " in "
+ elapsedMin
+ " minutes");
return new EventReaderSummary(
sourceInfo.getSourceId(),
sourceInfo.getSourceName(),
-1,
numRowsFetched,
totalEventSize,
(timeEnd - timeStart),
totProcessTime,
0,
0,
0);
}