/*
* Test test that event ID calls work as expected
*/
public void testExtractorEventID() throws Exception {
RawExtractor extractor = (RawExtractor) PluginLoader.load(DummyExtractor.class.getName());
extractor.configure(null);
extractor.prepare(null);
DBMSEvent event = extractor.extract();
String currentEventId = extractor.getCurrentResourceEventId();
Assert.assertEquals(event.getEventId(), currentEventId);
event = extractor.extract();
Assert.assertTrue(event.getEventId().compareTo(currentEventId) > 0);
currentEventId = extractor.getCurrentResourceEventId();
Assert.assertTrue(event.getEventId().compareTo(currentEventId) == 0);
extractor.release(null);
}
/*
* Test that dummy extractor works like expected,
*/
public void testExtractorBasic() throws Exception {
RawExtractor extractor = (RawExtractor) PluginLoader.load(DummyExtractor.class.getName());
extractor.configure(null);
extractor.prepare(null);
DBMSEvent event = extractor.extract();
Assert.assertEquals(event.getEventId(), "0");
event = extractor.extract();
Assert.assertEquals(event.getEventId(), "1");
extractor.setLastEventId("0");
event = extractor.extract();
Assert.assertEquals(event.getEventId(), "1");
extractor.setLastEventId(null);
event = extractor.extract();
Assert.assertEquals(event.getEventId(), "0");
for (Integer i = 1; i < 5; ++i) {
event = extractor.extract();
Assert.assertEquals(event.getEventId(), i.toString());
}
event = extractor.extract("0");
Assert.assertEquals(event.getEventId(), "0");
event = extractor.extract("4");
Assert.assertEquals(event.getEventId(), "4");
event = extractor.extract("5");
Assert.assertEquals(event, null);
extractor.release(null);
}
/**
* Post all committed changes to given queue
*
* @param q Queue to post
* @param minSCN Minimal SCN among all open transactions
* @param skipSeq If only part of the transaction should be posted, this is the last seq to skip
* @param lastObsoletePlogSeq Last obsolete plog sequence (min() over all open transactions - 1)
* @return lastProcessedEventId
*/
public String pushContentsToQueue(
BlockingQueue<DBMSEvent> q, long minSCN, int transactionFragSize, long lastObsoletePlogSeq)
throws UnsupportedEncodingException, ReplicatorException, SerialException,
InterruptedException, SQLException {
if (logger.isDebugEnabled()) {
logger.debug("pushContentsToQueue: " + this.XID + " (DML:" + this.transactionIsDML + ")");
}
if (transactionIsDML) {
int count = 0;
PlogLCR lastLCR = null;
String lastProcessedEventId = null;
ArrayList<DBMSData> data = new ArrayList<DBMSData>();
int fragSize = 0;
LargeObjectScanner<PlogLCR> scanner = null;
try {
scanner = LCRList.scanner();
while (scanner.hasNext()) {
PlogLCR LCR = scanner.next();
if (LCR.type == PlogLCR.ETYPE_LCR_DATA) {
/* add only data */
if (LCR.subtype == PlogLCR.ESTYPE_LCR_DDL)
throw new ReplicatorException(
"Internal corruption: DDL statement in a DML transaction.");
if (transactionFragSize > 0 && fragSize >= transactionFragSize) {
logger.debug("Fragmenting");
// Time to fragment
DBMSEvent event = new DBMSEvent(lastLCR.eventId, data, false, commitTime);
// Set metadata for the transaction. Source is
// Oracle, we normalize time data to GMT, and
// strings are in UTF8.
event.setMetaDataOption(ReplOptionParams.DBMS_TYPE, Database.ORACLE);
event.setMetaDataOption(ReplOptionParams.TIME_ZONE_AWARE, "true");
event.setMetaDataOption(ReplOptionParams.STRINGS, "utf8");
q.put(event);
// Clear array for next fragment.
data = new ArrayList<DBMSData>();
fragSize = 0;
}
count++;
fragSize++;
if (count > skipSeq) {
LCR.eventId =
""
+ commitSCN
+ "#"
+ XID
+ "#"
+ count
+ "#"
+ minSCN
+ "#"
+ lastObsoletePlogSeq;
lastLCR = LCR;
data.add(convertLCRtoDBMSDataDML(LCR));
lastProcessedEventId = LCR.eventId;
{
PlogLCR r2 = LCR;
if (logger.isDebugEnabled()) {
logger.debug(
"LCR: "
+ r2.type
+ "."
+ r2.subtype
+ ":"
+ r2.typeAsString()
+ ", XID="
+ r2.XID
+ ", LCRid="
+ r2.LCRid
+ " eventId="
+ r2.eventId);
logger.debug("EventId#1 set to " + r2.eventId);
}
}
}
} else {
throw new RuntimeException("Type " + LCR.type + " in queue, should be only data");
}
}
} catch (IOException e) {
logger.error(
"Transaction scanner error occured: XID="
+ this.XID
+ " key="
+ LCRList.getKey()
+ " cache="
+ LCRList.getByteCache().toString());
throw new RuntimeException("Unable to read transaction from cache: " + this.XID);
} finally {
if (scanner != null) scanner.close();
}
if (lastLCR != null) {
// Last LCR set = transaction is not empty
// Mark last LCR as "LAST", so we know transaction is complete
lastLCR.eventId =
"" + commitSCN + "#" + XID + "#" + "LAST" + "#" + minSCN + "#" + lastObsoletePlogSeq;
if (logger.isDebugEnabled()) {
logger.debug("EventId#2 set to " + lastLCR.eventId);
}
DBMSEvent event = new DBMSEvent(lastLCR.eventId, data, true, commitTime);
// Set metadata for the transaction. Source is Oracle, we
// normalize time data to GMT, and strings are in UTF8.
event.setMetaDataOption(ReplOptionParams.DBMS_TYPE, Database.ORACLE);
event.setMetaDataOption(ReplOptionParams.TIME_ZONE_AWARE, "true");
event.setMetaDataOption(ReplOptionParams.STRINGS, "utf8");
q.put(event);
}
return lastProcessedEventId;
} else {
ArrayList<DBMSData> data = new ArrayList<DBMSData>();
PlogLCR lastLCR = null;
String lastProcessedEventId = null;
LargeObjectScanner<PlogLCR> scanner = null;
try {
scanner = LCRList.scanner();
while (scanner.hasNext()) {
PlogLCR LCR = scanner.next();
if (LCR.type == PlogLCR.ETYPE_LCR_DATA) {
/* add only data */
if (LCR.subtype != PlogLCR.ESTYPE_LCR_DDL) {
throw new ReplicatorException(
"Internal corruption: DML statement in a DDL transaction.");
}
LCR.eventId =
""
+ commitSCN
+ "#"
+ XID
+ "#"
+ "LAST"
+ "#"
+ minSCN
+ "#"
+ lastObsoletePlogSeq;
data.add(convertLCRtoDBMSDataDDL(LCR));
lastLCR = LCR;
lastProcessedEventId = LCR.eventId;
if (logger.isDebugEnabled()) {
logger.info("DDL: [" + LCR.currentSchema + "][" + LCR.eventId + "]: " + LCR.SQLText);
}
} else {
throw new RuntimeException("Type " + LCR.type + " in queue, should be only data");
}
}
} catch (IOException e) {
logger.error(
"Transaction scanner error occured: XID="
+ this.XID
+ " key="
+ LCRList.getKey()
+ " cache="
+ LCRList.getByteCache().toString());
throw new RuntimeException("Unable to read transaction from cache: " + this.XID);
} finally {
if (scanner != null) scanner.close();
}
if (!data.isEmpty()) {
DBMSEvent event = new DBMSEvent(lastLCR.eventId, data, true, commitTime);
event.setMetaDataOption(ReplOptionParams.DBMS_TYPE, Database.ORACLE);
event.setMetaDataOption(ReplOptionParams.TIME_ZONE_AWARE, "true");
event.setMetaDataOption(ReplOptionParams.STRINGS, "utf8");
q.put(event);
}
return lastProcessedEventId;
}
}
/**
* {@inheritDoc}
*
* @see com.continuent.tungsten.replicator.extractor.RawExtractor#extract()
*/
@Override
public DBMSEvent extract() throws ReplicatorException, InterruptedException {
ArrayList<DBMSData> data = new ArrayList<DBMSData>();
long maxSCN = -1;
Timestamp sourceTStamp = null;
boolean noData = true;
try {
if (logger.isDebugEnabled()) logger.debug("Extending Window");
executeQuery(
"BEGIN DBMS_CDC_SUBSCRIBE.EXTEND_WINDOW(subscription_name => 'TUNGSTEN_PUB');END;",
false);
if (logger.isDebugEnabled())
logger.debug("Handling " + subscriberViews.keySet().size() + "views");
for (String view : subscriberViews.keySet()) {
OracleCDCSource cdcSource = subscriberViews.get(view);
Statement stmt = connection.getConnection().createStatement();
String statement;
if (lastSCN != null)
statement =
"SELECT "
+ cdcSource.getPublication(view).getColumnList()
+ " , CSCN$, COMMIT_TIMESTAMP$, OPERATION$"
+ " from "
+ view
+ " where cscn$ > "
+ lastSCN
+ " order by cscn$, rsid$";
else
statement =
"SELECT "
+ cdcSource.getPublication(view).getColumnList()
+ " , CSCN$, COMMIT_TIMESTAMP$, OPERATION$"
+ " from "
+ view
+ " order by cscn$, rsid$";
resultset = stmt.executeQuery(statement);
int userColumns = cdcSource.getPublication(view).getColumnsCount();
if (logger.isDebugEnabled()) logger.debug("Running " + statement);
OneRowChange updateRowChange = null;
OneRowChange oneRowChange = null;
while (resultset.next()) {
noData = false;
long currentSCN = resultset.getLong("CSCN$");
if (maxSCN < currentSCN) maxSCN = currentSCN;
if (sourceTStamp == null) sourceTStamp = resultset.getTimestamp("COMMIT_TIMESTAMP$");
if (logger.isDebugEnabled()) {
logger.debug("Receiving data");
StringBuffer buffer = new StringBuffer();
for (int i = 1; i <= resultset.getMetaData().getColumnCount(); i++) {
if (buffer.length() > 0) buffer.append('\t');
buffer.append(resultset.getString(i));
}
logger.debug("Received : " + buffer.toString());
}
String operation = resultset.getString("OPERATION$").trim();
RowChangeData rowData = new RowChangeData();
if (operation.equals("I")) {
if (oneRowChange == null || !oneRowChange.getAction().equals(ActionType.INSERT)) {
oneRowChange =
new OneRowChange(cdcSource.getSchema(), cdcSource.getTable(), ActionType.INSERT);
rowData.appendOneRowChange(oneRowChange);
data.add(rowData);
}
parseRowEvent(oneRowChange, false, userColumns);
} else if (operation.equals("D")) {
if (oneRowChange == null || !oneRowChange.getAction().equals(ActionType.DELETE)) {
oneRowChange =
new OneRowChange(cdcSource.getSchema(), cdcSource.getTable(), ActionType.DELETE);
rowData.appendOneRowChange(oneRowChange);
data.add(rowData);
}
parseRowEvent(oneRowChange, true, userColumns);
} else if (operation.startsWith("U")) {
if (updateRowChange == null) {
updateRowChange =
new OneRowChange(cdcSource.getSchema(), cdcSource.getTable(), ActionType.UPDATE);
rowData.appendOneRowChange(updateRowChange);
data.add(rowData);
if (operation.equals("UO")) {
parseRowEvent(updateRowChange, true, userColumns);
} else if (operation.equals("UN")) {
parseRowEvent(updateRowChange, false, userColumns);
}
} else {
if (operation.equals("UO")) {
parseRowEvent(updateRowChange, true, userColumns);
} else if (operation.equals("UN")) {
parseRowEvent(updateRowChange, false, userColumns);
}
}
} else {
logger.error(
"Unable to extract data from CDC (operation should be I, D, UO or UN - found "
+ operation
+ ")");
}
}
resultset.close();
resultset = null;
stmt.close();
}
lastSCN = null;
} catch (SQLException e) {
throw new ReplicatorException(e);
} finally {
if (resultset != null)
try {
resultset.close();
} catch (SQLException e) {
logger.warn("Failed to close resultset");
}
resultset = null;
}
if (noData) {
logger.warn("Retrieved empty resultset... no data available... sleeping");
Thread.sleep(1000);
}
if (logger.isDebugEnabled()) logger.debug("Purging window");
executeQuery(
"BEGIN DBMS_CDC_SUBSCRIBE.PURGE_WINDOW(subscription_name => 'TUNGSTEN_PUB');END;", false);
if (data.size() > 0) {
DBMSEvent event = new DBMSEvent(String.valueOf(maxSCN), data, sourceTStamp);
// Mark the event as coming from Oracle.
event.setMetaDataOption(ReplOptionParams.DBMS_TYPE, Database.ORACLE);
// Strings are converted to UTF8 rather than using bytes for this
// extractor.
event.setMetaDataOption(ReplOptionParams.STRINGS, "utf8");
return event;
} else {
return null;
}
}