/** Filter the event if it has already been executed. */
public ReplDBMSEvent filter(ReplDBMSEvent event)
throws ReplicatorException, InterruptedException {
totalEvents++;
if (appliedTimes == null) appliedTimes = new TreeMap<Long, Timestamp>();
Timestamp sourceTstamp = event.getDBMSEvent().getSourceTstamp();
appliedTimes.put(event.getSeqno(), sourceTstamp);
long currentTime = System.currentTimeMillis();
prefetchLatency = currentTime - sourceTstamp.getTime();
if (interval == 0 || lastChecked == 0 || (currentTime - lastChecked >= interval)) {
// It is now time to check CommitSeqnoTable again
checkSlavePosition(currentTime);
}
if (initTime == 0) initTime = sourceTstamp.getTime();
if (event.getSeqno() <= currentSeqno) {
if (logger.isDebugEnabled())
logger.debug("Discarding event " + event.getSeqno() + " as it is already applied");
return null;
} else
while (sourceTstamp.getTime() - initTime > (aheadMaxTime * 1000)) {
if (logger.isDebugEnabled())
logger.debug("Event is too far ahead of current slave position... sleeping");
// this event is too far ahead of the CommitSeqnoTable position:
// sleep some time and continue
long sleepStartMillis = System.currentTimeMillis();
try {
prefetchState = PrefetchState.sleeping;
Thread.sleep(sleepTime);
} catch (InterruptedException e) {
return null;
} finally {
prefetchState = PrefetchState.active;
sleepTimeMillis += (System.currentTimeMillis() - sleepStartMillis);
}
// Check again CommitSeqnoTable
checkSlavePosition(System.currentTimeMillis());
// and whereas the event got applied while sleeping
if (event.getSeqno() <= currentSeqno) {
if (logger.isDebugEnabled())
logger.debug("Discarding event " + event.getSeqno() + " as it is already applied");
return null;
}
}
prefetchEvents++;
if (logger.isDebugEnabled() && totalEvents % 20000 == 0)
logger.debug(
"Prefetched "
+ prefetchEvents
+ " events - Ratio "
+ (100 * prefetchEvents / totalEvents)
+ "%");
return event;
}
// Confirms that a particular event is ignored.
private void verifyIgnore(FilterVerificationHelper filterHelper, ReplDBMSEvent e)
throws ReplicatorException, InterruptedException {
ReplDBMSEvent e2 = filterHelper.filter(e);
assertNull("Should ignore event: seqno=" + e.getSeqno(), e2);
}
/**
* {@inheritDoc}
*
* @see
* com.continuent.tungsten.replicator.filter.Filter#filter(com.continuent.tungsten.replicator.event.ReplDBMSEvent)
*/
public ReplDBMSEvent filter(ReplDBMSEvent event) throws ReplicatorException {
ArrayList<DBMSData> data = event.getData();
for (Iterator<DBMSData> iterator = data.iterator(); iterator.hasNext(); ) {
DBMSData dataElem = iterator.next();
if (dataElem instanceof RowChangeData) {
RowChangeData rdata = (RowChangeData) dataElem;
for (Iterator<OneRowChange> iterator2 = rdata.getRowChanges().iterator();
iterator2.hasNext(); ) {
OneRowChange orc = iterator2.next();
// Don't analyze tables from Tungsten schema.
if (orc.getSchemaName().compareToIgnoreCase(tungstenSchema) == 0) {
if (logger.isDebugEnabled()) logger.debug("Ignoring " + tungstenSchema + " schema");
continue;
}
// Loop through defined transformations (usually one).
Iterator<String> it = definitions.keySet().iterator();
while (it.hasNext()) {
String transformation = it.next();
JSONArray array = definitions.get(transformation);
// Not using any hash, because simple loop should be
// fast enough for a few expected entries.
for (Object o : array) {
JSONObject jo = (JSONObject) o;
String defSchema = (String) jo.get("schema");
String defTable = (String) jo.get("table");
// Found a filter request for this schema & table?
if ((defSchema.equals("*") || defSchema.equals(orc.getSchemaName()))
&& (defTable.equals("*") || defTable.equals(orc.getTableName()))) {
// Defined columns to filter.
JSONArray defColumns = (JSONArray) jo.get("columns");
// Filter column values.
ArrayList<ColumnSpec> colSpecs = orc.getColumnSpec();
ArrayList<ArrayList<OneRowChange.ColumnVal>> colValues = orc.getColumnValues();
for (int c = 0; c < colSpecs.size(); c++) {
ColumnSpec colSpec = colSpecs.get(c);
if (colSpec.getName() != null) {
// Have this column in definitions?
if (defColumns.contains(colSpec.getName())) {
// Iterate through all rows in the
// column.
for (int row = 0; row < colValues.size(); row++) {
ColumnVal colValue = colValues.get(row).get(c);
if (colValue.getValue() != null) {
if (logger.isDebugEnabled())
logger.debug("Sending value: " + colValue.getValue());
// Send to server.
Object newValue =
sendToFilter(
transformation,
event.getSeqno(),
row,
orc.getSchemaName(),
orc.getTableName(),
colSpec.getName(),
colValue.getValue());
colValue.setValue((Serializable) newValue);
if (logger.isDebugEnabled()) logger.debug("Received value: " + newValue);
}
}
}
} else {
if (logger.isDebugEnabled()) {
logger.debug(
"Expected to filter column, but column name is undefined: "
+ orc.getSchemaName()
+ "."
+ orc.getTableName()
+ "["
+ colSpec.getIndex()
+ "]");
}
}
}
// Filter key values.
ArrayList<ColumnSpec> keySpecs = orc.getKeySpec();
ArrayList<ArrayList<OneRowChange.ColumnVal>> keyValues = orc.getKeyValues();
for (int k = 0; k < keySpecs.size(); k++) {
ColumnSpec keySpec = keySpecs.get(k);
if (keySpec.getName() != null) {
if (defColumns.contains(keySpec.getName())) {
// Iterate through all rows in the
// key.
for (int row = 0; row < keyValues.size(); row++) {
ColumnVal keyValue = keyValues.get(row).get(k);
if (keyValue.getValue() != null) {
if (logger.isDebugEnabled())
logger.debug("Sending value: " + keyValue.getValue());
// Send to server.
Object newValue =
sendToFilter(
transformation,
event.getSeqno(),
row,
orc.getSchemaName(),
orc.getTableName(),
keySpec.getName(),
keyValue.getValue());
keyValue.setValue((Serializable) newValue);
if (logger.isDebugEnabled()) logger.debug("Received value: " + newValue);
}
}
}
} else {
if (logger.isDebugEnabled()) {
logger.debug(
"Expected to filter key, but column name is undefined: "
+ orc.getSchemaName()
+ "."
+ orc.getTableName()
+ "["
+ keySpec.getIndex()
+ "]");
}
}
}
}
}
}
}
} else if (dataElem instanceof StatementData) {
// Not supported.
}
}
return event;
}
// Confirms that a particular event is accepted.
private void verifyAccept(FilterVerificationHelper filterHelper, ReplDBMSEvent e)
throws ReplicatorException, InterruptedException {
ReplDBMSEvent e2 = filterHelper.filter(e);
assertTrue("Should allow event, seqno=" + e.getSeqno(), e == e2);
}
