/**
* {@inheritDoc}
*
* @see Writable#readFields(DataInput)
*/
public void readFields(DataInput in) throws IOException {
BSONDecoder dec = new BSONDecoder();
BSONCallback cb = new BasicBSONCallback();
// Read the BSON length from the start of the record
int dataLen = in.readInt();
byte[] buf = new byte[dataLen];
in.readFully(buf);
dec.decode(buf, cb);
_doc = (BSONObject) cb.get();
log.info("Decoded a BSON Object: " + _doc);
}
@Override
public TStruct readStructBegin() throws TException {
if (!inFlight()) {
// This is the root struct, so first decode from the real transport.
try {
BSONDecoder decoder = new BasicBSONDecoder();
BSONObject srcObject = decoder.readObject(new TTransportInputStream(realTransport));
super.setSource(srcObject);
} catch (IOException e) {
throw new TException(e);
}
}
return super.readStructBegin();
}
protected void run() {
long startTime = System.currentTimeMillis();
// decide what collections to process
selectCollections();
OplogReplayWriter util = new OplogReplayWriter();
// create any re-mappings
Map<String, String> collectionMappings = new HashMap<String, String>();
Map<String, String> databaseMappings = new HashMap<String, String>();
createMappings(
DATABASE_MAPPING_STRING, COLLECTION_MAPPING_STRING, databaseMappings, collectionMappings);
// configure the writer
util.setCollectionMappings(collectionMappings);
util.setDatabaseMappings(databaseMappings);
util.setDestinationDatabaseUsername(DEST_DATABASE_USER_NAME);
util.setDestinationDatabasePassword(DEST_DATABASE_PASSWORD);
util.setDestinationDatabaseHost(DEST_DATABASE_HOST);
try {
File[] files = new File(INPUT_DIR).listFiles();
if (files != null) {
List<File> filesToProcess = new ArrayList<File>();
for (File file : files) {
if (file.getName().indexOf(".bson") > 0) {
filesToProcess.add(file);
}
}
long operationsRead = 0;
long operationsSkipped = 0;
long lastOutput = System.currentTimeMillis();
for (File file : filesToProcess) {
System.out.println("replaying file " + file.getName());
BufferedInputStream inputStream = null;
try {
if (file.getName().endsWith(".gz")) {
InputStream is = new GZIPInputStream(new FileInputStream(file));
inputStream = new BufferedInputStream(is);
} else {
inputStream = new BufferedInputStream(new FileInputStream(file));
}
BSONDecoder decoder = new DefaultDBDecoder();
while (true) {
if (inputStream.available() == 0) {
break;
}
BSONObject obj = decoder.readObject(inputStream);
if (obj == null) {
break;
}
BasicDBObject dbo = new BasicDBObject((BasicBSONObject) obj);
BSONTimestamp operationTimestamp = (BSONTimestamp) dbo.get("ts");
String namespace = dbo.getString("ns");
String collection = util.getUnmappedCollectionFromNamespace(namespace);
boolean shouldProcess = shouldProcessRecord(collection, operationTimestamp);
if (collection != null && shouldProcess) {
util.processRecord(dbo);
operationsRead++;
} else {
operationsSkipped++;
}
long durationSinceLastOutput = System.currentTimeMillis() - lastOutput;
if (durationSinceLastOutput > REPORT_INTERVAL) {
report(
util.getInsertCount(),
util.getUpdateCount(),
util.getDeleteCount(),
operationsRead,
operationsSkipped,
System.currentTimeMillis() - startTime);
lastOutput = System.currentTimeMillis();
}
}
} catch (Exception ex) {
ex.printStackTrace();
}
}
}
} catch (Exception e) {
e.printStackTrace();
}
}