Esempio n. 1
0
 /**
  * {@inheritDoc}
  *
  * @see Writable#readFields(DataInput)
  */
 public void readFields(DataInput in) throws IOException {
   BSONDecoder dec = new BSONDecoder();
   BSONCallback cb = new BasicBSONCallback();
   // Read the BSON length from the start of the record
   int dataLen = in.readInt();
   byte[] buf = new byte[dataLen];
   in.readFully(buf);
   dec.decode(buf, cb);
   _doc = (BSONObject) cb.get();
   log.info("Decoded a BSON Object: " + _doc);
 }
Esempio n. 2
0
 @Override
 public TStruct readStructBegin() throws TException {
   if (!inFlight()) {
     // This is the root struct, so first decode from the real transport.
     try {
       BSONDecoder decoder = new BasicBSONDecoder();
       BSONObject srcObject = decoder.readObject(new TTransportInputStream(realTransport));
       super.setSource(srcObject);
     } catch (IOException e) {
       throw new TException(e);
     }
   }
   return super.readStructBegin();
 }
Esempio n. 3
0
  protected void run() {
    long startTime = System.currentTimeMillis();
    //	decide what collections to process
    selectCollections();

    OplogReplayWriter util = new OplogReplayWriter();

    //	create any re-mappings
    Map<String, String> collectionMappings = new HashMap<String, String>();
    Map<String, String> databaseMappings = new HashMap<String, String>();
    createMappings(
        DATABASE_MAPPING_STRING, COLLECTION_MAPPING_STRING, databaseMappings, collectionMappings);

    //	configure the writer
    util.setCollectionMappings(collectionMappings);
    util.setDatabaseMappings(databaseMappings);
    util.setDestinationDatabaseUsername(DEST_DATABASE_USER_NAME);
    util.setDestinationDatabasePassword(DEST_DATABASE_PASSWORD);
    util.setDestinationDatabaseHost(DEST_DATABASE_HOST);

    try {
      File[] files = new File(INPUT_DIR).listFiles();
      if (files != null) {
        List<File> filesToProcess = new ArrayList<File>();
        for (File file : files) {
          if (file.getName().indexOf(".bson") > 0) {
            filesToProcess.add(file);
          }
        }
        long operationsRead = 0;
        long operationsSkipped = 0;
        long lastOutput = System.currentTimeMillis();
        for (File file : filesToProcess) {
          System.out.println("replaying file " + file.getName());
          BufferedInputStream inputStream = null;
          try {
            if (file.getName().endsWith(".gz")) {
              InputStream is = new GZIPInputStream(new FileInputStream(file));
              inputStream = new BufferedInputStream(is);
            } else {
              inputStream = new BufferedInputStream(new FileInputStream(file));
            }
            BSONDecoder decoder = new DefaultDBDecoder();
            while (true) {
              if (inputStream.available() == 0) {
                break;
              }
              BSONObject obj = decoder.readObject(inputStream);
              if (obj == null) {
                break;
              }
              BasicDBObject dbo = new BasicDBObject((BasicBSONObject) obj);

              BSONTimestamp operationTimestamp = (BSONTimestamp) dbo.get("ts");
              String namespace = dbo.getString("ns");
              String collection = util.getUnmappedCollectionFromNamespace(namespace);

              boolean shouldProcess = shouldProcessRecord(collection, operationTimestamp);

              if (collection != null && shouldProcess) {
                util.processRecord(dbo);
                operationsRead++;
              } else {
                operationsSkipped++;
              }

              long durationSinceLastOutput = System.currentTimeMillis() - lastOutput;
              if (durationSinceLastOutput > REPORT_INTERVAL) {
                report(
                    util.getInsertCount(),
                    util.getUpdateCount(),
                    util.getDeleteCount(),
                    operationsRead,
                    operationsSkipped,
                    System.currentTimeMillis() - startTime);
                lastOutput = System.currentTimeMillis();
              }
            }
          } catch (Exception ex) {
            ex.printStackTrace();
          }
        }
      }
    } catch (Exception e) {
      e.printStackTrace();
    }
  }