/**
* Forces the master server to fsync the RAM data to disk, then lock all writes. The database will
* be read-only after this command returns.
*
* @return result of the command execution
* @throws MongoException
* @mongodb.driver.manual reference/command/fsync/ fsync command
*/
public CommandResult fsyncAndLock() {
DBObject cmd = new BasicDBObject("fsync", 1);
cmd.put("lock", 1);
CommandResult result = getDB(ADMIN_DATABASE_NAME).command(cmd);
result.throwOnError();
return result;
}
protected void doBatch() throws KettleException {
WriteConcern concern = null;
if (log.getLogLevel().getLevel() >= LogLevel.DETAILED.getLevel()) {
concern = new WriteConcern(1);
}
WriteResult result = null;
if (concern != null) {
result = m_data.getCollection().insert(m_batch, concern);
} else {
result = m_data.getCollection().insert(m_batch);
}
CommandResult cmd = result.getLastError();
if (cmd != null && !cmd.ok()) {
String message = cmd.getErrorMessage();
logError(BaseMessages.getString(PKG, "MongoDbOutput.Messages.Error.MongoReported", message));
try {
cmd.throwOnError();
} catch (MongoException me) {
throw new KettleException(me.getMessage(), me);
}
}
m_batch.clear();
}
protected void throwOnError(WriteConcern wc, WriteResult wr) {
if (wc == null && wr.getLastConcern() == null) {
CommandResult cr = wr.getLastError();
if (cr != null && cr.getErrorMessage() != null && cr.getErrorMessage().length() > 0)
cr.throwOnError();
}
}
void checkAuth(DB db) throws IOException {
DB.AuthorizationCredentials credentials = db.getAuthorizationCredentials();
if (credentials == null) {
if (db._name.equals("admin")) return;
checkAuth(db._mongo.getDB("admin"));
return;
}
if (_authed.containsKey(db)) return;
CommandResult res = runCommand(db, credentials.getNonceCommand());
res.throwOnError();
res = runCommand(db, credentials.getAuthCommand(res.getString("nonce")));
res.throwOnError();
_authed.put(db, true);
}
/**
* Creates a collection with a given name and options. If the collection does not exist, a new
* collection is created.
*
* <p>Possible options:
*
* <ul>
* <li><b>capped</b> ({@code boolean}) - Enables a collection cap. False by default. If enabled,
* you must specify a size parameter.
* <li><b>size</b> ({@code int}) - If capped is true, size specifies a maximum size in bytes for
* the capped collection. When capped is false, you may use size to preallocate space.
* <li><b>max</b> ({@code int}) - Optional. Specifies a maximum "cap" in number of documents for
* capped collections. You must also specify size when specifying max.
* <p>
* </ul>
*
* <p>Note that if the {@code options} parameter is {@code null}, the creation will be deferred to
* when the collection is written to.
*
* @param name the name of the collection to return
* @param options options
* @return the collection
* @throws MongoException
*/
public DBCollection createCollection(String name, DBObject options) {
if (options != null) {
DBObject createCmd = new BasicDBObject("create", name);
createCmd.putAll(options);
CommandResult result = command(createCmd);
result.throwOnError();
}
return getCollection(name);
}
/**
* Forces the master server to fsync the RAM data to disk This is done automatically by the server
* at intervals, but can be forced for better reliability.
*
* @param async if true, the fsync will be done asynchronously on the server.
* @return result of the command execution
* @throws MongoException
* @mongodb.driver.manual reference/command/fsync/ fsync command
*/
public CommandResult fsync(boolean async) {
DBObject cmd = new BasicDBObject("fsync", 1);
if (async) {
cmd.put("async", 1);
}
CommandResult result = getDB(ADMIN_DATABASE_NAME).command(cmd);
result.throwOnError();
return result;
}
@Override
protected void refreshNode() {
// db.getStats can be slow..
// can't use driver's because doesnt use slaveOk
CommandResult res = db.command(new BasicDBObject("dbstats", 1), db.getMongo().getOptions());
// CommandResult res = db.command(new BasicDBObject("profile", -1));
res.throwOnError();
stats = res;
// db.getCollection("foo").save(new BasicDBObject("a", 1));
}
@Test
public void shouldTimeOutCommand() {
checkServerVersion(2.5);
enableMaxTimeFailPoint();
try {
CommandResult res =
getDatabase().command(new BasicDBObject("isMaster", 1).append("maxTimeMS", 1));
res.throwOnError();
fail("Show have thrown");
} catch (MongoExecutionTimeoutException e) {
assertEquals(50, e.getCode());
} finally {
disableMaxTimeFailPoint();
}
}
/**
* Gets a list of the names of all databases on the connected server.
*
* @return list of database names
* @throws MongoException
*/
public List<String> getDatabaseNames() {
BasicDBObject cmd = new BasicDBObject();
cmd.put("listDatabases", 1);
CommandResult res = getDB(ADMIN_DATABASE_NAME).command(cmd, getOptions());
res.throwOnError();
List l = (List) res.get("databases");
List<String> list = new ArrayList<String>();
for (Object o : l) {
list.add(((BasicDBObject) o).getString("name"));
}
return list;
}
protected <T> void ensureIndex(
Class<T> clazz,
String name,
BasicDBObject fields,
boolean unique,
boolean dropDupsOnCreate,
boolean background,
boolean sparse) {
// validate field names and translate them to the stored values
BasicDBObject keys = new BasicDBObject();
for (Entry<String, Object> entry : fields.entrySet()) {
StringBuffer sb = new StringBuffer(entry.getKey());
Mapper.validate(clazz, mapr, sb, FilterOperator.IN, "", true, false);
keys.put(sb.toString(), entry.getValue());
}
BasicDBObjectBuilder keyOpts = new BasicDBObjectBuilder();
if (name != null && name.length() > 0) {
keyOpts.add("name", name);
}
if (unique) {
keyOpts.add("unique", true);
if (dropDupsOnCreate) keyOpts.add("dropDups", true);
}
if (background) keyOpts.add("background", true);
if (sparse) keyOpts.add("sparse", true);
DBCollection dbColl = getCollection(clazz);
BasicDBObject opts = (BasicDBObject) keyOpts.get();
if (opts.isEmpty()) {
log.debug("Ensuring index for " + dbColl.getName() + " with keys:" + keys);
dbColl.ensureIndex(keys);
} else {
log.debug(
"Ensuring index for " + dbColl.getName() + " with keys:" + keys + " and opts:" + opts);
dbColl.ensureIndex(keys, opts);
}
// TODO: remove this once using 2.4 driver does this in ensureIndex
CommandResult cr = dbColl.getDB().getLastError();
cr.throwOnError();
}
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
Object[] row = getRow();
if (row == null) {
// no more output
// check any remaining buffered objects
if (m_batch != null && m_batch.size() > 0) {
doBatch();
}
// INDEXING - http://www.mongodb.org/display/DOCS/Indexes
// Indexing is computationally expensive - it needs to be
// done after all data is inserted and done in the BACKGROUND.
// UNIQUE indexes (prevent duplicates on the
// keys in the index) and SPARSE indexes (don't index docs that
// don't have the key field) - current limitation is that SPARSE
// indexes can only have a single field
List<MongoDbOutputMeta.MongoIndex> indexes = m_meta.getMongoIndexes();
if (indexes != null && indexes.size() > 0) {
logBasic(BaseMessages.getString(PKG, "MongoDbOutput.Messages.ApplyingIndexOpps"));
m_data.applyIndexes(indexes, log, m_meta.getTruncate());
}
disconnect();
setOutputDone();
return false;
}
if (first) {
first = false;
m_batchInsertSize = 100;
String batchInsert = environmentSubstitute(m_meta.getBatchInsertSize());
if (!Const.isEmpty(batchInsert)) {
m_batchInsertSize = Integer.parseInt(batchInsert);
}
m_batch = new ArrayList<DBObject>(m_batchInsertSize);
// output the same as the input
m_data.setOutputRowMeta(getInputRowMeta());
m_mongoTopLevelStructure =
MongoDbOutputData.checkTopLevelConsistency(m_meta.m_mongoFields, this);
if (m_mongoTopLevelStructure == MongoDbOutputData.MongoTopLevel.INCONSISTENT) {
throw new KettleException(
BaseMessages.getString(PKG, "MongoDbOutput.Messages.Error.InconsistentMongoTopLevel"));
}
// first check our incoming fields against our meta data for fields to insert
RowMetaInterface rmi = getInputRowMeta();
List<MongoDbOutputMeta.MongoField> mongoFields = m_meta.getMongoFields();
List<String> notToBeInserted = new ArrayList<String>();
for (int i = 0; i < rmi.size(); i++) {
ValueMetaInterface vm = rmi.getValueMeta(i);
boolean ok = false;
for (MongoDbOutputMeta.MongoField field : mongoFields) {
String mongoMatch = environmentSubstitute(field.m_incomingFieldName);
if (vm.getName().equals(mongoMatch)) {
ok = true;
break;
}
}
if (!ok) {
notToBeInserted.add(vm.getName());
}
}
if (notToBeInserted.size() == rmi.size()) {
throw new KettleException(
BaseMessages.getString(PKG, "MongoDbOutput.Messages.Error.NotInsertingAnyFields"));
}
if (notToBeInserted.size() > 0) {
StringBuffer b = new StringBuffer();
for (String s : notToBeInserted) {
b.append(s).append(" ");
}
logBasic(
BaseMessages.getString(PKG, "MongoDbOutput.Messages.FieldsNotToBeInserted"),
b.toString());
}
// init mongo fields
for (MongoDbOutputMeta.MongoField m : m_meta.getMongoFields()) {
m.init(this);
}
// check truncate
if (m_meta.getTruncate()) {
try {
logBasic(BaseMessages.getString(PKG, "MongoDbOutput.Messages.TruncatingCollection"));
m_data.getCollection().drop();
// re-establish the collection
String collection = environmentSubstitute(m_meta.getCollection());
m_data.createCollection(collection);
m_data.setCollection(m_data.getDB().getCollection(collection));
} catch (Exception m) {
disconnect();
throw new KettleException(m.getMessage(), m);
}
}
}
if (!isStopped()) {
if (m_meta.getUpsert()) {
/*DBObject updateQuery = MongoDbOutputData.getQueryObject(m_meta.getMongoFields(),
getInputRowMeta(), row, getParentVariableSpace(), m_mongoTopLevelStructure); */
DBObject updateQuery =
MongoDbOutputData.getQueryObject(m_meta.getMongoFields(), getInputRowMeta(), row, this);
if (log.isDebug()) {
logDebug(
BaseMessages.getString(
PKG, "MongoDbOutput.Messages.Debug.QueryForUpsert", updateQuery));
}
if (updateQuery != null) {
// i.e. we have some non-null incoming query field values
DBObject insertUpdate = null;
// get the record to update the match with
if (!m_meta.getModifierUpdate()) {
// complete record replace or insert
insertUpdate =
MongoDbOutputData.kettleRowToMongo(
m_meta.getMongoFields(),
getInputRowMeta(),
row,
this,
m_mongoTopLevelStructure);
} else {
// specific field update or insert
insertUpdate =
MongoDbOutputData.getModifierUpdateObject(
m_meta.getMongoFields(),
getInputRowMeta(),
row,
this,
m_mongoTopLevelStructure);
if (log.isDebug()) {
logDebug(
BaseMessages.getString(
PKG, "MongoDbOutput.Messages.Debug.ModifierUpdateObject", insertUpdate));
}
}
if (insertUpdate != null) {
WriteConcern concern = null;
if (log.getLogLevel().getLevel() >= LogLevel.DETAILED.getLevel()) {
concern = new WriteConcern(1);
}
WriteResult result = null;
if (concern != null) {
result =
m_data
.getCollection()
.update(updateQuery, insertUpdate, true, m_meta.getMulti(), concern);
} else {
result =
m_data.getCollection().update(updateQuery, insertUpdate, true, m_meta.getMulti());
}
CommandResult cmd = result.getLastError();
if (cmd != null && !cmd.ok()) {
String message = cmd.getErrorMessage();
logError(
BaseMessages.getString(
PKG, "MongoDbOutput.Messages.Error.MongoReported", message));
try {
cmd.throwOnError();
} catch (MongoException me) {
throw new KettleException(me.getMessage(), me);
}
}
}
}
} else {
// straight insert
DBObject mongoInsert =
MongoDbOutputData.kettleRowToMongo(
m_meta.getMongoFields(), getInputRowMeta(), row, this, m_mongoTopLevelStructure);
if (mongoInsert != null) {
m_batch.add(mongoInsert);
}
if (m_batch.size() == m_batchInsertSize) {
logBasic(BaseMessages.getString(PKG, "MongoDbOutput.Messages.CommitingABatch"));
doBatch();
}
}
}
return true;
}
/**
* Calls {@link DB#doEval(java.lang.String, java.lang.Object[]) }. If the command is successful,
* the "retval" field is extracted and returned. Otherwise an exception is thrown.
*
* @param code {@code String} representation of JavaScript function
* @param args arguments to pass to the JavaScript function
* @return result of the execution
* @throws MongoException if the operation failed
* @mongodb.driver.manual reference/method/db.eval/ db.eval()
*/
public Object eval(final String code, final Object... args) {
CommandResult result = doEval(code, args);
result.throwOnError();
return result.get("retval");
}
@SuppressWarnings("rawtypes")
public <T> MapreduceResults<T> mapReduce(
MapreduceType type,
Query query,
String map,
String reduce,
String finalize,
Map<String, Object> scopeFields,
Class<T> outputType) {
Assert.parametersNotNull("map", map);
Assert.parameterNotEmpty(map, "map");
Assert.parametersNotNull("reduce", reduce);
Assert.parameterNotEmpty(reduce, "reduce");
QueryImpl<T> qi = (QueryImpl<T>) query;
DBCollection dbColl = qi.getCollection();
// TODO remove this after testing.
if (dbColl == null) dbColl = getCollection(qi.getEntityClass());
if (log.isTraceEnabled())
log.info(
"Executing mapReduce("
+ dbColl.getName()
+ ") with query("
+ qi.toString()
+ ") map("
+ map
+ ") reduce("
+ reduce
+ ") finalize("
+ finalize
+ ") scope("
+ scopeFields
+ ")");
// TODO replace this with the 2.4 driver impl.
String outColl = mapr.getCollectionName(outputType);
BasicDBObjectBuilder bldr =
BasicDBObjectBuilder.start("mapreduce", mapr.getCollectionName(qi.getEntityClass()));
switch (type) {
case REDUCE:
bldr.push("out").add("reduce", outColl).pop();
break;
case MERGE:
bldr.push("out").add("merge", outColl).pop();
break;
case INLINE:
bldr.push("out").add("inline", 1).pop();
break;
default:
bldr.add("out", outColl);
break;
}
if (qi.getOffset() != 0 || qi.getFieldsObject() != null)
throw new QueryException(
"mapReduce does not allow the offset/retrievedFields query options.");
if (qi.getQueryObject() != null) bldr.add("query", qi.getQueryObject());
if (qi.getLimit() > 0) bldr.add("limit", qi.getLimit());
if (qi.getSortObject() != null) bldr.add("sort", qi.getSortObject());
bldr.add("map", map);
bldr.add("reduce", reduce);
if (finalize != null && finalize.length() > 0) bldr.add("finalize", finalize);
if (scopeFields != null && scopeFields.size() > 0)
bldr.add("scope", mapr.toMongoObject(null, null, scopeFields));
DBObject dbObj = bldr.get();
CommandResult cr = dbColl.getDB().command(dbObj);
cr.throwOnError();
MapreduceResults mrRes =
(MapreduceResults) mapr.fromDBObject(MapreduceResults.class, cr, createCache());
QueryImpl baseQ = null;
if (!MapreduceType.INLINE.equals(type))
baseQ = new QueryImpl(outputType, db.getCollection(mrRes.getOutputCollectionName()), this);
// TODO Handle inline case and create an iterator/able.
mrRes.setBits(type, baseQ);
return mrRes;
}
/**
* Drops this database, deleting the associated data files. Use with caution.
*
* @throws MongoException
*/
public void dropDatabase() {
CommandResult res = command(new BasicDBObject("dropDatabase", 1));
res.throwOnError();
_mongo._dbs.remove(this.getName());
}
/**
* Calls {@link DB#doEval(java.lang.String, java.lang.Object[]) }. If the command is successful,
* the "retval" field is extracted and returned. Otherwise an exception is thrown.
*
* @param code @{code String} representation of JavaScript function
* @param args arguments to pass to the JavaScript function
* @return result of the execution
* @throws MongoException
*/
public Object eval(String code, Object... args) {
CommandResult res = doEval(code, args);
res.throwOnError();
return res.get("retval");
}