示例#1
0
 /**
  * Forces the master server to fsync the RAM data to disk, then lock all writes. The database will
  * be read-only after this command returns.
  *
  * @return result of the command execution
  * @throws MongoException
  * @mongodb.driver.manual reference/command/fsync/ fsync command
  */
 public CommandResult fsyncAndLock() {
   DBObject cmd = new BasicDBObject("fsync", 1);
   cmd.put("lock", 1);
   CommandResult result = getDB(ADMIN_DATABASE_NAME).command(cmd);
   result.throwOnError();
   return result;
 }
  protected void doBatch() throws KettleException {
    WriteConcern concern = null;

    if (log.getLogLevel().getLevel() >= LogLevel.DETAILED.getLevel()) {
      concern = new WriteConcern(1);
    }
    WriteResult result = null;

    if (concern != null) {
      result = m_data.getCollection().insert(m_batch, concern);
    } else {
      result = m_data.getCollection().insert(m_batch);
    }

    CommandResult cmd = result.getLastError();

    if (cmd != null && !cmd.ok()) {
      String message = cmd.getErrorMessage();
      logError(BaseMessages.getString(PKG, "MongoDbOutput.Messages.Error.MongoReported", message));
      try {
        cmd.throwOnError();
      } catch (MongoException me) {
        throw new KettleException(me.getMessage(), me);
      }
    }

    m_batch.clear();
  }
 protected void throwOnError(WriteConcern wc, WriteResult wr) {
   if (wc == null && wr.getLastConcern() == null) {
     CommandResult cr = wr.getLastError();
     if (cr != null && cr.getErrorMessage() != null && cr.getErrorMessage().length() > 0)
       cr.throwOnError();
   }
 }
示例#4
0
  void checkAuth(DB db) throws IOException {
    DB.AuthorizationCredentials credentials = db.getAuthorizationCredentials();
    if (credentials == null) {
      if (db._name.equals("admin")) return;
      checkAuth(db._mongo.getDB("admin"));
      return;
    }
    if (_authed.containsKey(db)) return;

    CommandResult res = runCommand(db, credentials.getNonceCommand());
    res.throwOnError();

    res = runCommand(db, credentials.getAuthCommand(res.getString("nonce")));
    res.throwOnError();

    _authed.put(db, true);
  }
示例#5
0
 /**
  * Creates a collection with a given name and options. If the collection does not exist, a new
  * collection is created.
  *
  * <p>Possible options:
  *
  * <ul>
  *   <li><b>capped</b> ({@code boolean}) - Enables a collection cap. False by default. If enabled,
  *       you must specify a size parameter.
  *   <li><b>size</b> ({@code int}) - If capped is true, size specifies a maximum size in bytes for
  *       the capped collection. When capped is false, you may use size to preallocate space.
  *   <li><b>max</b> ({@code int}) - Optional. Specifies a maximum "cap" in number of documents for
  *       capped collections. You must also specify size when specifying max.
  *       <p>
  * </ul>
  *
  * <p>Note that if the {@code options} parameter is {@code null}, the creation will be deferred to
  * when the collection is written to.
  *
  * @param name the name of the collection to return
  * @param options options
  * @return the collection
  * @throws MongoException
  */
 public DBCollection createCollection(String name, DBObject options) {
   if (options != null) {
     DBObject createCmd = new BasicDBObject("create", name);
     createCmd.putAll(options);
     CommandResult result = command(createCmd);
     result.throwOnError();
   }
   return getCollection(name);
 }
示例#6
0
 /**
  * Forces the master server to fsync the RAM data to disk This is done automatically by the server
  * at intervals, but can be forced for better reliability.
  *
  * @param async if true, the fsync will be done asynchronously on the server.
  * @return result of the command execution
  * @throws MongoException
  * @mongodb.driver.manual reference/command/fsync/ fsync command
  */
 public CommandResult fsync(boolean async) {
   DBObject cmd = new BasicDBObject("fsync", 1);
   if (async) {
     cmd.put("async", 1);
   }
   CommandResult result = getDB(ADMIN_DATABASE_NAME).command(cmd);
   result.throwOnError();
   return result;
 }
示例#7
0
 @Override
 protected void refreshNode() {
   // db.getStats can be slow..
   // can't use driver's because doesnt use slaveOk
   CommandResult res = db.command(new BasicDBObject("dbstats", 1), db.getMongo().getOptions());
   //        CommandResult res = db.command(new BasicDBObject("profile", -1));
   res.throwOnError();
   stats = res;
   //        db.getCollection("foo").save(new BasicDBObject("a", 1));
 }
示例#8
0
 @Test
 public void shouldTimeOutCommand() {
   checkServerVersion(2.5);
   enableMaxTimeFailPoint();
   try {
     CommandResult res =
         getDatabase().command(new BasicDBObject("isMaster", 1).append("maxTimeMS", 1));
     res.throwOnError();
     fail("Show have thrown");
   } catch (MongoExecutionTimeoutException e) {
     assertEquals(50, e.getCode());
   } finally {
     disableMaxTimeFailPoint();
   }
 }
示例#9
0
  /**
   * Gets a list of the names of all databases on the connected server.
   *
   * @return list of database names
   * @throws MongoException
   */
  public List<String> getDatabaseNames() {

    BasicDBObject cmd = new BasicDBObject();
    cmd.put("listDatabases", 1);

    CommandResult res = getDB(ADMIN_DATABASE_NAME).command(cmd, getOptions());
    res.throwOnError();

    List l = (List) res.get("databases");

    List<String> list = new ArrayList<String>();

    for (Object o : l) {
      list.add(((BasicDBObject) o).getString("name"));
    }
    return list;
  }
示例#10
0
  protected <T> void ensureIndex(
      Class<T> clazz,
      String name,
      BasicDBObject fields,
      boolean unique,
      boolean dropDupsOnCreate,
      boolean background,
      boolean sparse) {
    // validate field names and translate them to the stored values
    BasicDBObject keys = new BasicDBObject();
    for (Entry<String, Object> entry : fields.entrySet()) {
      StringBuffer sb = new StringBuffer(entry.getKey());
      Mapper.validate(clazz, mapr, sb, FilterOperator.IN, "", true, false);
      keys.put(sb.toString(), entry.getValue());
    }

    BasicDBObjectBuilder keyOpts = new BasicDBObjectBuilder();
    if (name != null && name.length() > 0) {
      keyOpts.add("name", name);
    }
    if (unique) {
      keyOpts.add("unique", true);
      if (dropDupsOnCreate) keyOpts.add("dropDups", true);
    }

    if (background) keyOpts.add("background", true);
    if (sparse) keyOpts.add("sparse", true);

    DBCollection dbColl = getCollection(clazz);

    BasicDBObject opts = (BasicDBObject) keyOpts.get();
    if (opts.isEmpty()) {
      log.debug("Ensuring index for " + dbColl.getName() + " with keys:" + keys);
      dbColl.ensureIndex(keys);
    } else {
      log.debug(
          "Ensuring index for " + dbColl.getName() + " with keys:" + keys + " and opts:" + opts);
      dbColl.ensureIndex(keys, opts);
    }

    // TODO: remove this once using 2.4 driver does this in ensureIndex
    CommandResult cr = dbColl.getDB().getLastError();
    cr.throwOnError();
  }
示例#11
0
  public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {

    Object[] row = getRow();

    if (row == null) {
      // no more output

      // check any remaining buffered objects
      if (m_batch != null && m_batch.size() > 0) {
        doBatch();
      }

      // INDEXING - http://www.mongodb.org/display/DOCS/Indexes
      // Indexing is computationally expensive - it needs to be
      // done after all data is inserted and done in the BACKGROUND.

      // UNIQUE indexes (prevent duplicates on the
      // keys in the index) and SPARSE indexes (don't index docs that
      // don't have the key field) - current limitation is that SPARSE
      // indexes can only have a single field

      List<MongoDbOutputMeta.MongoIndex> indexes = m_meta.getMongoIndexes();
      if (indexes != null && indexes.size() > 0) {
        logBasic(BaseMessages.getString(PKG, "MongoDbOutput.Messages.ApplyingIndexOpps"));
        m_data.applyIndexes(indexes, log, m_meta.getTruncate());
      }

      disconnect();
      setOutputDone();
      return false;
    }

    if (first) {
      first = false;

      m_batchInsertSize = 100;

      String batchInsert = environmentSubstitute(m_meta.getBatchInsertSize());
      if (!Const.isEmpty(batchInsert)) {
        m_batchInsertSize = Integer.parseInt(batchInsert);
      }
      m_batch = new ArrayList<DBObject>(m_batchInsertSize);

      // output the same as the input
      m_data.setOutputRowMeta(getInputRowMeta());

      m_mongoTopLevelStructure =
          MongoDbOutputData.checkTopLevelConsistency(m_meta.m_mongoFields, this);
      if (m_mongoTopLevelStructure == MongoDbOutputData.MongoTopLevel.INCONSISTENT) {
        throw new KettleException(
            BaseMessages.getString(PKG, "MongoDbOutput.Messages.Error.InconsistentMongoTopLevel"));
      }

      // first check our incoming fields against our meta data for fields to insert
      RowMetaInterface rmi = getInputRowMeta();
      List<MongoDbOutputMeta.MongoField> mongoFields = m_meta.getMongoFields();
      List<String> notToBeInserted = new ArrayList<String>();
      for (int i = 0; i < rmi.size(); i++) {
        ValueMetaInterface vm = rmi.getValueMeta(i);
        boolean ok = false;
        for (MongoDbOutputMeta.MongoField field : mongoFields) {
          String mongoMatch = environmentSubstitute(field.m_incomingFieldName);
          if (vm.getName().equals(mongoMatch)) {
            ok = true;
            break;
          }
        }

        if (!ok) {
          notToBeInserted.add(vm.getName());
        }
      }

      if (notToBeInserted.size() == rmi.size()) {
        throw new KettleException(
            BaseMessages.getString(PKG, "MongoDbOutput.Messages.Error.NotInsertingAnyFields"));
      }

      if (notToBeInserted.size() > 0) {
        StringBuffer b = new StringBuffer();
        for (String s : notToBeInserted) {
          b.append(s).append(" ");
        }

        logBasic(
            BaseMessages.getString(PKG, "MongoDbOutput.Messages.FieldsNotToBeInserted"),
            b.toString());
      }

      // init mongo fields
      for (MongoDbOutputMeta.MongoField m : m_meta.getMongoFields()) {
        m.init(this);
      }

      // check truncate
      if (m_meta.getTruncate()) {
        try {
          logBasic(BaseMessages.getString(PKG, "MongoDbOutput.Messages.TruncatingCollection"));
          m_data.getCollection().drop();

          // re-establish the collection
          String collection = environmentSubstitute(m_meta.getCollection());
          m_data.createCollection(collection);
          m_data.setCollection(m_data.getDB().getCollection(collection));
        } catch (Exception m) {
          disconnect();
          throw new KettleException(m.getMessage(), m);
        }
      }
    }

    if (!isStopped()) {

      if (m_meta.getUpsert()) {
        /*DBObject updateQuery = MongoDbOutputData.getQueryObject(m_meta.getMongoFields(),
        getInputRowMeta(), row, getParentVariableSpace(), m_mongoTopLevelStructure); */
        DBObject updateQuery =
            MongoDbOutputData.getQueryObject(m_meta.getMongoFields(), getInputRowMeta(), row, this);

        if (log.isDebug()) {
          logDebug(
              BaseMessages.getString(
                  PKG, "MongoDbOutput.Messages.Debug.QueryForUpsert", updateQuery));
        }

        if (updateQuery != null) {
          // i.e. we have some non-null incoming query field values
          DBObject insertUpdate = null;

          // get the record to update the match with
          if (!m_meta.getModifierUpdate()) {
            // complete record replace or insert

            insertUpdate =
                MongoDbOutputData.kettleRowToMongo(
                    m_meta.getMongoFields(),
                    getInputRowMeta(),
                    row,
                    this,
                    m_mongoTopLevelStructure);
          } else {
            // specific field update or insert
            insertUpdate =
                MongoDbOutputData.getModifierUpdateObject(
                    m_meta.getMongoFields(),
                    getInputRowMeta(),
                    row,
                    this,
                    m_mongoTopLevelStructure);
            if (log.isDebug()) {
              logDebug(
                  BaseMessages.getString(
                      PKG, "MongoDbOutput.Messages.Debug.ModifierUpdateObject", insertUpdate));
            }
          }

          if (insertUpdate != null) {
            WriteConcern concern = null;

            if (log.getLogLevel().getLevel() >= LogLevel.DETAILED.getLevel()) {
              concern = new WriteConcern(1);
            }
            WriteResult result = null;
            if (concern != null) {
              result =
                  m_data
                      .getCollection()
                      .update(updateQuery, insertUpdate, true, m_meta.getMulti(), concern);
            } else {
              result =
                  m_data.getCollection().update(updateQuery, insertUpdate, true, m_meta.getMulti());
            }

            CommandResult cmd = result.getLastError();
            if (cmd != null && !cmd.ok()) {
              String message = cmd.getErrorMessage();
              logError(
                  BaseMessages.getString(
                      PKG, "MongoDbOutput.Messages.Error.MongoReported", message));
              try {
                cmd.throwOnError();
              } catch (MongoException me) {
                throw new KettleException(me.getMessage(), me);
              }
            }
          }
        }
      } else {
        // straight insert

        DBObject mongoInsert =
            MongoDbOutputData.kettleRowToMongo(
                m_meta.getMongoFields(), getInputRowMeta(), row, this, m_mongoTopLevelStructure);

        if (mongoInsert != null) {
          m_batch.add(mongoInsert);
        }
        if (m_batch.size() == m_batchInsertSize) {
          logBasic(BaseMessages.getString(PKG, "MongoDbOutput.Messages.CommitingABatch"));
          doBatch();
        }
      }
    }

    return true;
  }
示例#12
0
 /**
  * Calls {@link DB#doEval(java.lang.String, java.lang.Object[]) }. If the command is successful,
  * the "retval" field is extracted and returned. Otherwise an exception is thrown.
  *
  * @param code {@code String} representation of JavaScript function
  * @param args arguments to pass to the JavaScript function
  * @return result of the execution
  * @throws MongoException if the operation failed
  * @mongodb.driver.manual reference/method/db.eval/ db.eval()
  */
 public Object eval(final String code, final Object... args) {
   CommandResult result = doEval(code, args);
   result.throwOnError();
   return result.get("retval");
 }
示例#13
0
  @SuppressWarnings("rawtypes")
  public <T> MapreduceResults<T> mapReduce(
      MapreduceType type,
      Query query,
      String map,
      String reduce,
      String finalize,
      Map<String, Object> scopeFields,
      Class<T> outputType) {
    Assert.parametersNotNull("map", map);
    Assert.parameterNotEmpty(map, "map");
    Assert.parametersNotNull("reduce", reduce);
    Assert.parameterNotEmpty(reduce, "reduce");

    QueryImpl<T> qi = (QueryImpl<T>) query;

    DBCollection dbColl = qi.getCollection();
    // TODO remove this after testing.
    if (dbColl == null) dbColl = getCollection(qi.getEntityClass());

    if (log.isTraceEnabled())
      log.info(
          "Executing mapReduce("
              + dbColl.getName()
              + ") with query("
              + qi.toString()
              + ") map("
              + map
              + ") reduce("
              + reduce
              + ") finalize("
              + finalize
              + ") scope("
              + scopeFields
              + ")");

    // TODO replace this with the 2.4 driver impl.
    String outColl = mapr.getCollectionName(outputType);
    BasicDBObjectBuilder bldr =
        BasicDBObjectBuilder.start("mapreduce", mapr.getCollectionName(qi.getEntityClass()));

    switch (type) {
      case REDUCE:
        bldr.push("out").add("reduce", outColl).pop();
        break;
      case MERGE:
        bldr.push("out").add("merge", outColl).pop();
        break;
      case INLINE:
        bldr.push("out").add("inline", 1).pop();
        break;
      default:
        bldr.add("out", outColl);
        break;
    }

    if (qi.getOffset() != 0 || qi.getFieldsObject() != null)
      throw new QueryException(
          "mapReduce does not allow the offset/retrievedFields query options.");

    if (qi.getQueryObject() != null) bldr.add("query", qi.getQueryObject());
    if (qi.getLimit() > 0) bldr.add("limit", qi.getLimit());
    if (qi.getSortObject() != null) bldr.add("sort", qi.getSortObject());

    bldr.add("map", map);
    bldr.add("reduce", reduce);

    if (finalize != null && finalize.length() > 0) bldr.add("finalize", finalize);

    if (scopeFields != null && scopeFields.size() > 0)
      bldr.add("scope", mapr.toMongoObject(null, null, scopeFields));

    DBObject dbObj = bldr.get();
    CommandResult cr = dbColl.getDB().command(dbObj);
    cr.throwOnError();
    MapreduceResults mrRes =
        (MapreduceResults) mapr.fromDBObject(MapreduceResults.class, cr, createCache());

    QueryImpl baseQ = null;
    if (!MapreduceType.INLINE.equals(type))
      baseQ = new QueryImpl(outputType, db.getCollection(mrRes.getOutputCollectionName()), this);
    // TODO Handle inline case and create an iterator/able.

    mrRes.setBits(type, baseQ);
    return mrRes;
  }
示例#14
0
  /**
   * Drops this database, deleting the associated data files. Use with caution.
   *
   * @throws MongoException
   */
  public void dropDatabase() {

    CommandResult res = command(new BasicDBObject("dropDatabase", 1));
    res.throwOnError();
    _mongo._dbs.remove(this.getName());
  }
示例#15
0
  /**
   * Calls {@link DB#doEval(java.lang.String, java.lang.Object[]) }. If the command is successful,
   * the "retval" field is extracted and returned. Otherwise an exception is thrown.
   *
   * @param code @{code String} representation of JavaScript function
   * @param args arguments to pass to the JavaScript function
   * @return result of the execution
   * @throws MongoException
   */
  public Object eval(String code, Object... args) {

    CommandResult res = doEval(code, args);
    res.throwOnError();
    return res.get("retval");
  }