@Override
public Sequence eval(Sequence[] args, Sequence contextSequence) throws XPathException {
try {
// Verify clientid and get client
String mongodbClientId = args[0].itemAt(0).getStringValue();
MongodbClientStore.getInstance().validate(mongodbClientId);
MongoClient client = MongodbClientStore.getInstance().get(mongodbClientId);
// Get parameters
String dbname = args[1].itemAt(0).getStringValue();
// Retrieve database
DB db = client.getDB(dbname);
// Retrieve collection names
Set<String> collectionNames = db.getCollectionNames();
// Storage for results
ValueSequence valueSequence = new ValueSequence();
// Iterate over collection names ; only pairs of collections
// with names ending .chunks and .files are buckets
for (String collName : collectionNames) {
if (collName.endsWith(".chunks")) {
String bucketName = StringUtils.removeEnd(collName, ".chunks");
if (collectionNames.contains(bucketName + ".files")) {
valueSequence.add(new StringValue(bucketName));
}
}
}
return valueSequence;
} catch (XPathException ex) {
LOG.error(ex.getMessage(), ex);
throw new XPathException(this, ex.getMessage(), ex);
} catch (MongoException ex) {
LOG.error(ex.getMessage(), ex);
throw new XPathException(this, GridfsModule.GRFS0002, ex.getMessage());
} catch (Throwable ex) {
LOG.error(ex.getMessage(), ex);
throw new XPathException(this, GridfsModule.GRFS0003, ex.getMessage());
}
// return Sequence.EMPTY_SEQUENCE;
}
protected void doBatch() throws KettleException {
WriteConcern concern = null;
if (log.getLogLevel().getLevel() >= LogLevel.DETAILED.getLevel()) {
concern = new WriteConcern(1);
}
WriteResult result = null;
if (concern != null) {
result = m_data.getCollection().insert(m_batch, concern);
} else {
result = m_data.getCollection().insert(m_batch);
}
CommandResult cmd = result.getLastError();
if (cmd != null && !cmd.ok()) {
String message = cmd.getErrorMessage();
logError(BaseMessages.getString(PKG, "MongoDbOutput.Messages.Error.MongoReported", message));
try {
cmd.throwOnError();
} catch (MongoException me) {
throw new KettleException(me.getMessage(), me);
}
}
m_batch.clear();
}
/**
* Deletes a Database with the specified name in mongo database to which user is connected to.
*
* @param dbName Name of Database to be deleted
* @return Success if deleted else throws Exception
* @throws DatabaseException throw super type of
* UndefinedDatabaseException,DeleteDatabaseException
* @throws ValidationException throw super type of EmptyDatabaseNameException
*/
public String dropDb(String dbName) throws DatabaseException, ValidationException {
if (dbName == null) {
throw new DatabaseException(ErrorCodes.DB_NAME_EMPTY, "Database name is null");
}
if (dbName.equals("")) {
throw new DatabaseException(ErrorCodes.DB_NAME_EMPTY, "Database Name Empty");
}
try {
boolean dbPresent = getDbList().contains(dbName);
if (!dbPresent) {
throw new DatabaseException(
ErrorCodes.DB_DOES_NOT_EXISTS, "DB with name '" + dbName + "' DOES NOT EXIST");
}
mongoInstance.dropDatabase(dbName);
// newly added line
getDbList().remove(dbName);
} catch (MongoException e) {
throw new DatabaseException(ErrorCodes.DB_DELETION_EXCEPTION, e.getMessage());
}
return "Successfully dropped DB '" + dbName + "'. The page will reload now.";
}
/**
* Gets the result of the command
*
* @param dbName Name of Database
* @param command Name of the Command to be executed
* @param queryStr query to be performed. In case of empty query {} return all
* @param keys Keys to be present in the resulted docs.
* @param limit Number of docs to show.
* @param skip Docs to skip from the front.
* @return Result of executing the command.
* @throws DatabaseException throw super type of UndefinedDatabaseException
*/
public JSONObject executeQuery(
String dbName,
String command,
String queryStr,
String keys,
String sortBy,
int limit,
int skip)
throws DatabaseException, JSONException, InvalidMongoCommandException {
if (dbName == null) {
throw new DatabaseException(ErrorCodes.DB_NAME_EMPTY, "Database name is null");
}
if (dbName.equals("")) {
throw new DatabaseException(ErrorCodes.DB_NAME_EMPTY, "Database Name Empty");
}
// List<String> databaseNames = getDbList();
// if (!databaseNames.contains(dbName)) {
// throw new DatabaseException(ErrorCodes.DB_DOES_NOT_EXISTS,
// "DB with name [" + dbName + "]DOES_NOT_EXIST");
// }
try {
MongoDatabase db = mongoInstance.getDatabase(dbName);
return DatabaseQueryExecutor.executeQuery(db, command, queryStr, keys, sortBy, limit, skip);
} catch (MongoException e) {
throw new DatabaseException(ErrorCodes.QUERY_EXECUTION_EXCEPTION, e.getMessage());
}
}
/**
* Method invoked when a {@link MongoSession} needs to be created.
*
* @param username the username to use for authentication. NOTE: Please use a dummy user if you
* have disabled Mongo authentication
* @param password the password to use for authentication. NOTE: Please use a dummy password if
* you have disabled Mongo authentication
* @param database Name of the database
* @return the newly created {@link MongoSession}
* @throws org.mule.api.ConnectionException
*/
@Connect
public void connect(
@ConnectionKey String username,
@Password String password,
@Optional @Default("test") String database)
throws ConnectionException {
DB db = null;
try {
MongoOptions options = new MongoOptions();
if (connectionsPerHost != null) {
options.connectionsPerHost = connectionsPerHost;
}
if (threadsAllowedToBlockForConnectionMultiplier != null) {
options.threadsAllowedToBlockForConnectionMultiplier =
threadsAllowedToBlockForConnectionMultiplier;
}
if (maxWaitTime != null) {
options.maxWaitTime = maxWaitTime;
}
if (connectTimeout != null) {
options.connectTimeout = connectTimeout;
}
if (socketTimeout != null) {
options.socketTimeout = socketTimeout;
}
if (autoConnectRetry != null) {
options.autoConnectRetry = autoConnectRetry;
}
if (slaveOk != null) {
options.slaveOk = slaveOk;
}
if (safe != null) {
options.safe = safe;
}
if (w != null) {
options.w = w;
}
if (wtimeout != null) {
options.wtimeout = wtimeout;
}
if (fsync != null) {
options.fsync = fsync;
}
if (database != null) {
this.database = database;
}
mongo = getOrCreateMongoInstance(host, port, options);
db = getDatabase(mongo, username, password, database);
} catch (MongoException me) {
throw new ConnectionException(ConnectionExceptionCode.UNKNOWN, null, me.getMessage());
} catch (UnknownHostException e) {
throw new ConnectionException(ConnectionExceptionCode.UNKNOWN_HOST, null, e.getMessage());
}
this.client = new MongoClientImpl(db);
}
@Override
public void connectDb(String keyword) {
try {
initMongoDB();
items = db.getCollection(keyword);
BasicDBObject index = new BasicDBObject("tweet_ID", 1).append("unique", true);
// items.ensureIndex(index, new BasicDBObject("unique", true));
items.createIndex(index);
} catch (MongoException ex) {
System.out.println("MongoException :" + ex.getMessage());
}
}
public static MongoClient getMongoClient() {
if (mongo == null) {
try {
mongo = new MongoClient(host, port);
LOGGER.debug("New Mongo created with [{}] and [{}]", host, port);
} catch (UnknownHostException e) {
LOGGER.error(e.getMessage());
} catch (MongoException e2) {
LOGGER.error(e2.getMessage());
}
}
return mongo;
}
public void connectdb(String keyword) {
try {
// on constructor load initialize MongoDB and load collection
initMongoDB();
items = db.getCollection(keyword);
// make the tweet_ID unique in the database
BasicDBObject index = new BasicDBObject("tweet_ID", 1);
items.ensureIndex(index, new BasicDBObject("unique", true));
} catch (MongoException ex) {
System.out.println("MongoException :" + ex.getMessage());
}
}
@Override
protected void annotateInjectorExceptions(Collection<Message> messages) {
super.annotateInjectorExceptions(messages);
for (Message message : messages) {
if (message.getCause() instanceof MongoException) {
MongoException e = (MongoException) message.getCause();
LOG.error(
UI.wallString(
"Unable to connect to MongoDB. Is it running and the configuration correct?\n"
+ "Details: "
+ e.getMessage()));
System.exit(-1);
}
}
}
/**
* Gets the list of databases present in mongo database to which user is connected to.
*
* @return List of All Databases present in MongoDb
* @throws DatabaseException If any error while getting database list.
*/
public List<String> getDbList() throws DatabaseException {
try {
Set<String> authenticatedDbNames = connectionDetails.getAuthenticatedDbNames();
if (!connectionDetails.isAdminLogin()) {
return new ArrayList<String>(authenticatedDbNames);
}
List<String> dbs = new ArrayList<String>();
MongoCursor<String> dbsCursor = mongoInstance.listDatabaseNames().iterator();
while (dbsCursor.hasNext()) {
dbs.add(dbsCursor.next());
}
return dbs;
} catch (MongoException m) {
throw new DatabaseException(ErrorCodes.GET_DB_LIST_EXCEPTION, m.getMessage());
}
}
public MongoDB() {
try {
mongo = new MongoClient(HOST, PORT);
db = mongo.getDB(DB_NAME);
table = db.getCollection(COLLECTION_NAME);
dataLoader = new MongoDBDataLoader(table);
} catch (final UnknownHostException unknownHostException) {
System.err.println(unknownHostException.getMessage());
System.err.println(unknownHostException.getStackTrace());
} catch (final MongoException mongoException) {
System.err.println(mongoException.getMessage());
System.err.println(mongoException.getStackTrace());
}
}
@Override
public List<DependencyStatus> status() {
// note failures are tested manually for now, if you make changes test
// things still work
// TODO TEST add tests exercising failures
final String version;
try {
final CommandResult bi = gfs.getDB().command("buildInfo");
version = bi.getString("version");
} catch (MongoException e) {
LoggerFactory.getLogger(getClass()).error("Failed to connect to MongoDB", e);
return Arrays.asList(
new DependencyStatus(
false, "Couldn't connect to MongoDB: " + e.getMessage(), "GridFS", "Unknown"));
}
return Arrays.asList(new DependencyStatus(true, "OK", "GridFS", version));
}
public MongoDatabaseConnection(DatabaseConfig config) throws DatabaseConnectionException {
Mongo m = null;
String hostName = config.get("host");
String dbName = config.get("database");
String collectionName = config.get("collection");
try {
m = new Mongo(hostName);
} catch (UnknownHostException e) {
throw new DatabaseConnectionException(e.getMessage());
} catch (MongoException e) {
throw new DatabaseConnectionException(e.getMessage());
}
this.database = m.getDB(dbName);
this.collection = database.getCollection(collectionName);
}
/**
* Return Stats of a particular Database in mongo to which user is connected to.
*
* @param dbName Name of Database
* @return Array of JSON Objects each containing a key value pair in Db Stats.
* @throws JSONException While parsing JSON
* @throws DatabaseException Error while performing this operation
* @throws ValidationException throw super type of EmptyDatabaseNameException
*/
public JSONArray getDbStats(String dbName)
throws DatabaseException, ValidationException, JSONException {
if (dbName == null) {
throw new DatabaseException(ErrorCodes.DB_NAME_EMPTY, "Database name is null");
}
if (dbName.equals("")) {
throw new DatabaseException(ErrorCodes.DB_NAME_EMPTY, "Database Name Empty");
}
JSONArray dbStats = new JSONArray();
try {
List<String> dbList = getDbList();
boolean dbPresent = dbList.contains(dbName);
if (!dbPresent) {
throw new DatabaseException(
ErrorCodes.DB_DOES_NOT_EXISTS, "DB with name '" + dbName + "' DOES NOT EXIST");
}
MongoDatabase db = mongoInstance.getDatabase(dbName);
Document stats = db.runCommand(new Document("dbStats", "1"));
Set<String> keys = stats.keySet();
Iterator<String> keyIterator = keys.iterator();
while (keyIterator.hasNext()) {
JSONObject temp = new JSONObject();
String key = keyIterator.next();
temp.put("Key", key);
String value = stats.get(key).toString();
temp.put("Value", value);
String type = stats.get(key).getClass().toString();
temp.put("Type", type.substring(type.lastIndexOf('.') + 1));
dbStats.put(temp);
}
} catch (MongoException m) {
throw new DatabaseException(ErrorCodes.GET_DB_STATS_EXCEPTION, m.getMessage());
}
return dbStats;
}
@Test
public void insertDuplicateTest() throws Exception {
DB db = getDb();
BasicDBObject doc = new BasicDBObject();
doc.put("username", "insertduplicate");
WriteResult result = db.getCollection("users").insert(doc);
assertNull(result.getError());
// check we've created the collection
Set<String> colls = db.getCollectionNames();
assertTrue(colls.contains("users"));
// iterate the collection to ensure we can retrieve the object
doc = new BasicDBObject();
doc.put("username", "insertduplicate");
String message = null;
try {
result = db.getCollection("users").insert(doc);
} catch (MongoException me) {
message = me.getMessage();
}
assertNotNull(message);
assertTrue(
message.contains(
"Entity users requires that property named username be unique, value of insertduplicate exists"));
}
@Test
public void testCreateRetryWithError() {
Repository<Entity> mockRepo = Mockito.spy(repository);
Map<String, Object> studentBody = buildTestStudentEntity();
Map<String, Object> studentMetaData = new HashMap<String, Object>();
int noOfRetries = 5;
Mockito.doThrow(new MongoException("Test Exception"))
.when(((MongoEntityRepository) mockRepo))
.internalCreate("student", null, studentBody, studentMetaData, "student");
Mockito.doCallRealMethod()
.when(mockRepo)
.createWithRetries("student", null, studentBody, studentMetaData, "student", noOfRetries);
try {
mockRepo.createWithRetries(
"student", null, studentBody, studentMetaData, "student", noOfRetries);
} catch (MongoException ex) {
assertEquals(ex.getMessage(), "Test Exception");
}
Mockito.verify((MongoEntityRepository) mockRepo, Mockito.times(noOfRetries))
.internalCreate("student", null, studentBody, studentMetaData, "student");
}
/**
* Creates a Database with the specified name in mongo database to which user is connected to.
*
* @param dbName Name of Database to be created
* @return Success if Created else throws Exception
* @throws DatabaseException throw super type of
* DuplicateDatabaseException,InsertDatabaseException
* @throws ValidationException throw super type of EmptyDatabaseNameException
*/
public String createDb(String dbName) throws DatabaseException, ValidationException {
if (dbName == null) {
throw new DatabaseException(ErrorCodes.DB_NAME_EMPTY, "Database name is null");
}
if (dbName.equals("")) {
throw new DatabaseException(ErrorCodes.DB_NAME_EMPTY, "Database Name Empty");
}
try {
boolean dbAlreadyPresent = getDbList().contains(dbName);
if (dbAlreadyPresent) {
throw new DatabaseException(
ErrorCodes.DB_ALREADY_EXISTS, "DB with name '" + dbName + "' ALREADY EXISTS");
}
// mongoInstance.getDatabase(dbName).listCollectionNames();
mongoInstance.getDatabase(dbName).createCollection("temp");
connectionDetails.addToAuthenticatedDbNames(dbName);
} catch (MongoException e) {
throw new DatabaseException(ErrorCodes.DB_CREATION_EXCEPTION, e.getMessage());
}
return "Created DB with name '" + dbName + "'";
}
public void doPost(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
String email = request.getParameter("email");
String password = request.getParameter("password");
String verifypassword = request.getParameter("verifypassword");
Map<String, String> myResponse = new HashMap<String, String>();
PrintWriter out = response.getWriter();
if (email.matches(
"^[_A-Za-z0-9-\\+]+(\\.[_A-Za-z0-9-]+)*@"
+ "[A-Za-z0-9-]+(\\.[A-Za-z0-9]+)*(\\.[A-Za-z]{2,})$")) // make sure email is properly
// formatted
{
try {
MongoURI mongoURI = new MongoURI(System.getenv("MONGOHQ_URL"));
DB db = mongoURI.connectDB(); // instance of databse
db.authenticate(mongoURI.getUsername(), mongoURI.getPassword()); // authenticates d
// Set<string> accounts = db.getCollectionName("accounts");
// Mongo mongo = new Mongo("localhost", 27017); //creates new instance of mongo
// DB db = mongo.getDB("fourup"); //gets fourup database
DBCollection accounts = db.getCollection("accounts"); // creates collection for accounts
BasicDBObject query = new BasicDBObject(); // creates a basic object named query
query.put("email", email); // sets email to email
DBCursor cursor = accounts.find(query);
if (cursor.size() > 0) // check if email has already been registered
{
myResponse.put("Status", "Error");
myResponse.put("Error", "Account already exists using this email address.");
} else // since email doesn't currently exist in DB, go ahead and register user
{
if (password.equals(
verifypassword)) // check that both of the passwords entered match each other
{
BasicDBObject document = new BasicDBObject();
int salt = getSalt();
String hpass = passwrdHash(password, salt);
document.put("email", email);
document.put("salt", salt);
document.put("password", hpass); // this is where we need to hash the password
accounts.insert(document);
myResponse.put("Status", "Sucess");
myResponse.put("Sucess", "Account has been Created");
AccountObject user = new AccountObject(email, hpass);
// set session
HttpSession session = request.getSession();
session.setAttribute("currentUser", email);
// return cookie
Cookie cookie = new Cookie("fourupCookie", email); // add the login information here
response.addCookie(cookie);
// redirect to homepage
String message = "this is a test";
myResponse.put("html", "<html></html>");
response.setContentType("application/json");
response.setStatus(HttpServletResponse.SC_OK);
// response.sendRedirect("index.html"); //should add check to index page for cookie with
// login information
} else {
myResponse.put("Status", "Failed");
myResponse.put("Failed", "Passwords do not match.");
}
}
} catch (MongoException e) {
out.write(e.getMessage());
}
} else {
myResponse.put("Status", "Invalid");
myResponse.put(
"Invalid", "The email address has not been entered correctly."); // should output error
}
String strResponse = new Gson().toJson(myResponse);
response.getWriter().write(strResponse);
response.getWriter().close();
}
public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
Object[] row = getRow();
if (row == null) {
// no more output
// check any remaining buffered objects
if (m_batch != null && m_batch.size() > 0) {
doBatch();
}
// INDEXING - http://www.mongodb.org/display/DOCS/Indexes
// Indexing is computationally expensive - it needs to be
// done after all data is inserted and done in the BACKGROUND.
// UNIQUE indexes (prevent duplicates on the
// keys in the index) and SPARSE indexes (don't index docs that
// don't have the key field) - current limitation is that SPARSE
// indexes can only have a single field
List<MongoDbOutputMeta.MongoIndex> indexes = m_meta.getMongoIndexes();
if (indexes != null && indexes.size() > 0) {
logBasic(BaseMessages.getString(PKG, "MongoDbOutput.Messages.ApplyingIndexOpps"));
m_data.applyIndexes(indexes, log, m_meta.getTruncate());
}
disconnect();
setOutputDone();
return false;
}
if (first) {
first = false;
m_batchInsertSize = 100;
String batchInsert = environmentSubstitute(m_meta.getBatchInsertSize());
if (!Const.isEmpty(batchInsert)) {
m_batchInsertSize = Integer.parseInt(batchInsert);
}
m_batch = new ArrayList<DBObject>(m_batchInsertSize);
// output the same as the input
m_data.setOutputRowMeta(getInputRowMeta());
m_mongoTopLevelStructure =
MongoDbOutputData.checkTopLevelConsistency(m_meta.m_mongoFields, this);
if (m_mongoTopLevelStructure == MongoDbOutputData.MongoTopLevel.INCONSISTENT) {
throw new KettleException(
BaseMessages.getString(PKG, "MongoDbOutput.Messages.Error.InconsistentMongoTopLevel"));
}
// first check our incoming fields against our meta data for fields to insert
RowMetaInterface rmi = getInputRowMeta();
List<MongoDbOutputMeta.MongoField> mongoFields = m_meta.getMongoFields();
List<String> notToBeInserted = new ArrayList<String>();
for (int i = 0; i < rmi.size(); i++) {
ValueMetaInterface vm = rmi.getValueMeta(i);
boolean ok = false;
for (MongoDbOutputMeta.MongoField field : mongoFields) {
String mongoMatch = environmentSubstitute(field.m_incomingFieldName);
if (vm.getName().equals(mongoMatch)) {
ok = true;
break;
}
}
if (!ok) {
notToBeInserted.add(vm.getName());
}
}
if (notToBeInserted.size() == rmi.size()) {
throw new KettleException(
BaseMessages.getString(PKG, "MongoDbOutput.Messages.Error.NotInsertingAnyFields"));
}
if (notToBeInserted.size() > 0) {
StringBuffer b = new StringBuffer();
for (String s : notToBeInserted) {
b.append(s).append(" ");
}
logBasic(
BaseMessages.getString(PKG, "MongoDbOutput.Messages.FieldsNotToBeInserted"),
b.toString());
}
// init mongo fields
for (MongoDbOutputMeta.MongoField m : m_meta.getMongoFields()) {
m.init(this);
}
// check truncate
if (m_meta.getTruncate()) {
try {
logBasic(BaseMessages.getString(PKG, "MongoDbOutput.Messages.TruncatingCollection"));
m_data.getCollection().drop();
// re-establish the collection
String collection = environmentSubstitute(m_meta.getCollection());
m_data.createCollection(collection);
m_data.setCollection(m_data.getDB().getCollection(collection));
} catch (Exception m) {
disconnect();
throw new KettleException(m.getMessage(), m);
}
}
}
if (!isStopped()) {
if (m_meta.getUpsert()) {
/*DBObject updateQuery = MongoDbOutputData.getQueryObject(m_meta.getMongoFields(),
getInputRowMeta(), row, getParentVariableSpace(), m_mongoTopLevelStructure); */
DBObject updateQuery =
MongoDbOutputData.getQueryObject(m_meta.getMongoFields(), getInputRowMeta(), row, this);
if (log.isDebug()) {
logDebug(
BaseMessages.getString(
PKG, "MongoDbOutput.Messages.Debug.QueryForUpsert", updateQuery));
}
if (updateQuery != null) {
// i.e. we have some non-null incoming query field values
DBObject insertUpdate = null;
// get the record to update the match with
if (!m_meta.getModifierUpdate()) {
// complete record replace or insert
insertUpdate =
MongoDbOutputData.kettleRowToMongo(
m_meta.getMongoFields(),
getInputRowMeta(),
row,
this,
m_mongoTopLevelStructure);
} else {
// specific field update or insert
insertUpdate =
MongoDbOutputData.getModifierUpdateObject(
m_meta.getMongoFields(),
getInputRowMeta(),
row,
this,
m_mongoTopLevelStructure);
if (log.isDebug()) {
logDebug(
BaseMessages.getString(
PKG, "MongoDbOutput.Messages.Debug.ModifierUpdateObject", insertUpdate));
}
}
if (insertUpdate != null) {
WriteConcern concern = null;
if (log.getLogLevel().getLevel() >= LogLevel.DETAILED.getLevel()) {
concern = new WriteConcern(1);
}
WriteResult result = null;
if (concern != null) {
result =
m_data
.getCollection()
.update(updateQuery, insertUpdate, true, m_meta.getMulti(), concern);
} else {
result =
m_data.getCollection().update(updateQuery, insertUpdate, true, m_meta.getMulti());
}
CommandResult cmd = result.getLastError();
if (cmd != null && !cmd.ok()) {
String message = cmd.getErrorMessage();
logError(
BaseMessages.getString(
PKG, "MongoDbOutput.Messages.Error.MongoReported", message));
try {
cmd.throwOnError();
} catch (MongoException me) {
throw new KettleException(me.getMessage(), me);
}
}
}
}
} else {
// straight insert
DBObject mongoInsert =
MongoDbOutputData.kettleRowToMongo(
m_meta.getMongoFields(), getInputRowMeta(), row, this, m_mongoTopLevelStructure);
if (mongoInsert != null) {
m_batch.add(mongoInsert);
}
if (m_batch.size() == m_batchInsertSize) {
logBasic(BaseMessages.getString(PKG, "MongoDbOutput.Messages.CommitingABatch"));
doBatch();
}
}
}
return true;
}
