public void addCardInfoToNeo4j() {
DBCursor cursor = cardCollection.find();
DBObject sortObject = new BasicDBObject();
sortObject.put("name", 1);
// cursor = cursor.sort(sortObject);
cursor.addOption(Bytes.QUERYOPTION_NOTIMEOUT);
int count = 0;
while (cursor.hasNext()) {
DBObject cardObject = cursor.next();
String cardName = cardObject.get("name").toString().toUpperCase();
if (cardNodes.get(cardName) == null) {
Node newCardNode = graphDb.createNode();
newCardNode.setProperty("name", cardName);
for (String property : cardObject.keySet()) {
if (!property.equalsIgnoreCase("name")) {
newCardNode.setProperty(property, cardObject.get(property).toString());
}
}
cardNodes.put(cardName, newCardNode);
} else {
Node node = cardNodes.get(cardName);
for (String property : cardObject.keySet()) {
if (!property.equalsIgnoreCase("name")) {
node.setProperty(property, cardObject.get(property).toString());
}
}
}
count++;
System.out.println("Processed " + count + " card");
}
}
@Test(groups = {"basic"})
public void testOptions() {
DBCollection c = _db.getCollection("test");
DBCursor dbCursor = c.find();
assertEquals(0, dbCursor.getOptions());
dbCursor.setOptions(Bytes.QUERYOPTION_TAILABLE);
assertEquals(Bytes.QUERYOPTION_TAILABLE, dbCursor.getOptions());
dbCursor.addOption(Bytes.QUERYOPTION_SLAVEOK);
assertEquals(Bytes.QUERYOPTION_TAILABLE | Bytes.QUERYOPTION_SLAVEOK, dbCursor.getOptions());
dbCursor.resetOptions();
assertEquals(0, dbCursor.getOptions());
}
public static void main(String[] args) {
// TODO Auto-generated method stub
String[] repo = {
"c9s/App-gh", "joshsh/sesametools", "jbr/sibilant", "r1k0/kigen"
}; // "pouchdb/pouchdb"
Mongo mongo = new Mongo(MongoInfo.getMongoServerIp(), 27017);
DB db = mongo.getDB("ghcrawlerV3");
DBCollection pullcache = db.getCollection("pullcacheB");
DBCollection issuecache = db.getCollection("issuecacheB");
DBCollection pulls = db.getCollection("pullscp");
DBCollection issues = db.getCollection("issuescp");
for (int i = 0; i < repo.length; i++) {
pullcache.drop();
issuecache.drop();
PullCrawlerB pullCrawler = new PullCrawlerB();
IssueCrawlerB issueCrawler = new IssueCrawlerB();
issueCrawler.crawlIssues(repo[i]);
pullCrawler.crawlPulls(repo[i]);
DBCursor issuecursor = issuecache.find();
issuecursor.addOption(com.mongodb.Bytes.QUERYOPTION_NOTIMEOUT);
while (issuecursor.hasNext()) {
issues.save(issuecursor.next());
}
issuecursor.close();
DBCursor pullcursor = pullcache.find();
pullcursor.addOption(com.mongodb.Bytes.QUERYOPTION_NOTIMEOUT);
while (pullcursor.hasNext()) {
pulls.save(pullcursor.next());
}
pullcursor.close();
}
}
public void analysisClasses() throws Exception {
// Map<String,Integer> wordMap = new HashMap<String, Integer>();
// String sql = "select description from repotest";
// Connection connection = MysqlInfo.getMysqlConnection();
// PreparedStatement stmt = connection.prepareStatement(sql);
// ResultSet resultSet = stmt.executeQuery();
// while (resultSet.next()) {
// String description = resultSet.getString("description");
// String[] items = description.split(" ");
// for (String item : items) {
// if(!wordMap.containsKey(item)){
// wordMap.put(item, 1);
// }else{
// wordMap.put(item, wordMap.get(item)+1);
// System.out.println(item+"---------------");
// }
// }
// }
Mongo mongo = new Mongo(MongoInfo.getMongoServerIp(), 27017);
DB db = mongo.getDB("ghcrawlerV3");
DBCollection repo = db.getCollection("repository");
DBCursor repos = repo.find();
repos.addOption(com.mongodb.Bytes.QUERYOPTION_NOTIMEOUT);
HashMap<String, Integer> map = new HashMap<String, Integer>();
while (repos.hasNext()) {
DBObject object = repos.next();
if (object.get("description") != null) {
// System.out.println(object.get("description").toString().replaceAll("[^a-zA-Z'0-9]", "
// ").replaceAll("\\s+", " "));
String description =
object
.get("description")
.toString()
.replaceAll("[^a-zA-Z0-9]", " ")
.replaceAll("\\s+", " ");
for (int i = 0; i < description.split(" ").length; i++) {
String word = description.split(" ")[i].toLowerCase();
if (map.containsKey(word)) {
map.put(word, map.get(word) + 1);
} else {
map.put(word, 1);
}
}
}
}
Connection connection = MysqlInfo.getMysqlConnection();
String sqlInsert = "replace into word values(?,?)";
Set<String> words = map.keySet();
connection.setAutoCommit(false);
for (String word : words) {
int count = map.get(word);
if (count < 2) {
continue;
}
PreparedStatement stmt = connection.prepareStatement(sqlInsert);
stmt.setString(1, word);
stmt.setInt(2, count);
stmt.execute();
stmt.close();
}
connection.commit();
connection.close();
}
