public PooledConnection getConnection() throws SQLException {
PooledConnection res = null;
synchronized (connections) {
do {
res = connections.pollFirst();
if (res == null) {
if (numConnections < maxConnections) {
// Break out and create a new connection
++numConnections;
break;
} else {
try {
connections.wait();
} catch (InterruptedException ex) {
ex.printStackTrace();
Thread.currentThread().interrupt();
throw new ESXXException("Failed to get a pooled JDBC connection.", ex);
}
}
}
} while (res == null);
}
if (res == null) {
res = new PooledConnection(uri, props);
}
return res;
}
public void proses() {
System.out.println("Mulai memproses crawl!");
// nanti loop sampai 800
init();
// untuk test, satu kali dulu
for (int i = 0; i < 50; i++) {
try {
// debug, sleep satu menit dulu
// waktu = 1; //DEBUG, nanti dibuang!
int waktu = waktuTunggu();
System.out.println("Waktu tunggu:" + waktu);
System.out.println("Sleep dulu selama:" + waktu);
Thread.sleep(waktu * 1000 * 60); // sleep baru crawl, sleep dalam menit
prosesH();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
close();
}
@Override
public void run() {
while (true) {
String nv = null;
try {
nv = queue.take();
insertar(nv);
RFIDMain.ResultadoInsercionBD(nv, "OK");
Thread.sleep(100);
} catch (InterruptedException e) {
RFIDMain.ResultadoInsercionBD(nv, e.getMessage());
} catch (SQLException e) {
RFIDMain.ResultadoInsercionBD(nv, e.getMessage());
} catch (Exception e) {
RFIDMain.ResultadoInsercionBD(nv, e.getMessage());
break;
}
}
RFIDMain.ResultadoInsercionBD("Ha terminado de forma inesperada el hilo de procesamiento ", "");
}
// ----This function getting nodes from mongo's collection tree_nodes----
// ---------the function insert nodes to neo4j and deletes it from mongo-----------
// ----------------------------------------------------------------------------------------
public void update_tree() {
while (true) {
DBObject obj = new BasicDBObject();
obj.put("in_process", 0); // when other thread processing, 'in_process'=1
DBCursor cursor = this.colltree.find(obj); // find only documents not in use of other thread
// log4j.info(cursor.count() + " documents to process in collection tree nodes");
if (!cursor.hasNext()) { // no documents to process
try {
log4j.info("there is no nodes to process at the moment, going to sleep for 10 seconds");
Thread.currentThread();
Thread.sleep(1000 * 10);
log4j.info("update_tree woke up, continues");
} catch (InterruptedException e) {
log4j.error("InterruptedException caught, at update_all_tweets");
e.printStackTrace();
log4j.error(e);
}
} else // there are documents to process
{
try {
while (cursor.hasNext()) {
DBObject tr = cursor.next();
try {
String parent = tr.get("parent").toString();
String[] sons = tr.get("son").toString().split(","); // make array of sons
neo4j.addNode(
parent, sons, this.log4j); // create nodes and relationships if not exists
this.colltree.remove(tr); // remove document from collection
} catch (ConcurrentModificationException e) {
// log4j.error(e);
log4j.warn(e);
}
}
} catch (MongoException e) {
log4j.error(e);
}
}
}
}
@SuppressWarnings("unchecked")
private void multiPutOperation(DataStoreOperation op) {
Set<Future<Pair<String, Boolean>>> set = new HashSet<Future<Pair<String, Boolean>>>();
Map<String, Boolean> mapResults = new HashMap<String, Boolean>();
String tablename = ((MultiPutOperation) op).getTableName();
if (tablename.equals("friendsTimeLine")) {
Map<String, Pair<String, String>> map = ((MultiPutOperation) op).getMapKeyToDataAndTags();
Set<String> keys = map.keySet();
Iterator<String> iter = keys.iterator();
String follower;
while (iter.hasNext()) {
follower = iter.next();
Pair<String, String> pair = map.get(follower);
String date = pair.getFirst();
String tweetId = pair.getSecond();
Callable<Pair<String, Boolean>> callable =
new TMultiPutMySqlFriendsTimeline(this.borrowClient(), follower, date, tweetId);
Future<Pair<String, Boolean>> future = this.pool.submit(callable);
set.add(future);
}
for (Future<Pair<String, Boolean>> future : set) {
try {
Pair<String, Boolean> aux = future.get();
mapResults.put(aux.getFirst(), aux.getSecond());
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
}
}
((MultiPutOperation) op).setMapResults(mapResults);
op.notifyListeners();
} else {
if (tablename.equals("users")) {
Map<String, Pair<Map<String, String>, List<String>>> map =
((MultiPutOperation) op).getMapKeyToDataAndTags();
Set<String> keys = map.keySet();
Iterator<String> iter = keys.iterator();
String userid;
while (iter.hasNext()) {
userid = iter.next();
Pair<Map<String, String>, List<String>> hash = map.get(userid);
Map<String, String> userFields = hash.getFirst();
Callable<Pair<String, Boolean>> callable =
new TMultiPutMySqlUsers(this.borrowClient(), userid, userFields);
Future<Pair<String, Boolean>> future = this.pool.submit(callable);
set.add(future);
}
for (Future<Pair<String, Boolean>> future : set) {
try {
Pair<String, Boolean> aux = future.get();
mapResults.put(aux.getFirst(), aux.getSecond());
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
}
}
((MultiPutOperation) op).setMapResults(mapResults);
op.notifyListeners();
}
}
}
