public void callData() {
workingDir = System.getProperty("user.dir");
String fileName =
workingDir
+ File.separator
+ "data"
+ File.separator
+ "log"
+ File.separator
+ "karaf.log";
WatchedDir = workingDir + File.separator + "data" + File.separator + "log";
try {
LOG.info("log enabled ...");
Thread.sleep(10000);
infile = new File(theNewestFile.toString());
LOG.info("karaf file ..." + theNewestFile);
Thread.sleep(1000);
LOG.info("parsing karaf file ...");
Thread.sleep(9000);
Scanner scanner = new Scanner(infile);
while (scanner.hasNext()) {
line = scanner.nextLine();
if (line.contains("DEBUG") && line.contains("from")) {
String phrase1 = line;
String delims = "[,]+";
String[] tokens = phrase1.split(delims);
String phrase2 = line;
String delims2 = "[|]+";
String[] tokens2 = phrase2.split(delims2);
time = tokens[0];
attempt = tokens2[5];
String phrase3 = line;
String[] parts = phrase3.split(" ");
srcIP = parts[parts.length - 1];
usecpluginAAAstore.addData(time, srcIP, attempt);
LOG.info("Information stored in datastore is" + time + " " + srcIP + " " + attempt);
}
}
PublishNotif publishNotif = new PublishNotif();
publishNotif.setdataBroker(dataBroker);
publishNotif.Notify();
} catch (Exception e) {
e.printStackTrace();
}
}
/*----------------------------------------------------------------------------------------------------------------------*/
public static void main(String[] arg) throws Exception {
/*----------------------------------environment variable initialization-------------------------------------------------*/
System.out.println("Initializing environment variables...");
try {
Thread.sleep(2000); // 2 seconds
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
}
exit = false;
/*----------------------------------------------------------------------------------------------------------------------*/
jdbcInit();
System.out.println("--->Welcome to the PIKFLIX<---");
loginToDatabase();
/*---------------------------------------------super loop---------------------------------------------------------------*/
while (!exit) {
mainMenu();
}
/*----------------------------------------------------------------------------------------------------------------------*/
System.out.println("\nbye...");
}
public static void runTest3() throws Exception {
boolean exceptionOccured = false;
try {
Context ctx = cache.getJNDIContext();
DataSource ds1 = null;
DataSource ds2 = null;
ds1 = (DataSource) ctx.lookup("java:/XAPooledDataSource");
ds2 = (DataSource) ctx.lookup("java:/SimpleDataSource");
ds2.getConnection();
UserTransaction utx = (UserTransaction) ctx.lookup("java:/UserTransaction");
utx.begin();
utx.setTransactionTimeout(2);
ds1.getConnection();
Thread.sleep(4000);
try {
utx.commit();
} catch (Exception e) {
exceptionOccured = true;
}
if (!exceptionOccured)
fail("Exception (Transaction-Time-Out)did not occur although was supposed" + "to occur");
} catch (Exception e) {
fail("failed in runTest3 due to " + e);
}
}
public static void runTest1() throws Exception {
boolean exceptionOccured = false;
try {
Context ctx = cache.getJNDIContext();
DataSource ds1 = null;
DataSource ds2 = null;
ds1 = (DataSource) ctx.lookup("java:/XAPooledDataSource");
ds2 = (DataSource) ctx.lookup("java:/SimpleDataSource");
ds2.getConnection();
ds1 = (DataSource) ctx.lookup("java:/XAPooledDataSource");
UserTransaction utx = (UserTransaction) ctx.lookup("java:/UserTransaction");
utx.begin();
ds1.getConnection();
Thread.sleep(8000);
try {
utx.commit();
} catch (Exception e) {
exceptionOccured = true;
}
if (!exceptionOccured)
fail("Exception did not occur on commit although was supposed" + "occur");
} catch (Exception e) {
getLogWriter().fine("Exception caught in runTest1 due to : " + e);
fail("failed in runTest1 due to " + e);
}
}
public static File prereleaseSetup(
FungeneProps skelProps, String releaseName, String releaseDescription)
throws JAXBException, IOException {
PrereleaseHelper loader = new PrereleaseHelper(skelProps);
int nextRelease = loader.createRelease(releaseName, releaseDescription);
skelProps.setReleaseNo(nextRelease);
if (skelProps.getWorkDir().exists()) {
System.out.print("Release directory '" + skelProps.getWorkDir() + "' exists, nuking in ");
for (int index = 5; index >= 0; index--) {
System.out.print(index + "..");
try {
Thread.sleep(1000);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
System.out.println();
FileUtils.forceDelete(skelProps.getWorkDir());
}
skelProps.getWorkDir().mkdir();
File ret = new File(skelProps.getWorkDir(), "release_" + nextRelease + ".xml");
FungeneProps.writeProps(skelProps, ret);
return ret;
}
/**
* Execute Task locally and wait
*
* @param cmd command
* @return execution info
*/
public String executeLocal(String cmd) {
log.config(cmd);
if (m_task != null && m_task.isAlive()) m_task.interrupt();
m_task = new Task(cmd);
m_task.start();
StringBuffer sb = new StringBuffer();
while (true) {
// Give it a bit of time
try {
Thread.sleep(500);
} catch (InterruptedException ioe) {
log.log(Level.SEVERE, cmd, ioe);
}
// Info to user
sb.append(m_task.getOut())
.append("\n-----------\n")
.append(m_task.getErr())
.append("\n-----------");
// Are we done?
if (!m_task.isAlive()) break;
}
log.config("done");
return sb.toString();
} // executeLocal
public synchronized Connection getConnection() {
for (; ; ) {
try {
String url = cfg.get("jdbc.url");
String user = cfg.get("jdbc.user");
String pass = cfg.get("jdbc.password");
return DriverManager.getConnection(url, user, pass);
} catch (SQLException e) {
Logger.log(new LogEvent(this, "sql-connection", e));
try {
Thread.sleep(2000);
} catch (InterruptedException ex) {
}
}
}
}
@Test
public void testApplication() throws Exception {
LocalMode lma = LocalMode.newInstance();
Configuration conf = new Configuration(false);
conf.set("dt.operator.Unique.prop.tableName", "Test_Lookup_Cache");
conf.set("dt.operator.Unique.prop.store.dbUrl", "jdbc:hsqldb:mem:test;sql.syntax_mys=true");
conf.set("dt.operator.Unique.prop.store.dbDriver", "org.hsqldb.jdbcDriver");
lma.prepareDAG(new Application(), conf);
lma.cloneDAG();
LocalMode.Controller lc = lma.getController();
lc.setHeartbeatMonitoringEnabled(false);
lc.runAsync();
long now = System.currentTimeMillis();
while (System.currentTimeMillis() - now < 15000) {
Thread.sleep(1000);
}
lc.shutdown();
}
public void run() {
while (true) {
try {
Thread.sleep(25);
} catch (InterruptedException e) {
}
count++;
if (count == 20) {
count = 0;
eraseMazeMan();
drawMazeMan(mouth);
mouth = !mouth;
}
if (count == 10) {
eraseMazeMan();
drawMazeMan(mouth);
mouth = !mouth;
}
}
}
void run() throws Exception {
MySQL db = new MySQL();
db.setName("MySQL-Test");
db.setHost("localhost");
db.setDatabaseName("designstate");
db.setUsername("DesignState");
db.setPassword("zero0");
Connection con1 = db.connect();
Connection con2 = db.connect();
db.release(con1);
db.release(con2);
Thread.sleep(2000);
con1 = db.connect();
PreparedStatement s = con1.prepareStatement("show variables like 'wait_timeout'");
ResultSet r = s.executeQuery();
r.next();
long wait = r.getInt("Value");
{
} // System.out.println(wait);
}
@Override
public void run() {
while (true) {
String nv = null;
try {
nv = queue.take();
insertar(nv);
RFIDMain.ResultadoInsercionBD(nv, "OK");
Thread.sleep(100);
} catch (InterruptedException e) {
RFIDMain.ResultadoInsercionBD(nv, e.getMessage());
} catch (SQLException e) {
RFIDMain.ResultadoInsercionBD(nv, e.getMessage());
} catch (Exception e) {
RFIDMain.ResultadoInsercionBD(nv, e.getMessage());
break;
}
}
RFIDMain.ResultadoInsercionBD("Ha terminado de forma inesperada el hilo de procesamiento ", "");
}
public void proses() {
System.out.println("Mulai memproses crawl!");
// nanti loop sampai 800
init();
// untuk test, satu kali dulu
for (int i = 0; i < 50; i++) {
try {
// debug, sleep satu menit dulu
// waktu = 1; //DEBUG, nanti dibuang!
int waktu = waktuTunggu();
System.out.println("Waktu tunggu:" + waktu);
System.out.println("Sleep dulu selama:" + waktu);
Thread.sleep(waktu * 1000 * 60); // sleep baru crawl, sleep dalam menit
prosesH();
} catch (InterruptedException e) {
e.printStackTrace();
}
}
close();
}
// ----This function getting nodes from mongo's collection tree_nodes----
// ---------the function insert nodes to neo4j and deletes it from mongo-----------
// ----------------------------------------------------------------------------------------
public void update_tree() {
while (true) {
DBObject obj = new BasicDBObject();
obj.put("in_process", 0); // when other thread processing, 'in_process'=1
DBCursor cursor = this.colltree.find(obj); // find only documents not in use of other thread
// log4j.info(cursor.count() + " documents to process in collection tree nodes");
if (!cursor.hasNext()) { // no documents to process
try {
log4j.info("there is no nodes to process at the moment, going to sleep for 10 seconds");
Thread.currentThread();
Thread.sleep(1000 * 10);
log4j.info("update_tree woke up, continues");
} catch (InterruptedException e) {
log4j.error("InterruptedException caught, at update_all_tweets");
e.printStackTrace();
log4j.error(e);
}
} else // there are documents to process
{
try {
while (cursor.hasNext()) {
DBObject tr = cursor.next();
try {
String parent = tr.get("parent").toString();
String[] sons = tr.get("son").toString().split(","); // make array of sons
neo4j.addNode(
parent, sons, this.log4j); // create nodes and relationships if not exists
this.colltree.remove(tr); // remove document from collection
} catch (ConcurrentModificationException e) {
// log4j.error(e);
log4j.warn(e);
}
}
} catch (MongoException e) {
log4j.error(e);
}
}
}
}
public File getTheNewestFile() {
workingDir = System.getProperty("user.dir");
String fileName =
workingDir
+ File.separator
+ "data"
+ File.separator
+ "log"
+ File.separator
+ "karaf.log";
WatchedDir = workingDir + File.separator + "data" + File.separator + "log";
try {
LOG.info("log enabled ...");
File dir = new File(WatchedDir);
Thread.sleep(10000);
files = dir.listFiles();
Arrays.sort(files, LastModifiedFileComparator.LASTMODIFIED_COMPARATOR);
for (int i = 0; i < files.length; i++) {
file = files[i];
karafFiles += file.getName();
}
LOG.info("karaf files:" + karafFiles);
Arrays.sort(files, LastModifiedFileComparator.LASTMODIFIED_REVERSE);
for (int i = 0; i < files.length; i++) {
file = files[i];
RevOrder_ofFiles += file.getName();
}
LOG.info("list of karaf File in reverse order" + " " + RevOrder_ofFiles);
theNewestFile = files[0];
LOG.info("Latest karaf File" + theNewestFile);
} catch (Exception e) {
e.printStackTrace();
}
return theNewestFile;
}
/**
* Creates the database, deleting data that existed previously.
*
* <p>Won't actually remove symbol's tables if they aren't in the set SYMBOLS.
*
* @param pauseDuration - this is how long we sleep (in ms) for before hitting yahoo again
* @see SYMBOLS
*/
public static void createDatabase(String[] symbolArray, int pauseDuration) {
try {
Connection conn = initialize();
Statement stat = conn.createStatement();
try {
stat.executeUpdate("DROP TABLE IF EXISTS " + "listofsymbols" + ";");
stat.executeUpdate(
"CREATE TABLE IF NOT EXISTS " + "listofsymbols" + " (" + "'symbolname'" + ");");
} catch (Exception e) {
System.out.println("failed stat.executeUpdate");
System.out.println(e.getMessage());
}
for (int i = 0; i < symbolArray.length; i++) {
addSymbol(symbolArray[i]);
if (pauseDuration != 0) {
Thread.sleep(pauseDuration);
}
}
conn.close();
} catch (Exception e) {
e.printStackTrace();
}
}
public static void runTest2() throws Exception {
boolean exceptionOccured1 = false;
boolean exceptionOccured2 = false;
try {
Context ctx = cache.getJNDIContext();
DataSource ds1 = null;
DataSource ds2 = null;
ds1 = (DataSource) ctx.lookup("java:/XAPooledDataSource");
ds2 = (DataSource) ctx.lookup("java:/SimpleDataSource");
ds2.getConnection();
ds1.getConnection();
ds1.getConnection();
ds1.getConnection();
ds1.getConnection();
ds1.getConnection();
UserTransaction utx = (UserTransaction) ctx.lookup("java:/UserTransaction");
utx.begin();
try {
ds1.getConnection();
Thread.sleep(8000);
} catch (SQLException e) {
exceptionOccured1 = true;
}
try {
utx.commit();
} catch (Exception e) {
exceptionOccured2 = true;
}
if (!exceptionOccured1)
fail("Exception (Login-Time-Out)did not occur although was supposed" + "to occur");
if (exceptionOccured2)
fail("Exception did occur on commit, although was not supposed" + "to occur");
} catch (Exception e) {
fail("failed in runTest2 due to " + e);
}
}
public static boolean kill() {
int j = 0;
Thread meMySelfI = Thread.currentThread();
do {
/* iter across the thread group, killing all members. */
shutdown = true;
Thread list[] = new Thread[tg.activeCount()];
// get all members of the group (including submembers)
int i = tg.enumerate(list);
// no members means that we have gracefully suceeded
if (i == 0) return true;
// if some of the threads do IO during the shut down they will
// need time to accomplish the IO. So, I give it to 'em
// after the first attempt.
if (j > 0)
try {
meMySelfI.sleep(500);
} catch (Exception e) {
}
;
// try to shudown each thread in the group
while (i-- > 0) {
FlickrFtpd tftp = (FlickrFtpd) list[i];
tftp.interrupt(); // first, do it politely
meMySelfI.yield(); // give 'em time to respond
tftp.forceClose(); // second, use a big hammer
meMySelfI.yield(); // give 'em time to respond
}
} while (j++ <= 3);
return false;
}
/**
* Method to move files from HDFS to local filesystem
*
* <p>localPath: Path on the machines filesystem fs:FileSystem object from HDFS pathList:List of
* paths for files that might need to be backed up size:max size in bytes to be backed up
*
* <p>ReturnsDate of the last files backed up if reached size limit, else, zero
*/
public long backupFiles(
String localPath, String preservePath, FileSystem fs, ArrayList<Path> pathList, long size) {
Path fsPath;
long tmpSize = 0;
long tmpDate = 0;
// Start iterating over all paths
for (Path hdfsPath : pathList) {
try {
long nFileSize = fs.getContentSummary(hdfsPath).getLength();
tmpSize = tmpSize + nFileSize;
if ((tmpSize <= size) || (size == 0)) {
FileStatus stat = fs.getFileStatus(hdfsPath);
System.err.println(
"File "
+ hdfsPath.toUri().getPath()
+ " "
+ nFileSize
+ " bytes, "
+ "perms: "
+ stat.getOwner()
+ "/"
+ stat.getGroup()
+ ", "
+ stat.getPermission().toString());
tmpDate = stat.getModificationTime() / 1000;
String sFsPath = localPath + hdfsPath.toUri().getPath();
fsPath = new Path(sFsPath);
File f = new File(sFsPath);
// COMMENTED OUT: until a few backup cycles run
// and the mtime gets in fact set on all copied
// files.
//
// ignore it if the file exists and has the same mtime
// if (f.exists() && f.isFile() && f.lastModified() == stat.getModificationTime())
// {
// System.out.println("no need to backup " + f.toString() + ", mtime matches hdfs");
// continue;
// }
if (false == m_bDryRun) {
// check if we need to back up the local file
// (not directory), if it already exists.
if (f.exists() && f.isFile()) {
// ignore files with substrings in the
// no-preserve file
if (true == doPreserveFile(sFsPath)) {
// move it to the backup path
String sNewPath = preservePath + hdfsPath.toUri().getPath();
File newFile = new File(sNewPath);
// create directory structure for new file?
if (false == newFile.getParentFile().exists()) {
if (false == newFile.getParentFile().mkdirs()) {
System.err.println("Failed to mkdirs " + newFile.getParentFile().toString());
System.exit(1);
}
}
// rename existing file to new location
if (false == f.renameTo(newFile)) {
System.err.println(
"Failed to renameTo " + f.toString() + " to " + newFile.toString());
System.exit(1);
}
System.out.println("preserved " + f.toString() + " into " + newFile.toString());
} else {
System.out.println("skipped preservation of " + f.toString());
}
}
// copy from hdfs to local filesystem
fs.copyToLocalFile(hdfsPath, fsPath);
// set the mtime to match hdfs file
f.setLastModified(stat.getModificationTime());
// compare checksums on both files
compareChecksums(fs, hdfsPath, sFsPath);
}
// don't print the progress after every file -- go
// by at least 1% increments
long nPercentDone = (long) (100 * tmpSize / m_nTotalBytes);
if (nPercentDone > m_nLastPercentBytesDone) {
System.out.println(
"progress: copied "
+ prettyPrintBytes(tmpSize)
+ ", "
+ nPercentDone
+ "% done"
+ ", tstamp="
+ tmpDate);
m_nLastPercentBytesDone = nPercentDone;
}
if (m_nSleepSeconds > 0) {
try {
Thread.sleep(1000 * m_nSleepSeconds);
} catch (Exception e2) {
// ignore
}
}
} else {
return tmpDate;
}
} catch (IOException e) {
System.err.println("FATAL ERROR: Something wrong with the file");
System.err.println(e);
System.out.println(tmpDate);
System.exit(1);
return 0;
}
}
return 0;
}
public static void main(String[] argv) {
try {
ComboPooledDataSource cpds = new ComboPooledDataSource();
cpds.setJdbcUrl(argv[0]);
cpds.setUser(argv[1]);
cpds.setPassword(argv[2]);
cpds.setMinPoolSize(5);
cpds.setAcquireIncrement(5);
cpds.setMaxPoolSize(20);
System.err.println("Initial...");
display(cpds);
Thread.sleep(2000);
HashSet hs = new HashSet();
for (int i = 0; i < 20; ++i) {
Connection c = cpds.getConnection();
hs.add(c);
System.err.println("Adding (" + (i + 1) + ") " + c);
display(cpds);
Thread.sleep(1000);
// if (i == 9)
// {
// //System.err.println("hardReset()ing");
// //cpds.hardReset();
// System.err.println("softReset()ing");
// cpds.softReset();
// }
}
int count = 0;
for (Iterator ii = hs.iterator(); ii.hasNext(); ) {
Connection c = ((Connection) ii.next());
System.err.println("Removing " + ++count);
ii.remove();
try {
c.getMetaData().getTables(null, null, "PROBABLYNOT", new String[] {"TABLE"});
} catch (Exception e) {
System.err.println(e);
System.err.println();
continue;
} finally {
c.close();
}
Thread.sleep(2000);
display(cpds);
}
System.err.println(
"Closing data source, \"forcing\" garbage collection, and sleeping for 5 seconds...");
cpds.close();
System.gc();
System.err.println("Main Thread: Sleeping for five seconds!");
Thread.sleep(5000);
// System.gc();
// Thread.sleep(5000);
System.err.println("Bye!");
} catch (Exception e) {
e.printStackTrace();
}
}
private void executeParsedFile(NameListMessage nameListMessage, ArrayList cmds) {
BufferedWriter buffer = null;
int connectionId = nameListMessage.connectionId;
boolean commaSeparated = nameListMessage.commaSeparated;
// get java.sql.Connection from connectionId
Connection connection = ConnectionService.getConnection(connectionId);
if (connection == null) {
nameListMessage.errorMessage = kConnectionClosed;
} else {
try {
Thread th = null;
buffer = new BufferedWriter(writer);
// if reader exists, create reading thread
if (reader != null) {
final ObjectNameReader objectReader = new ObjectNameReader(nameListMessage.nameList);
// This thread will read stream until meets EOL (ie. when
// writer is closed)
th =
new Thread() {
public void run() {
objectReader.readInputFile(reader);
}
};
} // end if
// if no reader (offline reverse), never start the reading
// thread
// (because output is written down directly to the disk without
// processing in counterpart).
if (reader != null) th.start();
// execute each SQL statement
// we skip objects that don't need to be reversed
Iterator iter = cmds.iterator();
int index = -1;
while (iter.hasNext()) {
String cmd = (String) iter.next();
index++;
// check if this statement id is marked as <to skip>
if (nameListMessage.ignoredStatementIds != null
&& nameListMessage.ignoredStatementIds.contains(new Integer(index))) {
if (commaSeparated) {
buffer.flush();
buffer.newLine();
}
continue;
}
if (writer != null) {
if (cmd != null) {
try {
execute(connection, cmd, buffer, commaSeparated);
} catch (SQLException e) {
if (Debug.isDebug()) e.printStackTrace();
nameListMessage.errorMessage =
nameListMessage.errorMessage == null
? e.toString()
: nameListMessage.errorMessage.concat("\n" + e.toString()); // NOT
// LOCALIZABLE,
// escape
// code
}
}
if (commaSeparated) {
buffer.flush();
buffer.newLine();
}
}
}
}
/*
* catch (SQLException ex) { if (Debug.isDebug()) ex.printStackTrace();
* nameListMessage.errorMessage = nameListMessage.errorMessage == null ? ex.toString() :
* nameListMessage.errorMessage.concat("\n" + ex.toString()); //NOT LOCALIZABLE, escape
* code }
*/
catch (IOException ex) {
if (Debug.isDebug()) ex.printStackTrace();
nameListMessage.errorMessage =
nameListMessage.errorMessage == null
? ex.toString()
: nameListMessage.errorMessage.concat(
"\n" + ex.toString()); // NOT LOCALIZABLE, escape
// code
}
// close writer: the reading thread terminates.
try {
// WARNING: we must make a pause after closing the buffer
// otherwise
// the nameList variable won't be well initialized...strange!
// [FG]
buffer.close();
Thread.sleep(400);
} catch (IOException ex) {
} catch (InterruptedException ex) {
}
} // end if
} // end parseSqlFile()
// ----This function gets the raw json from raw_data collection----
// ---------the function separate the tweets and inserts it to all_tweets collection and calling
// all other data processing methods-----------
// ----------------------------------------------------------------------------------------
public void update_all_tweets(final double num_of_slots, final double max_time_frame_hours)
throws MongoException, ParseException {
log4j.info("starting update_all_tweets function");
String res = new String();
Integer countelements = 0;
while (true) {
// DBCursor cursor = this.collrd.find();
while (this.collrd.count() < 1) { // no documents to process
try {
log4j.info("there is no raw data at the moment, going to sleep for 10 seconds");
Thread.currentThread();
Thread.sleep(1000 * 10);
log4j.info("woke up, continues");
// cursor = this.collrd.find();
} catch (InterruptedException e) {
log4j.error("InterruptedException caught, at update_all_tweets");
log4j.error(e);
}
}
DBCursor cursor = this.collrd.find(); // get all documents from raw_data collection
try {
while (cursor.hasNext()) {
DBObject currdoc = cursor.next();
log4j.info("getting a document from the raw data db");
Object results = currdoc.get("results"); // result - json array of tweets
try {
res = results.toString();
} catch (NullPointerException e) {
res = "";
}
Object obj = JSONValue.parse(res);
log4j.info("making an array from the jsons tweets");
JSONArray array = (JSONArray) obj; // make an array of tweets
// JSONParser parser = new JSONParser();
try {
if (res != "") { // if there are tweets
@SuppressWarnings("rawtypes")
Iterator iterArray = array.iterator();
log4j.info("iterating over array tweets");
try {
while (iterArray.hasNext()) {
Object current = iterArray.next();
final DBObject dbObject =
(DBObject) JSON.parse(current.toString()); // parse all tweet data to json
countelements++;
// System.out.println("element number" + countelements.toString());
dbObject.put("max_id", currdoc.get("max_id")); // add max_id to tweet data
dbObject.put("query", currdoc.get("query")); // add query word to tweet data
dbObject.put(
"query_time", currdoc.get("query_time")); // add query time to tweet data
dbObject.put("query_time_string", currdoc.get("query_time_string"));
dbObject.put(
"text",
"@"
+ dbObject.get("from_user").toString()
+ ": "
+ dbObject.get("text").toString()); // add user_name to beginning of text
dbObject.put("count", 1L); // add appearance counter to tweet data
log4j.info("inserting tweet id: " + dbObject.get("id").toString());
try {
DBObject object = new BasicDBObject();
object.put(
"id", Long.parseLong(dbObject.get("id").toString())); // object to search
DBObject newobject = collat.findOne(object);
if (newobject != null) {
newobject.put(
"count",
Long.parseLong(newobject.get("count").toString())
+ 1); // update counter if id already exists
collat.update(object, newobject);
}
} catch (NullPointerException e) {
}
collat.insert(dbObject);
// collrd.findAndRemove(currdoc);
// log4j.info("calling function update_search_terms");
// final String text = "@" + dbObject.get("from_user").toString() + ": " +
// dbObject.get("text").toString();
/*Thread t10=new Thread(new Runnable(){
public void run(){
UpdateTweetCounterId(Long.parseLong(dbObject.get("id").toString()));
}
});*/
Thread t11 =
new Thread(
new Runnable() {
public void run() {
update_search_terms(
dbObject.get("text").toString(),
num_of_slots,
max_time_frame_hours,
dbObject.get("query").toString());
}
});
Thread t12 =
new Thread(
new Runnable() {
public void run() {
rate_user(
Long.parseLong(dbObject.get("from_user_id").toString()),
dbObject.get("from_user").toString(),
max_time_frame_hours);
// UpdateUserRate((long)num_of_slots,slot_time_millis,Long.parseLong(dbObject.get("from_user_id").toString()) , dbObject.get("from_user").toString() ,(long)0);
}
});
Thread t13 =
new Thread(
new Runnable() {
public void run() {
String quer = dbObject.get("query").toString();
quer = quer.replaceAll("%40", "@");
quer = quer.replaceAll("%23", "#");
long id =
(long)
(Double.parseDouble(dbObject.get("query_time").toString())
* 1000);
String idplus = dbObject.get("id").toString() + "," + id;
SearchResultId(quer, idplus);
}
});
// t10.start();
t11.start();
t12.start();
t13.start();
try {
log4j.info("Waiting for threads to finish.");
// t10.join();
t11.join();
t12.join();
t13.join();
} catch (InterruptedException e) {
log4j.error("Main thread (update_all_tweets) Interrupted");
}
}
} catch (Exception e) {
log4j.error(e);
e.printStackTrace();
}
}
} catch (NullPointerException e) {
log4j.error(e);
log4j.info("NullPointerException caught, at update_all_tweets");
}
log4j.info("removing processed document from raw_data collection");
try {
this.collrd.remove(currdoc);
} catch (Exception e) {
log4j.debug(e);
}
}
} catch (MongoException e) {
log4j.error(e);
}
}
}
/**
* Main function of the DOTS. Read the config file,create dotslogging,start performance client,
* check db connection,create keyboard thread. Create new testcase according the cpu_usage and
* connections, Write test summary and exception
*/
public static void main(String[] args) {
int sleepInterval = 1;
int waitInterval = 0;
int averageUsage = 0;
boolean reached = false;
Calendar currentTime;
/* Load DotsConfig value */
loadConfig(args);
/* Create the instance of DotsLogging */
DotsLogging logging = new DotsLogging(TESTCASENAME);
DotsLogging.logMessage("Database Opensource Test Suite V1.0");
DotsLogging.logMessage("Start to run JDBC API Test Case - " + TESTCASENAME);
DotsLogging.logSummary("Start to run JDBC API Test Case - " + TESTCASENAME);
if (!DotsConfig.RUN_AUTO) {
System.out.println("\nDatabase Opensource Test Suite V1.0");
System.out.println("\nStart to run JDBC API test case - " + TESTCASENAME);
}
DotsLogging.logMessage("Initialization started");
if (!DotsConfig.RUN_AUTO) {
System.out.println("\nTo stop running the test case, type STOP then press Enter\n");
}
/* Start performance monitor thread */
startPerfCtl();
/* Check database connection */
checkDBConn();
DotsLogging.logMessage("Testing Database Connections ...OK");
/* Start summary writer thread */
DotsSummary summary = new DotsSummary();
summary.start();
DotsLogging.logMessage("Starting Summary Writer ... OK ");
/* Set begin and end time */
Calendar endTime = Calendar.getInstance();
DotsSummary.startTime = System.currentTimeMillis();
endTime.add(Calendar.SECOND, DotsConfig.DURATION * 60);
/* Start keyboard thread */
if (!DotsConfig.RUN_AUTO) {
startKeyboard();
DotsLogging.logMessage("Starting Keyboard Thread ... OK");
}
boolean firstRun = true; // used for LOB manipulation
String lobFileName[] = new String[10]; // used for LOB manipulation
averageUsage = DotsConfig.CPU_USAGE;
int targetCpu = DotsConfig.CPU_TARGET;
try {
while (true) {
currentTime = Calendar.getInstance();
if (currentTime.after(endTime)) break;
if (DotsConfig.TERMINATION) break;
/* Check if need create a new database access thread*/
if ((DotsConfig.AUTO_MODE && (averageUsage < targetCpu))
|| (!DotsConfig.AUTO_MODE
&& (DotsConfig.THRDGRP.activeCount() < DotsConfig.CONNECTIONS))) {
Connection conn =
createConnection(
DotsConfig.DRIVER_CLASS_NAME,
DotsConfig.URL,
DotsConfig.DB_UID,
DotsConfig.DB_PASSWD);
if (conn != null) {
/* 5 second is a unit*/
sleepInterval = DotsConfig.INTERVAL * 12;
/* Create DB access thread */
switch (TESTCASENAME.charAt(4)) {
case '1':
if (TESTCASENAME.equals("BTCJ1")) {
BTCJ1 btcj1 = new BTCJ1(conn);
Thread thrdb1 = new Thread(DotsConfig.THRDGRP, btcj1);
thrdb1.start();
} else {
ATCJ1 atcj1 = new ATCJ1(conn);
Thread thrda1 = new Thread(DotsConfig.THRDGRP, atcj1);
thrda1.start();
}
break;
case '2':
if (TESTCASENAME.equals("BTCJ2")) {
BTCJ2 btcj2 = new BTCJ2(conn);
Thread thrdb2 = new Thread(DotsConfig.THRDGRP, btcj2);
thrdb2.start();
} else {
ATCJ2 atcj2 = new ATCJ2(conn);
Thread thrda2 = new Thread(DotsConfig.THRDGRP, atcj2);
thrda2.start();
}
break;
case '3':
BTCJ3 btcj3 = new BTCJ3(conn);
Thread thrdb3 = new Thread(DotsConfig.THRDGRP, btcj3);
thrdb3.start();
break;
case '4':
BTCJ4 btcj4 = new BTCJ4(conn);
Thread thrdb4 = new Thread(DotsConfig.THRDGRP, btcj4);
thrdb4.start();
break;
case '5':
BTCJ5 btcj5 = new BTCJ5(conn);
Thread thrdb5 = new Thread(DotsConfig.THRDGRP, btcj5);
thrdb5.start();
break;
case '6':
if (firstRun) {
for (int i = 0; i < 10; i++) {
lobFileName[i] = DotsGenerator.mdClob(DotsGenerator.mdInt(10, 100));
}
firstRun = false;
}
BTCJ6 btcj6 = new BTCJ6(conn, lobFileName);
Thread thrdb6 = new Thread(DotsConfig.THRDGRP, btcj6);
thrdb6.start();
break;
case '7':
if (firstRun) {
for (int i = 0; i < 10; i++) {
lobFileName[i] = DotsGenerator.mdBlob(DotsGenerator.mdInt(10, 100));
}
firstRun = false;
}
BTCJ7 btcj7 = new BTCJ7(conn, lobFileName);
Thread thrdb7 = new Thread(DotsConfig.THRDGRP, btcj7);
thrdb7.start();
break;
case '8':
BTCJ8 btcj8 = new BTCJ8(conn);
Thread thrdb8 = new Thread(DotsConfig.THRDGRP, btcj8);
thrdb8.start();
break;
}
} else {
/* If createConnection failed,sleepInterval will double,
but the maximum of sleepInterval is 1 hour */
sleepInterval = sleepInterval * 2;
if (sleepInterval > 720) sleepInterval = 720;
}
try {
averageUsage = 0;
/* Wait 5*sleepInterval seconds,check TERMINATION per 5 second */
for (int j = 0; j < sleepInterval; j++) {
averageUsage += DotsConfig.CPU_USAGE;
Thread.sleep(5000);
if ((j % 12) == 0)
DotsLogging.logMessage(
"Active Threads = "
+ DotsConfig.THRDGRP.activeCount()
+ " Average CPU Usage = "
+ DotsConfig.CPU_USAGE
+ "%");
if (DotsConfig.TERMINATION) break;
currentTime = Calendar.getInstance();
if (currentTime.after(endTime)) {
DotsConfig.TERMINATION = true;
break;
}
}
/* Calculate the average usage of this sleepInterval */
averageUsage = averageUsage / sleepInterval;
DotsLogging.logMessage(
sleepInterval / 12 + " Minutes Average CPU Usage = " + averageUsage + "%");
} catch (Exception e) {
DotsLogging.logException("Dots.main: " + e);
}
} else {
/* When CPU_TARGET is reached,DOTS will not create new thread until
average usage is less than CPU_TARGET-10 */
if (DotsConfig.AUTO_MODE && !reached) {
reached = true;
targetCpu = DotsConfig.CPU_TARGET - 10;
DotsLogging.logMessage("CPU Target " + DotsConfig.CPU_TARGET + "% is achieved now.");
}
/* Wait 5 minutes then check create connection condition */
try {
averageUsage = 0;
for (int i = 0; i < 60; i++) {
averageUsage += DotsConfig.CPU_USAGE;
Thread.sleep(5000);
if ((i % 12) == 0)
DotsLogging.logMessage(
"Active Threads = "
+ DotsConfig.THRDGRP.activeCount()
+ " Average CPU Usage = "
+ DotsConfig.CPU_USAGE
+ "%");
if (DotsConfig.TERMINATION) break;
currentTime = Calendar.getInstance();
if (currentTime.after(endTime)) {
DotsConfig.TERMINATION = true;
break;
}
}
averageUsage = averageUsage / 60;
DotsLogging.logMessage("5 Minutes Average CPU Usage = " + averageUsage + "%");
} catch (Exception e) {
DotsLogging.logException("Dots.main: " + e);
}
}
}
} catch (Throwable t) {
DotsLogging.logException("Dots.main: " + t);
System.out.println("Dots.main:" + t);
}
/* DURATION is reached or TERMINATION is set by other threads */
DotsConfig.TERMINATION = true;
DotsLogging.logMessage("Dots is Terminating ...");
/* Clear temporary file used by testcase 6&7 */
if ((TESTCASENAME.charAt(4) == '6') || (TESTCASENAME.charAt(4) == '7')) {
DotsLogging.logMessage("Cleaning temporary file ...");
if (!DotsConfig.RUN_AUTO) {
System.out.println("Cleaning temporary file ...");
}
try {
for (int j = 0; j < 10; j++) {
File tmpFile = new File(lobFileName[j]);
if (tmpFile.exists()) tmpFile.delete();
}
} catch (Exception e) {
DotsLogging.logException("Clean temporary file error.");
}
}
/* Write summary file and wait DB thread to exit */
try {
if (!DotsConfig.RUN_AUTO) {
System.out.println("\nWriting summary file ...");
}
summary.writeSummaryFile();
summary.canExit = true;
DotsPerf.canExit = true;
DotsLogging.logSummary("Waiting for the active threads to exit ......");
if (!DotsConfig.RUN_AUTO) {
System.out.println("\nWaiting for the active threads to exit ......");
}
while (DotsConfig.THRDGRP.activeCount() > 0) {
waitInterval++;
Thread.sleep(1000);
if (!DotsConfig.RUN_AUTO) {
System.out.println("Current active thread number is " + DotsConfig.THRDGRP.activeCount());
}
if (waitInterval > 60) break; // Max wait time is 1 minute
}
} catch (InterruptedException e) {
DotsLogging.logException("Dots.main:" + e);
}
DotsLogging.logSummary("All Database Access Threads exit.");
DotsLogging.close();
if (!DotsConfig.RUN_AUTO) {
System.out.println("DOTS closed successfully.");
}
System.exit(0);
}