Example #1
0
  public void callData() {
    workingDir = System.getProperty("user.dir");
    String fileName =
        workingDir
            + File.separator
            + "data"
            + File.separator
            + "log"
            + File.separator
            + "karaf.log";
    WatchedDir = workingDir + File.separator + "data" + File.separator + "log";
    try {

      LOG.info("log enabled ...");
      Thread.sleep(10000);
      infile = new File(theNewestFile.toString());
      LOG.info("karaf file ..." + theNewestFile);
      Thread.sleep(1000);
      LOG.info("parsing karaf file ...");
      Thread.sleep(9000);
      Scanner scanner = new Scanner(infile);

      while (scanner.hasNext()) {
        line = scanner.nextLine();
        if (line.contains("DEBUG") && line.contains("from")) {

          String phrase1 = line;
          String delims = "[,]+";
          String[] tokens = phrase1.split(delims);
          String phrase2 = line;
          String delims2 = "[|]+";
          String[] tokens2 = phrase2.split(delims2);
          time = tokens[0];
          attempt = tokens2[5];
          String phrase3 = line;
          String[] parts = phrase3.split(" ");
          srcIP = parts[parts.length - 1];
          usecpluginAAAstore.addData(time, srcIP, attempt);
          LOG.info("Information stored in datastore is" + time + " " + srcIP + " " + attempt);
        }
      }
      PublishNotif publishNotif = new PublishNotif();
      publishNotif.setdataBroker(dataBroker);
      publishNotif.Notify();

    } catch (Exception e) {

      e.printStackTrace();
    }
  }
Example #2
0
  /*----------------------------------------------------------------------------------------------------------------------*/
  public static void main(String[] arg) throws Exception {

    /*----------------------------------environment variable initialization-------------------------------------------------*/
    System.out.println("Initializing environment variables...");
    try {
      Thread.sleep(2000); // 2 seconds
    } catch (InterruptedException ex) {
      Thread.currentThread().interrupt();
    }
    exit = false;
    /*----------------------------------------------------------------------------------------------------------------------*/

    jdbcInit();

    System.out.println("--->Welcome to the PIKFLIX<---");

    loginToDatabase();

    /*---------------------------------------------super loop---------------------------------------------------------------*/
    while (!exit) {
      mainMenu();
    }
    /*----------------------------------------------------------------------------------------------------------------------*/
    System.out.println("\nbye...");
  }
 public static void runTest3() throws Exception {
   boolean exceptionOccured = false;
   try {
     Context ctx = cache.getJNDIContext();
     DataSource ds1 = null;
     DataSource ds2 = null;
     ds1 = (DataSource) ctx.lookup("java:/XAPooledDataSource");
     ds2 = (DataSource) ctx.lookup("java:/SimpleDataSource");
     ds2.getConnection();
     UserTransaction utx = (UserTransaction) ctx.lookup("java:/UserTransaction");
     utx.begin();
     utx.setTransactionTimeout(2);
     ds1.getConnection();
     Thread.sleep(4000);
     try {
       utx.commit();
     } catch (Exception e) {
       exceptionOccured = true;
     }
     if (!exceptionOccured)
       fail("Exception (Transaction-Time-Out)did not occur although was supposed" + "to occur");
   } catch (Exception e) {
     fail("failed in runTest3 due to " + e);
   }
 }
 public static void runTest1() throws Exception {
   boolean exceptionOccured = false;
   try {
     Context ctx = cache.getJNDIContext();
     DataSource ds1 = null;
     DataSource ds2 = null;
     ds1 = (DataSource) ctx.lookup("java:/XAPooledDataSource");
     ds2 = (DataSource) ctx.lookup("java:/SimpleDataSource");
     ds2.getConnection();
     ds1 = (DataSource) ctx.lookup("java:/XAPooledDataSource");
     UserTransaction utx = (UserTransaction) ctx.lookup("java:/UserTransaction");
     utx.begin();
     ds1.getConnection();
     Thread.sleep(8000);
     try {
       utx.commit();
     } catch (Exception e) {
       exceptionOccured = true;
     }
     if (!exceptionOccured)
       fail("Exception did not occur on commit although was supposed" + "occur");
   } catch (Exception e) {
     getLogWriter().fine("Exception caught in runTest1 due to : " + e);
     fail("failed in runTest1 due to " + e);
   }
 }
  public static File prereleaseSetup(
      FungeneProps skelProps, String releaseName, String releaseDescription)
      throws JAXBException, IOException {
    PrereleaseHelper loader = new PrereleaseHelper(skelProps);

    int nextRelease = loader.createRelease(releaseName, releaseDescription);
    skelProps.setReleaseNo(nextRelease);

    if (skelProps.getWorkDir().exists()) {
      System.out.print("Release directory '" + skelProps.getWorkDir() + "' exists, nuking in ");
      for (int index = 5; index >= 0; index--) {
        System.out.print(index + "..");

        try {
          Thread.sleep(1000);
        } catch (Exception e) {
          throw new RuntimeException(e);
        }
      }
      System.out.println();

      FileUtils.forceDelete(skelProps.getWorkDir());
    }

    skelProps.getWorkDir().mkdir();

    File ret = new File(skelProps.getWorkDir(), "release_" + nextRelease + ".xml");
    FungeneProps.writeProps(skelProps, ret);
    return ret;
  }
Example #6
0
  /**
   * Execute Task locally and wait
   *
   * @param cmd command
   * @return execution info
   */
  public String executeLocal(String cmd) {
    log.config(cmd);
    if (m_task != null && m_task.isAlive()) m_task.interrupt();

    m_task = new Task(cmd);
    m_task.start();

    StringBuffer sb = new StringBuffer();
    while (true) {
      //  Give it a bit of time
      try {
        Thread.sleep(500);
      } catch (InterruptedException ioe) {
        log.log(Level.SEVERE, cmd, ioe);
      }
      //  Info to user
      sb.append(m_task.getOut())
          .append("\n-----------\n")
          .append(m_task.getErr())
          .append("\n-----------");

      //  Are we done?
      if (!m_task.isAlive()) break;
    }
    log.config("done");
    return sb.toString();
  } //	executeLocal
Example #7
0
 public synchronized Connection getConnection() {
   for (; ; ) {
     try {
       String url = cfg.get("jdbc.url");
       String user = cfg.get("jdbc.user");
       String pass = cfg.get("jdbc.password");
       return DriverManager.getConnection(url, user, pass);
     } catch (SQLException e) {
       Logger.log(new LogEvent(this, "sql-connection", e));
       try {
         Thread.sleep(2000);
       } catch (InterruptedException ex) {
       }
     }
   }
 }
  @Test
  public void testApplication() throws Exception {
    LocalMode lma = LocalMode.newInstance();
    Configuration conf = new Configuration(false);
    conf.set("dt.operator.Unique.prop.tableName", "Test_Lookup_Cache");
    conf.set("dt.operator.Unique.prop.store.dbUrl", "jdbc:hsqldb:mem:test;sql.syntax_mys=true");
    conf.set("dt.operator.Unique.prop.store.dbDriver", "org.hsqldb.jdbcDriver");

    lma.prepareDAG(new Application(), conf);
    lma.cloneDAG();
    LocalMode.Controller lc = lma.getController();
    lc.setHeartbeatMonitoringEnabled(false);
    lc.runAsync();

    long now = System.currentTimeMillis();
    while (System.currentTimeMillis() - now < 15000) {
      Thread.sleep(1000);
    }
    lc.shutdown();
  }
Example #9
0
 public void run() {
   while (true) {
     try {
       Thread.sleep(25);
     } catch (InterruptedException e) {
     }
     count++;
     if (count == 20) {
       count = 0;
       eraseMazeMan();
       drawMazeMan(mouth);
       mouth = !mouth;
     }
     if (count == 10) {
       eraseMazeMan();
       drawMazeMan(mouth);
       mouth = !mouth;
     }
   }
 }
Example #10
0
 void run() throws Exception {
   MySQL db = new MySQL();
   db.setName("MySQL-Test");
   db.setHost("localhost");
   db.setDatabaseName("designstate");
   db.setUsername("DesignState");
   db.setPassword("zero0");
   Connection con1 = db.connect();
   Connection con2 = db.connect();
   db.release(con1);
   db.release(con2);
   Thread.sleep(2000);
   con1 = db.connect();
   PreparedStatement s = con1.prepareStatement("show variables like 'wait_timeout'");
   ResultSet r = s.executeQuery();
   r.next();
   long wait = r.getInt("Value");
   {
   } // System.out.println(wait);
 }
Example #11
0
  @Override
  public void run() {

    while (true) {
      String nv = null;
      try {
        nv = queue.take();
        insertar(nv);
        RFIDMain.ResultadoInsercionBD(nv, "OK");
        Thread.sleep(100);
      } catch (InterruptedException e) {
        RFIDMain.ResultadoInsercionBD(nv, e.getMessage());
      } catch (SQLException e) {
        RFIDMain.ResultadoInsercionBD(nv, e.getMessage());
      } catch (Exception e) {
        RFIDMain.ResultadoInsercionBD(nv, e.getMessage());
        break;
      }
    }
    RFIDMain.ResultadoInsercionBD("Ha terminado de forma inesperada el hilo de procesamiento ", "");
  }
Example #12
0
  public void proses() {
    System.out.println("Mulai memproses crawl!");

    // nanti loop sampai 800
    init();
    // untuk test, satu kali dulu
    for (int i = 0; i < 50; i++) {
      try {
        // debug, sleep satu menit dulu
        // waktu = 1; //DEBUG, nanti dibuang!
        int waktu = waktuTunggu();
        System.out.println("Waktu tunggu:" + waktu);
        System.out.println("Sleep dulu selama:" + waktu);
        Thread.sleep(waktu * 1000 * 60); // sleep baru crawl, sleep dalam menit
        prosesH();
      } catch (InterruptedException e) {
        e.printStackTrace();
      }
    }
    close();
  }
Example #13
0
  // ----This function getting nodes from mongo's collection tree_nodes----
  // ---------the function insert nodes to neo4j and deletes it from mongo-----------
  // ----------------------------------------------------------------------------------------
  public void update_tree() {
    while (true) {
      DBObject obj = new BasicDBObject();
      obj.put("in_process", 0); // when other thread processing, 'in_process'=1
      DBCursor cursor = this.colltree.find(obj); // find only documents not in use of other thread
      // log4j.info(cursor.count() + " documents to process in collection tree nodes");
      if (!cursor.hasNext()) { // no documents to process
        try {
          log4j.info("there is no nodes to process at the moment, going to sleep for 10 seconds");
          Thread.currentThread();
          Thread.sleep(1000 * 10);
          log4j.info("update_tree woke up, continues");
        } catch (InterruptedException e) {
          log4j.error("InterruptedException caught, at update_all_tweets");
          e.printStackTrace();
          log4j.error(e);
        }
      } else // there are documents to process
      {
        try {
          while (cursor.hasNext()) {
            DBObject tr = cursor.next();
            try {
              String parent = tr.get("parent").toString();
              String[] sons = tr.get("son").toString().split(","); // make array of sons
              neo4j.addNode(
                  parent, sons, this.log4j); // create nodes and relationships if not exists
              this.colltree.remove(tr); // remove document from collection

            } catch (ConcurrentModificationException e) {
              // log4j.error(e);
              log4j.warn(e);
            }
          }
        } catch (MongoException e) {
          log4j.error(e);
        }
      }
    }
  }
Example #14
0
  public File getTheNewestFile() {

    workingDir = System.getProperty("user.dir");
    String fileName =
        workingDir
            + File.separator
            + "data"
            + File.separator
            + "log"
            + File.separator
            + "karaf.log";
    WatchedDir = workingDir + File.separator + "data" + File.separator + "log";

    try {
      LOG.info("log enabled ...");
      File dir = new File(WatchedDir);
      Thread.sleep(10000);
      files = dir.listFiles();
      Arrays.sort(files, LastModifiedFileComparator.LASTMODIFIED_COMPARATOR);
      for (int i = 0; i < files.length; i++) {
        file = files[i];
        karafFiles += file.getName();
      }
      LOG.info("karaf files:" + karafFiles);
      Arrays.sort(files, LastModifiedFileComparator.LASTMODIFIED_REVERSE);
      for (int i = 0; i < files.length; i++) {
        file = files[i];
        RevOrder_ofFiles += file.getName();
      }
      LOG.info("list of karaf File in reverse order" + " " + RevOrder_ofFiles);
      theNewestFile = files[0];
      LOG.info("Latest karaf File" + theNewestFile);
    } catch (Exception e) {
      e.printStackTrace();
    }
    return theNewestFile;
  }
Example #15
0
  /**
   * Creates the database, deleting data that existed previously.
   *
   * <p>Won't actually remove symbol's tables if they aren't in the set SYMBOLS.
   *
   * @param pauseDuration - this is how long we sleep (in ms) for before hitting yahoo again
   * @see SYMBOLS
   */
  public static void createDatabase(String[] symbolArray, int pauseDuration) {
    try {
      Connection conn = initialize();
      Statement stat = conn.createStatement();
      try {
        stat.executeUpdate("DROP TABLE IF EXISTS " + "listofsymbols" + ";");
        stat.executeUpdate(
            "CREATE TABLE IF NOT EXISTS " + "listofsymbols" + " (" + "'symbolname'" + ");");
      } catch (Exception e) {
        System.out.println("failed stat.executeUpdate");
        System.out.println(e.getMessage());
      }

      for (int i = 0; i < symbolArray.length; i++) {
        addSymbol(symbolArray[i]);
        if (pauseDuration != 0) {
          Thread.sleep(pauseDuration);
        }
      }
      conn.close();
    } catch (Exception e) {
      e.printStackTrace();
    }
  }
 public static void runTest2() throws Exception {
   boolean exceptionOccured1 = false;
   boolean exceptionOccured2 = false;
   try {
     Context ctx = cache.getJNDIContext();
     DataSource ds1 = null;
     DataSource ds2 = null;
     ds1 = (DataSource) ctx.lookup("java:/XAPooledDataSource");
     ds2 = (DataSource) ctx.lookup("java:/SimpleDataSource");
     ds2.getConnection();
     ds1.getConnection();
     ds1.getConnection();
     ds1.getConnection();
     ds1.getConnection();
     ds1.getConnection();
     UserTransaction utx = (UserTransaction) ctx.lookup("java:/UserTransaction");
     utx.begin();
     try {
       ds1.getConnection();
       Thread.sleep(8000);
     } catch (SQLException e) {
       exceptionOccured1 = true;
     }
     try {
       utx.commit();
     } catch (Exception e) {
       exceptionOccured2 = true;
     }
     if (!exceptionOccured1)
       fail("Exception (Login-Time-Out)did not occur although was supposed" + "to occur");
     if (exceptionOccured2)
       fail("Exception did occur on commit, although was not supposed" + "to occur");
   } catch (Exception e) {
     fail("failed in runTest2 due to " + e);
   }
 }
Example #17
0
  public static boolean kill() {
    int j = 0;
    Thread meMySelfI = Thread.currentThread();

    do {
      /* iter across the thread group, killing all members. */
      shutdown = true;
      Thread list[] = new Thread[tg.activeCount()];

      // get all members of the group (including submembers)
      int i = tg.enumerate(list);

      // no members means that we have gracefully suceeded
      if (i == 0) return true;

      // if some of the threads do IO during the shut down they will
      // need time to accomplish the IO. So, I give it to 'em
      // after the first attempt.
      if (j > 0)
        try {
          meMySelfI.sleep(500);
        } catch (Exception e) {
        }
      ;

      // try to shudown each thread in the group
      while (i-- > 0) {
        FlickrFtpd tftp = (FlickrFtpd) list[i];
        tftp.interrupt(); // first, do it politely
        meMySelfI.yield(); // give 'em time to respond
        tftp.forceClose(); // second, use a big hammer
        meMySelfI.yield(); // give 'em time to respond
      }
    } while (j++ <= 3);
    return false;
  }
  /**
   * Method to move files from HDFS to local filesystem
   *
   * <p>localPath: Path on the machines filesystem fs:FileSystem object from HDFS pathList:List of
   * paths for files that might need to be backed up size:max size in bytes to be backed up
   *
   * <p>ReturnsDate of the last files backed up if reached size limit, else, zero
   */
  public long backupFiles(
      String localPath, String preservePath, FileSystem fs, ArrayList<Path> pathList, long size) {
    Path fsPath;
    long tmpSize = 0;
    long tmpDate = 0;

    // Start iterating over all paths
    for (Path hdfsPath : pathList) {
      try {
        long nFileSize = fs.getContentSummary(hdfsPath).getLength();
        tmpSize = tmpSize + nFileSize;

        if ((tmpSize <= size) || (size == 0)) {
          FileStatus stat = fs.getFileStatus(hdfsPath);

          System.err.println(
              "File "
                  + hdfsPath.toUri().getPath()
                  + " "
                  + nFileSize
                  + " bytes, "
                  + "perms: "
                  + stat.getOwner()
                  + "/"
                  + stat.getGroup()
                  + ", "
                  + stat.getPermission().toString());

          tmpDate = stat.getModificationTime() / 1000;

          String sFsPath = localPath + hdfsPath.toUri().getPath();
          fsPath = new Path(sFsPath);

          File f = new File(sFsPath);

          // COMMENTED OUT: until a few backup cycles run
          // and the mtime gets in fact set on all copied
          // files.
          //
          // ignore it if the file exists and has the same mtime
          // if (f.exists() && f.isFile() && f.lastModified() == stat.getModificationTime())
          // {
          // System.out.println("no need to backup " + f.toString() + ", mtime matches hdfs");
          // continue;
          // }

          if (false == m_bDryRun) {
            // check if we need to back up the local file
            // (not directory), if it already exists.
            if (f.exists() && f.isFile()) {
              // ignore files with substrings in the
              // no-preserve file
              if (true == doPreserveFile(sFsPath)) {
                // move it to the backup path
                String sNewPath = preservePath + hdfsPath.toUri().getPath();
                File newFile = new File(sNewPath);

                // create directory structure for new file?
                if (false == newFile.getParentFile().exists()) {
                  if (false == newFile.getParentFile().mkdirs()) {
                    System.err.println("Failed to mkdirs " + newFile.getParentFile().toString());
                    System.exit(1);
                  }
                }

                // rename existing file to new location
                if (false == f.renameTo(newFile)) {
                  System.err.println(
                      "Failed to renameTo " + f.toString() + " to " + newFile.toString());
                  System.exit(1);
                }

                System.out.println("preserved " + f.toString() + " into " + newFile.toString());
              } else {
                System.out.println("skipped preservation of " + f.toString());
              }
            }

            // copy from hdfs to local filesystem
            fs.copyToLocalFile(hdfsPath, fsPath);

            // set the mtime to match hdfs file
            f.setLastModified(stat.getModificationTime());

            // compare checksums on both files
            compareChecksums(fs, hdfsPath, sFsPath);
          }

          // don't print the progress after every file -- go
          // by at least 1% increments
          long nPercentDone = (long) (100 * tmpSize / m_nTotalBytes);
          if (nPercentDone > m_nLastPercentBytesDone) {
            System.out.println(
                "progress: copied "
                    + prettyPrintBytes(tmpSize)
                    + ", "
                    + nPercentDone
                    + "% done"
                    + ", tstamp="
                    + tmpDate);

            m_nLastPercentBytesDone = nPercentDone;
          }

          if (m_nSleepSeconds > 0) {
            try {
              Thread.sleep(1000 * m_nSleepSeconds);
            } catch (Exception e2) {
              // ignore
            }
          }
        } else {
          return tmpDate;
        }
      } catch (IOException e) {
        System.err.println("FATAL ERROR: Something wrong with the file");
        System.err.println(e);
        System.out.println(tmpDate);
        System.exit(1);

        return 0;
      }
    }

    return 0;
  }
Example #19
0
  public static void main(String[] argv) {
    try {
      ComboPooledDataSource cpds = new ComboPooledDataSource();
      cpds.setJdbcUrl(argv[0]);
      cpds.setUser(argv[1]);
      cpds.setPassword(argv[2]);
      cpds.setMinPoolSize(5);
      cpds.setAcquireIncrement(5);
      cpds.setMaxPoolSize(20);

      System.err.println("Initial...");
      display(cpds);
      Thread.sleep(2000);

      HashSet hs = new HashSet();
      for (int i = 0; i < 20; ++i) {
        Connection c = cpds.getConnection();
        hs.add(c);
        System.err.println("Adding (" + (i + 1) + ") " + c);
        display(cpds);
        Thread.sleep(1000);

        // 			if (i == 9)
        // 			    {
        //  				//System.err.println("hardReset()ing");
        //  				//cpds.hardReset();
        // 				System.err.println("softReset()ing");
        // 				cpds.softReset();
        // 			    }
      }

      int count = 0;
      for (Iterator ii = hs.iterator(); ii.hasNext(); ) {
        Connection c = ((Connection) ii.next());
        System.err.println("Removing " + ++count);
        ii.remove();
        try {
          c.getMetaData().getTables(null, null, "PROBABLYNOT", new String[] {"TABLE"});
        } catch (Exception e) {
          System.err.println(e);
          System.err.println();
          continue;
        } finally {
          c.close();
        }
        Thread.sleep(2000);
        display(cpds);
      }

      System.err.println(
          "Closing data source, \"forcing\" garbage collection, and sleeping for 5 seconds...");
      cpds.close();
      System.gc();
      System.err.println("Main Thread: Sleeping for five seconds!");
      Thread.sleep(5000);
      // 		System.gc();
      // 		Thread.sleep(5000);
      System.err.println("Bye!");
    } catch (Exception e) {
      e.printStackTrace();
    }
  }
  private void executeParsedFile(NameListMessage nameListMessage, ArrayList cmds) {
    BufferedWriter buffer = null;
    int connectionId = nameListMessage.connectionId;
    boolean commaSeparated = nameListMessage.commaSeparated;

    // get java.sql.Connection from connectionId
    Connection connection = ConnectionService.getConnection(connectionId);

    if (connection == null) {
      nameListMessage.errorMessage = kConnectionClosed;
    } else {

      try {
        Thread th = null;
        buffer = new BufferedWriter(writer);

        // if reader exists, create reading thread
        if (reader != null) {
          final ObjectNameReader objectReader = new ObjectNameReader(nameListMessage.nameList);

          // This thread will read stream until meets EOL (ie. when
          // writer is closed)
          th =
              new Thread() {
                public void run() {
                  objectReader.readInputFile(reader);
                }
              };
        } // end if

        // if no reader (offline reverse), never start the reading
        // thread
        // (because output is written down directly to the disk without
        // processing in counterpart).
        if (reader != null) th.start();

        // execute each SQL statement
        // we skip objects that don't need to be reversed
        Iterator iter = cmds.iterator();
        int index = -1;
        while (iter.hasNext()) {
          String cmd = (String) iter.next();
          index++;
          // check if this statement id is marked as <to skip>
          if (nameListMessage.ignoredStatementIds != null
              && nameListMessage.ignoredStatementIds.contains(new Integer(index))) {
            if (commaSeparated) {
              buffer.flush();
              buffer.newLine();
            }
            continue;
          }
          if (writer != null) {
            if (cmd != null) {
              try {
                execute(connection, cmd, buffer, commaSeparated);
              } catch (SQLException e) {
                if (Debug.isDebug()) e.printStackTrace();
                nameListMessage.errorMessage =
                    nameListMessage.errorMessage == null
                        ? e.toString()
                        : nameListMessage.errorMessage.concat("\n" + e.toString()); // NOT
                // LOCALIZABLE,
                // escape
                // code
              }
            }
            if (commaSeparated) {
              buffer.flush();
              buffer.newLine();
            }
          }
        }
      }
      /*
       * catch (SQLException ex) { if (Debug.isDebug()) ex.printStackTrace();
       * nameListMessage.errorMessage = nameListMessage.errorMessage == null ? ex.toString() :
       * nameListMessage.errorMessage.concat("\n" + ex.toString()); //NOT LOCALIZABLE, escape
       * code }
       */
      catch (IOException ex) {
        if (Debug.isDebug()) ex.printStackTrace();
        nameListMessage.errorMessage =
            nameListMessage.errorMessage == null
                ? ex.toString()
                : nameListMessage.errorMessage.concat(
                    "\n" + ex.toString()); // NOT LOCALIZABLE, escape
        // code
      }

      // close writer: the reading thread terminates.
      try {
        // WARNING: we must make a pause after closing the buffer
        // otherwise
        // the nameList variable won't be well initialized...strange!
        // [FG]
        buffer.close();
        Thread.sleep(400);
      } catch (IOException ex) {
      } catch (InterruptedException ex) {
      }
    } // end if
  } // end parseSqlFile()
Example #21
0
  // ----This function gets the raw json from raw_data collection----
  // ---------the function separate the tweets and inserts it to all_tweets collection and calling
  // all other data processing methods-----------
  // ----------------------------------------------------------------------------------------
  public void update_all_tweets(final double num_of_slots, final double max_time_frame_hours)
      throws MongoException, ParseException {
    log4j.info("starting update_all_tweets function");
    String res = new String();
    Integer countelements = 0;
    while (true) {
      // DBCursor cursor = this.collrd.find();
      while (this.collrd.count() < 1) { // no documents to process
        try {
          log4j.info("there is no raw data at the moment, going to sleep for 10 seconds");
          Thread.currentThread();
          Thread.sleep(1000 * 10);
          log4j.info("woke up, continues");
          // cursor = this.collrd.find();
        } catch (InterruptedException e) {
          log4j.error("InterruptedException caught, at update_all_tweets");
          log4j.error(e);
        }
      }
      DBCursor cursor = this.collrd.find(); // get all documents from raw_data collection
      try {
        while (cursor.hasNext()) {
          DBObject currdoc = cursor.next();
          log4j.info("getting a document from the raw data db");
          Object results = currdoc.get("results"); // result - json array of tweets
          try {
            res = results.toString();
          } catch (NullPointerException e) {
            res = "";
          }
          Object obj = JSONValue.parse(res);
          log4j.info("making an array from the jsons tweets");
          JSONArray array = (JSONArray) obj; // make an array of tweets
          // JSONParser parser = new JSONParser();
          try {
            if (res != "") { // if there are tweets
              @SuppressWarnings("rawtypes")
              Iterator iterArray = array.iterator();
              log4j.info("iterating over array tweets");
              try {
                while (iterArray.hasNext()) {
                  Object current = iterArray.next();
                  final DBObject dbObject =
                      (DBObject) JSON.parse(current.toString()); // parse all tweet data to json
                  countelements++;
                  // System.out.println("element number" + countelements.toString());
                  dbObject.put("max_id", currdoc.get("max_id")); // add max_id to tweet data
                  dbObject.put("query", currdoc.get("query")); // add query word to tweet data
                  dbObject.put(
                      "query_time", currdoc.get("query_time")); // add query time to tweet data
                  dbObject.put("query_time_string", currdoc.get("query_time_string"));
                  dbObject.put(
                      "text",
                      "@"
                          + dbObject.get("from_user").toString()
                          + ": "
                          + dbObject.get("text").toString()); // add user_name to beginning of text
                  dbObject.put("count", 1L); // add appearance counter to tweet data
                  log4j.info("inserting tweet id: " + dbObject.get("id").toString());
                  try {
                    DBObject object = new BasicDBObject();
                    object.put(
                        "id", Long.parseLong(dbObject.get("id").toString())); // object to search
                    DBObject newobject = collat.findOne(object);
                    if (newobject != null) {
                      newobject.put(
                          "count",
                          Long.parseLong(newobject.get("count").toString())
                              + 1); // update counter if id already exists
                      collat.update(object, newobject);
                    }
                  } catch (NullPointerException e) {

                  }
                  collat.insert(dbObject);
                  // collrd.findAndRemove(currdoc);
                  // log4j.info("calling function update_search_terms");
                  // final String text = "@" + dbObject.get("from_user").toString() + ": " +
                  // dbObject.get("text").toString();

                  /*Thread t10=new Thread(new Runnable(){
                  	public void run(){
                  		UpdateTweetCounterId(Long.parseLong(dbObject.get("id").toString()));
                  	}
                  });*/

                  Thread t11 =
                      new Thread(
                          new Runnable() {
                            public void run() {
                              update_search_terms(
                                  dbObject.get("text").toString(),
                                  num_of_slots,
                                  max_time_frame_hours,
                                  dbObject.get("query").toString());
                            }
                          });

                  Thread t12 =
                      new Thread(
                          new Runnable() {
                            public void run() {
                              rate_user(
                                  Long.parseLong(dbObject.get("from_user_id").toString()),
                                  dbObject.get("from_user").toString(),
                                  max_time_frame_hours);
                              // UpdateUserRate((long)num_of_slots,slot_time_millis,Long.parseLong(dbObject.get("from_user_id").toString()) , dbObject.get("from_user").toString() ,(long)0);
                            }
                          });

                  Thread t13 =
                      new Thread(
                          new Runnable() {
                            public void run() {
                              String quer = dbObject.get("query").toString();
                              quer = quer.replaceAll("%40", "@");
                              quer = quer.replaceAll("%23", "#");
                              long id =
                                  (long)
                                      (Double.parseDouble(dbObject.get("query_time").toString())
                                          * 1000);
                              String idplus = dbObject.get("id").toString() + "," + id;
                              SearchResultId(quer, idplus);
                            }
                          });
                  // t10.start();
                  t11.start();
                  t12.start();
                  t13.start();
                  try {
                    log4j.info("Waiting for threads to finish.");
                    // t10.join();
                    t11.join();
                    t12.join();
                    t13.join();
                  } catch (InterruptedException e) {
                    log4j.error("Main thread (update_all_tweets) Interrupted");
                  }
                }
              } catch (Exception e) {
                log4j.error(e);
                e.printStackTrace();
              }
            }
          } catch (NullPointerException e) {
            log4j.error(e);
            log4j.info("NullPointerException caught, at update_all_tweets");
          }
          log4j.info("removing processed document from raw_data collection");
          try {
            this.collrd.remove(currdoc);
          } catch (Exception e) {
            log4j.debug(e);
          }
        }
      } catch (MongoException e) {
        log4j.error(e);
      }
    }
  }
Example #22
0
  /**
   * Main function of the DOTS. Read the config file,create dotslogging,start performance client,
   * check db connection,create keyboard thread. Create new testcase according the cpu_usage and
   * connections, Write test summary and exception
   */
  public static void main(String[] args) {

    int sleepInterval = 1;
    int waitInterval = 0;
    int averageUsage = 0;
    boolean reached = false;
    Calendar currentTime;

    /* Load DotsConfig value */
    loadConfig(args);
    /* Create the instance of DotsLogging */
    DotsLogging logging = new DotsLogging(TESTCASENAME);

    DotsLogging.logMessage("Database Opensource Test Suite V1.0");
    DotsLogging.logMessage("Start to run JDBC API Test Case - " + TESTCASENAME);
    DotsLogging.logSummary("Start to run JDBC API Test Case - " + TESTCASENAME);
    if (!DotsConfig.RUN_AUTO) {
      System.out.println("\nDatabase Opensource Test Suite V1.0");
      System.out.println("\nStart to run JDBC API test case - " + TESTCASENAME);
    }

    DotsLogging.logMessage("Initialization started");
    if (!DotsConfig.RUN_AUTO) {
      System.out.println("\nTo stop running the test case, type STOP then press Enter\n");
    }
    /* Start performance monitor thread */
    startPerfCtl();

    /* Check database connection */
    checkDBConn();

    DotsLogging.logMessage("Testing Database Connections ...OK");

    /* Start summary writer thread */
    DotsSummary summary = new DotsSummary();
    summary.start();

    DotsLogging.logMessage("Starting Summary Writer ... OK ");

    /* Set begin and end time */
    Calendar endTime = Calendar.getInstance();
    DotsSummary.startTime = System.currentTimeMillis();
    endTime.add(Calendar.SECOND, DotsConfig.DURATION * 60);

    /* Start keyboard thread */
    if (!DotsConfig.RUN_AUTO) {
      startKeyboard();
      DotsLogging.logMessage("Starting Keyboard Thread ... OK");
    }

    boolean firstRun = true; // used for LOB manipulation
    String lobFileName[] = new String[10]; // used for LOB manipulation

    averageUsage = DotsConfig.CPU_USAGE;
    int targetCpu = DotsConfig.CPU_TARGET;
    try {
      while (true) {
        currentTime = Calendar.getInstance();
        if (currentTime.after(endTime)) break;
        if (DotsConfig.TERMINATION) break;
        /* Check if need create a new database access thread*/
        if ((DotsConfig.AUTO_MODE && (averageUsage < targetCpu))
            || (!DotsConfig.AUTO_MODE
                && (DotsConfig.THRDGRP.activeCount() < DotsConfig.CONNECTIONS))) {
          Connection conn =
              createConnection(
                  DotsConfig.DRIVER_CLASS_NAME,
                  DotsConfig.URL,
                  DotsConfig.DB_UID,
                  DotsConfig.DB_PASSWD);
          if (conn != null) {
            /* 5 second is a unit*/
            sleepInterval = DotsConfig.INTERVAL * 12;
            /* Create DB access thread */
            switch (TESTCASENAME.charAt(4)) {
              case '1':
                if (TESTCASENAME.equals("BTCJ1")) {
                  BTCJ1 btcj1 = new BTCJ1(conn);
                  Thread thrdb1 = new Thread(DotsConfig.THRDGRP, btcj1);
                  thrdb1.start();
                } else {
                  ATCJ1 atcj1 = new ATCJ1(conn);
                  Thread thrda1 = new Thread(DotsConfig.THRDGRP, atcj1);
                  thrda1.start();
                }
                break;
              case '2':
                if (TESTCASENAME.equals("BTCJ2")) {
                  BTCJ2 btcj2 = new BTCJ2(conn);
                  Thread thrdb2 = new Thread(DotsConfig.THRDGRP, btcj2);
                  thrdb2.start();
                } else {
                  ATCJ2 atcj2 = new ATCJ2(conn);
                  Thread thrda2 = new Thread(DotsConfig.THRDGRP, atcj2);
                  thrda2.start();
                }
                break;
              case '3':
                BTCJ3 btcj3 = new BTCJ3(conn);
                Thread thrdb3 = new Thread(DotsConfig.THRDGRP, btcj3);
                thrdb3.start();

                break;
              case '4':
                BTCJ4 btcj4 = new BTCJ4(conn);
                Thread thrdb4 = new Thread(DotsConfig.THRDGRP, btcj4);
                thrdb4.start();

                break;
              case '5':
                BTCJ5 btcj5 = new BTCJ5(conn);
                Thread thrdb5 = new Thread(DotsConfig.THRDGRP, btcj5);
                thrdb5.start();
                break;
              case '6':
                if (firstRun) {
                  for (int i = 0; i < 10; i++) {
                    lobFileName[i] = DotsGenerator.mdClob(DotsGenerator.mdInt(10, 100));
                  }
                  firstRun = false;
                }
                BTCJ6 btcj6 = new BTCJ6(conn, lobFileName);
                Thread thrdb6 = new Thread(DotsConfig.THRDGRP, btcj6);
                thrdb6.start();
                break;
              case '7':
                if (firstRun) {
                  for (int i = 0; i < 10; i++) {
                    lobFileName[i] = DotsGenerator.mdBlob(DotsGenerator.mdInt(10, 100));
                  }
                  firstRun = false;
                }
                BTCJ7 btcj7 = new BTCJ7(conn, lobFileName);
                Thread thrdb7 = new Thread(DotsConfig.THRDGRP, btcj7);
                thrdb7.start();
                break;
              case '8':
                BTCJ8 btcj8 = new BTCJ8(conn);
                Thread thrdb8 = new Thread(DotsConfig.THRDGRP, btcj8);
                thrdb8.start();
                break;
            }
          } else {
            /* If createConnection failed,sleepInterval will double,
            but the maximum of sleepInterval is 1 hour */
            sleepInterval = sleepInterval * 2;
            if (sleepInterval > 720) sleepInterval = 720;
          }
          try {
            averageUsage = 0;
            /* Wait 5*sleepInterval seconds,check TERMINATION per 5 second */
            for (int j = 0; j < sleepInterval; j++) {
              averageUsage += DotsConfig.CPU_USAGE;
              Thread.sleep(5000);
              if ((j % 12) == 0)
                DotsLogging.logMessage(
                    "Active Threads = "
                        + DotsConfig.THRDGRP.activeCount()
                        + " Average CPU Usage = "
                        + DotsConfig.CPU_USAGE
                        + "%");
              if (DotsConfig.TERMINATION) break;
              currentTime = Calendar.getInstance();
              if (currentTime.after(endTime)) {
                DotsConfig.TERMINATION = true;
                break;
              }
            }
            /* Calculate the average usage of this sleepInterval */
            averageUsage = averageUsage / sleepInterval;
            DotsLogging.logMessage(
                sleepInterval / 12 + " Minutes Average CPU Usage = " + averageUsage + "%");
          } catch (Exception e) {
            DotsLogging.logException("Dots.main: " + e);
          }

        } else {
          /* When CPU_TARGET is reached,DOTS will not create new thread until
          average usage is less than CPU_TARGET-10 */
          if (DotsConfig.AUTO_MODE && !reached) {
            reached = true;
            targetCpu = DotsConfig.CPU_TARGET - 10;
            DotsLogging.logMessage("CPU Target " + DotsConfig.CPU_TARGET + "% is achieved now.");
          }
          /* Wait 5 minutes then check create connection condition */
          try {
            averageUsage = 0;
            for (int i = 0; i < 60; i++) {
              averageUsage += DotsConfig.CPU_USAGE;
              Thread.sleep(5000);
              if ((i % 12) == 0)
                DotsLogging.logMessage(
                    "Active Threads = "
                        + DotsConfig.THRDGRP.activeCount()
                        + " Average CPU Usage = "
                        + DotsConfig.CPU_USAGE
                        + "%");
              if (DotsConfig.TERMINATION) break;
              currentTime = Calendar.getInstance();
              if (currentTime.after(endTime)) {
                DotsConfig.TERMINATION = true;
                break;
              }
            }
            averageUsage = averageUsage / 60;
            DotsLogging.logMessage("5 Minutes Average CPU Usage = " + averageUsage + "%");
          } catch (Exception e) {
            DotsLogging.logException("Dots.main: " + e);
          }
        }
      }
    } catch (Throwable t) {
      DotsLogging.logException("Dots.main: " + t);
      System.out.println("Dots.main:" + t);
    }

    /* DURATION is reached or TERMINATION is set by other threads */
    DotsConfig.TERMINATION = true;
    DotsLogging.logMessage("Dots is Terminating ...");
    /* Clear temporary file used by testcase 6&7 */
    if ((TESTCASENAME.charAt(4) == '6') || (TESTCASENAME.charAt(4) == '7')) {
      DotsLogging.logMessage("Cleaning temporary file ...");

      if (!DotsConfig.RUN_AUTO) {
        System.out.println("Cleaning temporary file ...");
      }
      try {
        for (int j = 0; j < 10; j++) {
          File tmpFile = new File(lobFileName[j]);
          if (tmpFile.exists()) tmpFile.delete();
        }
      } catch (Exception e) {
        DotsLogging.logException("Clean temporary file error.");
      }
    }
    /* Write summary file and wait DB thread to exit */
    try {
      if (!DotsConfig.RUN_AUTO) {
        System.out.println("\nWriting summary file ...");
      }
      summary.writeSummaryFile();
      summary.canExit = true;
      DotsPerf.canExit = true;
      DotsLogging.logSummary("Waiting for the active threads to exit ......");
      if (!DotsConfig.RUN_AUTO) {
        System.out.println("\nWaiting for the active threads to exit ......");
      }
      while (DotsConfig.THRDGRP.activeCount() > 0) {
        waitInterval++;
        Thread.sleep(1000);
        if (!DotsConfig.RUN_AUTO) {
          System.out.println("Current active thread number is " + DotsConfig.THRDGRP.activeCount());
        }
        if (waitInterval > 60) break; // Max wait time is 1 minute
      }
    } catch (InterruptedException e) {
      DotsLogging.logException("Dots.main:" + e);
    }
    DotsLogging.logSummary("All Database Access Threads exit.");
    DotsLogging.close();
    if (!DotsConfig.RUN_AUTO) {
      System.out.println("DOTS closed successfully.");
    }
    System.exit(0);
  }