Beispiel #1
0
  /**
   * Test batched prepared statement concurrency. Batch prepares must not disappear between the
   * moment when they were created and when they are executed.
   */
  public void testConcurrentBatching() throws Exception {
    // Create a connection with a batch size of 1. This should cause prepares and actual batch
    // execution to become
    // interspersed (if correct synchronization is not in place) and greatly increase the chance of
    // prepares
    // being rolled back before getting executed.
    Properties props = new Properties();
    props.setProperty(Messages.get(net.sourceforge.jtds.jdbc.Driver.BATCHSIZE), "1");
    props.setProperty(
        Messages.get(net.sourceforge.jtds.jdbc.Driver.PREPARESQL),
        String.valueOf(TdsCore.TEMPORARY_STORED_PROCEDURES));
    Connection con = getConnection(props);

    try {
      Statement stmt = con.createStatement();
      stmt.execute(
          "create table #testConcurrentBatch (v1 int, v2 int, v3 int, v4 int, v5 int, v6 int)");
      stmt.close();

      Vector exceptions = new Vector();
      con.setAutoCommit(false);

      Thread t1 = new ConcurrentBatchingHelper(con, exceptions);
      Thread t2 = new ConcurrentBatchingHelper(con, exceptions);
      t1.start();
      t2.start();
      t1.join();
      t2.join();

      assertEquals(0, exceptions.size());
    } finally {
      con.close();
    }
  }
Beispiel #2
0
  // ----This function gets the raw json from raw_data collection----
  // ---------the function separate the tweets and inserts it to all_tweets collection and calling
  // all other data processing methods-----------
  // ----------------------------------------------------------------------------------------
  public void update_all_tweets(final double num_of_slots, final double max_time_frame_hours)
      throws MongoException, ParseException {
    log4j.info("starting update_all_tweets function");
    String res = new String();
    Integer countelements = 0;
    while (true) {
      // DBCursor cursor = this.collrd.find();
      while (this.collrd.count() < 1) { // no documents to process
        try {
          log4j.info("there is no raw data at the moment, going to sleep for 10 seconds");
          Thread.currentThread();
          Thread.sleep(1000 * 10);
          log4j.info("woke up, continues");
          // cursor = this.collrd.find();
        } catch (InterruptedException e) {
          log4j.error("InterruptedException caught, at update_all_tweets");
          log4j.error(e);
        }
      }
      DBCursor cursor = this.collrd.find(); // get all documents from raw_data collection
      try {
        while (cursor.hasNext()) {
          DBObject currdoc = cursor.next();
          log4j.info("getting a document from the raw data db");
          Object results = currdoc.get("results"); // result - json array of tweets
          try {
            res = results.toString();
          } catch (NullPointerException e) {
            res = "";
          }
          Object obj = JSONValue.parse(res);
          log4j.info("making an array from the jsons tweets");
          JSONArray array = (JSONArray) obj; // make an array of tweets
          // JSONParser parser = new JSONParser();
          try {
            if (res != "") { // if there are tweets
              @SuppressWarnings("rawtypes")
              Iterator iterArray = array.iterator();
              log4j.info("iterating over array tweets");
              try {
                while (iterArray.hasNext()) {
                  Object current = iterArray.next();
                  final DBObject dbObject =
                      (DBObject) JSON.parse(current.toString()); // parse all tweet data to json
                  countelements++;
                  // System.out.println("element number" + countelements.toString());
                  dbObject.put("max_id", currdoc.get("max_id")); // add max_id to tweet data
                  dbObject.put("query", currdoc.get("query")); // add query word to tweet data
                  dbObject.put(
                      "query_time", currdoc.get("query_time")); // add query time to tweet data
                  dbObject.put("query_time_string", currdoc.get("query_time_string"));
                  dbObject.put(
                      "text",
                      "@"
                          + dbObject.get("from_user").toString()
                          + ": "
                          + dbObject.get("text").toString()); // add user_name to beginning of text
                  dbObject.put("count", 1L); // add appearance counter to tweet data
                  log4j.info("inserting tweet id: " + dbObject.get("id").toString());
                  try {
                    DBObject object = new BasicDBObject();
                    object.put(
                        "id", Long.parseLong(dbObject.get("id").toString())); // object to search
                    DBObject newobject = collat.findOne(object);
                    if (newobject != null) {
                      newobject.put(
                          "count",
                          Long.parseLong(newobject.get("count").toString())
                              + 1); // update counter if id already exists
                      collat.update(object, newobject);
                    }
                  } catch (NullPointerException e) {

                  }
                  collat.insert(dbObject);
                  // collrd.findAndRemove(currdoc);
                  // log4j.info("calling function update_search_terms");
                  // final String text = "@" + dbObject.get("from_user").toString() + ": " +
                  // dbObject.get("text").toString();

                  /*Thread t10=new Thread(new Runnable(){
                  	public void run(){
                  		UpdateTweetCounterId(Long.parseLong(dbObject.get("id").toString()));
                  	}
                  });*/

                  Thread t11 =
                      new Thread(
                          new Runnable() {
                            public void run() {
                              update_search_terms(
                                  dbObject.get("text").toString(),
                                  num_of_slots,
                                  max_time_frame_hours,
                                  dbObject.get("query").toString());
                            }
                          });

                  Thread t12 =
                      new Thread(
                          new Runnable() {
                            public void run() {
                              rate_user(
                                  Long.parseLong(dbObject.get("from_user_id").toString()),
                                  dbObject.get("from_user").toString(),
                                  max_time_frame_hours);
                              // UpdateUserRate((long)num_of_slots,slot_time_millis,Long.parseLong(dbObject.get("from_user_id").toString()) , dbObject.get("from_user").toString() ,(long)0);
                            }
                          });

                  Thread t13 =
                      new Thread(
                          new Runnable() {
                            public void run() {
                              String quer = dbObject.get("query").toString();
                              quer = quer.replaceAll("%40", "@");
                              quer = quer.replaceAll("%23", "#");
                              long id =
                                  (long)
                                      (Double.parseDouble(dbObject.get("query_time").toString())
                                          * 1000);
                              String idplus = dbObject.get("id").toString() + "," + id;
                              SearchResultId(quer, idplus);
                            }
                          });
                  // t10.start();
                  t11.start();
                  t12.start();
                  t13.start();
                  try {
                    log4j.info("Waiting for threads to finish.");
                    // t10.join();
                    t11.join();
                    t12.join();
                    t13.join();
                  } catch (InterruptedException e) {
                    log4j.error("Main thread (update_all_tweets) Interrupted");
                  }
                }
              } catch (Exception e) {
                log4j.error(e);
                e.printStackTrace();
              }
            }
          } catch (NullPointerException e) {
            log4j.error(e);
            log4j.info("NullPointerException caught, at update_all_tweets");
          }
          log4j.info("removing processed document from raw_data collection");
          try {
            this.collrd.remove(currdoc);
          } catch (Exception e) {
            log4j.debug(e);
          }
        }
      } catch (MongoException e) {
        log4j.error(e);
      }
    }
  }