예제 #1
0
  public static void getBlogs() throws Exception {

    BlockingQueue<String> queue = new ArrayBlockingQueue<String>(numCrawler * 4);

    CrawlerC[] crawler = new CrawlerC[numCrawler];
    for (int i = 0; i < crawler.length; i++) {
      crawler[i] = new CrawlerC(queue);
      crawler[i].start();
    }

    ResultSet rs = null;
    int offset = 1;
    while (true) {
      offset += 100;
      myStm.executeQuery("SELECT blogID from blogs where country = 'BR' LIMIT " + offset + ",100");
      System.out.println("\n---" + offset + "---");

      rs = myStm.getResultSet();
      try {
        if (!rs.first()) break;
        if (false) break;
        while (rs.next()) {
          // System.out.println(rs.getString("blogID"));
          if (!queue.offer(rs.getString("blogID"), 60, TimeUnit.SECONDS)) {
            System.out.println("Offer.Timeout");
          }
        }
      } catch (Exception e) {
      }
    }

    queue.clear();
    for (int i = 0; i < crawler.length; i++) queue.put(CrawlerC.NO_MORE_WORK);
    for (int i = 0; i < crawler.length; i++) crawler[i].join();
  }
 public final void put(T item) {
   if (!state) {
     throw new RuntimeException("Putting Forbidden");
   } else {
     try {
       queue.put(item);
     } catch (InterruptedException e) {
       e.printStackTrace();
     }
   }
 }
예제 #3
0
    /*package*/ List<O> doJob(List<I> batch) {
      List<O> generatedBatch;
      assert lastBatchSent == false;

      if (batch == POISON_PILL) {
        lastBatch = true;
        synchronized (name) {
          pendingJobs--;
        }
        //                System.out.println(name + " - lastBatch");
        generatedBatch = Collections.emptyList();
      } else {

        EXECUTOR task = taskQueue.poll();

        boolean nextNodesAvailable = true;
        for (Node<O, ?, ?> node : nodes) {
          nextNodesAvailable &= node.isAvailable();
        }

        if (task == null) { // No available task
          resubmit(batch);
          generatedBatch = null;
        } else if (!nextNodesAvailable) { // Next nodes have to many batches.
          try {
            taskQueue.put(task);
          } catch (InterruptedException e) {
            e.printStackTrace();
          }
          resubmit(batch);
          generatedBatch = null;
        } else { // Execute

          generatedBatch = execute(task, batch);
          //                    System.out.println(name + " - end job - " + generatedBatch.size());
          for (Node<O, ?, ?> node : nodes) {
            node.submit(generatedBatch);
          }

          try {
            taskQueue.put(task);
          } catch (InterruptedException e) {
            e.printStackTrace();
          }
          synchronized (name) {
            pendingJobs--;
          }
          //                    System.out.println(name + " - pendingJobs " + pendingJobs);
        }
      }

      if (isFinished()) {
        if (!lastBatchSent) {
          for (Node<O, ?, ?> node : nodes) {
            node.submit(POISON_PILL);
          }
          lastBatchSent = true;
        }
        System.out.println("Node '" + name + "' is finished");
        synchronized (syncObject) {
          syncObject.notify();
        }
      } else {
        System.out.println("Node '" + name + "' pendingJobs " + pendingJobs);
      }
      return generatedBatch;
    }
예제 #4
0
 private void handleChange(String line)
     throws IOException, InterruptedException, JsonParseException, JsonMappingException {
   changes.put(new StdDocumentChange(OBJECT_MAPPER.readTree(line)));
 }