Ejemplo n.º 1
0
  public List<Tweet> readFile(String fileLocation, int limit, String fromTweetId) {

    long startTime = System.currentTimeMillis();

    int errors = 0;
    int added = 0;
    int ignored = 0;
    int skipped = 0;

    List<Tweet> tweets = new ArrayList<Tweet>();

    try {

      // Read gzFile
      InputStream inputStream = new GZIPInputStream(new FileInputStream(fileLocation));
      Reader decoder = new InputStreamReader(inputStream);
      BufferedReader br = new BufferedReader(decoder);

      String status;

      while ((status = br.readLine()) != null) {

        // Ignore garbage and log output lines, all statuses start with JSON bracket
        if (!status.equals("") && status.charAt(0) == '{') {
          try {
            JSONObject jsonObject = new JSONObject(status);

            // We use created_at as an indicator that this is a Tweet.
            if (jsonObject.has("created_at")) {

              Status statusObject = TwitterObjectFactory.createStatus(status);

              Tweet tweet = this.getTweetObjectFromStatus(statusObject);

              if (fromTweetId != null) {

                if (fromTweetId.equals(tweet.getId())) {
                  this.log.write("StatusFileReader - Scanner pickup from " + fromTweetId);
                  fromTweetId = null;
                } else {
                  skipped++;
                }

                continue;
              }

              added++;
              tweets.add(tweet);

              if (limit > 0 && added >= limit) {
                break;
              }

            } else {
              ignored++;
            }

          } catch (JSONException e) {
            this.log.write(
                "Exception in StatusFileReader: Json Parse Failure on: "
                    + status
                    + ", "
                    + e.getMessage());
          }
        } else {
          ignored++;
        }
      }

      br.close();
      decoder.close();
      inputStream.close();

    } catch (Exception e) {
      this.log.write(
          "Exception in StatusFileReader: Error Reading File: "
              + e.getClass().getName()
              + ": "
              + e.getMessage());
    }

    long runTimeSeconds = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - startTime);

    double ops = ((double) added / (double) Math.max(runTimeSeconds, 1));

    this.log.write(
        "StatusFileReader - "
            + fileLocation
            + " processed in "
            + runTimeSeconds
            + "s. "
            + added
            + " ok / "
            + errors
            + " errors / "
            + ignored
            + " ignored / "
            + skipped
            + " skipped. "
            + ops
            + " ops. Limit: "
            + limit
            + ", Fetch: "
            + tweets.size());

    return tweets;
  }
Ejemplo n.º 2
0
  /**
   * Process HTTP request.
   *
   * @param act Action.
   * @param req Http request.
   * @param res Http response.
   */
  private void processRequest(String act, HttpServletRequest req, HttpServletResponse res) {
    res.setContentType("application/json");
    res.setCharacterEncoding("UTF-8");

    GridRestCommand cmd = command(req);

    if (cmd == null) {
      res.setStatus(HttpServletResponse.SC_BAD_REQUEST);

      return;
    }

    if (!authChecker.apply(req.getHeader("X-Signature"))) {
      res.setStatus(HttpServletResponse.SC_UNAUTHORIZED);

      return;
    }

    GridRestResponse cmdRes;

    Map<String, Object> params = parameters(req);

    try {
      GridRestRequest cmdReq = createRequest(cmd, params, req);

      if (log.isDebugEnabled()) log.debug("Initialized command request: " + cmdReq);

      cmdRes = hnd.handle(cmdReq);

      if (cmdRes == null)
        throw new IllegalStateException("Received null result from handler: " + hnd);

      byte[] sesTok = cmdRes.sessionTokenBytes();

      if (sesTok != null) cmdRes.setSessionToken(U.byteArray2HexString(sesTok));

      res.setStatus(HttpServletResponse.SC_OK);
    } catch (Exception e) {
      res.setStatus(HttpServletResponse.SC_OK);

      U.error(log, "Failed to process HTTP request [action=" + act + ", req=" + req + ']', e);

      cmdRes = new GridRestResponse(STATUS_FAILED, e.getMessage());
    } catch (Throwable e) {
      U.error(log, "Failed to process HTTP request [action=" + act + ", req=" + req + ']', e);

      throw e;
    }

    JsonConfig cfg = new GridJettyJsonConfig();

    // Workaround for not needed transformation of string into JSON object.
    if (cmdRes.getResponse() instanceof String)
      cfg.registerJsonValueProcessor(cmdRes.getClass(), "response", SKIP_STR_VAL_PROC);

    if (cmdRes.getResponse() instanceof GridClientTaskResultBean
        && ((GridClientTaskResultBean) cmdRes.getResponse()).getResult() instanceof String)
      cfg.registerJsonValueProcessor(cmdRes.getResponse().getClass(), "result", SKIP_STR_VAL_PROC);

    JSON json;

    try {
      json = JSONSerializer.toJSON(cmdRes, cfg);
    } catch (JSONException e) {
      U.error(log, "Failed to convert response to JSON: " + cmdRes, e);

      json = JSONSerializer.toJSON(new GridRestResponse(STATUS_FAILED, e.getMessage()), cfg);
    }

    try {
      if (log.isDebugEnabled())
        log.debug("Parsed command response into JSON object: " + json.toString(2));

      res.getWriter().write(json.toString());

      if (log.isDebugEnabled())
        log.debug(
            "Processed HTTP request [action=" + act + ", jsonRes=" + cmdRes + ", req=" + req + ']');
    } catch (IOException e) {
      U.error(log, "Failed to send HTTP response: " + json.toString(2), e);
    }
  }
      @Override
      public void onComplete(String response, Object state) {
        try {
          Log.i(TAG, response);
          json = Util.parseJson(response);

          // photos are in the form of a json array
          child = json.getJSONArray("data");

          int total = child.length();

          // contains links to photos
          links = new ArrayList<String>(total);

          // adds link to each photo to our list after replacing the "https" from url
          // DownloadManager does not support https in gingerbread
          for (int i = 0; i < total; i++) {
            photo_json = child.getJSONObject(i);

            if (dl_high_res_pics) {
              JSONArray image_set = photo_json.getJSONArray("images");

              // highest resolution picture has the index zero in the images jsonarray
              JSONObject highest_res_pic = image_set.getJSONObject(0);
              String http_replaced =
                  highest_res_pic.getString("source").replaceFirst("https", "http");
              links.add(i, http_replaced);
            } else {
              // source property of the json object points to the photo's link
              String http_replaced = photo_json.getString("source").replaceFirst("https", "http");
              links.add(i, http_replaced);
            }
          }

          download_photos.this.runOnUiThread(
              new Runnable() {
                public void run() {
                  //	mytask = new DownloadImageTask();
                  //	mytask.execute(links);
                  // start downloading using asynctask
                  // new DownloadImageTask().execute(links);
                  // downloadThread.setPath(path);
                  // downloadThread.setWholeTasks(links.size());
                  if (resume_file.exists()) {

                    initial_value = readProgress()[0];
                    final_value = links.size();
                    // case if the task is already completed
                    if (initial_value == final_value) {
                      completed = true;
                      resume_pause.setVisibility(View.GONE);

                      progress_download.setMax(final_value);
                      progress_download.setProgress(0); // bug in progress bar
                      progress_download.setProgress(initial_value);

                      progress_text.setText("Completed.");
                      mtext.setText(
                          getText(R.string.download_textview_message_1)
                              + " "
                              + path.toString()
                              + ". ");

                    }

                    // case if some of the photos are already downloaded
                    else {
                      progress_download.setMax(links.size());
                      // progress_download.setProgress(0);	//bug in progress bar
                      progress_download.setProgress(initial_value);

                      // N.B if i= initial_value -1, then one image will be downloaded again. so to
                      // save cost we start with i = initial_value
                      for (int i = initial_value; i < links.size(); i++) {
                        mtext.setText(
                            getText(R.string.download_textview_message_1)
                                + " "
                                + path.toString()
                                + ". ");
                        // downloadThread.setRunningTask(i + 1);
                        downloadThread.enqueueDownload(
                            new DownloadTask(
                                links.get(i), path, i + 1, final_value, download_photos.this));
                      }
                    }
                  }

                  // case if the task is entirely new task
                  else {
                    for (int i = 0; i < links.size(); i++) {
                      mtext.setText(
                          getText(R.string.download_textview_message_1)
                              + " "
                              + path.toString()
                              + ". ");
                      // downloadThread.setRunningTask(i + 1);
                      downloadThread.enqueueDownload(
                          new DownloadTask(
                              links.get(i), path, i + 1, links.size(), download_photos.this));
                    }
                  }
                }
              });
        } catch (JSONException ex) {
          Log.e(TAG, "JSONEXception : " + ex.getMessage());
        }
      }