public List<Tweet> readFile(String fileLocation, int limit, String fromTweetId) {
long startTime = System.currentTimeMillis();
int errors = 0;
int added = 0;
int ignored = 0;
int skipped = 0;
List<Tweet> tweets = new ArrayList<Tweet>();
try {
// Read gzFile
InputStream inputStream = new GZIPInputStream(new FileInputStream(fileLocation));
Reader decoder = new InputStreamReader(inputStream);
BufferedReader br = new BufferedReader(decoder);
String status;
while ((status = br.readLine()) != null) {
// Ignore garbage and log output lines, all statuses start with JSON bracket
if (!status.equals("") && status.charAt(0) == '{') {
try {
JSONObject jsonObject = new JSONObject(status);
// We use created_at as an indicator that this is a Tweet.
if (jsonObject.has("created_at")) {
Status statusObject = TwitterObjectFactory.createStatus(status);
Tweet tweet = this.getTweetObjectFromStatus(statusObject);
if (fromTweetId != null) {
if (fromTweetId.equals(tweet.getId())) {
this.log.write("StatusFileReader - Scanner pickup from " + fromTweetId);
fromTweetId = null;
} else {
skipped++;
}
continue;
}
added++;
tweets.add(tweet);
if (limit > 0 && added >= limit) {
break;
}
} else {
ignored++;
}
} catch (JSONException e) {
this.log.write(
"Exception in StatusFileReader: Json Parse Failure on: "
+ status
+ ", "
+ e.getMessage());
}
} else {
ignored++;
}
}
br.close();
decoder.close();
inputStream.close();
} catch (Exception e) {
this.log.write(
"Exception in StatusFileReader: Error Reading File: "
+ e.getClass().getName()
+ ": "
+ e.getMessage());
}
long runTimeSeconds = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - startTime);
double ops = ((double) added / (double) Math.max(runTimeSeconds, 1));
this.log.write(
"StatusFileReader - "
+ fileLocation
+ " processed in "
+ runTimeSeconds
+ "s. "
+ added
+ " ok / "
+ errors
+ " errors / "
+ ignored
+ " ignored / "
+ skipped
+ " skipped. "
+ ops
+ " ops. Limit: "
+ limit
+ ", Fetch: "
+ tweets.size());
return tweets;
}
/**
* Process HTTP request.
*
* @param act Action.
* @param req Http request.
* @param res Http response.
*/
private void processRequest(String act, HttpServletRequest req, HttpServletResponse res) {
res.setContentType("application/json");
res.setCharacterEncoding("UTF-8");
GridRestCommand cmd = command(req);
if (cmd == null) {
res.setStatus(HttpServletResponse.SC_BAD_REQUEST);
return;
}
if (!authChecker.apply(req.getHeader("X-Signature"))) {
res.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
return;
}
GridRestResponse cmdRes;
Map<String, Object> params = parameters(req);
try {
GridRestRequest cmdReq = createRequest(cmd, params, req);
if (log.isDebugEnabled()) log.debug("Initialized command request: " + cmdReq);
cmdRes = hnd.handle(cmdReq);
if (cmdRes == null)
throw new IllegalStateException("Received null result from handler: " + hnd);
byte[] sesTok = cmdRes.sessionTokenBytes();
if (sesTok != null) cmdRes.setSessionToken(U.byteArray2HexString(sesTok));
res.setStatus(HttpServletResponse.SC_OK);
} catch (Exception e) {
res.setStatus(HttpServletResponse.SC_OK);
U.error(log, "Failed to process HTTP request [action=" + act + ", req=" + req + ']', e);
cmdRes = new GridRestResponse(STATUS_FAILED, e.getMessage());
} catch (Throwable e) {
U.error(log, "Failed to process HTTP request [action=" + act + ", req=" + req + ']', e);
throw e;
}
JsonConfig cfg = new GridJettyJsonConfig();
// Workaround for not needed transformation of string into JSON object.
if (cmdRes.getResponse() instanceof String)
cfg.registerJsonValueProcessor(cmdRes.getClass(), "response", SKIP_STR_VAL_PROC);
if (cmdRes.getResponse() instanceof GridClientTaskResultBean
&& ((GridClientTaskResultBean) cmdRes.getResponse()).getResult() instanceof String)
cfg.registerJsonValueProcessor(cmdRes.getResponse().getClass(), "result", SKIP_STR_VAL_PROC);
JSON json;
try {
json = JSONSerializer.toJSON(cmdRes, cfg);
} catch (JSONException e) {
U.error(log, "Failed to convert response to JSON: " + cmdRes, e);
json = JSONSerializer.toJSON(new GridRestResponse(STATUS_FAILED, e.getMessage()), cfg);
}
try {
if (log.isDebugEnabled())
log.debug("Parsed command response into JSON object: " + json.toString(2));
res.getWriter().write(json.toString());
if (log.isDebugEnabled())
log.debug(
"Processed HTTP request [action=" + act + ", jsonRes=" + cmdRes + ", req=" + req + ']');
} catch (IOException e) {
U.error(log, "Failed to send HTTP response: " + json.toString(2), e);
}
}
@Override
public void onComplete(String response, Object state) {
try {
Log.i(TAG, response);
json = Util.parseJson(response);
// photos are in the form of a json array
child = json.getJSONArray("data");
int total = child.length();
// contains links to photos
links = new ArrayList<String>(total);
// adds link to each photo to our list after replacing the "https" from url
// DownloadManager does not support https in gingerbread
for (int i = 0; i < total; i++) {
photo_json = child.getJSONObject(i);
if (dl_high_res_pics) {
JSONArray image_set = photo_json.getJSONArray("images");
// highest resolution picture has the index zero in the images jsonarray
JSONObject highest_res_pic = image_set.getJSONObject(0);
String http_replaced =
highest_res_pic.getString("source").replaceFirst("https", "http");
links.add(i, http_replaced);
} else {
// source property of the json object points to the photo's link
String http_replaced = photo_json.getString("source").replaceFirst("https", "http");
links.add(i, http_replaced);
}
}
download_photos.this.runOnUiThread(
new Runnable() {
public void run() {
// mytask = new DownloadImageTask();
// mytask.execute(links);
// start downloading using asynctask
// new DownloadImageTask().execute(links);
// downloadThread.setPath(path);
// downloadThread.setWholeTasks(links.size());
if (resume_file.exists()) {
initial_value = readProgress()[0];
final_value = links.size();
// case if the task is already completed
if (initial_value == final_value) {
completed = true;
resume_pause.setVisibility(View.GONE);
progress_download.setMax(final_value);
progress_download.setProgress(0); // bug in progress bar
progress_download.setProgress(initial_value);
progress_text.setText("Completed.");
mtext.setText(
getText(R.string.download_textview_message_1)
+ " "
+ path.toString()
+ ". ");
}
// case if some of the photos are already downloaded
else {
progress_download.setMax(links.size());
// progress_download.setProgress(0); //bug in progress bar
progress_download.setProgress(initial_value);
// N.B if i= initial_value -1, then one image will be downloaded again. so to
// save cost we start with i = initial_value
for (int i = initial_value; i < links.size(); i++) {
mtext.setText(
getText(R.string.download_textview_message_1)
+ " "
+ path.toString()
+ ". ");
// downloadThread.setRunningTask(i + 1);
downloadThread.enqueueDownload(
new DownloadTask(
links.get(i), path, i + 1, final_value, download_photos.this));
}
}
}
// case if the task is entirely new task
else {
for (int i = 0; i < links.size(); i++) {
mtext.setText(
getText(R.string.download_textview_message_1)
+ " "
+ path.toString()
+ ". ");
// downloadThread.setRunningTask(i + 1);
downloadThread.enqueueDownload(
new DownloadTask(
links.get(i), path, i + 1, links.size(), download_photos.this));
}
}
}
});
} catch (JSONException ex) {
Log.e(TAG, "JSONEXception : " + ex.getMessage());
}
}