// A general method for calling api and receiving response
private String callAPI(final JSONObject param, final String api) {
URL apiURL;
try {
apiURL = new URL(url + api);
HttpURLConnection conn;
conn = (HttpURLConnection) apiURL.openConnection();
conn.setRequestMethod("POST");
conn.setDoOutput(true);
conn.setRequestProperty("Content-Type", "application/json; charset=UTF-8");
OutputStreamWriter wr = new OutputStreamWriter(conn.getOutputStream());
wr.write(param.toString());
wr.flush();
BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream()));
String inputLine;
StringBuffer response = new StringBuffer();
while ((inputLine = in.readLine()) != null) {
response.append(inputLine);
}
in.close();
// System.out.println(response.toString());
return response.toString();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return null;
}
/**
* Retrieves a list of parameters from a passed request in comma delimited format. If the
* parameter is not found, an NPE is thrown.
*
* @param req Request object to look in
* @param name the name of the field from the html form
* @return the value from the html form corresponding to the name parameter
* @exception NullPointerException thrown if the name was not in the html form
*/
public static String getOptionalParameterValues(ServletRequest req, String name)
throws NullPointerException {
String[] values = req.getParameterValues(name);
if (values == null) {
return Config.EMPTY_STR;
}
StringBuffer ret = new StringBuffer();
for (int i = 0; i < values.length; i++) {
if (i > 0) {
ret.append(",");
}
ret.append(values[i]);
}
return ret.toString();
}
/**
* Retrieves a list of parameters from a passed request in comma delimited format. If the
* parameter is not found, an NPE is thrown.
*
* @param req Request object to look in
* @param name the name of the field from the html form
* @return the value from the html form corresponding to the name parameter
* @exception NullPointerException thrown if the name was not in the html form
*/
public static String getRequiredParameterValues(ServletRequest req, String name)
throws NullPointerException {
String[] values = req.getParameterValues(name);
if (values == null) {
throw new NullPointerException(
"This form requires a \""
+ name
+ "\" parameter, which was missing from the submitted request.");
}
StringBuffer ret = new StringBuffer();
for (int i = 0; i < values.length; i++) {
if (i > 0) {
ret.append(",");
}
ret.append(values[i]);
}
return ret.toString();
}
public static ArrayList<ContentValues> parsePosts(
TagNode aThread,
int aThreadId,
int unreadIndex,
int opId,
AwfulPreferences prefs,
int startIndex) {
ArrayList<ContentValues> result = new ArrayList<ContentValues>();
boolean lastReadFound = false;
int index = startIndex;
String update_time = new Timestamp(System.currentTimeMillis()).toString();
Log.v(TAG, "Update time: " + update_time);
try {
if (!Constants.isICS()
|| !prefs.inlineYoutube) { // skipping youtube support for now, it kinda sucks.
aThread = convertVideos(aThread);
}
TagNode[] postNodes = aThread.getElementsByAttValue("class", "post", true, true);
for (TagNode node : postNodes) {
// fyad status, to prevent processing postbody twice if we are in fyad
ContentValues post = new ContentValues();
post.put(THREAD_ID, aThreadId);
// We'll just reuse the array of objects rather than create
// a ton of them
int id = Integer.parseInt(node.getAttributeByName("id").replaceAll("post", ""));
post.put(ID, id);
post.put(AwfulProvider.UPDATED_TIMESTAMP, update_time);
post.put(POST_INDEX, index);
if (index > unreadIndex) {
post.put(PREVIOUSLY_READ, 0);
lastReadFound = true;
} else {
post.put(PREVIOUSLY_READ, 1);
}
index++;
post.put(IS_MOD, 0);
post.put(IS_ADMIN, 0);
TagNode[] postContent = node.getElementsHavingAttribute("class", true);
for (TagNode pc : postContent) {
if (pc.getAttributeByName("class").contains("author")) {
post.put(USERNAME, pc.getText().toString().trim());
}
if (pc.getAttributeByName("class").contains("role-mod")) {
post.put(IS_MOD, 1);
}
if (pc.getAttributeByName("class").contains("role-admin")) {
post.put(IS_ADMIN, 1);
}
if (pc.getAttributeByName("class").equalsIgnoreCase("title")
&& pc.getChildTags().length > 0) {
TagNode[] avatar = pc.getElementsByName("img", true);
if (avatar.length > 0) {
post.put(AVATAR, avatar[0].getAttributeByName("src"));
}
post.put(AVATAR_TEXT, pc.getText().toString().trim());
}
if (pc.getAttributeByName("class").equalsIgnoreCase("postbody")
|| pc.getAttributeByName("class").contains("complete_shit")) {
TagNode[] images = pc.getElementsByName("img", true);
for (TagNode img : images) {
// don't alter video mock buttons
if ((img.hasAttribute("class")
&& img.getAttributeByName("class").contains("videoPlayButton"))) {
continue;
}
boolean dontLink = false;
TagNode parent = img.getParent();
String src = img.getAttributeByName("src");
if ((parent != null && parent.getName().equals("a"))
|| (img.hasAttribute("class")
&& img.getAttributeByName("class")
.contains("nolink"))) { // image is linked, don't override
dontLink = true;
}
if (src.contains(".gif")) {
img.setAttribute(
"class",
(img.hasAttribute("class") ? img.getAttributeByName("class") + " " : "")
+ "gif");
}
if (img.hasAttribute("title")) {
if (!prefs.showSmilies) { // kill all emotes
String name = img.getAttributeByName("title");
img.setName("p");
img.addChild(new ContentNode(name));
}
} else {
if (!lastReadFound && prefs.hideOldImages || !prefs.imagesEnabled) {
if (!dontLink) {
img.setName("a");
img.setAttribute("href", src);
img.addChild(new ContentNode(src));
} else {
img.setName("p");
img.addChild(new ContentNode(src));
}
} else {
if (!dontLink) {
img.setName("a");
img.setAttribute("href", src);
TagNode newimg = new TagNode("img");
if (!prefs.imgurThumbnails.equals("d") && src.contains("i.imgur.com")) {
int lastSlash = src.lastIndexOf('/');
if (src.length() - lastSlash <= 9) {
int pos = src.length() - 4;
src = src.substring(0, pos) + prefs.imgurThumbnails + src.substring(pos);
}
}
newimg.setAttribute("src", src);
img.addChild(newimg);
}
}
}
}
StringBuffer fixedContent = new StringBuffer();
Matcher fixCharMatch = fixCharacters_regex.matcher(NetworkUtils.getAsString(pc));
while (fixCharMatch.find()) {
fixCharMatch.appendReplacement(fixedContent, "");
}
fixCharMatch.appendTail(fixedContent);
post.put(CONTENT, fixedContent.toString());
}
if (pc.getAttributeByName("class").equalsIgnoreCase("postdate")) {
post.put(
DATE,
NetworkUtils.unencodeHtml(pc.getText().toString())
.replaceAll("[^\\w\\s:,]", "")
.trim());
}
if (pc.getAttributeByName("class").equalsIgnoreCase("profilelinks")) {
TagNode[] links = pc.getElementsHavingAttribute("href", true);
if (links.length > 0) {
String href = links[0].getAttributeByName("href").trim();
String userId = href.substring(href.lastIndexOf("rid=") + 4);
post.put(USER_ID, userId);
if (Integer.toString(opId).equals(userId)) { // ugh
post.put(IS_OP, 1);
} else {
post.put(IS_OP, 0);
}
}
}
if (pc.getAttributeByName("class").equalsIgnoreCase("editedby")
&& pc.getChildTags().length > 0) {
post.put(EDITED, "<i>" + pc.getChildTags()[0].getText().toString() + "</i>");
}
}
TagNode[] editImgs = node.getElementsByAttValue("alt", "Edit", true, true);
if (editImgs.length > 0) {
post.put(EDITABLE, 1);
} else {
post.put(EDITABLE, 0);
}
result.add(post);
}
Log.i(
TAG,
Integer.toString(postNodes.length) + " posts found, " + result.size() + " posts parsed.");
} catch (Exception e) {
e.printStackTrace();
}
return result;
}
public void doProcess(HttpServletRequest req, HttpServletResponse res, boolean isPost) {
StringBuffer bodyContent = null;
OutputStream out = null;
PrintWriter writer = null;
String serviceKey = null;
try {
BufferedReader in = req.getReader();
String line = null;
while ((line = in.readLine()) != null) {
if (bodyContent == null) bodyContent = new StringBuffer();
bodyContent.append(line);
}
} catch (Exception e) {
}
try {
if (requireSession) {
// check to see if there was a session created for this request
// if not assume it was from another domain and blow up
// Wrap this to prevent Portlet exeptions
HttpSession session = req.getSession(false);
if (session == null) {
res.setStatus(HttpServletResponse.SC_FORBIDDEN);
return;
}
}
serviceKey = req.getParameter("id");
// only to preven regressions - Remove before 1.0
if (serviceKey == null) serviceKey = req.getParameter("key");
// check if the services have been loaded or if they need to be reloaded
if (services == null || configUpdated()) {
getServices(res);
}
String urlString = null;
String xslURLString = null;
String userName = null;
String password = null;
String format = "json";
String callback = req.getParameter("callback");
String urlParams = req.getParameter("urlparams");
String countString = req.getParameter("count");
// encode the url to prevent spaces from being passed along
if (urlParams != null) {
urlParams = urlParams.replace(' ', '+');
}
try {
if (services.has(serviceKey)) {
JSONObject service = services.getJSONObject(serviceKey);
// default to the service default if no url parameters are specified
if (urlParams == null && service.has("defaultURLParams")) {
urlParams = service.getString("defaultURLParams");
}
String serviceURL = service.getString("url");
// build the URL
if (urlParams != null && serviceURL.indexOf("?") == -1) {
serviceURL += "?";
} else if (urlParams != null) {
serviceURL += "&";
}
String apikey = "";
if (service.has("username")) userName = service.getString("username");
if (service.has("password")) password = service.getString("password");
if (service.has("apikey")) apikey = service.getString("apikey");
urlString = serviceURL + apikey;
if (urlParams != null) urlString += "&" + urlParams;
if (service.has("xslStyleSheet")) {
xslURLString = service.getString("xslStyleSheet");
}
}
// code for passing the url directly through instead of using configuration file
else if (req.getParameter("url") != null) {
String serviceURL = req.getParameter("url");
// build the URL
if (urlParams != null && serviceURL.indexOf("?") == -1) {
serviceURL += "?";
} else if (urlParams != null) {
serviceURL += "&";
}
urlString = serviceURL;
if (urlParams != null) urlString += urlParams;
} else {
writer = res.getWriter();
if (serviceKey == null)
writer.write("XmlHttpProxyServlet Error: id parameter specifying serivce required.");
else
writer.write(
"XmlHttpProxyServlet Error : service for id '" + serviceKey + "' not found.");
writer.flush();
return;
}
} catch (Exception ex) {
getLogger().severe("XmlHttpProxyServlet Error loading service: " + ex);
}
Map paramsMap = new HashMap();
paramsMap.put("format", format);
// do not allow for xdomain unless the context level setting is enabled.
if (callback != null && allowXDomain) {
paramsMap.put("callback", callback);
}
if (countString != null) {
paramsMap.put("count", countString);
}
InputStream xslInputStream = null;
if (urlString == null) {
writer = res.getWriter();
writer.write(
"XmlHttpProxyServlet parameters: id[Required] urlparams[Optional] format[Optional] callback[Optional]");
writer.flush();
return;
}
// default to JSON
res.setContentType(responseContentType);
out = res.getOutputStream();
// get the stream for the xsl stylesheet
if (xslURLString != null) {
// check the web root for the resource
URL xslURL = null;
xslURL = ctx.getResource(resourcesDir + "xsl/" + xslURLString);
// if not in the web root check the classpath
if (xslURL == null) {
xslURL =
XmlHttpProxyServlet.class.getResource(classpathResourcesDir + "xsl/" + xslURLString);
}
if (xslURL != null) {
xslInputStream = xslURL.openStream();
} else {
String message =
"Could not locate the XSL stylesheet provided for service id "
+ serviceKey
+ ". Please check the XMLHttpProxy configuration.";
getLogger().severe(message);
try {
out.write(message.getBytes());
out.flush();
return;
} catch (java.io.IOException iox) {
}
}
}
if (!isPost) {
xhp.doGet(urlString, out, xslInputStream, paramsMap, userName, password);
} else {
if (bodyContent == null)
getLogger()
.info(
"XmlHttpProxyServlet attempting to post to url "
+ urlString
+ " with no body content");
xhp.doPost(
urlString,
out,
xslInputStream,
paramsMap,
bodyContent.toString(),
req.getContentType(),
userName,
password);
}
} catch (Exception iox) {
iox.printStackTrace();
getLogger().severe("XmlHttpProxyServlet: caught " + iox);
try {
writer = res.getWriter();
writer.write(iox.toString());
writer.flush();
} catch (java.io.IOException ix) {
ix.printStackTrace();
}
return;
} finally {
try {
if (out != null) out.close();
if (writer != null) writer.close();
} catch (java.io.IOException iox) {
}
}
}
/**
* Sends an HTTP GET request to a url
*
* @param url the url
* @return - HTTP response code
*/
public int sendGetRequest(String url)
throws IOException, OAuthMessageSignerException, OAuthExpectationFailedException,
OAuthCommunicationException {
System.out.println("url in sendrequest= " + url);
int responseCode = 500;
try {
HttpURLConnection uc = getConnection(url);
responseCode = uc.getResponseCode();
if (200 == responseCode || 401 == responseCode || 404 == responseCode) {
BufferedReader rd =
new BufferedReader(
new InputStreamReader(
responseCode == 200 ? uc.getInputStream() : uc.getErrorStream()));
StringBuffer sb = new StringBuffer();
String line;
while ((line = rd.readLine()) != null) {
System.out.println("line = " + line);
sb.append(line);
System.out.println("sb.append = " + sb);
}
String response = sb.toString();
try {
JSONObject json = new JSONObject(response);
System.out.println("\nResults:");
System.out.println(
"Total results = "
+ json.getJSONObject("bossresponse")
.getJSONObject("web")
.getString("totalresults"));
System.out.println();
JSONArray ja =
json.getJSONObject("bossresponse").getJSONObject("web").getJSONArray("results");
BufferedWriter out = new BufferedWriter(new FileWriter("outfile.csv", true));
String str = "";
System.out.println("\nResults:");
for (int i = 0; i < ja.length(); i++) {
// System.out.print((i+1) + ". ");
JSONObject j = ja.getJSONObject(i);
str += j.getString("url") + "," + j.getString("abstract") + "\n";
out.write(str);
}
} catch (Exception e) {
System.err.println("Something went wrong...");
e.printStackTrace();
}
rd.close();
setResponseBody(sb.toString());
}
} catch (MalformedURLException ex) {
throw new IOException(url + " is not valid");
} catch (IOException ie) {
throw new IOException("IO Exception " + ie.getMessage());
}
return responseCode;
}
protected String[] doInBackground(String... params) {
HttpURLConnection urlConnection = null;
// These two need to be declared outside the try/catch
// so that they can be closed in the finally block
BufferedReader reader = null;
// Will contain the raw JSON response as a string.
String forecastJsonStr = null;
String location = "";
if (params.length > 0) location = params[0];
// Log.v(LOG_TAG, "Location value: " + location);
String format = "json";
String units = "metric";
int numDays = 7;
String apiKey = "bd82977b86bf27fb59a04b61b657fb6f";
try {
// Construct the URL for the OpenWeatherMap query
// Possible parameters are avaiable at OWM's forecast API page, at
// http://openweathermap.org/API#forecast
final String FORECAST_BASE_URL = "http://api.openweathermap.org/data/2.5/forecast/daily?";
final String QUERY_PARAM = "zip";
final String FORMAT_PARAM = "mode";
final String UNIT_PARAM = "units";
final String DAYS_PARAM = "cnt";
final String KEY_PARAM = "appid";
Uri builtUri =
Uri.parse(FORECAST_BASE_URL)
.buildUpon()
.appendQueryParameter(QUERY_PARAM, location + ",us")
.appendQueryParameter(FORMAT_PARAM, format)
.appendQueryParameter(UNIT_PARAM, units)
.appendQueryParameter(DAYS_PARAM, Integer.toString(numDays))
.appendQueryParameter(KEY_PARAM, apiKey)
.build();
URL url = new URL(builtUri.toString());
// Log.v(LOG_TAG,"Built URI: " + builtUri.toString());
// URL url = new
// URL("http://api.openweathermap.org/data/2.5/forecast/daily?q=NewYork,us&mode=json&units=metric&cnt=7&appid=bd82977b86bf27fb59a04b61b657fb6f");
// Create the request to OpenWeatherMap, and open the connection
urlConnection = (HttpURLConnection) url.openConnection();
urlConnection.setRequestMethod("GET");
urlConnection.connect();
// Read the input stream into a String
InputStream inputStream = urlConnection.getInputStream();
StringBuffer buffer = new StringBuffer();
if (inputStream == null) {
// Nothing to do.
return null;
}
reader = new BufferedReader(new InputStreamReader(inputStream));
String line;
while ((line = reader.readLine()) != null) {
// Since it's JSON, adding a newline isn't necessary (it won't affect parsing)
// But it does make debugging a *lot* easier if you print out the completed
// buffer for debugging.
buffer.append(line + "\n");
}
if (buffer.length() == 0) {
// Stream was empty. No point in parsing.
return null;
}
forecastJsonStr = buffer.toString();
} catch (IOException e) {
Log.e(LOG_TAG, "Error ", e);
// If the code didn't successfully get the weather data, there's no point in attemping
// to parse it.
return null;
} finally {
if (urlConnection != null) {
urlConnection.disconnect();
}
if (reader != null) {
try {
reader.close();
} catch (final IOException e) {
Log.e(LOG_TAG, "Error closing stream", e);
}
}
}
try {
return getWeatherDataFromJson(forecastJsonStr, numDays);
} catch (JSONException e) {
Log.e(LOG_TAG, e.getMessage(), e);
e.printStackTrace();
}
return null;
}