@Override
protected String doInBackground(String... urls) {
Document doc = null;
try {
Connection.Response res =
Jsoup.connect(urls[0])
.data("eid", mUserName)
.timeout(3000)
.data("pw", mPassword)
.data("submit", "Login")
.method(Method.POST)
.execute();
doc = res.parse();
// get the cookie
mCookieValue = res.cookie(COOKIE_TYPE);
mLoginResponse = doc.toString();
/* Log.w("response", doc.toString());*/
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return mLoginResponse;
}
@Override
public void generateSessionId() {
LOGGER.info("login to DMM");
try {
Connection.Response res =
Jsoup.connect("https://www.dmm.co.jp/my/")
.data("login_id", userId)
.data("password", password)
.data("sava_password", "1")
.data("save_login_id", "1")
.data("act", "commit")
.method(Method.POST)
.execute();
String sesId = res.cookie(SESSION_ID_KEY);
LOGGER.info("sessionId={}", sesId);
this.sessionId = sesId;
} catch (SocketTimeoutException e) {
LOGGER.warn("login failed", e);
generateSessionIdRetry(3000);
} catch (SSLHandshakeException e) {
LOGGER.warn("login failed", e);
generateSessionIdRetry(3000);
} catch (IOException e) {
throw new EgetException("failed to login", e);
}
}
@Override
protected void onPostExecute(Connection.Response response) {
if (response != null && response.statusCode() == 200) {
try {
Log.v(TAG, "Response to JSON request: " + response.body());
JSONObject root = new JSONObject(response.body());
boolean success = "success".equals(root.getString("type"));
int points = root.getInt("points");
IHasEnterableGiveaways fragment = getFragment();
fragment.onEnterLeaveResult(giveawayId, getWhat(), success, true);
// Update the points we have.
SteamGiftsUserData.getCurrent().setPoints(points);
if (fragment instanceof Fragment
&& "error".equals(root.getString("type"))
&& "Sync Required".equals(root.getString("msg"))) {
((Fragment) fragment)
.getActivity()
.startActivity(new Intent(((Fragment) fragment).getContext(), SyncActivity.class));
}
return;
} catch (JSONException e) {
Log.e(TAG, "Failed to parse JSON object", e);
}
}
getFragment().onEnterLeaveResult(giveawayId, getWhat(), null, false);
}
public static String extractContent(String url) {
try {
Connection connection = Jsoup.connect(url);
connection.userAgent(USER_AGENT);
connection.followRedirects(true);
connection.timeout(GET_TIMEOUT);
long start = System.currentTimeMillis();
Connection.Response response = connection.execute();
long diff = System.currentTimeMillis() - start;
int responseCode = response.statusCode();
if (response.statusCode() == OK) {
String body = response.body();
Logger.info(
"%s retrieved, content length %d, time %s sec.",
url, body.length(), FormatUtil.millis2Seconds(diff));
return response.body();
} else {
Logger.error("%s returned %d", url, responseCode);
return "";
}
} catch (IOException e) {
Logger.error(e, "%s cannot be read.", url);
return "";
}
}
@Override
protected String doInBackground(Void... params) {
String html = "";
try {
Connection.Response loginForm;
loginForm =
Jsoup.connect("https://ta.yrdsb.ca/yrdsb/").method(Connection.Method.GET).execute();
// Login to page using user/pass entered in MainActivity
Document document =
Jsoup.connect("https://ta.yrdsb.ca/yrdsb/")
.data("cookieexists", "false")
.data("username", MainActivity.usernameString)
.data("password", MainActivity.passwordString)
.data("submit", "Login")
.cookies(loginForm.cookies())
.post();
// Convert document into string for easier processing
html = document.toString();
Document doc = Jsoup.parse(html);
// for (int x = 0; x < doc.select("[width=85%], [border=0], [cellspacing=0],
// [cellpadding=5]").size(); x++){
System.out.println(
doc.select("[width=85%], [border=0], [cellspacing=0], [cellpadding=5]")
.select("tr")
.size());
// System.out.println(doc.select("[width=85%], [border=0], [cellspacing=0],
// [cellpadding=5]").select("tr").get(5));
// }
// Prepare array to store grades
grades = new ArrayList<>();
courses = new ArrayList<>();
// Regex to search html string for grades, then add to array
Pattern p = Pattern.compile("current mark\\s?=\\s?(\\d+\\.?\\d*)");
Matcher m = p.matcher(html);
while (m.find()) {
grades.add(new Double(m.group(1)));
}
Pattern p1 = Pattern.compile("([a-zA-Z]{3}[0-9]{1}[a-zA-Z]{1}[0-9]{1})");
Matcher m1 = p1.matcher(html);
while (m1.find()) {
courses.add(new String(m1.group(1)));
}
} catch (IOException e) {
e.printStackTrace();
}
return html;
}
@Test
public void test03() throws Exception {
Connection connect = Jsoup.connect("http://www.jb51.net/article/16829.htm");
Connection.Response execute = connect.execute();
String body = execute.body();
System.out.println(body);
Document parse = Jsoup.parse(body);
Elements elements = parse.select("[src$=.gif]");
for (Element element : elements) {
System.out.println(element.attr("src"));
}
}
protected Integer doInBackground(String... params) {
String url = params[0];
int i = 0;
try {
Connection.Response res = Jsoup.connect(url).userAgent("Mozilla").execute();
Document doc = res.parse();
Elements pages = doc.getElementsByClass("last");
i = Integer.parseInt(pages.get(1).childNode(0).toString());
} catch (Exception e) {
}
return i;
}
@Override
public OvercastPlayer getPlayerByUrl(URL url) {
try {
Connection.Response response =
Jsoup.connect(url.toString())
.method(Connection.Method.GET)
.userAgent(userAgent)
.timeout(connectionTimeout)
.execute();
return new ParsedPlayer(this, response.parse());
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
public DeletePost(
ArrayList<String> postID, ArrayList<String> gallID, String id, String pw, String user_no) {
String UA =
"Mozilla/5.0 (iPad; CPU OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53";
// 디씨 모바일은 UA로 접속 거르니까 모바일 UA 추가.
Map<String, String> cookies = new LoginDC(id, pw).cookies; // 로그인 세션 정보가 담긴 쿠키.
for (int i = 0; i < postID.size(); i++) {
try {
Connection.Response res =
Jsoup.connect("http://m.dcinside.com/_option_write.php")
.data(
"no",
postID.get(i).replace(" ", ""),
"id",
gallID.get(i).replace(" ", ""),
"user_no",
user_no.replace(" ", ""),
"page",
"",
"mode",
"board_del")
// 지울 정보 POST 하는 부분. 갤 ID / 원글 고유번호 / 자신의 고유번호 / 지울 요소가 글이라는 모드 설정 및 자잘한 요소 보냄.
.method(Connection.Method.POST)
.userAgent(UA)
.header("Content-Type", "application/x-www-form-urlencoded")
.header("Accept-Encoding", "gzip, deflate, br")
.header("Accept-Language", "en-US,en;q=0.5")
.header("Host", "m.dcinside.com")
.referrer(
URLEncoder.encode(
"http://m.dcinside.com/view.php?id=doosanbears_new&no=5497159", "utf-8"))
.cookies(cookies)
.execute(); // 제거 POST 부분. 딱히 건드려야 할 보안 연결은 없음.
// 디씨 모바일 접속 및 POST 시에는 헤더들을 많이 요구함(리퍼러 필수!). 하나라도 빠질 시 POST 에러 뜨므로 모두 추가.
System.out.println(res.body());
System.out.println(gallID.get(i) + " " + postID.get(i));
System.out.println("\n");
// Response Body가 1이면 성공했다는 뜻.
} catch (MalformedURLException e) {
e.printStackTrace();
} catch (IOException e1) {
System.out.println("Post has been already deleted.");
}
}
}
@Override
public OvercastPlayer getPlayerByName(String username) {
String url = "https://oc.tc/users/" + username;
try {
Connection.Response response =
Jsoup.connect(url)
.method(Connection.Method.GET)
.userAgent(userAgent)
.timeout(connectionTimeout)
.execute();
return new ParsedPlayer(this, response.parse());
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
@Override
public List<OvercastPlayer> getPlayersByUrl(List<URL> urls) {
List<OvercastPlayer> playerList = new ArrayList<OvercastPlayer>();
for (URL url : urls) {
try {
Connection.Response response =
Jsoup.connect(url.toString())
.method(Connection.Method.GET)
.userAgent(userAgent)
.timeout(connectionTimeout)
.execute();
playerList.add(new ParsedPlayer(this, response.parse()));
} catch (IOException e) {
e.printStackTrace();
}
}
return playerList;
}
@Override
public List<OvercastPlayer> getPlayersByName(List<String> usernames) {
List<OvercastPlayer> playerList = new ArrayList<OvercastPlayer>();
for (String username : usernames) {
String url = "https://oc.tc/users/" + username;
try {
Connection.Response response =
Jsoup.connect(url)
.method(Connection.Method.GET)
.userAgent(userAgent)
.timeout(connectionTimeout)
.execute();
playerList.add(new ParsedPlayer(this, response.parse()));
} catch (IOException e) {
e.printStackTrace();
}
}
return playerList;
}
/**
* @param connection Jsoup connection object
* @param method HTTP method
* @return Jsoup Connection.Response object
*/
public Connection.Response execute(Connection connection, Connection.Method method) {
Connection.Response response;
if (method != null) {
connection.method(method);
}
try {
System.out.println("Calling " + connection.request().url());
if (props.getMode() == Mode.TEST) {
return null;
}
response = connection.execute();
} catch (IOException e) {
e.printStackTrace();
return null;
}
this.cookies.putAll(response.cookies());
return response;
}
@Override
public List<OvercastPlayer> getPlayersByUuid(List<UUID> uuids) {
List<OvercastPlayer> playerList = new ArrayList<OvercastPlayer>();
for (UUID uuid : uuids) {
String url = "https://oc.tc/users/" + MojangUtil.getUsername(String.valueOf(uuid));
try {
Connection.Response response =
Jsoup.connect(url)
.method(Connection.Method.GET)
.userAgent(userAgent)
.timeout(connectionTimeout)
.execute();
playerList.add(new ParsedPlayer(this, response.parse()));
} catch (IOException e) {
e.printStackTrace();
}
}
return playerList;
}
public void saveFromResponse(Connection.Response response, String filename) throws IOException {
File folder = new File(OUTPUT_FOLDER);
if (!folder.exists()) {
if (!folder.mkdir()) {
throw new IOException("Could not mkdir : " + folder);
}
}
FileOutputStream out = (new FileOutputStream(new java.io.File(OUTPUT_FOLDER + filename)));
out.write(
response.bodyAsBytes()); // resultImageResponse.body() is where the image's contents are.
out.close();
}
// set up url, method, header, cookies
private void setupFromConnection(HttpURLConnection conn, Connection.Response previousResponse)
throws IOException {
method = Connection.Method.valueOf(conn.getRequestMethod());
url = conn.getURL();
statusCode = conn.getResponseCode();
statusMessage = conn.getResponseMessage();
contentType = conn.getContentType();
Map<String, List<String>> resHeaders = conn.getHeaderFields();
processResponseHeaders(resHeaders);
// if from a redirect, map previous response cookies into this response
if (previousResponse != null) {
for (Map.Entry<String, String> prevCookie : previousResponse.cookies().entrySet()) {
if (!hasCookie(prevCookie.getKey())) cookie(prevCookie.getKey(), prevCookie.getValue());
}
}
}
private void jButton1ActionPerformed(
java.awt.event.ActionEvent evt) { // GEN-FIRST:event_jButton1ActionPerformed
Connection.Response res = null;
String url_country;
String prefix;
try {
url_search = url_search + jTextField_search.getText() + "\"";
res = Jsoup.connect(url_search).method(Method.GET).timeout(20000).execute();
doc = res.parse();
switch (jComboBox_country.getSelectedIndex()) {
case 0:
url_country =
"http://ru.kompass.com/en/searchCompanies/facet?value=RU&label=%20Russian%20Federation&filterType=countrynational&searchType=ALL&checked=true";
prefix = "/ru";
break;
case 1:
url_country =
"http://ru.kompass.com/en/searchCompanies/facet?value=BR&label=Brazil&filterType=country&searchType=ALL&checked=true";
prefix = "/br";
break;
case 2:
url_country =
"http://ru.kompass.com/en/searchCompanies/facet?value=CN&label=China&filterType=country&searchType=ALL&checked=true";
prefix = "/cn";
break;
case 3:
url_country =
"http://ru.kompass.com/en/searchCompanies/facet?value=ZA&label=South%20Africa&filterType=country&searchType=ALL&checked=true";
prefix = "/za";
break;
case 4:
url_country =
"http://ru.kompass.com/en/searchCompanies/facet?value=IN&label=India&filterType=country&searchType=ALL&checked=true";
prefix = "/in";
break;
default:
url_country =
"http://ru.kompass.com/en/searchCompanies/facet?value=RU&label=%20Russian%20Federation&filterType=countrynational&searchType=ALL&checked=true";
prefix = "/ru";
}
doc =
Jsoup.connect(url_country)
.cookies(res.cookies())
.userAgent(
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.154 Safari/537.36")
.timeout(20000)
.get();
Elements sheet = doc.select("#paginatorDivId > li > a");
for (int i = 0; i < sheet.size(); i++) {
sheet_urls[i] = "http://ru.kompass.com" + sheet.get(i).attr("href");
jTextArea_sheets.append(sheet_urls[i] + "\n");
}
jTextArea_company.setText("");
count_urls = 0;
int count_sheets = sheet.size();
jLabel_sheets.setText(Integer.toString(count_sheets));
for (int s = 0; s < count_sheets; s++) {
doc =
Jsoup.connect(sheet_urls[s])
.cookies(res.cookies())
.userAgent(
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.154 Safari/537.36")
.timeout(20000)
.get();
Elements links = doc.select("div.details > h2 > a[href]");
for (int i = 0; i < links.size(); i++) {
String buf_str = links.get(i).attr("href");
urls[count_urls] =
"http://ru.kompass.com" + buf_str.substring(buf_str.indexOf(prefix, 0));
jTextArea_company.append(urls[count_urls] + "\n");
count_urls++;
}
jLabel_name.setText(doc.title());
jLabel1.setText(String.valueOf(count_urls));
}
} catch (IOException ex) {
Logger.getLogger(main_window.class.getName()).log(Level.SEVERE, null, ex);
}
} // GEN-LAST:event_jButton1ActionPerformed
public Document post() throws IOException {
req.method(Method.POST);
execute();
return res.parse();
}