/**
* Send the command to Solr using a GET
*
* @param queryString
* @param url
* @return
* @throws IOException
*/
private static String sendGetCommand(String queryString, String url) throws IOException {
String results = null;
HttpClient client = new HttpClient();
GetMethod get = new GetMethod(url);
get.setQueryString(queryString.trim());
client.executeMethod(get);
try {
// Execute the method.
int statusCode = client.executeMethod(get);
if (statusCode != HttpStatus.SC_OK) {
System.err.println("Method failed: " + get.getStatusLine());
results = "Method failed: " + get.getStatusLine();
}
results = getStringFromStream(get.getResponseBodyAsStream());
} catch (HttpException e) {
System.err.println("Fatal protocol violation: " + e.getMessage());
e.printStackTrace();
} catch (IOException e) {
System.err.println("Fatal transport error: " + e.getMessage());
e.printStackTrace();
} finally {
// Release the connection.
get.releaseConnection();
}
return results;
}
public static void main(String[] args) {
HttpClient client = new HttpClient(); // Create an instance of HttpClient.
GetMethod method = new GetMethod(url); // Create a method instance.
method
.getParams()
.setParameter(
HttpMethodParams.RETRY_HANDLER,
new DefaultHttpMethodRetryHandler(
3, false)); // Provide custom retry handler is necessary
try {
int statusCode = client.executeMethod(method); // Execute the method.
if (statusCode != HttpStatus.SC_OK) {
System.err.println("Method failed: " + method.getStatusLine());
}
byte[] responseBody = method.getResponseBody(); // Read the response body.
System.out.println(
new String(
responseBody)); // Deal with the response.// Use caution: ensure correct character
// encoding and is not binary data
} catch (HttpException e) {
System.err.println("Fatal protocol violation: " + e.getMessage());
e.printStackTrace();
} catch (IOException e) {
System.err.println("Fatal transport error: " + e.getMessage());
e.printStackTrace();
} finally {
method.releaseConnection(); // Release the connection.
}
}
public List<DatastreamProfile> getDatastreamHistory(String pid, String dsId)
throws HttpException, IOException, FedoraException, SAXException,
ParserConfigurationException, XPathExpressionException {
String url =
this.fedoraBaseUrl + "/objects/" + pid + "/datastreams/" + dsId + "/history?format=xml";
GetMethod get = new GetMethod(url);
this.client.executeMethod(get);
if (get.getStatusCode() == 200) {
List<DatastreamProfile> history = new ArrayList<DatastreamProfile>();
Document xml = this.getDocumentBuilder().parse(get.getResponseBodyAsStream());
NodeList nl =
(NodeList)
this.getXPath()
.evaluate(
"/fedora-management:datastreamHistory/fedora-management:datastreamProfile",
xml,
XPathConstants.NODESET);
for (int i = 0; i < nl.getLength(); i++) {
Element el = (Element) nl.item(i);
history.add(new DatastreamProfile(this, el));
}
return history;
} else {
throw new FedoraException("REST action \"" + url + "\" failed: " + get.getStatusLine());
}
}
/**
* Fetches the object profile XML and parse out the property value for the given property.
*
* @param pid the pid of the object to be queried
* @param dsName the name of the datastream who's property is being queried
* @param prop the property to query.
* @return the value of the property
*/
public String getDatastreamProperty(
String pid, String dsName, DatastreamProfile.DatastreamProperty prop)
throws HttpException, IOException, SAXException, ParserConfigurationException,
FedoraException, XPathExpressionException {
String url = this.fedoraBaseUrl + "/objects/" + pid + "/datastreams/" + dsName + "?format=xml";
GetMethod get = new GetMethod(url);
this.client.executeMethod(get);
if (get.getStatusCode() == 200) {
Document xml = this.getDocumentBuilder().parse(get.getResponseBodyAsStream());
// compatible with fedora 3.4
String value =
(String)
this.getXPath()
.evaluate(
"/fedora-management:datastreamProfile/fedora-management:"
+ prop.getPropertyName(),
xml,
XPathConstants.STRING);
if (value != null) {
return value;
} else {
// compatible with fedora 3.2
return (String)
this.getXPath()
.evaluate(
"/datastreamProfile/" + prop.getPropertyName(), xml, XPathConstants.STRING);
}
} else {
throw new FedoraException("REST action \"" + url + "\" failed: " + get.getStatusLine());
}
}
private String getXmlToken() {
String token = null;
HttpClient httpclient = getHttpClient();
GetMethod get = new GetMethod(restUrl.getTokenUrl());
get.addRequestHeader("Accept", "application/xml");
NameValuePair userParam = new NameValuePair(USERNAME_PARAM, satelite.getUser());
NameValuePair passwordParam = new NameValuePair(PASSWORD_PARAM, satelite.getPassword());
NameValuePair[] params = new NameValuePair[] {userParam, passwordParam};
get.setQueryString(params);
try {
int statusCode = httpclient.executeMethod(get);
if (statusCode != HttpStatus.SC_OK) {
logger.error("Method failed: " + get.getStatusLine());
}
token = parseToken(get.getResponseBodyAsString());
} catch (HttpException e) {
logger.error(e.getMessage());
} catch (IOException e) {
logger.error(e.getMessage());
} catch (ParserConfigurationException e) {
logger.error(e.getMessage());
} catch (SAXException e) {
logger.error(e.getMessage());
} finally {
get.releaseConnection();
}
return token;
}
/**
* Retrieves the build number of this CloudTest server. Postcondition: The build number returned
* is never null.
*/
public VersionNumber getBuildNumber() throws IOException {
if (url == null) {
// User didn't enter a value in the Configure Jenkins page.
// Nothing we can do.
throw new IllegalStateException("No URL has been configured for this CloudTest server.");
}
final String[] v = new String[1];
try {
HttpClient hc = createClient();
GetMethod get = new GetMethod(url);
hc.executeMethod(get);
if (get.getStatusCode() != 200) {
throw new IOException(get.getStatusLine().toString());
}
SAXParser sp = SAXParserFactory.newInstance().newSAXParser();
sp.parse(
get.getResponseBodyAsStream(),
new DefaultHandler() {
@Override
public InputSource resolveEntity(String publicId, String systemId)
throws IOException, SAXException {
if (systemId.endsWith(".dtd")) return new InputSource(new StringReader(""));
return null;
}
@Override
public void startElement(
String uri, String localName, String qName, Attributes attributes)
throws SAXException {
if (qName.equals("meta")) {
if ("buildnumber".equals(attributes.getValue("name"))) {
v[0] = attributes.getValue("content");
throw new SAXException("found");
}
}
}
});
LOGGER.warning("Build number not found in " + url);
} catch (SAXException e) {
if (v[0] != null) return new VersionNumber(v[0]);
LOGGER.log(Level.WARNING, "Failed to load " + url, e);
} catch (ParserConfigurationException e) {
throw new Error(e);
}
// If we reach this point, then we failed to extract the build number.
throw new IOException(
"Failed to extract build number from \'"
+ this.getDescriptor().getDisplayName()
+ "\': <"
+ url
+ ">.");
}
private String sendRequest(TicketsRequest request) throws HttpException, IOException {
String html = "";
GetMethod get = new GetMethod(url2);
client.executeMethod(get);
int statusCodeInit = client.executeMethod(get);
if (statusCodeInit != HttpStatus.SC_OK) {
logger.error("sendRequest method failed. Get method failed: " + get.getStatusLine());
return null;
} else {
html = IOUtils.toString(get.getResponseBodyAsStream(), "UTF-8");
}
parseToken(html);
client.getHttpConnectionManager().getParams().setSoTimeout(10000);
PostMethod post = new PostMethod(URL);
post.addRequestHeader("Accept", "*/*");
post.addRequestHeader("Accept-Encoding", "gzip, deflate");
post.addRequestHeader("Accept-Language", "uk,ru;q=0.8,en-US;q=0.5,en;q=0.3");
// post.addRequestHeader("Cache-Control", "no-cache");
post.addRequestHeader("Connection", "keep-alive");
// post.addRequestHeader("Content-Length", "288"); //202. 196
post.addRequestHeader("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
post.addRequestHeader("GV-Ajax", "1");
post.addRequestHeader("GV-Referer", "http://booking.uz.gov.ua/");
post.addRequestHeader("GV-Screen", "1920x1080");
post.addRequestHeader("GV-Token", token);
post.addRequestHeader("GV-Unique-Host", "1");
post.addRequestHeader("Host", "booking.uz.gov.ua");
// post.addRequestHeader("Pragma", "no-cache");
post.addRequestHeader("Origin", "http://booking.uz.gov.ua");
post.addRequestHeader("Referer", "http://booking.uz.gov.ua/");
post.addRequestHeader(
"User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:43.0) Gecko/20100101 Firefox/39.0");
post.addParameter("another_ec", "0");
post.addParameter("date_dep", new SimpleDateFormat("dd.MM.yyyy").format(request.date));
post.addParameter("search", "");
post.addParameter("station_from", request.from.getName());
post.addParameter("station_id_from", request.from.getStationId());
post.addParameter("station_id_till", request.till.getStationId());
post.addParameter("station_till", request.till.getName());
post.addParameter("time_dep", "00:00");
post.addParameter("time_dep_till", "");
int statusCode = client.executeMethod(post);
if (statusCode != HttpStatus.SC_OK) {
logger.error("sendRequest method failed. Post method failed: " + post.getStatusLine());
return null;
}
return post.getResponseBodyAsString();
}
public static void commit(String updateUrl) throws IOException {
String url = updateUrl + "?stream.body=%3Ccommit/%3E";
GetMethod get = new GetMethod(url);
try {
HttpClient client = new HttpClient();
client.executeMethod(get);
int status = get.getStatusCode();
if (status != HttpStatus.SC_OK) {
throw new RuntimeException("REST action \"" + url + "\" failed: " + get.getStatusLine());
}
} finally {
get.releaseConnection();
}
}
/* 下载 url 指向的网页 */
public String downloadFile(String url) {
String filePath = null;
/* 1.生成 HttpClinet 对象并设置参数 */
HttpClient httpClient = new HttpClient();
// 设置 Http 连接超时 5s
httpClient.getHttpConnectionManager().getParams().setConnectionTimeout(5000);
/* 2.生成 GetMethod 对象并设置参数 */
GetMethod getMethod = new GetMethod(url);
// 设置 get 请求超时 5s
getMethod.getParams().setParameter(HttpMethodParams.SO_TIMEOUT, 5000);
// 设置请求重试处理
getMethod
.getParams()
.setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler());
/* 3.执行 HTTP GET 请求 */
try {
int statusCode = httpClient.executeMethod(getMethod);
// 判断访问的状态码
if (statusCode != HttpStatus.SC_OK) {
System.err.println("Method failed: " + getMethod.getStatusLine());
filePath = null;
}
/* 4.处理 HTTP 响应内容 */
byte[] responseBody = getMethod.getResponseBody(); // 读取为字节数组
// 根据网页 url 生成保存时的文件名
filePath =
"temp\\" + getFileNameByUrl(url, getMethod.getResponseHeader("Content-Type").getValue());
// createFile(filePath);
FileUtils.createFile(filePath);
saveToLocal(responseBody, filePath, false);
} catch (HttpException e) {
// 发生致命的异常，可能是协议不对或者返回的内容有问题
System.out.println("Please check your provided http address!");
e.printStackTrace();
} catch (IOException e) {
// 发生网络异常
e.printStackTrace();
} finally {
// 释放连接
getMethod.releaseConnection();
}
return filePath;
}
// ���� URL ָ�����ҳ
public String downloadFile(String url) {
String filePath = null;
// 1.���� HttpClinet��������
HttpClient httpClient = new HttpClient();
// ���� HTTP���ӳ�ʱ 5s
httpClient.getHttpConnectionManager().getParams().setConnectionTimeout(5000);
// 2.���� GetMethod��������
GetMethod getMethod = new GetMethod(url);
// ���� get����ʱ 5s
getMethod.getParams().setParameter(HttpMethodParams.SO_TIMEOUT, 5000);
// �����������Դ���
getMethod
.getParams()
.setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler());
// 3.ִ��GET����
try {
int statusCode = httpClient.executeMethod(getMethod);
// �жϷ��ʵ�״̬��
if (statusCode != HttpStatus.SC_OK) {
System.err.println("Method failed: " + getMethod.getStatusLine());
filePath = null;
}
// 4.���� HTTP ��Ӧ����
byte[] responseBody = getMethod.getResponseBody(); // ��ȡΪ�ֽ�����
// ������ҳ url ���ɱ���ʱ���ļ���
filePath =
"temp\\" + getFileNameByUrl(url, getMethod.getResponseHeader("Content-Type").getValue());
File file = new File(filePath);
if (!file.getParentFile().exists()) {
file.getParentFile().mkdir();
}
saveToLocal(responseBody, filePath);
} catch (HttpException e) {
// �����������쳣��������Э�鲻�Ի��߷��ص�����������
System.out.println("�������http��ַ�Ƿ���ȷ");
e.printStackTrace();
} catch (IOException e) {
// ���������쳣
e.printStackTrace();
} finally {
// �ͷ�����
getMethod.releaseConnection();
}
return filePath;
}
public static String getGetResponse(String url) {
String html = "";
GetMethod getMethod = new GetMethod(url);
try {
int statusCode = httpClient.executeMethod(getMethod);
if (statusCode != HttpStatus.SC_OK) {
System.err.println("Method failed: " + getMethod.getStatusLine());
}
html = getMethod.getResponseBodyAsString();
} catch (HttpException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} finally {
getMethod.releaseConnection();
}
return html;
}
@Override
public void process(Exchange exchange) throws Exception {
String q = query;
String location = exchange.getIn().getHeader(WeatherConstants.WEATHER_LOCATION, String.class);
if (location != null) {
q = getEndpoint().getConfiguration().getQuery(location);
}
HttpClient httpClient = ((WeatherComponent) getEndpoint().getComponent()).getHttpClient();
GetMethod method = new GetMethod(q);
try {
log.debug("Going to execute the Weather query {}", q);
int statusCode = httpClient.executeMethod(method);
if (statusCode != HttpStatus.SC_OK) {
throw new IllegalStateException(
"Got the invalid http status value '"
+ method.getStatusLine()
+ "' as the result of the query '"
+ query
+ "'");
}
String weather =
getEndpoint()
.getCamelContext()
.getTypeConverter()
.mandatoryConvertTo(String.class, method.getResponseBodyAsStream());
log.debug("Got back the Weather information {}", weather);
if (ObjectHelper.isEmpty(weather)) {
throw new IllegalStateException(
"Got the unexpected value '" + weather + "' as the result of the query '" + q + "'");
}
String header = getEndpoint().getConfiguration().getHeaderName();
if (header != null) {
exchange.getIn().setHeader(header, weather);
} else {
exchange.getIn().setBody(weather);
}
exchange.getIn().setHeader(WeatherConstants.WEATHER_QUERY, q);
} finally {
method.releaseConnection();
}
}
@Override
public void emitTuples() {
try {
int statusCode = client.executeMethod(method);
if (statusCode != HttpStatus.SC_OK) {
logger.error("Method failed: " + method.getStatusLine());
} else {
InputStream istream = method.getResponseBodyAsStream();
// Process response
InputStreamReader isr = new InputStreamReader(istream);
CSVReader reader = new CSVReader(isr);
List<String[]> myEntries = reader.readAll();
for (String[] stringArr : myEntries) {
ArrayList<String> tuple = new ArrayList<String>(Arrays.asList(stringArr));
if (tuple.size() != 4) {
return;
}
// input csv is <Symbol>,<Price>,<Volume>,<Time>
String symbol = tuple.get(0);
double currentPrice = Double.valueOf(tuple.get(1));
long currentVolume = Long.valueOf(tuple.get(2));
String timeStamp = tuple.get(3);
long vol = currentVolume;
// Sends total volume in first tick, and incremental volume afterwards.
if (lastVolume.containsKey(symbol)) {
vol -= lastVolume.get(symbol);
}
if (vol > 0 || outputEvenIfZeroVolume) {
price.emit(new KeyValPair<String, Double>(symbol, currentPrice));
volume.emit(new KeyValPair<String, Long>(symbol, vol));
time.emit(new KeyValPair<String, String>(symbol, timeStamp));
lastVolume.put(symbol, currentVolume);
}
}
}
Thread.sleep(readIntervalMillis);
} catch (InterruptedException ex) {
logger.debug(ex.toString());
} catch (IOException ex) {
logger.debug(ex.toString());
}
}
void loadBinary(String uri, int index, Target target) throws RepositoryException, IOException {
GetMethod method = new GetMethod(uri);
try {
int statusCode = client.executeMethod(method);
if (statusCode == DavServletResponse.SC_OK) {
target.setStream(method.getResponseBodyAsStream());
} else {
throw ExceptionConverter.generate(
new DavException(
statusCode,
("Unable to load binary at "
+ uri
+ " - Status line = "
+ method.getStatusLine().toString())));
}
} finally {
method.releaseConnection();
}
}
/**
* Get data set from CKAN repository
*
* <p>4 Big Source --> Direct Link
*/
@SuppressWarnings("unchecked")
private void getPackageList() {
HttpClient client = new HttpClient();
LOG.info(split.getAction());
GetMethod method = new GetMethod(split.getAction());
method
.getParams()
.setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false));
method.setRequestHeader("User-Agent", "Hammer Project - SantaMaria crawler");
method
.getParams()
.setParameter(HttpMethodParams.USER_AGENT, "Hammer Project - SantaMaria crawler");
try {
int statusCode = client.executeMethod(method);
if (statusCode != HttpStatus.SC_OK) {
throw new Exception("Method failed: " + method.getStatusLine());
}
byte[] responseBody = method.getResponseBody();
LOG.debug(new String(responseBody));
setOutput(new String(responseBody));
Document document = Document.parse(getOutput());
if (document.containsKey("result")) {
ArrayList<Document> docs =
(ArrayList<Document>) ((Document) document.get("result")).get("results");
for (Document doc : docs) {
this.dataset.add(doc.getString("id"));
}
LOG.info("SANTA MARIA CKAN3 RECORD READER found" + this.dataset.size());
}
} catch (Exception e) {
LOG.error(e);
} finally {
method.releaseConnection();
}
}
public static void executeHttpGet() throws Exception {
// Create an instance of HttpClient.
HttpClient client = new HttpClient();
// Create a method instance.
GetMethod method = new GetMethod(url);
method.addRequestHeader("command", "addCredit");
// method.addRequestHeader("value", "addCredit");
// Provide custom retry handler is necessary
method
.getParams()
.setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false));
try {
// Execute the method.
int statusCode = client.executeMethod(method);
if (statusCode != HttpStatus.SC_OK) {
System.err.println("Method failed: " + method.getStatusLine());
}
// Read the response body.
byte[] responseBody = method.getResponseBody();
System.out.println(responseBody);
System.out.println(method.getResponseHeader("response"));
// Deal with the response.
// Use caution: ensure correct character encoding and is not binary data
System.out.println(new String(responseBody));
} catch (HttpException e) {
System.err.println("Fatal protocol violation: " + e.getMessage());
e.printStackTrace();
} catch (IOException e) {
System.err.println("Fatal transport error: " + e.getMessage());
e.printStackTrace();
} finally {
// Release the connection.
method.releaseConnection();
}
}
/**
* Test
*
* @param pArgs
* @throws Exception
*/
@SuppressWarnings("unchecked")
public static void main(String[] pArgs) throws Exception {
GetCountByCkan3("http://catalog.data.gov/api/action/package_search?start=0&rows=1");
HttpClient client = new HttpClient();
GetMethod method =
new GetMethod("http://catalog.data.gov/api/action/package_search?start=0&rows=10");
method
.getParams()
.setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false));
method.setRequestHeader("User-Agent", "Hammer Project - SantaMaria crawler");
method
.getParams()
.setParameter(HttpMethodParams.USER_AGENT, "Hammer Project - SantaMaria crawler");
try {
int statusCode = client.executeMethod(method);
if (statusCode != HttpStatus.SC_OK) {
throw new Exception("Method failed: " + method.getStatusLine());
}
byte[] responseBody = method.getResponseBody();
LOG.info(new String(responseBody));
Document document = Document.parse(new String(responseBody));
if (document.containsKey("result")) {
ArrayList<Document> docs =
(ArrayList<Document>) ((Document) document.get("result")).get("results");
for (Document doc : docs) {
LOG.info(doc.getString("id"));
}
}
} catch (Exception e) {
LOG.error(e);
} finally {
method.releaseConnection();
}
}
public InputStream getDatastream(String pid, String dsName, String asOfDateTime)
throws IOException {
String url =
this.fedoraBaseUrl
+ "/objects/"
+ pid
+ "/datastreams/"
+ dsName
+ "/content"
+ (asOfDateTime != null
? "?asOfDateTime=" + URLEncoder.encode(asOfDateTime, "UTF-8")
: "");
GetMethod get = new GetMethod(url);
this.client.executeMethod(get);
int status = this.client.executeMethod(get);
if (status != HttpStatus.SC_CREATED && status != HttpStatus.SC_OK) {
throw new RuntimeException("REST action \"" + url + "\" failed: " + get.getStatusLine());
}
return get.getResponseBodyAsStream();
}
@Override
public String execute() throws Exception {
request.setAttribute("ips", ips);
request.setAttribute("startpage", pageno);
String connip = ips[pageno];
// 连接url
String conpath = "/pool/status";
HttpClient httpClient = new HttpClient();
GetMethod getMethod = new GetMethod("http://" + connip + conpath);
getMethod
.getParams()
.setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler());
try {
// 执行getMethod
int statusCode = httpClient.executeMethod(getMethod);
if (statusCode != HttpStatus.SC_OK) {
System.err.println("Method failed: " + getMethod.getStatusLine());
}
// 读取内容
byte[] responseBody = getMethod.getResponseBody();
// 处理内容
JSONObject jo = JSONObject.fromObject(new String(responseBody));
cinfo = (ConnInfo) jo.toBean(jo, ConnInfo.class);
} catch (HttpException e) {
// 发生致命的异常，可能是协议不对或者返回的内容有问题
System.err.println("Please check your provided http address!");
e.printStackTrace();
return ERROR;
} catch (IOException e) {
// 发生网络异常
e.printStackTrace();
return ERROR;
} finally {
// 释放连接
getMethod.releaseConnection();
}
return SUCCESS;
}
/**
* Returns list with all stations with their ID's from the server.
*
* @return List<Station>
* @throws HttpException
* @throws IOException
*/
public List<Station> getAllStations() {
if (stations != null && !stations.isEmpty()) {
return stations;
}
String url = "http://booking.uz.gov.ua/purchase/station/";
char letter;
GetMethod get = null;
String jsonResp = null;
List<Station> myStations = new ArrayList<>();
try {
for (int i = 0; i < 32; i++) {
letter = (char) (1072 + i); // 1072 - rus 'a' in ASCII
String currentUrl = URIUtil.encodeQuery(url + letter);
get = new GetMethod(currentUrl);
int statusCode = client.executeMethod(get);
if (statusCode != HttpStatus.SC_OK) {
logger.error("getAllStations method failed. Get method failed: " + get.getStatusLine());
return null;
}
jsonResp =
IOUtils.toString(
get.getResponseBodyAsStream(), "UTF-8"); // get.getResponseBodyAsString();
StationsListJson resp = new ObjectMapper().readValue(jsonResp, StationsListJson.class);
myStations.addAll(resp.getStations());
}
} catch (HttpException e) {
logger.error("Could not retrieve stations." + e.getMessage(), e);
throw new RuntimeException("Could not retrieve stations", e);
} catch (IOException e) {
logger.error("Could not retrieve stations." + e.getMessage(), e);
throw new RuntimeException("Could not retrieve stations", e);
}
Collections.sort(myStations);
return stations = myStations;
}
public boolean testHttpConnection() {
boolean result = true;
HttpClient httpclient = new HttpClient();
GetMethod httpget = new GetMethod("https://www.verisign.com/");
try {
try {
httpclient.executeMethod(httpget);
} catch (Exception e) {
result = false;
}
System.out.println(httpget.getStatusLine());
} finally {
httpget.releaseConnection();
}
if (httpget.getStatusCode() != 200) {
result = false;
}
return result;
}
/**
* Utility method to send message to kannel from this servlet.
*
* @param message
*/
public void sendMessageToKannel(
String message, String phoneNumber, String username, String password) {
String smsMessage = "";
try {
smsMessage = URLEncoder.encode(message, "UTF-8");
} catch (UnsupportedEncodingException ex) {
logger.debug(
SendSmsToKannelService.class
+ ":sendMessageToKannel() has thrown an exception\n"
+ ex.getMessage());
}
String kannelUrl = "http://10.210.7.193:13009/cgi-bin/sendsms";
String kannelParam =
"?username="******"&password="******"&to="
+ phoneNumber
+ "&text="
+ smsMessage
+ "&from=0802";
HttpClient client = new HttpClient();
GetMethod method = new GetMethod(kannelUrl + kannelParam);
method
.getParams()
.setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false));
try {
// Execute the method.
int statusCode = client.executeMethod(method);
if (statusCode != HttpStatus.SC_OK) {
System.err.println("Method failed: " + method.getStatusLine());
}
// Read the response body.
byte[] responseBody = method.getResponseBody();
// Deal with the response.
// Use caution: ensure correct character encoding and is not binary data
System.out.println(new String(responseBody));
} catch (HttpException e) {
System.err.println("Fatal protocol violation: " + e.getMessage());
e.printStackTrace();
} catch (IOException e) {
System.err.println("Fatal transport error: " + e.getMessage());
e.printStackTrace();
} finally {
// Release the connection.
method.releaseConnection();
}
}
public static void main(String[] args) throws Exception {
/**
* Use the key and secret provided when you registered your application
* http://www.flickr.com/services/api/keys/apply/
*/
String key = "";
String secret = "";
/**
* This example assumes that you have uploaded a photo to your Flickr account.
*
* <p>The photo id can be found in the URL of the photo
* http://flickr.com/photos/xxxxxxx@N03/<use_this_photo_id>/
*/
String photoId = "";
/**
* Request a frob to identify the login session. This call requires a signature. The signature
* starts with your shared secret and is followed by your API key and the method name. The API
* key and method name are prepended by the words "api_key" and "method" as shown in the
* following line.
*/
String methodGetFrob = "flickr.auth.getFrob";
String sig = secret + "api_key" + key + "method" + methodGetFrob;
/** The API signature must be MD5 encoded and appended to the request */
String signature = MD5(sig);
String request =
"http://api.flickr.com/services/rest/?method="
+ methodGetFrob
+ "&api_key="
+ key
+ "&api_sig="
+ signature;
System.out.println("GET frob request: " + request);
HttpClient client = new HttpClient();
GetMethod method = new GetMethod(request);
// Send GET request
int statusCode = client.executeMethod(method);
if (statusCode != HttpStatus.SC_OK) {
System.err.println("Method failed: " + method.getStatusLine());
}
InputStream rstream = null;
// Get the response body
rstream = method.getResponseBodyAsStream();
// Retrieve the XML response to the frob request and get the frob value.
Document response = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(rstream);
String frob = null;
// Check if frob is in the response
NodeList frobResponse = response.getElementsByTagName("frob");
Node frobNode = frobResponse.item(0);
if (frobNode != null) {
frob = frobNode.getTextContent();
System.out.println("Successfully retrieved frob: " + frob);
} else {
printFlickrError(response);
return;
}
System.out.println("Get frob XML response:");
/**
* Create a Flickr login link
* http://www.flickr.com/services/auth/?api_key=[api_key]&perms=[perms]&frob=[frob]&api_sig=[api_sig]
* We are using "write" for the perms value because we will be rotating an image.
*/
sig = secret + "api_key" + key + "frob" + frob + "permswrite";
signature = MD5(sig);
request =
"http://www.flickr.com/services/auth/?api_key="
+ key
+ "&perms=write&frob="
+ frob
+ "&api_sig="
+ signature;
/** Copy/paste the generated link into your favorite web browser and follow the instructions */
System.out.println(
"Browse to the following flickr url to authenticate yourself and then press enter.");
System.out.println(request);
BufferedReader infile = new BufferedReader(new InputStreamReader(System.in));
String line = infile.readLine();
/**
* Get auth token using frob. Once again, a signature is required for authenticated calls to the
* Flickr API.
*/
String methodGetToken = "flickr.auth.getToken";
sig = secret + "api_key" + key + "frob" + frob + "method" + methodGetToken;
signature = MD5(sig);
request =
"http://api.flickr.com/services/rest/?method="
+ methodGetToken
+ "&api_key="
+ key
+ "&frob="
+ frob
+ "&api_sig="
+ signature;
System.out.println("Token request: " + request);
method = new GetMethod(request);
// Send GET request
statusCode = client.executeMethod(method);
if (statusCode != HttpStatus.SC_OK) {
System.err.println("Method failed: " + method.getStatusLine());
}
rstream = null;
// Get the response body
rstream = method.getResponseBodyAsStream();
/** Retrieve the XML response to the token request and get the token value */
response = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(rstream);
String token = null;
// Check if token is in the response
NodeList tokenResponse = response.getElementsByTagName("token");
Node tokenNode = tokenResponse.item(0);
if (tokenNode != null) {
token = tokenNode.getTextContent();
System.out.println("Successfully retrieved token: " + token);
} else {
printFlickrError(response);
return;
}
/**
* Now that we have authenticated with Flickr and obtained an auth token with write privileges,
* it's time to rotate the image.
*
* <p>http://api.flickr.com/services/rest/?method=flickr.photos.transform.rotate&
* api_key=[api_key]&photo_id=[photo_id]°rees=180&auth_token=[auth_token]&api_sig=[api_sig]
*/
String methodRotate = "flickr.photos.transform.rotate";
sig =
secret
+ "api_key"
+ key
+ "auth_token"
+ token
+ "degrees180"
+ "method"
+ methodRotate
+ "photo_id"
+ photoId;
signature = MD5(sig);
/**
* Create POST data for rotate request. The Flickr API documentation specifies that calls to
* flickr.photos.transform.rotate be made via HTTP POST.
*/
request = "http://api.flickr.com/services/rest/";
PostMethod pmethod = new PostMethod(request);
pmethod.addParameter("method", methodRotate);
pmethod.addParameter("photo_id", photoId);
pmethod.addParameter("degrees", "180");
pmethod.addParameter("api_key", key);
pmethod.addParameter("auth_token", token);
pmethod.addParameter("api_sig", signature);
// Send POST request
statusCode = client.executeMethod(pmethod);
if (statusCode != HttpStatus.SC_OK) {
System.err.println("Method failed: " + pmethod.getStatusLine());
}
rstream = null;
// Get the response body
rstream = pmethod.getResponseBodyAsStream();
/** Retrieve the XML response to the rotate request and get the XML response */
response = DocumentBuilderFactory.newInstance().newDocumentBuilder().parse(rstream);
String photoid = null;
// Check if photoid is in the response
NodeList rotateResponse = response.getElementsByTagName("photoid");
Node rotateNode = rotateResponse.item(0);
if (rotateNode != null) {
photoid = rotateNode.getTextContent();
System.out.println("Successfully flipped photo " + photoid);
System.out.println(
"Refresh the photo in your web browser and you will see it has been flipped.");
} else {
printFlickrError(response);
return;
}
}
private void buscarOrganizacion(String token) {
Organizacion organizacion = null;
if (logger.isDebugEnabled()) logger.debug("Buscando Organizaciones para Satelite: " + satelite);
HttpClient httpclient = getHttpClient();
GetMethod get = new GetMethod(restUrl.getOrganizacionUrl());
get.addRequestHeader("Accept", "application/xml");
NameValuePair tokenParam = new NameValuePair("token", token);
int id = 0;
// if(satelite.getLastOrgId() != null)
// id = satelite.getLastOrgId();
int leap = idLeap;
while (true) {
id = id + 1;
if (logger.isTraceEnabled()) {
logger.trace("Buscando Organizacion con id: " + id + " en el Satelite: " + satelite);
}
NameValuePair passwordParam = new NameValuePair("id", String.valueOf(id));
NameValuePair[] params = new NameValuePair[] {tokenParam, passwordParam};
get.setQueryString(params);
String queryString = get.getQueryString();
int statusCode;
try {
if (logger.isTraceEnabled()) {
logger.trace("Query String: " + queryString);
}
statusCode = httpclient.executeMethod(get);
if (statusCode != HttpStatus.SC_OK) {
logger.error("Method failed: " + get.getStatusLine());
} else {
String xmlString = get.getResponseBodyAsString();
if (logger.isInfoEnabled()) {
logger.info("Xml Received.");
}
String xmlHash = constructXmlHash(xmlString);
organizacion = organizacionService.findByHash(xmlHash);
if (organizacion == null) {
organizacion = parseOrganizacion(xmlString);
if (organizacion == null) {
leap--;
if (leap <= 0) {
logger.info(
"Se llegó al fin de las Organizaciones. Saliendo del Satelite: " + satelite);
break;
} else {
logger.info("Leap: " + leap);
}
} else {
if (organizacion.getSatelites() == null) {
organizacion.setSatelites(new HashSet<OrganizacionSatelite>());
}
boolean sateliteFound = false;
for (OrganizacionSatelite sat : organizacion.getSatelites()) {
if (sat.getSatelite().getName().equals(satelite.getName())) {
sateliteFound = true;
break;
}
}
if (!sateliteFound) {
OrganizacionSatelite newSat = new OrganizacionSatelite();
newSat.setSatelite(satelite);
newSat.setOrganizacion(organizacion);
organizacion.getSatelites().add(newSat);
}
organizacion.setXmlHash(xmlHash);
organizacionService.saveOrganization(organizacion);
int numEmpresas = 0;
if (satelite.getNumEmpresas() != null) {
numEmpresas = satelite.getNumEmpresas();
}
numEmpresas++;
Date now = new Date();
satelite.setNumEmpresas(numEmpresas);
satelite.setLastRetrieval(new Timestamp(now.getTime()));
satelite.setLastOrgId(id);
sateliteService.saveSatelite(satelite);
}
} else {
if (logger.isInfoEnabled()) {
logger.info("Organizacion with id: " + id + " already in centraldir");
}
}
}
} catch (HttpException e) {
logger.error(e.getMessage());
} catch (IOException e) {
logger.error(e.getMessage());
} catch (ParserConfigurationException e) {
logger.error(e.getMessage());
} catch (SAXException e) {
logger.error(e.getMessage());
} catch (ServiceException e) {
logger.error(e.getMessage());
} catch (NoSuchAlgorithmException e) {
logger.error(e.getMessage());
} finally {
get.releaseConnection();
}
}
}
public static Document getRssDocument(Context context, InputStream is, Bundle feedBundle) {
GetMethod method = null;
BufferedReader in = null;
try {
// Download the feed
// url is the bitstream contents
in = new BufferedReader(new InputStreamReader(is));
String url = in.readLine();
// need to replace the feed prefix with the HTTP protocol ie replace feed:// with http://
url = url.replaceAll(PackageDetectorStep.FEED_PREFIX, "http://");
HttpClient client = new HttpClient();
// Create a method instance.
method = new GetMethod(url);
// Provide custom retry handler is necessary
method
.getParams()
.setParameter(
HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false));
// Execute the method.
int statusCode = client.executeMethod(method);
if (statusCode != HttpStatus.SC_OK) {
throw new Exception("Method failed: " + method.getStatusLine());
}
// Read the response body.
byte[] responseBody = method.getResponseBody();
// Store the response in a bitstream so that the ingester can use it later
if (feedBundle != null) {
BitstreamFormat bs_format = BitstreamFormat.findByShortDescription(context, "Text");
BundleUtils.setBitstreamFromBytes(
feedBundle, Constants.FEED_BUNDLE_CONTENTS_NAME, bs_format, responseBody, false);
}
// Now see if we have a RSS v2.0 document
SAXBuilder builder = new SAXBuilder(false);
Document xmlDoc = builder.build(new ByteArrayInputStream(responseBody));
// XMLOutputter outputPretty = new XMLOutputter(Format.getPrettyFormat());
// log.debug("Got RSS DOCUMENT:");
// log.debug(outputPretty.outputString(xmlDoc));
return xmlDoc;
} catch (Exception e) {
ExceptionLogger.logException(log, e);
} finally {
// Release the connection.
method.releaseConnection();
try {
in.close();
} catch (Exception e) {
}
}
return null;
}
@Override
protected void doGet(HttpServletRequest request, HttpServletResponse response)
throws ServletException, IOException {
GetMethod httpGet = null;
try {
String url = RequestUtil.getParameter(request, PARAM_URL, defaultProxyUrl);
String host = url.split("/")[2];
// Get the proxy parameters
// TODO: Add dependency injection to set proxy config from GeoNetwork settings, using also the
// credentials configured
String proxyHost = System.getProperty("http.proxyHost");
String proxyPort = System.getProperty("http.proxyPort");
// Get rest of parameters to pass to proxied url
HttpMethodParams urlParams = new HttpMethodParams();
Enumeration paramNames = request.getParameterNames();
while (paramNames.hasMoreElements()) {
String paramName = (String) paramNames.nextElement();
if (!paramName.equalsIgnoreCase(PARAM_URL)) {
urlParams.setParameter(paramName, request.getParameter(paramName));
}
}
// Checks if allowed host
if (!isAllowedHost(host)) {
// throw new ServletException("This proxy does not allow you to access that location.");
returnExceptionMessage(response, "This proxy does not allow you to access that location.");
return;
}
if (url.startsWith("http://") || url.startsWith("https://")) {
HttpClient client = new HttpClient();
// Added support for proxy
if (proxyHost != null && proxyPort != null) {
client.getHostConfiguration().setProxy(proxyHost, Integer.valueOf(proxyPort));
}
httpGet = new GetMethod(url);
httpGet.setParams(urlParams);
client.executeMethod(httpGet);
if (httpGet.getStatusCode() == HttpStatus.SC_OK) {
Header contentType = httpGet.getResponseHeader(HEADER_CONTENT_TYPE);
String[] contentTypesReturned = contentType.getValue().split(";");
if (!isValidContentType(contentTypesReturned[0])) {
contentTypesReturned = contentType.getValue().split(" ");
if (!isValidContentType(contentTypesReturned[0])) {
throw new ServletException("Status: 415 Unsupported media type");
}
}
// Sets response contentType
response.setContentType(getResponseContentType(contentTypesReturned));
String responseBody = httpGet.getResponseBodyAsString().trim();
PrintWriter out = response.getWriter();
out.print(responseBody);
out.flush();
out.close();
} else {
returnExceptionMessage(
response, "Unexpected failure: " + httpGet.getStatusLine().toString());
}
httpGet.releaseConnection();
} else {
// throw new ServletException("only HTTP(S) protocol supported");
returnExceptionMessage(response, "only HTTP(S) protocol supported");
}
} catch (Exception e) {
e.printStackTrace();
// throw new ServletException("Some unexpected error occurred. Error text was: " +
// e.getMessage());
returnExceptionMessage(
response, "Some unexpected error occurred. Error text was: " + e.getMessage());
} finally {
if (httpGet != null) httpGet.releaseConnection();
}
}
/** Run method of the thread */
public void run() {
queue = manager.workQueue;
while (manager.hasWorkLeft()) {
working = false;
// code to make the worker pause, if the pause button has been presed
// if the stop signal has been given stop the thread
if (stop) {
return;
}
// this pasuses the thread
synchronized (this) {
while (pleaseWait) {
try {
wait();
} catch (InterruptedException e) {
return;
} catch (Exception e) {
e.printStackTrace();
}
}
}
GetMethod httpget = null;
HeadMethod httphead = null;
try {
work = (WorkUnit) queue.take();
working = true;
url = work.getWork();
int code = 0;
String responce = "";
String rawResponce = "";
// if the work is a head request
if (work.getMethod().equalsIgnoreCase("HEAD")) {
if (Config.debug) {
System.out.println("DEBUG Worker[" + threadId + "]: HEAD " + url.toString());
}
httphead = new HeadMethod(url.toString());
// set the custom HTTP headers
Vector HTTPheaders = manager.getHTTPHeaders();
for (int a = 0; a < HTTPheaders.size(); a++) {
HTTPHeader httpHeader = (HTTPHeader) HTTPheaders.elementAt(a);
/*
* Host header has to be set in a different way!
*/
if (httpHeader.getHeader().startsWith("Host")) {
httphead.getParams().setVirtualHost(httpHeader.getValue());
} else {
httphead.setRequestHeader(httpHeader.getHeader(), httpHeader.getValue());
}
}
httphead.setFollowRedirects(Config.followRedirects);
/*
* this code is used to limit the number of request/sec
*/
if (manager.isLimitRequests()) {
while (manager.getTotalDone()
/ ((System.currentTimeMillis() - manager.getTimestarted()) / 1000.0)
> manager.getLimitRequestsTo()) {
Thread.sleep(100);
}
}
/*
* Send the head request
*/
code = httpclient.executeMethod(httphead);
if (Config.debug) {
System.out.println("DEBUG Worker[" + threadId + "]: " + code + " " + url.toString());
}
httphead.releaseConnection();
}
// if we are doing a get request
else if (work.getMethod().equalsIgnoreCase("GET")) {
// make the request;
if (Config.debug) {
System.out.println("DEBUG Worker[" + threadId + "]: GET " + url.toString());
}
httpget = new GetMethod(url.toString());
// set the custom HTTP headers
Vector HTTPheaders = manager.getHTTPHeaders();
for (int a = 0; a < HTTPheaders.size(); a++) {
HTTPHeader httpHeader = (HTTPHeader) HTTPheaders.elementAt(a);
/*
* Host header has to be set in a different way!
*/
if (httpHeader.getHeader().startsWith("Host")) {
httpget.getParams().setVirtualHost(httpHeader.getValue());
} else {
httpget.setRequestHeader(httpHeader.getHeader(), httpHeader.getValue());
}
}
httpget.setFollowRedirects(Config.followRedirects);
/*
* this code is used to limit the number of request/sec
*/
if (manager.isLimitRequests()) {
while (manager.getTotalDone()
/ ((System.currentTimeMillis() - manager.getTimestarted()) / 1000.0)
> manager.getLimitRequestsTo()) {
Thread.sleep(100);
}
}
code = httpclient.executeMethod(httpget);
if (Config.debug) {
System.out.println("DEBUG Worker[" + threadId + "]: " + code + " " + url.toString());
}
// set up the input stream
BufferedReader input =
new BufferedReader(new InputStreamReader(httpget.getResponseBodyAsStream()));
// save the headers into a string, used in viewing raw responce
String rawHeader;
rawHeader = httpget.getStatusLine() + "\r\n";
Header headers[] = httpget.getResponseHeaders();
StringBuffer buf = new StringBuffer();
for (int a = 0; a < headers.length; a++) {
buf.append(headers[a].getName() + ": " + headers[a].getValue() + "\r\n");
}
rawHeader = rawHeader + buf.toString();
buf = new StringBuffer();
// read in the responce body
String line;
while ((line = input.readLine()) != null) {
buf.append("\r\n" + line);
}
responce = buf.toString();
input.close();
rawResponce = rawHeader + responce;
// clean the responce
// parse the html of what we have found
if (Config.parseHTML && !work.getBaseCaseObj().isUseRegexInstead()) {
Header contentType = httpget.getResponseHeader("Content-Type");
if (contentType != null) {
if (contentType.getValue().startsWith("text")) {
manager.addHTMLToParseQueue(new HTMLparseWorkUnit(responce, work));
}
}
}
responce = FilterResponce.CleanResponce(responce, work);
Thread.sleep(10);
httpget.releaseConnection();
} else {
// There is no need to deal with requests other than HEAD or GET
}
// if we need to check the against the base case
if (work.getMethod().equalsIgnoreCase("GET")
&& work.getBaseCaseObj().useContentAnalysisMode()) {
if (code == 200) {
if (Config.debug) {
System.out.println(
"DEBUG Worker[" + threadId + "]: Base Case Check " + url.toString());
}
// TODO move this option to the Adv options
// if the responce does not match the base case
Pattern regexFindFile = Pattern.compile(".*file not found.*", Pattern.CASE_INSENSITIVE);
Matcher m = regexFindFile.matcher(responce);
// need to clean the base case of the item we are looking for
String basecase =
FilterResponce.removeItemCheckedFor(
work.getBaseCaseObj().getBaseCase(), work.getItemToCheck());
if (m.find()) {
// do nothing as we have a 404
} else if (!responce.equalsIgnoreCase(basecase)) {
if (work.isDir()) {
if (Config.debug) {
System.out.println(
"DEBUG Worker[" + threadId + "]: Found Dir (base case)" + url.toString());
}
// we found a dir
manager.foundDir(url, code, responce, basecase, rawResponce, work.getBaseCaseObj());
} else {
// found a file
if (Config.debug) {
System.out.println(
"DEBUG Worker[" + threadId + "]: Found File (base case)" + url.toString());
}
manager.foundFile(
url,
code,
responce,
work.getBaseCaseObj().getBaseCase(),
rawResponce,
work.getBaseCaseObj());
}
}
} else if (code == 404 || code == 400) {
// again do nothing as it is not there
} else {
if (work.isDir()) {
if (Config.debug) {
System.out.println(
"DEBUG Worker[" + threadId + "]: Found Dir (base case)" + url.toString());
}
// we found a dir
manager.foundDir(
url,
code,
responce,
work.getBaseCaseObj().getBaseCase(),
rawResponce,
work.getBaseCaseObj());
} else {
// found a file
if (Config.debug) {
System.out.println(
"DEBUG Worker[" + threadId + "]: Found File (base case)" + url.toString());
}
manager.foundFile(
url,
code,
responce,
work.getBaseCaseObj().getBaseCase(),
rawResponce,
work.getBaseCaseObj());
}
// manager.foundError(url, "Base Case Mode Error - Responce code came back as " + code +
// " it should have been 200");
// manager.workDone();
}
}
/*
* use the custom regex check instead
*/
else if (work.getBaseCaseObj().isUseRegexInstead()) {
Pattern regexFindFile = Pattern.compile(work.getBaseCaseObj().getRegex());
Matcher m = regexFindFile.matcher(rawResponce);
/*
System.out.println("======Trying to find======");
System.out.println(work.getBaseCaseObj().getRegex());
System.out.println("======In======");
System.out.println(responce);
System.out.println("======/In======");
*/
if (m.find()) {
// do nothing as we have a 404
if (Config.debug) {
System.out.println(
"DEBUG Worker[" + threadId + "]: Regex matched so it's a 404, " + url.toString());
}
} else {
if (Config.parseHTML) {
Header contentType = httpget.getResponseHeader("Content-Type");
if (contentType != null) {
if (contentType.getValue().startsWith("text")) {
manager.addHTMLToParseQueue(new HTMLparseWorkUnit(rawResponce, work));
}
}
}
if (work.isDir()) {
if (Config.debug) {
System.out.println(
"DEBUG Worker[" + threadId + "]: Found Dir (regex) " + url.toString());
}
// we found a dir
manager.foundDir(
url,
code,
responce,
work.getBaseCaseObj().getBaseCase(),
rawResponce,
work.getBaseCaseObj());
} else {
// found a file
if (Config.debug) {
System.out.println(
"DEBUG Worker[" + threadId + "]: Found File (regex) " + url.toString());
}
manager.foundFile(
url,
code,
responce,
work.getBaseCaseObj().getBaseCase(),
rawResponce,
work.getBaseCaseObj());
}
// manager.foundError(url, "Base Case Mode Error - Responce code came back as " + code +
// " it should have been 200");
// manager.workDone();
}
}
// just check the responce code
else {
// if is not the fail code, a 404 or a 400 then we have a possible
if (code != work.getBaseCaseObj().getFailCode()
&& code != 404
&& code != 0
&& code != 400) {
if (work.getMethod().equalsIgnoreCase("HEAD")) {
if (Config.debug) {
System.out.println(
"DEBUG Worker[" + threadId + "]: Getting responce via GET " + url.toString());
}
rawResponce = "";
httpget = new GetMethod(url.toString());
Vector HTTPheaders = manager.getHTTPHeaders();
for (int a = 0; a < HTTPheaders.size(); a++) {
HTTPHeader httpHeader = (HTTPHeader) HTTPheaders.elementAt(a);
httpget.setRequestHeader(httpHeader.getHeader(), httpHeader.getValue());
}
httpget.setFollowRedirects(Config.followRedirects);
/*
* this code is used to limit the number of request/sec
*/
if (manager.isLimitRequests()) {
while (manager.getTotalDone()
/ ((System.currentTimeMillis() - manager.getTimestarted()) / 1000.0)
> manager.getLimitRequestsTo()) {
Thread.sleep(100);
}
}
int newCode = httpclient.executeMethod(httpget);
// in some cases the second get can return a different result, than the first head
// request!
if (newCode != code) {
manager.foundError(
url,
"Return code for first HEAD, is different to the second GET: "
+ code
+ " - "
+ newCode);
}
rawResponce = "";
// build a string version of the headers
rawResponce = httpget.getStatusLine() + "\r\n";
Header headers[] = httpget.getResponseHeaders();
StringBuffer buf = new StringBuffer();
for (int a = 0; a < headers.length; a++) {
buf.append(headers[a].getName() + ": " + headers[a].getValue() + "\r\n");
}
buf.append("\r\n");
rawResponce = rawResponce + buf.toString();
if (httpget.getResponseContentLength() > 0) {
// get the http body
BufferedReader input =
new BufferedReader(new InputStreamReader(httpget.getResponseBodyAsStream()));
String line;
String tempResponce = "";
buf = new StringBuffer();
while ((line = input.readLine()) != null) {
buf.append("\r\n" + line);
}
tempResponce = buf.toString();
input.close();
rawResponce = rawResponce + tempResponce;
Header contentType = httpget.getResponseHeader("Content-Type");
if (Config.parseHTML) {
contentType = httpget.getResponseHeader("Content-Type");
if (contentType != null) {
if (contentType.getValue().startsWith("text")) {
manager.addHTMLToParseQueue(new HTMLparseWorkUnit(tempResponce, work));
}
}
}
}
httpget.releaseConnection();
}
if (work.isDir()) {
manager.foundDir(url, code, rawResponce, work.getBaseCaseObj());
} else {
manager.foundFile(url, code, rawResponce, work.getBaseCaseObj());
}
}
}
manager.workDone();
Thread.sleep(20);
} catch (NoHttpResponseException e) {
manager.foundError(url, "NoHttpResponseException " + e.getMessage());
manager.workDone();
} catch (ConnectTimeoutException e) {
manager.foundError(url, "ConnectTimeoutException " + e.getMessage());
manager.workDone();
} catch (URIException e) {
manager.foundError(url, "URIException " + e.getMessage());
manager.workDone();
} catch (IOException e) {
manager.foundError(url, "IOException " + e.getMessage());
manager.workDone();
} catch (InterruptedException e) {
// manager.foundError(url, "InterruptedException " + e.getMessage());
manager.workDone();
return;
} catch (IllegalArgumentException e) {
e.printStackTrace();
manager.foundError(url, "IllegalArgumentException " + e.getMessage());
manager.workDone();
} finally {
if (httpget != null) {
httpget.releaseConnection();
}
if (httphead != null) {
httphead.releaseConnection();
}
}
}
}
/**
* 通过httpclient方法获得网页内容
*
* @param url 目标的URL
* @return 页面内容字符串
* @throws IOException
*/
public static String getContentByHttpClient(String url, String encode, int timeout)
throws IOException {
HttpClient client = new HttpClient();
GetMethod method = new GetMethod(url);
// 设置失败后默认的重试次数为3
method
.getParams()
.setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false));
// 设置失败延时为5000ms，即5秒
method.getParams().setIntParameter(HttpMethodParams.SO_TIMEOUT, timeout);
ByteArrayOutputStream outstream = null;
BufferedInputStream bufferedInputStream = null;
try {
int statusCode = client.executeMethod(method);
if (statusCode != HttpStatus.SC_OK) {
System.out.println("方法失败: " + method.getStatusLine());
}
// 缓冲读取的内容4k，提高性能
bufferedInputStream = new BufferedInputStream(method.getResponseBodyAsStream());
outstream = new ByteArrayOutputStream();
byte[] buffer = new byte[4096];
int len;
while ((len = bufferedInputStream.read(buffer)) > 0) {
outstream.write(buffer, 0, len);
}
outstream.flush();
byte[] responseBody = outstream.toByteArray();
return new String(responseBody, encode);
} catch (HttpException e) {
System.out.println("致命协议错误：" + e.getMessage());
throw e;
} catch (IOException e) {
System.out.println("致命传输错误：" + e.getMessage());
throw e;
} finally {
if (bufferedInputStream != null) {
bufferedInputStream.close();
}
if (outstream != null) {
outstream.close();
}
// 注意，连接在完成后必须释放
method.releaseConnection();
}
}