private String extractResponse(GetMethod method) throws IOException {
Header header = method.getResponseHeader("FIRST");
assertNotNull("Received FIRST header", header);
String result = header.getValue();
assertNotNull("FIRST header not null", result);
header = method.getResponseHeader("SECOND");
assertNotNull("Received SECOND header", header);
String second = header.getValue();
assertNotNull("FIRST header not null", second);
result.concat(second);
// Read the response body.
byte[] responseBody = method.getResponseBody();
// Use caution: ensure correct character encoding and is not binary data
result.concat(new String(responseBody));
// Release the connection.
// method.releaseConnection();
return result;
}
/** Makes sure that a connection gets released after the content of the body is read. */
public void testResponseAutoRelease() throws Exception {
this.server.setHttpService(new EchoService());
MultiThreadedHttpConnectionManager connectionManager = new MultiThreadedHttpConnectionManager();
connectionManager.getParams().setDefaultMaxConnectionsPerHost(1);
client.setHttpConnectionManager(connectionManager);
// we shouldn't have to wait if a connection is available
client.getParams().setConnectionManagerTimeout(1);
GetMethod getMethod = new GetMethod("/");
try {
client.executeMethod(getMethod);
} catch (Exception e) {
fail("error reading from server: " + e);
}
// this should release the connection
getMethod.getResponseBody();
getMethod = new GetMethod("/");
try {
// this should fail quickly if the connection has not been released
client.executeMethod(getMethod);
} catch (HttpException e) {
fail("httpConnection does not appear to have been released: " + e);
} catch (IOException e) {
fail("error reading from server; " + e);
}
}
/**
* 抓取网页静态内容： 下载 void
*
* @author wx
*/
private void fetchStaticResources() {
HttpClient client = new HttpClient();
Set<String> urlSet = staticModels.keySet();
Iterator<String> ite = urlSet.iterator();
while (ite.hasNext()) {
String key = ite.next();
StaticResourceModel model = staticModels.get(key);
// 执行下载
GetMethod get = new GetMethod(model.getUrl());
try {
client.executeMethod(get);
File path = new File(siteSaveFolder + model.savePath());
if (!path.exists()) {
path.mkdirs();
}
File storeFile = new File(siteSaveFolder + model.getFilePath());
FileOutputStream output = new FileOutputStream(storeFile);
// 得到网络资源的字节数组,并写入文件
output.write(get.getResponseBody());
output.close();
} catch (Exception e) {
e.printStackTrace();
}
}
}
public static int GetCountByCkan3(String url) {
int count = 0;
HttpClient client = new HttpClient();
LOG.info("**** INPUT SPLIT COUNT *** " + url);
GetMethod method = new GetMethod(url);
method
.getParams()
.setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false));
method.setRequestHeader("User-Agent", "Hammer Project - SantaMaria crawler");
method
.getParams()
.setParameter(HttpMethodParams.USER_AGENT, "Hammer Project - SantaMaria crawler");
try {
client.executeMethod(method);
byte[] responseBody = method.getResponseBody();
Document doc = Document.parse(new String(responseBody));
if (doc.containsKey("result")) {
count = ((Document) doc.get("result")).getInteger("count");
LOG.info("Find --> " + count);
}
} catch (Exception e) {
e.printStackTrace();
LOG.error(e);
} finally {
method.releaseConnection();
}
return count;
}
/**
* 发生Get请求
*
* @param url 请求url
* @return
* @throws PortalException [参数说明]
* @return String [返回类型说明]
* @exception throws [违例类型] [违例说明]
* @see [类、类#方法、类#成员]
*/
public String get(String url) {
GetMethod httpMethod = new GetMethod(url);
// 设置header信息，传输XML格式的
httpMethod.setRequestHeader(CONTENT_TYPE_NAME, CONTENT_TYPE_VALUE_XML_UTF_8);
// 获取响应报文
String response = null;
try {
// 处理响应结果码
int resultCode = httpClient.executeMethod(httpMethod);
if (HttpStatus.SC_OK == resultCode) {
byte[] resBody = httpMethod.getResponseBody();
if (null != resBody && resBody.length > 0) {
response = new String(resBody, UTF_8);
} else {
LogUtil.error("Http resultCode=200, but responseBody is empty.");
}
} else {
LogUtil.error("Http resultCode=" + resultCode);
}
} catch (Exception ex) {
LogUtil.error("send http request error.", ex);
} finally {
if (null != httpMethod) {
httpMethod.releaseConnection();
}
}
return response;
}
protected void dialImpl(String number) throws ExceptionLP {
myLogger.info("Dialing <" + number + "> ...");
GetMethod method = getHttpMethod(number);
try {
prepareClient(client, method);
int status = client.executeMethod(method);
myLogger.info("Dialing done with HTTP Status: <" + status + ">");
if (status != HttpStatus.SC_OK) {
byte[] responseBody = method.getResponseBody();
String s = new String(responseBody);
myLogger.debug("Done dialing with response <" + s + ">");
}
} catch (HttpException ex) {
myLogger.debug("Got HttpException <" + ex.getMessage() + ">");
throw new ExceptionLP(EJBExceptionLP.FEHLER_TELEFON_WAHLVORGANG, ex.getMessage(), ex);
} catch (IOException ex) {
myLogger.debug("Got IOException <" + ex.getMessage() + ">");
throw new ExceptionLP(EJBExceptionLP.FEHLER_TELEFON_WAHLVORGANG, ex.getMessage(), ex);
} finally {
method.releaseConnection();
}
}
public static void main(String[] args) {
HttpClient client = new HttpClient(); // Create an instance of HttpClient.
GetMethod method = new GetMethod(url); // Create a method instance.
method
.getParams()
.setParameter(
HttpMethodParams.RETRY_HANDLER,
new DefaultHttpMethodRetryHandler(
3, false)); // Provide custom retry handler is necessary
try {
int statusCode = client.executeMethod(method); // Execute the method.
if (statusCode != HttpStatus.SC_OK) {
System.err.println("Method failed: " + method.getStatusLine());
}
byte[] responseBody = method.getResponseBody(); // Read the response body.
System.out.println(
new String(
responseBody)); // Deal with the response.// Use caution: ensure correct character
// encoding and is not binary data
} catch (HttpException e) {
System.err.println("Fatal protocol violation: " + e.getMessage());
e.printStackTrace();
} catch (IOException e) {
System.err.println("Fatal transport error: " + e.getMessage());
e.printStackTrace();
} finally {
method.releaseConnection(); // Release the connection.
}
}
/**
* 根据url获取ResponseBody,method=get
*
* @param url exp:http://192.168.1.1:8080/dir/target.html
* @return 以byte[]的方式放回
*/
public static byte[] getDataFromUrl(String url, int timeout) {
if (StringUtils.isBlank(url)) {
logger.error("url is blank!");
return null;
}
HttpClient httpClient = new HttpClient();
// 连接超时
httpClient.getHttpConnectionManager().getParams().setConnectionTimeout(8000);
// 等待数据返回超时
httpClient.getParams().setSoTimeout(timeout);
GetMethod method = new GetMethod(url);
// fix连接结束后不能正确关闭的问题
// method.setRequestHeader("Connection", "close");
// 如果发生错误则连续尝试1次
method
.getParams()
.setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(1, false));
try {
int statusCode = httpClient.executeMethod(method);
if (statusCode == HttpStatus.SC_OK) {
return method.getResponseBody();
} else {
throw new RuntimeException(
"http request error,return code:"
+ statusCode
+ ",msg:"
+ new String(method.getResponseBody()));
}
} catch (HttpException e) {
method.abort();
logger.error(e.getMessage());
} catch (IOException e) {
method.abort();
logger.error(e.getMessage());
} finally {
// Release the connection.
method.releaseConnection();
}
return null;
}
/**
* Download the resource at the given url
*
* @param url to download
* @param Use cache : store file in image cache
* @throws Exception
* @return result as an array of bytes, null if a problem occured
*/
public static byte[] download(URL url, boolean bUseCache) throws Exception {
byte[] bOut = null;
// check if file is not already downloaded or being downloaded
if (bUseCache) {
if (new File(Util.getCachePath(url)).exists()) {
return bOut;
}
}
GetMethod get = null;
HttpClient client = null;
int iConTO = 1000 * ConfigurationManager.getInt(CONF_NETWORK_CONNECTION_TO);
int iTraTO = 1000 * ConfigurationManager.getInt(CONF_NETWORK_TRANSFERT_TO);
if (ConfigurationManager.getBoolean(CONF_NETWORK_USE_PROXY)) {
client =
getHTTPClient(
ConfigurationManager.getProperty(CONF_NETWORK_PROXY_LOGIN),
DownloadManager.getProxyPwd(),
iConTO,
iTraTO);
} else {
client = getHTTPClient(iConTO, iTraTO);
}
get = new GetMethod(url.toString());
get.addRequestHeader(
"Accept",
"image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, */*"); //$NON-NLS-1$ //$NON-NLS-2$
get.addRequestHeader("Accept-Language", "en-us"); // $NON-NLS-1$ //$NON-NLS-2$
get.addRequestHeader(
"User-Agent",
"Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)"); //$NON-NLS-1$ //$NON-NLS-2$
get.addRequestHeader("Connection", "Keep-Alive"); // $NON-NLS-1$ //$NON-NLS-2$
int status = client.executeMethod(get);
if (bUseCache) {
BufferedOutputStream bos =
new BufferedOutputStream(new FileOutputStream(Util.getCachePath(url)));
BufferedInputStream bis = new BufferedInputStream(get.getResponseBodyAsStream());
int i;
while ((i = bis.read()) != -1) {
bos.write(i);
}
bos.close();
bis.close();
} else {
bOut = get.getResponseBody();
}
if (get != null && get.isRequestSent()) {
get.releaseConnection();
}
return bOut;
}
/* 下载 url 指向的网页 */
public String downloadFile(String url) {
String filePath = null;
/* 1.生成 HttpClinet 对象并设置参数 */
HttpClient httpClient = new HttpClient();
// 设置 Http 连接超时 5s
httpClient.getHttpConnectionManager().getParams().setConnectionTimeout(5000);
/* 2.生成 GetMethod 对象并设置参数 */
GetMethod getMethod = new GetMethod(url);
// 设置 get 请求超时 5s
getMethod.getParams().setParameter(HttpMethodParams.SO_TIMEOUT, 5000);
// 设置请求重试处理
getMethod
.getParams()
.setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler());
/* 3.执行 HTTP GET 请求 */
try {
int statusCode = httpClient.executeMethod(getMethod);
// 判断访问的状态码
if (statusCode != HttpStatus.SC_OK) {
System.err.println("Method failed: " + getMethod.getStatusLine());
filePath = null;
}
/* 4.处理 HTTP 响应内容 */
byte[] responseBody = getMethod.getResponseBody(); // 读取为字节数组
// 根据网页 url 生成保存时的文件名
filePath =
"temp\\" + getFileNameByUrl(url, getMethod.getResponseHeader("Content-Type").getValue());
// createFile(filePath);
FileUtils.createFile(filePath);
saveToLocal(responseBody, filePath, false);
} catch (HttpException e) {
// 发生致命的异常，可能是协议不对或者返回的内容有问题
System.out.println("Please check your provided http address!");
e.printStackTrace();
} catch (IOException e) {
// 发生网络异常
e.printStackTrace();
} finally {
// 释放连接
getMethod.releaseConnection();
}
return filePath;
}
// ���� URL ָ�����ҳ
public String downloadFile(String url) {
String filePath = null;
// 1.���� HttpClinet��������
HttpClient httpClient = new HttpClient();
// ���� HTTP���ӳ�ʱ 5s
httpClient.getHttpConnectionManager().getParams().setConnectionTimeout(5000);
// 2.���� GetMethod��������
GetMethod getMethod = new GetMethod(url);
// ���� get����ʱ 5s
getMethod.getParams().setParameter(HttpMethodParams.SO_TIMEOUT, 5000);
// �����������Դ���
getMethod
.getParams()
.setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler());
// 3.ִ��GET����
try {
int statusCode = httpClient.executeMethod(getMethod);
// �жϷ��ʵ�״̬��
if (statusCode != HttpStatus.SC_OK) {
System.err.println("Method failed: " + getMethod.getStatusLine());
filePath = null;
}
// 4.���� HTTP ��Ӧ����
byte[] responseBody = getMethod.getResponseBody(); // ��ȡΪ�ֽ�����
// ������ҳ url ���ɱ���ʱ���ļ���
filePath =
"temp\\" + getFileNameByUrl(url, getMethod.getResponseHeader("Content-Type").getValue());
File file = new File(filePath);
if (!file.getParentFile().exists()) {
file.getParentFile().mkdir();
}
saveToLocal(responseBody, filePath);
} catch (HttpException e) {
// �����������쳣��������Э�鲻�Ի��߷��ص�����������
System.out.println("�������http��ַ�Ƿ���ȷ");
e.printStackTrace();
} catch (IOException e) {
// ���������쳣
e.printStackTrace();
} finally {
// �ͷ�����
getMethod.releaseConnection();
}
return filePath;
}
/**
* Get data set from CKAN repository
*
* <p>4 Big Source --> Direct Link
*/
@SuppressWarnings("unchecked")
private void getPackageList() {
HttpClient client = new HttpClient();
LOG.info(split.getAction());
GetMethod method = new GetMethod(split.getAction());
method
.getParams()
.setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false));
method.setRequestHeader("User-Agent", "Hammer Project - SantaMaria crawler");
method
.getParams()
.setParameter(HttpMethodParams.USER_AGENT, "Hammer Project - SantaMaria crawler");
try {
int statusCode = client.executeMethod(method);
if (statusCode != HttpStatus.SC_OK) {
throw new Exception("Method failed: " + method.getStatusLine());
}
byte[] responseBody = method.getResponseBody();
LOG.debug(new String(responseBody));
setOutput(new String(responseBody));
Document document = Document.parse(getOutput());
if (document.containsKey("result")) {
ArrayList<Document> docs =
(ArrayList<Document>) ((Document) document.get("result")).get("results");
for (Document doc : docs) {
this.dataset.add(doc.getString("id"));
}
LOG.info("SANTA MARIA CKAN3 RECORD READER found" + this.dataset.size());
}
} catch (Exception e) {
LOG.error(e);
} finally {
method.releaseConnection();
}
}
/**
* Test
*
* @param pArgs
* @throws Exception
*/
@SuppressWarnings("unchecked")
public static void main(String[] pArgs) throws Exception {
GetCountByCkan3("http://catalog.data.gov/api/action/package_search?start=0&rows=1");
HttpClient client = new HttpClient();
GetMethod method =
new GetMethod("http://catalog.data.gov/api/action/package_search?start=0&rows=10");
method
.getParams()
.setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false));
method.setRequestHeader("User-Agent", "Hammer Project - SantaMaria crawler");
method
.getParams()
.setParameter(HttpMethodParams.USER_AGENT, "Hammer Project - SantaMaria crawler");
try {
int statusCode = client.executeMethod(method);
if (statusCode != HttpStatus.SC_OK) {
throw new Exception("Method failed: " + method.getStatusLine());
}
byte[] responseBody = method.getResponseBody();
LOG.info(new String(responseBody));
Document document = Document.parse(new String(responseBody));
if (document.containsKey("result")) {
ArrayList<Document> docs =
(ArrayList<Document>) ((Document) document.get("result")).get("results");
for (Document doc : docs) {
LOG.info(doc.getString("id"));
}
}
} catch (Exception e) {
LOG.error(e);
} finally {
method.releaseConnection();
}
}
public static void executeHttpGet() throws Exception {
// Create an instance of HttpClient.
HttpClient client = new HttpClient();
// Create a method instance.
GetMethod method = new GetMethod(url);
method.addRequestHeader("command", "addCredit");
// method.addRequestHeader("value", "addCredit");
// Provide custom retry handler is necessary
method
.getParams()
.setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false));
try {
// Execute the method.
int statusCode = client.executeMethod(method);
if (statusCode != HttpStatus.SC_OK) {
System.err.println("Method failed: " + method.getStatusLine());
}
// Read the response body.
byte[] responseBody = method.getResponseBody();
System.out.println(responseBody);
System.out.println(method.getResponseHeader("response"));
// Deal with the response.
// Use caution: ensure correct character encoding and is not binary data
System.out.println(new String(responseBody));
} catch (HttpException e) {
System.err.println("Fatal protocol violation: " + e.getMessage());
e.printStackTrace();
} catch (IOException e) {
System.err.println("Fatal transport error: " + e.getMessage());
e.printStackTrace();
} finally {
// Release the connection.
method.releaseConnection();
}
}
public WebResponse doGet(String url, String referer)
throws WebBrowserException, WebServerException {
try {
GetMethod getMethod = new GetMethod(url);
setHttpRequestHeader(getMethod);
if (referer != null && referer.trim().length() != 0) {
getMethod.setRequestHeader("Referer", referer);
}
logHttpGetRequest(getMethod);
int status = httpClient.executeMethod(getMethod);
String strResp = getMethod.getResponseBodyAsString();
byte[] byteResp = getMethod.getResponseBody();
String respEnc = getResponseEncoding(getMethod);
logHttpResponse(getMethod, strResp);
getMethod.releaseConnection();
// http:301,302,303,307
if (status == HttpStatus.SC_MOVED_PERMANENTLY
|| status == HttpStatus.SC_MOVED_TEMPORARILY
|| status == HttpStatus.SC_SEE_OTHER
|| status == HttpStatus.SC_TEMPORARY_REDIRECT) {
Header locationHeader = getMethod.getResponseHeader("Location");
String location = locationHeader.getValue();
if (logger.isDebugEnabled()) {
logger.debug("Redirect To Location = " + location);
}
if (location.startsWith("http")) {
return doGet(location);
} else {
return doGet("http://" + getResponseHost(getMethod) + location);
}
} else if (status == HttpStatus.SC_OK) { // http:200
return new WebResponse(getMethod.getURI().toString(), byteResp, respEnc);
} else {
throw new WebServerException("Server Exception[code=" + status + "]");
}
} catch (HttpException e) {
throw new WebBrowserException(e);
} catch (IOException e) {
throw new WebBrowserException(e);
}
}
@Override
public String execute() throws Exception {
request.setAttribute("ips", ips);
request.setAttribute("startpage", pageno);
String connip = ips[pageno];
// 连接url
String conpath = "/pool/status";
HttpClient httpClient = new HttpClient();
GetMethod getMethod = new GetMethod("http://" + connip + conpath);
getMethod
.getParams()
.setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler());
try {
// 执行getMethod
int statusCode = httpClient.executeMethod(getMethod);
if (statusCode != HttpStatus.SC_OK) {
System.err.println("Method failed: " + getMethod.getStatusLine());
}
// 读取内容
byte[] responseBody = getMethod.getResponseBody();
// 处理内容
JSONObject jo = JSONObject.fromObject(new String(responseBody));
cinfo = (ConnInfo) jo.toBean(jo, ConnInfo.class);
} catch (HttpException e) {
// 发生致命的异常，可能是协议不对或者返回的内容有问题
System.err.println("Please check your provided http address!");
e.printStackTrace();
return ERROR;
} catch (IOException e) {
// 发生网络异常
e.printStackTrace();
return ERROR;
} finally {
// 释放连接
getMethod.releaseConnection();
}
return SUCCESS;
}
/**
* Utility method to send message to kannel from this servlet.
*
* @param message
*/
public void sendMessageToKannel(
String message, String phoneNumber, String username, String password) {
String smsMessage = "";
try {
smsMessage = URLEncoder.encode(message, "UTF-8");
} catch (UnsupportedEncodingException ex) {
logger.debug(
SendSmsToKannelService.class
+ ":sendMessageToKannel() has thrown an exception\n"
+ ex.getMessage());
}
String kannelUrl = "http://10.210.7.193:13009/cgi-bin/sendsms";
String kannelParam =
"?username="******"&password="******"&to="
+ phoneNumber
+ "&text="
+ smsMessage
+ "&from=0802";
HttpClient client = new HttpClient();
GetMethod method = new GetMethod(kannelUrl + kannelParam);
method
.getParams()
.setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false));
try {
// Execute the method.
int statusCode = client.executeMethod(method);
if (statusCode != HttpStatus.SC_OK) {
System.err.println("Method failed: " + method.getStatusLine());
}
// Read the response body.
byte[] responseBody = method.getResponseBody();
// Deal with the response.
// Use caution: ensure correct character encoding and is not binary data
System.out.println(new String(responseBody));
} catch (HttpException e) {
System.err.println("Fatal protocol violation: " + e.getMessage());
e.printStackTrace();
} catch (IOException e) {
System.err.println("Fatal transport error: " + e.getMessage());
e.printStackTrace();
} finally {
// Release the connection.
method.releaseConnection();
}
}
public static Document getRssDocument(Context context, InputStream is, Bundle feedBundle) {
GetMethod method = null;
BufferedReader in = null;
try {
// Download the feed
// url is the bitstream contents
in = new BufferedReader(new InputStreamReader(is));
String url = in.readLine();
// need to replace the feed prefix with the HTTP protocol ie replace feed:// with http://
url = url.replaceAll(PackageDetectorStep.FEED_PREFIX, "http://");
HttpClient client = new HttpClient();
// Create a method instance.
method = new GetMethod(url);
// Provide custom retry handler is necessary
method
.getParams()
.setParameter(
HttpMethodParams.RETRY_HANDLER, new DefaultHttpMethodRetryHandler(3, false));
// Execute the method.
int statusCode = client.executeMethod(method);
if (statusCode != HttpStatus.SC_OK) {
throw new Exception("Method failed: " + method.getStatusLine());
}
// Read the response body.
byte[] responseBody = method.getResponseBody();
// Store the response in a bitstream so that the ingester can use it later
if (feedBundle != null) {
BitstreamFormat bs_format = BitstreamFormat.findByShortDescription(context, "Text");
BundleUtils.setBitstreamFromBytes(
feedBundle, Constants.FEED_BUNDLE_CONTENTS_NAME, bs_format, responseBody, false);
}
// Now see if we have a RSS v2.0 document
SAXBuilder builder = new SAXBuilder(false);
Document xmlDoc = builder.build(new ByteArrayInputStream(responseBody));
// XMLOutputter outputPretty = new XMLOutputter(Format.getPrettyFormat());
// log.debug("Got RSS DOCUMENT:");
// log.debug(outputPretty.outputString(xmlDoc));
return xmlDoc;
} catch (Exception e) {
ExceptionLogger.logException(log, e);
} finally {
// Release the connection.
method.releaseConnection();
try {
in.close();
} catch (Exception e) {
}
}
return null;
}
