/**
* Call api method.
*
* @param apiUrl the api url
* @param xmlContent the xml content
* @param contentType the content type
* @param method the method
* @param expected the expected
* @return the input stream
*/
protected InputStream callApiMethod(
String apiUrl, String xmlContent, String contentType, String method, int expected) {
try {
URL url = new URL(apiUrl);
HttpURLConnection request = (HttpURLConnection) url.openConnection();
if (ApplicationConstants.CONNECT_TIMEOUT > -1) {
request.setConnectTimeout(ApplicationConstants.CONNECT_TIMEOUT);
}
if (ApplicationConstants.READ_TIMEOUT > -1) {
request.setReadTimeout(ApplicationConstants.READ_TIMEOUT);
}
for (String headerName : requestHeaders.keySet()) {
request.setRequestProperty(headerName, requestHeaders.get(headerName));
}
request.setRequestMethod(method);
request.setDoOutput(true);
if (contentType != null) {
request.setRequestProperty("Content-Type", contentType);
}
if (xmlContent != null) {
PrintStream out = new PrintStream(new BufferedOutputStream(request.getOutputStream()));
out.print(xmlContent);
out.flush();
out.close();
}
request.connect();
if (request.getResponseCode() != expected) {
throw new GitHubException(
convertStreamToString(
getWrappedInputStream(
request.getErrorStream(),
GZIP_ENCODING.equalsIgnoreCase(request.getContentEncoding()))));
} else {
return getWrappedInputStream(
request.getInputStream(), GZIP_ENCODING.equalsIgnoreCase(request.getContentEncoding()));
}
} catch (IOException e) {
throw new GitHubException(e);
}
}
protected String sendPost() {
HttpURLConnection uRLConnection = null;
InputStream is = null;
BufferedReader buffer = null;
String result = null;
try {
URL url = new URL(Constants.APP_UPDATE_SERVER_URL);
uRLConnection = (HttpURLConnection) url.openConnection();
uRLConnection.setDoInput(true);
uRLConnection.setDoOutput(true);
uRLConnection.setRequestMethod("POST");
uRLConnection.setUseCaches(false);
uRLConnection.setConnectTimeout(10 * 1000);
uRLConnection.setReadTimeout(10 * 1000);
uRLConnection.setInstanceFollowRedirects(false);
uRLConnection.setRequestProperty("Connection", "Keep-Alive");
uRLConnection.setRequestProperty("Charset", "UTF-8");
uRLConnection.setRequestProperty("Accept-Encoding", "gzip, deflate");
uRLConnection.setRequestProperty("Content-Type", "application/json");
uRLConnection.connect();
is = uRLConnection.getInputStream();
String content_encode = uRLConnection.getContentEncoding();
if (null != content_encode && !"".equals(content_encode) && content_encode.equals("gzip")) {
is = new GZIPInputStream(is);
}
buffer = new BufferedReader(new InputStreamReader(is));
StringBuilder strBuilder = new StringBuilder();
String line;
while ((line = buffer.readLine()) != null) {
strBuilder.append(line);
}
result = strBuilder.toString();
} catch (Exception e) {
Log.e(TAG, "http post error", e);
} finally {
if (buffer != null) {
try {
buffer.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if (is != null) {
try {
is.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if (uRLConnection != null) {
uRLConnection.disconnect();
}
}
return result;
}
/**
* 获取网页字符串
*
* @param path
* @return
* @throws URISyntaxException
* @throws ClientProtocolException
* @throws IOException
*/
public static String getHtml(Context context, String path) throws Exception {
String html = "";
HttpURLConnection conn = getConnection(context, path);
conn.setConnectTimeout(8000);
conn.setRequestMethod("GET");
conn.setRequestProperty("Cache-Control", "no-cache");
conn.setRequestProperty("Host", "szlib.gov.cn");
conn.setRequestProperty("Accept-Encoding", "gzip, deflate");
conn.setUseCaches(false);
if (conn.getResponseCode() == 200) {
InputStream inputStream = null;
String encoding = conn.getContentEncoding();
if (encoding != null) {
if (-1 != encoding.indexOf("gzip")) {
inputStream = new GZIPInputStream(conn.getInputStream());
} else if (-1 != encoding.indexOf("deflate")) {
inputStream = new InflaterInputStream(conn.getInputStream());
}
} else {
inputStream = conn.getInputStream();
}
html = new String(read(inputStream), "UTF-8");
}
conn.disconnect();
return html;
}
/**
* Login with user name and password sent as String parameter to url using POST Interrogation
*
* @param username
* @param userpass
* @throws IOException
*/
@And("^I make post message with user: \"([^\"]*)\" and password: \"([^\"]*)\"$")
public void HttpPostForm(String username, String userpass) throws IOException {
URL url = new URL("http://dippy.trei.ro");
HttpURLConnection hConnection = (HttpURLConnection) url.openConnection();
HttpURLConnection.setFollowRedirects(true);
hConnection.setDoOutput(true);
hConnection.setRequestMethod("POST");
PrintStream ps = new PrintStream(hConnection.getOutputStream());
ps.print("user="******"&pass="******";do_login=Login");
ps.close();
hConnection.connect();
if (HttpURLConnection.HTTP_OK == hConnection.getResponseCode()) {
InputStream is = hConnection.getInputStream();
System.out.println("!!! encoding: " + hConnection.getContentEncoding());
System.out.println("!!! message: " + hConnection.getResponseMessage());
is.close();
hConnection.disconnect();
}
}
private String readResult(HttpURLConnection urlConnection) throws WeiboException {
InputStream is = null;
BufferedReader buffer = null;
GlobalContext globalContext = GlobalContext.getInstance();
String errorStr = globalContext.getString(R.string.timeout);
globalContext = null;
try {
is = urlConnection.getInputStream();
String content_encode = urlConnection.getContentEncoding();
if (null != content_encode && !"".equals(content_encode) && content_encode.equals("gzip")) {
is = new GZIPInputStream(is);
}
buffer = new BufferedReader(new InputStreamReader(is));
StringBuilder strBuilder = new StringBuilder();
String line;
while ((line = buffer.readLine()) != null) {
strBuilder.append(line);
}
AppLogger.d("result=" + strBuilder.toString());
return strBuilder.toString();
} catch (IOException e) {
e.printStackTrace();
throw new WeiboException(errorStr, e);
} finally {
Utility.closeSilently(is);
Utility.closeSilently(buffer);
urlConnection.disconnect();
}
}
/**
* Handles the "Content-Encoding" header.
*/
private InputStream wrapStream(HttpURLConnection uc, InputStream in) throws IOException {
String encoding = uc.getContentEncoding();
if (encoding==null || in==null) return in;
if (encoding.equals("gzip")) return new GZIPInputStream(in);
throw new UnsupportedOperationException("Unexpected Content-Encoding: "+encoding);
}
/**
* 从Http连接的头信息中获得字符集
*
* @param conn HTTP连接对象
* @return 字符集
*/
private static String getCharsetFromConn(HttpURLConnection conn) {
String charset = conn.getContentEncoding();
if (charset == null || "".equals(charset.trim())) {
String contentType = conn.getContentType();
charset = RegexUtil.get("charset=(.*)", contentType, 1);
}
return charset;
}
/**
* Call api post.
*
* @param apiUrl the api url
* @param parameters the parameters
* @param expected the expected
* @return the input stream
*/
protected InputStream callApiPost(String apiUrl, Map<String, String> parameters, int expected) {
try {
URL url = new URL(apiUrl);
HttpURLConnection request = (HttpURLConnection) url.openConnection();
if (ApplicationConstants.CONNECT_TIMEOUT > -1) {
request.setConnectTimeout(ApplicationConstants.CONNECT_TIMEOUT);
}
if (ApplicationConstants.READ_TIMEOUT > -1) {
request.setReadTimeout(ApplicationConstants.READ_TIMEOUT);
}
for (String headerName : requestHeaders.keySet()) {
request.setRequestProperty(headerName, requestHeaders.get(headerName));
}
parameters.putAll(requestParameters);
request.setRequestMethod("POST");
request.setDoOutput(true);
PrintStream out = new PrintStream(new BufferedOutputStream(request.getOutputStream()));
out.print(getParametersString(parameters));
out.flush();
out.close();
request.connect();
if (request.getResponseCode() != expected) {
throw new GitHubException(
convertStreamToString(
getWrappedInputStream(
request.getErrorStream(),
GZIP_ENCODING.equalsIgnoreCase(request.getContentEncoding()))));
} else {
return getWrappedInputStream(
request.getInputStream(), GZIP_ENCODING.equalsIgnoreCase(request.getContentEncoding()));
}
} catch (IOException e) {
throw new GitHubException(e);
} finally {
}
}
/**
* Download the URL and return as a String. Gzip handling from http://goo.gl/J88WG
*
* @param url the URL to download
* @return String of the URL contents
* @throws IOException when there is an error connecting or reading the URL
*/
public String downloadUrl(URL url) throws TVRenamerIOException {
InputStream inputStream = null;
StringBuilder contents = new StringBuilder();
try {
if (url != null) {
logger.log(Level.INFO, "Downloading URL {0}", url.toString());
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
HttpURLConnection.setFollowRedirects(true);
// allow both GZip and Deflate (ZLib) encodings
conn.setRequestProperty("Accept-Encoding", "gzip, deflate");
conn.setConnectTimeout(CONNECT_TIMEOUT_MS);
conn.setReadTimeout(READ_TIMEOUT_MS);
// create the appropriate stream wrapper based on the encoding type
String encoding = conn.getContentEncoding();
if (encoding != null && encoding.equalsIgnoreCase("gzip")) {
inputStream = new GZIPInputStream(conn.getInputStream());
} else if (encoding != null && encoding.equalsIgnoreCase("deflate")) {
inputStream = new InflaterInputStream(conn.getInputStream(), new Inflater(true));
} else {
inputStream = conn.getInputStream();
}
// always specify encoding while reading streams
BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8"));
logger.finer("Before reading url stream");
String s;
while ((s = reader.readLine()) != null) {
contents.append(s);
}
if (logger.isLoggable(Level.FINEST)) {
// no need to encode for logger output
logger.log(Level.FINEST, "Url stream:\n{0}", contents.toString());
}
}
} catch (Exception e) {
String message = "Exception when attempting to download and parse URL " + url;
logger.log(Level.SEVERE, message, e);
throw new TVRenamerIOException(message, e);
} finally {
try {
if (inputStream != null) {
inputStream.close();
}
} catch (IOException e) {
logger.log(Level.SEVERE, "Exception when attempting to close input stream", e);
}
}
return StringUtils.encodeSpecialCharacters(contents.toString());
}
public Response(HttpURLConnection con) throws IOException {
this.con = con;
this.statusCode = con.getResponseCode();
if (null == (is = con.getErrorStream())) {
is = con.getInputStream();
}
if (null != is && "gzip".equals(con.getContentEncoding())) {
// the response is gzipped
is = new GZIPInputStream(is);
}
}
/**
* 从Http连接的头信息中获得字符集
*
* @param conn HTTP连接对象
* @return 字符集
*/
public static String getCharset(HttpURLConnection conn) {
if (conn == null) {
return null;
}
String charset = conn.getContentEncoding();
if (StrUtil.isBlank(charset)) {
charset = ReUtil.get(CHARSET_PATTERN, conn.getContentType(), 1);
}
return charset;
}
/**
* Call api get.
*
* @param apiUrl the api url
* @param expected the expected
* @return the input stream
*/
protected InputStream callApiGet(String apiUrl, int expected) {
try {
URL url = new URL(apiUrl);
if (!requestParameters.isEmpty()) {
if (url.getQuery() == null) {
url = new URL(apiUrl + "?" + getParametersString(requestParameters));
} else {
url = new URL(apiUrl + "&" + getParametersString(requestParameters));
}
}
HttpURLConnection request = (HttpURLConnection) url.openConnection();
if (ApplicationConstants.CONNECT_TIMEOUT > -1) {
request.setConnectTimeout(ApplicationConstants.CONNECT_TIMEOUT);
}
if (ApplicationConstants.READ_TIMEOUT > -1) {
request.setReadTimeout(ApplicationConstants.READ_TIMEOUT);
}
for (String headerName : requestHeaders.keySet()) {
request.setRequestProperty(headerName, requestHeaders.get(headerName));
}
request.connect();
if (request.getResponseCode() != expected) {
throw new GitHubException(
convertStreamToString(
getWrappedInputStream(
request.getErrorStream(),
GZIP_ENCODING.equalsIgnoreCase(request.getContentEncoding()))));
} else {
return getWrappedInputStream(
request.getInputStream(), GZIP_ENCODING.equalsIgnoreCase(request.getContentEncoding()));
}
} catch (IOException e) {
throw new GitHubException(e);
}
}
/**
* 指定されたURLから、フィードのURLを検索する。
*
* @param url フィードURL検索先
* @return 検索結果リスト。見つからなかった場合は空のArrayListを返却する。
* @throws FeedParseException HTML解析中にエラーが発生した場合
*/
public ArrayList<FeedSource> findFeedUrl(String strUrl) throws FeedParseException {
ArrayList<FeedSource> feedList = new ArrayList<FeedSource>();
HttpURLConnection connection = null;
InputStream is = null;
try {
// 指定されたURLに接続する。
connection = connectURL(strUrl);
is = connection.getInputStream();
String enc = connection.getContentEncoding();
if (enc == null) {
// content-encodingヘッダが設定されて無い場合はとりあえずutf-8を設定。
enc = "utf-8";
}
// 接続したサイトのソースを取得し、パースする。
DOMResult result = parseDOM(is, enc);
// linkタグを取得する。
Document doc = (Document) result.getNode();
String docEnc = getDocumentEncoding(doc);
if (docEnc != null && !enc.equals(docEnc)) {
// Documentの文字コードが事前に設定したものと違う場合は、
// 文字コードを変更し、再接続する。
dispose(connection, is);
connection = connectURL(strUrl);
is = connection.getInputStream();
result = parseDOM(is, docEnc);
doc = (Document) result.getNode();
}
NodeList childs = doc.getElementsByTagName(TAG_NAME_LINK);
for (int i = 0; i < childs.getLength(); i++) {
Element element = (Element) childs.item(i);
if (isRssAutoDiscovery(element)) {
FeedSource entity = setFeedSource(element);
FeedParseFacade facade = new FeedParseFacade();
entity.setVersion(facade.getRssVersion(entity.getFeedUrl()));
feedList.add(entity);
}
}
} catch (Exception e) {
throw new FeedParseException(e);
} finally {
// 後始末
dispose(connection, is);
}
return feedList;
}
@Override
protected void runChild(FrameworkMethod method, RunNotifier notifier) {
Description description = describeChild(method);
if (method.getAnnotation(Ignore.class) != null) {
notifier.fireTestIgnored(description);
return;
}
String methodName = method.getName();
String allowedMethodName = methodNames.get(description);
if (methodName != null && !methodName.equals(allowedMethodName)) {
// notifier.fireTestIgnored(description);
return;
}
try {
notifier.fireTestStarted(description);
HttpURLConnection connection = getUrl(methodName, "POST");
handleError(connection);
String enc = connection.getContentEncoding();
if (enc == null) {
enc = "ISO-8859-1";
}
InputStream is = connection.getInputStream();
BufferedReader reader = new BufferedReader(new InputStreamReader(is, Charset.forName(enc)));
String line = null;
while ((line = reader.readLine()) != null) {
if (line.startsWith("E")) {
System.err.println(line.substring(1).trim());
} else if (line.startsWith("O")) {
System.out.println(line.substring(1).trim());
} else if (line.startsWith("RSUCCESS")) {
break;
} else if (line.startsWith("RERROR")) {
StringBuilder error = new StringBuilder(line.substring(6));
while ((line = reader.readLine()) != null) {
error.append(line).append("\n");
}
throw new AssertionFailedError(error.toString());
} else {
log.error("Protocol error in response: {}", line);
}
}
is.close();
connection.disconnect();
} catch (Throwable e) {
e.printStackTrace();
notifier.fireTestFailure(new Failure(description, e));
} finally {
notifier.fireTestFinished(description);
}
}
/**
* Initializes an {@link HttpEntity} from the given {@link HttpURLConnection}.
*
* @param connection
* @return an HttpEntity populated with data from <code>connection</code>.
*/
private static HttpEntity entityFromConnection(HttpURLConnection connection) {
BasicHttpEntity entity = new BasicHttpEntity();
InputStream inputStream;
try {
inputStream = connection.getInputStream();
} catch (IOException ioe) {
inputStream = connection.getErrorStream();
}
entity.setContent(inputStream);
entity.setContentLength(connection.getContentLength());
entity.setContentEncoding(connection.getContentEncoding());
entity.setContentType(connection.getContentType());
return entity;
}
private static HttpEntity a(HttpURLConnection httpurlconnection) {
BasicHttpEntity basichttpentity = new BasicHttpEntity();
java.io.InputStream inputstream;
try {
inputstream = httpurlconnection.getInputStream();
} catch (IOException ioexception) {
ioexception = httpurlconnection.getErrorStream();
}
basichttpentity.setContent(inputstream);
basichttpentity.setContentLength(httpurlconnection.getContentLength());
basichttpentity.setContentEncoding(httpurlconnection.getContentEncoding());
basichttpentity.setContentType(httpurlconnection.getContentType());
return basichttpentity;
}
/**
* Retrieve the {@link WeatherInformation} from a site containing an XML feed of weather
* information
*
* @param provider {@link WeatherInformation}
* @return {@link WeatherInformation} populated with data
* @throws IllegalArgumentException If there is an error parsing weather information
* @throws IOException If there is an error parsing weather information
*/
public WeatherInformation retrieveForecast(WeatherInformation provider)
throws IllegalArgumentException, IOException {
URL forecastUrl = new URL(provider.getProviderUrl());
URLConnection connection = forecastUrl.openConnection();
if (!(connection instanceof HttpURLConnection)) {
throw new IllegalArgumentException(forecastUrl.toExternalForm() + " is not a valid HTTP Url");
}
HttpURLConnection httpConnection = (HttpURLConnection) connection;
httpConnection.setRequestProperty("Accept-Encoding", WeatherConstants.GZIP);
httpConnection.setRequestProperty("User-Agent", WeatherConstants.BLOJSOM_WEATHER_USER_AGENT);
httpConnection.connect();
InputStream is = connection.getInputStream();
BufferedInputStream bis;
if (WeatherConstants.GZIP.equalsIgnoreCase(httpConnection.getContentEncoding())) {
bis = new BufferedInputStream(new GZIPInputStream(is));
} else {
bis = new BufferedInputStream(is);
}
StringBuffer buffer = new StringBuffer();
BufferedReader br = new BufferedReader(new InputStreamReader(bis));
String input;
while ((input = br.readLine()) != null) {
buffer.append(input).append(BlojsomConstants.LINE_SEPARATOR);
}
try {
Document document =
_documentBuilder.parse(new InputSource(new StringReader(buffer.toString())));
provider.parseDocument(document);
} catch (SAXException e) {
if (_logger.isErrorEnabled()) {
_logger.error(e);
}
}
bis.close();
return provider;
}
/**
* 执行HTTP请求之后获取到的数据流.
*
* @param connection
* @return
*/
private HttpEntity entityFromURLConnwction(HttpURLConnection connection) {
BasicHttpEntity entity = new BasicHttpEntity();
InputStream inputStream;
try {
inputStream = connection.getInputStream();
} catch (IOException e) {
e.printStackTrace();
inputStream = connection.getErrorStream();
}
// TODO : GZIP
entity.setContent(inputStream);
entity.setContentLength(connection.getContentLength());
entity.setContentEncoding(connection.getContentEncoding());
entity.setContentType(connection.getContentType());
return entity;
}
/** @param args */
public static void main(String[] args) {
// define the location of the resource
URL url = null;
try {
url = new URL("http://localhost:8080/ir/item/escidoc:ex5");
} catch (MalformedURLException e) {
System.err.println("Mal formed URL '" + url + "'");
System.exit(1);
}
// establish connection
try {
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("GET");
// check response code
if (conn.getResponseCode() != HttpURLConnection.HTTP_OK) {
System.err.println(conn.getResponseMessage());
} else {
// write out data
String contentEncoding = conn.getContentEncoding();
if (contentEncoding == null) {
contentEncoding = "UTF-8";
}
BufferedReader in =
new BufferedReader(new InputStreamReader(conn.getInputStream(), contentEncoding));
String line = in.readLine();
while (line != null) {
System.out.println(line);
line = in.readLine();
}
in.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
/** null rootTreeId implies "created from workDir" */
public ThreedModel createModelFromJs(SeriesKey seriesKey, URL url) throws IOException {
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
urlConnection.setRequestProperty("Accept-Encoding", "gzip, deflate");
try {
urlConnection.connect();
} catch (IOException e) {
log.log(Level.SEVERE, "RepoClient - urlConnection.connect() failed for url[" + url + "] ", e);
throw e;
}
String encoding = urlConnection.getContentEncoding();
InputStream is = null;
JsonNode jsThreedModel;
try {
if (encoding != null && encoding.equalsIgnoreCase("gzip")) {
is = new GZIPInputStream(urlConnection.getInputStream());
} else if (encoding != null && encoding.equalsIgnoreCase("deflate")) {
is = new InflaterInputStream(urlConnection.getInputStream(), new Inflater(true));
} else {
is = urlConnection.getInputStream();
}
ObjectMapper mapper = new ObjectMapper();
jsThreedModel = mapper.readValue(is, JsonNode.class);
} finally {
Closeables.closeQuietly(is);
}
JsonNode jsFm = jsThreedModel.get("featureModel");
JsonNode jsIm = jsThreedModel.get("imageModel");
JsonToFmJvm uFm = new JsonToFmJvm();
final FeatureModel featureModel = uFm.parseJson(seriesKey, jsFm);
ImageModel im = JsonToImJvm.parse(featureModel, jsIm);
ThreedModel threedModel = new ThreedModel(featureModel, im);
return threedModel;
}
/** connect to DAS server and return result as an InputStream */
private InputStream connectDASServer(URL url) throws IOException {
System.out.println(getTimeStamp());
System.out.println("opening connection to DAS Structure server");
HttpURLConnection huc = null;
// huc = (HttpURLConnection) dasUrl.openConnection();
// huc = proxyUrl.openConnection();
// System.out.println("opening "+url);
huc = (HttpURLConnection) url.openConnection();
System.out.println(huc);
// System.out.println("deacactivated encoding temporarily");
// should make communication much faster!
huc.setRequestProperty("Accept-Encoding", "gzip");
// System.out.println(huc.getResponseMessage());
System.out.println("getContentEncoding");
String contentEncoding = huc.getContentEncoding();
System.out.println("getInputStream");
InputStream inStream = huc.getInputStream();
if (contentEncoding != null) {
if (contentEncoding.indexOf("gzip") != -1) {
// we have gzip encoding
inStream = new GZIPInputStream(inStream);
System.out.println("using gzip encoding!");
}
}
System.out.println(getTimeStamp());
System.out.println("got InputStream from DAS Structure server");
System.out.println("encoding: " + contentEncoding);
System.out.println("code:" + huc.getResponseCode());
// System.out.println("message:" + huc.getResponseMessage());
// inStream = huc.getInputStream();
return inStream;
}
private String readError(HttpURLConnection urlConnection) throws WeiboException {
InputStream is = null;
BufferedReader buffer = null;
BeeboApplication globalContext = BeeboApplication.getInstance();
String errorStr = globalContext.getString(R.string.timeout);
try {
is = urlConnection.getErrorStream();
if (is == null) {
errorStr = globalContext.getString(R.string.unknown_sina_network_error);
throw new WeiboException(errorStr);
}
String content_encode = urlConnection.getContentEncoding();
if (!TextUtils.isEmpty(content_encode) && content_encode.equals("gzip")) {
is = new GZIPInputStream(is);
}
buffer = new BufferedReader(new InputStreamReader(is));
StringBuilder strBuilder = new StringBuilder();
String line;
while ((line = buffer.readLine()) != null) {
strBuilder.append(line);
}
AppLoggerUtils.d("error result=" + strBuilder.toString());
return strBuilder.toString();
} catch (IOException e) {
e.printStackTrace();
throw new WeiboException(errorStr, e);
} finally {
Utility.closeSilently(is);
Utility.closeSilently(buffer);
urlConnection.disconnect();
globalContext = null;
}
}
static byte[] getResponseContent(HttpURLConnection http) {
InputStream in;
try {
in = http.getInputStream();
String contentEncoding = http.getContentEncoding();
if (contentEncoding != null) {
if (contentEncoding.equalsIgnoreCase("gzip")) {
in =
new GZIPInputStream(http.getInputStream()); // reads 2 bytes to determine GZIP stream!
} else if (contentEncoding.equalsIgnoreCase("deflate")) {
in = new InflaterInputStream(http.getInputStream());
}
}
} catch (IOException e) {
in = http.getErrorStream();
} catch (Exception e) {
Log.e("URLConnection exception", e.toString());
return null;
}
try {
byte[] buffer = new byte[1024];
int size = 0;
ByteArrayOutputStream bytestream = new ByteArrayOutputStream();
while ((size = in.read(buffer, 0, 1024)) != -1) {
bytestream.write(buffer, 0, size);
}
byte retbuffer[] = bytestream.toByteArray();
bytestream.close();
return retbuffer;
} catch (Exception e) {
Log.e("URLConnection exception", e.toString());
}
return null;
}
public static void main(String[] args) {
try {
// Se obtiene el objeto URL.
URL url = new URL("http://getbootstrap.com");
// Se abre la conexión y se hace a HttpURLConnection, ya que es una
// conexión HTTP.
HttpURLConnection conexion = (HttpURLConnection) url.openConnection();
// Se obtienen los datos de la cabecera de la respuesta y se
// muestran por pantalla.
// Campos con método get propio.
System.out.println("\nCABECERA DE LA RESPUESTA\n");
System.out.println("CAMPOS CON MÉTODO GET PROPIO\n");
System.out.println("Método de petición [getRequestMethod()]: " + conexion.getRequestMethod());
System.out.println("Fecha petición [getDate()]: " + new Date(conexion.getDate()));
System.out.println("Código de respuesta [getResponseCode()]: " + conexion.getResponseCode());
System.out.println(
"Mensaje de respuesta [getResponseMessage()]: " + conexion.getResponseMessage());
System.out.println(
"Fecha última modificación [getLastModified()]: " + new Date(conexion.getLastModified()));
System.out.println("Tipo de contenido [getContentType()]: " + conexion.getContentType());
System.out.println("Codificación [getContentEncoding()]: " + conexion.getContentEncoding());
System.out.println(
"Tamaño del contenido [getContentLength()]: " + conexion.getContentLength());
System.out.println(
"Fecha expiración [getExpiration()]: " + new Date(conexion.getExpiration()));
// Todos los campos de la cabecera.
System.out.println("\nTODOS LOS CAMPOS DE LA CABECERA DE LA RESPUESTA [getHeaderFields()]\n");
Map<String, List<String>> cabeceraRespuesta = conexion.getHeaderFields();
for (Map.Entry<String, List<String>> campo : cabeceraRespuesta.entrySet()) {
System.out.println(campo.getKey() + " : " + campo.getValue());
}
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
@Override
public String getContentEncoding() {
return connection.getContentEncoding();
}
public boolean doGetSaveFile(
String urlStr, String path, FileDownloaderHttpHelper.DownloadListener downloadListener) {
File file = FileManager.createNewFileInSDCard(path);
if (file == null) {
return false;
}
boolean result = false;
BufferedOutputStream out = null;
InputStream in = null;
HttpURLConnection urlConnection = null;
try {
URL url = new URL(urlStr);
AppLoggerUtils.d("download request=" + urlStr);
Proxy proxy = getProxy();
if (proxy != null) {
urlConnection = (HttpURLConnection) url.openConnection(proxy);
} else {
urlConnection = (HttpURLConnection) url.openConnection();
}
urlConnection.setRequestMethod("GET");
urlConnection.setDoOutput(false);
urlConnection.setConnectTimeout(DOWNLOAD_CONNECT_TIMEOUT);
urlConnection.setReadTimeout(DOWNLOAD_READ_TIMEOUT);
urlConnection.setRequestProperty("Connection", "Keep-Alive");
urlConnection.setRequestProperty("Charset", "UTF-8");
urlConnection.setRequestProperty("Accept-Encoding", "gzip, deflate");
urlConnection.connect();
int status = urlConnection.getResponseCode();
if (status != HttpURLConnection.HTTP_OK) {
return false;
}
int bytetotal = (int) urlConnection.getContentLength();
int bytesum = 0;
int byteread = 0;
out = new BufferedOutputStream(new FileOutputStream(file));
InputStream is = urlConnection.getInputStream();
String content_encode = urlConnection.getContentEncoding();
if (!TextUtils.isEmpty(content_encode) && content_encode.equals("gzip")) {
is = new GZIPInputStream(is);
}
in = new BufferedInputStream(is);
final Thread thread = Thread.currentThread();
byte[] buffer = new byte[1444];
while ((byteread = in.read(buffer)) != -1) {
if (thread.isInterrupted()) {
if (((float) bytesum / (float) bytetotal) < 0.8f) {
file.delete();
throw new InterruptedIOException();
}
}
bytesum += byteread;
out.write(buffer, 0, byteread);
if (downloadListener != null && bytetotal > 0) {
downloadListener.pushProgress(bytesum, bytetotal);
}
}
if (downloadListener != null) {
downloadListener.completed();
}
AppLoggerUtils.v("download request= " + urlStr + " download finished");
result = true;
} catch (IOException e) {
e.printStackTrace();
AppLoggerUtils.v("download request= " + urlStr + " download failed");
} finally {
Utility.closeSilently(in);
Utility.closeSilently(out);
if (urlConnection != null) {
urlConnection.disconnect();
}
}
return result && ImageUtility.isThisBitmapCanRead(path);
}
/**
* Retrieves a file from a given URL and saves it to the outputPath.
*
* @param url the URL of the file to download
* @param outputPath the path to the save the file to
* @param useProxy whether to use the configured proxy when downloading files
* @throws DownloadFailedException is thrown if there is an error downloading the file
*/
public static void fetchFile(URL url, File outputPath, boolean useProxy)
throws DownloadFailedException {
if ("file".equalsIgnoreCase(url.getProtocol())) {
File file;
try {
file = new File(url.toURI());
} catch (URISyntaxException ex) {
final String msg = format("Download failed, unable to locate '%s'", url.toString());
throw new DownloadFailedException(msg);
}
if (file.exists()) {
try {
org.apache.commons.io.FileUtils.copyFile(file, outputPath);
} catch (IOException ex) {
final String msg =
format(
"Download failed, unable to copy '%s' to '%s'",
url.toString(), outputPath.getAbsolutePath());
throw new DownloadFailedException(msg);
}
} else {
final String msg = format("Download failed, file ('%s') does not exist", url.toString());
throw new DownloadFailedException(msg);
}
} else {
HttpURLConnection conn = null;
try {
LOGGER.debug("Attempting download of {}", url.toString());
conn = URLConnectionFactory.createHttpURLConnection(url, useProxy);
conn.setRequestProperty("Accept-Encoding", "gzip, deflate");
conn.connect();
int status = conn.getResponseCode();
int redirectCount = 0;
while ((status == HttpURLConnection.HTTP_MOVED_TEMP
|| status == HttpURLConnection.HTTP_MOVED_PERM
|| status == HttpURLConnection.HTTP_SEE_OTHER)
&& MAX_REDIRECT_ATTEMPTS > redirectCount++) {
final String location = conn.getHeaderField("Location");
try {
conn.disconnect();
} finally {
conn = null;
}
LOGGER.debug("Download is being redirected from {} to {}", url.toString(), location);
conn = URLConnectionFactory.createHttpURLConnection(new URL(location), useProxy);
conn.setRequestProperty("Accept-Encoding", "gzip, deflate");
conn.connect();
status = conn.getResponseCode();
}
if (status != 200) {
try {
conn.disconnect();
} finally {
conn = null;
}
final String msg =
format(
"Error downloading file %s; received response code %s.", url.toString(), status);
throw new DownloadFailedException(msg);
}
} catch (IOException ex) {
try {
if (conn != null) {
conn.disconnect();
}
} finally {
conn = null;
}
final String msg = format("Error downloading file %s; unable to connect.", url.toString());
throw new DownloadFailedException(msg, ex);
}
final String encoding = conn.getContentEncoding();
BufferedOutputStream writer = null;
InputStream reader = null;
try {
if (encoding != null && "gzip".equalsIgnoreCase(encoding)) {
reader = new GZIPInputStream(conn.getInputStream());
} else if (encoding != null && "deflate".equalsIgnoreCase(encoding)) {
reader = new InflaterInputStream(conn.getInputStream());
} else {
reader = conn.getInputStream();
}
writer = new BufferedOutputStream(new FileOutputStream(outputPath));
final byte[] buffer = new byte[4096];
int bytesRead;
while ((bytesRead = reader.read(buffer)) > 0) {
writer.write(buffer, 0, bytesRead);
}
LOGGER.debug("Download of {} complete", url.toString());
} catch (IOException ex) {
analyzeException(ex);
final String msg =
format(
"Error saving '%s' to file '%s'%nConnection Timeout: %d%nEncoding: %s%n",
url.toString(), outputPath.getAbsolutePath(), conn.getConnectTimeout(), encoding);
throw new DownloadFailedException(msg, ex);
} catch (Throwable ex) {
final String msg =
format(
"Unexpected exception saving '%s' to file '%s'%nConnection Timeout: %d%nEncoding: %s%n",
url.toString(), outputPath.getAbsolutePath(), conn.getConnectTimeout(), encoding);
throw new DownloadFailedException(msg, ex);
} finally {
if (writer != null) {
try {
writer.close();
} catch (IOException ex) {
LOGGER.trace("Error closing the writer in Downloader.", ex);
}
}
if (reader != null) {
try {
reader.close();
} catch (IOException ex) {
LOGGER.trace("Error closing the reader in Downloader.", ex);
}
}
try {
conn.disconnect();
} finally {
conn = null;
}
}
}
}
public void runInternal() {
connection = null;
try {
setProgressMessage(url.toString(), -1);
long startTimeStamp = System.currentTimeMillis();
delayedProgress =
coolReader.getEngine().showProgressDelayed(0, progressMessage, PROGRESS_DELAY_MILLIS);
URLConnection conn = url.openConnection();
if (conn instanceof HttpsURLConnection) {
onError("HTTPs is not supported yet");
return;
}
if (!(conn instanceof HttpURLConnection)) {
onError("Only HTTP supported");
return;
}
connection = (HttpURLConnection) conn;
connection.setRequestProperty("User-Agent", "CoolReader/3(Android)");
if (referer != null) connection.setRequestProperty("Referer", referer);
connection.setInstanceFollowRedirects(true);
connection.setAllowUserInteraction(false);
connection.setConnectTimeout(20000);
connection.setReadTimeout(40000);
connection.setDoInput(true);
String fileName = null;
String disp = connection.getHeaderField("Content-Disposition");
if (disp != null) {
int p = disp.indexOf("filename=");
if (p > 0) {
fileName = disp.substring(p + 9);
}
}
// connection.setDoOutput(true);
// connection.set
int response = -1;
response = connection.getResponseCode();
L.d("Response: " + response);
if (response != 200) {
onError("Error " + response);
return;
}
String contentType = connection.getContentType();
String contentEncoding = connection.getContentEncoding();
int contentLen = connection.getContentLength();
// connection.getC
L.d("Entity content length: " + contentLen);
L.d("Entity content type: " + contentType);
L.d("Entity content encoding: " + contentEncoding);
setProgressMessage(url.toString(), contentLen);
InputStream is = connection.getInputStream();
delayedProgress.cancel();
is = new ProgressInputStream(is, startTimeStamp, progressMessage, contentLen, 80);
final int MAX_CONTENT_LEN_TO_BUFFER = 256 * 1024;
boolean isZip = contentType != null && contentType.equals("application/zip");
if (expectedType != null) contentType = expectedType;
else if (contentLen > 0 && contentLen < MAX_CONTENT_LEN_TO_BUFFER) { // autodetect type
byte[] buf = new byte[contentLen];
if (is.read(buf) != contentLen) {
onError("Wrong content length");
return;
}
is.close();
is = null;
is = new ByteArrayInputStream(buf);
if (findSubstring(buf, "<?xml version=") >= 0 && findSubstring(buf, "<feed") >= 0)
contentType = "application/atom+xml"; // override type
}
if (contentType.startsWith("application/atom+xml")) {
L.d("Parsing feed");
parseFeed(is);
} else {
if (fileName == null) fileName = defaultFileName;
L.d("Downloading book: " + contentEncoding);
downloadBook(contentType, url.toString(), is, contentLen, fileName, isZip);
}
} catch (Exception e) {
L.e("Exception while trying to open URI " + url.toString(), e);
onError("Error occured while reading OPDS catalog");
} finally {
if (connection != null)
try {
connection.disconnect();
} catch (Exception e) {
// ignore
}
}
}
private static Map<String, ExchangeRate> requestExchangeRates(
final URL url,
float dogeBtcConversion,
final String userAgent,
final String source,
final String... fields) {
final long start = System.currentTimeMillis();
HttpURLConnection connection = null;
Reader reader = null;
try {
connection = (HttpURLConnection) url.openConnection();
connection.setInstanceFollowRedirects(false);
connection.setConnectTimeout(Constants.HTTP_TIMEOUT_MS);
connection.setReadTimeout(Constants.HTTP_TIMEOUT_MS);
connection.addRequestProperty("User-Agent", userAgent);
connection.addRequestProperty("Accept-Encoding", "gzip");
connection.connect();
final int responseCode = connection.getResponseCode();
if (responseCode == HttpURLConnection.HTTP_OK) {
final String contentEncoding = connection.getContentEncoding();
InputStream is = new BufferedInputStream(connection.getInputStream(), 1024);
if ("gzip".equalsIgnoreCase(contentEncoding)) is = new GZIPInputStream(is);
reader = new InputStreamReader(is, Constants.UTF_8);
final StringBuilder content = new StringBuilder();
final long length = Io.copy(reader, content);
final Map<String, ExchangeRate> rates = new TreeMap<String, ExchangeRate>();
final JSONObject head = new JSONObject(content.toString());
for (final Iterator<String> i = head.keys(); i.hasNext(); ) {
final String currencyCode = i.next();
if (!"timestamp".equals(currencyCode)) {
final JSONObject o = head.getJSONObject(currencyCode);
for (final String field : fields) {
final String rate = o.optString(field, null);
if (rate != null) {
try {
BigDecimal btcRate = new BigDecimal(GenericUtils.toNanoCoins(rate, 0));
BigInteger dogeRate =
btcRate.multiply(BigDecimal.valueOf(dogeBtcConversion)).toBigInteger();
if (dogeRate.signum() > 0) {
rates.put(currencyCode, new ExchangeRate(currencyCode, dogeRate, source));
break;
}
} catch (final ArithmeticException x) {
log.warn(
"problem fetching {} exchange rate from {} ({}): {}",
currencyCode,
url,
contentEncoding,
x.getMessage());
}
}
}
}
}
log.info(
"fetched exchange rates from {} ({}), {} chars, took {} ms",
url,
contentEncoding,
length,
System.currentTimeMillis() - start);
return rates;
} else {
log.warn("http status {} when fetching {}", responseCode, url);
}
} catch (final Exception x) {
log.warn("problem fetching exchange rates from " + url, x);
} finally {
if (reader != null) {
try {
reader.close();
} catch (final IOException x) {
// swallow
}
}
if (connection != null) connection.disconnect();
}
return null;
}
/** Scrapes the metadata on this page (can be called separately from {@link process} */
public void scrapeMetadata() {
final String logPrefix = "[" + this.getDestinationUrl() + "] ";
logger.trace(logPrefix + "Downloading and scraping page contents...");
InputStream inStr = null;
HttpURLConnection connection = null;
try {
URL url = new URL(this.getDestinationUrl());
connection = (HttpURLConnection) url.openConnection(Proxy.NO_PROXY);
connection.setConnectTimeout(LintedPage.HTTP_CONNECT_TIMEOUT);
connection.setReadTimeout(LintedPage.HTTP_READ_TIMEOUT);
connection.setRequestProperty("User-Agent", LintedPage.HTTP_USER_AGENT);
connection.setRequestProperty("Accept-Encoding", "gzip, deflate");
connection.setRequestProperty(
"Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
String contentType = connection.getContentType();
if (contentType == null) contentType = "unknown";
if (!contentType.toLowerCase().contains("text/html")
&& !contentType.toLowerCase().contains("text/plain")) {
if (contentType.toLowerCase().contains("image/png")
|| contentType.toLowerCase().contains("image/jpeg")) {
getMetaData().put("preview_image_url", this.getDestinationUrl());
_parseOk = true;
}
logger.warn(
logPrefix
+ "Not downloading or scraping page because content-type was: "
+ contentType);
return;
}
int contentLength = connection.getContentLength();
if (contentLength > LintedPage.HTTP_MAX_CONTENT_LENGTH) {
logger.warn(
logPrefix
+ "Not downloading or scraping page because content-length was too large: "
+ Integer.toString(contentLength)
+ " (max: "
+ Integer.toString(LintedPage.HTTP_MAX_CONTENT_LENGTH)
+ ")");
return;
}
String encoding = connection.getContentEncoding();
if (encoding != null && encoding.equalsIgnoreCase("gzip")) {
inStr = new GZIPInputStream(connection.getInputStream());
} else if (encoding != null && encoding.equalsIgnoreCase("deflate")) {
inStr = new InflaterInputStream(connection.getInputStream(), new Inflater(true));
} else {
inStr = connection.getInputStream();
}
} catch (FileNotFoundException fnf) {
_parseError = "HTTP ERROR 404";
logger.error(logPrefix + " " + _parseError);
return;
} catch (IOException ioe) {
try {
_parseError =
"Unable to download page [HTTP ERROR "
+ Integer.toString(connection.getResponseCode())
+ "]: "
+ ioe;
} catch (IOException e) {
// We'd get an ioexception on the above try{} clause if don't have a response code
_parseError = " Unable to download page: " + ioe;
}
logger.error(logPrefix + " " + _parseError);
return;
} catch (Exception ex) {
logger.error(logPrefix + "Unable to download page: " + ex);
_parseError = ex.toString();
return;
}
ServiceParser parser =
ServiceParserChainManager.getInstance().getServiceParser(this.getDestinationUrl());
parser.setRawContent(inStr);
parser.setRedirectUrlList(_redirectUrlList);
_parseOk = parser.parse();
_metaData = parser.getMetaData();
// Update the URL, if modified by the ServiceParser
String url = _metaData.getString("url");
if (url != null && !url.isEmpty()) {
_destinationUrl = url;
}
// Update alias URLs, if modified by the ServiceParser
if (_metaData.get("alias_urls") != null) {
Object[] arr = (Object[]) _metaData.get("alias_urls");
_aliases =
new ArrayList<String>(Arrays.asList(Arrays.copyOf(arr, arr.length, String[].class)));
}
// Get any parse error from the ServiceParser
String parseError = parser.getParseError();
if (parseError != null && !parseError.isEmpty()) {
_parseError = parseError;
}
}