/**
   * Call api method.
   *
   * @param apiUrl the api url
   * @param xmlContent the xml content
   * @param contentType the content type
   * @param method the method
   * @param expected the expected
   * @return the input stream
   */
  protected InputStream callApiMethod(
      String apiUrl, String xmlContent, String contentType, String method, int expected) {
    try {
      URL url = new URL(apiUrl);
      HttpURLConnection request = (HttpURLConnection) url.openConnection();

      if (ApplicationConstants.CONNECT_TIMEOUT > -1) {
        request.setConnectTimeout(ApplicationConstants.CONNECT_TIMEOUT);
      }

      if (ApplicationConstants.READ_TIMEOUT > -1) {
        request.setReadTimeout(ApplicationConstants.READ_TIMEOUT);
      }

      for (String headerName : requestHeaders.keySet()) {
        request.setRequestProperty(headerName, requestHeaders.get(headerName));
      }

      request.setRequestMethod(method);
      request.setDoOutput(true);

      if (contentType != null) {
        request.setRequestProperty("Content-Type", contentType);
      }

      if (xmlContent != null) {
        PrintStream out = new PrintStream(new BufferedOutputStream(request.getOutputStream()));

        out.print(xmlContent);
        out.flush();
        out.close();
      }

      request.connect();

      if (request.getResponseCode() != expected) {
        throw new GitHubException(
            convertStreamToString(
                getWrappedInputStream(
                    request.getErrorStream(),
                    GZIP_ENCODING.equalsIgnoreCase(request.getContentEncoding()))));
      } else {
        return getWrappedInputStream(
            request.getInputStream(), GZIP_ENCODING.equalsIgnoreCase(request.getContentEncoding()));
      }
    } catch (IOException e) {
      throw new GitHubException(e);
    }
  }
  protected String sendPost() {
    HttpURLConnection uRLConnection = null;
    InputStream is = null;
    BufferedReader buffer = null;
    String result = null;
    try {
      URL url = new URL(Constants.APP_UPDATE_SERVER_URL);
      uRLConnection = (HttpURLConnection) url.openConnection();
      uRLConnection.setDoInput(true);
      uRLConnection.setDoOutput(true);
      uRLConnection.setRequestMethod("POST");
      uRLConnection.setUseCaches(false);
      uRLConnection.setConnectTimeout(10 * 1000);
      uRLConnection.setReadTimeout(10 * 1000);
      uRLConnection.setInstanceFollowRedirects(false);
      uRLConnection.setRequestProperty("Connection", "Keep-Alive");
      uRLConnection.setRequestProperty("Charset", "UTF-8");
      uRLConnection.setRequestProperty("Accept-Encoding", "gzip, deflate");
      uRLConnection.setRequestProperty("Content-Type", "application/json");

      uRLConnection.connect();

      is = uRLConnection.getInputStream();

      String content_encode = uRLConnection.getContentEncoding();

      if (null != content_encode && !"".equals(content_encode) && content_encode.equals("gzip")) {
        is = new GZIPInputStream(is);
      }

      buffer = new BufferedReader(new InputStreamReader(is));
      StringBuilder strBuilder = new StringBuilder();
      String line;
      while ((line = buffer.readLine()) != null) {
        strBuilder.append(line);
      }
      result = strBuilder.toString();
    } catch (Exception e) {
      Log.e(TAG, "http post error", e);
    } finally {
      if (buffer != null) {
        try {
          buffer.close();
        } catch (IOException e) {
          e.printStackTrace();
        }
      }
      if (is != null) {
        try {
          is.close();
        } catch (IOException e) {
          e.printStackTrace();
        }
      }
      if (uRLConnection != null) {
        uRLConnection.disconnect();
      }
    }
    return result;
  }
Example #3
0
  /**
   * 获取网页字符串
   *
   * @param path
   * @return
   * @throws URISyntaxException
   * @throws ClientProtocolException
   * @throws IOException
   */
  public static String getHtml(Context context, String path) throws Exception {
    String html = "";
    HttpURLConnection conn = getConnection(context, path);
    conn.setConnectTimeout(8000);
    conn.setRequestMethod("GET");
    conn.setRequestProperty("Cache-Control", "no-cache");
    conn.setRequestProperty("Host", "szlib.gov.cn");
    conn.setRequestProperty("Accept-Encoding", "gzip, deflate");
    conn.setUseCaches(false);
    if (conn.getResponseCode() == 200) {
      InputStream inputStream = null;
      String encoding = conn.getContentEncoding();
      if (encoding != null) {
        if (-1 != encoding.indexOf("gzip")) {
          inputStream = new GZIPInputStream(conn.getInputStream());
        } else if (-1 != encoding.indexOf("deflate")) {
          inputStream = new InflaterInputStream(conn.getInputStream());
        }
      } else {
        inputStream = conn.getInputStream();
      }

      html = new String(read(inputStream), "UTF-8");
    }
    conn.disconnect();
    return html;
  }
  /**
   * Login with user name and password sent as String parameter to url using POST Interrogation
   *
   * @param username
   * @param userpass
   * @throws IOException
   */
  @And("^I make post message with user: \"([^\"]*)\" and password: \"([^\"]*)\"$")
  public void HttpPostForm(String username, String userpass) throws IOException {

    URL url = new URL("http://dippy.trei.ro");

    HttpURLConnection hConnection = (HttpURLConnection) url.openConnection();
    HttpURLConnection.setFollowRedirects(true);

    hConnection.setDoOutput(true);
    hConnection.setRequestMethod("POST");

    PrintStream ps = new PrintStream(hConnection.getOutputStream());
    ps.print("user="******"&pass="******";do_login=Login");
    ps.close();

    hConnection.connect();

    if (HttpURLConnection.HTTP_OK == hConnection.getResponseCode()) {
      InputStream is = hConnection.getInputStream();

      System.out.println("!!! encoding: " + hConnection.getContentEncoding());
      System.out.println("!!! message: " + hConnection.getResponseMessage());

      is.close();
      hConnection.disconnect();
    }
  }
Example #5
0
  private String readResult(HttpURLConnection urlConnection) throws WeiboException {
    InputStream is = null;
    BufferedReader buffer = null;
    GlobalContext globalContext = GlobalContext.getInstance();
    String errorStr = globalContext.getString(R.string.timeout);
    globalContext = null;
    try {
      is = urlConnection.getInputStream();

      String content_encode = urlConnection.getContentEncoding();

      if (null != content_encode && !"".equals(content_encode) && content_encode.equals("gzip")) {
        is = new GZIPInputStream(is);
      }

      buffer = new BufferedReader(new InputStreamReader(is));
      StringBuilder strBuilder = new StringBuilder();
      String line;
      while ((line = buffer.readLine()) != null) {
        strBuilder.append(line);
      }
      AppLogger.d("result=" + strBuilder.toString());
      return strBuilder.toString();
    } catch (IOException e) {
      e.printStackTrace();
      throw new WeiboException(errorStr, e);
    } finally {
      Utility.closeSilently(is);
      Utility.closeSilently(buffer);
      urlConnection.disconnect();
    }
  }
Example #6
0
    /**
     * Handles the "Content-Encoding" header.
     */
    private InputStream wrapStream(HttpURLConnection uc, InputStream in) throws IOException {
        String encoding = uc.getContentEncoding();
        if (encoding==null || in==null) return in;
        if (encoding.equals("gzip"))    return new GZIPInputStream(in);

        throw new UnsupportedOperationException("Unexpected Content-Encoding: "+encoding);
    }
Example #7
0
 /**
  * 从Http连接的头信息中获得字符集
  *
  * @param conn HTTP连接对象
  * @return 字符集
  */
 private static String getCharsetFromConn(HttpURLConnection conn) {
   String charset = conn.getContentEncoding();
   if (charset == null || "".equals(charset.trim())) {
     String contentType = conn.getContentType();
     charset = RegexUtil.get("charset=(.*)", contentType, 1);
   }
   return charset;
 }
  /**
   * Call api post.
   *
   * @param apiUrl the api url
   * @param parameters the parameters
   * @param expected the expected
   * @return the input stream
   */
  protected InputStream callApiPost(String apiUrl, Map<String, String> parameters, int expected) {
    try {
      URL url = new URL(apiUrl);
      HttpURLConnection request = (HttpURLConnection) url.openConnection();

      if (ApplicationConstants.CONNECT_TIMEOUT > -1) {
        request.setConnectTimeout(ApplicationConstants.CONNECT_TIMEOUT);
      }

      if (ApplicationConstants.READ_TIMEOUT > -1) {
        request.setReadTimeout(ApplicationConstants.READ_TIMEOUT);
      }

      for (String headerName : requestHeaders.keySet()) {
        request.setRequestProperty(headerName, requestHeaders.get(headerName));
      }

      parameters.putAll(requestParameters);

      request.setRequestMethod("POST");
      request.setDoOutput(true);

      PrintStream out = new PrintStream(new BufferedOutputStream(request.getOutputStream()));

      out.print(getParametersString(parameters));
      out.flush();
      out.close();

      request.connect();

      if (request.getResponseCode() != expected) {
        throw new GitHubException(
            convertStreamToString(
                getWrappedInputStream(
                    request.getErrorStream(),
                    GZIP_ENCODING.equalsIgnoreCase(request.getContentEncoding()))));
      } else {
        return getWrappedInputStream(
            request.getInputStream(), GZIP_ENCODING.equalsIgnoreCase(request.getContentEncoding()));
      }
    } catch (IOException e) {
      throw new GitHubException(e);
    } finally {
    }
  }
  /**
   * Download the URL and return as a String. Gzip handling from http://goo.gl/J88WG
   *
   * @param url the URL to download
   * @return String of the URL contents
   * @throws IOException when there is an error connecting or reading the URL
   */
  public String downloadUrl(URL url) throws TVRenamerIOException {
    InputStream inputStream = null;
    StringBuilder contents = new StringBuilder();

    try {
      if (url != null) {
        logger.log(Level.INFO, "Downloading URL {0}", url.toString());

        HttpURLConnection conn = (HttpURLConnection) url.openConnection();

        HttpURLConnection.setFollowRedirects(true);
        // allow both GZip and Deflate (ZLib) encodings
        conn.setRequestProperty("Accept-Encoding", "gzip, deflate");
        conn.setConnectTimeout(CONNECT_TIMEOUT_MS);
        conn.setReadTimeout(READ_TIMEOUT_MS);

        // create the appropriate stream wrapper based on the encoding type
        String encoding = conn.getContentEncoding();
        if (encoding != null && encoding.equalsIgnoreCase("gzip")) {
          inputStream = new GZIPInputStream(conn.getInputStream());
        } else if (encoding != null && encoding.equalsIgnoreCase("deflate")) {
          inputStream = new InflaterInputStream(conn.getInputStream(), new Inflater(true));
        } else {
          inputStream = conn.getInputStream();
        }

        // always specify encoding while reading streams
        BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, "UTF-8"));

        logger.finer("Before reading url stream");

        String s;
        while ((s = reader.readLine()) != null) {
          contents.append(s);
        }

        if (logger.isLoggable(Level.FINEST)) {
          // no need to encode for logger output
          logger.log(Level.FINEST, "Url stream:\n{0}", contents.toString());
        }
      }
    } catch (Exception e) {
      String message = "Exception when attempting to download and parse URL " + url;
      logger.log(Level.SEVERE, message, e);
      throw new TVRenamerIOException(message, e);
    } finally {
      try {
        if (inputStream != null) {
          inputStream.close();
        }
      } catch (IOException e) {
        logger.log(Level.SEVERE, "Exception when attempting to close input stream", e);
      }
    }

    return StringUtils.encodeSpecialCharacters(contents.toString());
  }
Example #10
0
 public Response(HttpURLConnection con) throws IOException {
   this.con = con;
   this.statusCode = con.getResponseCode();
   if (null == (is = con.getErrorStream())) {
     is = con.getInputStream();
   }
   if (null != is && "gzip".equals(con.getContentEncoding())) {
     // the response is gzipped
     is = new GZIPInputStream(is);
   }
 }
Example #11
0
  /**
   * 从Http连接的头信息中获得字符集
   *
   * @param conn HTTP连接对象
   * @return 字符集
   */
  public static String getCharset(HttpURLConnection conn) {
    if (conn == null) {
      return null;
    }

    String charset = conn.getContentEncoding();
    if (StrUtil.isBlank(charset)) {
      charset = ReUtil.get(CHARSET_PATTERN, conn.getContentType(), 1);
    }
    return charset;
  }
  /**
   * Call api get.
   *
   * @param apiUrl the api url
   * @param expected the expected
   * @return the input stream
   */
  protected InputStream callApiGet(String apiUrl, int expected) {
    try {
      URL url = new URL(apiUrl);
      if (!requestParameters.isEmpty()) {
        if (url.getQuery() == null) {
          url = new URL(apiUrl + "?" + getParametersString(requestParameters));
        } else {
          url = new URL(apiUrl + "&" + getParametersString(requestParameters));
        }
      }

      HttpURLConnection request = (HttpURLConnection) url.openConnection();

      if (ApplicationConstants.CONNECT_TIMEOUT > -1) {
        request.setConnectTimeout(ApplicationConstants.CONNECT_TIMEOUT);
      }

      if (ApplicationConstants.READ_TIMEOUT > -1) {
        request.setReadTimeout(ApplicationConstants.READ_TIMEOUT);
      }

      for (String headerName : requestHeaders.keySet()) {
        request.setRequestProperty(headerName, requestHeaders.get(headerName));
      }

      request.connect();

      if (request.getResponseCode() != expected) {
        throw new GitHubException(
            convertStreamToString(
                getWrappedInputStream(
                    request.getErrorStream(),
                    GZIP_ENCODING.equalsIgnoreCase(request.getContentEncoding()))));
      } else {
        return getWrappedInputStream(
            request.getInputStream(), GZIP_ENCODING.equalsIgnoreCase(request.getContentEncoding()));
      }
    } catch (IOException e) {
      throw new GitHubException(e);
    }
  }
  /**
   * 指定されたURLから、フィードのURLを検索する。
   *
   * @param url フィードURL検索先
   * @return 検索結果リスト。見つからなかった場合は空のArrayListを返却する。
   * @throws FeedParseException HTML解析中にエラーが発生した場合
   */
  public ArrayList<FeedSource> findFeedUrl(String strUrl) throws FeedParseException {
    ArrayList<FeedSource> feedList = new ArrayList<FeedSource>();
    HttpURLConnection connection = null;
    InputStream is = null;
    try {
      // 指定されたURLに接続する。
      connection = connectURL(strUrl);
      is = connection.getInputStream();

      String enc = connection.getContentEncoding();
      if (enc == null) {
        // content-encodingヘッダが設定されて無い場合はとりあえずutf-8を設定。
        enc = "utf-8";
      }

      // 接続したサイトのソースを取得し、パースする。
      DOMResult result = parseDOM(is, enc);

      // linkタグを取得する。
      Document doc = (Document) result.getNode();
      String docEnc = getDocumentEncoding(doc);

      if (docEnc != null && !enc.equals(docEnc)) {
        // Documentの文字コードが事前に設定したものと違う場合は、
        // 文字コードを変更し、再接続する。
        dispose(connection, is);
        connection = connectURL(strUrl);
        is = connection.getInputStream();

        result = parseDOM(is, docEnc);
        doc = (Document) result.getNode();
      }
      NodeList childs = doc.getElementsByTagName(TAG_NAME_LINK);

      for (int i = 0; i < childs.getLength(); i++) {
        Element element = (Element) childs.item(i);

        if (isRssAutoDiscovery(element)) {
          FeedSource entity = setFeedSource(element);
          FeedParseFacade facade = new FeedParseFacade();
          entity.setVersion(facade.getRssVersion(entity.getFeedUrl()));
          feedList.add(entity);
        }
      }

    } catch (Exception e) {
      throw new FeedParseException(e);
    } finally {
      // 後始末
      dispose(connection, is);
    }
    return feedList;
  }
  @Override
  protected void runChild(FrameworkMethod method, RunNotifier notifier) {
    Description description = describeChild(method);
    if (method.getAnnotation(Ignore.class) != null) {
      notifier.fireTestIgnored(description);
      return;
    }

    String methodName = method.getName();
    String allowedMethodName = methodNames.get(description);
    if (methodName != null && !methodName.equals(allowedMethodName)) {
      //			notifier.fireTestIgnored(description);
      return;
    }

    try {
      notifier.fireTestStarted(description);
      HttpURLConnection connection = getUrl(methodName, "POST");
      handleError(connection);

      String enc = connection.getContentEncoding();
      if (enc == null) {
        enc = "ISO-8859-1";
      }
      InputStream is = connection.getInputStream();
      BufferedReader reader = new BufferedReader(new InputStreamReader(is, Charset.forName(enc)));
      String line = null;
      while ((line = reader.readLine()) != null) {
        if (line.startsWith("E")) {
          System.err.println(line.substring(1).trim());
        } else if (line.startsWith("O")) {
          System.out.println(line.substring(1).trim());
        } else if (line.startsWith("RSUCCESS")) {
          break;
        } else if (line.startsWith("RERROR")) {
          StringBuilder error = new StringBuilder(line.substring(6));
          while ((line = reader.readLine()) != null) {
            error.append(line).append("\n");
          }
          throw new AssertionFailedError(error.toString());
        } else {
          log.error("Protocol error in response: {}", line);
        }
      }
      is.close();
      connection.disconnect();
    } catch (Throwable e) {
      e.printStackTrace();
      notifier.fireTestFailure(new Failure(description, e));
    } finally {
      notifier.fireTestFinished(description);
    }
  }
Example #15
0
 /**
  * Initializes an {@link HttpEntity} from the given {@link HttpURLConnection}.
  *
  * @param connection
  * @return an HttpEntity populated with data from <code>connection</code>.
  */
 private static HttpEntity entityFromConnection(HttpURLConnection connection) {
   BasicHttpEntity entity = new BasicHttpEntity();
   InputStream inputStream;
   try {
     inputStream = connection.getInputStream();
   } catch (IOException ioe) {
     inputStream = connection.getErrorStream();
   }
   entity.setContent(inputStream);
   entity.setContentLength(connection.getContentLength());
   entity.setContentEncoding(connection.getContentEncoding());
   entity.setContentType(connection.getContentType());
   return entity;
 }
Example #16
0
 private static HttpEntity a(HttpURLConnection httpurlconnection) {
   BasicHttpEntity basichttpentity = new BasicHttpEntity();
   java.io.InputStream inputstream;
   try {
     inputstream = httpurlconnection.getInputStream();
   } catch (IOException ioexception) {
     ioexception = httpurlconnection.getErrorStream();
   }
   basichttpentity.setContent(inputstream);
   basichttpentity.setContentLength(httpurlconnection.getContentLength());
   basichttpentity.setContentEncoding(httpurlconnection.getContentEncoding());
   basichttpentity.setContentType(httpurlconnection.getContentType());
   return basichttpentity;
 }
  /**
   * Retrieve the {@link WeatherInformation} from a site containing an XML feed of weather
   * information
   *
   * @param provider {@link WeatherInformation}
   * @return {@link WeatherInformation} populated with data
   * @throws IllegalArgumentException If there is an error parsing weather information
   * @throws IOException If there is an error parsing weather information
   */
  public WeatherInformation retrieveForecast(WeatherInformation provider)
      throws IllegalArgumentException, IOException {
    URL forecastUrl = new URL(provider.getProviderUrl());
    URLConnection connection = forecastUrl.openConnection();

    if (!(connection instanceof HttpURLConnection)) {
      throw new IllegalArgumentException(forecastUrl.toExternalForm() + " is not a valid HTTP Url");
    }

    HttpURLConnection httpConnection = (HttpURLConnection) connection;
    httpConnection.setRequestProperty("Accept-Encoding", WeatherConstants.GZIP);
    httpConnection.setRequestProperty("User-Agent", WeatherConstants.BLOJSOM_WEATHER_USER_AGENT);

    httpConnection.connect();
    InputStream is = connection.getInputStream();
    BufferedInputStream bis;

    if (WeatherConstants.GZIP.equalsIgnoreCase(httpConnection.getContentEncoding())) {
      bis = new BufferedInputStream(new GZIPInputStream(is));
    } else {
      bis = new BufferedInputStream(is);
    }

    StringBuffer buffer = new StringBuffer();
    BufferedReader br = new BufferedReader(new InputStreamReader(bis));

    String input;
    while ((input = br.readLine()) != null) {
      buffer.append(input).append(BlojsomConstants.LINE_SEPARATOR);
    }

    try {
      Document document =
          _documentBuilder.parse(new InputSource(new StringReader(buffer.toString())));
      provider.parseDocument(document);
    } catch (SAXException e) {
      if (_logger.isErrorEnabled()) {
        _logger.error(e);
      }
    }

    bis.close();

    return provider;
  }
  /**
   * 执行HTTP请求之后获取到的数据流.
   *
   * @param connection
   * @return
   */
  private HttpEntity entityFromURLConnwction(HttpURLConnection connection) {
    BasicHttpEntity entity = new BasicHttpEntity();
    InputStream inputStream;
    try {
      inputStream = connection.getInputStream();
    } catch (IOException e) {
      e.printStackTrace();
      inputStream = connection.getErrorStream();
    }

    // TODO : GZIP
    entity.setContent(inputStream);
    entity.setContentLength(connection.getContentLength());
    entity.setContentEncoding(connection.getContentEncoding());
    entity.setContentType(connection.getContentType());

    return entity;
  }
  /** @param args */
  public static void main(String[] args) {

    // define the location of the resource
    URL url = null;

    try {
      url = new URL("http://localhost:8080/ir/item/escidoc:ex5");
    } catch (MalformedURLException e) {
      System.err.println("Mal formed URL '" + url + "'");
      System.exit(1);
    }

    // establish connection
    try {
      HttpURLConnection conn = (HttpURLConnection) url.openConnection();
      conn.setRequestMethod("GET");

      // check response code
      if (conn.getResponseCode() != HttpURLConnection.HTTP_OK) {
        System.err.println(conn.getResponseMessage());
      } else {
        // write out data
        String contentEncoding = conn.getContentEncoding();
        if (contentEncoding == null) {
          contentEncoding = "UTF-8";
        }
        BufferedReader in =
            new BufferedReader(new InputStreamReader(conn.getInputStream(), contentEncoding));

        String line = in.readLine();
        while (line != null) {
          System.out.println(line);
          line = in.readLine();
        }

        in.close();
      }

    } catch (IOException e) {
      e.printStackTrace();
    }
  }
  /** null rootTreeId implies "created from workDir" */
  public ThreedModel createModelFromJs(SeriesKey seriesKey, URL url) throws IOException {

    HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
    urlConnection.setRequestProperty("Accept-Encoding", "gzip, deflate");
    try {
      urlConnection.connect();
    } catch (IOException e) {
      log.log(Level.SEVERE, "RepoClient - urlConnection.connect() failed for url[" + url + "] ", e);
      throw e;
    }

    String encoding = urlConnection.getContentEncoding();

    InputStream is = null;

    JsonNode jsThreedModel;
    try {
      if (encoding != null && encoding.equalsIgnoreCase("gzip")) {
        is = new GZIPInputStream(urlConnection.getInputStream());
      } else if (encoding != null && encoding.equalsIgnoreCase("deflate")) {
        is = new InflaterInputStream(urlConnection.getInputStream(), new Inflater(true));
      } else {
        is = urlConnection.getInputStream();
      }

      ObjectMapper mapper = new ObjectMapper();
      jsThreedModel = mapper.readValue(is, JsonNode.class);
    } finally {
      Closeables.closeQuietly(is);
    }

    JsonNode jsFm = jsThreedModel.get("featureModel");
    JsonNode jsIm = jsThreedModel.get("imageModel");

    JsonToFmJvm uFm = new JsonToFmJvm();
    final FeatureModel featureModel = uFm.parseJson(seriesKey, jsFm);

    ImageModel im = JsonToImJvm.parse(featureModel, jsIm);

    ThreedModel threedModel = new ThreedModel(featureModel, im);
    return threedModel;
  }
  /** connect to DAS server and return result as an InputStream */
  private InputStream connectDASServer(URL url) throws IOException {

    System.out.println(getTimeStamp());
    System.out.println("opening connection to DAS Structure server");
    HttpURLConnection huc = null;

    // huc = (HttpURLConnection) dasUrl.openConnection();
    // huc = proxyUrl.openConnection();
    // System.out.println("opening "+url);

    huc = (HttpURLConnection) url.openConnection();
    System.out.println(huc);
    // System.out.println("deacactivated encoding temporarily");
    // should make communication much faster!
    huc.setRequestProperty("Accept-Encoding", "gzip");

    // System.out.println(huc.getResponseMessage());

    System.out.println("getContentEncoding");
    String contentEncoding = huc.getContentEncoding();
    System.out.println("getInputStream");
    InputStream inStream = huc.getInputStream();

    if (contentEncoding != null) {
      if (contentEncoding.indexOf("gzip") != -1) {
        // we have gzip encoding
        inStream = new GZIPInputStream(inStream);
        System.out.println("using gzip encoding!");
      }
    }

    System.out.println(getTimeStamp());
    System.out.println("got InputStream from  DAS Structure server");
    System.out.println("encoding: " + contentEncoding);
    System.out.println("code:" + huc.getResponseCode());
    // System.out.println("message:" + huc.getResponseMessage());
    // inStream = huc.getInputStream();

    return inStream;
  }
Example #22
0
  private String readError(HttpURLConnection urlConnection) throws WeiboException {
    InputStream is = null;
    BufferedReader buffer = null;
    BeeboApplication globalContext = BeeboApplication.getInstance();
    String errorStr = globalContext.getString(R.string.timeout);

    try {
      is = urlConnection.getErrorStream();

      if (is == null) {
        errorStr = globalContext.getString(R.string.unknown_sina_network_error);
        throw new WeiboException(errorStr);
      }

      String content_encode = urlConnection.getContentEncoding();

      if (!TextUtils.isEmpty(content_encode) && content_encode.equals("gzip")) {
        is = new GZIPInputStream(is);
      }

      buffer = new BufferedReader(new InputStreamReader(is));
      StringBuilder strBuilder = new StringBuilder();
      String line;
      while ((line = buffer.readLine()) != null) {
        strBuilder.append(line);
      }
      AppLoggerUtils.d("error result=" + strBuilder.toString());
      return strBuilder.toString();
    } catch (IOException e) {
      e.printStackTrace();
      throw new WeiboException(errorStr, e);
    } finally {
      Utility.closeSilently(is);
      Utility.closeSilently(buffer);
      urlConnection.disconnect();
      globalContext = null;
    }
  }
  static byte[] getResponseContent(HttpURLConnection http) {
    InputStream in;
    try {
      in = http.getInputStream();
      String contentEncoding = http.getContentEncoding();
      if (contentEncoding != null) {
        if (contentEncoding.equalsIgnoreCase("gzip")) {
          in =
              new GZIPInputStream(http.getInputStream()); // reads 2 bytes to determine GZIP stream!
        } else if (contentEncoding.equalsIgnoreCase("deflate")) {
          in = new InflaterInputStream(http.getInputStream());
        }
      }
    } catch (IOException e) {
      in = http.getErrorStream();
    } catch (Exception e) {
      Log.e("URLConnection exception", e.toString());
      return null;
    }

    try {
      byte[] buffer = new byte[1024];
      int size = 0;
      ByteArrayOutputStream bytestream = new ByteArrayOutputStream();
      while ((size = in.read(buffer, 0, 1024)) != -1) {
        bytestream.write(buffer, 0, size);
      }
      byte retbuffer[] = bytestream.toByteArray();
      bytestream.close();
      return retbuffer;
    } catch (Exception e) {
      Log.e("URLConnection exception", e.toString());
    }

    return null;
  }
 public static void main(String[] args) {
   try {
     // Se obtiene el objeto URL.
     URL url = new URL("http://getbootstrap.com");
     // Se abre la conexión y se hace a HttpURLConnection, ya que es una
     // conexión HTTP.
     HttpURLConnection conexion = (HttpURLConnection) url.openConnection();
     // Se obtienen los datos de la cabecera de la respuesta y se
     // muestran por pantalla.
     // Campos con método get propio.
     System.out.println("\nCABECERA DE LA RESPUESTA\n");
     System.out.println("CAMPOS CON MÉTODO GET PROPIO\n");
     System.out.println("Método de petición [getRequestMethod()]: " + conexion.getRequestMethod());
     System.out.println("Fecha petición [getDate()]: " + new Date(conexion.getDate()));
     System.out.println("Código de respuesta [getResponseCode()]: " + conexion.getResponseCode());
     System.out.println(
         "Mensaje de respuesta [getResponseMessage()]: " + conexion.getResponseMessage());
     System.out.println(
         "Fecha última modificación [getLastModified()]: " + new Date(conexion.getLastModified()));
     System.out.println("Tipo de contenido [getContentType()]: " + conexion.getContentType());
     System.out.println("Codificación [getContentEncoding()]: " + conexion.getContentEncoding());
     System.out.println(
         "Tamaño del contenido [getContentLength()]: " + conexion.getContentLength());
     System.out.println(
         "Fecha expiración [getExpiration()]: " + new Date(conexion.getExpiration()));
     // Todos los campos de la cabecera.
     System.out.println("\nTODOS LOS CAMPOS DE LA CABECERA DE LA RESPUESTA [getHeaderFields()]\n");
     Map<String, List<String>> cabeceraRespuesta = conexion.getHeaderFields();
     for (Map.Entry<String, List<String>> campo : cabeceraRespuesta.entrySet()) {
       System.out.println(campo.getKey() + " : " + campo.getValue());
     }
   } catch (Exception e) {
     // TODO Auto-generated catch block
     e.printStackTrace();
   }
 }
Example #25
0
 @Override
 public String getContentEncoding() {
   return connection.getContentEncoding();
 }
Example #26
0
  public boolean doGetSaveFile(
      String urlStr, String path, FileDownloaderHttpHelper.DownloadListener downloadListener) {

    File file = FileManager.createNewFileInSDCard(path);
    if (file == null) {
      return false;
    }

    boolean result = false;

    BufferedOutputStream out = null;
    InputStream in = null;
    HttpURLConnection urlConnection = null;
    try {

      URL url = new URL(urlStr);
      AppLoggerUtils.d("download request=" + urlStr);
      Proxy proxy = getProxy();
      if (proxy != null) {
        urlConnection = (HttpURLConnection) url.openConnection(proxy);
      } else {
        urlConnection = (HttpURLConnection) url.openConnection();
      }

      urlConnection.setRequestMethod("GET");
      urlConnection.setDoOutput(false);
      urlConnection.setConnectTimeout(DOWNLOAD_CONNECT_TIMEOUT);
      urlConnection.setReadTimeout(DOWNLOAD_READ_TIMEOUT);
      urlConnection.setRequestProperty("Connection", "Keep-Alive");
      urlConnection.setRequestProperty("Charset", "UTF-8");
      urlConnection.setRequestProperty("Accept-Encoding", "gzip, deflate");

      urlConnection.connect();

      int status = urlConnection.getResponseCode();

      if (status != HttpURLConnection.HTTP_OK) {
        return false;
      }

      int bytetotal = (int) urlConnection.getContentLength();
      int bytesum = 0;
      int byteread = 0;
      out = new BufferedOutputStream(new FileOutputStream(file));

      InputStream is = urlConnection.getInputStream();
      String content_encode = urlConnection.getContentEncoding();
      if (!TextUtils.isEmpty(content_encode) && content_encode.equals("gzip")) {
        is = new GZIPInputStream(is);
      }
      in = new BufferedInputStream(is);

      final Thread thread = Thread.currentThread();
      byte[] buffer = new byte[1444];
      while ((byteread = in.read(buffer)) != -1) {
        if (thread.isInterrupted()) {
          if (((float) bytesum / (float) bytetotal) < 0.8f) {
            file.delete();
            throw new InterruptedIOException();
          }
        }

        bytesum += byteread;
        out.write(buffer, 0, byteread);
        if (downloadListener != null && bytetotal > 0) {
          downloadListener.pushProgress(bytesum, bytetotal);
        }
      }
      if (downloadListener != null) {
        downloadListener.completed();
      }
      AppLoggerUtils.v("download request= " + urlStr + " download finished");
      result = true;

    } catch (IOException e) {
      e.printStackTrace();
      AppLoggerUtils.v("download request= " + urlStr + " download failed");
    } finally {
      Utility.closeSilently(in);
      Utility.closeSilently(out);
      if (urlConnection != null) {
        urlConnection.disconnect();
      }
    }

    return result && ImageUtility.isThisBitmapCanRead(path);
  }
Example #27
0
  /**
   * Retrieves a file from a given URL and saves it to the outputPath.
   *
   * @param url the URL of the file to download
   * @param outputPath the path to the save the file to
   * @param useProxy whether to use the configured proxy when downloading files
   * @throws DownloadFailedException is thrown if there is an error downloading the file
   */
  public static void fetchFile(URL url, File outputPath, boolean useProxy)
      throws DownloadFailedException {
    if ("file".equalsIgnoreCase(url.getProtocol())) {
      File file;
      try {
        file = new File(url.toURI());
      } catch (URISyntaxException ex) {
        final String msg = format("Download failed, unable to locate '%s'", url.toString());
        throw new DownloadFailedException(msg);
      }
      if (file.exists()) {
        try {
          org.apache.commons.io.FileUtils.copyFile(file, outputPath);
        } catch (IOException ex) {
          final String msg =
              format(
                  "Download failed, unable to copy '%s' to '%s'",
                  url.toString(), outputPath.getAbsolutePath());
          throw new DownloadFailedException(msg);
        }
      } else {
        final String msg = format("Download failed, file ('%s') does not exist", url.toString());
        throw new DownloadFailedException(msg);
      }
    } else {
      HttpURLConnection conn = null;
      try {
        LOGGER.debug("Attempting download of {}", url.toString());
        conn = URLConnectionFactory.createHttpURLConnection(url, useProxy);
        conn.setRequestProperty("Accept-Encoding", "gzip, deflate");
        conn.connect();
        int status = conn.getResponseCode();
        int redirectCount = 0;
        while ((status == HttpURLConnection.HTTP_MOVED_TEMP
                || status == HttpURLConnection.HTTP_MOVED_PERM
                || status == HttpURLConnection.HTTP_SEE_OTHER)
            && MAX_REDIRECT_ATTEMPTS > redirectCount++) {
          final String location = conn.getHeaderField("Location");
          try {
            conn.disconnect();
          } finally {
            conn = null;
          }
          LOGGER.debug("Download is being redirected from {} to {}", url.toString(), location);
          conn = URLConnectionFactory.createHttpURLConnection(new URL(location), useProxy);
          conn.setRequestProperty("Accept-Encoding", "gzip, deflate");
          conn.connect();
          status = conn.getResponseCode();
        }
        if (status != 200) {
          try {
            conn.disconnect();
          } finally {
            conn = null;
          }
          final String msg =
              format(
                  "Error downloading file %s; received response code %s.", url.toString(), status);
          throw new DownloadFailedException(msg);
        }
      } catch (IOException ex) {
        try {
          if (conn != null) {
            conn.disconnect();
          }
        } finally {
          conn = null;
        }
        final String msg = format("Error downloading file %s; unable to connect.", url.toString());
        throw new DownloadFailedException(msg, ex);
      }

      final String encoding = conn.getContentEncoding();
      BufferedOutputStream writer = null;
      InputStream reader = null;
      try {
        if (encoding != null && "gzip".equalsIgnoreCase(encoding)) {
          reader = new GZIPInputStream(conn.getInputStream());
        } else if (encoding != null && "deflate".equalsIgnoreCase(encoding)) {
          reader = new InflaterInputStream(conn.getInputStream());
        } else {
          reader = conn.getInputStream();
        }

        writer = new BufferedOutputStream(new FileOutputStream(outputPath));
        final byte[] buffer = new byte[4096];
        int bytesRead;
        while ((bytesRead = reader.read(buffer)) > 0) {
          writer.write(buffer, 0, bytesRead);
        }
        LOGGER.debug("Download of {} complete", url.toString());
      } catch (IOException ex) {
        analyzeException(ex);
        final String msg =
            format(
                "Error saving '%s' to file '%s'%nConnection Timeout: %d%nEncoding: %s%n",
                url.toString(), outputPath.getAbsolutePath(), conn.getConnectTimeout(), encoding);
        throw new DownloadFailedException(msg, ex);
      } catch (Throwable ex) {
        final String msg =
            format(
                "Unexpected exception saving '%s' to file '%s'%nConnection Timeout: %d%nEncoding: %s%n",
                url.toString(), outputPath.getAbsolutePath(), conn.getConnectTimeout(), encoding);
        throw new DownloadFailedException(msg, ex);
      } finally {
        if (writer != null) {
          try {
            writer.close();
          } catch (IOException ex) {
            LOGGER.trace("Error closing the writer in Downloader.", ex);
          }
        }
        if (reader != null) {
          try {
            reader.close();
          } catch (IOException ex) {
            LOGGER.trace("Error closing the reader in Downloader.", ex);
          }
        }
        try {
          conn.disconnect();
        } finally {
          conn = null;
        }
      }
    }
  }
Example #28
0
    public void runInternal() {
      connection = null;

      try {
        setProgressMessage(url.toString(), -1);
        long startTimeStamp = System.currentTimeMillis();
        delayedProgress =
            coolReader.getEngine().showProgressDelayed(0, progressMessage, PROGRESS_DELAY_MILLIS);
        URLConnection conn = url.openConnection();
        if (conn instanceof HttpsURLConnection) {
          onError("HTTPs is not supported yet");
          return;
        }
        if (!(conn instanceof HttpURLConnection)) {
          onError("Only HTTP supported");
          return;
        }
        connection = (HttpURLConnection) conn;
        connection.setRequestProperty("User-Agent", "CoolReader/3(Android)");
        if (referer != null) connection.setRequestProperty("Referer", referer);
        connection.setInstanceFollowRedirects(true);
        connection.setAllowUserInteraction(false);
        connection.setConnectTimeout(20000);
        connection.setReadTimeout(40000);
        connection.setDoInput(true);
        String fileName = null;
        String disp = connection.getHeaderField("Content-Disposition");
        if (disp != null) {
          int p = disp.indexOf("filename=");
          if (p > 0) {
            fileName = disp.substring(p + 9);
          }
        }
        // connection.setDoOutput(true);
        // connection.set

        int response = -1;

        response = connection.getResponseCode();
        L.d("Response: " + response);
        if (response != 200) {
          onError("Error " + response);
          return;
        }
        String contentType = connection.getContentType();
        String contentEncoding = connection.getContentEncoding();
        int contentLen = connection.getContentLength();
        // connection.getC
        L.d("Entity content length: " + contentLen);
        L.d("Entity content type: " + contentType);
        L.d("Entity content encoding: " + contentEncoding);
        setProgressMessage(url.toString(), contentLen);
        InputStream is = connection.getInputStream();
        delayedProgress.cancel();
        is = new ProgressInputStream(is, startTimeStamp, progressMessage, contentLen, 80);
        final int MAX_CONTENT_LEN_TO_BUFFER = 256 * 1024;
        boolean isZip = contentType != null && contentType.equals("application/zip");
        if (expectedType != null) contentType = expectedType;
        else if (contentLen > 0 && contentLen < MAX_CONTENT_LEN_TO_BUFFER) { // autodetect type
          byte[] buf = new byte[contentLen];
          if (is.read(buf) != contentLen) {
            onError("Wrong content length");
            return;
          }
          is.close();
          is = null;
          is = new ByteArrayInputStream(buf);
          if (findSubstring(buf, "<?xml version=") >= 0 && findSubstring(buf, "<feed") >= 0)
            contentType = "application/atom+xml"; // override type
        }
        if (contentType.startsWith("application/atom+xml")) {
          L.d("Parsing feed");
          parseFeed(is);
        } else {
          if (fileName == null) fileName = defaultFileName;
          L.d("Downloading book: " + contentEncoding);
          downloadBook(contentType, url.toString(), is, contentLen, fileName, isZip);
        }
      } catch (Exception e) {
        L.e("Exception while trying to open URI " + url.toString(), e);
        onError("Error occured while reading OPDS catalog");
      } finally {
        if (connection != null)
          try {
            connection.disconnect();
          } catch (Exception e) {
            // ignore
          }
      }
    }
  private static Map<String, ExchangeRate> requestExchangeRates(
      final URL url,
      float dogeBtcConversion,
      final String userAgent,
      final String source,
      final String... fields) {
    final long start = System.currentTimeMillis();

    HttpURLConnection connection = null;
    Reader reader = null;

    try {
      connection = (HttpURLConnection) url.openConnection();

      connection.setInstanceFollowRedirects(false);
      connection.setConnectTimeout(Constants.HTTP_TIMEOUT_MS);
      connection.setReadTimeout(Constants.HTTP_TIMEOUT_MS);
      connection.addRequestProperty("User-Agent", userAgent);
      connection.addRequestProperty("Accept-Encoding", "gzip");
      connection.connect();

      final int responseCode = connection.getResponseCode();
      if (responseCode == HttpURLConnection.HTTP_OK) {
        final String contentEncoding = connection.getContentEncoding();

        InputStream is = new BufferedInputStream(connection.getInputStream(), 1024);
        if ("gzip".equalsIgnoreCase(contentEncoding)) is = new GZIPInputStream(is);

        reader = new InputStreamReader(is, Constants.UTF_8);
        final StringBuilder content = new StringBuilder();
        final long length = Io.copy(reader, content);

        final Map<String, ExchangeRate> rates = new TreeMap<String, ExchangeRate>();

        final JSONObject head = new JSONObject(content.toString());
        for (final Iterator<String> i = head.keys(); i.hasNext(); ) {
          final String currencyCode = i.next();
          if (!"timestamp".equals(currencyCode)) {
            final JSONObject o = head.getJSONObject(currencyCode);

            for (final String field : fields) {
              final String rate = o.optString(field, null);

              if (rate != null) {
                try {
                  BigDecimal btcRate = new BigDecimal(GenericUtils.toNanoCoins(rate, 0));
                  BigInteger dogeRate =
                      btcRate.multiply(BigDecimal.valueOf(dogeBtcConversion)).toBigInteger();

                  if (dogeRate.signum() > 0) {
                    rates.put(currencyCode, new ExchangeRate(currencyCode, dogeRate, source));
                    break;
                  }
                } catch (final ArithmeticException x) {
                  log.warn(
                      "problem fetching {} exchange rate from {} ({}): {}",
                      currencyCode,
                      url,
                      contentEncoding,
                      x.getMessage());
                }
              }
            }
          }
        }

        log.info(
            "fetched exchange rates from {} ({}), {} chars, took {} ms",
            url,
            contentEncoding,
            length,
            System.currentTimeMillis() - start);

        return rates;
      } else {
        log.warn("http status {} when fetching {}", responseCode, url);
      }
    } catch (final Exception x) {
      log.warn("problem fetching exchange rates from " + url, x);
    } finally {
      if (reader != null) {
        try {
          reader.close();
        } catch (final IOException x) {
          // swallow
        }
      }

      if (connection != null) connection.disconnect();
    }

    return null;
  }
Example #30
0
  /** Scrapes the metadata on this page (can be called separately from {@link process} */
  public void scrapeMetadata() {
    final String logPrefix = "[" + this.getDestinationUrl() + "] ";

    logger.trace(logPrefix + "Downloading and scraping page contents...");

    InputStream inStr = null;
    HttpURLConnection connection = null;
    try {
      URL url = new URL(this.getDestinationUrl());
      connection = (HttpURLConnection) url.openConnection(Proxy.NO_PROXY);
      connection.setConnectTimeout(LintedPage.HTTP_CONNECT_TIMEOUT);
      connection.setReadTimeout(LintedPage.HTTP_READ_TIMEOUT);
      connection.setRequestProperty("User-Agent", LintedPage.HTTP_USER_AGENT);
      connection.setRequestProperty("Accept-Encoding", "gzip, deflate");
      connection.setRequestProperty(
          "Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");

      String contentType = connection.getContentType();
      if (contentType == null) contentType = "unknown";
      if (!contentType.toLowerCase().contains("text/html")
          && !contentType.toLowerCase().contains("text/plain")) {

        if (contentType.toLowerCase().contains("image/png")
            || contentType.toLowerCase().contains("image/jpeg")) {
          getMetaData().put("preview_image_url", this.getDestinationUrl());
          _parseOk = true;
        }

        logger.warn(
            logPrefix
                + "Not downloading or scraping page because content-type was: "
                + contentType);
        return;
      }

      int contentLength = connection.getContentLength();
      if (contentLength > LintedPage.HTTP_MAX_CONTENT_LENGTH) {
        logger.warn(
            logPrefix
                + "Not downloading or scraping page because content-length was too large: "
                + Integer.toString(contentLength)
                + " (max: "
                + Integer.toString(LintedPage.HTTP_MAX_CONTENT_LENGTH)
                + ")");
        return;
      }

      String encoding = connection.getContentEncoding();
      if (encoding != null && encoding.equalsIgnoreCase("gzip")) {
        inStr = new GZIPInputStream(connection.getInputStream());
      } else if (encoding != null && encoding.equalsIgnoreCase("deflate")) {
        inStr = new InflaterInputStream(connection.getInputStream(), new Inflater(true));
      } else {
        inStr = connection.getInputStream();
      }
    } catch (FileNotFoundException fnf) {
      _parseError = "HTTP ERROR 404";
      logger.error(logPrefix + " " + _parseError);
      return;
    } catch (IOException ioe) {
      try {
        _parseError =
            "Unable to download page [HTTP ERROR "
                + Integer.toString(connection.getResponseCode())
                + "]: "
                + ioe;
      } catch (IOException e) {
        // We'd get an ioexception on the above try{} clause if don't have a response code
        _parseError = " Unable to download page: " + ioe;
      }
      logger.error(logPrefix + " " + _parseError);
      return;
    } catch (Exception ex) {
      logger.error(logPrefix + "Unable to download page: " + ex);
      _parseError = ex.toString();
      return;
    }

    ServiceParser parser =
        ServiceParserChainManager.getInstance().getServiceParser(this.getDestinationUrl());
    parser.setRawContent(inStr);
    parser.setRedirectUrlList(_redirectUrlList);
    _parseOk = parser.parse();
    _metaData = parser.getMetaData();

    // Update the URL, if modified by the ServiceParser
    String url = _metaData.getString("url");
    if (url != null && !url.isEmpty()) {
      _destinationUrl = url;
    }

    // Update alias URLs, if modified by the ServiceParser
    if (_metaData.get("alias_urls") != null) {
      Object[] arr = (Object[]) _metaData.get("alias_urls");
      _aliases =
          new ArrayList<String>(Arrays.asList(Arrays.copyOf(arr, arr.length, String[].class)));
    }

    // Get any parse error from the ServiceParser
    String parseError = parser.getParseError();
    if (parseError != null && !parseError.isEmpty()) {
      _parseError = parseError;
    }
  }