public BoardList() {
    URL url = null;
    HttpURLConnection http_url_connection = null;
    int response_code;
    String response_message;
    InputStreamReader in = null;
    BufferedReader reader = null;
    ParserDelegator pd = null;
    try {
      // httpでhtmlファイルを取得する一連の処理
      url = new URL("http://menu.2ch.net/bbsmenu.html");
      http_url_connection = (HttpURLConnection) url.openConnection();
      http_url_connection.setRequestMethod("GET");
      http_url_connection.setInstanceFollowRedirects(false);
      http_url_connection.setRequestProperty("User-Agent", "Monazilla/1.00");
      response_code = http_url_connection.getResponseCode();
      response_message = http_url_connection.getResponseMessage();
      in = new InputStreamReader(http_url_connection.getInputStream(), "SJIS");
      reader = new BufferedReader(in);

      pd = new ParserDelegator();
      pd.parse(reader, cb, true);
      in.close();
      reader.close();
      http_url_connection.disconnect();
    } catch (IOException e1) {
      e1.printStackTrace();
    }
  }
示例#2
0
 private void createConnection() throws IOException
 {
   String completeUrl = getCompleteUrl();
   if (connection == null)
   {
     System.setProperty("http.keepAlive", connectionKeepAlive ? "true" : "false");
     connection = (HttpURLConnection) new URL(completeUrl).openConnection();
     connection.setInstanceFollowRedirects(followRedirects);
   }
 }
示例#3
0
  public void execute() {

    CoreEvent event = null;
    URLConnection connection = null;

    InputStream inputStream = null;

    int httpStatus = 0;
    HTTPHeader[] headers = null;

    try {

      connection = url.openConnection();

      if (connection instanceof HttpURLConnection) {
        ((HttpURLConnection) connection).setInstanceFollowRedirects(false);
      }

      connection.setRequestProperty("User-agent", site.getUserAgent());
      context.preHandle(connection, site);

      long start = System.currentTimeMillis();
      connection.connect();

      if (connection instanceof HttpURLConnection) {
        httpStatus = ((HttpURLConnection) connection).getResponseCode();
        switch (httpStatus) {
          case HttpURLConnection.HTTP_MOVED_PERM:
          case HttpURLConnection.HTTP_MOVED_TEMP:
            String redirectURL = connection.getHeaderField("location");
            notifyEvent(
                url, new URLFoundEvent(context, url, URLUtil.normalize(new URL(redirectURL))));
            break;
          default:
            break;
        }
      }
      inputStream = new BufferedInputStream(connection.getInputStream());

      ByteArrayOutputStream os = new ByteArrayOutputStream();
      InputStream is = new BufferedInputStream(inputStream);
      // int size = connection.getContentLength();
      int size = 0;
      try {
        int i = is.read();
        while (i != -1) {
          size++;
          os.write(i);
          i = is.read();
        }
      } catch (IOException e) {
        LogFactory.getLog(SpiderHttpURLTask.class).error("i/o exception during fetch", e);
      }

      String contentType = connection.getContentType();
      int timeMs = (int) (System.currentTimeMillis() - start);

      headers = HTTPHeaderUtil.getHeaders(connection);

      if (httpStatus >= 200 && httpStatus < 303) {
        event =
            new URLSpideredOkEvent(
                context,
                url,
                httpStatus,
                connection,
                contentType,
                timeMs,
                size,
                os.toByteArray(),
                headers);
      } else {
        event = new URLSpideredErrorEvent(context, url, httpStatus, connection, headers, null);
      }

      context.postHandle(connection, site);

    } catch (FileNotFoundException e) {
      headers = HTTPHeaderUtil.getHeaders(connection);
      event = new URLSpideredErrorEvent(context, url, 404, connection, headers, e);
    } catch (Exception e) {
      LogFactory.getLog(this.getClass()).error("exception during spidering", e);
      event = new URLSpideredErrorEvent(context, url, httpStatus, connection, headers, e);
    } finally {
      notifyEvent(url, event);
      if (inputStream != null) {
        try {
          inputStream.close();
        } catch (IOException e) {
          LogFactory.getLog(SpiderHttpURLTask.class).error("i/o exception closing inputstream", e);
        }
      }
    }
  }
  SOAPMessage post(SOAPMessage message, URL endPoint) throws SOAPException {
    boolean isFailure = false;

    URL url = null;
    HttpURLConnection httpConnection = null;

    int responseCode = 0;
    try {
      if (endPoint.getProtocol().equals("https"))
        // if(!setHttps)
        initHttps();
      // Process the URL
      JaxmURI uri = new JaxmURI(endPoint.toString());
      String userInfo = uri.getUserinfo();

      url = endPoint;

      if (dL > 0) d("uri: " + userInfo + " " + url + " " + uri);

      // TBD
      //    Will deal with https later.
      if (!url.getProtocol().equalsIgnoreCase("http")
          && !url.getProtocol().equalsIgnoreCase("https")) {
        log.severe("SAAJ0052.p2p.protocol.mustbe.http.or.https");
        throw new IllegalArgumentException(
            "Protocol " + url.getProtocol() + " not supported in URL " + url);
      }
      httpConnection = (HttpURLConnection) createConnection(url);

      httpConnection.setRequestMethod("POST");

      httpConnection.setDoOutput(true);
      httpConnection.setDoInput(true);
      httpConnection.setUseCaches(false);
      httpConnection.setInstanceFollowRedirects(true);

      if (message.saveRequired()) message.saveChanges();

      MimeHeaders headers = message.getMimeHeaders();

      Iterator it = headers.getAllHeaders();
      boolean hasAuth = false; // true if we find explicit Auth header
      while (it.hasNext()) {
        MimeHeader header = (MimeHeader) it.next();

        String[] values = headers.getHeader(header.getName());
        if (values.length == 1)
          httpConnection.setRequestProperty(header.getName(), header.getValue());
        else {
          StringBuffer concat = new StringBuffer();
          int i = 0;
          while (i < values.length) {
            if (i != 0) concat.append(',');
            concat.append(values[i]);
            i++;
          }

          httpConnection.setRequestProperty(header.getName(), concat.toString());
        }

        if ("Authorization".equals(header.getName())) {
          hasAuth = true;
          log.fine("SAAJ0091.p2p.https.auth.in.POST.true");
        }
      }

      if (!hasAuth && userInfo != null) {
        initAuthUserInfo(httpConnection, userInfo);
      }

      OutputStream out = httpConnection.getOutputStream();
      message.writeTo(out);

      out.flush();
      out.close();

      httpConnection.connect();

      try {

        responseCode = httpConnection.getResponseCode();

        // let HTTP_INTERNAL_ERROR (500) through because it is used for SOAP faults
        if (responseCode == HttpURLConnection.HTTP_INTERNAL_ERROR) {
          isFailure = true;
        }
        // else if (responseCode != HttpURLConnection.HTTP_OK)
        // else if (!(responseCode >= HttpURLConnection.HTTP_OK && responseCode < 207))
        else if ((responseCode / 100) != 2) {
          log.log(
              Level.SEVERE,
              "SAAJ0008.p2p.bad.response",
              new String[] {httpConnection.getResponseMessage()});
          throw new SOAPExceptionImpl(
              "Bad response: (" + responseCode + httpConnection.getResponseMessage());
        }
      } catch (IOException e) {
        // on JDK1.3.1_01, we end up here, but then getResponseCode() succeeds!
        responseCode = httpConnection.getResponseCode();
        if (responseCode == HttpURLConnection.HTTP_INTERNAL_ERROR) {
          isFailure = true;
        } else {
          throw e;
        }
      }

    } catch (SOAPException ex) {
      throw ex;
    } catch (Exception ex) {
      log.severe("SAAJ0009.p2p.msg.send.failed");
      throw new SOAPExceptionImpl("Message send failed", ex);
    }

    SOAPMessage response = null;
    if (responseCode == HttpURLConnection.HTTP_OK || isFailure) {
      try {
        MimeHeaders headers = new MimeHeaders();

        String key, value;

        // Header field 0 is the status line so we skip it.

        int i = 1;

        while (true) {
          key = httpConnection.getHeaderFieldKey(i);
          value = httpConnection.getHeaderField(i);

          if (key == null && value == null) break;

          if (key != null) {
            StringTokenizer values = new StringTokenizer(value, ",");
            while (values.hasMoreTokens()) headers.addHeader(key, values.nextToken().trim());
          }
          i++;
        }

        InputStream httpIn =
            (isFailure ? httpConnection.getErrorStream() : httpConnection.getInputStream());

        byte[] bytes = readFully(httpIn);

        int length =
            httpConnection.getContentLength() == -1
                ? bytes.length
                : httpConnection.getContentLength();

        // If no reply message is returned,
        // content-Length header field value is expected to be zero.
        if (length == 0) {
          response = null;
          log.warning("SAAJ0014.p2p.content.zero");
        } else {
          ByteInputStream in = new ByteInputStream(bytes, length);
          response = messageFactory.createMessage(headers, in);
        }

        httpIn.close();
        httpConnection.disconnect();

      } catch (SOAPException ex) {
        throw ex;
      } catch (Exception ex) {
        log.log(Level.SEVERE, "SAAJ0010.p2p.cannot.read.resp", ex);
        throw new SOAPExceptionImpl("Unable to read response: " + ex.getMessage());
      }
    }
    return response;
  }