public BoardList() {
URL url = null;
HttpURLConnection http_url_connection = null;
int response_code;
String response_message;
InputStreamReader in = null;
BufferedReader reader = null;
ParserDelegator pd = null;
try {
// httpでhtmlファイルを取得する一連の処理
url = new URL("http://menu.2ch.net/bbsmenu.html");
http_url_connection = (HttpURLConnection) url.openConnection();
http_url_connection.setRequestMethod("GET");
http_url_connection.setInstanceFollowRedirects(false);
http_url_connection.setRequestProperty("User-Agent", "Monazilla/1.00");
response_code = http_url_connection.getResponseCode();
response_message = http_url_connection.getResponseMessage();
in = new InputStreamReader(http_url_connection.getInputStream(), "SJIS");
reader = new BufferedReader(in);
pd = new ParserDelegator();
pd.parse(reader, cb, true);
in.close();
reader.close();
http_url_connection.disconnect();
} catch (IOException e1) {
e1.printStackTrace();
}
}
private void createConnection() throws IOException
{
String completeUrl = getCompleteUrl();
if (connection == null)
{
System.setProperty("http.keepAlive", connectionKeepAlive ? "true" : "false");
connection = (HttpURLConnection) new URL(completeUrl).openConnection();
connection.setInstanceFollowRedirects(followRedirects);
}
}
public void execute() {
CoreEvent event = null;
URLConnection connection = null;
InputStream inputStream = null;
int httpStatus = 0;
HTTPHeader[] headers = null;
try {
connection = url.openConnection();
if (connection instanceof HttpURLConnection) {
((HttpURLConnection) connection).setInstanceFollowRedirects(false);
}
connection.setRequestProperty("User-agent", site.getUserAgent());
context.preHandle(connection, site);
long start = System.currentTimeMillis();
connection.connect();
if (connection instanceof HttpURLConnection) {
httpStatus = ((HttpURLConnection) connection).getResponseCode();
switch (httpStatus) {
case HttpURLConnection.HTTP_MOVED_PERM:
case HttpURLConnection.HTTP_MOVED_TEMP:
String redirectURL = connection.getHeaderField("location");
notifyEvent(
url, new URLFoundEvent(context, url, URLUtil.normalize(new URL(redirectURL))));
break;
default:
break;
}
}
inputStream = new BufferedInputStream(connection.getInputStream());
ByteArrayOutputStream os = new ByteArrayOutputStream();
InputStream is = new BufferedInputStream(inputStream);
// int size = connection.getContentLength();
int size = 0;
try {
int i = is.read();
while (i != -1) {
size++;
os.write(i);
i = is.read();
}
} catch (IOException e) {
LogFactory.getLog(SpiderHttpURLTask.class).error("i/o exception during fetch", e);
}
String contentType = connection.getContentType();
int timeMs = (int) (System.currentTimeMillis() - start);
headers = HTTPHeaderUtil.getHeaders(connection);
if (httpStatus >= 200 && httpStatus < 303) {
event =
new URLSpideredOkEvent(
context,
url,
httpStatus,
connection,
contentType,
timeMs,
size,
os.toByteArray(),
headers);
} else {
event = new URLSpideredErrorEvent(context, url, httpStatus, connection, headers, null);
}
context.postHandle(connection, site);
} catch (FileNotFoundException e) {
headers = HTTPHeaderUtil.getHeaders(connection);
event = new URLSpideredErrorEvent(context, url, 404, connection, headers, e);
} catch (Exception e) {
LogFactory.getLog(this.getClass()).error("exception during spidering", e);
event = new URLSpideredErrorEvent(context, url, httpStatus, connection, headers, e);
} finally {
notifyEvent(url, event);
if (inputStream != null) {
try {
inputStream.close();
} catch (IOException e) {
LogFactory.getLog(SpiderHttpURLTask.class).error("i/o exception closing inputstream", e);
}
}
}
}
SOAPMessage post(SOAPMessage message, URL endPoint) throws SOAPException {
boolean isFailure = false;
URL url = null;
HttpURLConnection httpConnection = null;
int responseCode = 0;
try {
if (endPoint.getProtocol().equals("https"))
// if(!setHttps)
initHttps();
// Process the URL
JaxmURI uri = new JaxmURI(endPoint.toString());
String userInfo = uri.getUserinfo();
url = endPoint;
if (dL > 0) d("uri: " + userInfo + " " + url + " " + uri);
// TBD
// Will deal with https later.
if (!url.getProtocol().equalsIgnoreCase("http")
&& !url.getProtocol().equalsIgnoreCase("https")) {
log.severe("SAAJ0052.p2p.protocol.mustbe.http.or.https");
throw new IllegalArgumentException(
"Protocol " + url.getProtocol() + " not supported in URL " + url);
}
httpConnection = (HttpURLConnection) createConnection(url);
httpConnection.setRequestMethod("POST");
httpConnection.setDoOutput(true);
httpConnection.setDoInput(true);
httpConnection.setUseCaches(false);
httpConnection.setInstanceFollowRedirects(true);
if (message.saveRequired()) message.saveChanges();
MimeHeaders headers = message.getMimeHeaders();
Iterator it = headers.getAllHeaders();
boolean hasAuth = false; // true if we find explicit Auth header
while (it.hasNext()) {
MimeHeader header = (MimeHeader) it.next();
String[] values = headers.getHeader(header.getName());
if (values.length == 1)
httpConnection.setRequestProperty(header.getName(), header.getValue());
else {
StringBuffer concat = new StringBuffer();
int i = 0;
while (i < values.length) {
if (i != 0) concat.append(',');
concat.append(values[i]);
i++;
}
httpConnection.setRequestProperty(header.getName(), concat.toString());
}
if ("Authorization".equals(header.getName())) {
hasAuth = true;
log.fine("SAAJ0091.p2p.https.auth.in.POST.true");
}
}
if (!hasAuth && userInfo != null) {
initAuthUserInfo(httpConnection, userInfo);
}
OutputStream out = httpConnection.getOutputStream();
message.writeTo(out);
out.flush();
out.close();
httpConnection.connect();
try {
responseCode = httpConnection.getResponseCode();
// let HTTP_INTERNAL_ERROR (500) through because it is used for SOAP faults
if (responseCode == HttpURLConnection.HTTP_INTERNAL_ERROR) {
isFailure = true;
}
// else if (responseCode != HttpURLConnection.HTTP_OK)
// else if (!(responseCode >= HttpURLConnection.HTTP_OK && responseCode < 207))
else if ((responseCode / 100) != 2) {
log.log(
Level.SEVERE,
"SAAJ0008.p2p.bad.response",
new String[] {httpConnection.getResponseMessage()});
throw new SOAPExceptionImpl(
"Bad response: (" + responseCode + httpConnection.getResponseMessage());
}
} catch (IOException e) {
// on JDK1.3.1_01, we end up here, but then getResponseCode() succeeds!
responseCode = httpConnection.getResponseCode();
if (responseCode == HttpURLConnection.HTTP_INTERNAL_ERROR) {
isFailure = true;
} else {
throw e;
}
}
} catch (SOAPException ex) {
throw ex;
} catch (Exception ex) {
log.severe("SAAJ0009.p2p.msg.send.failed");
throw new SOAPExceptionImpl("Message send failed", ex);
}
SOAPMessage response = null;
if (responseCode == HttpURLConnection.HTTP_OK || isFailure) {
try {
MimeHeaders headers = new MimeHeaders();
String key, value;
// Header field 0 is the status line so we skip it.
int i = 1;
while (true) {
key = httpConnection.getHeaderFieldKey(i);
value = httpConnection.getHeaderField(i);
if (key == null && value == null) break;
if (key != null) {
StringTokenizer values = new StringTokenizer(value, ",");
while (values.hasMoreTokens()) headers.addHeader(key, values.nextToken().trim());
}
i++;
}
InputStream httpIn =
(isFailure ? httpConnection.getErrorStream() : httpConnection.getInputStream());
byte[] bytes = readFully(httpIn);
int length =
httpConnection.getContentLength() == -1
? bytes.length
: httpConnection.getContentLength();
// If no reply message is returned,
// content-Length header field value is expected to be zero.
if (length == 0) {
response = null;
log.warning("SAAJ0014.p2p.content.zero");
} else {
ByteInputStream in = new ByteInputStream(bytes, length);
response = messageFactory.createMessage(headers, in);
}
httpIn.close();
httpConnection.disconnect();
} catch (SOAPException ex) {
throw ex;
} catch (Exception ex) {
log.log(Level.SEVERE, "SAAJ0010.p2p.cannot.read.resp", ex);
throw new SOAPExceptionImpl("Unable to read response: " + ex.getMessage());
}
}
return response;
}