private T requestInfo(URI baseUrl, RenderingContext context)
throws IOException, URISyntaxException, ParserConfigurationException, SAXException {
URL url = loader.createURL(baseUrl, context);
GetMethod method = null;
try {
final InputStream stream;
if ((url.getProtocol().equals("http") || url.getProtocol().equals("https"))
&& context.getConfig().localHostForwardIsFrom(url.getHost())) {
String scheme = url.getProtocol();
final String host = url.getHost();
if (url.getProtocol().equals("https")
&& context.getConfig().localHostForwardIsHttps2http()) {
scheme = "http";
}
URL localUrl = new URL(scheme, "localhost", url.getPort(), url.getFile());
HttpURLConnection connexion = (HttpURLConnection) localUrl.openConnection();
connexion.setRequestProperty("Host", host);
for (Map.Entry<String, String> entry : context.getHeaders().entrySet()) {
connexion.setRequestProperty(entry.getKey(), entry.getValue());
}
stream = connexion.getInputStream();
} else {
method = new GetMethod(url.toString());
for (Map.Entry<String, String> entry : context.getHeaders().entrySet()) {
method.setRequestHeader(entry.getKey(), entry.getValue());
}
context.getConfig().getHttpClient(baseUrl).executeMethod(method);
int code = method.getStatusCode();
if (code < 200 || code >= 300) {
throw new IOException(
"Error "
+ code
+ " while reading the Capabilities from "
+ url
+ ": "
+ method.getStatusText());
}
stream = method.getResponseBodyAsStream();
}
final T result;
try {
result = loader.parseInfo(stream);
} finally {
stream.close();
}
return result;
} finally {
if (method != null) {
method.releaseConnection();
}
}
}
private boolean processURL(URL url, String baseDir, StatusWindow status) throws IOException {
if (processedLinks.contains(url)) {
return false;
} else {
processedLinks.add(url);
}
URLConnection connection = url.openConnection();
InputStream in = new BufferedInputStream(connection.getInputStream());
ArrayList list = processPage(in, baseDir, url);
if ((status != null) && (list.size() > 0)) {
status.setMaximum(list.size());
}
for (int i = 0; i < list.size(); i++) {
if (status != null) {
status.setMessage(Utils.trimFileName(list.get(i).toString(), 40), i);
}
if ((!((String) list.get(i)).startsWith("RUN"))
&& (!((String) list.get(i)).startsWith("SAVE"))
&& (!((String) list.get(i)).startsWith("LOAD"))) {
processURL(
new URL(url.getProtocol(), url.getHost(), url.getPort(), (String) list.get(i)),
baseDir,
status);
}
}
in.close();
return true;
}
@Override
public void run() {
try {
URL url = buildPingUrl();
if (logger.isDebugEnabled()) {
logger.debug("Sending UDC information to {}...", url);
}
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setConnectTimeout((int) HTTP_TIMEOUT.millis());
conn.setReadTimeout((int) HTTP_TIMEOUT.millis());
if (conn.getResponseCode() >= 300) {
throw new Exception(
String.format("%s Responded with Code %d", url.getHost(), conn.getResponseCode()));
}
if (logger.isDebugEnabled()) {
BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream()));
String line = reader.readLine();
while (line != null) {
logger.debug(line);
line = reader.readLine();
}
reader.close();
} else {
conn.getInputStream().close();
}
successCounter.incrementAndGet();
} catch (Exception e) {
if (logger.isDebugEnabled()) {
logger.debug("Error sending UDC information", e);
}
failCounter.incrementAndGet();
}
}
private JarFile getCachedJarFile(URL url) {
JarFile result = (JarFile) fileCache.get(url);
/* if the JAR file is cached, the permission will always be there */
if (result != null) {
Permission perm = getPermission(result);
if (perm != null) {
SecurityManager sm = System.getSecurityManager();
if (sm != null) {
try {
sm.checkPermission(perm);
} catch (SecurityException se) {
// fallback to checkRead/checkConnect for pre 1.2
// security managers
if ((perm instanceof java.io.FilePermission)
&& perm.getActions().indexOf("read") != -1) {
sm.checkRead(perm.getName());
} else if ((perm instanceof java.net.SocketPermission)
&& perm.getActions().indexOf("connect") != -1) {
sm.checkConnect(url.getHost(), url.getPort());
} else {
throw se;
}
}
}
}
}
return result;
}
public static void setAZTracker(URL tracker_url, boolean az_tracker) {
String key = tracker_url.getHost() + ":" + tracker_url.getPort();
synchronized (az_trackers) {
boolean changed = false;
if (az_trackers.get(key) == null) {
if (az_tracker) {
az_trackers.put(key, new Long(SystemTime.getCurrentTime()));
changed = true;
}
} else {
if (!az_tracker) {
if (az_trackers.remove(key) != null) {
changed = true;
}
}
}
if (changed) {
COConfigurationManager.setParameter("Tracker Client AZ Instances", az_trackers);
}
}
}
public synchronized void logUnavailableHost(URL url) {
if (this.offlineMode) return;
if (url == null) {
String message = Logging.getMessage("nullValue.URLIsNull");
Logging.error(message);
throw new IllegalArgumentException(message);
}
String hostName = url.getHost();
HostInfo hi = this.hostMap.get(hostName);
if (hi != null) {
if (!hi.isUnavailable()) {
hi.logCount.incrementAndGet();
if (hi.isUnavailable()) // host just became unavailable
this.firePropertyChange(NetworkStatus.HOST_UNAVAILABLE, null, url);
}
hi.lastLogTime.set(System.currentTimeMillis());
} else {
hi = new HostInfo(this.attemptLimit.get(), this.tryAgainInterval.get());
hi.logCount.set(1);
if (hi.isUnavailable()) // the attempt limit may be as low as 1, so handle that case here
this.firePropertyChange(NetworkStatus.HOST_UNAVAILABLE, null, url);
this.hostMap.put(hostName, hi);
}
this.lastUnavailableLogTime.set(System.currentTimeMillis());
}
/**
* ** Gets the 'host' specification in the URI ** @return The 'host' specification in the URL, or
* null if unable to determine host
*/
public String getHost() {
try {
URL url = new URL(this.getURI());
return url.getHost();
} catch (MalformedURLException mue) {
return null;
}
}
/**
* 以阻塞的方式立即下载一个文件，该方法会覆盖已经存在的文件
*
* @param task
* @return "ok" if download success (else return errmessage);
*/
static String download(DownloadTask task) {
if (!openedStatus && show) openStatus();
URL url;
HttpURLConnection conn;
try {
url = new URL(task.getOrigin());
conn = (HttpURLConnection) url.openConnection();
conn.setConnectTimeout(30000);
conn.setReadTimeout(30000);
conn.setRequestProperty(
"User-Agent",
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.134 Safari/"
+ Math.random());
if ("www.imgjav.com".equals(url.getHost())) { // 该网站需要带cookie请求
conn.setRequestProperty(
"User-Agent",
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.134 Safari/537.36");
// conn.setRequestProperty("Cookie", "__cfduid=d219ea333c7a9b5743b572697b631925a1446093229;
// cf_clearance=6ae62d843f5d09acf393f9e4eb130d9366840c82-1446093303-28800");
conn.setRequestProperty(
"Cookie",
"__cfduid=d6ee846b378bb7d5d173a05541f8a2b6a1446090548; cf_clearance=ea10e8db31f8b6ee51570b118dd89b7e616d7b62-1446099714-28800");
conn.setRequestProperty("Host", "www.imgjav.com");
}
Path directory = Paths.get(task.getDest()).getParent();
if (!Files.exists(directory)) Files.createDirectories(directory);
} catch (Exception e) {
e.printStackTrace();
return e.getMessage();
}
try (InputStream is = conn.getInputStream();
BufferedInputStream in = new BufferedInputStream(is);
FileOutputStream fos = new FileOutputStream(task.getDest());
OutputStream out = new BufferedOutputStream(fos); ) {
int length = conn.getContentLength();
if (length < 1) throw new IOException("length<1");
byte[] binary = new byte[length];
byte[] buff = new byte[65536];
int len;
int index = 0;
while ((len = in.read(buff)) != -1) {
System.arraycopy(buff, 0, binary, index, len);
index += len;
allLen += len; // allLen有线程安全的问题 ，可能会不正确。无需精确数据，所以不同步了
task.setReceivePercent(String.format("%.2f", ((float) index / length) * 100) + "%");
}
out.write(binary);
} catch (IOException e) {
e.printStackTrace();
return e.getMessage();
}
return "ok";
}
// Strip off everything unneeded
private String getBaseUrl(String pUrl, String pServletPath) {
String sUrl;
try {
URL url = new URL(pUrl);
String host = getIpIfPossible(url.getHost());
sUrl = new URL(url.getProtocol(), host, url.getPort(), pServletPath).toExternalForm();
} catch (MalformedURLException exp) {
sUrl = plainReplacement(pUrl, pServletPath);
}
return sUrl;
}
/** Updates the host port attributes based on the URI. */
private void updateHostPort() {
if (uri != null && !"".equals(uri.trim())) {
try {
URL url = new URL(uri);
host = url.getHost();
port = url.getPort();
} catch (MalformedURLException e) {
// this won't happen
}
}
}
public static boolean isAZTracker(URL tracker_url) {
String host = tracker_url.getHost();
if (Constants.isAzureusDomain(host)) {
return (true);
}
synchronized (az_trackers) {
return (az_trackers.containsKey(host + ":" + tracker_url.getPort()));
}
}
public static boolean isUDPProbeOK(URL tracker_url) {
String host = tracker_url.getHost();
if (Constants.isAzureusDomain(host)) {
return (false);
}
synchronized (udp_probe_results) {
return (udp_probe_results.containsKey(host));
}
}
public void setAuthenticationOutcome(String realm, URL tracker, boolean success) {
try {
this_mon.enter();
setAuthenticationOutcome(
realm, tracker.getProtocol(), tracker.getHost(), tracker.getPort(), success);
} finally {
this_mon.exit();
}
}
public PasswordAuthentication getAuthentication(String realm, URL tracker) {
try {
this_mon.enter();
return (getAuthentication(
realm, tracker.getProtocol(), tracker.getHost(), tracker.getPort()));
} finally {
this_mon.exit();
}
}
public static void checkForBlacklistedURLs(URL url) throws IOException {
for (int i = 0; i < BLACKLISTED_HOSTS.length; i++) {
if (url.getHost().equalsIgnoreCase(BLACKLISTED_HOSTS[i])
&& url.getPort() == BLACKLISTED_PORTS[i]) {
throw (new IOException(
"http://" + BLACKLISTED_HOSTS[i] + ":" + BLACKLISTED_PORTS[i] + "/ is not a tracker"));
}
}
}
/** ** Sets the 'port' */
public boolean setPort(int _port) {
String uri = this.getURI();
if ((_port > 0) && URIArg.isAbsoluteURL(uri)) {
try {
URL oldURI = new URL(uri);
String proto = oldURI.getProtocol();
String host = oldURI.getHost();
int port = _port;
String file = oldURI.getFile();
URL newURI = new URL(proto, host, port, file);
this._setURI(newURI.toString());
return true;
} catch (MalformedURLException mue) {
// error
}
}
return false;
}
public synchronized void logAvailableHost(URL url) {
if (this.offlineMode) return;
if (url == null) {
String message = Logging.getMessage("nullValue.URLIsNull");
Logging.error(message);
throw new IllegalArgumentException(message);
}
String hostName = url.getHost();
HostInfo hi = this.hostMap.get(hostName);
if (hi != null) {
this.hostMap.remove(hostName); // host is available again
firePropertyChange(NetworkStatus.HOST_AVAILABLE, null, url);
}
this.lastAvailableLogTime.set(System.currentTimeMillis());
}
public boolean isHostUnavailable(URL url) {
if (this.offlineMode) return true;
if (url == null) {
String message = Logging.getMessage("nullValue.URLIsNull");
Logging.error(message);
throw new IllegalArgumentException(message);
}
String hostName = url.getHost();
HostInfo hi = this.hostMap.get(hostName);
if (hi == null) return false;
if (hi.isTimeToTryAgain()) {
hi.logCount.set(0); // info removed from table in logAvailableHost
return false;
}
return hi.isUnavailable();
}
/**
* mapUrlToFileLocation() is the method used to resolve urls into file names. This maps a given
* url to a file location, using the au top directory as the base. It creates directories which
* mirror the html string, so 'http://www.journal.org/issue1/index.html' would be cached in the
* file: <rootLocation>/www.journal.org/http/issue1/index.html
*
* @param rootLocation the top directory for ArchivalUnit this URL is in
* @param urlStr the url to translate
* @return the url file location
* @throws java.net.MalformedURLException
*/
public static String mapUrlToFileLocation(String rootLocation, String urlStr)
throws MalformedURLException {
int totalLength = rootLocation.length() + urlStr.length();
URL url = new URL(urlStr);
StringBuilder buffer = new StringBuilder(totalLength);
buffer.append(rootLocation);
if (!rootLocation.endsWith(File.separator)) {
buffer.append(File.separator);
}
buffer.append(url.getHost().toLowerCase());
int port = url.getPort();
if (port != -1) {
buffer.append(PORT_SEPARATOR);
buffer.append(port);
}
buffer.append(File.separator);
buffer.append(url.getProtocol());
if (RepositoryManager.isEnableLongComponents()) {
String escapedPath =
escapePath(
StringUtil.replaceString(url.getPath(), UrlUtil.URL_PATH_SEPARATOR, File.separator));
String query = url.getQuery();
if (query != null) {
escapedPath = escapedPath + "?" + escapeQuery(query);
}
String encodedPath = RepositoryNodeImpl.encodeUrl(escapedPath);
// encodeUrl strips leading / from path
buffer.append(File.separator);
buffer.append(encodedPath);
} else {
buffer.append(
escapePath(
StringUtil.replaceString(url.getPath(), UrlUtil.URL_PATH_SEPARATOR, File.separator)));
String query = url.getQuery();
if (query != null) {
buffer.append("?");
buffer.append(escapeQuery(query));
}
}
return buffer.toString();
}
/**
* Get a file name from a URL.
*
* @param url The URL.
* @param outputDirectoryName The output directory name.
* @return The file name stripped of its original path with the specified output directory
* attached.
*/
public static String getFileNameFromURL(URL url, String outputDirectoryName) {
String hostName = url.getHost();
String path = url.getPath();
String baseName = "";
if (path.length() == 0) {
if (hostName.length() == 0) {
baseName = "adorned";
} else {
baseName = hostName;
}
} else {
if (hostName.length() == 0) {
baseName = new File(path).getName();
} else {
baseName = new File(hostName + "." + path).getName();
}
}
return new File(outputDirectoryName, baseName).getPath();
}
public static void setUDPProbeResult(URL tracker_url, boolean probe_ok) {
String key = tracker_url.getHost();
synchronized (udp_probe_results) {
boolean changed = false;
if (udp_probe_results.get(key) == null) {
if (probe_ok) {
// arbitrary max size here just in case something weird happens
if (udp_probe_results.size() > 512) {
udp_probe_results.clear();
}
udp_probe_results.put(key, new Long(SystemTime.getCurrentTime()));
changed = true;
}
} else {
if (!probe_ok) {
if (udp_probe_results.remove(key) != null) {
changed = true;
}
}
}
if (changed) {
COConfigurationManager.setParameter("Tracker Client UDP Probe Results", udp_probe_results);
}
}
}
/**
* POST something to the given URL. The headers are put in as HTTP headers, the content length is
* calculated and the content is sent as content. Duh.
*
* @param cs Client socket items.
* @param url the url to post to
* @param headers additional headers to send as HTTP headers
* @param contentType type of the content
* @param content the body of the post
* @exception IOException if post fails
*/
private static Response post(
ClientSocket cs, URL url, Hashtable headers, String contentType, String content)
throws IOException {
// If the port is not specified in the URL, use the default http port.
int port = url.getPort();
if (port == -1) port = 80;
cs.s = new Socket(url.getHost(), +port);
cs.out = new PrintWriter(cs.s.getOutputStream());
cs.in = new BufferedReader(new InputStreamReader(cs.s.getInputStream()));
// Create the HTTP header
cs.out.print(HTTP_POST + " " + url.getFile() + " HTTP/" + HTTP_VERSION + "\r\n");
cs.out.print(HEADER_HOST + ": " + url.getHost() + ':' + port + "\r\n");
cs.out.print(HEADER_CONTENT_TYPE + ": " + contentType + "\r\n");
cs.out.print(HEADER_CONTENT_LENGTH + ": " + content.length() + "\r\n");
for (Enumeration e = headers.keys(); e.hasMoreElements(); ) {
Object key = e.nextElement();
cs.out.print(key + ": " + headers.get(key) + "\r\n");
}
// According to HTTP1.1 need another CRLF after the header
cs.out.print("\r\n");
Debug.log(Debug.MSG_LIFECYCLE, "HTTPUtils: Writing content to the socket... " + content);
cs.out.println(content);
cs.out.flush();
/* read the status line */
int statusCode = 0;
String statusString = null;
StringTokenizer st = new StringTokenizer(cs.in.readLine());
st.nextToken(); // ignore version part
statusCode = Integer.parseInt(st.nextToken());
StringBuffer sb = new StringBuffer();
while (st.hasMoreTokens()) {
sb.append(st.nextToken());
if (st.hasMoreTokens()) sb.append(" ");
}
statusString = sb.toString();
/* get the headers */
Hashtable respHeaders = new Hashtable();
int respContentLength = -1;
String respContentType = null;
String line = null;
while ((line = cs.in.readLine()) != null) {
if (line.length() == 0) break;
int colonIndex = line.indexOf(':');
String fieldName = line.substring(0, colonIndex);
String fieldValue = line.substring(colonIndex + 1).trim();
if (fieldName.equals(HEADER_CONTENT_LENGTH)) respContentLength = Integer.parseInt(fieldValue);
else if (fieldName.equals(HEADER_CONTENT_TYPE)) respContentType = fieldValue;
else respHeaders.put(fieldName, fieldValue);
}
Response response =
new Response(
statusCode, statusString, respHeaders, respContentLength, respContentType, cs.in);
return response;
}
String getRequestHost() {
return reqURL.getHost();
}
public static boolean isHosting(URL url_in) {
return (tracker_ip.length() > 0
&& UrlUtils.expandIPV6Host(url_in.getHost()).equalsIgnoreCase(tracker_ip));
}