private T requestInfo(URI baseUrl, RenderingContext context)
      throws IOException, URISyntaxException, ParserConfigurationException, SAXException {
    URL url = loader.createURL(baseUrl, context);

    GetMethod method = null;
    try {
      final InputStream stream;

      if ((url.getProtocol().equals("http") || url.getProtocol().equals("https"))
          && context.getConfig().localHostForwardIsFrom(url.getHost())) {
        String scheme = url.getProtocol();
        final String host = url.getHost();
        if (url.getProtocol().equals("https")
            && context.getConfig().localHostForwardIsHttps2http()) {
          scheme = "http";
        }
        URL localUrl = new URL(scheme, "localhost", url.getPort(), url.getFile());
        HttpURLConnection connexion = (HttpURLConnection) localUrl.openConnection();
        connexion.setRequestProperty("Host", host);
        for (Map.Entry<String, String> entry : context.getHeaders().entrySet()) {
          connexion.setRequestProperty(entry.getKey(), entry.getValue());
        }
        stream = connexion.getInputStream();
      } else {
        method = new GetMethod(url.toString());
        for (Map.Entry<String, String> entry : context.getHeaders().entrySet()) {
          method.setRequestHeader(entry.getKey(), entry.getValue());
        }
        context.getConfig().getHttpClient(baseUrl).executeMethod(method);
        int code = method.getStatusCode();
        if (code < 200 || code >= 300) {
          throw new IOException(
              "Error "
                  + code
                  + " while reading the Capabilities from "
                  + url
                  + ": "
                  + method.getStatusText());
        }
        stream = method.getResponseBodyAsStream();
      }
      final T result;
      try {
        result = loader.parseInfo(stream);
      } finally {
        stream.close();
      }
      return result;
    } finally {
      if (method != null) {
        method.releaseConnection();
      }
    }
  }
 private boolean processURL(URL url, String baseDir, StatusWindow status) throws IOException {
   if (processedLinks.contains(url)) {
     return false;
   } else {
     processedLinks.add(url);
   }
   URLConnection connection = url.openConnection();
   InputStream in = new BufferedInputStream(connection.getInputStream());
   ArrayList list = processPage(in, baseDir, url);
   if ((status != null) && (list.size() > 0)) {
     status.setMaximum(list.size());
   }
   for (int i = 0; i < list.size(); i++) {
     if (status != null) {
       status.setMessage(Utils.trimFileName(list.get(i).toString(), 40), i);
     }
     if ((!((String) list.get(i)).startsWith("RUN"))
         && (!((String) list.get(i)).startsWith("SAVE"))
         && (!((String) list.get(i)).startsWith("LOAD"))) {
       processURL(
           new URL(url.getProtocol(), url.getHost(), url.getPort(), (String) list.get(i)),
           baseDir,
           status);
     }
   }
   in.close();
   return true;
 }
Example #3
0
  @Override
  public void run() {
    try {
      URL url = buildPingUrl();
      if (logger.isDebugEnabled()) {
        logger.debug("Sending UDC information to {}...", url);
      }
      HttpURLConnection conn = (HttpURLConnection) url.openConnection();
      conn.setConnectTimeout((int) HTTP_TIMEOUT.millis());
      conn.setReadTimeout((int) HTTP_TIMEOUT.millis());

      if (conn.getResponseCode() >= 300) {
        throw new Exception(
            String.format("%s Responded with Code %d", url.getHost(), conn.getResponseCode()));
      }
      if (logger.isDebugEnabled()) {
        BufferedReader reader = new BufferedReader(new InputStreamReader(conn.getInputStream()));
        String line = reader.readLine();
        while (line != null) {
          logger.debug(line);
          line = reader.readLine();
        }
        reader.close();
      } else {
        conn.getInputStream().close();
      }
      successCounter.incrementAndGet();
    } catch (Exception e) {
      if (logger.isDebugEnabled()) {
        logger.debug("Error sending UDC information", e);
      }
      failCounter.incrementAndGet();
    }
  }
  private JarFile getCachedJarFile(URL url) {
    JarFile result = (JarFile) fileCache.get(url);

    /* if the JAR file is cached, the permission will always be there */
    if (result != null) {
      Permission perm = getPermission(result);
      if (perm != null) {
        SecurityManager sm = System.getSecurityManager();
        if (sm != null) {
          try {
            sm.checkPermission(perm);
          } catch (SecurityException se) {
            // fallback to checkRead/checkConnect for pre 1.2
            // security managers
            if ((perm instanceof java.io.FilePermission)
                && perm.getActions().indexOf("read") != -1) {
              sm.checkRead(perm.getName());
            } else if ((perm instanceof java.net.SocketPermission)
                && perm.getActions().indexOf("connect") != -1) {
              sm.checkConnect(url.getHost(), url.getPort());
            } else {
              throw se;
            }
          }
        }
      }
    }
    return result;
  }
Example #5
0
  public static void setAZTracker(URL tracker_url, boolean az_tracker) {
    String key = tracker_url.getHost() + ":" + tracker_url.getPort();

    synchronized (az_trackers) {
      boolean changed = false;

      if (az_trackers.get(key) == null) {

        if (az_tracker) {

          az_trackers.put(key, new Long(SystemTime.getCurrentTime()));

          changed = true;
        }
      } else {

        if (!az_tracker) {

          if (az_trackers.remove(key) != null) {

            changed = true;
          }
        }
      }

      if (changed) {

        COConfigurationManager.setParameter("Tracker Client AZ Instances", az_trackers);
      }
    }
  }
  public synchronized void logUnavailableHost(URL url) {
    if (this.offlineMode) return;

    if (url == null) {
      String message = Logging.getMessage("nullValue.URLIsNull");
      Logging.error(message);
      throw new IllegalArgumentException(message);
    }

    String hostName = url.getHost();
    HostInfo hi = this.hostMap.get(hostName);
    if (hi != null) {
      if (!hi.isUnavailable()) {
        hi.logCount.incrementAndGet();
        if (hi.isUnavailable()) // host just became unavailable
        this.firePropertyChange(NetworkStatus.HOST_UNAVAILABLE, null, url);
      }
      hi.lastLogTime.set(System.currentTimeMillis());
    } else {
      hi = new HostInfo(this.attemptLimit.get(), this.tryAgainInterval.get());
      hi.logCount.set(1);
      if (hi.isUnavailable()) // the attempt limit may be as low as 1, so handle that case here
      this.firePropertyChange(NetworkStatus.HOST_UNAVAILABLE, null, url);
      this.hostMap.put(hostName, hi);
    }

    this.lastUnavailableLogTime.set(System.currentTimeMillis());
  }
Example #7
0
 /**
  * ** Gets the 'host' specification in the URI ** @return The 'host' specification in the URL, or
  * null if unable to determine host
  */
 public String getHost() {
   try {
     URL url = new URL(this.getURI());
     return url.getHost();
   } catch (MalformedURLException mue) {
     return null;
   }
 }
Example #8
0
  /**
   * 以阻塞的方式立即下载一个文件,该方法会覆盖已经存在的文件
   *
   * @param task
   * @return "ok" if download success (else return errmessage);
   */
  static String download(DownloadTask task) {
    if (!openedStatus && show) openStatus();

    URL url;
    HttpURLConnection conn;
    try {
      url = new URL(task.getOrigin());
      conn = (HttpURLConnection) url.openConnection();
      conn.setConnectTimeout(30000);
      conn.setReadTimeout(30000);
      conn.setRequestProperty(
          "User-Agent",
          "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.134 Safari/"
              + Math.random());
      if ("www.imgjav.com".equals(url.getHost())) { // 该网站需要带cookie请求
        conn.setRequestProperty(
            "User-Agent",
            "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.134 Safari/537.36");
        // conn.setRequestProperty("Cookie", "__cfduid=d219ea333c7a9b5743b572697b631925a1446093229;
        // cf_clearance=6ae62d843f5d09acf393f9e4eb130d9366840c82-1446093303-28800");
        conn.setRequestProperty(
            "Cookie",
            "__cfduid=d6ee846b378bb7d5d173a05541f8a2b6a1446090548; cf_clearance=ea10e8db31f8b6ee51570b118dd89b7e616d7b62-1446099714-28800");
        conn.setRequestProperty("Host", "www.imgjav.com");
      }
      Path directory = Paths.get(task.getDest()).getParent();
      if (!Files.exists(directory)) Files.createDirectories(directory);
    } catch (Exception e) {
      e.printStackTrace();
      return e.getMessage();
    }

    try (InputStream is = conn.getInputStream();
        BufferedInputStream in = new BufferedInputStream(is);
        FileOutputStream fos = new FileOutputStream(task.getDest());
        OutputStream out = new BufferedOutputStream(fos); ) {
      int length = conn.getContentLength();
      if (length < 1) throw new IOException("length<1");
      byte[] binary = new byte[length];
      byte[] buff = new byte[65536];
      int len;
      int index = 0;
      while ((len = in.read(buff)) != -1) {
        System.arraycopy(buff, 0, binary, index, len);
        index += len;
        allLen += len; // allLen有线程安全的问题 ,可能会不正确。无需精确数据,所以不同步了
        task.setReceivePercent(String.format("%.2f", ((float) index / length) * 100) + "%");
      }
      out.write(binary);
    } catch (IOException e) {
      e.printStackTrace();
      return e.getMessage();
    }
    return "ok";
  }
Example #9
0
 // Strip off everything unneeded
 private String getBaseUrl(String pUrl, String pServletPath) {
   String sUrl;
   try {
     URL url = new URL(pUrl);
     String host = getIpIfPossible(url.getHost());
     sUrl = new URL(url.getProtocol(), host, url.getPort(), pServletPath).toExternalForm();
   } catch (MalformedURLException exp) {
     sUrl = plainReplacement(pUrl, pServletPath);
   }
   return sUrl;
 }
 /** Updates the host port attributes based on the URI. */
 private void updateHostPort() {
   if (uri != null && !"".equals(uri.trim())) {
     try {
       URL url = new URL(uri);
       host = url.getHost();
       port = url.getPort();
     } catch (MalformedURLException e) {
       // this won't happen
     }
   }
 }
Example #11
0
  public static boolean isAZTracker(URL tracker_url) {
    String host = tracker_url.getHost();

    if (Constants.isAzureusDomain(host)) {

      return (true);
    }

    synchronized (az_trackers) {
      return (az_trackers.containsKey(host + ":" + tracker_url.getPort()));
    }
  }
Example #12
0
  public static boolean isUDPProbeOK(URL tracker_url) {
    String host = tracker_url.getHost();

    if (Constants.isAzureusDomain(host)) {

      return (false);
    }

    synchronized (udp_probe_results) {
      return (udp_probe_results.containsKey(host));
    }
  }
  public void setAuthenticationOutcome(String realm, URL tracker, boolean success) {
    try {
      this_mon.enter();

      setAuthenticationOutcome(
          realm, tracker.getProtocol(), tracker.getHost(), tracker.getPort(), success);

    } finally {

      this_mon.exit();
    }
  }
  public PasswordAuthentication getAuthentication(String realm, URL tracker) {
    try {
      this_mon.enter();

      return (getAuthentication(
          realm, tracker.getProtocol(), tracker.getHost(), tracker.getPort()));

    } finally {

      this_mon.exit();
    }
  }
Example #15
0
  public static void checkForBlacklistedURLs(URL url) throws IOException {

    for (int i = 0; i < BLACKLISTED_HOSTS.length; i++) {

      if (url.getHost().equalsIgnoreCase(BLACKLISTED_HOSTS[i])
          && url.getPort() == BLACKLISTED_PORTS[i]) {

        throw (new IOException(
            "http://" + BLACKLISTED_HOSTS[i] + ":" + BLACKLISTED_PORTS[i] + "/ is not a tracker"));
      }
    }
  }
Example #16
0
 /** ** Sets the 'port' */
 public boolean setPort(int _port) {
   String uri = this.getURI();
   if ((_port > 0) && URIArg.isAbsoluteURL(uri)) {
     try {
       URL oldURI = new URL(uri);
       String proto = oldURI.getProtocol();
       String host = oldURI.getHost();
       int port = _port;
       String file = oldURI.getFile();
       URL newURI = new URL(proto, host, port, file);
       this._setURI(newURI.toString());
       return true;
     } catch (MalformedURLException mue) {
       // error
     }
   }
   return false;
 }
  public synchronized void logAvailableHost(URL url) {
    if (this.offlineMode) return;

    if (url == null) {
      String message = Logging.getMessage("nullValue.URLIsNull");
      Logging.error(message);
      throw new IllegalArgumentException(message);
    }

    String hostName = url.getHost();
    HostInfo hi = this.hostMap.get(hostName);
    if (hi != null) {
      this.hostMap.remove(hostName); // host is available again
      firePropertyChange(NetworkStatus.HOST_AVAILABLE, null, url);
    }

    this.lastAvailableLogTime.set(System.currentTimeMillis());
  }
  public boolean isHostUnavailable(URL url) {
    if (this.offlineMode) return true;

    if (url == null) {
      String message = Logging.getMessage("nullValue.URLIsNull");
      Logging.error(message);
      throw new IllegalArgumentException(message);
    }

    String hostName = url.getHost();
    HostInfo hi = this.hostMap.get(hostName);
    if (hi == null) return false;

    if (hi.isTimeToTryAgain()) {
      hi.logCount.set(0); // info removed from table in logAvailableHost
      return false;
    }

    return hi.isUnavailable();
  }
 /**
  * mapUrlToFileLocation() is the method used to resolve urls into file names. This maps a given
  * url to a file location, using the au top directory as the base. It creates directories which
  * mirror the html string, so 'http://www.journal.org/issue1/index.html' would be cached in the
  * file: <rootLocation>/www.journal.org/http/issue1/index.html
  *
  * @param rootLocation the top directory for ArchivalUnit this URL is in
  * @param urlStr the url to translate
  * @return the url file location
  * @throws java.net.MalformedURLException
  */
 public static String mapUrlToFileLocation(String rootLocation, String urlStr)
     throws MalformedURLException {
   int totalLength = rootLocation.length() + urlStr.length();
   URL url = new URL(urlStr);
   StringBuilder buffer = new StringBuilder(totalLength);
   buffer.append(rootLocation);
   if (!rootLocation.endsWith(File.separator)) {
     buffer.append(File.separator);
   }
   buffer.append(url.getHost().toLowerCase());
   int port = url.getPort();
   if (port != -1) {
     buffer.append(PORT_SEPARATOR);
     buffer.append(port);
   }
   buffer.append(File.separator);
   buffer.append(url.getProtocol());
   if (RepositoryManager.isEnableLongComponents()) {
     String escapedPath =
         escapePath(
             StringUtil.replaceString(url.getPath(), UrlUtil.URL_PATH_SEPARATOR, File.separator));
     String query = url.getQuery();
     if (query != null) {
       escapedPath = escapedPath + "?" + escapeQuery(query);
     }
     String encodedPath = RepositoryNodeImpl.encodeUrl(escapedPath);
     // encodeUrl strips leading / from path
     buffer.append(File.separator);
     buffer.append(encodedPath);
   } else {
     buffer.append(
         escapePath(
             StringUtil.replaceString(url.getPath(), UrlUtil.URL_PATH_SEPARATOR, File.separator)));
     String query = url.getQuery();
     if (query != null) {
       buffer.append("?");
       buffer.append(escapeQuery(query));
     }
   }
   return buffer.toString();
 }
  /**
   * Get a file name from a URL.
   *
   * @param url The URL.
   * @param outputDirectoryName The output directory name.
   * @return The file name stripped of its original path with the specified output directory
   *     attached.
   */
  public static String getFileNameFromURL(URL url, String outputDirectoryName) {
    String hostName = url.getHost();
    String path = url.getPath();
    String baseName = "";

    if (path.length() == 0) {
      if (hostName.length() == 0) {
        baseName = "adorned";
      } else {
        baseName = hostName;
      }
    } else {
      if (hostName.length() == 0) {
        baseName = new File(path).getName();
      } else {
        baseName = new File(hostName + "." + path).getName();
      }
    }

    return new File(outputDirectoryName, baseName).getPath();
  }
Example #21
0
  public static void setUDPProbeResult(URL tracker_url, boolean probe_ok) {
    String key = tracker_url.getHost();

    synchronized (udp_probe_results) {
      boolean changed = false;

      if (udp_probe_results.get(key) == null) {

        if (probe_ok) {

          // arbitrary max size here just in case something weird happens

          if (udp_probe_results.size() > 512) {

            udp_probe_results.clear();
          }

          udp_probe_results.put(key, new Long(SystemTime.getCurrentTime()));

          changed = true;
        }
      } else {

        if (!probe_ok) {

          if (udp_probe_results.remove(key) != null) {

            changed = true;
          }
        }
      }

      if (changed) {

        COConfigurationManager.setParameter("Tracker Client UDP Probe Results", udp_probe_results);
      }
    }
  }
Example #22
0
  /**
   * POST something to the given URL. The headers are put in as HTTP headers, the content length is
   * calculated and the content is sent as content. Duh.
   *
   * @param cs Client socket items.
   * @param url the url to post to
   * @param headers additional headers to send as HTTP headers
   * @param contentType type of the content
   * @param content the body of the post
   * @exception IOException if post fails
   */
  private static Response post(
      ClientSocket cs, URL url, Hashtable headers, String contentType, String content)
      throws IOException {

    // If the port is not specified in the URL, use the default http port.
    int port = url.getPort();
    if (port == -1) port = 80;

    cs.s = new Socket(url.getHost(), +port);
    cs.out = new PrintWriter(cs.s.getOutputStream());
    cs.in = new BufferedReader(new InputStreamReader(cs.s.getInputStream()));

    // Create the HTTP header
    cs.out.print(HTTP_POST + " " + url.getFile() + " HTTP/" + HTTP_VERSION + "\r\n");
    cs.out.print(HEADER_HOST + ": " + url.getHost() + ':' + port + "\r\n");
    cs.out.print(HEADER_CONTENT_TYPE + ": " + contentType + "\r\n");
    cs.out.print(HEADER_CONTENT_LENGTH + ": " + content.length() + "\r\n");

    for (Enumeration e = headers.keys(); e.hasMoreElements(); ) {
      Object key = e.nextElement();
      cs.out.print(key + ": " + headers.get(key) + "\r\n");
    }

    // According to HTTP1.1 need another CRLF after the header
    cs.out.print("\r\n");

    Debug.log(Debug.MSG_LIFECYCLE, "HTTPUtils: Writing content to the socket... " + content);

    cs.out.println(content);

    cs.out.flush();

    /* read the status line */
    int statusCode = 0;
    String statusString = null;
    StringTokenizer st = new StringTokenizer(cs.in.readLine());
    st.nextToken(); // ignore version part
    statusCode = Integer.parseInt(st.nextToken());

    StringBuffer sb = new StringBuffer();

    while (st.hasMoreTokens()) {
      sb.append(st.nextToken());
      if (st.hasMoreTokens()) sb.append(" ");
    }

    statusString = sb.toString();

    /* get the headers */
    Hashtable respHeaders = new Hashtable();
    int respContentLength = -1;
    String respContentType = null;
    String line = null;

    while ((line = cs.in.readLine()) != null) {

      if (line.length() == 0) break;

      int colonIndex = line.indexOf(':');
      String fieldName = line.substring(0, colonIndex);
      String fieldValue = line.substring(colonIndex + 1).trim();

      if (fieldName.equals(HEADER_CONTENT_LENGTH)) respContentLength = Integer.parseInt(fieldValue);
      else if (fieldName.equals(HEADER_CONTENT_TYPE)) respContentType = fieldValue;
      else respHeaders.put(fieldName, fieldValue);
    }

    Response response =
        new Response(
            statusCode, statusString, respHeaders, respContentLength, respContentType, cs.in);

    return response;
  }
Example #23
0
 String getRequestHost() {
   return reqURL.getHost();
 }
Example #24
0
 public static boolean isHosting(URL url_in) {
   return (tracker_ip.length() > 0
       && UrlUtils.expandIPV6Host(url_in.getHost()).equalsIgnoreCase(tracker_ip));
 }