void processPage(String pageUri) { String search = "href=\""; try { log.info("Calling to Google: " + pageUri); String inputString = UrlUtils.getURL(pageUri); log.info(inputString); int next = 0; Pattern cite = Pattern.compile("<cite>(.*?)</cite>"); Matcher matcher = cite.matcher(inputString); while (matcher.find()) { String newURI = "http://" + matcher .group(1) .replaceAll("\"", "") .replaceAll("<b>|</b>", "") .replaceAll("[ \\t\\n\\r]+", "") .trim(); log.info(newURI); profiles.addDeviceIfNotAlreadyKnown(newURI); } } catch (Exception e) { log.error(e.toString(), e); System.exit(0); } }
/** * get请求 * * @param context * @param url * @param params * @param responseListener * @throws IOException * @throws Exception */ public static InputStream request(Context context, String url, Bundle params) throws IOException, Exception { if (params != null) { if (url.contains("?")) { url = url + "&" + UrlUtils.encodeUrl(params); } else { url = url + "?" + UrlUtils.encodeUrl(params); } } Logger.getLogger().d("GET:" + url); HttpGet request = new HttpGet(url); HttpClient httpClient = getInstance(context); // 解决:HttpClient WARNING: Cookie rejected: Illegal domain attribute httpClient .getParams() .setParameter(ClientPNames.COOKIE_POLICY, CookiePolicy.BROWSER_COMPATIBILITY); HttpResponse response; InputStream in = null; response = httpClient.execute(request); in = response.getEntity().getContent(); return in; }
/** * get请求 * * @param context * @param url * @param params * @param responseListener * @throws NetworkErrorException */ public static void request( final Context context, String url, Bundle params, ResponseListener responseListener) { mLogger = Logger.getLogger(); if (params != null) { if (url.contains("?")) { url = url + "&" + UrlUtils.encodeUrl(params); } else { url = url + "?" + UrlUtils.encodeUrl(params); } } Logger.getLogger().d("GET:" + url); HttpGet request = new HttpGet(url); HttpClient httpClient = getInstance(context); // 解决:HttpClient WARNING: Cookie rejected: Illegal domain attribute httpClient .getParams() .setParameter(ClientPNames.COOKIE_POLICY, CookiePolicy.BROWSER_COMPATIBILITY); String response = ""; try { response = httpClient.execute(request, new BasicResponseHandler()); if (!TextUtils.isEmpty(response)) { responseListener.onComplete(response); } else { responseListener.onComplete(""); } } catch (IOException e) { e.printStackTrace(); if (e instanceof ConnectTimeoutException) { mLogger.e("error ConnectTimeoutException."); } mLogger.e("error begin."); mLogger.e("error params:"); mLogger.e("error.url(GET):" + url); mLogger.e("error.getMessage:" + e.getMessage()); mLogger.e("error end."); responseListener.onFail(new MyCosIOException("request url IOException", e)); } catch (Exception e) { e.printStackTrace(); mLogger.e("error:" + e.getMessage()); responseListener.onFail(new MyCosOtherException("request url Exception", e)); } }
/** callback from suggestion dropdown user selected a suggestion */ @Override public void onAction(String text, String extra, String source) { mUiController.getCurrentTopWebView().requestFocus(); if (UrlInputView.TYPED.equals(source)) { String url = UrlUtils.smartUrlFilter(text, false); Tab t = mBaseUi.getActiveTab(); // Only shortcut javascript URIs for now, as there is special // logic in UrlHandler for other schemas if (url != null && t != null && url.startsWith("javascript:")) { mUiController.loadUrl(t, url); setDisplayTitle(text); return; } } Intent i = new Intent(); String action = null; if (UrlInputView.VOICE.equals(source)) { action = RecognizerResultsIntent.ACTION_VOICE_SEARCH_RESULTS; source = null; } else { action = Intent.ACTION_SEARCH; } i.setAction(action); i.putExtra(SearchManager.QUERY, text); if (extra != null) { i.putExtra(SearchManager.EXTRA_DATA_KEY, extra); } if (source != null) { Bundle appData = new Bundle(); appData.putString(com.android.common.Search.SOURCE, source); i.putExtra(SearchManager.APP_DATA, appData); } mUiController.handleNewIntent(i); setDisplayTitle(text); }
/** Downloads and deploys the sourceFile */ private void deploySourceFile() { // download the source file... String downloadedFile = UrlUtils.download(srcUrl, this.workingFolderPath); // unzip content to src... ArchiveUtils.decompressZip(downloadedFile, this.workingFolderPath, true); }
@Test public void testAbsoluteUrl() throws MalformedURLException { URL baseUrl; baseUrl = new URL("http://domain.com/path/to/file"); String derivedUrl = UrlUtils.makeUrl(baseUrl, "http://domain2.com/newpath"); assertEquals("http://domain2.com/newpath", derivedUrl); }
@Test public void testLeadingDotSlash() throws MalformedURLException { URL baseUrl; baseUrl = new URL("http://domain.com/path/"); String derivedUrl = UrlUtils.makeUrl(baseUrl, "./file.html"); assertEquals("http://domain.com/path/file.html", derivedUrl); }
/** Adds a single sitemap to the index */ public SitemapIndexGenerator addUrl(SitemapIndexUrl url) { UrlUtils.checkUrl(url.url.toString(), baseUrlString); if (urls.size() >= maxUrls) { throw new RuntimeException("More than " + maxUrls + " urls"); } urls.add(url); return this; }
@Test public void testJavascript() throws MalformedURLException { URL baseUrl; baseUrl = new URL("http://domain.com/"); String relativeUrl = "JavaScript:GoURL(dowURL, '')"; String derivedUrl = UrlUtils.makeUrl(baseUrl, relativeUrl); assertEquals(relativeUrl, derivedUrl); }
@Test public void testProtocolAndDomain() throws MalformedURLException { assertEquals( "http://domain.com", UrlUtils.makeProtocolAndDomain("http://domain.com/index.html")); assertEquals( "http://domain.com", UrlUtils.makeProtocolAndDomain("http://domain.com:80/index.html")); assertEquals( "http://domain.com:8080", UrlUtils.makeProtocolAndDomain("http://domain.com:8080/index.html")); assertEquals("https://domain.com", UrlUtils.makeProtocolAndDomain("https://domain.com/")); try { UrlUtils.makeProtocolAndDomain("mailto:[email protected]"); fail("Exception should be thrown"); } catch (MalformedURLException e) { } }
@Override public void execute() throws BuildException { try { log("extracting " + this.url + " into " + this.outFile); String data = new HttpDocumentGrabber().grab(this.url); DocumentParser parser = new DocumentParser(); parser.setEnclosingTag("section"); String xhtml = parser.parse(this.url, UrlUtils.formatTitle(this.url), data); FileUtils.writeStringToFile(this.outFile, xhtml); } catch (Exception ex) { throw new BuildException(ex); } }
@Test public void testRawQuery() throws MalformedURLException { URL baseUrl; String derivedUrl; // See // http://www.communities.hp.com/securitysoftware/blogs/jeff/archive/2007/12/19/RFC-1808-vs-2396-vs-3986_3A00_-Browsers-vs.-programing-languages.aspx // Also http://www.ietf.org/rfc/rfc3986.txt // Also http://issues.apache.org/jira/browse/NUTCH-566 // Also http://issues.apache.org/jira/browse/NUTCH-436 baseUrl = new URL("http://domain.com/path/"); derivedUrl = UrlUtils.makeUrl(baseUrl, "?pid=1"); assertEquals("http://domain.com/path/?pid=1", derivedUrl); baseUrl = new URL("http://domain.com/file"); derivedUrl = UrlUtils.makeUrl(baseUrl, "?pid=1"); assertEquals("http://domain.com/file?pid=1", derivedUrl); baseUrl = new URL("http://domain.com/path/d;p?q#f"); derivedUrl = UrlUtils.makeUrl(baseUrl, "?pid=1"); assertEquals("http://domain.com/path/d;p?pid=1", derivedUrl); }
private Map executeHTTP(Map data_to_send, boolean v6) throws Exception { if (v6 && !enable_v6) { throw (new Exception("IPv6 is disabled")); } String host = getHost(v6, HTTP_SERVER_ADDRESS_V6, HTTP_SERVER_ADDRESS_V4); if (Logger.isEnabled()) Logger.log( new LogEvent( LOGID, "VersionCheckClient retrieving " + "version information from " + host + ":" + HTTP_SERVER_PORT + " via HTTP")); String url_str = "http://" + (v6 ? UrlUtils.convertIPV6Host(host) : host) + (HTTP_SERVER_PORT == 80 ? "" : (":" + HTTP_SERVER_PORT)) + "/version?"; url_str += URLEncoder.encode(new String(BEncoder.encode(data_to_send), "ISO-8859-1"), "ISO-8859-1"); URL url = new URL(url_str); HttpURLConnection url_connection = (HttpURLConnection) url.openConnection(); url_connection.connect(); try { InputStream is = url_connection.getInputStream(); Map reply = BDecoder.decode(new BufferedInputStream(is)); preProcessReply(reply, v6); return (reply); } finally { url_connection.disconnect(); } }
private static boolean checkForSpecialSearchTerm(String str) { str = str.trim(); String hit = UrlUtils.parseTextForURL(str, true, true); if (hit == null) { return (false); } UIFunctionsSWT uiFunctions = UIFunctionsManagerSWT.getUIFunctionsSWT(); new FileDownloadWindow(uiFunctions.getMainShell(), str, null, null, true); return (true); }
ProcessProfile(Resource device) { myARPReader.setProperty("WARN_RESOLVING_URI_AGAINST_EMPTY_BASE", "EM_IGNORE"); myARPReader.setErrorHandler( new RDFErrorHandler() { // ARP parser error handling routines public void warning(Exception e) { outputMsg("RDF parser warning:" + e.getMessage()); } public void error(Exception e) { outputMsg("RDF parser error:" + e.getMessage()); profileValidFlag = false; } public void fatalError(Exception e) { e.printStackTrace(); error(e); } }); this.deviceURI = device.getURI(); try { profile = UrlUtils.getURL(deviceURI).trim(); if (profile.contains("<html") || profile.contains("<head>")) { // this is HTML not a UAProf profile device.removeProperties(); } else if (!profile.contains("rdf") && !profile.contains("RDF")) { device.removeProperties(); } else if (!profile.contains("uaprof") && !profile.contains("UAPROF")) { device.removeProperties(); } else if (!profile.contains("openmobilealliance") && !profile.contains("wapforum")) { device.removeProperties(); } else { validate(device); profiles.fixMetadata(device, profile); } } catch (IOException io) { outputMsg("Could not retrieve " + device.getURI()); device.removeProperties(); unreachableProfiles++; } System.out.println(messages.toString()); }
private String getHTTPGetString(Map content, boolean for_proxy, boolean v6) { String host = getHost(v6, HTTP_SERVER_ADDRESS_V6, HTTP_SERVER_ADDRESS_V4); String get_str = "GET " + (for_proxy ? ("http://" + (v6 ? UrlUtils.convertIPV6Host(host) : host) + ":" + HTTP_SERVER_PORT) : "") + "/version?"; try { get_str += URLEncoder.encode(new String(BEncoder.encode(content), "ISO-8859-1"), "ISO-8859-1"); } catch (Throwable e) { } get_str += " HTTP/1.1" + "\015\012" + "\015\012"; return (get_str); }
private void createPropertiesFile(Model prfs) throws IOException { ProcessProperties p = new ProcessProperties(); HashMap<String, HashMap<String, Integer>> properties = p.createPropertyStructure(prfs); StringBuffer result = p.printResults(properties); UrlUtils.savePage(Constants.PROPERTIES_OUTPUT_FILE, result); }
public void runSupport() { try { new URL(url_str); // determine if this is already a proper URL } catch (Throwable t) { // it's not // //check if the string is just a base32/hex-encoded torrent infohash // // String magnet_uri = UrlUtils.normaliseMagnetURI( url_str ); // // if ( magnet_uri != null ){ // // url_str = magnet_uri; // } } try { url = AddressUtils.adjustURL(new URL(url_str)); String protocol = url.getProtocol().toLowerCase(); // hack here - the magnet download process requires an additional paramter to cause it to // stall on error so the error can be reported // if ( protocol.equals( "magnet" ) || protocol.equals( "dht" )){ // // url = AddressUtils.adjustURL( new URL(url_str+"&pause_on_error=true")); // } for (int i = 0; i < 2; i++) { try { // if ( protocol.equals("https")){ // // // see ConfigurationChecker for SSL client defaults // // HttpsURLConnection ssl_con = (HttpsURLConnection)url.openConnection(); // // // allow for certs that contain IP addresses rather than dns names // // ssl_con.setHostnameVerifier( // new HostnameVerifier() // { // public boolean // verify( // String host, // SSLSession session ) // { // return( true ); // } // }); // // con = ssl_con; // // }else{ // con = (HttpURLConnection) url.openConnection(); // } con.setRequestProperty( "User-Agent", Constants.AZUREUS_NAME + " " + Constants.AZUREUS_VERSION); if (referrer != null && referrer.length() > 0) { con.setRequestProperty("Referer", referrer); } if (request_properties != null) { Iterator it = request_properties.entrySet().iterator(); while (it.hasNext()) { Map.Entry entry = (Map.Entry) it.next(); String key = (String) entry.getKey(); String value = (String) entry.getValue(); // currently this code doesn't support gzip/deflate... if (!key.equalsIgnoreCase("Accept-Encoding")) { con.setRequestProperty(key, value); } } } this.con.connect(); break; // }catch( SSLException e ){ // // if ( i == 0 ){ // // if ( SESecurityManager.installServerCertificates( url ) != null ){ // // // certificate has been installed // // continue; // retry with new certificate // } // } // // throw( e ); // } catch (IOException e) { if (i == 0) { URL retry_url = UrlUtils.getIPV4Fallback(url); if (retry_url != null) { url = retry_url; } else { throw (e); } } } } int response = this.con.getResponseCode(); if (!ignoreReponseCode) { if ((response != HttpURLConnection.HTTP_ACCEPTED) && (response != HttpURLConnection.HTTP_OK)) { this.error(response, Integer.toString(response) + ": " + this.con.getResponseMessage()); return; } } /* Map headerFields = this.con.getHeaderFields(); System.out.println("Header of download of " + url_str); for (Iterator iter = headerFields.keySet().iterator(); iter.hasNext();) { String s = (String) iter.next(); System.out.println(s + ":" + headerFields.get(s)); } */ filename = this.con.getHeaderField("Content-Disposition"); if ((filename != null) && filename .toLowerCase() .matches(".*attachment.*")) // Some code to handle b0rked servers. while (filename.toLowerCase().charAt(0) != 'a') filename = filename.substring(1); if ((filename == null) || !filename.toLowerCase().startsWith("attachment") || (filename.indexOf('=') == -1)) { String tmp = this.url.getFile(); if (tmp.length() == 0 || tmp.equals("/")) { filename = url.getHost(); } else if (tmp.startsWith("?")) { // probably a magnet URI - use the hash // magnet:?xt=urn:sha1:VGC53ZWCUXUWVGX7LQPVZIYF4L6RXSU6 String query = tmp.toUpperCase(); int pos = query.indexOf("XT=URN:SHA1:"); if (pos == -1) { pos = query.indexOf("XT=URN:BTIH:"); } if (pos != -1) { pos += 12; int p2 = query.indexOf("&", pos); if (p2 == -1) { filename = query.substring(pos); } else { filename = query.substring(pos, p2); } } else { filename = "Torrent" + (long) (Math.random() * Long.MAX_VALUE); } filename += ".tmp"; } else { if (tmp.lastIndexOf('/') != -1) tmp = tmp.substring(tmp.lastIndexOf('/') + 1); // remove any params in the url int param_pos = tmp.indexOf('?'); if (param_pos != -1) { tmp = tmp.substring(0, param_pos); } filename = URLDecoder.decode(tmp, Constants.DEFAULT_ENCODING); } } else { filename = filename.substring(filename.indexOf('=') + 1); if (filename.startsWith("\"") && filename.endsWith("\"")) filename = filename.substring(1, filename.lastIndexOf('\"')); filename = URLDecoder.decode(filename, Constants.DEFAULT_ENCODING); // not sure of this piece of logic here but I'm not changing it at the moment File temp = new File(filename); filename = temp.getName(); } filename = FileUtil.convertOSSpecificChars(filename, false); // directoryname = // COConfigurationManager.getDirectoryParameter("General_sDefaultTorrent_Directory"); // boolean useTorrentSave = COConfigurationManager.getBooleanParameter("Save Torrent // Files"); directoryname = "D:\\Development\\testDownloads\\"; boolean useTorrentSave = true; if (file_str != null) { // not completely sure about the whole logic in this block File temp = new File(file_str); // if we're not using a default torrent save dir if (!useTorrentSave || directoryname.length() == 0) { // if it's already a dir if (temp.isDirectory()) { // use it directoryname = temp.getCanonicalPath(); } // it's a file else { // so use its parent dir directoryname = temp.getCanonicalFile().getParent(); } } // if it's a file if (!temp.isDirectory()) { // set the file name filename = temp.getName(); } } // what would happen here if directoryname == null and file_str == null?? this.state = STATE_INIT; this.notifyListener(); } catch (java.net.MalformedURLException e) { this.error(0, "Exception while parsing URL '" + url + "':" + e.getMessage()); } catch (java.net.UnknownHostException e) { this.error( 0, "Exception while initializing download of '" + url + "': Unknown Host '" + e.getMessage() + "'"); } catch (java.io.IOException ioe) { this.error(0, "I/O Exception while initializing download of '" + url + "':" + ioe.toString()); } catch (Throwable e) { this.error(0, "Exception while initializing download of '" + url + "':" + e.toString()); } if (this.state == STATE_ERROR) { return; } try { final boolean status_reader_run[] = {true}; this.state = STATE_START; notifyListener(); this.state = STATE_DOWNLOADING; notifyListener(); Thread status_reader = new AEThread("TorrentDownloader:statusreader") { public void runSupport() { boolean changed_status = false; while (true) { try { Thread.sleep(100); try { this_mon.enter(); if (!status_reader_run[0]) { break; } } finally { this_mon.exit(); } String s = con.getResponseMessage(); if (!s.equals(getStatus())) { if (!s.toLowerCase().startsWith("error:")) { if (s.toLowerCase().indexOf("alive") != -1) { if (percentDone < 10) { percentDone++; } } int pos = s.indexOf('%'); if (pos != -1) { int i; for (i = pos - 1; i >= 0; i--) { char c = s.charAt(i); if (!Character.isDigit(c) && c != ' ') { i++; break; } } try { percentDone = Integer.parseInt(s.substring(i, pos).trim()); } catch (Throwable e) { } } setStatus(s); } else { error(con.getResponseCode(), s.substring(6)); } changed_status = true; } } catch (Throwable e) { break; } } if (changed_status) { setStatus(""); } } }; status_reader.setDaemon(true); status_reader.start(); InputStream in; try { in = this.con.getInputStream(); } catch (FileNotFoundException e) { if (ignoreReponseCode) { in = this.con.getErrorStream(); } else { throw e; } } finally { try { this_mon.enter(); status_reader_run[0] = false; } finally { this_mon.exit(); } } // handle some servers that return gzip'd torrents even though we don't request it! String encoding = con.getHeaderField("content-encoding"); if (encoding != null) { if (encoding.equalsIgnoreCase("gzip")) { in = new GZIPInputStream(in); } else if (encoding.equalsIgnoreCase("deflate")) { in = new InflaterInputStream(in); } } if (this.state != STATE_ERROR) { this.file = new File(this.directoryname, filename); boolean useTempFile = false; try { this.file.createNewFile(); useTempFile = !this.file.exists(); } catch (Throwable t) { useTempFile = true; } if (useTempFile) { this.file = File.createTempFile("AZU", ".torrent", new File(this.directoryname)); this.file.createNewFile(); } FileOutputStream fileout = new FileOutputStream(this.file, false); bufBytes = 0; int size = (int) UrlUtils.getContentLength(con); this.percentDone = -1; do { if (this.cancel) { break; } try { bufBytes = in.read(buf); this.readTotal += bufBytes; if (size != 0) { this.percentDone = (100 * this.readTotal) / size; } notifyListener(); } catch (IOException e) { } if (bufBytes > 0) { fileout.write(buf, 0, bufBytes); } } while (bufBytes > 0); in.close(); fileout.flush(); fileout.close(); if (this.cancel) { this.state = STATE_CANCELLED; if (deleteFileOnCancel) { this.cleanUpFile(); } } else { if (this.readTotal <= 0) { this.error(0, "No data contained in '" + this.url.toString() + "'"); return; } // if the file has come down with a not-so-useful name then we try to rename // it to something more useful try { if (!filename.toLowerCase().endsWith(".torrent")) { TOTorrent torrent = TorrentUtils.readFromFile(file, false); String name = TorrentUtils.getLocalisedName(torrent) + ".torrent"; File new_file = new File(directoryname, name); if (file.renameTo(new_file)) { filename = name; file = new_file; } } } catch (Throwable e) { Debug.printStackTrace(e); } // TorrentUtils.setObtainedFrom( file, original_url ); this.state = STATE_FINISHED; } this.notifyListener(); } } catch (Exception e) { if (!cancel) { Debug.out("'" + this.directoryname + "' '" + filename + "'", e); } this.error(0, "Exception while downloading '" + this.url.toString() + "':" + e.getMessage()); } }
public static String getMediaFileMimeType(File mediaFile) { String originalFileName = mediaFile.getName().toLowerCase(); String mimeType = UrlUtils.getUrlMimeType(originalFileName); if (TextUtils.isEmpty(mimeType)) { try { String filePathForGuessingMime; if (mediaFile.getPath().contains("://")) { filePathForGuessingMime = Uri.encode(mediaFile.getPath(), ":/"); } else { filePathForGuessingMime = "file://" + Uri.encode(mediaFile.getPath(), "/"); } URL urlForGuessingMime = new URL(filePathForGuessingMime); URLConnection uc = urlForGuessingMime.openConnection(); String guessedContentType = uc.getContentType(); // internally calls guessContentTypeFromName(url.getFile()); and // guessContentTypeFromStream(is); // check if returned "content/unknown" if (!TextUtils.isEmpty(guessedContentType) && !guessedContentType.equals("content/unknown")) { mimeType = guessedContentType; } } catch (MalformedURLException e) { AppLog.e( AppLog.T.API, "MalformedURLException while trying to guess the content type for the file here " + mediaFile.getPath() + " with URLConnection", e); } catch (IOException e) { AppLog.e( AppLog.T.API, "Error while trying to guess the content type for the file here " + mediaFile.getPath() + " with URLConnection", e); } } // No mimeType yet? Try to decode the image and get the mimeType from there if (TextUtils.isEmpty(mimeType)) { try { DataInputStream inputStream = new DataInputStream(new FileInputStream(mediaFile)); String mimeTypeFromStream = getMimeTypeOfInputStream(inputStream); if (!TextUtils.isEmpty(mimeTypeFromStream)) { mimeType = mimeTypeFromStream; } inputStream.close(); } catch (FileNotFoundException e) { AppLog.e( AppLog.T.API, "FileNotFoundException while trying to guess the content type for the file " + mediaFile.getPath(), e); } catch (IOException e) { AppLog.e( AppLog.T.API, "IOException while trying to guess the content type for the file " + mediaFile.getPath(), e); } } if (TextUtils.isEmpty(mimeType)) { mimeType = ""; } else { if (mimeType.equalsIgnoreCase( "video/mp4v-es")) { // Fixes #533. See: http://tools.ietf.org/html/rfc3016 mimeType = "video/mp4"; } } return mimeType; }
@Test public void testQueryParamCase() throws MalformedURLException { URL baseUrl = new URL("http://domain.com"); String derivedUrl = UrlUtils.makeUrl(baseUrl, "?pid=1"); assertEquals("http://domain.com/?pid=1", derivedUrl); }