/** * Parse the login page for the GALX id. * * @return GALX id * @throws GoogleAuthenticatorException */ private String galx() throws GoogleAuthenticatorException { String galx = null; HttpGet get; try { DataConfiguration config = GoogleConfigurator.getConfiguration(); Pattern pattern = Pattern.compile(config.getString("google.auth.reGalx"), Pattern.CASE_INSENSITIVE); get = new HttpGet(config.getString("google.auth.loginUrl")); HttpResponse response = _httpClient.execute(get); String html = GoogleUtils.toString(response.getEntity().getContent()); // System.out.println("Page is : " + html); get.releaseConnection(); Matcher matcher = pattern.matcher(html); if (matcher.find()) { galx = matcher.group(1); } if (galx == null) { throw new GoogleAuthenticatorException("Cannot parse GALX!"); } } catch (ConfigurationException ex) { throw new GoogleAuthenticatorException(ex); } catch (ClientProtocolException ex) { throw new GoogleAuthenticatorException(ex); } catch (IOException ex) { throw new GoogleAuthenticatorException(ex); } // printCookies(); return galx; }
/** * This method retrieves data using HTTP or HTTPS protocol and 'get' method. * * @param url * @return */ protected byte[] httpGet(String url) { HttpGet httpRequest = null; HttpResponse httpResponse = null; try { final URI uri = URI.create(url.trim()); httpRequest = new HttpGet(uri); if (contentType != null) { httpRequest.setHeader(CONTENT_TYPE, contentType); } httpResponse = getHttpResponse(httpRequest, url); final byte[] returnedBytes = readHttpResponse(url, httpResponse); return returnedBytes; } finally { if (httpRequest != null) { httpRequest.releaseConnection(); } if (httpResponse != null) { EntityUtils.consumeQuietly(httpResponse.getEntity()); } } }
private boolean isClassAvailableToDeployment(String deploymentName, Class<?> clazz) throws IOException, HttpException { String className = clazz.getName(); String CONTEXT_URL = PortProviderUtil.generateURL("/", deploymentName); HttpGet httpget = new HttpGet( CONTEXT_URL + TestResource.LOAD_CLASS_PATH + "?" + TestResource.CLASSNAME_PARAM + "=" + className); try { String responseString = new String(); HttpClientContext context = HttpClientContext.create(); CloseableHttpResponse response = client.execute(httpget, context); HttpEntity entity = response.getEntity(); if (entity != null) { responseString = EntityUtils.toString(entity, StandardCharsets.UTF_8); } return (HttpStatus.SC_OK == response.getStatusLine().getStatusCode()) && (className.equals(responseString)); } finally { httpget.releaseConnection(); } }
@Override public Map<String, Optional<BigDecimal>> check(Check check) throws Exception { String formattedQuery = String.format(QUERY_STRING, new DateTime().getMillis(), check.getTarget()); URI uri = new URI(graphiteScheme, graphiteHost, graphitePath + "/render/", formattedQuery, null); System.out.println(uri.toString()); HttpGet get = new HttpGet(uri); Map<String, Optional<BigDecimal>> targetValues = new HashMap<String, Optional<BigDecimal>>(); try { JsonNode response = client.execute(get, handler); for (JsonNode metric : response) { String target = metric.path("target").asText(); try { BigDecimal value = getLatestValue(metric); targetValues.put(target, Optional.of(value)); } catch (InvalidGraphiteValueException e) { // Silence these - we don't know what's causing Graphite to return null values LOGGER.warn(check.getName() + " failed to read from Graphite", e); targetValues.put(target, Optional.<BigDecimal>absent()); } } } catch (Exception e) { LOGGER.warn(check.getName() + " failed to read from Graphite", e); } finally { get.releaseConnection(); } return targetValues; }
public String getTravelAds() throws HooklogicException { String hlFullUrl = winningTravelAdsUrlPrefix + "/WinningTravelAds" + "?domain=www.hotels.com" + "&culture=en_US" + "¤cy=GBP" + "&destRegion=3D931ACE-E3FE-46B4-A243-61D44A22053B" + "&checkIn=2013-10-16" + "&checkOut=2013-10-17" + "&rooms=1" + "&adults=2" + "&children=0" + "&userAuthenticated=false" + "&userIP=127.0.0.1" + "&userAgent=Mozillav" + "&hlUserData=DX-Lqi9WiZlSeb80sMGK4uQ0z.SHA&maxRequested=15" + "&userGuid=365cd652-79e8-4970-b823-b6b87d5e93b4"; DefaultHttpClient httpclient = new DefaultHttpClient(); HttpGet httpGet = new HttpGet(hlFullUrl); String responseStr = ""; try { HttpResponse testResponse = httpclient.execute(httpGet); responseStr = EntityUtils.toString(testResponse.getEntity()); } catch (Exception e) { throw new HooklogicException(e); } finally { httpGet.releaseConnection(); } return responseStr; }
public static IMHttpResponse get(String url) { IMHttpResponse response = new IMHttpResponse(); response.setStatusCode(404); if (StringUtils.isEmpty(url)) { return response; } int statusCode = 404; CloseableHttpClient httpClient = null; HttpGet httpGet = null; try { httpClient = HttpClients.createDefault(); httpGet = new HttpGet(url); // HTTP Get请求(POST雷同) RequestConfig requestConfig = RequestConfig.custom() .setSocketTimeout(2000) .setConnectTimeout(2000) .build(); // 设置请求和传输超时时间 httpGet.setConfig(requestConfig); CloseableHttpResponse hresp = httpClient.execute(httpGet); // 执行请求 String responseString = EntityUtils.toString(hresp.getEntity()); response.setStatusCode(hresp.getStatusLine().getStatusCode()); response.setResponseBody(responseString); return response; } catch (Exception e) { logger.error("error code: " + statusCode, e); } finally { if (httpGet != null) { httpGet.releaseConnection(); } } return response; }
/** * Given a URL to post a GET request, this method compares the actual status code of the response * with an expected status code. * * @param msg Error message that should be displayed if the actual response does not match the * expected response * @param url URL to be used to execute the GET request * @param statusCode expected status code of response * @throws Exception */ protected void assertUrlResponse(String msg, String url, int statusCode) throws Exception { HttpGet get = obtainGetMethod(new URI(null, url, null).toString()); HttpResponse httpResponse = perform(get); EntityUtils.consume(httpResponse.getEntity()); get.releaseConnection(); int status = getStatusCode(httpResponse); assertEquals(msg, statusCode, status); }
/** * @param url * @param paramMap 不定长度 如果长度为2则head+params,否则param */ public static String returnGet(String url, JSONObject... paramList) { HttpClient httpclient = new DefaultHttpClient(); JSONObject headMap = null; JSONObject paramMap = null; String result = ""; try { HttpGet httpget = new HttpGet(url); // httpget=setBrowHead(httpget); if (paramList == null || paramList.length == 0) { } else if (paramList.length == 1) { paramMap = paramList[0]; } else if (paramList.length == 2) { headMap = paramList[0]; paramMap = paramList[1]; } if (headMap != null && headMap.size() > 0) { for (Entry entry : headMap.entrySet()) { httpget.setHeader(entry.getKey().toString(), entry.getValue().toString()); } } if (paramMap != null && paramMap.size() > 0) { List<NameValuePair> params = new ArrayList<NameValuePair>(); for (Entry entry : paramMap.entrySet()) { params.add( new BasicNameValuePair(entry.getKey().toString(), entry.getValue().toString())); } String str = EntityUtils.toString(new UrlEncodedFormEntity(params, "utf-8")); httpget.setURI(new URI(httpget.getURI().toString() + "?" + str)); } // 执行get请求. HttpResponse response = httpclient.execute(httpget); // 获取响应实体 HttpEntity entity = response.getEntity(); try { System.out.println("--------------------------------------"); // 打印响应状态 System.out.println(response.getStatusLine()); if (entity != null) { // 打印响应内容长度 System.out.println("Response content length: " + entity.getContentLength()); result = EntityUtils.toString(entity); // 打印响应内容 System.out.println("Response content: " + result); } System.out.println("------------------------------------"); } finally { httpget.releaseConnection(); } } catch (Exception e) { e.printStackTrace(); } return result; }
public String get(String url) { HttpGet get = new HttpGet(getProtocol() + getHost() + "/" + url); get.setHeader("Content-Type", "application/xml"); get.setHeader( "User-Agent", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_4; en-us) AppleWebKit/533.16 (KHTML, like Gecko) Version/5.0 Safari/533.16"); try { HttpResponse response = getClient().execute(get); verify(response.getStatusLine().getStatusCode()); return EntityUtils.toString(response.getEntity()); } catch (IOException e) { throw new RuntimeException(e); } finally { get.releaseConnection(); } }
private void assertResponse(String deploymentName) throws IOException, HttpException { String CONTEXT_URL = PortProviderUtil.generateURL("/", deploymentName); HttpGet httpget = new HttpGet(CONTEXT_URL + TestResource.TEST_PATH); HttpClientContext context = HttpClientContext.create(); CloseableHttpResponse response = client.execute(httpget, context); assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); HttpEntity entity = response.getEntity(); String responseString = EntityUtils.toString(entity, StandardCharsets.UTF_8); assertEquals( "The server resource didn't send correct response", TestResource.TEST_RESPONSE, responseString); httpget.releaseConnection(); }
public String getBody(String url) throws IOException { DefaultHttpClient httpclient = new DefaultHttpClient(); HttpGet httpGet = new HttpGet(url); try { HttpResponse response = httpclient.execute(httpGet); HttpEntity entity = response.getEntity(); final String encoding = entity.getContentEncoding() != null ? entity.getContentEncoding().getValue() : "UTF-8"; final String body = (readBody(entity, encoding)); EntityUtils.consume(entity); return body; } finally { httpGet.releaseConnection(); } }
/** * Get data via HTTP as an InputStream, following redirects, and supplying credentials if the host * is the Fedora server. */ public InputStream get(String url) throws IOException { HttpGet get = null; boolean ok = false; try { // don't bother with challenges DefaultHttpClient client = new PreemptiveAuth(m_cManager); client.getParams().setIntParameter(CoreConnectionPNames.CONNECTION_TIMEOUT, 20000); client.getCredentialsProvider().setCredentials(m_authScope, m_creds); int redirectCount = 0; // how many redirects did we follow int resultCode = 300; // not really, but enter the loop that way Dimension d = null; HttpResponse response = null; while (resultCode > 299 && resultCode < 400 && redirectCount < 25) { get = new HttpGet(url); client.getParams().setBooleanParameter(ClientPNames.HANDLE_REDIRECTS, true); if (Administrator.INSTANCE != null) { d = Administrator.PROGRESS.getSize(); // if they're using Administrator, tell them we're downloading... Administrator.PROGRESS.setString("Downloading " + url + " . . ."); Administrator.PROGRESS.setValue(100); Administrator.PROGRESS.paintImmediately( 0, 0, (int) d.getWidth() - 1, (int) d.getHeight() - 1); } response = client.execute(get); resultCode = response.getStatusLine().getStatusCode(); if (resultCode > 299 && resultCode < 400) { redirectCount++; url = response.getFirstHeader(HttpHeaders.LOCATION).getValue(); } } if (resultCode != 200) { System.err.println(EntityUtils.toString(response.getEntity())); throw new IOException( "Server returned error: " + resultCode + " " + response.getStatusLine().getReasonPhrase()); } ok = true; if (Administrator.INSTANCE != null) { // cache it to a file File tempFile = File.createTempFile("fedora-client-download-", null); tempFile.deleteOnExit(); HashMap PARMS = new HashMap(); PARMS.put("in", response.getEntity().getContent()); PARMS.put("out", new FileOutputStream(tempFile)); // do the actual download in a safe thread SwingWorker worker = new SwingWorker(PARMS) { @Override public Object construct() { try { StreamUtility.pipeStream( (InputStream) parms.get("in"), (OutputStream) parms.get("out"), 8192); } catch (Exception e) { thrownException = e; } return ""; } }; worker.start(); // The following code will run in the (safe) // Swing event dispatcher thread. int ms = 200; while (!worker.done) { try { Administrator.PROGRESS.setValue(ms); Administrator.PROGRESS.paintImmediately( 0, 0, (int) d.getWidth() - 1, (int) d.getHeight() - 1); Thread.sleep(100); ms = ms + 100; if (ms >= 2000) { ms = 200; } } catch (InterruptedException ie) { } } if (worker.thrownException != null) { throw worker.thrownException; } Administrator.PROGRESS.setValue(2000); Administrator.PROGRESS.paintImmediately( 0, 0, (int) d.getWidth() - 1, (int) d.getHeight() - 1); try { Thread.sleep(100); } catch (InterruptedException ie) { } ((InputStream) PARMS.get("in")).close(); return new FileInputStream(tempFile); } return response.getEntity().getContent(); } catch (Exception e) { throw new IOException(e.getMessage()); } finally { if (get != null && !ok) { get.releaseConnection(); } if (Administrator.INSTANCE != null) { Administrator.PROGRESS.setValue(0); Administrator.PROGRESS.setString(""); } } }
/** @see org.xml.sax.EntityResolver#resolveEntity(java.lang.String, java.lang.String) */ @Override public InputSource resolveEntity(String publicId, String systemId) throws SAXException, IOException { if (requestsLeft > -1) { if (requestsLeft == 0) { throw new IOException("Number of permitted HTTP requests exceeded."); } else { requestsLeft--; } } HttpGet m = null; try { URL url; try { url = URL.parse(systemId); } catch (GalimatiasParseException e) { IOException ioe = (IOException) new IOException(e.getMessage()).initCause(e); SAXParseException spe = new SAXParseException(e.getMessage(), publicId, systemId, -1, -1, ioe); if (errorHandler != null) { errorHandler.fatalError(spe); } throw spe; } String scheme = url.scheme(); if (!("http".equals(scheme) || "https".equals(scheme))) { String msg = "Unsupported URI scheme: \u201C" + scheme + "\u201D."; SAXParseException spe = new SAXParseException(msg, publicId, systemId, -1, -1, new IOException(msg)); if (errorHandler != null) { errorHandler.fatalError(spe); } throw spe; } systemId = url.toString(); try { m = new HttpGet(systemId); } catch (IllegalArgumentException e) { SAXParseException spe = new SAXParseException( e.getMessage(), publicId, systemId, -1, -1, (IOException) new IOException(e.getMessage()).initCause(e)); if (errorHandler != null) { errorHandler.fatalError(spe); } throw spe; } m.setHeader("User-Agent", userAgent); m.setHeader("Accept", buildAccept()); m.setHeader("Accept-Encoding", "gzip"); log4j.info(systemId); HttpResponse response = client.execute(m); int statusCode = response.getStatusLine().getStatusCode(); if (statusCode != 200) { String msg = "HTTP resource not retrievable. The HTTP status from the remote server was: " + statusCode + "."; SAXParseException spe = new SAXParseException( msg, publicId, m.getURI().toString(), -1, -1, new IOException(msg)); if (errorHandler != null) { errorHandler.fatalError(spe); } throw spe; } HttpEntity entity = response.getEntity(); long len = entity.getContentLength(); if (sizeLimit > -1 && len > sizeLimit) { SAXParseException spe = new SAXParseException( "Resource size exceeds limit.", publicId, m.getURI().toString(), -1, -1, new StreamBoundException("Resource size exceeds limit.")); if (errorHandler != null) { errorHandler.fatalError(spe); } throw spe; } TypedInputSource is; org.apache.http.Header ct = response.getFirstHeader("Content-Type"); String contentType = null; final String baseUri = m.getURI().toString(); if (ct != null) { contentType = ct.getValue(); } is = contentTypeParser.buildTypedInputSource(baseUri, publicId, contentType); Header cl = response.getFirstHeader("Content-Language"); if (cl != null) { is.setLanguage(cl.getValue().trim()); } Header xuac = response.getFirstHeader("X-UA-Compatible"); if (xuac != null) { String val = xuac.getValue().trim(); if (!"ie=edge".equalsIgnoreCase(val)) { SAXParseException spe = new SAXParseException( "X-UA-Compatible HTTP header must have the value \u201CIE=edge\u201D," + " was \u201C" + val + "\u201D.", publicId, systemId, -1, -1); errorHandler.error(spe); } } Header csp = response.getFirstHeader("Content-Security-Policy"); if (csp != null) { try { ContentSecurityPolicy.THE_INSTANCE.checkValid(csp.getValue().trim()); } catch (DatatypeException e) { SAXParseException spe = new SAXParseException( "Content-Security-Policy HTTP header: " + e.getMessage(), publicId, systemId, -1, -1); Html5DatatypeException ex5 = (Html5DatatypeException) e; if (ex5.isWarning()) { errorHandler.warning(spe); } else { errorHandler.error(spe); } } } final HttpGet meth = m; InputStream stream = entity.getContent(); if (sizeLimit > -1) { stream = new BoundedInputStream(stream, sizeLimit, baseUri); } Header ce = response.getFirstHeader("Content-Encoding"); if (ce != null) { String val = ce.getValue().trim(); if ("gzip".equalsIgnoreCase(val) || "x-gzip".equalsIgnoreCase(val)) { stream = new GZIPInputStream(stream); if (sizeLimit > -1) { stream = new BoundedInputStream(stream, sizeLimit, baseUri); } } } is.setByteStream( new ObservableInputStream( stream, new StreamObserver() { private final Logger log4j = Logger.getLogger("nu.validator.xml.PrudentEntityResolver.StreamObserver"); private boolean released = false; @Override public void closeCalled() { log4j.debug("closeCalled"); if (!released) { log4j.debug("closeCalled, not yet released"); released = true; try { meth.releaseConnection(); } catch (Exception e) { log4j.debug("closeCalled, releaseConnection", e); } } } @Override public void exceptionOccurred(Exception ex) throws IOException { if (!released) { released = true; try { meth.abort(); } catch (Exception e) { log4j.debug("exceptionOccurred, abort", e); } finally { try { meth.releaseConnection(); } catch (Exception e) { log4j.debug("exceptionOccurred, releaseConnection", e); } } } if (ex instanceof SystemIdIOException) { throw (SystemIdIOException) ex; } else if (ex instanceof IOException) { IOException ioe = (IOException) ex; throw new SystemIdIOException(baseUri, ioe.getMessage(), ioe); } else if (ex instanceof RuntimeException) { throw (RuntimeException) ex; } else { throw new RuntimeException("API contract violation. Wrong exception type.", ex); } } @Override public void finalizerCalled() { if (!released) { released = true; try { meth.abort(); } catch (Exception e) { log4j.debug("finalizerCalled, abort", e); } finally { try { meth.releaseConnection(); } catch (Exception e) { log4j.debug("finalizerCalled, releaseConnection", e); } } } } })); return is; } catch (IOException | RuntimeException | SAXException e) { if (m != null) { try { m.abort(); } catch (Exception ex) { log4j.debug("abort", ex); } finally { try { m.releaseConnection(); } catch (Exception ex) { log4j.debug("releaseConnection", ex); } } } throw e; } }
/** * Retrieve the data for a resource using the given http client and endpoint definition. The * service is supposed to manage the connection handling itself. See {@link AbstractHttpProvider} * for a generic implementation of this method. * * @param resource the resource to be retrieved * @param endpoint the endpoint definition * @return a completely specified client response, including expiry information and the set of * triples */ @Override public ClientResponse retrieveResource(String resource, LDClientService client, Endpoint endpoint) throws DataRetrievalException { try { String contentType; if (endpoint != null && endpoint.getContentTypes().size() > 0) { contentType = CollectionUtils.fold( endpoint.getContentTypes(), new CollectionUtils.StringSerializer<ContentType>() { @Override public String serialize(ContentType contentType) { return contentType.toString("q"); } }, ","); } else { contentType = CollectionUtils.fold(Arrays.asList(listMimeTypes()), ","); } long defaultExpires = client.getClientConfiguration().getDefaultExpiry(); if (endpoint != null && endpoint.getDefaultExpiry() != null) { defaultExpires = endpoint.getDefaultExpiry(); } final ResponseHandler handler = new ResponseHandler(resource, endpoint); // a queue for queuing the request URLs needed to build the query response Queue<String> requestUrls = new LinkedList<String>(); requestUrls.addAll(buildRequestUrl(resource, endpoint)); Set<String> visited = new HashSet<String>(); String requestUrl = requestUrls.poll(); while (requestUrl != null) { if (!visited.contains(requestUrl)) { HttpGet get = new HttpGet(requestUrl); try { get.setHeader("Accept", contentType); get.setHeader("Accept-Language", "*"); // PoolParty compatibility log.info( "retrieving resource data for {} from '{}' endpoint, request URI is <{}>", new Object[] {resource, getName(), get.getURI().toASCIIString()}); handler.requestUrl = requestUrl; List<String> additionalRequestUrls = client.getClient().execute(get, handler); requestUrls.addAll(additionalRequestUrls); visited.add(requestUrl); } finally { get.releaseConnection(); } } requestUrl = requestUrls.poll(); } Date expiresDate = handler.expiresDate; if (expiresDate == null) { expiresDate = new Date(System.currentTimeMillis() + defaultExpires * 1000); } long min_expires = System.currentTimeMillis() + client.getClientConfiguration().getMinimumExpiry() * 1000; if (expiresDate.getTime() < min_expires) { log.info( "expiry time returned by request lower than minimum expiration time; using minimum time instead"); expiresDate = new Date(min_expires); } if (log.isInfoEnabled()) { RepositoryConnection con = handler.triples.getConnection(); log.info( "retrieved {} triples for resource {}; expiry date: {}", new Object[] {con.size(), resource, expiresDate}); con.close(); } ClientResponse result = new ClientResponse(handler.httpStatus, handler.triples); result.setExpires(expiresDate); return result; } catch (RepositoryException e) { log.error("error while initialising Sesame repository; classpath problem?", e); throw new DataRetrievalException( "error while initialising Sesame repository; classpath problem?", e); } catch (ClientProtocolException e) { log.error( "HTTP client error while trying to retrieve resource {}: {}", resource, e.getMessage()); throw new DataRetrievalException( "I/O error while trying to retrieve resource " + resource, e); } catch (IOException e) { log.error("I/O error while trying to retrieve resource {}: {}", resource, e.getMessage()); throw new DataRetrievalException( "I/O error while trying to retrieve resource " + resource, e); } catch (RuntimeException ex) { log.error( "Unknown error while trying to retrieve resource {}: {}", resource, ex.getMessage()); throw new DataRetrievalException( "Unknown error while trying to retrieve resource " + resource, ex); } }