private HttpUriRequest newHcRequest(HttpRequest request) throws IOException {
URI uri = request.getUri().toJavaUri();
HttpUriRequest httpUriRequest = HttpMethod.valueOf(request.getMethod()).newMessage(uri);
for (Map.Entry<String, List<String>> entry : request.getHeaders().entrySet()) {
httpUriRequest.addHeader(entry.getKey(), StringUtils.join(entry.getValue(), ','));
}
return httpUriRequest;
}
/**
* @see
* org.apache.shindig.gadgets.http.BasicHttpFetcher#fetch(org.apache.shindig.gadgets.http.HttpRequest)
*/
@Override
public HttpResponse fetch(HttpRequest request) throws GadgetException {
// Add authentication information if necessary
if (request.getOAuthArguments() == null
&& request.getOAuth2Arguments() == null
&& isAuthEndpoint(request)) {
this.authenticationProvider.provideAuthentication(request);
}
return super.fetch(request);
}
@Override
public boolean equals(Object obj) {
if (obj == this) {
return true;
}
if (!(obj instanceof HttpRequest)) {
return false;
}
HttpRequest req = (HttpRequest) obj;
return super.equals(obj)
&& req.getCacheTtl() == getCacheTtl()
&& req.getIgnoreCache() == getIgnoreCache();
}
public boolean rewrite(HttpRequest request, HttpResponse resp, MutableContent content) {
// Content fetched through the proxy can stipulate that it must be sanitized.
if (request.isSanitizationRequested()) {
ContentRewriterFeature rewriterFeature =
rewriterFeatureFactory.createRewriteAllFeature(request.getCacheTtl());
if (StringUtils.isEmpty(request.getRewriteMimeType())) {
logger.log(
Level.WARNING, "Request to sanitize without content type for " + request.getUri());
content.setContent("");
return true;
} else if (request.getRewriteMimeType().equalsIgnoreCase("text/css")) {
return rewriteProxiedCss(request, resp, content, rewriterFeature);
} else if (request.getRewriteMimeType().toLowerCase().startsWith("image/")) {
return rewriteProxiedImage(request, resp, content);
} else {
logger.log(
Level.WARNING,
"Request to sanitize unknown content type "
+ request.getRewriteMimeType()
+ " for "
+ request.getUri());
content.setContent("");
return true;
}
} else {
// No Op
return false;
}
}
@Override
public void fetch(HttpServletRequest request, HttpServletResponse response)
throws IOException, GadgetException {
if (request.getHeader("If-Modified-Since") != null) {
response.setStatus(HttpServletResponse.SC_NOT_MODIFIED);
return;
}
String host = request.getHeader("Host");
if (!lockedDomainService.isSafeForOpenProxy(host)) {
// Force embedded images and the like to their own domain to avoid XSS
// in gadget domains.
String msg =
"Embed request for url "
+ getParameter(request, URL_PARAM, "")
+ " made to wrong domain "
+ host;
logger.info(msg);
throw new GadgetException(GadgetException.Code.INVALID_PARAMETER, msg);
}
HttpRequest rcr = buildHttpRequest(request);
HttpResponse results = fetcher.fetch(rcr);
if (contentRewriterRegistry != null) {
results = contentRewriterRegistry.rewriteHttpResponse(rcr, results);
}
setResponseHeaders(request, response, results);
for (Map.Entry<String, List<String>> entry : results.getHeaders().entrySet()) {
String name = entry.getKey();
if (!DISALLOWED_RESPONSE_HEADERS.contains(name.toLowerCase())) {
for (String value : entry.getValue()) {
response.addHeader(name, value);
}
}
}
if (rcr.getRewriteMimeType() != null) {
response.setContentType(rcr.getRewriteMimeType());
}
if (results.getHttpStatusCode() != HttpResponse.SC_OK) {
response.sendError(results.getHttpStatusCode());
}
IOUtils.copy(results.getResponse(), response.getOutputStream());
}
public static String getMimeType(HttpRequest request, HttpResponseBuilder original) {
String mimeType = request.getRewriteMimeType();
if (mimeType == null) {
mimeType = original.getHeader("Content-Type");
}
return mimeType != null ? mimeType.toLowerCase() : null;
}
private HttpClient newHttpClient(String sslProtocol, HttpRequest request) {
HttpParams params = new BasicHttpParams();
HttpConnectionParams.setConnectionTimeout(params, httpTimeoutsProvider.getConnectionTimeout());
HttpConnectionParams.setSoTimeout(params, httpTimeoutsProvider.getSocketTimeout());
// AG-1059: Need to follow redirects to ensure that if the host app is setup with Apache and
// mod_jk we can still
// fetch internal gadgets.
if (request.getFollowRedirects()) {
params.setBooleanParameter(ClientPNames.HANDLE_REDIRECTS, true);
params.setIntParameter(ClientPNames.MAX_REDIRECTS, 3);
}
params.setParameter(
ClientPNames.DEFAULT_HEADERS,
ImmutableSet.of(new BasicHeader("Accept-Encoding", "gzip, deflate")));
DefaultHttpClient client = new DefaultHttpClient(params);
// AG-1044: Need to use the JVM's default proxy configuration for requests.
ProxySelectorRoutePlanner routePlanner =
new ProxySelectorRoutePlanner(
client.getConnectionManager().getSchemeRegistry(), ProxySelector.getDefault());
client.setRoutePlanner(routePlanner);
client.addResponseInterceptor(new GzipDeflatingInterceptor());
client
.getConnectionManager()
.getSchemeRegistry()
.register(new Scheme("https", new CustomSSLSocketFactory(sslProtocol), 443));
return client;
}
/**
* @param request the outbound http request
* @return true if the request should be authenticated
*/
private boolean isAuthEndpoint(HttpRequest request) {
Uri uri = request.getUri();
String path = uri.getPath();
for (String authEndpoint : authEndpoints) {
if (path.startsWith(authEndpoint)) {
return true;
}
}
return false;
}
private HttpResponse handleEchoUrl(HttpRequest request) throws Exception {
String query = request.getUri().getQuery();
if (query.contains("add_oauth_token")) {
query = query + "&oauth_token=abc";
}
return new HttpResponseBuilder()
.setHttpStatusCode(HttpResponse.SC_OK)
.setResponseString(query)
.create();
}
/** Generate a remote content request based on the parameters sent from the client. */
private HttpRequest buildHttpRequest(HttpServletRequest request) throws GadgetException {
Uri url = validateUrl(request.getParameter(URL_PARAM));
HttpRequest req = new HttpRequest(url);
req.setContainer(getContainer(request));
if (request.getParameter(GADGET_PARAM) != null) {
req.setGadget(Uri.parse(request.getParameter(GADGET_PARAM)));
}
// Allow the rewriter to use an externally forced mime type. This is needed
// allows proper rewriting of <script src="x"/> where x is returned with
// a content type like text/html which unfortunately happens all too often
req.setRewriteMimeType(request.getParameter(REWRITE_MIME_TYPE_PARAM));
req.setIgnoreCache(getIgnoreCache(request));
// If the proxy request specifies a refresh param then we want to force the min TTL for
// the retrieved entry in the cache regardless of the headers on the content when it
// is fetched from the original source.
if (request.getParameter(REFRESH_PARAM) != null) {
try {
req.setCacheTtl(Integer.parseInt(request.getParameter(REFRESH_PARAM)));
} catch (NumberFormatException nfe) {
// Ignore
}
}
return req;
}
/**
* We don't actually rewrite the image we just ensure that it is in fact a valid and known image
* type.
*/
private boolean rewriteProxiedImage(
HttpRequest request, HttpResponse resp, MutableContent content) {
boolean imageIsSafe = false;
try {
String contentType = resp.getHeader("Content-Type");
if (contentType == null || contentType.toLowerCase().startsWith("image/")) {
// Unspecified or unknown image mime type.
try {
ImageFormat imageFormat =
Sanselan.guessFormat(
new ByteSourceInputStream(resp.getResponse(), request.getUri().getPath()));
if (imageFormat == ImageFormat.IMAGE_FORMAT_UNKNOWN) {
logger.log(
Level.INFO, "Unable to sanitize unknown image type " + request.getUri().toString());
return true;
}
imageIsSafe = true;
// Return false to indicate that no rewriting occurred
return false;
} catch (IOException ioe) {
throw new RuntimeException(ioe);
} catch (ImageReadException ire) {
// Unable to read the image so its not safe
logger.log(
Level.INFO,
"Unable to detect image type for "
+ request.getUri().toString()
+ " for sanitized content",
ire);
return true;
}
} else {
return true;
}
} finally {
if (!imageIsSafe) {
content.setContent("");
}
}
}
@Test
public void testFetch_withToken() throws Exception {
Uri uri = Uri.parse("http://host?p=1&st=sometoken");
// We should get the request untouched.
HttpRequest request = createMock(HttpRequest.class);
expect(request.getUri()).andReturn(uri);
replay(request);
HttpResponse response = new HttpResponse();
HttpFetcher fetcher = createMock(HttpFetcher.class);
expect(fetcher.fetch(request)).andReturn(response);
replay(fetcher);
FakeUserHttpFetcher fakeFetcher;
fakeFetcher = new FakeUserHttpFetcher(config, fetcher, crypter);
fakeFetcher.fetch(request);
verify(request);
verify(fetcher);
}
@Test
public void testFetch_noToken() throws Exception {
Uri uri = Uri.parse("http://host?p=1");
HttpRequest request = createMock(HttpRequest.class);
expect(request.getUri()).andReturn(uri);
expect(request.setUri(Uri.parse("http://host?p=1&st=default%3Anull"))).andReturn(request);
replay(request);
HttpResponse response = new HttpResponse();
HttpFetcher fetcher = createMock(HttpFetcher.class);
expect(fetcher.fetch(request)).andReturn(response);
replay(fetcher);
FakeUserHttpFetcher fakeFetcher;
fakeFetcher = new FakeUserHttpFetcher(config, fetcher, crypter);
fakeFetcher.fetch(request);
verify(request);
verify(fetcher);
}
public HttpResponse fetch(HttpRequest request) throws GadgetException {
if (returnNull) {
return null;
}
if (gadgetException != null) {
throw gadgetException;
}
if (runtimeException != null) {
throw runtimeException;
}
if (request.getFollowRedirects()) {
throw new RuntimeException("Not supposed to follow OAuth redirects");
}
String url = request.getUri().toString();
try {
if (url.startsWith(REQUEST_TOKEN_URL)) {
++requestTokenCount;
return handleRequestTokenUrl(request);
} else if (url.startsWith(ACCESS_TOKEN_URL)) {
++accessTokenCount;
return handleAccessTokenUrl(request);
} else if (url.startsWith(RESOURCE_URL)) {
++resourceAccessCount;
return handleResourceUrl(request);
} else if (url.startsWith(NOT_FOUND_URL)) {
return handleNotFoundUrl(request);
} else if (url.startsWith(ERROR_400)) {
return handleError400Url(request);
} else if (url.startsWith(ECHO_URL)) {
return handleEchoUrl(request);
}
} catch (Exception e) {
throw new RuntimeException("Problem with request for URL " + url, e);
}
throw new RuntimeException("Unexpected request for " + url);
}
/** {@inheritDoc} */
public HttpResponse fetch(HttpRequest request) {
if (!whitelist.allows(request.getUri().toJavaUri())) {
log.warn(
"A request to "
+ request.getUri()
+ " has been denied. To allow requests to this URL add the application URL to your whitelist (http://confluence.atlassian.com/x/KQfCDQ ).");
return new HttpResponseBuilder()
.setHttpStatusCode(HttpResponse.SC_FORBIDDEN)
.setHeader("Content-Type", "text/plain")
.setResponseString(
"Requests to "
+ request.getUri()
+ " are not allowed. See your administrator about configuring a whitelist entry for this destination (http://confluence.atlassian.com/x/KQfCDQ ).")
.create();
}
HttpCacheKey cacheKey = new HttpCacheKey(request);
HttpResponse response = cache.getResponse(cacheKey, request);
if (response != null) {
return response;
}
try {
HttpUriRequest hcRequest = newHcRequest(request);
if (request.getPostBodyLength() > 0) {
((HttpEntityEnclosingRequest) hcRequest)
.setEntity(new InputStreamEntity(request.getPostBody(), request.getPostBodyLength()));
}
org.apache.http.HttpResponse hcResponse;
try {
// first try with TLS, the most common SSL protocol
hcResponse = newHttpClient("TLSv1", request).execute(hcRequest);
} catch (SSLException e) {
log.debug(
"SSL Exception establishing connection with TLSv1 protocol. Falling back to SSLv3.", e);
// if TLS failed, try with SSLv3
hcResponse = newHttpClient("SSLv3", request).execute(hcRequest);
}
response = makeResponse(hcResponse);
return cache.addResponse(cacheKey, request, response);
} catch (IOException e) {
if (e instanceof java.net.SocketTimeoutException || e instanceof java.net.SocketException) {
return HttpResponse.timeout();
} else {
log.error("Unable to retrieve response", e);
}
return HttpResponse.error();
}
}
/** Sanitize a CSS file. */
private boolean rewriteProxiedCss(
HttpRequest request,
HttpResponse response,
MutableContent content,
ContentRewriterFeature rewriterFeature) {
String sanitized = "";
try {
String contentType = response.getHeader("Content-Type");
if (contentType == null || contentType.toLowerCase().startsWith("text/")) {
SanitizingProxyingLinkRewriter cssImportRewriter =
sanitizingProxyingLinkRewriterFactory.create(
request.getGadget(),
rewriterFeature,
request.getContainer(),
"text/css",
false,
request.getIgnoreCache());
SanitizingProxyingLinkRewriter cssImageRewriter =
sanitizingProxyingLinkRewriterFactory.create(
request.getGadget(),
rewriterFeature,
request.getContainer(),
"image/*",
false,
request.getIgnoreCache());
sanitized =
cssSanitizer.sanitize(
content.getContent(), request.getUri(), cssImportRewriter, cssImageRewriter);
}
return true;
} finally {
// Set sanitized content in finally to ensure it is always cleared in
// the case of errors
content.setContent(sanitized);
}
}
public HttpResponse fetch(org.apache.shindig.gadgets.http.HttpRequest request)
throws GadgetException {
HttpUriRequest httpMethod = null;
Preconditions.checkNotNull(request);
final String methodType = request.getMethod();
final org.apache.http.HttpResponse response;
final long started = System.currentTimeMillis();
// Break the request Uri to its components:
Uri uri = request.getUri();
if (StringUtils.isEmpty(uri.getAuthority())) {
throw new GadgetException(
GadgetException.Code.INVALID_USER_DATA,
"Missing domain name for request: " + uri,
HttpServletResponse.SC_BAD_REQUEST);
}
if (StringUtils.isEmpty(uri.getScheme())) {
throw new GadgetException(
GadgetException.Code.INVALID_USER_DATA,
"Missing schema for request: " + uri,
HttpServletResponse.SC_BAD_REQUEST);
}
String[] hostparts = StringUtils.splitPreserveAllTokens(uri.getAuthority(), ':');
int port = -1; // default port
if (hostparts.length > 2) {
throw new GadgetException(
GadgetException.Code.INVALID_USER_DATA,
"Bad host name in request: " + uri.getAuthority(),
HttpServletResponse.SC_BAD_REQUEST);
}
if (hostparts.length == 2) {
try {
port = Integer.parseInt(hostparts[1]);
} catch (NumberFormatException e) {
throw new GadgetException(
GadgetException.Code.INVALID_USER_DATA,
"Bad port number in request: " + uri.getAuthority(),
HttpServletResponse.SC_BAD_REQUEST);
}
}
String requestUri = uri.getPath();
// Treat path as / if set as null.
if (uri.getPath() == null) {
requestUri = "/";
}
if (uri.getQuery() != null) {
requestUri += '?' + uri.getQuery();
}
// Get the http host to connect to.
HttpHost host = new HttpHost(hostparts[0], port, uri.getScheme());
try {
if ("POST".equals(methodType) || "PUT".equals(methodType)) {
HttpEntityEnclosingRequestBase enclosingMethod =
("POST".equals(methodType)) ? new HttpPost(requestUri) : new HttpPut(requestUri);
if (request.getPostBodyLength() > 0) {
enclosingMethod.setEntity(
new InputStreamEntity(request.getPostBody(), request.getPostBodyLength()));
}
httpMethod = enclosingMethod;
} else if ("GET".equals(methodType)) {
httpMethod = new HttpGet(requestUri);
} else if ("HEAD".equals(methodType)) {
httpMethod = new HttpHead(requestUri);
} else if ("DELETE".equals(methodType)) {
httpMethod = new HttpDelete(requestUri);
}
for (Map.Entry<String, List<String>> entry : request.getHeaders().entrySet()) {
httpMethod.addHeader(entry.getKey(), StringUtils.join(entry.getValue(), ','));
}
// Disable following redirects.
if (!request.getFollowRedirects()) {
httpMethod.getParams().setBooleanParameter(ClientPNames.HANDLE_REDIRECTS, false);
}
// HttpClient doesn't handle all cases when breaking url (specifically '_' in domain)
// So lets pass it the url parsed:
response = FETCHER.execute(host, httpMethod);
if (response == null) {
throw new IOException("Unknown problem with request");
}
long now = System.currentTimeMillis();
if (now - started > slowResponseWarning) {
slowResponseWarning(request, started, now);
}
return makeResponse(response);
} catch (Exception e) {
long now = System.currentTimeMillis();
// Find timeout exceptions, respond accordingly
if (TIMEOUT_EXCEPTIONS.contains(e.getClass())) {
LOG.info(
"Timeout for "
+ request.getUri()
+ " Exception: "
+ e.getClass().getName()
+ " - "
+ e.getMessage()
+ " - "
+ (now - started)
+ "ms");
return HttpResponse.timeout();
}
LOG.log(
Level.INFO,
"Got Exception fetching " + request.getUri() + " - " + (now - started) + "ms",
e);
// Separate shindig error from external error
throw new GadgetException(
GadgetException.Code.INTERNAL_SERVER_ERROR,
e,
HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
} finally {
// cleanup any outstanding resources..
if (httpMethod != null)
try {
httpMethod.abort();
} catch (UnsupportedOperationException e) {
// ignore
}
}
}
public static boolean isHtml(HttpRequest request, HttpResponseBuilder original) {
String mimeType = getMimeType(request, original);
return mimeType != null
&& (mimeType.contains("html"))
&& maybeAcceptHtml(request.getParam(UriCommon.Param.HTML_TAG_CONTEXT.getKey()));
}
// Loosely based off net.oauth.OAuthServlet, and even more loosely related
// to the OAuth specification
private MessageInfo parseMessage(HttpRequest request) {
MessageInfo info = new MessageInfo();
info.request = request;
String method = request.getMethod();
ParsedUrl parsed = new ParsedUrl(request.getUri().toString());
List<OAuth.Parameter> params = Lists.newArrayList();
params.addAll(parsed.getParsedQuery());
if (!validParamLocations.contains(OAuthParamLocation.URI_QUERY)) {
// Make sure nothing OAuth related ended up in the query string
for (OAuth.Parameter p : params) {
if (p.getKey().contains("oauth_")) {
throw new RuntimeException("Found unexpected query param " + p.getKey());
}
}
}
// Parse authorization header
if (validParamLocations.contains(OAuthParamLocation.AUTH_HEADER)) {
String aznHeader = request.getHeader("Authorization");
if (aznHeader != null) {
info.aznHeader = aznHeader;
for (OAuth.Parameter p : OAuthMessage.decodeAuthorization(aznHeader)) {
if (!p.getKey().equalsIgnoreCase("realm")) {
params.add(p);
}
}
}
}
// Parse body
info.body = request.getPostBodyAsString();
try {
info.rawBody = IOUtils.toByteArray(request.getPostBody());
} catch (IOException e) {
throw new RuntimeException("Can't read post body bytes", e);
}
if (OAuth.isFormEncoded(request.getHeader("Content-Type"))) {
params.addAll(OAuth.decodeForm(request.getPostBodyAsString()));
// If we're not configured to pass oauth parameters in the post body, double check
// that they didn't end up there.
if (!validParamLocations.contains(OAuthParamLocation.POST_BODY)) {
if (info.body.contains("oauth_")) {
throw new RuntimeException("Found unexpected post body data" + info.body);
}
}
}
// Return the lot
info.message = new OAuthMessage(method, parsed.getLocation(), params);
// Check for trusted parameters
if (checkTrustedParams) {
if (!"foo".equals(OAuthUtil.getParameter(info.message, "oauth_magic"))) {
throw new RuntimeException("no oauth_trusted=foo parameter");
}
if (!"bar".equals(OAuthUtil.getParameter(info.message, "opensocial_magic"))) {
throw new RuntimeException("no opensocial_trusted=foo parameter");
}
if (!"quux".equals(OAuthUtil.getParameter(info.message, "xoauth_magic"))) {
throw new RuntimeException("no xoauth_magic=quux parameter");
}
trustedParamCount += 3;
}
return info;
}
private HttpResponse handleResourceUrl(HttpRequest request) throws Exception {
MessageInfo info = parseMessage(request);
String consumerId = info.message.getParameter("oauth_consumer_key");
OAuthConsumer consumer;
if (CONSUMER_KEY.equals(consumerId)) {
consumer = oauthConsumer;
} else if ("signedfetch".equals(consumerId)) {
consumer = signedFetchConsumer;
} else if ("container.com".equals(consumerId)) {
consumer = signedFetchConsumer;
} else {
return makeOAuthProblemReport(
OAuthConstants.PROBLEM_PARAMETER_MISSING,
"oauth_consumer_key not found",
HttpResponse.SC_BAD_REQUEST);
}
OAuthAccessor accessor = new OAuthAccessor(consumer);
String responseBody = null;
if (throttled) {
return makeOAuthProblemReport(
OAuthConstants.PROBLEM_CONSUMER_KEY_REFUSED, "exceeded quota", HttpResponse.SC_FORBIDDEN);
}
if (unauthorized) {
return makeOAuthProblemReport(
OAuthConstants.PROBLEM_PERMISSION_DENIED,
"user refused access",
HttpResponse.SC_UNAUTHORIZED);
}
if (consumer == oauthConsumer) {
// for OAuth, check the access token. We skip this for signed fetch
String accessToken = info.message.getParameter("oauth_token");
TokenState state = tokenState.get(accessToken);
if (state == null) {
return makeOAuthProblemReport(
OAuthConstants.PROBLEM_TOKEN_REJECTED,
"Access token unknown",
HttpResponse.SC_UNAUTHORIZED);
}
// Check the signature
accessor.accessToken = accessToken;
accessor.tokenSecret = state.getSecret();
validateMessage(accessor, info, false);
if (state.getState() != State.APPROVED) {
return makeOAuthProblemReport(
OAuthConstants.PROBLEM_TOKEN_REVOKED,
"User revoked permissions",
HttpResponse.SC_UNAUTHORIZED);
}
if (sessionExtension) {
long expiration = state.issued + TOKEN_EXPIRATION_SECONDS * 1000;
if (expiration < clock.currentTimeMillis()) {
return makeOAuthProblemReport(
OAuthConstants.PROBLEM_ACCESS_TOKEN_EXPIRED,
"token needs to be refreshed",
HttpResponse.SC_UNAUTHORIZED);
}
}
responseBody = "User data is " + state.getUserData();
} else {
// Check the signature
validateMessage(accessor, info, false);
// For signed fetch, just echo back the query parameters in the body
responseBody = request.getUri().getQuery();
}
// Send back a response
HttpResponseBuilder resp =
new HttpResponseBuilder()
.setHttpStatusCode(HttpResponse.SC_OK)
.setResponseString(responseBody);
if (info.aznHeader != null) {
resp.setHeader(AUTHZ_ECHO_HEADER, info.aznHeader);
}
if (info.body != null) {
resp.setHeader(BODY_ECHO_HEADER, info.body);
}
if (info.rawBody != null) {
resp.setHeader(RAW_BODY_ECHO_HEADER, new String(Base64.encodeBase64(info.rawBody)));
}
return resp.create();
}