public String getNormalizedServerUri() throws URISyntaxException {
URI uri = new URI(originalServerUri);
if (uri.getAuthority() == null) uri = new URI("ws://" + originalServerUri);
if (uri.getPort() == -1)
if ("ws".equals(uri.getScheme().toLowerCase(Locale.ENGLISH)))
uri =
new URI(
"ws",
uri.getUserInfo(),
uri.getHost(),
PluginConfig.DEFAULT_TCP_PORT,
uri.getPath(),
uri.getQuery(),
uri.getFragment());
else if ("wss".equals(uri.getScheme().toLowerCase(Locale.ENGLISH)))
uri =
new URI(
"wss",
uri.getUserInfo(),
uri.getHost(),
PluginConfig.DEFAULT_SSL_PORT,
uri.getPath(),
uri.getQuery(),
uri.getFragment());
return uri.toString();
}
@SqlNullable
@Description("extract query parameter from url")
@ScalarFunction
@LiteralParameters({"x", "y"})
@SqlType("varchar(x)")
public static Slice urlExtractParameter(
@SqlType("varchar(x)") Slice url, @SqlType("varchar(y)") Slice parameterName) {
URI uri = parseUrl(url);
if ((uri == null) || (uri.getQuery() == null)) {
return null;
}
Slice query = slice(uri.getQuery());
String parameter = parameterName.toStringUtf8();
Iterable<String> queryArgs = QUERY_SPLITTER.split(query.toStringUtf8());
for (String queryArg : queryArgs) {
Iterator<String> arg = ARG_SPLITTER.split(queryArg).iterator();
if (arg.next().equals(parameter)) {
if (arg.hasNext()) {
return utf8Slice(arg.next());
}
// first matched key is empty
return Slices.EMPTY_SLICE;
}
}
// no key matched
return null;
}
public NetServerSpecFactoryBean configure(URI uri) {
setHost(null != uri.getHost() ? uri.getHost() : "0.0.0.0");
setPort(uri.getPort() > 0 ? uri.getPort() : 3000);
setFraming(null != uri.getPath() ? uri.getPath().substring(1) : "linefeed");
this.delegateCodec = StandardCodecs.STRING_CODEC;
if (null != uri.getQuery()) {
String[] params = StringUtils.split(uri.getQuery(), "&");
if (null == params) {
params = new String[] {uri.getQuery()};
}
for (String pair : params) {
String[] parts = StringUtils.split(pair, "=");
if (parts.length > 1) {
if ("codec".equals(parts[0])) {
setCodec(parts[1]);
} else if ("dispatcher".equals(parts[0])) {
setDispatcher(parts[1]);
} else if ("lengthFieldLength".equals(parts[0])) {
setLengthFieldLength(Integer.parseInt(parts[1]));
}
}
}
}
return this;
}
public static String URLFromURI(URI uri) {
String ret = "";
if (uri == null) {
log.error("");
}
ret += uri.getScheme() + "://" + uri.getHost() + uri.getPath();
if (uri.getQuery() != null && uri.getQuery().length() > 0) ret += "?" + uri.getQuery();
return ret;
}
/**
* decode the raw URI to get the underlying URI
*
* @param rawURI raw Har URI
* @return filtered URI of the underlying fileSystem
*/
private URI decodeHarURI(URI rawURI, Configuration conf) throws IOException {
String tmpAuth = rawURI.getAuthority();
// we are using the default file
// system in the config
// so create a underlying uri and
// return it
if (tmpAuth == null) {
// create a path
return FileSystem.getDefaultUri(conf);
}
String authority = rawURI.getAuthority();
if (authority == null) {
throw new IOException(
"URI: "
+ rawURI
+ " is an invalid Har URI since authority==null."
+ " Expecting har://<scheme>-<host>/<path>.");
}
int i = authority.indexOf('-');
if (i < 0) {
throw new IOException(
"URI: "
+ rawURI
+ " is an invalid Har URI since '-' not found."
+ " Expecting har://<scheme>-<host>/<path>.");
}
if (rawURI.getQuery() != null) {
// query component not allowed
throw new IOException("query component in Path not supported " + rawURI);
}
URI tmp;
try {
// convert <scheme>-<host> to <scheme>://<host>
URI baseUri = new URI(authority.replaceFirst("-", "://"));
tmp =
new URI(
baseUri.getScheme(),
baseUri.getAuthority(),
rawURI.getPath(),
rawURI.getQuery(),
rawURI.getFragment());
} catch (URISyntaxException e) {
throw new IOException(
"URI: " + rawURI + " is an invalid Har URI. Expecting har://<scheme>-<host>/<path>.");
}
return tmp;
}
public AhcWSRequest(AhcWSClient client, String url, Materializer materializer) {
this.client = client;
URI reference = URI.create(url);
this.url = url;
this.materializer = materializer;
String userInfo = reference.getUserInfo();
if (userInfo != null) {
this.setAuth(userInfo);
}
if (reference.getQuery() != null) {
this.setQueryString(reference.getQuery());
}
}
private void testUri(String input) {
URI uri = new UriBuilderImpl().uri(input).clone().build();
URI originalUri = URI.create(input);
assertEquals(originalUri.getScheme(), uri.getScheme());
assertEquals(originalUri.getHost(), uri.getHost());
assertEquals(originalUri.getPort(), uri.getPort());
assertEquals(originalUri.getUserInfo(), uri.getUserInfo());
assertEquals(originalUri.getPath(), uri.getPath());
assertEquals(originalUri.getQuery(), uri.getQuery());
assertEquals(originalUri.getFragment(), uri.getFragment());
assertEquals(originalUri.getRawSchemeSpecificPart(), uri.getRawSchemeSpecificPart());
assertEquals(originalUri.isAbsolute(), uri.isAbsolute());
assertEquals(input, uri.toString());
}
private static String getPath(URI surl) {
String path = surl.getPath();
String query = surl.getQuery();
if (query != null) {
int i = query.indexOf(SFN_STRING);
if (i != -1) {
path = query.substring(i + SFN_STRING.length());
}
}
/* REVISIT
*
* This is not correct in the presence of symlinked directories. The
* simplified path may refer to a different directory than the one
* we will delete.
*
* For now we ignore this problem - fixing it requires resolving the
* paths to an absolute path, which requires additional name space
* lookups.
*/
path = Files.simplifyPath(path);
if (!path.endsWith("/")) {
path = path + "/";
}
return path;
}
protected void bind() throws IOException {
URI bind = getBindLocation();
String host = bind.getHost();
host = (host == null || host.length() == 0) ? "localhost" : host;
InetAddress addr = InetAddress.getByName(host);
try {
if (host.trim().equals("localhost") || addr.equals(InetAddress.getLocalHost())) {
this.serverSocket = serverSocketFactory.createServerSocket(bind.getPort(), backlog);
} else {
this.serverSocket = serverSocketFactory.createServerSocket(bind.getPort(), backlog, addr);
}
this.serverSocket.setSoTimeout(2000);
} catch (IOException e) {
throw IOExceptionSupport.create(
"Failed to bind to server socket: " + bind + " due to: " + e, e);
}
try {
setConnectURI(
new URI(
bind.getScheme(),
bind.getUserInfo(),
resolveHostName(bind.getHost()),
serverSocket.getLocalPort(),
bind.getPath(),
bind.getQuery(),
bind.getFragment()));
} catch (URISyntaxException e) {
throw IOExceptionSupport.create(e);
}
}
/**
* Creates helper for archive.
*
* @param archive absolute location of archive in underlying filesystem
* @param originalBase directory for which Hadoop archive was created
*/
public HarPathHelper(HiveConf hconf, URI archive, URI originalBase) throws HiveException {
this.originalBase = addSlash(originalBase);
String parentHost = archive.getHost();
String harHost = null;
if (parentHost == null) {
harHost = archive.getScheme() + "-localhost";
} else {
harHost = archive.getScheme() + "-" + parentHost;
}
// have to make sure there's slash after .har, otherwise resolve doesn't work
String path = addSlash(archive.getPath());
if (!path.endsWith(".har/")) {
throw new HiveException("HAR archive path must end with .har");
}
// harUri is used to access the partition's files, which are in the archive
// The format of the RI is something like:
// har://underlyingfsscheme-host:port/archivepath
try {
base =
new URI(
"har",
archive.getUserInfo(),
harHost,
archive.getPort(),
path,
archive.getQuery(),
archive.getFragment());
} catch (URISyntaxException e) {
throw new HiveException("Couldn't create har URI from archive URI", e);
}
}
public WebSocketSession(URI requestURI, EventDriver websocket, LogicalConnection connection) {
if (requestURI == null) {
throw new RuntimeException("Request URI cannot be null");
}
this.requestURI = requestURI;
this.websocket = websocket;
this.connection = connection;
this.outgoingHandler = connection;
this.incomingHandler = websocket;
// Get the parameter map (use the jetty MultiMap to do this right)
MultiMap<String> params = new MultiMap<>();
String query = requestURI.getQuery();
if (StringUtil.isNotBlank(query)) {
UrlEncoded.decodeTo(query, params, StringUtil.__UTF8);
}
for (String name : params.keySet()) {
List<String> valueList = params.getValues(name);
String valueArr[] = new String[valueList.size()];
valueArr = valueList.toArray(valueArr);
parameterMap.put(name, valueArr);
}
}
private static boolean repeatedSegementsInUrl(final String url) {
if (url.indexOf("?view=foaf?view=foaf") >= 0) return true;
final URI uri;
try {
uri = new URI(url);
} catch (final URISyntaxException ex) {
return false;
}
final String path = uri.getPath();
if (path == null) return false;
final Map<String, Integer> pathCounts = new HashMap<>();
for (final String c : path.split("/", -1)) {
final Integer countBoxed = pathCounts.get(c);
final int count = countBoxed == null ? 0 : countBoxed;
if (count >= 2) return true;
pathCounts.put(c, count + 1);
}
final String query = uri.getQuery();
if (query == null) return false;
final Map<String, Integer> queryCounts = new HashMap<>();
for (final String c : query.split("&", -1)) {
final Integer countBoxed = queryCounts.get(c);
final int count = countBoxed == null ? 0 : countBoxed;
if (count >= 1) return true;
queryCounts.put(c, count + 1);
}
return false;
}
@Override
@Property(MessageContext.QUERY_STRING)
public String getQueryString() {
URI requestUri = httpExchange.getRequestURI();
String query = requestUri.getQuery();
if (query != null) return query;
return null;
}
@SqlNullable
@Description("extract query from url")
@ScalarFunction
@LiteralParameters("x")
@SqlType("varchar(x)")
public static Slice urlExtractQuery(@SqlType("varchar(x)") Slice url) {
URI uri = parseUrl(url);
return (uri == null) ? null : slice(uri.getQuery());
}
/**
* Generate a resource uri based off of the specified parameters.
*
* @param path path
* @return resource uri
*/
protected String generateResourceUri(final String... path) {
final UriBuilder uriBuilder = uriInfo.getBaseUriBuilder();
uriBuilder.segment(path);
URI uri = uriBuilder.build();
try {
// check for proxy settings
final String scheme = httpServletRequest.getHeader(PROXY_SCHEME_HTTP_HEADER);
final String host = httpServletRequest.getHeader(PROXY_HOST_HTTP_HEADER);
final String port = httpServletRequest.getHeader(PROXY_PORT_HTTP_HEADER);
String baseContextPath = httpServletRequest.getHeader(PROXY_CONTEXT_PATH_HTTP_HEADER);
// if necessary, prepend the context path
String resourcePath = uri.getPath();
if (baseContextPath != null) {
// normalize context path
if (!baseContextPath.startsWith("/")) {
baseContextPath = "/" + baseContextPath;
}
// determine the complete resource path
resourcePath = baseContextPath + resourcePath;
}
// determine the port uri
int uriPort = uri.getPort();
if (port != null) {
if (StringUtils.isWhitespace(port)) {
uriPort = -1;
} else {
try {
uriPort = Integer.parseInt(port);
} catch (final NumberFormatException nfe) {
logger.warn(
String.format(
"Unable to parse proxy port HTTP header '%s'. Using port from request URI '%s'.",
port, uriPort));
}
}
}
// construct the URI
uri =
new URI(
(StringUtils.isBlank(scheme)) ? uri.getScheme() : scheme,
uri.getUserInfo(),
(StringUtils.isBlank(host)) ? uri.getHost() : host,
uriPort,
resourcePath,
uri.getQuery(),
uri.getFragment());
} catch (final URISyntaxException use) {
throw new UriBuilderException(use);
}
return uri.toString();
}
public URI toURI() {
try {
URI base = getBase();
return new URI(
base.getScheme(), null, base.getHost(), 0, getPath().toString(), base.getQuery(), null);
} catch (URISyntaxException e) {
throw new RuntimeException(e);
}
}
@Test
public void getURI() throws Exception {
URI uri = new URI("http://example.com/path?query");
mockRequest.setServerName(uri.getHost());
mockRequest.setServerPort(uri.getPort());
mockRequest.setRequestURI(uri.getPath());
mockRequest.setQueryString(uri.getQuery());
assertEquals("Invalid uri", uri, request.getURI());
}
private static String ensureNonemptyUrlPath(String url) {
if (url == null) return null;
final URI uri = URI.create(url);
if (uri.getPath().isEmpty()) {
url = uri.getScheme() + "://" + uri.getRawAuthority() + '/';
if (uri.getRawQuery() != null) url += '?' + uri.getQuery();
}
return url;
}
/**
* Since a URI query can contain a single key multiple times, the responding parameter map will
* contain string values when a key appears only once, and string arrays when they key appears
* multiple times.
*
* @return a map representation of the query portion of the URI.
*/
public static Hashtable<String, Object> getParameters(URI uri) {
try {
return parseQuery(uri.getQuery(), true);
} catch (URISyntaxException e) {
// should never get here as URI query will already be validated.
e.printStackTrace();
throw new IllegalArgumentException(e.getMessage());
}
}
/**
* Return a proper URL with lowercase scheme, lowercase domain, stripped hash - used to compare
* against other previously seen URLs
*
* @param href A relative or absolute URL
* @param context Null or an absolute URL to use when href is relative
* @return
*/
public static URL getCanonicalURL(String href, String context) {
try {
URI normalized;
if (context != null) {
normalized = new URI(context);
if (normalized.getPath().equals("")) context += "/";
normalized = new URI(context).resolve(href);
} else {
normalized = new URI(href);
}
normalized = normalized.normalize();
return new URI(
normalized.getScheme().toLowerCase(),
normalized.getUserInfo(),
normalized.getHost(),
normalized.getPort(),
normalized.getPath().equals("") ? "/" : normalized.getPath(),
normalized.getQuery(),
null)
.toURL();
} catch (URISyntaxException e) {
LogFactory.getLog(URLCanonicalizer.class)
.error("Unable to canonicalize href: " + href + ", context" + context, e);
return null;
} catch (MalformedURLException e) {
LogFactory.getLog(URLCanonicalizer.class)
.error("Unable to canonicalize href: " + href + ", context" + context, e);
return null;
} catch (IllegalArgumentException e) {
LogFactory.getLog(URLCanonicalizer.class)
.error("Unable to canonicalize href: " + href + ", context" + context, e);
return null;
}
/*if (href.contains("#")) {
href = href.substring(0, href.indexOf("#"));
}
href = href.replace(" ", "%20");
try {
URL canonicalURL;
if (context == null) {
canonicalURL = new URL(href);
} else {
canonicalURL = new URL(new URL(context), href);
}
String path = canonicalURL.getPath();
if (path.startsWith("/../")) {
path = path.substring(3);
canonicalURL = new URL(canonicalURL.getProtocol(), canonicalURL.getHost(), canonicalURL.getPort(), path);
} else if (path.contains("..")) {
System.out.println("Found path with ..: " + path + " " + href + " " + context);
}
return canonicalURL;
} catch (MalformedURLException ex) {
return null;
}*/
}
/**
* Parse the URL parameters into a map of (key, value) pairs.
*
* @param uri The URL that was requested.
* @return A map of (key, value) pairs corresponding to the request parameters.
* @throws UnsupportedEncodingException Thrown if we could not decode the URL with utf8.
*/
private static Map<String, String> getURLParams(URI uri) throws UnsupportedEncodingException {
if (uri.getQuery() != null) {
Map<String, String> urlParams = new HashMap<>();
String query = uri.getQuery();
String[] queryFields = query.replace("\\&", "___AMP___").split("&");
for (String queryField : queryFields) {
queryField = queryField.replace("___AMP___", "&");
int firstEq = queryField.indexOf('=');
// Convention uses "+" for spaces.
String key = URLDecoder.decode(queryField.substring(0, firstEq), "utf8");
String value = URLDecoder.decode(queryField.substring(firstEq + 1), "utf8");
urlParams.put(key, value);
}
return urlParams;
} else {
return Collections.emptyMap();
}
}
public String getPath(URI uri) {
String path = uri.getPath();
String query = uri.getQuery();
if (query != null) {
int i = query.indexOf(SFN_STRING);
if (i != -1) {
path = query.substring(i + SFN_STRING.length()).replaceAll("//*", "/");
}
}
return path;
}
public static String normalizeIri(final String url) throws URISyntaxException {
final URI uri = new URI(url);
final String scheme = uri.getScheme().toLowerCase();
if (!"http".equals(scheme) && !"https".equals(scheme))
throw new URISyntaxException(url, "scheme");
String path = uri.getPath();
if (path == null || path.isEmpty()) path = "/";
String auth = uri.getAuthority();
while (auth.endsWith(".")) auth = auth.substring(0, auth.length() - 1);
return new URI(scheme, auth, path, uri.getQuery(), null).toString();
}
@Test
public void testGetUri() throws Exception {
StringBuilder queryString = buildBaseQueryString();
queryString.append("&itineraryId=1234&[email protected]");
final URI uri = itineraryRequest.getUri();
assertEquals("http", uri.getScheme());
assertEquals("api.ean.com", uri.getHost());
assertEquals("/ean-services/rs/hotel/v3/itin", uri.getPath());
assertEquals(queryString.toString(), uri.getQuery());
}
/**
* Update the port of this MutableUri.
*
* @param port new port number
* @throws URISyntaxException if the new equivalent URI is invalid
*/
public void setPort(final int port) throws URISyntaxException {
uri =
new URI(
uri.getScheme(),
uri.getUserInfo(),
uri.getHost(),
port,
uri.getPath(),
uri.getQuery(),
uri.getFragment());
}
/**
* Update the fragment (not encoded) of this MutableUri.
*
* @param fragment new fragment element (not encoded)
* @throws URISyntaxException if the new equivalent URI is invalid
*/
public void setFragment(final String fragment) throws URISyntaxException {
this.uri =
new URI(
uri.getScheme(),
uri.getUserInfo(),
uri.getHost(),
uri.getPort(),
uri.getPath(),
uri.getQuery(),
fragment);
}
/** Lists documents contained beneath the {@code options.input} prefix/directory. */
public static Set<URI> listInputDocuments(Options options)
throws URISyntaxException, IOException {
URI baseUri = new URI(options.getInput());
// List all documents in the directory or GCS prefix.
URI absoluteUri;
if (baseUri.getScheme() != null) {
absoluteUri = baseUri;
} else {
absoluteUri =
new URI(
"file",
baseUri.getAuthority(),
baseUri.getPath(),
baseUri.getQuery(),
baseUri.getFragment());
}
Set<URI> uris = new HashSet<>();
if (absoluteUri.getScheme().equals("file")) {
File directory = new File(absoluteUri);
for (String entry : directory.list()) {
File path = new File(directory, entry);
uris.add(path.toURI());
}
} else if (absoluteUri.getScheme().equals("gs")) {
GcsUtil gcsUtil = options.as(GcsOptions.class).getGcsUtil();
URI gcsUriGlob =
new URI(
absoluteUri.getScheme(),
absoluteUri.getAuthority(),
absoluteUri.getPath() + "*",
absoluteUri.getQuery(),
absoluteUri.getFragment());
for (GcsPath entry : gcsUtil.expand(GcsPath.fromUri(gcsUriGlob))) {
uris.add(entry.toUri());
}
}
return uris;
}
/**
* Makes sure, that URI points to directory by adding slash to it. Useful in relativizing URIs.
*/
public static URI addSlash(URI u) throws HiveException {
if (u.getPath().endsWith("/")) {
return u;
} else {
try {
return new URI(
u.getScheme(), u.getAuthority(), u.getPath() + "/", u.getQuery(), u.getFragment());
} catch (URISyntaxException e) {
throw new HiveException("Couldn't append slash to a URI", e);
}
}
}
private String getViewerId(UriInfo uriInfo) {
URI uri = uriInfo.getRequestUri();
String requestString = uri.getQuery();
if (requestString == null) return null;
String[] queryParts = requestString.split("&");
for (String queryPart : queryParts) {
if (queryPart.startsWith("opensocial_viewer_id")) {
return queryPart.substring(queryPart.indexOf("=") + 1, queryPart.length());
}
}
return null;
}
public void testToUri() {
if (!SystemInfo.isWindows) {
assertEquals("file:///asd", VfsUtil.toUri(new File("/asd")).toASCIIString());
assertEquals("file:///asd%20/sd", VfsUtil.toUri(new File("/asd /sd")).toASCIIString());
}
URI uri = VfsUtil.toUri("file:///asd");
assertNotNull(uri);
assertEquals("file", uri.getScheme());
assertEquals("/asd", uri.getPath());
uri = VfsUtil.toUri("file:///asd/ ads/ad#test");
assertNotNull(uri);
assertEquals("file", uri.getScheme());
assertEquals("/asd/ ads/ad", uri.getPath());
assertEquals("test", uri.getFragment());
uri = VfsUtil.toUri("file:///asd/ ads/ad#");
assertNotNull(uri);
assertEquals("file:///asd/%20ads/ad#", uri.toString());
uri = VfsUtil.toUri("mailto:[email protected]");
assertNotNull(uri);
assertEquals("*****@*****.**", uri.getSchemeSpecificPart());
if (SystemInfo.isWindows) {
uri = VfsUtil.toUri("file://C:/p");
assertNotNull(uri);
assertEquals("file", uri.getScheme());
assertEquals("/C:/p", uri.getPath());
}
uri = VfsUtil.toUri("file:///Users/S pace");
assertNotNull(uri);
assertEquals("file", uri.getScheme());
assertEquals("/Users/S pace", uri.getPath());
assertEquals("/Users/S%20pace", uri.getRawPath());
assertEquals("file:///Users/S%20pace", uri.toString());
uri = VfsUtil.toUri("http://developer.android.com/guide/developing/tools/avd.html");
assertNotNull(uri);
assertEquals("http", uri.getScheme());
assertEquals("/guide/developing/tools/avd.html", uri.getRawPath());
assertEquals("http://developer.android.com/guide/developing/tools/avd.html", uri.toString());
uri = VfsUtil.toUri("http://developer.android.com/guide/developing/tools/avd.html?f=23r2ewd");
assertNotNull(uri);
assertEquals("http", uri.getScheme());
assertEquals("/guide/developing/tools/avd.html", uri.getRawPath());
assertEquals(
"http://developer.android.com/guide/developing/tools/avd.html?f=23r2ewd", uri.toString());
assertEquals("f=23r2ewd", uri.getQuery());
}