/** Illegal characters unescaped in the string, for glob processing, etc. */
@Override
public String toString() {
StringBuffer sb = new StringBuffer();
if (mUri.getScheme() != null) {
sb.append(mUri.getScheme());
sb.append(":");
}
if (mUri.getAuthority() != null) {
sb.append("//");
sb.append(mUri.getAuthority());
}
if (mUri.getPath() != null) {
String path = mUri.getPath();
if (path.indexOf('/') == 0
&& hasWindowsDrive(path, true)
&& // has windows drive
mUri.getScheme() == null
&& // but no scheme
mUri.getAuthority() == null) { // or authority
path = path.substring(1); // remove slash before drive
}
sb.append(path);
}
return sb.toString();
}
/**
* Resolve a child path against a parent path.
*
* @param parent the parent path
* @param child the child path
*/
public Path(Path parent, Path child) {
// Add a slash to parent's path so resolution is compatible with URI's
URI parentUri = parent.uri;
final String parentPath = parentUri.getPath();
if (!(parentPath.equals("/") || parentPath.equals(""))) {
try {
parentUri =
new URI(
parentUri.getScheme(),
parentUri.getAuthority(),
parentUri.getPath() + "/",
null,
null);
} catch (URISyntaxException e) {
throw new IllegalArgumentException(e);
}
}
if (child.uri.getPath().startsWith(Path.SEPARATOR)) {
child =
new Path(
child.uri.getScheme(), child.uri.getAuthority(), child.uri.getPath().substring(1));
}
final URI resolved = parentUri.resolve(child.uri);
initialize(resolved.getScheme(), resolved.getAuthority(), normalizePath(resolved.getPath()));
}
/** Returns a qualified path object. */
@InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"})
public Path makeQualified(URI defaultUri, Path workingDir) {
Path path = this;
if (!isAbsolute()) {
path = new Path(workingDir, this);
}
URI pathUri = path.toUri();
String scheme = pathUri.getScheme();
String authority = pathUri.getAuthority();
String fragment = pathUri.getFragment();
if (scheme != null && (authority != null || defaultUri.getAuthority() == null)) return path;
if (scheme == null) {
scheme = defaultUri.getScheme();
}
if (authority == null) {
authority = defaultUri.getAuthority();
if (authority == null) {
authority = "";
}
}
URI newUri = null;
try {
newUri = new URI(scheme, authority, normalizePath(pathUri.getPath()), null, fragment);
} catch (URISyntaxException e) {
throw new IllegalArgumentException(e);
}
return new Path(newUri);
}
/**
* Returns a qualified path object.
*
* @param fs the FileSystem that should be used to obtain the current working directory
* @return the qualified path object
*/
public Path makeQualified(FileSystem fs) {
Path path = this;
if (!isAbsolute()) {
path = new Path(fs.getWorkingDirectory(), this);
}
final URI pathUri = path.toUri();
final URI fsUri = fs.getUri();
String scheme = pathUri.getScheme();
String authority = pathUri.getAuthority();
if (scheme != null && (authority != null || fsUri.getAuthority() == null)) {
return path;
}
if (scheme == null) {
scheme = fsUri.getScheme();
}
if (authority == null) {
authority = fsUri.getAuthority();
if (authority == null) {
authority = "";
}
}
return new Path(scheme + ":" + "//" + authority + pathUri.getPath());
}
public String toString() {
// we can't use uri.toString(), which escapes everything, because we want
// illegal characters unescaped in the string, for glob processing, etc.
StringBuilder buffer = new StringBuilder();
if (uri.getScheme() != null) {
buffer.append(uri.getScheme());
buffer.append(":");
}
if (uri.getAuthority() != null) {
buffer.append("//");
buffer.append(uri.getAuthority());
}
if (uri.getPath() != null) {
String path = uri.getPath();
if (path.indexOf('/') == 0
&& hasWindowsDrive(path, true)
&& // has windows drive
uri.getScheme() == null
&& // but no scheme
uri.getAuthority() == null) // or authority
path = path.substring(1); // remove slash before drive
buffer.append(path);
}
if (uri.getFragment() != null) {
buffer.append("#");
buffer.append(uri.getFragment());
}
return buffer.toString();
}
private String getMetastoreConnectURI(URI uri) {
String metastoreURI;
// For unit tests
if (uri.getAuthority().equals("unittest-local")) {
metastoreURI = "";
} else {
// Hardcoding hcat to thrift mapping till support for webhcat(templeton)
// is added
metastoreURI = "thrift://" + uri.getAuthority();
}
return metastoreURI;
}
/**
* decode the raw URI to get the underlying URI
*
* @param rawURI raw Har URI
* @return filtered URI of the underlying fileSystem
*/
private URI decodeHarURI(URI rawURI, Configuration conf) throws IOException {
String tmpAuth = rawURI.getAuthority();
// we are using the default file
// system in the config
// so create a underlying uri and
// return it
if (tmpAuth == null) {
// create a path
return FileSystem.getDefaultUri(conf);
}
String authority = rawURI.getAuthority();
if (authority == null) {
throw new IOException(
"URI: "
+ rawURI
+ " is an invalid Har URI since authority==null."
+ " Expecting har://<scheme>-<host>/<path>.");
}
int i = authority.indexOf('-');
if (i < 0) {
throw new IOException(
"URI: "
+ rawURI
+ " is an invalid Har URI since '-' not found."
+ " Expecting har://<scheme>-<host>/<path>.");
}
if (rawURI.getQuery() != null) {
// query component not allowed
throw new IOException("query component in Path not supported " + rawURI);
}
URI tmp;
try {
// convert <scheme>-<host> to <scheme>://<host>
URI baseUri = new URI(authority.replaceFirst("-", "://"));
tmp =
new URI(
baseUri.getScheme(),
baseUri.getAuthority(),
rawURI.getPath(),
rawURI.getQuery(),
rawURI.getFragment());
} catch (URISyntaxException e) {
throw new IOException(
"URI: " + rawURI + " is an invalid Har URI. Expecting har://<scheme>-<host>/<path>.");
}
return tmp;
}
private void applyConstraints(URI fromURI, URI toURI, Tree ast, boolean isLocal)
throws SemanticException {
// local mode implies that scheme should be "file"
// we can change this going forward
if (isLocal && !fromURI.getScheme().equals("file")) {
throw new SemanticException(
ErrorMsg.ILLEGAL_PATH.getMsg(
ast, "Source file system should be \"file\" if \"local\" is specified"));
}
try {
FileStatus[] srcs =
matchFilesOrDir(
FileSystem.get(fromURI, conf),
new Path(fromURI.getScheme(), fromURI.getAuthority(), fromURI.getPath()));
if (srcs == null || srcs.length == 0) {
throw new SemanticException(
ErrorMsg.INVALID_PATH.getMsg(ast, "No files matching path " + fromURI));
}
for (FileStatus oneSrc : srcs) {
if (oneSrc.isDir()) {
throw new SemanticException(
ErrorMsg.INVALID_PATH.getMsg(
ast, "source contains directory: " + oneSrc.getPath().toString()));
}
}
} catch (IOException e) {
// Has to use full name to make sure it does not conflict with
// org.apache.commons.lang.StringUtils
throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(ast), e);
}
// only in 'local' mode do we copy stuff from one place to another.
// reject different scheme/authority in other cases.
if (!isLocal
&& (!StringUtils.equals(fromURI.getScheme(), toURI.getScheme())
|| !StringUtils.equals(fromURI.getAuthority(), toURI.getAuthority()))) {
String reason =
"Move from: "
+ fromURI.toString()
+ " to: "
+ toURI.toString()
+ " is not valid. "
+ "Please check that values for params \"default.fs.name\" and "
+ "\"hive.metastore.warehouse.dir\" do not conflict.";
throw new SemanticException(ErrorMsg.ILLEGAL_PATH.getMsg(ast, reason));
}
}
@Override
public Connection connect(URI uri, Properties properties) throws ConnectionException {
if (PASSTHROUGH.equals(uri.getScheme())) {
if (STRIPES.containsKey(uri.getAuthority())) {
String serverName = uri.getHost();
PassthroughServer server =
PassthroughServerRegistry.getSharedInstance().getServerForName(serverName);
if (null != server) {
String connectionName = properties.getProperty("connection.name");
if (null == connectionName) {
connectionName = "Ehcache:ACTIVE-PASSIVE";
}
Connection connection = server.connectNewClient(connectionName);
STRIPES.get(uri.getAuthority()).add(connection);
return connection;
}
} else {
throw new IllegalArgumentException(
"UnitTestConnectionService failed to find stripe" + uri.getAuthority());
}
}
checkURI(uri);
ServerDescriptor serverDescriptor = SERVERS.get(uri);
if (serverDescriptor == null) {
throw new IllegalArgumentException("No server available for " + uri);
}
String name = properties.getProperty(ConnectionPropertyNames.CONNECTION_NAME);
if (name == null) {
name = "Ehcache:UNKNOWN";
}
Connection connection = serverDescriptor.server.connectNewClient(name);
serverDescriptor.add(connection, properties);
LOGGER.info("Client opened {} to PassthroughServer at {}", formatConnectionId(connection), uri);
/*
* Uses a Proxy around Connection so closed connections can be removed from the ServerDescriptor.
*/
return (Connection)
Proxy.newProxyInstance(
Connection.class.getClassLoader(),
new Class[] {Connection.class},
new ConnectionInvocationHandler(serverDescriptor, connection));
}
private URI buildURI(String url) {
URI uri = URI.create(url);
if (uri.getRawPath() == null) {
// AHC-96
// Let's try to derive it
StringBuilder buildedUrl = new StringBuilder();
if (uri.getScheme() != null) {
buildedUrl.append(uri.getScheme());
buildedUrl.append("://");
}
if (uri.getAuthority() != null) {
buildedUrl.append(uri.getAuthority());
}
if (url.indexOf("://") == -1) {
String s = buildedUrl.toString();
url = s + url.substring(uri.getScheme().length() + 1);
return buildURI(url);
} else {
throw new IllegalArgumentException("Invalid url " + uri.toString());
}
}
if (isNonEmpty(uri.getRawQuery())) {
String[] queries = uri.getRawQuery().split("&");
int pos;
for (String query : queries) {
pos = query.indexOf('=');
if (pos <= 0) {
addQueryParameter(query, null);
} else {
try {
if (useRawUrl) {
addQueryParameter(query.substring(0, pos), query.substring(pos + 1));
} else {
addQueryParameter(
URLDecoder.decode(query.substring(0, pos), "UTF-8"),
URLDecoder.decode(query.substring(pos + 1), "UTF-8"));
}
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
}
}
}
return uri;
}
/**
* Resolve a child TachyonURI against a parent TachyonURI.
*
* @param parent the parent
* @param child the child
*/
public TachyonURI(TachyonURI parent, TachyonURI child) {
// Add a slash to parent's path so resolution is compatible with URI's
URI parentUri = parent.mUri;
String parentPath = parentUri.getPath();
if (!parentPath.endsWith(SEPARATOR) && parentPath.length() > 0) {
parentPath += SEPARATOR;
}
try {
parentUri = new URI(parentUri.getScheme(), parentUri.getAuthority(), parentPath, null, null);
} catch (URISyntaxException e) {
throw new IllegalArgumentException(e);
}
URI resolved = parentUri.resolve(child.mUri);
mUri = createURI(resolved.getScheme(), resolved.getAuthority(), resolved.getPath());
}
@Override
public DependencyType getDependencyType(URI uri) throws URIHandlerException {
DependencyType depType = DependencyType.PULL;
// Not initializing in constructor as this will be part of oozie.services.ext
// and will be initialized after URIHandlerService
HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
if (hcatService != null) {
depType = dependencyTypes.get(uri.getAuthority());
if (depType == null) {
depType = hcatService.isKnownPublisher(uri) ? DependencyType.PUSH : DependencyType.PULL;
dependencyTypes.put(uri.getAuthority(), depType);
}
}
return depType;
}
private String encodeGroup(URI uri) {
String path = uri.getPath();
if (path.length() != 0) {
return "JMS/message containers/" + path.substring(1);
}
return "JMS/message containers/" + uri.getAuthority();
}
protected FileSystem getAppFileSystem(WorkflowJob workflow)
throws HadoopAccessorException, IOException, URISyntaxException {
URI uri = new URI(workflow.getAppPath());
HadoopAccessorService has = Services.get().get(HadoopAccessorService.class);
Configuration fsConf = has.createJobConf(uri.getAuthority());
return has.createFileSystem(workflow.getUser(), uri, fsConf);
}
/**
* Returns longest common path for the given list of files.
*
* @param files list of files.
* @return longest common path
*/
private static String commonPathForFiles(final List<String> files) {
if (files == null || files.isEmpty()) {
return "";
}
final int total = files.size();
final String[][] folders = new String[total][];
int shortest = Integer.MAX_VALUE;
for (int i = 0; i < total; i++) {
final Path path = new Path(files.get(i));
folders[i] = Path.getPathWithoutSchemeAndAuthority(path).toString().split(PATH_SEPARATOR);
shortest = Math.min(shortest, folders[i].length);
}
int latest;
out:
for (latest = 0; latest < shortest; latest++) {
final String current = folders[0][latest];
for (int i = 1; i < folders.length; i++) {
if (!current.equals(folders[i][latest])) {
break out;
}
}
}
final Path path = new Path(files.get(0));
final URI uri = path.toUri();
final String pathString = buildPath(folders[0], latest);
return new Path(uri.getScheme(), uri.getAuthority(), pathString).toString();
}
/**
* Creates a {@link FileSelection selection} with the given file statuses/files and selection
* root.
*
* @param statuses list of file statuses
* @param files list of files
* @param root root path for selections
* @return null if creation of {@link FileSelection} fails with an {@link
* IllegalArgumentException} otherwise a new selection.
* @see FileSelection#FileSelection(List, List, String)
*/
public static FileSelection create(
final List<FileStatus> statuses, final List<String> files, final String root) {
final boolean bothNonEmptySelection =
(statuses != null && statuses.size() > 0) && (files != null && files.size() > 0);
final boolean bothEmptySelection =
(statuses == null || statuses.size() == 0) && (files == null || files.size() == 0);
if (bothNonEmptySelection || bothEmptySelection) {
return null;
}
final String selectionRoot;
if (statuses == null || statuses.isEmpty()) {
selectionRoot = commonPathForFiles(files);
} else {
if (Strings.isNullOrEmpty(root)) {
throw new DrillRuntimeException("Selection root is null or empty" + root);
}
final Path rootPath = handleWildCard(root);
final URI uri = statuses.get(0).getPath().toUri();
final Path path = new Path(uri.getScheme(), uri.getAuthority(), rootPath.toUri().getPath());
selectionRoot = path.toString();
}
return new FileSelection(statuses, files, selectionRoot);
}
public String getNormalizedServerUri() throws URISyntaxException {
URI uri = new URI(originalServerUri);
if (uri.getAuthority() == null) uri = new URI("ws://" + originalServerUri);
if (uri.getPort() == -1)
if ("ws".equals(uri.getScheme().toLowerCase(Locale.ENGLISH)))
uri =
new URI(
"ws",
uri.getUserInfo(),
uri.getHost(),
PluginConfig.DEFAULT_TCP_PORT,
uri.getPath(),
uri.getQuery(),
uri.getFragment());
else if ("wss".equals(uri.getScheme().toLowerCase(Locale.ENGLISH)))
uri =
new URI(
"wss",
uri.getUserInfo(),
uri.getHost(),
PluginConfig.DEFAULT_SSL_PORT,
uri.getPath(),
uri.getQuery(),
uri.getFragment());
return uri.toString();
}
DnsNameResolver(
@Nullable String nsAuthority,
String name,
Attributes params,
Resource<ScheduledExecutorService> timerServiceResource,
Resource<ExecutorService> executorResource) {
// TODO: if a DNS server is provided as nsAuthority, use it.
// https://www.captechconsulting.com/blogs/accessing-the-dusty-corners-of-dns-with-java
this.timerServiceResource = timerServiceResource;
this.executorResource = executorResource;
// Must prepend a "//" to the name when constructing a URI, otherwise it will be treated as an
// opaque URI, thus the authority and host of the resulted URI would be null.
URI nameUri = URI.create("//" + name);
authority =
Preconditions.checkNotNull(
nameUri.getAuthority(), "nameUri (%s) doesn't have an authority", nameUri);
host = Preconditions.checkNotNull(nameUri.getHost(), "host");
if (nameUri.getPort() == -1) {
Integer defaultPort = params.get(NameResolver.Factory.PARAMS_DEFAULT_PORT);
if (defaultPort != null) {
port = defaultPort;
} else {
throw new IllegalArgumentException(
"name '" + name + "' doesn't contain a port, and default port is not set in params");
}
} else {
port = nameUri.getPort();
}
}
@Override
public void registerForNotification(URI uri, Configuration conf, String user, String actionID)
throws URIHandlerException {
HCatURI hcatURI;
try {
hcatURI = new HCatURI(uri);
} catch (URISyntaxException e) {
throw new URIHandlerException(ErrorCode.E0906, uri, e);
}
HCatAccessorService hcatService = Services.get().get(HCatAccessorService.class);
if (!hcatService.isRegisteredForNotification(hcatURI)) {
HCatClient client = getHCatClient(uri, conf, user);
try {
String topic = client.getMessageBusTopicName(hcatURI.getDb(), hcatURI.getTable());
if (topic == null) {
return;
}
hcatService.registerForNotification(
hcatURI, topic, new HCatMessageHandler(uri.getAuthority()));
} catch (HCatException e) {
throw new HCatAccessorException(ErrorCode.E1501, e);
} finally {
closeQuietly(client, true);
}
}
PartitionDependencyManagerService pdmService =
Services.get().get(PartitionDependencyManagerService.class);
pdmService.addMissingDependency(hcatURI, actionID);
}
/**
* Return an array containing hostnames, offset and size of portions of the given file. For a
* nonexistent file or regions, null will be returned.
*
* <p>This call is most helpful with DFS, where it returns hostnames of machines that contain the
* given file.
*
* <p>The FileSystem will simply return an elt containing 'localhost'.
*/
@Override
public BlockLocation[] getFileBlockLocations(FileStatus file, long start, long len)
throws IOException {
// Check if requested file in Swift is more than 5Gb. In this case
// each block has its own location -which may be determinable
// from the Swift client API, depending on the remote server
final FileStatus[] listOfFileBlocks = store.listSubPaths(file.getPath());
List<URI> locations = new ArrayList<URI>();
if (listOfFileBlocks.length > 1) {
for (FileStatus fileStatus : listOfFileBlocks) {
if (SwiftObjectPath.fromPath(uri, fileStatus.getPath())
.equals(SwiftObjectPath.fromPath(uri, file.getPath()))) {
continue;
}
locations.addAll(store.getObjectLocation(fileStatus.getPath()));
}
} else {
locations = store.getObjectLocation(file.getPath());
}
final String[] names = new String[locations.size()];
final String[] hosts = new String[locations.size()];
int i = 0;
for (URI location : locations) {
hosts[i] = location.getHost();
names[i] = location.getAuthority();
i++;
}
return new BlockLocation[] {new BlockLocation(names, hosts, 0, file.getLen())};
}
private URI createTaskLocation(URI baseLocation, String taskId) throws URISyntaxException {
return new URI(
baseLocation.getScheme(),
baseLocation.getAuthority(),
"/task/id/" + taskId,
null,
null); //$NON-NLS-1$
}
@Override
public boolean handlesURI(URI uri) {
if (PASSTHROUGH.equals(uri.getScheme())) {
return STRIPES.containsKey(uri.getAuthority());
}
checkURI(uri);
return SERVERS.containsKey(uri);
}
public void setDatabasePath(Map<String, String> sitexml, String dataPath) {
Path path = new Path(dataPath);
URI parentUri = path.toUri();
String authority = parentUri.getAuthority();
String fspath = parentUri.getScheme() + "://" + (authority == null ? "" : authority) + "/";
sitexml.put(AccumuloConfigFileOptions.INSTANCE_DFS_URI, fspath);
sitexml.put(AccumuloConfigFileOptions.INSTANCE_DFS_DIR, parentUri.getPath());
}
static String canonicalizeOrigin(String url) {
URI uri;
try {
uri = new URI(url);
} catch (URISyntaxException e) {
throw new RuntimeException("specified bad origin", e);
}
return uri.getScheme() + "://" + uri.getAuthority();
}
/** Create a local scratch directory on demand and return it. */
public String getLocalScratchDir(boolean mkdir) {
try {
FileSystem fs = FileSystem.getLocal(conf);
URI uri = fs.getUri();
return getScratchDir(uri.getScheme(), uri.getAuthority(), mkdir, localScratchDir);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/** {@inheritDoc} */
@Override
public void checkPath(Path path) {
URI uri = path.toUri();
if (uri.isAbsolute()) {
if (!F.eq(uri.getScheme(), IGFS_SCHEME))
throw new InvalidPathException(
"Wrong path scheme [expected=" + IGFS_SCHEME + ", actual=" + uri.getAuthority() + ']');
if (!F.eq(uri.getAuthority(), uriAuthority))
throw new InvalidPathException(
"Wrong path authority [expected="
+ uriAuthority
+ ", actual="
+ uri.getAuthority()
+ ']');
}
}
/**
* Run the query that has been set up in this instance. Next step would be to get the results with
* {@link getQueryResult()}
*/
public void doQuery() {
DefaultHttpClient client = new DefaultHttpClient();
client
.getCredentialsProvider()
.setCredentials(
new AuthScope(_targetHost.getHostName(), _targetHost.getPort()),
new UsernamePasswordCredentials(this.getAppid(), this.getAppid()));
URI uri;
try {
String full_path = getQueryPath();
String full_query = getUrlQuery();
uri = new URI(AZURESEARCH_SCHEME, AZURESEARCH_AUTHORITY, full_path, full_query, null);
// Bing and java URI disagree about how to represent + in query
// parameters. This is what we have to do instead...
uri =
new URI(
uri.getScheme()
+ "://"
+ uri.getAuthority()
+ uri.getPath()
+ "?"
+ uri.getRawQuery().replace("+", "%2b"));
// log.log(Level.WARNING, uri.toString());
} catch (URISyntaxException e1) {
e1.printStackTrace();
return;
}
HttpGet get = new HttpGet(uri);
get.addHeader("Accept", "application/xml");
get.addHeader("Content-Type", "application/xml");
try {
_responsePost = client.execute(get);
_resEntity = _responsePost.getEntity();
if (this.getProcessHTTPResults()) {
_rawResult = loadXMLFromStream(_resEntity.getContent());
this.loadResultsFromRawResults();
}
// Adding an automatic HTTP Result to String really requires
// Apache Commons IO. That would break
// Android compatibility. I'm not going to do that unless I
// re-implement IOUtils.
} catch (ClientProtocolException e) {
e.printStackTrace();
} catch (IOException e) {
e.printStackTrace();
} catch (IllegalStateException e) {
e.printStackTrace();
}
}
@Override
public void initialize(URI uri, Configuration conf) throws IOException {
super.initialize(uri, conf);
if (store == null) {
store = createDefaultStore(conf);
}
store.initialize(uri, conf);
setConf(conf);
this.uri = URI.create(uri.getScheme() + "://" + uri.getAuthority());
}
// NOTE this does not currently check anything
public static void checkHref(URI href) {
String uri = href.toASCIIString();
String auth = href.getAuthority();
String host = href.getHost();
String path = href.getPath();
// TODO inject the endpoint of the provider here for rudimentary checks as below
// assertEquals(auth + "://" + host + path, endpoint, "The Href must contain the provider
// endpoint");
// assertTrue(uri.startsWith(endpoint), "The Href must contain the provider endpoint");
}
public static BlobStore open(String url) throws IOException {
URI uri = URI.create(url);
String scheme = uri.getScheme();
if (scheme.equals("file")) {
return LocalBlobStore.open(Paths.get(uri));
} else if (scheme.equals("doss")) {
return RemoteBlobStore.openSecure(uri.getHost(), uri.getPort());
}
throw new IllegalArgumentException(
"Unknown URL scheme '" + uri.getAuthority() + "'. Must be file:// or doss://");
}