private void installSecondaryDexFiles() throws Exception {
final ImmutableMap<String, Path> hashToSources = getRequiredDexFiles();
final ImmutableSet<String> requiredHashes = hashToSources.keySet();
final ImmutableSet<String> presentHashes = prepareSecondaryDexDir(requiredHashes);
final Set<String> hashesToInstall = Sets.difference(requiredHashes, presentHashes);
Map<String, Path> filesToInstallByHash =
Maps.filterKeys(hashToSources, Predicates.in(hashesToInstall));
// This is a bit gross. It was a late addition. Ideally, we could eliminate this, but
// it wouldn't be terrible if we don't. We store the dexed jars on the device
// with the full SHA-1 hashes in their names. This is the format that the loader uses
// internally, so ideally we would just load them in place. However, the code currently
// expects to be able to copy the jars from a directory that matches the name in the
// metadata file, like "secondary-1.dex.jar". We don't want to give up putting the
// hashes in the file names (because we use that to skip re-uploads), so just hack
// the metadata file to have hash-like names.
String metadataContents =
com.google.common.io.Files.toString(
projectFilesystem
.resolve(exopackageInfo.getDexInfo().get().getMetadata())
.toFile(),
Charsets.UTF_8)
.replaceAll(
"secondary-(\\d+)\\.dex\\.jar (\\p{XDigit}{40}) ", "secondary-$2.dex.jar $2 ");
installFiles(
"secondary_dex",
ImmutableMap.copyOf(filesToInstallByHash),
metadataContents,
"secondary-%s.dex.jar",
SECONDARY_DEX_DIR);
}
@Override
public PlanNode visitTableScan(TableScanNode node, RewriteContext<Set<Symbol>> context) {
Set<Symbol> requiredTableScanOutputs =
context
.get()
.stream()
.filter(node.getOutputSymbols()::contains)
.collect(toImmutableSet());
List<Symbol> newOutputSymbols =
node.getOutputSymbols()
.stream()
.filter(requiredTableScanOutputs::contains)
.collect(toImmutableList());
Map<Symbol, ColumnHandle> newAssignments =
Maps.filterKeys(node.getAssignments(), in(requiredTableScanOutputs));
return new TableScanNode(
node.getId(),
node.getTable(),
newOutputSymbols,
newAssignments,
node.getLayout(),
node.getCurrentConstraint(),
node.getOriginalConstraint());
}
@Override
public PlanNode visitIndexSource(IndexSourceNode node, RewriteContext<Set<Symbol>> context) {
List<Symbol> newOutputSymbols =
node.getOutputSymbols()
.stream()
.filter(context.get()::contains)
.collect(toImmutableList());
Set<Symbol> newLookupSymbols =
node.getLookupSymbols()
.stream()
.filter(context.get()::contains)
.collect(toImmutableSet());
Set<Symbol> requiredAssignmentSymbols = context.get();
if (!node.getEffectiveTupleDomain().isNone()) {
Set<Symbol> requiredSymbols =
Maps.filterValues(
node.getAssignments(),
in(node.getEffectiveTupleDomain().getDomains().get().keySet()))
.keySet();
requiredAssignmentSymbols = Sets.union(context.get(), requiredSymbols);
}
Map<Symbol, ColumnHandle> newAssignments =
Maps.filterKeys(node.getAssignments(), in(requiredAssignmentSymbols));
return new IndexSourceNode(
node.getId(),
node.getIndexHandle(),
node.getTableHandle(),
newLookupSymbols,
newOutputSymbols,
newAssignments,
node.getEffectiveTupleDomain());
}
/**
* @return all known references under the "refs" namespace (i.e. not top level ones like HEAD,
* etc), key'ed by ref name
*/
@Override
public Map<String, String> getAll() {
Predicate<String> filter = Predicates.not(new RefPrefixPredicate(TRANSACTIONS_PREFIX));
Map<String, String> allButTransactions =
Maps.filterKeys(ImmutableMap.copyOf(this.refs), filter);
return allButTransactions;
}
public Map<String, File> FilterSourcesKeysAlone() {
return Maps.filterKeys(
srcMap,
new Predicate<String>() {
@Override
public boolean apply(String input) {
return input.startsWith(SRC_RESOURCE_PREFIX);
}
});
}
public Map<String, File> FilterTestsKeys() {
return Maps.filterKeys(
resources,
new Predicate<String>() {
@Override
public boolean apply(String input) {
return input.startsWith(TEST_RESOURCE_PREFIX);
}
});
}
private static int getNumberOfModules(Map<String, String> conf) {
return Maps.filterKeys(
conf,
new Predicate<String>() {
public boolean apply(String input) {
return StringUtils.startsWith(input, ReleaseAndPromotionAction.MODULE_KEY);
}
})
.size();
}
/**
* Deletes an item from the Project's metadata
*
* <p>Beyond it's use here it is also used as a cheap way of generating Operation's to both test
* the OperationApi and the pagination system.
*/
public static Operation deleteItemFromMetadata(
ProjectApi projectApi, String projectName, String key) {
Project project = projectApi.get(projectName);
assertNotNull(project);
ImmutableMap.Builder<String, String> metadataBuilder = ImmutableMap.builder();
metadataBuilder.putAll(
Maps.filterKeys(project.getCommonInstanceMetadata().getItems(), not(equalTo(key))));
return projectApi.setCommonInstanceMetadata(
projectName, metadataBuilder.build(), project.getCommonInstanceMetadata().getFingerprint());
}
public Map<String, File> FilterNonYUISourcesKeys() {
return Maps.filterKeys(
resources,
new Predicate<String>() {
@Override
public boolean apply(String input) {
return input.startsWith(SRC_RESOURCE_PREFIX)
&& !input.contains(SRC_YUI_RESOURCE_PREFIX);
}
});
}
private String buildOptions(Map<String, String> properties) {
StringBuilder options = new StringBuilder();
for (Map.Entry<String, String> attr :
Maps.filterKeys(properties, not(in(PROPERTIES))).entrySet()) {
String key = attr.getKey();
String val = attr.getValue();
options.append(" -").append(key);
if (!Strings.isNullOrEmpty(val)) {
options.append('=').append(val);
}
}
return options.toString();
}
protected Builder<P, D> addTransaction(
final DataModification<P, D> data, final Predicate<P> keyFilter) {
updatedOperational.putAll(Maps.filterKeys(data.getUpdatedOperationalData(), keyFilter));
updatedConfiguration.putAll(Maps.filterKeys(data.getUpdatedConfigurationData(), keyFilter));
originalConfiguration.putAll(Maps.filterKeys(data.getOriginalConfigurationData(), keyFilter));
originalOperational.putAll(Maps.filterKeys(data.getOriginalOperationalData(), keyFilter));
createdOperational.putAll(Maps.filterKeys(data.getCreatedOperationalData(), keyFilter));
createdConfiguration.putAll(Maps.filterKeys(data.getCreatedConfigurationData(), keyFilter));
removedOperational.addAll(Sets.filter(data.getRemovedOperationalData(), keyFilter));
removedConfiguration.addAll(Sets.filter(data.getRemovedConfigurationData(), keyFilter));
return this;
}
@Override
public Map<String, String> removeAll(final String namespace) {
Preconditions.checkNotNull(namespace, "provided namespace is null");
Predicate<String> keyPredicate =
new Predicate<String>() {
@Override
public boolean apply(String refName) {
return refName.startsWith(namespace);
}
};
Map<String, String> removed = Maps.filterKeys(ImmutableMap.copyOf(this.refs), keyPredicate);
for (String key : removed.keySet()) {
refs.remove(key);
}
return removed;
}
private SubPlan buildFragment(
PlanNode root, FragmentProperties properties, PlanFragmentId fragmentId) {
Set<Symbol> dependencies = SymbolExtractor.extract(root);
PlanFragment fragment =
new PlanFragment(
fragmentId,
root,
Maps.filterKeys(types, in(dependencies)),
properties.getOutputLayout(),
properties.getDistribution(),
properties.getDistributeBy(),
properties.getOutputPartitioning(),
properties.getPartitionBy(),
properties.getNullPartitionPolicy(),
properties.getHash());
return new SubPlan(fragment, properties.getChildren());
}
/**
* base jclouds tests expect properties to arrive in a different naming convention, based on
* provider name.
*
* <p>ex.
*
* <pre>
* test.jenkins.compute.provider=aws-ec2
* test.jenkins.compute.identity=access
* test.jenkins.compute.credential=secret
* </pre>
*
* should turn into
*
* <pre>
* test.aws-ec2.identity=access
* test.aws-ec2.credential=secret
* </pre>
*/
static {
PROVIDER =
checkNotNull(
System.getProperty("test.jenkins.compute.provider"),
"test.compute.provider variable must be set!");
Map<String, String> filtered =
Maps.filterKeys(
Map.class.cast(System.getProperties()),
Predicates.containsPattern("^test\\.jenkins\\.compute"));
Map<String, String> transformed =
Maps2.transformKeys(
filtered,
new Function<String, String>() {
public String apply(String arg0) {
return arg0.replaceAll("test.jenkins.compute", "test." + PROVIDER);
}
});
System.getProperties().putAll(transformed);
}
/**
* Drops capabilities that we shouldn't send over the wire.
*
* <p>Used for capabilities which aren't BeanToJson-convertable, and are only used by the local
* launcher.
*/
private static Capabilities dropCapabilities(Capabilities capabilities, String... keysToRemove) {
if (capabilities == null) {
return new DesiredCapabilities();
}
final Set<String> toRemove = Sets.newHashSet(keysToRemove);
DesiredCapabilities caps =
new DesiredCapabilities(
Maps.filterKeys(
capabilities.asMap(),
new Predicate<String>() {
public boolean apply(String key) {
return !toRemove.contains(key);
}
}));
// Ensure that the proxy is in a state fit to be sent to the extension
Proxy proxy = Proxy.extractFrom(capabilities);
if (proxy != null) {
caps.setCapability(PROXY, new BeanToJsonConverter().convert(proxy));
}
return caps;
}
private void installNativeLibrariesForAbi(String abi, ImmutableMap<String, Path> libraries)
throws Exception {
if (libraries.isEmpty()) {
return;
}
ImmutableSet<String> requiredHashes = libraries.keySet();
ImmutableSet<String> presentHashes = prepareNativeLibsDir(abi, requiredHashes);
Map<String, Path> filesToInstallByHash =
Maps.filterKeys(libraries, Predicates.not(Predicates.in(presentHashes)));
String metadataContents =
Joiner.on('\n')
.join(
FluentIterable.from(libraries.entrySet())
.transform(
new Function<Map.Entry<String, Path>, String>() {
@Override
public String apply(Map.Entry<String, Path> input) {
String hash = input.getKey();
String filename = input.getValue().getFileName().toString();
int index = filename.indexOf('.');
String libname =
index == -1 ? filename : filename.substring(0, index);
return String.format("%s native-%s.so", libname, hash);
}
}));
installFiles(
"native_library",
ImmutableMap.copyOf(filesToInstallByHash),
metadataContents,
"native-%s.so",
NATIVE_LIBS_DIR.resolve(abi));
}
/**
* Looks in the directory specified by {@code recursivePkgKey} for a package, does some work as
* specified by {@link Visitor} if such a package exists, then recursively does work in each
* non-excluded subdirectory as specified by {@link #getSkyKeyForSubdirectory}, and finally
* aggregates the {@link Visitor} value along with values from each subdirectory as specified by
* {@link #aggregateWithSubdirectorySkyValues}, and returns that aggregation.
*
* <p>Returns null if {@code env.valuesMissing()} is true, checked after each call to one of
* {@link RecursiveDirectoryTraversalFunction}'s abstract methods that were given {@code env}.
* (And after each of {@code visitDirectory}'s own uses of {@code env}, of course.)
*/
TReturn visitDirectory(RecursivePkgKey recursivePkgKey, Environment env) {
RootedPath rootedPath = recursivePkgKey.getRootedPath();
ImmutableSet<PathFragment> excludedPaths = recursivePkgKey.getExcludedPaths();
Path root = rootedPath.getRoot();
PathFragment rootRelativePath = rootedPath.getRelativePath();
SkyKey fileKey = FileValue.key(rootedPath);
FileValue fileValue;
try {
fileValue =
(FileValue)
env.getValueOrThrow(
fileKey,
InconsistentFilesystemException.class,
FileSymlinkException.class,
IOException.class);
} catch (InconsistentFilesystemException | FileSymlinkException | IOException e) {
return reportErrorAndReturn(
"Failed to get information about path", e, rootRelativePath, env.getListener());
}
if (fileValue == null) {
return null;
}
if (!fileValue.isDirectory()) {
return getEmptyReturn();
}
PackageIdentifier packageId =
PackageIdentifier.create(recursivePkgKey.getRepository(), rootRelativePath);
if ((packageId.getRepository().isDefault() || packageId.getRepository().isMain())
&& fileValue.isSymlink()
&& fileValue
.getUnresolvedLinkTarget()
.startsWith(directories.getOutputBase().asFragment())) {
// Symlinks back to the output base are not traversed so that we avoid convenience symlinks.
// Note that it's not enough to just check for the convenience symlinks themselves, because
// if the value of --symlink_prefix changes, the old symlinks are left in place. This
// algorithm also covers more creative use cases where people create convenience symlinks
// somewhere in the directory tree manually.
return getEmptyReturn();
}
SkyKey pkgLookupKey = PackageLookupValue.key(packageId);
SkyKey dirListingKey = DirectoryListingValue.key(rootedPath);
Map<
SkyKey,
ValueOrException4<
NoSuchPackageException,
InconsistentFilesystemException,
FileSymlinkException,
IOException>>
pkgLookupAndDirectoryListingDeps =
env.getValuesOrThrow(
ImmutableList.of(pkgLookupKey, dirListingKey),
NoSuchPackageException.class,
InconsistentFilesystemException.class,
FileSymlinkException.class,
IOException.class);
if (env.valuesMissing()) {
return null;
}
PackageLookupValue pkgLookupValue;
try {
pkgLookupValue =
(PackageLookupValue)
Preconditions.checkNotNull(
pkgLookupAndDirectoryListingDeps.get(pkgLookupKey).get(),
"%s %s",
recursivePkgKey,
pkgLookupKey);
} catch (NoSuchPackageException | InconsistentFilesystemException e) {
return reportErrorAndReturn("Failed to load package", e, rootRelativePath, env.getListener());
} catch (IOException | FileSymlinkException e) {
throw new IllegalStateException(e);
}
TVisitor visitor = getInitialVisitor();
DirectoryListingValue dirListingValue;
try {
dirListingValue =
(DirectoryListingValue)
Preconditions.checkNotNull(
pkgLookupAndDirectoryListingDeps.get(dirListingKey).get(),
"%s %s",
recursivePkgKey,
dirListingKey);
} catch (InconsistentFilesystemException | IOException e) {
return reportErrorAndReturn(
"Failed to list directory contents", e, rootRelativePath, env.getListener());
} catch (FileSymlinkException e) {
// DirectoryListingFunction only throws FileSymlinkCycleException when FileFunction throws it,
// but FileFunction was evaluated for rootedPath above, and didn't throw there. It shouldn't
// be able to avoid throwing there but throw here.
throw new IllegalStateException(
"Symlink cycle found after not being found for \"" + rootedPath + "\"");
} catch (NoSuchPackageException e) {
throw new IllegalStateException(e);
}
boolean followSymlinks = shouldFollowSymlinksWhenTraversing(dirListingValue.getDirents());
List<SkyKey> childDeps = new ArrayList<>();
for (Dirent dirent : dirListingValue.getDirents()) {
Type type = dirent.getType();
if (type != Type.DIRECTORY
&& (type != Type.SYMLINK || (type == Type.SYMLINK && !followSymlinks))) {
// Non-directories can never host packages. Symlinks to non-directories are weeded out at
// the next level of recursion when we check if its FileValue is a directory. This is slower
// if there are a lot of symlinks in the tree, but faster if there are only a few, which is
// the case most of the time.
//
// We are not afraid of weird symlink structure here: both cyclical ones and ones that give
// rise to infinite directory trees are diagnosed by FileValue.
continue;
}
String basename = dirent.getName();
if (rootRelativePath.equals(PathFragment.EMPTY_FRAGMENT)
&& PathPackageLocator.DEFAULT_TOP_LEVEL_EXCLUDES.contains(basename)) {
continue;
}
PathFragment subdirectory = rootRelativePath.getRelative(basename);
// If this subdirectory is one of the excluded paths, don't recurse into it.
if (excludedPaths.contains(subdirectory)) {
continue;
}
// If we have an excluded path that isn't below this subdirectory, we shouldn't pass that
// excluded path to our evaluation of the subdirectory, because the exclusion can't
// possibly match anything beneath the subdirectory.
//
// For example, if we're currently evaluating directory "a", are looking at its subdirectory
// "a/b", and we have an excluded path "a/c/d", there's no need to pass the excluded path
// "a/c/d" to our evaluation of "a/b".
//
// This strategy should help to get more skyframe sharing. Consider the example above. A
// subsequent request of "a/b/...", without any excluded paths, will be a cache hit.
//
// TODO(bazel-team): Replace the excludedPaths set with a trie or a SortedSet for better
// efficiency.
ImmutableSet<PathFragment> excludedSubdirectoriesBeneathThisSubdirectory =
PathFragment.filterPathsStartingWith(excludedPaths, subdirectory);
RootedPath subdirectoryRootedPath = RootedPath.toRootedPath(root, subdirectory);
childDeps.add(
getSkyKeyForSubdirectory(
recursivePkgKey.getRepository(),
subdirectoryRootedPath,
excludedSubdirectoriesBeneathThisSubdirectory));
}
Map<SkyKey, SkyValue> subdirectorySkyValues;
if (pkgLookupValue.packageExists() && pkgLookupValue.getRoot().equals(root)) {
SkyKey packageKey = PackageValue.key(packageId);
Map<SkyKey, ValueOrException<NoSuchPackageException>> dependentSkyValues =
env.getValuesOrThrow(
Iterables.concat(childDeps, ImmutableList.of(packageKey)),
NoSuchPackageException.class);
if (env.valuesMissing()) {
return null;
}
Package pkg = null;
try {
PackageValue pkgValue = (PackageValue) dependentSkyValues.get(packageKey).get();
if (pkgValue == null) {
return null;
}
pkg = pkgValue.getPackage();
if (pkg.containsErrors()) {
env.getListener()
.handle(Event.error("package contains errors: " + rootRelativePath.getPathString()));
}
} catch (NoSuchPackageException e) {
// The package had errors, but don't fail-fast as there might be subpackages below the
// current directory.
env.getListener()
.handle(Event.error("package contains errors: " + rootRelativePath.getPathString()));
}
if (pkg != null) {
visitor.visitPackageValue(pkg, env);
if (env.valuesMissing()) {
return null;
}
}
ImmutableMap.Builder<SkyKey, SkyValue> subdirectoryBuilder = ImmutableMap.builder();
for (Map.Entry<SkyKey, ValueOrException<NoSuchPackageException>> entry :
Maps.filterKeys(dependentSkyValues, Predicates.not(Predicates.equalTo(packageKey)))
.entrySet()) {
try {
subdirectoryBuilder.put(entry.getKey(), entry.getValue().get());
} catch (NoSuchPackageException e) {
// ignored.
}
}
subdirectorySkyValues = subdirectoryBuilder.build();
} else {
subdirectorySkyValues = env.getValues(childDeps);
}
if (env.valuesMissing()) {
return null;
}
return aggregateWithSubdirectorySkyValues(visitor, subdirectorySkyValues);
}
@Override
public Map<String, String> getAll(final String prefix) {
Preconditions.checkNotNull(prefix, "namespace can't be null");
Predicate<String> filter = new RefPrefixPredicate(prefix);
return Maps.filterKeys(ImmutableMap.copyOf(this.refs), filter);
}
protected Map<String, String> getProcessLayout(Profile profile, String layoutPath) {
return ByteToStringValues.INSTANCE.apply(
Maps.filterKeys(profile.getFileConfigurations(), new LayOutPredicate(layoutPath)));
}
public Message(final Map<String, Object> fields) {
this((String) fields.get(FIELD_ID), Maps.filterKeys(fields, not(equalTo(FIELD_ID))));
}
@Override
public ComputeService findComputeService(ConfigBag conf, boolean allowReuse) {
String provider = checkNotNull(conf.get(CLOUD_PROVIDER), "provider must not be null");
String identity =
checkNotNull(conf.get(CloudLocationConfig.ACCESS_IDENTITY), "identity must not be null");
String credential =
checkNotNull(
conf.get(CloudLocationConfig.ACCESS_CREDENTIAL), "credential must not be null");
Properties properties = new Properties();
properties.setProperty(Constants.PROPERTY_TRUST_ALL_CERTS, Boolean.toString(true));
properties.setProperty(Constants.PROPERTY_RELAX_HOSTNAME, Boolean.toString(true));
properties.setProperty(
"jclouds.ssh.max-retries",
conf.getStringKey("jclouds.ssh.max-retries") != null
? conf.getStringKey("jclouds.ssh.max-retries").toString()
: "50");
// Enable aws-ec2 lazy image fetching, if given a specific imageId; otherwise customize for
// specific owners; or all as a last resort
// See https://issues.apache.org/jira/browse/WHIRR-416
if ("aws-ec2".equals(provider)) {
// TODO convert AWS-only flags to config keys
if (groovyTruth(conf.get(IMAGE_ID))) {
properties.setProperty(PROPERTY_EC2_AMI_QUERY, "");
properties.setProperty(PROPERTY_EC2_CC_AMI_QUERY, "");
} else if (groovyTruth(conf.getStringKey("imageOwner"))) {
properties.setProperty(
PROPERTY_EC2_AMI_QUERY,
"owner-id=" + conf.getStringKey("imageOwner") + ";state=available;image-type=machine");
} else if (groovyTruth(conf.getStringKey("anyOwner"))) {
// set `anyOwner: true` to override the default query (which is restricted to certain owners
// as per below),
// allowing the AMI query to bind to any machine
// (note however, we sometimes pick defaults in JcloudsLocationFactory);
// (and be careful, this can give a LOT of data back, taking several minutes,
// and requiring extra memory allocated on the command-line)
properties.setProperty(PROPERTY_EC2_AMI_QUERY, "state=available;image-type=machine");
/*
* by default the following filters are applied:
* Filter.1.Name=owner-id&Filter.1.Value.1=137112412989&
* Filter.1.Value.2=063491364108&
* Filter.1.Value.3=099720109477&
* Filter.1.Value.4=411009282317&
* Filter.2.Name=state&Filter.2.Value.1=available&
* Filter.3.Name=image-type&Filter.3.Value.1=machine&
*/
}
// occasionally can get com.google.common.util.concurrent.UncheckedExecutionException:
// java.lang.RuntimeException:
// security group eu-central-1/jclouds#brooklyn-bxza-alex-eu-central-shoul-u2jy-nginx-ielm
// is not available after creating
// the default timeout was 500ms so let's raise it in case that helps
properties.setProperty(
EC2Constants.PROPERTY_EC2_TIMEOUT_SECURITYGROUP_PRESENT,
"" + Duration.seconds(30).toMilliseconds());
}
// FIXME Deprecated mechanism, should have a ConfigKey for overrides
Map<String, Object> extra =
Maps.filterKeys(conf.getAllConfig(), Predicates.containsPattern("^jclouds\\."));
if (extra.size() > 0) {
LOG.warn("Jclouds using deprecated property overrides: " + Sanitizer.sanitize(extra));
}
properties.putAll(extra);
String endpoint = conf.get(CloudLocationConfig.CLOUD_ENDPOINT);
if (!groovyTruth(endpoint)) endpoint = getDeprecatedProperty(conf, Constants.PROPERTY_ENDPOINT);
if (groovyTruth(endpoint)) properties.setProperty(Constants.PROPERTY_ENDPOINT, endpoint);
Map<?, ?> cacheKey =
MutableMap.builder()
.putAll(properties)
.put("provider", provider)
.put("identity", identity)
.put("credential", credential)
.putIfNotNull("endpoint", endpoint)
.build()
.asUnmodifiable();
if (allowReuse) {
ComputeService result = cachedComputeServices.get(cacheKey);
if (result != null) {
LOG.trace(
"jclouds ComputeService cache hit for compute service, for "
+ Sanitizer.sanitize(properties));
return result;
}
LOG.debug(
"jclouds ComputeService cache miss for compute service, creating, for "
+ Sanitizer.sanitize(properties));
}
Iterable<Module> modules = getCommonModules();
// Synchronizing to avoid deadlock from sun.reflect.annotation.AnnotationType.
// See https://github.com/brooklyncentral/brooklyn/issues/974
ComputeServiceContext computeServiceContext;
synchronized (createComputeServicesMutex) {
computeServiceContext =
ContextBuilder.newBuilder(provider)
.modules(modules)
.credentials(identity, credential)
.overrides(properties)
.build(ComputeServiceContext.class);
}
final ComputeService computeService = computeServiceContext.getComputeService();
if (allowReuse) {
synchronized (cachedComputeServices) {
ComputeService result = cachedComputeServices.get(cacheKey);
if (result != null) {
LOG.debug(
"jclouds ComputeService cache recovery for compute service, for "
+ Sanitizer.sanitize(cacheKey));
// keep the old one, discard the new one
computeService.getContext().close();
return result;
}
LOG.debug(
"jclouds ComputeService created "
+ computeService
+ ", adding to cache, for "
+ Sanitizer.sanitize(properties));
cachedComputeServices.put(cacheKey, computeService);
}
}
return computeService;
}
@Override
public ConnectorPartitionResult getPartitions(
ConnectorSession session,
ConnectorTableHandle tableHandle,
TupleDomain<ColumnHandle> tupleDomain) {
CassandraTableHandle cassandraTableHandle =
checkType(tableHandle, CassandraTableHandle.class, "tableHandle");
checkNotNull(tupleDomain, "tupleDomain is null");
CassandraTable table = schemaProvider.getTable(cassandraTableHandle);
List<CassandraColumnHandle> partitionKeys = table.getPartitionKeyColumns();
// fetch the partitions
List<CassandraPartition> allPartitions = getCassandraPartitions(table, tupleDomain);
log.debug(
"%s.%s #partitions: %d",
cassandraTableHandle.getSchemaName(),
cassandraTableHandle.getTableName(),
allPartitions.size());
// do a final pass to filter based on fields that could not be used to build the prefix
List<ConnectorPartition> partitions =
allPartitions
.stream()
.filter(partition -> tupleDomain.overlaps(partition.getTupleDomain()))
.collect(toList());
// All partition key domains will be fully evaluated, so we don't need to include those
TupleDomain<ColumnHandle> remainingTupleDomain = TupleDomain.none();
if (!tupleDomain.isNone()) {
if (partitions.size() == 1 && ((CassandraPartition) partitions.get(0)).isUnpartitioned()) {
remainingTupleDomain = tupleDomain;
} else {
@SuppressWarnings({"rawtypes", "unchecked"})
List<ColumnHandle> partitionColumns = (List) partitionKeys;
remainingTupleDomain =
TupleDomain.withColumnDomains(
Maps.filterKeys(tupleDomain.getDomains(), not(in(partitionColumns))));
}
}
// push down indexed column fixed value predicates only for unpartitioned partition which uses
// token range query
if (partitions.size() == 1 && ((CassandraPartition) partitions.get(0)).isUnpartitioned()) {
Map<ColumnHandle, Domain> domains = tupleDomain.getDomains();
List<ColumnHandle> indexedColumns = new ArrayList<>();
// compose partitionId by using indexed column
StringBuilder sb = new StringBuilder();
for (Map.Entry<ColumnHandle, Domain> entry : domains.entrySet()) {
CassandraColumnHandle column = (CassandraColumnHandle) entry.getKey();
Domain domain = entry.getValue();
if (column.isIndexed() && domain.isSingleValue()) {
sb.append(CassandraCqlUtils.validColumnName(column.getName()))
.append(" = ")
.append(
CassandraCqlUtils.cqlValue(
toCQLCompatibleString(entry.getValue().getSingleValue()),
column.getCassandraType()));
indexedColumns.add(column);
// Only one indexed column predicate can be pushed down.
break;
}
}
if (sb.length() > 0) {
CassandraPartition partition = (CassandraPartition) partitions.get(0);
TupleDomain<ColumnHandle> filterIndexedColumn =
TupleDomain.withColumnDomains(
Maps.filterKeys(remainingTupleDomain.getDomains(), not(in(indexedColumns))));
partitions = new ArrayList<>();
partitions.add(
new CassandraPartition(partition.getKey(), sb.toString(), filterIndexedColumn, true));
return new ConnectorPartitionResult(partitions, filterIndexedColumn);
}
}
return new ConnectorPartitionResult(partitions, remainingTupleDomain);
}