예제 #1
0
    private void installSecondaryDexFiles() throws Exception {
      final ImmutableMap<String, Path> hashToSources = getRequiredDexFiles();
      final ImmutableSet<String> requiredHashes = hashToSources.keySet();
      final ImmutableSet<String> presentHashes = prepareSecondaryDexDir(requiredHashes);
      final Set<String> hashesToInstall = Sets.difference(requiredHashes, presentHashes);

      Map<String, Path> filesToInstallByHash =
          Maps.filterKeys(hashToSources, Predicates.in(hashesToInstall));

      // This is a bit gross.  It was a late addition.  Ideally, we could eliminate this, but
      // it wouldn't be terrible if we don't.  We store the dexed jars on the device
      // with the full SHA-1 hashes in their names.  This is the format that the loader uses
      // internally, so ideally we would just load them in place.  However, the code currently
      // expects to be able to copy the jars from a directory that matches the name in the
      // metadata file, like "secondary-1.dex.jar".  We don't want to give up putting the
      // hashes in the file names (because we use that to skip re-uploads), so just hack
      // the metadata file to have hash-like names.
      String metadataContents =
          com.google.common.io.Files.toString(
                  projectFilesystem
                      .resolve(exopackageInfo.getDexInfo().get().getMetadata())
                      .toFile(),
                  Charsets.UTF_8)
              .replaceAll(
                  "secondary-(\\d+)\\.dex\\.jar (\\p{XDigit}{40}) ", "secondary-$2.dex.jar $2 ");

      installFiles(
          "secondary_dex",
          ImmutableMap.copyOf(filesToInstallByHash),
          metadataContents,
          "secondary-%s.dex.jar",
          SECONDARY_DEX_DIR);
    }
예제 #2
0
    @Override
    public PlanNode visitTableScan(TableScanNode node, RewriteContext<Set<Symbol>> context) {
      Set<Symbol> requiredTableScanOutputs =
          context
              .get()
              .stream()
              .filter(node.getOutputSymbols()::contains)
              .collect(toImmutableSet());

      List<Symbol> newOutputSymbols =
          node.getOutputSymbols()
              .stream()
              .filter(requiredTableScanOutputs::contains)
              .collect(toImmutableList());

      Map<Symbol, ColumnHandle> newAssignments =
          Maps.filterKeys(node.getAssignments(), in(requiredTableScanOutputs));

      return new TableScanNode(
          node.getId(),
          node.getTable(),
          newOutputSymbols,
          newAssignments,
          node.getLayout(),
          node.getCurrentConstraint(),
          node.getOriginalConstraint());
    }
예제 #3
0
    @Override
    public PlanNode visitIndexSource(IndexSourceNode node, RewriteContext<Set<Symbol>> context) {
      List<Symbol> newOutputSymbols =
          node.getOutputSymbols()
              .stream()
              .filter(context.get()::contains)
              .collect(toImmutableList());

      Set<Symbol> newLookupSymbols =
          node.getLookupSymbols()
              .stream()
              .filter(context.get()::contains)
              .collect(toImmutableSet());

      Set<Symbol> requiredAssignmentSymbols = context.get();
      if (!node.getEffectiveTupleDomain().isNone()) {
        Set<Symbol> requiredSymbols =
            Maps.filterValues(
                    node.getAssignments(),
                    in(node.getEffectiveTupleDomain().getDomains().get().keySet()))
                .keySet();
        requiredAssignmentSymbols = Sets.union(context.get(), requiredSymbols);
      }
      Map<Symbol, ColumnHandle> newAssignments =
          Maps.filterKeys(node.getAssignments(), in(requiredAssignmentSymbols));

      return new IndexSourceNode(
          node.getId(),
          node.getIndexHandle(),
          node.getTableHandle(),
          newLookupSymbols,
          newOutputSymbols,
          newAssignments,
          node.getEffectiveTupleDomain());
    }
예제 #4
0
  /**
   * @return all known references under the "refs" namespace (i.e. not top level ones like HEAD,
   *     etc), key'ed by ref name
   */
  @Override
  public Map<String, String> getAll() {

    Predicate<String> filter = Predicates.not(new RefPrefixPredicate(TRANSACTIONS_PREFIX));
    Map<String, String> allButTransactions =
        Maps.filterKeys(ImmutableMap.copyOf(this.refs), filter);
    return allButTransactions;
  }
예제 #5
0
 public Map<String, File> FilterSourcesKeysAlone() {
   return Maps.filterKeys(
       srcMap,
       new Predicate<String>() {
         @Override
         public boolean apply(String input) {
           return input.startsWith(SRC_RESOURCE_PREFIX);
         }
       });
 }
예제 #6
0
 public Map<String, File> FilterTestsKeys() {
   return Maps.filterKeys(
       resources,
       new Predicate<String>() {
         @Override
         public boolean apply(String input) {
           return input.startsWith(TEST_RESOURCE_PREFIX);
         }
       });
 }
 private static int getNumberOfModules(Map<String, String> conf) {
   return Maps.filterKeys(
           conf,
           new Predicate<String>() {
             public boolean apply(String input) {
               return StringUtils.startsWith(input, ReleaseAndPromotionAction.MODULE_KEY);
             }
           })
       .size();
 }
 /**
  * Deletes an item from the Project's metadata
  *
  * <p>Beyond it's use here it is also used as a cheap way of generating Operation's to both test
  * the OperationApi and the pagination system.
  */
 public static Operation deleteItemFromMetadata(
     ProjectApi projectApi, String projectName, String key) {
   Project project = projectApi.get(projectName);
   assertNotNull(project);
   ImmutableMap.Builder<String, String> metadataBuilder = ImmutableMap.builder();
   metadataBuilder.putAll(
       Maps.filterKeys(project.getCommonInstanceMetadata().getItems(), not(equalTo(key))));
   return projectApi.setCommonInstanceMetadata(
       projectName, metadataBuilder.build(), project.getCommonInstanceMetadata().getFingerprint());
 }
예제 #9
0
 public Map<String, File> FilterNonYUISourcesKeys() {
   return Maps.filterKeys(
       resources,
       new Predicate<String>() {
         @Override
         public boolean apply(String input) {
           return input.startsWith(SRC_RESOURCE_PREFIX)
               && !input.contains(SRC_YUI_RESOURCE_PREFIX);
         }
       });
 }
 private String buildOptions(Map<String, String> properties) {
   StringBuilder options = new StringBuilder();
   for (Map.Entry<String, String> attr :
       Maps.filterKeys(properties, not(in(PROPERTIES))).entrySet()) {
     String key = attr.getKey();
     String val = attr.getValue();
     options.append(" -").append(key);
     if (!Strings.isNullOrEmpty(val)) {
       options.append('=').append(val);
     }
   }
   return options.toString();
 }
 protected Builder<P, D> addTransaction(
     final DataModification<P, D> data, final Predicate<P> keyFilter) {
   updatedOperational.putAll(Maps.filterKeys(data.getUpdatedOperationalData(), keyFilter));
   updatedConfiguration.putAll(Maps.filterKeys(data.getUpdatedConfigurationData(), keyFilter));
   originalConfiguration.putAll(Maps.filterKeys(data.getOriginalConfigurationData(), keyFilter));
   originalOperational.putAll(Maps.filterKeys(data.getOriginalOperationalData(), keyFilter));
   createdOperational.putAll(Maps.filterKeys(data.getCreatedOperationalData(), keyFilter));
   createdConfiguration.putAll(Maps.filterKeys(data.getCreatedConfigurationData(), keyFilter));
   removedOperational.addAll(Sets.filter(data.getRemovedOperationalData(), keyFilter));
   removedConfiguration.addAll(Sets.filter(data.getRemovedConfigurationData(), keyFilter));
   return this;
 }
예제 #12
0
  @Override
  public Map<String, String> removeAll(final String namespace) {
    Preconditions.checkNotNull(namespace, "provided namespace is null");

    Predicate<String> keyPredicate =
        new Predicate<String>() {

          @Override
          public boolean apply(String refName) {
            return refName.startsWith(namespace);
          }
        };
    Map<String, String> removed = Maps.filterKeys(ImmutableMap.copyOf(this.refs), keyPredicate);
    for (String key : removed.keySet()) {
      refs.remove(key);
    }
    return removed;
  }
예제 #13
0
    private SubPlan buildFragment(
        PlanNode root, FragmentProperties properties, PlanFragmentId fragmentId) {
      Set<Symbol> dependencies = SymbolExtractor.extract(root);

      PlanFragment fragment =
          new PlanFragment(
              fragmentId,
              root,
              Maps.filterKeys(types, in(dependencies)),
              properties.getOutputLayout(),
              properties.getDistribution(),
              properties.getDistributeBy(),
              properties.getOutputPartitioning(),
              properties.getPartitionBy(),
              properties.getNullPartitionPolicy(),
              properties.getHash());

      return new SubPlan(fragment, properties.getChildren());
    }
  /**
   * base jclouds tests expect properties to arrive in a different naming convention, based on
   * provider name.
   *
   * <p>ex.
   *
   * <pre>
   *  test.jenkins.compute.provider=aws-ec2
   *  test.jenkins.compute.identity=access
   *  test.jenkins.compute.credential=secret
   * </pre>
   *
   * should turn into
   *
   * <pre>
   *  test.aws-ec2.identity=access
   *  test.aws-ec2.credential=secret
   * </pre>
   */
  static {
    PROVIDER =
        checkNotNull(
            System.getProperty("test.jenkins.compute.provider"),
            "test.compute.provider variable must be set!");
    Map<String, String> filtered =
        Maps.filterKeys(
            Map.class.cast(System.getProperties()),
            Predicates.containsPattern("^test\\.jenkins\\.compute"));
    Map<String, String> transformed =
        Maps2.transformKeys(
            filtered,
            new Function<String, String>() {

              public String apply(String arg0) {
                return arg0.replaceAll("test.jenkins.compute", "test." + PROVIDER);
              }
            });
    System.getProperties().putAll(transformed);
  }
예제 #15
0
  /**
   * Drops capabilities that we shouldn't send over the wire.
   *
   * <p>Used for capabilities which aren't BeanToJson-convertable, and are only used by the local
   * launcher.
   */
  private static Capabilities dropCapabilities(Capabilities capabilities, String... keysToRemove) {
    if (capabilities == null) {
      return new DesiredCapabilities();
    }
    final Set<String> toRemove = Sets.newHashSet(keysToRemove);
    DesiredCapabilities caps =
        new DesiredCapabilities(
            Maps.filterKeys(
                capabilities.asMap(),
                new Predicate<String>() {
                  public boolean apply(String key) {
                    return !toRemove.contains(key);
                  }
                }));

    // Ensure that the proxy is in a state fit to be sent to the extension
    Proxy proxy = Proxy.extractFrom(capabilities);
    if (proxy != null) {
      caps.setCapability(PROXY, new BeanToJsonConverter().convert(proxy));
    }

    return caps;
  }
예제 #16
0
    private void installNativeLibrariesForAbi(String abi, ImmutableMap<String, Path> libraries)
        throws Exception {
      if (libraries.isEmpty()) {
        return;
      }

      ImmutableSet<String> requiredHashes = libraries.keySet();
      ImmutableSet<String> presentHashes = prepareNativeLibsDir(abi, requiredHashes);

      Map<String, Path> filesToInstallByHash =
          Maps.filterKeys(libraries, Predicates.not(Predicates.in(presentHashes)));

      String metadataContents =
          Joiner.on('\n')
              .join(
                  FluentIterable.from(libraries.entrySet())
                      .transform(
                          new Function<Map.Entry<String, Path>, String>() {
                            @Override
                            public String apply(Map.Entry<String, Path> input) {
                              String hash = input.getKey();
                              String filename = input.getValue().getFileName().toString();
                              int index = filename.indexOf('.');
                              String libname =
                                  index == -1 ? filename : filename.substring(0, index);
                              return String.format("%s native-%s.so", libname, hash);
                            }
                          }));

      installFiles(
          "native_library",
          ImmutableMap.copyOf(filesToInstallByHash),
          metadataContents,
          "native-%s.so",
          NATIVE_LIBS_DIR.resolve(abi));
    }
  /**
   * Looks in the directory specified by {@code recursivePkgKey} for a package, does some work as
   * specified by {@link Visitor} if such a package exists, then recursively does work in each
   * non-excluded subdirectory as specified by {@link #getSkyKeyForSubdirectory}, and finally
   * aggregates the {@link Visitor} value along with values from each subdirectory as specified by
   * {@link #aggregateWithSubdirectorySkyValues}, and returns that aggregation.
   *
   * <p>Returns null if {@code env.valuesMissing()} is true, checked after each call to one of
   * {@link RecursiveDirectoryTraversalFunction}'s abstract methods that were given {@code env}.
   * (And after each of {@code visitDirectory}'s own uses of {@code env}, of course.)
   */
  TReturn visitDirectory(RecursivePkgKey recursivePkgKey, Environment env) {
    RootedPath rootedPath = recursivePkgKey.getRootedPath();
    ImmutableSet<PathFragment> excludedPaths = recursivePkgKey.getExcludedPaths();
    Path root = rootedPath.getRoot();
    PathFragment rootRelativePath = rootedPath.getRelativePath();

    SkyKey fileKey = FileValue.key(rootedPath);
    FileValue fileValue;
    try {
      fileValue =
          (FileValue)
              env.getValueOrThrow(
                  fileKey,
                  InconsistentFilesystemException.class,
                  FileSymlinkException.class,
                  IOException.class);
    } catch (InconsistentFilesystemException | FileSymlinkException | IOException e) {
      return reportErrorAndReturn(
          "Failed to get information about path", e, rootRelativePath, env.getListener());
    }
    if (fileValue == null) {
      return null;
    }

    if (!fileValue.isDirectory()) {
      return getEmptyReturn();
    }

    PackageIdentifier packageId =
        PackageIdentifier.create(recursivePkgKey.getRepository(), rootRelativePath);

    if ((packageId.getRepository().isDefault() || packageId.getRepository().isMain())
        && fileValue.isSymlink()
        && fileValue
            .getUnresolvedLinkTarget()
            .startsWith(directories.getOutputBase().asFragment())) {
      // Symlinks back to the output base are not traversed so that we avoid convenience symlinks.
      // Note that it's not enough to just check for the convenience symlinks themselves, because
      // if the value of --symlink_prefix changes, the old symlinks are left in place. This
      // algorithm also covers more creative use cases where people create convenience symlinks
      // somewhere in the directory tree manually.
      return getEmptyReturn();
    }

    SkyKey pkgLookupKey = PackageLookupValue.key(packageId);
    SkyKey dirListingKey = DirectoryListingValue.key(rootedPath);
    Map<
            SkyKey,
            ValueOrException4<
                NoSuchPackageException,
                InconsistentFilesystemException,
                FileSymlinkException,
                IOException>>
        pkgLookupAndDirectoryListingDeps =
            env.getValuesOrThrow(
                ImmutableList.of(pkgLookupKey, dirListingKey),
                NoSuchPackageException.class,
                InconsistentFilesystemException.class,
                FileSymlinkException.class,
                IOException.class);
    if (env.valuesMissing()) {
      return null;
    }
    PackageLookupValue pkgLookupValue;
    try {
      pkgLookupValue =
          (PackageLookupValue)
              Preconditions.checkNotNull(
                  pkgLookupAndDirectoryListingDeps.get(pkgLookupKey).get(),
                  "%s %s",
                  recursivePkgKey,
                  pkgLookupKey);
    } catch (NoSuchPackageException | InconsistentFilesystemException e) {
      return reportErrorAndReturn("Failed to load package", e, rootRelativePath, env.getListener());
    } catch (IOException | FileSymlinkException e) {
      throw new IllegalStateException(e);
    }

    TVisitor visitor = getInitialVisitor();

    DirectoryListingValue dirListingValue;
    try {
      dirListingValue =
          (DirectoryListingValue)
              Preconditions.checkNotNull(
                  pkgLookupAndDirectoryListingDeps.get(dirListingKey).get(),
                  "%s %s",
                  recursivePkgKey,
                  dirListingKey);
    } catch (InconsistentFilesystemException | IOException e) {
      return reportErrorAndReturn(
          "Failed to list directory contents", e, rootRelativePath, env.getListener());
    } catch (FileSymlinkException e) {
      // DirectoryListingFunction only throws FileSymlinkCycleException when FileFunction throws it,
      // but FileFunction was evaluated for rootedPath above, and didn't throw there. It shouldn't
      // be able to avoid throwing there but throw here.
      throw new IllegalStateException(
          "Symlink cycle found after not being found for \"" + rootedPath + "\"");
    } catch (NoSuchPackageException e) {
      throw new IllegalStateException(e);
    }

    boolean followSymlinks = shouldFollowSymlinksWhenTraversing(dirListingValue.getDirents());
    List<SkyKey> childDeps = new ArrayList<>();
    for (Dirent dirent : dirListingValue.getDirents()) {
      Type type = dirent.getType();
      if (type != Type.DIRECTORY
          && (type != Type.SYMLINK || (type == Type.SYMLINK && !followSymlinks))) {
        // Non-directories can never host packages. Symlinks to non-directories are weeded out at
        // the next level of recursion when we check if its FileValue is a directory. This is slower
        // if there are a lot of symlinks in the tree, but faster if there are only a few, which is
        // the case most of the time.
        //
        // We are not afraid of weird symlink structure here: both cyclical ones and ones that give
        // rise to infinite directory trees are diagnosed by FileValue.
        continue;
      }
      String basename = dirent.getName();
      if (rootRelativePath.equals(PathFragment.EMPTY_FRAGMENT)
          && PathPackageLocator.DEFAULT_TOP_LEVEL_EXCLUDES.contains(basename)) {
        continue;
      }
      PathFragment subdirectory = rootRelativePath.getRelative(basename);

      // If this subdirectory is one of the excluded paths, don't recurse into it.
      if (excludedPaths.contains(subdirectory)) {
        continue;
      }

      // If we have an excluded path that isn't below this subdirectory, we shouldn't pass that
      // excluded path to our evaluation of the subdirectory, because the exclusion can't
      // possibly match anything beneath the subdirectory.
      //
      // For example, if we're currently evaluating directory "a", are looking at its subdirectory
      // "a/b", and we have an excluded path "a/c/d", there's no need to pass the excluded path
      // "a/c/d" to our evaluation of "a/b".
      //
      // This strategy should help to get more skyframe sharing. Consider the example above. A
      // subsequent request of "a/b/...", without any excluded paths, will be a cache hit.
      //
      // TODO(bazel-team): Replace the excludedPaths set with a trie or a SortedSet for better
      // efficiency.
      ImmutableSet<PathFragment> excludedSubdirectoriesBeneathThisSubdirectory =
          PathFragment.filterPathsStartingWith(excludedPaths, subdirectory);
      RootedPath subdirectoryRootedPath = RootedPath.toRootedPath(root, subdirectory);
      childDeps.add(
          getSkyKeyForSubdirectory(
              recursivePkgKey.getRepository(),
              subdirectoryRootedPath,
              excludedSubdirectoriesBeneathThisSubdirectory));
    }

    Map<SkyKey, SkyValue> subdirectorySkyValues;
    if (pkgLookupValue.packageExists() && pkgLookupValue.getRoot().equals(root)) {
      SkyKey packageKey = PackageValue.key(packageId);
      Map<SkyKey, ValueOrException<NoSuchPackageException>> dependentSkyValues =
          env.getValuesOrThrow(
              Iterables.concat(childDeps, ImmutableList.of(packageKey)),
              NoSuchPackageException.class);
      if (env.valuesMissing()) {
        return null;
      }
      Package pkg = null;
      try {
        PackageValue pkgValue = (PackageValue) dependentSkyValues.get(packageKey).get();
        if (pkgValue == null) {
          return null;
        }
        pkg = pkgValue.getPackage();
        if (pkg.containsErrors()) {
          env.getListener()
              .handle(Event.error("package contains errors: " + rootRelativePath.getPathString()));
        }
      } catch (NoSuchPackageException e) {
        // The package had errors, but don't fail-fast as there might be subpackages below the
        // current directory.
        env.getListener()
            .handle(Event.error("package contains errors: " + rootRelativePath.getPathString()));
      }
      if (pkg != null) {
        visitor.visitPackageValue(pkg, env);
        if (env.valuesMissing()) {
          return null;
        }
      }
      ImmutableMap.Builder<SkyKey, SkyValue> subdirectoryBuilder = ImmutableMap.builder();
      for (Map.Entry<SkyKey, ValueOrException<NoSuchPackageException>> entry :
          Maps.filterKeys(dependentSkyValues, Predicates.not(Predicates.equalTo(packageKey)))
              .entrySet()) {
        try {
          subdirectoryBuilder.put(entry.getKey(), entry.getValue().get());
        } catch (NoSuchPackageException e) {
          // ignored.
        }
      }
      subdirectorySkyValues = subdirectoryBuilder.build();
    } else {
      subdirectorySkyValues = env.getValues(childDeps);
    }
    if (env.valuesMissing()) {
      return null;
    }
    return aggregateWithSubdirectorySkyValues(visitor, subdirectorySkyValues);
  }
예제 #18
0
 @Override
 public Map<String, String> getAll(final String prefix) {
   Preconditions.checkNotNull(prefix, "namespace can't be null");
   Predicate<String> filter = new RefPrefixPredicate(prefix);
   return Maps.filterKeys(ImmutableMap.copyOf(this.refs), filter);
 }
예제 #19
0
 protected Map<String, String> getProcessLayout(Profile profile, String layoutPath) {
   return ByteToStringValues.INSTANCE.apply(
       Maps.filterKeys(profile.getFileConfigurations(), new LayOutPredicate(layoutPath)));
 }
예제 #20
0
 public Message(final Map<String, Object> fields) {
   this((String) fields.get(FIELD_ID), Maps.filterKeys(fields, not(equalTo(FIELD_ID))));
 }
  @Override
  public ComputeService findComputeService(ConfigBag conf, boolean allowReuse) {
    String provider = checkNotNull(conf.get(CLOUD_PROVIDER), "provider must not be null");
    String identity =
        checkNotNull(conf.get(CloudLocationConfig.ACCESS_IDENTITY), "identity must not be null");
    String credential =
        checkNotNull(
            conf.get(CloudLocationConfig.ACCESS_CREDENTIAL), "credential must not be null");

    Properties properties = new Properties();
    properties.setProperty(Constants.PROPERTY_TRUST_ALL_CERTS, Boolean.toString(true));
    properties.setProperty(Constants.PROPERTY_RELAX_HOSTNAME, Boolean.toString(true));
    properties.setProperty(
        "jclouds.ssh.max-retries",
        conf.getStringKey("jclouds.ssh.max-retries") != null
            ? conf.getStringKey("jclouds.ssh.max-retries").toString()
            : "50");
    // Enable aws-ec2 lazy image fetching, if given a specific imageId; otherwise customize for
    // specific owners; or all as a last resort
    // See https://issues.apache.org/jira/browse/WHIRR-416
    if ("aws-ec2".equals(provider)) {
      // TODO convert AWS-only flags to config keys
      if (groovyTruth(conf.get(IMAGE_ID))) {
        properties.setProperty(PROPERTY_EC2_AMI_QUERY, "");
        properties.setProperty(PROPERTY_EC2_CC_AMI_QUERY, "");
      } else if (groovyTruth(conf.getStringKey("imageOwner"))) {
        properties.setProperty(
            PROPERTY_EC2_AMI_QUERY,
            "owner-id=" + conf.getStringKey("imageOwner") + ";state=available;image-type=machine");
      } else if (groovyTruth(conf.getStringKey("anyOwner"))) {
        // set `anyOwner: true` to override the default query (which is restricted to certain owners
        // as per below),
        // allowing the AMI query to bind to any machine
        // (note however, we sometimes pick defaults in JcloudsLocationFactory);
        // (and be careful, this can give a LOT of data back, taking several minutes,
        // and requiring extra memory allocated on the command-line)
        properties.setProperty(PROPERTY_EC2_AMI_QUERY, "state=available;image-type=machine");
        /*
         * by default the following filters are applied:
         * Filter.1.Name=owner-id&Filter.1.Value.1=137112412989&
         * Filter.1.Value.2=063491364108&
         * Filter.1.Value.3=099720109477&
         * Filter.1.Value.4=411009282317&
         * Filter.2.Name=state&Filter.2.Value.1=available&
         * Filter.3.Name=image-type&Filter.3.Value.1=machine&
         */
      }

      // occasionally can get com.google.common.util.concurrent.UncheckedExecutionException:
      // java.lang.RuntimeException:
      //     security group eu-central-1/jclouds#brooklyn-bxza-alex-eu-central-shoul-u2jy-nginx-ielm
      // is not available after creating
      // the default timeout was 500ms so let's raise it in case that helps
      properties.setProperty(
          EC2Constants.PROPERTY_EC2_TIMEOUT_SECURITYGROUP_PRESENT,
          "" + Duration.seconds(30).toMilliseconds());
    }

    // FIXME Deprecated mechanism, should have a ConfigKey for overrides
    Map<String, Object> extra =
        Maps.filterKeys(conf.getAllConfig(), Predicates.containsPattern("^jclouds\\."));
    if (extra.size() > 0) {
      LOG.warn("Jclouds using deprecated property overrides: " + Sanitizer.sanitize(extra));
    }
    properties.putAll(extra);

    String endpoint = conf.get(CloudLocationConfig.CLOUD_ENDPOINT);
    if (!groovyTruth(endpoint)) endpoint = getDeprecatedProperty(conf, Constants.PROPERTY_ENDPOINT);
    if (groovyTruth(endpoint)) properties.setProperty(Constants.PROPERTY_ENDPOINT, endpoint);

    Map<?, ?> cacheKey =
        MutableMap.builder()
            .putAll(properties)
            .put("provider", provider)
            .put("identity", identity)
            .put("credential", credential)
            .putIfNotNull("endpoint", endpoint)
            .build()
            .asUnmodifiable();

    if (allowReuse) {
      ComputeService result = cachedComputeServices.get(cacheKey);
      if (result != null) {
        LOG.trace(
            "jclouds ComputeService cache hit for compute service, for "
                + Sanitizer.sanitize(properties));
        return result;
      }
      LOG.debug(
          "jclouds ComputeService cache miss for compute service, creating, for "
              + Sanitizer.sanitize(properties));
    }

    Iterable<Module> modules = getCommonModules();

    // Synchronizing to avoid deadlock from sun.reflect.annotation.AnnotationType.
    // See https://github.com/brooklyncentral/brooklyn/issues/974
    ComputeServiceContext computeServiceContext;
    synchronized (createComputeServicesMutex) {
      computeServiceContext =
          ContextBuilder.newBuilder(provider)
              .modules(modules)
              .credentials(identity, credential)
              .overrides(properties)
              .build(ComputeServiceContext.class);
    }
    final ComputeService computeService = computeServiceContext.getComputeService();
    if (allowReuse) {
      synchronized (cachedComputeServices) {
        ComputeService result = cachedComputeServices.get(cacheKey);
        if (result != null) {
          LOG.debug(
              "jclouds ComputeService cache recovery for compute service, for "
                  + Sanitizer.sanitize(cacheKey));
          // keep the old one, discard the new one
          computeService.getContext().close();
          return result;
        }
        LOG.debug(
            "jclouds ComputeService created "
                + computeService
                + ", adding to cache, for "
                + Sanitizer.sanitize(properties));
        cachedComputeServices.put(cacheKey, computeService);
      }
    }
    return computeService;
  }
예제 #22
0
  @Override
  public ConnectorPartitionResult getPartitions(
      ConnectorSession session,
      ConnectorTableHandle tableHandle,
      TupleDomain<ColumnHandle> tupleDomain) {
    CassandraTableHandle cassandraTableHandle =
        checkType(tableHandle, CassandraTableHandle.class, "tableHandle");
    checkNotNull(tupleDomain, "tupleDomain is null");
    CassandraTable table = schemaProvider.getTable(cassandraTableHandle);
    List<CassandraColumnHandle> partitionKeys = table.getPartitionKeyColumns();

    // fetch the partitions
    List<CassandraPartition> allPartitions = getCassandraPartitions(table, tupleDomain);
    log.debug(
        "%s.%s #partitions: %d",
        cassandraTableHandle.getSchemaName(),
        cassandraTableHandle.getTableName(),
        allPartitions.size());

    // do a final pass to filter based on fields that could not be used to build the prefix
    List<ConnectorPartition> partitions =
        allPartitions
            .stream()
            .filter(partition -> tupleDomain.overlaps(partition.getTupleDomain()))
            .collect(toList());

    // All partition key domains will be fully evaluated, so we don't need to include those
    TupleDomain<ColumnHandle> remainingTupleDomain = TupleDomain.none();
    if (!tupleDomain.isNone()) {
      if (partitions.size() == 1 && ((CassandraPartition) partitions.get(0)).isUnpartitioned()) {
        remainingTupleDomain = tupleDomain;
      } else {
        @SuppressWarnings({"rawtypes", "unchecked"})
        List<ColumnHandle> partitionColumns = (List) partitionKeys;
        remainingTupleDomain =
            TupleDomain.withColumnDomains(
                Maps.filterKeys(tupleDomain.getDomains(), not(in(partitionColumns))));
      }
    }

    // push down indexed column fixed value predicates only for unpartitioned partition which uses
    // token range query
    if (partitions.size() == 1 && ((CassandraPartition) partitions.get(0)).isUnpartitioned()) {
      Map<ColumnHandle, Domain> domains = tupleDomain.getDomains();
      List<ColumnHandle> indexedColumns = new ArrayList<>();
      // compose partitionId by using indexed column
      StringBuilder sb = new StringBuilder();
      for (Map.Entry<ColumnHandle, Domain> entry : domains.entrySet()) {
        CassandraColumnHandle column = (CassandraColumnHandle) entry.getKey();
        Domain domain = entry.getValue();
        if (column.isIndexed() && domain.isSingleValue()) {
          sb.append(CassandraCqlUtils.validColumnName(column.getName()))
              .append(" = ")
              .append(
                  CassandraCqlUtils.cqlValue(
                      toCQLCompatibleString(entry.getValue().getSingleValue()),
                      column.getCassandraType()));
          indexedColumns.add(column);
          // Only one indexed column predicate can be pushed down.
          break;
        }
      }
      if (sb.length() > 0) {
        CassandraPartition partition = (CassandraPartition) partitions.get(0);
        TupleDomain<ColumnHandle> filterIndexedColumn =
            TupleDomain.withColumnDomains(
                Maps.filterKeys(remainingTupleDomain.getDomains(), not(in(indexedColumns))));
        partitions = new ArrayList<>();
        partitions.add(
            new CassandraPartition(partition.getKey(), sb.toString(), filterIndexedColumn, true));
        return new ConnectorPartitionResult(partitions, filterIndexedColumn);
      }
    }
    return new ConnectorPartitionResult(partitions, remainingTupleDomain);
  }