/** {@inheritDoc} */
  @Override
  public Map<K, V> peekAll(
      @Nullable Collection<? extends K> keys,
      @Nullable GridPredicate<? super GridCacheEntry<K, V>>[] filter) {
    if (keys == null || keys.isEmpty()) return emptyMap();

    final Collection<K> skipped = new GridLeanSet<K>();

    final Map<K, V> map = peekAll0(keys, filter, skipped);

    if (map.size() + skipped.size() != keys.size()) {
      map.putAll(
          dht.peekAll(
              F.view(
                  keys,
                  new P1<K>() {
                    @Override
                    public boolean apply(K k) {
                      return !map.containsKey(k) && !skipped.contains(k);
                    }
                  }),
              filter));
    }

    return map;
  }
  /** @param node Node to remove. */
  public void removeMappedNode(GridNode node) {
    if (mappedDhtNodes.contains(node))
      mappedDhtNodes = new ArrayList<>(F.view(mappedDhtNodes, F.notEqualTo(node)));

    if (mappedNearNodes != null && mappedNearNodes.contains(node))
      mappedNearNodes = new ArrayList<>(F.view(mappedNearNodes, F.notEqualTo(node)));
  }
  /** @throws Exception If failed. */
  public void testCreateFileColocated() throws Exception {
    GridGgfsPath path = new GridGgfsPath("/colocated");

    UUID uuid = UUID.randomUUID();

    GridUuid affKey;

    long idx = 0;

    while (true) {
      affKey = new GridUuid(uuid, idx);

      if (grid(0).mapKeyToNode(DATA_CACHE_NAME, affKey).id().equals(grid(0).localNode().id()))
        break;

      idx++;
    }

    try (GridGgfsOutputStream out = fs.create(path, 1024, true, affKey, 0, 1024, null)) {
      // Write 5M, should be enough to test distribution.
      for (int i = 0; i < 15; i++) out.write(new byte[1024 * 1024]);
    }

    GridGgfsFile info = fs.info(path);

    Collection<GridGgfsBlockLocation> affNodes = fs.affinity(path, 0, info.length());

    assertEquals(1, affNodes.size());
    Collection<UUID> nodeIds = F.first(affNodes).nodeIds();

    assertEquals(1, nodeIds.size());
    assertEquals(grid(0).localNode().id(), F.first(nodeIds));
  }
  /** {@inheritDoc} */
  @Override
  public final Map<UUID, GridNodeMetrics> metrics(Collection<UUID> nodeIds)
      throws GridSpiException {
    assert !F.isEmpty(nodeIds);

    long now = U.currentTimeMillis();

    Collection<UUID> expired = new LinkedList<>();

    for (UUID id : nodeIds) {
      GridNodeMetrics nodeMetrics = metricsMap.get(id);

      Long ts = tsMap.get(id);

      if (nodeMetrics == null || ts == null || ts < now - metricsExpireTime) expired.add(id);
    }

    if (!expired.isEmpty()) {
      Map<UUID, GridNodeMetrics> refreshed = metrics0(expired);

      for (UUID id : refreshed.keySet()) tsMap.put(id, now);

      metricsMap.putAll(refreshed);
    }

    return F.view(metricsMap, F.contains(nodeIds));
  }
Example #5
0
  /**
   * @param keys keys.
   * @param topVer Topology version.
   * @return Nodes for the keys.
   */
  public Collection<GridNode> remoteNodes(Iterable<? extends K> keys, long topVer) {
    Collection<Collection<GridNode>> colcol = new GridLeanSet<>();

    for (K key : keys) colcol.add(nodes(key, topVer));

    return F.view(F.flatCollections(colcol), F.remoteNodes(cctx.localNodeId()));
  }
Example #6
0
  /**
   * @param part Partition.
   * @param topVer Topology version.
   * @return Backup nodes.
   */
  public Collection<GridNode> backups(int part, long topVer) {
    Collection<GridNode> nodes = nodes(part, topVer);

    assert !F.isEmpty(nodes);

    if (nodes.size() <= 1) return Collections.emptyList();

    return F.view(nodes, F.notEqualTo(nodes.iterator().next()));
  }
  /** @throws Exception If failed. */
  public void testAffinityPut() throws Exception {
    Thread.sleep(2 * TOP_REFRESH_FREQ);

    assertEquals(NODES_CNT, client.compute().refreshTopology(false, false).size());

    Map<UUID, Grid> gridsByLocNode = new HashMap<>(NODES_CNT);

    GridClientData partitioned = client.data(PARTITIONED_CACHE_NAME);

    GridClientCompute compute = client.compute();

    for (int i = 0; i < NODES_CNT; i++) gridsByLocNode.put(grid(i).localNode().id(), grid(i));

    for (int i = 0; i < 100; i++) {
      String key = "key" + i;

      UUID primaryNodeId = grid(0).mapKeyToNode(PARTITIONED_CACHE_NAME, key).id();

      assertEquals("Affinity mismatch for key: " + key, primaryNodeId, partitioned.affinity(key));

      assertEquals(primaryNodeId, partitioned.affinity(key));

      // Must go to primary node only. Since backup count is 0, value must present on
      // primary node only.
      partitioned.put(key, "val" + key);

      for (Map.Entry<UUID, Grid> entry : gridsByLocNode.entrySet()) {
        Object val = entry.getValue().cache(PARTITIONED_CACHE_NAME).peek(key);

        if (primaryNodeId.equals(entry.getKey())) assertEquals("val" + key, val);
        else assertNull(val);
      }
    }

    // Now check that we will see value in near cache in pinned mode.
    for (int i = 100; i < 200; i++) {
      String pinnedKey = "key" + i;

      UUID primaryNodeId = grid(0).mapKeyToNode(PARTITIONED_CACHE_NAME, pinnedKey).id();

      UUID pinnedNodeId = F.first(F.view(gridsByLocNode.keySet(), F.notEqualTo(primaryNodeId)));

      GridClientNode node = compute.node(pinnedNodeId);

      partitioned.pinNodes(node).put(pinnedKey, "val" + pinnedKey);

      for (Map.Entry<UUID, Grid> entry : gridsByLocNode.entrySet()) {
        Object val = entry.getValue().cache(PARTITIONED_CACHE_NAME).peek(pinnedKey);

        if (primaryNodeId.equals(entry.getKey()) || pinnedNodeId.equals(entry.getKey()))
          assertEquals("val" + pinnedKey, val);
        else assertNull(val);
      }
    }
  }
Example #8
0
 /** {@inheritDoc} */
 @Override
 public void stopListenAsync(@Nullable GridInClosure<? super GridFuture<R>>... lsnr) {
   if (F.isEmpty(lsnr))
     synchronized (mux) {
       lsnrs.clear();
     }
   else
     synchronized (mux) {
       lsnrs.removeAll(F.asList(lsnr));
     }
 }
  /** @throws Exception If failed. */
  public void testTopologyListener() throws Exception {
    final Collection<UUID> added = new ArrayList<>(1);
    final Collection<UUID> rmvd = new ArrayList<>(1);

    final CountDownLatch addedLatch = new CountDownLatch(1);
    final CountDownLatch rmvLatch = new CountDownLatch(1);

    assertEquals(NODES_CNT, client.compute().refreshTopology(false, false).size());

    GridClientTopologyListener lsnr =
        new GridClientTopologyListener() {
          @Override
          public void onNodeAdded(GridClientNode node) {
            added.add(node.nodeId());

            addedLatch.countDown();
          }

          @Override
          public void onNodeRemoved(GridClientNode node) {
            rmvd.add(node.nodeId());

            rmvLatch.countDown();
          }
        };

    client.addTopologyListener(lsnr);

    try {
      Grid g = startGrid(NODES_CNT + 1);

      UUID id = g.localNode().id();

      assertTrue(addedLatch.await(2 * TOP_REFRESH_FREQ, MILLISECONDS));

      assertEquals(1, added.size());
      assertEquals(id, F.first(added));

      stopGrid(NODES_CNT + 1);

      assertTrue(rmvLatch.await(2 * TOP_REFRESH_FREQ, MILLISECONDS));

      assertEquals(1, rmvd.size());
      assertEquals(id, F.first(rmvd));
    } finally {
      client.removeTopologyListener(lsnr);

      stopGrid(NODES_CNT + 1);
    }
  }
 /** {@inheritDoc} */
 @Override
 public GridDeployment getDeployment(final GridUuid ldrId) {
   synchronized (mux) {
     return F.find(
         F.flat(cache.values()),
         null,
         new P1<SharedDeployment>() {
           @Override
           public boolean apply(SharedDeployment d) {
             return d.classLoaderId().equals(ldrId);
           }
         });
   }
 }
  /**
   * Creates signature for data.
   *
   * @param alg Algorithm.
   * @param prv Provider.
   * @param key Private key.
   * @param data Data.
   * @return Signature.
   * @throws GeneralSecurityException If any exception occurs while signature creation.
   */
  public static byte[] createSignature(String alg, String prv, PrivateKey key, byte[] data)
      throws GeneralSecurityException {
    assert !F.isEmpty(alg);
    assert !F.isEmpty(prv);
    assert key != null;
    assert data != null;

    Signature sign = Signature.getInstance(alg, prv);

    sign.initSign(key);
    sign.update(data);

    return sign.sign();
  }
  /**
   * @param rmtReducer Optional reducer.
   * @param rmtTransform Optional transformer.
   * @param args Arguments.
   * @return Future.
   */
  @SuppressWarnings("IfMayBeConditional")
  private <R> GridCacheQueryFuture<R> execute(
      @Nullable GridReducer<T, R> rmtReducer,
      @Nullable GridClosure<T, R> rmtTransform,
      @Nullable Object... args) {
    Collection<GridNode> nodes = nodes();

    cctx.checkSecurity(GridSecurityPermission.CACHE_READ);

    if (F.isEmpty(nodes))
      return new GridCacheQueryErrorFuture<>(
          cctx.kernalContext(),
          new GridEmptyProjectionException("There are no data nodes for cache: " + cctx.namexx()));

    if (log.isDebugEnabled())
      log.debug("Executing query [query=" + this + ", nodes=" + nodes + ']');

    if (cctx.deploymentEnabled()) {
      try {
        cctx.deploy().registerClasses(filter, rmtReducer, rmtTransform);
        cctx.deploy().registerClasses(args);
      } catch (GridException e) {
        return new GridCacheQueryErrorFuture<>(cctx.kernalContext(), e);
      }
    }

    if (subjId == null) subjId = cctx.localNodeId();

    taskHash = cctx.kernalContext().job().currentTaskNameHash();

    GridCacheQueryBean bean =
        new GridCacheQueryBean(
            this,
            (GridReducer<Object, Object>) rmtReducer,
            (GridClosure<Object, Object>) rmtTransform,
            args);

    GridCacheQueryManager qryMgr = cctx.queries();

    boolean loc = nodes.size() == 1 && F.first(nodes).id().equals(cctx.localNodeId());

    if (type == SQL_FIELDS)
      return (GridCacheQueryFuture<R>)
          (loc ? qryMgr.queryFieldsLocal(bean) : qryMgr.queryFieldsDistributed(bean, nodes));
    else
      return (GridCacheQueryFuture<R>)
          (loc ? qryMgr.queryLocal(bean) : qryMgr.queryDistributed(bean, nodes));
  }
Example #13
0
  /** {@inheritDoc} */
  @Override
  public boolean apply(@Nullable T1 t1, @Nullable T2 t2) {
    lazyCompile();

    if (expr != null) {
      JexlContext ctx = new MapContext();

      ctx.set(var1, t1);
      ctx.set(var2, t2);

      for (Map.Entry<String, Object> e : map.entrySet()) {
        ctx.set(e.getKey(), e.getValue());
      }

      try {
        Object obj = expr.evaluate(ctx);

        if (obj instanceof Boolean) {
          return (Boolean) obj;
        }
      } catch (Exception ex) {
        throw F.wrap(ex);
      }
    }

    return false;
  }
  /** @return Nodes to execute on. */
  private Collection<GridNode> nodes() {
    GridCacheMode cacheMode = cctx.config().getCacheMode();

    switch (cacheMode) {
      case LOCAL:
        if (prj != null)
          U.warn(
              log,
              "Ignoring query projection because it's executed over LOCAL cache "
                  + "(only local node will be queried): "
                  + this);

        return Collections.singletonList(cctx.localNode());

      case REPLICATED:
        if (prj != null) return nodes(cctx, prj);

        GridCacheDistributionMode mode = cctx.config().getDistributionMode();

        return mode == PARTITIONED_ONLY || mode == NEAR_PARTITIONED
            ? Collections.singletonList(cctx.localNode())
            : Collections.singletonList(F.rand(nodes(cctx, null)));

      case PARTITIONED:
        return nodes(cctx, prj);

      default:
        throw new IllegalStateException("Unknown cache distribution mode: " + cacheMode);
    }
  }
    /**
     * @return Query info.
     * @throws GridException In case of error.
     */
    @SuppressWarnings({"unchecked"})
    private GridCacheQueryInfo localQueryInfo() throws GridException {
      GridCacheQueryBean qry = query();

      GridPredicate<GridCacheEntry<Object, Object>> prjPred =
          qry.query().projectionFilter() == null
              ? F.<GridCacheEntry<Object, Object>>alwaysTrue()
              : qry.query().projectionFilter();

      GridMarshaller marsh = cctx.marshaller();

      GridReducer<Object, Object> rdc =
          qry.reducer() != null
              ? marsh.<GridReducer<Object, Object>>unmarshal(marsh.marshal(qry.reducer()), null)
              : null;

      GridClosure<Object, Object> trans =
          qry.transform() != null
              ? marsh.<GridClosure<Object, Object>>unmarshal(marsh.marshal(qry.transform()), null)
              : null;

      return new GridCacheQueryInfo(
          true,
          prjPred,
          trans,
          rdc,
          qry.query(),
          GridCacheLocalQueryFuture.this,
          ctx.localNodeId(),
          cctx.io().nextIoId(),
          qry.query().includeMetadata(),
          true,
          qry.arguments());
    }
  /**
   * Verifies signature for data.
   *
   * @param alg Algorithm.
   * @param prv Provider.
   * @param key Public key.
   * @param data Data.
   * @param sign Signature.
   * @return {@code true} if signature was verified, {@code false} - otherwise.
   * @throws GeneralSecurityException If any exception occurs while verifying.
   */
  public static boolean verifySignature(
      String alg, String prv, PublicKey key, byte[] data, byte[] sign)
      throws GeneralSecurityException {
    assert !F.isEmpty(alg);
    assert !F.isEmpty(prv);
    assert key != null;
    assert data != null;
    assert sign != null;

    Signature sign0 = Signature.getInstance(alg, prv);

    sign0.initVerify(key);
    sign0.update(data);

    return sign0.verify(sign);
  }
  /** {@inheritDoc} */
  @Override
  public V unswap(K key) throws GridException {
    ctx.denyOnFlags(F.asList(READ, SKIP_SWAP));

    // Unswap only from DHT. Near cache does not have swap storage.
    return dht.unswap(key);
  }
Example #18
0
  /**
   * @param mapping Mapping to order.
   * @param committedVers Committed versions.
   * @param rolledbackVers Rolled back versions.
   */
  void orderCompleted(
      GridDistributedTxMapping<K, V> mapping,
      Collection<GridCacheVersion> committedVers,
      Collection<GridCacheVersion> rolledbackVers) {
    for (GridCacheTxEntry<K, V> txEntry : F.concat(false, mapping.reads(), mapping.writes())) {
      while (true) {
        GridDistributedCacheEntry<K, V> entry = (GridDistributedCacheEntry<K, V>) txEntry.cached();

        try {
          // Handle explicit locks.
          GridCacheVersion base =
              txEntry.explicitVersion() != null ? txEntry.explicitVersion() : xidVer;

          entry.doneRemote(xidVer, base, committedVers, rolledbackVers);

          if (ec()) entry.recheck();

          break;
        } catch (GridCacheEntryRemovedException ignored) {
          assert entry.obsoleteVersion() != null;

          if (log.isDebugEnabled())
            log.debug(
                "Replacing obsolete entry in remote transaction [entry="
                    + entry
                    + ", tx="
                    + this
                    + ']');

          // Replace the entry.
          txEntry.cached(cctx.cache().entryEx(txEntry.key()), entry.keyBytes());
        }
      }
    }
  }
  /**
   * Prints a phrase on the grid nodes running anonymous closure objects and calculating total
   * number of letters.
   *
   * @param phrase Phrase to print on of the grid nodes.
   * @throws GridException If failed.
   */
  private static void countLettersClosure(String phrase) throws GridException {
    X.println(">>> Starting countLettersClosure() example...");

    // Explicitly execute the collection of callable objects and receive a result.
    Collection<Integer> results =
        G.grid()
            .call(
                SPREAD,
                new GridClosure<String, Integer>() { // Create executable logic.
                  @Override
                  public Integer apply(String word) {
                    // Print out a given word, just so we can
                    // see which node is doing what.
                    X.println(">>> Executing word: " + word);

                    // Return the length of a given word, i.e. number of letters.
                    return word.length();
                  }
                },
                Arrays.asList(phrase.split(" "))); // Collection of arguments for closures.

    // Add up all results using convenience 'sum()' method.
    int letterCnt = F.sum(results);

    X.println(">>>");
    X.println(">>> Finished execution of counting letters with closure based on GridGain 3.0 API.");
    X.println(">>> You should see the phrase '" + phrase + "' printed out on the nodes.");
    X.println(">>> Total number of letters in the phrase is '" + letterCnt + "'.");
    X.println(">>> Check all nodes for output (this node is also part of the grid).");
    X.println(">>>");
  }
  /** {@inheritDoc} */
  @Override
  public Map<K, V> peekAll(
      @Nullable Collection<? extends K> keys, @Nullable Collection<GridCachePeekMode> modes)
      throws GridException {
    if (keys == null || keys.isEmpty()) return emptyMap();

    final Collection<K> skipped = new GridLeanSet<K>();

    final Map<K, V> map =
        !modes.contains(PARTITIONED_ONLY)
            ? peekAll0(keys, modes, ctx.tm().localTxx(), skipped)
            : new GridLeanMap<K, V>(0);

    if (map.size() != keys.size() && !modes.contains(NEAR_ONLY)) {
      map.putAll(
          dht.peekAll(
              F.view(
                  keys,
                  new P1<K>() {
                    @Override
                    public boolean apply(K k) {
                      return !map.containsKey(k) && !skipped.contains(k);
                    }
                  }),
              modes));
    }

    return map;
  }
  /**
   * Prints a phrase on the grid nodes running anonymous callable objects and calculating total
   * number of letters.
   *
   * @param phrase Phrase to print on of the grid nodes.
   * @throws GridException If failed.
   */
  private static void countLettersCallable(String phrase) throws GridException {
    X.println(">>> Starting countLettersCallable() example...");

    Collection<Callable<Integer>> calls = new HashSet<Callable<Integer>>();

    for (final String word : phrase.split(" "))
      calls.add(
          new GridCallable<Integer>() { // Create executable logic.
            @Override
            public Integer call() throws Exception {
              // Print out a given word, just so we can
              // see which node is doing what.
              X.println(">>> Executing word: " + word);

              // Return the length of a given word, i.e. number of letters.
              return word.length();
            }
          });

    // Explicitly execute the collection of callable objects and receive a result.
    Collection<Integer> results = G.grid().call(SPREAD, calls);

    // Add up all results using convenience 'sum()' method on GridFunc class.
    int letterCnt = F.sum(results);

    X.println(">>>");
    X.println(
        ">>> Finished execution of counting letters with callables based on GridGain 3.0 API.");
    X.println(">>> You should see the phrase '" + phrase + "' printed out on the nodes.");
    X.println(">>> Total number of letters in the phrase is '" + letterCnt + "'.");
    X.println(">>> Check all nodes for output (this node is also part of the grid).");
    X.println(">>>");
  }
  /**
   * Prints every word a phrase on different nodes.
   *
   * @param phrase Phrase from which to print words on different nodes.
   * @throws GridException If failed.
   */
  private static void spreadWordsClosure(String phrase) throws GridException {
    X.println(">>> Starting spreadWordsClosure() example...");

    // Splits the passed in phrase into words and prints every word
    // on a individual grid node. If there are more words than nodes -
    // some nodes will print more than one word.
    G.grid()
        .run(
            SPREAD,
            F.yield(
                phrase.split(" "),
                new GridInClosure<String>() {
                  @Override
                  public void apply(String word) {
                    X.println(word);
                  }
                }));

    // NOTE:
    //
    // Alternatively, you can use existing closure 'F.println()' to
    // print any yield result in 'F.yield()' like so:
    //
    // G.grid().run(SPREAD, F.yield(phrase.split(" "), F.println()));
    //

    X.println(">>>");
    X.println(
        ">>> Finished printing individual words on different nodes based on GridGain 3.0 API.");
    X.println(">>> Check all nodes for output (this node is also part of the grid).");
    X.println(">>>");
  }
 /** {@inheritDoc} */
 @Override
 public boolean apply() {
   try {
     return applyx();
   } catch (GridException ex) {
     throw F.wrap(ex);
   }
 }
Example #24
0
 /** {@inheritDoc} */
 @Override
 public R apply() {
   try {
     return applyx();
   } catch (GridException e) {
     throw F.wrap(e);
   }
 }
 /** {@inheritDoc} */
 @Override
 public <T> T affinityKey() {
   try {
     return (T) job.getDeployment().annotatedValue(job.getJob(), GridCacheAffinityMapped.class);
   } catch (GridException e) {
     throw F.wrap(e);
   }
 }
 /** {@inheritDoc} */
 @Override
 public String cacheName() {
   try {
     return (String) job.getDeployment().annotatedValue(job.getJob(), GridCacheName.class);
   } catch (GridException e) {
     throw F.wrap(e);
   }
 }
Example #27
0
  /**
   * Creates node predicate that evaluates to {@code true} for all provided node IDs. Implementation
   * will make a defensive copy.
   *
   * @param ids Optional node IDs. If none provided - predicate will always return {@code false}.
   */
  public GridNodePredicate(@Nullable UUID... ids) {
    if (F.isEmpty(ids)) this.ids = Collections.emptySet();
    else if (ids.length == 1) this.ids = Collections.singleton(ids[0]);
    else {
      this.ids = new HashSet<>(ids.length);

      Collections.addAll(this.ids, ids);
    }
  }
Example #28
0
  /**
   * Creates node predicate that evaluates to {@code true} for all provided nodes. Implementation
   * will make a defensive copy.
   *
   * @param nodes Optional grid nodes. If none provided - predicate will always return {@code
   *     false}.
   */
  public GridNodePredicate(@Nullable GridNode... nodes) {
    if (F.isEmpty(nodes)) ids = Collections.emptySet();
    else if (nodes.length == 1) ids = Collections.singleton(nodes[0].id());
    else {
      ids = new HashSet<>(nodes.length);

      for (GridNode n : nodes) ids.add(n.id());
    }
  }
Example #29
0
  /**
   * Add information about key and version to request.
   *
   * <p>Other version has to be provided if suspect lock is DHT local.
   *
   * @param key Key.
   * @param cand Candidate.
   */
  @SuppressWarnings({"MismatchedQueryAndUpdateOfCollection"})
  void addCandidate(K key, GridCacheDgcLockCandidate cand) {
    Collection<GridCacheDgcLockCandidate> col =
        F.addIfAbsent(map, key, new ArrayList<GridCacheDgcLockCandidate>());

    assert col != null;

    col.add(cand);
  }
 /**
  * @param g Grid.
  * @return Non-system caches.
  */
 private Collection<GridCacheConfiguration> caches(Grid g) {
   return F.view(
       Arrays.asList(g.configuration().getCacheConfiguration()),
       new GridPredicate<GridCacheConfiguration>() {
         @Override
         public boolean apply(GridCacheConfiguration c) {
           return c.getName() == null || !c.getName().equals(CU.UTILITY_CACHE_NAME);
         }
       });
 }