/**
   * JUnit.
   *
   * @throws Exception If failed.
   */
  public void testGetAndIncrement() throws Exception {
    Collection<Long> res = new HashSet<>();

    String seqName = UUID.randomUUID().toString();

    for (int i = 0; i < GRID_CNT; i++) {
      Set<Long> retVal =
          compute(grid(i).cluster().forLocal()).call(new GetAndIncrementJob(seqName, RETRIES));

      for (Long l : retVal) assert !res.contains(l) : "Value already was used " + l;

      res.addAll(retVal);
    }

    assert res.size() == GRID_CNT * RETRIES;

    int gapSize = 0;

    for (long i = 0; i < GRID_CNT * RETRIES; i++) {
      if (!res.contains(i)) gapSize++;
      else gapSize = 0;

      assert gapSize <= BATCH_SIZE + 1
          : "Gap above id  " + i + " is " + gapSize + " more than batch size: " + (BATCH_SIZE + 1);
    }
  }
  /**
   * Change topology.
   *
   * @param parent Grid to execute tasks on.
   * @param add New nodes count.
   * @param rmv Remove nodes count.
   * @param type Type of nodes to manipulate.
   */
  private static void changeTopology(Ignite parent, int add, int rmv, String type) {
    Collection<ComputeTaskFuture<?>> tasks = new ArrayList<>();

    IgniteCompute comp = parent.compute().withAsync();

    // Start nodes in parallel.
    while (add-- > 0) {
      comp.execute(ClientStartNodeTask.class, type);

      tasks.add(comp.future());
    }

    for (ComputeTaskFuture<?> task : tasks) task.get();

    // Stop nodes in sequence.
    while (rmv-- > 0) parent.compute().execute(ClientStopNodeTask.class, type);

    // Wait for node stops.
    // U.sleep(1000);

    Collection<String> gridNames = new ArrayList<>();

    for (Ignite g : G.allGrids()) gridNames.add(g.name());

    parent.log().info(">>> Available grids: " + gridNames);
  }
  /** {@inheritDoc} */
  @Override
  public void finishUnmarshal(GridCacheSharedContext ctx, ClassLoader ldr)
      throws IgniteCheckedException {
    super.finishUnmarshal(ctx, ldr);

    if (writes != null) unmarshalTx(writes, false, ctx, ldr);

    if (reads != null) unmarshalTx(reads, false, ctx, ldr);

    if (grpLockKeyBytes != null && grpLockKey == null)
      grpLockKey = ctx.marshaller().unmarshal(grpLockKeyBytes, ldr);

    if (dhtVerKeys != null && dhtVers == null) {
      assert dhtVerVals != null;
      assert dhtVerKeys.size() == dhtVerVals.size();

      Iterator<IgniteTxKey> keyIt = dhtVerKeys.iterator();
      Iterator<GridCacheVersion> verIt = dhtVerVals.iterator();

      dhtVers = U.newHashMap(dhtVerKeys.size());

      while (keyIt.hasNext()) {
        IgniteTxKey key = keyIt.next();

        key.finishUnmarshal(ctx.cacheContext(key.cacheId()), ldr);

        dhtVers.put(key, verIt.next());
      }
    }

    if (txNodesBytes != null) txNodes = ctx.marshaller().unmarshal(txNodesBytes, ldr);
  }
  private void recheck() {
    // If this is the oldest node.
    if (oldestNode.get().id().equals(cctx.localNodeId())) {
      Collection<UUID> remaining = remaining();

      if (!remaining.isEmpty()) {
        try {
          cctx.io()
              .safeSend(
                  cctx.discovery().nodes(remaining),
                  new GridDhtPartitionsSingleRequest(exchId),
                  SYSTEM_POOL,
                  null);
        } catch (IgniteCheckedException e) {
          U.error(
              log,
              "Failed to request partitions from nodes [exchangeId="
                  + exchId
                  + ", nodes="
                  + remaining
                  + ']',
              e);
        }
      }
      // Resend full partition map because last attempt failed.
      else {
        if (spreadPartitions()) onDone(exchId.topologyVersion());
      }
    } else sendPartitions();

    // Schedule another send.
    scheduleRecheck();
  }
  /**
   * Sends query request.
   *
   * @param fut Distributed future.
   * @param req Request.
   * @param nodes Nodes.
   * @throws IgniteCheckedException In case of error.
   */
  @SuppressWarnings("unchecked")
  private void sendRequest(
      final GridCacheDistributedQueryFuture<?, ?, ?> fut,
      final GridCacheQueryRequest req,
      Collection<ClusterNode> nodes)
      throws IgniteCheckedException {
    assert fut != null;
    assert req != null;
    assert nodes != null;

    final UUID locNodeId = cctx.localNodeId();

    ClusterNode locNode = null;

    Collection<ClusterNode> rmtNodes = null;

    for (ClusterNode n : nodes) {
      if (n.id().equals(locNodeId)) locNode = n;
      else {
        if (rmtNodes == null) rmtNodes = new ArrayList<>(nodes.size());

        rmtNodes.add(n);
      }
    }

    // Request should be sent to remote nodes before the query is processed on the local node.
    // For example, a remote reducer has a state, we should not serialize and then send
    // the reducer changed by the local node.
    if (!F.isEmpty(rmtNodes)) {
      cctx.io()
          .safeSend(
              rmtNodes,
              req,
              cctx.ioPolicy(),
              new P1<ClusterNode>() {
                @Override
                public boolean apply(ClusterNode node) {
                  fut.onNodeLeft(node.id());

                  return !fut.isDone();
                }
              });
    }

    if (locNode != null) {
      cctx.closures()
          .callLocalSafe(
              new Callable<Object>() {
                @Override
                public Object call() throws Exception {
                  req.beforeLocalExecution(cctx);

                  processQueryRequest(locNodeId, req);

                  return null;
                }
              });
    }
  }
  /** Cleans up resources to avoid excessive memory usage. */
  public void cleanUp() {
    topSnapshot.set(null);
    singleMsgs.clear();
    fullMsgs.clear();
    rcvdIds.clear();
    oldestNode.set(null);
    partReleaseFut = null;

    Collection<ClusterNode> rmtNodes = this.rmtNodes;

    if (rmtNodes != null) rmtNodes.clear();
  }
  /**
   * @param cacheCtx Cache context.
   * @return {@code True} if local node can calculate affinity on it's own for this partition map
   *     exchange.
   */
  private boolean canCalculateAffinity(GridCacheContext cacheCtx) {
    AffinityFunction affFunc = cacheCtx.config().getAffinity();

    // Do not request affinity from remote nodes if affinity function is not centralized.
    if (!U.hasAnnotation(affFunc, AffinityCentralizedFunction.class)) return true;

    // If local node did not initiate exchange or local node is the only cache node in grid.
    Collection<ClusterNode> affNodes = CU.affinityNodes(cacheCtx, exchId.topologyVersion());

    return !exchId.nodeId().equals(cctx.localNodeId())
        || (affNodes.size() == 1 && affNodes.contains(cctx.localNode()));
  }
Exemplo n.º 8
0
  /**
   * Returns compact class host.
   *
   * @param obj Object to compact.
   * @return String.
   */
  @Nullable
  public static Object compactObject(Object obj) {
    if (obj == null) return null;

    if (obj instanceof Enum) return obj.toString();

    if (obj instanceof String || obj instanceof Boolean || obj instanceof Number) return obj;

    if (obj instanceof Collection) {
      Collection col = (Collection) obj;

      Object[] res = new Object[col.size()];

      int i = 0;

      for (Object elm : col) res[i++] = compactObject(elm);

      return res;
    }

    if (obj.getClass().isArray()) {
      Class<?> arrType = obj.getClass().getComponentType();

      if (arrType.isPrimitive()) {
        if (obj instanceof boolean[]) return Arrays.toString((boolean[]) obj);
        if (obj instanceof byte[]) return Arrays.toString((byte[]) obj);
        if (obj instanceof short[]) return Arrays.toString((short[]) obj);
        if (obj instanceof int[]) return Arrays.toString((int[]) obj);
        if (obj instanceof long[]) return Arrays.toString((long[]) obj);
        if (obj instanceof float[]) return Arrays.toString((float[]) obj);
        if (obj instanceof double[]) return Arrays.toString((double[]) obj);
      }

      Object[] arr = (Object[]) obj;

      int iMax = arr.length - 1;

      StringBuilder sb = new StringBuilder("[");

      for (int i = 0; i <= iMax; i++) {
        sb.append(compactObject(arr[i]));

        if (i != iMax) sb.append(", ");
      }

      sb.append("]");

      return sb.toString();
    }

    return U.compact(obj.getClass().getName());
  }
  /**
   * @param rmtReducer Optional reducer.
   * @param rmtTransform Optional transformer.
   * @param args Arguments.
   * @return Future.
   */
  @SuppressWarnings("IfMayBeConditional")
  private <R> CacheQueryFuture<R> execute(
      @Nullable IgniteReducer<T, R> rmtReducer,
      @Nullable IgniteClosure<T, R> rmtTransform,
      @Nullable Object... args) {
    Collection<ClusterNode> nodes = nodes();

    cctx.checkSecurity(SecurityPermission.CACHE_READ);

    if (nodes.isEmpty())
      return new GridCacheQueryErrorFuture<>(
          cctx.kernalContext(), new ClusterGroupEmptyCheckedException());

    if (log.isDebugEnabled())
      log.debug("Executing query [query=" + this + ", nodes=" + nodes + ']');

    if (cctx.deploymentEnabled()) {
      try {
        cctx.deploy().registerClasses(filter, rmtReducer, rmtTransform);
        cctx.deploy().registerClasses(args);
      } catch (IgniteCheckedException e) {
        return new GridCacheQueryErrorFuture<>(cctx.kernalContext(), e);
      }
    }

    if (subjId == null) subjId = cctx.localNodeId();

    taskHash = cctx.kernalContext().job().currentTaskNameHash();

    final GridCacheQueryBean bean =
        new GridCacheQueryBean(
            this,
            (IgniteReducer<Object, Object>) rmtReducer,
            (IgniteClosure<Object, Object>) rmtTransform,
            args);

    final GridCacheQueryManager qryMgr = cctx.queries();

    boolean loc = nodes.size() == 1 && F.first(nodes).id().equals(cctx.localNodeId());

    if (type == SQL_FIELDS || type == SPI)
      return (CacheQueryFuture<R>)
          (loc ? qryMgr.queryFieldsLocal(bean) : qryMgr.queryFieldsDistributed(bean, nodes));
    else if (type == SCAN && part != null && nodes.size() > 1)
      return new CacheQueryFallbackFuture<>(nodes, bean, qryMgr);
    else
      return (CacheQueryFuture<R>)
          (loc ? qryMgr.queryLocal(bean) : qryMgr.queryDistributed(bean, nodes));
  }
  /**
   * Processes cache query response.
   *
   * @param sndId Sender node id.
   * @param res Query response.
   */
  @SuppressWarnings("unchecked")
  private void processQueryResponse(UUID sndId, GridCacheQueryResponse res) {
    if (log.isDebugEnabled()) log.debug("Received query response: " + res);

    GridCacheQueryFutureAdapter fut = getQueryFuture(res.requestId());

    if (fut != null)
      if (res.fields())
        ((GridCacheDistributedFieldsQueryFuture) fut)
            .onPage(
                sndId,
                res.metadata(),
                (Collection<Map<String, Object>>) ((Collection) res.data()),
                res.error(),
                res.isFinished());
      else fut.onPage(sndId, res.data(), res.error(), res.isFinished());
    else if (!cancelled.contains(res.requestId()))
      U.warn(
          log,
          "Received response for finished or unknown query [rmtNodeId="
              + sndId
              + ", res="
              + res
              + ']');
  }
  /**
   * @param p Partition.
   * @param topVer Topology version ({@code -1} for all nodes).
   * @param state Partition state.
   * @param states Additional partition states.
   * @return List of nodes for the partition.
   */
  private List<ClusterNode> nodes(
      int p,
      AffinityTopologyVersion topVer,
      GridDhtPartitionState state,
      GridDhtPartitionState... states) {
    Collection<UUID> allIds =
        topVer.topologyVersion() > 0 ? F.nodeIds(CU.affinityNodes(cctx, topVer)) : null;

    lock.readLock().lock();

    try {
      assert node2part != null && node2part.valid()
          : "Invalid node-to-partitions map [topVer="
              + topVer
              + ", allIds="
              + allIds
              + ", node2part="
              + node2part
              + ", cache="
              + cctx.name()
              + ']';

      Collection<UUID> nodeIds = part2node.get(p);

      // Node IDs can be null if both, primary and backup, nodes disappear.
      int size = nodeIds == null ? 0 : nodeIds.size();

      if (size == 0) return Collections.emptyList();

      List<ClusterNode> nodes = new ArrayList<>(size);

      for (UUID id : nodeIds) {
        if (topVer.topologyVersion() > 0 && !allIds.contains(id)) continue;

        if (hasState(p, id, state, states)) {
          ClusterNode n = cctx.discovery().node(id);

          if (n != null && (topVer.topologyVersion() < 0 || n.order() <= topVer.topologyVersion()))
            nodes.add(n);
        }
      }

      return nodes;
    } finally {
      lock.readLock().unlock();
    }
  }
  /**
   * Processes cache query request.
   *
   * @param sndId Sender node id.
   * @param req Query request.
   */
  @SuppressWarnings("unchecked")
  @Override
  void processQueryRequest(UUID sndId, GridCacheQueryRequest req) {
    if (req.cancel()) {
      cancelIds.add(new CancelMessageId(req.id(), sndId));

      if (req.fields()) removeFieldsQueryResult(sndId, req.id());
      else removeQueryResult(sndId, req.id());
    } else {
      if (!cancelIds.contains(new CancelMessageId(req.id(), sndId))) {
        if (!F.eq(req.cacheName(), cctx.name())) {
          GridCacheQueryResponse res =
              new GridCacheQueryResponse(
                  cctx.cacheId(),
                  req.id(),
                  new IgniteCheckedException(
                      "Received request for incorrect cache [expected="
                          + cctx.name()
                          + ", actual="
                          + req.cacheName()));

          sendQueryResponse(sndId, res, 0);
        } else {
          threads.put(req.id(), Thread.currentThread());

          try {
            GridCacheQueryInfo info = distributedQueryInfo(sndId, req);

            if (info == null) return;

            if (req.fields()) runFieldsQuery(info);
            else runQuery(info);
          } catch (Throwable e) {
            U.error(log(), "Failed to run query.", e);

            sendQueryResponse(
                sndId, new GridCacheQueryResponse(cctx.cacheId(), req.id(), e.getCause()), 0);

            if (e instanceof Error) throw (Error) e;
          } finally {
            threads.remove(req.id());
          }
        }
      }
    }
  }
  /** {@inheritDoc} */
  @Override
  public void finishUnmarshal(GridCacheSharedContext ctx, ClassLoader ldr)
      throws IgniteCheckedException {
    super.finishUnmarshal(ctx, ldr);

    if (ownedValKeys != null && ownedVals == null) {
      ownedVals = U.newHashMap(ownedValKeys.size());

      assert ownedValKeys.size() == ownedValVals.size();

      Iterator<IgniteTxKey> keyIter = ownedValKeys.iterator();

      Iterator<CacheVersionedValue> valIter = ownedValVals.iterator();

      while (keyIter.hasNext()) {
        IgniteTxKey key = keyIter.next();

        GridCacheContext cctx = ctx.cacheContext(key.cacheId());

        CacheVersionedValue val = valIter.next();

        key.finishUnmarshal(cctx, ldr);

        val.finishUnmarshal(cctx, ldr);

        ownedVals.put(key, val);
      }
    }

    if (retVal != null && retVal.cacheId() != 0) {
      GridCacheContext cctx = ctx.cacheContext(retVal.cacheId());

      assert cctx != null : retVal.cacheId();

      retVal.finishUnmarshal(cctx, ldr);
    }

    if (filterFailedKeys != null) {
      for (IgniteTxKey key : filterFailedKeys) {
        GridCacheContext cctx = ctx.cacheContext(key.cacheId());

        key.finishUnmarshal(cctx, ldr);
      }
    }
  }
  /** @return {@code True} if all replies are received. */
  private boolean allReceived() {
    Collection<UUID> rmtIds = this.rmtIds;

    assert rmtIds != null : "Remote Ids can't be null: " + this;

    synchronized (rcvdIds) {
      return rcvdIds.containsAll(rmtIds);
    }
  }
  /** @throws Exception Thrown if test failed. */
  public void testC() throws Exception {
    final Collection<SampleBean> set = new GridConcurrentWeakHashSet<>();

    int threadCnt = 2;

    final int cnt = 5;

    final CountDownLatch start = new CountDownLatch(1);
    final CountDownLatch stop = new CountDownLatch(threadCnt);

    Runnable r =
        new Runnable() {
          @Override
          public void run() {
            try {
              start.await();

              for (int i = 0; i < cnt; i++) {
                for (int j = 0; j < cnt; j++) set.add(new SampleBean(i));
              }
            } catch (Exception e) {
              error(e.getMessage());
            }

            stop.countDown();
          }
        };

    for (int i = 0; i < threadCnt; i++) new Thread(r).start();

    start.countDown();

    stop.await();

    assert set.size() == cnt;

    gc();

    assert set.isEmpty();
  }
Exemplo n.º 16
0
  /**
   * Decode file charset.
   *
   * @param f File to process.
   * @return File charset.
   * @throws IOException in case of error.
   */
  public static Charset decode(File f) throws IOException {
    SortedMap<String, Charset> charsets = Charset.availableCharsets();

    String[] firstCharsets = {
      Charset.defaultCharset().name(), "US-ASCII", "UTF-8", "UTF-16BE", "UTF-16LE"
    };

    Collection<Charset> orderedCharsets = U.newLinkedHashSet(charsets.size());

    for (String c : firstCharsets)
      if (charsets.containsKey(c)) orderedCharsets.add(charsets.get(c));

    orderedCharsets.addAll(charsets.values());

    try (RandomAccessFile raf = new RandomAccessFile(f, "r")) {
      FileChannel ch = raf.getChannel();

      ByteBuffer buf = ByteBuffer.allocate(4096);

      ch.read(buf);

      buf.flip();

      for (Charset charset : orderedCharsets) {
        CharsetDecoder decoder = charset.newDecoder();

        decoder.reset();

        try {
          decoder.decode(buf);

          return charset;
        } catch (CharacterCodingException ignored) {
        }
      }
    }

    return Charset.defaultCharset();
  }
  /** {@inheritDoc} */
  @Override
  public Collection<ClusterNode> nodes(int p, AffinityTopologyVersion topVer) {
    Collection<ClusterNode> affNodes = cctx.affinity().nodes(p, topVer);

    lock.readLock().lock();

    try {
      assert node2part != null && node2part.valid()
          : "Invalid node-to-partitions map [topVer1="
              + topVer
              + ", topVer2="
              + this.topVer
              + ", cache="
              + cctx.name()
              + ", node2part="
              + node2part
              + ']';

      Collection<ClusterNode> nodes = null;

      Collection<UUID> nodeIds = part2node.get(p);

      if (!F.isEmpty(nodeIds)) {
        Collection<UUID> affIds = new HashSet<>(F.viewReadOnly(affNodes, F.node2id()));

        for (UUID nodeId : nodeIds) {
          if (!affIds.contains(nodeId) && hasState(p, nodeId, OWNING, MOVING, RENTING)) {
            ClusterNode n = cctx.discovery().node(nodeId);

            if (n != null
                && (topVer.topologyVersion() < 0 || n.order() <= topVer.topologyVersion())) {
              if (nodes == null) {
                nodes = new ArrayList<>(affNodes.size() + 2);

                nodes.addAll(affNodes);
              }

              nodes.add(n);
            }
          }
        }
      }

      return nodes != null ? nodes : affNodes;
    } finally {
      lock.readLock().unlock();
    }
  }
  /**
   * Sets data configurations.
   *
   * @param dataCfgs Data configurations.
   */
  public void setDataConfigurations(Collection<? extends GridClientDataConfiguration> dataCfgs) {
    this.dataCfgs = U.newHashMap(dataCfgs.size());

    for (GridClientDataConfiguration dataCfg : dataCfgs)
      this.dataCfgs.put(dataCfg.getName(), new GridClientDataConfiguration(dataCfg));
  }
  /** {@inheritDoc} */
  @SuppressWarnings("ErrorNotRethrown")
  @Override
  public IpcEndpoint accept() throws IgniteCheckedException {
    while (!Thread.currentThread().isInterrupted()) {
      Socket sock = null;

      boolean accepted = false;

      try {
        sock = srvSock.accept();

        accepted = true;

        InputStream inputStream = sock.getInputStream();
        ObjectInputStream in = new ObjectInputStream(inputStream);

        ObjectOutputStream out = new ObjectOutputStream(sock.getOutputStream());

        IpcSharedMemorySpace inSpace = null;

        IpcSharedMemorySpace outSpace = null;

        boolean err = true;

        try {
          IpcSharedMemoryInitRequest req = (IpcSharedMemoryInitRequest) in.readObject();

          if (log.isDebugEnabled()) log.debug("Processing request: " + req);

          IgnitePair<String> p = inOutToken(req.pid(), size);

          String file1 = p.get1();
          String file2 = p.get2();

          assert file1 != null;
          assert file2 != null;

          // Create tokens.
          new File(file1).createNewFile();
          new File(file2).createNewFile();

          if (log.isDebugEnabled()) log.debug("Created token files: " + p);

          inSpace = new IpcSharedMemorySpace(file1, req.pid(), pid, size, true, log);

          outSpace = new IpcSharedMemorySpace(file2, pid, req.pid(), size, false, log);

          IpcSharedMemoryClientEndpoint ret =
              new IpcSharedMemoryClientEndpoint(inSpace, outSpace, log);

          out.writeObject(
              new IpcSharedMemoryInitResponse(
                  file2, outSpace.sharedMemoryId(), file1, inSpace.sharedMemoryId(), pid, size));

          err = !in.readBoolean();

          endpoints.add(ret);

          return ret;
        } catch (UnsatisfiedLinkError e) {
          throw IpcSharedMemoryUtils.linkError(e);
        } catch (IOException e) {
          if (log.isDebugEnabled())
            log.debug(
                "Failed to process incoming connection "
                    + "(was connection closed by another party):"
                    + e.getMessage());
        } catch (ClassNotFoundException e) {
          U.error(log, "Failed to process incoming connection.", e);
        } catch (ClassCastException e) {
          String msg =
              "Failed to process incoming connection (most probably, shared memory "
                  + "rest endpoint has been configured by mistake).";

          LT.warn(log, null, msg);

          sendErrorResponse(out, e);
        } catch (IpcOutOfSystemResourcesException e) {
          if (!omitOutOfResourcesWarn) LT.warn(log, null, OUT_OF_RESOURCES_MSG);

          sendErrorResponse(out, e);
        } catch (IgniteCheckedException e) {
          LT.error(log, e, "Failed to process incoming shared memory connection.");

          sendErrorResponse(out, e);
        } finally {
          // Exception has been thrown, need to free system resources.
          if (err) {
            if (inSpace != null) inSpace.forceClose();

            // Safety.
            if (outSpace != null) outSpace.forceClose();
          }
        }
      } catch (IOException e) {
        if (!Thread.currentThread().isInterrupted() && !accepted)
          throw new IgniteCheckedException("Failed to accept incoming connection.", e);

        if (!closed)
          LT.error(
              log, null, "Failed to process incoming shared memory connection: " + e.getMessage());
      } finally {
        U.closeQuiet(sock);
      }
    } // while

    throw new IgniteInterruptedCheckedException("Socket accept was interrupted.");
  }
Exemplo n.º 20
0
  /**
   * Grabs local events and detects if events was lost since last poll.
   *
   * @param ignite Target grid.
   * @param evtOrderKey Unique key to take last order key from node local map.
   * @param evtThrottleCntrKey Unique key to take throttle count from node local map.
   * @param evtTypes Event types to collect.
   * @param evtMapper Closure to map grid events to Visor data transfer objects.
   * @return Collections of node events
   */
  public static Collection<VisorGridEvent> collectEvents(
      Ignite ignite,
      String evtOrderKey,
      String evtThrottleCntrKey,
      final int[] evtTypes,
      IgniteClosure<Event, VisorGridEvent> evtMapper) {
    assert ignite != null;
    assert evtTypes != null && evtTypes.length > 0;

    ConcurrentMap<String, Long> nl = ignite.cluster().nodeLocalMap();

    final long lastOrder = getOrElse(nl, evtOrderKey, -1L);
    final long throttle = getOrElse(nl, evtThrottleCntrKey, 0L);

    // When we first time arrive onto a node to get its local events,
    // we'll grab only last those events that not older than given period to make sure we are
    // not grabbing GBs of data accidentally.
    final long notOlderThan = System.currentTimeMillis() - EVENTS_COLLECT_TIME_WINDOW;

    // Flag for detecting gaps between events.
    final AtomicBoolean lastFound = new AtomicBoolean(lastOrder < 0);

    IgnitePredicate<Event> p =
        new IgnitePredicate<Event>() {
          /** */
          private static final long serialVersionUID = 0L;

          @Override
          public boolean apply(Event e) {
            // Detects that events were lost.
            if (!lastFound.get() && (lastOrder == e.localOrder())) lastFound.set(true);

            // Retains events by lastOrder, period and type.
            return e.localOrder() > lastOrder
                && e.timestamp() > notOlderThan
                && F.contains(evtTypes, e.type());
          }
        };

    Collection<Event> evts = ignite.events().localQuery(p);

    // Update latest order in node local, if not empty.
    if (!evts.isEmpty()) {
      Event maxEvt = Collections.max(evts, EVTS_ORDER_COMPARATOR);

      nl.put(evtOrderKey, maxEvt.localOrder());
    }

    // Update throttle counter.
    if (!lastFound.get())
      nl.put(evtThrottleCntrKey, throttle == 0 ? EVENTS_LOST_THROTTLE : throttle - 1);

    boolean lost = !lastFound.get() && throttle == 0;

    Collection<VisorGridEvent> res = new ArrayList<>(evts.size() + (lost ? 1 : 0));

    if (lost) res.add(new VisorGridEventsLost(ignite.cluster().localNode().id()));

    for (Event e : evts) {
      VisorGridEvent visorEvt = evtMapper.apply(e);

      if (visorEvt != null) res.add(visorEvt);
    }

    return res;
  }
Exemplo n.º 21
0
  /**
   * Remove particular entry from the trash directory or subdirectory.
   *
   * @param parentId Parent ID.
   * @param id Entry id.
   * @throws IgniteCheckedException If delete failed for some reason.
   */
  private void deleteDirectory(IgniteUuid parentId, IgniteUuid id) throws IgniteCheckedException {
    assert parentId != null;
    assert id != null;

    while (true) {
      IgfsFileInfo info = meta.info(id);

      if (info != null) {
        assert info.isDirectory();

        Map<String, IgfsListingEntry> listing = info.listing();

        if (listing.isEmpty()) return; // Directory is empty.

        Map<String, IgfsListingEntry> delListing;

        if (listing.size() <= MAX_DELETE_BATCH) delListing = listing;
        else {
          delListing = new HashMap<>(MAX_DELETE_BATCH, 1.0f);

          int i = 0;

          for (Map.Entry<String, IgfsListingEntry> entry : listing.entrySet()) {
            delListing.put(entry.getKey(), entry.getValue());

            if (++i == MAX_DELETE_BATCH) break;
          }
        }

        GridCompoundFuture<Object, ?> fut = new GridCompoundFuture<>();

        // Delegate to child folders.
        for (IgfsListingEntry entry : delListing.values()) {
          if (!cancelled) {
            if (entry.isDirectory()) deleteDirectory(id, entry.fileId());
            else {
              IgfsFileInfo fileInfo = meta.info(entry.fileId());

              if (fileInfo != null) {
                assert fileInfo.isFile();

                fut.add(data.delete(fileInfo));
              }
            }
          } else return;
        }

        fut.markInitialized();

        // Wait for data cache to delete values before clearing meta cache.
        try {
          fut.get();
        } catch (IgniteFutureCancelledCheckedException ignore) {
          // This future can be cancelled only due to IGFS shutdown.
          cancelled = true;

          return;
        }

        // Actual delete of folder content.
        Collection<IgniteUuid> delIds = meta.delete(id, delListing);

        if (delListing == listing && delListing.size() == delIds.size())
          break; // All entries were deleted.
      } else break; // Entry was deleted concurrently.
    }
  }
  /** @throws Exception Thrown if test failed. */
  public void testA() throws Exception {
    Collection<Integer> set = new GridConcurrentWeakHashSet<>();

    Integer i = 1;

    assert set.add(i);
    assert !set.add(i);

    assert set.contains(i);

    assert set.size() == 1;

    Collection<Integer> c = F.asList(2, 3, 4, 5);

    assert set.addAll(c);
    assert !set.addAll(c);

    assert set.containsAll(c);

    assert set.size() == 1 + c.size();

    assert set.remove(i);
    assert !set.remove(i);

    assert !set.contains(i);

    assert set.size() == c.size();

    assert set.removeAll(c);
    assert !set.removeAll(c);

    assert !set.containsAll(c);

    assert set.isEmpty();

    Collection<Integer> c1 = Arrays.asList(1, 3, 5, 7, 9);

    int cnt = 0;

    for (Iterator<Integer> iter = set.iterator(); iter.hasNext(); cnt++) c1.contains(iter.next());

    assert set.size() == cnt;

    assert set.size() == set.toArray().length;

    assert set.addAll(c1);

    assert set.retainAll(c);
    assert !set.retainAll(c);

    Collection<Integer> c2 = F.retain(c1, true, c);

    assert set.containsAll(c2);
    assert !set.containsAll(c1);
    assert !set.containsAll(c);

    assert set.size() == c2.size();

    set.clear();

    assert set.isEmpty();

    try {
      set.iterator().next();

      assert false;
    } catch (NoSuchElementException ignored) {
      assert true;
    }

    try {
      set.add(null);

      assert false;
    } catch (NullPointerException ignored) {
      assert true;
    }
  }
  /** @throws Exception Thrown if test failed. */
  public void testD() throws Exception {
    final Collection<SampleBean> set = new GridConcurrentWeakHashSet<>();

    final int cnt = 100;

    final CountDownLatch start = new CountDownLatch(1);
    final CountDownLatch stop = new CountDownLatch(3);

    new Thread() {
      @Override
      public void run() {
        try {
          start.await();

          for (int i = 0; i < cnt; i++) {
            for (int j = 0; j < cnt; j++) set.add(new SampleBean(i));
          }
        } catch (Exception e) {
          error(e.getMessage());
        }

        stop.countDown();
      }
    }.start();

    new Thread() {
      @Override
      public void run() {
        try {
          start.await();

          for (int i = 0; i < cnt; i++) {
            for (int j = 0; j < cnt; j++) set.remove(new SampleBean(i));
          }
        } catch (Exception e) {
          error(e.getMessage());
        }

        stop.countDown();
      }
    }.start();

    new Thread() {
      @SuppressWarnings({"UnusedDeclaration"})
      @Override
      public void run() {
        try {
          start.await();

          while (stop.getCount() > 1) {
            for (SampleBean b : set) {
              // No-op.
            }
          }
        } catch (Exception e) {
          error(e.getMessage());
        }

        stop.countDown();
      }
    }.start();

    start.countDown();

    stop.await();

    gc();

    assert set.isEmpty();
  }
  /** @throws Exception Thrown if test failed. */
  @SuppressWarnings({"UnusedAssignment"})
  public void testB() throws Exception {
    Collection<SampleBean> set = new GridConcurrentWeakHashSet<>();

    SampleBean bean1 = new SampleBean(1);

    assert set.add(bean1);
    assert !set.add(bean1);

    assert set.size() == 1;

    assert set.contains(bean1);

    bean1 = null;

    gc();

    assert set.isEmpty();

    Collection<SampleBean> c =
        F.asList(new SampleBean(1), new SampleBean(2), new SampleBean(3), new SampleBean(4));

    assert set.addAll(c);
    assert !set.addAll(c);

    assert set.size() == c.size();

    assert set.containsAll(c);

    c = null;

    gc();

    assert set.isEmpty();

    SampleBean b1 = new SampleBean(1);
    SampleBean b2 = new SampleBean(2);
    SampleBean b3 = new SampleBean(3);
    SampleBean b4 = new SampleBean(4);
    SampleBean b5 = new SampleBean(5);

    set.add(b1);
    set.add(b2);
    set.add(b3);
    set.add(b4);
    set.add(b5);

    Iterator iter = set.iterator();

    assert iter.hasNext();

    b2 = null;
    b3 = null;
    b4 = null;

    gc();

    int cnt = 0;

    while (iter.hasNext()) {
      info(iter.next().toString());

      cnt++;
    }

    assert set.size() == cnt;
  }
 /** {@inheritDoc} */
 @Override
 void onQueryFutureCanceled(long reqId) {
   cancelled.add(reqId);
 }
  /**
   * Load client configuration from the properties map.
   *
   * @param prefix Prefix for the client properties.
   * @param in Properties map to load configuration from.
   * @throws GridClientException If parsing configuration failed.
   */
  public void load(String prefix, Properties in) throws GridClientException {
    while (prefix.endsWith(".")) prefix = prefix.substring(0, prefix.length() - 1);

    if (!prefix.isEmpty()) prefix += ".";

    String balancer = in.getProperty(prefix + "balancer");
    String connectTimeout = in.getProperty(prefix + "connectTimeout");
    String cred = in.getProperty(prefix + "credentials");
    String autoFetchMetrics = in.getProperty(prefix + "autoFetchMetrics");
    String autoFetchAttrs = in.getProperty(prefix + "autoFetchAttributes");
    String maxConnIdleTime = in.getProperty(prefix + "idleTimeout");
    String proto = in.getProperty(prefix + "protocol");
    String srvrs = in.getProperty(prefix + "servers");
    String tcpNoDelay = in.getProperty(prefix + "tcp.noDelay");
    String topRefreshFreq = in.getProperty(prefix + "topology.refresh");

    String sslEnabled = in.getProperty(prefix + "ssl.enabled");

    String sslProto = in.getProperty(prefix + "ssl.protocol", "TLS");
    String sslKeyAlg = in.getProperty(prefix + "ssl.key.algorithm", "SunX509");

    String keyStorePath = in.getProperty(prefix + "ssl.keystore.location");
    String keyStorePwd = in.getProperty(prefix + "ssl.keystore.password");
    String keyStoreType = in.getProperty(prefix + "ssl.keystore.type");

    String trustStorePath = in.getProperty(prefix + "ssl.truststore.location");
    String trustStorePwd = in.getProperty(prefix + "ssl.truststore.password");
    String trustStoreType = in.getProperty(prefix + "ssl.truststore.type");

    String dataCfgs = in.getProperty(prefix + "data.configurations");

    setBalancer(resolveBalancer(balancer));

    if (!F.isEmpty(connectTimeout)) setConnectTimeout(Integer.parseInt(connectTimeout));

    if (!F.isEmpty(cred)) {
      int idx = cred.indexOf(':');

      if (idx >= 0 && idx < cred.length() - 1) {
        setSecurityCredentialsProvider(
            new SecurityCredentialsBasicProvider(
                new SecurityCredentials(cred.substring(0, idx), cred.substring(idx + 1))));
      } else {
        setSecurityCredentialsProvider(
            new SecurityCredentialsBasicProvider(new SecurityCredentials(null, null, cred)));
      }
    }

    if (!F.isEmpty(autoFetchMetrics)) setAutoFetchMetrics(Boolean.parseBoolean(autoFetchMetrics));

    if (!F.isEmpty(autoFetchAttrs)) setAutoFetchAttributes(Boolean.parseBoolean(autoFetchAttrs));

    if (!F.isEmpty(maxConnIdleTime)) setMaxConnectionIdleTime(Integer.parseInt(maxConnIdleTime));

    if (!F.isEmpty(proto)) setProtocol(GridClientProtocol.valueOf(proto));

    if (!F.isEmpty(srvrs)) setServers(Arrays.asList(srvrs.replaceAll("\\s+", "").split(",")));

    if (!F.isEmpty(tcpNoDelay)) setTcpNoDelay(Boolean.parseBoolean(tcpNoDelay));

    if (!F.isEmpty(topRefreshFreq)) setTopologyRefreshFrequency(Long.parseLong(topRefreshFreq));

    //
    // SSL configuration section
    //

    if (!F.isEmpty(sslEnabled) && Boolean.parseBoolean(sslEnabled)) {
      GridSslBasicContextFactory factory = new GridSslBasicContextFactory();

      factory.setProtocol(F.isEmpty(sslProto) ? "TLS" : sslProto);
      factory.setKeyAlgorithm(F.isEmpty(sslKeyAlg) ? "SunX509" : sslKeyAlg);

      if (F.isEmpty(keyStorePath))
        throw new IllegalArgumentException("SSL key store location is not specified.");

      factory.setKeyStoreFilePath(keyStorePath);

      if (keyStorePwd != null) factory.setKeyStorePassword(keyStorePwd.toCharArray());

      factory.setKeyStoreType(F.isEmpty(keyStoreType) ? "jks" : keyStoreType);

      if (F.isEmpty(trustStorePath))
        factory.setTrustManagers(GridSslBasicContextFactory.getDisabledTrustManager());
      else {
        factory.setTrustStoreFilePath(trustStorePath);

        if (trustStorePwd != null) factory.setTrustStorePassword(trustStorePwd.toCharArray());

        factory.setTrustStoreType(F.isEmpty(trustStoreType) ? "jks" : trustStoreType);
      }

      setSslContextFactory(factory);
    }

    //
    // Data configuration section
    //

    if (!F.isEmpty(dataCfgs)) {
      String[] names = dataCfgs.replaceAll("\\s+", "").split(",");
      Collection<GridClientDataConfiguration> list = new ArrayList<>();

      for (String cfgName : names) {
        if (F.isEmpty(cfgName)) continue;

        String name = in.getProperty(prefix + "data." + cfgName + ".name");
        String bal = in.getProperty(prefix + "data." + cfgName + ".balancer");
        String aff = in.getProperty(prefix + "data." + cfgName + ".affinity");

        GridClientDataConfiguration dataCfg = new GridClientDataConfiguration();

        dataCfg.setName(F.isEmpty(name) ? null : name);
        dataCfg.setBalancer(resolveBalancer(bal));
        dataCfg.setAffinity(resolveAffinity(aff));

        list.add(dataCfg);
      }

      setDataConfigurations(list);
    }
  }
  /**
   * @param updateSeq Update sequence.
   * @return Checks if any of the local partitions need to be evicted.
   */
  private boolean checkEvictions(long updateSeq) {
    assert lock.isWriteLockedByCurrentThread();

    boolean changed = false;

    UUID locId = cctx.nodeId();

    for (GridDhtLocalPartition part : locParts.values()) {
      GridDhtPartitionState state = part.state();

      if (state.active()) {
        int p = part.id();

        List<ClusterNode> affNodes = cctx.affinity().nodes(p, topVer);

        if (!affNodes.contains(cctx.localNode())) {
          Collection<UUID> nodeIds = F.nodeIds(nodes(p, topVer, OWNING));

          // If all affinity nodes are owners, then evict partition from local node.
          if (nodeIds.containsAll(F.nodeIds(affNodes))) {
            part.rent(false);

            updateLocal(part.id(), locId, part.state(), updateSeq);

            changed = true;

            if (log.isDebugEnabled())
              log.debug("Evicted local partition (all affinity nodes are owners): " + part);
          } else {
            int ownerCnt = nodeIds.size();
            int affCnt = affNodes.size();

            if (ownerCnt > affCnt) {
              List<ClusterNode> sorted = new ArrayList<>(cctx.discovery().nodes(nodeIds));

              // Sort by node orders in ascending order.
              Collections.sort(sorted, CU.nodeComparator(true));

              int diff = sorted.size() - affCnt;

              for (int i = 0; i < diff; i++) {
                ClusterNode n = sorted.get(i);

                if (locId.equals(n.id())) {
                  part.rent(false);

                  updateLocal(part.id(), locId, part.state(), updateSeq);

                  changed = true;

                  if (log.isDebugEnabled())
                    log.debug(
                        "Evicted local partition (this node is oldest non-affinity node): " + part);

                  break;
                }
              }
            }
          }
        }
      }
    }

    return changed;
  }