/**
   * @param sesId Session ID.
   * @param taskNodeId Task node ID.
   * @param taskName Task name.
   * @param dep Deployment.
   * @param taskClsName Task class name.
   * @param top Topology.
   * @param startTime Execution start time.
   * @param endTime Execution end time.
   * @param siblings Collection of siblings.
   * @param attrs Map of attributes.
   * @param fullSup {@code True} to enable distributed session attributes and checkpoints.
   * @return New session if one did not exist, or existing one.
   */
  public GridTaskSessionImpl createTaskSession(
      GridUuid sesId,
      UUID taskNodeId,
      String taskName,
      @Nullable GridDeployment dep,
      String taskClsName,
      @Nullable Collection<UUID> top,
      long startTime,
      long endTime,
      Collection<GridComputeJobSibling> siblings,
      Map<Object, Object> attrs,
      boolean fullSup) {
    if (!fullSup) {
      return new GridTaskSessionImpl(
          taskNodeId,
          taskName,
          dep,
          taskClsName,
          sesId,
          top,
          startTime,
          endTime,
          siblings,
          attrs,
          ctx,
          fullSup);
    }

    while (true) {
      GridTaskSessionImpl ses = sesMap.get(sesId);

      if (ses == null) {
        GridTaskSessionImpl old =
            sesMap.putIfAbsent(
                sesId,
                ses =
                    new GridTaskSessionImpl(
                        taskNodeId,
                        taskName,
                        dep,
                        taskClsName,
                        sesId,
                        top,
                        startTime,
                        endTime,
                        siblings,
                        attrs,
                        ctx,
                        fullSup));

        if (old != null) ses = old;
        else
          // Return without acquire.
          return ses;
      }

      if (ses.acquire()) return ses;
      else sesMap.remove(sesId, ses);
    }
  }
    /** {@inheritDoc} */
    @Nullable
    @Override
    public Map<String, Collection<?>> run(GridStreamerContext ctx, Collection<Bar> bars)
        throws GridException {
      ConcurrentMap<String, Bar> loc = ctx.localSpace();

      GridStreamerWindow win = ctx.window("stage2");

      // Add numbers to window.
      win.enqueueAll(bars);

      Collection<Bar> polled = win.pollEvictedBatch();

      if (!polled.isEmpty()) {
        Map<String, Bar> map = new HashMap<>();

        for (Bar polledBar : polled) {
          String symbol = polledBar.symbol();

          Bar bar = map.get(symbol);

          if (bar == null) map.put(symbol, bar = new Bar(symbol));

          bar.update(polledBar);
        }

        loc.putAll(map);
      }

      return null;
    }
  /**
   * @param p Partition number.
   * @param topVer Topology version.
   * @param create Create flag.
   * @param updateSeq Update sequence.
   * @return Local partition.
   */
  private GridDhtLocalPartition localPartition(
      int p, AffinityTopologyVersion topVer, boolean create, boolean updateSeq) {
    while (true) {
      boolean belongs = cctx.affinity().localNode(p, topVer);

      GridDhtLocalPartition loc = locParts.get(p);

      if (loc != null && loc.state() == EVICTED) {
        locParts.remove(p, loc);

        if (!create) return null;

        if (!belongs)
          throw new GridDhtInvalidPartitionException(
              p,
              "Adding entry to evicted partition [part="
                  + p
                  + ", topVer="
                  + topVer
                  + ", this.topVer="
                  + this.topVer
                  + ']');

        continue;
      }

      if (loc == null && create) {
        if (!belongs)
          throw new GridDhtInvalidPartitionException(
              p,
              "Creating partition which does not belong [part="
                  + p
                  + ", topVer="
                  + topVer
                  + ", this.topVer="
                  + this.topVer
                  + ']');

        lock.writeLock().lock();

        try {
          GridDhtLocalPartition old =
              locParts.putIfAbsent(p, loc = new GridDhtLocalPartition(cctx, p));

          if (old != null) loc = old;
          else {
            if (updateSeq) this.updateSeq.incrementAndGet();

            if (log.isDebugEnabled()) log.debug("Created local partition: " + loc);
          }
        } finally {
          lock.writeLock().unlock();
        }
      }

      return loc;
    }
  }
  /**
   * Removes session for a given session ID.
   *
   * @param sesId ID of session to remove.
   * @return {@code True} if session was removed.
   */
  public boolean removeSession(GridUuid sesId) {
    GridTaskSessionImpl ses = sesMap.get(sesId);

    assert ses == null || ses.isFullSupport();

    if (ses != null && ses.release()) {
      sesMap.remove(sesId, ses);

      return true;
    }

    return false;
  }
  /** {@inheritDoc} */
  @Override
  public void printMemoryStats() {
    super.printMemoryStats();

    X.println(">>>   threadsSize: " + threads.size());
    X.println(">>>   futsSize: " + futs.size());
  }
  /**
   * @param updateSeq Update sequence.
   * @return Future for evict attempt.
   */
  IgniteInternalFuture<Boolean> tryEvictAsync(boolean updateSeq) {
    if (map.isEmpty()
        && !GridQueryProcessor.isEnabled(cctx.config())
        && state.compareAndSet(RENTING, EVICTED, 0, 0)) {
      if (log.isDebugEnabled()) log.debug("Evicted partition: " + this);

      clearSwap();

      if (cctx.isDrEnabled()) cctx.dr().partitionEvicted(id);

      cctx.dataStructures().onPartitionEvicted(id);

      rent.onDone();

      ((GridDhtPreloader) cctx.preloader()).onPartitionEvicted(this, updateSeq);

      clearDeferredDeletes();

      return new GridFinishedFuture<>(true);
    }

    return cctx.closures()
        .callLocalSafe(
            new GPC<Boolean>() {
              @Override
              public Boolean call() {
                return tryEvict(true);
              }
            }, /*system pool*/
            true);
  }
  /**
   * @param updateSeq Update sequence.
   * @return {@code True} if entry has been transitioned to state EVICTED.
   */
  boolean tryEvict(boolean updateSeq) {
    if (state.getReference() != RENTING || state.getStamp() != 0 || groupReserved()) return false;

    // Attempt to evict partition entries from cache.
    clearAll();

    if (map.isEmpty() && state.compareAndSet(RENTING, EVICTED, 0, 0)) {
      if (log.isDebugEnabled()) log.debug("Evicted partition: " + this);

      if (!GridQueryProcessor.isEnabled(cctx.config())) clearSwap();

      if (cctx.isDrEnabled()) cctx.dr().partitionEvicted(id);

      cctx.dataStructures().onPartitionEvicted(id);

      rent.onDone();

      ((GridDhtPreloader) cctx.preloader()).onPartitionEvicted(this, updateSeq);

      clearDeferredDeletes();

      return true;
    }

    return false;
  }
  /**
   * Waits for renting partitions.
   *
   * @return {@code True} if mapping was changed.
   * @throws IgniteCheckedException If failed.
   */
  private boolean waitForRent() throws IgniteCheckedException {
    boolean changed = false;

    // Synchronously wait for all renting partitions to complete.
    for (Iterator<GridDhtLocalPartition> it = locParts.values().iterator(); it.hasNext(); ) {
      GridDhtLocalPartition p = it.next();

      GridDhtPartitionState state = p.state();

      if (state == RENTING || state == EVICTED) {
        if (log.isDebugEnabled()) log.debug("Waiting for renting partition: " + p);

        // Wait for partition to empty out.
        p.rent(true).get();

        if (log.isDebugEnabled()) log.debug("Finished waiting for renting partition: " + p);

        // Remove evicted partition.
        it.remove();

        changed = true;
      }
    }

    return changed;
  }
Example #9
0
  /**
   * @param key Key to lock.
   * @return Sync object that should be passed to {@link #unlock(Object, Object)} method.
   */
  public <T> Object lock(T key) {
    assert key != null;

    boolean interrupted = false;

    Sync t = new Sync();

    try {
      while (true) {
        Sync old = locks.putIfAbsent(key, t);

        if (old != null) {
          while (true) {
            try {
              old.await();

              break;
            } catch (InterruptedException ignored) {
              interrupted = true;
            }
          }
        } else return t;
      }
    } finally {
      if (interrupted) Thread.currentThread().interrupt();
    }
  }
Example #10
0
  /**
   * @param cancel {@code True} to close with cancellation.
   * @throws GridException If failed.
   */
  @Override
  public void close(boolean cancel) throws GridException {
    if (!closed.compareAndSet(false, true)) return;

    busyLock.block();

    if (log.isDebugEnabled())
      log.debug("Closing data loader [ldr=" + this + ", cancel=" + cancel + ']');

    GridException e = null;

    try {
      // Assuming that no methods are called on this loader after this method is called.
      if (cancel) {
        cancelled = true;

        for (Buffer buf : bufMappings.values()) buf.cancelAll();
      } else doFlush();

      ctx.event().removeLocalEventListener(discoLsnr);

      ctx.io().removeMessageListener(topic);
    } catch (GridException e0) {
      e = e0;
    }

    fut.onDone(null, e);

    if (e != null) throw e;
  }
  /**
   * Processes cache query request.
   *
   * @param sndId Sender node id.
   * @param req Query request.
   */
  @SuppressWarnings("unchecked")
  @Override
  void processQueryRequest(UUID sndId, GridCacheQueryRequest req) {
    if (req.cancel()) {
      cancelIds.add(new CancelMessageId(req.id(), sndId));

      if (req.fields()) removeFieldsQueryResult(sndId, req.id());
      else removeQueryResult(sndId, req.id());
    } else {
      if (!cancelIds.contains(new CancelMessageId(req.id(), sndId))) {
        if (!F.eq(req.cacheName(), cctx.name())) {
          GridCacheQueryResponse res =
              new GridCacheQueryResponse(
                  cctx.cacheId(),
                  req.id(),
                  new IgniteCheckedException(
                      "Received request for incorrect cache [expected="
                          + cctx.name()
                          + ", actual="
                          + req.cacheName()));

          sendQueryResponse(sndId, res, 0);
        } else {
          threads.put(req.id(), Thread.currentThread());

          try {
            GridCacheQueryInfo info = distributedQueryInfo(sndId, req);

            if (info == null) return;

            if (req.fields()) runFieldsQuery(info);
            else runQuery(info);
          } catch (Throwable e) {
            U.error(log(), "Failed to run query.", e);

            sendQueryResponse(
                sndId, new GridCacheQueryResponse(cctx.cacheId(), req.id(), e.getCause()), 0);

            if (e instanceof Error) throw (Error) e;
          } finally {
            threads.remove(req.id());
          }
        }
      }
    }
  }
Example #12
0
  /**
   * @param key Key to lock.
   * @return Sync object that should be passed to {@link #unlock(Object, Object)} method or {@code
   *     null} if try lock failed.
   */
  @Nullable
  public <T> Object tryLock(T key) {
    assert key != null;

    Sync t = new Sync();

    Sync old = locks.putIfAbsent(key, t);

    return old != null ? null : t;
  }
  /** @param entry Entry to add. */
  void onAdded(GridDhtCacheEntry entry) {
    GridDhtPartitionState state = state();

    if (state == EVICTED)
      throw new GridDhtInvalidPartitionException(
          id, "Adding entry to invalid partition [part=" + id + ']');

    map.put(entry.key(), entry);

    if (!entry.isInternal()) mapPubSize.increment();
  }
Example #14
0
  /**
   * Flushes every internal buffer if buffer was flushed before passed in threshold.
   *
   * <p>Does not wait for result and does not fail on errors assuming that this method should be
   * called periodically.
   */
  @Override
  public void tryFlush() throws GridInterruptedException {
    if (!busyLock.enterBusy()) return;

    try {
      for (Buffer buf : bufMappings.values()) buf.flush();

      lastFlushTime = U.currentTimeMillis();
    } finally {
      leaveBusy();
    }
  }
Example #15
0
  /**
   * @param key Key to lock.
   * @return Sync object that should be passed to {@link #unlock(Object, Object)} method.
   * @throws InterruptedException If interrupted while acquiring lock.
   */
  public <T> Object lockInterruptibly(T key) throws InterruptedException {
    assert key != null;

    Sync t = new Sync();

    while (true) {
      Sync old = locks.putIfAbsent(key, t);

      if (old != null) old.await();
      else return t;
    }
  }
  /** {@inheritDoc} */
  @Override
  void onCancelAtStop() {
    super.onCancelAtStop();

    for (GridCacheQueryFutureAdapter fut : futs.values())
      try {
        fut.cancel();
      } catch (IgniteCheckedException e) {
        U.error(log, "Failed to cancel running query future: " + fut, e);
      }

    U.interrupt(threads.values());
  }
  /** @param entry Entry to remove. */
  @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
  void onRemoved(GridDhtCacheEntry entry) {
    assert entry.obsolete();

    // Make sure to remove exactly this entry.
    synchronized (entry) {
      map.remove(entry.key(), entry);

      if (!entry.isInternal() && !entry.deleted()) mapPubSize.decrement();
    }

    // Attempt to evict.
    tryEvict(true);
  }
  /** {@inheritDoc} */
  @Override
  public void printMemoryStats(int threshold) {
    X.println(
        ">>>  Cache partition topology stats [grid="
            + cctx.gridName()
            + ", cache="
            + cctx.name()
            + ']');

    for (GridDhtLocalPartition part : locParts.values()) {
      int size = part.size();

      if (size >= threshold)
        X.println(">>>   Local partition [part=" + part.id() + ", size=" + size + ']');
    }
  }
 /** {@inheritDoc} */
 @Override
 public String toString() {
   return S.toString(
       GridDhtLocalPartition.class,
       this,
       "state",
       state(),
       "reservations",
       reservations(),
       "empty",
       map.isEmpty(),
       "createTime",
       U.format(createTime),
       "mapPubSize",
       mapPubSize);
 }
 /** @return Keys belonging to partition. */
 public Set<KeyCacheObject> keySet() {
   return map.keySet();
 }
Example #21
0
 /** {@inheritDoc} */
 @Override
 public String toString() {
   return S.toString(GridKeyLock.class, this, "locksSize", locks.size());
 }
Example #22
0
  /**
   * Performs flush.
   *
   * @throws GridException If failed.
   */
  private void doFlush() throws GridException {
    lastFlushTime = U.currentTimeMillis();

    List<GridFuture> activeFuts0 = null;

    int doneCnt = 0;

    for (GridFuture<?> f : activeFuts) {
      if (!f.isDone()) {
        if (activeFuts0 == null) activeFuts0 = new ArrayList<>((int) (activeFuts.size() * 1.2));

        activeFuts0.add(f);
      } else {
        f.get();

        doneCnt++;
      }
    }

    if (activeFuts0 == null || activeFuts0.isEmpty()) return;

    while (true) {
      Queue<GridFuture<?>> q = null;

      for (Buffer buf : bufMappings.values()) {
        GridFuture<?> flushFut = buf.flush();

        if (flushFut != null) {
          if (q == null) q = new ArrayDeque<>(bufMappings.size() * 2);

          q.add(flushFut);
        }
      }

      if (q != null) {
        assert !q.isEmpty();

        boolean err = false;

        for (GridFuture fut = q.poll(); fut != null; fut = q.poll()) {
          try {
            fut.get();
          } catch (GridException e) {
            if (log.isDebugEnabled()) log.debug("Failed to flush buffer: " + e);

            err = true;
          }
        }

        if (err)
          // Remaps needed - flush buffers.
          continue;
      }

      doneCnt = 0;

      for (int i = 0; i < activeFuts0.size(); i++) {
        GridFuture f = activeFuts0.get(i);

        if (f == null) doneCnt++;
        else if (f.isDone()) {
          f.get();

          doneCnt++;

          activeFuts0.set(i, null);
        } else break;
      }

      if (doneCnt == activeFuts0.size()) return;
    }
  }
Example #23
0
  /**
   * @param entries Entries.
   * @param resFut Result future.
   * @param activeKeys Active keys.
   * @param remaps Remaps count.
   */
  private void load0(
      Collection<? extends Map.Entry<K, V>> entries,
      final GridFutureAdapter<Object> resFut,
      final Collection<K> activeKeys,
      final int remaps) {
    assert entries != null;

    if (remaps >= MAX_REMAP_CNT) {
      resFut.onDone(new GridException("Failed to finish operation (too many remaps): " + remaps));

      return;
    }

    Map<GridNode, Collection<Map.Entry<K, V>>> mappings = new HashMap<>();

    boolean initPda = ctx.deploy().enabled() && jobPda == null;

    for (Map.Entry<K, V> entry : entries) {
      GridNode node;

      try {
        K key = entry.getKey();

        assert key != null;

        if (initPda) {
          jobPda = new DataLoaderPda(key, entry.getValue(), updater);

          initPda = false;
        }

        node = ctx.affinity().mapKeyToNode(cacheName, key);
      } catch (GridException e) {
        resFut.onDone(e);

        return;
      }

      if (node == null) {
        resFut.onDone(
            new GridTopologyException(
                "Failed to map key to node "
                    + "(no nodes with cache found in topology) [infos="
                    + entries.size()
                    + ", cacheName="
                    + cacheName
                    + ']'));

        return;
      }

      Collection<Map.Entry<K, V>> col = mappings.get(node);

      if (col == null) mappings.put(node, col = new ArrayList<>());

      col.add(entry);
    }

    for (final Map.Entry<GridNode, Collection<Map.Entry<K, V>>> e : mappings.entrySet()) {
      final UUID nodeId = e.getKey().id();

      Buffer buf = bufMappings.get(nodeId);

      if (buf == null) {
        Buffer old = bufMappings.putIfAbsent(nodeId, buf = new Buffer(e.getKey()));

        if (old != null) buf = old;
      }

      final Collection<Map.Entry<K, V>> entriesForNode = e.getValue();

      GridInClosure<GridFuture<?>> lsnr =
          new GridInClosure<GridFuture<?>>() {
            @Override
            public void apply(GridFuture<?> t) {
              try {
                t.get();

                for (Map.Entry<K, V> e : entriesForNode) activeKeys.remove(e.getKey());

                if (activeKeys.isEmpty()) resFut.onDone();
              } catch (GridException e1) {
                if (log.isDebugEnabled())
                  log.debug("Future finished with error [nodeId=" + nodeId + ", err=" + e1 + ']');

                if (cancelled) {
                  resFut.onDone(
                      new GridException(
                          "Data loader has been cancelled: " + GridDataLoaderImpl.this, e1));
                } else load0(entriesForNode, resFut, activeKeys, remaps + 1);
              }
            }
          };

      GridFutureAdapter<?> f;

      try {
        f = buf.update(entriesForNode, lsnr);
      } catch (GridInterruptedException e1) {
        resFut.onDone(e1);

        return;
      }

      if (ctx.discovery().node(nodeId) == null) {
        if (bufMappings.remove(nodeId, buf)) buf.onNodeLeft();

        if (f != null)
          f.onDone(
              new GridTopologyException(
                  "Failed to wait for request completion " + "(node has left): " + nodeId));
      }
    }
  }
  /** Clears values for this partition. */
  private void clearAll() {
    GridCacheVersion clearVer = cctx.versions().next();

    boolean swap = cctx.isSwapOrOffheapEnabled();

    boolean rec = cctx.events().isRecordable(EVT_CACHE_REBALANCE_OBJECT_UNLOADED);

    Iterator<GridDhtCacheEntry> it = map.values().iterator();

    GridCloseableIterator<Map.Entry<byte[], GridCacheSwapEntry>> swapIt = null;

    if (swap
        && GridQueryProcessor.isEnabled(cctx.config())) { // Indexing needs to unswap cache values.
      Iterator<GridDhtCacheEntry> unswapIt = null;

      try {
        swapIt = cctx.swap().iterator(id);
        unswapIt = unswapIterator(swapIt);
      } catch (Exception e) {
        U.error(log, "Failed to clear swap for evicted partition: " + this, e);
      }

      if (unswapIt != null) it = F.concat(it, unswapIt);
    }

    try {
      while (it.hasNext()) {
        GridDhtCacheEntry cached = it.next();

        try {
          if (cached.clearInternal(clearVer, swap)) {
            map.remove(cached.key(), cached);

            if (!cached.isInternal()) {
              mapPubSize.decrement();

              if (rec)
                cctx.events()
                    .addEvent(
                        cached.partition(),
                        cached.key(),
                        cctx.localNodeId(),
                        (IgniteUuid) null,
                        null,
                        EVT_CACHE_REBALANCE_OBJECT_UNLOADED,
                        null,
                        false,
                        cached.rawGet(),
                        cached.hasValue(),
                        null,
                        null,
                        null);
            }
          }
        } catch (IgniteCheckedException e) {
          U.error(log, "Failed to clear cache entry for evicted partition: " + cached, e);
        }
      }
    } finally {
      U.close(swapIt, log);
    }
  }
Example #25
0
  /**
   * @param key Key.
   * @param sync Sync that got from {@link #lockInterruptibly(Object)} call.
   */
  public <T> void unlock(T key, Object sync) {
    if (!locks.remove(key, sync))
      throw new IllegalStateException("Lock has not been acquired for key: " + key);

    ((Sync) sync).finish();
  }
 /** {@inheritDoc} */
 @Override
 public void printMemoryStats() {
   X.println(">>>");
   X.println(">>> Task session processor memory stats [grid=" + ctx.gridName() + ']');
   X.println(">>>  sesMapSize: " + sesMap.size());
 }
 /**
  * @param sesId Session ID.
  * @return Session for a given session ID.
  */
 @Nullable
 public GridTaskSessionImpl getSession(GridUuid sesId) {
   return sesMap.get(sesId);
 }
 /** @return Number of entries in this partition (constant-time method). */
 public int size() {
   return map.size();
 }
 /** @return {@code True} if partition is empty. */
 public boolean isEmpty() {
   return map.isEmpty();
 }
 /** @return Entries belonging to partition. */
 public Collection<GridDhtCacheEntry> entries() {
   return map.values();
 }