/**
  * Log task mapped.
  *
  * @param log Logger.
  * @param clazz Task class.
  * @param nodes Mapped nodes.
  */
 public static void logMapped(
     @Nullable IgniteLogger log, Class<?> clazz, Collection<ClusterNode> nodes) {
   log0(
       log,
       U.currentTimeMillis(),
       String.format("[%s]: MAPPED: %s", clazz.getSimpleName(), U.toShortString(nodes)));
 }
Example #2
0
  /**
   * Message received callback.
   *
   * @param src Sender node ID.
   * @param msg Received message.
   * @return {@code True}.
   */
  public boolean onMessageReceived(UUID src, GridHadoopMessage msg) {
    if (msg instanceof GridHadoopShuffleMessage) {
      GridHadoopShuffleMessage m = (GridHadoopShuffleMessage) msg;

      try {
        job(m.jobId()).onShuffleMessage(m);
      } catch (GridException e) {
        U.error(log, "Message handling failed.", e);
      }

      try {
        // Reply with ack.
        send0(src, new GridHadoopShuffleAck(m.id(), m.jobId()));
      } catch (GridException e) {
        U.error(
            log,
            "Failed to reply back to shuffle message sender [snd=" + src + ", msg=" + msg + ']',
            e);
      }
    } else if (msg instanceof GridHadoopShuffleAck) {
      GridHadoopShuffleAck m = (GridHadoopShuffleAck) msg;

      try {
        job(m.jobId()).onShuffleAck(m);
      } catch (GridException e) {
        U.error(log, "Message handling failed.", e);
      }
    } else
      throw new IllegalStateException(
          "Unknown message type received to Hadoop shuffle [src=" + src + ", msg=" + msg + ']');

    return true;
  }
  /**
   * @param cctx Context.
   * @param id Partition ID.
   */
  @SuppressWarnings("ExternalizableWithoutPublicNoArgConstructor")
  GridDhtLocalPartition(GridCacheContext cctx, int id) {
    assert cctx != null;

    this.id = id;
    this.cctx = cctx;

    log = U.logger(cctx.kernalContext(), logRef, this);

    rent =
        new GridFutureAdapter<Object>() {
          @Override
          public String toString() {
            return "PartitionRentFuture [part=" + GridDhtLocalPartition.this + ", map=" + map + ']';
          }
        };

    map = new ConcurrentHashMap8<>(cctx.config().getStartSize() / cctx.affinity().partitions());

    int delQueueSize =
        CU.isSystemCache(cctx.name())
            ? 100
            : Math.max(MAX_DELETE_QUEUE_SIZE / cctx.affinity().partitions(), 20);

    rmvQueue = new GridCircularBuffer<>(U.ceilPow2(delQueueSize));
  }
  /** @param e Error. */
  void onError(Throwable e) {
    tx.commitError(e);

    if (err.compareAndSet(null, e)) {
      boolean marked = tx.setRollbackOnly();

      if (e instanceof GridCacheTxRollbackException) {
        if (marked) {
          try {
            tx.rollback();
          } catch (GridException ex) {
            U.error(log, "Failed to automatically rollback transaction: " + tx, ex);
          }
        }
      } else if (tx.implicit()
          && tx.isSystemInvalidate()) { // Finish implicit transaction on heuristic error.
        try {
          tx.close();
        } catch (GridException ex) {
          U.error(log, "Failed to invalidate transaction: " + tx, ex);
        }
      }

      onComplete();
    }
  }
Example #5
0
  /**
   * Waits for all workers to finish.
   *
   * @param cancel Flag to indicate whether workers should be cancelled before waiting for them to
   *     finish.
   */
  public void join(boolean cancel) {
    if (cancel) U.cancel(workers);

    // Record current interrupted status of calling thread.
    boolean interrupted = Thread.interrupted();

    try {
      U.join(workers, log);
    } finally {
      // Reset interrupted flag on calling thread.
      if (interrupted) Thread.currentThread().interrupt();
    }
  }
  /** {@inheritDoc} */
  @Override
  void onCancelAtStop() {
    super.onCancelAtStop();

    for (GridCacheQueryFutureAdapter fut : futs.values())
      try {
        fut.cancel();
      } catch (IgniteCheckedException e) {
        U.error(log, "Failed to cancel running query future: " + fut, e);
      }

    U.interrupt(threads.values());
  }
    /**
     * Waits until total number of events processed is equal or greater then argument passed.
     *
     * @param cnt Number of events to wait.
     * @param timeout Timeout to wait.
     * @return {@code True} if successfully waited, {@code false} if timeout happened.
     * @throws InterruptedException If thread is interrupted.
     */
    public synchronized boolean awaitEvents(int cnt, long timeout) throws InterruptedException {
      long start = U.currentTimeMillis();

      long now = start;

      while (start + timeout > now) {
        if (evtCnt >= cnt) return true;

        wait(start + timeout - now);

        now = U.currentTimeMillis();
      }

      return false;
    }
  /** {@inheritDoc} */
  @Override
  public void loadCache(GridBiInClosure<K, V> c, @Nullable Object... args) throws GridException {
    ExecutorService exec =
        new ThreadPoolExecutor(
            threadsCnt,
            threadsCnt,
            0L,
            MILLISECONDS,
            new ArrayBlockingQueue<Runnable>(batchQueueSize),
            new BlockingRejectedExecutionHandler());

    Iterator<I> iter = inputIterator(args);

    Collection<I> buf = new ArrayList<>(batchSize);

    try {
      while (iter.hasNext()) {
        if (Thread.currentThread().isInterrupted()) {
          U.warn(log, "Working thread was interrupted while loading data.");

          break;
        }

        buf.add(iter.next());

        if (buf.size() == batchSize) {
          exec.submit(new Worker(c, buf, args));

          buf = new ArrayList<>(batchSize);
        }
      }

      if (!buf.isEmpty()) exec.submit(new Worker(c, buf, args));
    } catch (RejectedExecutionException ignored) {
      // Because of custom RejectedExecutionHandler.
      assert false : "RejectedExecutionException was thrown while it shouldn't.";
    } finally {
      exec.shutdown();

      try {
        exec.awaitTermination(Long.MAX_VALUE, MILLISECONDS);
      } catch (InterruptedException ignored) {
        U.warn(log, "Working thread was interrupted while waiting for put operations to complete.");

        Thread.currentThread().interrupt();
      }
    }
  }
  /**
   * Processes cache query response.
   *
   * @param sndId Sender node id.
   * @param res Query response.
   */
  @SuppressWarnings("unchecked")
  private void processQueryResponse(UUID sndId, GridCacheQueryResponse res) {
    if (log.isDebugEnabled()) log.debug("Received query response: " + res);

    GridCacheQueryFutureAdapter fut = getQueryFuture(res.requestId());

    if (fut != null)
      if (res.fields())
        ((GridCacheDistributedFieldsQueryFuture) fut)
            .onPage(
                sndId,
                res.metadata(),
                (Collection<Map<String, Object>>) ((Collection) res.data()),
                res.error(),
                res.isFinished());
      else fut.onPage(sndId, res.data(), res.error(), res.isFinished());
    else if (!cancelled.contains(res.requestId()))
      U.warn(
          log,
          "Received response for finished or unknown query [rmtNodeId="
              + sndId
              + ", res="
              + res
              + ']');
  }
  /** {@inheritDoc} */
  @Override
  public boolean onDone(GridCacheTx tx, Throwable err) {
    if ((initialized() || err != null) && super.onDone(tx, err)) {
      if (error() instanceof GridCacheTxHeuristicException) {
        long topVer = this.tx.topologyVersion();

        for (GridCacheTxEntry<K, V> e : this.tx.writeMap().values()) {
          try {
            if (e.op() != NOOP && !cctx.affinity().localNode(e.key(), topVer)) {
              GridCacheEntryEx<K, V> cacheEntry = cctx.cache().peekEx(e.key());

              if (cacheEntry != null) cacheEntry.invalidate(null, this.tx.xidVersion());
            }
          } catch (Throwable t) {
            U.error(log, "Failed to invalidate entry.", t);

            if (t instanceof Error) throw (Error) t;
          }
        }
      }

      // Don't forget to clean up.
      cctx.mvcc().removeFuture(this);

      return true;
    }

    return false;
  }
  /** Initializes future. */
  @SuppressWarnings({"unchecked"})
  void finish() {
    if (mappings != null) {
      finish(mappings.values());

      markInitialized();

      if (!isSync()) {
        boolean complete = true;

        for (GridFuture<?> f : pending())
          // Mini-future in non-sync mode gets done when message gets sent.
          if (isMini(f) && !f.isDone()) complete = false;

        if (complete) onComplete();
      }
    } else {
      assert !commit;

      try {
        tx.rollback();
      } catch (GridException e) {
        U.error(log, "Failed to rollback empty transaction: " + tx, e);
      }

      markInitialized();
    }
  }
  private void sendPartitions() {
    ClusterNode oldestNode = this.oldestNode.get();

    try {
      sendLocalPartitions(oldestNode, exchId);
    } catch (ClusterTopologyCheckedException ignore) {
      if (log.isDebugEnabled())
        log.debug(
            "Oldest node left during partition exchange [nodeId="
                + oldestNode.id()
                + ", exchId="
                + exchId
                + ']');
    } catch (IgniteCheckedException e) {
      scheduleRecheck();

      U.error(
          log,
          "Failed to send local partitions to oldest node (will retry after timeout) [oldestNodeId="
              + oldestNode.id()
              + ", exchId="
              + exchId
              + ']',
          e);
    }
  }
  /** @throws Exception If failed. */
  @SuppressWarnings("unchecked")
  public void testCancel() throws Exception {
    Grid grid = G.grid(getTestGridName());

    grid.compute()
        .localDeployTask(GridCancelTestTask.class, U.detectClassLoader(GridCancelTestTask.class));

    GridComputeTaskFuture<?> fut = grid.compute().execute(GridCancelTestTask.class.getName(), null);

    // Wait until jobs begin execution.
    boolean await = startSignal.await(WAIT_TIME, TimeUnit.MILLISECONDS);

    assert await : "Jobs did not start.";

    info("Test task result: " + fut);

    assert fut != null;

    // Only first job should successfully complete.
    Object res = fut.get();
    assert (Integer) res == 1;

    // Wait for all jobs to finish.
    await = stopSignal.await(WAIT_TIME, TimeUnit.MILLISECONDS);
    assert await : "Jobs did not stop.";

    // One is definitely processed. But there might be some more processed or cancelled or processed
    // and cancelled.
    // Thus total number should be at least SPLIT_COUNT and at most (SPLIT_COUNT - 1) *2 +1
    assert (cancelCnt + processedCnt) >= SPLIT_COUNT
            && (cancelCnt + processedCnt) <= (SPLIT_COUNT - 1) * 2 + 1
        : "Invalid cancel count value: " + cancelCnt;
  }
  /** Clears swap entries for evicted partition. */
  private void clearSwap() {
    assert state() == EVICTED;
    assert !GridQueryProcessor.isEnabled(cctx.config())
        : "Indexing needs to have unswapped values.";

    try {
      GridCloseableIterator<Map.Entry<byte[], GridCacheSwapEntry>> it = cctx.swap().iterator(id);

      boolean isLocStore = cctx.store().isLocal();

      if (it != null) {
        // We can safely remove these values because no entries will be created for evicted
        // partition.
        while (it.hasNext()) {
          Map.Entry<byte[], GridCacheSwapEntry> entry = it.next();

          byte[] keyBytes = entry.getKey();

          KeyCacheObject key = cctx.toCacheKeyObject(keyBytes);

          cctx.swap().remove(key);

          if (isLocStore) cctx.store().remove(null, key.value(cctx.cacheObjectContext(), false));
        }
      }
    } catch (IgniteCheckedException e) {
      U.error(log, "Failed to clear swap for evicted partition: " + this, e);
    }
  }
Example #15
0
  /** {@inheritDoc} */
  @SuppressWarnings({"unchecked"})
  @Override
  public void listenAsync(@Nullable final GridInClosure<? super GridFuture<R>> lsnr) {
    if (lsnr != null) {
      checkValid();

      boolean done;

      synchronized (mux) {
        done = this.done;

        if (!done) lsnrs.add(lsnr);
      }

      if (done) {
        try {
          if (syncNotify) notifyListener(lsnr);
          else
            ctx.closure()
                .runLocalSafe(
                    new GPR() {
                      @Override
                      public void run() {
                        notifyListener(lsnr);
                      }
                    },
                    true);
        } catch (IllegalStateException ignore) {
          U.warn(
              null,
              "Future notification will not proceed because grid is stopped: " + ctx.gridName());
        }
      }
    }
  }
  private void recheck() {
    // If this is the oldest node.
    if (oldestNode.get().id().equals(cctx.localNodeId())) {
      Collection<UUID> remaining = remaining();

      if (!remaining.isEmpty()) {
        try {
          cctx.io()
              .safeSend(
                  cctx.discovery().nodes(remaining),
                  new GridDhtPartitionsSingleRequest(exchId),
                  SYSTEM_POOL,
                  null);
        } catch (IgniteCheckedException e) {
          U.error(
              log,
              "Failed to request partitions from nodes [exchangeId="
                  + exchId
                  + ", nodes="
                  + remaining
                  + ']',
              e);
        }
      }
      // Resend full partition map because last attempt failed.
      else {
        if (spreadPartitions()) onDone(exchId.topologyVersion());
      }
    } else sendPartitions();

    // Schedule another send.
    scheduleRecheck();
  }
Example #17
0
  /**
   * Loads test configuration.
   *
   * @throws Exception if configuration is unawailable or broken.
   */
  private void loadTestConfiguration() throws Exception {
    assert TEST_CONFIGURATION_FILE.isFile();

    InputStream in = null;

    Properties p = new Properties();

    try {
      in = new FileInputStream(TEST_CONFIGURATION_FILE);

      p.load(in);
    } finally {
      U.closeQuiet(in);
    }

    clientNodes = Integer.parseInt(p.getProperty("client.nodes.count"));
    srvNodes = Integer.parseInt(p.getProperty("server.nodes.count"));
    threadsPerClient = Integer.parseInt(p.getProperty("threads.per.client"));
    cancelRate = Integer.parseInt(p.getProperty("cancel.rate"));
    submitDelay = Long.parseLong(p.getProperty("submit.delay"));

    taskParams =
        new GridJobLoadTestParams(
            Integer.parseInt(p.getProperty("jobs.count")),
            Integer.parseInt(p.getProperty("jobs.test.duration")),
            Integer.parseInt(p.getProperty("jobs.test.completion.delay")),
            Double.parseDouble(p.getProperty("jobs.failure.probability")));
  }
  /** {@inheritDoc} */
  @Override
  protected final boolean tryReleaseShared(int ignore) {
    endTime = U.currentTimeMillis();

    // Always signal after setting final done status.
    return true;
  }
Example #19
0
 static {
   try {
     TEST_CONF_DIR =
         new File(U.resolveGridGainUrl("/modules/core/src/test/config/job-loadtest").toURI());
   } catch (URISyntaxException e) {
     throw new RuntimeException("Failed to initialize directory.", e);
   }
 }
Example #20
0
 /**
  * Delegates to {@link #call()} method.
  *
  * <p>{@inheritDoc}
  *
  * @return {@inheritDoc}
  * @throws GridException {@inheritDoc}
  */
 @Override
 public final Object execute() throws GridException {
   try {
     return call();
   } catch (Throwable e) {
     throw U.cast(e);
   }
 }
  /**
   * Reconstructs object on demarshalling.
   *
   * @return Reconstructed object.
   * @throws ObjectStreamException Thrown in case of demarshalling error.
   */
  private Object readResolve() throws ObjectStreamException {
    GridTuple2<GridCacheContext, String> t = stash.get();

    try {
      return t.get1().dataStructures().sequence(t.get2(), 0L, false, false);
    } catch (GridException e) {
      throw U.withCause(new InvalidObjectException(e.getMessage()), e);
    }
  }
  /**
   * Log finished.
   *
   * @param log Logger.
   * @param clazz Class.
   * @param start Start time.
   */
  public static void logFinish(@Nullable IgniteLogger log, Class<?> clazz, long start) {
    final long end = U.currentTimeMillis();

    log0(
        log,
        end,
        String.format(
            "[%s]: FINISHED, duration: %s", clazz.getSimpleName(), formatDuration(end - start)));
  }
  /**
   * Reconstructs object on demarshalling.
   *
   * @return Reconstructed object.
   * @throws ObjectStreamException Thrown in case of demarshalling error.
   */
  @SuppressWarnings({"ConstantConditions"})
  private Object readResolve() throws ObjectStreamException {
    GridTuple2<GridCacheContext, String> t = stash.get();

    try {
      return t.get1().dataStructures().countDownLatch(t.get2(), 0, false, false);
    } catch (GridException e) {
      throw U.withCause(new InvalidObjectException(e.getMessage()), e);
    }
  }
  /** Perform cleanup of the trash directory. */
  private void delete() {
    IgfsFileInfo info = null;

    try {
      info = meta.info(TRASH_ID);
    } catch (ClusterTopologyServerNotFoundException e) {
      LT.warn(log, e, "Server nodes not found.");
    } catch (IgniteCheckedException e) {
      U.error(log, "Cannot obtain trash directory info.", e);
    }

    if (info != null) {
      for (Map.Entry<String, IgfsListingEntry> entry : info.listing().entrySet()) {
        IgniteUuid fileId = entry.getValue().fileId();

        if (log.isDebugEnabled())
          log.debug(
              "Deleting IGFS trash entry [name=" + entry.getKey() + ", fileId=" + fileId + ']');

        try {
          if (!cancelled) {
            if (delete(entry.getKey(), fileId)) {
              if (log.isDebugEnabled())
                log.debug(
                    "Sending delete confirmation message [name="
                        + entry.getKey()
                        + ", fileId="
                        + fileId
                        + ']');

              sendDeleteMessage(new IgfsDeleteMessage(fileId));
            }
          } else break;
        } catch (IgniteInterruptedCheckedException ignored) {
          // Ignore this exception while stopping.
        } catch (IgniteCheckedException e) {
          U.error(log, "Failed to delete entry from the trash directory: " + entry.getKey(), e);

          sendDeleteMessage(new IgfsDeleteMessage(fileId, e));
        }
      }
    }
  }
    /** {@inheritDoc} */
    @Override
    public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) {
      try {
        if (executor.isShutdown()) throw new RejectedExecutionException();
        else executor.getQueue().put(r);
      } catch (InterruptedException ignored) {
        U.warn(log, "Working thread was interrupted while loading data.");

        Thread.currentThread().interrupt();
      }
    }
  /**
   * Log message.
   *
   * @param log Logger.
   * @param msg Message to log.
   * @param clazz class.
   * @param start start time.
   * @return Time when message was logged.
   */
  public static long log(@Nullable IgniteLogger log, String msg, Class<?> clazz, long start) {
    final long end = U.currentTimeMillis();

    log0(
        log,
        end,
        String.format(
            "[%s]: %s, duration: %s", clazz.getSimpleName(), msg, formatDuration(end - start)));

    return end;
  }
Example #27
0
  /** @param jobId Job id. */
  public void jobFinished(GridHadoopJobId jobId) {
    GridHadoopShuffleJob job = jobs.remove(jobId);

    if (job != null) {
      try {
        job.close();
      } catch (GridException e) {
        U.error(log, "Failed to close job: " + jobId, e);
      }
    }
  }
  /**
   * @param cacheCtx Cache context.
   * @return {@code True} if local node can calculate affinity on it's own for this partition map
   *     exchange.
   */
  private boolean canCalculateAffinity(GridCacheContext cacheCtx) {
    AffinityFunction affFunc = cacheCtx.config().getAffinity();

    // Do not request affinity from remote nodes if affinity function is not centralized.
    if (!U.hasAnnotation(affFunc, AffinityCentralizedFunction.class)) return true;

    // If local node did not initiate exchange or local node is the only cache node in grid.
    Collection<ClusterNode> affNodes = CU.affinityNodes(cacheCtx, exchId.topologyVersion());

    return !exchId.nodeId().equals(cctx.localNodeId())
        || (affNodes.size() == 1 && affNodes.contains(cctx.localNode()));
  }
  /**
   * Reconstructs object on demarshalling.
   *
   * @return Reconstructed object.
   * @throws ObjectStreamException Thrown in case of demarshalling error.
   */
  @SuppressWarnings("unchecked")
  private Object readResolve() throws ObjectStreamException {
    try {
      GridBiTuple<GridCacheContext, String> t = stash.get();

      return t.get1().dataStructures().atomicReference(t.get2(), null, false);
    } catch (GridException e) {
      throw U.withCause(new InvalidObjectException(e.getMessage()), e);
    } finally {
      stash.remove();
    }
  }
Example #30
0
  /**
   * Flushes every internal buffer if buffer was flushed before passed in threshold.
   *
   * <p>Does not wait for result and does not fail on errors assuming that this method should be
   * called periodically.
   */
  @Override
  public void tryFlush() throws GridInterruptedException {
    if (!busyLock.enterBusy()) return;

    try {
      for (Buffer buf : bufMappings.values()) buf.flush();

      lastFlushTime = U.currentTimeMillis();
    } finally {
      leaveBusy();
    }
  }