/** @param e Error. */
  void onError(Throwable e) {
    tx.commitError(e);

    if (err.compareAndSet(null, e)) {
      boolean marked = tx.setRollbackOnly();

      if (e instanceof GridCacheTxRollbackException) {
        if (marked) {
          try {
            tx.rollback();
          } catch (GridException ex) {
            U.error(log, "Failed to automatically rollback transaction: " + tx, ex);
          }
        }
      } else if (tx.implicit()
          && tx.isSystemInvalidate()) { // Finish implicit transaction on heuristic error.
        try {
          tx.close();
        } catch (GridException ex) {
          U.error(log, "Failed to invalidate transaction: " + tx, ex);
        }
      }

      onComplete();
    }
  }
  /**
   * @param cctx Context.
   * @param id Partition ID.
   */
  @SuppressWarnings("ExternalizableWithoutPublicNoArgConstructor")
  GridDhtLocalPartition(GridCacheContext cctx, int id) {
    assert cctx != null;

    this.id = id;
    this.cctx = cctx;

    log = U.logger(cctx.kernalContext(), logRef, this);

    rent =
        new GridFutureAdapter<Object>() {
          @Override
          public String toString() {
            return "PartitionRentFuture [part=" + GridDhtLocalPartition.this + ", map=" + map + ']';
          }
        };

    map = new ConcurrentHashMap8<>(cctx.config().getStartSize() / cctx.affinity().partitions());

    int delQueueSize =
        CU.isSystemCache(cctx.name())
            ? 100
            : Math.max(MAX_DELETE_QUEUE_SIZE / cctx.affinity().partitions(), 20);

    rmvQueue = new GridCircularBuffer<>(U.ceilPow2(delQueueSize));
  }
Example #3
0
  /**
   * Decodes value from a given byte array to the object according to the flags given.
   *
   * @param flags Flags.
   * @param bytes Byte array to decode.
   * @return Decoded value.
   * @throws GridException If deserialization failed.
   */
  private Object decodeObj(short flags, byte[] bytes) throws GridException {
    assert bytes != null;

    if ((flags & SERIALIZED_FLAG) != 0)
      return jdkMarshaller.unmarshal(new ByteArrayInputStream(bytes), null);

    int masked = flags & 0xff00;

    switch (masked) {
      case BOOLEAN_FLAG:
        return bytes[0] == '1';
      case INT_FLAG:
        return U.bytesToInt(bytes, 0);
      case LONG_FLAG:
        return U.bytesToLong(bytes, 0);
      case DATE_FLAG:
        return new Date(U.bytesToLong(bytes, 0));
      case BYTE_FLAG:
        return bytes[0];
      case FLOAT_FLAG:
        return Float.intBitsToFloat(U.bytesToInt(bytes, 0));
      case DOUBLE_FLAG:
        return Double.longBitsToDouble(U.bytesToLong(bytes, 0));
      case BYTE_ARR_FLAG:
        return bytes;
      default:
        return new String(bytes);
    }
  }
  /** {@inheritDoc} */
  @Nullable
  @Override
  public Map<? extends GridComputeJob, GridNode> map(
      List<GridNode> subgrid, @Nullable GridBiTuple<Set<UUID>, A> arg) throws GridException {
    assert arg != null;
    assert arg.get1() != null;

    start = U.currentTimeMillis();

    boolean debug = debugState(g);

    if (debug) logStart(g.log(), getClass(), start);

    Set<UUID> nodeIds = arg.get1();

    Map<GridComputeJob, GridNode> map = U.newHashMap(nodeIds.size());

    try {
      taskArg = arg.get2();

      for (GridNode node : subgrid) if (nodeIds.contains(node.id())) map.put(job(taskArg), node);

      return map;
    } finally {
      if (debug) logMapped(g.log(), getClass(), map.values());
    }
  }
    /** @param workTokDir Token directory (common for multiple nodes). */
    private void cleanupResources(File workTokDir) {
      RandomAccessFile lockFile = null;

      FileLock lock = null;

      try {
        lockFile = new RandomAccessFile(new File(workTokDir, LOCK_FILE_NAME), "rw");

        lock = lockFile.getChannel().lock();

        if (lock != null) processTokenDirectory(workTokDir);
        else if (log.isDebugEnabled())
          log.debug(
              "Token directory is being processed concurrently: " + workTokDir.getAbsolutePath());
      } catch (OverlappingFileLockException ignored) {
        if (log.isDebugEnabled())
          log.debug(
              "Token directory is being processed concurrently: " + workTokDir.getAbsolutePath());
      } catch (FileLockInterruptionException ignored) {
        Thread.currentThread().interrupt();
      } catch (IOException e) {
        U.error(log, "Failed to process directory: " + workTokDir.getAbsolutePath(), e);
      } finally {
        U.releaseQuiet(lock);
        U.closeQuiet(lockFile);
      }
    }
  /**
   * Creates new HTTP requests handler.
   *
   * @param hnd Handler.
   * @param authChecker Authentication checking closure.
   * @param log Logger.
   */
  GridJettyRestHandler(
      GridRestProtocolHandler hnd, GridClosure<String, Boolean> authChecker, GridLogger log) {
    assert hnd != null;
    assert log != null;

    this.hnd = hnd;
    this.log = log;
    this.authChecker = authChecker;

    // Init default page and favicon.
    try {
      initDefaultPage();

      if (log.isDebugEnabled()) log.debug("Initialized default page.");
    } catch (IOException e) {
      U.warn(log, "Failed to initialize default page: " + e.getMessage());
    }

    try {
      initFavicon();

      if (log.isDebugEnabled())
        log.debug(
            favicon != null ? "Initialized favicon, size: " + favicon.length : "Favicon is null.");
    } catch (IOException e) {
      U.warn(log, "Failed to initialize favicon: " + e.getMessage());
    }
  }
Example #7
0
  /**
   * Message received callback.
   *
   * @param src Sender node ID.
   * @param msg Received message.
   * @return {@code True}.
   */
  public boolean onMessageReceived(UUID src, GridHadoopMessage msg) {
    if (msg instanceof GridHadoopShuffleMessage) {
      GridHadoopShuffleMessage m = (GridHadoopShuffleMessage) msg;

      try {
        job(m.jobId()).onShuffleMessage(m);
      } catch (GridException e) {
        U.error(log, "Message handling failed.", e);
      }

      try {
        // Reply with ack.
        send0(src, new GridHadoopShuffleAck(m.id(), m.jobId()));
      } catch (GridException e) {
        U.error(
            log,
            "Failed to reply back to shuffle message sender [snd=" + src + ", msg=" + msg + ']',
            e);
      }
    } else if (msg instanceof GridHadoopShuffleAck) {
      GridHadoopShuffleAck m = (GridHadoopShuffleAck) msg;

      try {
        job(m.jobId()).onShuffleAck(m);
      } catch (GridException e) {
        U.error(log, "Message handling failed.", e);
      }
    } else
      throw new IllegalStateException(
          "Unknown message type received to Hadoop shuffle [src=" + src + ", msg=" + msg + ']');

    return true;
  }
 /**
  * Log task mapped.
  *
  * @param log Logger.
  * @param clazz Task class.
  * @param nodes Mapped nodes.
  */
 public static void logMapped(
     @Nullable IgniteLogger log, Class<?> clazz, Collection<ClusterNode> nodes) {
   log0(
       log,
       U.currentTimeMillis(),
       String.format("[%s]: MAPPED: %s", clazz.getSimpleName(), U.toShortString(nodes)));
 }
Example #9
0
  /** {@inheritDoc} */
  @Override
  public void writeExternal(ObjectOutput out) throws IOException {
    originatingTxId.writeExternal(out);

    U.writeUuid(out, originatingNodeId);

    U.writeCollection(out, recoveryWrites);
  }
  /**
   * Processes unlock request.
   *
   * @param nodeId Sender node ID.
   * @param req Unlock request.
   */
  @SuppressWarnings({"unchecked"})
  private void processUnlockRequest(UUID nodeId, GridDistributedUnlockRequest req) {
    assert nodeId != null;

    try {
      ClassLoader ldr = ctx.deploy().globalLoader();
      List<byte[]> keys = req.keyBytes();

      for (byte[] keyBytes : keys) {
        K key = (K) U.unmarshal(ctx.marshaller(), new ByteArrayInputStream(keyBytes), ldr);

        while (true) {
          boolean created = false;

          GridDistributedCacheEntry<K, V> entry = peekexx(key);

          if (entry == null) {
            entry = entryexx(key);

            created = true;
          }

          try {
            entry.doneRemote(
                req.version(), req.version(), req.committedVersions(), req.rolledbackVersions());

            // Note that we don't reorder completed versions here,
            // as there is no point to reorder relative to the version
            // we are about to remove.
            if (entry.removeLock(req.version())) {
              if (log.isDebugEnabled())
                log.debug("Removed lock [lockId=" + req.version() + ", key=" + key + ']');

              if (created && entry.markObsolete(req.version())) removeIfObsolete(entry.key());
            } else if (log.isDebugEnabled())
              log.debug(
                  "Received unlock request for unknown candidate "
                      + "(added to cancelled locks set): "
                      + req);

            break;
          } catch (GridCacheEntryRemovedException ignored) {
            if (log.isDebugEnabled())
              log.debug(
                  "Received remove lock request for removed entry (will retry) [entry="
                      + entry
                      + ", req="
                      + req
                      + ']');
          }
        }
      }
    } catch (GridException e) {
      U.error(log, "Failed to unmarshal unlock key (unlock will not be performed): " + req, e);
    }
  }
  /** {@inheritDoc} */
  @Override
  public void writeExternal(ObjectOutput out) throws IOException {
    U.writeString(out, jobName);
    U.writeString(out, user);

    out.writeBoolean(hasCombiner);
    out.writeInt(numReduces);

    U.writeStringMap(out, props);
  }
  /** {@inheritDoc} */
  @Override
  public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
    jobName = U.readString(in);
    user = U.readString(in);

    hasCombiner = in.readBoolean();
    numReduces = in.readInt();

    props = U.readStringMap(in);
  }
Example #13
0
  /** {@inheritDoc} */
  @Override
  public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
    originatingTxId = new GridCacheVersion();

    originatingTxId.readExternal(in);

    originatingNodeId = U.readUuid(in);

    recoveryWrites = U.readCollection(in);
  }
  /** {@inheritDoc} */
  @Override
  public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
    src = U.readByteArray(in);

    depMode = GridDeploymentMode.fromOrdinal(in.readInt());

    clsLdrId = U.readGridUuid(in);
    srcClsName = U.readString(in);
    userVer = U.readString(in);
    ldrParties = U.readMap(in);
  }
  /** {@inheritDoc} */
  @Override
  public void writeExternal(ObjectOutput out) throws IOException {
    U.writeByteArray(out, src);

    out.writeInt(depMode.ordinal());

    U.writeGridUuid(out, clsLdrId);
    U.writeString(out, srcClsName);
    U.writeString(out, userVer);
    U.writeMap(out, ldrParties);
  }
Example #16
0
  /**
   * Waits for all workers to finish.
   *
   * @param cancel Flag to indicate whether workers should be cancelled before waiting for them to
   *     finish.
   */
  public void join(boolean cancel) {
    if (cancel) U.cancel(workers);

    // Record current interrupted status of calling thread.
    boolean interrupted = Thread.interrupted();

    try {
      U.join(workers, log);
    } finally {
      // Reset interrupted flag on calling thread.
      if (interrupted) Thread.currentThread().interrupt();
    }
  }
  /** {@inheritDoc} */
  @Override
  void onCancelAtStop() {
    super.onCancelAtStop();

    for (GridCacheQueryFutureAdapter fut : futs.values())
      try {
        fut.cancel();
      } catch (IgniteCheckedException e) {
        U.error(log, "Failed to cancel running query future: " + fut, e);
      }

    U.interrupt(threads.values());
  }
  /** {@inheritDoc} */
  @Override
  public void onClose() throws IgniteCheckedException {
    if (data == null)
      // Nothing to close.
      return;

    try {
      U.closeQuiet(data.getStatement());
    } catch (SQLException e) {
      throw new IgniteCheckedException(e);
    }

    U.closeQuiet(data);
  }
  /** {@inheritDoc} */
  @Override
  public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
    depEnabled = in.readBoolean();

    if (depEnabled) {
      topicBytes = U.readByteArray(in);
      predBytes = U.readByteArray(in);
      clsName = U.readString(in);
      depInfo = (GridDeploymentInfoBean) in.readObject();
    } else {
      topic = in.readObject();
      pred = (GridBiPredicate<UUID, Object>) in.readObject();
    }
  }
  /** {@inheritDoc} */
  @Override
  public void writeExternal(ObjectOutput out) throws IOException {
    out.writeBoolean(depEnabled);

    if (depEnabled) {
      U.writeByteArray(out, topicBytes);
      U.writeByteArray(out, predBytes);
      U.writeString(out, clsName);
      out.writeObject(depInfo);
    } else {
      out.writeObject(topic);
      out.writeObject(pred);
    }
  }
Example #21
0
  /**
   * Encodes given object to a byte array and returns flags that describe the type of serialized
   * object.
   *
   * @param obj Object to serialize.
   * @param out Output stream to which object should be written.
   * @return Serialization flags.
   * @throws GridException If JDK serialization failed.
   */
  private int encodeObj(Object obj, ByteArrayOutputStream out) throws GridException {
    int flags = 0;

    byte[] data = null;

    if (obj instanceof String) {
      data = ((String) obj).getBytes();
    } else if (obj instanceof Boolean) {
      data = new byte[] {(byte) ((Boolean) obj ? '1' : '0')};

      flags |= BOOLEAN_FLAG;
    } else if (obj instanceof Integer) {
      data = U.intToBytes((Integer) obj);

      flags |= INT_FLAG;
    } else if (obj instanceof Long) {
      data = U.longToBytes((Long) obj);

      flags |= LONG_FLAG;
    } else if (obj instanceof Date) {
      data = U.longToBytes(((Date) obj).getTime());

      flags |= DATE_FLAG;
    } else if (obj instanceof Byte) {
      data = new byte[] {(Byte) obj};

      flags |= BYTE_FLAG;
    } else if (obj instanceof Float) {
      data = U.intToBytes(Float.floatToIntBits((Float) obj));

      flags |= FLOAT_FLAG;
    } else if (obj instanceof Double) {
      data = U.longToBytes(Double.doubleToLongBits((Double) obj));

      flags |= DOUBLE_FLAG;
    } else if (obj instanceof byte[]) {
      data = (byte[]) obj;

      flags |= BYTE_ARR_FLAG;
    } else {
      jdkMarshaller.marshal(obj, out);

      flags |= SERIALIZED_FLAG;
    }

    if (data != null) out.write(data, 0, data.length);

    return flags;
  }
  /** {@inheritDoc} */
  @Override
  public void writeExternal(ObjectOutput out) throws IOException {
    super.writeExternal(out);

    out.writeLong(threadId);
    out.writeBoolean(commit);
    out.writeBoolean(invalidate);
    out.writeBoolean(reply);

    U.writeGridUuid(out, futId);
    CU.writeVersion(out, commitVer);
    CU.writeVersion(out, baseVer);

    U.writeCollection(out, writeEntries);
  }
  /** {@inheritDoc} */
  @Override
  public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
    super.readExternal(in);

    threadId = in.readLong();
    commit = in.readBoolean();
    invalidate = in.readBoolean();
    reply = in.readBoolean();

    futId = U.readGridUuid(in);
    commitVer = CU.readVersion(in);
    baseVer = CU.readVersion(in);

    writeEntries = U.readList(in);
  }
    /**
     * Waits until total number of events processed is equal or greater then argument passed.
     *
     * @param cnt Number of events to wait.
     * @param timeout Timeout to wait.
     * @return {@code True} if successfully waited, {@code false} if timeout happened.
     * @throws InterruptedException If thread is interrupted.
     */
    public synchronized boolean awaitEvents(int cnt, long timeout) throws InterruptedException {
      long start = U.currentTimeMillis();

      long now = start;

      while (start + timeout > now) {
        if (evtCnt >= cnt) return true;

        wait(start + timeout - now);

        now = U.currentTimeMillis();
      }

      return false;
    }
  /** {@inheritDoc} */
  @Override
  public void loadCache(GridBiInClosure<K, V> c, @Nullable Object... args) throws GridException {
    ExecutorService exec =
        new ThreadPoolExecutor(
            threadsCnt,
            threadsCnt,
            0L,
            MILLISECONDS,
            new ArrayBlockingQueue<Runnable>(batchQueueSize),
            new BlockingRejectedExecutionHandler());

    Iterator<I> iter = inputIterator(args);

    Collection<I> buf = new ArrayList<>(batchSize);

    try {
      while (iter.hasNext()) {
        if (Thread.currentThread().isInterrupted()) {
          U.warn(log, "Working thread was interrupted while loading data.");

          break;
        }

        buf.add(iter.next());

        if (buf.size() == batchSize) {
          exec.submit(new Worker(c, buf, args));

          buf = new ArrayList<>(batchSize);
        }
      }

      if (!buf.isEmpty()) exec.submit(new Worker(c, buf, args));
    } catch (RejectedExecutionException ignored) {
      // Because of custom RejectedExecutionHandler.
      assert false : "RejectedExecutionException was thrown while it shouldn't.";
    } finally {
      exec.shutdown();

      try {
        exec.awaitTermination(Long.MAX_VALUE, MILLISECONDS);
      } catch (InterruptedException ignored) {
        U.warn(log, "Working thread was interrupted while waiting for put operations to complete.");

        Thread.currentThread().interrupt();
      }
    }
  }
  /** @throws IOException If failed. */
  private void initFavicon() throws IOException {
    assert favicon == null;

    InputStream in = getClass().getResourceAsStream("favicon.ico");

    if (in != null) {
      BufferedInputStream bis = new BufferedInputStream(in);

      ByteArrayOutputStream bos = new ByteArrayOutputStream();

      try {
        byte[] buf = new byte[2048];

        while (true) {
          int n = bis.read(buf);

          if (n == -1) break;

          bos.write(buf, 0, n);
        }

        favicon = bos.toByteArray();
      } finally {
        U.closeQuiet(bis);
      }
    }
  }
  private void sendPartitions() {
    ClusterNode oldestNode = this.oldestNode.get();

    try {
      sendLocalPartitions(oldestNode, exchId);
    } catch (ClusterTopologyCheckedException ignore) {
      if (log.isDebugEnabled())
        log.debug(
            "Oldest node left during partition exchange [nodeId="
                + oldestNode.id()
                + ", exchId="
                + exchId
                + ']');
    } catch (IgniteCheckedException e) {
      scheduleRecheck();

      U.error(
          log,
          "Failed to send local partitions to oldest node (will retry after timeout) [oldestNodeId="
              + oldestNode.id()
              + ", exchId="
              + exchId
              + ']',
          e);
    }
  }
  private void recheck() {
    // If this is the oldest node.
    if (oldestNode.get().id().equals(cctx.localNodeId())) {
      Collection<UUID> remaining = remaining();

      if (!remaining.isEmpty()) {
        try {
          cctx.io()
              .safeSend(
                  cctx.discovery().nodes(remaining),
                  new GridDhtPartitionsSingleRequest(exchId),
                  SYSTEM_POOL,
                  null);
        } catch (IgniteCheckedException e) {
          U.error(
              log,
              "Failed to request partitions from nodes [exchangeId="
                  + exchId
                  + ", nodes="
                  + remaining
                  + ']',
              e);
        }
      }
      // Resend full partition map because last attempt failed.
      else {
        if (spreadPartitions()) onDone(exchId.topologyVersion());
      }
    } else sendPartitions();

    // Schedule another send.
    scheduleRecheck();
  }
  /** {@inheritDoc} */
  @Override
  public final Map<UUID, GridNodeMetrics> metrics(Collection<UUID> nodeIds)
      throws GridSpiException {
    assert !F.isEmpty(nodeIds);

    long now = U.currentTimeMillis();

    Collection<UUID> expired = new LinkedList<>();

    for (UUID id : nodeIds) {
      GridNodeMetrics nodeMetrics = metricsMap.get(id);

      Long ts = tsMap.get(id);

      if (nodeMetrics == null || ts == null || ts < now - metricsExpireTime) expired.add(id);
    }

    if (!expired.isEmpty()) {
      Map<UUID, GridNodeMetrics> refreshed = metrics0(expired);

      for (UUID id : refreshed.keySet()) tsMap.put(id, now);

      metricsMap.putAll(refreshed);
    }

    return F.view(metricsMap, F.contains(nodeIds));
  }
  /**
   * Load classes from given file. File could be either directory or JAR file.
   *
   * @param clsLdr Class loader to load files.
   * @param file Either directory or JAR file which contains classes or references to them.
   * @return Set of found and loaded classes or empty set if file does not exist.
   * @throws GridSpiException Thrown if given JAR file references to none existed class or
   *     IOException occurred during processing.
   */
  static Set<Class<? extends GridTask<?, ?>>> getClasses(ClassLoader clsLdr, File file)
      throws GridSpiException {
    Set<Class<? extends GridTask<?, ?>>> rsrcs = new HashSet<Class<? extends GridTask<?, ?>>>();

    if (file.exists() == false) {
      return rsrcs;
    }

    GridUriDeploymentFileResourceLoader fileRsrcLdr =
        new GridUriDeploymentFileResourceLoader(clsLdr, file);

    if (file.isDirectory()) {
      findResourcesInDirectory(fileRsrcLdr, file, rsrcs);
    } else {
      try {
        for (JarEntry entry : U.asIterable(new JarFile(file.getAbsolutePath()).entries())) {
          Class<? extends GridTask<?, ?>> rsrc = fileRsrcLdr.createResource(entry.getName(), false);

          if (rsrc != null) {
            rsrcs.add(rsrc);
          }
        }
      } catch (IOException e) {
        throw new GridSpiException(
            "Failed to discover classes in file: " + file.getAbsolutePath(), e);
      }
    }

    return rsrcs;
  }