Exemplo n.º 1
0
  /** {@inheritDoc} */
  @SuppressWarnings("TypeMayBeWeakened")
  @Nullable
  private Collection<Object> unmarshalFieldsCollection(
      @Nullable Collection<byte[]> byteCol, GridCacheContext<K, V> ctx, ClassLoader ldr)
      throws GridException {
    assert ctx != null;
    assert ldr != null;

    Collection<Object> col = unmarshalCollection(byteCol, ctx, ldr);
    Collection<Object> col0 = null;

    if (col != null) {
      col0 = new ArrayList<>(col.size());

      for (Object o : col) {
        List<Object> list = (List<Object>) o;
        List<Object> list0 = new ArrayList<>(list.size());

        for (Object obj : list)
          list0.add(obj != null ? ctx.marshaller().unmarshal((byte[]) obj, ldr) : null);

        col0.add(list0);
      }
    }

    return col0;
  }
Exemplo n.º 2
0
  /** {@inheritDoc} */
  @SuppressWarnings("TypeMayBeWeakened")
  @Nullable
  private Collection<byte[]> marshalFieldsCollection(
      @Nullable Collection<Object> col, GridCacheContext<K, V> ctx) throws GridException {
    assert ctx != null;

    if (col == null) return null;

    Collection<List<Object>> col0 = new ArrayList<>(col.size());

    for (Object o : col) {
      List<GridIndexingEntity<?>> list = (List<GridIndexingEntity<?>>) o;
      List<Object> list0 = new ArrayList<>(list.size());

      for (GridIndexingEntity<?> ent : list) {
        if (ent.bytes() != null) list0.add(ent.bytes());
        else {
          if (ctx.deploymentEnabled()) prepareObject(ent.value(), ctx);

          list0.add(CU.marshal(ctx, ent.value()));
        }
      }

      col0.add(list0);
    }

    return marshalCollection(col0, ctx);
  }
    /** {@inheritDoc} */
    @Override
    public Serializable reduce(List<GridComputeJobResult> results) throws GridException {
      if (log.isInfoEnabled()) log.info("Reducing job [job=" + this + ", results=" + results + ']');

      if (results.size() > 1) fail();

      return results.get(0).getData();
    }
  /**
   * Gets values referenced by sequential keys, e.g. {@code key1...keyN}.
   *
   * @param keyPrefix Key prefix, e.g. {@code key} for {@code key1...keyN}.
   * @param params Parameters map.
   * @return Values.
   */
  @Nullable
  protected List<Object> values(String keyPrefix, Map<String, Object> params) {
    assert keyPrefix != null;

    List<Object> vals = new LinkedList<>();

    for (int i = 1; ; i++) {
      String key = keyPrefix + i;

      if (params.containsKey(key)) vals.add(params.get(key));
      else break;
    }

    return vals;
  }
Exemplo n.º 5
0
  /** {@inheritDoc} */
  @Override
  public void run() {
    List<Grid> clientGrids = runGrid();

    assert clientGrids.size() == clientNodes;

    int threadsCnt = clientNodes * threadsPerClient;

    Executor e = Executors.newFixedThreadPool(threadsCnt);

    for (Grid grid : clientGrids) {
      for (int j = 0; j < threadsPerClient; j++)
        e.execute(new GridJobLoadTestSubmitter(grid, taskParams, cancelRate, submitDelay));
    }
  }
Exemplo n.º 6
0
  /**
   * Run all grid nodes as defined in test configuration.
   *
   * @return list of run nodes.
   */
  private List<Grid> runGrid() {
    List<Grid> clientGrids = new ArrayList<>(clientNodes);

    try {
      loadTestConfiguration();

      for (int i = 0; i < srvNodes; i++) startNode("server", SERVER_NODE_CONFIGURATION);

      // Start clients in the second order to cache a client node in GridGain.
      for (int i = 0; i < clientNodes; i++)
        clientGrids.add(startNode("client", CLIENT_NODE_CONFIGURATION));
    } catch (Exception e) {
      throw new RuntimeException(e);
    }

    return clientGrids;
  }
  /**
   * Creates REST request.
   *
   * @param cmd Command.
   * @param params Parameters.
   * @return REST request.
   * @throws GridException If creation failed.
   */
  @Nullable
  private GridRestRequest createRequest(
      GridRestCommand cmd, Map<String, Object> params, ServletRequest req) throws GridException {
    GridRestRequest restReq;

    switch (cmd) {
      case CACHE_GET:
      case CACHE_GET_ALL:
      case CACHE_PUT:
      case CACHE_PUT_ALL:
      case CACHE_REMOVE:
      case CACHE_REMOVE_ALL:
      case CACHE_ADD:
      case CACHE_CAS:
      case CACHE_METRICS:
      case CACHE_REPLACE:
      case CACHE_DECREMENT:
      case CACHE_INCREMENT:
      case CACHE_APPEND:
      case CACHE_PREPEND:
        {
          GridRestCacheRequest restReq0 = new GridRestCacheRequest();

          restReq0.cacheName((String) params.get("cacheName"));
          restReq0.key(params.get("key"));
          restReq0.value(params.get("val"));
          restReq0.value2(params.get("val2"));

          Object val1 = params.get("val1");

          if (val1 != null) restReq0.value(val1);

          restReq0.cacheFlags(intValue("cacheFlags", params, 0));
          restReq0.ttl(longValue("exp", params, null));
          restReq0.initial(longValue("init", params, null));
          restReq0.delta(longValue("delta", params, null));

          if (cmd == CACHE_GET_ALL || cmd == CACHE_PUT_ALL || cmd == CACHE_REMOVE_ALL) {
            List<Object> keys = values("k", params);
            List<Object> vals = values("v", params);

            if (keys.size() < vals.size())
              throw new GridException(
                  "Number of keys must be greater or equals to number of values.");

            Map<Object, Object> map = U.newHashMap(keys.size());

            Iterator<Object> keyIt = keys.iterator();
            Iterator<Object> valIt = vals.iterator();

            while (keyIt.hasNext()) map.put(keyIt.next(), valIt.hasNext() ? valIt.next() : null);

            restReq0.values(map);
          }

          restReq = restReq0;

          break;
        }

      case TOPOLOGY:
      case NODE:
        {
          GridRestTopologyRequest restReq0 = new GridRestTopologyRequest();

          restReq0.includeMetrics(Boolean.parseBoolean((String) params.get("mtr")));
          restReq0.includeAttributes(Boolean.parseBoolean((String) params.get("attr")));

          restReq0.nodeIp((String) params.get("ip"));

          restReq0.nodeId(uuidValue("id", params));

          restReq = restReq0;

          break;
        }

      case EXE:
      case RESULT:
      case NOOP:
        {
          GridRestTaskRequest restReq0 = new GridRestTaskRequest();

          restReq0.taskId((String) params.get("id"));
          restReq0.taskName((String) params.get("name"));

          restReq0.params(values("p", params));

          restReq0.async(Boolean.parseBoolean((String) params.get("async")));

          restReq0.timeout(longValue("timeout", params, 0L));

          restReq = restReq0;

          break;
        }

      case LOG:
        {
          GridRestLogRequest restReq0 = new GridRestLogRequest();

          restReq0.path((String) params.get("path"));

          restReq0.from(intValue("from", params, -1));
          restReq0.to(intValue("to", params, -1));

          restReq = restReq0;

          break;
        }

      case VERSION:
        {
          restReq = new GridRestRequest();

          break;
        }

      default:
        throw new GridException("Invalid command: " + cmd);
    }

    restReq.address(new InetSocketAddress(req.getRemoteAddr(), req.getRemotePort()));

    restReq.command(cmd);

    if (params.containsKey("gridgain.login") || params.containsKey("gridgain.password")) {
      GridSecurityCredentials cred =
          new GridSecurityCredentials(
              (String) params.get("gridgain.login"), (String) params.get("gridgain.password"));

      restReq.credentials(cred);
    }

    String clientId = (String) params.get("clientId");

    try {
      if (clientId != null) restReq.clientId(UUID.fromString(clientId));
    } catch (Exception ignored) {
      // Ignore invalid client id. Rest handler will process this logic.
    }

    String destId = (String) params.get("destId");

    try {
      if (destId != null) restReq.destinationId(UUID.fromString(destId));
    } catch (IllegalArgumentException ignored) {
      // Don't fail - try to execute locally.
    }

    String sesTokStr = (String) params.get("sessionToken");

    try {
      if (sesTokStr != null) restReq.sessionToken(U.hexString2ByteArray(sesTokStr));
    } catch (IllegalArgumentException ignored) {
      // Ignore invalid session token.
    }

    return restReq;
  }
Exemplo n.º 8
0
  /**
   * Performs flush.
   *
   * @throws GridException If failed.
   */
  private void doFlush() throws GridException {
    lastFlushTime = U.currentTimeMillis();

    List<GridFuture> activeFuts0 = null;

    int doneCnt = 0;

    for (GridFuture<?> f : activeFuts) {
      if (!f.isDone()) {
        if (activeFuts0 == null) activeFuts0 = new ArrayList<>((int) (activeFuts.size() * 1.2));

        activeFuts0.add(f);
      } else {
        f.get();

        doneCnt++;
      }
    }

    if (activeFuts0 == null || activeFuts0.isEmpty()) return;

    while (true) {
      Queue<GridFuture<?>> q = null;

      for (Buffer buf : bufMappings.values()) {
        GridFuture<?> flushFut = buf.flush();

        if (flushFut != null) {
          if (q == null) q = new ArrayDeque<>(bufMappings.size() * 2);

          q.add(flushFut);
        }
      }

      if (q != null) {
        assert !q.isEmpty();

        boolean err = false;

        for (GridFuture fut = q.poll(); fut != null; fut = q.poll()) {
          try {
            fut.get();
          } catch (GridException e) {
            if (log.isDebugEnabled()) log.debug("Failed to flush buffer: " + e);

            err = true;
          }
        }

        if (err)
          // Remaps needed - flush buffers.
          continue;
      }

      doneCnt = 0;

      for (int i = 0; i < activeFuts0.size(); i++) {
        GridFuture f = activeFuts0.get(i);

        if (f == null) doneCnt++;
        else if (f.isDone()) {
          f.get();

          doneCnt++;

          activeFuts0.set(i, null);
        } else break;
      }

      if (doneCnt == activeFuts0.size()) return;
    }
  }
  /** @throws Exception If failed. */
  public void testCreateFileFragmented() throws Exception {
    GridGgfsEx impl = (GridGgfsEx) grid(0).ggfs("ggfs");

    GridGgfsFragmentizerManager fragmentizer = impl.context().fragmentizer();

    GridTestUtils.setFieldValue(fragmentizer, "fragmentizerEnabled", false);

    GridGgfsPath path = new GridGgfsPath("/file");

    try {
      GridGgfs fs0 = grid(0).ggfs("ggfs");
      GridGgfs fs1 = grid(1).ggfs("ggfs");
      GridGgfs fs2 = grid(2).ggfs("ggfs");

      try (GridGgfsOutputStream out =
          fs0.create(
              path,
              128,
              false,
              1,
              CFG_GRP_SIZE,
              F.asMap(GridGgfs.PROP_PREFER_LOCAL_WRITES, "true"))) {
        // 1.5 blocks
        byte[] data = new byte[CFG_BLOCK_SIZE * 3 / 2];

        Arrays.fill(data, (byte) 1);

        out.write(data);
      }

      try (GridGgfsOutputStream out = fs1.append(path, false)) {
        // 1.5 blocks.
        byte[] data = new byte[CFG_BLOCK_SIZE * 3 / 2];

        Arrays.fill(data, (byte) 2);

        out.write(data);
      }

      // After this we should have first two block colocated with grid 0 and last block colocated
      // with grid 1.
      GridGgfsFileImpl fileImpl = (GridGgfsFileImpl) fs.info(path);

      GridCache<Object, Object> metaCache = grid(0).cachex(META_CACHE_NAME);

      GridGgfsFileInfo fileInfo = (GridGgfsFileInfo) metaCache.get(fileImpl.fileId());

      GridGgfsFileMap map = fileInfo.fileMap();

      List<GridGgfsFileAffinityRange> ranges = map.ranges();

      assertEquals(2, ranges.size());

      assertTrue(ranges.get(0).startOffset() == 0);
      assertTrue(ranges.get(0).endOffset() == 2 * CFG_BLOCK_SIZE - 1);

      assertTrue(ranges.get(1).startOffset() == 2 * CFG_BLOCK_SIZE);
      assertTrue(ranges.get(1).endOffset() == 3 * CFG_BLOCK_SIZE - 1);

      // Validate data read after colocated writes.
      try (GridGgfsInputStream in = fs2.open(path)) {
        // Validate first part of file.
        for (int i = 0; i < CFG_BLOCK_SIZE * 3 / 2; i++) assertEquals((byte) 1, in.read());

        // Validate second part of file.
        for (int i = 0; i < CFG_BLOCK_SIZE * 3 / 2; i++) assertEquals((byte) 2, in.read());

        assertEquals(-1, in.read());
      }
    } finally {
      GridTestUtils.setFieldValue(fragmentizer, "fragmentizerEnabled", true);

      boolean hasData = false;

      for (int i = 0; i < NODES_CNT; i++) hasData |= !grid(i).cachex(DATA_CACHE_NAME).isEmpty();

      assertTrue(hasData);

      fs.delete(path, true);
    }

    GridTestUtils.retryAssert(
        log,
        ASSERT_RETRIES,
        ASSERT_RETRY_INTERVAL,
        new CAX() {
          @Override
          public void applyx() {
            for (int i = 0; i < NODES_CNT; i++)
              assertTrue(grid(i).cachex(DATA_CACHE_NAME).isEmpty());
          }
        });
  }
  /**
   * This method is called to map or split grid task into multiple grid jobs. This is the first
   * method that gets called when task execution starts.
   *
   * @param data Task execution argument. Can be {@code null}. This is the same argument as the one
   *     passed into {@code Grid#execute(...)} methods.
   * @param subgrid Nodes available for this task execution. Note that order of nodes is guaranteed
   *     to be randomized by container. This ensures that every time you simply iterate through grid
   *     nodes, the order of nodes will be random which over time should result into all nodes being
   *     used equally.
   * @return Map of grid jobs assigned to subgrid node. Unless {@link
   *     GridComputeTaskContinuousMapper} is injected into task, if {@code null} or empty map is
   *     returned, exception will be thrown.
   * @throws GridException If mapping could not complete successfully. This exception will be thrown
   *     out of {@link GridComputeTaskFuture#get()} method.
   */
  @Override
  public Map<? extends GridComputeJob, GridNode> map(
      List<GridNode> subgrid, @Nullable final Collection<Integer> data) throws GridException {
    assert !subgrid.isEmpty();

    // Give preference to wanted node. Otherwise, take the first one.
    GridNode targetNode =
        F.find(
            subgrid,
            subgrid.get(0),
            new GridPredicate<GridNode>() {
              @Override
              public boolean apply(GridNode e) {
                return preferredNode.equals(e.id());
              }
            });

    return Collections.singletonMap(
        new GridComputeJobAdapter() {
          @GridLoggerResource private GridLogger log;

          @GridInstanceResource private Grid grid;

          @Override
          public Object execute() throws GridException {
            log.info("Going to put data: " + data.size());

            GridCache<Object, Object> cache = grid.cache(cacheName);

            assert cache != null;

            Map<Integer, T2<Integer, Collection<Integer>>> putMap = groupData(data);

            for (Map.Entry<Integer, T2<Integer, Collection<Integer>>> entry : putMap.entrySet()) {
              T2<Integer, Collection<Integer>> pair = entry.getValue();

              Object affKey = pair.get1();

              // Group lock partition.
              try (GridCacheTx tx =
                  cache.txStartPartition(
                      cache.affinity().partition(affKey),
                      optimistic ? OPTIMISTIC : PESSIMISTIC,
                      REPEATABLE_READ,
                      0,
                      pair.get2().size())) {
                for (Integer val : pair.get2()) cache.put(val, val);

                tx.commit();
              }
            }

            log.info("Finished put data: " + data.size());

            return data;
          }

          /**
           * Groups values by partitions.
           *
           * @param data Data to put.
           * @return Grouped map.
           */
          private Map<Integer, T2<Integer, Collection<Integer>>> groupData(Iterable<Integer> data) {
            GridCache<Object, Object> cache = grid.cache(cacheName);

            Map<Integer, T2<Integer, Collection<Integer>>> res = new HashMap<>();

            for (Integer val : data) {
              int part = cache.affinity().partition(val);

              T2<Integer, Collection<Integer>> tup = res.get(part);

              if (tup == null) {
                tup = new T2<Integer, Collection<Integer>>(val, new LinkedList<Integer>());

                res.put(part, tup);
              }

              tup.get2().add(val);
            }

            return res;
          }
        },
        targetNode);
  }