Esempio n. 1
0
  private Set<ByteArray> makeSlopKeys(
      Multimap<Integer, ByteArray> failedKeys, Slop.Operation operation) {
    Set<ByteArray> slopKeys = Sets.newHashSet();

    for (Map.Entry<Integer, ByteArray> entry : failedKeys.entries()) {
      byte[] opCode = new byte[] {operation.getOpCode()};
      byte[] spacer = new byte[] {(byte) 0};
      byte[] storeName = ByteUtils.getBytes(STORE_NAME, "UTF-8");
      byte[] nodeIdBytes = new byte[ByteUtils.SIZE_OF_INT];
      ByteUtils.writeInt(nodeIdBytes, entry.getKey(), 0);
      ByteArray slopKey =
          new ByteArray(
              ByteUtils.cat(
                  opCode, spacer, storeName, spacer, nodeIdBytes, spacer, entry.getValue().get()));
      slopKeys.add(slopKey);
    }
    return slopKeys;
  }
  /**
   * Create the voldemort key and value from the input key and value and map it out for each of the
   * responsible voldemort nodes
   *
   * <p>The output key is the md5 of the serialized key returned by makeKey(). The output value is
   * the node_id & partition_id of the responsible node followed by serialized value returned by
   * makeValue() OR if we have setKeys flag on the serialized key and serialized value
   */
  public void map(
      K key, V value, OutputCollector<BytesWritable, BytesWritable> output, Reporter reporter)
      throws IOException {
    byte[] keyBytes = keySerializer.toBytes(makeKey(key, value));
    byte[] valBytes = valueSerializer.toBytes(makeValue(key, value));

    // Compress key and values if required
    if (keySerializerDefinition.hasCompression()) {
      keyBytes = keyCompressor.deflate(keyBytes);
    }

    if (valueSerializerDefinition.hasCompression()) {
      valBytes = valueCompressor.deflate(valBytes);
    }

    // Get the output byte arrays ready to populate
    byte[] outputValue;
    BytesWritable outputKey;

    // Leave initial offset for (a) node id (b) partition id
    // since they are written later
    int offsetTillNow = 2 * ByteUtils.SIZE_OF_INT;

    // In order - 4 ( for node id ) + 4 ( partition id ) + 1 ( replica
    // type - primary | secondary | tertiary... ] + 4 ( key size )
    // size ) + 4 ( value size ) + key + value
    outputValue =
        new byte
            [valBytes.length
                + keyBytes.length
                + ByteUtils.SIZE_OF_BYTE
                + 4 * ByteUtils.SIZE_OF_INT];

    // Write key length - leave byte for replica type
    offsetTillNow += ByteUtils.SIZE_OF_BYTE;
    ByteUtils.writeInt(outputValue, keyBytes.length, offsetTillNow);

    // Write value length
    offsetTillNow += ByteUtils.SIZE_OF_INT;
    ByteUtils.writeInt(outputValue, valBytes.length, offsetTillNow);

    // Write key
    offsetTillNow += ByteUtils.SIZE_OF_INT;
    System.arraycopy(keyBytes, 0, outputValue, offsetTillNow, keyBytes.length);

    // Write value
    offsetTillNow += keyBytes.length;
    System.arraycopy(valBytes, 0, outputValue, offsetTillNow, valBytes.length);

    // Generate MR key - 16 byte md5
    outputKey = new BytesWritable(keyBytes);

    // Generate partition and node list this key is destined for
    List<Integer> partitionList = routingStrategy.getPartitionList(keyBytes);
    Node[] partitionToNode = routingStrategy.getPartitionToNode();

    for (int replicaType = 0; replicaType < partitionList.size(); replicaType++) {

      // Node id
      ByteUtils.writeInt(outputValue, partitionToNode[partitionList.get(replicaType)].getId(), 0);

      // Primary partition id
      ByteUtils.writeInt(outputValue, partitionList.get(0), ByteUtils.SIZE_OF_INT);

      // Replica type
      ByteUtils.writeBytes(
          outputValue, replicaType, 2 * ByteUtils.SIZE_OF_INT, ByteUtils.SIZE_OF_BYTE);

      BytesWritable outputVal = new BytesWritable(outputValue);

      output.collect(outputKey, outputVal);
    }
  }