// lame way to communicate with the coprocessor,
 // since it is loaded by a different class loader
 @Override
 public void prePut(
     final ObserverContext<RegionCoprocessorEnvironment> c,
     final Put put,
     final WALEdit edit,
     final Durability durability)
     throws IOException {
   if (put.getAttribute("ttl") != null) {
     Cell cell = put.getFamilyCellMap().values().iterator().next().get(0);
     KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
     ttls.put(TableName.valueOf(kv.getQualifier()), Bytes.toLong(kv.getValue()));
     c.bypass();
   } else if (put.getAttribute("versions") != null) {
     Cell cell = put.getFamilyCellMap().values().iterator().next().get(0);
     KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
     versions.put(TableName.valueOf(kv.getQualifier()), Bytes.toInt(kv.getValue()));
     c.bypass();
   }
 }
Beispiel #2
0
  /**
   * Verifies the result from get or scan using the dataGenerator (that was presumably also used to
   * generate said result).
   *
   * @param verifyValues verify that values in the result make sense for row/cf/column combination
   * @param verifyCfAndColumnIntegrity verify that cf/column set in the result is complete. Note
   *     that to use this multiPut should be used, or verification has to happen after writes,
   *     otherwise there can be races.
   * @return
   */
  public boolean verifyResultAgainstDataGenerator(
      Result result, boolean verifyValues, boolean verifyCfAndColumnIntegrity) {
    String rowKeyStr = Bytes.toString(result.getRow());
    // See if we have any data at all.
    if (result.isEmpty()) {
      LOG.error("Error checking data for key [" + rowKeyStr + "], no data returned");
      printLocations(result);
      return false;
    }

    if (!verifyValues && !verifyCfAndColumnIntegrity) {
      return true; // as long as we have something, we are good.
    }

    // See if we have all the CFs.
    byte[][] expectedCfs = dataGenerator.getColumnFamilies();
    if (verifyCfAndColumnIntegrity && (expectedCfs.length != result.getMap().size())) {
      LOG.error(
          "Error checking data for key ["
              + rowKeyStr
              + "], bad family count: "
              + result.getMap().size());
      printLocations(result);
      return false;
    }

    // Verify each column family from get in the result.
    for (byte[] cf : result.getMap().keySet()) {
      String cfStr = Bytes.toString(cf);
      Map<byte[], byte[]> columnValues = result.getFamilyMap(cf);
      if (columnValues == null) {
        LOG.error(
            "Error checking data for key [" + rowKeyStr + "], no data for family [" + cfStr + "]]");
        printLocations(result);
        return false;
      }

      Map<String, MutationType> mutateInfo = null;
      if (verifyCfAndColumnIntegrity || verifyValues) {
        if (!columnValues.containsKey(MUTATE_INFO)) {
          LOG.error(
              "Error checking data for key ["
                  + rowKeyStr
                  + "], column family ["
                  + cfStr
                  + "], column ["
                  + Bytes.toString(MUTATE_INFO)
                  + "]; value is not found");
          printLocations(result);
          return false;
        }

        long cfHash = Arrays.hashCode(cf);
        // Verify deleted columns, and make up column counts if deleted
        byte[] mutateInfoValue = columnValues.remove(MUTATE_INFO);
        mutateInfo = parseMutateInfo(mutateInfoValue);
        for (Map.Entry<String, MutationType> mutate : mutateInfo.entrySet()) {
          if (mutate.getValue() == MutationType.DELETE) {
            byte[] column = Bytes.toBytes(mutate.getKey());
            long columnHash = Arrays.hashCode(column);
            long hashCode = cfHash + columnHash;
            if (hashCode % 2 == 0) {
              if (columnValues.containsKey(column)) {
                LOG.error(
                    "Error checking data for key ["
                        + rowKeyStr
                        + "], column family ["
                        + cfStr
                        + "], column ["
                        + mutate.getKey()
                        + "]; should be deleted");
                printLocations(result);
                return false;
              }
              byte[] hashCodeBytes = Bytes.toBytes(hashCode);
              columnValues.put(column, hashCodeBytes);
            }
          }
        }

        // Verify increment
        if (!columnValues.containsKey(INCREMENT)) {
          LOG.error(
              "Error checking data for key ["
                  + rowKeyStr
                  + "], column family ["
                  + cfStr
                  + "], column ["
                  + Bytes.toString(INCREMENT)
                  + "]; value is not found");
          printLocations(result);
          return false;
        }
        long currentValue = Bytes.toLong(columnValues.remove(INCREMENT));
        if (verifyValues) {
          long amount = mutateInfo.isEmpty() ? 0 : cfHash;
          long originalValue = Arrays.hashCode(result.getRow());
          long extra = currentValue - originalValue;
          if (extra != 0 && (amount == 0 || extra % amount != 0)) {
            LOG.error(
                "Error checking data for key ["
                    + rowKeyStr
                    + "], column family ["
                    + cfStr
                    + "], column [increment], extra ["
                    + extra
                    + "], amount ["
                    + amount
                    + "]");
            printLocations(result);
            return false;
          }
          if (amount != 0 && extra != amount) {
            LOG.warn(
                "Warning checking data for key ["
                    + rowKeyStr
                    + "], column family ["
                    + cfStr
                    + "], column [increment], incremented ["
                    + (extra / amount)
                    + "] times");
          }
        }

        // See if we have correct columns.
        if (verifyCfAndColumnIntegrity
            && !dataGenerator.verify(result.getRow(), cf, columnValues.keySet())) {
          String colsStr = "";
          for (byte[] col : columnValues.keySet()) {
            if (colsStr.length() > 0) {
              colsStr += ", ";
            }
            colsStr += "[" + Bytes.toString(col) + "]";
          }
          LOG.error(
              "Error checking data for key ["
                  + rowKeyStr
                  + "], bad columns for family ["
                  + cfStr
                  + "]: "
                  + colsStr);
          printLocations(result);
          return false;
        }
        // See if values check out.
        if (verifyValues) {
          for (Map.Entry<byte[], byte[]> kv : columnValues.entrySet()) {
            String column = Bytes.toString(kv.getKey());
            MutationType mutation = mutateInfo.get(column);
            boolean verificationNeeded = true;
            byte[] bytes = kv.getValue();
            if (mutation != null) {
              boolean mutationVerified = true;
              long columnHash = Arrays.hashCode(kv.getKey());
              long hashCode = cfHash + columnHash;
              byte[] hashCodeBytes = Bytes.toBytes(hashCode);
              if (mutation == MutationType.APPEND) {
                int offset = bytes.length - hashCodeBytes.length;
                mutationVerified =
                    offset > 0
                        && Bytes.equals(
                            hashCodeBytes,
                            0,
                            hashCodeBytes.length,
                            bytes,
                            offset,
                            hashCodeBytes.length);
                if (mutationVerified) {
                  int n = 1;
                  while (true) {
                    int newOffset = offset - hashCodeBytes.length;
                    if (newOffset < 0
                        || !Bytes.equals(
                            hashCodeBytes,
                            0,
                            hashCodeBytes.length,
                            bytes,
                            newOffset,
                            hashCodeBytes.length)) {
                      break;
                    }
                    offset = newOffset;
                    n++;
                  }
                  if (n > 1) {
                    LOG.warn(
                        "Warning checking data for key ["
                            + rowKeyStr
                            + "], column family ["
                            + cfStr
                            + "], column ["
                            + column
                            + "], appended ["
                            + n
                            + "] times");
                  }
                  byte[] dest = new byte[offset];
                  System.arraycopy(bytes, 0, dest, 0, offset);
                  bytes = dest;
                }
              } else if (hashCode % 2 == 0) { // checkAndPut
                mutationVerified = Bytes.equals(bytes, hashCodeBytes);
                verificationNeeded = false;
              }
              if (!mutationVerified) {
                LOG.error(
                    "Error checking data for key ["
                        + rowKeyStr
                        + "], mutation checking failed for column family ["
                        + cfStr
                        + "], column ["
                        + column
                        + "]; mutation ["
                        + mutation
                        + "], hashCode ["
                        + hashCode
                        + "], verificationNeeded ["
                        + verificationNeeded
                        + "]");
                printLocations(result);
                return false;
              }
            } // end of mutation checking
            if (verificationNeeded
                && !dataGenerator.verify(result.getRow(), cf, kv.getKey(), bytes)) {
              LOG.error(
                  "Error checking data for key ["
                      + rowKeyStr
                      + "], column family ["
                      + cfStr
                      + "], column ["
                      + column
                      + "], mutation ["
                      + mutation
                      + "]; value of length "
                      + bytes.length);
              printLocations(result);
              return false;
            }
          }
        }
      }
    }
    return true;
  }