Example #1
0
  /**
   * Attempts to retrieve the specified tree rule from storage.
   *
   * @param tsdb The TSDB to use for storage access
   * @param tree_id ID of the tree the rule belongs to
   * @param level Level where the rule resides
   * @param order Order where the rule resides
   * @return A TreeRule object if found, null if it does not exist
   * @throws HBaseException if there was an issue
   * @throws IllegalArgumentException if the one of the required parameters was missing
   * @throws JSONException if the object could not be serialized
   */
  public static Deferred<TreeRule> fetchRule(
      final TSDB tsdb, final int tree_id, final int level, final int order) {
    if (tree_id < 1 || tree_id > 65535) {
      throw new IllegalArgumentException("Invalid Tree ID");
    }
    if (level < 0) {
      throw new IllegalArgumentException("Invalid rule level");
    }
    if (order < 0) {
      throw new IllegalArgumentException("Invalid rule order");
    }

    // fetch the whole row
    final GetRequest get = new GetRequest(tsdb.treeTable(), Tree.idToBytes(tree_id));
    get.family(Tree.TREE_FAMILY());
    get.qualifier(getQualifier(level, order));

    /** Called after fetching to parse the results */
    final class FetchCB implements Callback<Deferred<TreeRule>, ArrayList<KeyValue>> {

      @Override
      public Deferred<TreeRule> call(final ArrayList<KeyValue> row) {
        if (row == null || row.isEmpty()) {
          return Deferred.fromResult(null);
        }
        return Deferred.fromResult(parseFromStorage(row.get(0)));
      }
    }

    return tsdb.getClient().get(get).addCallbackDeferring(new FetchCB());
  }
Example #2
0
  public static void main(String[] args) throws Exception {
    ArgP argp = new ArgP();
    CliOptions.addCommon(argp);
    argp.addOption("--fix", "Fix errors as they're found.");
    args = CliOptions.parse(argp, args);
    if (args == null) {
      usage(argp, "Invalid usage.", 1);
    } else if (args.length < 3) {
      usage(argp, "Not enough arguments.", 2);
    }

    final HBaseClient client = CliOptions.clientFromOptions(argp);
    final byte[] table = argp.get("--table", "tsdb").getBytes();
    final TSDB tsdb =
        new TSDB(client, argp.get("--table", "tsdb"), argp.get("--uidtable", "tsdb-uid"));
    final boolean fix = argp.has("--fix");
    argp = null;
    int errors = 42;
    try {
      errors = fsck(tsdb, client, table, fix, args);
    } finally {
      tsdb.shutdown().joinUninterruptibly();
    }
    System.exit(errors == 0 ? 0 : 1);
  }
Example #3
0
 @Override
 public void initialize(TSDB tsdb) {
   if (tsdb == null) {
     throw new IllegalArgumentException("The TSDB object was null");
   }
   // a dummy config to check for throwing exceptions
   if (!tsdb.getConfig().hasProperty("tsd.core.storage_exception_handler.DummySEHPlugin.hosts")) {
     throw new IllegalArgumentException("Missing hosts config");
   }
 }
Example #4
0
  /**
   * Attempts to delete all rules belonging to the given tree.
   *
   * @param tsdb The TSDB to use for storage access
   * @param tree_id ID of the tree the rules belongs to
   * @return A deferred to wait on for completion. The value has no meaning and may be null.
   * @throws HBaseException if there was an issue
   * @throws IllegalArgumentException if the one of the required parameters was missing
   */
  public static Deferred<Object> deleteAllRules(final TSDB tsdb, final int tree_id) {
    if (tree_id < 1 || tree_id > 65535) {
      throw new IllegalArgumentException("Invalid Tree ID");
    }

    // fetch the whole row
    final GetRequest get = new GetRequest(tsdb.treeTable(), Tree.idToBytes(tree_id));
    get.family(Tree.TREE_FAMILY());

    /**
     * Called after fetching the requested row. If the row is empty, we just return, otherwise we
     * compile a list of qualifiers to delete and submit a single delete request to storage.
     */
    final class GetCB implements Callback<Deferred<Object>, ArrayList<KeyValue>> {

      @Override
      public Deferred<Object> call(final ArrayList<KeyValue> row) throws Exception {
        if (row == null || row.isEmpty()) {
          return Deferred.fromResult(null);
        }

        final ArrayList<byte[]> qualifiers = new ArrayList<byte[]>(row.size());

        for (KeyValue column : row) {
          if (column.qualifier().length > RULE_PREFIX.length
              && Bytes.memcmp(RULE_PREFIX, column.qualifier(), 0, RULE_PREFIX.length) == 0) {
            qualifiers.add(column.qualifier());
          }
        }

        final DeleteRequest delete =
            new DeleteRequest(
                tsdb.treeTable(),
                Tree.idToBytes(tree_id),
                Tree.TREE_FAMILY(),
                qualifiers.toArray(new byte[qualifiers.size()][]));
        return tsdb.getClient().delete(delete);
      }
    }

    return tsdb.getClient().get(get).addCallbackDeferring(new GetCB());
  }
Example #5
0
  /**
   * Attempts to delete the specified rule from storage
   *
   * @param tsdb The TSDB to use for storage access
   * @param tree_id ID of the tree the rule belongs to
   * @param level Level where the rule resides
   * @param order Order where the rule resides
   * @return A deferred without meaning. The response may be null and should only be used to track
   *     completion.
   * @throws HBaseException if there was an issue
   * @throws IllegalArgumentException if the one of the required parameters was missing
   */
  public static Deferred<Object> deleteRule(
      final TSDB tsdb, final int tree_id, final int level, final int order) {
    if (tree_id < 1 || tree_id > 65535) {
      throw new IllegalArgumentException("Invalid Tree ID");
    }
    if (level < 0) {
      throw new IllegalArgumentException("Invalid rule level");
    }
    if (order < 0) {
      throw new IllegalArgumentException("Invalid rule order");
    }

    final DeleteRequest delete =
        new DeleteRequest(
            tsdb.treeTable(),
            Tree.idToBytes(tree_id),
            Tree.TREE_FAMILY(),
            getQualifier(level, order));
    return tsdb.getClient().delete(delete);
  }
Example #6
0
  /**
   * Parses a query string "m=metric{tagk1=tagv1,...}" type query and returns a tsuid.
   *
   * @param data_query The query we're building
   * @throws BadRequestException if we are unable to parse the query or it is missing components
   * @todo - make this asynchronous
   */
  private String getTSUIDForMetric(final String query_string, TSDB tsdb) {
    if (query_string == null || query_string.isEmpty()) {
      throw new BadRequestException("The query string was empty");
    }

    // m is of the following forms:
    // metric[{tag=value,...}]
    // where the parts in square brackets `[' .. `]' are optional.
    final HashMap<String, String> tags = new HashMap<String, String>();
    String metric = null;
    try {
      metric = Tags.parseWithMetric(query_string, tags);
    } catch (IllegalArgumentException e) {
      throw new BadRequestException(e);
    }

    // sort the UIDs on tagk values
    final ByteMap<byte[]> tag_uids = new ByteMap<byte[]>();
    for (final Entry<String, String> pair : tags.entrySet()) {
      tag_uids.put(
          tsdb.getUID(UniqueIdType.TAGK, pair.getKey()),
          tsdb.getUID(UniqueIdType.TAGV, pair.getValue()));
    }

    // Byte Buffer to generate TSUID, pre allocated to the size of the TSUID
    final ByteArrayOutputStream buf =
        new ByteArrayOutputStream(
            TSDB.metrics_width() + tag_uids.size() * (TSDB.tagk_width() + TSDB.tagv_width()));
    try {
      buf.write(tsdb.getUID(UniqueIdType.METRIC, metric));
      for (final Entry<byte[], byte[]> uids : tag_uids.entrySet()) {
        buf.write(uids.getKey());
        buf.write(uids.getValue());
      }
    } catch (IOException e) {
      throw new BadRequestException(e);
    }
    final String tsuid = UniqueId.uidToString(buf.toByteArray());

    return tsuid;
  }
Example #7
0
 /**
  * Get the next random UID. Right now it uses 3 bytes as the metric with is 3. If it is 3 then it
  * can return only upto the max value a 3 byte integer can return, which is 2^31-1. In that case,
  * even though it is long, its range will be between 0 and 2^31-1
  *
  * @return random UID
  */
 public static long getRandomUID() {
   return getRandomUID(TSDB.metrics_width());
 }
Example #8
0
  /**
   * Assigns UIDs to the given metric, tagk or tagv names if applicable
   *
   * <p>This handler supports GET and POST whereby the GET command can parse query strings with the
   * {@code type} as their parameter and a comma separated list of values to assign UIDs to.
   *
   * <p>Multiple types and names can be provided in one call. Each name will be processed
   * independently and if there's an error (such as an invalid name or it is already assigned) the
   * error will be stored in a separate error map and other UIDs will be processed.
   *
   * @param tsdb The TSDB from the RPC router
   * @param query The query for this request
   */
  private void handleAssign(final TSDB tsdb, final HttpQuery query) {
    // only accept GET And POST
    if (query.method() != HttpMethod.GET && query.method() != HttpMethod.POST) {
      throw new BadRequestException(
          HttpResponseStatus.METHOD_NOT_ALLOWED,
          "Method not allowed",
          "The HTTP method [" + query.method().getName() + "] is not permitted for this endpoint");
    }

    final HashMap<String, List<String>> source;
    if (query.method() == HttpMethod.POST) {
      source = query.serializer().parseUidAssignV1();
    } else {
      source = new HashMap<String, List<String>>(3);
      // cut down on some repetitive code, split the query string values by
      // comma and add them to the source hash
      String[] types = {"metric", "tagk", "tagv"};
      for (int i = 0; i < types.length; i++) {
        final String values = query.getQueryStringParam(types[i]);
        if (values != null && !values.isEmpty()) {
          final String[] metrics = values.split(",");
          if (metrics != null && metrics.length > 0) {
            source.put(types[i], Arrays.asList(metrics));
          }
        }
      }
    }

    if (source.size() < 1) {
      throw new BadRequestException("Missing values to assign UIDs");
    }

    final Map<String, TreeMap<String, String>> response =
        new HashMap<String, TreeMap<String, String>>();

    int error_count = 0;
    for (Map.Entry<String, List<String>> entry : source.entrySet()) {
      final TreeMap<String, String> results = new TreeMap<String, String>();
      final TreeMap<String, String> errors = new TreeMap<String, String>();

      for (String name : entry.getValue()) {
        try {
          final byte[] uid = tsdb.assignUid(entry.getKey(), name);
          results.put(name, UniqueId.uidToString(uid));
        } catch (IllegalArgumentException e) {
          errors.put(name, e.getMessage());
          error_count++;
        }
      }

      response.put(entry.getKey(), results);
      if (errors.size() > 0) {
        response.put(entry.getKey() + "_errors", errors);
      }
    }

    if (error_count < 1) {
      query.sendReply(query.serializer().formatUidAssignV1(response));
    } else {
      query.sendReply(
          HttpResponseStatus.BAD_REQUEST, query.serializer().formatUidAssignV1(response));
    }
  }
Example #9
0
  /**
   * Handles CRUD calls to individual TSMeta data entries
   *
   * @param tsdb The TSDB from the RPC router
   * @param query The query for this request
   */
  private void handleTSMeta(final TSDB tsdb, final HttpQuery query) {

    final HttpMethod method = query.getAPIMethod();
    // GET
    if (method == HttpMethod.GET) {

      String tsuid = null;
      if (query.hasQueryStringParam("tsuid")) {
        tsuid = query.getQueryStringParam("tsuid");
        try {
          final TSMeta meta = TSMeta.getTSMeta(tsdb, tsuid).joinUninterruptibly();
          if (meta != null) {
            query.sendReply(query.serializer().formatTSMetaV1(meta));
          } else {
            throw new BadRequestException(
                HttpResponseStatus.NOT_FOUND, "Could not find Timeseries meta data");
          }
        } catch (NoSuchUniqueName e) {
          // this would only happen if someone deleted a UID but left the
          // the timeseries meta data
          throw new BadRequestException(
              HttpResponseStatus.NOT_FOUND, "Unable to find one of the UIDs", e);
        } catch (BadRequestException e) {
          throw e;
        } catch (Exception e) {
          throw new RuntimeException(e);
        }
      } else {
        String mquery = query.getRequiredQueryStringParam("m");
        // m is of the following forms:
        // metric[{tag=value,...}]
        // where the parts in square brackets `[' .. `]' are optional.
        final HashMap<String, String> tags = new HashMap<String, String>();
        String metric = null;
        try {
          metric = Tags.parseWithMetric(mquery, tags);
        } catch (IllegalArgumentException e) {
          throw new BadRequestException(e);
        }
        final TSUIDQuery tsuid_query = new TSUIDQuery(tsdb, metric, tags);
        try {
          final List<TSMeta> tsmetas = tsuid_query.getTSMetas().joinUninterruptibly();
          query.sendReply(query.serializer().formatTSMetaListV1(tsmetas));
        } catch (NoSuchUniqueName e) {
          throw new BadRequestException(
              HttpResponseStatus.NOT_FOUND, "Unable to find one of the UIDs", e);
        } catch (BadRequestException e) {
          throw e;
        } catch (RuntimeException e) {
          throw new BadRequestException(e);
        } catch (Exception e) {
          throw new RuntimeException(e);
        }
      }
      // POST / PUT
    } else if (method == HttpMethod.POST || method == HttpMethod.PUT) {

      final TSMeta meta;
      if (query.hasContent()) {
        meta = query.serializer().parseTSMetaV1();
      } else {
        meta = this.parseTSMetaQS(query);
      }

      /**
       * Storage callback used to determine if the storage call was successful or not. Also returns
       * the updated object from storage.
       */
      class SyncCB implements Callback<Deferred<TSMeta>, Boolean> {

        @Override
        public Deferred<TSMeta> call(Boolean success) throws Exception {
          if (!success) {
            throw new BadRequestException(
                HttpResponseStatus.INTERNAL_SERVER_ERROR,
                "Failed to save the TSMeta to storage",
                "This may be caused by another process modifying storage data");
          }

          return TSMeta.getTSMeta(tsdb, meta.getTSUID());
        }
      }

      if (meta.getTSUID() == null || meta.getTSUID().isEmpty()) {
        // we got a JSON object without TSUID. Try to find a timeseries spec of
        // the form "m": "metric{tagk=tagv,...}"
        final String metric = query.getRequiredQueryStringParam("m");
        final boolean create =
            query.getQueryStringParam("create") != null
                && query.getQueryStringParam("create").equals("true");
        final String tsuid = getTSUIDForMetric(metric, tsdb);

        class WriteCounterIfNotPresentCB implements Callback<Boolean, Boolean> {

          @Override
          public Boolean call(Boolean exists) throws Exception {
            if (!exists && create) {
              final PutRequest put =
                  new PutRequest(
                      tsdb.metaTable(),
                      UniqueId.stringToUid(tsuid),
                      TSMeta.FAMILY(),
                      TSMeta.COUNTER_QUALIFIER(),
                      Bytes.fromLong(0));
              tsdb.getClient().put(put);
            }

            return exists;
          }
        }

        try {
          // Check whether we have a TSMeta stored already
          final boolean exists = TSMeta.metaExistsInStorage(tsdb, tsuid).joinUninterruptibly();
          // set TSUID
          meta.setTSUID(tsuid);

          if (!exists && create) {
            // Write 0 to counter column if not present
            TSMeta.counterExistsInStorage(tsdb, UniqueId.stringToUid(tsuid))
                .addCallback(new WriteCounterIfNotPresentCB())
                .joinUninterruptibly();
            // set TSUID
            final Deferred<TSMeta> process_meta =
                meta.storeNew(tsdb).addCallbackDeferring(new SyncCB());
            final TSMeta updated_meta = process_meta.joinUninterruptibly();
            tsdb.indexTSMeta(updated_meta);
            tsdb.processTSMetaThroughTrees(updated_meta);
            query.sendReply(query.serializer().formatTSMetaV1(updated_meta));
          } else if (exists) {
            final Deferred<TSMeta> process_meta =
                meta.syncToStorage(tsdb, method == HttpMethod.PUT)
                    .addCallbackDeferring(new SyncCB());
            final TSMeta updated_meta = process_meta.joinUninterruptibly();
            tsdb.indexTSMeta(updated_meta);
            query.sendReply(query.serializer().formatTSMetaV1(updated_meta));
          } else {
            throw new BadRequestException(
                "Could not find TSMeta, specify \"create=true\" to create a new one.");
          }
        } catch (IllegalStateException e) {
          query.sendStatusOnly(HttpResponseStatus.NOT_MODIFIED);
        } catch (IllegalArgumentException e) {
          throw new BadRequestException(e);
        } catch (BadRequestException e) {
          throw e;
        } catch (NoSuchUniqueName e) {
          // this would only happen if someone deleted a UID but left the
          // the timeseries meta data
          throw new BadRequestException(
              HttpResponseStatus.NOT_FOUND, "Unable to find one or more UIDs", e);
        } catch (Exception e) {
          throw new RuntimeException(e);
        }
      } else {
        try {
          final Deferred<TSMeta> process_meta =
              meta.syncToStorage(tsdb, method == HttpMethod.PUT).addCallbackDeferring(new SyncCB());
          final TSMeta updated_meta = process_meta.joinUninterruptibly();
          tsdb.indexTSMeta(updated_meta);
          query.sendReply(query.serializer().formatTSMetaV1(updated_meta));
        } catch (IllegalStateException e) {
          query.sendStatusOnly(HttpResponseStatus.NOT_MODIFIED);
        } catch (IllegalArgumentException e) {
          throw new BadRequestException(e);
        } catch (NoSuchUniqueName e) {
          // this would only happen if someone deleted a UID but left the
          // the timeseries meta data
          throw new BadRequestException(
              HttpResponseStatus.NOT_FOUND, "Unable to find one or more UIDs", e);
        } catch (Exception e) {
          throw new RuntimeException(e);
        }
      }
      // DELETE
    } else if (method == HttpMethod.DELETE) {

      final TSMeta meta;
      if (query.hasContent()) {
        meta = query.serializer().parseTSMetaV1();
      } else {
        meta = this.parseTSMetaQS(query);
      }
      try {
        meta.delete(tsdb);
        tsdb.deleteTSMeta(meta.getTSUID());
      } catch (IllegalArgumentException e) {
        throw new BadRequestException("Unable to delete TSMeta information", e);
      }
      query.sendStatusOnly(HttpResponseStatus.NO_CONTENT);
    } else {
      throw new BadRequestException(
          HttpResponseStatus.METHOD_NOT_ALLOWED,
          "Method not allowed",
          "The HTTP method [" + method.getName() + "] is not permitted for this endpoint");
    }
  }
Example #10
0
  /**
   * Handles CRUD calls to individual UIDMeta data entries
   *
   * @param tsdb The TSDB from the RPC router
   * @param query The query for this request
   */
  private void handleUIDMeta(final TSDB tsdb, final HttpQuery query) {

    final HttpMethod method = query.getAPIMethod();
    // GET
    if (method == HttpMethod.GET) {

      final String uid = query.getRequiredQueryStringParam("uid");
      final UniqueIdType type =
          UniqueId.stringToUniqueIdType(query.getRequiredQueryStringParam("type"));
      try {
        final UIDMeta meta = UIDMeta.getUIDMeta(tsdb, type, uid).joinUninterruptibly();
        query.sendReply(query.serializer().formatUidMetaV1(meta));
      } catch (NoSuchUniqueId e) {
        throw new BadRequestException(
            HttpResponseStatus.NOT_FOUND, "Could not find the requested UID", e);
      } catch (Exception e) {
        throw new RuntimeException(e);
      }
      // POST
    } else if (method == HttpMethod.POST || method == HttpMethod.PUT) {

      final UIDMeta meta;
      if (query.hasContent()) {
        meta = query.serializer().parseUidMetaV1();
      } else {
        meta = this.parseUIDMetaQS(query);
      }

      /**
       * Storage callback used to determine if the storage call was successful or not. Also returns
       * the updated object from storage.
       */
      class SyncCB implements Callback<Deferred<UIDMeta>, Boolean> {

        @Override
        public Deferred<UIDMeta> call(Boolean success) throws Exception {
          if (!success) {
            throw new BadRequestException(
                HttpResponseStatus.INTERNAL_SERVER_ERROR,
                "Failed to save the UIDMeta to storage",
                "This may be caused by another process modifying storage data");
          }

          return UIDMeta.getUIDMeta(tsdb, meta.getType(), meta.getUID());
        }
      }

      try {
        final Deferred<UIDMeta> process_meta =
            meta.syncToStorage(tsdb, method == HttpMethod.PUT).addCallbackDeferring(new SyncCB());
        final UIDMeta updated_meta = process_meta.joinUninterruptibly();
        tsdb.indexUIDMeta(updated_meta);
        query.sendReply(query.serializer().formatUidMetaV1(updated_meta));
      } catch (IllegalStateException e) {
        query.sendStatusOnly(HttpResponseStatus.NOT_MODIFIED);
      } catch (IllegalArgumentException e) {
        throw new BadRequestException(e);
      } catch (NoSuchUniqueId e) {
        throw new BadRequestException(
            HttpResponseStatus.NOT_FOUND, "Could not find the requested UID", e);
      } catch (Exception e) {
        throw new RuntimeException(e);
      }
      // DELETE
    } else if (method == HttpMethod.DELETE) {

      final UIDMeta meta;
      if (query.hasContent()) {
        meta = query.serializer().parseUidMetaV1();
      } else {
        meta = this.parseUIDMetaQS(query);
      }
      try {
        meta.delete(tsdb).joinUninterruptibly();
        tsdb.deleteUIDMeta(meta);
      } catch (IllegalArgumentException e) {
        throw new BadRequestException("Unable to delete UIDMeta information", e);
      } catch (NoSuchUniqueId e) {
        throw new BadRequestException(
            HttpResponseStatus.NOT_FOUND, "Could not find the requested UID", e);
      } catch (Exception e) {
        throw new RuntimeException(e);
      }
      query.sendStatusOnly(HttpResponseStatus.NO_CONTENT);

    } else {
      throw new BadRequestException(
          HttpResponseStatus.METHOD_NOT_ALLOWED,
          "Method not allowed",
          "The HTTP method [" + method.getName() + "] is not permitted for this endpoint");
    }
  }