/**
* Attempts to retrieve the specified tree rule from storage.
*
* @param tsdb The TSDB to use for storage access
* @param tree_id ID of the tree the rule belongs to
* @param level Level where the rule resides
* @param order Order where the rule resides
* @return A TreeRule object if found, null if it does not exist
* @throws HBaseException if there was an issue
* @throws IllegalArgumentException if the one of the required parameters was missing
* @throws JSONException if the object could not be serialized
*/
public static Deferred<TreeRule> fetchRule(
final TSDB tsdb, final int tree_id, final int level, final int order) {
if (tree_id < 1 || tree_id > 65535) {
throw new IllegalArgumentException("Invalid Tree ID");
}
if (level < 0) {
throw new IllegalArgumentException("Invalid rule level");
}
if (order < 0) {
throw new IllegalArgumentException("Invalid rule order");
}
// fetch the whole row
final GetRequest get = new GetRequest(tsdb.treeTable(), Tree.idToBytes(tree_id));
get.family(Tree.TREE_FAMILY());
get.qualifier(getQualifier(level, order));
/** Called after fetching to parse the results */
final class FetchCB implements Callback<Deferred<TreeRule>, ArrayList<KeyValue>> {
@Override
public Deferred<TreeRule> call(final ArrayList<KeyValue> row) {
if (row == null || row.isEmpty()) {
return Deferred.fromResult(null);
}
return Deferred.fromResult(parseFromStorage(row.get(0)));
}
}
return tsdb.getClient().get(get).addCallbackDeferring(new FetchCB());
}
public static void main(String[] args) throws Exception {
ArgP argp = new ArgP();
CliOptions.addCommon(argp);
argp.addOption("--fix", "Fix errors as they're found.");
args = CliOptions.parse(argp, args);
if (args == null) {
usage(argp, "Invalid usage.", 1);
} else if (args.length < 3) {
usage(argp, "Not enough arguments.", 2);
}
final HBaseClient client = CliOptions.clientFromOptions(argp);
final byte[] table = argp.get("--table", "tsdb").getBytes();
final TSDB tsdb =
new TSDB(client, argp.get("--table", "tsdb"), argp.get("--uidtable", "tsdb-uid"));
final boolean fix = argp.has("--fix");
argp = null;
int errors = 42;
try {
errors = fsck(tsdb, client, table, fix, args);
} finally {
tsdb.shutdown().joinUninterruptibly();
}
System.exit(errors == 0 ? 0 : 1);
}
@Override
public void initialize(TSDB tsdb) {
if (tsdb == null) {
throw new IllegalArgumentException("The TSDB object was null");
}
// a dummy config to check for throwing exceptions
if (!tsdb.getConfig().hasProperty("tsd.core.storage_exception_handler.DummySEHPlugin.hosts")) {
throw new IllegalArgumentException("Missing hosts config");
}
}
/**
* Attempts to delete all rules belonging to the given tree.
*
* @param tsdb The TSDB to use for storage access
* @param tree_id ID of the tree the rules belongs to
* @return A deferred to wait on for completion. The value has no meaning and may be null.
* @throws HBaseException if there was an issue
* @throws IllegalArgumentException if the one of the required parameters was missing
*/
public static Deferred<Object> deleteAllRules(final TSDB tsdb, final int tree_id) {
if (tree_id < 1 || tree_id > 65535) {
throw new IllegalArgumentException("Invalid Tree ID");
}
// fetch the whole row
final GetRequest get = new GetRequest(tsdb.treeTable(), Tree.idToBytes(tree_id));
get.family(Tree.TREE_FAMILY());
/**
* Called after fetching the requested row. If the row is empty, we just return, otherwise we
* compile a list of qualifiers to delete and submit a single delete request to storage.
*/
final class GetCB implements Callback<Deferred<Object>, ArrayList<KeyValue>> {
@Override
public Deferred<Object> call(final ArrayList<KeyValue> row) throws Exception {
if (row == null || row.isEmpty()) {
return Deferred.fromResult(null);
}
final ArrayList<byte[]> qualifiers = new ArrayList<byte[]>(row.size());
for (KeyValue column : row) {
if (column.qualifier().length > RULE_PREFIX.length
&& Bytes.memcmp(RULE_PREFIX, column.qualifier(), 0, RULE_PREFIX.length) == 0) {
qualifiers.add(column.qualifier());
}
}
final DeleteRequest delete =
new DeleteRequest(
tsdb.treeTable(),
Tree.idToBytes(tree_id),
Tree.TREE_FAMILY(),
qualifiers.toArray(new byte[qualifiers.size()][]));
return tsdb.getClient().delete(delete);
}
}
return tsdb.getClient().get(get).addCallbackDeferring(new GetCB());
}
/**
* Attempts to delete the specified rule from storage
*
* @param tsdb The TSDB to use for storage access
* @param tree_id ID of the tree the rule belongs to
* @param level Level where the rule resides
* @param order Order where the rule resides
* @return A deferred without meaning. The response may be null and should only be used to track
* completion.
* @throws HBaseException if there was an issue
* @throws IllegalArgumentException if the one of the required parameters was missing
*/
public static Deferred<Object> deleteRule(
final TSDB tsdb, final int tree_id, final int level, final int order) {
if (tree_id < 1 || tree_id > 65535) {
throw new IllegalArgumentException("Invalid Tree ID");
}
if (level < 0) {
throw new IllegalArgumentException("Invalid rule level");
}
if (order < 0) {
throw new IllegalArgumentException("Invalid rule order");
}
final DeleteRequest delete =
new DeleteRequest(
tsdb.treeTable(),
Tree.idToBytes(tree_id),
Tree.TREE_FAMILY(),
getQualifier(level, order));
return tsdb.getClient().delete(delete);
}
/**
* Parses a query string "m=metric{tagk1=tagv1,...}" type query and returns a tsuid.
*
* @param data_query The query we're building
* @throws BadRequestException if we are unable to parse the query or it is missing components
* @todo - make this asynchronous
*/
private String getTSUIDForMetric(final String query_string, TSDB tsdb) {
if (query_string == null || query_string.isEmpty()) {
throw new BadRequestException("The query string was empty");
}
// m is of the following forms:
// metric[{tag=value,...}]
// where the parts in square brackets `[' .. `]' are optional.
final HashMap<String, String> tags = new HashMap<String, String>();
String metric = null;
try {
metric = Tags.parseWithMetric(query_string, tags);
} catch (IllegalArgumentException e) {
throw new BadRequestException(e);
}
// sort the UIDs on tagk values
final ByteMap<byte[]> tag_uids = new ByteMap<byte[]>();
for (final Entry<String, String> pair : tags.entrySet()) {
tag_uids.put(
tsdb.getUID(UniqueIdType.TAGK, pair.getKey()),
tsdb.getUID(UniqueIdType.TAGV, pair.getValue()));
}
// Byte Buffer to generate TSUID, pre allocated to the size of the TSUID
final ByteArrayOutputStream buf =
new ByteArrayOutputStream(
TSDB.metrics_width() + tag_uids.size() * (TSDB.tagk_width() + TSDB.tagv_width()));
try {
buf.write(tsdb.getUID(UniqueIdType.METRIC, metric));
for (final Entry<byte[], byte[]> uids : tag_uids.entrySet()) {
buf.write(uids.getKey());
buf.write(uids.getValue());
}
} catch (IOException e) {
throw new BadRequestException(e);
}
final String tsuid = UniqueId.uidToString(buf.toByteArray());
return tsuid;
}
/**
* Get the next random UID. Right now it uses 3 bytes as the metric with is 3. If it is 3 then it
* can return only upto the max value a 3 byte integer can return, which is 2^31-1. In that case,
* even though it is long, its range will be between 0 and 2^31-1
*
* @return random UID
*/
public static long getRandomUID() {
return getRandomUID(TSDB.metrics_width());
}
/**
* Assigns UIDs to the given metric, tagk or tagv names if applicable
*
* <p>This handler supports GET and POST whereby the GET command can parse query strings with the
* {@code type} as their parameter and a comma separated list of values to assign UIDs to.
*
* <p>Multiple types and names can be provided in one call. Each name will be processed
* independently and if there's an error (such as an invalid name or it is already assigned) the
* error will be stored in a separate error map and other UIDs will be processed.
*
* @param tsdb The TSDB from the RPC router
* @param query The query for this request
*/
private void handleAssign(final TSDB tsdb, final HttpQuery query) {
// only accept GET And POST
if (query.method() != HttpMethod.GET && query.method() != HttpMethod.POST) {
throw new BadRequestException(
HttpResponseStatus.METHOD_NOT_ALLOWED,
"Method not allowed",
"The HTTP method [" + query.method().getName() + "] is not permitted for this endpoint");
}
final HashMap<String, List<String>> source;
if (query.method() == HttpMethod.POST) {
source = query.serializer().parseUidAssignV1();
} else {
source = new HashMap<String, List<String>>(3);
// cut down on some repetitive code, split the query string values by
// comma and add them to the source hash
String[] types = {"metric", "tagk", "tagv"};
for (int i = 0; i < types.length; i++) {
final String values = query.getQueryStringParam(types[i]);
if (values != null && !values.isEmpty()) {
final String[] metrics = values.split(",");
if (metrics != null && metrics.length > 0) {
source.put(types[i], Arrays.asList(metrics));
}
}
}
}
if (source.size() < 1) {
throw new BadRequestException("Missing values to assign UIDs");
}
final Map<String, TreeMap<String, String>> response =
new HashMap<String, TreeMap<String, String>>();
int error_count = 0;
for (Map.Entry<String, List<String>> entry : source.entrySet()) {
final TreeMap<String, String> results = new TreeMap<String, String>();
final TreeMap<String, String> errors = new TreeMap<String, String>();
for (String name : entry.getValue()) {
try {
final byte[] uid = tsdb.assignUid(entry.getKey(), name);
results.put(name, UniqueId.uidToString(uid));
} catch (IllegalArgumentException e) {
errors.put(name, e.getMessage());
error_count++;
}
}
response.put(entry.getKey(), results);
if (errors.size() > 0) {
response.put(entry.getKey() + "_errors", errors);
}
}
if (error_count < 1) {
query.sendReply(query.serializer().formatUidAssignV1(response));
} else {
query.sendReply(
HttpResponseStatus.BAD_REQUEST, query.serializer().formatUidAssignV1(response));
}
}
/**
* Handles CRUD calls to individual TSMeta data entries
*
* @param tsdb The TSDB from the RPC router
* @param query The query for this request
*/
private void handleTSMeta(final TSDB tsdb, final HttpQuery query) {
final HttpMethod method = query.getAPIMethod();
// GET
if (method == HttpMethod.GET) {
String tsuid = null;
if (query.hasQueryStringParam("tsuid")) {
tsuid = query.getQueryStringParam("tsuid");
try {
final TSMeta meta = TSMeta.getTSMeta(tsdb, tsuid).joinUninterruptibly();
if (meta != null) {
query.sendReply(query.serializer().formatTSMetaV1(meta));
} else {
throw new BadRequestException(
HttpResponseStatus.NOT_FOUND, "Could not find Timeseries meta data");
}
} catch (NoSuchUniqueName e) {
// this would only happen if someone deleted a UID but left the
// the timeseries meta data
throw new BadRequestException(
HttpResponseStatus.NOT_FOUND, "Unable to find one of the UIDs", e);
} catch (BadRequestException e) {
throw e;
} catch (Exception e) {
throw new RuntimeException(e);
}
} else {
String mquery = query.getRequiredQueryStringParam("m");
// m is of the following forms:
// metric[{tag=value,...}]
// where the parts in square brackets `[' .. `]' are optional.
final HashMap<String, String> tags = new HashMap<String, String>();
String metric = null;
try {
metric = Tags.parseWithMetric(mquery, tags);
} catch (IllegalArgumentException e) {
throw new BadRequestException(e);
}
final TSUIDQuery tsuid_query = new TSUIDQuery(tsdb, metric, tags);
try {
final List<TSMeta> tsmetas = tsuid_query.getTSMetas().joinUninterruptibly();
query.sendReply(query.serializer().formatTSMetaListV1(tsmetas));
} catch (NoSuchUniqueName e) {
throw new BadRequestException(
HttpResponseStatus.NOT_FOUND, "Unable to find one of the UIDs", e);
} catch (BadRequestException e) {
throw e;
} catch (RuntimeException e) {
throw new BadRequestException(e);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
// POST / PUT
} else if (method == HttpMethod.POST || method == HttpMethod.PUT) {
final TSMeta meta;
if (query.hasContent()) {
meta = query.serializer().parseTSMetaV1();
} else {
meta = this.parseTSMetaQS(query);
}
/**
* Storage callback used to determine if the storage call was successful or not. Also returns
* the updated object from storage.
*/
class SyncCB implements Callback<Deferred<TSMeta>, Boolean> {
@Override
public Deferred<TSMeta> call(Boolean success) throws Exception {
if (!success) {
throw new BadRequestException(
HttpResponseStatus.INTERNAL_SERVER_ERROR,
"Failed to save the TSMeta to storage",
"This may be caused by another process modifying storage data");
}
return TSMeta.getTSMeta(tsdb, meta.getTSUID());
}
}
if (meta.getTSUID() == null || meta.getTSUID().isEmpty()) {
// we got a JSON object without TSUID. Try to find a timeseries spec of
// the form "m": "metric{tagk=tagv,...}"
final String metric = query.getRequiredQueryStringParam("m");
final boolean create =
query.getQueryStringParam("create") != null
&& query.getQueryStringParam("create").equals("true");
final String tsuid = getTSUIDForMetric(metric, tsdb);
class WriteCounterIfNotPresentCB implements Callback<Boolean, Boolean> {
@Override
public Boolean call(Boolean exists) throws Exception {
if (!exists && create) {
final PutRequest put =
new PutRequest(
tsdb.metaTable(),
UniqueId.stringToUid(tsuid),
TSMeta.FAMILY(),
TSMeta.COUNTER_QUALIFIER(),
Bytes.fromLong(0));
tsdb.getClient().put(put);
}
return exists;
}
}
try {
// Check whether we have a TSMeta stored already
final boolean exists = TSMeta.metaExistsInStorage(tsdb, tsuid).joinUninterruptibly();
// set TSUID
meta.setTSUID(tsuid);
if (!exists && create) {
// Write 0 to counter column if not present
TSMeta.counterExistsInStorage(tsdb, UniqueId.stringToUid(tsuid))
.addCallback(new WriteCounterIfNotPresentCB())
.joinUninterruptibly();
// set TSUID
final Deferred<TSMeta> process_meta =
meta.storeNew(tsdb).addCallbackDeferring(new SyncCB());
final TSMeta updated_meta = process_meta.joinUninterruptibly();
tsdb.indexTSMeta(updated_meta);
tsdb.processTSMetaThroughTrees(updated_meta);
query.sendReply(query.serializer().formatTSMetaV1(updated_meta));
} else if (exists) {
final Deferred<TSMeta> process_meta =
meta.syncToStorage(tsdb, method == HttpMethod.PUT)
.addCallbackDeferring(new SyncCB());
final TSMeta updated_meta = process_meta.joinUninterruptibly();
tsdb.indexTSMeta(updated_meta);
query.sendReply(query.serializer().formatTSMetaV1(updated_meta));
} else {
throw new BadRequestException(
"Could not find TSMeta, specify \"create=true\" to create a new one.");
}
} catch (IllegalStateException e) {
query.sendStatusOnly(HttpResponseStatus.NOT_MODIFIED);
} catch (IllegalArgumentException e) {
throw new BadRequestException(e);
} catch (BadRequestException e) {
throw e;
} catch (NoSuchUniqueName e) {
// this would only happen if someone deleted a UID but left the
// the timeseries meta data
throw new BadRequestException(
HttpResponseStatus.NOT_FOUND, "Unable to find one or more UIDs", e);
} catch (Exception e) {
throw new RuntimeException(e);
}
} else {
try {
final Deferred<TSMeta> process_meta =
meta.syncToStorage(tsdb, method == HttpMethod.PUT).addCallbackDeferring(new SyncCB());
final TSMeta updated_meta = process_meta.joinUninterruptibly();
tsdb.indexTSMeta(updated_meta);
query.sendReply(query.serializer().formatTSMetaV1(updated_meta));
} catch (IllegalStateException e) {
query.sendStatusOnly(HttpResponseStatus.NOT_MODIFIED);
} catch (IllegalArgumentException e) {
throw new BadRequestException(e);
} catch (NoSuchUniqueName e) {
// this would only happen if someone deleted a UID but left the
// the timeseries meta data
throw new BadRequestException(
HttpResponseStatus.NOT_FOUND, "Unable to find one or more UIDs", e);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
// DELETE
} else if (method == HttpMethod.DELETE) {
final TSMeta meta;
if (query.hasContent()) {
meta = query.serializer().parseTSMetaV1();
} else {
meta = this.parseTSMetaQS(query);
}
try {
meta.delete(tsdb);
tsdb.deleteTSMeta(meta.getTSUID());
} catch (IllegalArgumentException e) {
throw new BadRequestException("Unable to delete TSMeta information", e);
}
query.sendStatusOnly(HttpResponseStatus.NO_CONTENT);
} else {
throw new BadRequestException(
HttpResponseStatus.METHOD_NOT_ALLOWED,
"Method not allowed",
"The HTTP method [" + method.getName() + "] is not permitted for this endpoint");
}
}
/**
* Handles CRUD calls to individual UIDMeta data entries
*
* @param tsdb The TSDB from the RPC router
* @param query The query for this request
*/
private void handleUIDMeta(final TSDB tsdb, final HttpQuery query) {
final HttpMethod method = query.getAPIMethod();
// GET
if (method == HttpMethod.GET) {
final String uid = query.getRequiredQueryStringParam("uid");
final UniqueIdType type =
UniqueId.stringToUniqueIdType(query.getRequiredQueryStringParam("type"));
try {
final UIDMeta meta = UIDMeta.getUIDMeta(tsdb, type, uid).joinUninterruptibly();
query.sendReply(query.serializer().formatUidMetaV1(meta));
} catch (NoSuchUniqueId e) {
throw new BadRequestException(
HttpResponseStatus.NOT_FOUND, "Could not find the requested UID", e);
} catch (Exception e) {
throw new RuntimeException(e);
}
// POST
} else if (method == HttpMethod.POST || method == HttpMethod.PUT) {
final UIDMeta meta;
if (query.hasContent()) {
meta = query.serializer().parseUidMetaV1();
} else {
meta = this.parseUIDMetaQS(query);
}
/**
* Storage callback used to determine if the storage call was successful or not. Also returns
* the updated object from storage.
*/
class SyncCB implements Callback<Deferred<UIDMeta>, Boolean> {
@Override
public Deferred<UIDMeta> call(Boolean success) throws Exception {
if (!success) {
throw new BadRequestException(
HttpResponseStatus.INTERNAL_SERVER_ERROR,
"Failed to save the UIDMeta to storage",
"This may be caused by another process modifying storage data");
}
return UIDMeta.getUIDMeta(tsdb, meta.getType(), meta.getUID());
}
}
try {
final Deferred<UIDMeta> process_meta =
meta.syncToStorage(tsdb, method == HttpMethod.PUT).addCallbackDeferring(new SyncCB());
final UIDMeta updated_meta = process_meta.joinUninterruptibly();
tsdb.indexUIDMeta(updated_meta);
query.sendReply(query.serializer().formatUidMetaV1(updated_meta));
} catch (IllegalStateException e) {
query.sendStatusOnly(HttpResponseStatus.NOT_MODIFIED);
} catch (IllegalArgumentException e) {
throw new BadRequestException(e);
} catch (NoSuchUniqueId e) {
throw new BadRequestException(
HttpResponseStatus.NOT_FOUND, "Could not find the requested UID", e);
} catch (Exception e) {
throw new RuntimeException(e);
}
// DELETE
} else if (method == HttpMethod.DELETE) {
final UIDMeta meta;
if (query.hasContent()) {
meta = query.serializer().parseUidMetaV1();
} else {
meta = this.parseUIDMetaQS(query);
}
try {
meta.delete(tsdb).joinUninterruptibly();
tsdb.deleteUIDMeta(meta);
} catch (IllegalArgumentException e) {
throw new BadRequestException("Unable to delete UIDMeta information", e);
} catch (NoSuchUniqueId e) {
throw new BadRequestException(
HttpResponseStatus.NOT_FOUND, "Could not find the requested UID", e);
} catch (Exception e) {
throw new RuntimeException(e);
}
query.sendStatusOnly(HttpResponseStatus.NO_CONTENT);
} else {
throw new BadRequestException(
HttpResponseStatus.METHOD_NOT_ALLOWED,
"Method not allowed",
"The HTTP method [" + method.getName() + "] is not permitted for this endpoint");
}
}