public VAdminProto.GetMetadataResponse handleGetMetadata(VAdminProto.GetMetadataRequest request) {
VAdminProto.GetMetadataResponse.Builder response = VAdminProto.GetMetadataResponse.newBuilder();
try {
ByteArray key = ProtoUtils.decodeBytes(request.getKey());
String keyString = ByteUtils.getString(key.get(), "UTF-8");
if (MetadataStore.METADATA_KEYS.contains(keyString)) {
List<Versioned<byte[]>> versionedList = metadataStore.get(key);
int size = (versionedList.size() > 0) ? 1 : 0;
if (size > 0) {
Versioned<byte[]> versioned = versionedList.get(0);
response.setVersion(ProtoUtils.encodeVersioned(versioned));
}
} else {
throw new VoldemortException(
"Metadata Key passed " + keyString + " is not handled yet ...");
}
} catch (VoldemortException e) {
response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
logger.error("handleGetMetadata failed for request(" + request.toString() + ")", e);
}
return response.build();
}
public VAdminProto.AsyncOperationStatusResponse handleRebalanceNode(
VAdminProto.InitiateRebalanceNodeRequest request) {
VAdminProto.AsyncOperationStatusResponse.Builder response =
VAdminProto.AsyncOperationStatusResponse.newBuilder();
try {
if (!voldemortConfig.isEnableRebalanceService())
throw new VoldemortException(
"Rebalance service is not enabled for node:" + metadataStore.getNodeId());
RebalancePartitionsInfo rebalanceStealInfo =
new RebalancePartitionsInfo(
request.getStealerId(),
request.getDonorId(),
request.getPartitionsList(),
request.getDeletePartitionsList(),
request.getUnbalancedStoreList(),
request.getAttempt());
int requestId = rebalancer.rebalanceLocalNode(rebalanceStealInfo);
response
.setRequestId(requestId)
.setDescription(rebalanceStealInfo.toString())
.setStatus("started")
.setComplete(false);
} catch (VoldemortException e) {
response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
logger.error("handleRebalanceNode failed for request(" + request.toString() + ")", e);
}
return response.build();
}
private Message handleGetVersion(GetRequest request, Store<ByteArray, byte[]> store) {
VProto.GetVersionResponse.Builder response = VProto.GetVersionResponse.newBuilder();
try {
List<Version> versions = store.getVersions(ProtoUtils.decodeBytes(request.getKey()));
for (Version version : versions) response.addVersions(ProtoUtils.encodeClock(version));
} catch (VoldemortException e) {
response.setError(ProtoUtils.encodeError(getErrorMapper(), e));
}
return response.build();
}
private VProto.GetResponse handleGet(VProto.GetRequest request, Store<ByteArray, byte[]> store) {
VProto.GetResponse.Builder response = VProto.GetResponse.newBuilder();
try {
List<Versioned<byte[]>> values = store.get(ProtoUtils.decodeBytes(request.getKey()));
for (Versioned<byte[]> versioned : values)
response.addVersioned(ProtoUtils.encodeVersioned(versioned));
} catch (VoldemortException e) {
response.setError(ProtoUtils.encodeError(getErrorMapper(), e));
}
return response.build();
}
private VProto.PutResponse handlePut(VProto.PutRequest request, Store<ByteArray, byte[]> store) {
VProto.PutResponse.Builder response = VProto.PutResponse.newBuilder();
try {
ByteArray key = ProtoUtils.decodeBytes(request.getKey());
Versioned<byte[]> value = ProtoUtils.decodeVersioned(request.getVersioned());
store.put(key, value);
} catch (VoldemortException e) {
response.setError(ProtoUtils.encodeError(getErrorMapper(), e));
}
return response.build();
}
public VAdminProto.AsyncOperationStopResponse handleAsyncOperationStop(
VAdminProto.AsyncOperationStopRequest request) {
VAdminProto.AsyncOperationStopResponse.Builder response =
VAdminProto.AsyncOperationStopResponse.newBuilder();
int requestId = request.getRequestId();
try {
asyncService.stopOperation(requestId);
} catch (VoldemortException e) {
response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
logger.error("handleAsyncOperationStop failed for request(" + request.toString() + ")", e);
}
return response.build();
}
public VAdminProto.AsyncOperationListResponse handleAsyncOperationList(
VAdminProto.AsyncOperationListRequest request) {
VAdminProto.AsyncOperationListResponse.Builder response =
VAdminProto.AsyncOperationListResponse.newBuilder();
boolean showComplete = request.hasShowComplete() && request.getShowComplete();
try {
response.addAllRequestIds(asyncService.getAsyncOperationList(showComplete));
} catch (VoldemortException e) {
response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
logger.error("handleAsyncOperationList failed for request(" + request.toString() + ")", e);
}
return response.build();
}
private VProto.DeleteResponse handleDelete(
VProto.DeleteRequest request, Store<ByteArray, byte[]> store) {
VProto.DeleteResponse.Builder response = VProto.DeleteResponse.newBuilder();
try {
boolean success =
store.delete(
ProtoUtils.decodeBytes(request.getKey()),
ProtoUtils.decodeClock(request.getVersion()));
response.setSuccess(success);
} catch (VoldemortException e) {
response.setSuccess(false);
response.setError(ProtoUtils.encodeError(getErrorMapper(), e));
}
return response.build();
}
public VAdminProto.TruncateEntriesResponse handleTruncateEntries(
VAdminProto.TruncateEntriesRequest request) {
VAdminProto.TruncateEntriesResponse.Builder response =
VAdminProto.TruncateEntriesResponse.newBuilder();
try {
String storeName = request.getStore();
StorageEngine<ByteArray, byte[]> storageEngine = getStorageEngine(storeRepository, storeName);
storageEngine.truncate();
} catch (VoldemortException e) {
response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
logger.error("handleTruncateEntries failed for request(" + request.toString() + ")", e);
}
return response.build();
}
private VProto.GetAllResponse handleGetAll(
VProto.GetAllRequest request, Store<ByteArray, byte[]> store) {
VProto.GetAllResponse.Builder response = VProto.GetAllResponse.newBuilder();
try {
List<ByteArray> keys = new ArrayList<ByteArray>(request.getKeysCount());
for (ByteString string : request.getKeysList()) keys.add(ProtoUtils.decodeBytes(string));
Map<ByteArray, List<Versioned<byte[]>>> values = store.getAll(keys);
for (Map.Entry<ByteArray, List<Versioned<byte[]>>> entry : values.entrySet()) {
VProto.KeyedVersions.Builder keyedVersion =
VProto.KeyedVersions.newBuilder().setKey(ProtoUtils.encodeBytes(entry.getKey()));
for (Versioned<byte[]> version : entry.getValue())
keyedVersion.addVersions(ProtoUtils.encodeVersioned(version));
response.addValues(keyedVersion);
}
} catch (VoldemortException e) {
response.setError(ProtoUtils.encodeError(getErrorMapper(), e));
}
return response.build();
}
public VAdminProto.UpdateMetadataResponse handleUpdateMetadata(
VAdminProto.UpdateMetadataRequest request) {
VAdminProto.UpdateMetadataResponse.Builder response =
VAdminProto.UpdateMetadataResponse.newBuilder();
try {
ByteArray key = ProtoUtils.decodeBytes(request.getKey());
String keyString = ByteUtils.getString(key.get(), "UTF-8");
if (MetadataStore.METADATA_KEYS.contains(keyString)) {
Versioned<byte[]> versionedValue = ProtoUtils.decodeVersioned(request.getVersioned());
metadataStore.put(new ByteArray(ByteUtils.getBytes(keyString, "UTF-8")), versionedValue);
}
} catch (VoldemortException e) {
response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
logger.error("handleUpdateMetadata failed for request(" + request.toString() + ")", e);
}
return response.build();
}
public VAdminProto.DeletePartitionEntriesResponse handleDeletePartitionEntries(
VAdminProto.DeletePartitionEntriesRequest request) {
VAdminProto.DeletePartitionEntriesResponse.Builder response =
VAdminProto.DeletePartitionEntriesResponse.newBuilder();
ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> iterator = null;
try {
String storeName = request.getStore();
List<Integer> partitions = request.getPartitionsList();
StorageEngine<ByteArray, byte[]> storageEngine = getStorageEngine(storeRepository, storeName);
VoldemortFilter filter =
(request.hasFilter())
? getFilterFromRequest(request.getFilter(), voldemortConfig, networkClassLoader)
: new DefaultVoldemortFilter();
RoutingStrategy routingStrategy = metadataStore.getRoutingStrategy(storageEngine.getName());
EventThrottler throttler = new EventThrottler(voldemortConfig.getStreamMaxReadBytesPerSec());
iterator = storageEngine.entries();
int deleteSuccess = 0;
while (iterator.hasNext()) {
Pair<ByteArray, Versioned<byte[]>> entry = iterator.next();
ByteArray key = entry.getFirst();
Versioned<byte[]> value = entry.getSecond();
throttler.maybeThrottle(key.length() + valueSize(value));
if (checkKeyBelongsToDeletePartition(key.get(), partitions, routingStrategy)
&& filter.accept(key, value)) {
if (storageEngine.delete(key, value.getVersion())) deleteSuccess++;
}
}
response.setCount(deleteSuccess);
} catch (VoldemortException e) {
response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
logger.error(
"handleDeletePartitionEntries failed for request(" + request.toString() + ")", e);
} finally {
if (null != iterator) iterator.close();
}
return response.build();
}
public VAdminProto.AsyncOperationStatusResponse handleAsyncStatus(
VAdminProto.AsyncOperationStatusRequest request) {
VAdminProto.AsyncOperationStatusResponse.Builder response =
VAdminProto.AsyncOperationStatusResponse.newBuilder();
try {
int requestId = request.getRequestId();
AsyncOperationStatus operationStatus = asyncService.getOperationStatus(requestId);
boolean requestComplete = asyncService.isComplete(requestId);
response.setDescription(operationStatus.getDescription());
response.setComplete(requestComplete);
response.setStatus(operationStatus.getStatus());
response.setRequestId(requestId);
if (operationStatus.hasException())
throw new VoldemortException(operationStatus.getException());
} catch (VoldemortException e) {
response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
logger.error("handleAsyncStatus failed for request(" + request.toString().trim() + ")", e);
}
return response.build();
}
public VAdminProto.AddStoreResponse handleAddStore(VAdminProto.AddStoreRequest request) {
VAdminProto.AddStoreResponse.Builder response = VAdminProto.AddStoreResponse.newBuilder();
// don't try to add a store in the middle of rebalancing
if (metadataStore
.getServerState()
.equals(MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER)
|| metadataStore
.getServerState()
.equals(MetadataStore.VoldemortState.REBALANCING_CLUSTER)) {
response.setError(
ProtoUtils.encodeError(
errorCodeMapper, new VoldemortException("Rebalancing in progress")));
return response.build();
}
try {
// adding a store requires decoding the passed in store string
StoreDefinitionsMapper mapper = new StoreDefinitionsMapper();
StoreDefinition def = mapper.readStore(new StringReader(request.getStoreDefinition()));
synchronized (lock) {
// only allow a single store to be created at a time. We'll see concurrent errors when
// writing the
// stores.xml file out otherwise. (see ConfigurationStorageEngine.put for details)
if (!storeRepository.hasLocalStore(def.getName())) {
// open the store
storageService.openStore(def);
// update stores list in metadata store (this also has the
// effect of updating the stores.xml file)
List<StoreDefinition> currentStoreDefs;
List<Versioned<byte[]>> v = metadataStore.get(MetadataStore.STORES_KEY);
if (((v.size() > 0) ? 1 : 0) > 0) {
Versioned<byte[]> currentValue = v.get(0);
currentStoreDefs =
mapper.readStoreList(
new StringReader(ByteUtils.getString(currentValue.getValue(), "UTF-8")));
} else {
currentStoreDefs = Lists.newArrayList();
}
currentStoreDefs.add(def);
try {
metadataStore.put(MetadataStore.STORES_KEY, currentStoreDefs);
} catch (Exception e) {
throw new VoldemortException(e);
}
} else {
throw new StoreOperationFailureException(
String.format("Store '%s' already exists on this server", def.getName()));
}
}
} catch (VoldemortException e) {
response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
logger.error("handleAddStore failed for request(" + request.toString() + ")", e);
}
return response.build();
}
public VAdminProto.AsyncOperationStatusResponse handleFetchAndUpdate(
VAdminProto.InitiateFetchAndUpdateRequest request) {
final int nodeId = request.getNodeId();
final List<Integer> partitions = request.getPartitionsList();
final VoldemortFilter filter =
request.hasFilter()
? getFilterFromRequest(request.getFilter(), voldemortConfig, networkClassLoader)
: new DefaultVoldemortFilter();
final String storeName = request.getStore();
int requestId = asyncService.getUniqueRequestId();
VAdminProto.AsyncOperationStatusResponse.Builder response =
VAdminProto.AsyncOperationStatusResponse.newBuilder()
.setRequestId(requestId)
.setComplete(false)
.setDescription("Fetch and update")
.setStatus("started");
try {
asyncService.submitOperation(
requestId,
new AsyncOperation(requestId, "Fetch and Update") {
private final AtomicBoolean running = new AtomicBoolean(true);
@Override
public void stop() {
running.set(false);
}
@Override
public void operate() {
AdminClient adminClient =
RebalanceUtils.createTempAdminClient(
voldemortConfig, metadataStore.getCluster(), 4, 2);
try {
StorageEngine<ByteArray, byte[]> storageEngine =
getStorageEngine(storeRepository, storeName);
Iterator<Pair<ByteArray, Versioned<byte[]>>> entriesIterator =
adminClient.fetchEntries(nodeId, storeName, partitions, filter, false);
updateStatus("Initated fetchPartitionEntries");
EventThrottler throttler =
new EventThrottler(voldemortConfig.getStreamMaxWriteBytesPerSec());
for (long i = 0; running.get() && entriesIterator.hasNext(); i++) {
Pair<ByteArray, Versioned<byte[]>> entry = entriesIterator.next();
ByteArray key = entry.getFirst();
Versioned<byte[]> value = entry.getSecond();
try {
storageEngine.put(key, value);
} catch (ObsoleteVersionException e) {
// log and ignore
logger.debug("migratePartition threw ObsoleteVersionException, Ignoring.");
}
throttler.maybeThrottle(key.length() + valueSize(value));
if ((i % 1000) == 0) {
updateStatus(i + " entries processed");
}
}
} finally {
adminClient.stop();
}
}
});
} catch (VoldemortException e) {
response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
logger.error("handleFetchAndUpdate failed for request(" + request.toString() + ")", e);
}
return response.build();
}
