コード例 #1
0
  public VAdminProto.GetMetadataResponse handleGetMetadata(VAdminProto.GetMetadataRequest request) {
    VAdminProto.GetMetadataResponse.Builder response = VAdminProto.GetMetadataResponse.newBuilder();

    try {
      ByteArray key = ProtoUtils.decodeBytes(request.getKey());
      String keyString = ByteUtils.getString(key.get(), "UTF-8");
      if (MetadataStore.METADATA_KEYS.contains(keyString)) {
        List<Versioned<byte[]>> versionedList = metadataStore.get(key);
        int size = (versionedList.size() > 0) ? 1 : 0;

        if (size > 0) {
          Versioned<byte[]> versioned = versionedList.get(0);
          response.setVersion(ProtoUtils.encodeVersioned(versioned));
        }
      } else {
        throw new VoldemortException(
            "Metadata Key passed " + keyString + " is not handled yet ...");
      }
    } catch (VoldemortException e) {
      response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
      logger.error("handleGetMetadata failed for request(" + request.toString() + ")", e);
    }

    return response.build();
  }
コード例 #2
0
  public VAdminProto.AsyncOperationStatusResponse handleRebalanceNode(
      VAdminProto.InitiateRebalanceNodeRequest request) {
    VAdminProto.AsyncOperationStatusResponse.Builder response =
        VAdminProto.AsyncOperationStatusResponse.newBuilder();
    try {
      if (!voldemortConfig.isEnableRebalanceService())
        throw new VoldemortException(
            "Rebalance service is not enabled for node:" + metadataStore.getNodeId());

      RebalancePartitionsInfo rebalanceStealInfo =
          new RebalancePartitionsInfo(
              request.getStealerId(),
              request.getDonorId(),
              request.getPartitionsList(),
              request.getDeletePartitionsList(),
              request.getUnbalancedStoreList(),
              request.getAttempt());

      int requestId = rebalancer.rebalanceLocalNode(rebalanceStealInfo);

      response
          .setRequestId(requestId)
          .setDescription(rebalanceStealInfo.toString())
          .setStatus("started")
          .setComplete(false);
    } catch (VoldemortException e) {
      response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
      logger.error("handleRebalanceNode failed for request(" + request.toString() + ")", e);
    }

    return response.build();
  }
コード例 #3
0
 private Message handleGetVersion(GetRequest request, Store<ByteArray, byte[]> store) {
   VProto.GetVersionResponse.Builder response = VProto.GetVersionResponse.newBuilder();
   try {
     List<Version> versions = store.getVersions(ProtoUtils.decodeBytes(request.getKey()));
     for (Version version : versions) response.addVersions(ProtoUtils.encodeClock(version));
   } catch (VoldemortException e) {
     response.setError(ProtoUtils.encodeError(getErrorMapper(), e));
   }
   return response.build();
 }
コード例 #4
0
 private VProto.GetResponse handleGet(VProto.GetRequest request, Store<ByteArray, byte[]> store) {
   VProto.GetResponse.Builder response = VProto.GetResponse.newBuilder();
   try {
     List<Versioned<byte[]>> values = store.get(ProtoUtils.decodeBytes(request.getKey()));
     for (Versioned<byte[]> versioned : values)
       response.addVersioned(ProtoUtils.encodeVersioned(versioned));
   } catch (VoldemortException e) {
     response.setError(ProtoUtils.encodeError(getErrorMapper(), e));
   }
   return response.build();
 }
コード例 #5
0
 private VProto.PutResponse handlePut(VProto.PutRequest request, Store<ByteArray, byte[]> store) {
   VProto.PutResponse.Builder response = VProto.PutResponse.newBuilder();
   try {
     ByteArray key = ProtoUtils.decodeBytes(request.getKey());
     Versioned<byte[]> value = ProtoUtils.decodeVersioned(request.getVersioned());
     store.put(key, value);
   } catch (VoldemortException e) {
     response.setError(ProtoUtils.encodeError(getErrorMapper(), e));
   }
   return response.build();
 }
コード例 #6
0
  public VAdminProto.AsyncOperationStopResponse handleAsyncOperationStop(
      VAdminProto.AsyncOperationStopRequest request) {
    VAdminProto.AsyncOperationStopResponse.Builder response =
        VAdminProto.AsyncOperationStopResponse.newBuilder();
    int requestId = request.getRequestId();
    try {
      asyncService.stopOperation(requestId);
    } catch (VoldemortException e) {
      response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
      logger.error("handleAsyncOperationStop failed for request(" + request.toString() + ")", e);
    }

    return response.build();
  }
コード例 #7
0
  public VAdminProto.AsyncOperationListResponse handleAsyncOperationList(
      VAdminProto.AsyncOperationListRequest request) {
    VAdminProto.AsyncOperationListResponse.Builder response =
        VAdminProto.AsyncOperationListResponse.newBuilder();
    boolean showComplete = request.hasShowComplete() && request.getShowComplete();
    try {
      response.addAllRequestIds(asyncService.getAsyncOperationList(showComplete));
    } catch (VoldemortException e) {
      response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
      logger.error("handleAsyncOperationList failed for request(" + request.toString() + ")", e);
    }

    return response.build();
  }
コード例 #8
0
 private VProto.DeleteResponse handleDelete(
     VProto.DeleteRequest request, Store<ByteArray, byte[]> store) {
   VProto.DeleteResponse.Builder response = VProto.DeleteResponse.newBuilder();
   try {
     boolean success =
         store.delete(
             ProtoUtils.decodeBytes(request.getKey()),
             ProtoUtils.decodeClock(request.getVersion()));
     response.setSuccess(success);
   } catch (VoldemortException e) {
     response.setSuccess(false);
     response.setError(ProtoUtils.encodeError(getErrorMapper(), e));
   }
   return response.build();
 }
コード例 #9
0
  public VAdminProto.TruncateEntriesResponse handleTruncateEntries(
      VAdminProto.TruncateEntriesRequest request) {
    VAdminProto.TruncateEntriesResponse.Builder response =
        VAdminProto.TruncateEntriesResponse.newBuilder();
    try {
      String storeName = request.getStore();
      StorageEngine<ByteArray, byte[]> storageEngine = getStorageEngine(storeRepository, storeName);

      storageEngine.truncate();
    } catch (VoldemortException e) {
      response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
      logger.error("handleTruncateEntries failed for request(" + request.toString() + ")", e);
    }

    return response.build();
  }
コード例 #10
0
 private VProto.GetAllResponse handleGetAll(
     VProto.GetAllRequest request, Store<ByteArray, byte[]> store) {
   VProto.GetAllResponse.Builder response = VProto.GetAllResponse.newBuilder();
   try {
     List<ByteArray> keys = new ArrayList<ByteArray>(request.getKeysCount());
     for (ByteString string : request.getKeysList()) keys.add(ProtoUtils.decodeBytes(string));
     Map<ByteArray, List<Versioned<byte[]>>> values = store.getAll(keys);
     for (Map.Entry<ByteArray, List<Versioned<byte[]>>> entry : values.entrySet()) {
       VProto.KeyedVersions.Builder keyedVersion =
           VProto.KeyedVersions.newBuilder().setKey(ProtoUtils.encodeBytes(entry.getKey()));
       for (Versioned<byte[]> version : entry.getValue())
         keyedVersion.addVersions(ProtoUtils.encodeVersioned(version));
       response.addValues(keyedVersion);
     }
   } catch (VoldemortException e) {
     response.setError(ProtoUtils.encodeError(getErrorMapper(), e));
   }
   return response.build();
 }
コード例 #11
0
  public VAdminProto.UpdateMetadataResponse handleUpdateMetadata(
      VAdminProto.UpdateMetadataRequest request) {
    VAdminProto.UpdateMetadataResponse.Builder response =
        VAdminProto.UpdateMetadataResponse.newBuilder();

    try {
      ByteArray key = ProtoUtils.decodeBytes(request.getKey());
      String keyString = ByteUtils.getString(key.get(), "UTF-8");

      if (MetadataStore.METADATA_KEYS.contains(keyString)) {
        Versioned<byte[]> versionedValue = ProtoUtils.decodeVersioned(request.getVersioned());
        metadataStore.put(new ByteArray(ByteUtils.getBytes(keyString, "UTF-8")), versionedValue);
      }
    } catch (VoldemortException e) {
      response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
      logger.error("handleUpdateMetadata failed for request(" + request.toString() + ")", e);
    }

    return response.build();
  }
コード例 #12
0
  public VAdminProto.DeletePartitionEntriesResponse handleDeletePartitionEntries(
      VAdminProto.DeletePartitionEntriesRequest request) {
    VAdminProto.DeletePartitionEntriesResponse.Builder response =
        VAdminProto.DeletePartitionEntriesResponse.newBuilder();
    ClosableIterator<Pair<ByteArray, Versioned<byte[]>>> iterator = null;
    try {
      String storeName = request.getStore();
      List<Integer> partitions = request.getPartitionsList();
      StorageEngine<ByteArray, byte[]> storageEngine = getStorageEngine(storeRepository, storeName);
      VoldemortFilter filter =
          (request.hasFilter())
              ? getFilterFromRequest(request.getFilter(), voldemortConfig, networkClassLoader)
              : new DefaultVoldemortFilter();
      RoutingStrategy routingStrategy = metadataStore.getRoutingStrategy(storageEngine.getName());

      EventThrottler throttler = new EventThrottler(voldemortConfig.getStreamMaxReadBytesPerSec());
      iterator = storageEngine.entries();
      int deleteSuccess = 0;

      while (iterator.hasNext()) {
        Pair<ByteArray, Versioned<byte[]>> entry = iterator.next();

        ByteArray key = entry.getFirst();
        Versioned<byte[]> value = entry.getSecond();
        throttler.maybeThrottle(key.length() + valueSize(value));
        if (checkKeyBelongsToDeletePartition(key.get(), partitions, routingStrategy)
            && filter.accept(key, value)) {
          if (storageEngine.delete(key, value.getVersion())) deleteSuccess++;
        }
      }
      response.setCount(deleteSuccess);
    } catch (VoldemortException e) {
      response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
      logger.error(
          "handleDeletePartitionEntries failed for request(" + request.toString() + ")", e);
    } finally {
      if (null != iterator) iterator.close();
    }

    return response.build();
  }
コード例 #13
0
  public VAdminProto.AsyncOperationStatusResponse handleAsyncStatus(
      VAdminProto.AsyncOperationStatusRequest request) {
    VAdminProto.AsyncOperationStatusResponse.Builder response =
        VAdminProto.AsyncOperationStatusResponse.newBuilder();
    try {
      int requestId = request.getRequestId();
      AsyncOperationStatus operationStatus = asyncService.getOperationStatus(requestId);
      boolean requestComplete = asyncService.isComplete(requestId);
      response.setDescription(operationStatus.getDescription());
      response.setComplete(requestComplete);
      response.setStatus(operationStatus.getStatus());
      response.setRequestId(requestId);
      if (operationStatus.hasException())
        throw new VoldemortException(operationStatus.getException());
    } catch (VoldemortException e) {
      response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
      logger.error("handleAsyncStatus failed for request(" + request.toString().trim() + ")", e);
    }

    return response.build();
  }
コード例 #14
0
  public VAdminProto.AddStoreResponse handleAddStore(VAdminProto.AddStoreRequest request) {
    VAdminProto.AddStoreResponse.Builder response = VAdminProto.AddStoreResponse.newBuilder();

    // don't try to add a store in the middle of rebalancing
    if (metadataStore
            .getServerState()
            .equals(MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER)
        || metadataStore
            .getServerState()
            .equals(MetadataStore.VoldemortState.REBALANCING_CLUSTER)) {
      response.setError(
          ProtoUtils.encodeError(
              errorCodeMapper, new VoldemortException("Rebalancing in progress")));
      return response.build();
    }

    try {
      // adding a store requires decoding the passed in store string
      StoreDefinitionsMapper mapper = new StoreDefinitionsMapper();
      StoreDefinition def = mapper.readStore(new StringReader(request.getStoreDefinition()));

      synchronized (lock) {
        // only allow a single store to be created at a time. We'll see concurrent errors when
        // writing the
        // stores.xml file out otherwise. (see ConfigurationStorageEngine.put for details)

        if (!storeRepository.hasLocalStore(def.getName())) {
          // open the store
          storageService.openStore(def);

          // update stores list in metadata store (this also has the
          // effect of updating the stores.xml file)
          List<StoreDefinition> currentStoreDefs;
          List<Versioned<byte[]>> v = metadataStore.get(MetadataStore.STORES_KEY);

          if (((v.size() > 0) ? 1 : 0) > 0) {
            Versioned<byte[]> currentValue = v.get(0);
            currentStoreDefs =
                mapper.readStoreList(
                    new StringReader(ByteUtils.getString(currentValue.getValue(), "UTF-8")));
          } else {
            currentStoreDefs = Lists.newArrayList();
          }
          currentStoreDefs.add(def);
          try {
            metadataStore.put(MetadataStore.STORES_KEY, currentStoreDefs);
          } catch (Exception e) {
            throw new VoldemortException(e);
          }
        } else {
          throw new StoreOperationFailureException(
              String.format("Store '%s' already exists on this server", def.getName()));
        }
      }
    } catch (VoldemortException e) {
      response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
      logger.error("handleAddStore failed for request(" + request.toString() + ")", e);
    }

    return response.build();
  }
コード例 #15
0
  public VAdminProto.AsyncOperationStatusResponse handleFetchAndUpdate(
      VAdminProto.InitiateFetchAndUpdateRequest request) {
    final int nodeId = request.getNodeId();
    final List<Integer> partitions = request.getPartitionsList();
    final VoldemortFilter filter =
        request.hasFilter()
            ? getFilterFromRequest(request.getFilter(), voldemortConfig, networkClassLoader)
            : new DefaultVoldemortFilter();
    final String storeName = request.getStore();

    int requestId = asyncService.getUniqueRequestId();
    VAdminProto.AsyncOperationStatusResponse.Builder response =
        VAdminProto.AsyncOperationStatusResponse.newBuilder()
            .setRequestId(requestId)
            .setComplete(false)
            .setDescription("Fetch and update")
            .setStatus("started");

    try {
      asyncService.submitOperation(
          requestId,
          new AsyncOperation(requestId, "Fetch and Update") {

            private final AtomicBoolean running = new AtomicBoolean(true);

            @Override
            public void stop() {
              running.set(false);
            }

            @Override
            public void operate() {
              AdminClient adminClient =
                  RebalanceUtils.createTempAdminClient(
                      voldemortConfig, metadataStore.getCluster(), 4, 2);
              try {
                StorageEngine<ByteArray, byte[]> storageEngine =
                    getStorageEngine(storeRepository, storeName);
                Iterator<Pair<ByteArray, Versioned<byte[]>>> entriesIterator =
                    adminClient.fetchEntries(nodeId, storeName, partitions, filter, false);
                updateStatus("Initated fetchPartitionEntries");
                EventThrottler throttler =
                    new EventThrottler(voldemortConfig.getStreamMaxWriteBytesPerSec());
                for (long i = 0; running.get() && entriesIterator.hasNext(); i++) {
                  Pair<ByteArray, Versioned<byte[]>> entry = entriesIterator.next();

                  ByteArray key = entry.getFirst();
                  Versioned<byte[]> value = entry.getSecond();
                  try {
                    storageEngine.put(key, value);
                  } catch (ObsoleteVersionException e) {
                    // log and ignore
                    logger.debug("migratePartition threw ObsoleteVersionException, Ignoring.");
                  }

                  throttler.maybeThrottle(key.length() + valueSize(value));
                  if ((i % 1000) == 0) {
                    updateStatus(i + " entries processed");
                  }
                }
              } finally {
                adminClient.stop();
              }
            }
          });

    } catch (VoldemortException e) {
      response.setError(ProtoUtils.encodeError(errorCodeMapper, e));
      logger.error("handleFetchAndUpdate failed for request(" + request.toString() + ")", e);
    }

    return response.build();
  }