@Test
public void testGetAllWithBinaryData() throws Exception {
Store<ByteArray, byte[], byte[]> store = getStore();
List<ByteArray> keys = Lists.newArrayList();
List<byte[]> values = Lists.newArrayList();
// The Byte 0x8c is interesting, because if you use GetContent method of
// MimeBodyPart it gets converted to 0xc2 and 0x8c
// This thread tracks this question
// http://stackoverflow.com/questions/23023583/mimebodypart-getcontent-corrupts-binary-data
byte[] interestingByte = new byte[] {(byte) 0x8c};
ByteArray interestingKey = new ByteArray(interestingByte);
keys.add(interestingKey);
values.add(interestingByte);
// Add all possible byte values
byte[] allPossibleBytes = getAllPossibleBytes();
ByteArray allPossibleKey = new ByteArray(allPossibleBytes);
keys.add(allPossibleKey);
values.add(allPossibleBytes);
assertEquals(keys.size(), values.size());
int count = keys.size();
for (int i = 0; i < count; i++) {
VectorClock vc = getClock(0, 0);
store.put(keys.get(i), new Versioned<byte[]>(values.get(i), vc), null);
}
Map<ByteArray, List<Versioned<byte[]>>> result = store.getAll(keys, null);
assertGetAllValues(keys, values, result);
}
@Override
@Test
public void testFetchedEqualsPut() throws Exception {
System.out.println(" Testing Fetchhed equals put ");
ByteArray key = getKey();
Store<ByteArray, byte[], byte[]> store = getStore();
VectorClock clock = getClock(1, 1, 2, 3, 3, 4);
byte[] value = getValue();
System.out.println("Value chosen : " + value);
List<Versioned<byte[]>> resultList = store.get(key, null);
assertNotNull("Null result list obtained from a get request", resultList);
assertEquals("Store not empty at start!", 0, resultList.size());
Versioned<byte[]> versioned = new Versioned<byte[]>(value, clock);
store.put(key, versioned, null);
List<Versioned<byte[]>> found = store.get(key, null);
assertEquals("Should only be one version stored.", 1, found.size());
System.out.println("individual bytes");
System.out.println("input");
printBytes(versioned.getValue());
System.out.println("found");
printBytes(found.get(0).getValue());
assertTrue("Values not equal!", valuesEqual(versioned.getValue(), found.get(0).getValue()));
}
@Test
public void testFetchPartitionKeys() {
HashMap<ByteArray, byte[]> entrySet =
ServerTestUtils.createRandomKeyValuePairs(TEST_STREAM_KEYS_SIZE);
List<Integer> fetchPartitionsList = Arrays.asList(0, 2);
// insert it into server-0 store
int fetchPartitionKeyCount = 0;
Store<ByteArray, byte[]> store = getStore(0, testStoreName);
for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
store.put(entry.getKey(), new Versioned<byte[]>(entry.getValue()));
if (isKeyPartition(entry.getKey(), 0, testStoreName, fetchPartitionsList)) {
fetchPartitionKeyCount++;
}
}
Iterator<ByteArray> fetchIt =
getAdminClient().fetchKeys(0, testStoreName, fetchPartitionsList, null, false);
// check values
int count = 0;
while (fetchIt.hasNext()) {
assertEquals(
"Fetched key should belong to asked partitions",
true,
isKeyPartition(fetchIt.next(), 0, testStoreName, fetchPartitionsList));
count++;
}
// assert all keys for asked partitions are returned.
assertEquals("All keys for asked partitions should be received", fetchPartitionKeyCount, count);
}
@Test
public void testRecoverData() {
// use store with replication 2, required write 2 for this test.
String testStoreName = "test-recovery-data";
HashMap<ByteArray, byte[]> entrySet =
ServerTestUtils.createRandomKeyValuePairs(TEST_STREAM_KEYS_SIZE);
// insert it into server-0 store
Store<ByteArray, byte[]> store = getStore(0, testStoreName);
for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
store.put(entry.getKey(), new Versioned<byte[]>(entry.getValue()));
}
// assert server 1 is empty
store = getStore(1, testStoreName);
for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
assertSame("entry should NOT be present at store", 0, store.get(entry.getKey()).size());
}
// recover all data
adminClient.restoreDataFromReplications(1, 2);
// assert server 1 has all entries for its partitions
store = getStore(1, testStoreName);
for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
ByteArray key = entry.getKey();
assertSame("entry should be present for key " + key, 1, store.get(entry.getKey()).size());
assertEquals(
"entry value should match",
new String(entry.getValue()),
new String(store.get(entry.getKey()).get(0).getValue()));
}
}
protected void initializeMetadataVersions(List<StoreDefinition> storeDefs) {
Store<ByteArray, byte[], byte[]> versionStore =
storeRepository.getLocalStore(
SystemStoreConstants.SystemStoreName.voldsys$_metadata_version_persistence.name());
Properties props = new Properties();
try {
boolean isPropertyAdded = false;
ByteArray metadataVersionsKey =
new ByteArray(SystemStoreConstants.VERSIONS_METADATA_KEY.getBytes());
List<Versioned<byte[]>> versionList = versionStore.get(metadataVersionsKey, null);
VectorClock newClock = null;
if (versionList != null && versionList.size() > 0) {
byte[] versionsByteArray = versionList.get(0).getValue();
if (versionsByteArray != null) {
props.load(new ByteArrayInputStream(versionsByteArray));
}
newClock = (VectorClock) versionList.get(0).getVersion();
newClock = newClock.incremented(0, System.currentTimeMillis());
} else {
newClock = new VectorClock();
}
// Check if version exists for cluster.xml
if (!props.containsKey(SystemStoreConstants.CLUSTER_VERSION_KEY)) {
props.setProperty(SystemStoreConstants.CLUSTER_VERSION_KEY, "0");
isPropertyAdded = true;
}
// Check if version exists for stores.xml
if (!props.containsKey(SystemStoreConstants.STORES_VERSION_KEY)) {
props.setProperty(SystemStoreConstants.STORES_VERSION_KEY, "0");
isPropertyAdded = true;
}
// Check if version exists for each store
for (StoreDefinition def : storeDefs) {
if (!props.containsKey(def.getName())) {
props.setProperty(def.getName(), "0");
isPropertyAdded = true;
}
}
if (isPropertyAdded) {
StringBuilder finalVersionList = new StringBuilder();
for (String propName : props.stringPropertyNames()) {
finalVersionList.append(propName + "=" + props.getProperty(propName) + "\n");
}
versionStore.put(
metadataVersionsKey,
new Versioned<byte[]>(finalVersionList.toString().getBytes(), newClock),
null);
}
} catch (Exception e) {
logger.error("Error while intializing metadata versions ", e);
}
}
@Before
public void setup() throws IOException {
byte[] bytes1 = {(byte) 'A', (byte) 'B'};
byte[] bytes2 = {(byte) 'C', (byte) 'D'};
List<StoreDefinition> stores = ClusterTestUtils.getZZZ322StoreDefs("memory");
StoreDefinition storeDef = stores.get(0);
cluster = ClusterTestUtils.getZZZCluster();
ClientConfig clientConfig = new ClientConfig();
clientConfig.setBootstrapUrls(cluster.getNodeById(0).getSocketUrl().toString());
clientConfig.getZoneAffinity().setEnableGetOpZoneAffinity(true);
clientConfig.setClientZoneId(clientZoneId);
SocketStoreClientFactory socketStoreClientFactory = new SocketStoreClientFactory(clientConfig);
for (Integer nodeId : cluster.getNodeIds()) {
SocketStoreFactory socketStoreFactory = new ClientRequestExecutorPool(2, 10000, 100000, 1024);
VoldemortConfig config =
ServerTestUtils.createServerConfigWithDefs(
true,
nodeId,
TestUtils.createTempDir().getAbsolutePath(),
cluster,
stores,
new Properties());
VoldemortServer vs =
ServerTestUtils.startVoldemortServer(socketStoreFactory, config, cluster);
vservers.put(nodeId, vs);
socketStoreFactories.put(nodeId, socketStoreFactory);
Store<ByteArray, byte[], byte[]> store =
vs.getStoreRepository().getLocalStore(storeDef.getName());
Node node = cluster.getNodeById(nodeId);
VectorClock version1 = new VectorClock();
version1.incrementVersion(0, System.currentTimeMillis());
VectorClock version2 = version1.incremented(0, System.currentTimeMillis());
if (node.getZoneId() == clientZoneId) {
store.put(new ByteArray(bytes1), new Versioned<byte[]>(bytes1, version1), null);
} else {
store.put(new ByteArray(bytes1), new Versioned<byte[]>(bytes2, version2), null);
}
}
client = socketStoreClientFactory.getRawStore(storeDef.getName(), null);
}
private VProto.PutResponse handlePut(VProto.PutRequest request, Store<ByteArray, byte[]> store) {
VProto.PutResponse.Builder response = VProto.PutResponse.newBuilder();
try {
ByteArray key = ProtoUtils.decodeBytes(request.getKey());
Versioned<byte[]> value = ProtoUtils.decodeVersioned(request.getVersioned());
store.put(key, value);
} catch (VoldemortException e) {
response.setError(ProtoUtils.encodeError(getErrorMapper(), e));
}
return response.build();
}
@Override
@Test
public void testNullKeys() throws Exception {
System.out.println(" Testing null Keys ");
Store<ByteArray, byte[], byte[]> store = getStore();
try {
store.put(null, new Versioned<byte[]>(getValue(), getNewIncrementedVectorClock()), null);
fail("Store should not put null keys!");
} catch (IllegalArgumentException e) {
// this is good
} catch (NullPointerException npe) {
// this is good
}
try {
store.get(null, null);
fail("Store should not get null keys!");
} catch (IllegalArgumentException e) {
// this is good
} catch (NullPointerException npe) {
// this is good
}
try {
store.getAll(null, null);
fail("Store should not getAll null keys!");
} catch (IllegalArgumentException e) {
// this is good
} catch (NullPointerException npe) {
// this is good
}
try {
store.getAll(
Collections.<ByteArray>singleton(null),
Collections.<ByteArray, byte[]>singletonMap(null, null));
fail("Store should not getAll null keys!");
} catch (IllegalArgumentException e) {
// this is good
} catch (NullPointerException npe) {
// this is good
}
try {
store.delete(null, new VectorClock());
fail("Store should not delete null keys!");
} catch (IllegalArgumentException e) {
// this is good
} catch (NullPointerException npe) {
// this is good
}
}
/** @throws IOException */
@Test
public void testFetchAndUpdate() throws IOException {
HashMap<ByteArray, byte[]> entrySet =
ServerTestUtils.createRandomKeyValuePairs(TEST_STREAM_KEYS_SIZE);
List<Integer> fetchAndUpdatePartitionsList = Arrays.asList(0, 2);
// insert it into server-0 store
int fetchPartitionKeyCount = 0;
Store<ByteArray, byte[]> store = getStore(0, testStoreName);
for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
store.put(entry.getKey(), new Versioned<byte[]>(entry.getValue()));
if (isKeyPartition(entry.getKey(), 0, testStoreName, fetchAndUpdatePartitionsList)) {
fetchPartitionKeyCount++;
}
}
// assert that server1 is empty.
store = getStore(1, testStoreName);
for (Entry<ByteArray, byte[]> entry : entrySet.entrySet())
assertEquals("server1 should be empty at start.", 0, store.get(entry.getKey()).size());
// do fetch And update call server1 <-- server0
AdminClient client = getAdminClient();
int id = client.migratePartitions(0, 1, testStoreName, fetchAndUpdatePartitionsList, null);
client.waitForCompletion(1, id, 60, TimeUnit.SECONDS);
// check values
int count = 0;
store = getStore(1, testStoreName);
for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
if (isKeyPartition(entry.getKey(), 0, testStoreName, fetchAndUpdatePartitionsList)) {
assertEquals(
"server1 store should contain fetchAndupdated partitions.",
1,
store.get(entry.getKey()).size());
assertEquals(
"entry value should match",
new String(entry.getValue()),
new String(store.get(entry.getKey()).get(0).getValue()));
count++;
}
}
// assert all keys for asked partitions are returned.
assertEquals("All keys for asked partitions should be received", fetchPartitionKeyCount, count);
}
@Override
@Test
public void testGetVersions() throws Exception {
List<ByteArray> keys = getKeys(2);
ByteArray key = keys.get(0);
byte[] value = getValue();
VectorClock vc = getClock(0, 0);
Store<ByteArray, byte[], byte[]> store = getStore();
store.put(key, Versioned.value(value, vc), null);
List<Versioned<byte[]>> versioneds = store.get(key, null);
List<Version> versions = store.getVersions(key);
assertEquals(1, versioneds.size());
assertTrue(versions.size() > 0);
for (int i = 0; i < versions.size(); i++)
assertEquals(versioneds.get(0).getVersion(), versions.get(i));
assertEquals(0, store.getVersions(keys.get(1)).size());
}
@Override
@Test
public void testGetAll() throws Exception {
Store<ByteArray, byte[], byte[]> store = getStore();
int putCount = 10;
List<ByteArray> keys = getKeys(putCount);
List<byte[]> values = getValues(putCount);
assertEquals(putCount, values.size());
for (int i = 0; i < putCount; i++) {
VectorClock vc = getClock(0, 0);
store.put(keys.get(i), new Versioned<byte[]>(values.get(i), vc), null);
}
int countForGet = putCount / 2;
List<ByteArray> keysForGet = keys.subList(0, countForGet);
List<byte[]> valuesForGet = values.subList(0, countForGet);
Map<ByteArray, List<Versioned<byte[]>>> result = store.getAll(keysForGet, null);
assertGetAllValues(keysForGet, valuesForGet, result);
}
// check the basic rebalanceNode call.
@Test
public void testRebalanceNode() {
HashMap<ByteArray, byte[]> entrySet =
ServerTestUtils.createRandomKeyValuePairs(TEST_STREAM_KEYS_SIZE);
List<Integer> fetchAndUpdatePartitionsList = Arrays.asList(0, 2);
// insert it into server-0 store
int fetchPartitionKeyCount = 0;
Store<ByteArray, byte[]> store = getStore(0, testStoreName);
for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
store.put(entry.getKey(), new Versioned<byte[]>(entry.getValue()));
if (isKeyPartition(entry.getKey(), 0, testStoreName, fetchAndUpdatePartitionsList)) {
fetchPartitionKeyCount++;
}
}
List<Integer> rebalancePartitionList = Arrays.asList(1, 3);
RebalancePartitionsInfo stealInfo =
new RebalancePartitionsInfo(
1,
0,
rebalancePartitionList,
new ArrayList<Integer>(0),
Arrays.asList(testStoreName),
0);
int asyncId = adminClient.rebalanceNode(stealInfo);
assertNotSame("Got a valid rebalanceAsyncId", -1, asyncId);
getAdminClient().waitForCompletion(1, asyncId, 120, TimeUnit.SECONDS);
// assert data is copied correctly
store = getStore(1, testStoreName);
for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
if (isKeyPartition(entry.getKey(), 1, testStoreName, rebalancePartitionList)) {
assertSame("entry should be present at store", 1, store.get(entry.getKey()).size());
assertEquals(
"entry value should match",
new String(entry.getValue()),
new String(store.get(entry.getKey()).get(0).getValue()));
}
}
}
@Test
public void testTruncate() throws Exception {
HashMap<ByteArray, byte[]> entrySet =
ServerTestUtils.createRandomKeyValuePairs(TEST_STREAM_KEYS_SIZE);
// insert it into server-0 store
Store<ByteArray, byte[]> store = getStore(0, testStoreName);
for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
store.put(entry.getKey(), new Versioned<byte[]>(entry.getValue()));
}
// do truncate request
getAdminClient().truncate(0, testStoreName);
store = getStore(0, testStoreName);
for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
assertEquals("Deleted key should be missing.", 0, store.get(entry.getKey()).size());
}
}
@Test
public void testGetWithBinaryData() throws Exception {
Store<ByteArray, byte[], byte[]> store = getStore();
byte[] allPossibleBytes = getAllPossibleBytes();
ByteArray key = new ByteArray(allPossibleBytes);
VectorClock vc = getClock(0, 0);
Versioned<byte[]> versioned = new Versioned<byte[]>(allPossibleBytes, vc);
store.put(key, versioned, null);
List<Versioned<byte[]>> found = store.get(key, null);
assertEquals("Should only be one version stored.", 1, found.size());
System.out.println("individual bytes");
System.out.println("input");
printBytes(versioned.getValue());
System.out.println("found");
printBytes(found.get(0).getValue());
assertTrue("Values not equal!", valuesEqual(versioned.getValue(), found.get(0).getValue()));
}
public void testDeletePartitionEntries() {
HashMap<ByteArray, byte[]> entrySet =
ServerTestUtils.createRandomKeyValuePairs(TEST_STREAM_KEYS_SIZE);
// insert it into server-0 store
Store<ByteArray, byte[]> store = getStore(0, testStoreName);
for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
store.put(entry.getKey(), new Versioned<byte[]>(entry.getValue()));
}
List<Integer> deletePartitionsList = Arrays.asList(0, 2);
// do delete partitions request
getAdminClient().deletePartitions(0, testStoreName, deletePartitionsList, null);
store = getStore(0, testStoreName);
for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
if (isKeyPartition(entry.getKey(), 0, testStoreName, deletePartitionsList)) {
assertEquals("deleted partitions should be missing.", 0, store.get(entry.getKey()).size());
}
}
}
@Override
public Version put(K key, Versioned<V> versioned) throws ObsoleteVersionException {
clientStore.put(key, versioned, null);
return versioned.getVersion();
}
private void putInner(String key, Versioned<String> value) {
innerStore.put(key, value);
}
