/**
* Creates a store with the specified number of partitions.
*
* <p>NOTE: calling this method will set {@link Constants#KEY_VALUE_PARTITION_SIZE_BYTES_MAX} to
* {@link Constants#MB}.
*
* @param partitionNumber the number of partitions
* @param keyValuePairs the key-value pairs in the store, null if you don't want to know them
* @return the URI to the created store
*/
private TachyonURI createStoreOfMultiplePartitions(
int partitionNumber, List<KeyValuePair> keyValuePairs) throws Exception {
// These sizes are carefully selected, one partition holds only one key-value pair.
final long maxPartitionSize = Constants.MB; // Each partition is at most 1 MB
ClientContext.getConf()
.set(Constants.KEY_VALUE_PARTITION_SIZE_BYTES_MAX, String.valueOf(maxPartitionSize));
final int keyLength = 4; // 4Byte key
final int valueLength = 500 * Constants.KB; // 500KB value
TachyonURI storeUri = new TachyonURI(PathUtils.uniqPath());
mWriter = sKeyValueSystem.createStore(storeUri);
for (int i = 0; i < partitionNumber; i++) {
byte[] key = BufferUtils.getIncreasingByteArray(i, keyLength);
byte[] value = BufferUtils.getIncreasingByteArray(i, valueLength);
mWriter.put(key, value);
if (keyValuePairs != null) {
keyValuePairs.add(new KeyValuePair(key, value));
}
}
mWriter.close();
Assert.assertEquals(partitionNumber, getPartitionNumber(storeUri));
return storeUri;
}
/** Tests creating and opening a store with one key. */
@Test
public void createAndOpenStoreWithOneKeyTest() throws Exception {
mWriter = sKeyValueSystem.createStore(mStoreUri);
mWriter.put(KEY1, VALUE1);
mWriter.close();
mReader = sKeyValueSystem.openStore(mStoreUri);
Assert.assertArrayEquals(VALUE1, mReader.get(KEY1));
Assert.assertNull(mReader.get(KEY2));
mReader.close();
}
/**
* Creates a store with the specified number of key-value pairs. The key-value pairs are in the
* format specified in {@link #genBaseKey(int)} and {@link #genBaseValue(int)} with id starts from
* 0.
*
* <p>The created store's size is {@link Assert}ed before return.
*
* @param size the number of key-value pairs
* @param pairs the key-value pairs in the store, null if you don't want to know them
* @return the URI to the store
* @throws Exception if any error happens
*/
private TachyonURI createStoreOfSize(int size, List<KeyValuePair> pairs) throws Exception {
TachyonURI path = new TachyonURI(PathUtils.uniqPath());
KeyValueStoreWriter writer = sKeyValueSystem.createStore(path);
for (int i = 0; i < size; i++) {
byte[] key = genBaseKey(i).getBytes();
byte[] value = genBaseValue(i).getBytes();
writer.put(key, value);
if (pairs != null) {
pairs.add(new KeyValuePair(key, value));
}
}
writer.close();
Assert.assertEquals(size, sKeyValueSystem.openStore(path).size());
return path;
}
/** Tests that an iterator for an empty store has no next elements. */
@Test
public void emptyStoreIteratorTest() throws Exception {
mWriter = sKeyValueSystem.createStore(mStoreUri);
mWriter.close();
mReader = sKeyValueSystem.openStore(mStoreUri);
KeyValueIterator iterator = mReader.iterator();
Assert.assertFalse(iterator.hasNext());
}
/** Tests creating and opening an empty store. */
@Test
public void createAndOpenEmptyStoreTest() throws Exception {
mWriter = sKeyValueSystem.createStore(mStoreUri);
Assert.assertNotNull(mWriter);
mWriter.close();
mReader = sKeyValueSystem.openStore(mStoreUri);
Assert.assertNotNull(mReader);
mReader.close();
}
/** Tests creating and opening a store with a number of key. */
@Test
public void createAndOpenStoreWithMultiKeysTest() throws Exception {
final int numKeys = 100;
final int keyLength = 4; // 4Byte key
final int valueLength = 5 * Constants.KB; // 5KB value
mWriter = sKeyValueSystem.createStore(mStoreUri);
for (int i = 0; i < numKeys; i++) {
byte[] key = BufferUtils.getIncreasingByteArray(i, keyLength);
byte[] value = BufferUtils.getIncreasingByteArray(i, valueLength);
mWriter.put(key, value);
}
mWriter.close();
mReader = sKeyValueSystem.openStore(mStoreUri);
for (int i = 0; i < numKeys; i++) {
byte[] key = BufferUtils.getIncreasingByteArray(i, keyLength);
byte[] value = mReader.get(key);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(i, valueLength, value));
}
Assert.assertNull(mReader.get(KEY1));
Assert.assertNull(mReader.get(KEY2));
mReader.close();
}
/**
* Tests creating and opening a store with a number of keys, while each key-value pair is large
* enough to take a separate key-value partition.
*/
@Test
public void createMultiPartitionsTest() throws Exception {
// TODO(cc): Remove codes using createStoreOfMultiplePartitions.
final long maxPartitionSize = Constants.MB; // Each partition is at most 1 MB
final int numKeys = 10;
final int keyLength = 4; // 4Byte key
final int valueLength = 500 * Constants.KB; // 500KB value
FileSystem fs = FileSystem.Factory.get();
ClientContext.getConf()
.set(Constants.KEY_VALUE_PARTITION_SIZE_BYTES_MAX, String.valueOf(maxPartitionSize));
mWriter = sKeyValueSystem.createStore(mStoreUri);
for (int i = 0; i < numKeys; i++) {
byte[] key = BufferUtils.getIncreasingByteArray(i, keyLength);
byte[] value = BufferUtils.getIncreasingByteArray(i, valueLength);
mWriter.put(key, value);
}
mWriter.close();
List<URIStatus> files = fs.listStatus(mStoreUri);
Assert.assertEquals(numKeys, files.size());
for (URIStatus info : files) {
Assert.assertTrue(info.getLength() <= maxPartitionSize);
}
mReader = sKeyValueSystem.openStore(mStoreUri);
for (int i = 0; i < numKeys; i++) {
byte[] key = BufferUtils.getIncreasingByteArray(i, keyLength);
byte[] value = mReader.get(key);
Assert.assertTrue(BufferUtils.equalIncreasingByteArray(i, valueLength, value));
}
Assert.assertNull(mReader.get(KEY1));
Assert.assertNull(mReader.get(KEY2));
mReader.close();
}
/**
* Tests putting a key-value pair that is larger than the max key-value partition size, expecting
* exception thrown.
*/
@Test
public void putKeyValueTooLargeTest() throws Exception {
final long maxPartitionSize = 500 * Constants.KB; // Each partition is at most 500 KB
final int keyLength = 4; // 4Byte key
final int valueLength = 500 * Constants.KB; // 500KB value
ClientContext.getConf()
.set(Constants.KEY_VALUE_PARTITION_SIZE_BYTES_MAX, String.valueOf(maxPartitionSize));
mWriter = sKeyValueSystem.createStore(mStoreUri);
byte[] key = BufferUtils.getIncreasingByteArray(0, keyLength);
byte[] value = BufferUtils.getIncreasingByteArray(0, valueLength);
mThrown.expect(IOException.class);
mThrown.expectMessage(ExceptionMessage.KEY_VALUE_TOO_LARGE.getMessage(keyLength, valueLength));
mWriter.put(key, value);
}
