public void testIterable() throws Exception {
_store.clear();
// Generate keys
HashSet<String> keySet = new HashSet<String>(199);
while (keySet.size() < 100) {
keySet.add(UUID.randomUUID().toString());
}
assertEquals(100, keySet.size());
// Populate store
for (String key : keySet) {
byte[] value = RandomBytes.getBytes();
_store.put(key.getBytes(), value);
}
HashSet<String> keySet2 = new HashSet<String>(199);
for (Entry<byte[], byte[]> e : _store) {
keySet2.add(new String(e.getKey()));
}
assertEquals(keySet.size(), keySet2.size());
keySet2.removeAll(keySet);
assertEquals(0, keySet2.size());
}
@Override
protected void tearDown() {
try {
_store.close();
FileUtils.deleteDirectory(_homeDir);
} catch (Exception e) {
e.printStackTrace();
}
}
protected void iterate(DataStore<byte[], byte[]> store, int runTimeSeconds) {
int cnt = 0;
long total = runTimeSeconds * 1000;
long start = System.currentTimeMillis();
Iterator<byte[]> iter = store.keyIterator();
StatsLog.logger.info(">>> iterate");
byte[] key = null;
while (iter.hasNext()) {
key = iter.next();
store.get(key);
cnt++;
if ((System.currentTimeMillis() - start) > total) break;
}
StatsLog.logger.info(
"read " + cnt + " key-value(s) in " + (System.currentTimeMillis() - start) + " ms");
}
public void testKeyIndexedIterator() throws Exception {
IndexedIterator<byte[]> keyIter;
_store.clear();
keyIter = _store.keyIterator();
assertFalse(keyIter.hasNext());
// Generate keys
HashSet<String> keySet = new HashSet<String>(199);
while (keySet.size() < 100) {
keySet.add(UUID.randomUUID().toString());
}
assertEquals(100, keySet.size());
// Populate store
for (String key : keySet) {
byte[] value = RandomBytes.getBytes();
_store.put(key.getBytes(), value);
}
// Check keys
keyIter = _store.keyIterator();
byte[] key1a = keyIter.next();
keyIter.reset(0);
byte[] key1b = keyIter.next();
for (int i = 0; i < 10; i++) {
keyIter.next();
}
keyIter.reset(keyIter.index());
byte[] key1c = keyIter.next();
assertTrue(Arrays.equals(key1a, key1b));
// Re-open store
_store.close();
_store.open();
// check keys
keyIter = _store.keyIterator();
byte[] key2a = keyIter.next();
keyIter.reset(0);
byte[] key2b = keyIter.next();
for (int i = 0; i < 10; i++) {
keyIter.next();
}
keyIter.reset(keyIter.index());
byte[] key2c = keyIter.next();
assertTrue(Arrays.equals(key1a, key2a));
assertTrue(Arrays.equals(key1b, key2b));
assertTrue(Arrays.equals(key1c, key2c));
}
public void evalPerformance(int numOfReaders, int numOfWriters, int runDuration)
throws Exception {
File storeDir = getHomeDirectory();
if (!storeDir.exists()) storeDir.mkdirs();
cleanDirectory(storeDir);
DataStore<byte[], byte[]> store = getDataStore(storeDir);
StoreReader<DataStore<byte[], byte[]>, byte[], byte[]> storeReader = new DataStoreReader();
StoreWriter<DataStore<byte[], byte[]>, byte[], byte[]> storeWriter = new DataStoreWriter();
StoreTestDriver driver;
driver =
new StoreTestBytesDriver<DataStore<byte[], byte[]>>(
store, storeReader, storeWriter, _lineSeedData, _keyCount, _hitPercent);
driver.run(numOfReaders, numOfWriters, runDuration);
store.sync();
try {
iterate(store, 100);
} catch (UnsupportedOperationException e) {
}
}
@Override
public Position get(Position pos, Map<K, Event<V>> map) {
ArrayList<Event<K>> list = new ArrayList<Event<K>>(1000);
Position nextPos = get(pos, list);
for (Event<K> evt : list) {
K key = evt.getValue();
if (key != null) {
try {
V value = _store.get(key);
map.put(key, new SimpleEvent<V>(value, evt.getClock()));
} catch (Exception e) {
_logger.warn(e.getMessage());
}
}
}
return nextPos;
}
@Override
public final void put(DataStore<byte[], byte[]> store, byte[] key, byte[] value)
throws Exception {
store.put(key, value);
}
@Override
public final byte[] get(DataStore<byte[], byte[]> store, byte[] key) {
return (key == null) ? null : store.get(key);
}
@Override
public V get(K key) throws Exception {
return key == null ? null : _store.get(key);
}
/** Gets the index start of the underlying store. */
protected int getStoreIndexStart() {
return _store.keyIterator().index();
}
@Override
public Position get(Position pos, List<Event<K>> list) {
if (pos.getId() != _retention.getId()) {
if (pos.isIndexed()) {
throw new InvalidPositionException("Bootstrap reconnection rejected", pos);
} else {
Position newPos = getPosition(pos.getClock());
if (newPos == null) {
newPos =
new SimplePosition(
_retention.getId(), _retention.getOffset(), getStoreIndexStart(), pos.getClock());
_logger.warn("Reset position from " + pos + " to " + newPos);
}
pos = newPos;
}
}
// Reset position if necessary
if (pos.getOffset() < _retention.getOrigin()) {
Position newPos =
new SimplePosition(
_retention.getId(), _retention.getOffset(), getStoreIndexStart(), pos.getClock());
_logger.warn("Reset position from " + pos + " to " + newPos);
pos = newPos;
}
// Read from the retention directly
Position nextPos = _retention.get(pos, list);
// Out of retention and need to start bootstrap
if (nextPos == null && pos.isIndexed()) {
int index = pos.getIndex();
IndexedIterator<K> iter = _store.keyIterator();
try {
iter.reset(index);
} catch (ArrayIndexOutOfBoundsException e) {
Position newPos = new SimplePosition(_retention.getId(), pos.getOffset(), pos.getClock());
_logger.warn("Reset position from " + pos + " to " + newPos, e);
return newPos;
}
int cnt = 0;
int lastIndex = index;
while (iter.hasNext()) {
lastIndex = iter.index();
K key = iter.next();
index = iter.index();
list.add(new SimpleEvent<K>(key, pos.getClock()));
cnt++;
if (cnt >= _retention.getBatchSize()) {
if (lastIndex == index) {
while (iter.hasNext() && iter.index() == index) {
key = iter.next();
list.add(new SimpleEvent<K>(key, pos.getClock()));
cnt++;
}
index++;
}
// Exit loop when enough events are collected
break;
}
}
if (cnt > 0) {
_logger.info("Read[" + pos.getIndex() + "," + index + ") " + cnt);
}
if (iter.hasNext()) {
return new SimplePosition(_retention.getId(), pos.getOffset(), index, pos.getClock());
} else {
Clock newClock;
newClock = _retention.getClock(pos.getOffset());
if (newClock == null) newClock = pos.getClock();
return new SimplePosition(_retention.getId(), pos.getOffset(), newClock);
}
} else {
return nextPos;
}
}
