@BeforeMethod
public void setUp() throws Exception {
// do cleanup of other tests that may have started writers.
// this is necessary in a non-forked test suite environment
// where partition ids could collide
Utils.finishBackupWriters();
m_tmpDir = new File(System.getProperty("java.io.tmpdir"), "PartitionStoreStageTest");
if (!m_tmpDir.exists()) {
assertTrue("Failed to make the directories.", m_tmpDir.mkdirs());
}
// Add a dummy partition
String server = "test";
String storage = m_tmpDir + "/storage";
File storageDir = new File(storage);
if (!storageDir.exists()) {
assertTrue("Failed to make the storage directory.", storageDir.mkdirs());
if (!storageDir.exists()) {
s_logger.warn("Storage Directory " + storage + " does not exist.");
}
}
String mapped = m_tmpDir + "/mapped";
File mappedDir = new File(mapped);
if (!mappedDir.exists()) {
assertTrue("Failed to make the mapped directories.", mappedDir.mkdirs());
if (!mappedDir.exists()) {
s_logger.warn("Storage Directory " + mapped + " does not exist.");
}
}
if (m_part == null) {
IPartitionManager pm = ManagementContainer.getInstance().getPartitionManager();
m_part = pm.initializePartition(TEST_PID, server, storage, mapped);
}
Utils.startBackupWriters();
}
/** @author peter.lawrey */
public class MapWrapperTest {
static final String TMP = System.getProperty("java.io.tmpdir");
@Test
public void testMethods() throws IOException {
String name = TMP + "/set-methods";
ChronicleTools.deleteOnExit(name);
{
MapListener stringsListener = createMock("strings", MapListener.class);
stringsListener.eventStart(1, "strings");
stringsListener.add("Hello", "hi");
stringsListener.eventEnd(true);
stringsListener.eventStart(3, "strings");
stringsListener.add("World", "all");
stringsListener.eventEnd(true);
MapListener intListener = createMock("ints", MapListener.class);
for (int i = 0; i < 3; i++) {
intListener.eventStart(i * 2, "ints");
intListener.add(i, i + 1000);
intListener.eventEnd(true);
}
replay(stringsListener);
replay(intListener);
Chronicle chronicle = new IndexedChronicle(name);
DataStore dataStore = new DataStore(chronicle, ModelMode.MASTER);
MapWrapper<String, String> strings =
new MapWrapper<String, String>(
dataStore,
"strings",
String.class,
String.class,
new LinkedHashMap<String, String>(),
16);
strings.addListener(stringsListener);
MapWrapper<Integer, Integer> ints =
new MapWrapper<Integer, Integer>(
dataStore,
"ints",
Integer.class,
Integer.class,
new LinkedHashMap<Integer, Integer>(),
16);
ints.addListener(intListener);
dataStore.start();
ints.put(0, 1000);
strings.put("Hello", "hi");
ints.put(1, 1001);
strings.put("World", "all");
ints.put(2, 1002);
verify(stringsListener);
verify(intListener);
assertEquals("{Hello=hi, World=all}", strings.toString());
assertEquals("{0=1000, 1=1001, 2=1002}", ints.toString());
chronicle.close();
}
{
MapListener stringsListener = createMock("strings", MapListener.class);
stringsListener.eventStart(5, "strings");
stringsListener.add("!", "end");
stringsListener.eventEnd(true);
MapListener intListener = createMock("ints", MapListener.class);
intListener.eventStart(6, "ints");
intListener.add(3, 1003);
intListener.eventEnd(true);
replay(stringsListener);
replay(intListener);
Chronicle chronicle = new IndexedChronicle(name);
DataStore dataStore = new DataStore(chronicle, ModelMode.MASTER);
MapWrapper<String, String> strings =
new MapWrapper<String, String>(
dataStore,
"strings",
String.class,
String.class,
new LinkedHashMap<String, String>(),
16);
strings.addListener(stringsListener);
MapWrapper<Integer, Integer> ints =
new MapWrapper<Integer, Integer>(
dataStore,
"ints",
Integer.class,
Integer.class,
new LinkedHashMap<Integer, Integer>(),
16);
ints.addListener(intListener);
// assume we have all the events written so far
dataStore.start(chronicle.size());
strings.put("!", "end");
ints.put(3, 1003);
verify(stringsListener);
verify(intListener);
assertEquals("{Hello=hi, World=all, !=end}", strings.toString());
assertEquals("{0=1000, 1=1001, 2=1002, 3=1003}", ints.toString());
chronicle.close();
}
}
@Test
public void testMapPerformance() throws IOException {
String name = TMP + "/map-perf";
ChronicleTools.deleteOnExit(name);
long start = System.nanoTime();
int size = 0;
{
Chronicle chronicle = new IndexedChronicle(name);
DataStore dataStore = new DataStore(chronicle, ModelMode.MASTER);
MapWrapper<String, String> strings =
new MapWrapper<String, String>(
dataStore,
"strings",
String.class,
String.class,
new LinkedHashMap<String, String>(),
16);
MapWrapper<Integer, Integer> ints =
new MapWrapper<Integer, Integer>(
dataStore,
"ints",
Integer.class,
Integer.class,
new LinkedHashMap<Integer, Integer>(),
16);
dataStore.start();
ints.clear();
strings.clear();
for (int j = 0; j < 10000; j++) {
for (int i = 0; i < 100; i++) {
ints.put(i, i + j);
strings.put(Integer.toString(i), Integer.toString(i + j));
}
size += Math.min(strings.size(), ints.size());
for (int i = 0; i < 100; i++) {
ints.remove(i);
strings.remove(Integer.toString(i));
}
}
chronicle.close();
}
long mid = System.nanoTime();
{
Chronicle chronicle = new IndexedChronicle(name);
DataStore dataStore = new DataStore(chronicle, ModelMode.MASTER);
MapWrapper<String, String> strings =
new MapWrapper<String, String>(
dataStore,
"strings",
String.class,
String.class,
new LinkedHashMap<String, String>(),
16);
MapWrapper<Integer, Integer> ints =
new MapWrapper<Integer, Integer>(
dataStore,
"ints",
Integer.class,
Integer.class,
new LinkedHashMap<Integer, Integer>(),
16);
dataStore.start();
chronicle.close();
}
long end = System.nanoTime();
System.out.printf(
"Took %.1f seconds avg to add&remove %,d elements and %.1f seconds avg to reload them%n",
(mid - start) / 2e9, size, (end - mid) / 2e9);
}
@Test
public void testOverTcp() throws IOException, InterruptedException {
String name = TMP + "/testOverTcp0";
String name2 = TMP + "/testOverTcp2";
ChronicleTools.deleteOnExit(name);
ChronicleTools.deleteOnExit(name2);
long start = System.nanoTime();
int PORT = 12345;
int size = 0;
InProcessChronicleSource chronicle =
new InProcessChronicleSource(new IndexedChronicle(name), PORT);
DataStore dataStore = new DataStore(chronicle, ModelMode.MASTER);
MapWrapper<String, String> strings =
new MapWrapper<String, String>(
dataStore,
"strings",
String.class,
String.class,
new LinkedHashMap<String, String>(),
16);
MapWrapper<Integer, Integer> ints =
new MapWrapper<Integer, Integer>(
dataStore,
"ints",
Integer.class,
Integer.class,
new LinkedHashMap<Integer, Integer>(),
16);
dataStore.start();
ints.clear();
strings.clear();
InProcessChronicleSink chronicle2 =
new InProcessChronicleSink(new IndexedChronicle(name2), "localhost", PORT);
DataStore dataStore2 = new DataStore(chronicle2, ModelMode.READ_ONLY);
MapWrapper<String, String> strings2 =
new MapWrapper<String, String>(
dataStore2,
"strings",
String.class,
String.class,
new LinkedHashMap<String, String>(),
16);
MapWrapper<Integer, Integer> ints2 =
new MapWrapper<Integer, Integer>(
dataStore2,
"ints",
Integer.class,
Integer.class,
new LinkedHashMap<Integer, Integer>(),
16);
final AtomicInteger sai = new AtomicInteger();
MapListener<String, String> stringsListener =
new AbstractMapListener<String, String>() {
@Override
public void update(String key, String oldValue, String newValue) {
// System.out.println(key + " " + oldValue + " => " + newValue);
sai.incrementAndGet();
}
};
strings2.addListener(stringsListener);
final AtomicInteger iai = new AtomicInteger();
MapListener<Integer, Integer> intsListener =
new AbstractMapListener<Integer, Integer>() {
@Override
public void update(Integer key, Integer oldValue, Integer newValue) {
// System.out.println(key + " " + oldValue + " => " + newValue);
iai.incrementAndGet();
}
};
ints2.addListener(intsListener);
dataStore2.start();
int count = 0;
for (int j = 0; j < 1000; j++) {
int collectionSize = 1000;
for (int i = 0; i < collectionSize; i++) {
ints.put(i, i + j);
strings.put(Integer.toString(i), Integer.toString(i + j));
}
size += Math.min(strings.size(), ints.size());
for (int i = 0; i < collectionSize; i++) {
ints.remove(i);
strings.remove(Integer.toString(i));
}
count += 4 * collectionSize;
}
long mid = System.nanoTime();
// int timeout = 0;
while (dataStore2.events() < count) {
// if (timeout++ % 10000 == 0)
// System.out.println(dataStore2.events());
dataStore2.nextEvent();
}
chronicle.close();
chronicle2.close();
long end = System.nanoTime();
System.out.printf(
"Startup and write took %.2f us on average and read and shutdown took %.2f on average%n",
(mid - start) / count / 1e3, (end - mid) / count / 1e3);
}
@Test
public void testOverTcpPutAllClear() throws IOException, InterruptedException {
String name = TMP + "/testOverTcpPutAllClear0";
String name2 = TMP + "/testOverTcpPutAllClear2";
ChronicleTools.deleteOnExit(name);
ChronicleTools.deleteOnExit(name2);
long start = System.nanoTime();
int PORT = 12345;
int size = 0;
InProcessChronicleSource chronicle =
new InProcessChronicleSource(new IndexedChronicle(name), PORT);
DataStore dataStore = new DataStore(chronicle, ModelMode.MASTER);
MapWrapper<String, String> strings =
new MapWrapper<String, String>(
dataStore,
"strings",
String.class,
String.class,
new LinkedHashMap<String, String>(),
16);
MapWrapper<Integer, Integer> ints =
new MapWrapper<Integer, Integer>(
dataStore,
"ints",
Integer.class,
Integer.class,
new LinkedHashMap<Integer, Integer>(),
16);
dataStore.start();
ints.clear();
strings.clear();
InProcessChronicleSink chronicle2 =
new InProcessChronicleSink(new IndexedChronicle(name2), "localhost", PORT);
DataStore dataStore2 = new DataStore(chronicle2, ModelMode.READ_ONLY);
MapWrapper<String, String> strings2 =
new MapWrapper<String, String>(
dataStore2,
"strings",
String.class,
String.class,
new LinkedHashMap<String, String>(),
16);
MapWrapper<Integer, Integer> ints2 =
new MapWrapper<Integer, Integer>(
dataStore2,
"ints",
Integer.class,
Integer.class,
new LinkedHashMap<Integer, Integer>(),
16);
final AtomicInteger sai = new AtomicInteger();
MapListener<String, String> stringsListener =
new AbstractMapListener<String, String>() {
@Override
public void update(String key, String oldValue, String newValue) {
// System.out.println(key + " " + oldValue + " => " + newValue);
sai.incrementAndGet();
}
};
strings2.addListener(stringsListener);
final AtomicInteger iai = new AtomicInteger();
MapListener<Integer, Integer> intsListener =
new AbstractMapListener<Integer, Integer>() {
@Override
public void update(Integer key, Integer oldValue, Integer newValue) {
// System.out.println(key + " " + oldValue + " => " + newValue);
iai.incrementAndGet();
}
};
ints2.addListener(intsListener);
dataStore2.start();
Map<String, String> ssMap = new LinkedHashMap<String, String>();
Map<Integer, Integer> iiMap = new LinkedHashMap<Integer, Integer>();
int count = 0;
int collectionSize = 2000;
for (int i = 0; i < collectionSize; i++) {
iiMap.put(i, i);
ssMap.put(Integer.toString(i), Integer.toString(i));
}
for (int j = 0; j < 2500; j++) {
strings.putAll(ssMap);
ints.putAll(iiMap);
strings.clear();
ints.clear();
count += 4;
}
long mid = System.nanoTime();
// int timeout = 0;
while (dataStore2.events() < count) {
// if (timeout++ % 10000 == 0)
// System.out.println(dataStore2.events());
dataStore2.nextEvent();
}
chronicle.close();
chronicle2.close();
long end = System.nanoTime();
System.out.printf(
"Startup and write took %.2f us on average (per key) and read and shutdown took %.2f on average (per key)%n",
(mid - start) / count / collectionSize / 1e3, (end - mid) / count / collectionSize / 1e3);
}
@Test
public void testOverTcpGetPerf() throws IOException, InterruptedException {
String name = TMP + "/testOverTcpGetPerf0";
String name2 = TMP + "/testOverTcpGetPerf2";
ChronicleTools.deleteOnExit(name);
ChronicleTools.deleteOnExit(name2);
long start = System.nanoTime();
int PORT = 12345;
int size = 0;
InProcessChronicleSource chronicle =
new InProcessChronicleSource(new IndexedChronicle(name), PORT);
DataStore dataStore = new DataStore(chronicle, ModelMode.MASTER);
MapWrapper<String, String> strings =
new MapWrapper<String, String>(
dataStore,
"strings",
String.class,
String.class,
new LinkedHashMap<String, String>(),
16);
MapWrapper<Integer, Integer> ints =
new MapWrapper<Integer, Integer>(
dataStore,
"ints",
Integer.class,
Integer.class,
new LinkedHashMap<Integer, Integer>(),
16);
dataStore.start();
ints.clear();
strings.clear();
InProcessChronicleSink chronicle2 =
new InProcessChronicleSink(new IndexedChronicle(name2), "localhost", PORT);
DataStore dataStore2 = new DataStore(chronicle2, ModelMode.READ_ONLY);
MapWrapper<String, String> strings2 =
new MapWrapper<String, String>(
dataStore2,
"strings",
String.class,
String.class,
new LinkedHashMap<String, String>(),
16);
MapWrapper<Integer, Integer> ints2 =
new MapWrapper<Integer, Integer>(
dataStore2,
"ints",
Integer.class,
Integer.class,
new LinkedHashMap<Integer, Integer>(),
16);
final AtomicInteger sai = new AtomicInteger();
MapListener<String, String> stringsListener =
new AbstractMapListener<String, String>() {
@Override
public void update(String key, String oldValue, String newValue) {
// System.out.println(key + " " + oldValue + " => " + newValue);
sai.incrementAndGet();
}
};
strings2.addListener(stringsListener);
final AtomicInteger iai = new AtomicInteger();
MapListener<Integer, Integer> intsListener =
new AbstractMapListener<Integer, Integer>() {
@Override
public void update(Integer key, Integer oldValue, Integer newValue) {
// System.out.println(key + " " + oldValue + " => " + newValue);
iai.incrementAndGet();
}
};
ints2.addListener(intsListener);
dataStore2.start();
Map<String, String> ssMap = new LinkedHashMap<String, String>();
Map<Integer, Integer> iiMap = new LinkedHashMap<Integer, Integer>();
int count = 2; // one clear per collection
int collectionSize = 2000;
for (int i = 0; i < collectionSize; i++) {
iiMap.put(i, i);
ssMap.put(Integer.toString(i), Integer.toString(i));
}
strings.putAll(ssMap);
ints.putAll(iiMap);
count += 2;
// int timeout = 0;
while (dataStore2.events() < count) {
// if (timeout++ % 10000 == 0)
// System.out.println(dataStore2.events());
dataStore2.nextEvent();
}
assertEquals(collectionSize, strings.size());
assertEquals(collectionSize, strings2.size());
assertEquals(collectionSize, ints.size());
assertEquals(collectionSize, ints2.size());
int gets = 0;
for (int j = 0; j < 50000; j++) {
for (String s : ssMap.keySet()) {
String s1 = strings.get(s);
String s2 = strings2.get(s);
if (s1 == null) assertNotNull(s1);
if (!s1.equals(s2)) assertEquals(s1, s2);
}
gets += ssMap.size();
for (Integer i : iiMap.keySet()) {
Integer i1 = ints.get(i);
Integer i2 = ints2.get(i);
if (i1 == null) assertNotNull(i1);
if (!i1.equals(i2)) assertEquals(i1, i2);
}
gets += iiMap.size();
}
chronicle.close();
chronicle2.close();
long end = System.nanoTime();
System.out.printf(
"Average get time including startup, bootstrap and shutdown, took %.3f us average per key%n",
(end - start) / gets / 1e3);
}
}
