Ejemplo n.º 1
0
  @BeforeMethod
  public void setUp() throws Exception {
    // do cleanup of other tests that may have started writers.
    // this is necessary in a non-forked test suite environment
    // where partition ids could collide
    Utils.finishBackupWriters();

    m_tmpDir = new File(System.getProperty("java.io.tmpdir"), "PartitionStoreStageTest");
    if (!m_tmpDir.exists()) {
      assertTrue("Failed to make the directories.", m_tmpDir.mkdirs());
    }

    // Add a dummy partition
    String server = "test";
    String storage = m_tmpDir + "/storage";

    File storageDir = new File(storage);
    if (!storageDir.exists()) {
      assertTrue("Failed to make the storage directory.", storageDir.mkdirs());
      if (!storageDir.exists()) {
        s_logger.warn("Storage Directory " + storage + " does not exist.");
      }
    }

    String mapped = m_tmpDir + "/mapped";

    File mappedDir = new File(mapped);
    if (!mappedDir.exists()) {
      assertTrue("Failed to make the mapped directories.", mappedDir.mkdirs());
      if (!mappedDir.exists()) {
        s_logger.warn("Storage Directory " + mapped + " does not exist.");
      }
    }

    if (m_part == null) {
      IPartitionManager pm = ManagementContainer.getInstance().getPartitionManager();
      m_part = pm.initializePartition(TEST_PID, server, storage, mapped);
    }
    Utils.startBackupWriters();
  }
Ejemplo n.º 2
0
/** @author peter.lawrey */
public class MapWrapperTest {
  static final String TMP = System.getProperty("java.io.tmpdir");

  @Test
  public void testMethods() throws IOException {
    String name = TMP + "/set-methods";
    ChronicleTools.deleteOnExit(name);
    {
      MapListener stringsListener = createMock("strings", MapListener.class);
      stringsListener.eventStart(1, "strings");
      stringsListener.add("Hello", "hi");
      stringsListener.eventEnd(true);

      stringsListener.eventStart(3, "strings");
      stringsListener.add("World", "all");
      stringsListener.eventEnd(true);

      MapListener intListener = createMock("ints", MapListener.class);
      for (int i = 0; i < 3; i++) {
        intListener.eventStart(i * 2, "ints");
        intListener.add(i, i + 1000);
        intListener.eventEnd(true);
      }

      replay(stringsListener);
      replay(intListener);
      Chronicle chronicle = new IndexedChronicle(name);
      DataStore dataStore = new DataStore(chronicle, ModelMode.MASTER);
      MapWrapper<String, String> strings =
          new MapWrapper<String, String>(
              dataStore,
              "strings",
              String.class,
              String.class,
              new LinkedHashMap<String, String>(),
              16);
      strings.addListener(stringsListener);
      MapWrapper<Integer, Integer> ints =
          new MapWrapper<Integer, Integer>(
              dataStore,
              "ints",
              Integer.class,
              Integer.class,
              new LinkedHashMap<Integer, Integer>(),
              16);
      ints.addListener(intListener);

      dataStore.start();

      ints.put(0, 1000);
      strings.put("Hello", "hi");
      ints.put(1, 1001);
      strings.put("World", "all");
      ints.put(2, 1002);

      verify(stringsListener);
      verify(intListener);

      assertEquals("{Hello=hi, World=all}", strings.toString());
      assertEquals("{0=1000, 1=1001, 2=1002}", ints.toString());

      chronicle.close();
    }
    {
      MapListener stringsListener = createMock("strings", MapListener.class);
      stringsListener.eventStart(5, "strings");
      stringsListener.add("!", "end");
      stringsListener.eventEnd(true);

      MapListener intListener = createMock("ints", MapListener.class);

      intListener.eventStart(6, "ints");
      intListener.add(3, 1003);
      intListener.eventEnd(true);

      replay(stringsListener);
      replay(intListener);

      Chronicle chronicle = new IndexedChronicle(name);
      DataStore dataStore = new DataStore(chronicle, ModelMode.MASTER);
      MapWrapper<String, String> strings =
          new MapWrapper<String, String>(
              dataStore,
              "strings",
              String.class,
              String.class,
              new LinkedHashMap<String, String>(),
              16);
      strings.addListener(stringsListener);
      MapWrapper<Integer, Integer> ints =
          new MapWrapper<Integer, Integer>(
              dataStore,
              "ints",
              Integer.class,
              Integer.class,
              new LinkedHashMap<Integer, Integer>(),
              16);
      ints.addListener(intListener);

      // assume we have  all the events written so far
      dataStore.start(chronicle.size());

      strings.put("!", "end");
      ints.put(3, 1003);

      verify(stringsListener);
      verify(intListener);

      assertEquals("{Hello=hi, World=all, !=end}", strings.toString());
      assertEquals("{0=1000, 1=1001, 2=1002, 3=1003}", ints.toString());
      chronicle.close();
    }
  }

  @Test
  public void testMapPerformance() throws IOException {
    String name = TMP + "/map-perf";
    ChronicleTools.deleteOnExit(name);
    long start = System.nanoTime();
    int size = 0;
    {
      Chronicle chronicle = new IndexedChronicle(name);
      DataStore dataStore = new DataStore(chronicle, ModelMode.MASTER);
      MapWrapper<String, String> strings =
          new MapWrapper<String, String>(
              dataStore,
              "strings",
              String.class,
              String.class,
              new LinkedHashMap<String, String>(),
              16);
      MapWrapper<Integer, Integer> ints =
          new MapWrapper<Integer, Integer>(
              dataStore,
              "ints",
              Integer.class,
              Integer.class,
              new LinkedHashMap<Integer, Integer>(),
              16);
      dataStore.start();
      ints.clear();
      strings.clear();

      for (int j = 0; j < 10000; j++) {
        for (int i = 0; i < 100; i++) {
          ints.put(i, i + j);
          strings.put(Integer.toString(i), Integer.toString(i + j));
        }
        size += Math.min(strings.size(), ints.size());
        for (int i = 0; i < 100; i++) {
          ints.remove(i);
          strings.remove(Integer.toString(i));
        }
      }

      chronicle.close();
    }
    long mid = System.nanoTime();
    {
      Chronicle chronicle = new IndexedChronicle(name);
      DataStore dataStore = new DataStore(chronicle, ModelMode.MASTER);
      MapWrapper<String, String> strings =
          new MapWrapper<String, String>(
              dataStore,
              "strings",
              String.class,
              String.class,
              new LinkedHashMap<String, String>(),
              16);
      MapWrapper<Integer, Integer> ints =
          new MapWrapper<Integer, Integer>(
              dataStore,
              "ints",
              Integer.class,
              Integer.class,
              new LinkedHashMap<Integer, Integer>(),
              16);
      dataStore.start();
      chronicle.close();
    }
    long end = System.nanoTime();
    System.out.printf(
        "Took %.1f seconds avg to add&remove %,d elements and %.1f seconds avg to reload them%n",
        (mid - start) / 2e9, size, (end - mid) / 2e9);
  }

  @Test
  public void testOverTcp() throws IOException, InterruptedException {
    String name = TMP + "/testOverTcp0";
    String name2 = TMP + "/testOverTcp2";
    ChronicleTools.deleteOnExit(name);
    ChronicleTools.deleteOnExit(name2);

    long start = System.nanoTime();
    int PORT = 12345;
    int size = 0;

    InProcessChronicleSource chronicle =
        new InProcessChronicleSource(new IndexedChronicle(name), PORT);
    DataStore dataStore = new DataStore(chronicle, ModelMode.MASTER);
    MapWrapper<String, String> strings =
        new MapWrapper<String, String>(
            dataStore,
            "strings",
            String.class,
            String.class,
            new LinkedHashMap<String, String>(),
            16);
    MapWrapper<Integer, Integer> ints =
        new MapWrapper<Integer, Integer>(
            dataStore,
            "ints",
            Integer.class,
            Integer.class,
            new LinkedHashMap<Integer, Integer>(),
            16);
    dataStore.start();
    ints.clear();
    strings.clear();

    InProcessChronicleSink chronicle2 =
        new InProcessChronicleSink(new IndexedChronicle(name2), "localhost", PORT);
    DataStore dataStore2 = new DataStore(chronicle2, ModelMode.READ_ONLY);
    MapWrapper<String, String> strings2 =
        new MapWrapper<String, String>(
            dataStore2,
            "strings",
            String.class,
            String.class,
            new LinkedHashMap<String, String>(),
            16);
    MapWrapper<Integer, Integer> ints2 =
        new MapWrapper<Integer, Integer>(
            dataStore2,
            "ints",
            Integer.class,
            Integer.class,
            new LinkedHashMap<Integer, Integer>(),
            16);

    final AtomicInteger sai = new AtomicInteger();
    MapListener<String, String> stringsListener =
        new AbstractMapListener<String, String>() {
          @Override
          public void update(String key, String oldValue, String newValue) {
            //                System.out.println(key + " " + oldValue + " => " + newValue);
            sai.incrementAndGet();
          }
        };
    strings2.addListener(stringsListener);

    final AtomicInteger iai = new AtomicInteger();
    MapListener<Integer, Integer> intsListener =
        new AbstractMapListener<Integer, Integer>() {
          @Override
          public void update(Integer key, Integer oldValue, Integer newValue) {
            //                System.out.println(key + " " + oldValue + " => " + newValue);
            iai.incrementAndGet();
          }
        };
    ints2.addListener(intsListener);
    dataStore2.start();

    int count = 0;
    for (int j = 0; j < 1000; j++) {
      int collectionSize = 1000;
      for (int i = 0; i < collectionSize; i++) {
        ints.put(i, i + j);
        strings.put(Integer.toString(i), Integer.toString(i + j));
      }
      size += Math.min(strings.size(), ints.size());
      for (int i = 0; i < collectionSize; i++) {
        ints.remove(i);
        strings.remove(Integer.toString(i));
      }
      count += 4 * collectionSize;
    }
    long mid = System.nanoTime();

    //        int timeout = 0;
    while (dataStore2.events() < count) {
      //            if (timeout++ % 10000 == 0)
      //                System.out.println(dataStore2.events());
      dataStore2.nextEvent();
    }

    chronicle.close();
    chronicle2.close();
    long end = System.nanoTime();

    System.out.printf(
        "Startup and write took %.2f us on average and read and shutdown took %.2f on average%n",
        (mid - start) / count / 1e3, (end - mid) / count / 1e3);
  }

  @Test
  public void testOverTcpPutAllClear() throws IOException, InterruptedException {
    String name = TMP + "/testOverTcpPutAllClear0";
    String name2 = TMP + "/testOverTcpPutAllClear2";
    ChronicleTools.deleteOnExit(name);
    ChronicleTools.deleteOnExit(name2);

    long start = System.nanoTime();
    int PORT = 12345;
    int size = 0;

    InProcessChronicleSource chronicle =
        new InProcessChronicleSource(new IndexedChronicle(name), PORT);
    DataStore dataStore = new DataStore(chronicle, ModelMode.MASTER);
    MapWrapper<String, String> strings =
        new MapWrapper<String, String>(
            dataStore,
            "strings",
            String.class,
            String.class,
            new LinkedHashMap<String, String>(),
            16);
    MapWrapper<Integer, Integer> ints =
        new MapWrapper<Integer, Integer>(
            dataStore,
            "ints",
            Integer.class,
            Integer.class,
            new LinkedHashMap<Integer, Integer>(),
            16);
    dataStore.start();
    ints.clear();
    strings.clear();

    InProcessChronicleSink chronicle2 =
        new InProcessChronicleSink(new IndexedChronicle(name2), "localhost", PORT);
    DataStore dataStore2 = new DataStore(chronicle2, ModelMode.READ_ONLY);
    MapWrapper<String, String> strings2 =
        new MapWrapper<String, String>(
            dataStore2,
            "strings",
            String.class,
            String.class,
            new LinkedHashMap<String, String>(),
            16);
    MapWrapper<Integer, Integer> ints2 =
        new MapWrapper<Integer, Integer>(
            dataStore2,
            "ints",
            Integer.class,
            Integer.class,
            new LinkedHashMap<Integer, Integer>(),
            16);

    final AtomicInteger sai = new AtomicInteger();
    MapListener<String, String> stringsListener =
        new AbstractMapListener<String, String>() {
          @Override
          public void update(String key, String oldValue, String newValue) {
            //                System.out.println(key + " " + oldValue + " => " + newValue);
            sai.incrementAndGet();
          }
        };
    strings2.addListener(stringsListener);

    final AtomicInteger iai = new AtomicInteger();
    MapListener<Integer, Integer> intsListener =
        new AbstractMapListener<Integer, Integer>() {
          @Override
          public void update(Integer key, Integer oldValue, Integer newValue) {
            //                System.out.println(key + " " + oldValue + " => " + newValue);
            iai.incrementAndGet();
          }
        };
    ints2.addListener(intsListener);
    dataStore2.start();

    Map<String, String> ssMap = new LinkedHashMap<String, String>();
    Map<Integer, Integer> iiMap = new LinkedHashMap<Integer, Integer>();
    int count = 0;
    int collectionSize = 2000;
    for (int i = 0; i < collectionSize; i++) {
      iiMap.put(i, i);
      ssMap.put(Integer.toString(i), Integer.toString(i));
    }
    for (int j = 0; j < 2500; j++) {
      strings.putAll(ssMap);
      ints.putAll(iiMap);
      strings.clear();
      ints.clear();
      count += 4;
    }
    long mid = System.nanoTime();

    //        int timeout = 0;
    while (dataStore2.events() < count) {
      //            if (timeout++ % 10000 == 0)
      //                System.out.println(dataStore2.events());
      dataStore2.nextEvent();
    }

    chronicle.close();
    chronicle2.close();
    long end = System.nanoTime();

    System.out.printf(
        "Startup and write took %.2f us on average (per key) and read and shutdown took %.2f on average (per key)%n",
        (mid - start) / count / collectionSize / 1e3, (end - mid) / count / collectionSize / 1e3);
  }

  @Test
  public void testOverTcpGetPerf() throws IOException, InterruptedException {
    String name = TMP + "/testOverTcpGetPerf0";
    String name2 = TMP + "/testOverTcpGetPerf2";
    ChronicleTools.deleteOnExit(name);
    ChronicleTools.deleteOnExit(name2);

    long start = System.nanoTime();
    int PORT = 12345;
    int size = 0;

    InProcessChronicleSource chronicle =
        new InProcessChronicleSource(new IndexedChronicle(name), PORT);
    DataStore dataStore = new DataStore(chronicle, ModelMode.MASTER);
    MapWrapper<String, String> strings =
        new MapWrapper<String, String>(
            dataStore,
            "strings",
            String.class,
            String.class,
            new LinkedHashMap<String, String>(),
            16);
    MapWrapper<Integer, Integer> ints =
        new MapWrapper<Integer, Integer>(
            dataStore,
            "ints",
            Integer.class,
            Integer.class,
            new LinkedHashMap<Integer, Integer>(),
            16);
    dataStore.start();
    ints.clear();
    strings.clear();

    InProcessChronicleSink chronicle2 =
        new InProcessChronicleSink(new IndexedChronicle(name2), "localhost", PORT);
    DataStore dataStore2 = new DataStore(chronicle2, ModelMode.READ_ONLY);
    MapWrapper<String, String> strings2 =
        new MapWrapper<String, String>(
            dataStore2,
            "strings",
            String.class,
            String.class,
            new LinkedHashMap<String, String>(),
            16);
    MapWrapper<Integer, Integer> ints2 =
        new MapWrapper<Integer, Integer>(
            dataStore2,
            "ints",
            Integer.class,
            Integer.class,
            new LinkedHashMap<Integer, Integer>(),
            16);

    final AtomicInteger sai = new AtomicInteger();
    MapListener<String, String> stringsListener =
        new AbstractMapListener<String, String>() {
          @Override
          public void update(String key, String oldValue, String newValue) {
            //                System.out.println(key + " " + oldValue + " => " + newValue);
            sai.incrementAndGet();
          }
        };
    strings2.addListener(stringsListener);

    final AtomicInteger iai = new AtomicInteger();
    MapListener<Integer, Integer> intsListener =
        new AbstractMapListener<Integer, Integer>() {
          @Override
          public void update(Integer key, Integer oldValue, Integer newValue) {
            //                System.out.println(key + " " + oldValue + " => " + newValue);
            iai.incrementAndGet();
          }
        };
    ints2.addListener(intsListener);
    dataStore2.start();

    Map<String, String> ssMap = new LinkedHashMap<String, String>();
    Map<Integer, Integer> iiMap = new LinkedHashMap<Integer, Integer>();
    int count = 2; // one clear per collection
    int collectionSize = 2000;
    for (int i = 0; i < collectionSize; i++) {
      iiMap.put(i, i);
      ssMap.put(Integer.toString(i), Integer.toString(i));
    }

    strings.putAll(ssMap);
    ints.putAll(iiMap);
    count += 2;

    //        int timeout = 0;
    while (dataStore2.events() < count) {
      //            if (timeout++ % 10000 == 0)
      //                System.out.println(dataStore2.events());
      dataStore2.nextEvent();
    }
    assertEquals(collectionSize, strings.size());
    assertEquals(collectionSize, strings2.size());
    assertEquals(collectionSize, ints.size());
    assertEquals(collectionSize, ints2.size());

    int gets = 0;
    for (int j = 0; j < 50000; j++) {
      for (String s : ssMap.keySet()) {
        String s1 = strings.get(s);
        String s2 = strings2.get(s);
        if (s1 == null) assertNotNull(s1);
        if (!s1.equals(s2)) assertEquals(s1, s2);
      }
      gets += ssMap.size();
      for (Integer i : iiMap.keySet()) {
        Integer i1 = ints.get(i);
        Integer i2 = ints2.get(i);
        if (i1 == null) assertNotNull(i1);
        if (!i1.equals(i2)) assertEquals(i1, i2);
      }
      gets += iiMap.size();
    }

    chronicle.close();
    chronicle2.close();
    long end = System.nanoTime();

    System.out.printf(
        "Average get time including startup, bootstrap and shutdown, took %.3f us average per key%n",
        (end - start) / gets / 1e3);
  }
}