Example #1
0
  protected Vector /* of TTPNode */ queryPNodes1(String query) throws IOException {
    write(query);
    Vector lines = readlines();
    Vector r = new Vector();
    for (Enumeration e = lines.elements(); e.hasMoreElements(); ) {
      Vector line = stringToVector((String) e.nextElement(), ":");

      TTLexEntry le = new TTLexEntry();
      le.citationForm = (String) line.elementAt(0);
      le.features = (String) line.elementAt(1);
      le.inflection = (String) line.elementAt(2);
      le.inflFeatures = (String) line.elementAt(3);

      TTLexEntryToObj leo = new TTLexEntryToObj();
      leo.lexentry = le;
      leo.objname = (String) line.elementAt(4);
      leo.features = (String) line.elementAt(5);

      TTPNode pn = new TTPNode();
      pn.feature = TT.featureGet(le.features, TT.FT_POS, TT.F_NULL);
      pn.leo = leo;
      pn.startpos = TT.longParse((String) line.elementAt(8), true);
      pn.endpos = TT.longParse((String) line.elementAt(9), true);

      r.addElement(pn);
    }
    return r;
  }
Example #2
0
  @Test
  public void par_update_get_compact() throws InterruptedException {
    int scale = TT.scale();
    if (scale == 0) return;
    int threadNum = Math.min(4, scale * 4);
    final long end = TT.nowPlusMinutes(10);
    e = openEngine();
    final BlockingQueue<Fun.Pair<Long, byte[]>> q = new ArrayBlockingQueue(threadNum * 10);
    for (int i = 0; i < threadNum; i++) {
      byte[] b = TT.randomByteArray(new Random().nextInt(10000));
      long recid = e.put(b, BYTE_ARRAY_NOSIZE);
      q.put(new Fun.Pair(recid, b));
    }

    final CountDownLatch l = new CountDownLatch(2);
    Thread tt =
        new Thread() {
          @Override
          public void run() {
            try {
              while (l.getCount() > 1) e.compact();
            } finally {
              l.countDown();
            }
          }
        };
    tt.setDaemon(true);
    tt.run();

    Exec.execNTimes(
        threadNum,
        new Callable() {
          @Override
          public Object call() throws Exception {
            Random r = new Random();
            while (System.currentTimeMillis() < end) {
              Fun.Pair<Long, byte[]> t = q.take();
              assertTrue(
                  Serializer.BYTE_ARRAY.equals(t.b, e.get(t.a, Serializer.BYTE_ARRAY_NOSIZE)));
              int size = r.nextInt(1000);
              if (r.nextInt(10) == 1) size = size * 100;
              byte[] b = TT.randomByteArray(size);
              e.update(t.a, b, Serializer.BYTE_ARRAY_NOSIZE);
              q.put(new Fun.Pair<Long, byte[]>(t.a, b));
            }
            return null;
          }
        });
    l.countDown();
    l.await();

    for (Fun.Pair<Long, byte[]> t : q) {
      assertTrue(Serializer.BYTE_ARRAY.equals(t.b, e.get(t.a, Serializer.BYTE_ARRAY_NOSIZE)));
    }
    e.close();
  }
Example #3
0
  @Test
  @org.junit.Ignore
  public void large_node_size() {
    for (int i : new int[] {10, 200, 6000}) {

      int max = i * 100;
      File f = TT.tempDbFile();
      DB db = DBMaker.fileDB(f).transactionDisable().make();
      Map m =
          db.treeMapCreate("map")
              .nodeSize(i)
              .keySerializer(BTreeKeySerializer.INTEGER)
              .valueSerializer(Serializer.INTEGER)
              .make();

      for (int j = 0; j < max; j++) {
        m.put(j, j);
      }

      db.close();
      db = DBMaker.fileDB(f).deleteFilesAfterClose().transactionDisable().make();
      m = db.treeMap("map");

      for (Integer j = 0; j < max; j++) {
        assertEquals(j, m.get(j));
      }
      db.close();
    }
  }
Example #4
0
  @Test
  public void concurrent_first_key() {
    DB db = DBMaker.memoryDB().transactionDisable().make();
    final BTreeMap m = db.treeMap("name");

    // fill
    final int c = 1000000 * TT.scale();
    for (int i = 0; i <= c; i++) {
      m.put(i, i);
    }

    Thread t =
        new Thread() {
          @Override
          public void run() {
            for (int i = 0; i <= c; i++) {
              m.remove(c);
            }
          }
        };
    t.run();
    while (t.isAlive()) {
      assertNotNull(m.firstKey());
    }
  }
Example #5
0
  @Test
  public void open_ignores_rollback() {
    File f = TT.tempDbFile();
    WriteAheadLog wal = new WriteAheadLog(f.getPath());
    wal.walPutLong(1L, 11L);
    wal.commit();
    wal.walPutLong(2L, 33L);
    wal.rollback();
    wal.walPutLong(3L, 33L);
    wal.commit();
    wal.seal();
    wal.close();

    wal = new WriteAheadLog(f.getPath());
    wal.open(
        new WALSequence(
            new Object[] {WALSequence.beforeReplayStart},
            new Object[] {WALSequence.writeLong, 1L, 11L},
            new Object[] {WALSequence.commit},
            // 2L is ignored, rollback section is skipped on hard replay
            new Object[] {WALSequence.writeLong, 3L, 33L},
            new Object[] {WALSequence.commit}));
    wal.destroyWalFiles();
    wal.close();

    f.delete();
  }
Example #6
0
  @Test
  public void empty_update_commit() {
    if (TT.scale() == 0) return;

    e = openEngine();
    long recid = e.put("", Serializer.STRING_NOSIZE);
    assertEquals("", e.get(recid, Serializer.STRING_NOSIZE));

    for (int i = 0; i < 10000; i++) {
      String s = TT.randomString(80000);
      e.update(recid, s, Serializer.STRING_NOSIZE);
      assertEquals(s, e.get(recid, Serializer.STRING_NOSIZE));
      e.commit();
      assertEquals(s, e.get(recid, Serializer.STRING_NOSIZE));
    }
    e.close();
  }
Example #7
0
 protected Vector /* of Object */ queryAssertionLines1(String query) throws IOException {
   write(query);
   Vector v = readlines();
   Vector r = new Vector();
   for (Enumeration e = v.elements(); e.hasMoreElements(); ) {
     r.addElement(TT.stringToObject((String) e.nextElement()));
   }
   return r;
 }
  @Test
  public void string_formats_compatible_equal_size() throws IOException {
    ArrayList keys = new ArrayList();
    for (int i = 0; i < 1000; i++) {
      keys.add("common prefix " + TT.randomString(10, 0));
    }

    checkStringSerializers(keys);
  }
  @Test
  public void string_formats_compatible_no_prefix() throws IOException {
    ArrayList keys = new ArrayList();
    for (int i = 0; i < 1000; i++) {
      keys.add(TT.randomString(10 + new Random().nextInt(100), 0));
    }

    checkStringSerializers(keys);
  }
Example #10
0
 @Test
 public void compact_large_record() {
   e = openEngine();
   byte[] b = TT.randomByteArray(100000);
   long recid = e.put(b, Serializer.BYTE_ARRAY_NOSIZE);
   e.commit();
   e.compact();
   assertTrue(Serializer.BYTE_ARRAY.equals(b, e.get(recid, Serializer.BYTE_ARRAY_NOSIZE)));
   e.close();
 }
Example #11
0
  @Test
  public void replay_commit_over_file_edge() {
    String f = TT.tempDbFile().getPath();
    WriteAheadLog wal = new WriteAheadLog(f);

    byte[] b = TT.randomByteArray(20 * 1024 * 1024);
    wal.walPutRecord(11L, b, 0, b.length);
    wal.walPutRecord(33L, b, 0, b.length);
    wal.commit();
    wal.close();

    wal = new WriteAheadLog(f);
    wal.open(
        new WALSequence(
            new Object[] {WALSequence.beforeReplayStart},
            new Object[] {WALSequence.writeRecord, 11L, 16L, b},
            new Object[] {WALSequence.writeRecord, 33L, 4294967312L, b},
            new Object[] {WALSequence.commit}));
  }
Example #12
0
  @Test
  public void test_dir_node_serialization() throws IOException {

    BTreeMap.DirNode n =
        new BTreeMap.DirNode(new Object[] {1, 2, 3}, false, true, false, mkchild(4, 5, 6, 0));
    BTreeMap.DirNode n2 = (BTreeMap.DirNode) TT.clone(n, m.nodeSerializer);

    assertTrue(Arrays.equals(nodeKeysToArray(n), nodeKeysToArray(n2)));
    assertTrue(Arrays.equals((int[]) n.child, (int[]) n2.child));
  }
Example #13
0
  @Test
  public void empty_commit() {
    String f = TT.tempDbFile().getPath();
    WriteAheadLog wal = new WriteAheadLog(f);

    byte[] b = TT.randomByteArray(1024);
    wal.walPutRecord(33L, b, 0, b.length);
    wal.commit();
    wal.commit();
    wal.seal();
    wal.close();

    wal = new WriteAheadLog(f);
    wal.open(
        new WALSequence(
            new Object[] {WALSequence.beforeReplayStart},
            new Object[] {WALSequence.writeRecord, 33L, 16L, b},
            new Object[] {WALSequence.commit},
            new Object[] {WALSequence.commit}));
  }
Example #14
0
  @Test
  public void test_leaf_node_serialization() throws IOException {

    if (valsOutside) return;

    BTreeMap.LeafNode n =
        new BTreeMap.LeafNode(new Object[] {1, 2, 3}, true, true, false, new Object[] {1, 2, 3}, 0);
    BTreeMap.LeafNode n2 = (BTreeMap.LeafNode) TT.clone(n, m.nodeSerializer);
    assertTrue(Arrays.equals(nodeKeysToArray(n), nodeKeysToArray(n2)));
    assertEquals(n.next, n2.next);
  }
Example #15
0
  @Parameterized.Parameters
  public static List<Object[]> params() throws IOException {
    List ret = new ArrayList();
    int inc = TT.shortTest() ? 200 : 20;

    for (int commitNum = 1; commitNum < 1000; commitNum += inc) {
      for (int cutPointSeed = 0; cutPointSeed < 600; cutPointSeed += inc) {
        ret.add(new Object[] {commitNum, cutPointSeed});
      }
    }

    return ret;
  }
Example #16
0
 @Test
 public void update_reserved_recid_large() {
   e = openEngine();
   byte[] data = TT.randomByteArray((int) 1e7);
   e.update(Engine.RECID_NAME_CATALOG, data, Serializer.BYTE_ARRAY_NOSIZE);
   assertTrue(
       Serializer.BYTE_ARRAY.equals(
           data, e.get(Engine.RECID_NAME_CATALOG, Serializer.BYTE_ARRAY_NOSIZE)));
   e.commit();
   assertTrue(
       Serializer.BYTE_ARRAY.equals(
           data, e.get(Engine.RECID_NAME_CATALOG, Serializer.BYTE_ARRAY_NOSIZE)));
   e.close();
 }
Example #17
0
  @Test
  public void lazy_file_create() {
    File f = TT.tempDbFile();
    f.delete();
    File f2 = new File(f.getPath() + ".wal.0");
    WriteAheadLog wal = new WriteAheadLog(f.getPath());
    wal.open(WriteAheadLog.NOREPLAY);

    assertTrue(!f2.exists());
    wal.walPutLong(111L, 111L);
    assertTrue(f2.exists());
    wal.close();
    f2.delete();
  }
Example #18
0
  @Test
  public void issue_38() {
    int max = 50000 * TT.scale();
    Map<Integer, String[]> map = DBMaker.memoryDB().transactionDisable().make().treeMap("test");

    for (int i = 0; i < max; i++) {
      map.put(i, new String[5]);
    }

    for (int i = 0; i < max; i = i + 1000) {
      assertTrue(Arrays.equals(new String[5], map.get(i)));
      assertTrue(map.get(i).toString().contains("[Ljava.lang.String"));
    }
  }
Example #19
0
  @Test
  public void par_cas() throws InterruptedException {
    int scale = TT.scale();
    if (scale == 0) return;
    int threadNum = 8 * scale;
    final long end = TT.nowPlusMinutes(10);
    e = openEngine();
    final BlockingQueue<Fun.Pair<Long, byte[]>> q = new ArrayBlockingQueue(threadNum * 10);
    for (int i = 0; i < threadNum; i++) {
      byte[] b = TT.randomByteArray(new Random().nextInt(10000));
      long recid = e.put(b, BYTE_ARRAY_NOSIZE);
      q.put(new Fun.Pair(recid, b));
    }

    Exec.execNTimes(
        threadNum,
        new Callable() {
          @Override
          public Object call() throws Exception {
            Random r = new Random();
            while (System.currentTimeMillis() < end) {
              Fun.Pair<Long, byte[]> t = q.take();
              int size = r.nextInt(10000);
              if (r.nextInt(10) == 1) size = size * 100;
              byte[] b = TT.randomByteArray(size);
              assertTrue(e.compareAndSwap(t.a, t.b, b, Serializer.BYTE_ARRAY_NOSIZE));
              q.put(new Fun.Pair<Long, byte[]>(t.a, b));
            }
            return null;
          }
        });

    for (Fun.Pair<Long, byte[]> t : q) {
      assertTrue(Serializer.BYTE_ARRAY.equals(t.b, e.get(t.a, Serializer.BYTE_ARRAY_NOSIZE)));
    }
    e.close();
  }
Example #20
0
  @Test
  public void compact_double_recid_reuse() {
    e = openEngine();
    if (e instanceof StoreAppend) return; // TODO reenable once StoreAppend has compaction
    long recid1 = e.put("aa", Serializer.STRING);
    long recid2 = e.put("bb", Serializer.STRING);
    e.compact();
    e.delete(recid1, Serializer.STRING);
    e.compact();
    e.delete(recid2, Serializer.STRING);
    e.compact();

    TT.sortAndEquals(new long[] {recid1, recid2}, new long[] {e.preallocate(), e.preallocate()});

    e.close();
  }
Example #21
0
  @Test
  public void commit_huge() {
    if (TT.shortTest()) return;
    e = openEngine();
    long recid = e.put(new byte[1000 * 1000 * 1000], Serializer.BYTE_ARRAY_NOSIZE);
    e.commit();

    reopen();

    byte[] b = e.get(recid, Serializer.BYTE_ARRAY_NOSIZE);
    assertEquals(1000 * 1000 * 1000, b.length);
    for (byte bb : b) {
      assertEquals(0, bb);
    }
    e.close();
  }
Example #22
0
  @Test
  public void overflow_record() {
    File f = TT.tempDbFile();
    f.delete();
    File f0 = new File(f.getPath() + ".wal.0");
    File f1 = new File(f.getPath() + ".wal.1");
    WriteAheadLog wal = new WriteAheadLog(f.getPath());
    wal.open(WriteAheadLog.NOREPLAY);

    long lastPos = 0;
    while (!f1.exists()) {
      lastPos = wal.fileOffset;
      wal.walPutRecord(111L, new byte[100], 0, 100);
      assertTrue(f0.exists());
    }
    assertTrue(WriteAheadLog.MAX_FILE_SIZE - 1000 < lastPos);
    assertTrue(WriteAheadLog.MAX_FILE_SIZE + 120 > lastPos);
    wal.destroyWalFiles();
  }
Example #23
0
  @Test
  public void randomStructuralCheck() {
    Random r = new Random();
    BTreeMap map =
        DBMaker.memoryDB()
            .transactionDisable()
            .make()
            .treeMapCreate("aa")
            .keySerializer(BTreeKeySerializer.INTEGER)
            .valueSerializer(Serializer.INTEGER)
            .make();

    int max = 100000 * TT.scale();

    for (int i = 0; i < max * 10; i++) {
      map.put(r.nextInt(max), r.nextInt());
    }

    map.checkStructure();
  }
  protected void notifyModelListenersOnUpdate(TT targetModel) throws SQLException {

    Map<String, Object> originalValues = new HashMap<String, Object>();

    TT sourceModel = queryForId(extractId(targetModel));

    for (String syncNotificationFieldName :
        getSyncNotificationFieldNames(dataClass.getSimpleName())) {

      if (syncNotificationFieldName.equals("")) {
        continue;
      }

      if (syncNotificationFieldName.equals("uiEvent")) {
        if (targetModel.getUiEvent() != BaseModel.UI_EVENT_DEFAULT) {
          originalValues.put("uiEvent", null);
        }
      } else {
        FieldType fieldType = tableInfo.getFieldTypeByColumnName(syncNotificationFieldName);

        Object sourceFieldValue = fieldType.extractJavaFieldValue(sourceModel);
        Object targetFieldValue = fieldType.extractJavaFieldValue(targetModel);

        DataPersister dataPersister = fieldType.getDataPersister();

        if (!dataPersister.dataIsEqual(sourceFieldValue, targetFieldValue)) {

          originalValues.put(fieldType.getColumnName(), sourceFieldValue);
        }
      }
    }

    if (originalValues.isEmpty()) {
      return;
    }

    for (ModelListener<TT> modelListener : _modelListeners) {
      modelListener.onUpdate(targetModel, originalValues);
    }
  }
Example #25
0
  @Test
  public void compact2() {
    long max = TT.scale() * 10000;
    e = openEngine();
    Map<Long, Long> recids = new HashMap<Long, Long>();
    for (Long l = 0L; l < max; l++) {
      recids.put(l, e.put(l, Serializer.LONG));
    }

    e.commit();
    e.compact();
    for (Long l = max; l < max * 2; l++) {
      recids.put(l, e.put(l, Serializer.LONG));
    }

    for (Map.Entry<Long, Long> m : recids.entrySet()) {
      Long recid = m.getValue();
      Long value = m.getKey();
      assertEquals(value, e.get(recid, Serializer.LONG));
    }
    e.close();
  }
Example #26
0
  @Test
  public void cut_broken_end_rollback() {
    String f = TT.tempDbFile().getPath();
    WriteAheadLog wal = new WriteAheadLog(f);
    wal.walPutLong(1L, 11L);
    wal.commit();
    wal.walPutLong(2L, 22L);
    wal.commit();
    wal.walPutLong(3L, 33L);
    wal.rollback();
    wal.walPutLong(4L, 44L);
    wal.curVol.sync();
    wal.close();

    wal = new WriteAheadLog(f);
    wal.open(
        new WALSequence(
            new Object[] {WALSequence.beforeReplayStart},
            new Object[] {WALSequence.writeLong, 1L, 11L},
            new Object[] {WALSequence.commit},
            new Object[] {WALSequence.writeLong, 2L, 22L},
            new Object[] {WALSequence.commit}));
  }
Example #27
0
  @Test
  public void recover_with_interrupt() throws InterruptedException {
    int scale = TT.scale();
    if (scale == 0) return;
    e = openEngine();
    if (!e.canRollback()
        || e instanceof StoreHeap) // TODO engine might have crash recovery, but no rollbacks
    return;

    // fill recids
    final int max = scale * 1000;
    final ArrayList<Long> recids = new ArrayList<Long>();
    final AtomicLong a = new AtomicLong(10);
    final long counterRecid = e.put(a.get(), Serializer.LONG);
    Random r = new Random(a.get());
    for (int j = 0; j < max; j++) {
      byte[] b = new byte[r.nextInt(100000)];
      r.nextBytes(b);
      long recid = e.put(b, Serializer.BYTE_ARRAY_NOSIZE);
      recids.add(recid);
    }

    e.commit();

    long endTime = TT.nowPlusMinutes(10);

    while (endTime > System.currentTimeMillis()) {

      final CountDownLatch latch = new CountDownLatch(1);
      Thread t =
          new Thread() {
            @Override
            public void run() {
              try {
                for (; ; ) {
                  long A = a.incrementAndGet();
                  Random r = new Random(A);
                  e.update(counterRecid, A, Serializer.LONG);

                  for (long recid : recids) {
                    byte[] b = new byte[r.nextInt(100000)];
                    r.nextBytes(b);
                    e.update(recid, b, Serializer.BYTE_ARRAY_NOSIZE);
                  }
                  e.commit();
                }
              } finally {
                latch.countDown();
              }
            }
          };
      t.start();
      Thread.sleep(5000);
      t.stop();
      latch.await();
      if (!e.isClosed()) {
        close();
      }

      // reopen and check the content
      e = openEngine();

      // check if A-1 was commited
      long A = e.get(counterRecid, Serializer.LONG);
      assertTrue("" + A + " - " + a.get(), A == a.get() || A == a.get() - 1);
      r = new Random(A);
      for (long recid : recids) {
        byte[] b = new byte[r.nextInt(100000)];
        r.nextBytes(b);
        byte[] b2 = e.get(recid, Serializer.BYTE_ARRAY_NOSIZE);
        assertTrue("Data were not commited recid=" + recid, Arrays.equals(b, b2));
      }
      a.set(A);
    }
    e.close();
  }
Example #28
0
 /**
  * Generates the specified ThoughtTreasure assertion in natural language.
  *
  * @param feat a String containing single-character ThoughtTreasure features such as <code>
  *     F_ENGLISH</code> or <code>F_FRENCH</code>. The feature codes are defined in the <code>TT
  *     </code> class.
  * @param obj Java Object representing a ThoughtTreasure assertion
  * @exception IOException if an I/O error occurs communicating with the server
  * @see TT
  */
 public String generate(String feat, Object obj) throws IOException {
   return queryString("Generate " + feat.trim() + " " + TT.objectToString(obj) + "\n");
 }
Example #29
0
 @Test
 public void large_record() {
   testRecord(11111L, TT.randomByteArray(1000000));
 }
Example #30
0
 @Test
 public void ten_record() {
   testRecord(11111L, TT.randomByteArray(10));
 }