Example #1
0
  /**
   * Evaluates the full-text match.
   *
   * @param qc query context
   * @return number of tokens, used for scoring
   * @throws QueryException query exception
   */
  private int contains(final QueryContext qc) throws QueryException {
    first = true;
    final FTLexer lexer = ftt.lexer(qc.ftToken);

    // use faster evaluation for default options
    int num = 0;
    if (fast) {
      for (final byte[] t : tokens) {
        final FTTokens qtok = ftt.cache(t);
        num = Math.max(num, ftt.contains(qtok, lexer) * qtok.length());
      }
      return num;
    }

    // find and count all occurrences
    final boolean all = mode == FTMode.ALL || mode == FTMode.ALL_WORDS;
    int oc = 0;
    for (final byte[] w : unique(tokens(qc))) {
      final FTTokens qtok = ftt.cache(w);
      final int o = ftt.contains(qtok, lexer);
      if (all && o == 0) return 0;
      num = Math.max(num, o * qtok.length());
      oc += o;
    }

    // check if occurrences are in valid range. if yes, return number of tokens
    final long mn = occ != null ? toLong(occ[0], qc) : 1;
    final long mx = occ != null ? toLong(occ[1], qc) : Long.MAX_VALUE;
    if (mn == 0 && oc == 0) matches = FTNot.not(matches);
    return oc >= mn && oc <= mx ? Math.max(1, num) : 0;
  }
Example #2
0
  @Override
  public boolean indexAccessible(final IndexInfo ii) {
    /* If the following conditions yield true, the index is accessed:
     * - all query terms are statically available
     * - no FTTimes option is specified
     * - explicitly set case, diacritics and stemming match options do not
     *   conflict with index options. */
    data = ii.ic.data;
    final MetaData md = data.meta;
    final FTOpt fto = ftt.opt;

    /* Index will be applied if no explicit match options have been set
     * that conflict with the index options. As a consequence, though, index-
     * based querying might yield other results than sequential scanning. */
    if (occ != null
        || fto.cs != null && md.casesens == (fto.cs == FTCase.INSENSITIVE)
        || fto.isSet(DC) && md.diacritics != fto.is(DC)
        || fto.isSet(ST) && md.stemming != fto.is(ST)
        || fto.ln != null && !fto.ln.equals(md.language)) return false;

    // adopt database options to tokenizer
    fto.copy(md);

    // estimate costs if text is not known at compile time
    if (tokens == null) {
      ii.costs = Math.max(2, data.meta.size / 30);
      return true;
    }

    // summarize number of hits; break loop if no hits are expected
    final FTLexer ft = new FTLexer(fto);
    ii.costs = 0;
    for (byte[] t : tokens) {
      ft.init(t);
      while (ft.hasNext()) {
        final byte[] tok = ft.nextToken();
        if (fto.sw != null && fto.sw.contains(tok)) continue;

        if (fto.is(WC)) {
          // don't use index if one of the terms starts with a wildcard
          t = ft.get();
          if (t[0] == '.') return false;
          // don't use index if certain characters or more than 1 dot are found
          int d = 0;
          for (final byte w : t) {
            if (w == '{' || w == '\\' || w == '.' && ++d > 1) return false;
          }
        }
        // favor full-text index requests over exact queries
        final int costs = data.costs(ft);
        if (costs != 0) ii.costs += Math.max(2, costs / 100);
      }
    }
    return true;
  }
Example #3
0
  @Override
  public DiskData build() throws IOException {
    meta.assign(parser);
    meta.dirty = true;

    // calculate optimized output buffer sizes to reduce disk fragmentation
    final Runtime rt = Runtime.getRuntime();
    final long max = Math.min(1 << 22, rt.maxMemory() - rt.freeMemory() >> 2);
    int bs = (int) Math.min(meta.filesize, max);
    bs = Math.max(IO.BLOCKSIZE, bs - bs % IO.BLOCKSIZE);

    // drop old database (if available) and create new one
    DropDB.drop(dbname, sopts);
    sopts.dbpath(dbname).md();

    elemNames = new Names(meta);
    attrNames = new Names(meta);
    try {
      tout = new DataOutput(new TableOutput(meta, DATATBL));
      xout = new DataOutput(meta.dbfile(DATATXT), bs);
      vout = new DataOutput(meta.dbfile(DATAATV), bs);
      sout = new DataOutput(meta.dbfile(DATATMP), bs);

      final Performance perf = Prop.debug ? new Performance() : null;
      Util.debug(tit() + DOTS);
      parse();
      if (Prop.debug) Util.errln(" " + perf + " (" + Performance.getMemory() + ')');

    } catch (final IOException ex) {
      try {
        close();
      } catch (final IOException ignored) {
      }
      throw ex;
    }
    close();

    // copy temporary values into database table
    try (final DataInput in = new DataInput(meta.dbfile(DATATMP))) {
      final TableAccess ta = new TableDiskAccess(meta, true);
      for (; spos < ssize; ++spos) ta.write4(in.readNum(), 8, in.readNum());
      ta.close();
    }
    meta.dbfile(DATATMP).delete();

    // return database instance
    return new DiskData(meta, elemNames, attrNames, path, ns);
  }
Example #4
0
 /**
  * Fill the current buffer with bytes from the specified array from the specified offset.
  *
  * @param s source array
  * @param o offset from the beginning of the array
  * @return number of written bytes
  */
 private int write(final byte[] s, final int o) {
   final Buffer bf = bm.current();
   final int len = Math.min(IO.BLOCKSIZE, s.length - o);
   System.arraycopy(s, o, bf.data, 0, len);
   bf.dirty = true;
   return len;
 }
Example #5
0
  /**
   * Performs a wildcard search for the specified token.
   *
   * @param token token to look for
   * @return iterator
   */
  private synchronized IndexIterator wc(final byte[] token) {
    final FTIndexIterator it = FTIndexIterator.FTEMPTY;
    final FTWildcard wc = new FTWildcard(token);
    if (!wc.parse()) return it;

    final IntList pr = new IntList();
    final IntList ps = new IntList();
    final byte[] pref = wc.prefix();
    final int pl = pref.length, tl = tp.length;
    final int l = Math.min(tl - 1, wc.max());
    for (int ti = pl; ti <= l; ti++) {
      int i = tp[ti];
      if (i == -1) continue;
      int c = ti + 1;
      int e = -1;
      while (c < tl && e == -1) e = tp[c++];
      i = find(pref, i, e, ti);

      while (i < e) {
        final byte[] t = inY.readBytes(i, ti);
        if (!startsWith(t, pref)) break;
        if (wc.match(t)) {
          inZ.cursor(pointer(i, ti));
          final int s = size(i, ti);
          for (int d = 0; d < s; d++) {
            pr.add(inZ.readNum());
            ps.add(inZ.readNum());
          }
        }
        i += ti + ENTRY;
      }
    }
    return iter(new FTCache(pr, ps), token);
  }
Example #6
0
  /**
   * Performs a fuzzy search for the specified token with a maximum number of errors.
   *
   * @param token token to look for
   * @param k number of errors allowed
   * @return iterator
   */
  private synchronized IndexIterator fuzzy(final byte[] token, final int k) {
    FTIndexIterator it = FTIndexIterator.FTEMPTY;
    final int tokl = token.length, tl = tp.length;
    final int e = Math.min(tl - 1, tokl + k);
    int s = Math.max(1, tokl - k) - 1;

    while (++s <= e) {
      int p = tp[s];
      if (p == -1) continue;
      int t = s + 1, r = -1;
      while (t < tl && r == -1) r = tp[t++];
      while (p < r) {
        if (ls.similar(inY.readBytes(p, s), token, k)) {
          it = FTIndexIterator.union(iter(pointer(p, s), size(p, s), inZ, token), it);
        }
        p += s + ENTRY;
      }
    }
    return it;
  }
Example #7
0
  @Override
  public synchronized int costs(final IndexToken it) {
    final byte[] tok = it.get();
    if (tok.length > data.meta.maxlen) return Integer.MAX_VALUE;

    // estimate costs for queries which stretch over multiple index entries
    final FTOpt opt = ((FTLexer) it).ftOpt();
    if (opt.is(FZ) || opt.is(WC)) return Math.max(1, data.meta.size >> 4);

    return entry(tok).size;
  }
Example #8
0
  @Override
  public void insert(final int pre, final byte[] entries) {
    final int nnew = entries.length;
    if (nnew == 0) return;
    dirty();

    // number of records to be inserted
    final int nr = nnew >>> IO.NODEPOWER;

    int split = 0;
    if (used == 0) {
      // special case: insert new data into first block if database is empty
      readPage(0);
      usedPages.set(0);
      ++used;
    } else if (pre > 0) {
      // find the offset within the block where the new records will be inserted
      split = cursor(pre - 1) + IO.NODESIZE;
    } else {
      // all insert operations will add data after first node.
      // i.e., there is no "insert before first document" statement
      throw Util.notExpected("Insertion at beginning of populated table.");
    }

    // number of bytes occupied by old records in the current block
    final int nold = npre - fpre << IO.NODEPOWER;
    // number of bytes occupied by old records which will be moved at the end
    final int moved = nold - split;

    // special case: all entries fit in the current block
    Buffer bf = bm.current();
    if (nold + nnew <= IO.BLOCKSIZE) {
      Array.move(bf.data, split, nnew, moved);
      System.arraycopy(entries, 0, bf.data, split, nnew);
      bf.dirty = true;

      // increment first pre-values of blocks after the last modified block
      for (int i = page + 1; i < used; ++i) fpres[i] += nr;
      // update cached variables (fpre is not changed)
      npre += nr;
      meta.size += nr;
      return;
    }

    // append old entries at the end of the new entries
    final byte[] all = new byte[nnew + moved];
    System.arraycopy(entries, 0, all, 0, nnew);
    System.arraycopy(bf.data, split, all, nnew, moved);

    // fill in the current block with new entries
    // number of bytes which fit in the first block
    int nrem = IO.BLOCKSIZE - split;
    if (nrem > 0) {
      System.arraycopy(all, 0, bf.data, split, nrem);
      bf.dirty = true;
    }

    // number of new required blocks and remaining bytes
    final int req = all.length - nrem;
    int needed = req / IO.BLOCKSIZE;
    final int remain = req % IO.BLOCKSIZE;

    if (remain > 0) {
      // check if the last entries can fit in the block after the current one
      if (page + 1 < used) {
        final int o = occSpace(page + 1) << IO.NODEPOWER;
        if (remain <= IO.BLOCKSIZE - o) {
          // copy the last records
          readPage(page + 1);
          bf = bm.current();
          System.arraycopy(bf.data, 0, bf.data, remain, o);
          System.arraycopy(all, all.length - remain, bf.data, 0, remain);
          bf.dirty = true;
          // reduce the pre value, since it will be later incremented with nr
          fpres[page] -= remain >>> IO.NODEPOWER;
          // go back to the previous block
          readPage(page - 1);
        } else {
          // there is not enough space in the block - allocate a new one
          ++needed;
        }
      } else {
        // this is the last block - allocate a new one
        ++needed;
      }
    }

    // number of expected blocks: existing blocks + needed block - empty blocks
    final int exp = blocks + needed - (blocks - used);
    if (exp > fpres.length) {
      // resize directory arrays if existing ones are too small
      final int ns = Math.max(fpres.length << 1, exp);
      fpres = Arrays.copyOf(fpres, ns);
      pages = Arrays.copyOf(pages, ns);
    }

    // make place for the blocks where the new entries will be written
    Array.move(fpres, page + 1, needed, used - page - 1);
    Array.move(pages, page + 1, needed, used - page - 1);

    // write the all remaining entries
    while (needed-- > 0) {
      freeBlock();
      nrem += write(all, nrem);
      fpres[page] = fpres[page - 1] + IO.ENTRIES;
      pages[page] = (int) bm.current().pos;
    }

    // increment all fpre values after the last modified block
    for (int i = page + 1; i < used; ++i) fpres[i] += nr;

    meta.size += nr;

    // update cached variables
    fpre = fpres[page];
    npre = page + 1 < used && fpres[page + 1] < meta.size ? fpres[page + 1] : meta.size;
  }
Example #9
0
  @Override
  void drawRectangles(final Graphics g, final MapRects rects, final float scale) {
    // some additions to set up borders
    final MapRect l = view.layout.layout;
    l.x = (int) scale * l.x;
    l.y = (int) scale * l.y;
    l.w = (int) scale * l.w;
    l.h = (int) scale * l.h;
    final int ww = view.getWidth();
    final int hh = view.getWidth();

    final Data data = view.gui.context.data();
    final int fsz = GUIConstants.fontSize;

    final int off = gopts.get(GUIOptions.MAPOFFSETS);
    final int rs = rects.size;
    for (int ri = 0; ri < rs; ++ri) {
      // get rectangle information
      final MapRect r = rects.get(ri);
      final int pre = r.pre;

      // level 1: next context node, set marker pointer to 0
      final int lvl = r.level;

      final boolean full = r.w == ww && r.h == hh;
      Color col = color(rects, ri);
      final boolean mark = col != null;

      r.pos =
          view.gui.context.marked.ftpos != null
              ? view.gui.context.marked.ftpos.get(data, pre)
              : null;
      g.setColor(mark ? col : GUIConstants.color(lvl));

      if (r.w < l.x + l.w || r.h < l.y + l.h || off < 2 || ViewData.leaf(gopts, data, pre)) {
        g.fillRect(r.x, r.y, r.w, r.h);
      } else {
        // painting only border for non-leaf nodes..
        g.fillRect(r.x, r.y, l.x, r.h);
        g.fillRect(r.x, r.y, r.w, l.y);
        g.fillRect(r.x + r.w - l.w, r.y, l.w, r.h);
        g.fillRect(r.x, r.y + r.h - l.h, r.w, l.h);
      }

      if (!full) {
        col = mark ? GUIConstants.colormark3 : GUIConstants.color(lvl + 2);
        g.setColor(col);
        g.drawRect(r.x, r.y, r.w, r.h);
        col = mark ? GUIConstants.colormark4 : GUIConstants.color(Math.max(0, lvl - 2));
        g.setColor(col);
        g.drawLine(r.x + r.w, r.y, r.x + r.w, r.y + r.h);
        g.drawLine(r.x, r.y + r.h, r.x + r.w, r.y + r.h);
      }

      // skip drawing of string if there is no space
      if (r.w <= 3 || r.h < GUIConstants.fontSize) continue;

      r.x += 3;
      r.w -= 3;

      final int kind = data.kind(pre);
      if (kind == Data.ELEM || kind == Data.DOC) {
        g.setColor(Color.black);
        g.setFont(GUIConstants.font);
        BaseXLayout.chopString(g, ViewData.name(gopts, data, pre), r.x, r.y, r.w, fsz);
      } else {
        g.setColor(GUIConstants.color(r.level * 2 + 8));
        g.setFont(GUIConstants.mfont);
        final byte[] text = ViewData.content(data, pre, false);

        r.thumb = MapRenderer.calcHeight(g, r, text, fsz) >= r.h;
        if (r.thumb) {
          MapRenderer.drawThumbnails(g, r, text, fsz);
        } else {
          MapRenderer.drawText(g, r, text, fsz);
        }
      }
      r.x -= 3;
      r.w += 3;
    }
  }