Beispiel #1
0
  /**
   * Evaluates the full-text match.
   *
   * @param qc query context
   * @return number of tokens, used for scoring
   * @throws QueryException query exception
   */
  private int contains(final QueryContext qc) throws QueryException {
    first = true;
    final FTLexer lexer = ftt.lexer(qc.ftToken);

    // use faster evaluation for default options
    int num = 0;
    if (fast) {
      for (final byte[] t : tokens) {
        final FTTokens qtok = ftt.cache(t);
        num = Math.max(num, ftt.contains(qtok, lexer) * qtok.length());
      }
      return num;
    }

    // find and count all occurrences
    final boolean all = mode == FTMode.ALL || mode == FTMode.ALL_WORDS;
    int oc = 0;
    for (final byte[] w : unique(tokens(qc))) {
      final FTTokens qtok = ftt.cache(w);
      final int o = ftt.contains(qtok, lexer);
      if (all && o == 0) return 0;
      num = Math.max(num, o * qtok.length());
      oc += o;
    }

    // check if occurrences are in valid range. if yes, return number of tokens
    final long mn = occ != null ? toLong(occ[0], qc) : 1;
    final long mx = occ != null ? toLong(occ[1], qc) : Long.MAX_VALUE;
    if (mn == 0 && oc == 0) matches = FTNot.not(matches);
    return oc >= mn && oc <= mx ? Math.max(1, num) : 0;
  }
Beispiel #2
0
  @Override
  public final void mouseDragged(final MouseEvent e) {
    if (!SwingUtilities.isLeftMouseButton(e)) return;

    // selection mode
    select(e.getPoint(), false);
    final int y = Math.max(20, Math.min(e.getY(), getHeight() - 20));
    if (y != e.getY()) scroll.pos(scroll.pos() + e.getY() - y);
  }
Beispiel #3
0
  @Override
  public boolean indexAccessible(final IndexInfo ii) {
    /* If the following conditions yield true, the index is accessed:
     * - all query terms are statically available
     * - no FTTimes option is specified
     * - explicitly set case, diacritics and stemming match options do not
     *   conflict with index options. */
    data = ii.ic.data;
    final MetaData md = data.meta;
    final FTOpt fto = ftt.opt;

    /* Index will be applied if no explicit match options have been set
     * that conflict with the index options. As a consequence, though, index-
     * based querying might yield other results than sequential scanning. */
    if (occ != null
        || fto.cs != null && md.casesens == (fto.cs == FTCase.INSENSITIVE)
        || fto.isSet(DC) && md.diacritics != fto.is(DC)
        || fto.isSet(ST) && md.stemming != fto.is(ST)
        || fto.ln != null && !fto.ln.equals(md.language)) return false;

    // adopt database options to tokenizer
    fto.copy(md);

    // estimate costs if text is not known at compile time
    if (tokens == null) {
      ii.costs = Math.max(2, data.meta.size / 30);
      return true;
    }

    // summarize number of hits; break loop if no hits are expected
    final FTLexer ft = new FTLexer(fto);
    ii.costs = 0;
    for (byte[] t : tokens) {
      ft.init(t);
      while (ft.hasNext()) {
        final byte[] tok = ft.nextToken();
        if (fto.sw != null && fto.sw.contains(tok)) continue;

        if (fto.is(WC)) {
          // don't use index if one of the terms starts with a wildcard
          t = ft.get();
          if (t[0] == '.') return false;
          // don't use index if certain characters or more than 1 dot are found
          int d = 0;
          for (final byte w : t) {
            if (w == '{' || w == '\\' || w == '.' && ++d > 1) return false;
          }
        }
        // favor full-text index requests over exact queries
        final int costs = data.costs(ft);
        if (costs != 0) ii.costs += Math.max(2, costs / 100);
      }
    }
    return true;
  }
Beispiel #4
0
  @Override
  public DiskData build() throws IOException {
    meta.assign(parser);
    meta.dirty = true;

    // calculate optimized output buffer sizes to reduce disk fragmentation
    final Runtime rt = Runtime.getRuntime();
    final long max = Math.min(1 << 22, rt.maxMemory() - rt.freeMemory() >> 2);
    int bs = (int) Math.min(meta.filesize, max);
    bs = Math.max(IO.BLOCKSIZE, bs - bs % IO.BLOCKSIZE);

    // drop old database (if available) and create new one
    DropDB.drop(dbname, sopts);
    sopts.dbpath(dbname).md();

    elemNames = new Names(meta);
    attrNames = new Names(meta);
    try {
      tout = new DataOutput(new TableOutput(meta, DATATBL));
      xout = new DataOutput(meta.dbfile(DATATXT), bs);
      vout = new DataOutput(meta.dbfile(DATAATV), bs);
      sout = new DataOutput(meta.dbfile(DATATMP), bs);

      final Performance perf = Prop.debug ? new Performance() : null;
      Util.debug(tit() + DOTS);
      parse();
      if (Prop.debug) Util.errln(" " + perf + " (" + Performance.getMemory() + ')');

    } catch (final IOException ex) {
      try {
        close();
      } catch (final IOException ignored) {
      }
      throw ex;
    }
    close();

    // copy temporary values into database table
    try (final DataInput in = new DataInput(meta.dbfile(DATATMP))) {
      final TableAccess ta = new TableDiskAccess(meta, true);
      for (; spos < ssize; ++spos) ta.write4(in.readNum(), 8, in.readNum());
      ta.close();
    }
    meta.dbfile(DATATMP).delete();

    // return database instance
    return new DiskData(meta, elemNames, attrNames, path, ns);
  }
Beispiel #5
0
  @Override
  public void keyTyped(final KeyEvent e) {
    if (!hist.active()
        || control(e)
        || DELNEXT.is(e)
        || DELPREV.is(e)
        || ESCAPE.is(e)
        || CUT2.is(e)) return;

    final int caret = editor.pos();

    // remember if marked text is to be deleted
    final StringBuilder sb = new StringBuilder(1).append(e.getKeyChar());
    final boolean indent = TAB.is(e) && editor.indent(sb, e.isShiftDown());

    // delete marked text
    final boolean selected = editor.selected() && !indent;
    if (selected) editor.delete();

    final int move = ENTER.is(e) ? editor.enter(sb) : editor.add(sb, selected);

    // refresh history and adjust cursor position
    hist.store(editor.text(), caret, editor.pos());
    if (move != 0) editor.pos(Math.min(editor.size(), caret + move));

    // adjust text height
    scrollCode.invokeLater(true);
    e.consume();
  }
Beispiel #6
0
 /**
  * Fill the current buffer with bytes from the specified array from the specified offset.
  *
  * @param s source array
  * @param o offset from the beginning of the array
  * @return number of written bytes
  */
 private int write(final byte[] s, final int o) {
   final Buffer bf = bm.current();
   final int len = Math.min(IO.BLOCKSIZE, s.length - o);
   System.arraycopy(s, o, bf.data, 0, len);
   bf.dirty = true;
   return len;
 }
Beispiel #7
0
  /**
   * Performs a wildcard search for the specified token.
   *
   * @param token token to look for
   * @return iterator
   */
  private synchronized IndexIterator wc(final byte[] token) {
    final FTIndexIterator it = FTIndexIterator.FTEMPTY;
    final FTWildcard wc = new FTWildcard(token);
    if (!wc.parse()) return it;

    final IntList pr = new IntList();
    final IntList ps = new IntList();
    final byte[] pref = wc.prefix();
    final int pl = pref.length, tl = tp.length;
    final int l = Math.min(tl - 1, wc.max());
    for (int ti = pl; ti <= l; ti++) {
      int i = tp[ti];
      if (i == -1) continue;
      int c = ti + 1;
      int e = -1;
      while (c < tl && e == -1) e = tp[c++];
      i = find(pref, i, e, ti);

      while (i < e) {
        final byte[] t = inY.readBytes(i, ti);
        if (!startsWith(t, pref)) break;
        if (wc.match(t)) {
          inZ.cursor(pointer(i, ti));
          final int s = size(i, ti);
          for (int d = 0; d < s; d++) {
            pr.add(inZ.readNum());
            ps.add(inZ.readNum());
          }
        }
        i += ti + ENTRY;
      }
    }
    return iter(new FTCache(pr, ps), token);
  }
Beispiel #8
0
  /**
   * Initializes colors.
   *
   * @param prop gui properties
   */
  public static void init(final GUIProp prop) {
    final int r = prop.num(GUIProp.COLORRED);
    final int g = prop.num(GUIProp.COLORGREEN);
    final int b = prop.num(GUIProp.COLORBLUE);

    // calculate color c:
    // c = (255 - expectedColor) * 10 / factor (= GUIRED/BLUE/GREEN)
    color1 = new Color(col(r, 24), col(g, 25), col(b, 40));
    color2 = new Color(col(r, 32), col(g, 32), col(b, 44));
    color3 = new Color(col(r, 48), col(g, 50), col(b, 40));
    color4 = new Color(col(r, 140), col(g, 100), col(b, 70));
    color1A = new Color(col(r, 110), col(g, 150), col(b, 160), 100);

    colormark1A = new Color(col(r, 32), col(g, 160), col(b, 320), 100);
    colormark2A = new Color(col(r, 16), col(g, 80), col(b, 160), 100);
    colormark1 = new Color(col(r, 16), col(g, 120), col(b, 240));
    colormark2 = new Color(col(r, 16), col(g, 80), col(b, 160));
    colormark3 = new Color(col(r, 32), col(g, 160), col(b, 320));
    colormark4 = new Color(col(r, 1), col(g, 40), col(b, 80));

    // create color array
    for (int l = 1; l < COLORS.length + 1; ++l) {
      COLORS[l - 1] =
          new Color(Math.max(255 - l * r, 0), Math.max(255 - l * g, 0), Math.max(255 - l * b, 0));
    }
    final Color c = COLORS[16];
    color2A = new Color(c.getRed(), c.getGreen(), c.getBlue(), 40);
    color3A = new Color(c.getRed(), c.getGreen(), c.getBlue(), 100);

    final String f = prop.get(GUIProp.FONT);
    final int type = prop.num(GUIProp.FONTTYPE);
    fontSize = prop.num(GUIProp.FONTSIZE);
    font = new Font(f, type, fontSize);
    mfont = new Font(prop.get(GUIProp.MONOFONT), type, fontSize);
    bfont = new Font(f, Font.BOLD, fontSize);
    lfont = new Font(f, type, 18 + (fontSize >> 1));
    dmfont = new Font(prop.get(GUIProp.MONOFONT), 0, TFONT.getSize() - 1);

    final Container comp = new Container();
    dwidth = comp.getFontMetrics(dmfont).getWidths();
    fwidth = comp.getFontMetrics(font).getWidths();
    lwidth = comp.getFontMetrics(lfont).getWidths();
    mfwidth = comp.getFontMetrics(mfont).getWidths();
    bwidth = comp.getFontMetrics(bfont).getWidths();
  }
Beispiel #9
0
  /**
   * Performs a fuzzy search for the specified token with a maximum number of errors.
   *
   * @param token token to look for
   * @param k number of errors allowed
   * @return iterator
   */
  private synchronized IndexIterator fuzzy(final byte[] token, final int k) {
    FTIndexIterator it = FTIndexIterator.FTEMPTY;
    final int tokl = token.length, tl = tp.length;
    final int e = Math.min(tl - 1, tokl + k);
    int s = Math.max(1, tokl - k) - 1;

    while (++s <= e) {
      int p = tp[s];
      if (p == -1) continue;
      int t = s + 1, r = -1;
      while (t < tl && r == -1) r = tp[t++];
      while (p < r) {
        if (ls.similar(inY.readBytes(p, s), token, k)) {
          it = FTIndexIterator.union(iter(pointer(p, s), size(p, s), inZ, token), it);
        }
        p += s + ENTRY;
      }
    }
    return it;
  }
Beispiel #10
0
  @Override
  public synchronized int costs(final IndexToken it) {
    final byte[] tok = it.get();
    if (tok.length > data.meta.maxlen) return Integer.MAX_VALUE;

    // estimate costs for queries which stretch over multiple index entries
    final FTOpt opt = ((FTLexer) it).ftOpt();
    if (opt.is(FZ) || opt.is(WC)) return Math.max(1, data.meta.size >> 4);

    return entry(tok).size;
  }
Beispiel #11
0
 /**
  * Converts color value with specified factor.
  *
  * @param c color
  * @param f factor
  * @return converted color value
  */
 private static int col(final int c, final int f) {
   return Math.max(0, 255 - c * f / 10);
 }
Beispiel #12
0
 /**
  * Returns the specified color from the color gradient.
  *
  * @param i color index
  * @return color
  */
 public static Color color(final int i) {
   return COLORS[Math.min(COLORS.length - 1, i)];
 }
Beispiel #13
0
  /**
   * Evaluates the specified query.
   *
   * @param query query
   * @return success flag
   */
  final boolean query(final String query) {
    final Performance p = new Performance();
    String error;
    if (exception != null) {
      error = Util.message(exception);
    } else {
      try {
        long hits = 0;
        final boolean run = options.get(MainOptions.RUNQUERY);
        final boolean serial = options.get(MainOptions.SERIALIZE);
        final int runs = Math.max(1, options.get(MainOptions.RUNS));
        for (int r = 0; r < runs; ++r) {
          // reuse existing processor instance
          if (r != 0) qp = null;
          qp(query, context);
          parse(p);
          if (r == 0) plan(false);

          qp.compile();
          info.compiling += p.time();
          if (r == 0) plan(true);
          if (!run) continue;

          final PrintOutput po = r == 0 && serial ? out : new NullOutput();
          try (final Serializer ser = qp.getSerializer(po)) {
            if (maxResults >= 0) {
              result = qp.cache(maxResults);
              info.evaluating += p.time();
              result.serialize(ser);
              hits = result.size();
            } else {
              hits = 0;
              final Iter ir = qp.iter();
              info.evaluating += p.time();
              for (Item it; (it = ir.next()) != null; ) {
                ser.serialize(it);
                ++hits;
                checkStop();
              }
            }
          }
          qp.close();
          info.serializing += p.time();
        }
        // dump some query info
        // out.flush();

        // remove string list if global locking is used and if query is updating
        if (soptions.get(StaticOptions.GLOBALLOCK) && qp.updating) {
          info.readLocked = null;
          info.writeLocked = null;
        }
        return info(info.toString(qp, out.size(), hits, options.get(MainOptions.QUERYINFO)));

      } catch (final QueryException | IOException ex) {
        exception = ex;
        error = Util.message(ex);
      } catch (final ProcException ex) {
        error = INTERRUPTED;
      } catch (final StackOverflowError ex) {
        Util.debug(ex);
        error = BASX_STACKOVERFLOW.desc;
      } catch (final RuntimeException ex) {
        extError("");
        Util.debug(info());
        throw ex;
      } finally {
        // close processor after exceptions
        if (qp != null) qp.close();
      }
    }
    return extError(error);
  }
Beispiel #14
0
 /**
  * Prints the current stack trace to System.err.
  *
  * @param i number of steps to print
  */
 public static void stack(final int i) {
   errln("You're here:");
   final String[] stack = toArray(new Throwable());
   final int l = Math.min(Math.max(2, i + 2), stack.length);
   for (int s = 2; s < l; ++s) errln(stack[s]);
 }
Beispiel #15
0
  @Override
  public void insert(final int pre, final byte[] entries) {
    final int nnew = entries.length;
    if (nnew == 0) return;
    dirty();

    // number of records to be inserted
    final int nr = nnew >>> IO.NODEPOWER;

    int split = 0;
    if (used == 0) {
      // special case: insert new data into first block if database is empty
      readPage(0);
      usedPages.set(0);
      ++used;
    } else if (pre > 0) {
      // find the offset within the block where the new records will be inserted
      split = cursor(pre - 1) + IO.NODESIZE;
    } else {
      // all insert operations will add data after first node.
      // i.e., there is no "insert before first document" statement
      throw Util.notExpected("Insertion at beginning of populated table.");
    }

    // number of bytes occupied by old records in the current block
    final int nold = npre - fpre << IO.NODEPOWER;
    // number of bytes occupied by old records which will be moved at the end
    final int moved = nold - split;

    // special case: all entries fit in the current block
    Buffer bf = bm.current();
    if (nold + nnew <= IO.BLOCKSIZE) {
      Array.move(bf.data, split, nnew, moved);
      System.arraycopy(entries, 0, bf.data, split, nnew);
      bf.dirty = true;

      // increment first pre-values of blocks after the last modified block
      for (int i = page + 1; i < used; ++i) fpres[i] += nr;
      // update cached variables (fpre is not changed)
      npre += nr;
      meta.size += nr;
      return;
    }

    // append old entries at the end of the new entries
    final byte[] all = new byte[nnew + moved];
    System.arraycopy(entries, 0, all, 0, nnew);
    System.arraycopy(bf.data, split, all, nnew, moved);

    // fill in the current block with new entries
    // number of bytes which fit in the first block
    int nrem = IO.BLOCKSIZE - split;
    if (nrem > 0) {
      System.arraycopy(all, 0, bf.data, split, nrem);
      bf.dirty = true;
    }

    // number of new required blocks and remaining bytes
    final int req = all.length - nrem;
    int needed = req / IO.BLOCKSIZE;
    final int remain = req % IO.BLOCKSIZE;

    if (remain > 0) {
      // check if the last entries can fit in the block after the current one
      if (page + 1 < used) {
        final int o = occSpace(page + 1) << IO.NODEPOWER;
        if (remain <= IO.BLOCKSIZE - o) {
          // copy the last records
          readPage(page + 1);
          bf = bm.current();
          System.arraycopy(bf.data, 0, bf.data, remain, o);
          System.arraycopy(all, all.length - remain, bf.data, 0, remain);
          bf.dirty = true;
          // reduce the pre value, since it will be later incremented with nr
          fpres[page] -= remain >>> IO.NODEPOWER;
          // go back to the previous block
          readPage(page - 1);
        } else {
          // there is not enough space in the block - allocate a new one
          ++needed;
        }
      } else {
        // this is the last block - allocate a new one
        ++needed;
      }
    }

    // number of expected blocks: existing blocks + needed block - empty blocks
    final int exp = blocks + needed - (blocks - used);
    if (exp > fpres.length) {
      // resize directory arrays if existing ones are too small
      final int ns = Math.max(fpres.length << 1, exp);
      fpres = Arrays.copyOf(fpres, ns);
      pages = Arrays.copyOf(pages, ns);
    }

    // make place for the blocks where the new entries will be written
    Array.move(fpres, page + 1, needed, used - page - 1);
    Array.move(pages, page + 1, needed, used - page - 1);

    // write the all remaining entries
    while (needed-- > 0) {
      freeBlock();
      nrem += write(all, nrem);
      fpres[page] = fpres[page - 1] + IO.ENTRIES;
      pages[page] = (int) bm.current().pos;
    }

    // increment all fpre values after the last modified block
    for (int i = page + 1; i < used; ++i) fpres[i] += nr;

    meta.size += nr;

    // update cached variables
    fpre = fpres[page];
    npre = page + 1 < used && fpres[page + 1] < meta.size ? fpres[page + 1] : meta.size;
  }