/**
* Evaluates the full-text match.
*
* @param qc query context
* @return number of tokens, used for scoring
* @throws QueryException query exception
*/
private int contains(final QueryContext qc) throws QueryException {
first = true;
final FTLexer lexer = ftt.lexer(qc.ftToken);
// use faster evaluation for default options
int num = 0;
if (fast) {
for (final byte[] t : tokens) {
final FTTokens qtok = ftt.cache(t);
num = Math.max(num, ftt.contains(qtok, lexer) * qtok.length());
}
return num;
}
// find and count all occurrences
final boolean all = mode == FTMode.ALL || mode == FTMode.ALL_WORDS;
int oc = 0;
for (final byte[] w : unique(tokens(qc))) {
final FTTokens qtok = ftt.cache(w);
final int o = ftt.contains(qtok, lexer);
if (all && o == 0) return 0;
num = Math.max(num, o * qtok.length());
oc += o;
}
// check if occurrences are in valid range. if yes, return number of tokens
final long mn = occ != null ? toLong(occ[0], qc) : 1;
final long mx = occ != null ? toLong(occ[1], qc) : Long.MAX_VALUE;
if (mn == 0 && oc == 0) matches = FTNot.not(matches);
return oc >= mn && oc <= mx ? Math.max(1, num) : 0;
}
@Override
public boolean indexAccessible(final IndexInfo ii) {
/* If the following conditions yield true, the index is accessed:
* - all query terms are statically available
* - no FTTimes option is specified
* - explicitly set case, diacritics and stemming match options do not
* conflict with index options. */
data = ii.ic.data;
final MetaData md = data.meta;
final FTOpt fto = ftt.opt;
/* Index will be applied if no explicit match options have been set
* that conflict with the index options. As a consequence, though, index-
* based querying might yield other results than sequential scanning. */
if (occ != null
|| fto.cs != null && md.casesens == (fto.cs == FTCase.INSENSITIVE)
|| fto.isSet(DC) && md.diacritics != fto.is(DC)
|| fto.isSet(ST) && md.stemming != fto.is(ST)
|| fto.ln != null && !fto.ln.equals(md.language)) return false;
// adopt database options to tokenizer
fto.copy(md);
// estimate costs if text is not known at compile time
if (tokens == null) {
ii.costs = Math.max(2, data.meta.size / 30);
return true;
}
// summarize number of hits; break loop if no hits are expected
final FTLexer ft = new FTLexer(fto);
ii.costs = 0;
for (byte[] t : tokens) {
ft.init(t);
while (ft.hasNext()) {
final byte[] tok = ft.nextToken();
if (fto.sw != null && fto.sw.contains(tok)) continue;
if (fto.is(WC)) {
// don't use index if one of the terms starts with a wildcard
t = ft.get();
if (t[0] == '.') return false;
// don't use index if certain characters or more than 1 dot are found
int d = 0;
for (final byte w : t) {
if (w == '{' || w == '\\' || w == '.' && ++d > 1) return false;
}
}
// favor full-text index requests over exact queries
final int costs = data.costs(ft);
if (costs != 0) ii.costs += Math.max(2, costs / 100);
}
}
return true;
}
@Override
public synchronized int costs(final IndexToken it) {
final byte[] tok = it.get();
if (tok.length > data.meta.maxlen) return Integer.MAX_VALUE;
// estimate costs for queries which stretch over multiple index entries
final FTOpt opt = ((FTLexer) it).ftOpt();
if (opt.is(FZ) || opt.is(WC)) return Math.max(1, data.meta.size >> 4);
return entry(tok).size;
}
@Override
public DiskData build() throws IOException {
meta.assign(parser);
meta.dirty = true;
// calculate optimized output buffer sizes to reduce disk fragmentation
final Runtime rt = Runtime.getRuntime();
final long max = Math.min(1 << 22, rt.maxMemory() - rt.freeMemory() >> 2);
int bs = (int) Math.min(meta.filesize, max);
bs = Math.max(IO.BLOCKSIZE, bs - bs % IO.BLOCKSIZE);
// drop old database (if available) and create new one
DropDB.drop(dbname, sopts);
sopts.dbpath(dbname).md();
elemNames = new Names(meta);
attrNames = new Names(meta);
try {
tout = new DataOutput(new TableOutput(meta, DATATBL));
xout = new DataOutput(meta.dbfile(DATATXT), bs);
vout = new DataOutput(meta.dbfile(DATAATV), bs);
sout = new DataOutput(meta.dbfile(DATATMP), bs);
final Performance perf = Prop.debug ? new Performance() : null;
Util.debug(tit() + DOTS);
parse();
if (Prop.debug) Util.errln(" " + perf + " (" + Performance.getMemory() + ')');
} catch (final IOException ex) {
try {
close();
} catch (final IOException ignored) {
}
throw ex;
}
close();
// copy temporary values into database table
try (final DataInput in = new DataInput(meta.dbfile(DATATMP))) {
final TableAccess ta = new TableDiskAccess(meta, true);
for (; spos < ssize; ++spos) ta.write4(in.readNum(), 8, in.readNum());
ta.close();
}
meta.dbfile(DATATMP).delete();
// return database instance
return new DiskData(meta, elemNames, attrNames, path, ns);
}
/**
* Performs a fuzzy search for the specified token with a maximum number of errors.
*
* @param token token to look for
* @param k number of errors allowed
* @return iterator
*/
private synchronized IndexIterator fuzzy(final byte[] token, final int k) {
FTIndexIterator it = FTIndexIterator.FTEMPTY;
final int tokl = token.length, tl = tp.length;
final int e = Math.min(tl - 1, tokl + k);
int s = Math.max(1, tokl - k) - 1;
while (++s <= e) {
int p = tp[s];
if (p == -1) continue;
int t = s + 1, r = -1;
while (t < tl && r == -1) r = tp[t++];
while (p < r) {
if (ls.similar(inY.readBytes(p, s), token, k)) {
it = FTIndexIterator.union(iter(pointer(p, s), size(p, s), inZ, token), it);
}
p += s + ENTRY;
}
}
return it;
}
@Override
public void insert(final int pre, final byte[] entries) {
final int nnew = entries.length;
if (nnew == 0) return;
dirty();
// number of records to be inserted
final int nr = nnew >>> IO.NODEPOWER;
int split = 0;
if (used == 0) {
// special case: insert new data into first block if database is empty
readPage(0);
usedPages.set(0);
++used;
} else if (pre > 0) {
// find the offset within the block where the new records will be inserted
split = cursor(pre - 1) + IO.NODESIZE;
} else {
// all insert operations will add data after first node.
// i.e., there is no "insert before first document" statement
throw Util.notExpected("Insertion at beginning of populated table.");
}
// number of bytes occupied by old records in the current block
final int nold = npre - fpre << IO.NODEPOWER;
// number of bytes occupied by old records which will be moved at the end
final int moved = nold - split;
// special case: all entries fit in the current block
Buffer bf = bm.current();
if (nold + nnew <= IO.BLOCKSIZE) {
Array.move(bf.data, split, nnew, moved);
System.arraycopy(entries, 0, bf.data, split, nnew);
bf.dirty = true;
// increment first pre-values of blocks after the last modified block
for (int i = page + 1; i < used; ++i) fpres[i] += nr;
// update cached variables (fpre is not changed)
npre += nr;
meta.size += nr;
return;
}
// append old entries at the end of the new entries
final byte[] all = new byte[nnew + moved];
System.arraycopy(entries, 0, all, 0, nnew);
System.arraycopy(bf.data, split, all, nnew, moved);
// fill in the current block with new entries
// number of bytes which fit in the first block
int nrem = IO.BLOCKSIZE - split;
if (nrem > 0) {
System.arraycopy(all, 0, bf.data, split, nrem);
bf.dirty = true;
}
// number of new required blocks and remaining bytes
final int req = all.length - nrem;
int needed = req / IO.BLOCKSIZE;
final int remain = req % IO.BLOCKSIZE;
if (remain > 0) {
// check if the last entries can fit in the block after the current one
if (page + 1 < used) {
final int o = occSpace(page + 1) << IO.NODEPOWER;
if (remain <= IO.BLOCKSIZE - o) {
// copy the last records
readPage(page + 1);
bf = bm.current();
System.arraycopy(bf.data, 0, bf.data, remain, o);
System.arraycopy(all, all.length - remain, bf.data, 0, remain);
bf.dirty = true;
// reduce the pre value, since it will be later incremented with nr
fpres[page] -= remain >>> IO.NODEPOWER;
// go back to the previous block
readPage(page - 1);
} else {
// there is not enough space in the block - allocate a new one
++needed;
}
} else {
// this is the last block - allocate a new one
++needed;
}
}
// number of expected blocks: existing blocks + needed block - empty blocks
final int exp = blocks + needed - (blocks - used);
if (exp > fpres.length) {
// resize directory arrays if existing ones are too small
final int ns = Math.max(fpres.length << 1, exp);
fpres = Arrays.copyOf(fpres, ns);
pages = Arrays.copyOf(pages, ns);
}
// make place for the blocks where the new entries will be written
Array.move(fpres, page + 1, needed, used - page - 1);
Array.move(pages, page + 1, needed, used - page - 1);
// write the all remaining entries
while (needed-- > 0) {
freeBlock();
nrem += write(all, nrem);
fpres[page] = fpres[page - 1] + IO.ENTRIES;
pages[page] = (int) bm.current().pos;
}
// increment all fpre values after the last modified block
for (int i = page + 1; i < used; ++i) fpres[i] += nr;
meta.size += nr;
// update cached variables
fpre = fpres[page];
npre = page + 1 < used && fpres[page + 1] < meta.size ? fpres[page + 1] : meta.size;
}
@Override
void drawRectangles(final Graphics g, final MapRects rects, final float scale) {
// some additions to set up borders
final MapRect l = view.layout.layout;
l.x = (int) scale * l.x;
l.y = (int) scale * l.y;
l.w = (int) scale * l.w;
l.h = (int) scale * l.h;
final int ww = view.getWidth();
final int hh = view.getWidth();
final Data data = view.gui.context.data();
final int fsz = GUIConstants.fontSize;
final int off = gopts.get(GUIOptions.MAPOFFSETS);
final int rs = rects.size;
for (int ri = 0; ri < rs; ++ri) {
// get rectangle information
final MapRect r = rects.get(ri);
final int pre = r.pre;
// level 1: next context node, set marker pointer to 0
final int lvl = r.level;
final boolean full = r.w == ww && r.h == hh;
Color col = color(rects, ri);
final boolean mark = col != null;
r.pos =
view.gui.context.marked.ftpos != null
? view.gui.context.marked.ftpos.get(data, pre)
: null;
g.setColor(mark ? col : GUIConstants.color(lvl));
if (r.w < l.x + l.w || r.h < l.y + l.h || off < 2 || ViewData.leaf(gopts, data, pre)) {
g.fillRect(r.x, r.y, r.w, r.h);
} else {
// painting only border for non-leaf nodes..
g.fillRect(r.x, r.y, l.x, r.h);
g.fillRect(r.x, r.y, r.w, l.y);
g.fillRect(r.x + r.w - l.w, r.y, l.w, r.h);
g.fillRect(r.x, r.y + r.h - l.h, r.w, l.h);
}
if (!full) {
col = mark ? GUIConstants.colormark3 : GUIConstants.color(lvl + 2);
g.setColor(col);
g.drawRect(r.x, r.y, r.w, r.h);
col = mark ? GUIConstants.colormark4 : GUIConstants.color(Math.max(0, lvl - 2));
g.setColor(col);
g.drawLine(r.x + r.w, r.y, r.x + r.w, r.y + r.h);
g.drawLine(r.x, r.y + r.h, r.x + r.w, r.y + r.h);
}
// skip drawing of string if there is no space
if (r.w <= 3 || r.h < GUIConstants.fontSize) continue;
r.x += 3;
r.w -= 3;
final int kind = data.kind(pre);
if (kind == Data.ELEM || kind == Data.DOC) {
g.setColor(Color.black);
g.setFont(GUIConstants.font);
BaseXLayout.chopString(g, ViewData.name(gopts, data, pre), r.x, r.y, r.w, fsz);
} else {
g.setColor(GUIConstants.color(r.level * 2 + 8));
g.setFont(GUIConstants.mfont);
final byte[] text = ViewData.content(data, pre, false);
r.thumb = MapRenderer.calcHeight(g, r, text, fsz) >= r.h;
if (r.thumb) {
MapRenderer.drawThumbnails(g, r, text, fsz);
} else {
MapRenderer.drawText(g, r, text, fsz);
}
}
r.x -= 3;
r.w += 3;
}
}