@Override public IndexIterator iter(final IndexToken token) { final int id = values.id(token.get()); if (id == 0) return IndexIterator.EMPTY; final int len = lenList.get(id); final int[] ids = idsList.get(id), pres; if (data.meta.updindex) { final IntList tmp = new IntList(); for (int i = 0; i < len; ++i) tmp.add(data.pre(ids[i])); pres = tmp.sort().finish(); } else { pres = ids; } return new IndexIterator() { int p; @Override public boolean more() { return p < len; } @Override public int pre() { return pres[p++]; } @Override public int size() { return len; } }; }
@Override public void execute(final GUI gui) { final DialogExport dialog = new DialogExport(gui); if (!dialog.ok()) return; final IOFile root = new IOFile(dialog.path()); // check if existing files will be overwritten if (root.exists()) { IO file = null; boolean overwrite = false; final Data d = gui.context.data(); final IntList il = d.resources.docs(); final int is = il.size(); for (int i = 0; i < is; i++) { file = root.merge(Token.string(d.text(il.get(i), true))); if (file.exists()) { if (overwrite) { // more than one file will be overwritten; check remaining tests file = null; break; } overwrite = true; } } if (overwrite) { // show message for overwriting files or directories final String msg = file == null ? FILES_REPLACE_X : FILE_EXISTS_X; if (file == null) file = root; if (!BaseXDialog.confirm(gui, Util.info(msg, file))) return; } } DialogProgress.execute(gui, new Export(root.path())); }
/** * Performs a wildcard search for the specified token. * * @param token token to look for * @return iterator */ private synchronized IndexIterator wc(final byte[] token) { final FTIndexIterator it = FTIndexIterator.FTEMPTY; final FTWildcard wc = new FTWildcard(token); if (!wc.parse()) return it; final IntList pr = new IntList(); final IntList ps = new IntList(); final byte[] pref = wc.prefix(); final int pl = pref.length, tl = tp.length; final int l = Math.min(tl - 1, wc.max()); for (int ti = pl; ti <= l; ti++) { int i = tp[ti]; if (i == -1) continue; int c = ti + 1; int e = -1; while (c < tl && e == -1) e = tp[c++]; i = find(pref, i, e, ti); while (i < e) { final byte[] t = inY.readBytes(i, ti); if (!startsWith(t, pref)) break; if (wc.match(t)) { inZ.cursor(pointer(i, ti)); final int s = size(i, ti); for (int d = 0; d < s; d++) { pr.add(inZ.readNum()); ps.add(inZ.readNum()); } } i += ti + ENTRY; } } return iter(new FTCache(pr, ps), token); }
/** * Constructor. * * @param pr pre values * @param ps positions */ private FTCache(final IntList pr, final IntList ps) { final int s = pr.size(); final double[] v = new double[s]; for (int i = 0; i < s; i++) v[i] = (long) pr.get(i) << 32 | ps.get(i); order = Array.createOrder(v, true); pre = pr; pos = ps; }
@Override public void add(final ValueCache cache) { for (final byte[] key : cache) { final IntList vals = cache.ids(key); if (!vals.isEmpty()) add(key, vals.sort().finish()); } finish(); }
/** * Removes values from the index. * * @param key key * @param vals sorted values */ void delete(final byte[] key, final int... vals) { final int id = values.id(key), vl = vals.length, l = lenList.get(id), s = l - vl; final int[] ids = idsList.get(id); for (int i = 0, n = 0, v = 0; i < l; i++) { if (v == vl || ids[i] != vals[v]) ids[n++] = ids[i]; else v++; } lenList.set(id, s); if (s == 0) idsList.set(id, null); }
/** * Returns an iterator for an index entry. * * @param off offset on entries * @param size number of id/pos entries * @param da data source * @param token index token * @return iterator */ private static FTIndexIterator iter( final long off, final int size, final DataAccess da, final byte[] token) { da.cursor(off); final IntList pr = new IntList(size); final IntList ps = new IntList(size); for (int c = 0; c < size; c++) { pr.add(da.readNum()); ps.add(da.readNum()); } return iter(new FTCache(pr, ps), token); }
/** Finishes the index creation. */ void finish() { if (reorder == null) return; for (int i = 1; i < reorder.size(); i++) { if (reorder.get(i)) Arrays.sort(idsList.get(i), 0, lenList.get(i)); } reorder = null; }
@Override public int size() { // returns the actual number of indexed entries int s = 0; for (int c = 1; c < s; c++) if (lenList.get(c) > 0) s++; return s; }
@Override public boolean eq(final ANode node) { // no database node if (!(node instanceof DBNode)) return false; // ensure that the pre value is contained in the target documents final DBNode db = (DBNode) node; return data == db.data() && pres.contains(db.pre()); }
/** * Returns a document test. This test will be called by {@link AxisPath#index} if the context * value only consists of database nodes. * * @param rt root value * @return document test */ static Test get(final Value rt) { // use simple test if database contains only one document final Data data = rt.data(); if (data.meta.ndocs == 1) return Test.DOC; // adopt nodes from existing sequence if (rt instanceof DBNodeSeq) { final DBNodeSeq seq = (DBNodeSeq) rt; return seq.all() ? Test.DOC : new InvDocTest(new IntList(seq.pres()), data); } // loop through all documents and add pre values of documents // not more than 2^31 documents supported final IntList il = new IntList((int) rt.size()); for (final Item it : rt) il.add(((DBNode) it).pre()); return new InvDocTest(il, data); }
/** * Lists resources of the specified database. * * @return success flag * @throws IOException I/O exception */ private boolean listDB() throws IOException { final String db = args[0]; final String path = args[1] != null ? args[1] : ""; if (!Databases.validName(db)) return error(NAME_INVALID_X, db); final Table table = new Table(); table.description = RESOURCES; table.header.add(INPUT_PATH); table.header.add(TYPE); table.header.add(MimeTypes.CONTENT_TYPE); table.header.add(SIZE); try { // add xml documents final Data data = Open.open(db, context); final Resources res = data.resources; final IntList il = res.docs(path); final int ds = il.size(); for (int i = 0; i < ds; i++) { final int pre = il.get(i); final TokenList tl = new TokenList(3); final byte[] file = data.text(pre, true); tl.add(file); tl.add(DataText.M_XML); tl.add(MimeTypes.APP_XML); tl.add(data.size(pre, Data.DOC)); table.contents.add(tl); } // add binary resources for (final byte[] file : res.binaries(path)) { final String f = string(file); final TokenList tl = new TokenList(3); tl.add(file); tl.add(DataText.M_RAW); tl.add(MimeTypes.get(f)); tl.add(data.meta.binary(f).length()); table.contents.add(tl); } Close.close(data, context); } catch (final IOException ex) { return error(Util.message(ex)); } out.println(table.sort().finish()); return true; }
@Override public Item item(final QueryContext qc, final InputInfo ii) throws QueryException { final Data data = checkData(qc); final String path = path(1, qc); final Item item = toItem(exprs[2], qc); final Options opts = toOptions(3, Q_OPTIONS, new Options(), qc); final Updates updates = qc.resources.updates(); final IntList docs = data.resources.docs(path); int d = 0; // delete binary resources final IOFile bin = data.meta.binary(path); if (bin == null || bin.isDir()) throw BXDB_REPLACE_X.get(info, path); if (item instanceof Bin) { updates.add(new DBStore(data, path, item, info), qc); } else { if (bin.exists()) updates.add(new DBDelete(data, path, info), qc); final NewInput input = checkInput(item, token(path)); if (docs.isEmpty() || docs.get(0) == 0) { // no replacement of first document (because of TableDiskAccess#insert, used > 0, pre = 0) updates.add(new DBAdd(data, input, opts, qc, info), qc); } else { updates.add(new ReplaceDoc(docs.get(0), data, input, opts, qc, info), qc); d = 1; } } // delete old documents final int ds = docs.size(); for (; d < ds; d++) updates.add(new DeleteNode(docs.get(d), data, info), qc); return null; }
/** * Adds values to the index. * * @param key key to be indexed * @param vals sorted values */ void add(final byte[] key, final int... vals) { // token index: add values. otherwise, reference existing values final int id = type == IndexType.TOKEN ? values.put(key) : values.id(key), vl = vals.length; // updatable index: if required, resize existing arrays while (idsList.size() < id + 1) idsList.add(null); if (lenList.size() < id + 1) lenList.set(id, 0); final int len = lenList.get(id), size = len + vl; int[] ids = idsList.get(id); if (ids == null) { ids = vals; } else { if (ids.length < size) ids = Arrays.copyOf(ids, Array.newSize(size)); System.arraycopy(vals, 0, ids, len, vl); if (ids[len - 1] > vals[0]) { if (reorder == null) reorder = new BoolList(values.size()); reorder.set(id, true); } } idsList.set(id, ids); lenList.set(id, size); }
/** * Returns a string representation of the index structure. * * @param all include database contents in the representation. During updates, database lookups * must be avoided, as the data structures will be inconsistent. * @return string */ public String toString(final boolean all) { final TokenBuilder tb = new TokenBuilder(); tb.addExt(type).add(" INDEX, '").add(data.meta.name).add("':\n"); final int s = lenList.size(); for (int m = 1; m < s; m++) { final int len = lenList.get(m); if (len == 0) continue; final int[] ids = idsList.get(m); tb.add(" ").addInt(m); if (all) tb.add(", key: \"").add(data.text(data.pre(ids[0]), type == IndexType.TEXT)).add('"'); tb.add(", ids"); if (all) tb.add("/pres"); tb.add(": "); for (int n = 0; n < len; n++) { if (n != 0) tb.add(","); tb.addInt(ids[n]); if (all) tb.add('/').addInt(data.pre(ids[n])); } tb.add("\n"); } return tb.toString(); }
@Override public byte[] info(final MainOptions options) { final TokenBuilder tb = new TokenBuilder(); tb.add(LI_STRUCTURE).add(HASH).add(NL); tb.add(LI_NAMES).add(data.meta.names(type)).add(NL); final IndexStats stats = new IndexStats(options.get(MainOptions.MAXSTAT)); final int s = values.size(); for (int p = 1; p <= s; p++) { final int oc = lenList.get(p); if (oc > 0 && stats.adding(oc)) stats.add(values.key(p), oc); } stats.print(tb); return tb.finish(); }
/** * Optimizes the structures of a database. * * @param data data * @param enforceText enforce creation or deletion of text index * @param enforceAttr enforce creation or deletion of attribute index * @param enforceToken enforce creation or deletion of token index * @param enforceFt enforce creation or deletion of full-text index * @param cmd calling command instance (may be {@code null}) * @throws IOException I/O Exception during index rebuild */ public static void optimize( final Data data, final boolean enforceText, final boolean enforceAttr, final boolean enforceToken, final boolean enforceFt, final Optimize cmd) throws IOException { // initialize structural indexes final MetaData md = data.meta; if (!md.uptodate) { data.paths.init(); data.elemNames.init(); data.attrNames.init(); md.dirty = true; final IntList pars = new IntList(), elms = new IntList(); int n = 0; for (int pre = 0; pre < md.size; ++pre) { final byte kind = (byte) data.kind(pre); final int par = data.parent(pre, kind); while (!pars.isEmpty() && pars.peek() > par) { pars.pop(); elms.pop(); } final int level = pars.size(); if (kind == Data.DOC) { data.paths.put(0, Data.DOC, level); pars.push(pre); elms.push(0); ++n; } else if (kind == Data.ELEM) { final int id = data.nameId(pre); data.elemNames.index(data.elemNames.key(id), null, true); data.paths.put(id, Data.ELEM, level); pars.push(pre); elms.push(id); } else if (kind == Data.ATTR) { final int id = data.nameId(pre); final byte[] val = data.text(pre, false); data.attrNames.index(data.attrNames.key(id), val, true); data.paths.put(id, Data.ATTR, level, val, md); } else { final byte[] val = data.text(pre, true); if (kind == Data.TEXT && level > 1) data.elemNames.index(elms.peek(), val); data.paths.put(0, kind, level, val, md); } if (cmd != null) cmd.pre = pre; } md.ndocs = n; md.uptodate = true; } // rebuild value indexes optimize(IndexType.TEXT, data, md.createtext, md.textindex, enforceText, cmd); optimize(IndexType.ATTRIBUTE, data, md.createattr, md.attrindex, enforceAttr, cmd); optimize(IndexType.TOKEN, data, md.createtoken, md.tokenindex, enforceToken, cmd); optimize(IndexType.FULLTEXT, data, md.createft, md.ftindex, enforceFt, cmd); }
@Override public int costs(final IndexToken it) { return lenList.get(values.id(it.get())); }