/**
* Writes full-text data for a single token to disk.<br>
* Format: {@code score? pre1 pos1 pre2 pos2 ... (0 score)? pre...}
*
* @param out DataOutput for disk access
* @param vpre compressed pre values
* @param vpos compressed pos values
* @throws IOException IOException
*/
final void writeFTData(final DataOutput out, final byte[] vpre, final byte[] vpos)
throws IOException {
int np = 4, pp = 4, lp = -1, lu = -1;
final int ns = Num.size(vpre);
while (np < ns) {
if (scm > 0) {
final int p = Num.get(vpre, np);
if (lp != p) {
// new pre value: find document root
int u = unit.sortedIndexOf(p);
if (u < 0) u = -u - 1;
if (lu != u) {
// new unit: store scoring
final int s = Scoring.tfIDF(freq.get(fc++), maxfreq[u], unit.size(), ntoken[token]);
if (max < s) max = s;
if (min > s) min = s;
if (np != 4) out.write(0);
out.writeNum(s);
lu = u;
}
lp = p;
}
}
// full-text data is stored here, with -scoreU, pre1, pos1, ...,
// -scoreU, preU, posU
for (final int l = np + Num.length(vpre, np); np < l; ++np) out.write(vpre[np]);
for (final int l = pp + Num.length(vpos, pp); pp < l; ++pp) out.write(vpos[pp]);
}
++token;
}
@Override
protected void addText(final byte[] value, final int dist, final byte kind) throws IOException {
tout.write1(kind);
tout.write2(0);
tout.write5(textOff(value, true));
tout.write4(dist);
tout.write4(meta.size++);
}
@Override
protected void addDoc(final byte[] value) throws IOException {
tout.write1(Data.DOC);
tout.write2(0);
tout.write5(textOff(value, true));
tout.write4(0);
tout.write4(meta.size++);
}
@Override
protected void addAttr(final int name, final byte[] value, final int dist, final int uri)
throws IOException {
tout.write1(dist << 3 | Data.ATTR);
tout.write2(name);
tout.write5(textOff(value, false));
tout.write4(uri);
tout.write4(meta.size++);
}
/**
* Calculates the text offset and writes the text value.
*
* @param value value to be inlined
* @param text text/attribute flag
* @return inline value or text position
* @throws IOException I/O exception
*/
private long textOff(final byte[] value, final boolean text) throws IOException {
// inline integer values...
final long v = Token.toSimpleInt(value);
if (v != Integer.MIN_VALUE) return v | IO.OFFNUM;
// store text
final DataOutput store = text ? xout : vout;
final long off = store.size();
final byte[] val = COMP.get().pack(value);
store.writeToken(val);
return val == value ? off : off | IO.OFFCOMP;
}
@Override
public void close() throws IOException {
if (closed) return;
closed = true;
if (tout != null) tout.close();
if (xout != null) xout.close();
if (vout != null) vout.close();
if (sout != null) sout.close();
parser.close();
tout = null;
xout = null;
vout = null;
sout = null;
}
/**
* Writes the node to the specified output stream.
*
* @param out output stream
* @throws IOException I/O exception
*/
void write(final DataOutput out) throws IOException {
out.writeNum(name);
out.write1(kind);
out.writeNum(0);
out.writeNum(children.length);
out.writeDouble(1);
// update leaf flag
boolean leaf = stats.isLeaf();
for (final PathNode child : children) {
leaf &= child.kind == Data.TEXT || child.kind == Data.ATTR;
}
stats.setLeaf(leaf);
stats.write(out);
for (final PathNode child : children) child.write(out);
}
/**
* Write the map to the specified file.
*
* @param file file to write to
* @throws IOException I/O error while writing to the file
*/
public void write(final IOFile file) throws IOException {
try (final DataOutput out = new DataOutput(file)) {
out.writeNum(baseid);
out.writeNum(rows);
out.writeNums(pres);
out.writeNums(fids);
out.writeNums(nids);
out.writeNums(incs);
out.writeNums(oids);
}
}
@Override
public synchronized void flush() throws IOException {
for (final Buffer b : bm.all()) if (b.dirty) writeBlock(b);
if (!dirty) return;
try (final DataOutput out = new DataOutput(meta.dbfile(DATATBL + 'i'))) {
out.writeNum(blocks);
out.writeNum(used);
// due to legacy issues, number of blocks is written several times
out.writeNum(blocks);
for (int a = 0; a < blocks; a++) out.writeNum(fpres[a]);
out.writeNum(blocks);
for (int a = 0; a < blocks; a++) out.writeNum(pages[a]);
out.writeLongs(usedPages.toArray());
}
dirty = false;
}
@Override
protected void addElem(
final int dist, final int name, final int asize, final int uri, final boolean ne)
throws IOException {
tout.write1(asize << 3 | Data.ELEM);
tout.write2((ne ? 1 << 15 : 0) | name);
tout.write1(uri);
tout.write4(dist);
tout.write4(asize);
tout.write4(meta.size++);
if (Prop.debug && (c++ & 0x7FFFF) == 0) Util.err(".");
}
/**
* Writes all meta data to disk.
*
* @throws IOException I/O exception
*/
private void write() throws IOException {
if (meta.dirty) {
try (final DataOutput out = new DataOutput(meta.dbfile(DATAINF))) {
meta.write(out);
out.writeToken(token(DBTAGS));
elemNames.write(out);
out.writeToken(token(DBATTS));
attrNames.write(out);
out.writeToken(token(DBPATH));
paths.write(out);
out.writeToken(token(DBNS));
nspaces.write(out);
out.writeToken(token(DBDOCS));
resources.write(out);
out.write(0);
}
if (idmap != null) idmap.write(meta.dbfile(DATAIDP));
meta.dirty = false;
}
}
/**
* Writes a single node to disk.
*
* @param out output stream
* @throws IOException I/O exception
*/
void write(final DataOutput out) throws IOException {
out.writeNum(pr);
out.writeNums(values);
out.writeNum(sz);
for (int c = 0; c < sz; ++c) children[c].write(out);
}
/**
* Writes the path summary to the specified output.
*
* @param out output stream
* @throws IOException I/O exception
*/
public void write(final DataOutput out) throws IOException {
out.writeBool(root != null);
if (root != null) root.write(out);
}
@Override
protected void setSize(final int pre, final int size) throws IOException {
sout.writeNum(pre);
sout.writeNum(size);
++ssize;
}
