/**
* Identify where the end of the middle / payload section ends.
*
* @param row row of black/white values to search
* @return Array, containing index of start of 'end block' and end of 'end block'
* @throws NotFoundException
*/
int[] decodeEnd(BitArray row) throws NotFoundException {
// For convenience, reverse the row and then
// search from 'the start' for the end block
row.reverse();
try {
int endStart = skipWhiteSpace(row);
int[] endPattern = findGuardPattern(row, endStart, END_PATTERN_REVERSED);
// The start & end patterns must be pre/post fixed by a quiet zone. This
// zone must be at least 10 times the width of a narrow line.
// ref: http://www.barcode-1.net/i25code.html
validateQuietZone(row, endPattern[0]);
// Now recalculate the indices of where the 'endblock' starts & stops to
// accommodate
// the reversed nature of the search
int temp = endPattern[0];
endPattern[0] = row.getSize() - endPattern[1];
endPattern[1] = row.getSize() - temp;
return endPattern;
} finally {
// Put the row back the right way.
row.reverse();
}
}
/** Moves the cursor to a free block (either new or existing empty one). */
private void freeBlock() {
final int b = usedPages.nextFree(0);
usedPages.set(b);
readBlock(b);
++used;
++page;
}
// Applies simple sharpening to the row data to improve performance of the 1D Readers.
@Override
public BitArray getBlackRow(int y, BitArray row) throws NotFoundException {
LuminanceSource source = getLuminanceSource();
int width = source.getWidth();
if (row == null || row.getSize() < width) {
row = new BitArray(width);
} else {
row.clear();
}
initArrays(width);
byte[] localLuminances = source.getRow(y, luminances);
int[] localBuckets = buckets;
for (int x = 0; x < width; x++) {
int pixel = localLuminances[x] & 0xff;
localBuckets[pixel >> LUMINANCE_SHIFT]++;
}
int blackPoint = estimateBlackPoint(localBuckets);
int left = localLuminances[0] & 0xff;
int center = localLuminances[1] & 0xff;
for (int x = 1; x < width - 1; x++) {
int right = localLuminances[x + 1] & 0xff;
// A simple -1 4 -1 box filter with a weight of 2.
int luminance = ((center * 4) - left - right) / 2;
if (luminance < blackPoint) {
row.set(x);
}
left = center;
center = right;
}
return row;
}
/**
* Reads a bit from the stream. Returns a boolean (true for 1; false for 0) if a bit is available,
* or throws an EOFException if the end of stream is reached.
*
* @return the bit that was read (true = 1; false = 0)
* @throws IOException if the stream is closed or another I/O error occurs
* @throws EOFException when the next bit cannot be read because the end of stream is reached
*/
protected boolean doReadBit() throws IOException, EOFException {
if (currentIndex
>= bitArray
.length()) // also reads a new byte from underlying stream if needed! (will also check
// for closedness)
throw new EOFException("End of stream reached");
return bitArray.get(currentIndex++);
}
@AndroidIncompatible // OutOfMemoryError
public void testLargeBloomFilterDoesntOverflow() {
long numBits = Integer.MAX_VALUE;
numBits++;
BitArray bitArray = new BitArray(numBits);
assertTrue(
"BitArray.bitSize() must return a positive number, but was " + bitArray.bitSize(),
bitArray.bitSize() > 0);
// Ideally we would also test the bitSize() overflow of this BF, but it runs out of heap space
// BloomFilter.create(Funnels.unencodedCharsFunnel(), 244412641, 1e-11);
}
/**
* Skip all whitespace until we get to the first black line.
*
* @param row row of black/white values to search
* @return index of the first black line.
* @throws NotFoundException Throws exception if no black lines are found in the row
*/
private static int skipWhiteSpace(BitArray row) throws NotFoundException {
int width = row.getSize();
int endStart = 0;
while (endStart < width) {
if (row.get(endStart)) {
break;
}
endStart++;
}
if (endStart == width) {
throw NotFoundException.getNotFoundInstance();
}
return endStart;
}
@Override
public <T> void put(T object, Funnel<? super T> funnel, int numHashFunctions, BitArray bits) {
// TODO(user): when the murmur's shortcuts are implemented, update this code
long hash64 = Hashing.murmur3_128().newHasher().putObject(object, funnel).hash().asLong();
int hash1 = (int) hash64;
int hash2 = (int) (hash64 >>> 32);
for (int i = 1; i <= numHashFunctions; i++) {
int nextHash = hash1 + i * hash2;
if (nextHash < 0) {
nextHash = ~nextHash;
}
// up to here, the code is identical with the next method
bits.set(nextHash % bits.size());
}
}
/**
* Dispatch the character content of a node to an output handler.
*
* <p>The escape setting should be taken care of when outputting to a handler.
*/
public void characters(final int node, SerializationHandler handler) throws TransletException {
int nodeID = getNodeIdent(node);
if (nodeID == RTF_ROOT || nodeID == RTF_TEXT) {
boolean escapeBit = false;
boolean oldEscapeSetting = false;
try {
for (int i = 0; i < _size; i++) {
if (_dontEscape != null) {
escapeBit = _dontEscape.getBit(i);
if (escapeBit) {
oldEscapeSetting = handler.setEscaping(false);
}
}
handler.characters(_textArray[i]);
if (escapeBit) {
handler.setEscaping(oldEscapeSetting);
}
}
} catch (SAXException e) {
throw new TransletException(e);
}
}
}
@Override
public <T> boolean mightContain(
T object, Funnel<? super T> funnel, int numHashFunctions, BitArray bits) {
long hash64 = Hashing.murmur3_128().newHasher().putObject(object, funnel).hash().asLong();
int hash1 = (int) hash64;
int hash2 = (int) (hash64 >>> 32);
for (int i = 1; i <= numHashFunctions; i++) {
int nextHash = hash1 + i * hash2;
if (nextHash < 0) {
nextHash = ~nextHash;
}
// up to here, the code is identical with the previous method
if (!bits.get(nextHash % bits.size())) {
return false;
}
}
return true;
}
public void characters(char[] ch, int offset, int length) throws SAXException {
if (_size >= _textArray.length) {
String[] newTextArray = new String[_textArray.length * 2];
System.arraycopy(_textArray, 0, newTextArray, 0, _textArray.length);
_textArray = newTextArray;
}
if (!_escaping) {
if (_dontEscape == null) {
_dontEscape = new BitArray(8);
}
if (_size >= _dontEscape.size()) _dontEscape.resize(_dontEscape.size() * 2);
_dontEscape.setBit(_size);
}
_textArray[_size++] = new String(ch, offset, length);
}
@Override
boolean clean(final IntObjMap<Var> decl, final BitArray used) {
// [LW] does not fix {@link #vars}
final int len = preExpr.length;
for (int p = 0; p < post.length; p++) {
if (!used.get(post[p].id)) {
preExpr = Array.delete(preExpr, p);
post = Array.delete(post, p--);
}
}
return preExpr.length < len;
}
/**
* @param row row of black/white values to search
* @param rowOffset position to start search
* @param pattern pattern of counts of number of black and white pixels that are being searched
* for as a pattern
* @return start/end horizontal offset of guard pattern, as an array of two ints
* @throws NotFoundException if pattern is not found
*/
private static int[] findGuardPattern(BitArray row, int rowOffset, int[] pattern)
throws NotFoundException {
// TODO: This is very similar to implementation in UPCEANReader. Consider if they can be
// merged to a single method.
int patternLength = pattern.length;
int[] counters = new int[patternLength];
int width = row.getSize();
boolean isWhite = false;
int counterPosition = 0;
int patternStart = rowOffset;
for (int x = rowOffset; x < width; x++) {
boolean pixel = row.get(x);
if (pixel ^ isWhite) {
counters[counterPosition]++;
} else {
if (counterPosition == patternLength - 1) {
if (patternMatchVariance(counters, pattern, MAX_INDIVIDUAL_VARIANCE) < MAX_AVG_VARIANCE) {
return new int[] {patternStart, x};
}
patternStart += counters[0] + counters[1];
for (int y = 2; y < patternLength; y++) {
counters[y - 2] = counters[y];
}
counters[patternLength - 2] = 0;
counters[patternLength - 1] = 0;
counterPosition--;
} else {
counterPosition++;
}
counters[counterPosition] = 1;
isWhite = !isWhite;
}
}
throw NotFoundException.getNotFoundInstance();
}
public void characters(String str) throws SAXException {
// Resize the text array if necessary
if (_size >= _textArray.length) {
String[] newTextArray = new String[_textArray.length * 2];
System.arraycopy(_textArray, 0, newTextArray, 0, _textArray.length);
_textArray = newTextArray;
}
// If the escape setting is false, set the corresponding bit in
// the _dontEscape BitArray.
if (!_escaping) {
// The _dontEscape array is only created when needed.
if (_dontEscape == null) {
_dontEscape = new BitArray(8);
}
// Resize the _dontEscape array if necessary
if (_size >= _dontEscape.size()) _dontEscape.resize(_dontEscape.size() * 2);
_dontEscape.setBit(_size);
}
_textArray[_size++] = str;
}
/**
* The start & end patterns must be pre/post fixed by a quiet zone. This zone must be at least 10
* times the width of a narrow line. Scan back until we either get to the start of the barcode or
* match the necessary number of quiet zone pixels.
*
* <p>Note: Its assumed the row is reversed when using this method to find quiet zone after the
* end pattern.
*
* <p>ref: http://www.barcode-1.net/i25code.html
*
* @param row bit array representing the scanned barcode.
* @param startPattern index into row of the start or end pattern.
* @throws NotFoundException if the quiet zone cannot be found, a ReaderException is thrown.
*/
private void validateQuietZone(BitArray row, int startPattern) throws NotFoundException {
int quietCount = this.narrowLineWidth * 10; // expect to find this many pixels of quiet zone
for (int i = startPattern - 1; quietCount > 0 && i >= 0; i--) {
if (row.get(i)) {
break;
}
quietCount--;
}
if (quietCount != 0) {
// Unable to find the necessary number of quiet zone pixels.
throw NotFoundException.getNotFoundInstance();
}
}
@Override
public synchronized void flush() throws IOException {
for (final Buffer b : bm.all()) if (b.dirty) writeBlock(b);
if (!dirty) return;
try (final DataOutput out = new DataOutput(meta.dbfile(DATATBL + 'i'))) {
out.writeNum(blocks);
out.writeNum(used);
// due to legacy issues, number of blocks is written several times
out.writeNum(blocks);
for (int a = 0; a < blocks; a++) out.writeNum(fpres[a]);
out.writeNum(blocks);
for (int a = 0; a < blocks; a++) out.writeNum(pages[a]);
out.writeLongs(usedPages.toArray());
}
dirty = false;
}
/**
* The (estimated) number of bits left available for reading. Calls atEnd().
*
* @return
*/
public int bitsAvailable() throws IOException {
return bitArray.length() - currentIndex;
}
@Override
public void insert(final int pre, final byte[] entries) {
final int nnew = entries.length;
if (nnew == 0) return;
dirty();
// number of records to be inserted
final int nr = nnew >>> IO.NODEPOWER;
int split = 0;
if (used == 0) {
// special case: insert new data into first block if database is empty
readPage(0);
usedPages.set(0);
++used;
} else if (pre > 0) {
// find the offset within the block where the new records will be inserted
split = cursor(pre - 1) + IO.NODESIZE;
} else {
// all insert operations will add data after first node.
// i.e., there is no "insert before first document" statement
throw Util.notExpected("Insertion at beginning of populated table.");
}
// number of bytes occupied by old records in the current block
final int nold = npre - fpre << IO.NODEPOWER;
// number of bytes occupied by old records which will be moved at the end
final int moved = nold - split;
// special case: all entries fit in the current block
Buffer bf = bm.current();
if (nold + nnew <= IO.BLOCKSIZE) {
Array.move(bf.data, split, nnew, moved);
System.arraycopy(entries, 0, bf.data, split, nnew);
bf.dirty = true;
// increment first pre-values of blocks after the last modified block
for (int i = page + 1; i < used; ++i) fpres[i] += nr;
// update cached variables (fpre is not changed)
npre += nr;
meta.size += nr;
return;
}
// append old entries at the end of the new entries
final byte[] all = new byte[nnew + moved];
System.arraycopy(entries, 0, all, 0, nnew);
System.arraycopy(bf.data, split, all, nnew, moved);
// fill in the current block with new entries
// number of bytes which fit in the first block
int nrem = IO.BLOCKSIZE - split;
if (nrem > 0) {
System.arraycopy(all, 0, bf.data, split, nrem);
bf.dirty = true;
}
// number of new required blocks and remaining bytes
final int req = all.length - nrem;
int needed = req / IO.BLOCKSIZE;
final int remain = req % IO.BLOCKSIZE;
if (remain > 0) {
// check if the last entries can fit in the block after the current one
if (page + 1 < used) {
final int o = occSpace(page + 1) << IO.NODEPOWER;
if (remain <= IO.BLOCKSIZE - o) {
// copy the last records
readPage(page + 1);
bf = bm.current();
System.arraycopy(bf.data, 0, bf.data, remain, o);
System.arraycopy(all, all.length - remain, bf.data, 0, remain);
bf.dirty = true;
// reduce the pre value, since it will be later incremented with nr
fpres[page] -= remain >>> IO.NODEPOWER;
// go back to the previous block
readPage(page - 1);
} else {
// there is not enough space in the block - allocate a new one
++needed;
}
} else {
// this is the last block - allocate a new one
++needed;
}
}
// number of expected blocks: existing blocks + needed block - empty blocks
final int exp = blocks + needed - (blocks - used);
if (exp > fpres.length) {
// resize directory arrays if existing ones are too small
final int ns = Math.max(fpres.length << 1, exp);
fpres = Arrays.copyOf(fpres, ns);
pages = Arrays.copyOf(pages, ns);
}
// make place for the blocks where the new entries will be written
Array.move(fpres, page + 1, needed, used - page - 1);
Array.move(pages, page + 1, needed, used - page - 1);
// write the all remaining entries
while (needed-- > 0) {
freeBlock();
nrem += write(all, nrem);
fpres[page] = fpres[page - 1] + IO.ENTRIES;
pages[page] = (int) bm.current().pos;
}
// increment all fpre values after the last modified block
for (int i = page + 1; i < used; ++i) fpres[i] += nr;
meta.size += nr;
// update cached variables
fpre = fpres[page];
npre = page + 1 < used && fpres[page + 1] < meta.size ? fpres[page + 1] : meta.size;
}
@Override
public void delete(final int pre, final int nr) {
if (nr == 0) return;
dirty();
// get first block
cursor(pre);
// some useful variables to make code more readable
int from = pre - fpre;
final int last = pre + nr;
// check if all entries are in current block: handle and return
if (last - 1 < npre) {
final Buffer bf = bm.current();
copy(bf.data, from + nr, bf.data, from, npre - last);
updatePre(nr);
// if whole block was deleted, remove it from the index
if (npre == fpre) {
// mark the block as empty
usedPages.clear(pages[page]);
Array.move(fpres, page + 1, -1, used - page - 1);
Array.move(pages, page + 1, -1, used - page - 1);
--used;
readPage(page);
}
return;
}
// handle blocks whose entries are to be deleted entirely
// first count them
int unused = 0;
while (npre < last) {
if (from == 0) {
++unused;
// mark the blocks as empty; range clear cannot be used because the
// blocks may not be consecutive
usedPages.clear(pages[page]);
}
setPage(page + 1);
from = 0;
}
// if the last block is empty, clear the corresponding bit
readBlock(pages[page]);
final Buffer bf = bm.current();
if (npre == last) {
usedPages.clear((int) bf.pos);
++unused;
if (page < used - 1) readPage(page + 1);
else ++page;
} else {
// delete entries at beginning of current (last) block
copy(bf.data, last - fpre, bf.data, 0, npre - last);
}
// now remove them from the index
if (unused > 0) {
Array.move(fpres, page, -unused, used - page);
Array.move(pages, page, -unused, used - page);
used -= unused;
page -= unused;
}
// update index entry for this block
fpres[page] = pre;
fpre = pre;
updatePre(nr);
}
