@Test
public void flipTest6A() {
System.out.println("FlipTest6A");
final MutableRoaringBitmap rb = new MutableRoaringBitmap();
final MutableRoaringBitmap rb1 = MutableRoaringBitmap.flip(rb, 100000, 132000);
final MutableRoaringBitmap rb2 = MutableRoaringBitmap.flip(rb1, 99000, 2 * 65536);
final int rbcard = rb2.getCardinality();
Assert.assertEquals(1928, rbcard);
final BitSet bs = new BitSet();
for (int i = 99000; i < 100000; ++i) bs.set(i);
for (int i = 2 * 65536; i < 132000; ++i) bs.set(i);
Assert.assertTrue(equals(bs, rb2));
}
public static void main(String[] args) {
// Initialized to false by default
BitSet bitmap = new BitSet(10000000);
BufferedReader br = null;
String line = null;
try {
br = new BufferedReader(new FileReader(args[0]));
// Setting the bitmap for the input values
while ((line = br.readLine()) != null) {
int num = Integer.parseInt(line);
bitmap.set(num);
}
// Outputting the sorted value
for (int i = 0; i < bitmap.length(); i++) {
if (bitmap.get(i)) {
System.out.println(bitmap.get(i));
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
@Override
public void prob1step(
BitSet subset, BitSet u, BitSet v, boolean forall1, boolean forall2, BitSet result) {
boolean b1, b2, b3;
for (int i : new IterableStateSet(subset, numStates)) {
b1 = forall1; // there exists or for all player 1 choices
for (DistributionSet distrs : trans.get(i)) {
b2 = forall2; // there exists or for all player 2 choices
for (Distribution distr : distrs) {
b3 = distr.containsOneOf(v) && distr.isSubsetOf(u);
if (forall2) {
if (!b3) b2 = false;
} else {
if (b3) b2 = true;
}
}
if (forall1) {
if (!b2) b1 = false;
} else {
if (b2) b1 = true;
}
}
result.set(i, b1);
}
}
/**
* Updates the interal bitset from <code>iterator</code>. This will set <code>validMask</code> to
* true if <code>iterator</code> is non-null.
*/
private void updateMask(AttributedCharacterIterator iterator) {
if (iterator != null) {
validMask = true;
this.iterator = iterator;
// Update the literal mask
if (literalMask == null) {
literalMask = new BitSet();
} else {
for (int counter = literalMask.length() - 1; counter >= 0; counter--) {
literalMask.clear(counter);
}
}
iterator.first();
while (iterator.current() != CharacterIterator.DONE) {
Map attributes = iterator.getAttributes();
boolean set = isLiteral(attributes);
int start = iterator.getIndex();
int end = iterator.getRunLimit();
while (start < end) {
if (set) {
literalMask.set(start);
} else {
literalMask.clear(start);
}
start++;
}
iterator.setIndex(start);
}
}
}
public TOE(String s, int Nc, int Mc) {
this(Nc, Mc);
char[] C = s.toCharArray();
for (int i = 0; i < N; i++)
for (int j = 0; j < M; j++)
if (C[i * M + j] == '1') G.set(i * M + j);
else G.clear(i * M + j);
}
@Test
public void flipTest7A() { // within 1 word, first container
System.out.println("FlipTest7A");
final MutableRoaringBitmap rb = new MutableRoaringBitmap();
final MutableRoaringBitmap rb1 = MutableRoaringBitmap.flip(rb, 650, 132000);
final MutableRoaringBitmap rb2 = MutableRoaringBitmap.flip(rb1, 648, 651);
final int rbcard = rb2.getCardinality();
// 648, 649, 651-131999
Assert.assertEquals(132000 - 651 + 2, rbcard);
final BitSet bs = new BitSet();
bs.set(648);
bs.set(649);
for (int i = 651; i < 132000; ++i) bs.set(i);
Assert.assertTrue(equals(bs, rb2));
}
@Test
public void flipTest6() { // fits evenly on big end, multiple containers
System.out.println("FlipTest6");
final MutableRoaringBitmap rb = new MutableRoaringBitmap();
rb.flip(100000, 132000);
rb.flip(99000, 2 * 65536);
final int rbcard = rb.getCardinality();
// 99000 to 99999 are 1000 1s
// 131072 to 131999 are 928 1s
Assert.assertEquals(1928, rbcard);
final BitSet bs = new BitSet();
for (int i = 99000; i < 100000; ++i) bs.set(i);
for (int i = 2 * 65536; i < 132000; ++i) bs.set(i);
Assert.assertTrue(equals(bs, rb));
}
/**
* setup BitMap
*
* @exception ISOException
*/
public void recalcBitMap() throws ISOException {
if (!dirty) return;
int mf = Math.min(getMaxField(), 192);
BitSet bmap = new BitSet(((mf + 62) >> 6) << 6);
for (int i = 1; i <= mf; i++) if ((fields.get(i)) != null) bmap.set(i);
set(new ISOBitMap(-1, bmap));
dirty = false;
}
protected void doDamageRemoveTest(SimulatedArchivalUnit sau) throws Exception {
/* Cache the file again; this time the damage should be gone */
String file = sau.getUrlRoot() + DAMAGED_CACHED_URL;
UrlCacher uc = sau.makeUrlCacher(file);
BitSet fetchFlags = new BitSet();
fetchFlags.set(UrlCacher.REFETCH_FLAG);
uc.setFetchFlags(fetchFlags);
uc.cache();
checkUrlContent(sau, DAMAGED_CACHED_URL, 2, 2, 2, false, false);
}
@Test
public void flipTest5() { // fits evenly on small end, multiple
// containers
System.out.println("FlipTest5");
final MutableRoaringBitmap rb = new MutableRoaringBitmap();
rb.flip(100000, 132000);
rb.flip(65536, 120000);
final int rbcard = rb.getCardinality();
// 65536 to 99999 are 1s
// 120000 to 131999
Assert.assertEquals(46464, rbcard);
final BitSet bs = new BitSet();
for (int i = 65536; i < 100000; ++i) bs.set(i);
for (int i = 120000; i < 132000; ++i) bs.set(i);
Assert.assertTrue(equals(bs, rb));
}
@Test
public void flipTest4() { // fits evenly on both ends
System.out.println("FlipTest4");
final MutableRoaringBitmap rb = new MutableRoaringBitmap();
rb.flip(100000, 200000); // got 100k-199999
rb.flip(65536, 4 * 65536);
final int rbcard = rb.getCardinality();
// 65536 to 99999 are 1s
// 200000 to 262143 are 1s: total card
Assert.assertEquals(96608, rbcard);
final BitSet bs = new BitSet();
for (int i = 65536; i < 100000; ++i) bs.set(i);
for (int i = 200000; i < 262144; ++i) bs.set(i);
Assert.assertTrue(equals(bs, rb));
}
static void solve() {
int r = in.nextInt();
int c = in.nextInt();
int sy = in.nextInt() - 1;
int sx = in.nextInt() - 1;
int gy = in.nextInt() - 1;
int gx = in.nextInt() - 1;
char[][] t = new char[r][c];
for (int i = 0; i < r; i++) {
t[i] = in.next().toCharArray();
}
ArrayDeque<int[]> que = new ArrayDeque<>();
BitSet visited = new BitSet();
que.add(new int[] {sy, sx, 0});
visited.set(sy * c + sx);
int[] dx = {0, 1, 0, -1};
int[] dy = {1, 0, -1, 0};
int[][] dist = new int[r][c];
while (!que.isEmpty()) {
int[] p = que.pollFirst();
int cy = p[0];
int cx = p[1];
int d = p[2];
dist[cy][cx] = d;
for (int i = 0; i < 4; i++) {
int ny = cy + dy[i];
int nx = cx + dx[i];
if (ny < 0 || nx < 0 || r <= ny || c <= nx) continue;
if (visited.get(ny * c + nx) || t[ny][nx] == '#') continue;
que.add(new int[] {ny, nx, d + 1});
visited.set(ny * c + nx);
}
}
out.println(dist[gy][gx]);
}
@Test
public void flipTest1() {
final MutableRoaringBitmap rb = new MutableRoaringBitmap();
rb.flip(100000, 200000); // in-place on empty bitmap
final int rbcard = rb.getCardinality();
Assert.assertEquals(100000, rbcard);
final BitSet bs = new BitSet();
for (int i = 100000; i < 200000; ++i) bs.set(i);
Assert.assertTrue(equals(bs, rb));
}
@Test
public void flipTest1A() {
final MutableRoaringBitmap rb = new MutableRoaringBitmap();
final MutableRoaringBitmap rb1 = MutableRoaringBitmap.flip(rb, 100000, 200000);
final int rbcard = rb1.getCardinality();
Assert.assertEquals(100000, rbcard);
Assert.assertEquals(0, rb.getCardinality());
final BitSet bs = new BitSet();
Assert.assertTrue(equals(bs, rb)); // still empty?
for (int i = 100000; i < 200000; ++i) bs.set(i);
Assert.assertTrue(equals(bs, rb1));
}
@Test
public void flipTest2A() {
final MutableRoaringBitmap rb = new MutableRoaringBitmap();
final MutableRoaringBitmap rb1 = MutableRoaringBitmap.flip(rb, 100000, 100000);
rb.add(1); // will not affect rb1 (no shared container)
final int rbcard = rb1.getCardinality();
Assert.assertEquals(0, rbcard);
Assert.assertEquals(1, rb.getCardinality());
final BitSet bs = new BitSet();
Assert.assertTrue(equals(bs, rb1));
bs.set(1);
Assert.assertTrue(equals(bs, rb));
}
@Test
public void flipTest3() {
final MutableRoaringBitmap rb = new MutableRoaringBitmap();
rb.flip(100000, 200000); // got 100k-199999
rb.flip(100000, 199991); // give back 100k-199990
final int rbcard = rb.getCardinality();
Assert.assertEquals(9, rbcard);
final BitSet bs = new BitSet();
for (int i = 199991; i < 200000; ++i) bs.set(i);
Assert.assertTrue(equals(bs, rb));
}
@Test
public void flipTest3A() {
System.out.println("FlipTest3A");
final MutableRoaringBitmap rb = new MutableRoaringBitmap();
final MutableRoaringBitmap rb1 = MutableRoaringBitmap.flip(rb, 100000, 200000);
final MutableRoaringBitmap rb2 = MutableRoaringBitmap.flip(rb1, 100000, 199991);
final int rbcard = rb2.getCardinality();
Assert.assertEquals(9, rbcard);
final BitSet bs = new BitSet();
for (int i = 199991; i < 200000; ++i) bs.set(i);
Assert.assertTrue(equals(bs, rb2));
}
/**
* It adds a sequence from an array of string that we have to interpret
*
* @param integers
* @param sequenceID
*/
public void addSequence(String[] integers, int sequenceID) {
long timestamp = -1;
Sequence sequence = new Sequence(sequences.size());
sequence.setID(sequenceID);
Itemset itemset = new Itemset();
int inicio = 0;
Map<Item, Boolean> counted = new HashMap<Item, Boolean>();
for (int i = inicio; i < integers.length; i++) {
if (integers[i].codePointAt(0) == '<') { // Timestamp
String value = integers[i].substring(1, integers[i].length() - 1);
timestamp = Long.parseLong(value);
itemset.setTimestamp(timestamp);
} else if (integers[i].equals("-1")) { // end of an itemset
long time = itemset.getTimestamp() + 1;
sequence.addItemset(itemset);
itemset = new Itemset();
itemset.setTimestamp(time);
} else if (integers[i].equals("-2")) { // end of a sequence
sequences.add(sequence);
} else {
// extract the value for an item
Item item = itemFactory.getItem(Integer.parseInt(integers[i]));
if (counted.get(item) == null) {
counted.put(item, Boolean.TRUE);
BitSet appearances = frequentItems.get(item);
if (appearances == null) {
appearances = new BitSet();
frequentItems.put(item, appearances);
}
appearances.set(sequence.getId());
}
itemset.addItem(item);
}
}
}
/**
* Generates a zero crossing mask for a 2D function sets a Bitset object to 1 where a zero
* crossing is detected.
*
* @param xDim x dimension of image
* @param yDim y dimension of image
* @param buffer array in which to find zero crossing
* @param level level to generate zero crossings at
* @param detectionType the type of zero crossing method to use
* @return Bitset representing zero crossings
*/
public static BitSet genLevelMask(
int xDim, int yDim, float[] buffer, float level, int detectionType) {
int i0, i1, i2, i3;
float x0, x1, x2, x3;
int i, j;
int indexY;
int xxDim = xDim - 1;
int yyDim = yDim - 1;
BitSet edgeImage = new BitSet(xDim * yDim);
for (j = 0; j < yyDim; j++) {
indexY = j * xDim;
for (i = 0; i < xxDim; i++) {
i0 = indexY + i;
if (detectionType == MARCHING_SQUARES) {
i1 = i0 + 1;
i2 = i0 + xDim;
i3 = i0 + 1 + xDim;
x0 = buffer[i0];
x1 = buffer[i1];
x2 = buffer[i2];
x3 = buffer[i3];
if ((x0 >= level)
&& (x1 >= level)
&& (x2 >= level)
&& (x3 >= level)) { // case 0 - no edge
} else if ((x0 >= level) && (x1 >= level) && (x2 < level) && (x3 >= level)) {
// case 1 - edge in the lower left
edgeImage.set(i2);
} else if ((x0 >= level) && (x1 >= level) && (x2 >= level) && (x3 < level)) {
// case 2 - edge in the lower right
edgeImage.set(i3);
} else if ((x0 >= level) && (x1 >= level) && (x2 < level) && (x3 < level)) {
// case 3 - edge horizontally
edgeImage.set(i2);
edgeImage.set(i3);
} else if ((x0 >= level) && (x1 < level) && (x2 >= level) && (x3 >= level)) {
// case 4 - edge in the upper right
edgeImage.set(i1);
} else if ((x0 >= level) && (x1 < level) && (x2 < level) && (x3 >= level)) {
// case 5 - ambiguous case; either edge in upper right and lower left or
// edge that goes from the upper right to the lower left
edgeImage.set(i1);
edgeImage.set(i2);
} else if ((x0 >= level) && (x1 < level) && (x2 >= level) && (x3 < level)) {
// case 6 - edge going vertically along the right
edgeImage.set(i1);
edgeImage.set(i3);
} else if ((x0 >= level) && (x1 < level) && (x2 < level) && (x3 < level)) {
// case 7 - edge in the upper left
edgeImage.set(i0);
} else if ((x0 < level) && (x1 >= level) && (x2 >= level) && (x3 >= level)) {
// case 8 - edge in the upper left
edgeImage.set(i0);
} else if ((x0 < level) && (x1 >= level) && (x2 < level) && (x3 >= level)) {
// case 9 - edge going vertically along the left
edgeImage.set(i0);
edgeImage.set(i2);
} else if ((x0 < level) && (x1 >= level) && (x2 >= level) && (x3 < level)) {
// case 10 - ambiguous case; either edge in upper left and lower right or
// edge that goes from the upper left to the lower right
edgeImage.set(i0);
edgeImage.set(i3);
} else if ((x0 < level) && (x1 >= level) && (x2 < level) && (x3 < level)) {
// case 11 - edge in the upper right
edgeImage.set(i1);
} else if ((x0 < level) && (x1 < level) && (x2 >= level) && (x3 >= level)) {
// case 12 - edge going horizontally along the top
edgeImage.set(i0);
edgeImage.set(i1);
} else if ((x0 < level) && (x1 < level) && (x2 < level) && (x3 >= level)) {
// case 13 - edge in the lower right
edgeImage.set(i3);
} else if ((x0 < level) && (x1 < level) && (x2 >= level) && (x3 < level)) {
// case 14 - edge in the lower left
edgeImage.set(i2);
} else if ((x0 < level)
&& (x1 < level)
&& (x2 < level)
&& (x3 < level)) { // case 15 - no edge
}
} else if (detectionType == NEGATIVE_EDGES) {
if (buffer[i0] <= 0) {
edgeImage.set(i0);
}
} else if (detectionType == OLD_DETECTION) {
i1 = i0 + 1;
i2 = i0 + xDim;
i3 = i0 + 1 + xDim;
x0 = buffer[i0];
x1 = buffer[i1];
x2 = buffer[i2];
x3 = buffer[i3];
if ((x0 >= level) && (x1 >= level) && (x2 >= level) && (x3 >= level)) {
edgeImage.clear(i0);
} else if ((x0 <= level) && (x1 <= level) && (x2 <= level) && (x3 <= level)) {
edgeImage.clear(i0);
} else {
edgeImage.set(i0);
}
}
}
}
return edgeImage;
}
/** Simple test program */
public static void main(String args[]) {
STPGModelChecker mc;
STPGAbstrSimple stpg;
DistributionSet set;
Distribution distr;
// ModelCheckerResult res;
BitSet target;
// Simple example: Create and solve the stochastic game from:
// Mark Kattenbelt, Marta Kwiatkowska, Gethin Norman, David Parker
// A Game-based Abstraction-Refinement Framework for Markov Decision Processes
// Formal Methods in System Design 36(3): 246-280, 2010
try {
// Build game
stpg = new STPGAbstrSimple();
stpg.addStates(4);
// State 0 (s_0)
set = stpg.newDistributionSet(null);
distr = new Distribution();
distr.set(1, 1.0);
set.add(distr);
stpg.addDistributionSet(0, set);
// State 1 (s_1,s_2,s_3)
set = stpg.newDistributionSet(null);
distr = new Distribution();
distr.set(2, 1.0);
set.add(distr);
distr = new Distribution();
distr.set(1, 1.0);
set.add(distr);
stpg.addDistributionSet(1, set);
set = stpg.newDistributionSet(null);
distr = new Distribution();
distr.set(2, 0.5);
distr.set(3, 0.5);
set.add(distr);
distr = new Distribution();
distr.set(3, 1.0);
set.add(distr);
stpg.addDistributionSet(1, set);
// State 2 (s_4,s_5)
set = stpg.newDistributionSet(null);
distr = new Distribution();
distr.set(2, 1.0);
set.add(distr);
stpg.addDistributionSet(2, set);
// State 3 (s_6)
set = stpg.newDistributionSet(null);
distr = new Distribution();
distr.set(3, 1.0);
set.add(distr);
stpg.addDistributionSet(3, set);
// Print game
System.out.println(stpg);
// Model check
mc = new STPGModelChecker(null);
// mc.setVerbosity(2);
target = new BitSet();
target.set(3);
stpg.exportToDotFile("stpg.dot", target);
System.out.println("min min: " + mc.computeReachProbs(stpg, target, true, true).soln[0]);
System.out.println("max min: " + mc.computeReachProbs(stpg, target, false, true).soln[0]);
System.out.println("min max: " + mc.computeReachProbs(stpg, target, true, false).soln[0]);
System.out.println("max max: " + mc.computeReachProbs(stpg, target, false, false).soln[0]);
} catch (PrismException e) {
System.out.println(e);
}
}
public void compactScan(CharBuffer charBuffer, long size) {
for (int i = 0; i < size; i++) {
final char ch = charBuffer.get(i);
compactIndexUsed.set(ch);
}
}
public void rTest(final int N) {
System.out.println("rtest N=" + N);
for (int gap = 1; gap <= 65536; gap *= 2) {
final BitSet bs1 = new BitSet();
final MutableRoaringBitmap rb1 = new MutableRoaringBitmap();
for (int x = 0; x <= N; x += gap) {
bs1.set(x);
rb1.add(x);
}
if (bs1.cardinality() != rb1.getCardinality()) throw new RuntimeException("different card");
if (!equals(bs1, rb1)) throw new RuntimeException("basic bug");
for (int offset = 1; offset <= gap; offset *= 2) {
final BitSet bs2 = new BitSet();
final MutableRoaringBitmap rb2 = new MutableRoaringBitmap();
for (int x = 0; x <= N; x += gap) {
bs2.set(x + offset);
rb2.add(x + offset);
}
if (bs2.cardinality() != rb2.getCardinality()) throw new RuntimeException("different card");
if (!equals(bs2, rb2)) throw new RuntimeException("basic bug");
BitSet clonebs1;
// testing AND
clonebs1 = (BitSet) bs1.clone();
clonebs1.and(bs2);
if (!equals(clonebs1, MutableRoaringBitmap.and(rb1, rb2)))
throw new RuntimeException("bug and");
{
final MutableRoaringBitmap t = rb1.clone();
t.and(rb2);
if (!equals(clonebs1, t)) throw new RuntimeException("bug inplace and");
if (!t.equals(MutableRoaringBitmap.and(rb1, rb2))) {
System.out.println(
t.highLowContainer.getContainerAtIndex(0).getClass().getCanonicalName());
System.out.println(
MutableRoaringBitmap.and(rb1, rb2)
.highLowContainer
.getContainerAtIndex(0)
.getClass()
.getCanonicalName());
throw new RuntimeException("bug inplace and");
}
}
// testing OR
clonebs1 = (BitSet) bs1.clone();
clonebs1.or(bs2);
if (!equals(clonebs1, MutableRoaringBitmap.or(rb1, rb2)))
throw new RuntimeException("bug or");
{
final MutableRoaringBitmap t = rb1.clone();
t.or(rb2);
if (!equals(clonebs1, t)) throw new RuntimeException("bug or");
if (!t.equals(MutableRoaringBitmap.or(rb1, rb2))) throw new RuntimeException("bug or");
if (!t.toString().equals(MutableRoaringBitmap.or(rb1, rb2).toString()))
throw new RuntimeException("bug or");
}
// testing XOR
clonebs1 = (BitSet) bs1.clone();
clonebs1.xor(bs2);
if (!equals(clonebs1, MutableRoaringBitmap.xor(rb1, rb2))) {
throw new RuntimeException("bug xor");
}
{
final MutableRoaringBitmap t = rb1.clone();
t.xor(rb2);
if (!equals(clonebs1, t)) throw new RuntimeException("bug xor");
if (!t.equals(MutableRoaringBitmap.xor(rb1, rb2))) {
System.out.println(t);
System.out.println(MutableRoaringBitmap.xor(rb1, rb2));
System.out.println(
Arrays.equals(t.toArray(), MutableRoaringBitmap.xor(rb1, rb2).toArray()));
throw new RuntimeException("bug xor");
}
}
// testing NOTAND
clonebs1 = (BitSet) bs1.clone();
clonebs1.andNot(bs2);
if (!equals(clonebs1, MutableRoaringBitmap.andNot(rb1, rb2))) {
throw new RuntimeException("bug andnot");
}
clonebs1 = (BitSet) bs2.clone();
clonebs1.andNot(bs1);
if (!equals(clonebs1, MutableRoaringBitmap.andNot(rb2, rb1))) {
throw new RuntimeException("bug andnot");
}
{
final MutableRoaringBitmap t = rb2.clone();
t.andNot(rb1);
if (!equals(clonebs1, t)) {
throw new RuntimeException("bug inplace andnot");
}
final MutableRoaringBitmap g = MutableRoaringBitmap.andNot(rb2, rb1);
if (!equals(clonebs1, g)) {
throw new RuntimeException("bug andnot");
}
if (!t.equals(g)) throw new RuntimeException("bug");
}
clonebs1 = (BitSet) bs1.clone();
clonebs1.andNot(bs2);
if (!equals(clonebs1, MutableRoaringBitmap.andNot(rb1, rb2))) {
throw new RuntimeException("bug andnot");
}
{
final MutableRoaringBitmap t = rb1.clone();
t.andNot(rb2);
if (!equals(clonebs1, t)) {
throw new RuntimeException("bug andnot");
}
final MutableRoaringBitmap g = MutableRoaringBitmap.andNot(rb1, rb2);
if (!equals(clonebs1, g)) {
throw new RuntimeException("bug andnot");
}
if (!t.equals(g)) throw new RuntimeException("bug");
}
}
}
}
/**
* Dumps an {@link FST} to a GraphViz's <code>dot</code> language description for visualization.
* Example of use:
*
* <pre>
* PrintStream ps = new PrintStream("out.dot");
* fst.toDot(ps);
* ps.close();
* </pre>
*
* and then, from command line:
*
* <pre>
* dot -Tpng -o out.png out.dot
* </pre>
*
* <p>Note: larger FSTs (a few thousand nodes) won't even render, don't bother.
*
* @param sameRank If <code>true</code>, the resulting <code>dot</code> file will try to order
* states in layers of breadth-first traversal. This may mess up arcs, but makes the output
* FST's structure a bit clearer.
* @param labelStates If <code>true</code> states will have labels equal to their offsets in their
* binary format. Expands the graph considerably.
* @see "http://www.graphviz.org/"
*/
public static <T> void toDot(FST<T> fst, Writer out, boolean sameRank, boolean labelStates)
throws IOException {
final String expandedNodeColor = "blue";
// This is the start arc in the automaton (from the epsilon state to the first state
// with outgoing transitions.
final FST.Arc<T> startArc = fst.getFirstArc(new FST.Arc<T>());
// A queue of transitions to consider for the next level.
final List<FST.Arc<T>> thisLevelQueue = new ArrayList<FST.Arc<T>>();
// A queue of transitions to consider when processing the next level.
final List<FST.Arc<T>> nextLevelQueue = new ArrayList<FST.Arc<T>>();
nextLevelQueue.add(startArc);
// A list of states on the same level (for ranking).
final List<Integer> sameLevelStates = new ArrayList<Integer>();
// A bitset of already seen states (target offset).
final BitSet seen = new BitSet();
seen.set(startArc.target);
// Shape for states.
final String stateShape = "circle";
// Emit DOT prologue.
out.write("digraph FST {\n");
out.write(" rankdir = LR; splines=true; concentrate=true; ordering=out; ranksep=2.5; \n");
if (!labelStates) {
out.write(" node [shape=circle, width=.2, height=.2, style=filled]\n");
}
emitDotState(out, "initial", "point", "white", "");
emitDotState(
out,
Integer.toString(startArc.target),
stateShape,
fst.isExpandedTarget(startArc) ? expandedNodeColor : null,
"");
out.write(" initial -> " + startArc.target + "\n");
final T NO_OUTPUT = fst.outputs.getNoOutput();
int level = 0;
while (!nextLevelQueue.isEmpty()) {
// we could double buffer here, but it doesn't matter probably.
thisLevelQueue.addAll(nextLevelQueue);
nextLevelQueue.clear();
level++;
out.write("\n // Transitions and states at level: " + level + "\n");
while (!thisLevelQueue.isEmpty()) {
final FST.Arc<T> arc = thisLevelQueue.remove(thisLevelQueue.size() - 1);
if (fst.targetHasArcs(arc)) {
// scan all arcs
final int node = arc.target;
fst.readFirstTargetArc(arc, arc);
while (true) {
// Emit the unseen state and add it to the queue for the next level.
if (arc.target >= 0 && !seen.get(arc.target)) {
final boolean isExpanded = fst.isExpandedTarget(arc);
emitDotState(
out,
Integer.toString(arc.target),
stateShape,
isExpanded ? expandedNodeColor : null,
labelStates ? Integer.toString(arc.target) : "");
seen.set(arc.target);
nextLevelQueue.add(new FST.Arc<T>().copyFrom(arc));
sameLevelStates.add(arc.target);
}
String outs;
if (arc.output != NO_OUTPUT) {
outs = "/" + fst.outputs.outputToString(arc.output);
} else {
outs = "";
}
final String cl;
if (arc.label == FST.END_LABEL) {
cl = "~";
} else {
cl = printableLabel(arc.label);
}
out.write(" " + node + " -> " + arc.target + " [label=\"" + cl + outs + "\"]\n");
// Break the loop if we're on the last arc of this state.
if (arc.isLast()) {
break;
}
fst.readNextArc(arc);
}
}
}
// Emit state ranking information.
if (sameRank && sameLevelStates.size() > 1) {
out.write(" {rank=same; ");
for (int state : sameLevelStates) {
out.write(state + "; ");
}
out.write(" }\n");
}
sameLevelStates.clear();
}
// Emit terminating state (always there anyway).
out.write(" -1 [style=filled, color=black, shape=circle, label=\"\"]\n\n");
out.write(" {rank=sink; -1 }\n");
out.write("}\n");
out.flush();
}
public void testFormat() throws Exception {
JobConf job = new JobConf(conf);
FileSystem fs = FileSystem.getLocal(conf);
Path dir = new Path(System.getProperty("test.build.data", ".") + "/mapred");
Path file = new Path(dir, "test.seq");
Reporter reporter = Reporter.NULL;
int seed = new Random().nextInt();
// LOG.info("seed = "+seed);
Random random = new Random(seed);
fs.delete(dir, true);
FileInputFormat.setInputPaths(job, dir);
// for a variety of lengths
for (int length = 0; length < MAX_LENGTH; length += random.nextInt(MAX_LENGTH / 10) + 1) {
// LOG.info("creating; entries = " + length);
// create a file with length entries
SequenceFile.Writer writer =
SequenceFile.createWriter(fs, conf, file, IntWritable.class, BytesWritable.class);
try {
for (int i = 0; i < length; i++) {
IntWritable key = new IntWritable(i);
byte[] data = new byte[random.nextInt(10)];
random.nextBytes(data);
BytesWritable value = new BytesWritable(data);
writer.append(key, value);
}
} finally {
writer.close();
}
// try splitting the file in a variety of sizes
InputFormat<IntWritable, BytesWritable> format =
new SequenceFileInputFormat<IntWritable, BytesWritable>();
IntWritable key = new IntWritable();
BytesWritable value = new BytesWritable();
for (int i = 0; i < 3; i++) {
int numSplits = random.nextInt(MAX_LENGTH / (SequenceFile.SYNC_INTERVAL / 20)) + 1;
// LOG.info("splitting: requesting = " + numSplits);
InputSplit[] splits = format.getSplits(job, numSplits);
// LOG.info("splitting: got = " + splits.length);
// check each split
BitSet bits = new BitSet(length);
for (int j = 0; j < splits.length; j++) {
RecordReader<IntWritable, BytesWritable> reader =
format.getRecordReader(splits[j], job, reporter);
try {
int count = 0;
while (reader.next(key, value)) {
// if (bits.get(key.get())) {
// LOG.info("splits["+j+"]="+splits[j]+" : " +
// key.get());
// LOG.info("@"+reader.getPos());
// }
assertFalse("Key in multiple partitions.", bits.get(key.get()));
bits.set(key.get());
count++;
}
// LOG.info("splits["+j+"]="+splits[j]+" count=" +
// count);
} finally {
reader.close();
}
}
assertEquals("Some keys in no partition.", length, bits.cardinality());
}
}
}
/** Explode the archive into its constituent elements */
public void explode() throws CacheException {
CachedUrl cachedUrl = null;
int goodEntries = 0;
int badEntries = 0;
int ignoredEntries = 0;
int entriesBetweenSleep = 0;
ArchiveReader arcReader = null;
logger.info(
(storeArchive ? "Storing" : "Fetching") + " WARC file: " + origUrl + " will explode");
try {
if (storeArchive) {
UrlCacher uc = au.makeUrlCacher(new UrlData(arcStream, arcProps, fetchUrl));
BitSet bs = new BitSet();
bs.set(UrlCacher.DONT_CLOSE_INPUT_STREAM_FLAG);
uc.setFetchFlags(bs);
uc.storeContent();
archiveData.resetInputStream();
arcStream = archiveData.input;
}
// Wrap it in an ArchiveReader
logger.debug3("About to wrap stream");
arcReader = wrapStream(fetchUrl, arcStream);
logger.debug3("wrapStream() returns " + (arcReader == null ? "null" : "non-null"));
// Explode it
if (arcReader == null) {
throw new CacheException.ExploderException("no WarcReader for " + origUrl);
}
ArchivalUnit au = crawlFacade.getAu();
Set stemSet = new HashSet();
logger.debug("Exploding " + fetchUrl);
// Iterate through the elements in the WARC file, except the first
Iterator i = arcReader.iterator();
// Skip first record
for (i.next(); i.hasNext(); ) {
// XXX probably not necessary
helper.pokeWDog();
if ((++entriesBetweenSleep % sleepAfter) == 0) {
long pauseTime =
CurrentConfig.getTimeIntervalParam(PARAM_RETRY_PAUSE, DEFAULT_RETRY_PAUSE);
Deadline pause = Deadline.in(pauseTime);
logger.debug3("Sleeping for " + StringUtil.timeIntervalToString(pauseTime));
while (!pause.expired()) {
try {
pause.sleep();
} catch (InterruptedException ie) {
// no action
}
}
}
ArchiveRecord element = (ArchiveRecord) i.next();
// Each element is a URL to be cached in a suitable AU
ArchiveRecordHeader elementHeader = element.getHeader();
String elementUrl = elementHeader.getUrl();
String elementMimeType = elementHeader.getMimetype();
long elementLength = elementHeader.getLength();
logger.debug2("WARC url " + elementUrl + " mime " + elementMimeType);
if (elementUrl.startsWith("http:")) {
ArchiveEntry ae =
new ArchiveEntry(
elementUrl,
elementLength,
0, // XXX need to convert getDate string to long
element, // ArchiveRecord extends InputStream
this,
fetchUrl);
ae.setHeaderFields(makeCIProperties(elementHeader));
long bytesStored = elementLength;
logger.debug3("ArchiveEntry: " + ae.getName() + " bytes " + bytesStored);
try {
helper.process(ae);
} catch (PluginException ex) {
throw new CacheException.ExploderException("helper.process() threw", ex);
}
if (ae.getBaseUrl() != null) {
if (ae.getRestOfUrl() != null && ae.getHeaderFields() != null) {
storeEntry(ae);
handleAddText(ae);
goodEntries++;
crawlFacade.getCrawlerStatus().addContentBytesFetched(bytesStored);
} else {
ignoredEntries++;
}
} else {
badEntries++;
logger.debug2("Can't map " + elementUrl + " from " + archiveUrl);
}
}
}
} catch (IOException ex) {
throw new CacheException.ExploderException(ex);
} finally {
if (arcReader != null)
try {
arcReader.close();
arcReader = null;
} catch (IOException ex) {
throw new CacheException.ExploderException(ex);
}
if (cachedUrl != null) {
cachedUrl.release();
}
IOUtil.safeClose(arcStream);
}
if (badEntries == 0 && goodEntries > 0) {
// Make it look like a new crawl finished on each AU to which
// URLs were added.
for (Iterator it = touchedAus.iterator(); it.hasNext(); ) {
ArchivalUnit au = (ArchivalUnit) it.next();
logger.debug3(archiveUrl + " touching " + au.toString());
AuUtil.getDaemon(au).getNodeManager(au).newContentCrawlFinished();
}
} else {
ArchivalUnit au = crawlFacade.getAu();
String msg = archiveUrl + ": " + badEntries + "/" + goodEntries + " bad entries";
throw new CacheException.UnretryableException(msg);
}
}
