@Test
  public void flipTest6A() {
    System.out.println("FlipTest6A");
    final MutableRoaringBitmap rb = new MutableRoaringBitmap();
    final MutableRoaringBitmap rb1 = MutableRoaringBitmap.flip(rb, 100000, 132000);
    final MutableRoaringBitmap rb2 = MutableRoaringBitmap.flip(rb1, 99000, 2 * 65536);
    final int rbcard = rb2.getCardinality();

    Assert.assertEquals(1928, rbcard);

    final BitSet bs = new BitSet();
    for (int i = 99000; i < 100000; ++i) bs.set(i);
    for (int i = 2 * 65536; i < 132000; ++i) bs.set(i);
    Assert.assertTrue(equals(bs, rb2));
  }
  public static void main(String[] args) {

    // Initialized to false by default
    BitSet bitmap = new BitSet(10000000);
    BufferedReader br = null;
    String line = null;

    try {

      br = new BufferedReader(new FileReader(args[0]));

      // Setting the bitmap for the input values
      while ((line = br.readLine()) != null) {
        int num = Integer.parseInt(line);
        bitmap.set(num);
      }

      // Outputting the sorted value
      for (int i = 0; i < bitmap.length(); i++) {

        if (bitmap.get(i)) {
          System.out.println(bitmap.get(i));
        }
      }
    } catch (Exception e) {

      e.printStackTrace();
    }
  }
 @Override
 public void prob1step(
     BitSet subset, BitSet u, BitSet v, boolean forall1, boolean forall2, BitSet result) {
   boolean b1, b2, b3;
   for (int i : new IterableStateSet(subset, numStates)) {
     b1 = forall1; // there exists or for all player 1 choices
     for (DistributionSet distrs : trans.get(i)) {
       b2 = forall2; // there exists or for all player 2 choices
       for (Distribution distr : distrs) {
         b3 = distr.containsOneOf(v) && distr.isSubsetOf(u);
         if (forall2) {
           if (!b3) b2 = false;
         } else {
           if (b3) b2 = true;
         }
       }
       if (forall1) {
         if (!b2) b1 = false;
       } else {
         if (b2) b1 = true;
       }
     }
     result.set(i, b1);
   }
 }
  /**
   * Updates the interal bitset from <code>iterator</code>. This will set <code>validMask</code> to
   * true if <code>iterator</code> is non-null.
   */
  private void updateMask(AttributedCharacterIterator iterator) {
    if (iterator != null) {
      validMask = true;
      this.iterator = iterator;

      // Update the literal mask
      if (literalMask == null) {
        literalMask = new BitSet();
      } else {
        for (int counter = literalMask.length() - 1; counter >= 0; counter--) {
          literalMask.clear(counter);
        }
      }

      iterator.first();
      while (iterator.current() != CharacterIterator.DONE) {
        Map attributes = iterator.getAttributes();
        boolean set = isLiteral(attributes);
        int start = iterator.getIndex();
        int end = iterator.getRunLimit();

        while (start < end) {
          if (set) {
            literalMask.set(start);
          } else {
            literalMask.clear(start);
          }
          start++;
        }
        iterator.setIndex(start);
      }
    }
  }
예제 #5
0
 public TOE(String s, int Nc, int Mc) {
   this(Nc, Mc);
   char[] C = s.toCharArray();
   for (int i = 0; i < N; i++)
     for (int j = 0; j < M; j++)
       if (C[i * M + j] == '1') G.set(i * M + j);
       else G.clear(i * M + j);
 }
  @Test
  public void flipTest7A() { // within 1 word, first container
    System.out.println("FlipTest7A");
    final MutableRoaringBitmap rb = new MutableRoaringBitmap();
    final MutableRoaringBitmap rb1 = MutableRoaringBitmap.flip(rb, 650, 132000);
    final MutableRoaringBitmap rb2 = MutableRoaringBitmap.flip(rb1, 648, 651);
    final int rbcard = rb2.getCardinality();

    // 648, 649, 651-131999

    Assert.assertEquals(132000 - 651 + 2, rbcard);

    final BitSet bs = new BitSet();
    bs.set(648);
    bs.set(649);
    for (int i = 651; i < 132000; ++i) bs.set(i);
    Assert.assertTrue(equals(bs, rb2));
  }
  @Test
  public void flipTest6() { // fits evenly on big end, multiple containers
    System.out.println("FlipTest6");
    final MutableRoaringBitmap rb = new MutableRoaringBitmap();
    rb.flip(100000, 132000);
    rb.flip(99000, 2 * 65536);
    final int rbcard = rb.getCardinality();

    // 99000 to 99999 are 1000 1s
    // 131072 to 131999 are 928 1s

    Assert.assertEquals(1928, rbcard);

    final BitSet bs = new BitSet();
    for (int i = 99000; i < 100000; ++i) bs.set(i);
    for (int i = 2 * 65536; i < 132000; ++i) bs.set(i);
    Assert.assertTrue(equals(bs, rb));
  }
예제 #8
0
  /**
   * setup BitMap
   *
   * @exception ISOException
   */
  public void recalcBitMap() throws ISOException {
    if (!dirty) return;

    int mf = Math.min(getMaxField(), 192);

    BitSet bmap = new BitSet(((mf + 62) >> 6) << 6);
    for (int i = 1; i <= mf; i++) if ((fields.get(i)) != null) bmap.set(i);
    set(new ISOBitMap(-1, bmap));
    dirty = false;
  }
 protected void doDamageRemoveTest(SimulatedArchivalUnit sau) throws Exception {
   /* Cache the file again; this time the damage should be gone */
   String file = sau.getUrlRoot() + DAMAGED_CACHED_URL;
   UrlCacher uc = sau.makeUrlCacher(file);
   BitSet fetchFlags = new BitSet();
   fetchFlags.set(UrlCacher.REFETCH_FLAG);
   uc.setFetchFlags(fetchFlags);
   uc.cache();
   checkUrlContent(sau, DAMAGED_CACHED_URL, 2, 2, 2, false, false);
 }
예제 #10
0
  @Test
  public void flipTest5() { // fits evenly on small end, multiple
    // containers
    System.out.println("FlipTest5");
    final MutableRoaringBitmap rb = new MutableRoaringBitmap();
    rb.flip(100000, 132000);
    rb.flip(65536, 120000);
    final int rbcard = rb.getCardinality();

    // 65536 to 99999 are 1s
    // 120000 to 131999

    Assert.assertEquals(46464, rbcard);

    final BitSet bs = new BitSet();
    for (int i = 65536; i < 100000; ++i) bs.set(i);
    for (int i = 120000; i < 132000; ++i) bs.set(i);
    Assert.assertTrue(equals(bs, rb));
  }
예제 #11
0
  @Test
  public void flipTest4() { // fits evenly on both ends
    System.out.println("FlipTest4");
    final MutableRoaringBitmap rb = new MutableRoaringBitmap();
    rb.flip(100000, 200000); // got 100k-199999
    rb.flip(65536, 4 * 65536);
    final int rbcard = rb.getCardinality();

    // 65536 to 99999 are 1s
    // 200000 to 262143 are 1s: total card

    Assert.assertEquals(96608, rbcard);

    final BitSet bs = new BitSet();
    for (int i = 65536; i < 100000; ++i) bs.set(i);
    for (int i = 200000; i < 262144; ++i) bs.set(i);

    Assert.assertTrue(equals(bs, rb));
  }
예제 #12
0
파일: A.java 프로젝트: t8m8/AtCoder
  static void solve() {
    int r = in.nextInt();
    int c = in.nextInt();
    int sy = in.nextInt() - 1;
    int sx = in.nextInt() - 1;
    int gy = in.nextInt() - 1;
    int gx = in.nextInt() - 1;
    char[][] t = new char[r][c];
    for (int i = 0; i < r; i++) {
      t[i] = in.next().toCharArray();
    }

    ArrayDeque<int[]> que = new ArrayDeque<>();
    BitSet visited = new BitSet();

    que.add(new int[] {sy, sx, 0});
    visited.set(sy * c + sx);

    int[] dx = {0, 1, 0, -1};
    int[] dy = {1, 0, -1, 0};
    int[][] dist = new int[r][c];

    while (!que.isEmpty()) {
      int[] p = que.pollFirst();
      int cy = p[0];
      int cx = p[1];
      int d = p[2];
      dist[cy][cx] = d;

      for (int i = 0; i < 4; i++) {
        int ny = cy + dy[i];
        int nx = cx + dx[i];
        if (ny < 0 || nx < 0 || r <= ny || c <= nx) continue;
        if (visited.get(ny * c + nx) || t[ny][nx] == '#') continue;
        que.add(new int[] {ny, nx, d + 1});
        visited.set(ny * c + nx);
      }
    }

    out.println(dist[gy][gx]);
  }
예제 #13
0
  @Test
  public void flipTest1() {
    final MutableRoaringBitmap rb = new MutableRoaringBitmap();

    rb.flip(100000, 200000); // in-place on empty bitmap
    final int rbcard = rb.getCardinality();
    Assert.assertEquals(100000, rbcard);

    final BitSet bs = new BitSet();
    for (int i = 100000; i < 200000; ++i) bs.set(i);
    Assert.assertTrue(equals(bs, rb));
  }
예제 #14
0
  @Test
  public void flipTest1A() {
    final MutableRoaringBitmap rb = new MutableRoaringBitmap();

    final MutableRoaringBitmap rb1 = MutableRoaringBitmap.flip(rb, 100000, 200000);
    final int rbcard = rb1.getCardinality();
    Assert.assertEquals(100000, rbcard);
    Assert.assertEquals(0, rb.getCardinality());

    final BitSet bs = new BitSet();
    Assert.assertTrue(equals(bs, rb)); // still empty?
    for (int i = 100000; i < 200000; ++i) bs.set(i);
    Assert.assertTrue(equals(bs, rb1));
  }
예제 #15
0
  @Test
  public void flipTest2A() {
    final MutableRoaringBitmap rb = new MutableRoaringBitmap();

    final MutableRoaringBitmap rb1 = MutableRoaringBitmap.flip(rb, 100000, 100000);
    rb.add(1); // will not affect rb1 (no shared container)
    final int rbcard = rb1.getCardinality();
    Assert.assertEquals(0, rbcard);
    Assert.assertEquals(1, rb.getCardinality());

    final BitSet bs = new BitSet();
    Assert.assertTrue(equals(bs, rb1));
    bs.set(1);
    Assert.assertTrue(equals(bs, rb));
  }
예제 #16
0
  @Test
  public void flipTest3() {
    final MutableRoaringBitmap rb = new MutableRoaringBitmap();

    rb.flip(100000, 200000); // got 100k-199999
    rb.flip(100000, 199991); // give back 100k-199990
    final int rbcard = rb.getCardinality();

    Assert.assertEquals(9, rbcard);

    final BitSet bs = new BitSet();
    for (int i = 199991; i < 200000; ++i) bs.set(i);

    Assert.assertTrue(equals(bs, rb));
  }
예제 #17
0
  @Test
  public void flipTest3A() {
    System.out.println("FlipTest3A");
    final MutableRoaringBitmap rb = new MutableRoaringBitmap();
    final MutableRoaringBitmap rb1 = MutableRoaringBitmap.flip(rb, 100000, 200000);
    final MutableRoaringBitmap rb2 = MutableRoaringBitmap.flip(rb1, 100000, 199991);
    final int rbcard = rb2.getCardinality();

    Assert.assertEquals(9, rbcard);

    final BitSet bs = new BitSet();
    for (int i = 199991; i < 200000; ++i) bs.set(i);

    Assert.assertTrue(equals(bs, rb2));
  }
예제 #18
0
  /**
   * It adds a sequence from an array of string that we have to interpret
   *
   * @param integers
   * @param sequenceID
   */
  public void addSequence(String[] integers, int sequenceID) {
    long timestamp = -1;
    Sequence sequence = new Sequence(sequences.size());
    sequence.setID(sequenceID);
    Itemset itemset = new Itemset();
    int inicio = 0;
    Map<Item, Boolean> counted = new HashMap<Item, Boolean>();

    for (int i = inicio; i < integers.length; i++) {
      if (integers[i].codePointAt(0) == '<') { // Timestamp
        String value = integers[i].substring(1, integers[i].length() - 1);
        timestamp = Long.parseLong(value);
        itemset.setTimestamp(timestamp);
      } else if (integers[i].equals("-1")) { // end of an itemset
        long time = itemset.getTimestamp() + 1;
        sequence.addItemset(itemset);
        itemset = new Itemset();
        itemset.setTimestamp(time);
      } else if (integers[i].equals("-2")) { // end of a sequence
        sequences.add(sequence);
      } else {
        // extract the value for an item
        Item item = itemFactory.getItem(Integer.parseInt(integers[i]));
        if (counted.get(item) == null) {
          counted.put(item, Boolean.TRUE);
          BitSet appearances = frequentItems.get(item);
          if (appearances == null) {
            appearances = new BitSet();
            frequentItems.put(item, appearances);
          }
          appearances.set(sequence.getId());
        }
        itemset.addItem(item);
      }
    }
  }
  /**
   * Generates a zero crossing mask for a 2D function sets a Bitset object to 1 where a zero
   * crossing is detected.
   *
   * @param xDim x dimension of image
   * @param yDim y dimension of image
   * @param buffer array in which to find zero crossing
   * @param level level to generate zero crossings at
   * @param detectionType the type of zero crossing method to use
   * @return Bitset representing zero crossings
   */
  public static BitSet genLevelMask(
      int xDim, int yDim, float[] buffer, float level, int detectionType) {
    int i0, i1, i2, i3;
    float x0, x1, x2, x3;
    int i, j;
    int indexY;

    int xxDim = xDim - 1;
    int yyDim = yDim - 1;

    BitSet edgeImage = new BitSet(xDim * yDim);

    for (j = 0; j < yyDim; j++) {
      indexY = j * xDim;

      for (i = 0; i < xxDim; i++) {
        i0 = indexY + i;

        if (detectionType == MARCHING_SQUARES) {
          i1 = i0 + 1;
          i2 = i0 + xDim;
          i3 = i0 + 1 + xDim;

          x0 = buffer[i0];
          x1 = buffer[i1];
          x2 = buffer[i2];
          x3 = buffer[i3];

          if ((x0 >= level)
              && (x1 >= level)
              && (x2 >= level)
              && (x3 >= level)) { // case 0 - no edge
          } else if ((x0 >= level) && (x1 >= level) && (x2 < level) && (x3 >= level)) {

            // case 1 - edge in the lower left
            edgeImage.set(i2);
          } else if ((x0 >= level) && (x1 >= level) && (x2 >= level) && (x3 < level)) {

            // case 2 - edge in the lower right
            edgeImage.set(i3);
          } else if ((x0 >= level) && (x1 >= level) && (x2 < level) && (x3 < level)) {

            // case 3 - edge horizontally
            edgeImage.set(i2);
            edgeImage.set(i3);
          } else if ((x0 >= level) && (x1 < level) && (x2 >= level) && (x3 >= level)) {

            // case 4 - edge in the upper right
            edgeImage.set(i1);
          } else if ((x0 >= level) && (x1 < level) && (x2 < level) && (x3 >= level)) {

            // case 5 - ambiguous case; either edge in upper right and lower left or
            // edge that goes from the upper right to the lower left
            edgeImage.set(i1);
            edgeImage.set(i2);
          } else if ((x0 >= level) && (x1 < level) && (x2 >= level) && (x3 < level)) {

            // case 6 - edge going vertically along the right
            edgeImage.set(i1);
            edgeImage.set(i3);
          } else if ((x0 >= level) && (x1 < level) && (x2 < level) && (x3 < level)) {

            // case 7 - edge in the upper left
            edgeImage.set(i0);
          } else if ((x0 < level) && (x1 >= level) && (x2 >= level) && (x3 >= level)) {

            // case 8 - edge in the upper left
            edgeImage.set(i0);
          } else if ((x0 < level) && (x1 >= level) && (x2 < level) && (x3 >= level)) {

            // case 9 - edge going vertically along the left
            edgeImage.set(i0);
            edgeImage.set(i2);
          } else if ((x0 < level) && (x1 >= level) && (x2 >= level) && (x3 < level)) {

            // case 10 - ambiguous case; either edge in upper left and lower right or
            // edge that goes from the upper left to the lower right
            edgeImage.set(i0);
            edgeImage.set(i3);
          } else if ((x0 < level) && (x1 >= level) && (x2 < level) && (x3 < level)) {

            // case 11 - edge in the upper right
            edgeImage.set(i1);
          } else if ((x0 < level) && (x1 < level) && (x2 >= level) && (x3 >= level)) {

            // case 12 - edge going horizontally along the top
            edgeImage.set(i0);
            edgeImage.set(i1);
          } else if ((x0 < level) && (x1 < level) && (x2 < level) && (x3 >= level)) {

            // case 13 - edge in the lower right
            edgeImage.set(i3);
          } else if ((x0 < level) && (x1 < level) && (x2 >= level) && (x3 < level)) {

            // case 14 - edge in the lower left
            edgeImage.set(i2);
          } else if ((x0 < level)
              && (x1 < level)
              && (x2 < level)
              && (x3 < level)) { // case 15 - no edge
          }
        } else if (detectionType == NEGATIVE_EDGES) {

          if (buffer[i0] <= 0) {
            edgeImage.set(i0);
          }
        } else if (detectionType == OLD_DETECTION) {
          i1 = i0 + 1;
          i2 = i0 + xDim;
          i3 = i0 + 1 + xDim;

          x0 = buffer[i0];
          x1 = buffer[i1];
          x2 = buffer[i2];
          x3 = buffer[i3];

          if ((x0 >= level) && (x1 >= level) && (x2 >= level) && (x3 >= level)) {
            edgeImage.clear(i0);
          } else if ((x0 <= level) && (x1 <= level) && (x2 <= level) && (x3 <= level)) {
            edgeImage.clear(i0);
          } else {
            edgeImage.set(i0);
          }
        }
      }
    }

    return edgeImage;
  }
  /** Simple test program */
  public static void main(String args[]) {
    STPGModelChecker mc;
    STPGAbstrSimple stpg;
    DistributionSet set;
    Distribution distr;
    // ModelCheckerResult res;
    BitSet target;

    // Simple example: Create and solve the stochastic game from:
    // Mark Kattenbelt, Marta Kwiatkowska, Gethin Norman, David Parker
    // A Game-based Abstraction-Refinement Framework for Markov Decision Processes
    // Formal Methods in System Design 36(3): 246-280, 2010

    try {
      // Build game
      stpg = new STPGAbstrSimple();
      stpg.addStates(4);
      // State 0 (s_0)
      set = stpg.newDistributionSet(null);
      distr = new Distribution();
      distr.set(1, 1.0);
      set.add(distr);
      stpg.addDistributionSet(0, set);
      // State 1 (s_1,s_2,s_3)
      set = stpg.newDistributionSet(null);
      distr = new Distribution();
      distr.set(2, 1.0);
      set.add(distr);
      distr = new Distribution();
      distr.set(1, 1.0);
      set.add(distr);
      stpg.addDistributionSet(1, set);
      set = stpg.newDistributionSet(null);
      distr = new Distribution();
      distr.set(2, 0.5);
      distr.set(3, 0.5);
      set.add(distr);
      distr = new Distribution();
      distr.set(3, 1.0);
      set.add(distr);
      stpg.addDistributionSet(1, set);
      // State 2 (s_4,s_5)
      set = stpg.newDistributionSet(null);
      distr = new Distribution();
      distr.set(2, 1.0);
      set.add(distr);
      stpg.addDistributionSet(2, set);
      // State 3 (s_6)
      set = stpg.newDistributionSet(null);
      distr = new Distribution();
      distr.set(3, 1.0);
      set.add(distr);
      stpg.addDistributionSet(3, set);
      // Print game
      System.out.println(stpg);

      // Model check
      mc = new STPGModelChecker(null);
      // mc.setVerbosity(2);
      target = new BitSet();
      target.set(3);
      stpg.exportToDotFile("stpg.dot", target);
      System.out.println("min min: " + mc.computeReachProbs(stpg, target, true, true).soln[0]);
      System.out.println("max min: " + mc.computeReachProbs(stpg, target, false, true).soln[0]);
      System.out.println("min max: " + mc.computeReachProbs(stpg, target, true, false).soln[0]);
      System.out.println("max max: " + mc.computeReachProbs(stpg, target, false, false).soln[0]);
    } catch (PrismException e) {
      System.out.println(e);
    }
  }
 public void compactScan(CharBuffer charBuffer, long size) {
   for (int i = 0; i < size; i++) {
     final char ch = charBuffer.get(i);
     compactIndexUsed.set(ch);
   }
 }
예제 #22
0
  public void rTest(final int N) {
    System.out.println("rtest N=" + N);
    for (int gap = 1; gap <= 65536; gap *= 2) {
      final BitSet bs1 = new BitSet();
      final MutableRoaringBitmap rb1 = new MutableRoaringBitmap();
      for (int x = 0; x <= N; x += gap) {
        bs1.set(x);
        rb1.add(x);
      }
      if (bs1.cardinality() != rb1.getCardinality()) throw new RuntimeException("different card");
      if (!equals(bs1, rb1)) throw new RuntimeException("basic  bug");
      for (int offset = 1; offset <= gap; offset *= 2) {
        final BitSet bs2 = new BitSet();
        final MutableRoaringBitmap rb2 = new MutableRoaringBitmap();
        for (int x = 0; x <= N; x += gap) {
          bs2.set(x + offset);
          rb2.add(x + offset);
        }
        if (bs2.cardinality() != rb2.getCardinality()) throw new RuntimeException("different card");
        if (!equals(bs2, rb2)) throw new RuntimeException("basic  bug");

        BitSet clonebs1;
        // testing AND
        clonebs1 = (BitSet) bs1.clone();
        clonebs1.and(bs2);
        if (!equals(clonebs1, MutableRoaringBitmap.and(rb1, rb2)))
          throw new RuntimeException("bug and");
        {
          final MutableRoaringBitmap t = rb1.clone();
          t.and(rb2);
          if (!equals(clonebs1, t)) throw new RuntimeException("bug inplace and");
          if (!t.equals(MutableRoaringBitmap.and(rb1, rb2))) {
            System.out.println(
                t.highLowContainer.getContainerAtIndex(0).getClass().getCanonicalName());
            System.out.println(
                MutableRoaringBitmap.and(rb1, rb2)
                    .highLowContainer
                    .getContainerAtIndex(0)
                    .getClass()
                    .getCanonicalName());

            throw new RuntimeException("bug inplace and");
          }
        }

        // testing OR
        clonebs1 = (BitSet) bs1.clone();
        clonebs1.or(bs2);

        if (!equals(clonebs1, MutableRoaringBitmap.or(rb1, rb2)))
          throw new RuntimeException("bug or");
        {
          final MutableRoaringBitmap t = rb1.clone();
          t.or(rb2);
          if (!equals(clonebs1, t)) throw new RuntimeException("bug or");
          if (!t.equals(MutableRoaringBitmap.or(rb1, rb2))) throw new RuntimeException("bug or");
          if (!t.toString().equals(MutableRoaringBitmap.or(rb1, rb2).toString()))
            throw new RuntimeException("bug or");
        }
        // testing XOR
        clonebs1 = (BitSet) bs1.clone();
        clonebs1.xor(bs2);
        if (!equals(clonebs1, MutableRoaringBitmap.xor(rb1, rb2))) {
          throw new RuntimeException("bug xor");
        }
        {
          final MutableRoaringBitmap t = rb1.clone();
          t.xor(rb2);
          if (!equals(clonebs1, t)) throw new RuntimeException("bug xor");

          if (!t.equals(MutableRoaringBitmap.xor(rb1, rb2))) {
            System.out.println(t);
            System.out.println(MutableRoaringBitmap.xor(rb1, rb2));
            System.out.println(
                Arrays.equals(t.toArray(), MutableRoaringBitmap.xor(rb1, rb2).toArray()));
            throw new RuntimeException("bug xor");
          }
        }
        // testing NOTAND
        clonebs1 = (BitSet) bs1.clone();
        clonebs1.andNot(bs2);
        if (!equals(clonebs1, MutableRoaringBitmap.andNot(rb1, rb2))) {
          throw new RuntimeException("bug andnot");
        }
        clonebs1 = (BitSet) bs2.clone();
        clonebs1.andNot(bs1);
        if (!equals(clonebs1, MutableRoaringBitmap.andNot(rb2, rb1))) {
          throw new RuntimeException("bug andnot");
        }
        {
          final MutableRoaringBitmap t = rb2.clone();
          t.andNot(rb1);
          if (!equals(clonebs1, t)) {
            throw new RuntimeException("bug inplace andnot");
          }
          final MutableRoaringBitmap g = MutableRoaringBitmap.andNot(rb2, rb1);
          if (!equals(clonebs1, g)) {
            throw new RuntimeException("bug andnot");
          }
          if (!t.equals(g)) throw new RuntimeException("bug");
        }
        clonebs1 = (BitSet) bs1.clone();
        clonebs1.andNot(bs2);
        if (!equals(clonebs1, MutableRoaringBitmap.andNot(rb1, rb2))) {
          throw new RuntimeException("bug andnot");
        }
        {
          final MutableRoaringBitmap t = rb1.clone();
          t.andNot(rb2);
          if (!equals(clonebs1, t)) {
            throw new RuntimeException("bug andnot");
          }
          final MutableRoaringBitmap g = MutableRoaringBitmap.andNot(rb1, rb2);
          if (!equals(clonebs1, g)) {
            throw new RuntimeException("bug andnot");
          }
          if (!t.equals(g)) throw new RuntimeException("bug");
        }
      }
    }
  }
예제 #23
0
  /**
   * Dumps an {@link FST} to a GraphViz's <code>dot</code> language description for visualization.
   * Example of use:
   *
   * <pre>
   * PrintStream ps = new PrintStream(&quot;out.dot&quot;);
   * fst.toDot(ps);
   * ps.close();
   * </pre>
   *
   * and then, from command line:
   *
   * <pre>
   * dot -Tpng -o out.png out.dot
   * </pre>
   *
   * <p>Note: larger FSTs (a few thousand nodes) won't even render, don't bother.
   *
   * @param sameRank If <code>true</code>, the resulting <code>dot</code> file will try to order
   *     states in layers of breadth-first traversal. This may mess up arcs, but makes the output
   *     FST's structure a bit clearer.
   * @param labelStates If <code>true</code> states will have labels equal to their offsets in their
   *     binary format. Expands the graph considerably.
   * @see "http://www.graphviz.org/"
   */
  public static <T> void toDot(FST<T> fst, Writer out, boolean sameRank, boolean labelStates)
      throws IOException {
    final String expandedNodeColor = "blue";

    // This is the start arc in the automaton (from the epsilon state to the first state
    // with outgoing transitions.
    final FST.Arc<T> startArc = fst.getFirstArc(new FST.Arc<T>());

    // A queue of transitions to consider for the next level.
    final List<FST.Arc<T>> thisLevelQueue = new ArrayList<FST.Arc<T>>();

    // A queue of transitions to consider when processing the next level.
    final List<FST.Arc<T>> nextLevelQueue = new ArrayList<FST.Arc<T>>();
    nextLevelQueue.add(startArc);

    // A list of states on the same level (for ranking).
    final List<Integer> sameLevelStates = new ArrayList<Integer>();

    // A bitset of already seen states (target offset).
    final BitSet seen = new BitSet();
    seen.set(startArc.target);

    // Shape for states.
    final String stateShape = "circle";

    // Emit DOT prologue.
    out.write("digraph FST {\n");
    out.write("  rankdir = LR; splines=true; concentrate=true; ordering=out; ranksep=2.5; \n");

    if (!labelStates) {
      out.write("  node [shape=circle, width=.2, height=.2, style=filled]\n");
    }

    emitDotState(out, "initial", "point", "white", "");
    emitDotState(
        out,
        Integer.toString(startArc.target),
        stateShape,
        fst.isExpandedTarget(startArc) ? expandedNodeColor : null,
        "");
    out.write("  initial -> " + startArc.target + "\n");

    final T NO_OUTPUT = fst.outputs.getNoOutput();
    int level = 0;

    while (!nextLevelQueue.isEmpty()) {
      // we could double buffer here, but it doesn't matter probably.
      thisLevelQueue.addAll(nextLevelQueue);
      nextLevelQueue.clear();

      level++;
      out.write("\n  // Transitions and states at level: " + level + "\n");
      while (!thisLevelQueue.isEmpty()) {
        final FST.Arc<T> arc = thisLevelQueue.remove(thisLevelQueue.size() - 1);

        if (fst.targetHasArcs(arc)) {
          // scan all arcs
          final int node = arc.target;
          fst.readFirstTargetArc(arc, arc);

          while (true) {
            // Emit the unseen state and add it to the queue for the next level.
            if (arc.target >= 0 && !seen.get(arc.target)) {
              final boolean isExpanded = fst.isExpandedTarget(arc);
              emitDotState(
                  out,
                  Integer.toString(arc.target),
                  stateShape,
                  isExpanded ? expandedNodeColor : null,
                  labelStates ? Integer.toString(arc.target) : "");
              seen.set(arc.target);
              nextLevelQueue.add(new FST.Arc<T>().copyFrom(arc));
              sameLevelStates.add(arc.target);
            }

            String outs;
            if (arc.output != NO_OUTPUT) {
              outs = "/" + fst.outputs.outputToString(arc.output);
            } else {
              outs = "";
            }

            final String cl;
            if (arc.label == FST.END_LABEL) {
              cl = "~";
            } else {
              cl = printableLabel(arc.label);
            }

            out.write("  " + node + " -> " + arc.target + " [label=\"" + cl + outs + "\"]\n");

            // Break the loop if we're on the last arc of this state.
            if (arc.isLast()) {
              break;
            }
            fst.readNextArc(arc);
          }
        }
      }

      // Emit state ranking information.
      if (sameRank && sameLevelStates.size() > 1) {
        out.write("  {rank=same; ");
        for (int state : sameLevelStates) {
          out.write(state + "; ");
        }
        out.write(" }\n");
      }
      sameLevelStates.clear();
    }

    // Emit terminating state (always there anyway).
    out.write("  -1 [style=filled, color=black, shape=circle, label=\"\"]\n\n");
    out.write("  {rank=sink; -1 }\n");

    out.write("}\n");
    out.flush();
  }
  public void testFormat() throws Exception {
    JobConf job = new JobConf(conf);
    FileSystem fs = FileSystem.getLocal(conf);
    Path dir = new Path(System.getProperty("test.build.data", ".") + "/mapred");
    Path file = new Path(dir, "test.seq");

    Reporter reporter = Reporter.NULL;

    int seed = new Random().nextInt();
    // LOG.info("seed = "+seed);
    Random random = new Random(seed);

    fs.delete(dir, true);

    FileInputFormat.setInputPaths(job, dir);

    // for a variety of lengths
    for (int length = 0; length < MAX_LENGTH; length += random.nextInt(MAX_LENGTH / 10) + 1) {

      // LOG.info("creating; entries = " + length);

      // create a file with length entries
      SequenceFile.Writer writer =
          SequenceFile.createWriter(fs, conf, file, IntWritable.class, BytesWritable.class);
      try {
        for (int i = 0; i < length; i++) {
          IntWritable key = new IntWritable(i);
          byte[] data = new byte[random.nextInt(10)];
          random.nextBytes(data);
          BytesWritable value = new BytesWritable(data);
          writer.append(key, value);
        }
      } finally {
        writer.close();
      }

      // try splitting the file in a variety of sizes
      InputFormat<IntWritable, BytesWritable> format =
          new SequenceFileInputFormat<IntWritable, BytesWritable>();
      IntWritable key = new IntWritable();
      BytesWritable value = new BytesWritable();
      for (int i = 0; i < 3; i++) {
        int numSplits = random.nextInt(MAX_LENGTH / (SequenceFile.SYNC_INTERVAL / 20)) + 1;
        // LOG.info("splitting: requesting = " + numSplits);
        InputSplit[] splits = format.getSplits(job, numSplits);
        // LOG.info("splitting: got =        " + splits.length);

        // check each split
        BitSet bits = new BitSet(length);
        for (int j = 0; j < splits.length; j++) {
          RecordReader<IntWritable, BytesWritable> reader =
              format.getRecordReader(splits[j], job, reporter);
          try {
            int count = 0;
            while (reader.next(key, value)) {
              // if (bits.get(key.get())) {
              // LOG.info("splits["+j+"]="+splits[j]+" : " +
              // key.get());
              // LOG.info("@"+reader.getPos());
              // }
              assertFalse("Key in multiple partitions.", bits.get(key.get()));
              bits.set(key.get());
              count++;
            }
            // LOG.info("splits["+j+"]="+splits[j]+" count=" +
            // count);
          } finally {
            reader.close();
          }
        }
        assertEquals("Some keys in no partition.", length, bits.cardinality());
      }
    }
  }
예제 #25
0
  /** Explode the archive into its constituent elements */
  public void explode() throws CacheException {
    CachedUrl cachedUrl = null;
    int goodEntries = 0;
    int badEntries = 0;
    int ignoredEntries = 0;
    int entriesBetweenSleep = 0;
    ArchiveReader arcReader = null;

    logger.info(
        (storeArchive ? "Storing" : "Fetching") + " WARC file: " + origUrl + " will explode");
    try {
      if (storeArchive) {
        UrlCacher uc = au.makeUrlCacher(new UrlData(arcStream, arcProps, fetchUrl));
        BitSet bs = new BitSet();
        bs.set(UrlCacher.DONT_CLOSE_INPUT_STREAM_FLAG);
        uc.setFetchFlags(bs);
        uc.storeContent();
        archiveData.resetInputStream();
        arcStream = archiveData.input;
      }
      // Wrap it in an ArchiveReader
      logger.debug3("About to wrap stream");
      arcReader = wrapStream(fetchUrl, arcStream);
      logger.debug3("wrapStream() returns " + (arcReader == null ? "null" : "non-null"));
      // Explode it
      if (arcReader == null) {
        throw new CacheException.ExploderException("no WarcReader for " + origUrl);
      }
      ArchivalUnit au = crawlFacade.getAu();
      Set stemSet = new HashSet();
      logger.debug("Exploding " + fetchUrl);
      // Iterate through the elements in the WARC file, except the first
      Iterator i = arcReader.iterator();
      // Skip first record
      for (i.next(); i.hasNext(); ) {
        // XXX probably not necessary
        helper.pokeWDog();
        if ((++entriesBetweenSleep % sleepAfter) == 0) {
          long pauseTime =
              CurrentConfig.getTimeIntervalParam(PARAM_RETRY_PAUSE, DEFAULT_RETRY_PAUSE);
          Deadline pause = Deadline.in(pauseTime);
          logger.debug3("Sleeping for " + StringUtil.timeIntervalToString(pauseTime));
          while (!pause.expired()) {
            try {
              pause.sleep();
            } catch (InterruptedException ie) {
              // no action
            }
          }
        }
        ArchiveRecord element = (ArchiveRecord) i.next();
        // Each element is a URL to be cached in a suitable AU
        ArchiveRecordHeader elementHeader = element.getHeader();
        String elementUrl = elementHeader.getUrl();
        String elementMimeType = elementHeader.getMimetype();
        long elementLength = elementHeader.getLength();
        logger.debug2("WARC url " + elementUrl + " mime " + elementMimeType);
        if (elementUrl.startsWith("http:")) {
          ArchiveEntry ae =
              new ArchiveEntry(
                  elementUrl,
                  elementLength,
                  0, // XXX need to convert getDate string to long
                  element, // ArchiveRecord extends InputStream
                  this,
                  fetchUrl);
          ae.setHeaderFields(makeCIProperties(elementHeader));
          long bytesStored = elementLength;
          logger.debug3("ArchiveEntry: " + ae.getName() + " bytes " + bytesStored);
          try {
            helper.process(ae);
          } catch (PluginException ex) {
            throw new CacheException.ExploderException("helper.process() threw", ex);
          }
          if (ae.getBaseUrl() != null) {
            if (ae.getRestOfUrl() != null && ae.getHeaderFields() != null) {
              storeEntry(ae);
              handleAddText(ae);
              goodEntries++;
              crawlFacade.getCrawlerStatus().addContentBytesFetched(bytesStored);
            } else {
              ignoredEntries++;
            }
          } else {
            badEntries++;
            logger.debug2("Can't map " + elementUrl + " from " + archiveUrl);
          }
        }
      }
    } catch (IOException ex) {
      throw new CacheException.ExploderException(ex);
    } finally {
      if (arcReader != null)
        try {
          arcReader.close();
          arcReader = null;
        } catch (IOException ex) {
          throw new CacheException.ExploderException(ex);
        }
      if (cachedUrl != null) {
        cachedUrl.release();
      }
      IOUtil.safeClose(arcStream);
    }
    if (badEntries == 0 && goodEntries > 0) {
      // Make it look like a new crawl finished on each AU to which
      // URLs were added.
      for (Iterator it = touchedAus.iterator(); it.hasNext(); ) {
        ArchivalUnit au = (ArchivalUnit) it.next();
        logger.debug3(archiveUrl + " touching " + au.toString());
        AuUtil.getDaemon(au).getNodeManager(au).newContentCrawlFinished();
      }
    } else {
      ArchivalUnit au = crawlFacade.getAu();
      String msg = archiveUrl + ": " + badEntries + "/" + goodEntries + " bad entries";
      throw new CacheException.UnretryableException(msg);
    }
  }