/**
  * Update the piece availabilities for a given peer
  *
  * @param peerID String
  * @param has BitSet
  */
 public synchronized void peerAvailability(String peerID, BitSet has) {
   this.peerAvailabilies.put(peerID, has);
   BitSet interest = (BitSet) (has.clone());
   interest.andNot(this.isComplete);
   DownloadTask dt = this.task.get(peerID);
   if (dt != null) {
     if (interest.cardinality() > 0 && !dt.peer.isInteresting()) {
       dt.ms.addMessageToQueue(new Message_PP(PeerProtocol.INTERESTED, 2));
       dt.peer.setInteresting(true);
     }
   }
   dt = null;
 }
 public void compactEnd() {
   final int compactSize = compactIndexUsed.cardinality();
   if (compactSize == map.size()) {
     return;
   }
   for (int i = 1; i < list.size(); i++) {
     if (compactIndexUsed.get(i)) continue;
     // to be removed
     T t = list.get(i);
     list.set(i, null);
     map.remove(t);
     if (addPosition > i) addPosition = i;
   }
 }
  public void testFormat() throws Exception {
    JobConf job = new JobConf(conf);
    FileSystem fs = FileSystem.getLocal(conf);
    Path dir = new Path(System.getProperty("test.build.data", ".") + "/mapred");
    Path file = new Path(dir, "test.seq");

    Reporter reporter = Reporter.NULL;

    int seed = new Random().nextInt();
    // LOG.info("seed = "+seed);
    Random random = new Random(seed);

    fs.delete(dir, true);

    FileInputFormat.setInputPaths(job, dir);

    // for a variety of lengths
    for (int length = 0; length < MAX_LENGTH; length += random.nextInt(MAX_LENGTH / 10) + 1) {

      // LOG.info("creating; entries = " + length);

      // create a file with length entries
      SequenceFile.Writer writer =
          SequenceFile.createWriter(fs, conf, file, IntWritable.class, BytesWritable.class);
      try {
        for (int i = 0; i < length; i++) {
          IntWritable key = new IntWritable(i);
          byte[] data = new byte[random.nextInt(10)];
          random.nextBytes(data);
          BytesWritable value = new BytesWritable(data);
          writer.append(key, value);
        }
      } finally {
        writer.close();
      }

      // try splitting the file in a variety of sizes
      InputFormat<IntWritable, BytesWritable> format =
          new SequenceFileInputFormat<IntWritable, BytesWritable>();
      IntWritable key = new IntWritable();
      BytesWritable value = new BytesWritable();
      for (int i = 0; i < 3; i++) {
        int numSplits = random.nextInt(MAX_LENGTH / (SequenceFile.SYNC_INTERVAL / 20)) + 1;
        // LOG.info("splitting: requesting = " + numSplits);
        InputSplit[] splits = format.getSplits(job, numSplits);
        // LOG.info("splitting: got =        " + splits.length);

        // check each split
        BitSet bits = new BitSet(length);
        for (int j = 0; j < splits.length; j++) {
          RecordReader<IntWritable, BytesWritable> reader =
              format.getRecordReader(splits[j], job, reporter);
          try {
            int count = 0;
            while (reader.next(key, value)) {
              // if (bits.get(key.get())) {
              // LOG.info("splits["+j+"]="+splits[j]+" : " +
              // key.get());
              // LOG.info("@"+reader.getPos());
              // }
              assertFalse("Key in multiple partitions.", bits.get(key.get()));
              bits.set(key.get());
              count++;
            }
            // LOG.info("splits["+j+"]="+splits[j]+" count=" +
            // count);
          } finally {
            reader.close();
          }
        }
        assertEquals("Some keys in no partition.", length, bits.cardinality());
      }
    }
  }
예제 #4
0
 /** Returns the number of empty slots on this page. */
 public int getNumEmptySlots() {
   // hdj:!!! use getNumTuples instead of bitset.size() or bitset.length()
   BitSet bitset = BitSet.valueOf(header);
   return getNumTuples() - bitset.cardinality();
 }
  public void rTest(final int N) {
    System.out.println("rtest N=" + N);
    for (int gap = 1; gap <= 65536; gap *= 2) {
      final BitSet bs1 = new BitSet();
      final MutableRoaringBitmap rb1 = new MutableRoaringBitmap();
      for (int x = 0; x <= N; x += gap) {
        bs1.set(x);
        rb1.add(x);
      }
      if (bs1.cardinality() != rb1.getCardinality()) throw new RuntimeException("different card");
      if (!equals(bs1, rb1)) throw new RuntimeException("basic  bug");
      for (int offset = 1; offset <= gap; offset *= 2) {
        final BitSet bs2 = new BitSet();
        final MutableRoaringBitmap rb2 = new MutableRoaringBitmap();
        for (int x = 0; x <= N; x += gap) {
          bs2.set(x + offset);
          rb2.add(x + offset);
        }
        if (bs2.cardinality() != rb2.getCardinality()) throw new RuntimeException("different card");
        if (!equals(bs2, rb2)) throw new RuntimeException("basic  bug");

        BitSet clonebs1;
        // testing AND
        clonebs1 = (BitSet) bs1.clone();
        clonebs1.and(bs2);
        if (!equals(clonebs1, MutableRoaringBitmap.and(rb1, rb2)))
          throw new RuntimeException("bug and");
        {
          final MutableRoaringBitmap t = rb1.clone();
          t.and(rb2);
          if (!equals(clonebs1, t)) throw new RuntimeException("bug inplace and");
          if (!t.equals(MutableRoaringBitmap.and(rb1, rb2))) {
            System.out.println(
                t.highLowContainer.getContainerAtIndex(0).getClass().getCanonicalName());
            System.out.println(
                MutableRoaringBitmap.and(rb1, rb2)
                    .highLowContainer
                    .getContainerAtIndex(0)
                    .getClass()
                    .getCanonicalName());

            throw new RuntimeException("bug inplace and");
          }
        }

        // testing OR
        clonebs1 = (BitSet) bs1.clone();
        clonebs1.or(bs2);

        if (!equals(clonebs1, MutableRoaringBitmap.or(rb1, rb2)))
          throw new RuntimeException("bug or");
        {
          final MutableRoaringBitmap t = rb1.clone();
          t.or(rb2);
          if (!equals(clonebs1, t)) throw new RuntimeException("bug or");
          if (!t.equals(MutableRoaringBitmap.or(rb1, rb2))) throw new RuntimeException("bug or");
          if (!t.toString().equals(MutableRoaringBitmap.or(rb1, rb2).toString()))
            throw new RuntimeException("bug or");
        }
        // testing XOR
        clonebs1 = (BitSet) bs1.clone();
        clonebs1.xor(bs2);
        if (!equals(clonebs1, MutableRoaringBitmap.xor(rb1, rb2))) {
          throw new RuntimeException("bug xor");
        }
        {
          final MutableRoaringBitmap t = rb1.clone();
          t.xor(rb2);
          if (!equals(clonebs1, t)) throw new RuntimeException("bug xor");

          if (!t.equals(MutableRoaringBitmap.xor(rb1, rb2))) {
            System.out.println(t);
            System.out.println(MutableRoaringBitmap.xor(rb1, rb2));
            System.out.println(
                Arrays.equals(t.toArray(), MutableRoaringBitmap.xor(rb1, rb2).toArray()));
            throw new RuntimeException("bug xor");
          }
        }
        // testing NOTAND
        clonebs1 = (BitSet) bs1.clone();
        clonebs1.andNot(bs2);
        if (!equals(clonebs1, MutableRoaringBitmap.andNot(rb1, rb2))) {
          throw new RuntimeException("bug andnot");
        }
        clonebs1 = (BitSet) bs2.clone();
        clonebs1.andNot(bs1);
        if (!equals(clonebs1, MutableRoaringBitmap.andNot(rb2, rb1))) {
          throw new RuntimeException("bug andnot");
        }
        {
          final MutableRoaringBitmap t = rb2.clone();
          t.andNot(rb1);
          if (!equals(clonebs1, t)) {
            throw new RuntimeException("bug inplace andnot");
          }
          final MutableRoaringBitmap g = MutableRoaringBitmap.andNot(rb2, rb1);
          if (!equals(clonebs1, g)) {
            throw new RuntimeException("bug andnot");
          }
          if (!t.equals(g)) throw new RuntimeException("bug");
        }
        clonebs1 = (BitSet) bs1.clone();
        clonebs1.andNot(bs2);
        if (!equals(clonebs1, MutableRoaringBitmap.andNot(rb1, rb2))) {
          throw new RuntimeException("bug andnot");
        }
        {
          final MutableRoaringBitmap t = rb1.clone();
          t.andNot(rb2);
          if (!equals(clonebs1, t)) {
            throw new RuntimeException("bug andnot");
          }
          final MutableRoaringBitmap g = MutableRoaringBitmap.andNot(rb1, rb2);
          if (!equals(clonebs1, g)) {
            throw new RuntimeException("bug andnot");
          }
          if (!t.equals(g)) throw new RuntimeException("bug");
        }
      }
    }
  }
 public static boolean equals(BitSet bs, MutableRoaringBitmap rr) {
   final int[] a = new int[bs.cardinality()];
   int pos = 0;
   for (int x = bs.nextSetBit(0); x >= 0; x = bs.nextSetBit(x + 1)) a[pos++] = x;
   return Arrays.equals(rr.toArray(), a);
 }