/**
* Update the piece availabilities for a given peer
*
* @param peerID String
* @param has BitSet
*/
public synchronized void peerAvailability(String peerID, BitSet has) {
this.peerAvailabilies.put(peerID, has);
BitSet interest = (BitSet) (has.clone());
interest.andNot(this.isComplete);
DownloadTask dt = this.task.get(peerID);
if (dt != null) {
if (interest.cardinality() > 0 && !dt.peer.isInteresting()) {
dt.ms.addMessageToQueue(new Message_PP(PeerProtocol.INTERESTED, 2));
dt.peer.setInteresting(true);
}
}
dt = null;
}
public void compactEnd() {
final int compactSize = compactIndexUsed.cardinality();
if (compactSize == map.size()) {
return;
}
for (int i = 1; i < list.size(); i++) {
if (compactIndexUsed.get(i)) continue;
// to be removed
T t = list.get(i);
list.set(i, null);
map.remove(t);
if (addPosition > i) addPosition = i;
}
}
public void testFormat() throws Exception {
JobConf job = new JobConf(conf);
FileSystem fs = FileSystem.getLocal(conf);
Path dir = new Path(System.getProperty("test.build.data", ".") + "/mapred");
Path file = new Path(dir, "test.seq");
Reporter reporter = Reporter.NULL;
int seed = new Random().nextInt();
// LOG.info("seed = "+seed);
Random random = new Random(seed);
fs.delete(dir, true);
FileInputFormat.setInputPaths(job, dir);
// for a variety of lengths
for (int length = 0; length < MAX_LENGTH; length += random.nextInt(MAX_LENGTH / 10) + 1) {
// LOG.info("creating; entries = " + length);
// create a file with length entries
SequenceFile.Writer writer =
SequenceFile.createWriter(fs, conf, file, IntWritable.class, BytesWritable.class);
try {
for (int i = 0; i < length; i++) {
IntWritable key = new IntWritable(i);
byte[] data = new byte[random.nextInt(10)];
random.nextBytes(data);
BytesWritable value = new BytesWritable(data);
writer.append(key, value);
}
} finally {
writer.close();
}
// try splitting the file in a variety of sizes
InputFormat<IntWritable, BytesWritable> format =
new SequenceFileInputFormat<IntWritable, BytesWritable>();
IntWritable key = new IntWritable();
BytesWritable value = new BytesWritable();
for (int i = 0; i < 3; i++) {
int numSplits = random.nextInt(MAX_LENGTH / (SequenceFile.SYNC_INTERVAL / 20)) + 1;
// LOG.info("splitting: requesting = " + numSplits);
InputSplit[] splits = format.getSplits(job, numSplits);
// LOG.info("splitting: got = " + splits.length);
// check each split
BitSet bits = new BitSet(length);
for (int j = 0; j < splits.length; j++) {
RecordReader<IntWritable, BytesWritable> reader =
format.getRecordReader(splits[j], job, reporter);
try {
int count = 0;
while (reader.next(key, value)) {
// if (bits.get(key.get())) {
// LOG.info("splits["+j+"]="+splits[j]+" : " +
// key.get());
// LOG.info("@"+reader.getPos());
// }
assertFalse("Key in multiple partitions.", bits.get(key.get()));
bits.set(key.get());
count++;
}
// LOG.info("splits["+j+"]="+splits[j]+" count=" +
// count);
} finally {
reader.close();
}
}
assertEquals("Some keys in no partition.", length, bits.cardinality());
}
}
}
/** Returns the number of empty slots on this page. */
public int getNumEmptySlots() {
// hdj:!!! use getNumTuples instead of bitset.size() or bitset.length()
BitSet bitset = BitSet.valueOf(header);
return getNumTuples() - bitset.cardinality();
}
public void rTest(final int N) {
System.out.println("rtest N=" + N);
for (int gap = 1; gap <= 65536; gap *= 2) {
final BitSet bs1 = new BitSet();
final MutableRoaringBitmap rb1 = new MutableRoaringBitmap();
for (int x = 0; x <= N; x += gap) {
bs1.set(x);
rb1.add(x);
}
if (bs1.cardinality() != rb1.getCardinality()) throw new RuntimeException("different card");
if (!equals(bs1, rb1)) throw new RuntimeException("basic bug");
for (int offset = 1; offset <= gap; offset *= 2) {
final BitSet bs2 = new BitSet();
final MutableRoaringBitmap rb2 = new MutableRoaringBitmap();
for (int x = 0; x <= N; x += gap) {
bs2.set(x + offset);
rb2.add(x + offset);
}
if (bs2.cardinality() != rb2.getCardinality()) throw new RuntimeException("different card");
if (!equals(bs2, rb2)) throw new RuntimeException("basic bug");
BitSet clonebs1;
// testing AND
clonebs1 = (BitSet) bs1.clone();
clonebs1.and(bs2);
if (!equals(clonebs1, MutableRoaringBitmap.and(rb1, rb2)))
throw new RuntimeException("bug and");
{
final MutableRoaringBitmap t = rb1.clone();
t.and(rb2);
if (!equals(clonebs1, t)) throw new RuntimeException("bug inplace and");
if (!t.equals(MutableRoaringBitmap.and(rb1, rb2))) {
System.out.println(
t.highLowContainer.getContainerAtIndex(0).getClass().getCanonicalName());
System.out.println(
MutableRoaringBitmap.and(rb1, rb2)
.highLowContainer
.getContainerAtIndex(0)
.getClass()
.getCanonicalName());
throw new RuntimeException("bug inplace and");
}
}
// testing OR
clonebs1 = (BitSet) bs1.clone();
clonebs1.or(bs2);
if (!equals(clonebs1, MutableRoaringBitmap.or(rb1, rb2)))
throw new RuntimeException("bug or");
{
final MutableRoaringBitmap t = rb1.clone();
t.or(rb2);
if (!equals(clonebs1, t)) throw new RuntimeException("bug or");
if (!t.equals(MutableRoaringBitmap.or(rb1, rb2))) throw new RuntimeException("bug or");
if (!t.toString().equals(MutableRoaringBitmap.or(rb1, rb2).toString()))
throw new RuntimeException("bug or");
}
// testing XOR
clonebs1 = (BitSet) bs1.clone();
clonebs1.xor(bs2);
if (!equals(clonebs1, MutableRoaringBitmap.xor(rb1, rb2))) {
throw new RuntimeException("bug xor");
}
{
final MutableRoaringBitmap t = rb1.clone();
t.xor(rb2);
if (!equals(clonebs1, t)) throw new RuntimeException("bug xor");
if (!t.equals(MutableRoaringBitmap.xor(rb1, rb2))) {
System.out.println(t);
System.out.println(MutableRoaringBitmap.xor(rb1, rb2));
System.out.println(
Arrays.equals(t.toArray(), MutableRoaringBitmap.xor(rb1, rb2).toArray()));
throw new RuntimeException("bug xor");
}
}
// testing NOTAND
clonebs1 = (BitSet) bs1.clone();
clonebs1.andNot(bs2);
if (!equals(clonebs1, MutableRoaringBitmap.andNot(rb1, rb2))) {
throw new RuntimeException("bug andnot");
}
clonebs1 = (BitSet) bs2.clone();
clonebs1.andNot(bs1);
if (!equals(clonebs1, MutableRoaringBitmap.andNot(rb2, rb1))) {
throw new RuntimeException("bug andnot");
}
{
final MutableRoaringBitmap t = rb2.clone();
t.andNot(rb1);
if (!equals(clonebs1, t)) {
throw new RuntimeException("bug inplace andnot");
}
final MutableRoaringBitmap g = MutableRoaringBitmap.andNot(rb2, rb1);
if (!equals(clonebs1, g)) {
throw new RuntimeException("bug andnot");
}
if (!t.equals(g)) throw new RuntimeException("bug");
}
clonebs1 = (BitSet) bs1.clone();
clonebs1.andNot(bs2);
if (!equals(clonebs1, MutableRoaringBitmap.andNot(rb1, rb2))) {
throw new RuntimeException("bug andnot");
}
{
final MutableRoaringBitmap t = rb1.clone();
t.andNot(rb2);
if (!equals(clonebs1, t)) {
throw new RuntimeException("bug andnot");
}
final MutableRoaringBitmap g = MutableRoaringBitmap.andNot(rb1, rb2);
if (!equals(clonebs1, g)) {
throw new RuntimeException("bug andnot");
}
if (!t.equals(g)) throw new RuntimeException("bug");
}
}
}
}
public static boolean equals(BitSet bs, MutableRoaringBitmap rr) {
final int[] a = new int[bs.cardinality()];
int pos = 0;
for (int x = bs.nextSetBit(0); x >= 0; x = bs.nextSetBit(x + 1)) a[pos++] = x;
return Arrays.equals(rr.toArray(), a);
}