private void checkQueueGet(String key, KeyValue expected, String... entries) throws Exception {
PartitionAttributesFactory paf = new PartitionAttributesFactory();
paf.setTotalNumBuckets(1);
RegionFactory rf1 = this.c.createRegionFactory(RegionShortcut.PARTITION);
PartitionedRegion r1 =
(PartitionedRegion) rf1.setPartitionAttributes(paf.create()).create("r1");
// create the buckets
r1.put("blah", "blah");
// hack to get the queue.
HDFSParallelGatewaySenderQueue hopqueue = getHDFSQueue(r1, this.c);
HDFSBucketRegionQueue brq =
(HDFSBucketRegionQueue)
((PartitionedRegion) hopqueue.getRegion()).getDataStore().getLocalBucketById(0);
int seq = 0;
for (String s : entries) {
if (s.equals("roll")) {
brq.rolloverSkipList();
} else {
String[] kv = s.split("-");
hopqueue.put(getNewEvent(kv[0], kv[1], r1, 0, seq++));
}
}
byte[] bkey = EntryEventImpl.serialize(key);
HDFSGatewayEventImpl evt = hopqueue.get(r1, bkey, 0);
if (expected == null) {
assertNull(evt);
} else {
assertEquals(expected.key, evt.getKey());
assertEquals(expected.value, evt.getDeserializedValue());
}
}
public void testHopQueueWithOneBucket() throws Exception {
this.c.close();
this.c = createCache();
PartitionAttributesFactory paf = new PartitionAttributesFactory();
paf.setTotalNumBuckets(1);
RegionFactory rf1 = this.c.createRegionFactory(RegionShortcut.PARTITION);
PartitionedRegion r1 =
(PartitionedRegion) rf1.setPartitionAttributes(paf.create()).create("r1");
r1.put("K9", "x1");
r1.put("K8", "x2");
// hack to get the queue.
HDFSParallelGatewaySenderQueue hopqueue = getHDFSQueue(r1, this.c);
HDFSBucketRegionQueue hdfsBQ =
(HDFSBucketRegionQueue)
((PartitionedRegion) hopqueue.getRegion()).getDataStore().getLocalBucketById(0);
EntryEventImpl ev1 =
EntryEventImpl.create(
(LocalRegion) r1,
Operation.CREATE,
(Object) "K1",
(Object) "V1",
null,
false,
(DistributedMember) c.getMyId());
// put some keys with multiple updates.
hopqueue.put(getNewEvent("K2", "V2", r1, 0, 2));
hopqueue.put(getNewEvent("K3", "V3a", r1, 0, 8));
hopqueue.put(getNewEvent("K3", "V3", r1, 0, 7));
hopqueue.put(getNewEvent("K1", "V1", r1, 0, 3));
hopqueue.put(getNewEvent("K2", "V2a", r1, 0, 6));
hopqueue.put(getNewEvent("K3", "V3b", r1, 0, 9));
assertTrue(
" skip list size should be 6 ", getSortedEventQueue(hdfsBQ).currentSkipList.size() == 6);
// peek a key. it should be the lowesy
Object[] l = hopqueue.peek(1, 0).toArray();
assertTrue(
"First key should be K1 but is " + ((HDFSGatewayEventImpl) l[0]).getKey(),
((HDFSGatewayEventImpl) l[0]).getKey().equals("K1"));
assertTrue(
" Peeked skip list size should be 0 ",
getSortedEventQueue(hdfsBQ).getPeeked().size() == 6);
assertTrue(
" skip list size should be 6 ", getSortedEventQueue(hdfsBQ).currentSkipList.size() == 0);
// try to fetch the key. it would be in peeked skip list but still available
Object o = hopqueue.get(r1, CacheServerHelper.serialize("K1"), 0);
assertTrue("First key should be K1", ((HDFSGatewayEventImpl) o).getKey().equals("K1"));
assertTrue(
" skip lists size should be 6",
(getSortedEventQueue(hdfsBQ).getPeeked().size()
+ getSortedEventQueue(hdfsBQ).currentSkipList.size())
== 6);
o = hopqueue.get(r1, CacheServerHelper.serialize("K2"), 0);
Object v = ((HDFSGatewayEventImpl) o).getDeserializedValue();
assertTrue(" key should K2 with value V2a but the value was " + v, ((String) v).equals("V2a"));
o = hopqueue.get(r1, CacheServerHelper.serialize("K3"), 0);
v = ((HDFSGatewayEventImpl) o).getDeserializedValue();
assertTrue(" key should K3 with value V3b but the value was " + v, ((String) v).equals("V3b"));
}
public void testPeekABatch() throws Exception {
this.c.close();
this.c = createCache();
PartitionAttributesFactory paf = new PartitionAttributesFactory();
paf.setTotalNumBuckets(1);
RegionFactory rf1 = this.c.createRegionFactory(RegionShortcut.PARTITION);
PartitionedRegion r1 =
(PartitionedRegion) rf1.setPartitionAttributes(paf.create()).create("r1");
r1.put("K9", "x1");
r1.put("K8", "x2");
// hack to get the queue.
HDFSParallelGatewaySenderQueue hopqueue = getHDFSQueue(r1, this.c);
HDFSBucketRegionQueue hdfsBQ =
(HDFSBucketRegionQueue)
((PartitionedRegion) hopqueue.getRegion()).getDataStore().getLocalBucketById(0);
// put some keys with multiple updates.
hopqueue.put(getNewEvent("K2", "V2", r1, 0, 2));
hopqueue.put(getNewEvent("K3", "V3a", r1, 0, 8));
hopqueue.put(getNewEvent("K3", "V3", r1, 0, 7));
hopqueue.put(getNewEvent("K1", "V1", r1, 0, 3));
hopqueue.put(getNewEvent("K2", "V2a", r1, 0, 6));
hopqueue.put(getNewEvent("K3", "V3b", r1, 0, 9));
getSortedEventQueue(hdfsBQ).rollover(true);
hopqueue.put(getNewEvent("K1", "V12", r1, 0, 11));
hopqueue.put(getNewEvent("K5", "V3a", r1, 0, 12));
hopqueue.put(getNewEvent("K5", "V3b", r1, 0, 13));
assertTrue(
" skip list size should be 3 but is " + getSortedEventQueue(hdfsBQ).currentSkipList.size(),
getSortedEventQueue(hdfsBQ).currentSkipList.size() == 3);
assertTrue(
" skip list size should be 6 but is "
+ getSortedEventQueue(hdfsBQ).queueOfLists.peek().size(),
getSortedEventQueue(hdfsBQ).queueOfLists.peek().size() == 6);
Object o1 = hopqueue.get(r1, CacheServerHelper.serialize("K3"), 0);
;
Object o2 = hopqueue.get(r1, CacheServerHelper.serialize("K1"), 0);
;
Object v1 = ((HDFSGatewayEventImpl) o1).getDeserializedValue();
Object v2 = ((HDFSGatewayEventImpl) o2).getDeserializedValue();
assertTrue(
" key should K3 with value V3b but the value was " + v1, ((String) v1).equals("V3b"));
assertTrue(
" key should K1 with value V12 but the value was " + v2, ((String) v2).equals("V12"));
ArrayList a = hdfsBQ.peekABatch();
assertTrue(
"First key should be K1 but is " + ((HDFSGatewayEventImpl) a.get(0)).getKey(),
((HDFSGatewayEventImpl) a.get(0)).getKey().equals("K1"));
assertTrue(
"Second key should be K2 but is " + ((HDFSGatewayEventImpl) a.get(1)).getKey(),
((HDFSGatewayEventImpl) a.get(1)).getKey().equals("K2"));
assertTrue(
"Third key should be K2 but is " + ((HDFSGatewayEventImpl) a.get(2)).getKey(),
((HDFSGatewayEventImpl) a.get(2)).getKey().equals("K2"));
assertTrue(
" Peeked skip list size should be 6 ", getSortedEventQueue(hdfsBQ).getPeeked().size() == 6);
assertTrue(
" queueOfLists size should be 2 ", getSortedEventQueue(hdfsBQ).queueOfLists.size() == 2);
assertTrue(
" skip list size should be 3 ", getSortedEventQueue(hdfsBQ).currentSkipList.size() == 3);
o1 = hopqueue.get(r1, CacheServerHelper.serialize("K3"), 0);
;
o2 = hopqueue.get(r1, CacheServerHelper.serialize("K1"), 0);
;
v1 = ((HDFSGatewayEventImpl) o1).getDeserializedValue();
v2 = ((HDFSGatewayEventImpl) o2).getDeserializedValue();
assertTrue(
" key should K3 with value V3b but the value was " + v1, ((String) v1).equals("V3b"));
assertTrue(
" key should K1 with value V12 but the value was " + v2, ((String) v2).equals("V12"));
java.util.Iterator<KeyToSeqNumObject> iter1 =
getSortedEventQueue(hdfsBQ).getPeeked().iterator();
assertTrue("key in peeked list should be 3 ", iter1.next().getSeqNum() == 3);
assertTrue("key in peeked list should be 6 ", iter1.next().getSeqNum() == 6);
assertTrue("key in peeked list should be 2 ", iter1.next().getSeqNum() == 2);
assertTrue("key in peeked list should be 9 ", iter1.next().getSeqNum() == 9);
assertTrue("key in peeked list should be 8 ", iter1.next().getSeqNum() == 8);
assertTrue("key in peeked list should be 7 ", iter1.next().getSeqNum() == 7);
assertTrue(" Peeked list should not have any more elements. ", iter1.hasNext() == false);
java.util.Iterator<KeyToSeqNumObject> iter2 =
getSortedEventQueue(hdfsBQ).currentSkipList.iterator();
assertTrue("key in peeked list should be 11", iter2.next().getSeqNum() == 11);
assertTrue("key in peeked list should be 13", iter2.next().getSeqNum() == 13);
assertTrue("key in peeked list should be 12 ", iter2.next().getSeqNum() == 12);
iter2 = getSortedEventQueue(hdfsBQ).currentSkipList.iterator();
HashSet<Long> hs = new HashSet<Long>();
hs.add((long) 11);
hs.add((long) 13);
hs.add((long) 12);
hs.add((long) 3);
hs.add((long) 6);
hs.add((long) 2);
hs.add((long) 9);
hs.add((long) 8);
hs.add((long) 7);
hdfsBQ.hdfsEventQueue.handleRemainingElements(hs);
ArrayList a1 = hdfsBQ.peekABatch();
o1 = hopqueue.get(r1, CacheServerHelper.serialize("K3"), 0);
;
o2 = hopqueue.get(r1, CacheServerHelper.serialize("K1"), 0);
;
v2 = ((HDFSGatewayEventImpl) o2).getDeserializedValue();
assertTrue(" key should K3 should not have been found ", o1 == null);
assertTrue(
" key should K1 with value V12 but the value was " + v2, ((String) v2).equals("V12"));
assertTrue(
"First key should be K1 but is " + ((HDFSGatewayEventImpl) a1.get(0)).getKey(),
((HDFSGatewayEventImpl) a1.get(0)).getKey().equals("K1"));
assertTrue(
"Second key should be K5 but is " + ((HDFSGatewayEventImpl) a1.get(1)).getKey(),
((HDFSGatewayEventImpl) a1.get(1)).getKey().equals("K5"));
assertTrue(
"Third key should be K5 but is " + ((HDFSGatewayEventImpl) a1.get(2)).getKey(),
((HDFSGatewayEventImpl) a1.get(2)).getKey().equals("K5"));
assertTrue(
" Peeked skip list size should be 3 ",
getSortedEventQueue(hdfsBQ).getPeeked().size() == 3);
assertTrue(
" skip list size should be 0 but is " + getSortedEventQueue(hdfsBQ).currentSkipList.size(),
getSortedEventQueue(hdfsBQ).currentSkipList.size() == 0);
assertTrue(
" skip list size should be 3 but is "
+ getSortedEventQueue(hdfsBQ).queueOfLists.peek().size(),
getSortedEventQueue(hdfsBQ).queueOfLists.peek().size() == 3);
assertTrue(
" skip list size should be 2 but is " + getSortedEventQueue(hdfsBQ).queueOfLists.size(),
getSortedEventQueue(hdfsBQ).queueOfLists.size() == 2);
}