Esempio n. 1
0
 void subscribe(ByteString topic, ByteString subscriberId) throws Exception {
   org.apache.hedwig.protocol.PubSubProtocol.SubscriptionOptions options =
       org.apache.hedwig.protocol.PubSubProtocol.SubscriptionOptions.newBuilder()
           .setCreateOrAttach(
               org.apache.hedwig.protocol.PubSubProtocol.SubscribeRequest.CreateOrAttach
                   .CREATE_OR_ATTACH)
           .build();
   subscribe(topic, subscriberId, options);
 }
Esempio n. 2
0
    void sendXExpectLastY(ByteString topic, ByteString subid, final int x, final int y)
        throws Exception {
      for (int i = 0; i < x; i++) {
        publisher.publish(
            topic,
            org.apache.hedwig.protocol.PubSubProtocol.Message.newBuilder()
                .setBody(ByteString.copyFromUtf8(String.valueOf(i)))
                .build());
      }
      org.apache.hedwig.protocol.PubSubProtocol.SubscriptionOptions opts =
          org.apache.hedwig.protocol.PubSubProtocol.SubscriptionOptions.newBuilder()
              .setCreateOrAttach(
                  org.apache.hedwig.protocol.PubSubProtocol.SubscribeRequest.CreateOrAttach.ATTACH)
              .build();
      subscriber.subscribe(topic, subid, opts);

      final AtomicInteger expected = new AtomicInteger(x - y);
      final CountDownLatch latch = new CountDownLatch(1);
      subscriber.startDelivery(
          topic,
          subid,
          new org.apache.hedwig.client.api.MessageHandler() {
            @Override
            public synchronized void deliver(
                ByteString topic,
                ByteString subscriberId,
                org.apache.hedwig.protocol.PubSubProtocol.Message msg,
                org.apache.hedwig.util.Callback<Void> callback,
                Object context) {
              try {
                int value = Integer.valueOf(msg.getBody().toStringUtf8());
                if (value == expected.get()) {
                  expected.incrementAndGet();
                } else {
                  logger.error(
                      "Did not receive expected value, expected {}, got {}", expected.get(), value);
                  expected.set(0);
                  latch.countDown();
                }
                if (expected.get() == x) {
                  latch.countDown();
                }
                callback.operationFinished(context, null);
              } catch (Exception e) {
                logger.error("Received bad message", e);
                latch.countDown();
              }
            }
          });
      assertTrue(
          "Timed out waiting for messages Y is " + y + " expected is currently " + expected.get(),
          latch.await(10, TimeUnit.SECONDS));
      assertEquals("Should be expected message with " + x, x, expected.get());
      subscriber.stopDelivery(topic, subid);
      subscriber.closeSubscription(topic, subid);
    }
Esempio n. 3
0
    void receiveNumModM(
        final ByteString topic, final ByteString subid, final int start, final int num, final int M)
        throws Exception {
      org.apache.hedwig.filter.ServerMessageFilter filter =
          new org.apache.hedwig.filter.ServerMessageFilter() {

            @Override
            public org.apache.hedwig.filter.ServerMessageFilter initialize(Configuration conf) {
              // do nothing
              return this;
            }

            @Override
            public void uninitialize() {
              // do nothing;
            }

            @Override
            public org.apache.hedwig.filter.MessageFilterBase setSubscriptionPreferences(
                ByteString topic,
                ByteString subscriberId,
                org.apache.hedwig.protocol.PubSubProtocol.SubscriptionPreferences preferences) {
              // do nothing;
              return this;
            }

            @Override
            public boolean testMessage(org.apache.hedwig.protocol.PubSubProtocol.Message msg) {
              int value = Integer.valueOf(msg.getBody().toStringUtf8());
              return 0 == value % M;
            }
          };
      filter.initialize(conf.getConf());

      org.apache.hedwig.protocol.PubSubProtocol.SubscriptionOptions opts =
          org.apache.hedwig.protocol.PubSubProtocol.SubscriptionOptions.newBuilder()
              .setCreateOrAttach(
                  org.apache.hedwig.protocol.PubSubProtocol.SubscribeRequest.CreateOrAttach.ATTACH)
              .build();
      subscriber.subscribe(topic, subid, opts);
      final int base = start + M - start % M;
      final AtomicInteger expected = new AtomicInteger(base);
      final CountDownLatch latch = new CountDownLatch(1);
      subscriber.startDeliveryWithFilter(
          topic,
          subid,
          new org.apache.hedwig.client.api.MessageHandler() {
            public synchronized void deliver(
                ByteString topic,
                ByteString subscriberId,
                org.apache.hedwig.protocol.PubSubProtocol.Message msg,
                org.apache.hedwig.util.Callback<Void> callback,
                Object context) {
              try {
                int value = Integer.valueOf(msg.getBody().toStringUtf8());
                // duplicated messages received, ignore them
                if (value > start) {
                  if (value == expected.get()) {
                    expected.addAndGet(M);
                  } else {
                    logger.error(
                        "Did not receive expected value, expected {}, got {}",
                        expected.get(),
                        value);
                    expected.set(0);
                    latch.countDown();
                  }
                  if (expected.get() == (base + num * M)) {
                    latch.countDown();
                  }
                }
                callback.operationFinished(context, null);
              } catch (Exception e) {
                logger.error("Received bad message", e);
                latch.countDown();
              }
            }
          },
          (org.apache.hedwig.filter.ClientMessageFilter) filter);
      assertTrue(
          "Timed out waiting for messages mod " + M + " expected is " + expected.get(),
          latch.await(10, TimeUnit.SECONDS));
      assertEquals(
          "Should be expected message with " + (base + num * M), (base + num * M), expected.get());
      subscriber.stopDelivery(topic, subid);
      filter.uninitialize();
      subscriber.closeSubscription(topic, subid);
    }
Esempio n. 4
0
  /**
   * Test compatability between version 4.1.0 and the current version.
   *
   * <p>A 4.1.0 client could not update message bound, while current could do it.
   */
  @Test(timeout = 60000)
  public void testUpdateMessageBoundCompat410() throws Exception {
    ByteString topic = ByteString.copyFromUtf8("TestUpdateMessageBoundCompat410");
    ByteString subid = ByteString.copyFromUtf8("mysub");

    // start bookkeeper
    BookKeeperCluster420 bkc420 = new BookKeeperCluster420(3);
    bkc420.start();

    int port = PortManager.nextFreePort();
    int sslPort = PortManager.nextFreePort();

    // start hub server
    Server420 s420 = new Server420(zkUtil.getZooKeeperConnectString(), port, sslPort);
    s420.start();

    org.apache.hedwig.protocol.PubSubProtocol.SubscriptionOptions options5cur =
        org.apache.hedwig.protocol.PubSubProtocol.SubscriptionOptions.newBuilder()
            .setCreateOrAttach(
                org.apache.hedwig.protocol.PubSubProtocol.SubscribeRequest.CreateOrAttach
                    .CREATE_OR_ATTACH)
            .setMessageBound(5)
            .build();
    org.apache.hw_v4_1_0.hedwig.protocol.PubSubProtocol.SubscriptionOptions options5v410 =
        org.apache.hw_v4_1_0.hedwig.protocol.PubSubProtocol.SubscriptionOptions.newBuilder()
            .setCreateOrAttach(
                org.apache.hw_v4_1_0.hedwig.protocol.PubSubProtocol.SubscribeRequest.CreateOrAttach
                    .CREATE_OR_ATTACH)
            .setMessageBound(5)
            .build();
    org.apache.hw_v4_1_0.hedwig.protocol.PubSubProtocol.SubscriptionOptions options20v410 =
        org.apache.hw_v4_1_0.hedwig.protocol.PubSubProtocol.SubscriptionOptions.newBuilder()
            .setCreateOrAttach(
                org.apache.hw_v4_1_0.hedwig.protocol.PubSubProtocol.SubscribeRequest.CreateOrAttach
                    .CREATE_OR_ATTACH)
            .setMessageBound(20)
            .build();

    Client410 c410 = new Client410("localhost:" + port + ":" + sslPort);
    c410.subscribe(topic, subid, options20v410);
    c410.closeSubscription(topic, subid);
    Thread.sleep(1000); // give server time to run disconnect logic (BOOKKEEPER-513)

    c410.sendXExpectLastY(topic, subid, 50, 20);

    c410.subscribe(topic, subid, options5v410);
    c410.closeSubscription(topic, subid);
    Thread.sleep(1000); // give server time to run disconnect logic (BOOKKEEPER-513)

    // the message bound isn't updated.
    c410.sendXExpectLastY(topic, subid, 50, 20);

    ClientCurrent ccur = new ClientCurrent("localhost:" + port + ":" + sslPort);
    ccur.subscribe(topic, subid, options5cur);
    ccur.closeSubscription(topic, subid);
    Thread.sleep(1000); // give server time to run disconnect logic (BOOKKEEPER-513)

    // the message bound should be updated.
    c410.sendXExpectLastY(topic, subid, 50, 5);

    // stop 420 server
    s420.stop();

    c410.close();
    ccur.close();

    // stop bookkeeper cluster
    bkc420.stop();
  }
Esempio n. 5
0
  /**
   * Test compatability of message bound between version 4.0.0 and current version.
   *
   * <p>1) message bound doesn't take effects on 4.0.0 server. 2) message bound take effects on both
   * 4.1.0 and current server
   */
  @Test(timeout = 60000)
  public void testMessageBoundCompat() throws Exception {
    ByteString topic = ByteString.copyFromUtf8("testMessageBoundCompat");
    ByteString subid = ByteString.copyFromUtf8("mysub");

    int port = PortManager.nextFreePort();
    int sslPort = PortManager.nextFreePort();

    // start bookkeeper 400
    BookKeeperCluster400 bkc400 = new BookKeeperCluster400(3);
    bkc400.start();

    // start 400 server
    Server400 s400 = new Server400(zkUtil.getZooKeeperConnectString(), port, sslPort);
    s400.start();

    org.apache.hedwig.protocol.PubSubProtocol.SubscriptionOptions options5cur =
        org.apache.hedwig.protocol.PubSubProtocol.SubscriptionOptions.newBuilder()
            .setCreateOrAttach(
                org.apache.hedwig.protocol.PubSubProtocol.SubscribeRequest.CreateOrAttach
                    .CREATE_OR_ATTACH)
            .setMessageBound(5)
            .build();

    ClientCurrent ccur = new ClientCurrent("localhost:" + port + ":" + sslPort);
    ccur.subscribe(topic, subid, options5cur);
    ccur.closeSubscription(topic, subid);
    ccur.sendXExpectLastY(topic, subid, 50, 50);

    // stop 400 servers
    s400.stop();
    bkc400.stop();

    // start bookkeeper 410
    BookKeeperCluster410 bkc410 = new BookKeeperCluster410(3);
    bkc410.start();

    // start 410 server
    Server410 s410 = new Server410(zkUtil.getZooKeeperConnectString(), port, sslPort);
    s410.start();

    ccur.subscribe(topic, subid, options5cur);
    ccur.closeSubscription(topic, subid);
    ccur.sendXExpectLastY(topic, subid, 50, 5);

    // stop 410 servers
    s410.stop();
    bkc410.stop();

    // start bookkeeper current
    BookKeeperCluster420 bkc420 = new BookKeeperCluster420(3);
    bkc420.start();

    // start 420 server
    Server420 s420 = new Server420(zkUtil.getZooKeeperConnectString(), port, sslPort);
    s420.start();

    ccur.subscribe(topic, subid, options5cur);
    ccur.closeSubscription(topic, subid);
    ccur.sendXExpectLastY(topic, subid, 50, 5);

    // stop 420 server
    s420.stop();
    bkc420.stop();

    ccur.close();
  }
Esempio n. 6
0
    // throttle doesn't work talking with 41 server
    void throttleX41(ByteString topic, ByteString subid, final int X) throws Exception {
      org.apache.hedwig.protocol.PubSubProtocol.SubscriptionOptions options =
          org.apache.hedwig.protocol.PubSubProtocol.SubscriptionOptions.newBuilder()
              .setCreateOrAttach(
                  org.apache.hedwig.protocol.PubSubProtocol.SubscribeRequest.CreateOrAttach
                      .CREATE_OR_ATTACH)
              .setMessageWindowSize(X)
              .build();
      subscribe(topic, subid, options);
      closeSubscription(topic, subid);
      publishInts(topic, 1, 3 * X);
      subscribe(topic, subid);

      final AtomicInteger expected = new AtomicInteger(1);
      final CountDownLatch throttleLatch = new CountDownLatch(1);
      final CountDownLatch nonThrottleLatch = new CountDownLatch(1);
      subscriber.startDelivery(
          topic,
          subid,
          new org.apache.hedwig.client.api.MessageHandler() {
            @Override
            public synchronized void deliver(
                ByteString topic,
                ByteString subscriberId,
                org.apache.hedwig.protocol.PubSubProtocol.Message msg,
                org.apache.hedwig.util.Callback<Void> callback,
                Object context) {
              try {
                int value = Integer.valueOf(msg.getBody().toStringUtf8());
                logger.debug("Received message {},", value);

                if (value == expected.get()) {
                  expected.incrementAndGet();
                } else {
                  // error condition
                  logger.error(
                      "Did not receive expected value, expected {}, got {}", expected.get(), value);
                  expected.set(0);
                  throttleLatch.countDown();
                  nonThrottleLatch.countDown();
                }
                if (expected.get() > X + 1) {
                  throttleLatch.countDown();
                }
                if (expected.get() == (3 * X + 1)) {
                  nonThrottleLatch.countDown();
                }
                callback.operationFinished(context, null);
              } catch (Exception e) {
                logger.error("Received bad message", e);
                throttleLatch.countDown();
                nonThrottleLatch.countDown();
              }
            }
          });
      assertTrue(
          "Should Receive more messages than throttle value " + X,
          throttleLatch.await(10, TimeUnit.SECONDS));

      assertTrue(
          "Timed out waiting for messages " + (3 * X + 1),
          nonThrottleLatch.await(10, TimeUnit.SECONDS));
      assertEquals("Should be expected message with " + (3 * X + 1), 3 * X + 1, expected.get());

      subscriber.stopDelivery(topic, subid);
      closeSubscription(topic, subid);
    }