public void shutdown() throws IOException {
   kafka.shutdown();
   if (zookeeper.getState().equals(CuratorFrameworkState.STARTED)) {
     zookeeper.close();
   }
   server.close();
   FileUtils.deleteQuietly(new File(getLogDir()));
 }
Esempio n. 2
0
  public void start() throws Exception {
    zkServer = new TestingServer(zkPort, true);
    zkClient = KafkaUtils.createZkClient(zkServer.getConnectString());

    kafkaServer = new KafkaServerStartable(new KafkaConfig(getServerProperties()));
    kafkaServer.startup();
    LOGGER.debug("Started Kafka server at port {}", kafkaPort);
  }
Esempio n. 3
0
 public void stop() throws IOException {
   if (zkClient != null) {
     zkClient.close();
   }
   if (kafkaServer != null) {
     kafkaServer.shutdown();
   }
   if (zkServer != null) {
     zkServer.stop();
   }
 }
Esempio n. 4
0
  @AfterClass
  public static void shutdown() {
    kafkaGateway.stop();
    pluginRegistry.stop();

    simpleKafkaGateway.stop();
    pluginRegistry2.stop();

    try {
      if (kafkaServer != null) {
        kafkaServer.shutdown();
        kafkaServer.awaitShutdown();
      }
      zkTestServer.shutdownNetwork();
    } finally {
      try {
        FileUtils.deleteDirectory(kafkaLogFile);
      } catch (IOException e) {
        e.printStackTrace();
      }
    }
  }
 public KafkaTestBroker() {
   try {
     server = new TestingServer();
     String zookeeperConnectionString = server.getConnectString();
     ExponentialBackoffRetry retryPolicy = new ExponentialBackoffRetry(1000, 3);
     zookeeper = CuratorFrameworkFactory.newClient(zookeeperConnectionString, retryPolicy);
     zookeeper.start();
     port = InstanceSpec.getRandomPort();
     kafka.server.KafkaConfig config = buildKafkaConfig(zookeeperConnectionString);
     kafka = new KafkaServerStartable(config);
     kafka.startup();
   } catch (Exception ex) {
     throw new RuntimeException("Could not start test broker", ex);
   }
 }
Esempio n. 6
0
  public void start() throws Exception {
    zkServer = new TestingServer(zkPort, true);
    zkClient = new ZkClient(zkServer.getConnectString(), 10000, 10000, ZKStringSerializer$.MODULE$);

    File logs = Files.createTempDirectory("kafka_tmp").toFile();
    logs.deleteOnExit();
    LOGGER.debug("Created temp log dir: {}", logs.getAbsolutePath());

    Properties serverProperties = new Properties();
    serverProperties.put("zookeeper.connect", zkServer.getConnectString());
    serverProperties.put("broker.id", "1");
    serverProperties.put("host.name", "localhost");
    serverProperties.put("port", String.valueOf(kafkaPort));
    serverProperties.put("log.dir", logs.getAbsolutePath());
    serverProperties.put("log.flush.interval.messages", "1");

    kafkaServer = new KafkaServerStartable(new KafkaConfig(serverProperties));
    kafkaServer.startup();
    AdminUtils.createTopic(zkClient, topicName, topicPartitions, 1, new Properties());
  }
Esempio n. 7
0
 public void stop() throws IOException {
   if (zkClient != null) zkClient.close();
   if (kafkaServer != null) kafkaServer.shutdown();
   if (zkServer != null) zkServer.stop();
   LOGGER.debug("Zookeeper / Kafka services stopped!");
 }
Esempio n. 8
0
  @BeforeClass
  public static void init() throws Exception {
    final ShutdownRegistryImpl shutdownRegistry = new ShutdownRegistryImpl();

    try {
      zkTestServer =
          new ZooKeeperTestServer(0, shutdownRegistry, ZooKeeperTestServer.DEFAULT_SESSION_TIMEOUT);
      port = zkTestServer.startNetwork();
    } catch (Exception e) {
      throw new RuntimeException(e);
    }

    Properties kafkaProps = new Properties();
    kafkaProps.setProperty("num.partitions", "1");
    kafkaProps.setProperty("port", "9092");
    kafkaProps.setProperty("broker.id", "0");
    kafkaProps.setProperty("log.dir", "/tmp/sensei-gateway-test-kafka-logs");
    // override to the local running zk server
    kafkaProps.setProperty("zookeeper.connect", "localhost:" + port);

    kafkaLogFile = new File(kafkaProps.getProperty("log.dir"));
    FileUtils.deleteDirectory(kafkaLogFile);

    KafkaConfig kafkaConfig = new KafkaConfig(kafkaProps);
    kafkaServer = new KafkaServerStartable(kafkaConfig);

    kafkaServer.startup();

    Configuration config = new PropertiesConfiguration();
    config.addProperty(
        "sensei.gateway.class", "com.senseidb.gateway.kafka.KafkaDataProviderBuilder");
    config.addProperty("sensei.gateway.kafka.group.id", "1");
    config.addProperty("sensei.gateway.kafka.zookeeper.connect", "localhost:" + port);
    config.addProperty("sensei.gateway.kafka.auto.offset.reset", "smallest");
    config.addProperty("sensei.gateway.kafka.topic", "test");
    config.addProperty("sensei.gateway.provider.batchSize", "1");
    pluginRegistry = SenseiPluginRegistry.build(config);
    pluginRegistry.start();

    kafkaGateway = pluginRegistry.getBeanByFullPrefix("sensei.gateway", SenseiGateway.class);
    kafkaGateway.start();

    config = new PropertiesConfiguration();
    config.addProperty("sensei.gateway.class", "com.senseidb.gateway.kafka.SimpleKafkaGateway");
    config.addProperty("sensei.gateway.kafka.host", "localhost");
    config.addProperty("sensei.gateway.kafka.port", "9092");
    config.addProperty("sensei.gateway.kafka.topic", "test");
    config.addProperty("sensei.gateway.kafka.timeout", "3000");
    config.addProperty("sensei.gateway.provider.batchSize", "1");
    pluginRegistry2 = SenseiPluginRegistry.build(config);
    pluginRegistry2.start();

    simpleKafkaGateway = pluginRegistry2.getBeanByFullPrefix("sensei.gateway", SenseiGateway.class);
    simpleKafkaGateway.start();

    Properties props = new Properties();
    props.put("metadata.broker.list", "localhost:9092");
    props.put("serializer.class", "kafka.serializer.StringEncoder");

    ProducerConfig producerConfig = new ProducerConfig(props);
    Producer<String, String> kafkaProducer = new Producer<String, String>(producerConfig);

    for (JSONObject jsonObj : BaseGatewayTestUtil.dataList) {
      KeyedMessage<String, String> data =
          new KeyedMessage<String, String>("test", jsonObj.toString());
      kafkaProducer.send(data);
    }
  }