Example #1
0
  public SenseiServerBuilder(File confDir, Map<String, Object> properties) throws Exception {
    if (properties != null) {
      _senseiConfFile = null;
      _senseiConf = new MapConfiguration(properties);
      ((MapConfiguration) _senseiConf).setDelimiterParsingDisabled(true);
    } else {
      _senseiConfFile = new File(confDir, SENSEI_PROPERTIES);
      if (!_senseiConfFile.exists()) {
        throw new ConfigurationException(
            "configuration file: " + _senseiConfFile.getAbsolutePath() + " does not exist.");
      }
      _senseiConf = new PropertiesConfiguration();
      ((PropertiesConfiguration) _senseiConf).setDelimiterParsingDisabled(true);
      ((PropertiesConfiguration) _senseiConf).load(_senseiConfFile);
    }

    pluginRegistry = SenseiPluginRegistry.build(_senseiConf);
    pluginRegistry.start();

    processRelevanceFunctionPlugins(pluginRegistry);
    processRelevanceExternalObjectPlugins(pluginRegistry);

    _gateway = pluginRegistry.getBeanByFullPrefix(SENSEI_GATEWAY, SenseiGateway.class);
    _schemaDoc = loadSchema(confDir);
    _senseiSchema = SenseiSchema.build(_schemaDoc);
  }
Example #2
0
  public SenseiServerBuilder(Resource confDir, Map<String, Object> properties) throws Exception {
    _senseiConfFile = null;

    _senseiConf = new MapConfiguration(properties);
    ((MapConfiguration) _senseiConf).setDelimiterParsingDisabled(true);

    pluginRegistry = SenseiPluginRegistry.build(_senseiConf);
    pluginRegistry.start();

    processRelevanceFunctionPlugins(pluginRegistry);

    _gateway = pluginRegistry.getBeanByFullPrefix(SENSEI_GATEWAY, SenseiGateway.class);

    _schemaDoc = loadSchema(confDir);
    _senseiSchema = SenseiSchema.build(_schemaDoc);
  }
Example #3
0
  @BeforeClass
  public static void init() throws Exception {
    final ShutdownRegistryImpl shutdownRegistry = new ShutdownRegistryImpl();

    try {
      zkTestServer =
          new ZooKeeperTestServer(0, shutdownRegistry, ZooKeeperTestServer.DEFAULT_SESSION_TIMEOUT);
      port = zkTestServer.startNetwork();
    } catch (Exception e) {
      throw new RuntimeException(e);
    }

    Properties kafkaProps = new Properties();
    kafkaProps.setProperty("num.partitions", "1");
    kafkaProps.setProperty("port", "9092");
    kafkaProps.setProperty("broker.id", "0");
    kafkaProps.setProperty("log.dir", "/tmp/sensei-gateway-test-kafka-logs");
    // override to the local running zk server
    kafkaProps.setProperty("zookeeper.connect", "localhost:" + port);

    kafkaLogFile = new File(kafkaProps.getProperty("log.dir"));
    FileUtils.deleteDirectory(kafkaLogFile);

    KafkaConfig kafkaConfig = new KafkaConfig(kafkaProps);
    kafkaServer = new KafkaServerStartable(kafkaConfig);

    kafkaServer.startup();

    Configuration config = new PropertiesConfiguration();
    config.addProperty(
        "sensei.gateway.class", "com.senseidb.gateway.kafka.KafkaDataProviderBuilder");
    config.addProperty("sensei.gateway.kafka.group.id", "1");
    config.addProperty("sensei.gateway.kafka.zookeeper.connect", "localhost:" + port);
    config.addProperty("sensei.gateway.kafka.auto.offset.reset", "smallest");
    config.addProperty("sensei.gateway.kafka.topic", "test");
    config.addProperty("sensei.gateway.provider.batchSize", "1");
    pluginRegistry = SenseiPluginRegistry.build(config);
    pluginRegistry.start();

    kafkaGateway = pluginRegistry.getBeanByFullPrefix("sensei.gateway", SenseiGateway.class);
    kafkaGateway.start();

    config = new PropertiesConfiguration();
    config.addProperty("sensei.gateway.class", "com.senseidb.gateway.kafka.SimpleKafkaGateway");
    config.addProperty("sensei.gateway.kafka.host", "localhost");
    config.addProperty("sensei.gateway.kafka.port", "9092");
    config.addProperty("sensei.gateway.kafka.topic", "test");
    config.addProperty("sensei.gateway.kafka.timeout", "3000");
    config.addProperty("sensei.gateway.provider.batchSize", "1");
    pluginRegistry2 = SenseiPluginRegistry.build(config);
    pluginRegistry2.start();

    simpleKafkaGateway = pluginRegistry2.getBeanByFullPrefix("sensei.gateway", SenseiGateway.class);
    simpleKafkaGateway.start();

    Properties props = new Properties();
    props.put("metadata.broker.list", "localhost:9092");
    props.put("serializer.class", "kafka.serializer.StringEncoder");

    ProducerConfig producerConfig = new ProducerConfig(props);
    Producer<String, String> kafkaProducer = new Producer<String, String>(producerConfig);

    for (JSONObject jsonObj : BaseGatewayTestUtil.dataList) {
      KeyedMessage<String, String> data =
          new KeyedMessage<String, String>("test", jsonObj.toString());
      kafkaProducer.send(data);
    }
  }