public SenseiServerBuilder(File confDir, Map<String, Object> properties) throws Exception {
if (properties != null) {
_senseiConfFile = null;
_senseiConf = new MapConfiguration(properties);
((MapConfiguration) _senseiConf).setDelimiterParsingDisabled(true);
} else {
_senseiConfFile = new File(confDir, SENSEI_PROPERTIES);
if (!_senseiConfFile.exists()) {
throw new ConfigurationException(
"configuration file: " + _senseiConfFile.getAbsolutePath() + " does not exist.");
}
_senseiConf = new PropertiesConfiguration();
((PropertiesConfiguration) _senseiConf).setDelimiterParsingDisabled(true);
((PropertiesConfiguration) _senseiConf).load(_senseiConfFile);
}
pluginRegistry = SenseiPluginRegistry.build(_senseiConf);
pluginRegistry.start();
processRelevanceFunctionPlugins(pluginRegistry);
processRelevanceExternalObjectPlugins(pluginRegistry);
_gateway = pluginRegistry.getBeanByFullPrefix(SENSEI_GATEWAY, SenseiGateway.class);
_schemaDoc = loadSchema(confDir);
_senseiSchema = SenseiSchema.build(_schemaDoc);
}
public SenseiServerBuilder(Resource confDir, Map<String, Object> properties) throws Exception {
_senseiConfFile = null;
_senseiConf = new MapConfiguration(properties);
((MapConfiguration) _senseiConf).setDelimiterParsingDisabled(true);
pluginRegistry = SenseiPluginRegistry.build(_senseiConf);
pluginRegistry.start();
processRelevanceFunctionPlugins(pluginRegistry);
_gateway = pluginRegistry.getBeanByFullPrefix(SENSEI_GATEWAY, SenseiGateway.class);
_schemaDoc = loadSchema(confDir);
_senseiSchema = SenseiSchema.build(_schemaDoc);
}
public PersistentCacheManager createpersistentCacheManager() {
PersistentCacheManager cacheManager = new PersistentCacheManager();
Map map = new HashMap();
map.put("sensei.index.manager.default.maxpartition.id", 0);
cacheManager.init(
indexDir.getAbsolutePath(),
1,
null,
ZoieConfig.DEFAULT_VERSION_COMPARATOR,
SenseiPluginRegistry.build(new MapConfiguration(map)),
new ShardingStrategy() {
@Override
public int caculateShard(int maxShardId, JSONObject dataObj) throws JSONException {
return 0;
}
});
cache = new PersistentCache(indexDir, ZoieConfig.DEFAULT_VERSION_COMPARATOR);
cacheManager.getPersistentCaches().put(0, cache);
return cacheManager;
}
@BeforeClass
public static void init() throws Exception {
final ShutdownRegistryImpl shutdownRegistry = new ShutdownRegistryImpl();
try {
zkTestServer =
new ZooKeeperTestServer(0, shutdownRegistry, ZooKeeperTestServer.DEFAULT_SESSION_TIMEOUT);
port = zkTestServer.startNetwork();
} catch (Exception e) {
throw new RuntimeException(e);
}
Properties kafkaProps = new Properties();
kafkaProps.setProperty("num.partitions", "1");
kafkaProps.setProperty("port", "9092");
kafkaProps.setProperty("broker.id", "0");
kafkaProps.setProperty("log.dir", "/tmp/sensei-gateway-test-kafka-logs");
// override to the local running zk server
kafkaProps.setProperty("zookeeper.connect", "localhost:" + port);
kafkaLogFile = new File(kafkaProps.getProperty("log.dir"));
FileUtils.deleteDirectory(kafkaLogFile);
KafkaConfig kafkaConfig = new KafkaConfig(kafkaProps);
kafkaServer = new KafkaServerStartable(kafkaConfig);
kafkaServer.startup();
Configuration config = new PropertiesConfiguration();
config.addProperty(
"sensei.gateway.class", "com.senseidb.gateway.kafka.KafkaDataProviderBuilder");
config.addProperty("sensei.gateway.kafka.group.id", "1");
config.addProperty("sensei.gateway.kafka.zookeeper.connect", "localhost:" + port);
config.addProperty("sensei.gateway.kafka.auto.offset.reset", "smallest");
config.addProperty("sensei.gateway.kafka.topic", "test");
config.addProperty("sensei.gateway.provider.batchSize", "1");
pluginRegistry = SenseiPluginRegistry.build(config);
pluginRegistry.start();
kafkaGateway = pluginRegistry.getBeanByFullPrefix("sensei.gateway", SenseiGateway.class);
kafkaGateway.start();
config = new PropertiesConfiguration();
config.addProperty("sensei.gateway.class", "com.senseidb.gateway.kafka.SimpleKafkaGateway");
config.addProperty("sensei.gateway.kafka.host", "localhost");
config.addProperty("sensei.gateway.kafka.port", "9092");
config.addProperty("sensei.gateway.kafka.topic", "test");
config.addProperty("sensei.gateway.kafka.timeout", "3000");
config.addProperty("sensei.gateway.provider.batchSize", "1");
pluginRegistry2 = SenseiPluginRegistry.build(config);
pluginRegistry2.start();
simpleKafkaGateway = pluginRegistry2.getBeanByFullPrefix("sensei.gateway", SenseiGateway.class);
simpleKafkaGateway.start();
Properties props = new Properties();
props.put("metadata.broker.list", "localhost:9092");
props.put("serializer.class", "kafka.serializer.StringEncoder");
ProducerConfig producerConfig = new ProducerConfig(props);
Producer<String, String> kafkaProducer = new Producer<String, String>(producerConfig);
for (JSONObject jsonObj : BaseGatewayTestUtil.dataList) {
KeyedMessage<String, String> data =
new KeyedMessage<String, String>("test", jsonObj.toString());
kafkaProducer.send(data);
}
}