@Override
  public void setup() {
    LOGGER.setLevel(Level.DEBUG);
    LOGGER.debug("HBASE TEST SETUP!");

    if (!TestUtils.isSet(zookeeper)) {
      zookeeper = System.getProperty("zookeeperUrl");

      if (!TestUtils.isSet(zookeeper)) {

        PropertyParser propertyParser = null;

        try {
          propertyParser = new PropertyParser(HBASE_PROPS_FILE);
          propertyParser.parsePropsFile();
        } catch (final IOException e) {
          LOGGER.error("Unable to load property file: {}" + HBASE_PROPS_FILE);
        }

        if (System.getProperty("os.name").startsWith("Windows")) {
          System.setProperty("HADOOP_HOME", System.getenv().get("HADOOP_HOME"));
        }

        try {
          zookeeperLocalCluster =
              new ZookeeperLocalCluster.Builder()
                  .setPort(
                      Integer.parseInt(propertyParser.getProperty(ConfigVars.ZOOKEEPER_PORT_KEY)))
                  .setTempDir(propertyParser.getProperty(ConfigVars.ZOOKEEPER_TEMP_DIR_KEY))
                  .setZookeeperConnectionString(
                      propertyParser.getProperty(ConfigVars.ZOOKEEPER_CONNECTION_STRING_KEY))
                  .build();
          zookeeperLocalCluster.start();
        } catch (final Exception e) {
          LOGGER.error("Exception starting zookeeperLocalCluster: " + e);
          e.printStackTrace();
          Assert.fail();
        }

        zookeeper = zookeeperLocalCluster.getZookeeperConnectionString();

        LOGGER.debug("Using local zookeeper URL: " + zookeeper);

        try {
          Configuration conf = new Configuration();
          conf.set("hbase.online.schema.update.enable", "true");
          hbaseLocalCluster =
              new HbaseLocalCluster.Builder()
                  .setHbaseMasterPort(
                      Integer.parseInt(
                          propertyParser.getProperty(ConfigVars.HBASE_MASTER_PORT_KEY)))
                  .setHbaseMasterInfoPort(
                      Integer.parseInt(
                          propertyParser.getProperty(ConfigVars.HBASE_MASTER_INFO_PORT_KEY)))
                  .setNumRegionServers(
                      Integer.parseInt(
                          propertyParser.getProperty(ConfigVars.HBASE_NUM_REGION_SERVERS_KEY)))
                  .setHbaseRootDir(propertyParser.getProperty(ConfigVars.HBASE_ROOT_DIR_KEY))
                  .setZookeeperPort(
                      Integer.parseInt(propertyParser.getProperty(ConfigVars.ZOOKEEPER_PORT_KEY)))
                  .setZookeeperConnectionString(
                      propertyParser.getProperty(ConfigVars.ZOOKEEPER_CONNECTION_STRING_KEY))
                  .setZookeeperZnodeParent(
                      propertyParser.getProperty(ConfigVars.HBASE_ZNODE_PARENT_KEY))
                  .setHbaseWalReplicationEnabled(
                      Boolean.parseBoolean(
                          propertyParser.getProperty(ConfigVars.HBASE_WAL_REPLICATION_ENABLED_KEY)))
                  .setHbaseConfiguration(conf)
                  .build();
          hbaseLocalCluster.start();
        } catch (final Exception e) {
          LOGGER.error("Exception starting hbaseLocalCluster: " + e);
          e.printStackTrace();
          Assert.fail();
        }
      } else {
        LOGGER.debug("Using system zookeeper URL: " + zookeeper);
      }
    } else {
      LOGGER.debug("Using system zookeeper URL: " + zookeeper);
    }
  }
Exemplo n.º 2
0
  public static void main(String args[]) {

    final ZookeeperLocalCluster zookeeperLocalCluster =
        new ZookeeperLocalCluster.Builder()
            .setPort(Integer.parseInt(propertyParser.getProperty(ConfigVars.ZOOKEEPER_PORT_KEY)))
            .setTempDir(propertyParser.getProperty(ConfigVars.ZOOKEEPER_TEMP_DIR_KEY))
            .setZookeeperConnectionString(
                propertyParser.getProperty(ConfigVars.ZOOKEEPER_CONNECTION_STRING_KEY))
            .build();

    kafkaLocalBroker =
        new KafkaLocalBroker.Builder()
            .setKafkaHostname(propertyParser.getProperty(ConfigVars.KAFKA_HOSTNAME_KEY))
            .setKafkaPort(Integer.parseInt(propertyParser.getProperty(ConfigVars.KAFKA_PORT_KEY)))
            .setKafkaBrokerId(
                Integer.parseInt(propertyParser.getProperty(ConfigVars.KAFKA_TEST_BROKER_ID_KEY)))
            .setKafkaProperties(new Properties())
            .setKafkaTempDir(propertyParser.getProperty(ConfigVars.KAFKA_TEST_TEMP_DIR_KEY))
            .setZookeeperConnectionString(
                propertyParser.getProperty(ConfigVars.ZOOKEEPER_CONNECTION_STRING_KEY))
            .build();

    final StormLocalCluster stormLocalCluster =
        new StormLocalCluster.Builder()
            .setZookeeperHost(propertyParser.getProperty(ConfigVars.ZOOKEEPER_HOST_KEY))
            .setZookeeperPort(
                Long.parseLong(propertyParser.getProperty(ConfigVars.ZOOKEEPER_PORT_KEY)))
            .setEnableDebug(
                Boolean.parseBoolean(propertyParser.getProperty(ConfigVars.STORM_ENABLE_DEBUG_KEY)))
            .setNumWorkers(
                Integer.parseInt(propertyParser.getProperty(ConfigVars.STORM_NUM_WORKERS_KEY)))
            .setEnableDebug(true)
            .setStormConfig(new Config())
            .build();

    try {
      zookeeperLocalCluster.start();
      kafkaLocalBroker.start();
      stormLocalCluster.start();
    } catch (Exception e) {
      LOG.error("Couldn't start the services: " + e.getMessage());
      e.printStackTrace();
    }

    SensorEventsParam sensorEventsParam = new SensorEventsParam();
    sensorEventsParam.setEventEmitterClassName(
        "com.hortonworks.simulator.impl.domain.transport.Truck");
    sensorEventsParam.setEventCollectorClassName(
        "com.hortonworks.solution.KafkaSensorEventCollector");
    sensorEventsParam.setNumberOfEvents(200);
    sensorEventsParam.setDelayBetweenEvents(1000);
    sensorEventsParam.setRouteDirectory(
        Launcher.class.getResource("/" + "routes/midwest").getPath());
    sensorEventsParam.setTruckSymbolSize(10000);
    SensorEventsGenerator sensorEventsGenerator = new SensorEventsGenerator();
    sensorEventsGenerator.generateTruckEventsStream(sensorEventsParam);

    BrokerHosts hosts = new ZkHosts(zookeeperLocalCluster.getZookeeperConnectionString());
    String topic = "truck_events";
    String zkRoot = "/trucks";
    String consumerGroupId = "group1";
    SpoutConfig spoutConfig = new SpoutConfig(hosts, topic, zkRoot, consumerGroupId);
    spoutConfig.scheme = new SchemeAsMultiScheme(new TruckScheme2());
    KafkaSpout kafkaSpout = new KafkaSpout(spoutConfig);
    int spoutCount = 10;
    int boltCount = 1;

    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("kafkaSpout", kafkaSpout, spoutCount);

    StormTopology topology = builder.createTopology();
    stormLocalCluster.submitTopology(
        "part2: Truck events to Kafka", stormLocalCluster.getStormConf(), topology);

    Runtime.getRuntime()
        .addShutdownHook(
            new Thread() {
              @Override
              public void run() {
                try {
                  stormLocalCluster.stop(DO_CLEAN_UP);
                  kafkaLocalBroker.stop(DO_CLEAN_UP);
                  zookeeperLocalCluster.stop(DO_CLEAN_UP);
                } catch (Exception e) {
                  LOG.error("Couldn't shutdown the services: " + e.getLocalizedMessage());
                  e.printStackTrace();
                }
              }
            });

    while (true) {
      // run until ctrl-c'd or stopped from IDE
    }
  }