Exemplo n.º 1
0
 public void shutdown() {
   try {
     logger.info("Producer thread " + threadName + " shutting down");
     producerDataChannel.sendRequest(shutdownMessage);
   } catch (InterruptedException ie) {
     logger.warn("Interrupt during shutdown of ProducerThread", ie);
   }
 }
Exemplo n.º 2
0
 public void awaitShutdown() {
   try {
     shutdownComplete.await();
     producer.close();
     logger.info("Producer thread " + threadName + " shutdown complete");
   } catch (InterruptedException ie) {
     logger.warn("Interrupt during shutdown of ProducerThread", ie);
   }
 }
Exemplo n.º 3
0
 public void shutdown() {
   logger.info("Migration thread " + threadName + " shutting down");
   isRunning.set(false);
   interrupt();
   try {
     shutdownComplete.await();
   } catch (InterruptedException ie) {
     logger.warn("Interrupt during shutdown of MigrationThread", ie);
   }
   logger.info("Migration thread " + threadName + " shutdown complete");
 }
Exemplo n.º 4
0
  public static void main(String[] args) throws InterruptedException, IOException {
    OptionParser parser = new OptionParser();
    ArgumentAcceptingOptionSpec<String> consumerConfigOpt =
        parser
            .accepts(
                "consumer.config",
                "Kafka 0.7 consumer config to consume from the source 0.7 cluster. "
                    + "You man specify multiple of these.")
            .withRequiredArg()
            .describedAs("config file")
            .ofType(String.class);

    ArgumentAcceptingOptionSpec<String> producerConfigOpt =
        parser
            .accepts("producer.config", "Producer config.")
            .withRequiredArg()
            .describedAs("config file")
            .ofType(String.class);

    ArgumentAcceptingOptionSpec<Integer> numProducersOpt =
        parser
            .accepts("num.producers", "Number of producer instances")
            .withRequiredArg()
            .describedAs("Number of producers")
            .ofType(Integer.class)
            .defaultsTo(1);

    ArgumentAcceptingOptionSpec<String> zkClient01JarOpt =
        parser
            .accepts("zkclient.01.jar", "zkClient 0.1 jar file")
            .withRequiredArg()
            .describedAs("zkClient 0.1 jar file required by Kafka 0.7")
            .ofType(String.class);

    ArgumentAcceptingOptionSpec<String> kafka07JarOpt =
        parser
            .accepts("kafka.07.jar", "Kafka 0.7 jar file")
            .withRequiredArg()
            .describedAs("kafka 0.7 jar")
            .ofType(String.class);

    ArgumentAcceptingOptionSpec<Integer> numStreamsOpt =
        parser
            .accepts("num.streams", "Number of consumer streams")
            .withRequiredArg()
            .describedAs("Number of consumer threads")
            .ofType(Integer.class)
            .defaultsTo(1);

    ArgumentAcceptingOptionSpec<String> whitelistOpt =
        parser
            .accepts("whitelist", "Whitelist of topics to migrate from the 0.7 cluster")
            .withRequiredArg()
            .describedAs("Java regex (String)")
            .ofType(String.class);

    ArgumentAcceptingOptionSpec<String> blacklistOpt =
        parser
            .accepts("blacklist", "Blacklist of topics to migrate from the 0.7 cluster")
            .withRequiredArg()
            .describedAs("Java regex (String)")
            .ofType(String.class);

    ArgumentAcceptingOptionSpec<Integer> queueSizeOpt =
        parser
            .accepts(
                "queue.size",
                "Number of messages that are buffered between the 0.7 consumer and 0.8 producer")
            .withRequiredArg()
            .describedAs("Queue size in terms of number of messages")
            .ofType(Integer.class)
            .defaultsTo(10000);

    OptionSpecBuilder helpOpt = parser.accepts("help", "Print this message.");

    OptionSet options = parser.parse(args);

    if (options.has(helpOpt)) {
      parser.printHelpOn(System.out);
      System.exit(0);
    }

    checkRequiredArgs(
        parser,
        options,
        new OptionSpec[] {consumerConfigOpt, producerConfigOpt, zkClient01JarOpt, kafka07JarOpt});
    int whiteListCount = options.has(whitelistOpt) ? 1 : 0;
    int blackListCount = options.has(blacklistOpt) ? 1 : 0;
    if (whiteListCount + blackListCount != 1) {
      System.err.println("Exactly one of whitelist or blacklist is required.");
      System.exit(1);
    }

    String kafkaJarFile_07 = options.valueOf(kafka07JarOpt);
    String zkClientJarFile = options.valueOf(zkClient01JarOpt);
    String consumerConfigFile_07 = options.valueOf(consumerConfigOpt);
    int numConsumers = options.valueOf(numStreamsOpt);
    String producerConfigFile_08 = options.valueOf(producerConfigOpt);
    int numProducers = options.valueOf(numProducersOpt);
    final List<MigrationThread> migrationThreads = new ArrayList<MigrationThread>(numConsumers);
    final List<ProducerThread> producerThreads = new ArrayList<ProducerThread>(numProducers);

    try {
      File kafkaJar_07 = new File(kafkaJarFile_07);
      File zkClientJar = new File(zkClientJarFile);
      ParentLastURLClassLoader c1 =
          new ParentLastURLClassLoader(
              new URL[] {kafkaJar_07.toURI().toURL(), zkClientJar.toURI().toURL()});

      /** Construct the 07 consumer config * */
      ConsumerConfig_07 = c1.loadClass(KAFKA_07_CONSUMER_CONFIG_CLASS_NAME);
      KafkaStaticConsumer_07 = c1.loadClass(KAFKA_07_STATIC_CONSUMER_CLASS_NAME);
      ConsumerConnector_07 = c1.loadClass(KAFKA_07_CONSUMER_CONNECTOR_CLASS_NAME);
      KafkaStream_07 = c1.loadClass(KAFKA_07_CONSUMER_STREAM_CLASS_NAME);
      TopicFilter_07 = c1.loadClass(KAFKA_07_TOPIC_FILTER_CLASS_NAME);
      WhiteList_07 = c1.loadClass(KAFKA_07_WHITE_LIST_CLASS_NAME);
      BlackList_07 = c1.loadClass(KAFKA_07_BLACK_LIST_CLASS_NAME);
      KafkaMessageClass_07 = c1.loadClass(KAFKA_07_MESSAGE_CLASS_NAME);
      KafkaConsumerIteratorClass_07 = c1.loadClass(KAFKA_07_CONSUMER_ITERATOR_CLASS_NAME);
      KafkaMessageAndMetatDataClass_07 = c1.loadClass(KAFKA_07_MESSAGE_AND_METADATA_CLASS_NAME);

      Constructor ConsumerConfigConstructor_07 = ConsumerConfig_07.getConstructor(Properties.class);
      Properties kafkaConsumerProperties_07 = new Properties();
      kafkaConsumerProperties_07.load(new FileInputStream(consumerConfigFile_07));
      /**
       * Disable shallow iteration because the message format is different between 07 and 08, we
       * have to get each individual message *
       */
      if (kafkaConsumerProperties_07.getProperty("shallow.iterator.enable", "").equals("true")) {
        log.warn("Shallow iterator should not be used in the migration tool");
        kafkaConsumerProperties_07.setProperty("shallow.iterator.enable", "false");
      }
      Object consumerConfig_07 =
          ConsumerConfigConstructor_07.newInstance(kafkaConsumerProperties_07);

      /** Construct the 07 consumer connector * */
      Method ConsumerConnectorCreationMethod_07 =
          KafkaStaticConsumer_07.getMethod("createJavaConsumerConnector", ConsumerConfig_07);
      final Object consumerConnector_07 =
          ConsumerConnectorCreationMethod_07.invoke(null, consumerConfig_07);
      Method ConsumerConnectorCreateMessageStreamsMethod_07 =
          ConsumerConnector_07.getMethod("createMessageStreamsByFilter", TopicFilter_07, int.class);
      final Method ConsumerConnectorShutdownMethod_07 = ConsumerConnector_07.getMethod("shutdown");
      Constructor WhiteListConstructor_07 = WhiteList_07.getConstructor(String.class);
      Constructor BlackListConstructor_07 = BlackList_07.getConstructor(String.class);
      Object filterSpec = null;
      if (options.has(whitelistOpt))
        filterSpec = WhiteListConstructor_07.newInstance(options.valueOf(whitelistOpt));
      else filterSpec = BlackListConstructor_07.newInstance(options.valueOf(blacklistOpt));

      Object retKafkaStreams =
          ConsumerConnectorCreateMessageStreamsMethod_07.invoke(
              consumerConnector_07, filterSpec, numConsumers);

      Properties kafkaProducerProperties_08 = new Properties();
      kafkaProducerProperties_08.load(new FileInputStream(producerConfigFile_08));
      kafkaProducerProperties_08.setProperty("serializer.class", "kafka.serializer.DefaultEncoder");
      // create a producer channel instead
      int queueSize = options.valueOf(queueSizeOpt);
      ProducerDataChannel<KeyedMessage<byte[], byte[]>> producerDataChannel =
          new ProducerDataChannel<KeyedMessage<byte[], byte[]>>(queueSize);
      int threadId = 0;

      Runtime.getRuntime()
          .addShutdownHook(
              new Thread() {
                @Override
                public void run() {
                  try {
                    ConsumerConnectorShutdownMethod_07.invoke(consumerConnector_07);
                  } catch (Exception e) {
                    log.error("Error while shutting down Kafka consumer", e);
                  }
                  for (MigrationThread migrationThread : migrationThreads) {
                    migrationThread.shutdown();
                  }
                  for (ProducerThread producerThread : producerThreads) {
                    producerThread.shutdown();
                  }
                  for (ProducerThread producerThread : producerThreads) {
                    producerThread.awaitShutdown();
                  }
                  log.info("Kafka migration tool shutdown successfully");
                }
              });

      // start consumer threads
      for (Object stream : (List) retKafkaStreams) {
        MigrationThread thread = new MigrationThread(stream, producerDataChannel, threadId);
        threadId++;
        thread.start();
        migrationThreads.add(thread);
      }

      String clientId = kafkaProducerProperties_08.getProperty("client.id");
      // start producer threads
      for (int i = 0; i < numProducers; i++) {
        kafkaProducerProperties_08.put("client.id", clientId + "-" + i);
        ProducerConfig producerConfig_08 = new ProducerConfig(kafkaProducerProperties_08);
        Producer producer = new Producer(producerConfig_08);
        ProducerThread producerThread = new ProducerThread(producerDataChannel, producer, i);
        producerThread.start();
        producerThreads.add(producerThread);
      }
    } catch (Throwable e) {
      System.out.println("Kafka migration tool failed due to: " + Utils.stackTrace(e));
      log.error("Kafka migration tool failed: ", e);
    }
  }