示例#1
0
  @Override
  public void start() {
    logger.info("Starting {}...", this);

    Responder responder = new SpecificResponder(AvroSourceProtocol.class, this);
    if (maxThreads <= 0) {
      server = new NettyServer(responder, new InetSocketAddress(bindAddress, port));
    } else {
      server =
          new NettyServer(
              responder,
              new InetSocketAddress(bindAddress, port),
              new NioServerSocketChannelFactory(
                  Executors.newCachedThreadPool(), Executors.newFixedThreadPool(maxThreads)));
    }
    connectionCountUpdater = Executors.newSingleThreadScheduledExecutor();
    server.start();
    sourceCounter.start();
    super.start();
    final NettyServer srv = (NettyServer) server;
    connectionCountUpdater.scheduleWithFixedDelay(
        new Runnable() {

          @Override
          public void run() {
            sourceCounter.setOpenConnectionCount(Long.valueOf(srv.getNumActiveConnections()));
          }
        },
        0,
        60,
        TimeUnit.SECONDS);

    logger.info("Avro source {} started.", getName());
  }
示例#2
0
  @Override
  public void start() {
    logger.info("Starting thrift source");
    ExecutorService sourceService;
    ThreadFactory threadFactory =
        new ThreadFactoryBuilder().setNameFormat("Flume Thrift IPC Thread %d").build();
    if (maxThreads == 0) {
      sourceService = Executors.newCachedThreadPool(threadFactory);
    } else {
      sourceService = Executors.newFixedThreadPool(maxThreads, threadFactory);
    }
    try {
      serverTransport = new TNonblockingServerSocket(new InetSocketAddress(bindAddress, port));
    } catch (TTransportException e) {
      throw new FlumeException("Failed to start Thrift Source.", e);
    }

    THsHaServer.Args thhsArgs = new THsHaServer.Args(serverTransport);
    thhsArgs.processor(new ThriftSourceProtocol.Processor(new ThriftSourceHandler()));
    //	thhsArgs.transportFactory(new TFramedTransport.Factory());
    thhsArgs.protocolFactory(new TBinaryProtocol.Factory());
    thhsArgs.executorService(sourceService);
    server = new THsHaServer(thhsArgs); // 半同步半异步的服务模型

    servingExecutor =
        Executors.newSingleThreadExecutor(
            new ThreadFactoryBuilder().setNameFormat("Flume Thrift Source I/O Boss").build());
    /** Start serving. */
    servingExecutor.submit(
        new Runnable() {
          @Override
          public void run() {
            server.serve();
          }
        });

    long timeAfterStart = System.currentTimeMillis();
    while (!server.isServing()) {
      try {
        if (System.currentTimeMillis() - timeAfterStart >= 10000) {
          throw new FlumeException("Thrift server failed to start!");
        }
        TimeUnit.MILLISECONDS.sleep(1000);
      } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        throw new FlumeException("Interrupted while waiting for Thrift server" + " to start.", e);
      }
    }
    sourceCounter.start();
    logger.info("Started Thrift source.");
    super.start();
  }
  @Override
  public synchronized void start() {
    logger.info("SpoolDirectorySource source starting with directory: {}", spoolDirectory);

    executor = Executors.newSingleThreadScheduledExecutor();

    File directory = new File(spoolDirectory);
    try {
      reader =
          new ReliableSpoolingZipFileEventReader.Builder()
              .spoolDirectory(directory)
              .completedSuffix(completedSuffix)
              .ignorePattern(ignorePattern)
              .trackerDirPath(trackerDirPath)
              .annotateFileName(fileHeader)
              .fileNameHeader(fileHeaderKey)
              .annotateBaseName(basenameHeader)
              .baseNameHeader(basenameHeaderKey)
              .deserializerType(deserializerType)
              .deserializerContext(deserializerContext)
              .deletePolicy(deletePolicy)
              .inputCharset(inputCharset)
              .decodeErrorPolicy(decodeErrorPolicy)
              .consumeOrder(consumeOrder)
              .build();
    } catch (IOException ioe) {
      throw new FlumeException("Error instantiating spooling event parser", ioe);
    }

    Runnable runner = new SpoolDirectoryRunnable(reader, sourceCounter);
    executor.scheduleWithFixedDelay(runner, 0, POLL_DELAY_MS, TimeUnit.MILLISECONDS);

    super.start();
    logger.debug("SpoolDirectoryZipSource source started");
    sourceCounter.start();
  }