@Override
 public <T> StreamConsumer<T> chunkWriter(
     String aggregationId,
     List<String> dimensions,
     List<String> measures,
     Class<T> recordClass,
     long id) {
   chunkTypes.put(id, recordClass);
   ArrayList<Object> list = new ArrayList<>();
   lists.put(id, list);
   return (StreamConsumer<T>) StreamConsumers.toList(eventloop, list);
 }
  @Test
  public void test() throws Exception {
    final List<Integer> source = Lists.newArrayList();
    for (int i = 0; i < 5; i++) {
      source.add(i);
    }

    final NioEventloop eventloop = new NioEventloop();

    final StreamConsumers.ToList<Integer> consumerToList = StreamConsumers.toList(eventloop);

    SimpleNioServer server =
        new SimpleNioServer(eventloop) {
          @Override
          protected SocketConnection createConnection(SocketChannel socketChannel) {
            return new TcpStreamSocketConnection(eventloop, socketChannel) {
              @Override
              protected void wire(
                  StreamProducer<ByteBuf> socketReader, StreamConsumer<ByteBuf> socketWriter) {
                StreamBinaryDeserializer<Integer> streamDeserializer =
                    new StreamBinaryDeserializer<>(eventloop, intSerializer(), 10);
                streamDeserializer.streamTo(consumerToList);
                socketReader.streamTo(streamDeserializer);
              }
            };
          }
        };
    server.setListenAddress(address).acceptOnce();
    server.listen();

    final StreamBinarySerializer<Integer> streamSerializer =
        new StreamBinarySerializer<>(eventloop, intSerializer(), 1, 10, 0, false);
    reconnect(
        eventloop,
        address,
        defaultSocketSettings(),
        3,
        100L,
        new ConnectCallback() {
          @Override
          public void onConnect(SocketChannel socketChannel) {
            SocketConnection connection =
                new TcpStreamSocketConnection(eventloop, socketChannel) {
                  @Override
                  protected void wire(
                      StreamProducer<ByteBuf> socketReader, StreamConsumer<ByteBuf> socketWriter) {
                    streamSerializer.streamTo(socketWriter);
                    StreamProducers.ofIterable(eventloop, source).streamTo(streamSerializer);
                  }
                };
            connection.register();
          }

          @Override
          public void onException(Exception exception) {
            fail();
          }
        });

    eventloop.run();

    assertEquals(source, consumerToList.getList());

    assertEquals(getPoolItemsString(), ByteBufPool.getCreatedItems(), ByteBufPool.getPoolItems());
  }
Пример #3
0
  @Test
  public void testBinaryMessagingUploadAck() throws Exception {
    final List<Long> source = Lists.newArrayList();
    for (long i = 0; i < 100; i++) {
      source.add(i);
    }

    final AtomicBoolean ack = new AtomicBoolean(false);

    final NioEventloop eventloop = new NioEventloop();

    final StreamConsumers.ToList<Long> consumerToList = StreamConsumers.toList(eventloop);

    SimpleNioServer server =
        new SimpleNioServer(eventloop) {
          @Override
          protected SocketConnection createConnection(SocketChannel socketChannel) {
            return new StreamMessagingConnection<>(
                    eventloop,
                    socketChannel,
                    new StreamBinaryDeserializer<>(eventloop, stringSerializer(), 10),
                    new StreamBinarySerializer<>(eventloop, stringSerializer(), 2, 10, 0, false))
                .addHandler(
                    String.class,
                    new MessagingHandler<String, String>() {
                      @Override
                      public void onMessage(String item, final Messaging<String> messaging) {
                        assertEquals("start", item);
                        System.out.println("receive start");

                        StreamBinaryDeserializer<Long> streamDeserializer =
                            new StreamBinaryDeserializer<>(eventloop, longSerializer(), 10);
                        messaging.binarySocketReader().streamTo(streamDeserializer);
                        streamDeserializer.streamTo(consumerToList);

                        consumerToList.addCompletionCallback(
                            new CompletionCallback() {
                              @Override
                              public void onComplete() {
                                System.out.println("send ack");
                                messaging.sendMessage("ack");
                                messaging.shutdown();
                              }

                              @Override
                              public void onException(Exception exception) {
                                messaging.shutdown();
                              }
                            });
                      }
                    });
          }
        };
    server.setListenAddress(address).acceptOnce();
    server.listen();

    eventloop.connect(
        address,
        new SocketSettings(),
        new ConnectCallback() {
          @Override
          public void onConnect(SocketChannel socketChannel) {
            SocketConnection connection =
                new StreamMessagingConnection<>(
                        eventloop,
                        socketChannel,
                        new StreamBinaryDeserializer<>(eventloop, stringSerializer(), 10),
                        new StreamBinarySerializer<>(
                            eventloop, stringSerializer(), 2, 10, 0, false))
                    .addStarter(
                        new MessagingStarter<String>() {
                          @Override
                          public void onStart(Messaging<String> messaging) {
                            System.out.println("send start");
                            messaging.sendMessage("start");

                            StreamBinarySerializer<Long> streamSerializer =
                                new StreamBinarySerializer<>(
                                    eventloop, longSerializer(), 1, 10, 0, false);
                            StreamProducers.ofIterable(eventloop, source)
                                .streamTo(streamSerializer);
                            streamSerializer.streamTo(messaging.binarySocketWriter());
                          }
                        })
                    .addHandler(
                        String.class,
                        new MessagingHandler<String, String>() {
                          @Override
                          public void onMessage(String item, Messaging<String> messaging) {
                            ack.set(true);
                            assertEquals("ack", item);
                            System.out.println("receive ack");
                          }
                        });

            connection.register();
          }

          @Override
          public void onException(Exception e) {
            fail("Test Exception: " + e);
          }
        });

    eventloop.run();

    assertEquals(source, consumerToList.getList());
    assertTrue(ack.get());

    assertEquals(getPoolItemsString(), ByteBufPool.getCreatedItems(), ByteBufPool.getPoolItems());
  }
Пример #4
0
  @Test
  public void testGsonMessagingUpload() throws Exception {
    final List<Long> source = Lists.newArrayList();
    for (long i = 0; i < 100; i++) {
      source.add(i);
    }

    final NioEventloop eventloop = new NioEventloop();

    final StreamConsumers.ToList<Long> consumerToList = StreamConsumers.toList(eventloop);

    SimpleNioServer server =
        new SimpleNioServer(eventloop) {
          @Override
          protected SocketConnection createConnection(SocketChannel socketChannel) {
            return new StreamMessagingConnection<>(
                    eventloop,
                    socketChannel,
                    new StreamGsonDeserializer<>(eventloop, new Gson(), String.class, 10),
                    new StreamGsonSerializer<>(eventloop, new Gson(), String.class, 1, 50, 0))
                .addHandler(
                    String.class,
                    new MessagingHandler<String, String>() {
                      @Override
                      public void onMessage(String item, Messaging<String> messaging) {
                        assertEquals("start", item);

                        StreamBinaryDeserializer<Long> streamDeserializer =
                            new StreamBinaryDeserializer<>(eventloop, longSerializer(), 10);
                        messaging.binarySocketReader().streamTo(streamDeserializer);
                        streamDeserializer.streamTo(consumerToList);

                        messaging.shutdownWriter();
                      }
                    });
          }
        };
    server.setListenAddress(address).acceptOnce();
    server.listen();

    eventloop.connect(
        address,
        new SocketSettings(),
        new ConnectCallback() {
          @Override
          public void onConnect(SocketChannel socketChannel) {
            SocketConnection connection =
                new StreamMessagingConnection<>(
                        eventloop,
                        socketChannel,
                        new StreamGsonDeserializer<>(eventloop, new Gson(), String.class, 10),
                        new StreamGsonSerializer<>(eventloop, new Gson(), String.class, 1, 50, 0))
                    .addStarter(
                        new MessagingStarter<String>() {
                          @Override
                          public void onStart(Messaging<String> messaging) {
                            messaging.sendMessage("start");

                            StreamBinarySerializer<Long> streamSerializer =
                                new StreamBinarySerializer<>(
                                    eventloop, longSerializer(), 1, 10, 0, false);
                            StreamProducers.ofIterable(eventloop, source)
                                .streamTo(streamSerializer);
                            streamSerializer.streamTo(messaging.binarySocketWriter());

                            messaging.shutdownReader();
                          }
                        });
            connection.register();
          }

          @Override
          public void onException(Exception e) {
            fail("Test Exception: " + e);
          }
        });

    eventloop.run();

    assertEquals(source, consumerToList.getList());

    // FIXME: 1 ByteBuf Leak (ByteBuf(16) produced by StreamBinaryDeserializer)
    //		assertEquals(getPoolItemsString(), ByteBufPool.getCreatedItems(),
    // ByteBufPool.getPoolItems());
  }
Пример #5
0
  public static void main(String[] args) {
    StreamConsumers.ToList<TestAdvResult> consumerToList;
    int iterations = 30;
    root.setLevel(Level.ERROR);
    DefiningClassLoader classLoader = new DefiningClassLoader();
    ExecutorService executor = Executors.newCachedThreadPool();
    Path aggregationsDir = Paths.get("test/aggregations/");
    NioEventloop eventloop = new NioEventloop();

    for (int i = 0; i < iterations + 1; ++i) {
      if (i == iterations - 1) {
        Logger root = (Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME);
        root.setLevel(Level.INFO);
      }
      if (i == iterations) {
        classLoader = new DefiningClassLoader();
      }
      logger.info("Benchmark started.");

      logger.info("Initialization started.");
      deleteRecursivelyQuietly(aggregationsDir);
      CubeStructure structure = getStructure(classLoader);
      AggregationStorage aggregationStorage =
          new LocalFsAggregationStorage(eventloop, executor, structure, aggregationsDir);
      Cube cube =
          newCube(
              eventloop,
              classLoader,
              new LogToCubeMetadataStorageStub(),
              aggregationStorage,
              structure);
      cube.addAggregation(new Aggregation("pub", asList("pub"), asList("pubRequests")));
      cube.addAggregation(new Aggregation("adv", asList("adv"), asList("advRequests")));

      Path dir = Paths.get("test/logs/");
      deleteRecursivelyQuietly(dir);
      LogFileSystemImpl fileSystem = new LogFileSystemImpl(eventloop, executor, dir);
      SerializerFactory bufferSerializerFactory =
          SerializerFactory.createBufferSerializerFactory(classLoader, true, true);
      SerializerScanner registry = SerializerScanner.defaultScanner();
      SerializerGen serializerGen = registry.serializer(TypeToken.of(TestPubRequest.class));
      BufferSerializer<TestPubRequest> bufferSerializer =
          bufferSerializerFactory.createBufferSerializer(serializerGen);

      LogManager<TestPubRequest> logManager =
          new LogManagerImpl<>(eventloop, fileSystem, bufferSerializer);

      LogToCubeRunner<TestPubRequest> logToCubeRunner =
          new LogToCubeRunner<>(
              eventloop,
              cube,
              logManager,
              TestAggregatorSplitter.factory(),
              "testlog",
              asList("partitionA"),
              new LogToCubeMetadataStorageStub());
      logger.info("Initialization finished.");

      int numberOfTestRequests = 100_000;
      logger.info("Started generating {} random requests.", numberOfTestRequests);
      List<TestPubRequest> pubRequests = generatePubRequests(numberOfTestRequests);
      long advRequests = countAdvRequests(pubRequests);
      logger.info("Finished generating random requests. Adv requests generated: {}", advRequests);

      logger.info("Started streaming to LogManager.");
      new StreamProducers.OfIterator<>(eventloop, pubRequests.iterator())
          .streamTo(logManager.consumer("partitionA"));

      eventloop.run();
      logger.info("Finished streaming to LogManager.");

      CompletionCallbackObserver cb = new CompletionCallbackObserver();
      logger.info("Started processing log.");
      logToCubeRunner.processLog(cb);

      eventloop.run();
      logger.info("Finished processing log.");
      cb.check();

      consumerToList = StreamConsumers.toList(eventloop);
      cube.query(
              0,
              TestAdvResult.class,
              new CubeQuery()
                  .dimension("adv")
                  .measure("advRequests")
                  .between("adv", 2500, 7500)
                  .orderDesc("advRequests"))
          .streamTo(consumerToList);
      logger.info("Started executing query.");
      eventloop.run();
      logger.info("Finished executing query.");

      logger.info("Benchmark finished.");
    }

    executor.shutdownNow();
  }