@Override
public <T> StreamConsumer<T> chunkWriter(
String aggregationId,
List<String> dimensions,
List<String> measures,
Class<T> recordClass,
long id) {
chunkTypes.put(id, recordClass);
ArrayList<Object> list = new ArrayList<>();
lists.put(id, list);
return (StreamConsumer<T>) StreamConsumers.toList(eventloop, list);
}
@Test
public void test() throws Exception {
final List<Integer> source = Lists.newArrayList();
for (int i = 0; i < 5; i++) {
source.add(i);
}
final NioEventloop eventloop = new NioEventloop();
final StreamConsumers.ToList<Integer> consumerToList = StreamConsumers.toList(eventloop);
SimpleNioServer server =
new SimpleNioServer(eventloop) {
@Override
protected SocketConnection createConnection(SocketChannel socketChannel) {
return new TcpStreamSocketConnection(eventloop, socketChannel) {
@Override
protected void wire(
StreamProducer<ByteBuf> socketReader, StreamConsumer<ByteBuf> socketWriter) {
StreamBinaryDeserializer<Integer> streamDeserializer =
new StreamBinaryDeserializer<>(eventloop, intSerializer(), 10);
streamDeserializer.streamTo(consumerToList);
socketReader.streamTo(streamDeserializer);
}
};
}
};
server.setListenAddress(address).acceptOnce();
server.listen();
final StreamBinarySerializer<Integer> streamSerializer =
new StreamBinarySerializer<>(eventloop, intSerializer(), 1, 10, 0, false);
reconnect(
eventloop,
address,
defaultSocketSettings(),
3,
100L,
new ConnectCallback() {
@Override
public void onConnect(SocketChannel socketChannel) {
SocketConnection connection =
new TcpStreamSocketConnection(eventloop, socketChannel) {
@Override
protected void wire(
StreamProducer<ByteBuf> socketReader, StreamConsumer<ByteBuf> socketWriter) {
streamSerializer.streamTo(socketWriter);
StreamProducers.ofIterable(eventloop, source).streamTo(streamSerializer);
}
};
connection.register();
}
@Override
public void onException(Exception exception) {
fail();
}
});
eventloop.run();
assertEquals(source, consumerToList.getList());
assertEquals(getPoolItemsString(), ByteBufPool.getCreatedItems(), ByteBufPool.getPoolItems());
}
@Test
public void testBinaryMessagingUploadAck() throws Exception {
final List<Long> source = Lists.newArrayList();
for (long i = 0; i < 100; i++) {
source.add(i);
}
final AtomicBoolean ack = new AtomicBoolean(false);
final NioEventloop eventloop = new NioEventloop();
final StreamConsumers.ToList<Long> consumerToList = StreamConsumers.toList(eventloop);
SimpleNioServer server =
new SimpleNioServer(eventloop) {
@Override
protected SocketConnection createConnection(SocketChannel socketChannel) {
return new StreamMessagingConnection<>(
eventloop,
socketChannel,
new StreamBinaryDeserializer<>(eventloop, stringSerializer(), 10),
new StreamBinarySerializer<>(eventloop, stringSerializer(), 2, 10, 0, false))
.addHandler(
String.class,
new MessagingHandler<String, String>() {
@Override
public void onMessage(String item, final Messaging<String> messaging) {
assertEquals("start", item);
System.out.println("receive start");
StreamBinaryDeserializer<Long> streamDeserializer =
new StreamBinaryDeserializer<>(eventloop, longSerializer(), 10);
messaging.binarySocketReader().streamTo(streamDeserializer);
streamDeserializer.streamTo(consumerToList);
consumerToList.addCompletionCallback(
new CompletionCallback() {
@Override
public void onComplete() {
System.out.println("send ack");
messaging.sendMessage("ack");
messaging.shutdown();
}
@Override
public void onException(Exception exception) {
messaging.shutdown();
}
});
}
});
}
};
server.setListenAddress(address).acceptOnce();
server.listen();
eventloop.connect(
address,
new SocketSettings(),
new ConnectCallback() {
@Override
public void onConnect(SocketChannel socketChannel) {
SocketConnection connection =
new StreamMessagingConnection<>(
eventloop,
socketChannel,
new StreamBinaryDeserializer<>(eventloop, stringSerializer(), 10),
new StreamBinarySerializer<>(
eventloop, stringSerializer(), 2, 10, 0, false))
.addStarter(
new MessagingStarter<String>() {
@Override
public void onStart(Messaging<String> messaging) {
System.out.println("send start");
messaging.sendMessage("start");
StreamBinarySerializer<Long> streamSerializer =
new StreamBinarySerializer<>(
eventloop, longSerializer(), 1, 10, 0, false);
StreamProducers.ofIterable(eventloop, source)
.streamTo(streamSerializer);
streamSerializer.streamTo(messaging.binarySocketWriter());
}
})
.addHandler(
String.class,
new MessagingHandler<String, String>() {
@Override
public void onMessage(String item, Messaging<String> messaging) {
ack.set(true);
assertEquals("ack", item);
System.out.println("receive ack");
}
});
connection.register();
}
@Override
public void onException(Exception e) {
fail("Test Exception: " + e);
}
});
eventloop.run();
assertEquals(source, consumerToList.getList());
assertTrue(ack.get());
assertEquals(getPoolItemsString(), ByteBufPool.getCreatedItems(), ByteBufPool.getPoolItems());
}
@Test
public void testGsonMessagingUpload() throws Exception {
final List<Long> source = Lists.newArrayList();
for (long i = 0; i < 100; i++) {
source.add(i);
}
final NioEventloop eventloop = new NioEventloop();
final StreamConsumers.ToList<Long> consumerToList = StreamConsumers.toList(eventloop);
SimpleNioServer server =
new SimpleNioServer(eventloop) {
@Override
protected SocketConnection createConnection(SocketChannel socketChannel) {
return new StreamMessagingConnection<>(
eventloop,
socketChannel,
new StreamGsonDeserializer<>(eventloop, new Gson(), String.class, 10),
new StreamGsonSerializer<>(eventloop, new Gson(), String.class, 1, 50, 0))
.addHandler(
String.class,
new MessagingHandler<String, String>() {
@Override
public void onMessage(String item, Messaging<String> messaging) {
assertEquals("start", item);
StreamBinaryDeserializer<Long> streamDeserializer =
new StreamBinaryDeserializer<>(eventloop, longSerializer(), 10);
messaging.binarySocketReader().streamTo(streamDeserializer);
streamDeserializer.streamTo(consumerToList);
messaging.shutdownWriter();
}
});
}
};
server.setListenAddress(address).acceptOnce();
server.listen();
eventloop.connect(
address,
new SocketSettings(),
new ConnectCallback() {
@Override
public void onConnect(SocketChannel socketChannel) {
SocketConnection connection =
new StreamMessagingConnection<>(
eventloop,
socketChannel,
new StreamGsonDeserializer<>(eventloop, new Gson(), String.class, 10),
new StreamGsonSerializer<>(eventloop, new Gson(), String.class, 1, 50, 0))
.addStarter(
new MessagingStarter<String>() {
@Override
public void onStart(Messaging<String> messaging) {
messaging.sendMessage("start");
StreamBinarySerializer<Long> streamSerializer =
new StreamBinarySerializer<>(
eventloop, longSerializer(), 1, 10, 0, false);
StreamProducers.ofIterable(eventloop, source)
.streamTo(streamSerializer);
streamSerializer.streamTo(messaging.binarySocketWriter());
messaging.shutdownReader();
}
});
connection.register();
}
@Override
public void onException(Exception e) {
fail("Test Exception: " + e);
}
});
eventloop.run();
assertEquals(source, consumerToList.getList());
// FIXME: 1 ByteBuf Leak (ByteBuf(16) produced by StreamBinaryDeserializer)
// assertEquals(getPoolItemsString(), ByteBufPool.getCreatedItems(),
// ByteBufPool.getPoolItems());
}
public static void main(String[] args) {
StreamConsumers.ToList<TestAdvResult> consumerToList;
int iterations = 30;
root.setLevel(Level.ERROR);
DefiningClassLoader classLoader = new DefiningClassLoader();
ExecutorService executor = Executors.newCachedThreadPool();
Path aggregationsDir = Paths.get("test/aggregations/");
NioEventloop eventloop = new NioEventloop();
for (int i = 0; i < iterations + 1; ++i) {
if (i == iterations - 1) {
Logger root = (Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME);
root.setLevel(Level.INFO);
}
if (i == iterations) {
classLoader = new DefiningClassLoader();
}
logger.info("Benchmark started.");
logger.info("Initialization started.");
deleteRecursivelyQuietly(aggregationsDir);
CubeStructure structure = getStructure(classLoader);
AggregationStorage aggregationStorage =
new LocalFsAggregationStorage(eventloop, executor, structure, aggregationsDir);
Cube cube =
newCube(
eventloop,
classLoader,
new LogToCubeMetadataStorageStub(),
aggregationStorage,
structure);
cube.addAggregation(new Aggregation("pub", asList("pub"), asList("pubRequests")));
cube.addAggregation(new Aggregation("adv", asList("adv"), asList("advRequests")));
Path dir = Paths.get("test/logs/");
deleteRecursivelyQuietly(dir);
LogFileSystemImpl fileSystem = new LogFileSystemImpl(eventloop, executor, dir);
SerializerFactory bufferSerializerFactory =
SerializerFactory.createBufferSerializerFactory(classLoader, true, true);
SerializerScanner registry = SerializerScanner.defaultScanner();
SerializerGen serializerGen = registry.serializer(TypeToken.of(TestPubRequest.class));
BufferSerializer<TestPubRequest> bufferSerializer =
bufferSerializerFactory.createBufferSerializer(serializerGen);
LogManager<TestPubRequest> logManager =
new LogManagerImpl<>(eventloop, fileSystem, bufferSerializer);
LogToCubeRunner<TestPubRequest> logToCubeRunner =
new LogToCubeRunner<>(
eventloop,
cube,
logManager,
TestAggregatorSplitter.factory(),
"testlog",
asList("partitionA"),
new LogToCubeMetadataStorageStub());
logger.info("Initialization finished.");
int numberOfTestRequests = 100_000;
logger.info("Started generating {} random requests.", numberOfTestRequests);
List<TestPubRequest> pubRequests = generatePubRequests(numberOfTestRequests);
long advRequests = countAdvRequests(pubRequests);
logger.info("Finished generating random requests. Adv requests generated: {}", advRequests);
logger.info("Started streaming to LogManager.");
new StreamProducers.OfIterator<>(eventloop, pubRequests.iterator())
.streamTo(logManager.consumer("partitionA"));
eventloop.run();
logger.info("Finished streaming to LogManager.");
CompletionCallbackObserver cb = new CompletionCallbackObserver();
logger.info("Started processing log.");
logToCubeRunner.processLog(cb);
eventloop.run();
logger.info("Finished processing log.");
cb.check();
consumerToList = StreamConsumers.toList(eventloop);
cube.query(
0,
TestAdvResult.class,
new CubeQuery()
.dimension("adv")
.measure("advRequests")
.between("adv", 2500, 7500)
.orderDesc("advRequests"))
.streamTo(consumerToList);
logger.info("Started executing query.");
eventloop.run();
logger.info("Finished executing query.");
logger.info("Benchmark finished.");
}
executor.shutdownNow();
}