private Expression createExpression(String[] arguments, Operands operands) throws NoSuchOperatorException { OperatorFactory factory = new OperatorFactory(); Operator operator = factory.createOperator(arguments[1]); Expression expression = new Expression(operator, operands); return expression; }
@Override public void onEntitySpawned(Entity e, Layer layer) { if (e instanceof ControlableBody) { log.debug("adding entity operator"); OperatorFactory<Entity, ?> fac = entOpFac.get(0); ops.add(new OperatorBinding<Entity>(e, fac.create(e))); } }
@Override public void onLayerAdded(Layer layer) { // TODO access key for layer type and annotation for (OperatorFactory<Layer, ?> fac : layOpFac) { log.debug("binding layer operator"); ops.add(new OperatorBinding<Layer>(layer, fac.create(layer))); } }
@Test(dataProvider = "hashEnabledValues") public void testProbeOuterJoin( boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled) throws Exception { TaskContext taskContext = createTaskContext(); // build List<Type> buildTypes = ImmutableList.<Type>of(VARCHAR, BIGINT, BIGINT); RowPagesBuilder buildPages = rowPagesBuilder(buildHashEnabled, Ints.asList(0), ImmutableList.of(VARCHAR, BIGINT, BIGINT)) .addSequencePage(10, 20, 30, 40); LookupSourceSupplier lookupSourceSupplier = buildHash(parallelBuild, taskContext, Ints.asList(0), buildPages); // probe List<Type> probeTypes = ImmutableList.<Type>of(VARCHAR, BIGINT, BIGINT); RowPagesBuilder probePages = rowPagesBuilder(probeHashEnabled, Ints.asList(0), probeTypes); List<Page> probeInput = probePages.addSequencePage(15, 20, 1020, 2020).build(); OperatorFactory joinOperatorFactory = LookupJoinOperators.probeOuterJoin( 0, new PlanNodeId("test"), lookupSourceSupplier, probePages.getTypes(), Ints.asList(0), probePages.getHashChannel()); Operator joinOperator = joinOperatorFactory.createOperator( taskContext.addPipelineContext(true, true).addDriverContext()); // expected // expected MaterializedResult expected = MaterializedResult.resultBuilder(taskContext.getSession(), concat(probeTypes, buildTypes)) .row("20", 1020, 2020, "20", 30, 40) .row("21", 1021, 2021, "21", 31, 41) .row("22", 1022, 2022, "22", 32, 42) .row("23", 1023, 2023, "23", 33, 43) .row("24", 1024, 2024, "24", 34, 44) .row("25", 1025, 2025, "25", 35, 45) .row("26", 1026, 2026, "26", 36, 46) .row("27", 1027, 2027, "27", 37, 47) .row("28", 1028, 2028, "28", 38, 48) .row("29", 1029, 2029, "29", 39, 49) .row("30", 1030, 2030, null, null, null) .row("31", 1031, 2031, null, null, null) .row("32", 1032, 2032, null, null, null) .row("33", 1033, 2033, null, null, null) .row("34", 1034, 2034, null, null, null) .build(); assertOperatorEquals( joinOperator, probeInput, expected, true, getHashChannels(probePages, buildPages)); }
// 20 LET x = 3 public void parse(String line) { if (line.equalsIgnoreCase("RUN")) { this.run(); return; } if (line.equalsIgnoreCase("LIST")) { for (int l : code.keySet()) { System.out.println(l + " " + code.get(l)); } return; } try { String parts[] = line.split(" "); int lineNumber = Integer.parseInt(parts[0]); String opName = parts[1]; Operator operator = OperatorFactory.createOperator( opName, line.substring(parts[0].length() + parts[1].length() + 2)); code.put(lineNumber, operator); } catch (RuntimeException e) { System.err.println("Wrong operation!"); } }
@Test(dataProvider = "hashEnabledValues") public void testOuterJoinWithNullOnBothSides( boolean parallelBuild, boolean probeHashEnabled, boolean buildHashEnabled) throws Exception { TaskContext taskContext = createTaskContext(); // build RowPagesBuilder buildPages = rowPagesBuilder(buildHashEnabled, Ints.asList(0), ImmutableList.of(VARCHAR)) .row("a") .row((String) null) .row((String) null) .row("a") .row("b"); LookupSourceSupplier lookupSourceSupplier = buildHash(parallelBuild, taskContext, Ints.asList(0), buildPages); // probe List<Type> probeTypes = ImmutableList.<Type>of(VARCHAR); RowPagesBuilder probePages = rowPagesBuilder(probeHashEnabled, Ints.asList(0), probeTypes); List<Page> probeInput = probePages.row("a").row("b").row((String) null).row("c").build(); OperatorFactory joinOperatorFactory = LookupJoinOperators.probeOuterJoin( 0, new PlanNodeId("test"), lookupSourceSupplier, probePages.getTypes(), Ints.asList(0), probePages.getHashChannel()); Operator joinOperator = joinOperatorFactory.createOperator( taskContext.addPipelineContext(true, true).addDriverContext()); // expected MaterializedResult expected = MaterializedResult.resultBuilder( taskContext.getSession(), concat(probeTypes, buildPages.getTypes())) .row("a", "a") .row("a", "a") .row("b", "b") .row(null, null) .row("c", null) .build(); assertOperatorEquals( joinOperator, probeInput, expected, true, getHashChannels(probePages, buildPages)); }
@Test public void testLimitWithPageAlignment() throws Exception { List<Page> input = rowPagesBuilder(SINGLE_LONG) .addSequencePage(3, 1) .addSequencePage(2, 4) .addSequencePage(2, 6) .build(); OperatorFactory operatorFactory = new LimitOperatorFactory(0, ImmutableList.of(SINGLE_LONG), 5, Optional.<Integer>absent()); Operator operator = operatorFactory.createOperator(driverContext); List<Page> expected = rowPagesBuilder(SINGLE_LONG).addSequencePage(3, 1).addSequencePage(2, 4).build(); OperatorAssertion.assertOperatorEquals(operator, input, expected); }
@Test public void testAggregation() throws Exception { List<Page> input = rowPagesBuilder( SINGLE_VARBINARY, SINGLE_LONG, SINGLE_VARBINARY, SINGLE_LONG, SINGLE_DOUBLE, SINGLE_VARBINARY) .addSequencePage(100, 0, 0, 300, 500, 500, 500) .build(); OperatorFactory operatorFactory = new AggregationOperatorFactory( 0, Step.SINGLE, ImmutableList.of( aggregation(COUNT, new Input(0)), aggregation(LONG_SUM, new Input(1)), aggregation(LONG_AVERAGE, new Input(1)), aggregation(VAR_BINARY_MAX, new Input(2)), aggregation(COUNT_STRING_COLUMN, new Input(0)), aggregation(LONG_SUM, new Input(3)), aggregation(DOUBLE_SUM, new Input(4)), aggregation(VAR_BINARY_MAX, new Input(5)))); Operator operator = operatorFactory.createOperator(driverContext); MaterializedResult expected = resultBuilder( FIXED_INT_64, FIXED_INT_64, DOUBLE, VARIABLE_BINARY, FIXED_INT_64, FIXED_INT_64, DOUBLE, VARIABLE_BINARY) .row(100, 4950, 49.5, "399", 100, 54950, 54950.0, "599") .build(); assertOperatorEquals(operator, input, expected); }
@Test public void testSampledLimit() throws Exception { List<Page> input = rowPagesBuilder(SINGLE_LONG) .addSequencePage(2, 1) .addSequencePage(2, 4) .addSequencePage(2, 6) .build(); input = appendSampleWeight(input, 2); OperatorFactory operatorFactory = new LimitOperatorFactory( 0, ImmutableList.of(SINGLE_LONG), 5, Optional.of(input.get(0).getChannelCount() - 1)); Operator operator = operatorFactory.createOperator(driverContext); List<Page> expected = rowPagesBuilder(SINGLE_LONG, SINGLE_LONG).row(1, 2).row(2, 2).pageBreak().row(4, 1).build(); OperatorAssertion.assertOperatorEquals(operator, input, expected); }
@Test public void test() throws Exception { List<Page> input = rowPagesBuilder(SINGLE_VARBINARY, SINGLE_LONG).addSequencePage(100, 0, 0).build(); OperatorFactory operatorFactory = new FilterAndProjectOperatorFactory( 0, new FilterFunction() { @Override public boolean filter(TupleReadable... cursors) { long value = cursors[1].getLong(); return 10 <= value && value < 20; } @Override public boolean filter(RecordCursor cursor) { long value = cursor.getLong(0); return 10 <= value && value < 20; } }, ImmutableList.of(singleColumn(VARIABLE_BINARY, 0), new Add5Projection(1))); Operator operator = operatorFactory.createOperator(driverContext); MaterializedResult expected = MaterializedResult.resultBuilder(VARIABLE_BINARY, FIXED_INT_64) .row("10", 15) .row("11", 16) .row("12", 17) .row("13", 18) .row("14", 19) .row("15", 20) .row("16", 21) .row("17", 22) .row("18", 23) .row("19", 24) .build(); assertOperatorEquals(operator, input, expected); }
private static LookupSourceSupplier buildHash( boolean parallelBuild, TaskContext taskContext, List<Integer> hashChannels, RowPagesBuilder buildPages) { if (parallelBuild) { ParallelHashBuilder parallelHashBuilder = new ParallelHashBuilder( buildPages.getTypes(), hashChannels, buildPages.getHashChannel(), 100, PARTITION_COUNT); // collect input data DriverContext collectDriverContext = taskContext.addPipelineContext(true, true).addDriverContext(); ValuesOperatorFactory valuesOperatorFactory = new ValuesOperatorFactory( 0, new PlanNodeId("test"), buildPages.getTypes(), buildPages.build()); OperatorFactory collectOperatorFactory = parallelHashBuilder.getCollectOperatorFactory(1, new PlanNodeId("test")); Driver driver = new Driver( collectDriverContext, valuesOperatorFactory.createOperator(collectDriverContext), collectOperatorFactory.createOperator(collectDriverContext)); while (!driver.isFinished()) { driver.process(); } // build hash tables PipelineContext buildPipeline = taskContext.addPipelineContext(true, true); OperatorFactory buildOperatorFactory = parallelHashBuilder.getBuildOperatorFactory(new PlanNodeId("test")); for (int i = 0; i < PARTITION_COUNT; i++) { DriverContext buildDriverContext = buildPipeline.addDriverContext(); Driver buildDriver = new Driver(buildDriverContext, buildOperatorFactory.createOperator(buildDriverContext)); while (!buildDriver.isFinished()) { buildDriver.process(); } } return parallelHashBuilder.getLookupSourceSupplier(); } else { DriverContext driverContext = taskContext.addPipelineContext(true, true).addDriverContext(); ValuesOperatorFactory valuesOperatorFactory = new ValuesOperatorFactory( 0, new PlanNodeId("test"), buildPages.getTypes(), buildPages.build()); HashBuilderOperatorFactory hashBuilderOperatorFactory = new HashBuilderOperatorFactory( 1, new PlanNodeId("test"), buildPages.getTypes(), hashChannels, buildPages.getHashChannel(), 100); Driver driver = new Driver( driverContext, valuesOperatorFactory.createOperator(driverContext), hashBuilderOperatorFactory.createOperator(driverContext)); while (!driver.isFinished()) { driver.process(); } return hashBuilderOperatorFactory.getLookupSourceSupplier(); } }