@TestStep(VEND) public void vend() { Item item = machine.random(coins, rand); scripter.step("VEND (" + bottles + ") - " + item.name); test.addCoverage("item", item.name); coins -= item.cost; bottles--; req.covered(VEND); }
/** * Evaluates the test case to define if test generation has reached a plateau. The test case * generation is considered to have reached a plateau if the current test case is observed to gain * less than the plateau threshold of added coverage in the current best exploration option (up to * 'depth' new steps). The plateau threshold is defined in the exploration configuration. * * @param suite The suite where we grab the current test from to check for plateau. * @param steps How many steps to check for the plateau. * @return True if plateau is reached. False otherwise. */ private boolean isTestPlateau(TestSuite suite, int steps) { TestCase currentTest = suite.getCurrentTest(); int length = currentTest.getSteps().size(); // steps + 1 is to measure gain for last "steps" number of steps correctly // for example, suite has four steps: A,B,C,D and we request to look at last 3 // if we count different between D and B, we get gain of last two which is C and D // so we must look at different between gain in end of A and in end of D. this requires the + 1 int index = length - (steps + 1); if (index < steps) { // not possible to check the plateau if test is not long enough return false; } int now = currentTest.getCurrentStep().getAddedCoverage(); int before = currentTest.getSteps().get(index).getAddedCoverage(); // TestCoverage coverage = suite.getCoverage(); // int now = scoreCalculator.addedScoreFor(coverage, currentTest); //// int before = scoreCalculator.addedScoreFor(coverage, currentTest, length-steps); // int before = scoreCalculator.addedScoreFor(coverage, currentTest); int diff = now - before; // is the added value more than the given threshold? return diff < config.getTestPlateauThreshold(); }
@Override public boolean endTest(TestSuite suite, FSM fsm) { if (scenarioEndCondition != null) { // if scenario is not yet done, we do not end no matter what if (!scenarioEndCondition.endTest(suite, fsm)) return false; } if (isTimedOut()) { log.d("Exploration timeout"); return true; } int max = config.getMaxTestLength(); int testSteps = suite.currentSteps(); if (max > 0) { if (testSteps >= max) { log.d("test over maximum length"); return true; } } // have we achieved defined minimum length yet? int min = config.getMinTestLength(); if (min > 0) { if (testSteps < min) { return false; } } long mySeed = seed + suite.totalSteps(); // plateau should take precedence over score limit otherwise it never happens. thus the ordering // with this here int plateauThreshold = config.getTestPlateauThreshold(); if (plateauThreshold > 0) { if (isTestPlateau(suite, config.getTestPlateauLength())) { log.d("test has plateaued"); return checkProbability(mySeed); } } // have we achieved minimum defined added test coverage yet? int minScore = config.getMinTestScore(); if (minScore > 0) { if (suiteCoverage == null) suiteCoverage = suite.getCoverage(); TestCase test = suite.getCurrentTest(); TestCaseStep step = test.getCurrentStep(); // this is the case when starting generation if (step == null) return false; int added = step.getAddedCoverage(); if (added < minScore) { return false; } } // exploration only works nice and shiny until the end condition hits, then we have to go with // the pössis // this is because if we evaluate paths for overall score and one of them is longer but only due // to having // a different random "v" (below) value generated, it "seems" better although in fact should not // (or should it ?:) // for example: ++-+-+++++++++ //<-evaluates one step later for the random "v" and gets a // different value // ++-+++++++ //<-evaluates one step before for the random "v" and gets a // different value if (exploring) return true; return checkProbability(mySeed); }
@AfterTest public void storeTest() { test.setAttribute("script", scripter.script); }