diff --git a/README.md b/README.md index 0cef150..b6bff2d 100644 --- a/README.md +++ b/README.md @@ -4,15 +4,29 @@ A simple model-based testing (MBT) tool for python -pyosmo is python version of OSMO tester +Original OSMO tester can be found from: https://github.com/mukatee/osmo -Original OSMO tester can be found here: https://github.com/mukatee/osmo +## Motivation -Idea of model based testing is described in [introduction](doc/introduction.md) +Pyosmo is very useful tool when need to test system under testing logic very carefully or long time with automation. +This tool maximises the MBT tool flexibility and power by using the the pure python code as modelling language. + +From traditional testing tools perspective pyosmo provides automated test case creation based on programmed model. In +practise parametrized test cases (for +example: [pytest parametrized fixtures](https://docs.pytest.org/en/6.2.x/parametrize.html)) +are providing a bit similar functionality than simple test model can do. With true model it is able to plan a lot more +complex scenarions. + +From traditional [Model-based testing](https://en.wikipedia.org/wiki/Model-based_testing) tools perspective pyosmo is +providing much more flexible modelling language and simple integration to the system under testing or test generator. +Traditionally MBT tools have been using graphical modelling language which leads the stat explosion when test data is +included to the model. In pyosmo the model is pure python. Then it is able to model anything that is able to code with +python. All python libraries are helping the modelling work. ## Install using pip + ```bash pip install pyosmo ``` @@ -27,10 +41,10 @@ python -m pip install -e . ## Example model ```python -from pyosmo.osmo import Osmo +import pyosmo -class ExampleModel: +class ExampleCalculatorModel: def __init__(self): print('starting') @@ -44,20 +58,20 @@ class ExampleModel: def step_decrease(self): self._counter -= 1 - print("- {}".format(self._counter)) + print(f"- {self._counter}") def guard_increase(self): return self._counter < 100 def step_increase(self): self._counter += 1 - print("+ {}".format(self._counter)) + print(f"+ {self._counter}") # Initialize Osmo with model -osmo = Osmo(ExampleModel()) +osmo = pyosmo.Osmo(ExampleCalculatorModel()) # Generate tests -osmo.generate() +osmo.run() ``` # Select your approach @@ -100,16 +114,14 @@ Cons: ## Regression testing ```python -from pyosmo.osmo import Osmo -from pyosmo.end_conditions import StepCoverage -from pyosmo.end_conditions import Length +import pyosmo # This ues same example model than defined above -osmo = Osmo(ExampleModel()) +osmo = pyosmo.Osmo(ExampleCalculatorModel()) # Make sure that osmo go trough whole model in every test case -osmo.test_end_condition = StepCoverage(100) +osmo.test_end_condition = pyosmo.end_conditions.StepCoverage(100) # Do some test cases, which test do not take too long -osmo.test_suite_end_condition = Length(3) +osmo.test_suite_end_condition = pyosmo.end_conditions.Length(3) # Give seed to make sure that test is same every time osmo.seed = 333 # Run osmo @@ -120,24 +132,25 @@ osmo.generate() ```python import datetime -from pyosmo.osmo import Osmo -from pyosmo.end_conditions import Time +import pyosmo -osmo = Osmo(ExampleModel()) +osmo = pyosmo.Osmo(ExampleCalculatorModel()) # Run model for ten hours -osmo.test_end_condition = Time(int(datetime.timedelta(hours=10).total_seconds())) -osmo.test_suite_end_condition = Length(1) +osmo.test_end_condition = pyosmo.end_conditions.Time(int(datetime.timedelta(hours=10).total_seconds())) +osmo.test_suite_end_condition = pyosmo.end_conditions.Length(1) osmo.generate() ``` ## Run with pytest ```python +import pyosmo +# You can use your existing fixtures normally def test_smoke(): - osmo = Osmo(ExampleModel()) - osmo.test_end_condition = Length(10) - osmo.test_suite_end_condition = Length(1) - osmo.algorithm = RandomAlgorithm() + osmo = pyosmo.Osmo(ExampleCalculatorModel()) + osmo.test_end_condition = pyosmo.end_conditions.Length(10) + osmo.test_suite_end_condition = pyosmo.end_conditions.Length(1) + osmo.algorithm = pyosmo.algorithm.RandomAlgorithm() osmo.generate() ``` diff --git a/examples/offline_mbt/.gitignore b/examples/offline_mbt/.gitignore new file mode 100644 index 0000000..7ff40f2 --- /dev/null +++ b/examples/offline_mbt/.gitignore @@ -0,0 +1 @@ +generated_test.py diff --git a/examples/offline_mbt/README.md b/examples/offline_mbt/README.md new file mode 100644 index 0000000..c6ba6ea --- /dev/null +++ b/examples/offline_mbt/README.md @@ -0,0 +1,19 @@ +# Offline model-bases testing example + +This folder contains example of offline MBT + +Files: +* [offline_mbt.py](offline_mbt.py) contains simple calculator model and somo. Executing this generates tests in a file. +* [sut_calculator.py](sut_calculator.py) simulates system under testing in this case + +## How to use? + +Generate test cases +```bash +python offline_mbt.py +``` + +Execute test cases +```bash +pytest generated_test.py +``` \ No newline at end of file diff --git a/examples/offline_mbt/offline_mbt.py b/examples/offline_mbt/offline_mbt.py new file mode 100644 index 0000000..b4bdef4 --- /dev/null +++ b/examples/offline_mbt/offline_mbt.py @@ -0,0 +1,56 @@ +""" +Simple example model +""" + +import pyosmo + + +class CalculatorModel(pyosmo.OsmoModel): + + def __init__(self, f): + self.expected_result = 0 + self.file = f + self.max_number = 1000 + self.min_number = -1000 + self.test_number = 0 + + def before_suite(self): + self.file.write('# pylint: disable=too-many-lines\n') + self.file.write('from examples.offline_mbt.sut_calculator import CalculatorSUT\n') + + def before_test(self): + self.test_number += 1 + self.file.write(f'\n\ndef test_{self.test_number}_calculator():\n') + self.file.write(' calculator = CalculatorSUT()\n') + self.expected_result = 0 + + def guard_plus(self): + return self.expected_result < self.max_number + + def step_plus(self): + number_to_add = self.osmo_random.randint(0, self.max_number - self.expected_result) + self.expected_result += number_to_add + self.file.write(f' calculator.plus({number_to_add})\n') + self.file.write(f' assert calculator.display == {self.expected_result}\n') + + def guard_minus(self): + return self.expected_result > self.min_number + + def step_increase(self): + number_to_decrease = self.osmo_random.randint(0, abs(self.min_number - self.expected_result)) + self.expected_result -= number_to_decrease + self.file.write(f' calculator.minus({number_to_decrease})\n') + self.file.write(f' assert calculator.display == {self.expected_result}\n') + + +if __name__ == '__main__': + with open('generated_test.py', 'w', encoding='utf8') as file: + # Initialize Osmo with model + osmo = pyosmo.Osmo() + osmo.add_model(CalculatorModel(file)) + # Generate tests + osmo.test_end_condition = pyosmo.end_conditions.Length(10) + osmo.test_suite_end_condition = pyosmo.end_conditions.Length(100) + osmo.generate() + print('Tests generated in generated_test.py') + print('run "pytest generated_test.py" to executing tests') diff --git a/examples/offline_mbt/sut_calculator.py b/examples/offline_mbt/sut_calculator.py new file mode 100644 index 0000000..146b61c --- /dev/null +++ b/examples/offline_mbt/sut_calculator.py @@ -0,0 +1,10 @@ +class CalculatorSUT: + + def __init__(self): + self.display = 0 + + def plus(self, number): + self.display += number + + def minus(self, number): + self.display -= number diff --git a/examples/calculator_test_model.py b/examples/pytest/calculator_test_model.py similarity index 50% rename from examples/calculator_test_model.py rename to examples/pytest/calculator_test_model.py index 8da114d..7cd0b6c 100644 --- a/examples/calculator_test_model.py +++ b/examples/pytest/calculator_test_model.py @@ -1,66 +1,72 @@ # pylint: disable=no-self-use -from random import Random import pyosmo -from pyosmo import OsmoModel -from pyosmo.algorithm import WeightedAlgorithm -from pyosmo.end_conditions import Length -from pyosmo.osmo import Osmo -class CalculatorTestModel(OsmoModel): +@pyosmo.weight(10) # Set default weight for this class steps +class CalculatorTestModel(pyosmo.OsmoModel): + """ This is a simple test model for demonstration purposes """ - def __init__(self, random: Random = Random()): + def __init__(self): print('starting') self.expected_number = 0 - self.random = random def before_test(self): + """ This is executer always before new test case starts """ print('New test starts') self.expected_number = 0 def guard_minus(self): + """ It is always possible to reduce the number in this model, + then returning True always """ return True def step_minus(self): - numb = int(self.random.random() * 10) + """ The action what happens in case of minus button is pressed""" + numb = int(self.osmo_random.random() * 10) print(f"{self.expected_number} - {numb} = {self.expected_number - numb}") self.expected_number -= numb - @pyosmo.weight(2) # This happens two times more often than normally + @pyosmo.weight(2) def step_plus(self): - numb = int(self.random.random() * 10) + """ Number increase happens two times more often and then weight is 2 + step_plus do not have guard, which means that it is always possible step """ + numb = int(self.osmo_random.random() * 10) print(f"{self.expected_number} + {numb} = {self.expected_number + numb}") self.expected_number += numb def step_multiple(self): - numb = int(self.random.random() * 10) + """ Multiplying action """ + numb = int(self.osmo_random.random() * 10) print(f"{self.expected_number} * {numb} = {self.expected_number * numb}") self.expected_number *= numb def step_division(self): - numb = int(self.random.random() * 10) + """ Division action """ + numb = int(self.osmo_random.random() * 10) # Zero is not possible if numb == 0: numb = 2 print(f"{self.expected_number} / {numb} = {self.expected_number / numb}") self.expected_number /= numb - @pyosmo.weight(0.1) # this happens 10 times less ofthen usually + @pyosmo.weight(0.1) def step_clear(self): + """ Clearing the state happens very rarely """ self.expected_number = 0 print(0) def after(self): + """ This is executed after each test case """ display = self.expected_number print(f'Assert: {display = }') if __name__ == '__main__': # Initialize Osmo with model - osmo = Osmo(CalculatorTestModel()) - osmo.test_end_condition = Length(100) - osmo.algorithm = WeightedAlgorithm() + osmo = pyosmo.Osmo(CalculatorTestModel()) + osmo.test_end_condition = pyosmo.end_conditions.Length(100) + osmo.algorithm = pyosmo.algorithm.WeightedAlgorithm() # Generate tests osmo.generate() diff --git a/examples/pytest/test_calculator.py b/examples/pytest/test_calculator.py index bac56e4..54f5469 100644 --- a/examples/pytest/test_calculator.py +++ b/examples/pytest/test_calculator.py @@ -1,33 +1,41 @@ -from examples.calculator_test_model import CalculatorTestModel -from pyosmo import Osmo -from pyosmo.algorithm import WeightedAlgorithm, RandomAlgorithm -from pyosmo.end_conditions import Length -from pyosmo.models import RandomDelayModel - - -def test_smoke(): - # Initialize Osmo with model - osmo = Osmo(CalculatorTestModel()) - osmo.test_end_condition = Length(10) - osmo.test_suite_end_condition = Length(1) - osmo.algorithm = RandomAlgorithm() - osmo.generate() - - -def test_regression(): - # Initialize Osmo with model - osmo = Osmo(CalculatorTestModel()) - osmo.test_end_condition = Length(100) - osmo.test_suite_end_condition = Length(10) - osmo.algorithm = WeightedAlgorithm() +# pylint: disable=redefined-outer-name) +""" This demonstrates how to use model fom pytest side """ +import pytest + +import pyosmo +from examples.pytest.calculator_test_model import CalculatorTestModel + + +@pytest.fixture(scope='function') +def osmo() -> pyosmo.Osmo: + """ You can use common parts in fixtures as normally with pytest """ + return pyosmo.Osmo(CalculatorTestModel()) + + +@pytest.mark.smoke_test +def test_smoke(osmo): + """ Small test to run quickly and same way """ + osmo.seed = 1234 # Set seed to ensure that it runs same way every time + osmo.test_end_condition = pyosmo.end_conditions.Length(10) + osmo.test_suite_end_condition = pyosmo.end_conditions.Length(1) + osmo.algorithm = pyosmo.algorithm.RandomAlgorithm() + osmo.run() + + +@pytest.mark.regression_test +def test_regression(osmo): + """ Longer test to run in regression sets """ + osmo.test_end_condition = pyosmo.end_conditions.Length(100) + osmo.test_suite_end_condition = pyosmo.end_conditions.Length(10) + osmo.algorithm = pyosmo.algorithm.WeightedAlgorithm() osmo.run() -def test_random_timing(): - # Initialize Osmo with model - osmo = Osmo(CalculatorTestModel()) - osmo.add_model(RandomDelayModel(1, 2)) - osmo.test_end_condition = Length(10) - osmo.test_suite_end_condition = Length(1) - osmo.algorithm = WeightedAlgorithm() +@pytest.mark.long_test +def test_random_timing(osmo): + """ Longer test to test timings """ + osmo.add_model(pyosmo.models.RandomDelayModel(1, 2)) + osmo.test_end_condition = pyosmo.end_conditions.Length(10) + osmo.test_suite_end_condition = pyosmo.end_conditions.Length(1) + osmo.algorithm = pyosmo.algorithm.WeightedAlgorithm() osmo.run() diff --git a/examples/randomized_tests/2_randomize_input.py b/examples/randomized_tests/2_randomize_input.py new file mode 100644 index 0000000..70c13aa --- /dev/null +++ b/examples/randomized_tests/2_randomize_input.py @@ -0,0 +1,36 @@ +# pylint: disable=bare-except +import random +import time + + +def system_under_testing(data): + if data < 900: + # Flaky bug here + return "error" + return data * 2 + + +def test(): + # Send test data + data = random.randint(1, 10000) + output = system_under_testing(data) + + # Check that output is integer + try: + int(output) + except: + raise Exception("Data is not integer") from output + + # If we know logic of answer we can use that + # now output should be 2 times input + assert output == (2 * data) + + +if __name__ == '__main__': + # Loop test to see how easily test can find a bug + index = 0 + while True: + index += 1 + print(f"Test {index}") + time.sleep(0.1) + test() diff --git a/examples/randomized_tests/3_calculator.py b/examples/randomized_tests/3_calculator.py new file mode 100644 index 0000000..dc7e0a2 --- /dev/null +++ b/examples/randomized_tests/3_calculator.py @@ -0,0 +1,104 @@ +import random + +from pyosmo.end_conditions.length import Length +from pyosmo.osmo import Osmo + + +class Calculator: + """ + Calculator "implementation" which will be tested + """ + + def __init__(self): + self.result_number = 0 + + def add(self, num): + self.result_number += num + + def minus(self, num): + # Flaky bug here + if random.randint(1, 1000) > 995: + pass + else: + self.result_number -= num + + def result(self) -> int: + return self.result_number + + +class PositiveCalculator: + """ + This test model test only positive numbers + """ + # Reference number which will be compared to the system under testing + expected_count = 0 + # Connection to the system under testing, just instance in this case + calculator = Calculator() + + def __init__(self): + pass + + @staticmethod + def before_suite(): + """ This is executed at the begin of the test suite """ + print('START') + + def before_test(self): + """ This is executed before each test case """ + print('Test starts') + # Initializing variables + self.expected_count = 0 + self.calculator = Calculator() + + @staticmethod + def guard_add(): + """ It is always able to add when testing positive numbers + This guard can be deleted because this is default guard when guard is missing """ + return True + + def step_add(self): + """ Add a number """ + add_num = random.randint(1, 1000) + print(f'{self.expected_count} + {add_num} = {self.expected_count + add_num}') + # Add number to the system under testing + self.calculator.add(add_num) + # update reference variable + self.expected_count += add_num + + def guard_minus(self): + """ This is allowed only when count is positive """ + return self.expected_count > 0 + + def step_minus(self): + """ Minus random number """ + minus_num = random.randint(1, self.expected_count) + print(f'{self.expected_count} - {minus_num} = {self.expected_count - minus_num}') + # Minus number from system under testing + self.calculator.minus(minus_num) + # update reference variable + self.expected_count -= minus_num + + def after(self): + """ This happend after each test step """ + print(f'assert {self.calculator.result()} == {self.expected_count}') + assert self.calculator.result() == self.expected_count + + def after_test(self): + """ This is executed after each test """ + print(f'Test ends, final number: {self.expected_count}\n') + + @staticmethod + def after_suite(): + print('END') + + +if __name__ == '__main__': + # Add model to the osmo + osmo = Osmo(PositiveCalculator()) + # Setup test amount per suite + osmo.test_end_condition = Length(10) + # Set steps amount in test case + # Try to add bigger number of test steps to see when rare bug is cached + osmo.test_end_condition = Length(1) + # Run model + osmo.generate() diff --git a/examples/randomized_tests/4_randomized_timing.py b/examples/randomized_tests/4_randomized_timing.py new file mode 100644 index 0000000..b3526a7 --- /dev/null +++ b/examples/randomized_tests/4_randomized_timing.py @@ -0,0 +1,33 @@ +import random +import time + +from pyosmo.osmo import Osmo + + +class PositiveCalculator: + @staticmethod + def guard_something(): + return True + + @staticmethod + def step_something(): + print("1. inside step") + + # Random wait can be added inside test step + wait_ms = random.randint(200, 1000) + print(f"{wait_ms} sleep inside step") + time.sleep(wait_ms / 1000) + + print("2. inside step") + + @staticmethod + def after(): + # Random wait can be added also between test steps + wait_ms = random.randint(200, 3000) + print(f'Waiting for: {wait_ms}ms between steps') + time.sleep(wait_ms / 1000) + + +if __name__ == '__main__': + osmo = Osmo(PositiveCalculator()) + osmo.generate() diff --git a/examples/readme_example.py b/examples/readme_example.py new file mode 100644 index 0000000..8731ff3 --- /dev/null +++ b/examples/readme_example.py @@ -0,0 +1,35 @@ +""" +Simple example model +""" +from pyosmo.osmo import Osmo + + +class ExampleModel: + + def __init__(self): + print('starting') + self._counter = 0 + + def before_test(self): + self._counter = 0 + + def guard_decrease(self): + return self._counter > 1 + + def step_decrease(self): + self._counter -= 1 + print(f"- {self._counter}") + + def guard_increase(self): + return self._counter < 100 + + def step_increase(self): + self._counter += 1 + print(f"+ {self._counter}") + + +if __name__ == '__main__': + # Initialize Osmo with model + osmo = Osmo(ExampleModel()) + # Generate tests + osmo.generate() diff --git a/examples/split_model.py b/examples/split_model.py new file mode 100644 index 0000000..f45cf8a --- /dev/null +++ b/examples/split_model.py @@ -0,0 +1,61 @@ +""" This is simples possible example how to split model in two parts """ + +import pyosmo + + +class TestModel(pyosmo.OsmoModel): + + def __init__(self): + pass + + @staticmethod + def guard_first(): + return True + + @staticmethod + def step_first(): + print("1. step") + + @staticmethod + def before_suite(): + print('START') + + @staticmethod + def before_test(): + print('Test starts') + + @staticmethod + def after_test(): + print('Test ends') + + @staticmethod + def after_suite(): + print('END') + + +class TestModel2(pyosmo.OsmoModel): + + def __init__(self): + pass + + @staticmethod + def guard_second(): + return True + + @staticmethod + def pre_second(): + print("-->") + + @staticmethod + def step_second(): + print("2. step") + + @staticmethod + def post_second(): + print("<--") + + +if __name__ == '__main__': + osmo = pyosmo.Osmo(TestModel()) + osmo.add_model(TestModel2()) + osmo.generate() diff --git a/pyosmo/__init__.py b/pyosmo/__init__.py index 03cb0ed..81a1c4f 100644 --- a/pyosmo/__init__.py +++ b/pyosmo/__init__.py @@ -1,3 +1,6 @@ -from .decorators import weight -from .model import OsmoModel -from .osmo import Osmo +import pyosmo.algorithm +import pyosmo.end_conditions +import pyosmo.models +from pyosmo.decorators import weight +from pyosmo.models.osmo_model import OsmoModel +from pyosmo.osmo import Osmo diff --git a/pyosmo/algorithm/__init__.py b/pyosmo/algorithm/__init__.py index 022529c..b2f2d0d 100644 --- a/pyosmo/algorithm/__init__.py +++ b/pyosmo/algorithm/__init__.py @@ -1,2 +1,3 @@ +from .balancing import BalancingRandomAlgorithm, BalancingAlgorithm from .random import RandomAlgorithm -from .weighted import WeightedAlgorithm +from .weighted import WeightedAlgorithm, WeightedBalancingAlgorithm diff --git a/pyosmo/algorithm/balancing.py b/pyosmo/algorithm/balancing.py new file mode 100644 index 0000000..a0296ba --- /dev/null +++ b/pyosmo/algorithm/balancing.py @@ -0,0 +1,27 @@ +from typing import List + +from pyosmo.algorithm.base import OsmoAlgorithm +from pyosmo.history.history import OsmoHistory +from pyosmo.model import TestStep + + +class BalancingRandomAlgorithm(OsmoAlgorithm): + """ This is random algorithm but try to balance coverage. + In practise rare steps gets more priority when those are available """ + + def choose(self, history: OsmoHistory, choices: List[TestStep]) -> TestStep: + if len(choices) == 1: + return choices[0] + history_counts = [history.get_step_count(choice) for choice in choices] + weights = [(sum(history_counts) - h) for h in history_counts] + weights = [w - min(weights) + 1 for w in weights] + ret = self.random.choices(choices, weights=weights)[0] + return ret + + +class BalancingAlgorithm(OsmoAlgorithm): + """ Very simple and eager balancing algorithm """ + + def choose(self, history: OsmoHistory, choices: List[TestStep]) -> TestStep: + history_counts = [history.get_step_count(choice) for choice in choices] + return choices[history_counts.index(min(history_counts))] diff --git a/pyosmo/algorithm/weighted.py b/pyosmo/algorithm/weighted.py index 293770c..bd0b560 100644 --- a/pyosmo/algorithm/weighted.py +++ b/pyosmo/algorithm/weighted.py @@ -10,3 +10,26 @@ class WeightedAlgorithm(OsmoAlgorithm): def choose(self, history: OsmoHistory, choices: List[TestStep]) -> TestStep: return self.random.choices(choices, weights=[c.weight for c in choices])[0] + + +class WeightedBalancingAlgorithm(OsmoAlgorithm): + """ Weighted algorithm which balances based on history """ + + def choose(self, history: OsmoHistory, choices: List[TestStep]) -> TestStep: + weights = [c.weight for c in choices] + normalized_weights = [float(i) / max(weights) for i in weights] + + history_counts = [history.get_step_count(choice) for choice in choices] + if max(history_counts) == 0: + return self.random.choices(choices, weights=normalized_weights)[0] + + history_normalized_weights = [float(i) / max(history_counts) for i in history_counts] + + total_weights = [a - b if a - b != 0 else 0.1 for (a, b) in zip(normalized_weights, history_normalized_weights)] + + # Make sure that total weight is more than zero + if sum(total_weights) < 0: + temp_add = (abs(sum(total_weights)) + 0.2) / len(total_weights) + total_weights = [temp_add + x for x in total_weights] + + return self.random.choices(choices, weights=total_weights)[0] diff --git a/pyosmo/config.py b/pyosmo/config.py index c4d6e6e..c463b02 100644 --- a/pyosmo/config.py +++ b/pyosmo/config.py @@ -17,26 +17,13 @@ class OsmoConfig: def __init__(self): self._seed = randint(0, 10000) # pragma: no mutate - self._random = Random(self.seed) + self._random = Random(self._seed) self._algorithm = RandomAlgorithm() self._test_end_condition = Length(10) # pragma: no mutate self._test_suite_end_condition = Length(1) # pragma: no mutate self._test_error_strategy = AlwaysRaise() self._test_suite_error_strategy = AlwaysRaise() - @property - def seed(self) -> int: - return self._seed - - @seed.setter - def seed(self, value: int): - """ Set test generation algorithm """ - logger.debug(f'Set seed: {value}') - if not isinstance(value, int): - raise AttributeError("config needs to be OsmoConfig.") - self._seed = value - self._random = Random(self._seed) - @property def random(self) -> Random: return self._random diff --git a/pyosmo/model.py b/pyosmo/model.py index 87c977e..b0e2664 100644 --- a/pyosmo/model.py +++ b/pyosmo/model.py @@ -4,10 +4,6 @@ logger = logging.getLogger('osmo') -class OsmoModel: - pass - - class ModelFunction: """ Generic function class containing basic functionality of model functions""" diff --git a/pyosmo/models/osmo_model.py b/pyosmo/models/osmo_model.py new file mode 100644 index 0000000..abaa534 --- /dev/null +++ b/pyosmo/models/osmo_model.py @@ -0,0 +1,9 @@ +from random import Random + + +class OsmoModel: + """ Abstract Osmo model. When using cli tool need to make your models as child class of this one """ + + # This will be replaced with same random that osmo is using + # When using this random inside model you can give seed for osmo and get same run + osmo_random = Random() diff --git a/pyosmo/osmo.py b/pyosmo/osmo.py index 4c48fda..60dbdb8 100644 --- a/pyosmo/osmo.py +++ b/pyosmo/osmo.py @@ -1,6 +1,7 @@ # pylint: disable=bare-except,broad-except import logging from datetime import datetime +from random import Random from pyosmo.config import OsmoConfig from pyosmo.history.history import OsmoHistory @@ -20,6 +21,22 @@ def __init__(self, model: object = None): self.add_model(model) self.history = OsmoHistory() + @property + def seed(self) -> int: + return self._seed + + @seed.setter + def seed(self, value: int): + """ Set test generation algorithm """ + logger.debug(f'Set seed: {value}') + if not isinstance(value, int): + raise AttributeError("config needs to be OsmoConfig.") + self._seed = value + self._random = Random(self._seed) + # update osmo_random in all models + for model in self.model.sub_models: + model.osmo_random = self._random + @staticmethod def _check_model(model: object): """ Check that model is valid""" @@ -30,6 +47,8 @@ def add_model(self, model: object): """ Add model for osmo """ logger.debug(f'Add model:{model}') self._check_model(model) + # Set osmo_random + model.osmo_random = self._random self.model.add_model(model) def _run_step(self, step: TestStep): @@ -53,6 +72,7 @@ def run(self): def generate(self): """ Generate / run tests """ + self.history = OsmoHistory() # Restart the history logger.debug('Start generation..') logger.info(f'Using seed: {self.seed}') # Initialize algorithm diff --git a/pyosmo/tests/algorithms/__init__.py b/pyosmo/tests/algorithms/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pyosmo/tests/algorithms/test_balancing_algo.py b/pyosmo/tests/algorithms/test_balancing_algo.py new file mode 100644 index 0000000..4b78449 --- /dev/null +++ b/pyosmo/tests/algorithms/test_balancing_algo.py @@ -0,0 +1,53 @@ +from pyosmo import Osmo +from pyosmo.algorithm import BalancingRandomAlgorithm, BalancingAlgorithm +from pyosmo.end_conditions import Length + + +class WeightTestModel: + steps = [] + + def step_first(self): + self.steps.append('step_first') + + # Available every second time, balancing algoritm need to handle this + def guard_second(self): + return len(self.steps) % 2 == 0 + + def step_second(self): + self.steps.append('step_second') + + +def test_balancing_random_algorithm(): + steps_amount = 1000 + model = WeightTestModel() + osmo = Osmo(model) + osmo.test_end_condition = Length(1) + osmo.test_end_condition = Length(steps_amount) + osmo.algorithm = BalancingRandomAlgorithm() + osmo.generate() + + step_first_count = osmo.history.get_step_count(osmo.model.get_step_by_name("step_first")) + step_second_count = osmo.history.get_step_count(osmo.model.get_step_by_name("step_second")) + difference = step_first_count - step_second_count + + # Difference is less than 20% + assert difference < steps_amount * .2 + # Still need to be some difference + assert difference > 0 + + +def test_balancing_algorithm(): + steps_amount = 100 + model = WeightTestModel() + osmo = Osmo(model) + osmo.test_end_condition = Length(1) + osmo.test_end_condition = Length(steps_amount) + osmo.algorithm = BalancingAlgorithm() + osmo.generate() + + step_first_count = osmo.history.get_step_count(osmo.model.get_step_by_name("step_first")) + step_second_count = osmo.history.get_step_count(osmo.model.get_step_by_name("step_second")) + difference = step_first_count - step_second_count + + # Difference need to be very small + assert difference < 3 diff --git a/pyosmo/tests/algorithms/test_weight_algo.py b/pyosmo/tests/algorithms/test_weight_algo.py new file mode 100644 index 0000000..ceecf7e --- /dev/null +++ b/pyosmo/tests/algorithms/test_weight_algo.py @@ -0,0 +1,83 @@ +from pyosmo import Osmo, weight +from pyosmo.algorithm import WeightedAlgorithm, WeightedBalancingAlgorithm +from pyosmo.end_conditions import Length + + +class WeightTestModel1: + steps = [] + + @weight(1) + def step_first(self): + self.steps.append('step_first') + + @weight(2) + def step_second(self): + self.steps.append('step_second') + + @staticmethod + def weight_third(): + # More dynamic way to define weight + return 3 + + def step_third(self): + self.steps.append('step_third') + + +def test_weighted_algorithm(): + model = WeightTestModel1() + osmo = Osmo(model) + osmo.test_end_condition = Length(1) + osmo.test_end_condition = Length(100) + osmo.algorithm = WeightedAlgorithm() + osmo.generate() + + step_first_count = osmo.history.get_step_count(osmo.model.get_step_by_name("step_first")) + step_second_count = osmo.history.get_step_count(osmo.model.get_step_by_name("step_second")) + step_third_count = osmo.history.get_step_count(osmo.model.get_step_by_name("step_third")) + + assert step_first_count < step_second_count < step_third_count + + +@weight(10) # Set default weight for class functions +class WeightTestModel2: + steps = [] + + @weight(5) + def step_first(self): + self.steps.append('step_first') + + # Weight is 10 because of class basic weight + def step_second(self): + self.steps.append('step_second') + + def step_third(self): + self.steps.append('step_third') + + +def test_weighted_algorithm2(): + model = WeightTestModel2() + osmo = Osmo(model) + osmo.seed = 123 + osmo.test_suite_end_condition = Length(1) + osmo.test_end_condition = Length(100) + osmo.algorithm = WeightedAlgorithm() + osmo.generate() + + step_first_count = osmo.history.get_step_count(osmo.model.get_step_by_name("step_first")) + step_second_count = osmo.history.get_step_count(osmo.model.get_step_by_name("step_second")) + + assert step_first_count < step_second_count + + +def test_weighted_balancing_algorithm(): + model = WeightTestModel2() + osmo = Osmo(model) + osmo.test_suite_end_condition = Length(1) + osmo.test_end_condition = Length(100) + osmo.algorithm = WeightedBalancingAlgorithm() + osmo.generate() + + step_first_count = osmo.history.get_step_count(osmo.model.get_step_by_name("step_first")) + step_second_count = osmo.history.get_step_count(osmo.model.get_step_by_name("step_second")) + + assert step_first_count < step_second_count diff --git a/pyosmo/tests/test_error_strategy.py b/pyosmo/tests/test_error_strategy.py index a758488..dbaabde 100644 --- a/pyosmo/tests/test_error_strategy.py +++ b/pyosmo/tests/test_error_strategy.py @@ -31,10 +31,7 @@ def test_always_ignore(): osmo.test_suite_end_condition = Length(10) osmo.test_error_strategy = AlwaysIgnore() osmo.test_suite_error_strategy = AlwaysIgnore() - try: - osmo.generate() - except: - pass + osmo.generate() assert osmo.history.total_amount_of_steps == 10 * 100 diff --git a/pyosmo/tests/test_weight_algo.py b/pyosmo/tests/test_weight_algo.py deleted file mode 100644 index 4efea89..0000000 --- a/pyosmo/tests/test_weight_algo.py +++ /dev/null @@ -1,62 +0,0 @@ -from pyosmo import Osmo, weight -from pyosmo.algorithm import WeightedAlgorithm -from pyosmo.end_conditions import Length - - -def test_weighted_algorithm(): - class WeightTestModel: - steps = [] - - @weight(1) - def step_first(self): - self.steps.append('step_first') - - @weight(2) - def step_second(self): - self.steps.append('step_second') - - @staticmethod - def weight_third(): - # More dynamic way to define weight - return 3 - - def step_third(self): - self.steps.append('step_third') - - model = WeightTestModel() - osmo = Osmo(model) - osmo.test_end_condition = Length(1) - osmo.test_end_condition = Length(100) - osmo.algorithm = WeightedAlgorithm() - osmo.generate() - - step_first_count = osmo.history.get_step_count(osmo.model.get_step_by_name("step_first")) - step_second_count = osmo.history.get_step_count(osmo.model.get_step_by_name("step_second")) - step_third_count = osmo.history.get_step_count(osmo.model.get_step_by_name("step_third")) - - assert step_first_count < step_second_count < step_third_count - - -def test_weighted_algorithm_class_default(): - @weight(10) # Set default weight for class functions - class WeightTestModel: - steps = [] - - @weight(5) - def step_first(self): - self.steps.append('step_first') - - def step_second(self): - self.steps.append('step_second') - - model = WeightTestModel() - osmo = Osmo(model) - osmo.test_end_condition = Length(1) - osmo.test_end_condition = Length(100) - osmo.algorithm = WeightedAlgorithm() - osmo.generate() - - step_first_count = osmo.history.get_step_count(osmo.model.get_step_by_name("step_first")) - step_second_count = osmo.history.get_step_count(osmo.model.get_step_by_name("step_second")) - - assert step_first_count < step_second_count diff --git a/setup.py b/setup.py index 509c203..df7df1a 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ def read(fname): setup(name='pyosmo', - version='0.1.2', + version='0.1.3', python_requires='>=3.8', description=DESCRIPTION, long_description=read('README.md'),