Add smoke tests
- extend `pytest` with an option to run only the minimum number
of (unit) test cases to just keep the coverage at 100%
- rationale:
+ many of the unit test cases partly overlap with
respect to the lines of source code executed
+ also, integration tests, by definition, do not
contribute to a higher test coverage
- implementation: mark "redundant" test cases as one of:
+ `pytest.mark.integration_test`
=> code usage from the perspective of the end user
+ `pytest.mark.overlapping_test`
=> tests not contributing to the 100% coverage
+ `pytest.mark.sanity_test`
=> tests providing confidence in the test data
- add `tests.conftest` module
=> programatically convert the above markers into
`@pytest.mark.no_cover` and collect the non-"redundant" tests
- add nox session "test-fast" to run only the minimum
number of (unit) test while holding coverage at 100%
- refactor some test modules
+ wrap some test cases in a class
+ move sanity tests to the end of the files
This commit is contained in:
parent
de740ebb5f
commit
7e3e67c300
11 changed files with 182 additions and 35 deletions
59
tests/conftest.py
Normal file
59
tests/conftest.py
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
"""Configurations and utilities for all tests."""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def pytest_addoption(parser):
|
||||
"""Define custom CLI options for `pytest`."""
|
||||
parser.addoption(
|
||||
"--smoke-tests-only",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Run the minimum number of (unit) tests to achieve full coverage",
|
||||
)
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
"""Define custom markers explicitly."""
|
||||
config.addinivalue_line(
|
||||
"markers",
|
||||
"integration_test: non-unit test case; skipped during coverage reporting",
|
||||
)
|
||||
config.addinivalue_line(
|
||||
"markers",
|
||||
"overlapping_test: test case not contributing towards higher coverage",
|
||||
)
|
||||
config.addinivalue_line(
|
||||
"markers",
|
||||
"sanity_test: test case providing confidence in the test data",
|
||||
)
|
||||
|
||||
|
||||
def pytest_collection_modifyitems(config, items):
|
||||
"""Pre-process the test cases programatically.
|
||||
|
||||
- Add `no_cover` marker to test cases with any of these markers:
|
||||
+ "integration_test"
|
||||
+ "overlapping_test"
|
||||
+ "sanity_test"
|
||||
- Select test cases with none of the above markers as smoke tests,
|
||||
i.e., the minimum number of test cases to achieve 100% coverage
|
||||
"""
|
||||
smoke_tests = []
|
||||
|
||||
for item in items:
|
||||
if (
|
||||
"integration_test" in item.keywords
|
||||
or "overlapping_test" in item.keywords
|
||||
or "sanity_test" in item.keywords
|
||||
):
|
||||
item.add_marker(pytest.mark.no_cover)
|
||||
|
||||
elif config.getoption("--smoke-tests-only"):
|
||||
smoke_tests.append(item)
|
||||
|
||||
if config.getoption("--smoke-tests-only"):
|
||||
if not smoke_tests:
|
||||
pytest.exit("No smoke tests found")
|
||||
|
||||
items[:] = smoke_tests
|
||||
Loading…
Add table
Add a link
Reference in a new issue