floraachy
/
pytest2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Further "unknown marks warning" improvements (#5178) 

Further "unknown marks warning" improvements
This commit is contained in:
Bruno Oliveira 2019-05-11 13:28:10 -03:00 committed by GitHub
parent 184ef92f0b 0594dba5ce
commit 465b2d998a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 106 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
changelog/5023.feature.rst Normal file
View File

@ -0,0 +1,5 @@
New flag ``--strict-markers`` that triggers an error when unknown markers (e.g. those not registered using the `markers option`_ in the configuration file) are used in the test suite.
The existing ``--strict`` option has the same behavior currently, but can be augmented in the future for additional checks.
.. _`markers option`: https://docs.pytest.org/en/latest/reference.html#confval-markers

2
doc/en/example/markers.rst
View File

@ -259,7 +259,7 @@ For an example on how to add and work with markers from a plugin, see
* Asking for existing markers via ``pytest --markers`` gives good output
* Typos in function markers are treated as an error if you use
the ``--strict`` option.
the ``--strict-markers`` option.
.. _`scoped-marking`:

6
doc/en/mark.rst
View File

@ -41,15 +41,15 @@ marks by registering them in ``pytest.ini`` like this:
slow
serial
When the ``--strict`` command-line flag is passed, any unknown marks applied
When the ``--strict-markers`` command-line flag is passed, any unknown marks applied
with the ``@pytest.mark.name_of_the_mark`` decorator will trigger an error.
Marks added by pytest or by a plugin instead of the decorator will not trigger
this error. To enforce validation of markers, add ``--strict`` to ``addopts``:
this error. To enforce validation of markers, add ``--strict-markers`` to ``addopts``:
.. code-block:: ini
[pytest]
addopts = --strict
addopts = --strict-markers
markers =
slow
serial

10
doc/en/reference.rst
View File

@ -1269,15 +1269,17 @@ passed multiple times. The expected format is ``name=value``. For example::
.. confval:: markers
When the ``--strict`` command-line argument is used, only known markers -
defined in code by core pytest or some plugin - are allowed.
You can list additional markers in this setting to add them to the whitelist.
When the ``--strict-markers`` or ``--strict`` command-line arguments are used,
only known markers - defined in code by core pytest or some plugin - are allowed.
You can list one marker name per line, indented from the option name.
You can list additional markers in this setting to add them to the whitelist,
in which case you probably want to add ``--strict-markers`` to ``addopts``
to avoid future regressions:
.. code-block:: ini
[pytest]
addopts = --strict-markers
markers =
slow
serial

8
src/_pytest/main.py
View File

@ -47,11 +47,6 @@ def pytest_addoption(parser):
type="args",
default=[],
)
# parser.addini("dirpatterns",
# "patterns specifying possible locations of test files",
# type="linelist", default=["**/test_*.txt",
# "**/test_*.py", "**/*_test.py"]
# )
group = parser.getgroup("general", "running and selection options")
group._addoption(
"-x",
@ -71,9 +66,10 @@ def pytest_addoption(parser):
help="exit after first num failures or errors.",
)
group._addoption(
"--strict-markers",
"--strict",
action="store_true",
help="marks not registered in configuration file raise errors.",
help="markers not registered in the `markers` section of the configuration file raise errors.",
)
group._addoption(
"-c",

7
src/_pytest/mark/structures.py
View File

@ -311,8 +311,11 @@ class MarkGenerator(object):
# If the name is not in the set of known marks after updating,
# then it really is time to issue a warning or an error.
if name not in self._markers:
if self._config.option.strict:
fail("{!r} is not a registered marker".format(name), pytrace=False)
if self._config.option.strict_markers:
fail(
"{!r} not found in `markers` configuration option".format(name),
pytrace=False,
)
else:
warnings.warn(
"Unknown pytest.mark.%s - is this a typo? You can register "

2
testing/code/test_excinfo.py
View File

@ -11,7 +11,6 @@ import textwrap
import py
import six
from six.moves import queue
from test_source import astonly
import _pytest
import pytest
@ -147,7 +146,6 @@ class TestTraceback_f_g_h(object):
assert s.startswith("def f():")
assert s.endswith("raise ValueError")
@astonly
@failsonjython
def test_traceback_entry_getsource_in_construct(self):
source = _pytest._code.Source(

4
testing/code/test_source.py
View File

@ -16,7 +16,6 @@ import _pytest._code
import pytest
from _pytest._code import Source
astonly = pytest.mark.nothing
failsonjython = pytest.mark.xfail("sys.platform.startswith('java')")
@ -227,7 +226,6 @@ class TestSourceParsingAndCompiling(object):
s = source.getstatement(1)
assert s == str(source)
@astonly
def test_getstatementrange_within_constructs(self):
source = Source(
"""\
@ -630,7 +628,6 @@ x = 3
class TestTry(object):
pytestmark = astonly
source = """\
try:
raise ValueError
@ -675,7 +672,6 @@ finally:
class TestIf(object):
pytestmark = astonly
source = """\
if 1:
y = 3

9
testing/python/fixtures.py
View File

@ -1925,10 +1925,10 @@ class TestAutouseManagement(object):
reprec = testdir.inline_run()
reprec.assertoutcome(passed=1)
@pytest.mark.issue(226)
@pytest.mark.parametrize("param1", ["", "params=[1]"], ids=["p00", "p01"])
@pytest.mark.parametrize("param2", ["", "params=[1]"], ids=["p10", "p11"])
def test_ordering_dependencies_torndown_first(self, testdir, param1, param2):
"""#226"""
testdir.makepyfile(
"""
import pytest
@ -2707,9 +2707,9 @@ class TestFixtureMarker(object):
reprec = testdir.inline_run("-v")
reprec.assertoutcome(passed=5)
@pytest.mark.issue(246)
@pytest.mark.parametrize("scope", ["session", "function", "module"])
def test_finalizer_order_on_parametrization(self, scope, testdir):
"""#246"""
testdir.makepyfile(
"""
import pytest
@ -2744,8 +2744,8 @@ class TestFixtureMarker(object):
reprec = testdir.inline_run("-lvs")
reprec.assertoutcome(passed=3)
@pytest.mark.issue(396)
def test_class_scope_parametrization_ordering(self, testdir):
"""#396"""
testdir.makepyfile(
"""
import pytest
@ -2865,8 +2865,8 @@ class TestFixtureMarker(object):
res = testdir.runpytest("-v")
res.stdout.fnmatch_lines(["*test_foo*alpha*", "*test_foo*beta*"])
@pytest.mark.issue(920)
def test_deterministic_fixture_collection(self, testdir, monkeypatch):
"""#920"""
testdir.makepyfile(
"""
import pytest
@ -3649,7 +3649,6 @@ class TestScopeOrdering(object):
"""Class of tests that ensure fixtures are ordered based on their scopes (#2405)"""
@pytest.mark.parametrize("variant", ["mark", "autouse"])
@pytest.mark.issue(github="#2405")
def test_func_closure_module_auto(self, testdir, variant, monkeypatch):
"""Semantically identical to the example posted in #2405 when ``use_mark=True``"""
monkeypatch.setenv("FIXTURE_ACTIVATION_VARIANT", variant)

3
testing/python/integration.py
View File

@ -393,8 +393,9 @@ class TestNoselikeTestAttribute(object):
assert not call.items
@pytest.mark.issue(351)
class TestParameterize(object):
"""#351"""
def test_idfn_marker(self, testdir):
testdir.makepyfile(
"""

44
testing/python/metafunc.py
View File

@ -159,8 +159,9 @@ class TestMetafunc(object):
("x", "y"), [("abc", "def"), ("ghi", "jkl")], ids=["one"]
)
@pytest.mark.issue(510)
def test_parametrize_empty_list(self):
"""#510"""
def func(y):
pass
@ -262,8 +263,8 @@ class TestMetafunc(object):
for val, expected in values:
assert _idval(val, "a", 6, None, item=None, config=None) == expected
@pytest.mark.issue(250)
def test_idmaker_autoname(self):
"""#250"""
from _pytest.python import idmaker
result = idmaker(
@ -356,8 +357,8 @@ class TestMetafunc(object):
result = idmaker(("a", "b"), [pytest.param(e.one, e.two)])
assert result == ["Foo.one-Foo.two"]
@pytest.mark.issue(351)
def test_idmaker_idfn(self):
"""#351"""
from _pytest.python import idmaker
def ids(val):
@ -375,8 +376,8 @@ class TestMetafunc(object):
)
assert result == ["10.0-IndexError()", "20-KeyError()", "three-b2"]
@pytest.mark.issue(351)
def test_idmaker_idfn_unique_names(self):
"""#351"""
from _pytest.python import idmaker
def ids(val):
@ -459,8 +460,9 @@ class TestMetafunc(object):
)
assert result == ["a0", "a1", "b0", "c", "b1"]
@pytest.mark.issue(714)
def test_parametrize_indirect(self):
"""#714"""
def func(x, y):
pass
@ -473,8 +475,9 @@ class TestMetafunc(object):
assert metafunc._calls[0].params == dict(x=1, y=2)
assert metafunc._calls[1].params == dict(x=1, y=3)
@pytest.mark.issue(714)
def test_parametrize_indirect_list(self):
"""#714"""
def func(x, y):
pass
@ -483,8 +486,9 @@ class TestMetafunc(object):
assert metafunc._calls[0].funcargs == dict(y="b")
assert metafunc._calls[0].params == dict(x="a")
@pytest.mark.issue(714)
def test_parametrize_indirect_list_all(self):
"""#714"""
def func(x, y):
pass
@ -493,8 +497,9 @@ class TestMetafunc(object):
assert metafunc._calls[0].funcargs == {}
assert metafunc._calls[0].params == dict(x="a", y="b")
@pytest.mark.issue(714)
def test_parametrize_indirect_list_empty(self):
"""#714"""
def func(x, y):
pass
@ -503,9 +508,9 @@ class TestMetafunc(object):
assert metafunc._calls[0].funcargs == dict(x="a", y="b")
assert metafunc._calls[0].params == {}
@pytest.mark.issue(714)
def test_parametrize_indirect_list_functional(self, testdir):
"""
#714
Test parametrization with 'indirect' parameter applied on
particular arguments. As y is is direct, its value should
be used directly rather than being passed to the fixture
@ -532,8 +537,9 @@ class TestMetafunc(object):
result = testdir.runpytest("-v")
result.stdout.fnmatch_lines(["*test_simple*a-b*", "*1 passed*"])
@pytest.mark.issue(714)
def test_parametrize_indirect_list_error(self, testdir):
"""#714"""
def func(x, y):
pass
@ -541,12 +547,13 @@ class TestMetafunc(object):
with pytest.raises(pytest.fail.Exception):
metafunc.parametrize("x, y", [("a", "b")], indirect=["x", "z"])
@pytest.mark.issue(714)
def test_parametrize_uses_no_fixture_error_indirect_false(self, testdir):
"""The 'uses no fixture' error tells the user at collection time
that the parametrize data they've set up doesn't correspond to the
fixtures in their test function, rather than silently ignoring this
and letting the test potentially pass.
#714
"""
testdir.makepyfile(
"""
@ -560,8 +567,8 @@ class TestMetafunc(object):
result = testdir.runpytest("--collect-only")
result.stdout.fnmatch_lines(["*uses no argument 'y'*"])
@pytest.mark.issue(714)
def test_parametrize_uses_no_fixture_error_indirect_true(self, testdir):
"""#714"""
testdir.makepyfile(
"""
import pytest
@ -580,8 +587,8 @@ class TestMetafunc(object):
result = testdir.runpytest("--collect-only")
result.stdout.fnmatch_lines(["*uses no fixture 'y'*"])
@pytest.mark.issue(714)
def test_parametrize_indirect_uses_no_fixture_error_indirect_string(self, testdir):
"""#714"""
testdir.makepyfile(
"""
import pytest
@ -597,8 +604,8 @@ class TestMetafunc(object):
result = testdir.runpytest("--collect-only")
result.stdout.fnmatch_lines(["*uses no fixture 'y'*"])
@pytest.mark.issue(714)
def test_parametrize_indirect_uses_no_fixture_error_indirect_list(self, testdir):
"""#714"""
testdir.makepyfile(
"""
import pytest
@ -614,8 +621,8 @@ class TestMetafunc(object):
result = testdir.runpytest("--collect-only")
result.stdout.fnmatch_lines(["*uses no fixture 'y'*"])
@pytest.mark.issue(714)
def test_parametrize_argument_not_in_indirect_list(self, testdir):
"""#714"""
testdir.makepyfile(
"""
import pytest
@ -1201,9 +1208,9 @@ class TestMetafuncFunctional(object):
reprec = testdir.runpytest()
reprec.assert_outcomes(passed=4)
@pytest.mark.issue(463)
@pytest.mark.parametrize("attr", ["parametrise", "parameterize", "parameterise"])
def test_parametrize_misspelling(self, testdir, attr):
"""#463"""
testdir.makepyfile(
"""
import pytest
@ -1386,8 +1393,9 @@ class TestMetafuncFunctionalAuto(object):
assert output.count("preparing foo-3") == 1
@pytest.mark.issue(308)
class TestMarkersWithParametrization(object):
"""#308"""
def test_simple_mark(self, testdir):
s = """
import pytest
@ -1575,8 +1583,8 @@ class TestMarkersWithParametrization(object):
reprec = testdir.inline_run(SHOW_PYTEST_WARNINGS_ARG)
reprec.assertoutcome(passed=2, skipped=2)
@pytest.mark.issue(290)
def test_parametrize_ID_generation_string_int_works(self, testdir):
"""#290"""
testdir.makepyfile(
"""
import pytest

2
testing/test_capture.py
View File

@ -605,8 +605,8 @@ class TestCaptureFixture(object):
result.stdout.fnmatch_lines(["*KeyboardInterrupt*"])
assert result.ret == 2
@pytest.mark.issue(14)
def test_capture_and_logging(self, testdir):
"""#14"""
p = testdir.makepyfile(
"""\
import logging

2
testing/test_conftest.py
View File

@ -491,10 +491,10 @@ class TestConftestVisibility(object):
("snc", ".", 1),
],
)
@pytest.mark.issue(616)
def test_parsefactories_relative_node_ids(
self, testdir, chdir, testarg, expect_ntests_passed
):
"""#616"""
dirs = self._setup_tree(testdir)
print("pytest run in cwd: %s" % (dirs[chdir].relto(testdir.tmpdir)))
print("pytestarg : %s" % (testarg))

35
testing/test_mark.py
View File

@ -44,11 +44,11 @@ class TestMark(object):
class SomeClass(object):
pass
assert pytest.mark.fun(some_function) is some_function
assert pytest.mark.fun.with_args(some_function) is not some_function
assert pytest.mark.foo(some_function) is some_function
assert pytest.mark.foo.with_args(some_function) is not some_function
assert pytest.mark.fun(SomeClass) is SomeClass
assert pytest.mark.fun.with_args(SomeClass) is not SomeClass
assert pytest.mark.foo(SomeClass) is SomeClass
assert pytest.mark.foo.with_args(SomeClass) is not SomeClass
def test_pytest_mark_name_starts_with_underscore(self):
mark = Mark()
@ -130,7 +130,7 @@ def test_ini_markers_whitespace(testdir):
assert True
"""
)
rec = testdir.inline_run("--strict", "-m", "a1")
rec = testdir.inline_run("--strict-markers", "-m", "a1")
rec.assertoutcome(passed=1)
@ -150,7 +150,7 @@ def test_marker_without_description(testdir):
)
ftdir = testdir.mkdir("ft1_dummy")
testdir.tmpdir.join("conftest.py").move(ftdir.join("conftest.py"))
rec = testdir.runpytest("--strict")
rec = testdir.runpytest("--strict-markers")
rec.assert_outcomes()
@ -194,7 +194,8 @@ def test_mark_on_pseudo_function(testdir):
reprec.assertoutcome(passed=1)
def test_strict_prohibits_unregistered_markers(testdir):
@pytest.mark.parametrize("option_name", ["--strict-markers", "--strict"])
def test_strict_prohibits_unregistered_markers(testdir, option_name):
testdir.makepyfile(
"""
import pytest
@ -203,9 +204,11 @@ def test_strict_prohibits_unregistered_markers(testdir):
pass
"""
)
result = testdir.runpytest("--strict")
result = testdir.runpytest(option_name)
assert result.ret != 0
result.stdout.fnmatch_lines(["'unregisteredmark' is not a registered marker"])
result.stdout.fnmatch_lines(
["'unregisteredmark' not found in `markers` configuration option"]
)
@pytest.mark.parametrize(
@ -449,8 +452,8 @@ class TestFunctional(object):
items, rec = testdir.inline_genitems(p)
self.assert_markers(items, test_foo=("a", "b"), test_bar=("a",))
@pytest.mark.issue(568)
def test_mark_should_not_pass_to_siebling_class(self, testdir):
"""#568"""
p = testdir.makepyfile(
"""
import pytest
@ -652,9 +655,9 @@ class TestFunctional(object):
markers = {m.name for m in items[name].iter_markers()}
assert markers == set(expected_markers)
@pytest.mark.issue(1540)
@pytest.mark.filterwarnings("ignore")
def test_mark_from_parameters(self, testdir):
"""#1540"""
testdir.makepyfile(
"""
import pytest
@ -933,16 +936,16 @@ def test_mark_expressions_no_smear(testdir):
def test_addmarker_order():
node = Node("Test", config=mock.Mock(), session=mock.Mock(), nodeid="Test")
node.add_marker("a")
node.add_marker("b")
node.add_marker("c", append=False)
node.add_marker("foo")
node.add_marker("bar")
node.add_marker("baz", append=False)
extracted = [x.name for x in node.iter_markers()]
assert extracted == ["c", "a", "b"]
assert extracted == ["baz", "foo", "bar"]
@pytest.mark.issue("https://github.com/pytest-dev/pytest/issues/3605")
@pytest.mark.filterwarnings("ignore")
def test_markers_from_parametrize(testdir):
"""#3605"""
testdir.makepyfile(
"""
from __future__ import print_function

2
testing/test_recwarn.py
View File

@ -47,8 +47,8 @@ class TestWarningsRecorderChecker(object):
assert values is rec.list
pytest.raises(AssertionError, rec.pop)
@pytest.mark.issue(4243)
def test_warn_stacklevel(self):
"""#4243"""
rec = WarningsRecorder()
with rec:
warnings.warn("test", DeprecationWarning, 2)

29
testing/test_stepwise.py
View File

@ -76,7 +76,7 @@ def broken_testdir(testdir):
def test_run_without_stepwise(stepwise_testdir):
result = stepwise_testdir.runpytest("-v", "--strict", "--fail")
result = stepwise_testdir.runpytest("-v", "--strict-markers", "--fail")
result.stdout.fnmatch_lines(["*test_success_before_fail PASSED*"])
result.stdout.fnmatch_lines(["*test_fail_on_flag FAILED*"])
@ -85,7 +85,9 @@ def test_run_without_stepwise(stepwise_testdir):
def test_fail_and_continue_with_stepwise(stepwise_testdir):
# Run the tests with a failing second test.
result = stepwise_testdir.runpytest("-v", "--strict", "--stepwise", "--fail")
result = stepwise_testdir.runpytest(
"-v", "--strict-markers", "--stepwise", "--fail"
)
assert not result.stderr.str()
stdout = result.stdout.str()
@ -95,7 +97,7 @@ def test_fail_and_continue_with_stepwise(stepwise_testdir):
assert "test_success_after_fail" not in stdout
# "Fix" the test that failed in the last run and run it again.
result = stepwise_testdir.runpytest("-v", "--strict", "--stepwise")
result = stepwise_testdir.runpytest("-v", "--strict-markers", "--stepwise")
assert not result.stderr.str()
stdout = result.stdout.str()
@ -107,7 +109,12 @@ def test_fail_and_continue_with_stepwise(stepwise_testdir):
def test_run_with_skip_option(stepwise_testdir):
result = stepwise_testdir.runpytest(
"-v", "--strict", "--stepwise", "--stepwise-skip", "--fail", "--fail-last"
"-v",
"--strict-markers",
"--stepwise",
"--stepwise-skip",
"--fail",
"--fail-last",
)
assert not result.stderr.str()
@ -120,7 +127,7 @@ def test_run_with_skip_option(stepwise_testdir):
def test_fail_on_errors(error_testdir):
result = error_testdir.runpytest("-v", "--strict", "--stepwise")
result = error_testdir.runpytest("-v", "--strict-markers", "--stepwise")
assert not result.stderr.str()
stdout = result.stdout.str()
@ -131,7 +138,7 @@ def test_fail_on_errors(error_testdir):
def test_change_testfile(stepwise_testdir):
result = stepwise_testdir.runpytest(
"-v", "--strict", "--stepwise", "--fail", "test_a.py"
"-v", "--strict-markers", "--stepwise", "--fail", "test_a.py"
)
assert not result.stderr.str()
@ -140,7 +147,9 @@ def test_change_testfile(stepwise_testdir):
# Make sure the second test run starts from the beginning, since the
# test to continue from does not exist in testfile_b.
result = stepwise_testdir.runpytest("-v", "--strict", "--stepwise", "test_b.py")
result = stepwise_testdir.runpytest(
"-v", "--strict-markers", "--stepwise", "test_b.py"
)
assert not result.stderr.str()
stdout = result.stdout.str()
@ -149,7 +158,11 @@ def test_change_testfile(stepwise_testdir):
def test_stop_on_collection_errors(broken_testdir):
result = broken_testdir.runpytest(
"-v", "--strict", "--stepwise", "working_testfile.py", "broken_testfile.py"
"-v",
"--strict-markers",
"--stepwise",
"working_testfile.py",
"broken_testfile.py",
)
stdout = result.stdout.str()

2
testing/test_tmpdir.py
View File

@ -58,8 +58,8 @@ class TestTempdirHandler(object):
assert tmp2.relto(t.getbasetemp()).startswith("this")
assert tmp2 != tmp
@pytest.mark.issue(4425)
def test_tmppath_relative_basetemp_absolute(self, tmp_path, monkeypatch):
"""#4425"""
from _pytest.tmpdir import TempPathFactory
monkeypatch.chdir(tmp_path)

2
testing/test_unittest.py
View File

@ -930,11 +930,11 @@ def test_class_method_containing_test_issue1558(testdir):
reprec.assertoutcome(passed=1)
@pytest.mark.issue(3498)
@pytest.mark.parametrize(
"base", ["six.moves.builtins.object", "unittest.TestCase", "unittest2.TestCase"]
)
def test_usefixtures_marker_on_unittest(base, testdir):
"""#3498"""
module = base.rsplit(".", 1)[0]
pytest.importorskip(module)
testdir.makepyfile(

2
testing/test_warnings.py
View File

@ -302,7 +302,7 @@ def test_filterwarnings_mark_registration(testdir):
pass
"""
)
result = testdir.runpytest("--strict")
result = testdir.runpytest("--strict-markers")
assert result.ret == 0

8
tox.ini
View File

@ -139,7 +139,7 @@ commands = python scripts/release.py {posargs}
[pytest]
minversion = 2.0
addopts = -ra -p pytester
addopts = -ra -p pytester --strict-markers
rsyncdirs = tox.ini doc src testing
python_files = test_*.py *_test.py testing/*/*.py
python_classes = Test Acceptance
@ -170,7 +170,11 @@ filterwarnings =
ignore::_pytest.warning_types.PytestUnknownMarkWarning
pytester_example_dir = testing/example_scripts
markers =
issue
# dummy markers for testing
foo
bar
baz
# conftest.py reorders tests moving slow ones to the end of the list
slow
[flake8]