From 89c73582caf9dda84f237bd6d4986d4db7d11a2e Mon Sep 17 00:00:00 2001 From: John Still Date: Tue, 11 Jul 2017 11:52:16 -0500 Subject: [PATCH 01/10] ignore the active python installation unless told otherwise --- _pytest/main.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/_pytest/main.py b/_pytest/main.py index 1a6ba2781..5b6409664 100644 --- a/_pytest/main.py +++ b/_pytest/main.py @@ -70,6 +70,8 @@ def pytest_addoption(parser): group.addoption('--keepduplicates', '--keep-duplicates', action="store_true", dest="keepduplicates", default=False, help="Keep duplicate tests.") + group.addoption('--collect-in-virtualenv', action='store_true', + help="Collect tests in the current Python installation (default False)") group = parser.getgroup("debugconfig", "test session debugging and configuration") @@ -177,6 +179,16 @@ def pytest_ignore_collect(path, config): if py.path.local(path) in ignore_paths: return True + invenv = py.path.local(sys.prefix) == path + allow_invenv = config.getoption("collect_in_virtualenv") + if invenv and not allow_invenv: + config.warn(RuntimeWarning, + 'Path "%s" appears to be a Python installation; skipping\n' + 'Pass --collect-in-virtualenv to force collection of tests in "%s"\n' + 'Use --ignore="%s" to silence this warning' % (path, path, path) + ) + return True + # Skip duplicate paths. keepduplicates = config.getoption("keepduplicates") duplicate_paths = config.pluginmanager._duplicatepaths From c2d49e39a2603f3ee8ec3a0e13be4afc3303aca1 Mon Sep 17 00:00:00 2001 From: John Still Date: Tue, 11 Jul 2017 13:01:56 -0500 Subject: [PATCH 02/10] add news item --- changelog/2518.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/2518.feature diff --git a/changelog/2518.feature b/changelog/2518.feature new file mode 100644 index 000000000..3b97cc18f --- /dev/null +++ b/changelog/2518.feature @@ -0,0 +1 @@ +Collection ignores the currently active Python installation by default; `--collect-in-virtualenv` overrides this behavior. From 676c4f970d2d2f3dcece052adc65491e1f9e1588 Mon Sep 17 00:00:00 2001 From: John Still Date: Tue, 11 Jul 2017 13:31:11 -0500 Subject: [PATCH 03/10] trim trailing ws --- _pytest/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_pytest/main.py b/_pytest/main.py index 5b6409664..bd33ab95f 100644 --- a/_pytest/main.py +++ b/_pytest/main.py @@ -179,7 +179,7 @@ def pytest_ignore_collect(path, config): if py.path.local(path) in ignore_paths: return True - invenv = py.path.local(sys.prefix) == path + invenv = py.path.local(sys.prefix) == path allow_invenv = config.getoption("collect_in_virtualenv") if invenv and not allow_invenv: config.warn(RuntimeWarning, From b32cfc88daad55f6518fc828db7aa770d4e4c80a Mon Sep 17 00:00:00 2001 From: John Still Date: Tue, 11 Jul 2017 14:32:09 -0500 Subject: [PATCH 04/10] use presence of activate script rather than sys.prefix to determine if a dir is a virtualenv --- _pytest/main.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/_pytest/main.py b/_pytest/main.py index bd33ab95f..caf2ca813 100644 --- a/_pytest/main.py +++ b/_pytest/main.py @@ -169,6 +169,17 @@ def pytest_runtestloop(session): return True +def _in_venv(path): + """Attempts to detect if ``path`` is the root of a Virtual Environment by + checking for the existence of the appropriate activate script""" + bindir = path.join('Scripts' if sys.platform.startswith('win') else 'bin') + if not bindir.exists(): + return False + activates = ('activate', 'activate.csh', 'activate.fish', + 'Activate', 'Activate.bat', 'Activate.ps1') + return any([fname.basename in activates for fname in bindir.listdir()]) + + def pytest_ignore_collect(path, config): ignore_paths = config._getconftest_pathlist("collect_ignore", path=path.dirpath()) ignore_paths = ignore_paths or [] @@ -179,11 +190,10 @@ def pytest_ignore_collect(path, config): if py.path.local(path) in ignore_paths: return True - invenv = py.path.local(sys.prefix) == path - allow_invenv = config.getoption("collect_in_virtualenv") - if invenv and not allow_invenv: + allow_in_venv = config.getoption("collect_in_virtualenv") + if _in_venv(path) and not allow_in_venv: config.warn(RuntimeWarning, - 'Path "%s" appears to be a Python installation; skipping\n' + 'Path "%s" appears to be a Python virtual installation; skipping\n' 'Pass --collect-in-virtualenv to force collection of tests in "%s"\n' 'Use --ignore="%s" to silence this warning' % (path, path, path) ) From 67fca040503b30ff7d23f6962c1387c8f621c80c Mon Sep 17 00:00:00 2001 From: John Still Date: Tue, 11 Jul 2017 23:14:38 -0500 Subject: [PATCH 05/10] update docs and note; add virtualenv collection tests --- _pytest/main.py | 8 ++----- changelog/2518.feature | 2 +- doc/en/customize.rst | 11 ++++++++- testing/test_collection.py | 49 +++++++++++++++++++++++++++++++++++++- 4 files changed, 61 insertions(+), 9 deletions(-) diff --git a/_pytest/main.py b/_pytest/main.py index caf2ca813..a7ecc5149 100644 --- a/_pytest/main.py +++ b/_pytest/main.py @@ -71,7 +71,8 @@ def pytest_addoption(parser): dest="keepduplicates", default=False, help="Keep duplicate tests.") group.addoption('--collect-in-virtualenv', action='store_true', - help="Collect tests in the current Python installation (default False)") + dest='collect_in_virtualenv', default=False, + help="Don't ignore tests in a local virtualenv directory") group = parser.getgroup("debugconfig", "test session debugging and configuration") @@ -192,11 +193,6 @@ def pytest_ignore_collect(path, config): allow_in_venv = config.getoption("collect_in_virtualenv") if _in_venv(path) and not allow_in_venv: - config.warn(RuntimeWarning, - 'Path "%s" appears to be a Python virtual installation; skipping\n' - 'Pass --collect-in-virtualenv to force collection of tests in "%s"\n' - 'Use --ignore="%s" to silence this warning' % (path, path, path) - ) return True # Skip duplicate paths. diff --git a/changelog/2518.feature b/changelog/2518.feature index 3b97cc18f..2f6597a97 100644 --- a/changelog/2518.feature +++ b/changelog/2518.feature @@ -1 +1 @@ -Collection ignores the currently active Python installation by default; `--collect-in-virtualenv` overrides this behavior. +Collection ignores local virtualenvs by default; `--collect-in-virtualenv` overrides this behavior. diff --git a/doc/en/customize.rst b/doc/en/customize.rst index ce0a36c11..1920028a1 100644 --- a/doc/en/customize.rst +++ b/doc/en/customize.rst @@ -171,7 +171,16 @@ Builtin configuration file options norecursedirs = .svn _build tmp* This would tell ``pytest`` to not look into typical subversion or - sphinx-build directories or into any ``tmp`` prefixed directory. + sphinx-build directories or into any ``tmp`` prefixed directory. + + Additionally, ``pytest`` will attempt to intelligently identify and ignore a + virtualenv by the presence of an activation script. Any directory deemed to + be the root of a virtual environment will not be considered during test + collection unless ``‑‑collect‑in‑virtualenv`` is given. Note also that + ``norecursedirs`` takes precedence over ``‑‑collect‑in‑virtualenv``; e.g. if + you intend to run tests in a virtualenv with a base directory that matches + ``'.*'`` you *must* override ``norecursedirs`` in addition to using the + ``‑‑collect‑in‑virtualenv`` flag. .. confval:: testpaths diff --git a/testing/test_collection.py b/testing/test_collection.py index a90269789..a3c323e61 100644 --- a/testing/test_collection.py +++ b/testing/test_collection.py @@ -1,7 +1,7 @@ from __future__ import absolute_import, division, print_function import pytest, py -from _pytest.main import Session, EXIT_NOTESTSCOLLECTED +from _pytest.main import Session, EXIT_NOTESTSCOLLECTED, _in_venv class TestCollector(object): def test_collect_versus_item(self): @@ -121,6 +121,53 @@ class TestCollectFS(object): assert "test_notfound" not in s assert "test_found" in s + @pytest.mark.parametrize('fname', + ("activate", "activate.csh", "activate.fish", + "Activate", "Activate.bat", "Activate.ps1")) + def test_ignored_virtualenvs(self, testdir, fname): + bindir = "Scripts" if py.std.sys.platform.startswith("win") else "bin" + testdir.tmpdir.ensure("virtual", bindir, fname) + testfile = testdir.tmpdir.ensure("virtual", "test_invenv.py") + testfile.write("def test_hello(): pass") + + # by default, ignore tests inside a virtualenv + result = testdir.runpytest() + assert "test_invenv" not in result.stdout.str() + # allow test collection if user insists + result = testdir.runpytest("--collect-in-virtualenv") + assert "test_invenv" in result.stdout.str() + # allow test collection if user directly passes in the directory + result = testdir.runpytest("virtual") + assert "test_invenv" in result.stdout.str() + + @pytest.mark.parametrize('fname', + ("activate", "activate.csh", "activate.fish", + "Activate", "Activate.bat", "Activate.ps1")) + def test_ignored_virtualenvs_norecursedirs_precedence(self, testdir, fname): + bindir = "Scripts" if py.std.sys.platform.startswith("win") else "bin" + # norecursedirs takes priority + testdir.tmpdir.ensure(".virtual", bindir, fname) + testfile = testdir.tmpdir.ensure(".virtual", "test_invenv.py") + testfile.write("def test_hello(): pass") + result = testdir.runpytest("--collect-in-virtualenv") + assert "test_invenv" not in result.stdout.str() + # ...unless the virtualenv is explicitly given on the CLI + result = testdir.runpytest("--collect-in-virtualenv", ".virtual") + assert "test_invenv" in result.stdout.str() + + @pytest.mark.parametrize('fname', + ("activate", "activate.csh", "activate.fish", + "Activate", "Activate.bat", "Activate.ps1")) + def test__in_venv(self, testdir, fname): + """Directly test the virtual env detection function""" + bindir = "Scripts" if py.std.sys.platform.startswith("win") else "bin" + # no bin/activate, not a virtualenv + base_path = testdir.tmpdir.mkdir('venv') + assert _in_venv(base_path) is False + # with bin/activate, totally a virtualenv + base_path.ensure(bindir, fname) + assert _in_venv(base_path) is True + def test_custom_norecursedirs(self, testdir): testdir.makeini(""" [pytest] From 3a1c9c0e45de5d05b34b200c2491d24ce4b236b0 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Tue, 18 Jul 2017 15:37:01 -0300 Subject: [PATCH 06/10] Clarify in the docs how PYTEST_ADDOPTS and addopts ini option work together --- doc/en/customize.rst | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/doc/en/customize.rst b/doc/en/customize.rst index f50d8b46e..7be7ca2e5 100644 --- a/doc/en/customize.rst +++ b/doc/en/customize.rst @@ -112,15 +112,27 @@ progress output, you can write it into a configuration file: # content of pytest.ini # (or tox.ini or setup.cfg) [pytest] - addopts = -rsxX -q + addopts = -ra -q -Alternatively, you can set a PYTEST_ADDOPTS environment variable to add command +Alternatively, you can set a ``PYTEST_ADDOPTS`` environment variable to add command line options while the environment is in use:: - export PYTEST_ADDOPTS="-rsxX -q" + export PYTEST_ADDOPTS="-v" -From now on, running ``pytest`` will add the specified options. +Here's how the command-line is built in the presence of ``addopts`` or the environment variable:: + $PYTEST_ADDOTPS + +So if the user executes in the command-line:: + + pytest -m slow + +The actual command line executed is:: + + pytest -ra -q -v -m slow + +Note that as usual for other command-line applications, in case of conflicting options the last one wins, so the example +above will show verbose output because ``-v`` overwrites ``-q``. Builtin configuration file options From 637e566d05c677d9ec71177412c787ef1af3548d Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Tue, 18 Jul 2017 17:04:37 -0300 Subject: [PATCH 07/10] Separate all options for running/selecting tests into sections --- doc/en/usage.rst | 69 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 55 insertions(+), 14 deletions(-) diff --git a/doc/en/usage.rst b/doc/en/usage.rst index 763328f5a..64c072886 100644 --- a/doc/en/usage.rst +++ b/doc/en/usage.rst @@ -52,23 +52,64 @@ To stop the testing process after the first (N) failures:: Specifying tests / selecting tests --------------------------------------------------- -Several test run options:: +Pytest supports several ways to run and select tests from the command-line. - pytest test_mod.py # run tests in module - pytest somepath # run all tests below somepath - pytest -k stringexpr # only run tests with names that match the - # "string expression", e.g. "MyClass and not method" - # will select TestMyClass.test_something - # but not TestMyClass.test_method_simple - pytest test_mod.py::test_func # only run tests that match the "node ID", - # e.g. "test_mod.py::test_func" will select - # only test_func in test_mod.py - pytest test_mod.py::TestClass::test_method # run a single method in - # a single class +**Run tests in a module** -Import 'pkg' and use its filesystem location to find and run tests:: +:: - pytest --pyargs pkg # run all tests found below directory of pkg + pytest test_mod.py + +**Run tests in a directory** + +:: + + pytest testing/ + +**Run tests by keyword expressions** + +:: + + pytest -k "MyClass and not method" + +This will run tests which contain names that match the given *string expression*, which can +include Python operators that use filenames, class names and function names as variables. +The example above will run ``TestMyClass.test_something`` but not ``TestMyClass.test_method_simple``. + +.. _nodeids: + +**Run tests by node ids** + +Each collected test is assigned a unique ``nodeid`` which consist of the module filename followed +by specifiers like class names, function names and parameters from parametrization, separated by ``::`` characters. + +To run a specific test within a module:: + + pytest test_mod.py::test_func + + +Another example specifying a test method in the command line:: + + pytest test_mod.py::TestClass::test_method + +**Run tests by marker expressions** + +:: + + pytest -m slow + +Will run all tests which are decorated with the ``@pytest.mark.slow`` decorator. + +For more information see :ref:`marks `. + +**Run tests from packages** + +:: + + pytest --pyargs pkg.testing + +This will import ``pkg.testing`` and use its filesystem location to find and run tests from. + Modifying Python traceback printing ---------------------------------------------- From 62556bada660ca209e7e83bc775559d9039f31b1 Mon Sep 17 00:00:00 2001 From: Ronny Pfannschmidt Date: Tue, 18 Jul 2017 08:28:44 +0200 Subject: [PATCH 08/10] remove the MARK_INFO_ATTRIBUTE warning until we can fix internal usage fixes #2573 --- _pytest/mark.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/_pytest/mark.py b/_pytest/mark.py index 961c3c409..61562330f 100644 --- a/_pytest/mark.py +++ b/_pytest/mark.py @@ -6,7 +6,7 @@ import warnings from collections import namedtuple from operator import attrgetter from .compat import imap -from .deprecated import MARK_INFO_ATTRIBUTE, MARK_PARAMETERSET_UNPACKING +from .deprecated import MARK_PARAMETERSET_UNPACKING def alias(name, warning=None): getter = attrgetter(name) @@ -401,9 +401,9 @@ class MarkInfo(object): self.combined = mark self._marks = [mark] - name = alias('combined.name', warning=MARK_INFO_ATTRIBUTE) - args = alias('combined.args', warning=MARK_INFO_ATTRIBUTE) - kwargs = alias('combined.kwargs', warning=MARK_INFO_ATTRIBUTE) + name = alias('combined.name') + args = alias('combined.args') + kwargs = alias('combined.kwargs') def __repr__(self): return "".format(self.combined) From 2d4f1f022eb83d3029a35c3ef854534263af6c16 Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Tue, 18 Jul 2017 17:18:34 -0300 Subject: [PATCH 09/10] Introduce PYTEST_CURRENT_TEST environment variable Fix #2583 --- _pytest/runner.py | 19 ++++++++++++++++++ changelog/2583.feature | 2 ++ doc/en/example/simple.rst | 41 +++++++++++++++++++++++++++++++++++++++ testing/test_runner.py | 27 ++++++++++++++++++++++++++ 4 files changed, 89 insertions(+) create mode 100644 changelog/2583.feature diff --git a/_pytest/runner.py b/_pytest/runner.py index fd0b549a9..27be8f4d1 100644 --- a/_pytest/runner.py +++ b/_pytest/runner.py @@ -2,6 +2,7 @@ from __future__ import absolute_import, division, print_function import bdb +import os import sys from time import time @@ -91,9 +92,11 @@ def show_test_item(item): tw.write(' (fixtures used: {0})'.format(', '.join(used_fixtures))) def pytest_runtest_setup(item): + _update_current_test_var(item, 'setup') item.session._setupstate.prepare(item) def pytest_runtest_call(item): + _update_current_test_var(item, 'call') try: item.runtest() except Exception: @@ -107,7 +110,23 @@ def pytest_runtest_call(item): raise def pytest_runtest_teardown(item, nextitem): + _update_current_test_var(item, 'teardown') item.session._setupstate.teardown_exact(item, nextitem) + _update_current_test_var(item, None) + + +def _update_current_test_var(item, when): + """ + Update PYTEST_CURRENT_TEST to reflect the current item and stage. + + If ``when`` is None, delete PYTEST_CURRENT_TEST from the environment. + """ + var_name = 'PYTEST_CURRENT_TEST' + if when: + os.environ[var_name] = '{0} ({1})'.format(item.nodeid, when) + else: + os.environ.pop(var_name) + def pytest_report_teststatus(report): if report.when in ("setup", "teardown"): diff --git a/changelog/2583.feature b/changelog/2583.feature new file mode 100644 index 000000000..315f2378e --- /dev/null +++ b/changelog/2583.feature @@ -0,0 +1,2 @@ +Introduce the ``PYTEST_CURRENT_TEST`` environment variable that is set with the ``nodeid`` and stage (``setup``, ``call`` and +``teardown``) of the test being currently executed. See the `documentation `_ for more info. diff --git a/doc/en/example/simple.rst b/doc/en/example/simple.rst index da831244b..6b5d5a868 100644 --- a/doc/en/example/simple.rst +++ b/doc/en/example/simple.rst @@ -761,6 +761,47 @@ and run it:: You'll see that the fixture finalizers could use the precise reporting information. +``PYTEST_CURRENT_TEST`` environment variable +-------------------------------------------- + +.. versionadded:: 3.2 + +Sometimes a test session might get stuck and there might be no easy way to figure out +which test got stuck, for example if pytest was run in quiet mode (``-q``) or you don't have access to the console +output. This is particularly a problem if the problem helps only sporadically, the famous "flaky" kind of tests. + +``pytest`` sets a ``PYTEST_CURRENT_TEST`` environment variable when running tests, which can be inspected +by process monitoring utilities or libraries like `psutil `_ to discover which +test got stuck if necessary: + +.. code-block:: python + + import psutil + + for pid in psutil.pids(): + environ = psutil.Process(pid).environ() + if 'PYTEST_CURRENT_TEST' in environ: + print(f'pytest process {pid} running: {environ["PYTEST_CURRENT_TEST"]}') + +During the test session pytest will set ``PYTEST_CURRENT_TEST`` to the current test +:ref:`nodeid ` and the current stage, which can be ``setup``, ``call`` +and ``teardown``. + +For example, when running a single test function named ``test_foo`` from ``foo_module.py``, +``PYTEST_CURRENT_TEST`` will be set to: + +#. ``foo_module.py::test_foo (setup)`` +#. ``foo_module.py::test_foo (call)`` +#. ``foo_module.py::test_foo (teardown)`` + +In that order. + +.. note:: + + The contents of ``PYTEST_CURRENT_TEST`` is meant to be human readable and the actual format + can be changed between releases (even bug fixes) so it shouldn't be relied on for scripting + or automation. + Freezing pytest --------------- diff --git a/testing/test_runner.py b/testing/test_runner.py index def80ea5f..e70d955ac 100644 --- a/testing/test_runner.py +++ b/testing/test_runner.py @@ -681,6 +681,8 @@ def test_store_except_info_on_eror(): """ # Simulate item that raises a specific exception class ItemThatRaises(object): + nodeid = 'item_that_raises' + def runtest(self): raise IndexError('TEST') try: @@ -693,6 +695,31 @@ def test_store_except_info_on_eror(): assert sys.last_traceback +def test_current_test_env_var(testdir, monkeypatch): + pytest_current_test_vars = [] + monkeypatch.setattr(sys, 'pytest_current_test_vars', pytest_current_test_vars, raising=False) + testdir.makepyfile(''' + import pytest + import sys + import os + + @pytest.fixture + def fix(): + sys.pytest_current_test_vars.append(('setup', os.environ['PYTEST_CURRENT_TEST'])) + yield + sys.pytest_current_test_vars.append(('teardown', os.environ['PYTEST_CURRENT_TEST'])) + + def test(fix): + sys.pytest_current_test_vars.append(('call', os.environ['PYTEST_CURRENT_TEST'])) + ''') + result = testdir.runpytest_inprocess() + assert result.ret == 0 + test_id = 'test_current_test_env_var.py::test' + assert pytest_current_test_vars == [ + ('setup', test_id + ' (setup)'), ('call', test_id + ' (call)'), ('teardown', test_id + ' (teardown)')] + assert 'PYTEST_CURRENT_TEST' not in os.environ + + class TestReportContents(object): """ Test user-level API of ``TestReport`` objects. From d7f182ac4fcc1a416b456a02b0ed0508c1659d1c Mon Sep 17 00:00:00 2001 From: Bruno Oliveira Date: Wed, 19 Jul 2017 10:02:13 -0300 Subject: [PATCH 10/10] Remove SETUPTOOLS_SCM_PRETEND_VERSION during linting It was needed because of check-manifest, but we no longer have a MANIFEST file so it is not necessary --- tox.ini | 3 --- 1 file changed, 3 deletions(-) diff --git a/tox.ini b/tox.ini index cc9e64b1c..3dd094866 100644 --- a/tox.ini +++ b/tox.ini @@ -49,9 +49,6 @@ commands= skipsdist=True usedevelop=True basepython = python2.7 -# needed to keep check-manifest working -setenv = - SETUPTOOLS_SCM_PRETEND_VERSION=2.0.1 deps = flake8 # pygments required by rst-lint