diff --git a/_pytest/main.py b/_pytest/main.py index a4cc21a57..274b39782 100644 --- a/_pytest/main.py +++ b/_pytest/main.py @@ -72,6 +72,9 @@ def pytest_addoption(parser): group.addoption('--keepduplicates', '--keep-duplicates', action="store_true", dest="keepduplicates", default=False, help="Keep duplicate tests.") + group.addoption('--collect-in-virtualenv', action='store_true', + dest='collect_in_virtualenv', default=False, + help="Don't ignore tests in a local virtualenv directory") group = parser.getgroup("debugconfig", "test session debugging and configuration") @@ -168,6 +171,17 @@ def pytest_runtestloop(session): return True +def _in_venv(path): + """Attempts to detect if ``path`` is the root of a Virtual Environment by + checking for the existence of the appropriate activate script""" + bindir = path.join('Scripts' if sys.platform.startswith('win') else 'bin') + if not bindir.exists(): + return False + activates = ('activate', 'activate.csh', 'activate.fish', + 'Activate', 'Activate.bat', 'Activate.ps1') + return any([fname.basename in activates for fname in bindir.listdir()]) + + def pytest_ignore_collect(path, config): ignore_paths = config._getconftest_pathlist("collect_ignore", path=path.dirpath()) ignore_paths = ignore_paths or [] @@ -178,6 +192,10 @@ def pytest_ignore_collect(path, config): if py.path.local(path) in ignore_paths: return True + allow_in_venv = config.getoption("collect_in_virtualenv") + if _in_venv(path) and not allow_in_venv: + return True + # Skip duplicate paths. keepduplicates = config.getoption("keepduplicates") duplicate_paths = config.pluginmanager._duplicatepaths diff --git a/_pytest/mark.py b/_pytest/mark.py index c2959606c..5ded0afe8 100644 --- a/_pytest/mark.py +++ b/_pytest/mark.py @@ -6,7 +6,7 @@ import warnings from collections import namedtuple from operator import attrgetter from .compat import imap -from .deprecated import MARK_INFO_ATTRIBUTE, MARK_PARAMETERSET_UNPACKING +from .deprecated import MARK_PARAMETERSET_UNPACKING def alias(name, warning=None): @@ -407,9 +407,9 @@ class MarkInfo(object): self.combined = mark self._marks = [mark] - name = alias('combined.name', warning=MARK_INFO_ATTRIBUTE) - args = alias('combined.args', warning=MARK_INFO_ATTRIBUTE) - kwargs = alias('combined.kwargs', warning=MARK_INFO_ATTRIBUTE) + name = alias('combined.name') + args = alias('combined.args') + kwargs = alias('combined.kwargs') def __repr__(self): return "".format(self.combined) diff --git a/_pytest/runner.py b/_pytest/runner.py index d9d266361..b5829f46d 100644 --- a/_pytest/runner.py +++ b/_pytest/runner.py @@ -2,6 +2,7 @@ from __future__ import absolute_import, division, print_function import bdb +import os import sys from time import time @@ -99,10 +100,12 @@ def show_test_item(item): def pytest_runtest_setup(item): + _update_current_test_var(item, 'setup') item.session._setupstate.prepare(item) def pytest_runtest_call(item): + _update_current_test_var(item, 'call') try: item.runtest() except Exception: @@ -117,7 +120,22 @@ def pytest_runtest_call(item): def pytest_runtest_teardown(item, nextitem): + _update_current_test_var(item, 'teardown') item.session._setupstate.teardown_exact(item, nextitem) + _update_current_test_var(item, None) + + +def _update_current_test_var(item, when): + """ + Update PYTEST_CURRENT_TEST to reflect the current item and stage. + + If ``when`` is None, delete PYTEST_CURRENT_TEST from the environment. + """ + var_name = 'PYTEST_CURRENT_TEST' + if when: + os.environ[var_name] = '{0} ({1})'.format(item.nodeid, when) + else: + os.environ.pop(var_name) def pytest_report_teststatus(report): diff --git a/changelog/2518.feature b/changelog/2518.feature new file mode 100644 index 000000000..2f6597a97 --- /dev/null +++ b/changelog/2518.feature @@ -0,0 +1 @@ +Collection ignores local virtualenvs by default; `--collect-in-virtualenv` overrides this behavior. diff --git a/changelog/2583.feature b/changelog/2583.feature new file mode 100644 index 000000000..315f2378e --- /dev/null +++ b/changelog/2583.feature @@ -0,0 +1,2 @@ +Introduce the ``PYTEST_CURRENT_TEST`` environment variable that is set with the ``nodeid`` and stage (``setup``, ``call`` and +``teardown``) of the test being currently executed. See the `documentation `_ for more info. diff --git a/doc/en/customize.rst b/doc/en/customize.rst index b0c48f0e3..7be7ca2e5 100644 --- a/doc/en/customize.rst +++ b/doc/en/customize.rst @@ -112,15 +112,27 @@ progress output, you can write it into a configuration file: # content of pytest.ini # (or tox.ini or setup.cfg) [pytest] - addopts = -rsxX -q + addopts = -ra -q -Alternatively, you can set a PYTEST_ADDOPTS environment variable to add command +Alternatively, you can set a ``PYTEST_ADDOPTS`` environment variable to add command line options while the environment is in use:: - export PYTEST_ADDOPTS="-rsxX -q" + export PYTEST_ADDOPTS="-v" -From now on, running ``pytest`` will add the specified options. +Here's how the command-line is built in the presence of ``addopts`` or the environment variable:: + $PYTEST_ADDOTPS + +So if the user executes in the command-line:: + + pytest -m slow + +The actual command line executed is:: + + pytest -ra -q -v -m slow + +Note that as usual for other command-line applications, in case of conflicting options the last one wins, so the example +above will show verbose output because ``-v`` overwrites ``-q``. Builtin configuration file options @@ -171,7 +183,16 @@ Builtin configuration file options norecursedirs = .svn _build tmp* This would tell ``pytest`` to not look into typical subversion or - sphinx-build directories or into any ``tmp`` prefixed directory. + sphinx-build directories or into any ``tmp`` prefixed directory. + + Additionally, ``pytest`` will attempt to intelligently identify and ignore a + virtualenv by the presence of an activation script. Any directory deemed to + be the root of a virtual environment will not be considered during test + collection unless ``‑‑collect‑in‑virtualenv`` is given. Note also that + ``norecursedirs`` takes precedence over ``‑‑collect‑in‑virtualenv``; e.g. if + you intend to run tests in a virtualenv with a base directory that matches + ``'.*'`` you *must* override ``norecursedirs`` in addition to using the + ``‑‑collect‑in‑virtualenv`` flag. .. confval:: testpaths diff --git a/doc/en/example/simple.rst b/doc/en/example/simple.rst index da831244b..6b5d5a868 100644 --- a/doc/en/example/simple.rst +++ b/doc/en/example/simple.rst @@ -761,6 +761,47 @@ and run it:: You'll see that the fixture finalizers could use the precise reporting information. +``PYTEST_CURRENT_TEST`` environment variable +-------------------------------------------- + +.. versionadded:: 3.2 + +Sometimes a test session might get stuck and there might be no easy way to figure out +which test got stuck, for example if pytest was run in quiet mode (``-q``) or you don't have access to the console +output. This is particularly a problem if the problem helps only sporadically, the famous "flaky" kind of tests. + +``pytest`` sets a ``PYTEST_CURRENT_TEST`` environment variable when running tests, which can be inspected +by process monitoring utilities or libraries like `psutil `_ to discover which +test got stuck if necessary: + +.. code-block:: python + + import psutil + + for pid in psutil.pids(): + environ = psutil.Process(pid).environ() + if 'PYTEST_CURRENT_TEST' in environ: + print(f'pytest process {pid} running: {environ["PYTEST_CURRENT_TEST"]}') + +During the test session pytest will set ``PYTEST_CURRENT_TEST`` to the current test +:ref:`nodeid ` and the current stage, which can be ``setup``, ``call`` +and ``teardown``. + +For example, when running a single test function named ``test_foo`` from ``foo_module.py``, +``PYTEST_CURRENT_TEST`` will be set to: + +#. ``foo_module.py::test_foo (setup)`` +#. ``foo_module.py::test_foo (call)`` +#. ``foo_module.py::test_foo (teardown)`` + +In that order. + +.. note:: + + The contents of ``PYTEST_CURRENT_TEST`` is meant to be human readable and the actual format + can be changed between releases (even bug fixes) so it shouldn't be relied on for scripting + or automation. + Freezing pytest --------------- diff --git a/doc/en/usage.rst b/doc/en/usage.rst index 763328f5a..64c072886 100644 --- a/doc/en/usage.rst +++ b/doc/en/usage.rst @@ -52,23 +52,64 @@ To stop the testing process after the first (N) failures:: Specifying tests / selecting tests --------------------------------------------------- -Several test run options:: +Pytest supports several ways to run and select tests from the command-line. - pytest test_mod.py # run tests in module - pytest somepath # run all tests below somepath - pytest -k stringexpr # only run tests with names that match the - # "string expression", e.g. "MyClass and not method" - # will select TestMyClass.test_something - # but not TestMyClass.test_method_simple - pytest test_mod.py::test_func # only run tests that match the "node ID", - # e.g. "test_mod.py::test_func" will select - # only test_func in test_mod.py - pytest test_mod.py::TestClass::test_method # run a single method in - # a single class +**Run tests in a module** -Import 'pkg' and use its filesystem location to find and run tests:: +:: - pytest --pyargs pkg # run all tests found below directory of pkg + pytest test_mod.py + +**Run tests in a directory** + +:: + + pytest testing/ + +**Run tests by keyword expressions** + +:: + + pytest -k "MyClass and not method" + +This will run tests which contain names that match the given *string expression*, which can +include Python operators that use filenames, class names and function names as variables. +The example above will run ``TestMyClass.test_something`` but not ``TestMyClass.test_method_simple``. + +.. _nodeids: + +**Run tests by node ids** + +Each collected test is assigned a unique ``nodeid`` which consist of the module filename followed +by specifiers like class names, function names and parameters from parametrization, separated by ``::`` characters. + +To run a specific test within a module:: + + pytest test_mod.py::test_func + + +Another example specifying a test method in the command line:: + + pytest test_mod.py::TestClass::test_method + +**Run tests by marker expressions** + +:: + + pytest -m slow + +Will run all tests which are decorated with the ``@pytest.mark.slow`` decorator. + +For more information see :ref:`marks `. + +**Run tests from packages** + +:: + + pytest --pyargs pkg.testing + +This will import ``pkg.testing`` and use its filesystem location to find and run tests from. + Modifying Python traceback printing ---------------------------------------------- diff --git a/testing/test_collection.py b/testing/test_collection.py index a0b481628..5d1654410 100644 --- a/testing/test_collection.py +++ b/testing/test_collection.py @@ -2,7 +2,7 @@ from __future__ import absolute_import, division, print_function import pytest import py -from _pytest.main import Session, EXIT_NOTESTSCOLLECTED +from _pytest.main import Session, EXIT_NOTESTSCOLLECTED, _in_venv class TestCollector(object): @@ -123,6 +123,53 @@ class TestCollectFS(object): assert "test_notfound" not in s assert "test_found" in s + @pytest.mark.parametrize('fname', + ("activate", "activate.csh", "activate.fish", + "Activate", "Activate.bat", "Activate.ps1")) + def test_ignored_virtualenvs(self, testdir, fname): + bindir = "Scripts" if py.std.sys.platform.startswith("win") else "bin" + testdir.tmpdir.ensure("virtual", bindir, fname) + testfile = testdir.tmpdir.ensure("virtual", "test_invenv.py") + testfile.write("def test_hello(): pass") + + # by default, ignore tests inside a virtualenv + result = testdir.runpytest() + assert "test_invenv" not in result.stdout.str() + # allow test collection if user insists + result = testdir.runpytest("--collect-in-virtualenv") + assert "test_invenv" in result.stdout.str() + # allow test collection if user directly passes in the directory + result = testdir.runpytest("virtual") + assert "test_invenv" in result.stdout.str() + + @pytest.mark.parametrize('fname', + ("activate", "activate.csh", "activate.fish", + "Activate", "Activate.bat", "Activate.ps1")) + def test_ignored_virtualenvs_norecursedirs_precedence(self, testdir, fname): + bindir = "Scripts" if py.std.sys.platform.startswith("win") else "bin" + # norecursedirs takes priority + testdir.tmpdir.ensure(".virtual", bindir, fname) + testfile = testdir.tmpdir.ensure(".virtual", "test_invenv.py") + testfile.write("def test_hello(): pass") + result = testdir.runpytest("--collect-in-virtualenv") + assert "test_invenv" not in result.stdout.str() + # ...unless the virtualenv is explicitly given on the CLI + result = testdir.runpytest("--collect-in-virtualenv", ".virtual") + assert "test_invenv" in result.stdout.str() + + @pytest.mark.parametrize('fname', + ("activate", "activate.csh", "activate.fish", + "Activate", "Activate.bat", "Activate.ps1")) + def test__in_venv(self, testdir, fname): + """Directly test the virtual env detection function""" + bindir = "Scripts" if py.std.sys.platform.startswith("win") else "bin" + # no bin/activate, not a virtualenv + base_path = testdir.tmpdir.mkdir('venv') + assert _in_venv(base_path) is False + # with bin/activate, totally a virtualenv + base_path.ensure(bindir, fname) + assert _in_venv(base_path) is True + def test_custom_norecursedirs(self, testdir): testdir.makeini(""" [pytest] diff --git a/testing/test_runner.py b/testing/test_runner.py index 567b98eeb..ae081b4f0 100644 --- a/testing/test_runner.py +++ b/testing/test_runner.py @@ -701,6 +701,8 @@ def test_store_except_info_on_eror(): """ # Simulate item that raises a specific exception class ItemThatRaises(object): + nodeid = 'item_that_raises' + def runtest(self): raise IndexError('TEST') try: @@ -713,6 +715,31 @@ def test_store_except_info_on_eror(): assert sys.last_traceback +def test_current_test_env_var(testdir, monkeypatch): + pytest_current_test_vars = [] + monkeypatch.setattr(sys, 'pytest_current_test_vars', pytest_current_test_vars, raising=False) + testdir.makepyfile(''' + import pytest + import sys + import os + + @pytest.fixture + def fix(): + sys.pytest_current_test_vars.append(('setup', os.environ['PYTEST_CURRENT_TEST'])) + yield + sys.pytest_current_test_vars.append(('teardown', os.environ['PYTEST_CURRENT_TEST'])) + + def test(fix): + sys.pytest_current_test_vars.append(('call', os.environ['PYTEST_CURRENT_TEST'])) + ''') + result = testdir.runpytest_inprocess() + assert result.ret == 0 + test_id = 'test_current_test_env_var.py::test' + assert pytest_current_test_vars == [ + ('setup', test_id + ' (setup)'), ('call', test_id + ' (call)'), ('teardown', test_id + ' (teardown)')] + assert 'PYTEST_CURRENT_TEST' not in os.environ + + class TestReportContents(object): """ Test user-level API of ``TestReport`` objects. diff --git a/tox.ini b/tox.ini index fd5132281..1217f4032 100644 --- a/tox.ini +++ b/tox.ini @@ -50,9 +50,6 @@ commands = skipsdist = True usedevelop = True basepython = python2.7 -# needed to keep check-manifest working -setenv = - SETUPTOOLS_SCM_PRETEND_VERSION=2.0.1 deps = flake8 # pygments required by rst-lint