Merge pull request #3685 from nicoddemus/merge-master-into-features

Merge master into features

src/_pytest/_code/source.py

@@ -1,4 +1,4 @@
-from __future__ import absolute_import, division, generators, print_function
+from __future__ import absolute_import, division, print_function
 
 import ast
 from ast import PyCF_ONLY_AST as _AST_FLAG
@@ -152,12 +152,7 @@ class Source(object):
         return "\n".join(self.lines)
 
     def compile(
-        self,
-        filename=None,
-        mode="exec",
-        flag=generators.compiler_flag,
-        dont_inherit=0,
-        _genframe=None,
+        self, filename=None, mode="exec", flag=0, dont_inherit=0, _genframe=None
     ):
         """ return compiled code object. if filename is None
             invent an artificial filename which displays
@@ -201,9 +196,7 @@ class Source(object):
 #
 
 
-def compile_(
-    source, filename=None, mode="exec", flags=generators.compiler_flag, dont_inherit=0
-):
+def compile_(source, filename=None, mode="exec", flags=0, dont_inherit=0):
     """ compile the given source to a raw code object,
         and maintain an internal cache which allows later
         retrieval of the source code for the code object

src/_pytest/compat.py

@@ -47,8 +47,8 @@ else:
 
 
 if _PY3:
-    from collections.abc import MutableMapping as MappingMixin  # noqa
-    from collections.abc import Mapping, Sequence  # noqa
+    from collections.abc import MutableMapping as MappingMixin
+    from collections.abc import Mapping, Sequence
 else:
     # those raise DeprecationWarnings in Python >=3.7
     from collections import MutableMapping as MappingMixin  # noqa

src/_pytest/config/__init__.py

@@ -71,7 +71,7 @@ def main(args=None, plugins=None):
         return 4
 
 
-class cmdline(object):  # NOQA compatibility namespace
+class cmdline(object):  # compatibility namespace
     main = staticmethod(main)
 
 

src/_pytest/mark/structures.py

@@ -111,7 +111,19 @@ class ParameterSet(namedtuple("ParameterSet", "values, marks, id")):
         ]
         del argvalues
 
-        if not parameters:
+        if parameters:
+            # check all parameter sets have the correct number of values
+            for param in parameters:
+                if len(param.values) != len(argnames):
+                    raise ValueError(
+                        'In "parametrize" the number of values ({}) must be '
+                        "equal to the number of names ({})".format(
+                            param.values, argnames
+                        )
+                    )
+        else:
+            # empty parameter set (likely computed at runtime): create a single
+            # parameter set with NOSET values, with the "empty parameter set" mark applied to it
             mark = get_empty_parameterset_mark(config, argnames, func)
             parameters.append(
                 ParameterSet(values=(NOTSET,) * len(argnames), marks=[mark], id=None)
@@ -124,9 +136,9 @@ class Mark(object):
     #: name of the mark
     name = attr.ib(type=str)
     #: positional arguments of the mark decorator
-    args = attr.ib(type="List[object]")
+    args = attr.ib()  # type: List[object]
     #: keyword arguments of the mark decorator
-    kwargs = attr.ib(type="Dict[str, object]")
+    kwargs = attr.ib()  # type: Dict[str, object]
 
     def combined_with(self, other):
         """

src/_pytest/pytester.py

@@ -23,11 +23,6 @@ from _pytest.main import Session, EXIT_OK
 from _pytest.assertion.rewrite import AssertionRewritingHook
 from _pytest.compat import Path
 
-PYTEST_FULLPATH = os.path.abspath(pytest.__file__.rstrip("oc")).replace(
-    "$py.class", ".py"
-)
-
-
 IGNORE_PAM = [  # filenames added when obtaining details about the current user
     u"/var/lib/sss/mc/passwd"
 ]
@@ -1077,9 +1072,7 @@ class Testdir(object):
             print("couldn't print to %s because of encoding" % (fp,))
 
     def _getpytestargs(self):
-        # we cannot use `(sys.executable, script)` because on Windows the
-        # script is e.g. `pytest.exe`
-        return (sys.executable, PYTEST_FULLPATH)  # noqa
+        return (sys.executable, "-mpytest")
 
     def runpython(self, script):
         """Run a python script using sys.executable as interpreter.

src/_pytest/python.py

@@ -8,7 +8,6 @@ import os
 import collections
 import warnings
 from textwrap import dedent
-from itertools import count
 
 
 import py
@@ -945,46 +944,54 @@ class Metafunc(fixtures.FuncargnamesCompatAttr):
         """
         from _pytest.fixtures import scope2index
         from _pytest.mark import ParameterSet
-        from py.io import saferepr
 
         argnames, parameters = ParameterSet._for_parametrize(
             argnames, argvalues, self.function, self.config
         )
         del argvalues
-        default_arg_names = set(get_default_arg_names(self.function))
 
         if scope is None:
             scope = _find_parametrized_scope(argnames, self._arg2fixturedefs, indirect)
 
-        scopenum = scope2index(scope, descr="call to {}".format(self.parametrize))
-        valtypes = {}
-        for arg in argnames:
-            if arg not in self.fixturenames:
-                if arg in default_arg_names:
-                    raise ValueError(
-                        "%r already takes an argument %r with a default value"
-                        % (self.function, arg)
-                    )
-                else:
-                    if isinstance(indirect, (tuple, list)):
-                        name = "fixture" if arg in indirect else "argument"
-                    else:
-                        name = "fixture" if indirect else "argument"
-                    raise ValueError("%r uses no %s %r" % (self.function, name, arg))
+        self._validate_if_using_arg_names(argnames, indirect)
+
+        arg_values_types = self._resolve_arg_value_types(argnames, indirect)
+
+        ids = self._resolve_arg_ids(argnames, ids, parameters)
+
+        scopenum = scope2index(scope, descr="call to {}".format(self.parametrize))
+
+        # create the new calls: if we are parametrize() multiple times (by applying the decorator
+        # more than once) then we accumulate those calls generating the cartesian product
+        # of all calls
+        newcalls = []
+        for callspec in self._calls or [CallSpec2(self)]:
+            for param_index, (param_id, param_set) in enumerate(zip(ids, parameters)):
+                newcallspec = callspec.copy()
+                newcallspec.setmulti2(
+                    arg_values_types,
+                    argnames,
+                    param_set.values,
+                    param_id,
+                    param_set.marks,
+                    scopenum,
+                    param_index,
+                )
+                newcalls.append(newcallspec)
+        self._calls = newcalls
+
+    def _resolve_arg_ids(self, argnames, ids, parameters):
+        """Resolves the actual ids for the given argnames, based on the ``ids`` parameter given
+        to ``parametrize``.
+
+        :param List[str] argnames: list of argument names passed to ``parametrize()``.
+        :param ids: the ids parameter of the parametrized call (see docs).
+        :param List[ParameterSet] parameters: the list of parameter values, same size as ``argnames``.
+        :rtype: List[str]
+        :return: the list of ids for each argname given
+        """
+        from py.io import saferepr
 
-        if indirect is True:
-            valtypes = dict.fromkeys(argnames, "params")
-        elif indirect is False:
-            valtypes = dict.fromkeys(argnames, "funcargs")
-        elif isinstance(indirect, (tuple, list)):
-            valtypes = dict.fromkeys(argnames, "funcargs")
-            for arg in indirect:
-                if arg not in argnames:
-                    raise ValueError(
-                        "indirect given to %r: fixture %r doesn't exist"
-                        % (self.function, arg)
-                    )
-                valtypes[arg] = "params"
         idfn = None
         if callable(ids):
             idfn = ids
@@ -1001,29 +1008,57 @@ class Metafunc(fixtures.FuncargnamesCompatAttr):
                         msg % (saferepr(id_value), type(id_value).__name__)
                     )
         ids = idmaker(argnames, parameters, idfn, ids, self.config)
-        newcalls = []
-        for callspec in self._calls or [CallSpec2(self)]:
-            elements = zip(ids, parameters, count())
-            for a_id, param, param_index in elements:
-                if len(param.values) != len(argnames):
+        return ids
+
+    def _resolve_arg_value_types(self, argnames, indirect):
+        """Resolves if each parametrized argument must be considered a parameter to a fixture or a "funcarg"
+        to the function, based on the ``indirect`` parameter of the parametrized() call.
+
+        :param List[str] argnames: list of argument names passed to ``parametrize()``.
+        :param indirect: same ``indirect`` parameter of ``parametrize()``.
+        :rtype: Dict[str, str]
+            A dict mapping each arg name to either:
+            * "params" if the argname should be the parameter of a fixture of the same name.
+            * "funcargs" if the argname should be a parameter to the parametrized test function.
+        """
+        valtypes = {}
+        if indirect is True:
+            valtypes = dict.fromkeys(argnames, "params")
+        elif indirect is False:
+            valtypes = dict.fromkeys(argnames, "funcargs")
+        elif isinstance(indirect, (tuple, list)):
+            valtypes = dict.fromkeys(argnames, "funcargs")
+            for arg in indirect:
+                if arg not in argnames:
                     raise ValueError(
-                        'In "parametrize" the number of values ({}) must be '
-                        "equal to the number of names ({})".format(
-                            param.values, argnames
-                        )
+                        "indirect given to %r: fixture %r doesn't exist"
+                        % (self.function, arg)
                     )
-                newcallspec = callspec.copy()
-                newcallspec.setmulti2(
-                    valtypes,
-                    argnames,
-                    param.values,
-                    a_id,
-                    param.marks,
-                    scopenum,
-                    param_index,
-                )
-                newcalls.append(newcallspec)
-        self._calls = newcalls
+                valtypes[arg] = "params"
+        return valtypes
+
+    def _validate_if_using_arg_names(self, argnames, indirect):
+        """
+        Check if all argnames are being used, by default values, or directly/indirectly.
+
+        :param List[str] argnames: list of argument names passed to ``parametrize()``.
+        :param indirect: same ``indirect`` parameter of ``parametrize()``.
+        :raise ValueError: if validation fails.
+        """
+        default_arg_names = set(get_default_arg_names(self.function))
+        for arg in argnames:
+            if arg not in self.fixturenames:
+                if arg in default_arg_names:
+                    raise ValueError(
+                        "%r already takes an argument %r with a default value"
+                        % (self.function, arg)
+                    )
+                else:
+                    if isinstance(indirect, (tuple, list)):
+                        name = "fixture" if arg in indirect else "argument"
+                    else:
+                        name = "fixture" if indirect else "argument"
+                    raise ValueError("%r uses no %s %r" % (self.function, name, arg))
 
     def addcall(self, funcargs=None, id=NOTSET, param=NOTSET):
         """ Add a new call to the underlying test function during the collection phase of a test run.