diff --git a/doc/en/how-to/logging.rst b/doc/en/how-to/logging.rst index 300e9f6e6..2f75e57c1 100644 --- a/doc/en/how-to/logging.rst +++ b/doc/en/how-to/logging.rst @@ -208,6 +208,7 @@ option names are: If you need to record the whole test suite logging calls to a file, you can pass ``--log-file=/path/to/log/file``. This log file is opened in write mode by default which means that it will be overwritten at each run tests session. +You can specify the level of verbosity of the log file by passing ```--log-file-verbose=1`` If you'd like the file opened in append mode instead, then you can pass ``--log-file-mode=a``. Note that relative paths for the log-file location, whether passed on the CLI or declared in a config file, are always resolved relative to the current working directory. diff --git a/doc/en/reference/reference.rst b/doc/en/reference/reference.rst index 3675c7cb2..1e16f6c1b 100644 --- a/doc/en/reference/reference.rst +++ b/doc/en/reference/reference.rst @@ -2121,6 +2121,7 @@ All the command-line flags can be obtained by running ``pytest --help``:: --log-cli-date-format=LOG_CLI_DATE_FORMAT Log date format used by the logging module --log-file=LOG_FILE Path to a file when logging will be written to + --log-file-verbose={0,1} Log file verbose --log-file-mode={w,a} Log file open mode --log-file-level=LOG_FILE_LEVEL @@ -2221,6 +2222,8 @@ All the command-line flags can be obtained by running ``pytest --help``:: log_cli_date_format (string): Default value for --log-cli-date-format log_file (string): Default value for --log-file + log_file_verbose (int): + Default value for --log-file-verbose log_file_mode (string): Default value for --log-file-mode log_file_level (string): diff --git a/src/.DS_Store b/src/.DS_Store new file mode 100644 index 000000000..112d642aa Binary files /dev/null and b/src/.DS_Store differ diff --git a/src/_pytest/config/__init__.py b/src/_pytest/config/__init__.py index 058aaa1ff..bfc366baf 100644 --- a/src/_pytest/config/__init__.py +++ b/src/_pytest/config/__init__.py @@ -1574,6 +1574,7 @@ class Config: ``paths``, ``pathlist``, ``args`` and ``linelist`` : empty list ``[]`` ``bool`` : ``False`` ``string`` : empty string ``""`` + ``int`` : ``0`` If neither the ``default`` nor the ``type`` parameter is passed while registering the configuration through @@ -1641,6 +1642,8 @@ class Config: return value elif type == "bool": return _strtobool(str(value).strip()) + elif type == "int": + return int(str(value).strip()) elif type == "string": return value elif type is None: diff --git a/src/_pytest/config/argparsing.py b/src/_pytest/config/argparsing.py index f270b864c..4f4ef03fa 100644 --- a/src/_pytest/config/argparsing.py +++ b/src/_pytest/config/argparsing.py @@ -180,7 +180,7 @@ class Parser: name: str, help: str, type: Optional[ - Literal["string", "paths", "pathlist", "args", "linelist", "bool"] + Literal["string", "paths", "pathlist", "args", "linelist", "bool", "int"] ] = None, default: Any = NOT_SET, ) -> None: @@ -190,7 +190,7 @@ class Parser: Name of the ini-variable. :param type: Type of the variable. Can be: - + * ``int``: an integer * ``string``: a string * ``bool``: a boolean * ``args``: a list of strings, separated as in a shell @@ -215,7 +215,16 @@ class Parser: The value of ini-variables can be retrieved via a call to :py:func:`config.getini(name) `. """ - assert type in (None, "string", "paths", "pathlist", "args", "linelist", "bool") + assert type in ( + None, + "string", + "paths", + "pathlist", + "args", + "linelist", + "bool", + "int", + ) if default is NOT_SET: default = get_ini_default_for_type(type) @@ -224,7 +233,9 @@ class Parser: def get_ini_default_for_type( - type: Optional[Literal["string", "paths", "pathlist", "args", "linelist", "bool"]], + type: Optional[ + Literal["string", "paths", "pathlist", "args", "linelist", "bool", "int"] + ], ) -> Any: """ Used by addini to get the default value for a given ini-option type, when @@ -236,6 +247,8 @@ def get_ini_default_for_type( return [] elif type == "bool": return False + elif type == "int": + return 0 else: return "" diff --git a/src/_pytest/logging.py b/src/_pytest/logging.py index c9139d369..cf16fd6da 100644 --- a/src/_pytest/logging.py +++ b/src/_pytest/logging.py @@ -299,6 +299,13 @@ def pytest_addoption(parser: Parser) -> None: default=None, help="Path to a file when logging will be written to", ) + add_option_ini( + "--log-file-verbose", + dest="log_file_verbose", + default=0, + type="int", + help="Log file verbose", + ) add_option_ini( "--log-file-mode", dest="log_file_mode", @@ -677,6 +684,8 @@ class LoggingPlugin: if not os.path.isdir(directory): os.makedirs(directory) + self.log_file_verbose = get_option_ini(config, "log_file_verbose") + self.log_file_mode = get_option_ini(config, "log_file_mode") or "w" self.log_file_handler = _FileHandler( log_file, mode=self.log_file_mode, encoding="UTF-8" @@ -837,6 +846,22 @@ class LoggingPlugin: @hookimpl(wrapper=True) def pytest_runtest_setup(self, item: nodes.Item) -> Generator[None, None, None]: + if self.log_file_verbose and int(self.log_file_verbose) >= 1: + old_log_file_formatter = self.log_file_handler.formatter + self.log_file_handler.setFormatter(logging.Formatter()) + self.log_file_handler.emit( + logging.LogRecord( + "N/A", + logging.INFO, + "N/A", + 0, + f"Running at {item.nodeid}", + None, + None, + ) + ) + self.log_file_handler.setFormatter(old_log_file_formatter) + self.log_cli_handler.set_when("setup") empty: Dict[str, List[logging.LogRecord]] = {} diff --git a/src/_pytest/mark/structures.py b/src/_pytest/mark/structures.py index 456808063..0110233fb 100644 --- a/src/_pytest/mark/structures.py +++ b/src/_pytest/mark/structures.py @@ -166,22 +166,20 @@ class ParameterSet(NamedTuple): # Check all parameter sets have the correct number of values. for param in parameters: if len(param.values) != len(argnames): + # Construct a string representation of the expected parameter names + expected_argnames = ", ".join(argnames) + # Construct a string representation of the provided parameter values + provided_values = ", ".join(map(repr, param.values)) msg = ( - '{nodeid}: in "parametrize" the number of names ({names_len}):\n' - " {names}\n" - "must be equal to the number of values ({values_len}):\n" - " {values}" - ) - fail( - msg.format( - nodeid=nodeid, - values=param.values, - names=argnames, - names_len=len(argnames), - values_len=len(param.values), - ), - pytrace=False, + f'{nodeid}: \n\n Error in parameterization for test "{func.__name__}".\n ' + f"The number of specified parameters ({len(argnames)}) " + f"does not match the number of provided values ({len(param.values)}): {provided_values}. \n" + f" Please ensure that the correct number of parameter names " + f"({len(param.values)}) are separated by commas within quotes.\n\n" + f'Require more than parameter names: "{expected_argnames}"' ) + + fail(msg, pytrace=False) else: # Empty parameter set (likely computed at runtime): create a single # parameter set with NOTSET values, with the "empty parameter set" mark applied to it. diff --git a/src/_pytest/python.py b/src/_pytest/python.py index 4887614de..bd89f2389 100644 --- a/src/_pytest/python.py +++ b/src/_pytest/python.py @@ -1380,8 +1380,15 @@ class Metafunc: # num_ids == 0 is a special case: https://github.com/pytest-dev/pytest/issues/1849 if num_ids != len(parametersets) and num_ids != 0: - msg = "In {}: {} parameter sets specified, with different number of ids: {}" - fail(msg.format(func_name, len(parametersets), num_ids), pytrace=False) + # Construct a string representation of the expected parameter sets + expected_paramsets = len(parametersets) + + msg = ( + f"In {func_name}: \n\n" + f" Specified {expected_paramsets} parameter sets with {num_ids} different ids. \n" + f" Ensure all subparameters are correctly grouped within a single set of quotation marks." + ) + fail(msg, pytrace=False) return list(itertools.islice(ids, num_ids)) @@ -1411,8 +1418,24 @@ class Metafunc: arg_directness = dict.fromkeys(argnames, "direct") for arg in indirect: if arg not in argnames: + # Construct a list of valid parameter names + valid_params = ", ".join([f'"{name}"' for name in argnames]) + + # Construct a string representing the expected number of parameters + expected_param_count = len(indirect[0]) + + # Construct a string representing the actual number of parameters provided + actual_param_count = len(argnames) + fail( - f"In {self.function.__name__}: indirect fixture '{arg}' doesn't exist", + f"In function {self.function.__name__}: {argnames} is not a valid parameter. \n" + f"Expected {expected_param_count} sub parameters, " + f"but only {actual_param_count} were provided. \n\n" + f"Make sure to pass parameter names as strings without quotes, separated by commas, \n " + f"e.g., '@pytest.mark.parametrize({valid_params}, )'" + f"\n\n" + f"Or if multiple parameters are used, separate them by commas. \n " + f"e.g., '@pytest.mark.parametrize(\"arg1, arg2\", )'", pytrace=False, ) arg_directness[arg] = "indirect" diff --git a/testing/local_testing/add_function.py b/testing/local_testing/add_function.py new file mode 100644 index 000000000..e1829c359 --- /dev/null +++ b/testing/local_testing/add_function.py @@ -0,0 +1,2 @@ +def add(a: int, b: int) -> int: + return a + b diff --git a/testing/local_testing/local_test_0.py b/testing/local_testing/local_test_0.py new file mode 100644 index 000000000..fbc477f15 --- /dev/null +++ b/testing/local_testing/local_test_0.py @@ -0,0 +1,16 @@ +import pytest + + +@pytest.mark.parametrize("arg1, arg2", [(1, 1)]) +def test_parametrization(arg1: int, arg2: int) -> None: + assert arg1 == arg2 + assert arg1 + 1 == arg2 + 1 + + +# Gets the error: In test_parametrization: indirect fixture '(1, 1)' doesn't exist +# Goal is to change this message into a more beginner friendly message. + +# The error message lives in this path: pytest/src/_pytest/python.py + + +# ("arg1", "arg2"), and "arg1, arg2" works, but cannot put in default parameters as normal function diff --git a/testing/local_testing/local_test_test.py b/testing/local_testing/local_test_test.py new file mode 100644 index 000000000..968eff81e --- /dev/null +++ b/testing/local_testing/local_test_test.py @@ -0,0 +1,13 @@ +# test_math_operations.py + +import add_function + +import pytest + + +result_index = [(1, 2, 3), (-1, 1, 0), (0, 0, 0), (5, -3, 2), (1, 1, 2)] + + +@pytest.mark.parametrize("a, b, expected_result", result_index) +def test_add(a: int, b: int, expected_result: int) -> None: + assert add_function.add(a, b) == expected_result diff --git a/testing/test_log_file.py b/testing/test_log_file.py new file mode 100644 index 000000000..723203e5b --- /dev/null +++ b/testing/test_log_file.py @@ -0,0 +1,76 @@ +import os + +from _pytest.pytester import Pytester + + +def test_log_file_verbose(pytester: Pytester) -> None: + log_file = str(pytester.path.joinpath("pytest.log")) + + pytester.makepyfile( + """ + import logging + def test_verbose1(): + logging.info("test 1") + + def test_verbose2(): + logging.warning("test 2") + """ + ) + + result = pytester.runpytest( + "-s", f"--log-file={log_file}", "--log-file-verbose=1", "--log-file-level=INFO" + ) + + rec = pytester.inline_run() + rec.assertoutcome(passed=2) + + # make sure that we get a '0' exit code for the testsuite + assert result.ret == 0 + assert os.path.isfile(log_file) + with open(log_file, encoding="utf-8") as rfh: + contents = rfh.read() + + for s in [ + "Running at test_log_file_verbose.py::test_verbose1", + "test 1", + "Running at test_log_file_verbose.py::test_verbose2", + "test 2", + ]: + assert s in contents + + +def test_log_file_verbose0(pytester: Pytester) -> None: + log_file = str(pytester.path.joinpath("pytest.log")) + + pytester.makepyfile( + """ + import logging + def test_verbose1(): + logging.info("test 1") + + def test_verbose2(): + logging.warning("test 2") + """ + ) + + result = pytester.runpytest( + "-s", f"--log-file={log_file}", "--log-file-verbose=0", "--log-file-level=INFO" + ) + + rec = pytester.inline_run() + rec.assertoutcome(passed=2) + + # make sure that we get a '0' exit code for the testsuite + assert result.ret == 0 + assert os.path.isfile(log_file) + with open(log_file, encoding="utf-8") as rfh: + contents = rfh.read() + + for s in ["test 1", "test 2"]: + assert s in contents + + for s in [ + "Running at test_log_file_verbose0.py::test_verbose1", + "Running at test_log_file_verbose.py::test_verbose2", + ]: + assert s not in contents diff --git a/testing/test_mark.py b/testing/test_mark.py index 2896afa45..f9eee67cc 100644 --- a/testing/test_mark.py +++ b/testing/test_mark.py @@ -404,6 +404,35 @@ def test_parametrized_collected_from_command_line(pytester: Pytester) -> None: rec.assertoutcome(passed=3) +def test_parametrized_collect_with_wrong_format(pytester: Pytester) -> None: + """Parametrized test argument format not intuitive + line issue#8593""" + py_file = pytester.makepyfile( + """ + import pytest + + @pytest.mark.parametrize("arg1", "arg2", [(1, 1)]) + def test_parametrization(arg1: int, arg2: int) -> None: + assert arg1 == arg2 + assert arg1 + 1 == arg2 + 1 + """ + ) + + result = pytester.runpytest(py_file) + result.stdout.fnmatch_lines( + [ + "In function test_parametrization: ['arg1'] is not a valid parameter. ", + "Expected 2 sub parameters, but only 1 were provided. ", + "", + "Make sure to pass parameter names as strings without quotes, separated by commas, ", + " e.g., '@pytest.mark.parametrize(\"arg1\", )'", + "", + "Or if multiple parameters are used, separate them by commas. ", + " e.g., '@pytest.mark.parametrize(\"arg1, arg2\", )'", + ] + ) + + def test_parametrized_collect_with_wrong_args(pytester: Pytester) -> None: """Test collect parametrized func with wrong number of args.""" py_file = pytester.makepyfile( @@ -419,10 +448,13 @@ def test_parametrized_collect_with_wrong_args(pytester: Pytester) -> None: result = pytester.runpytest(py_file) result.stdout.fnmatch_lines( [ - 'test_parametrized_collect_with_wrong_args.py::test_func: in "parametrize" the number of names (2):', - " ['foo', 'bar']", - "must be equal to the number of values (3):", - " (1, 2, 3)", + "test_parametrized_collect_with_wrong_args.py::test_func: ", + "", + ' Error in parameterization for test "test_func".', + " The number of specified parameters (2) does not match the number of provided values (3): 1, 2, 3. ", + " Please ensure that the correct number of parameter names (3) are separated by commas within quotes.", + "", + 'Require more than parameter names: "foo, bar"', ] )