diff --git a/.github/workflows/downstream_testing.yml b/.github/workflows/downstream_testing.yml new file mode 100644 index 000000000..30640b216 --- /dev/null +++ b/.github/workflows/downstream_testing.yml @@ -0,0 +1,110 @@ +name: downstream_testing_2 + +on: + workflow_dispatch: + pull_request: + branches: + # Only run downstream tests on PRs for releases + - "[0-9]+.[0-9]+.x" + push: + branches: + - downstream_testing_2 + +jobs: + downstream-ubuntu: + runs-on: ubuntu-latest + timeout-minutes: 60 + + strategy: + fail-fast: false + matrix: + include: + - name: "pytest-django" + repo: "pytest-dev/pytest-django" + docker_profile: "postgres" + jobs: "test" + workflow_name: "main.yml" + matrix_exclude: | + linting,docs py39-dj40-mysql_innodb-coverage py38-dj32-sqlite-xdist-coverage + py38-dj40-sqlite-xdist-coverage py39-djmain-sqlite-coverage py36-dj32-mysql_myisam-coverage + - name: "pytest-django" + repo: "pytest-dev/pytest-django" + docker_profile: "mysql" + jobs: "test" + workflow_name: "main.yml" + matrix_exclude: | + linting,docs py310-dj40-postgres-xdist-coverage py310-dj32-postgres-xdist-coverage + py39-dj32-postgres-xdist-coverage py38-dj32-sqlite-xdist-coverage + py38-dj40-sqlite-xdist-coverage py39-djmain-sqlite-coverage pypy3-dj32-postgres + py36-dj32-mysql_myisam-coverage + - name: "pytest-django" + repo: "pytest-dev/pytest-django" + docker_profile: "nodb" + jobs: "test" + workflow_name: "main.yml" + matrix_exclude: | + linting,docs py310-dj40-postgres-xdist-coverage py310-dj32-postgres-xdist-coverage + py39-dj32-postgres-xdist-coverage py39-dj40-mysql_innodb-coverage + py36-dj32-mysql_myisam-coverage pypy3-dj32-postgres + + - name: "pytest-html" + repo: "pytest-dev/pytest-html" + docker_profile: "nodb" + jobs: "test_python" + workflow_name: "tests.yml" + matrix_exclude: py36-ubuntu + + - name: "pytest-order" + repo: "pytest-dev/pytest-order" + docker_profile: "nodb" + jobs: "test" + workflow_name: "pythontests.yml" + matrix_exclude: 3.6 + + - name: "pytest-mock" + repo: "pytest-dev/pytest-mock" + docker_profile: "nodb" + jobs: "test" + workflow_name: "test.yml" + matrix_exclude: "" + + - name: "pytest-cov" + repo: "pytest-dev/pytest-cov" + docker_profile: "nodb" + jobs: "test" + workflow_name: "test.yml" + # Exclude pypy3 for now; stil working on a suitable regex substitution + matrix_exclude: | + 3.6 pypy-3.6 pypy-3.7 + + - name: "pytest-bdd" + repo: "pytest-dev/pytest-bdd" + docker_profile: "nodb" + jobs: "build" + workflow_name: "main.yml" + matrix_exclude: "" + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Checkout ${{ matrix.name }} + uses: actions/checkout@v2 + with: + fetch-depth: 0 + repository: ${{ matrix.repo }} + path: ${{ matrix.name }} + - name: Run Downstream Tests - ${{ matrix.name }} + run: docker-compose -f ./testing/downstream_testing/docker-compose.yml --profile ${{ matrix.docker_profile }} up --exit-code-from base_${{ matrix.docker_profile }} + env: + DS_NAME: ${{ matrix.name }} + DS_YAML: ./${{ matrix.name }}/.github/workflows/${{ matrix.workflow_name }} + DS_JOBS: ${{ matrix.jobs }} + DS_MATRIX_EXCLUDE: ${{ matrix.matrix_exclude }} + DOCKER_BUILDKIT: 1 + COV_CMD: "" + - name: Test Results - ${{ matrix.name }} + if: ${{ always() }} + run: | + docker logs base -t &>docker.log + sort -b -k 1 docker.log diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 694498b08..eaa25508c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -68,6 +68,7 @@ repos: - packaging - tomli - types-pkg_resources + - types-PyYAML - repo: local hooks: - id: rst diff --git a/changelog/7342.trivial.rst b/changelog/7342.trivial.rst new file mode 100644 index 000000000..7cd4fecdc --- /dev/null +++ b/changelog/7342.trivial.rst @@ -0,0 +1,2 @@ +Added a system to run downstream plugin tests against current pytest, using GitHub Actions. Currently +constrained to pytest plugins that utilize GitHub Actions and ``tox`` to accomplish their own tests. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 3d683aebe..e9c518e5d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,6 +59,8 @@ markers = [ "slow", # experimental mark for all tests using pexpect "uses_pexpect", + # runs tests for ./testing/downstream_testing/downstream_runner.py + "downstream", ] diff --git a/setup.cfg b/setup.cfg index 3545a9503..554748cef 100644 --- a/setup.cfg +++ b/setup.cfg @@ -64,6 +64,7 @@ console_scripts = [options.extras_require] testing = + PyYAML argcomplete hypothesis>=3.56 mock diff --git a/testing/downstream_testing/Dockerfile b/testing/downstream_testing/Dockerfile new file mode 100644 index 000000000..aaa3057f9 --- /dev/null +++ b/testing/downstream_testing/Dockerfile @@ -0,0 +1,27 @@ +FROM sommersoft/pyenv-pytest:latest + +COPY entrypoint.sh /entrypoint.sh + +# Add mysql apt repository +RUN set -ex; \ +# gpg: key 5072E1F5: public key "MySQL Release Engineering " imported + key='859BE8D7C586F538430B19C2467B942D3A79BD29'; \ + export GNUPGHOME="$(mktemp -d)"; \ + #gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys "$key"; \ + for server in ha.pool.sks-keyservers.net \ + hkp://p80.pool.sks-keyservers.net:80 \ + keyserver.ubuntu.com \ + hkp://keyserver.ubuntu.com:80 \ + pgp.mit.edu; do \ + gpg --keyserver "$server" --recv-keys "$key" && break || echo "Trying new keyserver..."; \ + done; \ + gpg --batch --export "$key" > /etc/apt/trusted.gpg.d/mysql.gpg; \ + gpgconf --kill all; \ + rm -rf "$GNUPGHOME"; \ + apt-key list > /dev/null +RUN echo 'deb http://repo.mysql.com/apt/debian/ buster mysql-8.0' > /etc/apt/sources.list.d/mysql.list + +RUN apt-get update \ + && apt-get install -y postgresql-client mysql-client + +ENTRYPOINT [ "/entrypoint.sh" ] diff --git a/testing/downstream_testing/README.rst b/testing/downstream_testing/README.rst new file mode 100644 index 000000000..bea6aa6b8 --- /dev/null +++ b/testing/downstream_testing/README.rst @@ -0,0 +1,222 @@ +Automated Downstream Testing +++++++++++++++++++++++++++++ +Testing pytest Plugins With Changes to pytest +============================================= + +Overview +-------- + +- Utilizes Docker (compose) inside a GitHub Actions workflow to establish the necessary environment(s) to run tests. + + - The base image is currently Ubuntu only, and uses ``pyenv`` to have the necessary versions of Python installed. + + - The base image is on Docker Hub, which greatly speeds up each run by avoiding ``pyenv``'s compilation process each time. + + - The completed image contains a PostgreSQL & mySQL layer (needed by ``pytest-django``, at minimum). Additional layers can be added if necessary. + +- Utilizes each plugin's current test workflow to formulate the tests to run. + + - The plugin's GitHub Actions workflow is used to get ``tox`` environment settings. + + - The plugin's ``tox`` config is adjusted to ensure the local ``pytest`` is used. + +- Uses a static command set: ``tox -e {toxenv}``. + +How To Add Additional Plugin(s) +------------------------------- +**pytest/testing/downstream_testing/actions_schema.json:** +********************************************************** + +``actions_schema.json`` outlines both GitHub Actions workflow fields to get information, and how +that information should be manipulated to provide usable tox environments to run a plugin's tests. + +Example ``actions_schema.json`` entry: + + .. code:: JSON + + "pytest-django": { + "matrix": [ + "matrix", + "include" + ], + "tox_cmd_build": { + "base": "name", + "prefix": "py", + "sub": { + "pattern": "-coverage$", + "replace": "" + } + } + } + +**To add a new plugin, complete the following:** + +1. Start by adding a new JSON object, with the name of the repo. ``"pytest-django": {}`` in the example above. + +2. Add an array named ``matrix``. Array items should point to the strategy matrix from the repo's GitHub Actions + YAML file. For instance, the ``pytest-django`` example above has a strategy matrix defined as follows + + .. code:: YAML + + # pytest-django/.github/workflows/main.yml + jobs: + tests: + strategy: + fail-fast: false + matrix: + include: + - name: linting,docs + python: 3.8 + allow_failure: false + + - name: py310-dj40-postgres-xdist-coverage + python: '3.10' + allow_failure: false + + + This makes ``["matrix", "include"]`` our target to parse the ``pytest-django`` strategy. This is a "combination" + strategy based on the use of ``include``. + + For non-combination strategies, use the matrix field that points to the appropriate choices. Using + ``pytest-order`` as a non-combination example: + + .. code:: YAML + + # pytest-order/.github/workflows/pythontests.yml + jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + python-version: [3.6, 3.7, 3.8, 3.9, "3.10", pypy3] + + The corresponding entry in ``actions_schema.json`` points to ``["matrix", "python-version"]``: + + .. code:: JSON + + "pytest-order": { + "matrix": [ + "matrix", + "python-version" + ], + +3. Add a JSON object named ``tox_cmd_build``, with three items: ``base``, ``prefix``, and ``sub``. + + - ``base``: + + - For combination strategies (with ``include``), ``base`` is the field to be used as the basis + of the tox environment. ``base: "name"`` in the ``pytest-django`` example above. + + - For non-combination strategies, this field is an empty string. + + - ``prefix``: + + - For combination strategies, ``prefix`` is used to [dis]qualify entries in ``base``. + ``prefix: "py"`` in the ``pytest-django`` example above. + + - For non-combination strategies, this field is an emtpy string. + + - ``sub``: + + - For both combination and non-combination strategies, this JSON object gives a RegEx matching + (``pattern``) and a substituition (``replace``) string. Since these are JSON strings, they cannot be + represented as a Python raw string (``r""``); ensure to properly escape characters. + +**Testing additions locally:** + +1. Have a local copy of a plugin's GitHub Actions workflow YAML file, as well as the ``tox.ini``. These should + be placed in a sub-folder in ``pytest``; recommended folder name is the repo name. + + .. code:: shell + + . pytest/ + .. pytest-order/ + ... main.yml + ... tox.ini + +2. Utilize the ``--dry-run`` flag to run the parsing and command step building, without actually executing + the commands. This will provide debugging information. + + .. code:: shell + + (.venv) ~/pytest:$> python -m testing.downstream_testing.downstream_runner pytest-order pytest-order/main.yml test --matrix-exclude 3.6 --dry-run + + DEBUG | downstream_runner.load_matrix_schema | Loading schema: /home/pytest/testing/downstream_testing/action_schemas.json + DEBUG | downstream_runner.load_matrix_schema | 'pytest-order' schema loaded: {'matrix': ['matrix', 'python-version'], + 'tox_cmd_build': {'base': '', 'prefix': '', 'sub': {'pattern': '(\\d|py\\d)\\.*(\\d+)', 'replace': 'py\\1\\2'}}, 'python_version': 'python-version'} + DEBUG | downstream_runner.inject_pytest_dep | toxenv dependencies updated: {'!pytest{60,61,62,624,70}: pytest-xdist', '!pytest50: pytest @ file:///home/pytest'} + DEBUG | downstream_runner.build_run | job_name: test + DEBUG | downstream_runner.parse_matrix | parsed_matrix: [3.6, 3.7, 3.8, 3.9, '3.10', 'pypy3'] + DEBUG | downstream_runner.matrix | matrix: {'test': [{'name': 'py37', 'tox_cmd': 'py37'}, {'name': 'py38', 'tox_cmd': 'py38'}, {'name': 'py39', 'tox_cmd': 'py39'}, + {'name': 'py310', 'tox_cmd': 'py310'}, {'name': 'pypy3', 'tox_cmd': 'pypy3'}]} + DEBUG | downstream_runner.build_run | matrix[job]: {'name': 'py37', 'tox_cmd': 'py37'} + DEBUG | downstream_runner.build_run | matrix[job]: {'name': 'py38', 'tox_cmd': 'py38'} + DEBUG | downstream_runner.build_run | matrix[job]: {'name': 'py39', 'tox_cmd': 'py39'} + DEBUG | downstream_runner.build_run | matrix[job]: {'name': 'py310', 'tox_cmd': 'py310'} + DEBUG | downstream_runner.build_run | matrix[job]: {'name': 'pypy3', 'tox_cmd': 'pypy3'} + DEBUG | downstream_runner.build_run | built run: {'py37': ['tox -e py37'], 'py38': ['tox -e py38'], 'py39': ['tox -e py39'], 'py310': ['tox -e py310'], 'pypy3': ['tox -e pypy3']} + INFO | downstream_runner.run | --> running: 'tox -e py37' + INFO | downstream_runner.run | --> running: 'tox -e py38' + INFO | downstream_runner.run | --> running: 'tox -e py39' + INFO | downstream_runner.run | --> running: 'tox -e py310' + INFO | downstream_runner.run | --> running: 'tox -e pypy3' + + +**pytest/.github/workflows/downstream_testing.yml:** +***************************************************** +This GitHub Actions workflow orchestrates the various plugin tests, and only requires some minimal information. + +Add a new entry to the combination strategy matrix: + +1. ``name``: the plugin's repository name. + +2. ``repo``: the ``org/name`` of the plugin's GitHub repository. + +3. ``docker_profile``: the docker-compose profile to use for this entry. Docker profiles are defined in + ``pytest/testing/downstream_testing/docker-compose.yml``. At the time of this writing, the available + profiles are: ``nodb``, ``mysql``, & ``postgres``. + +4. ``jobs``: the list of job(s) to use from the plugin's GitHub Actions workflow. + +5. ``workflow_name``: the name of the GitHub Actions workflow file to use (located in ``plugin/.github/workflows``). + +6. ``matrix_exclude``: any entries to exclude from the selected matrix (combination & non-combination). + +.. code:: YAML + + - name: "pytest-django" + repo: "pytest-dev/pytest-django" + docker_profile: "postgres" + jobs: "test" + workflow_name: "main.yml" + matrix_exclude: | + linting,docs py39-dj40-mysql_innodb-coverage ... + + - name: "pytest-django" + repo: "pytest-dev/pytest-django" + docker_profile: "mysql" + jobs: "test" + workflow_name: "main.yml" + matrix_exclude: | + linting,docs py310-dj40-postgres-xdist-coverage ... + +.. epigraph:: + Example 1: using ``pytest-django``, which has a combination strategy matrix, we see two (of three) different + ``downstream_testing.yml`` entries. They each require a different database, so each entry uses the appropriate + ``docker_profile``. Additionally, to avoid known failures, ``matrix_exclude`` has all items that don't match + the database in use. + +.. code:: YAML + + - name: "pytest-order" + repo: "pytest-dev/pytest-order" + docker_profile: "nodb" + jobs: "test" + workflow_name: "pythontests.yml" + matrix_exclude: | + 3.6 + +.. epigraph:: + Example 2: using ``pytest-order``, which has a non-combination strategy matrix and requires no database. diff --git a/testing/downstream_testing/action_schemas.json b/testing/downstream_testing/action_schemas.json new file mode 100644 index 000000000..cd4ec8270 --- /dev/null +++ b/testing/downstream_testing/action_schemas.json @@ -0,0 +1,92 @@ +{ + "pytest-django": { + "matrix": [ + "matrix", + "include" + ], + "tox_cmd_build": { + "base": "name", + "prefix": "py", + "sub": + { + "pattern": "-coverage$", + "replace": "" + } + } + }, + "pytest-html": { + "matrix": [ + "matrix", + "include" + ], + "tox_cmd_build": { + "base": "name", + "prefix": "py", + "sub": + { + "pattern": "(py\\d+)-\\w+", + "replace": "\\1" + } + } + }, + "pytest-order": { + "matrix": [ + "matrix", + "python-version" + ], + "tox_cmd_build": { + "base": "", + "prefix": "", + "sub": + { + "pattern": "(\\d|py\\d)\\.*(\\d+)", + "replace": "py\\1\\2" + } + } + }, + "pytest-mock": { + "matrix": [ + "matrix", + "python" + ], + "tox_cmd_build": { + "base": "", + "prefix": "", + "sub": + { + "pattern": "(\\d|py\\d)\\.*(\\d+)", + "replace": "py\\1\\2" + } + } + }, + "pytest-cov": { + "matrix": [ + "matrix", + "python-version" + ], + "tox_cmd_build": { + "base": "", + "prefix": "", + "sub": + { + "pattern": "^\\d\\.(\\d+)(?:-dev)*", + "replace": "py3\\1" + } + } + }, + "pytest-bdd": { + "matrix": [ + "matrix", + "python-version" + ], + "tox_cmd_build": { + "base": "", + "prefix": "", + "sub": + { + "pattern": "^\\d\\.(\\d+)", + "replace": "py3\\1" + } + } + } +} diff --git a/testing/downstream_testing/docker-compose.yml b/testing/downstream_testing/docker-compose.yml new file mode 100644 index 000000000..73ffe12f8 --- /dev/null +++ b/testing/downstream_testing/docker-compose.yml @@ -0,0 +1,88 @@ +version: "3" +services: + base_nodb: + container_name: base + build: . + environment: + - DS_NAME + - DS_YAML + - DS_JOBS + - DS_MATRIX_EXCLUDE + - COV_CMD + - PY_COLORS=1 + - TOX_TESTENV_PASSENV=PY_COLORS + volumes: + - type: bind + source: /home/runner/work/pytest/pytest + target: /pytest + volume: + nocopy: true + profiles: + - nodb + + base_postgres: + container_name: base + build: . + environment: + - DS_NAME + - DS_YAML + - DS_JOBS + - DS_MATRIX_EXCLUDE + - TEST_DB_USER=$USER + - TEST_DB_PASSWORD=pytest_djang0 + - TEST_DB_HOST=postgres_db + - PY_COLORS=1 + - TOX_TESTENV_PASSENV=PY_COLORS + volumes: + - type: bind + source: /home/runner/work/pytest/pytest + target: /pytest + volume: + nocopy: true + depends_on: + - postgres_db + profiles: + - postgres + postgres_db: + image: "postgres:latest" + environment: + - POSTGRES_PASSWORD=pytest_djang0 + - POSTGRES_USER=$USER + volumes: + - /etc/passwd:/etc/passwd:ro + - ./data/db:/var/lib/postgresql/data + profiles: + - postgres + + base_mysql: + container_name: base + build: . + environment: + - DS_NAME + - DS_YAML + - DS_JOBS + - DS_MATRIX_EXCLUDE + - TEST_DB_USER=root + - TEST_DB_PASSWORD=root + - TEST_DB_HOST=mysql_db + - PY_COLORS=1 + - TOX_TESTENV_PASSENV=PY_COLORS + volumes: + - type: bind + source: /home/runner/work/pytest/pytest + target: /pytest + volume: + nocopy: true + depends_on: + - mysql_db + profiles: + - mysql + mysql_db: + image: "mysql:latest" + command: --default-authentication-plugin=mysql_native_password + environment: + - MYSQL_ROOT_PASSWORD=root + volumes: + - ./data/db:/var/lib/mysql + profiles: + - mysql diff --git a/testing/downstream_testing/downstream_runner.py b/testing/downstream_testing/downstream_runner.py new file mode 100644 index 000000000..5d2f39251 --- /dev/null +++ b/testing/downstream_testing/downstream_runner.py @@ -0,0 +1,328 @@ +from __future__ import annotations + +import argparse +import configparser +import json +import logging +import os.path +import re +import shlex +import subprocess +from collections import UserDict +from typing import Any +from typing import Iterable +from typing import Match +from typing import TYPE_CHECKING + +import yaml + +if TYPE_CHECKING: + from typing_extensions import TypedDict + +logging.basicConfig( + format="%(levelname)s | %(module)s.%(funcName)s | %(message)s", level="INFO" +) +logger = logging.getLogger(__name__) + + +parser = argparse.ArgumentParser(description="pytest downstream plugins test runner") +parser.add_argument("repo", help="Name of the repo.") +parser.add_argument("source", help="Path to source YAML file.") +parser.add_argument("jobs", nargs="+", help="Job names to use.") +parser.add_argument( + "--matrix-exclude", nargs="*", default=[], help="Exclude these matrix names." +) +parser.add_argument( + "--dry-run", + action="store_true", + help="Do not run parsed downstream action. Only display the generated command list and debug information.", +) + +if TYPE_CHECKING: + _BaseUserDict = UserDict[Any, Any] + + class SchemaBase(TypedDict): + repo: str + + class SchemaToxBase(TypedDict): + base: str + prefix: str + sub: dict[str, str] + + class SchemaType(SchemaBase, total=False): + matrix: list[str] + tox_cmd_build: SchemaToxBase + python_version: str + +else: + _BaseUserDict = UserDict + + +def load_matrix_schema(repo: str) -> SchemaType: + """Loads the matrix schema for ``repo``""" + schema: SchemaType = {"repo": repo} + working_dir = os.getcwd() + schema_path = os.path.join( + working_dir, "testing", "downstream_testing", "action_schemas.json" + ) + logger.debug("Loading schema: %s", schema_path) + if os.path.exists(schema_path): + with open(schema_path) as schema_file: + try: + schema = json.load(schema_file) + except json.JSONDecodeError as exc: + raise RuntimeError(f"Error decoding '{schema_path}'") from exc + else: + raise FileNotFoundError(f"'{schema_path}' not found.") + + if repo in schema: + logger.debug("'%s' schema loaded: %s", repo, schema[repo]) # type: ignore + return schema[repo] # type: ignore + else: + raise RuntimeError( + f"'{repo}' schema definition not found in actions_schema.json" + ) + + +TOX_DEP_FILTERS = { + "pytest-rerunfailures": { + "src": "pytest-rerunfailures @ git+https://github.com/pytest-dev/pytest-rerunfailures.git", + "condition": r"^pytest-rerunfailures.*", + "has_gen": r"pytest-rerunfailures\w*:", + }, + "pytest-xdist": { + "src": "pytest-xdist", + "condition": r"^pytest.*pytest-xdist", + "has_gen": r"pytest\{.*\,7\d.*\}", + }, + "pytest": { + "src": f"pytest @ file://{os.getcwd()}", + "condition": r"^pytest(?!\-)", + "has_gen": r"pytest\w*", + }, +} + + +class ToxDepFilter(_BaseUserDict): + def __init__(self) -> None: + self.data = TOX_DEP_FILTERS + + def matches_condition(self, match: str) -> str | None: + """Checks if ``match`` matches any conditions""" + match_found = None + for key, val in self.data.items(): + if re.search(val["condition"], match): + match_found = key + break + + return match_found + + def matches_gen_exp(self, dep: str, match: str) -> Match[str] | None: + """Checks if ``match`` matches ``dep``['has_gen'] condition.""" + return re.match(self.data[dep]["has_gen"], match) + + def filter_dep(self, match: str) -> dict[Any, Any] | None: + """Filters ``match`` based on conditions and returns the ``src`` dependency.""" + filtered_match = None + dep_condition = self.matches_condition(match) + if dep_condition is not None: + dep_gen_exp = self.matches_gen_exp(dep_condition, match) + if dep_gen_exp: + filtered_match = { + "src": self.data[dep_condition]["src"], + "gen_exp": dep_gen_exp[0], + } + return filtered_match + + +class DownstreamRunner: + def __init__( + self, + repo: str, + yaml_source: str, + jobs: Iterable[str], + matrix_exclude: Iterable[str] = (), + dry_run: bool = False, + ) -> None: + self.repo = repo + self.yaml_source = yaml_source + self.matrix_exclude = matrix_exclude + self.job_names = jobs + self.dry_run = dry_run + + self._yaml_tree: dict[str, Any] | None = None + self._matrix: dict[str, Any] | None = None + self.matrix_schema: SchemaType = load_matrix_schema(self.repo) + + @property + def yaml_tree(self) -> dict[str, Any]: + """The YAML tree built from the ``self.yaml_source`` file.""" + if self._yaml_tree is None: + with open(self.yaml_source) as f: + try: + _yaml_tree = yaml.safe_load(f.read()) + except yaml.YAMLError as exc: + raise RuntimeError( + f"Error while parsing '{self.yaml_source}'." + ) from exc + + if _yaml_tree is None: + raise RuntimeError(f"'{self.yaml_source}' failed to parse.") + else: + self._yaml_tree = _yaml_tree + + return self._yaml_tree + + def inject_pytest_dep(self) -> None: + """Ensure pytest is a dependency in tox.ini to allow us to use the 'local' + version of pytest. Also ensure other dependencies in ``TOX_DEP_FILTERS`` + are defined appropriately. + """ + ini_path = self.repo + "/tox.ini" + pytest_dep = TOX_DEP_FILTERS["pytest"]["src"] + tox_source = configparser.ConfigParser() + tox_source.read_file(open(ini_path)) + found_dep = [] + updated_deps = set() + section_deps = tox_source.get("testenv", "deps", fallback=None) + if section_deps: + tox_dep_filter = ToxDepFilter() + for dep in section_deps.split("\n"): + filtered_dep = tox_dep_filter.filter_dep(dep) + if filtered_dep and filtered_dep["src"] not in found_dep: + found_dep.append(filtered_dep["src"]) + updated_deps.add( + f"!{filtered_dep['gen_exp']}: {filtered_dep['src']}" + ) + + if not [item for item in updated_deps if pytest_dep in item]: + updated_deps.add(pytest_dep) + final_deps = "\n".join(updated_deps) + logger.debug("toxenv dependencies updated: %s", updated_deps) + tox_source["testenv"][ + "deps" + ] = f"{tox_source['testenv']['deps']}\n{final_deps}" + + with open(ini_path, "w") as f: + tox_source.write(f) + + def __repr__(self) -> str: + return str( + "DownstreamRunner(" + f"repo={self.repo}, " + f"yaml_source={self.yaml_source}, " + f"job_names={self.job_names}, " + f"matrix={self._matrix}, " + ")" + ) + + @property + def matrix(self) -> dict[str, Iterable[dict[str, str]]]: + """Iterates over ``self.yaml_tree``'s strategy matrix for each job in ``self.jobs``, and passes each + through ``parse_matrix``. + """ + + def parse_matrix(yaml_tree: dict[str, Any]) -> Iterable[Any]: + """Parses ``yaml_tree`` strategy matrix using ``self.matrix_schema`` information.""" + parsed_matrix = [] # type: ignore + pre_parsed: dict[str, Any] | Iterable[str | float] = yaml_tree + for key in self.matrix_schema["matrix"]: + if isinstance(pre_parsed, dict): + pre_parsed = pre_parsed[key] + else: + if isinstance(pre_parsed, list): + parsed_matrix = pre_parsed + else: + msg_info = f"repo: {self.repo} | matrix schema: {self.matrix_schema} | parsed result: {pre_parsed}" + raise TypeError( + f"Parsed Actions matrix is invalid. Should be list/array. {msg_info}" + ) + + logger.debug("parsed_matrix: %s", parsed_matrix) + if parsed_matrix: + tox_base = self.matrix_schema["tox_cmd_build"]["base"] + tox_prefix = self.matrix_schema["tox_cmd_build"]["prefix"] + skip_matrices = [] + if "include" in self.matrix_schema["matrix"]: + for item in parsed_matrix: + if ( + not item[tox_base].startswith(tox_prefix) + or item[tox_base] in self.matrix_exclude + or not item.get("os", "ubuntu").startswith("ubuntu") + ): + skip_matrices.append(item) + continue + + item["tox_cmd"] = re.sub( + self.matrix_schema["tox_cmd_build"]["sub"]["pattern"], + self.matrix_schema["tox_cmd_build"]["sub"]["replace"], + item[tox_base], + ) + logger.debug("re.sub: %s", item[tox_base]) + + for matrice in skip_matrices: + parsed_matrix.remove(matrice) + + else: + new_parsed_matrix = [] + for item in parsed_matrix: + if str(item) in self.matrix_exclude: + continue + tox_cmd = re.sub( + self.matrix_schema["tox_cmd_build"]["sub"]["pattern"], + self.matrix_schema["tox_cmd_build"]["sub"]["replace"], + str(item), + ) + new_parsed_matrix.append({"name": tox_cmd, "tox_cmd": tox_cmd}) + parsed_matrix = new_parsed_matrix + + return parsed_matrix + + if self._matrix is None: + matrix_items = {} + for job in self.job_names: + job_yaml = self.yaml_tree["jobs"][job]["strategy"] + parsed_matrix = parse_matrix(job_yaml) + matrix_items[job] = parsed_matrix + + self._matrix = matrix_items + logger.debug("matrix: %s", self._matrix) + return self._matrix + + def build_run(self) -> dict[str, list[str]]: + """Builds the list of commands for all necessary jobs via ``self.matrix``.""" + run = {} + for job in self.job_names: + logger.debug("job_name: %s", job) + for matrix in self.matrix[job]: + logger.debug("matrix[job]: %s", matrix) + run[matrix["name"]] = [f"tox -e {matrix['tox_cmd']}"] + + logger.debug("built run: %s", run) + return run + + def run(self) -> None: + self.inject_pytest_dep() + run_steps = self.build_run() + os.chdir(self.repo) + for matrix, steps in run_steps.items(): + for step in steps: + cmd = shlex.split(step) + logger.info("--> running: '%s'", step) + if not self.dry_run: + subprocess.run(cmd, encoding="utf-8", check=True) + + +if __name__ == "__main__": + cli_args = parser.parse_args() + if cli_args.dry_run: + logger.setLevel("DEBUG") + runner = DownstreamRunner( + cli_args.repo, + cli_args.source, + cli_args.jobs, + matrix_exclude=cli_args.matrix_exclude, + dry_run=cli_args.dry_run, + ) + + runner.run() diff --git a/testing/downstream_testing/entrypoint.sh b/testing/downstream_testing/entrypoint.sh new file mode 100755 index 000000000..151bc5458 --- /dev/null +++ b/testing/downstream_testing/entrypoint.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +python3.9 -m pip install --no-cache-dir pyyaml tox sh + +cd /pytest +python3.9 -u -m testing.downstream_testing.downstream_runner $DS_NAME $DS_YAML $DS_JOBS --matrix-exclude $DS_MATRIX_EXCLUDE diff --git a/testing/downstream_testing/test_downstream_runner.py b/testing/downstream_testing/test_downstream_runner.py new file mode 100644 index 000000000..a444d3945 --- /dev/null +++ b/testing/downstream_testing/test_downstream_runner.py @@ -0,0 +1,280 @@ +from __future__ import annotations + +import sys +from typing import Any + +import pytest + +sys.path.append("testing") +from downstream_testing import downstream_runner # noqa: E402 + +xfail = pytest.mark.xfail + +DUMMY_YAML_COMBINATION = """--- +name: dummy-include +jobs: + test: + runs-on: ubuntu-20.04 + + strategy: + fail-fast: false + matrix: + include: + - name: i-do-not-start-with-py + python: 3.6 + allow_failure: false + + - name: py310-dj40-postgres-xdist-coverage + python: '3.10' + allow_failure: false + + - name: py36-exclude-me + python: 3.6 + allow_failure: false + + - name: py37-exclude-me + python: 3.7 + allow_failure: false +""" + +DUMMY_YAML_NO_COMBINATION = """--- +name: dummy-no-include +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + python-version: [3.6, 3.7, 3.8, 3.9, "3.10", pypy3] + exclude: + - os: windows-latest + python-version: pypy3 +""" + + +@pytest.fixture() +def mock_schema_combination(monkeypatch): + def mock_load_schema_combination(repo: str) -> dict[str, Any]: + return { + "matrix": ["matrix", "include"], + "tox_cmd_build": { + "base": "name", + "prefix": "py", + "sub": {"pattern": "-coverage$", "replace": ""}, + }, + "python_version": "python", + } + + monkeypatch.setattr( + downstream_runner, "load_matrix_schema", mock_load_schema_combination + ) + + +@pytest.fixture() +def mock_schema_no_combination(monkeypatch): + def mock_load_schema_no_combination(repo: str) -> dict[str, Any]: + return { + "matrix": ["matrix", "python-version"], + "tox_cmd_build": { + "base": "", + "prefix": "", + "sub": {"pattern": "(\\d|py\\d)\\.*(\\d+)", "replace": "py\\1\\2"}, + }, + "python_version": "python", + } + + monkeypatch.setattr( + downstream_runner, "load_matrix_schema", mock_load_schema_no_combination + ) + + +@pytest.mark.downstream +class TestDownstreamRunnerConstructor: + @pytest.mark.parametrize( + "args", + [ + pytest.param(tuple(), marks=xfail), + pytest.param(("repo",), marks=xfail), + pytest.param(("repo", "yaml_source"), marks=xfail), + ("pytest-downstream", "yaml_source", ["jobs"]), + ], + ids=["no args", "repo only", "missing jobs", "all args"], + ) + def test_args(self, args, mock_schema_combination) -> None: + downstream_runner.DownstreamRunner(*args) + + @pytest.mark.parametrize( + "kwargs", + [ + ("matrix_exclude", "exclude"), + ("dry_run", True), + ], + ids=["matrix_exclude", "dry_run"], + ) + def test_kwargs(self, kwargs, mock_schema_combination) -> None: + args = ("pytest-downstream", "yaml_source", ["test"]) + new_kwargs = {kwargs[0]: kwargs[1]} + runner = downstream_runner.DownstreamRunner(*args, **new_kwargs) + + assert kwargs[1] == getattr(runner, kwargs[0]) + + +@pytest.mark.downstream +class TestDownstreamRunnerProperties: + def test_yaml_tree_file_doesnt_exist(self, mock_schema_combination) -> None: + runner = downstream_runner.DownstreamRunner( + "pytest-downstream", "yaml_source", ["test"], dry_run=True + ) + + with pytest.raises(FileNotFoundError): + runner.yaml_tree + + def test_yaml_tree_bad_yaml(self, mock_schema_combination, tmp_path) -> None: + yaml_source = tmp_path / "test.yml" + yaml_source.write_text("---\n:") + + runner = downstream_runner.DownstreamRunner( + "pytest-downstream", yaml_source, ["test"], dry_run=True + ) + + with pytest.raises(RuntimeError) as excinfo: + runner.yaml_tree + assert str(yaml_source) in excinfo.exconly(tryshort=True) + + def test_yaml_tree_empty_yaml(self, mock_schema_combination, tmp_path) -> None: + yaml_source = tmp_path / "test.yml" + yaml_source.write_text("---") + + runner = downstream_runner.DownstreamRunner( + "pytest-downstream", yaml_source, ["test"], dry_run=True + ) + + with pytest.raises(RuntimeError) as excinfo: + runner.yaml_tree + assert str(yaml_source) in excinfo.exconly(tryshort=True) + + def test_yaml_tree_passing_yaml(self, mock_schema_combination, tmp_path) -> None: + yaml_source = tmp_path / "test.yml" + yaml_source.write_text(DUMMY_YAML_COMBINATION) + + runner = downstream_runner.DownstreamRunner( + "pytest-downstream", yaml_source, ["test"], dry_run=True + ) + + assert runner.yaml_tree["name"] == "dummy-include" + assert "test" in runner.yaml_tree["jobs"] + + def test_matrix_combination(self, mock_schema_combination, tmp_path) -> None: + yaml_source = tmp_path / "test.yml" + yaml_source.write_text(DUMMY_YAML_COMBINATION) + + runner = downstream_runner.DownstreamRunner( + "pytest-downstream", yaml_source, ["test"], dry_run=True + ) + + assert all( + [ + matrice.get("tox_cmd", "").startswith("py") + for matrice in runner.matrix["test"] + ] + ) + + def test_matrix_no_combination(self, mock_schema_no_combination, tmp_path) -> None: + yaml_source = tmp_path / "test.yml" + yaml_source.write_text(DUMMY_YAML_NO_COMBINATION) + + runner = downstream_runner.DownstreamRunner( + "pytest-downstream", yaml_source, ["test"], dry_run=True + ) + + assert all( + [ + matrice.get("tox_cmd", "").startswith("py") + for matrice in runner.matrix["test"] + ] + ) + + def test_matrix_combination_matrix_exclude( + self, mock_schema_combination, tmp_path + ) -> None: + yaml_source = tmp_path / "test.yml" + yaml_source.write_text(DUMMY_YAML_COMBINATION) + matrix_to_exclude = ["py36-exclude-me", "py37-exclude-me"] + + runner = downstream_runner.DownstreamRunner( + "pytest-downstream", + yaml_source, + ["test"], + matrix_exclude=matrix_to_exclude, + dry_run=True, + ) + + matrix_names = {matrice["name"] for matrice in runner.matrix["test"]} + assert matrix_names.isdisjoint(set(matrix_to_exclude)) + + def test_matrix_no_combination_matrix_exclude( + self, mock_schema_no_combination, tmp_path + ) -> None: + yaml_source = tmp_path / "test.yml" + yaml_source.write_text(DUMMY_YAML_NO_COMBINATION) + matrix_to_exclude = ["3.6", "3.7"] + + runner = downstream_runner.DownstreamRunner( + "pytest-downstream", + yaml_source, + ["test"], + matrix_exclude=matrix_to_exclude, + dry_run=True, + ) + + matrix_names = {matrice["name"] for matrice in runner.matrix["test"]} + assert matrix_names.isdisjoint({"py36", "py37"}) + + +@pytest.mark.downstream +class TestDownstreamRunnerBuild: + def test_build_run_combination_matrix( + self, mock_schema_combination, tmp_path + ) -> None: + yaml_source = tmp_path / "test.yml" + yaml_source.write_text(DUMMY_YAML_COMBINATION) + matrix_to_exclude = ["py36-exclude-me", "py37-exclude-me"] + + runner = downstream_runner.DownstreamRunner( + "pytest-downstream", + yaml_source, + ["test"], + matrix_exclude=matrix_to_exclude, + dry_run=True, + ) + + run = runner.build_run() + assert run == { + "py310-dj40-postgres-xdist-coverage": [ + "tox -e py310-dj40-postgres-xdist", + ] + } + + def test_build_run_no_combination_matrix( + self, mock_schema_no_combination, tmp_path + ) -> None: + yaml_source = tmp_path / "test.yml" + yaml_source.write_text(DUMMY_YAML_NO_COMBINATION) + matrix_to_exclude = ["3.6", "3.7"] + + runner = downstream_runner.DownstreamRunner( + "pytest-downstream", + yaml_source, + ["test"], + matrix_exclude=matrix_to_exclude, + dry_run=True, + ) + + run = runner.build_run() + assert run == { + "py310": ["tox -e py310"], + "py38": ["tox -e py38"], + "py39": ["tox -e py39"], + "pypy3": ["tox -e pypy3"], + } diff --git a/tox.ini b/tox.ini index 93c390ffc..9ed36183a 100644 --- a/tox.ini +++ b/tox.ini @@ -16,6 +16,7 @@ envlist = py37-freeze docs docs-checklinks + downstream @@ -27,7 +28,7 @@ commands = coverage: coverage report -m passenv = USER USERNAME COVERAGE_* PYTEST_ADDOPTS TERM SETUPTOOLS_SCM_PRETEND_VERSION_FOR_PYTEST setenv = - _PYTEST_TOX_DEFAULT_POSARGS={env:_PYTEST_TOX_POSARGS_DOCTESTING:} {env:_PYTEST_TOX_POSARGS_LSOF:} {env:_PYTEST_TOX_POSARGS_XDIST:} + _PYTEST_TOX_DEFAULT_POSARGS={env:_PYTEST_TOX_POSARGS_DOCTESTING:} {env:_PYTEST_TOX_POSARGS_LSOF:} {env:_PYTEST_TOX_POSARGS_XDIST:} {env:_PYTEST_TOX_POSARGS_DOWNSTREAM:} # Configuration to run with coverage similar to CI, e.g. # "tox -e py37-coverage". @@ -43,6 +44,8 @@ setenv = lsof: _PYTEST_TOX_POSARGS_LSOF=--lsof xdist: _PYTEST_TOX_POSARGS_XDIST=-n auto + + downstream: _PYTEST_TOX_POSARGS_DOWNSTREAM=-m downstream extras = testing deps = doctesting: PyYAML