diff --git a/.github/workflows/downstream_testing.yml b/.github/workflows/downstream_testing.yml index b912be6b8..a57a9d595 100644 --- a/.github/workflows/downstream_testing.yml +++ b/.github/workflows/downstream_testing.yml @@ -21,7 +21,7 @@ jobs: workflow_name: "main.yml" matrix_exclude: | linting,docs py39-dj40-mysql_innodb-coverage py36-dj22-sqlite-xdist-coverage py37-dj22-sqlite-xdist-coverage - py38-dj32-sqlite-xdist-coverage py38-dj40-sqlite-xdist-coverage py39-djmain-sqlite-coverage + py38-dj32-sqlite-xdist-coverage py38-dj40-sqlite-xdist-coverage py39-djmain-sqlite-coverage py35-dj22-postgres-pytest54-coverage py35-dj22-sqlite_file-coverage py36-dj32-mysql_myisam-coverage - name: "pytest-django" repo: "pytest-dev/pytest-django" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 261c525c8..a028b0652 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/psf/black - rev: 22.1.0 + rev: 22.3.0 hooks: - id: black args: [--safe, --quiet] @@ -56,7 +56,7 @@ repos: hooks: - id: python-use-type-annotations - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.940 + rev: v0.942 hooks: - id: mypy files: ^(src/|testing/) @@ -69,6 +69,7 @@ repos: - tomli - types-atomicwrites - types-pkg_resources + - types-PyYAML - repo: local hooks: - id: rst diff --git a/testing/downstream_testing/Dockerfile b/testing/downstream_testing/Dockerfile index 4474f1d93..aaa3057f9 100644 --- a/testing/downstream_testing/Dockerfile +++ b/testing/downstream_testing/Dockerfile @@ -24,4 +24,4 @@ RUN echo 'deb http://repo.mysql.com/apt/debian/ buster mysql-8.0' > /etc/apt/sou RUN apt-get update \ && apt-get install -y postgresql-client mysql-client -ENTRYPOINT [ "/entrypoint.sh" ] \ No newline at end of file +ENTRYPOINT [ "/entrypoint.sh" ] diff --git a/testing/downstream_testing/action_schemas.json b/testing/downstream_testing/action_schemas.json index 96972fb40..7f16f2180 100644 --- a/testing/downstream_testing/action_schemas.json +++ b/testing/downstream_testing/action_schemas.json @@ -47,4 +47,4 @@ }, "python_version": "python-version" } -} \ No newline at end of file +} diff --git a/testing/downstream_testing/docker-compose.yml b/testing/downstream_testing/docker-compose.yml index fa21ba034..ee3fcd5e9 100644 --- a/testing/downstream_testing/docker-compose.yml +++ b/testing/downstream_testing/docker-compose.yml @@ -16,7 +16,7 @@ services: nocopy: true profiles: - nodb - + base_postgres: build: . environment: @@ -76,4 +76,4 @@ services: volumes: - ./data/db:/var/lib/mysql profiles: - - mysql \ No newline at end of file + - mysql diff --git a/testing/downstream_testing/downstream_runner.py b/testing/downstream_testing/downstream_runner.py index 6483cbb6b..feefcf081 100644 --- a/testing/downstream_testing/downstream_runner.py +++ b/testing/downstream_testing/downstream_runner.py @@ -2,58 +2,45 @@ import argparse import configparser import json import logging -import os import os.path import re import shlex import subprocess +from collections import UserDict +from typing import Any +from typing import Dict +from typing import List +from typing import Optional import yaml -logging.basicConfig(format="%(levelname)s | %(module)s.%(funcName)s | %(message)s", level="INFO") +logging.basicConfig( + format="%(levelname)s | %(module)s.%(funcName)s | %(message)s", level="INFO" +) logger = logging.getLogger(__name__) -parser = argparse.ArgumentParser( - description="downstream Actions runner" -) +parser = argparse.ArgumentParser(description="downstream Actions runner") +parser.add_argument("repo", help="Name of the repo.") +parser.add_argument("source", nargs=1, help="Path to source YAML file.") +parser.add_argument("jobs", nargs="+", help="Job names to use.") parser.add_argument( - "repo", - #nargs=1, - help="Name of the repo." -) -parser.add_argument( - "source", - nargs=1, - help="Path to source YAML file." -) -parser.add_argument( - "jobs", - nargs="+", - help="Job names to use." -) -parser.add_argument( - "--matrix-exclude", - nargs="*", - default=[], - help="Exclude these matrix names." + "--matrix-exclude", nargs="*", default=[], help="Exclude these matrix names." ) parser.add_argument( "--dry-run", action="store_true", - help="Do not run parsed downstream action. Only display the generated command list." + help="Do not run parsed downstream action. Only display the generated command list.", ) + def load_matrix_schema(repo): - """ Loads the matrix schema for `repo` """ + """Loads the matrix schema for `repo`""" schema = None working_dir = os.getcwd() schema_path = os.path.join( - working_dir, - "testing", - "downstream_testing", - "action_schemas.json" + working_dir, "testing", "downstream_testing", "action_schemas.json" ) logger.debug("loading schema: %s", schema_path) if os.path.exists(schema_path): @@ -74,6 +61,66 @@ def load_matrix_schema(repo): return schema + +TOX_DEP_FILTERS = { + "pytest": { + "src": f"pytest @ file://{os.getcwd()}", + "condition": r"^pytest(?!\-)", + "has_gen": r"pytest\w*", + }, + "pytest-rerunfailures": { + "src": "pytest-rerunfailures @ git+https://github.com/pytest-dev/pytest-rerunfailures.git", + "condition": r"^pytest-rerunfailures.*", + "has_gen": r"pytest-rerunfailures\w*:", + }, + "pytest-xdist": { + "src": "pytest-xdist", + "condition": r"^pytest.*pytest-xdist", + "has_gen": r"pytest\{.*\,7\d.*\}", + }, +} + + +class ToxDepFilter(UserDict[Any, Any]): + def __init__(self) -> None: + self.data = TOX_DEP_FILTERS + + def matches_condition(self, match: str) -> Optional[str]: + """Checks if `match` matches any conditions""" + match_found = None + for key, val in self.data.items(): + logger.debug( + "matches_condition: %s:%s -> %s", + key, + val, + re.search(val["condition"], match), + ) + if re.search(val["condition"], match): + match_found = key + break + + return match_found + + def matches_gen_exp(self, dep: str, match: str) -> Optional[Any]: + """Checks if `match` matches `dep`['has_gen'] condition.""" + logger.debug("matches_gen_exp: %s", re.match(self.data[dep]["has_gen"], match)) + return re.match(self.data[dep]["has_gen"], match) + + def filter_dep(self, match: str) -> Optional[Dict[Any, Any]]: + """Filters `match` based on conditions and returns the `src` dependency.""" + filtered_match = None + dep_condition = self.matches_condition(match) + if dep_condition is not None: + dep_gen_exp = self.matches_gen_exp(dep_condition, match) + if dep_gen_exp: + filtered_match = { + "src": self.data[dep_condition]["src"], + "gen_exp": dep_gen_exp, + } + logger.debug("filter_dep: %s", filtered_match) + return filtered_match + + class DownstreamRunner: def __init__(self, repo, yaml_source, jobs, matrix_exclude=None, dry_run=False): self.repo = repo @@ -87,61 +134,45 @@ class DownstreamRunner: self._steps = None self.matrix_schema = load_matrix_schema(self.repo) - @property def yaml_tree(self): - """ The YAML tree built from the `yaml_source` file. """ + """The YAML tree built from the `yaml_source` file.""" with open(self.yaml_source) as f: self._yaml_tree = yaml.safe_load(f.read()) - - if self._yaml_tree == None: + + if self._yaml_tree is None: raise SystemExit("Supplied YAML source failed to parse.") - + return self._yaml_tree def inject_pytest_dep(self): - """ Ensure pytest is a dependency in tox.ini to allow us to use the 'local' - version of pytest. + """Ensure pytest is a dependency in tox.ini to allow us to use the 'local' + version of pytest. """ ini_path = self.repo + "/tox.ini" pytest_dep = f"pytest @ file://{os.getcwd()}" - DEPS = { - "pytest": { - "src": f"pytest @ file://{os.getcwd()}", - "condition": lambda x: x.startswith("pytest") and not x.startswith("pytest-"), - "has_gen": lambda x: re.search(r"pytest\w*:", x) - }, - "pytest-rerunfailures": { - "src": "pytest-rerunfailures @ git+https://github.com/pytest-dev/pytest-rerunfailures.git", - "condition": lambda x: x.startswith("pytest-rerunfailures"), - "has_gen": lambda x: re.search(r"pytest-rerunfailures\w*:", x) - }, - "pytest-xdist": { - "src": "pytest-xdist", - "condition": lambda x: x.startswith("pytest{") and x.endswith("pytest-xdist"), - "has_gen": lambda x: re.search(r"pytest\{.*\,7\d.*\}:", x) - } - } tox_source = configparser.ConfigParser() tox_source.read_file(open(ini_path)) found_dep = [] - for section in tox_source.sections(): - updated_deps = set() - section_deps = tox_source.get(section, "deps", fallback=None) - if section_deps: - for dep in section_deps.split("\n"): - for check_dep in DEPS: - if DEPS[check_dep]["condition"](dep): - has_gen = DEPS[check_dep]["has_gen"](dep) - if has_gen is not None and check_dep not in found_dep: - found_dep.append(check_dep) - updated_deps.add(f"!{has_gen.group()} {DEPS[check_dep]['src']}") - - if not [item for item in updated_deps if pytest_dep in item]: - updated_deps.add(pytest_dep) - updated_deps = '\n'.join(updated_deps) - tox_source[section]["deps"] = f"{tox_source[section]['deps']}\n{updated_deps}" + updated_deps = set() + section_deps = tox_source.get("testenv", "deps", fallback=None) + if section_deps: + tox_dep_filter = ToxDepFilter() + for dep in section_deps.split("\n"): + filtered_dep = tox_dep_filter.filter_dep(dep) + if filtered_dep and filtered_dep not in found_dep: + found_dep.append(filtered_dep) + updated_deps.add( + f"!{filtered_dep['gen_exp']} {filtered_dep['src']}" + ) + + if not [item for item in updated_deps if pytest_dep in item]: + updated_deps.add(pytest_dep) + final_deps = "\n".join(updated_deps) + tox_source["testenv"][ + "deps" + ] = f"{tox_source['testenv']['deps']}\n{final_deps}" with open(ini_path, "w") as f: tox_source.write(f) @@ -163,7 +194,7 @@ class DownstreamRunner: parsed_matrix = yaml_tree for key in self.matrix_schema["matrix"]: parsed_matrix = parsed_matrix[key] - + logger.debug("parsed_matrix: %s", parsed_matrix) if parsed_matrix != yaml_tree: tox_base = self.matrix_schema["tox_cmd_build"]["base"] @@ -171,23 +202,24 @@ class DownstreamRunner: skip_matrices = [] if "include" in self.matrix_schema["matrix"]: for item in parsed_matrix: - if (not item[tox_base].startswith(tox_prefix) or - item[tox_base] in self.matrix_exclude or - not item.get("os", "ubuntu").startswith("ubuntu") + if ( + not item[tox_base].startswith(tox_prefix) + or item[tox_base] in self.matrix_exclude + or not item.get("os", "ubuntu").startswith("ubuntu") ): skip_matrices.append(item) continue - + item["tox_cmd"] = re.sub( self.matrix_schema["tox_cmd_build"]["sub"]["pattern"], self.matrix_schema["tox_cmd_build"]["sub"]["replace"], - item[tox_base] + item[tox_base], ) logger.debug("re.sub: %s", item[tox_base]) - + for matrice in skip_matrices: parsed_matrix.remove(matrice) - + else: new_parsed_matrix = [] for item in parsed_matrix: @@ -196,11 +228,11 @@ class DownstreamRunner: tox_cmd = re.sub( self.matrix_schema["tox_cmd_build"]["sub"]["pattern"], self.matrix_schema["tox_cmd_build"]["sub"]["replace"], - str(item) + str(item), ) new_parsed_matrix.append({"name": tox_cmd, "tox_cmd": tox_cmd}) parsed_matrix = new_parsed_matrix - + return parsed_matrix if self._matrix is None: @@ -209,7 +241,6 @@ class DownstreamRunner: job_yaml = self.yaml_tree["jobs"][job]["strategy"] parsed_matrix = parse_matrix(job_yaml) matrix_items[job] = parsed_matrix - self._matrix = matrix_items logger.debug("matrix: %s", self._matrix) @@ -218,7 +249,7 @@ class DownstreamRunner: @property def steps(self): if self._steps is None: - step_items = {} + step_items: Dict[str, List[Any]] = {} for job in self.job_names: if job not in step_items: step_items[job] = [] @@ -227,17 +258,14 @@ class DownstreamRunner: step_items[job].append(item) self._steps = step_items return self._steps - + def build_run(self): run = {} for job in self.job_names: logger.debug("job_name: %s", job) for matrix in self.matrix[job]: logger.debug("matrix[job]: %s", matrix) - run[matrix["name"]] = [ - "pip install tox", - f"tox -e {matrix['tox_cmd']}" - ] + run[matrix["name"]] = ["pip install tox", f"tox -e {matrix['tox_cmd']}"] logger.debug("built run: %s", run) return run @@ -251,11 +279,8 @@ class DownstreamRunner: cmd = shlex.split(step) logger.info("--> running: '%s'", step) if not self.dry_run: - subprocess.run( - cmd, - encoding="utf-8", - check=True - ) + subprocess.run(cmd, encoding="utf-8", check=True) + if __name__ == "__main__": cli_args = parser.parse_args() @@ -269,4 +294,4 @@ if __name__ == "__main__": dry_run=cli_args.dry_run, ) - runner.run() \ No newline at end of file + runner.run() diff --git a/testing/downstream_testing/entrypoint.sh b/testing/downstream_testing/entrypoint.sh index 1fbf660dc..9b0f35bd6 100755 --- a/testing/downstream_testing/entrypoint.sh +++ b/testing/downstream_testing/entrypoint.sh @@ -3,4 +3,4 @@ python3.9 -m pip install --no-cache-dir pyyaml sh cd /pytest -python3.9 -u -m testing.downstream_testing.downstream_runner $DS_NAME $DS_YAML $DS_JOBS --matrix-exclude $DS_MATRIX_EXCLUDE \ No newline at end of file +python3.9 -u -m testing.downstream_testing.downstream_runner $DS_NAME $DS_YAML $DS_JOBS --matrix-exclude $DS_MATRIX_EXCLUDE