From 5612367f3f944443de17cf7dc392a40da129a575 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Tue, 8 Feb 2022 07:31:54 -0600
Subject: [PATCH 01/59] add working docker-compose elements

---
 testing/downstream_testing/Dockerfile         | 31 ++++++++
 testing/downstream_testing/docker-compose.yml | 78 +++++++++++++++++++
 testing/downstream_testing/entrypoint.sh      |  6 ++
 3 files changed, 115 insertions(+)
 create mode 100644 testing/downstream_testing/Dockerfile
 create mode 100644 testing/downstream_testing/docker-compose.yml
 create mode 100644 testing/downstream_testing/entrypoint.sh

diff --git a/testing/downstream_testing/Dockerfile b/testing/downstream_testing/Dockerfile
new file mode 100644
index 000000000..c40cce6bb
--- /dev/null
+++ b/testing/downstream_testing/Dockerfile
@@ -0,0 +1,31 @@
+FROM sommersoft/pyenv-pytest:latest
+
+COPY entrypoint.sh /entrypoint.sh
+
+# Add mysql apt repository
+RUN set -ex; \
+# gpg: key 5072E1F5: public key "MySQL Release Engineering <mysql-build@oss.oracle.com>" imported
+	key='859BE8D7C586F538430B19C2467B942D3A79BD29'; \
+	export GNUPGHOME="$(mktemp -d)"; \
+	#gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys "$key"; \
+	for server in ha.pool.sks-keyservers.net \
+				  hkp://p80.pool.sks-keyservers.net:80 \
+				  keyserver.ubuntu.com \
+				  hkp://keyserver.ubuntu.com:80 \
+				  pgp.mit.edu; do \
+		gpg --keyserver "$server" --recv-keys "$key" && break || echo "Trying new keyserver..."; \
+	done; \
+	gpg --batch --export "$key" > /etc/apt/trusted.gpg.d/mysql.gpg; \
+	gpgconf --kill all; \
+	rm -rf "$GNUPGHOME"; \
+	apt-key list > /dev/null
+RUN echo 'deb http://repo.mysql.com/apt/debian/ buster mysql-8.0' > /etc/apt/sources.list.d/mysql.list
+
+RUN apt-get update \
+	&& apt-get install --no-install-recommends -y make build-essential libssl-dev zlib1g-dev \
+    libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev xz-utils tk-dev \
+    libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev
+
+RUN apt-get install -y postgresql-client mysql-client
+
+ENTRYPOINT [ "/entrypoint.sh" ]
\ No newline at end of file
diff --git a/testing/downstream_testing/docker-compose.yml b/testing/downstream_testing/docker-compose.yml
new file mode 100644
index 000000000..90bef1dc8
--- /dev/null
+++ b/testing/downstream_testing/docker-compose.yml
@@ -0,0 +1,78 @@
+version: "3"
+services:
+  base_nodb:
+    build: .
+    environment:
+      - DS_NAME
+      - DS_YAML
+      - DS_JOBS
+      - DS_MATRIX_EXCLUDE
+    volumes:
+      - type: bind
+        source: /home/runner/work/pytest/pytest
+        target: /pytest
+        volume:
+          nocopy: true
+    profiles:
+      - nodb
+  
+  base_postgres:
+    build: .
+    environment:
+      - DS_NAME
+      - DS_YAML
+      - DS_JOBS
+      - DS_MATRIX_EXCLUDE
+      - TEST_DB_USER=$USER
+      - TEST_DB_PASSWORD=pytest_djang0
+      - TEST_DB_HOST=postgres_db
+    volumes:
+      - type: bind
+        source: /home/runner/work/pytest/pytest
+        target: /pytest
+        volume:
+          nocopy: true
+    depends_on:
+      - postgres_db
+    profiles:
+      - postgres
+  postgres_db:
+    image: "postgres:latest"
+    environment:
+      - POSTGRES_PASSWORD=pytest_djang0
+      - POSTGRES_USER=$USER
+    volumes:
+      - /etc/passwd:/etc/passwd:ro
+      - ./data/db:/var/lib/postgresql/data
+    profiles:
+      - postgres
+
+  base_mysql:
+    build: .
+    environment:
+      - DS_NAME
+      - DS_YAML
+      - DS_JOBS
+      - DS_MATRIX_EXCLUDE
+      - TEST_DB_USER=root
+      - TEST_DB_PASSWORD=root
+      - TEST_DB_HOST=mysql_db
+    volumes:
+      - type: bind
+        source: /home/runner/work/pytest/pytest
+        target: /pytest
+        volume:
+          nocopy: true
+    depends_on:
+      - mysql_db
+    profiles:
+      - mysql
+  mysql_db:
+    image: "mysql:latest"
+    command: --default-authentication-plugin=mysql_native_password
+    environment:
+      - MYSQL_ROOT_PASSWORD=root
+    volumes:
+      - ./data/db:/var/lib/mysql
+    profiles:
+      - mysql
\ No newline at end of file
diff --git a/testing/downstream_testing/entrypoint.sh b/testing/downstream_testing/entrypoint.sh
new file mode 100644
index 000000000..1fbf660dc
--- /dev/null
+++ b/testing/downstream_testing/entrypoint.sh
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+python3.9 -m pip install --no-cache-dir pyyaml sh
+
+cd /pytest
+python3.9 -u -m testing.downstream_testing.downstream_runner $DS_NAME $DS_YAML $DS_JOBS --matrix-exclude $DS_MATRIX_EXCLUDE
\ No newline at end of file

From c791725fda1e6b2a7c646a9dc742297a567a23c9 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Wed, 23 Mar 2022 09:32:13 -0500
Subject: [PATCH 02/59] trying new approach with static steps and local
 'config'

---
 .github/workflows/downstream_testing.yml      |  62 ++++
 .../downstream_testing/action_schemas.json    |  18 ++
 .../downstream_testing/downstream_runner.py   | 289 ++++++++++++++++++
 3 files changed, 369 insertions(+)
 create mode 100644 .github/workflows/downstream_testing.yml
 create mode 100644 testing/downstream_testing/action_schemas.json
 create mode 100644 testing/downstream_testing/downstream_runner.py

diff --git a/.github/workflows/downstream_testing.yml b/.github/workflows/downstream_testing.yml
new file mode 100644
index 000000000..e292af9ee
--- /dev/null
+++ b/.github/workflows/downstream_testing.yml
@@ -0,0 +1,62 @@
+name: downstream_compose
+
+on:
+  push:
+    branches:
+      - downstream_compose
+
+jobs:
+  downstream-ubuntu:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: "pytest-django"
+            repo: "pytest-dev/pytest-django"
+            docker_profile: "postgres"
+            jobs: "test"
+            workflow_name: "main.yml"
+            matrix_exclude: |
+              linting,docs py39-dj40-mysql_innodb-coverage py36-dj22-sqlite-xdist-coverage py37-dj22-sqlite-xdist-coverage
+              py38-dj32-sqlite-xdist-coverage py38-dj40-sqlite-xdist-coverage py39-djmain-sqlite-coverage 
+              py35-dj22-postgres-pytest54-coverage py35-dj22-sqlite_file-coverage py36-dj32-mysql_myisam-coverage
+          - name: "pytest-django"
+            repo: "pytest-dev/pytest-django"
+            docker_profile: "mysql"
+            jobs: "test"
+            workflow_name: "main.yml"
+            matrix_exclude: |
+              linting,docs py310-dj40-postgres-xdist-coverage py310-dj32-postgres-xdist-coverage
+              py39-dj32-postgres-xdist-coverage py35-dj22-postgres-pytest54-coverage pypy3-dj22-postgres
+              py36-dj22-sqlite-xdist-coverage py37-dj22-sqlite-xdist-coverage py38-dj32-sqlite-xdist-coverage
+              py38-dj40-sqlite-xdist-coverage py39-djmain-sqlite-coverage py35-dj22-sqlite_file-coverage
+          - name: "pytest-django"
+            repo: "pytest-dev/pytest-django"
+            docker_profile: "nodb"
+            jobs: "test"
+            workflow_name: "main.yml"
+            matrix_exclude: |
+              linting,docs py39-dj40-mysql_innodb-coverage py35-dj22-postgres-pytest54-coverage 
+              py39-dj40-mysql_innodb-coverage py35-dj22-postgres-pytest54-coverage py36-dj32-mysql_myisam-coverage
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+    - name: Checkout ${{ matrix.name }}
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+        repository: ${{ matrix.repo }}
+        path: ${{ matrix.name }}
+    - name: Run Downstream Tests
+      run: docker-compose -f ./testing/downstream_testing/docker-compose.yml --profile ${{ matrix.docker_profile }} up --exit-code-from base_${{ matrix.docker_profile }}
+      env:
+        DS_NAME: ${{ matrix.name }}
+        DS_YAML: ./${{ matrix.name }}/.github/workflows/${{ matrix.workflow_name }}
+        DS_JOBS: ${{ matrix.jobs }}
+        DS_MATRIX_EXCLUDE: ${{ matrix.matrix_exclude }}
+        DOCKER_BUILDKIT: 1
\ No newline at end of file
diff --git a/testing/downstream_testing/action_schemas.json b/testing/downstream_testing/action_schemas.json
new file mode 100644
index 000000000..b4fdeb3fa
--- /dev/null
+++ b/testing/downstream_testing/action_schemas.json
@@ -0,0 +1,18 @@
+{
+    "pytest-django": {
+        "matrix": [
+            "matrix",
+            "include"
+        ],
+        "tox_cmd_build": {
+            "base": "name",
+            "prefix": "py",
+            "sub":
+                {
+                    "pattern": "\\-coverage$",
+                    "replace": ""
+                }
+        },
+        "python_version": "python"
+    }
+}
\ No newline at end of file
diff --git a/testing/downstream_testing/downstream_runner.py b/testing/downstream_testing/downstream_runner.py
new file mode 100644
index 000000000..625b33d34
--- /dev/null
+++ b/testing/downstream_testing/downstream_runner.py
@@ -0,0 +1,289 @@
+import argparse
+from collections import namedtuple, UserDict
+import configparser
+import json
+import logging
+import os
+import os.path
+from pprint import pprint
+import re
+import shlex
+import subprocess
+
+import sh
+from sh.contrib import git
+import yaml
+
+logging.basicConfig(format="%(levelname)s | %(module)s.%(funcName)s | %(message)s", level="INFO")
+logger = logging.getLogger(__name__)
+
+
+parser = argparse.ArgumentParser(
+    description="downstream Actions runner"
+)
+parser.add_argument(
+    "repo",
+    #nargs=1,
+    help="Name of the repo."
+)
+parser.add_argument(
+    "source",
+    nargs=1,
+    help="Path to source YAML file."
+)
+parser.add_argument(
+    "jobs",
+    nargs="+",
+    help="Job names to use."
+)
+parser.add_argument(
+    "--matrix-exclude",
+    nargs="*",
+    default=[],
+    help="Exclude these matrix names."
+)
+
+parser.add_argument(
+    "--dry-run",
+    action="store_true",
+    help="Do not run parsed downstream action. Only display the generated command list."
+)
+
+def load_matrix_schema(repo):
+    """ Loads the matrix schema for `repo` """
+    schema = None
+    working_dir = os.getcwd()
+    schema_path = os.path.join(
+        working_dir,
+        "testing",
+        "downstream_testing",
+        "action_schemas.json"
+    )
+    logger.debug("loading schema: %s", schema_path)
+    if os.path.exists(schema_path):
+        with open(schema_path) as schema_file:
+            try:
+                schema = json.load(schema_file)
+            except json.JSONDecodeError:
+                logger.error("failed to read action_schemas.json")
+    else:
+        logger.warning("action_schemas.json not found.")
+
+    if schema is not None and repo in schema:
+        schema = schema[repo]
+        logger.debug("'%s' schema loaded: %s", repo, schema)
+    else:
+        schema = None
+        logger.warning("'%s' schema not found in actions_schema.json", repo)
+
+    return schema
+
+class DownstreamRunner:
+    # TODO: actualize args vs **kwargs
+    def __init__(self, **kwargs):
+        self.yaml_source = kwargs.get("yaml_source", None)
+        if self.yaml_source == None:
+            raise SystemExit("No YAML source provided.")
+        self._yaml_tree = None
+        self.repo = kwargs.get("repo")
+        self.matrix_exclude = kwargs.get("matrix_exclude")
+        self.job_names = kwargs.get("jobs")
+        self._matrix = None
+        self._steps = None
+        self.dry_run = kwargs.get("dry_run", False)
+        self.matrix_schema = load_matrix_schema(self.repo)
+
+
+    @property
+    def yaml_tree(self):
+        """ The YAML tree built from the `yaml_source` file. """
+
+        with open(self.yaml_source) as f:
+            self._yaml_tree = yaml.safe_load(f.read())
+        
+        if self._yaml_tree == None:
+            raise SystemExit("Supplied YAML source failed to parse.")
+        
+        return self._yaml_tree
+
+    def apply_git_patches(self):
+        """ Apply any applicable git patches to the repo."""
+        if self.dry_run:
+            logger.info("Skipping all git patches (dry-run).")
+            return
+
+        working_dir = os.getcwd()
+        patches_dir = os.path.join(working_dir, "testing", "downstream_testing", "patches")
+        repo_dir = os.path.join(working_dir, self.repo)
+
+        for patch in os.listdir(patches_dir):
+            if patch.startswith(self.repo) and patch.endswith(".patch"):
+                patch_full_path = os.path.join(patches_dir, patch)
+                logger.info("Applying patch: %s", patch)
+                if not self.dry_run:
+                    if not os.getcwd() == repo_dir:
+                        os.chdir(repo_dir)
+                    try:
+                        git.config("user.email", "noreply@noreply.no")
+                        git.config("user.name", "pytest-dev")
+                        logger.info("%s", git.am(patch_full_path))
+                    except sh.ErrorReturnCode as err:
+                        logger.info(
+                            "Failed to apply '%s' to '%s': %s",
+                            patch,
+                            self.repo,
+                            str(err.stderr)
+                        )
+            else:
+                logger.info("Skipping patch: %s", patch)
+
+        if not os.getcwd() == working_dir:
+            os.chdir(working_dir)
+
+    def inject_pytest_dep(self):
+        """ Ensure pytest is a dependency in tox.ini to allow us to use the 'local'
+            version of pytest.
+        """
+        ini_path = self.repo + "/tox.ini"
+        pytest_dep = f"pytest @ file://{os.getcwd()}"
+        DEPS = {
+            "pytest": {
+                "src": f"pytest @ file://{os.getcwd()}",
+                "condition": lambda x: x.startswith("pytest") and not x.startswith("pytest-"),
+                "has_gen": lambda x: re.search(r"pytest\w*:", x)
+            },
+            "pytest-rerunfailures": {
+                "src": "pytest-rerunfailures @ git+https://github.com/pytest-dev/pytest-rerunfailures.git",
+                "condition": lambda x: x.startswith("pytest-rerunfailures"),
+                "has_gen": lambda x: re.search(r"pytest-rerunfailures\w*:", x)
+            }
+        }
+        tox_source = configparser.ConfigParser()
+        tox_source.read_file(open(ini_path))
+        #updated_deps = set()
+        found_pytest = False
+        for section in tox_source.sections():
+            updated_deps = set()
+            section_deps = tox_source.get(section, "deps", fallback=None)
+            if section_deps:
+                for dep in section_deps.split("\n"):
+                    for check_dep in DEPS:
+                        if DEPS[check_dep]["condition"](dep):
+                            has_gen = DEPS[check_dep]["has_gen"](dep)
+                            if has_gen is not None and not found_pytest:
+                                found_pytest = True
+                                updated_deps.add(f"!{has_gen.group()} {DEPS[check_dep]['src']}")
+                updated_deps = '\n'.join(updated_deps)
+                tox_source[section]["deps"] = f"{tox_source[section]['deps']}\n{updated_deps}"
+
+        with open(ini_path, "w") as f:
+            tox_source.write(f)
+
+    def __repr__(self):
+        return str(
+            "DownstreamRunner("
+            f"repo={self.repo}, "
+            f"yaml_source={self.yaml_source}, "
+            f"job_names={self.job_names}, "
+            f"matrix={self.matrix}, "
+            f"steps={self.steps}"
+            ")"
+        )
+
+    @property
+    def matrix(self):
+        def parse_matrix(yaml_tree):
+            parsed_matrix = yaml_tree
+            for key in self.matrix_schema["matrix"]:
+                parsed_matrix = parsed_matrix[key]
+                #logger.debug("%s", parsed_matrix)
+            if parsed_matrix != yaml_tree:
+                tox_base = self.matrix_schema["tox_cmd_build"]["base"]
+                tox_prefix = self.matrix_schema["tox_cmd_build"]["prefix"]
+                skip_matrices = []
+                for item in parsed_matrix:
+                    if (not item[tox_base].startswith(tox_prefix) or
+                        item[tox_base] in self.matrix_exclude
+                    ):
+                        skip_matrices.append(item)
+                        continue
+                    item["tox_cmd"] = re.sub(
+                        self.matrix_schema["tox_cmd_build"]["sub"]["pattern"],
+                        self.matrix_schema["tox_cmd_build"]["sub"]["replace"],
+                        item[tox_base]
+                    )
+                    #logger.debug("re.sub: %s", item[tox_base])
+                for matrice in skip_matrices:
+                    parsed_matrix.remove(matrice)
+                
+            return parsed_matrix
+
+        if self._matrix is None:
+            matrix_items = {}
+            for job in self.job_names:
+                job_yaml = self.yaml_tree["jobs"][job]["strategy"]
+                parsed_matrix = parse_matrix(job_yaml)
+                matrix_items[job] = parsed_matrix
+                
+
+            self._matrix = matrix_items
+        logger.debug("matrix: %s", self._matrix)
+        return self._matrix
+
+    @property
+    def steps(self):
+        if self._steps is None:
+            step_items = {}
+            for job in self.job_names:
+                if job not in step_items:
+                    step_items[job] = []
+                for item in self.yaml_tree["jobs"][job]["steps"]:
+                    if "run" in item:
+                        step_items[job].append(item)
+            self._steps = step_items
+        return self._steps
+    
+    def build_run(self):
+        run = {}
+        for job in self.job_names:
+            logger.debug("job_name: %s", job)
+            for matrix in self.matrix[job]:
+                logger.debug("matrix[job]: %s", matrix)
+                run[matrix["name"]] = [
+                    "pip install tox",
+                    f"tox -e {matrix['tox_cmd']}"
+                ]
+
+        logger.debug("built run: %s", run)
+        return run
+
+    def run(self):
+        self.apply_git_patches()
+        self.inject_pytest_dep()
+        run_steps = self.build_run()
+        os.chdir(self.repo)
+        for matrix, steps in run_steps.items():
+            for step in steps:
+                cmd = shlex.split(step)
+                logger.info("--> running: '%s'", step)
+                if not self.dry_run:
+                    subprocess.run(
+                        cmd,
+                        encoding="utf-8",
+                        check=True
+                    )
+
+if __name__ == "__main__":
+    cli_args = parser.parse_args()
+    if cli_args.dry_run:
+        logger.setLevel("DEBUG")
+    runner_args = {
+        "repo": cli_args.repo,
+        "yaml_source": cli_args.source[0],
+        "jobs": cli_args.jobs,
+        "matrix_exclude": cli_args.matrix_exclude,
+        "dry_run": cli_args.dry_run,
+    }
+    runner = DownstreamRunner(**runner_args)
+
+    runner.run()
\ No newline at end of file

From bc4723810d8df7661529c5a2df1fe04f7fa4fada Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Wed, 23 Mar 2022 09:33:02 -0500
Subject: [PATCH 03/59] run on this branch

---
 .github/workflows/downstream_testing.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/downstream_testing.yml b/.github/workflows/downstream_testing.yml
index e292af9ee..44b8e05a1 100644
--- a/.github/workflows/downstream_testing.yml
+++ b/.github/workflows/downstream_testing.yml
@@ -1,9 +1,9 @@
-name: downstream_compose
+name: downstream_testing_2
 
 on:
   push:
     branches:
-      - downstream_compose
+      - downstream_testing_2
 
 jobs:
   downstream-ubuntu:

From d4264666efdb94bd330f23e3841016b6043515b0 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Wed, 23 Mar 2022 09:54:23 -0500
Subject: [PATCH 04/59] chmod entrypoint.sh

---
 testing/downstream_testing/entrypoint.sh | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 testing/downstream_testing/entrypoint.sh

diff --git a/testing/downstream_testing/entrypoint.sh b/testing/downstream_testing/entrypoint.sh
old mode 100644
new mode 100755

From 1be1b31e1afb74135a4fc9721fc86b6d867e13bc Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Wed, 23 Mar 2022 10:00:54 -0500
Subject: [PATCH 05/59] remove git patching

---
 .../downstream_testing/downstream_runner.py   | 35 -------------------
 1 file changed, 35 deletions(-)

diff --git a/testing/downstream_testing/downstream_runner.py b/testing/downstream_testing/downstream_runner.py
index 625b33d34..ee1129679 100644
--- a/testing/downstream_testing/downstream_runner.py
+++ b/testing/downstream_testing/downstream_runner.py
@@ -106,40 +106,6 @@ class DownstreamRunner:
         
         return self._yaml_tree
 
-    def apply_git_patches(self):
-        """ Apply any applicable git patches to the repo."""
-        if self.dry_run:
-            logger.info("Skipping all git patches (dry-run).")
-            return
-
-        working_dir = os.getcwd()
-        patches_dir = os.path.join(working_dir, "testing", "downstream_testing", "patches")
-        repo_dir = os.path.join(working_dir, self.repo)
-
-        for patch in os.listdir(patches_dir):
-            if patch.startswith(self.repo) and patch.endswith(".patch"):
-                patch_full_path = os.path.join(patches_dir, patch)
-                logger.info("Applying patch: %s", patch)
-                if not self.dry_run:
-                    if not os.getcwd() == repo_dir:
-                        os.chdir(repo_dir)
-                    try:
-                        git.config("user.email", "noreply@noreply.no")
-                        git.config("user.name", "pytest-dev")
-                        logger.info("%s", git.am(patch_full_path))
-                    except sh.ErrorReturnCode as err:
-                        logger.info(
-                            "Failed to apply '%s' to '%s': %s",
-                            patch,
-                            self.repo,
-                            str(err.stderr)
-                        )
-            else:
-                logger.info("Skipping patch: %s", patch)
-
-        if not os.getcwd() == working_dir:
-            os.chdir(working_dir)
-
     def inject_pytest_dep(self):
         """ Ensure pytest is a dependency in tox.ini to allow us to use the 'local'
             version of pytest.
@@ -258,7 +224,6 @@ class DownstreamRunner:
         return run
 
     def run(self):
-        self.apply_git_patches()
         self.inject_pytest_dep()
         run_steps = self.build_run()
         os.chdir(self.repo)

From 37e1cd363c0414759c6937a231ae4b7e8b569925 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Wed, 23 Mar 2022 11:54:18 -0500
Subject: [PATCH 06/59] fix matrix exclusions

---
 .github/workflows/downstream_testing.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/downstream_testing.yml b/.github/workflows/downstream_testing.yml
index 44b8e05a1..325a5bf2c 100644
--- a/.github/workflows/downstream_testing.yml
+++ b/.github/workflows/downstream_testing.yml
@@ -39,8 +39,10 @@ jobs:
             jobs: "test"
             workflow_name: "main.yml"
             matrix_exclude: |
-              linting,docs py39-dj40-mysql_innodb-coverage py35-dj22-postgres-pytest54-coverage 
-              py39-dj40-mysql_innodb-coverage py35-dj22-postgres-pytest54-coverage py36-dj32-mysql_myisam-coverage
+              linting,docs y39-dj40-mysql_innodb-coverage py35-dj22-postgres-pytest54-coverage
+              py36-dj32-mysql_myisam-coverage py310-dj40-postgres-xdist-coverage
+              py310-dj32-postgres-xdist-coverage py39-dj32-postgres-xdist-coverage
+              py35-dj22-postgres-pytest54-coverage pypy3-dj22-postgres
 
     steps:
     - uses: actions/checkout@v2

From 28e4833b4edcbcb5b88280e8d5ffebc8d40c6b3c Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Wed, 23 Mar 2022 14:15:50 -0500
Subject: [PATCH 07/59] add pytest-html

---
 .github/workflows/downstream_testing.yml       |  7 +++++++
 testing/downstream_testing/action_schemas.json | 18 +++++++++++++++++-
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/downstream_testing.yml b/.github/workflows/downstream_testing.yml
index 325a5bf2c..a3a7c176c 100644
--- a/.github/workflows/downstream_testing.yml
+++ b/.github/workflows/downstream_testing.yml
@@ -44,6 +44,13 @@ jobs:
               py310-dj32-postgres-xdist-coverage py39-dj32-postgres-xdist-coverage
               py35-dj22-postgres-pytest54-coverage pypy3-dj22-postgres
 
+          - name: "pytest-html"
+            repo: "pytest-dev/pytest-html"
+            docker_profile: "nodb"
+            jobs: "test_python"
+            workflow_name: "tests.yml"
+            matrix_exclude: ""
+
     steps:
     - uses: actions/checkout@v2
       with:
diff --git a/testing/downstream_testing/action_schemas.json b/testing/downstream_testing/action_schemas.json
index b4fdeb3fa..a0954d11f 100644
--- a/testing/downstream_testing/action_schemas.json
+++ b/testing/downstream_testing/action_schemas.json
@@ -9,10 +9,26 @@
             "prefix": "py",
             "sub":
                 {
-                    "pattern": "\\-coverage$",
+                    "pattern": "-coverage$",
                     "replace": ""
                 }
         },
         "python_version": "python"
+    },
+    "pytest-html": {
+        "matrix": [
+            "matrix",
+            "include"
+        ],
+        "tox_cmd_build": {
+            "base": "name",
+            "prefix": "py",
+            "sub":
+                {
+                    "pattern": "(py\\d+)-\\w+",
+                    "replace": "\\1"
+                }
+        },
+        "python_version": "python"
     }
 }
\ No newline at end of file

From 52e9944dad132d5c839d69e07a918a73d40b8c13 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Wed, 23 Mar 2022 21:50:12 -0500
Subject: [PATCH 08/59] finalize django matrix exclusions

---
 .github/workflows/downstream_testing.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/downstream_testing.yml b/.github/workflows/downstream_testing.yml
index a3a7c176c..b015e71ed 100644
--- a/.github/workflows/downstream_testing.yml
+++ b/.github/workflows/downstream_testing.yml
@@ -33,6 +33,7 @@ jobs:
               py39-dj32-postgres-xdist-coverage py35-dj22-postgres-pytest54-coverage pypy3-dj22-postgres
               py36-dj22-sqlite-xdist-coverage py37-dj22-sqlite-xdist-coverage py38-dj32-sqlite-xdist-coverage
               py38-dj40-sqlite-xdist-coverage py39-djmain-sqlite-coverage py35-dj22-sqlite_file-coverage
+              py36-dj32-mysql_myisam-coverage
           - name: "pytest-django"
             repo: "pytest-dev/pytest-django"
             docker_profile: "nodb"
@@ -42,7 +43,7 @@ jobs:
               linting,docs y39-dj40-mysql_innodb-coverage py35-dj22-postgres-pytest54-coverage
               py36-dj32-mysql_myisam-coverage py310-dj40-postgres-xdist-coverage
               py310-dj32-postgres-xdist-coverage py39-dj32-postgres-xdist-coverage
-              py35-dj22-postgres-pytest54-coverage pypy3-dj22-postgres
+              pypy3-dj22-postgres
 
           - name: "pytest-html"
             repo: "pytest-dev/pytest-html"

From 7ed76d51313e05eb0f4d6039bc1d9ed95938cd70 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Wed, 23 Mar 2022 22:15:59 -0500
Subject: [PATCH 09/59] fix typo

---
 .github/workflows/downstream_testing.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/downstream_testing.yml b/.github/workflows/downstream_testing.yml
index b015e71ed..bfc9a2f2e 100644
--- a/.github/workflows/downstream_testing.yml
+++ b/.github/workflows/downstream_testing.yml
@@ -40,7 +40,7 @@ jobs:
             jobs: "test"
             workflow_name: "main.yml"
             matrix_exclude: |
-              linting,docs y39-dj40-mysql_innodb-coverage py35-dj22-postgres-pytest54-coverage
+              linting,docs py39-dj40-mysql_innodb-coverage py35-dj22-postgres-pytest54-coverage
               py36-dj32-mysql_myisam-coverage py310-dj40-postgres-xdist-coverage
               py310-dj32-postgres-xdist-coverage py39-dj32-postgres-xdist-coverage
               pypy3-dj22-postgres

From e74cd734ec05ffb1e29948a3a1abb404832f7f00 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Wed, 23 Mar 2022 22:24:08 -0500
Subject: [PATCH 10/59] last django matrix exclusion. promise.

---
 .github/workflows/downstream_testing.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/downstream_testing.yml b/.github/workflows/downstream_testing.yml
index bfc9a2f2e..265d6fcf5 100644
--- a/.github/workflows/downstream_testing.yml
+++ b/.github/workflows/downstream_testing.yml
@@ -43,7 +43,7 @@ jobs:
               linting,docs py39-dj40-mysql_innodb-coverage py35-dj22-postgres-pytest54-coverage
               py36-dj32-mysql_myisam-coverage py310-dj40-postgres-xdist-coverage
               py310-dj32-postgres-xdist-coverage py39-dj32-postgres-xdist-coverage
-              pypy3-dj22-postgres
+              pypy3-dj22-postgres py36-dj22-sqlite-xdist
 
           - name: "pytest-html"
             repo: "pytest-dev/pytest-html"

From 603de7a25f62eb626203276dfc27eaf2d400a16f Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Wed, 23 Mar 2022 22:32:59 -0500
Subject: [PATCH 11/59] last last one. i hope.

---
 .github/workflows/downstream_testing.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/downstream_testing.yml b/.github/workflows/downstream_testing.yml
index 265d6fcf5..ff34730ee 100644
--- a/.github/workflows/downstream_testing.yml
+++ b/.github/workflows/downstream_testing.yml
@@ -43,7 +43,7 @@ jobs:
               linting,docs py39-dj40-mysql_innodb-coverage py35-dj22-postgres-pytest54-coverage
               py36-dj32-mysql_myisam-coverage py310-dj40-postgres-xdist-coverage
               py310-dj32-postgres-xdist-coverage py39-dj32-postgres-xdist-coverage
-              pypy3-dj22-postgres py36-dj22-sqlite-xdist
+              pypy3-dj22-postgres py36-dj22-sqlite-xdist-coverage
 
           - name: "pytest-html"
             repo: "pytest-dev/pytest-html"

From 9ba1afe022b6cc5b74653e94dba3da23a488fcf9 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Thu, 24 Mar 2022 09:10:56 -0500
Subject: [PATCH 12/59] def the last one. no more py <=3.6

---
 .github/workflows/downstream_testing.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/downstream_testing.yml b/.github/workflows/downstream_testing.yml
index ff34730ee..38989c8d8 100644
--- a/.github/workflows/downstream_testing.yml
+++ b/.github/workflows/downstream_testing.yml
@@ -43,7 +43,7 @@ jobs:
               linting,docs py39-dj40-mysql_innodb-coverage py35-dj22-postgres-pytest54-coverage
               py36-dj32-mysql_myisam-coverage py310-dj40-postgres-xdist-coverage
               py310-dj32-postgres-xdist-coverage py39-dj32-postgres-xdist-coverage
-              pypy3-dj22-postgres py36-dj22-sqlite-xdist-coverage
+              pypy3-dj22-postgres py36-dj22-sqlite-xdist-coverage py35-dj22-sqlite_file-coverage
 
           - name: "pytest-html"
             repo: "pytest-dev/pytest-html"

From f39f6443fee6584da6f59967272dbb0d8386c892 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Fri, 25 Mar 2022 08:43:47 -0500
Subject: [PATCH 13/59] test to ensure docker buildx is used

---
 .github/workflows/downstream_testing.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/downstream_testing.yml b/.github/workflows/downstream_testing.yml
index 38989c8d8..5eb58cbb8 100644
--- a/.github/workflows/downstream_testing.yml
+++ b/.github/workflows/downstream_testing.yml
@@ -69,4 +69,5 @@ jobs:
         DS_YAML: ./${{ matrix.name }}/.github/workflows/${{ matrix.workflow_name }}
         DS_JOBS: ${{ matrix.jobs }}
         DS_MATRIX_EXCLUDE: ${{ matrix.matrix_exclude }}
-        DOCKER_BUILDKIT: 1
\ No newline at end of file
+        DOCKER_BUILDKIT: 1
+        COMPOSE_DOCKER_CLI_BUILD: 1
\ No newline at end of file

From 1b07091100c330fa2d717ac9d54f097a2e00d20a Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Fri, 25 Mar 2022 08:55:05 -0500
Subject: [PATCH 14/59] test without docker buildkit

---
 .github/workflows/downstream_testing.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/downstream_testing.yml b/.github/workflows/downstream_testing.yml
index 5eb58cbb8..7f65d07fd 100644
--- a/.github/workflows/downstream_testing.yml
+++ b/.github/workflows/downstream_testing.yml
@@ -69,5 +69,3 @@ jobs:
         DS_YAML: ./${{ matrix.name }}/.github/workflows/${{ matrix.workflow_name }}
         DS_JOBS: ${{ matrix.jobs }}
         DS_MATRIX_EXCLUDE: ${{ matrix.matrix_exclude }}
-        DOCKER_BUILDKIT: 1
-        COMPOSE_DOCKER_CLI_BUILD: 1
\ No newline at end of file

From 33ababbaed0debace97038cde4f0287c46e3dae6 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Fri, 25 Mar 2022 10:14:21 -0500
Subject: [PATCH 15/59] revert buildkit tests

---
 .github/workflows/downstream_testing.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/downstream_testing.yml b/.github/workflows/downstream_testing.yml
index 7f65d07fd..2508ae300 100644
--- a/.github/workflows/downstream_testing.yml
+++ b/.github/workflows/downstream_testing.yml
@@ -69,3 +69,4 @@ jobs:
         DS_YAML: ./${{ matrix.name }}/.github/workflows/${{ matrix.workflow_name }}
         DS_JOBS: ${{ matrix.jobs }}
         DS_MATRIX_EXCLUDE: ${{ matrix.matrix_exclude }}
+        DOCKER_BUILDKIT: 1

From 4d2cd34b944dfdce03adc71a6909422d1023bd97 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Sun, 27 Mar 2022 09:01:05 -0500
Subject: [PATCH 16/59] add pytest-order

---
 .github/workflows/downstream_testing.yml      |  8 ++++
 .../downstream_testing/action_schemas.json    | 16 +++++++
 .../downstream_testing/downstream_runner.py   | 47 +++++++++++++------
 3 files changed, 56 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/downstream_testing.yml b/.github/workflows/downstream_testing.yml
index 2508ae300..4e5ec535e 100644
--- a/.github/workflows/downstream_testing.yml
+++ b/.github/workflows/downstream_testing.yml
@@ -52,6 +52,14 @@ jobs:
             workflow_name: "tests.yml"
             matrix_exclude: ""
 
+          - name: "pytest-order"
+            repo: "pytest-dev/pytest-order"
+            docker_profile: "nodb"
+            jobs: "test"
+            workflow_name: "pythontests.yml"
+            matrix_exclude: |
+              3.6
+
     steps:
     - uses: actions/checkout@v2
       with:
diff --git a/testing/downstream_testing/action_schemas.json b/testing/downstream_testing/action_schemas.json
index a0954d11f..96972fb40 100644
--- a/testing/downstream_testing/action_schemas.json
+++ b/testing/downstream_testing/action_schemas.json
@@ -30,5 +30,21 @@
                 }
         },
         "python_version": "python"
+    },
+    "pytest-order": {
+        "matrix": [
+            "matrix",
+            "python-version"
+        ],
+        "tox_cmd_build": {
+            "base": "",
+            "prefix": "",
+            "sub":
+                {
+                    "pattern": "(\\d|py\\d)\\.*(\\d+)",
+                    "replace": "py\\1\\2"
+                }
+        },
+        "python_version": "python-version"
     }
 }
\ No newline at end of file
diff --git a/testing/downstream_testing/downstream_runner.py b/testing/downstream_testing/downstream_runner.py
index ee1129679..72e750189 100644
--- a/testing/downstream_testing/downstream_runner.py
+++ b/testing/downstream_testing/downstream_runner.py
@@ -162,26 +162,43 @@ class DownstreamRunner:
             parsed_matrix = yaml_tree
             for key in self.matrix_schema["matrix"]:
                 parsed_matrix = parsed_matrix[key]
-                #logger.debug("%s", parsed_matrix)
+                
+            logger.debug("parsed_matrix: %s", parsed_matrix)
             if parsed_matrix != yaml_tree:
                 tox_base = self.matrix_schema["tox_cmd_build"]["base"]
                 tox_prefix = self.matrix_schema["tox_cmd_build"]["prefix"]
                 skip_matrices = []
-                for item in parsed_matrix:
-                    if (not item[tox_base].startswith(tox_prefix) or
-                        item[tox_base] in self.matrix_exclude
-                    ):
-                        skip_matrices.append(item)
-                        continue
-                    item["tox_cmd"] = re.sub(
-                        self.matrix_schema["tox_cmd_build"]["sub"]["pattern"],
-                        self.matrix_schema["tox_cmd_build"]["sub"]["replace"],
-                        item[tox_base]
-                    )
-                    #logger.debug("re.sub: %s", item[tox_base])
-                for matrice in skip_matrices:
-                    parsed_matrix.remove(matrice)
+                if isinstance(parsed_matrix, dict):
+                    for item in parsed_matrix:
+                        if (not item[tox_base].startswith(tox_prefix) or
+                            item[tox_base] in self.matrix_exclude
+                        ):
+                            skip_matrices.append(item)
+                            continue
+                        
+                        item["tox_cmd"] = re.sub(
+                            self.matrix_schema["tox_cmd_build"]["sub"]["pattern"],
+                            self.matrix_schema["tox_cmd_build"]["sub"]["replace"],
+                            item[tox_base]
+                        )
+                        #logger.debug("re.sub: %s", item[tox_base])
+                    
+                    for matrice in skip_matrices:
+                        parsed_matrix.remove(matrice)
                 
+                elif isinstance(parsed_matrix, list):
+                    new_parsed_matrix = []
+                    for item in parsed_matrix:
+                        if str(item) in self.matrix_exclude:
+                            continue
+                        tox_cmd = re.sub(
+                            self.matrix_schema["tox_cmd_build"]["sub"]["pattern"],
+                            self.matrix_schema["tox_cmd_build"]["sub"]["replace"],
+                            str(item)
+                        )
+                        new_parsed_matrix.append({"name": tox_cmd, "tox_cmd": tox_cmd})
+                    parsed_matrix = new_parsed_matrix
+                        
             return parsed_matrix
 
         if self._matrix is None:

From 71fa7faa60c13e54f186391a9e22674af3946cbc Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Sun, 27 Mar 2022 09:58:57 -0500
Subject: [PATCH 17/59] attempt to set COV_CMD

---
 testing/downstream_testing/action_schemas.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/testing/downstream_testing/action_schemas.json b/testing/downstream_testing/action_schemas.json
index 96972fb40..51904139c 100644
--- a/testing/downstream_testing/action_schemas.json
+++ b/testing/downstream_testing/action_schemas.json
@@ -42,7 +42,7 @@
             "sub":
                 {
                     "pattern": "(\\d|py\\d)\\.*(\\d+)",
-                    "replace": "py\\1\\2"
+                    "replace": "py\\1\\2 COV_CMD=''"
                 }
         },
         "python_version": "python-version"

From d6e808d5f11b123f02d37013036b5c280d557347 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Sun, 27 Mar 2022 11:27:33 -0500
Subject: [PATCH 18/59] COV_CMD attempt 2

---
 .github/workflows/downstream_testing.yml       | 1 +
 testing/downstream_testing/action_schemas.json | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/downstream_testing.yml b/.github/workflows/downstream_testing.yml
index 4e5ec535e..17902eeb1 100644
--- a/.github/workflows/downstream_testing.yml
+++ b/.github/workflows/downstream_testing.yml
@@ -78,3 +78,4 @@ jobs:
         DS_JOBS: ${{ matrix.jobs }}
         DS_MATRIX_EXCLUDE: ${{ matrix.matrix_exclude }}
         DOCKER_BUILDKIT: 1
+        COV_CMD: ""
diff --git a/testing/downstream_testing/action_schemas.json b/testing/downstream_testing/action_schemas.json
index 51904139c..96972fb40 100644
--- a/testing/downstream_testing/action_schemas.json
+++ b/testing/downstream_testing/action_schemas.json
@@ -42,7 +42,7 @@
             "sub":
                 {
                     "pattern": "(\\d|py\\d)\\.*(\\d+)",
-                    "replace": "py\\1\\2 COV_CMD=''"
+                    "replace": "py\\1\\2"
                 }
         },
         "python_version": "python-version"

From 0b12f28a63fed23d7d7ffd1278bcaab17cce93b2 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Sun, 27 Mar 2022 21:42:59 -0500
Subject: [PATCH 19/59] better handle the different matrix types

---
 testing/downstream_testing/downstream_runner.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/testing/downstream_testing/downstream_runner.py b/testing/downstream_testing/downstream_runner.py
index 72e750189..bf9d9d96e 100644
--- a/testing/downstream_testing/downstream_runner.py
+++ b/testing/downstream_testing/downstream_runner.py
@@ -168,10 +168,11 @@ class DownstreamRunner:
                 tox_base = self.matrix_schema["tox_cmd_build"]["base"]
                 tox_prefix = self.matrix_schema["tox_cmd_build"]["prefix"]
                 skip_matrices = []
-                if isinstance(parsed_matrix, dict):
+                if "include" in self.matrix_schema["matrix"]:
                     for item in parsed_matrix:
                         if (not item[tox_base].startswith(tox_prefix) or
-                            item[tox_base] in self.matrix_exclude
+                            item[tox_base] in self.matrix_exclude or
+                            not item.get("os", "").startswith("ubuntu")
                         ):
                             skip_matrices.append(item)
                             continue
@@ -181,12 +182,12 @@ class DownstreamRunner:
                             self.matrix_schema["tox_cmd_build"]["sub"]["replace"],
                             item[tox_base]
                         )
-                        #logger.debug("re.sub: %s", item[tox_base])
+                        logger.debug("re.sub: %s", item[tox_base])
                     
                     for matrice in skip_matrices:
                         parsed_matrix.remove(matrice)
                 
-                elif isinstance(parsed_matrix, list):
+                else:
                     new_parsed_matrix = []
                     for item in parsed_matrix:
                         if str(item) in self.matrix_exclude:

From c0ac0ffb6c5f12d648d12b4ea78362e18e40d3e2 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Sun, 27 Mar 2022 21:43:22 -0500
Subject: [PATCH 20/59] exclude py36 in pytest-html

---
 .github/workflows/downstream_testing.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/downstream_testing.yml b/.github/workflows/downstream_testing.yml
index 17902eeb1..b912be6b8 100644
--- a/.github/workflows/downstream_testing.yml
+++ b/.github/workflows/downstream_testing.yml
@@ -50,7 +50,7 @@ jobs:
             docker_profile: "nodb"
             jobs: "test_python"
             workflow_name: "tests.yml"
-            matrix_exclude: ""
+            matrix_exclude: py36-ubuntu
 
           - name: "pytest-order"
             repo: "pytest-dev/pytest-order"

From eff1ed5f1ea3a05985bdcc479e466a04723240f2 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Mon, 28 Mar 2022 08:23:46 -0500
Subject: [PATCH 21/59] make COV_CMD available to base_nodb container

---
 testing/downstream_testing/docker-compose.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/testing/downstream_testing/docker-compose.yml b/testing/downstream_testing/docker-compose.yml
index 90bef1dc8..fa21ba034 100644
--- a/testing/downstream_testing/docker-compose.yml
+++ b/testing/downstream_testing/docker-compose.yml
@@ -7,6 +7,7 @@ services:
       - DS_YAML
       - DS_JOBS
       - DS_MATRIX_EXCLUDE
+      - COV_CMD
     volumes:
       - type: bind
         source: /home/runner/work/pytest/pytest

From 8314dc6fb132e3c681a679900e8295d51d7e7aeb Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Mon, 28 Mar 2022 09:33:20 -0500
Subject: [PATCH 22/59] fix tox.ini dep resolution

---
 testing/downstream_testing/downstream_runner.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/testing/downstream_testing/downstream_runner.py b/testing/downstream_testing/downstream_runner.py
index bf9d9d96e..df3e215ff 100644
--- a/testing/downstream_testing/downstream_runner.py
+++ b/testing/downstream_testing/downstream_runner.py
@@ -122,12 +122,17 @@ class DownstreamRunner:
                 "src": "pytest-rerunfailures @ git+https://github.com/pytest-dev/pytest-rerunfailures.git",
                 "condition": lambda x: x.startswith("pytest-rerunfailures"),
                 "has_gen": lambda x: re.search(r"pytest-rerunfailures\w*:", x)
+            },
+            "pytest-xdist": {
+                "src": "pytest-xdist",
+                "condition": lambda x: x.startswith("pytest{") and x.endswith("pytest-xdist"),
+                "has_gen": lambda x: re.search(r"pytest\{.*\,7\d.*\}:", x)
             }
         }
         tox_source = configparser.ConfigParser()
         tox_source.read_file(open(ini_path))
         #updated_deps = set()
-        found_pytest = False
+        found_dep = []
         for section in tox_source.sections():
             updated_deps = set()
             section_deps = tox_source.get(section, "deps", fallback=None)
@@ -136,8 +141,9 @@ class DownstreamRunner:
                     for check_dep in DEPS:
                         if DEPS[check_dep]["condition"](dep):
                             has_gen = DEPS[check_dep]["has_gen"](dep)
-                            if has_gen is not None and not found_pytest:
-                                found_pytest = True
+                            if has_gen is not None and check_dep not in found_dep:
+                                #found_pytest = True
+                                found_dep.append(check_dep)
                                 updated_deps.add(f"!{has_gen.group()} {DEPS[check_dep]['src']}")
                 updated_deps = '\n'.join(updated_deps)
                 tox_source[section]["deps"] = f"{tox_source[section]['deps']}\n{updated_deps}"

From f8a47b305478aea9dafd6f15e8bac3734e68ca18 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Mon, 28 Mar 2022 16:49:29 -0500
Subject: [PATCH 23/59] handle a matrix not having 'os' defined

---
 testing/downstream_testing/downstream_runner.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/testing/downstream_testing/downstream_runner.py b/testing/downstream_testing/downstream_runner.py
index df3e215ff..c9368c8d1 100644
--- a/testing/downstream_testing/downstream_runner.py
+++ b/testing/downstream_testing/downstream_runner.py
@@ -178,7 +178,7 @@ class DownstreamRunner:
                     for item in parsed_matrix:
                         if (not item[tox_base].startswith(tox_prefix) or
                             item[tox_base] in self.matrix_exclude or
-                            not item.get("os", "").startswith("ubuntu")
+                            not item.get("os", "ubuntu").startswith("ubuntu")
                         ):
                             skip_matrices.append(item)
                             continue

From 542be267d81a0eb770b8d8d26fce3745b1b9056d Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Mon, 28 Mar 2022 21:57:16 -0500
Subject: [PATCH 24/59] cover edge case of pytest not being a dep in tox.ini

---
 testing/downstream_testing/downstream_runner.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/testing/downstream_testing/downstream_runner.py b/testing/downstream_testing/downstream_runner.py
index c9368c8d1..92f901364 100644
--- a/testing/downstream_testing/downstream_runner.py
+++ b/testing/downstream_testing/downstream_runner.py
@@ -131,7 +131,6 @@ class DownstreamRunner:
         }
         tox_source = configparser.ConfigParser()
         tox_source.read_file(open(ini_path))
-        #updated_deps = set()
         found_dep = []
         for section in tox_source.sections():
             updated_deps = set()
@@ -142,9 +141,11 @@ class DownstreamRunner:
                         if DEPS[check_dep]["condition"](dep):
                             has_gen = DEPS[check_dep]["has_gen"](dep)
                             if has_gen is not None and check_dep not in found_dep:
-                                #found_pytest = True
                                 found_dep.append(check_dep)
                                 updated_deps.add(f"!{has_gen.group()} {DEPS[check_dep]['src']}")
+                
+                if not [item for item in updated_deps if pytest_dep in item]:
+                    updated_deps.add(pytest_dep)
                 updated_deps = '\n'.join(updated_deps)
                 tox_source[section]["deps"] = f"{tox_source[section]['deps']}\n{updated_deps}"
 

From d396741d7061cfa7d540a6248e63ad4b5ce28ebf Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Sat, 2 Apr 2022 08:35:35 -0500
Subject: [PATCH 25/59] remove apt deps leftover for pyenv installs

---
 testing/downstream_testing/Dockerfile | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/testing/downstream_testing/Dockerfile b/testing/downstream_testing/Dockerfile
index c40cce6bb..4474f1d93 100644
--- a/testing/downstream_testing/Dockerfile
+++ b/testing/downstream_testing/Dockerfile
@@ -22,10 +22,6 @@ RUN set -ex; \
 RUN echo 'deb http://repo.mysql.com/apt/debian/ buster mysql-8.0' > /etc/apt/sources.list.d/mysql.list
 
 RUN apt-get update \
-	&& apt-get install --no-install-recommends -y make build-essential libssl-dev zlib1g-dev \
-    libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev xz-utils tk-dev \
-    libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev
-
-RUN apt-get install -y postgresql-client mysql-client
+	&& apt-get install -y postgresql-client mysql-client
 
 ENTRYPOINT [ "/entrypoint.sh" ]
\ No newline at end of file

From 76592c888d93539b94e48ec816b1b66124015d61 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Sat, 2 Apr 2022 08:44:07 -0500
Subject: [PATCH 26/59] remove unsused imports

---
 testing/downstream_testing/downstream_runner.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/testing/downstream_testing/downstream_runner.py b/testing/downstream_testing/downstream_runner.py
index 92f901364..d457ed5ce 100644
--- a/testing/downstream_testing/downstream_runner.py
+++ b/testing/downstream_testing/downstream_runner.py
@@ -1,17 +1,13 @@
 import argparse
-from collections import namedtuple, UserDict
 import configparser
 import json
 import logging
 import os
 import os.path
-from pprint import pprint
 import re
 import shlex
 import subprocess
 
-import sh
-from sh.contrib import git
 import yaml
 
 logging.basicConfig(format="%(levelname)s | %(module)s.%(funcName)s | %(message)s", level="INFO")

From b2c47bde102d6e276a76e36012aeb8fe09877ff2 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Sat, 2 Apr 2022 09:18:17 -0500
Subject: [PATCH 27/59] finalize signature of DownstreamRunner; remove **kwargs
 usage

---
 .../downstream_testing/downstream_runner.py   | 31 +++++++++----------
 1 file changed, 14 insertions(+), 17 deletions(-)

diff --git a/testing/downstream_testing/downstream_runner.py b/testing/downstream_testing/downstream_runner.py
index d457ed5ce..6483cbb6b 100644
--- a/testing/downstream_testing/downstream_runner.py
+++ b/testing/downstream_testing/downstream_runner.py
@@ -75,18 +75,16 @@ def load_matrix_schema(repo):
     return schema
 
 class DownstreamRunner:
-    # TODO: actualize args vs **kwargs
-    def __init__(self, **kwargs):
-        self.yaml_source = kwargs.get("yaml_source", None)
-        if self.yaml_source == None:
-            raise SystemExit("No YAML source provided.")
+    def __init__(self, repo, yaml_source, jobs, matrix_exclude=None, dry_run=False):
+        self.repo = repo
+        self.yaml_source = yaml_source
+        self.matrix_exclude = matrix_exclude
+        self.job_names = jobs
+        self.dry_run = dry_run
+
         self._yaml_tree = None
-        self.repo = kwargs.get("repo")
-        self.matrix_exclude = kwargs.get("matrix_exclude")
-        self.job_names = kwargs.get("jobs")
         self._matrix = None
         self._steps = None
-        self.dry_run = kwargs.get("dry_run", False)
         self.matrix_schema = load_matrix_schema(self.repo)
 
 
@@ -263,13 +261,12 @@ if __name__ == "__main__":
     cli_args = parser.parse_args()
     if cli_args.dry_run:
         logger.setLevel("DEBUG")
-    runner_args = {
-        "repo": cli_args.repo,
-        "yaml_source": cli_args.source[0],
-        "jobs": cli_args.jobs,
-        "matrix_exclude": cli_args.matrix_exclude,
-        "dry_run": cli_args.dry_run,
-    }
-    runner = DownstreamRunner(**runner_args)
+    runner = DownstreamRunner(
+        cli_args.repo,
+        cli_args.source[0],
+        cli_args.jobs,
+        matrix_exclude=cli_args.matrix_exclude,
+        dry_run=cli_args.dry_run,
+    )
 
     runner.run()
\ No newline at end of file

From d2807824d2b3e60d936165eb237c323a14086e66 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Sun, 3 Apr 2022 08:53:22 -0500
Subject: [PATCH 28/59] added pre-commit; linted; made mypy happy with tox.ini
 injection re-factor; started type annotations

---
 .github/workflows/downstream_testing.yml      |   2 +-
 .pre-commit-config.yaml                       |   5 +-
 testing/downstream_testing/Dockerfile         |   2 +-
 .../downstream_testing/action_schemas.json    |   2 +-
 testing/downstream_testing/docker-compose.yml |   4 +-
 .../downstream_testing/downstream_runner.py   | 211 ++++++++++--------
 testing/downstream_testing/entrypoint.sh      |   2 +-
 7 files changed, 127 insertions(+), 101 deletions(-)

diff --git a/.github/workflows/downstream_testing.yml b/.github/workflows/downstream_testing.yml
index b912be6b8..a57a9d595 100644
--- a/.github/workflows/downstream_testing.yml
+++ b/.github/workflows/downstream_testing.yml
@@ -21,7 +21,7 @@ jobs:
             workflow_name: "main.yml"
             matrix_exclude: |
               linting,docs py39-dj40-mysql_innodb-coverage py36-dj22-sqlite-xdist-coverage py37-dj22-sqlite-xdist-coverage
-              py38-dj32-sqlite-xdist-coverage py38-dj40-sqlite-xdist-coverage py39-djmain-sqlite-coverage 
+              py38-dj32-sqlite-xdist-coverage py38-dj40-sqlite-xdist-coverage py39-djmain-sqlite-coverage
               py35-dj22-postgres-pytest54-coverage py35-dj22-sqlite_file-coverage py36-dj32-mysql_myisam-coverage
           - name: "pytest-django"
             repo: "pytest-dev/pytest-django"
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 261c525c8..a028b0652 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,6 +1,6 @@
 repos:
 -   repo: https://github.com/psf/black
-    rev: 22.1.0
+    rev: 22.3.0
     hooks:
     -   id: black
         args: [--safe, --quiet]
@@ -56,7 +56,7 @@ repos:
     hooks:
     -   id: python-use-type-annotations
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.940
+    rev: v0.942
     hooks:
     -   id: mypy
         files: ^(src/|testing/)
@@ -69,6 +69,7 @@ repos:
           - tomli
           - types-atomicwrites
           - types-pkg_resources
+          - types-PyYAML
 -   repo: local
     hooks:
     -   id: rst
diff --git a/testing/downstream_testing/Dockerfile b/testing/downstream_testing/Dockerfile
index 4474f1d93..aaa3057f9 100644
--- a/testing/downstream_testing/Dockerfile
+++ b/testing/downstream_testing/Dockerfile
@@ -24,4 +24,4 @@ RUN echo 'deb http://repo.mysql.com/apt/debian/ buster mysql-8.0' > /etc/apt/sou
 RUN apt-get update \
 	&& apt-get install -y postgresql-client mysql-client
 
-ENTRYPOINT [ "/entrypoint.sh" ]
\ No newline at end of file
+ENTRYPOINT [ "/entrypoint.sh" ]
diff --git a/testing/downstream_testing/action_schemas.json b/testing/downstream_testing/action_schemas.json
index 96972fb40..7f16f2180 100644
--- a/testing/downstream_testing/action_schemas.json
+++ b/testing/downstream_testing/action_schemas.json
@@ -47,4 +47,4 @@
         },
         "python_version": "python-version"
     }
-}
\ No newline at end of file
+}
diff --git a/testing/downstream_testing/docker-compose.yml b/testing/downstream_testing/docker-compose.yml
index fa21ba034..ee3fcd5e9 100644
--- a/testing/downstream_testing/docker-compose.yml
+++ b/testing/downstream_testing/docker-compose.yml
@@ -16,7 +16,7 @@ services:
           nocopy: true
     profiles:
       - nodb
-  
+
   base_postgres:
     build: .
     environment:
@@ -76,4 +76,4 @@ services:
     volumes:
       - ./data/db:/var/lib/mysql
     profiles:
-      - mysql
\ No newline at end of file
+      - mysql
diff --git a/testing/downstream_testing/downstream_runner.py b/testing/downstream_testing/downstream_runner.py
index 6483cbb6b..feefcf081 100644
--- a/testing/downstream_testing/downstream_runner.py
+++ b/testing/downstream_testing/downstream_runner.py
@@ -2,58 +2,45 @@ import argparse
 import configparser
 import json
 import logging
-import os
 import os.path
 import re
 import shlex
 import subprocess
+from collections import UserDict
+from typing import Any
+from typing import Dict
+from typing import List
+from typing import Optional
 
 import yaml
 
-logging.basicConfig(format="%(levelname)s | %(module)s.%(funcName)s | %(message)s", level="INFO")
+logging.basicConfig(
+    format="%(levelname)s | %(module)s.%(funcName)s | %(message)s", level="INFO"
+)
 logger = logging.getLogger(__name__)
 
 
-parser = argparse.ArgumentParser(
-    description="downstream Actions runner"
-)
+parser = argparse.ArgumentParser(description="downstream Actions runner")
+parser.add_argument("repo", help="Name of the repo.")
+parser.add_argument("source", nargs=1, help="Path to source YAML file.")
+parser.add_argument("jobs", nargs="+", help="Job names to use.")
 parser.add_argument(
-    "repo",
-    #nargs=1,
-    help="Name of the repo."
-)
-parser.add_argument(
-    "source",
-    nargs=1,
-    help="Path to source YAML file."
-)
-parser.add_argument(
-    "jobs",
-    nargs="+",
-    help="Job names to use."
-)
-parser.add_argument(
-    "--matrix-exclude",
-    nargs="*",
-    default=[],
-    help="Exclude these matrix names."
+    "--matrix-exclude", nargs="*", default=[], help="Exclude these matrix names."
 )
 
 parser.add_argument(
     "--dry-run",
     action="store_true",
-    help="Do not run parsed downstream action. Only display the generated command list."
+    help="Do not run parsed downstream action. Only display the generated command list.",
 )
 
+
 def load_matrix_schema(repo):
-    """ Loads the matrix schema for `repo` """
+    """Loads the matrix schema for `repo`"""
     schema = None
     working_dir = os.getcwd()
     schema_path = os.path.join(
-        working_dir,
-        "testing",
-        "downstream_testing",
-        "action_schemas.json"
+        working_dir, "testing", "downstream_testing", "action_schemas.json"
     )
     logger.debug("loading schema: %s", schema_path)
     if os.path.exists(schema_path):
@@ -74,6 +61,66 @@ def load_matrix_schema(repo):
 
     return schema
 
+
+TOX_DEP_FILTERS = {
+    "pytest": {
+        "src": f"pytest @ file://{os.getcwd()}",
+        "condition": r"^pytest(?!\-)",
+        "has_gen": r"pytest\w*",
+    },
+    "pytest-rerunfailures": {
+        "src": "pytest-rerunfailures @ git+https://github.com/pytest-dev/pytest-rerunfailures.git",
+        "condition": r"^pytest-rerunfailures.*",
+        "has_gen": r"pytest-rerunfailures\w*:",
+    },
+    "pytest-xdist": {
+        "src": "pytest-xdist",
+        "condition": r"^pytest.*pytest-xdist",
+        "has_gen": r"pytest\{.*\,7\d.*\}",
+    },
+}
+
+
+class ToxDepFilter(UserDict[Any, Any]):
+    def __init__(self) -> None:
+        self.data = TOX_DEP_FILTERS
+
+    def matches_condition(self, match: str) -> Optional[str]:
+        """Checks if `match` matches any conditions"""
+        match_found = None
+        for key, val in self.data.items():
+            logger.debug(
+                "matches_condition: %s:%s -> %s",
+                key,
+                val,
+                re.search(val["condition"], match),
+            )
+            if re.search(val["condition"], match):
+                match_found = key
+                break
+
+        return match_found
+
+    def matches_gen_exp(self, dep: str, match: str) -> Optional[Any]:
+        """Checks if `match` matches `dep`['has_gen'] condition."""
+        logger.debug("matches_gen_exp: %s", re.match(self.data[dep]["has_gen"], match))
+        return re.match(self.data[dep]["has_gen"], match)
+
+    def filter_dep(self, match: str) -> Optional[Dict[Any, Any]]:
+        """Filters `match` based on conditions and returns the `src` dependency."""
+        filtered_match = None
+        dep_condition = self.matches_condition(match)
+        if dep_condition is not None:
+            dep_gen_exp = self.matches_gen_exp(dep_condition, match)
+            if dep_gen_exp:
+                filtered_match = {
+                    "src": self.data[dep_condition]["src"],
+                    "gen_exp": dep_gen_exp,
+                }
+        logger.debug("filter_dep: %s", filtered_match)
+        return filtered_match
+
+
 class DownstreamRunner:
     def __init__(self, repo, yaml_source, jobs, matrix_exclude=None, dry_run=False):
         self.repo = repo
@@ -87,61 +134,45 @@ class DownstreamRunner:
         self._steps = None
         self.matrix_schema = load_matrix_schema(self.repo)
 
-
     @property
     def yaml_tree(self):
-        """ The YAML tree built from the `yaml_source` file. """
+        """The YAML tree built from the `yaml_source` file."""
 
         with open(self.yaml_source) as f:
             self._yaml_tree = yaml.safe_load(f.read())
-        
-        if self._yaml_tree == None:
+
+        if self._yaml_tree is None:
             raise SystemExit("Supplied YAML source failed to parse.")
-        
+
         return self._yaml_tree
 
     def inject_pytest_dep(self):
-        """ Ensure pytest is a dependency in tox.ini to allow us to use the 'local'
-            version of pytest.
+        """Ensure pytest is a dependency in tox.ini to allow us to use the 'local'
+        version of pytest.
         """
         ini_path = self.repo + "/tox.ini"
         pytest_dep = f"pytest @ file://{os.getcwd()}"
-        DEPS = {
-            "pytest": {
-                "src": f"pytest @ file://{os.getcwd()}",
-                "condition": lambda x: x.startswith("pytest") and not x.startswith("pytest-"),
-                "has_gen": lambda x: re.search(r"pytest\w*:", x)
-            },
-            "pytest-rerunfailures": {
-                "src": "pytest-rerunfailures @ git+https://github.com/pytest-dev/pytest-rerunfailures.git",
-                "condition": lambda x: x.startswith("pytest-rerunfailures"),
-                "has_gen": lambda x: re.search(r"pytest-rerunfailures\w*:", x)
-            },
-            "pytest-xdist": {
-                "src": "pytest-xdist",
-                "condition": lambda x: x.startswith("pytest{") and x.endswith("pytest-xdist"),
-                "has_gen": lambda x: re.search(r"pytest\{.*\,7\d.*\}:", x)
-            }
-        }
         tox_source = configparser.ConfigParser()
         tox_source.read_file(open(ini_path))
         found_dep = []
-        for section in tox_source.sections():
-            updated_deps = set()
-            section_deps = tox_source.get(section, "deps", fallback=None)
-            if section_deps:
-                for dep in section_deps.split("\n"):
-                    for check_dep in DEPS:
-                        if DEPS[check_dep]["condition"](dep):
-                            has_gen = DEPS[check_dep]["has_gen"](dep)
-                            if has_gen is not None and check_dep not in found_dep:
-                                found_dep.append(check_dep)
-                                updated_deps.add(f"!{has_gen.group()} {DEPS[check_dep]['src']}")
-                
-                if not [item for item in updated_deps if pytest_dep in item]:
-                    updated_deps.add(pytest_dep)
-                updated_deps = '\n'.join(updated_deps)
-                tox_source[section]["deps"] = f"{tox_source[section]['deps']}\n{updated_deps}"
+        updated_deps = set()
+        section_deps = tox_source.get("testenv", "deps", fallback=None)
+        if section_deps:
+            tox_dep_filter = ToxDepFilter()
+            for dep in section_deps.split("\n"):
+                filtered_dep = tox_dep_filter.filter_dep(dep)
+                if filtered_dep and filtered_dep not in found_dep:
+                    found_dep.append(filtered_dep)
+                    updated_deps.add(
+                        f"!{filtered_dep['gen_exp']} {filtered_dep['src']}"
+                    )
+
+            if not [item for item in updated_deps if pytest_dep in item]:
+                updated_deps.add(pytest_dep)
+            final_deps = "\n".join(updated_deps)
+            tox_source["testenv"][
+                "deps"
+            ] = f"{tox_source['testenv']['deps']}\n{final_deps}"
 
         with open(ini_path, "w") as f:
             tox_source.write(f)
@@ -163,7 +194,7 @@ class DownstreamRunner:
             parsed_matrix = yaml_tree
             for key in self.matrix_schema["matrix"]:
                 parsed_matrix = parsed_matrix[key]
-                
+
             logger.debug("parsed_matrix: %s", parsed_matrix)
             if parsed_matrix != yaml_tree:
                 tox_base = self.matrix_schema["tox_cmd_build"]["base"]
@@ -171,23 +202,24 @@ class DownstreamRunner:
                 skip_matrices = []
                 if "include" in self.matrix_schema["matrix"]:
                     for item in parsed_matrix:
-                        if (not item[tox_base].startswith(tox_prefix) or
-                            item[tox_base] in self.matrix_exclude or
-                            not item.get("os", "ubuntu").startswith("ubuntu")
+                        if (
+                            not item[tox_base].startswith(tox_prefix)
+                            or item[tox_base] in self.matrix_exclude
+                            or not item.get("os", "ubuntu").startswith("ubuntu")
                         ):
                             skip_matrices.append(item)
                             continue
-                        
+
                         item["tox_cmd"] = re.sub(
                             self.matrix_schema["tox_cmd_build"]["sub"]["pattern"],
                             self.matrix_schema["tox_cmd_build"]["sub"]["replace"],
-                            item[tox_base]
+                            item[tox_base],
                         )
                         logger.debug("re.sub: %s", item[tox_base])
-                    
+
                     for matrice in skip_matrices:
                         parsed_matrix.remove(matrice)
-                
+
                 else:
                     new_parsed_matrix = []
                     for item in parsed_matrix:
@@ -196,11 +228,11 @@ class DownstreamRunner:
                         tox_cmd = re.sub(
                             self.matrix_schema["tox_cmd_build"]["sub"]["pattern"],
                             self.matrix_schema["tox_cmd_build"]["sub"]["replace"],
-                            str(item)
+                            str(item),
                         )
                         new_parsed_matrix.append({"name": tox_cmd, "tox_cmd": tox_cmd})
                     parsed_matrix = new_parsed_matrix
-                        
+
             return parsed_matrix
 
         if self._matrix is None:
@@ -209,7 +241,6 @@ class DownstreamRunner:
                 job_yaml = self.yaml_tree["jobs"][job]["strategy"]
                 parsed_matrix = parse_matrix(job_yaml)
                 matrix_items[job] = parsed_matrix
-                
 
             self._matrix = matrix_items
         logger.debug("matrix: %s", self._matrix)
@@ -218,7 +249,7 @@ class DownstreamRunner:
     @property
     def steps(self):
         if self._steps is None:
-            step_items = {}
+            step_items: Dict[str, List[Any]] = {}
             for job in self.job_names:
                 if job not in step_items:
                     step_items[job] = []
@@ -227,17 +258,14 @@ class DownstreamRunner:
                         step_items[job].append(item)
             self._steps = step_items
         return self._steps
-    
+
     def build_run(self):
         run = {}
         for job in self.job_names:
             logger.debug("job_name: %s", job)
             for matrix in self.matrix[job]:
                 logger.debug("matrix[job]: %s", matrix)
-                run[matrix["name"]] = [
-                    "pip install tox",
-                    f"tox -e {matrix['tox_cmd']}"
-                ]
+                run[matrix["name"]] = ["pip install tox", f"tox -e {matrix['tox_cmd']}"]
 
         logger.debug("built run: %s", run)
         return run
@@ -251,11 +279,8 @@ class DownstreamRunner:
                 cmd = shlex.split(step)
                 logger.info("--> running: '%s'", step)
                 if not self.dry_run:
-                    subprocess.run(
-                        cmd,
-                        encoding="utf-8",
-                        check=True
-                    )
+                    subprocess.run(cmd, encoding="utf-8", check=True)
+
 
 if __name__ == "__main__":
     cli_args = parser.parse_args()
@@ -269,4 +294,4 @@ if __name__ == "__main__":
         dry_run=cli_args.dry_run,
     )
 
-    runner.run()
\ No newline at end of file
+    runner.run()
diff --git a/testing/downstream_testing/entrypoint.sh b/testing/downstream_testing/entrypoint.sh
index 1fbf660dc..9b0f35bd6 100755
--- a/testing/downstream_testing/entrypoint.sh
+++ b/testing/downstream_testing/entrypoint.sh
@@ -3,4 +3,4 @@
 python3.9 -m pip install --no-cache-dir pyyaml sh
 
 cd /pytest
-python3.9 -u -m testing.downstream_testing.downstream_runner $DS_NAME $DS_YAML $DS_JOBS --matrix-exclude $DS_MATRIX_EXCLUDE
\ No newline at end of file
+python3.9 -u -m testing.downstream_testing.downstream_runner $DS_NAME $DS_YAML $DS_JOBS --matrix-exclude $DS_MATRIX_EXCLUDE

From e718e30aa646c9f1a933d3adc6d92046eef586e0 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Sun, 3 Apr 2022 10:12:00 -0500
Subject: [PATCH 29/59] fix genexp in filtered_match; use TYPE_CHECKING to keep
 mypy happy; remove some debug msgs

---
 .../downstream_testing/downstream_runner.py   | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/testing/downstream_testing/downstream_runner.py b/testing/downstream_testing/downstream_runner.py
index feefcf081..d820aa5ba 100644
--- a/testing/downstream_testing/downstream_runner.py
+++ b/testing/downstream_testing/downstream_runner.py
@@ -11,6 +11,7 @@ from typing import Any
 from typing import Dict
 from typing import List
 from typing import Optional
+from typing import TYPE_CHECKING
 
 import yaml
 
@@ -81,7 +82,13 @@ TOX_DEP_FILTERS = {
 }
 
 
-class ToxDepFilter(UserDict[Any, Any]):
+if TYPE_CHECKING:
+    _BaseUserDict = UserDict[Any, Any]
+else:
+    _BaseUserDict = UserDict
+
+
+class ToxDepFilter(_BaseUserDict):
     def __init__(self) -> None:
         self.data = TOX_DEP_FILTERS
 
@@ -89,12 +96,6 @@ class ToxDepFilter(UserDict[Any, Any]):
         """Checks if `match` matches any conditions"""
         match_found = None
         for key, val in self.data.items():
-            logger.debug(
-                "matches_condition: %s:%s -> %s",
-                key,
-                val,
-                re.search(val["condition"], match),
-            )
             if re.search(val["condition"], match):
                 match_found = key
                 break
@@ -103,7 +104,6 @@ class ToxDepFilter(UserDict[Any, Any]):
 
     def matches_gen_exp(self, dep: str, match: str) -> Optional[Any]:
         """Checks if `match` matches `dep`['has_gen'] condition."""
-        logger.debug("matches_gen_exp: %s", re.match(self.data[dep]["has_gen"], match))
         return re.match(self.data[dep]["has_gen"], match)
 
     def filter_dep(self, match: str) -> Optional[Dict[Any, Any]]:
@@ -115,9 +115,9 @@ class ToxDepFilter(UserDict[Any, Any]):
             if dep_gen_exp:
                 filtered_match = {
                     "src": self.data[dep_condition]["src"],
-                    "gen_exp": dep_gen_exp,
+                    "gen_exp": dep_gen_exp[0],
                 }
-        logger.debug("filter_dep: %s", filtered_match)
+                logger.debug("toxenv dependency updated: %s", filtered_match)
         return filtered_match
 
 

From 2343f8b4fdc8da44615c7299dee268ebdb1cd413 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Sun, 3 Apr 2022 10:21:01 -0500
Subject: [PATCH 30/59] fix final dependency formatting

---
 testing/downstream_testing/downstream_runner.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/testing/downstream_testing/downstream_runner.py b/testing/downstream_testing/downstream_runner.py
index d820aa5ba..7a8e90273 100644
--- a/testing/downstream_testing/downstream_runner.py
+++ b/testing/downstream_testing/downstream_runner.py
@@ -164,7 +164,7 @@ class DownstreamRunner:
                 if filtered_dep and filtered_dep not in found_dep:
                     found_dep.append(filtered_dep)
                     updated_deps.add(
-                        f"!{filtered_dep['gen_exp']} {filtered_dep['src']}"
+                        f"!{filtered_dep['gen_exp']}: {filtered_dep['src']}"
                     )
 
             if not [item for item in updated_deps if pytest_dep in item]:

From 5197801025ca557085156a585c886195ccb5171b Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Sun, 3 Apr 2022 11:46:44 -0500
Subject: [PATCH 31/59] fix duplicated filtered dependencies

---
 testing/downstream_testing/downstream_runner.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/testing/downstream_testing/downstream_runner.py b/testing/downstream_testing/downstream_runner.py
index 7a8e90273..8e18aab14 100644
--- a/testing/downstream_testing/downstream_runner.py
+++ b/testing/downstream_testing/downstream_runner.py
@@ -161,8 +161,8 @@ class DownstreamRunner:
             tox_dep_filter = ToxDepFilter()
             for dep in section_deps.split("\n"):
                 filtered_dep = tox_dep_filter.filter_dep(dep)
-                if filtered_dep and filtered_dep not in found_dep:
-                    found_dep.append(filtered_dep)
+                if filtered_dep and filtered_dep["src"] not in found_dep:
+                    found_dep.append(filtered_dep["src"])
                     updated_deps.add(
                         f"!{filtered_dep['gen_exp']}: {filtered_dep['src']}"
                     )

From 26b2a5016d952f54f15c17bad8a885e0af2f1a25 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Mon, 4 Apr 2022 07:32:38 -0500
Subject: [PATCH 32/59] reorder tox dependency filters to avoid skipping

---
 testing/downstream_testing/downstream_runner.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/testing/downstream_testing/downstream_runner.py b/testing/downstream_testing/downstream_runner.py
index 8e18aab14..2d36a7a65 100644
--- a/testing/downstream_testing/downstream_runner.py
+++ b/testing/downstream_testing/downstream_runner.py
@@ -28,7 +28,6 @@ parser.add_argument("jobs", nargs="+", help="Job names to use.")
 parser.add_argument(
     "--matrix-exclude", nargs="*", default=[], help="Exclude these matrix names."
 )
-
 parser.add_argument(
     "--dry-run",
     action="store_true",
@@ -64,11 +63,6 @@ def load_matrix_schema(repo):
 
 
 TOX_DEP_FILTERS = {
-    "pytest": {
-        "src": f"pytest @ file://{os.getcwd()}",
-        "condition": r"^pytest(?!\-)",
-        "has_gen": r"pytest\w*",
-    },
     "pytest-rerunfailures": {
         "src": "pytest-rerunfailures @ git+https://github.com/pytest-dev/pytest-rerunfailures.git",
         "condition": r"^pytest-rerunfailures.*",
@@ -79,6 +73,11 @@ TOX_DEP_FILTERS = {
         "condition": r"^pytest.*pytest-xdist",
         "has_gen": r"pytest\{.*\,7\d.*\}",
     },
+    "pytest": {
+        "src": f"pytest @ file://{os.getcwd()}",
+        "condition": r"^pytest(?!\-)",
+        "has_gen": r"pytest\w*",
+    },
 }
 
 
@@ -96,6 +95,10 @@ class ToxDepFilter(_BaseUserDict):
         """Checks if `match` matches any conditions"""
         match_found = None
         for key, val in self.data.items():
+            if "xdist" in match:
+                logging.debug(
+                    "matches_condition: %s", re.search(val["condition"], match)
+                )
             if re.search(val["condition"], match):
                 match_found = key
                 break

From 6815cfea52e5bd5b463728eb939cf1187b9dac28 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Mon, 4 Apr 2022 07:32:38 -0500
Subject: [PATCH 33/59] reorder tox dependency filters to avoid skipping

---
 testing/downstream_testing/downstream_runner.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/testing/downstream_testing/downstream_runner.py b/testing/downstream_testing/downstream_runner.py
index 8e18aab14..628d1344a 100644
--- a/testing/downstream_testing/downstream_runner.py
+++ b/testing/downstream_testing/downstream_runner.py
@@ -28,7 +28,6 @@ parser.add_argument("jobs", nargs="+", help="Job names to use.")
 parser.add_argument(
     "--matrix-exclude", nargs="*", default=[], help="Exclude these matrix names."
 )
-
 parser.add_argument(
     "--dry-run",
     action="store_true",
@@ -64,11 +63,6 @@ def load_matrix_schema(repo):
 
 
 TOX_DEP_FILTERS = {
-    "pytest": {
-        "src": f"pytest @ file://{os.getcwd()}",
-        "condition": r"^pytest(?!\-)",
-        "has_gen": r"pytest\w*",
-    },
     "pytest-rerunfailures": {
         "src": "pytest-rerunfailures @ git+https://github.com/pytest-dev/pytest-rerunfailures.git",
         "condition": r"^pytest-rerunfailures.*",
@@ -79,6 +73,11 @@ TOX_DEP_FILTERS = {
         "condition": r"^pytest.*pytest-xdist",
         "has_gen": r"pytest\{.*\,7\d.*\}",
     },
+    "pytest": {
+        "src": f"pytest @ file://{os.getcwd()}",
+        "condition": r"^pytest(?!\-)",
+        "has_gen": r"pytest\w*",
+    },
 }
 
 

From 6baeec08ec8aeb369020dfad1ad747ece6937e2e Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Thu, 7 Apr 2022 09:17:34 -0500
Subject: [PATCH 34/59] continue adding/updating type annotations

---
 .../downstream_testing/downstream_runner.py   | 87 ++++++++++---------
 1 file changed, 46 insertions(+), 41 deletions(-)

diff --git a/testing/downstream_testing/downstream_runner.py b/testing/downstream_testing/downstream_runner.py
index 628d1344a..a08ab0091 100644
--- a/testing/downstream_testing/downstream_runner.py
+++ b/testing/downstream_testing/downstream_runner.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import argparse
 import configparser
 import json
@@ -8,9 +10,8 @@ import shlex
 import subprocess
 from collections import UserDict
 from typing import Any
-from typing import Dict
-from typing import List
-from typing import Optional
+from typing import Iterable
+from typing import Match
 from typing import TYPE_CHECKING
 
 import yaml
@@ -91,7 +92,7 @@ class ToxDepFilter(_BaseUserDict):
     def __init__(self) -> None:
         self.data = TOX_DEP_FILTERS
 
-    def matches_condition(self, match: str) -> Optional[str]:
+    def matches_condition(self, match: str) -> str | None:
         """Checks if `match` matches any conditions"""
         match_found = None
         for key, val in self.data.items():
@@ -101,11 +102,11 @@ class ToxDepFilter(_BaseUserDict):
 
         return match_found
 
-    def matches_gen_exp(self, dep: str, match: str) -> Optional[Any]:
+    def matches_gen_exp(self, dep: str, match: str) -> Match[str] | None:
         """Checks if `match` matches `dep`['has_gen'] condition."""
         return re.match(self.data[dep]["has_gen"], match)
 
-    def filter_dep(self, match: str) -> Optional[Dict[Any, Any]]:
+    def filter_dep(self, match: str) -> dict[Any, Any] | None:
         """Filters `match` based on conditions and returns the `src` dependency."""
         filtered_match = None
         dep_condition = self.matches_condition(match)
@@ -121,36 +122,44 @@ class ToxDepFilter(_BaseUserDict):
 
 
 class DownstreamRunner:
-    def __init__(self, repo, yaml_source, jobs, matrix_exclude=None, dry_run=False):
+    def __init__(
+        self,
+        repo: str,
+        yaml_source: str,
+        jobs: str,
+        matrix_exclude: str = "",
+        dry_run: bool = False,
+    ) -> None:
         self.repo = repo
         self.yaml_source = yaml_source
         self.matrix_exclude = matrix_exclude
         self.job_names = jobs
         self.dry_run = dry_run
 
-        self._yaml_tree = None
-        self._matrix = None
-        self._steps = None
+        self._yaml_tree: dict[str, Any] | None = None
+        self._matrix: dict[str, Any] | None = None
         self.matrix_schema = load_matrix_schema(self.repo)
 
     @property
-    def yaml_tree(self):
-        """The YAML tree built from the `yaml_source` file."""
-
-        with open(self.yaml_source) as f:
-            self._yaml_tree = yaml.safe_load(f.read())
-
+    def yaml_tree(self) -> dict[str, Any]:
+        """The YAML tree built from the `self.yaml_source` file."""
         if self._yaml_tree is None:
-            raise SystemExit("Supplied YAML source failed to parse.")
+            with open(self.yaml_source) as f:
+                _yaml_tree = yaml.safe_load(f.read())
+
+            if _yaml_tree is None:
+                raise SystemExit("Supplied YAML source failed to parse.")
+            else:
+                self._yaml_tree = _yaml_tree
 
         return self._yaml_tree
 
-    def inject_pytest_dep(self):
+    def inject_pytest_dep(self) -> None:
         """Ensure pytest is a dependency in tox.ini to allow us to use the 'local'
         version of pytest.
         """
         ini_path = self.repo + "/tox.ini"
-        pytest_dep = f"pytest @ file://{os.getcwd()}"
+        pytest_dep = TOX_DEP_FILTERS["pytest"]["src"]
         tox_source = configparser.ConfigParser()
         tox_source.read_file(open(ini_path))
         found_dep = []
@@ -176,26 +185,35 @@ class DownstreamRunner:
         with open(ini_path, "w") as f:
             tox_source.write(f)
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         return str(
             "DownstreamRunner("
             f"repo={self.repo}, "
             f"yaml_source={self.yaml_source}, "
             f"job_names={self.job_names}, "
             f"matrix={self.matrix}, "
-            f"steps={self.steps}"
             ")"
         )
 
     @property
-    def matrix(self):
-        def parse_matrix(yaml_tree):
-            parsed_matrix = yaml_tree
+    def matrix(self) -> dict[str, Iterable[dict[str, str]]]:
+        def parse_matrix(yaml_tree: dict[str, Any]) -> Iterable[Any]:
+            parsed_matrix = []  # type: ignore
+            pre_parsed: dict[str, Any] | Iterable[str | float] = yaml_tree
             for key in self.matrix_schema["matrix"]:
-                parsed_matrix = parsed_matrix[key]
+                if isinstance(pre_parsed, dict):
+                    pre_parsed = pre_parsed[key]
+            else:
+                if isinstance(pre_parsed, list):
+                    parsed_matrix = pre_parsed
+                else:
+                    msg_info = f"repo: {self.repo} | matrix schema: {self.matrix_schema} | parsed result: {pre_parsed}"
+                    raise TypeError(
+                        f"Parsed Actions matrix is invalid. Should be list/array. {msg_info}"
+                    )
 
             logger.debug("parsed_matrix: %s", parsed_matrix)
-            if parsed_matrix != yaml_tree:
+            if parsed_matrix:
                 tox_base = self.matrix_schema["tox_cmd_build"]["base"]
                 tox_prefix = self.matrix_schema["tox_cmd_build"]["prefix"]
                 skip_matrices = []
@@ -245,20 +263,7 @@ class DownstreamRunner:
         logger.debug("matrix: %s", self._matrix)
         return self._matrix
 
-    @property
-    def steps(self):
-        if self._steps is None:
-            step_items: Dict[str, List[Any]] = {}
-            for job in self.job_names:
-                if job not in step_items:
-                    step_items[job] = []
-                for item in self.yaml_tree["jobs"][job]["steps"]:
-                    if "run" in item:
-                        step_items[job].append(item)
-            self._steps = step_items
-        return self._steps
-
-    def build_run(self):
+    def build_run(self) -> dict[str, list[str]]:
         run = {}
         for job in self.job_names:
             logger.debug("job_name: %s", job)
@@ -269,7 +274,7 @@ class DownstreamRunner:
         logger.debug("built run: %s", run)
         return run
 
-    def run(self):
+    def run(self) -> None:
         self.inject_pytest_dep()
         run_steps = self.build_run()
         os.chdir(self.repo)

From cb15c08ddb48f852bc27465f7e4ce164d22dbb7a Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Fri, 8 Apr 2022 00:15:25 -0500
Subject: [PATCH 35/59] load_matrix_schema: raise exceptions for errors vs
 logging; add type annotations

---
 .../downstream_testing/downstream_runner.py   | 51 ++++++++++++-------
 1 file changed, 32 insertions(+), 19 deletions(-)

diff --git a/testing/downstream_testing/downstream_runner.py b/testing/downstream_testing/downstream_runner.py
index a08ab0091..7648d31a2 100644
--- a/testing/downstream_testing/downstream_runner.py
+++ b/testing/downstream_testing/downstream_runner.py
@@ -15,6 +15,7 @@ from typing import Match
 from typing import TYPE_CHECKING
 
 import yaml
+from typing_extensions import TypedDict
 
 logging.basicConfig(
     format="%(levelname)s | %(module)s.%(funcName)s | %(message)s", level="INFO"
@@ -35,32 +36,50 @@ parser.add_argument(
     help="Do not run parsed downstream action. Only display the generated command list.",
 )
 
+if TYPE_CHECKING:
+    _BaseUserDict = UserDict[Any, Any]
 
-def load_matrix_schema(repo):
+    class SchemaBase(TypedDict):
+        repo: str
+
+    class SchemaToxBase(TypedDict):
+        base: str
+        prefix: str
+        sub: dict[str, str]
+
+    class SchemaType(SchemaBase, total=False):
+        matrix: list[str]
+        tox_cmd_build: SchemaToxBase
+        python_version: str
+
+else:
+    _BaseUserDict = UserDict
+
+
+def load_matrix_schema(repo: str) -> SchemaType:
     """Loads the matrix schema for `repo`"""
-    schema = None
+    schema: SchemaType = {"repo": repo}
     working_dir = os.getcwd()
     schema_path = os.path.join(
         working_dir, "testing", "downstream_testing", "action_schemas.json"
     )
-    logger.debug("loading schema: %s", schema_path)
+    logger.debug("Loading schema: %s", schema_path)
     if os.path.exists(schema_path):
         with open(schema_path) as schema_file:
             try:
                 schema = json.load(schema_file)
-            except json.JSONDecodeError:
-                logger.error("failed to read action_schemas.json")
+            except json.JSONDecodeError as exc:
+                raise RuntimeError(f"Error decoding '{schema_path}'") from exc
     else:
-        logger.warning("action_schemas.json not found.")
+        raise FileNotFoundError(f"'{schema_path}' not found.")
 
-    if schema is not None and repo in schema:
-        schema = schema[repo]
+    if repo in schema:
         logger.debug("'%s' schema loaded: %s", repo, schema)
+        return schema[repo]  # type: ignore
     else:
-        schema = None
-        logger.warning("'%s' schema not found in actions_schema.json", repo)
-
-    return schema
+        raise RuntimeError(
+            f"'{repo}' schema definition not found in actions_schema.json"
+        )
 
 
 TOX_DEP_FILTERS = {
@@ -82,12 +101,6 @@ TOX_DEP_FILTERS = {
 }
 
 
-if TYPE_CHECKING:
-    _BaseUserDict = UserDict[Any, Any]
-else:
-    _BaseUserDict = UserDict
-
-
 class ToxDepFilter(_BaseUserDict):
     def __init__(self) -> None:
         self.data = TOX_DEP_FILTERS
@@ -138,7 +151,7 @@ class DownstreamRunner:
 
         self._yaml_tree: dict[str, Any] | None = None
         self._matrix: dict[str, Any] | None = None
-        self.matrix_schema = load_matrix_schema(self.repo)
+        self.matrix_schema: SchemaType = load_matrix_schema(self.repo)
 
     @property
     def yaml_tree(self) -> dict[str, Any]:

From 67734859626725fc6ed27e5cabe8796710da774f Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Fri, 8 Apr 2022 13:43:38 -0500
Subject: [PATCH 36/59] only import typing-extenstions during TYPE_CHECKING

---
 testing/downstream_testing/downstream_runner.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/testing/downstream_testing/downstream_runner.py b/testing/downstream_testing/downstream_runner.py
index a3fd44109..2f2ab4d2a 100644
--- a/testing/downstream_testing/downstream_runner.py
+++ b/testing/downstream_testing/downstream_runner.py
@@ -15,7 +15,9 @@ from typing import Match
 from typing import TYPE_CHECKING
 
 import yaml
-from typing_extensions import TypedDict
+
+if TYPE_CHECKING:
+    from typing_extensions import TypedDict
 
 logging.basicConfig(
     format="%(levelname)s | %(module)s.%(funcName)s | %(message)s", level="INFO"

From 32b76444740d9f97b83ce204e9af7db9453d786c Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Sun, 10 Apr 2022 08:03:29 -0500
Subject: [PATCH 37/59] correct some type annotations

---
 testing/downstream_testing/downstream_runner.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/testing/downstream_testing/downstream_runner.py b/testing/downstream_testing/downstream_runner.py
index 2f2ab4d2a..48696b666 100644
--- a/testing/downstream_testing/downstream_runner.py
+++ b/testing/downstream_testing/downstream_runner.py
@@ -145,8 +145,8 @@ class DownstreamRunner:
         self,
         repo: str,
         yaml_source: str,
-        jobs: str,
-        matrix_exclude: str = "",
+        jobs: Iterable[str],
+        matrix_exclude: Iterable[str] = (),
         dry_run: bool = False,
     ) -> None:
         self.repo = repo

From dea5bc43b5d8af72852763bd561783d198b1b7e1 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Sun, 10 Apr 2022 22:31:57 -0500
Subject: [PATCH 38/59] adjusted some exceptions & argparse; discovered via
 test writing

---
 testing/downstream_testing/downstream_runner.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/testing/downstream_testing/downstream_runner.py b/testing/downstream_testing/downstream_runner.py
index 48696b666..bed1fe56e 100644
--- a/testing/downstream_testing/downstream_runner.py
+++ b/testing/downstream_testing/downstream_runner.py
@@ -27,7 +27,7 @@ logger = logging.getLogger(__name__)
 
 parser = argparse.ArgumentParser(description="downstream Actions runner")
 parser.add_argument("repo", help="Name of the repo.")
-parser.add_argument("source", nargs=1, help="Path to source YAML file.")
+parser.add_argument("source", help="Path to source YAML file.")
 parser.add_argument("jobs", nargs="+", help="Job names to use.")
 parser.add_argument(
     "--matrix-exclude", nargs="*", default=[], help="Exclude these matrix names."
@@ -164,10 +164,15 @@ class DownstreamRunner:
         """The YAML tree built from the `self.yaml_source` file."""
         if self._yaml_tree is None:
             with open(self.yaml_source) as f:
-                _yaml_tree = yaml.safe_load(f.read())
+                try:
+                    _yaml_tree = yaml.safe_load(f.read())
+                except yaml.YAMLError as exc:
+                    raise RuntimeError(
+                        f"Error while parsing '{self.yaml_source}'."
+                    ) from exc
 
             if _yaml_tree is None:
-                raise SystemExit("Supplied YAML source failed to parse.")
+                raise RuntimeError(f"'{self.yaml_source}' failed to parse.")
             else:
                 self._yaml_tree = _yaml_tree
 
@@ -311,7 +316,7 @@ if __name__ == "__main__":
         logger.setLevel("DEBUG")
     runner = DownstreamRunner(
         cli_args.repo,
-        cli_args.source[0],
+        cli_args.source,
         cli_args.jobs,
         matrix_exclude=cli_args.matrix_exclude,
         dry_run=cli_args.dry_run,

From 657865e9ae6337615e8e7bd52a25fc180fd9394d Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Mon, 11 Apr 2022 15:59:49 -0500
Subject: [PATCH 39/59] add tests for downstream_runner.py

---
 pyproject.toml                                |   2 +
 setup.cfg                                     |   1 +
 .../test_downstream_runner.py                 | 281 ++++++++++++++++++
 tox.ini                                       |   5 +-
 4 files changed, 288 insertions(+), 1 deletion(-)
 create mode 100644 testing/downstream_testing/test_downstream_runner.py

diff --git a/pyproject.toml b/pyproject.toml
index 3d683aebe..e9c518e5d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -59,6 +59,8 @@ markers = [
     "slow",
     # experimental mark for all tests using pexpect
     "uses_pexpect",
+    # runs tests for ./testing/downstream_testing/downstream_runner.py
+    "downstream",
 ]
 
 
diff --git a/setup.cfg b/setup.cfg
index fe6ea4095..a63ec5fcd 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -65,6 +65,7 @@ console_scripts =
 
 [options.extras_require]
 testing =
+    PyYAML
     argcomplete
     hypothesis>=3.56
     mock
diff --git a/testing/downstream_testing/test_downstream_runner.py b/testing/downstream_testing/test_downstream_runner.py
new file mode 100644
index 000000000..1b22ba226
--- /dev/null
+++ b/testing/downstream_testing/test_downstream_runner.py
@@ -0,0 +1,281 @@
+from __future__ import annotations
+
+import sys
+from typing import Any
+
+import pytest
+
+sys.path.append("testing")
+from downstream_testing import downstream_runner  # noqa: E402
+
+xfail = pytest.mark.xfail
+
+DUMMY_YAML_COMBINATION = """---
+name: dummy-include
+jobs:
+  test:
+    runs-on: ubuntu-20.04
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - name: i-do-not-start-with-py
+          python: 3.6
+          allow_failure: false
+
+        - name: py310-dj40-postgres-xdist-coverage
+          python: '3.10'
+          allow_failure: false
+
+        - name: py36-exclude-me
+          python: 3.6
+          allow_failure: false
+
+        - name: py37-exclude-me
+          python: 3.7
+          allow_failure: false
+"""
+
+DUMMY_YAML_NO_COMBINATION = """---
+name: dummy-no-include
+jobs:
+  test:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        python-version: [3.6, 3.7, 3.8, 3.9, "3.10", pypy3]
+        exclude:
+          - os: windows-latest
+            python-version: pypy3
+"""
+
+
+@pytest.fixture()
+def mock_schema_combination(monkeypatch):
+    def mock_load_schema_combination(repo: str) -> dict[str, Any]:
+        return {
+            "matrix": ["matrix", "include"],
+            "tox_cmd_build": {
+                "base": "name",
+                "prefix": "py",
+                "sub": {"pattern": "-coverage$", "replace": ""},
+            },
+            "python_version": "python",
+        }
+
+    monkeypatch.setattr(
+        downstream_runner, "load_matrix_schema", mock_load_schema_combination
+    )
+
+
+@pytest.fixture()
+def mock_schema_no_combination(monkeypatch):
+    def mock_load_schema_no_combination(repo: str) -> dict[str, Any]:
+        return {
+            "matrix": ["matrix", "python-version"],
+            "tox_cmd_build": {
+                "base": "",
+                "prefix": "",
+                "sub": {"pattern": "(\\d|py\\d)\\.*(\\d+)", "replace": "py\\1\\2"},
+            },
+            "python_version": "python",
+        }
+
+    monkeypatch.setattr(
+        downstream_runner, "load_matrix_schema", mock_load_schema_no_combination
+    )
+
+
+@pytest.mark.downstream
+class TestDownstreamRunnerConstructor:
+    @pytest.mark.parametrize(
+        "args",
+        [
+            pytest.param(tuple(), marks=xfail),
+            pytest.param(("repo",), marks=xfail),
+            pytest.param(("repo", "yaml_source"), marks=xfail),
+            ("pytest-downstream", "yaml_source", ["jobs"]),
+        ],
+        ids=["no args", "repo only", "missing jobs", "all args"],
+    )
+    def test_args(self, args, mock_schema_combination) -> None:
+        downstream_runner.DownstreamRunner(*args)
+
+    @pytest.mark.parametrize(
+        "kwargs",
+        [
+            ("matrix_exclude", "exclude"),
+            ("dry_run", True),
+        ],
+        ids=["matrix_exclude", "dry_run"],
+    )
+    def test_kwargs(self, kwargs, mock_schema_combination) -> None:
+        args = ("pytest-downstream", "yaml_source", ["test"])
+        new_kwargs = {kwargs[0]: kwargs[1]}
+        runner = downstream_runner.DownstreamRunner(*args, **new_kwargs)
+
+        assert kwargs[1] == getattr(runner, kwargs[0])
+
+
+@pytest.mark.downstream
+class TestDownstreamRunnerProperties:
+    def test_yaml_tree_file_doesnt_exist(self, mock_schema_combination) -> None:
+        runner = downstream_runner.DownstreamRunner(
+            "pytest-downstream", "yaml_source", ["test"], dry_run=True
+        )
+
+        with pytest.raises(FileNotFoundError):
+            runner.yaml_tree
+
+    def test_yaml_tree_bad_yaml(self, mock_schema_combination, tmp_path) -> None:
+        yaml_source = tmp_path / "test.yml"
+        yaml_source.write_text("---\n:")
+
+        runner = downstream_runner.DownstreamRunner(
+            "pytest-downstream", yaml_source, ["test"], dry_run=True
+        )
+
+        with pytest.raises(RuntimeError) as excinfo:
+            runner.yaml_tree
+        assert str(yaml_source) in excinfo.exconly(tryshort=True)
+
+    def test_yaml_tree_empty_yaml(self, mock_schema_combination, tmp_path) -> None:
+        yaml_source = tmp_path / "test.yml"
+        yaml_source.write_text("---")
+
+        runner = downstream_runner.DownstreamRunner(
+            "pytest-downstream", yaml_source, ["test"], dry_run=True
+        )
+
+        with pytest.raises(RuntimeError) as excinfo:
+            runner.yaml_tree
+        assert str(yaml_source) in excinfo.exconly(tryshort=True)
+
+    def test_yaml_tree_passing_yaml(self, mock_schema_combination, tmp_path) -> None:
+        yaml_source = tmp_path / "test.yml"
+        yaml_source.write_text(DUMMY_YAML_COMBINATION)
+
+        runner = downstream_runner.DownstreamRunner(
+            "pytest-downstream", yaml_source, ["test"], dry_run=True
+        )
+
+        assert runner.yaml_tree["name"] == "dummy-include"
+        assert "test" in runner.yaml_tree["jobs"]
+
+    def test_matrix_combination(self, mock_schema_combination, tmp_path) -> None:
+        yaml_source = tmp_path / "test.yml"
+        yaml_source.write_text(DUMMY_YAML_COMBINATION)
+
+        runner = downstream_runner.DownstreamRunner(
+            "pytest-downstream", yaml_source, ["test"], dry_run=True
+        )
+
+        assert all(
+            [
+                matrice.get("tox_cmd", "").startswith("py")
+                for matrice in runner.matrix["test"]
+            ]
+        )
+
+    def test_matrix_no_combination(self, mock_schema_no_combination, tmp_path) -> None:
+        yaml_source = tmp_path / "test.yml"
+        yaml_source.write_text(DUMMY_YAML_NO_COMBINATION)
+
+        runner = downstream_runner.DownstreamRunner(
+            "pytest-downstream", yaml_source, ["test"], dry_run=True
+        )
+
+        assert all(
+            [
+                matrice.get("tox_cmd", "").startswith("py")
+                for matrice in runner.matrix["test"]
+            ]
+        )
+
+    def test_matrix_combination_matrix_exclude(
+        self, mock_schema_combination, tmp_path
+    ) -> None:
+        yaml_source = tmp_path / "test.yml"
+        yaml_source.write_text(DUMMY_YAML_COMBINATION)
+        matrix_to_exclude = ["py36-exclude-me", "py37-exclude-me"]
+
+        runner = downstream_runner.DownstreamRunner(
+            "pytest-downstream",
+            yaml_source,
+            ["test"],
+            matrix_exclude=matrix_to_exclude,
+            dry_run=True,
+        )
+
+        matrix_names = {matrice["name"] for matrice in runner.matrix["test"]}
+        assert matrix_names.isdisjoint(set(matrix_to_exclude))
+
+    def test_matrix_no_combination_matrix_exclude(
+        self, mock_schema_no_combination, tmp_path
+    ) -> None:
+        yaml_source = tmp_path / "test.yml"
+        yaml_source.write_text(DUMMY_YAML_NO_COMBINATION)
+        matrix_to_exclude = ["3.6", "3.7"]
+
+        runner = downstream_runner.DownstreamRunner(
+            "pytest-downstream",
+            yaml_source,
+            ["test"],
+            matrix_exclude=matrix_to_exclude,
+            dry_run=True,
+        )
+
+        matrix_names = {matrice["name"] for matrice in runner.matrix["test"]}
+        assert matrix_names.isdisjoint({"py36", "py37"})
+
+
+@pytest.mark.downstream
+class TestDownstreamRunnerBuild:
+    def test_build_run_combination_matrix(
+        self, mock_schema_combination, tmp_path
+    ) -> None:
+        yaml_source = tmp_path / "test.yml"
+        yaml_source.write_text(DUMMY_YAML_COMBINATION)
+        matrix_to_exclude = ["py36-exclude-me", "py37-exclude-me"]
+
+        runner = downstream_runner.DownstreamRunner(
+            "pytest-downstream",
+            yaml_source,
+            ["test"],
+            matrix_exclude=matrix_to_exclude,
+            dry_run=True,
+        )
+
+        run = runner.build_run()
+        assert run == {
+            "py310-dj40-postgres-xdist-coverage": [
+                "pip install tox",
+                "tox -e py310-dj40-postgres-xdist",
+            ]
+        }
+
+    def test_build_run_no_combination_matrix(
+        self, mock_schema_no_combination, tmp_path
+    ) -> None:
+        yaml_source = tmp_path / "test.yml"
+        yaml_source.write_text(DUMMY_YAML_NO_COMBINATION)
+        matrix_to_exclude = ["3.6", "3.7"]
+
+        runner = downstream_runner.DownstreamRunner(
+            "pytest-downstream",
+            yaml_source,
+            ["test"],
+            matrix_exclude=matrix_to_exclude,
+            dry_run=True,
+        )
+
+        run = runner.build_run()
+        assert run == {
+            "py310": ["pip install tox", "tox -e py310"],
+            "py38": ["pip install tox", "tox -e py38"],
+            "py39": ["pip install tox", "tox -e py39"],
+            "pypy3": ["pip install tox", "tox -e pypy3"],
+        }
diff --git a/tox.ini b/tox.ini
index 93c390ffc..9ed36183a 100644
--- a/tox.ini
+++ b/tox.ini
@@ -16,6 +16,7 @@ envlist =
     py37-freeze
     docs
     docs-checklinks
+    downstream
 
 
 
@@ -27,7 +28,7 @@ commands =
     coverage: coverage report -m
 passenv = USER USERNAME COVERAGE_* PYTEST_ADDOPTS TERM SETUPTOOLS_SCM_PRETEND_VERSION_FOR_PYTEST
 setenv =
-    _PYTEST_TOX_DEFAULT_POSARGS={env:_PYTEST_TOX_POSARGS_DOCTESTING:} {env:_PYTEST_TOX_POSARGS_LSOF:} {env:_PYTEST_TOX_POSARGS_XDIST:}
+    _PYTEST_TOX_DEFAULT_POSARGS={env:_PYTEST_TOX_POSARGS_DOCTESTING:} {env:_PYTEST_TOX_POSARGS_LSOF:} {env:_PYTEST_TOX_POSARGS_XDIST:} {env:_PYTEST_TOX_POSARGS_DOWNSTREAM:}
 
     # Configuration to run with coverage similar to CI, e.g.
     # "tox -e py37-coverage".
@@ -43,6 +44,8 @@ setenv =
     lsof: _PYTEST_TOX_POSARGS_LSOF=--lsof
 
     xdist: _PYTEST_TOX_POSARGS_XDIST=-n auto
+
+    downstream: _PYTEST_TOX_POSARGS_DOWNSTREAM=-m downstream
 extras = testing
 deps =
     doctesting: PyYAML

From a802ecc367bf235e174ded14eda41ea551d46377 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Sat, 16 Apr 2022 08:44:32 -0500
Subject: [PATCH 40/59] logging & documentation improvements

---
 .../downstream_testing/downstream_runner.py   | 20 ++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/testing/downstream_testing/downstream_runner.py b/testing/downstream_testing/downstream_runner.py
index bed1fe56e..e90856bcb 100644
--- a/testing/downstream_testing/downstream_runner.py
+++ b/testing/downstream_testing/downstream_runner.py
@@ -25,7 +25,7 @@ logging.basicConfig(
 logger = logging.getLogger(__name__)
 
 
-parser = argparse.ArgumentParser(description="downstream Actions runner")
+parser = argparse.ArgumentParser(description="pytest downstream plugins test runner")
 parser.add_argument("repo", help="Name of the repo.")
 parser.add_argument("source", help="Path to source YAML file.")
 parser.add_argument("jobs", nargs="+", help="Job names to use.")
@@ -35,7 +35,7 @@ parser.add_argument(
 parser.add_argument(
     "--dry-run",
     action="store_true",
-    help="Do not run parsed downstream action. Only display the generated command list.",
+    help="Do not run parsed downstream action. Only display the generated command list and debug information.",
 )
 
 if TYPE_CHECKING:
@@ -76,7 +76,7 @@ def load_matrix_schema(repo: str) -> SchemaType:
         raise FileNotFoundError(f"'{schema_path}' not found.")
 
     if repo in schema:
-        logger.debug("'%s' schema loaded: %s", repo, schema)
+        logger.debug("'%s' schema loaded: %s", repo, schema[repo])  # type: ignore
         return schema[repo]  # type: ignore
     else:
         raise RuntimeError(
@@ -108,7 +108,7 @@ class ToxDepFilter(_BaseUserDict):
         self.data = TOX_DEP_FILTERS
 
     def matches_condition(self, match: str) -> str | None:
-        """Checks if `match` matches any conditions"""
+        """Checks if ``match`` matches any conditions"""
         match_found = None
         for key, val in self.data.items():
             if "xdist" in match:
@@ -122,11 +122,11 @@ class ToxDepFilter(_BaseUserDict):
         return match_found
 
     def matches_gen_exp(self, dep: str, match: str) -> Match[str] | None:
-        """Checks if `match` matches `dep`['has_gen'] condition."""
+        """Checks if ``match`` matches ``dep``['has_gen'] condition."""
         return re.match(self.data[dep]["has_gen"], match)
 
     def filter_dep(self, match: str) -> dict[Any, Any] | None:
-        """Filters `match` based on conditions and returns the `src` dependency."""
+        """Filters ``match`` based on conditions and returns the ``src`` dependency."""
         filtered_match = None
         dep_condition = self.matches_condition(match)
         if dep_condition is not None:
@@ -136,7 +136,6 @@ class ToxDepFilter(_BaseUserDict):
                     "src": self.data[dep_condition]["src"],
                     "gen_exp": dep_gen_exp[0],
                 }
-                logger.debug("toxenv dependency updated: %s", filtered_match)
         return filtered_match
 
 
@@ -202,6 +201,7 @@ class DownstreamRunner:
             if not [item for item in updated_deps if pytest_dep in item]:
                 updated_deps.add(pytest_dep)
             final_deps = "\n".join(updated_deps)
+            logger.debug("toxenv dependencies updated: %s", updated_deps)
             tox_source["testenv"][
                 "deps"
             ] = f"{tox_source['testenv']['deps']}\n{final_deps}"
@@ -221,7 +221,12 @@ class DownstreamRunner:
 
     @property
     def matrix(self) -> dict[str, Iterable[dict[str, str]]]:
+        """Iterates over ``self.yaml_tree`` strategy matrix for each job in ``self.jobs``, and passes each
+        through ``parse_matrix``.
+        """
+
         def parse_matrix(yaml_tree: dict[str, Any]) -> Iterable[Any]:
+            """Parses `yaml_tree` strategy matrix using ``self.matrix_schema`` information."""
             parsed_matrix = []  # type: ignore
             pre_parsed: dict[str, Any] | Iterable[str | float] = yaml_tree
             for key in self.matrix_schema["matrix"]:
@@ -288,6 +293,7 @@ class DownstreamRunner:
         return self._matrix
 
     def build_run(self) -> dict[str, list[str]]:
+        """Builds the list of commands for all necessary jobs via ``self.matrix``."""
         run = {}
         for job in self.job_names:
             logger.debug("job_name: %s", job)

From 1b947b269c24b49bd134802ddaaafcd25e7d087f Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Sat, 16 Apr 2022 09:13:59 -0500
Subject: [PATCH 41/59] add a README outlining plugin additions

---
 testing/downstream_testing/README.rst | 220 ++++++++++++++++++++++++++
 1 file changed, 220 insertions(+)
 create mode 100644 testing/downstream_testing/README.rst

diff --git a/testing/downstream_testing/README.rst b/testing/downstream_testing/README.rst
new file mode 100644
index 000000000..025a05649
--- /dev/null
+++ b/testing/downstream_testing/README.rst
@@ -0,0 +1,220 @@
+Automated Downstream Testing
+++++++++++++++++++++++++++++
+Testing pytest Plugins With Changes to pytest
+=============================================
+
+Overview
+--------
+
+- Utilizes Docker (compose) inside a GitHub Actions workflow to establish the necessary environment(s) to run tests.
+
+  - The base image is currently Ubuntu only, and uses ``pyenv`` to have the necessary versions of Python installed.
+
+  - The base image is on Docker Hub, which greatly speeds up each run by avoiding ``pyenv``'s compilation process each time.
+
+  - The completed image contains a PostgreSQL & mySQL layer (needed by ``pytest-django``, at minimum). Additional layers can be added if necessary.
+
+- Utilizes each plugin's current test workflow to formulate the tests to run.
+
+  - The plugin's GitHub Actions workflow is used to get ``tox`` environment settings.
+
+  - The plugin's ``tox`` config is adjusted to ensure the local ``pytest`` is used.
+
+- Uses a static command set: ``pip install tox`` & ``tox -e {toxenv}``.
+
+How To Add Additional Plugin(s)
+-------------------------------
+**pytest/testing/downstream_testing/actions_schema.json:**
+**********************************************************
+
+``actions_schema.json`` outlines both GitHub Actions workflow fields to get information, and how
+that information should be manipulated to provide usable tox environments to run a plugin's tests.
+
+Example ``actions_schema.json`` entry:
+
+  .. code:: JSON
+
+    "pytest-django": {
+      "matrix": [
+        "matrix",
+        "include"
+      ],
+      "tox_cmd_build": {
+        "base": "name",
+        "prefix": "py",
+        "sub": {
+          "pattern": "-coverage$",
+          "replace": ""
+        }
+      }
+    }
+
+Below are the steps to add a new plugin.
+
+1. Start by adding a new JSON object, with the name of the repo. ``"pytest-django": {}`` in the example above.
+
+2. Add an array named ``matrix``. Array items should point to the strategy matrix from the repo's GitHub Actions
+   YAML file. For instance, the ``pytest-django`` example above has a strategy matrix defined as follows:
+
+    .. code:: YAML
+
+      # pytest-django/.github/workflows/main.yml
+      jobs:
+        tests:
+          strategy:
+            fail-fast: false
+            matrix:
+              include:
+              - name: linting,docs
+                python: 3.8
+                allow_failure: false
+
+              - name: py310-dj40-postgres-xdist-coverage
+                python: '3.10'
+                allow_failure: false
+
+
+  This makes ``["matrix", "include"]`` our target to parse the ``pytest-django`` strategy. This is a "combination"
+  strategy based on the use of ``include``.
+
+  For non-combination strategies, use the matrix field that points to the appropriate choices. Using
+  ``pytest-order`` as a non-combination example:
+
+    .. code:: YAML
+
+      # pytest-order/.github/workflows/pythontests.yml
+      jobs:
+        test:
+          runs-on: ${{ matrix.os }}
+          strategy:
+            fail-fast: false
+            matrix:
+              os: [ubuntu-latest, windows-latest]
+              python-version: [3.6, 3.7, 3.8, 3.9, "3.10", pypy3]
+
+  The corresponding entry in ``actions_schema.json`` points to ``["matrix", "python-version"]``:
+
+    .. code:: JSON
+
+      "pytest-order": {
+          "matrix": [
+              "matrix",
+              "python-version"
+          ],
+
+3. Add a JSON object named ``tox_cmd_build``, with three items: ``base``, ``prefix``, and ``sub``.
+
+   - ``base``: For combination strategies (with ``include``), ``base`` is the field to be used as the basis
+      of the tox environment. For non-combination strategies, this field is an empty string. ``base: "name"``
+      in the ``pytest-django`` example above.
+
+   - ``prefix``: For combination strategies, ``prefix`` is used to [dis]qualify entries in ``base``. For
+       non-combination strategies, this field is an emtpy string. ``prefix: "py"`` in the ``pytest-django``
+       example above.
+
+   - ``sub``: For both combination and non-combination strategies, this JSON object gives a RegEx matching
+      (``pattern``) and a substituition (``replace``) string. Since these are JSON strings, they cannot be
+      represented as a Python raw string (``r""``); ensure to properly escape characters.
+
+Any additions can be verified locally with the following process:
+
+1. Have a local copy of a plugin's GitHub Actions workflow YAML file, as well as a ``tox.ini``. These should
+   be placed in a folder; recommended folder name is the repo name.
+
+   .. code:: shell
+
+     .
+     .. pytest-order/
+     .... main.yml
+     .... tox.ini
+
+2. Utilize the ``--dry-run`` flag to run the parsing and command step building, without actually executing
+   the commands. This will provide debugging information.
+
+   .. code::
+
+      (.venv) ~/pytest:$> python -m testing.downstream_testing.downstream_runner pytest-order pytest-order/pytest-order.yml test --matrix-exclude 3.6 --dry-run
+
+      DEBUG | downstream_runner.load_matrix_schema | Loading schema: /home/sommersoft/Dev/pytest-dev/pytest/testing/downstream_testing/action_schemas.json
+      DEBUG | downstream_runner.load_matrix_schema | 'pytest-order' schema loaded: {'matrix': ['matrix', 'python-version'],
+      'tox_cmd_build': {'base': '', 'prefix': '', 'sub': {'pattern': '(\\d|py\\d)\\.*(\\d+)', 'replace': 'py\\1\\2'}}, 'python_version': 'python-version'}
+      DEBUG | downstream_runner.inject_pytest_dep | toxenv dependencies updated: {'!pytest{60,61,62,624,70}: pytest-xdist', '!pytest50: pytest @ file:///home/pytest'}
+      DEBUG | downstream_runner.build_run | job_name: test
+      DEBUG | downstream_runner.parse_matrix | parsed_matrix: [3.6, 3.7, 3.8, 3.9, '3.10', 'pypy3']
+      DEBUG | downstream_runner.matrix | matrix: {'test': [{'name': 'py37', 'tox_cmd': 'py37'}, {'name': 'py38', 'tox_cmd': 'py38'}, {'name': 'py39', 'tox_cmd': 'py39'},
+      {'name': 'py310', 'tox_cmd': 'py310'}, {'name': 'pypy3', 'tox_cmd': 'pypy3'}]}
+      DEBUG | downstream_runner.build_run | matrix[job]: {'name': 'py37', 'tox_cmd': 'py37'}
+      DEBUG | downstream_runner.build_run | matrix[job]: {'name': 'py38', 'tox_cmd': 'py38'}
+      DEBUG | downstream_runner.build_run | matrix[job]: {'name': 'py39', 'tox_cmd': 'py39'}
+      DEBUG | downstream_runner.build_run | matrix[job]: {'name': 'py310', 'tox_cmd': 'py310'}
+      DEBUG | downstream_runner.build_run | matrix[job]: {'name': 'pypy3', 'tox_cmd': 'pypy3'}
+      DEBUG | downstream_runner.build_run | built run: {'py37': ['pip install tox', 'tox -e py37'], 'py38': ['pip install tox', 'tox -e py38'], 'py39': ['pip install tox',
+      'tox -e py39'], 'py310': ['pip install tox', 'tox -e py310'], 'pypy3': ['pip install tox', 'tox -e pypy3']}
+      INFO | downstream_runner.run | --> running: 'pip install tox'
+      INFO | downstream_runner.run | --> running: 'tox -e py37'
+      INFO | downstream_runner.run | --> running: 'pip install tox'
+      INFO | downstream_runner.run | --> running: 'tox -e py38'
+      INFO | downstream_runner.run | --> running: 'pip install tox'
+      INFO | downstream_runner.run | --> running: 'tox -e py39'
+      INFO | downstream_runner.run | --> running: 'pip install tox'
+      INFO | downstream_runner.run | --> running: 'tox -e py310'
+      INFO | downstream_runner.run | --> running: 'pip install tox'
+      INFO | downstream_runner.run | --> running: 'tox -e pypy3'
+
+
+**pytest/.github/workflows/downstream_testing.yml:**
+*****************************************************
+This GitHub Actions workflow orchestrates the various plugin tests, and only requires some minimal information.
+
+Add a new entry to the combination strategy matrix:
+
+1. ``name``: the plugin's repository name.
+
+2. ``repo``: the ``org/name`` of the plugin's GitHub repository.
+
+3. ``docker_profile``: the docker-compose profile to use for this entry. Docker profiles are defined in
+   ``pytest/testing/downstream_testing/docker-compose.yml``. At the time of this writing, the available
+   profiles are: ``nodb``, ``mysql``, & ``postgres``.
+
+4. ``jobs``: the list of job(s) to use from the plugin's GitHub Actions workflow.
+
+5. ``workflow_name``: the name of the GitHub Actions workflow file to use (located in ``plugin/.github/workflows``).
+
+6. ``matrix_exclude``: any entries to exclude from the selected matrix (combination & non-combination).
+
+.. code:: YAML
+
+  - name: "pytest-django"
+    repo: "pytest-dev/pytest-django"
+    docker_profile: "postgres"
+    jobs: "test"
+    workflow_name: "main.yml"
+    matrix_exclude: |
+      linting,docs py39-dj40-mysql_innodb-coverage ...
+
+  - name: "pytest-django"
+    repo: "pytest-dev/pytest-django"
+    docker_profile: "mysql"
+    jobs: "test"
+    workflow_name: "main.yml"
+    matrix_exclude: |
+      linting,docs py310-dj40-postgres-xdist-coverage ...
+
+.. epigraph::
+  Example 1: using ``pytest-django``, which has a combination strategy matrix, we see two (of three) different
+  ``downstream_testing.yml`` entries. They each require a different database, so each entry uses the appropriate
+  ``docker_profile``. Additionally, to avoid known failures, ``matrix_exclude`` has all items that don't match
+  the database in use.
+
+.. code:: YAML
+
+  - name: "pytest-order"
+    repo: "pytest-dev/pytest-order"
+    docker_profile: "nodb"
+    jobs: "test"
+    workflow_name: "pythontests.yml"
+    matrix_exclude: |
+      3.6
+
+.. epigraph::
+  Example 2: using ``pytest-order``, which has a non-combination strategy matrix and requires no database.

From e6b66d1da116eaa7487a3e37e73e7f6419f91d17 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Sat, 16 Apr 2022 13:05:31 -0500
Subject: [PATCH 42/59] fix rST formatting

---
 testing/downstream_testing/README.rst | 209 ++++++++++++++------------
 1 file changed, 109 insertions(+), 100 deletions(-)

diff --git a/testing/downstream_testing/README.rst b/testing/downstream_testing/README.rst
index 025a05649..6887d14a3 100644
--- a/testing/downstream_testing/README.rst
+++ b/testing/downstream_testing/README.rst
@@ -34,87 +34,96 @@ Example ``actions_schema.json`` entry:
 
   .. code:: JSON
 
-    "pytest-django": {
-      "matrix": [
-        "matrix",
-        "include"
-      ],
-      "tox_cmd_build": {
-        "base": "name",
-        "prefix": "py",
-        "sub": {
-          "pattern": "-coverage$",
-          "replace": ""
+      "pytest-django": {
+        "matrix": [
+          "matrix",
+          "include"
+        ],
+        "tox_cmd_build": {
+          "base": "name",
+          "prefix": "py",
+          "sub": {
+            "pattern": "-coverage$",
+            "replace": ""
+          }
         }
       }
-    }
 
 Below are the steps to add a new plugin.
 
 1. Start by adding a new JSON object, with the name of the repo. ``"pytest-django": {}`` in the example above.
 
 2. Add an array named ``matrix``. Array items should point to the strategy matrix from the repo's GitHub Actions
-   YAML file. For instance, the ``pytest-django`` example above has a strategy matrix defined as follows:
+   YAML file. For instance, the ``pytest-django`` example above has a strategy matrix defined as follows
 
     .. code:: YAML
 
-      # pytest-django/.github/workflows/main.yml
-      jobs:
-        tests:
-          strategy:
-            fail-fast: false
-            matrix:
-              include:
-              - name: linting,docs
-                python: 3.8
-                allow_failure: false
+        # pytest-django/.github/workflows/main.yml
+        jobs:
+          tests:
+            strategy:
+              fail-fast: false
+              matrix:
+                include:
+                - name: linting,docs
+                  python: 3.8
+                  allow_failure: false
 
-              - name: py310-dj40-postgres-xdist-coverage
-                python: '3.10'
-                allow_failure: false
+                - name: py310-dj40-postgres-xdist-coverage
+                  python: '3.10'
+                  allow_failure: false
 
 
-  This makes ``["matrix", "include"]`` our target to parse the ``pytest-django`` strategy. This is a "combination"
-  strategy based on the use of ``include``.
+   This makes ``["matrix", "include"]`` our target to parse the ``pytest-django`` strategy. This is a "combination"
+   strategy based on the use of ``include``.
 
-  For non-combination strategies, use the matrix field that points to the appropriate choices. Using
-  ``pytest-order`` as a non-combination example:
+   For non-combination strategies, use the matrix field that points to the appropriate choices. Using
+   ``pytest-order`` as a non-combination example:
 
     .. code:: YAML
 
-      # pytest-order/.github/workflows/pythontests.yml
-      jobs:
-        test:
-          runs-on: ${{ matrix.os }}
-          strategy:
-            fail-fast: false
-            matrix:
-              os: [ubuntu-latest, windows-latest]
-              python-version: [3.6, 3.7, 3.8, 3.9, "3.10", pypy3]
+        # pytest-order/.github/workflows/pythontests.yml
+        jobs:
+          test:
+            runs-on: ${{ matrix.os }}
+            strategy:
+              fail-fast: false
+              matrix:
+                os: [ubuntu-latest, windows-latest]
+                python-version: [3.6, 3.7, 3.8, 3.9, "3.10", pypy3]
 
-  The corresponding entry in ``actions_schema.json`` points to ``["matrix", "python-version"]``:
+   The corresponding entry in ``actions_schema.json`` points to ``["matrix", "python-version"]``:
 
     .. code:: JSON
 
-      "pytest-order": {
-          "matrix": [
-              "matrix",
-              "python-version"
-          ],
+        "pytest-order": {
+            "matrix": [
+                "matrix",
+                "python-version"
+            ],
 
 3. Add a JSON object named ``tox_cmd_build``, with three items: ``base``, ``prefix``, and ``sub``.
 
-   - ``base``: For combination strategies (with ``include``), ``base`` is the field to be used as the basis
-      of the tox environment. For non-combination strategies, this field is an empty string. ``base: "name"``
-      in the ``pytest-django`` example above.
+   - ``base``:
 
-   - ``prefix``: For combination strategies, ``prefix`` is used to [dis]qualify entries in ``base``. For
-       non-combination strategies, this field is an emtpy string. ``prefix: "py"`` in the ``pytest-django``
+     - For combination strategies (with ``include``), ``base`` is the field to be used as the basis
+       of the tox environment.
+
+     - For non-combination strategies, this field is an empty string. ``base: "name"``
+       in the ``pytest-django`` example above.
+
+   - ``prefix``:
+
+     - For combination strategies, ``prefix`` is used to [dis]qualify entries in ``base``.
+
+     - For non-combination strategies, this field is an emtpy string. ``prefix: "py"`` in the ``pytest-django``
        example above.
 
-   - ``sub``: For both combination and non-combination strategies, this JSON object gives a RegEx matching
-      (``pattern``) and a substituition (``replace``) string. Since these are JSON strings, they cannot be
-      represented as a Python raw string (``r""``); ensure to properly escape characters.
+   - ``sub``:
+
+     - For both combination and non-combination strategies, this JSON object gives a RegEx matching
+       (``pattern``) and a substituition (``replace``) string. Since these are JSON strings, they cannot be
+       represented as a Python raw string (``r""``); ensure to properly escape characters.
 
 Any additions can be verified locally with the following process:
 
@@ -133,33 +142,33 @@ Any additions can be verified locally with the following process:
 
    .. code::
 
-      (.venv) ~/pytest:$> python -m testing.downstream_testing.downstream_runner pytest-order pytest-order/pytest-order.yml test --matrix-exclude 3.6 --dry-run
+       (.venv) ~/pytest:$> python -m testing.downstream_testing.downstream_runner pytest-order pytest-order/main.yml test --matrix-exclude 3.6 --dry-run
 
-      DEBUG | downstream_runner.load_matrix_schema | Loading schema: /home/sommersoft/Dev/pytest-dev/pytest/testing/downstream_testing/action_schemas.json
-      DEBUG | downstream_runner.load_matrix_schema | 'pytest-order' schema loaded: {'matrix': ['matrix', 'python-version'],
-      'tox_cmd_build': {'base': '', 'prefix': '', 'sub': {'pattern': '(\\d|py\\d)\\.*(\\d+)', 'replace': 'py\\1\\2'}}, 'python_version': 'python-version'}
-      DEBUG | downstream_runner.inject_pytest_dep | toxenv dependencies updated: {'!pytest{60,61,62,624,70}: pytest-xdist', '!pytest50: pytest @ file:///home/pytest'}
-      DEBUG | downstream_runner.build_run | job_name: test
-      DEBUG | downstream_runner.parse_matrix | parsed_matrix: [3.6, 3.7, 3.8, 3.9, '3.10', 'pypy3']
-      DEBUG | downstream_runner.matrix | matrix: {'test': [{'name': 'py37', 'tox_cmd': 'py37'}, {'name': 'py38', 'tox_cmd': 'py38'}, {'name': 'py39', 'tox_cmd': 'py39'},
-      {'name': 'py310', 'tox_cmd': 'py310'}, {'name': 'pypy3', 'tox_cmd': 'pypy3'}]}
-      DEBUG | downstream_runner.build_run | matrix[job]: {'name': 'py37', 'tox_cmd': 'py37'}
-      DEBUG | downstream_runner.build_run | matrix[job]: {'name': 'py38', 'tox_cmd': 'py38'}
-      DEBUG | downstream_runner.build_run | matrix[job]: {'name': 'py39', 'tox_cmd': 'py39'}
-      DEBUG | downstream_runner.build_run | matrix[job]: {'name': 'py310', 'tox_cmd': 'py310'}
-      DEBUG | downstream_runner.build_run | matrix[job]: {'name': 'pypy3', 'tox_cmd': 'pypy3'}
-      DEBUG | downstream_runner.build_run | built run: {'py37': ['pip install tox', 'tox -e py37'], 'py38': ['pip install tox', 'tox -e py38'], 'py39': ['pip install tox',
-      'tox -e py39'], 'py310': ['pip install tox', 'tox -e py310'], 'pypy3': ['pip install tox', 'tox -e pypy3']}
-      INFO | downstream_runner.run | --> running: 'pip install tox'
-      INFO | downstream_runner.run | --> running: 'tox -e py37'
-      INFO | downstream_runner.run | --> running: 'pip install tox'
-      INFO | downstream_runner.run | --> running: 'tox -e py38'
-      INFO | downstream_runner.run | --> running: 'pip install tox'
-      INFO | downstream_runner.run | --> running: 'tox -e py39'
-      INFO | downstream_runner.run | --> running: 'pip install tox'
-      INFO | downstream_runner.run | --> running: 'tox -e py310'
-      INFO | downstream_runner.run | --> running: 'pip install tox'
-      INFO | downstream_runner.run | --> running: 'tox -e pypy3'
+       DEBUG | downstream_runner.load_matrix_schema | Loading schema: /home/pytest/testing/downstream_testing/action_schemas.json
+       DEBUG | downstream_runner.load_matrix_schema | 'pytest-order' schema loaded: {'matrix': ['matrix', 'python-version'],
+       'tox_cmd_build': {'base': '', 'prefix': '', 'sub': {'pattern': '(\\d|py\\d)\\.*(\\d+)', 'replace': 'py\\1\\2'}}, 'python_version': 'python-version'}
+       DEBUG | downstream_runner.inject_pytest_dep | toxenv dependencies updated: {'!pytest{60,61,62,624,70}: pytest-xdist', '!pytest50: pytest @ file:///home/pytest'}
+       DEBUG | downstream_runner.build_run | job_name: test
+       DEBUG | downstream_runner.parse_matrix | parsed_matrix: [3.6, 3.7, 3.8, 3.9, '3.10', 'pypy3']
+       DEBUG | downstream_runner.matrix | matrix: {'test': [{'name': 'py37', 'tox_cmd': 'py37'}, {'name': 'py38', 'tox_cmd': 'py38'}, {'name': 'py39', 'tox_cmd': 'py39'},
+       {'name': 'py310', 'tox_cmd': 'py310'}, {'name': 'pypy3', 'tox_cmd': 'pypy3'}]}
+       DEBUG | downstream_runner.build_run | matrix[job]: {'name': 'py37', 'tox_cmd': 'py37'}
+       DEBUG | downstream_runner.build_run | matrix[job]: {'name': 'py38', 'tox_cmd': 'py38'}
+       DEBUG | downstream_runner.build_run | matrix[job]: {'name': 'py39', 'tox_cmd': 'py39'}
+       DEBUG | downstream_runner.build_run | matrix[job]: {'name': 'py310', 'tox_cmd': 'py310'}
+       DEBUG | downstream_runner.build_run | matrix[job]: {'name': 'pypy3', 'tox_cmd': 'pypy3'}
+       DEBUG | downstream_runner.build_run | built run: {'py37': ['pip install tox', 'tox -e py37'], 'py38': ['pip install tox', 'tox -e py38'], 'py39': ['pip install tox',
+       'tox -e py39'], 'py310': ['pip install tox', 'tox -e py310'], 'pypy3': ['pip install tox', 'tox -e pypy3']}
+       INFO | downstream_runner.run | --> running: 'pip install tox'
+       INFO | downstream_runner.run | --> running: 'tox -e py37'
+       INFO | downstream_runner.run | --> running: 'pip install tox'
+       INFO | downstream_runner.run | --> running: 'tox -e py38'
+       INFO | downstream_runner.run | --> running: 'pip install tox'
+       INFO | downstream_runner.run | --> running: 'tox -e py39'
+       INFO | downstream_runner.run | --> running: 'pip install tox'
+       INFO | downstream_runner.run | --> running: 'tox -e py310'
+       INFO | downstream_runner.run | --> running: 'pip install tox'
+       INFO | downstream_runner.run | --> running: 'tox -e pypy3'
 
 
 **pytest/.github/workflows/downstream_testing.yml:**
@@ -184,21 +193,21 @@ Add a new entry to the combination strategy matrix:
 
 .. code:: YAML
 
-  - name: "pytest-django"
-    repo: "pytest-dev/pytest-django"
-    docker_profile: "postgres"
-    jobs: "test"
-    workflow_name: "main.yml"
-    matrix_exclude: |
-      linting,docs py39-dj40-mysql_innodb-coverage ...
+    - name: "pytest-django"
+      repo: "pytest-dev/pytest-django"
+      docker_profile: "postgres"
+      jobs: "test"
+      workflow_name: "main.yml"
+      matrix_exclude: |
+        linting,docs py39-dj40-mysql_innodb-coverage ...
 
-  - name: "pytest-django"
-    repo: "pytest-dev/pytest-django"
-    docker_profile: "mysql"
-    jobs: "test"
-    workflow_name: "main.yml"
-    matrix_exclude: |
-      linting,docs py310-dj40-postgres-xdist-coverage ...
+    - name: "pytest-django"
+      repo: "pytest-dev/pytest-django"
+      docker_profile: "mysql"
+      jobs: "test"
+      workflow_name: "main.yml"
+      matrix_exclude: |
+        linting,docs py310-dj40-postgres-xdist-coverage ...
 
 .. epigraph::
   Example 1: using ``pytest-django``, which has a combination strategy matrix, we see two (of three) different
@@ -208,13 +217,13 @@ Add a new entry to the combination strategy matrix:
 
 .. code:: YAML
 
-  - name: "pytest-order"
-    repo: "pytest-dev/pytest-order"
-    docker_profile: "nodb"
-    jobs: "test"
-    workflow_name: "pythontests.yml"
-    matrix_exclude: |
-      3.6
+    - name: "pytest-order"
+      repo: "pytest-dev/pytest-order"
+      docker_profile: "nodb"
+      jobs: "test"
+      workflow_name: "pythontests.yml"
+      matrix_exclude: |
+        3.6
 
 .. epigraph::
   Example 2: using ``pytest-order``, which has a non-combination strategy matrix and requires no database.

From f2f21b1a2b7c7cc0190247841e25c363d8fe1cdb Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Sun, 17 Apr 2022 08:45:34 -0500
Subject: [PATCH 43/59] add Actions step for more concise results readability

---
 .github/workflows/downstream_testing.yml      | 5 ++++-
 testing/downstream_testing/docker-compose.yml | 3 +++
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/downstream_testing.yml b/.github/workflows/downstream_testing.yml
index a57a9d595..bccc144f2 100644
--- a/.github/workflows/downstream_testing.yml
+++ b/.github/workflows/downstream_testing.yml
@@ -70,7 +70,7 @@ jobs:
         fetch-depth: 0
         repository: ${{ matrix.repo }}
         path: ${{ matrix.name }}
-    - name: Run Downstream Tests
+    - name: Run Downstream Tests - {{ matrix.name }}
       run: docker-compose -f ./testing/downstream_testing/docker-compose.yml --profile ${{ matrix.docker_profile }} up --exit-code-from base_${{ matrix.docker_profile }}
       env:
         DS_NAME: ${{ matrix.name }}
@@ -79,3 +79,6 @@ jobs:
         DS_MATRIX_EXCLUDE: ${{ matrix.matrix_exclude }}
         DOCKER_BUILDKIT: 1
         COV_CMD: ""
+    - name: Test Results - {{ matrix.name }}
+      if: ${{ always() }}
+      run: docker logs base
diff --git a/testing/downstream_testing/docker-compose.yml b/testing/downstream_testing/docker-compose.yml
index ee3fcd5e9..ea5bf33a8 100644
--- a/testing/downstream_testing/docker-compose.yml
+++ b/testing/downstream_testing/docker-compose.yml
@@ -1,6 +1,7 @@
 version: "3"
 services:
   base_nodb:
+    container_name: base
     build: .
     environment:
       - DS_NAME
@@ -18,6 +19,7 @@ services:
       - nodb
 
   base_postgres:
+    container_name: base
     build: .
     environment:
       - DS_NAME
@@ -49,6 +51,7 @@ services:
       - postgres
 
   base_mysql:
+    container_name: base
     build: .
     environment:
       - DS_NAME

From 909f699b73a3aebe7f766cc699ad5efd1a3c33e2 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Sun, 17 Apr 2022 17:43:27 -0500
Subject: [PATCH 44/59] fix expression

---
 .github/workflows/downstream_testing.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/downstream_testing.yml b/.github/workflows/downstream_testing.yml
index bccc144f2..6f568bf21 100644
--- a/.github/workflows/downstream_testing.yml
+++ b/.github/workflows/downstream_testing.yml
@@ -70,7 +70,7 @@ jobs:
         fetch-depth: 0
         repository: ${{ matrix.repo }}
         path: ${{ matrix.name }}
-    - name: Run Downstream Tests - {{ matrix.name }}
+    - name: Run Downstream Tests - ${{ matrix.name }}
       run: docker-compose -f ./testing/downstream_testing/docker-compose.yml --profile ${{ matrix.docker_profile }} up --exit-code-from base_${{ matrix.docker_profile }}
       env:
         DS_NAME: ${{ matrix.name }}
@@ -79,6 +79,6 @@ jobs:
         DS_MATRIX_EXCLUDE: ${{ matrix.matrix_exclude }}
         DOCKER_BUILDKIT: 1
         COV_CMD: ""
-    - name: Test Results - {{ matrix.name }}
+    - name: Test Results - ${{ matrix.name }}
       if: ${{ always() }}
       run: docker logs base

From 15ee41d22d67b3d32e081896e932ca13ecdd9485 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Sun, 17 Apr 2022 18:38:13 -0500
Subject: [PATCH 45/59] remove 'python_version' from action_schemas

---
 testing/downstream_testing/action_schemas.json | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/testing/downstream_testing/action_schemas.json b/testing/downstream_testing/action_schemas.json
index 7f16f2180..70e2fe3f8 100644
--- a/testing/downstream_testing/action_schemas.json
+++ b/testing/downstream_testing/action_schemas.json
@@ -12,8 +12,7 @@
                     "pattern": "-coverage$",
                     "replace": ""
                 }
-        },
-        "python_version": "python"
+        }
     },
     "pytest-html": {
         "matrix": [
@@ -28,8 +27,7 @@
                     "pattern": "(py\\d+)-\\w+",
                     "replace": "\\1"
                 }
-        },
-        "python_version": "python"
+        }
     },
     "pytest-order": {
         "matrix": [
@@ -44,7 +42,6 @@
                     "pattern": "(\\d|py\\d)\\.*(\\d+)",
                     "replace": "py\\1\\2"
                 }
-        },
-        "python_version": "python-version"
+        }
     }
 }

From 2c1a1bf68edf33da5336db8b5fd88ae1394899b2 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Sun, 17 Apr 2022 18:40:41 -0500
Subject: [PATCH 46/59] add pytest-mock

---
 .github/workflows/downstream_testing.yml       |  7 +++++++
 testing/downstream_testing/action_schemas.json | 15 +++++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/.github/workflows/downstream_testing.yml b/.github/workflows/downstream_testing.yml
index 6f568bf21..0301a4da4 100644
--- a/.github/workflows/downstream_testing.yml
+++ b/.github/workflows/downstream_testing.yml
@@ -60,6 +60,13 @@ jobs:
             matrix_exclude: |
               3.6
 
+          - name: "pytest-mock"
+            repo: "pytest-dev/pytest-mock"
+            docker_profile: "nodb"
+            jobs: "test"
+            workflow_name: "test.yml"
+            matrix_exclude: ""
+
     steps:
     - uses: actions/checkout@v2
       with:
diff --git a/testing/downstream_testing/action_schemas.json b/testing/downstream_testing/action_schemas.json
index 70e2fe3f8..f822db873 100644
--- a/testing/downstream_testing/action_schemas.json
+++ b/testing/downstream_testing/action_schemas.json
@@ -43,5 +43,20 @@
                     "replace": "py\\1\\2"
                 }
         }
+    },
+    "pytest-mock": {
+        "matrix": [
+            "matrix",
+            "python"
+        ],
+        "tox_cmd_build": {
+            "base": "",
+            "prefix": "",
+            "sub":
+                {
+                    "pattern": "(\\d|py\\d)\\.*(\\d+)",
+                    "replace": "py\\1\\2"
+                }
+        }
     }
 }

From a9011880a343302f9b1e1228c1c2a3c81da58783 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Mon, 18 Apr 2022 08:36:07 -0500
Subject: [PATCH 47/59] do not install tox repeatedly

---
 testing/downstream_testing/README.rst           | 10 ++--------
 testing/downstream_testing/downstream_runner.py |  2 +-
 testing/downstream_testing/entrypoint.sh        |  2 +-
 3 files changed, 4 insertions(+), 10 deletions(-)

diff --git a/testing/downstream_testing/README.rst b/testing/downstream_testing/README.rst
index 6887d14a3..ea127f109 100644
--- a/testing/downstream_testing/README.rst
+++ b/testing/downstream_testing/README.rst
@@ -20,7 +20,7 @@ Overview
 
   - The plugin's ``tox`` config is adjusted to ensure the local ``pytest`` is used.
 
-- Uses a static command set: ``pip install tox`` & ``tox -e {toxenv}``.
+- Uses a static command set: ``tox -e {toxenv}``.
 
 How To Add Additional Plugin(s)
 -------------------------------
@@ -157,17 +157,11 @@ Any additions can be verified locally with the following process:
        DEBUG | downstream_runner.build_run | matrix[job]: {'name': 'py39', 'tox_cmd': 'py39'}
        DEBUG | downstream_runner.build_run | matrix[job]: {'name': 'py310', 'tox_cmd': 'py310'}
        DEBUG | downstream_runner.build_run | matrix[job]: {'name': 'pypy3', 'tox_cmd': 'pypy3'}
-       DEBUG | downstream_runner.build_run | built run: {'py37': ['pip install tox', 'tox -e py37'], 'py38': ['pip install tox', 'tox -e py38'], 'py39': ['pip install tox',
-       'tox -e py39'], 'py310': ['pip install tox', 'tox -e py310'], 'pypy3': ['pip install tox', 'tox -e pypy3']}
-       INFO | downstream_runner.run | --> running: 'pip install tox'
+       DEBUG | downstream_runner.build_run | built run: {'py37': ['tox -e py37'], 'py38': ['tox -e py38'], 'py39': ['tox -e py39'], 'py310': ['tox -e py310'], 'pypy3': ['tox -e pypy3']}
        INFO | downstream_runner.run | --> running: 'tox -e py37'
-       INFO | downstream_runner.run | --> running: 'pip install tox'
        INFO | downstream_runner.run | --> running: 'tox -e py38'
-       INFO | downstream_runner.run | --> running: 'pip install tox'
        INFO | downstream_runner.run | --> running: 'tox -e py39'
-       INFO | downstream_runner.run | --> running: 'pip install tox'
        INFO | downstream_runner.run | --> running: 'tox -e py310'
-       INFO | downstream_runner.run | --> running: 'pip install tox'
        INFO | downstream_runner.run | --> running: 'tox -e pypy3'
 
 
diff --git a/testing/downstream_testing/downstream_runner.py b/testing/downstream_testing/downstream_runner.py
index e90856bcb..c1917ea6c 100644
--- a/testing/downstream_testing/downstream_runner.py
+++ b/testing/downstream_testing/downstream_runner.py
@@ -299,7 +299,7 @@ class DownstreamRunner:
             logger.debug("job_name: %s", job)
             for matrix in self.matrix[job]:
                 logger.debug("matrix[job]: %s", matrix)
-                run[matrix["name"]] = ["pip install tox", f"tox -e {matrix['tox_cmd']}"]
+                run[matrix["name"]] = [f"tox -e {matrix['tox_cmd']}"]
 
         logger.debug("built run: %s", run)
         return run
diff --git a/testing/downstream_testing/entrypoint.sh b/testing/downstream_testing/entrypoint.sh
index 9b0f35bd6..151bc5458 100755
--- a/testing/downstream_testing/entrypoint.sh
+++ b/testing/downstream_testing/entrypoint.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-python3.9 -m pip install --no-cache-dir pyyaml sh
+python3.9 -m pip install --no-cache-dir pyyaml tox sh
 
 cd /pytest
 python3.9 -u -m testing.downstream_testing.downstream_runner $DS_NAME $DS_YAML $DS_JOBS --matrix-exclude $DS_MATRIX_EXCLUDE

From d18ce867be1b57de5a957071b794fb58754966b8 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Fri, 22 Apr 2022 08:18:18 -0500
Subject: [PATCH 48/59] force pytest color output for all tests via PY_COLORS

---
 testing/downstream_testing/docker-compose.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/testing/downstream_testing/docker-compose.yml b/testing/downstream_testing/docker-compose.yml
index ea5bf33a8..73ffe12f8 100644
--- a/testing/downstream_testing/docker-compose.yml
+++ b/testing/downstream_testing/docker-compose.yml
@@ -9,6 +9,8 @@ services:
       - DS_JOBS
       - DS_MATRIX_EXCLUDE
       - COV_CMD
+      - PY_COLORS=1
+      - TOX_TESTENV_PASSENV=PY_COLORS
     volumes:
       - type: bind
         source: /home/runner/work/pytest/pytest
@@ -29,6 +31,8 @@ services:
       - TEST_DB_USER=$USER
       - TEST_DB_PASSWORD=pytest_djang0
       - TEST_DB_HOST=postgres_db
+      - PY_COLORS=1
+      - TOX_TESTENV_PASSENV=PY_COLORS
     volumes:
       - type: bind
         source: /home/runner/work/pytest/pytest
@@ -61,6 +65,8 @@ services:
       - TEST_DB_USER=root
       - TEST_DB_PASSWORD=root
       - TEST_DB_HOST=mysql_db
+      - PY_COLORS=1
+      - TOX_TESTENV_PASSENV=PY_COLORS
     volumes:
       - type: bind
         source: /home/runner/work/pytest/pytest

From 084ffa41aa3364030b5c179397d4daa9c17622a0 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Sat, 23 Apr 2022 08:27:24 -0500
Subject: [PATCH 49/59] sort the docker logs by timestamp; docker-compose
 writes log entries FIFO which may not be chronological

---
 .github/workflows/downstream_testing.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/downstream_testing.yml b/.github/workflows/downstream_testing.yml
index 0301a4da4..b2293f484 100644
--- a/.github/workflows/downstream_testing.yml
+++ b/.github/workflows/downstream_testing.yml
@@ -88,4 +88,6 @@ jobs:
         COV_CMD: ""
     - name: Test Results - ${{ matrix.name }}
       if: ${{ always() }}
-      run: docker logs base
+      run: |
+        docker logs base -t &>docker.log
+        sort -b -k 1 docker.log

From 4b9000c83213d233dd688cb4ee1bfe80dc1d3a2f Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Sat, 23 Apr 2022 10:43:35 -0500
Subject: [PATCH 50/59] add pytest-cov

---
 .github/workflows/downstream_testing.yml |  9 +++
 pytest-cov/test.yml                      | 81 ++++++++++++++++++++++++
 2 files changed, 90 insertions(+)
 create mode 100644 pytest-cov/test.yml

diff --git a/.github/workflows/downstream_testing.yml b/.github/workflows/downstream_testing.yml
index b2293f484..4299511f3 100644
--- a/.github/workflows/downstream_testing.yml
+++ b/.github/workflows/downstream_testing.yml
@@ -67,6 +67,15 @@ jobs:
             workflow_name: "test.yml"
             matrix_exclude: ""
 
+          - name: "pytest-cov"
+            repo: "pytest-dev/pytest-cov"
+            docker_profile: "nodb"
+            jobs: "test"
+            workflow_name: "test.yml"
+            # Exclude pypy3 for now; stil working on a suitable regex substitution
+            matrix_exclude: |
+              3.6 pypy-3.6 pypy-3.7
+
     steps:
     - uses: actions/checkout@v2
       with:
diff --git a/pytest-cov/test.yml b/pytest-cov/test.yml
new file mode 100644
index 000000000..fa420f547
--- /dev/null
+++ b/pytest-cov/test.yml
@@ -0,0 +1,81 @@
+name: Test
+
+on: [push, pull_request, workflow_dispatch]
+
+env:
+  FORCE_COLOR: 1
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["pypy-3.6", "pypy-3.7", "3.6", "3.7", "3.8", "3.9", "3.10-dev"]
+        tox-extra-versions: [
+            "pytest46-xdist127",
+            "pytest46-xdist133",
+            "pytest54-xdist133",
+            "pytest62-xdist202",
+        ]
+        include:
+          # Add new helper variables to existing jobs
+          - {python-version: "pypy-3.6", tox-python-version: "pypy3"}
+          - {python-version: "pypy-3.7", tox-python-version: "pypy3"}
+          - {python-version: "3.6", tox-python-version: "py36"}
+          - {python-version: "3.7", tox-python-version: "py37"}
+          - {python-version: "3.8", tox-python-version: "py38"}
+          - {python-version: "3.9", tox-python-version: "py39"}
+          - {python-version: "3.10-dev", tox-python-version: "py310"}
+        exclude:
+          # Remove some jobs from the matrix
+          - {tox-extra-versions: "pytest46-xdist127", python-version: "3.8"}
+          - {tox-extra-versions: "pytest46-xdist127", python-version: "3.9"}
+          - {tox-extra-versions: "pytest46-xdist133", python-version: "3.9"}
+          - {tox-extra-versions: "pytest54-xdist133", python-version: "3.9"}
+          - {tox-extra-versions: "pytest46-xdist127", python-version: "3.10-dev"}
+          - {tox-extra-versions: "pytest46-xdist133", python-version: "3.10-dev"}
+          - {tox-extra-versions: "pytest54-xdist133", python-version: "3.10-dev"}
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Get pip cache dir
+        id: pip-cache
+        run: |
+          echo "::set-output name=dir::$(pip cache dir)"
+
+      - name: Cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key:
+            test-${{ matrix.python-version }}-v1-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            test-${{ matrix.python-version }}-v1-
+
+      - name: Install dependencies
+        run: |
+          python -m pip install -U pip
+          python -m pip install -U wheel
+          python -m pip install --progress-bar=off tox -rci/requirements.txt
+          virtualenv --version
+          pip --version
+          tox --version
+
+      - name: Tox tests
+        run: |
+          tox -v -e ${{ matrix.tox-python-version }}-${{ matrix.tox-extra-versions }}-coverage55
+
+  allgood:
+    needs: test
+    runs-on: ubuntu-latest
+    name: Test successful
+    steps:
+      - name: Success
+        run: echo Test successful
\ No newline at end of file

From 00787d3a4d512224d56845a51e8f5a3f5c62fc29 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Sat, 23 Apr 2022 11:01:47 -0500
Subject: [PATCH 51/59] Revert "add pytest-cov"

This reverts commit 4b9000c83213d233dd688cb4ee1bfe80dc1d3a2f.
---
 .github/workflows/downstream_testing.yml |  9 ---
 pytest-cov/test.yml                      | 81 ------------------------
 2 files changed, 90 deletions(-)
 delete mode 100644 pytest-cov/test.yml

diff --git a/.github/workflows/downstream_testing.yml b/.github/workflows/downstream_testing.yml
index 4299511f3..b2293f484 100644
--- a/.github/workflows/downstream_testing.yml
+++ b/.github/workflows/downstream_testing.yml
@@ -67,15 +67,6 @@ jobs:
             workflow_name: "test.yml"
             matrix_exclude: ""
 
-          - name: "pytest-cov"
-            repo: "pytest-dev/pytest-cov"
-            docker_profile: "nodb"
-            jobs: "test"
-            workflow_name: "test.yml"
-            # Exclude pypy3 for now; stil working on a suitable regex substitution
-            matrix_exclude: |
-              3.6 pypy-3.6 pypy-3.7
-
     steps:
     - uses: actions/checkout@v2
       with:
diff --git a/pytest-cov/test.yml b/pytest-cov/test.yml
deleted file mode 100644
index fa420f547..000000000
--- a/pytest-cov/test.yml
+++ /dev/null
@@ -1,81 +0,0 @@
-name: Test
-
-on: [push, pull_request, workflow_dispatch]
-
-env:
-  FORCE_COLOR: 1
-
-jobs:
-  test:
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["pypy-3.6", "pypy-3.7", "3.6", "3.7", "3.8", "3.9", "3.10-dev"]
-        tox-extra-versions: [
-            "pytest46-xdist127",
-            "pytest46-xdist133",
-            "pytest54-xdist133",
-            "pytest62-xdist202",
-        ]
-        include:
-          # Add new helper variables to existing jobs
-          - {python-version: "pypy-3.6", tox-python-version: "pypy3"}
-          - {python-version: "pypy-3.7", tox-python-version: "pypy3"}
-          - {python-version: "3.6", tox-python-version: "py36"}
-          - {python-version: "3.7", tox-python-version: "py37"}
-          - {python-version: "3.8", tox-python-version: "py38"}
-          - {python-version: "3.9", tox-python-version: "py39"}
-          - {python-version: "3.10-dev", tox-python-version: "py310"}
-        exclude:
-          # Remove some jobs from the matrix
-          - {tox-extra-versions: "pytest46-xdist127", python-version: "3.8"}
-          - {tox-extra-versions: "pytest46-xdist127", python-version: "3.9"}
-          - {tox-extra-versions: "pytest46-xdist133", python-version: "3.9"}
-          - {tox-extra-versions: "pytest54-xdist133", python-version: "3.9"}
-          - {tox-extra-versions: "pytest46-xdist127", python-version: "3.10-dev"}
-          - {tox-extra-versions: "pytest46-xdist133", python-version: "3.10-dev"}
-          - {tox-extra-versions: "pytest54-xdist133", python-version: "3.10-dev"}
-
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Get pip cache dir
-        id: pip-cache
-        run: |
-          echo "::set-output name=dir::$(pip cache dir)"
-
-      - name: Cache
-        uses: actions/cache@v2
-        with:
-          path: ${{ steps.pip-cache.outputs.dir }}
-          key:
-            test-${{ matrix.python-version }}-v1-${{ hashFiles('**/requirements.txt') }}
-          restore-keys: |
-            test-${{ matrix.python-version }}-v1-
-
-      - name: Install dependencies
-        run: |
-          python -m pip install -U pip
-          python -m pip install -U wheel
-          python -m pip install --progress-bar=off tox -rci/requirements.txt
-          virtualenv --version
-          pip --version
-          tox --version
-
-      - name: Tox tests
-        run: |
-          tox -v -e ${{ matrix.tox-python-version }}-${{ matrix.tox-extra-versions }}-coverage55
-
-  allgood:
-    needs: test
-    runs-on: ubuntu-latest
-    name: Test successful
-    steps:
-      - name: Success
-        run: echo Test successful
\ No newline at end of file

From cd9fd6a1ce7a90f69613943afb3c9fa14e7b9181 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Sat, 23 Apr 2022 11:04:08 -0500
Subject: [PATCH 52/59] re-add pytest-cov; this time with the correct changes

---
 .github/workflows/downstream_testing.yml       |  9 +++++++++
 testing/downstream_testing/action_schemas.json | 15 +++++++++++++++
 2 files changed, 24 insertions(+)

diff --git a/.github/workflows/downstream_testing.yml b/.github/workflows/downstream_testing.yml
index b2293f484..4299511f3 100644
--- a/.github/workflows/downstream_testing.yml
+++ b/.github/workflows/downstream_testing.yml
@@ -67,6 +67,15 @@ jobs:
             workflow_name: "test.yml"
             matrix_exclude: ""
 
+          - name: "pytest-cov"
+            repo: "pytest-dev/pytest-cov"
+            docker_profile: "nodb"
+            jobs: "test"
+            workflow_name: "test.yml"
+            # Exclude pypy3 for now; stil working on a suitable regex substitution
+            matrix_exclude: |
+              3.6 pypy-3.6 pypy-3.7
+
     steps:
     - uses: actions/checkout@v2
       with:
diff --git a/testing/downstream_testing/action_schemas.json b/testing/downstream_testing/action_schemas.json
index f822db873..c6fa03c86 100644
--- a/testing/downstream_testing/action_schemas.json
+++ b/testing/downstream_testing/action_schemas.json
@@ -58,5 +58,20 @@
                     "replace": "py\\1\\2"
                 }
         }
+    },
+    "pytest-cov": {
+        "matrix": [
+            "matrix",
+            "python-version"
+        ],
+        "tox_cmd_build": {
+            "base": "",
+            "prefix": "",
+            "sub":
+                {
+                    "pattern": "^\\d\\.(\\d+)(?:-dev)*",
+                    "replace": "py3\\1"
+                }
+        }
     }
 }

From 868967241fd8b13b09bce30efea82628b1ab62c6 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Sat, 23 Apr 2022 20:58:52 -0500
Subject: [PATCH 53/59] add pytest-bdd

---
 .github/workflows/downstream_testing.yml       |  8 ++++++++
 testing/downstream_testing/action_schemas.json | 15 +++++++++++++++
 2 files changed, 23 insertions(+)

diff --git a/.github/workflows/downstream_testing.yml b/.github/workflows/downstream_testing.yml
index 4299511f3..5d95770da 100644
--- a/.github/workflows/downstream_testing.yml
+++ b/.github/workflows/downstream_testing.yml
@@ -76,6 +76,14 @@ jobs:
             matrix_exclude: |
               3.6 pypy-3.6 pypy-3.7
 
+          - name: "pytest-bdd"
+            repo: "pytest-dev/pytest-bdd"
+            docker_profile: "nodb"
+            jobs: "build"
+            workflow_name: "main.yml"
+            # Exclude pypy3 for now; stil working on a suitable regex substitution
+            matrix_exclude: ""
+
     steps:
     - uses: actions/checkout@v2
       with:
diff --git a/testing/downstream_testing/action_schemas.json b/testing/downstream_testing/action_schemas.json
index c6fa03c86..cd4ec8270 100644
--- a/testing/downstream_testing/action_schemas.json
+++ b/testing/downstream_testing/action_schemas.json
@@ -73,5 +73,20 @@
                     "replace": "py3\\1"
                 }
         }
+    },
+    "pytest-bdd": {
+        "matrix": [
+            "matrix",
+            "python-version"
+        ],
+        "tox_cmd_build": {
+            "base": "",
+            "prefix": "",
+            "sub":
+                {
+                    "pattern": "^\\d\\.(\\d+)",
+                    "replace": "py3\\1"
+                }
+        }
     }
 }

From 5f1137b3475a19be775777fb24f0933d69fc2a01 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Sat, 23 Apr 2022 21:13:00 -0500
Subject: [PATCH 54/59] fix build_run test after change in a901188

---
 testing/downstream_testing/test_downstream_runner.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/testing/downstream_testing/test_downstream_runner.py b/testing/downstream_testing/test_downstream_runner.py
index 1b22ba226..a444d3945 100644
--- a/testing/downstream_testing/test_downstream_runner.py
+++ b/testing/downstream_testing/test_downstream_runner.py
@@ -252,7 +252,6 @@ class TestDownstreamRunnerBuild:
         run = runner.build_run()
         assert run == {
             "py310-dj40-postgres-xdist-coverage": [
-                "pip install tox",
                 "tox -e py310-dj40-postgres-xdist",
             ]
         }
@@ -274,8 +273,8 @@ class TestDownstreamRunnerBuild:
 
         run = runner.build_run()
         assert run == {
-            "py310": ["pip install tox", "tox -e py310"],
-            "py38": ["pip install tox", "tox -e py38"],
-            "py39": ["pip install tox", "tox -e py39"],
-            "pypy3": ["pip install tox", "tox -e pypy3"],
+            "py310": ["tox -e py310"],
+            "py38": ["tox -e py38"],
+            "py39": ["tox -e py39"],
+            "pypy3": ["tox -e pypy3"],
         }

From 4d027263f3d28dcd23169f99c9aeeef622e5c817 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Sun, 24 Apr 2022 18:27:42 -0500
Subject: [PATCH 55/59] misc cleanup and standardizing

---
 .github/workflows/downstream_testing.yml        |  4 +---
 testing/downstream_testing/downstream_runner.py | 17 +++++++----------
 2 files changed, 8 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/downstream_testing.yml b/.github/workflows/downstream_testing.yml
index 5d95770da..9b4d4cd7e 100644
--- a/.github/workflows/downstream_testing.yml
+++ b/.github/workflows/downstream_testing.yml
@@ -57,8 +57,7 @@ jobs:
             docker_profile: "nodb"
             jobs: "test"
             workflow_name: "pythontests.yml"
-            matrix_exclude: |
-              3.6
+            matrix_exclude: 3.6
 
           - name: "pytest-mock"
             repo: "pytest-dev/pytest-mock"
@@ -81,7 +80,6 @@ jobs:
             docker_profile: "nodb"
             jobs: "build"
             workflow_name: "main.yml"
-            # Exclude pypy3 for now; stil working on a suitable regex substitution
             matrix_exclude: ""
 
     steps:
diff --git a/testing/downstream_testing/downstream_runner.py b/testing/downstream_testing/downstream_runner.py
index c1917ea6c..5d2f39251 100644
--- a/testing/downstream_testing/downstream_runner.py
+++ b/testing/downstream_testing/downstream_runner.py
@@ -59,7 +59,7 @@ else:
 
 
 def load_matrix_schema(repo: str) -> SchemaType:
-    """Loads the matrix schema for `repo`"""
+    """Loads the matrix schema for ``repo``"""
     schema: SchemaType = {"repo": repo}
     working_dir = os.getcwd()
     schema_path = os.path.join(
@@ -111,10 +111,6 @@ class ToxDepFilter(_BaseUserDict):
         """Checks if ``match`` matches any conditions"""
         match_found = None
         for key, val in self.data.items():
-            if "xdist" in match:
-                logging.debug(
-                    "matches_condition: %s", re.search(val["condition"], match)
-                )
             if re.search(val["condition"], match):
                 match_found = key
                 break
@@ -160,7 +156,7 @@ class DownstreamRunner:
 
     @property
     def yaml_tree(self) -> dict[str, Any]:
-        """The YAML tree built from the `self.yaml_source` file."""
+        """The YAML tree built from the ``self.yaml_source`` file."""
         if self._yaml_tree is None:
             with open(self.yaml_source) as f:
                 try:
@@ -179,7 +175,8 @@ class DownstreamRunner:
 
     def inject_pytest_dep(self) -> None:
         """Ensure pytest is a dependency in tox.ini to allow us to use the 'local'
-        version of pytest.
+        version of pytest. Also ensure other dependencies in ``TOX_DEP_FILTERS``
+        are defined appropriately.
         """
         ini_path = self.repo + "/tox.ini"
         pytest_dep = TOX_DEP_FILTERS["pytest"]["src"]
@@ -215,18 +212,18 @@ class DownstreamRunner:
             f"repo={self.repo}, "
             f"yaml_source={self.yaml_source}, "
             f"job_names={self.job_names}, "
-            f"matrix={self.matrix}, "
+            f"matrix={self._matrix}, "
             ")"
         )
 
     @property
     def matrix(self) -> dict[str, Iterable[dict[str, str]]]:
-        """Iterates over ``self.yaml_tree`` strategy matrix for each job in ``self.jobs``, and passes each
+        """Iterates over ``self.yaml_tree``'s strategy matrix for each job in ``self.jobs``, and passes each
         through ``parse_matrix``.
         """
 
         def parse_matrix(yaml_tree: dict[str, Any]) -> Iterable[Any]:
-            """Parses `yaml_tree` strategy matrix using ``self.matrix_schema`` information."""
+            """Parses ``yaml_tree`` strategy matrix using ``self.matrix_schema`` information."""
             parsed_matrix = []  # type: ignore
             pre_parsed: dict[str, Any] | Iterable[str | float] = yaml_tree
             for key in self.matrix_schema["matrix"]:

From 2eaf3cdc67c3ef1a4ed006352e2b319bc963657c Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Mon, 25 Apr 2022 07:36:29 -0500
Subject: [PATCH 56/59] README fixes and improvements

---
 testing/downstream_testing/README.rst | 25 ++++++++++++-------------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/testing/downstream_testing/README.rst b/testing/downstream_testing/README.rst
index ea127f109..bea6aa6b8 100644
--- a/testing/downstream_testing/README.rst
+++ b/testing/downstream_testing/README.rst
@@ -49,7 +49,7 @@ Example ``actions_schema.json`` entry:
         }
       }
 
-Below are the steps to add a new plugin.
+**To add a new plugin, complete the following:**
 
 1. Start by adding a new JSON object, with the name of the repo. ``"pytest-django": {}`` in the example above.
 
@@ -107,17 +107,16 @@ Below are the steps to add a new plugin.
    - ``base``:
 
      - For combination strategies (with ``include``), ``base`` is the field to be used as the basis
-       of the tox environment.
+       of the tox environment. ``base: "name"`` in the ``pytest-django`` example above.
 
-     - For non-combination strategies, this field is an empty string. ``base: "name"``
-       in the ``pytest-django`` example above.
+     - For non-combination strategies, this field is an empty string.
 
    - ``prefix``:
 
      - For combination strategies, ``prefix`` is used to [dis]qualify entries in ``base``.
+       ``prefix: "py"`` in the ``pytest-django`` example above.
 
-     - For non-combination strategies, this field is an emtpy string. ``prefix: "py"`` in the ``pytest-django``
-       example above.
+     - For non-combination strategies, this field is an emtpy string.
 
    - ``sub``:
 
@@ -125,22 +124,22 @@ Below are the steps to add a new plugin.
        (``pattern``) and a substituition (``replace``) string. Since these are JSON strings, they cannot be
        represented as a Python raw string (``r""``); ensure to properly escape characters.
 
-Any additions can be verified locally with the following process:
+**Testing additions locally:**
 
-1. Have a local copy of a plugin's GitHub Actions workflow YAML file, as well as a ``tox.ini``. These should
-   be placed in a folder; recommended folder name is the repo name.
+1. Have a local copy of a plugin's GitHub Actions workflow YAML file, as well as the ``tox.ini``. These should
+   be placed in a sub-folder in ``pytest``; recommended folder name is the repo name.
 
    .. code:: shell
 
-     .
+     . pytest/
      .. pytest-order/
-     .... main.yml
-     .... tox.ini
+     ... main.yml
+     ... tox.ini
 
 2. Utilize the ``--dry-run`` flag to run the parsing and command step building, without actually executing
    the commands. This will provide debugging information.
 
-   .. code::
+   .. code:: shell
 
        (.venv) ~/pytest:$> python -m testing.downstream_testing.downstream_runner pytest-order pytest-order/main.yml test --matrix-exclude 3.6 --dry-run
 

From 86b8cbdd406ac4706d78378a58209e313a0718a2 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Tue, 26 Apr 2022 09:08:33 -0500
Subject: [PATCH 57/59] prep for PR to upstream

---
 .github/workflows/downstream_testing.yml | 5 +++++
 changelog/7342.trivial.rst               | 2 ++
 2 files changed, 7 insertions(+)
 create mode 100644 changelog/7342.trivial.rst

diff --git a/.github/workflows/downstream_testing.yml b/.github/workflows/downstream_testing.yml
index 9b4d4cd7e..b39b798c8 100644
--- a/.github/workflows/downstream_testing.yml
+++ b/.github/workflows/downstream_testing.yml
@@ -1,6 +1,11 @@
 name: downstream_testing_2
 
 on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      # Only run downstream tests on PRs for releases
+      - "[0-9]+.[0-9]+.x"
   push:
     branches:
       - downstream_testing_2
diff --git a/changelog/7342.trivial.rst b/changelog/7342.trivial.rst
new file mode 100644
index 000000000..7cd4fecdc
--- /dev/null
+++ b/changelog/7342.trivial.rst
@@ -0,0 +1,2 @@
+Added a system to run downstream plugin tests against current pytest, using GitHub Actions. Currently 
+constrained to pytest plugins that utilize GitHub Actions and ``tox`` to accomplish their own tests.
\ No newline at end of file

From a36538c33b771621d5013fe83612a79d2b16c447 Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Tue, 26 Apr 2022 09:39:09 -0500
Subject: [PATCH 58/59] update pytest-django matrix with upstream changes

---
 .github/workflows/downstream_testing.yml | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/downstream_testing.yml b/.github/workflows/downstream_testing.yml
index b39b798c8..bdf7a352e 100644
--- a/.github/workflows/downstream_testing.yml
+++ b/.github/workflows/downstream_testing.yml
@@ -25,9 +25,8 @@ jobs:
             jobs: "test"
             workflow_name: "main.yml"
             matrix_exclude: |
-              linting,docs py39-dj40-mysql_innodb-coverage py36-dj22-sqlite-xdist-coverage py37-dj22-sqlite-xdist-coverage
-              py38-dj32-sqlite-xdist-coverage py38-dj40-sqlite-xdist-coverage py39-djmain-sqlite-coverage
-              py35-dj22-postgres-pytest54-coverage py35-dj22-sqlite_file-coverage py36-dj32-mysql_myisam-coverage
+              linting,docs py39-dj40-mysql_innodb-coverage py38-dj32-sqlite-xdist-coverage 
+              py38-dj40-sqlite-xdist-coverage py39-djmain-sqlite-coverage py36-dj32-mysql_myisam-coverage
           - name: "pytest-django"
             repo: "pytest-dev/pytest-django"
             docker_profile: "mysql"
@@ -35,20 +34,17 @@ jobs:
             workflow_name: "main.yml"
             matrix_exclude: |
               linting,docs py310-dj40-postgres-xdist-coverage py310-dj32-postgres-xdist-coverage
-              py39-dj32-postgres-xdist-coverage py35-dj22-postgres-pytest54-coverage pypy3-dj22-postgres
-              py36-dj22-sqlite-xdist-coverage py37-dj22-sqlite-xdist-coverage py38-dj32-sqlite-xdist-coverage
-              py38-dj40-sqlite-xdist-coverage py39-djmain-sqlite-coverage py35-dj22-sqlite_file-coverage
-              py36-dj32-mysql_myisam-coverage
+              py39-dj32-postgres-xdist-coverage py38-dj32-sqlite-xdist-coverage
+              py38-dj40-sqlite-xdist-coverage py39-djmain-sqlite-coverage pypy3-dj32-postgres
           - name: "pytest-django"
             repo: "pytest-dev/pytest-django"
             docker_profile: "nodb"
             jobs: "test"
             workflow_name: "main.yml"
             matrix_exclude: |
-              linting,docs py39-dj40-mysql_innodb-coverage py35-dj22-postgres-pytest54-coverage
-              py36-dj32-mysql_myisam-coverage py310-dj40-postgres-xdist-coverage
-              py310-dj32-postgres-xdist-coverage py39-dj32-postgres-xdist-coverage
-              pypy3-dj22-postgres py36-dj22-sqlite-xdist-coverage py35-dj22-sqlite_file-coverage
+              linting,docs py310-dj40-postgres-xdist-coverage py310-dj32-postgres-xdist-coverage
+              py39-dj32-postgres-xdist-coverage py39-dj40-mysql_innodb-coverage
+              py36-dj32-mysql_myisam-coverage pypy3-dj32-postgres
 
           - name: "pytest-html"
             repo: "pytest-dev/pytest-html"

From 65f95cf4f5cde21f22ec8c5ecb1cd18a9e0c05eb Mon Sep 17 00:00:00 2001
From: sommersoft <sommersoft@gmail.com>
Date: Tue, 26 Apr 2022 09:48:27 -0500
Subject: [PATCH 59/59] missed an exclusion of py36-mysql

---
 .github/workflows/downstream_testing.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/downstream_testing.yml b/.github/workflows/downstream_testing.yml
index bdf7a352e..30640b216 100644
--- a/.github/workflows/downstream_testing.yml
+++ b/.github/workflows/downstream_testing.yml
@@ -36,6 +36,7 @@ jobs:
               linting,docs py310-dj40-postgres-xdist-coverage py310-dj32-postgres-xdist-coverage
               py39-dj32-postgres-xdist-coverage py38-dj32-sqlite-xdist-coverage
               py38-dj40-sqlite-xdist-coverage py39-djmain-sqlite-coverage pypy3-dj32-postgres
+              py36-dj32-mysql_myisam-coverage
           - name: "pytest-django"
             repo: "pytest-dev/pytest-django"
             docker_profile: "nodb"