From 49f7972d489c9c203fb82f67b8ab074065d37dd0 Mon Sep 17 00:00:00 2001 From: holger krekel Date: Mon, 18 Jan 2010 16:18:59 +0100 Subject: [PATCH] some finalizing docs bit, regen plugin docs --HG-- branch : trunk --- bin-for-dist/makepluginlist.py | 17 +++-- doc/announce/release-1.2.0.txt | 115 ++++++++++++++++++++++++++++ doc/test/dist.txt | 132 --------------------------------- doc/test/features.txt | 105 +++++++++++++------------- doc/test/index.txt | 1 - doc/test/plugin/index.txt | 30 +++----- doc/test/plugin/links.txt | 5 +- doc/test/plugin/xdist.txt | 2 +- 8 files changed, 188 insertions(+), 219 deletions(-) create mode 100644 doc/announce/release-1.2.0.txt delete mode 100644 doc/test/dist.txt diff --git a/bin-for-dist/makepluginlist.py b/bin-for-dist/makepluginlist.py index bc1055dd8..1f4840619 100644 --- a/bin-for-dist/makepluginlist.py +++ b/bin-for-dist/makepluginlist.py @@ -6,12 +6,10 @@ plugins = [ ('advanced python testing', 'skipping mark pdb figleaf coverage ' 'monkeypatch capture recwarn tmpdir',), - ('other testing domains, misc', - 'oejskit django xdist genscript'), - ('reporting and failure logging', - 'pastebin junitxml xmlresult resultlog terminal',), - ('other testing conventions', - 'unittest nose doctest restdoc'), + ('distributed testing, CI and deployment', + 'xdist pastebin junitxml resultlog genscript',), + ('testing domains and conventions', + 'oejskit django unittest nose doctest restdoc'), ('core debugging / help functionality', 'helpconfig hooklog') #('internal plugins / core functionality', @@ -22,6 +20,8 @@ plugins = [ externals = { 'oejskit': "run javascript tests in real life browsers", + 'xdist': None, + 'figleaf': None, 'django': "for testing django applications", 'coverage': "for testing with Ned's coverage module ", 'xmlresult': "for generating xml reports " @@ -143,7 +143,10 @@ class PluginOverview(RestWriter): doc = PluginDoc(docpath) doc.make(config=config, name=name) self.add_internal_link(name, doc.target) - self.para("%s_ %s" %(name, doc.oneliner)) + if name in externals: + self.para("%s_ (external) %s" %(name, doc.oneliner)) + else: + self.para("%s_ %s" %(name, doc.oneliner)) self.Print() class HookSpec(RestWriter): diff --git a/doc/announce/release-1.2.0.txt b/doc/announce/release-1.2.0.txt new file mode 100644 index 000000000..0c592234c --- /dev/null +++ b/doc/announce/release-1.2.0.txt @@ -0,0 +1,115 @@ +py.test/pylib 1.2.0: junitxml, standalone test scripts, pluginization +-------------------------------------------------------------------------------- + +py.test is an advanced automated testing tool working with +Python2, Python3 and Jython versions on all major operating +systems. It has a simple plugin architecture and can run many +existing common Python test suites without modification. It offers +some unique features not found in other testing tools. +See http://pytest.org for more info. + +py.test 1.2.0 brings many bug fixes and interesting new abilities: + +* --junitxml=path will create an XML file for use with CI processing +* --genscript=path creates a standalone py.test-equivalent test-script +* --ignore=path prevents collection of anything below that path +* --confcutdir=path only lookup conftest.py test configs below that path +* a new "pytestconfig" function argument gives direct access to option values +* parametrized tests can now be specified per-class as well +* on CPython py.test additionally installs as "py.test-VERSION" + +Apart from many bug fixes 1.2.0 also has better pluginization. +Distributed testing and looponfailing testing have been moved +out into its own "pytest-xdist" plugin which can be installed separately. +The same is true for "pytest-figleaf" for doing coverage reporting. +Those can also serve well now as blue prints for doing your own. +separately released plugins. + +thanks to all who helped and gave feedback, +have fun, + +holger krekel, January 2010 + +Changes between 1.2.0 and 1.1.1 +===================================== + +- moved dist/looponfailing from py.test core into a new + separately released pytest-xdist plugin. + +- new junitxml plugin: --junitxml=path will generate a junit style xml file + which is processable e.g. by the Hudson CI system. + +- new option: --genscript=path will generate a standalone py.test script + which will not need any libraries installed. thanks to Ralf Schmitt. + +- new option: --ignore will prevent specified path from collection. + Can be specified multiple times. + +- new option: --confcutdir=dir will make py.test only consider conftest + files that are relative to the specified dir. + +- new funcarg: "pytestconfig" is the pytest config object for access + to command line args and can now be easily used in a test. + +- install 'py.test' and `py.which` with a ``-$VERSION`` suffix to + disambiguate between Python3, python2.X, Jython and PyPy installed versions. + +- new "pytestconfig" funcarg allows access to test config object + +- new "pytest_report_header" hook can return additional lines + to be displayed at the header of a test run. + +- (experimental) allow "py.test path::name1::name2::..." for pointing + to a test within a test collection directly. This might eventually + evolve as a full substitute to "-k" specifications. + +- streamlined plugin loading: order is now as documented in + customize.html: setuptools, ENV, commandline, conftest. + also setuptools entry point names are turned to canonical namees ("pytest_*") + +- automatically skip tests that need 'capfd' but have no os.dup + +- allow pytest_generate_tests to be defined in classes as well + +- deprecate usage of 'disabled' attribute in favour of pytestmark +- deprecate definition of Directory, Module, Class and Function nodes + in conftest.py files. Use pytest collect hooks instead. + +- collection/item node specific runtest/collect hooks are only called exactly + on matching conftest.py files, i.e. ones which are exactly below + the filesystem path of an item + +- change: the first pytest_collect_directory hook to return something + will now prevent further hooks to be called. + +- change: figleaf plugin now requires --figleaf to run. Also + change its long command line options to be a bit shorter (see py.test -h). + +- change: pytest doctest plugin is now enabled by default and has a + new option --doctest-glob to set a pattern for file matches. + +- change: remove internal py._* helper vars, only keep py._pydir + +- robustify capturing to survive if custom pytest_runtest_setup + code failed and prevented the capturing setup code from running. + +- make py.test.* helpers provided by default plugins visible early - + works transparently both for pydoc and for interactive sessions + which will regularly see e.g. py.test.mark and py.test.importorskip. + +- simplify internal plugin manager machinery +- simplify internal collection tree by introducing a RootCollector node + +- fix assert reinterpreation that sees a call containing "keyword=..." + +- fix issue66: invoke pytest_sessionstart and pytest_sessionfinish + hooks on slaves during dist-testing, report module/session teardown + hooks correctly. + +- fix issue65: properly handle dist-testing if no + execnet/py lib installed remotely. + +- skip some install-tests if no execnet is available + +- fix docs, fix internal bin/ script generation + diff --git a/doc/test/dist.txt b/doc/test/dist.txt deleted file mode 100644 index 2df764a45..000000000 --- a/doc/test/dist.txt +++ /dev/null @@ -1,132 +0,0 @@ -.. _`distribute tests across machines`: - -=================== -Distributed testing -=================== - -``py.test`` can ad-hoc distribute test runs to multiple CPUs or remote -machines. This allows to speed up development or to use special resources -of remote machines. Before running tests remotely, ``py.test`` efficiently -synchronizes your program source code to the remote place. All test results -are reported back and displayed to your local test session. You may -specify different Python versions and interpreters. - -**Requirements**: you need to install the `execnet`_ package -(at least version 1.0.0b4) to perform distributed test runs. - -**NOTE**: Version 1.1.x is not able to distribute tests across Python3/Python2 barriers. - -Speed up test runs by sending tests to multiple CPUs ----------------------------------------------------------- - -To send tests to multiple CPUs, type:: - - py.test -n NUM - -Especially for longer running tests or tests requiring -a lot of IO this can lead to considerable speed ups. - - -Running tests in a Python subprocess ----------------------------------------- - -To instantiate a python2.4 sub process and send tests to it, you may type:: - - py.test -d --tx popen//python=python2.4 - -This will start a subprocess which is run with the "python2.4" -Python interpreter, found in your system binary lookup path. - -If you prefix the --tx option value like this:: - - --tx 3*popen//python=python2.4 - -then three subprocesses would be created and tests -will be load-balanced across these three processes. - - -Sending tests to remote SSH accounts ------------------------------------------------ - -Suppose you have a package ``mypkg`` which contains some -tests that you can successfully run locally. And you -have a ssh-reachable machine ``myhost``. Then -you can ad-hoc distribute your tests by typing:: - - py.test -d --tx ssh=myhostpopen --rsyncdir mypkg mypkg - -This will synchronize your ``mypkg`` package directory -to an remote ssh account and then locally collect tests -and send them to remote places for execution. - -You can specify multiple ``--rsyncdir`` directories -to be sent to the remote side. - -**NOTE:** For py.test to collect and send tests correctly -you not only need to make sure all code and tests -directories are rsynced, but that any test (sub) directory -also has an ``__init__.py`` file because internally -py.test references tests as a fully qualified python -module path. **You will otherwise get strange errors** -during setup of the remote side. - -Sending tests to remote Socket Servers ----------------------------------------- - -Download the single-module `socketserver.py`_ Python program -and run it like this:: - - python socketserver.py - -It will tell you that it starts listening on the default -port. You can now on your home machine specify this -new socket host with something like this:: - - py.test -d --tx socket=192.168.1.102:8888 --rsyncdir mypkg mypkg - - -.. _`atonce`: - -Running tests on many platforms at once -------------------------------------------------------------- - -The basic command to run tests on multiple platforms is:: - - py.test --dist=each --tx=spec1 --tx=spec2 - -If you specify a windows host, an OSX host and a Linux -environment this command will send each tests to all -platforms - and report back failures from all platforms -at once. The specifications strings use the `xspec syntax`_. - -.. _`xspec syntax`: http://codespeak.net/execnet/trunk/basics.html#xspec - -.. _`socketserver.py`: http://codespeak.net/svn/py/dist/py/execnet/script/socketserver.py - -.. _`execnet`: http://codespeak.net/execnet - -Specifying test exec environments in a conftest.py -------------------------------------------------------------- - -Instead of specifying command line options, you can -put options values in a ``conftest.py`` file like this:: - - pytest_option_tx = ['ssh=myhost//python=python2.5', 'popen//python=python2.5'] - pytest_option_dist = True - -Any commandline ``--tx`` specifictions will add to the list of available execution -environments. - -Specifying "rsync" dirs in a conftest.py -------------------------------------------------------------- - -In your ``mypkg/conftest.py`` you may specify directories to synchronise -or to exclude:: - - rsyncdirs = ['.', '../plugins'] - rsyncignore = ['_cache'] - -These directory specifications are relative to the directory -where the ``conftest.py`` is found. - - diff --git a/doc/test/features.txt b/doc/test/features.txt index da72fb7ac..be153a034 100644 --- a/doc/test/features.txt +++ b/doc/test/features.txt @@ -9,22 +9,40 @@ py.test feature overview mature command line testing tool ==================================================== -py.test is a command line tool to collect and run automated tests. It -runs well on Linux, Windows and OSX Python 2.4 through to 2.6 versions. -It can distribute a single test run to multiple machines. It is used in -many projects, ranging from running 10 thousands of tests integrated -with buildbot to a few inlined tests on a command line script. +py.test is a command line tool to collect, run and report about automated tests. It runs well on Linux, Windows and OSX and on Python 2.4 through to 3.1 versions. +It is used in many projects, ranging from running 10 thousands of tests +to a few inlined tests on a command line script. As of version 1.2 you can also +generate a no-dependency py.test-equivalent standalone script that you +can distribute along with your application. -.. _`autocollect`: +extensive easy plugin system +====================================================== -automatically collects and executes tests -=============================================== +.. _`suprisingly easy`: http://bruynooghe.blogspot.com/2009/12/skipping-slow-test-by-default-in-pytest.html -py.test discovers tests automatically by looking at -specified directories and its files for common -naming patterns. As ``py.test`` operates as a separate -cmdline tool you can easily have a command line utility and -some tests in the same file. +py.test delegates almost all aspects of its operation to plugins_. +It is `suprisingly easy`_ to add command line options or +do other kind of add-ons and customizations. This can +be done per-project or by distributing a global plugin. +One can can thus modify or add aspects for purposes such as: + +* reporting extensions +* customizing collection and execution of tests +* running and managing non-python tests +* managing domain-specific test state setup +* adding non-python tests into the run, e.g. driven by data files + +.. _`plugins`: plugin/index.html + +distributing tests to your CPUs and SSH accounts +========================================================== + +.. _`pytest-xdist`: plugin/xdist.html + +Through the use of the separately released `pytest-xdist`_ plugin you +can seemlessly distribute runs to multiple CPUs or remote computers +through SSH and sockets. This plugin also offers a ``--looponfailing`` +mode which will continously re-run only failing tests in a subprocess. supports several testing practises and methods ================================================================== @@ -40,9 +58,20 @@ with figleaf`_ or `Javasript unit- and functional testing`_. .. _`Javasript unit- and functional testing`: plugin/oejskit.html .. _`coverage testing with figleaf`: plugin/figleaf.html +integrates well with CI systems +==================================================== + +py.test can produce JUnitXML style output as well as formatted +"resultlog" files that can be postprocessed by Continous Integration +systems such as Hudson or Buildbot easily. It also provides command +line options to control test configuration lookup behaviour or ignoring +certain tests or directories. + no-boilerplate test functions with Python =================================================== +.. _`autocollect`: + automatic Python test discovery ------------------------------------ @@ -53,13 +82,18 @@ filename are inspected for finding tests: * classes with a leading ``Test`` name and ``test`` prefixed methods. * ``unittest.TestCase`` subclasses -test functions can run with different argument sets ------------------------------------------------------------ +parametrizing test functions and advanced functional testing +-------------------------------------------------------------- py.test offers the unique `funcargs mechanism`_ for setting up and passing project-specific objects to Python test functions. Test Parametrization happens by triggering a call to the same test -functions with different argument values. +function with different argument values. For doing fixtures +using the funcarg mechanism makes your test and setup code +more efficient and more readable. This is especially true +for functional tests which might depend on command line +options and a setup that needs to be shared across +a whole test run. per-test capturing of output, including subprocesses ---------------------------------------------------- @@ -137,30 +171,6 @@ can make use of this feature. .. _`xUnit style setup`: xunit_setup.html .. _`pytest_nose`: plugin/nose.html -load-balance test runs to multiple CPUs -======================================== - -For large test suites you can distribute your -tests to multiple CPUs by issuing for example:: - - py.test -n 3 - -Read more on `distributed testing`_. - -.. _`distributed testing`: dist.html - -ad-hoc run tests cross-platform -================================================== - -py.test supports the sending of tests to -remote ssh-accounts, socket servers. -It can `ad-hoc run your test on multiple -platforms one a single test run`. Ad-hoc -means that there are **no installation -requirements whatsoever** on the remote side. - -.. _`ad-hoc run your test on multiple platforms one a single test run`: dist.html#atonce - advanced test selection and running modes ========================================================= @@ -202,21 +212,6 @@ plugin for more information. .. _`pytest_keyword`: plugin/mark.html -easy to extend -========================================= - -py.test has advanced `extension mechanisms`_ -with a growing `list of default plugins`_. -One can can easily modify or add aspects for for -purposes such as: - -* reporting extensions -* customizing collection and execution of tests -* running and managing non-python tests -* managing domain-specific test state setup - -.. _`list of default plugins`: plugin/index.html -.. _`extension mechanisms`: customize.html#extensions .. _`reStructured Text`: http://docutils.sourceforge.net .. _`Python debugger`: http://docs.python.org/lib/module-pdb.html diff --git a/doc/test/index.txt b/doc/test/index.txt index aa440d772..11b665914 100644 --- a/doc/test/index.txt +++ b/doc/test/index.txt @@ -24,6 +24,5 @@ changelog_: history of changes covering last releases .. _features: features.html .. _funcargs: funcargs.html .. _customize: customize.html -.. _`distributed testing`: dist.html diff --git a/doc/test/plugin/index.txt b/doc/test/plugin/index.txt index 7da09814d..d39bc2666 100644 --- a/doc/test/plugin/index.txt +++ b/doc/test/plugin/index.txt @@ -8,7 +8,7 @@ mark_ generic mechanism for marking python functions. pdb_ interactive debugging with the Python Debugger. -figleaf_ report test coverage using the 'figleaf' package. +figleaf_ (external) report test coverage using the 'figleaf' package. coverage_ (external) for testing with Ned's coverage module @@ -21,34 +21,26 @@ recwarn_ helpers for asserting deprecation and other warnings. tmpdir_ provide temporary directories to test functions. -other testing domains, misc -=========================== +distributed testing, CI and deployment +====================================== -oejskit_ (external) run javascript tests in real life browsers - -django_ (external) for testing django applications - -xdist_ loop on failing tests, distribute test runs to CPUs and hosts. - -genscript_ generate standalone test script to be distributed along with an application. - - -reporting and failure logging -============================= +xdist_ (external) loop on failing tests, distribute test runs to CPUs and hosts. pastebin_ submit failure or test session information to a pastebin service. junitxml_ logging of test results in JUnit-XML format, for use with Hudson -xmlresult_ (external) for generating xml reports and CruiseControl integration - resultlog_ non-xml machine-readable logging of test results. -terminal_ Implements terminal reporting of the full testing process. +genscript_ generate standalone test script to be distributed along with an application. -other testing conventions -========================= +testing domains and conventions +=============================== + +oejskit_ (external) run javascript tests in real life browsers + +django_ (external) for testing django applications unittest_ automatically discover and run traditional "unittest.py" style tests. diff --git a/doc/test/plugin/links.txt b/doc/test/plugin/links.txt index 87fc5d9f1..e71073d09 100644 --- a/doc/test/plugin/links.txt +++ b/doc/test/plugin/links.txt @@ -1,5 +1,4 @@ .. _`helpconfig`: helpconfig.html -.. _`terminal`: terminal.html .. _`pytest_recwarn.py`: http://bitbucket.org/hpk42/py-trunk/raw/1.2.0/py/plugin/pytest_recwarn.py .. _`unittest`: unittest.html .. _`pytest_monkeypatch.py`: http://bitbucket.org/hpk42/py-trunk/raw/1.2.0/py/plugin/pytest_monkeypatch.py @@ -15,7 +14,6 @@ .. _`pytest_nose.py`: http://bitbucket.org/hpk42/py-trunk/raw/1.2.0/py/plugin/pytest_nose.py .. _`pytest_restdoc.py`: http://bitbucket.org/hpk42/py-trunk/raw/1.2.0/py/plugin/pytest_restdoc.py .. _`restdoc`: restdoc.html -.. _`xdist`: xdist.html .. _`pytest_pastebin.py`: http://bitbucket.org/hpk42/py-trunk/raw/1.2.0/py/plugin/pytest_pastebin.py .. _`pytest_tmpdir.py`: http://bitbucket.org/hpk42/py-trunk/raw/1.2.0/py/plugin/pytest_tmpdir.py .. _`pytest_figleaf.py`: http://bitbucket.org/hpk42/py-trunk/raw/1.2.0/py/plugin/pytest_figleaf.py @@ -33,15 +31,14 @@ .. _`figleaf`: figleaf.html .. _`customize`: ../customize.html .. _`hooklog`: hooklog.html -.. _`pytest_terminal.py`: http://bitbucket.org/hpk42/py-trunk/raw/1.2.0/py/plugin/pytest_terminal.py .. _`recwarn`: recwarn.html +.. _`xdist`: xdist.html .. _`pytest_pdb.py`: http://bitbucket.org/hpk42/py-trunk/raw/1.2.0/py/plugin/pytest_pdb.py .. _`monkeypatch`: monkeypatch.html .. _`coverage`: coverage.html .. _`resultlog`: resultlog.html .. _`pytest_junitxml.py`: http://bitbucket.org/hpk42/py-trunk/raw/1.2.0/py/plugin/pytest_junitxml.py .. _`django`: django.html -.. _`xmlresult`: xmlresult.html .. _`pytest_unittest.py`: http://bitbucket.org/hpk42/py-trunk/raw/1.2.0/py/plugin/pytest_unittest.py .. _`nose`: nose.html .. _`pytest_resultlog.py`: http://bitbucket.org/hpk42/py-trunk/raw/1.2.0/py/plugin/pytest_resultlog.py diff --git a/doc/test/plugin/xdist.txt b/doc/test/plugin/xdist.txt index a711ca86b..cb4fee920 100644 --- a/doc/test/plugin/xdist.txt +++ b/doc/test/plugin/xdist.txt @@ -26,7 +26,7 @@ program source code to the remote place. All test results are reported back and displayed to your local test session. You may specify different Python versions and interpreters. -.. _`pytest-xdist`: http://pytest.org/plugin/xdist.html +.. _`pytest-xdist`: http://pypi.python.org/pypi/pytest-xdist Usage examples ---------------------