[svn r57321] merging the event branch:

* moving in test, misc, code, io directories and
  py/__init__.py
* py/bin/_find.py does not print to stderr anymore
* a few fixes to conftest files in other dirs
some more fixes and adjustments pending

--HG--
branch : trunk

py/test/dsession/testing/__init__.py

@@ -0,0 +1 @@
+#

py/test/dsession/testing/basetest.py

@@ -0,0 +1,32 @@
+
+""" Support module for running tests
+"""
+
+import py
+from py.__.test.testing.setupdata import getexamplefile 
+
+class DirSetup(object):
+    def setup_method(self, method):
+        name = "%s.%s" %(self.__class__.__name__, method.func_name)
+        self.tmpdir = py.test.ensuretemp(name)
+        self.source = self.tmpdir.ensure("source", dir=1)
+        self.dest = self.tmpdir.join("dest")
+
+
+class BasicRsessionTest(object):
+    def setup_class(cls):
+        path = getexamplefile("funcexamples.py")
+        cls.config = py.test.config._reparse([path.dirpath()])
+        cls.modulecol = cls.config.getfsnode(path)
+
+    def setup_method(self, method):
+        self.session = self.config.initsession()
+        
+    def getfunc(self, name):
+        funcname = "func" + name
+        col = self.modulecol.join(funcname) 
+        assert col is not None, funcname
+        return col
+
+    def getdocexample(self):
+        return getexamplefile("docexample.txt")

py/test/dsession/testing/test_dsession.py

@@ -0,0 +1,361 @@
+from py.__.test.testing.suptest import InlineCollection
+from py.__.test.dsession.dsession import DSession, LoopState
+from py.__.test.dsession.hostmanage import Host, makehostup
+from py.__.test.runner import basic_collect_report 
+from py.__.test import event
+from py.__.test import outcome
+import py
+
+def run(item):
+    runner = item._getrunner()
+    return runner(item)
+
+class MockNode:
+    def __init__(self):
+        self.sent = []
+
+    def sendlist(self, items):
+        self.sent.append(items)
+
+    def shutdown(self):
+        self._shutdown=True
+
+def dumpqueue(queue):
+    while queue.qsize():
+        print queue.get()
+
+class TestDSession(InlineCollection):
+    def test_fixoptions(self):
+        config = self.parseconfig("--exec=xxx")
+        config.initsession().fixoptions()
+        assert config.option.numprocesses == 1
+        config = self.parseconfig("--exec=xxx", '-n3')
+        config.initsession().fixoptions()
+        assert config.option.numprocesses == 3
+
+    def test_add_remove_host(self):
+        item = self.getitem("def test_func(): pass")
+        rep = run(item)
+        session = DSession(item._config)
+        host = Host("localhost")
+        host.node = MockNode()
+        assert not session.host2pending
+        session.addhost(host)
+        assert len(session.host2pending) == 1
+        session.senditems([item])
+        pending = session.removehost(host)
+        assert pending == [item]
+        assert item not in session.item2host
+        py.test.raises(Exception, "session.removehost(host)")
+
+    def test_senditems_removeitems(self):
+        item = self.getitem("def test_func(): pass")
+        rep = run(item)
+        session = DSession(item._config)
+        host = Host("localhost")
+        host.node = MockNode()
+        session.addhost(host)
+        session.senditems([item])  
+        assert session.host2pending[host] == [item]
+        assert session.item2host[item] == host
+        session.removeitem(item)
+        assert not session.host2pending[host] 
+        assert not session.item2host
+
+    def test_triggertesting_collect(self):
+        modcol = self.getmodulecol("""
+            def test_func():
+                pass
+        """)
+        session = DSession(modcol._config)
+        session.triggertesting([modcol])
+        rep = session.queue.get(block=False)
+        assert isinstance(rep, event.CollectionReport)
+        assert len(rep.result) == 1
+
+    def test_triggertesting_item(self):
+        item = self.getitem("def test_func(): pass")
+        session = DSession(item._config)
+        host1 = Host("localhost")
+        host1.node = MockNode()
+        host2 = Host("localhost")
+        host2.node = MockNode()
+        session.addhost(host1)
+        session.addhost(host2)
+        session.triggertesting([item] * (session.MAXITEMSPERHOST*2 + 1))
+        host1_sent = host1.node.sent[0]
+        host2_sent = host2.node.sent[0]
+        assert host1_sent == [item] * session.MAXITEMSPERHOST
+        assert host2_sent == [item] * session.MAXITEMSPERHOST
+        assert session.host2pending[host1] == host1_sent
+        assert session.host2pending[host2] == host2_sent
+        ev = session.queue.get(block=False)
+        assert isinstance(ev, event.RescheduleItems)
+        assert ev.items == [item]
+
+    def test_keyboardinterrupt(self):
+        item = self.getitem("def test_func(): pass")
+        session = DSession(item._config)
+        def raise_(timeout=None): raise KeyboardInterrupt()
+        session.queue.get = raise_
+        exitstatus = session.loop([])
+        assert exitstatus == outcome.EXIT_INTERRUPTED
+
+    def test_internalerror(self):
+        item = self.getitem("def test_func(): pass")
+        session = DSession(item._config)
+        def raise_(): raise ValueError()
+        session.queue.get = raise_
+        exitstatus = session.loop([])
+        assert exitstatus == outcome.EXIT_INTERNALERROR
+
+    def test_rescheduleevent(self):
+        item = self.getitem("def test_func(): pass")
+        session = DSession(item._config)
+        host1 = Host("localhost")
+        host1.node = MockNode()
+        session.addhost(host1)
+        ev = event.RescheduleItems([item])
+        loopstate = LoopState([])
+        session.queue.put(ev)
+        session.loop_once(loopstate)
+        # check that RescheduleEvents are not immediately
+        # rescheduled if there are no hosts 
+        assert loopstate.dowork == False 
+        session.queue.put(event.NOP())
+        session.loop_once(loopstate)
+        session.queue.put(event.NOP())
+        session.loop_once(loopstate)
+        assert host1.node.sent == [[item]]
+        session.queue.put(run(item))
+        session.loop_once(loopstate)
+        assert loopstate.shuttingdown 
+        assert not loopstate.testsfailed 
+
+    def test_no_hosts_remaining_for_tests(self):
+        item = self.getitem("def test_func(): pass")
+        # setup a session with one host
+        session = DSession(item._config)
+        host1 = Host("localhost")
+        host1.node = MockNode()
+        session.addhost(host1)
+       
+        # setup a HostDown event
+        ev = event.HostDown(host1, None)
+        session.queue.put(ev)
+
+        loopstate = LoopState([item])
+        loopstate.dowork = False
+        session.loop_once(loopstate)
+        dumpqueue(session.queue)
+        assert loopstate.exitstatus == outcome.EXIT_NOHOSTS
+
+    def test_hostdown_causes_reschedule_pending(self):
+        modcol = self.getmodulecol("""
+            def test_crash(): 
+                assert 0
+            def test_fail(): 
+                x
+        """)
+        item1, item2 = [modcol.join(x) for x in modcol.listdir()]
+
+        # setup a session with two hosts 
+        session = DSession(item1._config)
+        host1 = Host("localhost")
+        host1.node = MockNode()
+        session.addhost(host1)
+        host2 = Host("localhost")
+        host2.node = MockNode()
+        session.addhost(host2)
+      
+        # have one test pending for a host that goes down 
+        session.senditems([item1, item2])
+        host = session.item2host[item1]
+        ev = event.HostDown(host, None)
+        session.queue.put(ev)
+
+        events = [] ; session.bus.subscribe(events.append)
+        loopstate = LoopState([])
+        session.loop_once(loopstate)
+
+        assert loopstate.colitems == [item2] # do not reschedule crash item
+        testrep = [x for x in events if isinstance(x, event.ItemTestReport)][0]
+        assert testrep.failed
+        assert testrep.colitem == item1
+        assert str(testrep.outcome.longrepr).find("CRASHED") != -1
+        assert str(testrep.outcome.longrepr).find(host.hostname) != -1
+
+    def test_hostup_adds_to_available(self):
+        item = self.getitem("def test_func(): pass")
+        # setup a session with two hosts 
+        session = DSession(item._config)
+        host1 = Host("localhost")
+        hostup = makehostup(host1)
+        session.queue.put(hostup)
+        loopstate = LoopState([item])
+        loopstate.dowork = False
+        assert len(session.host2pending) == 0
+        session.loop_once(loopstate)
+        assert len(session.host2pending) == 1
+
+    def test_event_propagation(self):
+        item = self.getitem("def test_func(): pass")
+        session = DSession(item._config)
+      
+        ev = event.NOP()
+        events = [] ; session.bus.subscribe(events.append)
+        session.queue.put(ev)
+        session.loop_once(LoopState([]))
+        assert events[0] == ev  
+
+    def runthrough(self, item):
+        session = DSession(item._config)
+        host1 = Host("localhost")
+        host1.node = MockNode()
+        session.addhost(host1)
+        loopstate = LoopState([item])
+
+        session.queue.put(event.NOP())
+        session.loop_once(loopstate)
+
+        assert host1.node.sent == [[item]]
+        ev = run(item)
+        session.queue.put(ev)
+        session.loop_once(loopstate)
+        assert loopstate.shuttingdown  
+        session.queue.put(event.HostDown(host1, None))
+        session.loop_once(loopstate)
+        dumpqueue(session.queue)
+        return session, loopstate.exitstatus 
+
+    def test_exit_completed_tests_ok(self):
+        item = self.getitem("def test_func(): pass")
+        session, exitstatus = self.runthrough(item)
+        assert exitstatus == outcome.EXIT_OK
+
+    def test_exit_completed_tests_fail(self):
+        item = self.getitem("def test_func(): 0/0")
+        session, exitstatus = self.runthrough(item)
+        assert exitstatus == outcome.EXIT_TESTSFAILED
+
+    def test_exit_on_first_failing(self):
+        modcol = self.getmodulecol("""
+            def test_fail(): 
+                assert 0
+            def test_pass(): 
+                pass
+        """)
+        modcol._config.option.exitfirst = True
+        session = DSession(modcol._config)
+        host1 = Host("localhost")
+        host1.node = MockNode()
+        session.addhost(host1)
+        items = basic_collect_report(modcol).result
+
+        # trigger testing  - this sends tests to host1
+        session.triggertesting(items)
+
+        # run tests ourselves and produce reports 
+        ev1 = run(items[0])
+        ev2 = run(items[1])
+        session.queue.put(ev1) # a failing one
+        session.queue.put(ev2)
+        # now call the loop
+        loopstate = LoopState(items)
+        session.loop_once(loopstate)
+        assert loopstate.testsfailed
+        assert loopstate.shuttingdown
+
+    def test_shuttingdown_filters_events(self):
+        item = self.getitem("def test_func(): pass")
+        session = DSession(item._config)
+        host = Host("localhost")
+        session.addhost(host)
+        loopstate = LoopState([])
+        loopstate.shuttingdown = True
+        l = []
+        session.bus.subscribe(l.append)
+        session.queue.put(run(item))
+        session.loop_once(loopstate)
+        assert not l
+        ev = event.HostDown(host)
+        session.queue.put(ev)
+        session.loop_once(loopstate)
+        assert l == [ev]
+
+    def test_filteritems(self):
+        modcol = self.getmodulecol("""
+            def test_fail(): 
+                assert 0
+            def test_pass(): 
+                pass
+        """)
+        session = DSession(modcol._config)
+
+        modcol._config.option.keyword = "nothing"
+        dsel = session.filteritems([modcol])
+        assert dsel == [modcol] 
+        items = [modcol.join(x) for x in modcol.listdir()]
+        events = [] ; session.bus.subscribe(events.append)
+        remaining = session.filteritems(items)
+        assert remaining == []
+        
+        ev = events[-1]
+        assert isinstance(ev, event.Deselected)
+        assert ev.items == items 
+
+        modcol._config.option.keyword = "test_fail"
+        remaining = session.filteritems(items)
+        assert remaining == [items[0]]
+
+        ev = events[-1]
+        assert isinstance(ev, event.Deselected)
+        assert ev.items == [items[1]]
+
+    def test_hostdown_shutdown_after_completion(self):
+        item = self.getitem("def test_func(): pass")
+        session = DSession(item._config)
+
+        host = Host("localhost")
+        host.node = MockNode()
+        session.addhost(host)
+        session.senditems([item])
+        session.queue.put(run(item))
+        loopstate = LoopState([]) 
+        session.loop_once(loopstate)
+        assert host.node._shutdown is True
+        assert loopstate.exitstatus is None, "loop did not wait for hostdown"
+        assert loopstate.shuttingdown 
+        session.queue.put(event.HostDown(host, None))
+        session.loop_once(loopstate)
+        assert loopstate.exitstatus == 0
+
+    def test_nopending_but_collection_remains(self):
+        modcol = self.getmodulecol("""
+            def test_fail(): 
+                assert 0
+            def test_pass(): 
+                pass
+        """)
+        session = DSession(modcol._config)
+        host = Host("localhost")
+        host.node = MockNode()
+        session.addhost(host)
+
+        colreport = basic_collect_report(modcol)
+        item1, item2 = colreport.result
+        session.senditems([item1])
+        # host2pending will become empty when the loop sees
+        # the report 
+        session.queue.put(run(item1)) 
+
+        # but we have a collection pending
+        session.queue.put(colreport) 
+
+        loopstate = LoopState([]) 
+        session.loop_once(loopstate)
+        assert loopstate.exitstatus is None, "loop did not care for collection report"
+        assert not loopstate.colitems 
+        session.loop_once(loopstate)
+        assert loopstate.colitems == colreport.result
+        assert loopstate.exitstatus is None, "loop did not care for colitems"

py/test/dsession/testing/test_functional_dsession.py

@@ -0,0 +1,152 @@
+
+""" Tests various aspects of dist, like ssh hosts setup/teardown
+"""
+
+import py
+from py.__.test import event
+from py.__.test.dsession.dsession import DSession
+from py.__.test.dsession.hostmanage import HostManager, Host
+from basetest import BasicRsessionTest, DirSetup
+from py.__.test.testing import suptest
+import os
+
+def eventreader(session):
+    queue = py.std.Queue.Queue()
+    session.bus.subscribe(queue.put)
+    def readevent(eventtype=event.ItemTestReport, timeout=2.0):
+        events = []
+        while 1:
+            try:
+                ev = queue.get(timeout=timeout)
+            except py.std.Queue.Empty:
+                print "seen events", events
+                raise IOError("did not see %r events" % (eventtype))
+            else:
+                if isinstance(ev, eventtype):
+                    #print "other events seen", events
+                    return ev
+                events.append(ev)
+    return readevent
+
+class TestAsyncFunctional(suptest.InlineCollection):
+    def test_dist_no_disthost(self):
+        config = self.parseconfig(self.tmpdir, '-d')
+        py.test.raises(SystemExit, "config.initsession()")
+
+    def test_session_eventlog_dist(self):
+        self.makepyfile(conftest="dist_hosts=['localhost']\n")
+        eventlog = self.tmpdir.join("mylog")
+        config = self.parseconfig(self.tmpdir, '-d', '--eventlog=%s' % eventlog)
+        session = config.initsession()
+        session.bus.notify(event.TestrunStart())
+        s = eventlog.read()
+        assert s.find("TestrunStart") != -1
+
+    def test_conftest_options(self):
+        self.makepyfile(conftest="""
+            print "importing conftest"
+            import py
+            Option = py.test.config.Option 
+            option = py.test.config.addoptions("someopt", 
+                Option('', '--forcegen', action="store_true", dest="forcegen", default=False))
+        """)
+        self.makepyfile(__init__="#")
+        p1 = self.makepyfile(test_one="""
+            def test_1(): 
+                import py, conftest
+                print "test_1: py.test.config.option.forcegen", py.test.config.option.forcegen
+                print "test_1: conftest", conftest
+                print "test_1: conftest.option.forcegen", conftest.option.forcegen
+                assert conftest.option.forcegen 
+        """)
+        print p1
+        config = self.parseconfig('-n1', p1, '--forcegen')
+        dsession = DSession(config)
+        readevent = eventreader(dsession)
+        dsession.main()
+        ev = readevent(event.ItemTestReport)
+        if not ev.passed:
+            print ev.outcome.longrepr
+            assert 0
+
+    def test_dist_some_tests(self):
+        self.makepyfile(conftest="dist_hosts=['localhost']\n")
+        p1 = self.makepyfile(test_one="""
+            def test_1(): 
+                pass
+            def test_x():
+                import py
+                py.test.skip("aaa")
+            def test_fail():
+                assert 0
+        """)
+        config = self.parseconfig('-d', p1)
+        dsession = DSession(config)
+        readevent = eventreader(dsession)
+        dsession.main([config.getfsnode(p1)])
+        ev = readevent(event.ItemTestReport)
+        assert ev.passed
+        ev = readevent(event.ItemTestReport)
+        assert ev.skipped
+        ev = readevent(event.ItemTestReport)
+        assert ev.failed
+        # see that the host is really down 
+        ev = readevent(event.HostDown)
+        assert ev.host.hostname == "localhost"
+        ev = readevent(event.TestrunFinish)
+
+    def test_distribution_rsync_roots_example(self):
+        py.test.skip("testing for root rsync needs rework")
+        destdir = py.test.ensuretemp("example_dist_destdir")
+        subdir = "sub_example_dist"
+        sourcedir = self.tmpdir.mkdir("source")
+        sourcedir.ensure(subdir, "conftest.py").write(py.code.Source("""
+            dist_hosts = ["localhost:%s"]
+            dist_rsync_roots = ["%s", "../py"]
+        """ % (destdir, tmpdir.join(subdir), )))
+        tmpdir.ensure(subdir, "__init__.py")
+        tmpdir.ensure(subdir, "test_one.py").write(py.code.Source("""
+            def test_1(): 
+                pass
+            def test_2():
+                assert 0
+            def test_3():
+                raise ValueError(23)
+            def test_4(someargs):
+                pass
+            def test_5():
+                assert __file__ != '%s'
+            #def test_6():
+            #    import py
+            #    assert py.__file__ != '%s'
+        """ % (tmpdir.join(subdir), py.__file__)))
+        destdir.join("py").mksymlinkto(py.path.local(py.__file__).dirpath())
+        config = py.test.config._reparse([tmpdir.join(subdir)])
+        assert config.topdir == tmpdir
+        assert not tmpdir.join("__init__.py").check()
+        dist = DSession(config)
+        sorter = suptest.events_from_session(dist)
+        testevents = sorter.get(event.ItemTestReport)
+        assert len([x for x in testevents if x.passed]) == 2
+        assert len([x for x in testevents if x.failed]) == 3
+        assert len([x for x in testevents if x.skipped]) == 0
+
+    def test_nice_level(self):
+        """ Tests if nice level behaviour is ok """
+        if not hasattr(os, 'nice'):
+            py.test.skip("no os.nice() available")
+        self.makepyfile(conftest="""
+                dist_hosts=['localhost']
+                dist_nicelevel = 10
+        """)
+        p1 = self.makepyfile(test_nice="""
+            def test_nice():
+                import os
+                assert os.nice(0) == 10
+        """)
+        config = self.parseconfig('-d', p1)
+        session = config.initsession()
+        events = suptest.events_from_session(session)
+        ev = events.getreport('test_nice')
+        assert ev.passed
+

py/test/dsession/testing/test_hostmanage.py

@@ -0,0 +1,321 @@
+
+""" RSync filter test
+"""
+
+import py
+from basetest import DirSetup 
+from py.__.test.dsession.hostmanage import HostRSync, Host, HostManager, gethosts
+from py.__.test.dsession.hostmanage import sethomedir, gethomedir, getpath_relto_home
+from py.__.test import event
+
+class TestHost(DirSetup):
+    def _gethostinfo(self, relpath=""):
+        exampledir = self.tmpdir.join("gethostinfo")
+        if relpath:
+            exampledir = exampledir.join(relpath)
+        assert not exampledir.check()
+        hostinfo = Host("localhost:%s" % exampledir)
+        return hostinfo
+
+    def test_defaultpath(self):
+        x = Host("localhost:")
+        assert x.hostname == "localhost"
+        assert not x.relpath
+
+    def test_addrel(self):
+        host = Host("localhost:", addrel="whatever")
+        assert host.inplacelocal 
+        assert not host.relpath 
+        host = Host("localhost:/tmp", addrel="base")
+        assert host.relpath == "/tmp/base"
+        host = Host("localhost:tmp", addrel="base2")
+        assert host.relpath == "tmp/base2"
+
+    def test_path(self):
+        x = Host("localhost:/tmp")
+        assert x.relpath == "/tmp"
+        assert x.hostname == "localhost"
+        assert not x.inplacelocal 
+
+    def test_equality(self):
+        x = Host("localhost:")
+        y = Host("localhost:")
+        assert x != y
+        assert not (x == y)
+
+    def test_hostid(self):
+        x = Host("localhost:")
+        y = Host("localhost:")
+        assert x.hostid != y.hostid 
+        x = Host("localhost:/tmp")
+        y = Host("localhost:")
+        assert x.hostid != y.hostid 
+
+    def test_non_existing_hosts(self):
+        host = Host("alskdjalsdkjasldkajlsd")
+        py.test.raises((py.process.cmdexec.Error, IOError, EOFError), 
+                       host.initgateway)
+
+    def test_remote_has_homedir_as_currentdir(self):
+        host = self._gethostinfo()
+        old = py.path.local.get_temproot().chdir()
+        try:
+            host.initgateway()
+            channel = host.gw.remote_exec(py.code.Source(
+                gethomedir, """
+                import os
+                homedir = gethomedir()
+                curdir = os.getcwd()
+                channel.send((curdir, homedir))
+            """))
+            remote_curdir, remote_homedir = channel.receive()
+            assert remote_curdir == remote_homedir 
+        finally:
+            old.chdir()
+
+    def test_initgateway_localhost_relpath(self):
+        host = Host("localhost:somedir")
+        host.initgateway()
+        assert host.gw
+        try:
+            homedir = py.path.local._gethomedir() 
+            expected = homedir.join("somedir")
+            assert host.gw_remotepath == str(expected)
+        finally:
+            host.gw.exit()
+
+    def test_initgateway_python(self):
+        host = Host("localhost", python="python2.4")
+        l = []
+        def p(python):
+            l.append(python)
+            raise ValueError
+        py.magic.patch(py.execnet, 'PopenGateway', p)
+        try:
+            py.test.raises(ValueError, host.initgateway)
+        finally:
+            py.magic.revert(py.execnet, 'PopenGateway')
+        assert l[0] == host.python
+
+    def test_initgateway_ssh_and_remotepath(self):
+        from py.__.conftest import option
+        if not option.sshtarget:
+            py.test.skip("no known ssh target, use -S to set one")
+        host = Host("%s" % (option.sshtarget, ))
+        # this test should be careful to not write/rsync anything
+        # as the remotepath is the default location 
+        # and may be used in the real world 
+        host.initgateway()
+        assert host.gw
+        assert host.gw_remotepath.endswith(host.relpath)
+        channel = host.gw.remote_exec("""
+            import os
+            homedir = os.environ['HOME']
+            relpath = channel.receive()
+            path = os.path.join(homedir, relpath)
+            channel.send(path) 
+        """)
+        channel.send(host.relpath)
+        res = channel.receive()
+        assert res == host.gw_remotepath
+
+class TestSyncing(DirSetup): 
+    def _gethostinfo(self):
+        hostinfo = Host("localhost:%s" % self.dest)
+        return hostinfo 
+        
+    def test_hrsync_filter(self):
+        self.source.ensure("dir", "file.txt")
+        self.source.ensure(".svn", "entries")
+        self.source.ensure(".somedotfile", "moreentries")
+        self.source.ensure("somedir", "editfile~")
+        syncer = HostRSync(self.source)
+        l = list(self.source.visit(rec=syncer.filter,
+                                   fil=syncer.filter))
+        assert len(l) == 3
+        basenames = [x.basename for x in l]
+        assert 'dir' in basenames
+        assert 'file.txt' in basenames
+        assert 'somedir' in basenames
+
+    def test_hrsync_localhost_inplace(self):
+        h1 = Host("localhost")
+        events = []
+        rsync = HostRSync(self.source)
+        h1.initgateway()
+        rsync.add_target_host(h1, notify=events.append)
+        assert events
+        l = [x for x in events 
+                if isinstance(x, event.HostRSyncing)]
+        assert len(l) == 1
+        ev = l[0]
+        assert ev.host == h1
+        assert ev.root == ev.remotepath 
+        l = [x for x in events 
+                if isinstance(x, event.HostRSyncRootReady)]
+        assert len(l) == 1
+        ev = l[0]
+        assert ev.root == self.source
+        assert ev.host == h1
+
+    def test_hrsync_one_host(self):
+        h1 = self._gethostinfo()
+        finished = []
+        rsync = HostRSync(self.source)
+        h1.initgateway()
+        rsync.add_target_host(h1)
+        self.source.join("hello.py").write("world")
+        rsync.send()
+        assert self.dest.join("hello.py").check()
+
+    def test_hrsync_same_host_twice(self):
+        h1 = self._gethostinfo()
+        h2 = self._gethostinfo()
+        finished = []
+        rsync = HostRSync(self.source)
+        l = []
+        h1.initgateway()
+        h2.initgateway()
+        res1 = rsync.add_target_host(h1)
+        assert res1
+        res2 = rsync.add_target_host(h2)
+        assert not res2
+
+class TestHostManager(DirSetup):
+    def gethostmanager(self, dist_hosts, dist_rsync_roots=None):
+        l = ["dist_hosts = %r" % dist_hosts]
+        if dist_rsync_roots:
+            l.append("dist_rsync_roots = %r" % dist_rsync_roots)
+        self.source.join("conftest.py").write("\n".join(l))
+        config = py.test.config._reparse([self.source])
+        assert config.topdir == self.source
+        session = config.initsession()
+        hm = HostManager(session)
+        assert hm.hosts
+        return hm
+        
+    def test_hostmanager_custom_hosts(self):
+        session = py.test.config._reparse([self.source]).initsession()
+        hm = HostManager(session, hosts=[1,2,3])
+        assert hm.hosts == [1,2,3]
+
+    def test_hostmanager_init_rsync_topdir(self):
+        dir2 = self.source.ensure("dir1", "dir2", dir=1)
+        dir2.ensure("hello")
+        hm = self.gethostmanager(
+            dist_hosts = ["localhost:%s" % self.dest]
+        )
+        assert hm.session.config.topdir == self.source
+        hm.init_rsync() 
+        dest = self.dest.join(self.source.basename)
+        assert dest.join("dir1").check()
+        assert dest.join("dir1", "dir2").check()
+        assert dest.join("dir1", "dir2", 'hello').check()
+
+    def test_hostmanager_init_rsync_topdir_explicit(self):
+        dir2 = self.source.ensure("dir1", "dir2", dir=1)
+        dir2.ensure("hello")
+        hm = self.gethostmanager(
+            dist_hosts = ["localhost:%s" % self.dest],
+            dist_rsync_roots = [str(self.source)]
+        )
+        assert hm.session.config.topdir == self.source
+        hm.init_rsync()
+        dest = self.dest.join(self.source.basename)
+        assert dest.join("dir1").check()
+        assert dest.join("dir1", "dir2").check()
+        assert dest.join("dir1", "dir2", 'hello').check()
+
+    def test_hostmanager_init_rsync_roots(self):
+        dir2 = self.source.ensure("dir1", "dir2", dir=1)
+        self.source.ensure("dir1", "somefile", dir=1)
+        dir2.ensure("hello")
+        self.source.ensure("bogusdir", "file")
+        self.source.join("conftest.py").write(py.code.Source("""
+            dist_rsync_roots = ['dir1/dir2']
+        """))
+        session = py.test.config._reparse([self.source]).initsession()
+        hm = HostManager(session, 
+                         hosts=[Host("localhost:" + str(self.dest))])
+        hm.init_rsync()
+        assert self.dest.join("dir2").check()
+        assert not self.dest.join("dir1").check()
+        assert not self.dest.join("bogus").check()
+
+    def test_hostmanager_rsync_ignore(self):
+        dir2 = self.source.ensure("dir1", "dir2", dir=1)
+        dir5 = self.source.ensure("dir5", "dir6", "bogus")
+        dirf = self.source.ensure("dir5", "file")
+        dir2.ensure("hello")
+        self.source.join("conftest.py").write(py.code.Source("""
+            dist_rsync_ignore = ['dir1/dir2', 'dir5/dir6']
+        """))
+        session = py.test.config._reparse([self.source]).initsession()
+        hm = HostManager(session,
+                         hosts=[Host("localhost:" + str(self.dest))])
+        hm.init_rsync()
+        assert self.dest.join("dir1").check()
+        assert not self.dest.join("dir1", "dir2").check()
+        assert self.dest.join("dir5","file").check()
+        assert not self.dest.join("dir6").check()
+
+    def test_hostmanage_optimise_localhost(self):
+        hosts = [Host("localhost") for i in range(3)]
+        session = py.test.config._reparse([self.source]).initsession()
+        hm = HostManager(session, hosts=hosts)
+        hm.init_rsync()
+        for host in hosts:
+            assert host.inplacelocal
+            assert host.gw_remotepath is None
+            assert not host.relpath 
+
+    def test_hostmanage_setup_hosts(self):
+        hosts = [Host("localhost") for i in range(3)]
+        session = py.test.config._reparse([self.source]).initsession()
+        hm = HostManager(session, hosts=hosts)
+        queue = py.std.Queue.Queue()
+        hm.setup_hosts(notify=queue.put)
+        for host in hm.hosts:
+            ev = queue.get(timeout=2.0)
+            assert isinstance(ev, event.HostUp)
+        for host in hm.hosts:
+            host.node.shutdown()
+        for host in hm.hosts:
+            ev = queue.get(timeout=2.0)
+            assert isinstance(ev, event.HostDown)
+
+    def XXXtest_ssh_rsync_samehost_twice(self):
+        #XXX we have no easy way to have a temp directory remotely!
+        option = py.test.config.option
+        if option.sshtarget is None: 
+            py.test.skip("no known ssh target, use -S to set one")
+        host1 = Host("%s" % (option.sshtarget, ))
+        host2 = Host("%s" % (option.sshtarget, ))
+        hm = HostManager(config, hosts=[host1, host2])
+        events = []
+        hm.init_rsync(events.append)
+        print events
+        assert 0
+
+
+def test_getpath_relto_home():
+    x = getpath_relto_home("hello")
+    assert x == py.path.local._gethomedir().join("hello")
+    x = getpath_relto_home(".")
+    assert x == py.path.local._gethomedir()
+
+def test_sethomedir():
+    old = py.path.local.get_temproot().chdir()
+    try:
+        sethomedir()
+        curdir = py.path.local()
+    finally:
+        old.chdir()
+
+    assert py.path.local._gethomedir() == curdir
+
+def test_gethosts():
+    config = py.test.config._reparse(['-n3'])
+    hosts = gethosts(config, '')
+    assert len(hosts) == 3
+

py/test/dsession/testing/test_masterslave.py

@@ -0,0 +1,90 @@
+
+import py
+from py.__.test.dsession.masterslave import MasterNode
+from py.__.test.dsession.hostmanage import Host 
+from basetest import BasicRsessionTest
+from py.__.test import event 
+
+class TestMasterSlaveConnection(BasicRsessionTest): 
+    def getevent(self, eventtype=event.ItemTestReport, timeout=2.0):
+        events = []
+        while 1:
+            try:
+                ev = self.queue.get(timeout=timeout)
+            except py.std.Queue.Empty:
+                print "node channel", self.node.channel
+                print "remoteerror", self.node.channel._getremoteerror()
+                print "seen events", events
+                raise IOError("did not see %r events" % (eventtype))
+            else:
+                if isinstance(ev, eventtype):
+                    return ev
+                events.append(ev)
+
+    def setup_method(self, method):
+        super(TestMasterSlaveConnection, self).setup_method(method)
+        self.queue = py.std.Queue.Queue()
+        self.host = Host("localhost") 
+        self.host.initgateway()
+        self.node = MasterNode(self.host, self.session.config, 
+                               self.queue.put)
+        assert not self.node.channel.isclosed()
+       
+    def teardown_method(self, method):
+        print "at teardown:", self.node.channel
+        self.host.gw.exit()
+
+    def test_crash_invalid_item(self):
+        self.node.send(123) # invalid item 
+        ev = self.getevent(event.HostDown)
+        assert ev.host == self.host
+        assert str(ev.error).find("AttributeError") != -1
+
+    def test_crash_killed(self):
+        if not hasattr(py.std.os, 'kill'):
+            py.test.skip("no os.kill")
+        item = self.getfunc("kill15")
+        self.node.send(item) 
+        ev = self.getevent(event.HostDown)
+        assert ev.host == self.host
+        assert str(ev.error).find("TERMINATED") != -1
+
+    def test_node_down(self):
+        self.node.shutdown()
+        ev = self.getevent(event.HostDown)
+        assert ev.host == self.host 
+        assert not ev.error
+        self.node.callback(self.node.ENDMARK)
+        excinfo = py.test.raises(IOError, 
+            "self.getevent(event.HostDown, timeout=0.01)")
+
+    def test_send_on_closed_channel(self):
+        item = self.getfunc("passed")
+        self.node.channel.close()
+        py.test.raises(IOError, "self.node.send(item)")
+        #ev = self.getevent(event.InternalException)
+        #assert ev.excinfo.errisinstance(IOError)
+
+    def test_send_one(self):
+        item = self.getfunc("passed")
+        self.node.send(self.getfunc("passed"))
+        ev = self.getevent()
+        assert ev.passed 
+        assert ev.colitem == item
+        #assert event.item == item 
+        #assert event.item is not item 
+
+    def test_send_some(self):
+        for outcome in "passed failed skipped".split():
+            self.node.send(self.getfunc(outcome))
+            ev = self.getevent()
+            assert getattr(ev, outcome) 
+
+    def test_send_list(self):
+        l = []
+        for outcome in "passed failed skipped".split():
+            l.append(self.getfunc(outcome))
+        self.node.sendlist(l)
+        for outcome in "passed failed skipped".split():
+            ev = self.getevent()
+            assert getattr(ev, outcome) 

py/test/dsession/testing/test_mypickle.py

@@ -0,0 +1,214 @@
+
+import py
+from py.__.test.dsession.mypickle import ImmutablePickler, PickleChannel, UnpickleError
+
+class A: 
+    pass
+def test_pickle_and_back_IS_same():
+    def pickle_band_back_IS_same(obj, proto):
+        p1 = ImmutablePickler(uneven=False, protocol=proto)
+        p2 = ImmutablePickler(uneven=True, protocol=proto)
+        s1 = p1.dumps(obj)
+        d2 = p2.loads(s1)
+        s2 = p2.dumps(d2)
+        obj_back = p1.loads(s2)
+        assert obj is obj_back 
+
+    a1 = A()
+    a2 = A()
+    a2.a1 = a1
+    for proto in (0,1,2, -1):
+        for obj in {1:2}, [1,2,3], a1, a2:
+            yield pickle_band_back_IS_same, obj, proto
+
+def test_pickling_twice_before_unpickling():
+    p1 = ImmutablePickler(uneven=False)
+    p2 = ImmutablePickler(uneven=True)
+
+    a1 = A()
+    a2 = A()
+    a3 = A() 
+    a3.a1 = a1
+    a2.a1 = a1
+    s1 = p1.dumps(a1)
+    a1.a3 = a3
+    s2 = p1.dumps(a2)
+    other_a1 = p2.loads(s1)
+    other_a2 = p2.loads(s2)
+    back_a1 = p1.loads(p2.dumps(other_a1))
+    other_a3 = p2.loads(p1.dumps(a3))
+    back_a3 = p1.loads(p2.dumps(other_a3))
+    back_a2 = p1.loads(p2.dumps(other_a2))
+    back_a1 = p1.loads(p2.dumps(other_a1))
+    assert back_a1 is a1
+    assert back_a2 is a2
+
+def test_pickling_concurrently():
+    p1 = ImmutablePickler(uneven=False)
+    p2 = ImmutablePickler(uneven=True)
+
+    a1 = A()
+    a1.hasattr = 42
+    a2 = A()
+
+    s1 = p1.dumps(a1)  
+    s2 = p2.dumps(a2)
+    other_a1 = p2.loads(s1)
+    other_a2 = p1.loads(s2)
+    a1_back = p1.loads(p2.dumps(other_a1))
+
+def test_self_memoize():
+    p1 = ImmutablePickler(uneven=False)
+    a1 = A()
+    p1.selfmemoize(a1)
+    x = p1.loads(p1.dumps(a1))
+    assert x is a1
+
+TESTTIMEOUT = 2.0
+class TestPickleChannelFunctional:
+    def setup_class(cls):
+        cls.gw = py.execnet.PopenGateway()
+        cls.gw.remote_init_threads(5)
+
+    def teardown_class(cls):
+        cls.gw.exit()
+
+    def test_popen_send_instance(self):
+        channel = self.gw.remote_exec("""
+            from py.__.test.dsession.mypickle import PickleChannel
+            channel = PickleChannel(channel)
+            from py.__.test.dsession.testing.test_mypickle import A
+            a1 = A()
+            a1.hello = 10
+            channel.send(a1)
+            a2 = channel.receive()
+            channel.send(a2 is a1)
+        """)
+        channel = PickleChannel(channel)
+        a_received = channel.receive()
+        assert isinstance(a_received, A)
+        assert a_received.hello == 10
+        channel.send(a_received)
+        remote_a2_is_a1 = channel.receive()
+        assert remote_a2_is_a1 
+
+    def test_send_concurrent(self):
+        channel = self.gw.remote_exec("""
+            from py.__.test.dsession.mypickle import PickleChannel
+            channel = PickleChannel(channel)
+            from py.__.test.dsession.testing.test_mypickle import A
+            l = [A() for i in range(10)]
+            channel.send(l)
+            other_l = channel.receive() 
+            channel.send((l, other_l))
+            channel.send(channel.receive())
+            channel.receive()
+        """)
+        channel = PickleChannel(channel)
+        l = [A() for i in range(10)]
+        channel.send(l)
+        other_l = channel.receive()
+        channel.send(other_l)
+        ret = channel.receive()
+        assert ret[0] is other_l
+        assert ret[1] is l 
+        back = channel.receive()
+        assert other_l is other_l 
+        channel.send(None)
+
+    #s1 = p1.dumps(a1)  
+    #s2 = p2.dumps(a2)
+    #other_a1 = p2.loads(s1)
+    #other_a2 = p1.loads(s2)
+    #a1_back = p1.loads(p2.dumps(other_a1))
+        
+    def test_popen_with_callback(self):
+        channel = self.gw.remote_exec("""
+            from py.__.test.dsession.mypickle import PickleChannel
+            channel = PickleChannel(channel)
+            from py.__.test.dsession.testing.test_mypickle import A
+            a1 = A()
+            a1.hello = 10
+            channel.send(a1)
+            a2 = channel.receive()
+            channel.send(a2 is a1)
+        """)
+        channel = PickleChannel(channel)
+        queue = py.std.Queue.Queue()
+        channel.setcallback(queue.put)
+        a_received = queue.get(timeout=TESTTIMEOUT)
+        assert isinstance(a_received, A)
+        assert a_received.hello == 10
+        channel.send(a_received)
+        #remote_a2_is_a1 = queue.get(timeout=TESTTIMEOUT)
+        #assert remote_a2_is_a1 
+
+    def test_popen_with_callback_with_endmarker(self):
+        channel = self.gw.remote_exec("""
+            from py.__.test.dsession.mypickle import PickleChannel
+            channel = PickleChannel(channel)
+            from py.__.test.dsession.testing.test_mypickle import A
+            a1 = A()
+            a1.hello = 10
+            channel.send(a1)
+            a2 = channel.receive()
+            channel.send(a2 is a1)
+        """)
+        channel = PickleChannel(channel)
+        queue = py.std.Queue.Queue()
+        channel.setcallback(queue.put, endmarker=-1)
+          
+        a_received = queue.get(timeout=TESTTIMEOUT)
+        assert isinstance(a_received, A)
+        assert a_received.hello == 10
+        channel.send(a_received)
+        remote_a2_is_a1 = queue.get(timeout=TESTTIMEOUT)
+        assert remote_a2_is_a1 
+        endmarker = queue.get(timeout=TESTTIMEOUT)
+        assert endmarker == -1
+
+    def test_popen_with_callback_with_endmarker_and_unpickling_error(self):
+        channel = self.gw.remote_exec("""
+            from py.__.test.dsession.mypickle import PickleChannel
+            channel = PickleChannel(channel)
+            from py.__.test.dsession.testing.test_mypickle import A
+            a1 = A()
+            channel.send(a1)
+            channel.send(a1)
+        """)
+        channel = PickleChannel(channel)
+        queue = py.std.Queue.Queue()
+        a = channel.receive()
+        channel._ipickle._unpicklememo.clear()
+        channel.setcallback(queue.put, endmarker=-1)
+        next = queue.get(timeout=TESTTIMEOUT)
+        assert next == -1 
+        error = channel._getremoteerror()
+        assert isinstance(error, UnpickleError)
+
+    def test_popen_with_newchannel(self):
+        channel = self.gw.remote_exec("""
+            from py.__.test.dsession.mypickle import PickleChannel
+            channel = PickleChannel(channel)
+            newchannel = channel.receive()
+            newchannel.send(42)
+        """)
+        channel = PickleChannel(channel)
+        newchannel = self.gw.newchannel()
+        channel.send(newchannel)
+        channel.waitclose()
+        res = newchannel.receive()
+        assert res == 42
+
+    def test_popen_with_various_methods(self):
+        channel = self.gw.remote_exec("""
+            from py.__.test.dsession.mypickle import PickleChannel
+            channel = PickleChannel(channel)
+            channel.receive()
+        """)
+        channel = PickleChannel(channel)
+        assert not channel.isclosed()
+        assert not channel._getremoteerror()
+        channel.send(2)
+        channel.waitclose(timeout=2)
+