diff --git a/py/test/rsession/hostmanage.py b/py/test/rsession/hostmanage.py index 83f3ff02d..3df8c22df 100644 --- a/py/test/rsession/hostmanage.py +++ b/py/test/rsession/hostmanage.py @@ -2,32 +2,59 @@ import sys, os import py import time import thread, threading -from py.__.test.rsession.master import \ - setup_slave, MasterNode +from py.__.test.rsession.master import MasterNode +from py.__.test.rsession.slave import setup_slave + from py.__.test.rsession import report class HostInfo(object): """ Class trying to store all necessary attributes for host """ - host_ids = {} + _hostname2list = {} + localdest = None - def __init__(self, hostname, relpath=None): - self.hostid = self._getuniqueid(hostname) - self.hostname = hostname - self.relpath = relpath + def __init__(self, spec): + parts = spec.split(':', 1) + self.hostname = parts.pop(0) + if parts: + self.relpath = parts[0] + else: + self.relpath = "pytestcache-" + self.hostname + self.hostid = self._getuniqueid(self.hostname) - def _getuniqueid(cls, hostname): - if not hostname in cls.host_ids: - cls.host_ids[hostname] = 0 - return hostname - retval = hostname + '_' + str(cls.host_ids[hostname]) - cls.host_ids[hostname] += 1 - return retval - _getuniqueid = classmethod(_getuniqueid) + def _getuniqueid(self, hostname): + l = self._hostname2list.setdefault(hostname, []) + hostid = hostname + "_" * len(l) + l.append(hostid) + return hostid + + def initgateway(self, python="python"): + assert not hasattr(self, 'gw') + if self.hostname == "localhost": + gw = py.execnet.PopenGateway(python=python) + else: + gw = py.execnet.SshGateway(self.hostname, + remotepython=python) + self.gw = gw + channel = gw.remote_exec(""" + import os + targetdir = %r + if not os.path.isabs(targetdir): + homedir = os.environ['HOME'] + targetdir = os.path.join(homedir, targetdir) + if not os.path.exists(targetdir): + os.makedirs(targetdir) + channel.send(os.path.abspath(targetdir)) + """ % self.relpath) + self.gw_remotepath = channel.receive() + #print "initialized", gw, "with remotepath", self.gw_remotepath + if self.hostname == "localhost": + self.localdest = py.path.local(self.gw_remotepath) + assert self.localdest.check(dir=1) def __str__(self): - return "" % (self.hostname,) + return "" % (self.hostname, self.relpath) def __hash__(self): return hash(self.hostid) @@ -39,138 +66,77 @@ class HostInfo(object): return not self == other class HostRSync(py.execnet.RSync): - """ An rsync wrapper which filters out *~, .svn/ and *.pyc + """ RSyncer that filters out common files """ - def __init__(self, config): - py.execnet.RSync.__init__(self, delete=True) - self.config = config + def __init__(self, *args, **kwargs): + self._synced = {} + super(HostRSync, self).__init__(*args, **kwargs) def filter(self, path): - if path.endswith('.pyc') or path.endswith('~'): - return False - dir, base = os.path.split(path) - try: - name = "dist_rsync_roots" - rsync_roots = self.config.conftest.rget_path(name, dir) - except AttributeError: - rsync_roots = None - if base == '.svn': - return False - if rsync_roots is None: - return True - return base in rsync_roots + path = py.path.local(path) + if not path.ext in ('.pyc', '.pyo'): + if not path.basename.endswith('~'): + if path.check(dotfile=0): + return True -class DummyGateway(object): - pass - -class HostOptions(object): - """ Dummy container for host options, not to keep that - as different function parameters, mostly related to - tests only - """ - def __init__(self, remote_python="python", - optimise_localhost=True, do_sync=True, - create_gateways=True): - self.remote_python = remote_python - self.optimise_localhost = optimise_localhost - self.do_sync = do_sync - self.create_gateways = create_gateways + def add_target_host(self, host, destrelpath=None, finishedcallback=None): + key = host.hostname, host.relpath + if key in self._synced: + if finishedcallback: + finishedcallback() + return False + self._synced[key] = True + # the follow attributes are set from host.initgateway() + gw = host.gw + remotepath = host.gw_remotepath + if destrelpath is not None: + remotepath = os.path.join(remotepath, destrelpath) + super(HostRSync, self).add_target(gw, + remotepath, + finishedcallback) + return True # added the target class HostManager(object): - def __init__(self, sshhosts, config, options=HostOptions()): + def __init__(self, sshhosts, config): self.sshhosts = sshhosts self.config = config - self.options = options - if not options.create_gateways: - self.prepare_gateways = self.prepare_dummy_gateways - #assert pkgdir.join("__init__.py").check(), ( - # "%s probably wrong" %(pkgdir,)) - - def prepare_dummy_gateways(self): - for host in self.sshhosts: - gw = DummyGateway() - host.gw = gw - gw.host = host - return self.sshhosts - - def prepare_ssh_gateway(self, host): - if self.options.remote_python is None: - gw = py.execnet.SshGateway(host.hostname) - else: - gw = py.execnet.SshGateway(host.hostname, - remotepython=self.options.remote_python) - return gw - - def prepare_popen_rsync_gateway(self, host): - """ Popen gateway, but with forced rsync - """ - from py.__.execnet.register import PopenCmdGateway - gw = PopenCmdGateway("cd ~; python -u -c 'exec input()'") - if not host.relpath.startswith("/"): - host.relpath = os.environ['HOME'] + '/' + host.relpath - return gw - - def prepare_popen_gateway(self, host): - if self.options.remote_python is None: - gw = py.execnet.PopenGateway() - else: - gw = py.execnet.PopenGateway(python=self.options.remote_python) - host.relpath = str(self.config.topdir) - return gw def prepare_gateways(self): + dist_remotepython = self.config.getvalue("dist_remotepython") for host in self.sshhosts: - if host.hostname == 'localhost': - if not self.options.optimise_localhost: - gw = self.prepare_popen_rsync_gateway(host) - else: - gw = self.prepare_popen_gateway(host) - else: - gw = self.prepare_ssh_gateway(host) - host.gw = gw - gw.host = host - return self.sshhosts + host.initgateway(python=dist_remotepython) + host.gw.host = host # XXX would like to avoid it - def need_rsync(self, rsynced, hostname, remoterootpath): - if (hostname, remoterootpath) in rsynced: - return False - if hostname == 'localhost' and self.options.optimise_localhost: - return False - return True + def init_rsync(self, reporter): + # send each rsync roots + roots = self.config.getvalue_pathlist("dist_rsync_roots") + if roots is None: + roots = [self.config.topdir] + self.prepare_gateways() + rsync = HostRSync() + for root in roots: + destrelpath = root.relto(self.config.topdir) + for host in self.sshhosts: + reporter(report.HostRSyncing(host)) + def donecallback(): + reporter(report.HostReady(host)) + rsync.add_target_host(host, destrelpath, + finishedcallback=donecallback) + rsync.send(root) - def init_hosts(self, reporter, done_dict={}): + def init_hosts(self, reporter, done_dict=None): if done_dict is None: done_dict = {} - - hosts = self.prepare_gateways() - - # rsyncing - rsynced = {} - - if self.options.do_sync: - rsync = HostRSync(self.config) - for host in hosts: - if not self.need_rsync(rsynced, host.hostname, host.relpath): - reporter(report.HostReady(host)) - continue - rsynced[(host.hostname, host.relpath)] = True - def done(host=host): - reporter(report.HostReady(host)) - reporter(report.HostRSyncing(host)) - if self.options.do_sync: - rsync.add_target(host.gw, host.relpath, done) - if not self.options.do_sync: - return # for testing only - rsync.send(self.config.topdir) # hosts ready + self.init_rsync(reporter) return self.setup_nodes(reporter, done_dict) def setup_nodes(self, reporter, done_dict): nodes = [] for host in self.sshhosts: - ch = setup_slave(host.gw, host.relpath, self.config) - nodes.append(MasterNode(ch, reporter, done_dict)) - + if hasattr(host.gw, 'remote_exec'): # otherwise dummy for tests :/ + ch = setup_slave(host, self.config) + nodes.append(MasterNode(ch, reporter, done_dict)) return nodes def teardown_hosts(self, reporter, channels, nodes, diff --git a/py/test/rsession/master.py b/py/test/rsession/master.py index a1a31c29b..a4b51d89a 100644 --- a/py/test/rsession/master.py +++ b/py/test/rsession/master.py @@ -74,14 +74,3 @@ def dispatch_loop(masternodes, itemgenerator, shouldstop, waiter() return all_tests -def setup_slave(gateway, pkgpath, config): - from py.__.test.rsession import slave - import os - ch = gateway.remote_exec(str(py.code.Source(slave.setup, "setup()"))) - #if hasattr(gateway, 'sshaddress'): - # assert not os.path.isabs(pkgpath) - ch.send(str(pkgpath)) - ch.send(config.make_repr(defaultconftestnames)) - return ch - -defaultconftestnames = ['dist_nicelevel'] diff --git a/py/test/rsession/rsession.py b/py/test/rsession/rsession.py index 7b440b1b4..31faf141f 100644 --- a/py/test/rsession/rsession.py +++ b/py/test/rsession/rsession.py @@ -10,9 +10,8 @@ import time from py.__.test.rsession import report from py.__.test.rsession.master import \ - setup_slave, MasterNode, dispatch_loop, itemgen, randomgen -from py.__.test.rsession.hostmanage import HostInfo, HostOptions, HostManager - + MasterNode, dispatch_loop, itemgen, randomgen +from py.__.test.rsession.hostmanage import HostInfo, HostManager from py.__.test.rsession.local import local_loop, plain_runner, apigen_runner,\ box_runner from py.__.test.rsession.reporter import LocalReporter, RemoteReporter @@ -24,24 +23,12 @@ class AbstractSession(Session): An abstract session executes collectors/items through a runner. """ - def __init__(self, config, optimise_localhost=True): - super(AbstractSession, self).__init__(config=config) - self.optimise_localhost = optimise_localhost - def fixoptions(self): option = self.config.option if option.runbrowser and not option.startserver: #print "--runbrowser implies --startserver" option.startserver = True super(AbstractSession, self).fixoptions() - - def getpkgdir(path): - path = py.path.local(path) - pkgpath = path.pypkgpath() - if pkgpath is None: - pkgpath = path - return pkgpath - getpkgdir = staticmethod(getpkgdir) def init_reporter(self, reporter, sshhosts, reporter_class, arg=""): """ This initialises so called `reporter` class, which will @@ -115,18 +102,6 @@ class AbstractSession(Session): self.checkfun = checkfun return new_reporter, checkfun -def parse_directories(sshhosts): - """ Parse sshadresses of hosts to have distinct hostname/hostdir - """ - directories = {} - for host in sshhosts: - m = re.match("^(.*?):(.*)$", host.hostname) - if m: - host.hostname = m.group(1) - host.relpath = m.group(2) + "-" + host.hostname - else: - host.relpath = "pytestcache-%s" % host.hostname - class RSession(AbstractSession): """ Remote version of session """ @@ -151,7 +126,7 @@ class RSession(AbstractSession): """ main loop for running tests. """ args = self.config.args - sshhosts, remotepython = self.read_distributed_config() + sshhosts = self._getconfighosts() reporter, startserverflag = self.init_reporter(reporter, sshhosts, RemoteReporter) reporter, checkfun = self.wrap_reporter(reporter) @@ -159,9 +134,7 @@ class RSession(AbstractSession): reporter(report.TestStarted(sshhosts)) done_dict = {} - hostopts = HostOptions(remote_python=remotepython, - optimise_localhost=self.optimise_localhost) - hostmanager = HostManager(sshhosts, self.config, hostopts) + hostmanager = HostManager(sshhosts, self.config) try: nodes = hostmanager.init_hosts(reporter, done_dict) reporter(report.RsyncFinished()) @@ -191,14 +164,9 @@ class RSession(AbstractSession): self.kill_server(startserverflag) raise - def read_distributed_config(self): - """ Read from conftest file the configuration of distributed testing - """ - sshhosts = [HostInfo(i) for i in - self.config.getvalue("dist_hosts")] - parse_directories(sshhosts) - remotepython = self.config.getvalue("dist_remotepython") - return sshhosts, remotepython + def _getconfighosts(self): + return [HostInfo(spec) for spec in + self.config.getvalue("dist_hosts")] def dispatch_tests(self, nodes, reporter, checkfun, done_dict): colitems = self.config.getcolitems() @@ -262,7 +230,7 @@ class LSession(AbstractSession): print >>sys.stderr, 'building documentation' capture = py.io.StdCaptureFD() try: - pkgdir = self.getpkgdir(self.config.args[0]) + pkgdir = py.path.local(self.config.args[0]).pypkgpath() apigen.build(pkgdir, DocStorageAccessor(self.docstorage), capture) @@ -272,7 +240,7 @@ class LSession(AbstractSession): def init_runner(self): if self.config.option.apigen: from py.__.apigen.tracer.tracer import Tracer, DocStorage - pkgdir = self.getpkgdir(self.config.args[0]) + pkgdir = py.path.local(self.config.args[0]).pypkgpath() apigen = py.path.local(self.config.option.apigen).pyimport() if not hasattr(apigen, 'get_documentable_items'): raise NotImplementedError("Provided script does not seem " @@ -288,4 +256,3 @@ class LSession(AbstractSession): return box_runner return plain_runner - diff --git a/py/test/rsession/slave.py b/py/test/rsession/slave.py index 56c7e1f43..0c13743e7 100644 --- a/py/test/rsession/slave.py +++ b/py/test/rsession/slave.py @@ -106,6 +106,15 @@ def slave_main(receive, send, path, config, pidinfo): while nextitem is not None: nextitem = receive() +defaultconftestnames = ['dist_nicelevel'] +def setup_slave(host, config): + channel = host.gw.remote_exec(str(py.code.Source(setup, "setup()"))) + configrepr = config.make_repr(defaultconftestnames) + #print "sending configrepr", configrepr + channel.send(host.gw_remotepath) + channel.send(configrepr) + return channel + def setup(): def callback_gen(channel, queue, info): def callback(item): @@ -116,19 +125,16 @@ def setup(): sys.exit(0) queue.put(item) return callback - + # our current dir is the topdir import os, sys - basedir = channel.receive() # path is ready + basedir = channel.receive() config_repr = channel.receive() # setup defaults... sys.path.insert(0, basedir) import py config = py.test.config - if config._initialized: - config = config._reparse([basedir]) - config.merge_repr(config_repr) - else: - config.initdirect(basedir, config_repr) + assert not config._initialized + config.initdirect(basedir, config_repr) if not config.option.nomagic: py.magic.invoke(assertion=1) from py.__.test.rsession.slave import slave_main, PidInfo diff --git a/py/test/rsession/testing/test_hostmanage.py b/py/test/rsession/testing/test_hostmanage.py new file mode 100644 index 000000000..48d8a868d --- /dev/null +++ b/py/test/rsession/testing/test_hostmanage.py @@ -0,0 +1,137 @@ + +""" RSync filter test +""" + +import py +from py.__.test.rsession.hostmanage import HostRSync +from py.__.test.rsession.hostmanage import HostInfo, HostManager + +class DirSetup: + def setup_method(self, method): + name = "%s.%s" %(self.__class__.__name__, method.func_name) + self.tmpdir = py.test.ensuretemp(name) + self.source = self.tmpdir.ensure("source", dir=1) + self.dest = self.tmpdir.join("dest") + +class TestHostInfo: + def test_defaultpath(self): + x = HostInfo("localhost") + assert x.hostname == "localhost" + assert x.relpath == "pytestcache-" + x.hostname + + def test_path(self): + x = HostInfo("localhost:/tmp") + assert x.relpath == "/tmp" + assert x.hostname == "localhost" + + def test_hostid(self): + x = HostInfo("localhost") + y = HostInfo("localhost") + assert x.hostid != y.hostid + x = HostInfo("localhost:/tmp") + y = HostInfo("localhost") + assert x.hostid != y.hostid + + def test_non_existing_hosts(self): + host = HostInfo("alskdjalsdkjasldkajlsd") + py.test.raises((py.process.cmdexec.Error, IOError, EOFError), + host.initgateway) + + def test_initgateway_localhost_relpath(self): + name = "pytestcache-localhost" + x = HostInfo("localhost:%s" % name) + x.initgateway() + assert x.gw + try: + homedir = py.path.local(py.std.os.environ['HOME']) + expected = homedir.join(name) + assert x.gw_remotepath == str(expected) + assert x.localdest == expected + finally: + x.gw.exit() + + + def test_initgateway_ssh_and_remotepath(self): + option = py.test.config.option + if option.sshtarget is None: + py.test.skip("no known ssh target, use -S to set one") + x = HostInfo("%s" % (option.sshtarget, )) + x.initgateway() + assert x.gw + assert x.gw_remotepath.endswith(x.relpath) + channel = x.gw.remote_exec(""" + import os + homedir = os.environ['HOME'] + relpath = channel.receive() + path = os.path.join(homedir, relpath) + channel.send(path) + """) + channel.send(x.relpath) + res = channel.receive() + assert res == x.gw_remotepath + assert x.localdest is None + +class TestSyncing(DirSetup): + def test_hrsync_filter(self): + self.source.ensure("dir", "file.txt") + self.source.ensure(".svn", "entries") + self.source.ensure(".somedotfile", "moreentries") + self.source.ensure("somedir", "editfile~") + syncer = HostRSync() + l = list(self.source.visit(rec=syncer.filter, + fil=syncer.filter)) + assert len(l) == 3 + basenames = [x.basename for x in l] + assert 'dir' in basenames + assert 'file.txt' in basenames + assert 'somedir' in basenames + + def test_hrsync_one_host(self): + h1 = HostInfo("localhost:%s" % self.dest) + finished = [] + rsync = HostRSync() + h1.initgateway() + rsync.add_target_host(h1) + self.source.join("hello.py").write("world") + rsync.send(self.source) + assert self.dest.join("hello.py").check() + + def test_hrsync_same_host_twice(self): + h1 = HostInfo("localhost:%s" % self.dest) + h2 = HostInfo("localhost:%s" % self.dest) + finished = [] + rsync = HostRSync() + l = [] + h1.initgateway() + res1 = rsync.add_target_host(h1) + assert res1 + res2 = rsync.add_target_host(h2) + assert not res2 + +class TestHostManager(DirSetup): + def test_hostmanager_init_rsync_topdir(self): + dir2 = self.source.ensure("dir1", "dir2", dir=1) + dir2.ensure("hello") + config = py.test.config._reparse([self.source]) + hm = HostManager([HostInfo("localhost:" + str(self.dest))], config) + events = [] + hm.init_rsync(reporter=events.append) + assert self.dest.join("dir1").check() + assert self.dest.join("dir1", "dir2").check() + assert self.dest.join("dir1", "dir2", 'hello').check() + + def test_hostmanager_init_rsync_rsync_roots(self): + dir2 = self.source.ensure("dir1", "dir2", dir=1) + dir2.ensure("hello") + self.source.ensure("bogusdir", "file") + self.source.join("conftest.py").write(py.code.Source(""" + dist_rsync_roots = ['dir1/dir2'] + """)) + config = py.test.config._reparse([self.source]) + hm = HostManager([HostInfo("localhost:" + str(self.dest))], config) + events = [] + hm.init_rsync(reporter=events.append) + assert self.dest.join("dir1").check() + assert self.dest.join("dir1", "dir2").check() + assert self.dest.join("dir1", "dir2", 'hello').check() + assert not self.dest.join("bogus").check() diff --git a/py/test/rsession/testing/test_master.py b/py/test/rsession/testing/test_master.py index c45310fda..cc33b3807 100644 --- a/py/test/rsession/testing/test_master.py +++ b/py/test/rsession/testing/test_master.py @@ -9,18 +9,19 @@ import py, sys if sys.platform == 'win32': py.test.skip("rsession is unsupported on Windows.") -from py.__.test.rsession.master import dispatch_loop, setup_slave, MasterNode, randomgen +from py.__.test.rsession.master import dispatch_loop, MasterNode, randomgen +from py.__.test.rsession.slave import setup_slave from py.__.test.rsession.outcome import ReprOutcome, Outcome -from py.__.test.rsession.testing.test_slave import funcpass_spec, funcfail_spec, funchang_spec from py.__.test.rsession import report from py.__.test.rsession.hostmanage import HostInfo def setup_module(mod): # bind an empty config - config = py.test.config._reparse([]) + mod.tmpdir = tmpdir = py.test.ensuretemp(mod.__name__) + # to avoid rsyncing + config = py.test.config._reparse([tmpdir]) config._overwrite('dist_taskspernode', 10) - mod.pkgdir = py.path.local(py.__file__).dirpath().dirpath() - mod.rootcol = py.test.collect.Directory(mod.pkgdir) + mod.rootcol = config._getcollector(tmpdir) class DummyGateway(object): def __init__(self): @@ -92,46 +93,69 @@ def test_dispatch_loop(): node.pending.pop() dispatch_loop(masternodes, itemgenerator, shouldstop, waiter=waiter) -def test_slave_setup(): - gw = py.execnet.PopenGateway() - config = py.test.config._reparse([]) - channel = setup_slave(gw, pkgdir, config) - spec = rootcol._getitembynames(funcpass_spec)._get_collector_trail() - channel.send(spec) - output = ReprOutcome(channel.receive()) - assert output.passed - channel.send(42) - channel.waitclose(10) - gw.exit() +class TestSlave: + def setup_class(cls): + cls.tmpdir = tmpdir = py.test.ensuretemp(cls.__name__) + pkgpath = tmpdir.join("pkg") + pkgpath.ensure("__init__.py") + pkgpath.join("test_something.py").write(py.code.Source(""" + def funcpass(): + pass -def test_slave_running(): - def simple_report(event): - if not isinstance(event, report.ReceivedItemOutcome): - return - item = event.item - if item.code.name == 'funcpass': - assert event.outcome.passed - else: - assert not event.outcome.passed - - def open_gw(): - gw = py.execnet.PopenGateway() - gw.host = HostInfo("localhost") - gw.host.gw = gw - config = py.test.config._reparse([]) - channel = setup_slave(gw, pkgdir, config) - mn = MasterNode(channel, simple_report, {}) - return mn - - master_nodes = [open_gw(), open_gw(), open_gw()] - funcpass_item = rootcol._getitembynames(funcpass_spec) - funcfail_item = rootcol._getitembynames(funcfail_spec) - itemgenerator = iter([funcfail_item] + - [funcpass_item] * 5 + [funcfail_item] * 5) - shouldstop = lambda : False - dispatch_loop(master_nodes, itemgenerator, shouldstop) + def funcfail(): + raise AssertionError("hello world") + """)) + cls.config = py.test.config._reparse([tmpdir]) + assert cls.config.topdir == tmpdir + cls.rootcol = cls.config._getcollector(tmpdir) + + def _gettrail(self, *names): + item = self.rootcol._getitembynames(names) + return self.config.get_collector_trail(item) + + def test_slave_setup(self): + host = HostInfo("localhost:%s" %(self.tmpdir,)) + host.initgateway() + channel = setup_slave(host, self.config) + spec = self._gettrail("pkg", "test_something.py", "funcpass") + print "sending", spec + channel.send(spec) + output = ReprOutcome(channel.receive()) + assert output.passed + channel.send(42) + channel.waitclose(10) + host.gw.exit() + + def test_slave_running(self): + py.test.skip("XXX test broken, needs refactoring") + def simple_report(event): + if not isinstance(event, report.ReceivedItemOutcome): + return + item = event.item + if item.code.name == 'funcpass': + assert event.outcome.passed + else: + assert not event.outcome.passed + + def open_gw(): + gw = py.execnet.PopenGateway() + gw.host = HostInfo("localhost") + gw.host.gw = gw + config = py.test.config._reparse([tmpdir]) + channel = setup_slave(gw.host, config) + mn = MasterNode(channel, simple_report, {}) + return mn + + master_nodes = [open_gw(), open_gw(), open_gw()] + funcpass_item = rootcol._getitembynames(funcpass_spec) + funcfail_item = rootcol._getitembynames(funcfail_spec) + itemgenerator = iter([funcfail_item] + + [funcpass_item] * 5 + [funcfail_item] * 5) + shouldstop = lambda : False + dispatch_loop(master_nodes, itemgenerator, shouldstop) def test_slave_running_interrupted(): + py.test.skip("XXX test broken, needs refactoring") #def simple_report(event): # if not isinstance(event, report.ReceivedItemOutcome): # return @@ -146,7 +170,7 @@ def test_slave_running_interrupted(): gw = py.execnet.PopenGateway() gw.host = HostInfo("localhost") gw.host.gw = gw - config = py.test.config._reparse([]) + config = py.test.config._reparse([tmpdir]) channel = setup_slave(gw, pkgdir, config) mn = MasterNode(channel, reports.append, {}) return mn, gw, channel diff --git a/py/test/rsession/testing/test_rest.py b/py/test/rsession/testing/test_rest.py index 5c426b719..d666aec1b 100644 --- a/py/test/rsession/testing/test_rest.py +++ b/py/test/rsession/testing/test_rest.py @@ -52,7 +52,7 @@ class TestRestUnits(object): 'localhost\n\n') def test_report_HostRSyncing(self): - event = report.HostRSyncing(HostInfo('localhost', '/foo/bar')) + event = report.HostRSyncing(HostInfo('localhost:/foo/bar')) reporter.report(event) assert stdout.getvalue() == ('::\n\n localhost: RSYNC ==> ' '/foo/bar\n\n') diff --git a/py/test/rsession/testing/test_rsession.py b/py/test/rsession/testing/test_rsession.py index 932c3c200..d9dc4109d 100644 --- a/py/test/rsession/testing/test_rsession.py +++ b/py/test/rsession/testing/test_rsession.py @@ -4,38 +4,16 @@ import py from py.__.test.rsession import report -from py.__.test.rsession.rsession import RSession, parse_directories,\ - parse_directories -from py.__.test.rsession.hostmanage import HostOptions, HostManager,\ - HostInfo +from py.__.test.rsession.rsession import RSession +from py.__.test.rsession.hostmanage import HostManager, HostInfo from py.__.test.rsession.testing.test_slave import funcfail_spec,\ funcpass_spec, funcskip_spec, funcprint_spec, funcprintfail_spec, \ funcoptioncustom_spec def setup_module(mod): mod.pkgdir = py.path.local(py.__file__).dirpath() + mod.tmpdir = py.test.ensuretemp(mod.__name__) -def test_setup_non_existing_hosts(): - setup_events = [] - hosts = [HostInfo("alskdjalsdkjasldkajlsd")] - config = py.test.config._reparse([]) - hm = HostManager(hosts, config) - cmd = "hm.init_hosts(setup_events.append)" - py.test.raises((py.process.cmdexec.Error, IOError, EOFError), cmd) - #assert setup_events - -def test_getpkdir(): - one = pkgdir.join("initpkg.py") - two = pkgdir.join("path", "__init__.py") - p1 = RSession.getpkgdir(one) - p2 = RSession.getpkgdir(two) - assert p1 == p2 - assert p1 == pkgdir - -def test_getpkdir_no_inits(): - tmp = py.test.ensuretemp("getpkdir1") - fn = tmp.ensure("hello.py") - assert RSession.getpkgdir(fn) == fn #def test_make_colitems(): # one = pkgdir.join("initpkg.py") @@ -74,10 +52,11 @@ def test_example_tryiter(): class TestRSessionRemote: def test_example_distribution_minus_x(self): + destdir = py.test.ensuretemp("example_dist_dest_x") tmpdir = py.test.ensuretemp("example_distribution_minus_x") tmpdir.ensure("sub", "conftest.py").write(py.code.Source(""" - dist_hosts = [%r] - """ % ('localhost',))) + dist_hosts = ['localhost:%s'] + """ % destdir)) tmpdir.ensure("sub", "__init__.py") tmpdir.ensure("sub", "test_one.py").write(py.code.Source(""" def test_1(): @@ -92,8 +71,7 @@ class TestRSessionRemote: def test_4(someargs): pass """)) - args = [str(tmpdir.join("sub")), "-x"] - config = py.test.config._reparse(args) + config = py.test.config._reparse([tmpdir.join("sub"), '-x']) rsession = RSession(config) allevents = [] rsession.main(reporter=allevents.append) @@ -102,13 +80,14 @@ class TestRSessionRemote: assert len(testevents) == 3 assert rsession.checkfun() - def test_example_distribution(self): + def test_distribution_rsync_roots_example(self): + destdir = py.test.ensuretemp("example_dist_destdir") subdir = "sub_example_dist" tmpdir = py.test.ensuretemp("example_distribution") tmpdir.ensure(subdir, "conftest.py").write(py.code.Source(""" - dist_hosts = [%r] + dist_hosts = ["localhost:%s"] dist_rsync_roots = ["%s", "../py"] - """ % ('localhost', tmpdir.join(subdir), ))) + """ % (destdir, tmpdir.join(subdir), ))) tmpdir.ensure(subdir, "__init__.py") tmpdir.ensure(subdir, "test_one.py").write(py.code.Source(""" def test_1(): @@ -124,16 +103,18 @@ class TestRSessionRemote: def test_6(): import py assert py.__file__ != '%s' - """ % (str(tmpdir.join(subdir)), py.__file__))) - tmpdir.join("py").mksymlinkto(py.path.local(py.__file__).dirpath()) - args = [str(tmpdir.join(subdir))] - config = py.test.config._reparse(args) - rsession = RSession(config, optimise_localhost=False) + """ % (tmpdir.join(subdir), py.__file__))) + destdir.join("py").mksymlinkto(py.path.local(py.__file__).dirpath()) + config = py.test.config._reparse([tmpdir.join(subdir)]) + assert config.topdir == tmpdir + assert not tmpdir.join("__init__.py").check() + rsession = RSession(config) allevents = [] rsession.main(reporter=allevents.append) testevents = [x for x in allevents if isinstance(x, report.ReceivedItemOutcome)] assert len(testevents) + print testevents passevents = [i for i in testevents if i.outcome.passed] failevents = [i for i in testevents if i.outcome.excinfo] skippedevents = [i for i in testevents if i.outcome.skipped] @@ -156,13 +137,11 @@ class TestRSessionRemote: def test_setup_teardown_ssh(self): hosts = [HostInfo('localhost')] - parse_directories(hosts) setup_events = [] teardown_events = [] - config = py.test.config._reparse([]) - opts = HostOptions(optimise_localhost=False) - hm = HostManager(hosts, config, opts) + config = py.test.config._reparse([tmpdir]) + hm = HostManager(hosts, config) nodes = hm.init_hosts(setup_events.append) hm.teardown_hosts(teardown_events.append, [node.channel for node in nodes], nodes) @@ -184,12 +163,10 @@ class TestRSessionRemote: def test_setup_teardown_run_ssh(self): hosts = [HostInfo('localhost')] - parse_directories(hosts) allevents = [] config = py.test.config._reparse([]) - opts = HostOptions(optimise_localhost=False) - hm = HostManager(hosts, config, opts) + hm = HostManager(hosts, config) nodes = hm.init_hosts(allevents.append) from py.__.test.rsession.testing.test_executor \ @@ -223,47 +200,11 @@ class TestRSessionRemote: passed_stdout = [i for i in passed if i.outcome.stdout.find('samfing') != -1] assert len(passed_stdout) == len(nodes), passed - def test_config_pass(self): - """ Tests options object passing master -> server - """ - allevents = [] - hosts = [HostInfo('localhost')] - parse_directories(hosts) - config = py.test.config._reparse([]) - config._overwrite('custom', 'custom') - # we need to overwrite default list to serialize - from py.__.test.rsession.master import defaultconftestnames - defaultconftestnames.append("custom") - try: - opts = HostOptions(optimise_localhost=False) - hm = HostManager(hosts, config, opts) - nodes = hm.init_hosts(allevents.append) - - rootcol = py.test.collect.Directory(pkgdir.dirpath()) - itempass = rootcol._getitembynames(funcoptioncustom_spec) - - for node in nodes: - node.send(itempass) - - hm.teardown_hosts(allevents.append, [node.channel for node in nodes], nodes) - events = [i for i in allevents - if isinstance(i, report.ReceivedItemOutcome)] - passed = [i for i in events - if i.outcome.passed] - skipped = [i for i in events - if i.outcome.skipped] - assert len(passed) == 1 * len(nodes) - assert len(skipped) == 0 - assert len(events) == len(passed) - finally: - defaultconftestnames.remove("custom") - def test_nice_level(self): """ Tests if nice level behaviour is ok """ allevents = [] hosts = [HostInfo('localhost')] - parse_directories(hosts) tmpdir = py.test.ensuretemp("nice") tmpdir.ensure("__init__.py") tmpdir.ensure("conftest.py").write(py.code.Source(""" @@ -283,45 +224,6 @@ class TestRSessionRemote: if isinstance(x, report.ReceivedItemOutcome)] passevents = [x for x in testevents if x.outcome.passed] assert len(passevents) == 1 - -class XxxTestDirectories(object): - # need complete rewrite, and unsure if it makes sense at all - def test_simple_parse(self): - sshhosts = [HostInfo(i) for i in ['h1', 'h2', 'h3']] - parse_directories(sshhosts) - - def test_sophisticated_parse(self): - sshhosts = ['a@h1:/tmp', 'h2:tmp', 'h3'] - dirs = parse_directories(sshhosts) - assert py.builtin.sorted( - dirs.values()) == ['/tmp', 'pytestcache', 'tmp'] - - def test_parse_multiple_hosts(self): - hosts = ['h1', 'h1', 'h1:/tmp'] - dirs = parse_directories(hosts) - assert dirs == {(0, 'h1'): 'pytestcache', (1, 'h1'): 'pytestcache', - (2, 'h1'):'/tmp'} - -class TestInithosts(object): - def test_inithosts(self): - testevents = [] - hostnames = ['h1:/tmp', 'h1:/tmp', 'h1:/other', 'h2', 'h2:home'] - hosts = [HostInfo(i) for i in hostnames] - parse_directories(hosts) - config = py.test.config._reparse([]) - opts = HostOptions(do_sync=False, create_gateways=False) - hm = HostManager(hosts, config, opts) - nodes = hm.init_hosts(testevents.append) - events = [i for i in testevents if isinstance(i, report.HostRSyncing)] - assert len(events) == 4 - assert events[0].host.hostname == 'h1' - assert events[0].host.relpath == '/tmp-h1' - assert events[1].host.hostname == 'h1' - assert events[1].host.relpath == '/other-h1' - assert events[2].host.hostname == 'h2' - assert events[2].host.relpath == 'pytestcache-h2' - assert events[3].host.hostname == 'h2' - assert events[3].host.relpath == 'home-h2' def test_rsession_no_disthost(): tmpdir = py.test.ensuretemp("rsession_no_disthost") diff --git a/py/test/rsession/testing/test_rsync.py b/py/test/rsession/testing/test_rsync.py deleted file mode 100644 index 821f10d99..000000000 --- a/py/test/rsession/testing/test_rsync.py +++ /dev/null @@ -1,30 +0,0 @@ - -""" RSync filter test -""" - -import py -from py.__.test.rsession.hostmanage import HostRSync - -def test_rsync(): - tmpdir = py.test.ensuretemp("rsync_rsession") - tmpdir.ensure("a", dir=True) - tmpdir.ensure("b", dir=True) - tmpdir.ensure("conftest.py").write(py.code.Source(""" - dist_rsyncroots = ['a'] - """)) - tmpdir.join("a").ensure("x") - adir = tmpdir.join("a").ensure("xy", dir=True) - adir.ensure("conftest.py").write(py.code.Source(""" - dist_rsyncroots = ['b', 'conftest.py'] - """)) - adir.ensure("a", dir=True) - adir.ensure("b", dir=True) - config = py.test.config._reparse([str(tmpdir)]) - h = HostRSync(config) - h.sourcedir = config.topdir - assert h.filter(str(tmpdir.join("a"))) - assert not h.filter(str(tmpdir.join("b"))) - assert h.filter(str(tmpdir.join("a").join("x"))) - assert h.filter(str(adir.join("conftest.py"))) - assert not h.filter(str(adir.join("a"))) - assert h.filter(str(adir.join("b"))) diff --git a/py/test/rsession/testing/test_slave.py b/py/test/rsession/testing/test_slave.py index 4a73742d3..6fd867d24 100644 --- a/py/test/rsession/testing/test_slave.py +++ b/py/test/rsession/testing/test_slave.py @@ -11,6 +11,7 @@ if sys.platform == 'win32': py.test.skip("rsession is unsupported on Windows.") def setup_module(module): + module.tmpdir = py.test.ensuretemp(module.__name__) module.rootdir = py.path.local(py.__file__).dirpath().dirpath() module.rootcol = py.test.collect.Directory(rootdir) @@ -107,64 +108,11 @@ def test_slave_run_failing_wrapped(): assert not outcome.setupfailure assert outcome.excinfo -def test_slave_main_simple(): - res = [] - failitem = rootcol._getitembynames(funcfail_spec) - passitem = rootcol._getitembynames(funcpass_spec) - q = [None, - passitem._get_collector_trail(), - failitem._get_collector_trail() - ] - config = py.test.config._reparse([]) - pidinfo = PidInfo() - slave_main(q.pop, res.append, str(rootdir), config, pidinfo) - assert len(res) == 2 - res_repr = [ReprOutcome(r) for r in res] - assert not res_repr[0].passed and res_repr[1].passed - def test_slave_run_different_stuff(): node = gettestnode() node.run(rootcol._getitembynames("py doc log.txt".split()). _get_collector_trail()) -def test_slave_setup_exit(): - tmp = py.test.ensuretemp("slaveexit") - tmp.ensure("__init__.py") - q = py.std.Queue.Queue() - config = py.test.config._reparse([tmp]) - - class C: - res = [] - def __init__(self): - self.q = [str(tmp), - config.make_repr(conftestnames=['dist_nicelevel']), - funchang_spec, - 42, - funcpass_spec] - self.q.reverse() - - def receive(self): - return self.q.pop() - - def setcallback(self, callback): - import thread - def f(): - while 1: - callback(self.q.pop()) - f() - #thread.start_new_thread(f, ()) - - def close(self): - pass - - send = res.append - try: - exec py.code.Source(setup, "setup()").compile() in {'channel':C()} - except SystemExit: - pass - else: - py.test.fail("Did not exit") - def test_pidinfo(): if not hasattr(os, 'fork') or not hasattr(os, 'waitpid'): py.test.skip("Platform does not support fork")