floraachy/pytest2
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[svn r63210] * implement "--dist=each" to distribute each to each available node

Browse Source 
* improved node-management and nice showing of results
* streamline command line options

--HG--
branch : trunk
This commit is contained in:
hpk
2009-03-22 21:44:45 +01:00
parent ad6afe21ff
commit 2c2bfb5513
13 changed files with 123 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
py/test/dist/dsession.py
Vendored
+8 -3
View File
@@ -177,8 +177,11 @@ class DSession(Session):
else:
self.bus.notify("collectionstart", event.CollectionStart(next))
self.queueevent("collectionreport", basic_collect_report(next))
self.senditems(senditems)
#self.senditems_each(senditems)
if self.config.option.dist == "each":
self.senditems_each(senditems)
else:
# XXX assert self.config.option.dist == "load"
self.senditems_load(senditems)
def queueevent(self, eventname, *args, **kwargs):
self.queue.put((eventname, args, kwargs))
@@ -201,7 +204,7 @@ class DSession(Session):
# we have some left, give it to the main loop
self.queueevent("rescheduleitems", event.RescheduleItems(tosend))
def senditems(self, tosend):
def senditems_load(self, tosend):
if not tosend:
return
for node, pending in self.node2pending.items():
@@ -242,6 +245,8 @@ class DSession(Session):
""" setup any neccessary resources ahead of the test run. """
self.nodemanager = NodeManager(self.config)
self.nodemanager.setup_nodes(putevent=self.queue.put)
if self.config.option.dist == "each":
self.nodemanager.wait_nodesready(5.0)
def teardown(self):
""" teardown any resources after a test run. """
py/test/dist/nodemanage.py
Vendored
+20 -15
View File
@@ -1,6 +1,6 @@
import py
import sys, os
from py.__.test.dist.txnode import MasterNode
from py.__.test.dist.txnode import TXNode
from py.__.execnet.gwmanage import GatewayManager
@@ -12,19 +12,11 @@ class NodeManager(object):
self.roots = self.config.getrsyncdirs()
self.gwmanager = GatewayManager(specs)
self.nodes = []
self._nodesready = py.std.threading.Event()
def trace(self, msg):
self.config.bus.notify("trace", "nodemanage", msg)
def trace_nodestatus(self):
if self.config.option.debug:
for ch, result in self.gwmanager.multi_exec("""
import sys, os
channel.send((sys.executable, os.getcwd(), sys.path))
""").receive_each(withchannel=True):
self.trace("spec %r, execuable %r, cwd %r, syspath %r" %(
ch.gateway.spec, result[0], result[1], result[2]))
def config_getignores(self):
return self.config.getconftest_pathlist("rsyncignore")
@@ -55,20 +47,33 @@ class NodeManager(object):
# such that unpickling configs will
# pick it up as the right topdir
# (for other gateways this chdir is irrelevant)
self.trace("making gateways")
old = self.config.topdir.chdir()
try:
self.gwmanager.makegateways()
finally:
old.chdir()
self.trace_nodestatus()
def setup_nodes(self, putevent):
self.rsync_roots()
self.trace_nodestatus()
self.trace("setting up nodes")
for gateway in self.gwmanager.gateways:
node = MasterNode(gateway, self.config, putevent)
self.nodes.append(node)
node = TXNode(gateway, self.config, putevent, slaveready=self._slaveready)
gateway.node = node # to keep node alive
self.trace("started node %r" % node)
def _slaveready(self, node):
#assert node.gateway == node.gateway
#assert node.gateway.node == node
self.nodes.append(node)
self.trace("%s slave node ready %r" % (node.gateway.id, node))
if len(self.nodes) == len(self.gwmanager.gateways):
self._nodesready.set()
def wait_nodesready(self, timeout=None):
self._nodesready.wait(timeout)
if not self._nodesready.isSet():
raise IOError("nodes did not get ready for %r secs" % timeout)
def teardown_nodes(self):
# XXX do teardown nodes?
py/test/dist/testing/test_dsession.py
Vendored
+7 -7
View File
@@ -35,14 +35,14 @@ class TestDSession:
assert not session.node2pending
session.addnode(node)
assert len(session.node2pending) == 1
session.senditems([item])
session.senditems_load([item])
pending = session.removenode(node)
assert pending == [item]
assert item not in session.item2nodes
l = session.removenode(node)
assert not l
def test_send_remove_to_two_nodes(self, testdir):
def test_senditems_each_and_receive_with_two_nodes(self, testdir):
item = testdir.getitem("def test_func(): pass")
node1 = MockNode()
node2 = MockNode()
@@ -60,13 +60,13 @@ class TestDSession:
assert not session.node2pending[node1]
assert not session.item2nodes
def test_senditems_removeitems(self, testdir):
def test_senditems_load_and_receive_one_node(self, testdir):
item = testdir.getitem("def test_func(): pass")
node = MockNode()
rep = run(item, node)
session = DSession(item.config)
session.addnode(node)
session.senditems([item])
session.senditems_load([item])
assert session.node2pending[node] == [item]
assert session.item2nodes[item] == [node]
session.removeitem(item, node)
@@ -174,7 +174,7 @@ class TestDSession:
session.addnode(node2)
# have one test pending for a node that goes down
session.senditems([item1, item2])
session.senditems_load([item1, item2])
node = session.item2nodes[item1] [0]
session.queueevent("testnodedown", node, None)
evrec = EventRecorder(session.bus)
@@ -316,7 +316,7 @@ class TestDSession:
node = MockNode()
session.addnode(node)
session.senditems([item])
session.senditems_load([item])
session.queueevent("itemtestreport", run(item, node))
loopstate = session._initloopstate([])
session.loop_once(loopstate)
@@ -340,7 +340,7 @@ class TestDSession:
colreport = basic_collect_report(modcol)
item1, item2 = colreport.result
session.senditems([item1])
session.senditems_load([item1])
# node2pending will become empty when the loop sees the report
rep = run(item1, node)
py/test/dist/testing/test_nodemanage.py
Vendored
+7
View File
@@ -28,6 +28,13 @@ class TestNodeManager:
assert p.join("dir1").check()
assert p.join("dir1", "file1").check()
def test_popen_nodes_are_ready(self, testdir):
nodemanager = NodeManager(testdir.parseconfig(
"--tx", "3*popen"))
nodemanager.setup_nodes([].append)
nodemanager.wait_nodesready(timeout=2.0)
def test_popen_rsync_subdir(self, testdir, source, dest):
dir1 = source.mkdir("dir1")
dir2 = dir1.mkdir("dir2")
py/test/dist/testing/test_txnode.py
Vendored
+2 -2
View File
@@ -1,6 +1,6 @@
import py
from py.__.test.dist.txnode import MasterNode
from py.__.test.dist.txnode import TXNode
class EventQueue:
def __init__(self, bus, queue=None):
@@ -44,7 +44,7 @@ class MySetup:
self.queue = py.std.Queue.Queue()
self.xspec = py.execnet.XSpec("popen")
self.gateway = py.execnet.makegateway(self.xspec)
self.node = MasterNode(self.gateway, self.config, putevent=self.queue.put)
self.node = TXNode(self.gateway, self.config, putevent=self.queue.put)
assert not self.node.channel.isclosed()
return self.node
py/test/dist/txnode.py
Vendored
+11 -3
View File
@@ -5,16 +5,22 @@ import py
from py.__.test import event
from py.__.test.dist.mypickle import PickleChannel
class MasterNode(object):
""" Install slave code, manage sending test tasks & receiving results """
class TXNode(object):
""" Represents a Test Execution environment in the controlling process.
- sets up a slave node through an execnet gateway
- manages sending of test-items and receival of results and events
- creates events when the remote side crashes
"""
ENDMARK = -1
def __init__(self, gateway, config, putevent):
def __init__(self, gateway, config, putevent, slaveready=None):
self.config = config
self.putevent = putevent
self.gateway = gateway
self.channel = install_slave(gateway, config)
self.channel.setcallback(self.callback, endmarker=self.ENDMARK)
self._sendslaveready = slaveready
self._down = False
def notify(self, eventname, *args, **kwargs):
@@ -39,6 +45,8 @@ class MasterNode(object):
return
eventname, args, kwargs = eventcall
if eventname == "slaveready":
if self._sendslaveready:
self._sendslaveready(self)
self.notify("testnodeready", self)
elif eventname == "slavefinished":
self._down = True