removing some py.execnet references and moving scripts to execnet repo
--HG-- branch : trunk
This commit is contained in:
holger krekel
parent
6c2b1c4363
commit
fb159b0d40
|
|
@ -2,7 +2,6 @@ The py lib is a Python development support library featuring
|
||||||
the following tools and modules:
|
the following tools and modules:
|
||||||
|
|
||||||
* py.test: tool for distributed automated testing
|
* py.test: tool for distributed automated testing
|
||||||
* py.execnet: ad-hoc distributed execution
|
|
||||||
* py.code: dynamic code generation and introspection
|
* py.code: dynamic code generation and introspection
|
||||||
* py.path: uniform local and svn path objects
|
* py.path: uniform local and svn path objects
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,116 +0,0 @@
|
||||||
#!/usr/bin/env python
|
|
||||||
|
|
||||||
"""
|
|
||||||
|
|
||||||
small utility for hot-syncing a svn repository through ssh.
|
|
||||||
uses execnet.
|
|
||||||
|
|
||||||
"""
|
|
||||||
|
|
||||||
import py
|
|
||||||
import sys, os
|
|
||||||
|
|
||||||
def usage():
|
|
||||||
arg0 = sys.argv[0]
|
|
||||||
print """%s [user@]remote-host:/repo/location localrepo [identity keyfile]""" % (arg0,)
|
|
||||||
|
|
||||||
|
|
||||||
def main(args):
|
|
||||||
remote = args[0]
|
|
||||||
localrepo = py.path.local(args[1])
|
|
||||||
if not localrepo.check(dir=1):
|
|
||||||
raise SystemExit("localrepo %s does not exist" %(localrepo,))
|
|
||||||
if len(args) == 3:
|
|
||||||
keyfile = py.path.local(args[2])
|
|
||||||
else:
|
|
||||||
keyfile = None
|
|
||||||
remote_host, path = remote.split(':', 1)
|
|
||||||
print "ssh-connecting to", remote_host
|
|
||||||
gw = getgateway(remote_host, keyfile)
|
|
||||||
|
|
||||||
local_rev = get_svn_youngest(localrepo)
|
|
||||||
|
|
||||||
# local protocol
|
|
||||||
# 1. client sends rev/repo -> server
|
|
||||||
# 2. server checks for newer revisions and sends dumps
|
|
||||||
# 3. client receives dumps, updates local repo
|
|
||||||
# 4. client goes back to step 1
|
|
||||||
c = gw.remote_exec("""
|
|
||||||
import py
|
|
||||||
import os
|
|
||||||
remote_rev, repopath = channel.receive()
|
|
||||||
while 1:
|
|
||||||
rev = py.process.cmdexec('svnlook youngest "%s"' % repopath)
|
|
||||||
rev = int(rev)
|
|
||||||
if rev > remote_rev:
|
|
||||||
revrange = (remote_rev+1, rev)
|
|
||||||
dumpchannel = channel.gateway.newchannel()
|
|
||||||
channel.send(revrange)
|
|
||||||
channel.send(dumpchannel)
|
|
||||||
|
|
||||||
f = os.popen(
|
|
||||||
"svnadmin dump -q --incremental -r %s:%s %s"
|
|
||||||
% (revrange[0], revrange[1], repopath), 'r')
|
|
||||||
try:
|
|
||||||
maxcount = dumpchannel.receive()
|
|
||||||
count = maxcount
|
|
||||||
while 1:
|
|
||||||
s = f.read(8192)
|
|
||||||
if not s:
|
|
||||||
raise EOFError
|
|
||||||
dumpchannel.send(s)
|
|
||||||
count = count - 1
|
|
||||||
if count <= 0:
|
|
||||||
ack = dumpchannel.receive()
|
|
||||||
count = maxcount
|
|
||||||
|
|
||||||
except EOFError:
|
|
||||||
dumpchannel.close()
|
|
||||||
remote_rev = rev
|
|
||||||
else:
|
|
||||||
# using svn-hook instead would be nice here
|
|
||||||
py.std.time.sleep(30)
|
|
||||||
""")
|
|
||||||
|
|
||||||
c.send((local_rev, path))
|
|
||||||
print "checking revisions from %d in %s" %(local_rev, remote)
|
|
||||||
while 1:
|
|
||||||
revstart, revend = c.receive()
|
|
||||||
dumpchannel = c.receive()
|
|
||||||
print "receiving revisions", revstart, "-", revend, "replaying..."
|
|
||||||
svn_load(localrepo, dumpchannel)
|
|
||||||
print "current revision", revend
|
|
||||||
|
|
||||||
def svn_load(repo, dumpchannel, maxcount=100):
|
|
||||||
# every maxcount we will send an ACK to the other
|
|
||||||
# side in order to synchronise and avoid our side
|
|
||||||
# growing buffers (py.execnet does not control
|
|
||||||
# RAM usage or receive queue sizes)
|
|
||||||
dumpchannel.send(maxcount)
|
|
||||||
f = os.popen("svnadmin load -q %s" %(repo, ), "w")
|
|
||||||
count = maxcount
|
|
||||||
for x in dumpchannel:
|
|
||||||
sys.stdout.write(".")
|
|
||||||
sys.stdout.flush()
|
|
||||||
f.write(x)
|
|
||||||
count = count - 1
|
|
||||||
if count <= 0:
|
|
||||||
dumpchannel.send(maxcount)
|
|
||||||
count = maxcount
|
|
||||||
print >>sys.stdout
|
|
||||||
f.close()
|
|
||||||
|
|
||||||
def get_svn_youngest(repo):
|
|
||||||
rev = py.process.cmdexec('svnlook youngest "%s"' % repo)
|
|
||||||
return int(rev)
|
|
||||||
|
|
||||||
def getgateway(host, keyfile=None):
|
|
||||||
return execnet.SshGateway(host, identity=keyfile)
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
if len(sys.argv) < 3:
|
|
||||||
usage()
|
|
||||||
raise SystemExit(1)
|
|
||||||
|
|
||||||
main(sys.argv[1:])
|
|
||||||
|
|
||||||
|
|
@ -1,139 +0,0 @@
|
||||||
"""
|
|
||||||
sysinfo.py [host1] [host2] [options]
|
|
||||||
|
|
||||||
obtain system info from remote machine.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import py
|
|
||||||
import sys
|
|
||||||
|
|
||||||
|
|
||||||
parser = py.std.optparse.OptionParser(usage=__doc__)
|
|
||||||
parser.add_option("-f", "--sshconfig", action="store", dest="ssh_config", default=None,
|
|
||||||
help="use given ssh config file, and add info all contained hosts for getting info")
|
|
||||||
parser.add_option("-i", "--ignore", action="store", dest="ignores", default=None,
|
|
||||||
help="ignore hosts (useful if the list of hostnames come from a file list)")
|
|
||||||
|
|
||||||
def parsehosts(path):
|
|
||||||
path = py.path.local(path)
|
|
||||||
l = []
|
|
||||||
rex = py.std.re.compile(r'Host\s*(\S+)')
|
|
||||||
for line in path.readlines():
|
|
||||||
m = rex.match(line)
|
|
||||||
if m is not None:
|
|
||||||
sshname, = m.groups()
|
|
||||||
l.append(sshname)
|
|
||||||
return l
|
|
||||||
|
|
||||||
class RemoteInfo:
|
|
||||||
def __init__(self, gateway):
|
|
||||||
self.gw = gateway
|
|
||||||
self._cache = {}
|
|
||||||
|
|
||||||
def exreceive(self, execstring):
|
|
||||||
if execstring not in self._cache:
|
|
||||||
channel = self.gw.remote_exec(execstring)
|
|
||||||
self._cache[execstring] = channel.receive()
|
|
||||||
return self._cache[execstring]
|
|
||||||
|
|
||||||
def getmodattr(self, modpath):
|
|
||||||
module = modpath.split(".")[0]
|
|
||||||
return self.exreceive("""
|
|
||||||
import %s
|
|
||||||
channel.send(%s)
|
|
||||||
""" %(module, modpath))
|
|
||||||
|
|
||||||
def islinux(self):
|
|
||||||
return self.getmodattr('sys.platform').find("linux") != -1
|
|
||||||
|
|
||||||
def getfqdn(self):
|
|
||||||
return self.exreceive("""
|
|
||||||
import socket
|
|
||||||
channel.send(socket.getfqdn())
|
|
||||||
""")
|
|
||||||
|
|
||||||
def getmemswap(self):
|
|
||||||
if self.islinux():
|
|
||||||
return self.exreceive("""
|
|
||||||
import commands, re
|
|
||||||
out = commands.getoutput("free")
|
|
||||||
mem = re.search(r"Mem:\s+(\S*)", out).group(1)
|
|
||||||
swap = re.search(r"Swap:\s+(\S*)", out).group(1)
|
|
||||||
channel.send((mem, swap))
|
|
||||||
""")
|
|
||||||
|
|
||||||
def getcpuinfo(self):
|
|
||||||
if self.islinux():
|
|
||||||
return self.exreceive("""
|
|
||||||
# a hyperthreaded cpu core only counts as 1, although it
|
|
||||||
# is present as 2 in /proc/cpuinfo. Counting it as 2 is
|
|
||||||
# misleading because it is *by far* not as efficient as
|
|
||||||
# two independent cores.
|
|
||||||
cpus = {}
|
|
||||||
cpuinfo = {}
|
|
||||||
f = open("/proc/cpuinfo")
|
|
||||||
lines = f.readlines()
|
|
||||||
f.close()
|
|
||||||
for line in lines + ['']:
|
|
||||||
if line.strip():
|
|
||||||
key, value = line.split(":", 1)
|
|
||||||
cpuinfo[key.strip()] = value.strip()
|
|
||||||
else:
|
|
||||||
corekey = (cpuinfo.get("physical id"),
|
|
||||||
cpuinfo.get("core id"))
|
|
||||||
cpus[corekey] = 1
|
|
||||||
numcpus = len(cpus)
|
|
||||||
model = cpuinfo.get("model name")
|
|
||||||
channel.send((numcpus, model))
|
|
||||||
""")
|
|
||||||
|
|
||||||
def debug(*args):
|
|
||||||
print >>sys.stderr, " ".join(map(str, args))
|
|
||||||
def error(*args):
|
|
||||||
debug("ERROR", args[0] + ":", *args[1:])
|
|
||||||
|
|
||||||
def getinfo(sshname, ssh_config=None, loginfo=sys.stdout):
|
|
||||||
debug("connecting to", sshname)
|
|
||||||
try:
|
|
||||||
gw = execnet.SshGateway(sshname, ssh_config=ssh_config)
|
|
||||||
except IOError:
|
|
||||||
error("could not get sshagteway", sshname)
|
|
||||||
else:
|
|
||||||
ri = RemoteInfo(gw)
|
|
||||||
#print "%s info:" % sshname
|
|
||||||
prefix = sshname.upper() + " "
|
|
||||||
print >>loginfo, prefix, "fqdn:", ri.getfqdn()
|
|
||||||
for attr in (
|
|
||||||
"sys.platform",
|
|
||||||
"sys.version_info",
|
|
||||||
):
|
|
||||||
loginfo.write("%s %s: " %(prefix, attr,))
|
|
||||||
loginfo.flush()
|
|
||||||
value = ri.getmodattr(attr)
|
|
||||||
loginfo.write(str(value))
|
|
||||||
loginfo.write("\n")
|
|
||||||
loginfo.flush()
|
|
||||||
memswap = ri.getmemswap()
|
|
||||||
if memswap:
|
|
||||||
mem,swap = memswap
|
|
||||||
print >>loginfo, prefix, "Memory:", mem, "Swap:", swap
|
|
||||||
cpuinfo = ri.getcpuinfo()
|
|
||||||
if cpuinfo:
|
|
||||||
numcpu, model = cpuinfo
|
|
||||||
print >>loginfo, prefix, "number of cpus:", numcpu
|
|
||||||
print >>loginfo, prefix, "cpu model", model
|
|
||||||
return ri
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
options, args = parser.parse_args()
|
|
||||||
hosts = list(args)
|
|
||||||
ssh_config = options.ssh_config
|
|
||||||
if ssh_config:
|
|
||||||
hosts.extend(parsehosts(ssh_config))
|
|
||||||
ignores = options.ignores or ()
|
|
||||||
if ignores:
|
|
||||||
ignores = ignores.split(",")
|
|
||||||
for host in hosts:
|
|
||||||
if host not in ignores:
|
|
||||||
getinfo(host, ssh_config=ssh_config)
|
|
||||||
|
|
||||||
|
|
@ -73,7 +73,6 @@ pageTracker._trackPageview();
|
||||||
html.div(
|
html.div(
|
||||||
html.h3("supporting APIs:"),
|
html.h3("supporting APIs:"),
|
||||||
self.a_docref("pylib index", "index.html"),
|
self.a_docref("pylib index", "index.html"),
|
||||||
self.a_docref("py.execnet", "execnet.html"),
|
|
||||||
self.a_docref("py.path", "path.html"),
|
self.a_docref("py.path", "path.html"),
|
||||||
self.a_docref("py.code", "code.html"),
|
self.a_docref("py.code", "code.html"),
|
||||||
)
|
)
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue