some release preps and cleanups
- update setup.py for release - use distributes_setup on python3 - remove unncessary package_data - remove execnet example --HG-- branch : trunk
This commit is contained in:
parent
92d482069c
commit
270ac2bc0d
|
@ -1,16 +1,14 @@
|
||||||
include CHANGELOG
|
include CHANGELOG
|
||||||
include README.txt
|
include README.txt
|
||||||
include setup.py
|
include setup.py
|
||||||
|
include distribute_setup.py
|
||||||
include LICENSE
|
include LICENSE
|
||||||
include py/LICENSE
|
|
||||||
include py/path/testing/repotest.dump
|
|
||||||
include py/rest/rest.sty.template
|
|
||||||
graft doc
|
graft doc
|
||||||
graft contrib
|
graft contrib
|
||||||
graft example
|
graft example
|
||||||
graft py/bin
|
graft bin
|
||||||
graft py/rest/testing/data
|
|
||||||
exclude *.orig
|
exclude *.orig
|
||||||
exclude *.rej
|
exclude *.rej
|
||||||
|
exclude .hginore
|
||||||
prune .svn
|
prune .svn
|
||||||
prune .hg
|
prune .hg
|
||||||
|
|
|
@ -1,6 +1,7 @@
|
||||||
import py
|
import py
|
||||||
import subprocess
|
import subprocess
|
||||||
import os
|
import os
|
||||||
|
import execnet
|
||||||
|
|
||||||
|
|
||||||
#
|
#
|
||||||
|
|
|
@ -0,0 +1,456 @@
|
||||||
|
#!python
|
||||||
|
"""Bootstrap distribute installation
|
||||||
|
|
||||||
|
If you want to use setuptools in your package's setup.py, just include this
|
||||||
|
file in the same directory with it, and add this to the top of your setup.py::
|
||||||
|
|
||||||
|
from distribute_setup import use_setuptools
|
||||||
|
use_setuptools()
|
||||||
|
|
||||||
|
If you want to require a specific version of setuptools, set a download
|
||||||
|
mirror, or use an alternate download directory, you can do so by supplying
|
||||||
|
the appropriate options to ``use_setuptools()``.
|
||||||
|
|
||||||
|
This file can also be run as a script to install or upgrade setuptools.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import fnmatch
|
||||||
|
import tempfile
|
||||||
|
import tarfile
|
||||||
|
from distutils import log
|
||||||
|
|
||||||
|
try:
|
||||||
|
from site import USER_SITE
|
||||||
|
except ImportError:
|
||||||
|
USER_SITE = None
|
||||||
|
|
||||||
|
try:
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
def _python_cmd(*args):
|
||||||
|
args = (sys.executable,) + args
|
||||||
|
return subprocess.call(args) == 0
|
||||||
|
|
||||||
|
except ImportError:
|
||||||
|
# will be used for python 2.3
|
||||||
|
def _python_cmd(*args):
|
||||||
|
args = (sys.executable,) + args
|
||||||
|
# quoting arguments if windows
|
||||||
|
if sys.platform == 'win32':
|
||||||
|
def quote(arg):
|
||||||
|
if ' ' in arg:
|
||||||
|
return '"%s"' % arg
|
||||||
|
return arg
|
||||||
|
args = [quote(arg) for arg in args]
|
||||||
|
return os.spawnl(os.P_WAIT, sys.executable, *args) == 0
|
||||||
|
|
||||||
|
DEFAULT_VERSION = "0.6.6"
|
||||||
|
DEFAULT_URL = "http://pypi.python.org/packages/source/d/distribute/"
|
||||||
|
SETUPTOOLS_PKG_INFO = """\
|
||||||
|
Metadata-Version: 1.0
|
||||||
|
Name: setuptools
|
||||||
|
Version: 0.6c9
|
||||||
|
Summary: xxxx
|
||||||
|
Home-page: xxx
|
||||||
|
Author: xxx
|
||||||
|
Author-email: xxx
|
||||||
|
License: xxx
|
||||||
|
Description: xxx
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def _install(tarball):
|
||||||
|
# extracting the tarball
|
||||||
|
tmpdir = tempfile.mkdtemp()
|
||||||
|
log.warn('Extracting in %s', tmpdir)
|
||||||
|
old_wd = os.getcwd()
|
||||||
|
try:
|
||||||
|
os.chdir(tmpdir)
|
||||||
|
tar = tarfile.open(tarball)
|
||||||
|
_extractall(tar)
|
||||||
|
tar.close()
|
||||||
|
|
||||||
|
# going in the directory
|
||||||
|
subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0])
|
||||||
|
os.chdir(subdir)
|
||||||
|
log.warn('Now working in %s', subdir)
|
||||||
|
|
||||||
|
# installing
|
||||||
|
log.warn('Installing Distribute')
|
||||||
|
assert _python_cmd('setup.py', 'install')
|
||||||
|
finally:
|
||||||
|
os.chdir(old_wd)
|
||||||
|
|
||||||
|
|
||||||
|
def _build_egg(tarball, to_dir):
|
||||||
|
# extracting the tarball
|
||||||
|
tmpdir = tempfile.mkdtemp()
|
||||||
|
log.warn('Extracting in %s', tmpdir)
|
||||||
|
old_wd = os.getcwd()
|
||||||
|
try:
|
||||||
|
os.chdir(tmpdir)
|
||||||
|
tar = tarfile.open(tarball)
|
||||||
|
_extractall(tar)
|
||||||
|
tar.close()
|
||||||
|
|
||||||
|
# going in the directory
|
||||||
|
subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0])
|
||||||
|
os.chdir(subdir)
|
||||||
|
log.warn('Now working in %s', subdir)
|
||||||
|
|
||||||
|
# building an egg
|
||||||
|
log.warn('Building a Distribute egg in %s', to_dir)
|
||||||
|
_python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir)
|
||||||
|
|
||||||
|
# returning the result
|
||||||
|
for file in os.listdir(to_dir):
|
||||||
|
if fnmatch.fnmatch(file, 'distribute-%s*.egg' % DEFAULT_VERSION):
|
||||||
|
return os.path.join(to_dir, file)
|
||||||
|
|
||||||
|
raise IOError('Could not build the egg.')
|
||||||
|
finally:
|
||||||
|
os.chdir(old_wd)
|
||||||
|
|
||||||
|
|
||||||
|
def _do_download(version, download_base, to_dir, download_delay):
|
||||||
|
tarball = download_setuptools(version, download_base,
|
||||||
|
to_dir, download_delay)
|
||||||
|
egg = _build_egg(tarball, to_dir)
|
||||||
|
sys.path.insert(0, egg)
|
||||||
|
import setuptools
|
||||||
|
setuptools.bootstrap_install_from = egg
|
||||||
|
|
||||||
|
|
||||||
|
def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL,
|
||||||
|
to_dir=os.curdir, download_delay=15, no_fake=False):
|
||||||
|
# making sure we use the absolute path
|
||||||
|
to_dir = os.path.abspath(to_dir)
|
||||||
|
was_imported = 'pkg_resources' in sys.modules or \
|
||||||
|
'setuptools' in sys.modules
|
||||||
|
try:
|
||||||
|
try:
|
||||||
|
import pkg_resources
|
||||||
|
if not hasattr(pkg_resources, '_distribute'):
|
||||||
|
if not no_fake:
|
||||||
|
fake_setuptools()
|
||||||
|
raise ImportError
|
||||||
|
except ImportError:
|
||||||
|
return _do_download(version, download_base, to_dir, download_delay)
|
||||||
|
try:
|
||||||
|
pkg_resources.require("distribute>="+version)
|
||||||
|
return
|
||||||
|
except pkg_resources.VersionConflict:
|
||||||
|
e = sys.exc_info()[1]
|
||||||
|
if was_imported:
|
||||||
|
sys.stderr.write(
|
||||||
|
"The required version of distribute (>=%s) is not available,\n"
|
||||||
|
"and can't be installed while this script is running. Please\n"
|
||||||
|
"install a more recent version first, using\n"
|
||||||
|
"'easy_install -U distribute'."
|
||||||
|
"\n\n(Currently using %r)\n" % (version, e.args[0]))
|
||||||
|
sys.exit(2)
|
||||||
|
else:
|
||||||
|
del pkg_resources, sys.modules['pkg_resources'] # reload ok
|
||||||
|
return _do_download(version, download_base, to_dir,
|
||||||
|
download_delay)
|
||||||
|
except pkg_resources.DistributionNotFound:
|
||||||
|
return _do_download(version, download_base, to_dir,
|
||||||
|
download_delay)
|
||||||
|
finally:
|
||||||
|
_create_fake_setuptools_pkg_info(to_dir)
|
||||||
|
|
||||||
|
def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL,
|
||||||
|
to_dir=os.curdir, delay=15):
|
||||||
|
"""Download distribute from a specified location and return its filename
|
||||||
|
|
||||||
|
`version` should be a valid distribute version number that is available
|
||||||
|
as an egg for download under the `download_base` URL (which should end
|
||||||
|
with a '/'). `to_dir` is the directory where the egg will be downloaded.
|
||||||
|
`delay` is the number of seconds to pause before an actual download
|
||||||
|
attempt.
|
||||||
|
"""
|
||||||
|
# making sure we use the absolute path
|
||||||
|
to_dir = os.path.abspath(to_dir)
|
||||||
|
try:
|
||||||
|
from urllib.request import urlopen
|
||||||
|
except ImportError:
|
||||||
|
from urllib2 import urlopen
|
||||||
|
tgz_name = "distribute-%s.tar.gz" % version
|
||||||
|
url = download_base + tgz_name
|
||||||
|
saveto = os.path.join(to_dir, tgz_name)
|
||||||
|
src = dst = None
|
||||||
|
if not os.path.exists(saveto): # Avoid repeated downloads
|
||||||
|
try:
|
||||||
|
log.warn("Downloading %s", url)
|
||||||
|
src = urlopen(url)
|
||||||
|
# Read/write all in one block, so we don't create a corrupt file
|
||||||
|
# if the download is interrupted.
|
||||||
|
data = src.read()
|
||||||
|
dst = open(saveto, "wb")
|
||||||
|
dst.write(data)
|
||||||
|
finally:
|
||||||
|
if src:
|
||||||
|
src.close()
|
||||||
|
if dst:
|
||||||
|
dst.close()
|
||||||
|
return os.path.realpath(saveto)
|
||||||
|
|
||||||
|
|
||||||
|
def _patch_file(path, content):
|
||||||
|
"""Will backup the file then patch it"""
|
||||||
|
existing_content = open(path).read()
|
||||||
|
if existing_content == content:
|
||||||
|
# already patched
|
||||||
|
log.warn('Already patched.')
|
||||||
|
return False
|
||||||
|
log.warn('Patching...')
|
||||||
|
_rename_path(path)
|
||||||
|
f = open(path, 'w')
|
||||||
|
try:
|
||||||
|
f.write(content)
|
||||||
|
finally:
|
||||||
|
f.close()
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _same_content(path, content):
|
||||||
|
return open(path).read() == content
|
||||||
|
|
||||||
|
|
||||||
|
def _rename_path(path):
|
||||||
|
new_name = path + '.OLD.%s' % time.time()
|
||||||
|
log.warn('Renaming %s into %s', path, new_name)
|
||||||
|
try:
|
||||||
|
from setuptools.sandbox import DirectorySandbox
|
||||||
|
def _violation(*args):
|
||||||
|
pass
|
||||||
|
DirectorySandbox._violation = _violation
|
||||||
|
except ImportError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
os.rename(path, new_name)
|
||||||
|
return new_name
|
||||||
|
|
||||||
|
|
||||||
|
def _remove_flat_installation(placeholder):
|
||||||
|
if not os.path.isdir(placeholder):
|
||||||
|
log.warn('Unkown installation at %s', placeholder)
|
||||||
|
return False
|
||||||
|
found = False
|
||||||
|
for file in os.listdir(placeholder):
|
||||||
|
if fnmatch.fnmatch(file, 'setuptools*.egg-info'):
|
||||||
|
found = True
|
||||||
|
break
|
||||||
|
if not found:
|
||||||
|
log.warn('Could not locate setuptools*.egg-info')
|
||||||
|
return
|
||||||
|
|
||||||
|
log.warn('Removing elements out of the way...')
|
||||||
|
pkg_info = os.path.join(placeholder, file)
|
||||||
|
if os.path.isdir(pkg_info):
|
||||||
|
patched = _patch_egg_dir(pkg_info)
|
||||||
|
else:
|
||||||
|
patched = _patch_file(pkg_info, SETUPTOOLS_PKG_INFO)
|
||||||
|
|
||||||
|
if not patched:
|
||||||
|
log.warn('%s already patched.', pkg_info)
|
||||||
|
return False
|
||||||
|
# now let's move the files out of the way
|
||||||
|
for element in ('setuptools', 'pkg_resources.py', 'site.py'):
|
||||||
|
element = os.path.join(placeholder, element)
|
||||||
|
if os.path.exists(element):
|
||||||
|
_rename_path(element)
|
||||||
|
else:
|
||||||
|
log.warn('Could not find the %s element of the '
|
||||||
|
'Setuptools distribution', element)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _after_install(dist):
|
||||||
|
log.warn('After install bootstrap.')
|
||||||
|
placeholder = dist.get_command_obj('install').install_purelib
|
||||||
|
_create_fake_setuptools_pkg_info(placeholder)
|
||||||
|
|
||||||
|
def _create_fake_setuptools_pkg_info(placeholder):
|
||||||
|
if not placeholder or not os.path.exists(placeholder):
|
||||||
|
log.warn('Could not find the install location')
|
||||||
|
return
|
||||||
|
pyver = '%s.%s' % (sys.version_info[0], sys.version_info[1])
|
||||||
|
setuptools_file = 'setuptools-0.6c9-py%s.egg-info' % pyver
|
||||||
|
pkg_info = os.path.join(placeholder, setuptools_file)
|
||||||
|
if os.path.exists(pkg_info):
|
||||||
|
log.warn('%s already exists', pkg_info)
|
||||||
|
return
|
||||||
|
log.warn('Creating %s', pkg_info)
|
||||||
|
f = open(pkg_info, 'w')
|
||||||
|
try:
|
||||||
|
f.write(SETUPTOOLS_PKG_INFO)
|
||||||
|
finally:
|
||||||
|
f.close()
|
||||||
|
pth_file = os.path.join(placeholder, 'setuptools.pth')
|
||||||
|
log.warn('Creating %s', pth_file)
|
||||||
|
f = open(pth_file, 'w')
|
||||||
|
try:
|
||||||
|
f.write(os.path.join(os.curdir, setuptools_file))
|
||||||
|
finally:
|
||||||
|
f.close()
|
||||||
|
|
||||||
|
|
||||||
|
def _patch_egg_dir(path):
|
||||||
|
# let's check if it's already patched
|
||||||
|
pkg_info = os.path.join(path, 'EGG-INFO', 'PKG-INFO')
|
||||||
|
if os.path.exists(pkg_info):
|
||||||
|
if _same_content(pkg_info, SETUPTOOLS_PKG_INFO):
|
||||||
|
log.warn('%s already patched.', pkg_info)
|
||||||
|
return False
|
||||||
|
_rename_path(path)
|
||||||
|
os.mkdir(path)
|
||||||
|
os.mkdir(os.path.join(path, 'EGG-INFO'))
|
||||||
|
pkg_info = os.path.join(path, 'EGG-INFO', 'PKG-INFO')
|
||||||
|
f = open(pkg_info, 'w')
|
||||||
|
try:
|
||||||
|
f.write(SETUPTOOLS_PKG_INFO)
|
||||||
|
finally:
|
||||||
|
f.close()
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _before_install():
|
||||||
|
log.warn('Before install bootstrap.')
|
||||||
|
fake_setuptools()
|
||||||
|
|
||||||
|
|
||||||
|
def _under_prefix(location):
|
||||||
|
if 'install' not in sys.argv:
|
||||||
|
return True
|
||||||
|
args = sys.argv[sys.argv.index('install')+1:]
|
||||||
|
for index, arg in enumerate(args):
|
||||||
|
for option in ('--root', '--prefix'):
|
||||||
|
if arg.startswith('%s=' % option):
|
||||||
|
top_dir = arg.split('root=')[-1]
|
||||||
|
return location.startswith(top_dir)
|
||||||
|
elif arg == option:
|
||||||
|
if len(args) > index:
|
||||||
|
top_dir = args[index+1]
|
||||||
|
return location.startswith(top_dir)
|
||||||
|
elif option == '--user' and USER_SITE is not None:
|
||||||
|
return location.startswith(USER_SITE)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def fake_setuptools():
|
||||||
|
log.warn('Scanning installed packages')
|
||||||
|
try:
|
||||||
|
import pkg_resources
|
||||||
|
except ImportError:
|
||||||
|
# we're cool
|
||||||
|
log.warn('Setuptools or Distribute does not seem to be installed.')
|
||||||
|
return
|
||||||
|
ws = pkg_resources.working_set
|
||||||
|
try:
|
||||||
|
setuptools_dist = ws.find(pkg_resources.Requirement.parse('setuptools',
|
||||||
|
replacement=False))
|
||||||
|
except TypeError:
|
||||||
|
# old distribute API
|
||||||
|
setuptools_dist = ws.find(pkg_resources.Requirement.parse('setuptools'))
|
||||||
|
|
||||||
|
if setuptools_dist is None:
|
||||||
|
log.warn('No setuptools distribution found')
|
||||||
|
return
|
||||||
|
# detecting if it was already faked
|
||||||
|
setuptools_location = setuptools_dist.location
|
||||||
|
log.warn('Setuptools installation detected at %s', setuptools_location)
|
||||||
|
|
||||||
|
# if --root or --preix was provided, and if
|
||||||
|
# setuptools is not located in them, we don't patch it
|
||||||
|
if not _under_prefix(setuptools_location):
|
||||||
|
log.warn('Not patching, --root or --prefix is installing Distribute'
|
||||||
|
' in another location')
|
||||||
|
return
|
||||||
|
|
||||||
|
# let's see if its an egg
|
||||||
|
if not setuptools_location.endswith('.egg'):
|
||||||
|
log.warn('Non-egg installation')
|
||||||
|
res = _remove_flat_installation(setuptools_location)
|
||||||
|
if not res:
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
log.warn('Egg installation')
|
||||||
|
pkg_info = os.path.join(setuptools_location, 'EGG-INFO', 'PKG-INFO')
|
||||||
|
if (os.path.exists(pkg_info) and
|
||||||
|
_same_content(pkg_info, SETUPTOOLS_PKG_INFO)):
|
||||||
|
log.warn('Already patched.')
|
||||||
|
return
|
||||||
|
log.warn('Patching...')
|
||||||
|
# let's create a fake egg replacing setuptools one
|
||||||
|
res = _patch_egg_dir(setuptools_location)
|
||||||
|
if not res:
|
||||||
|
return
|
||||||
|
log.warn('Patched done.')
|
||||||
|
_relaunch()
|
||||||
|
|
||||||
|
|
||||||
|
def _relaunch():
|
||||||
|
log.warn('Relaunching...')
|
||||||
|
# we have to relaunch the process
|
||||||
|
args = [sys.executable] + sys.argv
|
||||||
|
sys.exit(subprocess.call(args))
|
||||||
|
|
||||||
|
|
||||||
|
def _extractall(self, path=".", members=None):
|
||||||
|
"""Extract all members from the archive to the current working
|
||||||
|
directory and set owner, modification time and permissions on
|
||||||
|
directories afterwards. `path' specifies a different directory
|
||||||
|
to extract to. `members' is optional and must be a subset of the
|
||||||
|
list returned by getmembers().
|
||||||
|
"""
|
||||||
|
import copy
|
||||||
|
import operator
|
||||||
|
from tarfile import ExtractError
|
||||||
|
directories = []
|
||||||
|
|
||||||
|
if members is None:
|
||||||
|
members = self
|
||||||
|
|
||||||
|
for tarinfo in members:
|
||||||
|
if tarinfo.isdir():
|
||||||
|
# Extract directories with a safe mode.
|
||||||
|
directories.append(tarinfo)
|
||||||
|
tarinfo = copy.copy(tarinfo)
|
||||||
|
tarinfo.mode = 448 # decimal for oct 0700
|
||||||
|
self.extract(tarinfo, path)
|
||||||
|
|
||||||
|
# Reverse sort directories.
|
||||||
|
if sys.version_info < (2, 4):
|
||||||
|
def sorter(dir1, dir2):
|
||||||
|
return cmp(dir1.name, dir2.name)
|
||||||
|
directories.sort(sorter)
|
||||||
|
directories.reverse()
|
||||||
|
else:
|
||||||
|
directories.sort(key=operator.attrgetter('name'), reverse=True)
|
||||||
|
|
||||||
|
# Set correct owner, mtime and filemode on directories.
|
||||||
|
for tarinfo in directories:
|
||||||
|
dirpath = os.path.join(path, tarinfo.name)
|
||||||
|
try:
|
||||||
|
self.chown(tarinfo, dirpath)
|
||||||
|
self.utime(tarinfo, dirpath)
|
||||||
|
self.chmod(tarinfo, dirpath)
|
||||||
|
except ExtractError:
|
||||||
|
e = sys.exc_info()[1]
|
||||||
|
if self.errorlevel > 1:
|
||||||
|
raise
|
||||||
|
else:
|
||||||
|
self._dbg(1, "tarfile: %s" % e)
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv, version=DEFAULT_VERSION):
|
||||||
|
"""Install or upgrade setuptools and EasyInstall"""
|
||||||
|
tarball = download_setuptools()
|
||||||
|
_install(tarball)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main(sys.argv[1:])
|
|
@ -1,37 +0,0 @@
|
||||||
"""
|
|
||||||
example
|
|
||||||
|
|
||||||
reading results from possibly blocking code running in sub processes.
|
|
||||||
"""
|
|
||||||
import py
|
|
||||||
|
|
||||||
NUM_PROCESSES = 5
|
|
||||||
|
|
||||||
channels = []
|
|
||||||
for i in range(NUM_PROCESSES):
|
|
||||||
gw = execnet.PopenGateway() # or use SSH or socket gateways
|
|
||||||
channel = gw.remote_exec("""
|
|
||||||
import time
|
|
||||||
secs = channel.receive()
|
|
||||||
time.sleep(secs)
|
|
||||||
channel.send("waited %d secs" % secs)
|
|
||||||
""")
|
|
||||||
channels.append(channel)
|
|
||||||
print "*** instantiated subprocess", gw
|
|
||||||
|
|
||||||
mc = execnet.MultiChannel(channels)
|
|
||||||
queue = mc.make_receive_queue()
|
|
||||||
|
|
||||||
print "***", "verifying that timeout on receiving results from blocked subprocesses works"
|
|
||||||
try:
|
|
||||||
queue.get(timeout=1.0)
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
print "*** sending subprocesses some data to have them unblock"
|
|
||||||
mc.send_each(1)
|
|
||||||
|
|
||||||
print "*** receiving results asynchronously"
|
|
||||||
for i in range(NUM_PROCESSES):
|
|
||||||
channel, result = queue.get(timeout=2.0)
|
|
||||||
print "result", channel.gateway, result
|
|
|
@ -1,31 +0,0 @@
|
||||||
"""
|
|
||||||
redirect output from remote to a local function
|
|
||||||
showcasing features of the channel object:
|
|
||||||
|
|
||||||
- sending a channel over a channel
|
|
||||||
- adapting a channel to a file object
|
|
||||||
- setting a callback for receiving channel data
|
|
||||||
|
|
||||||
"""
|
|
||||||
|
|
||||||
import py
|
|
||||||
|
|
||||||
gw = execnet.PopenGateway()
|
|
||||||
|
|
||||||
outchan = gw.remote_exec("""
|
|
||||||
import sys
|
|
||||||
outchan = channel.gateway.newchannel()
|
|
||||||
sys.stdout = outchan.makefile("w")
|
|
||||||
channel.send(outchan)
|
|
||||||
""").receive()
|
|
||||||
|
|
||||||
# note: callbacks execute in receiver thread!
|
|
||||||
def write(data):
|
|
||||||
print "received:", repr(data)
|
|
||||||
outchan.setcallback(write)
|
|
||||||
|
|
||||||
gw.remote_exec("""
|
|
||||||
print 'hello world'
|
|
||||||
print 'remote execution ends'
|
|
||||||
""").waitclose()
|
|
||||||
|
|
|
@ -1,93 +0,0 @@
|
||||||
#!/usr/bin/env python
|
|
||||||
|
|
||||||
import py
|
|
||||||
import sys, os
|
|
||||||
|
|
||||||
def usage():
|
|
||||||
arg0 = sys.argv[0]
|
|
||||||
print """%s [user@]remote-host:/repo/location localrepo [identity keyfile]""" % (arg0,)
|
|
||||||
|
|
||||||
|
|
||||||
def main(args):
|
|
||||||
remote = args[0]
|
|
||||||
localrepo = py.path.local(args[1])
|
|
||||||
if not localrepo.check(dir=1):
|
|
||||||
raise SystemExit("localrepo %s does not exist" %(localrepo,))
|
|
||||||
if len(args) == 3:
|
|
||||||
keyfile = py.path.local(args[2])
|
|
||||||
else:
|
|
||||||
keyfile = None
|
|
||||||
remote_host, path = remote.split(':', 1)
|
|
||||||
print "ssh-connecting to", remote_host
|
|
||||||
gw = getgateway(remote_host, keyfile)
|
|
||||||
|
|
||||||
local_rev = get_svn_youngest(localrepo)
|
|
||||||
|
|
||||||
# local protocol
|
|
||||||
# 1. client sends rev/repo -> server
|
|
||||||
# 2. server checks for newer revisions and sends dumps
|
|
||||||
# 3. client receives dumps, updates local repo
|
|
||||||
# 4. client goes back to step 1
|
|
||||||
c = gw.remote_exec("""
|
|
||||||
import py
|
|
||||||
import os
|
|
||||||
remote_rev, repopath = channel.receive()
|
|
||||||
while 1:
|
|
||||||
rev = py.process.cmdexec('svnlook youngest "%s"' % repopath)
|
|
||||||
rev = int(rev)
|
|
||||||
if rev > remote_rev:
|
|
||||||
revrange = (remote_rev+1, rev)
|
|
||||||
dumpchannel = channel.gateway.newchannel()
|
|
||||||
channel.send(revrange)
|
|
||||||
channel.send(dumpchannel)
|
|
||||||
|
|
||||||
f = os.popen(
|
|
||||||
"svnadmin dump -q --incremental -r %s:%s %s"
|
|
||||||
% (revrange[0], revrange[1], repopath), 'r')
|
|
||||||
try:
|
|
||||||
while 1:
|
|
||||||
s = f.read(8192)
|
|
||||||
if not s:
|
|
||||||
raise EOFError
|
|
||||||
dumpchannel.send(s)
|
|
||||||
except EOFError:
|
|
||||||
dumpchannel.close()
|
|
||||||
remote_rev = rev
|
|
||||||
else:
|
|
||||||
# using svn-hook instead would be nice here
|
|
||||||
py.std.time.sleep(30)
|
|
||||||
""")
|
|
||||||
|
|
||||||
c.send((local_rev, path))
|
|
||||||
print "checking revisions from %d in %s" %(local_rev, remote)
|
|
||||||
while 1:
|
|
||||||
revstart, revend = c.receive()
|
|
||||||
dumpchannel = c.receive()
|
|
||||||
|
|
||||||
print "receiving revisions", revstart, "-", revend, "replaying..."
|
|
||||||
svn_load(localrepo, dumpchannel)
|
|
||||||
print "current revision", revend
|
|
||||||
|
|
||||||
def svn_load(repo, dumpchannel):
|
|
||||||
f = os.popen("svnadmin load -q %s" %(repo, ), "w")
|
|
||||||
for x in dumpchannel:
|
|
||||||
sys.stdout.write(".")
|
|
||||||
sys.stdout.flush()
|
|
||||||
f.write(x)
|
|
||||||
print >>sys.stdout
|
|
||||||
f.close()
|
|
||||||
|
|
||||||
def get_svn_youngest(repo):
|
|
||||||
rev = py.process.cmdexec('svnlook youngest "%s"' % repo)
|
|
||||||
return int(rev)
|
|
||||||
|
|
||||||
def getgateway(host, keyfile=None):
|
|
||||||
return execnet.SshGateway(host, identity=keyfile)
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
if len(sys.argv) < 3:
|
|
||||||
usage()
|
|
||||||
raise SystemExit(1)
|
|
||||||
|
|
||||||
main(sys.argv[1:])
|
|
||||||
|
|
|
@ -1,139 +0,0 @@
|
||||||
"""
|
|
||||||
sysinfo.py [host1] [host2] [options]
|
|
||||||
|
|
||||||
obtain system info from remote machine.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import py
|
|
||||||
import sys
|
|
||||||
|
|
||||||
|
|
||||||
parser = py.std.optparse.OptionParser(usage=__doc__)
|
|
||||||
parser.add_option("-f", "--sshconfig", action="store", dest="ssh_config", default=None,
|
|
||||||
help="use given ssh config file, and add info all contained hosts for getting info")
|
|
||||||
parser.add_option("-i", "--ignore", action="store", dest="ignores", default=None,
|
|
||||||
help="ignore hosts (useful if the list of hostnames come from a file list)")
|
|
||||||
|
|
||||||
def parsehosts(path):
|
|
||||||
path = py.path.local(path)
|
|
||||||
l = []
|
|
||||||
rex = py.std.re.compile(r'Host\s*(\S+)')
|
|
||||||
for line in path.readlines():
|
|
||||||
m = rex.match(line)
|
|
||||||
if m is not None:
|
|
||||||
sshname, = m.groups()
|
|
||||||
l.append(sshname)
|
|
||||||
return l
|
|
||||||
|
|
||||||
class RemoteInfo:
|
|
||||||
def __init__(self, gateway):
|
|
||||||
self.gw = gateway
|
|
||||||
self._cache = {}
|
|
||||||
|
|
||||||
def exreceive(self, execstring):
|
|
||||||
if execstring not in self._cache:
|
|
||||||
channel = self.gw.remote_exec(execstring)
|
|
||||||
self._cache[execstring] = channel.receive()
|
|
||||||
return self._cache[execstring]
|
|
||||||
|
|
||||||
def getmodattr(self, modpath):
|
|
||||||
module = modpath.split(".")[0]
|
|
||||||
return self.exreceive("""
|
|
||||||
import %s
|
|
||||||
channel.send(%s)
|
|
||||||
""" %(module, modpath))
|
|
||||||
|
|
||||||
def islinux(self):
|
|
||||||
return self.getmodattr('sys.platform').find("linux") != -1
|
|
||||||
|
|
||||||
def getfqdn(self):
|
|
||||||
return self.exreceive("""
|
|
||||||
import socket
|
|
||||||
channel.send(socket.getfqdn())
|
|
||||||
""")
|
|
||||||
|
|
||||||
def getmemswap(self):
|
|
||||||
if self.islinux():
|
|
||||||
return self.exreceive("""
|
|
||||||
import commands, re
|
|
||||||
out = commands.getoutput("free")
|
|
||||||
mem = re.search(r"Mem:\s+(\S*)", out).group(1)
|
|
||||||
swap = re.search(r"Swap:\s+(\S*)", out).group(1)
|
|
||||||
channel.send((mem, swap))
|
|
||||||
""")
|
|
||||||
|
|
||||||
def getcpuinfo(self):
|
|
||||||
if self.islinux():
|
|
||||||
return self.exreceive("""
|
|
||||||
# a hyperthreaded cpu core only counts as 1, although it
|
|
||||||
# is present as 2 in /proc/cpuinfo. Counting it as 2 is
|
|
||||||
# misleading because it is *by far* not as efficient as
|
|
||||||
# two independent cores.
|
|
||||||
cpus = {}
|
|
||||||
cpuinfo = {}
|
|
||||||
f = open("/proc/cpuinfo")
|
|
||||||
lines = f.readlines()
|
|
||||||
f.close()
|
|
||||||
for line in lines + ['']:
|
|
||||||
if line.strip():
|
|
||||||
key, value = line.split(":", 1)
|
|
||||||
cpuinfo[key.strip()] = value.strip()
|
|
||||||
else:
|
|
||||||
corekey = (cpuinfo.get("physical id"),
|
|
||||||
cpuinfo.get("core id"))
|
|
||||||
cpus[corekey] = 1
|
|
||||||
numcpus = len(cpus)
|
|
||||||
model = cpuinfo.get("model name")
|
|
||||||
channel.send((numcpus, model))
|
|
||||||
""")
|
|
||||||
|
|
||||||
def debug(*args):
|
|
||||||
print >>sys.stderr, " ".join(map(str, args))
|
|
||||||
def error(*args):
|
|
||||||
debug("ERROR", args[0] + ":", *args[1:])
|
|
||||||
|
|
||||||
def getinfo(sshname, ssh_config=None, loginfo=sys.stdout):
|
|
||||||
debug("connecting to", sshname)
|
|
||||||
try:
|
|
||||||
gw = execnet.SshGateway(sshname, ssh_config=ssh_config)
|
|
||||||
except IOError:
|
|
||||||
error("could not get sshagteway", sshname)
|
|
||||||
else:
|
|
||||||
ri = RemoteInfo(gw)
|
|
||||||
#print "%s info:" % sshname
|
|
||||||
prefix = sshname.upper() + " "
|
|
||||||
print >>loginfo, prefix, "fqdn:", ri.getfqdn()
|
|
||||||
for attr in (
|
|
||||||
"sys.platform",
|
|
||||||
"sys.version_info",
|
|
||||||
):
|
|
||||||
loginfo.write("%s %s: " %(prefix, attr,))
|
|
||||||
loginfo.flush()
|
|
||||||
value = ri.getmodattr(attr)
|
|
||||||
loginfo.write(str(value))
|
|
||||||
loginfo.write("\n")
|
|
||||||
loginfo.flush()
|
|
||||||
memswap = ri.getmemswap()
|
|
||||||
if memswap:
|
|
||||||
mem,swap = memswap
|
|
||||||
print >>loginfo, prefix, "Memory:", mem, "Swap:", swap
|
|
||||||
cpuinfo = ri.getcpuinfo()
|
|
||||||
if cpuinfo:
|
|
||||||
numcpu, model = cpuinfo
|
|
||||||
print >>loginfo, prefix, "number of cpus:", numcpu
|
|
||||||
print >>loginfo, prefix, "cpu model", model
|
|
||||||
return ri
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
options, args = parser.parse_args()
|
|
||||||
hosts = list(args)
|
|
||||||
ssh_config = options.ssh_config
|
|
||||||
if ssh_config:
|
|
||||||
hosts.extend(parsehosts(ssh_config))
|
|
||||||
ignores = options.ignores or ()
|
|
||||||
if ignores:
|
|
||||||
ignores = ignores.split(",")
|
|
||||||
for host in hosts:
|
|
||||||
if host not in ignores:
|
|
||||||
getinfo(host, ssh_config=ssh_config)
|
|
||||||
|
|
29
setup.py
29
setup.py
|
@ -1,15 +1,19 @@
|
||||||
"""py lib / py.test setup.py file"""
|
"""py lib / py.test setup.py file"""
|
||||||
import os, sys
|
import os, sys
|
||||||
|
if sys.version_info >= (3,0):
|
||||||
|
from distribute_setup import use_setuptools
|
||||||
|
use_setuptools()
|
||||||
from setuptools import setup
|
from setuptools import setup
|
||||||
long_description = """
|
|
||||||
|
|
||||||
|
long_description = """
|
||||||
py.test and pylib: rapid testing and development utils
|
py.test and pylib: rapid testing and development utils
|
||||||
|
|
||||||
- `py.test`_: cross-project testing tool with many advanced features
|
- `py.test`_: cross-project testing tool with many advanced features
|
||||||
- `py.path`_: path abstractions over local and subversion files
|
- `py.path`_: path abstractions over local and subversion files
|
||||||
- `py.code`_: dynamic code compile and traceback printing support
|
- `py.code`_: dynamic code compile and traceback printing support
|
||||||
|
|
||||||
Compatibility: Linux, Win32, OSX, Python versions 2.4 through to 3.1.
|
Platforms: Linux, Win32, OSX
|
||||||
|
Interpreters: Python versions 2.4 through to 3.1, Jython 2.5.1.
|
||||||
For questions please check out http://pylib.org/contact.html
|
For questions please check out http://pylib.org/contact.html
|
||||||
|
|
||||||
.. _`py.test`: http://pylib.org/test.html
|
.. _`py.test`: http://pylib.org/test.html
|
||||||
|
@ -17,7 +21,6 @@ For questions please check out http://pylib.org/contact.html
|
||||||
.. _`py.code`: http://pylib.org/code.html
|
.. _`py.code`: http://pylib.org/code.html
|
||||||
|
|
||||||
(c) Holger Krekel and others, 2009
|
(c) Holger Krekel and others, 2009
|
||||||
|
|
||||||
"""
|
"""
|
||||||
trunk = None
|
trunk = None
|
||||||
def main():
|
def main():
|
||||||
|
@ -58,30 +61,12 @@ def main():
|
||||||
'_py.io',
|
'_py.io',
|
||||||
'_py.log',
|
'_py.log',
|
||||||
'_py.path',
|
'_py.path',
|
||||||
'_py.path.gateway',
|
|
||||||
'_py.process',
|
'_py.process',
|
||||||
'_py.test',
|
'_py.test',
|
||||||
'_py.test.dist',
|
'_py.test.dist',
|
||||||
'_py.test.looponfail',
|
'_py.test.looponfail',
|
||||||
'_py.test.plugin',],
|
'_py.test.plugin',],
|
||||||
package_data={'py': ['bin/_findpy.py',
|
zip_safe=False,
|
||||||
'bin/env.cmd',
|
|
||||||
'bin/env.py',
|
|
||||||
'bin/py.cleanup',
|
|
||||||
'bin/py.convert_unittest',
|
|
||||||
'bin/py.countloc',
|
|
||||||
'bin/py.lookup',
|
|
||||||
'bin/py.svnwcrevert',
|
|
||||||
'bin/py.test',
|
|
||||||
'bin/py.which',
|
|
||||||
'bin/win32/py.cleanup.cmd',
|
|
||||||
'bin/win32/py.convert_unittest.cmd',
|
|
||||||
'bin/win32/py.countloc.cmd',
|
|
||||||
'bin/win32/py.lookup.cmd',
|
|
||||||
'bin/win32/py.svnwcrevert.cmd',
|
|
||||||
'bin/win32/py.test.cmd',
|
|
||||||
'bin/win32/py.which.cmd',],
|
|
||||||
zip_safe=True,
|
|
||||||
)
|
)
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|
Loading…
Reference in New Issue