some release preps and cleanups
- update setup.py for release - use distributes_setup on python3 - remove unncessary package_data - remove execnet example --HG-- branch : trunk
This commit is contained in:
		
							parent
							
								
									92d482069c
								
							
						
					
					
						commit
						270ac2bc0d
					
				|  | @ -1,16 +1,14 @@ | ||||||
| include CHANGELOG | include CHANGELOG | ||||||
| include README.txt | include README.txt | ||||||
| include setup.py | include setup.py | ||||||
|  | include distribute_setup.py | ||||||
| include LICENSE  | include LICENSE  | ||||||
| include py/LICENSE  |  | ||||||
| include py/path/testing/repotest.dump |  | ||||||
| include py/rest/rest.sty.template |  | ||||||
| graft doc | graft doc | ||||||
| graft contrib | graft contrib | ||||||
| graft example | graft example | ||||||
| graft py/bin  | graft bin  | ||||||
| graft py/rest/testing/data |  | ||||||
| exclude *.orig  | exclude *.orig  | ||||||
| exclude *.rej  | exclude *.rej  | ||||||
|  | exclude .hginore | ||||||
| prune .svn  | prune .svn  | ||||||
| prune .hg  | prune .hg  | ||||||
|  |  | ||||||
|  | @ -1,6 +1,7 @@ | ||||||
| import py | import py | ||||||
| import subprocess | import subprocess | ||||||
| import os | import os | ||||||
|  | import execnet | ||||||
| 
 | 
 | ||||||
| 
 | 
 | ||||||
| # | # | ||||||
|  |  | ||||||
|  | @ -0,0 +1,456 @@ | ||||||
|  | #!python | ||||||
|  | """Bootstrap distribute installation | ||||||
|  | 
 | ||||||
|  | If you want to use setuptools in your package's setup.py, just include this | ||||||
|  | file in the same directory with it, and add this to the top of your setup.py:: | ||||||
|  | 
 | ||||||
|  |     from distribute_setup import use_setuptools | ||||||
|  |     use_setuptools() | ||||||
|  | 
 | ||||||
|  | If you want to require a specific version of setuptools, set a download | ||||||
|  | mirror, or use an alternate download directory, you can do so by supplying | ||||||
|  | the appropriate options to ``use_setuptools()``. | ||||||
|  | 
 | ||||||
|  | This file can also be run as a script to install or upgrade setuptools. | ||||||
|  | """ | ||||||
|  | import os | ||||||
|  | import sys | ||||||
|  | import time | ||||||
|  | import fnmatch | ||||||
|  | import tempfile | ||||||
|  | import tarfile | ||||||
|  | from distutils import log | ||||||
|  | 
 | ||||||
|  | try: | ||||||
|  |     from site import USER_SITE | ||||||
|  | except ImportError: | ||||||
|  |     USER_SITE = None | ||||||
|  | 
 | ||||||
|  | try: | ||||||
|  |     import subprocess | ||||||
|  | 
 | ||||||
|  |     def _python_cmd(*args): | ||||||
|  |         args = (sys.executable,) + args | ||||||
|  |         return subprocess.call(args) == 0 | ||||||
|  | 
 | ||||||
|  | except ImportError: | ||||||
|  |     # will be used for python 2.3 | ||||||
|  |     def _python_cmd(*args): | ||||||
|  |         args = (sys.executable,) + args | ||||||
|  |         # quoting arguments if windows | ||||||
|  |         if sys.platform == 'win32': | ||||||
|  |             def quote(arg): | ||||||
|  |                 if ' ' in arg: | ||||||
|  |                     return '"%s"' % arg | ||||||
|  |                 return arg | ||||||
|  |             args = [quote(arg) for arg in args] | ||||||
|  |         return os.spawnl(os.P_WAIT, sys.executable, *args) == 0 | ||||||
|  | 
 | ||||||
|  | DEFAULT_VERSION = "0.6.6" | ||||||
|  | DEFAULT_URL = "http://pypi.python.org/packages/source/d/distribute/" | ||||||
|  | SETUPTOOLS_PKG_INFO = """\ | ||||||
|  | Metadata-Version: 1.0 | ||||||
|  | Name: setuptools | ||||||
|  | Version: 0.6c9 | ||||||
|  | Summary: xxxx | ||||||
|  | Home-page: xxx | ||||||
|  | Author: xxx | ||||||
|  | Author-email: xxx | ||||||
|  | License: xxx | ||||||
|  | Description: xxx | ||||||
|  | """ | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  | def _install(tarball): | ||||||
|  |     # extracting the tarball | ||||||
|  |     tmpdir = tempfile.mkdtemp() | ||||||
|  |     log.warn('Extracting in %s', tmpdir) | ||||||
|  |     old_wd = os.getcwd() | ||||||
|  |     try: | ||||||
|  |         os.chdir(tmpdir) | ||||||
|  |         tar = tarfile.open(tarball) | ||||||
|  |         _extractall(tar) | ||||||
|  |         tar.close() | ||||||
|  | 
 | ||||||
|  |         # going in the directory | ||||||
|  |         subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) | ||||||
|  |         os.chdir(subdir) | ||||||
|  |         log.warn('Now working in %s', subdir) | ||||||
|  | 
 | ||||||
|  |         # installing | ||||||
|  |         log.warn('Installing Distribute') | ||||||
|  |         assert _python_cmd('setup.py', 'install') | ||||||
|  |     finally: | ||||||
|  |         os.chdir(old_wd) | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  | def _build_egg(tarball, to_dir): | ||||||
|  |     # extracting the tarball | ||||||
|  |     tmpdir = tempfile.mkdtemp() | ||||||
|  |     log.warn('Extracting in %s', tmpdir) | ||||||
|  |     old_wd = os.getcwd() | ||||||
|  |     try: | ||||||
|  |         os.chdir(tmpdir) | ||||||
|  |         tar = tarfile.open(tarball) | ||||||
|  |         _extractall(tar) | ||||||
|  |         tar.close() | ||||||
|  | 
 | ||||||
|  |         # going in the directory | ||||||
|  |         subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) | ||||||
|  |         os.chdir(subdir) | ||||||
|  |         log.warn('Now working in %s', subdir) | ||||||
|  | 
 | ||||||
|  |         # building an egg | ||||||
|  |         log.warn('Building a Distribute egg in %s', to_dir) | ||||||
|  |         _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir) | ||||||
|  | 
 | ||||||
|  |         # returning the result | ||||||
|  |         for file in os.listdir(to_dir): | ||||||
|  |             if fnmatch.fnmatch(file, 'distribute-%s*.egg' % DEFAULT_VERSION): | ||||||
|  |                 return os.path.join(to_dir, file) | ||||||
|  | 
 | ||||||
|  |         raise IOError('Could not build the egg.') | ||||||
|  |     finally: | ||||||
|  |         os.chdir(old_wd) | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  | def _do_download(version, download_base, to_dir, download_delay): | ||||||
|  |     tarball = download_setuptools(version, download_base, | ||||||
|  |                                   to_dir, download_delay) | ||||||
|  |     egg = _build_egg(tarball, to_dir) | ||||||
|  |     sys.path.insert(0, egg) | ||||||
|  |     import setuptools | ||||||
|  |     setuptools.bootstrap_install_from = egg | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  | def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, | ||||||
|  |                    to_dir=os.curdir, download_delay=15, no_fake=False): | ||||||
|  |     # making sure we use the absolute path | ||||||
|  |     to_dir = os.path.abspath(to_dir) | ||||||
|  |     was_imported = 'pkg_resources' in sys.modules or \ | ||||||
|  |         'setuptools' in sys.modules | ||||||
|  |     try: | ||||||
|  |         try: | ||||||
|  |             import pkg_resources | ||||||
|  |             if not hasattr(pkg_resources, '_distribute'): | ||||||
|  |                 if not no_fake: | ||||||
|  |                     fake_setuptools() | ||||||
|  |                 raise ImportError | ||||||
|  |         except ImportError: | ||||||
|  |             return _do_download(version, download_base, to_dir, download_delay) | ||||||
|  |         try: | ||||||
|  |             pkg_resources.require("distribute>="+version) | ||||||
|  |             return | ||||||
|  |         except pkg_resources.VersionConflict: | ||||||
|  |             e = sys.exc_info()[1] | ||||||
|  |             if was_imported: | ||||||
|  |                 sys.stderr.write( | ||||||
|  |                 "The required version of distribute (>=%s) is not available,\n" | ||||||
|  |                 "and can't be installed while this script is running. Please\n" | ||||||
|  |                 "install a more recent version first, using\n" | ||||||
|  |                 "'easy_install -U distribute'." | ||||||
|  |                 "\n\n(Currently using %r)\n" % (version, e.args[0])) | ||||||
|  |                 sys.exit(2) | ||||||
|  |             else: | ||||||
|  |                 del pkg_resources, sys.modules['pkg_resources']    # reload ok | ||||||
|  |                 return _do_download(version, download_base, to_dir, | ||||||
|  |                                     download_delay) | ||||||
|  |         except pkg_resources.DistributionNotFound: | ||||||
|  |             return _do_download(version, download_base, to_dir, | ||||||
|  |                                 download_delay) | ||||||
|  |     finally: | ||||||
|  |         _create_fake_setuptools_pkg_info(to_dir) | ||||||
|  | 
 | ||||||
|  | def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, | ||||||
|  |                         to_dir=os.curdir, delay=15): | ||||||
|  |     """Download distribute from a specified location and return its filename | ||||||
|  | 
 | ||||||
|  |     `version` should be a valid distribute version number that is available | ||||||
|  |     as an egg for download under the `download_base` URL (which should end | ||||||
|  |     with a '/'). `to_dir` is the directory where the egg will be downloaded. | ||||||
|  |     `delay` is the number of seconds to pause before an actual download | ||||||
|  |     attempt. | ||||||
|  |     """ | ||||||
|  |     # making sure we use the absolute path | ||||||
|  |     to_dir = os.path.abspath(to_dir) | ||||||
|  |     try: | ||||||
|  |         from urllib.request import urlopen | ||||||
|  |     except ImportError: | ||||||
|  |         from urllib2 import urlopen | ||||||
|  |     tgz_name = "distribute-%s.tar.gz" % version | ||||||
|  |     url = download_base + tgz_name | ||||||
|  |     saveto = os.path.join(to_dir, tgz_name) | ||||||
|  |     src = dst = None | ||||||
|  |     if not os.path.exists(saveto):  # Avoid repeated downloads | ||||||
|  |         try: | ||||||
|  |             log.warn("Downloading %s", url) | ||||||
|  |             src = urlopen(url) | ||||||
|  |             # Read/write all in one block, so we don't create a corrupt file | ||||||
|  |             # if the download is interrupted. | ||||||
|  |             data = src.read() | ||||||
|  |             dst = open(saveto, "wb") | ||||||
|  |             dst.write(data) | ||||||
|  |         finally: | ||||||
|  |             if src: | ||||||
|  |                 src.close() | ||||||
|  |             if dst: | ||||||
|  |                 dst.close() | ||||||
|  |     return os.path.realpath(saveto) | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  | def _patch_file(path, content): | ||||||
|  |     """Will backup the file then patch it""" | ||||||
|  |     existing_content = open(path).read() | ||||||
|  |     if existing_content == content: | ||||||
|  |         # already patched | ||||||
|  |         log.warn('Already patched.') | ||||||
|  |         return False | ||||||
|  |     log.warn('Patching...') | ||||||
|  |     _rename_path(path) | ||||||
|  |     f = open(path, 'w') | ||||||
|  |     try: | ||||||
|  |         f.write(content) | ||||||
|  |     finally: | ||||||
|  |         f.close() | ||||||
|  |     return True | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  | def _same_content(path, content): | ||||||
|  |     return open(path).read() == content | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  | def _rename_path(path): | ||||||
|  |     new_name = path + '.OLD.%s' % time.time() | ||||||
|  |     log.warn('Renaming %s into %s', path, new_name) | ||||||
|  |     try: | ||||||
|  |         from setuptools.sandbox import DirectorySandbox | ||||||
|  |         def _violation(*args): | ||||||
|  |             pass | ||||||
|  |         DirectorySandbox._violation = _violation | ||||||
|  |     except ImportError: | ||||||
|  |         pass | ||||||
|  | 
 | ||||||
|  |     os.rename(path, new_name) | ||||||
|  |     return new_name | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  | def _remove_flat_installation(placeholder): | ||||||
|  |     if not os.path.isdir(placeholder): | ||||||
|  |         log.warn('Unkown installation at %s', placeholder) | ||||||
|  |         return False | ||||||
|  |     found = False | ||||||
|  |     for file in os.listdir(placeholder): | ||||||
|  |         if fnmatch.fnmatch(file, 'setuptools*.egg-info'): | ||||||
|  |             found = True | ||||||
|  |             break | ||||||
|  |     if not found: | ||||||
|  |         log.warn('Could not locate setuptools*.egg-info') | ||||||
|  |         return | ||||||
|  | 
 | ||||||
|  |     log.warn('Removing elements out of the way...') | ||||||
|  |     pkg_info = os.path.join(placeholder, file) | ||||||
|  |     if os.path.isdir(pkg_info): | ||||||
|  |         patched = _patch_egg_dir(pkg_info) | ||||||
|  |     else: | ||||||
|  |         patched = _patch_file(pkg_info, SETUPTOOLS_PKG_INFO) | ||||||
|  | 
 | ||||||
|  |     if not patched: | ||||||
|  |         log.warn('%s already patched.', pkg_info) | ||||||
|  |         return False | ||||||
|  |     # now let's move the files out of the way | ||||||
|  |     for element in ('setuptools', 'pkg_resources.py', 'site.py'): | ||||||
|  |         element = os.path.join(placeholder, element) | ||||||
|  |         if os.path.exists(element): | ||||||
|  |             _rename_path(element) | ||||||
|  |         else: | ||||||
|  |             log.warn('Could not find the %s element of the ' | ||||||
|  |                      'Setuptools distribution', element) | ||||||
|  |     return True | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  | def _after_install(dist): | ||||||
|  |     log.warn('After install bootstrap.') | ||||||
|  |     placeholder = dist.get_command_obj('install').install_purelib | ||||||
|  |     _create_fake_setuptools_pkg_info(placeholder) | ||||||
|  | 
 | ||||||
|  | def _create_fake_setuptools_pkg_info(placeholder): | ||||||
|  |     if not placeholder or not os.path.exists(placeholder): | ||||||
|  |         log.warn('Could not find the install location') | ||||||
|  |         return | ||||||
|  |     pyver = '%s.%s' % (sys.version_info[0], sys.version_info[1]) | ||||||
|  |     setuptools_file = 'setuptools-0.6c9-py%s.egg-info' % pyver | ||||||
|  |     pkg_info = os.path.join(placeholder, setuptools_file) | ||||||
|  |     if os.path.exists(pkg_info): | ||||||
|  |         log.warn('%s already exists', pkg_info) | ||||||
|  |         return | ||||||
|  |     log.warn('Creating %s', pkg_info) | ||||||
|  |     f = open(pkg_info, 'w') | ||||||
|  |     try: | ||||||
|  |         f.write(SETUPTOOLS_PKG_INFO) | ||||||
|  |     finally: | ||||||
|  |         f.close() | ||||||
|  |     pth_file = os.path.join(placeholder, 'setuptools.pth') | ||||||
|  |     log.warn('Creating %s', pth_file) | ||||||
|  |     f = open(pth_file, 'w') | ||||||
|  |     try: | ||||||
|  |         f.write(os.path.join(os.curdir, setuptools_file)) | ||||||
|  |     finally: | ||||||
|  |         f.close() | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  | def _patch_egg_dir(path): | ||||||
|  |     # let's check if it's already patched | ||||||
|  |     pkg_info = os.path.join(path, 'EGG-INFO', 'PKG-INFO') | ||||||
|  |     if os.path.exists(pkg_info): | ||||||
|  |         if _same_content(pkg_info, SETUPTOOLS_PKG_INFO): | ||||||
|  |             log.warn('%s already patched.', pkg_info) | ||||||
|  |             return False | ||||||
|  |     _rename_path(path) | ||||||
|  |     os.mkdir(path) | ||||||
|  |     os.mkdir(os.path.join(path, 'EGG-INFO')) | ||||||
|  |     pkg_info = os.path.join(path, 'EGG-INFO', 'PKG-INFO') | ||||||
|  |     f = open(pkg_info, 'w') | ||||||
|  |     try: | ||||||
|  |         f.write(SETUPTOOLS_PKG_INFO) | ||||||
|  |     finally: | ||||||
|  |         f.close() | ||||||
|  |     return True | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  | def _before_install(): | ||||||
|  |     log.warn('Before install bootstrap.') | ||||||
|  |     fake_setuptools() | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  | def _under_prefix(location): | ||||||
|  |     if 'install' not in sys.argv: | ||||||
|  |         return True | ||||||
|  |     args = sys.argv[sys.argv.index('install')+1:] | ||||||
|  |     for index, arg in enumerate(args): | ||||||
|  |         for option in ('--root', '--prefix'): | ||||||
|  |             if arg.startswith('%s=' % option): | ||||||
|  |                 top_dir = arg.split('root=')[-1] | ||||||
|  |                 return location.startswith(top_dir) | ||||||
|  |             elif arg == option: | ||||||
|  |                 if len(args) > index: | ||||||
|  |                     top_dir = args[index+1] | ||||||
|  |                     return location.startswith(top_dir) | ||||||
|  |             elif option == '--user' and USER_SITE is not None: | ||||||
|  |                 return location.startswith(USER_SITE) | ||||||
|  |     return True | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  | def fake_setuptools(): | ||||||
|  |     log.warn('Scanning installed packages') | ||||||
|  |     try: | ||||||
|  |         import pkg_resources | ||||||
|  |     except ImportError: | ||||||
|  |         # we're cool | ||||||
|  |         log.warn('Setuptools or Distribute does not seem to be installed.') | ||||||
|  |         return | ||||||
|  |     ws = pkg_resources.working_set | ||||||
|  |     try: | ||||||
|  |         setuptools_dist = ws.find(pkg_resources.Requirement.parse('setuptools', | ||||||
|  |                                   replacement=False)) | ||||||
|  |     except TypeError: | ||||||
|  |         # old distribute API | ||||||
|  |         setuptools_dist = ws.find(pkg_resources.Requirement.parse('setuptools')) | ||||||
|  | 
 | ||||||
|  |     if setuptools_dist is None: | ||||||
|  |         log.warn('No setuptools distribution found') | ||||||
|  |         return | ||||||
|  |     # detecting if it was already faked | ||||||
|  |     setuptools_location = setuptools_dist.location | ||||||
|  |     log.warn('Setuptools installation detected at %s', setuptools_location) | ||||||
|  | 
 | ||||||
|  |     # if --root or --preix was provided, and if | ||||||
|  |     # setuptools is not located in them, we don't patch it | ||||||
|  |     if not _under_prefix(setuptools_location): | ||||||
|  |         log.warn('Not patching, --root or --prefix is installing Distribute' | ||||||
|  |                  ' in another location') | ||||||
|  |         return | ||||||
|  | 
 | ||||||
|  |     # let's see if its an egg | ||||||
|  |     if not setuptools_location.endswith('.egg'): | ||||||
|  |         log.warn('Non-egg installation') | ||||||
|  |         res = _remove_flat_installation(setuptools_location) | ||||||
|  |         if not res: | ||||||
|  |             return | ||||||
|  |     else: | ||||||
|  |         log.warn('Egg installation') | ||||||
|  |         pkg_info = os.path.join(setuptools_location, 'EGG-INFO', 'PKG-INFO') | ||||||
|  |         if (os.path.exists(pkg_info) and | ||||||
|  |             _same_content(pkg_info, SETUPTOOLS_PKG_INFO)): | ||||||
|  |             log.warn('Already patched.') | ||||||
|  |             return | ||||||
|  |         log.warn('Patching...') | ||||||
|  |         # let's create a fake egg replacing setuptools one | ||||||
|  |         res = _patch_egg_dir(setuptools_location) | ||||||
|  |         if not res: | ||||||
|  |             return | ||||||
|  |     log.warn('Patched done.') | ||||||
|  |     _relaunch() | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  | def _relaunch(): | ||||||
|  |     log.warn('Relaunching...') | ||||||
|  |     # we have to relaunch the process | ||||||
|  |     args = [sys.executable] + sys.argv | ||||||
|  |     sys.exit(subprocess.call(args)) | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  | def _extractall(self, path=".", members=None): | ||||||
|  |     """Extract all members from the archive to the current working | ||||||
|  |        directory and set owner, modification time and permissions on | ||||||
|  |        directories afterwards. `path' specifies a different directory | ||||||
|  |        to extract to. `members' is optional and must be a subset of the | ||||||
|  |        list returned by getmembers(). | ||||||
|  |     """ | ||||||
|  |     import copy | ||||||
|  |     import operator | ||||||
|  |     from tarfile import ExtractError | ||||||
|  |     directories = [] | ||||||
|  | 
 | ||||||
|  |     if members is None: | ||||||
|  |         members = self | ||||||
|  | 
 | ||||||
|  |     for tarinfo in members: | ||||||
|  |         if tarinfo.isdir(): | ||||||
|  |             # Extract directories with a safe mode. | ||||||
|  |             directories.append(tarinfo) | ||||||
|  |             tarinfo = copy.copy(tarinfo) | ||||||
|  |             tarinfo.mode = 448 # decimal for oct 0700 | ||||||
|  |         self.extract(tarinfo, path) | ||||||
|  | 
 | ||||||
|  |     # Reverse sort directories. | ||||||
|  |     if sys.version_info < (2, 4): | ||||||
|  |         def sorter(dir1, dir2): | ||||||
|  |             return cmp(dir1.name, dir2.name) | ||||||
|  |         directories.sort(sorter) | ||||||
|  |         directories.reverse() | ||||||
|  |     else: | ||||||
|  |         directories.sort(key=operator.attrgetter('name'), reverse=True) | ||||||
|  | 
 | ||||||
|  |     # Set correct owner, mtime and filemode on directories. | ||||||
|  |     for tarinfo in directories: | ||||||
|  |         dirpath = os.path.join(path, tarinfo.name) | ||||||
|  |         try: | ||||||
|  |             self.chown(tarinfo, dirpath) | ||||||
|  |             self.utime(tarinfo, dirpath) | ||||||
|  |             self.chmod(tarinfo, dirpath) | ||||||
|  |         except ExtractError: | ||||||
|  |             e = sys.exc_info()[1] | ||||||
|  |             if self.errorlevel > 1: | ||||||
|  |                 raise | ||||||
|  |             else: | ||||||
|  |                 self._dbg(1, "tarfile: %s" % e) | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  | def main(argv, version=DEFAULT_VERSION): | ||||||
|  |     """Install or upgrade setuptools and EasyInstall""" | ||||||
|  |     tarball = download_setuptools() | ||||||
|  |     _install(tarball) | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  | if __name__ == '__main__': | ||||||
|  |     main(sys.argv[1:]) | ||||||
|  | @ -1,37 +0,0 @@ | ||||||
| """ |  | ||||||
| example |  | ||||||
| 
 |  | ||||||
| reading results from possibly blocking code running in sub processes.  |  | ||||||
| """ |  | ||||||
| import py |  | ||||||
| 
 |  | ||||||
| NUM_PROCESSES = 5 |  | ||||||
| 
 |  | ||||||
| channels = [] |  | ||||||
| for i in range(NUM_PROCESSES): |  | ||||||
|     gw = execnet.PopenGateway() # or use SSH or socket gateways  |  | ||||||
|     channel = gw.remote_exec(""" |  | ||||||
|         import time |  | ||||||
|         secs = channel.receive() |  | ||||||
|         time.sleep(secs) |  | ||||||
|         channel.send("waited %d secs" % secs) |  | ||||||
|     """) |  | ||||||
|     channels.append(channel) |  | ||||||
|     print "*** instantiated subprocess", gw |  | ||||||
| 
 |  | ||||||
| mc = execnet.MultiChannel(channels) |  | ||||||
| queue = mc.make_receive_queue() |  | ||||||
| 
 |  | ||||||
| print "***", "verifying that timeout on receiving results from blocked subprocesses works" |  | ||||||
| try: |  | ||||||
|     queue.get(timeout=1.0)  |  | ||||||
| except Exception: |  | ||||||
|     pass |  | ||||||
| 
 |  | ||||||
| print "*** sending subprocesses some data to have them unblock" |  | ||||||
| mc.send_each(1)  |  | ||||||
| 
 |  | ||||||
| print "*** receiving results asynchronously" |  | ||||||
| for i in range(NUM_PROCESSES): |  | ||||||
|     channel, result = queue.get(timeout=2.0) |  | ||||||
|     print "result", channel.gateway, result |  | ||||||
|  | @ -1,31 +0,0 @@ | ||||||
| """ |  | ||||||
| redirect output from remote to a local function  |  | ||||||
| showcasing features of the channel object: |  | ||||||
| 
 |  | ||||||
| - sending a channel over a channel  |  | ||||||
| - adapting a channel to a file object  |  | ||||||
| - setting a callback for receiving channel data  |  | ||||||
| 
 |  | ||||||
| """ |  | ||||||
| 
 |  | ||||||
| import py |  | ||||||
| 
 |  | ||||||
| gw = execnet.PopenGateway() |  | ||||||
| 
 |  | ||||||
| outchan = gw.remote_exec(""" |  | ||||||
|     import sys |  | ||||||
|     outchan = channel.gateway.newchannel() |  | ||||||
|     sys.stdout = outchan.makefile("w") |  | ||||||
|     channel.send(outchan)  |  | ||||||
| """).receive() |  | ||||||
| 
 |  | ||||||
| # note: callbacks execute in receiver thread!  |  | ||||||
| def write(data): |  | ||||||
|     print "received:", repr(data) |  | ||||||
| outchan.setcallback(write) |  | ||||||
| 
 |  | ||||||
| gw.remote_exec(""" |  | ||||||
|     print 'hello world' |  | ||||||
|     print 'remote execution ends' |  | ||||||
| """).waitclose() |  | ||||||
| 
 |  | ||||||
|  | @ -1,93 +0,0 @@ | ||||||
| #!/usr/bin/env python  |  | ||||||
| 
 |  | ||||||
| import py |  | ||||||
| import sys, os |  | ||||||
| 
 |  | ||||||
| def usage(): |  | ||||||
|     arg0 = sys.argv[0] |  | ||||||
|     print """%s [user@]remote-host:/repo/location localrepo [identity keyfile]""" % (arg0,) |  | ||||||
| 
 |  | ||||||
| 
 |  | ||||||
| def main(args): |  | ||||||
|     remote = args[0] |  | ||||||
|     localrepo = py.path.local(args[1]) |  | ||||||
|     if not localrepo.check(dir=1): |  | ||||||
|         raise SystemExit("localrepo %s does not exist" %(localrepo,)) |  | ||||||
|     if len(args) == 3: |  | ||||||
|         keyfile = py.path.local(args[2]) |  | ||||||
|     else: |  | ||||||
|         keyfile = None |  | ||||||
|     remote_host, path = remote.split(':', 1) |  | ||||||
|     print "ssh-connecting to", remote_host  |  | ||||||
|     gw = getgateway(remote_host, keyfile) |  | ||||||
| 
 |  | ||||||
|     local_rev = get_svn_youngest(localrepo) |  | ||||||
| 
 |  | ||||||
|     # local protocol  |  | ||||||
|     # 1. client sends rev/repo -> server  |  | ||||||
|     # 2. server checks for newer revisions and sends dumps  |  | ||||||
|     # 3. client receives dumps, updates local repo  |  | ||||||
|     # 4. client goes back to step 1 |  | ||||||
|     c = gw.remote_exec(""" |  | ||||||
|         import py |  | ||||||
|         import os |  | ||||||
|         remote_rev, repopath = channel.receive() |  | ||||||
|         while 1:  |  | ||||||
|             rev = py.process.cmdexec('svnlook youngest "%s"' % repopath)  |  | ||||||
|             rev = int(rev) |  | ||||||
|             if rev > remote_rev: |  | ||||||
|                 revrange = (remote_rev+1, rev) |  | ||||||
|                 dumpchannel = channel.gateway.newchannel() |  | ||||||
|                 channel.send(revrange) |  | ||||||
|                 channel.send(dumpchannel) |  | ||||||
| 
 |  | ||||||
|                 f = os.popen( |  | ||||||
|                         "svnadmin dump -q --incremental -r %s:%s %s"  |  | ||||||
|                          % (revrange[0], revrange[1], repopath), 'r') |  | ||||||
|                 try: |  | ||||||
|                     while 1: |  | ||||||
|                         s = f.read(8192) |  | ||||||
|                         if not s: |  | ||||||
|                             raise EOFError |  | ||||||
|                         dumpchannel.send(s) |  | ||||||
|                 except EOFError: |  | ||||||
|                     dumpchannel.close() |  | ||||||
|                 remote_rev = rev  |  | ||||||
|             else: |  | ||||||
|                 # using svn-hook instead would be nice here |  | ||||||
|                 py.std.time.sleep(30) |  | ||||||
|     """) |  | ||||||
| 
 |  | ||||||
|     c.send((local_rev, path)) |  | ||||||
|     print "checking revisions from %d in %s" %(local_rev, remote) |  | ||||||
|     while 1:  |  | ||||||
|         revstart, revend = c.receive() |  | ||||||
|         dumpchannel = c.receive()  |  | ||||||
| 
 |  | ||||||
|         print "receiving revisions", revstart, "-", revend, "replaying..." |  | ||||||
|         svn_load(localrepo, dumpchannel) |  | ||||||
|         print "current revision", revend  |  | ||||||
| 
 |  | ||||||
| def svn_load(repo, dumpchannel): |  | ||||||
|     f = os.popen("svnadmin load -q %s" %(repo, ), "w") |  | ||||||
|     for x in dumpchannel: |  | ||||||
|         sys.stdout.write(".") |  | ||||||
|         sys.stdout.flush() |  | ||||||
|         f.write(x) |  | ||||||
|     print >>sys.stdout |  | ||||||
|     f.close()  |  | ||||||
| 
 |  | ||||||
| def get_svn_youngest(repo): |  | ||||||
|     rev = py.process.cmdexec('svnlook youngest "%s"' % repo)  |  | ||||||
|     return int(rev) |  | ||||||
| 
 |  | ||||||
| def getgateway(host, keyfile=None): |  | ||||||
|     return execnet.SshGateway(host, identity=keyfile) |  | ||||||
| 
 |  | ||||||
| if __name__ == '__main__': |  | ||||||
|     if len(sys.argv) < 3: |  | ||||||
|         usage() |  | ||||||
|         raise SystemExit(1) |  | ||||||
| 
 |  | ||||||
|     main(sys.argv[1:]) |  | ||||||
| 
 |  | ||||||
|  | @ -1,139 +0,0 @@ | ||||||
| """ |  | ||||||
| sysinfo.py [host1] [host2] [options] |  | ||||||
| 
 |  | ||||||
| obtain system info from remote machine.  |  | ||||||
| """ |  | ||||||
| 
 |  | ||||||
| import py |  | ||||||
| import sys |  | ||||||
| 
 |  | ||||||
| 
 |  | ||||||
| parser = py.std.optparse.OptionParser(usage=__doc__) |  | ||||||
| parser.add_option("-f", "--sshconfig", action="store", dest="ssh_config", default=None, |  | ||||||
|                   help="use given ssh config file, and add info all contained hosts for getting info") |  | ||||||
| parser.add_option("-i", "--ignore", action="store", dest="ignores", default=None, |  | ||||||
|                   help="ignore hosts (useful if the list of hostnames come from a file list)") |  | ||||||
| 
 |  | ||||||
| def parsehosts(path): |  | ||||||
|     path = py.path.local(path) |  | ||||||
|     l = [] |  | ||||||
|     rex = py.std.re.compile(r'Host\s*(\S+)') |  | ||||||
|     for line in path.readlines(): |  | ||||||
|         m = rex.match(line) |  | ||||||
|         if m is not None: |  | ||||||
|             sshname, = m.groups() |  | ||||||
|             l.append(sshname) |  | ||||||
|     return l |  | ||||||
| 
 |  | ||||||
| class RemoteInfo: |  | ||||||
|     def __init__(self, gateway): |  | ||||||
|         self.gw = gateway |  | ||||||
|         self._cache = {} |  | ||||||
| 
 |  | ||||||
|     def exreceive(self, execstring): |  | ||||||
|         if execstring not in self._cache: |  | ||||||
|             channel = self.gw.remote_exec(execstring) |  | ||||||
|             self._cache[execstring] = channel.receive() |  | ||||||
|         return self._cache[execstring] |  | ||||||
| 
 |  | ||||||
|     def getmodattr(self, modpath): |  | ||||||
|         module = modpath.split(".")[0] |  | ||||||
|         return self.exreceive(""" |  | ||||||
|             import %s |  | ||||||
|             channel.send(%s) |  | ||||||
|         """ %(module, modpath)) |  | ||||||
| 
 |  | ||||||
|     def islinux(self): |  | ||||||
|         return self.getmodattr('sys.platform').find("linux") != -1 |  | ||||||
| 
 |  | ||||||
|     def getfqdn(self): |  | ||||||
|         return self.exreceive(""" |  | ||||||
|             import socket |  | ||||||
|             channel.send(socket.getfqdn()) |  | ||||||
|         """) |  | ||||||
| 
 |  | ||||||
|     def getmemswap(self): |  | ||||||
|         if self.islinux(): |  | ||||||
|             return self.exreceive("""  |  | ||||||
|             import commands, re |  | ||||||
|             out = commands.getoutput("free") |  | ||||||
|             mem = re.search(r"Mem:\s+(\S*)", out).group(1) |  | ||||||
|             swap = re.search(r"Swap:\s+(\S*)", out).group(1) |  | ||||||
|             channel.send((mem, swap)) |  | ||||||
|             """) |  | ||||||
| 
 |  | ||||||
|     def getcpuinfo(self): |  | ||||||
|         if self.islinux(): |  | ||||||
|             return self.exreceive(""" |  | ||||||
|                 # a hyperthreaded cpu core only counts as 1, although it |  | ||||||
|                 # is present as 2 in /proc/cpuinfo.  Counting it as 2 is |  | ||||||
|                 # misleading because it is *by far* not as efficient as |  | ||||||
|                 # two independent cores. |  | ||||||
|                 cpus = {} |  | ||||||
|                 cpuinfo = {} |  | ||||||
|                 f = open("/proc/cpuinfo") |  | ||||||
|                 lines = f.readlines() |  | ||||||
|                 f.close() |  | ||||||
|                 for line in lines + ['']: |  | ||||||
|                     if line.strip(): |  | ||||||
|                         key, value = line.split(":", 1) |  | ||||||
|                         cpuinfo[key.strip()] = value.strip() |  | ||||||
|                     else: |  | ||||||
|                         corekey = (cpuinfo.get("physical id"), |  | ||||||
|                                    cpuinfo.get("core id")) |  | ||||||
|                         cpus[corekey] = 1 |  | ||||||
|                 numcpus = len(cpus) |  | ||||||
|                 model = cpuinfo.get("model name") |  | ||||||
|                 channel.send((numcpus, model)) |  | ||||||
|             """) |  | ||||||
| 
 |  | ||||||
| def debug(*args): |  | ||||||
|     print >>sys.stderr, " ".join(map(str, args)) |  | ||||||
| def error(*args): |  | ||||||
|     debug("ERROR", args[0] + ":", *args[1:]) |  | ||||||
| 
 |  | ||||||
| def getinfo(sshname, ssh_config=None, loginfo=sys.stdout): |  | ||||||
|     debug("connecting to", sshname) |  | ||||||
|     try: |  | ||||||
|         gw = execnet.SshGateway(sshname, ssh_config=ssh_config) |  | ||||||
|     except IOError: |  | ||||||
|         error("could not get sshagteway", sshname) |  | ||||||
|     else: |  | ||||||
|         ri = RemoteInfo(gw) |  | ||||||
|         #print "%s info:" % sshname |  | ||||||
|         prefix = sshname.upper() + " " |  | ||||||
|         print >>loginfo, prefix, "fqdn:", ri.getfqdn() |  | ||||||
|         for attr in ( |  | ||||||
|             "sys.platform",  |  | ||||||
|             "sys.version_info",  |  | ||||||
|         ): |  | ||||||
|             loginfo.write("%s %s: " %(prefix, attr,)) |  | ||||||
|             loginfo.flush() |  | ||||||
|             value = ri.getmodattr(attr) |  | ||||||
|             loginfo.write(str(value)) |  | ||||||
|             loginfo.write("\n") |  | ||||||
|             loginfo.flush() |  | ||||||
|         memswap = ri.getmemswap() |  | ||||||
|         if memswap: |  | ||||||
|             mem,swap = memswap |  | ||||||
|             print >>loginfo, prefix, "Memory:", mem, "Swap:", swap  |  | ||||||
|         cpuinfo = ri.getcpuinfo() |  | ||||||
|         if cpuinfo: |  | ||||||
|             numcpu, model = cpuinfo |  | ||||||
|             print >>loginfo, prefix, "number of cpus:",  numcpu |  | ||||||
|             print >>loginfo, prefix, "cpu model", model  |  | ||||||
|         return ri |  | ||||||
|              |  | ||||||
| if __name__ == '__main__': |  | ||||||
|     options, args = parser.parse_args() |  | ||||||
|     hosts = list(args) |  | ||||||
|     ssh_config = options.ssh_config |  | ||||||
|     if ssh_config: |  | ||||||
|         hosts.extend(parsehosts(ssh_config)) |  | ||||||
|     ignores = options.ignores or () |  | ||||||
|     if ignores: |  | ||||||
|         ignores = ignores.split(",") |  | ||||||
|     for host in hosts: |  | ||||||
|         if host not in ignores: |  | ||||||
|             getinfo(host, ssh_config=ssh_config) |  | ||||||
|          |  | ||||||
							
								
								
									
										29
									
								
								setup.py
								
								
								
								
							
							
						
						
									
										29
									
								
								setup.py
								
								
								
								
							|  | @ -1,15 +1,19 @@ | ||||||
| """py lib / py.test setup.py file""" | """py lib / py.test setup.py file""" | ||||||
| import os, sys | import os, sys | ||||||
|  | if sys.version_info >= (3,0): | ||||||
|  |     from distribute_setup import use_setuptools | ||||||
|  |     use_setuptools() | ||||||
| from setuptools import setup | from setuptools import setup | ||||||
| long_description = """ |  | ||||||
| 
 | 
 | ||||||
|  | long_description = """ | ||||||
| py.test and pylib: rapid testing and development utils | py.test and pylib: rapid testing and development utils | ||||||
| 
 | 
 | ||||||
| - `py.test`_: cross-project testing tool with many advanced features | - `py.test`_: cross-project testing tool with many advanced features | ||||||
| - `py.path`_: path abstractions over local and subversion files | - `py.path`_: path abstractions over local and subversion files | ||||||
| - `py.code`_: dynamic code compile and traceback printing support | - `py.code`_: dynamic code compile and traceback printing support | ||||||
| 
 | 
 | ||||||
| Compatibility: Linux, Win32, OSX, Python versions 2.4 through to 3.1. | Platforms: Linux, Win32, OSX | ||||||
|  | Interpreters: Python versions 2.4 through to 3.1, Jython 2.5.1.  | ||||||
| For questions please check out http://pylib.org/contact.html | For questions please check out http://pylib.org/contact.html | ||||||
| 
 | 
 | ||||||
| .. _`py.test`: http://pylib.org/test.html | .. _`py.test`: http://pylib.org/test.html | ||||||
|  | @ -17,7 +21,6 @@ For questions please check out http://pylib.org/contact.html | ||||||
| .. _`py.code`: http://pylib.org/code.html | .. _`py.code`: http://pylib.org/code.html | ||||||
| 
 | 
 | ||||||
| (c) Holger Krekel and others, 2009 | (c) Holger Krekel and others, 2009 | ||||||
| 
 |  | ||||||
| """ | """ | ||||||
| trunk = None | trunk = None | ||||||
| def main(): | def main(): | ||||||
|  | @ -58,30 +61,12 @@ def main(): | ||||||
|                   '_py.io', |                   '_py.io', | ||||||
|                   '_py.log', |                   '_py.log', | ||||||
|                   '_py.path', |                   '_py.path', | ||||||
|                   '_py.path.gateway', |  | ||||||
|                   '_py.process', |                   '_py.process', | ||||||
|                   '_py.test', |                   '_py.test', | ||||||
|                   '_py.test.dist', |                   '_py.test.dist', | ||||||
|                   '_py.test.looponfail', |                   '_py.test.looponfail', | ||||||
|                   '_py.test.plugin',], |                   '_py.test.plugin',], | ||||||
|         package_data={'py': ['bin/_findpy.py', |         zip_safe=False, | ||||||
|                              'bin/env.cmd', |  | ||||||
|                              'bin/env.py', |  | ||||||
|                              'bin/py.cleanup', |  | ||||||
|                              'bin/py.convert_unittest', |  | ||||||
|                              'bin/py.countloc', |  | ||||||
|                              'bin/py.lookup', |  | ||||||
|                              'bin/py.svnwcrevert', |  | ||||||
|                              'bin/py.test', |  | ||||||
|                              'bin/py.which', |  | ||||||
|                              'bin/win32/py.cleanup.cmd', |  | ||||||
|                              'bin/win32/py.convert_unittest.cmd', |  | ||||||
|                              'bin/win32/py.countloc.cmd', |  | ||||||
|                              'bin/win32/py.lookup.cmd', |  | ||||||
|                              'bin/win32/py.svnwcrevert.cmd', |  | ||||||
|                              'bin/win32/py.test.cmd', |  | ||||||
|                              'bin/win32/py.which.cmd',], |  | ||||||
|         zip_safe=True, |  | ||||||
|     ) |     ) | ||||||
| 
 | 
 | ||||||
| if __name__ == '__main__': | if __name__ == '__main__': | ||||||
|  |  | ||||||
		Loading…
	
		Reference in New Issue