[svn r37264] create the new development trunk

--HG--
branch : trunk

py/apigen/apigen.js

@@ -0,0 +1,11 @@
+function loadloc() {
+    /* load iframe content using # part of the url */
+    var loc = document.location.toString();
+    if (loc.indexOf('#') == -1) {
+        return;
+    };
+    var chunks = loc.split('#');
+    var anchor = chunks[chunks.length - 1];
+    var iframe = document.getElementsByTagName('iframe')[0];
+    iframe.src = anchor;
+};

py/apigen/apigen.py

@@ -0,0 +1,55 @@
+""" run 'py.test --apigen=<this script>' to get documentation exported
+
+    exports to /tmp/output by default, set the environment variable
+    'APIGEN_TARGET' to override
+"""
+
+import os
+import py
+from py.__.apigen import htmlgen
+from py.__.apigen import linker
+from py.__.apigen import project
+
+def import_pkgdir(pkgdir):
+    if pkgdir.check(dir=True):
+        return pkgdir.join('__init__.py').getpymodule()
+    else:
+        # XXX not sure if this is ever used normally...
+        return pkgdir.getpymodule()
+
+def get_documentable_items(pkgdir):
+    rootmod = import_pkgdir(pkgdir)
+    if hasattr(rootmod, '__package__'):
+        return rootmod
+    # XXX fix non-initpkg situations(?)
+    return {}
+
+def build(pkgdir, dsa):
+    l = linker.Linker()
+    proj = project.Project()
+
+    if 'APIGEN_TARGET' in os.environ:
+        targetdir = py.path.local(os.environ['APIGEN_TARGET'])
+    else:
+        targetdir = pkgdir.dirpath().join('apigen')
+    targetdir.ensure(dir=True)
+
+    all_names = dsa._get_names(filter=lambda x, y: True)
+    namespace_tree = htmlgen.create_namespace_tree(all_names)
+    apb = htmlgen.ApiPageBuilder(targetdir, l, dsa, pkgdir)
+    spb = htmlgen.SourcePageBuilder(targetdir, l, pkgdir)
+
+    ns_data = apb.prepare_namespace_pages(namespace_tree)
+    class_names = dsa.get_class_names()
+    class_data, method_data = apb.prepare_class_pages(namespace_tree,
+                                                      class_names)
+    function_names = dsa.get_function_names()
+    func_data = apb.prepare_function_pages(namespace_tree, function_names)
+    source_data = spb.prepare_pages(pkgdir)
+
+    apb.build_namespace_pages(ns_data, proj)
+    apb.build_class_pages(class_data, proj)
+    apb.build_method_pages(method_data, proj)
+    apb.build_function_pages(func_data, proj)
+    spb.build_pages(source_data, proj, pkgdir)
+

py/apigen/conftest.py

@@ -0,0 +1,10 @@
+import py
+
+Option = py.test.config.Option
+option = py.test.config.addoptions("apigen test options",
+        Option('', '--webcheck',
+               action="store_true", dest="webcheck", default=False,
+               help="run XHTML validation tests"
+        ),
+)
+

py/apigen/htmlgen.py

@@ -0,0 +1,567 @@
+import py
+import inspect
+from py.__.apigen.layout import LayoutPage
+from py.__.apigen.source import browser as source_browser
+from py.__.apigen.source import html as source_html
+from py.__.apigen.tracer.description import is_private
+from py.__.apigen.rest.genrest import split_of_last_part
+from py.__.apigen.linker import relpath
+
+sorted = py.builtin.sorted
+html = py.xml.html
+raw = py.xml.raw
+
+# HTML related stuff
+class H(html):
+    class Description(html.div):
+        style = html.Style(margin_left='15em')
+    
+    class NamespaceDescription(Description):
+        pass
+
+    class NamespaceItem(html.div):
+        pass
+
+    class NamespaceDef(html.h1):
+        pass
+
+    class ClassDescription(Description):
+        pass
+
+    class ClassDef(html.h1):
+        pass
+
+    class MethodDescription(Description):
+        pass
+
+    class MethodDef(html.h2):
+        pass
+
+    class FunctionDescription(Description):
+        pass
+
+    class FunctionDef(html.h2):
+        pass
+
+    class ParameterDescription(html.div):
+        pass
+
+    class Docstring(html.div):
+        style = html.Style(white_space='pre', min_height='3em')
+
+    class Navigation(html.div):
+        style = html.Style(min_height='99%', float='left', margin_top='1.2em',
+                           overflow='auto', width='15em', white_space='nowrap')
+
+    class NavigationItem(html.div):
+        pass
+
+    class BaseDescription(html.a):
+        pass
+
+    class SourceDef(html.div):
+        pass
+
+    class NonPythonSource(html.pre):
+        style = html.Style(margin_left='15em')
+
+    class DirList(html.div):
+        style = html.Style(margin_left='15em')
+
+    class DirListItem(html.div):
+        pass
+
+    class ValueDescList(html.ul):
+        def __init__(self, *args, **kwargs):
+            super(H.ValueDescList, self).__init__(*args, **kwargs)
+
+    class ValueDescItem(html.li):
+        pass
+
+def get_param_htmldesc(linker, func):
+    """ get the html for the parameters of a function """
+    import inspect
+    # XXX copy and modify formatargspec to produce html
+    return H.em(inspect.formatargspec(*inspect.getargspec(func)))
+
+def build_navitem_html(linker, name, linkid, indent, selected):
+    href = linker.get_lazyhref(linkid)
+    navitem = H.NavigationItem((indent * 2 * u'\xa0'), H.a(name, href=href))
+    if selected:
+        navitem.attr.class_ = 'selected'
+    return navitem
+
+# some helper functionality
+def source_dirs_files(fspath):
+    """ returns a tuple (dirs, files) for fspath
+
+        dirs are all the subdirs, files are the files which are interesting
+        in building source documentation for a Python code tree (basically all
+        normal files excluding .pyc and .pyo ones)
+
+        all files and dirs that have a name starting with . are considered
+        hidden
+    """
+    dirs = []
+    files = []
+    for child in fspath.listdir():
+        if child.basename.startswith('.'):
+            continue
+        if child.check(dir=True):
+            dirs.append(child)
+        elif child.check(file=True):
+            if child.ext in ['.pyc', '.pyo']:
+                continue
+            files.append(child)
+    return sorted(dirs), sorted(files)
+
+def create_namespace_tree(dotted_names):
+    """ creates a tree (in dict form) from a set of dotted names
+    """
+    ret = {}
+    for dn in dotted_names:
+        path = dn.split('.')
+        for i in xrange(len(path)):
+            ns = '.'.join(path[:i])
+            itempath = '.'.join(path[:i + 1])
+            if ns not in ret:
+                ret[ns] = []
+            if itempath not in ret[ns]:
+                ret[ns].append(itempath)
+    return ret
+
+def wrap_page(project, title, contentel, navel, outputpath, stylesheeturl):
+    page = LayoutPage(project, title, nav=navel, encoding='UTF-8',
+                      stylesheeturl=stylesheeturl)
+    page.set_content(contentel)
+    here = py.magic.autopath().dirpath()
+    style = here.join('style.css').read()
+    outputpath.join('style.css').write(style)
+    return page
+
+# the PageBuilder classes take care of producing the docs (using the stuff
+# above)
+class AbstractPageBuilder(object):
+    def write_page(self, title, reltargetpath, project, tag, nav):
+        targetpath = self.base.join(reltargetpath)
+        stylesheeturl = relpath('%s/' % (targetpath.dirpath(),),
+                                self.base.join('style.css').strpath)
+        page = wrap_page(project, title,
+                         tag, nav, self.base, stylesheeturl)
+        content = self.linker.call_withbase(reltargetpath, page.unicode)
+        targetpath.ensure()
+        targetpath.write(content.encode("utf8"))
+
+class SourcePageBuilder(AbstractPageBuilder):
+    """ builds the html for a source docs page """
+    def __init__(self, base, linker, projroot):
+        self.base = base
+        self.linker = linker
+        self.projroot = projroot
+    
+    def build_navigation(self, fspath):
+        nav = H.Navigation()
+        relpath = fspath.relto(self.projroot)
+        path = relpath.split('/')
+        indent = 0
+        # build links to parents
+        for i in xrange(len(path)):
+            dirpath = '/'.join(path[:i])
+            abspath = self.projroot.join(dirpath).strpath
+            if i == 0:
+                text = 'root'
+            else:
+                text = path[i-1]
+            nav.append(build_navitem_html(self.linker, text, abspath,
+                                          indent, False))
+            indent += 1
+        # build siblings or children and self
+        if fspath.check(dir=True):
+            # we're a dir, build ourselves and our children
+            dirpath = fspath
+            nav.append(build_navitem_html(self.linker, dirpath.basename,
+                                          dirpath.strpath, indent, True))
+            indent += 1
+        elif fspath.strpath == self.projroot.strpath:
+            dirpath = fspath
+        else:
+            # we're a file, build our parent's children only
+            dirpath = fspath.dirpath()
+        diritems, fileitems = source_dirs_files(dirpath)
+        for dir in diritems:
+            nav.append(build_navitem_html(self.linker, dir.basename,
+                                          dir.strpath, indent, False))
+        for file in fileitems:
+            selected = (fspath.check(file=True) and
+                        file.basename == fspath.basename)
+            nav.append(build_navitem_html(self.linker, file.basename,
+                                          file.strpath, indent, selected))
+        return nav
+
+    re = py.std.re
+    _reg_body = re.compile(r'<body[^>]*>(.*)</body>', re.S)
+    def build_python_page(self, fspath):
+        mod = source_browser.parse_path(fspath)
+        # XXX let's cheat a bit here... there should be a different function 
+        # using the linker, and returning a proper py.xml.html element,
+        # at some point
+        html = source_html.create_html(mod)
+        snippet = self._reg_body.search(html).group(1)
+        tag = H.SourceDef(raw(snippet))
+        nav = self.build_navigation(fspath)
+        return tag, nav
+
+    def build_dir_page(self, fspath):
+        tag = H.DirList()
+        dirs, files = source_dirs_files(fspath)
+        tag.append(H.h2('directories'))
+        for path in dirs:
+            tag.append(H.DirListItem(H.a(path.basename,
+                            href=self.linker.get_lazyhref(str(path)))))
+        tag.append(H.h2('files'))
+        for path in files:
+            tag.append(H.DirListItem(H.a(path.basename,
+                            href=self.linker.get_lazyhref(str(path)))))
+        nav = self.build_navigation(fspath)
+        return tag, nav
+
+    def build_nonpython_page(self, fspath):
+        try:
+            tag = H.NonPythonSource(unicode(fspath.read(), 'utf-8'))
+        except UnicodeError:
+            tag = H.NonPythonSource('no source available (binary file?)')
+        nav = self.build_navigation(fspath)
+        return tag, nav
+
+    def prepare_pages(self, base):
+        passed = []
+        for fspath in [base] + list(base.visit()):
+            if fspath.ext in ['.pyc', '.pyo']:
+                continue
+            relfspath = fspath.relto(base)
+            if relfspath.find('/.') > -1:
+                # skip hidden dirs and files
+                continue
+            elif fspath.check(dir=True):
+                if relfspath != '':
+                    relfspath += '/'
+                reloutputpath = 'source/%sindex.html' % (relfspath,)
+            else:
+                reloutputpath = "source/%s.html" % (relfspath,)
+            outputpath = self.base.join(reloutputpath)
+            self.linker.set_link(str(fspath), reloutputpath)
+            passed.append((fspath, outputpath))
+        return passed
+
+    def build_pages(self, data, project, base):
+        """ build syntax-colored source views """
+        for fspath, outputpath in data:
+            if fspath.check(ext='.py'):
+                try:
+                    tag, nav = self.build_python_page(fspath)
+                except (KeyboardInterrupt, SystemError):
+                    raise
+                except: # XXX strange stuff going wrong at times... need to fix
+                    exc, e, tb = py.std.sys.exc_info()
+                    print '%s - %s' % (exc, e)
+                    print
+                    print ''.join(py.std.traceback.format_tb(tb))
+                    print '-' * 79
+                    del tb
+                    tag, nav = self.build_nonpython_page(fspath)
+            elif fspath.check(dir=True):
+                tag, nav = self.build_dir_page(fspath)
+            else:
+                tag, nav = self.build_nonpython_page(fspath)
+            title = 'sources for %s' % (fspath.basename,)
+            reltargetpath = outputpath.relto(self.base)
+            self.write_page(title, reltargetpath, project, tag, nav)
+
+class ApiPageBuilder(AbstractPageBuilder):
+    """ builds the html for an api docs page """
+    def __init__(self, base, linker, dsa, projroot):
+        self.base = base
+        self.linker = linker
+        self.dsa = dsa
+        self.projroot = projroot
+        self.projpath = py.path.local(projroot)
+        
+    def build_callable_view(self, dotted_name):
+        """ build the html for a class method """
+        # XXX we may want to have seperate
+        func = self.dsa.get_obj(dotted_name)
+        docstring = func.__doc__
+        localname = func.__name__
+        argdesc = get_param_htmldesc(self.linker, func)
+        valuedesc = self.build_callable_value_description(dotted_name)
+
+        sourcefile = inspect.getsourcefile(func)
+        callable_source = self.dsa.get_function_source(dotted_name)
+        is_in_pkg = py.path.local(sourcefile).relto(self.projpath)
+        # i assume they're both either available or unavailable(XXX ?)
+        if is_in_pkg and sourcefile and callable_source:
+            csource = H.div(H.br(),
+                            H.a('origin: %s' % (sourcefile,),
+                                href=self.linker.get_lazyhref(sourcefile)),
+                            H.br(),
+                            H.SourceDef(H.pre(callable_source)))
+        elif not is_in_pkg and sourcefile and callable_source:
+            csource = H.div(H.br(),
+                            H.em('origin: %s' % (sourcefile,)),
+                            H.br(),
+                            H.SourceDef(H.pre(callable_source)))
+        else:
+            csource = H.SourceDef('could not get source file')
+
+        snippet = H.FunctionDescription(
+            H.FunctionDef(localname, argdesc),
+            valuedesc,
+            H.Docstring(docstring or H.em('no docstring available')),
+            csource,
+        )
+        
+        return snippet
+
+    def build_class_view(self, dotted_name):
+        """ build the html for a class """
+        cls = self.dsa.get_obj(dotted_name)
+        # XXX is this a safe check?
+        try:
+            sourcefile = inspect.getsourcefile(cls)
+        except TypeError:
+            sourcelink = 'builtin file, no source available'
+        else:
+            if sourcefile is None:
+                sourcelink = H.div('no source available')
+            else:
+                if sourcefile[-1] in ['o', 'c']:
+                    sourcefile = sourcefile[:-1]
+                sourcelink = H.div(H.a('view source',
+                    href=self.linker.get_lazyhref(sourcefile)))
+
+        docstring = cls.__doc__
+        methods = self.dsa.get_class_methods(dotted_name)
+        basehtml = []
+        bases = self.dsa.get_possible_base_classes(dotted_name)
+        for base in bases:
+            try:
+                obj = self.dsa.get_obj(base.name)
+            except KeyError:
+                basehtml.append(base.name)
+            else:
+                href = self.linker.get_lazyhref(base.name)
+                basehtml.append(H.BaseDescription(base.name, href=href))
+            basehtml.append(',')
+        if basehtml:
+            basehtml.pop()
+        basehtml.append('):')
+        if not hasattr(cls, '__name__'):
+            clsname = 'instance of %s' % (cls.__class__.__name__,)
+        else:
+            clsname = cls.__name__
+        snippet = H.ClassDescription(
+            # XXX bases HTML
+            H.ClassDef('%s(' % (clsname,), *basehtml),
+            H.Docstring(docstring or H.em('no docstring available')),
+            sourcelink,
+        )
+        if methods:
+            snippet.append(H.h2('methods:'))
+            for method in methods:
+                snippet += self.build_callable_view('%s.%s' % (dotted_name,
+                                                    method))
+        # XXX properties
+        return snippet
+
+    def build_namespace_view(self, namespace_dotted_name, item_dotted_names):
+        """ build the html for a namespace (module) """
+        try:
+            obj = self.dsa.get_obj(namespace_dotted_name)
+        except KeyError:
+            docstring = None
+        else:
+            docstring = obj.__doc__
+        snippet = H.NamespaceDescription(
+            H.NamespaceDef(namespace_dotted_name),
+            H.Docstring(docstring or H.em('no docstring available'))
+        )
+        for dotted_name in item_dotted_names:
+            itemname = dotted_name.split('.')[-1]
+            if is_private(itemname):
+                continue
+            snippet.append(
+                H.NamespaceItem(
+                    H.a(itemname,
+                        href=self.linker.get_lazyhref(dotted_name)
+                    )
+                )
+            )
+        return snippet
+
+    def prepare_class_pages(self, namespace_tree, classes_dotted_names):
+        passed = []
+        methodsdata = []
+        for dotted_name in classes_dotted_names:
+            parent_dotted_name, _ = split_of_last_part(dotted_name)
+            try:
+                sibling_dotted_names = namespace_tree[parent_dotted_name]
+            except KeyError:
+                # no siblings (built-in module or sth)
+                sibling_dotted_names = []
+            tag = self.build_class_view(dotted_name)
+            nav = self.build_navigation(parent_dotted_name,
+                                        sibling_dotted_names, dotted_name)
+            reltargetpath = "api/%s.html" % (dotted_name,)
+            self.linker.set_link(dotted_name, reltargetpath)
+            passed.append((dotted_name, tag, nav, reltargetpath))
+            method_dotted_names = ['%s.%s' % (dotted_name, method_name) for
+                                   method_name in
+                                   self.dsa.get_class_methods(dotted_name)]
+            methodsdata += self.prepare_method_pages(namespace_tree,
+                                                     method_dotted_names)
+        return passed, methodsdata
+        
+    def build_class_pages(self, data, project):
+        """ build the full api pages for a set of classes """
+        for dotted_name, tag, nav, reltargetpath in data:
+            title = 'api documentation for %s' % (dotted_name,)
+            self.write_page(title, reltargetpath, project, tag, nav)
+
+    def prepare_method_pages(self, namespace_tree, method_dotted_names):
+        # XXX note that even though these pages are still built, there's no nav
+        # pointing to them anymore...
+        passed = []
+        for dotted_name in method_dotted_names:
+            parent_dotted_name, _ = split_of_last_part(dotted_name)
+            module_dotted_name, _ = split_of_last_part(parent_dotted_name)
+            sibling_dotted_names = namespace_tree[module_dotted_name]
+            tag = self.build_callable_view(dotted_name)
+            nav = self.build_navigation(parent_dotted_name,
+                                        sibling_dotted_names, dotted_name)
+            reltargetpath = "api/%s.html" % (dotted_name,)
+            self.linker.set_link(dotted_name, reltargetpath)
+            passed.append((dotted_name, tag, nav, reltargetpath))
+        return passed
+
+    def build_method_pages(self, data, project):
+        for dotted_name, tag, nav, reltargetpath in data:
+            title = 'api documentation for %s' % (dotted_name,)
+            self.write_page(title, reltargetpath, project, tag, nav)
+
+    def prepare_function_pages(self, namespace_tree, method_dotted_names):
+        passed = []
+        for dotted_name in method_dotted_names:
+            # XXX should we create a build_function_view instead?
+            parent_dotted_name, _ = split_of_last_part(dotted_name)
+            sibling_dotted_names = namespace_tree[parent_dotted_name]
+            tag = self.build_callable_view(dotted_name)
+            nav = self.build_navigation(parent_dotted_name,
+                                        sibling_dotted_names, dotted_name)
+            reltargetpath = "api/%s.html" % (dotted_name,)
+            self.linker.set_link(dotted_name, reltargetpath)
+            passed.append((dotted_name, tag, nav, reltargetpath))
+        return passed
+
+    def build_function_pages(self, data, project):
+        for dotted_name, tag, nav, reltargetpath in data:
+            title = 'api documentation for %s' % (dotted_name,)
+            self.write_page(title, reltargetpath, project, tag, nav)
+
+    def prepare_namespace_pages(self, namespace_tree):
+        passed = []
+        module_name = self.dsa.get_module_name().split('/')[-1]
+
+        names = namespace_tree.keys()
+        names.sort()
+        for dotted_name in names:
+            subitem_dotted_names = namespace_tree[dotted_name]
+            tag = self.build_namespace_view(dotted_name, subitem_dotted_names)
+            nav = self.build_navigation(dotted_name, subitem_dotted_names,
+                                        dotted_name)
+            if dotted_name == '':
+                reltargetpath = 'api/index.html'
+            else:
+                reltargetpath = 'api/%s.html' % (dotted_name,)
+            self.linker.set_link(dotted_name, reltargetpath)
+            passed.append((dotted_name, tag, nav, reltargetpath))
+        return passed
+
+    def build_namespace_pages(self, data, project):
+        for dotted_name, tag, nav, reltargetpath in data:
+            if dotted_name == '':
+                dotted_name = self.dsa.get_module_name().split('/')[-1]
+            title = 'index of %s namespace' % (dotted_name,)
+            self.write_page(title, reltargetpath, project, tag, nav)
+
+    def build_navigation(self, dotted_name, item_dotted_names, selection):
+        navitems = []
+
+        # top namespace, index.html
+        module_name = self.dsa.get_module_name().split('/')[-1]
+        navitems.append(build_navitem_html(self.linker, module_name, '', 0,
+                                           (selection == '')))
+
+        indent = 1
+        path = dotted_name.split('.')
+        if dotted_name != '':
+            # build html for each item in path to dotted_name item
+            for i in xrange(len(path)):
+                name = path[i]
+                item_dotted_name = '.'.join(path[:i+1])
+                selected = (selection == item_dotted_name)
+                navitems.append(build_navitem_html(self.linker, name,
+                                                   item_dotted_name, indent,
+                                                   selected))
+                indent += 1
+
+        # build sub items of dotted_name item
+        for item_dotted_name in py.builtin.sorted(item_dotted_names):
+            itemname = item_dotted_name.split('.')[-1]
+            if is_private(itemname):
+                continue
+            selected = (item_dotted_name == selection)
+            navitems.append(build_navitem_html(self.linker, itemname,
+                                               item_dotted_name, indent,
+                                               selected))
+        return H.Navigation(*navitems)
+
+    def build_callable_value_description(self, dotted_name):
+        args, retval = self.dsa.get_function_signature(dotted_name)
+        valuedesc = H.ValueDescList()
+        for name, _type in args + [('return value', retval)]:
+            l = self.process_type_link(_type)
+            items = []
+            next = "%s :: " % name
+            for item in l:
+                if isinstance(item, str):
+                    next += item
+                else:
+                    if next:
+                        items.append(next)
+                        next = ""
+                    items.append(item)
+            if next:
+                items.append(next)
+            valuedesc.append(H.ValueDescItem(*items))
+        return H.div(H.div('where:'), valuedesc)
+
+    def process_type_link(self, _type):
+        # now we do simple type dispatching and provide a link in this case
+        lst = []
+        data = self.dsa.get_type_desc(_type)
+        if not data:
+            for i in _type.striter():
+                if isinstance(i, str):
+                    lst.append(i)
+                else:
+                    lst += self.process_type_link(i)
+            return lst
+        name, _desc_type, is_degenerated = data
+        if not is_degenerated:
+            linktarget = self.linker.get_lazyhref(name)
+            lst.append(H.a(str(_type), href=linktarget))
+        else:
+            # we should provide here some way of linking to sourcegen directly
+            lst.append(name)
+        return lst
+

py/apigen/layout.py

@@ -0,0 +1,23 @@
+""" layout definition for generating api/source documents
+
+    this is the place where customization can be done
+"""
+
+import py
+from py.__.doc.confrest import Page
+
+class LayoutPage(Page):
+    """ this provides the layout and style information """
+
+    def __init__(self, *args, **kwargs):
+        self.nav = kwargs.pop('nav')
+        super(LayoutPage, self).__init__(*args, **kwargs)
+
+    def set_content(self, contentel):
+        self.contentspace.append(contentel)
+
+    def fill(self):
+        super(LayoutPage, self).fill()
+        self.menubar[:] = []
+        self.menubar.append(self.nav)
+

py/apigen/linker.py

@@ -0,0 +1,75 @@
+import py
+html = py.xml.html
+
+def getrelfspath(dotted_name):
+    # XXX need to make sure its imported on non-py lib 
+    return eval(dotted_name, {"py": py})
+
+class LazyHref(object):
+    def __init__(self, linker, linkid):
+        self._linker = linker
+        self._linkid = linkid
+
+    def __unicode__(self):
+        return unicode(self._linker.get_target(self._linkid))
+
+class Linker(object):
+    fromlocation = None
+
+    def __init__(self):
+        self._linkid2target = {}
+
+    def get_lazyhref(self, linkid):
+        return LazyHref(self, linkid)
+
+    def set_link(self, linkid, target):
+        assert (linkid not in self._linkid2target,
+                'linkid %r already used' % (linkid,))
+        self._linkid2target[linkid] = target
+
+    def get_target(self, linkid):
+        linktarget = self._linkid2target[linkid]
+        if self.fromlocation is not None:
+            linktarget = relpath(self.fromlocation, linktarget)
+        return linktarget
+
+    def call_withbase(self, base, func, *args, **kwargs):
+        assert self.fromlocation is None
+        self.fromlocation = base 
+        try:
+            return func(*args, **kwargs)
+        finally:
+            del self.fromlocation 
+    
+def relpath(p1, p2, sep='/', back='..'):
+    if (p1.startswith(sep) ^ p2.startswith(sep)): 
+        raise ValueError("mixed absolute relative path: %r -> %r" %(p1, p2))
+    fromlist = p1.split(sep)
+    tolist = p2.split(sep)
+
+    # AA
+    # AA BB     -> AA/BB
+    #
+    # AA BB
+    # AA CC     -> CC
+    #
+    # AA BB 
+    # AA      -> ../AA
+
+    diffindex = 0
+    for x1, x2 in zip(fromlist, tolist):
+        if x1 != x2:
+            break
+        diffindex += 1
+    commonindex = diffindex - 1
+
+    fromlist_diff = fromlist[diffindex:]
+    tolist_diff = tolist[diffindex:]
+
+    if not fromlist_diff:
+        return sep.join(tolist[commonindex:])
+    backcount = len(fromlist_diff)
+    if tolist_diff:
+        return sep.join([back,]*(backcount-1) + tolist_diff)
+    return sep.join([back,]*(backcount) + tolist[commonindex:])
+

py/apigen/project.py

@@ -0,0 +1,65 @@
+""" this contains the code that actually builds the pages using layout.py
+
+    building the docs happens in two passes: the first one takes care of
+    collecting contents and navigation items, the second builds the actual
+    HTML
+"""
+
+import py
+from layout import LayoutPage
+
+class Project(py.__.doc.confrest.Project):
+    """ a full project
+
+        this takes care of storing information on the first pass, and building
+        pages + indexes on the second
+    """
+
+    def __init__(self):
+        self.content_items = {}
+    
+    def add_item(self, path, content):
+        """ add a single item (page)
+
+            path is a (relative) path to the object, used for building links
+            and navigation
+
+            content is an instance of some py.xml.html item
+        """
+        assert path not in self.content_items, 'duplicate path %s' % (path,)
+        self.content_items[path] = content
+
+    def build(self, outputpath):
+        """ convert the tree to actual HTML
+        
+            uses the LayoutPage class below for each page and takes care of
+            building index documents for the root and each sub directory
+        """
+        opath = py.path.local(outputpath)
+        opath.ensure(dir=True)
+        paths = self.content_items.keys()
+        paths.sort()
+        for path in paths:
+            # build the page using the LayoutPage class
+            page = self.Page(self, path, stylesheeturl=self.stylesheet)
+            page.contentspace.append(self.content_items[path])
+            ipath = opath.join(path)
+            if not ipath.dirpath().check():
+                # XXX create index.html(?)
+                ipath.ensure(file=True)
+            ipath.write(page.unicode().encode(self.encoding))
+
+    def process(self, txtpath):
+        """ this allows using the project from confrest """
+        # XXX not interesting yet, but who knows later (because of the
+        # cool nav)
+
+if __name__ == '__main__':
+    # XXX just to have an idea of how to use this...
+    proj = Project()
+    here = py.path.local('.')
+    for fpath in here.visit():
+        if fpath.check(file=True):
+            proj.add_item(fpath, convert_to_html_somehow(fpath))
+    proj.build()
+

py/apigen/rest/genrest.py

@@ -0,0 +1,524 @@
+
+""" Generating ReST output (raw, not python)
+out of data that we know about function calls
+"""
+
+import py
+import sys
+import re
+
+from py.__.apigen.tracer.docstorage import DocStorageAccessor
+from py.__.rest.rst import * # XXX Maybe we should list it here
+from py.__.apigen.tracer import model
+from py.__.rest.transform import RestTransformer
+
+def split_of_last_part(name):
+    name = name.split(".")
+    return ".".join(name[:-1]), name[-1]
+
+class AbstractLinkWriter(object):
+    """ Class implementing writing links to source code.
+    There should exist various classes for that, different for Trac,
+    different for CVSView, etc.
+    """
+    def getlinkobj(self, obj, name):
+        return None
+    
+    def getlink(self, filename, lineno, funcname):
+        raise NotImplementedError("Abstract link writer")
+    
+    def getpkgpath(self, filename):
+        # XXX: very simple thing
+        path = py.path.local(filename).dirpath()
+        while 1:
+            try:
+                path.join('__init__.py').stat()
+                path = path.dirpath()
+            except py.error.ENOENT:
+                return path
+    
+class ViewVC(AbstractLinkWriter):
+    """ Link writer for ViewVC version control viewer
+    """
+    def __init__(self, basepath):
+        # XXX: should try to guess from a working copy of svn
+        self.basepath = basepath
+    
+    def getlink(self, filename, lineno, funcname):
+        path = str(self.getpkgpath(filename))
+        assert filename.startswith(path), (
+            "%s does not belong to package %s" % (filename, path))
+        relname = filename[len(path):]
+        if relname.endswith('.pyc'):
+            relname = relname[:-1]
+        sep = py.std.os.sep
+        if sep != '/':
+            relname = relname.replace(sep, '/')
+        return ('%s:%s' % (filename, lineno),
+                self.basepath + relname[1:] + '?view=markup')
+    
+class SourceView(AbstractLinkWriter):
+    def __init__(self, baseurl):
+        self.baseurl = baseurl
+        if self.baseurl.endswith("/"):
+            self.baseurl = baseurl[:-1]
+
+    def getlink(self, filename, lineno, funcname):
+        if filename.endswith('.pyc'):
+            filename = filename[:-1]
+        if filename is None:
+            return "<UNKNOWN>:%s" % funcname,""
+        pkgpath = self.getpkgpath(filename)
+        if not filename.startswith(str(pkgpath)):
+            # let's leave it
+            return "<UNKNOWN>:%s" % funcname,""
+        
+        relname = filename[len(str(pkgpath)):]
+        if relname.endswith('.pyc'):
+            relname = relname[:-1]
+        sep = py.std.os.sep
+        if sep != '/':
+            relname = relname.replace(sep, '/')
+        return "%s:%s" % (relname, funcname),\
+            "%s%s#%s" % (self.baseurl, relname, funcname)
+
+    def getlinkobj(self, name, obj):
+        try:
+            filename = sys.modules[obj.__module__].__file__
+            return self.getlink(filename, 0, name)
+        except AttributeError:
+            return None
+
+class DirectPaste(AbstractLinkWriter):
+    """ No-link writer (inliner)
+    """
+    def getlink(self, filename, lineno, funcname):
+        return ('%s:%s' % (filename, lineno), "")
+
+class DirectFS(AbstractLinkWriter):
+    """ Creates links to the files on the file system (for local docs)
+    """
+    def getlink(self, filename, lineno, funcname):
+        return ('%s:%s' % (filename, lineno), 'file://%s' % (filename,))
+
+class PipeWriter(object):
+    def __init__(self, output=sys.stdout):
+        self.output = output
+    
+    def write_section(self, name, rest):
+        text = "Contents of file %s.txt:" % (name,)
+        self.output.write(text + "\n")
+        self.output.write("=" * len(text) + "\n")
+        self.output.write("\n")
+        self.output.write(rest.text() + "\n")
+
+    def getlink(self, type, targetname, targetfilename):
+        return '%s.txt' % (targetfilename,)
+
+class DirWriter(object):
+    def __init__(self, directory=None):
+        if directory is None:
+            self.directory = py.test.ensuretemp("rstoutput")
+        else:
+            self.directory = py.path.local(directory)
+    
+    def write_section(self, name, rest):
+        filename = '%s.txt' % (name,)
+        self.directory.ensure(filename).write(rest.text())
+
+    def getlink(self, type, targetname, targetfilename):
+        # we assume the result will get converted to HTML...
+        return '%s.html' % (targetfilename,)
+
+class FileWriter(object):
+    def __init__(self, fpath):
+        self.fpath = fpath
+        self.fp = fpath.open('w+')
+        self._defined_targets = []
+
+    def write_section(self, name, rest):
+        self.fp.write(rest.text())
+        self.fp.flush()
+
+    def getlink(self, type, targetname, targetbasename):
+        # XXX problem: because of docutils' named anchor generation scheme,
+        # a method Foo.__init__ would clash with Foo.init (underscores are 
+        # removed)
+        if targetname in self._defined_targets:
+            return None
+        self._defined_targets.append(targetname)
+        targetname = targetname.lower().replace('.', '-').replace('_', '-')
+        while '--' in targetname:
+            targetname = targetname.replace('--', '-')
+        if targetname.startswith('-'):
+            targetname = targetname[1:]
+        if targetname.endswith('-'):
+            targetname = targetname[:-1]
+        return '#%s-%s' % (type, targetname)
+
+class HTMLDirWriter(object):
+    def __init__(self, indexhandler, filehandler, directory=None):
+        self.indexhandler = indexhandler
+        self.filehandler = filehandler
+        if directory is None:
+            self.directory = py.test.ensuretemp('dirwriter')
+        else:
+            self.directory = py.path.local(directory)
+
+    def write_section(self, name, rest):
+        if name == 'index':
+            handler = self.indexhandler
+        else:
+            handler = self.filehandler
+        h = handler(name)
+        t = RestTransformer(rest)
+        t.parse(h)
+        self.directory.ensure('%s.html' % (name,)).write(h.html)
+
+    def getlink(self, type, targetname, targetfilename):
+        return '%s.html' % (targetfilename,)
+
+class RestGen(object):
+    def __init__(self, dsa, linkgen, writer=PipeWriter()):
+        #assert isinstance(linkgen, DirectPaste), (
+        #                        "Cannot use different linkgen by now")
+        self.dsa = dsa
+        self.linkgen = linkgen
+        self.writer = writer
+        self.tracebacks = {}
+
+    def write(self):
+        """write the data to the writer"""
+        modlist = self.get_module_list()
+        classlist = self.get_class_list(module='')
+        funclist = self.get_function_list()
+        modlist.insert(0, ['', classlist, funclist])
+
+        indexrest = self.build_index([t[0] for t in modlist])
+        self.writer.write_section('index', Rest(*indexrest))
+        
+        self.build_modrest(modlist)
+        
+    def build_modrest(self, modlist):
+        modrest = self.build_modules(modlist)
+        for name, rest, classlist, funclist in modrest:
+            mname = name
+            if mname == '':
+                mname = self.dsa.get_module_name()
+            self.writer.write_section('module_%s' % (mname,),
+                                      Rest(*rest))
+            for cname, crest, cfunclist in classlist:
+                self.writer.write_section('class_%s' % (cname,),
+                                          Rest(*crest))
+                for fname, frest, tbdata in cfunclist:
+                    self.writer.write_section('method_%s' % (fname,),
+                                              Rest(*frest))
+                    for tbname, tbrest in tbdata:
+                        self.writer.write_section('traceback_%s' % (tbname,),
+                                                  Rest(*tbrest))
+            for fname, frest, tbdata in funclist:
+                self.writer.write_section('function_%s' % (fname,),
+                                          Rest(*frest))
+                for tbname, tbrest in tbdata:
+                    self.writer.write_section('traceback_%s' % (tbname,),
+                                              Rest(*tbrest))
+    
+    def build_classrest(self, classlist):
+        classrest = self.build_classes(classlist)
+        for cname, rest, cfunclist in classrest:
+            self.writer.write_section('class_%s' % (cname,),
+                                      Rest(*rest))
+            for fname, rest in cfunclist:
+                self.writer.write_section('method_%s' % (fname,),
+                                          Rest(*rest))
+
+    def build_funcrest(self, funclist):
+        funcrest = self.build_functions(funclist)
+        for fname, rest, tbdata in funcrest:
+            self.writer.write_section('function_%s' % (fname,),
+                                      Rest(*rest))
+            for tbname, tbrest in tbdata:
+                self.writer.write_section('traceback_%s' % (tbname,),
+                                          Rest(*tbrest))
+
+    def build_index(self, modules):
+        rest = [Title('index', abovechar='=', belowchar='=')]
+        rest.append(Title('exported modules:', belowchar='='))
+        for module in modules:
+            mtitle = module
+            if module == '':
+                module = self.dsa.get_module_name()
+                mtitle = '%s (top-level)' % (module,)
+            linktarget = self.writer.getlink('module', module,
+                                             'module_%s' % (module,))
+            rest.append(ListItem(Link(mtitle, linktarget)))
+        return rest
+
+    def build_modules(self, modules):
+        ret = []
+        for module, classes, functions in modules:
+            mname = module
+            if mname == '':
+                mname = self.dsa.get_module_name()
+            rest = [Title('module: %s' % (mname,), abovechar='=',
+                                                   belowchar='='),
+                    Title('index:', belowchar='=')]
+            if classes:
+                rest.append(Title('classes:', belowchar='^'))
+                for cls, bases, cfunclist in classes:
+                    linktarget = self.writer.getlink('class', cls,
+                        'class_%s' % (cls,))
+                    rest.append(ListItem(Link(cls, linktarget)))
+            classrest = self.build_classes(classes)
+            if functions:
+                rest.append(Title('functions:', belowchar='^'))
+                for func in functions:
+                    if module:
+                        func = '%s.%s' % (module, func)
+                    linktarget = self.writer.getlink('function',
+                                                     func,
+                                                     'function_%s' % (func,))
+                    rest.append(ListItem(Link(func, linktarget)))
+            funcrest = self.build_functions(functions, module, False)
+            ret.append((module, rest, classrest, funcrest))
+        return ret
+    
+    def build_classes(self, classes):
+        ret = []
+        for cls, bases, functions in classes:
+            rest = [Title('class: %s' % (cls,), belowchar='='),
+                    LiteralBlock(self.dsa.get_doc(cls))]
+            # link to source
+            link_to_class = self.linkgen.getlinkobj(cls, self.dsa.get_obj(cls))
+            if link_to_class:
+                rest.append(Paragraph(Text("source: "), Link(*link_to_class)))
+        
+            if bases:
+                rest.append(Title('base classes:', belowchar='^')),
+                for base in bases:
+                    rest.append(ListItem(self.make_class_link(base)))
+            if functions:
+                rest.append(Title('functions:', belowchar='^'))
+                for (func, origin) in functions:
+                    linktarget = self.writer.getlink('method',
+                                                     '%s.%s' % (cls, func),
+                                                     'method_%s.%s' % (cls,
+                                                                       func))
+                    rest.append(ListItem(Link('%s.%s' % (cls, func),
+                                              linktarget)))
+            funcrest = self.build_functions(functions, cls, True)
+            ret.append((cls, rest, funcrest))
+        return ret
+    
+    def build_functions(self, functions, parent='', methods=False):
+        ret = []
+        for function in functions:
+            origin = None
+            if methods:
+                function, origin = function
+            if parent:
+                function = '%s.%s' % (parent, function)
+            rest, tbrest = self.write_function(function, origin=origin,
+                                               ismethod=methods)
+            ret.append((function, rest, tbrest))
+        return ret
+
+    def get_module_list(self):
+        visited = []
+        ret = []
+        for name in self.dsa.get_class_names():
+            if '.' in name:
+                module, classname = split_of_last_part(name)
+                if module in visited:
+                    continue
+                visited.append(module)
+                ret.append((module, self.get_class_list(module),
+                                    self.get_function_list(module)))
+        return ret
+
+    def get_class_list(self, module):
+        ret = []
+        for name in self.dsa.get_class_names():
+            classname = name
+            if '.' in name:
+                classmodule, classname = split_of_last_part(name)
+                if classmodule != module:
+                    continue
+            elif module != '':
+                continue
+            bases = self.dsa.get_possible_base_classes(name)
+            ret.append((name, bases, self.get_method_list(name)))
+        return ret
+
+    def get_function_list(self, module=''):
+        ret = []
+        for name in self.dsa.get_function_names():
+            funcname = name
+            if '.' in name:
+                funcpath, funcname = split_of_last_part(name)
+                if funcpath != module:
+                    continue
+            elif module != '':
+                continue
+            ret.append(funcname)
+        return ret
+
+    def get_method_list(self, classname):
+        methodnames = self.dsa.get_class_methods(classname)
+        return [(mn, self.dsa.get_method_origin('%s.%s' % (classname, mn)))
+                for mn in methodnames]
+
+    def process_type_link(self, _type):
+        # now we do simple type dispatching and provide a link in this case
+        lst = []
+        data = self.dsa.get_type_desc(_type)
+        if not data:
+            for i in _type.striter():
+                if isinstance(i, str):
+                    lst.append(i)
+                else:
+                    lst += self.process_type_link(i)
+            return lst
+        name, _desc_type, is_degenerated = data
+        if not is_degenerated:
+            linktarget = self.writer.getlink(_desc_type, name,
+                                             '%s_%s' % (_desc_type, name))
+            lst.append(Link(str(_type), linktarget))
+        else:
+            # we should provide here some way of linking to sourcegen directly
+            lst.append(name)
+        return lst
+
+    def write_function(self, functionname, origin=None, ismethod=False,
+                       belowchar='-'):
+        # XXX I think the docstring should either be split on \n\n and cleaned
+        # from indentation, or treated as ReST too (although this is obviously
+        # dangerous for non-ReST docstrings)...
+        if ismethod:
+            title = Title('method: %s' % (functionname,), belowchar=belowchar)
+        else:
+            title = Title('function: %s' % (functionname,),
+                          belowchar=belowchar)
+        
+        lst = [title, LiteralBlock(self.dsa.get_doc(functionname)),
+               LiteralBlock(self.dsa.get_function_definition(functionname))]
+        link_to_function = self.linkgen.getlinkobj(functionname, self.dsa.get_obj(functionname))
+        if link_to_function:
+            lst.insert(1, Paragraph(Text("source: "), Link(*link_to_function)))
+        
+        opar = Paragraph(Strong('origin'), ":")
+        if origin:
+            opar.add(self.make_class_link(origin))
+        else:
+            opar.add(Text('<UNKNOWN>'))
+        lst.append(opar)
+
+        lst.append(Paragraph(Strong("where"), ":"))
+        args, retval = self.dsa.get_function_signature(functionname)
+        for name, _type in args + [('return value', retval)]:
+            l = self.process_type_link(_type)
+            items = []
+            next = "%s :: " % name
+            for item in l:
+                if isinstance(item, str):
+                    next += item
+                else:
+                    if next:
+                        items.append(Text(next))
+                        next = ""
+                    items.append(item)
+            if next:
+                items.append(Text(next))
+            lst.append(ListItem(*items))
+        
+        local_changes = self.dsa.get_function_local_changes(functionname)
+        if local_changes:
+            lst.append(Paragraph(Strong('changes in __dict__ after execution'), ":"))
+            for k, changeset in local_changes.iteritems():
+                lst.append(ListItem('%s: %s' % (k, ', '.join(changeset))))
+        
+        exceptions = self.dsa.get_function_exceptions(functionname)
+        if exceptions:
+            lst.append(Paragraph(Strong('exceptions that might appear during '
+                                 'execution'), ":"))
+            for exc in exceptions:
+                lst.append(ListItem(exc))
+                # XXX: right now we leave it alone
+        
+        # XXX missing implementation of dsa.get_function_location()
+        #filename, lineno = self.dsa.get_function_location(functionname)
+        #linkname, linktarget = self.linkgen.getlink(filename, lineno)
+        #if linktarget:
+        #    lst.append(Paragraph("Function source: ",
+        #               Link(linkname, linktarget)))
+        #else:
+        source = self.dsa.get_function_source(functionname)
+        if source:
+            lst.append(Paragraph(Strong('function source'), ":"))
+            lst.append(LiteralBlock(source))
+        
+        # call sites..
+        call_sites = self.dsa.get_function_callpoints(functionname)
+        tbrest = []
+        if call_sites:
+            call_site_title = Title("call sites:", belowchar='+')
+            lst.append(call_site_title)
+            
+            # we have to think differently here. I would go for:
+            # 1. A quick'n'dirty statement where call has appeared first 
+            #    (topmost)
+            # 2. Link to short traceback
+            # 3. Link to long traceback
+            for call_site, _ in call_sites:
+                fdata, tbdata = self.call_site_link(functionname, call_site)
+                lst += fdata
+                tbrest.append(tbdata)
+        
+        return lst, tbrest
+
+    def call_site_link(self, functionname, call_site):
+        tbid, tbrest = self.gen_traceback(functionname, call_site)
+        tbname = '%s.%s' % (functionname, tbid)
+        linktarget = self.writer.getlink('traceback',
+                                         tbname,
+                                         'traceback_%s' % (tbname,))
+        frest = [Paragraph("called in %s" % call_site[0].filename),
+                 Paragraph(Link("traceback %s" % (tbname,),
+                 linktarget))]
+        return frest, (tbname, tbrest)
+    
+    def gen_traceback(self, funcname, call_site):
+        tbid = len(self.tracebacks.setdefault(funcname, []))
+        self.tracebacks[funcname].append(call_site)
+        tbrest = [Title('traceback for %s' % (funcname,))]
+        for line in call_site:
+            lineno = line.lineno - line.firstlineno
+            linkname, linktarget = self.linkgen.getlink(line.filename,
+                                                        line.lineno + 1,
+                                                        funcname)
+            if linktarget:
+                tbrest.append(Paragraph(Link(linkname, linktarget)))
+            else:
+                tbrest.append(Paragraph(linkname))
+            try:
+                source = line.source
+            except IOError:
+                source = "*cannot get source*"
+            mangled = []
+            for i, sline in enumerate(str(source).split('\n')):
+                if i == lineno:
+                    line = '-> %s' % (sline,)
+                else:
+                    line = '   %s' % (sline,)
+                mangled.append(line)
+            tbrest.append(LiteralBlock('\n'.join(mangled)))
+        return tbid, tbrest
+
+    def make_class_link(self, desc):
+        if not desc or desc.is_degenerated:
+            # create dummy link here, or no link at all
+            return Strong(desc.name)
+        else:
+            linktarget = self.writer.getlink('class', desc.name,
+                'class_%s' % (desc.name,))
+            return Link(desc.name, linktarget)

py/apigen/rest/htmlhandlers.py

@@ -0,0 +1,84 @@
+from py.__.rest.transform import HTMLHandler, entitize
+from py.xml import html, raw
+
+class PageHandler(HTMLHandler):
+    def startDocument(self):
+        super(PageHandler, self).startDocument()
+        self.head.append(html.link(type='text/css', rel='stylesheet',
+                                   href='style.css'))
+        title = self.title[0]
+        breadcrumb = ''.join([unicode(el) for el in self.breadcrumb(title)])
+        self.body.append(html.div(raw(breadcrumb), class_='breadcrumb'))
+
+    def handleLink(self, text, target):
+        self.tagstack[-1].append(html.a(text, href=target,
+                                        target='content'))
+
+    def breadcrumb(self, title):
+        if title != 'index':
+            type, path = title.split('_', 1)
+            path = path.split('.')
+            module = None
+            cls = None
+            func = None
+            meth = None
+            if type == 'module':
+                module = '.'.join(path)
+            elif type == 'class':
+                module = '.'.join(path[:-1])
+                cls = path[-1]
+            elif type  == 'method':
+                module = '.'.join(path[:-2])
+                cls = path[-2]
+                meth = path[-1]
+            else:
+                module = '.'.join(path[:-1])
+                func = path[-1]
+            if module:
+                yield html.a(module, href='module_%s.html' % (module,))
+                if type != 'module':
+                    yield u'.'
+            if cls:
+                s = cls
+                if module:
+                    s = '%s.%s' % (module, cls)
+                yield html.a(cls, href='class_%s.html' % (s,))
+                if type != 'class':
+                    yield u'.'
+            if meth:
+                s = '%s.%s' % (cls, meth)
+                if module:
+                    s = '%s.%s.%s' % (module, cls, meth)
+                yield html.a(meth, href='method_%s.html' % (s,))
+            if func:
+                s = func
+                if module:
+                    s = '%s.%s' % (module, func)
+                yield html.a(func, href='function_%s.html' % (s,))
+
+class IndexHandler(PageHandler):
+    ignore_text = False
+
+    def startDocument(self):
+        super(IndexHandler, self).startDocument()
+        self.head.append(html.script(type='text/javascript', src='apigen.js'))
+        self._push(html.div(id='sidebar'))
+
+    def endDocument(self):
+        maindiv = html.div(id="main")
+        maindiv.append(html.div(id="breadcrumb"))
+        maindiv.append(html.iframe(name='content', id='content',
+                                   src='module_py.html'))
+        self.body.append(maindiv)
+    
+    def startTitle(self, depth):
+        self.ignore_text = True
+    
+    def endTitle(self, depth):
+        self.ignore_text = False
+
+    def handleText(self, text):
+        if self.ignore_text:
+            return
+        super(IndexHandler, self).handleText(text)
+

py/apigen/rest/testing/somemodule.py

@@ -0,0 +1,10 @@
+class SomeClass(object):
+    """Some class definition"""
+    
+    def __init__(self, a):
+        self.a = a
+    
+    def method(self, a, b, c):
+        """method docstring"""
+        return a + b + c
+

py/apigen/rest/testing/someothermodule.py

@@ -0,0 +1,21 @@
+from somemodule import SomeClass
+
+class SomeSubClass(SomeClass):
+    """Some subclass definition"""
+
+def fun(a, b, c):
+    """Some docstring
+    
+        Let's make it span a couple of lines to be interesting...
+
+        Note:
+
+         * rest
+         * should
+         * be
+         * supported
+         * or
+         * ignored...
+    """
+    return "d"
+

py/apigen/rest/testing/test_htmlhandlers.py

@@ -0,0 +1,41 @@
+import py
+from py.__.apigen.rest.htmlhandlers import PageHandler
+
+def test_breadcrumb():
+    h = PageHandler()
+    for fname, expected in [
+            ('module_py', '<a href="module_py.html">py</a>'),
+            ('module_py.test',
+                '<a href="module_py.test.html">py.test</a>'),
+            ('class_py.test',
+                ('<a href="module_py.html">py</a>.'
+                 '<a href="class_py.test.html">test</a>')),
+            ('class_py.test.foo',
+                ('<a href="module_py.test.html">py.test</a>.'
+                 '<a href="class_py.test.foo.html">foo</a>')),
+            ('class_py.test.foo.bar',
+                ('<a href="module_py.test.foo.html">py.test.foo</a>.'
+                 '<a href="class_py.test.foo.bar.html">bar</a>')),
+            ('function_foo', '<a href="function_foo.html">foo</a>'),
+            ('function_foo.bar',
+                ('<a href="module_foo.html">foo</a>.'
+                 '<a href="function_foo.bar.html">bar</a>')),
+            ('function_foo.bar.baz',
+                ('<a href="module_foo.bar.html">foo.bar</a>.'
+                 '<a href="function_foo.bar.baz.html">baz</a>')),
+            ('method_foo.bar',
+                ('<a href="class_foo.html">foo</a>.'
+                 '<a href="method_foo.bar.html">bar</a>')),
+            ('method_foo.bar.baz',
+                ('<a href="module_foo.html">foo</a>.'
+                 '<a href="class_foo.bar.html">bar</a>.'
+                 '<a href="method_foo.bar.baz.html">baz</a>')),
+            ('method_foo.bar.baz.qux',
+                ('<a href="module_foo.bar.html">foo.bar</a>.'
+                 '<a href="class_foo.bar.baz.html">baz</a>.'
+                 '<a href="method_foo.bar.baz.qux.html">qux</a>')),
+            ]:
+        html = ''.join([unicode(el) for el in h.breadcrumb(fname)])
+        print fname
+        print html
+        assert html == expected

py/apigen/rest/testing/test_rest.py

@@ -0,0 +1,486 @@
+
+""" tests document generation
+"""
+
+import py
+from StringIO import StringIO
+
+from py.__.apigen.rest.genrest import ViewVC, RestGen, PipeWriter, \
+                                            DirWriter, FileWriter, \
+                                            DirectPaste, DirectFS, \
+                                            HTMLDirWriter, SourceView
+from py.__.apigen.tracer.tracer import Tracer
+from py.__.apigen.tracer.docstorage import DocStorage, DocStorageAccessor
+from py.__.apigen.tracer.permastore import PermaDocStorage
+import pickle
+
+from py.__.apigen.tracer.testing.runtest import cut_pyc
+from py.__.doc.conftest import genlinkchecks
+from py.__.rest.rst import Rest, Paragraph
+from py.__.rest.transform import HTMLHandler
+# XXX: UUuuuuuuuuuuuuuuuuuuuuuuu, dangerous import
+
+sorted = py.builtin.sorted
+
+def _nl(s):
+    """normalize newlines (converting to \n)"""
+    s = s.replace('\r\n', '\n')
+    s = s.replace('\r', '\n')
+    return s
+
+def setup_module(mod):
+    mod.temppath = py.test.ensuretemp('restgen')
+
+def fun_():
+    pass
+
+class SomeClass(object):
+    """Some class definition"""
+    
+    def __init__(self, a):
+        self.a = a
+    
+    def method(self, a, b, c):
+        """method docstring"""
+        return a + b + c
+    
+class SomeSubClass(SomeClass):
+    """Some subclass definition"""
+
+def fun(a, b, c):
+    """Some docstring
+    
+        Let's make it span a couple of lines to be interesting...
+
+        Note:
+
+         * rest
+         * should
+         * be
+         * supported
+         * or
+         * ignored...
+    """
+    return "d"
+
+def test_direct_link():
+    fname = cut_pyc(__file__)
+    title, link = DirectPaste().getlink(fname, 2, "")
+    assert title == '%s:%s' % (fname, 2)
+    assert link == ''
+
+def test_viewvc_link():
+    vcview = ViewVC("http://codespeak.net/viewvc/")
+    fname = cut_pyc(__file__)
+    title, link = vcview.getlink(fname, 0, "")
+    assert title == '%s:%s' % (fname, 0)
+    assert link == ('http://codespeak.net/viewvc/py/apigen/rest/'
+                        'testing/test_rest.py?view=markup')
+
+def test_fs_link():
+    title, link = DirectFS().getlink('/foo/bar/baz.py', 100, "func")
+    assert title == '/foo/bar/baz.py:100'
+    assert link == 'file:///foo/bar/baz.py'
+
+class WriterTest(object):
+    def get_filled_writer(self, writerclass, *args, **kwargs):
+        dw = writerclass(*args, **kwargs)
+        dw.write_section('foo', Rest(Paragraph('foo data')))
+        dw.write_section('bar', Rest(Paragraph('bar data')))
+        return dw
+
+class TestDirWriter(WriterTest):
+    def test_write_section(self):
+        tempdir = temppath.ensure('dirwriter', dir=True)
+        dw = self.get_filled_writer(DirWriter, tempdir)
+        fpaths = tempdir.listdir('*.txt')
+        assert len(fpaths) == 2
+        assert sorted([f.basename for f in fpaths]) == ['bar.txt', 'foo.txt']
+        assert _nl(tempdir.join('foo.txt').read()) == 'foo data\n'
+        assert _nl(tempdir.join('bar.txt').read()) == 'bar data\n'
+    
+    def test_getlink(self):
+        dw = DirWriter(temppath.join('dirwriter_getlink'))
+        link = dw.getlink('function', 'Foo.bar', 'method_foo_bar')
+        assert link == 'method_foo_bar.html'
+
+class TestFileWriter(WriterTest):
+    def test_write_section(self):
+        tempfile = temppath.ensure('filewriter', file=True)
+        fw = self.get_filled_writer(FileWriter, tempfile)
+        data = tempfile.read()
+        assert len(data)
+    
+    def test_getlink(self):
+        fw = FileWriter(temppath.join('filewriter_getlink'))
+        link = fw.getlink('function', 'Foo.bar', 'method_foo_bar')
+        assert link == '#function-foo-bar'
+        # only produce the same link target once...
+        link = fw.getlink('function', 'Foo.bar', 'method_foo_bar')
+        assert link is None
+        link = fw.getlink('function', 'Foo.__init__', 'method_foo___init__')
+        assert link == '#function-foo-init'
+
+class TestPipeWriter(WriterTest):
+    def test_write_section(self):
+        s = StringIO()
+        pw = self.get_filled_writer(PipeWriter, s)
+        data = s.getvalue()
+        assert len(data)
+
+    def test_getlink(self):
+        pw = PipeWriter(StringIO())
+        link = pw.getlink('function', 'Foo.bar', 'method_foo_bar')
+        assert link == 'method_foo_bar.txt'
+
+class TestHTMLDirWriter(WriterTest):
+    def test_write_section(self):
+        tempdir = temppath.ensure('htmldirwriter', dir=1)
+        hdw = self.get_filled_writer(HTMLDirWriter, HTMLHandler, HTMLHandler,
+                                     tempdir)
+        assert tempdir.join('foo.html').check(file=1)
+        assert tempdir.join('bar.html').check(file=1)
+        assert tempdir.join('foo.html').read().startswith('<html>')
+
+class TestRest(object):
+    def get_filled_docstorage(self):
+        descs = {'SomeClass': SomeClass,
+                 'SomeSubClass': SomeSubClass,
+                 'fun':fun}
+        ds = DocStorage().from_dict(descs)
+        t = Tracer(ds)
+        t.start_tracing()
+        s1 = SomeClass("a")
+        fun(1, 2, s1)
+        s2 = SomeSubClass("b")
+        s2.method(1,2,3)
+        fun(1, 3, s2)
+        t.end_tracing()
+        return DocStorageAccessor(ds)
+
+    def get_filled_docstorage_modules(self):
+        import somemodule
+        import someothermodule
+        descs = {
+            'somemodule.SomeClass': somemodule.SomeClass,
+            'someothermodule.SomeSubClass': someothermodule.SomeSubClass,
+            'someothermodule.fun': someothermodule.fun,
+        }
+        ds = DocStorage().from_dict(descs)
+        t = Tracer(ds)
+        t.start_tracing()
+        s1 = somemodule.SomeClass("a")
+        someothermodule.fun(1, 2, s1)
+        s2 = someothermodule.SomeSubClass("b")
+        s2.method(1, 2, 3)
+        someothermodule.fun(1, 3, s2)
+        t.end_tracing()
+        return DocStorageAccessor(ds)
+    
+    def check_rest(self, tempdir):
+        from py.__.misc import rest
+        for path in tempdir.listdir('*.txt'):
+            try:
+                rest.process(path)
+            except ImportError:
+                py.test.skip('skipping rest generation because docutils is '
+                             'not installed (this is a partial skip, the rest '
+                             'of the test was successful)')
+        for path in tempdir.listdir('*.txt'):
+            for item, arg1, arg2, arg3 in genlinkchecks(path):
+                item(arg1, arg2, arg3)
+    
+    def test_generation_simple_api(self):
+        ds = self.get_filled_docstorage()
+        lg = DirectPaste()
+        tempdir = temppath.ensure("simple_api", dir=True)
+        r = RestGen(ds, lg, DirWriter(tempdir))
+        r.write()
+        basenames = [p.basename for p in tempdir.listdir('*.txt')]
+        expected = [
+            'class_SomeClass.txt',
+            'class_SomeSubClass.txt',
+            'function_fun.txt',
+            'index.txt',
+            'method_SomeClass.__init__.txt',
+            'method_SomeClass.method.txt',
+            'method_SomeSubClass.__init__.txt',
+            'method_SomeSubClass.method.txt',
+            'module_Unknown module.txt',
+            'traceback_SomeClass.__init__.0.txt',
+            'traceback_SomeSubClass.__init__.0.txt',
+            'traceback_SomeSubClass.method.0.txt',
+            'traceback_fun.0.txt',
+            'traceback_fun.1.txt',
+        ]
+        print sorted(basenames)
+        assert sorted(basenames) == expected
+        # now we check out...
+        self.check_rest(tempdir)
+        tempdir = temppath.ensure("simple_api_ps", dir=True)
+        if 0:
+            ps = PermaDocStorage(ds)
+            r = RestGen(ps, lg, DirWriter(tempdir))
+            r.write()
+            basenames = [p.basename for p in tempdir.listdir('*.txt')]
+            assert sorted(basenames) == expected
+            self.check_rest(tempdir)
+            pickle.dumps(ps)
+
+    def test_generation_modules(self):
+        ds = self.get_filled_docstorage_modules()
+        lg = DirectPaste()
+        tempdir = temppath.ensure('module_api', dir=True)
+        r = RestGen(ds, lg, DirWriter(tempdir))
+        r.write()
+        basenames = [p.basename for p in tempdir.listdir('*.txt')]
+        expected = [
+            'class_somemodule.SomeClass.txt',
+            'class_someothermodule.SomeSubClass.txt',
+            'function_someothermodule.fun.txt',
+            'index.txt',
+            'method_somemodule.SomeClass.__init__.txt',
+            'method_somemodule.SomeClass.method.txt',
+            'method_someothermodule.SomeSubClass.__init__.txt',
+            'method_someothermodule.SomeSubClass.method.txt',
+            'module_Unknown module.txt',
+            'module_somemodule.txt',
+            'module_someothermodule.txt',
+            'traceback_somemodule.SomeClass.__init__.0.txt',
+            'traceback_someothermodule.SomeSubClass.__init__.0.txt',
+            'traceback_someothermodule.SomeSubClass.method.0.txt',
+            'traceback_someothermodule.fun.0.txt',
+            'traceback_someothermodule.fun.1.txt',
+        ]
+        print sorted(basenames)
+        assert sorted(basenames) == expected
+
+    def test_check_internal_links(self):
+        ds = self.get_filled_docstorage()
+        lg = DirectFS()
+        tempdir = temppath.ensure('internal_links', dir=True)
+        r = RestGen(ds, lg, DirWriter(tempdir))
+        r.write()
+        index = tempdir.join('module_Unknown module.txt')
+        assert index.check(file=True)
+        data = _nl(index.read())
+        assert data.find('.. _`fun`: function_fun.html\n') > -1
+        assert data.find('.. _`fun`: #function-fun\n') == -1
+
+        tempfile = temppath.ensure('internal_links.txt',
+                                  file=True)
+        r = RestGen(ds, lg, FileWriter(tempfile))
+        r.write()
+        data = _nl(tempfile.read())
+        assert data.find('.. _`fun`: #function-fun\n') > -1
+        assert data.find('.. _`fun`: function_fun.html') == -1
+        tempfile = temppath.ensure("internal_links_ps.txt", file=True)
+        if 0:
+            ps = PermaDocStorage(ds)
+            r = RestGen(ps, lg, FileWriter(tempfile))
+            r.write()
+            data = _nl(tempfile.read())
+            assert data.find('.. _`fun`: #function-fun\n') > -1
+            assert data.find('.. _`fun`: function_fun.html') == -1
+            pickle.dumps(ps)
+
+    def test_check_section_order(self):
+        # we use the previous method's data
+        tempfile = temppath.join('internal_links.txt')
+        if not tempfile.check():
+            py.test.skip('depends on previous test, which failed')
+        data = _nl(tempfile.read())
+        # index should be above the rest
+        assert data.find('classes\\:') > -1
+        assert data.find('classes\\:') < data.find('function\\: fun')
+        assert data.find('classes\\:') < data.find(
+                                                    'class\\: SomeClass')
+        # function definitions should be above class ones
+        assert data.find('function\\: fun') > data.find('class\\: SomeClass')
+        # class method definitions should be below the class defs
+        assert data.find('class\\: SomeClass') < data.find(
+                                            'method\\: SomeClass.method')
+        # __init__ should be above other methods
+        assert data.find('method\\: SomeClass.\\_\\_init\\_\\_') > -1
+        assert data.find('method\\: SomeClass.\\_\\_init\\_\\_') < data.find(
+                                                'method\\: SomeClass.method')
+        # base class info
+        assert py.std.re.search(r'class\\\: SomeSubClass.*'
+                                r'base classes\\\:\n\^+[\n ]+\* `SomeClass`_.*'
+                                r'`SomeSubClass.__init__',
+                                data, py.std.re.S)
+
+    def test_som_fun(self):
+        descs = {'fun_': fun_}
+        ds = DocStorage().from_dict(descs)
+        t = Tracer(ds)
+        t.start_tracing()
+        fun_()
+        t.end_tracing()
+        lg = DirectPaste()
+        tempdir = temppath.ensure("some_fun", dir=True)
+        r = RestGen(DocStorageAccessor(ds), lg, DirWriter(tempdir))
+        r.write()
+        self.check_rest(tempdir)
+
+    def test_function_source(self):
+        def blah():
+            a = 3
+            b = 4
+            return a + b
+        
+        descs = {'blah': blah}
+        ds = DocStorage().from_dict(descs)
+        t = Tracer(ds)
+        t.start_tracing()
+        blah()
+        t.end_tracing()
+        lg = DirectPaste()
+        tempdir = temppath.ensure("function_source", dir=True)
+        r = RestGen(DocStorageAccessor(ds), lg, DirWriter(tempdir))
+        r.write()
+        assert tempdir.join("function_blah.txt").read().find("a = 3") != -1
+        self.check_rest(tempdir)
+        ps = DocStorageAccessor(ds)
+        r = RestGen(ps, lg, DirWriter(tempdir))
+        r.write()
+        assert tempdir.join("function_blah.txt").read().find("a = 3") != -1
+
+    def test_function_arguments(self):
+        def blah(a, b, c):
+            return "axx"
+        
+        class C:
+            pass
+        
+        descs = {'blah':blah}
+        ds = DocStorage().from_dict(descs)
+        t = Tracer(ds)
+        t.start_tracing()
+        blah(3, "x", C())
+        t.end_tracing()
+        lg = DirectPaste()
+        tempdir = temppath.ensure("function_args", dir=True)
+        r = RestGen(DocStorageAccessor(ds), lg, DirWriter(tempdir))
+        r.write()
+        source = tempdir.join("function_blah.txt").read()
+        call_point = source.find("call sites\:")
+        assert call_point != -1
+        assert source.find("a \:\: <Int>") < call_point
+        assert source.find("b \:\: <String>") < call_point
+        assert source.find("c \:\: <Instance of Class C>") < call_point
+        self.check_rest(tempdir)
+
+    def test_class_typedefs(self):
+        class A(object):
+            def __init__(self, x):
+                pass
+
+            def a(self):
+                pass
+        
+        class B(A):
+            def __init__(self, y):
+                pass
+        
+        def xxx(x):
+            return x
+        
+        descs = {'A': A, 'B': B, 'xxx':xxx}
+        ds = DocStorage().from_dict(descs)
+        t = Tracer(ds)
+        t.start_tracing()
+        xxx(A(3))
+        xxx(B("f"))
+        t.end_tracing()
+        lg = DirectPaste()
+        tempdir = temppath.ensure("classargs", dir=True)
+        r = RestGen(DocStorageAccessor(ds), lg, DirWriter(tempdir))
+        r.write()
+        source = tempdir.join("function_xxx.txt").read()
+        call_point = source.find("call sites\:")
+        assert call_point != -1
+        print source
+        assert -1 < source.find("x \:\: <Instance of AnyOf( `Class B`_ , "
+                                "`Class A`_ )>") < call_point
+        source = tempdir.join('method_B.a.txt').read()
+        assert source.find('**origin** \: `A`_') > -1
+        self.check_rest(tempdir)
+
+    def test_exc_raising(self):
+        def x():
+            try:
+                1/0
+            except:
+                pass
+        
+        descs = {'x':x}
+        ds = DocStorage().from_dict(descs)
+        t = Tracer(ds)
+        t.start_tracing()
+        x()
+        t.end_tracing()
+        lg = DirectPaste()
+        tempdir = temppath.ensure("exc_raising", dir=True)
+        r = RestGen(DocStorageAccessor(ds), lg, DirWriter(tempdir))
+        r.write()
+        source = tempdir.join('function_x.txt').open().read()
+        assert source.find('ZeroDivisionError') < source.find('call sites\:')
+
+
+    def test_nonexist_origin(self):
+        class A:
+            def method(self):
+                pass
+        
+        class B(A):
+            pass
+        
+        descs = {'B':B}
+        ds = DocStorage().from_dict(descs)
+        t = Tracer(ds)
+        t.start_tracing()
+        B().method()
+        t.end_tracing()
+        lg = DirectPaste()
+        tempdir = temppath.ensure("nonexit_origin", dir=True)
+        r = RestGen(DocStorageAccessor(ds), lg, DirWriter(tempdir))
+        r.write()
+        self.check_rest(tempdir)
+
+    def test_sourceview(self):
+        class A:
+            def method(self):
+                pass
+        
+        descs = {'A':A}
+        ds = DocStorage().from_dict(descs)
+        t = Tracer(ds)
+        t.start_tracing()
+        A().method()
+        t.end_tracing()
+        lg = SourceView('http://localhost:8000')
+        tempdir = temppath.ensure("sourceview", dir=True)
+        r = RestGen(DocStorageAccessor(ds), lg, DirWriter(tempdir))
+        r.write()
+        self.check_rest(tempdir)
+        assert tempdir.join('traceback_A.method.0.txt').open().read().find(
+           '.. _`/py/apigen/rest/testing/test\_rest.py\:A.method`: http://localhost:8000/py/apigen/rest/testing/test_rest.py#A.method') != -1
+
+    def test_sourceview_fun(self):
+        def f():
+            pass
+
+        descs = {'f':f}
+        ds = DocStorage().from_dict(descs)
+        t = Tracer(ds)
+        t.start_tracing()
+        f()
+        t.end_tracing()
+        tempdir = temppath.ensure("sourceview_fun", dir=True)
+        lg = SourceView('http://localhost:8000')
+        r = RestGen(DocStorageAccessor(ds), lg, DirWriter(tempdir))
+        r.write()
+        self.check_rest(tempdir)
+        assert tempdir.join('function_f.txt').open().read().find(
+            '.. _`/py/apigen/rest/testing/test\_rest.py\:f`: http://localhost:8000/py/apigen/rest/testing/test_rest.py#f') != -1

py/apigen/source/browser.py

@@ -0,0 +1,143 @@
+
+""" source browser using compiler module
+
+WARNING!!!
+
+This is very simple and very silly attempt to make so.
+
+"""
+
+from compiler import parse, ast
+import py
+
+from py.__.path.common import PathBase
+
+blockers = [ast.Function, ast.Class]
+
+class BaseElem(object):
+    def listnames(self):
+        if getattr(self, 'parent', None):
+            return self.parent.listnames() + '.' + self.name
+        return self.name
+
+class Module(BaseElem):
+    def __init__(self, path, _dict):
+        self.path = path
+        self.dict = _dict
+    
+    def __getattr__(self, attr):
+        try:
+            return self.dict[attr]
+        except KeyError:
+            raise AttributeError(attr)
+    
+    def get_children(self):
+        values = self.dict.values()
+        all = values[:]
+        for v in values:
+            all += v.get_children()
+        return all
+
+def get_endline(start, lst):
+    l = lst[::-1]
+    for i in l:
+        if i.lineno:
+            return i.lineno
+        end_ch = get_endline(None, i.getChildNodes())
+        if end_ch:
+            return end_ch
+    return start
+
+class Function(BaseElem):
+    def __init__(self, name, parent, firstlineno, endlineno):
+        self.firstlineno = firstlineno
+        self.endlineno = endlineno
+        self.name = name
+        self.parent = parent
+
+    def get_children(self):
+        return []
+
+class Method(BaseElem):
+    def __init__(self, name, parent, firstlineno, endlineno):
+        self.name = name
+        self.firstlineno = firstlineno
+        self.endlineno = endlineno
+        self.parent = parent
+
+def function_from_ast(ast, cls_ast, cls=Function):
+    startline = ast.lineno
+    endline = get_endline(startline, ast.getChildNodes())
+    assert endline
+    return cls(ast.name, cls_ast, startline, endline)
+
+def class_from_ast(cls_ast):
+    bases = [i.name for i in cls_ast.bases if isinstance(i, ast.Name)]
+    # XXX
+    methods = {}
+    startline = cls_ast.lineno
+    name = cls_ast.name
+    endline = get_endline(startline, cls_ast.getChildNodes())
+    cls = Class(name, startline, endline, bases, [])
+    cls.methods = dict([(i.name, function_from_ast(i, cls, Method)) for i in \
+        cls_ast.code.nodes if isinstance(i, ast.Function)])
+    return cls
+
+class Class(BaseElem):
+    def __init__(self, name, firstlineno, endlineno, bases, methods):
+        self.bases = bases
+        self.firstlineno = firstlineno
+        self.endlineno = endlineno
+        self.name = name
+        self.methods = methods
+
+    def __getattr__(self, attr):
+        try:
+            return self.methods[attr]
+        except KeyError:
+            raise AttributeError(attr)
+    
+    def get_children(self):
+        return self.methods.values()
+
+def dir_nodes(st):
+    """ List all the subnodes, which are not blockers
+    """
+    res = []
+    for i in st.getChildNodes():
+        res.append(i)
+        if not i.__class__ in blockers:
+            res += dir_nodes(i)
+    return res
+
+def update_mod_dict(imp_mod, mod_dict):
+    # make sure that things that are in mod_dict, and not in imp_mod,
+    # are not shown
+    for key, value in mod_dict.items():
+        if not hasattr(imp_mod, key):
+            del mod_dict[key]
+
+def parse_path(path):
+    if not isinstance(path, PathBase):
+        path = py.path.local(path)
+    buf = path.open().read()
+    st = parse(buf)
+    # first go - we get all functions and classes defined on top-level
+    nodes = dir_nodes(st)
+    function_ast = [i for i in nodes if isinstance(i, ast.Function)]
+    classes_ast = [i for i in nodes if isinstance(i, ast.Class)]
+    mod_dict = dict([(i.name, function_from_ast(i, None)) for i in function_ast]
+       + [(i.name, class_from_ast(i)) for i in classes_ast])
+    # we check all the elements, if they're really there
+    try:
+        mod = path.pyimport()
+    except (KeyboardInterrupt, SystemExit):
+        raise
+    except:  # catch all other import problems generically
+        # XXX some import problem: we probably should not
+        # pretend to have an empty module 
+        pass
+    else:
+        update_mod_dict(mod, mod_dict)
+    return Module(path, mod_dict)
+

py/apigen/source/color.py

@@ -0,0 +1,174 @@
+""" simple Python syntax coloring """
+
+import re
+
+class PythonSchema(object):
+    """ contains information for syntax coloring """
+    comment = [('#', '\n')]
+    multiline_string = ['"""', "'''"]
+    string = ['"""', "'''", '"', "'"]
+    # XXX not complete
+    keyword = ['for', 'if', 'not', 'then', 'else', 'while', 'from', 'import',
+               'try', 'except', 'finally', 'raise', 'print', 'exec', 'eval',
+               'break', 'in', 'assert', 'None']
+    alt_keyword = ['def', 'class', 'return', 'pass', 'yield']
+
+class Token(object):
+    data = None
+    type = 'unknown'
+
+    def __init__(self, data, type='unknown'):
+        self.data = data
+        self.type = type
+
+    def __repr__(self):
+        return '<Token type="%s" %r>' % (self.type, self.data)
+
+    def __eq__(self, other):
+        return self.data == other.data and self.type == other.type
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+class Tokenizer(object):
+    """ when fed lists strings, it will return tokens with type info
+    
+        very naive tokenizer, state is recorded for multi-line strings, etc.
+    """
+
+    _re_word = re.compile('[\w_]+')
+    _re_space = re.compile('\s+')
+    _re_number = re.compile('[\d\.]*\d[\d\.]*l?', re.I)
+    _re_rest = re.compile('[^\w\s\d\'"]+') # XXX cheating a bit with the quotes
+
+    # these will be filled using the schema
+    _re_strings_full = None
+    _re_strings_multiline = None
+    _re_strings_comments = None
+
+    def __init__(self, schema):
+        self.schema = schema
+        self._inside_multiline = False
+        
+        self._re_strings_full = []
+        self._re_strings_multiline = []
+        self._re_strings_empty = []
+        for d in schema.string + schema.multiline_string:
+            self._re_strings_full.append(
+                re.compile(r'%s[^\\%s]*(\\.[^\\%s]*)+%s' % (d, d, d, d)))
+            self._re_strings_full.append(
+                re.compile(r'%s[^\\%s]+(\\.[^\\%s]*)*%s' % (d, d, d, d)))
+            self._re_strings_empty.append(re.compile('%s%s' % (d, d)))
+        for d in schema.multiline_string:
+            self._re_strings_multiline.append((re.compile('%s.*' % (d,), re.S),
+                                               re.compile('.*?%s' % (d,))))
+        # no multi-line comments in Python... phew :)
+        self._re_comments = []
+        for start, end in schema.comment:
+            self._re_comments.append(re.compile('%s.*?%s' % (start, end)))
+
+    def tokenize(self, data):
+        if self._inside_multiline:
+            m = self._inside_multiline.match(data)
+            if not m:
+                yield Token(data, 'string')
+                data = ''
+            else:
+                s = m.group(0)
+                data = data[len(s):]
+                self._inside_multiline = False
+                yield Token(s, 'string')
+        while data:
+            for f in [self._check_full_strings, self._check_multiline_strings,
+                      self._check_empty_strings, self._check_comments,
+                      self._check_number, self._check_space, self._check_word,
+                      self._check_rest]:
+                data, t = f(data)
+                if t:
+                    yield t
+                    break
+            else:
+                raise ValueError(
+                        'no token found in %r (bug in tokenizer)' % (data,))
+                
+    def _check_full_strings(self, data):
+        token = None
+        for r in self._re_strings_full:
+            m = r.match(data)
+            if m:
+                s = m.group(0)
+                data = data[len(s):]
+                token = Token(s, type='string')
+                break
+        return data, token
+
+    def _check_multiline_strings(self, data):
+        token = None
+        for start, end in self._re_strings_multiline:
+            m = start.match(data)
+            if m:
+                s = m.group(0)
+                data = ''
+                self._inside_multiline = end
+                token = Token(s, 'string')
+                break
+        return data, token
+
+    def _check_empty_strings(self, data):
+        token = None
+        for r in self._re_strings_empty:
+            m = r.match(data)
+            if m:
+                s = m.group(0)
+                data = data[len(s):]
+                token = Token(s, type='string')
+                break
+        return data, token
+
+
+    def _check_comments(self, data):
+        # fortunately we don't have to deal with multi-line comments
+        token = None
+        for r in self._re_comments:
+            m = r.match(data)
+            if m:
+                s = m.group(0)
+                data = data[len(s):]
+                token = Token(s, 'comment')
+                break
+        return data, token
+
+    def _check_word(self, data):
+        m = self._re_word.match(data)
+        if m:
+            s = m.group(0)
+            type = 'word'
+            if s in self.schema.keyword:
+                type = 'keyword'
+            elif s in self.schema.alt_keyword:
+                type = 'alt_keyword'
+            return data[len(s):], Token(s, type)
+        return data, None
+
+    def _check_space(self, data):
+        m = self._re_space.match(data)
+        if m:
+            s = m.group(0)
+            return data[len(s):], Token(s, 'whitespace')
+        return data, None
+
+    def _check_number(self, data):
+        m = self._re_number.match(data)
+        if m:
+            s = m.group(0)
+            return data[len(s):], Token(s, 'number')
+        return data, None
+
+    def _check_rest(self, data):
+        m = self._re_rest.match(data)
+        if m:
+            s = m.group(0)
+            return data[len(s):], Token(s, 'unknown')
+        return data, None
+
+

py/apigen/source/html.py

@@ -0,0 +1,250 @@
+
+""" html - generating ad-hoc html out of source browser
+"""
+
+from py.xml import html, raw
+from compiler import ast
+import time
+from py.__.apigen.source.color import Tokenizer, PythonSchema
+
+class HtmlEnchanter(object):
+    reserved_words = ['if', 'for', 'return', 'yield']
+
+    def __init__(self, mod):
+        self.mod = mod
+        self.create_caches()
+    
+    def create_caches(self):
+        mod = self.mod
+        linecache = {}
+        for item in mod.get_children():
+            linecache[item.firstlineno] = item
+        self.linecache = linecache
+    
+    def enchant_row(self, num, row):
+        # add some informations to row, like functions defined in that
+        # line, etc.
+        try:
+            item = self.linecache[num]
+            # XXX: this should not be assertion, rather check, but we want to
+            #      know if stuff is working
+            pos = row.find(item.name)
+            assert pos != -1
+            end = len(item.name) + pos
+            chunk = html.a(row[pos:end], href="#" + item.listnames(),
+                     name=item.listnames())
+            return [row[:pos], chunk, row[end:]]
+        except KeyError:
+            return [row] # no more info
+
+class HTMLDocument(object):
+    def __init__(self, tokenizer=None):
+        self.html = root = html.html()
+        self.head = head = self.create_head()
+        root.append(head)
+        self.body = body = self.create_body()
+        root.append(body)
+        self.table, self.tbody = table, tbody = self.create_table()
+        body.append(table)
+
+        if tokenizer is None:
+            tokenizer = Tokenizer(PythonSchema)
+        self.tokenizer = tokenizer
+
+    def create_head(self):
+        return html.head(
+            html.title('source view'),
+            html.style("""
+                body, td {
+                    background-color: #FFF;
+                    color: black;
+                    font-family: monospace, Monaco;
+                }
+
+                table, tr {
+                    margin: 0px;
+                    padding: 0px;
+                    border-width: 0px;
+                }
+
+                a {
+                    color: blue;
+                    font-weight: bold;
+                    text-decoration: none;
+                }
+                
+                a:hover {
+                    color: #005;
+                }
+                
+                .lineno {
+                    text-align: right;
+                    color: #555;
+                    width: 3em;
+                    padding-right: 1em;
+                    border: 0px solid black;
+                    border-right-width: 1px;
+                }
+                
+                .code {
+                    padding-left: 1em;
+                    white-space: pre;
+                }
+                
+                .comment {
+                    color: purple;
+                }
+
+                .string {
+                    color: #777;
+                }
+
+                .keyword {
+                    color: blue;
+                }
+
+                .alt_keyword {
+                    color: green;
+                }
+                
+            """, type='text/css'),
+        )
+
+    def create_body(self):
+        return html.body()
+
+    def create_table(self):
+        table = html.table(cellpadding='0', cellspacing='0')
+        tbody = html.tbody()
+        table.append(tbody)
+        return table, tbody
+
+    def prepare_line(self, text):
+        """ adds html formatting to text items (list)
+
+            only processes items if they're of a string type (or unicode)
+        """
+        ret = []
+        for item in text:
+            if type(item) in [str, unicode]:
+                tokens = self.tokenizer.tokenize(item)
+                for t in tokens:
+                    if t.type in ['keyword', 'alt_keyword', 'number',
+                                  'string', 'comment']:
+                        ret.append(html.span(t.data, class_=t.type))
+                    else:
+                        ret.append(t.data)
+            else:
+                ret.append(item)
+        return ret
+
+    def add_row(self, lineno, text):
+        if text == ['']:
+            text = [raw(' ')]
+        else:
+            text = self.prepare_line(text)
+        self.tbody.append(html.tr(html.td(str(lineno), class_='lineno'),
+                                  html.td(class_='code', *text)))
+
+    def __unicode__(self):
+        # XXX don't like to use indent=0 here, but else py.xml's indentation
+        # messes up the html inside the table cells (which displays formatting)
+        return self.html.unicode(indent=0)
+
+def create_html(mod):
+    # out is some kind of stream
+    #*[html.tr(html.td(i.name)) for i in mod.get_children()]
+    lines = mod.path.open().readlines()
+    
+    enchanter = HtmlEnchanter(mod)
+    doc = HTMLDocument()
+    for i, row in enumerate(lines):
+        row = enchanter.enchant_row(i + 1, row)
+        doc.add_row(i + 1, row)
+    return unicode(doc)
+
+style = html.style("""
+  
+  body, p, td {
+    background-color: #FFF;
+    color: black;
+    font-family: monospace, Monaco;
+  }
+
+  td.type {
+    width: 2em;
+  }
+
+  td.name {
+    width: 30em;
+  }
+
+  td.mtime {
+    width: 13em;
+  }
+
+  td.size {
+    text-alignment: right;
+  }
+
+""")
+
+def create_dir_html(path, href_prefix=''):
+    h = html.html(
+        html.head(
+            html.title('directory listing of %s' % (path,)),
+            style,
+        ),
+    )
+    body = html.body(
+        html.h1('directory listing of %s' % (path,)),
+    )
+    h.append(body)
+    table = html.table()
+    body.append(table)
+    tbody = html.tbody()
+    table.append(tbody)
+    items = list(path.listdir())
+    items.sort(key=lambda p: p.basename)
+    items.sort(key=lambda p: not p.check(dir=True))
+    for fpath in items:
+        tr = html.tr()
+        tbody.append(tr)
+        td1 = html.td(fpath.check(dir=True) and 'D' or 'F', class_='type')
+        tr.append(td1)
+        href = fpath.basename
+        if href_prefix:
+            href = '%s%s' % (href_prefix, href)
+        if fpath.check(dir=True):
+            href += '/'
+        td2 = html.td(html.a(fpath.basename, href=href), class_='name')
+        tr.append(td2)
+        td3 = html.td(time.strftime('%Y-%m-%d %H:%M:%S',
+                      time.gmtime(fpath.mtime())), class_='mtime')
+        tr.append(td3)
+        if fpath.check(dir=True):
+            size = ''
+            unit = ''
+        else:
+            size = fpath.size()
+            unit = 'B'
+            for u in ['kB', 'MB', 'GB', 'TB']:
+                if size > 1024:
+                    size = round(size / 1024.0, 2)
+                    unit = u
+        td4 = html.td('%s %s' % (size, unit), class_='size')
+        tr.append(td4)
+    return unicode(h)
+
+def create_unknown_html(path):
+    h = html.html(
+        html.head(
+            html.title('Can not display page'),
+            style,
+        ),
+        html.body(
+            html.p('The data URL (%s) does not contain Python code.' % (path,))
+        ),
+    )
+    return h.unicode()
+

py/apigen/source/index.cgi

@@ -0,0 +1,29 @@
+#!/usr/bin/python
+
+import cgitb;cgitb.enable()
+import path
+import py
+from py.__.apigen.source.browser import parse_path
+from py.__.apigen.source.html import create_html, create_dir_html, \
+                                     create_unknown_html
+
+BASE_URL='http://codespeak.net/svn/py/dist'
+def cgi_main():
+    import os
+    reqpath = os.environ.get('PATH_INFO', '')
+    path = py.path.svnurl('%s%s' % (BASE_URL, reqpath))
+    if not path.check():
+        return create_unknown_html(path)
+    if path.check(file=True):
+        return unicode(create_html(parse_path(path)))
+    elif path.check(dir=True):
+        prefix = ''
+        if not reqpath:
+            prefix = 'index.cgi/'
+        return create_dir_html(path, href_prefix=prefix)
+    else:
+        return create_unknown_html(path)
+
+print 'Content-Type: text/html; charset=UTF-8'
+print 
+print cgi_main()

py/apigen/source/path.py

@@ -0,0 +1,2 @@
+import os, sys
+sys.path = ['/'.join(os.path.dirname(__file__).split(os.sep)[:-3])] + sys.path

py/apigen/source/server.py

@@ -0,0 +1,48 @@
+
+""" web server for displaying source
+"""
+
+import py
+try:
+    from pypy.translator.js.examples import server
+except ImportError:
+    py.test.skip("PyPy not found")
+from py.__.apigen.source.browser import parse_path
+from py.__.apigen.source.html import create_html, create_dir_html, create_unknown_html
+from py.xml import html
+
+class Handler(server.TestHandler):
+    BASE_URL='http://codespeak.net/svn/py/dist'
+
+    def __getattr__(self, attr):
+        if attr == 'index':
+            attr = ''
+        url = self.BASE_URL + "/" + attr
+        if url.endswith('_py'):
+            url = url[:-3] + '.py'
+        path = py.path.svnurl(url)
+        if not path.check():
+            def f(rev=None):
+                return create_unknown_html(path)
+            f.exposed = True
+            f.func_name = attr
+            return f
+        def f(rev='HEAD'):
+            path = py.path.svnurl(url, rev)
+            # some try.. except.. here
+            if path.check(file=True):
+                return unicode(create_html(parse_path(path)))
+            elif path.check(dir=True):
+                return create_dir_html(path)
+            else:
+                return create_unknown_html(path)
+        f.exposed = True
+        f.func_name = attr
+        return f
+
+def _main():
+    server.start_server(handler=Handler)
+
+if __name__ == '__main__':
+    _main()
+

py/apigen/source/testing/test_browser.py

@@ -0,0 +1,80 @@
+
+""" test source browser abilities
+"""
+
+from py.__.apigen.source.browser import parse_path, Class, Function, Method
+import py
+
+def test_browser():
+    tmp = py.test.ensuretemp("sourcebrowser")
+    tmp.ensure("a.py").write(py.code.Source("""
+    def f():
+        pass
+    
+    def g():
+        pass
+        
+    class X:
+        pass
+        
+    class Z(object):
+        x = 1
+        def zzz(self):
+            1
+            2
+            3
+            4
+    """))
+    mod = parse_path(tmp.join("a.py"))
+    assert isinstance(mod.g, Function)
+    assert isinstance(mod.Z, Class)
+    py.test.raises(AttributeError, "mod.zzz")
+    assert mod.g.firstlineno == 5
+    assert mod.g.name == "g"
+    assert mod.g.endlineno == 6
+    assert mod.X.firstlineno == 8
+    assert mod.X.endlineno == 9
+    assert mod.Z.bases == ["object"]
+    assert isinstance(mod.Z.zzz, Method)
+    assert mod.Z.zzz.firstlineno == 13
+    assert mod.Z.zzz.endlineno == 17
+
+def test_if_browser():
+    tmp = py.test.ensuretemp("sourcebrowser")
+    tmp.ensure("b.py").write(py.code.Source("""
+        if 1:
+            def f():
+                pass
+        if 0:
+            def g():
+                pass
+    """))
+    mod = parse_path(tmp.join("b.py"))
+    assert isinstance(mod.f, Function)
+    py.test.raises(AttributeError, 'mod.g')
+
+def test_bases():
+    tmp = py.test.ensuretemp("sourcebrowser")
+    tmp.ensure("c.py").write(py.code.Source("""
+    import py
+    class Dir(py.test.collect.Directory):
+        pass
+    """))
+    mod = parse_path(tmp.join("c.py"))
+    # if it does not rise it's ok for now
+    #
+    
+def test_importing_goes_wrong():
+    tmp = py.test.ensuretemp("sourcebrowserimport")
+    tmp.ensure("x.py").write(py.code.Source("""
+        import aslkdjaslkdjasdl
+    """))
+    mod = parse_path(tmp.join("x.py"))
+
+    tmp.ensure("y.py").write(py.code.Source("""
+        raise KeyboardInterrupt 
+    """))
+    py.test.raises(KeyboardInterrupt, 'parse_path(tmp.join("y.py"))')
+    
+    # if it does not rise it's ok for now
+    #

py/apigen/source/testing/test_color.py

@@ -0,0 +1,77 @@
+import py
+from py.__.apigen.source.color import Tokenizer, Token, PythonSchema
+
+class TestTokenizer(object):
+    def tokens(self, data):
+        t = Tokenizer(PythonSchema)
+        return list(t.tokenize(data))
+
+    def test_word(self):
+        assert self.tokens('foo') == [Token('foo', type='word')]
+        assert self.tokens('_1_word') == [Token('_1_word', type='word')]
+
+    def test_keyword(self):
+        assert 'if' in PythonSchema.keyword
+        assert self.tokens('see if it works') == [Token('see', type='word'),
+                                                  Token(' ',
+                                                        type='whitespace'),
+                                                  Token('if', type='keyword'),
+                                                  Token(' ',
+                                                        type='whitespace'),
+                                                  Token('it', type='word'),
+                                                  Token(' ',
+                                                        type='whitespace'),
+                                                  Token('works', type='word')]
+
+    def test_space(self):
+        assert self.tokens(' ') == [Token(' ', type='whitespace')]
+        assert self.tokens(' \n') == [Token(' \n', type='whitespace')]
+
+    def test_number(self):
+        # XXX incomplete
+        assert self.tokens('1') == [Token('1', type='number')]
+        assert self.tokens('1.1') == [Token('1.1', type='number')]
+        assert self.tokens('.1') == [Token('.1', type='number')]
+        assert self.tokens('1.') == [Token('1.', type='number')]
+        assert self.tokens('1.1l') == [Token('1.1l', type='number')]
+
+    def test_printable(self):
+        assert self.tokens('.') == [Token('.', 'unknown')]
+        assert self.tokens(';#$@\n') == [Token(';#$@', type='unknown'),
+                                         Token('\n', type='whitespace')]
+
+    def test_comment(self):
+        assert self.tokens('# foo\n') == [Token('# foo\n', type='comment')]
+        assert self.tokens('foo # bar\n') == [Token('foo', type='word'),
+                                              Token(' ', type='whitespace'),
+                                              Token('# bar\n', type='comment')]
+
+    def test_string_simple(self):
+        assert self.tokens('"foo"') == [Token('"foo"', type='string')]
+        assert self.tokens('"foo"\'bar\'') == [Token('"foo"', type='string'),
+                                               Token("'bar'", type='string')]
+
+    def test_string_escape(self):
+        assert self.tokens('"foo \\" bar"') == [Token('"foo \\" bar"',
+                                                      type='string')]
+
+    def test_string_multiline(self):
+        t = Tokenizer(PythonSchema)
+        res = list(t.tokenize('"""foo\n'))
+        assert res == [Token('"""foo\n', type='string')]
+        res = list(t.tokenize('bar\n'))
+        assert res == [Token('bar\n', type='string')]
+        res = list(t.tokenize('"""\n'))
+        assert res == [Token('"""', type='string'),
+                       Token('\n', type='whitespace')]
+        # tricky problem: the following line must not put the tokenizer in
+        # 'multiline state'...
+        res = list(t.tokenize('"""foo"""'))
+        assert res == [Token('"""foo"""', type='string')]
+        res = list(t.tokenize('bar'))
+        assert res == [Token('bar', type='word')]
+
+    def test_string_following_printable(self):
+        assert self.tokens('."foo"') == [Token('.', type='unknown'),
+                                         Token('"foo"', type='string')]
+

py/apigen/source/testing/test_html.py

@@ -0,0 +1,149 @@
+
+""" test of html generation
+"""
+
+from py.__.apigen.source.html import create_html, HTMLDocument
+from py.__.apigen.source.browser import parse_path
+from py.xml import html
+
+import py
+import os
+
+def create_html_and_show(path):
+    mod = parse_path(path)
+    html = create_html(mod)
+    testfile = py.test.ensuretemp("htmloutput").ensure("test.html")
+    testfile.write(unicode(html))
+    return testfile
+
+def test_basic():
+    tmp = py.test.ensuretemp("sourcehtml")
+    inp = tmp.ensure("one.py")
+    inp.write(py.code.Source("""
+    def func_one():
+        pass
+    
+    def func_two(x, y):
+        x = 1
+        y = 2
+        return x + y
+    
+    class B:
+        pass
+    
+    class A(B):
+        def meth1(self):
+            pass
+        
+        def meth2(self):
+            pass
+    """))
+    
+    testfile = create_html_and_show(inp)
+    data = testfile.open().read()
+    assert data.find('<a href="#func_one"') != -1
+    assert data.find('<a href="#func_two"') != -1
+    assert data.find('<a href="#B"') != -1
+    assert data.find('<a href="#A"') != -1
+    assert data.find('<a href="#A.meth1"') != -1
+
+class _HTMLDocument(HTMLDocument):
+    def __init__(self):
+        pass
+
+class TestHTMLDocument(object):
+    def test_head(self):
+        doc = _HTMLDocument()
+        head = doc.create_head()
+        assert isinstance(head, html.head)
+        rendered = unicode(head)
+        assert rendered.find('<title>source view</title>') > -1
+        assert py.std.re.search('<style type="text/css">[^<]+</style>',
+                                rendered)
+
+    def test_body(self):
+        doc = _HTMLDocument()
+        body = doc.create_body()
+        assert unicode(body) == '<body></body>'
+
+    def test_table(self):
+        doc = _HTMLDocument()
+        table, tbody = doc.create_table()
+        assert isinstance(table, html.table)
+        assert isinstance(tbody, html.tbody)
+        assert tbody == table[0]
+
+    def prepare_line(self, line, doc=None):
+        if doc is None:
+            doc = HTMLDocument()
+        l = doc.prepare_line(line)
+        return ''.join([unicode(i) for i in l])
+
+    def test_prepare_line_basic(self):
+        result = self.prepare_line(['see if this works'])
+        assert result == 'see <span class="keyword">if</span> this works'
+        result = self.prepare_line(['see if this ',
+                                    html.a('works', name='works'),' too'])
+        assert result == ('see <span class="keyword">if</span> this '
+                          '<a name="works">works</a> too')
+        result = self.prepare_line(['see if something else works'])
+        assert result == ('see <span class="keyword">if</span> something '
+                          '<span class="keyword">else</span> works')
+        result = self.prepare_line(['see if something ',
+                                    html.a('else', name='else'), ' works too'])
+        assert result == ('see <span class="keyword">if</span> something '
+                          '<a name="else">else</a> works too')
+
+    def test_prepare_line_strings(self):
+        result = self.prepare_line(['foo = "bar"'])
+        assert result == 'foo = <span class="string">&quot;bar&quot;</span>'
+
+        result = self.prepare_line(['"spam"'])
+        assert result == '<span class="string">&quot;spam&quot;</span>'
+        
+        # test multiline strings
+        doc = HTMLDocument()
+        result = self.prepare_line(['"""start of multiline'], doc)
+        assert result == ('<span class="string">&quot;&quot;&quot;start of '
+                          'multiline</span>')
+        # doc should now be in 'string mode'
+        result = self.prepare_line(['see if it doesn\'t touch this'], doc)
+        assert result == ('<span class="string">see if it doesn&apos;t touch '
+                          'this</span>')
+        result = self.prepare_line(['"""'], doc)
+        assert result == '<span class="string">&quot;&quot;&quot;</span>'
+        result = self.prepare_line(['see if it colours this again'], doc)
+        assert result == ('see <span class="keyword">if</span> it colours '
+                          'this again')
+
+    def test_add_row(self):
+        doc = HTMLDocument()
+        doc.add_row(1, ['""" this is a foo implementation """'])
+        doc.add_row(2, [''])
+        doc.add_row(3, ['class ', html.a('Foo', name='Foo'), ':'])
+        doc.add_row(4, ['    pass'])
+        tbody = doc.tbody
+        assert len(tbody) == 4
+        assert unicode(tbody[0][0]) == '<td class="lineno">1</td>'
+        assert unicode(tbody[0][1]) == ('<td class="code">'
+                                        '<span class="string">'
+                                        '&quot;&quot;&quot; '
+                                        'this is a foo implementation '
+                                        '&quot;&quot;&quot;'
+                                        '</span></td>')
+        assert unicode(tbody[1][1]) == '<td class="code">&#xa0;</td>'
+        assert unicode(tbody[2][1]) == ('<td class="code">'
+                                        '<span class="alt_keyword">class'
+                                        '</span> '
+                                        '<a name="Foo">Foo</a>:</td>')
+        assert unicode(tbody[3][1]) == ('<td class="code">    '
+                                        '<span class="alt_keyword">pass'
+                                        '</span></td>')
+
+    def test_unicode(self):
+        doc = HTMLDocument()
+        h = unicode(doc)
+        print h
+        assert py.std.re.match(r'<html>\s*<head>\s*<title>[^<]+</title>'
+                                '.*</body>\w*</html>$', h, py.std.re.S)
+

py/apigen/style.css

@@ -0,0 +1,105 @@
+#sidebar {
+  width: 9em;
+  float: left;
+  vertical-align: top;
+  margin-top: 0.5em;
+}
+
+#main {
+  margin-left: 10em;
+}
+
+#content {
+  border: 0px;
+  height: 95%;
+  width: 100%;
+}
+
+#breadcrumb {
+  height: 5%;
+  display: none;
+}
+
+#logo {
+  /* float: left; */
+}
+
+#logo img {
+  border-width: 0px;
+}
+
+#menubar a {
+  text-decoration: none;
+  color: blue;
+}
+
+#menubar div.selected a {
+  color: purple;
+}
+
+.project_title {
+  font-size: 2em;
+  font-weight: bold;
+}
+
+body, div, p, h1, h2, h3, h4 {
+  font-family: Trebuchet MS, Verdana, Arial;
+  background-color: #FFE;
+  color: black;
+}
+
+a {
+  color: #006;
+  text-decoration: none;
+}
+
+ul {
+  padding-left: 0em;
+  margin-top: 0px;
+}
+
+ul li {
+  list-style-type: none;
+}
+
+.code a {
+    color: blue;
+    font-weight: bold;
+    text-decoration: none;
+}
+
+a:hover {
+    color: #005;
+}
+                
+.lineno {
+    text-align: right;
+    color: #555;
+    width: 3em;
+    padding-right: 1em;
+    border: 0px solid black;
+    border-right-width: 1px;
+}
+
+.code {
+    padding-left: 1em;
+    white-space: pre;
+    font-family: monospace, Monaco;
+}
+
+.comment {
+    color: purple;
+}
+
+.string {
+    color: #777;
+}
+
+.keyword {
+    color: blue;
+}
+
+.alt_keyword {
+    color: green;
+}
+                

py/apigen/testing/test_apigen_example.py

@@ -0,0 +1,360 @@
+import py
+html = py.xml.html
+from py.__.apigen.linker import Linker
+from py.__.apigen.htmlgen import *
+from py.__.apigen.tracer.docstorage import DocStorage, DocStorageAccessor
+from py.__.apigen.tracer.tracer import Tracer
+from py.__.apigen.project import Project
+from py.__.test.web import webcheck
+from py.__.apigen.conftest import option
+
+def run_string_sequence_test(data, seq):
+    currpos = -1
+    for s in seq:
+        newpos = data.find(s)
+        if currpos >= newpos:
+            if newpos == -1:
+                message = 'not found'
+            else:
+                message = 'unexpected position: %s' % (newpos,)
+            py.test.fail('string %r: %s' % (s, message))
+        currpos = newpos
+
+def setup_fs_project():
+    temp = py.test.ensuretemp('apigen_example')
+    temp.ensure("pkg/func.py").write(py.code.Source("""\
+        def func(arg1):
+            "docstring"
+    """))
+    temp.ensure('pkg/someclass.py').write(py.code.Source("""\
+        class SomeClass(object):
+            " docstring someclass "
+            def __init__(self, somevar):
+                self.somevar = somevar
+                
+            def get_somevar(self):
+                " get_somevar docstring "
+                return self.somevar
+        SomeInstance = SomeClass(10)
+    """))
+    temp.ensure('pkg/somesubclass.py').write(py.code.Source("""\
+        from someclass import SomeClass
+        class SomeSubClass(SomeClass):
+            " docstring somesubclass "
+            def get_somevar(self):
+                return self.somevar + 1
+    """))
+    temp.ensure('pkg/somenamespace.py').write(py.code.Source("""\
+        def foo():
+            return 'bar'
+        def baz(qux):
+            return qux
+    """))
+    temp.ensure("pkg/__init__.py").write(py.code.Source("""\
+        from py.initpkg import initpkg
+        initpkg(__name__, exportdefs = {
+            'main.sub.func': ("./func.py", "func"),
+            'main.SomeClass': ('./someclass.py', 'SomeClass'),
+            'main.SomeInstance': ('./someclass.py', 'SomeInstance'),
+            'main.SomeSubClass': ('./somesubclass.py', 'SomeSubClass'),
+            'main.SomeSubClass': ('./somesubclass.py', 'SomeSubClass'),
+            'other':             ('./somenamespace.py', '*'),
+        })
+    """))
+    return temp, 'pkg'
+
+def get_dsa(fsroot, pkgname):
+    py.std.sys.path.insert(0, str(fsroot))
+    pkg = __import__(pkgname)
+    ds = DocStorage()
+    ds.from_pkg(pkg)
+    dsa = DocStorageAccessor(ds)
+    return ds, dsa
+
+def _checkhtml(htmlstring):
+    if isinstance(htmlstring, unicode):
+        htmlstring = htmlstring.encode('UTF-8', 'replace')
+    assert isinstance(htmlstring, str)
+    if option.webcheck:
+        webcheck.check_html(htmlstring)
+    else:
+        py.test.skip("pass --webcheck to validate html produced in tests "
+                     "(partial skip: the test has succeeded up until here)")
+
+def _checkhtmlsnippet(htmlstring):
+    # XXX wrap page around snippet and validate 
+    pass
+    #newstring = """<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
+    #"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">\n""" + unicode(h)
+    #_checkhtml(newstring)
+
+class AbstractBuilderTest(object):
+    def setup_class(cls):
+        cls.fs_root, cls.pkg_name = setup_fs_project()
+        cls.ds, cls.dsa = get_dsa(cls.fs_root, cls.pkg_name)
+        cls.project = Project()
+
+    def setup_method(self, meth):
+        class LinkerForTests(Linker):
+            def get_target(self, linkid):
+                try:
+                    return super(LinkerForTests, self).get_target(linkid)
+                except KeyError:
+                    return 'unknown_link_%s' % (linkid,)
+        self.base = base = py.test.ensuretemp('%s_%s' % (
+                            self.__class__.__name__, meth.im_func.func_name))
+        self.linker = linker = LinkerForTests()
+        self.apb = ApiPageBuilder(base, linker, self.dsa, self.fs_root)
+        self.spb = SourcePageBuilder(base, linker, self.fs_root)
+        self.namespace_tree = create_namespace_tree(['main.sub',
+                                                     'main.sub.func',
+                                                     'main.SomeClass',
+                                                     'main.SomeSubClass',
+                                                     'main.SomeInstance',
+                                                     'other.foo',
+                                                     'other.bar'])
+
+class TestApiPageBuilder(AbstractBuilderTest):
+    def test_build_callable_view(self):
+        ds, dsa = get_dsa(self.fs_root, self.pkg_name)
+        t = Tracer(ds)
+        t.start_tracing()
+        pkg = __import__(self.pkg_name)
+        pkg.main.sub.func(10)
+        pkg.main.sub.func(pkg.main.SomeClass(10))
+        t.end_tracing()
+        apb = ApiPageBuilder(self.base, self.linker, dsa, self.fs_root)
+        snippet = apb.build_callable_view('main.sub.func')
+        html = snippet.unicode()
+        print html
+        run_string_sequence_test(html, [
+            'arg1 :: AnyOf(',
+            'href="',
+            'Class SomeClass',
+            'Int&gt;',
+            'return value :: &lt;None&gt;',
+            'origin: %s' % (self.fs_root.join('pkg/func.py'),),
+            'def func(arg1):',
+        ])
+        _checkhtmlsnippet(html)
+
+    def test_build_function_pages(self):
+        data = self.apb.prepare_function_pages(self.namespace_tree,
+                                               ['main.sub.func'])
+        self.apb.build_function_pages(data, self.project)
+        funcfile = self.base.join('api/main.sub.func.html')
+        assert funcfile.check()
+        html = funcfile.read()
+        _checkhtml(html)
+
+    def test_build_class_view(self):
+        snippet = self.apb.build_class_view('main.SomeClass')
+        html = snippet.unicode()
+        _checkhtmlsnippet(html)
+
+    def test_build_class_pages(self):
+        data, methodsdata = self.apb.prepare_class_pages(self.namespace_tree,
+                                                         ['main.SomeClass',
+                                                          'main.SomeSubClass'])
+        self.apb.build_class_pages(data, self.project)
+        clsfile = self.base.join('api/main.SomeClass.html')
+        assert clsfile.check()
+        html = clsfile.read()
+        _checkhtml(html)
+
+    def test_build_class_pages_instance(self):
+        data, methodsdata = self.apb.prepare_class_pages(self.namespace_tree,
+                                                         ['main.SomeClass',
+                                                          'main.SomeSubClass',
+                                                          'main.SomeInstance'])
+        self.apb.build_class_pages(data, self.project)
+        clsfile = self.base.join('api/main.SomeInstance.html')
+        assert clsfile.check()
+        html = clsfile.read()
+        print html
+        run_string_sequence_test(html, [
+            'instance of SomeClass()',
+        ])
+
+    def test_build_class_pages_nav_links(self):
+        data, methodsdata = self.apb.prepare_class_pages(self.namespace_tree,
+                                                         ['main.SomeSubClass',
+                                                          'main.SomeClass'])
+        # fake some stuff that would be built from other methods
+        self.linker.set_link('', 'api/index.html')
+        self.linker.set_link('main', 'api/main.html')
+        self.apb.build_class_pages(data, self.project)
+        clsfile = self.base.join('api/main.SomeClass.html')
+        assert clsfile.check()
+        html = clsfile.read()
+        print html
+        run_string_sequence_test(html, [
+            'href="../style.css"',
+            'href="index.html">pkg',
+            'href="main.html">main',
+            'href="main.SomeClass.html">SomeClass',
+            'href="main.SomeSubClass.html">SomeSubClass',
+        ])
+        assert not 'href="main.sub.func.html"' in html
+        py.test.skip('WOP from here')
+        assert 'href="main.sub.html">sub' in html
+        _checkhtml(html)
+
+    def test_build_class_pages_base_link(self):
+        data, methodsdata = self.apb.prepare_class_pages(self.namespace_tree,
+                                                        ['main.SomeSubClass',
+                                                         'main.SomeClass'])
+        self.apb.build_class_pages(data, self.project)
+        clsfile = self.base.join('api/main.SomeSubClass.html')
+        assert clsfile.check()
+        html = clsfile.read()
+        print html
+        run_string_sequence_test(html, [
+            'href="../style.css"',
+            'href="main.SomeClass.html">main.SomeClass',
+        ])
+        _checkhtml(html)
+
+    def test_source_links(self):
+        data, methodsdata = self.apb.prepare_class_pages(self.namespace_tree,
+                                                         ['main.SomeSubClass',
+                                                          'main.SomeClass'])
+        sourcedata = self.spb.prepare_pages(self.fs_root)
+        self.apb.build_class_pages(data, self.project)
+        self.spb.build_pages(sourcedata, self.project, self.fs_root)
+        funchtml = self.base.join('api/main.SomeClass.html').read()
+        assert funchtml.find('href="../source/pkg/someclass.py.html"') > -1
+        _checkhtml(funchtml)
+
+    def test_build_namespace_pages(self):
+        data = self.apb.prepare_namespace_pages(self.namespace_tree)
+        self.apb.build_namespace_pages(data, self.project)
+        mainfile = self.base.join('api/main.html')
+        assert mainfile.check()
+        html = mainfile.read()
+        print html
+        run_string_sequence_test(html, [
+            'index of main namespace',
+        ])
+        otherfile = self.base.join('api/other.html')
+        assert otherfile.check()
+        otherhtml = otherfile.read()
+        print otherhtml
+        run_string_sequence_test(otherhtml, [
+            'index of other namespace',
+        ])
+        _checkhtml(html)
+        _checkhtml(otherhtml)
+
+    def test_build_namespace_pages_index(self):
+        data = self.apb.prepare_namespace_pages(self.namespace_tree)
+        self.apb.build_namespace_pages(data, self.project)
+        pkgfile = self.base.join('api/index.html')
+        assert pkgfile.check()
+        html = pkgfile.read()
+        assert 'index of project pkg namespace'
+        _checkhtml(html)
+
+    def test_build_namespace_pages_subnamespace(self):
+        data = self.apb.prepare_namespace_pages(self.namespace_tree)
+        self.apb.build_namespace_pages(data, self.project)
+        subfile = self.base.join('api/main.sub.html')
+        assert subfile.check()
+        html = subfile.read()
+        _checkhtml(html)
+
+    def test_build_function_api_pages_nav(self):
+        data = self.apb.prepare_function_pages(self.namespace_tree,
+                                                   ['main.sub.func'])
+        self.linker.set_link('', 'api/index.html')
+        self.linker.set_link('main', 'api/main.html')
+        self.linker.set_link('main.sub', 'api/main.sub.html')
+        self.apb.build_function_pages(data, self.project)
+        funcfile = self.base.join('api/main.sub.func.html')
+        html = funcfile.read()
+        print html
+        run_string_sequence_test(html, [
+            '<a href="index.html">',
+            '<a href="main.html">',
+            '<a href="main.sub.html">',
+            '<a href="main.sub.func.html">',
+        ])
+        _checkhtml(html)
+
+    def test_build_function_navigation(self):
+        self.apb.prepare_namespace_pages(self.namespace_tree)
+        self.apb.prepare_function_pages(self.namespace_tree, ['main.sub.func'])
+        nav = self.apb.build_navigation('main.sub', ['main.sub.func'],
+                                        'main.sub.func')
+        html = nav.unicode(indent=0)
+        print html.encode('UTF-8')
+        assert (u'<div><a href="api/index.html">pkg</a></div>'
+                u'<div>\xa0\xa0<a href="api/main.html">main</a></div>'
+                u'<div>\xa0\xa0\xa0\xa0'
+                    u'<a href="api/main.sub.html">sub</a></div>'
+                u'<div class="selected">\xa0\xa0\xa0\xa0\xa0\xa0'
+                    u'<a href="api/main.sub.func.html">func</a></div>'
+        ) in html
+
+    def test_build_root_namespace_view(self):
+        data = self.apb.prepare_namespace_pages(self.namespace_tree)
+        self.apb.build_namespace_pages(data, self.project)
+        rootfile = self.base.join('api/index.html')
+        assert rootfile.check()
+        html = rootfile.read()
+        assert '<a href="main.html">' in html
+        _checkhtml(html)
+
+class TestSourcePageBuilder(AbstractBuilderTest):
+    def test_build_pages(self):
+        data = self.spb.prepare_pages(self.fs_root)
+        self.spb.build_pages(data, self.project, self.fs_root)
+        somesource = self.base.join('source/pkg/func.py.html').read()
+        _checkhtml(somesource)
+
+    def test_build_pages_nav(self):
+        data = self.spb.prepare_pages(self.fs_root)
+        self.spb.build_pages(data, self.project, self.fs_root)
+        funcsource = self.base.join('source/pkg/func.py.html')
+        assert funcsource.check(file=True)
+        html = funcsource.read()
+        print html
+        run_string_sequence_test(html, [
+            'href="../../style.css"',
+            '<a href="../index.html">root</a>',
+            '<a href="index.html">pkg</a>',
+            '<a href="someclass.py.html">someclass.py</a>',
+            '<a href="somesubclass.py.html">somesubclass.py</a>',
+        ])
+
+    def test_build_dir_page(self):
+        data = self.spb.prepare_pages(self.fs_root)
+        self.spb.build_pages(data, self.project, self.fs_root)
+        pkgindex = self.base.join('source/pkg/index.html')
+        assert pkgindex.check(file=True)
+        html = pkgindex.read()
+        print html
+        run_string_sequence_test(html, [
+            'href="../../style.css"',
+            '<a href="../index.html">root</a>',
+            '<a href="index.html">pkg</a>',
+            '<a href="func.py.html">func.py</a>',
+            '<a href="someclass.py.html">someclass.py</a>',
+            '<a href="somesubclass.py.html">somesubclass.py</a>',
+            '<h2>directories</h2>',
+            '<h2>files</h2>'])
+        _checkhtml(html)
+
+    def test_build_navigation_root(self):
+        self.spb.prepare_pages(self.fs_root)
+        nav = self.spb.build_navigation(self.fs_root.join('pkg'))
+        html = nav.unicode(indent=0)
+        print html.encode('UTF-8')
+        run_string_sequence_test(html, [
+            'href="source/index.html">root',
+            'href="source/pkg/index.html">pkg',
+            'href="source/pkg/func.py.html">func.py',
+            'href="source/pkg/someclass.py.html">someclass.py',
+            'href="source/pkg/somesubclass.py.html">somesubclass.py',
+        ])
+

py/apigen/testing/test_apigen_functional.py

@@ -0,0 +1,116 @@
+""" functional test for apigen.py
+
+    script to build api + source docs from py.test
+"""
+
+import py
+
+def setup_fs_project():
+    temp = py.test.ensuretemp('apigen_functional')
+    temp.ensure("pkg/func.py").write(py.code.Source("""\
+        def func(arg1):
+            "docstring"
+
+        def func_2(arg1, arg2):
+            return arg1(arg2)
+    """))
+    temp.ensure('pkg/sometestclass.py').write(py.code.Source("""\
+        class SomeTestClass(object):
+            " docstring sometestclass "
+            def __init__(self, somevar):
+                self.somevar = somevar
+                
+            def get_somevar(self):
+                " get_somevar docstring "
+                return self.somevar
+    """))
+    temp.ensure('pkg/sometestsubclass.py').write(py.code.Source("""\
+        from sometestclass import SomeTestClass
+        class SomeTestSubClass(SomeTestClass):
+            " docstring sometestsubclass "
+            def get_somevar(self):
+                return self.somevar + 1
+    """))
+    temp.ensure('pkg/somenamespace.py').write(py.code.Source("""\
+        def foo():
+            return 'bar'
+        def baz(qux):
+            return qux
+    """))
+    temp.ensure("pkg/__init__.py").write(py.code.Source("""\
+        from py.initpkg import initpkg
+        initpkg(__name__, exportdefs = {
+            'main.sub.func': ("./func.py", "func"),
+            'main.func': ("./func.py", "func_2"),
+            'main.SomeTestClass': ('./sometestclass.py', 'SomeTestClass'),
+            'main.SomeTestSubClass': ('./sometestsubclass.py',
+                                      'SomeTestSubClass'),
+        })
+    """))
+    temp.ensure('pkg/test/test_pkg.py').write(py.code.Source("""\
+        import py
+        py.std.sys.path.insert(0,
+            py.magic.autopath().dirpath().dirpath().dirpath().strpath)
+        import pkg
+
+        # this mainly exists to provide some data to the tracer
+        def test_pkg():
+            s = pkg.main.SomeTestClass(10)
+            assert s.get_somevar() == 10
+            s = pkg.main.SomeTestClass('10')
+            assert s.get_somevar() == '10'
+            s = pkg.main.SomeTestSubClass(10)
+            assert s.get_somevar() == 11
+            s = pkg.main.SomeTestSubClass('10')
+            py.test.raises(TypeError, 's.get_somevar()')
+            assert pkg.main.sub.func(10) is None
+            assert pkg.main.sub.func(20) is None
+            s = pkg.main.func(pkg.main.SomeTestClass, 10)
+            assert isinstance(s, pkg.main.SomeTestClass)
+    """))
+    return temp, 'pkg'
+
+def test_apigen_functional():
+    fs_root, package_name = setup_fs_project()
+    tempdir = py.test.ensuretemp('test_apigen_functional_results')
+    parentdir = py.magic.autopath().dirpath().dirpath()
+    pkgdir = fs_root.join('pkg')
+    try:
+        output = py.process.cmdexec('APIGEN_TARGET="%s" py.test --session=L '
+                                    '--apigen="%s/apigen.py" "%s"' % (
+                                        tempdir, parentdir, pkgdir))
+    except py.error.Error, e:
+        print e.out
+        raise
+    assert output.lower().find('traceback') == -1
+
+    # just some quick content checks
+    apidir = tempdir.join('api')
+    assert apidir.check(dir=True)
+    sometestclass_api = apidir.join('main.SomeTestClass.html')
+    assert sometestclass_api.check(file=True)
+    html = sometestclass_api.read()
+    assert '<a href="main.SomeTestClass.html">SomeTestClass</a>' in html
+    # XXX not linking to method files anymore
+    #sometestclass_init_api = apidir.join('main.SomeTestClass.__init__.html')
+    #assert sometestclass_init_api.check(file=True)
+    #assert sometestclass_init_api.read().find(
+    #        '<a href="main.SomeTestClass.__init__.html">__init__</a>') > -1
+    namespace_api = apidir.join('main.html')
+    assert namespace_api.check(file=True)
+    html = namespace_api.read()
+    assert '<a href="main.SomeTestClass.html">SomeTestClass</a>' in html
+
+    sourcedir = tempdir.join('source')
+    assert sourcedir.check(dir=True)
+    sometestclass_source = sourcedir.join('sometestclass.py.html')
+    assert sometestclass_source.check(file=True)
+    html = sometestclass_source.read()
+    assert '<div class="project_title">sources for sometestclass.py</div>' in html
+
+    # XXX later...
+    #index = sourcedir.join('index.html')
+    #assert index.check(file=True)
+    #html = index.read()
+    #assert '<a href="main/index.html">main</a>' in html
+

py/apigen/testing/test_htmlgen.py

@@ -0,0 +1,45 @@
+import py
+from py.__.apigen import htmlgen
+from py.__.apigen.linker import Linker
+
+def test_create_namespace_tree():
+    tree = htmlgen.create_namespace_tree(['foo.bar.baz'])
+    assert tree == {'': ['foo'],
+                    'foo': ['foo.bar'],
+                    'foo.bar': ['foo.bar.baz']}
+    tree = htmlgen.create_namespace_tree(['foo.bar.baz', 'foo.bar.qux'])
+    assert tree == {'': ['foo'],
+                    'foo': ['foo.bar'],
+                    'foo.bar': ['foo.bar.baz', 'foo.bar.qux']}
+    tree = htmlgen.create_namespace_tree(['pkg.sub.func',
+                                          'pkg.SomeClass',
+                                          'pkg.SomeSubClass'])
+    assert tree == {'': ['pkg'],
+                    'pkg.sub': ['pkg.sub.func'],
+                    'pkg': ['pkg.sub', 'pkg.SomeClass',
+                            'pkg.SomeSubClass']}
+
+def test_build_navitem_html():
+    l = Linker()
+    l.set_link('spam.eggs.foo', 'foo.html')
+    h = htmlgen.build_navitem_html(l, 'foo', 'spam.eggs.foo', 0, False)
+    assert unicode(h) == u'<div><a href="foo.html">foo</a></div>'
+    h = htmlgen.build_navitem_html(l, 'bar', 'spam.eggs.foo', 1, True)
+    assert unicode(h) == (u'<div class="selected">\xa0\xa0'
+                          u'<a href="foo.html">bar</a></div>')
+
+def test_source_dirs_files():
+    temp = py.test.ensuretemp('test_source_dirs_files')
+    temp.join('dir').ensure(dir=True)
+    temp.join('dir/file1.py').ensure(file=True)
+    temp.join('dir/file2.pyc').ensure(file=True)
+    temp.join('dir/file3.c').ensure(file=True)
+    temp.join('dir/.hidden_file').ensure(file=True)
+    temp.join('dir/sub').ensure(dir=True)
+    temp.join('dir/.hidden_dir').ensure(dir=True)
+    dirs, files = htmlgen.source_dirs_files(temp.join('dir'))
+    dirnames = py.builtin.sorted([d.basename for d in dirs])
+    filenames = py.builtin.sorted([f.basename for f in files])
+    assert dirnames == ['sub']
+    assert filenames == ['file1.py', 'file3.c']
+

py/apigen/testing/test_linker.py

@@ -0,0 +1,44 @@
+import py
+from py.__.apigen.linker import Linker, getrelfspath, relpath
+
+class TestLinker(object):
+    def test_get_target(self):
+        linker = Linker()
+        lazyhref = linker.get_lazyhref('py.path.local')
+        linker.set_link('py.path.local', 'py/path/local.html')
+        relpath = linker.get_target('py.path.local')
+        assert relpath == 'py/path/local.html'
+   
+    def test_target_relative(self):
+        linker = Linker()
+        lazyhref = linker.get_lazyhref('py.path.local')
+        linker.set_link('py.path.local', 'py/path/local.html')
+        relpath = linker.call_withbase('py/index.html', 
+                    linker.get_target, 'py.path.local')
+        assert relpath == 'path/local.html'
+
+
+        
+testspec = [
+    'a    a/b   a/b', 
+    '/a   /a/b  a/b', 
+    'a    b     b',
+    '/a   /b    b',
+    'a/b  c/d   ../c/d', 
+    '/a/b  /c/d   ../c/d', 
+    'a/b  a     ../a', 
+    '/a/b /a   ../a', 
+]     
+
+def gen_check(frompath, topath, expected):
+    result = relpath(frompath, topath)
+    print "linking", frompath, "to", topath
+    assert result == expected
+
+def test_gen_check():
+    for line in testspec:
+        frompath, topath, expected = line.split()
+        yield gen_check, frompath, topath, expected
+
+def test_check_incompatible():
+    py.test.raises(ValueError, "relpath('/a', 'b')")

py/apigen/todo-apigen.txt

@@ -0,0 +1,68 @@
+
+* format docstrings more nicely (with tests) 
+
+* have the API function view be as informative as possible
+  without having to go to the "single method" view
+  (do we even need a single method view?), for example: 
+
+  browsing the class views (and clicking on methods) 
+  should always make it obvious which class is being
+  viewed.  method views (when navigating there through
+  the class view) should also have the source there 
+
+* have class-level attributes be displayed 
+
+* use "inherited" doc strings, i.e. for
+    class A:
+        def meth(self):
+            "doc1"
+    class B(A):
+        def meth(self): 
+            pass
+
+    B.meth should display the A.meth docstring, probably 
+    with special formatting (italics or so). 
+
+* factor out some common code in the build_* functions
+
+* refactor the apigen/rsession interaction to become
+  cleaner (e.g. apigen's get_documentable_items should
+  be separately tested and the caller should not need
+  to guess what it will get, i think) 
+
+* look out for and streamline all apigen/source-viewer
+  documentation into one document 
+
+
+
+* consider automating dependencies: 
+
+  e.g. something like: queue_render(page, fspath, linker, ...) 
+  would defer the rendering until later. 
+  then a loop does:
+
+  maxlength = len(queue)
+  while queue: 
+       page, fspath, linker, ... = queue.get()
+       # fill outputpath/link here or even earlier
+       if all_links_resolve(page, linker): 
+           render it and write to filesystem
+           maxlength = len(queue) 
+       else:
+           queue.append(...)
+           maxlength -= 1
+           if maxlength <= 0:  
+               print "ERROR: seems i can't make progress"
+               print "unresolved links follow: "
+               ... 
+               print "unresolved pages/fspaths:"
+               ... 
+               XXX maybe: print "filling linker with dummy hrefs, and rendering anyway"
+               ... 
+               raise ... 
+
+* also we might have a support function for tests that
+  fills the linker with "dummy hrefs" for certain types
+  like source links
+       
+* XXX list more here 

py/apigen/tracer/description.py

@@ -0,0 +1,358 @@
+
+import py
+from py.__.apigen.tracer import model
+
+import types
+import inspect
+import copy
+
+MAX_CALL_SITES = 20
+
+set = py.builtin.set
+
+def is_private(name):
+    return name.startswith('_') and not name.startswith('__')
+
+class CallFrame(object):
+    def __init__(self, frame):
+        self.filename = frame.code.raw.co_filename
+        self.lineno = frame.lineno
+        self.firstlineno = frame.code.firstlineno
+        self.source = frame.code.source()
+
+    def _getval(self):
+        return (self.filename, self.lineno)
+
+    def __hash__(self):
+        return hash(self._getval())
+
+    def __eq__(self, other):
+        return self._getval() == other._getval()
+
+    def __ne__(self, other):
+        return not self == other
+
+class CallStack(object):
+    def __init__(self, tb):
+        #if isinstance(tb, py.code.Traceback):
+        #    self.tb = tb
+        #else:
+        #    self.tb = py.code.Traceback(tb)
+        self.tb = [CallFrame(frame) for frame in tb]
+    
+    #def _getval(self):
+    #    return [(frame.code.raw.co_filename, frame.lineno+1) for frame
+    #        in self]
+    
+    def __hash__(self):
+        return hash(tuple(self.tb))
+    
+    def __eq__(self, other):
+        return self.tb == other.tb
+    
+    def __ne__(self, other):
+        return not self == other
+    
+    #def __getattr__(self, attr):
+    #    return getattr(self.tb, attr)
+    
+    def __iter__(self):
+        return iter(self.tb)
+    
+    def __getitem__(self, item):
+        return self.tb[item]
+    
+    def __len__(self):
+        return len(self.tb)
+    
+    def __cmp__(self, other):
+        return cmp(self.tb, other.tb)
+
+def cut_stack(stack, frame, upward_frame=None):
+    if hasattr(frame, 'raw'):
+        frame = frame.raw
+    if upward_frame:
+        if hasattr(upward_frame, 'raw'):
+            upward_frame = upward_frame.raw
+        return CallStack([py.code.Frame(i) for i in stack[stack.index(frame):\
+                stack.index(upward_frame)+1]])
+    return CallStack([py.code.Frame(i) for i in stack[stack.index(frame):]])
+
+##class CallSite(object):
+##    def __init__(self, filename, lineno):
+##        self.filename = filename
+##        self.lineno = lineno
+##    
+##    def get_tuple(self):
+##        return self.filename, self.lineno
+##    
+##    def __hash__(self):
+##        return hash((self.filename, self.lineno))
+##    
+##    def __eq__(self, other):
+##        return (self.filename, self.lineno) == (other.filename, other.lineno)
+##    
+##    def __ne__(self, other):
+##        return not self == other
+##    
+##    def __cmp__(self, other):
+##        if self.filename < other.filename:
+##            return -1
+##        if self.filename > other.filename:
+##            return 1
+##        if self.lineno < other.lineno:
+##            return -1
+##        if self.lineno > other.lineno:
+##            return 1
+##        return 0
+    
+class NonHashableObject(object):
+    def __init__(self, cls):
+        self.cls = cls
+    
+    def __hash__(self):
+        raise NotImplementedError("Object of type %s are unhashable" % self.cls)
+
+class Desc(object):
+    def __init__(self, name, pyobj, **kwargs):
+        self.pyobj = pyobj
+        self.is_degenerated = False
+        self.name = name
+        if type(self) is Desc:
+            # do not override property...
+            self.code = NonHashableObject(self.__class__) # dummy think that makes code unhashable
+    # we make new base class instead of using pypy's one because
+    # of type restrictions of pypy descs
+    
+    def __hash__(self):
+        return hash(self.code)
+    
+    def __eq__(self, other):
+        if isinstance(other, Desc):
+            return self.code == other.code
+        if isinstance(other, types.CodeType):
+            return self.code == other
+        if isinstance(other, tuple) and len(other) == 2:
+            return self.code == other
+        return False
+    
+    def __ne__(self, other):
+        return not self == other
+    # This set of functions will not work on Desc, because we need to
+    # define code somehow
+
+class FunctionDesc(Desc):
+    def __init__(self, *args, **kwargs):
+        super(FunctionDesc, self).__init__(*args, **kwargs)
+        self.inputcells = [model.s_ImpossibleValue for i in xrange(self.\
+            code.co_argcount)]
+        self.call_sites = {}
+        self.keep_frames = kwargs.get('keep_frames', False)
+        self.frame_copier = kwargs.get('frame_copier', lambda x:x)
+        self.retval = model.s_ImpossibleValue
+        self.exceptions = {}
+    
+    def consider_call(self, inputcells):
+        for cell_num, cell in enumerate(inputcells):
+            self.inputcells[cell_num] = model.unionof(cell, self.inputcells[cell_num])
+
+    def consider_call_site(self, frame, cut_frame):
+        if len(self.call_sites) > MAX_CALL_SITES:
+            return
+        stack = [i[0] for i in inspect.stack()]
+        cs = cut_stack(stack, frame, cut_frame)
+        self.call_sites[cs] = cs
+    
+    def consider_exception(self, exc, value):
+        self.exceptions[exc] = True
+    
+    def get_call_sites(self):
+        # convinient accessor for various data which we keep there
+        if not self.keep_frames:
+            return [(key, val) for key, val in self.call_sites.iteritems()]
+        else:
+            lst = []
+            for key, val in self.call_sites.iteritems():
+                for frame in val:
+                    lst.append((key, frame))
+            return lst
+    
+    def consider_return(self, arg):
+        self.retval = model.unionof(arg, self.retval)
+
+    def consider_start_locals(self, frame):
+        pass
+
+    def consider_end_locals(self, frame):
+        pass
+    
+    def getcode(self):
+        return self.pyobj.func_code
+    code = property(getcode)
+    
+    def get_local_changes(self):
+        return {}
+    
+class ClassDesc(Desc):
+    def __init__(self, *args, **kwargs):
+        super(ClassDesc, self).__init__(*args, **kwargs)
+        self.fields = {}
+        # we'll gather informations about methods and possibly
+        # other variables encountered here
+    
+    def getcode(self):
+        # This is a hack. We're trying to return as much close to __init__
+        # of us as possible, but still hashable object
+        if hasattr(self.pyobj, '__init__'):
+            if hasattr(self.pyobj.__init__, 'im_func') and \
+                hasattr(self.pyobj.__init__.im_func, 'func_code'):
+                result = self.pyobj.__init__.im_func.func_code
+            else:
+                result = self.pyobj.__init__
+        else:
+            result = self.pyobj
+        try:
+            hash(result)
+        except KeyboardInterrupt, SystemExit:
+            raise
+        except: # XXX UUuuuu bare except here. What can it really rise???
+            try:
+                hash(self.pyobj)
+                result = self.pyobj
+            except:
+                result = self
+        return result
+    code = property(getcode)
+    
+    def consider_call(self, inputcells):
+        if '__init__' in self.fields:
+            md = self.fields['__init__']
+        else:
+            md = MethodDesc(self.name + '.__init__', self.pyobj.__init__)
+            self.fields['__init__'] = md
+        md.consider_call(inputcells)
+    
+    def consider_return(self, arg):
+        pass # we *know* what return value we do have
+    
+    def consider_exception(self, exc, value):
+        if '__init__' in self.fields:
+            md = self.fields['__init__']
+        else:
+            md = MethodDesc(self.name + '.__init__', self.pyobj.__init__)
+            self.fields['__init__'] = md
+        md.consider_exception(exc, value)
+
+    def consider_start_locals(self, frame):
+        if '__init__' in self.fields:
+            md = self.fields['__init__']
+            md.consider_start_locals(frame)
+
+    def consider_end_locals(self, frame):
+        if '__init__' in self.fields:
+            md = self.fields['__init__']
+            md.consider_end_locals(frame)
+    
+    def consider_call_site(self, frame, cut_frame):
+        self.fields['__init__'].consider_call_site(frame, cut_frame)
+    
+    def add_method_desc(self, name, methoddesc):
+        self.fields[name] = methoddesc
+    
+    def getfields(self):
+        # return fields of values that has been used
+        l = [i for i, v in self.fields.iteritems() if not is_private(i)]
+        return l
+
+    def getbases(self):
+        bases = []
+        tovisit = [self.pyobj]
+        while tovisit:
+            current = tovisit.pop()
+            if current is not self.pyobj:
+                bases.append(current)
+            tovisit += [b for b in current.__bases__ if b not in bases]
+        return bases
+    bases = property(getbases)
+    
+##    def has_code(self, code):
+##        # check __init__ method
+##        return self.pyobj.__init__.im_func.func_code is code
+##    
+##    def consider_call(self, inputcells):
+##        # special thing, make MethodDesc for __init__
+##        
+##
+class MethodDesc(FunctionDesc):
+    def __init__(self, *args, **kwargs):
+        super(MethodDesc, self).__init__(*args, **kwargs)
+        self.old_dict = {}
+        self.changeset = {}
+
+    # right now it's not different than method desc, only code is different
+    def getcode(self):
+        return self.pyobj.im_func.func_code
+    code = property(getcode)
+##    def has_code(self, code):
+##        return self.pyobj.im_func.func_code is code
+
+    def __hash__(self):
+        return hash((self.code, self.pyobj.im_class))
+    
+    def __eq__(self, other):
+        if isinstance(other, tuple):
+            return self.code is other[0] and self.pyobj.im_class is other[1]
+        if isinstance(other, MethodDesc):
+            return self.pyobj is other.pyobj
+        return False
+
+    def consider_start_locals(self, frame):
+        # XXX recursion issues?
+        obj = frame.f_locals[self.pyobj.im_func.func_code.co_varnames[0]]
+        try:
+            if not obj:
+                # static method
+                return
+        except AttributeError:
+            return
+        self.old_dict = self.perform_dict_copy(obj.__dict__)
+
+    def perform_dict_copy(self, d):
+        if d is None:
+            return {}
+        return d.copy()
+
+    def consider_end_locals(self, frame):
+        obj = frame.f_locals[self.pyobj.im_func.func_code.co_varnames[0]]
+        try:
+            if not obj:
+                # static method
+                return
+        except AttributeError:
+            return
+        # store the local changes
+        # update self.changeset
+        self.update_changeset(obj.__dict__)
+
+    def get_local_changes(self):
+        return self.changeset
+    
+    def set_changeset(changeset, key, value):
+        if key not in changeset:
+            changeset[key] = set([value])
+        else:
+            changeset[key].add(value)
+    set_changeset = staticmethod(set_changeset)
+    
+    def update_changeset(self, new_dict):
+        changeset = self.changeset
+        for k, v in self.old_dict.iteritems():
+            if k not in new_dict:
+                self.set_changeset(changeset, k, "deleted")
+            elif new_dict[k] != v:
+                self.set_changeset(changeset, k, "changed")
+        for k, v in new_dict.iteritems():
+            if k not in self.old_dict:
+                self.set_changeset(changeset, k, "created")
+        return changeset
+

py/apigen/tracer/docstorage.py

@@ -0,0 +1,334 @@
+
+""" This module is keeping track about API informations as well as
+providing some interface to easily access stored data
+"""
+
+import py
+import sys
+import types
+import inspect
+
+from py.__.apigen.tracer.description import FunctionDesc, ClassDesc, \
+                                            MethodDesc, Desc
+
+from py.__.apigen.tracer import model
+
+sorted = py.builtin.sorted
+
+class DocStorage(object):
+    """ Class storing info about API
+    """
+    def consider_call(self, frame, caller_frame, upward_cut_frame=None):
+        assert isinstance(frame, py.code.Frame)
+        desc = self.find_desc(frame.code, frame.raw.f_locals)
+        if desc:
+            self.generalize_args(desc, frame)
+            desc.consider_call_site(caller_frame, upward_cut_frame)
+            desc.consider_start_locals(frame)
+
+    def generalize_args(self, desc, frame):
+        args = [arg for key, arg in frame.getargs()]
+        #self.call_stack.append((desc, args))
+        desc.consider_call([model.guess_type(arg) for arg in args])
+    
+    def generalize_retval(self, desc, arg):
+        desc.consider_return(model.guess_type(arg))
+
+    def consider_return(self, frame, arg):
+        assert isinstance(frame, py.code.Frame)
+        desc = self.find_desc(frame.code, frame.raw.f_locals)
+        if desc:
+            self.generalize_retval(desc, arg)
+            desc.consider_end_locals(frame)
+    
+    def consider_exception(self, frame, arg):
+        desc = self.find_desc(frame.code, frame.raw.f_locals)
+        if desc:
+            exc_class, value, _ = arg
+            desc.consider_exception(exc_class, value)
+
+    def find_desc(self, code, locals):
+        try:
+            # argh, very fragile specialcasing
+            return self.desc_cache[(code.raw,
+                                    locals[code.raw.co_varnames[0]].__class__)]
+        except (KeyError, IndexError, AttributeError): # XXX hrmph
+            return self.desc_cache.get(code.raw, None)
+        #for desc in self.descs.values():
+        #    if desc.has_code(frame.code.raw):
+        #        return desc
+        #return None
+    
+    def make_cache(self):
+        self.desc_cache = {}
+        for key, desc in self.descs.iteritems():
+            self.desc_cache[desc] = desc
+    
+    def from_dict(self, _dict, keep_frames = False):
+        self.descs = {}
+        for key, val in _dict.iteritems():
+            to_key, to_val = self.make_desc(key, val)
+            if to_key:
+                self.descs[to_key] = to_val
+        self.make_cache()
+        # XXX
+        return self
+    
+    # XXX: This function becomes slowly outdated and even might go away at some
+    #      point. The question is whether we want to use tracer.magic or not
+    #      at all
+    def add_desc(self, name, value, **kwargs):
+        key = name
+        count = 1
+        while key in self.descs:
+            key = "%s_%d" % (name, count)
+            count += 1
+        key, desc = self.make_desc(key, value, **kwargs)
+        if key:
+            self.descs[key] = desc
+            self.desc_cache[desc] = desc
+            return desc
+        else:
+            return None
+        
+    def make_desc(self, key, value, add_desc=True, **kwargs):
+        if isinstance(value, types.FunctionType):
+            desc = FunctionDesc(key, value, **kwargs)
+        elif isinstance(value, (types.ObjectType, types.ClassType)):
+            desc = ClassDesc(key, value, **kwargs)
+            # XXX: This is the special case when we do not have __init__
+            #      in dir(value) for uknown reason. Need to investigate it
+            for name in dir(value) + ['__init__']:
+                field = getattr(value, name, None)
+                if isinstance(field, types.MethodType) and \
+                    isinstance(field.im_func, types.FunctionType):
+                    real_name = key + '.' + name
+                    md = MethodDesc(real_name, field)
+                    if add_desc: # XXX hack
+                        self.descs[real_name] = md
+                    desc.add_method_desc(name, md)
+                # Some other fields as well?
+        elif isinstance(value, types.MethodType):
+            desc = MethodDesc(key, value, **kwargs)
+        else:
+            desc = Desc(value)
+        return (key, desc) # How to do it better? I want a desc to be a key
+            # value, but I cannot get full object if I do a lookup
+
+    def from_pkg(self, module, keep_frames=False):
+        self.module = module
+        defs = module.__package__.exportdefs
+        d = {}
+        for key, value in defs.iteritems():
+            chain = key.split('.')
+            base = module
+            for elem in chain:
+                base = getattr(base, elem)
+            if value[1] == '*':
+                d.update(self.get_star_import_tree(base, key))
+            else:
+                d[key] = base
+        self.from_dict(d, keep_frames)
+        # XXX
+        return self
+
+    def get_star_import_tree(self, module, modname):
+        """ deal with '*' entries in an initpkg situation """
+        ret = {}
+        modpath = py.path.local(inspect.getsourcefile(module))
+        pkgpath = module.__package__.getpath()
+        for objname in dir(module):
+            if objname.startswith('_'):
+                continue # also skip __*__ attributes
+            obj = getattr(module, objname)
+            if (isinstance(obj, types.ClassType) or
+                    isinstance(obj, types.ObjectType)):
+                try:
+                    sourcefile_object = py.path.local(
+                                            inspect.getsourcefile(obj))
+                except TypeError:
+                    continue
+                else:
+                    if sourcefile_object.strpath != modpath.strpath:
+                        # not in this package
+                        continue
+                dotted_name = '%s.%s' % (modname, objname)
+                ret[dotted_name] = obj
+        return ret
+    
+    def from_module(self, func):
+        raise NotImplementedError("From module")
+
+class AbstractDocStorageAccessor(object):
+    def __init__(self):
+        raise NotImplementedError("Purely virtual object")
+    
+    def get_function_names(self):
+        """ Returning names of all functions
+        """
+    
+    def get_class_names(self):
+        """ Returning names of all classess
+        """
+
+    def get_doc(self, name):
+        """ Returning __doc__ of a function
+        """
+
+    def get_function_definition(self, name):
+        """ Returns definition of a function (source)
+        """
+    
+    def get_function_signature(self, name):
+        """ Returns types of a function
+        """
+
+    def get_function_callpoints(self, name):
+        """ Returns list of all callpoints
+        """
+    
+    def get_module_name(self):
+        pass
+    
+    def get_class_methods(self, name):
+        """ Returns all methods of a class
+        """
+    
+    #def get_object_info(self, key):
+    #    
+    
+    def get_module_info(self):
+        """ Returns module information
+        """
+
+class DocStorageAccessor(AbstractDocStorageAccessor):
+    """ Set of helper functions to access DocStorage, separated in different
+    class to keep abstraction
+    """
+    def __init__(self, ds):
+        self.ds = ds
+
+    def _get_names(self, filter):
+        return [i for i, desc in self.ds.descs.iteritems() if filter(i, desc)]
+
+    def get_function_names(self):
+        return sorted(self._get_names(lambda i, desc: type(desc) is
+                                                      FunctionDesc))
+    
+    def get_class_names(self):
+        return sorted(self._get_names(lambda i, desc: isinstance(desc,
+                                                                 ClassDesc)))
+    
+    #def get_function(self, name):
+    #    return self.ds.descs[name].pyobj
+    
+    def get_doc(self, name):
+        return self.ds.descs[name].pyobj.__doc__ or "*Not documented*"
+    
+    def get_function_definition(self, name):
+        desc = self.ds.descs[name]
+        assert isinstance(desc, FunctionDesc)
+        code = py.code.Code(desc.code)
+        return code.fullsource[code.firstlineno]
+    
+    def get_function_signature(self, name):
+        desc = self.ds.descs[name]
+        # we return pairs of (name, type) here
+        names = desc.pyobj.func_code.co_varnames[
+                    :desc.pyobj.func_code.co_argcount]
+        types = desc.inputcells
+        return zip(names, types), desc.retval
+    
+    def get_function_source(self, name):
+        desc = self.ds.descs[name]
+        try:
+            return str(py.code.Source(desc.pyobj))
+        except IOError:
+            return "Cannot get source"
+
+    def get_function_callpoints(self, name):
+        # return list of tuple (filename, fileline, frame)
+        return self.ds.descs[name].get_call_sites()
+
+    def get_function_local_changes(self, name):
+        return self.ds.descs[name].get_local_changes()
+
+    def get_function_exceptions(self, name):
+        return sorted([i.__name__ for i in self.ds.descs[name].exceptions.keys()])
+
+    def get_module_name(self):
+        if hasattr(self.ds, 'module'):
+            return self.ds.module.__name__
+        return "Unknown module"
+    
+    def get_class_methods(self, name):
+        desc = self.ds.descs[name]
+        assert isinstance(desc, ClassDesc)
+        return sorted(desc.getfields())
+        
+    def get_module_info(self):
+        module = getattr(self.ds, 'module', None)
+        if module is None:
+            return "Lack of module info"
+        try:
+            retval = module.__doc__ or "*undocumented*"
+            retval = module.__package__.description
+            retval = module.__package__.long_description
+        except AttributeError:
+            pass
+        return retval
+
+    def get_type_desc(self, _type):
+        # XXX We provide only classes here
+        if not isinstance(_type, model.SomeClass):
+            return None
+        # XXX we might want to cache it at some point
+        for key, desc in self.ds.descs.iteritems():
+            if desc.pyobj == _type.cls:
+                return key, 'class', desc.is_degenerated
+        return None
+
+    def get_method_origin(self, name):
+        method = self.ds.descs[name].pyobj
+        cls = method.im_class
+        if not cls.__bases__:
+            return self.desc_from_pyobj(cls, cls.__name__)
+        curr = cls
+        while curr:
+            for base in curr.__bases__:
+                basefunc = getattr(base, method.im_func.func_name, None)
+                if (basefunc is not None and hasattr(basefunc, 'im_func') and
+                        hasattr(basefunc.im_func, 'func_code') and
+                        basefunc.im_func.func_code is
+                            method.im_func.func_code):
+                    curr = base
+                    break
+            else:
+                break
+        return self.desc_from_pyobj(curr, curr.__name__)
+
+    def get_possible_base_classes(self, name):
+        cls = self.ds.descs[name].pyobj
+        if not hasattr(cls, '__bases__'):
+            return []
+        retval = []
+        for base in cls.__bases__:
+            desc = self.desc_from_pyobj(base, base.__name__)
+            if desc is not None:
+                retval.append(desc)
+        return retval
+
+    def desc_from_pyobj(self, pyobj, name):
+        for desc in self.ds.descs.values():
+            if isinstance(desc, ClassDesc) and desc.pyobj is pyobj:
+                return desc
+        # otherwise create empty desc
+        key, desc = self.ds.make_desc(name, pyobj, False)
+        #self.ds.descs[key] = desc
+        desc.is_degenerated = True
+        # and make sure we'll not try to link to it directly
+        return desc
+
+    def get_obj(self, name):
+        return self.ds.descs[name].pyobj
+

py/apigen/tracer/magic.py

@@ -0,0 +1,63 @@
+
+""" magic - some operations which helps to extend PDB with some magic data.
+Actually there is only explicit tracking of data, might be extended to
+automatic at some point.
+"""
+
+# some magic stuff to have singleton of DocStorage, but initialised explicitely
+
+import weakref
+
+import py
+from py.__.apigen.tracer.docstorage import DocStorage
+from py.__.apigen.tracer.tracer import Tracer
+import sys
+
+class DocStorageKeeper(object):
+    doc_storage = DocStorage()
+    doc_storage.tracer = Tracer(doc_storage)
+    doc_storage.from_dict({})
+    
+    def set_storage(cl, ds):
+        cl.doc_storage = ds
+        cl.doc_storage.tracer = Tracer(ds)
+    set_storage = classmethod(set_storage)
+
+def get_storage():
+    return DocStorageKeeper.doc_storage
+
+def stack_copier(frame):
+    # copy all stack, not only frame
+    num = 0
+    gather = False
+    stack = []
+    try:
+        while 1:
+            if gather:
+                stack.append(py.code.Frame(sys._getframe(num)))
+            else:
+                if sys._getframe(num) is frame.raw:
+                    gather = True
+            num += 1
+    except ValueError:
+        pass
+    return stack
+
+def trace(keep_frames=False, frame_copier=lambda x:x):
+    def decorator(fun):
+        ds = get_storage()
+        # in case we do not have this function inside doc storage, we
+        # want to have it
+        desc = ds.find_desc(py.code.Code(fun.func_code))
+        if desc is None:
+            desc = ds.add_desc(fun.func_name, fun, keep_frames=keep_frames,
+                frame_copier=frame_copier)
+        
+        def wrapper(*args, **kwargs):
+            ds.tracer.start_tracing()
+            retval = fun(*args, **kwargs)
+            ds.tracer.end_tracing()
+            return retval
+        
+        return wrapper
+    return decorator

py/apigen/tracer/model.py

@@ -0,0 +1,331 @@
+
+""" model - type system model for apigen
+"""
+
+# we implement all the types which are in the types.*, naming
+# scheme after pypy's
+
+import py
+import types
+
+set = py.builtin.set
+
+
+# __extend__ and pairtype?
+class SomeObject(object):
+    typedef = types.ObjectType
+    
+    def __repr__(self):
+        return "<%s>" % self.__class__.__name__[4:]
+        return str(self.typedef)[7:-2]
+    
+    def unionof(self, other):
+        if isinstance(other, SomeImpossibleValue):
+            return self
+        if isinstance(other, SomeUnion):
+            return other.unionof(self)
+        if self == other:
+            return self
+        return SomeUnion([self, other])
+    
+    def gettypedef(self):
+        return self.typedef
+    
+    def __hash__(self):
+        return hash(self.__class__)
+    
+    def __eq__(self, other):
+        return self.__class__ == other.__class__
+    
+    def __ne__(self, other):
+        return not self == other
+        
+    # this is to provide possibility of eventually linking some stuff
+    def striter(self):
+        yield str(self)
+
+class SomeUnion(object):
+    # empty typedef
+    def __init__(self, possibilities):
+        self.possibilities = set(possibilities)
+    
+    def unionof(self, other):
+        if isinstance(other, SomeUnion):
+            return SomeUnion(self.possibilities.union(other.possibilities))
+        return SomeUnion(list(self.possibilities) + [other])
+    
+    def __eq__(self, other):
+        if type(other) is not SomeUnion:
+            return False
+        return self.possibilities == other.possibilities
+    
+    def __ne__(self, other):
+        return not self == other
+    
+    def __repr__(self):
+        return "AnyOf(%s)" % ", ".join([str(i) for i in list(self.possibilities)])
+    
+    def gettypedef(self):
+        return (None, None)
+    
+    def striter(self):
+        yield "AnyOf("
+        for num, i in enumerate(self.possibilities):
+            yield i
+            if num != len(self.possibilities) - 1:
+                yield ", "
+        yield ")"
+
+class SomeBoolean(SomeObject):
+    typedef = types.BooleanType
+
+class SomeBuffer(SomeObject):
+    typedef = types.BufferType
+
+class SomeBuiltinFunction(SomeObject):
+    typedef = types.BuiltinFunctionType
+
+#class SomeBuiltinMethod(SomeObject):
+#    typedef = types.BuiltinMethodType
+
+class SomeClass(SomeObject):
+    typedef = types.ClassType
+    
+    def __init__(self, cls):
+        self.cls = cls
+        self.name = cls.__name__
+        self.id = id(cls)
+
+    def __getstate__(self):
+        return (self.name, self.id)
+
+    def __setstate__(self, state):
+        self.name, self.id = state
+        self.cls = None
+    
+    def __hash__(self):
+        return hash("Class") ^ hash(self.id)
+    
+    def __eq__(self, other):
+        if type(other) is not SomeClass:
+            return False
+        return self.id == other.id
+    
+    def unionof(self, other):
+        if type(other) is not SomeClass or self.id is not other.id:
+            return super(SomeClass, self).unionof(other)
+        return self
+    
+    def __repr__(self):
+        return "Class %s" % self.name
+    
+class SomeCode(SomeObject):
+    typedef = types.CodeType
+
+class SomeComplex(SomeObject):
+    typedef = types.ComplexType
+
+class SomeDictProxy(SomeObject):
+    typedef = types.DictProxyType
+
+class SomeDict(SomeObject):
+    typedef = types.DictType
+
+class SomeEllipsis(SomeObject):
+    typedef = types.EllipsisType
+
+class SomeFile(SomeObject):
+    typedef = types.FileType
+
+class SomeFloat(SomeObject):
+    typedef = types.FloatType
+
+class SomeFrame(SomeObject):
+    typedef = types.FrameType
+
+class SomeFunction(SomeObject):
+    typedef = types.FunctionType
+
+class SomeGenerator(SomeObject):
+    typedef = types.GeneratorType
+
+class SomeInstance(SomeObject):
+    def __init__(self, classdef):
+        self.classdef = classdef
+        
+    def __hash__(self):
+        return hash("SomeInstance") ^ hash(self.classdef)
+    
+    def __eq__(self, other):
+        if type(other) is not SomeInstance:
+            return False
+        return other.classdef == self.classdef
+    
+    def unionof(self, other):
+        if type(other) is not SomeInstance:
+            return super(SomeInstance, self).unionof(other)
+        if self.classdef == other.classdef:
+            return self
+        return SomeInstance(unionof(self.classdef, other.classdef))
+    
+    def __repr__(self):
+        return "<Instance of %s>" % str(self.classdef)
+    
+    def striter(self):
+        yield "<Instance of "
+        yield self.classdef
+        yield ">"
+    
+    typedef = types.InstanceType
+
+class SomeInt(SomeObject):
+    typedef = types.IntType
+
+class SomeLambda(SomeObject):
+    typedef = types.LambdaType
+
+class SomeList(SomeObject):
+    typedef = types.ListType
+
+class SomeLong(SomeObject):
+    typedef = types.LongType
+
+class SomeMethod(SomeObject):
+    typedef = types.MethodType
+
+class SomeModule(SomeObject):
+    typedef = types.ModuleType
+
+class SomeNone(SomeObject):
+    typedef = types.NoneType
+
+class SomeNotImplemented(SomeObject):
+    typedef = types.NotImplementedType
+
+class SomeObject(SomeObject):
+    typedef = types.ObjectType
+
+class SomeSlice(SomeObject):
+    typedef = types.SliceType
+
+class SomeString(SomeObject):
+    typedef = types.StringType
+
+class SomeTraceback(SomeObject):
+    typedef = types.TracebackType
+
+class SomeTuple(SomeObject):
+    typedef = types.TupleType
+
+class SomeType(SomeObject):
+    typedef = types.TypeType
+
+class SomeUnboundMethod(SomeObject):
+    typedef = types.UnboundMethodType
+
+class SomeUnicode(SomeObject):
+    typedef = types.UnicodeType
+
+class SomeXRange(SomeObject):
+    typedef = types.XRangeType
+
+class SomeImpossibleValue(SomeObject):
+    def unionof(self, other):
+        return other
+    
+    def __repr__(self):
+        return "<UNKNOWN>"
+
+s_ImpossibleValue = SomeImpossibleValue()
+s_None = SomeNone()
+s_Ellipsis = SomeEllipsis()
+
+def guess_type(x):
+    # this is mostly copy of immutablevalue
+    if hasattr(x, 'im_self') and x.im_self is None:
+        x = x.im_func
+        assert not hasattr(x, 'im_self')
+    tp = type(x)
+    if tp is bool:
+        result = SomeBoolean()
+    elif tp is int:
+        result = SomeInt()
+    elif issubclass(tp, str):
+        result = SomeString()
+    elif tp is unicode:
+        result = SomeUnicode()
+    elif tp is tuple:
+        result = SomeTuple()
+        #result = SomeTuple(items = [self.immutablevalue(e, need_const) for e in x])
+    elif tp is float:
+        result = SomeFloat()
+    elif tp is list:
+        #else:
+        #    listdef = ListDef(self, s_ImpossibleValue)
+        #    for e in x:
+        #        listdef.generalize(self.annotation_from_example(e))
+        result = SomeList()
+    elif tp is dict:
+##        dictdef = DictDef(self, 
+##        s_ImpossibleValue,
+##        s_ImpossibleValue,
+##        is_r_dict = tp is r_dict)
+##        if tp is r_dict:
+##            s_eqfn = self.immutablevalue(x.key_eq)
+##            s_hashfn = self.immutablevalue(x.key_hash)
+##            dictdef.dictkey.update_rdict_annotations(s_eqfn,
+##                s_hashfn)
+##        for ek, ev in x.iteritems():
+##            dictdef.generalize_key(self.annotation_from_example(ek))
+##            dictdef.generalize_value(self.annotation_from_example(ev))
+        result = SomeDict()
+    elif tp is types.ModuleType:
+        result = SomeModule()
+    elif callable(x):
+        #if hasattr(x, '__self__') and x.__self__ is not None:
+        #    # for cases like 'l.append' where 'l' is a global constant list
+        #    s_self = self.immutablevalue(x.__self__, need_const)
+        #    result = s_self.find_method(x.__name__)
+        #    if result is None:
+        #        result = SomeObject()
+        #elif hasattr(x, 'im_self') and hasattr(x, 'im_func'):
+        #    # on top of PyPy, for cases like 'l.append' where 'l' is a
+        #    # global constant list, the find_method() returns non-None
+        #    s_self = self.immutablevalue(x.im_self, need_const)
+        #    result = s_self.find_method(x.im_func.__name__)
+        #else:
+        #    result = None
+        #if result is None:
+        #    if (self.annotator.policy.allow_someobjects
+        #        and getattr(x, '__module__', None) == '__builtin__'
+        #        # XXX note that the print support functions are __builtin__
+        #        and tp not in (types.FunctionType, types.MethodType)):
+        ##        result = SomeObject()
+        #        result.knowntype = tp # at least for types this needs to be correct
+        #    else:
+        #        result = SomePBC([self.getdesc(x)])
+        if tp is types.BuiltinFunctionType or tp is types.BuiltinMethodType:
+            result = SomeBuiltinFunction()
+        elif hasattr(x, 'im_func'):
+            result = SomeMethod()
+        elif hasattr(x, 'func_code'):
+            result = SomeFunction()
+        elif hasattr(x, '__class__'):
+            if x.__class__ is type:
+                result = SomeClass(x)
+            else:
+                result = SomeInstance(SomeClass(x.__class__))
+        elif tp is types.ClassType:
+            result = SomeClass(x)
+    elif x is None:
+        return s_None
+    elif hasattr(x, '__class__'):
+        result = SomeInstance(SomeClass(x.__class__))
+    else:
+        result = SomeObject()
+    # XXX here we might want to consider stuff like
+    # buffer, slice, etc. etc. Let's leave it for now
+    return result
+
+def unionof(first, other):
+    return first.unionof(other)

py/apigen/tracer/permastore.py

@@ -0,0 +1,106 @@
+import py
+
+class DescPlaceholder(object):
+    pass
+
+class ClassPlaceholder(object):
+    pass
+
+class SerialisableClassDesc(object):
+    def __init__(self, original_desc):
+        self.is_degenerated = original_desc.is_degenerated
+        self.name = original_desc.name
+
+class PermaDocStorage(object):
+    """ Picklable version of docstorageaccessor
+    """
+    function_fields = ['source', 'signature', 'definition', 'callpoints',
+                       'local_changes', 'exceptions']
+    
+    def __init__(self, dsa):
+        """ Initialise from original doc storage accessor
+        """
+        self.names = {}
+        self.module_info = dsa.get_module_info()
+        self.module_name = dsa.get_module_name()
+        self._save_functions(dsa)
+        self._save_classes(dsa)
+
+    def _save_functions(self, dsa):
+        names = dsa.get_function_names()
+        self.function_names = names
+        for name in names:
+            self._save_function(dsa, name)
+
+    def _save_function(self, dsa, name):
+        ph = DescPlaceholder()
+        ph.__doc__ = dsa.get_doc(name)
+        for field in self.function_fields:
+            setattr(ph, field, getattr(dsa, 'get_function_%s' % field)(name))
+        self.names[name] = ph
+        return ph
+
+    def _save_classes(self, dsa):
+        names = dsa.get_class_names()
+        self.class_names = names
+        for name in names:
+            ph = ClassPlaceholder()
+            ph.__doc__ = dsa.get_doc(name)
+            methods = dsa.get_class_methods(name)
+            ph.methods = methods
+            ph.base_classes = [SerialisableClassDesc(i) for i in
+                               dsa.get_possible_base_classes(name)]
+
+            for method in methods:
+                method_name = name + "." + method
+                mh = self._save_function(dsa, name + "." + method)
+                mh.origin = SerialisableClassDesc(dsa.get_method_origin(
+                    method_name))
+            self.names[name] = ph
+
+    def get_class_methods(self, name):
+        desc = self.names[name]
+        assert isinstance(desc, ClassPlaceholder)
+        return desc.methods
+
+    def get_doc(self, name):
+        return self.names[name].__doc__
+
+    def get_module_info(self):
+        return self.module_info
+
+    def get_module_name(self):
+        return self.module_name
+
+    def get_class_names(self):
+        return self.class_names
+
+    def get_function_names(self):
+        return self.function_names
+
+    def get_method_origin(self, name):
+        # returns a DESCRIPTION of a method origin, to make sure where we
+        # write it
+        return self.names[name].origin
+
+    def get_possible_base_classes(self, name):
+        # returns list of descs of base classes
+        return self.names[name].base_classes
+
+    # This are placeholders to provide something more reliable
+    def get_type_desc(self, _type):
+        return None
+
+    #def get_obj(self, name):
+    #    This is quite hairy, get rid of it soon
+    #    # returns a pyobj
+    #    pass
+
+for field in PermaDocStorage.function_fields:
+    d = {"field": field}
+    func_name = "get_function_%s" % (field, )
+    exec py.code.Source("""
+    def %s(self, name, field=field):
+        return getattr(self.names[name], field)
+""" % (func_name, )).compile() in d
+    setattr(PermaDocStorage, func_name, d[func_name])

py/apigen/tracer/testing/package/submodule/__init__.py

@@ -0,0 +1,13 @@
+import py
+
+from py.__.initpkg import initpkg
+
+initpkg(__name__,
+        description="test package",
+        exportdefs = {
+         'pak.mod.one': ('./pak/mod.py', 'one'),
+         'pak.mod.two': ('./pak/mod.py', 'nottwo'),
+         'notpak.notmod.notclass': ('./pak/mod.py', 'cls'),
+         'somenamespace': ('./pak/somenamespace.py', '*'),
+        })
+

py/apigen/tracer/testing/package/submodule/pak/mod.py

@@ -0,0 +1,10 @@
+class cls(object):
+    def __init__(self, x):
+        self.x = x
+
+def one(x):
+    return x+3
+
+def nottwo():
+    pass
+

py/apigen/tracer/testing/package/submodule/pak/somenamespace.py

@@ -0,0 +1,5 @@
+def foo(x):
+    return x + 1
+
+def bar(x):
+    return x + 2

py/apigen/tracer/testing/runtest.py

@@ -0,0 +1,5 @@
+
+def cut_pyc(f_name):
+    if f_name.endswith('.pyc'):
+        return f_name[:-1]
+    return f_name

py/apigen/tracer/testing/test_desc.py

@@ -0,0 +1,28 @@
+
+""" Some additional tests about descriptions
+"""
+
+from py.__.apigen.tracer.description import *
+
+class A:
+    pass
+
+class B(object):
+    def __init__(self):
+        pass
+
+class C(object):
+    pass
+
+class D:
+    def __init__(self):
+        pass
+
+def test_getcode():
+    assert hash(ClassDesc("a", A).code)
+    assert hash(ClassDesc("b", B).code)
+    assert hash(ClassDesc("c", C).code)
+    assert hash(ClassDesc("d", D).code)
+
+def test_eq():
+    assert ClassDesc('a', A) == ClassDesc('a', A)

py/apigen/tracer/testing/test_docgen.py

@@ -0,0 +1,420 @@
+
+""" test doc generation
+"""
+
+import py
+import sys
+
+#try:
+from py.__.apigen.tracer.tracer import DocStorage, Tracer
+from py.__.apigen.tracer.docstorage import DocStorageAccessor
+from py.__.apigen.tracer.testing.runtest import cut_pyc
+from py.__.apigen.tracer.description import FunctionDesc
+from py.__.apigen.tracer import model
+from py.__.apigen.tracer.permastore import PermaDocStorage
+#    from pypy.annotation import model
+#except ImportError, s:
+#    py.test.skip("Cannot import: %s" % str(s))
+
+#def setup_module(mod):
+#    data_path = py.path.local(mod.__file__).dirpath().join("data")
+#    sys.path.insert(0, str(data_path))
+
+# XXX: Perma doc storage disabled a bit
+
+sorted = py.builtin.sorted
+set = py.builtin.set
+
+def fun(a, b, c):
+    "Some docstring"
+    return "d"
+
+def test_basic():
+    descs = {"fun":fun}
+    ds = DocStorage().from_dict(descs)
+    t = Tracer(ds)
+    t.start_tracing()
+    fun(1, ("g", 3), 8)
+    fun(2., ("a", 1.), "a")
+    t.end_tracing()
+    desc = ds.descs['fun']
+    inputcells = desc.inputcells
+    assert len(inputcells) == 3
+    assert isinstance(inputcells[0], model.SomeUnion)
+    assert isinstance(inputcells[1], model.SomeTuple)
+    assert isinstance(inputcells[2], model.SomeUnion)
+    assert isinstance(desc.retval, model.SomeString)
+    cs = sorted(desc.call_sites.keys())
+    assert len(cs) == 2
+    f_name = cut_pyc(__file__)
+    assert len(cs[0]) == 1
+    assert len(cs[1]) == 1
+    assert cs[1][0].filename == f_name
+    # lines are counted from 0
+    num = test_basic.func_code.co_firstlineno
+    assert cs[1][0].lineno == num + 4 or cs[1][0].lineno == num + 5
+    assert cs[0][0].filename == f_name
+    assert cs[0][0].lineno == num + 5 or cs[0][0].lineno == num + 4
+    if 0:
+        pds = PermaDocStorage(DocStorageAccessor(ds))
+        assert pds.get_function_names() == ['fun']
+        sig = pds.get_function_signature('fun')
+        assert sig[0][0][0] == 'a'
+        assert isinstance(sig[0][0][1], model.SomeUnion)
+        assert len(pds.get_function_callpoints('fun')) == 2
+
+class AClass(object):
+    """ Class docstring
+    """
+    def __init__(self, b="blah"):
+        pass
+    
+    def exposed_method(self, a, b, c):
+        """ method docstring
+        """
+        return self._hidden_method()
+    
+    def _hidden_method(self):
+        """ should not appear
+        """
+        return "z"
+
+class ANotherClass(AClass):
+    def another_exposed_method(self, a):
+        # no docstring
+        return a
+
+def test_class():
+    descs = {'AClass':AClass}
+    ds = DocStorage().from_dict(descs)
+    t = Tracer(ds)
+    t.start_tracing()
+    s = AClass()
+    s.exposed_method(1, 2., [1,2,3])
+    t.end_tracing()
+    desc = ds.descs['AClass']
+    inputcells = desc.fields['__init__'].inputcells
+    assert len(inputcells) == 2
+    assert isinstance(inputcells[0], model.SomeInstance)
+    #assert inputcells[0].classdef.classdesc.pyobj is SomeClass
+    # XXX: should work
+    assert isinstance(inputcells[1], model.SomeString)
+    f_name = __file__
+    if f_name.endswith('.pyc'):
+        f_name = f_name[:-1]
+    cs = sorted(desc.fields['__init__'].call_sites.keys())
+    assert len(cs) == 1
+    assert len(cs[0]) == 1
+    assert cs[0][0].filename == f_name
+    assert cs[0][0].lineno == test_class.func_code.co_firstlineno + 4
+    # method check
+    assert sorted(desc.getfields()) == ['__init__', 'exposed_method']
+    inputcells = desc.fields['exposed_method'].inputcells
+    assert len(inputcells) == 4
+    assert isinstance(inputcells[0], model.SomeInstance)
+    #assert inputcells[0].classdef.classdesc.pyobj is SomeClass
+    # XXX should work
+    assert isinstance(inputcells[1], model.SomeInt)
+    assert isinstance(inputcells[2], model.SomeFloat)
+    assert isinstance(inputcells[3], model.SomeList)
+    assert isinstance(desc.fields['exposed_method'].retval, model.SomeString)
+    if 0:
+        pds = PermaDocStorage(DocStorageAccessor(ds))
+        assert pds.get_class_names() == ['AClass']
+        assert len(pds.get_function_signature('AClass.exposed_method')[0]) == 4
+
+def other_fun():
+    pass
+
+def test_add_desc():
+    ds = DocStorage().from_dict({})
+    ds.add_desc("one", fun)
+    ds.add_desc("one", other_fun)
+    assert sorted(ds.descs.keys()) == ["one", "one_1"]
+    assert isinstance(ds.descs["one"], FunctionDesc)
+    assert isinstance(ds.descs["one_1"], FunctionDesc)
+    assert ds.descs["one"].pyobj is fun
+    assert ds.descs["one_1"].pyobj is other_fun
+    assert ds.desc_cache[ds.descs["one"]] is ds.descs["one"]
+    assert ds.desc_cache[ds.descs["one_1"]] is ds.descs["one_1"]
+
+def test_while_call():
+    ds = DocStorage().from_dict({"other_fun":other_fun})
+    t = Tracer(ds)
+    t.start_tracing()
+    for x in xrange(8):
+        other_fun()
+    t.end_tracing()
+    desc = ds.descs["other_fun"]
+    assert len(desc.call_sites.keys()) == 1
+    #assert isinstance(desc.call_sites.values()[0][0], py.code.Frame)
+    if 0:
+        pds = PermaDocStorage(DocStorageAccessor(ds))
+        assert len(pds.get_function_callpoints("other_fun")) == 1
+
+class A(object):
+    def method(self, x):
+        self.x = x
+
+class B:
+    def method(self, x):
+        self.x = x
+
+def test_without_init():
+    ds = DocStorage().from_dict({'A':A, 'B':B})
+    t = Tracer(ds)
+    t.start_tracing()
+    x = A()
+    y = B()
+    x.method(3)
+    y.method(4)
+    t.end_tracing()
+    assert isinstance(ds.descs['A'].fields['method'].inputcells[1],
+                      model.SomeInt)
+    assert isinstance(ds.descs['B'].fields['method'].inputcells[1],
+                      model.SomeInt)
+    if 0:
+        pds = PermaDocStorage(DocStorageAccessor(ds))
+
+def test_local_changes():
+    class testclass(object):
+        def __init__(self):
+            self.foo = 0
+        def bar(self, x):
+            self.foo = x
+    ds = DocStorage().from_dict({'testclass': testclass})
+    t = Tracer(ds)
+    t.start_tracing()
+    c = testclass()
+    c.bar(1)
+    t.end_tracing()
+    desc = ds.descs['testclass']
+    methdesc = desc.fields['bar']
+    #assert methdesc.old_dict != methdesc.new_dict
+    assert methdesc.get_local_changes() == {'foo': set(['changed'])}
+    return ds
+
+def test_local_changes_nochange():
+    class testclass(object):
+        def __init__(self):
+            self.foo = 0
+        def bar(self, x):
+            self.foo = x
+    ds = DocStorage().from_dict({'testclass': testclass})
+    t = Tracer(ds)
+    t.start_tracing()
+    c = testclass()
+    t.end_tracing()
+    desc = ds.descs['testclass']
+    methdesc = desc.fields['bar']
+    assert methdesc.get_local_changes() == {}
+    return ds
+
+def test_multiple_classes_with_same_init():
+    class A:
+        def __init__(self, x):
+            self.x = x
+    
+    class B(A):
+        pass
+    
+    ds = DocStorage().from_dict({'A':A, 'B':B})
+    t = Tracer(ds)
+    t.start_tracing()
+    c = A(3)
+    d = B(4)
+    t.end_tracing()
+    assert len(ds.descs['A'].fields['__init__'].call_sites) == 1
+    assert len(ds.descs['B'].fields['__init__'].call_sites) == 1
+    return ds
+
+def test_exception_raise():
+    def x():
+        1/0
+    
+    def y():
+        try:
+            x()
+        except ZeroDivisionError:
+            pass
+    
+    def z():
+        y()
+    
+    ds = DocStorage().from_dict({'x':x, 'y':y, 'z':z})
+    t = Tracer(ds)
+    t.start_tracing()
+    z()
+    t.end_tracing()
+    assert ds.descs['x'].exceptions.keys() == [ZeroDivisionError]
+    assert ds.descs['y'].exceptions.keys() == [ZeroDivisionError]
+    assert ds.descs['z'].exceptions.keys() == []
+    return ds
+
+def test_subclass():
+    descs = {'ANotherClass': ANotherClass}
+    ds = DocStorage().from_dict(descs)
+    t = Tracer(ds)
+    t.start_tracing()
+    s = ANotherClass('blah blah')
+    s.another_exposed_method(1)
+    t.end_tracing()
+    desc = ds.descs['ANotherClass']
+    assert len(desc.fields) == 4
+    inputcells = desc.fields['__init__'].inputcells
+    assert len(inputcells) == 2
+    inputcells = desc.fields['another_exposed_method'].inputcells
+    assert len(inputcells) == 2
+    bases = desc.bases
+    assert len(bases) == 2
+    return ds
+    
+def test_bases():
+    class A:
+        pass
+    
+    class B:
+        pass
+    
+    class C(A,B):
+        pass
+    
+    ds = DocStorage().from_dict({'C':C, 'B':B})
+    dsa = DocStorageAccessor(ds)
+    for desc in dsa.get_possible_base_classes('C'):
+        assert desc is ds.descs['B'] or desc.is_degenerated
+    return ds
+
+def test_desc_from_pyobj():
+    class A:
+        pass
+
+    class B(A):
+        pass
+
+    ds = DocStorage().from_dict({'A': A, 'B': B})
+    dsa = DocStorageAccessor(ds)
+    assert dsa.desc_from_pyobj(A, 'A') is ds.descs['A']
+    return ds
+
+def test_method_origin():
+    class A:
+        def foo(self):
+            pass
+
+    class B(A):
+        def bar(self):
+            pass
+
+    class C(B):
+        pass
+
+    ds = DocStorage().from_dict({'C': C, 'B': B})
+    dsa = DocStorageAccessor(ds)
+    origin = dsa.get_method_origin('C.bar')
+    assert origin is ds.descs['B']
+    return ds
+
+def test_multiple_methods():
+    class A(object):
+        def meth(self):
+            pass
+    
+    class B(A):
+        pass
+    
+    class C(A):
+        pass
+    
+    ds = DocStorage().from_dict({'C':C, 'B':B})
+    dsa = DocStorageAccessor(ds)
+    t = Tracer(ds)
+    t.start_tracing()
+    B().meth()
+    C().meth()
+    t.end_tracing()
+    assert len(ds.descs['B'].fields['meth'].call_sites) == 1
+    assert len(ds.descs['C'].fields['meth'].call_sites) == 1
+    return ds
+
+def test_is_private():
+    # XXX implicit test, but so are the rest :|
+    class Foo(object):
+        def foo(self):
+            pass
+        def _foo(self):
+            pass
+        def __foo(self):
+            pass
+        def trigger__foo(self):
+            self.__foo()
+        def __foo__(self):
+            pass
+
+    ds = DocStorage().from_dict({'Foo': Foo})
+    dsa = DocStorageAccessor(ds)
+    t = Tracer(ds)
+    t.start_tracing()
+    f = Foo()
+    f.foo()
+    f._foo()
+    f.trigger__foo()
+    f.__foo__()
+    t.end_tracing()
+    assert sorted(ds.descs['Foo'].getfields()) == ['__foo__', 'foo',
+                                                     'trigger__foo']
+
+def setup_fs_project():
+    temp = py.test.ensuretemp('test_get_initpkg_star_items')
+    temp.ensure("pkg/func.py").write(py.code.Source("""\
+        def func(arg1):
+            "docstring"
+    """))
+    temp.ensure('pkg/someclass.py').write(py.code.Source("""\
+        class SomeClass(object):
+            " docstring someclass "
+            def __init__(self, somevar):
+                self.somevar = somevar
+                
+            def get_somevar(self):
+                " get_somevar docstring "
+                return self.somevar
+        SomeInstance = SomeClass(10)
+    """))
+    temp.ensure('pkg/somesubclass.py').write(py.code.Source("""\
+        from someclass import SomeClass
+        class SomeSubClass(SomeClass):
+            " docstring somesubclass "
+            def get_somevar(self):
+                return self.somevar + 1
+    """))
+    temp.ensure('pkg/somenamespace.py').write(py.code.Source("""\
+        from pkg.main.sub import func
+    
+        def foo():
+            return 'bar'
+        def baz(qux):
+            return qux
+    """))
+    temp.ensure("pkg/__init__.py").write(py.code.Source("""\
+        from py.initpkg import initpkg
+        initpkg(__name__, exportdefs = {
+            'main.sub.func': ("./func.py", "func"),
+            'main.SomeClass': ('./someclass.py', 'SomeClass'),
+            'main.SomeInstance': ('./someclass.py', 'SomeInstance'),
+            'main.SomeSubClass': ('./somesubclass.py', 'SomeSubClass'),
+            'main.SomeSubClass': ('./somesubclass.py', 'SomeSubClass'),
+            'other':             ('./somenamespace.py', '*'),
+        })
+    """))
+    return temp, 'pkg'
+
+def test_get_initpkg_star_items():
+    pkgdir, pkgname = setup_fs_project()
+    py.std.sys.path.insert(0, str(pkgdir))
+    pkg = __import__(pkgname)
+    ds = DocStorage().from_pkg(pkg)
+    sit = ds.get_star_import_tree(pkg.other, 'pkg.other')
+    print sit
+    assert sorted(sit.keys()) == ['pkg.other.baz', 'pkg.other.foo']
+

py/apigen/tracer/testing/test_magic.py

@@ -0,0 +1,48 @@
+
+""" test magic abilities of tracer
+"""
+
+import py
+py.test.skip("These features has been disabled")
+
+from py.__.apigen.tracer.magic import trace, get_storage, stack_copier, \
+    DocStorageKeeper
+from py.__.apigen.tracer.docstorage import DocStorage
+from py.__.apigen.tracer import model
+
+#def setup_function(f):
+#    DocStorageKeeper.set_storage(DocStorage().from_dict({}))
+
+def fun(a, b, c):
+    return "a"
+fun = trace()(fun)
+
+def test_magic():
+    fun(1, 2, 3)
+    
+    ds = get_storage()
+    assert 'fun' in ds.descs
+    assert len(ds.descs.keys()) == 2
+    desc = ds.descs['fun']
+    inputcells = desc.inputcells
+    assert isinstance(inputcells[0], model.SomeInt)
+    assert isinstance(inputcells[1], model.SomeInt)
+    assert isinstance(inputcells[2], model.SomeInt)
+    assert isinstance(desc.retval, model.SomeString)
+
+def g(x):
+    return f(x)
+
+def f(x):
+    return x + 3
+f = trace(keep_frames=True, frame_copier=stack_copier)(f)
+
+def test_fancy_copier():
+    g(1)
+    
+    ds = get_storage()
+    assert 'f' in ds.descs
+    desc = ds.descs['f']
+    stack = desc.call_sites.values()[0][0]
+    assert str(stack[0].statement) == '    return f(x)'
+    assert str(stack[1].statement) == '    g(1)'

py/apigen/tracer/testing/test_model.py

@@ -0,0 +1,112 @@
+
+""" test_model - our (very simple) type system
+model tests
+"""
+
+from py.__.apigen.tracer.model import *
+
+import types
+import py
+
+def check_guess(val, t):
+    assert isinstance(guess_type(val), t)
+
+def test_basic():
+    """ This tests checks every object that we might want
+    to track
+    """
+    check_guess(3, SomeInt)
+    check_guess(3., SomeFloat)
+    check_guess(True, SomeBoolean)
+    check_guess(lambda x: None, SomeFunction)
+
+    class A:
+        pass
+
+    check_guess(A, SomeClass)
+    check_guess(A(), SomeInstance)
+
+    class B(object):
+        def meth(self):
+            pass
+    
+    class C(object):
+        def __call__(self):
+            pass
+
+    check_guess(B, SomeClass)
+    check_guess(B.meth, SomeFunction)
+    check_guess(B(), SomeInstance)
+    check_guess(B().meth, SomeMethod)
+    check_guess([1], SomeList)
+    check_guess(None, SomeNone)
+    check_guess((1,), SomeTuple)
+    check_guess(C(), SomeInstance)
+    import sys
+    check_guess(sys, SomeModule)
+    check_guess({}, SomeDict)
+    check_guess(sys.exc_info, SomeBuiltinFunction)
+
+def test_anyof():
+    def check_lst(lst):
+        a = guess_type(lst[0])
+        for i in lst[1:]:
+            a = unionof(a, guess_type(i))
+        d = dict([(i, True) for i in a.possibilities])
+        assert len(a.possibilities) == len(d)
+        for i in a.possibilities:
+            assert not isinstance(i, SomeUnion)
+        return a
+        
+    class C(object):
+        pass
+    
+    ret = check_lst([3, 4, 3., "aa"])
+    assert len(ret.possibilities) == 3
+    ret = check_lst([3, 4, 3.])
+    ret2 = check_lst([1, "aa"])
+    ret3 = unionof(ret, ret2)
+    assert len(ret3.possibilities) == 3
+    ret = check_lst([3, 1.])
+    ret = unionof(ret, guess_type("aa"))
+    ret = unionof(guess_type("aa"), ret)
+    ret = unionof(guess_type(C()), ret)
+    ret = unionof(ret, guess_type("aa"))
+    ret = unionof(ret, guess_type(C()))
+    assert len(ret.possibilities) == 4
+
+def test_union():
+    class A(object):
+        pass
+    
+    class B(object):
+        pass
+    
+    f = guess_type(A).unionof(guess_type(A))
+    assert isinstance(f, SomeClass)
+    assert f.cls is A
+    f = guess_type(A).unionof(guess_type(B)).unionof(guess_type(A))
+    assert isinstance(f, SomeUnion)
+    assert len(f.possibilities) == 2
+    f = guess_type(A()).unionof(guess_type(A()))
+    assert isinstance(f, SomeInstance)
+    assert isinstance(f.classdef, SomeClass)
+    assert f.classdef.cls is A
+    f = guess_type(B()).unionof(guess_type(A())).unionof(guess_type(B()))
+    assert isinstance(f, SomeInstance)
+    assert isinstance(f.classdef, SomeUnion)
+    assert len(f.classdef.possibilities) == 2
+    
+def test_striter():
+    class A(object):
+        pass
+    
+    class B(object):
+        pass
+    
+    g = guess_type(A).unionof(guess_type(A()))
+    l = py.builtin.sorted(list(g.striter()))
+    assert l[4] == "AnyOf("
+    assert isinstance(l[0], SomeClass)
+    assert l[3] == ", "
+    assert isinstance(l[1], SomeInstance)

py/apigen/tracer/testing/test_package.py

@@ -0,0 +1,51 @@
+
+""" some tests for from_package
+"""
+
+from py.__.apigen.tracer.docstorage import DocStorage
+from py.__.apigen.tracer.tracer import Tracer
+from py.__.apigen.tracer import model
+import sys
+import py
+
+def setup_module(mod):
+    sys.path.insert(0, str(py.path.local(__file__).dirpath().join("package")))
+    import submodule
+    mod.submodule = submodule
+
+def teardown_module(mod):
+    sys.path = sys.path[1:]
+
+class TestFullModule(object):
+    def setup_class(cls):
+        cls.ds = DocStorage().from_pkg(submodule)
+        cls.tracer = Tracer(cls.ds)
+    
+    def test_init(self):
+        ds = self.ds
+        print sorted(ds.descs.keys())
+        assert len(ds.descs) == 6
+        assert py.builtin.sorted(ds.descs.keys()) == [
+                'notpak.notmod.notclass', 'notpak.notmod.notclass.__init__',
+                'pak.mod.one', 'pak.mod.two', 'somenamespace.bar',
+                'somenamespace.foo']
+
+    def test_simple_call(self):
+        ds = self.ds
+        self.tracer.start_tracing()
+        submodule.pak.mod.one(3)
+        self.tracer.end_tracing()
+        desc = self.ds.descs['pak.mod.one']
+        assert isinstance(desc.retval, model.SomeInt)
+        assert isinstance(desc.inputcells[0], model.SomeInt)
+    
+    def test_call_class(self):
+        ds = self.ds
+        self.tracer.start_tracing()
+        c = submodule.notpak.notmod.notclass(3)
+        self.tracer.end_tracing()
+        desc = self.ds.descs['notpak.notmod.notclass']
+        methdesc = desc.fields['__init__']
+        assert isinstance(methdesc.inputcells[0], model.SomeInstance)
+        assert isinstance(methdesc.inputcells[1], model.SomeInt)
+

py/apigen/tracer/tracer.py

@@ -0,0 +1,55 @@
+
+""" simple tracer for API generation
+"""
+
+import py
+import sys
+import types
+
+from py.__.apigen.tracer.description import FunctionDesc
+from py.__.apigen.tracer.docstorage import DocStorage
+
+class UnionError(Exception):
+    pass
+
+class NoValue(object):
+    pass
+
+class Tracer(object):
+    """ Basic tracer object, used for gathering additional info
+    about API functions
+    """
+    def __init__(self, docstorage):
+        self.docstorage = docstorage
+        self.tracing = False
+    
+    _locals = {}
+    def _tracer(self, frame, event, arg):
+        
+        # perform actuall tracing
+        frame = py.code.Frame(frame)
+        if event == 'call':
+            assert arg is None
+            try:
+                self.docstorage.consider_call(frame,
+                                          py.code.Frame(sys._getframe(2)),
+                                          self.frame)
+            except ValueError:
+                self.docstorage.consider_call(frame, None, self.frame)
+        elif event == 'return':
+            self.docstorage.consider_return(frame, arg)
+        elif event == 'exception':
+            self.docstorage.consider_exception(frame, arg)
+        return self._tracer
+    
+    def start_tracing(self):
+        if self.tracing:
+            return
+        self.tracing = True
+        self.frame = py.code.Frame(sys._getframe(1))
+        sys.settrace(self._tracer)
+    
+    def end_tracing(self):
+        self.tracing = False
+        sys.settrace(None)
+