remove code.new() function and store lines directly into linecache.cache instead.

This avoids the need for custom code objects, improving compatibility for jython
and pypy-c.

--HG--
branch : trunk

CHANGELOG

@@ -11,6 +11,11 @@ Changes between 1.3.0 and 1.3.1
 - improve tracebacks presentation: 
   - raises shows shorter more relevant tracebacks
 
+- improve support for raises and other dynamically compiled code by
+  manipulating python's linecache.cache instead of the previous
+  rather hacky way of creating custom code objects.  This makes 
+  it seemlessly work on Jython and PyPy at least. 
+
 Changes between 1.2.1 and 1.3.0
 ==================================================
 

py/_builtin.py

@@ -151,6 +151,9 @@ else:
         return getattr(function, "__dict__", None)
 
     def _getcode(function):
+        try:
+            return getattr(function, "__code__")
+        except AttributeError:
             return getattr(function, "func_code", None)
 
     def print_(*args, **kwargs):

py/_code/code.py

@@ -23,58 +23,8 @@ class Code(object):
     def __ne__(self, other):
         return not self == other
 
-    def new(self, rec=False, **kwargs): 
-        """ return new code object with modified attributes. 
-            if rec-cursive is true then dive into code 
-            objects contained in co_consts. 
-        """ 
-        if sys.platform.startswith("java"):
-            # XXX jython does not support the below co_filename hack
-            return self.raw 
-        names = [x for x in dir(self.raw) if x[:3] == 'co_']
-        for name in kwargs: 
-            if name not in names: 
-                raise TypeError("unknown code attribute: %r" %(name, ))
-        if rec and hasattr(self.raw, 'co_consts'):  # jython 
-            newconstlist = []
-            co = self.raw
-            cotype = type(co)
-            for c in co.co_consts:
-                if isinstance(c, cotype):
-                    c = self.__class__(c).new(rec=True, **kwargs) 
-                newconstlist.append(c)
-            return self.new(rec=False, co_consts=tuple(newconstlist), **kwargs) 
-        for name in names:
-            if name not in kwargs:
-                kwargs[name] = getattr(self.raw, name)
-        arglist = [
-                 kwargs['co_argcount'],
-                 kwargs['co_nlocals'],
-                 kwargs.get('co_stacksize', 0), # jython
-                 kwargs.get('co_flags', 0), # jython
-                 kwargs.get('co_code', ''), # jython
-                 kwargs.get('co_consts', ()), # jython
-                 kwargs.get('co_names', []), # 
-                 kwargs['co_varnames'],
-                 kwargs['co_filename'],
-                 kwargs['co_name'],
-                 kwargs['co_firstlineno'],
-                 kwargs.get('co_lnotab', ''), #jython
-                 kwargs.get('co_freevars', None), #jython
-                 kwargs.get('co_cellvars', None), # jython
-        ]
-        if sys.version_info >= (3,0):
-            arglist.insert(1, kwargs['co_kwonlyargcount'])
-            return self.raw.__class__(*arglist)
-        else:
-            return py.std.new.code(*arglist)
-
     def path(self):
         """ return a path object pointing to source code"""
-        fn = self.raw.co_filename 
-        try:
-            return fn.__path__
-        except AttributeError:
         p = py.path.local(self.raw.co_filename)
         if not p.check():
             # XXX maybe try harder like the weird logic 

py/_code/source.py

@@ -212,10 +212,10 @@ class Source(object):
         else:
             if flag & _AST_FLAG:
                 return co
-            co_filename = MyStr(filename)
+            from types import ModuleType
-            co_filename.__source__ = self
+            lines = [(x + "\n") for x in self.lines]
-            return py.code.Code(co).new(rec=1, co_filename=co_filename) 
+            py.std.linecache.cache[filename] = (1, None, lines, filename)
-            #return newcode_withfilename(co, co_filename)
+            return co
 
 #
 # public API shortcut functions
@@ -224,11 +224,9 @@ class Source(object):
 def compile_(source, filename=None, mode='exec', flags=
             generators.compiler_flag, dont_inherit=0):
     """ compile the given source to a raw code object,
-        which points back to the source code through
+        and maintain an internal cache which allows later
-        "co_filename.__source__".  All code objects
+        retrieval of the source code for the code object 
-        contained in the code object will recursively
+        and any recursively created code objects. 
-        also have this special subclass-of-string
-        filename.
     """
     if _ast is not None and isinstance(source, _ast.AST):
         # XXX should Source support having AST?
@@ -262,14 +260,8 @@ def getfslineno(obj):
 #
 # helper functions
 #
-class MyStr(str):
-    """ custom string which allows to add attributes. """
 
 def findsource(obj):
-    obj = py.code.getrawcode(obj)
-    try:
-        fullsource = obj.co_filename.__source__
-    except AttributeError:
     try:
         sourcelines, lineno = py.std.inspect.findsource(obj)
     except (KeyboardInterrupt, SystemExit):
@@ -279,27 +271,15 @@ def findsource(obj):
     source = Source()
     source.lines = [line.rstrip() for line in sourcelines]
     return source, lineno
-    else:
-        lineno = obj.co_firstlineno - 1        
-        return fullsource, lineno
-
 
 def getsource(obj, **kwargs):
     obj = py.code.getrawcode(obj)
-    try:
-        fullsource = obj.co_filename.__source__
-    except AttributeError:
     try:
         strsrc = inspect.getsource(obj)
     except IndentationError:
         strsrc = "\"Buggy python version consider upgrading, cannot get source\""
     assert isinstance(strsrc, str)
     return Source(strsrc, **kwargs)
-    else:
-        lineno = obj.co_firstlineno - 1
-        end = fullsource.getblockend(lineno)
-        return Source(fullsource[lineno:end+1], deident=True)
-
 
 def deindent(lines, offset=None):
     if offset is None:

testing/code/test_code.py

@@ -1,87 +1,12 @@
-from __future__ import generators
 import py
 import sys
 
-failsonjython = py.test.mark.xfail("sys.platform.startswith('java')")
-
-def test_newcode(): 
-    source = "i = 3"
-    co = compile(source, '', 'exec') 
-    code = py.code.Code(co) 
-    newco = code.new() 
-    assert co == newco 
-
 def test_ne():
     code1 = py.code.Code(compile('foo = "bar"', '', 'exec'))
     assert code1 == code1
     code2 = py.code.Code(compile('foo = "baz"', '', 'exec'))
     assert code2 != code1
 
-@failsonjython
-def test_newcode_unknown_args(): 
-    code = py.code.Code(compile("", '', 'exec'))
-    py.test.raises(TypeError, 'code.new(filename="hello")')
-
-@failsonjython
-def test_newcode_withfilename():
-    source = py.code.Source("""
-        def f():
-            def g():
-                pass
-    """)
-    co = compile(str(source)+'\n', 'nada', 'exec')
-    obj = 'hello'
-    newco = py.code.Code(co).new(rec=True, co_filename=obj)
-    def walkcode(co):
-        for x in co.co_consts:
-            if isinstance(x, type(co)):
-                for y in walkcode(x):
-                    yield y
-        yield co
-
-    names = []
-    for code in walkcode(newco):
-        assert newco.co_filename == obj
-        assert newco.co_filename is obj
-        names.append(code.co_name)
-    assert 'f' in names
-    assert 'g' in names
-
-@failsonjython
-def test_newcode_with_filename(): 
-    source = "i = 3"
-    co = compile(source, '', 'exec') 
-    code = py.code.Code(co) 
-    class MyStr(str): 
-        pass 
-    filename = MyStr("hello") 
-    filename.__source__ = py.code.Source(source) 
-    newco = code.new(rec=True, co_filename=filename) 
-    assert newco.co_filename.__source__ == filename.__source__
-    s = py.code.Source(newco) 
-    assert str(s) == source 
-
-
-@failsonjython
-def test_new_code_object_carries_filename_through():
-    class mystr(str):
-        pass
-    filename = mystr("dummy")
-    co = compile("hello\n", filename, 'exec')
-    assert not isinstance(co.co_filename, mystr)
-    args = [
-            co.co_argcount, co.co_nlocals, co.co_stacksize,
-             co.co_flags, co.co_code, co.co_consts,
-             co.co_names, co.co_varnames,
-             filename,
-             co.co_name, co.co_firstlineno, co.co_lnotab,
-             co.co_freevars, co.co_cellvars
-    ]
-    if sys.version_info > (3,0):
-        args.insert(1, co.co_kwonlyargcount)
-    c2 = py.std.types.CodeType(*args)
-    assert c2.co_filename is filename
-
 def test_code_gives_back_name_for_not_existing_file():
     name = 'abc-123'
     co_code = compile("pass\n", name, 'exec')

testing/code/test_excinfo.py

@@ -293,7 +293,6 @@ class TestFormattedExcinfo:
         assert lines[0] == "|   def f(x):"
         assert lines[1] == "        pass"
 
-    @failsonjython
     def test_repr_source_excinfo(self):
         """ check if indentation is right """
         pr = FormattedExcinfo()

testing/code/test_source.py

@@ -80,11 +80,10 @@ def test_source_strip_multiline():
     source2 = source.strip() 
     assert source2.lines == [" hello"]
 
-@failsonjython
 def test_syntaxerror_rerepresentation():
-    ex = py.test.raises(SyntaxError, py.code.compile, 'x x')
+    ex = py.test.raises(SyntaxError, py.code.compile, 'xyz xyz')
     assert ex.value.lineno == 1
-    assert ex.value.offset == 3
+    assert ex.value.offset in (4,7) # XXX pypy/jython versus cpython? 
     assert ex.value.text.strip(), 'x x'
 
 def test_isparseable():
@@ -132,7 +131,6 @@ class TestSourceParsingAndCompiling:
         exec (co, d)
         assert d['x'] == 3
 
-    @failsonjython
     def test_compile_and_getsource_simple(self):
         co = py.code.compile("x=3")
         exec (co)
@@ -203,7 +201,6 @@ class TestSourceParsingAndCompiling:
         assert isinstance(mod, ast.Module)
         compile(mod, "<filename>", "exec")
 
-    @failsonjython
     def test_compile_and_getsource(self):
         co = self.source.compile()
         py.builtin.exec_(co, globals())
@@ -260,7 +257,6 @@ def test_getstartingblock_multiline():
     l = [i for i in x.source.lines if i.strip()]
     assert len(l) == 4
 
-@failsonjython
 def test_getline_finally():
     def c(): pass
     excinfo = py.test.raises(TypeError, """
@@ -274,7 +270,6 @@ def test_getline_finally():
     source = excinfo.traceback[-1].statement
     assert str(source).strip() == 'c(1)'
 
-@failsonjython
 def test_getfuncsource_dynamic():
     source = """
         def f():
@@ -341,7 +336,6 @@ def test_getsource_fallback():
     src = getsource(x)
     assert src == expected
 
-@failsonjython
 def test_idem_compile_and_getsource():
     from py._code.source import getsource
     expected = "def x(): pass"
@@ -355,8 +349,7 @@ def test_findsource_fallback():
     assert 'test_findsource_simple' in str(src)
     assert src[lineno] == '    def x():'
 
-@failsonjython
+def test_findsource():
-def test_findsource___source__():
     from py._code.source import findsource
     co = py.code.compile("""if 1:
     def x():
@@ -373,7 +366,6 @@ def test_findsource___source__():
     assert src[lineno] == "    def x():"
     
 
-@failsonjython
 def test_getfslineno():
     from py.code import getfslineno
 

testing/plugin/test_pytest_runner.py

@@ -316,7 +316,11 @@ def test_runtest_in_module_ordering(testdir):
 
 class TestRaises:
     def test_raises(self):
-        py.test.raises(ValueError, "int('qwe')")
+        source = "int('qwe')"
+        excinfo = py.test.raises(ValueError, source)
+        code = excinfo.traceback[-1].frame.code
+        s = str(code.fullsource)
+        assert s == source 
 
     def test_raises_exec(self):
         py.test.raises(ValueError, "a,x = []")