Something went wrong. Try again.
What happens if I push all of nixpkgs
Something went wrong. Try again.
12 kB · 354 lines
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355From 1b77e35d848340f2c5f4c9b82965c25a0572d48f Mon Sep 17 00:00:00 2001From: Jeroen Demeyer <J.Demeyer@UGent.be>Date: Thu, 14 Feb 2019 10:02:41 +0100Subject: [PATCH] @cython.trashcan directive to enable the Python trashcan for deallocations
--- Cython/Compiler/ModuleNode.py | 10 +++ Cython/Compiler/Options.py | 2 + Cython/Compiler/PyrexTypes.py | 8 +- Cython/Compiler/Symtab.py | 18 +++- Cython/Utility/ExtensionTypes.c | 43 ++++++++++ tests/run/trashcan.pyx | 148 ++++++++++++++++++++++++++++++++ 6 files changed, 227 insertions(+), 2 deletions(-) create mode 100644 tests/run/trashcan.pyx
diff --git a/Cython/Compiler/ModuleNode.py b/Cython/Compiler/ModuleNode.pyindex 56845330d..3a3e8a956 100644--- a/Cython/Compiler/ModuleNode.py+++ b/Cython/Compiler/ModuleNode.py@@ -1443,6 +1443,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): is_final_type = scope.parent_type.is_final_type needs_gc = scope.needs_gc()+ needs_trashcan = scope.needs_trashcan() weakref_slot = scope.lookup_here("__weakref__") if not scope.is_closure_class_scope else None if weakref_slot not in scope.var_entries:@@ -1481,6 +1482,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # running this destructor. code.putln("PyObject_GC_UnTrack(o);") + if needs_trashcan:+ code.globalstate.use_utility_code(+ UtilityCode.load_cached("PyTrashcan", "ExtensionTypes.c"))+ code.putln("__Pyx_TRASHCAN_BEGIN(o, %s)" % slot_func_cname)+ # call the user's __dealloc__ self.generate_usr_dealloc_call(scope, code) @@ -1554,6 +1560,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("(*Py_TYPE(o)->tp_free)(o);") if freelist_size: code.putln("}")++ if needs_trashcan:+ code.putln("__Pyx_TRASHCAN_END")+ code.putln( "}") diff --git a/Cython/Compiler/Options.py b/Cython/Compiler/Options.pyindex d03119fca..05a728135 100644--- a/Cython/Compiler/Options.py+++ b/Cython/Compiler/Options.py@@ -319,6 +319,7 @@ directive_types = { 'freelist': int, 'c_string_type': one_of('bytes', 'bytearray', 'str', 'unicode'), 'c_string_encoding': normalise_encoding_name,+ 'trashcan': bool, 'cpow': bool } @@ -362,6 +363,7 @@ directive_scopes = { # defaults to available everywhere 'np_pythran': ('module',), 'fast_gil': ('module',), 'iterable_coroutine': ('module', 'function'),+ 'trashcan' : ('cclass',), } diff --git a/Cython/Compiler/PyrexTypes.py b/Cython/Compiler/PyrexTypes.pyindex c309bd04b..9231130b5 100644--- a/Cython/Compiler/PyrexTypes.py+++ b/Cython/Compiler/PyrexTypes.py@@ -1129,6 +1129,7 @@ class PyObjectType(PyrexType): is_extern = False is_subclassed = False is_gc_simple = False+ builtin_trashcan = False # builtin type using trashcan def __str__(self): return "Python object"@@ -1183,10 +1184,14 @@ class PyObjectType(PyrexType): builtin_types_that_cannot_create_refcycles = set([- 'bool', 'int', 'long', 'float', 'complex',+ 'object', 'bool', 'int', 'long', 'float', 'complex', 'bytearray', 'bytes', 'unicode', 'str', 'basestring' ]) +builtin_types_with_trashcan = set([+ 'dict', 'list', 'set', 'frozenset', 'tuple', 'type',+])+ class BuiltinObjectType(PyObjectType): # objstruct_cname string Name of PyObject struct@@ -1211,6 +1216,7 @@ class BuiltinObjectType(PyObjectType): self.typeptr_cname = "(&%s)" % cname self.objstruct_cname = objstruct_cname self.is_gc_simple = name in builtin_types_that_cannot_create_refcycles+ self.builtin_trashcan = name in builtin_types_with_trashcan if name == 'type': # Special case the type type, as many C API calls (and other # libraries) actually expect a PyTypeObject* for type arguments.diff --git a/Cython/Compiler/Symtab.py b/Cython/Compiler/Symtab.pyindex 7361a55ae..f0c311ba6 100644--- a/Cython/Compiler/Symtab.py+++ b/Cython/Compiler/Symtab.py@@ -2043,7 +2043,7 @@ class PyClassScope(ClassScope): class CClassScope(ClassScope): # Namespace of an extension type. #- # parent_type CClassType+ # parent_type PyExtensionType # #typeobj_cname string or None # #objstruct_cname string # method_table_cname string@@ -2087,6 +2087,22 @@ class CClassScope(ClassScope): return not self.parent_type.is_gc_simple return False + def needs_trashcan(self):+ # If the trashcan directive is explicitly set to False,+ # unconditionally disable the trashcan.+ directive = self.directives.get('trashcan')+ if directive is False:+ return False+ # If the directive is set to True and the class has Python-valued+ # C attributes, then it should use the trashcan in tp_dealloc.+ if directive and self.has_cyclic_pyobject_attrs:+ return True+ # Use the trashcan if the base class uses it+ base_type = self.parent_type.base_type+ if base_type and base_type.scope is not None:+ return base_type.scope.needs_trashcan()+ return self.parent_type.builtin_trashcan+ def needs_tp_clear(self): """ Do we need to generate an implementation for the tp_clear slot? Candiff --git a/Cython/Utility/ExtensionTypes.c b/Cython/Utility/ExtensionTypes.cindex dc187ab49..f359165df 100644--- a/Cython/Utility/ExtensionTypes.c+++ b/Cython/Utility/ExtensionTypes.c@@ -119,6 +119,49 @@ static int __Pyx_PyType_Ready(PyTypeObject *t) { return r; } +/////////////// PyTrashcan.proto ///////////////++// These macros are taken from https://github.com/python/cpython/pull/11841+// Unlike the Py_TRASHCAN_SAFE_BEGIN/Py_TRASHCAN_SAFE_END macros, they+// allow dealing correctly with subclasses.++// This requires CPython version >= 2.7.4+// (or >= 3.2.4 but we don't support such old Python 3 versions anyway)+#if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x02070400+#define __Pyx_TRASHCAN_BEGIN_CONDITION(op, cond) \+ do { \+ PyThreadState *_tstate = NULL; \+ // If "cond" is false, then _tstate remains NULL and the deallocator+ // is run normally without involving the trashcan+ if (cond) { \+ _tstate = PyThreadState_GET(); \+ if (_tstate->trash_delete_nesting >= PyTrash_UNWIND_LEVEL) { \+ // Store the object (to be deallocated later) and jump past+ // Py_TRASHCAN_END, skipping the body of the deallocator+ _PyTrash_thread_deposit_object((PyObject*)(op)); \+ break; \+ } \+ ++_tstate->trash_delete_nesting; \+ }+ // The body of the deallocator is here.+#define __Pyx_TRASHCAN_END \+ if (_tstate) { \+ --_tstate->trash_delete_nesting; \+ if (_tstate->trash_delete_later && _tstate->trash_delete_nesting <= 0) \+ _PyTrash_thread_destroy_chain(); \+ } \+ } while (0);++#define __Pyx_TRASHCAN_BEGIN(op, dealloc) __Pyx_TRASHCAN_BEGIN_CONDITION(op, \+ Py_TYPE(op)->tp_dealloc == (destructor)(dealloc))++#else+// The trashcan is a no-op on other Python implementations+// or old CPython versions+#define __Pyx_TRASHCAN_BEGIN(op, dealloc)+#define __Pyx_TRASHCAN_END+#endif+ /////////////// CallNextTpDealloc.proto /////////////// static void __Pyx_call_next_tp_dealloc(PyObject* obj, destructor current_tp_dealloc);diff --git a/tests/run/trashcan.pyx b/tests/run/trashcan.pyxnew file mode 100644index 000000000..93a501ff8--- /dev/null+++ b/tests/run/trashcan.pyx@@ -0,0 +1,148 @@+# mode: run++cimport cython+++# Count number of times an object was deallocated twice. This should remain 0.+cdef int double_deallocations = 0+def assert_no_double_deallocations():+ global double_deallocations+ err = double_deallocations+ double_deallocations = 0+ assert not err+++# Compute x = f(f(f(...(None)...))) nested n times and throw away the result.+# The real test happens when exiting this function: then a big recursive+# deallocation of x happens. We are testing two things in the tests below:+# that Python does not crash and that no double deallocation happens.+# See also https://github.com/python/cpython/pull/11841+def recursion_test(f, int n=2**20):+ x = None+ cdef int i+ for i in range(n):+ x = f(x)+++@cython.trashcan(True)+cdef class Recurse:+ """+ >>> recursion_test(Recurse)+ >>> assert_no_double_deallocations()+ """+ cdef public attr+ cdef int deallocated++ def __init__(self, x):+ self.attr = x++ def __dealloc__(self):+ # Check that we're not being deallocated twice+ global double_deallocations+ double_deallocations += self.deallocated+ self.deallocated = 1+++cdef class RecurseSub(Recurse):+ """+ >>> recursion_test(RecurseSub)+ >>> assert_no_double_deallocations()+ """+ cdef int subdeallocated++ def __dealloc__(self):+ # Check that we're not being deallocated twice+ global double_deallocations+ double_deallocations += self.subdeallocated+ self.subdeallocated = 1+++@cython.freelist(4)+@cython.trashcan(True)+cdef class RecurseFreelist:+ """+ >>> recursion_test(RecurseFreelist)+ >>> recursion_test(RecurseFreelist, 1000)+ >>> assert_no_double_deallocations()+ """+ cdef public attr+ cdef int deallocated++ def __init__(self, x):+ self.attr = x++ def __dealloc__(self):+ # Check that we're not being deallocated twice+ global double_deallocations+ double_deallocations += self.deallocated+ self.deallocated = 1+++# Subclass of list => uses trashcan by default+# As long as https://github.com/python/cpython/pull/11841 is not fixed,+# this does lead to double deallocations, so we skip that check.+cdef class RecurseList(list):+ """+ >>> RecurseList(42)+ [42]+ >>> recursion_test(RecurseList)+ """+ def __init__(self, x):+ super().__init__((x,))+++# Some tests where the trashcan is NOT used. When the trashcan is not used+# in a big recursive deallocation, the __dealloc__s of the base classs are+# only run after the __dealloc__s of the subclasses.+# We use this to detect trashcan usage.+cdef int base_deallocated = 0+cdef int trashcan_used = 0+def assert_no_trashcan_used():+ global base_deallocated, trashcan_used+ err = trashcan_used+ trashcan_used = base_deallocated = 0+ assert not err+++cdef class Base:+ def __dealloc__(self):+ global base_deallocated+ base_deallocated = 1+++# Trashcan disabled by default+cdef class Sub1(Base):+ """+ >>> recursion_test(Sub1, 100)+ >>> assert_no_trashcan_used()+ """+ cdef public attr++ def __init__(self, x):+ self.attr = x++ def __dealloc__(self):+ global base_deallocated, trashcan_used+ trashcan_used += base_deallocated+++@cython.trashcan(True)+cdef class Middle(Base):+ cdef public foo+++# Trashcan disabled explicitly+@cython.trashcan(False)+cdef class Sub2(Middle):+ """+ >>> recursion_test(Sub2, 1000)+ >>> assert_no_trashcan_used()+ """+ cdef public attr++ def __init__(self, x):+ self.attr = x++ def __dealloc__(self):+ global base_deallocated, trashcan_used+ trashcan_used += base_deallocated-- 2.39.0