gh-93649: Split tracemalloc tests from _testcapimodule.c (#99551)

Lib/test/test_capi/test_mem.py

@@ -0,0 +1,169 @@
+import re
+import textwrap
+import unittest
+
+
+from test import support
+from test.support import import_helper, requires_subprocess
+from test.support.script_helper import assert_python_failure, assert_python_ok
+
+
+# Skip this test if the _testcapi module isn't available.
+_testcapi = import_helper.import_module('_testcapi')
+
+@requires_subprocess()
+class PyMemDebugTests(unittest.TestCase):
+    PYTHONMALLOC = 'debug'
+    # '0x04c06e0' or '04C06E0'
+    PTR_REGEX = r'(?:0x)?[0-9a-fA-F]+'
+
+    def check(self, code):
+        with support.SuppressCrashReport():
+            out = assert_python_failure(
+                '-c', code,
+                PYTHONMALLOC=self.PYTHONMALLOC,
+                # FreeBSD: instruct jemalloc to not fill freed() memory
+                # with junk byte 0x5a, see JEMALLOC(3)
+                MALLOC_CONF="junk:false",
+            )
+        stderr = out.err
+        return stderr.decode('ascii', 'replace')
+
+    def test_buffer_overflow(self):
+        out = self.check('import _testcapi; _testcapi.pymem_buffer_overflow()')
+        regex = (r"Debug memory block at address p={ptr}: API 'm'\n"
+                 r"    16 bytes originally requested\n"
+                 r"    The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n"
+                 r"    The [0-9] pad bytes at tail={ptr} are not all FORBIDDENBYTE \(0x[0-9a-f]{{2}}\):\n"
+                 r"        at tail\+0: 0x78 \*\*\* OUCH\n"
+                 r"        at tail\+1: 0xfd\n"
+                 r"        at tail\+2: 0xfd\n"
+                 r"        .*\n"
+                 r"(    The block was made by call #[0-9]+ to debug malloc/realloc.\n)?"
+                 r"    Data at p: cd cd cd .*\n"
+                 r"\n"
+                 r"Enable tracemalloc to get the memory block allocation traceback\n"
+                 r"\n"
+                 r"Fatal Python error: _PyMem_DebugRawFree: bad trailing pad byte")
+        regex = regex.format(ptr=self.PTR_REGEX)
+        regex = re.compile(regex, flags=re.DOTALL)
+        self.assertRegex(out, regex)
+
+    def test_api_misuse(self):
+        out = self.check('import _testcapi; _testcapi.pymem_api_misuse()')
+        regex = (r"Debug memory block at address p={ptr}: API 'm'\n"
+                 r"    16 bytes originally requested\n"
+                 r"    The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n"
+                 r"    The [0-9] pad bytes at tail={ptr} are FORBIDDENBYTE, as expected.\n"
+                 r"(    The block was made by call #[0-9]+ to debug malloc/realloc.\n)?"
+                 r"    Data at p: cd cd cd .*\n"
+                 r"\n"
+                 r"Enable tracemalloc to get the memory block allocation traceback\n"
+                 r"\n"
+                 r"Fatal Python error: _PyMem_DebugRawFree: bad ID: Allocated using API 'm', verified using API 'r'\n")
+        regex = regex.format(ptr=self.PTR_REGEX)
+        self.assertRegex(out, regex)
+
+    def check_malloc_without_gil(self, code):
+        out = self.check(code)
+        expected = ('Fatal Python error: _PyMem_DebugMalloc: '
+                    'Python memory allocator called without holding the GIL')
+        self.assertIn(expected, out)
+
+    def test_pymem_malloc_without_gil(self):
+        # Debug hooks must raise an error if PyMem_Malloc() is called
+        # without holding the GIL
+        code = 'import _testcapi; _testcapi.pymem_malloc_without_gil()'
+        self.check_malloc_without_gil(code)
+
+    def test_pyobject_malloc_without_gil(self):
+        # Debug hooks must raise an error if PyObject_Malloc() is called
+        # without holding the GIL
+        code = 'import _testcapi; _testcapi.pyobject_malloc_without_gil()'
+        self.check_malloc_without_gil(code)
+
+    def check_pyobject_is_freed(self, func_name):
+        code = textwrap.dedent(f'''
+            import gc, os, sys, _testcapi
+            # Disable the GC to avoid crash on GC collection
+            gc.disable()
+            try:
+                _testcapi.{func_name}()
+                # Exit immediately to avoid a crash while deallocating
+                # the invalid object
+                os._exit(0)
+            except _testcapi.error:
+                os._exit(1)
+        ''')
+        assert_python_ok(
+            '-c', code,
+            PYTHONMALLOC=self.PYTHONMALLOC,
+            MALLOC_CONF="junk:false",
+        )
+
+    def test_pyobject_null_is_freed(self):
+        self.check_pyobject_is_freed('check_pyobject_null_is_freed')
+
+    def test_pyobject_uninitialized_is_freed(self):
+        self.check_pyobject_is_freed('check_pyobject_uninitialized_is_freed')
+
+    def test_pyobject_forbidden_bytes_is_freed(self):
+        self.check_pyobject_is_freed('check_pyobject_forbidden_bytes_is_freed')
+
+    def test_pyobject_freed_is_freed(self):
+        self.check_pyobject_is_freed('check_pyobject_freed_is_freed')
+
+    def test_set_nomemory(self):
+        code = """if 1:
+            import _testcapi
+
+            class C(): pass
+
+            # The first loop tests both functions and that remove_mem_hooks()
+            # can be called twice in a row. The second loop checks a call to
+            # set_nomemory() after a call to remove_mem_hooks(). The third
+            # loop checks the start and stop arguments of set_nomemory().
+            for outer_cnt in range(1, 4):
+                start = 10 * outer_cnt
+                for j in range(100):
+                    if j == 0:
+                        if outer_cnt != 3:
+                            _testcapi.set_nomemory(start)
+                        else:
+                            _testcapi.set_nomemory(start, start + 1)
+                    try:
+                        C()
+                    except MemoryError as e:
+                        if outer_cnt != 3:
+                            _testcapi.remove_mem_hooks()
+                        print('MemoryError', outer_cnt, j)
+                        _testcapi.remove_mem_hooks()
+                        break
+        """
+        rc, out, err = assert_python_ok('-c', code)
+        lines = out.splitlines()
+        for i, line in enumerate(lines, 1):
+            self.assertIn(b'MemoryError', out)
+            *_, count = line.split(b' ')
+            count = int(count)
+            self.assertLessEqual(count, i*5)
+            self.assertGreaterEqual(count, i*5-2)
+
+
+class PyMemMallocDebugTests(PyMemDebugTests):
+    PYTHONMALLOC = 'malloc_debug'
+
+
+@unittest.skipUnless(support.with_pymalloc(), 'need pymalloc')
+class PyMemPymallocDebugTests(PyMemDebugTests):
+    PYTHONMALLOC = 'pymalloc_debug'
+
+
+@unittest.skipUnless(support.Py_DEBUG, 'need Py_DEBUG')
+class PyMemDefaultTests(PyMemDebugTests):
+    # test default allocator of Python compiled in debug mode
+    PYTHONMALLOC = ''
+
+
+if __name__ == "__main__":
+    unittest.main()

Lib/test/test_capi/test_misc.py

@@ -323,42 +323,6 @@ class CAPITest(unittest.TestCase):
     def test_buildvalue_N(self):
         _testcapi.test_buildvalue_N()
 
-    def test_set_nomemory(self):
-        code = """if 1:
-            import _testcapi
-
-            class C(): pass
-
-            # The first loop tests both functions and that remove_mem_hooks()
-            # can be called twice in a row. The second loop checks a call to
-            # set_nomemory() after a call to remove_mem_hooks(). The third
-            # loop checks the start and stop arguments of set_nomemory().
-            for outer_cnt in range(1, 4):
-                start = 10 * outer_cnt
-                for j in range(100):
-                    if j == 0:
-                        if outer_cnt != 3:
-                            _testcapi.set_nomemory(start)
-                        else:
-                            _testcapi.set_nomemory(start, start + 1)
-                    try:
-                        C()
-                    except MemoryError as e:
-                        if outer_cnt != 3:
-                            _testcapi.remove_mem_hooks()
-                        print('MemoryError', outer_cnt, j)
-                        _testcapi.remove_mem_hooks()
-                        break
-        """
-        rc, out, err = assert_python_ok('-c', code)
-        lines = out.splitlines()
-        for i, line in enumerate(lines, 1):
-            self.assertIn(b'MemoryError', out)
-            *_, count = line.split(b' ')
-            count = int(count)
-            self.assertLessEqual(count, i*5)
-            self.assertGreaterEqual(count, i*5-2)
-
     def test_mapping_keys_values_items(self):
         class Mapping1(dict):
             def keys(self):
@@ -1470,124 +1434,6 @@ class Test_testinternalcapi(unittest.TestCase):
                     if name.startswith('test_'))
 
 
-@support.requires_subprocess()
-class PyMemDebugTests(unittest.TestCase):
-    PYTHONMALLOC = 'debug'
-    # '0x04c06e0' or '04C06E0'
-    PTR_REGEX = r'(?:0x)?[0-9a-fA-F]+'
-
-    def check(self, code):
-        with support.SuppressCrashReport():
-            out = assert_python_failure(
-                '-c', code,
-                PYTHONMALLOC=self.PYTHONMALLOC,
-                # FreeBSD: instruct jemalloc to not fill freed() memory
-                # with junk byte 0x5a, see JEMALLOC(3)
-                MALLOC_CONF="junk:false",
-            )
-        stderr = out.err
-        return stderr.decode('ascii', 'replace')
-
-    def test_buffer_overflow(self):
-        out = self.check('import _testcapi; _testcapi.pymem_buffer_overflow()')
-        regex = (r"Debug memory block at address p={ptr}: API 'm'\n"
-                 r"    16 bytes originally requested\n"
-                 r"    The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n"
-                 r"    The [0-9] pad bytes at tail={ptr} are not all FORBIDDENBYTE \(0x[0-9a-f]{{2}}\):\n"
-                 r"        at tail\+0: 0x78 \*\*\* OUCH\n"
-                 r"        at tail\+1: 0xfd\n"
-                 r"        at tail\+2: 0xfd\n"
-                 r"        .*\n"
-                 r"(    The block was made by call #[0-9]+ to debug malloc/realloc.\n)?"
-                 r"    Data at p: cd cd cd .*\n"
-                 r"\n"
-                 r"Enable tracemalloc to get the memory block allocation traceback\n"
-                 r"\n"
-                 r"Fatal Python error: _PyMem_DebugRawFree: bad trailing pad byte")
-        regex = regex.format(ptr=self.PTR_REGEX)
-        regex = re.compile(regex, flags=re.DOTALL)
-        self.assertRegex(out, regex)
-
-    def test_api_misuse(self):
-        out = self.check('import _testcapi; _testcapi.pymem_api_misuse()')
-        regex = (r"Debug memory block at address p={ptr}: API 'm'\n"
-                 r"    16 bytes originally requested\n"
-                 r"    The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n"
-                 r"    The [0-9] pad bytes at tail={ptr} are FORBIDDENBYTE, as expected.\n"
-                 r"(    The block was made by call #[0-9]+ to debug malloc/realloc.\n)?"
-                 r"    Data at p: cd cd cd .*\n"
-                 r"\n"
-                 r"Enable tracemalloc to get the memory block allocation traceback\n"
-                 r"\n"
-                 r"Fatal Python error: _PyMem_DebugRawFree: bad ID: Allocated using API 'm', verified using API 'r'\n")
-        regex = regex.format(ptr=self.PTR_REGEX)
-        self.assertRegex(out, regex)
-
-    def check_malloc_without_gil(self, code):
-        out = self.check(code)
-        expected = ('Fatal Python error: _PyMem_DebugMalloc: '
-                    'Python memory allocator called without holding the GIL')
-        self.assertIn(expected, out)
-
-    def test_pymem_malloc_without_gil(self):
-        # Debug hooks must raise an error if PyMem_Malloc() is called
-        # without holding the GIL
-        code = 'import _testcapi; _testcapi.pymem_malloc_without_gil()'
-        self.check_malloc_without_gil(code)
-
-    def test_pyobject_malloc_without_gil(self):
-        # Debug hooks must raise an error if PyObject_Malloc() is called
-        # without holding the GIL
-        code = 'import _testcapi; _testcapi.pyobject_malloc_without_gil()'
-        self.check_malloc_without_gil(code)
-
-    def check_pyobject_is_freed(self, func_name):
-        code = textwrap.dedent(f'''
-            import gc, os, sys, _testcapi
-            # Disable the GC to avoid crash on GC collection
-            gc.disable()
-            try:
-                _testcapi.{func_name}()
-                # Exit immediately to avoid a crash while deallocating
-                # the invalid object
-                os._exit(0)
-            except _testcapi.error:
-                os._exit(1)
-        ''')
-        assert_python_ok(
-            '-c', code,
-            PYTHONMALLOC=self.PYTHONMALLOC,
-            MALLOC_CONF="junk:false",
-        )
-
-    def test_pyobject_null_is_freed(self):
-        self.check_pyobject_is_freed('check_pyobject_null_is_freed')
-
-    def test_pyobject_uninitialized_is_freed(self):
-        self.check_pyobject_is_freed('check_pyobject_uninitialized_is_freed')
-
-    def test_pyobject_forbidden_bytes_is_freed(self):
-        self.check_pyobject_is_freed('check_pyobject_forbidden_bytes_is_freed')
-
-    def test_pyobject_freed_is_freed(self):
-        self.check_pyobject_is_freed('check_pyobject_freed_is_freed')
-
-
-class PyMemMallocDebugTests(PyMemDebugTests):
-    PYTHONMALLOC = 'malloc_debug'
-
-
-@unittest.skipUnless(support.with_pymalloc(), 'need pymalloc')
-class PyMemPymallocDebugTests(PyMemDebugTests):
-    PYTHONMALLOC = 'pymalloc_debug'
-
-
-@unittest.skipUnless(support.Py_DEBUG, 'need Py_DEBUG')
-class PyMemDefaultTests(PyMemDebugTests):
-    # test default allocator of Python compiled in debug mode
-    PYTHONMALLOC = ''
-
-
 @unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module")
 class Test_ModuleStateAccess(unittest.TestCase):
     """Test access to module start (PEP 573)"""

Modules/_testcapi/mem.c

@@ -596,6 +596,82 @@ check_pyobject_freed_is_freed(PyObject *self, PyObject *Py_UNUSED(args))
 #endif
 }
 
+// Tracemalloc tests
+static PyObject *
+tracemalloc_track(PyObject *self, PyObject *args)
+{
+    unsigned int domain;
+    PyObject *ptr_obj;
+    Py_ssize_t size;
+    int release_gil = 0;
+
+    if (!PyArg_ParseTuple(args, "IOn|i",
+                          &domain, &ptr_obj, &size, &release_gil))
+    {
+        return NULL;
+    }
+    void *ptr = PyLong_AsVoidPtr(ptr_obj);
+    if (PyErr_Occurred()) {
+        return NULL;
+    }
+
+    int res;
+    if (release_gil) {
+        Py_BEGIN_ALLOW_THREADS
+        res = PyTraceMalloc_Track(domain, (uintptr_t)ptr, size);
+        Py_END_ALLOW_THREADS
+    }
+    else {
+        res = PyTraceMalloc_Track(domain, (uintptr_t)ptr, size);
+    }
+    if (res < 0) {
+        PyErr_SetString(PyExc_RuntimeError, "PyTraceMalloc_Track error");
+        return NULL;
+    }
+
+    Py_RETURN_NONE;
+}
+
+static PyObject *
+tracemalloc_untrack(PyObject *self, PyObject *args)
+{
+    unsigned int domain;
+    PyObject *ptr_obj;
+
+    if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj)) {
+        return NULL;
+    }
+    void *ptr = PyLong_AsVoidPtr(ptr_obj);
+    if (PyErr_Occurred()) {
+        return NULL;
+    }
+
+    int res = PyTraceMalloc_Untrack(domain, (uintptr_t)ptr);
+    if (res < 0) {
+        PyErr_SetString(PyExc_RuntimeError, "PyTraceMalloc_Untrack error");
+        return NULL;
+    }
+
+    Py_RETURN_NONE;
+}
+
+static PyObject *
+tracemalloc_get_traceback(PyObject *self, PyObject *args)
+{
+    unsigned int domain;
+    PyObject *ptr_obj;
+
+    if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj)) {
+        return NULL;
+    }
+    void *ptr = PyLong_AsVoidPtr(ptr_obj);
+    if (PyErr_Occurred()) {
+        return NULL;
+    }
+
+    return _PyTraceMalloc_GetTraceback(domain, (uintptr_t)ptr);
+}
+
 static PyMethodDef test_methods[] = {
     {"check_pyobject_forbidden_bytes_is_freed",
                             check_pyobject_forbidden_bytes_is_freed, METH_NOARGS},
@@ -617,6 +693,11 @@ static PyMethodDef test_methods[] = {
     {"test_pymem_setrawallocators",   test_pymem_setrawallocators,   METH_NOARGS},
     {"test_pyobject_new",             test_pyobject_new,             METH_NOARGS},
     {"test_pyobject_setallocators",   test_pyobject_setallocators,   METH_NOARGS},
+
+    // Tracemalloc tests
+    {"tracemalloc_track",             tracemalloc_track,             METH_VARARGS},
+    {"tracemalloc_untrack",           tracemalloc_untrack,           METH_VARARGS},
+    {"tracemalloc_get_traceback",     tracemalloc_get_traceback,     METH_VARARGS},
     {NULL},
 };
 

Modules/_testcapimodule.c

@@ -2065,78 +2065,6 @@ getitem_with_error(PyObject *self, PyObject *args)
     return PyObject_GetItem(map, key);
 }
 
-static PyObject *
-tracemalloc_track(PyObject *self, PyObject *args)
-{
-    unsigned int domain;
-    PyObject *ptr_obj;
-    void *ptr;
-    Py_ssize_t size;
-    int release_gil = 0;
-    int res;
-
-    if (!PyArg_ParseTuple(args, "IOn|i", &domain, &ptr_obj, &size, &release_gil))
-        return NULL;
-    ptr = PyLong_AsVoidPtr(ptr_obj);
-    if (PyErr_Occurred())
-        return NULL;
-
-    if (release_gil) {
-        Py_BEGIN_ALLOW_THREADS
-        res = PyTraceMalloc_Track(domain, (uintptr_t)ptr, size);
-        Py_END_ALLOW_THREADS
-    }
-    else {
-        res = PyTraceMalloc_Track(domain, (uintptr_t)ptr, size);
-    }
-
-    if (res < 0) {
-        PyErr_SetString(PyExc_RuntimeError, "PyTraceMalloc_Track error");
-        return NULL;
-    }
-
-    Py_RETURN_NONE;
-}
-
-static PyObject *
-tracemalloc_untrack(PyObject *self, PyObject *args)
-{
-    unsigned int domain;
-    PyObject *ptr_obj;
-    void *ptr;
-    int res;
-
-    if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj))
-        return NULL;
-    ptr = PyLong_AsVoidPtr(ptr_obj);
-    if (PyErr_Occurred())
-        return NULL;
-
-    res = PyTraceMalloc_Untrack(domain, (uintptr_t)ptr);
-    if (res < 0) {
-        PyErr_SetString(PyExc_RuntimeError, "PyTraceMalloc_Untrack error");
-        return NULL;
-    }
-
-    Py_RETURN_NONE;
-}
-
-static PyObject *
-tracemalloc_get_traceback(PyObject *self, PyObject *args)
-{
-    unsigned int domain;
-    PyObject *ptr_obj;
-    void *ptr;
-
-    if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj))
-        return NULL;
-    ptr = PyLong_AsVoidPtr(ptr_obj);
-    if (PyErr_Occurred())
-        return NULL;
-
-    return _PyTraceMalloc_GetTraceback(domain, (uintptr_t)ptr);
-}
-
 static PyObject *
 dict_get_version(PyObject *self, PyObject *args)
 {
@@ -3301,9 +3229,6 @@ static PyMethodDef TestMethods[] = {
     {"return_result_with_error", return_result_with_error, METH_NOARGS},
     {"getitem_with_error", getitem_with_error, METH_VARARGS},
     {"Py_CompileString",     pycompilestring, METH_O},
-    {"tracemalloc_track", tracemalloc_track, METH_VARARGS},
-    {"tracemalloc_untrack", tracemalloc_untrack, METH_VARARGS},
-    {"tracemalloc_get_traceback", tracemalloc_get_traceback, METH_VARARGS},
     {"dict_get_version", dict_get_version, METH_VARARGS},
     {"raise_SIGINT_then_send_None", raise_SIGINT_then_send_None, METH_VARARGS},
     {"stack_pointer", stack_pointer, METH_NOARGS},