diff options
| -rw-r--r-- | Doc/library/sys.rst | 22 | ||||
| -rw-r--r-- | Doc/whatsnew/3.10.rst | 4 | ||||
| -rw-r--r-- | Lib/test/test_capi.py | 17 | ||||
| -rw-r--r-- | Lib/test/test_faulthandler.py | 8 | ||||
| -rw-r--r-- | Lib/test/test_sys.py | 5 | ||||
| -rw-r--r-- | Misc/NEWS.d/next/Library/2021-01-18-11-59-46.bpo-42955.CSWLC9.rst | 2 | ||||
| -rw-r--r-- | Python/module_names.h | 295 | ||||
| -rw-r--r-- | Python/pylifecycle.c | 59 | ||||
| -rw-r--r-- | Python/sysmodule.c | 62 | ||||
| -rw-r--r-- | Tools/scripts/generate_module_names.py | 102 |
10 files changed, 303 insertions, 273 deletions
diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 0f13adcf0e5..d536fc9322e 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -153,10 +153,12 @@ always available. .. data:: builtin_module_names - A tuple of strings giving the names of all modules that are compiled into this + A tuple of strings containing the names of all modules that are compiled into this Python interpreter. (This information is not available in any other way --- ``modules.keys()`` only lists the imported modules.) + See also the :attr:`sys.module_names` list. + .. function:: call_tracing(func, args) @@ -1060,6 +1062,24 @@ always available. This is still called as a fallback if a :data:`meta_path` entry doesn't have a :meth:`~importlib.abc.MetaPathFinder.find_spec` method. +.. data:: module_names + + A frozenset of strings containing the names of standard library modules. + + It is the same on all platforms. Modules which are not available on + some platforms and modules disabled at Python build are also listed. + All module kinds are listed: pure Python, built-in, frozen and extension + modules. Test modules are excluded. + + For packages, only sub-packages are listed, not sub-modules. For example, + ``concurrent`` package and ``concurrent.futures`` sub-package are listed, + but not ``concurrent.futures.base`` sub-module. + + See also the :attr:`sys.builtin_module_names` list. + + .. versionchanged:: 3.10 + + .. data:: modules This is a dictionary that maps module names to modules which have already been diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index d822dda09d2..a6c3fbbff91 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -396,6 +396,10 @@ Add :data:`sys.orig_argv` attribute: the list of the original command line arguments passed to the Python executable. (Contributed by Victor Stinner in :issue:`23427`.) +Add :data:`sys.module_names`, containing the list of the standard library +module names. +(Contributed by Victor Stinner in :issue:`42955`.) + threading --------- diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 67175cd044a..5f5c0d038d9 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -569,12 +569,23 @@ class CAPITest(unittest.TestCase): self.assertEqual(len(modules), total) def test_fatal_error(self): + # By default, stdlib extension modules are ignored, + # but not test modules. expected = ('_testcapi',) - not_expected = ('sys', 'builtins', '_imp', '_thread', '_weakref', - '_io', 'marshal', '_signal', '_abc') - code = 'import _testcapi; _testcapi.fatal_error(b"MESSAGE")' + not_expected = ('sys',) + code = 'import _testcapi, sys; _testcapi.fatal_error(b"MESSAGE")' self.check_fatal_error(code, expected, not_expected) + # Mark _testcapi as stdlib module, but not sys + expected = ('sys',) + not_expected = ('_testcapi',) + code = textwrap.dedent(''' + import _testcapi, sys + sys.module_names = frozenset({"_testcapi"}) + _testcapi.fatal_error(b"MESSAGE") + ''') + self.check_fatal_error(code, expected) + class TestPendingCalls(unittest.TestCase): diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index b4a654f8a9c..02077a69bb4 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -334,8 +334,9 @@ class FaultHandlerTests(unittest.TestCase): def test_dump_ext_modules(self): code = """ import faulthandler - # _testcapi is a test module and not considered as a stdlib module - import _testcapi + import sys + # Don't filter stdlib module names + sys.module_names = frozenset() faulthandler.enable() faulthandler._sigsegv() """ @@ -346,7 +347,8 @@ class FaultHandlerTests(unittest.TestCase): if not match: self.fail(f"Cannot find 'Extension modules:' in {stderr!r}") modules = set(match.group(1).strip().split(', ')) - self.assertIn('_testcapi', modules) + for name in ('sys', 'faulthandler'): + self.assertIn(name, modules) def test_is_enabled(self): orig_stderr = sys.stderr diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 3af5b117aff..729b8667fc8 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -986,6 +986,11 @@ class SysModuleTest(unittest.TestCase): self.assertEqual(proc.stdout.rstrip().splitlines(), expected, proc) + def test_module_names(self): + self.assertIsInstance(sys.module_names, frozenset) + for name in sys.module_names: + self.assertIsInstance(name, str) + @test.support.cpython_only class UnraisableHookTest(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2021-01-18-11-59-46.bpo-42955.CSWLC9.rst b/Misc/NEWS.d/next/Library/2021-01-18-11-59-46.bpo-42955.CSWLC9.rst new file mode 100644 index 00000000000..0631acd7a98 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-01-18-11-59-46.bpo-42955.CSWLC9.rst @@ -0,0 +1,2 @@ +Add :data:`sys.module_names`, containing the list of the standard library +module names. Patch by Victor Stinner. diff --git a/Python/module_names.h b/Python/module_names.h index 533a73260ef..0dc2633916d 100644 --- a/Python/module_names.h +++ b/Python/module_names.h @@ -1,63 +1,123 @@ // Auto-generated by Tools/scripts/generate_module_names.py. +// List used to create sys.module_names. static const char* _Py_module_names[] = { - -// Built-in modules +"__future__", "_abc", +"_aix_support", "_ast", +"_asyncio", +"_bisect", +"_blake2", +"_bootsubprocess", +"_bz2", "_codecs", +"_codecs_cn", +"_codecs_hk", +"_codecs_iso2022", +"_codecs_jp", +"_codecs_kr", +"_codecs_tw", "_collections", +"_collections_abc", +"_compat_pickle", +"_compression", +"_contextvars", +"_crypt", +"_csv", +"_ctypes", +"_curses", +"_curses_panel", +"_datetime", +"_dbm", +"_decimal", +"_elementtree", "_functools", +"_gdbm", +"_hashlib", +"_heapq", "_imp", "_io", +"_json", "_locale", +"_lsprof", +"_lzma", +"_markupbase", +"_md5", +"_msi", +"_multibytecodec", +"_multiprocessing", +"_opcode", "_operator", +"_osx_support", +"_pickle", +"_posixshmem", +"_posixsubprocess", +"_py_abc", +"_pydecimal", +"_pyio", +"_queue", +"_random", +"_sha1", +"_sha256", +"_sha3", +"_sha512", "_signal", +"_sitebuiltins", +"_socket", +"_sqlite3", "_sre", +"_ssl", "_stat", +"_statistics", "_string", +"_strptime", +"_struct", "_symtable", "_thread", +"_threading_local", +"_tkinter", "_tracemalloc", +"_uuid", "_warnings", "_weakref", -"atexit", -"builtins", -"errno", -"faulthandler", -"gc", -"itertools", -"marshal", -"posix", -"pwd", -"sys", -"time", - -// Pure Python modules (Lib/*.py) -"__future__", +"_weakrefset", +"_winapi", +"_xxsubinterpreters", +"_zoneinfo", "abc", "aifc", "antigravity", "argparse", +"array", "ast", "asynchat", +"asyncio", "asyncore", +"atexit", +"audioop", "base64", "bdb", +"binascii", "binhex", "bisect", +"builtins", "bz2", "cProfile", "calendar", "cgi", "cgitb", "chunk", +"cmath", "cmd", "code", "codecs", "codeop", +"collections", "colorsys", "compileall", +"concurrent", +"concurrent.futures", "configparser", "contextlib", "contextvars", @@ -65,45 +125,80 @@ static const char* _Py_module_names[] = { "copyreg", "crypt", "csv", +"ctypes", +"ctypes.macholib", +"curses", "dataclasses", "datetime", +"dbm", "decimal", "difflib", "dis", +"distutils", +"distutils.command", "doctest", +"email", +"email.mime", +"encodings", +"ensurepip", +"ensurepip._bundled", "enum", +"errno", +"faulthandler", +"fcntl", "filecmp", "fileinput", "fnmatch", "fractions", "ftplib", "functools", +"gc", "genericpath", "getopt", "getpass", "gettext", "glob", "graphlib", +"grp", "gzip", "hashlib", "heapq", "hmac", +"html", +"http", +"idlelib", "imaplib", "imghdr", "imp", +"importlib", "inspect", "io", "ipaddress", +"itertools", +"json", "keyword", +"lib2to3", +"lib2to3.fixes", +"lib2to3.pgen2", "linecache", "locale", +"logging", "lzma", "mailbox", "mailcap", +"marshal", +"math", "mimetypes", +"mmap", "modulefinder", +"msilib", +"msvcrt", +"multiprocessing", +"multiprocessing.dummy", "netrc", +"nis", "nntplib", +"nt", "ntpath", "nturl2path", "numbers", @@ -111,6 +206,7 @@ static const char* _Py_module_names[] = { "operator", "optparse", "os", +"ossaudiodev", "pathlib", "pdb", "pickle", @@ -120,23 +216,30 @@ static const char* _Py_module_names[] = { "platform", "plistlib", "poplib", +"posix", "posixpath", "pprint", "profile", "pstats", "pty", +"pwd", "py_compile", "pyclbr", "pydoc", +"pydoc_data", +"pyexpat", "queue", "quopri", "random", "re", +"readline", "reprlib", +"resource", "rlcompleter", "runpy", "sched", "secrets", +"select", "selectors", "shelve", "shlex", @@ -148,6 +251,8 @@ static const char* _Py_module_names[] = { "sndhdr", "socket", "socketserver", +"spwd", +"sqlite3", "sre_compile", "sre_constants", "sre_parse", @@ -160,15 +265,20 @@ static const char* _Py_module_names[] = { "subprocess", "sunau", "symtable", +"sys", "sysconfig", +"syslog", "tabnanny", "tarfile", "telnetlib", "tempfile", +"termios", "textwrap", "this", "threading", +"time", "timeit", +"tkinter", "token", "tokenize", "trace", @@ -176,161 +286,32 @@ static const char* _Py_module_names[] = { "tracemalloc", "tty", "turtle", +"turtledemo", "types", "typing", +"unicodedata", +"unittest", +"urllib", "uu", "uuid", +"venv", "warnings", "wave", "weakref", "webbrowser", -"xdrlib", -"zipapp", -"zipfile", -"zipimport", - -// Packages and sub-packages -"asyncio", -"collections", -"concurrent", -"concurrent.futures", -"ctypes", -"ctypes.macholib", -"curses", -"dbm", -"distutils", -"distutils.command", -"email", -"email.mime", -"encodings", -"ensurepip", -"ensurepip._bundled", -"html", -"http", -"idlelib", -"importlib", -"json", -"lib2to3", -"lib2to3.fixes", -"lib2to3.pgen2", -"logging", -"msilib", -"multiprocessing", -"multiprocessing.dummy", -"pydoc_data", -"sqlite3", -"tkinter", -"turtledemo", -"unittest", -"urllib", -"venv", +"winreg", +"winsound", "wsgiref", +"xdrlib", "xml", "xml.dom", "xml.etree", "xml.parsers", "xml.sax", "xmlrpc", -"zoneinfo", - -// Extension modules built by setup.py -"_asyncio", -"_bisect", -"_blake2", -"_bz2", -"_codecs_cn", -"_codecs_hk", -"_codecs_iso2022", -"_codecs_jp", -"_codecs_kr", -"_codecs_tw", -"_contextvars", -"_crypt", -"_csv", -"_ctypes", -"_curses", -"_curses_panel", -"_datetime", -"_dbm", -"_decimal", -"_elementtree", -"_gdbm", -"_hashlib", -"_heapq", -"_json", -"_lsprof", -"_lzma", -"_md5", -"_multibytecodec", -"_multiprocessing", -"_opcode", -"_pickle", -"_posixshmem", -"_posixsubprocess", -"_queue", -"_random", -"_sha1", -"_sha256", -"_sha3", -"_sha512", -"_socket", -"_sqlite3", -"_ssl", -"_statistics", -"_struct", -"_tkinter", -"_uuid", -"_xxsubinterpreters", -"_zoneinfo", -"array", -"audioop", -"binascii", -"cmath", -"fcntl", -"grp", -"math", -"mmap", -"nis", -"ossaudiodev", -"pyexpat", -"readline", -"resource", -"select", -"spwd", -"syslog", -"termios", -"unicodedata", +"zipapp", +"zipfile", +"zipimport", "zlib", - -// Built-in and extension modules built by Modules/Setup -"_abc", -"_codecs", -"_collections", -"_functools", -"_io", -"_locale", -"_operator", -"_signal", -"_sre", -"_stat", -"_symtable", -"_thread", -"_tracemalloc", -"_weakref", -"atexit", -"errno", -"faulthandler", -"itertools", -"posix", -"pwd", -"time", - -// Windows extension modules -"_msi", -"_winapi", -"msvcrt", -"nt", -"winreg", -"winsound", - +"zoneinfo", }; diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index e9df8fb5d27..a97f45d0d5d 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -18,8 +18,6 @@ #include "pycore_sysmodule.h" // _PySys_ClearAuditHooks() #include "pycore_traceback.h" // _Py_DumpTracebackThreads() -#include "module_names.h" // _Py_module_names - #include <locale.h> // setlocale() #ifdef HAVE_SIGNAL_H @@ -2499,7 +2497,7 @@ fatal_error_exit(int status) // Dump the list of extension modules of sys.modules, excluding stdlib modules -// (_Py_module_names), into fd file descriptor. +// (sys.module_names), into fd file descriptor. // // This function is called by a signal handler in faulthandler: avoid memory // allocations and keep the implementation simple. For example, the list is not @@ -2515,10 +2513,31 @@ _Py_DumpExtensionModules(int fd, PyInterpreterState *interp) return; } + Py_ssize_t pos; + PyObject *key, *value; + + // Avoid PyDict_GetItemString() which calls PyUnicode_FromString(), + // memory cannot be allocated on the heap in a signal handler. + // Iterate on the dict instead. + PyObject *module_names = NULL; + pos = 0; + while (PyDict_Next(interp->sysdict, &pos, &key, &value)) { + if (PyUnicode_Check(key) + && PyUnicode_CompareWithASCIIString(key, "module_names") == 0) { + module_names = value; + break; + } + } + // If we failed to get sys.module_names or it's not a frozenset, + // don't exclude stdlib modules. + if (module_names != NULL && !PyFrozenSet_Check(module_names)) { + module_names = NULL; + } + + // List extensions int header = 1; Py_ssize_t count = 0; - Py_ssize_t pos = 0; - PyObject *key, *value; + pos = 0; while (PyDict_Next(modules, &pos, &key, &value)) { if (!PyUnicode_Check(key)) { continue; @@ -2526,22 +2545,26 @@ _Py_DumpExtensionModules(int fd, PyInterpreterState *interp) if (!_PyModule_IsExtension(value)) { continue; } - - // Check if it is a stdlib extension module. // Use the module name from the sys.modules key, // don't attempt to get the module object name. - const Py_ssize_t names_len = Py_ARRAY_LENGTH(_Py_module_names); - int is_stdlib_mod = 0; - for (Py_ssize_t i=0; i < names_len; i++) { - const char *name = _Py_module_names[i]; - if (PyUnicode_CompareWithASCIIString(key, name) == 0) { - is_stdlib_mod = 1; - break; + if (module_names != NULL) { + int is_stdlib_ext = 0; + + Py_ssize_t i; + PyObject *item; + Py_hash_t hash; + for (i=0; _PySet_NextEntry(module_names, &i, &item, &hash); ) { + if (PyUnicode_Check(item) + && PyUnicode_Compare(key, item) == 0) + { + is_stdlib_ext = 1; + break; + } + } + if (is_stdlib_ext) { + // Ignore stdlib extension + continue; } - } - if (is_stdlib_mod) { - // Ignore stdlib extension module. - continue; } if (header) { diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 720532eade2..e2f7e39f333 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -29,6 +29,7 @@ Data members: #include "frameobject.h" // PyFrame_GetBack() #include "pydtrace.h" #include "osdefs.h" // DELIM +#include "module_names.h" // _Py_module_names #include <locale.h> #ifdef MS_WINDOWS @@ -2020,33 +2021,63 @@ static PyMethodDef sys_methods[] = { {NULL, NULL} /* sentinel */ }; + static PyObject * list_builtin_module_names(void) { PyObject *list = PyList_New(0); - int i; - if (list == NULL) + if (list == NULL) { return NULL; - for (i = 0; PyImport_Inittab[i].name != NULL; i++) { - PyObject *name = PyUnicode_FromString( - PyImport_Inittab[i].name); - if (name == NULL) - break; - PyList_Append(list, name); + } + for (Py_ssize_t i = 0; PyImport_Inittab[i].name != NULL; i++) { + PyObject *name = PyUnicode_FromString(PyImport_Inittab[i].name); + if (name == NULL) { + goto error; + } + if (PyList_Append(list, name) < 0) { + Py_DECREF(name); + goto error; + } Py_DECREF(name); } if (PyList_Sort(list) != 0) { - Py_DECREF(list); - list = NULL; + goto error; + } + PyObject *tuple = PyList_AsTuple(list); + Py_DECREF(list); + return tuple; + +error: + Py_DECREF(list); + return NULL; +} + + +static PyObject * +list_module_names(void) +{ + Py_ssize_t len = Py_ARRAY_LENGTH(_Py_module_names); + PyObject *names = PyTuple_New(len); + if (names == NULL) { + return NULL; } - if (list) { - PyObject *v = PyList_AsTuple(list); - Py_DECREF(list); - list = v; + + for (Py_ssize_t i = 0; i < len; i++) { + PyObject *name = PyUnicode_FromString(_Py_module_names[i]); + if (name == NULL) { + Py_DECREF(names); + return NULL; + } + PyTuple_SET_ITEM(names, i, name); } - return list; + + PyObject *set = PyObject_CallFunction((PyObject *)&PyFrozenSet_Type, + "(O)", names); + Py_DECREF(names); + return set; } + /* Pre-initialization support for sys.warnoptions and sys._xoptions * * Modern internal code paths: @@ -2753,6 +2784,7 @@ _PySys_InitCore(PyThreadState *tstate, PyObject *sysdict) SET_SYS("hash_info", get_hash_info(tstate)); SET_SYS("maxunicode", PyLong_FromLong(0x10FFFF)); SET_SYS("builtin_module_names", list_builtin_module_names()); + SET_SYS("module_names", list_module_names()); #if PY_BIG_ENDIAN SET_SYS_FROM_STRING("byteorder", "big"); #else diff --git a/Tools/scripts/generate_module_names.py b/Tools/scripts/generate_module_names.py index 985a1a5e5a2..9d363aa04b3 100644 --- a/Tools/scripts/generate_module_names.py +++ b/Tools/scripts/generate_module_names.py @@ -17,27 +17,6 @@ IGNORE = { '__pycache__', 'site-packages', - # Helper modules of public modules. - # For example, sysconfig uses _osx_support. - '_aix_support', - '_collections_abc', - '_compat_pickle', - '_compression', - '_markupbase', - '_osx_support', - '_sitebuiltins', - '_strptime', - '_threading_local', - '_weakrefset', - - # Used to bootstrap setup.py - '_bootsubprocess', - - # pure Python implementation - '_py_abc', - '_pydecimal', - '_pyio', - # test modules '__phello__.foo', '_ctypes_test', @@ -69,40 +48,20 @@ WINDOWS_MODULES = ( ) -def write_comment(fp, comment): - print(f"// {comment}", file=fp) - - -def write_modules(fp, names): - for name in sorted(names): - if name in IGNORE: - continue - print(f'"{name}",', file=fp) - print(file=fp) - - -def list_builtin_modules(fp): - write_comment(fp, "Built-in modules") - write_modules(fp, sys.builtin_module_names) - - # Pure Python modules (Lib/*.py) -def list_python_modules(fp): - write_comment(fp, "Pure Python modules (Lib/*.py)") - names = [] +def list_python_modules(names): for filename in os.listdir(STDLIB_PATH): if not filename.endswith(".py"): continue name = filename.removesuffix(".py") - names.append(name) - write_modules(fp, names) + names.add(name) def _list_sub_packages(path, names, parent=None): for name in os.listdir(path): - package_path = os.path.join(path, name) if name in IGNORE: continue + package_path = os.path.join(path, name) if not os.path.isdir(package_path): continue if not any(package_file.endswith(".py") @@ -114,40 +73,28 @@ def _list_sub_packages(path, names, parent=None): qualname = name if qualname in IGNORE: continue - names.append(qualname) + names.add(qualname) _list_sub_packages(package_path, names, qualname) # Packages and sub-packages -def list_packages(fp): - write_comment(fp, "Packages and sub-packages") - names = [] +def list_packages(names): _list_sub_packages(STDLIB_PATH, names) - write_modules(fp, names) - - -# Windows extensions -def list_windows_extensions(fp): - write_comment(fp, "Windows extension modules") - write_modules(fp, WINDOWS_MODULES) # Extension modules built by setup.py -def list_setup(fp): +def list_setup_extensions(names): cmd = [sys.executable, SETUP_PY, "-q", "build", "--list-module-names"] output = subprocess.check_output(cmd) output = output.decode("utf8") - names = output.splitlines() - - write_comment(fp, "Extension modules built by setup.py") - write_modules(fp, names) + extensions = output.splitlines() + names |= set(extensions) # Built-in and extension modules built by Modules/Setup -def list_modules_setup(fp): +def list_modules_setup_extensions(names): assign_var = re.compile("^[A-Z]+=") - names = [] with open(MODULES_SETUP, encoding="utf-8") as modules_fp: for line in modules_fp: # Strip comment @@ -165,25 +112,26 @@ def list_modules_setup(fp): continue # "errno errnomodule.c" => write "errno" name = parts[0] - names.append(name) + names.add(name) - write_comment(fp, "Built-in and extension modules built by Modules/Setup") - write_modules(fp, names) + +def list_modules(): + names = set(sys.builtin_module_names) | set(WINDOWS_MODULES) + list_modules_setup_extensions(names) + list_setup_extensions(names) + list_packages(names) + list_python_modules(names) + names -= set(IGNORE) + return names -def list_modules(fp): +def write_modules(fp, names): print("// Auto-generated by Tools/scripts/generate_module_names.py.", file=fp) + print("// List used to create sys.module_names.", file=fp) print(file=fp) print("static const char* _Py_module_names[] = {", file=fp) - print(file=fp) - - list_builtin_modules(fp) - list_python_modules(fp) - list_packages(fp) - list_setup(fp) - list_modules_setup(fp) - list_windows_extensions(fp) - + for name in sorted(names): + print(f'"{name}",', file=fp) print("};", file=fp) @@ -193,7 +141,9 @@ def main(): file=sys.stderr) sys.exit(1) - list_modules(sys.stdout) + fp = sys.stdout + names = list_modules() + write_modules(fp, names) if __name__ == "__main__": |