Skip to content

Commit cf6e7c5

Browse files
ericsnowcurrentlyericsnowcurrently
authored
gh-100227: Isolate the Import State to Each Interpreter (gh-101941)
Specific changes: * move the import lock to PyInterpreterState * move the "find_and_load" diagnostic state to PyInterpreterState Note that the import lock exists to keep multiple imports of the same module in the same interpreter (but in different threads) from stomping on each other. Independently, we use a distinct global lock to protect globally shared import state, especially related to loaded extension modules. For now we can rely on the GIL as that lock but with a per-interpreter GIL we'll need a new global lock. The remaining state in _PyRuntimeState.imports will (probably) continue being global. #100227
1 parent b45d14b commit cf6e7c5

File tree

5 files changed

+85
-74
lines changed

5 files changed

+85
-74
lines changed

‎Include/cpython/import.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@ PyAPI_FUNC(PyObject *) _PyImport_GetModuleId(_Py_Identifier *name);
1010
PyAPI_FUNC(int) _PyImport_SetModule(PyObject *name, PyObject *module);
1111
PyAPI_FUNC(int) _PyImport_SetModuleString(const char *name, PyObject* module);
1212

13-
PyAPI_FUNC(void) _PyImport_AcquireLock(void);
14-
PyAPI_FUNC(int) _PyImport_ReleaseLock(void);
13+
PyAPI_FUNC(void) _PyImport_AcquireLock(PyInterpreterState *interp);
14+
PyAPI_FUNC(int) _PyImport_ReleaseLock(PyInterpreterState *interp);
1515

1616
PyAPI_FUNC(int) _PyImport_FixupBuiltin(
1717
PyObject *mod,

‎Include/internal/pycore_import.h‎

Lines changed: 21 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -21,17 +21,6 @@ struct _import_runtime_state {
2121
This is initialized lazily in _PyImport_FixupExtensionObject().
2222
Modules are added there and looked up in _imp.find_extension(). */
2323
PyObject *extensions;
24-
/* The global import lock. */
25-
struct {
26-
PyThread_type_lock mutex;
27-
unsigned long thread;
28-
int level;
29-
} lock;
30-
struct {
31-
int import_level;
32-
_PyTime_t accumulated;
33-
int header;
34-
} find_and_load;
3524
/* Package context -- the full module name for package imports */
3625
const char * pkgcontext;
3726
};
@@ -69,6 +58,18 @@ struct _import_state {
6958
int dlopenflags;
7059
#endif
7160
PyObject *import_func;
61+
/* The global import lock. */
62+
struct {
63+
PyThread_type_lock mutex;
64+
unsigned long thread;
65+
int level;
66+
} lock;
67+
/* diagnostic info in PyImport_ImportModuleLevelObject() */
68+
struct {
69+
int import_level;
70+
_PyTime_t accumulated;
71+
int header;
72+
} find_and_load;
7273
};
7374

7475
#ifdef HAVE_DLOPEN
@@ -86,8 +87,15 @@ struct _import_state {
8687

8788
#define IMPORTS_INIT \
8889
{ \
89-
.override_frozen_modules = 0, \
9090
DLOPENFLAGS_INIT \
91+
.lock = { \
92+
.mutex = NULL, \
93+
.thread = PYTHREAD_INVALID_THREAD_ID, \
94+
.level = 0, \
95+
}, \
96+
.find_and_load = { \
97+
.header = 1, \
98+
}, \
9199
}
92100

93101
extern void _PyImport_ClearCore(PyInterpreterState *interp);
@@ -138,7 +146,7 @@ extern void _PyImport_FiniExternal(PyInterpreterState *interp);
138146

139147

140148
#ifdef HAVE_FORK
141-
extern PyStatus _PyImport_ReInitLock(void);
149+
extern PyStatus _PyImport_ReInitLock(PyInterpreterState *interp);
142150
#endif
143151

144152

‎Include/internal/pycore_runtime_init.h‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -40,16 +40,6 @@ extern PyTypeObject _PyExc_MemoryError;
4040
in accordance with the specification. */ \
4141
.autoTSSkey = Py_tss_NEEDS_INIT, \
4242
.parser = _parser_runtime_state_INIT, \
43-
.imports = { \
44-
.lock = { \
45-
.mutex = NULL, \
46-
.thread = PYTHREAD_INVALID_THREAD_ID, \
47-
.level = 0, \
48-
}, \
49-
.find_and_load = { \
50-
.header = 1, \
51-
}, \
52-
}, \
5343
.ceval = { \
5444
.perf = _PyEval_RUNTIME_PERF_INIT, \
5545
}, \

‎Modules/posixmodule.c‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -567,18 +567,21 @@ run_at_forkers(PyObject *lst, int reverse)
567567
void
568568
PyOS_BeforeFork(void)
569569
{
570-
run_at_forkers(_PyInterpreterState_GET()->before_forkers, 1);
570+
PyInterpreterState *interp = _PyInterpreterState_GET();
571+
run_at_forkers(interp->before_forkers, 1);
571572

572-
_PyImport_AcquireLock();
573+
_PyImport_AcquireLock(interp);
573574
}
574575

575576
void
576577
PyOS_AfterFork_Parent(void)
577578
{
578-
if (_PyImport_ReleaseLock() <= 0)
579+
PyInterpreterState *interp = _PyInterpreterState_GET();
580+
if (_PyImport_ReleaseLock(interp) <= 0) {
579581
Py_FatalError("failed releasing import lock after fork");
582+
}
580583

581-
run_at_forkers(_PyInterpreterState_GET()->after_forkers_parent, 0);
584+
run_at_forkers(interp->after_forkers_parent, 0);
582585
}
583586

584587
void
@@ -604,7 +607,7 @@ PyOS_AfterFork_Child(void)
604607
goto fatal_error;
605608
}
606609

607-
status = _PyImport_ReInitLock();
610+
status = _PyImport_ReInitLock(tstate->interp);
608611
if (_PyStatus_EXCEPTION(status)) {
609612
goto fatal_error;
610613
}

‎Python/import.c‎

Lines changed: 54 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -56,11 +56,6 @@ static struct _inittab *inittab_copy = NULL;
5656
#define LAST_MODULE_INDEX _PyRuntime.imports.last_module_index
5757
#define EXTENSIONS _PyRuntime.imports.extensions
5858

59-
#define import_lock _PyRuntime.imports.lock.mutex
60-
#define import_lock_thread _PyRuntime.imports.lock.thread
61-
#define import_lock_level _PyRuntime.imports.lock.level
62-
63-
#define FIND_AND_LOAD _PyRuntime.imports.find_and_load
6459
#define PKGCONTEXT (_PyRuntime.imports.pkgcontext)
6560

6661

@@ -85,6 +80,16 @@ static struct _inittab *inittab_copy = NULL;
8580
#define IMPORT_FUNC(interp) \
8681
(interp)->imports.import_func
8782

83+
#define IMPORT_LOCK(interp) \
84+
(interp)->imports.lock.mutex
85+
#define IMPORT_LOCK_THREAD(interp) \
86+
(interp)->imports.lock.thread
87+
#define IMPORT_LOCK_LEVEL(interp) \
88+
(interp)->imports.lock.level
89+
90+
#define FIND_AND_LOAD(interp) \
91+
(interp)->imports.find_and_load
92+
8893

8994
/*******************/
9095
/* the import lock */
@@ -95,45 +100,45 @@ static struct _inittab *inittab_copy = NULL;
95100
These calls are serialized by the global interpreter lock. */
96101

97102
void
98-
_PyImport_AcquireLock(void)
103+
_PyImport_AcquireLock(PyInterpreterState *interp)
99104
{
100105
unsigned long me = PyThread_get_thread_ident();
101106
if (me == PYTHREAD_INVALID_THREAD_ID)
102107
return; /* Too bad */
103-
if (import_lock == NULL) {
104-
import_lock = PyThread_allocate_lock();
105-
if (import_lock == NULL)
108+
if (IMPORT_LOCK(interp) == NULL) {
109+
IMPORT_LOCK(interp) = PyThread_allocate_lock();
110+
if (IMPORT_LOCK(interp) == NULL)
106111
return; /* Nothing much we can do. */
107112
}
108-
if (import_lock_thread == me) {
109-
import_lock_level++;
113+
if (IMPORT_LOCK_THREAD(interp) == me) {
114+
IMPORT_LOCK_LEVEL(interp)++;
110115
return;
111116
}
112-
if (import_lock_thread != PYTHREAD_INVALID_THREAD_ID ||
113-
!PyThread_acquire_lock(import_lock, 0))
117+
if (IMPORT_LOCK_THREAD(interp) != PYTHREAD_INVALID_THREAD_ID ||
118+
!PyThread_acquire_lock(IMPORT_LOCK(interp), 0))
114119
{
115120
PyThreadState *tstate = PyEval_SaveThread();
116-
PyThread_acquire_lock(import_lock, WAIT_LOCK);
121+
PyThread_acquire_lock(IMPORT_LOCK(interp), WAIT_LOCK);
117122
PyEval_RestoreThread(tstate);
118123
}
119-
assert(import_lock_level == 0);
120-
import_lock_thread = me;
121-
import_lock_level = 1;
124+
assert(IMPORT_LOCK_LEVEL(interp) == 0);
125+
IMPORT_LOCK_THREAD(interp) = me;
126+
IMPORT_LOCK_LEVEL(interp) = 1;
122127
}
123128

124129
int
125-
_PyImport_ReleaseLock(void)
130+
_PyImport_ReleaseLock(PyInterpreterState *interp)
126131
{
127132
unsigned long me = PyThread_get_thread_ident();
128-
if (me == PYTHREAD_INVALID_THREAD_ID || import_lock == NULL)
133+
if (me == PYTHREAD_INVALID_THREAD_ID || IMPORT_LOCK(interp) == NULL)
129134
return 0; /* Too bad */
130-
if (import_lock_thread != me)
135+
if (IMPORT_LOCK_THREAD(interp) != me)
131136
return -1;
132-
import_lock_level--;
133-
assert(import_lock_level >= 0);
134-
if (import_lock_level == 0) {
135-
import_lock_thread = PYTHREAD_INVALID_THREAD_ID;
136-
PyThread_release_lock(import_lock);
137+
IMPORT_LOCK_LEVEL(interp)--;
138+
assert(IMPORT_LOCK_LEVEL(interp) >= 0);
139+
if (IMPORT_LOCK_LEVEL(interp) == 0) {
140+
IMPORT_LOCK_THREAD(interp) = PYTHREAD_INVALID_THREAD_ID;
141+
PyThread_release_lock(IMPORT_LOCK(interp));
137142
}
138143
return 1;
139144
}
@@ -144,23 +149,23 @@ _PyImport_ReleaseLock(void)
144149
We now acquire the import lock around fork() calls but on some platforms
145150
(Solaris 9 and earlier? see isue7242) that still left us with problems. */
146151
PyStatus
147-
_PyImport_ReInitLock(void)
152+
_PyImport_ReInitLock(PyInterpreterState *interp)
148153
{
149-
if (import_lock != NULL) {
150-
if (_PyThread_at_fork_reinit(&import_lock) < 0) {
154+
if (IMPORT_LOCK(interp) != NULL) {
155+
if (_PyThread_at_fork_reinit(&IMPORT_LOCK(interp)) < 0) {
151156
return _PyStatus_ERR("failed to create a new lock");
152157
}
153158
}
154159

155-
if (import_lock_level > 1) {
160+
if (IMPORT_LOCK_LEVEL(interp) > 1) {
156161
/* Forked as a side effect of import */
157162
unsigned long me = PyThread_get_thread_ident();
158-
PyThread_acquire_lock(import_lock, WAIT_LOCK);
159-
import_lock_thread = me;
160-
import_lock_level--;
163+
PyThread_acquire_lock(IMPORT_LOCK(interp), WAIT_LOCK);
164+
IMPORT_LOCK_THREAD(interp) = me;
165+
IMPORT_LOCK_LEVEL(interp)--;
161166
} else {
162-
import_lock_thread = PYTHREAD_INVALID_THREAD_ID;
163-
import_lock_level = 0;
167+
IMPORT_LOCK_THREAD(interp) = PYTHREAD_INVALID_THREAD_ID;
168+
IMPORT_LOCK_LEVEL(interp) = 0;
164169
}
165170
return _PyStatus_OK();
166171
}
@@ -2506,8 +2511,8 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name)
25062511
PyObject *mod = NULL;
25072512
PyInterpreterState *interp = tstate->interp;
25082513
int import_time = _PyInterpreterState_GetConfig(interp)->import_time;
2509-
#define import_level FIND_AND_LOAD.import_level
2510-
#define accumulated FIND_AND_LOAD.accumulated
2514+
#define import_level FIND_AND_LOAD(interp).import_level
2515+
#define accumulated FIND_AND_LOAD(interp).accumulated
25112516

25122517
_PyTime_t t1 = 0, accumulated_copy = accumulated;
25132518

@@ -2528,7 +2533,7 @@ import_find_and_load(PyThreadState *tstate, PyObject *abs_name)
25282533
* _PyDict_GetItemIdWithError().
25292534
*/
25302535
if (import_time) {
2531-
#define header FIND_AND_LOAD.header
2536+
#define header FIND_AND_LOAD(interp).header
25322537
if (header) {
25332538
fputs("import time: self [us] | cumulative | imported package\n",
25342539
stderr);
@@ -2867,10 +2872,6 @@ _PyImport_Fini(void)
28672872
{
28682873
/* Destroy the database used by _PyImport_{Fixup,Find}Extension */
28692874
_extensions_cache_clear_all();
2870-
if (import_lock != NULL) {
2871-
PyThread_free_lock(import_lock);
2872-
import_lock = NULL;
2873-
}
28742875

28752876
/* Use the same memory allocator as _PyImport_Init(). */
28762877
PyMemAllocatorEx old_alloc;
@@ -2959,6 +2960,11 @@ _PyImport_FiniCore(PyInterpreterState *interp)
29592960
PyErr_WriteUnraisable(NULL);
29602961
}
29612962

2963+
if (IMPORT_LOCK(interp) != NULL) {
2964+
PyThread_free_lock(IMPORT_LOCK(interp));
2965+
IMPORT_LOCK(interp) = NULL;
2966+
}
2967+
29622968
_PyImport_ClearCore(interp);
29632969
}
29642970

@@ -3090,7 +3096,9 @@ static PyObject *
30903096
_imp_lock_held_impl(PyObject *module)
30913097
/*[clinic end generated code: output=8b89384b5e1963fc input=9b088f9b217d9bdf]*/
30923098
{
3093-
return PyBool_FromLong(import_lock_thread != PYTHREAD_INVALID_THREAD_ID);
3099+
PyInterpreterState *interp = _PyInterpreterState_GET();
3100+
return PyBool_FromLong(
3101+
IMPORT_LOCK_THREAD(interp) != PYTHREAD_INVALID_THREAD_ID);
30943102
}
30953103

30963104
/*[clinic input]
@@ -3106,7 +3114,8 @@ static PyObject *
31063114
_imp_acquire_lock_impl(PyObject *module)
31073115
/*[clinic end generated code: output=1aff58cb0ee1b026 input=4a2d4381866d5fdc]*/
31083116
{
3109-
_PyImport_AcquireLock();
3117+
PyInterpreterState *interp = _PyInterpreterState_GET();
3118+
_PyImport_AcquireLock(interp);
31103119
Py_RETURN_NONE;
31113120
}
31123121

@@ -3122,7 +3131,8 @@ static PyObject *
31223131
_imp_release_lock_impl(PyObject *module)
31233132
/*[clinic end generated code: output=7faab6d0be178b0a input=934fb11516dd778b]*/
31243133
{
3125-
if (_PyImport_ReleaseLock() < 0) {
3134+
PyInterpreterState *interp = _PyInterpreterState_GET();
3135+
if (_PyImport_ReleaseLock(interp) < 0) {
31263136
PyErr_SetString(PyExc_RuntimeError,
31273137
"not holding the import lock");
31283138
return NULL;

0 commit comments

Comments
 (0)