[Python-checkins] bpo-32030: Add _PyCoreConfig.module_search_path_env (#4504)
Victor Stinner
webhook-mailer at python.org
Wed Nov 22 18:12:12 EST 2017
https://github.com/python/cpython/commit/d4341109746aa15e1909e63b30b93b6133ffe401
commit: d4341109746aa15e1909e63b30b93b6133ffe401
branch: master
author: Victor Stinner <victor.stinner at gmail.com>
committer: GitHub <noreply at github.com>
date: 2017-11-23T00:12:09+01:00
summary:
bpo-32030: Add _PyCoreConfig.module_search_path_env (#4504)
Changes:
* Py_Main() initializes _PyCoreConfig.module_search_path_env from
the PYTHONPATH environment variable.
* PyInterpreterState_New() now initializes core_config and config
fields
* Compute sys.path a little bit ealier in
_Py_InitializeMainInterpreter() and new_interpreter()
* Add _Py_GetPathWithConfig() private function.
files:
M Include/pylifecycle.h
M Include/pystate.h
M Modules/getpath.c
M Modules/main.c
M PC/getpathp.c
M Python/pylifecycle.c
M Python/pystate.c
diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h
index a75b77cc733..2a5e73a79ca 100644
--- a/Include/pylifecycle.h
+++ b/Include/pylifecycle.h
@@ -93,6 +93,9 @@ PyAPI_FUNC(wchar_t *) Py_GetProgramFullPath(void);
PyAPI_FUNC(wchar_t *) Py_GetPrefix(void);
PyAPI_FUNC(wchar_t *) Py_GetExecPrefix(void);
PyAPI_FUNC(wchar_t *) Py_GetPath(void);
+#ifdef Py_BUILD_CORE
+PyAPI_FUNC(wchar_t *) _Py_GetPathWithConfig(_PyCoreConfig *config);
+#endif
PyAPI_FUNC(void) Py_SetPath(const wchar_t *);
#ifdef MS_WINDOWS
int _Py_CheckPython3();
diff --git a/Include/pystate.h b/Include/pystate.h
index a3840c96cb0..c5d3c3345f0 100644
--- a/Include/pystate.h
+++ b/Include/pystate.h
@@ -30,6 +30,7 @@ typedef struct {
unsigned long hash_seed;
int _disable_importlib; /* Needed by freeze_importlib */
const char *allocator; /* Memory allocator: _PyMem_SetupAllocators() */
+ wchar_t *module_search_path_env; /* PYTHONPATH environment variable */
int dev_mode; /* -X dev */
int faulthandler; /* -X faulthandler */
int tracemalloc; /* -X tracemalloc=N */
@@ -39,11 +40,13 @@ typedef struct {
} _PyCoreConfig;
#define _PyCoreConfig_INIT \
- {.ignore_environment = 0, \
+ (_PyCoreConfig){\
+ .ignore_environment = 0, \
.use_hash_seed = -1, \
.hash_seed = 0, \
._disable_importlib = 0, \
.allocator = NULL, \
+ .module_search_path_env = NULL, \
.dev_mode = 0, \
.faulthandler = 0, \
.tracemalloc = 0, \
@@ -61,7 +64,9 @@ typedef struct {
int install_signal_handlers;
} _PyMainInterpreterConfig;
-#define _PyMainInterpreterConfig_INIT {-1}
+#define _PyMainInterpreterConfig_INIT \
+ (_PyMainInterpreterConfig){\
+ .install_signal_handlers = -1}
typedef struct _is {
diff --git a/Modules/getpath.c b/Modules/getpath.c
index dd3387a9d77..ad4a4e5e718 100644
--- a/Modules/getpath.c
+++ b/Modules/getpath.c
@@ -456,14 +456,12 @@ search_for_exec_prefix(wchar_t *argv0_path, wchar_t *home,
}
static void
-calculate_path(void)
+calculate_path(_PyCoreConfig *core_config)
{
extern wchar_t *Py_GetProgramName(void);
static const wchar_t delimiter[2] = {DELIM, '\0'};
static const wchar_t separator[2] = {SEP, '\0'};
- char *_rtpypath = Py_GETENV("PYTHONPATH"); /* XXX use wide version on Windows */
- wchar_t *rtpypath = NULL;
wchar_t *home = Py_GetPythonHome();
char *_path = getenv("PATH");
wchar_t *path_buffer = NULL;
@@ -707,11 +705,22 @@ calculate_path(void)
*/
bufsz = 0;
- if (_rtpypath && _rtpypath[0] != '\0') {
- size_t rtpypath_len;
- rtpypath = Py_DecodeLocale(_rtpypath, &rtpypath_len);
- if (rtpypath != NULL)
- bufsz += rtpypath_len + 1;
+ wchar_t *env_path = NULL;
+ if (core_config) {
+ if (core_config->module_search_path_env) {
+ bufsz += wcslen(core_config->module_search_path_env) + 1;
+ }
+ }
+ else {
+ char *env_pathb = Py_GETENV("PYTHONPATH");
+ if (env_pathb && env_pathb[0] != '\0') {
+ size_t env_path_len;
+ env_path = Py_DecodeLocale(env_pathb, &env_path_len);
+ /* FIXME: handle decoding and memory error */
+ if (env_path != NULL) {
+ bufsz += env_path_len + 1;
+ }
+ }
}
defpath = _pythonpath;
@@ -742,12 +751,20 @@ calculate_path(void)
}
/* Run-time value of $PYTHONPATH goes first */
- if (rtpypath) {
- wcscpy(buf, rtpypath);
- wcscat(buf, delimiter);
+ buf[0] = '\0';
+ if (core_config) {
+ if (core_config->module_search_path_env) {
+ wcscpy(buf, core_config->module_search_path_env);
+ wcscat(buf, delimiter);
+ }
}
- else
- buf[0] = '\0';
+ else {
+ if (env_path) {
+ wcscpy(buf, env_path);
+ wcscat(buf, delimiter);
+ }
+ }
+ PyMem_RawFree(env_path);
/* Next is the default zip path */
wcscat(buf, zip_path);
@@ -818,7 +835,6 @@ calculate_path(void)
PyMem_RawFree(_prefix);
PyMem_RawFree(_exec_prefix);
PyMem_RawFree(lib_python);
- PyMem_RawFree(rtpypath);
}
@@ -841,11 +857,20 @@ Py_SetPath(const wchar_t *path)
}
}
+wchar_t *
+_Py_GetPathWithConfig(_PyCoreConfig *core_config)
+{
+ if (!module_search_path) {
+ calculate_path(core_config);
+ }
+ return module_search_path;
+}
+
wchar_t *
Py_GetPath(void)
{
if (!module_search_path)
- calculate_path();
+ calculate_path(NULL);
return module_search_path;
}
@@ -853,7 +878,7 @@ wchar_t *
Py_GetPrefix(void)
{
if (!module_search_path)
- calculate_path();
+ calculate_path(NULL);
return prefix;
}
@@ -861,7 +886,7 @@ wchar_t *
Py_GetExecPrefix(void)
{
if (!module_search_path)
- calculate_path();
+ calculate_path(NULL);
return exec_prefix;
}
@@ -869,7 +894,7 @@ wchar_t *
Py_GetProgramFullPath(void)
{
if (!module_search_path)
- calculate_path();
+ calculate_path(NULL);
return progpath;
}
diff --git a/Modules/main.c b/Modules/main.c
index e5e4f33f51c..3bd93e37601 100644
--- a/Modules/main.c
+++ b/Modules/main.c
@@ -399,6 +399,7 @@ typedef struct {
_PyInitError err;
/* PYTHONWARNINGS env var */
_Py_OptList env_warning_options;
+ /* PYTHONPATH env var */
int argc;
wchar_t **argv;
} _PyMain;
@@ -441,6 +442,8 @@ pymain_free_impl(_PyMain *pymain)
Py_CLEAR(pymain->main_importer_path);
PyMem_RawFree(pymain->program_name);
+ PyMem_RawFree(pymain->core_config.module_search_path_env);
+
#ifdef __INSURE__
/* Insure++ is a memory analysis tool that aids in discovering
* memory leaks and other memory problems. On Python exit, the
@@ -502,15 +505,22 @@ pymain_run_main_from_importer(_PyMain *pymain)
static wchar_t*
-pymain_strdup(_PyMain *pymain, wchar_t *str)
+pymain_wstrdup(_PyMain *pymain, wchar_t *str)
{
size_t len = wcslen(str) + 1; /* +1 for NUL character */
- wchar_t *str2 = PyMem_RawMalloc(sizeof(wchar_t) * len);
+ if (len > (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t)) {
+ pymain->err = INIT_NO_MEMORY();
+ return NULL;
+ }
+
+ size_t size = len * sizeof(wchar_t);
+ wchar_t *str2 = PyMem_RawMalloc(size);
if (str2 == NULL) {
pymain->err = INIT_NO_MEMORY();
return NULL;
}
- memcpy(str2, str, len * sizeof(wchar_t));
+
+ memcpy(str2, str, size);
return str2;
}
@@ -518,7 +528,7 @@ pymain_strdup(_PyMain *pymain, wchar_t *str)
static int
pymain_optlist_append(_PyMain *pymain, _Py_OptList *list, wchar_t *str)
{
- wchar_t *str2 = pymain_strdup(pymain, str);
+ wchar_t *str2 = pymain_wstrdup(pymain, str);
if (str2 == NULL) {
return -1;
}
@@ -762,14 +772,12 @@ pymain_warnings_envvar(_PyMain *pymain)
wchar_t *wp;
if ((wp = _wgetenv(L"PYTHONWARNINGS")) && *wp != L'\0') {
- wchar_t *buf, *warning, *context = NULL;
+ wchar_t *warning, *context = NULL;
- buf = (wchar_t *)PyMem_RawMalloc((wcslen(wp) + 1) * sizeof(wchar_t));
+ wchar_t *buf = pymain_wstrdup(pymain, wp);
if (buf == NULL) {
- pymain->err = INIT_NO_MEMORY();
return -1;
}
- wcscpy(buf, wp);
for (warning = wcstok_s(buf, L",", &context);
warning != NULL;
warning = wcstok_s(NULL, L",", &context)) {
@@ -805,12 +813,11 @@ pymain_warnings_envvar(_PyMain *pymain)
if (len == (size_t)-2) {
pymain->err = _Py_INIT_ERR("failed to decode "
"PYTHONWARNINGS");
- return -1;
}
else {
pymain->err = INIT_NO_MEMORY();
- return -1;
}
+ return -1;
}
if (pymain_optlist_append(pymain, &pymain->env_warning_options,
warning) < 0) {
@@ -929,7 +936,7 @@ pymain_get_program_name(_PyMain *pymain)
if (pymain->program_name == NULL) {
/* Use argv[0] by default */
- pymain->program_name = pymain_strdup(pymain, pymain->argv[0]);
+ pymain->program_name = pymain_wstrdup(pymain, pymain->argv[0]);
if (pymain->program_name == NULL) {
return -1;
}
@@ -1362,6 +1369,48 @@ pymain_set_flags_from_env(_PyMain *pymain)
}
+static int
+pymain_init_pythonpath(_PyMain *pymain)
+{
+ if (Py_IgnoreEnvironmentFlag) {
+ return 0;
+ }
+
+#ifdef MS_WINDOWS
+ wchar_t *path = _wgetenv(L"PYTHONPATH");
+ if (!path || path[0] == '\0') {
+ return 0;
+ }
+
+ wchar_t *path2 = pymain_wstrdup(pymain, path);
+ if (path2 == NULL) {
+ return -1;
+ }
+
+ pymain->core_config.module_search_path_env = path2;
+#else
+ char *path = pymain_get_env_var("PYTHONPATH");
+ if (!path) {
+ return 0;
+ }
+
+ size_t len;
+ wchar_t *wpath = Py_DecodeLocale(path, &len);
+ if (!wpath) {
+ if (len == (size_t)-2) {
+ pymain->err = _Py_INIT_ERR("failed to decode PYTHONHOME");
+ }
+ else {
+ pymain->err = INIT_NO_MEMORY();
+ }
+ return -1;
+ }
+ pymain->core_config.module_search_path_env = wpath;
+#endif
+ return 0;
+}
+
+
static int
pymain_parse_envvars(_PyMain *pymain)
{
@@ -1383,6 +1432,9 @@ pymain_parse_envvars(_PyMain *pymain)
return -1;
}
core_config->allocator = Py_GETENV("PYTHONMALLOC");
+ if (pymain_init_pythonpath(pymain) < 0) {
+ return -1;
+ }
/* -X options */
if (pymain_get_xoption(pymain, L"showrefcount")) {
diff --git a/PC/getpathp.c b/PC/getpathp.c
index 9bbc7bf0b27..b182ae6a58d 100644
--- a/PC/getpathp.c
+++ b/PC/getpathp.c
@@ -624,7 +624,7 @@ read_pth_file(const wchar_t *path, wchar_t *prefix, int *isolated, int *nosite)
static void
-calculate_path(void)
+calculate_path(_PyCoreConfig *core_config)
{
wchar_t argv0_path[MAXPATHLEN+1];
wchar_t *buf;
@@ -637,8 +637,13 @@ calculate_path(void)
wchar_t *userpath = NULL;
wchar_t zip_path[MAXPATHLEN+1];
- if (!Py_IgnoreEnvironmentFlag) {
+ if (core_config) {
+ envpath = core_config->module_search_path_env;
+ }
+ else if (!Py_IgnoreEnvironmentFlag) {
envpath = _wgetenv(L"PYTHONPATH");
+ if (envpath && *envpath == '\0')
+ envpath = NULL;
}
get_progpath();
@@ -709,9 +714,6 @@ calculate_path(void)
else
wcscpy_s(prefix, MAXPATHLEN+1, pythonhome);
- if (envpath && *envpath == '\0')
- envpath = NULL;
-
skiphome = pythonhome==NULL ? 0 : 1;
#ifdef Py_ENABLE_SHARED
@@ -896,11 +898,20 @@ Py_SetPath(const wchar_t *path)
}
}
+wchar_t *
+_Py_GetPathWithConfig(_PyCoreConfig *core_config)
+{
+ if (!module_search_path) {
+ calculate_path(core_config);
+ }
+ return module_search_path;
+}
+
wchar_t *
Py_GetPath(void)
{
if (!module_search_path)
- calculate_path();
+ calculate_path(NULL);
return module_search_path;
}
@@ -908,7 +919,7 @@ wchar_t *
Py_GetPrefix(void)
{
if (!module_search_path)
- calculate_path();
+ calculate_path(NULL);
return prefix;
}
@@ -922,7 +933,7 @@ wchar_t *
Py_GetProgramFullPath(void)
{
if (!module_search_path)
- calculate_path();
+ calculate_path(NULL);
return progpath;
}
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index 36fcf61c33d..9eeac9d33be 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -48,8 +48,6 @@ _Py_IDENTIFIER(threading);
extern "C" {
#endif
-extern wchar_t *Py_GetPath(void);
-
extern grammar _PyParser_Grammar; /* From graminit.c */
/* Forward */
@@ -842,6 +840,11 @@ _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *config)
/* Now finish configuring the main interpreter */
interp->config = *config;
+ /* GetPath may initialize state that _PySys_EndInit locks
+ in, and so has to be called first. */
+ /* TODO: Call Py_GetPath() in Py_ReadConfig, rather than here */
+ wchar_t *sys_path = _Py_GetPathWithConfig(&interp->core_config);
+
if (interp->core_config._disable_importlib) {
/* Special mode for freeze_importlib: run with no import system
*
@@ -857,10 +860,7 @@ _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *config)
return _Py_INIT_ERR("can't initialize time");
/* Finish setting up the sys module and import system */
- /* GetPath may initialize state that _PySys_EndInit locks
- in, and so has to be called first. */
- /* TODO: Call Py_GetPath() in Py_ReadConfig, rather than here */
- PySys_SetPath(Py_GetPath());
+ PySys_SetPath(sys_path);
if (_PySys_EndInit(interp->sysdict) < 0)
return _Py_INIT_ERR("can't finish initializing sys");
@@ -1301,6 +1301,8 @@ new_interpreter(PyThreadState **tstate_p)
/* XXX The following is lax in error checking */
+ wchar_t *sys_path = _Py_GetPathWithConfig(&interp->core_config);
+
PyObject *modules = PyDict_New();
if (modules == NULL) {
return _Py_INIT_ERR("can't make modules dictionary");
@@ -1314,7 +1316,7 @@ new_interpreter(PyThreadState **tstate_p)
goto handle_error;
Py_INCREF(interp->sysdict);
PyDict_SetItemString(interp->sysdict, "modules", modules);
- PySys_SetPath(Py_GetPath());
+ PySys_SetPath(sys_path);
_PySys_EndInit(interp->sysdict);
}
diff --git a/Python/pystate.c b/Python/pystate.c
index 807ac4eb9d1..f6fbb4d041e 100644
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -106,55 +106,60 @@ PyInterpreterState_New(void)
PyInterpreterState *interp = (PyInterpreterState *)
PyMem_RawMalloc(sizeof(PyInterpreterState));
- if (interp != NULL) {
- interp->modules = NULL;
- interp->modules_by_index = NULL;
- interp->sysdict = NULL;
- interp->builtins = NULL;
- interp->builtins_copy = NULL;
- interp->tstate_head = NULL;
- interp->check_interval = 100;
- interp->num_threads = 0;
- interp->pythread_stacksize = 0;
- interp->codec_search_path = NULL;
- interp->codec_search_cache = NULL;
- interp->codec_error_registry = NULL;
- interp->codecs_initialized = 0;
- interp->fscodec_initialized = 0;
- interp->importlib = NULL;
- interp->import_func = NULL;
- interp->eval_frame = _PyEval_EvalFrameDefault;
- interp->co_extra_user_count = 0;
+ if (interp == NULL) {
+ return NULL;
+ }
+
+
+ interp->modules = NULL;
+ interp->modules_by_index = NULL;
+ interp->sysdict = NULL;
+ interp->builtins = NULL;
+ interp->builtins_copy = NULL;
+ interp->tstate_head = NULL;
+ interp->check_interval = 100;
+ interp->num_threads = 0;
+ interp->pythread_stacksize = 0;
+ interp->codec_search_path = NULL;
+ interp->codec_search_cache = NULL;
+ interp->codec_error_registry = NULL;
+ interp->codecs_initialized = 0;
+ interp->fscodec_initialized = 0;
+ interp->core_config = _PyCoreConfig_INIT;
+ interp->config = _PyMainInterpreterConfig_INIT;
+ interp->importlib = NULL;
+ interp->import_func = NULL;
+ interp->eval_frame = _PyEval_EvalFrameDefault;
+ interp->co_extra_user_count = 0;
#ifdef HAVE_DLOPEN
#if HAVE_DECL_RTLD_NOW
- interp->dlopenflags = RTLD_NOW;
+ interp->dlopenflags = RTLD_NOW;
#else
- interp->dlopenflags = RTLD_LAZY;
+ interp->dlopenflags = RTLD_LAZY;
#endif
#endif
#ifdef HAVE_FORK
- interp->before_forkers = NULL;
- interp->after_forkers_parent = NULL;
- interp->after_forkers_child = NULL;
+ interp->before_forkers = NULL;
+ interp->after_forkers_parent = NULL;
+ interp->after_forkers_child = NULL;
#endif
- HEAD_LOCK();
- interp->next = _PyRuntime.interpreters.head;
- if (_PyRuntime.interpreters.main == NULL) {
- _PyRuntime.interpreters.main = interp;
- }
- _PyRuntime.interpreters.head = interp;
- if (_PyRuntime.interpreters.next_id < 0) {
- /* overflow or Py_Initialize() not called! */
- PyErr_SetString(PyExc_RuntimeError,
- "failed to get an interpreter ID");
- interp = NULL;
- } else {
- interp->id = _PyRuntime.interpreters.next_id;
- _PyRuntime.interpreters.next_id += 1;
- }
- HEAD_UNLOCK();
+ HEAD_LOCK();
+ interp->next = _PyRuntime.interpreters.head;
+ if (_PyRuntime.interpreters.main == NULL) {
+ _PyRuntime.interpreters.main = interp;
+ }
+ _PyRuntime.interpreters.head = interp;
+ if (_PyRuntime.interpreters.next_id < 0) {
+ /* overflow or Py_Initialize() not called! */
+ PyErr_SetString(PyExc_RuntimeError,
+ "failed to get an interpreter ID");
+ interp = NULL;
+ } else {
+ interp->id = _PyRuntime.interpreters.next_id;
+ _PyRuntime.interpreters.next_id += 1;
}
+ HEAD_UNLOCK();
return interp;
}
More information about the Python-checkins
mailing list