diff --git a/Include/compile.h b/Include/compile.h index 3cc351d409..837e4afbea 100644 --- a/Include/compile.h +++ b/Include/compile.h @@ -47,18 +47,18 @@ typedef struct { #define FUTURE_GENERATOR_STOP "generator_stop" struct _mod; /* Declare the existence of this type */ -#define PyAST_Compile(mod, s, f, ar) PyAST_CompileEx(mod, s, f, -1, ar) +#define PyAST_Compile(mod, s, f, ar) PyAST_CompileEx(mod, s, f, NULL, ar) PyAPI_FUNC(PyCodeObject *) PyAST_CompileEx( struct _mod *mod, const char *filename, /* decoded from the filesystem encoding */ PyCompilerFlags *flags, - int optimize, + PyObject *optimizations, PyArena *arena); PyAPI_FUNC(PyCodeObject *) PyAST_CompileObject( struct _mod *mod, PyObject *filename, PyCompilerFlags *flags, - int optimize, + PyObject *optimizations, PyArena *arena); PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromAST( struct _mod * mod, diff --git a/Include/pythonrun.h b/Include/pythonrun.h index 6f0c6fc655..31625d002c 100644 --- a/Include/pythonrun.h +++ b/Include/pythonrun.h @@ -100,19 +100,19 @@ PyAPI_FUNC(PyObject *) PyRun_FileExFlags( #ifdef Py_LIMITED_API PyAPI_FUNC(PyObject *) Py_CompileString(const char *, const char *, int); #else -#define Py_CompileString(str, p, s) Py_CompileStringExFlags(str, p, s, NULL, -1) -#define Py_CompileStringFlags(str, p, s, f) Py_CompileStringExFlags(str, p, s, f, -1) +#define Py_CompileString(str, p, s) Py_CompileStringExFlags(str, p, s, NULL, NULL) +#define Py_CompileStringFlags(str, p, s, f) Py_CompileStringExFlags(str, p, s, f, NULL) PyAPI_FUNC(PyObject *) Py_CompileStringExFlags( const char *str, const char *filename, /* decoded from the filesystem encoding */ int start, PyCompilerFlags *flags, - int optimize); + PyObject *optimizations); PyAPI_FUNC(PyObject *) Py_CompileStringObject( const char *str, PyObject *filename, int start, PyCompilerFlags *flags, - int optimize); + PyObject *optimizations); #endif PyAPI_FUNC(struct symtable *) Py_SymtableString( const char *str, diff --git a/Include/sysmodule.h b/Include/sysmodule.h index c5547ff674..62bba3bd94 100644 --- a/Include/sysmodule.h +++ b/Include/sysmodule.h @@ -25,6 +25,9 @@ PyAPI_FUNC(void) PySys_WriteStderr(const char *format, ...) PyAPI_FUNC(void) PySys_FormatStdout(const char *format, ...); PyAPI_FUNC(void) PySys_FormatStderr(const char *format, ...); +PyAPI_FUNC(void) PySys_SetOptimizations(PyObject *); +PyAPI_FUNC(PyObject *) PySys_GetOptimizations(void); + PyAPI_FUNC(void) PySys_ResetWarnOptions(void); PyAPI_FUNC(void) PySys_AddWarnOption(const wchar_t *); PyAPI_FUNC(void) PySys_AddWarnOptionUnicode(PyObject *); diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 9d949b74cb..87dcda7b43 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -328,19 +328,22 @@ class BuiltinTest(unittest.TestCase): codestr = '''def f(): """doc""" + debug_enabled = False + if __debug__: + debug_enabled = True try: assert False except AssertionError: - return (True, f.__doc__) + return (True, f.__doc__, debug_enabled) else: - return (False, f.__doc__) + return (False, f.__doc__, debug_enabled) ''' def f(): """doc""" - values = [(-1, __debug__, f.__doc__), - (0, True, 'doc'), - (1, False, 'doc'), - (2, False, None)] - for optval, debugval, docstring in values: + values = [(-1, __debug__, f.__doc__, __debug__), + (0, True, 'doc', True), + (1, False, 'doc', False), + (2, False, None, False)] + for optval, assertval, docstring, debugval in values: # test both direct compilation and compilation via AST codeobjs = [] codeobjs.append(compile(codestr, "", "exec", optimize=optval)) @@ -350,7 +353,7 @@ class BuiltinTest(unittest.TestCase): ns = {} exec(code, ns) rv = ns['f']() - self.assertEqual(rv, (debugval, docstring)) + self.assertEqual(rv, (assertval, docstring, debugval)) def test_delattr(self): sys.spam = 1 diff --git a/Modules/main.c b/Modules/main.c index 08b22760de..051d9e4792 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -40,7 +40,7 @@ static wchar_t **orig_argv; static int orig_argc; /* command line options */ -#define BASE_OPTS L"bBc:dEhiIJm:OqRsStuvVW:xX:?" +#define BASE_OPTS L"bBc:dEhiIJm:N:OqRsStuvVW:xX:?" #define PROGRAM_OPTS BASE_OPTS @@ -64,6 +64,7 @@ static const char usage_2[] = "\ if stdin does not appear to be a terminal; also PYTHONINSPECT=x\n\ -I : isolate Python from the user's environment (implies -E and -s)\n\ -m mod : run library module as a script (terminates option list)\n\ +-N : optimization flags: nodebug, noassert, nodocstring\n\ -O : optimize generated bytecode slightly; also PYTHONOPTIMIZE=x\n\ -OO : remove doc-strings in addition to the -O optimizations\n\ -q : don't print version and copyright messages on interactive startup\n\ @@ -354,6 +355,7 @@ typedef struct { wchar_t *module; /* -m argument */ PyObject *warning_options; /* -W options */ PyObject *extra_options; /* -X options */ + PyObject *optimizations; /* -N optimization flags */ int print_help; /* -h, -? options */ int print_version; /* -V option */ int bytes_warning; /* Py_BytesWarningFlag */ @@ -372,7 +374,7 @@ typedef struct { } _Py_CommandLineDetails; #define _Py_CommandLineDetails_INIT \ - {NULL, NULL, NULL, NULL, NULL, \ + {NULL, NULL, NULL, NULL, NULL, NULL, \ 0, 0, 0, 0, 0, 0, 0, 0, \ 0, 0, 0, 0, 0, 0, 0} @@ -380,6 +382,7 @@ static int read_command_line(int argc, wchar_t **argv, _Py_CommandLineDetails *cmdline) { PyObject *warning_option = NULL; + PyObject *optimization = NULL; wchar_t *command = NULL; wchar_t *module = NULL; int c; @@ -435,6 +438,14 @@ read_command_line(int argc, wchar_t **argv, _Py_CommandLineDetails *cmdline) /* case 'J': reserved for Jython */ + case 'N': + if (cmdline->optimizations == NULL) + cmdline->optimizations = PySet_New(NULL); + optimization = PyUnicode_FromWideChar(_PyOS_optarg, -1); + PySet_Add(cmdline->optimizations, optimization); + Py_DECREF(optimization); + break; + case 'O': cmdline->optimization_level++; break; @@ -515,6 +526,46 @@ read_command_line(int argc, wchar_t **argv, _Py_CommandLineDetails *cmdline) } } + /* + * Map legacy optimization_level int value to optimization flags. + * 0: {}, no optimization flags + * 1: {"nodebug", "noassert"} + * 2: {"nodebug", "noassert", "nodocstring"} + */ + if (cmdline->optimization_level >= 0) { + if (cmdline->optimizations == NULL) { + cmdline->optimizations = PySet_New(NULL); + } + + if (cmdline->optimization_level == 1) { + optimization = PyUnicode_FromString("nodebug"); + PySet_Add(cmdline->optimizations, optimization); + Py_DECREF(optimization); + + optimization = PyUnicode_FromString("noassert"); + PySet_Add(cmdline->optimizations, optimization); + Py_DECREF(optimization); + } + + if (cmdline->optimization_level == 2) { + optimization = PyUnicode_FromString("nodebug"); + PySet_Add(cmdline->optimizations, optimization); + Py_DECREF(optimization); + + optimization = PyUnicode_FromString("noassert"); + PySet_Add(cmdline->optimizations, optimization); + Py_DECREF(optimization); + + optimization = PyUnicode_FromString("nodocstring"); + PySet_Add(cmdline->optimizations, optimization); + Py_DECREF(optimization); + } + } + + if (cmdline->optimizations != NULL) { + PySys_SetOptimizations(cmdline->optimizations); + } + if (command == NULL && module == NULL && _PyOS_optind < argc && wcscmp(argv[_PyOS_optind], L"-") != 0) { diff --git a/Modules/parsermodule.c b/Modules/parsermodule.c index 929f2deb16..91e0286a6b 100644 --- a/Modules/parsermodule.c +++ b/Modules/parsermodule.c @@ -515,7 +515,7 @@ parser_compilest(PyST_Object *self, PyObject *args, PyObject *kw) goto error; res = (PyObject *)PyAST_CompileObject(mod, filename, - &self->st_flags, -1, arena); + &self->st_flags, NULL, arena); error: Py_XDECREF(filename); if (arena != NULL) diff --git a/Modules/zipimport.c b/Modules/zipimport.c index fad1b1f5ab..302a8b8bd3 100644 --- a/Modules/zipimport.c +++ b/Modules/zipimport.c @@ -1365,7 +1365,7 @@ compile_source(PyObject *pathname, PyObject *source) } code = Py_CompileStringObject(PyBytes_AsString(fixed_source), - pathname, Py_file_input, NULL, -1); + pathname, Py_file_input, NULL, NULL); Py_DECREF(fixed_source); return code; diff --git a/Programs/_freeze_importlib.c b/Programs/_freeze_importlib.c index 1069966a18..7b9be6cfd1 100644 --- a/Programs/_freeze_importlib.c +++ b/Programs/_freeze_importlib.c @@ -90,7 +90,12 @@ main(int argc, char *argv[]) code_name = is_bootstrap ? "" : ""; - code = Py_CompileStringExFlags(text, code_name, Py_file_input, NULL, 0); + + PyObject *optimizations = PySet_New(NULL); // empty: no optimizations + code = Py_CompileStringExFlags(text, code_name, Py_file_input, NULL, + optimizations); + Py_DECREF(optimizations); + if (code == NULL) goto error; free(text); diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 85f207b68e..542ae337e2 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -683,7 +683,7 @@ in addition to any features explicitly specified. static PyObject * builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, const char *mode, int flags, int dont_inherit, - int optimize) + PyObject *optimizations) /*[clinic end generated code: output=1fa176e33452bb63 input=0ff726f595eb9fcd]*/ { PyObject *source_copy; @@ -705,11 +705,11 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, } /* XXX Warn if (supplied_flags & PyCF_MASK_OBSOLETE) != 0? */ - if (optimize < -1 || optimize > 2) { - PyErr_SetString(PyExc_ValueError, - "compile(): invalid optimize value"); - goto error; - } +// if (optimize < -1 || optimize > 2) { +// PyErr_SetString(PyExc_ValueError, +// "compile(): invalid optimize value"); +// goto error; +// } if (!dont_inherit) { PyEval_MergeCompilerFlags(&cf); @@ -751,8 +751,8 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, PyArena_Free(arena); goto error; } - result = (PyObject*)PyAST_CompileObject(mod, filename, - &cf, optimize, arena); + result = (PyObject*)PyAST_CompileObject(mod, filename, &cf, + optimizations, arena); PyArena_Free(arena); } goto finally; @@ -762,7 +762,8 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, if (str == NULL) goto error; - result = Py_CompileStringObject(str, filename, start[compile_mode], &cf, optimize); + result = Py_CompileStringObject(str, filename, start[compile_mode], &cf, + optimizations); Py_XDECREF(source_copy); goto finally; @@ -2737,7 +2738,7 @@ _PyBuiltin_Init(void) SETBUILTIN("tuple", &PyTuple_Type); SETBUILTIN("type", &PyType_Type); SETBUILTIN("zip", &PyZip_Type); - debug = PyBool_FromLong(Py_OptimizeFlag == 0); + debug = PyBool_FromLong(Py_OptimizeFlag == 0); // TODO if (PyDict_SetItemString(dict, "__debug__", debug) < 0) { Py_DECREF(debug); return NULL; diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index fa327da0ff..35accc0701 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -133,7 +133,7 @@ exit: PyDoc_STRVAR(builtin_compile__doc__, "compile($module, /, source, filename, mode, flags=0,\n" -" dont_inherit=False, optimize=-1)\n" +" dont_inherit=False, optimizations=None, optimize=-1)\n" "--\n" "\n" "Compile source into a code object that can be executed by exec() or eval().\n" @@ -155,26 +155,61 @@ PyDoc_STRVAR(builtin_compile__doc__, static PyObject * builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, const char *mode, int flags, int dont_inherit, - int optimize); + PyObject *optimizations); static PyObject * builtin_compile(PyObject *module, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; - static const char * const _keywords[] = {"source", "filename", "mode", "flags", "dont_inherit", "optimize", NULL}; - static _PyArg_Parser _parser = {"OO&s|iii:compile", _keywords, 0}; + static const char * const _keywords[] = { + "source", + "filename", + "mode", + "flags", + "dont_inherit", + "optimizations", + "optimize", + NULL + }; + static _PyArg_Parser _parser = {"OO&s|iiOi:compile", _keywords, 0}; PyObject *source; PyObject *filename; const char *mode; int flags = 0; int dont_inherit = 0; + PyObject *optimizations = NULL; int optimize = -1; if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, - &source, PyUnicode_FSDecoder, &filename, &mode, &flags, &dont_inherit, &optimize)) { - goto exit; + &source, PyUnicode_FSDecoder, &filename, &mode, &flags, &dont_inherit, + &optimizations, &optimize)) { + goto exit; + } + + /* + * Map legacy optimize int value to optimization flags. + * - 1: NULL, use optimization level of interpreter + * 0: {}, no optimization flags + * 1: {"nodebug", "noassert"} + * 2: {"nodebug", "noassert", "nodocstring"} + */ + if (optimize >= 0) { + if (optimizations == NULL) { + optimizations = PySet_New(NULL); + } + + if (optimize == 1) { + PySet_Add(optimizations, PyUnicode_FromString("nodebug")); + PySet_Add(optimizations, PyUnicode_FromString("noassert")); + } else if (optimize == 2) { + PySet_Add(optimizations, PyUnicode_FromString("nodebug")); + PySet_Add(optimizations, PyUnicode_FromString("noassert")); + PySet_Add(optimizations, PyUnicode_FromString("nodocstring")); + } } - return_value = builtin_compile_impl(module, source, filename, mode, flags, dont_inherit, optimize); + + return_value = builtin_compile_impl(module, source, filename, mode, flags, + dont_inherit, optimizations); exit: return return_value; diff --git a/Python/compile.c b/Python/compile.c index 280ddc39e3..a062f3e0ee 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -154,7 +154,7 @@ struct compiler { PyFutureFeatures *c_future; /* pointer to module's __future__ */ PyCompilerFlags *c_flags; - int c_optimize; /* optimization level */ + PyObject *c_optimizations; /* optimization flags */ int c_interactive; /* true if in interactive mode */ int c_nestlevel; @@ -285,6 +285,12 @@ _Py_Mangle(PyObject *privateobj, PyObject *ident) return result; } +static int +optimization_flag_absent(struct compiler *c, const char *flag) +{ + return PySet_Contains(c->c_optimizations, PyUnicode_FromString(flag)) == 0; +} + static int compiler_init(struct compiler *c) { @@ -299,7 +305,7 @@ compiler_init(struct compiler *c) PyCodeObject * PyAST_CompileObject(mod_ty mod, PyObject *filename, PyCompilerFlags *flags, - int optimize, PyArena *arena) + PyObject *optimizations, PyArena *arena) { struct compiler c; PyCodeObject *co = NULL; @@ -328,9 +334,14 @@ PyAST_CompileObject(mod_ty mod, PyObject *filename, PyCompilerFlags *flags, c.c_future->ff_features = merged; flags->cf_flags = merged; c.c_flags = flags; - c.c_optimize = (optimize == -1) ? Py_OptimizeFlag : optimize; c.c_nestlevel = 0; + if (optimizations == NULL) { + c.c_optimizations = PySys_GetOptimizations(); + } else { + c.c_optimizations = optimizations; + } + c.c_st = PySymtable_BuildObject(mod, filename, c.c_future); if (c.c_st == NULL) { if (!PyErr_Occurred()) @@ -348,14 +359,14 @@ PyAST_CompileObject(mod_ty mod, PyObject *filename, PyCompilerFlags *flags, PyCodeObject * PyAST_CompileEx(mod_ty mod, const char *filename_str, PyCompilerFlags *flags, - int optimize, PyArena *arena) + PyObject *optimizations, PyArena *arena) { PyObject *filename; PyCodeObject *co; filename = PyUnicode_DecodeFSDefault(filename_str); if (filename == NULL) return NULL; - co = PyAST_CompileObject(mod, filename, flags, optimize, arena); + co = PyAST_CompileObject(mod, filename, flags, optimizations, arena); Py_DECREF(filename); return co; @@ -1432,7 +1443,8 @@ compiler_body(struct compiler *c, asdl_seq *stmts, string docstring) ADDOP(c, SETUP_ANNOTATIONS); } /* if not -OO mode, set docstring */ - if (c->c_optimize < 2 && docstring) { + int include_doc = optimization_flag_absent(c, "nodocstring"); + if (include_doc && docstring) { ADDOP_O(c, LOAD_CONST, docstring, consts); ADDOP_NAME(c, STORE_NAME, __doc__, names); } @@ -1837,7 +1849,8 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async) } /* if not -OO mode, add docstring */ - if (c->c_optimize < 2 && s->v.FunctionDef.docstring) + int include_doc = optimization_flag_absent(c, "nodocstring"); + if (include_doc && s->v.FunctionDef.docstring) docstring = s->v.FunctionDef.docstring; if (compiler_add_o(c, c->u->u_consts, docstring) < 0) { compiler_exit_scope(c); @@ -2825,8 +2838,10 @@ compiler_assert(struct compiler *c, stmt_ty s) basicblock *end; PyObject* msg; - if (c->c_optimize) + int include_assert = optimization_flag_absent(c, "noassert"); + if (!include_assert) { return 1; + } if (assertion_error == NULL) { assertion_error = PyUnicode_InternFromString("AssertionError"); if (assertion_error == NULL) @@ -4142,8 +4157,10 @@ expr_constant(struct compiler *c, expr_ty e) case Name_kind: /* optimize away names that can't be reassigned */ id = PyUnicode_AsUTF8(e->v.Name.id); - if (id && strcmp(id, "__debug__") == 0) - return !c->c_optimize; + if (id && strcmp(id, "__debug__") == 0) { + int include_debug = optimization_flag_absent(c, "nodebug"); + return include_debug ? 1 : 0; + } return -1; case NameConstant_kind: { PyObject *o = e->v.NameConstant.value; @@ -5453,5 +5470,5 @@ PyAPI_FUNC(PyCodeObject *) PyAST_Compile(mod_ty mod, const char *filename, PyCompilerFlags *flags, PyArena *arena) { - return PyAST_CompileEx(mod, filename, flags, -1, arena); + return PyAST_CompileEx(mod, filename, flags, NULL, arena); } diff --git a/Python/pythonrun.c b/Python/pythonrun.c index f31b3ee5a5..0225a9c382 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -976,7 +976,7 @@ run_mod(mod_ty mod, PyObject *filename, PyObject *globals, PyObject *locals, { PyCodeObject *co; PyObject *v; - co = PyAST_CompileObject(mod, filename, flags, -1, arena); + co = PyAST_CompileObject(mod, filename, flags, NULL, arena); if (co == NULL) return NULL; v = PyEval_EvalCode((PyObject*)co, globals, locals); @@ -1023,7 +1023,7 @@ run_pyc_file(FILE *fp, const char *filename, PyObject *globals, PyObject * Py_CompileStringObject(const char *str, PyObject *filename, int start, - PyCompilerFlags *flags, int optimize) + PyCompilerFlags *flags, PyObject *optimizations) { PyCodeObject *co; mod_ty mod; @@ -1041,20 +1041,20 @@ Py_CompileStringObject(const char *str, PyObject *filename, int start, PyArena_Free(arena); return result; } - co = PyAST_CompileObject(mod, filename, flags, optimize, arena); + co = PyAST_CompileObject(mod, filename, flags, optimizations, arena); PyArena_Free(arena); return (PyObject *)co; } PyObject * Py_CompileStringExFlags(const char *str, const char *filename_str, int start, - PyCompilerFlags *flags, int optimize) + PyCompilerFlags *flags, PyObject *optimizations) { PyObject *filename, *co; filename = PyUnicode_DecodeFSDefault(filename_str); if (filename == NULL) return NULL; - co = Py_CompileStringObject(str, filename, start, flags, optimize); + co = Py_CompileStringObject(str, filename, start, flags, optimizations); Py_DECREF(filename); return co; } @@ -1523,7 +1523,7 @@ PyRun_SimpleString(const char *s) PyAPI_FUNC(PyObject *) Py_CompileString(const char *str, const char *p, int s) { - return Py_CompileStringExFlags(str, p, s, NULL, -1); + return Py_CompileStringExFlags(str, p, s, NULL, NULL); } #undef Py_CompileStringFlags @@ -1531,7 +1531,7 @@ PyAPI_FUNC(PyObject *) Py_CompileStringFlags(const char *str, const char *p, int s, PyCompilerFlags *flags) { - return Py_CompileStringExFlags(str, p, s, flags, -1); + return Py_CompileStringExFlags(str, p, s, flags, NULL); } #undef PyRun_InteractiveOne diff --git a/Python/sysmodule.c b/Python/sysmodule.c index ab435c8310..2e325ad031 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1481,6 +1481,31 @@ list_builtin_module_names(void) return list; } +static PyObject *optimizations = NULL; + +void +PySys_SetOptimizations(PyObject *options) +{ + if (optimizations == NULL) + optimizations = PySet_New(NULL); + + Py_ssize_t i; + int size = PySet_GET_SIZE(options); + for (i = 0; i < size; i++) { + PySet_Add(optimizations, PySet_Pop(options)); + } +} + +PyObject * +PySys_GetOptimizations(void) +{ + if (optimizations == NULL || !PySet_Check(optimizations)) { + Py_XDECREF(optimizations); + optimizations = PySet_New(NULL); + } + return optimizations; +} + static PyObject *warnoptions = NULL; void