[Python-checkins] bpo-45434: Mark the PyTokenizer C API as private (GH-28924)
vstinner
webhook-mailer at python.org
Wed Oct 13 11:22:19 EDT 2021
https://github.com/python/cpython/commit/713bb19356bce9b8f2b95461834fe1dae505f889
commit: 713bb19356bce9b8f2b95461834fe1dae505f889
branch: main
author: Victor Stinner <vstinner at python.org>
committer: vstinner <vstinner at python.org>
date: 2021-10-13T17:22:14+02:00
summary:
bpo-45434: Mark the PyTokenizer C API as private (GH-28924)
Rename PyTokenize functions to mark them as private:
* PyTokenizer_FindEncodingFilename() => _PyTokenizer_FindEncodingFilename()
* PyTokenizer_FromString() => _PyTokenizer_FromString()
* PyTokenizer_FromFile() => _PyTokenizer_FromFile()
* PyTokenizer_FromUTF8() => _PyTokenizer_FromUTF8()
* PyTokenizer_Free() => _PyTokenizer_Free()
* PyTokenizer_Get() => _PyTokenizer_Get()
Remove the unused PyTokenizer_FindEncoding() function.
import.c: remove unused #include "errcode.h".
files:
M Parser/pegen.c
M Parser/string_parser.c
M Parser/tokenizer.c
M Parser/tokenizer.h
M Python/Python-tokenize.c
M Python/import.c
M Python/traceback.c
diff --git a/Parser/pegen.c b/Parser/pegen.c
index a9896356e5b58..e4d2692539264 100644
--- a/Parser/pegen.c
+++ b/Parser/pegen.c
@@ -729,7 +729,7 @@ _PyPegen_fill_token(Parser *p)
{
const char *start;
const char *end;
- int type = PyTokenizer_Get(p->tok, &start, &end);
+ int type = _PyTokenizer_Get(p->tok, &start, &end);
// Record and skip '# type: ignore' comments
while (type == TYPE_IGNORE) {
@@ -746,7 +746,7 @@ _PyPegen_fill_token(Parser *p)
PyErr_NoMemory();
return -1;
}
- type = PyTokenizer_Get(p->tok, &start, &end);
+ type = _PyTokenizer_Get(p->tok, &start, &end);
}
// If we have reached the end and we are in single input mode we need to insert a newline and reset the parsing
@@ -1306,7 +1306,7 @@ _PyPegen_check_tokenizer_errors(Parser *p) {
for (;;) {
const char *start;
const char *end;
- switch (PyTokenizer_Get(p->tok, &start, &end)) {
+ switch (_PyTokenizer_Get(p->tok, &start, &end)) {
case ERRORTOKEN:
if (p->tok->level != 0) {
int error_lineno = p->tok->parenlinenostack[p->tok->level-1];
@@ -1411,7 +1411,7 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena
const char *enc, const char *ps1, const char *ps2,
PyCompilerFlags *flags, int *errcode, PyArena *arena)
{
- struct tok_state *tok = PyTokenizer_FromFile(fp, enc, ps1, ps2);
+ struct tok_state *tok = _PyTokenizer_FromFile(fp, enc, ps1, ps2);
if (tok == NULL) {
if (PyErr_Occurred()) {
raise_tokenizer_init_error(filename_ob);
@@ -1441,7 +1441,7 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena
_PyPegen_Parser_Free(p);
error:
- PyTokenizer_Free(tok);
+ _PyTokenizer_Free(tok);
return result;
}
@@ -1453,9 +1453,9 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen
struct tok_state *tok;
if (flags == NULL || flags->cf_flags & PyCF_IGNORE_COOKIE) {
- tok = PyTokenizer_FromUTF8(str, exec_input);
+ tok = _PyTokenizer_FromUTF8(str, exec_input);
} else {
- tok = PyTokenizer_FromString(str, exec_input);
+ tok = _PyTokenizer_FromString(str, exec_input);
}
if (tok == NULL) {
if (PyErr_Occurred()) {
@@ -1483,7 +1483,7 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen
_PyPegen_Parser_Free(p);
error:
- PyTokenizer_Free(tok);
+ _PyTokenizer_Free(tok);
return result;
}
diff --git a/Parser/string_parser.c b/Parser/string_parser.c
index 2880d07e5f6fe..cffe24e992e44 100644
--- a/Parser/string_parser.c
+++ b/Parser/string_parser.c
@@ -386,7 +386,7 @@ fstring_compile_expr(Parser *p, const char *expr_start, const char *expr_end,
str[0] = '(';
str[len+1] = ')';
- struct tok_state* tok = PyTokenizer_FromString(str, 1);
+ struct tok_state* tok = _PyTokenizer_FromString(str, 1);
if (tok == NULL) {
PyMem_Free(str);
return NULL;
@@ -409,7 +409,7 @@ fstring_compile_expr(Parser *p, const char *expr_start, const char *expr_end,
exit:
PyMem_Free(str);
_PyPegen_Parser_Free(p2);
- PyTokenizer_Free(tok);
+ _PyTokenizer_Free(tok);
return result;
}
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index c7a014de42d8f..ae3874b09de48 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -108,7 +108,7 @@ static char *
error_ret(struct tok_state *tok) /* XXX */
{
tok->decoding_erred = 1;
- if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */
+ if (tok->fp != NULL && tok->buf != NULL) /* see _PyTokenizer_Free */
PyMem_Free(tok->buf);
tok->buf = tok->cur = tok->inp = NULL;
tok->start = NULL;
@@ -702,7 +702,7 @@ decode_str(const char *input, int single, struct tok_state *tok)
/* Set up tokenizer for string */
struct tok_state *
-PyTokenizer_FromString(const char *str, int exec_input)
+_PyTokenizer_FromString(const char *str, int exec_input)
{
struct tok_state *tok = tok_new();
char *decoded;
@@ -711,7 +711,7 @@ PyTokenizer_FromString(const char *str, int exec_input)
return NULL;
decoded = decode_str(str, exec_input, tok);
if (decoded == NULL) {
- PyTokenizer_Free(tok);
+ _PyTokenizer_Free(tok);
return NULL;
}
@@ -723,7 +723,7 @@ PyTokenizer_FromString(const char *str, int exec_input)
/* Set up tokenizer for UTF-8 string */
struct tok_state *
-PyTokenizer_FromUTF8(const char *str, int exec_input)
+_PyTokenizer_FromUTF8(const char *str, int exec_input)
{
struct tok_state *tok = tok_new();
char *translated;
@@ -731,7 +731,7 @@ PyTokenizer_FromUTF8(const char *str, int exec_input)
return NULL;
tok->input = translated = translate_newlines(str, exec_input, tok);
if (translated == NULL) {
- PyTokenizer_Free(tok);
+ _PyTokenizer_Free(tok);
return NULL;
}
tok->decoding_state = STATE_NORMAL;
@@ -739,7 +739,7 @@ PyTokenizer_FromUTF8(const char *str, int exec_input)
tok->str = translated;
tok->encoding = new_string("utf-8", 5, tok);
if (!tok->encoding) {
- PyTokenizer_Free(tok);
+ _PyTokenizer_Free(tok);
return NULL;
}
@@ -751,14 +751,14 @@ PyTokenizer_FromUTF8(const char *str, int exec_input)
/* Set up tokenizer for file */
struct tok_state *
-PyTokenizer_FromFile(FILE *fp, const char* enc,
- const char *ps1, const char *ps2)
+_PyTokenizer_FromFile(FILE *fp, const char* enc,
+ const char *ps1, const char *ps2)
{
struct tok_state *tok = tok_new();
if (tok == NULL)
return NULL;
if ((tok->buf = (char *)PyMem_Malloc(BUFSIZ)) == NULL) {
- PyTokenizer_Free(tok);
+ _PyTokenizer_Free(tok);
return NULL;
}
tok->cur = tok->inp = tok->buf;
@@ -771,7 +771,7 @@ PyTokenizer_FromFile(FILE *fp, const char* enc,
gets copied into the parse tree. */
tok->encoding = new_string(enc, strlen(enc), tok);
if (!tok->encoding) {
- PyTokenizer_Free(tok);
+ _PyTokenizer_Free(tok);
return NULL;
}
tok->decoding_state = STATE_NORMAL;
@@ -782,7 +782,7 @@ PyTokenizer_FromFile(FILE *fp, const char* enc,
/* Free a tok_state structure */
void
-PyTokenizer_Free(struct tok_state *tok)
+_PyTokenizer_Free(struct tok_state *tok)
{
if (tok->encoding != NULL) {
PyMem_Free(tok->encoding);
@@ -2049,7 +2049,8 @@ tok_get(struct tok_state *tok, const char **p_start, const char **p_end)
}
int
-PyTokenizer_Get(struct tok_state *tok, const char **p_start, const char **p_end)
+_PyTokenizer_Get(struct tok_state *tok,
+ const char **p_start, const char **p_end)
{
int result = tok_get(tok, p_start, p_end);
if (tok->decoding_erred) {
@@ -2062,7 +2063,7 @@ PyTokenizer_Get(struct tok_state *tok, const char **p_start, const char **p_end)
/* Get the encoding of a Python file. Check for the coding cookie and check if
the file starts with a BOM.
- PyTokenizer_FindEncodingFilename() returns NULL when it can't find the
+ _PyTokenizer_FindEncodingFilename() returns NULL when it can't find the
encoding in the first or second line of the file (in which case the encoding
should be assumed to be UTF-8).
@@ -2070,7 +2071,7 @@ PyTokenizer_Get(struct tok_state *tok, const char **p_start, const char **p_end)
by the caller. */
char *
-PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)
+_PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)
{
struct tok_state *tok;
FILE *fp;
@@ -2087,7 +2088,7 @@ PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)
if (fp == NULL) {
return NULL;
}
- tok = PyTokenizer_FromFile(fp, NULL, NULL, NULL);
+ tok = _PyTokenizer_FromFile(fp, NULL, NULL, NULL);
if (tok == NULL) {
fclose(fp);
return NULL;
@@ -2100,12 +2101,12 @@ PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)
tok->filename = PyUnicode_FromString("<string>");
if (tok->filename == NULL) {
fclose(fp);
- PyTokenizer_Free(tok);
+ _PyTokenizer_Free(tok);
return encoding;
}
}
while (tok->lineno < 2 && tok->done == E_OK) {
- PyTokenizer_Get(tok, &p_start, &p_end);
+ _PyTokenizer_Get(tok, &p_start, &p_end);
}
fclose(fp);
if (tok->encoding) {
@@ -2114,18 +2115,11 @@ PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)
strcpy(encoding, tok->encoding);
}
}
- PyTokenizer_Free(tok);
+ _PyTokenizer_Free(tok);
return encoding;
}
-char *
-PyTokenizer_FindEncoding(int fd)
-{
- return PyTokenizer_FindEncodingFilename(fd, NULL);
-}
-
#ifdef Py_DEBUG
-
void
tok_dump(int type, char *start, char *end)
{
@@ -2133,5 +2127,4 @@ tok_dump(int type, char *start, char *end)
if (type == NAME || type == NUMBER || type == STRING || type == OP)
printf("(%.*s)", (int)(end - start), start);
}
-
-#endif
+#endif // Py_DEBUG
diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h
index 677f9dba490be..1d1cfd639d9d5 100644
--- a/Parser/tokenizer.h
+++ b/Parser/tokenizer.h
@@ -86,12 +86,12 @@ struct tok_state {
enum interactive_underflow_t interactive_underflow;
};
-extern struct tok_state *PyTokenizer_FromString(const char *, int);
-extern struct tok_state *PyTokenizer_FromUTF8(const char *, int);
-extern struct tok_state *PyTokenizer_FromFile(FILE *, const char*,
+extern struct tok_state *_PyTokenizer_FromString(const char *, int);
+extern struct tok_state *_PyTokenizer_FromUTF8(const char *, int);
+extern struct tok_state *_PyTokenizer_FromFile(FILE *, const char*,
const char *, const char *);
-extern void PyTokenizer_Free(struct tok_state *);
-extern int PyTokenizer_Get(struct tok_state *, const char **, const char **);
+extern void _PyTokenizer_Free(struct tok_state *);
+extern int _PyTokenizer_Get(struct tok_state *, const char **, const char **);
#define tok_dump _Py_tok_dump
diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c
index fa713282558b0..d3ebbe1331a15 100644
--- a/Python/Python-tokenize.c
+++ b/Python/Python-tokenize.c
@@ -47,7 +47,7 @@ tokenizeriter_new_impl(PyTypeObject *type, const char *source)
if (filename == NULL) {
return NULL;
}
- self->tok = PyTokenizer_FromUTF8(source, 1);
+ self->tok = _PyTokenizer_FromUTF8(source, 1);
if (self->tok == NULL) {
Py_DECREF(filename);
return NULL;
@@ -61,7 +61,7 @@ tokenizeriter_next(tokenizeriterobject *it)
{
const char *start;
const char *end;
- int type = PyTokenizer_Get(it->tok, &start, &end);
+ int type = _PyTokenizer_Get(it->tok, &start, &end);
if (type == ERRORTOKEN && PyErr_Occurred()) {
return NULL;
}
@@ -105,7 +105,7 @@ static void
tokenizeriter_dealloc(tokenizeriterobject *it)
{
PyTypeObject *tp = Py_TYPE(it);
- PyTokenizer_Free(it->tok);
+ _PyTokenizer_Free(it->tok);
tp->tp_free(it);
Py_DECREF(tp);
}
diff --git a/Python/import.c b/Python/import.c
index 731f0f59118d6..4bc1e518bf525 100644
--- a/Python/import.c
+++ b/Python/import.c
@@ -11,7 +11,6 @@
#include "pycore_interp.h" // _PyInterpreterState_ClearModules()
#include "pycore_pystate.h" // _PyInterpreterState_GET()
#include "pycore_sysmodule.h"
-#include "errcode.h"
#include "marshal.h"
#include "code.h"
#include "importdl.h"
diff --git a/Python/traceback.c b/Python/traceback.c
index ffa7c3494eafc..b18cbb91ce8ed 100644
--- a/Python/traceback.c
+++ b/Python/traceback.c
@@ -29,7 +29,7 @@
#define MAX_NTHREADS 100
/* Function from Parser/tokenizer.c */
-extern char * PyTokenizer_FindEncodingFilename(int, PyObject *);
+extern char* _PyTokenizer_FindEncodingFilename(int, PyObject *);
_Py_IDENTIFIER(TextIOWrapper);
_Py_IDENTIFIER(close);
@@ -431,7 +431,7 @@ _Py_DisplaySourceLine(PyObject *f, PyObject *filename, int lineno, int indent, i
Py_DECREF(binary);
return 0;
}
- found_encoding = PyTokenizer_FindEncodingFilename(fd, filename);
+ found_encoding = _PyTokenizer_FindEncodingFilename(fd, filename);
if (found_encoding == NULL)
PyErr_Clear();
encoding = (found_encoding != NULL) ? found_encoding : "utf-8";
More information about the Python-checkins
mailing list