[Python-checkins] cpython (3.5): Issue #16182: Fix readline begidx, endidx, and use locale encoding
martin.panter
python-checkins at python.org
Mon Jun 13 23:03:40 EDT 2016
https://hg.python.org/cpython/rev/5122b3465a38
changeset: 101988:5122b3465a38
branch: 3.5
parent: 101986:0cf4bda2882c
user: Martin Panter <vadmium+py at gmail.com>
date: Tue Jun 14 01:16:16 2016 +0000
summary:
Issue #16182: Fix readline begidx, endidx, and use locale encoding
Based on patch by Serhiy Storchaka.
files:
Lib/test/test_readline.py | 141 +++++++++++++++++++++++++-
Misc/NEWS | 4 +
Modules/readline.c | 132 +++++++++++++++++-------
3 files changed, 235 insertions(+), 42 deletions(-)
diff --git a/Lib/test/test_readline.py b/Lib/test/test_readline.py
--- a/Lib/test/test_readline.py
+++ b/Lib/test/test_readline.py
@@ -1,15 +1,23 @@
"""
Very minimal unittests for parts of the readline module.
"""
+from contextlib import ExitStack
+from errno import EIO
import os
+import selectors
+import subprocess
+import sys
import tempfile
import unittest
-from test.support import import_module, unlink
+from test.support import import_module, unlink, TESTFN
from test.support.script_helper import assert_python_ok
# Skip tests if there is no readline module
readline = import_module('readline')
+ at unittest.skipUnless(hasattr(readline, "clear_history"),
+ "The history update test cannot be run because the "
+ "clear_history method is not available.")
class TestHistoryManipulation (unittest.TestCase):
"""
These tests were added to check that the libedit emulation on OSX and the
@@ -17,9 +25,6 @@
why the tests cover only a small subset of the interface.
"""
- @unittest.skipUnless(hasattr(readline, "clear_history"),
- "The history update test cannot be run because the "
- "clear_history method is not available.")
def testHistoryUpdates(self):
readline.clear_history()
@@ -82,6 +87,21 @@
# write_history_file can create the target
readline.write_history_file(hfilename)
+ def test_nonascii_history(self):
+ readline.clear_history()
+ try:
+ readline.add_history("entrée 1")
+ except UnicodeEncodeError as err:
+ self.skipTest("Locale cannot encode test data: " + format(err))
+ readline.add_history("entrée 2")
+ readline.replace_history_item(1, "entrée 22")
+ readline.write_history_file(TESTFN)
+ self.addCleanup(os.remove, TESTFN)
+ readline.clear_history()
+ readline.read_history_file(TESTFN)
+ self.assertEqual(readline.get_history_item(1), "entrée 1")
+ self.assertEqual(readline.get_history_item(2), "entrée 22")
+
class TestReadline(unittest.TestCase):
@@ -96,6 +116,119 @@
TERM='xterm-256color')
self.assertEqual(stdout, b'')
+ def test_nonascii(self):
+ try:
+ readline.add_history("\xEB\xEF")
+ except UnicodeEncodeError as err:
+ self.skipTest("Locale cannot encode test data: " + format(err))
+
+ script = r"""import readline
+
+if readline.__doc__ and "libedit" in readline.__doc__:
+ readline.parse_and_bind(r'bind ^B ed-prev-char')
+ readline.parse_and_bind(r'bind "\t" rl_complete')
+ readline.parse_and_bind('bind -s ^A "|t\xEB[after]"')
+else:
+ readline.parse_and_bind(r'Control-b: backward-char')
+ readline.parse_and_bind(r'"\t": complete')
+ readline.parse_and_bind(r'set disable-completion off')
+ readline.parse_and_bind(r'set show-all-if-ambiguous off')
+ readline.parse_and_bind(r'set show-all-if-unmodified off')
+ readline.parse_and_bind('Control-a: "|t\xEB[after]"')
+
+def pre_input_hook():
+ readline.insert_text("[\xEFnserted]")
+ readline.redisplay()
+readline.set_pre_input_hook(pre_input_hook)
+
+def completer(text, state):
+ if text == "t\xEB":
+ if state == 0:
+ print("text", ascii(text))
+ print("line", ascii(readline.get_line_buffer()))
+ print("indexes", readline.get_begidx(), readline.get_endidx())
+ return "t\xEBnt"
+ if state == 1:
+ return "t\xEBxt"
+ if text == "t\xEBx" and state == 0:
+ return "t\xEBxt"
+ return None
+readline.set_completer(completer)
+
+def display(substitution, matches, longest_match_length):
+ print("substitution", ascii(substitution))
+ print("matches", ascii(matches))
+readline.set_completion_display_matches_hook(display)
+
+print("result", ascii(input()))
+print("history", ascii(readline.get_history_item(1)))
+"""
+
+ input = b"\x01" # Ctrl-A, expands to "|t\xEB[after]"
+ input += b"\x02" * len("[after]") # Move cursor back
+ input += b"\t\t" # Display possible completions
+ input += b"x\t" # Complete "t\xEBx" -> "t\xEBxt"
+ input += b"\r"
+ output = run_pty(script, input)
+ self.assertIn(b"text 't\\xeb'\r\n", output)
+ self.assertIn(b"line '[\\xefnserted]|t\\xeb[after]'\r\n", output)
+ self.assertIn(b"indexes 11 13\r\n", output)
+ self.assertIn(b"substitution 't\\xeb'\r\n", output)
+ self.assertIn(b"matches ['t\\xebnt', 't\\xebxt']\r\n", output)
+ expected = br"'[\xefnserted]|t\xebxt[after]'"
+ self.assertIn(b"result " + expected + b"\r\n", output)
+ self.assertIn(b"history " + expected + b"\r\n", output)
+
+
+def run_pty(script, input=b"dummy input\r"):
+ pty = import_module('pty')
+ output = bytearray()
+ [master, slave] = pty.openpty()
+ args = (sys.executable, '-c', script)
+ proc = subprocess.Popen(args, stdin=slave, stdout=slave, stderr=slave)
+ os.close(slave)
+ with ExitStack() as cleanup:
+ cleanup.enter_context(proc)
+ def terminate(proc):
+ try:
+ proc.terminate()
+ except ProcessLookupError:
+ # Workaround for Open/Net BSD bug (Issue 16762)
+ pass
+ cleanup.callback(terminate, proc)
+ cleanup.callback(os.close, master)
+ # Avoid using DefaultSelector and PollSelector. Kqueue() does not
+ # work with pseudo-terminals on OS X < 10.9 (Issue 20365) and Open
+ # BSD (Issue 20667). Poll() does not work with OS X 10.6 or 10.4
+ # either (Issue 20472). Hopefully the file descriptor is low enough
+ # to use with select().
+ sel = cleanup.enter_context(selectors.SelectSelector())
+ sel.register(master, selectors.EVENT_READ | selectors.EVENT_WRITE)
+ os.set_blocking(master, False)
+ while True:
+ for [_, events] in sel.select():
+ if events & selectors.EVENT_READ:
+ try:
+ chunk = os.read(master, 0x10000)
+ except OSError as err:
+ # Linux raises EIO when slave is closed (Issue 5380)
+ if err.errno != EIO:
+ raise
+ chunk = b""
+ if not chunk:
+ return output
+ output.extend(chunk)
+ if events & selectors.EVENT_WRITE:
+ try:
+ input = input[os.write(master, input):]
+ except OSError as err:
+ # Apparently EIO means the slave was closed
+ if err.errno != EIO:
+ raise
+ input = b"" # Stop writing
+ if not input:
+ sel.modify(master, selectors.EVENT_READ)
+
if __name__ == "__main__":
unittest.main()
diff --git a/Misc/NEWS b/Misc/NEWS
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -13,6 +13,10 @@
Library
-------
+- Issue #16182: Fix various functions in the "readline" module to use the
+ locale encoding, and fix get_begidx() and get_endidx() to return code point
+ indexes.
+
What's New in Python 3.5.2 final?
=================================
diff --git a/Modules/readline.c b/Modules/readline.c
--- a/Modules/readline.c
+++ b/Modules/readline.c
@@ -128,20 +128,40 @@
#define readlinestate_global ((readlinestate *)PyModule_GetState(PyState_FindModule(&readlinemodule)))
+/* Convert to/from multibyte C strings */
+
+static PyObject *
+encode(PyObject *b)
+{
+ return PyUnicode_EncodeLocale(b, "surrogateescape");
+}
+
+static PyObject *
+decode(const char *s)
+{
+ return PyUnicode_DecodeLocale(s, "surrogateescape");
+}
+
+
/* Exported function to send one line to readline's init file parser */
static PyObject *
-parse_and_bind(PyObject *self, PyObject *args)
+parse_and_bind(PyObject *self, PyObject *string)
{
- char *s, *copy;
- if (!PyArg_ParseTuple(args, "s:parse_and_bind", &s))
+ char *copy;
+ PyObject *encoded = encode(string);
+ if (encoded == NULL) {
return NULL;
+ }
/* Make a copy -- rl_parse_and_bind() modifies its argument */
/* Bernard Herzog */
- copy = PyMem_Malloc(1 + strlen(s));
- if (copy == NULL)
+ copy = PyMem_Malloc(1 + PyBytes_GET_SIZE(encoded));
+ if (copy == NULL) {
+ Py_DECREF(encoded);
return PyErr_NoMemory();
- strcpy(copy, s);
+ }
+ strcpy(copy, PyBytes_AS_STRING(encoded));
+ Py_DECREF(encoded);
rl_parse_and_bind(copy);
PyMem_Free(copy); /* Free the copy */
Py_RETURN_NONE;
@@ -441,17 +461,18 @@
/* Set the tab-completion word-delimiters that readline uses */
static PyObject *
-set_completer_delims(PyObject *self, PyObject *args)
+set_completer_delims(PyObject *self, PyObject *string)
{
char *break_chars;
-
- if (!PyArg_ParseTuple(args, "s:set_completer_delims", &break_chars)) {
+ PyObject *encoded = encode(string);
+ if (encoded == NULL) {
return NULL;
}
/* Keep a reference to the allocated memory in the module state in case
some other module modifies rl_completer_word_break_characters
(see issue #17289). */
- break_chars = strdup(break_chars);
+ break_chars = strdup(PyBytes_AS_STRING(encoded));
+ Py_DECREF(encoded);
if (break_chars) {
free(completer_word_break_characters);
completer_word_break_characters = break_chars;
@@ -531,10 +552,11 @@
py_replace_history(PyObject *self, PyObject *args)
{
int entry_number;
- char *line;
+ PyObject *line;
+ PyObject *encoded;
HIST_ENTRY *old_entry;
- if (!PyArg_ParseTuple(args, "is:replace_history_item", &entry_number,
+ if (!PyArg_ParseTuple(args, "iU:replace_history_item", &entry_number,
&line)) {
return NULL;
}
@@ -543,7 +565,12 @@
"History index cannot be negative");
return NULL;
}
- old_entry = replace_history_entry(entry_number, line, (void *)NULL);
+ encoded = encode(line);
+ if (encoded == NULL) {
+ return NULL;
+ }
+ old_entry = replace_history_entry(entry_number, PyBytes_AS_STRING(encoded), (void *)NULL);
+ Py_DECREF(encoded);
if (!old_entry) {
PyErr_Format(PyExc_ValueError,
"No history item at position %d",
@@ -562,14 +589,14 @@
/* Add a line to the history buffer */
static PyObject *
-py_add_history(PyObject *self, PyObject *args)
+py_add_history(PyObject *self, PyObject *string)
{
- char *line;
-
- if(!PyArg_ParseTuple(args, "s:add_history", &line)) {
+ PyObject *encoded = encode(string);
+ if (encoded == NULL) {
return NULL;
}
- add_history(line);
+ add_history(PyBytes_AS_STRING(encoded));
+ Py_DECREF(encoded);
Py_RETURN_NONE;
}
@@ -583,7 +610,7 @@
static PyObject *
get_completer_delims(PyObject *self, PyObject *noarg)
{
- return PyUnicode_FromString(rl_completer_word_break_characters);
+ return decode(rl_completer_word_break_characters);
}
PyDoc_STRVAR(doc_get_completer_delims,
@@ -673,7 +700,7 @@
}
#endif /* __APPLE__ */
if ((hist_ent = history_get(idx)))
- return PyUnicode_FromString(hist_ent->line);
+ return decode(hist_ent->line);
else {
Py_RETURN_NONE;
}
@@ -702,7 +729,7 @@
static PyObject *
get_line_buffer(PyObject *self, PyObject *noarg)
{
- return PyUnicode_FromString(rl_line_buffer);
+ return decode(rl_line_buffer);
}
PyDoc_STRVAR(doc_get_line_buffer,
@@ -730,12 +757,14 @@
/* Exported function to insert text into the line buffer */
static PyObject *
-insert_text(PyObject *self, PyObject *args)
+insert_text(PyObject *self, PyObject *string)
{
- char *s;
- if (!PyArg_ParseTuple(args, "s:insert_text", &s))
+ PyObject *encoded = encode(string);
+ if (encoded == NULL) {
return NULL;
- rl_insert_text(s);
+ }
+ rl_insert_text(PyBytes_AS_STRING(encoded));
+ Py_DECREF(encoded);
Py_RETURN_NONE;
}
@@ -763,9 +792,9 @@
static struct PyMethodDef readline_methods[] =
{
- {"parse_and_bind", parse_and_bind, METH_VARARGS, doc_parse_and_bind},
+ {"parse_and_bind", parse_and_bind, METH_O, doc_parse_and_bind},
{"get_line_buffer", get_line_buffer, METH_NOARGS, doc_get_line_buffer},
- {"insert_text", insert_text, METH_VARARGS, doc_insert_text},
+ {"insert_text", insert_text, METH_O, doc_insert_text},
{"redisplay", redisplay, METH_NOARGS, doc_redisplay},
{"read_init_file", read_init_file, METH_VARARGS, doc_read_init_file},
{"read_history_file", read_history_file,
@@ -792,8 +821,8 @@
{"get_endidx", get_endidx, METH_NOARGS, doc_get_endidx},
{"set_completer_delims", set_completer_delims,
- METH_VARARGS, doc_set_completer_delims},
- {"add_history", py_add_history, METH_VARARGS, doc_add_history},
+ METH_O, doc_set_completer_delims},
+ {"add_history", py_add_history, METH_O, doc_add_history},
{"remove_history_item", py_remove_history, METH_VARARGS, doc_remove_history},
{"replace_history_item", py_replace_history, METH_VARARGS, doc_replace_history},
{"get_completer_delims", get_completer_delims,
@@ -890,7 +919,7 @@
int num_matches, int max_length)
{
int i;
- PyObject *m=NULL, *s=NULL, *r=NULL;
+ PyObject *sub, *m=NULL, *s=NULL, *r=NULL;
#ifdef WITH_THREAD
PyGILState_STATE gilstate = PyGILState_Ensure();
#endif
@@ -898,16 +927,17 @@
if (m == NULL)
goto error;
for (i = 0; i < num_matches; i++) {
- s = PyUnicode_FromString(matches[i+1]);
+ s = decode(matches[i+1]);
if (s == NULL)
goto error;
if (PyList_SetItem(m, i, s) == -1)
goto error;
}
+ sub = decode(matches[0]);
r = PyObject_CallFunction(readlinestate_global->completion_display_matches_hook,
- "sOi", matches[0], m, max_length);
+ "NNi", sub, m, max_length);
- Py_DECREF(m); m=NULL;
+ m=NULL;
if (r == NULL ||
(r != Py_None && PyLong_AsLong(r) == -1 && PyErr_Occurred())) {
@@ -955,22 +985,24 @@
{
char *result = NULL;
if (readlinestate_global->completer != NULL) {
- PyObject *r;
+ PyObject *r = NULL, *t;
#ifdef WITH_THREAD
PyGILState_STATE gilstate = PyGILState_Ensure();
#endif
rl_attempted_completion_over = 1;
- r = PyObject_CallFunction(readlinestate_global->completer, "si", text, state);
+ t = decode(text);
+ r = PyObject_CallFunction(readlinestate_global->completer, "Ni", t, state);
if (r == NULL)
goto error;
if (r == Py_None) {
result = NULL;
}
else {
- char *s = _PyUnicode_AsString(r);
- if (s == NULL)
+ PyObject *encoded = encode(r);
+ if (encoded == NULL)
goto error;
- result = strdup(s);
+ result = strdup(PyBytes_AS_STRING(encoded));
+ Py_DECREF(encoded);
}
Py_DECREF(r);
goto done;
@@ -994,6 +1026,9 @@
flex_complete(const char *text, int start, int end)
{
char **result;
+ char saved;
+ size_t start_size, end_size;
+ wchar_t *s;
#ifdef WITH_THREAD
PyGILState_STATE gilstate = PyGILState_Ensure();
#endif
@@ -1003,6 +1038,27 @@
#ifdef HAVE_RL_COMPLETION_SUPPRESS_APPEND
rl_completion_suppress_append = 0;
#endif
+
+ saved = rl_line_buffer[start];
+ rl_line_buffer[start] = 0;
+ s = Py_DecodeLocale(rl_line_buffer, &start_size);
+ rl_line_buffer[start] = saved;
+ if (s == NULL) {
+ goto done;
+ }
+ PyMem_RawFree(s);
+ saved = rl_line_buffer[end];
+ rl_line_buffer[end] = 0;
+ s = Py_DecodeLocale(rl_line_buffer + start, &end_size);
+ rl_line_buffer[end] = saved;
+ if (s == NULL) {
+ goto done;
+ }
+ PyMem_RawFree(s);
+ start = (int)start_size;
+ end = start + (int)end_size;
+
+done:
Py_XDECREF(readlinestate_global->begidx);
Py_XDECREF(readlinestate_global->endidx);
readlinestate_global->begidx = PyLong_FromLong((long) start);
--
Repository URL: https://hg.python.org/cpython
More information about the Python-checkins
mailing list