[Python-checkins] r71796 - in python/trunk: Include/pystrtod.h Include/stringobject.h Lib/test/test_format.py Lib/test/test_types.py Misc/NEWS Objects/stringlib/formatter.h Objects/stringlib/localeutil.h Objects/stringlib/stringdefs.h Python/pystrtod.c
eric.smith
python-checkins at python.org
Wed Apr 22 15:29:05 CEST 2009
Author: eric.smith
Date: Wed Apr 22 15:29:05 2009
New Revision: 71796
Log:
Backport of some of the work in r71665 to trunk. This reworks much of
int, long, and float __format__(), and it keeps their implementation
in sync with py3k.
Also added PyOS_double_to_string. This is the "fallback" version
that's also available in trunk, and should be kept in sync with that
code. I'll add an issue to document PyOS_double_to_string in the C
API.
There are many internal cleanups. Externally visible changes include:
- Implement PEP 378, Format Specifier for Thousands Separator, for
floats, ints, and longs.
- Issue #5515: 'n' formatting for ints, longs, and floats handles
leading zero formatting poorly.
- Issue #5772: For float.__format__, don't add a trailing ".0" if
we're using no type code and we have an exponent.
Modified:
python/trunk/Include/pystrtod.h
python/trunk/Include/stringobject.h
python/trunk/Lib/test/test_format.py
python/trunk/Lib/test/test_types.py
python/trunk/Misc/NEWS
python/trunk/Objects/stringlib/formatter.h
python/trunk/Objects/stringlib/localeutil.h
python/trunk/Objects/stringlib/stringdefs.h
python/trunk/Python/pystrtod.c
Modified: python/trunk/Include/pystrtod.h
==============================================================================
--- python/trunk/Include/pystrtod.h (original)
+++ python/trunk/Include/pystrtod.h Wed Apr 22 15:29:05 2009
@@ -10,6 +10,25 @@
PyAPI_FUNC(double) PyOS_ascii_atof(const char *str);
PyAPI_FUNC(char *) PyOS_ascii_formatd(char *buffer, size_t buf_len, const char *format, double d);
+/* The caller is responsible for calling PyMem_Free to free the buffer
+ that's is returned. */
+PyAPI_FUNC(char *) PyOS_double_to_string(double val,
+ char format_code,
+ int precision,
+ int flags,
+ int *type);
+
+
+/* PyOS_double_to_string's "flags" parameter can be set to 0 or more of: */
+#define Py_DTSF_SIGN 0x01 /* always add the sign */
+#define Py_DTSF_ADD_DOT_0 0x02 /* if the result is an integer add ".0" */
+#define Py_DTSF_ALT 0x04 /* "alternate" formatting. it's format_code
+ specific */
+
+/* PyOS_double_to_string's "type", if non-NULL, will be set to one of: */
+#define Py_DTST_FINITE 0
+#define Py_DTST_INFINITE 1
+#define Py_DTST_NAN 2
#ifdef __cplusplus
}
Modified: python/trunk/Include/stringobject.h
==============================================================================
--- python/trunk/Include/stringobject.h (original)
+++ python/trunk/Include/stringobject.h Wed Apr 22 15:29:05 2009
@@ -177,16 +177,26 @@
strings) */
);
+
/* Using the current locale, insert the thousands grouping
into the string pointed to by buffer. For the argument descriptions,
see Objects/stringlib/localeutil.h */
+PyAPI_FUNC(Py_ssize_t) _PyString_InsertThousandsGroupingLocale(char *buffer,
+ Py_ssize_t n_buffer,
+ char *digits,
+ Py_ssize_t n_digits,
+ Py_ssize_t min_width);
-PyAPI_FUNC(int) _PyString_InsertThousandsGrouping(char *buffer,
- Py_ssize_t n_buffer,
- Py_ssize_t n_digits,
- Py_ssize_t buf_size,
- Py_ssize_t *count,
- int append_zero_char);
+/* Using explicit passed-in values, insert the thousands grouping
+ into the string pointed to by buffer. For the argument descriptions,
+ see Objects/stringlib/localeutil.h */
+PyAPI_FUNC(Py_ssize_t) _PyString_InsertThousandsGrouping(char *buffer,
+ Py_ssize_t n_buffer,
+ char *digits,
+ Py_ssize_t n_digits,
+ Py_ssize_t min_width,
+ const char *grouping,
+ const char *thousands_sep);
/* Format the object based on the format_spec, as defined in PEP 3101
(Advanced String Formatting). */
Modified: python/trunk/Lib/test/test_format.py
==============================================================================
--- python/trunk/Lib/test/test_format.py (original)
+++ python/trunk/Lib/test/test_format.py Wed Apr 22 15:29:05 2009
@@ -232,6 +232,10 @@
testboth("%o", -042L, "-42")
testboth("%o", float(042), "42")
+ # alternate float formatting
+ testformat('%g', 1.1, '1.1')
+ testformat('%#g', 1.1, '1.10000')
+
# Test exception for unknown format characters
if verbose:
print 'Testing exceptions'
Modified: python/trunk/Lib/test/test_types.py
==============================================================================
--- python/trunk/Lib/test/test_types.py (original)
+++ python/trunk/Lib/test/test_types.py Wed Apr 22 15:29:05 2009
@@ -113,6 +113,9 @@
self.assertEqual(1.5e-101.__format__('e'), '1.500000e-101')
self.assertEqual('%e' % 1.5e-101, '1.500000e-101')
+ self.assertEqual('%g' % 1.0, '1')
+ self.assertEqual('%#g' % 1.0, '1.00000')
+
def test_normal_integers(self):
# Ensure the first 256 integers are shared
a = 256
@@ -412,6 +415,9 @@
self.assertRaises(TypeError, 3 .__format__, None)
self.assertRaises(TypeError, 3 .__format__, 0)
+ # can't have ',' with 'c'
+ self.assertRaises(ValueError, 3 .__format__, ",c")
+
# ensure that only int and float type specifiers work
for format_spec in ([chr(x) for x in range(ord('a'), ord('z')+1)] +
[chr(x) for x in range(ord('A'), ord('Z')+1)]):
@@ -609,11 +615,37 @@
# a totaly empty format specifier means something else.
# So, just use a sign flag
test(1e200, '+g', '+1e+200')
- test(1e200, '+', '+1.0e+200')
+ test(1e200, '+', '+1e+200')
+ test(1.1e200, '+g', '+1.1e+200')
+ test(1.1e200, '+', '+1.1e+200')
+
test(1.1e200, '+g', '+1.1e+200')
test(1.1e200, '+', '+1.1e+200')
- # % formatting
+ # 0 padding
+ test(1234., '010f', '1234.000000')
+ test(1234., '011f', '1234.000000')
+ test(1234., '012f', '01234.000000')
+ test(-1234., '011f', '-1234.000000')
+ test(-1234., '012f', '-1234.000000')
+ test(-1234., '013f', '-01234.000000')
+ test(-1234.12341234, '013f', '-01234.123412')
+ test(-123456.12341234, '011.2f', '-0123456.12')
+
+ # 0 padding with commas
+ test(1234., '011,f', '1,234.000000')
+ test(1234., '012,f', '1,234.000000')
+ test(1234., '013,f', '01,234.000000')
+ test(-1234., '012,f', '-1,234.000000')
+ test(-1234., '013,f', '-1,234.000000')
+ test(-1234., '014,f', '-01,234.000000')
+ test(-12345., '015,f', '-012,345.000000')
+ test(-123456., '016,f', '-0,123,456.000000')
+ test(-123456., '017,f', '-0,123,456.000000')
+ test(-123456.12341234, '017,f', '-0,123,456.123412')
+ test(-123456.12341234, '013,.2f', '-0,123,456.12')
+
+ # % formatting
test(-1.0, '%', '-100.000000%')
# format spec must be string
@@ -637,6 +669,24 @@
self.assertRaises(ValueError, format, 0.0, '#')
self.assertRaises(ValueError, format, 0.0, '#20f')
+ def test_format_spec_errors(self):
+ # int, float, and string all share the same format spec
+ # mini-language parser.
+
+ # Check that we can't ask for too many digits. This is
+ # probably a CPython specific test. It tries to put the width
+ # into a C long.
+ self.assertRaises(ValueError, format, 0, '1'*10000 + 'd')
+
+ # Similar with the precision.
+ self.assertRaises(ValueError, format, 0, '.' + '1'*10000 + 'd')
+
+ # And may as well test both.
+ self.assertRaises(ValueError, format, 0, '1'*1000 + '.' + '1'*10000 + 'd')
+
+ # Make sure commas aren't allowed with various type codes
+ for code in 'xXobns':
+ self.assertRaises(ValueError, format, 0, ',' + code)
def test_main():
run_unittest(TypesTests)
Modified: python/trunk/Misc/NEWS
==============================================================================
--- python/trunk/Misc/NEWS (original)
+++ python/trunk/Misc/NEWS Wed Apr 22 15:29:05 2009
@@ -12,6 +12,15 @@
Core and Builtins
-----------------
+- Implement PEP 378, Format Specifier for Thousands Separator, for
+ floats, ints, and longs.
+
+- Issue #5515: 'n' formatting for ints, longs, and floats handles
+ leading zero formatting poorly.
+
+- Issue #5772: For float.__format__, don't add a trailing ".0" if
+ we're using no type code and we have an exponent.
+
- Issue #3166: Make long -> float (and int -> float) conversions
correctly rounded.
Modified: python/trunk/Objects/stringlib/formatter.h
==============================================================================
--- python/trunk/Objects/stringlib/formatter.h (original)
+++ python/trunk/Objects/stringlib/formatter.h Wed Apr 22 15:29:05 2009
@@ -1,6 +1,8 @@
/* implements the string, long, and float formatters. that is,
string.__format__, etc. */
+#include <locale.h>
+
/* Before including this, you must include either:
stringlib/unicodedefs.h
stringlib/stringdefs.h
@@ -13,8 +15,6 @@
be. These are the only non-static functions defined here.
*/
-#define ALLOW_PARENS_FOR_SIGN 0
-
/* Raises an exception about an unknown presentation type for this
* type. */
@@ -104,9 +104,6 @@
{
switch (c) {
case ' ': case '+': case '-':
-#if ALLOW_PARENS_FOR_SIGN
- case '(':
-#endif
return 1;
default:
return 0;
@@ -120,6 +117,7 @@
int alternate;
STRINGLIB_CHAR sign;
Py_ssize_t width;
+ int thousands_separators;
Py_ssize_t precision;
STRINGLIB_CHAR type;
} InternalFormatSpec;
@@ -132,7 +130,7 @@
*/
static int
parse_internal_render_format_spec(STRINGLIB_CHAR *format_spec,
- Py_ssize_t format_spec_len,
+ Py_ssize_t format_spec_len,
InternalFormatSpec *format,
char default_type)
{
@@ -142,13 +140,14 @@
/* end-ptr is used throughout this code to specify the length of
the input string */
- Py_ssize_t specified_width;
+ Py_ssize_t consumed;
format->fill_char = '\0';
format->align = '\0';
format->alternate = 0;
format->sign = '\0';
format->width = -1;
+ format->thousands_separators = 0;
format->precision = -1;
format->type = default_type;
@@ -168,18 +167,13 @@
if (end-ptr >= 1 && is_sign_element(ptr[0])) {
format->sign = ptr[0];
++ptr;
-#if ALLOW_PARENS_FOR_SIGN
- if (end-ptr >= 1 && ptr[0] == ')') {
- ++ptr;
- }
-#endif
}
/* If the next character is #, we're in alternate mode. This only
applies to integers. */
if (end-ptr >= 1 && ptr[0] == '#') {
- format->alternate = 1;
- ++ptr;
+ format->alternate = 1;
+ ++ptr;
}
/* The special case for 0-padding (backwards compat) */
@@ -191,25 +185,35 @@
++ptr;
}
- /* XXX add error checking */
- specified_width = get_integer(&ptr, end, &format->width);
+ consumed = get_integer(&ptr, end, &format->width);
+ if (consumed == -1)
+ /* Overflow error. Exception already set. */
+ return 0;
- /* if specified_width is 0, we didn't consume any characters for
- the width. in that case, reset the width to -1, because
- get_integer() will have set it to zero */
- if (specified_width == 0) {
+ /* If consumed is 0, we didn't consume any characters for the
+ width. In that case, reset the width to -1, because
+ get_integer() will have set it to zero. -1 is how we record
+ that the width wasn't specified. */
+ if (consumed == 0)
format->width = -1;
+
+ /* Comma signifies add thousands separators */
+ if (end-ptr && ptr[0] == ',') {
+ format->thousands_separators = 1;
+ ++ptr;
}
/* Parse field precision */
if (end-ptr && ptr[0] == '.') {
++ptr;
- /* XXX add error checking */
- specified_width = get_integer(&ptr, end, &format->precision);
+ consumed = get_integer(&ptr, end, &format->precision);
+ if (consumed == -1)
+ /* Overflow error. Exception already set. */
+ return 0;
- /* not having a precision after a dot is an error */
- if (specified_width == 0) {
+ /* Not having a precision after a dot is an error. */
+ if (consumed == 0) {
PyErr_Format(PyExc_ValueError,
"Format specifier missing precision");
return 0;
@@ -217,10 +221,10 @@
}
- /* Finally, parse the type field */
+ /* Finally, parse the type field. */
if (end-ptr > 1) {
- /* invalid conversion spec */
+ /* More than one char remain, invalid conversion spec. */
PyErr_Format(PyExc_ValueError, "Invalid conversion specification");
return 0;
}
@@ -230,6 +234,29 @@
++ptr;
}
+ /* Do as much validating as we can, just by looking at the format
+ specifier. Do not take into account what type of formatting
+ we're doing (int, float, string). */
+
+ if (format->thousands_separators) {
+ switch (format->type) {
+ case 'd':
+ case 'e':
+ case 'f':
+ case 'g':
+ case 'E':
+ case 'G':
+ case '%':
+ case 'F':
+ /* These are allowed. See PEP 378.*/
+ break;
+ default:
+ PyErr_Format(PyExc_ValueError,
+ "Cannot specify ',' with '%c'.", format->type);
+ return 0;
+ }
+ }
+
return 1;
}
@@ -238,6 +265,20 @@
/*********** common routines for numeric formatting *********************/
/************************************************************************/
+/* Locale type codes. */
+#define LT_CURRENT_LOCALE 0
+#define LT_DEFAULT_LOCALE 1
+#define LT_NO_LOCALE 2
+
+/* Locale info needed for formatting integers and the part of floats
+ before and including the decimal. Note that locales only support
+ 8-bit chars, not unicode. */
+typedef struct {
+ char *decimal_point;
+ char *thousands_sep;
+ char *grouping;
+} LocaleInfo;
+
/* describes the layout for an integer, see the comment in
calc_number_widths() for details */
typedef struct {
@@ -245,38 +286,84 @@
Py_ssize_t n_prefix;
Py_ssize_t n_spadding;
Py_ssize_t n_rpadding;
- char lsign;
- Py_ssize_t n_lsign;
- char rsign;
- Py_ssize_t n_rsign;
- Py_ssize_t n_total; /* just a convenience, it's derivable from the
- other fields */
+ char sign;
+ Py_ssize_t n_sign; /* number of digits needed for sign (0/1) */
+ Py_ssize_t n_grouped_digits; /* Space taken up by the digits, including
+ any grouping chars. */
+ Py_ssize_t n_decimal; /* 0 if only an integer */
+ Py_ssize_t n_remainder; /* Digits in decimal and/or exponent part,
+ excluding the decimal itself, if
+ present. */
+
+ /* These 2 are not the widths of fields, but are needed by
+ STRINGLIB_GROUPING. */
+ Py_ssize_t n_digits; /* The number of digits before a decimal
+ or exponent. */
+ Py_ssize_t n_min_width; /* The min_width we used when we computed
+ the n_grouped_digits width. */
} NumberFieldWidths;
+/* Given a number of the form:
+ digits[remainder]
+ where ptr points to the start and end points to the end, find where
+ the integer part ends. This could be a decimal, an exponent, both,
+ or neither.
+ If a decimal point is present, set *has_decimal and increment
+ remainder beyond it.
+ Results are undefined (but shouldn't crash) for improperly
+ formatted strings.
+*/
+static void
+parse_number(STRINGLIB_CHAR *ptr, Py_ssize_t len,
+ Py_ssize_t *n_remainder, int *has_decimal)
+{
+ STRINGLIB_CHAR *end = ptr + len;
+ STRINGLIB_CHAR *remainder;
+
+ while (ptr<end && isdigit(*ptr))
+ ++ptr;
+ remainder = ptr;
+
+ /* Does remainder start with a decimal point? */
+ *has_decimal = ptr<end && *remainder == '.';
+
+ /* Skip the decimal point. */
+ if (*has_decimal)
+ remainder++;
+
+ *n_remainder = end - remainder;
+}
+
/* not all fields of format are used. for example, precision is
unused. should this take discrete params in order to be more clear
about what it does? or is passing a single format parameter easier
and more efficient enough to justify a little obfuscation? */
-static void
-calc_number_widths(NumberFieldWidths *spec, STRINGLIB_CHAR actual_sign,
- Py_ssize_t n_prefix, Py_ssize_t n_digits,
- const InternalFormatSpec *format)
+static Py_ssize_t
+calc_number_widths(NumberFieldWidths *spec, Py_ssize_t n_prefix,
+ STRINGLIB_CHAR sign_char, STRINGLIB_CHAR *number,
+ Py_ssize_t n_number, Py_ssize_t n_remainder,
+ int has_decimal, const LocaleInfo *locale,
+ const InternalFormatSpec *format)
{
+ Py_ssize_t n_non_digit_non_padding;
+ Py_ssize_t n_padding;
+
+ spec->n_digits = n_number - n_remainder - (has_decimal?1:0);
spec->n_lpadding = 0;
- spec->n_prefix = 0;
+ spec->n_prefix = n_prefix;
+ spec->n_decimal = has_decimal ? strlen(locale->decimal_point) : 0;
+ spec->n_remainder = n_remainder;
spec->n_spadding = 0;
spec->n_rpadding = 0;
- spec->lsign = '\0';
- spec->n_lsign = 0;
- spec->rsign = '\0';
- spec->n_rsign = 0;
+ spec->sign = '\0';
+ spec->n_sign = 0;
/* the output will look like:
- | |
- | <lpadding> <lsign> <prefix> <spadding> <digits> <rsign> <rpadding> |
- | |
+ | |
+ | <lpadding> <sign> <prefix> <spadding> <grouped_digits> <decimal> <remainder> <rpadding> |
+ | |
- lsign and rsign are computed from format->sign and the actual
+ sign is computed from format->sign and the actual
sign of the number
prefix is given (it's for the '0x' prefix)
@@ -291,108 +378,191 @@
*/
/* compute the various parts we're going to write */
- if (format->sign == '+') {
+ switch (format->sign) {
+ case '+':
/* always put a + or - */
- spec->n_lsign = 1;
- spec->lsign = (actual_sign == '-' ? '-' : '+');
- }
-#if ALLOW_PARENS_FOR_SIGN
- else if (format->sign == '(') {
- if (actual_sign == '-') {
- spec->n_lsign = 1;
- spec->lsign = '(';
- spec->n_rsign = 1;
- spec->rsign = ')';
- }
- }
-#endif
- else if (format->sign == ' ') {
- spec->n_lsign = 1;
- spec->lsign = (actual_sign == '-' ? '-' : ' ');
- }
- else {
- /* non specified, or the default (-) */
- if (actual_sign == '-') {
- spec->n_lsign = 1;
- spec->lsign = '-';
+ spec->n_sign = 1;
+ spec->sign = (sign_char == '-' ? '-' : '+');
+ break;
+ case ' ':
+ spec->n_sign = 1;
+ spec->sign = (sign_char == '-' ? '-' : ' ');
+ break;
+ default:
+ /* Not specified, or the default (-) */
+ if (sign_char == '-') {
+ spec->n_sign = 1;
+ spec->sign = '-';
}
}
- spec->n_prefix = n_prefix;
+ /* The number of chars used for non-digits and non-padding. */
+ n_non_digit_non_padding = spec->n_sign + spec->n_prefix + spec->n_decimal +
+ spec->n_remainder;
+
+ /* min_width can go negative, that's okay. format->width == -1 means
+ we don't care. */
+ if (format->fill_char == '0')
+ spec->n_min_width = format->width - n_non_digit_non_padding;
+ else
+ spec->n_min_width = 0;
- /* now the number of padding characters */
- if (format->width == -1) {
- /* no padding at all, nothing to do */
- }
- else {
- /* see if any padding is needed */
- if (spec->n_lsign + n_digits + spec->n_rsign +
- spec->n_prefix >= format->width) {
- /* no padding needed, we're already bigger than the
- requested width */
- }
- else {
- /* determine which of left, space, or right padding is
- needed */
- Py_ssize_t padding = format->width -
- (spec->n_lsign + spec->n_prefix +
- n_digits + spec->n_rsign);
- if (format->align == '<')
- spec->n_rpadding = padding;
- else if (format->align == '>')
- spec->n_lpadding = padding;
- else if (format->align == '^') {
- spec->n_lpadding = padding / 2;
- spec->n_rpadding = padding - spec->n_lpadding;
- }
- else if (format->align == '=')
- spec->n_spadding = padding;
- else
- spec->n_lpadding = padding;
+ if (spec->n_digits == 0)
+ /* This case only occurs when using 'c' formatting, we need
+ to special case it because the grouping code always wants
+ to have at least one character. */
+ spec->n_grouped_digits = 0;
+ else
+ spec->n_grouped_digits = STRINGLIB_GROUPING(NULL, 0, NULL,
+ spec->n_digits,
+ spec->n_min_width,
+ locale->grouping,
+ locale->thousands_sep);
+
+ /* Given the desired width and the total of digit and non-digit
+ space we consume, see if we need any padding. format->width can
+ be negative (meaning no padding), but this code still works in
+ that case. */
+ n_padding = format->width -
+ (n_non_digit_non_padding + spec->n_grouped_digits);
+ if (n_padding > 0) {
+ /* Some padding is needed. Determine if it's left, space, or right. */
+ switch (format->align) {
+ case '<':
+ spec->n_rpadding = n_padding;
+ break;
+ case '^':
+ spec->n_lpadding = n_padding / 2;
+ spec->n_rpadding = n_padding - spec->n_lpadding;
+ break;
+ case '=':
+ spec->n_spadding = n_padding;
+ break;
+ default:
+ /* Handles '>', plus catch-all just in case. */
+ spec->n_lpadding = n_padding;
+ break;
}
}
- spec->n_total = spec->n_lpadding + spec->n_lsign + spec->n_prefix +
- spec->n_spadding + n_digits + spec->n_rsign + spec->n_rpadding;
+ return spec->n_lpadding + spec->n_sign + spec->n_prefix +
+ spec->n_spadding + spec->n_grouped_digits + spec->n_decimal +
+ spec->n_remainder + spec->n_rpadding;
}
-/* fill in the non-digit parts of a numbers's string representation,
- as determined in calc_number_widths(). returns the pointer to
- where the digits go. */
-static STRINGLIB_CHAR *
-fill_non_digits(STRINGLIB_CHAR *p_buf, const NumberFieldWidths *spec,
- STRINGLIB_CHAR *prefix, Py_ssize_t n_digits,
- STRINGLIB_CHAR fill_char)
+/* Fill in the digit parts of a numbers's string representation,
+ as determined in calc_number_widths().
+ No error checking, since we know the buffer is the correct size. */
+static void
+fill_number(STRINGLIB_CHAR *buf, const NumberFieldWidths *spec,
+ STRINGLIB_CHAR *digits, Py_ssize_t n_digits,
+ STRINGLIB_CHAR *prefix, STRINGLIB_CHAR fill_char,
+ LocaleInfo *locale, int toupper)
{
- STRINGLIB_CHAR *p_digits;
+ /* Used to keep track of digits, decimal, and remainder. */
+ STRINGLIB_CHAR *p = digits;
+
+#ifndef NDEBUG
+ Py_ssize_t r;
+#endif
if (spec->n_lpadding) {
- STRINGLIB_FILL(p_buf, fill_char, spec->n_lpadding);
- p_buf += spec->n_lpadding;
+ STRINGLIB_FILL(buf, fill_char, spec->n_lpadding);
+ buf += spec->n_lpadding;
}
- if (spec->n_lsign == 1) {
- *p_buf++ = spec->lsign;
+ if (spec->n_sign == 1) {
+ *buf++ = spec->sign;
}
if (spec->n_prefix) {
- memmove(p_buf,
- prefix,
- spec->n_prefix * sizeof(STRINGLIB_CHAR));
- p_buf += spec->n_prefix;
+ memmove(buf,
+ prefix,
+ spec->n_prefix * sizeof(STRINGLIB_CHAR));
+ if (toupper) {
+ Py_ssize_t t;
+ for (t = 0; t < spec->n_prefix; ++t)
+ buf[t] = STRINGLIB_TOUPPER(buf[t]);
+ }
+ buf += spec->n_prefix;
}
if (spec->n_spadding) {
- STRINGLIB_FILL(p_buf, fill_char, spec->n_spadding);
- p_buf += spec->n_spadding;
+ STRINGLIB_FILL(buf, fill_char, spec->n_spadding);
+ buf += spec->n_spadding;
}
- p_digits = p_buf;
- p_buf += n_digits;
- if (spec->n_rsign == 1) {
- *p_buf++ = spec->rsign;
+
+ /* Only for type 'c' special case, it has no digits. */
+ if (spec->n_digits != 0) {
+ /* Fill the digits with InsertThousandsGrouping. */
+#ifndef NDEBUG
+ r =
+#endif
+ STRINGLIB_GROUPING(buf, spec->n_grouped_digits, digits,
+ spec->n_digits, spec->n_min_width,
+ locale->grouping, locale->thousands_sep);
+#ifndef NDEBUG
+ assert(r == spec->n_grouped_digits);
+#endif
+ p += spec->n_digits;
+ }
+ if (toupper) {
+ Py_ssize_t t;
+ for (t = 0; t < spec->n_grouped_digits; ++t)
+ buf[t] = STRINGLIB_TOUPPER(buf[t]);
+ }
+ buf += spec->n_grouped_digits;
+
+ if (spec->n_decimal) {
+ Py_ssize_t t;
+ for (t = 0; t < spec->n_decimal; ++t)
+ buf[t] = locale->decimal_point[t];
+ buf += spec->n_decimal;
+ p += 1;
+ }
+
+ if (spec->n_remainder) {
+ memcpy(buf, p, spec->n_remainder * sizeof(STRINGLIB_CHAR));
+ buf += spec->n_remainder;
+ p += spec->n_remainder;
}
+
if (spec->n_rpadding) {
- STRINGLIB_FILL(p_buf, fill_char, spec->n_rpadding);
- p_buf += spec->n_rpadding;
+ STRINGLIB_FILL(buf, fill_char, spec->n_rpadding);
+ buf += spec->n_rpadding;
+ }
+}
+
+static char no_grouping[1] = {CHAR_MAX};
+
+/* Find the decimal point character(s?), thousands_separator(s?), and
+ grouping description, either for the current locale if type is
+ LT_CURRENT_LOCALE, a hard-coded locale if LT_DEFAULT_LOCALE, or
+ none if LT_NO_LOCALE. */
+static void
+get_locale_info(int type, LocaleInfo *locale_info)
+{
+ switch (type) {
+ case LT_CURRENT_LOCALE: {
+ struct lconv *locale_data = localeconv();
+ locale_info->decimal_point = locale_data->decimal_point;
+ locale_info->thousands_sep = locale_data->thousands_sep;
+ locale_info->grouping = locale_data->grouping;
+ break;
+ }
+ case LT_DEFAULT_LOCALE:
+ locale_info->decimal_point = ".";
+ locale_info->thousands_sep = ",";
+ locale_info->grouping = "\3"; /* Group every 3 characters,
+ trailing 0 means repeat
+ infinitely. */
+ break;
+ case LT_NO_LOCALE:
+ locale_info->decimal_point = ".";
+ locale_info->thousands_sep = "";
+ locale_info->grouping = no_grouping;
+ break;
+ default:
+ assert(0);
}
- return p_digits;
}
+
#endif /* FORMAT_FLOAT || FORMAT_LONG */
/************************************************************************/
@@ -420,7 +590,7 @@
if (format->alternate) {
PyErr_SetString(PyExc_ValueError,
"Alternate form (#) not allowed in string format "
- "specifier");
+ "specifier");
goto done;
}
@@ -504,25 +674,27 @@
static PyObject *
format_int_or_long_internal(PyObject *value, const InternalFormatSpec *format,
- IntOrLongToString tostring)
+ IntOrLongToString tostring)
{
PyObject *result = NULL;
PyObject *tmp = NULL;
STRINGLIB_CHAR *pnumeric_chars;
STRINGLIB_CHAR numeric_char;
- STRINGLIB_CHAR sign = '\0';
- STRINGLIB_CHAR *p;
+ STRINGLIB_CHAR sign_char = '\0';
Py_ssize_t n_digits; /* count of digits need from the computed
string */
- Py_ssize_t n_leading_chars;
- Py_ssize_t n_grouping_chars = 0; /* Count of additional chars to
- allocate, used for 'n'
- formatting. */
+ Py_ssize_t n_remainder = 0; /* Used only for 'c' formatting, which
+ produces non-digits */
Py_ssize_t n_prefix = 0; /* Count of prefix chars, (e.g., '0x') */
+ Py_ssize_t n_total;
STRINGLIB_CHAR *prefix = NULL;
NumberFieldWidths spec;
long x;
+ /* Locale settings, either from the actual locale or
+ from a hard-code pseudo-locale */
+ LocaleInfo locale;
+
/* no precision allowed on integers */
if (format->precision != -1) {
PyErr_SetString(PyExc_ValueError,
@@ -530,7 +702,6 @@
goto done;
}
-
/* special case for character formatting */
if (format->type == 'c') {
/* error to specify a sign */
@@ -541,6 +712,14 @@
goto done;
}
+ /* Error to specify a comma. */
+ if (format->thousands_separators) {
+ PyErr_SetString(PyExc_ValueError,
+ "Thousands separators not allowed with integer"
+ " format specifier 'c'");
+ goto done;
+ }
+
/* taken from unicodeobject.c formatchar() */
/* Integer input truncated to a character */
/* XXX: won't work for int */
@@ -562,31 +741,38 @@
goto done;
}
#endif
- numeric_char = (STRINGLIB_CHAR)x;
- pnumeric_chars = &numeric_char;
+ numeric_char = (STRINGLIB_CHAR)x;
+ pnumeric_chars = &numeric_char;
n_digits = 1;
+
+ /* As a sort-of hack, we tell calc_number_widths that we only
+ have "remainder" characters. calc_number_widths thinks
+ these are characters that don't get formatted, only copied
+ into the output string. We do this for 'c' formatting,
+ because the characters are likely to be non-digits. */
+ n_remainder = 1;
}
else {
int base;
- int leading_chars_to_skip = 0; /* Number of characters added by
- PyNumber_ToBase that we want to
- skip over. */
+ int leading_chars_to_skip = 0; /* Number of characters added by
+ PyNumber_ToBase that we want to
+ skip over. */
/* Compute the base and how many characters will be added by
PyNumber_ToBase */
switch (format->type) {
case 'b':
base = 2;
- leading_chars_to_skip = 2; /* 0b */
+ leading_chars_to_skip = 2; /* 0b */
break;
case 'o':
base = 8;
- leading_chars_to_skip = 2; /* 0o */
+ leading_chars_to_skip = 2; /* 0o */
break;
case 'x':
case 'X':
base = 16;
- leading_chars_to_skip = 2; /* 0x */
+ leading_chars_to_skip = 2; /* 0x */
break;
default: /* shouldn't be needed, but stops a compiler warning */
case 'd':
@@ -595,102 +781,58 @@
break;
}
- /* The number of prefix chars is the same as the leading
- chars to skip */
- if (format->alternate)
- n_prefix = leading_chars_to_skip;
+ /* The number of prefix chars is the same as the leading
+ chars to skip */
+ if (format->alternate)
+ n_prefix = leading_chars_to_skip;
/* Do the hard part, converting to a string in a given base */
- tmp = tostring(value, base);
+ tmp = tostring(value, base);
if (tmp == NULL)
goto done;
- pnumeric_chars = STRINGLIB_STR(tmp);
+ pnumeric_chars = STRINGLIB_STR(tmp);
n_digits = STRINGLIB_LEN(tmp);
- prefix = pnumeric_chars;
+ prefix = pnumeric_chars;
- /* Remember not to modify what pnumeric_chars points to. it
- might be interned. Only modify it after we copy it into a
- newly allocated output buffer. */
+ /* Remember not to modify what pnumeric_chars points to. it
+ might be interned. Only modify it after we copy it into a
+ newly allocated output buffer. */
/* Is a sign character present in the output? If so, remember it
and skip it */
- sign = pnumeric_chars[0];
- if (sign == '-') {
- ++prefix;
- ++leading_chars_to_skip;
+ if (pnumeric_chars[0] == '-') {
+ sign_char = pnumeric_chars[0];
+ ++prefix;
+ ++leading_chars_to_skip;
}
- /* Skip over the leading chars (0x, 0b, etc.) */
- n_digits -= leading_chars_to_skip;
- pnumeric_chars += leading_chars_to_skip;
- }
-
- if (format->type == 'n')
- /* Compute how many additional chars we need to allocate
- to hold the thousands grouping. */
- STRINGLIB_GROUPING(NULL, n_digits, n_digits,
- 0, &n_grouping_chars, 0);
-
- /* Calculate the widths of the various leading and trailing parts */
- calc_number_widths(&spec, sign, n_prefix, n_digits + n_grouping_chars,
- format);
+ /* Skip over the leading chars (0x, 0b, etc.) */
+ n_digits -= leading_chars_to_skip;
+ pnumeric_chars += leading_chars_to_skip;
+ }
+
+ /* Determine the grouping, separator, and decimal point, if any. */
+ get_locale_info(format->type == 'n' ? LT_CURRENT_LOCALE :
+ (format->thousands_separators ?
+ LT_DEFAULT_LOCALE :
+ LT_NO_LOCALE),
+ &locale);
+
+ /* Calculate how much memory we'll need. */
+ n_total = calc_number_widths(&spec, n_prefix, sign_char, pnumeric_chars,
+ n_digits, n_remainder, 0, &locale, format);
- /* Allocate a new string to hold the result */
- result = STRINGLIB_NEW(NULL, spec.n_total);
+ /* Allocate the memory. */
+ result = STRINGLIB_NEW(NULL, n_total);
if (!result)
- goto done;
- p = STRINGLIB_STR(result);
-
- /* XXX There is too much magic here regarding the internals of
- spec and the location of the prefix and digits. It would be
- better if calc_number_widths returned a number of logical
- offsets into the buffer, and those were used. Maybe in a
- future code cleanup. */
-
- /* Fill in the digit parts */
- n_leading_chars = spec.n_lpadding + spec.n_lsign +
- spec.n_prefix + spec.n_spadding;
- memmove(p + n_leading_chars,
- pnumeric_chars,
- n_digits * sizeof(STRINGLIB_CHAR));
-
- /* If type is 'X', convert the filled in digits to uppercase */
- if (format->type == 'X') {
- Py_ssize_t t;
- for (t = 0; t < n_digits; ++t)
- p[t + n_leading_chars] = STRINGLIB_TOUPPER(p[t + n_leading_chars]);
- }
-
- /* Insert the grouping, if any, after the uppercasing of the digits, so
- we can ensure that grouping chars won't be affected. */
- if (n_grouping_chars) {
- /* We know this can't fail, since we've already
- reserved enough space. */
- STRINGLIB_CHAR *pstart = p + n_leading_chars;
-#ifndef NDEBUG
- int r =
-#endif
- STRINGLIB_GROUPING(pstart, n_digits, n_digits,
- spec.n_total+n_grouping_chars-n_leading_chars,
- NULL, 0);
- assert(r);
- }
-
- /* Fill in the non-digit parts (padding, sign, etc.) */
- fill_non_digits(p, &spec, prefix, n_digits + n_grouping_chars,
- format->fill_char == '\0' ? ' ' : format->fill_char);
-
- /* If type is 'X', uppercase the prefix. This has to be done after the
- prefix is filled in by fill_non_digits */
- if (format->type == 'X') {
- Py_ssize_t t;
- for (t = 0; t < n_prefix; ++t)
- p[t + spec.n_lpadding + spec.n_lsign] =
- STRINGLIB_TOUPPER(p[t + spec.n_lpadding + spec.n_lsign]);
- }
+ goto done;
+ /* Populate the memory. */
+ fill_number(STRINGLIB_STR(result), &spec, pnumeric_chars, n_digits,
+ prefix, format->fill_char == '\0' ? ' ' : format->fill_char,
+ &locale, format->type == 'X');
done:
Py_XDECREF(tmp);
@@ -704,149 +846,152 @@
#ifdef FORMAT_FLOAT
#if STRINGLIB_IS_UNICODE
-/* taken from unicodeobject.c */
-static Py_ssize_t
-strtounicode(Py_UNICODE *buffer, const char *charbuffer)
+static void
+strtounicode(Py_UNICODE *buffer, const char *charbuffer, Py_ssize_t len)
{
- register Py_ssize_t i;
- Py_ssize_t len = strlen(charbuffer);
- for (i = len - 1; i >= 0; --i)
- buffer[i] = (Py_UNICODE) charbuffer[i];
-
- return len;
+ Py_ssize_t i;
+ for (i = 0; i < len; ++i)
+ buffer[i] = (Py_UNICODE)charbuffer[i];
}
#endif
-/* see FORMATBUFLEN in unicodeobject.c */
-#define FLOAT_FORMATBUFLEN 120
-
/* much of this is taken from unicodeobject.c */
static PyObject *
format_float_internal(PyObject *value,
- const InternalFormatSpec *format)
+ const InternalFormatSpec *format)
{
- /* fmt = '%.' + `prec` + `type` + '%%'
- worst case length = 2 + 10 (len of INT_MAX) + 1 + 2 = 15 (use 20)*/
- char fmt[20];
-
- /* taken from unicodeobject.c */
- /* Worst case length calc to ensure no buffer overrun:
-
- 'g' formats:
- fmt = %#.<prec>g
- buf = '-' + [0-9]*prec + '.' + 'e+' + (longest exp
- for any double rep.)
- len = 1 + prec + 1 + 2 + 5 = 9 + prec
-
- 'f' formats:
- buf = '-' + [0-9]*x + '.' + [0-9]*prec (with x < 50)
- len = 1 + 50 + 1 + prec = 52 + prec
-
- If prec=0 the effective precision is 1 (the leading digit is
- always given), therefore increase the length by one.
-
- */
- char charbuf[FLOAT_FORMATBUFLEN];
+ char *buf = NULL; /* buffer returned from PyOS_double_to_string */
Py_ssize_t n_digits;
- double x;
+ Py_ssize_t n_remainder;
+ Py_ssize_t n_total;
+ int has_decimal;
+ double val;
Py_ssize_t precision = format->precision;
- PyObject *result = NULL;
- STRINGLIB_CHAR sign;
- char* trailing = "";
+ STRINGLIB_CHAR type = format->type;
+ int add_pct = 0;
STRINGLIB_CHAR *p;
NumberFieldWidths spec;
- STRINGLIB_CHAR type = format->type;
+ int flags = 0;
+ PyObject *result = NULL;
+ STRINGLIB_CHAR sign_char = '\0';
+ int float_type; /* Used to see if we have a nan, inf, or regular float. */
#if STRINGLIB_IS_UNICODE
- Py_UNICODE unicodebuf[FLOAT_FORMATBUFLEN];
+ Py_UNICODE *unicode_tmp = NULL;
#endif
- /* alternate is not allowed on floats. */
+ /* Locale settings, either from the actual locale or
+ from a hard-code pseudo-locale */
+ LocaleInfo locale;
+
+ /* Alternate is not allowed on floats. */
if (format->alternate) {
PyErr_SetString(PyExc_ValueError,
"Alternate form (#) not allowed in float format "
- "specifier");
+ "specifier");
goto done;
}
- /* first, do the conversion as 8-bit chars, using the platform's
- snprintf. then, if needed, convert to unicode. */
+ if (type == '\0') {
+ /* Omitted type specifier. This is like 'g' but with at least
+ one digit after the decimal point. */
+ type = 'g';
+ flags |= Py_DTSF_ADD_DOT_0;
+ }
+
+ if (type == 'n')
+ /* 'n' is the same as 'g', except for the locale used to
+ format the result. We take care of that later. */
+ type = 'g';
/* 'F' is the same as 'f', per the PEP */
if (type == 'F')
type = 'f';
- x = PyFloat_AsDouble(value);
-
- if (x == -1.0 && PyErr_Occurred())
+ val = PyFloat_AsDouble(value);
+ if (val == -1.0 && PyErr_Occurred())
goto done;
if (type == '%') {
type = 'f';
- x *= 100;
- trailing = "%";
+ val *= 100;
+ add_pct = 1;
}
if (precision < 0)
precision = 6;
- if (type == 'f' && fabs(x) >= 1e50)
+ if ((type == 'f' || type == 'F') && fabs(val) >= 1e50)
type = 'g';
- /* cast "type", because if we're in unicode we need to pass a
- 8-bit char. this is safe, because we've restricted what "type"
- can be */
- PyOS_snprintf(fmt, sizeof(fmt), "%%.%" PY_FORMAT_SIZE_T "d%c", precision,
- (char)type);
-
- /* do the actual formatting */
- PyOS_ascii_formatd(charbuf, sizeof(charbuf), fmt, x);
-
- /* adding trailing to fmt with PyOS_snprintf doesn't work, not
- sure why. we'll just concatentate it here, no harm done. we
- know we can't have a buffer overflow from the fmt size
- analysis */
- strcat(charbuf, trailing);
-
- /* rather than duplicate the code for snprintf for both unicode
- and 8 bit strings, we just use the 8 bit version and then
- convert to unicode in a separate code path. that's probably
- the lesser of 2 evils. */
+ /* Cast "type", because if we're in unicode we need to pass a
+ 8-bit char. This is safe, because we've restricted what "type"
+ can be. */
+ buf = PyOS_double_to_string(val, (char)type, precision, flags,
+ &float_type);
+ if (buf == NULL)
+ goto done;
+ n_digits = strlen(buf);
+
+ if (add_pct) {
+ /* We know that buf has a trailing zero (since we just called
+ strlen() on it), and we don't use that fact any more. So we
+ can just write over the trailing zero. */
+ buf[n_digits] = '%';
+ n_digits += 1;
+ }
+
+ /* Since there is no unicode version of PyOS_double_to_string,
+ just use the 8 bit version and then convert to unicode. */
#if STRINGLIB_IS_UNICODE
- n_digits = strtounicode(unicodebuf, charbuf);
- p = unicodebuf;
+ unicode_tmp = (Py_UNICODE*)PyMem_Malloc((n_digits)*sizeof(Py_UNICODE));
+ if (unicode_tmp == NULL) {
+ PyErr_NoMemory();
+ goto done;
+ }
+ strtounicode(unicode_tmp, buf, n_digits);
+ p = unicode_tmp;
#else
- /* compute the length. I believe this is done because the return
- value from snprintf above is unreliable */
- n_digits = strlen(charbuf);
- p = charbuf;
+ p = buf;
#endif
- /* is a sign character present in the output? if so, remember it
+ /* Is a sign character present in the output? If so, remember it
and skip it */
- sign = p[0];
- if (sign == '-') {
+ if (*p == '-') {
+ sign_char = *p;
++p;
--n_digits;
}
- calc_number_widths(&spec, sign, 0, n_digits, format);
+ /* Determine if we have any "remainder" (after the digits, might include
+ decimal or exponent or both (or neither)) */
+ parse_number(p, n_digits, &n_remainder, &has_decimal);
+
+ /* Determine the grouping, separator, and decimal point, if any. */
+ get_locale_info(format->type == 'n' ? LT_CURRENT_LOCALE :
+ (format->thousands_separators ?
+ LT_DEFAULT_LOCALE :
+ LT_NO_LOCALE),
+ &locale);
+
+ /* Calculate how much memory we'll need. */
+ n_total = calc_number_widths(&spec, 0, sign_char, p, n_digits,
+ n_remainder, has_decimal, &locale, format);
- /* allocate a string with enough space */
- result = STRINGLIB_NEW(NULL, spec.n_total);
+ /* Allocate the memory. */
+ result = STRINGLIB_NEW(NULL, n_total);
if (result == NULL)
goto done;
- /* Fill in the non-digit parts (padding, sign, etc.) */
- fill_non_digits(STRINGLIB_STR(result), &spec, NULL, n_digits,
- format->fill_char == '\0' ? ' ' : format->fill_char);
-
- /* fill in the digit parts */
- memmove(STRINGLIB_STR(result) +
- (spec.n_lpadding + spec.n_lsign + spec.n_spadding),
- p,
- n_digits * sizeof(STRINGLIB_CHAR));
+ /* Populate the memory. */
+ fill_number(STRINGLIB_STR(result), &spec, p, n_digits, NULL,
+ format->fill_char == '\0' ? ' ' : format->fill_char, &locale,
+ 0);
done:
+ PyMem_Free(buf);
+#if STRINGLIB_IS_UNICODE
+ PyMem_Free(unicode_tmp);
+#endif
return result;
}
#endif /* FORMAT_FLOAT */
@@ -856,8 +1001,8 @@
/************************************************************************/
PyObject *
FORMAT_STRING(PyObject *obj,
- STRINGLIB_CHAR *format_spec,
- Py_ssize_t format_spec_len)
+ STRINGLIB_CHAR *format_spec,
+ Py_ssize_t format_spec_len)
{
InternalFormatSpec format;
PyObject *result = NULL;
@@ -871,7 +1016,7 @@
/* parse the format_spec */
if (!parse_internal_render_format_spec(format_spec, format_spec_len,
- &format, 's'))
+ &format, 's'))
goto done;
/* type conversion? */
@@ -893,9 +1038,9 @@
#if defined FORMAT_LONG || defined FORMAT_INT
static PyObject*
format_int_or_long(PyObject* obj,
- STRINGLIB_CHAR *format_spec,
- Py_ssize_t format_spec_len,
- IntOrLongToString tostring)
+ STRINGLIB_CHAR *format_spec,
+ Py_ssize_t format_spec_len,
+ IntOrLongToString tostring)
{
PyObject *result = NULL;
PyObject *tmp = NULL;
@@ -910,8 +1055,8 @@
/* parse the format_spec */
if (!parse_internal_render_format_spec(format_spec,
- format_spec_len,
- &format, 'd'))
+ format_spec_len,
+ &format, 'd'))
goto done;
/* type conversion? */
@@ -924,8 +1069,8 @@
case 'X':
case 'n':
/* no type conversion needed, already an int (or long). do
- the formatting */
- result = format_int_or_long_internal(obj, &format, tostring);
+ the formatting */
+ result = format_int_or_long_internal(obj, &format, tostring);
break;
case 'e':
@@ -974,11 +1119,11 @@
PyObject *
FORMAT_LONG(PyObject *obj,
- STRINGLIB_CHAR *format_spec,
- Py_ssize_t format_spec_len)
+ STRINGLIB_CHAR *format_spec,
+ Py_ssize_t format_spec_len)
{
return format_int_or_long(obj, format_spec, format_spec_len,
- long_format);
+ long_format);
}
#endif /* FORMAT_LONG */
@@ -995,19 +1140,19 @@
PyObject *
FORMAT_INT(PyObject *obj,
- STRINGLIB_CHAR *format_spec,
- Py_ssize_t format_spec_len)
+ STRINGLIB_CHAR *format_spec,
+ Py_ssize_t format_spec_len)
{
return format_int_or_long(obj, format_spec, format_spec_len,
- int_format);
+ int_format);
}
#endif /* FORMAT_INT */
#ifdef FORMAT_FLOAT
PyObject *
FORMAT_FLOAT(PyObject *obj,
- STRINGLIB_CHAR *format_spec,
- Py_ssize_t format_spec_len)
+ STRINGLIB_CHAR *format_spec,
+ Py_ssize_t format_spec_len)
{
PyObject *result = NULL;
InternalFormatSpec format;
@@ -1021,17 +1166,13 @@
/* parse the format_spec */
if (!parse_internal_render_format_spec(format_spec,
- format_spec_len,
- &format, '\0'))
+ format_spec_len,
+ &format, '\0'))
goto done;
/* type conversion? */
switch (format.type) {
- case '\0':
- /* 'Z' means like 'g', but with at least one decimal. See
- PyOS_ascii_formatd */
- format.type = 'Z';
- /* Deliberate fall through to the next case statement */
+ case '\0': /* No format code: like 'g', but with at least one decimal. */
case 'e':
case 'E':
case 'f':
Modified: python/trunk/Objects/stringlib/localeutil.h
==============================================================================
--- python/trunk/Objects/stringlib/localeutil.h (original)
+++ python/trunk/Objects/stringlib/localeutil.h Wed Apr 22 15:29:05 2009
@@ -5,126 +5,208 @@
#include <locale.h>
+#define MAX(x, y) ((x) < (y) ? (y) : (x))
+#define MIN(x, y) ((x) < (y) ? (x) : (y))
+
+typedef struct {
+ const char *grouping;
+ char previous;
+ Py_ssize_t i; /* Where we're currently pointing in grouping. */
+} GroupGenerator;
+
+static void
+_GroupGenerator_init(GroupGenerator *self, const char *grouping)
+{
+ self->grouping = grouping;
+ self->i = 0;
+ self->previous = 0;
+}
+
+/* Returns the next grouping, or 0 to signify end. */
+static Py_ssize_t
+_GroupGenerator_next(GroupGenerator *self)
+{
+ /* Note that we don't really do much error checking here. If a
+ grouping string contains just CHAR_MAX, for example, then just
+ terminate the generator. That shouldn't happen, but at least we
+ fail gracefully. */
+ switch (self->grouping[self->i]) {
+ case 0:
+ return self->previous;
+ case CHAR_MAX:
+ /* Stop the generator. */
+ return 0;
+ default: {
+ char ch = self->grouping[self->i];
+ self->previous = ch;
+ self->i++;
+ return (Py_ssize_t)ch;
+ }
+ }
+}
+
+/* Fill in some digits, leading zeros, and thousands separator. All
+ are optional, depending on when we're called. */
+static void
+fill(STRINGLIB_CHAR **digits_end, STRINGLIB_CHAR **buffer_end,
+ Py_ssize_t n_chars, Py_ssize_t n_zeros, const char* thousands_sep,
+ Py_ssize_t thousands_sep_len)
+{
+#if STRINGLIB_IS_UNICODE
+ Py_ssize_t i;
+#endif
+
+ if (thousands_sep) {
+ *buffer_end -= thousands_sep_len;
+
+ /* Copy the thousands_sep chars into the buffer. */
+#if STRINGLIB_IS_UNICODE
+ /* Convert from the char's of the thousands_sep from
+ the locale into unicode. */
+ for (i = 0; i < thousands_sep_len; ++i)
+ (*buffer_end)[i] = thousands_sep[i];
+#else
+ /* No conversion, just memcpy the thousands_sep. */
+ memcpy(*buffer_end, thousands_sep, thousands_sep_len);
+#endif
+ }
+
+ *buffer_end -= n_chars;
+ *digits_end -= n_chars;
+ memcpy(*buffer_end, *digits_end, n_chars * sizeof(STRINGLIB_CHAR));
+
+ *buffer_end -= n_zeros;
+ STRINGLIB_FILL(*buffer_end, '0', n_zeros);
+}
+
/**
* _Py_InsertThousandsGrouping:
* @buffer: A pointer to the start of a string.
- * @n_buffer: The length of the string.
+ * @n_buffer: Number of characters in @buffer.
+ * @digits: A pointer to the digits we're reading from. If count
+ * is non-NULL, this is unused.
* @n_digits: The number of digits in the string, in which we want
* to put the grouping chars.
- * @buf_size: The maximum size of the buffer pointed to by buffer.
- * @count: If non-NULL, points to a variable that will receive the
- * number of characters we need to insert (and no formatting
- * will actually occur).
- * @append_zero_char: If non-zero, put a trailing zero at the end of
- * of the resulting string, if and only if we modified the
- * string.
+ * @min_width: The minimum width of the digits in the output string.
+ * Output will be zero-padded on the left to fill.
+ * @grouping: see definition in localeconv().
+ * @thousands_sep: see definition in localeconv().
*
- * Inserts thousand grouping characters (as defined in the current
- * locale) into the string between buffer and buffer+n_digits. If
- * count is non-NULL, don't do any formatting, just count the number
- * of characters to insert. This is used by the caller to
- * appropriately resize the buffer, if needed. If count is non-NULL,
- * buffer can be NULL (it is not dereferenced at all in that case).
+ * There are 2 modes: counting and filling. If @buffer is NULL,
+ * we are in counting mode, else filling mode.
+ * If counting, the required buffer size is returned.
+ * If filling, we know the buffer will be large enough, so we don't
+ * need to pass in the buffer size.
+ * Inserts thousand grouping characters (as defined by grouping and
+ * thousands_sep) into the string between buffer and buffer+n_digits.
*
* Return value: 0 on error, else 1. Note that no error can occur if
* count is non-NULL.
*
* This name won't be used, the includer of this file should define
* it to be the actual function name, based on unicode or string.
+ *
+ * As closely as possible, this code mimics the logic in decimal.py's
+ _insert_thousands_sep().
**/
-int
+Py_ssize_t
_Py_InsertThousandsGrouping(STRINGLIB_CHAR *buffer,
- Py_ssize_t n_buffer,
- Py_ssize_t n_digits,
- Py_ssize_t buf_size,
- Py_ssize_t *count,
- int append_zero_char)
+ Py_ssize_t n_buffer,
+ STRINGLIB_CHAR *digits,
+ Py_ssize_t n_digits,
+ Py_ssize_t min_width,
+ const char *grouping,
+ const char *thousands_sep)
{
- struct lconv *locale_data = localeconv();
- const char *grouping = locale_data->grouping;
- const char *thousands_sep = locale_data->thousands_sep;
- Py_ssize_t thousands_sep_len = strlen(thousands_sep);
- STRINGLIB_CHAR *pend = NULL; /* current end of buffer */
- STRINGLIB_CHAR *pmax = NULL; /* max of buffer */
- char current_grouping;
- Py_ssize_t remaining = n_digits; /* Number of chars remaining to
- be looked at */
-
- /* Initialize the character count, if we're just counting. */
- if (count)
- *count = 0;
- else {
- /* We're not just counting, we're modifying buffer */
- pend = buffer + n_buffer;
- pmax = buffer + buf_size;
- }
-
- /* Starting at the end and working right-to-left, keep track of
- what grouping needs to be added and insert that. */
- current_grouping = *grouping++;
-
- /* If the first character is 0, perform no grouping at all. */
- if (current_grouping == 0)
- return 1;
-
- while (remaining > current_grouping) {
- /* Always leave buffer and pend valid at the end of this
- loop, since we might leave with a return statement. */
-
- remaining -= current_grouping;
- if (count) {
- /* We're only counting, not touching the memory. */
- *count += thousands_sep_len;
- }
- else {
- /* Do the formatting. */
-
- STRINGLIB_CHAR *plast = buffer + remaining;
-
- /* Is there room to insert thousands_sep_len chars? */
- if (pmax - pend < thousands_sep_len)
- /* No room. */
- return 0;
-
- /* Move the rest of the string down. */
- memmove(plast + thousands_sep_len,
- plast,
- (pend - plast) * sizeof(STRINGLIB_CHAR));
- /* Copy the thousands_sep chars into the buffer. */
-#if STRINGLIB_IS_UNICODE
- /* Convert from the char's of the thousands_sep from
- the locale into unicode. */
- {
- Py_ssize_t i;
- for (i = 0; i < thousands_sep_len; ++i)
- plast[i] = thousands_sep[i];
- }
-#else
- /* No conversion, just memcpy the thousands_sep. */
- memcpy(plast, thousands_sep, thousands_sep_len);
-#endif
- }
+ Py_ssize_t count = 0;
+ Py_ssize_t n_zeros;
+ int loop_broken = 0;
+ int use_separator = 0; /* First time through, don't append the
+ separator. They only go between
+ groups. */
+ STRINGLIB_CHAR *buffer_end = NULL;
+ STRINGLIB_CHAR *digits_end = NULL;
+ Py_ssize_t l;
+ Py_ssize_t n_chars;
+ Py_ssize_t thousands_sep_len = strlen(thousands_sep);
+ Py_ssize_t remaining = n_digits; /* Number of chars remaining to
+ be looked at */
+ /* A generator that returns all of the grouping widths, until it
+ returns 0. */
+ GroupGenerator groupgen;
+ _GroupGenerator_init(&groupgen, grouping);
+
+ if (buffer) {
+ buffer_end = buffer + n_buffer;
+ digits_end = digits + n_digits;
+ }
+
+ while ((l = _GroupGenerator_next(&groupgen)) > 0) {
+ l = MIN(l, MAX(MAX(remaining, min_width), 1));
+ n_zeros = MAX(0, l - remaining);
+ n_chars = MAX(0, MIN(remaining, l));
+
+ /* Use n_zero zero's and n_chars chars */
+
+ /* Count only, don't do anything. */
+ count += (use_separator ? thousands_sep_len : 0) + n_zeros + n_chars;
+
+ if (buffer) {
+ /* Copy into the output buffer. */
+ fill(&digits_end, &buffer_end, n_chars, n_zeros,
+ use_separator ? thousands_sep : NULL, thousands_sep_len);
+ }
+
+ /* Use a separator next time. */
+ use_separator = 1;
+
+ remaining -= n_chars;
+ min_width -= l;
+
+ if (remaining <= 0 && min_width <= 0) {
+ loop_broken = 1;
+ break;
+ }
+ min_width -= thousands_sep_len;
+ }
+ if (!loop_broken) {
+ /* We left the loop without using a break statement. */
+
+ l = MAX(MAX(remaining, min_width), 1);
+ n_zeros = MAX(0, l - remaining);
+ n_chars = MAX(0, MIN(remaining, l));
+
+ /* Use n_zero zero's and n_chars chars */
+ count += (use_separator ? thousands_sep_len : 0) + n_zeros + n_chars;
+ if (buffer) {
+ /* Copy into the output buffer. */
+ fill(&digits_end, &buffer_end, n_chars, n_zeros,
+ use_separator ? thousands_sep : NULL, thousands_sep_len);
+ }
+ }
+ return count;
+}
- /* Adjust end pointer. */
- pend += thousands_sep_len;
+/**
+ * _Py_InsertThousandsGroupingLocale:
+ * @buffer: A pointer to the start of a string.
+ * @n_digits: The number of digits in the string, in which we want
+ * to put the grouping chars.
+ *
+ * Reads thee current locale and calls _Py_InsertThousandsGrouping().
+ **/
+Py_ssize_t
+_Py_InsertThousandsGroupingLocale(STRINGLIB_CHAR *buffer,
+ Py_ssize_t n_buffer,
+ STRINGLIB_CHAR *digits,
+ Py_ssize_t n_digits,
+ Py_ssize_t min_width)
+{
+ struct lconv *locale_data = localeconv();
+ const char *grouping = locale_data->grouping;
+ const char *thousands_sep = locale_data->thousands_sep;
- /* Move to the next grouping character, unless we're
- repeating (which is designated by a grouping of 0). */
- if (*grouping != 0) {
- current_grouping = *grouping++;
- if (current_grouping == CHAR_MAX)
- /* We're done. */
- break;
- }
- }
- if (append_zero_char) {
- /* Append a zero character to mark the end of the string,
- if there's room. */
- if (pend - (buffer + remaining) < 1)
- /* No room, error. */
- return 0;
- *pend = 0;
- }
- return 1;
+ return _Py_InsertThousandsGrouping(buffer, n_buffer, digits, n_digits,
+ min_width, grouping, thousands_sep);
}
#endif /* STRINGLIB_LOCALEUTIL_H */
Modified: python/trunk/Objects/stringlib/stringdefs.h
==============================================================================
--- python/trunk/Objects/stringlib/stringdefs.h (original)
+++ python/trunk/Objects/stringlib/stringdefs.h Wed Apr 22 15:29:05 2009
@@ -6,6 +6,15 @@
compiled as unicode. */
#define STRINGLIB_IS_UNICODE 0
+/* _tolower and _toupper are defined by SUSv2, but they're not ISO C */
+/* This needs to be cleaned up. See issue 5793. */
+#ifndef _tolower
+#define _tolower tolower
+#endif
+#ifndef _toupper
+#define _toupper toupper
+#endif
+
#define STRINGLIB_OBJECT PyStringObject
#define STRINGLIB_CHAR char
#define STRINGLIB_TYPE_NAME "string"
@@ -13,8 +22,8 @@
#define STRINGLIB_EMPTY nullstring
#define STRINGLIB_ISDECIMAL(x) ((x >= '0') && (x <= '9'))
#define STRINGLIB_TODECIMAL(x) (STRINGLIB_ISDECIMAL(x) ? (x - '0') : -1)
-#define STRINGLIB_TOUPPER toupper
-#define STRINGLIB_TOLOWER tolower
+#define STRINGLIB_TOUPPER(x) _toupper(Py_CHARMASK(x))
+#define STRINGLIB_TOLOWER(x) _tolower(Py_CHARMASK(x))
#define STRINGLIB_FILL memset
#define STRINGLIB_STR PyString_AS_STRING
#define STRINGLIB_LEN PyString_GET_SIZE
@@ -24,5 +33,6 @@
#define STRINGLIB_CMP memcmp
#define STRINGLIB_TOSTR PyObject_Str
#define STRINGLIB_GROUPING _PyString_InsertThousandsGrouping
+#define STRINGLIB_GROUPING_LOCALE _PyString_InsertThousandsGroupingLocale
#endif /* !STRINGLIB_STRINGDEFS_H */
Modified: python/trunk/Python/pystrtod.c
==============================================================================
--- python/trunk/Python/pystrtod.c (original)
+++ python/trunk/Python/pystrtod.c Wed Apr 22 15:29:05 2009
@@ -37,6 +37,15 @@
*
* Return value: the #gdouble value.
**/
+
+/*
+ Use system strtod; since strtod is locale aware, we may
+ have to first fix the decimal separator.
+
+ Note that unlike _Py_dg_strtod, the system strtod may not always give
+ correctly rounded results.
+*/
+
double
PyOS_ascii_strtod(const char *nptr, char **endptr)
{
@@ -187,6 +196,13 @@
return val;
}
+double
+PyOS_ascii_atof(const char *nptr)
+{
+ return PyOS_ascii_strtod(nptr, NULL);
+}
+
+
/* Given a string that may have a decimal point in the current
locale, change it back to a dot. Since the string cannot get
longer, no need for a maximum buffer size parameter. */
@@ -292,8 +308,9 @@
}
}
-/* Ensure that buffer has a decimal point in it. The decimal point
- will not be in the current locale, it will always be '.' */
+/* Ensure that buffer has a decimal point in it. The decimal point will not
+ be in the current locale, it will always be '.'. Don't add a decimal if an
+ exponent is present. */
Py_LOCAL_INLINE(void)
ensure_decimal_point(char* buffer, size_t buf_size)
{
@@ -322,7 +339,8 @@
insert_count = 1;
}
}
- else {
+ else if (!(*p == 'e' || *p == 'E')) {
+ /* Don't add ".0" if we have an exponent. */
chars_to_insert = ".0";
insert_count = 2;
}
@@ -341,37 +359,6 @@
}
}
-/* Add the locale specific grouping characters to buffer. Note
- that any decimal point (if it's present) in buffer is already
- locale-specific. Return 0 on error, else 1. */
-Py_LOCAL_INLINE(int)
-add_thousands_grouping(char* buffer, size_t buf_size)
-{
- Py_ssize_t len = strlen(buffer);
- struct lconv *locale_data = localeconv();
- const char *decimal_point = locale_data->decimal_point;
-
- /* Find the decimal point, if any. We're only concerned
- about the characters to the left of the decimal when
- adding grouping. */
- char *p = strstr(buffer, decimal_point);
- if (!p) {
- /* No decimal, use the entire string. */
-
- /* If any exponent, adjust p. */
- p = strpbrk(buffer, "eE");
- if (!p)
- /* No exponent and no decimal. Use the entire
- string. */
- p = buffer + len;
- }
- /* At this point, p points just past the right-most character we
- want to format. We need to add the grouping string for the
- characters between buffer and p. */
- return _PyString_InsertThousandsGrouping(buffer, len, p-buffer,
- buf_size, NULL, 1);
-}
-
/* see FORMATBUFLEN in unicodeobject.c */
#define FLOAT_FORMATBUFLEN 120
@@ -386,9 +373,8 @@
* Converts a #gdouble to a string, using the '.' as
* decimal point. To format the number you pass in
* a printf()-style format string. Allowed conversion
- * specifiers are 'e', 'E', 'f', 'F', 'g', 'G', and 'n'.
+ * specifiers are 'e', 'E', 'f', 'F', 'g', 'G', and 'Z'.
*
- * 'n' is the same as 'g', except it uses the current locale.
* 'Z' is the same as 'g', except it always has a decimal and
* at least one digit after the decimal.
*
@@ -403,11 +389,6 @@
char format_char;
size_t format_len = strlen(format);
- /* For type 'n', we need to make a copy of the format string, because
- we're going to modify 'n' -> 'g', and format is const char*, so we
- can't modify it directly. FLOAT_FORMATBUFLEN should be longer than
- we ever need this to be. There's an upcoming check to ensure it's
- big enough. */
/* Issue 2264: code 'Z' requires copying the format. 'Z' is 'g', but
also with at least one character past the decimal. */
char tmp_format[FLOAT_FORMATBUFLEN];
@@ -433,12 +414,12 @@
if (!(format_char == 'e' || format_char == 'E' ||
format_char == 'f' || format_char == 'F' ||
format_char == 'g' || format_char == 'G' ||
- format_char == 'n' || format_char == 'Z'))
+ format_char == 'Z'))
return NULL;
- /* Map 'n' or 'Z' format_char to 'g', by copying the format string and
+ /* Map 'Z' format_char to 'g', by copying the format string and
replacing the final char with a 'g' */
- if (format_char == 'n' || format_char == 'Z') {
+ if (format_char == 'Z') {
if (format_len + 1 >= sizeof(tmp_format)) {
/* The format won't fit in our copy. Error out. In
practice, this will never happen and will be
@@ -457,11 +438,8 @@
/* Do various fixups on the return string */
/* Get the current locale, and find the decimal point string.
- Convert that string back to a dot. Do not do this if using the
- 'n' (number) format code, since we want to keep the localized
- decimal point in that case. */
- if (format_char != 'n')
- change_decimal_from_locale_to_dot(buffer);
+ Convert that string back to a dot. */
+ change_decimal_from_locale_to_dot(buffer);
/* If an exponent exists, ensure that the exponent is at least
MIN_EXPONENT_DIGITS digits, providing the buffer is large enough
@@ -475,16 +453,111 @@
if (format_char == 'Z')
ensure_decimal_point(buffer, buf_size);
- /* If format_char is 'n', add the thousands grouping. */
- if (format_char == 'n')
- if (!add_thousands_grouping(buffer, buf_size))
- return NULL;
-
return buffer;
}
-double
-PyOS_ascii_atof(const char *nptr)
+PyAPI_FUNC(char *) PyOS_double_to_string(double val,
+ char format_code,
+ int precision,
+ int flags,
+ int *type)
{
- return PyOS_ascii_strtod(nptr, NULL);
+ char buf[128];
+ char format[32];
+ Py_ssize_t len;
+ char *result;
+ char *p;
+ int t;
+ int upper = 0;
+
+ /* Validate format_code, and map upper and lower case */
+ switch (format_code) {
+ case 'e': /* exponent */
+ case 'f': /* fixed */
+ case 'g': /* general */
+ break;
+ case 'E':
+ upper = 1;
+ format_code = 'e';
+ break;
+ case 'F':
+ upper = 1;
+ format_code = 'f';
+ break;
+ case 'G':
+ upper = 1;
+ format_code = 'g';
+ break;
+ case 'r': /* repr format */
+ /* Supplied precision is unused, must be 0. */
+ if (precision != 0) {
+ PyErr_BadInternalCall();
+ return NULL;
+ }
+ precision = 17;
+ format_code = 'g';
+ break;
+ case 's': /* str format */
+ /* Supplied precision is unused, must be 0. */
+ if (precision != 0) {
+ PyErr_BadInternalCall();
+ return NULL;
+ }
+ precision = 12;
+ format_code = 'g';
+ break;
+ default:
+ PyErr_BadInternalCall();
+ return NULL;
+ }
+
+ /* Handle nan and inf. */
+ if (Py_IS_NAN(val)) {
+ strcpy(buf, "nan");
+ t = Py_DTST_NAN;
+ } else if (Py_IS_INFINITY(val)) {
+ if (copysign(1., val) == 1.)
+ strcpy(buf, "inf");
+ else
+ strcpy(buf, "-inf");
+ t = Py_DTST_INFINITE;
+ } else {
+ t = Py_DTST_FINITE;
+
+
+ if (flags & Py_DTSF_ADD_DOT_0)
+ format_code = 'Z';
+
+ PyOS_snprintf(format, 32, "%%%s.%i%c", (flags & Py_DTSF_ALT ? "#" : ""), precision, format_code);
+ PyOS_ascii_formatd(buf, sizeof(buf), format, val);
+ }
+
+ len = strlen(buf);
+
+ /* Add 1 for the trailing 0 byte.
+ Add 1 because we might need to make room for the sign.
+ */
+ result = PyMem_Malloc(len + 2);
+ if (result == NULL) {
+ PyErr_NoMemory();
+ return NULL;
+ }
+ p = result;
+
+ /* Never add sign for nan/inf, even if asked. */
+ if (flags & Py_DTSF_SIGN && buf[0] != '-' && t == Py_DTST_FINITE)
+ *p++ = '+';
+
+ strcpy(p, buf);
+
+ if (upper) {
+ /* Convert to upper case. */
+ char *p1;
+ for (p1 = p; *p1; p1++)
+ *p1 = toupper(*p1);
+ }
+
+ if (type)
+ *type = t;
+ return result;
}
More information about the Python-checkins
mailing list