Skip to content

C
cpython
Commit 7e303731 authored by Benjamin Peterson's avatar Benjamin Peterson
Browse files
Options

remove MAX_MAXCHAR because it's unsafe for computing maximum codepoitn value (see #18183)

parent 0e547b66
Show whitespace changes
Inline Side-by-side
Showing with 32 additions and 31 deletions
Lib/test/test_unicode.py
View file @ 7e303731
......@@ -566,6 +566,9 @@ class UnicodeTest(string_tests.CommonTest,
self.assertEqual('\U0008fffe'.lower(), '\U0008fffe')
self.assertEqual('\u2177'.lower(), '\u2177')
# See issue #18183 for this one.
'\U00010000\U00100000'.lower()
def test_casefold(self):
self.assertEqual('hello'.casefold(), 'hello')
self.assertEqual('hELlo'.casefold(), 'hello')
......
Misc/NEWS
View file @ 7e303731
......@@ -12,6 +12,9 @@ What's New in Python 3.3.3 release candidate 1?
Core and Builtins
-----------------
- Issue #18183: Fix various unicode operations on strings with large unicode
codepoints.
- Issue #18180: Fix ref leak in _PyImport_GetDynLoadWindows().
- Issue #18038: SyntaxError raised during compilation sources with illegal
......
Objects/unicodeobject.c
View file @ 7e303731
......@@ -112,11 +112,6 @@ extern "C" {
#define _PyUnicode_DATA_ANY(op) \
(((PyUnicodeObject*)(op))->data.any)
/* Optimized version of Py_MAX() to compute the maximum character:
use it when your are computing the second argument of PyUnicode_New() */
#define MAX_MAXCHAR(maxchar1, maxchar2) \
((maxchar1) | (maxchar2))
#undef PyUnicode_READY
#define PyUnicode_READY(op) \
(assert(_PyUnicode_CHECK(op)), \
......@@ -2495,7 +2490,7 @@ PyUnicode_FromFormatV(const char *format, va_list vargs)
case 'c':
{
Py_UCS4 ordinal = va_arg(count, int);
maxchar = MAX_MAXCHAR(maxchar, ordinal);
maxchar = Py_MAX(maxchar, ordinal);
n++;
break;
}
......@@ -2591,7 +2586,7 @@ PyUnicode_FromFormatV(const char *format, va_list vargs)
/* since PyUnicode_DecodeUTF8 returns already flexible
unicode objects, there is no need to call ready on them */
argmaxchar = PyUnicode_MAX_CHAR_VALUE(str);
maxchar = MAX_MAXCHAR(maxchar, argmaxchar);
maxchar = Py_MAX(maxchar, argmaxchar);
n += PyUnicode_GET_LENGTH(str);
/* Remember the str and switch to the next slot */
*callresult++ = str;
......@@ -2604,7 +2599,7 @@ PyUnicode_FromFormatV(const char *format, va_list vargs)
if (PyUnicode_READY(obj) == -1)
goto fail;
argmaxchar = PyUnicode_MAX_CHAR_VALUE(obj);
maxchar = MAX_MAXCHAR(maxchar, argmaxchar);
maxchar = Py_MAX(maxchar, argmaxchar);
n += PyUnicode_GET_LENGTH(obj);
break;
}
......@@ -2619,7 +2614,7 @@ PyUnicode_FromFormatV(const char *format, va_list vargs)
if (PyUnicode_READY(obj) == -1)
goto fail;
argmaxchar = PyUnicode_MAX_CHAR_VALUE(obj);
maxchar = MAX_MAXCHAR(maxchar, argmaxchar);
maxchar = Py_MAX(maxchar, argmaxchar);
n += PyUnicode_GET_LENGTH(obj);
*callresult++ = NULL;
}
......@@ -2632,7 +2627,7 @@ PyUnicode_FromFormatV(const char *format, va_list vargs)
goto fail;
}
argmaxchar = PyUnicode_MAX_CHAR_VALUE(str_obj);
maxchar = MAX_MAXCHAR(maxchar, argmaxchar);
maxchar = Py_MAX(maxchar, argmaxchar);
n += PyUnicode_GET_LENGTH(str_obj);
*callresult++ = str_obj;
}
......@@ -2651,7 +2646,7 @@ PyUnicode_FromFormatV(const char *format, va_list vargs)
goto fail;
}
argmaxchar = PyUnicode_MAX_CHAR_VALUE(str);
maxchar = MAX_MAXCHAR(maxchar, argmaxchar);
maxchar = Py_MAX(maxchar, argmaxchar);
n += PyUnicode_GET_LENGTH(str);
/* Remember the str and switch to the next slot */
*callresult++ = str;
......@@ -2670,7 +2665,7 @@ PyUnicode_FromFormatV(const char *format, va_list vargs)
goto fail;
}
argmaxchar = PyUnicode_MAX_CHAR_VALUE(repr);
maxchar = MAX_MAXCHAR(maxchar, argmaxchar);
maxchar = Py_MAX(maxchar, argmaxchar);
n += PyUnicode_GET_LENGTH(repr);
/* Remember the repr and switch to the next slot */
*callresult++ = repr;
......@@ -2689,7 +2684,7 @@ PyUnicode_FromFormatV(const char *format, va_list vargs)
goto fail;
}
argmaxchar = PyUnicode_MAX_CHAR_VALUE(ascii);
maxchar = MAX_MAXCHAR(maxchar, argmaxchar);
maxchar = Py_MAX(maxchar, argmaxchar);
n += PyUnicode_GET_LENGTH(ascii);
/* Remember the repr and switch to the next slot */
*callresult++ = ascii;
......@@ -8628,11 +8623,11 @@ fix_decimal_and_space_to_ascii(PyObject *self)
}
if (fixed != 0) {
modified = 1;
maxchar = MAX_MAXCHAR(maxchar, fixed);
maxchar = Py_MAX(maxchar, fixed);
PyUnicode_WRITE(kind, data, i, fixed);
}
else
maxchar = MAX_MAXCHAR(maxchar, ch);
maxchar = Py_MAX(maxchar, ch);
}
}
......@@ -8673,7 +8668,7 @@ PyUnicode_TransformDecimalToASCII(Py_UNICODE *s,
int decimal = Py_UNICODE_TODECIMAL(ch);
if (decimal >= 0)
ch = '0' + decimal;
maxchar = MAX_MAXCHAR(maxchar, ch);
maxchar = Py_MAX(maxchar, ch);
}
}
......@@ -8914,7 +8909,7 @@ _PyUnicode_InsertThousandsGrouping(
if (unicode == NULL) {
*maxchar = 127;
if (len != n_digits) {
*maxchar = MAX_MAXCHAR(*maxchar,
*maxchar = Py_MAX(*maxchar,
PyUnicode_MAX_CHAR_VALUE(thousands_sep));
}
}
......@@ -9309,14 +9304,14 @@ do_capitalize(int kind, void *data, Py_ssize_t length, Py_UCS4 *res, Py_UCS4 *ma
c = PyUnicode_READ(kind, data, 0);
n_res = _PyUnicode_ToUpperFull(c, mapped);
for (j = 0; j < n_res; j++) {
*maxchar = MAX_MAXCHAR(*maxchar, mapped[j]);
*maxchar = Py_MAX(*maxchar, mapped[j]);
res[k++] = mapped[j];
}
for (i = 1; i < length; i++) {
c = PyUnicode_READ(kind, data, i);
n_res = lower_ucs4(kind, data, length, i, c, mapped);
for (j = 0; j < n_res; j++) {
*maxchar = MAX_MAXCHAR(*maxchar, mapped[j]);
*maxchar = Py_MAX(*maxchar, mapped[j]);
res[k++] = mapped[j];
}
}
......@@ -9341,7 +9336,7 @@ do_swapcase(int kind, void *data, Py_ssize_t length, Py_UCS4 *res, Py_UCS4 *maxc
mapped[0] = c;
}
for (j = 0; j < n_res; j++) {
*maxchar = MAX_MAXCHAR(*maxchar, mapped[j]);
*maxchar = Py_MAX(*maxchar, mapped[j]);
res[k++] = mapped[j];
}
}
......@@ -9362,7 +9357,7 @@ do_upper_or_lower(int kind, void *data, Py_ssize_t length, Py_UCS4 *res,
else
n_res = _PyUnicode_ToUpperFull(c, mapped);
for (j = 0; j < n_res; j++) {
*maxchar = MAX_MAXCHAR(*maxchar, mapped[j]);
*maxchar = Py_MAX(*maxchar, mapped[j]);
res[k++] = mapped[j];
}
}
......@@ -9391,7 +9386,7 @@ do_casefold(int kind, void *data, Py_ssize_t length, Py_UCS4 *res, Py_UCS4 *maxc
Py_UCS4 mapped[3];
int j, n_res = _PyUnicode_ToFoldedFull(c, mapped);
for (j = 0; j < n_res; j++) {
*maxchar = MAX_MAXCHAR(*maxchar, mapped[j]);
*maxchar = Py_MAX(*maxchar, mapped[j]);
res[k++] = mapped[j];
}
}
......@@ -9416,7 +9411,7 @@ do_title(int kind, void *data, Py_ssize_t length, Py_UCS4 *res, Py_UCS4 *maxchar
n_res = _PyUnicode_ToTitleFull(c, mapped);
for (j = 0; j < n_res; j++) {
*maxchar = MAX_MAXCHAR(*maxchar, mapped[j]);
*maxchar = Py_MAX(*maxchar, mapped[j]);
res[k++] = mapped[j];
}
......@@ -9571,7 +9566,7 @@ PyUnicode_Join(PyObject *separator, PyObject *seq)
goto onError;
sz += PyUnicode_GET_LENGTH(item);
item_maxchar = PyUnicode_MAX_CHAR_VALUE(item);
maxchar = MAX_MAXCHAR(maxchar, item_maxchar);
maxchar = Py_MAX(maxchar, item_maxchar);
if (i != 0)
sz += seplen;
if (sz < old_sz || sz > PY_SSIZE_T_MAX) {
......@@ -9747,7 +9742,7 @@ pad(PyObject *self,
return NULL;
}
maxchar = PyUnicode_MAX_CHAR_VALUE(self);
maxchar = MAX_MAXCHAR(maxchar, fill);
maxchar = Py_MAX(maxchar, fill);
u = PyUnicode_New(left + _PyUnicode_LENGTH(self) + right, maxchar);
if (!u)
return NULL;
......@@ -10061,7 +10056,7 @@ replace(PyObject *self, PyObject *str1,
/* Replacing str1 with str2 may cause a maxchar reduction in the
result string. */
mayshrink = (maxchar_str2 < maxchar);
maxchar = MAX_MAXCHAR(maxchar, maxchar_str2);
maxchar = Py_MAX(maxchar, maxchar_str2);
if (len1 == len2) {
/* same length */
......@@ -10647,7 +10642,7 @@ PyUnicode_Concat(PyObject *left, PyObject *right)
maxchar = PyUnicode_MAX_CHAR_VALUE(u);
maxchar2 = PyUnicode_MAX_CHAR_VALUE(v);
maxchar = MAX_MAXCHAR(maxchar, maxchar2);
maxchar = Py_MAX(maxchar, maxchar2);
/* Concat the two Unicode strings */
w = PyUnicode_New(new_len, maxchar);
......@@ -10734,7 +10729,7 @@ PyUnicode_Append(PyObject **p_left, PyObject *right)
else {
maxchar = PyUnicode_MAX_CHAR_VALUE(left);
maxchar2 = PyUnicode_MAX_CHAR_VALUE(right);
maxchar = MAX_MAXCHAR(maxchar, maxchar2);
maxchar = Py_MAX(maxchar, maxchar2);
/* Concat the two Unicode strings */
res = PyUnicode_New(new_len, maxchar);
......@@ -13846,15 +13841,15 @@ PyUnicode_Format(PyObject *format, PyObject *args)
if (!(flags & F_LJUST)) {
if (sign) {
if ((width-1) > len)
bufmaxchar = MAX_MAXCHAR(bufmaxchar, fill);
bufmaxchar = Py_MAX(bufmaxchar, fill);
}
else {
if (width > len)
bufmaxchar = MAX_MAXCHAR(bufmaxchar, fill);
bufmaxchar = Py_MAX(bufmaxchar, fill);
}
}
maxchar = _PyUnicode_FindMaxChar(temp, 0, pindex+len);
bufmaxchar = MAX_MAXCHAR(bufmaxchar, maxchar);
bufmaxchar = Py_MAX(bufmaxchar, maxchar);
buflen = width;
if (sign && len == width)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7