Skip to content

C
cpython
Commit eac71820 authored by Georg Brandl's avatar Georg Brandl
Browse files
Options

Issue #3282: str.isprintable() should return False for undefined Unicode characters.

parent edda13ca
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 8936 additions and 8817 deletions
Lib/test/test_unicode.py
View file @ eac71820
......@@ -420,12 +420,17 @@ class UnicodeTest(
self.assertFalse(" ".isidentifier())
self.assertFalse("[".isidentifier())
self.assertFalse("".isidentifier())
self.assertFalse("0".isidentifier())
def test_isprintable(self):
self.assertTrue("".isprintable())
self.assertTrue("abcdefg".isprintable())
self.assertFalse("abcdefg\n".isprintable())
self.assertTrue("\u0370".isprintable())
# some defined Unicode character
self.assertTrue("\u0374".isprintable())
# undefined character
self.assertFalse("\u0370".isprintable())
# single surrogate character
self.assertFalse("\ud800".isprintable())
def test_contains(self):
......@@ -598,7 +603,7 @@ class UnicodeTest(
# format specifiers for user defined type
self.assertEqual('{0:abc}'.format(C()), 'abc')
# !r, !s and !a coersions
# !r, !s and !a coercions
self.assertEqual('{0!s}'.format('Hello'), 'Hello')
self.assertEqual('{0!s:}'.format('Hello'), 'Hello')
self.assertEqual('{0!s:15}'.format('Hello'), 'Hello ')
......@@ -606,11 +611,15 @@ class UnicodeTest(
self.assertEqual('{0!r}'.format('Hello'), "'Hello'")
self.assertEqual('{0!r:}'.format('Hello'), "'Hello'")
self.assertEqual('{0!r}'.format(F('Hello')), 'F(Hello)')
self.assertEqual('{0!r}'.format(F('\u0370')), 'F(\u0370)')
self.assertEqual('{0!r}'.format('\u0370'), "'\\u0370'") # nonprintable
self.assertEqual('{0!r}'.format('\u0374'), "'\u0374'") # printable
self.assertEqual('{0!r}'.format(F('\u0374')), 'F(\u0374)')
self.assertEqual('{0!a}'.format('Hello'), "'Hello'")
self.assertEqual('{0!a}'.format('\u0370'), "'\\u0370'") # nonprintable
self.assertEqual('{0!a}'.format('\u0374'), "'\\u0374'") # printable
self.assertEqual('{0!a:}'.format('Hello'), "'Hello'")
self.assertEqual('{0!a}'.format(F('Hello')), 'F(Hello)')
self.assertEqual('{0!a}'.format(F('\u0370')), 'F(\\u0370)')
self.assertEqual('{0!a}'.format(F('\u0374')), 'F(\\u0374)')
# test fallback to object.__format__
self.assertEqual('{0}'.format({}), '{}')
......
Misc/NEWS
View file @ eac71820
......@@ -12,6 +12,9 @@ What's new in Python 3.0b2?
Core and Builtins
-----------------
- Issue #3282: str.isprintable() should return False for undefined
Unicode characters.
- Issue #3236: Return small longs from PyLong_FromString.
Library
......
Modules/unicodename_db.h
View file @ eac71820
This diff is collapsed.
Objects/unicodectype.c
View file @ eac71820
......@@ -21,7 +21,7 @@
#define UPPER_MASK 0x80
#define XID_START_MASK 0x100
#define XID_CONTINUE_MASK 0x200
#define NONPRINTABLE_MASK 0x400
#define PRINTABLE_MASK 0x400
typedef struct {
const Py_UNICODE upper;
......@@ -693,7 +693,7 @@ int _PyUnicode_IsPrintable(Py_UNICODE ch)
{
const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
return (ctype->flags & NONPRINTABLE_MASK) == 0;
return (ctype->flags & PRINTABLE_MASK) != 0;
}
#ifndef WANT_WCTYPE_FUNCTIONS
......
Objects/unicodetype_db.h
View file @ eac71820
......@@ -3,150 +3,150 @@
/* a list of unique character type descriptors */
const _PyUnicode_TypeRecord _PyUnicode_TypeRecords[] = {
{0, 0, 0, 0, 0, 0},
{0, 0, 0, 0, 0, 1024},
{0, 0, 0, 0, 0, 1056},
{0, 0, 0, 0, 0, 1072},
{0, 0, 0, 0, 0, 0},
{0, 0, 0, 0, 0, 518},
{0, 0, 0, 1, 1, 518},
{0, 0, 0, 2, 2, 518},
{0, 0, 0, 3, 3, 518},
{0, 0, 0, 4, 4, 518},
{0, 0, 0, 5, 5, 518},
{0, 0, 0, 6, 6, 518},
{0, 0, 0, 7, 7, 518},
{0, 0, 0, 8, 8, 518},
{0, 0, 0, 9, 9, 518},
{0, 32, 0, 0, 0, 897},
{0, 0, 0, 0, 0, 512},
{65504, 0, 65504, 0, 0, 777},
{0, 0, 0, 0, 0, 777},
{0, 0, 0, 0, 2, 4},
{0, 0, 0, 0, 3, 4},
{743, 0, 743, 0, 0, 777},
{0, 0, 0, 0, 1, 4},
{121, 0, 121, 0, 0, 777},
{0, 1, 0, 0, 0, 897},
{65535, 0, 65535, 0, 0, 777},
{0, 65337, 0, 0, 0, 897},
{65304, 0, 65304, 0, 0, 777},
{0, 65415, 0, 0, 0, 897},
{65236, 0, 65236, 0, 0, 777},
{0, 210, 0, 0, 0, 897},
{0, 206, 0, 0, 0, 897},
{0, 205, 0, 0, 0, 897},
{0, 79, 0, 0, 0, 897},
{0, 202, 0, 0, 0, 897},
{0, 203, 0, 0, 0, 897},
{0, 207, 0, 0, 0, 897},
{97, 0, 97, 0, 0, 777},
{0, 211, 0, 0, 0, 897},
{0, 209, 0, 0, 0, 897},
{163, 0, 163, 0, 0, 777},
{0, 213, 0, 0, 0, 897},
{130, 0, 130, 0, 0, 777},
{0, 214, 0, 0, 0, 897},
{0, 218, 0, 0, 0, 897},
{0, 217, 0, 0, 0, 897},
{0, 219, 0, 0, 0, 897},
{0, 0, 0, 0, 0, 769},
{56, 0, 56, 0, 0, 777},
{0, 2, 1, 0, 0, 897},
{65535, 1, 0, 0, 0, 833},
{65534, 0, 65535, 0, 0, 777},
{65457, 0, 65457, 0, 0, 777},
{0, 65439, 0, 0, 0, 897},
{0, 65480, 0, 0, 0, 897},
{0, 65406, 0, 0, 0, 897},
{0, 0, 0, 0, 0, 897},
{0, 65373, 0, 0, 0, 897},
{0, 83, 0, 0, 0, 897},
{65326, 0, 65326, 0, 0, 777},
{65330, 0, 65330, 0, 0, 777},
{65331, 0, 65331, 0, 0, 777},
{65334, 0, 65334, 0, 0, 777},
{65333, 0, 65333, 0, 0, 777},
{65329, 0, 65329, 0, 0, 777},
{65327, 0, 65327, 0, 0, 777},
{65325, 0, 65325, 0, 0, 777},
{65323, 0, 65323, 0, 0, 777},
{65322, 0, 65322, 0, 0, 777},
{65318, 0, 65318, 0, 0, 777},
{65319, 0, 65319, 0, 0, 777},
{65317, 0, 65317, 0, 0, 777},
{65453, 0, 65453, 0, 0, 777},
{84, 0, 84, 0, 0, 512},
{0, 0, 0, 0, 0, 1},
{0, 38, 0, 0, 0, 897},
{0, 37, 0, 0, 0, 897},
{0, 64, 0, 0, 0, 897},
{0, 63, 0, 0, 0, 897},
{65498, 0, 65498, 0, 0, 777},
{65499, 0, 65499, 0, 0, 777},
{65505, 0, 65505, 0, 0, 777},
{65472, 0, 65472, 0, 0, 777},
{65473, 0, 65473, 0, 0, 777},
{65474, 0, 65474, 0, 0, 777},
{65479, 0, 65479, 0, 0, 777},
{65489, 0, 65489, 0, 0, 777},
{65482, 0, 65482, 0, 0, 777},
{65450, 0, 65450, 0, 0, 777},
{65456, 0, 65456, 0, 0, 777},
{7, 0, 7, 0, 0, 777},
{0, 65476, 0, 0, 0, 897},
{65440, 0, 65440, 0, 0, 777},
{0, 65529, 0, 0, 0, 897},
{0, 80, 0, 0, 0, 897},
{0, 48, 0, 0, 0, 897},
{65488, 0, 65488, 0, 0, 777},
{0, 0, 0, 0, 0, 513},
{0, 7264, 0, 0, 0, 897},
{0, 0, 0, 0, 1, 516},
{0, 0, 0, 0, 2, 516},
{0, 0, 0, 0, 3, 516},
{0, 0, 0, 0, 4, 516},
{0, 0, 0, 0, 5, 516},
{0, 0, 0, 0, 6, 516},
{0, 0, 0, 0, 7, 516},
{0, 0, 0, 0, 8, 516},
{0, 0, 0, 0, 9, 516},
{0, 0, 0, 0, 0, 768},
{65477, 0, 65477, 0, 0, 777},
{8, 0, 8, 0, 0, 777},
{0, 65528, 0, 0, 0, 897},
{74, 0, 74, 0, 0, 777},
{86, 0, 86, 0, 0, 777},
{100, 0, 100, 0, 0, 777},
{128, 0, 128, 0, 0, 777},
{112, 0, 112, 0, 0, 777},
{126, 0, 126, 0, 0, 777},
{0, 65528, 0, 0, 0, 833},
{9, 0, 9, 0, 0, 777},
{0, 65462, 0, 0, 0, 897},
{0, 65527, 0, 0, 0, 833},
{58331, 0, 58331, 0, 0, 777},
{0, 65450, 0, 0, 0, 897},
{0, 65436, 0, 0, 0, 897},
{0, 65424, 0, 0, 0, 897},
{0, 65408, 0, 0, 0, 897},
{0, 65410, 0, 0, 0, 897},
{0, 0, 0, 0, 0, 4},
{0, 0, 0, 0, 4, 4},
{0, 0, 0, 0, 5, 4},
{0, 0, 0, 0, 6, 4},
{0, 0, 0, 0, 7, 4},
{0, 0, 0, 0, 8, 4},
{0, 0, 0, 0, 9, 4},
{0, 58019, 0, 0, 0, 897},
{0, 57153, 0, 0, 0, 897},
{0, 57274, 0, 0, 0, 897},
{0, 16, 0, 0, 0, 768},
{65520, 0, 65520, 0, 0, 768},
{0, 26, 0, 0, 0, 0},
{65510, 0, 65510, 0, 0, 0},
{58272, 0, 58272, 0, 0, 777},
{0, 40, 0, 0, 0, 897},
{65496, 0, 65496, 0, 0, 777},
{0, 0, 0, 0, 0, 32},
{0, 0, 0, 0, 0, 48},
{0, 0, 0, 0, 0, 1024},
{0, 0, 0, 0, 0, 1542},
{0, 0, 0, 1, 1, 1542},
{0, 0, 0, 2, 2, 1542},
{0, 0, 0, 3, 3, 1542},
{0, 0, 0, 4, 4, 1542},
{0, 0, 0, 5, 5, 1542},
{0, 0, 0, 6, 6, 1542},
{0, 0, 0, 7, 7, 1542},
{0, 0, 0, 8, 8, 1542},
{0, 0, 0, 9, 9, 1542},
{0, 32, 0, 0, 0, 1921},
{0, 0, 0, 0, 0, 1536},
{65504, 0, 65504, 0, 0, 1801},
{0, 0, 0, 0, 0, 1801},
{0, 0, 0, 0, 2, 1028},
{0, 0, 0, 0, 3, 1028},
{743, 0, 743, 0, 0, 1801},
{0, 0, 0, 0, 1, 1028},
{121, 0, 121, 0, 0, 1801},
{0, 1, 0, 0, 0, 1921},
{65535, 0, 65535, 0, 0, 1801},
{0, 65337, 0, 0, 0, 1921},
{65304, 0, 65304, 0, 0, 1801},
{0, 65415, 0, 0, 0, 1921},
{65236, 0, 65236, 0, 0, 1801},
{0, 210, 0, 0, 0, 1921},
{0, 206, 0, 0, 0, 1921},
{0, 205, 0, 0, 0, 1921},
{0, 79, 0, 0, 0, 1921},
{0, 202, 0, 0, 0, 1921},
{0, 203, 0, 0, 0, 1921},
{0, 207, 0, 0, 0, 1921},
{97, 0, 97, 0, 0, 1801},
{0, 211, 0, 0, 0, 1921},
{0, 209, 0, 0, 0, 1921},
{163, 0, 163, 0, 0, 1801},
{0, 213, 0, 0, 0, 1921},
{130, 0, 130, 0, 0, 1801},
{0, 214, 0, 0, 0, 1921},
{0, 218, 0, 0, 0, 1921},
{0, 217, 0, 0, 0, 1921},
{0, 219, 0, 0, 0, 1921},
{0, 0, 0, 0, 0, 1793},
{56, 0, 56, 0, 0, 1801},
{0, 2, 1, 0, 0, 1921},
{65535, 1, 0, 0, 0, 1857},
{65534, 0, 65535, 0, 0, 1801},
{65457, 0, 65457, 0, 0, 1801},
{0, 65439, 0, 0, 0, 1921},
{0, 65480, 0, 0, 0, 1921},
{0, 65406, 0, 0, 0, 1921},
{0, 0, 0, 0, 0, 1921},
{0, 65373, 0, 0, 0, 1921},
{0, 83, 0, 0, 0, 1921},
{65326, 0, 65326, 0, 0, 1801},
{65330, 0, 65330, 0, 0, 1801},
{65331, 0, 65331, 0, 0, 1801},
{65334, 0, 65334, 0, 0, 1801},
{65333, 0, 65333, 0, 0, 1801},
{65329, 0, 65329, 0, 0, 1801},
{65327, 0, 65327, 0, 0, 1801},
{65325, 0, 65325, 0, 0, 1801},
{65323, 0, 65323, 0, 0, 1801},
{65322, 0, 65322, 0, 0, 1801},
{65318, 0, 65318, 0, 0, 1801},
{65319, 0, 65319, 0, 0, 1801},
{65317, 0, 65317, 0, 0, 1801},
{65453, 0, 65453, 0, 0, 1801},
{84, 0, 84, 0, 0, 1536},
{0, 0, 0, 0, 0, 1025},
{0, 38, 0, 0, 0, 1921},
{0, 37, 0, 0, 0, 1921},
{0, 64, 0, 0, 0, 1921},
{0, 63, 0, 0, 0, 1921},
{65498, 0, 65498, 0, 0, 1801},
{65499, 0, 65499, 0, 0, 1801},
{65505, 0, 65505, 0, 0, 1801},
{65472, 0, 65472, 0, 0, 1801},
{65473, 0, 65473, 0, 0, 1801},
{65474, 0, 65474, 0, 0, 1801},
{65479, 0, 65479, 0, 0, 1801},
{65489, 0, 65489, 0, 0, 1801},
{65482, 0, 65482, 0, 0, 1801},
{65450, 0, 65450, 0, 0, 1801},
{65456, 0, 65456, 0, 0, 1801},
{7, 0, 7, 0, 0, 1801},
{0, 65476, 0, 0, 0, 1921},
{65440, 0, 65440, 0, 0, 1801},
{0, 65529, 0, 0, 0, 1921},
{0, 80, 0, 0, 0, 1921},
{0, 48, 0, 0, 0, 1921},
{65488, 0, 65488, 0, 0, 1801},
{0, 0, 0, 0, 0, 1537},
{0, 7264, 0, 0, 0, 1921},
{0, 0, 0, 0, 1, 1540},
{0, 0, 0, 0, 2, 1540},
{0, 0, 0, 0, 3, 1540},
{0, 0, 0, 0, 4, 1540},
{0, 0, 0, 0, 5, 1540},
{0, 0, 0, 0, 6, 1540},
{0, 0, 0, 0, 7, 1540},
{0, 0, 0, 0, 8, 1540},
{0, 0, 0, 0, 9, 1540},
{0, 0, 0, 0, 0, 1792},
{65477, 0, 65477, 0, 0, 1801},
{8, 0, 8, 0, 0, 1801},
{0, 65528, 0, 0, 0, 1921},
{74, 0, 74, 0, 0, 1801},
{86, 0, 86, 0, 0, 1801},
{100, 0, 100, 0, 0, 1801},
{128, 0, 128, 0, 0, 1801},
{112, 0, 112, 0, 0, 1801},
{126, 0, 126, 0, 0, 1801},
{0, 65528, 0, 0, 0, 1857},
{9, 0, 9, 0, 0, 1801},
{0, 65462, 0, 0, 0, 1921},
{0, 65527, 0, 0, 0, 1857},
{58331, 0, 58331, 0, 0, 1801},
{0, 65450, 0, 0, 0, 1921},
{0, 65436, 0, 0, 0, 1921},
{0, 65424, 0, 0, 0, 1921},
{0, 65408, 0, 0, 0, 1921},
{0, 65410, 0, 0, 0, 1921},
{0, 0, 0, 0, 0, 1028},
{0, 0, 0, 0, 4, 1028},
{0, 0, 0, 0, 5, 1028},
{0, 0, 0, 0, 6, 1028},
{0, 0, 0, 0, 7, 1028},
{0, 0, 0, 0, 8, 1028},
{0, 0, 0, 0, 9, 1028},
{0, 58019, 0, 0, 0, 1921},
{0, 57153, 0, 0, 0, 1921},
{0, 57274, 0, 0, 0, 1921},
{0, 16, 0, 0, 0, 1792},
{65520, 0, 65520, 0, 0, 1792},
{0, 26, 0, 0, 0, 1024},
{65510, 0, 65510, 0, 0, 1024},
{58272, 0, 58272, 0, 0, 1801},
{0, 40, 0, 0, 0, 1921},
{65496, 0, 65496, 0, 0, 1801},
};
/* type indexes */
......@@ -1323,6 +1323,3 @@ static unsigned char index2[] = {
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0,
};
Tools/unicode/makeunicodedata.py
View file @ eac71820
......@@ -20,7 +20,7 @@
# 2002-11-25 mvl add UNIDATA_VERSION
# 2004-05-29 perky add east asian width information
# 2006-03-10 mvl update to Unicode 4.1; add UCD 3.2 delta
# 2008-06-11 gb add NONPRINTABLE_MASK for Atsuo Ishimoto's ascii() patch
# 2008-06-11 gb add PRINTABLE_MASK for Atsuo Ishimoto's ascii() patch
#
# written by Fredrik Lundh (fredrik@pythonware.com)
#
......@@ -61,7 +61,7 @@ TITLE_MASK = 0x40
UPPER_MASK = 0x80
XID_START_MASK = 0x100
XID_CONTINUE_MASK = 0x200
NONPRINTABLE_MASK = 0x400
PRINTABLE_MASK = 0x400
def maketables(trace=0):
......@@ -373,10 +373,8 @@ def makeunicodetype(unicode, trace):
flags |= TITLE_MASK
if category == "Lu":
flags |= UPPER_MASK
if category[0] == "C":
flags |= NONPRINTABLE_MASK
if category[0] == "Z" and char != " ":
flags |= NONPRINTABLE_MASK
if char == " " or category[0] not in ("C", "Z"):
flags |= PRINTABLE_MASK
if "XID_Start" in properties:
flags |= XID_START_MASK
if "XID_Continue" in properties:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7