Skip to content

C
cpython
Commit eac71820 authored by Georg Brandl's avatar Georg Brandl
Browse files
Options

Issue #3282: str.isprintable() should return False for undefined Unicode characters.

parent edda13ca
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 8936 additions and 8817 deletions
Lib/test/test_unicode.py
View file @ eac71820
...@@ -420,12 +420,17 @@ class UnicodeTest( ...@@ -420,12 +420,17 @@ class UnicodeTest(
self.assertFalse(" ".isidentifier()) self.assertFalse(" ".isidentifier())
self.assertFalse("[".isidentifier()) self.assertFalse("[".isidentifier())
self.assertFalse("".isidentifier()) self.assertFalse("".isidentifier())
self.assertFalse("0".isidentifier())
def test_isprintable(self): def test_isprintable(self):
self.assertTrue("".isprintable()) self.assertTrue("".isprintable())
self.assertTrue("abcdefg".isprintable()) self.assertTrue("abcdefg".isprintable())
self.assertFalse("abcdefg\n".isprintable()) self.assertFalse("abcdefg\n".isprintable())
self.assertTrue("\u0370".isprintable()) # some defined Unicode character
self.assertTrue("\u0374".isprintable())
# undefined character
self.assertFalse("\u0370".isprintable())
# single surrogate character
self.assertFalse("\ud800".isprintable()) self.assertFalse("\ud800".isprintable())
def test_contains(self): def test_contains(self):
...@@ -598,7 +603,7 @@ class UnicodeTest( ...@@ -598,7 +603,7 @@ class UnicodeTest(
# format specifiers for user defined type # format specifiers for user defined type
self.assertEqual('{0:abc}'.format(C()), 'abc') self.assertEqual('{0:abc}'.format(C()), 'abc')
# !r, !s and !a coersions # !r, !s and !a coercions
self.assertEqual('{0!s}'.format('Hello'), 'Hello') self.assertEqual('{0!s}'.format('Hello'), 'Hello')
self.assertEqual('{0!s:}'.format('Hello'), 'Hello') self.assertEqual('{0!s:}'.format('Hello'), 'Hello')
self.assertEqual('{0!s:15}'.format('Hello'), 'Hello ') self.assertEqual('{0!s:15}'.format('Hello'), 'Hello ')
...@@ -606,11 +611,15 @@ class UnicodeTest( ...@@ -606,11 +611,15 @@ class UnicodeTest(
self.assertEqual('{0!r}'.format('Hello'), "'Hello'") self.assertEqual('{0!r}'.format('Hello'), "'Hello'")
self.assertEqual('{0!r:}'.format('Hello'), "'Hello'") self.assertEqual('{0!r:}'.format('Hello'), "'Hello'")
self.assertEqual('{0!r}'.format(F('Hello')), 'F(Hello)') self.assertEqual('{0!r}'.format(F('Hello')), 'F(Hello)')
self.assertEqual('{0!r}'.format(F('\u0370')), 'F(\u0370)') self.assertEqual('{0!r}'.format('\u0370'), "'\\u0370'") # nonprintable
self.assertEqual('{0!r}'.format('\u0374'), "'\u0374'") # printable
self.assertEqual('{0!r}'.format(F('\u0374')), 'F(\u0374)')
self.assertEqual('{0!a}'.format('Hello'), "'Hello'") self.assertEqual('{0!a}'.format('Hello'), "'Hello'")
self.assertEqual('{0!a}'.format('\u0370'), "'\\u0370'") # nonprintable
self.assertEqual('{0!a}'.format('\u0374'), "'\\u0374'") # printable
self.assertEqual('{0!a:}'.format('Hello'), "'Hello'") self.assertEqual('{0!a:}'.format('Hello'), "'Hello'")
self.assertEqual('{0!a}'.format(F('Hello')), 'F(Hello)') self.assertEqual('{0!a}'.format(F('Hello')), 'F(Hello)')
self.assertEqual('{0!a}'.format(F('\u0370')), 'F(\\u0370)') self.assertEqual('{0!a}'.format(F('\u0374')), 'F(\\u0374)')
# test fallback to object.__format__ # test fallback to object.__format__
self.assertEqual('{0}'.format({}), '{}') self.assertEqual('{0}'.format({}), '{}')
......
Misc/NEWS
View file @ eac71820
...@@ -12,6 +12,9 @@ What's new in Python 3.0b2? ...@@ -12,6 +12,9 @@ What's new in Python 3.0b2?
Core and Builtins Core and Builtins
----------------- -----------------
- Issue #3282: str.isprintable() should return False for undefined
Unicode characters.
- Issue #3236: Return small longs from PyLong_FromString. - Issue #3236: Return small longs from PyLong_FromString.
Library Library
......
Modules/unicodename_db.h
View file @ eac71820
This diff is collapsed.
Objects/unicodectype.c
View file @ eac71820
...@@ -21,7 +21,7 @@ ...@@ -21,7 +21,7 @@
#define UPPER_MASK 0x80 #define UPPER_MASK 0x80
#define XID_START_MASK 0x100 #define XID_START_MASK 0x100
#define XID_CONTINUE_MASK 0x200 #define XID_CONTINUE_MASK 0x200
#define NONPRINTABLE_MASK 0x400 #define PRINTABLE_MASK 0x400
typedef struct { typedef struct {
const Py_UNICODE upper; const Py_UNICODE upper;
...@@ -693,7 +693,7 @@ int _PyUnicode_IsPrintable(Py_UNICODE ch) ...@@ -693,7 +693,7 @@ int _PyUnicode_IsPrintable(Py_UNICODE ch)
{ {
const _PyUnicode_TypeRecord *ctype = gettyperecord(ch); const _PyUnicode_TypeRecord *ctype = gettyperecord(ch);
return (ctype->flags & NONPRINTABLE_MASK) == 0; return (ctype->flags & PRINTABLE_MASK) != 0;
} }
#ifndef WANT_WCTYPE_FUNCTIONS #ifndef WANT_WCTYPE_FUNCTIONS
......
Objects/unicodetype_db.h
View file @ eac71820
...@@ -3,150 +3,150 @@ ...@@ -3,150 +3,150 @@
/* a list of unique character type descriptors */ /* a list of unique character type descriptors */
const _PyUnicode_TypeRecord _PyUnicode_TypeRecords[] = { const _PyUnicode_TypeRecord _PyUnicode_TypeRecords[] = {
{0, 0, 0, 0, 0, 0}, {0, 0, 0, 0, 0, 0},
{0, 0, 0, 0, 0, 1024},
{0, 0, 0, 0, 0, 1056},
{0, 0, 0, 0, 0, 1072},
{0, 0, 0, 0, 0, 0}, {0, 0, 0, 0, 0, 0},
{0, 0, 0, 0, 0, 518}, {0, 0, 0, 0, 0, 32},
{0, 0, 0, 1, 1, 518}, {0, 0, 0, 0, 0, 48},
{0, 0, 0, 2, 2, 518}, {0, 0, 0, 0, 0, 1024},
{0, 0, 0, 3, 3, 518}, {0, 0, 0, 0, 0, 1542},
{0, 0, 0, 4, 4, 518}, {0, 0, 0, 1, 1, 1542},
{0, 0, 0, 5, 5, 518}, {0, 0, 0, 2, 2, 1542},
{0, 0, 0, 6, 6, 518}, {0, 0, 0, 3, 3, 1542},
{0, 0, 0, 7, 7, 518}, {0, 0, 0, 4, 4, 1542},
{0, 0, 0, 8, 8, 518}, {0, 0, 0, 5, 5, 1542},
{0, 0, 0, 9, 9, 518}, {0, 0, 0, 6, 6, 1542},
{0, 32, 0, 0, 0, 897}, {0, 0, 0, 7, 7, 1542},
{0, 0, 0, 0, 0, 512}, {0, 0, 0, 8, 8, 1542},
{65504, 0, 65504, 0, 0, 777}, {0, 0, 0, 9, 9, 1542},
{0, 0, 0, 0, 0, 777}, {0, 32, 0, 0, 0, 1921},
{0, 0, 0, 0, 2, 4}, {0, 0, 0, 0, 0, 1536},
{0, 0, 0, 0, 3, 4}, {65504, 0, 65504, 0, 0, 1801},
{743, 0, 743, 0, 0, 777}, {0, 0, 0, 0, 0, 1801},
{0, 0, 0, 0, 1, 4}, {0, 0, 0, 0, 2, 1028},
{121, 0, 121, 0, 0, 777}, {0, 0, 0, 0, 3, 1028},
{0, 1, 0, 0, 0, 897}, {743, 0, 743, 0, 0, 1801},
{65535, 0, 65535, 0, 0, 777}, {0, 0, 0, 0, 1, 1028},
{0, 65337, 0, 0, 0, 897}, {121, 0, 121, 0, 0, 1801},
{65304, 0, 65304, 0, 0, 777}, {0, 1, 0, 0, 0, 1921},
{0, 65415, 0, 0, 0, 897}, {65535, 0, 65535, 0, 0, 1801},
{65236, 0, 65236, 0, 0, 777}, {0, 65337, 0, 0, 0, 1921},
{0, 210, 0, 0, 0, 897}, {65304, 0, 65304, 0, 0, 1801},
{0, 206, 0, 0, 0, 897}, {0, 65415, 0, 0, 0, 1921},
{0, 205, 0, 0, 0, 897}, {65236, 0, 65236, 0, 0, 1801},
{0, 79, 0, 0, 0, 897}, {0, 210, 0, 0, 0, 1921},
{0, 202, 0, 0, 0, 897}, {0, 206, 0, 0, 0, 1921},
{0, 203, 0, 0, 0, 897}, {0, 205, 0, 0, 0, 1921},
{0, 207, 0, 0, 0, 897}, {0, 79, 0, 0, 0, 1921},
{97, 0, 97, 0, 0, 777}, {0, 202, 0, 0, 0, 1921},
{0, 211, 0, 0, 0, 897}, {0, 203, 0, 0, 0, 1921},
{0, 209, 0, 0, 0, 897}, {0, 207, 0, 0, 0, 1921},
{163, 0, 163, 0, 0, 777}, {97, 0, 97, 0, 0, 1801},
{0, 213, 0, 0, 0, 897}, {0, 211, 0, 0, 0, 1921},
{130, 0, 130, 0, 0, 777}, {0, 209, 0, 0, 0, 1921},
{0, 214, 0, 0, 0, 897}, {163, 0, 163, 0, 0, 1801},
{0, 218, 0, 0, 0, 897}, {0, 213, 0, 0, 0, 1921},
{0, 217, 0, 0, 0, 897}, {130, 0, 130, 0, 0, 1801},
{0, 219, 0, 0, 0, 897}, {0, 214, 0, 0, 0, 1921},
{0, 0, 0, 0, 0, 769}, {0, 218, 0, 0, 0, 1921},
{56, 0, 56, 0, 0, 777}, {0, 217, 0, 0, 0, 1921},
{0, 2, 1, 0, 0, 897}, {0, 219, 0, 0, 0, 1921},
{65535, 1, 0, 0, 0, 833}, {0, 0, 0, 0, 0, 1793},
{65534, 0, 65535, 0, 0, 777}, {56, 0, 56, 0, 0, 1801},
{65457, 0, 65457, 0, 0, 777}, {0, 2, 1, 0, 0, 1921},
{0, 65439, 0, 0, 0, 897}, {65535, 1, 0, 0, 0, 1857},
{0, 65480, 0, 0, 0, 897}, {65534, 0, 65535, 0, 0, 1801},
{0, 65406, 0, 0, 0, 897}, {65457, 0, 65457, 0, 0, 1801},
{0, 0, 0, 0, 0, 897}, {0, 65439, 0, 0, 0, 1921},
{0, 65373, 0, 0, 0, 897}, {0, 65480, 0, 0, 0, 1921},
{0, 83, 0, 0, 0, 897}, {0, 65406, 0, 0, 0, 1921},
{65326, 0, 65326, 0, 0, 777}, {0, 0, 0, 0, 0, 1921},
{65330, 0, 65330, 0, 0, 777}, {0, 65373, 0, 0, 0, 1921},
{65331, 0, 65331, 0, 0, 777}, {0, 83, 0, 0, 0, 1921},
{65334, 0, 65334, 0, 0, 777}, {65326, 0, 65326, 0, 0, 1801},
{65333, 0, 65333, 0, 0, 777}, {65330, 0, 65330, 0, 0, 1801},
{65329, 0, 65329, 0, 0, 777}, {65331, 0, 65331, 0, 0, 1801},
{65327, 0, 65327, 0, 0, 777}, {65334, 0, 65334, 0, 0, 1801},
{65325, 0, 65325, 0, 0, 777}, {65333, 0, 65333, 0, 0, 1801},
{65323, 0, 65323, 0, 0, 777}, {65329, 0, 65329, 0, 0, 1801},
{65322, 0, 65322, 0, 0, 777}, {65327, 0, 65327, 0, 0, 1801},
{65318, 0, 65318, 0, 0, 777}, {65325, 0, 65325, 0, 0, 1801},
{65319, 0, 65319, 0, 0, 777}, {65323, 0, 65323, 0, 0, 1801},
{65317, 0, 65317, 0, 0, 777}, {65322, 0, 65322, 0, 0, 1801},
{65453, 0, 65453, 0, 0, 777}, {65318, 0, 65318, 0, 0, 1801},
{84, 0, 84, 0, 0, 512}, {65319, 0, 65319, 0, 0, 1801},
{0, 0, 0, 0, 0, 1}, {65317, 0, 65317, 0, 0, 1801},
{0, 38, 0, 0, 0, 897}, {65453, 0, 65453, 0, 0, 1801},
{0, 37, 0, 0, 0, 897}, {84, 0, 84, 0, 0, 1536},
{0, 64, 0, 0, 0, 897}, {0, 0, 0, 0, 0, 1025},
{0, 63, 0, 0, 0, 897}, {0, 38, 0, 0, 0, 1921},
{65498, 0, 65498, 0, 0, 777}, {0, 37, 0, 0, 0, 1921},
{65499, 0, 65499, 0, 0, 777}, {0, 64, 0, 0, 0, 1921},
{65505, 0, 65505, 0, 0, 777}, {0, 63, 0, 0, 0, 1921},
{65472, 0, 65472, 0, 0, 777}, {65498, 0, 65498, 0, 0, 1801},
{65473, 0, 65473, 0, 0, 777}, {65499, 0, 65499, 0, 0, 1801},
{65474, 0, 65474, 0, 0, 777}, {65505, 0, 65505, 0, 0, 1801},
{65479, 0, 65479, 0, 0, 777}, {65472, 0, 65472, 0, 0, 1801},
{65489, 0, 65489, 0, 0, 777}, {65473, 0, 65473, 0, 0, 1801},
{65482, 0, 65482, 0, 0, 777}, {65474, 0, 65474, 0, 0, 1801},
{65450, 0, 65450, 0, 0, 777}, {65479, 0, 65479, 0, 0, 1801},
{65456, 0, 65456, 0, 0, 777}, {65489, 0, 65489, 0, 0, 1801},
{7, 0, 7, 0, 0, 777}, {65482, 0, 65482, 0, 0, 1801},
{0, 65476, 0, 0, 0, 897}, {65450, 0, 65450, 0, 0, 1801},
{65440, 0, 65440, 0, 0, 777}, {65456, 0, 65456, 0, 0, 1801},
{0, 65529, 0, 0, 0, 897}, {7, 0, 7, 0, 0, 1801},
{0, 80, 0, 0, 0, 897}, {0, 65476, 0, 0, 0, 1921},
{0, 48, 0, 0, 0, 897}, {65440, 0, 65440, 0, 0, 1801},
{65488, 0, 65488, 0, 0, 777}, {0, 65529, 0, 0, 0, 1921},
{0, 0, 0, 0, 0, 513}, {0, 80, 0, 0, 0, 1921},
{0, 7264, 0, 0, 0, 897}, {0, 48, 0, 0, 0, 1921},
{0, 0, 0, 0, 1, 516}, {65488, 0, 65488, 0, 0, 1801},
{0, 0, 0, 0, 2, 516}, {0, 0, 0, 0, 0, 1537},
{0, 0, 0, 0, 3, 516}, {0, 7264, 0, 0, 0, 1921},
{0, 0, 0, 0, 4, 516}, {0, 0, 0, 0, 1, 1540},
{0, 0, 0, 0, 5, 516}, {0, 0, 0, 0, 2, 1540},
{0, 0, 0, 0, 6, 516}, {0, 0, 0, 0, 3, 1540},
{0, 0, 0, 0, 7, 516}, {0, 0, 0, 0, 4, 1540},
{0, 0, 0, 0, 8, 516}, {0, 0, 0, 0, 5, 1540},
{0, 0, 0, 0, 9, 516}, {0, 0, 0, 0, 6, 1540},
{0, 0, 0, 0, 0, 768}, {0, 0, 0, 0, 7, 1540},
{65477, 0, 65477, 0, 0, 777}, {0, 0, 0, 0, 8, 1540},
{8, 0, 8, 0, 0, 777}, {0, 0, 0, 0, 9, 1540},
{0, 65528, 0, 0, 0, 897}, {0, 0, 0, 0, 0, 1792},
{74, 0, 74, 0, 0, 777}, {65477, 0, 65477, 0, 0, 1801},
{86, 0, 86, 0, 0, 777}, {8, 0, 8, 0, 0, 1801},
{100, 0, 100, 0, 0, 777}, {0, 65528, 0, 0, 0, 1921},
{128, 0, 128, 0, 0, 777}, {74, 0, 74, 0, 0, 1801},
{112, 0, 112, 0, 0, 777}, {86, 0, 86, 0, 0, 1801},
{126, 0, 126, 0, 0, 777}, {100, 0, 100, 0, 0, 1801},
{0, 65528, 0, 0, 0, 833}, {128, 0, 128, 0, 0, 1801},
{9, 0, 9, 0, 0, 777}, {112, 0, 112, 0, 0, 1801},
{0, 65462, 0, 0, 0, 897}, {126, 0, 126, 0, 0, 1801},
{0, 65527, 0, 0, 0, 833}, {0, 65528, 0, 0, 0, 1857},
{58331, 0, 58331, 0, 0, 777}, {9, 0, 9, 0, 0, 1801},
{0, 65450, 0, 0, 0, 897}, {0, 65462, 0, 0, 0, 1921},
{0, 65436, 0, 0, 0, 897}, {0, 65527, 0, 0, 0, 1857},
{0, 65424, 0, 0, 0, 897}, {58331, 0, 58331, 0, 0, 1801},
{0, 65408, 0, 0, 0, 897}, {0, 65450, 0, 0, 0, 1921},
{0, 65410, 0, 0, 0, 897}, {0, 65436, 0, 0, 0, 1921},
{0, 0, 0, 0, 0, 4}, {0, 65424, 0, 0, 0, 1921},
{0, 0, 0, 0, 4, 4}, {0, 65408, 0, 0, 0, 1921},
{0, 0, 0, 0, 5, 4}, {0, 65410, 0, 0, 0, 1921},
{0, 0, 0, 0, 6, 4}, {0, 0, 0, 0, 0, 1028},
{0, 0, 0, 0, 7, 4}, {0, 0, 0, 0, 4, 1028},
{0, 0, 0, 0, 8, 4}, {0, 0, 0, 0, 5, 1028},
{0, 0, 0, 0, 9, 4}, {0, 0, 0, 0, 6, 1028},
{0, 58019, 0, 0, 0, 897}, {0, 0, 0, 0, 7, 1028},
{0, 57153, 0, 0, 0, 897}, {0, 0, 0, 0, 8, 1028},
{0, 57274, 0, 0, 0, 897}, {0, 0, 0, 0, 9, 1028},
{0, 16, 0, 0, 0, 768}, {0, 58019, 0, 0, 0, 1921},
{65520, 0, 65520, 0, 0, 768}, {0, 57153, 0, 0, 0, 1921},
{0, 26, 0, 0, 0, 0}, {0, 57274, 0, 0, 0, 1921},
{65510, 0, 65510, 0, 0, 0}, {0, 16, 0, 0, 0, 1792},
{58272, 0, 58272, 0, 0, 777}, {65520, 0, 65520, 0, 0, 1792},
{0, 40, 0, 0, 0, 897}, {0, 26, 0, 0, 0, 1024},
{65496, 0, 65496, 0, 0, 777}, {65510, 0, 65510, 0, 0, 1024},
{58272, 0, 58272, 0, 0, 1801},
{0, 40, 0, 0, 0, 1921},
{65496, 0, 65496, 0, 0, 1801},
}; };
/* type indexes */ /* type indexes */
...@@ -1323,6 +1323,3 @@ static unsigned char index2[] = { ...@@ -1323,6 +1323,3 @@ static unsigned char index2[] = {
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0,
}; };
Tools/unicode/makeunicodedata.py
View file @ eac71820
...@@ -20,7 +20,7 @@ ...@@ -20,7 +20,7 @@
# 2002-11-25 mvl add UNIDATA_VERSION # 2002-11-25 mvl add UNIDATA_VERSION
# 2004-05-29 perky add east asian width information # 2004-05-29 perky add east asian width information
# 2006-03-10 mvl update to Unicode 4.1; add UCD 3.2 delta # 2006-03-10 mvl update to Unicode 4.1; add UCD 3.2 delta
# 2008-06-11 gb add NONPRINTABLE_MASK for Atsuo Ishimoto's ascii() patch # 2008-06-11 gb add PRINTABLE_MASK for Atsuo Ishimoto's ascii() patch
# #
# written by Fredrik Lundh (fredrik@pythonware.com) # written by Fredrik Lundh (fredrik@pythonware.com)
# #
...@@ -61,7 +61,7 @@ TITLE_MASK = 0x40 ...@@ -61,7 +61,7 @@ TITLE_MASK = 0x40
UPPER_MASK = 0x80 UPPER_MASK = 0x80
XID_START_MASK = 0x100 XID_START_MASK = 0x100
XID_CONTINUE_MASK = 0x200 XID_CONTINUE_MASK = 0x200
NONPRINTABLE_MASK = 0x400 PRINTABLE_MASK = 0x400
def maketables(trace=0): def maketables(trace=0):
...@@ -373,10 +373,8 @@ def makeunicodetype(unicode, trace): ...@@ -373,10 +373,8 @@ def makeunicodetype(unicode, trace):
flags |= TITLE_MASK flags |= TITLE_MASK
if category == "Lu": if category == "Lu":
flags |= UPPER_MASK flags |= UPPER_MASK
if category[0] == "C": if char == " " or category[0] not in ("C", "Z"):
flags |= NONPRINTABLE_MASK flags |= PRINTABLE_MASK
if category[0] == "Z" and char != " ":
flags |= NONPRINTABLE_MASK
if "XID_Start" in properties: if "XID_Start" in properties:
flags |= XID_START_MASK flags |= XID_START_MASK
if "XID_Continue" in properties: if "XID_Continue" in properties:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7