Skip to content

C
cpython
Commit 44531cb2 authored by Alexandre Vassalotti's avatar Alexandre Vassalotti
Browse files
Options

Optimize built-in unicode codecs by avoiding unnecessary copying. 

The approach used is similiar to what is currently used in the version
of unicodeobject.c in Python 2.x. The only difference is we use
_PyBytes_Resize instead of _PyString_Resize.
parent 9cb6f7f7
Hide whitespace changes
Inline Side-by-side
Showing with 58 additions and 50 deletions
Objects/unicodeobject.c
View file @ 44531cb2
......@@ -1873,7 +1873,7 @@ PyObject *PyUnicode_EncodeUTF7(const Py_UNICODE *s,
int encodeWhiteSpace,
const char *errors)
{
PyObject *v, *result;
PyObject *v;
/* It might be possible to tighten this worst case */
Py_ssize_t cbAllocated = 5 * size;
int inShift = 0;
......@@ -1889,11 +1889,11 @@ PyObject *PyUnicode_EncodeUTF7(const Py_UNICODE *s,
if (cbAllocated / 5 != size)
return PyErr_NoMemory();
v = PyByteArray_FromStringAndSize(NULL, cbAllocated);
v = PyBytes_FromStringAndSize(NULL, cbAllocated);
if (v == NULL)
return NULL;
start = out = PyByteArray_AS_STRING(v);
start = out = PyBytes_AS_STRING(v);
for (;i < size; ++i) {
Py_UNICODE ch = s[i];
......@@ -1958,10 +1958,9 @@ PyObject *PyUnicode_EncodeUTF7(const Py_UNICODE *s,
*out++= B64(charsleft << (6-bitsleft) );
*out++ = '-';
}
result = PyBytes_FromStringAndSize(PyByteArray_AS_STRING(v), out - start);
Py_DECREF(v);
return result;
if (_PyBytes_Resize(&v, out - start) < 0)
return NULL;
return v;
}
#undef SPECIAL
......@@ -2479,7 +2478,7 @@ PyUnicode_EncodeUTF32(const Py_UNICODE *s,
const char *errors,
int byteorder)
{
PyObject *v, *result;
PyObject *v;
unsigned char *p;
Py_ssize_t nsize, bytesize;
#ifndef Py_UNICODE_WIDE
......@@ -2515,11 +2514,11 @@ PyUnicode_EncodeUTF32(const Py_UNICODE *s,
bytesize = nsize * 4;
if (bytesize / 4 != nsize)
return PyErr_NoMemory();
v = PyByteArray_FromStringAndSize(NULL, bytesize);
v = PyBytes_FromStringAndSize(NULL, bytesize);
if (v == NULL)
return NULL;
p = (unsigned char *)PyByteArray_AS_STRING(v);
p = (unsigned char *)PyBytes_AS_STRING(v);
if (byteorder == 0)
STORECHAR(0xFEFF);
if (size == 0)
......@@ -2556,9 +2555,7 @@ PyUnicode_EncodeUTF32(const Py_UNICODE *s,
}
done:
result = PyBytes_FromStringAndSize(PyByteArray_AS_STRING(v), Py_SIZE(v));
Py_DECREF(v);
return result;
return v;
#undef STORECHAR
}
......@@ -2757,7 +2754,7 @@ PyUnicode_EncodeUTF16(const Py_UNICODE *s,
const char *errors,
int byteorder)
{
PyObject *v, *result;
PyObject *v;
unsigned char *p;
Py_ssize_t nsize, bytesize;
#ifdef Py_UNICODE_WIDE
......@@ -2792,11 +2789,11 @@ PyUnicode_EncodeUTF16(const Py_UNICODE *s,
bytesize = nsize * 2;
if (bytesize / 2 != nsize)
return PyErr_NoMemory();
v = PyByteArray_FromStringAndSize(NULL, bytesize);
v = PyBytes_FromStringAndSize(NULL, bytesize);
if (v == NULL)
return NULL;
p = (unsigned char *)PyByteArray_AS_STRING(v);
p = (unsigned char *)PyBytes_AS_STRING(v);
if (byteorder == 0)
STORECHAR(0xFEFF);
if (size == 0)
......@@ -2828,9 +2825,7 @@ PyUnicode_EncodeUTF16(const Py_UNICODE *s,
}
done:
result = PyBytes_FromStringAndSize(PyByteArray_AS_STRING(v), Py_SIZE(v));
Py_DECREF(v);
return result;
return v;
#undef STORECHAR
}
......@@ -3120,7 +3115,7 @@ static const char *hexdigits = "0123456789abcdef";
PyObject *PyUnicode_EncodeUnicodeEscape(const Py_UNICODE *s,
Py_ssize_t size)
{
PyObject *repr, *result;
PyObject *repr;
char *p;
#ifdef Py_UNICODE_WIDE
......@@ -3147,17 +3142,20 @@ PyObject *PyUnicode_EncodeUnicodeEscape(const Py_UNICODE *s,
escape.
*/
if (size == 0)
return PyBytes_FromStringAndSize(NULL, 0);
if (size > (PY_SSIZE_T_MAX - 2 - 1) / expandsize)
return PyErr_NoMemory();
repr = PyByteArray_FromStringAndSize(NULL,
repr = PyBytes_FromStringAndSize(NULL,
2
+ expandsize*size
+ 1);
if (repr == NULL)
return NULL;
p = PyByteArray_AS_STRING(repr);
p = PyBytes_AS_STRING(repr);
while (size-- > 0) {
Py_UNICODE ch = *s++;
......@@ -3249,13 +3247,13 @@ PyObject *PyUnicode_EncodeUnicodeEscape(const Py_UNICODE *s,
*p++ = (char) ch;
}
result = PyBytes_FromStringAndSize(PyByteArray_AS_STRING(repr),
p - PyByteArray_AS_STRING(repr));
Py_DECREF(repr);
return result;
assert(p - PyBytes_AS_STRING(repr) > 0);
if (_PyBytes_Resize(&repr, p - PyBytes_AS_STRING(repr)) < 0)
return NULL;
return repr;
}
PyObject *PyUnicode_AsUnicodeEscapeString(PyObject *unicode)
PyObject *PyUnicodeAsUnicodeEscapeString(PyObject *unicode)
{
PyObject *s;
if (!PyUnicode_Check(unicode)) {
......@@ -3389,7 +3387,7 @@ PyObject *PyUnicode_DecodeRawUnicodeEscape(const char *s,
PyObject *PyUnicode_EncodeRawUnicodeEscape(const Py_UNICODE *s,
Py_ssize_t size)
{
PyObject *repr, *result;
PyObject *repr;
char *p;
char *q;
......@@ -3402,13 +3400,13 @@ PyObject *PyUnicode_EncodeRawUnicodeEscape(const Py_UNICODE *s,
if (size > PY_SSIZE_T_MAX / expandsize)
return PyErr_NoMemory();
repr = PyByteArray_FromStringAndSize(NULL, expandsize * size);
repr = PyBytes_FromStringAndSize(NULL, expandsize * size);
if (repr == NULL)
return NULL;
if (size == 0)
goto done;
return repr;
p = q = PyByteArray_AS_STRING(repr);
p = q = PyBytes_AS_STRING(repr);
while (size-- > 0) {
Py_UNICODE ch = *s++;
#ifdef Py_UNICODE_WIDE
......@@ -3468,10 +3466,10 @@ PyObject *PyUnicode_EncodeRawUnicodeEscape(const Py_UNICODE *s,
}
size = p - q;
done:
result = PyBytes_FromStringAndSize(PyByteArray_AS_STRING(repr), size);
Py_DECREF(repr);
return result;
assert(size > 0);
if (_PyBytes_Resize(&repr, size) < 0)
return NULL;
return repr;
}
PyObject *PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode)
......@@ -3706,7 +3704,6 @@ static PyObject *unicode_encode_ucs1(const Py_UNICODE *p,
const char *reason = (limit == 256) ? "ordinal not in range(256)" : "ordinal not in range(128)";
PyObject *errorHandler = NULL;
PyObject *exc = NULL;
PyObject *result = NULL;
/* the following variable is used for caching string comparisons
* -1=not initialized, 0=unknown, 1=strict, 2=replace, 3=ignore, 4=xmlcharrefreplace */
int known_errorHandler = -1;
......@@ -3715,10 +3712,10 @@ static PyObject *unicode_encode_ucs1(const Py_UNICODE *p,
replacements, if we need more, we'll resize */
if (size == 0)
return PyBytes_FromStringAndSize(NULL, 0);
res = PyByteArray_FromStringAndSize(NULL, size);
res = PyBytes_FromStringAndSize(NULL, size);
if (res == NULL)
return NULL;
str = PyByteArray_AS_STRING(res);
str = PyBytes_AS_STRING(res);
ressize = size;
while (p<endp) {
......@@ -3768,7 +3765,7 @@ static PyObject *unicode_encode_ucs1(const Py_UNICODE *p,
p = collend;
break;
case 4: /* xmlcharrefreplace */
respos = str - PyByteArray_AS_STRING(res);
respos = str - PyBytes_AS_STRING(res);
/* determine replacement size (temporarily (mis)uses p) */
for (p = collstart, repsize = 0; p < collend; ++p) {
if (*p<10)
......@@ -3795,9 +3792,9 @@ static PyObject *unicode_encode_ucs1(const Py_UNICODE *p,
if (requiredsize > ressize) {
if (requiredsize<2*ressize)
requiredsize = 2*ressize;
if (PyByteArray_Resize(res, requiredsize))
if (_PyBytes_Resize(&res, requiredsize))
goto onError;
str = PyByteArray_AS_STRING(res) + respos;
str = PyBytes_AS_STRING(res) + respos;
ressize = requiredsize;
}
/* generate replacement (temporarily (mis)uses p) */
......@@ -3815,17 +3812,17 @@ static PyObject *unicode_encode_ucs1(const Py_UNICODE *p,
/* need more space? (at least enough for what we
have+the replacement+the rest of the string, so
we won't have to check space for encodable characters) */
respos = str - PyByteArray_AS_STRING(res);
respos = str - PyBytes_AS_STRING(res);
repsize = PyUnicode_GET_SIZE(repunicode);
requiredsize = respos+repsize+(endp-collend);
if (requiredsize > ressize) {
if (requiredsize<2*ressize)
requiredsize = 2*ressize;
if (PyByteArray_Resize(res, requiredsize)) {
if (_PyBytes_Resize(&res, requiredsize)) {
Py_DECREF(repunicode);
goto onError;
}
str = PyByteArray_AS_STRING(res) + respos;
str = PyBytes_AS_STRING(res) + respos;
ressize = requiredsize;
}
/* check if there is anything unencodable in the replacement
......@@ -3845,13 +3842,23 @@ static PyObject *unicode_encode_ucs1(const Py_UNICODE *p,
}
}
}
result = PyBytes_FromStringAndSize(PyByteArray_AS_STRING(res),
str - PyByteArray_AS_STRING(res));
/* Resize if we allocated to much */
size = str - PyBytes_AS_STRING(res);
if (size < ressize) { /* If this falls res will be NULL */
assert(size > 0);
if (_PyBytes_Resize(&res, size) < 0)
goto onError;
}
Py_XDECREF(errorHandler);
Py_XDECREF(exc);
return res;
onError:
Py_DECREF(res);
Py_XDECREF(res);
Py_XDECREF(errorHandler);
Py_XDECREF(exc);
return result;
return NULL;
}
PyObject *PyUnicode_EncodeLatin1(const Py_UNICODE *p,
......@@ -4104,7 +4111,7 @@ static int encode_mbcs(PyObject **repr,
else {
/* Extend string object */
n = PyBytes_Size(*repr);
if (_PyBytes_Resize(repr, n + mbcssize) < 0)
if (_PyBytes_Resize(&repr, n + mbcssize) < 0)
return -1;
}
......@@ -4834,7 +4841,8 @@ PyObject *PyUnicode_EncodeCharmap(const Py_UNICODE *p,
/* Resize if we allocated to much */
if (respos<PyBytes_GET_SIZE(res))
_PyBytes_Resize(&res, respos);
if (_PyBytes_Resize(&res, respos) < 0)
goto onError;
Py_XDECREF(exc);
Py_XDECREF(errorHandler);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7