[2.7] bpo-37170: Fix the cast on error in PyLong_AsUnsignedLongLongMask() (GH-13860) (GH-13898)

(cherry picked from commit dc247650)
Co-authored-by: Zackery Spytz <zspytz@gmail.com>

Doc/c-api/long.rst

@@ -217,6 +217,9 @@ Long Integer Objects
    Return a C :c:type:`unsigned long` from a Python long integer, without checking
    for overflow.
 
+   Returns ``(unsigned long)-1`` on error.  Use :c:func:`PyErr_Occurred` to
+   disambiguate.
+
    .. versionadded:: 2.3
 
 
@@ -225,6 +228,9 @@ Long Integer Objects
    Return a C :c:type:`unsigned long long` from a Python long integer, without
    checking for overflow.
 
+   Returns ``(unsigned PY_LONG_LONG)-1`` on error.  Use
+   :c:func:`PyErr_Occurred` to disambiguate.
+
    .. versionadded:: 2.3
 
 

Misc/NEWS.d/next/C API/2019-06-06-08-47-04.bpo-37170.hO_fpM.rst

+Fix the cast on error in :c:func:`PyLong_AsUnsignedLongLongMask()`.

Modules/_testcapimodule.c

@@ -888,6 +888,26 @@ test_long_long_and_overflow(PyObject *self)
     return Py_None;
 }
 
+static PyObject *
+test_long_as_unsigned_long_long_mask(PyObject *self)
+{
+    unsigned PY_LONG_LONG res = PyLong_AsUnsignedLongLongMask(NULL);
+
+    if (res != (unsigned PY_LONG_LONG)-1 || !PyErr_Occurred()) {
+        return raiseTestError("test_long_as_unsigned_long_long_mask",
+                              "PyLong_AsUnsignedLongLongMask(NULL) didn't "
+                              "complain");
+    }
+    if (!PyErr_ExceptionMatches(PyExc_SystemError)) {
+        return raiseTestError("test_long_as_unsigned_long_long_mask",
+                              "PyLong_AsUnsignedLongLongMask(NULL) raised "
+                              "something other than SystemError");
+    }
+    PyErr_Clear();
+    Py_INCREF(Py_None);
+    return Py_None;
+}
+
 /* Test the L code for PyArg_ParseTuple.  This should deliver a PY_LONG_LONG
    for both long and int arguments.  The test may leak a little memory if
    it fails.
@@ -2715,6 +2735,8 @@ static PyMethodDef TestMethods[] = {
     {"test_longlong_api",       test_longlong_api,               METH_NOARGS},
     {"test_long_long_and_overflow",
         (PyCFunction)test_long_long_and_overflow, METH_NOARGS},
+    {"test_long_as_unsigned_long_long_mask",
+        (PyCFunction)test_long_as_unsigned_long_long_mask, METH_NOARGS},
     {"test_L_code",             (PyCFunction)test_L_code,        METH_NOARGS},
 #endif
     {"getargs_f",               getargs_f,                       METH_VARARGS},

Objects/longobject.c

@@ -1022,7 +1022,7 @@ PyLong_AsUnsignedLongLongMask(PyObject *vv)
 
     if (vv == NULL || !PyLong_Check(vv)) {
         PyErr_BadInternalCall();
-        return (unsigned long) -1;
+        return (unsigned PY_LONG_LONG) -1;
     }
     v = (PyLongObject *)vv;
     i = Py_SIZE(v);