Issue #19730: Argument Clinic now supports all the existing PyArg

"format units" as legacy converters, as well as two new features:
"self converters" and the "version" directive.

Include/pyport.h

@@ -188,6 +188,13 @@ typedef Py_ssize_t Py_hash_t;
 #define SIZEOF_PY_UHASH_T SIZEOF_SIZE_T
 typedef size_t Py_uhash_t;
 
+/* Only used for compatibility with code that may not be PY_SSIZE_T_CLEAN. */
+#ifdef PY_SSIZE_T_CLEAN
+typedef Py_ssize_t Py_ssize_clean_t;
+#else
+typedef int Py_ssize_clean_t;
+#endif
+
 /* Largest possible value of size_t.
    SIZE_MAX is part of C99, so it might be defined on some
    platforms. If it is not defined, (size_t)-1 is a portable

Misc/NEWS

@@ -9,7 +9,6 @@ Projected release date: 2013-11-24
 
 Core and Builtins
 -----------------
-
 - Use the repr of a module name in more places in import, especially
   exceptions.
 
@@ -450,6 +449,9 @@ Build
 
 Tools/Demos
 -----------
+- Issue #19730: Argument Clinic now supports all the existing PyArg
+  "format units" as legacy converters, as well as two new features:
+  "self converters" and the "version" directive.
 
 - Issue #19552: pyvenv now bootstraps pip into virtual environments by
   default (pass --without-pip to request the old behaviour)

Modules/_datetimemodule.c

@@ -4167,10 +4167,10 @@ PyDoc_STRVAR(datetime_datetime_now__doc__,
     {"now", (PyCFunction)datetime_datetime_now, METH_VARARGS|METH_KEYWORDS|METH_CLASS, datetime_datetime_now__doc__},
 
 static PyObject *
-datetime_datetime_now_impl(PyObject *cls, PyObject *tz);
+datetime_datetime_now_impl(PyTypeObject *cls, PyObject *tz);
 
 static PyObject *
-datetime_datetime_now(PyObject *cls, PyObject *args, PyObject *kwargs)
+datetime_datetime_now(PyTypeObject *cls, PyObject *args, PyObject *kwargs)
 {
     PyObject *return_value = NULL;
     static char *_keywords[] = {"tz", NULL};
@@ -4187,8 +4187,8 @@ exit:
 }
 
 static PyObject *
-datetime_datetime_now_impl(PyObject *cls, PyObject *tz)
-/*[clinic checksum: cde1daca68c9b7dca6df51759db2de1d43a39774]*/
+datetime_datetime_now_impl(PyTypeObject *cls, PyObject *tz)
+/*[clinic checksum: 5e61647d5d1feaf1ab096c5406ccea17bb7b061c]*/
 {
     PyObject *self;
 
@@ -4198,7 +4198,7 @@ datetime_datetime_now_impl(PyObject *cls, PyObject *tz)
     if (check_tzinfo_subclass(tz) < 0)
         return NULL;
 
-    self = datetime_best_possible(cls,
+    self = datetime_best_possible((PyObject *)cls,
                                   tz == Py_None ? localtime : gmtime,
                                   tz);
     if (self != NULL && tz != Py_None) {

Modules/_dbmmodule.c

@@ -43,6 +43,20 @@ static PyTypeObject Dbmtype;
 
 static PyObject *DbmError;
 
+/*[clinic]
+module dbm
+class dbm.dbm
+[clinic]*/
+/*[clinic checksum: da39a3ee5e6b4b0d3255bfef95601890afd80709]*/
+
+/*[python]
+class dbmobject_converter(self_converter):
+    type = "dbmobject *"
+    def converter_init(self):
+        self.name = 'dp'
+[python]*/
+/*[python checksum: da39a3ee5e6b4b0d3255bfef95601890afd80709]*/
+
 static PyObject *
 newdbmobject(const char *file, int flags, int mode)
 {
@@ -248,27 +262,77 @@ static PySequenceMethods dbm_as_sequence = {
     0,                          /* sq_inplace_repeat */
 };
 
+/*[clinic]
+
+dbm.dbm.get
+
+    self: dbmobject
+
+    key: str(length=True)
+    [
+    default: object
+    ]
+    /
+
+Return the value for key if present, otherwise default.
+[clinic]*/
+
+PyDoc_STRVAR(dbm_dbm_get__doc__,
+"Return the value for key if present, otherwise default.\n"
+"\n"
+"dbm.dbm.get(key, [default])");
+
+#define DBM_DBM_GET_METHODDEF    \
+    {"get", (PyCFunction)dbm_dbm_get, METH_VARARGS, dbm_dbm_get__doc__},
+
+static PyObject *
+dbm_dbm_get_impl(dbmobject *dp, const char *key, Py_ssize_clean_t key_length, int group_right_1, PyObject *default_value);
+
 static PyObject *
-dbm_get(dbmobject *dp, PyObject *args)
+dbm_dbm_get(PyObject *self, PyObject *args)
 {
-    datum key, val;
-    PyObject *defvalue = Py_None;
-    char *tmp_ptr;
-    Py_ssize_t tmp_size;
+    PyObject *return_value = NULL;
+    const char *key;
+    Py_ssize_clean_t key_length;
+    int group_right_1 = 0;
+    PyObject *default_value = NULL;
+
+    switch (PyTuple_Size(args)) {
+        case 1:
+            if (!PyArg_ParseTuple(args, "s#:get", &key, &key_length))
+                return NULL;
+            break;
+        case 2:
+            if (!PyArg_ParseTuple(args, "s#O:get", &key, &key_length, &default_value))
+                return NULL;
+            group_right_1 = 1;
+            break;
+        default:
+            PyErr_SetString(PyExc_TypeError, "dbm.dbm.get requires 1 to 2 arguments");
+            return NULL;
+    }
+    return_value = dbm_dbm_get_impl((dbmobject *)self, key, key_length, group_right_1, default_value);
 
-    if (!PyArg_ParseTuple(args, "s#|O:get",
-                          &tmp_ptr, &tmp_size, &defvalue))
-        return NULL;
-    key.dptr = tmp_ptr;
-    key.dsize = tmp_size;
+    return return_value;
+}
+
+static PyObject *
+dbm_dbm_get_impl(dbmobject *dp, const char *key, Py_ssize_clean_t key_length, int group_right_1, PyObject *default_value)
+/*[clinic checksum: 5b4265e66568f163ef0fc7efec09410eaf793508]*/
+{
+    datum dbm_key, val;
+
+    if (!group_right_1)
+        default_value = Py_None;
+    dbm_key.dptr = (char *)key;
+    dbm_key.dsize = key_length;
     check_dbmobject_open(dp);
-    val = dbm_fetch(dp->di_dbm, key);
+    val = dbm_fetch(dp->di_dbm, dbm_key);
     if (val.dptr != NULL)
         return PyBytes_FromStringAndSize(val.dptr, val.dsize);
-    else {
-        Py_INCREF(defvalue);
-        return defvalue;
-    }
+
+    Py_INCREF(default_value);
+    return default_value;
 }
 
 static PyObject *
@@ -333,9 +397,7 @@ static PyMethodDef dbm_methods[] = {
      "close()\nClose the database."},
     {"keys",            (PyCFunction)dbm_keys,          METH_NOARGS,
      "keys() -> list\nReturn a list of all keys in the database."},
-    {"get",             (PyCFunction)dbm_get,           METH_VARARGS,
-     "get(key[, default]) -> value\n"
-     "Return the value for key if present, otherwise default."},
+    DBM_DBM_GET_METHODDEF
     {"setdefault",      (PyCFunction)dbm_setdefault,    METH_VARARGS,
      "setdefault(key[, default]) -> value\n"
      "Return the value for key if present, otherwise default.  If key\n"
@@ -379,7 +441,6 @@ static PyTypeObject Dbmtype = {
 /* ----------------------------------------------------------------- */
 
 /*[clinic]
-module dbm
 
 dbm.open as dbmopen
 
@@ -415,10 +476,10 @@ PyDoc_STRVAR(dbmopen__doc__,
     {"open", (PyCFunction)dbmopen, METH_VARARGS, dbmopen__doc__},
 
 static PyObject *
-dbmopen_impl(PyObject *module, const char *filename, const char *flags, int mode);
+dbmopen_impl(PyModuleDef *module, const char *filename, const char *flags, int mode);
 
 static PyObject *
-dbmopen(PyObject *module, PyObject *args)
+dbmopen(PyModuleDef *module, PyObject *args)
 {
     PyObject *return_value = NULL;
     const char *filename;
@@ -436,8 +497,8 @@ exit:
 }
 
 static PyObject *
-dbmopen_impl(PyObject *module, const char *filename, const char *flags, int mode)
-/*[clinic checksum: 2b0ec9e3c6ecd19e06d16c9f0ba33848245cb1ab]*/
+dbmopen_impl(PyModuleDef *module, const char *filename, const char *flags, int mode)
+/*[clinic checksum: c1f2036017ec36a43ac6f59893732751e67c19d5]*/
 {
     int iflags;
 

Modules/_weakref.c

@@ -25,10 +25,10 @@ PyDoc_STRVAR(_weakref_getweakrefcount__doc__,
     {"getweakrefcount", (PyCFunction)_weakref_getweakrefcount, METH_O, _weakref_getweakrefcount__doc__},
 
 static Py_ssize_t
-_weakref_getweakrefcount_impl(PyObject *module, PyObject *object);
+_weakref_getweakrefcount_impl(PyModuleDef *module, PyObject *object);
 
 static PyObject *
-_weakref_getweakrefcount(PyObject *module, PyObject *object)
+_weakref_getweakrefcount(PyModuleDef *module, PyObject *object)
 {
     PyObject *return_value = NULL;
     Py_ssize_t _return_value;
@@ -42,8 +42,8 @@ exit:
 }
 
 static Py_ssize_t
-_weakref_getweakrefcount_impl(PyObject *module, PyObject *object)
-/*[clinic checksum: 05cffbc3a4b193a0b7e645da81be281748704f69]*/
+_weakref_getweakrefcount_impl(PyModuleDef *module, PyObject *object)
+/*[clinic checksum: 015113be0c9a0a8672d35df10c63e3642cc23da4]*/
 {
     PyWeakReference **list;
 

Modules/posixmodule.c

@@ -2460,10 +2460,10 @@ PyDoc_STRVAR(os_stat__doc__,
     {"stat", (PyCFunction)os_stat, METH_VARARGS|METH_KEYWORDS, os_stat__doc__},
 
 static PyObject *
-os_stat_impl(PyObject *module, path_t *path, int dir_fd, int follow_symlinks);
+os_stat_impl(PyModuleDef *module, path_t *path, int dir_fd, int follow_symlinks);
 
 static PyObject *
-os_stat(PyObject *module, PyObject *args, PyObject *kwargs)
+os_stat(PyModuleDef *module, PyObject *args, PyObject *kwargs)
 {
     PyObject *return_value = NULL;
     static char *_keywords[] = {"path", "dir_fd", "follow_symlinks", NULL};
@@ -2485,8 +2485,8 @@ exit:
 }
 
 static PyObject *
-os_stat_impl(PyObject *module, path_t *path, int dir_fd, int follow_symlinks)
-/*[clinic checksum: 89390f78327e3f045a81974d758d3996e2a71f68]*/
+os_stat_impl(PyModuleDef *module, path_t *path, int dir_fd, int follow_symlinks)
+/*[clinic checksum: b08112eff0ceab3ec2c72352da95ce73f245d104]*/
 {
     return posix_do_stat("stat", path, dir_fd, follow_symlinks);
 }
@@ -2600,10 +2600,10 @@ PyDoc_STRVAR(os_access__doc__,
     {"access", (PyCFunction)os_access, METH_VARARGS|METH_KEYWORDS, os_access__doc__},
 
 static PyObject *
-os_access_impl(PyObject *module, path_t *path, int mode, int dir_fd, int effective_ids, int follow_symlinks);
+os_access_impl(PyModuleDef *module, path_t *path, int mode, int dir_fd, int effective_ids, int follow_symlinks);
 
 static PyObject *
-os_access(PyObject *module, PyObject *args, PyObject *kwargs)
+os_access(PyModuleDef *module, PyObject *args, PyObject *kwargs)
 {
     PyObject *return_value = NULL;
     static char *_keywords[] = {"path", "mode", "dir_fd", "effective_ids", "follow_symlinks", NULL};
@@ -2627,8 +2627,8 @@ exit:
 }
 
 static PyObject *
-os_access_impl(PyObject *module, path_t *path, int mode, int dir_fd, int effective_ids, int follow_symlinks)
-/*[clinic checksum: aa3e145816a748172e62df8e44af74169c7e1247]*/
+os_access_impl(PyModuleDef *module, path_t *path, int mode, int dir_fd, int effective_ids, int follow_symlinks)
+/*[clinic checksum: b9f8ececb061d31b64220c29526bfee642d1b602]*/
 {
     PyObject *return_value = NULL;
 
@@ -2734,10 +2734,10 @@ PyDoc_STRVAR(os_ttyname__doc__,
     {"ttyname", (PyCFunction)os_ttyname, METH_VARARGS, os_ttyname__doc__},
 
 static char *
-os_ttyname_impl(PyObject *module, int fd);
+os_ttyname_impl(PyModuleDef *module, int fd);
 
 static PyObject *
-os_ttyname(PyObject *module, PyObject *args)
+os_ttyname(PyModuleDef *module, PyObject *args)
 {
     PyObject *return_value = NULL;
     int fd;
@@ -2757,8 +2757,8 @@ exit:
 }
 
 static char *
-os_ttyname_impl(PyObject *module, int fd)
-/*[clinic checksum: c742dd621ec98d0f81d37d264e1d3c89c7a5fb1a]*/
+os_ttyname_impl(PyModuleDef *module, int fd)
+/*[clinic checksum: 61e4e525984cb293f949ccae6ae393c0011dfe8e]*/
 {
     char *ret;
 

Modules/zlibmodule.c

@@ -81,6 +81,13 @@ zlib_error(z_stream zst, int err, char *msg)
         PyErr_Format(ZlibError, "Error %d %s: %.200s", err, msg, zmsg);
 }
 
+/*[clinic]
+module zlib
+class zlib.Compress
+class zlib.Decompress
+[clinic]*/
+/*[clinic checksum: da39a3ee5e6b4b0d3255bfef95601890afd80709]*/
+
 PyDoc_STRVAR(compressobj__doc__,
 "compressobj(level=-1, method=DEFLATED, wbits=15, memlevel=8,\n"
 "            strategy=Z_DEFAULT_STRATEGY[, zdict])\n"
@@ -157,32 +164,86 @@ PyZlib_Free(voidpf ctx, void *ptr)
     PyMem_RawFree(ptr);
 }
 
-PyDoc_STRVAR(compress__doc__,
-"compress(string[, level]) -- Returned compressed string.\n"
+/*[clinic]
+zlib.compress
+    bytes: Py_buffer
+        Binary data to be compressed.
+    [
+    level: int
+        Compression level, in 0-9.
+    ]
+    /
+
+Returns compressed string.
+
+[clinic]*/
+
+PyDoc_STRVAR(zlib_compress__doc__,
+"Returns compressed string.\n"
 "\n"
-"Optional arg level is the compression level, in 0-9.");
+"zlib.compress(bytes, [level])\n"
+"  bytes\n"
+"    Binary data to be compressed.\n"
+"  level\n"
+"    Compression level, in 0-9.");
+
+#define ZLIB_COMPRESS_METHODDEF    \
+    {"compress", (PyCFunction)zlib_compress, METH_VARARGS, zlib_compress__doc__},
 
 static PyObject *
-PyZlib_compress(PyObject *self, PyObject *args)
+zlib_compress_impl(PyModuleDef *module, Py_buffer *bytes, int group_right_1, int level);
+
+static PyObject *
+zlib_compress(PyModuleDef *module, PyObject *args)
+{
+    PyObject *return_value = NULL;
+    Py_buffer bytes;
+    int group_right_1 = 0;
+    int level = 0;
+
+    switch (PyTuple_Size(args)) {
+        case 1:
+            if (!PyArg_ParseTuple(args, "y*:compress", &bytes))
+                return NULL;
+            break;
+        case 2:
+            if (!PyArg_ParseTuple(args, "y*i:compress", &bytes, &level))
+                return NULL;
+            group_right_1 = 1;
+            break;
+        default:
+            PyErr_SetString(PyExc_TypeError, "zlib.compress requires 1 to 2 arguments");
+            return NULL;
+    }
+    return_value = zlib_compress_impl(module, &bytes, group_right_1, level);
+
+    /* Cleanup for bytes */
+    if (bytes.buf)
+       PyBuffer_Release(&bytes);
+
+    return return_value;
+}
+
+static PyObject *
+zlib_compress_impl(PyModuleDef *module, Py_buffer *bytes, int group_right_1, int level)
+/*[clinic checksum: 03e857836db25448d4d572da537eb7faf7695d71]*/
 {
     PyObject *ReturnVal = NULL;
-    Py_buffer pinput;
     Byte *input, *output = NULL;
     unsigned int length;
-    int level=Z_DEFAULT_COMPRESSION, err;
+    int err;
     z_stream zst;
 
-    /* require Python string object, optional 'level' arg */
-    if (!PyArg_ParseTuple(args, "y*|i:compress", &pinput, &level))
-        return NULL;
+    if (!group_right_1)
+        level = Z_DEFAULT_COMPRESSION;
 
-    if ((size_t)pinput.len > UINT_MAX) {
+    if ((size_t)bytes->len > UINT_MAX) {
         PyErr_SetString(PyExc_OverflowError,
                         "Size does not fit in an unsigned int");
         goto error;
     }
-    input = pinput.buf;
-    length = (unsigned int)pinput.len;
+    input = bytes->buf;
+    length = (unsigned int)bytes->len;
 
     zst.avail_out = length + length/1000 + 12 + 1;
 
@@ -239,7 +300,6 @@ PyZlib_compress(PyObject *self, PyObject *args)
         zlib_error(zst, err, "while finishing compression");
 
  error:
-    PyBuffer_Release(&pinput);
     PyMem_Free(output);
 
     return ReturnVal;
@@ -682,10 +742,6 @@ save_unconsumed_input(compobject *self, int err)
 }
 
 /*[clinic]
-
-module zlib
-class zlib.Decompress
-
 zlib.Decompress.decompress
 
     data: Py_buffer
@@ -739,14 +795,15 @@ zlib_Decompress_decompress(PyObject *self, PyObject *args)
 
 exit:
     /* Cleanup for data */
-    PyBuffer_Release(&data);
+    if (data.buf)
+       PyBuffer_Release(&data);
 
     return return_value;
 }
 
 static PyObject *
 zlib_Decompress_decompress_impl(PyObject *self, Py_buffer *data, unsigned int max_length)
-/*[clinic checksum: 76ca9259e3f5ca86bae9da3d0e75637b5d492234]*/
+/*[clinic checksum: f83e91728d327462d7ccbee95299514f26b92253]*/
 {
     compobject *zself = (compobject *)self;
     int err;
@@ -966,8 +1023,6 @@ PyZlib_flush(compobject *self, PyObject *args)
 #ifdef HAVE_ZLIB_COPY
 
 /*[clinic]
-
-class zlib.Compress
 zlib.Compress.copy
 
 Return a copy of the compression object.
@@ -1295,8 +1350,7 @@ static PyMethodDef zlib_methods[] =
 {
     {"adler32", (PyCFunction)PyZlib_adler32, METH_VARARGS,
                 adler32__doc__},
-    {"compress", (PyCFunction)PyZlib_compress,  METH_VARARGS,
-                 compress__doc__},
+    ZLIB_COMPRESS_METHODDEF
     {"compressobj", (PyCFunction)PyZlib_compressobj, METH_VARARGS|METH_KEYWORDS,
                     compressobj__doc__},
     {"crc32", (PyCFunction)PyZlib_crc32, METH_VARARGS,

Objects/unicodeobject.c

@@ -12924,10 +12924,10 @@ PyDoc_STRVAR(unicode_maketrans__doc__,
     {"maketrans", (PyCFunction)unicode_maketrans, METH_VARARGS|METH_STATIC, unicode_maketrans__doc__},
 
 static PyObject *
-unicode_maketrans_impl(PyObject *x, PyObject *y, PyObject *z);
+unicode_maketrans_impl(void *null, PyObject *x, PyObject *y, PyObject *z);
 
 static PyObject *
-unicode_maketrans(PyObject *null, PyObject *args)
+unicode_maketrans(void *null, PyObject *args)
 {
     PyObject *return_value = NULL;
     PyObject *x;
@@ -12938,15 +12938,15 @@ unicode_maketrans(PyObject *null, PyObject *args)
         "O|UU:maketrans",
         &x, &y, &z))
         goto exit;
-    return_value = unicode_maketrans_impl(x, y, z);
+    return_value = unicode_maketrans_impl(null, x, y, z);
 
 exit:
     return return_value;
 }
 
 static PyObject *
-unicode_maketrans_impl(PyObject *x, PyObject *y, PyObject *z)
-/*[clinic checksum: 137db9c3199e7906b7967009f511c24fa3235b5f]*/
+unicode_maketrans_impl(void *null, PyObject *x, PyObject *y, PyObject *z)
+/*[clinic checksum: 6d522e3aea2f2e123da3c5d367132a99d803f9b9]*/
 {
     PyObject *new = NULL, *key, *value;
     Py_ssize_t i = 0;

Tools/clinic/clinic.py

@@ -23,7 +23,6 @@ import sys
 import tempfile
 import textwrap
 
-
 # TODO:
 # converters for
 #
@@ -52,6 +51,8 @@ import textwrap
 #   is too new for us.
 #
 
+version = '1'
+
 _empty = inspect._empty
 _void = inspect._void
 
@@ -195,6 +196,46 @@ def linear_format(s, **kwargs):
 
     return output()[:-1]
 
+def version_splitter(s):
+    """Splits a version string into a tuple of integers.
+
+    The following ASCII characters are allowed, and employ
+    the following conversions:
+        a -> -3
+        b -> -2
+        c -> -1
+    (This permits Python-style version strings such as "1.4b3".)
+    """
+    version = []
+    accumulator = []
+    def flush():
+        if not accumulator:
+            raise ValueError('Malformed version string: ' + repr(s))
+        version.append(int(''.join(accumulator)))
+        accumulator.clear()
+
+    for c in s:
+        if c.isdigit():
+            accumulator.append(c)
+        elif c == '.':
+            flush()
+        elif c in 'abc':
+            flush()
+            version.append('abc'.index(c) - 3)
+        else:
+            raise ValueError('Illegal character ' + repr(c) + ' in version string ' + repr(s))
+    flush()
+    return tuple(version)
+
+def version_comparitor(version1, version2):
+    iterator = itertools.zip_longest(version_splitter(version1), version_splitter(version2), fillvalue=0)
+    for i, (a, b) in enumerate(iterator):
+        if a < b:
+            return -1
+        if a > b:
+            return 1
+    return 0
+
 
 class CRenderData:
     def __init__(self):
@@ -373,22 +414,22 @@ PyDoc_STRVAR({c_basename}__doc__,
 {docstring});
 
 #define {methoddef_name}    \\
-    {{"{name}", (PyCFunction){c_basename}, {meth_flags}, {c_basename}__doc__}},
-""".replace('{meth_flags}', flags)
+    {{"{name}", (PyCFunction){c_basename}, {methoddef_flags}, {c_basename}__doc__}},
+""".replace('{methoddef_flags}', flags)
 
-    def meth_noargs_pyobject_template(self, meth_flags=""):
-        return self.template_base("METH_NOARGS", meth_flags) + """
+    def meth_noargs_pyobject_template(self, methoddef_flags=""):
+        return self.template_base("METH_NOARGS", methoddef_flags) + """
 static PyObject *
-{c_basename}(PyObject *{self_name})
+{c_basename}({self_type}{self_name})
 """
 
-    def meth_noargs_template(self, meth_flags=""):
-        return self.template_base("METH_NOARGS", meth_flags) + """
+    def meth_noargs_template(self, methoddef_flags=""):
+        return self.template_base("METH_NOARGS", methoddef_flags) + """
 static {impl_return_type}
 {impl_prototype};
 
 static PyObject *
-{c_basename}(PyObject *{self_name})
+{c_basename}({self_type}{self_name})
 {{
     PyObject *return_value = NULL;
     {declarations}
@@ -406,14 +447,14 @@ static {impl_return_type}
 {impl_prototype}
 """
 
-    def meth_o_template(self, meth_flags=""):
-        return self.template_base("METH_O", meth_flags) + """
+    def meth_o_template(self, methoddef_flags=""):
+        return self.template_base("METH_O", methoddef_flags) + """
 static PyObject *
 {c_basename}({impl_parameters})
 """
 
-    def meth_o_return_converter_template(self, meth_flags=""):
-        return self.template_base("METH_O", meth_flags) + """
+    def meth_o_return_converter_template(self, methoddef_flags=""):
+        return self.template_base("METH_O", methoddef_flags) + """
 static {impl_return_type}
 {impl_prototype};
 
@@ -435,13 +476,13 @@ static {impl_return_type}
 {impl_prototype}
 """
 
-    def option_group_template(self, meth_flags=""):
-        return self.template_base("METH_VARARGS", meth_flags) + """
+    def option_group_template(self, methoddef_flags=""):
+        return self.template_base("METH_VARARGS", methoddef_flags) + """
 static {impl_return_type}
 {impl_prototype};
 
 static PyObject *
-{c_basename}(PyObject *{self_name}, PyObject *args)
+{c_basename}({self_type}{self_name}, PyObject *args)
 {{
     PyObject *return_value = NULL;
     {declarations}
@@ -460,13 +501,13 @@ static {impl_return_type}
 {impl_prototype}
 """
 
-    def keywords_template(self, meth_flags=""):
-        return self.template_base("METH_VARARGS|METH_KEYWORDS", meth_flags) + """
+    def keywords_template(self, methoddef_flags=""):
+        return self.template_base("METH_VARARGS|METH_KEYWORDS", methoddef_flags) + """
 static {impl_return_type}
 {impl_prototype};
 
 static PyObject *
-{c_basename}(PyObject *{self_name}, PyObject *args, PyObject *kwargs)
+{c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs)
 {{
     PyObject *return_value = NULL;
     static char *_keywords[] = {{{keywords}, NULL}};
@@ -489,13 +530,13 @@ static {impl_return_type}
 {impl_prototype}
 """
 
-    def positional_only_template(self, meth_flags=""):
-        return self.template_base("METH_VARARGS", meth_flags) + """
+    def positional_only_template(self, methoddef_flags=""):
+        return self.template_base("METH_VARARGS", methoddef_flags) + """
 static {impl_return_type}
 {impl_prototype};
 
 static PyObject *
-{c_basename}(PyObject *{self_name}, PyObject *args)
+{c_basename}({self_type}{self_name}, PyObject *args)
 {{
     PyObject *return_value = NULL;
     {declarations}
@@ -614,27 +655,6 @@ static {impl_return_type}
         add, output = text_accumulator()
         data = CRenderData()
 
-        if f.kind == STATIC_METHOD:
-            meth_flags = 'METH_STATIC'
-            self_name = "null"
-        else:
-            if f.kind == CALLABLE:
-                meth_flags = ''
-                self_name = "self" if f.cls else "module"
-            elif f.kind == CLASS_METHOD:
-                meth_flags = 'METH_CLASS'
-                self_name = "cls"
-            else:
-                fail("Unrecognized 'kind' " + repr(f.kind) + " for function " + f.name)
-
-            data.impl_parameters.append("PyObject *" + self_name)
-            data.impl_arguments.append(self_name)
-
-        if f.coexist:
-            if meth_flags:
-                meth_flags += '|'
-            meth_flags += 'METH_COEXIST'
-
         parameters = list(f.parameters.values())
         converters = [p.converter for p in parameters]
 
@@ -654,8 +674,6 @@ static {impl_return_type}
 
         template_dict['docstring'] = self.docstring_for_c_string(f)
 
-        template_dict['self_name'] = self_name
-
         positional = has_option_groups =  False
 
         if parameters:
@@ -680,6 +698,18 @@ static {impl_return_type}
             if has_option_groups:
                 assert positional
 
+        # now insert our "self" (or whatever) parameters
+        # (we deliberately don't call render on self converters)
+        stock_self = self_converter('self', f)
+        template_dict['self_name'] = stock_self.name
+        template_dict['self_type'] = stock_self.type
+        data.impl_parameters.insert(0, f.self_converter.type + ("" if f.self_converter.type.endswith('*') else " ") + f.self_converter.name)
+        if f.self_converter.type != stock_self.type:
+            self_cast = '(' + f.self_converter.type + ')'
+        else:
+            self_cast = ''
+        data.impl_arguments.insert(0, self_cast + stock_self.name)
+
         f.return_converter.render(f, data)
         template_dict['impl_return_type'] = f.return_converter.type
 
@@ -701,25 +731,26 @@ static {impl_return_type}
 
         if not parameters:
             if default_return_converter:
-                template = self.meth_noargs_pyobject_template(meth_flags)
+                template = self.meth_noargs_pyobject_template(f.methoddef_flags)
             else:
-                template = self.meth_noargs_template(meth_flags)
+                template = self.meth_noargs_template(f.methoddef_flags)
         elif (len(parameters) == 1 and
               parameters[0].kind == inspect.Parameter.POSITIONAL_ONLY and
               not converters[0].is_optional() and
               isinstance(converters[0], object_converter) and
               converters[0].format_unit == 'O'):
             if default_return_converter:
-                template = self.meth_o_template(meth_flags)
+                template = self.meth_o_template(f.methoddef_flags)
             else:
                 # HACK
                 # we're using "impl_parameters" for the
                 # non-impl function, because that works
                 # better for METH_O.  but that means we
-                # must surpress actually declaring the
+                # must supress actually declaring the
                 # impl's parameters as variables in the
                 # non-impl.  but since it's METH_O, we
-                # only have one anyway, and it's the first one.
+                # only have one anyway, so
+                # we don't have any problem finding it.
                 declarations_copy = list(data.declarations)
                 before, pyobject, after = declarations_copy[0].partition('PyObject *')
                 assert not before, "hack failed, see comment"
@@ -727,16 +758,16 @@ static {impl_return_type}
                 assert after and after[0].isalpha(), "hack failed, see comment"
                 del declarations_copy[0]
                 template_dict['declarations'] = "\n".join(declarations_copy)
-                template = self.meth_o_return_converter_template(meth_flags)
+                template = self.meth_o_return_converter_template(f.methoddef_flags)
         elif has_option_groups:
             self.render_option_group_parsing(f, template_dict)
-            template = self.option_group_template(meth_flags)
+            template = self.option_group_template(f.methoddef_flags)
             template = linear_format(template,
                 option_group_parsing=template_dict['option_group_parsing'])
         elif positional:
-            template = self.positional_only_template(meth_flags)
+            template = self.positional_only_template(f.methoddef_flags)
         else:
-            template = self.keywords_template(meth_flags)
+            template = self.keywords_template(f.methoddef_flags)
 
         template = linear_format(template,
             declarations=template_dict['declarations'],
@@ -1178,6 +1209,20 @@ class Function:
         self.docstring = docstring or ''
         self.kind = kind
         self.coexist = coexist
+        self.self_converter = None
+
+    @property
+    def methoddef_flags(self):
+        flags = []
+        if self.kind == CLASS_METHOD:
+            flags.append('METH_CLASS')
+        elif self.kind == STATIC_METHOD:
+            flags.append('METH_STATIC')
+        else:
+            assert self.kind == CALLABLE, "unknown kind: " + repr(self.kind)
+        if self.coexist:
+            flags.append('METH_COEXIST')
+        return '|'.join(flags)
 
     def __repr__(self):
         return '<clinic.Function ' + self.name + '>'
@@ -1307,6 +1352,7 @@ class CConverter(metaclass=CConverterAutoRegister):
     # The C converter *function* to be used, if any.
     # (If this is not None, format_unit must be 'O&'.)
     converter = None
+
     encoding = None
     impl_by_reference = False
     parse_by_reference = True
@@ -1354,6 +1400,8 @@ class CConverter(metaclass=CConverterAutoRegister):
         # impl_arguments
         s = ("&" if self.impl_by_reference else "") + name
         data.impl_arguments.append(s)
+        if self.length:
+            data.impl_arguments.append(self.length_name())
 
         # keywords
         data.keywords.append(name)
@@ -1370,12 +1418,20 @@ class CConverter(metaclass=CConverterAutoRegister):
 
         # impl_parameters
         data.impl_parameters.append(self.simple_declaration(by_reference=self.impl_by_reference))
+        if self.length:
+            data.impl_parameters.append("Py_ssize_clean_t " + self.length_name())
 
         # cleanup
         cleanup = self.cleanup()
         if cleanup:
             data.cleanup.append('/* Cleanup for ' + name + ' */\n' + cleanup.rstrip() + "\n")
 
+    def length_name(self):
+        """Computes the name of the associated "length" variable."""
+        if not self.length:
+            return None
+        return ensure_legal_c_identifier(self.name) + "_length"
+
     # Why is this one broken out separately?
     # For "positional-only" function parsing,
     # which generates a bunch of PyArg_ParseTuple calls.
@@ -1388,9 +1444,13 @@ class CConverter(metaclass=CConverterAutoRegister):
         if self.encoding:
             list.append(self.encoding)
 
-        s = ("&" if self.parse_by_reference else "") + ensure_legal_c_identifier(self.name)
+        legal_name = ensure_legal_c_identifier(self.name)
+        s = ("&" if self.parse_by_reference else "") + legal_name
         list.append(s)
 
+        if self.length:
+            list.append("&" + self.length_name())
+
     #
     # All the functions after here are intended as extension points.
     #
@@ -1421,6 +1481,10 @@ class CConverter(metaclass=CConverterAutoRegister):
             declaration.append(" = ")
             declaration.append(default)
         declaration.append(";")
+        if self.length:
+            declaration.append('\nPy_ssize_clean_t ')
+            declaration.append(self.length_name())
+            declaration.append(';')
         return "".join(declaration)
 
     def initialize(self):
@@ -1462,7 +1526,7 @@ class byte_converter(CConverter):
 
     def converter_init(self, *, bitwise=False):
         if bitwise:
-            format_unit = 'B'
+            self.format_unit = 'B'
 
 class short_converter(CConverter):
     type = 'short'
@@ -1478,15 +1542,17 @@ class unsigned_short_converter(CConverter):
         if not bitwise:
             fail("Unsigned shorts must be bitwise (for now).")
 
-@add_legacy_c_converter('C', from_str=True)
+@add_legacy_c_converter('C', types='str')
 class int_converter(CConverter):
     type = 'int'
     format_unit = 'i'
     c_ignored_default = "0"
 
-    def converter_init(self, *, from_str=False):
-        if from_str:
-            format_unit = 'C'
+    def converter_init(self, *, types='int'):
+        if types == 'str':
+            self.format_unit = 'C'
+        elif types != 'int':
+            fail("int_converter: illegal 'types' argument")
 
 class unsigned_int_converter(CConverter):
     type = 'unsigned int'
@@ -1568,18 +1634,66 @@ class object_converter(CConverter):
             self.encoding = type
 
 
-@add_legacy_c_converter('y', from_bytes=True)
+@add_legacy_c_converter('s#', length=True)
+@add_legacy_c_converter('y', type="bytes")
+@add_legacy_c_converter('y#', type="bytes", length=True)
 @add_legacy_c_converter('z', nullable=True)
+@add_legacy_c_converter('z#', nullable=True, length=True)
 class str_converter(CConverter):
     type = 'const char *'
     format_unit = 's'
 
-    def converter_init(self, *, nullable=False, from_bytes=False):
-        if from_bytes:
-            assert not nullable
-            format_unit = 'y'
-        if nullable:
-            format_unit = 'z'
+    def converter_init(self, *, encoding=None, types="str",
+        length=False, nullable=False, zeroes=False):
+
+        types = set(types.strip().split())
+        bytes_type = set(("bytes",))
+        str_type = set(("str",))
+        all_3_type = set(("bytearray",)) | bytes_type | str_type
+        is_bytes = types == bytes_type
+        is_str = types == str_type
+        is_all_3 = types == all_3_type
+
+        self.length = bool(length)
+        format_unit = None
+
+        if encoding:
+            self.encoding = encoding
+
+            if is_str and not (length or zeroes or nullable):
+                format_unit = 'es'
+            elif is_all_3 and not (length or zeroes or nullable):
+                format_unit = 'et'
+            elif is_str and length and zeroes and not nullable:
+                format_unit = 'es#'
+            elif is_all_3 and length and not (nullable or zeroes):
+                format_unit = 'et#'
+
+            if format_unit.endswith('#'):
+                # TODO set pointer to NULL
+                # TODO add cleanup for buffer
+                pass
+
+        else:
+            if zeroes:
+                fail("str_converter: illegal combination of arguments (zeroes is only legal with an encoding)")
+
+            if is_bytes and not (nullable or length):
+                format_unit = 'y'
+            elif is_bytes and length and not nullable:
+                format_unit = 'y#'
+            elif is_str and not (nullable or length):
+                format_unit = 's'
+            elif is_str and length and not nullable:
+                format_unit = 's#'
+            elif is_str and nullable  and not length:
+                format_unit = 'z'
+            elif is_str and nullable and length:
+                format_unit = 'z#'
+
+        if not format_unit:
+            fail("str_converter: illegal combination of arguments")
+        self.format_unit = format_unit
 
 
 class PyBytesObject_converter(CConverter):
@@ -1594,36 +1708,89 @@ class unicode_converter(CConverter):
     type = 'PyObject *'
     format_unit = 'U'
 
+@add_legacy_c_converter('u#', length=True)
 @add_legacy_c_converter('Z', nullable=True)
+@add_legacy_c_converter('Z#', nullable=True, length=True)
 class Py_UNICODE_converter(CConverter):
     type = 'Py_UNICODE *'
     format_unit = 'u'
 
-    def converter_init(self, *, nullable=False):
-        if nullable:
-            format_unit = 'Z'
+    def converter_init(self, *, nullable=False, length=False):
+        format_unit = 'Z' if nullable else 'u'
+        if length:
+            format_unit += '#'
+            self.length = True
+        self.format_unit = format_unit
 
-@add_legacy_c_converter('s*', zeroes=True)
-@add_legacy_c_converter('w*', read_write=True)
-@add_legacy_c_converter('z*', zeroes=True, nullable=True)
+#
+# We define three string conventions for buffer types in the 'types' argument:
+#  'buffer' : any object supporting the buffer interface
+#  'rwbuffer': any object supporting the buffer interface, but must be writeable
+#  'robuffer': any object supporting the buffer interface, but must not be writeable
+#
+@add_legacy_c_converter('s*', types='str bytes bytearray buffer')
+@add_legacy_c_converter('z*', types='str bytes bytearray buffer', nullable=True)
+@add_legacy_c_converter('w*', types='bytearray rwbuffer')
 class Py_buffer_converter(CConverter):
     type = 'Py_buffer'
     format_unit = 'y*'
     impl_by_reference = True
     c_ignored_default = "{NULL, NULL, 0, 0, 0, 0, NULL, NULL, NULL, NULL, NULL}"
 
-    def converter_init(self, *, str=False, zeroes=False, nullable=False, read_write=False):
-        if not str:
-            assert not (zeroes or nullable or read_write)
-        elif read_write:
-            assert not (zeroes or nullable)
-            self.format_unit = 'w*'
+    def converter_init(self, *, types='bytes bytearray buffer', nullable=False):
+        types = set(types.strip().split())
+        bytes_type = set(('bytes',))
+        bytearray_type = set(('bytearray',))
+        buffer_type = set(('buffer',))
+        rwbuffer_type = set(('rwbuffer',))
+        robuffer_type = set(('robuffer',))
+        str_type = set(('str',))
+        bytes_bytearray_buffer_type = bytes_type | bytearray_type | buffer_type
+
+        format_unit = None
+        if types == (str_type | bytes_bytearray_buffer_type):
+            format_unit = 's*' if not nullable else 'z*'
         else:
-            assert zeroes
-            self.format_unit = 'z*' if nullable else 's*'
+            if nullable:
+                fail('Py_buffer_converter: illegal combination of arguments (nullable=True)')
+            elif types == (bytes_bytearray_buffer_type):
+                format_unit = 'y*'
+            elif types == (bytearray_type | rwuffer_type):
+                format_unit = 'w*'
+        if not format_unit:
+            fail("Py_buffer_converter: illegal combination of arguments")
+
+        self.format_unit = format_unit
 
     def cleanup(self):
-        return "PyBuffer_Release(&" + ensure_legal_c_identifier(self.name) + ");\n"
+        name = ensure_legal_c_identifier(self.name)
+        return "".join(["if (", name, ".buf)\n   PyBuffer_Release(&", name, ");\n"])
+
+
+class self_converter(CConverter):
+    """
+    A special-case converter:
+    this is the default converter used for "self".
+    """
+    type = "PyObject *"
+    def converter_init(self):
+        f = self.function
+        if f.kind == CALLABLE:
+            if f.cls:
+                self.name = "self"
+            else:
+                self.name = "module"
+                self.type = "PyModuleDef *"
+        elif f.kind == STATIC_METHOD:
+            self.name = "null"
+            self.type = "void *"
+        elif f.kind == CLASS_METHOD:
+            self.name = "cls"
+            self.type = "PyTypeObject *"
+
+    def render(self, parameter, data):
+        fail("render() should never be called on self_converter instances")
+
 
 
 def add_c_return_converter(f, name=None):
@@ -1830,6 +1997,11 @@ class DSLParser:
         self.kind = CALLABLE
         self.coexist = False
 
+    def directive_version(self, required):
+        global version
+        if version_comparitor(version, required) < 0:
+            fail("Insufficient Clinic version!\n  Version: " + version + "\n  Required: " + required)
+
     def directive_module(self, name):
         fields = name.split('.')
         new = fields.pop()
@@ -1867,6 +2039,7 @@ class DSLParser:
         assert self.coexist == False
         self.coexist = True
 
+
     def parse(self, block):
         self.reset()
         self.block = block
@@ -2128,6 +2301,17 @@ class DSLParser:
             fail('{} is not a valid {}converter'.format(name, legacy_str))
         converter = dict[name](parameter_name, self.function, value, **kwargs)
 
+        # special case: if it's the self converter,
+        # don't actually add it to the parameter list
+        if isinstance(converter, self_converter):
+            if self.function.parameters or (self.parameter_state != self.ps_required):
+                fail("The 'self' parameter, if specified, must be the very first thing in the parameter block.")
+            if self.function.self_converter:
+                fail("You can't specify the 'self' parameter more than once.")
+            self.function.self_converter = converter
+            self.parameter_state = self.ps_start
+            return
+
         kind = inspect.Parameter.KEYWORD_ONLY if self.keyword_only else inspect.Parameter.POSITIONAL_OR_KEYWORD
         p = Parameter(parameter_name, kind, function=self.function, converter=converter, default=value, group=self.group)
         self.function.parameters[parameter_name] = p
@@ -2224,6 +2408,9 @@ class DSLParser:
 
     # the final stanza of the DSL is the docstring.
     def state_function_docstring(self, line):
+        if not self.function.self_converter:
+            self.function.self_converter = self_converter("self", self.function)
+
         if self.group:
             fail("Function " + self.function.name + " has a ] without a matching [.")