Patch #15916658: implement the __dir__() special function lookup in PyObject_Dir.

Doc/lib/libfuncs.tex

@@ -274,21 +274,34 @@ class C:
 \end{funcdesc}
 
 \begin{funcdesc}{dir}{\optional{object}}
-  Without arguments, return the list of names in the current local
-  symbol table.  With an argument, attempts to return a list of valid
-  attributes for that object.  This information is gleaned from the
-  object's \member{__dict__} attribute, if defined, and from the class
-  or type object.  The list is not necessarily complete.
-  If the object is a module object, the list contains the names of the
-  module's attributes.
-  If the object is a type or class object,
-  the list contains the names of its attributes,
-  and recursively of the attributes of its bases.
-  Otherwise, the list contains the object's attributes' names,
-  the names of its class's attributes,
-  and recursively of the attributes of its class's base classes.
-  The resulting list is sorted alphabetically.
-  For example:
+  Without arguments, return the list of names in the current local scope.  With
+  an argument, attempt to return a list of valid attributes for that object.
+
+  If the object has a method named \method{__dir__()}, this method will be
+  called and must return the list of attributes. This allows objects that
+  implement a custom \function{__getattr__()} or \function{__getattribute__()}
+  function to customize the way \function{dir()} reports their attributes.
+
+  If the object does not provide \method{__dir__()}, the function tries its best
+  to gather information from the object's \member{__dict__} attribute, if
+  defined, and from its type object.  The resulting list is not necessarily
+  complete, and may be inaccurate when the object has a custom
+  \function{__getattr__()}.
+  
+  The default \function{dir()} mechanism behaves differently with different
+  types of objects, as it attempts to produce the most relevant, rather than
+  complete, information:
+  \begin{itemize}
+  \item If the object is a module object, the list contains the names of the
+    module's attributes.
+  \item If the object is a type or class object, the list contains the names of
+    its attributes, and recursively of the attributes of its bases.
+  \item Otherwise, the list contains the object's attributes' names, the names
+    of its class's attributes, and recursively of the attributes of its class's
+    base classes.
+  \end{itemize}
+  
+  The resulting list is sorted alphabetically.  For example:
 
 \begin{verbatim}
 >>> import struct
@@ -296,13 +309,19 @@ class C:
 ['__builtins__', '__doc__', '__name__', 'struct']
 >>> dir(struct)
 ['__doc__', '__name__', 'calcsize', 'error', 'pack', 'unpack']
+>>> class Foo(object):
+...     def __dir__(self):
+...         return ["kan", "ga", "roo"]
+...
+>>> f = Foo()
+>>> dir(f)
+['ga', 'kan', 'roo']
 \end{verbatim}
 
-  \note{Because \function{dir()} is supplied primarily as a convenience
-  for use at an interactive prompt,
-  it tries to supply an interesting set of names more than it tries to
-  supply a rigorously or consistently defined set of names,
-  and its detailed behavior may change across releases.}
+  \note{Because \function{dir()} is supplied primarily as a convenience for use
+    at an interactive prompt, it tries to supply an interesting set of names
+    more than it tries to supply a rigorously or consistently defined set of
+    names, and its detailed behavior may change across releases.}
 \end{funcdesc}
 
 \begin{funcdesc}{divmod}{a, b}

Lib/test/test_builtin.py

@@ -223,12 +223,67 @@ class BuiltinTest(unittest.TestCase):
         self.assertRaises(TypeError, delattr)
 
     def test_dir(self):
-        x = 1
-        self.assert_('x' in dir())
-        import sys
-        self.assert_('modules' in dir(sys))
+        # dir(wrong number of arguments)
         self.assertRaises(TypeError, dir, 42, 42)
 
+        # dir() - local scope
+        local_var = 1
+        self.assert_('local_var' in dir())
+
+        # dir(module)
+        import sys
+        self.assert_('exit' in dir(sys))
+
+        # dir(module_with_invalid__dict__)
+        import types
+        class Foo(types.ModuleType):
+            __dict__ = 8
+        f = Foo("foo")
+        self.assertRaises(TypeError, dir, f)
+
+        # dir(type)
+        self.assert_("strip" in dir(str))
+        self.assert_("__mro__" not in dir(str))
+
+        # dir(obj)
+        class Foo(object):
+            def __init__(self):
+                self.x = 7
+                self.y = 8
+                self.z = 9
+        f = Foo()
+        self.assert_("y" in dir(f))
+
+        # dir(obj_no__dict__)
+        class Foo(object):
+            __slots__ = []
+        f = Foo()
+        self.assert_("__repr__" in dir(f))
+
+        # dir(obj_no__class__with__dict__)
+        # (an ugly trick to cause getattr(f, "__class__") to fail)
+        class Foo(object):
+            __slots__ = ["__class__", "__dict__"]
+            def __init__(self):
+                self.bar = "wow"
+        f = Foo()
+        self.assert_("__repr__" not in dir(f))
+        self.assert_("bar" in dir(f))
+
+        # dir(obj_using __dir__)
+        class Foo(object):
+            def __dir__(self):
+                return ["kan", "ga", "roo"]
+        f = Foo()
+        self.assert_(dir(f) == ["ga", "kan", "roo"])
+
+        # dir(obj__dir__not_list)
+        class Foo(object):
+            def __dir__(self):
+                return 7
+        f = Foo()
+        self.assertRaises(TypeError, dir, f)
+
     def test_divmod(self):
         self.assertEqual(divmod(12, 7), (1, 5))
         self.assertEqual(divmod(-12, 7), (-2, 2))

Misc/NEWS

@@ -28,11 +28,15 @@ TO DO
 Core and Builtins
 -----------------
 
-- Removing indexing/slicing on BaseException.
+- The dir() function has been extended to call the __dir__() method on
+  its argument, if it exists. If not, it will work like before. This allows
+  customizing the output of dir() in the presence of a __getattr__().
 
-- Remove the exceptions module, all the exceptions are already builtin.
+- Removed indexing/slicing on BaseException.
 
-- input() becomes raw_input(): the name input() now implements the
+- Removed the exceptions module, all the exceptions are already builtin.
+
+- input() became raw_input(): the name input() now implements the
   functionality formerly known as raw_input(); the name raw_input()
   is no longer defined.
 

Objects/object.c

@@ -1284,6 +1284,8 @@ PyCallable_Check(PyObject *x)
 	return x->ob_type->tp_call != NULL;
 }
 
+/* ------------------------- PyObject_Dir() helpers ------------------------- */
+
 /* Helper for PyObject_Dir.
    Merge the __dict__ of aclass into dict, and recursively also all
    the __dict__s of aclass's base classes.  The order of merging isn't
@@ -1343,158 +1345,174 @@ merge_class_dict(PyObject* dict, PyObject* aclass)
 	return 0;
 }
 
-/* Helper for PyObject_Dir.
-   If obj has an attr named attrname that's a list, merge its string
-   elements into keys of dict.
-   Return 0 on success, -1 on error.  Errors due to not finding the attr,
-   or the attr not being a list, are suppressed.
-*/
+/* Helper for PyObject_Dir without arguments: returns the local scope. */
+static PyObject *
+_dir_locals()
+{
+	PyObject *locals = PyEval_GetLocals();
 
-static int
-merge_list_attr(PyObject* dict, PyObject* obj, const char *attrname)
+	if (locals == NULL) {
+		PyErr_SetString(PyExc_SystemError, "frame does not exist");
+		return NULL;
+	}
+
+	/* the locals don't need to be DECREF'd */
+	return PyMapping_Keys(locals);
+}
+
+/* Helper for PyObject_Dir of type objects: returns __dict__ and __bases__.
+   We deliberately don't suck up its __class__, as methods belonging to the 
+   metaclass would probably be more confusing than helpful. 
+*/
+static PyObject * 
+_specialized_dir_type(PyObject *obj)
 {
-	PyObject *list;
-	int result = 0;
+	PyObject *result = NULL;
+	PyObject *dict = PyDict_New();
 
-	assert(PyDict_Check(dict));
-	assert(obj);
-	assert(attrname);
+	if (dict != NULL && merge_class_dict(dict, obj) == 0)
+		result = PyDict_Keys(dict);
 
-	list = PyObject_GetAttrString(obj, attrname);
-	if (list == NULL)
-		PyErr_Clear();
+	Py_XDECREF(dict);
+	return result;
+}
 
-	else if (PyList_Check(list)) {
-		int i;
-		for (i = 0; i < PyList_GET_SIZE(list); ++i) {
-			PyObject *item = PyList_GET_ITEM(list, i);
-			if (PyString_Check(item)) {
-				result = PyDict_SetItem(dict, item, Py_None);
-				if (result < 0)
-					break;
-			}
+/* Helper for PyObject_Dir of module objects: returns the module's __dict__. */
+static PyObject *
+_specialized_dir_module(PyObject *obj)
+{
+	PyObject *result = NULL;
+	PyObject *dict = PyObject_GetAttrString(obj, "__dict__");
+
+	if (dict != NULL) {
+		if (PyDict_Check(dict))
+			result = PyDict_Keys(dict);
+		else {
+			PyErr_Format(PyExc_TypeError,
+				     "%.200s.__dict__ is not a dictionary",
+				     PyModule_GetName(obj));
 		}
 	}
 
-	Py_XDECREF(list);
+	Py_XDECREF(dict);
 	return result;
 }
 
-/* Like __builtin__.dir(arg).  See bltinmodule.c's builtin_dir for the
-   docstring, which should be kept in synch with this implementation. */
-
-PyObject *
-PyObject_Dir(PyObject *arg)
+/* Helper for PyObject_Dir of generic objects: returns __dict__, __class__,
+   and recursively up the __class__.__bases__ chain.
+*/
+static PyObject *
+_generic_dir(PyObject *obj)
 {
-	/* Set exactly one of these non-NULL before the end. */
-	PyObject *result = NULL;	/* result list */
-	PyObject *masterdict = NULL;	/* result is masterdict.keys() */
-
-	/* If NULL arg, return the locals. */
-	if (arg == NULL) {
-		PyObject *locals = PyEval_GetLocals();
-		if (locals == NULL)
-			goto error;
-		result = PyMapping_Keys(locals);
-		if (result == NULL)
-			goto error;
+	PyObject *result = NULL;
+	PyObject *dict = NULL;
+	PyObject *itsclass = NULL;
+	
+	/* Get __dict__ (which may or may not be a real dict...) */
+	dict = PyObject_GetAttrString(obj, "__dict__");
+	if (dict == NULL) {
+		PyErr_Clear();
+		dict = PyDict_New();
 	}
-
-	/* Elif this is some form of module, we only want its dict. */
-	else if (PyModule_Check(arg)) {
-		masterdict = PyObject_GetAttrString(arg, "__dict__");
-		if (masterdict == NULL)
-			goto error;
-		if (!PyDict_Check(masterdict)) {
-			PyErr_SetString(PyExc_TypeError,
-					"module.__dict__ is not a dictionary");
-			goto error;
-		}
+	else if (!PyDict_Check(dict)) {
+		Py_DECREF(dict);
+		dict = PyDict_New();
 	}
-
-	/* Elif some form of type or class, grab its dict and its bases.
-	   We deliberately don't suck up its __class__, as methods belonging
-	   to the metaclass would probably be more confusing than helpful. */
-	else if (PyType_Check(arg)) {
-		masterdict = PyDict_New();
-		if (masterdict == NULL)
-			goto error;
-		if (merge_class_dict(masterdict, arg) < 0)
-			goto error;
+	else {
+		/* Copy __dict__ to avoid mutating it. */
+		PyObject *temp = PyDict_Copy(dict);
+		Py_DECREF(dict);
+		dict = temp;
 	}
 
-	/* Else look at its dict, and the attrs reachable from its class. */
+	if (dict == NULL)
+		goto error;
+
+	/* Merge in attrs reachable from its class. */
+	itsclass = PyObject_GetAttrString(obj, "__class__");
+	if (itsclass == NULL)
+		/* XXX(tomer): Perhaps fall back to obj->ob_type if no
+		               __class__ exists? */
+		PyErr_Clear();
 	else {
-		PyObject *itsclass;
-		/* Create a dict to start with.  CAUTION:  Not everything
-		   responding to __dict__ returns a dict! */
-		masterdict = PyObject_GetAttrString(arg, "__dict__");
-		if (masterdict == NULL) {
-			PyErr_Clear();
-			masterdict = PyDict_New();
-		}
-		else if (!PyDict_Check(masterdict)) {
-			Py_DECREF(masterdict);
-			masterdict = PyDict_New();
-		}
-		else {
-			/* The object may have returned a reference to its
-			   dict, so copy it to avoid mutating it. */
-			PyObject *temp = PyDict_Copy(masterdict);
-			Py_DECREF(masterdict);
-			masterdict = temp;
-		}
-		if (masterdict == NULL)
+		if (merge_class_dict(dict, itsclass) != 0)
 			goto error;
+	}
 
-		/* Merge in __members__ and __methods__ (if any).
-		   XXX Would like this to go away someday; for now, it's
-		   XXX needed to get at im_self etc of method objects. */
-		if (merge_list_attr(masterdict, arg, "__members__") < 0)
-			goto error;
-		if (merge_list_attr(masterdict, arg, "__methods__") < 0)
-			goto error;
+	result = PyDict_Keys(dict);
+	/* fall through */
+error:
+	Py_XDECREF(itsclass);
+	Py_XDECREF(dict);
+	return result;
+}
 
-		/* Merge in attrs reachable from its class.
-		   CAUTION:  Not all objects have a __class__ attr. */
-		itsclass = PyObject_GetAttrString(arg, "__class__");
-		if (itsclass == NULL)
-			PyErr_Clear();
-		else {
-			int status = merge_class_dict(masterdict, itsclass);
-			Py_DECREF(itsclass);
-			if (status < 0)
-				goto error;
-		}
-	}
+/* Helper for PyObject_Dir: object introspection.
+   This calls one of the above specialized versions if no __dir__ method
+   exists. */
+static PyObject *
+_dir_object(PyObject *obj)
+{
+	PyObject * result = NULL;
+	PyObject * dirfunc = PyObject_GetAttrString((PyObject*)obj->ob_type,
+						    "__dir__");
 
-	assert((result == NULL) ^ (masterdict == NULL));
-	if (masterdict != NULL) {
-		/* The result comes from its keys. */
-		assert(result == NULL);
-		result = PyDict_Keys(masterdict);
-		if (result == NULL)
-			goto error;
+	assert(obj);
+	if (dirfunc == NULL) {
+		/* use default implementation */
+		PyErr_Clear();
+		if (PyModule_Check(obj))
+			result = _specialized_dir_module(obj);
+		else if (PyType_Check(obj))
+			result = _specialized_dir_type(obj);
+		else
+			result = _generic_dir(obj);
 	}
+	else {
+		/* use __dir__ */
+		result = PyObject_CallFunctionObjArgs(dirfunc, obj, NULL);
+		Py_DECREF(dirfunc);
+		if (result == NULL)
+			return NULL;
 
-	assert(result);
-	if (!PyList_Check(result)) {
-		PyErr_Format(PyExc_TypeError,
-			"Expected keys() to be a list, not '%.200s'",
-			result->ob_type->tp_name);
-		goto error;
+		/* result must be a list */
+		/* XXX(gbrandl): could also check if all items are strings */
+		if (!PyList_Check(result)) {
+			PyErr_Format(PyExc_TypeError,
+				     "__dir__() must return a list, not %.200s",
+				     result->ob_type->tp_name);
+			Py_DECREF(result);
+			result = NULL;
+		}
 	}
-	if (PyList_Sort(result) != 0)
-		goto error;
+
+	return result;
+}
+
+/* Implementation of dir() -- if obj is NULL, returns the names in the current
+   (local) scope.  Otherwise, performs introspection of the object: returns a
+   sorted list of attribute names (supposedly) accessible from the object
+*/
+PyObject *
+PyObject_Dir(PyObject *obj)
+{
+	PyObject * result;
+
+	if (obj == NULL)
+		/* no object -- introspect the locals */
+		result = _dir_locals();
 	else
-		goto normal_return;
+		/* object -- introspect the object */
+		result = _dir_object(obj);
 
-  error:
-	Py_XDECREF(result);
-	result = NULL;
-	/* fall through */
-  normal_return:
-  	Py_XDECREF(masterdict);
+	assert(result == NULL || PyList_Check(result));
+
+	if (result != NULL && PyList_Sort(result) != 0) {
+		/* sorting the list failed */
+		Py_DECREF(result);
+		result = NULL;
+	}
+	
 	return result;
 }
 

Python/bltinmodule.c

@@ -427,15 +427,16 @@ builtin_dir(PyObject *self, PyObject *args)
 PyDoc_STRVAR(dir_doc,
 "dir([object]) -> list of strings\n"
 "\n"
-"Return an alphabetized list of names comprising (some of) the attributes\n"
-"of the given object, and of attributes reachable from it:\n"
-"\n"
-"No argument:  the names in the current scope.\n"
-"Module object:  the module attributes.\n"
-"Type or class object:  its attributes, and recursively the attributes of\n"
-"    its bases.\n"
-"Otherwise:  its attributes, its class's attributes, and recursively the\n"
-"    attributes of its class's base classes.");
+"If called without an argument, return the names in the current scope.\n"
+"Else, return an alphabetized list of names comprising (some of) the attributes\n"
+"of the given object, and of attributes reachable from it.\n"
+"If the object supplies a method named __dir__, it will be used; otherwise\n"
+"the default dir() logic is used and returns:\n"
+"  for a module object: the module's attributes.\n"
+"  for a class object:  its attributes, and recursively the attributes\n"
+"    of its bases.\n"
+"  for an other object: its attributes, its class's attributes, and\n"
+"    recursively the attributes of its class's base classes.");
 
 static PyObject *
 builtin_divmod(PyObject *self, PyObject *args)