bpo-11410: Standardize and use symbol visibility attributes across POSIX and Windows. (GH-16347)

Include/exports.h

+#ifndef Py_EXPORTS_H
+#define Py_EXPORTS_H
+
+#if defined(_WIN32) || defined(__CYGWIN__)
+    #define Py_IMPORTED_SYMBOL __declspec(dllimport)
+    #define Py_EXPORTED_SYMBOL __declspec(dllexport)
+    #define Py_LOCAL_SYMBOL
+#else
+/*
+ * If we only ever used gcc >= 5, we could use __has_attribute(visibility)
+ * as a cross-platform way to determine if visibility is supported. However,
+ * we may still need to support gcc >= 4, as some Ubuntu LTS and Centos versions
+ * have 4 < gcc < 5.
+ */
+    #ifndef __has_attribute
+      #define __has_attribute(x) 0  // Compatibility with non-clang compilers.
+    #endif
+    #if (defined(__GNUC__) && (__GNUC__ >= 4)) ||\
+        (defined(__clang__) && __has_attribute(visibility))
+        #define Py_IMPORTED_SYMBOL __attribute__ ((visibility ("default")))
+        #define Py_EXPORTED_SYMBOL __attribute__ ((visibility ("default")))
+        #define Py_LOCAL_SYMBOL  __attribute__ ((visibility ("hidden")))
+    #else
+        #define Py_IMPORTED_SYMBOL
+        #define Py_EXPORTED_SYMBOL
+        #define Py_LOCAL_SYMBOL
+    #endif
+#endif
+
+#endif /* Py_EXPORTS_H */

Include/pyport.h

@@ -638,16 +638,18 @@ extern char * _getpty(int *, int, mode_t, int);
 #       define HAVE_DECLSPEC_DLL
 #endif
 
+#include "exports.h"
+
 /* only get special linkage if built as shared or platform is Cygwin */
 #if defined(Py_ENABLE_SHARED) || defined(__CYGWIN__)
 #       if defined(HAVE_DECLSPEC_DLL)
 #               if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
-#                       define PyAPI_FUNC(RTYPE) __declspec(dllexport) RTYPE
-#                       define PyAPI_DATA(RTYPE) extern __declspec(dllexport) RTYPE
+#                       define PyAPI_FUNC(RTYPE) Py_EXPORTED_SYMBOL RTYPE
+#                       define PyAPI_DATA(RTYPE) extern Py_EXPORTED_SYMBOL RTYPE
         /* module init functions inside the core need no external linkage */
         /* except for Cygwin to handle embedding */
 #                       if defined(__CYGWIN__)
-#                               define PyMODINIT_FUNC __declspec(dllexport) PyObject*
+#                               define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject*
 #                       else /* __CYGWIN__ */
 #                               define PyMODINIT_FUNC PyObject*
 #                       endif /* __CYGWIN__ */
@@ -658,14 +660,14 @@ extern char * _getpty(int *, int, mode_t, int);
         /* failures similar to those described at the bottom of 4.1: */
         /* http://docs.python.org/extending/windows.html#a-cookbook-approach */
 #                       if !defined(__CYGWIN__)
-#                               define PyAPI_FUNC(RTYPE) __declspec(dllimport) RTYPE
+#                               define PyAPI_FUNC(RTYPE) Py_IMPORTED_SYMBOL RTYPE
 #                       endif /* !__CYGWIN__ */
-#                       define PyAPI_DATA(RTYPE) extern __declspec(dllimport) RTYPE
+#                       define PyAPI_DATA(RTYPE) extern Py_IMPORTED_SYMBOL RTYPE
         /* module init functions outside the core must be exported */
 #                       if defined(__cplusplus)
-#                               define PyMODINIT_FUNC extern "C" __declspec(dllexport) PyObject*
+#                               define PyMODINIT_FUNC extern "C" Py_EXPORTED_SYMBOL PyObject*
 #                       else /* __cplusplus */
-#                               define PyMODINIT_FUNC __declspec(dllexport) PyObject*
+#                               define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject*
 #                       endif /* __cplusplus */
 #               endif /* Py_BUILD_CORE */
 #       endif /* HAVE_DECLSPEC_DLL */
@@ -673,16 +675,16 @@ extern char * _getpty(int *, int, mode_t, int);
 
 /* If no external linkage macros defined by now, create defaults */
 #ifndef PyAPI_FUNC
-#       define PyAPI_FUNC(RTYPE) RTYPE
+#       define PyAPI_FUNC(RTYPE) Py_EXPORTED_SYMBOL RTYPE
 #endif
 #ifndef PyAPI_DATA
-#       define PyAPI_DATA(RTYPE) extern RTYPE
+#       define PyAPI_DATA(RTYPE) extern Py_EXPORTED_SYMBOL RTYPE
 #endif
 #ifndef PyMODINIT_FUNC
 #       if defined(__cplusplus)
-#               define PyMODINIT_FUNC extern "C" PyObject*
+#               define PyMODINIT_FUNC extern "C" Py_EXPORTED_SYMBOL PyObject*
 #       else /* __cplusplus */
-#               define PyMODINIT_FUNC PyObject*
+#               define PyMODINIT_FUNC Py_EXPORTED_SYMBOL PyObject*
 #       endif /* __cplusplus */
 #endif
 

Misc/NEWS.d/next/Core and Builtins/2019-09-24-05-32-27.bpo-11410.vS182p.rst

+Better control over symbol visibility is provided through use of the
+visibility attributes available in gcc >= 4.0, provided in a uniform way
+across POSIX and Windows. The POSIX build files have been updated to compile
+with -fvisibility=hidden, minimising exported symbols.

Modules/_ctypes/_ctypes_test.c

@@ -4,11 +4,7 @@
 #include <windows.h>
 #endif
 
-#if defined(MS_WIN32) || defined(__CYGWIN__)
-#define EXPORT(x) __declspec(dllexport) x
-#else
-#define EXPORT(x) x
-#endif
+#define EXPORT(x) Py_EXPORTED_SYMBOL x
 
 /* some functions handy for testing */
 

Modules/_io/_iomodule.h

@@ -2,6 +2,8 @@
  * Declarations shared between the different parts of the io module
  */
 
+#include "exports.h"
+
 /* ABCs */
 extern PyTypeObject PyIOBase_Type;
 extern PyTypeObject PyRawIOBase_Type;
@@ -183,4 +185,4 @@ extern PyObject *_PyIO_str_write;
 extern PyObject *_PyIO_empty_str;
 extern PyObject *_PyIO_empty_bytes;
 
-extern PyTypeObject _PyBytesIOBuffer_Type;
+extern Py_EXPORTED_SYMBOL PyTypeObject _PyBytesIOBuffer_Type;

Modules/_io/bytesio.c

@@ -1124,7 +1124,7 @@ static PyBufferProcs bytesiobuf_as_buffer = {
     (releasebufferproc) bytesiobuf_releasebuffer,
 };
 
-PyTypeObject _PyBytesIOBuffer_Type = {
+Py_EXPORTED_SYMBOL PyTypeObject _PyBytesIOBuffer_Type = {
     PyVarObject_HEAD_INIT(NULL, 0)
     "_io._BytesIOBuffer",                      /*tp_name*/
     sizeof(bytesiobuf),                        /*tp_basicsize*/

Parser/pgen/grammar.py

@@ -61,13 +61,14 @@ class Grammar:
     def produce_graminit_c(self, writer):
         writer("/* Generated by Parser/pgen */\n\n")
 
+        writer('#include "exports.h"\n')
         writer('#include "grammar.h"\n')
-        writer("grammar _PyParser_Grammar;\n")
+        writer("Py_EXPORTED_SYMBOL grammar _PyParser_Grammar;\n")
 
         self.print_dfas(writer)
         self.print_labels(writer)
 
-        writer("grammar _PyParser_Grammar = {\n")
+        writer("Py_EXPORTED_SYMBOL grammar _PyParser_Grammar = {\n")
         writer("    {n_dfas},\n".format(n_dfas=len(self.dfas)))
         writer("    dfas,\n")
         writer("    {{{n_labels}, labels}},\n".format(n_labels=len(self.labels)))

Python/getargs.c

@@ -106,7 +106,7 @@ PyArg_Parse(PyObject *args, const char *format, ...)
     return retval;
 }
 
-int
+PyAPI_FUNC(int)
 _PyArg_Parse_SizeT(PyObject *args, const char *format, ...)
 {
     int retval;
@@ -131,7 +131,7 @@ PyArg_ParseTuple(PyObject *args, const char *format, ...)
     return retval;
 }
 
-int
+PyAPI_FUNC(int)
 _PyArg_ParseTuple_SizeT(PyObject *args, const char *format, ...)
 {
     int retval;
@@ -156,7 +156,7 @@ _PyArg_ParseStack(PyObject *const *args, Py_ssize_t nargs, const char *format, .
     return retval;
 }
 
-int
+PyAPI_FUNC(int)
 _PyArg_ParseStack_SizeT(PyObject *const *args, Py_ssize_t nargs, const char *format, ...)
 {
     int retval;
@@ -182,7 +182,7 @@ PyArg_VaParse(PyObject *args, const char *format, va_list va)
     return retval;
 }
 
-int
+PyAPI_FUNC(int)
 _PyArg_VaParse_SizeT(PyObject *args, const char *format, va_list va)
 {
     va_list lva;
@@ -1442,7 +1442,7 @@ PyArg_ParseTupleAndKeywords(PyObject *args,
     return retval;
 }
 
-int
+PyAPI_FUNC(int)
 _PyArg_ParseTupleAndKeywords_SizeT(PyObject *args,
                                   PyObject *keywords,
                                   const char *format,
@@ -1493,7 +1493,7 @@ PyArg_VaParseTupleAndKeywords(PyObject *args,
     return retval;
 }
 
-int
+PyAPI_FUNC(int)
 _PyArg_VaParseTupleAndKeywords_SizeT(PyObject *args,
                                     PyObject *keywords,
                                     const char *format,
@@ -1519,7 +1519,7 @@ _PyArg_VaParseTupleAndKeywords_SizeT(PyObject *args,
     return retval;
 }
 
-int
+PyAPI_FUNC(int)
 _PyArg_ParseTupleAndKeywordsFast(PyObject *args, PyObject *keywords,
                             struct _PyArg_Parser *parser, ...)
 {
@@ -1532,7 +1532,7 @@ _PyArg_ParseTupleAndKeywordsFast(PyObject *args, PyObject *keywords,
     return retval;
 }
 
-int
+PyAPI_FUNC(int)
 _PyArg_ParseTupleAndKeywordsFast_SizeT(PyObject *args, PyObject *keywords,
                             struct _PyArg_Parser *parser, ...)
 {
@@ -1545,7 +1545,7 @@ _PyArg_ParseTupleAndKeywordsFast_SizeT(PyObject *args, PyObject *keywords,
     return retval;
 }
 
-int
+PyAPI_FUNC(int)
 _PyArg_ParseStackAndKeywords(PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames,
                   struct _PyArg_Parser *parser, ...)
 {
@@ -1558,7 +1558,7 @@ _PyArg_ParseStackAndKeywords(PyObject *const *args, Py_ssize_t nargs, PyObject *
     return retval;
 }
 
-int
+PyAPI_FUNC(int)
 _PyArg_ParseStackAndKeywords_SizeT(PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames,
                         struct _PyArg_Parser *parser, ...)
 {
@@ -1572,7 +1572,7 @@ _PyArg_ParseStackAndKeywords_SizeT(PyObject *const *args, Py_ssize_t nargs, PyOb
 }
 
 
-int
+PyAPI_FUNC(int)
 _PyArg_VaParseTupleAndKeywordsFast(PyObject *args, PyObject *keywords,
                             struct _PyArg_Parser *parser, va_list va)
 {
@@ -1586,7 +1586,7 @@ _PyArg_VaParseTupleAndKeywordsFast(PyObject *args, PyObject *keywords,
     return retval;
 }
 
-int
+PyAPI_FUNC(int)
 _PyArg_VaParseTupleAndKeywordsFast_SizeT(PyObject *args, PyObject *keywords,
                             struct _PyArg_Parser *parser, va_list va)
 {

Python/graminit.c

 /* Generated by Parser/pgen */
 
+#include "exports.h"
 #include "grammar.h"
-grammar _PyParser_Grammar;
+Py_EXPORTED_SYMBOL grammar _PyParser_Grammar;
 static const arc arcs_0_0[3] = {
     {2, 1},
     {3, 2},
@@ -2693,7 +2694,7 @@ static const label labels[183] = {
     {346, 0},
     {347, 0},
 };
-grammar _PyParser_Grammar = {
+Py_EXPORTED_SYMBOL grammar _PyParser_Grammar = {
     92,
     dfas,
     {183, labels},

Python/pythonrun.c

@@ -57,7 +57,7 @@ _Py_static_string(PyId_string, "<string>");
 extern "C" {
 #endif
 
-extern grammar _PyParser_Grammar; /* From graminit.c */
+extern Py_EXPORTED_SYMBOL grammar _PyParser_Grammar; /* From graminit.c */
 
 /* Forward */
 static void flush_io(void);

configure

@@ -7341,6 +7341,47 @@ $as_echo "$ac_cv_enable_implicit_function_declaration_error" >&6; }
       CFLAGS_NODIST="$CFLAGS_NODIST -Werror=implicit-function-declaration"
     fi
 
+    { $as_echo "$as_me:${as_lineno-$LINENO}: checking if we can use visibility in $CC" >&5
+$as_echo_n "checking if we can use visibility in $CC... " >&6; }
+     ac_save_cc="$CC"
+     CC="$CC -fvisibility=hidden"
+     if ${ac_cv_enable_visibility+:} false; then :
+  $as_echo_n "(cached) " >&6
+else
+  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+
+
+int
+main ()
+{
+
+  ;
+  return 0;
+}
+
+_ACEOF
+if ac_fn_c_try_compile "$LINENO"; then :
+
+           ac_cv_enable_visibility=yes
+
+else
+
+           ac_cv_enable_visibility=no
+
+fi
+rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+
+     CC="$ac_save_cc"
+    { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_enable_visibility" >&5
+$as_echo "$ac_cv_enable_visibility" >&6; }
+
+    if test $ac_cv_enable_visibility = yes
+    then
+      CFLAGS_NODIST="$CFLAGS_NODIST -fvisibility=hidden"
+    fi
+
     # if using gcc on alpha, use -mieee to get (near) full IEEE 754
     # support.  Without this, treatment of subnormals doesn't follow
     # the standard.

configure.ac

@@ -1787,6 +1787,26 @@ yes)
       CFLAGS_NODIST="$CFLAGS_NODIST -Werror=implicit-function-declaration"
     fi
 
+    AC_MSG_CHECKING(if we can use visibility in $CC)
+     ac_save_cc="$CC"
+     CC="$CC -fvisibility=hidden"
+     AC_CACHE_VAL(ac_cv_enable_visibility,
+       AC_COMPILE_IFELSE(
+         [
+	   AC_LANG_PROGRAM([[]], [[]])
+	 ],[
+           ac_cv_enable_visibility=yes
+	 ],[
+           ac_cv_enable_visibility=no
+	 ]))
+     CC="$ac_save_cc"
+    AC_MSG_RESULT($ac_cv_enable_visibility)
+
+    if test $ac_cv_enable_visibility = yes
+    then
+      CFLAGS_NODIST="$CFLAGS_NODIST -fvisibility=hidden"
+    fi
+
     # if using gcc on alpha, use -mieee to get (near) full IEEE 754
     # support.  Without this, treatment of subnormals doesn't follow
     # the standard.