Issue #13590: Improve support for OS X Xcode 4:

- Try to avoid building Python or extension modules with problematic
  llvm-gcc compiler.
- Since Xcode 4 removes ppc support, extension module builds now
  check for ppc compiler support and automatically remove ppc and
  ppc64 archs when not available.
- Since Xcode 4 no longer install SDKs in default locations,
  extension module builds now revert to using installed headers
  and libs if the SDK used to build the interpreter is not
  available.
- Update ./configure to use better defaults for universal builds;
  in particular, --enable-universalsdk=yes uses the Xcode default
  SDK and --with-universal-archs now defaults to "intel" if ppc
  not available.

Lib/distutils/sysconfig.py

@@ -162,7 +162,7 @@ def get_python_lib(plat_specific=0, standard_lib=0, prefix=None):
             "I don't know where Python installs its library "
             "on platform '%s'" % os.name)
 
-_USE_CLANG = None
+
 
 def customize_compiler(compiler):
     """Do any platform-specific customization of a CCompiler instance.
@@ -177,36 +177,7 @@ def customize_compiler(compiler):
 
         newcc = None
         if 'CC' in os.environ:
-            newcc = os.environ['CC']
-        elif sys.platform == 'darwin' and cc == 'gcc-4.2':
-            # Issue #13590:
-            #       Since Apple removed gcc-4.2 in Xcode 4.2, we can no
-            #       longer assume it is available for extension module builds.
-            #       If Python was built with gcc-4.2, check first to see if
-            #       it is available on this system; if not, try to use clang
-            #       instead unless the caller explicitly set CC.
-            global _USE_CLANG
-            if _USE_CLANG is None:
-                from distutils import log
-                from subprocess import Popen, PIPE
-                p = Popen("! type gcc-4.2 && type clang && exit 2",
-                                shell=True, stdout=PIPE, stderr=PIPE)
-                p.wait()
-                if p.returncode == 2:
-                    _USE_CLANG = True
-                    log.warn("gcc-4.2 not found, using clang instead")
-                else:
-                    _USE_CLANG = False
-            if _USE_CLANG:
-                newcc = 'clang'
-        if newcc:
-            # On OS X, if CC is overridden, use that as the default
-            #       command for LDSHARED as well
-            if (sys.platform == 'darwin'
-                    and 'LDSHARED' not in os.environ
-                    and ldshared.startswith(cc)):
-                ldshared = newcc + ldshared[len(cc):]
-            cc = newcc
+            cc = os.environ['CC']
         if 'CXX' in os.environ:
             cxx = os.environ['CXX']
         if 'LDSHARED' in os.environ:
@@ -522,6 +493,29 @@ def _init_os2():
     _config_vars = g
 
 
+def _read_output(commandstring):
+    """
+    Returns os.popen(commandstring, "r").read(), but
+    without actually using os.popen because that
+    function is not usable during python bootstrap
+    """
+    # NOTE: tempfile is also not useable during
+    # bootstrap
+    import contextlib
+    try:
+        import tempfile
+        fp = tempfile.NamedTemporaryFile()
+    except ImportError:
+        fp = open("/tmp/distutils.%s"%(
+            os.getpid(),), "w+b")
+
+    with contextlib.closing(fp) as fp:
+        cmd = "%s >'%s'"%(commandstring, fp.name)
+        os.system(cmd)
+        data = fp.read()
+
+    return data.decode('utf-8')
+
 def get_config_vars(*args):
     """With no arguments, return a dictionary of all configuration
     variables relevant for the current platform.  Generally this includes
@@ -561,9 +555,70 @@ def get_config_vars(*args):
                 _config_vars['srcdir'] = os.path.normpath(srcdir)
 
         if sys.platform == 'darwin':
+            from distutils.spawn import find_executable
+
             kernel_version = os.uname()[2] # Kernel version (8.4.3)
             major_version = int(kernel_version.split('.')[0])
 
+            # Issue #13590:
+            #    The OSX location for the compiler varies between OSX
+            #    (or rather Xcode) releases.  With older releases (up-to 10.5)
+            #    the compiler is in /usr/bin, with newer releases the compiler
+            #    can only be found inside Xcode.app if the "Command Line Tools"
+            #    are not installed.
+            #
+            #    Futhermore, the compiler that can be used varies between
+            #    Xcode releases. Upto Xcode 4 it was possible to use 'gcc-4.2'
+            #    as the compiler, after that 'clang' should be used because
+            #    gcc-4.2 is either not present, or a copy of 'llvm-gcc' that
+            #    miscompiles Python.
+
+            # skip checks if the compiler was overriden with a CC env variable
+            if 'CC' not in os.environ:
+                cc = oldcc = _config_vars['CC']
+                if not find_executable(cc):
+                    # Compiler is not found on the shell search PATH.
+                    # Now search for clang, first on PATH (if the Command LIne
+                    # Tools have been installed in / or if the user has provided
+                    # another location via CC).  If not found, try using xcrun
+                    # to find an uninstalled clang (within a selected Xcode).
+
+                    # NOTE: Cannot use subprocess here because of bootstrap
+                    # issues when building Python itself (and os.popen is
+                    # implemented on top of subprocess and is therefore not
+                    # usable as well)
+
+                    data = (find_executable('clang') or
+                            _read_output(
+                                "/usr/bin/xcrun -find clang 2>/dev/null").strip())
+                    if not data:
+                        raise DistutilsPlatformError(
+                               "Cannot locate working compiler")
+
+                    _config_vars['CC'] = cc = data
+                    _config_vars['CXX'] = cc + '++'
+
+                elif os.path.basename(cc).startswith('gcc'):
+                    # Compiler is GCC, check if it is LLVM-GCC
+                    data = _read_output("'%s' --version 2>/dev/null"
+                                         % (cc.replace("'", "'\"'\"'"),))
+                    if 'llvm-gcc' in data:
+                        # Found LLVM-GCC, fall back to clang
+                        data = (find_executable('clang') or
+                                _read_output(
+                                    "/usr/bin/xcrun -find clang 2>/dev/null").strip())
+                        if find_executable(data):
+                            _config_vars['CC'] = cc = data
+                            _config_vars['CXX'] = cc + '++'
+
+                if (cc != oldcc
+                        and 'LDSHARED' in _config_vars
+                        and 'LDSHARED' not in os.environ):
+                    # modify LDSHARED if we modified CC
+                    ldshared = _config_vars['LDSHARED']
+                    if ldshared.startswith(oldcc):
+                        _config_vars['LDSHARED'] = cc + ldshared[len(oldcc):]
+
             if major_version < 8:
                 # On Mac OS X before 10.4, check if -arch and -isysroot
                 # are in CFLAGS or LDFLAGS and remove them if they are.
@@ -579,19 +634,45 @@ def get_config_vars(*args):
                     _config_vars[key] = flags
 
             else:
+                # Different Xcode releases support different sets for '-arch'
+                # flags. In particular, Xcode 4.x no longer supports the
+                # PPC architectures.
+                #
+                # This code automatically removes '-arch ppc' and '-arch ppc64'
+                # when these are not supported. That makes it possible to
+                # build extensions on OSX 10.7 and later with the prebuilt
+                # 32-bit installer on the python.org website.
+                flags = _config_vars['CFLAGS']
+                if re.search('-arch\s+ppc', flags) is not None:
+                    # NOTE: Cannot use subprocess here because of bootstrap
+                    # issues when building Python itself
+                    status = os.system("'%s' -arch ppc -x c /dev/null 2>/dev/null"%(
+                        _config_vars['CC'].replace("'", "'\"'\"'"),))
+
+                    if status != 0:
+                        # Compiler doesn't support PPC, remove the related
+                        # '-arch' flags.
+                        for key in ('LDFLAGS', 'BASECFLAGS',
+                            # a number of derived variables. These need to be
+                            # patched up as well.
+                            'CFLAGS', 'PY_CFLAGS', 'BLDSHARED', 'LDSHARED'):
+
+                            flags = _config_vars[key]
+                            flags = re.sub('-arch\s+ppc\w*\s', ' ', flags)
+                            _config_vars[key] = flags
+
 
                 # Allow the user to override the architecture flags using
                 # an environment variable.
                 # NOTE: This name was introduced by Apple in OSX 10.5 and
                 # is used by several scripting languages distributed with
                 # that OS release.
-
                 if 'ARCHFLAGS' in os.environ:
                     arch = os.environ['ARCHFLAGS']
                     for key in ('LDFLAGS', 'BASECFLAGS',
                         # a number of derived variables. These need to be
                         # patched up as well.
-                        'CFLAGS', 'PY_CFLAGS', 'BLDSHARED'):
+                        'CFLAGS', 'PY_CFLAGS', 'BLDSHARED', 'LDSHARED'):
 
                         flags = _config_vars[key]
                         flags = re.sub('-arch\s+\w+\s', ' ', flags)

Lib/distutils/unixccompiler.py

@@ -83,9 +83,8 @@ def _darwin_compiler_fixup(compiler_so, cc_args):
         except ValueError:
             pass
 
-    # Check if the SDK that is used during compilation actually exists,
-    # the universal build requires the usage of a universal SDK and not all
-    # users have that installed by default.
+    # Check if the SDK that is used during compilation actually exists.
+    # If not, revert to using the installed headers and hope for the best.
     sysroot = None
     if '-isysroot' in cc_args:
         idx = cc_args.index('-isysroot')
@@ -97,7 +96,21 @@ def _darwin_compiler_fixup(compiler_so, cc_args):
     if sysroot and not os.path.isdir(sysroot):
         log.warn("Compiling with an SDK that doesn't seem to exist: %s",
                 sysroot)
-        log.warn("Please check your Xcode installation")
+        log.warn("Attempting to compile without the SDK")
+        while True:
+            try:
+                index = cc_args.index('-isysroot')
+                # Strip this argument and the next one:
+                del cc_args[index:index+2]
+            except ValueError:
+                break
+        while True:
+            try:
+                index = compiler_so.index('-isysroot')
+                # Strip this argument and the next one:
+                del compiler_so[index:index+2]
+            except ValueError:
+                break
 
     return compiler_so
 

Misc/NEWS

@@ -234,6 +234,21 @@ Tests
 Build
 -----
 
+- Issue #13590: Improve support for OS X Xcode 4:
+    * Try to avoid building Python or extension modules with problematic
+      llvm-gcc compiler.
+    * Since Xcode 4 removes ppc support, extension module builds now
+      check for ppc compiler support and automatically remove ppc and
+      ppc64 archs when not available.
+    * Since Xcode 4 no longer install SDKs in default locations,
+      extension module builds now revert to using installed headers
+      and libs if the SDK used to build the interpreter is not
+      available.
+    * Update ./configure to use better defaults for universal builds;
+      in particular, --enable-universalsdk=yes uses the Xcode default
+      SDK and --with-universal-archs now defaults to "intel" if ppc
+      not available.
+
 - Issue #14225: Fix Unicode support for curses (#12567) on OS X
 
 - Issue #14928: Fix importlib bootstrap issues by using a custom executable

configure.ac

@@ -104,14 +104,20 @@ CONFIG_ARGS="$ac_configure_args"
 
 AC_MSG_CHECKING([for --enable-universalsdk])
 AC_ARG_ENABLE(universalsdk,
-	AS_HELP_STRING([--enable-universalsdk@<:@=SDKDIR@:>@], [Build against Mac OS X 10.4u SDK (ppc/i386)]),
+	AS_HELP_STRING([--enable-universalsdk@<:@=SDKDIR@:>@], [Build fat binary against Mac OS X SDK]),
 [
 	case $enableval in
 	yes)
-		enableval=/Developer/SDKs/MacOSX10.4u.sdk
-		if test ! -d "${enableval}"
+		# Locate the best usable SDK, see Mac/README.txt for more
+		# information
+		enableval="`/usr/bin/xcodebuild -version -sdk macosx Path 2>/dev/null`"
+		if test -z "${enableval}"
 		then
-			enableval=/
+			enableval=/Developer/SDKs/MacOSX10.4u.sdk
+			if test ! -d "${enableval}"
+			then
+				enableval=/
+			fi
 		fi
 		;;
 	esac
@@ -143,7 +149,20 @@ AC_SUBST(UNIVERSALSDK)
 
 AC_SUBST(ARCH_RUN_32BIT)
 
+# For backward compatibility reasons we prefer to select '32-bit' if available,
+# otherwise use 'intel'
 UNIVERSAL_ARCHS="32-bit"
+if test "`uname -s`" = "Darwin"
+then
+	if test -n "${UNIVERSALSDK}"
+	then
+		if test -z "`/usr/bin/file "${UNIVERSALSDK}/usr/lib/libSystem.dylib" | grep ppc`"
+		then
+			UNIVERSAL_ARCHS="intel"
+		fi
+	fi
+fi
+
 AC_SUBST(LIPO_32BIT_FLAGS)
 AC_MSG_CHECKING(for --with-universal-archs)
 AC_ARG_WITH(universal-archs,
@@ -153,7 +172,7 @@ AC_ARG_WITH(universal-archs,
 	UNIVERSAL_ARCHS="$withval"
 ],
 [
- 	AC_MSG_RESULT(32-bit)
+ 	AC_MSG_RESULT(${UNIVERSAL_ARCHS})
 ])
 
 
@@ -501,6 +520,63 @@ fi
 if test -z "$CFLAGS"; then
         CFLAGS=
 fi
+
+if test "$ac_sys_system" = "Darwin"
+then
+	# Compiler selection on MacOSX is more complicated than
+	# AC_PROG_CC can handle, see Mac/README.txt for more
+	# information
+	if test -z "${CC}"
+	then
+		found_gcc=
+		found_clang=
+		as_save_IFS=$IFS; IFS=:
+		for as_dir in $PATH
+		do
+			IFS=$as_save_IFS
+			if test -x $as_dir/gcc; then
+				if test -z "${found_gcc}"; then
+					found_gcc=$as_dir/gcc
+				fi
+			fi
+			if test -x $as_dir/clang; then
+				if test -z "${found_clang}"; then
+					found_clang=$as_dir/clang
+				fi
+			fi
+		done
+		IFS=$as_save_IFS
+
+		if test -n "$found_gcc" -a -n "$found_clang"
+		then
+			if test -n "`"$found_gcc" --version | grep llvm-gcc`"
+			then
+				AC_MSG_NOTICE([Detected llvm-gcc, falling back to clang])
+				CC="$found_clang"
+				CXX="$found_clang++"
+			fi
+
+
+		elif test -z "$found_gcc" -a -n "$found_clang"
+		then
+			AC_MSG_NOTICE([No GCC found, use CLANG])
+			CC="$found_clang"
+			CXX="$found_clang++"
+
+		elif test -z "$found_gcc" -a -z "$found_clang"
+		then
+			found_clang=`/usr/bin/xcrun -find clang 2>/dev/null`
+			if test -n "${found_clang}"
+			then
+				AC_MSG_NOTICE([Using clang from Xcode.app])
+				CC="${found_clang}"
+				CXX="`/usr/bin/xcrun -find clang++`"
+
+			# else: use default behaviour
+			fi
+		fi
+	fi
+fi
 AC_PROG_CC
 
 AC_SUBST(CXX)
@@ -534,6 +610,7 @@ then
         case "$CC" in
         gcc)    AC_PATH_PROG(CXX, [g++], [g++], [notfound]) ;;
         cc)     AC_PATH_PROG(CXX, [c++], [c++], [notfound]) ;;
+        clang|*/clang)     AC_PATH_PROG(CXX, [clang++], [clang++], [notfound]) ;;
         esac
 	if test "$CXX" = "notfound"
 	then