build-installer.py 40 KB
Newer Older
1
#!/usr/bin/python
2 3 4
"""
This script is used to build the "official unofficial" universal build on
Mac OS X. It requires Mac OS X 10.4, Xcode 2.2 and the 10.4u SDK to do its
5 6
work.  64-bit or four-way universal builds require at least OS X 10.5 and
the 10.5 SDK.
7 8 9 10 11 12

Please ensure that this script keeps working with Python 2.3, to avoid
bootstrap issues (/usr/bin/python is Python 2.3 on OSX 10.4)

Usage: see USAGE variable in the script.
"""
13
import platform, os, sys, getopt, textwrap, shutil, urllib2, stat, time, pwd
14
import grp
15

16 17
INCLUDE_TIMESTAMP = 1
VERBOSE = 1
18 19 20 21 22 23 24 25 26 27 28 29 30 31

from plistlib import Plist

import MacOS

try:
    from plistlib import writePlist
except ImportError:
    # We're run using python2.3
    def writePlist(plist, path):
        plist.write(path)

def shellQuote(value):
    """
32
    Return the string value in a form that can safely be inserted into
33 34 35 36 37 38 39 40 41 42
    a shell command.
    """
    return "'%s'"%(value.replace("'", "'\"'\"'"))

def grepValue(fn, variable):
    variable = variable + '='
    for ln in open(fn, 'r'):
        if ln.startswith(variable):
            value = ln[len(variable):].strip()
            return value[1:-1]
43
    raise RuntimeError, "Cannot find variable %s" % variable[:-1]
44 45 46 47

def getVersion():
    return grepValue(os.path.join(SRCDIR, 'configure'), 'PACKAGE_VERSION')

48 49 50
def getVersionTuple():
    return tuple([int(n) for n in getVersion().split('.')])

51 52 53 54 55
def getFullVersion():
    fn = os.path.join(SRCDIR, 'Include', 'patchlevel.h')
    for ln in open(fn):
        if 'PY_VERSION' in ln:
            return ln.split()[-1][1:-1]
56
    raise RuntimeError, "Cannot find full version??"
57

58 59
# The directory we'll use to create the build (will be erased and recreated)
WORKDIR = "/tmp/_py"
60

61
# The directory we'll use to store third-party sources. Set this to something
62
# else if you don't want to re-fetch required libraries every time.
63 64
DEPSRC = os.path.join(WORKDIR, 'third-party')
DEPSRC = os.path.expanduser('~/Universal/other-sources')
65 66

# Location of the preferred SDK
67 68 69 70 71 72 73 74 75 76 77 78 79 80

### There are some issues with the SDK selection below here,
### The resulting binary doesn't work on all platforms that
### it should. Always default to the 10.4u SDK until that
### isue is resolved.
###
##if int(os.uname()[2].split('.')[0]) == 8:
##    # Explicitly use the 10.4u (universal) SDK when
##    # building on 10.4, the system headers are not
##    # useable for a universal build
##    SDKPATH = "/Developer/SDKs/MacOSX10.4u.sdk"
##else:
##    SDKPATH = "/"

81
SDKPATH = "/Developer/SDKs/MacOSX10.4u.sdk"
82

83 84
universal_opts_map = { '32-bit': ('i386', 'ppc',),
                       '64-bit': ('x86_64', 'ppc64',),
85 86 87 88 89 90 91 92 93
                       'intel':  ('i386', 'x86_64'),
                       '3-way':  ('ppc', 'i386', 'x86_64'),
                       'all':    ('i386', 'ppc', 'x86_64', 'ppc64',) }
default_target_map = {
        '64-bit': '10.5',
        '3-way': '10.5',
        'intel': '10.5',
        'all': '10.5',
}
94 95 96 97 98 99

UNIVERSALOPTS = tuple(universal_opts_map.keys())

UNIVERSALARCHS = '32-bit'

ARCHLIST = universal_opts_map[UNIVERSALARCHS]
100

101
# Source directory (asume we're in Mac/BuildScript)
102
SRCDIR = os.path.dirname(
103 104
        os.path.dirname(
            os.path.dirname(
105 106
                os.path.abspath(__file__
        ))))
107

108 109 110
# $MACOSX_DEPLOYMENT_TARGET -> minimum OS X level
DEPTARGET = '10.3'

111 112 113 114 115 116 117 118 119 120 121
target_cc_map = {
        '10.3': 'gcc-4.0',
        '10.4': 'gcc-4.0',
        '10.5': 'gcc-4.0',
        '10.6': 'gcc-4.2',
}

CC = target_cc_map[DEPTARGET]

PYTHON_3 = getVersionTuple() >= (3, 0)

122
USAGE = textwrap.dedent("""\
123 124 125 126 127 128 129 130 131
    Usage: build_python [options]

    Options:
    -? or -h:            Show this message
    -b DIR
    --build-dir=DIR:     Create build here (default: %(WORKDIR)r)
    --third-party=DIR:   Store third-party sources here (default: %(DEPSRC)r)
    --sdk-path=DIR:      Location of the SDK (default: %(SDKPATH)r)
    --src-dir=DIR:       Location of the Python sources (default: %(SRCDIR)r)
132 133
    --dep-target=10.n    OS X deployment target (default: %(DEPTARGET)r)
    --universal-archs=x  universal architectures (options: %(UNIVERSALOPTS)r, default: %(UNIVERSALARCHS)r)
134 135 136 137 138
""")% globals()


# Instructions for building libraries that are necessary for building a
# batteries included python.
139 140 141
#   [The recipes are defined here for convenience but instantiated later after
#    command line options have been processed.]
def library_recipes():
142 143 144 145 146 147 148 149 150 151 152 153 154 155 156
    result = []

    if DEPTARGET < '10.5':
        result.extend([
          dict(
              name="Bzip2 1.0.5",
              url="http://www.bzip.org/1.0.5/bzip2-1.0.5.tar.gz",
              checksum='3c15a0c8d1d3ee1c46a1634d00617b1a',
              configure=None,
              install='make install CC=%s PREFIX=%s/usr/local/ CFLAGS="-arch %s -isysroot %s"'%(
                  CC,
                  shellQuote(os.path.join(WORKDIR, 'libraries')),
                  ' -arch '.join(ARCHLIST),
                  SDKPATH,
              ),
157
          ),
158 159 160 161 162 163 164 165 166 167 168
          dict(
              name="ZLib 1.2.3",
              url="http://www.gzip.org/zlib/zlib-1.2.3.tar.gz",
              checksum='debc62758716a169df9f62e6ab2bc634',
              configure=None,
              install='make install CC=%s prefix=%s/usr/local/ CFLAGS="-arch %s -isysroot %s"'%(
                  CC,
                  shellQuote(os.path.join(WORKDIR, 'libraries')),
                  ' -arch '.join(ARCHLIST),
                  SDKPATH,
              ),
169
          ),
170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226
          dict(
              # Note that GNU readline is GPL'd software
              name="GNU Readline 5.1.4",
              url="http://ftp.gnu.org/pub/gnu/readline/readline-5.1.tar.gz" ,
              checksum='7ee5a692db88b30ca48927a13fd60e46',
              patchlevel='0',
              patches=[
                  # The readline maintainers don't do actual micro releases, but
                  # just ship a set of patches.
                  'http://ftp.gnu.org/pub/gnu/readline/readline-5.1-patches/readline51-001',
                  'http://ftp.gnu.org/pub/gnu/readline/readline-5.1-patches/readline51-002',
                  'http://ftp.gnu.org/pub/gnu/readline/readline-5.1-patches/readline51-003',
                  'http://ftp.gnu.org/pub/gnu/readline/readline-5.1-patches/readline51-004',
              ]
          ),
          dict(
              name="SQLite 3.6.11",
              url="http://www.sqlite.org/sqlite-3.6.11.tar.gz",
              checksum='7ebb099696ab76cc6ff65dd496d17858',
              configure_pre=[
                  '--enable-threadsafe',
                  '--enable-tempstore',
                  '--enable-shared=no',
                  '--enable-static=yes',
                  '--disable-tcl',
              ]
          ),
          dict(
              name="NCurses 5.5",
              url="http://ftp.gnu.org/pub/gnu/ncurses/ncurses-5.5.tar.gz",
              checksum='e73c1ac10b4bfc46db43b2ddfd6244ef',
              configure_pre=[
                  "--without-cxx",
                  "--without-ada",
                  "--without-progs",
                  "--without-curses-h",
                  "--enable-shared",
                  "--with-shared",
                  "--datadir=/usr/share",
                  "--sysconfdir=/etc",
                  "--sharedstatedir=/usr/com",
                  "--with-terminfo-dirs=/usr/share/terminfo",
                  "--with-default-terminfo-dir=/usr/share/terminfo",
                  "--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib"%(getVersion(),),
                  "--enable-termcap",
              ],
              patches=[
                  "ncurses-5.5.patch",
              ],
              useLDFlags=False,
              install='make && make install DESTDIR=%s && cd %s/usr/local/lib && ln -fs ../../../Library/Frameworks/Python.framework/Versions/%s/lib/lib* .'%(
                  shellQuote(os.path.join(WORKDIR, 'libraries')),
                  shellQuote(os.path.join(WORKDIR, 'libraries')),
                  getVersion(),
                  ),
          ),
        ])
227

228
    result.extend([
229
      dict(
230 231 232 233 234
          name="Sleepycat DB 4.7.25",
          url="http://download.oracle.com/berkeley-db/db-4.7.25.tar.gz",
          checksum='ec2b87e833779681a0c3a814aa71359e',
          buildDir="build_unix",
          configure="../dist/configure",
235
          configure_pre=[
236
              '--includedir=/usr/local/include/db4',
237 238
          ]
      ),
239 240 241
    ])

    return result
242 243 244


# Instructions for building packages inside the .mpkg.
245 246 247 248 249 250 251 252 253 254 255
def pkg_recipes():
    unselected_for_python3 = ('selected', 'unselected')[PYTHON_3]
    result = [
        dict(
            name="PythonFramework",
            long_name="Python Framework",
            source="/Library/Frameworks/Python.framework",
            readme="""\
                This package installs Python.framework, that is the python
                interpreter and the standard library. This also includes Python
                wrappers for lots of Mac OS X API's.
256
            """,
257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338
            postflight="scripts/postflight.framework",
            selected='selected',
        ),
        dict(
            name="PythonApplications",
            long_name="GUI Applications",
            source="/Applications/Python %(VER)s",
            readme="""\
                This package installs IDLE (an interactive Python IDE),
                Python Launcher and Build Applet (create application bundles
                from python scripts).

                It also installs a number of examples and demos.
                """,
            required=False,
            selected='selected',
        ),
        dict(
            name="PythonUnixTools",
            long_name="UNIX command-line tools",
            source="/usr/local/bin",
            readme="""\
                This package installs the unix tools in /usr/local/bin for
                compatibility with older releases of Python. This package
                is not necessary to use Python.
                """,
            required=False,
            selected='selected',
        ),
        dict(
            name="PythonDocumentation",
            long_name="Python Documentation",
            topdir="/Library/Frameworks/Python.framework/Versions/%(VER)s/Resources/English.lproj/Documentation",
            source="/pydocs",
            readme="""\
                This package installs the python documentation at a location
                that is useable for pydoc and IDLE. If you have installed Xcode
                it will also install a link to the documentation in
                /Developer/Documentation/Python
                """,
            postflight="scripts/postflight.documentation",
            required=False,
            selected='selected',
        ),
        dict(
            name="PythonProfileChanges",
            long_name="Shell profile updater",
            readme="""\
                This packages updates your shell profile to make sure that
                the Python tools are found by your shell in preference of
                the system provided Python tools.

                If you don't install this package you'll have to add
                "/Library/Frameworks/Python.framework/Versions/%(VER)s/bin"
                to your PATH by hand.
                """,
            postflight="scripts/postflight.patch-profile",
            topdir="/Library/Frameworks/Python.framework",
            source="/empty-dir",
            required=False,
            selected=unselected_for_python3,
        ),
    ]

    if DEPTARGET < '10.4':
        result.append(
            dict(
                name="PythonSystemFixes",
                long_name="Fix system Python",
                readme="""\
                    This package updates the system python installation on
                    Mac OS X 10.3 to ensure that you can build new python extensions
                    using that copy of python after installing this version.
                    """,
                postflight="../Tools/fixapplepython23.py",
                topdir="/Library/Frameworks/Python.framework",
                source="/empty-dir",
                required=False,
                selected=unselected_for_python3,
            )
        )
    return result
339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362

def fatal(msg):
    """
    A fatal error, bail out.
    """
    sys.stderr.write('FATAL: ')
    sys.stderr.write(msg)
    sys.stderr.write('\n')
    sys.exit(1)

def fileContents(fn):
    """
    Return the contents of the named file
    """
    return open(fn, 'rb').read()

def runCommand(commandline):
    """
    Run a command and raise RuntimeError if it fails. Output is surpressed
    unless the command fails.
    """
    fd = os.popen(commandline, 'r')
    data = fd.read()
    xit = fd.close()
363
    if xit is not None:
364
        sys.stdout.write(data)
365
        raise RuntimeError, "command failed: %s"%(commandline,)
366 367 368 369 370 371 372 373

    if VERBOSE:
        sys.stdout.write(data); sys.stdout.flush()

def captureCommand(commandline):
    fd = os.popen(commandline, 'r')
    data = fd.read()
    xit = fd.close()
374
    if xit is not None:
375
        sys.stdout.write(data)
376
        raise RuntimeError, "command failed: %s"%(commandline,)
377 378 379

    return data

380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396
def getTclTkVersion(configfile, versionline):
    """
    search Tcl or Tk configuration file for version line
    """
    try:
        f = open(configfile, "r")
    except:
        fatal("Framework configuration file not found: %s" % configfile)

    for l in f:
        if l.startswith(versionline):
            f.close()
            return l

    fatal("Version variable %s not found in framework configuration file: %s"
            % (versionline, configfile))

397 398 399 400 401 402
def checkEnvironment():
    """
    Check that we're running on a supported system.
    """

    if platform.system() != 'Darwin':
403
        fatal("This script should be run on a Mac OS X 10.4 (or later) system")
404

405 406
    if int(platform.release().split('.')[0]) < 8:
        fatal("This script should be run on a Mac OS X 10.4 (or later) system")
407 408 409 410 411

    if not os.path.exists(SDKPATH):
        fatal("Please install the latest version of Xcode and the %s SDK"%(
            os.path.basename(SDKPATH[:-4])))

412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443
    # Because we only support dynamic load of only one major/minor version of
    # Tcl/Tk, ensure:
    # 1. there are no user-installed frameworks of Tcl/Tk with version
    #       higher than the Apple-supplied system version
    # 2. there is a user-installed framework in /Library/Frameworks with the
    #       same version as the system version.  This allows users to choose
    #       to install a newer patch level.

    for framework in ['Tcl', 'Tk']:
        fw = dict(lower=framework.lower(),
                    upper=framework.upper(),
                    cap=framework.capitalize())
        fwpth = "Library/Frameworks/%(cap)s.framework/%(lower)sConfig.sh" % fw
        sysfw = os.path.join('/System', fwpth)
        libfw = os.path.join('/', fwpth)
        usrfw = os.path.join(os.getenv('HOME'), fwpth)
        version = "%(upper)s_VERSION" % fw
        if getTclTkVersion(libfw, version) != getTclTkVersion(sysfw, version):
            fatal("Version of %s must match %s" % (libfw, sysfw) )
        if os.path.exists(usrfw):
            fatal("Please rename %s to avoid possible dynamic load issues."
                    % usrfw)

    # Remove inherited environment variables which might influence build
    environ_var_prefixes = ['CPATH', 'C_INCLUDE_', 'DYLD_', 'LANG', 'LC_',
                            'LD_', 'LIBRARY_', 'PATH', 'PYTHON']
    for ev in list(os.environ):
        for prefix in environ_var_prefixes:
            if ev.startswith(prefix) :
                print "INFO: deleting environment variable %s=%s" % (
                                                    ev, os.environ[ev])
                del os.environ[ev]
444

445 446 447
    os.environ['PATH'] = '/bin:/sbin:/usr/bin:/usr/sbin'
    print "Setting default PATH: %s"%(os.environ['PATH'])

448

449
def parseOptions(args=None):
450 451 452
    """
    Parse arguments and update global settings.
    """
453
    global WORKDIR, DEPSRC, SDKPATH, SRCDIR, DEPTARGET
454
    global UNIVERSALOPTS, UNIVERSALARCHS, ARCHLIST, CC
455 456 457 458 459 460

    if args is None:
        args = sys.argv[1:]

    try:
        options, args = getopt.getopt(args, '?hb',
461 462
                [ 'build-dir=', 'third-party=', 'sdk-path=' , 'src-dir=',
                  'dep-target=', 'universal-archs=', 'help' ])
463 464
    except getopt.error, msg:
        print msg
465 466 467
        sys.exit(1)

    if args:
468
        print "Additional arguments"
469 470
        sys.exit(1)

471
    deptarget = None
472
    for k, v in options:
473
        if k in ('-h', '-?', '--help'):
474
            print USAGE
475 476 477 478 479 480 481 482 483 484 485 486 487 488
            sys.exit(0)

        elif k in ('-d', '--build-dir'):
            WORKDIR=v

        elif k in ('--third-party',):
            DEPSRC=v

        elif k in ('--sdk-path',):
            SDKPATH=v

        elif k in ('--src-dir',):
            SRCDIR=v

489 490
        elif k in ('--dep-target', ):
            DEPTARGET=v
491
            deptarget=v
492 493 494 495 496

        elif k in ('--universal-archs', ):
            if v in UNIVERSALOPTS:
                UNIVERSALARCHS = v
                ARCHLIST = universal_opts_map[UNIVERSALARCHS]
497 498 499 500
                if deptarget is None:
                    # Select alternate default deployment
                    # target
                    DEPTARGET = default_target_map.get(v, '10.3')
501 502 503
            else:
                raise NotImplementedError, v

504
        else:
505
            raise NotImplementedError, k
506 507 508 509 510 511

    SRCDIR=os.path.abspath(SRCDIR)
    WORKDIR=os.path.abspath(WORKDIR)
    SDKPATH=os.path.abspath(SDKPATH)
    DEPSRC=os.path.abspath(DEPSRC)

512 513
    CC=target_cc_map[DEPTARGET]

514 515 516 517
    print "Settings:"
    print " * Source directory:", SRCDIR
    print " * Build directory: ", WORKDIR
    print " * SDK location:    ", SDKPATH
518 519 520
    print " * Third-party source:", DEPSRC
    print " * Deployment target:", DEPTARGET
    print " * Universal architectures:", ARCHLIST
521
    print " * C compiler:", CC
522
    print ""
523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566




def extractArchive(builddir, archiveName):
    """
    Extract a source archive into 'builddir'. Returns the path of the
    extracted archive.

    XXX: This function assumes that archives contain a toplevel directory
    that is has the same name as the basename of the archive. This is
    save enough for anything we use.
    """
    curdir = os.getcwd()
    try:
        os.chdir(builddir)
        if archiveName.endswith('.tar.gz'):
            retval = os.path.basename(archiveName[:-7])
            if os.path.exists(retval):
                shutil.rmtree(retval)
            fp = os.popen("tar zxf %s 2>&1"%(shellQuote(archiveName),), 'r')

        elif archiveName.endswith('.tar.bz2'):
            retval = os.path.basename(archiveName[:-8])
            if os.path.exists(retval):
                shutil.rmtree(retval)
            fp = os.popen("tar jxf %s 2>&1"%(shellQuote(archiveName),), 'r')

        elif archiveName.endswith('.tar'):
            retval = os.path.basename(archiveName[:-4])
            if os.path.exists(retval):
                shutil.rmtree(retval)
            fp = os.popen("tar xf %s 2>&1"%(shellQuote(archiveName),), 'r')

        elif archiveName.endswith('.zip'):
            retval = os.path.basename(archiveName[:-4])
            if os.path.exists(retval):
                shutil.rmtree(retval)
            fp = os.popen("unzip %s 2>&1"%(shellQuote(archiveName),), 'r')

        data = fp.read()
        xit = fp.close()
        if xit is not None:
            sys.stdout.write(data)
567
            raise RuntimeError, "Cannot extract %s"%(archiveName,)
568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588

        return os.path.join(builddir, retval)

    finally:
        os.chdir(curdir)

KNOWNSIZES = {
    "http://ftp.gnu.org/pub/gnu/readline/readline-5.1.tar.gz": 7952742,
    "http://downloads.sleepycat.com/db-4.4.20.tar.gz": 2030276,
}

def downloadURL(url, fname):
    """
    Download the contents of the url into the file.
    """
    try:
        size = os.path.getsize(fname)
    except OSError:
        pass
    else:
        if KNOWNSIZES.get(url) == size:
589
            print "Using existing file for", url
590
            return
591
    fpIn = urllib2.urlopen(url)
592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627
    fpOut = open(fname, 'wb')
    block = fpIn.read(10240)
    try:
        while block:
            fpOut.write(block)
            block = fpIn.read(10240)
        fpIn.close()
        fpOut.close()
    except:
        try:
            os.unlink(fname)
        except:
            pass

def buildRecipe(recipe, basedir, archList):
    """
    Build software using a recipe. This function does the
    'configure;make;make install' dance for C software, with a possibility
    to customize this process, basically a poor-mans DarwinPorts.
    """
    curdir = os.getcwd()

    name = recipe['name']
    url = recipe['url']
    configure = recipe.get('configure', './configure')
    install = recipe.get('install', 'make && make install DESTDIR=%s'%(
        shellQuote(basedir)))

    archiveName = os.path.split(url)[-1]
    sourceArchive = os.path.join(DEPSRC, archiveName)

    if not os.path.exists(DEPSRC):
        os.mkdir(DEPSRC)


    if os.path.exists(sourceArchive):
628
        print "Using local copy of %s"%(name,)
629 630

    else:
631
        print "Did not find local copy of %s"%(name,)
632
        print "Downloading %s"%(name,)
633
        downloadURL(url, sourceArchive)
634
        print "Archive for %s stored as %s"%(name, sourceArchive)
635

636
    print "Extracting archive for %s"%(name,)
637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657
    buildDir=os.path.join(WORKDIR, '_bld')
    if not os.path.exists(buildDir):
        os.mkdir(buildDir)

    workDir = extractArchive(buildDir, sourceArchive)
    os.chdir(workDir)
    if 'buildDir' in recipe:
        os.chdir(recipe['buildDir'])


    for fn in recipe.get('patches', ()):
        if fn.startswith('http://'):
            # Download the patch before applying it.
            path = os.path.join(DEPSRC, os.path.basename(fn))
            downloadURL(fn, path)
            fn = path

        fn = os.path.join(curdir, fn)
        runCommand('patch -p%s < %s'%(recipe.get('patchlevel', 1),
            shellQuote(fn),))

658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680
    if configure is not None:
        configure_args = [
            "--prefix=/usr/local",
            "--enable-static",
            "--disable-shared",
            #"CPP=gcc -arch %s -E"%(' -arch '.join(archList,),),
        ]

        if 'configure_pre' in recipe:
            args = list(recipe['configure_pre'])
            if '--disable-static' in args:
                configure_args.remove('--enable-static')
            if '--enable-shared' in args:
                configure_args.remove('--disable-shared')
            configure_args.extend(args)

        if recipe.get('useLDFlags', 1):
            configure_args.extend([
                "CFLAGS=-arch %s -isysroot %s -I%s/usr/local/include"%(
                        ' -arch '.join(archList),
                        shellQuote(SDKPATH)[1:-1],
                        shellQuote(basedir)[1:-1],),
                "LDFLAGS=-syslibroot,%s -L%s/usr/local/lib -arch %s"%(
681
                    shellQuote(SDKPATH)[1:-1],
682 683 684 685 686 687 688 689 690 691
                    shellQuote(basedir)[1:-1],
                    ' -arch '.join(archList)),
            ])
        else:
            configure_args.extend([
                "CFLAGS=-arch %s -isysroot %s -I%s/usr/local/include"%(
                        ' -arch '.join(archList),
                        shellQuote(SDKPATH)[1:-1],
                        shellQuote(basedir)[1:-1],),
            ])
692

693 694
        if 'configure_post' in recipe:
            configure_args = configure_args = list(recipe['configure_post'])
695

696 697
        configure_args.insert(0, configure)
        configure_args = [ shellQuote(a) for a in configure_args ]
698

699
        print "Running configure for %s"%(name,)
700
        runCommand(' '.join(configure_args) + ' 2>&1')
701

702
    print "Running install for %s"%(name,)
703 704
    runCommand('{ ' + install + ' ;} 2>&1')

705 706
    print "Done %s"%(name,)
    print ""
707 708 709 710 711 712 713

    os.chdir(curdir)

def buildLibraries():
    """
    Build our dependencies into $WORKDIR/libraries/usr/local
    """
714 715 716
    print ""
    print "Building required libraries"
    print ""
717 718 719 720 721
    universal = os.path.join(WORKDIR, 'libraries')
    os.mkdir(universal)
    os.makedirs(os.path.join(universal, 'usr', 'local', 'lib'))
    os.makedirs(os.path.join(universal, 'usr', 'local', 'include'))

722
    for recipe in library_recipes():
723
        buildRecipe(recipe, universal, ARCHLIST)
724 725 726 727



def buildPythonDocs():
728
    # This stores the documentation as Resources/English.lproj/Documentation
729
    # inside the framwork. pydoc and IDLE will pick it up there.
730
    print "Install python documentation"
731
    rootDir = os.path.join(WORKDIR, '_root')
732
    buildDir = os.path.join('../../Doc')
733
    docdir = os.path.join(rootDir, 'pydocs')
734 735 736
    curDir = os.getcwd()
    os.chdir(buildDir)
    runCommand('make update')
737
    runCommand("make html PYTHON='%s'" % os.path.abspath(sys.executable))
738
    os.chdir(curDir)
739 740
    if not os.path.exists(docdir):
        os.mkdir(docdir)
Ronald Oussoren's avatar
Ronald Oussoren committed
741
    os.rename(os.path.join(buildDir, 'build', 'html'), docdir)
742 743 744


def buildPython():
745
    print "Building a universal python for %s architectures" % UNIVERSALARCHS
746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767

    buildDir = os.path.join(WORKDIR, '_bld', 'python')
    rootDir = os.path.join(WORKDIR, '_root')

    if os.path.exists(buildDir):
        shutil.rmtree(buildDir)
    if os.path.exists(rootDir):
        shutil.rmtree(rootDir)
    os.mkdir(buildDir)
    os.mkdir(rootDir)
    os.mkdir(os.path.join(rootDir, 'empty-dir'))
    curdir = os.getcwd()
    os.chdir(buildDir)

    # Not sure if this is still needed, the original build script
    # claims that parts of the install assume python.exe exists.
    os.symlink('python', os.path.join(buildDir, 'python.exe'))

    # Extract the version from the configure file, needed to calculate
    # several paths.
    version = getVersion()

768 769 770 771 772
    # Since the extra libs are not in their installed framework location
    # during the build, augment the library path so that the interpreter
    # will find them during its extension import sanity checks.
    os.environ['DYLD_LIBRARY_PATH'] = os.path.join(WORKDIR,
                                        'libraries', 'usr', 'local', 'lib')
773
    print "Running configure..."
774
    runCommand("%s -C --enable-framework --enable-universalsdk=%s "
775 776
               "--with-universal-archs=%s "
               "%s "
777 778 779 780
               "LDFLAGS='-g -L%s/libraries/usr/local/lib' "
               "OPT='-g -O3 -I%s/libraries/usr/local/include' 2>&1"%(
        shellQuote(os.path.join(SRCDIR, 'configure')), shellQuote(SDKPATH),
        UNIVERSALARCHS,
781
        (' ', '--with-computed-gotos ')[PYTHON_3],
782
        shellQuote(WORKDIR)[1:-1],
783 784
        shellQuote(WORKDIR)[1:-1]))

785
    print "Running make"
786 787
    runCommand("make")

788
    print "Running make install"
Ronald Oussoren's avatar
Ronald Oussoren committed
789
    runCommand("make install DESTDIR=%s"%(
790 791
        shellQuote(rootDir)))

792
    print "Running make frameworkinstallextras"
793 794 795
    runCommand("make frameworkinstallextras DESTDIR=%s"%(
        shellQuote(rootDir)))

796
    del os.environ['DYLD_LIBRARY_PATH']
797
    print "Copying required shared libraries"
798 799 800 801 802 803 804 805 806 807
    if os.path.exists(os.path.join(WORKDIR, 'libraries', 'Library')):
        runCommand("mv %s/* %s"%(
            shellQuote(os.path.join(
                WORKDIR, 'libraries', 'Library', 'Frameworks',
                'Python.framework', 'Versions', getVersion(),
                'lib')),
            shellQuote(os.path.join(WORKDIR, '_root', 'Library', 'Frameworks',
                'Python.framework', 'Versions', getVersion(),
                'lib'))))

808
    print "Fix file modes"
809
    frmDir = os.path.join(rootDir, 'Library', 'Frameworks', 'Python.framework')
810 811
    gid = grp.getgrnam('admin').gr_gid

812 813 814
    for dirpath, dirnames, filenames in os.walk(frmDir):
        for dn in dirnames:
            os.chmod(os.path.join(dirpath, dn), 0775)
815 816
            os.chown(os.path.join(dirpath, dn), -1, gid)

817 818 819 820 821 822 823 824

        for fn in filenames:
            if os.path.islink(fn):
                continue

            # "chmod g+w $fn"
            p = os.path.join(dirpath, fn)
            st = os.stat(p)
825 826
            os.chmod(p, stat.S_IMODE(st.st_mode) | stat.S_IWGRP)
            os.chown(p, -1, gid)
827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857

    # We added some directories to the search path during the configure
    # phase. Remove those because those directories won't be there on
    # the end-users system.
    path =os.path.join(rootDir, 'Library', 'Frameworks', 'Python.framework',
                'Versions', version, 'lib', 'python%s'%(version,),
                'config', 'Makefile')
    fp = open(path, 'r')
    data = fp.read()
    fp.close()

    data = data.replace('-L%s/libraries/usr/local/lib'%(WORKDIR,), '')
    data = data.replace('-I%s/libraries/usr/local/include'%(WORKDIR,), '')
    fp = open(path, 'w')
    fp.write(data)
    fp.close()

    # Add symlinks in /usr/local/bin, using relative links
    usr_local_bin = os.path.join(rootDir, 'usr', 'local', 'bin')
    to_framework = os.path.join('..', '..', '..', 'Library', 'Frameworks',
            'Python.framework', 'Versions', version, 'bin')
    if os.path.exists(usr_local_bin):
        shutil.rmtree(usr_local_bin)
    os.makedirs(usr_local_bin)
    for fn in os.listdir(
                os.path.join(frmDir, 'Versions', version, 'bin')):
        os.symlink(os.path.join(to_framework, fn),
                   os.path.join(usr_local_bin, fn))

    os.chdir(curdir)

858 859 860 861 862
    if PYTHON_3:
        # Remove the 'Current' link, that way we don't accidently mess
        # with an already installed version of python 2
        os.unlink(os.path.join(rootDir, 'Library', 'Frameworks',
                            'Python.framework', 'Versions', 'Current'))
863 864 865 866 867

def patchFile(inPath, outPath):
    data = fileContents(inPath)
    data = data.replace('$FULL_VERSION', getFullVersion())
    data = data.replace('$VERSION', getVersion())
868
    data = data.replace('$MACOSX_DEPLOYMENT_TARGET', ''.join((DEPTARGET, ' or later')))
869
    data = data.replace('$ARCHITECTURES', ", ".join(universal_opts_map[UNIVERSALARCHS]))
870
    data = data.replace('$INSTALL_SIZE', installSize())
871 872 873

    # This one is not handy as a template variable
    data = data.replace('$PYTHONFRAMEWORKINSTALLDIR', '/Library/Frameworks/Python.framework')
874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890
    fp = open(outPath, 'wb')
    fp.write(data)
    fp.close()

def patchScript(inPath, outPath):
    data = fileContents(inPath)
    data = data.replace('@PYVER@', getVersion())
    fp = open(outPath, 'wb')
    fp.write(data)
    fp.close()
    os.chmod(outPath, 0755)



def packageFromRecipe(targetDir, recipe):
    curdir = os.getcwd()
    try:
891 892
        # The major version (such as 2.5) is included in the package name
        # because having two version of python installed at the same time is
893 894
        # common.
        pkgname = '%s-%s'%(recipe['name'], getVersion())
895 896 897 898 899 900
        srcdir  = recipe.get('source')
        pkgroot = recipe.get('topdir', srcdir)
        postflight = recipe.get('postflight')
        readme = textwrap.dedent(recipe['readme'])
        isRequired = recipe.get('required', True)

901
        print "- building package %s"%(pkgname,)
902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947

        # Substitute some variables
        textvars = dict(
            VER=getVersion(),
            FULLVER=getFullVersion(),
        )
        readme = readme % textvars

        if pkgroot is not None:
            pkgroot = pkgroot % textvars
        else:
            pkgroot = '/'

        if srcdir is not None:
            srcdir = os.path.join(WORKDIR, '_root', srcdir[1:])
            srcdir = srcdir % textvars

        if postflight is not None:
            postflight = os.path.abspath(postflight)

        packageContents = os.path.join(targetDir, pkgname + '.pkg', 'Contents')
        os.makedirs(packageContents)

        if srcdir is not None:
            os.chdir(srcdir)
            runCommand("pax -wf %s . 2>&1"%(shellQuote(os.path.join(packageContents, 'Archive.pax')),))
            runCommand("gzip -9 %s 2>&1"%(shellQuote(os.path.join(packageContents, 'Archive.pax')),))
            runCommand("mkbom . %s 2>&1"%(shellQuote(os.path.join(packageContents, 'Archive.bom')),))

        fn = os.path.join(packageContents, 'PkgInfo')
        fp = open(fn, 'w')
        fp.write('pmkrpkg1')
        fp.close()

        rsrcDir = os.path.join(packageContents, "Resources")
        os.mkdir(rsrcDir)
        fp = open(os.path.join(rsrcDir, 'ReadMe.txt'), 'w')
        fp.write(readme)
        fp.close()

        if postflight is not None:
            patchScript(postflight, os.path.join(rsrcDir, 'postflight'))

        vers = getFullVersion()
        major, minor = map(int, getVersion().split('.', 2))
        pl = Plist(
948 949 950
                CFBundleGetInfoString="Python.%s %s"%(pkgname, vers,),
                CFBundleIdentifier='org.python.Python.%s'%(pkgname,),
                CFBundleName='Python.%s'%(pkgname,),
951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970
                CFBundleShortVersionString=vers,
                IFMajorVersion=major,
                IFMinorVersion=minor,
                IFPkgFormatVersion=0.10000000149011612,
                IFPkgFlagAllowBackRev=False,
                IFPkgFlagAuthorizationAction="RootAuthorization",
                IFPkgFlagDefaultLocation=pkgroot,
                IFPkgFlagFollowLinks=True,
                IFPkgFlagInstallFat=True,
                IFPkgFlagIsRequired=isRequired,
                IFPkgFlagOverwritePermissions=False,
                IFPkgFlagRelocatable=False,
                IFPkgFlagRestartAction="NoRestart",
                IFPkgFlagRootVolumeOnly=True,
                IFPkgFlagUpdateInstalledLangauges=False,
            )
        writePlist(pl, os.path.join(packageContents, 'Info.plist'))

        pl = Plist(
                    IFPkgDescriptionDescription=readme,
971
                    IFPkgDescriptionTitle=recipe.get('long_name', "Python.%s"%(pkgname,)),
972 973 974 975 976 977 978 979 980 981 982 983 984 985
                    IFPkgDescriptionVersion=vers,
                )
        writePlist(pl, os.path.join(packageContents, 'Resources', 'Description.plist'))

    finally:
        os.chdir(curdir)


def makeMpkgPlist(path):

    vers = getFullVersion()
    major, minor = map(int, getVersion().split('.', 2))

    pl = Plist(
986 987 988
            CFBundleGetInfoString="Python %s"%(vers,),
            CFBundleIdentifier='org.python.Python',
            CFBundleName='Python',
989 990 991 992 993 994
            CFBundleShortVersionString=vers,
            IFMajorVersion=major,
            IFMinorVersion=minor,
            IFPkgFlagComponentDirectory="Contents/Packages",
            IFPkgFlagPackageList=[
                dict(
995
                    IFPkgFlagPackageLocation='%s-%s.pkg'%(item['name'], getVersion()),
996
                    IFPkgFlagPackageSelection=item.get('selected', 'selected'),
997
                )
998
                for item in pkg_recipes()
999 1000 1001 1002
            ],
            IFPkgFormatVersion=0.10000000149011612,
            IFPkgFlagBackgroundScaling="proportional",
            IFPkgFlagBackgroundAlignment="left",
1003
            IFPkgFlagAuthorizationAction="RootAuthorization",
1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021
        )

    writePlist(pl, path)


def buildInstaller():

    # Zap all compiled files
    for dirpath, _, filenames in os.walk(os.path.join(WORKDIR, '_root')):
        for fn in filenames:
            if fn.endswith('.pyc') or fn.endswith('.pyo'):
                os.unlink(os.path.join(dirpath, fn))

    outdir = os.path.join(WORKDIR, 'installer')
    if os.path.exists(outdir):
        shutil.rmtree(outdir)
    os.mkdir(outdir)

1022
    pkgroot = os.path.join(outdir, 'Python.mpkg', 'Contents')
1023 1024
    pkgcontents = os.path.join(pkgroot, 'Packages')
    os.makedirs(pkgcontents)
1025
    for recipe in pkg_recipes():
1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038
        packageFromRecipe(pkgcontents, recipe)

    rsrcDir = os.path.join(pkgroot, 'Resources')

    fn = os.path.join(pkgroot, 'PkgInfo')
    fp = open(fn, 'w')
    fp.write('pmkrpkg1')
    fp.close()

    os.mkdir(rsrcDir)

    makeMpkgPlist(os.path.join(pkgroot, 'Info.plist'))
    pl = Plist(
1039
                IFPkgDescriptionTitle="Python",
1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050
                IFPkgDescriptionVersion=getVersion(),
            )

    writePlist(pl, os.path.join(pkgroot, 'Resources', 'Description.plist'))
    for fn in os.listdir('resources'):
        if fn == '.svn': continue
        if fn.endswith('.jpg'):
            shutil.copy(os.path.join('resources', fn), os.path.join(rsrcDir, fn))
        else:
            patchFile(os.path.join('resources', fn), os.path.join(rsrcDir, fn))

1051
    shutil.copy("../../LICENSE", os.path.join(rsrcDir, 'License.txt'))
1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065


def installSize(clear=False, _saved=[]):
    if clear:
        del _saved[:]
    if not _saved:
        data = captureCommand("du -ks %s"%(
                    shellQuote(os.path.join(WORKDIR, '_root'))))
        _saved.append("%d"%((0.5 + (int(data.split()[0]) / 1024.0)),))
    return _saved[0]


def buildDMG():
    """
1066
    Create DMG containing the rootDir.
1067 1068 1069 1070 1071 1072
    """
    outdir = os.path.join(WORKDIR, 'diskimage')
    if os.path.exists(outdir):
        shutil.rmtree(outdir)

    imagepath = os.path.join(outdir,
1073
                    'python-%s-macosx%s'%(getFullVersion(),DEPTARGET))
1074
    if INCLUDE_TIMESTAMP:
1075
        imagepath = imagepath + '-%04d-%02d-%02d'%(time.localtime()[:3])
1076 1077 1078
    imagepath = imagepath + '.dmg'

    os.mkdir(outdir)
1079 1080 1081
    volname='Python %s'%(getFullVersion())
    runCommand("hdiutil create -format UDRW -volname %s -srcfolder %s %s"%(
            shellQuote(volname),
1082
            shellQuote(os.path.join(WORKDIR, 'installer')),
1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104
            shellQuote(imagepath + ".tmp.dmg" )))


    if not os.path.exists(os.path.join(WORKDIR, "mnt")):
        os.mkdir(os.path.join(WORKDIR, "mnt"))
    runCommand("hdiutil attach %s -mountroot %s"%(
        shellQuote(imagepath + ".tmp.dmg"), shellQuote(os.path.join(WORKDIR, "mnt"))))

    # Custom icon for the DMG, shown when the DMG is mounted.
    shutil.copy("../Icons/Disk Image.icns",
            os.path.join(WORKDIR, "mnt", volname, ".VolumeIcon.icns"))
    runCommand("/Developer/Tools/SetFile -a C %s/"%(
            shellQuote(os.path.join(WORKDIR, "mnt", volname)),))

    runCommand("hdiutil detach %s"%(shellQuote(os.path.join(WORKDIR, "mnt", volname))))

    setIcon(imagepath + ".tmp.dmg", "../Icons/Disk Image.icns")
    runCommand("hdiutil convert %s -format UDZO -o %s"%(
            shellQuote(imagepath + ".tmp.dmg"), shellQuote(imagepath)))
    setIcon(imagepath, "../Icons/Disk Image.icns")

    os.unlink(imagepath + ".tmp.dmg")
1105 1106 1107 1108 1109 1110 1111 1112 1113

    return imagepath


def setIcon(filePath, icnsPath):
    """
    Set the custom icon for the specified file or directory.
    """

1114 1115
    dirPath = os.path.normpath(os.path.dirname(__file__))
    toolPath = os.path.join(dirPath, "seticon.app/Contents/MacOS/seticon")
1116 1117 1118
    if not os.path.exists(toolPath) or os.stat(toolPath).st_mtime < os.stat(dirPath + '/seticon.m').st_mtime:
        # NOTE: The tool is created inside an .app bundle, otherwise it won't work due
        # to connections to the window server.
1119 1120 1121
        appPath = os.path.join(dirPath, "seticon.app/Contents/MacOS")
        if not os.path.exists(appPath):
            os.makedirs(appPath)
1122 1123
        runCommand("cc -o %s %s/seticon.m -framework Cocoa"%(
            shellQuote(toolPath), shellQuote(dirPath)))
1124

1125 1126
    runCommand("%s %s %s"%(shellQuote(os.path.abspath(toolPath)), shellQuote(icnsPath),
        shellQuote(filePath)))
1127 1128 1129 1130 1131 1132

def main():
    # First parse options and check if we can perform our work
    parseOptions()
    checkEnvironment()

1133
    os.environ['MACOSX_DEPLOYMENT_TARGET'] = DEPTARGET
1134
    os.environ['CC'] = CC
1135 1136 1137 1138 1139

    if os.path.exists(WORKDIR):
        shutil.rmtree(WORKDIR)
    os.mkdir(WORKDIR)

1140 1141
    os.environ['LC_ALL'] = 'C'

1142 1143 1144 1145 1146
    # Then build third-party libraries such as sleepycat DB4.
    buildLibraries()

    # Now build python itself
    buildPython()
1147 1148 1149 1150 1151 1152 1153 1154

    # And then build the documentation
    # Remove the Deployment Target from the shell
    # environment, it's no longer needed and
    # an unexpected build target can cause problems
    # when Sphinx and its dependencies need to
    # be (re-)installed.
    del os.environ['MACOSX_DEPLOYMENT_TARGET']
1155
    buildPythonDocs()
1156 1157 1158


    # Prepare the applications folder
1159
    fn = os.path.join(WORKDIR, "_root", "Applications",
Benjamin Peterson's avatar
Benjamin Peterson committed
1160
                "Python %s"%(getVersion(),), "Update Shell Profile.command")
1161
    patchScript("scripts/postflight.patch-profile",  fn)
1162

Benjamin Peterson's avatar
Benjamin Peterson committed
1163
    folder = os.path.join(WORKDIR, "_root", "Applications", "Python %s"%(
1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174
        getVersion(),))
    os.chmod(folder, 0755)
    setIcon(folder, "../Icons/Python Folder.icns")

    # Create the installer
    buildInstaller()

    # And copy the readme into the directory containing the installer
    patchFile('resources/ReadMe.txt', os.path.join(WORKDIR, 'installer', 'ReadMe.txt'))

    # Ditto for the license file.
1175
    shutil.copy('../../LICENSE', os.path.join(WORKDIR, 'installer', 'License.txt'))
1176 1177

    fp = open(os.path.join(WORKDIR, 'installer', 'Build.txt'), 'w')
1178 1179 1180
    print >> fp, "# BUILD INFO"
    print >> fp, "# Date:", time.ctime()
    print >> fp, "# By:", pwd.getpwuid(os.getuid()).pw_gecos
1181 1182 1183 1184 1185 1186 1187
    fp.close()

    # And copy it to a DMG
    buildDMG()

if __name__ == "__main__":
    main()