cmd/go, go/build, misc/swig: add SWIG support to Go tool

R=adg, rsc, franciscossouza, seb.binet, gen.battle
CC=golang-dev
https://golang.org/cl/5845071

api/next.txt

@@ -23,6 +23,8 @@ pkg go/ast, method (CommentMap) Filter(Node) CommentMap
 pkg go/ast, method (CommentMap) String() string
 pkg go/ast, method (CommentMap) Update(Node) Node
 pkg go/ast, type CommentMap map[Node][]*CommentGroup
+pkg go/build, type Package struct, SwigCXXFiles []string
+pkg go/build, type Package struct, SwigFiles []string
 pkg go/doc, var IllegalPrefixes []string
 pkg image, const YCbCrSubsampleRatio440 YCbCrSubsampleRatio
 pkg math/big, method (*Int) MarshalJSON() ([]byte, error)

misc/swig/callback/Makefile

-# Copyright 2011 The Go Authors.  All rights reserved.
-# Use of this source code is governed by a BSD-style
-# license that can be found in the LICENSE file.
-
-include ../../../src/Make.inc
-
-TARG=swig/callback
-SWIGFILES=\
-	callback.swigcxx
-
-CLEANFILES+=run
-
-include ../../../src/Make.pkg
-
-%: install %.go
-	$(GC) $(GCFLAGS) $(GCIMPORTS) $*.go
-	$(LD) $(SWIG_RPATH) -o $@ $*.$O

misc/swig/stdio/hello.go → misc/swig/callback/callback.go

-// Copyright 2011 The Go Authors.  All rights reserved.
+// Copyright 2012 The Go Authors.  All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package main
+package callback
 
-import "swig/file"
+type GoCallback struct{}
 
-func main() {
-	file.Puts("Hello, world")
+func (p *GoCallback) Run() string {
+	return "GoCallback.Run"
 }

misc/swig/callback/run.go → misc/swig/callback/callback_test.go

-// Copyright 2011 The Go Authors.  All rights reserved.
+// Copyright 2012 The Go Authors.  All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-package main
+package callback_test
 
 import (
-	"fmt"
-	"swig/callback"
+	"../callback"
+	"testing"
 )
 
-type GoCallback struct{}
-
-func (p *GoCallback) Run() string {
-	return "GoCallback.Run"
-}
-
-func main() {
+func TestCall(t *testing.T) {
 	c := callback.NewCaller()
 	cb := callback.NewCallback()
 
 	c.SetCallback(cb)
 	s := c.Call()
-	fmt.Println(s)
 	if s != "Callback::run" {
-		panic(s)
+		t.Errorf("unexpected string from Call: %q", s)
 	}
 	c.DelCallback()
+}
 
-	cb = callback.NewDirectorCallback(&GoCallback{})
+func TestCallback(t *testing.T) {
+	c := callback.NewCaller()
+	cb := callback.NewDirectorCallback(&callback.GoCallback{})
 	c.SetCallback(cb)
-	s = c.Call()
-	fmt.Println(s)
+	s := c.Call()
 	if s != "GoCallback.Run" {
-		panic(s)
+		t.Errorf("unexpected string from Call with callback: %q", s)
 	}
 	c.DelCallback()
 	callback.DeleteDirectorCallback(cb)

misc/swig/stdio/file.swig

@@ -6,6 +6,19 @@
 
 %{
 #include <stdio.h>
+#include <stdlib.h>
 %}
 
-int puts(const char *);
+%typemap(gotype) const char * "string"
+%typemap(in) const char * %{
+	$1 = malloc($input.n + 1);
+	memcpy($1, $input.p, $input.n);
+	$1[$input.n] = '\0';
+%}
+%typemap(freearg) const char * %{
+	free($1);
+%}
+
+FILE *fopen(const char *name, const char *mode);
+int fclose(FILE *);
+int fgetc(FILE *);

misc/swig/stdio/file_test.go

+// Copyright 2012 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package file
+
+import "testing"
+
+// Open this file itself and verify that the first few characters are
+// as expected.
+func TestRead(t *testing.T) {
+	f := Fopen("file_test.go", "r")
+	if f == nil {
+		t.Fatal("fopen failed")
+	}
+	if Fgetc(f) != '/' || Fgetc(f) != '/' || Fgetc(f) != ' ' || Fgetc(f) != 'C' {
+		t.Error("read unexpected characters")
+	}
+	if Fclose(f) != 0 {
+		t.Error("fclose failed")
+	}
+}

src/cmd/go/build.go

@@ -684,6 +684,21 @@ func (b *builder) build(a *action) (err error) {
 		gofiles = append(gofiles, outGo...)
 	}
 
+	// Run SWIG.
+	if a.p.usesSwig() {
+		// In a package using SWIG, any .c or .s files are
+		// compiled with gcc.
+		gccfiles := append(cfiles, sfiles...)
+		cfiles = nil
+		sfiles = nil
+		outGo, outObj, err := b.swig(a.p, obj, gccfiles)
+		if err != nil {
+			return err
+		}
+		cgoObjects = append(cgoObjects, outObj...)
+		gofiles = append(gofiles, outGo...)
+	}
+
 	// Prepare Go import path list.
 	inc := b.includeArgs("-I", a.deps)
 
@@ -799,6 +814,20 @@ func (b *builder) install(a *action) (err error) {
 		defer os.Remove(a1.target)
 	}
 
+	if a.p.usesSwig() {
+		for _, f := range stringList(a.p.SwigFiles, a.p.SwigCXXFiles) {
+			dir = a.p.swigDir(&buildContext)
+			if err := b.mkdir(dir); err != nil {
+				return err
+			}
+			soname := a.p.swigSoname(f)
+			target := filepath.Join(dir, soname)
+			if err = b.copyFile(a, target, soname, perm); err != nil {
+				return err
+			}
+		}
+	}
+
 	return b.copyFile(a, a.target, a1.target, perm)
 }
 
@@ -1275,7 +1304,21 @@ func (gcToolchain) pack(b *builder, p *Package, objDir, afile string, ofiles []s
 
 func (gcToolchain) ld(b *builder, p *Package, out string, allactions []*action, mainpkg string, ofiles []string) error {
 	importArgs := b.includeArgs("-L", allactions)
-	return b.run(".", p.ImportPath, tool(archChar+"l"), "-o", out, importArgs, buildLdflags, mainpkg)
+	swigDirs := make(map[string]bool)
+	swigArg := []string{}
+	for _, a := range allactions {
+		if a.p != nil && a.p.usesSwig() {
+			sd := a.p.swigDir(&buildContext)
+			if len(swigArg) == 0 {
+				swigArg = []string{"-r", sd}
+			} else if !swigDirs[sd] {
+				swigArg[1] += ":"
+				swigArg[1] += sd
+			}
+			swigDirs[sd] = true
+		}
+	}
+	return b.run(".", p.ImportPath, tool(archChar+"l"), "-o", out, importArgs, swigArg, buildLdflags, mainpkg)
 }
 
 func (gcToolchain) cc(b *builder, p *Package, objdir, ofile, cfile string) error {
@@ -1336,6 +1379,7 @@ func (tools gccgcToolchain) ld(b *builder, p *Package, out string, allactions []
 	// gccgo needs explicit linking with all package dependencies,
 	// and all LDFLAGS from cgo dependencies.
 	afiles := make(map[*Package]string)
+	sfiles := make(map[*Package][]string)
 	ldflags := []string{}
 	cgoldflags := []string{}
 	for _, a := range allactions {
@@ -1346,11 +1390,21 @@ func (tools gccgcToolchain) ld(b *builder, p *Package, out string, allactions []
 				}
 			}
 			cgoldflags = append(cgoldflags, a.p.CgoLDFLAGS...)
+			if a.p.usesSwig() {
+				sd := a.p.swigDir(&buildContext)
+				for _, f := range stringList(a.p.SwigFiles, a.p.SwigCXXFiles) {
+					soname := a.p.swigSoname(f)
+					sfiles[a.p] = append(sfiles[a.p], filepath.Join(sd, soname))
+				}
+			}
 		}
 	}
 	for _, afile := range afiles {
 		ldflags = append(ldflags, afile)
 	}
+	for _, sfiles := range sfiles {
+		ldflags = append(ldflags, sfiles...)
+	}
 	ldflags = append(ldflags, cgoldflags...)
 	return b.run(".", p.ImportPath, "gccgo", "-o", out, buildGccgoflags, ofiles, "-Wl,-(", ldflags, "-Wl,-)")
 }
@@ -1558,6 +1612,104 @@ func (b *builder) cgo(p *Package, cgoExe, obj string, gccfiles []string) (outGo,
 	return outGo, outObj, nil
 }
 
+// Run SWIG on all SWIG input files.
+func (b *builder) swig(p *Package, obj string, gccfiles []string) (outGo, outObj []string, err error) {
+	for _, f := range p.SwigFiles {
+		goFile, objFile, err := b.swigOne(p, f, obj, false)
+		if err != nil {
+			return nil, nil, err
+		}
+		if goFile != "" {
+			outGo = append(outGo, goFile)
+		}
+		if objFile != "" {
+			outObj = append(outObj, objFile)
+		}
+	}
+	for _, f := range p.SwigCXXFiles {
+		goFile, objFile, err := b.swigOne(p, f, obj, true)
+		if err != nil {
+			return nil, nil, err
+		}
+		if goFile != "" {
+			outGo = append(outGo, goFile)
+		}
+		if objFile != "" {
+			outObj = append(outObj, objFile)
+		}
+	}
+	return outGo, outObj, nil
+}
+
+// Run SWIG on one SWIG input file.
+func (b *builder) swigOne(p *Package, file, obj string, cxx bool) (outGo, outObj string, err error) {
+	n := 5 // length of ".swig"
+	if cxx {
+		n = 8 // length of ".swigcxx"
+	}
+	base := file[:len(file)-n]
+	goFile := base + ".go"
+	cBase := base + "_gc."
+	gccBase := base + "_wrap."
+	gccExt := "c"
+	if cxx {
+		gccExt = "cxx"
+	}
+	soname := p.swigSoname(file)
+
+	_, gccgo := buildToolchain.(gccgcToolchain)
+
+	// swig
+	args := []string{
+		"-go",
+		"-module", base,
+		"-soname", soname,
+		"-o", obj + gccBase + gccExt,
+		"-outdir", obj,
+	}
+	if gccgo {
+		args = append(args, "-gccgo")
+	}
+	if cxx {
+		args = append(args, "-c++")
+	}
+
+	if err := b.run(p.Dir, p.ImportPath, "swig", args, file); err != nil {
+		return "", "", err
+	}
+
+	var cObj string
+	if !gccgo {
+		// cc
+		cObj = obj + cBase + archChar
+		if err := buildToolchain.cc(b, p, obj, cObj, obj+cBase+"c"); err != nil {
+			return "", "", err
+		}
+	}
+
+	// gcc
+	gccObj := obj + gccBase + "o"
+	if err := b.gcc(p, gccObj, []string{"-g", "-fPIC", "-O2"}, obj+gccBase+gccExt); err != nil {
+		return "", "", err
+	}
+
+	// create shared library
+	osldflags := map[string][]string{
+		"darwin":  []string{"-dynamiclib", "-Wl,-undefined,dynamic_lookup"},
+		"freebsd": []string{"-shared", "-lpthread", "-lm"},
+		"linux":   []string{"-shared", "-lpthread", "-lm"},
+		"windows": []string{"-shared", "-lm", "-mthreads"},
+	}
+	var cxxlib []string
+	if cxx {
+		cxxlib = []string{"-lstdc++"}
+	}
+	ldflags := stringList(osldflags[goos], cxxlib)
+	b.run(p.Dir, p.ImportPath, b.gccCmd(p.Dir), "-o", soname, gccObj, ldflags)
+
+	return obj + goFile, cObj, nil
+}
+
 // An actionQueue is a priority queue of actions.
 type actionQueue []*action
 

src/cmd/go/clean.go

@@ -34,6 +34,7 @@ source directories corresponding to the import paths:
 	DIR(.exe)        from go build
 	DIR.test(.exe)   from go test -c
 	MAINFILE(.exe)   from go build MAINFILE.go
+	*.so             from SWIG
 
 In the list, DIR represents the final path element of the
 directory, and MAINFILE is the base name of any Go source
@@ -93,11 +94,12 @@ var cleanFile = map[string]bool{
 }
 
 var cleanExt = map[string]bool{
-	".5": true,
-	".6": true,
-	".8": true,
-	".a": true,
-	".o": true,
+	".5":  true,
+	".6":  true,
+	".8":  true,
+	".a":  true,
+	".o":  true,
+	".so": true,
 }
 
 func clean(p *Package) {
@@ -191,6 +193,20 @@ func clean(p *Package) {
 		}
 	}
 
+	if cleanI && p.usesSwig() {
+		for _, f := range stringList(p.SwigFiles, p.SwigCXXFiles) {
+			dir := p.swigDir(&buildContext)
+			soname := p.swigSoname(f)
+			target := filepath.Join(dir, soname)
+			if cleanN || cleanX {
+				b.showcmd("", "rm -f %s", target)
+			}
+			if !cleanN {
+				os.Remove(target)
+			}
+		}
+	}
+
 	if cleanR {
 		for _, p1 := range p.imports {
 			clean(p1)

src/cmd/go/doc.go

@@ -293,12 +293,14 @@ being passed to the template is:
         Root       string // Go root or Go path dir containing this package
 
         // Source files
-        GoFiles  []string  // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles)
-        CgoFiles []string  // .go sources files that import "C"
-        CFiles   []string  // .c source files
-        HFiles   []string  // .h source files
-        SFiles   []string  // .s source files
-        SysoFiles []string // .syso object files to add to archive
+        GoFiles  []string     // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles)
+        CgoFiles []string     // .go sources files that import "C"
+        CFiles   []string     // .c source files
+        HFiles   []string     // .h source files
+        SFiles   []string     // .s source files
+        SysoFiles []string    // .syso object files to add to archive
+        SwigFiles []string    // .swig files
+        SwigCXXFiles []string // .swigcxx files
 
         // Cgo directives
         CgoCFLAGS    []string // cgo: flags for C compiler

src/cmd/go/list.go

@@ -41,12 +41,14 @@ being passed to the template is:
         Root       string // Go root or Go path dir containing this package
 
         // Source files
-        GoFiles  []string  // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles)
-        CgoFiles []string  // .go sources files that import "C"
-        CFiles   []string  // .c source files
-        HFiles   []string  // .h source files
-        SFiles   []string  // .s source files
-        SysoFiles []string // .syso object files to add to archive
+        GoFiles  []string     // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles)
+        CgoFiles []string     // .go sources files that import "C"
+        CFiles   []string     // .c source files
+        HFiles   []string     // .h source files
+        SFiles   []string     // .s source files
+        SysoFiles []string    // .syso object files to add to archive
+        SwigFiles []string    // .swig files
+        SwigCXXFiles []string // .swigcxx files
 
         // Cgo directives
         CgoCFLAGS    []string // cgo: flags for C compiler

src/cmd/go/pkg.go

@@ -36,12 +36,14 @@ type Package struct {
 	Root       string `json:",omitempty"` // Go root or Go path dir containing this package
 
 	// Source files
-	GoFiles   []string `json:",omitempty"` // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles)
-	CgoFiles  []string `json:",omitempty"` // .go sources files that import "C"
-	CFiles    []string `json:",omitempty"` // .c source files
-	HFiles    []string `json:",omitempty"` // .h source files
-	SFiles    []string `json:",omitempty"` // .s source files
-	SysoFiles []string `json:",omitempty"` // .syso system object files added to package
+	GoFiles      []string `json:",omitempty"` // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles)
+	CgoFiles     []string `json:",omitempty"` // .go sources files that import "C"
+	CFiles       []string `json:",omitempty"` // .c source files
+	HFiles       []string `json:",omitempty"` // .h source files
+	SFiles       []string `json:",omitempty"` // .s source files
+	SysoFiles    []string `json:",omitempty"` // .syso system object files added to package
+	SwigFiles    []string `json:",omitempty"` // .swig files
+	SwigCXXFiles []string `json:",omitempty"` // .swigcxx files
 
 	// Cgo directives
 	CgoCFLAGS    []string `json:",omitempty"` // cgo: flags for C compiler
@@ -94,6 +96,8 @@ func (p *Package) copyBuild(pp *build.Package) {
 	p.HFiles = pp.HFiles
 	p.SFiles = pp.SFiles
 	p.SysoFiles = pp.SysoFiles
+	p.SwigFiles = pp.SwigFiles
+	p.SwigCXXFiles = pp.SwigCXXFiles
 	p.CgoCFLAGS = pp.CgoCFLAGS
 	p.CgoLDFLAGS = pp.CgoLDFLAGS
 	p.CgoPkgConfig = pp.CgoPkgConfig
@@ -408,6 +412,29 @@ func (p *Package) load(stk *importStack, bp *build.Package, err error) *Package
 	return p
 }
 
+// usesSwig returns whether the package needs to run SWIG.
+func (p *Package) usesSwig() bool {
+	return len(p.SwigFiles) > 0 || len(p.SwigCXXFiles) > 0
+}
+
+// swigSoname returns the name of the shared library we create for a
+// SWIG input file.
+func (p *Package) swigSoname(file string) string {
+	return strings.Replace(p.ImportPath, "/", "-", -1) + "-" + strings.Replace(file, ".", "-", -1) + ".so"
+}
+
+// swigDir returns the name of the shared SWIG directory for a
+// package.
+func (p *Package) swigDir(ctxt *build.Context) string {
+	dir := p.build.PkgRoot
+	if ctxt.Compiler == "gccgo" {
+		dir = filepath.Join(dir, "gccgo")
+	} else {
+		dir = filepath.Join(dir, ctxt.GOOS+"_"+ctxt.GOARCH)
+	}
+	return filepath.Join(dir, "swig")
+}
+
 // packageList returns the list of packages in the dag rooted at roots
 // as visited in a depth-first post-order traversal.
 func packageList(roots []*Package) []*Package {
@@ -459,7 +486,7 @@ func isStale(p *Package, topRoot map[string]bool) bool {
 	// distributions of Go packages, although such binaries are
 	// only useful with the specific version of the toolchain that
 	// created them.
-	if len(p.gofiles) == 0 {
+	if len(p.gofiles) == 0 && !p.usesSwig() {
 		return false
 	}
 
@@ -522,6 +549,21 @@ func isStale(p *Package, topRoot map[string]bool) bool {
 		}
 	}
 
+	for _, src := range stringList(p.SwigFiles, p.SwigCXXFiles) {
+		if olderThan(filepath.Join(p.Dir, src)) {
+			return true
+		}
+		soname := p.swigSoname(src)
+		fi, err := os.Stat(soname)
+		if err != nil {
+			return true
+		}
+		fiSrc, err := os.Stat(src)
+		if err != nil || fiSrc.ModTime().After(fi.ModTime()) {
+			return true
+		}
+	}
+
 	return false
 }
 

src/cmd/go/test.go

@@ -595,6 +595,25 @@ func (b *builder) runTest(a *action) error {
 		cmd.Stderr = &buf
 	}
 
+	// If there are any local SWIG dependencies, we want to load
+	// the shared library from the build directory.
+	if a.p.usesSwig() {
+		env := os.Environ()
+		found := false
+		prefix := "LD_LIBRARY_PATH="
+		for i, v := range env {
+			if strings.HasPrefix(v, prefix) {
+				env[i] = v + ":."
+				found = true
+				break
+			}
+		}
+		if !found {
+			env = append(env, "LD_LIBRARY_PATH=.")
+		}
+		cmd.Env = env
+	}
+
 	t0 := time.Now()
 	err := cmd.Start()
 

src/pkg/go/build/build.go

@@ -281,12 +281,14 @@ type Package struct {
 	PkgObj     string // installed .a file
 
 	// Source files
-	GoFiles   []string // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles)
-	CgoFiles  []string // .go source files that import "C"
-	CFiles    []string // .c source files
-	HFiles    []string // .h source files
-	SFiles    []string // .s source files
-	SysoFiles []string // .syso system object files to add to archive
+	GoFiles      []string // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles)
+	CgoFiles     []string // .go source files that import "C"
+	CFiles       []string // .c source files
+	HFiles       []string // .h source files
+	SFiles       []string // .s source files
+	SysoFiles    []string // .syso system object files to add to archive
+	SwigFiles    []string // .swig files
+	SwigCXXFiles []string // .swigcxx files
 
 	// Cgo directives
 	CgoPkgConfig []string // Cgo pkg-config directives
@@ -489,7 +491,7 @@ Found:
 		}
 		ext := name[i:]
 		switch ext {
-		case ".go", ".c", ".s", ".h", ".S":
+		case ".go", ".c", ".s", ".h", ".S", ".swig", ".swigcxx":
 			// tentatively okay - read to make sure
 		case ".syso":
 			// binary objects to add to package archive
@@ -532,6 +534,12 @@ Found:
 		case ".S":
 			Sfiles = append(Sfiles, name)
 			continue
+		case ".swig":
+			p.SwigFiles = append(p.SwigFiles, name)
+			continue
+		case ".swigcxx":
+			p.SwigCXXFiles = append(p.SwigCXXFiles, name)
+			continue
 		}
 
 		pf, err := parser.ParseFile(fset, filename, data, parser.ImportsOnly|parser.ParseComments)