Skip to content

G
go
Commit 0e5b4eb0 authored by Russ Cox's avatar Russ Cox Committed by Brad Fitzpatrick
Browse files
Options

cmd/dist: build packages in parallel, make code more Go-like 

(Changed modified by bradfitz from original rsc version)

Change-Id: I8ea40044c325f333a13d48b59b4795b02c579533
Reviewed-on: https://go-review.googlesource.com/14026Reviewed-by: default avatarIan Lance Taylor <iant@golang.org>
Run-TryBot: Brad Fitzpatrick <bradfitz@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
parent c73df92b
Hide whitespace changes
Inline Side-by-side
Showing with 162 additions and 105 deletions
src/cmd/dist/build.go
View file @ 0e5b4eb0
...@@ -11,7 +11,9 @@ import ( ...@@ -11,7 +11,9 @@ import (
"os" "os"
"os/exec" "os/exec"
"path/filepath" "path/filepath"
"sort"
"strings" "strings"
"sync"
) )
// Initialization for any invocation. // Initialization for any invocation.
...@@ -487,9 +489,20 @@ var gentab = []struct { ...@@ -487,9 +489,20 @@ var gentab = []struct {
{"anames9.c", nil}, {"anames9.c", nil},
} }
// installed maps from a dir name (as given to install) to a chan
// closed when the dir's package is installed.
var installed = make(map[string]chan struct{})
// install installs the library, package, or binary associated with dir, // install installs the library, package, or binary associated with dir,
// which is relative to $GOROOT/src. // which is relative to $GOROOT/src.
func install(dir string) { func install(dir string) {
if ch, ok := installed[dir]; ok {
defer close(ch)
}
for _, dep := range builddeps[dir] {
<-installed[dep]
}
if vflag > 0 { if vflag > 0 {
if goos != gohostos || goarch != gohostarch { if goos != gohostos || goarch != gohostarch {
errprintf("%s (%s/%s)\n", dir, goos, goarch) errprintf("%s (%s/%s)\n", dir, goos, goarch)
...@@ -498,6 +511,9 @@ func install(dir string) { ...@@ -498,6 +511,9 @@ func install(dir string) {
} }
} }
workdir := pathf("%s/%s", workdir, dir)
xmkdirall(workdir)
var clean []string var clean []string
defer func() { defer func() {
for _, name := range clean { for _, name := range clean {
...@@ -673,6 +689,7 @@ func install(dir string) { ...@@ -673,6 +689,7 @@ func install(dir string) {
run(path, CheckExit|ShowOutput, compile...) run(path, CheckExit|ShowOutput, compile...)
// Compile the files. // Compile the files.
var wg sync.WaitGroup
for _, p := range files { for _, p := range files {
if !strings.HasSuffix(p, ".s") { if !strings.HasSuffix(p, ".s") {
continue continue
...@@ -695,14 +712,14 @@ func install(dir string) { ...@@ -695,14 +712,14 @@ func install(dir string) {
// Change the last character of the output file (which was c or s). // Change the last character of the output file (which was c or s).
b = b[:len(b)-1] + "o" b = b[:len(b)-1] + "o"
compile = append(compile, "-o", b, p) compile = append(compile, "-o", b, p)
bgrun(path, compile...) bgrun(&wg, path, compile...)
link = append(link, b) link = append(link, b)
if doclean { if doclean {
clean = append(clean, b) clean = append(clean, b)
} }
} }
bgwait() bgwait(&wg)
if ispackcmd { if ispackcmd {
xremove(link[targ]) xremove(link[targ])
...@@ -839,62 +856,19 @@ func dopack(dst, src string, extra []string) { ...@@ -839,62 +856,19 @@ func dopack(dst, src string, extra []string) {
writefile(bdst.String(), dst, 0) writefile(bdst.String(), dst, 0)
} }
// buildorder records the order of builds for the 'go bootstrap' command. // builddeps records the build dependencies for the 'go bootstrap' command.
// The Go packages and commands must be in dependency order, // It is a map[string][]string and generated by mkdeps.bash into deps.go.
// maintained by hand, but the order doesn't change often.
var buildorder = []string{ // buildlist is the list of directories being built, sorted by name.
// Go libraries and programs for bootstrap. var buildlist = makeBuildlist()
"runtime",
"errors", func makeBuildlist() []string {
"sync/atomic", var all []string
"sync", for dir := range builddeps {
"internal/singleflight", all = append(all, dir)
"io", }
"unicode", sort.Strings(all)
"unicode/utf8", return all
"unicode/utf16",
"bytes",
"math",
"strings",
"strconv",
"bufio",
"sort",
"container/heap",
"encoding/base64",
"syscall",
"internal/syscall/windows/registry",
"time",
"internal/syscall/windows",
"os",
"reflect",
"fmt",
"encoding",
"encoding/binary",
"encoding/json",
"flag",
"path/filepath",
"path",
"io/ioutil",
"log",
"regexp/syntax",
"regexp",
"go/token",
"go/scanner",
"go/ast",
"go/parser",
"os/exec",
"os/signal",
"net/url",
"text/template/parse",
"text/template",
"go/doc",
"go/build",
"hash",
"crypto",
"crypto/sha1",
"debug/dwarf",
"debug/elf",
"cmd/go",
} }
var runtimegen = []string{ var runtimegen = []string{
...@@ -903,7 +877,7 @@ var runtimegen = []string{ ...@@ -903,7 +877,7 @@ var runtimegen = []string{
} }
func clean() { func clean() {
for _, name := range buildorder { for _, name := range buildlist {
path := pathf("%s/src/%s", goroot, name) path := pathf("%s/src/%s", goroot, name)
// Remove generated files. // Remove generated files.
for _, elem := range xreaddir(path) { for _, elem := range xreaddir(path) {
...@@ -1044,19 +1018,30 @@ func cmdbootstrap() { ...@@ -1044,19 +1018,30 @@ func cmdbootstrap() {
// than in a standard release like Go 1.4, so don't do this rebuild by default. // than in a standard release like Go 1.4, so don't do this rebuild by default.
if false { if false {
xprintf("##### Building Go toolchain using itself.\n") xprintf("##### Building Go toolchain using itself.\n")
for _, dir := range buildorder { for _, dir := range buildlist {
if dir == "cmd/go" { installed[dir] = make(chan struct{})
break
}
install(dir)
} }
var wg sync.WaitGroup
for _, dir := range builddeps["cmd/go"] {
wg.Add(1)
dir := dir
go func() {
defer wg.Done()
install(dir)
}()
}
wg.Wait()
xprintf("\n") xprintf("\n")
} }
xprintf("##### Building go_bootstrap for host, %s/%s.\n", gohostos, gohostarch) xprintf("##### Building go_bootstrap for host, %s/%s.\n", gohostos, gohostarch)
for _, dir := range buildorder { for _, dir := range buildlist {
install(dir) installed[dir] = make(chan struct{})
}
for _, dir := range buildlist {
go install(dir)
} }
<-installed["cmd/go"]
goos = oldgoos goos = oldgoos
goarch = oldgoarch goarch = oldgoarch
...@@ -1065,6 +1050,7 @@ func cmdbootstrap() { ...@@ -1065,6 +1050,7 @@ func cmdbootstrap() {
// Build runtime for actual goos/goarch too. // Build runtime for actual goos/goarch too.
if goos != gohostos || goarch != gohostarch { if goos != gohostos || goarch != gohostarch {
installed["runtime"] = make(chan struct{})
install("runtime") install("runtime")
} }
} }
......
src/cmd/dist/deps.go 0 → 100644
View file @ 0e5b4eb0
// generated by mkdeps.bash
package main
var builddeps = map[string][]string{
"bufio": {"bytes", "errors", "io", "runtime", "sync", "sync/atomic", "unicode", "unicode/utf8"},
"bytes": {"errors", "io", "runtime", "sync", "sync/atomic", "unicode", "unicode/utf8"},
"container/heap": {"runtime", "sort"},
"crypto": {"errors", "hash", "io", "math", "runtime", "strconv", "sync", "sync/atomic", "unicode/utf8"},
"crypto/sha1": {"crypto", "errors", "hash", "io", "math", "runtime", "strconv", "sync", "sync/atomic", "unicode/utf8"},
"debug/dwarf": {"encoding/binary", "errors", "fmt", "internal/syscall/windows", "internal/syscall/windows/registry", "io", "math", "os", "path", "reflect", "runtime", "sort", "strconv", "strings", "sync", "sync/atomic", "syscall", "time", "unicode", "unicode/utf16", "unicode/utf8"},
"debug/elf": {"bytes", "debug/dwarf", "encoding/binary", "errors", "fmt", "internal/syscall/windows", "internal/syscall/windows/registry", "io", "math", "os", "path", "reflect", "runtime", "sort", "strconv", "strings", "sync", "sync/atomic", "syscall", "time", "unicode", "unicode/utf16", "unicode/utf8"},
"encoding": {"runtime"},
"encoding/base64": {"errors", "io", "math", "runtime", "strconv", "sync", "sync/atomic", "unicode/utf8"},
"encoding/binary": {"errors", "io", "math", "reflect", "runtime", "strconv", "sync", "sync/atomic", "unicode/utf8"},
"encoding/json": {"bytes", "encoding", "encoding/base64", "errors", "fmt", "internal/syscall/windows", "internal/syscall/windows/registry", "io", "math", "os", "reflect", "runtime", "sort", "strconv", "strings", "sync", "sync/atomic", "syscall", "time", "unicode", "unicode/utf16", "unicode/utf8"},
"errors": {"runtime"},
"flag": {"errors", "fmt", "internal/syscall/windows", "internal/syscall/windows/registry", "io", "math", "os", "reflect", "runtime", "sort", "strconv", "sync", "sync/atomic", "syscall", "time", "unicode/utf16", "unicode/utf8"},
"fmt": {"errors", "internal/syscall/windows", "internal/syscall/windows/registry", "io", "math", "os", "reflect", "runtime", "strconv", "sync", "sync/atomic", "syscall", "time", "unicode/utf16", "unicode/utf8"},
"go/ast": {"bytes", "errors", "fmt", "go/scanner", "go/token", "internal/syscall/windows", "internal/syscall/windows/registry", "io", "math", "os", "path/filepath", "reflect", "runtime", "sort", "strconv", "strings", "sync", "sync/atomic", "syscall", "time", "unicode", "unicode/utf16", "unicode/utf8"},
"go/build": {"bufio", "bytes", "errors", "fmt", "go/ast", "go/doc", "go/parser", "go/scanner", "go/token", "internal/syscall/windows", "internal/syscall/windows/registry", "io", "io/ioutil", "log", "math", "net/url", "os", "path", "path/filepath", "reflect", "regexp", "regexp/syntax", "runtime", "sort", "strconv", "strings", "sync", "sync/atomic", "syscall", "text/template", "text/template/parse", "time", "unicode", "unicode/utf16", "unicode/utf8"},
"go/doc": {"bytes", "errors", "fmt", "go/ast", "go/scanner", "go/token", "internal/syscall/windows", "internal/syscall/windows/registry", "io", "io/ioutil", "math", "net/url", "os", "path", "path/filepath", "reflect", "regexp", "regexp/syntax", "runtime", "sort", "strconv", "strings", "sync", "sync/atomic", "syscall", "text/template", "text/template/parse", "time", "unicode", "unicode/utf16", "unicode/utf8"},
"go/parser": {"bytes", "errors", "fmt", "go/ast", "go/scanner", "go/token", "internal/syscall/windows", "internal/syscall/windows/registry", "io", "io/ioutil", "math", "os", "path/filepath", "reflect", "runtime", "sort", "strconv", "strings", "sync", "sync/atomic", "syscall", "time", "unicode", "unicode/utf16", "unicode/utf8"},
"go/scanner": {"bytes", "errors", "fmt", "go/token", "internal/syscall/windows", "internal/syscall/windows/registry", "io", "math", "os", "path/filepath", "reflect", "runtime", "sort", "strconv", "strings", "sync", "sync/atomic", "syscall", "time", "unicode", "unicode/utf16", "unicode/utf8"},
"go/token": {"errors", "fmt", "internal/syscall/windows", "internal/syscall/windows/registry", "io", "math", "os", "reflect", "runtime", "sort", "strconv", "sync", "sync/atomic", "syscall", "time", "unicode/utf16", "unicode/utf8"},
"hash": {"errors", "io", "runtime", "sync", "sync/atomic"},
"internal/singleflight": {"runtime", "sync", "sync/atomic"},
"internal/syscall/windows": {"errors", "runtime", "sync", "sync/atomic", "syscall", "unicode/utf16"},
"internal/syscall/windows/registry": {"errors", "io", "runtime", "sync", "sync/atomic", "syscall", "unicode/utf16"},
"io": {"errors", "runtime", "sync", "sync/atomic"},
"io/ioutil": {"bytes", "errors", "internal/syscall/windows", "internal/syscall/windows/registry", "io", "math", "os", "path/filepath", "runtime", "sort", "strconv", "strings", "sync", "sync/atomic", "syscall", "time", "unicode", "unicode/utf16", "unicode/utf8"},
"log": {"errors", "fmt", "internal/syscall/windows", "internal/syscall/windows/registry", "io", "math", "os", "reflect", "runtime", "strconv", "sync", "sync/atomic", "syscall", "time", "unicode/utf16", "unicode/utf8"},
"math": {"runtime"},
"net/url": {"bytes", "errors", "fmt", "internal/syscall/windows", "internal/syscall/windows/registry", "io", "math", "os", "reflect", "runtime", "sort", "strconv", "strings", "sync", "sync/atomic", "syscall", "time", "unicode", "unicode/utf16", "unicode/utf8"},
"os": {"errors", "internal/syscall/windows", "internal/syscall/windows/registry", "io", "runtime", "sync", "sync/atomic", "syscall", "time", "unicode/utf16", "unicode/utf8"},
"os/exec": {"bytes", "errors", "internal/syscall/windows", "internal/syscall/windows/registry", "io", "math", "os", "path/filepath", "runtime", "sort", "strconv", "strings", "sync", "sync/atomic", "syscall", "time", "unicode", "unicode/utf16", "unicode/utf8"},
"os/signal": {"errors", "internal/syscall/windows", "internal/syscall/windows/registry", "io", "os", "runtime", "sync", "sync/atomic", "syscall", "time", "unicode/utf16", "unicode/utf8"},
"path": {"errors", "io", "runtime", "strings", "sync", "sync/atomic", "unicode", "unicode/utf8"},
"path/filepath": {"bytes", "errors", "internal/syscall/windows", "internal/syscall/windows/registry", "io", "os", "runtime", "sort", "strings", "sync", "sync/atomic", "syscall", "time", "unicode", "unicode/utf16", "unicode/utf8"},
"reflect": {"errors", "math", "runtime", "strconv", "sync", "sync/atomic", "unicode/utf8"},
"regexp": {"bytes", "errors", "io", "math", "regexp/syntax", "runtime", "sort", "strconv", "strings", "sync", "sync/atomic", "unicode", "unicode/utf8"},
"regexp/syntax": {"bytes", "errors", "io", "math", "runtime", "sort", "strconv", "strings", "sync", "sync/atomic", "unicode", "unicode/utf8"},
"runtime": {},
"sort": {"runtime"},
"strconv": {"errors", "math", "runtime", "unicode/utf8"},
"strings": {"errors", "io", "runtime", "sync", "sync/atomic", "unicode", "unicode/utf8"},
"sync": {"runtime", "sync/atomic"},
"sync/atomic": {"runtime"},
"syscall": {"errors", "runtime", "sync", "sync/atomic", "unicode/utf16"},
"text/template": {"bytes", "errors", "fmt", "internal/syscall/windows", "internal/syscall/windows/registry", "io", "io/ioutil", "math", "net/url", "os", "path/filepath", "reflect", "runtime", "sort", "strconv", "strings", "sync", "sync/atomic", "syscall", "text/template/parse", "time", "unicode", "unicode/utf16", "unicode/utf8"},
"text/template/parse": {"bytes", "errors", "fmt", "internal/syscall/windows", "internal/syscall/windows/registry", "io", "math", "os", "reflect", "runtime", "strconv", "strings", "sync", "sync/atomic", "syscall", "time", "unicode", "unicode/utf16", "unicode/utf8"},
"time": {"errors", "internal/syscall/windows/registry", "io", "runtime", "sync", "sync/atomic", "syscall", "unicode/utf16"},
"unicode": {"runtime"},
"unicode/utf16": {"runtime"},
"unicode/utf8": {"runtime"},
"cmd/go": {"bufio", "bytes", "container/heap", "crypto", "crypto/sha1", "debug/dwarf", "debug/elf", "encoding", "encoding/base64", "encoding/binary", "encoding/json", "errors", "flag", "fmt", "go/ast", "go/build", "go/doc", "go/parser", "go/scanner", "go/token", "hash", "internal/singleflight", "internal/syscall/windows", "internal/syscall/windows/registry", "io", "io/ioutil", "log", "math", "net/url", "os", "os/exec", "os/signal", "path", "path/filepath", "reflect", "regexp", "regexp/syntax", "runtime", "sort", "strconv", "strings", "sync", "sync/atomic", "syscall", "text/template", "text/template/parse", "time", "unicode", "unicode/utf16", "unicode/utf8"},
}
src/cmd/dist/mkdeps.bash 0 → 100755
View file @ 0e5b4eb0
#!/bin/bash
set -e
# Windows has the most dependencies.
export GOOS=windows
(
echo '// generated by mkdeps.bash'
echo
echo 'package main'
echo
echo 'var builddeps = map[string][]string{'
deps=$(GOOS=windows go list -tags cmd_go_bootstrap -f '{{join .Deps "\n"}}' cmd/go | grep -v '^unsafe$')
GOOS=windows go list -tags cmd_go_bootstrap -f '{{printf "%q" .ImportPath}}: { {{range .Deps}}{{if not (eq . "unsafe")}}{{printf "%q" .}}, {{end}}{{end}} },' $deps cmd/go
echo '}'
) |gofmt >deps.go
src/cmd/dist/util.go
View file @ 0e5b4eb0
...@@ -16,7 +16,6 @@ import ( ...@@ -16,7 +16,6 @@ import (
"strconv" "strconv"
"strings" "strings"
"sync" "sync"
"sync/atomic"
"time" "time"
) )
...@@ -109,7 +108,9 @@ func run(dir string, mode int, cmd ...string) string { ...@@ -109,7 +108,9 @@ func run(dir string, mode int, cmd ...string) string {
} }
outputLock.Unlock() outputLock.Unlock()
if mode&Background != 0 { if mode&Background != 0 {
bgdied.Done() // Prevent fatal from waiting on our own goroutine's
// bghelper to exit:
bghelpers.Done()
} }
fatal("FAILED: %v: %v", strings.Join(cmd, " "), err) fatal("FAILED: %v: %v", strings.Join(cmd, " "), err)
} }
...@@ -130,62 +131,60 @@ var ( ...@@ -130,62 +131,60 @@ var (
bgwork = make(chan func(), 1e5) bgwork = make(chan func(), 1e5)
bgdone = make(chan struct{}, 1e5) bgdone = make(chan struct{}, 1e5)
bgdied sync.WaitGroup bghelpers sync.WaitGroup
nwork int32
ndone int32
dying = make(chan bool) dieOnce sync.Once // guards close of dying
nfatal int32 dying = make(chan struct{})
) )
func bginit() { func bginit() {
bgdied.Add(maxbg) bghelpers.Add(maxbg)
for i := 0; i < maxbg; i++ { for i := 0; i < maxbg; i++ {
go bghelper() go bghelper()
} }
} }
func bghelper() { func bghelper() {
defer bghelpers.Done()
for { for {
w := <-bgwork select {
w() case <-dying:
// Stop if we're dying.
if atomic.LoadInt32(&nfatal) > 0 {
bgdied.Done()
return return
case w := <-bgwork:
// Dying takes precedence over doing more work.
select {
case <-dying:
return
default:
w()
}
} }
} }
} }
// bgrun is like run but runs the command in the background. // bgrun is like run but runs the command in the background.
// CheckExit|ShowOutput mode is implied (since output cannot be returned). // CheckExit|ShowOutput mode is implied (since output cannot be returned).
func bgrun(dir string, cmd ...string) { // bgrun adds 1 to wg immediately, and calls Done when the work completes.
func bgrun(wg *sync.WaitGroup, dir string, cmd ...string) {
wg.Add(1)
bgwork <- func() { bgwork <- func() {
defer wg.Done()
run(dir, CheckExit|ShowOutput|Background, cmd...) run(dir, CheckExit|ShowOutput|Background, cmd...)
} }
} }
// bgwait waits for pending bgruns to finish. // bgwait waits for pending bgruns to finish.
// bgwait must be called from only a single goroutine at a time. // bgwait must be called from only a single goroutine at a time.
func bgwait() { func bgwait(wg *sync.WaitGroup) {
var wg sync.WaitGroup done := make(chan struct{})
wg.Add(maxbg) go func() {
done := make(chan bool) wg.Wait()
for i := 0; i < maxbg; i++ { close(done)
bgwork <- func() { }()
wg.Done() select {
case <-done:
// Hold up bg goroutine until either the wait finishes case <-dying:
// or the program starts dying due to a call to fatal.
select {
case <-dying:
case <-done:
}
}
} }
wg.Wait()
close(done)
} }
// xgetwd returns the current directory. // xgetwd returns the current directory.
...@@ -355,16 +354,12 @@ func xworkdir() string { ...@@ -355,16 +354,12 @@ func xworkdir() string {
func fatal(format string, args ...interface{}) { func fatal(format string, args ...interface{}) {
fmt.Fprintf(os.Stderr, "go tool dist: %s\n", fmt.Sprintf(format, args...)) fmt.Fprintf(os.Stderr, "go tool dist: %s\n", fmt.Sprintf(format, args...))
dieOnce.Do(func() { close(dying) })
// Wait for background goroutines to finish, // Wait for background goroutines to finish,
// so that exit handler that removes the work directory // so that exit handler that removes the work directory
// is not fighting with active writes or open files. // is not fighting with active writes or open files.
if atomic.AddInt32(&nfatal, 1) == 1 { bghelpers.Wait()
close(dying)
}
for i := 0; i < maxbg; i++ {
bgwork <- func() {} // wake up workers so they notice nfatal > 0
}
bgdied.Wait()
xexit(2) xexit(2)
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7