dashboard: fix for branches

In the new world, one builder runs
        gobuilder -commit
which uploads information about commits to the dashboard,
which then hands the work out to the builders by hash.
There is no assumption anymore that the commit numbers
are consistent across builders.

New builders will need to be deployed.  For now darwin-amd64
is running the new builder to test the code.

The new JSON-based protocol for handing out work via /todo
should be easy to extend if we want to add support for sending
trial CLs to the builders.

This code is already running on godashboard.appspot.com.

R=adg, dave
CC=golang-dev
https://golang.org/cl/4519047

misc/dashboard/builder/Makefile

@@ -7,7 +7,6 @@ include ../../../src/Make.inc
 TARG=gobuilder
 GOFILES=\
 	exec.go\
-	hg.go\
 	http.go\
 	main.go\
 	package.go\

misc/dashboard/builder/doc.go

@@ -14,9 +14,6 @@ It periodically pulls updates from the Go Mercurial repository.
 When a newer revision is found, Go Builder creates a clone of the repository,
 runs all.bash, and reports build success or failure to the Go Dashboard. 
 
-For a successful build, Go Builder will also run benchmarks 
-(cd $GOROOT/src/pkg; make bench) and send the results to the Go Dashboard.
-
 For a release revision (a change description that matches "release.YYYY-MM-DD"),
 Go Builder will create a tar.gz archive of the GOROOT and deliver it to the
 Go Google Code project's downloads section.
@@ -34,8 +31,6 @@ Optional flags:
     The location of the Go Dashboard application to which Go Builder will
     report its results.
 
-  -bench: Run benchmarks
-
   -release: Build and deliver binary release archive
 
   -rev=N: Build revision N and exit
@@ -45,7 +40,7 @@ Optional flags:
   -v: Verbose logging
 
   -external: External package builder mode (will not report Go build
-     state to dashboard, issue releases, or run benchmarks)
+     state to dashboard or issue releases)
 
 The key file should be located at $HOME/.gobuildkey or, for a builder-specific
 key, $HOME/.gobuildkey-$BUILDER (eg, $HOME/.gobuildkey-linux-amd64).

misc/dashboard/builder/exec.go

@@ -18,7 +18,7 @@ func run(envv []string, dir string, argv ...string) os.Error {
 	if *verbose {
 		log.Println("run", argv)
 	}
-	bin, err := pathLookup(argv[0])
+	bin, err := lookPath(argv[0])
 	if err != nil {
 		return err
 	}
@@ -36,7 +36,7 @@ func runLog(envv []string, logfile, dir string, argv ...string) (output string,
 	if *verbose {
 		log.Println("runLog", argv)
 	}
-	bin, err := pathLookup(argv[0])
+	bin, err := lookPath(argv[0])
 	if err != nil {
 		return
 	}
@@ -67,10 +67,10 @@ func runLog(envv []string, logfile, dir string, argv ...string) (output string,
 	return b.String(), wait.WaitStatus.ExitStatus(), nil
 }
 
-// Find bin in PATH if a relative or absolute path hasn't been specified
-func pathLookup(s string) (string, os.Error) {
-	if strings.HasPrefix(s, "/") || strings.HasPrefix(s, "./") || strings.HasPrefix(s, "../") {
-		return s, nil
+// lookPath looks for cmd in $PATH if cmd does not begin with / or ./ or ../.
+func lookPath(cmd string) (string, os.Error) {
+	if strings.HasPrefix(cmd, "/") || strings.HasPrefix(cmd, "./") || strings.HasPrefix(cmd, "../") {
+		return cmd, nil
 	}
-	return exec.LookPath(s)
+	return exec.LookPath(cmd)
 }

misc/dashboard/builder/hg.go

-// Copyright 2011 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-package main
-
-import (
-	"fmt"
-	"os"
-	"regexp"
-	"strconv"
-	"strings"
-)
-
-type Commit struct {
-	num    int    // mercurial revision number
-	node   string // mercurial hash
-	parent string // hash of commit's parent
-	user   string // author's Name <email>
-	date   string // date of commit
-	desc   string // description
-}
-
-// getCommit returns details about the Commit specified by the revision hash
-func getCommit(rev string) (c Commit, err os.Error) {
-	defer func() {
-		if err != nil {
-			err = fmt.Errorf("getCommit: %s: %s", rev, err)
-		}
-	}()
-	parts, err := getCommitParts(rev)
-	if err != nil {
-		return
-	}
-	num, err := strconv.Atoi(parts[0])
-	if err != nil {
-		return
-	}
-	parent := ""
-	if num > 0 {
-		prev := strconv.Itoa(num - 1)
-		if pparts, err := getCommitParts(prev); err == nil {
-			parent = pparts[1]
-		}
-	}
-	user := strings.Replace(parts[2], "&lt;", "<", -1)
-	user = strings.Replace(user, "&gt;", ">", -1)
-	return Commit{num, parts[1], parent, user, parts[3], parts[4]}, nil
-}
-
-func getCommitParts(rev string) (parts []string, err os.Error) {
-	const format = "{rev}>{node}>{author|escape}>{date}>{desc}"
-	s, _, err := runLog(nil, "", goroot,
-		"hg", "log",
-		"--encoding", "utf-8",
-		"--rev", rev,
-		"--limit", "1",
-		"--template", format,
-	)
-	if err != nil {
-		return
-	}
-	return strings.Split(s, ">", 5), nil
-}
-
-var revisionRe = regexp.MustCompile(`([0-9]+):[0-9a-f]+$`)
-
-// getTag fetches a Commit by finding the first hg tag that matches re.
-func getTag(re *regexp.Regexp) (c Commit, tag string, err os.Error) {
-	o, _, err := runLog(nil, "", goroot, "hg", "tags")
-	for _, l := range strings.Split(o, "\n", -1) {
-		tag = re.FindString(l)
-		if tag == "" {
-			continue
-		}
-		s := revisionRe.FindStringSubmatch(l)
-		if s == nil {
-			err = os.NewError("couldn't find revision number")
-			return
-		}
-		c, err = getCommit(s[1])
-		return
-	}
-	err = os.NewError("no matching tag found")
-	return
-}

misc/dashboard/builder/http.go

@@ -6,84 +6,104 @@ package main
 
 import (
 	"bytes"
-	"encoding/base64"
-	"encoding/binary"
 	"fmt"
 	"http"
 	"json"
 	"log"
 	"os"
-	"regexp"
 	"strconv"
 )
 
-// getHighWater returns the current highwater revision hash for this builder
-func (b *Builder) getHighWater() (rev string, err os.Error) {
-	url := fmt.Sprintf("http://%s/hw-get?builder=%s", *dashboard, b.name)
-	r, _, err := http.Get(url)
-	if err != nil {
-		return
+type param map[string]string
+
+// dash runs the given method and command on the dashboard.
+// If args is not nil, it is the query or post parameters.
+// If resp is not nil, dash unmarshals the body as JSON into resp.
+func dash(meth, cmd string, resp interface{}, args param) os.Error {
+	var r *http.Response
+	var err os.Error
+	if *verbose {
+		log.Println("dash", cmd, args)
+	}
+	cmd = "http://" + *dashboard + "/" + cmd
+	switch meth {
+	case "GET":
+		if args != nil {
+			m := make(map[string][]string)
+			for k, v := range args {
+				m[k] = []string{v}
+			}
+			cmd += "?" + http.EncodeQuery(m)
+		}
+		r, _, err = http.Get(cmd)
+	case "POST":
+		r, err = http.PostForm(cmd, args)
+	default:
+		return fmt.Errorf("unknown method %q", meth)
 	}
-	buf := new(bytes.Buffer)
-	_, err = buf.ReadFrom(r.Body)
 	if err != nil {
-		return
+		return err
+	}
+	defer r.Body.Close()
+	var buf bytes.Buffer
+	buf.ReadFrom(r.Body)
+	if resp != nil {
+		if err = json.Unmarshal(buf.Bytes(), resp); err != nil {
+			log.Printf("json unmarshal %#q: %s\n", buf.Bytes(), err)
+			return err
+		}
 	}
-	r.Body.Close()
-	return buf.String(), nil
+	return nil
 }
 
-// recordResult sends build results to the dashboard
-func (b *Builder) recordResult(buildLog string, c Commit) os.Error {
-	return httpCommand("build", map[string]string{
-		"builder": b.name,
-		"key":     b.key,
-		"node":    c.node,
-		"parent":  c.parent,
-		"user":    c.user,
-		"date":    c.date,
-		"desc":    c.desc,
-		"log":     buildLog,
-	})
+func dashStatus(meth, cmd string, args param) os.Error {
+	var resp struct {
+		Status string
+		Error string
+	}
+	err := dash(meth, cmd, &resp, args)
+	if err != nil {
+		return err
+	}
+	if resp.Status != "OK" {
+		return os.NewError("/build: " + resp.Error)
+	}
+	return nil	
 }
 	
-// match lines like: "package.BechmarkFunc	100000	    999 ns/op"
-var benchmarkRegexp = regexp.MustCompile("([^\n\t ]+)[\t ]+([0-9]+)[\t ]+([0-9]+) ns/op")
-
-// recordBenchmarks sends benchmark results to the dashboard
-func (b *Builder) recordBenchmarks(benchLog string, c Commit) os.Error {
-	results := benchmarkRegexp.FindAllStringSubmatch(benchLog, -1)
-	var buf bytes.Buffer
-	b64 := base64.NewEncoder(base64.StdEncoding, &buf)
-	for _, r := range results {
-		for _, s := range r[1:] {
-			binary.Write(b64, binary.BigEndian, uint16(len(s)))
-			b64.Write([]byte(s))
+// todo returns the next hash to build.
+func (b *Builder) todo() (rev string, err os.Error) {
+	var resp []struct{
+		Hash string
 	}
+	if err = dash("GET", "todo", &resp, param{"builder": b.name}); err != nil {
+		return
+	}
+	if len(resp) > 0 {
+		rev = resp[0].Hash
 	}
-	b64.Close()
-	return httpCommand("benchmarks", map[string]string{
+	return
+}
+
+// recordResult sends build results to the dashboard
+func (b *Builder) recordResult(buildLog string, hash string) os.Error {
+	return dash("POST", "build", nil, param{
 		"builder": b.name,
 		"key":     b.key,
-		"node":          c.node,
-		"benchmarkdata": buf.String(),
+		"node":    hash,
+		"log":     buildLog,
 	})
 }
 
-// getPackages fetches a list of package paths from the dashboard
-func getPackages() (pkgs []string, err os.Error) {
-	r, _, err := http.Get(fmt.Sprintf("http://%v/package?fmt=json", *dashboard))
-	if err != nil {
-		return
-	}
-	defer r.Body.Close()
-	d := json.NewDecoder(r.Body)
+// packages fetches a list of package paths from the dashboard
+func packages() (pkgs []string, err os.Error) {
 	var resp struct {
 		Packages []struct {
 			Path string
 		}
 	}
-	if err = d.Decode(&resp); err != nil {
+	err = dash("GET", "package", &resp, param{"fmt": "json"})
+	if err != nil {
 		return
 	}
 	for _, p := range resp.Packages {
@@ -93,24 +113,36 @@ func getPackages() (pkgs []string, err os.Error) {
 }
 
 // updatePackage sends package build results and info to the dashboard
-func (b *Builder) updatePackage(pkg string, state bool, buildLog, info string, c Commit) os.Error {
-	args := map[string]string{
+func (b *Builder) updatePackage(pkg string, state bool, buildLog, info string, hash string) os.Error {
+	return dash("POST", "package", nil, param{
 		"builder": b.name,
 		"key":     b.key,
 		"path":    pkg,
 		"state":   strconv.Btoa(state),
 		"log":     buildLog,
 		"info":    info,
-		"go_rev":  strconv.Itoa(c.num),
-	}
-	return httpCommand("package", args)
+		"go_rev":  hash[:12],
+	})
 }
 
-func httpCommand(cmd string, args map[string]string) os.Error {
-	if *verbose {
-		log.Println("httpCommand", cmd, args)
+// postCommit informs the dashboard of a new commit
+func postCommit(key string, l *HgLog) os.Error {
+	return dashStatus("POST", "commit", param{
+		"key": key,
+		"node": l.Hash,
+		"date": l.Date,
+		"user": l.Author,
+		"parent": l.Parent,
+		"desc": l.Desc,
+	})
+}
+
+// dashboardCommit returns true if the dashboard knows about hash.
+func dashboardCommit(hash string) bool {
+	err := dashStatus("GET", "commit", param{"node": hash})
+	if err != nil {
+		log.Printf("check %s: %s", hash, err)
+		return false
 	}
-	url := fmt.Sprintf("http://%v/%v", *dashboard, cmd)
-	_, err := http.PostForm(url, args)
-	return err
+	return true
 }

misc/dashboard/builder/package.go

@@ -13,8 +13,8 @@ import (
 	"path"
 )
 
-func (b *Builder) buildPackages(workpath string, c Commit) os.Error {
-	pkgs, err := getPackages()
+func (b *Builder) buildPackages(workpath string, hash string) os.Error {
+	pkgs, err := packages()
 	if err != nil {
 		return err
 	}
@@ -32,13 +32,13 @@ func (b *Builder) buildPackages(workpath string, c Commit) os.Error {
 		built := code != 0
 
 		// get doc comment from package source
-		info, err := getPackageComment(p, path.Join(goroot, "pkg", p))
+		info, err := packageComment(p, path.Join(goroot, "pkg", p))
 		if err != nil {
 			log.Printf("goinstall %v: %v", p, err)
 		}
 
 		// update dashboard with build state + info
-		err = b.updatePackage(p, built, buildLog, info, c)
+		err = b.updatePackage(p, built, buildLog, info, hash)
 		if err != nil {
 			log.Printf("updatePackage %v: %v", p, err)
 		}
@@ -46,7 +46,7 @@ func (b *Builder) buildPackages(workpath string, c Commit) os.Error {
 	return nil
 }
 
-func getPackageComment(pkg, pkgpath string) (info string, err os.Error) {
+func packageComment(pkg, pkgpath string) (info string, err os.Error) {
 	fset := token.NewFileSet()
 	pkgs, err := parser.ParseDir(fset, pkgpath, nil, parser.PackageClauseOnly|parser.ParseComments)
 	if err != nil {

misc/dashboard/godashboard/app.yaml

 application: godashboard
-version: 5
+version: 6
 runtime: python
 api_version: 1
 

misc/dashboard/godashboard/index.yaml

@@ -23,6 +23,12 @@ indexes:
   - name: __key__
     direction: desc
 
+- kind: Commit
+  ancestor: yes
+  properties:
+  - name: __key__
+    direction: desc
+
 - kind: Project
   properties:
   - name: approved