main_test.go 26.4 KB
Newer Older
1 2 3
package main

import (
4
	"./internal/api"
5
	"./internal/helper"
6
	"./internal/testhelper"
7
	"./internal/upstream"
8
	"bytes"
9
	"crypto/sha1"
10
	"encoding/base64"
11
	"encoding/json"
12
	"fmt"
13
	"io"
14
	"io/ioutil"
15
	"log"
16
	"mime/multipart"
17 18
	"net/http"
	"net/http/httptest"
19 20
	"net/http/httputil"
	"net/url"
21 22 23
	"os"
	"os/exec"
	"path"
24
	"regexp"
25
	"strings"
26 27 28 29
	"testing"
	"time"
)

30 31
const scratchDir = "testdata/scratch"
const testRepoRoot = "testdata/data"
32
const testDocumentRoot = "testdata/public"
33 34
const testRepo = "group/test.git"
const testProject = "group/test"
35

36
var checkoutDir = path.Join(scratchDir, "test")
37
var cacheDir = path.Join(scratchDir, "cache")
38 39

func TestAllowedClone(t *testing.T) {
40 41 42 43 44 45
	// Prepare clone directory
	if err := os.RemoveAll(scratchDir); err != nil {
		t.Fatal(err)
	}

	// Prepare test server and backend
46
	ts := testAuthServer(nil, 200, gitOkBody(t))
47
	defer ts.Close()
48 49
	ws := startWorkhorseServer(ts.URL)
	defer ws.Close()
50 51

	// Do the git clone
52
	cloneCmd := exec.Command("git", "clone", fmt.Sprintf("%s/%s", ws.URL, testRepo), checkoutDir)
53
	runOrFail(t, cloneCmd)
54

55 56 57 58
	// We may have cloned an 'empty' repository, 'git log' will fail in it
	logCmd := exec.Command("git", "log", "-1", "--oneline")
	logCmd.Dir = checkoutDir
	runOrFail(t, logCmd)
59 60
}

Jacob Vosmaer's avatar
Jacob Vosmaer committed
61 62 63 64 65 66 67
func TestDeniedClone(t *testing.T) {
	// Prepare clone directory
	if err := os.RemoveAll(scratchDir); err != nil {
		t.Fatal(err)
	}

	// Prepare test server and backend
68
	ts := testAuthServer(nil, 403, "Access denied")
Jacob Vosmaer's avatar
Jacob Vosmaer committed
69
	defer ts.Close()
70 71
	ws := startWorkhorseServer(ts.URL)
	defer ws.Close()
Jacob Vosmaer's avatar
Jacob Vosmaer committed
72 73

	// Do the git clone
74
	cloneCmd := exec.Command("git", "clone", fmt.Sprintf("%s/%s", ws.URL, testRepo), checkoutDir)
75 76 77
	out, err := cloneCmd.CombinedOutput()
	t.Logf("%s", out)
	if err == nil {
Jacob Vosmaer's avatar
Jacob Vosmaer committed
78 79 80 81
		t.Fatal("git clone should have failed")
	}
}

82
func TestAllowedPush(t *testing.T) {
Jacob Vosmaer's avatar
Jacob Vosmaer committed
83
	preparePushRepo(t)
84 85

	// Prepare the test server and backend
86
	ts := testAuthServer(nil, 200, gitOkBody(t))
87
	defer ts.Close()
88 89
	ws := startWorkhorseServer(ts.URL)
	defer ws.Close()
90 91

	// Perform the git push
92
	pushCmd := exec.Command("git", "push", fmt.Sprintf("%s/%s", ws.URL, testRepo), fmt.Sprintf("master:%s", newBranch()))
93
	pushCmd.Dir = checkoutDir
94
	runOrFail(t, pushCmd)
95 96
}

Jacob Vosmaer's avatar
Jacob Vosmaer committed
97
func TestDeniedPush(t *testing.T) {
Jacob Vosmaer's avatar
Jacob Vosmaer committed
98
	preparePushRepo(t)
Jacob Vosmaer's avatar
Jacob Vosmaer committed
99 100

	// Prepare the test server and backend
101
	ts := testAuthServer(nil, 403, "Access denied")
Jacob Vosmaer's avatar
Jacob Vosmaer committed
102
	defer ts.Close()
103 104
	ws := startWorkhorseServer(ts.URL)
	defer ws.Close()
Jacob Vosmaer's avatar
Jacob Vosmaer committed
105 106

	// Perform the git push
107
	pushCmd := exec.Command("git", "push", "-v", fmt.Sprintf("%s/%s", ws.URL, testRepo), fmt.Sprintf("master:%s", newBranch()))
Jacob Vosmaer's avatar
Jacob Vosmaer committed
108
	pushCmd.Dir = checkoutDir
109 110 111
	out, err := pushCmd.CombinedOutput()
	t.Logf("%s", out)
	if err == nil {
Jacob Vosmaer's avatar
Jacob Vosmaer committed
112 113 114 115
		t.Fatal("git push should have failed")
	}
}

116 117 118 119 120
func TestAllowedDownloadZip(t *testing.T) {
	prepareDownloadDir(t)

	// Prepare test server and backend
	archiveName := "foobar.zip"
121
	ts := archiveOKServer(t, archiveName)
122
	defer ts.Close()
123 124
	ws := startWorkhorseServer(ts.URL)
	defer ws.Close()
125

126
	downloadCmd := exec.Command("curl", "-J", "-O", fmt.Sprintf("%s/%s/repository/archive.zip", ws.URL, testProject))
127 128 129 130 131 132 133 134 135 136 137 138 139
	downloadCmd.Dir = scratchDir
	runOrFail(t, downloadCmd)

	extractCmd := exec.Command("unzip", archiveName)
	extractCmd.Dir = scratchDir
	runOrFail(t, extractCmd)
}

func TestAllowedDownloadTar(t *testing.T) {
	prepareDownloadDir(t)

	// Prepare test server and backend
	archiveName := "foobar.tar"
140
	ts := archiveOKServer(t, archiveName)
141
	defer ts.Close()
142 143
	ws := startWorkhorseServer(ts.URL)
	defer ws.Close()
144

145
	downloadCmd := exec.Command("curl", "-J", "-O", fmt.Sprintf("%s/%s/repository/archive.tar", ws.URL, testProject))
146 147 148 149 150 151 152 153 154 155 156 157 158
	downloadCmd.Dir = scratchDir
	runOrFail(t, downloadCmd)

	extractCmd := exec.Command("tar", "xf", archiveName)
	extractCmd.Dir = scratchDir
	runOrFail(t, extractCmd)
}

func TestAllowedDownloadTarGz(t *testing.T) {
	prepareDownloadDir(t)

	// Prepare test server and backend
	archiveName := "foobar.tar.gz"
159
	ts := archiveOKServer(t, archiveName)
160
	defer ts.Close()
161 162
	ws := startWorkhorseServer(ts.URL)
	defer ws.Close()
163

164
	downloadCmd := exec.Command("curl", "-J", "-O", fmt.Sprintf("%s/%s/repository/archive.tar.gz", ws.URL, testProject))
165 166 167 168 169 170 171 172 173 174 175 176 177
	downloadCmd.Dir = scratchDir
	runOrFail(t, downloadCmd)

	extractCmd := exec.Command("tar", "zxf", archiveName)
	extractCmd.Dir = scratchDir
	runOrFail(t, extractCmd)
}

func TestAllowedDownloadTarBz2(t *testing.T) {
	prepareDownloadDir(t)

	// Prepare test server and backend
	archiveName := "foobar.tar.bz2"
178
	ts := archiveOKServer(t, archiveName)
179
	defer ts.Close()
180 181
	ws := startWorkhorseServer(ts.URL)
	defer ws.Close()
182

183
	downloadCmd := exec.Command("curl", "-J", "-O", fmt.Sprintf("%s/%s/repository/archive.tar.bz2", ws.URL, testProject))
184 185 186 187
	downloadCmd.Dir = scratchDir
	runOrFail(t, downloadCmd)

	extractCmd := exec.Command("tar", "jxf", archiveName)
Jacob Vosmaer's avatar
Jacob Vosmaer committed
188 189 190 191 192 193 194 195 196
	extractCmd.Dir = scratchDir
	runOrFail(t, extractCmd)
}

func TestAllowedApiDownloadZip(t *testing.T) {
	prepareDownloadDir(t)

	// Prepare test server and backend
	archiveName := "foobar.zip"
197
	ts := archiveOKServer(t, archiveName)
Jacob Vosmaer's avatar
Jacob Vosmaer committed
198
	defer ts.Close()
199 200
	ws := startWorkhorseServer(ts.URL)
	defer ws.Close()
Jacob Vosmaer's avatar
Jacob Vosmaer committed
201

202
	downloadCmd := exec.Command("curl", "-J", "-O", fmt.Sprintf("%s/api/v3/projects/123/repository/archive.zip", ws.URL))
Jacob Vosmaer's avatar
Jacob Vosmaer committed
203 204 205 206
	downloadCmd.Dir = scratchDir
	runOrFail(t, downloadCmd)

	extractCmd := exec.Command("unzip", archiveName)
207 208 209 210
	extractCmd.Dir = scratchDir
	runOrFail(t, extractCmd)
}

211 212 213 214 215
func TestAllowedApiDownloadZipWithSlash(t *testing.T) {
	prepareDownloadDir(t)

	// Prepare test server and backend
	archiveName := "foobar.zip"
216
	ts := archiveOKServer(t, archiveName)
217 218 219 220 221 222 223 224 225 226 227
	defer ts.Close()
	ws := startWorkhorseServer(ts.URL)
	defer ws.Close()

	// Use foo%2Fbar instead of a numeric ID
	downloadCmd := exec.Command("curl", "-J", "-O", fmt.Sprintf("%s/api/v3/projects/foo%%2Fbar/repository/archive.zip", ws.URL))
	if !strings.Contains(downloadCmd.Args[3], `projects/foo%2Fbar/repository`) {
		t.Fatalf("Cannot find percent-2F: %v", downloadCmd.Args)
	}
	downloadCmd.Dir = scratchDir
	runOrFail(t, downloadCmd)
Jacob Vosmaer's avatar
Jacob Vosmaer committed
228 229

	extractCmd := exec.Command("unzip", archiveName)
230 231 232 233
	extractCmd.Dir = scratchDir
	runOrFail(t, extractCmd)
}

234 235 236 237 238
func TestDownloadCacheHit(t *testing.T) {
	prepareDownloadDir(t)

	// Prepare test server and backend
	archiveName := "foobar.zip"
239
	ts := archiveOKServer(t, archiveName)
240
	defer ts.Close()
241 242
	ws := startWorkhorseServer(ts.URL)
	defer ws.Close()
243 244 245 246

	if err := os.MkdirAll(cacheDir, 0755); err != nil {
		t.Fatal(err)
	}
247
	cachedContent := []byte("cached")
248 249 250 251
	if err := ioutil.WriteFile(path.Join(cacheDir, archiveName), cachedContent, 0644); err != nil {
		t.Fatal(err)
	}

252
	downloadCmd := exec.Command("curl", "-J", "-O", fmt.Sprintf("%s/api/v3/projects/123/repository/archive.zip", ws.URL))
253 254 255 256 257 258 259 260 261 262 263 264
	downloadCmd.Dir = scratchDir
	runOrFail(t, downloadCmd)

	actual, err := ioutil.ReadFile(path.Join(scratchDir, archiveName))
	if err != nil {
		t.Fatal(err)
	}
	if bytes.Compare(actual, cachedContent) != 0 {
		t.Fatal("Unexpected file contents in download")
	}
}

265 266 267 268 269
func TestDownloadCacheCreate(t *testing.T) {
	prepareDownloadDir(t)

	// Prepare test server and backend
	archiveName := "foobar.zip"
270
	ts := archiveOKServer(t, archiveName)
271
	defer ts.Close()
272 273
	ws := startWorkhorseServer(ts.URL)
	defer ws.Close()
274

275
	downloadCmd := exec.Command("curl", "-J", "-O", fmt.Sprintf("%s/api/v3/projects/123/repository/archive.zip", ws.URL))
276 277 278 279 280 281 282 283 284
	downloadCmd.Dir = scratchDir
	runOrFail(t, downloadCmd)

	compareCmd := exec.Command("cmp", path.Join(cacheDir, archiveName), path.Join(scratchDir, archiveName))
	if err := compareCmd.Run(); err != nil {
		t.Fatalf("Comparison between downloaded file and cache item failed: %s", err)
	}
}

285 286 287 288 289 290 291 292 293
func TestRegularProjectsAPI(t *testing.T) {
	apiResponse := "API RESPONSE"
	ts := testAuthServer(nil, 200, apiResponse)
	defer ts.Close()
	ws := startWorkhorseServer(ts.URL)
	defer ws.Close()

	for _, resource := range []string{
		"/api/v3/projects/123/repository/not/special",
Jacob Vosmaer's avatar
Jacob Vosmaer committed
294
		"/api/v3/projects/foo%2Fbar/repository/not/special",
295
		"/api/v3/projects/123/not/special",
Jacob Vosmaer's avatar
Jacob Vosmaer committed
296
		"/api/v3/projects/foo%2Fbar/not/special",
297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315
	} {
		resp, err := http.Get(ws.URL + resource)
		if err != nil {
			t.Fatal(err)
		}
		defer resp.Body.Close()
		buf := &bytes.Buffer{}
		if _, err := io.Copy(buf, resp.Body); err != nil {
			t.Error(err)
		}
		if buf.String() != apiResponse {
			t.Errorf("GET %q: Expected %q, got %q", resource, apiResponse, buf.String())
		}
		if resp.StatusCode != 200 {
			t.Errorf("GET %q: expected 200, got %d", resource, resp.StatusCode)
		}
	}
}

Jacob Vosmaer's avatar
Jacob Vosmaer committed
316 317 318 319
func TestAllowedXSendfileDownload(t *testing.T) {
	contentFilename := "my-content"
	prepareDownloadDir(t)

320
	allowedXSendfileDownload(t, contentFilename, "foo/uploads/bar")
Jacob Vosmaer's avatar
Jacob Vosmaer committed
321 322 323 324 325 326
}

func TestDeniedXSendfileDownload(t *testing.T) {
	contentFilename := "my-content"
	prepareDownloadDir(t)

327
	deniedXSendfileDownload(t, contentFilename, "foo/uploads/bar")
Jacob Vosmaer's avatar
Jacob Vosmaer committed
328 329
}

330 331 332 333 334 335 336
func TestAllowedStaticFile(t *testing.T) {
	content := "PUBLIC"
	if err := setupStaticFile("static file.txt", content); err != nil {
		t.Fatalf("create public/static file.txt: %v", err)
	}

	proxied := false
337
	ts := testhelper.TestServerWithHandler(regexp.MustCompile(`.`), func(w http.ResponseWriter, r *http.Request) {
338 339 340 341 342 343 344 345 346 347 348 349 350
		proxied = true
		w.WriteHeader(404)
	})
	defer ts.Close()
	ws := startWorkhorseServer(ts.URL)
	defer ws.Close()

	for _, resource := range []string{
		"/static%20file.txt",
		"/static file.txt",
	} {
		resp, err := http.Get(ws.URL + resource)
		if err != nil {
351
			t.Error(err)
352 353 354 355
		}
		defer resp.Body.Close()
		buf := &bytes.Buffer{}
		if _, err := io.Copy(buf, resp.Body); err != nil {
356
			t.Error(err)
357 358
		}
		if buf.String() != content {
359
			t.Errorf("GET %q: Expected %q, got %q", resource, content, buf.String())
360 361
		}
		if resp.StatusCode != 200 {
362
			t.Errorf("GET %q: expected 200, got %d", resource, resp.StatusCode)
363 364
		}
		if proxied {
365
			t.Errorf("GET %q: should not have made it to backend", resource)
366 367 368 369
		}
	}
}

370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400
func TestStaticFileRelativeURL(t *testing.T) {
	content := "PUBLIC"
	if err := setupStaticFile("static.txt", content); err != nil {
		t.Fatalf("create public/static.txt: %v", err)
	}

	ts := testhelper.TestServerWithHandler(regexp.MustCompile(`.`), http.HandlerFunc(http.NotFound))
	defer ts.Close()
	backendURLString := ts.URL + "/my-relative-url"
	log.Print(backendURLString)
	ws := startWorkhorseServer(backendURLString)
	defer ws.Close()

	resource := "/my-relative-url/static.txt"
	resp, err := http.Get(ws.URL + resource)
	if err != nil {
		t.Error(err)
	}
	defer resp.Body.Close()
	buf := &bytes.Buffer{}
	if _, err := io.Copy(buf, resp.Body); err != nil {
		t.Error(err)
	}
	if buf.String() != content {
		t.Errorf("GET %q: Expected %q, got %q", resource, content, buf.String())
	}
	if resp.StatusCode != 200 {
		t.Errorf("GET %q: expected 200, got %d", resource, resp.StatusCode)
	}
}

401
func TestAllowedPublicUploadsFile(t *testing.T) {
402 403
	content := "PRIVATE but allowed"
	if err := setupStaticFile("uploads/static file.txt", content); err != nil {
404 405 406 407
		t.Fatalf("create public/uploads/static file.txt: %v", err)
	}

	proxied := false
408
	ts := testhelper.TestServerWithHandler(regexp.MustCompile(`.`), func(w http.ResponseWriter, r *http.Request) {
409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429
		proxied = true
		w.Header().Add("X-Sendfile", *documentRoot+r.URL.Path)
		w.WriteHeader(200)
	})
	defer ts.Close()
	ws := startWorkhorseServer(ts.URL)
	defer ws.Close()

	for _, resource := range []string{
		"/uploads/static%20file.txt",
		"/uploads/static file.txt",
	} {
		resp, err := http.Get(ws.URL + resource)
		if err != nil {
			t.Fatal(err)
		}
		defer resp.Body.Close()
		buf := &bytes.Buffer{}
		if _, err := io.Copy(buf, resp.Body); err != nil {
			t.Fatal(err)
		}
430 431
		if buf.String() != content {
			t.Fatalf("GET %q: Expected %q, got %q", resource, content, buf.String())
432 433 434 435 436 437 438 439 440 441 442
		}
		if resp.StatusCode != 200 {
			t.Fatalf("GET %q: expected 200, got %d", resource, resp.StatusCode)
		}
		if !proxied {
			t.Fatalf("GET %q: never made it to backend", resource)
		}
	}
}

func TestDeniedPublicUploadsFile(t *testing.T) {
443 444
	content := "PRIVATE"
	if err := setupStaticFile("uploads/static.txt", content); err != nil {
445 446 447 448
		t.Fatalf("create public/uploads/static.txt: %v", err)
	}

	proxied := false
449
	ts := testhelper.TestServerWithHandler(regexp.MustCompile(`.`), func(w http.ResponseWriter, _ *http.Request) {
450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469
		proxied = true
		w.WriteHeader(404)
	})
	defer ts.Close()
	ws := startWorkhorseServer(ts.URL)
	defer ws.Close()

	for _, resource := range []string{
		"/uploads/static.txt",
		"/uploads%2Fstatic.txt",
	} {
		resp, err := http.Get(ws.URL + resource)
		if err != nil {
			t.Fatal(err)
		}
		defer resp.Body.Close()
		buf := &bytes.Buffer{}
		if _, err := io.Copy(buf, resp.Body); err != nil {
			t.Fatal(err)
		}
470
		if buf.String() == content {
471 472 473 474 475 476 477 478 479 480 481
			t.Fatalf("GET %q: Got private file contents which should have been blocked by upstream", resource)
		}
		if resp.StatusCode != 404 {
			t.Fatalf("GET %q: expected 404, got %d", resource, resp.StatusCode)
		}
		if !proxied {
			t.Fatalf("GET %q: never made it to backend", resource)
		}
	}
}

482 483 484 485 486 487 488 489 490 491
func TestArtifactsUpload(t *testing.T) {
	reqBody := &bytes.Buffer{}
	writer := multipart.NewWriter(reqBody)
	file, err := writer.CreateFormFile("file", "my.file")
	if err != nil {
		t.Fatal(err)
	}
	fmt.Fprint(file, "SHOULD BE ON DISK, NOT IN MULTIPART")
	writer.Close()

492
	ts := testhelper.TestServerWithHandler(regexp.MustCompile(`.`), func(w http.ResponseWriter, r *http.Request) {
493 494 495 496 497 498 499 500 501 502
		if strings.HasSuffix(r.URL.Path, "/authorize") {
			if _, err := fmt.Fprintf(w, `{"TempPath":"%s"}`, scratchDir); err != nil {
				t.Fatal(err)
			}
			return
		}
		err := r.ParseMultipartForm(100000)
		if err != nil {
			t.Fatal(err)
		}
503 504 505
		nValues := 2 // filename + path for just the upload (no metadata because we are not POSTing a valid zip file)
		if len(r.MultipartForm.Value) != nValues {
			t.Errorf("Expected to receive exactly %d values", nValues)
506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526
		}
		if len(r.MultipartForm.File) != 0 {
			t.Error("Expected to not receive any files")
		}
		w.WriteHeader(200)
	})
	defer ts.Close()
	ws := startWorkhorseServer(ts.URL)
	defer ws.Close()

	resource := `/ci/api/v1/builds/123/artifacts`
	resp, err := http.Post(ws.URL+resource, writer.FormDataContentType(), reqBody)
	if err != nil {
		t.Error(err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		t.Errorf("GET %q: expected 200, got %d", resource, resp.StatusCode)
	}
}

527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560
func TestArtifactsGetSingleFile(t *testing.T) {
	// We manually created this zip file in the gitlab-workhorse Git repository
	archivePath := `testdata/artifacts-archive.zip`
	fileName := "myfile"
	fileContents := "MY FILE"
	resourcePath := `/namespace/project/builds/123/artifacts/file/` + fileName
	ts := testhelper.TestServerWithHandler(regexp.MustCompile(`\A`+resourcePath+`\z`), func(w http.ResponseWriter, r *http.Request) {
		encodedFilename := base64.StdEncoding.EncodeToString([]byte(fileName))
		if _, err := fmt.Fprintf(w, `{"Archive":"%s","Entry":"%s"}`, archivePath, encodedFilename); err != nil {
			t.Fatal(err)
		}
		return
	})
	defer ts.Close()
	ws := startWorkhorseServer(ts.URL)
	defer ws.Close()

	resp, err := http.Get(ws.URL + resourcePath)
	if err != nil {
		t.Error(err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		t.Errorf("GET %q: expected 200, got %d", resourcePath, resp.StatusCode)
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		t.Fatal(err)
	}
	if string(body) != fileContents {
		t.Fatalf("Expected file contents %q, got %q", fileContents, body)
	}
}

561 562 563
func TestGetGitBlob(t *testing.T) {
	blobId := "50b27c6518be44c42c4d87966ae2481ce895624c" // the LICENSE file in the test repository
	blobLength := 1075
564
	headerKey := http.CanonicalHeaderKey("Gitlab-Workhorse-Send-Data")
565 566 567
	ts := testhelper.TestServerWithHandler(regexp.MustCompile(`.`), func(w http.ResponseWriter, r *http.Request) {
		responseJSON := fmt.Sprintf(`{"RepoPath":"%s","BlobId":"%s"}`, path.Join(testRepoRoot, testRepo), blobId)
		encodedJSON := base64.StdEncoding.EncodeToString([]byte(responseJSON))
568
		w.Header().Set(headerKey, "git-blob:"+encodedJSON)
569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585
		if _, err := fmt.Fprintf(w, "GNU General Public License"); err != nil {
			t.Fatal(err)
		}
		return
	})
	defer ts.Close()
	ws := startWorkhorseServer(ts.URL)
	defer ws.Close()
	resourcePath := "/something"
	resp, err := http.Get(ws.URL + resourcePath)
	if err != nil {
		t.Error(err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		t.Errorf("GET %q: expected 200, got %d", resourcePath, resp.StatusCode)
	}
586 587 588
	if len(resp.Header[headerKey]) != 0 {
		t.Fatalf("Unexpected response header: %s: %q", headerKey, resp.Header.Get(headerKey))
	}
589 590 591 592 593 594 595
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		t.Fatal(err)
	}
	if len(body) != blobLength {
		t.Fatalf("Expected body of %d bytes, got %d", blobLength, len(body))
	}
596 597 598
	if cl := resp.Header.Get("Content-Length"); cl != fmt.Sprintf("%d", blobLength) {
		t.Fatalf("Expected Content-Length %v, got %q", blobLength, cl)
	}
599 600 601 602 603
	if !strings.HasPrefix(string(body), "The MIT License (MIT)") {
		t.Fatalf("Expected MIT license, got %q", body)
	}
}

604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619
func setupStaticFile(fpath, content string) error {
	cwd, err := os.Getwd()
	if err != nil {
		return err
	}
	*documentRoot = path.Join(cwd, testDocumentRoot)
	if err := os.MkdirAll(path.Join(*documentRoot, path.Dir(fpath)), 0755); err != nil {
		return err
	}
	static_file := path.Join(*documentRoot, fpath)
	if err := ioutil.WriteFile(static_file, []byte(content), 0666); err != nil {
		return err
	}
	return nil
}

620 621 622 623 624 625 626 627 628
func prepareDownloadDir(t *testing.T) {
	if err := os.RemoveAll(scratchDir); err != nil {
		t.Fatal(err)
	}
	if err := os.MkdirAll(scratchDir, 0755); err != nil {
		t.Fatal(err)
	}
}

Jacob Vosmaer's avatar
Jacob Vosmaer committed
629
func preparePushRepo(t *testing.T) {
Jacob Vosmaer's avatar
Jacob Vosmaer committed
630 631 632 633 634
	if err := os.RemoveAll(scratchDir); err != nil {
		t.Fatal(err)
	}
	cloneCmd := exec.Command("git", "clone", path.Join(testRepoRoot, testRepo), checkoutDir)
	runOrFail(t, cloneCmd)
Jacob Vosmaer's avatar
Jacob Vosmaer committed
635 636 637 638
}

func newBranch() string {
	return fmt.Sprintf("branch-%d", time.Now().UnixNano())
Jacob Vosmaer's avatar
Jacob Vosmaer committed
639 640
}

641
func testAuthServer(url *regexp.Regexp, code int, body interface{}) *httptest.Server {
642
	return testhelper.TestServerWithHandler(url, func(w http.ResponseWriter, r *http.Request) {
643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659
		// Write pure string
		if data, ok := body.(string); ok {
			log.Println("UPSTREAM", r.Method, r.URL, code)
			w.WriteHeader(code)
			fmt.Fprint(w, data)
			return
		}

		// Write json string
		data, err := json.Marshal(body)
		if err != nil {
			log.Println("UPSTREAM", r.Method, r.URL, "FAILURE", err)
			w.WriteHeader(503)
			fmt.Fprint(w, err)
			return
		}

660
		log.Println("UPSTREAM", r.Method, r.URL, code)
661
		w.WriteHeader(code)
662 663
		w.Write(data)
	})
664 665
}

666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688
func archiveOKServer(t *testing.T, archiveName string) *httptest.Server {
	return testhelper.TestServerWithHandler(regexp.MustCompile("."), func(w http.ResponseWriter, r *http.Request) {
		cwd, err := os.Getwd()
		if err != nil {
			t.Fatal(err)
		}
		archivePath := path.Join(cwd, cacheDir, archiveName)

		params := struct{ RepoPath, ArchivePath, CommitId, ArchivePrefix string }{
			repoPath(t),
			archivePath,
			"c7fbe50c7c7419d9701eebe64b1fdacc3df5b9dd",
			"foobar123",
		}
		jsonData, err := json.Marshal(params)
		if err != nil {
			t.Fatal(err)
		}
		encodedJSON := base64.StdEncoding.EncodeToString(jsonData)
		w.Header().Set("Gitlab-Workhorse-Send-Data", "git-archive:"+encodedJSON)
	})
}

689
func startWorkhorseServer(authBackend string) *httptest.Server {
690 691 692 693 694 695
	u := upstream.NewUpstream(
		helper.URLMustParse(authBackend),
		"",
		"123",
		testDocumentRoot,
		false,
696
		0,
697
	)
698
	return httptest.NewServer(u)
699
}
700 701

func runOrFail(t *testing.T, cmd *exec.Cmd) {
702 703 704
	out, err := cmd.CombinedOutput()
	t.Logf("%s", out)
	if err != nil {
705 706 707
		t.Fatal(err)
	}
}
708

709
func gitOkBody(t *testing.T) interface{} {
710
	return &api.Response{
711 712 713
		GL_ID:    "user-123",
		RepoPath: repoPath(t),
	}
714 715 716 717 718 719 720 721 722
}

func repoPath(t *testing.T) string {
	cwd, err := os.Getwd()
	if err != nil {
		t.Fatal(err)
	}
	return path.Join(cwd, testRepoRoot, testRepo)
}
723 724 725 726 727 728 729

// sha1(data) as human-readable string
func sha1s(data []byte) string {
	return fmt.Sprintf("%x", sha1.Sum(data))
}

// download an URL
730
func download(t *testing.T, url, username, password string, h http.Header) (*http.Response, []byte) {
731 732 733 734
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		t.Fatal(err)
	}
735 736 737
	if !(username == "" && password == "") {
		req.SetBasicAuth(username, password)
	}
738 739 740 741
	// copy header to request
	for k, v := range h {
		req.Header[k] = v
	}
742 743 744
	client := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error {
		return http.ErrUseLastResponse // don't follow redirects
	}}
745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761
	resp, err := client.Do(req)
	if err != nil {
		t.Fatal(err)
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		t.Fatal(err)
	}
	return resp, body
}

// Context for downloading & verifying paths under URL prefix
type DownloadContext struct {
	t         *testing.T
	urlPrefix string
	Header    http.Header
762 763
	username  string
	password  string
764 765 766 767
}

func NewDownloadContext(t *testing.T, urlPrefix string) *DownloadContext {
	h := make(http.Header)
768
	return &DownloadContext{t, urlPrefix, h, "", ""}
769 770 771
}

func (dl DownloadContext) download(path string) (*http.Response, []byte) {
772
	return download(dl.t, dl.urlPrefix+path, dl.username, dl.password, dl.Header)
773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799
}

// download `path` and expect content sha1 to be `expectSha1`
func (dl DownloadContext) ExpectSha1(path, expectSha1 string) {
	resp, out := dl.download(path)
	if resp.StatusCode != 200 {
		dl.t.Fatalf("Unexpected status code (expected 200, got %v)", resp.StatusCode)
	}
	outSha1 := sha1s(out)
	if outSha1 != expectSha1 {
		dl.t.Fatal("Unexpected content in blob download")
	}
}

// download `path` and expect content data to be `expect`
func (dl DownloadContext) Expect(path, expect string) {
	dl.ExpectSha1(path, sha1s([]byte(expect)))
}

// download `path` and expect HTTP status code to be `code`
func (dl DownloadContext) ExpectCode(path string, code int) {
	resp, _ := dl.download(path)
	if resp.StatusCode != code {
		dl.t.Fatalf("Unexpected status code (expected %v, got %v)", code, resp.StatusCode)
	}
}

800 801 802 803 804 805 806 807
// download `path` and expect HTTP reply header to be set to `value`.
func (dl DownloadContext) ExpectHeader(path, header, value string) {
	resp, _ := dl.download(path)
	if h := resp.Header.Get(header); h != value {
		dl.t.Fatalf("Header %q: expected %q; got %q", header, value, h)
	}
}

808 809
func TestBlobDownload(t *testing.T) {
	// Prepare test server and "all-ok" auth backend
810
	ts := archiveOKServer(t, "")
811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841
	defer ts.Close()
	ws := startWorkhorseServer(ts.URL)
	defer ws.Close()
	dl := NewDownloadContext(t, fmt.Sprintf("%s/%s/raw", ws.URL, testProject))

	dl.Expect("/5f923865/README.md", "testme\n======\n\nSample repo for testing gitlab features\n")
	dl.ExpectSha1("/5f923865/README.md", "5f7af35c185a9e5face2f4afb6d7c4f00328d04c")
	dl.ExpectSha1("/5f923865/files/ruby/popen.rb", "68990cc20fa74383358797a27967fa2b45d7d8f6")
	dl.ExpectSha1("/874797c3/files/ruby/popen.rb", "4c266708f2bfd7ca3fed3f7ec74253f92ff3fe73")
	dl.ExpectCode("/master/non-existing-file", 404)
}

func TestDeniedBlobDownload(t *testing.T) {
	// Prepare test server and "all-deny" auth backend
	ts := testAuthServer(nil, 403, "Access denied")
	defer ts.Close()
	ws := startWorkhorseServer(ts.URL)
	defer ws.Close()
	dl := NewDownloadContext(t, fmt.Sprintf("%s/%s/raw", ws.URL, testProject))

	dl.ExpectCode("/5f923865/README.md", 403)
	dl.ExpectCode("/5f923865/files/ruby/popen.rb", 403)
	dl.ExpectCode("/874797c3/files/ruby/popen.rb", 403)
	dl.ExpectCode("/master/non-existing-file", 403)
}

func TestPrivateBlobDownload(t *testing.T) {
	// Prepare test server and auth backend:
	// access is ok if token is provided either via query or via header
	ts := testhelper.TestServerWithHandler(nil, func(w http.ResponseWriter, r *http.Request) {
		log.Println("UPSTREAM", r.Method, r.URL)
842
		gitfetch := (strings.HasSuffix(r.URL.Path, "/info/refs") && r.URL.RawQuery == "service=git-upload-pack")
843 844 845 846 847 848 849 850
		token1 := r.URL.Query().Get("aaa_token")
		token2 := r.Header.Get("BBB-TOKEN")
		cookie := ""
		if c, err := r.Cookie("_gitlab_session"); err == nil {
			cookie = c.Value
		}
		username, password, user_ok := r.BasicAuth()
		if user_ok {
851
			// user:password only accepted for `git fetch` requests
852 853 854 855 856 857 858 859
			user_ok = (gitfetch && username == "user-ddd" && password == "password-eee")
		}

		// simulate rails which gives "302 location: .../users/sign_in" when no access by token
		if !gitfetch && (token1 == "" && token2 == "" && cookie == "") {
			w.Header().Set("location", ".../users/sign_in")
			w.WriteHeader(302)
			return
860
		}
861 862

		if !(token1 == "TOKEN-4AAA" || token2 == "TOKEN-4BBB" || cookie == "COOKIE-CCC" || user_ok) {
863 864 865 866 867
			w.WriteHeader(403)
			fmt.Fprintf(w, "Access denied")
			return
		}

868 869 870 871 872 873 874 875 876 877 878 879
		// `git fetch` expects json in body, not senddata-compatible headers
		if gitfetch {
			data, err := json.Marshal(gitOkBody(t))
			if err != nil {
				t.Fatal(err)
			}

			w.WriteHeader(200)
			w.Write(data)
			return
		}

880 881 882 883
		// for authorized .../repository/archive.zip reply the same way archiveOKServer does.
		aok := archiveOKServer(t, "")
		defer aok.Close()
		aokurl, err := url.Parse(aok.URL)
884 885 886
		if err != nil {
			t.Fatal(err)
		}
887 888
		proxy := httputil.NewSingleHostReverseProxy(aokurl)
		proxy.ServeHTTP(w, r)
889 890 891 892 893 894
	})
	defer ts.Close()
	ws := startWorkhorseServer(ts.URL)
	defer ws.Close()
	dl := NewDownloadContext(t, fmt.Sprintf("%s/%s/raw", ws.URL, testProject))

895 896 897
	dl.ExpectCode("/5f923865/README.md", 401)
	dl.ExpectCode("/5f923865/README.md?bbb_token=TOKEN-4BBB", 302)
	dl.ExpectCode("/5f923865/README.md?aaa_token=TOKEN-XXXX", 403)
898 899 900
	dl.ExpectCode("/5f923865/README.md?aaa_token=TOKEN-4AAA", 200)
	dl.ExpectSha1("/5f923865/README.md?aaa_token=TOKEN-4AAA", "5f7af35c185a9e5face2f4afb6d7c4f00328d04c")

901 902 903
	dl.Header.Set("AAA-TOKEN", "TOKEN-4AAA")
	dl.ExpectCode("/5f923865/README.md", 302)
	dl.Header.Set("BBB-TOKEN", "TOKEN-XXX")
904
	dl.ExpectCode("/5f923865/README.md", 403)
905
	dl.Header.Set("BBB-TOKEN", "TOKEN-4BBB")
906 907 908 909
	dl.ExpectCode("/5f923865/README.md", 200)
	dl.ExpectSha1("/5f923865/README.md", "5f7af35c185a9e5face2f4afb6d7c4f00328d04c")

	dl.Header = make(http.Header) // clear
910
	dl.ExpectCode("/5f923865/README.md", 401)
911
	dl.Header.Set("Cookie", "alpha=1")
912
	dl.ExpectCode("/5f923865/README.md", 401)
913
	dl.Header.Set("Cookie", "alpha=1; beta=2")
914 915
	dl.ExpectCode("/5f923865/README.md", 401)
	dl.Header.Set("Cookie", "alpha=1; _gitlab_session=COOKIE-XXX; beta=2")
916 917 918 919
	dl.ExpectCode("/5f923865/README.md", 403)
	dl.Header.Set("Cookie", "alpha=1; _gitlab_session=COOKIE-CCC; beta=2")
	dl.ExpectCode("/5f923865/README.md", 200)
	dl.ExpectSha1("/5f923865/README.md", "5f7af35c185a9e5face2f4afb6d7c4f00328d04c")
920 921

	dl.Header = make(http.Header) // clear
922 923
	dl.ExpectCode("/5f923865/README.md", 401)
	dl.ExpectHeader("/5f923865/README.md", "www-authenticate", "Basic realm=\"\"")
924 925 926 927 928
	dl.username = "user-aaa"
	dl.password = "password-bbb"
	dl.ExpectCode("/5f923865/README.md", 403)
	dl.username = "user-ddd"
	dl.password = "password-eee"
929
	dl.ExpectCode("/5f923865/README.md?qqq_token=1", 302)
930 931
	dl.ExpectCode("/5f923865/README.md", 200)
	dl.ExpectSha1("/5f923865/README.md", "5f7af35c185a9e5face2f4afb6d7c4f00328d04c")
932
}