Merge branch 'qmnguyen0711/upgrade-gitaly-image-for-workhorse-integration-test' into 'master'

Use matching gitaly binary for workhorse integration tests

See merge request gitlab-org/gitlab!72271

.gitlab/ci/workhorse.gitlab-ci.yml

@@ -9,24 +9,21 @@ workhorse:verify:
 
 .workhorse:test:
   extends: .workhorse:rules:workhorse
-  services:
-    - name: registry.gitlab.com/gitlab-org/build/cng/gitaly:latest
-      # Disable the hooks so we don't have to stub the GitLab API
-      command: ["/usr/bin/env", "GITALY_TESTING_NO_GIT_HOOKS=1", "/scripts/process-wrapper"]
-      alias: gitaly
   variables:
-    GITALY_ADDRESS: "tcp://gitaly:8075"
+    GITALY_ADDRESS: "tcp://127.0.0.1:8075"
   stage: test
-  needs: []
+  needs:
+    - setup-test-env
   script:
     - go version
     - apt-get update && apt-get -y install libimage-exiftool-perl
+    - scripts/gitaly-test-build
     - make -C workhorse test
 
 workhorse:test using go 1.16:
   extends: .workhorse:test
-  image: ${GITLAB_DEPENDENCY_PROXY}golang:1.16
+  image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7-golang-1.16-git-2.31
 
 workhorse:test using go 1.17:
   extends: .workhorse:test
-  image: ${GITLAB_DEPENDENCY_PROXY}golang:1.17
+  image: registry.gitlab.com/gitlab-org/gitlab-build-images:ruby-2.7-golang-1.17-git-2.31

workhorse/Makefile

@@ -13,6 +13,8 @@ else
 	BUILD_TIME := $(shell date -u "$(DATE_FMT)")
 endif
 GOBUILD := go build -ldflags "-X main.Version=$(VERSION_STRING) -X main.BuildTime=$(BUILD_TIME)"
+GITALY  := tmp/tests/gitaly/_build/bin/gitaly
+GITALY_PID_FILE := gitaly.pid
 EXE_ALL := gitlab-resize-image gitlab-zip-cat gitlab-zip-metadata gitlab-workhorse
 INSTALL := install
 BUILD_TAGS := tracer_static tracer_static_jaeger continuous_profiler_stackdriver
@@ -63,7 +65,16 @@ install: $(EXE_ALL)
 .PHONY:	test
 test: prepare-tests
 	$(call message,$@)
-	@go test -tags "$(BUILD_TAGS)" ./...
+	go test -tags "$(BUILD_TAGS)" ./... ;\
+	status="$$?" ;\
+	if [ -f "$(GITALY_PID_FILE)" ] ; then \
+		echo "Clean up Gitaly server for workhorse integration test" ;\
+		kill -9 $$(cat $(GITALY_PID_FILE)) ;\
+		rm $(GITALY_PID_FILE) ;\
+	else \
+		echo "Gitaly integration test not running" ;\
+	fi ;\
+	exit "$$status"
 	@echo SUCCESS
 
 .PHONY:	clean
@@ -82,9 +93,27 @@ clean-build:
 	rm -rf $(TARGET_DIR)
 
 .PHONY:	prepare-tests
+prepare-tests:	run-gitaly
 prepare-tests:	testdata/data/group/test.git $(EXE_ALL)
 prepare-tests:	testdata/scratch
 
+.PHONY: run-gitaly
+run-gitaly: gitaly.pid
+
+$(GITALY_PID_FILE): gitaly.toml
+	@{ \
+	if [ -z "$${GITALY_ADDRESS+x}" ] ; then \
+		echo "To run gitaly integration tests set GITALY_ADDRESS=tcp://127.0.0.1:8075" ; \
+	else \
+		cd .. ; \
+		GITALY_TESTING_NO_GIT_HOOKS=1 GITALY_PID_FILE=workhorse/$(GITALY_PID_FILE) $(GITALY) workhorse/gitaly.toml ; \
+	fi \
+	} &
+
+gitaly.toml: ../tmp/tests/gitaly/config.toml
+	sed -e 's/^socket_path.*$$/listen_addr = "0.0.0.0:8075"/;s/^\[auth\]$$//;s/^token.*$$//;s/^internal_socket_dir.*$$//' \
+		$< > $@
+
 testdata/data/group/test.git:
 	$(call message,$@)
 	git clone --quiet --bare https://gitlab.com/gitlab-org/gitlab-test.git $@

workhorse/gitaly_integration_test.go

@@ -16,6 +16,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/require"
 	"gitlab.com/gitlab-org/gitaly/v14/proto/go/gitalypb"
 
@@ -53,6 +54,10 @@ func realGitalyOkBody(t *testing.T) *api.Response {
 	return realGitalyAuthResponse(gitOkBody(t))
 }
 
+func realGitalyOkBodyWithSidechannel(t *testing.T) *api.Response {
+	return realGitalyAuthResponse(gitOkBodyWithSidechannel(t))
+}
+
 func ensureGitalyRepository(t *testing.T, apiResponse *api.Response) error {
 	ctx, namespace, err := gitaly.NewNamespaceClient(context.Background(), apiResponse.GitalyServer)
 	if err != nil {
@@ -83,10 +88,18 @@ func ensureGitalyRepository(t *testing.T, apiResponse *api.Response) error {
 }
 
 func TestAllowedClone(t *testing.T) {
+	testAllowedClone(t, realGitalyOkBody(t))
+}
+
+func TestAllowedCloneWithSidechannel(t *testing.T) {
+	gitaly.InitializeSidechannelRegistry(logrus.StandardLogger())
+	testAllowedClone(t, realGitalyOkBodyWithSidechannel(t))
+}
+
+func testAllowedClone(t *testing.T, apiResponse *api.Response) {
 	skipUnlessRealGitaly(t)
 
 	// Create the repository in the Gitaly server
-	apiResponse := realGitalyOkBody(t)
 	require.NoError(t, ensureGitalyRepository(t, apiResponse))
 
 	// Prepare test server and backend
@@ -107,10 +120,18 @@ func TestAllowedClone(t *testing.T) {
 }
 
 func TestAllowedShallowClone(t *testing.T) {
+	testAllowedShallowClone(t, realGitalyOkBody(t))
+}
+
+func TestAllowedShallowCloneWithSidechannel(t *testing.T) {
+	gitaly.InitializeSidechannelRegistry(logrus.StandardLogger())
+	testAllowedShallowClone(t, realGitalyOkBodyWithSidechannel(t))
+}
+
+func testAllowedShallowClone(t *testing.T, apiResponse *api.Response) {
 	skipUnlessRealGitaly(t)
 
 	// Create the repository in the Gitaly server
-	apiResponse := realGitalyOkBody(t)
 	require.NoError(t, ensureGitalyRepository(t, apiResponse))
 
 	// Prepare test server and backend