Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
G
gitlab-workhorse
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
1
Merge Requests
1
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
nexedi
gitlab-workhorse
Commits
51d1256c
Commit
51d1256c
authored
Jan 25, 2016
by
Jacob Vosmaer
Browse files
Options
Browse Files
Download
Plain Diff
Merge branch 'master' of
https://gitlab.com/gitlab-org/gitlab-workhorse
into raw-blob
parents
56ac73c7
4fe063cf
Changes
29
Hide whitespace changes
Inline
Side-by-side
Showing
29 changed files
with
1011 additions
and
101 deletions
+1011
-101
.gitignore
.gitignore
+2
-0
.gitlab-ci.yml
.gitlab-ci.yml
+4
-4
CHANGELOG
CHANGELOG
+10
-1
Makefile
Makefile
+16
-6
VERSION
VERSION
+1
-1
cmd/gitlab-zip-cat/main.go
cmd/gitlab-zip-cat/main.go
+86
-0
cmd/gitlab-zip-metadata/main.go
cmd/gitlab-zip-metadata/main.go
+36
-0
internal/api/api.go
internal/api/api.go
+4
-0
internal/artifacts/artifact_download.go
internal/artifacts/artifact_download.go
+99
-0
internal/artifacts/artifact_download_test.go
internal/artifacts/artifact_download_test.go
+116
-0
internal/artifacts/artifacts_upload.go
internal/artifacts/artifacts_upload.go
+86
-0
internal/artifacts/artifacts_upload_test.go
internal/artifacts/artifacts_upload_test.go
+174
-0
internal/artifacts/escape_quotes.go
internal/artifacts/escape_quotes.go
+10
-0
internal/git/archive.go
internal/git/archive.go
+4
-3
internal/git/command.go
internal/git/command.go
+0
-15
internal/git/git-http.go
internal/git/git-http.go
+4
-4
internal/helper/helpers.go
internal/helper/helpers.go
+31
-0
internal/lfs/lfs.go
internal/lfs/lfs.go
+1
-1
internal/upload/artifacts.go
internal/upload/artifacts.go
+0
-13
internal/upload/uploads.go
internal/upload/uploads.go
+52
-41
internal/upload/uploads_test.go
internal/upload/uploads_test.go
+75
-6
internal/upstream/routes.go
internal/upstream/routes.go
+4
-3
internal/upstream/upstream.go
internal/upstream/upstream.go
+1
-1
internal/zipartifacts/codes.go
internal/zipartifacts/codes.go
+7
-0
internal/zipartifacts/entry.go
internal/zipartifacts/entry.go
+13
-0
internal/zipartifacts/metadata.go
internal/zipartifacts/metadata.go
+104
-0
main_test.go
main_test.go
+69
-2
support/path
support/path
+2
-0
testdata/artifacts-archive.zip
testdata/artifacts-archive.zip
+0
-0
No files found.
.gitignore
View file @
51d1256c
...
@@ -2,3 +2,5 @@ gitlab-workhorse
...
@@ -2,3 +2,5 @@ gitlab-workhorse
testdata/data
testdata/data
testdata/scratch
testdata/scratch
testdata/public
testdata/public
gitlab-zip-cat
gitlab-zip-metadata
.gitlab-ci.yml
View file @
51d1256c
before_script
:
before_script
:
-
test -f /.dockerinit &&
apt-get update -qq && apt-get install -y curl unzip bzip2
-
apt-get update -qq && apt-get install -y curl unzip bzip2
-
curl -O https://storage.googleapis.com/golang/go1.5.
2
.linux-amd64.tar.gz
-
curl -O https://storage.googleapis.com/golang/go1.5.
3
.linux-amd64.tar.gz
-
echo '
cae87ed095e8d94a81871281d35da7829bd1234e go1.5.2.linux-amd64.tar.gz' | shasum -c
-
-
echo '
43afe0c5017e502630b1aea4d44b8a7f059bf60d7f29dfd58db454d4e4e0ae53 go1.5.3.linux-amd64.tar.gz' | shasum -c -a256
-
-
test -f /.dockerinit && rm -rf /usr/local/go && tar -C /usr/local -xzf go1.5.2
.linux-amd64.tar.gz
-
rm -rf /usr/local/go && tar -C /usr/local -xzf go1.5.3
.linux-amd64.tar.gz
-
export PATH=/usr/local/go/bin:$PATH
-
export PATH=/usr/local/go/bin:$PATH
test
:
test
:
...
...
CHANGELOG
View file @
51d1256c
...
@@ -2,11 +2,20 @@
...
@@ -2,11 +2,20 @@
Formerly known as 'gitlab-git-http-server'.
Formerly known as 'gitlab-git-http-server'.
0.6.1
Add support for generating zip artifacts metadata and serving single
files from zip archives.
Gitlab-workhorse now consists of multiple executables. We also fixed a
routing bug introduced by the 0.6.0 refactor that broke relative URL
support.
0.6.0
0.6.0
Overhauled the source code organization; no user-facing changes
Overhauled the source code organization; no user-facing changes
(intended). The application code is now split into Go 'packages'
(intended). The application code is now split into Go 'packages'
(modules).
(modules).
As of 0.6.0 gitlab-workhorse requires Go 1.5 or newer.
0.5.4
0.5.4
...
...
Makefile
View file @
51d1256c
PREFIX
=
/usr/local
PREFIX
=
/usr/local
VERSION
=
$(
shell
git describe
)
-
$(
shell
date
-u
+%Y%m%d.%H%M%S
)
VERSION
=
$(
shell
git describe
)
-
$(
shell
date
-u
+%Y%m%d.%H%M%S
)
GOBUILD
=
go build
-ldflags
"-X main.Version=
${VERSION}
"
all
:
gitlab-zip-cat gitlab-zip-metadata gitlab-workhorse
gitlab-zip-cat
:
$(shell find cmd/gitlab-zip-cat/ -name '*.go')
${GOBUILD}
-o
$@
./cmd/
$@
gitlab-zip-metadata
:
$(shell find cmd/gitlab-zip-metadata/ -name '*.go')
${GOBUILD}
-o
$@
./cmd/
$@
gitlab-workhorse
:
$(shell find . -name '*.go')
gitlab-workhorse
:
$(shell find . -name '*.go')
go build
-ldflags
"-X main.Version=
${VERSION}
"
-o
gitlab-workhorse
${GOBUILD}
-o
$@
install
:
gitlab-workhorse
install
:
gitlab-workhorse gitlab-zip-cat gitlab-zip-metadata
install
gitlab-workhorse
${PREFIX}
/bin/
mkdir
-p
$(DESTDIR)${PREFIX}
/bin/
install
gitlab-workhorse gitlab-zip-cat gitlab-zip-metadata
${DESTDIR}${PREFIX}
/bin/
.PHONY
:
test
.PHONY
:
test
test
:
testdata/data/group/test.git clean-workhorse
gitlab-workhorse
test
:
testdata/data/group/test.git clean-workhorse
all
go
fmt
./... |
awk
'{ print } END { if (NR > 0) { print "Please run go fmt"; exit 1 } }'
go
fmt
./... |
awk
'{ print } END { if (NR > 0) { print "Please run go fmt"; exit 1 } }'
go
test
./...
support/path
go
test
./...
@
echo
SUCCESS
@
echo
SUCCESS
coverage
:
testdata/data/group/test.git
coverage
:
testdata/data/group/test.git
...
@@ -30,4 +40,4 @@ clean: clean-workhorse
...
@@ -30,4 +40,4 @@ clean: clean-workhorse
.PHONY
:
clean-workhorse
.PHONY
:
clean-workhorse
clean-workhorse
:
clean-workhorse
:
rm
-f
gitlab-workhorse
rm
-f
gitlab-workhorse
gitlab-zip-cat gitlab-zip-metadata
VERSION
View file @
51d1256c
0.6.
0
0.6.
1
cmd/gitlab-zip-cat/main.go
0 → 100644
View file @
51d1256c
package
main
import
(
"../../internal/zipartifacts"
"archive/zip"
"flag"
"fmt"
"io"
"os"
)
const
progName
=
"gitlab-zip-cat"
var
Version
=
"unknown"
var
printVersion
=
flag
.
Bool
(
"version"
,
false
,
"Print version and exit"
)
func
main
()
{
flag
.
Parse
()
version
:=
fmt
.
Sprintf
(
"%s %s"
,
progName
,
Version
)
if
*
printVersion
{
fmt
.
Println
(
version
)
os
.
Exit
(
0
)
}
if
len
(
os
.
Args
)
!=
3
{
fmt
.
Fprintf
(
os
.
Stderr
,
"Usage: %s FILE.ZIP ENTRY"
,
progName
)
os
.
Exit
(
1
)
}
archiveFileName
:=
os
.
Args
[
1
]
fileName
,
err
:=
zipartifacts
.
DecodeFileEntry
(
os
.
Args
[
2
])
if
err
!=
nil
{
fatalError
(
fmt
.
Errorf
(
"decode entry %q: %v"
,
os
.
Args
[
2
],
err
))
}
archive
,
err
:=
zip
.
OpenReader
(
archiveFileName
)
if
err
!=
nil
{
notFoundError
(
fmt
.
Errorf
(
"open %q: %v"
,
archiveFileName
,
err
))
}
defer
archive
.
Close
()
file
:=
findFileInZip
(
fileName
,
&
archive
.
Reader
)
if
file
==
nil
{
notFoundError
(
fmt
.
Errorf
(
"find %q in %q: not found"
,
fileName
,
archiveFileName
))
}
// Start decompressing the file
reader
,
err
:=
file
.
Open
()
if
err
!=
nil
{
fatalError
(
fmt
.
Errorf
(
"open %q in %q: %v"
,
fileName
,
archiveFileName
,
err
))
}
defer
reader
.
Close
()
if
_
,
err
:=
fmt
.
Printf
(
"%d
\n
"
,
file
.
UncompressedSize64
);
err
!=
nil
{
fatalError
(
fmt
.
Errorf
(
"write file size: %v"
,
err
))
}
if
_
,
err
:=
io
.
Copy
(
os
.
Stdout
,
reader
);
err
!=
nil
{
fatalError
(
fmt
.
Errorf
(
"write %q from %q to stdout: %v"
,
fileName
,
archiveFileName
,
err
))
}
}
func
findFileInZip
(
fileName
string
,
archive
*
zip
.
Reader
)
*
zip
.
File
{
for
_
,
file
:=
range
archive
.
File
{
if
file
.
Name
==
fileName
{
return
file
}
}
return
nil
}
func
printError
(
err
error
)
{
fmt
.
Fprintf
(
os
.
Stderr
,
"%s: %v"
,
progName
,
err
)
}
func
fatalError
(
err
error
)
{
printError
(
err
)
os
.
Exit
(
1
)
}
func
notFoundError
(
err
error
)
{
printError
(
err
)
os
.
Exit
(
zipartifacts
.
StatusEntryNotFound
)
}
cmd/gitlab-zip-metadata/main.go
0 → 100644
View file @
51d1256c
package
main
import
(
"../../internal/zipartifacts"
"flag"
"fmt"
"os"
)
const
progName
=
"gitlab-zip-metadata"
var
Version
=
"unknown"
var
printVersion
=
flag
.
Bool
(
"version"
,
false
,
"Print version and exit"
)
func
main
()
{
flag
.
Parse
()
version
:=
fmt
.
Sprintf
(
"%s %s"
,
progName
,
Version
)
if
*
printVersion
{
fmt
.
Println
(
version
)
os
.
Exit
(
0
)
}
if
len
(
os
.
Args
)
!=
2
{
fmt
.
Fprintf
(
os
.
Stderr
,
"Usage: %s FILE.ZIP"
,
progName
)
os
.
Exit
(
1
)
}
if
err
:=
zipartifacts
.
GenerateZipMetadataFromFile
(
os
.
Args
[
1
],
os
.
Stdout
);
err
!=
nil
{
fmt
.
Fprintf
(
os
.
Stderr
,
"%s: %v
\n
"
,
progName
,
err
)
if
err
==
os
.
ErrInvalid
{
os
.
Exit
(
zipartifacts
.
StatusNotZip
)
}
os
.
Exit
(
1
)
}
}
internal/api/api.go
View file @
51d1256c
...
@@ -57,6 +57,10 @@ type Response struct {
...
@@ -57,6 +57,10 @@ type Response struct {
// TmpPath is the path where we should store temporary files
// TmpPath is the path where we should store temporary files
// This is set by authorization middleware
// This is set by authorization middleware
TempPath
string
TempPath
string
// Archive is the path where the artifacts archive is stored
Archive
string
`json:"archive"`
// Entry is a filename inside the archive point to file that needs to be extracted
Entry
string
`json:"entry"`
}
}
// singleJoiningSlash is taken from reverseproxy.go:NewSingleHostReverseProxy
// singleJoiningSlash is taken from reverseproxy.go:NewSingleHostReverseProxy
...
...
internal/artifacts/artifact_download.go
0 → 100644
View file @
51d1256c
package
artifacts
import
(
"../api"
"../helper"
"../zipartifacts"
"bufio"
"errors"
"fmt"
"io"
"mime"
"net/http"
"os"
"os/exec"
"path/filepath"
"strings"
"syscall"
)
func
detectFileContentType
(
fileName
string
)
string
{
contentType
:=
mime
.
TypeByExtension
(
filepath
.
Ext
(
fileName
))
if
contentType
==
""
{
contentType
=
"application/octet-stream"
}
return
contentType
}
func
unpackFileFromZip
(
archiveFileName
,
encodedFilename
string
,
headers
http
.
Header
,
output
io
.
Writer
)
error
{
fileName
,
err
:=
zipartifacts
.
DecodeFileEntry
(
encodedFilename
)
if
err
!=
nil
{
return
err
}
catFile
:=
exec
.
Command
(
"gitlab-zip-cat"
,
archiveFileName
,
encodedFilename
)
catFile
.
Stderr
=
os
.
Stderr
catFile
.
SysProcAttr
=
&
syscall
.
SysProcAttr
{
Setpgid
:
true
}
stdout
,
err
:=
catFile
.
StdoutPipe
()
if
err
!=
nil
{
return
fmt
.
Errorf
(
"create gitlab-zip-cat stdout pipe: %v"
,
err
)
}
if
err
:=
catFile
.
Start
();
err
!=
nil
{
return
fmt
.
Errorf
(
"start %v: %v"
,
catFile
.
Args
,
err
)
}
defer
helper
.
CleanUpProcessGroup
(
catFile
)
basename
:=
filepath
.
Base
(
fileName
)
reader
:=
bufio
.
NewReader
(
stdout
)
contentLength
,
err
:=
reader
.
ReadString
(
'\n'
)
if
err
!=
nil
{
if
catFileErr
:=
waitCatFile
(
catFile
);
catFileErr
!=
nil
{
return
catFileErr
}
return
fmt
.
Errorf
(
"read content-length: %v"
,
err
)
}
contentLength
=
strings
.
TrimSuffix
(
contentLength
,
"
\n
"
)
// Write http headers about the file
headers
.
Set
(
"Content-Length"
,
contentLength
)
headers
.
Set
(
"Content-Type"
,
detectFileContentType
(
fileName
))
headers
.
Set
(
"Content-Disposition"
,
"attachment; filename=
\"
"
+
escapeQuotes
(
basename
)
+
"
\"
"
)
// Copy file body to client
if
_
,
err
:=
io
.
Copy
(
output
,
reader
);
err
!=
nil
{
return
fmt
.
Errorf
(
"copy stdout of %v: %v"
,
catFile
.
Args
,
err
)
}
return
waitCatFile
(
catFile
)
}
func
waitCatFile
(
cmd
*
exec
.
Cmd
)
error
{
err
:=
cmd
.
Wait
()
if
err
==
nil
{
return
nil
}
if
st
,
ok
:=
helper
.
ExitStatus
(
err
);
ok
&&
st
==
zipartifacts
.
StatusEntryNotFound
{
return
os
.
ErrNotExist
}
return
fmt
.
Errorf
(
"wait for %v to finish: %v"
,
cmd
.
Args
,
err
)
}
// Artifacts downloader doesn't support ranges when downloading a single file
func
DownloadArtifact
(
myAPI
*
api
.
API
)
http
.
Handler
{
return
myAPI
.
PreAuthorizeHandler
(
func
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
,
a
*
api
.
Response
)
{
if
a
.
Archive
==
""
||
a
.
Entry
==
""
{
helper
.
Fail500
(
w
,
errors
.
New
(
"DownloadArtifact: Archive or Path is empty"
))
return
}
err
:=
unpackFileFromZip
(
a
.
Archive
,
a
.
Entry
,
w
.
Header
(),
w
)
if
os
.
IsNotExist
(
err
)
{
http
.
NotFound
(
w
,
r
)
return
}
else
if
err
!=
nil
{
helper
.
Fail500
(
w
,
fmt
.
Errorf
(
"DownloadArtifact: %v"
,
err
))
}
},
""
)
}
internal/artifacts/artifact_download_test.go
0 → 100644
View file @
51d1256c
package
artifacts
import
(
"../api"
"../helper"
"../testhelper"
"archive/zip"
"encoding/base64"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"os"
"testing"
)
func
testArtifactDownloadServer
(
t
*
testing
.
T
,
archive
string
,
entry
string
)
*
httptest
.
Server
{
mux
:=
http
.
NewServeMux
()
mux
.
HandleFunc
(
"/url/path"
,
func
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
)
{
if
r
.
Method
!=
"GET"
{
t
.
Fatal
(
"Expected GET request"
)
}
w
.
Header
()
.
Set
(
"Content-Type"
,
"application/json"
)
data
,
err
:=
json
.
Marshal
(
&
api
.
Response
{
Archive
:
archive
,
Entry
:
base64
.
StdEncoding
.
EncodeToString
([]
byte
(
entry
)),
})
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
w
.
Write
(
data
)
})
return
testhelper
.
TestServerWithHandler
(
nil
,
mux
.
ServeHTTP
)
}
func
testDownloadArtifact
(
t
*
testing
.
T
,
ts
*
httptest
.
Server
)
*
httptest
.
ResponseRecorder
{
httpRequest
,
err
:=
http
.
NewRequest
(
"GET"
,
ts
.
URL
+
"/url/path"
,
nil
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
response
:=
httptest
.
NewRecorder
()
apiClient
:=
api
.
NewAPI
(
helper
.
URLMustParse
(
ts
.
URL
),
"123"
,
nil
)
DownloadArtifact
(
apiClient
)
.
ServeHTTP
(
response
,
httpRequest
)
return
response
}
func
TestDownloadingFromValidArchive
(
t
*
testing
.
T
)
{
tempFile
,
err
:=
ioutil
.
TempFile
(
""
,
"uploads"
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
defer
tempFile
.
Close
()
defer
os
.
Remove
(
tempFile
.
Name
())
archive
:=
zip
.
NewWriter
(
tempFile
)
defer
archive
.
Close
()
fileInArchive
,
err
:=
archive
.
Create
(
"test.txt"
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
fmt
.
Fprint
(
fileInArchive
,
"testtest"
)
archive
.
Close
()
ts
:=
testArtifactDownloadServer
(
t
,
tempFile
.
Name
(),
"test.txt"
)
defer
ts
.
Close
()
response
:=
testDownloadArtifact
(
t
,
ts
)
testhelper
.
AssertResponseCode
(
t
,
response
,
200
)
testhelper
.
AssertResponseHeader
(
t
,
response
,
"Content-Type"
,
"text/plain; charset=utf-8"
)
testhelper
.
AssertResponseHeader
(
t
,
response
,
"Content-Disposition"
,
"attachment; filename=
\"
test.txt
\"
"
)
testhelper
.
AssertResponseBody
(
t
,
response
,
"testtest"
)
}
func
TestDownloadingNonExistingFile
(
t
*
testing
.
T
)
{
tempFile
,
err
:=
ioutil
.
TempFile
(
""
,
"uploads"
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
defer
tempFile
.
Close
()
defer
os
.
Remove
(
tempFile
.
Name
())
archive
:=
zip
.
NewWriter
(
tempFile
)
defer
archive
.
Close
()
archive
.
Close
()
ts
:=
testArtifactDownloadServer
(
t
,
tempFile
.
Name
(),
"test"
)
defer
ts
.
Close
()
response
:=
testDownloadArtifact
(
t
,
ts
)
testhelper
.
AssertResponseCode
(
t
,
response
,
404
)
}
func
TestDownloadingFromInvalidArchive
(
t
*
testing
.
T
)
{
ts
:=
testArtifactDownloadServer
(
t
,
"path/to/non/existing/file"
,
"test"
)
defer
ts
.
Close
()
response
:=
testDownloadArtifact
(
t
,
ts
)
testhelper
.
AssertResponseCode
(
t
,
response
,
404
)
}
func
TestIncompleteApiResponse
(
t
*
testing
.
T
)
{
ts
:=
testArtifactDownloadServer
(
t
,
""
,
""
)
defer
ts
.
Close
()
response
:=
testDownloadArtifact
(
t
,
ts
)
testhelper
.
AssertResponseCode
(
t
,
response
,
500
)
}
internal/artifacts/artifacts_upload.go
0 → 100644
View file @
51d1256c
package
artifacts
import
(
"../api"
"../helper"
"../upload"
"../zipartifacts"
"errors"
"fmt"
"io/ioutil"
"mime/multipart"
"net/http"
"os"
"os/exec"
"syscall"
)
type
artifactsUploadProcessor
struct
{
TempPath
string
metadataFile
string
}
func
(
a
*
artifactsUploadProcessor
)
ProcessFile
(
formName
,
fileName
string
,
writer
*
multipart
.
Writer
)
error
{
// ProcessFile for artifacts requires file form-data field name to eq `file`
if
formName
!=
"file"
{
return
fmt
.
Errorf
(
"Invalid form field: %q"
,
formName
)
}
if
a
.
metadataFile
!=
""
{
return
fmt
.
Errorf
(
"Artifacts request contains more than one file!"
)
}
// Create temporary file for metadata and store it's path
tempFile
,
err
:=
ioutil
.
TempFile
(
a
.
TempPath
,
"metadata_"
)
if
err
!=
nil
{
return
err
}
defer
tempFile
.
Close
()
a
.
metadataFile
=
tempFile
.
Name
()
// Generate metadata and save to file
zipMd
:=
exec
.
Command
(
"gitlab-zip-metadata"
,
fileName
)
zipMd
.
Stderr
=
os
.
Stderr
zipMd
.
SysProcAttr
=
&
syscall
.
SysProcAttr
{
Setpgid
:
true
}
zipMd
.
Stdout
=
tempFile
if
err
:=
zipMd
.
Start
();
err
!=
nil
{
return
err
}
defer
helper
.
CleanUpProcessGroup
(
zipMd
)
if
err
:=
zipMd
.
Wait
();
err
!=
nil
{
if
st
,
ok
:=
helper
.
ExitStatus
(
err
);
ok
&&
st
==
zipartifacts
.
StatusNotZip
{
return
nil
}
return
err
}
// Pass metadata file path to Rails
writer
.
WriteField
(
"metadata.path"
,
a
.
metadataFile
)
writer
.
WriteField
(
"metadata.name"
,
"metadata.gz"
)
return
nil
}
func
(
a
*
artifactsUploadProcessor
)
ProcessField
(
formName
string
,
writer
*
multipart
.
Writer
)
error
{
return
nil
}
func
(
a
*
artifactsUploadProcessor
)
Cleanup
()
{
if
a
.
metadataFile
!=
""
{
os
.
Remove
(
a
.
metadataFile
)
}
}
func
UploadArtifacts
(
myAPI
*
api
.
API
,
h
http
.
Handler
)
http
.
Handler
{
return
myAPI
.
PreAuthorizeHandler
(
func
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
,
a
*
api
.
Response
)
{
if
a
.
TempPath
==
""
{
helper
.
Fail500
(
w
,
errors
.
New
(
"UploadArtifacts: TempPath is empty"
))
return
}
mg
:=
&
artifactsUploadProcessor
{
TempPath
:
a
.
TempPath
}
defer
mg
.
Cleanup
()
upload
.
HandleFileUploads
(
w
,
r
,
h
,
a
.
TempPath
,
mg
)
},
"/authorize"
)
}
internal/artifacts/artifacts_upload_test.go
0 → 100644
View file @
51d1256c
package
artifacts
import
(
"../api"
"../helper"
"../proxy"
"../testhelper"
"../zipartifacts"
"archive/zip"
"bytes"
"compress/gzip"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"mime/multipart"
"net/http"
"net/http/httptest"
"os"
"testing"
)
func
testArtifactsUploadServer
(
t
*
testing
.
T
,
tempPath
string
)
*
httptest
.
Server
{
mux
:=
http
.
NewServeMux
()
mux
.
HandleFunc
(
"/url/path/authorize"
,
func
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
)
{
if
r
.
Method
!=
"POST"
{
t
.
Fatal
(
"Expected POST request"
)
}
w
.
Header
()
.
Set
(
"Content-Type"
,
"application/json"
)
data
,
err
:=
json
.
Marshal
(
&
api
.
Response
{
TempPath
:
tempPath
,
})
if
err
!=
nil
{
t
.
Fatal
(
"Expected to marshal"
)
}
w
.
Write
(
data
)
})
mux
.
HandleFunc
(
"/url/path"
,
func
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
)
{
if
r
.
Method
!=
"POST"
{
t
.
Fatal
(
"Expected POST request"
)
}
if
r
.
FormValue
(
"file.path"
)
==
""
{
w
.
WriteHeader
(
501
)
return
}
if
r
.
FormValue
(
"metadata.path"
)
==
""
{
w
.
WriteHeader
(
502
)
return
}
_
,
err
:=
ioutil
.
ReadFile
(
r
.
FormValue
(
"file.path"
))
if
err
!=
nil
{
w
.
WriteHeader
(
404
)
return
}
metadata
,
err
:=
ioutil
.
ReadFile
(
r
.
FormValue
(
"metadata.path"
))
if
err
!=
nil
{
w
.
WriteHeader
(
404
)
return
}
gz
,
err
:=
gzip
.
NewReader
(
bytes
.
NewReader
(
metadata
))
if
err
!=
nil
{
w
.
WriteHeader
(
405
)
return
}
defer
gz
.
Close
()
metadata
,
err
=
ioutil
.
ReadAll
(
gz
)
if
err
!=
nil
{
w
.
WriteHeader
(
404
)
return
}
if
!
bytes
.
HasPrefix
(
metadata
,
[]
byte
(
zipartifacts
.
MetadataHeaderPrefix
+
zipartifacts
.
MetadataHeader
))
{
w
.
WriteHeader
(
400
)
return
}
w
.
WriteHeader
(
200
)
})
return
testhelper
.
TestServerWithHandler
(
nil
,
mux
.
ServeHTTP
)
}
func
testUploadArtifacts
(
contentType
string
,
body
io
.
Reader
,
t
*
testing
.
T
,
ts
*
httptest
.
Server
)
*
httptest
.
ResponseRecorder
{
httpRequest
,
err
:=
http
.
NewRequest
(
"POST"
,
ts
.
URL
+
"/url/path"
,
body
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
httpRequest
.
Header
.
Set
(
"Content-Type"
,
contentType
)
response
:=
httptest
.
NewRecorder
()
apiClient
:=
api
.
NewAPI
(
helper
.
URLMustParse
(
ts
.
URL
),
"123"
,
nil
)
proxyClient
:=
proxy
.
NewProxy
(
helper
.
URLMustParse
(
ts
.
URL
),
"123"
,
nil
)
UploadArtifacts
(
apiClient
,
proxyClient
)
.
ServeHTTP
(
response
,
httpRequest
)
return
response
}
func
TestUploadHandlerAddingMetadata
(
t
*
testing
.
T
)
{
tempPath
,
err
:=
ioutil
.
TempDir
(
""
,
"uploads"
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
defer
os
.
RemoveAll
(
tempPath
)
ts
:=
testArtifactsUploadServer
(
t
,
tempPath
)
defer
ts
.
Close
()
var
buffer
bytes
.
Buffer
writer
:=
multipart
.
NewWriter
(
&
buffer
)
file
,
err
:=
writer
.
CreateFormFile
(
"file"
,
"my.file"
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
archive
:=
zip
.
NewWriter
(
file
)
defer
archive
.
Close
()
fileInArchive
,
err
:=
archive
.
Create
(
"test.file"
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
fmt
.
Fprint
(
fileInArchive
,
"test"
)
archive
.
Close
()
writer
.
Close
()
response
:=
testUploadArtifacts
(
writer
.
FormDataContentType
(),
&
buffer
,
t
,
ts
)
testhelper
.
AssertResponseCode
(
t
,
response
,
200
)
}
func
TestUploadHandlerForUnsupportedArchive
(
t
*
testing
.
T
)
{
tempPath
,
err
:=
ioutil
.
TempDir
(
""
,
"uploads"
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
defer
os
.
RemoveAll
(
tempPath
)
ts
:=
testArtifactsUploadServer
(
t
,
tempPath
)
defer
ts
.
Close
()
var
buffer
bytes
.
Buffer
writer
:=
multipart
.
NewWriter
(
&
buffer
)
file
,
err
:=
writer
.
CreateFormFile
(
"file"
,
"my.file"
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
fmt
.
Fprint
(
file
,
"test"
)
writer
.
Close
()
response
:=
testUploadArtifacts
(
writer
.
FormDataContentType
(),
&
buffer
,
t
,
ts
)
// 502 is a custom response code from the mock server in testUploadArtifacts
testhelper
.
AssertResponseCode
(
t
,
response
,
502
)
}
func
TestUploadFormProcessing
(
t
*
testing
.
T
)
{
tempPath
,
err
:=
ioutil
.
TempDir
(
""
,
"uploads"
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
defer
os
.
RemoveAll
(
tempPath
)
ts
:=
testArtifactsUploadServer
(
t
,
tempPath
)
defer
ts
.
Close
()
var
buffer
bytes
.
Buffer
writer
:=
multipart
.
NewWriter
(
&
buffer
)
file
,
err
:=
writer
.
CreateFormFile
(
"metadata"
,
"my.file"
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
fmt
.
Fprint
(
file
,
"test"
)
writer
.
Close
()
response
:=
testUploadArtifacts
(
writer
.
FormDataContentType
(),
&
buffer
,
t
,
ts
)
testhelper
.
AssertResponseCode
(
t
,
response
,
500
)
}
internal/artifacts/escape_quotes.go
0 → 100644
View file @
51d1256c
package
artifacts
import
"strings"
// taken from mime/multipart/writer.go
var
quoteEscaper
=
strings
.
NewReplacer
(
"
\\
"
,
"
\\\\
"
,
`"`
,
"
\\\"
"
)
func
escapeQuotes
(
s
string
)
string
{
return
quoteEscaper
.
Replace
(
s
)
}
internal/git/archive.go
View file @
51d1256c
...
@@ -23,6 +23,7 @@ import (
...
@@ -23,6 +23,7 @@ import (
func
GetArchive
(
a
*
api
.
API
)
http
.
Handler
{
func
GetArchive
(
a
*
api
.
API
)
http
.
Handler
{
return
repoPreAuthorizeHandler
(
a
,
handleGetArchive
)
return
repoPreAuthorizeHandler
(
a
,
handleGetArchive
)
}
}
func
handleGetArchive
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
,
a
*
api
.
Response
)
{
func
handleGetArchive
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
,
a
*
api
.
Response
)
{
var
format
string
var
format
string
urlPath
:=
r
.
URL
.
Path
urlPath
:=
r
.
URL
.
Path
...
@@ -78,7 +79,7 @@ func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
...
@@ -78,7 +79,7 @@ func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
helper
.
Fail500
(
w
,
fmt
.
Errorf
(
"handleGetArchive: start %v: %v"
,
archiveCmd
.
Args
,
err
))
helper
.
Fail500
(
w
,
fmt
.
Errorf
(
"handleGetArchive: start %v: %v"
,
archiveCmd
.
Args
,
err
))
return
return
}
}
defer
c
leanUpProcessGroup
(
archiveCmd
)
// Ensure brute force subprocess clean-up
defer
helper
.
C
leanUpProcessGroup
(
archiveCmd
)
// Ensure brute force subprocess clean-up
var
stdout
io
.
ReadCloser
var
stdout
io
.
ReadCloser
if
compressCmd
==
nil
{
if
compressCmd
==
nil
{
...
@@ -98,7 +99,7 @@ func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
...
@@ -98,7 +99,7 @@ func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
helper
.
Fail500
(
w
,
fmt
.
Errorf
(
"handleGetArchive: start %v: %v"
,
compressCmd
.
Args
,
err
))
helper
.
Fail500
(
w
,
fmt
.
Errorf
(
"handleGetArchive: start %v: %v"
,
compressCmd
.
Args
,
err
))
return
return
}
}
defer
c
leanUpProcessGroup
(
compressCmd
)
defer
helper
.
C
leanUpProcessGroup
(
compressCmd
)
archiveStdout
.
Close
()
archiveStdout
.
Close
()
}
}
...
@@ -110,7 +111,7 @@ func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
...
@@ -110,7 +111,7 @@ func handleGetArchive(w http.ResponseWriter, r *http.Request, a *api.Response) {
setArchiveHeaders
(
w
,
format
,
archiveFilename
)
setArchiveHeaders
(
w
,
format
,
archiveFilename
)
w
.
WriteHeader
(
200
)
// Don't bother with HTTP 500 from this point on, just return
w
.
WriteHeader
(
200
)
// Don't bother with HTTP 500 from this point on, just return
if
_
,
err
:=
io
.
Copy
(
w
,
archiveReader
);
err
!=
nil
{
if
_
,
err
:=
io
.
Copy
(
w
,
archiveReader
);
err
!=
nil
{
helper
.
LogError
(
fmt
.
Errorf
(
"handleGetArchive:
read
: %v"
,
err
))
helper
.
LogError
(
fmt
.
Errorf
(
"handleGetArchive:
copy 'git archive' output
: %v"
,
err
))
return
return
}
}
if
err
:=
archiveCmd
.
Wait
();
err
!=
nil
{
if
err
:=
archiveCmd
.
Wait
();
err
!=
nil
{
...
...
internal/git/command.go
View file @
51d1256c
...
@@ -23,18 +23,3 @@ func gitCommand(gl_id string, name string, args ...string) *exec.Cmd {
...
@@ -23,18 +23,3 @@ func gitCommand(gl_id string, name string, args ...string) *exec.Cmd {
cmd
.
Stderr
=
os
.
Stderr
cmd
.
Stderr
=
os
.
Stderr
return
cmd
return
cmd
}
}
func
cleanUpProcessGroup
(
cmd
*
exec
.
Cmd
)
{
if
cmd
==
nil
{
return
}
process
:=
cmd
.
Process
if
process
!=
nil
&&
process
.
Pid
>
0
{
// Send SIGTERM to the process group of cmd
syscall
.
Kill
(
-
process
.
Pid
,
syscall
.
SIGTERM
)
}
// reap our child process
cmd
.
Wait
()
}
internal/git/git-http.go
View file @
51d1256c
...
@@ -72,7 +72,7 @@ func handleGetInfoRefs(w http.ResponseWriter, r *http.Request, a *api.Response)
...
@@ -72,7 +72,7 @@ func handleGetInfoRefs(w http.ResponseWriter, r *http.Request, a *api.Response)
helper
.
Fail500
(
w
,
fmt
.
Errorf
(
"handleGetInfoRefs: start %v: %v"
,
cmd
.
Args
,
err
))
helper
.
Fail500
(
w
,
fmt
.
Errorf
(
"handleGetInfoRefs: start %v: %v"
,
cmd
.
Args
,
err
))
return
return
}
}
defer
c
leanUpProcessGroup
(
cmd
)
// Ensure brute force subprocess clean-up
defer
helper
.
C
leanUpProcessGroup
(
cmd
)
// Ensure brute force subprocess clean-up
// Start writing the response
// Start writing the response
w
.
Header
()
.
Add
(
"Content-Type"
,
fmt
.
Sprintf
(
"application/x-%s-advertisement"
,
rpc
))
w
.
Header
()
.
Add
(
"Content-Type"
,
fmt
.
Sprintf
(
"application/x-%s-advertisement"
,
rpc
))
...
@@ -87,7 +87,7 @@ func handleGetInfoRefs(w http.ResponseWriter, r *http.Request, a *api.Response)
...
@@ -87,7 +87,7 @@ func handleGetInfoRefs(w http.ResponseWriter, r *http.Request, a *api.Response)
return
return
}
}
if
_
,
err
:=
io
.
Copy
(
w
,
stdout
);
err
!=
nil
{
if
_
,
err
:=
io
.
Copy
(
w
,
stdout
);
err
!=
nil
{
helper
.
LogError
(
fmt
.
Errorf
(
"handleGetInfoRefs:
read from
%v: %v"
,
cmd
.
Args
,
err
))
helper
.
LogError
(
fmt
.
Errorf
(
"handleGetInfoRefs:
copy output of
%v: %v"
,
cmd
.
Args
,
err
))
return
return
}
}
if
err
:=
cmd
.
Wait
();
err
!=
nil
{
if
err
:=
cmd
.
Wait
();
err
!=
nil
{
...
@@ -125,7 +125,7 @@ func handlePostRPC(w http.ResponseWriter, r *http.Request, a *api.Response) {
...
@@ -125,7 +125,7 @@ func handlePostRPC(w http.ResponseWriter, r *http.Request, a *api.Response) {
helper
.
Fail500
(
w
,
fmt
.
Errorf
(
"handlePostRPC: start %v: %v"
,
cmd
.
Args
,
err
))
helper
.
Fail500
(
w
,
fmt
.
Errorf
(
"handlePostRPC: start %v: %v"
,
cmd
.
Args
,
err
))
return
return
}
}
defer
c
leanUpProcessGroup
(
cmd
)
// Ensure brute force subprocess clean-up
defer
helper
.
C
leanUpProcessGroup
(
cmd
)
// Ensure brute force subprocess clean-up
// Write the client request body to Git's standard input
// Write the client request body to Git's standard input
if
_
,
err
:=
io
.
Copy
(
stdin
,
r
.
Body
);
err
!=
nil
{
if
_
,
err
:=
io
.
Copy
(
stdin
,
r
.
Body
);
err
!=
nil
{
...
@@ -146,7 +146,7 @@ func handlePostRPC(w http.ResponseWriter, r *http.Request, a *api.Response) {
...
@@ -146,7 +146,7 @@ func handlePostRPC(w http.ResponseWriter, r *http.Request, a *api.Response) {
// This io.Copy may take a long time, both for Git push and pull.
// This io.Copy may take a long time, both for Git push and pull.
if
_
,
err
:=
io
.
Copy
(
w
,
stdout
);
err
!=
nil
{
if
_
,
err
:=
io
.
Copy
(
w
,
stdout
);
err
!=
nil
{
helper
.
LogError
(
fmt
.
Errorf
(
"handlePostRPC
read from
%v: %v"
,
cmd
.
Args
,
err
))
helper
.
LogError
(
fmt
.
Errorf
(
"handlePostRPC
copy output of
%v: %v"
,
cmd
.
Args
,
err
))
return
return
}
}
if
err
:=
cmd
.
Wait
();
err
!=
nil
{
if
err
:=
cmd
.
Wait
();
err
!=
nil
{
...
...
internal/helper/helpers.go
View file @
51d1256c
...
@@ -6,6 +6,8 @@ import (
...
@@ -6,6 +6,8 @@ import (
"net/http"
"net/http"
"net/url"
"net/url"
"os"
"os"
"os/exec"
"syscall"
)
)
func
Fail500
(
w
http
.
ResponseWriter
,
err
error
)
{
func
Fail500
(
w
http
.
ResponseWriter
,
err
error
)
{
...
@@ -79,3 +81,32 @@ func HeaderClone(h http.Header) http.Header {
...
@@ -79,3 +81,32 @@ func HeaderClone(h http.Header) http.Header {
}
}
return
h2
return
h2
}
}
func
CleanUpProcessGroup
(
cmd
*
exec
.
Cmd
)
{
if
cmd
==
nil
{
return
}
process
:=
cmd
.
Process
if
process
!=
nil
&&
process
.
Pid
>
0
{
// Send SIGTERM to the process group of cmd
syscall
.
Kill
(
-
process
.
Pid
,
syscall
.
SIGTERM
)
}
// reap our child process
cmd
.
Wait
()
}
func
ExitStatus
(
err
error
)
(
int
,
bool
)
{
exitError
,
ok
:=
err
.
(
*
exec
.
ExitError
)
if
!
ok
{
return
0
,
false
}
waitStatus
,
ok
:=
exitError
.
Sys
()
.
(
syscall
.
WaitStatus
)
if
!
ok
{
return
0
,
false
}
return
waitStatus
.
ExitStatus
(),
true
}
internal/lfs/lfs.go
View file @
51d1256c
...
@@ -61,7 +61,7 @@ func handleStoreLfsObject(h http.Handler) api.HandleFunc {
...
@@ -61,7 +61,7 @@ func handleStoreLfsObject(h http.Handler) api.HandleFunc {
written
,
err
:=
io
.
Copy
(
hw
,
r
.
Body
)
written
,
err
:=
io
.
Copy
(
hw
,
r
.
Body
)
if
err
!=
nil
{
if
err
!=
nil
{
helper
.
Fail500
(
w
,
fmt
.
Errorf
(
"handleStoreLfsObject:
write
tempfile: %v"
,
err
))
helper
.
Fail500
(
w
,
fmt
.
Errorf
(
"handleStoreLfsObject:
copy body to
tempfile: %v"
,
err
))
return
return
}
}
file
.
Close
()
file
.
Close
()
...
...
internal/upload/artifacts.go
deleted
100644 → 0
View file @
56ac73c7
package
upload
import
(
"../api"
"net/http"
)
func
Artifacts
(
myAPI
*
api
.
API
,
h
http
.
Handler
)
http
.
Handler
{
return
myAPI
.
PreAuthorizeHandler
(
func
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
,
a
*
api
.
Response
)
{
r
.
Header
.
Set
(
tempPathHeader
,
a
.
TempPath
)
handleFileUploads
(
h
)
.
ServeHTTP
(
w
,
r
)
},
"/authorize"
)
}
internal/upload/uploads.go
View file @
51d1256c
...
@@ -3,7 +3,6 @@ package upload
...
@@ -3,7 +3,6 @@ package upload
import
(
import
(
"../helper"
"../helper"
"bytes"
"bytes"
"errors"
"fmt"
"fmt"
"io"
"io"
"io/ioutil"
"io/ioutil"
...
@@ -12,13 +11,20 @@ import (
...
@@ -12,13 +11,20 @@ import (
"os"
"os"
)
)
const
tempPathHeader
=
"Gitlab-Workhorse-Temp-Path"
type
MultipartFormProcessor
interface
{
ProcessFile
(
formName
,
fileName
string
,
writer
*
multipart
.
Writer
)
error
ProcessField
(
formName
string
,
writer
*
multipart
.
Writer
)
error
}
func
rewriteFormFilesFromMultipart
(
r
*
http
.
Request
,
writer
*
multipart
.
Writer
,
tempPath
string
)
(
cleanup
func
(),
err
error
)
{
func
rewriteFormFilesFromMultipart
(
r
*
http
.
Request
,
writer
*
multipart
.
Writer
,
tempPath
string
,
filter
MultipartFormProcessor
)
(
cleanup
func
(),
err
error
)
{
// Create multipart reader
// Create multipart reader
reader
,
err
:=
r
.
MultipartReader
()
reader
,
err
:=
r
.
MultipartReader
()
if
err
!=
nil
{
if
err
!=
nil
{
return
nil
,
err
if
err
==
http
.
ErrNotMultipart
{
// We want to be able to recognize http.ErrNotMultipart elsewhere so no fmt.Errorf
return
nil
,
http
.
ErrNotMultipart
}
return
nil
,
fmt
.
Errorf
(
"get multipart reader: %v"
,
err
)
}
}
var
files
[]
string
var
files
[]
string
...
@@ -51,13 +57,13 @@ func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, te
...
@@ -51,13 +57,13 @@ func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, te
if
filename
:=
p
.
FileName
();
filename
!=
""
{
if
filename
:=
p
.
FileName
();
filename
!=
""
{
// Create temporary directory where the uploaded file will be stored
// Create temporary directory where the uploaded file will be stored
if
err
:=
os
.
MkdirAll
(
tempPath
,
0700
);
err
!=
nil
{
if
err
:=
os
.
MkdirAll
(
tempPath
,
0700
);
err
!=
nil
{
return
cleanup
,
err
return
cleanup
,
fmt
.
Errorf
(
"mkdir for tempfile: %v"
,
err
)
}
}
// Create temporary file in path returned by Authorization filter
// Create temporary file in path returned by Authorization filter
file
,
err
:=
ioutil
.
TempFile
(
tempPath
,
"upload_"
)
file
,
err
:=
ioutil
.
TempFile
(
tempPath
,
"upload_"
)
if
err
!=
nil
{
if
err
!=
nil
{
return
cleanup
,
err
return
cleanup
,
fmt
.
Errorf
(
"create tempfile: %v"
,
err
)
}
}
defer
file
.
Close
()
defer
file
.
Close
()
...
@@ -67,62 +73,67 @@ func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, te
...
@@ -67,62 +73,67 @@ func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, te
files
=
append
(
files
,
file
.
Name
())
files
=
append
(
files
,
file
.
Name
())
_
,
err
=
io
.
Copy
(
file
,
p
)
_
,
err
=
io
.
Copy
(
file
,
p
)
file
.
Close
()
if
err
!=
nil
{
if
err
!=
nil
{
return
cleanup
,
fmt
.
Errorf
(
"copy from multipart to tempfile: %v"
,
err
)
}
file
.
Close
()
if
err
:=
filter
.
ProcessFile
(
name
,
file
.
Name
(),
writer
);
err
!=
nil
{
return
cleanup
,
err
return
cleanup
,
err
}
}
}
else
{
}
else
{
np
,
err
:=
writer
.
CreatePart
(
p
.
Header
)
np
,
err
:=
writer
.
CreatePart
(
p
.
Header
)
if
err
!=
nil
{
if
err
!=
nil
{
return
cleanup
,
err
return
cleanup
,
fmt
.
Errorf
(
"create multipart field: %v"
,
err
)
}
}
_
,
err
=
io
.
Copy
(
np
,
p
)
_
,
err
=
io
.
Copy
(
np
,
p
)
if
err
!=
nil
{
if
err
!=
nil
{
return
cleanup
,
err
return
cleanup
,
fmt
.
Errorf
(
"duplicate multipart field: %v"
,
err
)
}
if
err
:=
filter
.
ProcessField
(
name
,
writer
);
err
!=
nil
{
return
cleanup
,
fmt
.
Errorf
(
"process multipart field: %v"
,
err
)
}
}
}
}
}
}
return
cleanup
,
nil
return
cleanup
,
nil
}
}
func
handleFileUploads
(
h
http
.
Handler
)
http
.
Handler
{
func
HandleFileUploads
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
,
h
http
.
Handler
,
tempPath
string
,
filter
MultipartFormProcessor
)
{
return
http
.
HandlerFunc
(
func
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
)
{
if
tempPath
==
""
{
tempPath
:=
r
.
Header
.
Get
(
tempPathHeader
)
helper
.
Fail500
(
w
,
fmt
.
Errorf
(
"handleFileUploads: tempPath empty"
))
if
tempPath
==
""
{
return
helper
.
Fail500
(
w
,
errors
.
New
(
"handleFileUploads: TempPath empty"
))
}
return
}
r
.
Header
.
Del
(
tempPathHeader
)
var
body
bytes
.
Buffer
var
body
bytes
.
Buffer
writer
:=
multipart
.
NewWriter
(
&
body
)
writer
:=
multipart
.
NewWriter
(
&
body
)
defer
writer
.
Close
()
defer
writer
.
Close
()
// Rewrite multipart form data
// Rewrite multipart form data
cleanup
,
err
:=
rewriteFormFilesFromMultipart
(
r
,
writer
,
tempPath
)
cleanup
,
err
:=
rewriteFormFilesFromMultipart
(
r
,
writer
,
tempPath
,
filter
)
if
err
!=
nil
{
if
err
!=
nil
{
if
err
==
http
.
ErrNotMultipart
{
if
err
==
http
.
ErrNotMultipart
{
h
.
ServeHTTP
(
w
,
r
)
h
.
ServeHTTP
(
w
,
r
)
}
else
{
}
else
{
helper
.
Fail500
(
w
,
fmt
.
Errorf
(
"handleFileUploads: extract files from multipart: %v"
,
err
))
helper
.
Fail500
(
w
,
fmt
.
Errorf
(
"handleFileUploads: extract files from multipart: %v"
,
err
))
}
return
}
}
return
}
if
cleanup
!=
nil
{
if
cleanup
!=
nil
{
defer
cleanup
()
defer
cleanup
()
}
}
// Close writer
// Close writer
writer
.
Close
()
writer
.
Close
()
// Hijack the request
// Hijack the request
r
.
Body
=
ioutil
.
NopCloser
(
&
body
)
r
.
Body
=
ioutil
.
NopCloser
(
&
body
)
r
.
ContentLength
=
int64
(
body
.
Len
())
r
.
ContentLength
=
int64
(
body
.
Len
())
r
.
Header
.
Set
(
"Content-Type"
,
writer
.
FormDataContentType
())
r
.
Header
.
Set
(
"Content-Type"
,
writer
.
FormDataContentType
())
// Proxy the request
// Proxy the request
h
.
ServeHTTP
(
w
,
r
)
h
.
ServeHTTP
(
w
,
r
)
})
}
}
internal/upload/uploads_test.go
View file @
51d1256c
...
@@ -5,6 +5,7 @@ import (
...
@@ -5,6 +5,7 @@ import (
"../proxy"
"../proxy"
"../testhelper"
"../testhelper"
"bytes"
"bytes"
"errors"
"fmt"
"fmt"
"io"
"io"
"io/ioutil"
"io/ioutil"
...
@@ -19,10 +20,27 @@ import (
...
@@ -19,10 +20,27 @@ import (
var
nilHandler
=
http
.
HandlerFunc
(
func
(
http
.
ResponseWriter
,
*
http
.
Request
)
{})
var
nilHandler
=
http
.
HandlerFunc
(
func
(
http
.
ResponseWriter
,
*
http
.
Request
)
{})
type
testFormProcessor
struct
{
}
func
(
a
*
testFormProcessor
)
ProcessFile
(
formName
,
fileName
string
,
writer
*
multipart
.
Writer
)
error
{
if
formName
!=
"file"
&&
fileName
!=
"my.file"
{
return
errors
.
New
(
"illegal file"
)
}
return
nil
}
func
(
a
*
testFormProcessor
)
ProcessField
(
formName
string
,
writer
*
multipart
.
Writer
)
error
{
if
formName
!=
"token"
{
return
errors
.
New
(
"illegal field"
)
}
return
nil
}
func
TestUploadTempPathRequirement
(
t
*
testing
.
T
)
{
func
TestUploadTempPathRequirement
(
t
*
testing
.
T
)
{
response
:=
httptest
.
NewRecorder
()
response
:=
httptest
.
NewRecorder
()
request
:=
&
http
.
Request
{}
request
:=
&
http
.
Request
{}
handleFileUploads
(
nilHandler
)
.
ServeHTTP
(
response
,
request
)
HandleFileUploads
(
response
,
request
,
nilHandler
,
""
,
nil
)
testhelper
.
AssertResponseCode
(
t
,
response
,
500
)
testhelper
.
AssertResponseCode
(
t
,
response
,
500
)
}
}
...
@@ -56,9 +74,8 @@ func TestUploadHandlerForwardingRawData(t *testing.T) {
...
@@ -56,9 +74,8 @@ func TestUploadHandlerForwardingRawData(t *testing.T) {
response
:=
httptest
.
NewRecorder
()
response
:=
httptest
.
NewRecorder
()
httpRequest
.
Header
.
Set
(
tempPathHeader
,
tempPath
)
handler
:=
proxy
.
NewProxy
(
helper
.
URLMustParse
(
ts
.
URL
),
"123"
,
nil
)
HandleFileUploads
(
response
,
httpRequest
,
handler
,
tempPath
,
nil
)
handleFileUploads
(
proxy
.
NewProxy
(
helper
.
URLMustParse
(
ts
.
URL
),
"123"
,
nil
))
.
ServeHTTP
(
response
,
httpRequest
)
testhelper
.
AssertResponseCode
(
t
,
response
,
202
)
testhelper
.
AssertResponseCode
(
t
,
response
,
202
)
if
response
.
Body
.
String
()
!=
"RESPONSE"
{
if
response
.
Body
.
String
()
!=
"RESPONSE"
{
t
.
Fatal
(
"Expected RESPONSE in response body"
)
t
.
Fatal
(
"Expected RESPONSE in response body"
)
...
@@ -129,13 +146,65 @@ func TestUploadHandlerRewritingMultiPartData(t *testing.T) {
...
@@ -129,13 +146,65 @@ func TestUploadHandlerRewritingMultiPartData(t *testing.T) {
httpRequest
.
Body
=
ioutil
.
NopCloser
(
&
buffer
)
httpRequest
.
Body
=
ioutil
.
NopCloser
(
&
buffer
)
httpRequest
.
ContentLength
=
int64
(
buffer
.
Len
())
httpRequest
.
ContentLength
=
int64
(
buffer
.
Len
())
httpRequest
.
Header
.
Set
(
"Content-Type"
,
writer
.
FormDataContentType
())
httpRequest
.
Header
.
Set
(
"Content-Type"
,
writer
.
FormDataContentType
())
httpRequest
.
Header
.
Set
(
tempPathHeader
,
tempPath
)
response
:=
httptest
.
NewRecorder
()
response
:=
httptest
.
NewRecorder
()
handleFileUploads
(
proxy
.
NewProxy
(
helper
.
URLMustParse
(
ts
.
URL
),
"123"
,
nil
))
.
ServeHTTP
(
response
,
httpRequest
)
handler
:=
proxy
.
NewProxy
(
helper
.
URLMustParse
(
ts
.
URL
),
"123"
,
nil
)
HandleFileUploads
(
response
,
httpRequest
,
handler
,
tempPath
,
&
testFormProcessor
{})
testhelper
.
AssertResponseCode
(
t
,
response
,
202
)
testhelper
.
AssertResponseCode
(
t
,
response
,
202
)
if
_
,
err
:=
os
.
Stat
(
filePath
);
!
os
.
IsNotExist
(
err
)
{
if
_
,
err
:=
os
.
Stat
(
filePath
);
!
os
.
IsNotExist
(
err
)
{
t
.
Fatal
(
"expected the file to be deleted"
)
t
.
Fatal
(
"expected the file to be deleted"
)
}
}
}
}
func
TestUploadProcessingField
(
t
*
testing
.
T
)
{
tempPath
,
err
:=
ioutil
.
TempDir
(
""
,
"uploads"
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
defer
os
.
RemoveAll
(
tempPath
)
var
buffer
bytes
.
Buffer
writer
:=
multipart
.
NewWriter
(
&
buffer
)
writer
.
WriteField
(
"token2"
,
"test"
)
writer
.
Close
()
httpRequest
,
err
:=
http
.
NewRequest
(
"PUT"
,
"/url/path"
,
&
buffer
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
httpRequest
.
Header
.
Set
(
"Content-Type"
,
writer
.
FormDataContentType
())
response
:=
httptest
.
NewRecorder
()
HandleFileUploads
(
response
,
httpRequest
,
nilHandler
,
tempPath
,
&
testFormProcessor
{})
testhelper
.
AssertResponseCode
(
t
,
response
,
500
)
}
func
TestUploadProcessingFile
(
t
*
testing
.
T
)
{
tempPath
,
err
:=
ioutil
.
TempDir
(
""
,
"uploads"
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
defer
os
.
RemoveAll
(
tempPath
)
var
buffer
bytes
.
Buffer
writer
:=
multipart
.
NewWriter
(
&
buffer
)
file
,
err
:=
writer
.
CreateFormFile
(
"file2"
,
"my.file"
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
fmt
.
Fprint
(
file
,
"test"
)
writer
.
Close
()
httpRequest
,
err
:=
http
.
NewRequest
(
"PUT"
,
"/url/path"
,
&
buffer
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
httpRequest
.
Header
.
Set
(
"Content-Type"
,
writer
.
FormDataContentType
())
response
:=
httptest
.
NewRecorder
()
HandleFileUploads
(
response
,
httpRequest
,
nilHandler
,
tempPath
,
&
testFormProcessor
{})
testhelper
.
AssertResponseCode
(
t
,
response
,
500
)
}
internal/upstream/routes.go
View file @
51d1256c
...
@@ -2,11 +2,11 @@ package upstream
...
@@ -2,11 +2,11 @@ package upstream
import
(
import
(
apipkg
"../api"
apipkg
"../api"
"../artifacts"
"../git"
"../git"
"../lfs"
"../lfs"
proxypkg
"../proxy"
proxypkg
"../proxy"
"../staticpages"
"../staticpages"
"../upload"
"net/http"
"net/http"
"regexp"
"regexp"
)
)
...
@@ -64,8 +64,9 @@ func (u *Upstream) configureRoutes() {
...
@@ -64,8 +64,9 @@ func (u *Upstream) configureRoutes() {
route
{
"GET"
,
regexp
.
MustCompile
(
projectsAPIPattern
+
`repository/archive.tar.gz\z`
),
git
.
GetArchive
(
api
)},
route
{
"GET"
,
regexp
.
MustCompile
(
projectsAPIPattern
+
`repository/archive.tar.gz\z`
),
git
.
GetArchive
(
api
)},
route
{
"GET"
,
regexp
.
MustCompile
(
projectsAPIPattern
+
`repository/archive.tar.bz2\z`
),
git
.
GetArchive
(
api
)},
route
{
"GET"
,
regexp
.
MustCompile
(
projectsAPIPattern
+
`repository/archive.tar.bz2\z`
),
git
.
GetArchive
(
api
)},
// CI Artifacts API
// CI Artifacts
route
{
"POST"
,
regexp
.
MustCompile
(
ciAPIPattern
+
`v1/builds/[0-9]+/artifacts\z`
),
contentEncodingHandler
(
upload
.
Artifacts
(
api
,
proxy
))},
route
{
"GET"
,
regexp
.
MustCompile
(
projectPattern
+
`builds/[0-9]+/artifacts/file/`
),
contentEncodingHandler
(
artifacts
.
DownloadArtifact
(
api
))},
route
{
"POST"
,
regexp
.
MustCompile
(
ciAPIPattern
+
`v1/builds/[0-9]+/artifacts\z`
),
contentEncodingHandler
(
artifacts
.
UploadArtifacts
(
api
,
proxy
))},
// Explicitly proxy API requests
// Explicitly proxy API requests
route
{
""
,
regexp
.
MustCompile
(
apiPattern
),
proxy
},
route
{
""
,
regexp
.
MustCompile
(
apiPattern
),
proxy
},
...
...
internal/upstream/upstream.go
View file @
51d1256c
...
@@ -41,8 +41,8 @@ func NewUpstream(backend *url.URL, socket string, version string, documentRoot s
...
@@ -41,8 +41,8 @@ func NewUpstream(backend *url.URL, socket string, version string, documentRoot s
if
backend
==
nil
{
if
backend
==
nil
{
up
.
Backend
=
DefaultBackend
up
.
Backend
=
DefaultBackend
}
}
up
.
configureRoutes
()
up
.
configureURLPrefix
()
up
.
configureURLPrefix
()
up
.
configureRoutes
()
return
&
up
return
&
up
}
}
...
...
internal/zipartifacts/codes.go
0 → 100644
View file @
51d1256c
package
zipartifacts
// These are exit codes used by subprocesses in cmd/gitlab-zip-xxx
const
(
StatusNotZip
=
10
+
iota
StatusEntryNotFound
)
internal/zipartifacts/entry.go
0 → 100644
View file @
51d1256c
package
zipartifacts
import
(
"encoding/base64"
)
func
DecodeFileEntry
(
entry
string
)
(
string
,
error
)
{
decoded
,
err
:=
base64
.
StdEncoding
.
DecodeString
(
entry
)
if
err
!=
nil
{
return
""
,
err
}
return
string
(
decoded
),
nil
}
internal/zipartifacts/metadata.go
0 → 100644
View file @
51d1256c
package
zipartifacts
import
(
"archive/zip"
"compress/gzip"
"encoding/binary"
"encoding/json"
"io"
"os"
"strconv"
)
type
metadata
struct
{
Modified
int64
`json:"modified"`
Mode
string
`json:"mode"`
CRC
uint32
`json:"crc,omitempty"`
Size
uint64
`json:"size,omitempty"`
Zipped
uint64
`json:"zipped,omitempty"`
Comment
string
`json:"comment,omitempty"`
}
const
MetadataHeaderPrefix
=
"
\x00\x00\x00
&"
// length of string below, encoded properly
const
MetadataHeader
=
"GitLab Build Artifacts Metadata 0.0.2
\n
"
func
newMetadata
(
file
*
zip
.
File
)
metadata
{
return
metadata
{
Modified
:
file
.
ModTime
()
.
Unix
(),
Mode
:
strconv
.
FormatUint
(
uint64
(
file
.
Mode
()
.
Perm
()),
8
),
CRC
:
file
.
CRC32
,
Size
:
file
.
UncompressedSize64
,
Zipped
:
file
.
CompressedSize64
,
Comment
:
file
.
Comment
,
}
}
func
(
m
metadata
)
writeEncoded
(
output
io
.
Writer
)
error
{
j
,
err
:=
json
.
Marshal
(
m
)
if
err
!=
nil
{
return
err
}
j
=
append
(
j
,
byte
(
'\n'
))
return
writeBytes
(
output
,
j
)
}
func
writeZipEntryMetadata
(
output
io
.
Writer
,
entry
*
zip
.
File
)
error
{
err
:=
writeString
(
output
,
entry
.
Name
)
if
err
!=
nil
{
return
err
}
err
=
newMetadata
(
entry
)
.
writeEncoded
(
output
)
if
err
!=
nil
{
return
err
}
return
nil
}
func
generateZipMetadata
(
output
io
.
Writer
,
archive
*
zip
.
Reader
)
error
{
err
:=
writeString
(
output
,
MetadataHeader
)
if
err
!=
nil
{
return
err
}
// Write empty error string
err
=
writeString
(
output
,
"{}"
)
if
err
!=
nil
{
return
err
}
// Write all files
for
_
,
entry
:=
range
archive
.
File
{
err
=
writeZipEntryMetadata
(
output
,
entry
)
if
err
!=
nil
{
return
err
}
}
return
nil
}
func
GenerateZipMetadataFromFile
(
fileName
string
,
w
io
.
Writer
)
error
{
archive
,
err
:=
zip
.
OpenReader
(
fileName
)
if
err
!=
nil
{
// Ignore non-zip archives
return
os
.
ErrInvalid
}
defer
archive
.
Close
()
gz
:=
gzip
.
NewWriter
(
w
)
defer
gz
.
Close
()
return
generateZipMetadata
(
gz
,
&
archive
.
Reader
)
}
func
writeBytes
(
output
io
.
Writer
,
data
[]
byte
)
error
{
err
:=
binary
.
Write
(
output
,
binary
.
BigEndian
,
uint32
(
len
(
data
)))
if
err
==
nil
{
_
,
err
=
output
.
Write
(
data
)
}
return
err
}
func
writeString
(
output
io
.
Writer
,
str
string
)
error
{
return
writeBytes
(
output
,
[]
byte
(
str
))
}
main_test.go
View file @
51d1256c
...
@@ -6,6 +6,7 @@ import (
...
@@ -6,6 +6,7 @@ import (
"./internal/testhelper"
"./internal/testhelper"
"./internal/upstream"
"./internal/upstream"
"bytes"
"bytes"
"encoding/base64"
"encoding/json"
"encoding/json"
"fmt"
"fmt"
"io"
"io"
...
@@ -363,6 +364,37 @@ func TestAllowedStaticFile(t *testing.T) {
...
@@ -363,6 +364,37 @@ func TestAllowedStaticFile(t *testing.T) {
}
}
}
}
func
TestStaticFileRelativeURL
(
t
*
testing
.
T
)
{
content
:=
"PUBLIC"
if
err
:=
setupStaticFile
(
"static.txt"
,
content
);
err
!=
nil
{
t
.
Fatalf
(
"create public/static.txt: %v"
,
err
)
}
ts
:=
testhelper
.
TestServerWithHandler
(
regexp
.
MustCompile
(
`.`
),
http
.
HandlerFunc
(
http
.
NotFound
))
defer
ts
.
Close
()
backendURLString
:=
ts
.
URL
+
"/my-relative-url"
log
.
Print
(
backendURLString
)
ws
:=
startWorkhorseServer
(
backendURLString
)
defer
ws
.
Close
()
resource
:=
"/my-relative-url/static.txt"
resp
,
err
:=
http
.
Get
(
ws
.
URL
+
resource
)
if
err
!=
nil
{
t
.
Error
(
err
)
}
defer
resp
.
Body
.
Close
()
buf
:=
&
bytes
.
Buffer
{}
if
_
,
err
:=
io
.
Copy
(
buf
,
resp
.
Body
);
err
!=
nil
{
t
.
Error
(
err
)
}
if
buf
.
String
()
!=
content
{
t
.
Errorf
(
"GET %q: Expected %q, got %q"
,
resource
,
content
,
buf
.
String
())
}
if
resp
.
StatusCode
!=
200
{
t
.
Errorf
(
"GET %q: expected 200, got %d"
,
resource
,
resp
.
StatusCode
)
}
}
func
TestAllowedPublicUploadsFile
(
t
*
testing
.
T
)
{
func
TestAllowedPublicUploadsFile
(
t
*
testing
.
T
)
{
content
:=
"PRIVATE but allowed"
content
:=
"PRIVATE but allowed"
if
err
:=
setupStaticFile
(
"uploads/static file.txt"
,
content
);
err
!=
nil
{
if
err
:=
setupStaticFile
(
"uploads/static file.txt"
,
content
);
err
!=
nil
{
...
@@ -465,8 +497,9 @@ func TestArtifactsUpload(t *testing.T) {
...
@@ -465,8 +497,9 @@ func TestArtifactsUpload(t *testing.T) {
if
err
!=
nil
{
if
err
!=
nil
{
t
.
Fatal
(
err
)
t
.
Fatal
(
err
)
}
}
if
len
(
r
.
MultipartForm
.
Value
)
!=
2
{
// 1 file name, 1 file path
nValues
:=
2
// filename + path for just the upload (no metadata because we are not POSTing a valid zip file)
t
.
Error
(
"Expected to receive exactly 2 values"
)
if
len
(
r
.
MultipartForm
.
Value
)
!=
nValues
{
t
.
Errorf
(
"Expected to receive exactly %d values"
,
nValues
)
}
}
if
len
(
r
.
MultipartForm
.
File
)
!=
0
{
if
len
(
r
.
MultipartForm
.
File
)
!=
0
{
t
.
Error
(
"Expected to not receive any files"
)
t
.
Error
(
"Expected to not receive any files"
)
...
@@ -488,6 +521,40 @@ func TestArtifactsUpload(t *testing.T) {
...
@@ -488,6 +521,40 @@ func TestArtifactsUpload(t *testing.T) {
}
}
}
}
func
TestArtifactsGetSingleFile
(
t
*
testing
.
T
)
{
// We manually created this zip file in the gitlab-workhorse Git repository
archivePath
:=
`testdata/artifacts-archive.zip`
fileName
:=
"myfile"
fileContents
:=
"MY FILE"
resourcePath
:=
`/namespace/project/builds/123/artifacts/file/`
+
fileName
ts
:=
testhelper
.
TestServerWithHandler
(
regexp
.
MustCompile
(
`\A`
+
resourcePath
+
`\z`
),
func
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
)
{
encodedFilename
:=
base64
.
StdEncoding
.
EncodeToString
([]
byte
(
fileName
))
if
_
,
err
:=
fmt
.
Fprintf
(
w
,
`{"Archive":"%s","Entry":"%s"}`
,
archivePath
,
encodedFilename
);
err
!=
nil
{
t
.
Fatal
(
err
)
}
return
})
defer
ts
.
Close
()
ws
:=
startWorkhorseServer
(
ts
.
URL
)
defer
ws
.
Close
()
resp
,
err
:=
http
.
Get
(
ws
.
URL
+
resourcePath
)
if
err
!=
nil
{
t
.
Error
(
err
)
}
defer
resp
.
Body
.
Close
()
if
resp
.
StatusCode
!=
200
{
t
.
Errorf
(
"GET %q: expected 200, got %d"
,
resourcePath
,
resp
.
StatusCode
)
}
body
,
err
:=
ioutil
.
ReadAll
(
resp
.
Body
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
if
string
(
body
)
!=
fileContents
{
t
.
Fatalf
(
"Expected file contents %q, got %q"
,
fileContents
,
body
)
}
}
func
setupStaticFile
(
fpath
,
content
string
)
error
{
func
setupStaticFile
(
fpath
,
content
string
)
error
{
cwd
,
err
:=
os
.
Getwd
()
cwd
,
err
:=
os
.
Getwd
()
if
err
!=
nil
{
if
err
!=
nil
{
...
...
support/path
0 → 100755
View file @
51d1256c
#!/bin/sh
exec env
PATH
=
$(
pwd
)
:
${
PATH
}
"
$@
"
testdata/artifacts-archive.zip
0 → 100644
View file @
51d1256c
File added
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment