Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
G
gitlab-workhorse
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
1
Merge Requests
1
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
nexedi
gitlab-workhorse
Commits
e0a2ea65
Commit
e0a2ea65
authored
Jan 18, 2016
by
Kamil Trzcinski
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Refactor and implement tests for artifacts upload (injecting the metadata) and single file download
parent
49b0cdef
Changes
8
Show whitespace changes
Inline
Side-by-side
Showing
8 changed files
with
571 additions
and
70 deletions
+571
-70
internal/artifacts/artifact_download.go
internal/artifacts/artifact_download.go
+93
-0
internal/artifacts/artifact_download_test.go
internal/artifacts/artifact_download_test.go
+109
-0
internal/artifacts/artifacts_upload.go
internal/artifacts/artifacts_upload.go
+73
-0
internal/artifacts/artifacts_upload_test.go
internal/artifacts/artifacts_upload_test.go
+172
-0
internal/artifacts/metadata.go
internal/artifacts/metadata.go
+37
-8
internal/artifacts/upload_filter.go
internal/artifacts/upload_filter.go
+0
-47
internal/upload/uploads.go
internal/upload/uploads.go
+12
-9
internal/upload/uploads_test.go
internal/upload/uploads_test.go
+75
-6
No files found.
internal/artifacts/artifact
s
.go
→
internal/artifacts/artifact
_download
.go
View file @
e0a2ea65
...
@@ -3,7 +3,6 @@ package artifacts
...
@@ -3,7 +3,6 @@ package artifacts
import
(
import
(
"../api"
"../api"
"../helper"
"../helper"
"../upload"
"archive/zip"
"archive/zip"
"encoding/base64"
"encoding/base64"
"errors"
"errors"
...
@@ -15,15 +14,58 @@ import (
...
@@ -15,15 +14,58 @@ import (
"strconv"
"strconv"
)
)
func
UploadArtifacts
(
myAPI
*
api
.
API
,
h
http
.
Handler
)
http
.
Handler
{
func
decodeFileEntry
(
entry
string
)
(
string
,
error
)
{
return
myAPI
.
PreAuthorizeHandler
(
func
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
,
a
*
api
.
Response
)
{
decoded
,
err
:=
base64
.
StdEncoding
.
DecodeString
(
entry
)
if
a
.
TempPath
==
""
{
if
err
!=
nil
{
helper
.
Fail500
(
w
,
errors
.
New
(
"UploadArtifacts: TempPath is empty"
))
return
""
,
err
return
}
return
string
(
decoded
),
nil
}
func
detectFileContentType
(
fileName
string
)
string
{
contentType
:=
mime
.
TypeByExtension
(
filepath
.
Ext
(
fileName
))
if
contentType
==
""
{
contentType
=
"application/octet-stream"
}
return
contentType
}
func
findFileInZip
(
fileName
string
,
archive
*
zip
.
Reader
)
*
zip
.
File
{
for
_
,
file
:=
range
archive
.
File
{
if
file
.
Name
==
fileName
{
return
file
}
}
return
nil
}
func
unpackFileFromZip
(
archiveFileName
,
fileName
string
,
headers
http
.
Header
,
output
io
.
Writer
)
error
{
archive
,
err
:=
zip
.
OpenReader
(
archiveFileName
)
if
err
!=
nil
{
return
err
}
defer
archive
.
Close
()
file
:=
findFileInZip
(
fileName
,
&
archive
.
Reader
)
if
file
==
nil
{
return
os
.
ErrNotExist
}
// Start decompressing the file
reader
,
err
:=
file
.
Open
()
if
err
!=
nil
{
return
err
}
}
defer
reader
.
Close
()
// Write http headers about the file
headers
.
Set
(
"Content-Length"
,
strconv
.
FormatInt
(
int64
(
file
.
UncompressedSize64
),
10
))
headers
.
Set
(
"Content-Type"
,
detectFileContentType
(
file
.
Name
))
headers
.
Set
(
"Content-Disposition"
,
"attachment; filename=
\"
"
+
filepath
.
Base
(
file
.
Name
)
+
"
\"
"
)
upload
.
HandleFileUploads
(
w
,
r
,
h
,
a
.
TempPath
,
&
artifactsFormFilter
{})
// Copy file body to client
},
"/authorize"
)
_
,
err
=
io
.
Copy
(
output
,
reader
)
return
err
}
}
// Artifacts downloader doesn't support ranges when downloading a single file
// Artifacts downloader doesn't support ranges when downloading a single file
...
@@ -34,51 +76,18 @@ func DownloadArtifact(myAPI *api.API) http.Handler {
...
@@ -34,51 +76,18 @@ func DownloadArtifact(myAPI *api.API) http.Handler {
return
return
}
}
fileName
Decoded
,
err
:=
base64
.
StdEncoding
.
DecodeString
(
a
.
Entry
)
fileName
,
err
:=
decodeFileEntry
(
a
.
Entry
)
if
err
!=
nil
{
if
err
!=
nil
{
helper
.
Fail500
(
w
,
err
)
helper
.
Fail500
(
w
,
err
)
return
return
}
}
fileName
:=
string
(
fileNameDecoded
)
// TODO:
err
=
unpackFileFromZip
(
a
.
Archive
,
fileName
,
w
.
Header
(),
w
)
// This should be moved to sub process to reduce memory pressue on workhorse
archive
,
err
:=
zip
.
OpenReader
(
a
.
Archive
)
if
os
.
IsNotExist
(
err
)
{
if
os
.
IsNotExist
(
err
)
{
http
.
NotFound
(
w
,
r
)
http
.
NotFound
(
w
,
r
)
return
return
}
else
if
err
!=
nil
{
}
else
if
err
!=
nil
{
helper
.
Fail500
(
w
,
err
)
helper
.
Fail500
(
w
,
err
)
}
}
defer
archive
.
Close
()
var
file
*
zip
.
File
for
_
,
file
=
range
archive
.
File
{
if
file
.
Name
==
fileName
{
break
}
}
if
file
==
nil
{
http
.
NotFound
(
w
,
r
)
return
}
contentType
:=
mime
.
TypeByExtension
(
filepath
.
Ext
(
file
.
Name
))
if
contentType
==
""
{
contentType
=
"application/octet-stream"
}
w
.
Header
()
.
Set
(
"Content-Length"
,
strconv
.
FormatInt
(
int64
(
file
.
UncompressedSize64
),
10
))
w
.
Header
()
.
Set
(
"Content-Type"
,
contentType
)
w
.
Header
()
.
Set
(
"Content-Disposition"
,
"attachment; filename=
\"
"
+
filepath
.
Base
(
file
.
Name
)
+
"
\"
"
)
reader
,
err
:=
file
.
Open
()
if
err
!=
nil
{
helper
.
Fail500
(
w
,
err
)
}
defer
reader
.
Close
()
// Copy file body
io
.
Copy
(
w
,
reader
)
},
""
)
},
""
)
}
}
internal/artifacts/artifact_download_test.go
0 → 100644
View file @
e0a2ea65
package
artifacts
import
(
"../api"
"../helper"
"../testhelper"
"archive/zip"
"encoding/base64"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"os"
"testing"
)
func
testArtifactDownloadServer
(
t
*
testing
.
T
,
archive
string
,
entry
string
)
*
httptest
.
Server
{
mux
:=
http
.
NewServeMux
()
mux
.
HandleFunc
(
"/url/path"
,
func
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
)
{
if
r
.
Method
!=
"GET"
{
t
.
Fatal
(
"Expected GET request"
)
}
w
.
Header
()
.
Set
(
"Content-Type"
,
"application/json"
)
data
,
err
:=
json
.
Marshal
(
&
api
.
Response
{
Archive
:
archive
,
Entry
:
base64
.
StdEncoding
.
EncodeToString
([]
byte
(
entry
)),
})
if
err
!=
nil
{
t
.
Fatal
(
"Expected to marshal"
)
}
w
.
Write
(
data
)
})
return
testhelper
.
TestServerWithHandler
(
nil
,
mux
.
ServeHTTP
)
}
func
testDownloadArtifact
(
t
*
testing
.
T
,
ts
*
httptest
.
Server
)
*
httptest
.
ResponseRecorder
{
httpRequest
,
err
:=
http
.
NewRequest
(
"GET"
,
ts
.
URL
+
"/url/path"
,
nil
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
response
:=
httptest
.
NewRecorder
()
apiClient
:=
api
.
NewAPI
(
helper
.
URLMustParse
(
ts
.
URL
),
"123"
,
nil
)
DownloadArtifact
(
apiClient
)
.
ServeHTTP
(
response
,
httpRequest
)
return
response
}
func
TestDownloadingFromValidArchive
(
t
*
testing
.
T
)
{
tempFile
,
err
:=
ioutil
.
TempFile
(
""
,
"uploads"
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
defer
tempFile
.
Close
()
defer
os
.
Remove
(
tempFile
.
Name
())
archive
:=
zip
.
NewWriter
(
tempFile
)
defer
archive
.
Close
()
fileInArchive
,
err
:=
archive
.
Create
(
"test.txt"
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
fmt
.
Fprint
(
fileInArchive
,
"testtest"
)
archive
.
Close
()
ts
:=
testArtifactDownloadServer
(
t
,
tempFile
.
Name
(),
"test.txt"
)
defer
ts
.
Close
()
response
:=
testDownloadArtifact
(
t
,
ts
)
testhelper
.
AssertResponseCode
(
t
,
response
,
200
)
testhelper
.
AssertResponseHeader
(
t
,
response
,
"Content-Type"
,
"text/plain; charset=utf-8"
)
testhelper
.
AssertResponseBody
(
t
,
response
,
"testtest"
)
}
func
TestDownloadingNonExistingFile
(
t
*
testing
.
T
)
{
tempFile
,
err
:=
ioutil
.
TempFile
(
""
,
"uploads"
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
defer
tempFile
.
Close
()
defer
os
.
Remove
(
tempFile
.
Name
())
archive
:=
zip
.
NewWriter
(
tempFile
)
defer
archive
.
Close
()
archive
.
Close
()
ts
:=
testArtifactDownloadServer
(
t
,
tempFile
.
Name
(),
"test"
)
defer
ts
.
Close
()
response
:=
testDownloadArtifact
(
t
,
ts
)
testhelper
.
AssertResponseCode
(
t
,
response
,
404
)
}
func
TestDownloadingFromInvalidArchive
(
t
*
testing
.
T
)
{
ts
:=
testArtifactDownloadServer
(
t
,
"path/to/non/existing/file"
,
"test"
)
defer
ts
.
Close
()
response
:=
testDownloadArtifact
(
t
,
ts
)
testhelper
.
AssertResponseCode
(
t
,
response
,
404
)
}
func
TestIncompleteApiResponse
(
t
*
testing
.
T
)
{
ts
:=
testArtifactDownloadServer
(
t
,
""
,
""
)
defer
ts
.
Close
()
response
:=
testDownloadArtifact
(
t
,
ts
)
testhelper
.
AssertResponseCode
(
t
,
response
,
500
)
}
internal/artifacts/artifacts_upload.go
0 → 100644
View file @
e0a2ea65
package
artifacts
import
(
"../api"
"../helper"
"../upload"
"errors"
"fmt"
"io/ioutil"
"mime/multipart"
"net/http"
"os"
)
// The artifactsFormFilter allows to pass only the `file` as file in body
type
artifactsUploadProcessor
struct
{
TempPath
string
metadataFile
string
}
func
(
a
*
artifactsUploadProcessor
)
ProcessFile
(
formName
,
fileName
string
,
writer
*
multipart
.
Writer
)
error
{
if
formName
!=
"file"
{
return
fmt
.
Errorf
(
"Invalid form field: %q"
,
formName
)
}
if
a
.
metadataFile
!=
""
{
return
fmt
.
Errorf
(
"Multiple files"
)
}
// Create temporary file for metadata and store it's path
tempFile
,
err
:=
ioutil
.
TempFile
(
a
.
TempPath
,
"metadata_"
)
if
err
!=
nil
{
return
err
}
defer
tempFile
.
Close
()
a
.
metadataFile
=
tempFile
.
Name
()
// Generate metadata and save to file
err
=
generateZipMetadataFromFile
(
fileName
,
tempFile
)
if
err
==
os
.
ErrInvalid
{
return
nil
}
else
if
err
!=
nil
{
return
err
}
// Pass metadata file path to Rails
writer
.
WriteField
(
"metadata.path"
,
a
.
metadataFile
)
writer
.
WriteField
(
"metadata.name"
,
"metadata.gz"
)
return
nil
}
func
(
a
*
artifactsUploadProcessor
)
ProcessField
(
formName
string
,
writer
*
multipart
.
Writer
)
error
{
return
nil
}
func
(
a
*
artifactsUploadProcessor
)
Cleanup
()
{
if
a
.
metadataFile
!=
""
{
os
.
Remove
(
a
.
metadataFile
)
}
}
func
UploadArtifacts
(
myAPI
*
api
.
API
,
h
http
.
Handler
)
http
.
Handler
{
return
myAPI
.
PreAuthorizeHandler
(
func
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
,
a
*
api
.
Response
)
{
if
a
.
TempPath
==
""
{
helper
.
Fail500
(
w
,
errors
.
New
(
"UploadArtifacts: TempPath is empty"
))
return
}
mg
:=
&
artifactsUploadProcessor
{
TempPath
:
a
.
TempPath
}
defer
mg
.
Cleanup
()
upload
.
HandleFileUploads
(
w
,
r
,
h
,
a
.
TempPath
,
mg
)
},
"/authorize"
)
}
internal/artifacts/artifacts_upload_test.go
0 → 100644
View file @
e0a2ea65
package
artifacts
import
(
"../api"
"../helper"
"../proxy"
"../testhelper"
"archive/zip"
"bytes"
"compress/gzip"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"mime/multipart"
"net/http"
"net/http/httptest"
"os"
"testing"
)
func
testArtifactsUploadServer
(
t
*
testing
.
T
,
tempPath
string
)
*
httptest
.
Server
{
mux
:=
http
.
NewServeMux
()
mux
.
HandleFunc
(
"/url/path/authorize"
,
func
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
)
{
if
r
.
Method
!=
"POST"
{
t
.
Fatal
(
"Expected POST request"
)
}
w
.
Header
()
.
Set
(
"Content-Type"
,
"application/json"
)
data
,
err
:=
json
.
Marshal
(
&
api
.
Response
{
TempPath
:
tempPath
,
})
if
err
!=
nil
{
t
.
Fatal
(
"Expected to marshal"
)
}
w
.
Write
(
data
)
})
mux
.
HandleFunc
(
"/url/path"
,
func
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
)
{
if
r
.
Method
!=
"POST"
{
t
.
Fatal
(
"Expected POST request"
)
}
if
r
.
FormValue
(
"file.path"
)
==
""
{
w
.
WriteHeader
(
501
)
return
}
if
r
.
FormValue
(
"metadata.path"
)
==
""
{
w
.
WriteHeader
(
502
)
return
}
_
,
err
:=
ioutil
.
ReadFile
(
r
.
FormValue
(
"file.path"
))
if
err
!=
nil
{
w
.
WriteHeader
(
404
)
return
}
metadata
,
err
:=
ioutil
.
ReadFile
(
r
.
FormValue
(
"metadata.path"
))
if
err
!=
nil
{
w
.
WriteHeader
(
404
)
return
}
gz
,
err
:=
gzip
.
NewReader
(
bytes
.
NewReader
(
metadata
))
if
err
!=
nil
{
w
.
WriteHeader
(
405
)
return
}
defer
gz
.
Close
()
metadata
,
err
=
ioutil
.
ReadAll
(
gz
)
if
err
!=
nil
{
w
.
WriteHeader
(
404
)
return
}
if
!
bytes
.
Contains
(
metadata
,
[]
byte
(
metadataHeader
))
{
w
.
WriteHeader
(
400
)
return
}
w
.
WriteHeader
(
200
)
})
return
testhelper
.
TestServerWithHandler
(
nil
,
mux
.
ServeHTTP
)
}
func
testUploadArtifacts
(
contentType
string
,
body
io
.
Reader
,
t
*
testing
.
T
,
ts
*
httptest
.
Server
)
*
httptest
.
ResponseRecorder
{
httpRequest
,
err
:=
http
.
NewRequest
(
"POST"
,
ts
.
URL
+
"/url/path"
,
body
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
httpRequest
.
Header
.
Set
(
"Content-Type"
,
contentType
)
response
:=
httptest
.
NewRecorder
()
apiClient
:=
api
.
NewAPI
(
helper
.
URLMustParse
(
ts
.
URL
),
"123"
,
nil
)
proxyClient
:=
proxy
.
NewProxy
(
helper
.
URLMustParse
(
ts
.
URL
),
"123"
,
nil
)
UploadArtifacts
(
apiClient
,
proxyClient
)
.
ServeHTTP
(
response
,
httpRequest
)
return
response
}
func
TestUploadHandlerAddingMetadata
(
t
*
testing
.
T
)
{
tempPath
,
err
:=
ioutil
.
TempDir
(
""
,
"uploads"
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
defer
os
.
RemoveAll
(
tempPath
)
ts
:=
testArtifactsUploadServer
(
t
,
tempPath
)
defer
ts
.
Close
()
var
buffer
bytes
.
Buffer
writer
:=
multipart
.
NewWriter
(
&
buffer
)
file
,
err
:=
writer
.
CreateFormFile
(
"file"
,
"my.file"
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
archive
:=
zip
.
NewWriter
(
file
)
defer
archive
.
Close
()
fileInArchive
,
err
:=
archive
.
Create
(
"test.file"
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
fmt
.
Fprint
(
fileInArchive
,
"test"
)
archive
.
Close
()
writer
.
Close
()
response
:=
testUploadArtifacts
(
writer
.
FormDataContentType
(),
&
buffer
,
t
,
ts
)
testhelper
.
AssertResponseCode
(
t
,
response
,
200
)
}
func
TestUploadHandlerForUnsupportedArchive
(
t
*
testing
.
T
)
{
tempPath
,
err
:=
ioutil
.
TempDir
(
""
,
"uploads"
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
defer
os
.
RemoveAll
(
tempPath
)
ts
:=
testArtifactsUploadServer
(
t
,
tempPath
)
defer
ts
.
Close
()
var
buffer
bytes
.
Buffer
writer
:=
multipart
.
NewWriter
(
&
buffer
)
file
,
err
:=
writer
.
CreateFormFile
(
"file"
,
"my.file"
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
fmt
.
Fprint
(
file
,
"test"
)
writer
.
Close
()
response
:=
testUploadArtifacts
(
writer
.
FormDataContentType
(),
&
buffer
,
t
,
ts
)
testhelper
.
AssertResponseCode
(
t
,
response
,
502
)
}
func
TestUploadFormProcessing
(
t
*
testing
.
T
)
{
tempPath
,
err
:=
ioutil
.
TempDir
(
""
,
"uploads"
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
defer
os
.
RemoveAll
(
tempPath
)
ts
:=
testArtifactsUploadServer
(
t
,
tempPath
)
defer
ts
.
Close
()
var
buffer
bytes
.
Buffer
writer
:=
multipart
.
NewWriter
(
&
buffer
)
file
,
err
:=
writer
.
CreateFormFile
(
"metadata"
,
"my.file"
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
fmt
.
Fprint
(
file
,
"test"
)
writer
.
Close
()
response
:=
testUploadArtifacts
(
writer
.
FormDataContentType
(),
&
buffer
,
t
,
ts
)
testhelper
.
AssertResponseCode
(
t
,
response
,
500
)
}
internal/artifacts/metadata.go
View file @
e0a2ea65
...
@@ -2,9 +2,11 @@ package artifacts
...
@@ -2,9 +2,11 @@ package artifacts
import
(
import
(
"archive/zip"
"archive/zip"
"compress/gzip"
"encoding/binary"
"encoding/binary"
"encoding/json"
"encoding/json"
"io"
"io"
"os"
"strconv"
"strconv"
)
)
...
@@ -17,6 +19,8 @@ type metadata struct {
...
@@ -17,6 +19,8 @@ type metadata struct {
Comment
string
`json:"comment,omitempty"`
Comment
string
`json:"comment,omitempty"`
}
}
const
metadataHeader
=
"GitLab Build Artifacts Metadata 0.0.2
\n
"
func
newMetadata
(
file
*
zip
.
File
)
metadata
{
func
newMetadata
(
file
*
zip
.
File
)
metadata
{
return
metadata
{
return
metadata
{
Modified
:
file
.
ModTime
()
.
Unix
(),
Modified
:
file
.
ModTime
()
.
Unix
(),
...
@@ -28,7 +32,7 @@ func newMetadata(file *zip.File) metadata {
...
@@ -28,7 +32,7 @@ func newMetadata(file *zip.File) metadata {
}
}
}
}
func
(
m
metadata
)
write
(
output
io
.
Writer
)
error
{
func
(
m
metadata
)
write
Encoded
(
output
io
.
Writer
)
error
{
j
,
err
:=
json
.
Marshal
(
m
)
j
,
err
:=
json
.
Marshal
(
m
)
if
err
!=
nil
{
if
err
!=
nil
{
return
err
return
err
...
@@ -37,23 +41,34 @@ func (m metadata) write(output io.Writer) error {
...
@@ -37,23 +41,34 @@ func (m metadata) write(output io.Writer) error {
return
writeBytes
(
output
,
j
)
return
writeBytes
(
output
,
j
)
}
}
func
generateZipMetadata
(
output
io
.
Writer
,
archive
*
zip
.
Reader
)
error
{
func
writeZipEntryMetadata
(
output
io
.
Writer
,
entry
*
zip
.
File
)
error
{
err
:=
writeString
(
output
,
"GitLab Build Artifacts Metadata 0.0.2
\n
"
)
err
:=
writeString
(
output
,
entry
.
Name
)
if
err
!=
nil
{
if
err
!=
nil
{
return
err
return
err
}
}
err
=
writeString
(
output
,
"{}"
)
err
=
newMetadata
(
entry
)
.
writeEncoded
(
output
)
if
err
!=
nil
{
if
err
!=
nil
{
return
err
return
err
}
}
return
nil
}
for
_
,
entry
:=
range
archive
.
File
{
func
generateZipMetadata
(
output
io
.
Writer
,
archive
*
zip
.
Reader
)
error
{
err
=
writeString
(
output
,
entry
.
Name
)
err
:=
writeString
(
output
,
metadataHeader
)
if
err
!=
nil
{
return
err
}
// Write empty error string
err
=
writeString
(
output
,
"{}"
)
if
err
!=
nil
{
if
err
!=
nil
{
return
err
return
err
}
}
err
=
newMetadata
(
entry
)
.
write
(
output
)
// Write all files
for
_
,
entry
:=
range
archive
.
File
{
err
=
writeZipEntryMetadata
(
output
,
entry
)
if
err
!=
nil
{
if
err
!=
nil
{
return
err
return
err
}
}
...
@@ -61,6 +76,20 @@ func generateZipMetadata(output io.Writer, archive *zip.Reader) error {
...
@@ -61,6 +76,20 @@ func generateZipMetadata(output io.Writer, archive *zip.Reader) error {
return
nil
return
nil
}
}
func
generateZipMetadataFromFile
(
fileName
string
,
w
io
.
Writer
)
error
{
archive
,
err
:=
zip
.
OpenReader
(
fileName
)
if
err
!=
nil
{
// Ignore non-zip archives
return
os
.
ErrInvalid
}
defer
archive
.
Close
()
gz
:=
gzip
.
NewWriter
(
w
)
defer
gz
.
Close
()
return
generateZipMetadata
(
gz
,
&
archive
.
Reader
)
}
func
writeBytes
(
output
io
.
Writer
,
data
[]
byte
)
error
{
func
writeBytes
(
output
io
.
Writer
,
data
[]
byte
)
error
{
err
:=
binary
.
Write
(
output
,
binary
.
BigEndian
,
uint32
(
len
(
data
)))
err
:=
binary
.
Write
(
output
,
binary
.
BigEndian
,
uint32
(
len
(
data
)))
if
err
==
nil
{
if
err
==
nil
{
...
...
internal/artifacts/upload_filter.go
deleted
100644 → 0
View file @
49b0cdef
package
artifacts
import
(
"mime/multipart"
"fmt"
"archive/zip"
"compress/gzip"
)
type
artifactsFormFilter
struct
{
}
func
(
a
*
artifactsFormFilter
)
FilterFile
(
formName
,
fileName
string
,
writer
*
multipart
.
Writer
)
error
{
if
formName
!=
"file"
{
return
fmt
.
Errorf
(
"Invalid form field: %q"
,
formName
)
}
archive
,
err
:=
zip
.
OpenReader
(
fileName
)
if
err
!=
nil
{
// Ignore non-zip archives
return
nil
}
defer
archive
.
Close
()
// TODO:
// we could create a temporary file and save to this file instead of writing to mulipart.Writer
// doing it like this is simpler, but puts more pressure on memory
metadataFile
,
err
:=
writer
.
CreateFormFile
(
"metadata"
,
"metadata.gz"
)
if
err
!=
nil
{
return
err
}
defer
writer
.
Close
()
gz
:=
gzip
.
NewWriter
(
metadataFile
)
defer
gz
.
Close
()
err
=
generateZipMetadata
(
gz
,
&
archive
.
Reader
)
if
err
!=
nil
{
return
err
}
return
nil
}
func
(
a
*
artifactsFormFilter
)
FilterField
(
formName
string
,
writer
*
multipart
.
Writer
)
error
{
return
nil
}
internal/upload/uploads.go
View file @
e0a2ea65
...
@@ -11,14 +11,12 @@ import (
...
@@ -11,14 +11,12 @@ import (
"os"
"os"
)
)
const
tempPathHeader
=
"Gitlab-Workhorse-Temp-Path"
type
MultipartFormProcessor
interface
{
ProcessFile
(
formName
,
fileName
string
,
writer
*
multipart
.
Writer
)
error
type
MultipartFormFilter
interface
{
ProcessField
(
formName
string
,
writer
*
multipart
.
Writer
)
error
FilterFile
(
formName
,
fileName
string
,
writer
*
multipart
.
Writer
)
error
FilterField
(
formName
string
,
writer
*
multipart
.
Writer
)
error
}
}
func
rewriteFormFilesFromMultipart
(
r
*
http
.
Request
,
writer
*
multipart
.
Writer
,
tempPath
string
,
filter
MultipartForm
Filte
r
)
(
cleanup
func
(),
err
error
)
{
func
rewriteFormFilesFromMultipart
(
r
*
http
.
Request
,
writer
*
multipart
.
Writer
,
tempPath
string
,
filter
MultipartForm
Processo
r
)
(
cleanup
func
(),
err
error
)
{
// Create multipart reader
// Create multipart reader
reader
,
err
:=
r
.
MultipartReader
()
reader
,
err
:=
r
.
MultipartReader
()
if
err
!=
nil
{
if
err
!=
nil
{
...
@@ -78,7 +76,7 @@ func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, te
...
@@ -78,7 +76,7 @@ func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, te
file
.
Close
()
file
.
Close
()
if
filter
!=
nil
{
if
filter
!=
nil
{
err
=
filter
.
Filter
File
(
name
,
file
.
Name
(),
writer
)
err
=
filter
.
Process
File
(
name
,
file
.
Name
(),
writer
)
if
err
!=
nil
{
if
err
!=
nil
{
return
cleanup
,
err
return
cleanup
,
err
}
}
...
@@ -95,7 +93,7 @@ func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, te
...
@@ -95,7 +93,7 @@ func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, te
}
}
if
filter
!=
nil
{
if
filter
!=
nil
{
err
=
filter
.
Filter
Field
(
name
,
writer
)
err
=
filter
.
Process
Field
(
name
,
writer
)
if
err
!=
nil
{
if
err
!=
nil
{
return
cleanup
,
err
return
cleanup
,
err
}
}
...
@@ -105,7 +103,12 @@ func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, te
...
@@ -105,7 +103,12 @@ func rewriteFormFilesFromMultipart(r *http.Request, writer *multipart.Writer, te
return
cleanup
,
nil
return
cleanup
,
nil
}
}
func
HandleFileUploads
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
,
h
http
.
Handler
,
tempPath
string
,
filter
MultipartFormFilter
)
{
func
HandleFileUploads
(
w
http
.
ResponseWriter
,
r
*
http
.
Request
,
h
http
.
Handler
,
tempPath
string
,
filter
MultipartFormProcessor
)
{
if
tempPath
==
""
{
helper
.
Fail500
(
w
,
fmt
.
Errorf
(
"handleFileUploads: temporary path not defined"
))
return
}
var
body
bytes
.
Buffer
var
body
bytes
.
Buffer
writer
:=
multipart
.
NewWriter
(
&
body
)
writer
:=
multipart
.
NewWriter
(
&
body
)
defer
writer
.
Close
()
defer
writer
.
Close
()
...
...
internal/upload/uploads_test.go
View file @
e0a2ea65
...
@@ -5,6 +5,7 @@ import (
...
@@ -5,6 +5,7 @@ import (
"../proxy"
"../proxy"
"../testhelper"
"../testhelper"
"bytes"
"bytes"
"errors"
"fmt"
"fmt"
"io"
"io"
"io/ioutil"
"io/ioutil"
...
@@ -19,10 +20,27 @@ import (
...
@@ -19,10 +20,27 @@ import (
var
nilHandler
=
http
.
HandlerFunc
(
func
(
http
.
ResponseWriter
,
*
http
.
Request
)
{})
var
nilHandler
=
http
.
HandlerFunc
(
func
(
http
.
ResponseWriter
,
*
http
.
Request
)
{})
type
testFormProcessor
struct
{
}
func
(
a
*
testFormProcessor
)
ProcessFile
(
formName
,
fileName
string
,
writer
*
multipart
.
Writer
)
error
{
if
formName
!=
"file"
&&
fileName
!=
"my.file"
{
return
errors
.
New
(
"illegal file"
)
}
return
nil
}
func
(
a
*
testFormProcessor
)
ProcessField
(
formName
string
,
writer
*
multipart
.
Writer
)
error
{
if
formName
!=
"token"
{
return
errors
.
New
(
"illegal field"
)
}
return
nil
}
func
TestUploadTempPathRequirement
(
t
*
testing
.
T
)
{
func
TestUploadTempPathRequirement
(
t
*
testing
.
T
)
{
response
:=
httptest
.
NewRecorder
()
response
:=
httptest
.
NewRecorder
()
request
:=
&
http
.
Request
{}
request
:=
&
http
.
Request
{}
handleFileUploads
(
nilHandler
)
.
ServeHTTP
(
response
,
request
)
HandleFileUploads
(
response
,
request
,
nilHandler
,
""
,
nil
)
testhelper
.
AssertResponseCode
(
t
,
response
,
500
)
testhelper
.
AssertResponseCode
(
t
,
response
,
500
)
}
}
...
@@ -56,9 +74,8 @@ func TestUploadHandlerForwardingRawData(t *testing.T) {
...
@@ -56,9 +74,8 @@ func TestUploadHandlerForwardingRawData(t *testing.T) {
response
:=
httptest
.
NewRecorder
()
response
:=
httptest
.
NewRecorder
()
httpRequest
.
Header
.
Set
(
tempPathHeader
,
tempPath
)
handler
:=
proxy
.
NewProxy
(
helper
.
URLMustParse
(
ts
.
URL
),
"123"
,
nil
)
HandleFileUploads
(
response
,
httpRequest
,
handler
,
tempPath
,
nil
)
handleFileUploads
(
proxy
.
NewProxy
(
helper
.
URLMustParse
(
ts
.
URL
),
"123"
,
nil
))
.
ServeHTTP
(
response
,
httpRequest
)
testhelper
.
AssertResponseCode
(
t
,
response
,
202
)
testhelper
.
AssertResponseCode
(
t
,
response
,
202
)
if
response
.
Body
.
String
()
!=
"RESPONSE"
{
if
response
.
Body
.
String
()
!=
"RESPONSE"
{
t
.
Fatal
(
"Expected RESPONSE in response body"
)
t
.
Fatal
(
"Expected RESPONSE in response body"
)
...
@@ -129,13 +146,65 @@ func TestUploadHandlerRewritingMultiPartData(t *testing.T) {
...
@@ -129,13 +146,65 @@ func TestUploadHandlerRewritingMultiPartData(t *testing.T) {
httpRequest
.
Body
=
ioutil
.
NopCloser
(
&
buffer
)
httpRequest
.
Body
=
ioutil
.
NopCloser
(
&
buffer
)
httpRequest
.
ContentLength
=
int64
(
buffer
.
Len
())
httpRequest
.
ContentLength
=
int64
(
buffer
.
Len
())
httpRequest
.
Header
.
Set
(
"Content-Type"
,
writer
.
FormDataContentType
())
httpRequest
.
Header
.
Set
(
"Content-Type"
,
writer
.
FormDataContentType
())
httpRequest
.
Header
.
Set
(
tempPathHeader
,
tempPath
)
response
:=
httptest
.
NewRecorder
()
response
:=
httptest
.
NewRecorder
()
handleFileUploads
(
proxy
.
NewProxy
(
helper
.
URLMustParse
(
ts
.
URL
),
"123"
,
nil
))
.
ServeHTTP
(
response
,
httpRequest
)
handler
:=
proxy
.
NewProxy
(
helper
.
URLMustParse
(
ts
.
URL
),
"123"
,
nil
)
HandleFileUploads
(
response
,
httpRequest
,
handler
,
tempPath
,
&
testFormProcessor
{})
testhelper
.
AssertResponseCode
(
t
,
response
,
202
)
testhelper
.
AssertResponseCode
(
t
,
response
,
202
)
if
_
,
err
:=
os
.
Stat
(
filePath
);
!
os
.
IsNotExist
(
err
)
{
if
_
,
err
:=
os
.
Stat
(
filePath
);
!
os
.
IsNotExist
(
err
)
{
t
.
Fatal
(
"expected the file to be deleted"
)
t
.
Fatal
(
"expected the file to be deleted"
)
}
}
}
}
func
TestUploadProcessingField
(
t
*
testing
.
T
)
{
tempPath
,
err
:=
ioutil
.
TempDir
(
""
,
"uploads"
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
defer
os
.
RemoveAll
(
tempPath
)
var
buffer
bytes
.
Buffer
writer
:=
multipart
.
NewWriter
(
&
buffer
)
writer
.
WriteField
(
"token2"
,
"test"
)
writer
.
Close
()
httpRequest
,
err
:=
http
.
NewRequest
(
"PUT"
,
"/url/path"
,
&
buffer
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
httpRequest
.
Header
.
Set
(
"Content-Type"
,
writer
.
FormDataContentType
())
response
:=
httptest
.
NewRecorder
()
HandleFileUploads
(
response
,
httpRequest
,
nilHandler
,
tempPath
,
&
testFormProcessor
{})
testhelper
.
AssertResponseCode
(
t
,
response
,
500
)
}
func
TestUploadProcessingFile
(
t
*
testing
.
T
)
{
tempPath
,
err
:=
ioutil
.
TempDir
(
""
,
"uploads"
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
defer
os
.
RemoveAll
(
tempPath
)
var
buffer
bytes
.
Buffer
writer
:=
multipart
.
NewWriter
(
&
buffer
)
file
,
err
:=
writer
.
CreateFormFile
(
"file2"
,
"my.file"
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
fmt
.
Fprint
(
file
,
"test"
)
writer
.
Close
()
httpRequest
,
err
:=
http
.
NewRequest
(
"PUT"
,
"/url/path"
,
&
buffer
)
if
err
!=
nil
{
t
.
Fatal
(
err
)
}
httpRequest
.
Header
.
Set
(
"Content-Type"
,
writer
.
FormDataContentType
())
response
:=
httptest
.
NewRecorder
()
HandleFileUploads
(
response
,
httpRequest
,
nilHandler
,
tempPath
,
&
testFormProcessor
{})
testhelper
.
AssertResponseCode
(
t
,
response
,
500
)
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment