Skip to content

P
packer
Commit 8fdb4f77 authored by Chris Bednarski's avatar Chris Bednarski
Browse files
Options

WIP 2/4 tests passing, still need to re-implement ZIP and bare compression... 

WIP 2/4 tests passing, still need to re-implement ZIP and bare compression files and do some cleanup
parent 47bb5ae8
Hide whitespace changes
Inline Side-by-side
Showing with 321 additions and 176 deletions
post-processor/compress/post-processor.go
View file @ 8fdb4f77
...@@ -3,19 +3,14 @@ package compress ...@@ -3,19 +3,14 @@ package compress
import ( import (
"archive/tar" "archive/tar"
"archive/zip" "archive/zip"
"compress/flate"
"compress/gzip" "compress/gzip"
"fmt" "fmt"
"io" "io"
"os" "os"
"path/filepath" "path/filepath"
"regexp"
"runtime" "runtime"
"strings"
"time"
"gopkg.in/yaml.v2"
"github.com/biogo/hts/bgzf"
"github.com/klauspost/pgzip" "github.com/klauspost/pgzip"
"github.com/mitchellh/packer/common" "github.com/mitchellh/packer/common"
"github.com/mitchellh/packer/helper/config" "github.com/mitchellh/packer/helper/config"
...@@ -24,24 +19,13 @@ import ( ...@@ -24,24 +19,13 @@ import (
"github.com/pierrec/lz4" "github.com/pierrec/lz4"
) )
type Metadata map[string]Metaitem
type Metaitem struct {
CompSize int64 `yaml:"compsize"`
OrigSize int64 `yaml:"origsize"`
CompType string `yaml:"comptype"`
CompDate string `yaml:"compdate"`
}
type Config struct { type Config struct {
common.PackerConfig `mapstructure:",squash"` common.PackerConfig `mapstructure:",squash"`
OutputPath string `mapstructure:"output"` OutputPath string `mapstructure:"output"`
OutputFile string `mapstructure:"file"` Level int `mapstructure:"level"`
Compression int `mapstructure:"compression"`
Metadata bool `mapstructure:"metadata"`
NumCPU int `mapstructure:"numcpu"`
Format string `mapstructure:"format"`
KeepInputArtifact bool `mapstructure:"keep_input_artifact"` KeepInputArtifact bool `mapstructure:"keep_input_artifact"`
Archive string
Algorithm string
ctx *interpolate.Context ctx *interpolate.Context
} }
...@@ -49,8 +33,52 @@ type PostProcessor struct { ...@@ -49,8 +33,52 @@ type PostProcessor struct {
config Config config Config
} }
// ErrInvalidCompressionLevel is returned when the compression level passed to
// gzip is not in the expected range. See compress/flate for details.
var ErrInvalidCompressionLevel = fmt.Errorf(
"Invalid compression level. Expected an integer from -1 to 9.")
var ErrWrongInputCount = fmt.Errorf(
"Can only have 1 input file when not using tar/zip")
func detectFromFilename(config *Config) error {
re := regexp.MustCompile("^.+?(?:\\.([a-z0-9]+))?\\.([a-z0-9]+)$")
extensions := map[string]string{
"tar": "tar",
"zip": "zip",
"gz": "pgzip",
"lz4": "lz4",
}
result := re.FindAllString(config.OutputPath, -1)
// Should we make an archive? E.g. tar or zip?
if result[0] == "tar" {
config.Archive = "tar"
}
if result[1] == "zip" || result[1] == "tar" {
config.Archive = result[1]
// Tar or zip is our final artifact. Bail out.
return nil
}
// Should we compress the artifact?
algorithm, ok := extensions[result[1]]
if ok {
config.Algorithm = algorithm
// We found our compression algorithm something. Bail out.
return nil
}
// We didn't find anything. Default to tar + pgzip
config.Algorithm = "pgzip"
config.Archive = "tar"
return fmt.Errorf("Unable to detect compression algorithm")
}
func (p *PostProcessor) Configure(raws ...interface{}) error { func (p *PostProcessor) Configure(raws ...interface{}) error {
p.config.Compression = -1 p.config.Level = -1
err := config.Decode(&p.config, &config.DecodeOpts{ err := config.Decode(&p.config, &config.DecodeOpts{
Interpolate: true, Interpolate: true,
InterpolateFilter: &interpolate.RenderFilter{ InterpolateFilter: &interpolate.RenderFilter{
...@@ -73,19 +101,13 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { ...@@ -73,19 +101,13 @@ func (p *PostProcessor) Configure(raws ...interface{}) error {
"output": &p.config.OutputPath, "output": &p.config.OutputPath,
} }
if p.config.Compression > flate.BestCompression { if p.config.Level > gzip.BestCompression {
p.config.Compression = flate.BestCompression p.config.Level = gzip.BestCompression
}
if p.config.Compression == -1 {
p.config.Compression = flate.DefaultCompression
} }
if p.config.Level == -1 {
if p.config.NumCPU < 1 { p.config.Level = gzip.DefaultCompression
p.config.NumCPU = runtime.NumCPU()
} }
runtime.GOMAXPROCS(p.config.NumCPU)
for key, ptr := range templates { for key, ptr := range templates {
if *ptr == "" { if *ptr == "" {
errs = packer.MultiErrorAppend( errs = packer.MultiErrorAppend(
...@@ -107,123 +129,113 @@ func (p *PostProcessor) Configure(raws ...interface{}) error { ...@@ -107,123 +129,113 @@ func (p *PostProcessor) Configure(raws ...interface{}) error {
} }
func (p *PostProcessor) fillMetadata(metadata Metadata, files []string) Metadata { func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) {
// layout shows by example how the reference time should be represented.
const layout = "2006-01-02_15-04-05" newArtifact := &Artifact{Path: p.config.OutputPath}
t := time.Now()
if !p.config.Metadata { outputFile, err := os.Create(p.config.OutputPath)
return metadata if err != nil {
return nil, false, fmt.Errorf(
"Unable to create archive %s: %s", p.config.OutputPath, err)
} }
for _, f := range files { defer outputFile.Close()
if fi, err := os.Stat(f); err != nil {
continue // Setup output interface. If we're using compression, output is a
} else { // compression writer. Otherwise it's just a file.
if i, ok := metadata[filepath.Base(f)]; !ok { var output io.WriteCloser
metadata[filepath.Base(f)] = Metaitem{CompType: p.config.Format, OrigSize: fi.Size(), CompDate: t.Format(layout)} switch p.config.Algorithm {
} else { case "lz4":
i.CompSize = fi.Size() lzwriter := lz4.NewWriter(outputFile)
i.CompDate = t.Format(layout) if p.config.Level > gzip.DefaultCompression {
metadata[filepath.Base(f)] = i lzwriter.Header.HighCompression = true
} }
defer lzwriter.Close()
output = lzwriter
case "pgzip":
output, err = pgzip.NewWriterLevel(outputFile, p.config.Level)
if err != nil {
return nil, false, ErrInvalidCompressionLevel
} }
defer output.Close()
default:
output = outputFile
} }
return metadata
//Archive
switch p.config.Archive {
case "tar":
archiveTar(artifact.Files(), output)
case "zip":
archive := zip.NewWriter(output)
defer archive.Close()
default:
// We have a regular file, so we'll just do an io.Copy
if len(artifact.Files()) != 1 {
return nil, false, fmt.Errorf(
"Can only have 1 input file when not using tar/zip. Found %d "+
"files: %v", len(artifact.Files()), artifact.Files())
}
source, err := os.Open(artifact.Files()[0])
if err != nil {
return nil, false, fmt.Errorf(
"Failed to open source file %s for reading: %s",
artifact.Files()[0], err)
}
defer source.Close()
io.Copy(output, source)
}
return newArtifact, p.config.KeepInputArtifact, nil
} }
func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) { func archiveTar(files []string, output io.WriteCloser) error {
newartifact := &Artifact{Path: p.config.OutputPath} archive := tar.NewWriter(output)
metafile := filepath.Join(p.config.OutputPath, "metadata") defer archive.Close()
ui.Say(fmt.Sprintf("[CBEDNARSKI] Creating archive at %s", newartifact.Path)) for _, path := range files {
_, err := os.Stat(newartifact.Path) file, err := os.Open(path)
if err == nil { if err != nil {
return nil, false, fmt.Errorf("output dir %s must not exists", newartifact.Path) return fmt.Errorf("Unable to read file %s: %s", path, err)
} }
err = os.MkdirAll(newartifact.Path, 0755) defer file.Close()
if err != nil {
return nil, false, fmt.Errorf("failed to create output: %s", err)
}
p.config.Format += "tar.gzip" fi, err := file.Stat()
formats := strings.Split(p.config.Format, ".") if err != nil {
ui.Say(fmt.Sprintf("[CBEDNARSKI] Formats length %d", len(formats))) return fmt.Errorf("Unable to get fileinfo for %s: %s", path, err)
if len(p.config.Format) == 0 {
ui.Say("[CBEDNARSKI] Formats is empty")
formats[0] = "tar.gzip"
}
files := artifact.Files()
metadata := make(Metadata, 0)
metadata = p.fillMetadata(metadata, files)
ui.Say(fmt.Sprintf("[CBEDNARSKI] Formats %#v", formats))
for _, compress := range formats {
switch compress {
case "tar":
files, err = p.cmpTAR(files, filepath.Join(p.config.OutputPath, p.config.OutputFile))
metadata = p.fillMetadata(metadata, files)
case "zip":
files, err = p.cmpZIP(files, filepath.Join(p.config.OutputPath, p.config.OutputFile))
metadata = p.fillMetadata(metadata, files)
case "pgzip":
files, err = p.cmpPGZIP(files, p.config.OutputPath)
metadata = p.fillMetadata(metadata, files)
case "gzip":
files, err = p.cmpGZIP(files, p.config.OutputPath)
metadata = p.fillMetadata(metadata, files)
case "bgzf":
files, err = p.cmpBGZF(files, p.config.OutputPath)
metadata = p.fillMetadata(metadata, files)
case "lz4":
files, err = p.cmpLZ4(files, p.config.OutputPath)
metadata = p.fillMetadata(metadata, files)
case "e2fs":
files, err = p.cmpE2FS(files, filepath.Join(p.config.OutputPath, p.config.OutputFile))
metadata = p.fillMetadata(metadata, files)
} }
target, err := os.Readlink(path)
if err != nil { if err != nil {
return nil, false, fmt.Errorf("Failed to compress: %s", err) return fmt.Errorf("Failed to readlink for %s: %s", path, err)
} }
}
if p.config.Metadata { header, err := tar.FileInfoHeader(fi, target)
fp, err := os.Create(metafile)
if err != nil { if err != nil {
return nil, false, err return fmt.Errorf("Failed to create tar header for %s: %s", path, err)
}
if buf, err := yaml.Marshal(metadata); err != nil {
fp.Close()
return nil, false, err
} else {
if _, err = fp.Write(buf); err != nil {
fp.Close()
return nil, false, err
}
fp.Close()
} }
}
newartifact.files = append(newartifact.files, files...) if err := archive.WriteHeader(header); err != nil {
if p.config.Metadata { return fmt.Errorf("Failed to write tar header for %s: %s", path, err)
newartifact.files = append(newartifact.files, metafile) }
}
return newartifact, p.config.KeepInputArtifact, nil if _, err := io.Copy(archive, file); err != nil {
return fmt.Errorf("Failed to copy %s data to archive: %s", path, err)
}
}
return nil
} }
func (p *PostProcessor) cmpTAR(src []string, dst string) ([]string, error) { func (p *PostProcessor) cmpTAR(files []string, target string) ([]string, error) {
fw, err := os.Create(dst) fw, err := os.Create(target)
if err != nil { if err != nil {
return nil, fmt.Errorf("tar error creating tar %s: %s", dst, err) return nil, fmt.Errorf("tar error creating tar %s: %s", target, err)
} }
defer fw.Close() defer fw.Close()
tw := tar.NewWriter(fw) tw := tar.NewWriter(fw)
defer tw.Close() defer tw.Close()
for _, name := range src { for _, name := range files {
fi, err := os.Stat(name) fi, err := os.Stat(name)
if err != nil { if err != nil {
return nil, fmt.Errorf("tar error on stat of %s: %s", name, err) return nil, fmt.Errorf("tar error on stat of %s: %s", name, err)
...@@ -250,18 +262,18 @@ func (p *PostProcessor) cmpTAR(src []string, dst string) ([]string, error) { ...@@ -250,18 +262,18 @@ func (p *PostProcessor) cmpTAR(src []string, dst string) ([]string, error) {
} }
fr.Close() fr.Close()
} }
return []string{dst}, nil return []string{target}, nil
} }
func (p *PostProcessor) cmpGZIP(src []string, dst string) ([]string, error) { func (p *PostProcessor) cmpGZIP(files []string, target string) ([]string, error) {
var res []string var res []string
for _, name := range src { for _, name := range files {
filename := filepath.Join(dst, filepath.Base(name)) filename := filepath.Join(target, filepath.Base(name))
fw, err := os.Create(filename) fw, err := os.Create(filename)
if err != nil { if err != nil {
return nil, fmt.Errorf("gzip error: %s", err) return nil, fmt.Errorf("gzip error creating archive: %s", err)
} }
cw, err := gzip.NewWriterLevel(fw, p.config.Compression) cw, err := gzip.NewWriterLevel(fw, p.config.Level)
if err != nil { if err != nil {
fw.Close() fw.Close()
return nil, fmt.Errorf("gzip error: %s", err) return nil, fmt.Errorf("gzip error: %s", err)
...@@ -286,15 +298,16 @@ func (p *PostProcessor) cmpGZIP(src []string, dst string) ([]string, error) { ...@@ -286,15 +298,16 @@ func (p *PostProcessor) cmpGZIP(src []string, dst string) ([]string, error) {
return res, nil return res, nil
} }
func (p *PostProcessor) cmpPGZIP(src []string, dst string) ([]string, error) { func (p *PostProcessor) cmpPGZIP(files []string, target string) ([]string, error) {
var res []string var res []string
for _, name := range src { for _, name := range files {
filename := filepath.Join(dst, filepath.Base(name)) filename := filepath.Join(target, filepath.Base(name))
fw, err := os.Create(filename) fw, err := os.Create(filename)
if err != nil { if err != nil {
return nil, fmt.Errorf("pgzip error: %s", err) return nil, fmt.Errorf("pgzip error: %s", err)
} }
cw, err := pgzip.NewWriterLevel(fw, p.config.Compression) cw, err := pgzip.NewWriterLevel(fw, p.config.Level)
cw.SetConcurrency(500000, runtime.GOMAXPROCS(-1))
if err != nil { if err != nil {
fw.Close() fw.Close()
return nil, fmt.Errorf("pgzip error: %s", err) return nil, fmt.Errorf("pgzip error: %s", err)
...@@ -332,7 +345,7 @@ func (p *PostProcessor) cmpLZ4(src []string, dst string) ([]string, error) { ...@@ -332,7 +345,7 @@ func (p *PostProcessor) cmpLZ4(src []string, dst string) ([]string, error) {
fw.Close() fw.Close()
return nil, fmt.Errorf("lz4 error: %s", err) return nil, fmt.Errorf("lz4 error: %s", err)
} }
if p.config.Compression > flate.DefaultCompression { if p.config.Level > gzip.DefaultCompression {
cw.Header.HighCompression = true cw.Header.HighCompression = true
} }
fr, err := os.Open(name) fr, err := os.Open(name)
...@@ -355,43 +368,6 @@ func (p *PostProcessor) cmpLZ4(src []string, dst string) ([]string, error) { ...@@ -355,43 +368,6 @@ func (p *PostProcessor) cmpLZ4(src []string, dst string) ([]string, error) {
return res, nil return res, nil
} }
func (p *PostProcessor) cmpBGZF(src []string, dst string) ([]string, error) {
var res []string
for _, name := range src {
filename := filepath.Join(dst, filepath.Base(name))
fw, err := os.Create(filename)
if err != nil {
return nil, fmt.Errorf("bgzf error: %s", err)
}
cw, err := bgzf.NewWriterLevel(fw, p.config.Compression, runtime.NumCPU())
if err != nil {
return nil, fmt.Errorf("bgzf error: %s", err)
}
fr, err := os.Open(name)
if err != nil {
cw.Close()
fw.Close()
return nil, fmt.Errorf("bgzf error: %s", err)
}
if _, err = io.Copy(cw, fr); err != nil {
cw.Close()
fr.Close()
fw.Close()
return nil, fmt.Errorf("bgzf error: %s", err)
}
cw.Close()
fr.Close()
fw.Close()
res = append(res, filename)
}
return res, nil
}
func (p *PostProcessor) cmpE2FS(src []string, dst string) ([]string, error) {
panic("not implemented")
}
func (p *PostProcessor) cmpZIP(src []string, dst string) ([]string, error) { func (p *PostProcessor) cmpZIP(src []string, dst string) ([]string, error) {
fw, err := os.Create(dst) fw, err := os.Create(dst)
if err != nil { if err != nil {
......
post-processor/compress/post-processor_test.go
View file @ 8fdb4f77
...@@ -83,6 +83,111 @@ func TestSimpleCompress(t *testing.T) { ...@@ -83,6 +83,111 @@ func TestSimpleCompress(t *testing.T) {
} }
} }
func TestZipArchive(t *testing.T) {
if os.Getenv(env.TestEnvVar) == "" {
t.Skip(fmt.Sprintf(
"Acceptance tests skipped unless env '%s' set", env.TestEnvVar))
}
ui, artifact, err := setup(t)
if err != nil {
t.Fatalf("Error bootstrapping test: %s", err)
}
if artifact != nil {
defer artifact.Destroy()
}
tpl, err := template.Parse(strings.NewReader(tarTestCase))
if err != nil {
t.Fatalf("Unable to parse test config: %s", err)
}
compressor := PostProcessor{}
compressor.Configure(tpl.PostProcessors[0][0].Config)
artifactOut, _, err := compressor.PostProcess(ui, artifact)
if err != nil {
t.Fatalf("Failed to archive artifact: %s", err)
}
// Cleanup after the test completes
defer artifactOut.Destroy()
// Verify things look good
_, err = os.Stat("package.zip")
if err != nil {
t.Errorf("Unable to read archive: %s", err)
}
}
func TestTarArchive(t *testing.T) {
if os.Getenv(env.TestEnvVar) == "" {
t.Skip(fmt.Sprintf(
"Acceptance tests skipped unless env '%s' set", env.TestEnvVar))
}
ui, artifact, err := setup(t)
if err != nil {
t.Fatalf("Error bootstrapping test: %s", err)
}
if artifact != nil {
defer artifact.Destroy()
}
tpl, err := template.Parse(strings.NewReader(tarTestCase))
if err != nil {
t.Fatalf("Unable to parse test config: %s", err)
}
compressor := PostProcessor{}
compressor.Configure(tpl.PostProcessors[0][0].Config)
artifactOut, _, err := compressor.PostProcess(ui, artifact)
if err != nil {
t.Fatalf("Failed to archive artifact: %s", err)
}
// Cleanup after the test completes
defer artifactOut.Destroy()
// Verify things look good
_, err = os.Stat("package.tar")
if err != nil {
t.Errorf("Unable to read archive: %s", err)
}
}
func TestCompressOptions(t *testing.T) {
if os.Getenv(env.TestEnvVar) == "" {
t.Skip(fmt.Sprintf(
"Acceptance tests skipped unless env '%s' set", env.TestEnvVar))
}
ui, artifact, err := setup(t)
if err != nil {
t.Fatalf("Error bootstrapping test: %s", err)
}
if artifact != nil {
defer artifact.Destroy()
}
tpl, err := template.Parse(strings.NewReader(zipTestCase))
if err != nil {
t.Fatalf("Unable to parse test config: %s", err)
}
compressor := PostProcessor{}
compressor.Configure(tpl.PostProcessors[0][0].Config)
artifactOut, _, err := compressor.PostProcess(ui, artifact)
if err != nil {
t.Fatalf("Failed to archive artifact: %s", err)
}
// Cleanup after the test completes
defer artifactOut.Destroy()
// Verify things look good
_, err = os.Stat("package.gz")
if err != nil {
t.Errorf("Unable to read archive: %s", err)
}
}
const simpleTestCase = ` const simpleTestCase = `
{ {
"post-processors": [ "post-processors": [
...@@ -93,3 +198,38 @@ const simpleTestCase = ` ...@@ -93,3 +198,38 @@ const simpleTestCase = `
] ]
} }
` `
const zipTestCase = `
{
"post-processors": [
{
"type": "compress",
"output": "package.zip"
}
]
}
`
const tarTestCase = `
{
"post-processors": [
{
"type": "compress",
"output": "package.tar"
}
]
}
`
const optionsTestCase = `
{
"post-processors": [
{
"type": "compress",
"output": "package.gz",
"level": 9,
"parallel": false
}
]
}
`
website/source/docs/post-processors/compress.html.markdown
View file @ 8fdb4f77
...@@ -15,17 +15,46 @@ archive. ...@@ -15,17 +15,46 @@ archive.
## Configuration ## Configuration
The configuration for this post-processor is extremely simple. The minimal required configuration is to specify the output file. This will create a gzipped tarball.
* `output` (string) - The path to save the compressed archive. * `output` (required, string) - The path to save the compressed archive. The archive format is inferred from the filename. E.g. `.tar.gz` will be a gzipped tarball. `.zip` will be a zip file.
If the extension can't be detected tar+gzip will be used as a fallback.
If you want more control over how the archive is created you can specify the following settings:
* `level` (optional, integer) - Specify the compression level, for algorithms that support it. Value from -1 through 9 inclusive. 9 offers the smallest file size, but takes longer
* `keep_input_artifact` (optional, bool) - Keep source files; defaults to false
## Supported Formats
Supported file extensions include `.zip`, `.tar`, `.gz`, `.tar.gz`, `.lz4` and `.tar.lz4`.
## Example ## Example
An example is shown below, showing only the post-processor configuration: Some minimal examples are shown below, showing only the post-processor configuration:
```json
{
"type": "compress",
"output": "archive.tar.gz"
}
```
```json
{
"type": "compress",
"output": "archive.zip"
}
```
A more complex example, again showing only the post-processor configuration:
```javascript ```json
{ {
"type": "compress", "type": "compress",
"output": "foo.tar.gz" "output": "archive.gz",
"compression": 9,
"parallel": false
} }
``` ```
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7