Skip to content

P
packer
Commit 12cf6650 authored by Chris Bednarski's avatar Chris Bednarski
Browse files
Options

Revert compress post-processor to master to get baseline test

parent f2f8ad16
Show whitespace changes
Inline Side-by-side
Showing with 60 additions and 373 deletions
post-processor/compress/artifact.go
View file @ 12cf6650
...@@ -8,31 +8,37 @@ import ( ...@@ -8,31 +8,37 @@ import (
const BuilderId = "packer.post-processor.compress" const BuilderId = "packer.post-processor.compress"
type Artifact struct { type Artifact struct {
builderId string Path string
dir string Provider string
f []string
} }
func (a *Artifact) BuilderId() string { func NewArtifact(provider, path string) *Artifact {
return &Artifact{
Path: path,
Provider: provider,
}
}
func (*Artifact) BuilderId() string {
return BuilderId return BuilderId
} }
func (a *Artifact) Files() []string { func (self *Artifact) Id() string {
return a.f return ""
} }
func (*Artifact) Id() string { func (self *Artifact) Files() []string {
return "COMPRESS" return []string{self.Path}
} }
func (a *Artifact) String() string { func (self *Artifact) String() string {
return fmt.Sprintf("VM compressed files in directory: %s", a.dir) return fmt.Sprintf("'%s' compressing: %s", self.Provider, self.Path)
} }
func (*Artifact) State(name string) interface{} { func (*Artifact) State(name string) interface{} {
return nil return nil
} }
func (a *Artifact) Destroy() error { func (self *Artifact) Destroy() error {
return os.RemoveAll(a.dir) return os.Remove(self.Path)
} }
post-processor/compress/post-processor.go
View file @ 12cf6650
...@@ -2,416 +2,97 @@ package compress ...@@ -2,416 +2,97 @@ package compress
import ( import (
"archive/tar" "archive/tar"
"archive/zip"
"compress/flate"
"compress/gzip" "compress/gzip"
"fmt" "fmt"
"io" "io"
"os" "os"
"path/filepath"
"runtime"
"strings"
"time"
"github.com/biogo/hts/bgzf"
"github.com/klauspost/pgzip"
"github.com/mitchellh/packer/common" "github.com/mitchellh/packer/common"
"github.com/mitchellh/packer/helper/config" "github.com/mitchellh/packer/helper/config"
"github.com/mitchellh/packer/packer" "github.com/mitchellh/packer/packer"
"github.com/mitchellh/packer/template/interpolate" "github.com/mitchellh/packer/template/interpolate"
"github.com/pierrec/lz4"
"gopkg.in/yaml.v2"
) )
type Metadata map[string]Metaitem
type Metaitem struct {
CompSize int64 `yaml:"compsize"`
OrigSize int64 `yaml:"origsize"`
CompType string `yaml:"comptype"`
CompDate string `yaml:"compdate"`
}
type Config struct { type Config struct {
common.PackerConfig `mapstructure:",squash"` common.PackerConfig `mapstructure:",squash"`
OutputPath string `mapstructure:"output"` OutputPath string `mapstructure:"output"`
OutputFile string `mapstructure:"file"`
Compression int `mapstructure:"compression"` ctx interpolate.Context
Metadata bool `mapstructure:"metadata"`
NumCPU int `mapstructure:"numcpu"`
Format string `mapstructure:"format"`
KeepInputArtifact bool `mapstructure:"keep_input_artifact"`
ctx *interpolate.Context
} }
type PostProcessor struct { type PostProcessor struct {
cfg Config config Config
} }
func (p *PostProcessor) Configure(raws ...interface{}) error { func (self *PostProcessor) Configure(raws ...interface{}) error {
p.cfg.Compression = -1 err := config.Decode(&self.config, &config.DecodeOpts{
err := config.Decode(&p.cfg, &config.DecodeOpts{
Interpolate: true, Interpolate: true,
InterpolateFilter: &interpolate.RenderFilter{ InterpolateFilter: &interpolate.RenderFilter{
Exclude: []string{ Exclude: []string{},
// TODO figure out if something needs to go here.
},
}, },
}, raws...) }, raws...)
errs := new(packer.MultiError)
if p.cfg.OutputPath == "" {
p.cfg.OutputPath = "packer_{{.BuildName}}_{{.Provider}}"
}
if err = interpolate.Validate(p.cfg.OutputPath, p.cfg.ctx); err != nil {
errs = packer.MultiErrorAppend(
errs, fmt.Errorf("Error parsing target template: %s", err))
}
templates := map[string]*string{
"output": &p.cfg.OutputPath,
}
if p.cfg.Compression > flate.BestCompression {
p.cfg.Compression = flate.BestCompression
}
if p.cfg.Compression == -1 {
p.cfg.Compression = flate.DefaultCompression
}
if p.cfg.NumCPU < 1 {
p.cfg.NumCPU = runtime.NumCPU()
}
runtime.GOMAXPROCS(p.cfg.NumCPU)
for key, ptr := range templates {
if *ptr == "" {
errs = packer.MultiErrorAppend(
errs, fmt.Errorf("%s must be set", key))
}
*ptr, err = interpolate.Render(p.cfg.OutputPath, p.cfg.ctx)
if err != nil { if err != nil {
errs = packer.MultiErrorAppend( return err
errs, fmt.Errorf("Error processing %s: %s", key, err))
}
}
if len(errs.Errors) > 0 {
return errs
} }
return nil return nil
} }
func (p *PostProcessor) fillMetadata(metadata Metadata, files []string) Metadata { func (self *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) {
// layout shows by example how the reference time should be represented. ui.Say(fmt.Sprintf("Creating archive for '%s'", artifact.BuilderId()))
const layout = "2006-01-02_15-04-05"
t := time.Now()
if !p.cfg.Metadata { // Create the compressed archive file at the appropriate OutputPath.
return metadata fw, err := os.Create(self.config.OutputPath)
}
for _, f := range files {
if fi, err := os.Stat(f); err != nil {
continue
} else {
if i, ok := metadata[filepath.Base(f)]; !ok {
metadata[filepath.Base(f)] = Metaitem{CompType: p.cfg.Format, OrigSize: fi.Size(), CompDate: t.Format(layout)}
} else {
i.CompSize = fi.Size()
i.CompDate = t.Format(layout)
metadata[filepath.Base(f)] = i
}
}
}
return metadata
}
func (p *PostProcessor) PostProcess(ui packer.Ui, artifact packer.Artifact) (packer.Artifact, bool, error) {
newartifact := &Artifact{builderId: artifact.BuilderId(), dir: p.cfg.OutputPath}
metafile := filepath.Join(p.cfg.OutputPath, "metadata")
_, err := os.Stat(newartifact.dir)
if err == nil {
return nil, false, fmt.Errorf("output dir must not exists: %s", err)
}
err = os.MkdirAll(newartifact.dir, 0755)
if err != nil {
return nil, false, fmt.Errorf("failed to create output: %s", err)
}
formats := strings.Split(p.cfg.Format, ".")
files := artifact.Files()
metadata := make(Metadata, 0)
metadata = p.fillMetadata(metadata, files)
for _, compress := range formats {
switch compress {
case "tar":
files, err = p.cmpTAR(files, filepath.Join(p.cfg.OutputPath, p.cfg.OutputFile))
metadata = p.fillMetadata(metadata, files)
case "zip":
files, err = p.cmpZIP(files, filepath.Join(p.cfg.OutputPath, p.cfg.OutputFile))
metadata = p.fillMetadata(metadata, files)
case "pgzip":
files, err = p.cmpPGZIP(files, p.cfg.OutputPath)
metadata = p.fillMetadata(metadata, files)
case "gzip":
files, err = p.cmpGZIP(files, p.cfg.OutputPath)
metadata = p.fillMetadata(metadata, files)
case "bgzf":
files, err = p.cmpBGZF(files, p.cfg.OutputPath)
metadata = p.fillMetadata(metadata, files)
case "lz4":
files, err = p.cmpLZ4(files, p.cfg.OutputPath)
metadata = p.fillMetadata(metadata, files)
case "e2fs":
files, err = p.cmpE2FS(files, filepath.Join(p.cfg.OutputPath, p.cfg.OutputFile))
metadata = p.fillMetadata(metadata, files)
}
if err != nil {
return nil, false, fmt.Errorf("Failed to compress: %s", err)
}
}
if p.cfg.Metadata {
fp, err := os.Create(metafile)
if err != nil {
return nil, false, err
}
if buf, err := yaml.Marshal(metadata); err != nil {
fp.Close()
return nil, false, err
} else {
if _, err = fp.Write(buf); err != nil {
fp.Close()
return nil, false, err
}
fp.Close()
}
}
newartifact.f = append(newartifact.f, files...)
if p.cfg.Metadata {
newartifact.f = append(newartifact.f, metafile)
}
return newartifact, p.cfg.KeepInputArtifact, nil
}
func (p *PostProcessor) cmpTAR(src []string, dst string) ([]string, error) {
fw, err := os.Create(dst)
if err != nil { if err != nil {
return nil, fmt.Errorf("tar error: %s", err) return nil, false, fmt.Errorf(
"Failed creating file for compressed archive: %s", self.config.OutputPath)
} }
defer fw.Close() defer fw.Close()
tw := tar.NewWriter(fw) gw := gzip.NewWriter(fw)
defer tw.Close() defer gw.Close()
for _, name := range src { // Iterate through all of the artifact's files and put them into the
fi, err := os.Stat(name) // compressed archive using the tar/gzip writers.
for _, path := range artifact.Files() {
fi, err := os.Stat(path)
if err != nil { if err != nil {
return nil, fmt.Errorf("tar error: %s", err) return nil, false, fmt.Errorf(
"Failed stating file: %s", path)
} }
target, _ := os.Readlink(name) target, _ := os.Readlink(path)
header, err := tar.FileInfoHeader(fi, target) header, err := tar.FileInfoHeader(fi, target)
if err != nil { if err != nil {
return nil, fmt.Errorf("tar error: %s", err) return nil, false, fmt.Errorf(
} "Failed creating archive header: %s", path)
if err = tw.WriteHeader(header); err != nil {
return nil, fmt.Errorf("tar error: %s", err)
}
fr, err := os.Open(name)
if err != nil {
return nil, fmt.Errorf("tar error: %s", err)
}
if _, err = io.Copy(tw, fr); err != nil {
fr.Close()
return nil, fmt.Errorf("tar error: %s", err)
} }
fr.Close()
}
return []string{dst}, nil
}
func (p *PostProcessor) cmpGZIP(src []string, dst string) ([]string, error) { tw := tar.NewWriter(gw)
var res []string defer tw.Close()
for _, name := range src {
filename := filepath.Join(dst, filepath.Base(name))
fw, err := os.Create(filename)
if err != nil {
return nil, fmt.Errorf("gzip error: %s", err)
}
cw, err := gzip.NewWriterLevel(fw, p.cfg.Compression)
if err != nil {
fw.Close()
return nil, fmt.Errorf("gzip error: %s", err)
}
fr, err := os.Open(name)
if err != nil {
cw.Close()
fw.Close()
return nil, fmt.Errorf("gzip error: %s", err)
}
if _, err = io.Copy(cw, fr); err != nil {
cw.Close()
fr.Close()
fw.Close()
return nil, fmt.Errorf("gzip error: %s", err)
}
cw.Close()
fr.Close()
fw.Close()
res = append(res, filename)
}
return res, nil
}
func (p *PostProcessor) cmpPGZIP(src []string, dst string) ([]string, error) {
var res []string
for _, name := range src {
filename := filepath.Join(dst, filepath.Base(name))
fw, err := os.Create(filename)
if err != nil {
return nil, fmt.Errorf("pgzip error: %s", err)
}
cw, err := pgzip.NewWriterLevel(fw, p.cfg.Compression)
if err != nil {
fw.Close()
return nil, fmt.Errorf("pgzip error: %s", err)
}
fr, err := os.Open(name)
if err != nil {
cw.Close()
fw.Close()
return nil, fmt.Errorf("pgzip error: %s", err)
}
if _, err = io.Copy(cw, fr); err != nil {
cw.Close()
fr.Close()
fw.Close()
return nil, fmt.Errorf("pgzip error: %s", err)
}
cw.Close()
fr.Close()
fw.Close()
res = append(res, filename)
}
return res, nil
}
func (p *PostProcessor) cmpLZ4(src []string, dst string) ([]string, error) {
var res []string
for _, name := range src {
filename := filepath.Join(dst, filepath.Base(name))
fw, err := os.Create(filename)
if err != nil {
return nil, fmt.Errorf("lz4 error: %s", err)
}
cw := lz4.NewWriter(fw)
if err != nil {
fw.Close()
return nil, fmt.Errorf("lz4 error: %s", err)
}
if p.cfg.Compression > flate.DefaultCompression {
cw.Header.HighCompression = true
}
fr, err := os.Open(name)
if err != nil {
cw.Close()
fw.Close()
return nil, fmt.Errorf("lz4 error: %s", err)
}
if _, err = io.Copy(cw, fr); err != nil {
cw.Close()
fr.Close()
fw.Close()
return nil, fmt.Errorf("lz4 error: %s", err)
}
cw.Close()
fr.Close()
fw.Close()
res = append(res, filename)
}
return res, nil
}
func (p *PostProcessor) cmpBGZF(src []string, dst string) ([]string, error) {
var res []string
for _, name := range src {
filename := filepath.Join(dst, filepath.Base(name))
fw, err := os.Create(filename)
if err != nil {
return nil, fmt.Errorf("bgzf error: %s", err)
}
cw, err := bgzf.NewWriterLevel(fw, p.cfg.Compression, runtime.NumCPU())
if err != nil {
return nil, fmt.Errorf("bgzf error: %s", err)
}
fr, err := os.Open(name)
if err != nil {
cw.Close()
fw.Close()
return nil, fmt.Errorf("bgzf error: %s", err)
}
if _, err = io.Copy(cw, fr); err != nil {
cw.Close()
fr.Close()
fw.Close()
return nil, fmt.Errorf("bgzf error: %s", err)
}
cw.Close()
fr.Close()
fw.Close()
res = append(res, filename)
}
return res, nil
}
func (p *PostProcessor) cmpE2FS(src []string, dst string) ([]string, error) {
panic("not implemented")
}
func (p *PostProcessor) cmpZIP(src []string, dst string) ([]string, error) {
fw, err := os.Create(dst)
if err != nil {
return nil, fmt.Errorf("zip error: %s", err)
}
defer fw.Close()
zw := zip.NewWriter(fw)
defer zw.Close()
for _, name := range src { // Write the header first to the archive. This takes partial data
header, err := zw.Create(name) // from the FileInfo that is grabbed by running the stat command.
if err != nil { if err := tw.WriteHeader(header); err != nil {
return nil, fmt.Errorf("zip error: %s", err) return nil, false, fmt.Errorf(
"Failed writing archive header: %s", path)
} }
fr, err := os.Open(name) // Open the target file for archiving and compressing.
fr, err := os.Open(path)
if err != nil { if err != nil {
return nil, fmt.Errorf("zip error: %s", err) return nil, false, fmt.Errorf(
"Failed opening file '%s' to write compressed archive.", path)
} }
defer fr.Close()
if _, err = io.Copy(header, fr); err != nil { if _, err = io.Copy(tw, fr); err != nil {
fr.Close() return nil, false, fmt.Errorf(
return nil, fmt.Errorf("zip error: %s", err) "Failed copying file to archive: %s", path)
} }
fr.Close()
} }
return []string{dst}, nil
return NewArtifact(artifact.BuilderId(), self.config.OutputPath), false, nil
} }
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7