Commit 1edbbd80 authored by Chris Bednarski's avatar Chris Bednarski

Merge pull request #2205 from mitchellh/f-vtolstov-compress

Added new compress post-processor from Vasiliy Tolstov
parents 8a2ec1ca 2d92fd87
package file
import (
"fmt"
"log"
"os"
)
type FileArtifact struct {
filename string
}
func (*FileArtifact) BuilderId() string {
return BuilderId
}
func (a *FileArtifact) Files() []string {
return []string{a.filename}
}
func (a *FileArtifact) Id() string {
return "File"
}
func (a *FileArtifact) String() string {
return fmt.Sprintf("Stored file: %s", a.filename)
}
func (a *FileArtifact) State(name string) interface{} {
return nil
}
func (a *FileArtifact) Destroy() error {
log.Printf("Deleting %s", a.filename)
return os.Remove(a.filename)
}
package file
import (
"testing"
"github.com/mitchellh/packer/packer"
)
func TestNullArtifact(t *testing.T) {
var _ packer.Artifact = new(FileArtifact)
}
package file
/*
The File builder creates an artifact from a file. Because it does not require
any virutalization or network resources, it's very fast and useful for testing.
*/
import (
"fmt"
"io"
"io/ioutil"
"os"
"github.com/mitchellh/multistep"
"github.com/mitchellh/packer/packer"
)
const BuilderId = "packer.file"
type Builder struct {
config *Config
runner multistep.Runner
}
func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
c, warnings, errs := NewConfig(raws...)
if errs != nil {
return warnings, errs
}
b.config = c
return warnings, nil
}
// Run is where the actual build should take place. It takes a Build and a Ui.
func (b *Builder) Run(ui packer.Ui, hook packer.Hook, cache packer.Cache) (packer.Artifact, error) {
artifact := new(FileArtifact)
if b.config.Source != "" {
source, err := os.Open(b.config.Source)
defer source.Close()
if err != nil {
return nil, err
}
// Create will truncate an existing file
target, err := os.Create(b.config.Target)
defer target.Close()
if err != nil {
return nil, err
}
ui.Say(fmt.Sprintf("Copying %s to %s", source.Name(), target.Name()))
bytes, err := io.Copy(target, source)
if err != nil {
return nil, err
}
ui.Say(fmt.Sprintf("Copied %d bytes", bytes))
artifact.filename = target.Name()
} else {
// We're going to write Contents; if it's empty we'll just create an
// empty file.
err := ioutil.WriteFile(b.config.Target, []byte(b.config.Content), 0600)
if err != nil {
return nil, err
}
artifact.filename = b.config.Target
}
return artifact, nil
}
// Cancel cancels a possibly running Builder. This should block until
// the builder actually cancels and cleans up after itself.
func (b *Builder) Cancel() {
b.runner.Cancel()
}
package file
import (
"fmt"
"io/ioutil"
"testing"
builderT "github.com/mitchellh/packer/helper/builder/testing"
"github.com/mitchellh/packer/packer"
)
func TestBuilder_implBuilder(t *testing.T) {
var _ packer.Builder = new(Builder)
}
func TestBuilderFileAcc_content(t *testing.T) {
builderT.Test(t, builderT.TestCase{
Builder: &Builder{},
Template: fileContentTest,
Check: checkContent,
})
}
func TestBuilderFileAcc_copy(t *testing.T) {
builderT.Test(t, builderT.TestCase{
Builder: &Builder{},
Template: fileCopyTest,
Check: checkCopy,
})
}
func checkContent(artifacts []packer.Artifact) error {
content, err := ioutil.ReadFile("contentTest.txt")
if err != nil {
return err
}
contentString := string(content)
if contentString != "hello world!" {
return fmt.Errorf("Unexpected file contents: %s", contentString)
}
return nil
}
func checkCopy(artifacts []packer.Artifact) error {
content, err := ioutil.ReadFile("copyTest.txt")
if err != nil {
return err
}
contentString := string(content)
if contentString != "Hello world.\n" {
return fmt.Errorf("Unexpected file contents: %s", contentString)
}
return nil
}
const fileContentTest = `
{
"builders": [
{
"type":"test",
"target":"contentTest.txt",
"content":"hello world!"
}
]
}
`
const fileCopyTest = `
{
"builders": [
{
"type":"test",
"target":"copyTest.txt",
"source":"test-fixtures/artifact.txt"
}
]
}
`
package file
import (
"fmt"
"github.com/mitchellh/packer/common"
"github.com/mitchellh/packer/helper/config"
"github.com/mitchellh/packer/packer"
"github.com/mitchellh/packer/template/interpolate"
)
var ErrTargetRequired = fmt.Errorf("target required")
var ErrContentSourceConflict = fmt.Errorf("Cannot specify source file AND content")
type Config struct {
common.PackerConfig `mapstructure:",squash"`
Source string `mapstructure:"source"`
Target string `mapstructure:"target"`
Content string `mapstructure:"content"`
}
func NewConfig(raws ...interface{}) (*Config, []string, error) {
c := new(Config)
warnings := []string{}
err := config.Decode(c, &config.DecodeOpts{
Interpolate: true,
InterpolateFilter: &interpolate.RenderFilter{
Exclude: []string{},
},
}, raws...)
if err != nil {
return nil, warnings, err
}
var errs *packer.MultiError
if c.Target == "" {
errs = packer.MultiErrorAppend(errs, ErrTargetRequired)
}
if c.Content == "" && c.Source == "" {
warnings = append(warnings, "Both source file and contents are blank; target will have no content")
}
if c.Content != "" && c.Source != "" {
errs = packer.MultiErrorAppend(errs, ErrContentSourceConflict)
}
if errs != nil && len(errs.Errors) > 0 {
return nil, warnings, errs
}
return c, warnings, nil
}
package file
import (
"strings"
"testing"
)
func testConfig() map[string]interface{} {
return map[string]interface{}{
"source": "src.txt",
"target": "dst.txt",
"content": "Hello, world!",
}
}
func TestContentSourceConflict(t *testing.T) {
raw := testConfig()
_, _, errs := NewConfig(raw)
if !strings.Contains(errs.Error(), ErrContentSourceConflict.Error()) {
t.Errorf("Expected config error: %s", ErrContentSourceConflict.Error())
}
}
func TestNoFilename(t *testing.T) {
raw := testConfig()
delete(raw, "filename")
_, _, errs := NewConfig(raw)
if errs == nil {
t.Errorf("Expected config error: %s", ErrTargetRequired.Error())
}
}
func TestNoContent(t *testing.T) {
raw := testConfig()
delete(raw, "content")
delete(raw, "source")
_, warns, _ := NewConfig(raw)
if len(warns) == 0 {
t.Error("Expected config warning without any content")
}
}
package main
import (
"github.com/mitchellh/packer/builder/file"
"github.com/mitchellh/packer/packer/plugin"
)
func main() {
server, err := plugin.Server()
if err != nil {
panic(err)
}
server.RegisterBuilder(new(file.Builder))
server.Serve()
}
The MIT License (MIT)
Copyright (c) 2014 Vasiliy Tolstov
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
......@@ -10,6 +10,7 @@ const BuilderId = "packer.post-processor.compress"
type Artifact struct {
Path string
Provider string
files []string
}
func NewArtifact(provider, path string) *Artifact {
......@@ -19,26 +20,26 @@ func NewArtifact(provider, path string) *Artifact {
}
}
func (*Artifact) BuilderId() string {
func (a *Artifact) BuilderId() string {
return BuilderId
}
func (self *Artifact) Id() string {
func (*Artifact) Id() string {
return ""
}
func (self *Artifact) Files() []string {
return []string{self.Path}
func (a *Artifact) Files() []string {
return []string{a.Path}
}
func (self *Artifact) String() string {
return fmt.Sprintf("'%s' compressing: %s", self.Provider, self.Path)
func (a *Artifact) String() string {
return fmt.Sprintf("'%s' compressing: %s", a.Provider, a.Path)
}
func (*Artifact) State(name string) interface{} {
return nil
}
func (self *Artifact) Destroy() error {
return os.Remove(self.Path)
func (a *Artifact) Destroy() error {
return os.Remove(a.Path)
}
// +build ignore
package main
import (
"compress/flate"
"compress/gzip"
"fmt"
"io"
"io/ioutil"
"os"
"runtime"
"testing"
"github.com/biogo/hts/bgzf"
"github.com/klauspost/pgzip"
"github.com/pierrec/lz4"
)
type Compressor struct {
r *os.File
w *os.File
sr int64
sw int64
}
func (c *Compressor) Close() error {
var err error
fi, _ := c.w.Stat()
c.sw = fi.Size()
if err = c.w.Close(); err != nil {
return err
}
fi, _ = c.r.Stat()
c.sr = fi.Size()
if err = c.r.Close(); err != nil {
return err
}
return nil
}
func NewCompressor(src, dst string) (*Compressor, error) {
r, err := os.Open(src)
if err != nil {
return nil, err
}
w, err := os.Create(dst)
if err != nil {
r.Close()
return nil, err
}
c := &Compressor{r: r, w: w}
return c, nil
}
func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
var resw testing.BenchmarkResult
var resr testing.BenchmarkResult
c, err := NewCompressor("/tmp/image.r", "/tmp/image.w")
if err != nil {
panic(err)
}
resw = testing.Benchmark(c.BenchmarkGZIPWriter)
c.w.Seek(0, 0)
resr = testing.Benchmark(c.BenchmarkGZIPReader)
c.Close()
fmt.Printf("gzip:\twriter %s\treader %s\tsize %d\n", resw.T.String(), resr.T.String(), c.sw)
c, err = NewCompressor("/tmp/image.r", "/tmp/image.w")
if err != nil {
panic(err)
}
resw = testing.Benchmark(c.BenchmarkBGZFWriter)
c.w.Seek(0, 0)
resr = testing.Benchmark(c.BenchmarkBGZFReader)
c.Close()
fmt.Printf("bgzf:\twriter %s\treader %s\tsize %d\n", resw.T.String(), resr.T.String(), c.sw)
c, err = NewCompressor("/tmp/image.r", "/tmp/image.w")
if err != nil {
panic(err)
}
resw = testing.Benchmark(c.BenchmarkPGZIPWriter)
c.w.Seek(0, 0)
resr = testing.Benchmark(c.BenchmarkPGZIPReader)
c.Close()
fmt.Printf("pgzip:\twriter %s\treader %s\tsize %d\n", resw.T.String(), resr.T.String(), c.sw)
c, err = NewCompressor("/tmp/image.r", "/tmp/image.w")
if err != nil {
panic(err)
}
resw = testing.Benchmark(c.BenchmarkLZ4Writer)
c.w.Seek(0, 0)
resr = testing.Benchmark(c.BenchmarkLZ4Reader)
c.Close()
fmt.Printf("lz4:\twriter %s\treader %s\tsize %d\n", resw.T.String(), resr.T.String(), c.sw)
}
func (c *Compressor) BenchmarkGZIPWriter(b *testing.B) {
cw, _ := gzip.NewWriterLevel(c.w, flate.BestSpeed)
b.ResetTimer()
_, err := io.Copy(cw, c.r)
if err != nil {
b.Fatal(err)
}
cw.Close()
c.w.Sync()
}
func (c *Compressor) BenchmarkGZIPReader(b *testing.B) {
cr, _ := gzip.NewReader(c.w)
b.ResetTimer()
_, err := io.Copy(ioutil.Discard, cr)
if err != nil {
b.Fatal(err)
}
}
func (c *Compressor) BenchmarkBGZFWriter(b *testing.B) {
cw, _ := bgzf.NewWriterLevel(c.w, flate.BestSpeed, runtime.NumCPU())
b.ResetTimer()
_, err := io.Copy(cw, c.r)
if err != nil {
b.Fatal(err)
}
c.w.Sync()
}
func (c *Compressor) BenchmarkBGZFReader(b *testing.B) {
cr, _ := bgzf.NewReader(c.w, 0)
b.ResetTimer()
_, err := io.Copy(ioutil.Discard, cr)
if err != nil {
b.Fatal(err)
}
}
func (c *Compressor) BenchmarkPGZIPWriter(b *testing.B) {
cw, _ := pgzip.NewWriterLevel(c.w, flate.BestSpeed)
b.ResetTimer()
_, err := io.Copy(cw, c.r)
if err != nil {
b.Fatal(err)
}
cw.Close()
c.w.Sync()
}
func (c *Compressor) BenchmarkPGZIPReader(b *testing.B) {
cr, _ := pgzip.NewReader(c.w)
b.ResetTimer()
_, err := io.Copy(ioutil.Discard, cr)
if err != nil {
b.Fatal(err)
}
}
func (c *Compressor) BenchmarkLZ4Writer(b *testing.B) {
cw := lz4.NewWriter(c.w)
// cw.Header.HighCompression = true
cw.Header.NoChecksum = true
b.ResetTimer()
_, err := io.Copy(cw, c.r)
if err != nil {
b.Fatal(err)
}
cw.Close()
c.w.Sync()
}
func (c *Compressor) BenchmarkLZ4Reader(b *testing.B) {
cr := lz4.NewReader(c.w)
b.ResetTimer()
_, err := io.Copy(ioutil.Discard, cr)
if err != nil {
b.Fatal(err)
}
}
This diff is collapsed.
package compress
import ()
import (
"compress/gzip"
"fmt"
"io/ioutil"
"os"
"strings"
"testing"
"github.com/mitchellh/packer/builder/file"
env "github.com/mitchellh/packer/helper/builder/testing"
"github.com/mitchellh/packer/packer"
"github.com/mitchellh/packer/template"
)
func TestDetectFilename(t *testing.T) {
// Test default / fallback with no file extension
nakedFilename := Config{OutputPath: "test"}
nakedFilename.detectFromFilename()
if nakedFilename.Archive != "tar" {
t.Error("Expected to find tar archive setting")
}
if nakedFilename.Algorithm != "pgzip" {
t.Error("Expected to find pgzip algorithm setting")
}
// Test .archive
zipFilename := Config{OutputPath: "test.zip"}
zipFilename.detectFromFilename()
if zipFilename.Archive != "zip" {
t.Error("Expected to find zip archive setting")
}
if zipFilename.Algorithm != "" {
t.Error("Expected to find empty algorithm setting")
}
// Test .compress
lz4Filename := Config{OutputPath: "test.lz4"}
lz4Filename.detectFromFilename()
if lz4Filename.Archive != "" {
t.Error("Expected to find empty archive setting")
}
if lz4Filename.Algorithm != "lz4" {
t.Error("Expected to find lz4 algorithm setting")
}
// Test .archive.compress with some.extra.dots...
lotsOfDots := Config{OutputPath: "test.blah.bloo.blee.tar.lz4"}
lotsOfDots.detectFromFilename()
if lotsOfDots.Archive != "tar" {
t.Error("Expected to find tar archive setting")
}
if lotsOfDots.Algorithm != "lz4" {
t.Error("Expected to find lz4 algorithm setting")
}
}
const expectedFileContents = "Hello world!"
func TestSimpleCompress(t *testing.T) {
const config = `
{
"post-processors": [
{
"type": "compress",
"output": "package.tar.gz"
}
]
}
`
artifact := testArchive(t, config)
defer artifact.Destroy()
fi, err := os.Stat("package.tar.gz")
if err != nil {
t.Errorf("Unable to read archive: %s", err)
}
if fi.IsDir() {
t.Error("Archive should not be a directory")
}
}
func TestZipArchive(t *testing.T) {
const config = `
{
"post-processors": [
{
"type": "compress",
"output": "package.zip"
}
]
}
`
artifact := testArchive(t, config)
defer artifact.Destroy()
// Verify things look good
_, err := os.Stat("package.zip")
if err != nil {
t.Errorf("Unable to read archive: %s", err)
}
}
func TestTarArchive(t *testing.T) {
const config = `
{
"post-processors": [
{
"type": "compress",
"output": "package.tar"
}
]
}
`
artifact := testArchive(t, config)
defer artifact.Destroy()
// Verify things look good
_, err := os.Stat("package.tar")
if err != nil {
t.Errorf("Unable to read archive: %s", err)
}
}
func TestCompressOptions(t *testing.T) {
const config = `
{
"post-processors": [
{
"type": "compress",
"output": "package.gz",
"compression_level": 9
}
]
}
`
artifact := testArchive(t, config)
defer artifact.Destroy()
filename := "package.gz"
archive, _ := os.Open(filename)
gzipReader, _ := gzip.NewReader(archive)
data, _ := ioutil.ReadAll(gzipReader)
if string(data) != expectedFileContents {
t.Errorf("Expected:\n%s\nFound:\n%s\n", expectedFileContents, data)
}
}
// Test Helpers
func setup(t *testing.T) (packer.Ui, packer.Artifact, error) {
// Create fake UI and Cache
ui := packer.TestUi(t)
cache := &packer.FileCache{CacheDir: os.TempDir()}
// Create config for file builder
const fileConfig = `{"builders":[{"type":"file","target":"package.txt","content":"Hello world!"}]}`
tpl, err := template.Parse(strings.NewReader(fileConfig))
if err != nil {
return nil, nil, fmt.Errorf("Unable to parse setup configuration: %s", err)
}
// Prepare the file builder
builder := file.Builder{}
warnings, err := builder.Prepare(tpl.Builders["file"].Config)
if len(warnings) > 0 {
for _, warn := range warnings {
return nil, nil, fmt.Errorf("Configuration warning: %s", warn)
}
}
if err != nil {
return nil, nil, fmt.Errorf("Invalid configuration: %s", err)
}
// Run the file builder
artifact, err := builder.Run(ui, nil, cache)
if err != nil {
return nil, nil, fmt.Errorf("Failed to build artifact: %s", err)
}
return ui, artifact, err
}
func testArchive(t *testing.T, config string) packer.Artifact {
if os.Getenv(env.TestEnvVar) == "" {
t.Skip(fmt.Sprintf(
"Acceptance tests skipped unless env '%s' set", env.TestEnvVar))
}
ui, artifact, err := setup(t)
if err != nil {
t.Fatalf("Error bootstrapping test: %s", err)
}
if artifact != nil {
defer artifact.Destroy()
}
tpl, err := template.Parse(strings.NewReader(config))
if err != nil {
t.Fatalf("Unable to parse test config: %s", err)
}
compressor := PostProcessor{}
compressor.Configure(tpl.PostProcessors[0][0].Config)
artifactOut, _, err := compressor.PostProcess(ui, artifact)
if err != nil {
t.Fatalf("Failed to compress artifact: %s", err)
}
return artifactOut
}
......@@ -2,7 +2,7 @@
layout: "docs"
page_title: "compress Post-Processor"
description: |-
The Packer compress post-processor takes an artifact with files (such as from VMware or VirtualBox) and gzip compresses the artifact into a single archive.
The Packer compress post-processor takes an artifact with files (such as from VMware or VirtualBox) and compresses the artifact into a single archive.
---
# Compress Post-Processor
......@@ -10,22 +10,58 @@ description: |-
Type: `compress`
The Packer compress post-processor takes an artifact with files (such as from
VMware or VirtualBox) and gzip compresses the artifact into a single
archive.
VMware or VirtualBox) and compresses the artifact into a single archive.
## Configuration
The configuration for this post-processor is extremely simple.
### Required:
* `output` (string) - The path to save the compressed archive.
You must specify the output filename. The archive format is derived from the filename.
## Example
* `output` (string) - The path to save the compressed archive. The archive
format is inferred from the filename. E.g. `.tar.gz` will be a gzipped
tarball. `.zip` will be a zip file. If the extension can't be detected packer
defaults to `.tar.gz` behavior but will not change the filename.
An example is shown below, showing only the post-processor configuration:
If you are executing multiple builders in parallel you should make sure
`output` is unique for each one. For example `packer_{{.BuildName}}_{{.Provider}}.zip`.
```javascript
### Optional:
If you want more control over how the archive is created you can specify the following settings:
* `compression_level` (integer) - Specify the compression level, for algorithms
that support it, from 1 through 9 inclusive. Typically higher compression
levels take longer but produce smaller files. Defaults to `6`
* `keep_input_artifact` (bool) - Keep source files; defaults to `false`
### Supported Formats
Supported file extensions include `.zip`, `.tar`, `.gz`, `.tar.gz`, `.lz4` and `.tar.lz4`. Note that `.gz` and `.lz4` will fail if you have multiple files to compress.
## Examples
Some minimal examples are shown below, showing only the post-processor configuration:
```json
{
"type": "compress",
"output": "archive.tar.lz4"
}
```
```json
{
"type": "compress",
"output": "archive.zip"
}
```
```json
{
"type": "compress",
"output": "foo.tar.gz"
"output": "archive.gz",
"compression": 9
}
```
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment