diff --git a/.travis.yml b/.travis.yml index 78592598..39d2ac44 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,8 @@ language: go go: - - 1.9.x - - 1.10.3 + - 1.11.x + - 1.12.1 install: - go build -v diff --git a/fs/fs_test.go b/fs/fs_test.go index 2cf799ef..32522d2a 100644 --- a/fs/fs_test.go +++ b/fs/fs_test.go @@ -15,18 +15,17 @@ package fs import ( "archive/zip" - "bytes" "io" "io/ioutil" "os" "path" - "path/filepath" "reflect" "sort" - "strings" "sync" "testing" "time" + + "github.com/rakyll/statik/ziptree" ) type wantFile struct { @@ -388,46 +387,11 @@ func BenchmarkOpen(b *testing.B) { // mustZipTree walks on the source path and returns the zipped file contents // as a string. Panics on any errors. func mustZipTree(srcPath string) string { - var out bytes.Buffer - w := zip.NewWriter(&out) - if err := filepath.Walk(srcPath, func(path string, fi os.FileInfo, err error) error { - if err != nil { - return err - } - // Ignore directories and hidden files. - // No entry is needed for directories in a zip file. - // Each file is represented with a path, no directory - // entities are required to build the hierarchy. - if fi.IsDir() || strings.HasPrefix(fi.Name(), ".") { - return nil - } - relPath, err := filepath.Rel(srcPath, path) - if err != nil { - return err - } - b, err := ioutil.ReadFile(path) - if err != nil { - return err - } - fHeader, err := zip.FileInfoHeader(fi) - if err != nil { - return err - } - fHeader.Name = filepath.ToSlash(relPath) - fHeader.Method = zip.Deflate - f, err := w.CreateHeader(fHeader) - if err != nil { - return err - } - _, err = f.Write(b) - return err - }); err != nil { - panic(err) - } - if err := w.Close(); err != nil { + bs, err := ziptree.Zip(srcPath) + if err != nil { panic(err) } - return out.String() + return string(bs) } // mustReadFile returns the file contents. Panics on any errors. diff --git a/statik.go b/statik.go index 91b2a643..c799a7fb 100644 --- a/statik.go +++ b/statik.go @@ -17,7 +17,6 @@ package main import ( - "archive/zip" "bytes" "flag" "fmt" @@ -25,9 +24,10 @@ import ( "io/ioutil" "os" "path" - "path/filepath" "strings" "time" + + "github.com/rakyll/statik/ziptree" ) const ( @@ -45,6 +45,7 @@ var ( flagTags = flag.String("tags", "", "Write build constraint tags") flagPkg = flag.String("p", "statik", "Name of the generated package") flagPkgCmt = flag.String("c", "Package statik contains static assets.", "The package comment. An empty value disables this comment.\n") + flagDotfiles = flag.Bool("dotfiles", false, "Collect dotfiles inside source or not.") ) // mtimeDate holds the arbitrary mtime that we assign to files when @@ -119,65 +120,19 @@ func rename(src, dest string) error { // Generates source registers generated zip contents data to // be read by the statik/fs HTTP file system. func generateSource(srcPath string) (file *os.File, err error) { - var ( - buffer bytes.Buffer - zipWriter io.Writer - ) - - zipWriter = &buffer - f, err := ioutil.TempFile("", namePackage) - if err != nil { - return + var opts []ziptree.Option + if *flagNoCompress { + opts = append(opts, ziptree.Compress(false)) } - - zipWriter = io.MultiWriter(zipWriter, f) - defer f.Close() - - w := zip.NewWriter(zipWriter) - if err = filepath.Walk(srcPath, func(path string, fi os.FileInfo, err error) error { - if err != nil { - return err - } - // Ignore directories and hidden files. - // No entry is needed for directories in a zip file. - // Each file is represented with a path, no directory - // entities are required to build the hierarchy. - if fi.IsDir() || strings.HasPrefix(fi.Name(), ".") { - return nil - } - relPath, err := filepath.Rel(srcPath, path) - if err != nil { - return err - } - b, err := ioutil.ReadFile(path) - if err != nil { - return err - } - fHeader, err := zip.FileInfoHeader(fi) - if err != nil { - return err - } - if *flagNoMtime { - // Always use the same modification time so that - // the output is deterministic with respect to the file contents. - // Do NOT use fHeader.Modified as it only works on go >= 1.10 - fHeader.SetModTime(mtimeDate) - } - fHeader.Name = filepath.ToSlash(relPath) - if !*flagNoCompress { - fHeader.Method = zip.Deflate - } - f, err := w.CreateHeader(fHeader) - if err != nil { - return err - } - _, err = f.Write(b) - return err - }); err != nil { - return + if *flagNoMtime { + opts = append(opts, ziptree.FixMtime(mtimeDate)) } - if err = w.Close(); err != nil { - return + if *flagDotfiles { + opts = append(opts, ziptree.IncludeDotFiles(true)) + } + bs, err := ziptree.Zip(srcPath, opts...) + if err != nil { + return nil, err } var tags string @@ -202,41 +157,22 @@ import ( func init() { data := "`, tags, comment, namePackage) - FprintZipData(&qb, buffer.Bytes()) + ziptree.FprintZipData(&qb, bs) fmt.Fprint(&qb, `" fs.Register(data) } `) - + f, err := ioutil.TempFile("", namePackage) + if err != nil { + return nil, err + } + f.Close() if err = ioutil.WriteFile(f.Name(), qb.Bytes(), 0644); err != nil { return } return f, nil } -// FprintZipData converts zip binary contents to a string literal. -func FprintZipData(dest *bytes.Buffer, zipData []byte) { - for _, b := range zipData { - if b == '\n' { - dest.WriteString(`\n`) - continue - } - if b == '\\' { - dest.WriteString(`\\`) - continue - } - if b == '"' { - dest.WriteString(`\"`) - continue - } - if (b >= 32 && b <= 126) || b == '\t' { - dest.WriteByte(b) - continue - } - fmt.Fprintf(dest, "\\x%02x", b) - } -} - // comment lines prefixes each line in lines with "// ". func commentLines(lines string) string { lines = "// " + strings.Replace(lines, "\n", "\n// ", -1) diff --git a/testdata/ziptree-skipdir/.dot/inside-dot b/testdata/ziptree-skipdir/.dot/inside-dot new file mode 100644 index 00000000..e69de29b diff --git a/testdata/ziptree-skipdir/.dotfile b/testdata/ziptree-skipdir/.dotfile new file mode 100644 index 00000000..e69de29b diff --git a/testdata/ziptree-skipdir/general-file b/testdata/ziptree-skipdir/general-file new file mode 100644 index 00000000..e69de29b diff --git a/testdata/ziptree/hello b/testdata/ziptree/hello new file mode 100644 index 00000000..e69de29b diff --git a/ziptree/fprint.go b/ziptree/fprint.go new file mode 100644 index 00000000..b109f4ab --- /dev/null +++ b/ziptree/fprint.go @@ -0,0 +1,33 @@ +package ziptree + +import ( + "bufio" + "fmt" + "io" +) + +// FprintZipData converts zip binary contents to a string literal. +func FprintZipData(w io.Writer, zipData []byte) error { + dest := bufio.NewWriter(w) + for _, b := range zipData { + if err := func(b byte) error { + switch b { + case '\n': + _, err := dest.WriteString(`\n`) + return err + case '\\', '"': + _, err := dest.WriteString(`\` + string(b)) + return err + default: + if (b >= 32 && b <= 126) || b == '\t' { + return dest.WriteByte(b) + } + _, err := fmt.Fprintf(dest, "\\x%02x", b) + return err + } + }(b); err != nil { + return err + } + } + return dest.Flush() +} diff --git a/ziptree/zipper.go b/ziptree/zipper.go new file mode 100644 index 00000000..60541fd2 --- /dev/null +++ b/ziptree/zipper.go @@ -0,0 +1,106 @@ +// Package ziptree contains code to zip a directory tree and write it out. +package ziptree + +import ( + "archive/zip" + "bytes" + "io/ioutil" + "os" + "path/filepath" + "strings" + "time" +) + +// Option for zipping +type Option func(*config) + +type config struct { + Compress bool + Mtime time.Time + IncludeDotFiles bool +} + +func newConfig() *config { + return &config{Compress: true} +} + +// Compress is an Option to compress a zip or not +func Compress(f bool) Option { + return func(c *config) { + c.Compress = f + } +} + +// FixMtime is an Option to fix mtimes of the file in the zip +func FixMtime(t time.Time) Option { + return func(c *config) { + c.Mtime = t + } +} + +// Compress is an Option to include dotfiles or not +func IncludeDotFiles(f bool) Option { + return func(c *config) { + c.IncludeDotFiles = f + } +} + +// Zip a directory tree +func Zip(srcPath string, opts ...Option) ([]byte, error) { + c := newConfig() + for _, opt := range opts { + opt(c) + } + + var buffer bytes.Buffer + w := zip.NewWriter(&buffer) + if err := filepath.Walk(srcPath, func(path string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + // Ignore directories and hidden files. + // No entry is needed for directories in a zip file. + // Each file is represented with a path, no directory + // entities are required to build the hierarchy. + if fi.IsDir() || (!c.IncludeDotFiles && strings.HasPrefix(fi.Name(), ".")) { + if fi.IsDir() && strings.HasPrefix(fi.Name(), ".") && !c.IncludeDotFiles { + return filepath.SkipDir + } + return nil + } + relPath, err := filepath.Rel(srcPath, path) + if err != nil { + return err + } + b, err := ioutil.ReadFile(path) + if err != nil { + return err + } + fHeader, err := zip.FileInfoHeader(fi) + if err != nil { + return err + } + if !c.Mtime.IsZero() { + // Always use the same modification time so that + // the output is deterministic with respect to the file contents. + // Do NOT use fHeader.Modified as it only works on go >= 1.10 + fHeader.SetModTime(c.Mtime) + } + fHeader.Name = filepath.ToSlash(relPath) + if c.Compress { + fHeader.Method = zip.Deflate + } + f, err := w.CreateHeader(fHeader) + if err != nil { + return err + } + _, err = f.Write(b) + return err + }); err != nil { + return nil, err + } + if err := w.Close(); err != nil { + return nil, err + } + return buffer.Bytes(), nil +} diff --git a/ziptree/zipper_test.go b/ziptree/zipper_test.go new file mode 100644 index 00000000..3b9180b1 --- /dev/null +++ b/ziptree/zipper_test.go @@ -0,0 +1,106 @@ +package ziptree_test + +import ( + "archive/zip" + "bytes" + "os" + "reflect" + "sort" + "testing" + "time" + + "github.com/rakyll/statik/ziptree" +) + +var zipData = []byte{ + 0x50, 0x4b, 0x03, 0x04, 0x14, 0x00, 0x08, 0x00, + 0x08, 0x00, 0x00, 0x00, 0x21, 0x28, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x05, 0x00, 0x09, 0x00, 0x68, 0x65, + 0x6c, 0x6c, 0x6f, 0x55, 0x54, 0x05, 0x00, 0x01, + 0x80, 0x43, 0x6d, 0x38, 0x01, 0x00, 0x00, 0xff, + 0xff, 0x50, 0x4b, 0x07, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x50, 0x4b, 0x01, 0x02, 0x14, 0x03, 0x14, + 0x00, 0x08, 0x00, 0x08, 0x00, 0x00, 0x00, 0x21, + 0x28, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x09, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0xa4, 0x81, 0x00, 0x00, 0x00, 0x00, 0x68, + 0x65, 0x6c, 0x6c, 0x6f, 0x55, 0x54, 0x05, 0x00, + 0x01, 0x80, 0x43, 0x6d, 0x38, 0x50, 0x4b, 0x05, + 0x06, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x01, + 0x00, 0x3c, 0x00, 0x00, 0x00, 0x41, 0x00, 0x00, + 0x00, 0x00, 0x00} +var zipData4Print = zipData + +func TestZip(t *testing.T) { + err := os.Chmod("../testdata/ziptree/hello", 0644) + if err != nil { + t.Fatal(err) + } + out, err := ziptree.Zip( + "../testdata/ziptree/", + ziptree.FixMtime(time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC))) + if err != nil { + t.Errorf("error should be nil but: %s", err) + } + wantData := zipData + if !reflect.DeepEqual(out, wantData) { + t.Errorf("got: %#v\nexpect: %#v", out, wantData) + } +} + +func TestFprintZipData(t *testing.T) { + buf := &bytes.Buffer{} + err := ziptree.FprintZipData(buf, zipData4Print) + if err != nil { + t.Errorf("error should be nil but: %s", err) + } + out := buf.String() + want := `PK\x03\x04\x14\x00\x08\x00\x08\x00\x00\x00!(\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x05\x00 \x00helloUT\x05\x00\x01\x80Cm8\x01\x00\x00\xff\xffPK\x07\x08\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00PK\x01\x02\x14\x03\x14\x00\x08\x00\x08\x00\x00\x00!(\x00\x00\x00\x00\x05\x00\x00\x00\x00\x00\x00\x00\x05\x00 \x00\x00\x00\x00\x00\x00\x00\x00\x00\xa4\x81\x00\x00\x00\x00helloUT\x05\x00\x01\x80Cm8PK\x05\x06\x00\x00\x00\x00\x01\x00\x01\x00<\x00\x00\x00A\x00\x00\x00\x00\x00` + if out != want { + t.Errorf("got: %s\nexpect: %s", out, want) + } +} + +func TestZip_CollectFile(t *testing.T) { + tests := []struct { + description string + dir string + opts []ziptree.Option + wantFiles []string + }{ + { + description: "dot files and files inside dot dirs are not collected", + dir: "../testdata/ziptree-skipdir", + opts: nil, + wantFiles: []string{"general-file"}, + }, + { + description: "include dot files", + dir: "../testdata/ziptree-skipdir", + opts: []ziptree.Option{ziptree.IncludeDotFiles(true)}, + wantFiles: []string{".dot/inside-dot", ".dotfile", "general-file"}, + }, + } + + for _, tc := range tests { + t.Run(tc.description, func(t *testing.T) { + out, err := ziptree.Zip(tc.dir, tc.opts...) + if err != nil { + t.Errorf("error should be nil, but: %s", err) + } + zipReader, err := zip.NewReader(bytes.NewReader(out), int64(len(out))) + l := len(zipReader.File) + files := make([]string, l) + for i := 0; i < l; i++ { + files[i] = zipReader.File[i].Name + } + sort.Strings(files) + if !reflect.DeepEqual(tc.wantFiles, files) { + t.Errorf("got: %v\nwant: %v", files, tc.wantFiles) + } + }) + } +}