diff --git a/Gopkg.lock b/Gopkg.lock index 98a3c3a4bce..02d13883d4e 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -85,12 +85,12 @@ version = "v1.2.2" [[projects]] - digest = "1:db8e2f3c8cc717afe53c6941776312bd7046d45d9fb757c5f3f32fd3cc46562d" + digest = "1:359173d7e6eb3d35eeb3f2ef5c87bd2aff6e775dba5a1dcf8b2a53b86d0af79e" name = "github.com/codeclysm/extract" packages = ["."] pruneopts = "UT" - revision = "de8493db4b3e06921c3ee97fa0b200435a8a796d" - version = "v2.0.0" + revision = "cb78af9c8af24b50533f931fa83bdfd4170bf840" + version = "v2.2.0" [[projects]] digest = "1:7cb4fdca4c251b3ef8027c90ea35f70c7b661a593b9eeae34753c65499098bb1" @@ -170,6 +170,19 @@ pruneopts = "UT" revision = "36ee7e946282a3fb1cfecd476ddc9b35d8847e42" +[[projects]] + digest = "1:4442692c7525ec403b23a5b3e169a089bb1e6ed3b69795be877d39578744bf7a" + name = "github.com/h2non/filetype" + packages = [ + ".", + "matchers", + "matchers/isobmff", + "types", + ] + pruneopts = "UT" + revision = "2248f2e2f77cd8cf9694216e3f9589d71005de37" + version = "v1.0.8" + [[projects]] digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be" name = "github.com/inconshreveable/mousetrap" @@ -409,18 +422,6 @@ revision = "291d751a997645784a6067bc8a4c528e819ee171" source = "github.com/cmaglie/pb" -[[projects]] - digest = "1:8b1a1e6d3dcef7c542f604f8b9f8013d1d13c3d61f8e9cbe368afa2735c898c2" - name = "gopkg.in/h2non/filetype.v1" - packages = [ - ".", - "matchers", - "types", - ] - pruneopts = "UT" - revision = "cc14fdc9ca0e4c2bafad7458f6ff79fd3947cfbb" - version = "v1.0.5" - [[projects]] digest = "1:4d2e5a73dc1500038e504a8d78b986630e3626dc027bc030ba5c75da257cdb96" name = "gopkg.in/yaml.v2" diff --git a/arduino/resources/install.go b/arduino/resources/install.go index 376562b849b..7c63ee5d18e 100644 --- a/arduino/resources/install.go +++ b/arduino/resources/install.go @@ -123,5 +123,8 @@ func findPackageRoot(parent *paths.Path) (*paths.Path, error) { return nil, fmt.Errorf("no unique root dir in archive, found '%s' and '%s'", root, file) } } + if root == nil { + return nil, fmt.Errorf("files in archive must be placed in a subdirectory") + } return root, nil } diff --git a/vendor/github.com/codeclysm/extract/Gopkg.lock b/vendor/github.com/codeclysm/extract/Gopkg.lock new file mode 100644 index 00000000000..cbbef323f39 --- /dev/null +++ b/vendor/github.com/codeclysm/extract/Gopkg.lock @@ -0,0 +1,61 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + digest = "1:ffe9824d294da03b391f44e1ae8281281b4afc1bdaa9588c9097785e3af10cec" + name = "github.com/davecgh/go-spew" + packages = ["spew"] + pruneopts = "NUT" + revision = "8991bc29aa16c548c550c7ff78260e27b9ab7c73" + version = "v1.1.1" + +[[projects]] + digest = "1:31e4b5b5379d4069d96209534017b11ceee83e4edcea0aecace10e21a22d41e7" + name = "github.com/h2non/filetype" + packages = [ + ".", + "matchers", + "types", + ] + pruneopts = "NUT" + revision = "b2d66fbb5aed4a1fe9f0ca71a109b9161373866a" + version = "v1.0.6" + +[[projects]] + branch = "master" + digest = "1:af67a9ee74095b67773293bfd43713ec6ef617b53ff1a7718868a1127fb50153" + name = "github.com/juju/errors" + packages = ["."] + pruneopts = "NUT" + revision = "089d3ea4e4d597bd98acac068193d341983326a3" + +[[projects]] + digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe" + name = "github.com/pmezard/go-difflib" + packages = ["difflib"] + pruneopts = "NUT" + revision = "792786c7400a136282c1664665ae0a8db921c6c2" + version = "v1.0.0" + +[[projects]] + digest = "1:85adecf1dbfae5769cc62a8bcea6498f8e9f0e2452e4e6686eb36fa4428a5a6e" + name = "github.com/stretchr/testify" + packages = [ + "assert", + "require", + ] + pruneopts = "NUT" + revision = "ffdc059bfe9ce6a4e144ba849dbedead332c6053" + version = "v1.3.0" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + input-imports = [ + "github.com/h2non/filetype", + "github.com/h2non/filetype/types", + "github.com/juju/errors", + "github.com/stretchr/testify/require", + ] + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/vendor/github.com/codeclysm/extract/Gopkg.toml b/vendor/github.com/codeclysm/extract/Gopkg.toml new file mode 100644 index 00000000000..54f737f21e7 --- /dev/null +++ b/vendor/github.com/codeclysm/extract/Gopkg.toml @@ -0,0 +1,43 @@ +# Gopkg.toml example +# +# Refer to https://golang.github.io/dep/docs/Gopkg.toml.html +# for detailed Gopkg.toml documentation. +# +# required = ["github.com/user/thing/cmd/thing"] +# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] +# +# [[constraint]] +# name = "github.com/user/project" +# version = "1.0.0" +# +# [[constraint]] +# name = "github.com/user/project2" +# branch = "dev" +# source = "github.com/myfork/project2" +# +# [[override]] +# name = "github.com/x/y" +# version = "2.4.0" +# +# [prune] +# non-go = false +# go-tests = true +# unused-packages = true + + +[[constraint]] + name = "github.com/h2non/filetype" + version = "1.0.6" + +[[constraint]] + branch = "master" + name = "github.com/juju/errors" + +[[constraint]] + name = "github.com/stretchr/testify" + version = "1.3.0" + +[prune] + go-tests = true + unused-packages = true + non-go = true diff --git a/vendor/github.com/codeclysm/extract/README.md b/vendor/github.com/codeclysm/extract/README.md index e06d7c8d173..e46ff5a011e 100644 --- a/vendor/github.com/codeclysm/extract/README.md +++ b/vendor/github.com/codeclysm/extract/README.md @@ -36,4 +36,28 @@ If you don't know which archive you're dealing with (life really is always a sur ```go extract.Archive(data, "/path/where/to/extract", nil) +``` + +If you need more control over how your files will be extracted you can use an Extractor. + +It Needs a FS object that implements the FS interface: + +``` +type FS interface { + Link(string, string) error + MkdirAll(string, os.FileMode) error + OpenFile(name string, flag int, perm os.FileMode) (*os.File, error) + Symlink(string, string) error + } +``` + +which contains only the required function to perform an extraction. This way it's easy to wrap os functions to +chroot the path, or scramble the files, or send an event for each operation or even reimplementing them for an in-memory store, I don't know. + +```go +extractor := extract.Extractor{ + FS: fs, +} + +extractor.Archive(data, "path/where/to/extract", nil) ``` \ No newline at end of file diff --git a/vendor/github.com/codeclysm/extract/extract.go b/vendor/github.com/codeclysm/extract/extract.go index 617b7178b97..adb8ed172b9 100644 --- a/vendor/github.com/codeclysm/extract/extract.go +++ b/vendor/github.com/codeclysm/extract/extract.go @@ -27,21 +27,10 @@ package extract import ( - "archive/tar" - "archive/zip" "bytes" - "compress/bzip2" - "compress/gzip" "context" "io" - "io/ioutil" "os" - "path/filepath" - - filetype "gopkg.in/h2non/filetype.v1" - "gopkg.in/h2non/filetype.v1/types" - - "github.com/juju/errors" ) // Renamer is a function that can be used to rename the files when you're extracting @@ -54,67 +43,31 @@ type Renamer func(string) string // handle the names of the files. // If the file is not an archive, an error is returned. func Archive(ctx context.Context, body io.Reader, location string, rename Renamer) error { - body, kind, err := match(body) - if err != nil { - errors.Annotatef(err, "Detect archive type") + extractor := Extractor{ + FS: fs{}, } - switch kind.Extension { - case "zip": - return Zip(ctx, body, location, rename) - case "gz": - return Gz(ctx, body, location, rename) - case "bz2": - return Bz2(ctx, body, location, rename) - case "tar": - return Tar(ctx, body, location, rename) - default: - return errors.New("Not a supported archive") - } + return extractor.Archive(ctx, body, location, rename) } // Bz2 extracts a .bz2 or .tar.bz2 archived stream of data in the specified location. // It accepts a rename function to handle the names of the files (see the example) func Bz2(ctx context.Context, body io.Reader, location string, rename Renamer) error { - reader := bzip2.NewReader(body) - - body, kind, err := match(reader) - if err != nil { - return errors.Annotatef(err, "extract bz2: detect") - } - - if kind.Extension == "tar" { - return Tar(ctx, body, location, rename) + extractor := Extractor{ + FS: fs{}, } - err = copy(ctx, location, 0666, body) - if err != nil { - return err - } - return nil + return extractor.Bz2(ctx, body, location, rename) } // Gz extracts a .gz or .tar.gz archived stream of data in the specified location. // It accepts a rename function to handle the names of the files (see the example) func Gz(ctx context.Context, body io.Reader, location string, rename Renamer) error { - reader, err := gzip.NewReader(body) - if err != nil { - return errors.Annotatef(err, "Gunzip") - } - - body, kind, err := match(reader) - if err != nil { - return err + extractor := Extractor{ + FS: fs{}, } - if kind.Extension == "tar" { - return Tar(ctx, body, location, rename) - } - err = copy(ctx, location, 0666, body) - if err != nil { - return err - } - return nil + return extractor.Gz(ctx, body, location, rename) } type file struct { @@ -130,210 +83,37 @@ type link struct { // Tar extracts a .tar archived stream of data in the specified location. // It accepts a rename function to handle the names of the files (see the example) func Tar(ctx context.Context, body io.Reader, location string, rename Renamer) error { - files := []file{} - links := []link{} - symlinks := []link{} - - // We make the first pass creating the directory structure, or we could end up - // attempting to create a file where there's no folder - tr := tar.NewReader(body) - for { - select { - case <-ctx.Done(): - return errors.New("interrupted") - default: - } - - header, err := tr.Next() - if err == io.EOF { - break - } - - if err != nil { - return errors.Annotatef(err, "Read tar stream") - } - - path := header.Name - if rename != nil { - path = rename(path) - } - - if path == "" { - continue - } - - path = filepath.Join(location, path) - info := header.FileInfo() - - switch header.Typeflag { - case tar.TypeDir: - if err := os.MkdirAll(path, info.Mode()); err != nil { - return errors.Annotatef(err, "Create directory %s", path) - } - case tar.TypeReg, tar.TypeRegA: - var data bytes.Buffer - if _, err := copyCancel(ctx, &data, tr); err != nil { - return errors.Annotatef(err, "Read contents of file %s", path) - } - files = append(files, file{Path: path, Mode: info.Mode(), Data: data}) - case tar.TypeLink: - name := header.Linkname - if rename != nil { - name = rename(name) - } - - name = filepath.Join(location, name) - links = append(links, link{Path: path, Name: name}) - case tar.TypeSymlink: - symlinks = append(symlinks, link{Path: path, Name: header.Linkname}) - } - } - - // Now we make another pass creating the files and links - for i := range files { - if err := copy(ctx, files[i].Path, files[i].Mode, &files[i].Data); err != nil { - return errors.Annotatef(err, "Create file %s", files[i].Path) - } + extractor := Extractor{ + FS: fs{}, } - for i := range links { - select { - case <-ctx.Done(): - return errors.New("interrupted") - default: - } - if err := os.Link(links[i].Name, links[i].Path); err != nil { - return errors.Annotatef(err, "Create link %s", links[i].Path) - } - } - - for i := range symlinks { - select { - case <-ctx.Done(): - return errors.New("interrupted") - default: - } - if err := os.Symlink(symlinks[i].Name, symlinks[i].Path); err != nil { - return errors.Annotatef(err, "Create link %s", symlinks[i].Path) - } - } - return nil + return extractor.Tar(ctx, body, location, rename) } // Zip extracts a .zip archived stream of data in the specified location. // It accepts a rename function to handle the names of the files (see the example). func Zip(ctx context.Context, body io.Reader, location string, rename Renamer) error { - // read the whole body into a buffer. Not sure this is the best way to do it - buffer := bytes.NewBuffer([]byte{}) - copyCancel(ctx, buffer, body) - - archive, err := zip.NewReader(bytes.NewReader(buffer.Bytes()), int64(buffer.Len())) - if err != nil { - return errors.Annotatef(err, "Read the zip file") - } - - files := []file{} - links := []link{} - - // We make the first pass creating the directory structure, or we could end up - // attempting to create a file where there's no folder - for _, header := range archive.File { - select { - case <-ctx.Done(): - return errors.New("interrupted") - default: - } - - path := header.Name - if rename != nil { - path = rename(path) - } - - if path == "" { - continue - } - - path = filepath.Join(location, path) - info := header.FileInfo() - - switch { - case info.IsDir(): - if err := os.MkdirAll(path, info.Mode()|os.ModeDir|100); err != nil { - return errors.Annotatef(err, "Create directory %s", path) - } - // We only check for symlinks because hard links aren't possible - case info.Mode()&os.ModeSymlink != 0: - f, err := header.Open() - if err != nil { - return errors.Annotatef(err, "Open link %s", path) - } - name, err := ioutil.ReadAll(f) - if err != nil { - return errors.Annotatef(err, "Read address of link %s", path) - } - links = append(links, link{Path: path, Name: string(name)}) - default: - f, err := header.Open() - if err != nil { - return errors.Annotatef(err, "Open file %s", path) - } - var data bytes.Buffer - if _, err := copyCancel(ctx, &data, f); err != nil { - return errors.Annotatef(err, "Read contents of file %s", path) - } - files = append(files, file{Path: path, Mode: info.Mode(), Data: data}) - } + extractor := Extractor{ + FS: fs{}, } - // Now we make another pass creating the files and links - for i := range files { - if err := copy(ctx, files[i].Path, files[i].Mode, &files[i].Data); err != nil { - return errors.Annotatef(err, "Create file %s", files[i].Path) - } - } + return extractor.Zip(ctx, body, location, rename) +} - for i := range links { - select { - case <-ctx.Done(): - return errors.New("interrupted") - default: - } - if err := os.Symlink(links[i].Name, links[i].Path); err != nil { - return errors.Annotatef(err, "Create link %s", links[i].Path) - } - } +type fs struct{} - return nil +func (f fs) Link(oldname, newname string) error { + return os.Link(oldname, newname) } -func copy(ctx context.Context, path string, mode os.FileMode, src io.Reader) error { - // We add the execution permission to be able to create files inside it - err := os.MkdirAll(filepath.Dir(path), mode|os.ModeDir|100) - if err != nil { - return err - } - file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode) - if err != nil { - return err - } - defer file.Close() - _, err = copyCancel(ctx, file, src) - return err +func (f fs) MkdirAll(path string, perm os.FileMode) error { + return os.MkdirAll(path, perm) } -// match reads the first 512 bytes, calls types.Match and returns a reader -// for the whole stream -func match(r io.Reader) (io.Reader, types.Type, error) { - buffer := make([]byte, 512) - - n, err := r.Read(buffer) - if err != nil && err != io.EOF { - return nil, types.Unknown, err - } - - r = io.MultiReader(bytes.NewBuffer(buffer[:n]), r) - - typ, err := filetype.Match(buffer) +func (f fs) Symlink(oldname, newname string) error { + return os.Symlink(oldname, newname) +} - return r, typ, err +func (f fs) OpenFile(name string, flag int, perm os.FileMode) (*os.File, error) { + return os.OpenFile(name, flag, perm) } diff --git a/vendor/github.com/codeclysm/extract/extractor.go b/vendor/github.com/codeclysm/extract/extractor.go new file mode 100644 index 00000000000..6bf9085a99d --- /dev/null +++ b/vendor/github.com/codeclysm/extract/extractor.go @@ -0,0 +1,318 @@ +package extract + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/bzip2" + "compress/gzip" + "context" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + + filetype "github.com/h2non/filetype" + "github.com/h2non/filetype/types" + "github.com/juju/errors" +) + +// Extractor is more sophisticated than the base functions. It allows to write over an interface +// rather than directly on the filesystem +type Extractor struct { + FS interface { + Link(string, string) error + MkdirAll(string, os.FileMode) error + OpenFile(name string, flag int, perm os.FileMode) (*os.File, error) + Symlink(string, string) error + } +} + +// Archive extracts a generic archived stream of data in the specified location. +// It automatically detects the archive type and accepts a rename function to +// handle the names of the files. +// If the file is not an archive, an error is returned. +func (e *Extractor) Archive(ctx context.Context, body io.Reader, location string, rename Renamer) error { + body, kind, err := match(body) + if err != nil { + errors.Annotatef(err, "Detect archive type") + } + + switch kind.Extension { + case "zip": + return e.Zip(ctx, body, location, rename) + case "gz": + return e.Gz(ctx, body, location, rename) + case "bz2": + return e.Bz2(ctx, body, location, rename) + case "tar": + return e.Tar(ctx, body, location, rename) + default: + return errors.New("Not a supported archive") + } +} + +// Bz2 extracts a .bz2 or .tar.bz2 archived stream of data in the specified location. +// It accepts a rename function to handle the names of the files (see the example) +func (e *Extractor) Bz2(ctx context.Context, body io.Reader, location string, rename Renamer) error { + reader := bzip2.NewReader(body) + + body, kind, err := match(reader) + if err != nil { + return errors.Annotatef(err, "extract bz2: detect") + } + + if kind.Extension == "tar" { + return Tar(ctx, body, location, rename) + } + + err = e.copy(ctx, location, 0666, body) + if err != nil { + return err + } + return nil +} + +// Gz extracts a .gz or .tar.gz archived stream of data in the specified location. +// It accepts a rename function to handle the names of the files (see the example) +func (e *Extractor) Gz(ctx context.Context, body io.Reader, location string, rename Renamer) error { + reader, err := gzip.NewReader(body) + if err != nil { + return errors.Annotatef(err, "Gunzip") + } + + body, kind, err := match(reader) + if err != nil { + return err + } + + if kind.Extension == "tar" { + return e.Tar(ctx, body, location, rename) + } + err = e.copy(ctx, location, 0666, body) + if err != nil { + return err + } + return nil +} + +// Tar extracts a .tar archived stream of data in the specified location. +// It accepts a rename function to handle the names of the files (see the example) +func (e *Extractor) Tar(ctx context.Context, body io.Reader, location string, rename Renamer) error { + files := []file{} + links := []link{} + symlinks := []link{} + + // We make the first pass creating the directory structure, or we could end up + // attempting to create a file where there's no folder + tr := tar.NewReader(body) + for { + select { + case <-ctx.Done(): + return errors.New("interrupted") + default: + } + + header, err := tr.Next() + if err == io.EOF { + break + } + + if err != nil { + return errors.Annotatef(err, "Read tar stream") + } + + path := header.Name + if rename != nil { + path = rename(path) + } + + if path == "" { + continue + } + + path = filepath.Join(location, path) + info := header.FileInfo() + + switch header.Typeflag { + case tar.TypeDir: + if err := e.FS.MkdirAll(path, info.Mode()); err != nil { + return errors.Annotatef(err, "Create directory %s", path) + } + case tar.TypeReg, tar.TypeRegA: + var data bytes.Buffer + if _, err := copyCancel(ctx, &data, tr); err != nil { + return errors.Annotatef(err, "Read contents of file %s", path) + } + files = append(files, file{Path: path, Mode: info.Mode(), Data: data}) + case tar.TypeLink: + name := header.Linkname + if rename != nil { + name = rename(name) + } + + name = filepath.Join(location, name) + links = append(links, link{Path: path, Name: name}) + case tar.TypeSymlink: + symlinks = append(symlinks, link{Path: path, Name: header.Linkname}) + } + } + + // Now we make another pass creating the files and links + for i := range files { + if err := e.copy(ctx, files[i].Path, files[i].Mode, &files[i].Data); err != nil { + return errors.Annotatef(err, "Create file %s", files[i].Path) + } + } + + for i := range links { + select { + case <-ctx.Done(): + return errors.New("interrupted") + default: + } + if err := e.FS.Link(links[i].Name, links[i].Path); err != nil { + return errors.Annotatef(err, "Create link %s", links[i].Path) + } + } + + for i := range symlinks { + select { + case <-ctx.Done(): + return errors.New("interrupted") + default: + } + if err := e.FS.Symlink(symlinks[i].Name, symlinks[i].Path); err != nil { + return errors.Annotatef(err, "Create link %s", symlinks[i].Path) + } + } + return nil +} + +// Zip extracts a .zip archived stream of data in the specified location. +// It accepts a rename function to handle the names of the files (see the example). +func (e *Extractor) Zip(ctx context.Context, body io.Reader, location string, rename Renamer) error { + // read the whole body into a buffer. Not sure this is the best way to do it + buffer := bytes.NewBuffer([]byte{}) + copyCancel(ctx, buffer, body) + + archive, err := zip.NewReader(bytes.NewReader(buffer.Bytes()), int64(buffer.Len())) + if err != nil { + return errors.Annotatef(err, "Read the zip file") + } + + files := []file{} + links := []link{} + + // We make the first pass creating the directory structure, or we could end up + // attempting to create a file where there's no folder + for _, header := range archive.File { + select { + case <-ctx.Done(): + return errors.New("interrupted") + default: + } + + path := header.Name + + // Replace backslash with forward slash. There are archives in the wild made with + // buggy compressors that use backslash as path separator. The ZIP format explicitly + // denies the use of "\" so we just replace it with slash "/". + // Moreover it seems that folders are stored as "files" but with a final "\" in the + // filename... oh, well... + forceDir := strings.HasSuffix(path, "\\") + path = strings.Replace(path, "\\", "/", -1) + + if rename != nil { + path = rename(path) + } + + if path == "" { + continue + } + + path = filepath.Join(location, path) + info := header.FileInfo() + + switch { + case info.IsDir() || forceDir: + if err := e.FS.MkdirAll(path, info.Mode()|os.ModeDir|100); err != nil { + return errors.Annotatef(err, "Create directory %s", path) + } + // We only check for symlinks because hard links aren't possible + case info.Mode()&os.ModeSymlink != 0: + f, err := header.Open() + if err != nil { + return errors.Annotatef(err, "Open link %s", path) + } + name, err := ioutil.ReadAll(f) + if err != nil { + return errors.Annotatef(err, "Read address of link %s", path) + } + links = append(links, link{Path: path, Name: string(name)}) + default: + f, err := header.Open() + if err != nil { + return errors.Annotatef(err, "Open file %s", path) + } + var data bytes.Buffer + if _, err := copyCancel(ctx, &data, f); err != nil { + return errors.Annotatef(err, "Read contents of file %s", path) + } + files = append(files, file{Path: path, Mode: info.Mode(), Data: data}) + } + } + + // Now we make another pass creating the files and links + for i := range files { + if err := e.copy(ctx, files[i].Path, files[i].Mode, &files[i].Data); err != nil { + return errors.Annotatef(err, "Create file %s", files[i].Path) + } + } + + for i := range links { + select { + case <-ctx.Done(): + return errors.New("interrupted") + default: + } + if err := e.FS.Symlink(links[i].Name, links[i].Path); err != nil { + return errors.Annotatef(err, "Create link %s", links[i].Path) + } + } + + return nil +} + +func (e *Extractor) copy(ctx context.Context, path string, mode os.FileMode, src io.Reader) error { + // We add the execution permission to be able to create files inside it + err := e.FS.MkdirAll(filepath.Dir(path), mode|os.ModeDir|100) + if err != nil { + return err + } + file, err := e.FS.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, mode) + if err != nil { + return err + } + defer file.Close() + _, err = copyCancel(ctx, file, src) + return err +} + +// match reads the first 512 bytes, calls types.Match and returns a reader +// for the whole stream +func match(r io.Reader) (io.Reader, types.Type, error) { + buffer := make([]byte, 512) + + n, err := r.Read(buffer) + if err != nil && err != io.EOF { + return nil, types.Unknown, err + } + + r = io.MultiReader(bytes.NewBuffer(buffer[:n]), r) + + typ, err := filetype.Match(buffer) + + return r, typ, err +} diff --git a/vendor/gopkg.in/h2non/filetype.v1/.editorconfig b/vendor/github.com/h2non/filetype/.editorconfig similarity index 100% rename from vendor/gopkg.in/h2non/filetype.v1/.editorconfig rename to vendor/github.com/h2non/filetype/.editorconfig diff --git a/vendor/gopkg.in/h2non/filetype.v1/.gitignore b/vendor/github.com/h2non/filetype/.gitignore similarity index 100% rename from vendor/gopkg.in/h2non/filetype.v1/.gitignore rename to vendor/github.com/h2non/filetype/.gitignore diff --git a/vendor/gopkg.in/h2non/filetype.v1/.travis.yml b/vendor/github.com/h2non/filetype/.travis.yml similarity index 67% rename from vendor/gopkg.in/h2non/filetype.v1/.travis.yml rename to vendor/github.com/h2non/filetype/.travis.yml index 739e14ebd2d..370dc0472a8 100644 --- a/vendor/gopkg.in/h2non/filetype.v1/.travis.yml +++ b/vendor/github.com/h2non/filetype/.travis.yml @@ -1,14 +1,12 @@ language: go go: - - 1.9 - - 1.8 - - 1.7 - - 1.6 - - tip + - "1.11" + - "1.10" + - "tip" before_install: - - go get -u -v github.com/golang/lint/golint + - go get -u -v golang.org/x/lint/golint script: - diff -u <(echo -n) <(gofmt -s -d ./) diff --git a/vendor/github.com/h2non/filetype/History.md b/vendor/github.com/h2non/filetype/History.md new file mode 100644 index 00000000000..3c02be1fce9 --- /dev/null +++ b/vendor/github.com/h2non/filetype/History.md @@ -0,0 +1,109 @@ + +v1.0.8 / 2019-02-10 +=================== + + * refactor(images): heic -> heif + * feat(docs): add heif format + * Merge pull request #60 from rikonor/master + * add heif/heic support + * fix(docs): dicom -> dcm + * feat: add dicom type + * Merge pull request #58 from Fentonz/master + * Merge pull request #59 from kmanley/master + * fix example; related to h2non/filetype#43 + * Added DICOM type to archive + + +v1.0.7 / 2019-02-09 +=================== + + * Merge pull request #56 from akupila/wasm + * add wasm to readme + * detect wasm file type + +v1.0.6 / 2019-01-22 +=================== + + * Merge pull request #55 from ivanlemeshev/master + * Added ftypmp4v to MP4 matcher + * Merge pull request #54 from aofei/master + * chore: add support for Go modules + * feat: add support for AAC (audio/aac) + * Merge pull request #53 from lynxbyorion/check-for-docoments + * Added checks for documents. + * Merge pull request #51 from eriken/master + * fixed bad mime and import paths + * Merge pull request #50 from eriken/jpeg2000_support + * fix import paths + * jpeg2000 support + * Merge pull request #47 from Ma124/master + * Merge pull request #49 from amoore614/master + * more robust check for .mov files + * bugfix: reverse order of matcher key list so user registered matchers appear first + * bugfix: store ptr to MatcherKeys in case user registered matchers are used. + * update comment + * Bump buffer size to 8K to allow for more custom file matching + * refactor(readme): update package import path + * Merge pull request #48 from kumakichi/support_msooxml + * do not use v1 + * ok, master already changed travis + * add fixtures, but MatchReader may not work for some msooxml files, 4096 bytes maybe not enough + * support ms ooxml, #40 + * Fixed misspells + * fix(travis): use string notation for matrix items + * Merge pull request #42 from bruth/patch-2 + * refactor(travis): remove Go 1.6, add Go 1.10 + * Change maximum bytes required for detection + * Merge pull request #36 from yiiTT/patch-1 + * Add MP4 dash and additional ISO formats + * Merge pull request #34 from RangelReale/fix-mp4-case + * Merge pull request #32 from yiiTT/fix-m4v + * Fixed mp4 detection case-sensitivity according to http://www.ftyps.com/ + * Fix M4v matcher + +v1.0.5 / 2017-12-12 +=================== + + * Merge pull request #30 from RangelReale/fix_mp4 + * Fix duplicated item in mp4 fix + * Fix MP4 matcher, with information from http://www.file-recovery.com/mp4-signature-format.htm + * Merge pull request #28 from ikovic/master + * Updated file header example. + +v1.0.4 / 2017-11-29 +=================== + + * fix: tests and document types matchers + * refactor(docs): remove codesponsor + * Merge pull request #26 from bienkma/master + * Add support check file type: .doc, .docx, .pptx, .ppt, .xls, .xlsx + * feat(docs): add code sponsor banner + * feat(travis): add go 1.9 + * Merge pull request #24 from strazzere/patch-1 + * Fix typo in unknown + +v1.0.3 / 2017-08-03 +=================== + + * Merge pull request #21 from elemeta/master + * Add Elf file as supported matcher archive type + +v1.0.2 / 2017-07-26 +=================== + + * Merge pull request #20 from marshyski/master + * Added RedHat RPM as supported matcher archive type + * Merge pull request #19 from nlamirault/patch-1 + * Fix typo in documentation + +v1.0.1 / 2017-02-24 +=================== + + * Merge pull request #18 from Impyy/enable-webm + * Enable the webm matcher + * feat(docs): add Go version badge + +1.0.0 / 2016-12-11 +================== + +- Initial stable version (v1.0.0). diff --git a/vendor/gopkg.in/h2non/filetype.v1/LICENSE b/vendor/github.com/h2non/filetype/LICENSE similarity index 100% rename from vendor/gopkg.in/h2non/filetype.v1/LICENSE rename to vendor/github.com/h2non/filetype/LICENSE diff --git a/vendor/gopkg.in/h2non/filetype.v1/README.md b/vendor/github.com/h2non/filetype/README.md similarity index 90% rename from vendor/gopkg.in/h2non/filetype.v1/README.md rename to vendor/github.com/h2non/filetype/README.md index 8d45324447f..2968ae244c4 100644 --- a/vendor/gopkg.in/h2non/filetype.v1/README.md +++ b/vendor/github.com/h2non/filetype/README.md @@ -14,14 +14,14 @@ For SVG file type checking, see [go-is-svg](https://github.com/h2non/go-is-svg) - [Pluggable](#add-additional-file-type-matchers): add custom new types and matchers - Simple and semantic API - [Blazing fast](#benchmarks), even processing large files -- Only first 261 bytes representing the max file header is required, so you can just [pass a slice](#file-header) +- Only first 262 bytes representing the max file header is required, so you can just [pass a slice](#file-header) - Dependency free (just Go code, no C compilation needed) - Cross-platform file recognition ## Installation ```bash -go get gopkg.in/h2non/filetype.v1 +go get github.com/h2non/filetype ``` ## API @@ -30,8 +30,8 @@ See [Godoc](https://godoc.org/github.com/h2non/filetype) reference. ### Subpackages -- [`gopkg.in/h2non/filetype.v1/types`](https://godoc.org/github.com/h2non/filetype/types) -- [`gopkg.in/h2non/filetype.v1/matchers`](https://godoc.org/github.com/h2non/filetype/matchers) +- [`github.com/h2non/filetype/types`](https://godoc.org/github.com/h2non/filetype/types) +- [`github.com/h2non/filetype/matchers`](https://godoc.org/github.com/h2non/filetype/matchers) ## Examples @@ -42,16 +42,17 @@ package main import ( "fmt" - "gopkg.in/h2non/filetype.v1" "io/ioutil" + + "github.com/h2non/filetype" ) func main() { buf, _ := ioutil.ReadFile("sample.jpg") - kind, unknown := filetype.Match(buf) - if unknown != nil { - fmt.Printf("Unknown: %s", unknown) + kind, _ := filetype.Match(buf) + if kind == filetype.Unknown { + fmt.Println("Unknown file type") return } @@ -66,8 +67,9 @@ package main import ( "fmt" - "gopkg.in/h2non/filetype.v1" "io/ioutil" + + "github.com/h2non/filetype" ) func main() { @@ -88,7 +90,8 @@ package main import ( "fmt" - "gopkg.in/h2non/filetype.v1" + + "github.com/h2non/filetype" ) func main() { @@ -115,8 +118,9 @@ package main import ( "fmt" - "gopkg.in/h2non/filetype.v1" "io/ioutil" + + "github.com/h2non/filetype" ) func main() { @@ -142,7 +146,8 @@ package main import ( "fmt" - "gopkg.in/h2non/filetype.v1" + + "github.com/h2non/filetype" ) var fooType = filetype.NewType("foo", "foo/foo") @@ -187,6 +192,7 @@ func main() { - **cr2** - `image/x-canon-cr2` - **tif** - `image/tiff` - **bmp** - `image/bmp` +- **heif** - `image/heif` - **jxr** - `image/vnd.ms-photo` - **psd** - `image/vnd.adobe.photoshop` - **ico** - `image/x-icon` @@ -212,6 +218,7 @@ func main() { - **flac** - `audio/x-flac` - **wav** - `audio/x-wav` - **amr** - `audio/amr` +- **aac** - `audio/aac` #### Archive @@ -239,6 +246,7 @@ func main() { - **lz** - `application/x-lzip` - **rpm** - `application/x-rpm` - **elf** - `application/x-executable` +- **dcm** - `application/dicom` #### Documents @@ -256,6 +264,10 @@ func main() { - **ttf** - `application/font-sfnt` - **otf** - `application/font-sfnt` +#### Application + +- **wasm** - `application/wasm` + ## Benchmarks Measured using [real files](https://github.com/h2non/filetype/tree/master/fixtures). diff --git a/vendor/gopkg.in/h2non/filetype.v1/filetype.go b/vendor/github.com/h2non/filetype/filetype.go similarity index 96% rename from vendor/gopkg.in/h2non/filetype.v1/filetype.go rename to vendor/github.com/h2non/filetype/filetype.go index 3753c03a234..933058c8532 100644 --- a/vendor/gopkg.in/h2non/filetype.v1/filetype.go +++ b/vendor/github.com/h2non/filetype/filetype.go @@ -3,8 +3,8 @@ package filetype import ( "errors" - "gopkg.in/h2non/filetype.v1/matchers" - "gopkg.in/h2non/filetype.v1/types" + "github.com/h2non/filetype/matchers" + "github.com/h2non/filetype/types" ) // Types stores a map of supported types diff --git a/vendor/github.com/h2non/filetype/go.mod b/vendor/github.com/h2non/filetype/go.mod new file mode 100644 index 00000000000..765d393f80c --- /dev/null +++ b/vendor/github.com/h2non/filetype/go.mod @@ -0,0 +1 @@ +module github.com/h2non/filetype diff --git a/vendor/gopkg.in/h2non/filetype.v1/kind.go b/vendor/github.com/h2non/filetype/kind.go similarity index 81% rename from vendor/gopkg.in/h2non/filetype.v1/kind.go rename to vendor/github.com/h2non/filetype/kind.go index 49397258c84..de8473507dc 100644 --- a/vendor/gopkg.in/h2non/filetype.v1/kind.go +++ b/vendor/github.com/h2non/filetype/kind.go @@ -1,8 +1,8 @@ package filetype import ( - "gopkg.in/h2non/filetype.v1/matchers" - "gopkg.in/h2non/filetype.v1/types" + "github.com/h2non/filetype/matchers" + "github.com/h2non/filetype/types" ) // Image tries to match a file as image type @@ -60,6 +60,17 @@ func IsArchive(buf []byte) bool { return kind != types.Unknown } +// Document tries to match a file as document type +func Document(buf []byte) (types.Type, error) { + return doMatchMap(buf, matchers.Document) +} + +// IsDocument checks if the given buffer is an document type +func IsDocument(buf []byte) bool { + kind, _ := Document(buf) + return kind != types.Unknown +} + func doMatchMap(buf []byte, machers matchers.Map) (types.Type, error) { kind := MatchMap(buf, machers) if kind != types.Unknown { diff --git a/vendor/gopkg.in/h2non/filetype.v1/match.go b/vendor/github.com/h2non/filetype/match.go similarity index 78% rename from vendor/gopkg.in/h2non/filetype.v1/match.go rename to vendor/github.com/h2non/filetype/match.go index 9b6e376f161..82cf8046845 100644 --- a/vendor/gopkg.in/h2non/filetype.v1/match.go +++ b/vendor/github.com/h2non/filetype/match.go @@ -4,13 +4,16 @@ import ( "io" "os" - "gopkg.in/h2non/filetype.v1/matchers" - "gopkg.in/h2non/filetype.v1/types" + "github.com/h2non/filetype/matchers" + "github.com/h2non/filetype/types" ) // Matchers is an alias to matchers.Matchers var Matchers = matchers.Matchers +// MatcherKeys is an alias to matchers.MatcherKeys +var MatcherKeys = &matchers.MatcherKeys + // NewMatcher is an alias to matchers.NewMatcher var NewMatcher = matchers.NewMatcher @@ -21,7 +24,8 @@ func Match(buf []byte) (types.Type, error) { return types.Unknown, ErrEmptyBuffer } - for _, checker := range Matchers { + for _, kind := range *MatcherKeys { + checker := Matchers[kind] match := checker(buf) if match != types.Unknown && match.Extension != "" { return match, nil @@ -49,7 +53,7 @@ func MatchFile(filepath string) (types.Type, error) { // MatchReader is convenient wrapper to Match() any Reader func MatchReader(reader io.Reader) (types.Type, error) { - buffer := make([]byte, 512) + buffer := make([]byte, 8192) // 8K makes msooxml tests happy and allows for expanded custom file checks _, err := reader.Read(buffer) if err != nil && err != io.EOF { @@ -70,7 +74,7 @@ func Matches(buf []byte) bool { return kind != types.Unknown } -// MatchMap performs a file matching againts a map of match functions +// MatchMap performs a file matching against a map of match functions func MatchMap(buf []byte, matchers matchers.Map) types.Type { for kind, matcher := range matchers { if matcher(buf) { @@ -80,7 +84,7 @@ func MatchMap(buf []byte, matchers matchers.Map) types.Type { return types.Unknown } -// MatchesMap is an alias to Matches() but using matching againts a map of match functions +// MatchesMap is an alias to Matches() but using matching against a map of match functions func MatchesMap(buf []byte, matchers matchers.Map) bool { return MatchMap(buf, matchers) != types.Unknown } diff --git a/vendor/github.com/h2non/filetype/matchers/application.go b/vendor/github.com/h2non/filetype/matchers/application.go new file mode 100644 index 00000000000..f482062d6c0 --- /dev/null +++ b/vendor/github.com/h2non/filetype/matchers/application.go @@ -0,0 +1,20 @@ +package matchers + +var ( + TypeWasm = newType("wasm", "application/wasm") +) + +var Application = Map{ + TypeWasm: Wasm, +} + +// Wasm detects a Web Assembly 1.0 filetype. +func Wasm(buf []byte) bool { + // WASM has starts with `\0asm`, followed by the version. + // http://webassembly.github.io/spec/core/binary/modules.html#binary-magic + return len(buf) >= 8 && + buf[0] == 0x00 && buf[1] == 0x61 && + buf[2] == 0x73 && buf[3] == 0x6D && + buf[4] == 0x01 && buf[5] == 0x00 && + buf[6] == 0x00 && buf[7] == 0x00 +} diff --git a/vendor/gopkg.in/h2non/filetype.v1/matchers/archive.go b/vendor/github.com/h2non/filetype/matchers/archive.go similarity index 96% rename from vendor/gopkg.in/h2non/filetype.v1/matchers/archive.go rename to vendor/github.com/h2non/filetype/matchers/archive.go index 9c1270ffbcc..0e88af144fe 100644 --- a/vendor/gopkg.in/h2non/filetype.v1/matchers/archive.go +++ b/vendor/github.com/h2non/filetype/matchers/archive.go @@ -25,6 +25,7 @@ var ( TypeLz = newType("lz", "application/x-lzip") TypeRpm = newType("rpm", "application/x-rpm") TypeElf = newType("elf", "application/x-executable") + TypeDcm = newType("dcm", "application/dicom") ) var Archive = Map{ @@ -52,6 +53,7 @@ var Archive = Map{ TypeLz: Lz, TypeRpm: Rpm, TypeElf: Elf, + TypeDcm: Dcm, } func Epub(buf []byte) bool { @@ -215,3 +217,9 @@ func Elf(buf []byte) bool { buf[0] == 0x7F && buf[1] == 0x45 && buf[2] == 0x4C && buf[3] == 0x46 } + +func Dcm(buf []byte) bool { + return len(buf) > 131 && + buf[128] == 0x44 && buf[129] == 0x49 && + buf[130] == 0x43 && buf[131] == 0x4D +} diff --git a/vendor/gopkg.in/h2non/filetype.v1/matchers/audio.go b/vendor/github.com/h2non/filetype/matchers/audio.go similarity index 89% rename from vendor/gopkg.in/h2non/filetype.v1/matchers/audio.go rename to vendor/github.com/h2non/filetype/matchers/audio.go index 7b27caf1ddc..6d532630af0 100644 --- a/vendor/gopkg.in/h2non/filetype.v1/matchers/audio.go +++ b/vendor/github.com/h2non/filetype/matchers/audio.go @@ -8,6 +8,7 @@ var ( TypeFlac = newType("flac", "audio/x-flac") TypeWav = newType("wav", "audio/x-wav") TypeAmr = newType("amr", "audio/amr") + TypeAac = newType("aac", "audio/aac") ) var Audio = Map{ @@ -18,6 +19,7 @@ var Audio = Map{ TypeFlac: Flac, TypeWav: Wav, TypeAmr: Amr, + TypeAac: Aac, } func Midi(buf []byte) bool { @@ -65,3 +67,9 @@ func Amr(buf []byte) bool { buf[2] == 0x41 && buf[3] == 0x4D && buf[4] == 0x52 && buf[5] == 0x0A } + +func Aac(buf []byte) bool { + return len(buf) > 1 && + ((buf[0] == 0xFF && buf[1] == 0xF1) || + (buf[0] == 0xFF && buf[1] == 0xF9)) +} diff --git a/vendor/github.com/h2non/filetype/matchers/document.go b/vendor/github.com/h2non/filetype/matchers/document.go new file mode 100644 index 00000000000..7e1ef851de5 --- /dev/null +++ b/vendor/github.com/h2non/filetype/matchers/document.go @@ -0,0 +1,182 @@ +package matchers + +import ( + "bytes" + "encoding/binary" +) + +var ( + TypeDoc = newType("doc", "application/msword") + TypeDocx = newType("docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document") + TypeXls = newType("xls", "application/vnd.ms-excel") + TypeXlsx = newType("xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") + TypePpt = newType("ppt", "application/vnd.ms-powerpoint") + TypePptx = newType("pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation") +) + +var Document = Map{ + TypeDoc: Doc, + TypeDocx: Docx, + TypeXls: Xls, + TypeXlsx: Xlsx, + TypePpt: Ppt, + TypePptx: Pptx, +} + +type docType int + +const ( + TYPE_DOC docType = iota + TYPE_DOCX + TYPE_XLS + TYPE_XLSX + TYPE_PPT + TYPE_PPTX + TYPE_OOXML +) + +func Doc(buf []byte) bool { + return len(buf) > 7 && + buf[0] == 0xD0 && buf[1] == 0xCF && + buf[2] == 0x11 && buf[3] == 0xE0 && + buf[4] == 0xA1 && buf[5] == 0xB1 && + buf[6] == 0x1A && buf[7] == 0xE1 +} + +func Docx(buf []byte) bool { + typ, ok := msooxml(buf) + return ok && typ == TYPE_DOCX +} + +func Xls(buf []byte) bool { + return len(buf) > 7 && + buf[0] == 0xD0 && buf[1] == 0xCF && + buf[2] == 0x11 && buf[3] == 0xE0 && + buf[4] == 0xA1 && buf[5] == 0xB1 && + buf[6] == 0x1A && buf[7] == 0xE1 +} + +func Xlsx(buf []byte) bool { + typ, ok := msooxml(buf) + return ok && typ == TYPE_XLSX +} + +func Ppt(buf []byte) bool { + return len(buf) > 7 && + buf[0] == 0xD0 && buf[1] == 0xCF && + buf[2] == 0x11 && buf[3] == 0xE0 && + buf[4] == 0xA1 && buf[5] == 0xB1 && + buf[6] == 0x1A && buf[7] == 0xE1 +} + +func Pptx(buf []byte) bool { + typ, ok := msooxml(buf) + return ok && typ == TYPE_PPTX +} + +func msooxml(buf []byte) (typ docType, found bool) { + signature := []byte{'P', 'K', 0x03, 0x04} + + // start by checking for ZIP local file header signature + if ok := compareBytes(buf, signature, 0); !ok { + return + } + + // make sure the first file is correct + if v, ok := checkMSOoml(buf, 0x1E); ok { + return v, ok + } + + if !compareBytes(buf, []byte("[Content_Types].xml"), 0x1E) && !compareBytes(buf, []byte("_rels/.rels"), 0x1E) { + return + } + + // skip to the second local file header + // since some documents include a 520-byte extra field following the file + // header, we need to scan for the next header + startOffset := int(binary.LittleEndian.Uint32(buf[18:22]) + 49) + idx := search(buf, startOffset, 6000) + if idx == -1 { + return + } + + // now skip to the *third* local file header; again, we need to scan due to a + // 520-byte extra field following the file header + startOffset += idx + 4 + 26 + idx = search(buf, startOffset, 6000) + if idx == -1 { + return + } + + // and check the subdirectory name to determine which type of OOXML + // file we have. Correct the mimetype with the registered ones: + // http://technet.microsoft.com/en-us/library/cc179224.aspx + startOffset += idx + 4 + 26 + if typ, ok := checkMSOoml(buf, startOffset); ok { + return typ, ok + } + + // OpenOffice/Libreoffice orders ZIP entry differently, so check the 4th file + startOffset += 26 + idx = search(buf, startOffset, 6000) + if idx == -1 { + return TYPE_OOXML, true + } + + startOffset += idx + 4 + 26 + if typ, ok := checkMSOoml(buf, startOffset); ok { + return typ, ok + } else { + return TYPE_OOXML, true + } +} + +func compareBytes(slice, subSlice []byte, startOffset int) bool { + sl := len(subSlice) + + if startOffset+sl > len(slice) { + return false + } + + s := slice[startOffset : startOffset+sl] + for i := range s { + if subSlice[i] != s[i] { + return false + } + } + + return true +} + +func checkMSOoml(buf []byte, offset int) (typ docType, ok bool) { + ok = true + + switch { + case compareBytes(buf, []byte("word/"), offset): + typ = TYPE_DOCX + case compareBytes(buf, []byte("ppt/"), offset): + typ = TYPE_PPTX + case compareBytes(buf, []byte("xl/"), offset): + typ = TYPE_XLSX + default: + ok = false + } + + return +} + +func search(buf []byte, start, rangeNum int) int { + length := len(buf) + end := start + rangeNum + signature := []byte{'P', 'K', 0x03, 0x04} + + if end > length { + end = length + } + + if start >= end { + return -1 + } + + return bytes.Index(buf[start:end], signature) +} diff --git a/vendor/gopkg.in/h2non/filetype.v1/matchers/font.go b/vendor/github.com/h2non/filetype/matchers/font.go similarity index 100% rename from vendor/gopkg.in/h2non/filetype.v1/matchers/font.go rename to vendor/github.com/h2non/filetype/matchers/font.go diff --git a/vendor/github.com/h2non/filetype/matchers/image.go b/vendor/github.com/h2non/filetype/matchers/image.go new file mode 100644 index 00000000000..5405477a693 --- /dev/null +++ b/vendor/github.com/h2non/filetype/matchers/image.go @@ -0,0 +1,133 @@ +package matchers + +import "github.com/h2non/filetype/matchers/isobmff" + +var ( + TypeJpeg = newType("jpg", "image/jpeg") + TypeJpeg2000 = newType("jp2", "image/jp2") + TypePng = newType("png", "image/png") + TypeGif = newType("gif", "image/gif") + TypeWebp = newType("webp", "image/webp") + TypeCR2 = newType("cr2", "image/x-canon-cr2") + TypeTiff = newType("tif", "image/tiff") + TypeBmp = newType("bmp", "image/bmp") + TypeJxr = newType("jxr", "image/vnd.ms-photo") + TypePsd = newType("psd", "image/vnd.adobe.photoshop") + TypeIco = newType("ico", "image/x-icon") + TypeHeif = newType("heif", "image/heif") +) + +var Image = Map{ + TypeJpeg: Jpeg, + TypeJpeg2000: Jpeg2000, + TypePng: Png, + TypeGif: Gif, + TypeWebp: Webp, + TypeCR2: CR2, + TypeTiff: Tiff, + TypeBmp: Bmp, + TypeJxr: Jxr, + TypePsd: Psd, + TypeIco: Ico, + TypeHeif: Heif, +} + +func Jpeg(buf []byte) bool { + return len(buf) > 2 && + buf[0] == 0xFF && + buf[1] == 0xD8 && + buf[2] == 0xFF +} + +func Jpeg2000(buf []byte) bool { + return len(buf) > 12 && + buf[0] == 0x0 && + buf[1] == 0x0 && + buf[2] == 0x0 && + buf[3] == 0xC && + buf[4] == 0x6A && + buf[5] == 0x50 && + buf[6] == 0x20 && + buf[7] == 0x20 && + buf[8] == 0xD && + buf[9] == 0xA && + buf[10] == 0x87 && + buf[11] == 0xA && + buf[12] == 0x0 +} + +func Png(buf []byte) bool { + return len(buf) > 3 && + buf[0] == 0x89 && buf[1] == 0x50 && + buf[2] == 0x4E && buf[3] == 0x47 +} + +func Gif(buf []byte) bool { + return len(buf) > 2 && + buf[0] == 0x47 && buf[1] == 0x49 && buf[2] == 0x46 +} + +func Webp(buf []byte) bool { + return len(buf) > 11 && + buf[8] == 0x57 && buf[9] == 0x45 && + buf[10] == 0x42 && buf[11] == 0x50 +} + +func CR2(buf []byte) bool { + return len(buf) > 9 && + ((buf[0] == 0x49 && buf[1] == 0x49 && buf[2] == 0x2A && buf[3] == 0x0) || + (buf[0] == 0x4D && buf[1] == 0x4D && buf[2] == 0x0 && buf[3] == 0x2A)) && + buf[8] == 0x43 && buf[9] == 0x52 +} + +func Tiff(buf []byte) bool { + return len(buf) > 3 && + ((buf[0] == 0x49 && buf[1] == 0x49 && buf[2] == 0x2A && buf[3] == 0x0) || + (buf[0] == 0x4D && buf[1] == 0x4D && buf[2] == 0x0 && buf[3] == 0x2A)) +} + +func Bmp(buf []byte) bool { + return len(buf) > 1 && + buf[0] == 0x42 && + buf[1] == 0x4D +} + +func Jxr(buf []byte) bool { + return len(buf) > 2 && + buf[0] == 0x49 && + buf[1] == 0x49 && + buf[2] == 0xBC +} + +func Psd(buf []byte) bool { + return len(buf) > 3 && + buf[0] == 0x38 && buf[1] == 0x42 && + buf[2] == 0x50 && buf[3] == 0x53 +} + +func Ico(buf []byte) bool { + return len(buf) > 3 && + buf[0] == 0x00 && buf[1] == 0x00 && + buf[2] == 0x01 && buf[3] == 0x00 +} + +func Heif(buf []byte) bool { + if !isobmff.IsISOBMFF(buf) { + return false + } + + majorBrand, _, compatibleBrands := isobmff.GetFtyp(buf) + if majorBrand == "heic" { + return true + } + + if majorBrand == "mif1" || majorBrand == "msf1" { + for _, compatibleBrand := range compatibleBrands { + if compatibleBrand == "heic" { + return true + } + } + } + + return false +} diff --git a/vendor/github.com/h2non/filetype/matchers/isobmff/isobmff.go b/vendor/github.com/h2non/filetype/matchers/isobmff/isobmff.go new file mode 100644 index 00000000000..963e80568ff --- /dev/null +++ b/vendor/github.com/h2non/filetype/matchers/isobmff/isobmff.go @@ -0,0 +1,31 @@ +package isobmff + +import "encoding/binary" + +// IsISOBMFF checks whether the given buffer represents ISO Base Media File Format data +func IsISOBMFF(buf []byte) bool { + if len(buf) < 16 || string(buf[4:8]) != "ftyp" { + return false + } + + if ftypLength := binary.BigEndian.Uint32(buf[0:4]); len(buf) < int(ftypLength) { + return false + } + + return true +} + +// GetFtyp returns the major brand, minor version and compatible brands of the ISO-BMFF data +func GetFtyp(buf []byte) (string, string, []string) { + ftypLength := binary.BigEndian.Uint32(buf[0:4]) + + majorBrand := string(buf[8:12]) + minorVersion := string(buf[12:16]) + + compatibleBrands := []string{} + for i := 16; i < int(ftypLength); i += 4 { + compatibleBrands = append(compatibleBrands, string(buf[i:i+4])) + } + + return majorBrand, minorVersion, compatibleBrands +} diff --git a/vendor/gopkg.in/h2non/filetype.v1/matchers/matchers.go b/vendor/github.com/h2non/filetype/matchers/matchers.go similarity index 70% rename from vendor/gopkg.in/h2non/filetype.v1/matchers/matchers.go rename to vendor/github.com/h2non/filetype/matchers/matchers.go index 4525c02a4bf..20d74d080d8 100644 --- a/vendor/gopkg.in/h2non/filetype.v1/matchers/matchers.go +++ b/vendor/github.com/h2non/filetype/matchers/matchers.go @@ -1,6 +1,8 @@ package matchers -import "gopkg.in/h2non/filetype.v1/types" +import ( + "github.com/h2non/filetype/types" +) // Internal shortcut to NewType var newType = types.NewType @@ -16,6 +18,7 @@ type TypeMatcher func([]byte) types.Type // Store registered file type matchers var Matchers = make(map[types.Type]TypeMatcher) +var MatcherKeys []types.Type // Create and register a new type matcher function func NewMatcher(kind types.Type, fn Matcher) TypeMatcher { @@ -27,10 +30,13 @@ func NewMatcher(kind types.Type, fn Matcher) TypeMatcher { } Matchers[kind] = matcher + // prepend here so any user defined matchers get added first + MatcherKeys = append([]types.Type{kind}, MatcherKeys...) return matcher } func register(matchers ...Map) { + MatcherKeys = MatcherKeys[:0] for _, m := range matchers { for kind, matcher := range m { NewMatcher(kind, matcher) @@ -40,5 +46,6 @@ func register(matchers ...Map) { func init() { // Arguments order is intentional - register(Image, Video, Audio, Font, Document, Archive) + // Archive files will be checked last due to prepend above in func NewMatcher + register(Archive, Document, Font, Audio, Video, Image, Application) } diff --git a/vendor/gopkg.in/h2non/filetype.v1/matchers/video.go b/vendor/github.com/h2non/filetype/matchers/video.go similarity index 60% rename from vendor/gopkg.in/h2non/filetype.v1/matchers/video.go rename to vendor/github.com/h2non/filetype/matchers/video.go index 9b8350b3821..2c78038102b 100644 --- a/vendor/gopkg.in/h2non/filetype.v1/matchers/video.go +++ b/vendor/github.com/h2non/filetype/matchers/video.go @@ -26,8 +26,6 @@ var Video = Map{ func M4v(buf []byte) bool { return len(buf) > 10 && - buf[0] == 0x0 && buf[1] == 0x0 && - buf[2] == 0x0 && buf[3] == 0x1C && buf[4] == 0x66 && buf[5] == 0x74 && buf[6] == 0x79 && buf[7] == 0x70 && buf[8] == 0x4D && buf[9] == 0x34 && @@ -58,11 +56,13 @@ func Webm(buf []byte) bool { } func Mov(buf []byte) bool { - return len(buf) > 7 && - buf[0] == 0x0 && buf[1] == 0x0 && + return len(buf) > 15 && ((buf[0] == 0x0 && buf[1] == 0x0 && buf[2] == 0x0 && buf[3] == 0x14 && buf[4] == 0x66 && buf[5] == 0x74 && - buf[6] == 0x79 && buf[7] == 0x70 + buf[6] == 0x79 && buf[7] == 0x70) || + (buf[4] == 0x6d && buf[5] == 0x6f && buf[6] == 0x6f && buf[7] == 0x76) || + (buf[4] == 0x6d && buf[5] == 0x64 && buf[6] == 0x61 && buf[7] == 0x74) || + (buf[12] == 0x6d && buf[13] == 0x64 && buf[14] == 0x61 && buf[15] == 0x74)) } func Avi(buf []byte) bool { @@ -99,22 +99,31 @@ func Mp4(buf []byte) bool { return len(buf) > 11 && (buf[4] == 'f' && buf[5] == 't' && buf[6] == 'y' && buf[7] == 'p') && ((buf[8] == 'a' && buf[9] == 'v' && buf[10] == 'c' && buf[11] == '1') || + (buf[8] == 'd' && buf[9] == 'a' && buf[10] == 's' && buf[11] == 'h') || (buf[8] == 'i' && buf[9] == 's' && buf[10] == 'o' && buf[11] == '2') || + (buf[8] == 'i' && buf[9] == 's' && buf[10] == 'o' && buf[11] == '3') || + (buf[8] == 'i' && buf[9] == 's' && buf[10] == 'o' && buf[11] == '4') || + (buf[8] == 'i' && buf[9] == 's' && buf[10] == 'o' && buf[11] == '5') || + (buf[8] == 'i' && buf[9] == 's' && buf[10] == 'o' && buf[11] == '6') || (buf[8] == 'i' && buf[9] == 's' && buf[10] == 'o' && buf[11] == 'm') || (buf[8] == 'm' && buf[9] == 'm' && buf[10] == 'p' && buf[11] == '4') || (buf[8] == 'm' && buf[9] == 'p' && buf[10] == '4' && buf[11] == '1') || (buf[8] == 'm' && buf[9] == 'p' && buf[10] == '4' && buf[11] == '2') || + (buf[8] == 'm' && buf[9] == 'p' && buf[10] == '4' && buf[11] == 'v') || (buf[8] == 'm' && buf[9] == 'p' && buf[10] == '7' && buf[11] == '1') || - (buf[8] == 'm' && buf[9] == 's' && buf[10] == 'n' && buf[11] == 'v') || - (buf[8] == 'n' && buf[9] == 'd' && buf[10] == 'a' && buf[11] == 's') || - (buf[8] == 'n' && buf[9] == 'd' && buf[10] == 's' && buf[11] == 'c') || - (buf[8] == 'n' && buf[9] == 'd' && buf[10] == 's' && buf[11] == 'h') || - (buf[8] == 'n' && buf[9] == 'd' && buf[10] == 's' && buf[11] == 'm') || - (buf[8] == 'n' && buf[9] == 'd' && buf[10] == 's' && buf[11] == 'p') || - (buf[8] == 'n' && buf[9] == 'd' && buf[10] == 's' && buf[11] == 's') || - (buf[8] == 'n' && buf[9] == 'd' && buf[10] == 'x' && buf[11] == 'c') || - (buf[8] == 'n' && buf[9] == 'd' && buf[10] == 'x' && buf[11] == 'h') || - (buf[8] == 'n' && buf[9] == 'd' && buf[10] == 'x' && buf[11] == 'm') || - (buf[8] == 'n' && buf[9] == 'd' && buf[10] == 'x' && buf[11] == 'p') || - (buf[8] == 'n' && buf[9] == 'd' && buf[10] == 'x' && buf[11] == 's')) + (buf[8] == 'M' && buf[9] == 'S' && buf[10] == 'N' && buf[11] == 'V') || + (buf[8] == 'N' && buf[9] == 'D' && buf[10] == 'A' && buf[11] == 'S') || + (buf[8] == 'N' && buf[9] == 'D' && buf[10] == 'S' && buf[11] == 'C') || + (buf[8] == 'N' && buf[9] == 'S' && buf[10] == 'D' && buf[11] == 'C') || + (buf[8] == 'N' && buf[9] == 'D' && buf[10] == 'S' && buf[11] == 'H') || + (buf[8] == 'N' && buf[9] == 'D' && buf[10] == 'S' && buf[11] == 'M') || + (buf[8] == 'N' && buf[9] == 'D' && buf[10] == 'S' && buf[11] == 'P') || + (buf[8] == 'N' && buf[9] == 'D' && buf[10] == 'S' && buf[11] == 'S') || + (buf[8] == 'N' && buf[9] == 'D' && buf[10] == 'X' && buf[11] == 'C') || + (buf[8] == 'N' && buf[9] == 'D' && buf[10] == 'X' && buf[11] == 'H') || + (buf[8] == 'N' && buf[9] == 'D' && buf[10] == 'X' && buf[11] == 'M') || + (buf[8] == 'N' && buf[9] == 'D' && buf[10] == 'X' && buf[11] == 'P') || + (buf[8] == 'N' && buf[9] == 'D' && buf[10] == 'X' && buf[11] == 'S') || + (buf[8] == 'F' && buf[9] == '4' && buf[10] == 'V' && buf[11] == ' ') || + (buf[8] == 'F' && buf[9] == '4' && buf[10] == 'P' && buf[11] == ' ')) } diff --git a/vendor/gopkg.in/h2non/filetype.v1/types/defaults.go b/vendor/github.com/h2non/filetype/types/defaults.go similarity index 68% rename from vendor/gopkg.in/h2non/filetype.v1/types/defaults.go rename to vendor/github.com/h2non/filetype/types/defaults.go index bb1ea62ee6d..0d985a05d68 100644 --- a/vendor/gopkg.in/h2non/filetype.v1/types/defaults.go +++ b/vendor/github.com/h2non/filetype/types/defaults.go @@ -1,4 +1,4 @@ package types -// Unkown default type +// Unknown default type var Unknown = NewType("unknown", "") diff --git a/vendor/gopkg.in/h2non/filetype.v1/types/mime.go b/vendor/github.com/h2non/filetype/types/mime.go similarity index 100% rename from vendor/gopkg.in/h2non/filetype.v1/types/mime.go rename to vendor/github.com/h2non/filetype/types/mime.go diff --git a/vendor/gopkg.in/h2non/filetype.v1/types/split.go b/vendor/github.com/h2non/filetype/types/split.go similarity index 100% rename from vendor/gopkg.in/h2non/filetype.v1/types/split.go rename to vendor/github.com/h2non/filetype/types/split.go diff --git a/vendor/gopkg.in/h2non/filetype.v1/types/type.go b/vendor/github.com/h2non/filetype/types/type.go similarity index 100% rename from vendor/gopkg.in/h2non/filetype.v1/types/type.go rename to vendor/github.com/h2non/filetype/types/type.go diff --git a/vendor/gopkg.in/h2non/filetype.v1/types/types.go b/vendor/github.com/h2non/filetype/types/types.go similarity index 100% rename from vendor/gopkg.in/h2non/filetype.v1/types/types.go rename to vendor/github.com/h2non/filetype/types/types.go diff --git a/vendor/gopkg.in/h2non/filetype.v1/version.go b/vendor/github.com/h2non/filetype/version.go similarity index 73% rename from vendor/gopkg.in/h2non/filetype.v1/version.go rename to vendor/github.com/h2non/filetype/version.go index 4ef1e52ddbe..be84ec228e6 100644 --- a/vendor/gopkg.in/h2non/filetype.v1/version.go +++ b/vendor/github.com/h2non/filetype/version.go @@ -1,4 +1,4 @@ package filetype // Version exposes the current package version. -const Version = "1.0.5" +const Version = "1.0.8" diff --git a/vendor/gopkg.in/h2non/filetype.v1/History.md b/vendor/gopkg.in/h2non/filetype.v1/History.md deleted file mode 100644 index a33f4e7406c..00000000000 --- a/vendor/gopkg.in/h2non/filetype.v1/History.md +++ /dev/null @@ -1,47 +0,0 @@ - -v1.0.5 / 2017-12-12 -=================== - - * Merge pull request #30 from RangelReale/fix_mp4 - * Fix duplicated item in mp4 fix - * Fix MP4 matcher, with information from http://www.file-recovery.com/mp4-signature-format.htm - * Merge pull request #28 from ikovic/master - * Updated file header example. - -v1.0.4 / 2017-11-29 -=================== - - * fix: tests and document types matchers - * refactor(docs): remove codesponsor - * Merge pull request #26 from bienkma/master - * Add support check file type: .doc, .docx, .pptx, .ppt, .xls, .xlsx - * feat(docs): add code sponsor banner - * feat(travis): add go 1.9 - * Merge pull request #24 from strazzere/patch-1 - * Fix typo in unknown - -v1.0.3 / 2017-08-03 -=================== - - * Merge pull request #21 from elemeta/master - * Add Elf file as supported matcher archive type - -v1.0.2 / 2017-07-26 -=================== - - * Merge pull request #20 from marshyski/master - * Added RedHat RPM as supported matcher archive type - * Merge pull request #19 from nlamirault/patch-1 - * Fix typo in documentation - -v1.0.1 / 2017-02-24 -=================== - - * Merge pull request #18 from Impyy/enable-webm - * Enable the webm matcher - * feat(docs): add Go version badge - -1.0.0 / 2016-12-11 -================== - -- Initial stable version (v1.0.0). diff --git a/vendor/gopkg.in/h2non/filetype.v1/matchers/document.go b/vendor/gopkg.in/h2non/filetype.v1/matchers/document.go deleted file mode 100644 index cc5ded202b7..00000000000 --- a/vendor/gopkg.in/h2non/filetype.v1/matchers/document.go +++ /dev/null @@ -1,66 +0,0 @@ -package matchers - -import "bytes" - -var ( - TypeDoc = newType("doc", "application/msword") - TypeDocx = newType("docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document") - TypeXls = newType("xls", "application/vnd.ms-excel") - TypeXlsx = newType("xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") - TypePpt = newType("ppt", "application/vnd.ms-powerpoint") - TypePptx = newType("pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation") -) - -var Document = Map{ - TypeDoc: Doc, - TypeDocx: Docx, - TypeXls: Xls, - TypeXlsx: Xlsx, - TypePpt: Ppt, - TypePptx: Pptx, -} - -func Doc(buf []byte) bool { - return len(buf) > 7 && - buf[0] == 0xD0 && buf[1] == 0xCF && - buf[2] == 0x11 && buf[3] == 0xE0 && - buf[4] == 0xA1 && buf[5] == 0xB1 && - buf[6] == 0x1A && buf[7] == 0xE1 -} - -func Docx(buf []byte) bool { - return len(buf) > 3 && - buf[0] == 0x50 && buf[1] == 0x4B && - buf[2] == 0x03 && buf[3] == 0x04 && - bytes.Contains(buf[:256], []byte(TypeDocx.MIME.Value)) -} - -func Xls(buf []byte) bool { - return len(buf) > 7 && - buf[0] == 0xD0 && buf[1] == 0xCF && - buf[2] == 0x11 && buf[3] == 0xE0 && - buf[4] == 0xA1 && buf[5] == 0xB1 && - buf[6] == 0x1A && buf[7] == 0xE1 -} - -func Xlsx(buf []byte) bool { - return len(buf) > 3 && - buf[0] == 0x50 && buf[1] == 0x4B && - buf[2] == 0x03 && buf[3] == 0x04 && - bytes.Contains(buf[:256], []byte(TypeXlsx.MIME.Value)) -} - -func Ppt(buf []byte) bool { - return len(buf) > 7 && - buf[0] == 0xD0 && buf[1] == 0xCF && - buf[2] == 0x11 && buf[3] == 0xE0 && - buf[4] == 0xA1 && buf[5] == 0xB1 && - buf[6] == 0x1A && buf[7] == 0xE1 -} - -func Pptx(buf []byte) bool { - return len(buf) > 3 && - buf[0] == 0x50 && buf[1] == 0x4B && - buf[2] == 0x07 && buf[3] == 0x08 && - bytes.Contains(buf[:256], []byte(TypePptx.MIME.Value)) -} diff --git a/vendor/gopkg.in/h2non/filetype.v1/matchers/image.go b/vendor/gopkg.in/h2non/filetype.v1/matchers/image.go deleted file mode 100644 index bc3378d6c61..00000000000 --- a/vendor/gopkg.in/h2non/filetype.v1/matchers/image.go +++ /dev/null @@ -1,89 +0,0 @@ -package matchers - -var ( - TypeJpeg = newType("jpg", "image/jpeg") - TypePng = newType("png", "image/png") - TypeGif = newType("gif", "image/gif") - TypeWebp = newType("webp", "image/webp") - TypeCR2 = newType("cr2", "image/x-canon-cr2") - TypeTiff = newType("tif", "image/tiff") - TypeBmp = newType("bmp", "image/bmp") - TypeJxr = newType("jxr", "image/vnd.ms-photo") - TypePsd = newType("psd", "image/vnd.adobe.photoshop") - TypeIco = newType("ico", "image/x-icon") -) - -var Image = Map{ - TypeJpeg: Jpeg, - TypePng: Png, - TypeGif: Gif, - TypeWebp: Webp, - TypeCR2: CR2, - TypeTiff: Tiff, - TypeBmp: Bmp, - TypeJxr: Jxr, - TypePsd: Psd, - TypeIco: Ico, -} - -func Jpeg(buf []byte) bool { - return len(buf) > 2 && - buf[0] == 0xFF && - buf[1] == 0xD8 && - buf[2] == 0xFF -} - -func Png(buf []byte) bool { - return len(buf) > 3 && - buf[0] == 0x89 && buf[1] == 0x50 && - buf[2] == 0x4E && buf[3] == 0x47 -} - -func Gif(buf []byte) bool { - return len(buf) > 2 && - buf[0] == 0x47 && buf[1] == 0x49 && buf[2] == 0x46 -} - -func Webp(buf []byte) bool { - return len(buf) > 11 && - buf[8] == 0x57 && buf[9] == 0x45 && - buf[10] == 0x42 && buf[11] == 0x50 -} - -func CR2(buf []byte) bool { - return len(buf) > 9 && - ((buf[0] == 0x49 && buf[1] == 0x49 && buf[2] == 0x2A && buf[3] == 0x0) || - (buf[0] == 0x4D && buf[1] == 0x4D && buf[2] == 0x0 && buf[3] == 0x2A)) && - buf[8] == 0x43 && buf[9] == 0x52 -} - -func Tiff(buf []byte) bool { - return len(buf) > 3 && - ((buf[0] == 0x49 && buf[1] == 0x49 && buf[2] == 0x2A && buf[3] == 0x0) || - (buf[0] == 0x4D && buf[1] == 0x4D && buf[2] == 0x0 && buf[3] == 0x2A)) -} - -func Bmp(buf []byte) bool { - return len(buf) > 1 && - buf[0] == 0x42 && - buf[1] == 0x4D -} - -func Jxr(buf []byte) bool { - return len(buf) > 2 && - buf[0] == 0x49 && - buf[1] == 0x49 && - buf[2] == 0xBC -} - -func Psd(buf []byte) bool { - return len(buf) > 3 && - buf[0] == 0x38 && buf[1] == 0x42 && - buf[2] == 0x50 && buf[3] == 0x53 -} - -func Ico(buf []byte) bool { - return len(buf) > 3 && - buf[0] == 0x00 && buf[1] == 0x00 && - buf[2] == 0x01 && buf[3] == 0x00 -}