From 99fb6d574af2a64698591bda01e6f251e178423d Mon Sep 17 00:00:00 2001 From: Mauri de Souza Meneguzzo Date: Wed, 26 Jul 2023 19:12:10 -0300 Subject: [PATCH 1/6] archive/tar: add AddFS method to Writer The method AddFS can be used to add the contents of a fs.FS filesystem to a tar archive. This method walks the directory tree starting at the root of the filesystem and adds each file to the archive. Fixes: #58000 --- api/next/58000.txt | 1 + src/archive/tar/writer.go | 34 ++++++++++++++++++++++++++++++++ src/archive/tar/writer_test.go | 36 ++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+) create mode 100644 api/next/58000.txt diff --git a/api/next/58000.txt b/api/next/58000.txt new file mode 100644 index 00000000000000..b7595e84ac577c --- /dev/null +++ b/api/next/58000.txt @@ -0,0 +1 @@ +pkg archive/tar, method (*Writer) AddFS(fsys fs.FS) error #58000 \ No newline at end of file diff --git a/src/archive/tar/writer.go b/src/archive/tar/writer.go index 1c95f0738a864f..d7d9ca479437af 100644 --- a/src/archive/tar/writer.go +++ b/src/archive/tar/writer.go @@ -7,6 +7,7 @@ package tar import ( "fmt" "io" + "io/fs" "path" "sort" "strings" @@ -403,6 +404,39 @@ func (tw *Writer) writeRawHeader(blk *block, size int64, flag byte) error { return nil } +// AddFS adds the files from fs.FS to the archive. +// It walks the directory tree starting at the root of the filesystem +// adding each file to the tar archive while maintaining the directory structure. +func (tw *Writer) AddFS(fsys fs.FS) error { + return fs.WalkDir(fsys, ".", func(name string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + info, err := d.Info() + if err != nil { + return err + } + h, err := FileInfoHeader(info, "") + if err != nil { + return err + } + h.Name = name + if err := tw.WriteHeader(h); err != nil { + return err + } + f, err := fsys.Open(name) + if err != nil { + return err + } + defer f.Close() + _, err = io.Copy(tw, f) + return err + }) +} + // splitUSTARPath splits a path according to USTAR prefix and suffix rules. // If the path is not splittable, then it will return ("", "", false). func splitUSTARPath(name string) (prefix, suffix string, ok bool) { diff --git a/src/archive/tar/writer_test.go b/src/archive/tar/writer_test.go index f6d75c58032305..0ca3ce49b8ebe3 100644 --- a/src/archive/tar/writer_test.go +++ b/src/archive/tar/writer_test.go @@ -15,6 +15,7 @@ import ( "sort" "strings" "testing" + "testing/fstest" "testing/iotest" "time" ) @@ -1333,3 +1334,38 @@ func TestFileWriter(t *testing.T) { } } } + +func TestWriterAddFs(t *testing.T) { + expectedFiles := []string{ + "file.go", + "subfolder/another.go", + } + fsys := fstest.MapFS{ + "file.go": {}, + "subfolder/another.go": {}, + } + var buf bytes.Buffer + tw := NewWriter(&buf) + if err := tw.AddFS(fsys); err != nil { + t.Fatal(err) + } + + // Test that we can get the files back from the archive + tr := NewReader(&buf) + var foundFiles []string + for { + hdr, err := tr.Next() + if err == io.EOF { + break // End of archive + } + if err != nil { + t.Fatal(err) + } + foundFiles = append(foundFiles, hdr.Name) + } + + if !reflect.DeepEqual(expectedFiles, foundFiles) { + t.Fatalf("got %+v, want %+v", + foundFiles, expectedFiles) + } +} From d01fe22f614d4056c403751abb81efc7596dad22 Mon Sep 17 00:00:00 2001 From: Mauri de Souza Meneguzzo Date: Wed, 26 Jul 2023 20:49:16 -0300 Subject: [PATCH 2/6] simplify test --- src/archive/tar/writer_test.go | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/src/archive/tar/writer_test.go b/src/archive/tar/writer_test.go index 0ca3ce49b8ebe3..f77841dd6284fa 100644 --- a/src/archive/tar/writer_test.go +++ b/src/archive/tar/writer_test.go @@ -1336,13 +1336,9 @@ func TestFileWriter(t *testing.T) { } func TestWriterAddFs(t *testing.T) { - expectedFiles := []string{ - "file.go", - "subfolder/another.go", - } fsys := fstest.MapFS{ - "file.go": {}, - "subfolder/another.go": {}, + "file.go": {Data: []byte("hello")}, + "subfolder/another.go": {Data: []byte("world")}, } var buf bytes.Buffer tw := NewWriter(&buf) @@ -1352,7 +1348,7 @@ func TestWriterAddFs(t *testing.T) { // Test that we can get the files back from the archive tr := NewReader(&buf) - var foundFiles []string + foundFiles := make(map[string][]byte) for { hdr, err := tr.Next() if err == io.EOF { @@ -1361,11 +1357,22 @@ func TestWriterAddFs(t *testing.T) { if err != nil { t.Fatal(err) } - foundFiles = append(foundFiles, hdr.Name) + + data := make([]byte, hdr.Size) + _, err = tr.Read(data) + foundFiles[hdr.Name] = data } - if !reflect.DeepEqual(expectedFiles, foundFiles) { - t.Fatalf("got %+v, want %+v", - foundFiles, expectedFiles) + for name, file := range fsys { + got, ok := foundFiles[name] + if !ok { + t.Fatalf("got filename %s, want %s", + got, name) + } + + if !reflect.DeepEqual(foundFiles[name], file.Data) { + t.Fatalf("got file content %#v, want %#v", + foundFiles[name], file.Data) + } } } From 9951132da58d0e05c715f9fdccc3d9c5856fd015 Mon Sep 17 00:00:00 2001 From: Mauri de Souza Meneguzzo Date: Thu, 27 Jul 2023 19:53:16 -0300 Subject: [PATCH 3/6] fix api signature --- api/next/58000.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/next/58000.txt b/api/next/58000.txt index b7595e84ac577c..de8292466d78c4 100644 --- a/api/next/58000.txt +++ b/api/next/58000.txt @@ -1 +1 @@ -pkg archive/tar, method (*Writer) AddFS(fsys fs.FS) error #58000 \ No newline at end of file +pkg archive/tar, method (*Writer) AddFS(fs.FS) error #58000 \ No newline at end of file From 842df81696bd9e1d4d33d4a1df623213b8116259 Mon Sep 17 00:00:00 2001 From: Mauri de Souza Meneguzzo Date: Thu, 27 Jul 2023 20:10:39 -0300 Subject: [PATCH 4/6] add newline to api sig --- api/next/58000.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/next/58000.txt b/api/next/58000.txt index de8292466d78c4..94db9637cbd26f 100644 --- a/api/next/58000.txt +++ b/api/next/58000.txt @@ -1 +1 @@ -pkg archive/tar, method (*Writer) AddFS(fs.FS) error #58000 \ No newline at end of file +pkg archive/tar, method (*Writer) AddFS(fs.FS) error #58000 From 21ed9e1e795e237d41e016707a71691c708891fb Mon Sep 17 00:00:00 2001 From: Mauri de Souza Meneguzzo Date: Sat, 29 Jul 2023 18:35:08 -0300 Subject: [PATCH 5/6] refactor test, handle err properly --- src/archive/tar/writer_test.go | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/src/archive/tar/writer_test.go b/src/archive/tar/writer_test.go index f77841dd6284fa..9887f8a0da9995 100644 --- a/src/archive/tar/writer_test.go +++ b/src/archive/tar/writer_test.go @@ -1348,8 +1348,8 @@ func TestWriterAddFs(t *testing.T) { // Test that we can get the files back from the archive tr := NewReader(&buf) - foundFiles := make(map[string][]byte) - for { + + for name, file := range fsys { hdr, err := tr.Next() if err == io.EOF { break // End of archive @@ -1359,20 +1359,19 @@ func TestWriterAddFs(t *testing.T) { } data := make([]byte, hdr.Size) - _, err = tr.Read(data) - foundFiles[hdr.Name] = data - } + _, err = io.ReadFull(tr, data) + if err != nil { + t.Fatal(err) + } - for name, file := range fsys { - got, ok := foundFiles[name] - if !ok { - t.Fatalf("got filename %s, want %s", - got, name) + if name != hdr.Name { + t.Fatalf("got filename %v, want %v", + name, hdr.Name) } - if !reflect.DeepEqual(foundFiles[name], file.Data) { - t.Fatalf("got file content %#v, want %#v", - foundFiles[name], file.Data) + if string(data) != string(file.Data) { + t.Fatalf("got file content %v, want %v", + data, file.Data) } } } From 11a62bfd7ebc7ca93b1832da15018e6805c92520 Mon Sep 17 00:00:00 2001 From: Mauri de Souza Meneguzzo Date: Wed, 9 Aug 2023 21:00:10 -0300 Subject: [PATCH 6/6] add fix and test for non-regular files --- src/archive/tar/writer.go | 5 +++++ src/archive/tar/writer_test.go | 40 +++++++++++++++++++++++++++------- 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/src/archive/tar/writer.go b/src/archive/tar/writer.go index d7d9ca479437af..d805e266d0594c 100644 --- a/src/archive/tar/writer.go +++ b/src/archive/tar/writer.go @@ -5,6 +5,7 @@ package tar import ( + "errors" "fmt" "io" "io/fs" @@ -419,6 +420,10 @@ func (tw *Writer) AddFS(fsys fs.FS) error { if err != nil { return err } + // TODO(#49580): Handle symlinks when fs.ReadLinkFS is available. + if !info.Mode().IsRegular() { + return errors.New("tar: cannot add non-regular file") + } h, err := FileInfoHeader(info, "") if err != nil { return err diff --git a/src/archive/tar/writer_test.go b/src/archive/tar/writer_test.go index 9887f8a0da9995..a9936d6cd526fe 100644 --- a/src/archive/tar/writer_test.go +++ b/src/archive/tar/writer_test.go @@ -9,6 +9,7 @@ import ( "encoding/hex" "errors" "io" + "io/fs" "os" "path" "reflect" @@ -1335,7 +1336,7 @@ func TestFileWriter(t *testing.T) { } } -func TestWriterAddFs(t *testing.T) { +func TestWriterAddFS(t *testing.T) { fsys := fstest.MapFS{ "file.go": {Data: []byte("hello")}, "subfolder/another.go": {Data: []byte("world")}, @@ -1349,7 +1350,18 @@ func TestWriterAddFs(t *testing.T) { // Test that we can get the files back from the archive tr := NewReader(&buf) - for name, file := range fsys { + entries, err := fsys.ReadDir(".") + if err != nil { + t.Fatal(err) + } + + var curfname string + for _, entry := range entries { + curfname = entry.Name() + if entry.IsDir() { + curfname += "/" + continue + } hdr, err := tr.Next() if err == io.EOF { break // End of archive @@ -1358,20 +1370,32 @@ func TestWriterAddFs(t *testing.T) { t.Fatal(err) } - data := make([]byte, hdr.Size) - _, err = io.ReadFull(tr, data) + data, err := io.ReadAll(tr) if err != nil { t.Fatal(err) } - if name != hdr.Name { + if hdr.Name != curfname { t.Fatalf("got filename %v, want %v", - name, hdr.Name) + curfname, hdr.Name) } - if string(data) != string(file.Data) { + origdata := fsys[curfname].Data + if string(data) != string(origdata) { t.Fatalf("got file content %v, want %v", - data, file.Data) + data, origdata) } } } + +func TestWriterAddFSNonRegularFiles(t *testing.T) { + fsys := fstest.MapFS{ + "device": {Data: []byte("hello"), Mode: 0755 | fs.ModeDevice}, + "symlink": {Data: []byte("world"), Mode: 0755 | fs.ModeSymlink}, + } + var buf bytes.Buffer + tw := NewWriter(&buf) + if err := tw.AddFS(fsys); err == nil { + t.Fatal("expected error, got nil") + } +}