diff --git a/cmd/root.go b/cmd/root.go index b22678c6..d1b25231 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -167,6 +167,29 @@ func getImageForName(imageName string) (pkgutil.Image, error) { // TODO(nkubala): implement caching // create tempdir and extract fs into it + var layers []pkgutil.Layer + if includeLayers() { + imgLayers, err := img.Layers() + if err != nil { + return pkgutil.Image{}, err + } + for _, layer := range imgLayers { + path, err := ioutil.TempDir("", strings.Replace(imageName, "/", "", -1)) + if err != nil { + return pkgutil.Image{ + Layers: layers, + }, err + } + if err := pkgutil.GetFileSystemForLayer(layer, path, nil); err != nil { + return pkgutil.Image{ + Layers: layers, + }, err + } + layers = append(layers, pkgutil.Layer{ + FSPath: path, + }) + } + } path, err := ioutil.TempDir("", strings.Replace(imageName, "/", "", -1)) if err != nil { return pkgutil.Image{}, err @@ -174,15 +197,26 @@ func getImageForName(imageName string) (pkgutil.Image, error) { if err := pkgutil.GetFileSystemForImage(img, path, nil); err != nil { return pkgutil.Image{ FSPath: path, + Layers: layers, }, err } return pkgutil.Image{ Image: img, Source: imageName, FSPath: path, + Layers: layers, }, nil } +func includeLayers() bool { + for _, t := range types { + if t == "layer" { + return true + } + } + return false +} + func init() { RootCmd.PersistentFlags().StringVarP(&LogLevel, "verbosity", "v", "warning", "This flag controls the verbosity of container-diff.") RootCmd.PersistentFlags().StringVarP(&format, "format", "", "", "Format to output diff in.") diff --git a/differs/differs.go b/differs/differs.go index e12a01d8..1658e540 100644 --- a/differs/differs.go +++ b/differs/differs.go @@ -45,6 +45,7 @@ var Analyzers = map[string]Analyzer{ "history": HistoryAnalyzer{}, "metadata": MetadataAnalyzer{}, "file": FileAnalyzer{}, + "layer": FileLayerAnalyzer{}, "apt": AptAnalyzer{}, "rpm": RPMAnalyzer{}, "pip": PipAnalyzer{}, diff --git a/differs/file_diff.go b/differs/file_diff.go index 5dae3db2..500d1d13 100644 --- a/differs/file_diff.go +++ b/differs/file_diff.go @@ -19,6 +19,7 @@ package differs import ( pkgutil "github.com/GoogleContainerTools/container-diff/pkg/util" "github.com/GoogleContainerTools/container-diff/util" + "github.com/sirupsen/logrus" ) type FileAnalyzer struct { @@ -30,7 +31,7 @@ func (a FileAnalyzer) Name() string { // FileDiff diffs two packages and compares their contents func (a FileAnalyzer) Diff(image1, image2 pkgutil.Image) (util.Result, error) { - diff, err := diffImageFiles(image1, image2) + diff, err := diffImageFiles(image1.FSPath, image2.FSPath) return &util.DirDiffResult{ Image1: image1.Source, Image2: image2.Source, @@ -53,10 +54,7 @@ func (a FileAnalyzer) Analyze(image pkgutil.Image) (util.Result, error) { return &result, err } -func diffImageFiles(image1, image2 pkgutil.Image) (util.DirDiff, error) { - img1 := image1.FSPath - img2 := image2.FSPath - +func diffImageFiles(img1, img2 string) (util.DirDiff, error) { var diff util.DirDiff img1Dir, err := pkgutil.GetDirectory(img1, true) @@ -71,3 +69,63 @@ func diffImageFiles(image1, image2 pkgutil.Image) (util.DirDiff, error) { diff, _ = util.DiffDirectory(img1Dir, img2Dir) return diff, nil } + +type FileLayerAnalyzer struct { +} + +func (a FileLayerAnalyzer) Name() string { + return "FileLayerAnalyzer" +} + +// FileDiff diffs two packages and compares their contents +func (a FileLayerAnalyzer) Diff(image1, image2 pkgutil.Image) (util.Result, error) { + var dirDiffs []util.DirDiff + + // Go through each layer of the first image... + for index, layer := range image1.Layers { + if index >= len(image2.Layers) { + continue + } + // ...else, diff as usual + layer2 := image2.Layers[index] + diff, err := diffImageFiles(layer.FSPath, layer2.FSPath) + if err != nil { + return &util.MultipleDirDiffResult{}, err + } + dirDiffs = append(dirDiffs, diff) + } + + // check if there are any additional layers in either image + if len(image1.Layers) != len(image2.Layers) { + if len(image1.Layers) > len(image2.Layers) { + logrus.Infof("%s has additional layers, please use container-diff analyze to view the files in these layers", image1.Source) + } else { + logrus.Infof("%s has additional layers, please use container-diff analyze to view the files in these layers", image2.Source) + } + } + return &util.MultipleDirDiffResult{ + Image1: image1.Source, + Image2: image2.Source, + DiffType: "FileLayer", + Diff: util.MultipleDirDiff{ + DirDiffs: dirDiffs, + }, + }, nil +} + +func (a FileLayerAnalyzer) Analyze(image pkgutil.Image) (util.Result, error) { + var directoryEntries [][]pkgutil.DirectoryEntry + for _, layer := range image.Layers { + layerDir, err := pkgutil.GetDirectory(layer.FSPath, true) + if err != nil { + return util.FileLayerAnalyzeResult{}, err + } + directoryEntries = append(directoryEntries, pkgutil.GetDirectoryEntries(layerDir)) + } + + return &util.FileLayerAnalyzeResult{ + Image: image.Source, + AnalyzeType: "FileLayer", + Analysis: directoryEntries, + }, nil +} diff --git a/pkg/util/image_utils.go b/pkg/util/image_utils.go index a21536e5..7e514e41 100644 --- a/pkg/util/image_utils.go +++ b/pkg/util/image_utils.go @@ -33,10 +33,15 @@ import ( "github.com/sirupsen/logrus" ) +type Layer struct { + FSPath string +} + type Image struct { Image v1.Image Source string FSPath string + Layers []Layer } type ImageHistoryItem struct { @@ -50,6 +55,13 @@ func CleanupImage(image Image) { logrus.Warn(err.Error()) } } + if image.Layers != nil { + for _, layer := range image.Layers { + if err := os.RemoveAll(layer.FSPath); err != nil { + logrus.Warn(err.Error()) + } + } + } } func SortMap(m map[string]struct{}) string { @@ -61,6 +73,15 @@ func SortMap(m map[string]struct{}) string { return strings.Join(pairs, " ") } +// GetFileSystemForLayer unpacks a layer to local disk +func GetFileSystemForLayer(layer v1.Layer, root string, whitelist []string) error { + contents, err := layer.Uncompressed() + if err != nil { + return err + } + return unpackTar(tar.NewReader(contents), root, whitelist) +} + // unpack image filesystem to local disk func GetFileSystemForImage(image v1.Image, root string, whitelist []string) error { if err := unpackTar(tar.NewReader(mutate.Extract(image)), root, whitelist); err != nil { diff --git a/tests/file_layer_analysis_expected.json b/tests/file_layer_analysis_expected.json new file mode 100644 index 00000000..f7c02256 --- /dev/null +++ b/tests/file_layer_analysis_expected.json @@ -0,0 +1,26 @@ +[ + { + "Image": "gcr.io/gcp-runtimes/diff-layer-base", + "AnalyzeType": "FileLayer", + "Analysis": [ + [ + { + "Name": "/first", + "Size": 6 + } + ], + [ + { + "Name": "/second", + "Size": 7 + } + ], + [ + { + "Name": "/third", + "Size": 6 + } + ] + ] + } +] \ No newline at end of file diff --git a/tests/file_layer_diff_expected.json b/tests/file_layer_diff_expected.json new file mode 100644 index 00000000..b741d8c6 --- /dev/null +++ b/tests/file_layer_diff_expected.json @@ -0,0 +1,31 @@ +[ + { + "Image1": "gcr.io/gcp-runtimes/diff-layer-base", + "Image2": "gcr.io/gcp-runtimes/diff-layer-modified", + "DiffType": "FileLayer", + "Diff": { + "DirDiffs": [ + { + "Adds": null, + "Dels": null, + "Mods": null + }, + { + "Adds": [ + { + "Name": "/modified", + "Size": 9 + } + ], + "Dels": [ + { + "Name": "/second", + "Size": 7 + } + ], + "Mods": null + } + ] + } + } +] \ No newline at end of file diff --git a/tests/integration_test.go b/tests/integration_test.go index 378e253c..7ca2dbe4 100644 --- a/tests/integration_test.go +++ b/tests/integration_test.go @@ -38,6 +38,9 @@ const ( diffBase = "gcr.io/gcp-runtimes/diff-base" diffModified = "gcr.io/gcp-runtimes/diff-modified" + diffLayerBase = "gcr.io/gcp-runtimes/diff-layer-base" + diffLayerModifed = "gcr.io/gcp-runtimes/diff-layer-modified" + metadataBase = "gcr.io/gcp-runtimes/metadata-base" metadataModified = "gcr.io/gcp-runtimes/metadata-modified" @@ -106,6 +109,14 @@ func TestDiffAndAnalysis(t *testing.T) { differFlags: []string{"--type=file"}, expectedFile: "file_diff_expected.json", }, + { + description: "file layer differ", + subcommand: "diff", + imageA: diffLayerBase, + imageB: diffLayerModifed, + differFlags: []string{"--type=layer"}, + expectedFile: "file_layer_diff_expected.json", + }, { description: "apt differ", subcommand: "diff", @@ -191,6 +202,13 @@ func TestDiffAndAnalysis(t *testing.T) { differFlags: []string{"--type=file", "-o"}, expectedFile: "file_sorted_analysis_expected.json", }, + { + description: "file layer analysis", + subcommand: "analyze", + imageA: diffLayerBase, + differFlags: []string{"--type=layer"}, + expectedFile: "file_layer_analysis_expected.json", + }, { description: "pip analysis", subcommand: "analyze", diff --git a/util/analyze_output_utils.go b/util/analyze_output_utils.go index 3460a812..7f07ee14 100644 --- a/util/analyze_output_utils.go +++ b/util/analyze_output_utils.go @@ -216,3 +216,55 @@ func (r FileAnalyzeResult) OutputText(analyzeType string, format string) error { } return TemplateOutputFromFormat(strResult, "FileAnalyze", format) } + +type FileLayerAnalyzeResult AnalyzeResult + +func (r FileLayerAnalyzeResult) OutputStruct() interface{} { + analysis, valid := r.Analysis.([][]util.DirectoryEntry) + if !valid { + logrus.Error("Unexpected structure of Analysis. Should be of type []DirectoryEntry") + return errors.New("Could not output FileAnalyzer analysis result") + } + + for _, a := range analysis { + if SortSize { + directoryBy(directorySizeSort).Sort(a) + } else { + directoryBy(directoryNameSort).Sort(a) + } + } + + r.Analysis = analysis + return r +} + +func (r FileLayerAnalyzeResult) OutputText(analyzeType string, format string) error { + analysis, valid := r.Analysis.([][]util.DirectoryEntry) + if !valid { + logrus.Error("Unexpected structure of Analysis. Should be of type []DirectoryEntry") + return errors.New("Could not output FileAnalyzer analysis result") + } + + var strDirectoryEntries [][]StrDirectoryEntry + + for _, a := range analysis { + if SortSize { + directoryBy(directorySizeSort).Sort(a) + } else { + directoryBy(directoryNameSort).Sort(a) + } + strAnalysis := stringifyDirectoryEntries(a) + strDirectoryEntries = append(strDirectoryEntries, strAnalysis) + } + + strResult := struct { + Image string + AnalyzeType string + Analysis [][]StrDirectoryEntry + }{ + Image: r.Image, + AnalyzeType: r.AnalyzeType, + Analysis: strDirectoryEntries, + } + return TemplateOutputFromFormat(strResult, "FileLayerAnalyze", format) +} diff --git a/util/diff_output_utils.go b/util/diff_output_utils.go index 33748d8d..4f2a72c0 100644 --- a/util/diff_output_utils.go +++ b/util/diff_output_utils.go @@ -230,3 +230,65 @@ func (r DirDiffResult) OutputText(diffType string, format string) error { } return TemplateOutputFromFormat(strResult, "DirDiff", format) } + +type MultipleDirDiffResult DiffResult + +func (r MultipleDirDiffResult) OutputStruct() interface{} { + diff, valid := r.Diff.(MultipleDirDiff) + if !valid { + logrus.Error("Unexpected structure of Diff. Should follow the MultipleDirDiff struct") + return errors.New("Could not output FileLayerAnalyzer diff result") + } + for i, d := range diff.DirDiffs { + diff.DirDiffs[i] = sortDirDiff(d) + } + r.Diff = diff + return r +} + +func (r MultipleDirDiffResult) OutputText(diffType string, format string) error { + diff, valid := r.Diff.(MultipleDirDiff) + if !valid { + logrus.Error("Unexpected structure of Diff. Should follow the MultipleDirDiff struct") + return errors.New("Could not output FileLayerAnalyzer diff result") + } + for i, d := range diff.DirDiffs { + diff.DirDiffs[i] = sortDirDiff(d) + } + + type StrDiff struct { + Adds []StrDirectoryEntry + Dels []StrDirectoryEntry + Mods []StrEntryDiff + } + + var strDiffs []StrDiff + for _, d := range diff.DirDiffs { + strAdds := stringifyDirectoryEntries(d.Adds) + strDels := stringifyDirectoryEntries(d.Dels) + strMods := stringifyEntryDiffs(d.Mods) + + strDiffs = append(strDiffs, StrDiff{ + Adds: strAdds, + Dels: strDels, + Mods: strMods, + }) + + } + + type ImageDiff struct { + StrDiffs []StrDiff + } + strResult := struct { + Image1 string + Image2 string + DiffType string + Diff []StrDiff + }{ + Image1: r.Image1, + Image2: r.Image2, + DiffType: r.DiffType, + Diff: strDiffs, + } + return TemplateOutputFromFormat(strResult, "MultipleDirDiff", format) +} diff --git a/util/diff_utils.go b/util/diff_utils.go index 558611e2..9b3dd9ca 100644 --- a/util/diff_utils.go +++ b/util/diff_utils.go @@ -34,6 +34,10 @@ type DirDiff struct { Mods []EntryDiff } +type MultipleDirDiff struct { + DirDiffs []DirDiff +} + type FileNameDiff struct { Filename string Description string diff --git a/util/format_utils.go b/util/format_utils.go index dad03811..fc263bb8 100644 --- a/util/format_utils.go +++ b/util/format_utils.go @@ -34,9 +34,11 @@ var templates = map[string]string{ "HistDiff": HistoryDiffOutput, "MetadataDiff": MetadataDiffOutput, "DirDiff": FSDiffOutput, + "MultipleDirDiff": FSLayerDiffOutput, "FilenameDiff": FilenameDiffOutput, "ListAnalyze": ListAnalysisOutput, "FileAnalyze": FileAnalysisOutput, + "FileLayerAnalyze": FileLayerAnalysisOutput, "MultiVersionPackageAnalyze": MultiVersionPackageOutput, "SingleVersionPackageAnalyze": SingleVersionPackageOutput, } diff --git a/util/template_utils.go b/util/template_utils.go index 9366f09d..c235e68b 100644 --- a/util/template_utils.go +++ b/util/template_utils.go @@ -29,6 +29,23 @@ These entries have been changed between {{.Image1}} and {{.Image2}}:{{if not .Di FILE SIZE1 SIZE2{{range .Diff.Mods}}{{"\n"}}{{.Name}} {{.Size1}} {{.Size2}}{{end}} {{end}} ` +const FSLayerDiffOutput = ` +-----{{.DiffType}}----- + +{{range $index, $diff := .Diff}} + +Diff for Layer {{$index}}: +These entries have been added to {{$.Image1}}:{{if not $diff.Adds}} None{{else}} +FILE SIZE{{range $diff.Adds}}{{"\n"}}{{.Name}} {{.Size}}{{end}}{{end}} + +These entries have been deleted from {{$.Image1}}:{{if not $diff.Dels}} None{{else}} +FILE SIZE{{range $diff.Dels}}{{"\n"}}{{.Name}} {{.Size}}{{end}}{{end}} + +These entries have been changed between {{$.Image1}} and {{$.Image2}}:{{if not $diff.Mods}} None{{else}} +FILE SIZE1 SIZE2{{range $diff.Mods}}{{"\n"}}{{.Name}} {{.Size1}} {{.Size2}}{{end}} +{{end}} +{{end}} +` const SingleVersionDiffOutput = ` -----{{.DiffType}}----- @@ -97,6 +114,16 @@ FILE SIZE{{range .Analysis}}{{"\n"}}{{.Name}} {{.Size}}{{end}} {{end}} ` +const FileLayerAnalysisOutput = ` +-----{{.AnalyzeType}}----- +{{range $index, $analysis := .Analysis}} + +Analysis for {{$.Image}} Layer {{$index}}:{{if not $analysis}} None{{else}} +FILE SIZE{{range $analysis}}{{"\n"}}{{.Name}} {{.Size}}{{end}} +{{end}} +{{end}} +` + const MultiVersionPackageOutput = ` -----{{.AnalyzeType}}-----