Merge pull request #248 from davidcassany/layered_analysis_for_single_version_packages

nkubala · web-flow · commit 509496ef97a9 · 2018-07-23T12:39:52.000-07:00
Layered analysis for single version packages
diff --git a/cmd/root.go b/cmd/root.go
@@ -59,6 +59,8 @@ const (
 	RemotePrefix = "remote://"
 )
 
+var layerAnalyzers = [...]string{"layer", "aptlayer"}
+
 var RootCmd = &cobra.Command{
 	Use:   "container-diff",
 	Short: "container-diff is a tool for analyzing and comparing container images",
@@ -268,8 +270,10 @@ func getExtractPathForName(name string) (string, error) {
 
 func includeLayers() bool {
 	for _, t := range types {
-		if t == "layer" {
-			return true
+		for _, a := range layerAnalyzers {
+			if t == a {
+				return true
+			}
 		}
 	}
 	return false
diff --git a/differs/apt_diff.go b/differs/apt_diff.go
@@ -28,6 +28,9 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+//APT package database location
+const dpkgStatusFile string = "var/lib/dpkg/status"
+
 type AptAnalyzer struct {
 }
 
@@ -47,13 +50,16 @@ func (a AptAnalyzer) Analyze(image pkgutil.Image) (util.Result, error) {
 }
 
 func (a AptAnalyzer) getPackages(image pkgutil.Image) (map[string]util.PackageInfo, error) {
-	path := image.FSPath
+	return readStatusFile(image.FSPath)
+}
+
+func readStatusFile(root string) (map[string]util.PackageInfo, error) {
 	packages := make(map[string]util.PackageInfo)
-	if _, err := os.Stat(path); err != nil {
+	if _, err := os.Stat(root); err != nil {
 		// invalid image directory path
 		return packages, err
 	}
-	statusFile := filepath.Join(path, "var/lib/dpkg/status")
+	statusFile := filepath.Join(root, dpkgStatusFile)
 	if _, err := os.Stat(statusFile); err != nil {
 		// status file does not exist in this layer
 		return packages, nil
@@ -120,3 +126,43 @@ func parseLine(text string, currPackage string, packages map[string]util.Package
 	}
 	return currPackage
 }
+
+type AptLayerAnalyzer struct {
+}
+
+func (a AptLayerAnalyzer) Name() string {
+	return "AptLayerAnalyzer"
+}
+
+// AptDiff compares the packages installed by apt-get.
+func (a AptLayerAnalyzer) Diff(image1, image2 pkgutil.Image) (util.Result, error) {
+	diff, err := singleVersionLayerDiff(image1, image2, a)
+	return diff, err
+}
+
+func (a AptLayerAnalyzer) Analyze(image pkgutil.Image) (util.Result, error) {
+	analysis, err := singleVersionLayerAnalysis(image, a)
+	return analysis, err
+}
+
+func (a AptLayerAnalyzer) getPackages(image pkgutil.Image) ([]map[string]util.PackageInfo, error) {
+	var packages []map[string]util.PackageInfo
+	if _, err := os.Stat(image.FSPath); err != nil {
+		// invalid image directory path
+		return packages, err
+	}
+	statusFile := filepath.Join(image.FSPath, dpkgStatusFile)
+	if _, err := os.Stat(statusFile); err != nil {
+		// status file does not exist in this image
+		return packages, nil
+	}
+	for _, layer := range image.Layers {
+		layerPackages, err := readStatusFile(layer.FSPath)
+		if err != nil {
+			return packages, err
+		}
+		packages = append(packages, layerPackages)
+	}
+
+	return packages, nil
+}
diff --git a/differs/differs.go b/differs/differs.go
@@ -47,6 +47,7 @@ var Analyzers = map[string]Analyzer{
 	"file":     FileAnalyzer{},
 	"layer":    FileLayerAnalyzer{},
 	"apt":      AptAnalyzer{},
+	"aptlayer": AptLayerAnalyzer{},
 	"rpm":      RPMAnalyzer{},
 	"pip":      PipAnalyzer{},
 	"node":     NodeAnalyzer{},
diff --git a/differs/package_differs.go b/differs/package_differs.go
@@ -17,10 +17,12 @@ limitations under the License.
 package differs
 
 import (
+	"errors"
 	"strings"
 
 	pkgutil "github.com/GoogleContainerTools/container-diff/pkg/util"
 	"github.com/GoogleContainerTools/container-diff/util"
+	"github.com/sirupsen/logrus"
 )
 
 type MultiVersionPackageAnalyzer interface {
@@ -33,6 +35,11 @@ type SingleVersionPackageAnalyzer interface {
 	Name() string
 }
 
+type SingleVersionPackageLayerAnalyzer interface {
+	getPackages(image pkgutil.Image) ([]map[string]util.PackageInfo, error)
+	Name() string
+}
+
 func multiVersionDiff(image1, image2 pkgutil.Image, differ MultiVersionPackageAnalyzer) (*util.MultiVersionPackageDiffResult, error) {
 	pack1, err := differ.getPackages(image1)
 	if err != nil {
@@ -71,6 +78,13 @@ func singleVersionDiff(image1, image2 pkgutil.Image, differ SingleVersionPackage
 	}, nil
 }
 
+// singleVersionLayerDiff returns an error as this diff is not supported as
+// it is far from obvious to define it in meaningful way
+func singleVersionLayerDiff(image1, image2 pkgutil.Image, differ SingleVersionPackageLayerAnalyzer) (*util.SingleVersionPackageLayerDiffResult, error) {
+	logrus.Warning("'diff' command for packages on layers is not supported, consider using 'analyze' on each image instead")
+	return &util.SingleVersionPackageLayerDiffResult{}, errors.New("Diff for packages on layers is not supported, only analysis is supported")
+}
+
 func multiVersionAnalysis(image pkgutil.Image, analyzer MultiVersionPackageAnalyzer) (*util.MultiVersionPackageAnalyzeResult, error) {
 	pack, err := analyzer.getPackages(image)
 	if err != nil {
@@ -98,3 +112,39 @@ func singleVersionAnalysis(image pkgutil.Image, analyzer SingleVersionPackageAna
 	}
 	return &analysis, nil
 }
+
+// singleVersionLayerAnalysis returns the packages included, deleted or
+// updated in each layer
+func singleVersionLayerAnalysis(image pkgutil.Image, analyzer SingleVersionPackageLayerAnalyzer) (*util.SingleVersionPackageLayerAnalyzeResult, error) {
+	pack, err := analyzer.getPackages(image)
+	if err != nil {
+		return &util.SingleVersionPackageLayerAnalyzeResult{}, err
+	}
+	var pkgDiffs []util.PackageDiff
+
+	// Each layer with modified packages includes a complete list of packages
+	// in its package database. Thus we diff the current layer with the
+	// previous one that contains a package database. Layers that do not
+	// include a package database are omitted.
+	preInd := -1
+	for i := range pack {
+		var pkgDiff util.PackageDiff
+		if preInd < 0 && len(pack[i]) > 0 {
+			pkgDiff = util.GetMapDiff(make(map[string]util.PackageInfo), pack[i])
+			preInd = i
+		} else if preInd >= 0 && len(pack[i]) > 0 {
+			pkgDiff = util.GetMapDiff(pack[preInd], pack[i])
+			preInd = i
+		}
+
+		pkgDiffs = append(pkgDiffs, pkgDiff)
+	}
+
+	return &util.SingleVersionPackageLayerAnalyzeResult{
+		Image:       image.Source,
+		AnalyzeType: strings.TrimSuffix(analyzer.Name(), "Analyzer"),
+		Analysis: util.PackageLayerDiff{
+			PackageDiffs: pkgDiffs,
+		},
+	}, nil
+}
diff --git a/util/analyze_output_utils.go b/util/analyze_output_utils.go
@@ -136,6 +136,78 @@ func (r SingleVersionPackageAnalyzeResult) OutputText(diffType string, format st
 	return TemplateOutputFromFormat(strResult, "SingleVersionPackageAnalyze", format)
 }
 
+type SingleVersionPackageLayerAnalyzeResult AnalyzeResult
+
+func (r SingleVersionPackageLayerAnalyzeResult) OutputStruct() interface{} {
+	analysis, valid := r.Analysis.(PackageLayerDiff)
+	if !valid {
+		logrus.Error("Unexpected structure of Analysis.  Should be of type PackageLayerDiff")
+		return fmt.Errorf("Could not output %s analysis result", r.AnalyzeType)
+	}
+
+	type PkgDiff struct {
+		Packages1 []PackageOutput
+		Packages2 []PackageOutput
+		InfoDiff  []Info
+	}
+
+	var analysisOutput []PkgDiff
+	for _, d := range analysis.PackageDiffs {
+		diffOutput := PkgDiff{
+			Packages1: getSingleVersionPackageOutput(d.Packages1),
+			Packages2: getSingleVersionPackageOutput(d.Packages2),
+			InfoDiff:  getSingleVersionInfoDiffOutput(d.InfoDiff),
+		}
+		analysisOutput = append(analysisOutput, diffOutput)
+	}
+
+	output := struct {
+		Image       string
+		AnalyzeType string
+		Analysis    []PkgDiff
+	}{
+		Image:       r.Image,
+		AnalyzeType: r.AnalyzeType,
+		Analysis:    analysisOutput,
+	}
+	return output
+}
+
+func (r SingleVersionPackageLayerAnalyzeResult) OutputText(diffType string, format string) error {
+	analysis, valid := r.Analysis.(PackageLayerDiff)
+	if !valid {
+		logrus.Error("Unexpected structure of Analysis.  Should be of type PackageLayerDiff")
+		return fmt.Errorf("Could not output %s analysis result", r.AnalyzeType)
+	}
+
+	type StrDiff struct {
+		Packages1 []StrPackageOutput
+		Packages2 []StrPackageOutput
+		InfoDiff  []StrInfo
+	}
+
+	var analysisOutput []StrDiff
+	for _, d := range analysis.PackageDiffs {
+		diffOutput := StrDiff{
+			Packages1: stringifyPackages(getSingleVersionPackageOutput(d.Packages1)),
+			Packages2: stringifyPackages(getSingleVersionPackageOutput(d.Packages2)),
+			InfoDiff:  stringifyPackageDiff(getSingleVersionInfoDiffOutput(d.InfoDiff)),
+		}
+		analysisOutput = append(analysisOutput, diffOutput)
+	}
+
+	strResult := struct {
+		Image       string
+		AnalyzeType string
+		Analysis    []StrDiff
+	}{
+		Image:       r.Image,
+		AnalyzeType: r.AnalyzeType,
+		Analysis:    analysisOutput,
+	}
+	return TemplateOutputFromFormat(strResult, "SingleVersionPackageLayerAnalyze", format)
+}
+
 type PackageOutput struct {
 	Name    string
 	Path    string `json:",omitempty"`
diff --git a/util/diff_output_utils.go b/util/diff_output_utils.go
@@ -162,6 +162,72 @@ func getSingleVersionInfoDiffOutput(infoDiff []Info) []Info {
 	return infoDiff
 }
 
+type SingleVersionPackageLayerDiffResult DiffResult
+
+func (r SingleVersionPackageLayerDiffResult) OutputStruct() interface{} {
+	diff, valid := r.Diff.(PackageLayerDiff)
+	if !valid {
+		logrus.Error("Unexpected structure of Diff.  Should follow the PackageLayerDiff struct")
+		return fmt.Errorf("Could not output %s diff result", r.DiffType)
+	}
+
+	type PkgDiff struct {
+		Packages1 []PackageOutput
+		Packages2 []PackageOutput
+		InfoDiff  []Info
+	}
+
+	var diffOutputs []PkgDiff
+	for _, d := range diff.PackageDiffs {
+		diffOutput := PkgDiff{
+			Packages1: getSingleVersionPackageOutput(d.Packages1),
+			Packages2: getSingleVersionPackageOutput(d.Packages2),
+			InfoDiff:  getSingleVersionInfoDiffOutput(d.InfoDiff),
+		}
+		diffOutputs = append(diffOutputs, diffOutput)
+	}
+
+	r.Diff = diffOutputs
+	return r
+}
+
+func (r SingleVersionPackageLayerDiffResult) OutputText(diffType string, format string) error {
+	diff, valid := r.Diff.(PackageLayerDiff)
+	if !valid {
+		logrus.Error("Unexpected structure of Diff.  Should follow the PackageLayerDiff struct")
+		return fmt.Errorf("Could not output %s diff result", r.DiffType)
+	}
+
+	type StrDiff struct {
+		Packages1 []StrPackageOutput
+		Packages2 []StrPackageOutput
+		InfoDiff  []StrInfo
+	}
+
+	var diffOutputs []StrDiff
+	for _, d := range diff.PackageDiffs {
+		diffOutput := StrDiff{
+			Packages1: stringifyPackages(getSingleVersionPackageOutput(d.Packages1)),
+			Packages2: stringifyPackages(getSingleVersionPackageOutput(d.Packages2)),
+			InfoDiff:  stringifyPackageDiff(getSingleVersionInfoDiffOutput(d.InfoDiff)),
+		}
+		diffOutputs = append(diffOutputs, diffOutput)
+	}
+
+	strResult := struct {
+		Image1   string
+		Image2   string
+		DiffType string
+		Diff     []StrDiff
+	}{
+		Image1:   r.Image1,
+		Image2:   r.Image2,
+		DiffType: r.DiffType,
+		Diff:     diffOutputs,
+	}
+	return TemplateOutputFromFormat(strResult, "SingleVersionPackageLayerDiff", format)
+}
+
 type HistDiffResult DiffResult
 
 func (r HistDiffResult) OutputStruct() interface{} {
diff --git a/util/format_utils.go b/util/format_utils.go
@@ -29,18 +29,19 @@ import (
 )
 
 var templates = map[string]string{
-	"SingleVersionPackageDiff":    SingleVersionDiffOutput,
-	"MultiVersionPackageDiff":     MultiVersionDiffOutput,
-	"HistDiff":                    HistoryDiffOutput,
-	"MetadataDiff":                MetadataDiffOutput,
-	"DirDiff":                     FSDiffOutput,
-	"MultipleDirDiff":             FSLayerDiffOutput,
-	"FilenameDiff":                FilenameDiffOutput,
-	"ListAnalyze":                 ListAnalysisOutput,
-	"FileAnalyze":                 FileAnalysisOutput,
-	"FileLayerAnalyze":            FileLayerAnalysisOutput,
-	"MultiVersionPackageAnalyze":  MultiVersionPackageOutput,
-	"SingleVersionPackageAnalyze": SingleVersionPackageOutput,
+	"SingleVersionPackageDiff":         SingleVersionDiffOutput,
+	"MultiVersionPackageDiff":          MultiVersionDiffOutput,
+	"HistDiff":                         HistoryDiffOutput,
+	"MetadataDiff":                     MetadataDiffOutput,
+	"DirDiff":                          FSDiffOutput,
+	"MultipleDirDiff":                  FSLayerDiffOutput,
+	"FilenameDiff":                     FilenameDiffOutput,
+	"ListAnalyze":                      ListAnalysisOutput,
+	"FileAnalyze":                      FileAnalysisOutput,
+	"FileLayerAnalyze":                 FileLayerAnalysisOutput,
+	"MultiVersionPackageAnalyze":       MultiVersionPackageOutput,
+	"SingleVersionPackageAnalyze":      SingleVersionPackageOutput,
+	"SingleVersionPackageLayerAnalyze": SingleVersionPackageLayerOutput,
 }
 
 func JSONify(diff interface{}) error {
diff --git a/util/package_diff_utils.go b/util/package_diff_utils.go
@@ -46,6 +46,12 @@ type PackageDiff struct {
 	InfoDiff  []Info
 }
 
+// PackageLayerDiff stores the difference information between two images
+// layer by layer in PackageDiff array
+type PackageLayerDiff struct {
+	PackageDiffs []PackageDiff
+}
+
 // Info stores the information for one package in two different images.
 type Info struct {
 	Package string
diff --git a/util/template_utils.go b/util/template_utils.go
@@ -139,3 +139,19 @@ Packages found in {{.Image}}:{{if not .Analysis}} None{{else}}
 NAME	VERSION	SIZE{{range .Analysis}}{{"\n"}}{{print "-"}}{{.Name}}	{{.Version}}	{{.Size}}{{end}}
 {{end}}
 `
+
+const SingleVersionPackageLayerOutput = `
+-----{{.AnalyzeType}}-----
+{{range $index, $analysis := .Analysis}}
+For Layer {{$index}}:{{if not (or (or $analysis.Packages1 $analysis.Packages2) $analysis.InfoDiff)}} No package changes {{else}}
+{{if ne $index 0}}Deleted packages from previous layers:{{if not $analysis.Packages1}} None{{else}}
+NAME	VERSION	SIZE{{range $analysis.Packages1}}{{"\n"}}{{print "-"}}{{.Name}}	{{.Version}}	{{.Size}}{{end}}{{end}}
+
+{{end}}Packages added in this layer:{{if not $analysis.Packages2}} None{{else}}
+NAME	VERSION	SIZE{{range $analysis.Packages2}}{{"\n"}}{{print "-"}}{{.Name}}	{{.Version}}	{{.Size}}{{end}}{{end}}
+{{if ne $index 0}}
+Version differences:{{if not $analysis.InfoDiff}} None{{else}}
+PACKAGE	PREV_LAYER	CURRENT_LAYER {{range $analysis.InfoDiff}}{{"\n"}}{{print "-"}}{{.Package}}	{{.Info1.Version}}, {{.Info1.Size}}	{{.Info2.Version}}, {{.Info2.Size}}{{end}}
+{{end}}{{end}}{{end}}
+{{end}}
+`

Original file line number	Diff line number	Diff line change
`@@ -59,6 +59,8 @@ const (`
`59`	`59`	`RemotePrefix = "remote://"`
`60`	`60`	`)`
`61`	`61`
	`62`	`+var layerAnalyzers = [...]string{"layer", "aptlayer"}`
	`63`	`+`
`62`	`64`	`var RootCmd = &cobra.Command{`
`63`	`65`	`Use: "container-diff",`
`64`	`66`	`Short: "container-diff is a tool for analyzing and comparing container images",`
`@@ -268,8 +270,10 @@ func getExtractPathForName(name string) (string, error) {`
`268`	`270`
`269`	`271`	`func includeLayers() bool {`
`270`	`272`	`for _, t := range types {`
`271`		`- if t == "layer" {`
`272`		`- return true`
	`273`	`+ for _, a := range layerAnalyzers {`
	`274`	`+ if t == a {`
	`275`	`+ return true`
	`276`	`+ }`
`273`	`277`	`}`
`274`	`278`	`}`
`275`	`279`	`return false`