diff --git a/README.md b/README.md index 82940fc8..36f3ddd5 100644 --- a/README.md +++ b/README.md @@ -50,33 +50,6 @@ scripts: - ... ``` -## Package -A package is an entry in a `BUILD.yaml` in the `packages` section. All packages share the following fields: -```YAML -# name is the component-wide unique name of this package -name: must-not-contain-spaces -# Package type must be one of: go, yarn, docker, generic -type: generic -# Sources list all sources of this package. Entries can be double-star globs and are relative to the component root. -# Avoid listing sources outside the component folder. -srcs: -- "**/*.yaml" -- "glob/**/path" -# Deps list dependencies to other packages which must be built prior to building this package. How these dependencies are made -# available during build depends on the package type. -deps: -- some/other:package -# Argdeps makes build arguments version relevant. I.e. if the value of a build arg listed here changes, so does the package version. -argdeps: -- someBuildArg -# Env is a list of key=value pair environment variables available during package build -env: -- CGO_ENABLED=0 -# Config configures the package build depending on the package type. See below for details -config: - ... -``` - ## Script Scripts are a great way to automate tasks during development time (think [`yarn scripts`](https://classic.yarnpkg.com/en/docs/package-json#toc-scripts)). Unlike packages they do not run in isolation by default, but have access to the original workspace. @@ -113,13 +86,32 @@ script: | echo "build args work to: ${myBuildArg}" ``` -### Build arguments - -In a package definition one can use _build arguments_. Build args have the form of `${argumentName}` and are string-replaced when the package is loaded. -**It's advisable to use build args only within the `config` section of packages**. Constants and built-in build args do not even work outside of the config section. - -Leeway supports built-in build arguments: -- `__pkg_version` resolves to the leeway version hash of a component. +## Package +A package is an entry in a `BUILD.yaml` in the `packages` section. All packages share the following fields: +```YAML +# name is the component-wide unique name of this package +name: must-not-contain-spaces +# Package type must be one of: go, yarn, docker, generic +type: generic +# Sources list all sources of this package. Entries can be double-star globs and are relative to the component root. +# Avoid listing sources outside the component folder. +srcs: +- "**/*.yaml" +- "glob/**/path" +# Deps list dependencies to other packages which must be built prior to building this package. How these dependencies are made +# available during build depends on the package type. +deps: +- some/other:package +# Argdeps makes build arguments version relevant. I.e. if the value of a build arg listed here changes, so does the package version. +argdeps: +- someBuildArg +# Env is a list of key=value pair environment variables available during package build +env: +- CGO_ENABLED=0 +# Config configures the package build depending on the package type. See below for details +config: + ... +``` ### Go packages ```YAML @@ -200,6 +192,90 @@ config: - ["sh", "-c", "ls *"] ``` +## Dynaimc package scripts +Packages can be dynamically produced within a component using a dynamic package script named `BUILD.js`. This ECMAScript 5.1 file is executed using [Goja](https://github.com/dop251/goja) and produces a `packages` array which contains the package struct much like they'd exist within the `BUILD.yaml`. For example: + +Leeway interacts with the script using global variables, specifically: +- `args` [input] a JavaScript object containing the build arguments which have explicitely been passed to leeway. +- `packages` [output] where the script produces an array of package structures akin to those found in a `BUILD.yaml` file. + + + + + + +
+ +`BUILD.js` file + +```JavaScript +let packages = []; + +let deps = []; +for(let i = 0; i < 5; i++) { + const name = "hello-"+i; + deps.push(name); + packages.push({ + name: name, + type: "generic", + config: { + commands: [ + ["echo", args.msg + ": hello from "+i] + ] + } + }); +} + +packages.push({ + name: "all", + type: "generic", + deps: deps.map(d => ":" + d), +}) +``` + + +Equivalent `BUILD.yaml` +```YAML +pacakages: +- name: all + type: generic + deps: + - hello-1 + - hello-2 + - hello-3 + - hello-4 + - hello-5 +- name: hello-1 + type: generic + config: + commands: + - ["echo", "${msg}: hello from 1"] +- name: hello-2 + type: generic + config: + commands: + - ["echo", "${msg}: hello from 2"] +- name: hello-3 + type: generic + config: + commands: + - ["echo", "${msg}: hello from 3"] +... +``` + +
+ +> **Note** that for a `BUILD.js` to become effective/be recodnized there needs to a (possibly empty) `BUILD.yaml` in the same directory. + +## Build arguments + +In a package definition one can use _build arguments_. Build args have the form of `${argumentName}` and are string-replaced when the package is loaded. +**It's advisable to use build args only within the `config` section of packages**. Constants and built-in build args do not even work outside of the config section. + +Leeway supports built-in build arguments: +- `__pkg_version` resolves to the leeway version hash of a component. +- `__git_commit` contains the current Git commit if the build is executed from within a Git working copy. If this variable is used and the build is not executed from within a Git working copy the variable resolution will fail. + ## Package Variants Leeway supports build-time variance through "package variants". Those variants are defined on the workspace level and can modify the list of sources, environment variables and config of packages. For example consider a `WORKSPACE.YAML` with this variants section: diff --git a/fixtures/pkgs/generic/BUILD.js b/fixtures/pkgs/generic/BUILD.js new file mode 100644 index 00000000..e9592ffb --- /dev/null +++ b/fixtures/pkgs/generic/BUILD.js @@ -0,0 +1,22 @@ +let packages = []; + +let deps = []; +for(let i = 0; i < 10; i++) { + const name = "hello-"+i; + deps.push(name); + packages.push({ + name: name, + type: "generic", + config: { + commands: [ + ["echo", args.msg + " hello from "+i] + ] + } + }); +} + +packages.push({ + name: "all", + type: "generic", + deps: deps.map(d => ":" + d), +}) diff --git a/go.mod b/go.mod index 82bba3dd..b4af03b2 100644 --- a/go.mod +++ b/go.mod @@ -27,17 +27,21 @@ require ( sigs.k8s.io/bom v0.1.0 ) +require github.com/dop251/goja v0.0.0-20220815083517-0c74f9139fd6 + require ( github.com/Microsoft/go-winio v0.4.16 // indirect github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 // indirect github.com/acomagu/bufpipe v1.0.3 // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/cyphar/filepath-securejoin v0.2.3 // indirect + github.com/dlclark/regexp2 v1.7.0 // indirect github.com/emirpasic/gods v1.12.0 // indirect github.com/fatih/color v1.12.0 // indirect github.com/go-git/gcfg v1.5.0 // indirect github.com/go-git/go-billy/v5 v5.3.1 // indirect github.com/go-git/go-git/v5 v5.4.2 // indirect + github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/godbus/dbus/v5 v5.0.6 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect @@ -59,7 +63,7 @@ require ( golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b // indirect golang.org/x/net v0.0.0-20211111160137-58aab5ef257a // indirect golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c // indirect - golang.org/x/text v0.3.6 // indirect + golang.org/x/text v0.3.7 // indirect golang.org/x/tools v0.1.7 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect sigs.k8s.io/release-utils v0.3.0 // indirect diff --git a/go.sum b/go.sum index bc8ff1bb..8fbf0c48 100644 --- a/go.sum +++ b/go.sum @@ -97,7 +97,15 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/disiqueira/gotree v1.0.0 h1:en5wk87n7/Jyk6gVME3cx3xN9KmUCstJ1IjHr4Se4To= github.com/disiqueira/gotree v1.0.0/go.mod h1:7CwL+VWsWAU95DovkdRZAtA7YbtHwGk+tLV/kNi8niU= +github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= +github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo= +github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= +github.com/dop251/goja v0.0.0-20220815083517-0c74f9139fd6 h1:xHdUVG+c8SWJnct16Z3QJOVlaYo3OwoJyamo6kR6OL0= +github.com/dop251/goja v0.0.0-20220815083517-0c74f9139fd6/go.mod h1:yRkwfj0CBpOGre+TwBsqPV0IH0Pk73e4PXJOeNDboGs= +github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= +github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= @@ -134,6 +142,8 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU= +github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.6 h1:mkgN1ofwASrYnJ5W6U/BxG15eXXXjirgZc7CLqkcaro= github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -256,8 +266,9 @@ github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -326,6 +337,8 @@ github.com/praetorian-inc/gokart v0.3.0/go.mod h1:P/ADPPKodL3GOPTpA9eeyAmtfQfu0/ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sclevine/spec v1.4.0/go.mod h1:LvpgJaFyvQzRvc1kaDs0bulYwzC70PbiYjC4QnFHkOM= @@ -586,8 +599,9 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/pkg/leeway/workspace.go b/pkg/leeway/workspace.go index 73540438..219f3c09 100644 --- a/pkg/leeway/workspace.go +++ b/pkg/leeway/workspace.go @@ -19,6 +19,7 @@ import ( "sync" "time" + "github.com/dop251/goja" "github.com/imdario/mergo" "github.com/in-toto/in-toto-golang/in_toto" "github.com/minio/highwayhash" @@ -656,6 +657,36 @@ func loadComponent(ctx context.Context, workspace *Workspace, path string, args err = nil } + builderFN := strings.TrimSuffix(path, ".yaml") + ".js" + if _, err := os.Stat(builderFN); err == nil { + addFC, err := runPackageBuilder(builderFN, args) + if err != nil { + return Component{}, err + } + for _, p := range addFC { + fc, err := yaml.Marshal(p) + if err != nil { + return Component{}, err + } + log.WithField("fc", string(fc)).WithField("component", comp.Name).Debug("adding dynamic package") + + var nd yaml.Node + err = yaml.Unmarshal(fc, &nd) + if err != nil { + return Component{}, err + } + + var pkg Package + err = yaml.Unmarshal(fc, &pkg) + if err != nil { + return Component{}, err + } + + comp.Packages = append(comp.Packages, &pkg) + rawcomp.Packages = append(rawcomp.Packages, nd) + } + } + for i, pkg := range comp.Packages { pkg.C = &comp if pkg.Type == "typescript" { @@ -864,3 +895,36 @@ func mergeEnv(pkg *Package, src []string) error { } return nil } + +func runPackageBuilder(fn string, args Arguments) (fc []map[string]interface{}, err error) { + defer func() { + if err != nil { + err = fmt.Errorf("failed to run package builder script at %s: %w", fn, err) + } + }() + + prog, err := os.ReadFile(fn) + if err != nil { + return nil, err + } + + vm := goja.New() + err = vm.Set("args", args) + if err != nil { + return nil, err + } + _, err = vm.RunString(string(prog)) + if err != nil { + return nil, err + } + + var res []map[string]interface{} + err = vm.ExportTo(vm.Get("packages"), &res) + if err != nil { + return nil, err + } + + log.WithField("res", res).WithField("fn", fn).Debug("ran package builder script") + + return res, nil +}