gitpod-io · csweichel · Sep 9, 2022 · Aug 31, 2022 · Aug 31, 2022
diff --git a/README.md b/README.md
@@ -50,33 +50,6 @@ scripts:
 - ...
 ```
 
-## Package
-A package is an entry in a `BUILD.yaml` in the `packages` section. All packages share the following fields:
-```YAML
-# name is the component-wide unique name of this package
-name: must-not-contain-spaces
-# Package type must be one of: go, yarn, docker, generic
-type: generic
-# Sources list all sources of this package. Entries can be double-star globs and are relative to the component root.
-# Avoid listing sources outside the component folder.
-srcs:
-- "**/*.yaml"
-- "glob/**/path"
-# Deps list dependencies to other packages which must be built prior to building this package. How these dependencies are made
-# available during build depends on the package type.
-deps:
-- some/other:package
-# Argdeps makes build arguments version relevant. I.e. if the value of a build arg listed here changes, so does the package version.
-argdeps:
-- someBuildArg
-# Env is a list of key=value pair environment variables available during package build
-env:
-- CGO_ENABLED=0
-# Config configures the package build depending on the package type. See below for details
-config:
-  ...
-```
-
 ## Script
 Scripts are a great way to automate tasks during development time (think [`yarn scripts`](https://classic.yarnpkg.com/en/docs/package-json#toc-scripts)).
 Unlike packages they do not run in isolation by default, but have access to the original workspace.
@@ -113,13 +86,32 @@ script: |
   echo "build args work to: ${myBuildArg}"
 ```
 
-### Build arguments
-
-In a package definition one can use _build arguments_. Build args have the form of `${argumentName}` and are string-replaced when the package is loaded.
-**It's advisable to use build args only within the `config` section of packages**. Constants and built-in build args do not even work outside of the config section.
-
-Leeway supports built-in build arguments:
-- `__pkg_version` resolves to the leeway version hash of a component.
+## Package
+A package is an entry in a `BUILD.yaml` in the `packages` section. All packages share the following fields:
+```YAML
+# name is the component-wide unique name of this package
+name: must-not-contain-spaces
+# Package type must be one of: go, yarn, docker, generic
+type: generic
+# Sources list all sources of this package. Entries can be double-star globs and are relative to the component root.
+# Avoid listing sources outside the component folder.
+srcs:
+- "**/*.yaml"
+- "glob/**/path"
+# Deps list dependencies to other packages which must be built prior to building this package. How these dependencies are made
+# available during build depends on the package type.
+deps:
+- some/other:package
+# Argdeps makes build arguments version relevant. I.e. if the value of a build arg listed here changes, so does the package version.
+argdeps:
+- someBuildArg
+# Env is a list of key=value pair environment variables available during package build
+env:
+- CGO_ENABLED=0
+# Config configures the package build depending on the package type. See below for details
+config:
+  ...
+```
 
 ### Go packages
 ```YAML
@@ -200,6 +192,90 @@ config:
   - ["sh", "-c", "ls *"]
 ```
 
+## Dynaimc package scripts
+Packages can be dynamically produced within a component using a dynamic package script named `BUILD.js`. This ECMAScript 5.1 file is executed using [Goja](https://github.com/dop251/goja) and produces a `packages` array which contains the package struct much like they'd exist within the `BUILD.yaml`. For example:
+
+Leeway interacts with the script using global variables, specifically:
+- `args` [input] a JavaScript object containing the build arguments which have explicitely been passed to leeway.
+- `packages` [output] where the script produces an array of package structures akin to those found in a `BUILD.yaml` file.
+
+<table>
+<tr>
+<td valign="top">
+
+`BUILD.js` file
+
+```JavaScript
+let packages = [];
+
+let deps = [];
+for(let i = 0; i < 5; i++) {
+  const name = "hello-"+i;
+  deps.push(name);
+  packages.push({
+    name: name,
+    type: "generic",
+    config: {
+      commands: [
+        ["echo", args.msg + ": hello from "+i]
+      ]
+    }
+  });
+}
+
+packages.push({
+  name: "all",
+  type: "generic",
+  deps: deps.map(d => ":" + d),
+})
+```
+</td>
+<td>
+
+Equivalent `BUILD.yaml`
+```YAML
+pacakages:
+- name: all
+  type: generic
+  deps:
+    - hello-1
+    - hello-2
+    - hello-3
+    - hello-4
+    - hello-5
+- name: hello-1
+  type: generic
+  config:
+    commands:
+      - ["echo", "${msg}: hello from 1"]
+- name: hello-2
+  type: generic
+  config:
+    commands:
+      - ["echo", "${msg}: hello from 2"]
+- name: hello-3
+  type: generic
+  config:
+    commands:
+      - ["echo", "${msg}: hello from 3"]
+...
+```
+
+</td>
+</tr>
+</table>
+
+> **Note** that for a `BUILD.js` to become effective/be recodnized there needs to a (possibly empty) `BUILD.yaml` in the same directory.
+
+## Build arguments
+
+In a package definition one can use _build arguments_. Build args have the form of `${argumentName}` and are string-replaced when the package is loaded.
+**It's advisable to use build args only within the `config` section of packages**. Constants and built-in build args do not even work outside of the config section.
+
+Leeway supports built-in build arguments:
+- `__pkg_version` resolves to the leeway version hash of a component.
+- `__git_commit` contains the current Git commit if the build is executed from within a Git working copy. If this variable is used and the build is not executed from within a Git working copy the variable resolution will fail.
+
 ## Package Variants
 Leeway supports build-time variance through "package variants". Those variants are defined on the workspace level and can modify the list of sources, environment variables and config of packages.
 For example consider a `WORKSPACE.YAML` with this variants section:

diff --git a/fixtures/pkgs/generic/BUILD.js b/fixtures/pkgs/generic/BUILD.js
@@ -0,0 +1,22 @@
+let packages = [];
+
+let deps = [];
+for(let i = 0; i < 10; i++) {
+  const name = "hello-"+i;
+  deps.push(name);
+  packages.push({
+    name: name,
+    type: "generic",
+    config: {
+      commands: [
+        ["echo", args.msg + " hello from "+i]
+      ]
+    }
+  });
+}
+
+packages.push({
+  name: "all",
+  type: "generic",
+  deps: deps.map(d => ":" + d),
+})
diff --git a/go.mod b/go.mod
@@ -27,17 +27,21 @@ require (
 	sigs.k8s.io/bom v0.1.0
 )
 
+require github.com/dop251/goja v0.0.0-20220815083517-0c74f9139fd6
+
 require (
 	github.com/Microsoft/go-winio v0.4.16 // indirect
 	github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 // indirect
 	github.com/acomagu/bufpipe v1.0.3 // indirect
 	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.3 // indirect
+	github.com/dlclark/regexp2 v1.7.0 // indirect
 	github.com/emirpasic/gods v1.12.0 // indirect
 	github.com/fatih/color v1.12.0 // indirect
 	github.com/go-git/gcfg v1.5.0 // indirect
 	github.com/go-git/go-billy/v5 v5.3.1 // indirect
 	github.com/go-git/go-git/v5 v5.4.2 // indirect
+	github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
 	github.com/godbus/dbus/v5 v5.0.6 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
@@ -59,7 +63,7 @@ require (
 	golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b // indirect
 	golang.org/x/net v0.0.0-20211111160137-58aab5ef257a // indirect
 	golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c // indirect
-	golang.org/x/text v0.3.6 // indirect
+	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/tools v0.1.7 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	sigs.k8s.io/release-utils v0.3.0 // indirect

diff --git a/go.sum b/go.sum
@@ -97,7 +97,15 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/disiqueira/gotree v1.0.0 h1:en5wk87n7/Jyk6gVME3cx3xN9KmUCstJ1IjHr4Se4To=
 github.com/disiqueira/gotree v1.0.0/go.mod h1:7CwL+VWsWAU95DovkdRZAtA7YbtHwGk+tLV/kNi8niU=
+github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
+github.com/dlclark/regexp2 v1.7.0 h1:7lJfhqlPssTb1WQx4yvTHN0uElPEv52sbaECrAQxjAo=
+github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk=
+github.com/dop251/goja v0.0.0-20220815083517-0c74f9139fd6 h1:xHdUVG+c8SWJnct16Z3QJOVlaYo3OwoJyamo6kR6OL0=
+github.com/dop251/goja v0.0.0-20220815083517-0c74f9139fd6/go.mod h1:yRkwfj0CBpOGre+TwBsqPV0IH0Pk73e4PXJOeNDboGs=
+github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y=
+github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM=
 github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
 github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
 github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
@@ -134,6 +142,8 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
+github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU=
+github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/godbus/dbus/v5 v5.0.6 h1:mkgN1ofwASrYnJ5W6U/BxG15eXXXjirgZc7CLqkcaro=
 github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
@@ -256,8 +266,9 @@ github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQ
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
+github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -326,6 +337,8 @@ github.com/praetorian-inc/gokart v0.3.0/go.mod h1:P/ADPPKodL3GOPTpA9eeyAmtfQfu0/
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
+github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/sclevine/spec v1.4.0/go.mod h1:LvpgJaFyvQzRvc1kaDs0bulYwzC70PbiYjC4QnFHkOM=
@@ -586,8 +599,9 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

diff --git a/pkg/leeway/workspace.go b/pkg/leeway/workspace.go
@@ -19,6 +19,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/dop251/goja"
 	"github.com/imdario/mergo"
 	"github.com/in-toto/in-toto-golang/in_toto"
 	"github.com/minio/highwayhash"
@@ -656,6 +657,36 @@ func loadComponent(ctx context.Context, workspace *Workspace, path string, args
 		err = nil
 	}
 
+	builderFN := strings.TrimSuffix(path, ".yaml") + ".js"
+	if _, err := os.Stat(builderFN); err == nil {
+		addFC, err := runPackageBuilder(builderFN, args)
+		if err != nil {
+			return Component{}, err
+		}
+		for _, p := range addFC {
+			fc, err := yaml.Marshal(p)
+			if err != nil {
+				return Component{}, err
+			}
+			log.WithField("fc", string(fc)).WithField("component", comp.Name).Debug("adding dynamic package")
+
+			var nd yaml.Node
+			err = yaml.Unmarshal(fc, &nd)
+			if err != nil {
+				return Component{}, err
+			}
+
+			var pkg Package
+			err = yaml.Unmarshal(fc, &pkg)
+			if err != nil {
+				return Component{}, err
+			}
+
+			comp.Packages = append(comp.Packages, &pkg)
+			rawcomp.Packages = append(rawcomp.Packages, nd)
+		}
+	}
+
 	for i, pkg := range comp.Packages {
 		pkg.C = &comp
 		if pkg.Type == "typescript" {
@@ -864,3 +895,36 @@ func mergeEnv(pkg *Package, src []string) error {
 	}
 	return nil
 }
+
+func runPackageBuilder(fn string, args Arguments) (fc []map[string]interface{}, err error) {
+	defer func() {
+		if err != nil {
+			err = fmt.Errorf("failed to run package builder script at %s: %w", fn, err)
+		}
+	}()
+
+	prog, err := os.ReadFile(fn)
+	if err != nil {
+		return nil, err
+	}
+
+	vm := goja.New()
+	err = vm.Set("args", args)
+	if err != nil {
+		return nil, err
+	}
+	_, err = vm.RunString(string(prog))
+	if err != nil {
+		return nil, err
+	}
+
+	var res []map[string]interface{}
+	err = vm.ExportTo(vm.Get("packages"), &res)
+	if err != nil {
+		return nil, err
+	}
+
+	log.WithField("res", res).WithField("fn", fn).Debug("ran package builder script")
+
+	return res, nil
+}