Merge pull request #23 from CosmWasm/workspace-optimizer

Add workspace optimizer

Author: ethanfrey

.editorconfig

@@ -10,3 +10,7 @@ insert_final_newline = true
 [Makefile]
 indent_style = tab
 indent_size = 4
+
+[*.py]
+indent_style = space
+indent_size = 4

Makefile

@@ -1,20 +1,21 @@
-.PHONY: build publish run debug
+.PHONY: build-rust-optimizer build-workspace-optimizer build publish-rust-optimizer publish-workspace-optimizer publish
 
-DOCKER_NAME := "cosmwasm/rust-optimizer"
-DOCKER_TAG := 0.10.1
-CODE ?= "/path/to/contract"
-USER_ID := $(shell id -u)
-USER_GROUP = $(shell id -g)
+DOCKER_NAME_RUST_OPTIMIZER := "cosmwasm/rust-optimizer"
+DOCKER_NAME_WORKSPACE_OPTIMIZER := "cosmwasm/workspace-optimizer"
+DOCKER_TAG := 0.10.2
 
-build:
-	docker build . -t $(DOCKER_NAME):$(DOCKER_TAG)
+build-rust-optimizer:
+	docker build -t $(DOCKER_NAME_RUST_OPTIMIZER):$(DOCKER_TAG) --file rust-optimizer.Dockerfile .
 
-publish: build
-	docker push $(DOCKER_NAME):$(DOCKER_TAG)
+build-workspace-optimizer:
+	docker build -t $(DOCKER_NAME_WORKSPACE_OPTIMIZER):$(DOCKER_TAG) --file workspace-optimizer.Dockerfile .
 
-# Usage: make run CODE=/path/to/contract
-run:
-	docker run --rm -u $(USER_ID):$(USER_GROUP) -v "$(CODE)":/code $(DOCKER_NAME):$(DOCKER_TAG)
+publish-rust-optimizer: build-rust-optimizer
+	docker push $(DOCKER_NAME_RUST_OPTIMIZER):$(DOCKER_TAG)
 
-debug:
-	docker run --rm -it -u $(USER_ID):$(USER_GROUP) -v "$(CODE)":/code $(DOCKER_NAME):$(DOCKER_TAG) /bin/bash
+publish-workspace-optimizer: build-workspace-optimizer
+	docker push $(DOCKER_NAME_WORKSPACE_OPTIMIZER):$(DOCKER_TAG)
+
+build: build-rust-optimizer build-workspace-optimizer
+
+publish: publish-rust-optimizer publish-workspace-optimizer

README.md

@@ -8,7 +8,7 @@ optimization on the build size, using binary stripping and `wasm-opt`.
 
 ## Usage
 
-*This works for most cases, for cosmwasm builds see advanced*
+*This works for most cases, for monorepo builds see advanced*
 
 The easiest way is to simply use the [published docker image](https://hub.docker.com/r/cosmwasm/rust-optimizer).
 You must set the local path to the smart contract you wish to compile and
@@ -24,7 +24,7 @@ you to produce a smaller build that works with the cosmwasm integration tests
 docker run --rm -v "$(pwd)":/code \
   --mount type=volume,source="$(basename "$(pwd)")_cache",target=/code/target \
   --mount type=volume,source=registry_cache,target=/usr/local/cargo/registry \
-  cosmwasm/rust-optimizer:0.10.1
+  cosmwasm/rust-optimizer:0.10.2
 ```
 
 Demo this with `cosmwasm-examples` (going into eg. `erc20` subdir before running),
@@ -34,7 +34,38 @@ Note that we use one registry cache (to avoid excessive downloads), but the targ
 contract that we compile. This means no interference between contracts, but very fast recompile times when making
 minor adjustments to a contract you had previously created an optimized build for.
 
-## Advanced usage
+## Mono Repos
+
+### Contracts as Workspace Members
+
+*This is designed for cosmwasm-plus samples. We use a separate docker image*
+
+Sometime you want many contracts to be related and import common functionality. This is
+exactly the case of [`cosmwasm-plus`](https://github.com/CosmWasm/cosmwasm-plus).
+In such a case, we can often not just compile from root, as the compile order is
+not deterministic and there are feature flags shared among the repos.
+This has lead to [issues in the past](https://github.com/CosmWasm/rust-optimizer/issues/21).
+
+For this use-case we made a second docker image, which will compile all the
+`contracts/*` folders inside the workspace and do so one-by-one in alphabetical order.
+It will then add all the generated wasm files to an `artifacts` directory with a checksum,
+just like the basic docker image (same output format).
+
+To compile all contracts in the workspace deterministically, you can run:
+
+```shell
+docker run --rm -v "$(pwd)":/code \
+  --mount type=volume,source="$(basename "$(pwd)")_cache",target=/code/target \
+  --mount type=volume,source=registry_cache,target=/usr/local/cargo/registry \
+  cosmwasm/workspace-optimizer:0.10.2
+```
+
+The downside is that to verify one contract in the workspace, you need to compile them
+all, but the majority of the build time is in dependencies, which are shared and cached
+between the various contracts and thus the time is sub-linear with respect to number
+of contracts.
+
+### Contracts excluded from Workspace
 
 *This is designed for cosmwasm samples. You cannot provide automatic verification for these*
 
@@ -52,7 +83,7 @@ case, we can use the optimize.sh command:
 docker run --rm -v "$(pwd)":/code \
   --mount type=volume,source="devcontract_cache_burner",target=/code/contracts/burner/target \
   --mount type=volume,source=registry_cache,target=/usr/local/cargo/registry \
-  cosmwasm/rust-optimizer:0.10.1 ./contracts/burner
+  cosmwasm/rust-optimizer:0.10.2 ./contracts/burner
 ```
 
 ## Development

optimize_workspace.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+
+# Build script for cargo workspaces
+
+CARGO_PATH="cargo"
+PACKAGE_PREFIX="contracts/"
+
+import glob
+import os
+import shutil
+import stat
+import subprocess
+import toml
+
+def log(*args):
+    print(*args, flush=True)
+
+with open("Cargo.toml") as file:
+    document = toml.load(file)
+    members = document['workspace']['members']
+
+log("Found workspace member entries:", members)
+
+all_packages = []
+for member in members:
+    all_packages.extend(glob.glob(member))
+all_packages.sort()
+log("Package directories:", all_packages)
+
+contract_packages = [p for p in all_packages if p.startswith(PACKAGE_PREFIX)]
+log("Contracts to be built:", contract_packages)
+
+artifacts_dir = os.path.realpath("artifacts")
+os.makedirs(artifacts_dir, exist_ok=True)
+
+for contract in contract_packages:
+    log("Building {} ...".format(contract))
+    # make a tmp dir for the output (*.wasm and other) to not touch the host filesystem
+    tmp_dir = "/tmp/" + contract
+    os.makedirs(tmp_dir, exist_ok=True)
+
+    # Rust nightly and unstable-options is needed to use --out-dir
+    cmd = [CARGO_PATH, "-Z=unstable-options", "build", "--release", "--target=wasm32-unknown-unknown", "--locked", "--out-dir={}".format(tmp_dir)]
+    os.environ["RUSTFLAGS"] = "-C link-arg=-s"
+    subprocess.check_call(cmd, cwd=contract)
+
+    for build_result in glob.glob("{}/*.wasm".format(tmp_dir)):
+        log("Optimizing built {} ...".format(build_result))
+        name = os.path.basename(build_result)
+        cmd = ["wasm-opt", "-Os", "-o", "artifacts/{}".format(name), build_result]
+        subprocess.check_call(cmd)
+

optimize_workspace.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -o errexit -o nounset -o pipefail
+command -v shellcheck > /dev/null && shellcheck "$0"
+
+export PATH="$PATH:/root/.cargo/bin"
+
+rustup toolchain list
+cargo --version
+
+optimize_workspace.py
+
+# Postprocess artifacts
+(
+  cd artifacts
+  chmod -x ./*.wasm
+  sha256sum -- *.wasm > checksums.txt
+)
+
+echo "done"

Dockerfile renamed to rust-optimizer.Dockerfile

@@ -0,0 +1,37 @@
+# See https://github.com/emscripten-core/emscripten/pull/9119/files for fastcomp vs. upstream docs
+# Using the traditional fastcomp for now.
+FROM trzeci/emscripten:1.39.8-fastcomp
+
+# This version of Rust will not be used for compilation but just serves as a stable base image to get debian+rustup.
+# See Rust nightly config below.
+FROM rust:1.45.2
+RUN rustup toolchain remove 1.45.2
+
+RUN apt update
+RUN apt install python3 python3-toml -y
+
+RUN python3 --version
+
+# Install Rust nightly
+# Choose version from: https://rust-lang.github.io/rustup-components-history/x86_64-unknown-linux-gnu.html
+RUN rustup toolchain install nightly-2020-08-20 --allow-downgrade --profile minimal --target wasm32-unknown-unknown
+RUN rustup default nightly-2020-08-20
+RUN rustup toolchain list
+# Check cargo version
+RUN cargo --version
+
+# copy wasm-opt into our path
+COPY --from=0 /emsdk_portable/binaryen/bin/wasm-opt /usr/local/bin
+
+# Assume we mount the source code in /code
+WORKDIR /code
+
+# Add our scripts as entry point
+ADD optimize_workspace.py /usr/local/bin/
+ADD optimize_workspace.sh /usr/local/bin/
+RUN chmod +x /usr/local/bin/optimize_workspace.py
+RUN chmod +x /usr/local/bin/optimize_workspace.sh
+
+ENTRYPOINT ["optimize_workspace.sh"]
+# Default argument when none is provided
+CMD ["."]