Slim down Nix builds (#530)

* Disable debug info for Nix builds

For example, this slims down the airflow-operator image from 615MiB to
147MiB.

This improvement is a combination of the size of the debuginfo
itself (~200MiB per binary), and not having to include build
dependencies just because our debuginfo refers to them (primarily, this
ended up pulling in GCC, which sits at ~260MiB just on its own).

* Also remove the shell (by default)

* Fix a remote build awareness warning

* Re-enable shell for now

Author: nightkr

template/default.nix

@@ -36,45 +36,90 @@
     # We're only using this for dev builds at the moment,
     # so don't pay for release optimization.
     release = false;
-    defaultCrateOverrides = pkgs.defaultCrateOverrides // {
-      prost-build = attrs: {
-        buildInputs = [ pkgs.protobuf ];
-      };
-      tonic-reflection = attrs: {
-        buildInputs = [ pkgs.rustfmt ];
-      };
-      csi-grpc = attrs: {
-        nativeBuildInputs = [ pkgs.protobuf ];
-      };
-      stackable-secret-operator = attrs: {
-        buildInputs = [ pkgs.protobuf pkgs.rustfmt ];
-      };
-      stackable-opa-user-info-fetcher = attrs: {
-        # TODO: why is this not pulled in via libgssapi-sys?
-        buildInputs = [ pkgs.krb5 ];
-      };
-      krb5-sys = attrs: {
-        nativeBuildInputs = [ pkgs.pkg-config ];
-        buildInputs = [ pkgs.krb5 ];
-        LIBCLANG_PATH = "${pkgs.libclang.lib}/lib";
-        # Clang's resource directory is located at ${pkgs.clang.cc.lib}/lib/clang/<version>.
-        # Starting with Clang 16, only the major version is used for the resource directory,
-        # whereas the full version was used in prior Clang versions (see
-        # https://github.com/llvm/llvm-project/commit/e1b88c8a09be25b86b13f98755a9bd744b4dbf14).
-        # The clang wrapper ${pkgs.clang} provides a symlink to the resource directory, which
-        # we use instead.
-        BINDGEN_EXTRA_CLANG_ARGS = "-I${pkgs.glibc.dev}/include -I${pkgs.clang}/resource-root/include";
-      };
-      libgssapi-sys = attrs: {
-        buildInputs = [ pkgs.krb5 ];
-        LIBCLANG_PATH = "${pkgs.libclang.lib}/lib";
-        BINDGEN_EXTRA_CLANG_ARGS = "-I${pkgs.glibc.dev}/include -I${pkgs.clang}/resource-root/include";
+
+    buildRustCrateForPkgs = pkgs: attrs: pkgs.buildRustCrate.override {
+      # Consider migrating to mold for faster linking, but in my (@nightkr's)
+      # quick testing so far it actually seems to perform slightly worse than
+      # the default one.
+      # stdenv = pkgs.stdenvAdapters.useMoldLinker pkgs.stdenv;
+
+      defaultCrateOverrides = pkgs.defaultCrateOverrides // {
+        # Attributes applied here apply to a single crate
+
+        prost-build = attrs: {
+          buildInputs = [ pkgs.protobuf ];
+        };
+        tonic-reflection = attrs: {
+          buildInputs = [ pkgs.rustfmt ];
+        };
+        csi-grpc = attrs: {
+          nativeBuildInputs = [ pkgs.protobuf ];
+        };
+        stackable-secret-operator = attrs: {
+          buildInputs = [ pkgs.protobuf pkgs.rustfmt ];
+        };
+        stackable-opa-user-info-fetcher = attrs: {
+          # TODO: why is this not pulled in via libgssapi-sys?
+          buildInputs = [ pkgs.krb5 ];
+        };
+        krb5-sys = attrs: {
+          nativeBuildInputs = [ pkgs.pkg-config ];
+          buildInputs = [ pkgs.krb5 ];
+          LIBCLANG_PATH = "${pkgs.libclang.lib}/lib";
+          # Clang's resource directory is located at ${pkgs.clang.cc.lib}/lib/clang/<version>.
+          # Starting with Clang 16, only the major version is used for the resource directory,
+          # whereas the full version was used in prior Clang versions (see
+          # https://github.com/llvm/llvm-project/commit/e1b88c8a09be25b86b13f98755a9bd744b4dbf14).
+          # The clang wrapper ${pkgs.clang} provides a symlink to the resource directory, which
+          # we use instead.
+          BINDGEN_EXTRA_CLANG_ARGS = "-I${pkgs.glibc.dev}/include -I${pkgs.clang}/resource-root/include";
+        };
+        libgssapi-sys = attrs: {
+          buildInputs = [ pkgs.krb5 ];
+          LIBCLANG_PATH = "${pkgs.libclang.lib}/lib";
+          BINDGEN_EXTRA_CLANG_ARGS = "-I${pkgs.glibc.dev}/include -I${pkgs.clang}/resource-root/include";
+        };
       };
-    };
+    } (attrs // {
+      # Attributes applied here apply to all built crates
+      # Note that these *take precedence over* per-crate overrides
+
+      dontStrip = !strip;
+
+      extraRustcOpts = [
+        "-C debuginfo=${toString debuginfo}"
+        # Enabling optimization shrinks the binaries further, but also *vastly*
+        # increases the build time.
+        # "-C opt-level=3"
+      ] ++ attrs.extraRustcOpts;
+
+      # Parallel codegen allows Rustc to use more cores.
+      # This should help speed up compiling "bottleneck" crates that Nix can't
+      # parallelize (like the operator binary itself).
+      codegenUnits = 32;
+    });
   }
 , meta ? pkgsLocal.lib.importJSON ./nix/meta.json
 , dockerName ? "oci.stackable.tech/sandbox/${meta.operator.name}"
 , dockerTag ? null
+# Controls the amount of debug information included in the built operator binaries,
+# see https://doc.rust-lang.org/rustc/codegen-options/index.html#debuginfo
+# For comparison, `cargo build --release` defaults to 0, and the debug profile
+# (no `--release`) defaults to 2.
+# see https://doc.rust-lang.org/cargo/reference/profiles.html#debug
+# Set to 2 if you want to run a debugger, but note that it bloats the Docker
+# images *significantly* (hundreds of megabytes).
+, debuginfo ? 0
+# Strip operator binaries if we don't include debuginfo, because *something*
+# still something still includes a reference to gcc (~230MiB), causing it to be
+# added to the docker images.
+, strip ? if debuginfo == 0 then true else false
+# We normally don't include a shell in the (dev) operator images, but it can be
+# enabled by enabling this flag.
+# TODO(@nightkr): Re-enabled for now, since some operators ship with bash init
+# scripts (like secret-operator's CSI path migration job). Consider either
+# removing them or integrating them into the main operator binary instead.
+, includeShell ? true
 }:
 rec {
   inherit cargo sources pkgsLocal pkgsTarget meta;
@@ -96,14 +141,14 @@ rec {
     name = dockerName;
     tag = dockerTag;
     contents = [
-      # Common debugging tools
-      pkgsTarget.bashInteractive
-      pkgsTarget.coreutils
-      pkgsTarget.util-linuxMinimal
       # Kerberos 5 must be installed globally to load plugins correctly
       pkgsTarget.krb5
       # Make the whole cargo workspace available on $PATH
       build
+    ] ++ lib.optional includeShell [
+      pkgsTarget.bashInteractive
+      pkgsTarget.coreutils
+      pkgsTarget.util-linuxMinimal
     ];
     config = {
       Env =
@@ -156,6 +201,6 @@ rec {
     # (see https://github.com/pre-commit/pre-commit-hooks?tab=readme-ov-file#trailing-whitespace).
     # So, remove the trailing newline already here to avoid that an
     # unnecessary change is shown in Git.
-    ${pkgs.gnused}/bin/sed -i '$d' Cargo.nix
+    ${pkgsLocal.gnused}/bin/sed -i '$d' Cargo.nix
   '';
 }