Migrate all rake tasks to use rbwasm build

Author: kateinoigakukun

Rakefile

@@ -8,78 +8,14 @@ require "ruby_wasm/rake_task"
 
 Dir.glob("tasks/**.rake").each { |f| import f }
 
-BUILD_SOURCES = {
-  "head" => {
-    type: "github",
-    repo: "ruby/ruby",
-    rev: "master",
-    patches: Dir["./patches/*.patch"].map { |p| File.expand_path(p) }
-  },
-  "3.3" => {
-    type: "tarball",
-    url: "https://cache.ruby-lang.org/pub/ruby/3.3/ruby-3.3.0.tar.gz"
-  },
-  "3.2" => {
-    type: "tarball",
-    url: "https://cache.ruby-lang.org/pub/ruby/3.2/ruby-3.2.2.tar.gz"
-  }
-}
-
-# Respect revisions specified in build_manifest.json, which is usually generated on GitHub Actions.
-if File.exist?("build_manifest.json")
-  begin
-    manifest = JSON.parse(File.read("build_manifest.json"))
-    manifest["ruby_revisions"].each do |name, rev|
-      BUILD_SOURCES[name][:rev] = rev
-    end
-  rescue StandardError
-    $stderr.puts "Failed to load build_manifest.json"
-  end
-end
-
-FULL_EXTS =
-  "bigdecimal,cgi/escape,continuation,coverage,date,dbm,digest/bubblebabble,digest,digest/md5,digest/rmd160,digest/sha1,digest/sha2,etc,fcntl,fiber,gdbm,json,json/generator,json/parser,nkf,objspace,pathname,psych,racc/cparse,rbconfig/sizeof,ripper,stringio,strscan,monitor,zlib,openssl"
+BUILD_SOURCES = ["3.3", "3.2", "head"]
+BUILD_PROFILES = ["full", "minimal"]
 
-BUILD_PROFILES = {
-  "minimal" => {
-    debug: false,
-    default_exts: "",
-    user_exts: []
-  },
-  "minimal-debug" => {
-    debug: true,
-    default_exts: "",
-    user_exts: []
-  },
-  "full" => {
-    debug: false,
-    default_exts: FULL_EXTS,
-    user_exts: []
-  },
-  "full-debug" => {
-    debug: true,
-    default_exts: FULL_EXTS,
-    user_exts: []
-  },
-  "full-js-debug" => {
-    debug: true,
-    default_exts: FULL_EXTS,
-    user_exts: %w[js witapi]
-  }
-}
-
-BUILDS =
-  %w[wasm32-unknown-wasi wasm32-unknown-emscripten]
-    .product(BUILD_SOURCES.keys, BUILD_PROFILES.keys)
-    .select do |target, _, profile_name|
-      if target == "wasm32-unknown-emscripten"
-        # Builds only full for Emscripten since minimal, js, debug
-        # builds are rarely used with Emscripten.
-        next profile_name == "full"
-      end
-      next true
-    end
-    .map { |t, s, p| { src: s, target: t, profile: p } }
+BUILDS = BUILD_SOURCES.product(BUILD_PROFILES).map do |src, profile|
+  [src, "wasm32-unknown-wasi", profile]
+end + BUILD_SOURCES.map do |src|
+  [src, "wasm32-unknown-emscripten", "full"]
+end
 
 NPM_PACKAGES = [
   {
@@ -117,45 +53,19 @@ STANDALONE_PACKAGES = [
 LIB_ROOT = File.dirname(__FILE__)
 
 TOOLCHAINS = {}
+BUILDS.map { |_, target, _| target }.uniq.each do |target|
+  toolchain = RubyWasm::Toolchain.get(target)
+  TOOLCHAINS[toolchain.name] = toolchain
+end
 
 namespace :build do
   BUILD_TASKS =
-    BUILDS.map do |params|
-      name = "#{params[:src]}-#{params[:target]}-#{params[:profile]}"
-      source = BUILD_SOURCES[params[:src]].merge(name: params[:src])
-      profile = BUILD_PROFILES[params[:profile]]
-      options = {
-        src: source,
-        target: params[:target],
-        default_exts: profile[:default_exts]
-      }
-      debug = profile[:debug]
-      RubyWasm::BuildTask.new(name, **options) do |t|
-        if debug
-          t.crossruby.debugflags = %w[-g]
-          t.crossruby.wasmoptflags = %w[-O3 -g]
-          t.crossruby.ldflags = %w[
-            -Xlinker
-            --stack-first
-            -Xlinker
-            -z
-            -Xlinker
-            stack-size=16777216
-          ]
-        else
-          t.crossruby.debugflags = %w[-g0]
-          t.crossruby.ldflags = %w[-Xlinker -zstack-size=16777216]
-        end
-
-        toolchain = t.toolchain
-        t.crossruby.user_exts =
-          profile[:user_exts].map do |ext|
-            srcdir = File.join(LIB_ROOT, "packages", "gems", "js", "ext", ext)
-            RubyWasm::CrossRubyExtProduct.new(srcdir, toolchain)
-          end
-        unless TOOLCHAINS.key? toolchain.name
-          TOOLCHAINS[toolchain.name] = toolchain
-        end
+    BUILDS.map do |src, target, profile|
+      name = "#{src}-#{target}-#{profile}"
+
+      desc "Cross-build Ruby for #{@target}"
+      task name do
+        sh *["exe/rbwasm", "build", "--ruby-version", src, "--target", target, "--build-profile", profile, "-o", "/dev/null"]
       end
     end
 

lib/ruby_wasm/cli.rb

@@ -45,7 +45,9 @@ def build(args)
         save_temps: false,
         optimize: false,
         remake: false,
+        ruby_version: "3.3",
         target_triplet: "wasm32-unknown-wasi",
+        profile: "full",
         stdlib: true
       }
       OptionParser
@@ -60,10 +62,18 @@ def build(args)
             options[:save_temps] = true
           end
 
+          opts.on("--ruby-version VERSION", "Ruby version") do |version|
+            options[:ruby_version] = version
+          end
+
           opts.on("--target TRIPLET", "Target triplet") do |triplet|
             options[:target_triplet] = triplet
           end
 
+          opts.on("--build-profile PROFILE", "Build profile. full or minimal") do |profile|
+            options[:profile] = profile
+          end
+
           opts.on("--optimize", "Optimize the output") do
             options[:optimize] = true
           end
@@ -85,28 +95,54 @@ def build(args)
       verbose = RubyWasm.logger.level == :debug
       executor = RubyWasm::BuildExecutor.new(verbose: verbose)
 
+      unless options[:output]
+        @stderr.puts "Output file is not specified"
+        exit 1
+      end
+
       require "tmpdir"
 
       if options[:save_temps]
         tmpdir = Dir.mktmpdir
         self.do_build(executor, tmpdir, options)
-        @stdout.puts "Temporary files are saved to #{tmpdir}"
+        @stderr.puts "Temporary files are saved to #{tmpdir}"
         exit
       else
         Dir.mktmpdir { |tmpdir| self.do_build(executor, tmpdir, options) }
       end
     end
 
-    private def do_build(executor, tmp_dir, options)
-      packager = RubyWasm::Packager.new(tmp_dir, options[:target_triplet])
-      wasm_bytes = packager.package(executor, options)
-      @stdout.puts "Size: #{SizeFormatter.format(wasm_bytes.size)}"
-      if options[:output]
-        File.binwrite(options[:output], wasm_bytes.pack("C*"))
-        @stdout.puts "Wrote #{options[:output]}"
+    private
+
+    def build_config(options)
+      config = {
+        target: options[:target_triplet],
+        src: options[:ruby_version],
+      }
+      case options[:profile]
+      when "full"
+        config[:profile] = RubyWasm::Packager::ALL_DEFAULT_EXTS
+      when "minimal"
+        config[:profile] = ""
       else
-        @stderr.puts "Writing to stdout"
+        RubyWasm.logger.error "Unknown profile: #{options[:profile]} (available: full, minimal)"
+        exit 1
+      end
+    end
+
+    def do_build(executor, tmp_dir, options)
+      if defined?(Bundler)
+        definition = Bundler.definition
+      end
+      packager = RubyWasm::Packager.new(tmp_dir, options[:target_triplet], definition)
+      wasm_bytes = packager.package(executor, options)
+      @stderr.puts "Size: #{SizeFormatter.format(wasm_bytes.size)}"
+      case options[:output]
+      when "-"
         @stdout.write wasm_bytes.pack("C*")
+      else
+        File.binwrite(options[:output], wasm_bytes.pack("C*"))
+        @stderr.puts "Wrote #{options[:output]}"
       end
     end
   end

lib/ruby_wasm/packager.rb

@@ -1,10 +1,21 @@
+# A class responsible for packaging whole Ruby project
 class RubyWasm::Packager
-  def initialize(dest_dir, target_triplet, definition = Bundler.definition)
+  # Initializes a new instance of the RubyWasm::Packager class.
+  #
+  # @param dest_dir [String] The destination used to construct the filesystem.
+  # @param config [Hash] The build config used for building Ruby.
+  # @param definition [Bundler::Definition] The Bundler definition.
+  def initialize(dest_dir, config = nil, definition = nil)
     @dest_dir = dest_dir
     @definition = definition
-    @target_triplet = target_triplet
+    @config = config
   end
 
+  # Packages the Ruby code into a Wasm binary. (including extensions)
+  #
+  # @param executor [RubyWasm::BuildExecutor] The executor for building the Wasm binary.
+  # @param options [Hash] The packaging options.
+  # @return [Array<Integer>] The bytes of the packaged Wasm binary.
   def package(executor, options)
     ruby_core = RubyWasm::Packager::Core.new(self)
     tarball = ruby_core.build(executor, options)
@@ -29,42 +40,111 @@ def package(executor, options)
     wasm_bytes
   end
 
+  # The list of excluded gems from the Bundler definition.
   EXCLUDED_GEMS = %w[ruby_wasm bundler]
 
+  # Retrieves the specs from the Bundler definition, excluding the excluded gems.
   def specs
+    return [] unless @definition
     @definition.specs.reject { |spec| EXCLUDED_GEMS.include?(spec.name) }
   end
 
+  # Checks if dynamic linking is supported.
   def support_dynamic_linking?
     @ruby_channel == "head"
   end
 
+  # Retrieves the root directory of the Ruby project.
+  # The root directory contains the following stuff:
+  #  * patches/*.patch
+  #  * build_manifest.json
+  #  * rubies
+  #  * build
   def root
     @root ||=
       begin
         if explicit = ENV["RUBY_WASM_ROOT"]
           File.expand_path(explicit)
-        else
+        elsif defined?(Bundler)
           Bundler.root
+        else
+          Dir.pwd
         end
       rescue Bundler::GemfileNotFound
         Dir.pwd
       end
   end
 
-  def build_options
-    {
-      target: @target_triplet,
-      src: {
-        name: "3.3",
+  # Retrieves the alias definitions for the Ruby sources.
+  def build_source_aliases
+    patches = Dir[File.join(root, "patches", "*.patch")]
+    sources = {
+      "head" => {
+        type: "github",
+        repo: "ruby/ruby",
+        rev: "master",
+        patches: patches.map { |p| File.expand_path(p) }
+      },
+      "3.3" => {
         type: "tarball",
         url: "https://cache.ruby-lang.org/pub/ruby/3.3/ruby-3.3.0.tar.gz"
       },
-      default_exts:
-        "bigdecimal,cgi/escape,continuation,coverage,date,dbm,digest/bubblebabble,digest,digest/md5,digest/rmd160,digest/sha1,digest/sha2,etc,fcntl,fiber,gdbm,json,json/generator,json/parser,nkf,objspace,pathname,psych,racc/cparse,rbconfig/sizeof,ripper,stringio,strscan,monitor,zlib,openssl",
-      toolchain: RubyWasm::Toolchain.get(@target_triplet),
-      build_dir: File.join(root, "build"),
-      rubies_dir: File.join(root, "rubies")
+      "3.2" => {
+        type: "tarball",
+        url: "https://cache.ruby-lang.org/pub/ruby/3.2/ruby-3.2.2.tar.gz"
+      }
+    }
+    sources.each do |name, source|
+      source[:name] = name
+    end
+
+    build_manifest = File.join(root, "build_manifest.json")
+    if File.exist?(build_manifest)
+      begin
+        manifest = JSON.parse(File.read(build_manifest))
+        manifest["ruby_revisions"].each do |name, rev|
+          sources[name][:rev] = rev
+        end
+      rescue StandardError => e
+        RubyWasm.logger.warn "Failed to load build_manifest.json: #{e}"
+      end
+    end
+    sources
+  end
+
+  ALL_DEFAULT_EXTS = "bigdecimal,cgi/escape,continuation,coverage,date,dbm,digest/bubblebabble,digest,digest/md5,digest/rmd160,digest/sha1,digest/sha2,etc,fcntl,fiber,gdbm,json,json/generator,json/parser,nkf,objspace,pathname,psych,racc/cparse,rbconfig/sizeof,ripper,stringio,strscan,monitor,zlib,openssl"
+
+  # Retrieves the build options used for building Ruby itself.
+  def build_options
+    default = {
+      target: "wasm32-unknown-wasi",
+      src: "3.3",
+      default_exts: ALL_DEFAULT_EXTS,
     }
+    override = {}
+    if @config
+      override = @config["build_options"] || {}
+    end
+    # Merge the default options with the config options
+    default.merge(override)
+  end
+
+  # Retrieves the resolved build options
+  def full_build_options
+    options = build_options
+    toolchain = RubyWasm::Toolchain.get(options[:target])
+    build_dir = File.join(root, "build")
+    rubies_dir = File.join(root, "rubies")
+    src = if options[:src].is_a?(Hash)
+      options[:src]
+    else
+      src_name = options[:src]
+      aliases = build_source_aliases
+      aliases[src_name] || raise("Unknown Ruby source: #{src_name} (available: #{aliases.keys.join(", ")})")
+    end
+    options.merge(
+      toolchain: toolchain, build_dir: build_dir,
+      rubies_dir: rubies_dir, src: src
+    )
   end
 end