From 3b6e7b10c5427697648635394c0aaec77fd47218 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Fri, 15 Nov 2024 10:26:21 +0100 Subject: [PATCH 001/175] Add github workflow --- .github/workflows/ci.yml | 51 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000000..2b448f8ee8bf --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,51 @@ +name: CI + +# Trigger the workflow on push or pull request, but only for the master branch +on: + pull_request: + types: + - opened + - synchronize + push: + branches: [master] + +jobs: + cabal: + name: ${{ matrix.os }} / ghc ${{ matrix.ghc }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + ghc: ['9.6.5'] # bootstrapping compiler + + steps: + - uses: actions/checkout@v4 + with: + submodules: "recursive" + + - uses: haskell-actions/setup@v2 + id: setup + name: Setup Haskell tools + with: + ghc-version: ${{ matrix.ghc }} + cabal-version: "latest" + cabal-update: true + + - name: Install Alex and Happy + run: | + cabal install alex + cabal install happy + + - name: Configure the build + run: | + ./boot + ./configure + + - name: Build Hadrian + run: | + ./hadrian/build --version + + - name: Build the bindist + run: | + ./hadrian/build --flavour=release -j binary-dist-dir --docs=none From 7ef5f050266e1b41f995b1c0fe2ee1080d187251 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 27 Nov 2024 12:19:19 +0100 Subject: [PATCH 002/175] Build GHC with cabal-install and a Makefile --- Makefile | 50 ++++++++++++++++++++++++++++++++++++++++++++ cabal.project-stage0 | 41 ++++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 Makefile create mode 100644 cabal.project-stage0 diff --git a/Makefile b/Makefile new file mode 100644 index 000000000000..822889ee63c1 --- /dev/null +++ b/Makefile @@ -0,0 +1,50 @@ + +build: + rm -rf _build + + # Preparing source files... + mkdir -p _build/stage0/src/ + cp -rf ./libraries _build/stage0/src/ + cp -rf ./compiler _build/stage0/src/libraries/ghc + cp -rf ./ghc _build/stage0/src/ghc-bin + cp -rf ./utils _build/stage0/src/ + + ## Substituting variables + cp _build/stage0/src/ghc-bin/ghc-bin.cabal{.in,} + cp _build/stage0/src/libraries/ghc/ghc.cabal{.in,} + cp _build/stage0/src/libraries/ghc/GHC/CmmToLlvm/Version/Bounds.hs{.in,} + cp _build/stage0/src/libraries/ghc-boot/ghc-boot.cabal{.in,} + cp _build/stage0/src/libraries/ghc-boot-th/ghc-boot-th.cabal{.in,} + cp _build/stage0/src/libraries/ghc-heap/ghc-heap.cabal{.in,} + cp _build/stage0/src/libraries/ghci/ghci.cabal{.in,} + cp _build/stage0/src/utils/ghc-pkg/ghc-pkg.cabal{.in,} + + sed -i 's/@ProjectVersion@/9.13/' _build/stage0/src/ghc-bin/ghc-bin.cabal + sed -i 's/@ProjectVersionMunged@/9.13/' _build/stage0/src/ghc-bin/ghc-bin.cabal + sed -i 's/@ProjectVersion@/9.13/' _build/stage0/src/libraries/ghc/ghc.cabal + sed -i 's/@ProjectVersionMunged@/9.13/' _build/stage0/src/libraries/ghc/ghc.cabal + sed -i 's/@ProjectVersion@/9.13/' _build/stage0/src/libraries/ghc-boot/ghc-boot.cabal + sed -i 's/@ProjectVersionMunged@/9.13/' _build/stage0/src/libraries/ghc-boot/ghc-boot.cabal + sed -i 's/@ProjectVersion@/9.13/' _build/stage0/src/libraries/ghc-boot-th/ghc-boot-th.cabal + sed -i 's/@ProjectVersionMunged@/9.13/' _build/stage0/src/libraries/ghc-boot-th/ghc-boot-th.cabal + sed -i 's/@Suffix@//' _build/stage0/src/libraries/ghc-boot-th/ghc-boot-th.cabal + sed -i 's/@SourceRoot@/./' _build/stage0/src/libraries/ghc-boot-th/ghc-boot-th.cabal + sed -i 's/@ProjectVersion@/9.13/' _build/stage0/src/libraries/ghc-heap/ghc-heap.cabal + sed -i 's/@ProjectVersionMunged@/9.13/' _build/stage0/src/libraries/ghc-heap/ghc-heap.cabal + sed -i 's/@ProjectVersionForLib@/9.13/' _build/stage0/src/libraries/ghc-heap/ghc-heap.cabal + sed -i 's/@ProjectVersion@/9.13/' _build/stage0/src/libraries/ghci/ghci.cabal + sed -i 's/@ProjectVersionMunged@/9.13/' _build/stage0/src/libraries/ghci/ghci.cabal + sed -i 's/@ProjectVersionForLib@/9.13/' _build/stage0/src/libraries/ghci/ghci.cabal + sed -i 's/@ProjectVersion@/9.13/' _build/stage0/src/utils/ghc-pkg/ghc-pkg.cabal + sed -i 's/@ProjectVersionMunged@/9.13/' _build/stage0/src/utils/ghc-pkg/ghc-pkg.cabal + sed -i 's/@ProjectVersionForLib@/9.13/' _build/stage0/src/utils/ghc-pkg/ghc-pkg.cabal + + sed -i 's/@LlvmMinVersion@/13/' _build/stage0/src/libraries/ghc/GHC/CmmToLlvm/Version/Bounds.hs + sed -i 's/@LlvmMaxVersion@/20/' _build/stage0/src/libraries/ghc/GHC/CmmToLlvm/Version/Bounds.hs + + # Building... + mkdir -p _build/stage0/cabal/ + mkdir -p _build/stage0/bin/ + cabal configure --project-file=cabal.project-stage0 + cabal build --project-file=cabal.project-stage0 ghc-bin:ghc -j --builddir=_build/stage0/cabal/ + cabal install --project-file=cabal.project-stage0 ghc-bin:ghc -j --builddir=_build/stage0/cabal/ --installdir=_build/stage0/bin --overwrite-policy=always --install-method=copy diff --git a/cabal.project-stage0 b/cabal.project-stage0 new file mode 100644 index 000000000000..67100ab90e57 --- /dev/null +++ b/cabal.project-stage0 @@ -0,0 +1,41 @@ +packages: + ./_build/stage0/src/ghc-bin/ + ./_build/stage0/src/libraries/ghc + ./_build/stage0/src/libraries/directory/ + ./_build/stage0/src/libraries/file-io/ + ./_build/stage0/src/libraries/filepath/ + ./_build/stage0/src/libraries/ghc-platform/ + ./_build/stage0/src/libraries/ghc-boot/ + ./_build/stage0/src/libraries/ghc-boot-th/ + ./_build/stage0/src/libraries/ghc-heap + ./_build/stage0/src/libraries/ghci + ./_build/stage0/src/libraries/os-string/ + ./_build/stage0/src/libraries/process/ + ./_build/stage0/src/libraries/semaphore-compat + ./_build/stage0/src/libraries/time + ./_build/stage0/src/libraries/unix/ + ./_build/stage0/src/libraries/Win32/ + ./_build/stage0/src/utils/ghc-pkg + ./_build/stage0/src/utils/hsc2hs + ./_build/stage0/src/utils/unlit + ./_build/stage0/src/utils/genprimopcode/ + ./_build/stage0/src/utils/deriveConstants/ + +benchmarks: False +tests: False +allow-boot-library-installs: True + +program-options + ghc-options: -fhide-source-paths + +package * + library-vanilla: True + shared: False + executable-profiling: False + executable-dynamic: False + ghc-options: -fhide-source-paths + +package ghc-boot-th + flags: +bootstrap + +allow-newer: ghc-boot-th From a03c95cb3d0abec0cb95a085fca583fcfe47ff13 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 27 Nov 2024 15:01:30 +0100 Subject: [PATCH 003/175] Generate proper settings for ghc-boot --- GenSettings.hs | 26 ++++++++++++++++++++++++++ Makefile | 6 ++++-- cabal.project-stage0 | 5 +++-- 3 files changed, 33 insertions(+), 4 deletions(-) create mode 100644 GenSettings.hs diff --git a/GenSettings.hs b/GenSettings.hs new file mode 100644 index 000000000000..b380a5ba8593 --- /dev/null +++ b/GenSettings.hs @@ -0,0 +1,26 @@ +module Main where + +import Data.Maybe + +main :: IO () +main = interact update_settings + +-- | Update settings using those of the boot compilers +update_settings :: String -> String +update_settings input = output + where + output = show out_settings + + in_settings,out_settings :: [(String,String)] + in_settings = read input + + out_settings = + [ ("hostPlatformArch", fromMaybe (error "Couldn't read 'target arch' setting") (lookup "target arch" in_settings)) + , ("hostPlatformOS", fromMaybe (error "Couldn't read 'target os' setting") (lookup "target os" in_settings)) + , ("cProjectGitCommitId", "DEADBEEF") -- FIXME + , ("cProjectVersion", "9.13") + , ("cProjectVersionInt", "913") + , ("cProjectPatchLevel", "0") + , ("cProjectPatchLevel1", "0") + , ("cProjectPatchLevel2", "0") + ] diff --git a/Makefile b/Makefile index 822889ee63c1..129e9d404447 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,5 @@ +HADRIAN_SETTINGS_STAGE0 := $(shell ghc --info | runghc GenSettings.hs) + build: rm -rf _build @@ -46,5 +48,5 @@ build: mkdir -p _build/stage0/cabal/ mkdir -p _build/stage0/bin/ cabal configure --project-file=cabal.project-stage0 - cabal build --project-file=cabal.project-stage0 ghc-bin:ghc -j --builddir=_build/stage0/cabal/ - cabal install --project-file=cabal.project-stage0 ghc-bin:ghc -j --builddir=_build/stage0/cabal/ --installdir=_build/stage0/bin --overwrite-policy=always --install-method=copy + HADRIAN_SETTINGS='$(HADRIAN_SETTINGS_STAGE0)' \ + cabal install --project-file=cabal.project-stage0 ghc-bin:ghc -j --builddir=_build/stage0/cabal/ --installdir=_build/stage0/bin --overwrite-policy=always --install-method=copy diff --git a/cabal.project-stage0 b/cabal.project-stage0 index 67100ab90e57..0e4bff4838b8 100644 --- a/cabal.project-stage0 +++ b/cabal.project-stage0 @@ -26,14 +26,15 @@ tests: False allow-boot-library-installs: True program-options - ghc-options: -fhide-source-paths + ghc-options: -fhide-source-paths -j package * library-vanilla: True shared: False executable-profiling: False executable-dynamic: False - ghc-options: -fhide-source-paths + executable-static: False + ghc-options: -fhide-source-paths -j package ghc-boot-th flags: +bootstrap From b9d460fe5acc297b737802f76c1d416882219d85 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 27 Nov 2024 15:06:38 +0100 Subject: [PATCH 004/175] Build stage0 tools: ghc-pkg, deriveConstants, genprimopcode --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 129e9d404447..902371a6e76a 100644 --- a/Makefile +++ b/Makefile @@ -49,4 +49,6 @@ build: mkdir -p _build/stage0/bin/ cabal configure --project-file=cabal.project-stage0 HADRIAN_SETTINGS='$(HADRIAN_SETTINGS_STAGE0)' \ - cabal install --project-file=cabal.project-stage0 ghc-bin:ghc -j --builddir=_build/stage0/cabal/ --installdir=_build/stage0/bin --overwrite-policy=always --install-method=copy + cabal install --project-file=cabal.project-stage0 \ + ghc-bin:ghc ghc-pkg:ghc-pkg genprimopcode:genprimopcode deriveConstants:deriveConstants \ + -j --builddir=_build/stage0/cabal/ --installdir=_build/stage0/bin --overwrite-policy=always --install-method=copy From af7a7cec20f2c0bb9a53923233d983c2d68e6b65 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 27 Nov 2024 17:10:54 +0100 Subject: [PATCH 005/175] Generate proper stage1 settings --- GenSettings.hs | 101 +++++++++++++++++++++++++++++++++++++++++++++++-- Makefile | 19 ++++++++-- 2 files changed, 113 insertions(+), 7 deletions(-) diff --git a/GenSettings.hs b/GenSettings.hs index b380a5ba8593..8aebd6dfc064 100644 --- a/GenSettings.hs +++ b/GenSettings.hs @@ -1,13 +1,18 @@ module Main where import Data.Maybe +import System.Environment main :: IO () -main = interact update_settings +main = do + args <- getArgs + case args of + ["ghc-boot"] -> interact ghcboot_settings + ["stage1"] -> interact stage1_settings --- | Update settings using those of the boot compilers -update_settings :: String -> String -update_settings input = output +-- | Generate HADRIAN_SETTINGS for ghc-boot's Setup.hs, based on given settings +ghcboot_settings :: String -> String +ghcboot_settings input = output where output = show out_settings @@ -24,3 +29,91 @@ update_settings input = output , ("cProjectPatchLevel1", "0") , ("cProjectPatchLevel2", "0") ] + +-- | Generate settings for stage1 compiler, based on given settings (stage0's +-- compiler settings) +stage1_settings :: String -> String +stage1_settings input = output + where + output = show out_settings + + in_settings,out_settings :: [(String,String)] + in_settings = read input + + -- keep the previous setting, fail if it doesn't exist + keep_fail s = keep_def s (error ("Couldn't find setting "<> show s)) + + -- keep the previous setting, default to the given value if it doesn't exist + keep_def s d = case lookup s in_settings of + Nothing -> (s,d) + Just v -> (s,v) + + -- use the previous setting, or if it doesn't exist use the setting for the + -- second key. Fail if both don't exist. This is useful to support + -- bootstrapping with old compilers that mingled some settings. + keep_or_fail s s2 = case lookup s in_settings of + Nothing -> case lookup s2 in_settings of + Nothing -> error ("Couldn't find any of " <> show s <> " and " <> show s2) + Just v -> (s,v) + Just v -> (s,v) + + + out_settings = + [ keep_fail "C compiler command" + , keep_fail "C compiler flags" + , keep_fail "C++ compiler command" + , keep_fail "C++ compiler flags" + , keep_fail "C compiler link flags" + , keep_fail "C compiler supports -no-pie" + , keep_or_fail "CPP command" "Haskell CPP command" + , keep_or_fail "CPP flags" "Haskell CPP flags" + , keep_fail "Haskell CPP command" + , keep_fail "Haskell CPP flags" + , keep_or_fail "JavaScript CPP command" "Haskell CPP command" + , keep_or_fail "JavaScript CPP flags" "Haskell CPP flags" + , keep_or_fail "C-- CPP command" "Haskell CPP command" + , keep_or_fail "C-- CPP flags" "Haskell CPP flags" + , keep_def "C-- CPP supports -g0" "NO" + , keep_fail "ld supports compact unwind" + , keep_fail "ld supports filelist" + , keep_fail "ld supports single module" + , keep_fail "ld is GNU ld" + , keep_fail "Merge objects command" + , keep_fail "Merge objects flags" + , keep_def "Merge objects supports response files" "NO" + , keep_fail "ar command" + , keep_fail "ar flags" + , keep_fail "ar supports at file" + , keep_fail "ar supports -L" + , keep_fail "ranlib command" + , keep_fail "otool command" + , keep_fail "install_name_tool command" + , keep_fail "windres command" + , keep_fail "unlit command" + , keep_fail "cross compiling" + , keep_fail "target platform string" + , keep_fail "target os" + , keep_fail "target arch" + , keep_fail "target word size" + , keep_fail "target word big endian" + , keep_fail "target has GNU nonexec stack" + , keep_fail "target has .ident directive" + , keep_fail "target has subsections via symbols" + , keep_fail "target has libm" + , keep_fail "Unregisterised" + , keep_fail "LLVM target" + , keep_fail "LLVM llc command" + , keep_fail "LLVM opt command" + , keep_def "LLVM llvm-as command" "llvm-as" + , keep_fail "Use inplace MinGW toolchain" + + , keep_def "target RTS linker only supports shared libraries" "NO" + , ("Use interpreter", "NO") + , keep_fail "Support SMP" + , keep_fail "RTS ways" + , keep_fail "Tables next to code" + , keep_fail "Leading underscore" + , keep_fail "Use LibFFI" + , keep_fail "RTS expects libdw" + , ("Relative Global Package DB", "_build/stage1/pkgs") + ] diff --git a/Makefile b/Makefile index 902371a6e76a..2d5e1abea297 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,10 @@ -HADRIAN_SETTINGS_STAGE0 := $(shell ghc --info | runghc GenSettings.hs) +HADRIAN_SETTINGS_STAGE0 := $(shell ghc --info | runghc GenSettings.hs ghc-boot) +SETTINGS_STAGE1 := $(shell ghc --info | runghc GenSettings.hs stage1) build: rm -rf _build - + # Preparing source files... mkdir -p _build/stage0/src/ cp -rf ./libraries _build/stage0/src/ @@ -48,7 +49,19 @@ build: mkdir -p _build/stage0/cabal/ mkdir -p _build/stage0/bin/ cabal configure --project-file=cabal.project-stage0 + HADRIAN_SETTINGS='$(HADRIAN_SETTINGS_STAGE0)' \ + cabal build --project-file=cabal.project-stage0 \ + ghc-bin:ghc ghc-pkg:ghc-pkg genprimopcode:genprimopcode deriveConstants:deriveConstants \ + -j --builddir=_build/stage0/cabal/ HADRIAN_SETTINGS='$(HADRIAN_SETTINGS_STAGE0)' \ cabal install --project-file=cabal.project-stage0 \ ghc-bin:ghc ghc-pkg:ghc-pkg genprimopcode:genprimopcode deriveConstants:deriveConstants \ - -j --builddir=_build/stage0/cabal/ --installdir=_build/stage0/bin --overwrite-policy=always --install-method=copy + -j --builddir=_build/stage0/cabal/ \ + --installdir=_build/stage0/bin --overwrite-policy=always --install-method=copy + + mkdir -p _build/stage0/lib + mkdir -p _build/stage0/pkgs + echo '$(SETTINGS_STAGE1)' > _build/stage0/lib/settings + + _build/stage0/bin/ghc --version + _build/stage0/bin/ghc --info From 38f96a9d91777bd97949a725397fa3a4cf8f55d5 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Thu, 28 Nov 2024 11:35:48 +0100 Subject: [PATCH 006/175] Fix installation of stage1 programs. WIP using stage1 compiler --- Makefile | 83 ++++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 72 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 2d5e1abea297..d13dc49de34c 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,11 @@ HADRIAN_SETTINGS_STAGE0 := $(shell ghc --info | runghc GenSettings.hs ghc-boot) +HADRIAN_SETTINGS_STAGE1 := $(shell ghc --info | runghc GenSettings.hs ghc-boot) SETTINGS_STAGE1 := $(shell ghc --info | runghc GenSettings.hs stage1) -build: - rm -rf _build +all: _build/stage1/bin/ghc +_build/stage0/bin/ghc: # Preparing source files... mkdir -p _build/stage0/src/ cp -rf ./libraries _build/stage0/src/ @@ -41,7 +42,7 @@ build: sed -i 's/@ProjectVersion@/9.13/' _build/stage0/src/utils/ghc-pkg/ghc-pkg.cabal sed -i 's/@ProjectVersionMunged@/9.13/' _build/stage0/src/utils/ghc-pkg/ghc-pkg.cabal sed -i 's/@ProjectVersionForLib@/9.13/' _build/stage0/src/utils/ghc-pkg/ghc-pkg.cabal - + sed -i 's/@LlvmMinVersion@/13/' _build/stage0/src/libraries/ghc/GHC/CmmToLlvm/Version/Bounds.hs sed -i 's/@LlvmMaxVersion@/20/' _build/stage0/src/libraries/ghc/GHC/CmmToLlvm/Version/Bounds.hs @@ -53,15 +54,75 @@ build: cabal build --project-file=cabal.project-stage0 \ ghc-bin:ghc ghc-pkg:ghc-pkg genprimopcode:genprimopcode deriveConstants:deriveConstants \ -j --builddir=_build/stage0/cabal/ - HADRIAN_SETTINGS='$(HADRIAN_SETTINGS_STAGE0)' \ - cabal install --project-file=cabal.project-stage0 \ - ghc-bin:ghc ghc-pkg:ghc-pkg genprimopcode:genprimopcode deriveConstants:deriveConstants \ - -j --builddir=_build/stage0/cabal/ \ - --installdir=_build/stage0/bin --overwrite-policy=always --install-method=copy - + + # Installing binaries + cp `cabal list-bin --project-file=cabal.project-stage0 --builddir=_build/stage0/cabal/ ghc-bin:ghc` _build/stage0/bin/ghc + cp `cabal list-bin --project-file=cabal.project-stage0 --builddir=_build/stage0/cabal/ ghc-pkg:ghc-pkg` _build/stage0/bin/ghc-pkg + cp `cabal list-bin --project-file=cabal.project-stage0 --builddir=_build/stage0/cabal/ deriveConstants:deriveConstants` _build/stage0/bin/deriveConstants + cp `cabal list-bin --project-file=cabal.project-stage0 --builddir=_build/stage0/cabal/ genprimopcode:genprimopcode` _build/stage0/bin/genprimopcode + + # Generate settings mkdir -p _build/stage0/lib - mkdir -p _build/stage0/pkgs echo '$(SETTINGS_STAGE1)' > _build/stage0/lib/settings - + _build/stage0/bin/ghc --version _build/stage0/bin/ghc --info + + +_build/stage1/bin/ghc: _build/stage0/bin/ghc + rm -rf _build/stage1 + mkdir -p _build/stage1 + + # Initialize empty package db + _build/stage0/bin/ghc-pkg init _build/stage1/pkgs + _build/stage0/bin/ghc-pkg recache --global-package-db=_build/stage1/pkgs --no-user-package-db + + # Preparing source files... + mkdir -p _build/stage1/src/ + cp -rf ./libraries _build/stage1/src/ + cp -rf ./compiler _build/stage1/src/libraries/ghc + cp -rf ./rts _build/stage1/src/libraries/ + cp -rf ./ghc _build/stage1/src/ghc-bin + + ## Substituting variables + cp _build/stage1/src/ghc-bin/ghc-bin.cabal{.in,} + cp _build/stage1/src/libraries/ghc/ghc.cabal{.in,} + cp _build/stage1/src/libraries/ghc/GHC/CmmToLlvm/Version/Bounds.hs{.in,} + cp _build/stage1/src/libraries/ghc-boot/ghc-boot.cabal{.in,} + cp _build/stage1/src/libraries/ghc-boot-th/ghc-boot-th.cabal{.in,} + cp _build/stage1/src/libraries/ghc-heap/ghc-heap.cabal{.in,} + cp _build/stage1/src/libraries/ghci/ghci.cabal{.in,} + + sed -i 's/@ProjectVersion@/9.13/' _build/stage1/src/ghc-bin/ghc-bin.cabal + sed -i 's/@ProjectVersionMunged@/9.13/' _build/stage1/src/ghc-bin/ghc-bin.cabal + sed -i 's/@ProjectVersion@/9.13/' _build/stage1/src/libraries/ghc/ghc.cabal + sed -i 's/@ProjectVersionMunged@/9.13/' _build/stage1/src/libraries/ghc/ghc.cabal + sed -i 's/@ProjectVersion@/9.13/' _build/stage1/src/libraries/ghc-boot/ghc-boot.cabal + sed -i 's/@ProjectVersionMunged@/9.13/' _build/stage1/src/libraries/ghc-boot/ghc-boot.cabal + sed -i 's/@ProjectVersion@/9.13/' _build/stage1/src/libraries/ghc-boot-th/ghc-boot-th.cabal + sed -i 's/@ProjectVersionMunged@/9.13/' _build/stage1/src/libraries/ghc-boot-th/ghc-boot-th.cabal + sed -i 's/@Suffix@//' _build/stage1/src/libraries/ghc-boot-th/ghc-boot-th.cabal + sed -i 's/@SourceRoot@/./' _build/stage1/src/libraries/ghc-boot-th/ghc-boot-th.cabal + sed -i 's/@ProjectVersion@/9.13/' _build/stage1/src/libraries/ghc-heap/ghc-heap.cabal + sed -i 's/@ProjectVersionMunged@/9.13/' _build/stage1/src/libraries/ghc-heap/ghc-heap.cabal + sed -i 's/@ProjectVersionForLib@/9.13/' _build/stage1/src/libraries/ghc-heap/ghc-heap.cabal + sed -i 's/@ProjectVersion@/9.13/' _build/stage1/src/libraries/ghci/ghci.cabal + sed -i 's/@ProjectVersionMunged@/9.13/' _build/stage1/src/libraries/ghci/ghci.cabal + sed -i 's/@ProjectVersionForLib@/9.13/' _build/stage1/src/libraries/ghci/ghci.cabal + + sed -i 's/@LlvmMinVersion@/13/' _build/stage1/src/libraries/ghc/GHC/CmmToLlvm/Version/Bounds.hs + sed -i 's/@LlvmMaxVersion@/20/' _build/stage1/src/libraries/ghc/GHC/CmmToLlvm/Version/Bounds.hs + + # Building boot libraries + mkdir -p _build/stage1/cabal/ + + cabal configure --project-file=cabal.project-stage1 + + HADRIAN_SETTINGS='$(HADRIAN_SETTINGS_STAGE1)' \ + cabal build --project-file=cabal.project-stage1 \ + rts \ + --builddir=_build/stage1/cabal/ -v + +clean: + rm -rf _build + From 4137c767c78567065e9cb677f4b04869b3b36f0f Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Thu, 28 Nov 2024 12:05:09 +0100 Subject: [PATCH 007/175] Fix path to stage1 package db --- GenSettings.hs | 2 +- cabal.project-stage1 | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 cabal.project-stage1 diff --git a/GenSettings.hs b/GenSettings.hs index 8aebd6dfc064..fcd7d6525c72 100644 --- a/GenSettings.hs +++ b/GenSettings.hs @@ -115,5 +115,5 @@ stage1_settings input = output , keep_fail "Leading underscore" , keep_fail "Use LibFFI" , keep_fail "RTS expects libdw" - , ("Relative Global Package DB", "_build/stage1/pkgs") + , ("Relative Global Package DB", "../../stage1/pkgs") ] diff --git a/cabal.project-stage1 b/cabal.project-stage1 new file mode 100644 index 000000000000..a799ff545a08 --- /dev/null +++ b/cabal.project-stage1 @@ -0,0 +1,22 @@ +with-compiler: ./_build/stage0/bin/ghc +with-hc-pkg: ./_build/stage0/bin/ghc-pkg +package-dbs: clear, global + +packages: + ./_build/stage1/src/libraries/rts + +benchmarks: False +tests: False +allow-boot-library-installs: True + +program-options + ghc-options: -fhide-source-paths -j + +package * + library-vanilla: True + shared: False + executable-profiling: False + executable-dynamic: False + executable-static: False + ghc-options: -fhide-source-paths -j + From 304d91713659dca031e9ac01e3224176e2ef9008 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Thu, 28 Nov 2024 16:13:33 +0100 Subject: [PATCH 008/175] Progress towards building the RTS --- Makefile | 46 ++++++++++++++++++++++++++++++++++++++------ cabal.project-stage1 | 6 +++--- 2 files changed, 43 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index d13dc49de34c..f51c29b84791 100644 --- a/Makefile +++ b/Makefile @@ -79,10 +79,15 @@ _build/stage1/bin/ghc: _build/stage0/bin/ghc # Preparing source files... mkdir -p _build/stage1/src/ - cp -rf ./libraries _build/stage1/src/ - cp -rf ./compiler _build/stage1/src/libraries/ghc - cp -rf ./rts _build/stage1/src/libraries/ - cp -rf ./ghc _build/stage1/src/ghc-bin + cp -rf ./libraries _build/stage1/src/ + cp -rf ./compiler _build/stage1/src/libraries/ghc + cp -rf ./rts _build/stage1/src/libraries/ + cp -rf ./ghc _build/stage1/src/ghc-bin + cp -rf ./config.sub _build/stage1/src/libraries/rts/ + cp -rf ./config.guess _build/stage1/src/libraries/rts/ + + python rts/gen_event_types.py --event-types-defines _build/stage1/src/libraries/rts/include/rts/EventLogConstants.h + python rts/gen_event_types.py --event-types-array _build/stage1/src/libraries/rts/include/rts/EventTypes.h ## Substituting variables cp _build/stage1/src/ghc-bin/ghc-bin.cabal{.in,} @@ -92,6 +97,7 @@ _build/stage1/bin/ghc: _build/stage0/bin/ghc cp _build/stage1/src/libraries/ghc-boot-th/ghc-boot-th.cabal{.in,} cp _build/stage1/src/libraries/ghc-heap/ghc-heap.cabal{.in,} cp _build/stage1/src/libraries/ghci/ghci.cabal{.in,} + cp _build/stage1/src/libraries/rts/include/ghcversion.h{.in,} sed -i 's/@ProjectVersion@/9.13/' _build/stage1/src/ghc-bin/ghc-bin.cabal sed -i 's/@ProjectVersionMunged@/9.13/' _build/stage1/src/ghc-bin/ghc-bin.cabal @@ -109,6 +115,10 @@ _build/stage1/bin/ghc: _build/stage0/bin/ghc sed -i 's/@ProjectVersion@/9.13/' _build/stage1/src/libraries/ghci/ghci.cabal sed -i 's/@ProjectVersionMunged@/9.13/' _build/stage1/src/libraries/ghci/ghci.cabal sed -i 's/@ProjectVersionForLib@/9.13/' _build/stage1/src/libraries/ghci/ghci.cabal + sed -i 's/@ProjectVersion@/9.13/' _build/stage1/src/libraries/rts/include/ghcversion.h + sed -i 's/@ProjectVersionInt@/913/' _build/stage1/src/libraries/rts/include/ghcversion.h + sed -i 's/@ProjectPatchLevel1@/0/' _build/stage1/src/libraries/rts/include/ghcversion.h + sed -i 's/@ProjectPatchLevel2@/0/' _build/stage1/src/libraries/rts/include/ghcversion.h sed -i 's/@LlvmMinVersion@/13/' _build/stage1/src/libraries/ghc/GHC/CmmToLlvm/Version/Bounds.hs sed -i 's/@LlvmMaxVersion@/20/' _build/stage1/src/libraries/ghc/GHC/CmmToLlvm/Version/Bounds.hs @@ -116,12 +126,36 @@ _build/stage1/bin/ghc: _build/stage0/bin/ghc # Building boot libraries mkdir -p _build/stage1/cabal/ - cabal configure --project-file=cabal.project-stage1 + cabal configure --project-file=cabal.project-stage1 \ + --with-compiler=`pwd`/_build/stage0/bin/ghc \ + --with-hc-pkg=`pwd`/_build/stage0/bin/ghc-pkg \ + --builddir=_build/stage1/cabal/ HADRIAN_SETTINGS='$(HADRIAN_SETTINGS_STAGE1)' \ cabal build --project-file=cabal.project-stage1 \ rts \ - --builddir=_build/stage1/cabal/ -v + --with-compiler=`pwd`/_build/stage0/bin/ghc \ + --with-hc-pkg=`pwd`/_build/stage0/bin/ghc-pkg \ + --ghc-options="-ghcversion-file=`pwd`/_build/stage1/src/libraries/rts/include/ghcversion.h" \ + --ghc-options="-I`pwd`/_build/stage1/src/libraries/rts/include/" \ + --ghc-options="-I`pwd`/_build/stage1/src/libraries/rts/" \ + --ghc-options='"-optc=-DProjectVersion=\"913\""' \ + --ghc-options='"-optc=-DRtsWay=\"FIXME\""' \ + --ghc-options='"-optc=-DHostPlatform=\"FIXME\""' \ + --ghc-options='"-optc=-DHostArch=\"FIXME\""' \ + --ghc-options='"-optc=-DHostOS=\"FIXME\""' \ + --ghc-options='"-optc=-DHostVendor=\"FIXME\""' \ + --ghc-options='"-optc=-DBuildPlatform=\"FIXME\""' \ + --ghc-options='"-optc=-DBuildArch=\"FIXME\""' \ + --ghc-options='"-optc=-DBuildOS=\"FIXME\""' \ + --ghc-options='"-optc=-DBuildVendor=\"FIXME\""' \ + --ghc-options='"-optc=-DTargetPlatform=\"FIXME\""' \ + --ghc-options='"-optc=-DTargetArch=\"FIXME\""' \ + --ghc-options='"-optc=-DTargetOS=\"FIXME\""' \ + --ghc-options='"-optc=-DTargetVendor=\"FIXME\""' \ + --ghc-options='"-optc=-DGhcUnregisterised=\"FIXME\""' \ + --ghc-options='"-optc=-DTablesNextToCode=\"FIXME\""' \ + --builddir=_build/stage1/cabal/ clean: rm -rf _build diff --git a/cabal.project-stage1 b/cabal.project-stage1 index a799ff545a08..3381b2a1d897 100644 --- a/cabal.project-stage1 +++ b/cabal.project-stage1 @@ -1,5 +1,6 @@ -with-compiler: ./_build/stage0/bin/ghc -with-hc-pkg: ./_build/stage0/bin/ghc-pkg +-- for some reason it doesn't work... +--with-compiler: ./_build/stage0/bin/ghc +--with-hc-pkg: ./_build/stage0/bin/ghc-pkg package-dbs: clear, global packages: @@ -19,4 +20,3 @@ package * executable-dynamic: False executable-static: False ghc-options: -fhide-source-paths -j - From ff24e405eda0706b26db7eb053eeace4da971c49 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Thu, 28 Nov 2024 17:47:33 +0100 Subject: [PATCH 009/175] Fix CPP for Cmm --- GenSettings.hs | 8 ++++++-- Makefile | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/GenSettings.hs b/GenSettings.hs index fcd7d6525c72..a8c5b577f411 100644 --- a/GenSettings.hs +++ b/GenSettings.hs @@ -57,6 +57,10 @@ stage1_settings input = output Just v -> (s,v) Just v -> (s,v) + --FIXME: we default to these flags for Cmm CPP, otherwise CPP fails + -- with error: missing '(' after "__has_feature" + -- because we pass `-traditional` while compiling Apply.cmm (in TSANUtils.h) + default_cpp_flags = "-E" out_settings = [ keep_fail "C compiler command" @@ -66,13 +70,13 @@ stage1_settings input = output , keep_fail "C compiler link flags" , keep_fail "C compiler supports -no-pie" , keep_or_fail "CPP command" "Haskell CPP command" - , keep_or_fail "CPP flags" "Haskell CPP flags" + , keep_def "CPP flags" default_cpp_flags , keep_fail "Haskell CPP command" , keep_fail "Haskell CPP flags" , keep_or_fail "JavaScript CPP command" "Haskell CPP command" , keep_or_fail "JavaScript CPP flags" "Haskell CPP flags" , keep_or_fail "C-- CPP command" "Haskell CPP command" - , keep_or_fail "C-- CPP flags" "Haskell CPP flags" + , keep_def "C-- CPP flags" default_cpp_flags , keep_def "C-- CPP supports -g0" "NO" , keep_fail "ld supports compact unwind" , keep_fail "ld supports filelist" diff --git a/Makefile b/Makefile index f51c29b84791..6fa1d6622d89 100644 --- a/Makefile +++ b/Makefile @@ -155,7 +155,7 @@ _build/stage1/bin/ghc: _build/stage0/bin/ghc --ghc-options='"-optc=-DTargetVendor=\"FIXME\""' \ --ghc-options='"-optc=-DGhcUnregisterised=\"FIXME\""' \ --ghc-options='"-optc=-DTablesNextToCode=\"FIXME\""' \ - --builddir=_build/stage1/cabal/ + --builddir=_build/stage1/cabal/ clean: rm -rf _build From cd7e255921f3438f2255a804bfde5188d668cdce Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Fri, 29 Nov 2024 10:47:14 +0100 Subject: [PATCH 010/175] Apply suggestions --- Makefile | 7 +------ cabal.project-stage1 | 1 + 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 6fa1d6622d89..84167b1eceee 100644 --- a/Makefile +++ b/Makefile @@ -49,7 +49,7 @@ _build/stage0/bin/ghc: # Building... mkdir -p _build/stage0/cabal/ mkdir -p _build/stage0/bin/ - cabal configure --project-file=cabal.project-stage0 + HADRIAN_SETTINGS='$(HADRIAN_SETTINGS_STAGE0)' \ cabal build --project-file=cabal.project-stage0 \ ghc-bin:ghc ghc-pkg:ghc-pkg genprimopcode:genprimopcode deriveConstants:deriveConstants \ @@ -126,11 +126,6 @@ _build/stage1/bin/ghc: _build/stage0/bin/ghc # Building boot libraries mkdir -p _build/stage1/cabal/ - cabal configure --project-file=cabal.project-stage1 \ - --with-compiler=`pwd`/_build/stage0/bin/ghc \ - --with-hc-pkg=`pwd`/_build/stage0/bin/ghc-pkg \ - --builddir=_build/stage1/cabal/ - HADRIAN_SETTINGS='$(HADRIAN_SETTINGS_STAGE1)' \ cabal build --project-file=cabal.project-stage1 \ rts \ diff --git a/cabal.project-stage1 b/cabal.project-stage1 index 3381b2a1d897..dfa9d54f072b 100644 --- a/cabal.project-stage1 +++ b/cabal.project-stage1 @@ -9,6 +9,7 @@ packages: benchmarks: False tests: False allow-boot-library-installs: True +active-repositories: :none program-options ghc-options: -fhide-source-paths -j From b878f1a5b4c723eb93255999e1208ea8eb9b3d38 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 3 Dec 2024 09:18:14 +0100 Subject: [PATCH 011/175] Allow specifying custom cabal --- Makefile | 19 ++++++++++++------- rts/rts.cabal | 4 ++-- utils/genprimopcode/genprimopcode.cabal | 2 ++ 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 84167b1eceee..1fe5ddfd8735 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,9 @@ HADRIAN_SETTINGS_STAGE1 := $(shell ghc --info | runghc GenSettings.hs ghc-boot) SETTINGS_STAGE1 := $(shell ghc --info | runghc GenSettings.hs stage1) +# CABAL := /home/hsyl20/repo/cabal/dist-newstyle/build/x86_64-linux/ghc-9.8.2/cabal-install-3.15.0.0/x/cabal/build/cabal/cabal +CABAL := cabal + all: _build/stage1/bin/ghc _build/stage0/bin/ghc: @@ -51,15 +54,15 @@ _build/stage0/bin/ghc: mkdir -p _build/stage0/bin/ HADRIAN_SETTINGS='$(HADRIAN_SETTINGS_STAGE0)' \ - cabal build --project-file=cabal.project-stage0 \ + $(CABAL) build --project-file=cabal.project-stage0 \ ghc-bin:ghc ghc-pkg:ghc-pkg genprimopcode:genprimopcode deriveConstants:deriveConstants \ -j --builddir=_build/stage0/cabal/ # Installing binaries - cp `cabal list-bin --project-file=cabal.project-stage0 --builddir=_build/stage0/cabal/ ghc-bin:ghc` _build/stage0/bin/ghc - cp `cabal list-bin --project-file=cabal.project-stage0 --builddir=_build/stage0/cabal/ ghc-pkg:ghc-pkg` _build/stage0/bin/ghc-pkg - cp `cabal list-bin --project-file=cabal.project-stage0 --builddir=_build/stage0/cabal/ deriveConstants:deriveConstants` _build/stage0/bin/deriveConstants - cp `cabal list-bin --project-file=cabal.project-stage0 --builddir=_build/stage0/cabal/ genprimopcode:genprimopcode` _build/stage0/bin/genprimopcode + cp `$(CABAL) list-bin --project-file=cabal.project-stage0 --builddir=_build/stage0/cabal/ ghc-bin:ghc` _build/stage0/bin/ghc + cp `$(CABAL) list-bin --project-file=cabal.project-stage0 --builddir=_build/stage0/cabal/ ghc-pkg:ghc-pkg` _build/stage0/bin/ghc-pkg + cp `$(CABAL) list-bin --project-file=cabal.project-stage0 --builddir=_build/stage0/cabal/ deriveConstants:deriveConstants` _build/stage0/bin/deriveConstants + cp `$(CABAL) list-bin --project-file=cabal.project-stage0 --builddir=_build/stage0/cabal/ genprimopcode:genprimopcode` _build/stage0/bin/genprimopcode # Generate settings mkdir -p _build/stage0/lib @@ -72,6 +75,7 @@ _build/stage0/bin/ghc: _build/stage1/bin/ghc: _build/stage0/bin/ghc rm -rf _build/stage1 mkdir -p _build/stage1 + $(CABAL) --version # Initialize empty package db _build/stage0/bin/ghc-pkg init _build/stage1/pkgs @@ -127,7 +131,7 @@ _build/stage1/bin/ghc: _build/stage0/bin/ghc mkdir -p _build/stage1/cabal/ HADRIAN_SETTINGS='$(HADRIAN_SETTINGS_STAGE1)' \ - cabal build --project-file=cabal.project-stage1 \ + $(CABAL) build --project-file=cabal.project-stage1 \ rts \ --with-compiler=`pwd`/_build/stage0/bin/ghc \ --with-hc-pkg=`pwd`/_build/stage0/bin/ghc-pkg \ @@ -150,7 +154,8 @@ _build/stage1/bin/ghc: _build/stage0/bin/ghc --ghc-options='"-optc=-DTargetVendor=\"FIXME\""' \ --ghc-options='"-optc=-DGhcUnregisterised=\"FIXME\""' \ --ghc-options='"-optc=-DTablesNextToCode=\"FIXME\""' \ - --builddir=_build/stage1/cabal/ + --ghc-options='-v3' \ + --builddir=_build/stage1/cabal/ -v3 clean: rm -rf _build diff --git a/rts/rts.cabal b/rts/rts.cabal index f996cfa0e2b7..f526ac59a601 100644 --- a/rts/rts.cabal +++ b/rts/rts.cabal @@ -1,4 +1,4 @@ -cabal-version: 3.0 +cabal-version: 3.4 name: rts version: 1.0.3 synopsis: The GHC runtime system @@ -399,7 +399,7 @@ library if arch(ppc) || arch(ppc64) || arch(s390x) || arch(riscv64) || arch(loongarch64) asm-sources: StgCRunAsm.S - c-sources: Adjustor.c + c-sources: Adjustor.c (-O3) adjustor/AdjustorPool.c ExecPage.c Arena.c diff --git a/utils/genprimopcode/genprimopcode.cabal b/utils/genprimopcode/genprimopcode.cabal index 8db8827e1594..946c5e576ae9 100644 --- a/utils/genprimopcode/genprimopcode.cabal +++ b/utils/genprimopcode/genprimopcode.cabal @@ -31,5 +31,7 @@ Executable genprimopcode AccessOps Build-Depends: base >= 4 && < 5, array + default-extensions: + UnboxedSums if flag(build-tool-depends) build-tool-depends: alex:alex >= 3.2.6, happy:happy >= 1.20.0 From 07abc0d871e5cd40cd83c2552c9f2f1ea2031d2b Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 3 Dec 2024 11:07:14 +0100 Subject: [PATCH 012/175] Start with deriveConstants --- Makefile | 14 ++++++++++++-- rts/rts.cabal | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 1fe5ddfd8735..08acee087533 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,6 @@ HADRIAN_SETTINGS_STAGE0 := $(shell ghc --info | runghc GenSettings.hs ghc-boot) HADRIAN_SETTINGS_STAGE1 := $(shell ghc --info | runghc GenSettings.hs ghc-boot) SETTINGS_STAGE1 := $(shell ghc --info | runghc GenSettings.hs stage1) - # CABAL := /home/hsyl20/repo/cabal/dist-newstyle/build/x86_64-linux/ghc-9.8.2/cabal-install-3.15.0.0/x/cabal/build/cabal/cabal CABAL := cabal @@ -126,10 +125,21 @@ _build/stage1/bin/ghc: _build/stage0/bin/ghc sed -i 's/@LlvmMinVersion@/13/' _build/stage1/src/libraries/ghc/GHC/CmmToLlvm/Version/Bounds.hs sed -i 's/@LlvmMaxVersion@/20/' _build/stage1/src/libraries/ghc/GHC/CmmToLlvm/Version/Bounds.hs + + # Generating headers + # FIXME: deriveConstants requires ghcautoconf.h and ghcplatform.h + mkdir -p _build/stage1/temp/derive_constants + _build/stage0/bin/deriveConstants --gen-header -o _build/stage1/src/libraries/rts/include/DerivedConstants.h \ + --target-os linux \ + --tmpdir _build/stage1/temp/derive_constants \ + --gcc-program gcc \ + --nm-program nm \ + --objdump-program objdump \ + --gcc-flag "-I_build/stage1/src/libraries/rts/include" # Building boot libraries mkdir -p _build/stage1/cabal/ - + HADRIAN_SETTINGS='$(HADRIAN_SETTINGS_STAGE1)' \ $(CABAL) build --project-file=cabal.project-stage1 \ rts \ diff --git a/rts/rts.cabal b/rts/rts.cabal index f526ac59a601..b0c48afd3370 100644 --- a/rts/rts.cabal +++ b/rts/rts.cabal @@ -399,7 +399,7 @@ library if arch(ppc) || arch(ppc64) || arch(s390x) || arch(riscv64) || arch(loongarch64) asm-sources: StgCRunAsm.S - c-sources: Adjustor.c (-O3) + c-sources: Adjustor.c adjustor/AdjustorPool.c ExecPage.c Arena.c From 8b4cdfcd0bc4d75bc6cbbce8fbc8b327fafc68d1 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 3 Dec 2024 13:03:39 +0100 Subject: [PATCH 013/175] Generate DerivedConstants.h with a hack --- Makefile | 35 ++++++++++++++++++++++++++++++++--- utils/deriveConstants/Main.hs | 1 + 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 08acee087533..4f184e897358 100644 --- a/Makefile +++ b/Makefile @@ -128,6 +128,34 @@ _build/stage1/bin/ghc: _build/stage0/bin/ghc # Generating headers # FIXME: deriveConstants requires ghcautoconf.h and ghcplatform.h + # Let's run cabal until it fails so that these files are generated... + HADRIAN_SETTINGS='$(HADRIAN_SETTINGS_STAGE1)' \ + $(CABAL) build --project-file=cabal.project-stage1 \ + rts \ + --with-compiler=`pwd`/_build/stage0/bin/ghc \ + --with-hc-pkg=`pwd`/_build/stage0/bin/ghc-pkg \ + --ghc-options="-ghcversion-file=`pwd`/_build/stage1/src/libraries/rts/include/ghcversion.h" \ + --ghc-options="-I`pwd`/_build/stage1/src/libraries/rts/include/" \ + --ghc-options="-I`pwd`/_build/stage1/src/libraries/rts/" \ + --ghc-options='"-optc=-DProjectVersion=\"913\""' \ + --ghc-options='"-optc=-DRtsWay=\"FIXME\""' \ + --ghc-options='"-optc=-DHostPlatform=\"FIXME\""' \ + --ghc-options='"-optc=-DHostArch=\"FIXME\""' \ + --ghc-options='"-optc=-DHostOS=\"FIXME\""' \ + --ghc-options='"-optc=-DHostVendor=\"FIXME\""' \ + --ghc-options='"-optc=-DBuildPlatform=\"FIXME\""' \ + --ghc-options='"-optc=-DBuildArch=\"FIXME\""' \ + --ghc-options='"-optc=-DBuildOS=\"FIXME\""' \ + --ghc-options='"-optc=-DBuildVendor=\"FIXME\""' \ + --ghc-options='"-optc=-DTargetPlatform=\"FIXME\""' \ + --ghc-options='"-optc=-DTargetArch=\"FIXME\""' \ + --ghc-options='"-optc=-DTargetOS=\"FIXME\""' \ + --ghc-options='"-optc=-DTargetVendor=\"FIXME\""' \ + --ghc-options='"-optc=-DGhcUnregisterised=\"FIXME\""' \ + --ghc-options='"-optc=-DTablesNextToCode=\"FIXME\""' \ + --builddir=_build/stage1/cabal/ >/dev/null 2>&1 || true + + # Deriving constants mkdir -p _build/stage1/temp/derive_constants _build/stage0/bin/deriveConstants --gen-header -o _build/stage1/src/libraries/rts/include/DerivedConstants.h \ --target-os linux \ @@ -135,7 +163,9 @@ _build/stage1/bin/ghc: _build/stage0/bin/ghc --gcc-program gcc \ --nm-program nm \ --objdump-program objdump \ - --gcc-flag "-I_build/stage1/src/libraries/rts/include" + --gcc-flag "-I_build/stage1/src/libraries/rts/include" \ + --gcc-flag "-I_build/stage1/src/libraries/rts" \ + --gcc-flag "-I_build/stage1/cabal/build/x86_64-linux/ghc-9.13/rts-1.0.2/build/include" # Building boot libraries mkdir -p _build/stage1/cabal/ @@ -164,8 +194,7 @@ _build/stage1/bin/ghc: _build/stage0/bin/ghc --ghc-options='"-optc=-DTargetVendor=\"FIXME\""' \ --ghc-options='"-optc=-DGhcUnregisterised=\"FIXME\""' \ --ghc-options='"-optc=-DTablesNextToCode=\"FIXME\""' \ - --ghc-options='-v3' \ - --builddir=_build/stage1/cabal/ -v3 + --builddir=_build/stage1/cabal/ clean: rm -rf _build diff --git a/utils/deriveConstants/Main.hs b/utils/deriveConstants/Main.hs index 68acf8ec8150..044ba9339b41 100644 --- a/utils/deriveConstants/Main.hs +++ b/utils/deriveConstants/Main.hs @@ -841,6 +841,7 @@ getWanted verbose os tmpdir gccProgram gccFlags nmProgram mobjdumpProgram = case words line of ('_' : n) : "C" : s : _ -> mkP n s n : "C" : s : _ -> mkP n s + n : "B" : _ : s : _ -> mkP n s [n, "D", _, s] -> mkP n s [s, "O", "*COM*", _, n] -> mkP n s _ -> Nothing From c3eac1bb22acb38980151a98374ece1cf3a0765b Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 3 Dec 2024 13:16:39 +0100 Subject: [PATCH 014/175] Pass -this-unit-id because cabal doesn't when it compiles Cmm files --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 4f184e897358..d6407ebbf902 100644 --- a/Makefile +++ b/Makefile @@ -194,6 +194,7 @@ _build/stage1/bin/ghc: _build/stage0/bin/ghc --ghc-options='"-optc=-DTargetVendor=\"FIXME\""' \ --ghc-options='"-optc=-DGhcUnregisterised=\"FIXME\""' \ --ghc-options='"-optc=-DTablesNextToCode=\"FIXME\""' \ + --ghc-options='-this-unit-id rts' \ --builddir=_build/stage1/cabal/ clean: From 5fadf5f390dc287ddf9b4a641356721aab7d4212 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 11 Dec 2024 16:34:05 +0100 Subject: [PATCH 015/175] Try using per source flags --- GenSettings.hs | 1 + Makefile | 4 ++-- cabal.project-stage0 | 3 +++ rts/rts.cabal | 11 +++++++++-- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/GenSettings.hs b/GenSettings.hs index a8c5b577f411..8d9fffc22521 100644 --- a/GenSettings.hs +++ b/GenSettings.hs @@ -113,6 +113,7 @@ stage1_settings input = output , keep_def "target RTS linker only supports shared libraries" "NO" , ("Use interpreter", "NO") + , ("base unit-id", "base") , keep_fail "Support SMP" , keep_fail "RTS ways" , keep_fail "Tables next to code" diff --git a/Makefile b/Makefile index d6407ebbf902..b017f5784bed 100644 --- a/Makefile +++ b/Makefile @@ -2,8 +2,8 @@ HADRIAN_SETTINGS_STAGE0 := $(shell ghc --info | runghc GenSettings.hs ghc-boot) HADRIAN_SETTINGS_STAGE1 := $(shell ghc --info | runghc GenSettings.hs ghc-boot) SETTINGS_STAGE1 := $(shell ghc --info | runghc GenSettings.hs stage1) -# CABAL := /home/hsyl20/repo/cabal/dist-newstyle/build/x86_64-linux/ghc-9.8.2/cabal-install-3.15.0.0/x/cabal/build/cabal/cabal -CABAL := cabal +CABAL := /home/hsyl20/repo/cabal/dist-newstyle/build/x86_64-linux/ghc-9.8.2/cabal-install-3.15.0.0/x/cabal/build/cabal/cabal +# CABAL := cabal all: _build/stage1/bin/ghc diff --git a/cabal.project-stage0 b/cabal.project-stage0 index 0e4bff4838b8..8c272e5e89aa 100644 --- a/cabal.project-stage0 +++ b/cabal.project-stage0 @@ -39,4 +39,7 @@ package * package ghc-boot-th flags: +bootstrap +-- package genprimopcode +-- flags: -build-tool-depends + allow-newer: ghc-boot-th diff --git a/rts/rts.cabal b/rts/rts.cabal index b0c48afd3370..515f42bf2300 100644 --- a/rts/rts.cabal +++ b/rts/rts.cabal @@ -376,8 +376,15 @@ library -- AutoApply is generated AutoApply.cmm AutoApply_V16.cmm - AutoApply_V32.cmm - AutoApply_V64.cmm + + if arch(x86_64) + cmm-sources: + AutoApply_V32.cmm (-mavx2) + AutoApply_V64.cmm (-mavx512f) + else + cmm-sources: + AutoApply_V32.cmm + AutoApply_V64.cmm -- Adjustor stuff if flag(libffi-adjustors) From a15c85d8d61aa02258f76d05f1296cd111f41375 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 11 Dec 2024 17:36:30 +0100 Subject: [PATCH 016/175] Fix rts per file flags --- rts/rts.cabal | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rts/rts.cabal b/rts/rts.cabal index 515f42bf2300..0a54f13fa714 100644 --- a/rts/rts.cabal +++ b/rts/rts.cabal @@ -366,8 +366,6 @@ library HeapStackCheck.cmm Jumps_D.cmm Jumps_V16.cmm - Jumps_V32.cmm - Jumps_V64.cmm PrimOps.cmm StgMiscClosures.cmm StgStartup.cmm @@ -381,10 +379,14 @@ library cmm-sources: AutoApply_V32.cmm (-mavx2) AutoApply_V64.cmm (-mavx512f) + Jumps_V32.cmm (-mavx2) + Jumps_V64.cmm (-mavx512f) else cmm-sources: AutoApply_V32.cmm AutoApply_V64.cmm + Jumps_V32.cmm + Jumps_V64.cmm -- Adjustor stuff if flag(libffi-adjustors) From 0b275d3df8743ac9052718dbfd738aa5bb10701f Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Fri, 13 Dec 2024 13:06:19 +0100 Subject: [PATCH 017/175] Generate AutoApply*.cmm --- Makefile | 9 ++++++++- cabal.project-stage0 | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b017f5784bed..b49b9dd99f63 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ _build/stage0/bin/ghc: HADRIAN_SETTINGS='$(HADRIAN_SETTINGS_STAGE0)' \ $(CABAL) build --project-file=cabal.project-stage0 \ - ghc-bin:ghc ghc-pkg:ghc-pkg genprimopcode:genprimopcode deriveConstants:deriveConstants \ + ghc-bin:ghc ghc-pkg:ghc-pkg genprimopcode:genprimopcode deriveConstants:deriveConstants genapply:genapply \ -j --builddir=_build/stage0/cabal/ # Installing binaries @@ -62,6 +62,7 @@ _build/stage0/bin/ghc: cp `$(CABAL) list-bin --project-file=cabal.project-stage0 --builddir=_build/stage0/cabal/ ghc-pkg:ghc-pkg` _build/stage0/bin/ghc-pkg cp `$(CABAL) list-bin --project-file=cabal.project-stage0 --builddir=_build/stage0/cabal/ deriveConstants:deriveConstants` _build/stage0/bin/deriveConstants cp `$(CABAL) list-bin --project-file=cabal.project-stage0 --builddir=_build/stage0/cabal/ genprimopcode:genprimopcode` _build/stage0/bin/genprimopcode + cp `$(CABAL) list-bin --project-file=cabal.project-stage0 --builddir=_build/stage0/cabal/ genapply:genapply` _build/stage0/bin/genapply # Generate settings mkdir -p _build/stage0/lib @@ -166,6 +167,12 @@ _build/stage1/bin/ghc: _build/stage0/bin/ghc --gcc-flag "-I_build/stage1/src/libraries/rts/include" \ --gcc-flag "-I_build/stage1/src/libraries/rts" \ --gcc-flag "-I_build/stage1/cabal/build/x86_64-linux/ghc-9.13/rts-1.0.2/build/include" + + # Generate autoapply + _build/stage0/bin/genapply _build/stage1/src/libraries/rts/include/DerivedConstants.h > _build/stage1/src/libraries/rts/AutoApply.cmm + _build/stage0/bin/genapply _build/stage1/src/libraries/rts/include/DerivedConstants.h -V16 > _build/stage1/src/libraries/rts/AutoApply_V16.cmm + _build/stage0/bin/genapply _build/stage1/src/libraries/rts/include/DerivedConstants.h -V32 > _build/stage1/src/libraries/rts/AutoApply_V32.cmm + _build/stage0/bin/genapply _build/stage1/src/libraries/rts/include/DerivedConstants.h -V64 > _build/stage1/src/libraries/rts/AutoApply_V64.cmm # Building boot libraries mkdir -p _build/stage1/cabal/ diff --git a/cabal.project-stage0 b/cabal.project-stage0 index 8c272e5e89aa..2b98ddf588a5 100644 --- a/cabal.project-stage0 +++ b/cabal.project-stage0 @@ -19,6 +19,7 @@ packages: ./_build/stage0/src/utils/hsc2hs ./_build/stage0/src/utils/unlit ./_build/stage0/src/utils/genprimopcode/ + ./_build/stage0/src/utils/genapply/ ./_build/stage0/src/utils/deriveConstants/ benchmarks: False From 4388e769e6ef394ca87edb8bfc5b3105679ce8cb Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Mon, 16 Dec 2024 15:10:13 +0100 Subject: [PATCH 018/175] Build libffi --- Makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Makefile b/Makefile index b49b9dd99f63..604885893795 100644 --- a/Makefile +++ b/Makefile @@ -174,6 +174,14 @@ _build/stage1/bin/ghc: _build/stage0/bin/ghc _build/stage0/bin/genapply _build/stage1/src/libraries/rts/include/DerivedConstants.h -V32 > _build/stage1/src/libraries/rts/AutoApply_V32.cmm _build/stage0/bin/genapply _build/stage1/src/libraries/rts/include/DerivedConstants.h -V64 > _build/stage1/src/libraries/rts/AutoApply_V64.cmm + # Build libffi + mkdir -p _build/stage1/src/libffi + mkdir -p _build/stage1/libffi + (cd _build/stage1/src/libffi; tar -xvf ../../../../libffi-tarballs/libffi-3.4.6.tar.gz) + (cd _build/stage1/src/libffi/libffi-3.4.6; ./configure --disable-docs --with-pics=yes --disable-multi-os-directory --prefix=`pwd`/../../../../../_build/stage1/libffi/ && make install -j) + cp -f _build/stage1/libffi/include/* _build/stage1/src/libraries/rts/include/ + cp -f _build/stage1/libffi/lib/libffi.a _build/stage1/cabal/build/x86_64-linux/ghc-9.13/rts-1.0.2/build/libCffi.a + # Building boot libraries mkdir -p _build/stage1/cabal/ From fce763adb2333ee44b523d69b0e60a31399f9cb6 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 18 Dec 2024 09:54:54 +0100 Subject: [PATCH 019/175] Progress building ghc-prim --- Makefile | 37 ++++++++++++++++++++++++++++++++++--- cabal.project-stage1 | 1 + cabal.project-stage1-rts | 23 +++++++++++++++++++++++ 3 files changed, 58 insertions(+), 3 deletions(-) create mode 100644 cabal.project-stage1-rts diff --git a/Makefile b/Makefile index 604885893795..64f886d0f3d3 100644 --- a/Makefile +++ b/Makefile @@ -131,7 +131,7 @@ _build/stage1/bin/ghc: _build/stage0/bin/ghc # FIXME: deriveConstants requires ghcautoconf.h and ghcplatform.h # Let's run cabal until it fails so that these files are generated... HADRIAN_SETTINGS='$(HADRIAN_SETTINGS_STAGE1)' \ - $(CABAL) build --project-file=cabal.project-stage1 \ + $(CABAL) build --project-file=cabal.project-stage1-rts \ rts \ --with-compiler=`pwd`/_build/stage0/bin/ghc \ --with-hc-pkg=`pwd`/_build/stage0/bin/ghc-pkg \ @@ -185,8 +185,11 @@ _build/stage1/bin/ghc: _build/stage0/bin/ghc # Building boot libraries mkdir -p _build/stage1/cabal/ + # we need to pass "-this-unit-id=rts", otherwise GHC tries to lookup the + # platform constants in the package db and fails. The flag is already + # set in rts.cabal but for some reason it isn't always passed :shrug: HADRIAN_SETTINGS='$(HADRIAN_SETTINGS_STAGE1)' \ - $(CABAL) build --project-file=cabal.project-stage1 \ + $(CABAL) build --project-file=cabal.project-stage1-rts \ rts \ --with-compiler=`pwd`/_build/stage0/bin/ghc \ --with-hc-pkg=`pwd`/_build/stage0/bin/ghc-pkg \ @@ -209,7 +212,35 @@ _build/stage1/bin/ghc: _build/stage0/bin/ghc --ghc-options='"-optc=-DTargetVendor=\"FIXME\""' \ --ghc-options='"-optc=-DGhcUnregisterised=\"FIXME\""' \ --ghc-options='"-optc=-DTablesNextToCode=\"FIXME\""' \ - --ghc-options='-this-unit-id rts' \ + --ghc-options='-this-unit-id=rts' \ + --builddir=_build/stage1/cabal/ + + + + HADRIAN_SETTINGS='$(HADRIAN_SETTINGS_STAGE1)' \ + $(CABAL) build --project-file=cabal.project-stage1 \ + rts ghc-prim \ + --with-compiler=`pwd`/_build/stage0/bin/ghc \ + --with-hc-pkg=`pwd`/_build/stage0/bin/ghc-pkg \ + --ghc-options="-ghcversion-file=`pwd`/_build/stage1/src/libraries/rts/include/ghcversion.h" \ + --ghc-options="-I`pwd`/_build/stage1/src/libraries/rts/include/" \ + --ghc-options="-I`pwd`/_build/stage1/src/libraries/rts/" \ + --ghc-options='"-optc=-DProjectVersion=\"913\""' \ + --ghc-options='"-optc=-DRtsWay=\"FIXME\""' \ + --ghc-options='"-optc=-DHostPlatform=\"FIXME\""' \ + --ghc-options='"-optc=-DHostArch=\"FIXME\""' \ + --ghc-options='"-optc=-DHostOS=\"FIXME\""' \ + --ghc-options='"-optc=-DHostVendor=\"FIXME\""' \ + --ghc-options='"-optc=-DBuildPlatform=\"FIXME\""' \ + --ghc-options='"-optc=-DBuildArch=\"FIXME\""' \ + --ghc-options='"-optc=-DBuildOS=\"FIXME\""' \ + --ghc-options='"-optc=-DBuildVendor=\"FIXME\""' \ + --ghc-options='"-optc=-DTargetPlatform=\"FIXME\""' \ + --ghc-options='"-optc=-DTargetArch=\"FIXME\""' \ + --ghc-options='"-optc=-DTargetOS=\"FIXME\""' \ + --ghc-options='"-optc=-DTargetVendor=\"FIXME\""' \ + --ghc-options='"-optc=-DGhcUnregisterised=\"FIXME\""' \ + --ghc-options='"-optc=-DTablesNextToCode=\"FIXME\""' \ --builddir=_build/stage1/cabal/ clean: diff --git a/cabal.project-stage1 b/cabal.project-stage1 index dfa9d54f072b..99466392bde6 100644 --- a/cabal.project-stage1 +++ b/cabal.project-stage1 @@ -5,6 +5,7 @@ package-dbs: clear, global packages: ./_build/stage1/src/libraries/rts + ./_build/stage1/src/libraries/ghc-prim benchmarks: False tests: False diff --git a/cabal.project-stage1-rts b/cabal.project-stage1-rts new file mode 100644 index 000000000000..dfa9d54f072b --- /dev/null +++ b/cabal.project-stage1-rts @@ -0,0 +1,23 @@ +-- for some reason it doesn't work... +--with-compiler: ./_build/stage0/bin/ghc +--with-hc-pkg: ./_build/stage0/bin/ghc-pkg +package-dbs: clear, global + +packages: + ./_build/stage1/src/libraries/rts + +benchmarks: False +tests: False +allow-boot-library-installs: True +active-repositories: :none + +program-options + ghc-options: -fhide-source-paths -j + +package * + library-vanilla: True + shared: False + executable-profiling: False + executable-dynamic: False + executable-static: False + ghc-options: -fhide-source-paths -j From 2abe03e0d1f5a5d5a160f0e656a911b22dc06810 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 18 Dec 2024 10:31:11 +0100 Subject: [PATCH 020/175] Build ghc-prim --- Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile b/Makefile index 64f886d0f3d3..d2cc4ab17825 100644 --- a/Makefile +++ b/Makefile @@ -215,6 +215,11 @@ _build/stage1/bin/ghc: _build/stage0/bin/ghc --ghc-options='-this-unit-id=rts' \ --builddir=_build/stage1/cabal/ + # generate files related to primops + + gcc -E -undef -traditional -P -x c _build/stage1/src/libraries/ghc/GHC/Builtin/primops.txt.pp > _build/stage1/src/libraries/ghc/GHC/Builtin/primops.txt + _build/stage0/bin/genprimopcode --make-haskell-source < _build/stage1/src/libraries/ghc/GHC/Builtin/primops.txt > _build/stage1/src/libraries/ghc-prim/GHC/Prim.hs + _build/stage0/bin/genprimopcode --make-haskell-wrappers < _build/stage1/src/libraries/ghc/GHC/Builtin/primops.txt > _build/stage1/src/libraries/ghc-prim/GHC/PrimopWrappers.hs HADRIAN_SETTINGS='$(HADRIAN_SETTINGS_STAGE1)' \ From 51994a3c0c1b80610ee9fca9746aa969b58f30ce Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Thu, 19 Dec 2024 14:30:41 +0800 Subject: [PATCH 021/175] refactor: avoid hack when checking size of struct MD5Context --- libraries/ghc-internal/configure.ac | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/libraries/ghc-internal/configure.ac b/libraries/ghc-internal/configure.ac index b87652f61d25..d92117246de1 100644 --- a/libraries/ghc-internal/configure.ac +++ b/libraries/ghc-internal/configure.ac @@ -6,6 +6,8 @@ AC_CONFIG_SRCDIR([include/HsBase.h]) AC_CONFIG_HEADERS([include/HsBaseConfig.h include/EventConfig.h]) +CPPFLAGS="-I$srcdir $CPPFLAGS" + AC_PROG_CC dnl make extensions visible to allow feature-tests to detect them later on AC_USE_SYSTEM_EXTENSIONS @@ -402,10 +404,10 @@ AS_IF([test "x$with_libcharset" != xno], fi -dnl Calling AC_CHECK_TYPE(T) makes AC_CHECK_SIZEOF(T) abort on failure -dnl instead of considering sizeof(T) as 0. -AC_CHECK_TYPE([struct MD5Context], [], [AC_MSG_ERROR([internal error])], [#include "include/md5.h"]) AC_CHECK_SIZEOF([struct MD5Context], [], [#include "include/md5.h"]) +AS_IF([test "$ac_cv_sizeof_struct_MD5Context" -eq 0],[ + AC_MSG_ERROR([cannot determine sizeof(struct MD5Context)]) +]) AC_SUBST(EXTRA_LIBS) AC_CONFIG_FILES([ghc-internal.buildinfo include/HsIntegerGmp.h]) From ad48a085ab0feb1260d8d1b4ff64bebcf4f73dc6 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 18 Dec 2024 11:07:02 +0100 Subject: [PATCH 022/175] Build ghc-bignum --- Makefile | 2 +- cabal.project-stage1 | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d2cc4ab17825..09dd1e9c3782 100644 --- a/Makefile +++ b/Makefile @@ -224,7 +224,7 @@ _build/stage1/bin/ghc: _build/stage0/bin/ghc HADRIAN_SETTINGS='$(HADRIAN_SETTINGS_STAGE1)' \ $(CABAL) build --project-file=cabal.project-stage1 \ - rts ghc-prim \ + rts ghc-prim ghc-bignum \ --with-compiler=`pwd`/_build/stage0/bin/ghc \ --with-hc-pkg=`pwd`/_build/stage0/bin/ghc-pkg \ --ghc-options="-ghcversion-file=`pwd`/_build/stage1/src/libraries/rts/include/ghcversion.h" \ diff --git a/cabal.project-stage1 b/cabal.project-stage1 index 99466392bde6..2ce53a4fe611 100644 --- a/cabal.project-stage1 +++ b/cabal.project-stage1 @@ -6,6 +6,7 @@ package-dbs: clear, global packages: ./_build/stage1/src/libraries/rts ./_build/stage1/src/libraries/ghc-prim + ./_build/stage1/src/libraries/ghc-bignum benchmarks: False tests: False @@ -22,3 +23,7 @@ package * executable-dynamic: False executable-static: False ghc-options: -fhide-source-paths -j + +package ghc-bignum + -- make our life easier for now by using the native bignum backend + flags: +native From af0275d41b66aa0176a6fa62036a875cf6c32b9f Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 7 Jan 2025 16:19:30 +0100 Subject: [PATCH 023/175] WIP: build ghc-internal --- Makefile | 6 ++++-- cabal.project-stage1 | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 09dd1e9c3782..b10b8d00e066 100644 --- a/Makefile +++ b/Makefile @@ -97,6 +97,7 @@ _build/stage1/bin/ghc: _build/stage0/bin/ghc cp _build/stage1/src/ghc-bin/ghc-bin.cabal{.in,} cp _build/stage1/src/libraries/ghc/ghc.cabal{.in,} cp _build/stage1/src/libraries/ghc/GHC/CmmToLlvm/Version/Bounds.hs{.in,} + cp _build/stage1/src/libraries/ghc-internal/ghc-internal.cabal{.in,} cp _build/stage1/src/libraries/ghc-boot/ghc-boot.cabal{.in,} cp _build/stage1/src/libraries/ghc-boot-th/ghc-boot-th.cabal{.in,} cp _build/stage1/src/libraries/ghc-heap/ghc-heap.cabal{.in,} @@ -119,6 +120,7 @@ _build/stage1/bin/ghc: _build/stage0/bin/ghc sed -i 's/@ProjectVersion@/9.13/' _build/stage1/src/libraries/ghci/ghci.cabal sed -i 's/@ProjectVersionMunged@/9.13/' _build/stage1/src/libraries/ghci/ghci.cabal sed -i 's/@ProjectVersionForLib@/9.13/' _build/stage1/src/libraries/ghci/ghci.cabal + sed -i 's/@ProjectVersionForLib@/9.13/' _build/stage1/src/libraries/ghc-internal/ghc-internal.cabal sed -i 's/@ProjectVersion@/9.13/' _build/stage1/src/libraries/rts/include/ghcversion.h sed -i 's/@ProjectVersionInt@/913/' _build/stage1/src/libraries/rts/include/ghcversion.h sed -i 's/@ProjectPatchLevel1@/0/' _build/stage1/src/libraries/rts/include/ghcversion.h @@ -224,7 +226,7 @@ _build/stage1/bin/ghc: _build/stage0/bin/ghc HADRIAN_SETTINGS='$(HADRIAN_SETTINGS_STAGE1)' \ $(CABAL) build --project-file=cabal.project-stage1 \ - rts ghc-prim ghc-bignum \ + rts ghc-prim ghc-bignum ghc-internal \ --with-compiler=`pwd`/_build/stage0/bin/ghc \ --with-hc-pkg=`pwd`/_build/stage0/bin/ghc-pkg \ --ghc-options="-ghcversion-file=`pwd`/_build/stage1/src/libraries/rts/include/ghcversion.h" \ @@ -246,7 +248,7 @@ _build/stage1/bin/ghc: _build/stage0/bin/ghc --ghc-options='"-optc=-DTargetVendor=\"FIXME\""' \ --ghc-options='"-optc=-DGhcUnregisterised=\"FIXME\""' \ --ghc-options='"-optc=-DTablesNextToCode=\"FIXME\""' \ - --builddir=_build/stage1/cabal/ + --builddir=_build/stage1/cabal/ -v clean: rm -rf _build diff --git a/cabal.project-stage1 b/cabal.project-stage1 index 2ce53a4fe611..87a1a6eaf97a 100644 --- a/cabal.project-stage1 +++ b/cabal.project-stage1 @@ -7,6 +7,7 @@ packages: ./_build/stage1/src/libraries/rts ./_build/stage1/src/libraries/ghc-prim ./_build/stage1/src/libraries/ghc-bignum + ./_build/stage1/src/libraries/ghc-internal benchmarks: False tests: False From 13f1d3ccc35a41a25d26df276d6cbafed40bf6ed Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 8 Jan 2025 13:47:29 +0100 Subject: [PATCH 024/175] Fix new error while bootstrapping --- Makefile | 16 +++++++++++++++- cabal.project-stage0 | 6 ++++++ compiler/GHC/Parser/Lexer.x | 2 -- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index b10b8d00e066..90cdaa8f654d 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ HADRIAN_SETTINGS_STAGE0 := $(shell ghc --info | runghc GenSettings.hs ghc-boot) HADRIAN_SETTINGS_STAGE1 := $(shell ghc --info | runghc GenSettings.hs ghc-boot) SETTINGS_STAGE1 := $(shell ghc --info | runghc GenSettings.hs stage1) -CABAL := /home/hsyl20/repo/cabal/dist-newstyle/build/x86_64-linux/ghc-9.8.2/cabal-install-3.15.0.0/x/cabal/build/cabal/cabal +CABAL := /home/hsyl20/projects/cabal/dist-newstyle/build/x86_64-linux/ghc-9.10.1/cabal-install-3.15.0.0/x/cabal/build/cabal/cabal # CABAL := cabal all: _build/stage1/bin/ghc @@ -14,6 +14,13 @@ _build/stage0/bin/ghc: cp -rf ./compiler _build/stage0/src/libraries/ghc cp -rf ./ghc _build/stage0/src/ghc-bin cp -rf ./utils _build/stage0/src/ + + cp -f rts/include/rts/Bytecodes.h _build/stage0/src/libraries/ghc/ + cp -f rts/include/rts/storage/ClosureTypes.h _build/stage0/src/libraries/ghc/ + cp -f rts/include/rts/storage/FunTypes.h _build/stage0/src/libraries/ghc/ + cp -f rts/include/stg/MachRegs.h _build/stage0/src/libraries/ghc/ + mkdir -p _build/stage0/src/libraries/ghc/MachRegs + cp -f rts/include/stg/MachRegs/*.h _build/stage0/src/libraries/ghc/MachRegs/ ## Substituting variables cp _build/stage0/src/ghc-bin/ghc-bin.cabal{.in,} @@ -89,6 +96,13 @@ _build/stage1/bin/ghc: _build/stage0/bin/ghc cp -rf ./ghc _build/stage1/src/ghc-bin cp -rf ./config.sub _build/stage1/src/libraries/rts/ cp -rf ./config.guess _build/stage1/src/libraries/rts/ + + cp -f rts/include/rts/Bytecodes.h _build/stage1/src/libraries/ghc/ + cp -f rts/include/rts/storage/ClosureTypes.h _build/stage1/src/libraries/ghc/ + cp -f rts/include/rts/storage/FunTypes.h _build/stage1/src/libraries/ghc/ + cp -f rts/include/stg/MachRegs.h _build/stage1/src/libraries/ghc/ + mkdir -p _build/stage1/src/libraries/ghc/MachRegs + cp -f rts/include/stg/MachRegs/*.h _build/stage1/src/libraries/ghc/MachRegs/ python rts/gen_event_types.py --event-types-defines _build/stage1/src/libraries/rts/include/rts/EventLogConstants.h python rts/gen_event_types.py --event-types-array _build/stage1/src/libraries/rts/include/rts/EventTypes.h diff --git a/cabal.project-stage0 b/cabal.project-stage0 index 2b98ddf588a5..45eb059be6a9 100644 --- a/cabal.project-stage0 +++ b/cabal.project-stage0 @@ -22,6 +22,8 @@ packages: ./_build/stage0/src/utils/genapply/ ./_build/stage0/src/utils/deriveConstants/ +with-compiler: ghc-9.8.4 + benchmarks: False tests: False allow-boot-library-installs: True @@ -37,6 +39,10 @@ package * executable-static: False ghc-options: -fhide-source-paths -j +constraints: + -- for some reason 2.23 doesn't build + template-haskell <= 2.22 + package ghc-boot-th flags: +bootstrap diff --git a/compiler/GHC/Parser/Lexer.x b/compiler/GHC/Parser/Lexer.x index aa9d22fcb704..29d31565f989 100644 --- a/compiler/GHC/Parser/Lexer.x +++ b/compiler/GHC/Parser/Lexer.x @@ -3369,8 +3369,6 @@ topNoLayoutContainsCommas (ALRNoLayout b _ : _) = b -- If the generated alexScan/alexScanUser functions are called multiple times -- in this file, alexScanUser gets broken out into a separate function and -- increases memory usage. Make sure GHC inlines this function and optimizes it. --- https://github.com/haskell/alex/pull/262 -{-# INLINE alexScanUser #-} lexToken :: P (PsLocated Token) lexToken = do From 3589356ee07813eecbc105d9ae52e8ef826f20ea Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 8 Jan 2025 14:25:50 +0100 Subject: [PATCH 025/175] Fix more stuff --- Makefile | 10 +++++++--- cabal.project-stage1 | 5 ++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 90cdaa8f654d..f71aee034f65 100644 --- a/Makefile +++ b/Makefile @@ -103,6 +103,10 @@ _build/stage1/bin/ghc: _build/stage0/bin/ghc cp -f rts/include/stg/MachRegs.h _build/stage1/src/libraries/ghc/ mkdir -p _build/stage1/src/libraries/ghc/MachRegs cp -f rts/include/stg/MachRegs/*.h _build/stage1/src/libraries/ghc/MachRegs/ + + cp -f utils/fs/fs.h _build/stage1/src/libraries/ghc-internal/include + cp -f utils/fs/fs.c _build/stage1/src/libraries/ghc-internal/cbits + cp -f utils/fs/fs.* _build/stage1/src/libraries/rts/ python rts/gen_event_types.py --event-types-defines _build/stage1/src/libraries/rts/include/rts/EventLogConstants.h python rts/gen_event_types.py --event-types-array _build/stage1/src/libraries/rts/include/rts/EventTypes.h @@ -182,7 +186,7 @@ _build/stage1/bin/ghc: _build/stage0/bin/ghc --objdump-program objdump \ --gcc-flag "-I_build/stage1/src/libraries/rts/include" \ --gcc-flag "-I_build/stage1/src/libraries/rts" \ - --gcc-flag "-I_build/stage1/cabal/build/x86_64-linux/ghc-9.13/rts-1.0.2/build/include" + --gcc-flag "-I_build/stage1/cabal/build/x86_64-linux/ghc-9.13/rts-1.0.3/build/include" # Generate autoapply _build/stage0/bin/genapply _build/stage1/src/libraries/rts/include/DerivedConstants.h > _build/stage1/src/libraries/rts/AutoApply.cmm @@ -196,7 +200,7 @@ _build/stage1/bin/ghc: _build/stage0/bin/ghc (cd _build/stage1/src/libffi; tar -xvf ../../../../libffi-tarballs/libffi-3.4.6.tar.gz) (cd _build/stage1/src/libffi/libffi-3.4.6; ./configure --disable-docs --with-pics=yes --disable-multi-os-directory --prefix=`pwd`/../../../../../_build/stage1/libffi/ && make install -j) cp -f _build/stage1/libffi/include/* _build/stage1/src/libraries/rts/include/ - cp -f _build/stage1/libffi/lib/libffi.a _build/stage1/cabal/build/x86_64-linux/ghc-9.13/rts-1.0.2/build/libCffi.a + cp -f _build/stage1/libffi/lib/libffi.a _build/stage1/cabal/build/x86_64-linux/ghc-9.13/rts-1.0.3/build/libCffi.a # Building boot libraries mkdir -p _build/stage1/cabal/ @@ -240,7 +244,7 @@ _build/stage1/bin/ghc: _build/stage0/bin/ghc HADRIAN_SETTINGS='$(HADRIAN_SETTINGS_STAGE1)' \ $(CABAL) build --project-file=cabal.project-stage1 \ - rts ghc-prim ghc-bignum ghc-internal \ + rts ghc-prim ghc-internal \ --with-compiler=`pwd`/_build/stage0/bin/ghc \ --with-hc-pkg=`pwd`/_build/stage0/bin/ghc-pkg \ --ghc-options="-ghcversion-file=`pwd`/_build/stage1/src/libraries/rts/include/ghcversion.h" \ diff --git a/cabal.project-stage1 b/cabal.project-stage1 index 87a1a6eaf97a..5e6ec7c7c6e9 100644 --- a/cabal.project-stage1 +++ b/cabal.project-stage1 @@ -6,7 +6,6 @@ package-dbs: clear, global packages: ./_build/stage1/src/libraries/rts ./_build/stage1/src/libraries/ghc-prim - ./_build/stage1/src/libraries/ghc-bignum ./_build/stage1/src/libraries/ghc-internal benchmarks: False @@ -25,6 +24,6 @@ package * executable-static: False ghc-options: -fhide-source-paths -j -package ghc-bignum +package ghc-internal -- make our life easier for now by using the native bignum backend - flags: +native + flags: +bignum-native From a63608f89e0dbc827060d47a99e3f5f94f17669d Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 8 Jan 2025 14:32:02 +0100 Subject: [PATCH 026/175] Build base --- Makefile | 6 ++++-- cabal.project-stage1 | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index f71aee034f65..7138756a0b25 100644 --- a/Makefile +++ b/Makefile @@ -120,6 +120,7 @@ _build/stage1/bin/ghc: _build/stage0/bin/ghc cp _build/stage1/src/libraries/ghc-boot-th/ghc-boot-th.cabal{.in,} cp _build/stage1/src/libraries/ghc-heap/ghc-heap.cabal{.in,} cp _build/stage1/src/libraries/ghci/ghci.cabal{.in,} + cp _build/stage1/src/libraries/base/base.cabal{.in,} cp _build/stage1/src/libraries/rts/include/ghcversion.h{.in,} sed -i 's/@ProjectVersion@/9.13/' _build/stage1/src/ghc-bin/ghc-bin.cabal @@ -143,6 +144,7 @@ _build/stage1/bin/ghc: _build/stage0/bin/ghc sed -i 's/@ProjectVersionInt@/913/' _build/stage1/src/libraries/rts/include/ghcversion.h sed -i 's/@ProjectPatchLevel1@/0/' _build/stage1/src/libraries/rts/include/ghcversion.h sed -i 's/@ProjectPatchLevel2@/0/' _build/stage1/src/libraries/rts/include/ghcversion.h + sed -i 's/@ProjectVersionForLib@/9.13/' _build/stage1/src/libraries/base/base.cabal sed -i 's/@LlvmMinVersion@/13/' _build/stage1/src/libraries/ghc/GHC/CmmToLlvm/Version/Bounds.hs sed -i 's/@LlvmMaxVersion@/20/' _build/stage1/src/libraries/ghc/GHC/CmmToLlvm/Version/Bounds.hs @@ -244,7 +246,7 @@ _build/stage1/bin/ghc: _build/stage0/bin/ghc HADRIAN_SETTINGS='$(HADRIAN_SETTINGS_STAGE1)' \ $(CABAL) build --project-file=cabal.project-stage1 \ - rts ghc-prim ghc-internal \ + rts ghc-prim ghc-internal base \ --with-compiler=`pwd`/_build/stage0/bin/ghc \ --with-hc-pkg=`pwd`/_build/stage0/bin/ghc-pkg \ --ghc-options="-ghcversion-file=`pwd`/_build/stage1/src/libraries/rts/include/ghcversion.h" \ @@ -266,7 +268,7 @@ _build/stage1/bin/ghc: _build/stage0/bin/ghc --ghc-options='"-optc=-DTargetVendor=\"FIXME\""' \ --ghc-options='"-optc=-DGhcUnregisterised=\"FIXME\""' \ --ghc-options='"-optc=-DTablesNextToCode=\"FIXME\""' \ - --builddir=_build/stage1/cabal/ -v + --builddir=_build/stage1/cabal/ clean: rm -rf _build diff --git a/cabal.project-stage1 b/cabal.project-stage1 index 5e6ec7c7c6e9..9749bd18c690 100644 --- a/cabal.project-stage1 +++ b/cabal.project-stage1 @@ -7,6 +7,7 @@ packages: ./_build/stage1/src/libraries/rts ./_build/stage1/src/libraries/ghc-prim ./_build/stage1/src/libraries/ghc-internal + ./_build/stage1/src/libraries/base benchmarks: False tests: False From 3c0c08dc1d3850c10b4643bb7e18f21df61a9d33 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Fri, 10 Jan 2025 16:42:31 +0100 Subject: [PATCH 027/175] Build statically linked stage0 programs. --- cabal.project-stage0 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cabal.project-stage0 b/cabal.project-stage0 index 45eb059be6a9..c1c9d1f90abf 100644 --- a/cabal.project-stage0 +++ b/cabal.project-stage0 @@ -36,7 +36,7 @@ package * shared: False executable-profiling: False executable-dynamic: False - executable-static: False + executable-static: True ghc-options: -fhide-source-paths -j constraints: From e2397b403c9145370e0bb368583ae464617f6f97 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Fri, 10 Jan 2025 16:41:42 +0100 Subject: [PATCH 028/175] Add Haskell build script We need Haskell anyway for GenSettings and Makefile syntax isn't high-level enough to write quality code. --- Build.hs | 326 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 326 insertions(+) create mode 100755 Build.hs diff --git a/Build.hs b/Build.hs new file mode 100755 index 000000000000..98df88b92474 --- /dev/null +++ b/Build.hs @@ -0,0 +1,326 @@ +#!/usr/bin/env cabal +{- cabal: +build-depends: + base, + directory, + filepath, + process, + text +-} + +{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE ImportQualifiedPost #-} +{-# LANGUAGE OverloadedStrings #-} + +-- | GHC builder +-- +-- Importantly, it doesn't link with the cabal library but use cabal-install +-- program instead (compared to e.g. Hadrian). +module Main where + +import Data.Maybe +import Data.List qualified as List +import Data.Text (Text) +import Data.Text qualified as Text +import Data.Text.IO qualified as Text +import Control.Monad +import System.Environment +import System.Directory +import System.Process +import System.FilePath +import System.Exit + +main :: IO () +main = do + -- FIXME: specific patched cabal-install for now and GHC that is known to + -- work... + setEnv "CABAL" "/home/hsyl20/projects/cabal/dist-newstyle/build/x86_64-linux/ghc-9.10.1/cabal-install-3.15.0.0/x/cabal/build/cabal/cabal" + setEnv "GHC" "ghc-9.8.4" + + -- detect GHC and cabal-install to use for bootstrapping + ghc0 <- do + ghc_path <- fromMaybe "ghc" <$> lookupEnv "GHC" + findExecutable ghc_path >>= \case + Nothing -> error ("Couldn't find GHC: " ++ show ghc_path) + Just x -> pure (Ghc x) + + cabal <- do + cabal_path <- fromMaybe "cabal" <$> lookupEnv "CABAL" + findExecutable cabal_path >>= \case + Nothing -> error ("Couldn't find cabal-install: " ++ show cabal_path) + Just x -> pure (Cabal x) + + void $ readCreateProcessWithExitCode (runGhc ghc0 ["--version"]) "" + + -- build GHC stage1 + buildGhcStage1 defaultGhcBuildOptions cabal ghc0 + + +-- | Build stage1 GHC program +buildGhcStage1 :: GhcBuildOptions -> Cabal -> Ghc -> IO () +buildGhcStage1 opts cabal ghc0 = do + putStrLn "Preparing GHC sources to build GHC stage1..." + prepareGhcSources opts "_build/stage0/src/" + + let builddir = "_build/stage0/cabal/" + createDirectoryIfMissing True builddir + + putStrLn "Prepare GHC stage1 configuration..." + -- we need to augment the current environment to pass HADRIAN_SETTINGS + -- environment variable to ghc-boot's Setup.hs script. + stage0_settings <- read <$> readCreateProcess (runGhc ghc0 ["--info"]) "" + stage1_ghc_boot_settings <- do + commit_id <- readCreateProcess (proc "git" ["rev-parse", "HEAD"]) "" + -- we infer stage1's host platform from stage0's settings + let settings = + [ ("hostPlatformArch", fromMaybe (error "Couldn't read 'target arch' setting") (lookup "target arch" stage0_settings)) + , ("hostPlatformOS", fromMaybe (error "Couldn't read 'target os' setting") (lookup "target os" stage0_settings)) + , ("cProjectGitCommitId", commit_id) + , ("cProjectVersion", Text.unpack $ gboVersion opts) + , ("cProjectVersionInt", Text.unpack $ gboVersionInt opts) + , ("cProjectPatchLevel", Text.unpack $ gboVersionPatchLevel opts) + , ("cProjectPatchLevel1", Text.unpack $ gboVersionPatchLevel1 opts) + , ("cProjectPatchLevel2", Text.unpack $ gboVersionPatchLevel2 opts) + ] + pure (show settings) + + current_env <- getEnvironment + let stage1_env = ("HADRIAN_SETTINGS", stage1_ghc_boot_settings) : current_env + + putStrLn "Building GHC stage1 and bootstrapping utility programs..." + let build_cmd = (runCabal cabal + [ "build" + , "--project-file=cabal.project-stage0" + , "--builddir=" ++ builddir + , "-j" + -- the targets + , "ghc-bin:ghc" + , "ghc-pkg:ghc-pkg" + , "genprimopcode:genprimopcode" + , "deriveConstants:deriveConstants" + , "genapply:genapply" + ]) + { env = Just stage1_env + } + (exit_code, cabal_stdout, cabal_stderr) <- readCreateProcessWithExitCode build_cmd "" + writeFile "_build/stage0/cabal.stdout" cabal_stdout + writeFile "_build/stage0/cabal.stderr" cabal_stderr + case exit_code of + ExitSuccess -> pure () + ExitFailure n -> do + putStrLn $ "cabal-install failed with error code: " ++ show n + putStrLn "Logs can be found in \"_build/stage0/cabal.{stdout,stderr}\"" + exitFailure + + putStrLn "Copying stage0 programs and generating settings to use them..." + let listbin_cmd p = runCabal cabal + [ "list-bin" + , "--project-file=cabal.project-stage0" + , "--builddir=" ++ builddir + , p + ] + let copy_bin target bin = do + (list_bin_exit_code, list_bin_stdout, list_bin_stderr) <- readCreateProcessWithExitCode (listbin_cmd target) "" + case list_bin_exit_code of + ExitSuccess + | (bin_src:_) <- lines list_bin_stdout + -> cp bin_src ("_build/stage0/bin" bin) + _ -> do + putStrLn $ "Failed to run cabal list-bin for the target: " ++ show target + putStrLn list_bin_stderr + exitFailure + createDirectoryIfMissing True "_build/stage0/bin" + copy_bin "ghc-bin:ghc" "ghc" + copy_bin "ghc-pkg:ghc-pkg" "ghc-pkg" + copy_bin "deriveConstants:deriveConstants" "deriveConstants" + copy_bin "genprimopcode:genprimopcode" "genprimopcode" + copy_bin "genapply:genapply" "genapply" + + -- generate settings based on stage1 compiler settings + createDirectoryIfMissing True "_build/stage0/lib" + let stage1_settings = makeStage1Settings stage0_settings + writeFile "_build/stage0/lib/settings" (show stage1_settings) + + -- try to run the stage1 compiler (no package db yet, so just display the + -- version) + (test_exit_code, test_stdout, _test_stderr) <- readCreateProcessWithExitCode (proc "_build/stage0/bin/ghc" ["--version"]) "" + case test_exit_code of + ExitSuccess -> pure () + ExitFailure n -> do + putStrLn $ "Failed to run stage1 compiler with error code " ++ show n + exitFailure + + + + +data GhcBuildOptions = GhcBuildOptions + { gboVersion :: !Text -- ^ GHC version + , gboVersionInt :: !Text -- ^ GHC version as an Int + , gboVersionMunged :: !Text -- ^ GHC version "munged" + , gboVersionForLib :: !Text -- ^ GHC version for libraries? + , gboVersionPatchLevel :: !Text -- ^ GHC patchlevel version + , gboVersionPatchLevel1 :: !Text -- ^ GHC patchlevel1 version + , gboVersionPatchLevel2 :: !Text -- ^ GHC patchlevel2 version + , gboLlvmMinVersion :: !Text -- ^ Min LLVM version supported + , gboLlvmMaxVersion :: !Text -- ^ Max LLVM version supported + } + +defaultGhcBuildOptions :: GhcBuildOptions +defaultGhcBuildOptions = GhcBuildOptions + { gboVersion = "9.13" + , gboVersionInt = "913" + , gboVersionMunged = "9.13" + , gboVersionForLib = "9.13" + , gboVersionPatchLevel = "0" + , gboVersionPatchLevel1 = "0" + , gboVersionPatchLevel2 = "0" + , gboLlvmMinVersion = "13" + , gboLlvmMaxVersion = "20" + } + + +-- | Prepare GHC sources in the given directory +prepareGhcSources :: GhcBuildOptions -> FilePath -> IO () +prepareGhcSources opts dst = do + createDirectoryIfMissing True dst + + cp "./libraries" dst + cp "./compiler" (dst "libraries/ghc") + cp "./ghc" (dst "ghc-bin") + cp "./utils" dst + + cp "rts/include/rts/Bytecodes.h" (dst "libraries/ghc/") + cp "rts/include/rts/storage/ClosureTypes.h" (dst "libraries/ghc/") + cp "rts/include/rts/storage/FunTypes.h" (dst "libraries/ghc/") + cp "rts/include/stg/MachRegs.h" (dst "libraries/ghc/") + createDirectoryIfMissing True (dst "libraries/ghc/MachRegs") + cp "rts/include/stg/MachRegs/*.h" (dst "libraries/ghc/MachRegs/") + + -- substitute variables in files + let subst fin fout rs = do + t <- Text.readFile fin + Text.writeFile fout (List.foldl' (\v (needle,rep) -> Text.replace needle rep v) t rs) + let subst_in f = subst (f <.> "in") f + let common_substs = + [ (,) "@ProjectVersion@" (gboVersion opts) + , (,) "@ProjectVersionMunged@" (gboVersionMunged opts) + , (,) "@ProjectVersionForLib@" (gboVersionForLib opts) + ] + llvm_substs = + [ (,) "@LlvmMinVersion@" (gboLlvmMinVersion opts) + , (,) "@LlvmMaxVersion@" (gboLlvmMaxVersion opts) + ] + boot_th_substs = + [ (,) "@Suffix@" "" + , (,) "@SourceRoot@" "." + ] + + subst_in (dst "ghc-bin/ghc-bin.cabal") common_substs + subst_in (dst "libraries/ghc/ghc.cabal") common_substs + subst_in (dst "libraries/ghc-boot/ghc-boot.cabal") common_substs + subst_in (dst "libraries/ghc-boot-th/ghc-boot-th.cabal") (common_substs ++ boot_th_substs) + subst_in (dst "libraries/ghc-heap/ghc-heap.cabal") common_substs + subst_in (dst "libraries/ghci/ghci.cabal") common_substs + subst_in (dst "libraries/ghc/GHC/CmmToLlvm/Version/Bounds.hs") llvm_substs + subst_in (dst "utils/ghc-pkg/ghc-pkg.cabal") common_substs + +-- Avoid FilePath blindness by using type aliases for programs. +newtype Ghc = Ghc FilePath +newtype Cabal = Cabal FilePath + +runGhc :: Ghc -> [String] -> CreateProcess +runGhc (Ghc f) = proc f + +runCabal :: Cabal -> [String] -> CreateProcess +runCabal (Cabal f) = proc f + +cp :: String -> String -> IO () +cp src dst = void (readCreateProcessWithExitCode (shell $ "cp -rf " ++ src ++ " " ++ dst) "") + +-- | Generate settings for stage1 compiler, based on given settings (stage0's +-- compiler settings) +makeStage1Settings :: [(String,String)] -> [(String,String)] +makeStage1Settings in_settings = out_settings + where + -- keep the previous setting, fail if it doesn't exist + keep_fail s = keep_def s (error ("Couldn't find setting "<> show s)) + + -- keep the previous setting, default to the given value if it doesn't exist + keep_def s d = case lookup s in_settings of + Nothing -> (s,d) + Just v -> (s,v) + + -- use the previous setting, or if it doesn't exist use the setting for the + -- second key. Fail if both don't exist. This is useful to support + -- bootstrapping with old compilers that mingled some settings. + keep_or_fail s s2 = case lookup s in_settings of + Nothing -> case lookup s2 in_settings of + Nothing -> error ("Couldn't find any of " <> show s <> " and " <> show s2) + Just v -> (s,v) + Just v -> (s,v) + + --FIXME: we default to these flags for Cmm CPP, otherwise CPP fails + -- with error: missing '(' after "__has_feature" + -- because we pass `-traditional` while compiling Apply.cmm (in TSANUtils.h) + default_cpp_flags = "-E" + + out_settings = + [ keep_fail "C compiler command" + , keep_fail "C compiler flags" + , keep_fail "C++ compiler command" + , keep_fail "C++ compiler flags" + , keep_fail "C compiler link flags" + , keep_fail "C compiler supports -no-pie" + , keep_or_fail "CPP command" "Haskell CPP command" + , keep_def "CPP flags" default_cpp_flags + , keep_fail "Haskell CPP command" + , keep_fail "Haskell CPP flags" + , keep_or_fail "JavaScript CPP command" "Haskell CPP command" + , keep_or_fail "JavaScript CPP flags" "Haskell CPP flags" + , keep_or_fail "C-- CPP command" "Haskell CPP command" + , keep_def "C-- CPP flags" default_cpp_flags + , keep_def "C-- CPP supports -g0" "NO" + , keep_fail "ld supports compact unwind" + , keep_fail "ld supports filelist" + , keep_fail "ld supports single module" + , keep_fail "ld is GNU ld" + , keep_fail "Merge objects command" + , keep_fail "Merge objects flags" + , keep_def "Merge objects supports response files" "NO" + , keep_fail "ar command" + , keep_fail "ar flags" + , keep_fail "ar supports at file" + , keep_fail "ar supports -L" + , keep_fail "ranlib command" + , keep_fail "otool command" + , keep_fail "install_name_tool command" + , keep_fail "windres command" + , keep_fail "unlit command" + , keep_fail "cross compiling" + , keep_fail "target platform string" + , keep_fail "target os" + , keep_fail "target arch" + , keep_fail "target word size" + , keep_fail "target word big endian" + , keep_fail "target has GNU nonexec stack" + , keep_fail "target has .ident directive" + , keep_fail "target has subsections via symbols" + , keep_fail "target has libm" + , keep_fail "Unregisterised" + , keep_fail "LLVM target" + , keep_fail "LLVM llc command" + , keep_fail "LLVM opt command" + , keep_def "LLVM llvm-as command" "llvm-as" + , keep_fail "Use inplace MinGW toolchain" + + , keep_def "target RTS linker only supports shared libraries" "NO" + , ("Use interpreter", "NO") + , ("base unit-id", "base") + , keep_fail "Support SMP" + , keep_fail "RTS ways" + , keep_fail "Tables next to code" + , keep_fail "Leading underscore" + , keep_fail "Use LibFFI" + , keep_fail "RTS expects libdw" + , ("Relative Global Package DB", "../../stage1/pkgs") + ] From 4fee18653426b55fb91f61ee8845aaf39a626e62 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Mon, 13 Jan 2025 17:31:08 +0100 Subject: [PATCH 029/175] Build stage1 boot libraries with Build.hs --- Build.hs | 232 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 228 insertions(+), 4 deletions(-) diff --git a/Build.hs b/Build.hs index 98df88b92474..fdd2fdaa47b6 100755 --- a/Build.hs +++ b/Build.hs @@ -5,12 +5,15 @@ build-depends: directory, filepath, process, - text + text, + temporary -} {-# LANGUAGE LambdaCase #-} {-# LANGUAGE ImportQualifiedPost #-} {-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE DerivingStrategies #-} +{-# LANGUAGE GeneralizedNewtypeDeriving #-} -- | GHC builder -- @@ -29,6 +32,7 @@ import System.Directory import System.Process import System.FilePath import System.Exit +import System.IO.Temp main :: IO () main = do @@ -55,6 +59,15 @@ main = do -- build GHC stage1 buildGhcStage1 defaultGhcBuildOptions cabal ghc0 + ghc1 <- Ghc <$> makeAbsolute "_build/stage0/bin/ghc" + ghcPkg1 <- GhcPkg <$> makeAbsolute "_build/stage0/bin/ghc-pkg" + deriveConstants <- DeriveConstants <$> makeAbsolute "_build/stage0/bin/deriveConstants" + genapply <- GenApply <$> makeAbsolute "_build/stage0/bin/genapply" + genprimop <- GenPrimop <$> makeAbsolute "_build/stage0/bin/genprimopcode" + + -- build boot libraries with stage1 compiler + buildBootLibraries cabal ghc1 ghcPkg1 deriveConstants genapply genprimop defaultGhcBuildOptions + -- | Build stage1 GHC program buildGhcStage1 :: GhcBuildOptions -> Cabal -> Ghc -> IO () @@ -136,6 +149,20 @@ buildGhcStage1 opts cabal ghc0 = do copy_bin "genprimopcode:genprimopcode" "genprimopcode" copy_bin "genapply:genapply" "genapply" + -- initialize empty global package database + pkgdb <- makeAbsolute "_build/stage1/pkgs" + doesDirectoryExist pkgdb >>= \case + True -> pure () -- don't try to recreate the DB if it already exist as it would fail + False -> do + ghcpkg <- GhcPkg <$> makeAbsolute "_build/stage0/bin/ghc-pkg" + void $ readCreateProcess (runGhcPkg ghcpkg ["init", pkgdb]) "" + void $ readCreateProcess (runGhcPkg ghcpkg + [ "recache" + , "--global-package-db="++pkgdb + , "--no-user-package-db" + ]) "" + + -- generate settings based on stage1 compiler settings createDirectoryIfMissing True "_build/stage0/lib" let stage1_settings = makeStage1Settings stage0_settings @@ -151,8 +178,6 @@ buildGhcStage1 opts cabal ghc0 = do exitFailure - - data GhcBuildOptions = GhcBuildOptions { gboVersion :: !Text -- ^ GHC version , gboVersionInt :: !Text -- ^ GHC version as an Int @@ -186,9 +211,13 @@ prepareGhcSources opts dst = do cp "./libraries" dst cp "./compiler" (dst "libraries/ghc") + cp "./rts" (dst "libraries/") cp "./ghc" (dst "ghc-bin") cp "./utils" dst + cp "./config.sub" (dst "libraries/rts/") + cp "./config.guess" (dst "libraries/rts/") + cp "rts/include/rts/Bytecodes.h" (dst "libraries/ghc/") cp "rts/include/rts/storage/ClosureTypes.h" (dst "libraries/ghc/") cp "rts/include/rts/storage/FunTypes.h" (dst "libraries/ghc/") @@ -196,6 +225,25 @@ prepareGhcSources opts dst = do createDirectoryIfMissing True (dst "libraries/ghc/MachRegs") cp "rts/include/stg/MachRegs/*.h" (dst "libraries/ghc/MachRegs/") + cp "utils/fs/fs.h" (dst "libraries/ghc-internal/include") + cp "utils/fs/fs.c" (dst "libraries/ghc-internal/cbits") + cp "utils/fs/fs.*" (dst "libraries/rts/") + + python <- findExecutable "python" >>= \case + Nothing -> error "Couldn't find 'python'" + Just r -> pure r + + void $ readCreateProcess (proc python + [ "rts/gen_event_types.py" + , "--event-types-defines" + , dst "libraries/rts/include/rts/EventLogConstants.h" + ]) "" + void $ readCreateProcess (proc python + [ "rts/gen_event_types.py" + , "--event-types-array" + , dst "libraries/rts/include/rts/EventTypes.h" + ]) "" + -- substitute variables in files let subst fin fout rs = do t <- Text.readFile fin @@ -205,6 +253,9 @@ prepareGhcSources opts dst = do [ (,) "@ProjectVersion@" (gboVersion opts) , (,) "@ProjectVersionMunged@" (gboVersionMunged opts) , (,) "@ProjectVersionForLib@" (gboVersionForLib opts) + , (,) "@ProjectPatchLevel1@" (gboVersionPatchLevel1 opts) + , (,) "@ProjectPatchLevel2@" (gboVersionPatchLevel2 opts) + , (,) "@ProjectVersionInt@" (gboVersionInt opts) ] llvm_substs = [ (,) "@LlvmMinVersion@" (gboLlvmMinVersion opts) @@ -224,18 +275,44 @@ prepareGhcSources opts dst = do subst_in (dst "libraries/ghc/GHC/CmmToLlvm/Version/Bounds.hs") llvm_substs subst_in (dst "utils/ghc-pkg/ghc-pkg.cabal") common_substs + subst_in (dst "libraries/ghc-internal/ghc-internal.cabal") common_substs + subst_in (dst "libraries/base/base.cabal") common_substs + subst_in (dst "libraries/rts/include/ghcversion.h") common_substs + -- Avoid FilePath blindness by using type aliases for programs. newtype Ghc = Ghc FilePath +newtype GhcPkg = GhcPkg FilePath newtype Cabal = Cabal FilePath +newtype DeriveConstants = DeriveConstants FilePath +newtype GenApply = GenApply FilePath +newtype GenPrimop = GenPrimop FilePath runGhc :: Ghc -> [String] -> CreateProcess runGhc (Ghc f) = proc f +ghcPath :: Ghc -> FilePath +ghcPath (Ghc x) = x + +runGhcPkg :: GhcPkg -> [String] -> CreateProcess +runGhcPkg (GhcPkg f) = proc f + +ghcPkgPath :: GhcPkg -> FilePath +ghcPkgPath (GhcPkg x) = x + runCabal :: Cabal -> [String] -> CreateProcess runCabal (Cabal f) = proc f +runDeriveConstants :: DeriveConstants -> [String] -> CreateProcess +runDeriveConstants (DeriveConstants f) = proc f + +runGenApply :: GenApply -> [String] -> CreateProcess +runGenApply (GenApply f) = proc f + +runGenPrimop :: GenPrimop -> [String] -> CreateProcess +runGenPrimop (GenPrimop f) = proc f + cp :: String -> String -> IO () -cp src dst = void (readCreateProcessWithExitCode (shell $ "cp -rf " ++ src ++ " " ++ dst) "") +cp src dst = void (readCreateProcess (shell $ "cp -rf " ++ src ++ " " ++ dst) "") -- | Generate settings for stage1 compiler, based on given settings (stage0's -- compiler settings) @@ -324,3 +401,150 @@ makeStage1Settings in_settings = out_settings , keep_fail "RTS expects libdw" , ("Relative Global Package DB", "../../stage1/pkgs") ] + +buildBootLibraries :: Cabal -> Ghc -> GhcPkg -> DeriveConstants -> GenApply -> GenPrimop -> GhcBuildOptions -> IO () +buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts = do + -- FIXME: should be parameters + let dst = "_build/stage1/" + src <- makeAbsolute "_build/stage1/src" + + putStrLn "Preparing GHC sources to build GHC stage2..." + prepareGhcSources opts src + + -- Build the RTS + putStrLn "Building the RTS..." + src_rts <- makeAbsolute (src "libraries/rts") + build_dir <- makeAbsolute (dst "cabal") + ghcversionh <- makeAbsolute (src_rts "include/ghcversion.h") + + putStrLn "Generating RTS headers..." + + let build_rts_cmd = runCabal cabal + [ "build" + , "--project-file=cabal.project-stage1-rts" -- TODO: replace with command-line args + , "rts" + , "--with-compiler=" ++ ghcPath ghc -- FIXME: escape path + , "--with-hc-pkg=" ++ ghcPkgPath ghcpkg -- FIXME: escape path + , "--ghc-options=\"-ghcversion-file=" ++ ghcversionh ++ "\"" + , "--ghc-options=\"-I" ++ (src_rts "include") ++ "\"" + , "--ghc-options=\"-I" ++ src_rts ++ "\"" + , "--builddir=" ++ build_dir + -- we need to pass "-this-unit-id=rts", otherwise GHC tries to lookup the + -- platform constants in the package db and fails. The flag is already + -- set in rts.cabal but for some reason it isn't always passed :shrug: + , "--ghc-options=-this-unit-id=rts" + , "--ghc-options=-DProjectVersion=913" + , "--ghc-options=\"-optc=-DProjectVersion=\\\"913\\\"\"" + , "--ghc-options=\"-optc=-DRtsWay=\\\"FIXME\\\"\"" + , "--ghc-options=\"-optc=-DHostPlatform=\\\"FIXME\\\"\"" + , "--ghc-options=\"-optc=-DHostArch=\\\"FIXME\\\"\"" + , "--ghc-options=\"-optc=-DHostOS=\\\"FIXME\\\"\"" + , "--ghc-options=\"-optc=-DHostVendor=\\\"FIXME\\\"\"" + , "--ghc-options=\"-optc=-DBuildPlatform=\\\"FIXME\\\"\"" + , "--ghc-options=\"-optc=-DBuildArch=\\\"FIXME\\\"\"" + , "--ghc-options=\"-optc=-DBuildOS=\\\"FIXME\\\"\"" + , "--ghc-options=\"-optc=-DBuildVendor=\\\"FIXME\\\"\"" + , "--ghc-options=\"-optc=-DTargetPlatform=\\\"FIXME\\\"\"" + , "--ghc-options=\"-optc=-DTargetArch=\\\"FIXME\\\"\"" + , "--ghc-options=\"-optc=-DTargetOS=\\\"FIXME\\\"\"" + , "--ghc-options=\"-optc=-DTargetVendor=\\\"FIXME\\\"\"" + , "--ghc-options=\"-optc=-DGhcUnregisterised=\\\"FIXME\\\"\"" + , "--ghc-options=\"-optc=-DTablesNextToCode=\\\"FIXME\\\"\"" + ] + + -- FIXME: deriveConstants requires ghcautoconf.h and ghcplatform.h but these + -- files are generated by the configure script of the RTS... + -- We use the following hack: + -- 1. run cabal until it fails. This should generate the headers we need before failing. + -- 2. use deriveConstants to generate the other files + -- 3. rerun cabal to build the rts + + -- first run is expected to fail because of misssing headers + void $ readCreateProcessWithExitCode build_rts_cmd "" + ghcplatform_dir <- takeDirectory <$> readCreateProcess (shell ("find " ++ build_dir ++ " -name ghcplatform.h")) "" + + -- deriving constants + let derived_constants = src_rts "include/DerivedConstants.h" + withSystemTempDirectory "derive-constants" $ \tmp_dir -> do + void $ readCreateProcess (runDeriveConstants derive_constants + [ "--gen-header" + , "-o", derived_constants + , "--target-os", "linux" -- FIXME + , "--tmpdir", tmp_dir + , "--gcc-program", "gcc" -- FIXME + , "--nm-program", "nm" -- FIXME + , "--objdump-program", "objdump" -- FIXME + , "--gcc-flag", "-I" ++ src_rts "include" + , "--gcc-flag", "-I" ++ src_rts + , "--gcc-flag", "-I" ++ ghcplatform_dir + ]) "" + + -- Generate autoapply + let run_genapply args out = writeFile out =<< readCreateProcess (runGenApply genapply args) "" + run_genapply [derived_constants] (src_rts "AutoApply.cmm") + run_genapply [derived_constants, "-V16"] (src_rts "AutoApply_V16.cmm") + run_genapply [derived_constants, "-V32"] (src_rts "AutoApply_V32.cmm") + run_genapply [derived_constants, "-V64"] (src_rts "AutoApply_V64.cmm") + + -- Generate genprimopcode + let primops_txt = src "libraries/ghc/GHC/Builtin/primops.txt" + let primops_txt_pp = primops_txt <.> ".pp" + primops <- readCreateProcess (shell $ "gcc -E -undef -traditional -P -x c " ++ primops_txt_pp) "" + writeFile primops_txt primops + writeFile (src "libraries/ghc-prim/GHC/Prim.hs") =<< readCreateProcess (runGenPrimop genprimop ["--make-haskell-source"]) primops + writeFile (src "libraries/ghc-prim/GHC/PrimopWrappers.hs") =<< readCreateProcess (runGenPrimop genprimop ["--make-haskell-wrappers"]) primops + + -- build libffi + putStrLn "Building libffi..." + src_libffi <- makeAbsolute (src "libffi") + dst_libffi <- makeAbsolute (dst "libffi") + let libffi_version = "3.4.6" + createDirectoryIfMissing True src_libffi + createDirectoryIfMissing True dst_libffi + void $ readCreateProcess (shell ("tar -xvf libffi-tarballs/libffi-" ++ libffi_version ++ ".tar.gz -C " ++ src_libffi)) "" + let build_libffi = mconcat + [ "cd " ++ src_libffi "libffi-" ++ libffi_version ++ "; " + , "./configure --disable-docs --with-pic=yes --disable-multi-os-directory --prefix=" ++ dst_libffi + , " && make install -j" + ] + (libffi_exit_code, libffi_stdout, libffi_stderr) <- readCreateProcessWithExitCode (shell build_libffi) "" + case libffi_exit_code of + ExitSuccess -> pure () + ExitFailure r -> do + putStrLn $ "Failed to build libffi with error code " ++ show r + putStrLn libffi_stdout + putStrLn libffi_stderr + exitFailure + cp (dst_libffi "include" "*") (src_rts "include") + cp (dst_libffi "lib" "libffi.a") (takeDirectory ghcplatform_dir "libCffi.a") + + -- second run of cabal is expected to succeed now that have generated all the headers! + (rts_exit_code, rts_stdout, rts_stderr) <- readCreateProcessWithExitCode build_rts_cmd "" + case rts_exit_code of + ExitSuccess -> pure () + ExitFailure r -> do + putStrLn $ "Failed to build the RTS with error code " ++ show r + putStrLn rts_stdout + putStrLn rts_stderr + exitFailure + + -- build boot libraries: ghc-internal, base... but not GHC itself + let build_boot_cmd = runCabal cabal + [ "build" + , "--project-file=cabal.project-stage1" -- TODO: replace with command-line args + , "ghc-prim", "ghc-internal", "base" + , "--with-compiler=" ++ ghcPath ghc -- FIXME: escape path + , "--with-hc-pkg=" ++ ghcPkgPath ghcpkg -- FIXME: escape path + , "--ghc-options=\"-ghcversion-file=" ++ ghcversionh ++ "\"" + , "--builddir=" ++ build_dir + ] + + putStrLn "Building boot libraries..." + (boot_exit_code, boot_stdout, boot_stderr) <- readCreateProcessWithExitCode build_boot_cmd "" + case boot_exit_code of + ExitSuccess -> pure () + ExitFailure r -> do + putStrLn $ "Failed to build boot libraries with error code " ++ show r + putStrLn boot_stdout + putStrLn boot_stderr + exitFailure From c9aa65dcd9fa1df2769c91465212748651db294f Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Mon, 13 Jan 2025 17:47:25 +0100 Subject: [PATCH 030/175] Add timing to messages --- Build.hs | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/Build.hs b/Build.hs index fdd2fdaa47b6..b80a0b573735 100755 --- a/Build.hs +++ b/Build.hs @@ -14,6 +14,7 @@ build-depends: {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE DerivingStrategies #-} {-# LANGUAGE GeneralizedNewtypeDeriving #-} +{-# LANGUAGE NumericUnderscores #-} -- | GHC builder -- @@ -33,6 +34,7 @@ import System.Process import System.FilePath import System.Exit import System.IO.Temp +import System.CPUTime main :: IO () main = do @@ -72,13 +74,13 @@ main = do -- | Build stage1 GHC program buildGhcStage1 :: GhcBuildOptions -> Cabal -> Ghc -> IO () buildGhcStage1 opts cabal ghc0 = do - putStrLn "Preparing GHC sources to build GHC stage1..." + msg "Preparing GHC sources to build GHC stage1..." prepareGhcSources opts "_build/stage0/src/" let builddir = "_build/stage0/cabal/" createDirectoryIfMissing True builddir - putStrLn "Prepare GHC stage1 configuration..." + msg "Prepare GHC stage1 configuration..." -- we need to augment the current environment to pass HADRIAN_SETTINGS -- environment variable to ghc-boot's Setup.hs script. stage0_settings <- read <$> readCreateProcess (runGhc ghc0 ["--info"]) "" @@ -100,7 +102,7 @@ buildGhcStage1 opts cabal ghc0 = do current_env <- getEnvironment let stage1_env = ("HADRIAN_SETTINGS", stage1_ghc_boot_settings) : current_env - putStrLn "Building GHC stage1 and bootstrapping utility programs..." + msg "Building GHC stage1 and bootstrapping utility programs..." let build_cmd = (runCabal cabal [ "build" , "--project-file=cabal.project-stage0" @@ -125,7 +127,7 @@ buildGhcStage1 opts cabal ghc0 = do putStrLn "Logs can be found in \"_build/stage0/cabal.{stdout,stderr}\"" exitFailure - putStrLn "Copying stage0 programs and generating settings to use them..." + msg "Copying stage0 programs and generating settings to use them..." let listbin_cmd p = runCabal cabal [ "list-bin" , "--project-file=cabal.project-stage0" @@ -408,16 +410,16 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts = d let dst = "_build/stage1/" src <- makeAbsolute "_build/stage1/src" - putStrLn "Preparing GHC sources to build GHC stage2..." + msg "Preparing GHC sources to build GHC stage2..." prepareGhcSources opts src -- Build the RTS - putStrLn "Building the RTS..." + msg "Building the RTS..." src_rts <- makeAbsolute (src "libraries/rts") build_dir <- makeAbsolute (dst "cabal") ghcversionh <- makeAbsolute (src_rts "include/ghcversion.h") - putStrLn "Generating RTS headers..." + msg "Generating RTS headers..." let build_rts_cmd = runCabal cabal [ "build" @@ -495,7 +497,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts = d writeFile (src "libraries/ghc-prim/GHC/PrimopWrappers.hs") =<< readCreateProcess (runGenPrimop genprimop ["--make-haskell-wrappers"]) primops -- build libffi - putStrLn "Building libffi..." + msg "Building libffi..." src_libffi <- makeAbsolute (src "libffi") dst_libffi <- makeAbsolute (dst "libffi") let libffi_version = "3.4.6" @@ -539,7 +541,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts = d , "--builddir=" ++ build_dir ] - putStrLn "Building boot libraries..." + msg "Building boot libraries..." (boot_exit_code, boot_stdout, boot_stderr) <- readCreateProcessWithExitCode build_boot_cmd "" case boot_exit_code of ExitSuccess -> pure () @@ -548,3 +550,11 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts = d putStrLn boot_stdout putStrLn boot_stderr exitFailure + + + +msg :: String -> IO () +msg x = do + t <- getCPUTime + let d = t `div` 1_000_000_000 + putStrLn ("[" ++ show d ++ "] " ++ x) From cc421cf87f7d64bc6b404b0413edc7fc009550f8 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Mon, 13 Jan 2025 17:57:04 +0100 Subject: [PATCH 031/175] Fix messages --- Build.hs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Build.hs b/Build.hs index b80a0b573735..5b329448687b 100755 --- a/Build.hs +++ b/Build.hs @@ -70,6 +70,8 @@ main = do -- build boot libraries with stage1 compiler buildBootLibraries cabal ghc1 ghcPkg1 deriveConstants genapply genprimop defaultGhcBuildOptions + msg "Done" + -- | Build stage1 GHC program buildGhcStage1 :: GhcBuildOptions -> Cabal -> Ghc -> IO () @@ -419,7 +421,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts = d build_dir <- makeAbsolute (dst "cabal") ghcversionh <- makeAbsolute (src_rts "include/ghcversion.h") - msg "Generating RTS headers..." + msg " - Generating RTS headers..." let build_rts_cmd = runCabal cabal [ "build" @@ -497,7 +499,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts = d writeFile (src "libraries/ghc-prim/GHC/PrimopWrappers.hs") =<< readCreateProcess (runGenPrimop genprimop ["--make-haskell-wrappers"]) primops -- build libffi - msg "Building libffi..." + msg " - Building libffi..." src_libffi <- makeAbsolute (src "libffi") dst_libffi <- makeAbsolute (dst "libffi") let libffi_version = "3.4.6" From 7b0e5ca46bd2050f910b0b53ae0a8c1a2cb51be8 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 14 Jan 2025 10:42:46 +0100 Subject: [PATCH 032/175] Don't wire in the compiler --- Build.hs | 1 + cabal.project-stage0 | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Build.hs b/Build.hs index 5b329448687b..af9ffd5f1c9c 100755 --- a/Build.hs +++ b/Build.hs @@ -110,6 +110,7 @@ buildGhcStage1 opts cabal ghc0 = do , "--project-file=cabal.project-stage0" , "--builddir=" ++ builddir , "-j" + , "--with-compiler=" ++ ghcPath ghc0 -- FIXME: escape path -- the targets , "ghc-bin:ghc" , "ghc-pkg:ghc-pkg" diff --git a/cabal.project-stage0 b/cabal.project-stage0 index c1c9d1f90abf..069fc5116941 100644 --- a/cabal.project-stage0 +++ b/cabal.project-stage0 @@ -22,8 +22,6 @@ packages: ./_build/stage0/src/utils/genapply/ ./_build/stage0/src/utils/deriveConstants/ -with-compiler: ghc-9.8.4 - benchmarks: False tests: False allow-boot-library-installs: True From dbada6e7e6427b7d45a35bc42a1a3982e2e89fea Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 14 Jan 2025 10:47:41 +0100 Subject: [PATCH 033/175] Fix options --- cabal.project-stage0 | 1 - cabal.project-stage1 | 4 ---- cabal.project-stage1-rts | 4 ---- 3 files changed, 9 deletions(-) diff --git a/cabal.project-stage0 b/cabal.project-stage0 index 069fc5116941..7fa4dfb9a98c 100644 --- a/cabal.project-stage0 +++ b/cabal.project-stage0 @@ -35,7 +35,6 @@ package * executable-profiling: False executable-dynamic: False executable-static: True - ghc-options: -fhide-source-paths -j constraints: -- for some reason 2.23 doesn't build diff --git a/cabal.project-stage1 b/cabal.project-stage1 index 9749bd18c690..1e76d2d406ad 100644 --- a/cabal.project-stage1 +++ b/cabal.project-stage1 @@ -1,6 +1,3 @@ --- for some reason it doesn't work... ---with-compiler: ./_build/stage0/bin/ghc ---with-hc-pkg: ./_build/stage0/bin/ghc-pkg package-dbs: clear, global packages: @@ -23,7 +20,6 @@ package * executable-profiling: False executable-dynamic: False executable-static: False - ghc-options: -fhide-source-paths -j package ghc-internal -- make our life easier for now by using the native bignum backend diff --git a/cabal.project-stage1-rts b/cabal.project-stage1-rts index dfa9d54f072b..8deeec5d01c3 100644 --- a/cabal.project-stage1-rts +++ b/cabal.project-stage1-rts @@ -1,6 +1,3 @@ --- for some reason it doesn't work... ---with-compiler: ./_build/stage0/bin/ghc ---with-hc-pkg: ./_build/stage0/bin/ghc-pkg package-dbs: clear, global packages: @@ -20,4 +17,3 @@ package * executable-profiling: False executable-dynamic: False executable-static: False - ghc-options: -fhide-source-paths -j From d611659c5fd513f0be108ac7d4722e9770a73ac9 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 14 Jan 2025 11:49:41 +0100 Subject: [PATCH 034/175] Fix compiler path --- Build.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/Build.hs b/Build.hs index af9ffd5f1c9c..27f6f4ab8ea8 100755 --- a/Build.hs +++ b/Build.hs @@ -134,6 +134,7 @@ buildGhcStage1 opts cabal ghc0 = do let listbin_cmd p = runCabal cabal [ "list-bin" , "--project-file=cabal.project-stage0" + , "--with-compiler=" ++ ghcPath ghc0 -- FIXME: escape path , "--builddir=" ++ builddir , p ] From 8822716600b3da0ddc856cc1290f5e8b0b33ca6f Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 14 Jan 2025 12:10:31 +0100 Subject: [PATCH 035/175] Minor refactor --- Build.hs | 103 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 55 insertions(+), 48 deletions(-) diff --git a/Build.hs b/Build.hs index 27f6f4ab8ea8..aef759cbf75e 100755 --- a/Build.hs +++ b/Build.hs @@ -56,9 +56,10 @@ main = do Nothing -> error ("Couldn't find cabal-install: " ++ show cabal_path) Just x -> pure (Cabal x) - void $ readCreateProcessWithExitCode (runGhc ghc0 ["--version"]) "" + ghc0_version <- readCreateProcess (runGhc ghc0 ["--version"]) "" + msg $ "Bootstrapping GHC version: " ++ init ghc0_version - -- build GHC stage1 + msg "Building stage1 GHC program and utility programs" buildGhcStage1 defaultGhcBuildOptions cabal ghc0 ghc1 <- Ghc <$> makeAbsolute "_build/stage0/bin/ghc" @@ -67,7 +68,7 @@ main = do genapply <- GenApply <$> makeAbsolute "_build/stage0/bin/genapply" genprimop <- GenPrimop <$> makeAbsolute "_build/stage0/bin/genprimopcode" - -- build boot libraries with stage1 compiler + msg "Building boot libraries with stage1 compiler..." buildBootLibraries cabal ghc1 ghcPkg1 deriveConstants genapply genprimop defaultGhcBuildOptions msg "Done" @@ -76,13 +77,12 @@ main = do -- | Build stage1 GHC program buildGhcStage1 :: GhcBuildOptions -> Cabal -> Ghc -> IO () buildGhcStage1 opts cabal ghc0 = do - msg "Preparing GHC sources to build GHC stage1..." + msg " - Preparing sources to build with GHC stage0..." prepareGhcSources opts "_build/stage0/src/" let builddir = "_build/stage0/cabal/" createDirectoryIfMissing True builddir - msg "Prepare GHC stage1 configuration..." -- we need to augment the current environment to pass HADRIAN_SETTINGS -- environment variable to ghc-boot's Setup.hs script. stage0_settings <- read <$> readCreateProcess (runGhc ghc0 ["--info"]) "" @@ -104,7 +104,7 @@ buildGhcStage1 opts cabal ghc0 = do current_env <- getEnvironment let stage1_env = ("HADRIAN_SETTINGS", stage1_ghc_boot_settings) : current_env - msg "Building GHC stage1 and bootstrapping utility programs..." + msg " - Building GHC stage1 and bootstrapping utility programs..." let build_cmd = (runCabal cabal [ "build" , "--project-file=cabal.project-stage0" @@ -130,7 +130,7 @@ buildGhcStage1 opts cabal ghc0 = do putStrLn "Logs can be found in \"_build/stage0/cabal.{stdout,stderr}\"" exitFailure - msg "Copying stage0 programs and generating settings to use them..." + msg " - Copying stage1 programs and generating settings to use them..." let listbin_cmd p = runCabal cabal [ "list-bin" , "--project-file=cabal.project-stage0" @@ -285,40 +285,6 @@ prepareGhcSources opts dst = do subst_in (dst "libraries/base/base.cabal") common_substs subst_in (dst "libraries/rts/include/ghcversion.h") common_substs --- Avoid FilePath blindness by using type aliases for programs. -newtype Ghc = Ghc FilePath -newtype GhcPkg = GhcPkg FilePath -newtype Cabal = Cabal FilePath -newtype DeriveConstants = DeriveConstants FilePath -newtype GenApply = GenApply FilePath -newtype GenPrimop = GenPrimop FilePath - -runGhc :: Ghc -> [String] -> CreateProcess -runGhc (Ghc f) = proc f - -ghcPath :: Ghc -> FilePath -ghcPath (Ghc x) = x - -runGhcPkg :: GhcPkg -> [String] -> CreateProcess -runGhcPkg (GhcPkg f) = proc f - -ghcPkgPath :: GhcPkg -> FilePath -ghcPkgPath (GhcPkg x) = x - -runCabal :: Cabal -> [String] -> CreateProcess -runCabal (Cabal f) = proc f - -runDeriveConstants :: DeriveConstants -> [String] -> CreateProcess -runDeriveConstants (DeriveConstants f) = proc f - -runGenApply :: GenApply -> [String] -> CreateProcess -runGenApply (GenApply f) = proc f - -runGenPrimop :: GenPrimop -> [String] -> CreateProcess -runGenPrimop (GenPrimop f) = proc f - -cp :: String -> String -> IO () -cp src dst = void (readCreateProcess (shell $ "cp -rf " ++ src ++ " " ++ dst) "") -- | Generate settings for stage1 compiler, based on given settings (stage0's -- compiler settings) @@ -398,7 +364,7 @@ makeStage1Settings in_settings = out_settings , keep_def "target RTS linker only supports shared libraries" "NO" , ("Use interpreter", "NO") - , ("base unit-id", "base") + , ("base unit-id", "base") -- FIXME , keep_fail "Support SMP" , keep_fail "RTS ways" , keep_fail "Tables next to code" @@ -414,17 +380,14 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts = d let dst = "_build/stage1/" src <- makeAbsolute "_build/stage1/src" - msg "Preparing GHC sources to build GHC stage2..." + msg " - Preparing sources to build with GHC stage1..." prepareGhcSources opts src -- Build the RTS - msg "Building the RTS..." src_rts <- makeAbsolute (src "libraries/rts") build_dir <- makeAbsolute (dst "cabal") ghcversionh <- makeAbsolute (src_rts "include/ghcversion.h") - msg " - Generating RTS headers..." - let build_rts_cmd = runCabal cabal [ "build" , "--project-file=cabal.project-stage1-rts" -- TODO: replace with command-line args @@ -465,6 +428,8 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts = d -- 2. use deriveConstants to generate the other files -- 3. rerun cabal to build the rts + msg " - Generating headers and sources..." + -- first run is expected to fail because of misssing headers void $ readCreateProcessWithExitCode build_rts_cmd "" ghcplatform_dir <- takeDirectory <$> readCreateProcess (shell ("find " ++ build_dir ++ " -name ghcplatform.h")) "" @@ -525,6 +490,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts = d cp (dst_libffi "lib" "libffi.a") (takeDirectory ghcplatform_dir "libCffi.a") -- second run of cabal is expected to succeed now that have generated all the headers! + msg " - Building the RTS..." (rts_exit_code, rts_stdout, rts_stderr) <- readCreateProcessWithExitCode build_rts_cmd "" case rts_exit_code of ExitSuccess -> pure () @@ -545,7 +511,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts = d , "--builddir=" ++ build_dir ] - msg "Building boot libraries..." + msg " - Building boot libraries..." (boot_exit_code, boot_stdout, boot_stderr) <- readCreateProcessWithExitCode build_boot_cmd "" case boot_exit_code of ExitSuccess -> pure () @@ -557,8 +523,49 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts = d +--------------------------- +-- Utilities +--------------------------- + +-- | Display a message to the user with some timestamp msg :: String -> IO () msg x = do t <- getCPUTime let d = t `div` 1_000_000_000 - putStrLn ("[" ++ show d ++ "] " ++ x) + let stp = "[" ++ show d ++ "]" + putStrLn (stp ++ replicate (6 - length stp) ' ' ++ x) + +-- Avoid FilePath blindness by using type aliases for programs. +newtype Ghc = Ghc FilePath +newtype GhcPkg = GhcPkg FilePath +newtype Cabal = Cabal FilePath +newtype DeriveConstants = DeriveConstants FilePath +newtype GenApply = GenApply FilePath +newtype GenPrimop = GenPrimop FilePath + +runGhc :: Ghc -> [String] -> CreateProcess +runGhc (Ghc f) = proc f + +ghcPath :: Ghc -> FilePath +ghcPath (Ghc x) = x + +runGhcPkg :: GhcPkg -> [String] -> CreateProcess +runGhcPkg (GhcPkg f) = proc f + +ghcPkgPath :: GhcPkg -> FilePath +ghcPkgPath (GhcPkg x) = x + +runCabal :: Cabal -> [String] -> CreateProcess +runCabal (Cabal f) = proc f + +runDeriveConstants :: DeriveConstants -> [String] -> CreateProcess +runDeriveConstants (DeriveConstants f) = proc f + +runGenApply :: GenApply -> [String] -> CreateProcess +runGenApply (GenApply f) = proc f + +runGenPrimop :: GenPrimop -> [String] -> CreateProcess +runGenPrimop (GenPrimop f) = proc f + +cp :: String -> String -> IO () +cp src dst = void (readCreateProcess (shell $ "cp -rf " ++ src ++ " " ++ dst) "") From 7fd82b116545dd2421e38c99e70027cf52859442 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 14 Jan 2025 16:59:10 +0100 Subject: [PATCH 036/175] Fix Makefile to use Build.hs --- Build.hs | 57 ++++++----- GenSettings.hs | 124 ---------------------- Makefile | 273 +------------------------------------------------ 3 files changed, 32 insertions(+), 422 deletions(-) delete mode 100644 GenSettings.hs diff --git a/Build.hs b/Build.hs index aef759cbf75e..9cc2e559ef00 100755 --- a/Build.hs +++ b/Build.hs @@ -77,7 +77,6 @@ main = do -- | Build stage1 GHC program buildGhcStage1 :: GhcBuildOptions -> Cabal -> Ghc -> IO () buildGhcStage1 opts cabal ghc0 = do - msg " - Preparing sources to build with GHC stage0..." prepareGhcSources opts "_build/stage0/src/" let builddir = "_build/stage0/cabal/" @@ -184,35 +183,10 @@ buildGhcStage1 opts cabal ghc0 = do exitFailure -data GhcBuildOptions = GhcBuildOptions - { gboVersion :: !Text -- ^ GHC version - , gboVersionInt :: !Text -- ^ GHC version as an Int - , gboVersionMunged :: !Text -- ^ GHC version "munged" - , gboVersionForLib :: !Text -- ^ GHC version for libraries? - , gboVersionPatchLevel :: !Text -- ^ GHC patchlevel version - , gboVersionPatchLevel1 :: !Text -- ^ GHC patchlevel1 version - , gboVersionPatchLevel2 :: !Text -- ^ GHC patchlevel2 version - , gboLlvmMinVersion :: !Text -- ^ Min LLVM version supported - , gboLlvmMaxVersion :: !Text -- ^ Max LLVM version supported - } - -defaultGhcBuildOptions :: GhcBuildOptions -defaultGhcBuildOptions = GhcBuildOptions - { gboVersion = "9.13" - , gboVersionInt = "913" - , gboVersionMunged = "9.13" - , gboVersionForLib = "9.13" - , gboVersionPatchLevel = "0" - , gboVersionPatchLevel1 = "0" - , gboVersionPatchLevel2 = "0" - , gboLlvmMinVersion = "13" - , gboLlvmMaxVersion = "20" - } - - -- | Prepare GHC sources in the given directory prepareGhcSources :: GhcBuildOptions -> FilePath -> IO () prepareGhcSources opts dst = do + msg $ " - Preparing sources in " ++ dst ++ "..." createDirectoryIfMissing True dst cp "./libraries" dst @@ -521,6 +495,35 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts = d putStrLn boot_stderr exitFailure +--------------------------- +-- Options +--------------------------- + +data GhcBuildOptions = GhcBuildOptions + { gboVersion :: !Text -- ^ GHC version + , gboVersionInt :: !Text -- ^ GHC version as an Int + , gboVersionMunged :: !Text -- ^ GHC version "munged" + , gboVersionForLib :: !Text -- ^ GHC version for libraries? + , gboVersionPatchLevel :: !Text -- ^ GHC patchlevel version + , gboVersionPatchLevel1 :: !Text -- ^ GHC patchlevel1 version + , gboVersionPatchLevel2 :: !Text -- ^ GHC patchlevel2 version + , gboLlvmMinVersion :: !Text -- ^ Min LLVM version supported + , gboLlvmMaxVersion :: !Text -- ^ Max LLVM version supported + } + +defaultGhcBuildOptions :: GhcBuildOptions +defaultGhcBuildOptions = GhcBuildOptions + { gboVersion = "9.13" + , gboVersionInt = "913" + , gboVersionMunged = "9.13" + , gboVersionForLib = "9.13" + , gboVersionPatchLevel = "0" + , gboVersionPatchLevel1 = "0" + , gboVersionPatchLevel2 = "0" + , gboLlvmMinVersion = "13" + , gboLlvmMaxVersion = "20" + } + --------------------------- diff --git a/GenSettings.hs b/GenSettings.hs deleted file mode 100644 index 8d9fffc22521..000000000000 --- a/GenSettings.hs +++ /dev/null @@ -1,124 +0,0 @@ -module Main where - -import Data.Maybe -import System.Environment - -main :: IO () -main = do - args <- getArgs - case args of - ["ghc-boot"] -> interact ghcboot_settings - ["stage1"] -> interact stage1_settings - --- | Generate HADRIAN_SETTINGS for ghc-boot's Setup.hs, based on given settings -ghcboot_settings :: String -> String -ghcboot_settings input = output - where - output = show out_settings - - in_settings,out_settings :: [(String,String)] - in_settings = read input - - out_settings = - [ ("hostPlatformArch", fromMaybe (error "Couldn't read 'target arch' setting") (lookup "target arch" in_settings)) - , ("hostPlatformOS", fromMaybe (error "Couldn't read 'target os' setting") (lookup "target os" in_settings)) - , ("cProjectGitCommitId", "DEADBEEF") -- FIXME - , ("cProjectVersion", "9.13") - , ("cProjectVersionInt", "913") - , ("cProjectPatchLevel", "0") - , ("cProjectPatchLevel1", "0") - , ("cProjectPatchLevel2", "0") - ] - --- | Generate settings for stage1 compiler, based on given settings (stage0's --- compiler settings) -stage1_settings :: String -> String -stage1_settings input = output - where - output = show out_settings - - in_settings,out_settings :: [(String,String)] - in_settings = read input - - -- keep the previous setting, fail if it doesn't exist - keep_fail s = keep_def s (error ("Couldn't find setting "<> show s)) - - -- keep the previous setting, default to the given value if it doesn't exist - keep_def s d = case lookup s in_settings of - Nothing -> (s,d) - Just v -> (s,v) - - -- use the previous setting, or if it doesn't exist use the setting for the - -- second key. Fail if both don't exist. This is useful to support - -- bootstrapping with old compilers that mingled some settings. - keep_or_fail s s2 = case lookup s in_settings of - Nothing -> case lookup s2 in_settings of - Nothing -> error ("Couldn't find any of " <> show s <> " and " <> show s2) - Just v -> (s,v) - Just v -> (s,v) - - --FIXME: we default to these flags for Cmm CPP, otherwise CPP fails - -- with error: missing '(' after "__has_feature" - -- because we pass `-traditional` while compiling Apply.cmm (in TSANUtils.h) - default_cpp_flags = "-E" - - out_settings = - [ keep_fail "C compiler command" - , keep_fail "C compiler flags" - , keep_fail "C++ compiler command" - , keep_fail "C++ compiler flags" - , keep_fail "C compiler link flags" - , keep_fail "C compiler supports -no-pie" - , keep_or_fail "CPP command" "Haskell CPP command" - , keep_def "CPP flags" default_cpp_flags - , keep_fail "Haskell CPP command" - , keep_fail "Haskell CPP flags" - , keep_or_fail "JavaScript CPP command" "Haskell CPP command" - , keep_or_fail "JavaScript CPP flags" "Haskell CPP flags" - , keep_or_fail "C-- CPP command" "Haskell CPP command" - , keep_def "C-- CPP flags" default_cpp_flags - , keep_def "C-- CPP supports -g0" "NO" - , keep_fail "ld supports compact unwind" - , keep_fail "ld supports filelist" - , keep_fail "ld supports single module" - , keep_fail "ld is GNU ld" - , keep_fail "Merge objects command" - , keep_fail "Merge objects flags" - , keep_def "Merge objects supports response files" "NO" - , keep_fail "ar command" - , keep_fail "ar flags" - , keep_fail "ar supports at file" - , keep_fail "ar supports -L" - , keep_fail "ranlib command" - , keep_fail "otool command" - , keep_fail "install_name_tool command" - , keep_fail "windres command" - , keep_fail "unlit command" - , keep_fail "cross compiling" - , keep_fail "target platform string" - , keep_fail "target os" - , keep_fail "target arch" - , keep_fail "target word size" - , keep_fail "target word big endian" - , keep_fail "target has GNU nonexec stack" - , keep_fail "target has .ident directive" - , keep_fail "target has subsections via symbols" - , keep_fail "target has libm" - , keep_fail "Unregisterised" - , keep_fail "LLVM target" - , keep_fail "LLVM llc command" - , keep_fail "LLVM opt command" - , keep_def "LLVM llvm-as command" "llvm-as" - , keep_fail "Use inplace MinGW toolchain" - - , keep_def "target RTS linker only supports shared libraries" "NO" - , ("Use interpreter", "NO") - , ("base unit-id", "base") - , keep_fail "Support SMP" - , keep_fail "RTS ways" - , keep_fail "Tables next to code" - , keep_fail "Leading underscore" - , keep_fail "Use LibFFI" - , keep_fail "RTS expects libdw" - , ("Relative Global Package DB", "../../stage1/pkgs") - ] diff --git a/Makefile b/Makefile index 7138756a0b25..fe6b75ba35a3 100644 --- a/Makefile +++ b/Makefile @@ -1,274 +1,5 @@ -HADRIAN_SETTINGS_STAGE0 := $(shell ghc --info | runghc GenSettings.hs ghc-boot) -HADRIAN_SETTINGS_STAGE1 := $(shell ghc --info | runghc GenSettings.hs ghc-boot) -SETTINGS_STAGE1 := $(shell ghc --info | runghc GenSettings.hs stage1) - -CABAL := /home/hsyl20/projects/cabal/dist-newstyle/build/x86_64-linux/ghc-9.10.1/cabal-install-3.15.0.0/x/cabal/build/cabal/cabal -# CABAL := cabal - -all: _build/stage1/bin/ghc - -_build/stage0/bin/ghc: - # Preparing source files... - mkdir -p _build/stage0/src/ - cp -rf ./libraries _build/stage0/src/ - cp -rf ./compiler _build/stage0/src/libraries/ghc - cp -rf ./ghc _build/stage0/src/ghc-bin - cp -rf ./utils _build/stage0/src/ - - cp -f rts/include/rts/Bytecodes.h _build/stage0/src/libraries/ghc/ - cp -f rts/include/rts/storage/ClosureTypes.h _build/stage0/src/libraries/ghc/ - cp -f rts/include/rts/storage/FunTypes.h _build/stage0/src/libraries/ghc/ - cp -f rts/include/stg/MachRegs.h _build/stage0/src/libraries/ghc/ - mkdir -p _build/stage0/src/libraries/ghc/MachRegs - cp -f rts/include/stg/MachRegs/*.h _build/stage0/src/libraries/ghc/MachRegs/ - - ## Substituting variables - cp _build/stage0/src/ghc-bin/ghc-bin.cabal{.in,} - cp _build/stage0/src/libraries/ghc/ghc.cabal{.in,} - cp _build/stage0/src/libraries/ghc/GHC/CmmToLlvm/Version/Bounds.hs{.in,} - cp _build/stage0/src/libraries/ghc-boot/ghc-boot.cabal{.in,} - cp _build/stage0/src/libraries/ghc-boot-th/ghc-boot-th.cabal{.in,} - cp _build/stage0/src/libraries/ghc-heap/ghc-heap.cabal{.in,} - cp _build/stage0/src/libraries/ghci/ghci.cabal{.in,} - cp _build/stage0/src/utils/ghc-pkg/ghc-pkg.cabal{.in,} - - sed -i 's/@ProjectVersion@/9.13/' _build/stage0/src/ghc-bin/ghc-bin.cabal - sed -i 's/@ProjectVersionMunged@/9.13/' _build/stage0/src/ghc-bin/ghc-bin.cabal - sed -i 's/@ProjectVersion@/9.13/' _build/stage0/src/libraries/ghc/ghc.cabal - sed -i 's/@ProjectVersionMunged@/9.13/' _build/stage0/src/libraries/ghc/ghc.cabal - sed -i 's/@ProjectVersion@/9.13/' _build/stage0/src/libraries/ghc-boot/ghc-boot.cabal - sed -i 's/@ProjectVersionMunged@/9.13/' _build/stage0/src/libraries/ghc-boot/ghc-boot.cabal - sed -i 's/@ProjectVersion@/9.13/' _build/stage0/src/libraries/ghc-boot-th/ghc-boot-th.cabal - sed -i 's/@ProjectVersionMunged@/9.13/' _build/stage0/src/libraries/ghc-boot-th/ghc-boot-th.cabal - sed -i 's/@Suffix@//' _build/stage0/src/libraries/ghc-boot-th/ghc-boot-th.cabal - sed -i 's/@SourceRoot@/./' _build/stage0/src/libraries/ghc-boot-th/ghc-boot-th.cabal - sed -i 's/@ProjectVersion@/9.13/' _build/stage0/src/libraries/ghc-heap/ghc-heap.cabal - sed -i 's/@ProjectVersionMunged@/9.13/' _build/stage0/src/libraries/ghc-heap/ghc-heap.cabal - sed -i 's/@ProjectVersionForLib@/9.13/' _build/stage0/src/libraries/ghc-heap/ghc-heap.cabal - sed -i 's/@ProjectVersion@/9.13/' _build/stage0/src/libraries/ghci/ghci.cabal - sed -i 's/@ProjectVersionMunged@/9.13/' _build/stage0/src/libraries/ghci/ghci.cabal - sed -i 's/@ProjectVersionForLib@/9.13/' _build/stage0/src/libraries/ghci/ghci.cabal - sed -i 's/@ProjectVersion@/9.13/' _build/stage0/src/utils/ghc-pkg/ghc-pkg.cabal - sed -i 's/@ProjectVersionMunged@/9.13/' _build/stage0/src/utils/ghc-pkg/ghc-pkg.cabal - sed -i 's/@ProjectVersionForLib@/9.13/' _build/stage0/src/utils/ghc-pkg/ghc-pkg.cabal - - sed -i 's/@LlvmMinVersion@/13/' _build/stage0/src/libraries/ghc/GHC/CmmToLlvm/Version/Bounds.hs - sed -i 's/@LlvmMaxVersion@/20/' _build/stage0/src/libraries/ghc/GHC/CmmToLlvm/Version/Bounds.hs - - # Building... - mkdir -p _build/stage0/cabal/ - mkdir -p _build/stage0/bin/ - - HADRIAN_SETTINGS='$(HADRIAN_SETTINGS_STAGE0)' \ - $(CABAL) build --project-file=cabal.project-stage0 \ - ghc-bin:ghc ghc-pkg:ghc-pkg genprimopcode:genprimopcode deriveConstants:deriveConstants genapply:genapply \ - -j --builddir=_build/stage0/cabal/ - - # Installing binaries - cp `$(CABAL) list-bin --project-file=cabal.project-stage0 --builddir=_build/stage0/cabal/ ghc-bin:ghc` _build/stage0/bin/ghc - cp `$(CABAL) list-bin --project-file=cabal.project-stage0 --builddir=_build/stage0/cabal/ ghc-pkg:ghc-pkg` _build/stage0/bin/ghc-pkg - cp `$(CABAL) list-bin --project-file=cabal.project-stage0 --builddir=_build/stage0/cabal/ deriveConstants:deriveConstants` _build/stage0/bin/deriveConstants - cp `$(CABAL) list-bin --project-file=cabal.project-stage0 --builddir=_build/stage0/cabal/ genprimopcode:genprimopcode` _build/stage0/bin/genprimopcode - cp `$(CABAL) list-bin --project-file=cabal.project-stage0 --builddir=_build/stage0/cabal/ genapply:genapply` _build/stage0/bin/genapply - - # Generate settings - mkdir -p _build/stage0/lib - echo '$(SETTINGS_STAGE1)' > _build/stage0/lib/settings - - _build/stage0/bin/ghc --version - _build/stage0/bin/ghc --info - - -_build/stage1/bin/ghc: _build/stage0/bin/ghc - rm -rf _build/stage1 - mkdir -p _build/stage1 - $(CABAL) --version - - # Initialize empty package db - _build/stage0/bin/ghc-pkg init _build/stage1/pkgs - _build/stage0/bin/ghc-pkg recache --global-package-db=_build/stage1/pkgs --no-user-package-db - - # Preparing source files... - mkdir -p _build/stage1/src/ - cp -rf ./libraries _build/stage1/src/ - cp -rf ./compiler _build/stage1/src/libraries/ghc - cp -rf ./rts _build/stage1/src/libraries/ - cp -rf ./ghc _build/stage1/src/ghc-bin - cp -rf ./config.sub _build/stage1/src/libraries/rts/ - cp -rf ./config.guess _build/stage1/src/libraries/rts/ - - cp -f rts/include/rts/Bytecodes.h _build/stage1/src/libraries/ghc/ - cp -f rts/include/rts/storage/ClosureTypes.h _build/stage1/src/libraries/ghc/ - cp -f rts/include/rts/storage/FunTypes.h _build/stage1/src/libraries/ghc/ - cp -f rts/include/stg/MachRegs.h _build/stage1/src/libraries/ghc/ - mkdir -p _build/stage1/src/libraries/ghc/MachRegs - cp -f rts/include/stg/MachRegs/*.h _build/stage1/src/libraries/ghc/MachRegs/ - - cp -f utils/fs/fs.h _build/stage1/src/libraries/ghc-internal/include - cp -f utils/fs/fs.c _build/stage1/src/libraries/ghc-internal/cbits - cp -f utils/fs/fs.* _build/stage1/src/libraries/rts/ - - python rts/gen_event_types.py --event-types-defines _build/stage1/src/libraries/rts/include/rts/EventLogConstants.h - python rts/gen_event_types.py --event-types-array _build/stage1/src/libraries/rts/include/rts/EventTypes.h - - ## Substituting variables - cp _build/stage1/src/ghc-bin/ghc-bin.cabal{.in,} - cp _build/stage1/src/libraries/ghc/ghc.cabal{.in,} - cp _build/stage1/src/libraries/ghc/GHC/CmmToLlvm/Version/Bounds.hs{.in,} - cp _build/stage1/src/libraries/ghc-internal/ghc-internal.cabal{.in,} - cp _build/stage1/src/libraries/ghc-boot/ghc-boot.cabal{.in,} - cp _build/stage1/src/libraries/ghc-boot-th/ghc-boot-th.cabal{.in,} - cp _build/stage1/src/libraries/ghc-heap/ghc-heap.cabal{.in,} - cp _build/stage1/src/libraries/ghci/ghci.cabal{.in,} - cp _build/stage1/src/libraries/base/base.cabal{.in,} - cp _build/stage1/src/libraries/rts/include/ghcversion.h{.in,} - - sed -i 's/@ProjectVersion@/9.13/' _build/stage1/src/ghc-bin/ghc-bin.cabal - sed -i 's/@ProjectVersionMunged@/9.13/' _build/stage1/src/ghc-bin/ghc-bin.cabal - sed -i 's/@ProjectVersion@/9.13/' _build/stage1/src/libraries/ghc/ghc.cabal - sed -i 's/@ProjectVersionMunged@/9.13/' _build/stage1/src/libraries/ghc/ghc.cabal - sed -i 's/@ProjectVersion@/9.13/' _build/stage1/src/libraries/ghc-boot/ghc-boot.cabal - sed -i 's/@ProjectVersionMunged@/9.13/' _build/stage1/src/libraries/ghc-boot/ghc-boot.cabal - sed -i 's/@ProjectVersion@/9.13/' _build/stage1/src/libraries/ghc-boot-th/ghc-boot-th.cabal - sed -i 's/@ProjectVersionMunged@/9.13/' _build/stage1/src/libraries/ghc-boot-th/ghc-boot-th.cabal - sed -i 's/@Suffix@//' _build/stage1/src/libraries/ghc-boot-th/ghc-boot-th.cabal - sed -i 's/@SourceRoot@/./' _build/stage1/src/libraries/ghc-boot-th/ghc-boot-th.cabal - sed -i 's/@ProjectVersion@/9.13/' _build/stage1/src/libraries/ghc-heap/ghc-heap.cabal - sed -i 's/@ProjectVersionMunged@/9.13/' _build/stage1/src/libraries/ghc-heap/ghc-heap.cabal - sed -i 's/@ProjectVersionForLib@/9.13/' _build/stage1/src/libraries/ghc-heap/ghc-heap.cabal - sed -i 's/@ProjectVersion@/9.13/' _build/stage1/src/libraries/ghci/ghci.cabal - sed -i 's/@ProjectVersionMunged@/9.13/' _build/stage1/src/libraries/ghci/ghci.cabal - sed -i 's/@ProjectVersionForLib@/9.13/' _build/stage1/src/libraries/ghci/ghci.cabal - sed -i 's/@ProjectVersionForLib@/9.13/' _build/stage1/src/libraries/ghc-internal/ghc-internal.cabal - sed -i 's/@ProjectVersion@/9.13/' _build/stage1/src/libraries/rts/include/ghcversion.h - sed -i 's/@ProjectVersionInt@/913/' _build/stage1/src/libraries/rts/include/ghcversion.h - sed -i 's/@ProjectPatchLevel1@/0/' _build/stage1/src/libraries/rts/include/ghcversion.h - sed -i 's/@ProjectPatchLevel2@/0/' _build/stage1/src/libraries/rts/include/ghcversion.h - sed -i 's/@ProjectVersionForLib@/9.13/' _build/stage1/src/libraries/base/base.cabal - - sed -i 's/@LlvmMinVersion@/13/' _build/stage1/src/libraries/ghc/GHC/CmmToLlvm/Version/Bounds.hs - sed -i 's/@LlvmMaxVersion@/20/' _build/stage1/src/libraries/ghc/GHC/CmmToLlvm/Version/Bounds.hs - - # Generating headers - # FIXME: deriveConstants requires ghcautoconf.h and ghcplatform.h - # Let's run cabal until it fails so that these files are generated... - HADRIAN_SETTINGS='$(HADRIAN_SETTINGS_STAGE1)' \ - $(CABAL) build --project-file=cabal.project-stage1-rts \ - rts \ - --with-compiler=`pwd`/_build/stage0/bin/ghc \ - --with-hc-pkg=`pwd`/_build/stage0/bin/ghc-pkg \ - --ghc-options="-ghcversion-file=`pwd`/_build/stage1/src/libraries/rts/include/ghcversion.h" \ - --ghc-options="-I`pwd`/_build/stage1/src/libraries/rts/include/" \ - --ghc-options="-I`pwd`/_build/stage1/src/libraries/rts/" \ - --ghc-options='"-optc=-DProjectVersion=\"913\""' \ - --ghc-options='"-optc=-DRtsWay=\"FIXME\""' \ - --ghc-options='"-optc=-DHostPlatform=\"FIXME\""' \ - --ghc-options='"-optc=-DHostArch=\"FIXME\""' \ - --ghc-options='"-optc=-DHostOS=\"FIXME\""' \ - --ghc-options='"-optc=-DHostVendor=\"FIXME\""' \ - --ghc-options='"-optc=-DBuildPlatform=\"FIXME\""' \ - --ghc-options='"-optc=-DBuildArch=\"FIXME\""' \ - --ghc-options='"-optc=-DBuildOS=\"FIXME\""' \ - --ghc-options='"-optc=-DBuildVendor=\"FIXME\""' \ - --ghc-options='"-optc=-DTargetPlatform=\"FIXME\""' \ - --ghc-options='"-optc=-DTargetArch=\"FIXME\""' \ - --ghc-options='"-optc=-DTargetOS=\"FIXME\""' \ - --ghc-options='"-optc=-DTargetVendor=\"FIXME\""' \ - --ghc-options='"-optc=-DGhcUnregisterised=\"FIXME\""' \ - --ghc-options='"-optc=-DTablesNextToCode=\"FIXME\""' \ - --builddir=_build/stage1/cabal/ >/dev/null 2>&1 || true - - # Deriving constants - mkdir -p _build/stage1/temp/derive_constants - _build/stage0/bin/deriveConstants --gen-header -o _build/stage1/src/libraries/rts/include/DerivedConstants.h \ - --target-os linux \ - --tmpdir _build/stage1/temp/derive_constants \ - --gcc-program gcc \ - --nm-program nm \ - --objdump-program objdump \ - --gcc-flag "-I_build/stage1/src/libraries/rts/include" \ - --gcc-flag "-I_build/stage1/src/libraries/rts" \ - --gcc-flag "-I_build/stage1/cabal/build/x86_64-linux/ghc-9.13/rts-1.0.3/build/include" - - # Generate autoapply - _build/stage0/bin/genapply _build/stage1/src/libraries/rts/include/DerivedConstants.h > _build/stage1/src/libraries/rts/AutoApply.cmm - _build/stage0/bin/genapply _build/stage1/src/libraries/rts/include/DerivedConstants.h -V16 > _build/stage1/src/libraries/rts/AutoApply_V16.cmm - _build/stage0/bin/genapply _build/stage1/src/libraries/rts/include/DerivedConstants.h -V32 > _build/stage1/src/libraries/rts/AutoApply_V32.cmm - _build/stage0/bin/genapply _build/stage1/src/libraries/rts/include/DerivedConstants.h -V64 > _build/stage1/src/libraries/rts/AutoApply_V64.cmm - - # Build libffi - mkdir -p _build/stage1/src/libffi - mkdir -p _build/stage1/libffi - (cd _build/stage1/src/libffi; tar -xvf ../../../../libffi-tarballs/libffi-3.4.6.tar.gz) - (cd _build/stage1/src/libffi/libffi-3.4.6; ./configure --disable-docs --with-pics=yes --disable-multi-os-directory --prefix=`pwd`/../../../../../_build/stage1/libffi/ && make install -j) - cp -f _build/stage1/libffi/include/* _build/stage1/src/libraries/rts/include/ - cp -f _build/stage1/libffi/lib/libffi.a _build/stage1/cabal/build/x86_64-linux/ghc-9.13/rts-1.0.3/build/libCffi.a - - # Building boot libraries - mkdir -p _build/stage1/cabal/ - - # we need to pass "-this-unit-id=rts", otherwise GHC tries to lookup the - # platform constants in the package db and fails. The flag is already - # set in rts.cabal but for some reason it isn't always passed :shrug: - HADRIAN_SETTINGS='$(HADRIAN_SETTINGS_STAGE1)' \ - $(CABAL) build --project-file=cabal.project-stage1-rts \ - rts \ - --with-compiler=`pwd`/_build/stage0/bin/ghc \ - --with-hc-pkg=`pwd`/_build/stage0/bin/ghc-pkg \ - --ghc-options="-ghcversion-file=`pwd`/_build/stage1/src/libraries/rts/include/ghcversion.h" \ - --ghc-options="-I`pwd`/_build/stage1/src/libraries/rts/include/" \ - --ghc-options="-I`pwd`/_build/stage1/src/libraries/rts/" \ - --ghc-options='"-optc=-DProjectVersion=\"913\""' \ - --ghc-options='"-optc=-DRtsWay=\"FIXME\""' \ - --ghc-options='"-optc=-DHostPlatform=\"FIXME\""' \ - --ghc-options='"-optc=-DHostArch=\"FIXME\""' \ - --ghc-options='"-optc=-DHostOS=\"FIXME\""' \ - --ghc-options='"-optc=-DHostVendor=\"FIXME\""' \ - --ghc-options='"-optc=-DBuildPlatform=\"FIXME\""' \ - --ghc-options='"-optc=-DBuildArch=\"FIXME\""' \ - --ghc-options='"-optc=-DBuildOS=\"FIXME\""' \ - --ghc-options='"-optc=-DBuildVendor=\"FIXME\""' \ - --ghc-options='"-optc=-DTargetPlatform=\"FIXME\""' \ - --ghc-options='"-optc=-DTargetArch=\"FIXME\""' \ - --ghc-options='"-optc=-DTargetOS=\"FIXME\""' \ - --ghc-options='"-optc=-DTargetVendor=\"FIXME\""' \ - --ghc-options='"-optc=-DGhcUnregisterised=\"FIXME\""' \ - --ghc-options='"-optc=-DTablesNextToCode=\"FIXME\""' \ - --ghc-options='-this-unit-id=rts' \ - --builddir=_build/stage1/cabal/ - - # generate files related to primops - - gcc -E -undef -traditional -P -x c _build/stage1/src/libraries/ghc/GHC/Builtin/primops.txt.pp > _build/stage1/src/libraries/ghc/GHC/Builtin/primops.txt - _build/stage0/bin/genprimopcode --make-haskell-source < _build/stage1/src/libraries/ghc/GHC/Builtin/primops.txt > _build/stage1/src/libraries/ghc-prim/GHC/Prim.hs - _build/stage0/bin/genprimopcode --make-haskell-wrappers < _build/stage1/src/libraries/ghc/GHC/Builtin/primops.txt > _build/stage1/src/libraries/ghc-prim/GHC/PrimopWrappers.hs - - - HADRIAN_SETTINGS='$(HADRIAN_SETTINGS_STAGE1)' \ - $(CABAL) build --project-file=cabal.project-stage1 \ - rts ghc-prim ghc-internal base \ - --with-compiler=`pwd`/_build/stage0/bin/ghc \ - --with-hc-pkg=`pwd`/_build/stage0/bin/ghc-pkg \ - --ghc-options="-ghcversion-file=`pwd`/_build/stage1/src/libraries/rts/include/ghcversion.h" \ - --ghc-options="-I`pwd`/_build/stage1/src/libraries/rts/include/" \ - --ghc-options="-I`pwd`/_build/stage1/src/libraries/rts/" \ - --ghc-options='"-optc=-DProjectVersion=\"913\""' \ - --ghc-options='"-optc=-DRtsWay=\"FIXME\""' \ - --ghc-options='"-optc=-DHostPlatform=\"FIXME\""' \ - --ghc-options='"-optc=-DHostArch=\"FIXME\""' \ - --ghc-options='"-optc=-DHostOS=\"FIXME\""' \ - --ghc-options='"-optc=-DHostVendor=\"FIXME\""' \ - --ghc-options='"-optc=-DBuildPlatform=\"FIXME\""' \ - --ghc-options='"-optc=-DBuildArch=\"FIXME\""' \ - --ghc-options='"-optc=-DBuildOS=\"FIXME\""' \ - --ghc-options='"-optc=-DBuildVendor=\"FIXME\""' \ - --ghc-options='"-optc=-DTargetPlatform=\"FIXME\""' \ - --ghc-options='"-optc=-DTargetArch=\"FIXME\""' \ - --ghc-options='"-optc=-DTargetOS=\"FIXME\""' \ - --ghc-options='"-optc=-DTargetVendor=\"FIXME\""' \ - --ghc-options='"-optc=-DGhcUnregisterised=\"FIXME\""' \ - --ghc-options='"-optc=-DTablesNextToCode=\"FIXME\""' \ - --builddir=_build/stage1/cabal/ +all: + ./Build.hs clean: rm -rf _build From d49d25decc31c3a66409a760baf5a305043ee84e Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 14 Jan 2025 17:44:56 +0100 Subject: [PATCH 037/175] Fix Setup scripts --- Build.hs | 16 +++++------ cabal.project-stage1 | 42 +++++++++++++++++++++++++++- compiler/Setup.hs | 19 ++++++++++--- compiler/ghc.cabal.in | 2 +- libraries/ghc-boot/Setup.hs | 17 ++++++++--- libraries/ghc-boot/ghc-boot.cabal.in | 2 +- 6 files changed, 79 insertions(+), 19 deletions(-) diff --git a/Build.hs b/Build.hs index 9cc2e559ef00..defe12a0e6f6 100755 --- a/Build.hs +++ b/Build.hs @@ -188,12 +188,13 @@ prepareGhcSources :: GhcBuildOptions -> FilePath -> IO () prepareGhcSources opts dst = do msg $ " - Preparing sources in " ++ dst ++ "..." createDirectoryIfMissing True dst + createDirectoryIfMissing True (dst "libraries/ghc/MachRegs") - cp "./libraries" dst - cp "./compiler" (dst "libraries/ghc") - cp "./rts" (dst "libraries/") - cp "./ghc" (dst "ghc-bin") - cp "./utils" dst + cp "./libraries" dst + cp "./compiler/*" (dst "libraries/ghc/") + cp "./rts" (dst "libraries/") + cp "./ghc" (dst "ghc-bin") + cp "./utils" dst cp "./config.sub" (dst "libraries/rts/") cp "./config.guess" (dst "libraries/rts/") @@ -202,7 +203,6 @@ prepareGhcSources opts dst = do cp "rts/include/rts/storage/ClosureTypes.h" (dst "libraries/ghc/") cp "rts/include/rts/storage/FunTypes.h" (dst "libraries/ghc/") cp "rts/include/stg/MachRegs.h" (dst "libraries/ghc/") - createDirectoryIfMissing True (dst "libraries/ghc/MachRegs") cp "rts/include/stg/MachRegs/*.h" (dst "libraries/ghc/MachRegs/") cp "utils/fs/fs.h" (dst "libraries/ghc-internal/include") @@ -251,6 +251,7 @@ prepareGhcSources opts dst = do subst_in (dst "libraries/ghc-boot/ghc-boot.cabal") common_substs subst_in (dst "libraries/ghc-boot-th/ghc-boot-th.cabal") (common_substs ++ boot_th_substs) subst_in (dst "libraries/ghc-heap/ghc-heap.cabal") common_substs + subst_in (dst "libraries/template-haskell/template-haskell.cabal") common_substs subst_in (dst "libraries/ghci/ghci.cabal") common_substs subst_in (dst "libraries/ghc/GHC/CmmToLlvm/Version/Bounds.hs") llvm_substs subst_in (dst "utils/ghc-pkg/ghc-pkg.cabal") common_substs @@ -354,7 +355,6 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts = d let dst = "_build/stage1/" src <- makeAbsolute "_build/stage1/src" - msg " - Preparing sources to build with GHC stage1..." prepareGhcSources opts src -- Build the RTS @@ -516,7 +516,7 @@ defaultGhcBuildOptions = GhcBuildOptions { gboVersion = "9.13" , gboVersionInt = "913" , gboVersionMunged = "9.13" - , gboVersionForLib = "9.13" + , gboVersionForLib = "9.1300" , gboVersionPatchLevel = "0" , gboVersionPatchLevel1 = "0" , gboVersionPatchLevel2 = "0" diff --git a/cabal.project-stage1 b/cabal.project-stage1 index 1e76d2d406ad..4236b4d9917f 100644 --- a/cabal.project-stage1 +++ b/cabal.project-stage1 @@ -5,6 +5,42 @@ packages: ./_build/stage1/src/libraries/ghc-prim ./_build/stage1/src/libraries/ghc-internal ./_build/stage1/src/libraries/base + ./_build/stage1/src/libraries/ghc + ./_build/stage1/src/libraries/ghc-platform/ + ./_build/stage1/src/libraries/ghc-boot/ + ./_build/stage1/src/libraries/ghc-boot-th/ + ./_build/stage1/src/libraries/ghc-heap + ./_build/stage1/src/libraries/ghci + ./_build/stage1/src/libraries/stm + ./_build/stage1/src/libraries/template-haskell + ./_build/stage1/src/libraries/hpc + ./_build/stage1/src/ghc-bin/ + ./_build/stage1/src/utils/ghc-pkg + ./_build/stage1/src/utils/hsc2hs + ./_build/stage1/src/utils/unlit + + ./_build/stage1/src/libraries/array + ./_build/stage1/src/libraries/binary + ./_build/stage1/src/libraries/bytestring + ./_build/stage1/src/libraries/containers/containers + ./_build/stage1/src/libraries/deepseq + ./_build/stage1/src/libraries/directory/ + ./_build/stage1/src/libraries/exceptions + ./_build/stage1/src/libraries/file-io/ + ./_build/stage1/src/libraries/filepath/ + ./_build/stage1/src/libraries/mtl + ./_build/stage1/src/libraries/os-string/ + ./_build/stage1/src/libraries/parsec + ./_build/stage1/src/libraries/pretty/ + ./_build/stage1/src/libraries/process/ + ./_build/stage1/src/libraries/semaphore-compat + ./_build/stage1/src/libraries/text + ./_build/stage1/src/libraries/time + ./_build/stage1/src/libraries/transformers + ./_build/stage1/src/libraries/unix/ + ./_build/stage1/src/libraries/Win32/ + ./_build/stage1/src/libraries/Cabal/Cabal-syntax + ./_build/stage1/src/libraries/Cabal/Cabal benchmarks: False tests: False @@ -22,5 +58,9 @@ package * executable-static: False package ghc-internal - -- make our life easier for now by using the native bignum backend + -- FIXME: make our life easier for now by using the native bignum backend flags: +bignum-native + +package text + -- FIXME: avoid having to deal with system-cxx-std-lib fake package for now + flags: -simdutf diff --git a/compiler/Setup.hs b/compiler/Setup.hs index 42a58b5bb639..c51d27b3ff9b 100644 --- a/compiler/Setup.hs +++ b/compiler/Setup.hs @@ -1,4 +1,5 @@ {-# LANGUAGE NamedFieldPuns #-} +{-# LANGUAGE CPP #-} module Main where import Distribution.Simple @@ -11,6 +12,9 @@ import Distribution.Verbosity import Distribution.Simple.Program import Distribution.Simple.Utils import Distribution.Simple.Setup +#if MIN_VERSION_Cabal(3,14,0) +import Distribution.Simple.LocalBuildInfo (interpretSymbolicPathLBI) +#endif import System.IO import System.Process @@ -58,8 +62,15 @@ primopIncls = ghcAutogen :: Verbosity -> LocalBuildInfo -> IO () ghcAutogen verbosity lbi@LocalBuildInfo{pkgDescrFile,withPrograms,componentNameMap} = do + +#if MIN_VERSION_Cabal(3,14,0) + let fromSymPath = interpretSymbolicPathLBI lbi +#else + let fromSymPath = id +#endif + -- Get compiler/ root directory from the cabal file - let Just compilerRoot = takeDirectory <$> pkgDescrFile + let Just compilerRoot = (takeDirectory . fromSymPath) <$> pkgDescrFile -- Require the necessary programs (gcc ,withPrograms) <- requireProgram normal gccProgram withPrograms @@ -79,10 +90,10 @@ ghcAutogen verbosity lbi@LocalBuildInfo{pkgDescrFile,withPrograms,componentNameM -- Call genprimopcode to generate *.hs-incl forM_ primopIncls $ \(file,command) -> do contents <- readProcess "genprimopcode" [command] primopsStr - rewriteFileEx verbosity (buildDir lbi file) contents + rewriteFileEx verbosity (fromSymPath (buildDir lbi) file) contents -- Write GHC.Platform.Constants - let platformConstantsPath = autogenPackageModulesDir lbi "GHC/Platform/Constants.hs" + let platformConstantsPath = fromSymPath (autogenPackageModulesDir lbi) "GHC/Platform/Constants.hs" targetOS = case lookup "target os" settings of Nothing -> error "no target os in settings" Just os -> os @@ -97,7 +108,7 @@ ghcAutogen verbosity lbi@LocalBuildInfo{pkgDescrFile,withPrograms,componentNameM _ -> error "Couldn't find unique cabal library when building ghc" -- Write GHC.Settings.Config - configHsPath = autogenPackageModulesDir lbi "GHC/Settings/Config.hs" + configHsPath = fromSymPath (autogenPackageModulesDir lbi) "GHC/Settings/Config.hs" configHs = generateConfigHs cProjectUnitId settings createDirectoryIfMissingVerbose verbosity True (takeDirectory configHsPath) rewriteFileEx verbosity configHsPath configHs diff --git a/compiler/ghc.cabal.in b/compiler/ghc.cabal.in index 214f61cafc37..51a5dd0bbd7d 100644 --- a/compiler/ghc.cabal.in +++ b/compiler/ghc.cabal.in @@ -50,7 +50,7 @@ extra-source-files: custom-setup - setup-depends: base >= 3 && < 5, Cabal >= 1.6 && <3.14, directory, process, filepath, containers + setup-depends: base >= 3 && < 5, Cabal >= 1.6 && <3.16, directory, process, filepath, containers Flag internal-interpreter Description: Build with internal interpreter support. diff --git a/libraries/ghc-boot/Setup.hs b/libraries/ghc-boot/Setup.hs index 0995ee3f8ff6..715b7e553596 100644 --- a/libraries/ghc-boot/Setup.hs +++ b/libraries/ghc-boot/Setup.hs @@ -1,6 +1,6 @@ {-# LANGUAGE RecordWildCards #-} {-# LANGUAGE LambdaCase #-} -{-# LANGUAGE LambdaCase #-} +{-# LANGUAGE CPP #-} module Main where import Distribution.Simple @@ -10,6 +10,9 @@ import Distribution.Verbosity import Distribution.Simple.Program import Distribution.Simple.Utils import Distribution.Simple.Setup +#if MIN_VERSION_Cabal(3,14,0) +import Distribution.Simple.LocalBuildInfo (interpretSymbolicPathLBI) +#endif import System.IO import System.Directory @@ -31,13 +34,19 @@ main = defaultMainWithHooks ghcHooks ghcAutogen :: Verbosity -> LocalBuildInfo -> IO () ghcAutogen verbosity lbi@LocalBuildInfo{..} = do +#if MIN_VERSION_Cabal(3,14,0) + let fromSymPath = interpretSymbolicPathLBI lbi +#else + let fromSymPath = id +#endif + -- Get compiler/ root directory from the cabal file - let Just compilerRoot = takeDirectory <$> pkgDescrFile + let Just compilerRoot = (takeDirectory . fromSymPath) <$> pkgDescrFile let platformHostFile = "GHC/Platform/Host.hs" - platformHostPath = autogenPackageModulesDir lbi platformHostFile + platformHostPath = fromSymPath (autogenPackageModulesDir lbi) platformHostFile ghcVersionFile = "GHC/Version.hs" - ghcVersionPath = autogenPackageModulesDir lbi ghcVersionFile + ghcVersionPath = fromSymPath (autogenPackageModulesDir lbi) ghcVersionFile -- Get compiler settings settings <- lookupEnv "HADRIAN_SETTINGS" >>= \case diff --git a/libraries/ghc-boot/ghc-boot.cabal.in b/libraries/ghc-boot/ghc-boot.cabal.in index d61f6809fef4..992c7834739e 100644 --- a/libraries/ghc-boot/ghc-boot.cabal.in +++ b/libraries/ghc-boot/ghc-boot.cabal.in @@ -28,7 +28,7 @@ build-type: Custom extra-source-files: changelog.md custom-setup - setup-depends: base >= 3 && < 5, Cabal >= 1.6 && <3.14, directory, filepath + setup-depends: base >= 3 && < 5, Cabal >= 1.6 && <3.16, directory, filepath source-repository head type: git From 999b935eb18219bee4e15fa4b96d3de166c1583e Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 14 Jan 2025 18:07:00 +0100 Subject: [PATCH 038/175] Add Alex --- cabal.project-stage1 | 1 + 1 file changed, 1 insertion(+) diff --git a/cabal.project-stage1 b/cabal.project-stage1 index 4236b4d9917f..eb9295b7b6ea 100644 --- a/cabal.project-stage1 +++ b/cabal.project-stage1 @@ -41,6 +41,7 @@ packages: ./_build/stage1/src/libraries/Win32/ ./_build/stage1/src/libraries/Cabal/Cabal-syntax ./_build/stage1/src/libraries/Cabal/Cabal + https://github.com/haskell/alex/archive/refs/tags/v3.5.2.0.tar.gz benchmarks: False tests: False From e6e97c7470e6131bdb83429eca2e822349288e7d Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 14 Jan 2025 18:14:51 +0100 Subject: [PATCH 039/175] Use the Makefile in CI --- .github/workflows/ci.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2b448f8ee8bf..f872684db452 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,20 +32,10 @@ jobs: cabal-version: "latest" cabal-update: true - - name: Install Alex and Happy - run: | - cabal install alex - cabal install happy - - name: Configure the build run: | ./boot - ./configure - - - name: Build Hadrian - run: | - ./hadrian/build --version - name: Build the bindist run: | - ./hadrian/build --flavour=release -j binary-dist-dir --docs=none + make From 9da37f0f1f7c98fe44f6d43ad743194137c3093f Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 15 Jan 2025 14:29:13 +0100 Subject: [PATCH 040/175] Comments --- Build.hs | 20 ++++++++++++++++---- cabal.project-stage0 | 4 +++- libraries/ghc-prim/ghc-prim.cabal | 3 +++ 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/Build.hs b/Build.hs index defe12a0e6f6..4acf05323b2e 100755 --- a/Build.hs +++ b/Build.hs @@ -376,8 +376,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts = d -- platform constants in the package db and fails. The flag is already -- set in rts.cabal but for some reason it isn't always passed :shrug: , "--ghc-options=-this-unit-id=rts" - , "--ghc-options=-DProjectVersion=913" - , "--ghc-options=\"-optc=-DProjectVersion=\\\"913\\\"\"" + , "--ghc-options=\"-optc=-DProjectVersion=\\\"" ++ Text.unpack (gboVersionInt opts) ++ "\\\"\"" , "--ghc-options=\"-optc=-DRtsWay=\\\"FIXME\\\"\"" , "--ghc-options=\"-optc=-DHostPlatform=\\\"FIXME\\\"\"" , "--ghc-options=\"-optc=-DHostArch=\\\"FIXME\\\"\"" @@ -431,7 +430,11 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts = d run_genapply [derived_constants, "-V32"] (src_rts "AutoApply_V32.cmm") run_genapply [derived_constants, "-V64"] (src_rts "AutoApply_V64.cmm") - -- Generate genprimopcode + -- Generate primop code for ghc-prim + -- + -- Note that this can't be done in a Setup.hs for ghc-prim because + -- cabal-install can't build it because t depends on base, Cabal, etc. + -- libraries that aren't built yet. let primops_txt = src "libraries/ghc/GHC/Builtin/primops.txt" let primops_txt_pp = primops_txt <.> ".pp" primops <- readCreateProcess (shell $ "gcc -E -undef -traditional -P -x c " ++ primops_txt_pp) "" @@ -478,11 +481,20 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts = d let build_boot_cmd = runCabal cabal [ "build" , "--project-file=cabal.project-stage1" -- TODO: replace with command-line args - , "ghc-prim", "ghc-internal", "base" , "--with-compiler=" ++ ghcPath ghc -- FIXME: escape path , "--with-hc-pkg=" ++ ghcPkgPath ghcpkg -- FIXME: escape path , "--ghc-options=\"-ghcversion-file=" ++ ghcversionh ++ "\"" , "--builddir=" ++ build_dir + -- never reinstall the RTS during this step: we should use the one + -- installed at the previous step. Otherwise we risk using invalid + -- generated files. + -- FIXME: but it doesn't work because the rts isn't really installed + -- , "--constraint=rts installed" + + -- targets + , "ghc-prim" + , "ghc-internal" + , "base" ] msg " - Building boot libraries..." diff --git a/cabal.project-stage0 b/cabal.project-stage0 index 7fa4dfb9a98c..80976f21b7a3 100644 --- a/cabal.project-stage0 +++ b/cabal.project-stage0 @@ -1,6 +1,6 @@ packages: ./_build/stage0/src/ghc-bin/ - ./_build/stage0/src/libraries/ghc + ./_build/stage0/src/libraries/ghc/ ./_build/stage0/src/libraries/directory/ ./_build/stage0/src/libraries/file-io/ ./_build/stage0/src/libraries/filepath/ @@ -43,7 +43,9 @@ constraints: package ghc-boot-th flags: +bootstrap + -- package genprimopcode -- flags: -build-tool-depends +-- allow template-haskell with newer ghc-boot-th allow-newer: ghc-boot-th diff --git a/libraries/ghc-prim/ghc-prim.cabal b/libraries/ghc-prim/ghc-prim.cabal index 8cb39c0b932d..add096d36d0b 100644 --- a/libraries/ghc-prim/ghc-prim.cabal +++ b/libraries/ghc-prim/ghc-prim.cabal @@ -8,6 +8,9 @@ category: GHC maintainer: libraries@haskell.org bug-reports: https://gitlab.haskell.org/ghc/ghc/issues/new synopsis: GHC primitives +-- We can't use Custom build-type with boot packages: building Setup.hs with +-- cabal-install requires that base, Cabal, directory, process, filepath, +-- containers, etc. libraries be available, but they aren't when we bootstrap. build-type: Simple description: This package used to contain the primitive types and operations supplied by From 4ff9370a160f764751e90875e2f6e92633860a5b Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 15 Jan 2025 15:20:30 +0100 Subject: [PATCH 041/175] Cleanup --- Build.hs | 45 +++++++++++++++++++++------------------------ 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/Build.hs b/Build.hs index 4acf05323b2e..f068402b6157 100755 --- a/Build.hs +++ b/Build.hs @@ -60,7 +60,7 @@ main = do msg $ "Bootstrapping GHC version: " ++ init ghc0_version msg "Building stage1 GHC program and utility programs" - buildGhcStage1 defaultGhcBuildOptions cabal ghc0 + buildGhcStage1 defaultGhcBuildOptions cabal ghc0 "_build/stage0/" ghc1 <- Ghc <$> makeAbsolute "_build/stage0/bin/ghc" ghcPkg1 <- GhcPkg <$> makeAbsolute "_build/stage0/bin/ghc-pkg" @@ -69,17 +69,17 @@ main = do genprimop <- GenPrimop <$> makeAbsolute "_build/stage0/bin/genprimopcode" msg "Building boot libraries with stage1 compiler..." - buildBootLibraries cabal ghc1 ghcPkg1 deriveConstants genapply genprimop defaultGhcBuildOptions + buildBootLibraries cabal ghc1 ghcPkg1 deriveConstants genapply genprimop defaultGhcBuildOptions "_build/stage1/" msg "Done" -- | Build stage1 GHC program -buildGhcStage1 :: GhcBuildOptions -> Cabal -> Ghc -> IO () -buildGhcStage1 opts cabal ghc0 = do - prepareGhcSources opts "_build/stage0/src/" +buildGhcStage1 :: GhcBuildOptions -> Cabal -> Ghc -> FilePath -> IO () +buildGhcStage1 opts cabal ghc0 dst = do + prepareGhcSources opts (dst "src/") - let builddir = "_build/stage0/cabal/" + let builddir = dst "cabal" createDirectoryIfMissing True builddir -- we need to augment the current environment to pass HADRIAN_SETTINGS @@ -120,13 +120,13 @@ buildGhcStage1 opts cabal ghc0 = do { env = Just stage1_env } (exit_code, cabal_stdout, cabal_stderr) <- readCreateProcessWithExitCode build_cmd "" - writeFile "_build/stage0/cabal.stdout" cabal_stdout - writeFile "_build/stage0/cabal.stderr" cabal_stderr + writeFile (dst "cabal.stdout") cabal_stdout + writeFile (dst "cabal.stderr") cabal_stderr case exit_code of ExitSuccess -> pure () ExitFailure n -> do putStrLn $ "cabal-install failed with error code: " ++ show n - putStrLn "Logs can be found in \"_build/stage0/cabal.{stdout,stderr}\"" + putStrLn $ "Logs can be found in \"" ++ dst ++ "/cabal.{stdout,stderr}\"" exitFailure msg " - Copying stage1 programs and generating settings to use them..." @@ -142,12 +142,12 @@ buildGhcStage1 opts cabal ghc0 = do case list_bin_exit_code of ExitSuccess | (bin_src:_) <- lines list_bin_stdout - -> cp bin_src ("_build/stage0/bin" bin) + -> cp bin_src (dst "bin" bin) _ -> do putStrLn $ "Failed to run cabal list-bin for the target: " ++ show target putStrLn list_bin_stderr exitFailure - createDirectoryIfMissing True "_build/stage0/bin" + createDirectoryIfMissing True (dst "bin") copy_bin "ghc-bin:ghc" "ghc" copy_bin "ghc-pkg:ghc-pkg" "ghc-pkg" copy_bin "deriveConstants:deriveConstants" "deriveConstants" @@ -155,11 +155,11 @@ buildGhcStage1 opts cabal ghc0 = do copy_bin "genapply:genapply" "genapply" -- initialize empty global package database - pkgdb <- makeAbsolute "_build/stage1/pkgs" + pkgdb <- makeAbsolute (dst "pkgs") doesDirectoryExist pkgdb >>= \case True -> pure () -- don't try to recreate the DB if it already exist as it would fail False -> do - ghcpkg <- GhcPkg <$> makeAbsolute "_build/stage0/bin/ghc-pkg" + ghcpkg <- GhcPkg <$> makeAbsolute (dst "bin/ghc-pkg") void $ readCreateProcess (runGhcPkg ghcpkg ["init", pkgdb]) "" void $ readCreateProcess (runGhcPkg ghcpkg [ "recache" @@ -169,13 +169,13 @@ buildGhcStage1 opts cabal ghc0 = do -- generate settings based on stage1 compiler settings - createDirectoryIfMissing True "_build/stage0/lib" + createDirectoryIfMissing True (dst "lib") let stage1_settings = makeStage1Settings stage0_settings - writeFile "_build/stage0/lib/settings" (show stage1_settings) + writeFile (dst "lib/settings") (show stage1_settings) -- try to run the stage1 compiler (no package db yet, so just display the -- version) - (test_exit_code, test_stdout, _test_stderr) <- readCreateProcessWithExitCode (proc "_build/stage0/bin/ghc" ["--version"]) "" + (test_exit_code, test_stdout, _test_stderr) <- readCreateProcessWithExitCode (proc (dst "bin/ghc") ["--version"]) "" case test_exit_code of ExitSuccess -> pure () ExitFailure n -> do @@ -339,22 +339,19 @@ makeStage1Settings in_settings = out_settings , keep_def "target RTS linker only supports shared libraries" "NO" , ("Use interpreter", "NO") - , ("base unit-id", "base") -- FIXME + , ("base unit-id", "base") -- there is no base yet... Anyway this isn't really useful to set , keep_fail "Support SMP" , keep_fail "RTS ways" , keep_fail "Tables next to code" , keep_fail "Leading underscore" , keep_fail "Use LibFFI" , keep_fail "RTS expects libdw" - , ("Relative Global Package DB", "../../stage1/pkgs") + , ("Relative Global Package DB", "../pkgs") ] -buildBootLibraries :: Cabal -> Ghc -> GhcPkg -> DeriveConstants -> GenApply -> GenPrimop -> GhcBuildOptions -> IO () -buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts = do - -- FIXME: should be parameters - let dst = "_build/stage1/" - src <- makeAbsolute "_build/stage1/src" - +buildBootLibraries :: Cabal -> Ghc -> GhcPkg -> DeriveConstants -> GenApply -> GenPrimop -> GhcBuildOptions -> FilePath -> IO () +buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst = do + src <- makeAbsolute (dst "src") prepareGhcSources opts src -- Build the RTS From fd3029fbb5fea9d786777c5eae121dd2dc9667f2 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Thu, 16 Jan 2025 14:26:40 +0100 Subject: [PATCH 042/175] Generate cabal.project files for easier maintenance and parameterization --- Build.hs | 166 ++++++++++++++++++++++++++++++++++++--- cabal.project-stage0 | 51 ------------ cabal.project-stage1 | 67 ---------------- cabal.project-stage1-rts | 19 ----- 4 files changed, 155 insertions(+), 148 deletions(-) delete mode 100644 cabal.project-stage0 delete mode 100644 cabal.project-stage1 delete mode 100644 cabal.project-stage1-rts diff --git a/Build.hs b/Build.hs index f068402b6157..3a3c65dbb528 100755 --- a/Build.hs +++ b/Build.hs @@ -77,7 +77,8 @@ main = do -- | Build stage1 GHC program buildGhcStage1 :: GhcBuildOptions -> Cabal -> Ghc -> FilePath -> IO () buildGhcStage1 opts cabal ghc0 dst = do - prepareGhcSources opts (dst "src/") + let src = dst "src" + prepareGhcSources opts src let builddir = dst "cabal" createDirectoryIfMissing True builddir @@ -104,12 +105,59 @@ buildGhcStage1 opts cabal ghc0 dst = do let stage1_env = ("HADRIAN_SETTINGS", stage1_ghc_boot_settings) : current_env msg " - Building GHC stage1 and bootstrapping utility programs..." + let cabal_project_path = dst "cabal.project-stage0" + makeCabalProject cabal_project_path + [ "packages:" + , " " ++ src "ghc-bin/" + , " " ++ src "libraries/ghc/" + , " " ++ src "libraries/directory/" + , " " ++ src "libraries/file-io/" + , " " ++ src "libraries/filepath/" + , " " ++ src "libraries/ghc-platform/" + , " " ++ src "libraries/ghc-boot/" + , " " ++ src "libraries/ghc-boot-th/" + , " " ++ src "libraries/ghc-heap" + , " " ++ src "libraries/ghci" + , " " ++ src "libraries/os-string/" + , " " ++ src "libraries/process/" + , " " ++ src "libraries/semaphore-compat" + , " " ++ src "libraries/time" + , " " ++ src "libraries/unix/" + , " " ++ src "libraries/Win32/" + , " " ++ src "utils/ghc-pkg" + , " " ++ src "utils/hsc2hs" + , " " ++ src "utils/unlit" + , " " ++ src "utils/genprimopcode/" + , " " ++ src "utils/genapply/" + , " " ++ src "utils/deriveConstants/" + , "" + , "benchmarks: False" + , "tests: False" + , "allow-boot-library-installs: True" + , "" + , "package *" + , " library-vanilla: True" + , " shared: False" + , " executable-profiling: False" + , " executable-dynamic: False" + , " executable-static: True" + , "" + , "constraints:" + -- for some reason 2.23 doesn't build + , " template-haskell <= 2.22" + , "" + , "package ghc-boot-th" + , " flags: +bootstrap" + , "" + -- allow template-haskell with newer ghc-boot-th + , "allow-newer: ghc-boot-th" + ] let build_cmd = (runCabal cabal [ "build" - , "--project-file=cabal.project-stage0" + , "--project-file=" ++ cabal_project_path , "--builddir=" ++ builddir , "-j" - , "--with-compiler=" ++ ghcPath ghc0 -- FIXME: escape path + , "--with-compiler=" ++ ghcPath ghc0 -- the targets , "ghc-bin:ghc" , "ghc-pkg:ghc-pkg" @@ -132,8 +180,8 @@ buildGhcStage1 opts cabal ghc0 dst = do msg " - Copying stage1 programs and generating settings to use them..." let listbin_cmd p = runCabal cabal [ "list-bin" - , "--project-file=cabal.project-stage0" - , "--with-compiler=" ++ ghcPath ghc0 -- FIXME: escape path + , "--project-file=" ++ cabal_project_path + , "--with-compiler=" ++ ghcPath ghc0 , "--builddir=" ++ builddir , p ] @@ -359,12 +407,32 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst build_dir <- makeAbsolute (dst "cabal") ghcversionh <- makeAbsolute (src_rts "include/ghcversion.h") + let cabal_project_rts_path = dst "cabal.project-rts" + makeCabalProject cabal_project_rts_path + [ "package-dbs: clear, global" + , "" + , "packages:" + , " ./_build/stage1/src/libraries/rts" + , "" + , "benchmarks: False" + , "tests: False" + , "allow-boot-library-installs: True" + , "active-repositories: :none" + , "" + , "package *" + , " library-vanilla: True" + , " shared: False" + , " executable-profiling: False" + , " executable-dynamic: False" + , " executable-static: False" + ] + let build_rts_cmd = runCabal cabal [ "build" - , "--project-file=cabal.project-stage1-rts" -- TODO: replace with command-line args + , "--project-file=" ++ cabal_project_rts_path , "rts" - , "--with-compiler=" ++ ghcPath ghc -- FIXME: escape path - , "--with-hc-pkg=" ++ ghcPkgPath ghcpkg -- FIXME: escape path + , "--with-compiler=" ++ ghcPath ghc + , "--with-hc-pkg=" ++ ghcPkgPath ghcpkg , "--ghc-options=\"-ghcversion-file=" ++ ghcversionh ++ "\"" , "--ghc-options=\"-I" ++ (src_rts "include") ++ "\"" , "--ghc-options=\"-I" ++ src_rts ++ "\"" @@ -475,11 +543,78 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst exitFailure -- build boot libraries: ghc-internal, base... but not GHC itself + let cabal_project_bootlibs_path = dst "cabal-project-boot-libs" + makeCabalProject cabal_project_bootlibs_path + [ "package-dbs: clear, global" + , "" + , "packages:" + , " ./_build/stage1/src/libraries/rts" + , " ./_build/stage1/src/libraries/ghc-prim" + , " ./_build/stage1/src/libraries/ghc-internal" + , " ./_build/stage1/src/libraries/base" + , " ./_build/stage1/src/libraries/ghc" + , " ./_build/stage1/src/libraries/ghc-platform/" + , " ./_build/stage1/src/libraries/ghc-boot/" + , " ./_build/stage1/src/libraries/ghc-boot-th/" + , " ./_build/stage1/src/libraries/ghc-heap" + , " ./_build/stage1/src/libraries/ghci" + , " ./_build/stage1/src/libraries/stm" + , " ./_build/stage1/src/libraries/template-haskell" + , " ./_build/stage1/src/libraries/hpc" + , " ./_build/stage1/src/ghc-bin/" + , " ./_build/stage1/src/utils/ghc-pkg" + , " ./_build/stage1/src/utils/hsc2hs" + , " ./_build/stage1/src/utils/unlit" + , "" + , " ./_build/stage1/src/libraries/array" + , " ./_build/stage1/src/libraries/binary" + , " ./_build/stage1/src/libraries/bytestring" + , " ./_build/stage1/src/libraries/containers/containers" + , " ./_build/stage1/src/libraries/deepseq" + , " ./_build/stage1/src/libraries/directory/" + , " ./_build/stage1/src/libraries/exceptions" + , " ./_build/stage1/src/libraries/file-io/" + , " ./_build/stage1/src/libraries/filepath/" + , " ./_build/stage1/src/libraries/mtl" + , " ./_build/stage1/src/libraries/os-string/" + , " ./_build/stage1/src/libraries/parsec" + , " ./_build/stage1/src/libraries/pretty/" + , " ./_build/stage1/src/libraries/process/" + , " ./_build/stage1/src/libraries/semaphore-compat" + , " ./_build/stage1/src/libraries/text" + , " ./_build/stage1/src/libraries/time" + , " ./_build/stage1/src/libraries/transformers" + , " ./_build/stage1/src/libraries/unix/" + , " ./_build/stage1/src/libraries/Win32/" + , " ./_build/stage1/src/libraries/Cabal/Cabal-syntax" + , " ./_build/stage1/src/libraries/Cabal/Cabal" + , " https://github.com/haskell/alex/archive/refs/tags/v3.5.2.0.tar.gz" + , "" + , "benchmarks: False" + , "tests: False" + , "allow-boot-library-installs: True" + , "active-repositories: :none" + , "" + , "package *" + , " library-vanilla: True" + , " shared: False" + , " executable-profiling: False" + , " executable-dynamic: False" + , " executable-static: False" + , "" + , "package ghc-internal" + -- FIXME: make our life easier for now by using the native bignum backend + , " flags: +bignum-native" + , "" + , "package text" + -- FIXME: avoid having to deal with system-cxx-std-lib fake package for now + , " flags: -simdutf" + ] let build_boot_cmd = runCabal cabal [ "build" - , "--project-file=cabal.project-stage1" -- TODO: replace with command-line args - , "--with-compiler=" ++ ghcPath ghc -- FIXME: escape path - , "--with-hc-pkg=" ++ ghcPkgPath ghcpkg -- FIXME: escape path + , "--project-file=" ++ cabal_project_bootlibs_path + , "--with-compiler=" ++ ghcPath ghc + , "--with-hc-pkg=" ++ ghcPkgPath ghcpkg , "--ghc-options=\"-ghcversion-file=" ++ ghcversionh ++ "\"" , "--builddir=" ++ build_dir -- never reinstall the RTS during this step: we should use the one @@ -581,3 +716,12 @@ runGenPrimop (GenPrimop f) = proc f cp :: String -> String -> IO () cp src dst = void (readCreateProcess (shell $ "cp -rf " ++ src ++ " " ++ dst) "") + +makeCabalProject :: FilePath -> [String] -> IO () +makeCabalProject path xs = writeFile path $ unlines (xs ++ common) + where + common = + [ "" + , "program-options" + , " ghc-options: -fhide-source-paths -j" + ] diff --git a/cabal.project-stage0 b/cabal.project-stage0 deleted file mode 100644 index 80976f21b7a3..000000000000 --- a/cabal.project-stage0 +++ /dev/null @@ -1,51 +0,0 @@ -packages: - ./_build/stage0/src/ghc-bin/ - ./_build/stage0/src/libraries/ghc/ - ./_build/stage0/src/libraries/directory/ - ./_build/stage0/src/libraries/file-io/ - ./_build/stage0/src/libraries/filepath/ - ./_build/stage0/src/libraries/ghc-platform/ - ./_build/stage0/src/libraries/ghc-boot/ - ./_build/stage0/src/libraries/ghc-boot-th/ - ./_build/stage0/src/libraries/ghc-heap - ./_build/stage0/src/libraries/ghci - ./_build/stage0/src/libraries/os-string/ - ./_build/stage0/src/libraries/process/ - ./_build/stage0/src/libraries/semaphore-compat - ./_build/stage0/src/libraries/time - ./_build/stage0/src/libraries/unix/ - ./_build/stage0/src/libraries/Win32/ - ./_build/stage0/src/utils/ghc-pkg - ./_build/stage0/src/utils/hsc2hs - ./_build/stage0/src/utils/unlit - ./_build/stage0/src/utils/genprimopcode/ - ./_build/stage0/src/utils/genapply/ - ./_build/stage0/src/utils/deriveConstants/ - -benchmarks: False -tests: False -allow-boot-library-installs: True - -program-options - ghc-options: -fhide-source-paths -j - -package * - library-vanilla: True - shared: False - executable-profiling: False - executable-dynamic: False - executable-static: True - -constraints: - -- for some reason 2.23 doesn't build - template-haskell <= 2.22 - -package ghc-boot-th - flags: +bootstrap - - --- package genprimopcode --- flags: -build-tool-depends - --- allow template-haskell with newer ghc-boot-th -allow-newer: ghc-boot-th diff --git a/cabal.project-stage1 b/cabal.project-stage1 deleted file mode 100644 index eb9295b7b6ea..000000000000 --- a/cabal.project-stage1 +++ /dev/null @@ -1,67 +0,0 @@ -package-dbs: clear, global - -packages: - ./_build/stage1/src/libraries/rts - ./_build/stage1/src/libraries/ghc-prim - ./_build/stage1/src/libraries/ghc-internal - ./_build/stage1/src/libraries/base - ./_build/stage1/src/libraries/ghc - ./_build/stage1/src/libraries/ghc-platform/ - ./_build/stage1/src/libraries/ghc-boot/ - ./_build/stage1/src/libraries/ghc-boot-th/ - ./_build/stage1/src/libraries/ghc-heap - ./_build/stage1/src/libraries/ghci - ./_build/stage1/src/libraries/stm - ./_build/stage1/src/libraries/template-haskell - ./_build/stage1/src/libraries/hpc - ./_build/stage1/src/ghc-bin/ - ./_build/stage1/src/utils/ghc-pkg - ./_build/stage1/src/utils/hsc2hs - ./_build/stage1/src/utils/unlit - - ./_build/stage1/src/libraries/array - ./_build/stage1/src/libraries/binary - ./_build/stage1/src/libraries/bytestring - ./_build/stage1/src/libraries/containers/containers - ./_build/stage1/src/libraries/deepseq - ./_build/stage1/src/libraries/directory/ - ./_build/stage1/src/libraries/exceptions - ./_build/stage1/src/libraries/file-io/ - ./_build/stage1/src/libraries/filepath/ - ./_build/stage1/src/libraries/mtl - ./_build/stage1/src/libraries/os-string/ - ./_build/stage1/src/libraries/parsec - ./_build/stage1/src/libraries/pretty/ - ./_build/stage1/src/libraries/process/ - ./_build/stage1/src/libraries/semaphore-compat - ./_build/stage1/src/libraries/text - ./_build/stage1/src/libraries/time - ./_build/stage1/src/libraries/transformers - ./_build/stage1/src/libraries/unix/ - ./_build/stage1/src/libraries/Win32/ - ./_build/stage1/src/libraries/Cabal/Cabal-syntax - ./_build/stage1/src/libraries/Cabal/Cabal - https://github.com/haskell/alex/archive/refs/tags/v3.5.2.0.tar.gz - -benchmarks: False -tests: False -allow-boot-library-installs: True -active-repositories: :none - -program-options - ghc-options: -fhide-source-paths -j - -package * - library-vanilla: True - shared: False - executable-profiling: False - executable-dynamic: False - executable-static: False - -package ghc-internal - -- FIXME: make our life easier for now by using the native bignum backend - flags: +bignum-native - -package text - -- FIXME: avoid having to deal with system-cxx-std-lib fake package for now - flags: -simdutf diff --git a/cabal.project-stage1-rts b/cabal.project-stage1-rts deleted file mode 100644 index 8deeec5d01c3..000000000000 --- a/cabal.project-stage1-rts +++ /dev/null @@ -1,19 +0,0 @@ -package-dbs: clear, global - -packages: - ./_build/stage1/src/libraries/rts - -benchmarks: False -tests: False -allow-boot-library-installs: True -active-repositories: :none - -program-options - ghc-options: -fhide-source-paths -j - -package * - library-vanilla: True - shared: False - executable-profiling: False - executable-dynamic: False - executable-static: False From c8fd283d618cb065c5114746807c2600a6179558 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Thu, 16 Jan 2025 14:35:34 +0100 Subject: [PATCH 043/175] Fix passing ghc-options to the rts when building Cmm --- Build.hs | 6 +----- rts/rts.cabal | 1 + 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/Build.hs b/Build.hs index 3a3c65dbb528..ef171fff3a58 100755 --- a/Build.hs +++ b/Build.hs @@ -437,10 +437,6 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , "--ghc-options=\"-I" ++ (src_rts "include") ++ "\"" , "--ghc-options=\"-I" ++ src_rts ++ "\"" , "--builddir=" ++ build_dir - -- we need to pass "-this-unit-id=rts", otherwise GHC tries to lookup the - -- platform constants in the package db and fails. The flag is already - -- set in rts.cabal but for some reason it isn't always passed :shrug: - , "--ghc-options=-this-unit-id=rts" , "--ghc-options=\"-optc=-DProjectVersion=\\\"" ++ Text.unpack (gboVersionInt opts) ++ "\\\"\"" , "--ghc-options=\"-optc=-DRtsWay=\\\"FIXME\\\"\"" , "--ghc-options=\"-optc=-DHostPlatform=\\\"FIXME\\\"\"" @@ -498,7 +494,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst -- Generate primop code for ghc-prim -- -- Note that this can't be done in a Setup.hs for ghc-prim because - -- cabal-install can't build it because t depends on base, Cabal, etc. + -- cabal-install can't build Setup.hs because it depends on base, Cabal, etc. -- libraries that aren't built yet. let primops_txt = src "libraries/ghc/GHC/Builtin/primops.txt" let primops_txt_pp = primops_txt <.> ".pp" diff --git a/rts/rts.cabal b/rts/rts.cabal index 0a54f13fa714..fa61ac4f6d2a 100644 --- a/rts/rts.cabal +++ b/rts/rts.cabal @@ -109,6 +109,7 @@ library -- expects the unit-id to be -- set without version ghc-options: -this-unit-id rts + cmm-options: -this-unit-id rts exposed: True exposed-modules: From 872ec28e607d7c024bb84e0a2fb137be4827e086 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Thu, 16 Jan 2025 14:37:34 +0100 Subject: [PATCH 044/175] Fix hard-wired paths --- Build.hs | 81 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 40 insertions(+), 41 deletions(-) diff --git a/Build.hs b/Build.hs index ef171fff3a58..ea19c3901a5b 100755 --- a/Build.hs +++ b/Build.hs @@ -412,7 +412,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst [ "package-dbs: clear, global" , "" , "packages:" - , " ./_build/stage1/src/libraries/rts" + , " " ++ src "libraries/rts" , "" , "benchmarks: False" , "tests: False" @@ -544,46 +544,45 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst [ "package-dbs: clear, global" , "" , "packages:" - , " ./_build/stage1/src/libraries/rts" - , " ./_build/stage1/src/libraries/ghc-prim" - , " ./_build/stage1/src/libraries/ghc-internal" - , " ./_build/stage1/src/libraries/base" - , " ./_build/stage1/src/libraries/ghc" - , " ./_build/stage1/src/libraries/ghc-platform/" - , " ./_build/stage1/src/libraries/ghc-boot/" - , " ./_build/stage1/src/libraries/ghc-boot-th/" - , " ./_build/stage1/src/libraries/ghc-heap" - , " ./_build/stage1/src/libraries/ghci" - , " ./_build/stage1/src/libraries/stm" - , " ./_build/stage1/src/libraries/template-haskell" - , " ./_build/stage1/src/libraries/hpc" - , " ./_build/stage1/src/ghc-bin/" - , " ./_build/stage1/src/utils/ghc-pkg" - , " ./_build/stage1/src/utils/hsc2hs" - , " ./_build/stage1/src/utils/unlit" - , "" - , " ./_build/stage1/src/libraries/array" - , " ./_build/stage1/src/libraries/binary" - , " ./_build/stage1/src/libraries/bytestring" - , " ./_build/stage1/src/libraries/containers/containers" - , " ./_build/stage1/src/libraries/deepseq" - , " ./_build/stage1/src/libraries/directory/" - , " ./_build/stage1/src/libraries/exceptions" - , " ./_build/stage1/src/libraries/file-io/" - , " ./_build/stage1/src/libraries/filepath/" - , " ./_build/stage1/src/libraries/mtl" - , " ./_build/stage1/src/libraries/os-string/" - , " ./_build/stage1/src/libraries/parsec" - , " ./_build/stage1/src/libraries/pretty/" - , " ./_build/stage1/src/libraries/process/" - , " ./_build/stage1/src/libraries/semaphore-compat" - , " ./_build/stage1/src/libraries/text" - , " ./_build/stage1/src/libraries/time" - , " ./_build/stage1/src/libraries/transformers" - , " ./_build/stage1/src/libraries/unix/" - , " ./_build/stage1/src/libraries/Win32/" - , " ./_build/stage1/src/libraries/Cabal/Cabal-syntax" - , " ./_build/stage1/src/libraries/Cabal/Cabal" + , " " ++ src "libraries/rts" + , " " ++ src "libraries/ghc-prim" + , " " ++ src "libraries/ghc-internal" + , " " ++ src "libraries/base" + , " " ++ src "libraries/ghc" + , " " ++ src "libraries/ghc-platform/" + , " " ++ src "libraries/ghc-boot/" + , " " ++ src "libraries/ghc-boot-th/" + , " " ++ src "libraries/ghc-heap" + , " " ++ src "libraries/ghci" + , " " ++ src "libraries/stm" + , " " ++ src "libraries/template-haskell" + , " " ++ src "libraries/hpc" + , " " ++ src "ghc-bin/" + , " " ++ src "utils/ghc-pkg" + , " " ++ src "utils/hsc2hs" + , " " ++ src "utils/unlit" + , " " ++ src "libraries/array" + , " " ++ src "libraries/binary" + , " " ++ src "libraries/bytestring" + , " " ++ src "libraries/containers/containers" + , " " ++ src "libraries/deepseq" + , " " ++ src "libraries/directory/" + , " " ++ src "libraries/exceptions" + , " " ++ src "libraries/file-io/" + , " " ++ src "libraries/filepath/" + , " " ++ src "libraries/mtl" + , " " ++ src "libraries/os-string/" + , " " ++ src "libraries/parsec" + , " " ++ src "libraries/pretty/" + , " " ++ src "libraries/process/" + , " " ++ src "libraries/semaphore-compat" + , " " ++ src "libraries/text" + , " " ++ src "libraries/time" + , " " ++ src "libraries/transformers" + , " " ++ src "libraries/unix/" + , " " ++ src "libraries/Win32/" + , " " ++ src "libraries/Cabal/Cabal-syntax" + , " " ++ src "libraries/Cabal/Cabal" , " https://github.com/haskell/alex/archive/refs/tags/v3.5.2.0.tar.gz" , "" , "benchmarks: False" From cc91b7e8c5ee611a82f8114f1de32399e86bceec Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Mon, 20 Jan 2025 11:06:49 +0100 Subject: [PATCH 045/175] Fix fs.h for unlit --- Build.hs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Build.hs b/Build.hs index ea19c3901a5b..f1dedc713e88 100755 --- a/Build.hs +++ b/Build.hs @@ -256,6 +256,7 @@ prepareGhcSources opts dst = do cp "utils/fs/fs.h" (dst "libraries/ghc-internal/include") cp "utils/fs/fs.c" (dst "libraries/ghc-internal/cbits") cp "utils/fs/fs.*" (dst "libraries/rts/") + cp "utils/fs/fs.*" (dst "utils/unlit/") python <- findExecutable "python" >>= \case Nothing -> error "Couldn't find 'python'" @@ -612,9 +613,11 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , "--with-hc-pkg=" ++ ghcPkgPath ghcpkg , "--ghc-options=\"-ghcversion-file=" ++ ghcversionh ++ "\"" , "--builddir=" ++ build_dir + , "-j" -- never reinstall the RTS during this step: we should use the one -- installed at the previous step. Otherwise we risk using invalid - -- generated files. + -- generated files and we don't pass enough options here to build the + -- rts anyway. -- FIXME: but it doesn't work because the rts isn't really installed -- , "--constraint=rts installed" From 2ece465258a15c8e4076e512ba9eb1e2e6cac390 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Mon, 20 Jan 2025 11:33:57 +0100 Subject: [PATCH 046/175] Build all the boot libraries at once, including the rts --- Build.hs | 150 +++++++++++++++++++++++++++---------------------------- 1 file changed, 73 insertions(+), 77 deletions(-) diff --git a/Build.hs b/Build.hs index f1dedc713e88..5dc01da86492 100755 --- a/Build.hs +++ b/Build.hs @@ -15,6 +15,7 @@ build-depends: {-# LANGUAGE DerivingStrategies #-} {-# LANGUAGE GeneralizedNewtypeDeriving #-} {-# LANGUAGE NumericUnderscores #-} +{-# OPTIONS_GHC -Wall #-} -- | GHC builder -- @@ -91,14 +92,14 @@ buildGhcStage1 opts cabal ghc0 dst = do -- we infer stage1's host platform from stage0's settings let settings = [ ("hostPlatformArch", fromMaybe (error "Couldn't read 'target arch' setting") (lookup "target arch" stage0_settings)) - , ("hostPlatformOS", fromMaybe (error "Couldn't read 'target os' setting") (lookup "target os" stage0_settings)) + , ("hostPlatformOS", fromMaybe (error "Couldn't read 'target os' setting") (lookup "target os" stage0_settings)) , ("cProjectGitCommitId", commit_id) - , ("cProjectVersion", Text.unpack $ gboVersion opts) + , ("cProjectVersion", Text.unpack $ gboVersion opts) , ("cProjectVersionInt", Text.unpack $ gboVersionInt opts) , ("cProjectPatchLevel", Text.unpack $ gboVersionPatchLevel opts) , ("cProjectPatchLevel1", Text.unpack $ gboVersionPatchLevel1 opts) , ("cProjectPatchLevel2", Text.unpack $ gboVersionPatchLevel2 opts) - ] + ] :: [(String,String)] pure (show settings) current_env <- getEnvironment @@ -179,28 +180,28 @@ buildGhcStage1 opts cabal ghc0 dst = do msg " - Copying stage1 programs and generating settings to use them..." let listbin_cmd p = runCabal cabal - [ "list-bin" - , "--project-file=" ++ cabal_project_path + [ "list-bin" + , "--project-file=" ++ cabal_project_path , "--with-compiler=" ++ ghcPath ghc0 - , "--builddir=" ++ builddir - , p - ] + , "--builddir=" ++ builddir + , p + ] let copy_bin target bin = do - (list_bin_exit_code, list_bin_stdout, list_bin_stderr) <- readCreateProcessWithExitCode (listbin_cmd target) "" - case list_bin_exit_code of - ExitSuccess - | (bin_src:_) <- lines list_bin_stdout - -> cp bin_src (dst "bin" bin) - _ -> do - putStrLn $ "Failed to run cabal list-bin for the target: " ++ show target - putStrLn list_bin_stderr - exitFailure + (list_bin_exit_code, list_bin_stdout, list_bin_stderr) <- readCreateProcessWithExitCode (listbin_cmd target) "" + case list_bin_exit_code of + ExitSuccess + | (bin_src:_) <- lines list_bin_stdout + -> cp bin_src (dst "bin" bin) + _ -> do + putStrLn $ "Failed to run cabal list-bin for the target: " ++ show target + putStrLn list_bin_stderr + exitFailure createDirectoryIfMissing True (dst "bin") - copy_bin "ghc-bin:ghc" "ghc" - copy_bin "ghc-pkg:ghc-pkg" "ghc-pkg" + copy_bin "ghc-bin:ghc" "ghc" + copy_bin "ghc-pkg:ghc-pkg" "ghc-pkg" copy_bin "deriveConstants:deriveConstants" "deriveConstants" copy_bin "genprimopcode:genprimopcode" "genprimopcode" - copy_bin "genapply:genapply" "genapply" + copy_bin "genapply:genapply" "genapply" -- initialize empty global package database pkgdb <- makeAbsolute (dst "pkgs") @@ -223,7 +224,7 @@ buildGhcStage1 opts cabal ghc0 dst = do -- try to run the stage1 compiler (no package db yet, so just display the -- version) - (test_exit_code, test_stdout, _test_stderr) <- readCreateProcessWithExitCode (proc (dst "bin/ghc") ["--version"]) "" + (test_exit_code, _test_stdout, _test_stderr) <- readCreateProcessWithExitCode (proc (dst "bin/ghc") ["--version"]) "" case test_exit_code of ExitSuccess -> pure () ExitFailure n -> do @@ -241,7 +242,7 @@ prepareGhcSources opts dst = do cp "./libraries" dst cp "./compiler/*" (dst "libraries/ghc/") cp "./rts" (dst "libraries/") - cp "./ghc" (dst "ghc-bin") + cp "./ghc" (dst "ghc-bin") cp "./utils" dst cp "./config.sub" (dst "libraries/rts/") @@ -275,25 +276,25 @@ prepareGhcSources opts dst = do -- substitute variables in files let subst fin fout rs = do - t <- Text.readFile fin - Text.writeFile fout (List.foldl' (\v (needle,rep) -> Text.replace needle rep v) t rs) + t <- Text.readFile fin + Text.writeFile fout (List.foldl' (\v (needle,rep) -> Text.replace needle rep v) t rs) let subst_in f = subst (f <.> "in") f let common_substs = - [ (,) "@ProjectVersion@" (gboVersion opts) - , (,) "@ProjectVersionMunged@" (gboVersionMunged opts) - , (,) "@ProjectVersionForLib@" (gboVersionForLib opts) - , (,) "@ProjectPatchLevel1@" (gboVersionPatchLevel1 opts) - , (,) "@ProjectPatchLevel2@" (gboVersionPatchLevel2 opts) - , (,) "@ProjectVersionInt@" (gboVersionInt opts) - ] + [ (,) "@ProjectVersion@" (gboVersion opts) + , (,) "@ProjectVersionMunged@" (gboVersionMunged opts) + , (,) "@ProjectVersionForLib@" (gboVersionForLib opts) + , (,) "@ProjectPatchLevel1@" (gboVersionPatchLevel1 opts) + , (,) "@ProjectPatchLevel2@" (gboVersionPatchLevel2 opts) + , (,) "@ProjectVersionInt@" (gboVersionInt opts) + ] llvm_substs = - [ (,) "@LlvmMinVersion@" (gboLlvmMinVersion opts) - , (,) "@LlvmMaxVersion@" (gboLlvmMaxVersion opts) - ] + [ (,) "@LlvmMinVersion@" (gboLlvmMinVersion opts) + , (,) "@LlvmMaxVersion@" (gboLlvmMaxVersion opts) + ] boot_th_substs = - [ (,) "@Suffix@" "" - , (,) "@SourceRoot@" "." - ] + [ (,) "@Suffix@" "" + , (,) "@SourceRoot@" "." + ] subst_in (dst "ghc-bin/ghc-bin.cabal") common_substs subst_in (dst "libraries/ghc/ghc.cabal") common_substs @@ -409,7 +410,32 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst ghcversionh <- makeAbsolute (src_rts "include/ghcversion.h") let cabal_project_rts_path = dst "cabal.project-rts" - makeCabalProject cabal_project_rts_path + -- cabal's code handling escaping is bonkers. We need to wrap the whole + -- option into \" otherwise it does weird things (like keeping only the + -- last double-quote). + let def_string k v = " ghc-options: \"-optc-D" ++ k ++ "=\\\"" ++ v ++ "\\\"\"" + let rts_options = + [ "package rts" + , def_string "ProjectVersion" (Text.unpack (gboVersionInt opts)) + , def_string "RtsWay" "FIXME" + , def_string "HostPlatform" "FIXME" + , def_string "HostArch" "FIXME" + , def_string "HostOS" "FIXME" + , def_string "HostVendor" "FIXME" + , def_string "BuildPlatform" "FIXME" + , def_string "BuildArch" "FIXME" + , def_string "BuildOS" "FIXME" + , def_string "BuildVendor" "FIXME" + , def_string "TargetPlatform" "FIXME" + , def_string "TargetArch" "FIXME" + , def_string "TargetOS" "FIXME" + , def_string "TargetVendor" "FIXME" + , def_string "GhcUnregisterised" "FIXME" + , def_string "TablesNextToCode" "FIXME" + , " ghc-options: -I" ++ (src_rts "include") + , " ghc-options: -I" ++ src_rts + ] + makeCabalProject cabal_project_rts_path $ [ "package-dbs: clear, global" , "" , "packages:" @@ -426,7 +452,8 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , " executable-profiling: False" , " executable-dynamic: False" , " executable-static: False" - ] + , "" + ] ++ rts_options let build_rts_cmd = runCabal cabal [ "build" @@ -435,25 +462,8 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , "--with-compiler=" ++ ghcPath ghc , "--with-hc-pkg=" ++ ghcPkgPath ghcpkg , "--ghc-options=\"-ghcversion-file=" ++ ghcversionh ++ "\"" - , "--ghc-options=\"-I" ++ (src_rts "include") ++ "\"" - , "--ghc-options=\"-I" ++ src_rts ++ "\"" , "--builddir=" ++ build_dir - , "--ghc-options=\"-optc=-DProjectVersion=\\\"" ++ Text.unpack (gboVersionInt opts) ++ "\\\"\"" - , "--ghc-options=\"-optc=-DRtsWay=\\\"FIXME\\\"\"" - , "--ghc-options=\"-optc=-DHostPlatform=\\\"FIXME\\\"\"" - , "--ghc-options=\"-optc=-DHostArch=\\\"FIXME\\\"\"" - , "--ghc-options=\"-optc=-DHostOS=\\\"FIXME\\\"\"" - , "--ghc-options=\"-optc=-DHostVendor=\\\"FIXME\\\"\"" - , "--ghc-options=\"-optc=-DBuildPlatform=\\\"FIXME\\\"\"" - , "--ghc-options=\"-optc=-DBuildArch=\\\"FIXME\\\"\"" - , "--ghc-options=\"-optc=-DBuildOS=\\\"FIXME\\\"\"" - , "--ghc-options=\"-optc=-DBuildVendor=\\\"FIXME\\\"\"" - , "--ghc-options=\"-optc=-DTargetPlatform=\\\"FIXME\\\"\"" - , "--ghc-options=\"-optc=-DTargetArch=\\\"FIXME\\\"\"" - , "--ghc-options=\"-optc=-DTargetOS=\\\"FIXME\\\"\"" - , "--ghc-options=\"-optc=-DTargetVendor=\\\"FIXME\\\"\"" - , "--ghc-options=\"-optc=-DGhcUnregisterised=\\\"FIXME\\\"\"" - , "--ghc-options=\"-optc=-DTablesNextToCode=\\\"FIXME\\\"\"" + , "-v3" ] -- FIXME: deriveConstants requires ghcautoconf.h and ghcplatform.h but these @@ -528,20 +538,9 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst cp (dst_libffi "include" "*") (src_rts "include") cp (dst_libffi "lib" "libffi.a") (takeDirectory ghcplatform_dir "libCffi.a") - -- second run of cabal is expected to succeed now that have generated all the headers! - msg " - Building the RTS..." - (rts_exit_code, rts_stdout, rts_stderr) <- readCreateProcessWithExitCode build_rts_cmd "" - case rts_exit_code of - ExitSuccess -> pure () - ExitFailure r -> do - putStrLn $ "Failed to build the RTS with error code " ++ show r - putStrLn rts_stdout - putStrLn rts_stderr - exitFailure - -- build boot libraries: ghc-internal, base... but not GHC itself let cabal_project_bootlibs_path = dst "cabal-project-boot-libs" - makeCabalProject cabal_project_bootlibs_path + makeCabalProject cabal_project_bootlibs_path $ [ "package-dbs: clear, global" , "" , "packages:" @@ -605,7 +604,9 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , "package text" -- FIXME: avoid having to deal with system-cxx-std-lib fake package for now , " flags: -simdutf" - ] + , "" + ] ++ rts_options + let build_boot_cmd = runCabal cabal [ "build" , "--project-file=" ++ cabal_project_bootlibs_path @@ -614,14 +615,9 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , "--ghc-options=\"-ghcversion-file=" ++ ghcversionh ++ "\"" , "--builddir=" ++ build_dir , "-j" - -- never reinstall the RTS during this step: we should use the one - -- installed at the previous step. Otherwise we risk using invalid - -- generated files and we don't pass enough options here to build the - -- rts anyway. - -- FIXME: but it doesn't work because the rts isn't really installed - -- , "--constraint=rts installed" -- targets + , "rts" , "ghc-prim" , "ghc-internal" , "base" @@ -642,7 +638,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst --------------------------- data GhcBuildOptions = GhcBuildOptions - { gboVersion :: !Text -- ^ GHC version + { gboVersion :: !Text -- ^ GHC version , gboVersionInt :: !Text -- ^ GHC version as an Int , gboVersionMunged :: !Text -- ^ GHC version "munged" , gboVersionForLib :: !Text -- ^ GHC version for libraries? From cc3e24c86b8fb4225c2eebe70f5596f1c3b48ff8 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Mon, 20 Jan 2025 11:59:47 +0100 Subject: [PATCH 047/175] Dump logs into files --- Build.hs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Build.hs b/Build.hs index 5dc01da86492..d2e6ab774a99 100755 --- a/Build.hs +++ b/Build.hs @@ -625,14 +625,16 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst msg " - Building boot libraries..." (boot_exit_code, boot_stdout, boot_stderr) <- readCreateProcessWithExitCode build_boot_cmd "" + writeFile (dst "boot-libs.stdout") boot_stdout + writeFile (dst "boot-libs.stderr") boot_stderr case boot_exit_code of ExitSuccess -> pure () ExitFailure r -> do putStrLn $ "Failed to build boot libraries with error code " ++ show r - putStrLn boot_stdout - putStrLn boot_stderr + putStrLn $ "Logs can be found in " ++ dst ++ "/boot-libs.{stdout,stderr}" exitFailure + --------------------------- -- Options --------------------------- From 5bc6c2d304d2a76cabdea04dc8af4673fc595aeb Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Mon, 20 Jan 2025 15:29:50 +0100 Subject: [PATCH 048/175] Fix a few things found while trying to use "cabal install --lib" --- Build.hs | 2 +- rts/rts.cabal | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Build.hs b/Build.hs index d2e6ab774a99..fb2657167da3 100755 --- a/Build.hs +++ b/Build.hs @@ -433,6 +433,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , def_string "GhcUnregisterised" "FIXME" , def_string "TablesNextToCode" "FIXME" , " ghc-options: -I" ++ (src_rts "include") + , " ghc-options: -I" ++ (src_rts "adjustor") , " ghc-options: -I" ++ src_rts ] makeCabalProject cabal_project_rts_path $ @@ -463,7 +464,6 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , "--with-hc-pkg=" ++ ghcPkgPath ghcpkg , "--ghc-options=\"-ghcversion-file=" ++ ghcversionh ++ "\"" , "--builddir=" ++ build_dir - , "-v3" ] -- FIXME: deriveConstants requires ghcautoconf.h and ghcplatform.h but these diff --git a/rts/rts.cabal b/rts/rts.cabal index fa61ac4f6d2a..017a79837010 100644 --- a/rts/rts.cabal +++ b/rts/rts.cabal @@ -13,6 +13,11 @@ build-type: Configure extra-source-files: configure + config.guess + config.sub + ghcplatform.h.top.in + ghcplatform.h.bottom + ghcautoconf.h.autoconf.in configure.ac external-symbols.list.in rts.buildinfo.in From a7570d2729aa9f38dab332f94d7f5cacc73ffe59 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Tue, 21 Jan 2025 16:59:48 +0800 Subject: [PATCH 049/175] Move Cabal submodule to our patched version --- .gitmodules | 3 ++- libraries/Cabal | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 46f1db3e7cc5..01d9e8110ed1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -8,8 +8,9 @@ ignore = untracked [submodule "libraries/Cabal"] path = libraries/Cabal - url = https://gitlab.haskell.org/ghc/packages/Cabal.git + url = https://github.com/hsyl20/cabal ignore = untracked + branch = hsyl20/per-file-extra-source-options [submodule "libraries/containers"] path = libraries/containers url = https://gitlab.haskell.org/ghc/packages/containers.git diff --git a/libraries/Cabal b/libraries/Cabal index 269fd808e5d8..e8538c6597cc 160000 --- a/libraries/Cabal +++ b/libraries/Cabal @@ -1 +1 @@ -Subproject commit 269fd808e5d80223a229b6b19edfe6f5b109007a +Subproject commit e8538c6597cc355eca705827845f5341d1d815f2 From fa5b181415c89e45b5fff8407771f18b00cfd793 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Tue, 21 Jan 2025 17:24:19 +0800 Subject: [PATCH 050/175] Add patched cabal to Makefile --- Build.hs | 8 +++----- Makefile | 9 +++++++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Build.hs b/Build.hs index fb2657167da3..67f3849d4917 100755 --- a/Build.hs +++ b/Build.hs @@ -39,11 +39,6 @@ import System.CPUTime main :: IO () main = do - -- FIXME: specific patched cabal-install for now and GHC that is known to - -- work... - setEnv "CABAL" "/home/hsyl20/projects/cabal/dist-newstyle/build/x86_64-linux/ghc-9.10.1/cabal-install-3.15.0.0/x/cabal/build/cabal/cabal" - setEnv "GHC" "ghc-9.8.4" - -- detect GHC and cabal-install to use for bootstrapping ghc0 <- do ghc_path <- fromMaybe "ghc" <$> lookupEnv "GHC" @@ -106,6 +101,7 @@ buildGhcStage1 opts cabal ghc0 dst = do let stage1_env = ("HADRIAN_SETTINGS", stage1_ghc_boot_settings) : current_env msg " - Building GHC stage1 and bootstrapping utility programs..." + let cabal_project_path = dst "cabal.project-stage0" makeCabalProject cabal_project_path [ "packages:" @@ -153,6 +149,7 @@ buildGhcStage1 opts cabal ghc0 dst = do -- allow template-haskell with newer ghc-boot-th , "allow-newer: ghc-boot-th" ] + let build_cmd = (runCabal cabal [ "build" , "--project-file=" ++ cabal_project_path @@ -168,6 +165,7 @@ buildGhcStage1 opts cabal ghc0 dst = do ]) { env = Just stage1_env } + (exit_code, cabal_stdout, cabal_stderr) <- readCreateProcessWithExitCode build_cmd "" writeFile (dst "cabal.stdout") cabal_stdout writeFile (dst "cabal.stderr") cabal_stderr diff --git a/Makefile b/Makefile index fe6b75ba35a3..89fac3092df2 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,10 @@ -all: - ./Build.hs +CABAL := $(shell cabal list-bin -v0 --project-dir libraries/Cabal cabal-install:exe:cabal) + +all: $(CABAL) + CABAL=$(CABAL) GHC=ghc-9.8.4 ./Build.hs + +$(CABAL): + cabal build --project-dir libraries/Cabal cabal clean: rm -rf _build From 8a91f42946dec7bf63e4b0de3fada3c206e9e685 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Tue, 21 Jan 2025 17:53:25 +0800 Subject: [PATCH 051/175] Add TODO list --- Build.hs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Build.hs b/Build.hs index 67f3849d4917..a45ff56456ea 100755 --- a/Build.hs +++ b/Build.hs @@ -230,6 +230,11 @@ buildGhcStage1 opts cabal ghc0 dst = do exitFailure +-- TODO: +-- - headers shared between different packages should be a common dependency +-- - event types should be generated by Setup.hs +-- package versions: do they need to be all the same? + -- | Prepare GHC sources in the given directory prepareGhcSources :: GhcBuildOptions -> FilePath -> IO () prepareGhcSources opts dst = do @@ -246,12 +251,14 @@ prepareGhcSources opts dst = do cp "./config.sub" (dst "libraries/rts/") cp "./config.guess" (dst "libraries/rts/") + -- These needs to shared cp "rts/include/rts/Bytecodes.h" (dst "libraries/ghc/") cp "rts/include/rts/storage/ClosureTypes.h" (dst "libraries/ghc/") cp "rts/include/rts/storage/FunTypes.h" (dst "libraries/ghc/") cp "rts/include/stg/MachRegs.h" (dst "libraries/ghc/") cp "rts/include/stg/MachRegs/*.h" (dst "libraries/ghc/MachRegs/") + -- shared among ghc-internal rts and unlit cp "utils/fs/fs.h" (dst "libraries/ghc-internal/include") cp "utils/fs/fs.c" (dst "libraries/ghc-internal/cbits") cp "utils/fs/fs.*" (dst "libraries/rts/") @@ -266,6 +273,7 @@ prepareGhcSources opts dst = do , "--event-types-defines" , dst "libraries/rts/include/rts/EventLogConstants.h" ]) "" + void $ readCreateProcess (proc python [ "rts/gen_event_types.py" , "--event-types-array" @@ -301,7 +309,10 @@ prepareGhcSources opts dst = do subst_in (dst "libraries/ghc-heap/ghc-heap.cabal") common_substs subst_in (dst "libraries/template-haskell/template-haskell.cabal") common_substs subst_in (dst "libraries/ghci/ghci.cabal") common_substs + + -- This is only used for a warning message. Nuke the check! subst_in (dst "libraries/ghc/GHC/CmmToLlvm/Version/Bounds.hs") llvm_substs + subst_in (dst "utils/ghc-pkg/ghc-pkg.cabal") common_substs subst_in (dst "libraries/ghc-internal/ghc-internal.cabal") common_substs @@ -434,6 +445,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , " ghc-options: -I" ++ (src_rts "adjustor") , " ghc-options: -I" ++ src_rts ] + makeCabalProject cabal_project_rts_path $ [ "package-dbs: clear, global" , "" From 0f029f94b5b482f9b91f67e8c6a7f73d00c8bae0 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Tue, 21 Jan 2025 17:53:37 +0800 Subject: [PATCH 052/175] Simplify Build.hs dependencies --- Build.hs | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/Build.hs b/Build.hs index a45ff56456ea..5c83bb62bfb9 100755 --- a/Build.hs +++ b/Build.hs @@ -1,13 +1,4 @@ -#!/usr/bin/env cabal -{- cabal: -build-depends: - base, - directory, - filepath, - process, - text, - temporary --} +#!/usr/bin/env runhaskell {-# LANGUAGE LambdaCase #-} {-# LANGUAGE ImportQualifiedPost #-} @@ -29,12 +20,12 @@ import Data.Text (Text) import Data.Text qualified as Text import Data.Text.IO qualified as Text import Control.Monad +import Control.Exception (bracket) import System.Environment import System.Directory import System.Process import System.FilePath import System.Exit -import System.IO.Temp import System.CPUTime main :: IO () @@ -731,3 +722,15 @@ makeCabalProject path xs = writeFile path $ unlines (xs ++ common) , "program-options" , " ghc-options: -fhide-source-paths -j" ] + + +withSystemTempDirectory :: String -> (String -> IO a) -> IO a +withSystemTempDirectory prefix = do + bracket + (do + tmpdir <- getTemporaryDirectory + let dir = tmpdir prefix + createDirectory dir + return dir + ) + removeDirectoryRecursive From 92816b6c42bb5d11b8291aa609fea7443b0c9388 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 21 Jan 2025 11:03:15 +0100 Subject: [PATCH 053/175] Progress towards cabal-install install --lib --- Build.hs | 5 +- rts/rts.cabal | 234 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 236 insertions(+), 3 deletions(-) diff --git a/Build.hs b/Build.hs index 5c83bb62bfb9..38e725856fcd 100755 --- a/Build.hs +++ b/Build.hs @@ -432,9 +432,6 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , def_string "TargetVendor" "FIXME" , def_string "GhcUnregisterised" "FIXME" , def_string "TablesNextToCode" "FIXME" - , " ghc-options: -I" ++ (src_rts "include") - , " ghc-options: -I" ++ (src_rts "adjustor") - , " ghc-options: -I" ++ src_rts ] makeCabalProject cabal_project_rts_path $ @@ -609,6 +606,8 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst ] ++ rts_options let build_boot_cmd = runCabal cabal + -- [ "install" + -- , "--lib" [ "build" , "--project-file=" ++ cabal_project_bootlibs_path , "--with-compiler=" ++ ghcPath ghc diff --git a/rts/rts.cabal b/rts/rts.cabal index 017a79837010..364df2458a90 100644 --- a/rts/rts.cabal +++ b/rts/rts.cabal @@ -21,6 +21,239 @@ extra-source-files: configure.ac external-symbols.list.in rts.buildinfo.in + linker/ELFRelocs/AArch64.def + linker/ELFRelocs/ARM.def + linker/ELFRelocs/i386.def + linker/ELFRelocs/x86_64.def + win32/libHSffi.def + win32/libHSghc-internal.def + win32/libHSghc-prim.def + posix/ticker/Pthread.c + posix/ticker/Setitimer.c + posix/ticker/TimerCreate.c + posix/ticker/TimerFd.c + -- headers files that are not installed by the rts package but only used to + -- build the rts C code + adjustor/AdjustorPool.h + Adjustor.h + Apply.h + Arena.h + ARMOutlineAtomicsSymbols.h + AutoApply.h + AutoApplyVecs.h + BeginPrivate.h + Capability.h + CheckUnload.h + CheckVectorSupport.h + CloneStack.h + Continuation.h + Disassembler.h + EndPrivate.h + eventlog/EventLog.h + Excn.h + FileLock.h + ForeignExports.h + fs_rts.h + fs.h + GetEnv.h + GetTime.h + Globals.h + Hash.h + hooks/Hooks.h + include/Cmm.h + include/ghcconfig.h + include/HsFFI.h + include/MachDeps.h + include/rts/Adjustor.h + include/RtsAPI.h + include/rts/BlockSignals.h + include/rts/Bytecodes.h + include/rts/Config.h + include/rts/Constants.h + include/rts/EventLogFormat.h + include/rts/EventLogWriter.h + include/rts/ExecPage.h + include/rts/FileLock.h + include/rts/Flags.h + include/rts/ForeignExports.h + include/rts/GetTime.h + include/rts/ghc_ffi.h + include/rts/Globals.h + include/Rts.h + include/rts/Hpc.h + include/rts/IOInterface.h + include/rts/IPE.h + include/rts/Libdw.h + include/rts/LibdwPool.h + include/rts/Linker.h + include/rts/Main.h + include/rts/Messages.h + include/rts/NonMoving.h + include/rts/OSThreads.h + include/rts/Parallel.h + include/rts/PosixSource.h + include/rts/PrimFloat.h + include/rts/prof/CCS.h + include/rts/prof/Heap.h + include/rts/Profiling.h + include/rts/prof/LDV.h + include/rts/Signals.h + include/rts/SpinLock.h + include/rts/StableName.h + include/rts/StablePtr.h + include/rts/StaticPtrTable.h + include/rts/storage/Block.h + include/rts/storage/ClosureMacros.h + include/rts/storage/Closures.h + include/rts/storage/ClosureTypes.h + include/rts/storage/FunTypes.h + include/rts/storage/GC.h + include/rts/storage/HeapAlloc.h + include/rts/storage/Heap.h + include/rts/storage/InfoTables.h + include/rts/storage/MBlock.h + include/rts/storage/TSO.h + include/rts/Threads.h + include/rts/Ticky.h + include/rts/Time.h + include/rts/Timer.h + include/rts/TSANUtils.h + include/rts/TTY.h + include/rts/Types.h + include/rts/Utils.h + include/stg/DLL.h + include/Stg.h + include/stg/MachRegs/arm32.h + include/stg/MachRegs/arm64.h + include/stg/MachRegsForHost.h + include/stg/MachRegs.h + include/stg/MachRegs/loongarch64.h + include/stg/MachRegs/ppc.h + include/stg/MachRegs/riscv64.h + include/stg/MachRegs/s390x.h + include/stg/MachRegs/wasm32.h + include/stg/MachRegs/x86.h + include/stg/MiscClosures.h + include/stg/Prim.h + include/stg/Regs.h + include/stg/SMP.h + include/stg/Ticky.h + include/stg/Types.h + Interpreter.h + IOManager.h + IOManagerInternals.h + IPE.h + Jumps.h + LdvProfile.h + Libdw.h + LibdwPool.h + linker/CacheFlush.h + linker/elf_compat.h + linker/elf_got.h + linker/Elf.h + linker/elf_plt_aarch64.h + linker/elf_plt_arm.h + linker/elf_plt.h + linker/elf_plt_riscv64.h + linker/elf_reloc_aarch64.h + linker/elf_reloc.h + linker/elf_reloc_riscv64.h + linker/ElfTypes.h + linker/elf_util.h + linker/InitFini.h + LinkerInternals.h + linker/LoadNativeObjPosix.h + linker/M32Alloc.h + linker/MachO.h + linker/macho/plt_aarch64.h + linker/macho/plt.h + linker/MachOTypes.h + linker/MMap.h + linker/PEi386.h + linker/PEi386Types.h + linker/SymbolExtras.h + linker/util.h + linker/Wasm32Types.h + Messages.h + PathUtils.h + Pool.h + posix/Clock.h + posix/Select.h + posix/Signals.h + posix/TTY.h + Prelude.h + Printer.h + ProfHeap.h + ProfHeapInternal.h + ProfilerReport.h + ProfilerReportJson.h + Profiling.h + Proftimer.h + RaiseAsync.h + ReportMemoryMap.h + RetainerProfile.h + RetainerSet.h + RtsDllMain.h + RtsFlags.h + RtsSignals.h + RtsSymbolInfo.h + RtsSymbols.h + RtsUtils.h + Schedule.h + sm/BlockAlloc.h + sm/CNF.h + sm/Compact.h + sm/Evac.h + sm/GC.h + sm/GCTDecl.h + sm/GCThread.h + sm/GCUtils.h + sm/HeapUtils.h + sm/MarkStack.h + sm/MarkWeak.h + sm/NonMovingAllocate.h + sm/NonMovingCensus.h + sm/NonMoving.h + sm/NonMovingMark.h + sm/NonMovingScav.h + sm/NonMovingShortcut.h + sm/NonMovingSweep.h + sm/OSMem.h + SMPClosureOps.h + sm/Sanity.h + sm/Scav.h + sm/ShouldCompact.h + sm/Storage.h + sm/Sweep.h + Sparks.h + StableName.h + StablePtr.h + StaticPtrTable.h + Stats.h + StgPrimFloat.h + StgRun.h + STM.h + Task.h + ThreadLabels.h + ThreadPaused.h + Threads.h + Ticker.h + Ticky.h + Timer.h + TopHandler.h + Trace.h + TraverseHeap.h + Updates.h + Weak.h + win32/AsyncMIO.h + win32/AsyncWinIO.h + win32/AwaitEvent.h + win32/ConsoleHandler.h + win32/MIOManager.h + win32/ThrIOManager.h + win32/veh_excn.h + win32/WorkQueue.h + WSDeque.h extra-tmp-files: autom4te.cache @@ -276,6 +509,7 @@ library cpp-options: -DNOSMP include-dirs: include + include-dirs: . includes: Rts.h autogen-includes: ghcautoconf.h ghcplatform.h install-includes: Cmm.h HsFFI.h MachDeps.h Jumps.h Rts.h RtsAPI.h RtsSymbols.h Stg.h From 7e6164616d5248dbc590ecef8876dd1a0171ccea Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 21 Jan 2025 11:19:18 +0100 Subject: [PATCH 054/175] Fix timestamps --- Build.hs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/Build.hs b/Build.hs index 38e725856fcd..f65303e077d2 100755 --- a/Build.hs +++ b/Build.hs @@ -26,7 +26,10 @@ import System.Directory import System.Process import System.FilePath import System.Exit -import System.CPUTime +import System.IO.Unsafe +import Data.Time.Clock +import Data.IORef +import Data.Fixed main :: IO () main = do @@ -670,12 +673,17 @@ defaultGhcBuildOptions = GhcBuildOptions -- Utilities --------------------------- +{-# NOINLINE init_time #-} +init_time :: IORef UTCTime +init_time = unsafePerformIO (newIORef =<< getCurrentTime) + -- | Display a message to the user with some timestamp msg :: String -> IO () msg x = do - t <- getCPUTime - let d = t `div` 1_000_000_000 - let stp = "[" ++ show d ++ "]" + it <- readIORef init_time + t <- getCurrentTime + let d = realToFrac (nominalDiffTimeToSeconds (diffUTCTime t it)) :: Centi + let stp = "[" ++ show d ++ "s] " putStrLn (stp ++ replicate (6 - length stp) ' ' ++ x) -- Avoid FilePath blindness by using type aliases for programs. From cb880672cf5f3321bef9dab2f19b6cdcde07feef Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 22 Jan 2025 12:21:06 +0100 Subject: [PATCH 055/175] Use cabal install --lib to build boot libraries --- Build.hs | 41 +++++++++++++------- libraries/ghc-internal/ghc-internal.cabal.in | 8 ++-- 2 files changed, 32 insertions(+), 17 deletions(-) diff --git a/Build.hs b/Build.hs index f65303e077d2..566866c4601e 100755 --- a/Build.hs +++ b/Build.hs @@ -197,16 +197,8 @@ buildGhcStage1 opts cabal ghc0 dst = do -- initialize empty global package database pkgdb <- makeAbsolute (dst "pkgs") - doesDirectoryExist pkgdb >>= \case - True -> pure () -- don't try to recreate the DB if it already exist as it would fail - False -> do - ghcpkg <- GhcPkg <$> makeAbsolute (dst "bin/ghc-pkg") - void $ readCreateProcess (runGhcPkg ghcpkg ["init", pkgdb]) "" - void $ readCreateProcess (runGhcPkg ghcpkg - [ "recache" - , "--global-package-db="++pkgdb - , "--no-user-package-db" - ]) "" + ghcpkg <- GhcPkg <$> makeAbsolute (dst "bin/ghc-pkg") + initEmptyDB ghcpkg pkgdb -- generate settings based on stage1 compiler settings @@ -435,6 +427,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , def_string "TargetVendor" "FIXME" , def_string "GhcUnregisterised" "FIXME" , def_string "TablesNextToCode" "FIXME" + , " flags: +use-system-libffi" -- FIXME: deal with libffi (add package?) ] makeCabalProject cabal_project_rts_path $ @@ -525,6 +518,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst void $ readCreateProcess (shell ("tar -xvf libffi-tarballs/libffi-" ++ libffi_version ++ ".tar.gz -C " ++ src_libffi)) "" let build_libffi = mconcat [ "cd " ++ src_libffi "libffi-" ++ libffi_version ++ "; " + -- FIXME: pass the appropriate toolchain (CC, LD...) , "./configure --disable-docs --with-pic=yes --disable-multi-os-directory --prefix=" ++ dst_libffi , " && make install -j" ] @@ -608,10 +602,17 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , "" ] ++ rts_options + + let pkgdb = dst "pkgs" + initEmptyDB ghcpkg pkgdb + let build_boot_cmd = runCabal cabal - -- [ "install" - -- , "--lib" - [ "build" + [ "install" + , "--lib" + , "--package-db=" ++ pkgdb + , "--package-env=" ++ dst + , "-v3" + -- [ "build" , "--project-file=" ++ cabal_project_bootlibs_path , "--with-compiler=" ++ ghcPath ghc , "--with-hc-pkg=" ++ ghcPkgPath ghcpkg @@ -634,7 +635,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst ExitSuccess -> pure () ExitFailure r -> do putStrLn $ "Failed to build boot libraries with error code " ++ show r - putStrLn $ "Logs can be found in " ++ dst ++ "/boot-libs.{stdout,stderr}" + putStrLn $ "Logs can be found in " ++ dst ++ "boot-libs.{stdout,stderr}" exitFailure @@ -741,3 +742,15 @@ withSystemTempDirectory prefix = do return dir ) removeDirectoryRecursive + +initEmptyDB :: GhcPkg -> FilePath -> IO () +initEmptyDB ghcpkg pkgdb = do + doesDirectoryExist pkgdb >>= \case + True -> pure () -- don't try to recreate the DB if it already exist as it would fail + False -> do + void $ readCreateProcess (runGhcPkg ghcpkg ["init", pkgdb]) "" + void $ readCreateProcess (runGhcPkg ghcpkg + [ "recache" + , "--global-package-db="++pkgdb + , "--no-user-package-db" + ]) "" diff --git a/libraries/ghc-internal/ghc-internal.cabal.in b/libraries/ghc-internal/ghc-internal.cabal.in index 76cdcb31be66..a35b5707853e 100644 --- a/libraries/ghc-internal/ghc-internal.cabal.in +++ b/libraries/ghc-internal/ghc-internal.cabal.in @@ -343,9 +343,11 @@ Library GHC.Internal.Tuple GHC.Internal.Types - autogen-modules: - GHC.Internal.Prim - GHC.Internal.PrimopWrappers + -- Cabal expects autogen modules to be some specific directories, not in the + -- source dirs... + -- autogen-modules: + -- GHC.Internal.Prim + -- GHC.Internal.PrimopWrappers other-modules: GHC.Internal.Data.Typeable.Internal From 9eaa75e872baff81e1852297053c4e0c7f925418 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 22 Jan 2025 13:14:49 +0100 Subject: [PATCH 056/175] Parse package env --- Build.hs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/Build.hs b/Build.hs index 566866c4601e..d2cd3e77ca0f 100755 --- a/Build.hs +++ b/Build.hs @@ -603,14 +603,12 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst ] ++ rts_options - let pkgdb = dst "pkgs" - initEmptyDB ghcpkg pkgdb - + let boot_libs_env = dst "boot-libs.env" let build_boot_cmd = runCabal cabal [ "install" , "--lib" - , "--package-db=" ++ pkgdb - , "--package-env=" ++ dst + , "--package-env=" ++ boot_libs_env + , "--force-reinstalls" , "-v3" -- [ "build" , "--project-file=" ++ cabal_project_bootlibs_path @@ -638,6 +636,13 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst putStrLn $ "Logs can be found in " ++ dst ++ "boot-libs.{stdout,stderr}" exitFailure + -- The libraries have been installed globally. + (global_gb:pkg_ids) <- (map (drop 11) . drop 2 . lines) <$> readFile boot_libs_env + putStrLn $ "We've built boot libraries in " ++ global_gb ++ ":" + mapM_ (putStrLn . (" - " ++)) pkg_ids + + -- TODO: copy the libs in another db + --------------------------- -- Options From 3e48043046409a5eba5454b6e1f991906d289545 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Thu, 23 Jan 2025 14:41:36 +0800 Subject: [PATCH 057/175] Do not hardcode GHC version into Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 89fac3092df2..13278bcd0469 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ CABAL := $(shell cabal list-bin -v0 --project-dir libraries/Cabal cabal-install:exe:cabal) all: $(CABAL) - CABAL=$(CABAL) GHC=ghc-9.8.4 ./Build.hs + CABAL=$(CABAL) ./Build.hs $(CABAL): cabal build --project-dir libraries/Cabal cabal From 9ceb81427601b27478b23308caea307a3933bf5d Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Thu, 23 Jan 2025 15:53:24 +0800 Subject: [PATCH 058/175] chore: add Build.hs to hie.yaml --- hie.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/hie.yaml b/hie.yaml index 34dde0452ad1..b666fd0f0517 100644 --- a/hie.yaml +++ b/hie.yaml @@ -5,4 +5,13 @@ # cradle: {bios: {program: "./hadrian/hie-bios.bat"}} # # The format is documented here - https://github.com/mpickering/hie-bios -cradle: {bios: {program: "./hadrian/hie-bios"}} +cradle: + multi: + - path: Build.hs + config: + cradle: + direct: + arguments: [] + - path: "*" + config: + cradle: {bios: {program: "./hadrian/hie-bios"}} From 650c647ad94d67ad9f06ce386f821f9f0863f782 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Thu, 23 Jan 2025 17:51:04 +0800 Subject: [PATCH 059/175] Display output from cabal build --- Build.hs | 81 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 44 insertions(+), 37 deletions(-) diff --git a/Build.hs b/Build.hs index d2cd3e77ca0f..8c49260ad4f2 100755 --- a/Build.hs +++ b/Build.hs @@ -20,12 +20,13 @@ import Data.Text (Text) import Data.Text qualified as Text import Data.Text.IO qualified as Text import Control.Monad -import Control.Exception (bracket) +import Control.Exception (bracket, IOException, catch) import System.Environment import System.Directory import System.Process import System.FilePath import System.Exit +import System.IO import System.IO.Unsafe import Data.Time.Clock import Data.IORef @@ -144,50 +145,49 @@ buildGhcStage1 opts cabal ghc0 dst = do , "allow-newer: ghc-boot-th" ] - let build_cmd = (runCabal cabal - [ "build" - , "--project-file=" ++ cabal_project_path - , "--builddir=" ++ builddir - , "-j" - , "--with-compiler=" ++ ghcPath ghc0 - -- the targets - , "ghc-bin:ghc" - , "ghc-pkg:ghc-pkg" - , "genprimopcode:genprimopcode" - , "deriveConstants:deriveConstants" - , "genapply:genapply" - ]) - { env = Just stage1_env - } - - (exit_code, cabal_stdout, cabal_stderr) <- readCreateProcessWithExitCode build_cmd "" - writeFile (dst "cabal.stdout") cabal_stdout - writeFile (dst "cabal.stderr") cabal_stderr + + exit_code <- + let args = [ "build" + , "--project-file=" ++ cabal_project_path + , "--builddir=" ++ builddir + , "-j" + , "--with-compiler=" ++ ghcPath ghc0 + -- the targets + , "ghc-bin:ghc" + , "ghc-pkg:ghc-pkg" + , "genprimopcode:genprimopcode" + , "deriveConstants:deriveConstants" + , "genapply:genapply" + ] + in runProcess' $ (runCabal cabal args) { env = Just stage1_env } case exit_code of ExitSuccess -> pure () ExitFailure n -> do putStrLn $ "cabal-install failed with error code: " ++ show n - putStrLn $ "Logs can be found in \"" ++ dst ++ "/cabal.{stdout,stderr}\"" exitFailure msg " - Copying stage1 programs and generating settings to use them..." - let listbin_cmd p = runCabal cabal - [ "list-bin" - , "--project-file=" ++ cabal_project_path - , "--with-compiler=" ++ ghcPath ghc0 - , "--builddir=" ++ builddir - , p - ] + let copy_bin target bin = do - (list_bin_exit_code, list_bin_stdout, list_bin_stderr) <- readCreateProcessWithExitCode (listbin_cmd target) "" - case list_bin_exit_code of - ExitSuccess - | (bin_src:_) <- lines list_bin_stdout - -> cp bin_src (dst "bin" bin) - _ -> do - putStrLn $ "Failed to run cabal list-bin for the target: " ++ show target - putStrLn list_bin_stderr - exitFailure + let args = + [ "list-bin" + , "--project-file=" ++ cabal_project_path + , "--with-compiler=" ++ ghcPath ghc0 + , "--builddir=" ++ builddir + , target + ] + + bin_src <- + readCreateProcess (runCabal cabal args) "" + `catch` + (\e -> do + hPutStr stderr $ "Failed to run cabal list-bin for the target: " ++ show target + hPutStr stderr $ show (e :: IOException) + exitFailure + ) + + cp bin_src (dst "bin" bin) + createDirectoryIfMissing True (dst "bin") copy_bin "ghc-bin:ghc" "ghc" copy_bin "ghc-pkg:ghc-pkg" "ghc-pkg" @@ -759,3 +759,10 @@ initEmptyDB ghcpkg pkgdb = do , "--global-package-db="++pkgdb , "--no-user-package-db" ]) "" + +runProcess' :: CreateProcess -> IO ExitCode +runProcess' p = + withCreateProcess p { + std_out = Inherit, + std_err = Inherit + } $ \_ _ _ ph -> waitForProcess ph From ea16e5e13b52223ab561e5b1921f7e4952ba9ced Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Thu, 23 Jan 2025 17:57:55 +0800 Subject: [PATCH 060/175] Separate cabal from GHC in GitHub workflow --- .github/workflows/ci.yml | 9 +++++---- Makefile | 8 +++++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f872684db452..31b59c2ab9f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,9 +33,10 @@ jobs: cabal-update: true - name: Configure the build - run: | - ./boot + run: ./boot + + - name: Build patched cabal + run: make cabal - name: Build the bindist - run: | - make + run: make diff --git a/Makefile b/Makefile index 13278bcd0469..3f9ffc83f293 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,12 @@ -CABAL := $(shell cabal list-bin -v0 --project-dir libraries/Cabal cabal-install:exe:cabal) +export CABAL := $(shell cabal list-bin -v0 --project-dir libraries/Cabal cabal-install:exe:cabal) all: $(CABAL) - CABAL=$(CABAL) ./Build.hs + ./Build.hs +cabal: $(CABAL) + $(CABAL): - cabal build --project-dir libraries/Cabal cabal + cabal build --project-dir libraries/Cabal cabal-install:exe:cabal clean: rm -rf _build From 2e79dba882822d59bbbf157ccefbae28669f5772 Mon Sep 17 00:00:00 2001 From: Andrea Bedini Date: Fri, 24 Jan 2025 13:33:31 +0800 Subject: [PATCH 061/175] No need to recache a newly initialised packagedb --- Build.hs | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/Build.hs b/Build.hs index 8c49260ad4f2..bed5eabae1a1 100755 --- a/Build.hs +++ b/Build.hs @@ -750,15 +750,8 @@ withSystemTempDirectory prefix = do initEmptyDB :: GhcPkg -> FilePath -> IO () initEmptyDB ghcpkg pkgdb = do - doesDirectoryExist pkgdb >>= \case - True -> pure () -- don't try to recreate the DB if it already exist as it would fail - False -> do - void $ readCreateProcess (runGhcPkg ghcpkg ["init", pkgdb]) "" - void $ readCreateProcess (runGhcPkg ghcpkg - [ "recache" - , "--global-package-db="++pkgdb - , "--no-user-package-db" - ]) "" + exists <- doesDirectoryExist pkgdb + unless exists $ void $ readCreateProcess (runGhcPkg ghcpkg ["init", pkgdb]) "" runProcess' :: CreateProcess -> IO ExitCode runProcess' p = From d942040963fc70130ad1fe1351e3d32986bcf1b9 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Fri, 24 Jan 2025 09:54:35 +0100 Subject: [PATCH 062/175] Use GHC 9.8.4 on CI --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 31b59c2ab9f2..87d31ed05b3a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest] - ghc: ['9.6.5'] # bootstrapping compiler + ghc: ['9.8.4'] # bootstrapping compiler steps: - uses: actions/checkout@v4 From f89fd48e542634a286676968abeae05ca1c9fcc3 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Fri, 24 Jan 2025 11:31:50 +0100 Subject: [PATCH 063/175] Revert "Display output from cabal build" This reverts commit f9ce6017e0468e405ef8fdf722d63bb43dc953c9. --- Build.hs | 82 ++++++++++++++++++++++++++------------------------------ 1 file changed, 38 insertions(+), 44 deletions(-) diff --git a/Build.hs b/Build.hs index bed5eabae1a1..ebfd85752c5f 100755 --- a/Build.hs +++ b/Build.hs @@ -20,13 +20,12 @@ import Data.Text (Text) import Data.Text qualified as Text import Data.Text.IO qualified as Text import Control.Monad -import Control.Exception (bracket, IOException, catch) +import Control.Exception (bracket) import System.Environment import System.Directory import System.Process import System.FilePath import System.Exit -import System.IO import System.IO.Unsafe import Data.Time.Clock import Data.IORef @@ -145,49 +144,50 @@ buildGhcStage1 opts cabal ghc0 dst = do , "allow-newer: ghc-boot-th" ] - - exit_code <- - let args = [ "build" - , "--project-file=" ++ cabal_project_path - , "--builddir=" ++ builddir - , "-j" - , "--with-compiler=" ++ ghcPath ghc0 - -- the targets - , "ghc-bin:ghc" - , "ghc-pkg:ghc-pkg" - , "genprimopcode:genprimopcode" - , "deriveConstants:deriveConstants" - , "genapply:genapply" - ] - in runProcess' $ (runCabal cabal args) { env = Just stage1_env } + let build_cmd = (runCabal cabal + [ "build" + , "--project-file=" ++ cabal_project_path + , "--builddir=" ++ builddir + , "-j" + , "--with-compiler=" ++ ghcPath ghc0 + -- the targets + , "ghc-bin:ghc" + , "ghc-pkg:ghc-pkg" + , "genprimopcode:genprimopcode" + , "deriveConstants:deriveConstants" + , "genapply:genapply" + ]) + { env = Just stage1_env + } + + (exit_code, cabal_stdout, cabal_stderr) <- readCreateProcessWithExitCode build_cmd "" + writeFile (dst "cabal.stdout") cabal_stdout + writeFile (dst "cabal.stderr") cabal_stderr case exit_code of ExitSuccess -> pure () ExitFailure n -> do putStrLn $ "cabal-install failed with error code: " ++ show n + putStrLn $ "Logs can be found in \"" ++ dst ++ "/cabal.{stdout,stderr}\"" exitFailure msg " - Copying stage1 programs and generating settings to use them..." - + let listbin_cmd p = runCabal cabal + [ "list-bin" + , "--project-file=" ++ cabal_project_path + , "--with-compiler=" ++ ghcPath ghc0 + , "--builddir=" ++ builddir + , p + ] let copy_bin target bin = do - let args = - [ "list-bin" - , "--project-file=" ++ cabal_project_path - , "--with-compiler=" ++ ghcPath ghc0 - , "--builddir=" ++ builddir - , target - ] - - bin_src <- - readCreateProcess (runCabal cabal args) "" - `catch` - (\e -> do - hPutStr stderr $ "Failed to run cabal list-bin for the target: " ++ show target - hPutStr stderr $ show (e :: IOException) - exitFailure - ) - - cp bin_src (dst "bin" bin) - + (list_bin_exit_code, list_bin_stdout, list_bin_stderr) <- readCreateProcessWithExitCode (listbin_cmd target) "" + case list_bin_exit_code of + ExitSuccess + | (bin_src:_) <- lines list_bin_stdout + -> cp bin_src (dst "bin" bin) + _ -> do + putStrLn $ "Failed to run cabal list-bin for the target: " ++ show target + putStrLn list_bin_stderr + exitFailure createDirectoryIfMissing True (dst "bin") copy_bin "ghc-bin:ghc" "ghc" copy_bin "ghc-pkg:ghc-pkg" "ghc-pkg" @@ -750,12 +750,6 @@ withSystemTempDirectory prefix = do initEmptyDB :: GhcPkg -> FilePath -> IO () initEmptyDB ghcpkg pkgdb = do + -- don't try to recreate the DB if it already exist as it would fail exists <- doesDirectoryExist pkgdb unless exists $ void $ readCreateProcess (runGhcPkg ghcpkg ["init", pkgdb]) "" - -runProcess' :: CreateProcess -> IO ExitCode -runProcess' p = - withCreateProcess p { - std_out = Inherit, - std_err = Inherit - } $ \_ _ _ ph -> waitForProcess ph From 678821bb37a3af1d3ebf7d6f2fa779cb5dccf1aa Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Fri, 24 Jan 2025 11:35:45 +0100 Subject: [PATCH 064/175] Revert "Do not hardcode GHC version into Makefile" This reverts commit ebb77c582863782916b715adf409254e27d22910. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3f9ffc83f293..89c58316468b 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ export CABAL := $(shell cabal list-bin -v0 --project-dir libraries/Cabal cabal-install:exe:cabal) all: $(CABAL) - ./Build.hs + GHC=ghc-9.8.4 ./Build.hs cabal: $(CABAL) From 461b10892d45b6d4360709fa38e3c9b84edfc06a Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Fri, 24 Jan 2025 12:18:14 +0100 Subject: [PATCH 065/175] Show logs on failure when building boot libs --- Build.hs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Build.hs b/Build.hs index ebfd85752c5f..c157a0459fde 100755 --- a/Build.hs +++ b/Build.hs @@ -633,6 +633,8 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst ExitSuccess -> pure () ExitFailure r -> do putStrLn $ "Failed to build boot libraries with error code " ++ show r + putStrLn boot_stdout + putStrLn boot_stderr putStrLn $ "Logs can be found in " ++ dst ++ "boot-libs.{stdout,stderr}" exitFailure From 2ee86bda42a1b69bdfe62bf4f2147d1c709a7a3e Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Fri, 24 Jan 2025 14:22:30 +0100 Subject: [PATCH 066/175] Add xxhash.h --- rts/rts.cabal | 1 + 1 file changed, 1 insertion(+) diff --git a/rts/rts.cabal b/rts/rts.cabal index 364df2458a90..d4d2e98a0b7e 100644 --- a/rts/rts.cabal +++ b/rts/rts.cabal @@ -34,6 +34,7 @@ extra-source-files: posix/ticker/TimerFd.c -- headers files that are not installed by the rts package but only used to -- build the rts C code + xxhash.h adjustor/AdjustorPool.h Adjustor.h Apply.h From 733f9e1e663da714afea58d71c0b71371d039025 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Mon, 27 Jan 2025 11:39:35 +0100 Subject: [PATCH 067/175] Fix after merge of ghc-prim into ghc-internal --- Build.hs | 10 ++++++---- compiler/GHC/Driver/Pipeline.hs | 11 ++++++++++- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/Build.hs b/Build.hs index c157a0459fde..26b02b2a8802 100755 --- a/Build.hs +++ b/Build.hs @@ -167,6 +167,8 @@ buildGhcStage1 opts cabal ghc0 dst = do ExitSuccess -> pure () ExitFailure n -> do putStrLn $ "cabal-install failed with error code: " ++ show n + putStrLn cabal_stdout + putStrLn cabal_stderr putStrLn $ "Logs can be found in \"" ++ dst ++ "/cabal.{stdout,stderr}\"" exitFailure @@ -496,17 +498,17 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst run_genapply [derived_constants, "-V32"] (src_rts "AutoApply_V32.cmm") run_genapply [derived_constants, "-V64"] (src_rts "AutoApply_V64.cmm") - -- Generate primop code for ghc-prim + -- Generate primop code for ghc-internal -- - -- Note that this can't be done in a Setup.hs for ghc-prim because + -- Note that this can't be done in a Setup.hs for ghc-internal because -- cabal-install can't build Setup.hs because it depends on base, Cabal, etc. -- libraries that aren't built yet. let primops_txt = src "libraries/ghc/GHC/Builtin/primops.txt" let primops_txt_pp = primops_txt <.> ".pp" primops <- readCreateProcess (shell $ "gcc -E -undef -traditional -P -x c " ++ primops_txt_pp) "" writeFile primops_txt primops - writeFile (src "libraries/ghc-prim/GHC/Prim.hs") =<< readCreateProcess (runGenPrimop genprimop ["--make-haskell-source"]) primops - writeFile (src "libraries/ghc-prim/GHC/PrimopWrappers.hs") =<< readCreateProcess (runGenPrimop genprimop ["--make-haskell-wrappers"]) primops + writeFile (src "libraries/ghc-internal/src/GHC/Internal/Prim.hs") =<< readCreateProcess (runGenPrimop genprimop ["--make-haskell-source"]) primops + writeFile (src "libraries/ghc-internal/src/GHC/Internal/PrimopWrappers.hs") =<< readCreateProcess (runGenPrimop genprimop ["--make-haskell-wrappers"]) primops -- build libffi msg " - Building libffi..." diff --git a/compiler/GHC/Driver/Pipeline.hs b/compiler/GHC/Driver/Pipeline.hs index 10a0a11d39e7..81517c4b9ff6 100644 --- a/compiler/GHC/Driver/Pipeline.hs +++ b/compiler/GHC/Driver/Pipeline.hs @@ -49,6 +49,8 @@ import GHC.Platform import GHC.Utils.Monad ( MonadIO(liftIO), mapMaybeM ) +import GHC.Builtin.Names + import GHC.Driver.Main import GHC.Driver.Env hiding ( Hsc ) import GHC.Driver.Errors @@ -91,6 +93,7 @@ import GHC.Data.StringBuffer ( hPutStringBuffer ) import GHC.Data.Maybe ( expectJust ) import GHC.Iface.Make ( mkFullIface ) +import GHC.Iface.Load ( getGhcPrimIface ) import GHC.Runtime.Loader ( initializePlugins ) @@ -817,7 +820,13 @@ hscGenBackendPipeline pipe_env hsc_env mod_sum result = do let !linkable = Linkable part_time (ms_mod mod_sum) (NE.singleton (DotO final_object ModuleObject)) -- Add the object linkable to the potential bytecode linkable which was generated in HscBackend. return (mlinkable { homeMod_object = Just linkable }) - return (miface, final_linkable) + + -- when building ghc-internal with cabal-install, we still want the virtual + -- interface for gHC_PRIM in the cache + let miface_final + | ms_mod mod_sum == gHC_PRIM = getGhcPrimIface hsc_env + | otherwise = miface + return (miface_final, final_linkable) asPipeline :: P m => Bool -> PipeEnv -> HscEnv -> Maybe ModLocation -> FilePath -> m (Maybe ObjFile) asPipeline use_cpp pipe_env hsc_env location input_fn = From f92289ae112269a1d98cb2e74e87ab9dbe37f134 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Mon, 27 Jan 2025 12:01:11 +0100 Subject: [PATCH 068/175] Workaround environment file parsing --- Build.hs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Build.hs b/Build.hs index 26b02b2a8802..2382225facee 100755 --- a/Build.hs +++ b/Build.hs @@ -641,8 +641,13 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst exitFailure -- The libraries have been installed globally. - (global_gb:pkg_ids) <- (map (drop 11) . drop 2 . lines) <$> readFile boot_libs_env - putStrLn $ "We've built boot libraries in " ++ global_gb ++ ":" + -- + -- FIXME: Sometimes the package environment contains the path to the global db, + -- sometimes not... I don't know why yet. + -- (global_db:pkg_ids) <- (map (drop 11) . drop 2 . lines) <$> readFile boot_libs_env + let global_db = "~/.cabal/store/ghc-9.13-inplace/package.db" + pkg_ids <- (map (drop 11) . drop 2 . lines) <$> readFile boot_libs_env + putStrLn $ "We've built boot libraries in " ++ global_db ++ ":" mapM_ (putStrLn . (" - " ++)) pkg_ids -- TODO: copy the libs in another db From 768340e7e17aaea6b00a1b3fbd4e4819b5e74ee7 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Mon, 27 Jan 2025 14:11:52 +0100 Subject: [PATCH 069/175] Better fix to package env without package dbs... --- Build.hs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Build.hs b/Build.hs index 2382225facee..1f1f8b626987 100755 --- a/Build.hs +++ b/Build.hs @@ -641,12 +641,18 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst exitFailure -- The libraries have been installed globally. - -- + boot_libs_env_lines <- lines <$> readFile boot_libs_env -- FIXME: Sometimes the package environment contains the path to the global db, -- sometimes not... I don't know why yet. - -- (global_db:pkg_ids) <- (map (drop 11) . drop 2 . lines) <$> readFile boot_libs_env - let global_db = "~/.cabal/store/ghc-9.13-inplace/package.db" - pkg_ids <- (map (drop 11) . drop 2 . lines) <$> readFile boot_libs_env + (global_db,pkg_ids) <- case drop 2 boot_libs_env_lines of + [] -> error "Unexpected empty package environment" + (x:xs) + | not ("package-db" `List.isPrefixOf` x) + -> do + putStrLn "For some reason cabal-install didn't generate a valid package environment (package-db is missing)." + putStrLn "It happens sometimes for unknown reasons... Rerun 'make' to workaround this..." + exitFailure + | otherwise -> pure (drop 11 x, map (drop 11) xs) putStrLn $ "We've built boot libraries in " ++ global_db ++ ":" mapM_ (putStrLn . (" - " ++)) pkg_ids From 0c2ae8cdc25628483b3689dbba59092bd4de5891 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Mon, 27 Jan 2025 14:32:54 +0100 Subject: [PATCH 070/175] Prepare stage1's package db --- Build.hs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Build.hs b/Build.hs index 1f1f8b626987..7a1cf2d628c0 100755 --- a/Build.hs +++ b/Build.hs @@ -656,7 +656,18 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst putStrLn $ "We've built boot libraries in " ++ global_db ++ ":" mapM_ (putStrLn . (" - " ++)) pkg_ids - -- TODO: copy the libs in another db + -- copy the libs in another db + createDirectoryIfMissing True (dst "pkgs") + initEmptyDB ghcpkg (dst "pkgs") + let pkg_root = takeDirectory global_db + forM_ pkg_ids $ \pid -> do + conf <- Text.readFile (global_db pid <.> "conf") + -- replace full path with ${pkgroot} + Text.writeFile (dst "pkgs" pid <.> "conf") + (Text.replace (Text.pack pkg_root) "${pkgroot}" conf) + cp (pkg_root pid) (dst "pkgs" pid) + void $ readCreateProcess (runGhcPkg ghcpkg ["recache", "--package-db=" ++ (dst "pkgs")]) "" + --------------------------- From f2470741b2d836dec4d5308e849149d7de73e141 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Mon, 27 Jan 2025 16:19:31 +0100 Subject: [PATCH 071/175] Progress towards building more boot libraries and building stage2 compiler --- Build.hs | 120 +++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 103 insertions(+), 17 deletions(-) diff --git a/Build.hs b/Build.hs index 7a1cf2d628c0..180e29c775c0 100755 --- a/Build.hs +++ b/Build.hs @@ -61,12 +61,35 @@ main = do msg "Building boot libraries with stage1 compiler..." buildBootLibraries cabal ghc1 ghcPkg1 deriveConstants genapply genprimop defaultGhcBuildOptions "_build/stage1/" + -- now we copy the stage1 compiler into _build/stage1 and we generate settings + -- to use the newly installed packages. That's not what Hadrian does but it's + -- easier for us to nuke the stage1 directory to remove only stage1's built + -- libs without nuking the stage1 compiler which is slow to build. + createDirectoryIfMissing True "_build/stage1/bin" + cp "_build/stage0/bin/ghc" "_build/stage1/bin/ghc" + cp "_build/stage0/bin/ghc-pkg" "_build/stage1/bin/ghc-pkg" + createDirectoryIfMissing True "_build/stage1/lib" + cp "_build/stage0/lib/settings" "_build/stage1/lib/settings" + + msg "Building stage2 GHC program" + createDirectoryIfMissing True "_build/stage2" + ghc1' <- Ghc <$> makeAbsolute "_build/stage1/bin/ghc" + buildGhcStage2 defaultGhcBuildOptions cabal ghc1' "_build/stage2/" + msg "Done" -- | Build stage1 GHC program buildGhcStage1 :: GhcBuildOptions -> Cabal -> Ghc -> FilePath -> IO () -buildGhcStage1 opts cabal ghc0 dst = do +buildGhcStage1 = buildGhcStage True + +-- | Build stage2 GHC program +buildGhcStage2 :: GhcBuildOptions -> Cabal -> Ghc -> FilePath -> IO () +buildGhcStage2 = buildGhcStage False + +-- | Build GHC program +buildGhcStage :: Bool -> GhcBuildOptions -> Cabal -> Ghc -> FilePath -> IO () +buildGhcStage booting opts cabal ghc0 dst = do let src = dst "src" prepareGhcSources opts src @@ -94,10 +117,9 @@ buildGhcStage1 opts cabal ghc0 dst = do current_env <- getEnvironment let stage1_env = ("HADRIAN_SETTINGS", stage1_ghc_boot_settings) : current_env - msg " - Building GHC stage1 and bootstrapping utility programs..." + let cabal_project_path = dst "cabal.project-ghc" - let cabal_project_path = dst "cabal.project-stage0" - makeCabalProject cabal_project_path + let stage1_project = [ "packages:" , " " ++ src "ghc-bin/" , " " ++ src "libraries/ghc/" @@ -133,30 +155,90 @@ buildGhcStage1 opts cabal ghc0 dst = do , " executable-dynamic: False" , " executable-static: True" , "" + , "package ghc-boot-th" + , " flags: +bootstrap" + , "" + -- allow template-haskell with newer ghc-boot-th + , "allow-newer: ghc-boot-th" + , "" , "constraints:" - -- for some reason 2.23 doesn't build + -- FIXME: template-haskell 2.23 is too recent when booting with 9.8.4 , " template-haskell <= 2.22" + ] + + let stage2_project = + [ "packages:" + , " " ++ src "ghc-bin/" + , " " ++ src "libraries/deepseq/" + , " " ++ src "libraries/hpc/" + , " " ++ src "libraries/stm/" + , " " ++ src "libraries/text/" + , " " ++ src "libraries/ghc/" + , " " ++ src "libraries/directory/" + , " " ++ src "libraries/file-io/" + , " " ++ src "libraries/filepath/" + , " " ++ src "libraries/ghc-platform/" + , " " ++ src "libraries/ghc-boot/" + , " " ++ src "libraries/ghc-boot-th/" + , " " ++ src "libraries/ghc-heap" + , " " ++ src "libraries/ghci" + , " " ++ src "libraries/os-string/" + , " " ++ src "libraries/process/" + , " " ++ src "libraries/semaphore-compat" + , " " ++ src "libraries/time" + , " " ++ src "libraries/unix/" + , " " ++ src "libraries/Win32/" + , " " ++ src "utils/ghc-pkg" + , " " ++ src "utils/hsc2hs" + , " " ++ src "utils/unlit" + , " " ++ src "utils/genprimopcode/" + , " " ++ src "utils/genapply/" + , " " ++ src "utils/deriveConstants/" , "" - , "package ghc-boot-th" - , " flags: +bootstrap" + , "benchmarks: False" + , "tests: False" + , "allow-boot-library-installs: True" + -- we need even after booting because cabal thinks `template-haskell` isn't reinstallable otherwise + , "" + , "package *" + , " library-vanilla: True" + , " shared: False" + , " executable-profiling: False" + , " executable-dynamic: False" + , " executable-static: True" , "" -- allow template-haskell with newer ghc-boot-th , "allow-newer: ghc-boot-th" + , "" + , "package text" + -- FIXME: avoid having to deal with system-cxx-std-lib fake package for now + , " flags: -simdutf" + , "" ] - let build_cmd = (runCabal cabal + makeCabalProject cabal_project_path (if booting then stage1_project else stage2_project) + + -- the targets + let targets + | booting = + [ "ghc-bin:ghc" + , "ghc-pkg:ghc-pkg" + , "genprimopcode:genprimopcode" + , "deriveConstants:deriveConstants" + , "genapply:genapply" + ] + | otherwise = + [ "ghc-bin:ghc" + , "ghc-pkg:ghc-pkg" + ] + + let build_cmd = (runCabal cabal $ [ "build" , "--project-file=" ++ cabal_project_path , "--builddir=" ++ builddir , "-j" , "--with-compiler=" ++ ghcPath ghc0 - -- the targets - , "ghc-bin:ghc" - , "ghc-pkg:ghc-pkg" - , "genprimopcode:genprimopcode" - , "deriveConstants:deriveConstants" - , "genapply:genapply" - ]) + ] ++ targets) { env = Just stage1_env } @@ -169,7 +251,7 @@ buildGhcStage1 opts cabal ghc0 dst = do putStrLn $ "cabal-install failed with error code: " ++ show n putStrLn cabal_stdout putStrLn cabal_stderr - putStrLn $ "Logs can be found in \"" ++ dst ++ "/cabal.{stdout,stderr}\"" + putStrLn $ "Logs can be found in \"" ++ (dst "cabal.{stdout,stderr}\"") exitFailure msg " - Copying stage1 programs and generating settings to use them..." @@ -394,6 +476,7 @@ makeStage1Settings in_settings = out_settings , keep_fail "Use LibFFI" , keep_fail "RTS expects libdw" , ("Relative Global Package DB", "../pkgs") + -- relative to $topdir (i.e. /lib) ] buildBootLibraries :: Cabal -> Ghc -> GhcPkg -> DeriveConstants -> GenApply -> GenPrimop -> GhcBuildOptions -> FilePath -> IO () @@ -663,8 +746,11 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst forM_ pkg_ids $ \pid -> do conf <- Text.readFile (global_db pid <.> "conf") -- replace full path with ${pkgroot} + -- NOTE: GHC assumes that pkgroot is just one directory above the directory + -- containing the package db. In our case where everything is at the same + -- level in "pkgs" we need to re-add "/pkgs" Text.writeFile (dst "pkgs" pid <.> "conf") - (Text.replace (Text.pack pkg_root) "${pkgroot}" conf) + (Text.replace (Text.pack pkg_root) "${pkgroot}/pkgs/" conf) cp (pkg_root pid) (dst "pkgs" pid) void $ readCreateProcess (runGhcPkg ghcpkg ["recache", "--package-db=" ++ (dst "pkgs")]) "" From 77c6ed41d591fce969e8aae2bc7fe194ff12cb24 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Mon, 27 Jan 2025 16:37:32 +0100 Subject: [PATCH 072/175] Fix package copy --- Build.hs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Build.hs b/Build.hs index 180e29c775c0..1851e0818abf 100755 --- a/Build.hs +++ b/Build.hs @@ -751,7 +751,8 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst -- level in "pkgs" we need to re-add "/pkgs" Text.writeFile (dst "pkgs" pid <.> "conf") (Text.replace (Text.pack pkg_root) "${pkgroot}/pkgs/" conf) - cp (pkg_root pid) (dst "pkgs" pid) + cp (pkg_root pid) (dst "pkgs") + void $ readCreateProcess (runGhcPkg ghcpkg ["recache", "--package-db=" ++ (dst "pkgs")]) "" From 652d44760ed475c97b06800c65ea42a40e4cda8b Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Mon, 27 Jan 2025 17:43:06 +0100 Subject: [PATCH 073/175] Enable TNTC to avoid link issue for now We copy settings from the boot compiler and it is likely to have TNTC enabled. So just enable it too. --- Build.hs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Build.hs b/Build.hs index 1851e0818abf..debac87fe197 100755 --- a/Build.hs +++ b/Build.hs @@ -513,6 +513,12 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , def_string "GhcUnregisterised" "FIXME" , def_string "TablesNextToCode" "FIXME" , " flags: +use-system-libffi" -- FIXME: deal with libffi (add package?) + , " flags: +tables-next-to-code" + -- FIXME: we should make this optional here and in the compiler + -- settings. Ideally, GHC should even look into the rts's + -- ghcautoconf.h to check whether TABLES_NEXT_TO_CODE is defined or + -- not. It would be cleaner than duplicating this information into the + -- settings (similar to what we do with platform constants). ] makeCabalProject cabal_project_rts_path $ From c35ce295237d610f50606c96a3d7673375a399b7 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 28 Jan 2025 14:25:40 +0100 Subject: [PATCH 074/175] Workaround cabal bug with multiple flags --- Build.hs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/Build.hs b/Build.hs index debac87fe197..e8601f15f216 100755 --- a/Build.hs +++ b/Build.hs @@ -512,13 +512,20 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , def_string "TargetVendor" "FIXME" , def_string "GhcUnregisterised" "FIXME" , def_string "TablesNextToCode" "FIXME" - , " flags: +use-system-libffi" -- FIXME: deal with libffi (add package?) - , " flags: +tables-next-to-code" - -- FIXME: we should make this optional here and in the compiler - -- settings. Ideally, GHC should even look into the rts's + , " flags: +use-system-libffi +tables-next-to-code" + -- FIXME: deal with libffi (add package?) + -- + -- FIXME: we should make tables-next-to-code optional here and in the + -- compiler settings. Ideally, GHC should even look into the rts's -- ghcautoconf.h to check whether TABLES_NEXT_TO_CODE is defined or -- not. It would be cleaner than duplicating this information into the -- settings (similar to what we do with platform constants). + + -- FIXME: Cabal doesn't like when flags are on separate lines like + -- this: + -- flags: +use-system-libffi + -- flags: +tables-next-to-code + -- Apparently it makes it ignore the first set of flags... ] makeCabalProject cabal_project_rts_path $ From e25b51aad56ce28f6744f013177cd172847630cf Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 28 Jan 2025 16:37:03 +0100 Subject: [PATCH 075/175] Try adding ffi to extra-libraries-static --- Build.hs | 18 ++++++++++-------- rts/rts.cabal | 1 + 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Build.hs b/Build.hs index e8601f15f216..cb776a253162 100755 --- a/Build.hs +++ b/Build.hs @@ -254,7 +254,7 @@ buildGhcStage booting opts cabal ghc0 dst = do putStrLn $ "Logs can be found in \"" ++ (dst "cabal.{stdout,stderr}\"") exitFailure - msg " - Copying stage1 programs and generating settings to use them..." + msg " - Copying programs and generating GHC settings..." let listbin_cmd p = runCabal cabal [ "list-bin" , "--project-file=" ++ cabal_project_path @@ -273,11 +273,14 @@ buildGhcStage booting opts cabal ghc0 dst = do putStrLn list_bin_stderr exitFailure createDirectoryIfMissing True (dst "bin") - copy_bin "ghc-bin:ghc" "ghc" - copy_bin "ghc-pkg:ghc-pkg" "ghc-pkg" - copy_bin "deriveConstants:deriveConstants" "deriveConstants" - copy_bin "genprimopcode:genprimopcode" "genprimopcode" - copy_bin "genapply:genapply" "genapply" + + copy_bin "ghc-bin:ghc" "ghc" + copy_bin "ghc-pkg:ghc-pkg" "ghc-pkg" + + when booting $ do + copy_bin "deriveConstants:deriveConstants" "deriveConstants" + copy_bin "genprimopcode:genprimopcode" "genprimopcode" + copy_bin "genapply:genapply" "genapply" -- initialize empty global package database pkgdb <- makeAbsolute (dst "pkgs") @@ -708,7 +711,6 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , "--package-env=" ++ boot_libs_env , "--force-reinstalls" , "-v3" - -- [ "build" , "--project-file=" ++ cabal_project_bootlibs_path , "--with-compiler=" ++ ghcPath ghc , "--with-hc-pkg=" ++ ghcPkgPath ghcpkg @@ -763,7 +765,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst -- containing the package db. In our case where everything is at the same -- level in "pkgs" we need to re-add "/pkgs" Text.writeFile (dst "pkgs" pid <.> "conf") - (Text.replace (Text.pack pkg_root) "${pkgroot}/pkgs/" conf) + (Text.replace (Text.pack pkg_root) "${pkgroot}/pkgs" conf) cp (pkg_root pid) (dst "pkgs") void $ readCreateProcess (runGhcPkg ghcpkg ["recache", "--package-db=" ++ (dst "pkgs")]) "" diff --git a/rts/rts.cabal b/rts/rts.cabal index d4d2e98a0b7e..41d6d93041da 100644 --- a/rts/rts.cabal +++ b/rts/rts.cabal @@ -467,6 +467,7 @@ library extra-libraries: dl if flag(use-system-libffi) extra-libraries: ffi + extra-libraries-static: ffi if os(windows) extra-libraries: -- for the linker From 706168fdada2867ff3b9af3c006549c9f2130430 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 28 Jan 2025 17:14:31 +0100 Subject: [PATCH 076/175] Fix FS_NAMESPACE for the rts --- Build.hs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Build.hs b/Build.hs index cb776a253162..3e7d02efe6ec 100755 --- a/Build.hs +++ b/Build.hs @@ -497,6 +497,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst -- option into \" otherwise it does weird things (like keeping only the -- last double-quote). let def_string k v = " ghc-options: \"-optc-D" ++ k ++ "=\\\"" ++ v ++ "\\\"\"" + let def k v = " ghc-options: \"-optc-D" ++ k ++ "=" ++ v ++ "\"" let rts_options = [ "package rts" , def_string "ProjectVersion" (Text.unpack (gboVersionInt opts)) @@ -515,6 +516,8 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , def_string "TargetVendor" "FIXME" , def_string "GhcUnregisterised" "FIXME" , def_string "TablesNextToCode" "FIXME" + -- Set the namespace for the rts fs functions + , def "FS_NAMESPACE" "rts" , " flags: +use-system-libffi +tables-next-to-code" -- FIXME: deal with libffi (add package?) -- From 8aae70a0dae1db2c74cfc7509c94a14f752ba1de Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 29 Jan 2025 14:33:08 +0100 Subject: [PATCH 077/175] Enable the internal interpreter in stage2 compiler --- Build.hs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Build.hs b/Build.hs index 3e7d02efe6ec..53ab1f6eff7d 100755 --- a/Build.hs +++ b/Build.hs @@ -210,6 +210,12 @@ buildGhcStage booting opts cabal ghc0 dst = do -- allow template-haskell with newer ghc-boot-th , "allow-newer: ghc-boot-th" , "" + , "package ghc" + , " flags: +internal-interpreter" + , "" + , "package ghci" + , " flags: +internal-interpreter" + , "" , "package text" -- FIXME: avoid having to deal with system-cxx-std-lib fake package for now , " flags: -simdutf" From bb262b9c58b58499c6a538eed0f18d7c225227fe Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 29 Jan 2025 15:02:57 +0100 Subject: [PATCH 078/175] Fully enable internal-interpreter in stage2 (it works!) --- Build.hs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Build.hs b/Build.hs index 53ab1f6eff7d..04dafb8e0fd7 100755 --- a/Build.hs +++ b/Build.hs @@ -76,6 +76,9 @@ main = do ghc1' <- Ghc <$> makeAbsolute "_build/stage1/bin/ghc" buildGhcStage2 defaultGhcBuildOptions cabal ghc1' "_build/stage2/" + -- copy stage1's boot packages for stage2 to use. + cp "_build/stage1/pkgs/*" "_build/stage2/pkgs" + msg "Done" @@ -177,6 +180,8 @@ buildGhcStage booting opts cabal ghc0 dst = do , " " ++ src "libraries/directory/" , " " ++ src "libraries/file-io/" , " " ++ src "libraries/filepath/" + , " " ++ src "libraries/haskeline/" + , " " ++ src "libraries/terminfo/" , " " ++ src "libraries/ghc-platform/" , " " ++ src "libraries/ghc-boot/" , " " ++ src "libraries/ghc-boot-th/" @@ -216,6 +221,12 @@ buildGhcStage booting opts cabal ghc0 dst = do , "package ghci" , " flags: +internal-interpreter" , "" + , "package ghc-bin" + , " flags: +internal-interpreter" + , "" + , "package haskeline" + , " flags: -terminfo" -- FIXME: should be enabled but I don't have the static libs for terminfo on ArchLinux... + , "" , "package text" -- FIXME: avoid having to deal with system-cxx-std-lib fake package for now , " flags: -simdutf" From a831d5341e826f313f6f1c6b4c0467423c81e952 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Thu, 30 Jan 2025 11:42:28 +0100 Subject: [PATCH 079/175] Build ghc-toolchain --- Build.hs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Build.hs b/Build.hs index 04dafb8e0fd7..96b61f57d80a 100755 --- a/Build.hs +++ b/Build.hs @@ -146,6 +146,8 @@ buildGhcStage booting opts cabal ghc0 dst = do , " " ++ src "utils/genprimopcode/" , " " ++ src "utils/genapply/" , " " ++ src "utils/deriveConstants/" + , " " ++ src "utils/ghc-toolchain/" + , " " ++ src "utils/ghc-toolchain/exe" , "" , "benchmarks: False" , "tests: False" @@ -243,6 +245,7 @@ buildGhcStage booting opts cabal ghc0 dst = do , "genprimopcode:genprimopcode" , "deriveConstants:deriveConstants" , "genapply:genapply" + , "ghc-toolchain-bin:ghc-toolchain-bin" ] | otherwise = [ "ghc-bin:ghc" @@ -295,9 +298,10 @@ buildGhcStage booting opts cabal ghc0 dst = do copy_bin "ghc-pkg:ghc-pkg" "ghc-pkg" when booting $ do - copy_bin "deriveConstants:deriveConstants" "deriveConstants" - copy_bin "genprimopcode:genprimopcode" "genprimopcode" - copy_bin "genapply:genapply" "genapply" + copy_bin "deriveConstants:deriveConstants" "deriveConstants" + copy_bin "genprimopcode:genprimopcode" "genprimopcode" + copy_bin "genapply:genapply" "genapply" + copy_bin "ghc-toolchain-bin:ghc-toolchain-bin" "ghc-toolchain" -- initialize empty global package database pkgdb <- makeAbsolute (dst "pkgs") From dfd1b1b7747c5babb3e2c32ddfea63a78e011b05 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Fri, 31 Jan 2025 09:26:11 +0100 Subject: [PATCH 080/175] Add TODO --- rts/rts.cabal | 1 + 1 file changed, 1 insertion(+) diff --git a/rts/rts.cabal b/rts/rts.cabal index 41d6d93041da..31ebdd4eb82a 100644 --- a/rts/rts.cabal +++ b/rts/rts.cabal @@ -511,6 +511,7 @@ library cpp-options: -DNOSMP include-dirs: include + -- TODO: move internal headers into include/private? include-dirs: . includes: Rts.h autogen-includes: ghcautoconf.h ghcplatform.h From 2e5dff44cc1cc3bc0b70464aa478aab02f9206f9 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Fri, 31 Jan 2025 15:20:01 +0100 Subject: [PATCH 081/175] ghc-toolchain: allow generation of settings file instead of Target --- utils/ghc-toolchain/exe/Main.hs | 204 +++++++++++++++++++++++++++++++- 1 file changed, 202 insertions(+), 2 deletions(-) diff --git a/utils/ghc-toolchain/exe/Main.hs b/utils/ghc-toolchain/exe/Main.hs index 71faaf79a171..09f009f554a1 100644 --- a/utils/ghc-toolchain/exe/Main.hs +++ b/utils/ghc-toolchain/exe/Main.hs @@ -1,12 +1,14 @@ {-# LANGUAGE NamedFieldPuns #-} {-# LANGUAGE LambdaCase #-} {-# LANGUAGE TypeApplications #-} +{-# LANGUAGE RecordWildCards #-} module Main where import Control.Monad -import Data.Char (toUpper) +import Data.Char (toUpper,isSpace) import Data.Maybe (isNothing,fromMaybe) +import qualified Data.List as List import System.Exit import System.Console.GetOpt import System.Environment @@ -62,6 +64,7 @@ data Opts = Opts , optLdOverride :: Maybe Bool , optVerbosity :: Int , optKeepTemp :: Bool + , optOutputSettings :: Bool -- ^ Output settings file, not Target } data FormatOpts = FormatOpts @@ -107,6 +110,7 @@ emptyOpts = Opts , optLdOverride = Nothing , optVerbosity = 1 , optKeepTemp = False + , optOutputSettings = False } where po0 = emptyProgOpt @@ -148,6 +152,9 @@ _optTablesNextToCode = Lens optTablesNextToCode (\x o -> o {optTablesNextToCode= _optUseLibFFIForAdjustors = Lens optUseLibFFIForAdjustors (\x o -> o {optUseLibFFIForAdjustors=x}) _optLdOvveride = Lens optLdOverride (\x o -> o {optLdOverride=x}) +_optOutputSettings :: Lens Opts Bool +_optOutputSettings = Lens optOutputSettings (\x o -> o {optOutputSettings=x}) + _optVerbosity :: Lens Opts Int _optVerbosity = Lens optVerbosity (\x o -> o {optVerbosity=x}) @@ -161,6 +168,7 @@ options = , llvmTripleOpt , verbosityOpt , keepTempOpt + , outputSettingsOpt , outputOpt ] ++ concat @@ -233,6 +241,9 @@ options = keepTempOpt = Option [] ["keep-temp"] (NoArg (set _optKeepTemp True)) "do not remove temporary files" + outputSettingsOpt = Option [] ["output-settings"] (NoArg (set _optOutputSettings True)) + "output settings instead of Target" + outputOpt = Option ['o'] ["output"] (ReqArg (set _optOutput . Just) "OUTPUT") "The output path for the generated target toolchain configuration" @@ -309,7 +320,10 @@ run opts = do tgt <- mkTarget opts logDebug $ "Final Target: " ++ show tgt let file = fromMaybe (error "undefined --output") (optOutput opts) - writeFile file (show tgt) + let output = case optOutputSettings opts of + False -> show tgt + True -> show (targetToSettings tgt) + writeFile file output optional :: M a -> M (Maybe a) optional k = fmap Just k <|> pure Nothing @@ -493,3 +507,189 @@ mkTarget opts = do return t --- ROMES:TODO: fp_settings.m4 in general which I don't think was ported completely (e.g. the basenames and windows llvm-XX and such) + + +targetToSettings :: Target -> [(String,String)] +targetToSettings tgt@Target{..} = + [ ("C compiler command", ccPath) + , ("C compiler flags", ccFlags) + , ("C++ compiler command", cxxPath) + , ("C++ compiler flags", cxxFlags) + , ("C compiler link flags", clinkFlags) + , ("C compiler supports -no-pie", linkSupportsNoPie) + , ("CPP command", cppPath) + , ("CPP flags", cppFlags) + , ("Haskell CPP command", hsCppPath) + , ("Haskell CPP flags", hsCppFlags) + , ("JavaScript CPP command", jsCppPath) + , ("JavaScript CPP flags", jsCppFlags) + , ("C-- CPP command", cmmCppPath) + , ("C-- CPP flags", cmmCppFlags) + , ("C-- CPP supports -g0", cmmCppSupportsG0') + , ("ld supports compact unwind", linkSupportsCompactUnwind) + , ("ld supports filelist", linkSupportsFilelist) + , ("ld supports single module", linkSupportsSingleModule) + , ("ld is GNU ld", linkIsGnu) + , ("Merge objects command", mergeObjsPath) + , ("Merge objects flags", mergeObjsFlags) + , ("Merge objects supports response files", mergeObjsSupportsResponseFiles') + , ("ar command", arPath) + , ("ar flags", arFlags) + , ("ar supports at file", arSupportsAtFile') + , ("ar supports -L", arSupportsDashL') + , ("ranlib command", ranlibPath) + , ("otool command", otool_cmd) + , ("install_name_tool command", install_name_cmd) + , ("windres command", (maybe "/bin/false" prgPath tgtWindres)) -- TODO: /bin/false is not available on many distributions by default, but we keep it as it were before the ghc-toolchain patch. Fix-me. + , ("unlit command", "$topdir/../bin/unlit") -- FIXME + , ("cross compiling", yesNo False) -- FIXME: why do we need this settings at all? + , ("target platform string", targetPlatformTriple tgt) + , ("target os", (show $ archOS_OS tgtArchOs)) + , ("target arch", (show $ archOS_arch tgtArchOs)) + , ("target word size", wordSize) + , ("target word big endian", isBigEndian) + , ("target has GNU nonexec stack", (yesNo tgtSupportsGnuNonexecStack)) + , ("target has .ident directive", (yesNo tgtSupportsIdentDirective)) + , ("target has subsections via symbols", (yesNo tgtSupportsSubsectionsViaSymbols)) + , ("target has libm", has_libm) + , ("Unregisterised", (yesNo tgtUnregisterised)) + , ("LLVM target", tgtLlvmTarget) + , ("LLVM llc command", llc_cmd) + , ("LLVM opt command", llvm_opt_cmd) + , ("LLVM llvm-as command", llvm_as_cmd) + , ("Use inplace MinGW toolchain", use_inplace_mingw) + , ("target RTS linker only supports shared libraries", yesNo (targetRTSLinkerOnlySupportsSharedLibs tgt)) + , ("Use interpreter", yesNo (targetSupportsInterpreter tgt)) + , ("Support SMP", yesNo (targetSupportsSMP tgt)) + , ("RTS ways", "") -- FIXME: should be a property of the RTS, not of the target + , ("Tables next to code", (yesNo tgtTablesNextToCode)) + , ("Leading underscore", (yesNo tgtSymbolsHaveLeadingUnderscore)) + , ("Use LibFFI", yesNo tgtUseLibffiForAdjustors) + , ("RTS expects libdw", yesNo False) -- FIXME + , ("Relative Global Package DB", "") + , ("base unit-id", "") + ] + where + yesNo True = "YES" + yesNo False = "NO" + + wordSize = show (wordSize2Bytes tgtWordSize) + isBigEndian = yesNo $ (\case BigEndian -> True; LittleEndian -> False) tgtEndianness + + otool_cmd = "" -- FIXME + install_name_cmd = "" -- FIXME + has_libm = "NO" -- FIXME + llc_cmd = "llc" -- FIXME + llvm_opt_cmd = "opt" -- FIXME + llvm_as_cmd = "llvm-as" -- FIXME + use_inplace_mingw = "NO" -- FIXME + + ccPath = prgPath $ ccProgram tgtCCompiler + ccFlags = escapeArgs $ prgFlags $ ccProgram tgtCCompiler + cxxPath = prgPath $ cxxProgram tgtCxxCompiler + cxxFlags = escapeArgs $ prgFlags $ cxxProgram tgtCxxCompiler + clinkFlags = escapeArgs $ prgFlags $ ccLinkProgram tgtCCompilerLink + linkSupportsNoPie = yesNo $ ccLinkSupportsNoPie tgtCCompilerLink + cppPath = prgPath $ cppProgram tgtCPreprocessor + cppFlags = escapeArgs $ prgFlags $ cppProgram tgtCPreprocessor + hsCppPath = prgPath $ hsCppProgram tgtHsCPreprocessor + hsCppFlags = escapeArgs $ prgFlags $ hsCppProgram tgtHsCPreprocessor + jsCppPath = maybe "" (prgPath . jsCppProgram) tgtJsCPreprocessor + jsCppFlags = maybe "" (escapeArgs . prgFlags . jsCppProgram) tgtJsCPreprocessor + cmmCppPath = prgPath $ cmmCppProgram tgtCmmCPreprocessor + cmmCppFlags = escapeArgs $ prgFlags $ cmmCppProgram tgtCmmCPreprocessor + cmmCppSupportsG0' = yesNo $ cmmCppSupportsG0 tgtCmmCPreprocessor + mergeObjsPath = maybe "" (prgPath . mergeObjsProgram) tgtMergeObjs + mergeObjsFlags = maybe "" (escapeArgs . prgFlags . mergeObjsProgram) tgtMergeObjs + linkSupportsSingleModule = yesNo $ ccLinkSupportsSingleModule tgtCCompilerLink + linkSupportsFilelist = yesNo $ ccLinkSupportsFilelist tgtCCompilerLink + linkSupportsCompactUnwind = yesNo $ ccLinkSupportsCompactUnwind tgtCCompilerLink + linkIsGnu = yesNo $ ccLinkIsGnu tgtCCompilerLink + arPath = prgPath $ arMkArchive tgtAr + arFlags = escapeArgs $ prgFlags (arMkArchive tgtAr) + arSupportsAtFile' = yesNo (arSupportsAtFile tgtAr) + arSupportsDashL' = yesNo (arSupportsDashL tgtAr) + ranlibPath = maybe "" (prgPath . ranlibProgram) tgtRanlib + mergeObjsSupportsResponseFiles' = maybe "NO" (yesNo . mergeObjsSupportsResponseFiles) tgtMergeObjs + +-- | Just like 'GHC.ResponseFile.escapeArgs', but use spaces instead of newlines +-- for splitting elements. +escapeArgs :: [String] -> String +escapeArgs = unwords . map escapeArg + +escapeArg :: String -> String +escapeArg = reverse . List.foldl' escape [] + +escape :: String -> Char -> String +escape cs c + | isSpace c + || '\\' == c + || '\'' == c + || '"' == c = c:'\\':cs -- n.b., our caller must reverse the result + | otherwise = c:cs + +-- | Does the target support the -N RTS flag? +-- +-- Adapated from hadrian: Oracles.Flag.targetSupportsSMP +targetSupportsSMP :: Target -> Bool +targetSupportsSMP Target{..} = case archOS_arch tgtArchOs of + -- The THREADED_RTS requires `BaseReg` to be in a register and the + -- Unregisterised mode doesn't allow that. + _ | tgtUnregisterised -> False + ArchARM isa _ _ + -- We don't support load/store barriers pre-ARMv7. See #10433. + | isa < ARMv7 -> False + | otherwise -> True + ArchX86 -> True + ArchX86_64 -> True + ArchPPC -> True + ArchPPC_64 ELF_V1 -> True + ArchPPC_64 ELF_V2 -> True + ArchAArch64 -> True + ArchS390X -> True + ArchRISCV64 -> True + ArchLoongArch64 -> True + ArchAArch64 -> True + _ -> False + + + +-- | Check whether the target supports GHCi. +-- +-- Adapted from hadrian:Oracles.Settings.ghcWithInterpreter +targetSupportsInterpreter :: Target -> Bool +targetSupportsInterpreter Target{..} = goodOs && goodArch + where + goodOs = case archOS_OS tgtArchOs of + OSMinGW32 -> True + OSLinux -> True + OSSolaris2 -> True + OSFreeBSD -> True + OSDragonFly -> True + OSNetBSD -> True + OSOpenBSD -> True + OSDarwin -> True + OSKFreeBSD -> True + OSWasi -> True + _ -> False + -- TODO "cygwin32"? + + goodArch = case archOS_arch tgtArchOs of + ArchX86 -> True + ArchX86_64 -> True + ArchPPC -> True + ArchAArch64 -> True + ArchS390X -> True + ArchPPC_64 ELF_V1 -> True + ArchPPC_64 ELF_V2 -> True + ArchRISCV64 -> True + ArchWasm32 -> True + ArchAArch64 -> True + ArchARM {} -> True + _ -> False + + +targetRTSLinkerOnlySupportsSharedLibs :: Target -> Bool +targetRTSLinkerOnlySupportsSharedLibs tgt = case archOS_arch (tgtArchOs tgt) of + ArchWasm32 -> True + _ -> False From ae8d29f9e982cadd3d8eccf180c635d2ad02c48d Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Fri, 31 Jan 2025 15:31:02 +0100 Subject: [PATCH 082/175] Factor ghcTargetArchOS --- Build.hs | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/Build.hs b/Build.hs index 96b61f57d80a..dfddaf064ad1 100755 --- a/Build.hs +++ b/Build.hs @@ -52,6 +52,8 @@ main = do msg "Building stage1 GHC program and utility programs" buildGhcStage1 defaultGhcBuildOptions cabal ghc0 "_build/stage0/" + + ghc1 <- Ghc <$> makeAbsolute "_build/stage0/bin/ghc" ghcPkg1 <- GhcPkg <$> makeAbsolute "_build/stage0/bin/ghc-pkg" deriveConstants <- DeriveConstants <$> makeAbsolute "_build/stage0/bin/deriveConstants" @@ -101,13 +103,13 @@ buildGhcStage booting opts cabal ghc0 dst = do -- we need to augment the current environment to pass HADRIAN_SETTINGS -- environment variable to ghc-boot's Setup.hs script. - stage0_settings <- read <$> readCreateProcess (runGhc ghc0 ["--info"]) "" + (arch,os) <- ghcTargetArchOS ghc0 stage1_ghc_boot_settings <- do commit_id <- readCreateProcess (proc "git" ["rev-parse", "HEAD"]) "" -- we infer stage1's host platform from stage0's settings let settings = - [ ("hostPlatformArch", fromMaybe (error "Couldn't read 'target arch' setting") (lookup "target arch" stage0_settings)) - , ("hostPlatformOS", fromMaybe (error "Couldn't read 'target os' setting") (lookup "target os" stage0_settings)) + [ ("hostPlatformArch", arch) + , ("hostPlatformOS", os) , ("cProjectGitCommitId", commit_id) , ("cProjectVersion", Text.unpack $ gboVersion opts) , ("cProjectVersionInt", Text.unpack $ gboVersionInt opts) @@ -308,20 +310,12 @@ buildGhcStage booting opts cabal ghc0 dst = do ghcpkg <- GhcPkg <$> makeAbsolute (dst "bin/ghc-pkg") initEmptyDB ghcpkg pkgdb - -- generate settings based on stage1 compiler settings createDirectoryIfMissing True (dst "lib") + stage0_settings <- read <$> readCreateProcess (runGhc ghc0 ["--info"]) "" let stage1_settings = makeStage1Settings stage0_settings writeFile (dst "lib/settings") (show stage1_settings) - -- try to run the stage1 compiler (no package db yet, so just display the - -- version) - (test_exit_code, _test_stdout, _test_stderr) <- readCreateProcessWithExitCode (proc (dst "bin/ghc") ["--version"]) "" - case test_exit_code of - ExitSuccess -> pure () - ExitFailure n -> do - putStrLn $ "Failed to run stage1 compiler with error code " ++ show n - exitFailure -- TODO: @@ -905,3 +899,11 @@ initEmptyDB ghcpkg pkgdb = do -- don't try to recreate the DB if it already exist as it would fail exists <- doesDirectoryExist pkgdb unless exists $ void $ readCreateProcess (runGhcPkg ghcpkg ["init", pkgdb]) "" + +-- | Retrieve GHC's target arch/os from ghc --info +ghcTargetArchOS :: Ghc -> IO (String,String) +ghcTargetArchOS ghc = do + is <- read <$> readCreateProcess (runGhc ghc ["--info"]) "" :: IO [(String,String)] + let arch = fromMaybe (error "Couldn't read 'target arch' setting") (lookup "target arch" is) + let os = fromMaybe (error "Couldn't read 'target os' setting") (lookup "target os" is) + pure (arch,os) From 92472a29983b390675c48d232a7f1becb94e4545 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Fri, 31 Jan 2025 16:40:14 +0100 Subject: [PATCH 083/175] Use ghc-toolchain to generate settings --- Build.hs | 233 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 135 insertions(+), 98 deletions(-) diff --git a/Build.hs b/Build.hs index dfddaf064ad1..6291f91b74a6 100755 --- a/Build.hs +++ b/Build.hs @@ -6,6 +6,7 @@ {-# LANGUAGE DerivingStrategies #-} {-# LANGUAGE GeneralizedNewtypeDeriving #-} {-# LANGUAGE NumericUnderscores #-} +{-# LANGUAGE RecordWildCards #-} {-# OPTIONS_GHC -Wall #-} -- | GHC builder @@ -16,6 +17,8 @@ module Main where import Data.Maybe import Data.List qualified as List +import Data.Map qualified as Map +import Data.Map (Map) import Data.Text (Text) import Data.Text qualified as Text import Data.Text.IO qualified as Text @@ -52,13 +55,21 @@ main = do msg "Building stage1 GHC program and utility programs" buildGhcStage1 defaultGhcBuildOptions cabal ghc0 "_build/stage0/" - - ghc1 <- Ghc <$> makeAbsolute "_build/stage0/bin/ghc" ghcPkg1 <- GhcPkg <$> makeAbsolute "_build/stage0/bin/ghc-pkg" deriveConstants <- DeriveConstants <$> makeAbsolute "_build/stage0/bin/deriveConstants" genapply <- GenApply <$> makeAbsolute "_build/stage0/bin/genapply" genprimop <- GenPrimop <$> makeAbsolute "_build/stage0/bin/genprimopcode" + ghcToolchain <- GhcToolchain <$> makeAbsolute "_build/stage0/bin/ghc-toolchain" + + -- generate settings based on stage1 compiler settings: stage1 should never be + -- a cross-compiler! Hence we reuse the same target platform as the bootstrap + -- compiler. + stage0_target_triple <- ghcTargetTriple ghc0 + let stage1_settings = emptySettings + { settingsTriple = Just stage0_target_triple + } + generateSettings ghcToolchain stage1_settings "_build/stage0/" msg "Building boot libraries with stage1 compiler..." buildBootLibraries cabal ghc1 ghcPkg1 deriveConstants genapply genprimop defaultGhcBuildOptions "_build/stage1/" @@ -70,6 +81,7 @@ main = do createDirectoryIfMissing True "_build/stage1/bin" cp "_build/stage0/bin/ghc" "_build/stage1/bin/ghc" cp "_build/stage0/bin/ghc-pkg" "_build/stage1/bin/ghc-pkg" + cp "_build/stage0/bin/unlit" "_build/stage1/bin/unlit" createDirectoryIfMissing True "_build/stage1/lib" cp "_build/stage0/lib/settings" "_build/stage1/lib/settings" @@ -78,8 +90,15 @@ main = do ghc1' <- Ghc <$> makeAbsolute "_build/stage1/bin/ghc" buildGhcStage2 defaultGhcBuildOptions cabal ghc1' "_build/stage2/" - -- copy stage1's boot packages for stage2 to use. + -- Reuse stage1 settings for stage2 and copy stage1's built boot package for + -- stage2 to use. + -- TODO: in the future we want to generate different settings for cross + -- targets and build boot libraries with stage2 using these settings. In any + -- case, we need non-cross boot packages to build plugins for use with + -- -fplugin-library. + createDirectoryIfMissing True "_build/stage2/lib/" cp "_build/stage1/pkgs/*" "_build/stage2/pkgs" + cp "_build/stage1/lib/settings" "_build/stage2/lib/settings" msg "Done" @@ -248,10 +267,12 @@ buildGhcStage booting opts cabal ghc0 dst = do , "deriveConstants:deriveConstants" , "genapply:genapply" , "ghc-toolchain-bin:ghc-toolchain-bin" + , "unlit:unlit" ] | otherwise = [ "ghc-bin:ghc" , "ghc-pkg:ghc-pkg" + , "unlit:unlit" ] let build_cmd = (runCabal cabal $ @@ -298,6 +319,7 @@ buildGhcStage booting opts cabal ghc0 dst = do copy_bin "ghc-bin:ghc" "ghc" copy_bin "ghc-pkg:ghc-pkg" "ghc-pkg" + copy_bin "unlit:unlit" "unlit" when booting $ do copy_bin "deriveConstants:deriveConstants" "deriveConstants" @@ -310,12 +332,6 @@ buildGhcStage booting opts cabal ghc0 dst = do ghcpkg <- GhcPkg <$> makeAbsolute (dst "bin/ghc-pkg") initEmptyDB ghcpkg pkgdb - -- generate settings based on stage1 compiler settings - createDirectoryIfMissing True (dst "lib") - stage0_settings <- read <$> readCreateProcess (runGhc ghc0 ["--info"]) "" - let stage1_settings = makeStage1Settings stage0_settings - writeFile (dst "lib/settings") (show stage1_settings) - -- TODO: @@ -408,95 +424,6 @@ prepareGhcSources opts dst = do subst_in (dst "libraries/rts/include/ghcversion.h") common_substs --- | Generate settings for stage1 compiler, based on given settings (stage0's --- compiler settings) -makeStage1Settings :: [(String,String)] -> [(String,String)] -makeStage1Settings in_settings = out_settings - where - -- keep the previous setting, fail if it doesn't exist - keep_fail s = keep_def s (error ("Couldn't find setting "<> show s)) - - -- keep the previous setting, default to the given value if it doesn't exist - keep_def s d = case lookup s in_settings of - Nothing -> (s,d) - Just v -> (s,v) - - -- use the previous setting, or if it doesn't exist use the setting for the - -- second key. Fail if both don't exist. This is useful to support - -- bootstrapping with old compilers that mingled some settings. - keep_or_fail s s2 = case lookup s in_settings of - Nothing -> case lookup s2 in_settings of - Nothing -> error ("Couldn't find any of " <> show s <> " and " <> show s2) - Just v -> (s,v) - Just v -> (s,v) - - --FIXME: we default to these flags for Cmm CPP, otherwise CPP fails - -- with error: missing '(' after "__has_feature" - -- because we pass `-traditional` while compiling Apply.cmm (in TSANUtils.h) - default_cpp_flags = "-E" - - out_settings = - [ keep_fail "C compiler command" - , keep_fail "C compiler flags" - , keep_fail "C++ compiler command" - , keep_fail "C++ compiler flags" - , keep_fail "C compiler link flags" - , keep_fail "C compiler supports -no-pie" - , keep_or_fail "CPP command" "Haskell CPP command" - , keep_def "CPP flags" default_cpp_flags - , keep_fail "Haskell CPP command" - , keep_fail "Haskell CPP flags" - , keep_or_fail "JavaScript CPP command" "Haskell CPP command" - , keep_or_fail "JavaScript CPP flags" "Haskell CPP flags" - , keep_or_fail "C-- CPP command" "Haskell CPP command" - , keep_def "C-- CPP flags" default_cpp_flags - , keep_def "C-- CPP supports -g0" "NO" - , keep_fail "ld supports compact unwind" - , keep_fail "ld supports filelist" - , keep_fail "ld supports single module" - , keep_fail "ld is GNU ld" - , keep_fail "Merge objects command" - , keep_fail "Merge objects flags" - , keep_def "Merge objects supports response files" "NO" - , keep_fail "ar command" - , keep_fail "ar flags" - , keep_fail "ar supports at file" - , keep_fail "ar supports -L" - , keep_fail "ranlib command" - , keep_fail "otool command" - , keep_fail "install_name_tool command" - , keep_fail "windres command" - , keep_fail "unlit command" - , keep_fail "cross compiling" - , keep_fail "target platform string" - , keep_fail "target os" - , keep_fail "target arch" - , keep_fail "target word size" - , keep_fail "target word big endian" - , keep_fail "target has GNU nonexec stack" - , keep_fail "target has .ident directive" - , keep_fail "target has subsections via symbols" - , keep_fail "target has libm" - , keep_fail "Unregisterised" - , keep_fail "LLVM target" - , keep_fail "LLVM llc command" - , keep_fail "LLVM opt command" - , keep_def "LLVM llvm-as command" "llvm-as" - , keep_fail "Use inplace MinGW toolchain" - - , keep_def "target RTS linker only supports shared libraries" "NO" - , ("Use interpreter", "NO") - , ("base unit-id", "base") -- there is no base yet... Anyway this isn't really useful to set - , keep_fail "Support SMP" - , keep_fail "RTS ways" - , keep_fail "Tables next to code" - , keep_fail "Leading underscore" - , keep_fail "Use LibFFI" - , keep_fail "RTS expects libdw" - , ("Relative Global Package DB", "../pkgs") - -- relative to $topdir (i.e. /lib) - ] - buildBootLibraries :: Cabal -> Ghc -> GhcPkg -> DeriveConstants -> GenApply -> GenPrimop -> GhcBuildOptions -> FilePath -> IO () buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst = do src <- makeAbsolute (dst "src") @@ -841,6 +768,7 @@ msg x = do -- Avoid FilePath blindness by using type aliases for programs. newtype Ghc = Ghc FilePath newtype GhcPkg = GhcPkg FilePath +newtype GhcToolchain = GhcToolchain FilePath newtype Cabal = Cabal FilePath newtype DeriveConstants = DeriveConstants FilePath newtype GenApply = GenApply FilePath @@ -855,6 +783,9 @@ ghcPath (Ghc x) = x runGhcPkg :: GhcPkg -> [String] -> CreateProcess runGhcPkg (GhcPkg f) = proc f +runGhcToolchain :: GhcToolchain -> [String] -> CreateProcess +runGhcToolchain (GhcToolchain f) = proc f + ghcPkgPath :: GhcPkg -> FilePath ghcPkgPath (GhcPkg x) = x @@ -907,3 +838,109 @@ ghcTargetArchOS ghc = do let arch = fromMaybe (error "Couldn't read 'target arch' setting") (lookup "target arch" is) let os = fromMaybe (error "Couldn't read 'target os' setting") (lookup "target os" is) pure (arch,os) + +-- | Retrieve GHC's target triple +ghcTargetTriple :: Ghc -> IO String +ghcTargetTriple ghc = do + is <- read <$> readCreateProcess (runGhc ghc ["--info"]) "" :: IO [(String,String)] + pure $ fromMaybe (error "Couldn't read 'Target platform setting") (lookup "Target platform" is) + + +data Settings = Settings + { settingsTriple :: Maybe String + , settingsTargetPrefix :: Maybe String + , settingsLocallyExecutable :: Maybe Bool + , settingsLlvmTriple :: Maybe String + , settingsCc :: ProgOpt + , settingsCxx :: ProgOpt + , settingsCpp :: ProgOpt + , settingsHsCpp :: ProgOpt + , settingsJsCpp :: ProgOpt + , settingsCmmCpp :: ProgOpt + , settingsCcLink :: ProgOpt + , settingsAr :: ProgOpt + , settingsRanlib :: ProgOpt + , settingsNm :: ProgOpt + , settingsReadelf :: ProgOpt + , settingsMergeObjs :: ProgOpt + , settingsWindres :: ProgOpt + -- Note we don't actually configure LD into anything but + -- see #23857 and #22550 for the very unfortunate story. + , settingsLd :: ProgOpt + , settingsUnregisterised :: Maybe Bool + , settingsTablesNextToCode :: Maybe Bool + , settingsUseLibFFIForAdjustors :: Maybe Bool + , settingsLdOverride :: Maybe Bool + } + +-- | Program specifier from the command-line. +data ProgOpt = ProgOpt + { poPath :: Maybe String + -- ^ Refers to the path to an executable, or simply the + -- executable name. + , poFlags :: Maybe [String] + } + +emptyProgOpt :: ProgOpt +emptyProgOpt = ProgOpt Nothing Nothing + +emptySettings :: Settings +emptySettings = Settings + { settingsTriple = Nothing + , settingsTargetPrefix = Nothing + , settingsLocallyExecutable = Nothing + , settingsLlvmTriple = Nothing + , settingsCc = po0 + , settingsCxx = po0 + , settingsCpp = po0 + , settingsHsCpp = po0 + , settingsJsCpp = po0 + , settingsCmmCpp = po0 + , settingsCcLink = po0 + , settingsAr = po0 + , settingsRanlib = po0 + , settingsNm = po0 + , settingsReadelf = po0 + , settingsMergeObjs = po0 + , settingsWindres = po0 + , settingsLd = po0 + , settingsUnregisterised = Nothing + , settingsTablesNextToCode = Nothing + , settingsUseLibFFIForAdjustors = Nothing + , settingsLdOverride = Nothing + } + where + po0 = emptyProgOpt + +generateSettings :: GhcToolchain -> Settings -> FilePath -> IO () +generateSettings ghc_toolchain Settings{..} dst = do + createDirectoryIfMissing True (dst "lib") + + let gen_settings_path = dst "lib/settings.generated" + let common_args = + [ "--output-settings" + , "-o", gen_settings_path + ] + + let opt m f = fmap f m + let args = mconcat (catMaybes + [ opt settingsTriple $ \x -> ["--triple", x] + -- FIXME: add other options for ghc-toolchain from Settings + ]) ++ common_args + + (exit_code, toolchain_stdout, toolchain_stderr) <- readCreateProcessWithExitCode (runGhcToolchain ghc_toolchain args) "" + writeFile (dst "ghc-toolchain.stdout") toolchain_stdout + writeFile (dst "ghc-toolchain.stderr") toolchain_stderr + case exit_code of + ExitSuccess -> pure () + ExitFailure n -> do + putStrLn $ "ghc-toolchain failed with error code: " ++ show n + putStrLn toolchain_stdout + putStrLn toolchain_stderr + putStrLn $ "Logs can be found in \"" ++ (dst "ghc-toolchain.{stdout,stderr}\"") + exitFailure + + -- fixup settings generated by ghc-toolchain + kvs <- (Map.fromList . read) <$> readFile gen_settings_path :: IO (Map String String) + let kvs' = Map.insert "Relative Global Package DB" "../pkgs" kvs + writeFile (dst "lib/settings") (show $ Map.toList kvs') From 0b8e2179fc2e2d4ad5d5a7674147271e36be014b Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Mon, 3 Feb 2025 17:30:31 +0100 Subject: [PATCH 084/175] Build auxiliary programs to allow running the testsuite --- Build.hs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Build.hs b/Build.hs index 6291f91b74a6..f9d6165f432d 100755 --- a/Build.hs +++ b/Build.hs @@ -218,6 +218,8 @@ buildGhcStage booting opts cabal ghc0 dst = do , " " ++ src "libraries/Win32/" , " " ++ src "utils/ghc-pkg" , " " ++ src "utils/hsc2hs" + , " " ++ src "utils/hp2ps" + , " " ++ src "utils/hpc" , " " ++ src "utils/unlit" , " " ++ src "utils/genprimopcode/" , " " ++ src "utils/genapply/" @@ -268,11 +270,15 @@ buildGhcStage booting opts cabal ghc0 dst = do , "genapply:genapply" , "ghc-toolchain-bin:ghc-toolchain-bin" , "unlit:unlit" + , "hsc2hs:hsc2hs" ] | otherwise = [ "ghc-bin:ghc" , "ghc-pkg:ghc-pkg" , "unlit:unlit" + , "hsc2hs:hsc2hs" + , "hp2ps:hp2ps" + , "hpc-bin:hpc" ] let build_cmd = (runCabal cabal $ @@ -320,6 +326,11 @@ buildGhcStage booting opts cabal ghc0 dst = do copy_bin "ghc-bin:ghc" "ghc" copy_bin "ghc-pkg:ghc-pkg" "ghc-pkg" copy_bin "unlit:unlit" "unlit" + copy_bin "hsc2hs:hsc2hs" "hsc2hs" + + unless booting $ do + copy_bin "hp2ps:hp2ps" "hp2ps" + copy_bin "hpc-bin:hpc" "hpc" when booting $ do copy_bin "deriveConstants:deriveConstants" "deriveConstants" From 871d63595f53d8d27cff1e6ab12bf4d6e952bc5a Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Mon, 3 Feb 2025 17:35:24 +0100 Subject: [PATCH 085/175] Build shallow libraries on top of ghc-internal --- Build.hs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Build.hs b/Build.hs index f9d6165f432d..cb433bf10052 100755 --- a/Build.hs +++ b/Build.hs @@ -602,6 +602,8 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , " " ++ src "libraries/base" , " " ++ src "libraries/ghc" , " " ++ src "libraries/ghc-platform/" + , " " ++ src "libraries/ghc-bignum/" + , " " ++ src "libraries/integer-gmp/" , " " ++ src "libraries/ghc-boot/" , " " ++ src "libraries/ghc-boot-th/" , " " ++ src "libraries/ghc-heap" @@ -676,9 +678,12 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst -- targets , "rts" - , "ghc-prim" , "ghc-internal" , "base" + -- shallow compat packages over ghc-internal + , "ghc-prim" + , "ghc-bignum" + , "integer-gmp" ] msg " - Building boot libraries..." From 83b45c98fd8b133fe5223e0e516c76e892a56802 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Mon, 3 Feb 2025 17:46:19 +0100 Subject: [PATCH 086/175] Try to add template-haskell and explain what's failing --- Build.hs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Build.hs b/Build.hs index cb433bf10052..020d12c78e3c 100755 --- a/Build.hs +++ b/Build.hs @@ -684,6 +684,11 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , "ghc-prim" , "ghc-bignum" , "integer-gmp" + -- , "template-haskell" -- FIXME: adding this fails because cabal runs + -- ghc for some reason, without having the rts built (obviously, we're + -- building it!), hence it crashes because ghc can't find + -- `ghcversion.h`... which is the in the rts package it hasn't built yet + -- ffs! ] msg " - Building boot libraries..." From 7c5fdd8a2c152626308006758a0aed74b5bb6d21 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 4 Feb 2025 10:06:24 +0100 Subject: [PATCH 087/175] Allow hsc2hs to find template-hsc.h in GHC's lib directory --- Build.hs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Build.hs b/Build.hs index 020d12c78e3c..53d978f28074 100755 --- a/Build.hs +++ b/Build.hs @@ -84,6 +84,7 @@ main = do cp "_build/stage0/bin/unlit" "_build/stage1/bin/unlit" createDirectoryIfMissing True "_build/stage1/lib" cp "_build/stage0/lib/settings" "_build/stage1/lib/settings" + cp "_build/stage0/lib/template-hsc.h" "_build/stage1/lib/template-hsc.h" msg "Building stage2 GHC program" createDirectoryIfMissing True "_build/stage2" @@ -183,6 +184,9 @@ buildGhcStage booting opts cabal ghc0 dst = do , "" , "package ghc-boot-th" , " flags: +bootstrap" + , "" + , "package hcs2hs" + , " flags: +in-ghc-tree" -- allow finding template-hsc.h in GHC's /lib , "" -- allow template-haskell with newer ghc-boot-th , "allow-newer: ghc-boot-th" @@ -249,6 +253,9 @@ buildGhcStage booting opts cabal ghc0 dst = do , "package ghc-bin" , " flags: +internal-interpreter" , "" + , "package hcs2hs" + , " flags: +in-ghc-tree" -- allow finding template-hsc.h in GHC's /lib + , "" , "package haskeline" , " flags: -terminfo" -- FIXME: should be enabled but I don't have the static libs for terminfo on ArchLinux... , "" @@ -327,6 +334,8 @@ buildGhcStage booting opts cabal ghc0 dst = do copy_bin "ghc-pkg:ghc-pkg" "ghc-pkg" copy_bin "unlit:unlit" "unlit" copy_bin "hsc2hs:hsc2hs" "hsc2hs" + createDirectoryIfMissing True (dst "lib") + cp (src "utils/hsc2hs/data/template-hsc.h") (dst "lib/template-hsc.h") unless booting $ do copy_bin "hp2ps:hp2ps" "hp2ps" From 88e70bdf988a521c5f4d7351e702bee4f271e92f Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 4 Feb 2025 10:51:31 +0100 Subject: [PATCH 088/175] Use custom version of hsc2hs with debug print --- .gitmodules | 2 +- utils/hsc2hs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 01d9e8110ed1..6bfcda55ba81 100644 --- a/.gitmodules +++ b/.gitmodules @@ -98,7 +98,7 @@ ignore = untracked [submodule "utils/hsc2hs"] path = utils/hsc2hs - url = https://gitlab.haskell.org/ghc/hsc2hs.git + url = https://github.com/hsyl20/hsc2hs.git ignore = untracked [submodule "libffi-tarballs"] path = libffi-tarballs diff --git a/utils/hsc2hs b/utils/hsc2hs index c3b21800a673..df910e92de53 160000 --- a/utils/hsc2hs +++ b/utils/hsc2hs @@ -1 +1 @@ -Subproject commit c3b21800a67366c9591dc85a471d1dfdb1efcf29 +Subproject commit df910e92de531bc63670a0c531d01cae7f155d3a From daf7768638d6e18eb4f9833cf92fe5feb0764737 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 4 Feb 2025 11:40:23 +0100 Subject: [PATCH 089/175] Fix typo... --- Build.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Build.hs b/Build.hs index 53d978f28074..0ea6374b9bae 100755 --- a/Build.hs +++ b/Build.hs @@ -185,7 +185,7 @@ buildGhcStage booting opts cabal ghc0 dst = do , "package ghc-boot-th" , " flags: +bootstrap" , "" - , "package hcs2hs" + , "package hsc2hs" , " flags: +in-ghc-tree" -- allow finding template-hsc.h in GHC's /lib , "" -- allow template-haskell with newer ghc-boot-th @@ -253,7 +253,7 @@ buildGhcStage booting opts cabal ghc0 dst = do , "package ghc-bin" , " flags: +internal-interpreter" , "" - , "package hcs2hs" + , "package hsc2hs" , " flags: +in-ghc-tree" -- allow finding template-hsc.h in GHC's /lib , "" , "package haskeline" From 9c0efcfe365b7e599cd7ad8de56dd4739abb2fc0 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 4 Feb 2025 12:16:16 +0100 Subject: [PATCH 090/175] Revert "Use custom version of hsc2hs with debug print" This reverts commit 88e70bdf988a521c5f4d7351e702bee4f271e92f. --- .gitmodules | 2 +- utils/hsc2hs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 6bfcda55ba81..01d9e8110ed1 100644 --- a/.gitmodules +++ b/.gitmodules @@ -98,7 +98,7 @@ ignore = untracked [submodule "utils/hsc2hs"] path = utils/hsc2hs - url = https://github.com/hsyl20/hsc2hs.git + url = https://gitlab.haskell.org/ghc/hsc2hs.git ignore = untracked [submodule "libffi-tarballs"] path = libffi-tarballs diff --git a/utils/hsc2hs b/utils/hsc2hs index df910e92de53..c3b21800a673 160000 --- a/utils/hsc2hs +++ b/utils/hsc2hs @@ -1 +1 @@ -Subproject commit df910e92de531bc63670a0c531d01cae7f155d3a +Subproject commit c3b21800a67366c9591dc85a471d1dfdb1efcf29 From 4ac8ec7a34da254ba465d9d9c13970d700ec618c Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 4 Feb 2025 14:30:33 +0100 Subject: [PATCH 091/175] GHC: support compilation without ghcversion.h --- compiler/GHC/Driver/Pipeline/Execute.hs | 4 +- compiler/GHC/SysTools/Cpp.hs | 54 +++++++++++++++---------- 2 files changed, 35 insertions(+), 23 deletions(-) diff --git a/compiler/GHC/Driver/Pipeline/Execute.hs b/compiler/GHC/Driver/Pipeline/Execute.hs index 5ba39e4bb234..4f72eff1b737 100644 --- a/compiler/GHC/Driver/Pipeline/Execute.hs +++ b/compiler/GHC/Driver/Pipeline/Execute.hs @@ -478,7 +478,7 @@ runCcPhase cc_phase pipe_env hsc_env location input_fn = do -- very weakly typed, being derived from C--. ["-fno-strict-aliasing"] - ghcVersionH <- getGhcVersionPathName dflags unit_env + include_ghcVersionH <- getGhcVersionIncludeFlags dflags unit_env withAtomicRename output_fn $ \temp_outputFilename -> GHC.SysTools.runCc (phaseForeignLanguage cc_phase) logger tmpfs dflags ( @@ -523,7 +523,7 @@ runCcPhase cc_phase pipe_env hsc_env location input_fn = do else []) ++ verbFlags ++ cc_opt - ++ [ "-include", ghcVersionH ] + ++ include_ghcVersionH ++ framework_paths ++ include_paths ++ pkg_extra_cc_opts diff --git a/compiler/GHC/SysTools/Cpp.hs b/compiler/GHC/SysTools/Cpp.hs index 4237c6c60360..2f7f60afb92e 100644 --- a/compiler/GHC/SysTools/Cpp.hs +++ b/compiler/GHC/SysTools/Cpp.hs @@ -7,6 +7,7 @@ module GHC.SysTools.Cpp ( doCpp , CppOpts(..) , getGhcVersionPathName + , getGhcVersionIncludeFlags , applyCDefs , offsetIncludePaths ) @@ -175,8 +176,7 @@ doCpp logger tmpfs dflags unit_env opts input_fn output_fn = do let asserts_def = [ "-D__GLASGOW_HASKELL_ASSERTS_IGNORED__" | gopt Opt_IgnoreAsserts dflags] -- Default CPP defines in Haskell source - ghcVersionH <- getGhcVersionPathName dflags unit_env - let hsSourceCppOpts = [ "-include", ghcVersionH ] + hsSourceCppOpts <- getGhcVersionIncludeFlags dflags unit_env -- MIN_VERSION macros let uids = explicitUnits unit_state @@ -262,28 +262,40 @@ generateMacros prefix name version = _ -> error "take3" (major1,major2,minor) = take3 $ map show (versionBranch version) ++ repeat "0" +getGhcVersionIncludeFlags :: DynFlags -> UnitEnv -> IO [String] +getGhcVersionIncludeFlags dflags unit_env = do + mghcversionh <- getGhcVersionPathName dflags unit_env + pure $ case mghcversionh of + Nothing -> [] + Just p -> ["-include", p] -- | Find out path to @ghcversion.h@ file -getGhcVersionPathName :: DynFlags -> UnitEnv -> IO FilePath +getGhcVersionPathName :: DynFlags -> UnitEnv -> IO (Maybe FilePath) getGhcVersionPathName dflags unit_env = do - let candidates = case ghcVersionFile dflags of - -- the user has provided an explicit `ghcversion.h` file to use. - Just path -> [path] - -- otherwise, try to find it in the rts' include-dirs. - -- Note: only in the RTS include-dirs! not all preload units less we may - -- use a wrong file. See #25106 where a globally installed - -- /usr/include/ghcversion.h file was used instead of the one provided - -- by the rts. - Nothing -> case lookupUnitId (ue_homeUnitState unit_env) rtsUnitId of - Nothing -> [] - Just info -> ( "ghcversion.h") <$> collectIncludeDirs [info] - - found <- filterM doesFileExist candidates - case found of - [] -> throwGhcExceptionIO (InstallationError - ("ghcversion.h missing; tried: " - ++ intercalate ", " candidates)) - (x:_) -> return x + case ghcVersionFile dflags of + -- the user has provided an explicit `ghcversion.h` file to use. + Just path -> doesFileExist path >>= \case + True -> pure (Just path) + False -> throwGhcExceptionIO (InstallationError + ("ghcversion.h missing; tried user-supplied path: " ++ path)) + -- otherwise, try to find it in the rts' include-dirs. + -- Note: only in the RTS include-dirs! not all preload units less we may + -- use a wrong file. See #25106 where a globally installed + -- /usr/include/ghcversion.h file was used instead of the one provided + -- by the rts. + Nothing -> case lookupUnitId (ue_homeUnitState unit_env) rtsUnitId of + Nothing -> do + -- print warning and return nothing + putStrLn "Couldn't find ghcversion.h file: no rts unit available and -ghcversion-file flag not passed" + pure Nothing + Just info -> do + let candidates = ( "ghcversion.h") <$> collectIncludeDirs [info] + found <- filterM doesFileExist candidates + case found of + [] -> throwGhcExceptionIO (InstallationError + ("ghcversion.h missing; tried: " + ++ intercalate ", " candidates)) + (x:_) -> return (Just x) applyCDefs :: DefunctionalizedCDefs -> Logger -> DynFlags -> IO [String] applyCDefs NoCDefs _ _ = return [] From 70d1c87d7738515abfe0163856f3ac334a7b4747 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 4 Feb 2025 15:08:14 +0100 Subject: [PATCH 092/175] Try to enable template-haskell again --- Build.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Build.hs b/Build.hs index 0ea6374b9bae..5267dbab1bfb 100755 --- a/Build.hs +++ b/Build.hs @@ -693,7 +693,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , "ghc-prim" , "ghc-bignum" , "integer-gmp" - -- , "template-haskell" -- FIXME: adding this fails because cabal runs + , "template-haskell" -- FIXME: adding this fails because cabal runs -- ghc for some reason, without having the rts built (obviously, we're -- building it!), hence it crashes because ghc can't find -- `ghcversion.h`... which is the in the rts package it hasn't built yet From 2f31e5535758cdb49b9f557e5686bce8f7e9d0f8 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 4 Feb 2025 15:50:17 +0100 Subject: [PATCH 093/175] Remove old comment --- Build.hs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/Build.hs b/Build.hs index 5267dbab1bfb..a03b6e0a60a3 100755 --- a/Build.hs +++ b/Build.hs @@ -693,11 +693,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , "ghc-prim" , "ghc-bignum" , "integer-gmp" - , "template-haskell" -- FIXME: adding this fails because cabal runs - -- ghc for some reason, without having the rts built (obviously, we're - -- building it!), hence it crashes because ghc can't find - -- `ghcversion.h`... which is the in the rts package it hasn't built yet - -- ffs! + , "template-haskell" ] msg " - Building boot libraries..." From c6f9ab0333d743b9e5d400fb2abf30c9f38f78b9 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 5 Feb 2025 10:13:56 +0100 Subject: [PATCH 094/175] Install more boot libraries --- Build.hs | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/Build.hs b/Build.hs index a03b6e0a60a3..7d0c9cf597e6 100755 --- a/Build.hs +++ b/Build.hs @@ -99,6 +99,11 @@ main = do -- -fplugin-library. createDirectoryIfMissing True "_build/stage2/lib/" cp "_build/stage1/pkgs/*" "_build/stage2/pkgs" + cp "_build/stage1/pkgs/package.cache" "_build/stage2/pkgs/package.cache" + -- copy package.cache last to make the date younger than other files, + -- otherwise we get a load of warnings like this: + -- WARNING: cache is out of date: {..}/_build/stage2/lib/../pkgs/package.cache + -- ghc will see an old view of this package db. Use 'ghc-pkg recache' to fix. cp "_build/stage1/lib/settings" "_build/stage2/lib/settings" msg "Done" @@ -689,11 +694,42 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , "rts" , "ghc-internal" , "base" + , "stm" -- shallow compat packages over ghc-internal , "ghc-prim" , "ghc-bignum" , "integer-gmp" , "template-haskell" + -- target dependencies + , "ghc-boot-th" -- dependency of template-haskell + , "pretty" -- dependency of ghc-boot-th + -- other boot libraries used by tests + , "array" + , "binary" + , "bytestring" + , "Cabal" + , "Cabal-syntax" + , "containers" + , "deepseq" + , "directory" + , "exceptions" + , "file-io" + , "filepath" + , "hpc" + , "mtl" + , "os-string" + , "parsec" + , "process" + , "semaphore-compat" + , "text" + , "time" + , "transformers" + , "unix" -- FIXME: we'd have to install Win32 for Windows target. Maybe --libs could install dependencies too.. + -- ghc related + , "ghc-boot" + , "ghc-heap" + , "ghci" + -- , "ghc" -- FIXME: somehow it breaks the build ] msg " - Building boot libraries..." From 3e1f4176304f8f663b9b36b6be22366926d2f551 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Mon, 10 Feb 2025 11:19:49 +0100 Subject: [PATCH 095/175] Create bindist dir --- Build.hs | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/Build.hs b/Build.hs index 7d0c9cf597e6..69c55e19ba37 100755 --- a/Build.hs +++ b/Build.hs @@ -79,10 +79,11 @@ main = do -- easier for us to nuke the stage1 directory to remove only stage1's built -- libs without nuking the stage1 compiler which is slow to build. createDirectoryIfMissing True "_build/stage1/bin" + createDirectoryIfMissing True "_build/stage1/lib" cp "_build/stage0/bin/ghc" "_build/stage1/bin/ghc" cp "_build/stage0/bin/ghc-pkg" "_build/stage1/bin/ghc-pkg" cp "_build/stage0/bin/unlit" "_build/stage1/bin/unlit" - createDirectoryIfMissing True "_build/stage1/lib" + cp "_build/stage0/bin/hsc2hs" "_build/stage1/bin/hsc2hs" cp "_build/stage0/lib/settings" "_build/stage1/lib/settings" cp "_build/stage0/lib/template-hsc.h" "_build/stage1/lib/template-hsc.h" @@ -93,18 +94,24 @@ main = do -- Reuse stage1 settings for stage2 and copy stage1's built boot package for -- stage2 to use. + createDirectoryIfMissing True "_build/stage2/lib/" + cp "_build/stage1/pkgs/*" "_build/stage2/pkgs" + cp "_build/stage1/lib/settings" "_build/stage2/lib/settings" + -- TODO: in the future we want to generate different settings for cross -- targets and build boot libraries with stage2 using these settings. In any -- case, we need non-cross boot packages to build plugins for use with -- -fplugin-library. - createDirectoryIfMissing True "_build/stage2/lib/" - cp "_build/stage1/pkgs/*" "_build/stage2/pkgs" - cp "_build/stage1/pkgs/package.cache" "_build/stage2/pkgs/package.cache" - -- copy package.cache last to make the date younger than other files, - -- otherwise we get a load of warnings like this: - -- WARNING: cache is out of date: {..}/_build/stage2/lib/../pkgs/package.cache - -- ghc will see an old view of this package db. Use 'ghc-pkg recache' to fix. - cp "_build/stage1/lib/settings" "_build/stage2/lib/settings" + + + -- Finally create bindist directory + msg "Creating bindist" + createDirectoryIfMissing True "_build/bindist/lib/" + createDirectoryIfMissing True "_build/bindist/bin/" + createDirectoryIfMissing True "_build/bindist/pkgs/" + cp "_build/stage2/bin/*" "_build/bindist/bin/" + cp "_build/stage2/lib/*" "_build/bindist/lib/" + cp "_build/stage2/pkgs/*" "_build/bindist/pkgs/" msg "Done" @@ -863,8 +870,13 @@ runGenApply (GenApply f) = proc f runGenPrimop :: GenPrimop -> [String] -> CreateProcess runGenPrimop (GenPrimop f) = proc f +-- | Copy +-- +-- Recursively, force overwrite, and preserve timestamps (important for package +-- dbs) cp :: String -> String -> IO () -cp src dst = void (readCreateProcess (shell $ "cp -rf " ++ src ++ " " ++ dst) "") +cp src dst = void (readCreateProcess (shell $ "cp -rfp " ++ src ++ " " ++ dst) "") + makeCabalProject :: FilePath -> [String] -> IO () makeCabalProject path xs = writeFile path $ unlines (xs ++ common) From 4dea6380a545e7618a8ce32cdc8212b0de4a6ae3 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Mon, 10 Feb 2025 11:20:01 +0100 Subject: [PATCH 096/175] Try to upload bindist and to run tests on CI --- .github/workflows/ci.yml | 9 +++++++++ Makefile | 2 ++ 2 files changed, 11 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 87d31ed05b3a..c1171e23ae3c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,3 +40,12 @@ jobs: - name: Build the bindist run: make + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: bindist + path: _build/bindist + + - name: Run the testsuite + run: make test diff --git a/Makefile b/Makefile index 89c58316468b..4f8216f6ca19 100644 --- a/Makefile +++ b/Makefile @@ -11,3 +11,5 @@ $(CABAL): clean: rm -rf _build +test: + TEST_HC=`pwd`/_build/bindist/bin/ghc make -C testsuite/tests test From 22dae3134390fa86a57e8d87e8d77cfe23a9ec2a Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Mon, 10 Feb 2025 11:27:15 +0100 Subject: [PATCH 097/175] Add Happy as a source package --- Build.hs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Build.hs b/Build.hs index 69c55e19ba37..4e27d4d42e9e 100755 --- a/Build.hs +++ b/Build.hs @@ -665,6 +665,17 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , "allow-boot-library-installs: True" , "active-repositories: :none" , "" + , "source-repository-package" + , " type: git" + , " location: git@github.com:haskell/happy.git" + , " tag: 2.1.5" + , " subdir: lib" + , "" + , "source-repository-package" + , " type: git" + , " location: git@github.com:haskell/happy.git" + , " tag: 2.1.5" + , "" , "package *" , " library-vanilla: True" , " shared: False" From fca96b02dc6569281bb262b11c6a305a56f1c643 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Mon, 10 Feb 2025 12:05:56 +0100 Subject: [PATCH 098/175] Run stage1 compiler from _build/stage1 instead of _build/stage0 + misc --- Build.hs | 61 +++++++++++++++++++++++++---------------------- Makefile | 2 +- compiler/Setup.hs | 6 +++++ 3 files changed, 40 insertions(+), 29 deletions(-) diff --git a/Build.hs b/Build.hs index 4e27d4d42e9e..e622673aa694 100755 --- a/Build.hs +++ b/Build.hs @@ -55,12 +55,24 @@ main = do msg "Building stage1 GHC program and utility programs" buildGhcStage1 defaultGhcBuildOptions cabal ghc0 "_build/stage0/" - ghc1 <- Ghc <$> makeAbsolute "_build/stage0/bin/ghc" - ghcPkg1 <- GhcPkg <$> makeAbsolute "_build/stage0/bin/ghc-pkg" - deriveConstants <- DeriveConstants <$> makeAbsolute "_build/stage0/bin/deriveConstants" - genapply <- GenApply <$> makeAbsolute "_build/stage0/bin/genapply" - genprimop <- GenPrimop <$> makeAbsolute "_build/stage0/bin/genprimopcode" - ghcToolchain <- GhcToolchain <$> makeAbsolute "_build/stage0/bin/ghc-toolchain" + -- now we copy the stage1 compiler and other tools into _build/stage1 and we + -- generate settings to use the newly installed packages. That's not what + -- Hadrian does but it's easier for us to nuke the stage1 directory to remove + -- only stage1's built libs without nuking the stage1 compiler which is slow + -- to build. + createDirectoryIfMissing True "_build/stage1/bin" + createDirectoryIfMissing True "_build/stage1/lib" + createDirectoryIfMissing True "_build/stage1/pkgs" + cp "_build/stage0/bin/*" "_build/stage1/bin/" + cp "_build/stage0/lib/template-hsc.h" "_build/stage1/lib/template-hsc.h" + cp "_build/stage0/pkgs/*" "_build/stage1/pkgs/" + + ghc1 <- Ghc <$> makeAbsolute "_build/stage1/bin/ghc" + ghcPkg1 <- GhcPkg <$> makeAbsolute "_build/stage1/bin/ghc-pkg" + deriveConstants <- DeriveConstants <$> makeAbsolute "_build/stage1/bin/deriveConstants" + genapply <- GenApply <$> makeAbsolute "_build/stage1/bin/genapply" + genprimop <- GenPrimop <$> makeAbsolute "_build/stage1/bin/genprimopcode" + ghcToolchain <- GhcToolchain <$> makeAbsolute "_build/stage1/bin/ghc-toolchain" -- generate settings based on stage1 compiler settings: stage1 should never be -- a cross-compiler! Hence we reuse the same target platform as the bootstrap @@ -69,24 +81,11 @@ main = do let stage1_settings = emptySettings { settingsTriple = Just stage0_target_triple } - generateSettings ghcToolchain stage1_settings "_build/stage0/" + generateSettings ghcToolchain stage1_settings "_build/stage1/" msg "Building boot libraries with stage1 compiler..." buildBootLibraries cabal ghc1 ghcPkg1 deriveConstants genapply genprimop defaultGhcBuildOptions "_build/stage1/" - -- now we copy the stage1 compiler into _build/stage1 and we generate settings - -- to use the newly installed packages. That's not what Hadrian does but it's - -- easier for us to nuke the stage1 directory to remove only stage1's built - -- libs without nuking the stage1 compiler which is slow to build. - createDirectoryIfMissing True "_build/stage1/bin" - createDirectoryIfMissing True "_build/stage1/lib" - cp "_build/stage0/bin/ghc" "_build/stage1/bin/ghc" - cp "_build/stage0/bin/ghc-pkg" "_build/stage1/bin/ghc-pkg" - cp "_build/stage0/bin/unlit" "_build/stage1/bin/unlit" - cp "_build/stage0/bin/hsc2hs" "_build/stage1/bin/hsc2hs" - cp "_build/stage0/lib/settings" "_build/stage1/lib/settings" - cp "_build/stage0/lib/template-hsc.h" "_build/stage1/lib/template-hsc.h" - msg "Building stage2 GHC program" createDirectoryIfMissing True "_build/stage2" ghc1' <- Ghc <$> makeAbsolute "_build/stage1/bin/ghc" @@ -130,6 +129,8 @@ buildGhcStage booting opts cabal ghc0 dst = do let src = dst "src" prepareGhcSources opts src + msg " - Building GHC and utility programs..." + let builddir = dst "cabal" createDirectoryIfMissing True builddir @@ -365,12 +366,6 @@ buildGhcStage booting opts cabal ghc0 dst = do initEmptyDB ghcpkg pkgdb - --- TODO: --- - headers shared between different packages should be a common dependency --- - event types should be generated by Setup.hs --- package versions: do they need to be all the same? - -- | Prepare GHC sources in the given directory prepareGhcSources :: GhcBuildOptions -> FilePath -> IO () prepareGhcSources opts dst = do @@ -461,6 +456,8 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst src <- makeAbsolute (dst "src") prepareGhcSources opts src + msg " - Building boot libraries..." + -- Build the RTS src_rts <- makeAbsolute (src "libraries/rts") build_dir <- makeAbsolute (dst "cabal") @@ -548,8 +545,16 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst msg " - Generating headers and sources..." -- first run is expected to fail because of misssing headers - void $ readCreateProcessWithExitCode build_rts_cmd "" - ghcplatform_dir <- takeDirectory <$> readCreateProcess (shell ("find " ++ build_dir ++ " -name ghcplatform.h")) "" + (_exit_code, rts_conf_stdout, rts_conf_stderr) <- readCreateProcessWithExitCode build_rts_cmd "" + writeFile (dst "rts-conf.stdout") rts_conf_stdout + writeFile (dst "rts-conf.stderr") rts_conf_stderr + ghcplatform_dir <- do + ghcplatform_h <- readCreateProcess (shell ("find " ++ build_dir ++ " -name ghcplatform.h")) "" + case ghcplatform_h of + "" -> do + putStrLn "Couldn't find ghcplatform.h" + exitFailure + d -> pure (takeDirectory d) -- deriving constants let derived_constants = src_rts "include/DerivedConstants.h" diff --git a/Makefile b/Makefile index 4f8216f6ca19..0f325c79606c 100644 --- a/Makefile +++ b/Makefile @@ -11,5 +11,5 @@ $(CABAL): clean: rm -rf _build -test: +test: all TEST_HC=`pwd`/_build/bindist/bin/ghc make -C testsuite/tests test diff --git a/compiler/Setup.hs b/compiler/Setup.hs index c51d27b3ff9b..3ce749e05e82 100644 --- a/compiler/Setup.hs +++ b/compiler/Setup.hs @@ -98,7 +98,13 @@ ghcAutogen verbosity lbi@LocalBuildInfo{pkgDescrFile,withPrograms,componentNameM Nothing -> error "no target os in settings" Just os -> os createDirectoryIfMissingVerbose verbosity True (takeDirectory platformConstantsPath) +#if MIN_VERSION_Cabal(3,15,0) + -- temp files are now always created in system temp directory + -- (cf 8161f5f99dbe5d6c7564d9e163754935ddde205d) + withTempFile "Constants_tmp.hs" $ \tmp h -> do +#else withTempFile (takeDirectory platformConstantsPath) "Constants_tmp.hs" $ \tmp h -> do +#endif hClose h callProcess "deriveConstants" ["--gen-haskell-type","-o",tmp,"--target-os",targetOS] renameFile tmp platformConstantsPath From 11d8e40edd983ce51768aa357e188062cd2388b2 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Mon, 10 Feb 2025 12:17:27 +0100 Subject: [PATCH 099/175] Install more tools --- Build.hs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Build.hs b/Build.hs index e622673aa694..c664d58ae7be 100755 --- a/Build.hs +++ b/Build.hs @@ -295,6 +295,9 @@ buildGhcStage booting opts cabal ghc0 dst = do | otherwise = [ "ghc-bin:ghc" , "ghc-pkg:ghc-pkg" + , "genprimopcode:genprimopcode" + , "deriveConstants:deriveConstants" + , "genapply:genapply" , "unlit:unlit" , "hsc2hs:hsc2hs" , "hp2ps:hp2ps" @@ -347,6 +350,12 @@ buildGhcStage booting opts cabal ghc0 dst = do copy_bin "ghc-pkg:ghc-pkg" "ghc-pkg" copy_bin "unlit:unlit" "unlit" copy_bin "hsc2hs:hsc2hs" "hsc2hs" + -- always install these tools: they are needed to build the ghc library (e.g. + -- for a different target) + copy_bin "deriveConstants:deriveConstants" "deriveConstants" + copy_bin "genprimopcode:genprimopcode" "genprimopcode" + copy_bin "genapply:genapply" "genapply" + createDirectoryIfMissing True (dst "lib") cp (src "utils/hsc2hs/data/template-hsc.h") (dst "lib/template-hsc.h") @@ -355,9 +364,6 @@ buildGhcStage booting opts cabal ghc0 dst = do copy_bin "hpc-bin:hpc" "hpc" when booting $ do - copy_bin "deriveConstants:deriveConstants" "deriveConstants" - copy_bin "genprimopcode:genprimopcode" "genprimopcode" - copy_bin "genapply:genapply" "genapply" copy_bin "ghc-toolchain-bin:ghc-toolchain-bin" "ghc-toolchain" -- initialize empty global package database @@ -456,8 +462,6 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst src <- makeAbsolute (dst "src") prepareGhcSources opts src - msg " - Building boot libraries..." - -- Build the RTS src_rts <- makeAbsolute (src "libraries/rts") build_dir <- makeAbsolute (dst "cabal") From 08b61eaf5e54f970e5ad8b095e83af05dbd2b545 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Mon, 10 Feb 2025 12:18:41 +0100 Subject: [PATCH 100/175] Comment --- Build.hs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Build.hs b/Build.hs index c664d58ae7be..58a1f5fa2f79 100755 --- a/Build.hs +++ b/Build.hs @@ -756,7 +756,9 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , "ghc-boot" , "ghc-heap" , "ghci" - -- , "ghc" -- FIXME: somehow it breaks the build + -- , "ghc" -- FIXME: somehow it breaks the build: genprimopcode isn't + -- found. Maybe we should put it in path somehow? Or add it as a + -- build-depends? ] msg " - Building boot libraries..." From a2d9f9b846a1cf426b66360509f5b8871cd922ee Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Mon, 10 Feb 2025 13:49:53 +0100 Subject: [PATCH 101/175] Fix github url to use https --- Build.hs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Build.hs b/Build.hs index 58a1f5fa2f79..c42ef8996245 100755 --- a/Build.hs +++ b/Build.hs @@ -676,13 +676,13 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , "" , "source-repository-package" , " type: git" - , " location: git@github.com:haskell/happy.git" + , " location: https://github.com/haskell/happy.git" , " tag: 2.1.5" , " subdir: lib" , "" , "source-repository-package" , " type: git" - , " location: git@github.com:haskell/happy.git" + , " location: https://github.com/haskell/happy.git" , " tag: 2.1.5" , "" , "package *" From 7eea216bae027c1c6cdaa0b202b6c8e06e0ef6f2 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 11 Feb 2025 08:56:57 +0100 Subject: [PATCH 102/175] Install happy directly... --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 0f325c79606c..ffaff64f0498 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ export CABAL := $(shell cabal list-bin -v0 --project-dir libraries/Cabal cabal-install:exe:cabal) all: $(CABAL) + cabal install happy GHC=ghc-9.8.4 ./Build.hs cabal: $(CABAL) From 2e4305d9ab0ce721ffd796089e11643073cff35b Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 11 Feb 2025 09:15:31 +0100 Subject: [PATCH 103/175] Use happy from Hackage --- Build.hs | 15 +++------------ Makefile | 1 - 2 files changed, 3 insertions(+), 13 deletions(-) diff --git a/Build.hs b/Build.hs index c42ef8996245..507ae4d8b248 100755 --- a/Build.hs +++ b/Build.hs @@ -667,24 +667,15 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , " " ++ src "libraries/Win32/" , " " ++ src "libraries/Cabal/Cabal-syntax" , " " ++ src "libraries/Cabal/Cabal" - , " https://github.com/haskell/alex/archive/refs/tags/v3.5.2.0.tar.gz" + -- use alex from Hackage, not git, as it already has preprocessed + -- alex/happy files. + , " https://hackage.haskell.org/package/alex-3.5.2.0/alex-3.5.2.0.tar.gz" , "" , "benchmarks: False" , "tests: False" , "allow-boot-library-installs: True" , "active-repositories: :none" , "" - , "source-repository-package" - , " type: git" - , " location: https://github.com/haskell/happy.git" - , " tag: 2.1.5" - , " subdir: lib" - , "" - , "source-repository-package" - , " type: git" - , " location: https://github.com/haskell/happy.git" - , " tag: 2.1.5" - , "" , "package *" , " library-vanilla: True" , " shared: False" diff --git a/Makefile b/Makefile index ffaff64f0498..0f325c79606c 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,6 @@ export CABAL := $(shell cabal list-bin -v0 --project-dir libraries/Cabal cabal-install:exe:cabal) all: $(CABAL) - cabal install happy GHC=ghc-9.8.4 ./Build.hs cabal: $(CABAL) From 8fdb2f0c858c5c985991e555ae1db7efc42feb81 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 11 Feb 2025 09:28:20 +0100 Subject: [PATCH 104/175] Install GHC lib --- Build.hs | 12 +++++++++--- compiler/Setup.hs | 2 +- compiler/ghc.cabal.in | 1 + 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Build.hs b/Build.hs index 507ae4d8b248..b46da20cf6a1 100755 --- a/Build.hs +++ b/Build.hs @@ -645,6 +645,8 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , " " ++ src "utils/ghc-pkg" , " " ++ src "utils/hsc2hs" , " " ++ src "utils/unlit" + , " " ++ src "utils/genprimopcode" + , " " ++ src "utils/deriveConstants" , " " ++ src "libraries/array" , " " ++ src "libraries/binary" , " " ++ src "libraries/bytestring" @@ -670,6 +672,8 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst -- use alex from Hackage, not git, as it already has preprocessed -- alex/happy files. , " https://hackage.haskell.org/package/alex-3.5.2.0/alex-3.5.2.0.tar.gz" + , " https://hackage.haskell.org/package/happy-2.1.5/happy-2.1.5.tar.gz" + , " https://hackage.haskell.org/package/happy-lib-2.1.5/happy-lib-2.1.5.tar.gz" , "" , "benchmarks: False" , "tests: False" @@ -683,6 +687,10 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , " executable-dynamic: False" , " executable-static: False" , "" + , "package ghc" + -- Require genprimopcode, etc. used by Setup.hs + , " flags: +build-tool-depends" + , "" , "package ghc-internal" -- FIXME: make our life easier for now by using the native bignum backend , " flags: +bignum-native" @@ -747,9 +755,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , "ghc-boot" , "ghc-heap" , "ghci" - -- , "ghc" -- FIXME: somehow it breaks the build: genprimopcode isn't - -- found. Maybe we should put it in path somehow? Or add it as a - -- build-depends? + , "ghc" ] msg " - Building boot libraries..." diff --git a/compiler/Setup.hs b/compiler/Setup.hs index 3ce749e05e82..23d233877e88 100644 --- a/compiler/Setup.hs +++ b/compiler/Setup.hs @@ -107,7 +107,7 @@ ghcAutogen verbosity lbi@LocalBuildInfo{pkgDescrFile,withPrograms,componentNameM #endif hClose h callProcess "deriveConstants" ["--gen-haskell-type","-o",tmp,"--target-os",targetOS] - renameFile tmp platformConstantsPath + copyFile tmp platformConstantsPath let cProjectUnitId = case Map.lookup (CLibName LMainLibName) componentNameMap of Just [LibComponentLocalBuildInfo{componentUnitId}] -> unUnitId componentUnitId diff --git a/compiler/ghc.cabal.in b/compiler/ghc.cabal.in index 51a5dd0bbd7d..55083b5cdb39 100644 --- a/compiler/ghc.cabal.in +++ b/compiler/ghc.cabal.in @@ -67,6 +67,7 @@ Flag dynamic-system-linker Flag build-tool-depends Description: Use build-tool-depends Default: True + Manual: True Flag with-libzstd Default: False From 31c186f666a4188f2af4c4757902b9313a367bd9 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 11 Feb 2025 09:33:57 +0100 Subject: [PATCH 105/175] Install ghc-platform --- Build.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/Build.hs b/Build.hs index b46da20cf6a1..de3e86eead21 100755 --- a/Build.hs +++ b/Build.hs @@ -754,6 +754,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst -- ghc related , "ghc-boot" , "ghc-heap" + , "ghc-platform" , "ghci" , "ghc" ] From 4bcba6e7262a20226063c84732950d58209cb651 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 11 Feb 2025 10:59:01 +0100 Subject: [PATCH 106/175] Normalize testsuite backtrace paths --- testsuite/driver/testlib.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py index 5cc174403f4f..fe2fb0f63cb9 100644 --- a/testsuite/driver/testlib.py +++ b/testsuite/driver/testlib.py @@ -2931,6 +2931,11 @@ def normalise_errmsg(s: str) -> str: # hpc executable is given ghc suffix s = re.sub('hpc-ghc', 'hpc', s) + # backtrace paths contain the package path when building with Hadrian + s = re.sub(r'called at libraries/\w+(-\w+)*/', 'called at ', s) + s = re.sub(r'called at utils/\w+(-\w+)*/', 'called at ', s) + s = re.sub(r'called at compiler/', 'called at ', s) + # The inplace ghc's are called ghc-stage[123] to avoid filename # collisions, so we need to normalise that to just "ghc" s = re.sub('ghc-stage[123]', 'ghc', s) From 63abcb6d2ae1d5653cd6a7cdaeb0d3ab649769d2 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 11 Feb 2025 11:43:29 +0100 Subject: [PATCH 107/175] Upload test results --- .github/workflows/ci.yml | 9 +++++++++ Makefile | 6 +++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1171e23ae3c..147183ce5e89 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,3 +49,12 @@ jobs: - name: Run the testsuite run: make test + + - name: Upload test results + uses: actions/upload-artifact@v4 + with: + name: testsuite-results + path: | + _build/test-perf.csv + _build/test-summary.txt + _build/test-junit.xml diff --git a/Makefile b/Makefile index 0f325c79606c..0845d9e27805 100644 --- a/Makefile +++ b/Makefile @@ -12,4 +12,8 @@ clean: rm -rf _build test: all - TEST_HC=`pwd`/_build/bindist/bin/ghc make -C testsuite/tests test + TEST_HC=`pwd`/_build/bindist/bin/ghc \ + METRICS_FILE=`pwd`/_build/test-perf.csv \ + SUMMARY_FILE=`pwd`/_build/test-summary.txt \ + JUNIT_FILE=`pwd`/_build/test-junit.xml \ + make -C testsuite/tests test From 004e48512a2d24093f3361cc639cbd5a36d4ccd5 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 11 Feb 2025 14:24:49 +0100 Subject: [PATCH 108/175] Upload test results always --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 147183ce5e89..591dae769c80 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,6 +52,7 @@ jobs: - name: Upload test results uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} # upload test results even if the testsuite failed to pass with: name: testsuite-results path: | From 5e3672c9859128efa4b83c1367ab1de3cb7928a9 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 12 Feb 2025 10:13:52 +0100 Subject: [PATCH 109/175] Build iserv --- Build.hs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Build.hs b/Build.hs index de3e86eead21..b88b90405037 100755 --- a/Build.hs +++ b/Build.hs @@ -238,6 +238,7 @@ buildGhcStage booting opts cabal ghc0 dst = do , " " ++ src "utils/hp2ps" , " " ++ src "utils/hpc" , " " ++ src "utils/unlit" + , " " ++ src "utils/iserv" , " " ++ src "utils/genprimopcode/" , " " ++ src "utils/genapply/" , " " ++ src "utils/deriveConstants/" @@ -302,6 +303,7 @@ buildGhcStage booting opts cabal ghc0 dst = do , "hsc2hs:hsc2hs" , "hp2ps:hp2ps" , "hpc-bin:hpc" + , "iserv:iserv" ] let build_cmd = (runCabal cabal $ @@ -362,6 +364,7 @@ buildGhcStage booting opts cabal ghc0 dst = do unless booting $ do copy_bin "hp2ps:hp2ps" "hp2ps" copy_bin "hpc-bin:hpc" "hpc" + copy_bin "iserv:iserv" "ghc-iserv" -- vanilla iserv when booting $ do copy_bin "ghc-toolchain-bin:ghc-toolchain-bin" "ghc-toolchain" @@ -451,6 +454,7 @@ prepareGhcSources opts dst = do subst_in (dst "libraries/ghc/GHC/CmmToLlvm/Version/Bounds.hs") llvm_substs subst_in (dst "utils/ghc-pkg/ghc-pkg.cabal") common_substs + subst_in (dst "utils/iserv/iserv.cabal") common_substs subst_in (dst "libraries/ghc-internal/ghc-internal.cabal") common_substs subst_in (dst "libraries/base/base.cabal") common_substs From 2dbee36f3e061127689e70beaa554f7b0327a167 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 12 Feb 2025 10:17:04 +0100 Subject: [PATCH 110/175] Build ghc-experimental --- Build.hs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Build.hs b/Build.hs index b88b90405037..3d475bf3a89d 100755 --- a/Build.hs +++ b/Build.hs @@ -457,6 +457,7 @@ prepareGhcSources opts dst = do subst_in (dst "utils/iserv/iserv.cabal") common_substs subst_in (dst "libraries/ghc-internal/ghc-internal.cabal") common_substs + subst_in (dst "libraries/ghc-experimental/ghc-experimental.cabal") common_substs subst_in (dst "libraries/base/base.cabal") common_substs subst_in (dst "libraries/rts/include/ghcversion.h") common_substs @@ -624,7 +625,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst cp (dst_libffi "include" "*") (src_rts "include") cp (dst_libffi "lib" "libffi.a") (takeDirectory ghcplatform_dir "libCffi.a") - -- build boot libraries: ghc-internal, base... but not GHC itself + -- build boot libraries: ghc-internal, base... let cabal_project_bootlibs_path = dst "cabal-project-boot-libs" makeCabalProject cabal_project_bootlibs_path $ [ "package-dbs: clear, global" @@ -633,6 +634,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , " " ++ src "libraries/rts" , " " ++ src "libraries/ghc-prim" , " " ++ src "libraries/ghc-internal" + , " " ++ src "libraries/ghc-experimental" , " " ++ src "libraries/base" , " " ++ src "libraries/ghc" , " " ++ src "libraries/ghc-platform/" @@ -724,6 +726,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , "rts" , "ghc-internal" , "base" + , "ghc-experimental" , "stm" -- shallow compat packages over ghc-internal , "ghc-prim" From 83970a34c81bbc46427d7fa557f0c6b35f7ac4a0 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 12 Feb 2025 11:01:12 +0100 Subject: [PATCH 111/175] Reduce verbosity --- Build.hs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Build.hs b/Build.hs index 3d475bf3a89d..7e1fd9758231 100755 --- a/Build.hs +++ b/Build.hs @@ -714,7 +714,6 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , "--lib" , "--package-env=" ++ boot_libs_env , "--force-reinstalls" - , "-v3" , "--project-file=" ++ cabal_project_bootlibs_path , "--with-compiler=" ++ ghcPath ghc , "--with-hc-pkg=" ++ ghcPkgPath ghcpkg @@ -725,8 +724,8 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst -- targets , "rts" , "ghc-internal" - , "base" , "ghc-experimental" + , "base" , "stm" -- shallow compat packages over ghc-internal , "ghc-prim" From 5a2f2c97fbc74b1ce73e7ef543387bc6b5ac615a Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 12 Feb 2025 13:15:21 +0100 Subject: [PATCH 112/175] Copy usage files --- Build.hs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Build.hs b/Build.hs index 7e1fd9758231..9f211bd35ce8 100755 --- a/Build.hs +++ b/Build.hs @@ -111,6 +111,8 @@ main = do cp "_build/stage2/bin/*" "_build/bindist/bin/" cp "_build/stage2/lib/*" "_build/bindist/lib/" cp "_build/stage2/pkgs/*" "_build/bindist/pkgs/" + cp "driver/ghc-usage.txt" "_build/bindist/lib/" + cp "driver/ghci-usage.txt" "_build/bindist/lib/" msg "Done" From d98596b35cebf491a9fc0c6939098045895ce364 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 12 Feb 2025 13:15:31 +0100 Subject: [PATCH 113/175] Normalise callstacks in both and stderr and stdout --- testsuite/driver/testlib.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py index fe2fb0f63cb9..5295b010a707 100644 --- a/testsuite/driver/testlib.py +++ b/testsuite/driver/testlib.py @@ -2876,6 +2876,11 @@ def normalise_callstacks(s: str) -> str: def repl(matches): location = matches.group(1) location = normalise_slashes_(location) + # backtrace paths contain the package path when building with Hadrian + location = re.sub(r'libraries/\w+(-\w+)*/', '', location) + location = re.sub(r'utils/\w+(-\w+)*/', '', location) + location = re.sub(r'compiler/', '', location) + location = re.sub(r'\./', '', location) return ', called at {0}:: in :'.format(location) # Ignore line number differences in call stacks (#10834). s = re.sub(callSite_re, repl, s) @@ -2931,11 +2936,6 @@ def normalise_errmsg(s: str) -> str: # hpc executable is given ghc suffix s = re.sub('hpc-ghc', 'hpc', s) - # backtrace paths contain the package path when building with Hadrian - s = re.sub(r'called at libraries/\w+(-\w+)*/', 'called at ', s) - s = re.sub(r'called at utils/\w+(-\w+)*/', 'called at ', s) - s = re.sub(r'called at compiler/', 'called at ', s) - # The inplace ghc's are called ghc-stage[123] to avoid filename # collisions, so we need to normalise that to just "ghc" s = re.sub('ghc-stage[123]', 'ghc', s) From 16affa6067d82a35e9ac6df7d08a1e9532fcec5a Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 12 Feb 2025 13:52:52 +0100 Subject: [PATCH 114/175] Use same options for ghc-lib when building ghc and boot libs --- Build.hs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Build.hs b/Build.hs index 9f211bd35ce8..70e770b5e3fd 100755 --- a/Build.hs +++ b/Build.hs @@ -261,7 +261,7 @@ buildGhcStage booting opts cabal ghc0 dst = do , "allow-newer: ghc-boot-th" , "" , "package ghc" - , " flags: +internal-interpreter" + , " flags: +build-tool-depends +internal-interpreter" , "" , "package ghci" , " flags: +internal-interpreter" @@ -693,11 +693,17 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , " shared: False" , " executable-profiling: False" , " executable-dynamic: False" - , " executable-static: False" + , " executable-static: True" , "" , "package ghc" - -- Require genprimopcode, etc. used by Setup.hs - , " flags: +build-tool-depends" + -- build-tool-depends: require genprimopcode, etc. used by Setup.hs + -- internal-interpreter: otherwise our compiler has the internal + -- interpreter but not the boot library we install + -- FIXME: we should really install the lib we used to build stage2 + , " flags: +build-tool-depends +internal-interpreter" + , "" + , "package ghci" + , " flags: +internal-interpreter" , "" , "package ghc-internal" -- FIXME: make our life easier for now by using the native bignum backend From 666f8249ff25f49738f7a196589f4c8676b26aee Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 12 Feb 2025 14:14:52 +0100 Subject: [PATCH 115/175] Fix encodingAllocations test --- libraries/base/tests/perf/encodingAllocations.hs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libraries/base/tests/perf/encodingAllocations.hs b/libraries/base/tests/perf/encodingAllocations.hs index cd136963cb94..e3a252655f1d 100755 --- a/libraries/base/tests/perf/encodingAllocations.hs +++ b/libraries/base/tests/perf/encodingAllocations.hs @@ -13,7 +13,11 @@ import Distribution.Simple.Utils main :: IO () -main = withTempFile "." "encodingAllocations.tmp" (const $ loop 1000000) +main = withTempFile +#if !MIN_VERSION_Cabal(3,15,0) + "." +#endif + "encodingAllocations.tmp" (const $ loop 1000000) loop :: Int -> Handle -> IO () loop 0 !_ = pure () From 35442b09edac333dfbbd4e2608b47bedae6a9fae Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Thu, 13 Feb 2025 10:44:23 +0100 Subject: [PATCH 116/175] Build system-cxx-std-lib package --- Build.hs | 2 ++ .../system-cxx-std-lib.cabal | 21 +++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 libraries/system-cxx-std-lib/system-cxx-std-lib.cabal diff --git a/Build.hs b/Build.hs index 70e770b5e3fd..1e3e6ac66e7b 100755 --- a/Build.hs +++ b/Build.hs @@ -649,6 +649,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , " " ++ src "libraries/stm" , " " ++ src "libraries/template-haskell" , " " ++ src "libraries/hpc" + , " " ++ src "libraries/system-cxx-std-lib" , " " ++ src "ghc-bin/" , " " ++ src "utils/ghc-pkg" , " " ++ src "utils/hsc2hs" @@ -735,6 +736,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , "ghc-experimental" , "base" , "stm" + , "system-cxx-std-lib" -- shallow compat packages over ghc-internal , "ghc-prim" , "ghc-bignum" diff --git a/libraries/system-cxx-std-lib/system-cxx-std-lib.cabal b/libraries/system-cxx-std-lib/system-cxx-std-lib.cabal new file mode 100644 index 000000000000..72f86a4d0b69 --- /dev/null +++ b/libraries/system-cxx-std-lib/system-cxx-std-lib.cabal @@ -0,0 +1,21 @@ +cabal-version: 2.0 +name: system-cxx-std-lib +version: 1.0 +license: BSD-3-Clause +synopsis: A placeholder for the system's C++ standard library implementation. +description: Building against C++ libraries requires that the C++ standard + library be included when linking. Typically when compiling a C++ + project this is done automatically by the C++ compiler. However, + as GHC uses the C compiler for linking, users needing the C++ + standard library must declare this dependency explicitly. + . + This "virtual" package can be used to depend upon the host system's + C++ standard library implementation in a platform agnostic manner. +category: System +build-type: Simple + +library + -- empty library: this is just a placeholder for GHC to use to inject C++ + -- standard libraries when linking with the C toolchain, or to directly use + -- the C++ toolchain to link. + From 4a04f48cb63445c6b80810f3ca599dec974af964 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Thu, 13 Feb 2025 11:16:40 +0100 Subject: [PATCH 117/175] Build and install runghc --- Build.hs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Build.hs b/Build.hs index 1e3e6ac66e7b..fe36af0180c6 100755 --- a/Build.hs +++ b/Build.hs @@ -244,6 +244,7 @@ buildGhcStage booting opts cabal ghc0 dst = do , " " ++ src "utils/genprimopcode/" , " " ++ src "utils/genapply/" , " " ++ src "utils/deriveConstants/" + , " " ++ src "utils/runghc/" , "" , "benchmarks: False" , "tests: False" @@ -306,6 +307,7 @@ buildGhcStage booting opts cabal ghc0 dst = do , "hp2ps:hp2ps" , "hpc-bin:hpc" , "iserv:iserv" + , "runghc:runghc" ] let build_cmd = (runCabal cabal $ @@ -366,6 +368,7 @@ buildGhcStage booting opts cabal ghc0 dst = do unless booting $ do copy_bin "hp2ps:hp2ps" "hp2ps" copy_bin "hpc-bin:hpc" "hpc" + copy_bin "runghc:runghc" "runghc" copy_bin "iserv:iserv" "ghc-iserv" -- vanilla iserv when booting $ do @@ -457,6 +460,7 @@ prepareGhcSources opts dst = do subst_in (dst "utils/ghc-pkg/ghc-pkg.cabal") common_substs subst_in (dst "utils/iserv/iserv.cabal") common_substs + subst_in (dst "utils/runghc/runghc.cabal") common_substs subst_in (dst "libraries/ghc-internal/ghc-internal.cabal") common_substs subst_in (dst "libraries/ghc-experimental/ghc-experimental.cabal") common_substs From 2004ddd03d0ac4b0465773c92d32a21b86dbba6f Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Thu, 13 Feb 2025 11:18:44 +0100 Subject: [PATCH 118/175] Build and install ghc-compact --- Build.hs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Build.hs b/Build.hs index fe36af0180c6..6d3f0ccede3c 100755 --- a/Build.hs +++ b/Build.hs @@ -644,6 +644,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , " " ++ src "libraries/base" , " " ++ src "libraries/ghc" , " " ++ src "libraries/ghc-platform/" + , " " ++ src "libraries/ghc-compact/" , " " ++ src "libraries/ghc-bignum/" , " " ++ src "libraries/integer-gmp/" , " " ++ src "libraries/ghc-boot/" @@ -738,6 +739,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , "rts" , "ghc-internal" , "ghc-experimental" + , "ghc-compact" , "base" , "stm" , "system-cxx-std-lib" From 49807984159986514f964787e16f0d024b4e540a Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Thu, 13 Feb 2025 11:29:15 +0100 Subject: [PATCH 119/175] Use cabal from stable-haskell repository --- .gitmodules | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 01d9e8110ed1..fc634597fb31 100644 --- a/.gitmodules +++ b/.gitmodules @@ -8,9 +8,9 @@ ignore = untracked [submodule "libraries/Cabal"] path = libraries/Cabal - url = https://github.com/hsyl20/cabal + url = https://github.com/stable-haskell/cabal ignore = untracked - branch = hsyl20/per-file-extra-source-options + branch = wip/make-build [submodule "libraries/containers"] path = libraries/containers url = https://gitlab.haskell.org/ghc/packages/containers.git From 3e82764cf8b4add8fc0de6e60a6eeeed22d85d3e Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Thu, 13 Feb 2025 14:29:07 +0100 Subject: [PATCH 120/175] Distribute ghc-toolchain lib (used by some tests) --- Build.hs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Build.hs b/Build.hs index 6d3f0ccede3c..3d299ee666a0 100755 --- a/Build.hs +++ b/Build.hs @@ -661,6 +661,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , " " ++ src "utils/unlit" , " " ++ src "utils/genprimopcode" , " " ++ src "utils/deriveConstants" + , " " ++ src "utils/ghc-toolchain/" , " " ++ src "libraries/array" , " " ++ src "libraries/binary" , " " ++ src "libraries/bytestring" @@ -777,6 +778,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , "ghc-boot" , "ghc-heap" , "ghc-platform" + , "ghc-toolchain" -- some test requires this , "ghci" , "ghc" ] From 42bc76ef80cf8e88d7f3a148fae79e9b1d349bd2 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Thu, 13 Feb 2025 16:45:29 +0100 Subject: [PATCH 121/175] Fix stage2 build to support plugins --- Build.hs | 31 ++++++------------------------- 1 file changed, 6 insertions(+), 25 deletions(-) diff --git a/Build.hs b/Build.hs index 3d299ee666a0..0a1ed6e79c99 100755 --- a/Build.hs +++ b/Build.hs @@ -213,28 +213,15 @@ buildGhcStage booting opts cabal ghc0 dst = do let stage2_project = [ "packages:" + -- ghc *library* mustn't be listed here: otherwise its unit-id becomes + -- ghc-9.xx-inplace and it's wrong when we load plugins + -- (wired-in thisGhcUnitId is wrong) + -- + -- actually we don't need any of the boot packages we already + -- installed. , " " ++ src "ghc-bin/" - , " " ++ src "libraries/deepseq/" - , " " ++ src "libraries/hpc/" - , " " ++ src "libraries/stm/" - , " " ++ src "libraries/text/" - , " " ++ src "libraries/ghc/" - , " " ++ src "libraries/directory/" - , " " ++ src "libraries/file-io/" - , " " ++ src "libraries/filepath/" , " " ++ src "libraries/haskeline/" , " " ++ src "libraries/terminfo/" - , " " ++ src "libraries/ghc-platform/" - , " " ++ src "libraries/ghc-boot/" - , " " ++ src "libraries/ghc-boot-th/" - , " " ++ src "libraries/ghc-heap" - , " " ++ src "libraries/ghci" - , " " ++ src "libraries/os-string/" - , " " ++ src "libraries/process/" - , " " ++ src "libraries/semaphore-compat" - , " " ++ src "libraries/time" - , " " ++ src "libraries/unix/" - , " " ++ src "libraries/Win32/" , " " ++ src "utils/ghc-pkg" , " " ++ src "utils/hsc2hs" , " " ++ src "utils/hp2ps" @@ -261,12 +248,6 @@ buildGhcStage booting opts cabal ghc0 dst = do -- allow template-haskell with newer ghc-boot-th , "allow-newer: ghc-boot-th" , "" - , "package ghc" - , " flags: +build-tool-depends +internal-interpreter" - , "" - , "package ghci" - , " flags: +internal-interpreter" - , "" , "package ghc-bin" , " flags: +internal-interpreter" , "" From 6d0d86405cb72c4caa32c4b7f17bd83b61dc8968 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Thu, 13 Feb 2025 16:59:24 +0100 Subject: [PATCH 122/175] Remove useless boot-lib-reinstall --- Build.hs | 2 -- 1 file changed, 2 deletions(-) diff --git a/Build.hs b/Build.hs index 0a1ed6e79c99..b54d6d7e0c7e 100755 --- a/Build.hs +++ b/Build.hs @@ -235,8 +235,6 @@ buildGhcStage booting opts cabal ghc0 dst = do , "" , "benchmarks: False" , "tests: False" - , "allow-boot-library-installs: True" - -- we need even after booting because cabal thinks `template-haskell` isn't reinstallable otherwise , "" , "package *" , " library-vanilla: True" From d0319385431fcd74b2ebbdf155d69a18d17f64a7 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Thu, 13 Feb 2025 17:00:19 +0100 Subject: [PATCH 123/175] Remove useless flags --- Build.hs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Build.hs b/Build.hs index b54d6d7e0c7e..52e99fd1ac51 100755 --- a/Build.hs +++ b/Build.hs @@ -242,9 +242,6 @@ buildGhcStage booting opts cabal ghc0 dst = do , " executable-profiling: False" , " executable-dynamic: False" , " executable-static: True" - , "" - -- allow template-haskell with newer ghc-boot-th - , "allow-newer: ghc-boot-th" , "" , "package ghc-bin" , " flags: +internal-interpreter" @@ -255,10 +252,6 @@ buildGhcStage booting opts cabal ghc0 dst = do , "package haskeline" , " flags: -terminfo" -- FIXME: should be enabled but I don't have the static libs for terminfo on ArchLinux... , "" - , "package text" - -- FIXME: avoid having to deal with system-cxx-std-lib fake package for now - , " flags: -simdutf" - , "" ] makeCabalProject cabal_project_path (if booting then stage1_project else stage2_project) From 8d8d9a5bc2a0560af26ac9bf2053090ddf5d9230 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Mon, 17 Feb 2025 10:43:44 +0100 Subject: [PATCH 124/175] Only run compact sanity tests when the debug rts is available --- libraries/ghc-compact/tests/all.T | 2 +- testsuite/mk/test.mk | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/libraries/ghc-compact/tests/all.T b/libraries/ghc-compact/tests/all.T index 9a666161ff99..2bb90c4dbf1f 100644 --- a/libraries/ghc-compact/tests/all.T +++ b/libraries/ghc-compact/tests/all.T @@ -1,5 +1,5 @@ setTestOpts( - [extra_ways(['sanity', 'compacting_gc']), + [extra_ways(['compacting_gc'] + (['sanity'] if debug_rts() else [])), js_skip # compact API not supported by the JS backend ]) diff --git a/testsuite/mk/test.mk b/testsuite/mk/test.mk index 0d8899fe51b9..9fb28a05ad85 100644 --- a/testsuite/mk/test.mk +++ b/testsuite/mk/test.mk @@ -124,6 +124,13 @@ else RUNTEST_OPTS += -e ghc_with_dynamic_rts=False endif +ifeq "$(filter thr, $(GhcRTSWays))" "debug" +RUNTEST_OPTS += -e config.debug_rts=True +else +RUNTEST_OPTS += -e config.debug_rts=False +endif + + ifeq "$(GhcWithInterpreter)" "NO" RUNTEST_OPTS += -e config.have_interp=False else ifeq "$(GhcStage)" "1" From 643072e8182ad69663f38b16d3d2edcba67c0245 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Mon, 17 Feb 2025 11:26:40 +0100 Subject: [PATCH 125/175] Flush C buffers in T7388 --- testsuite/tests/ghci/scripts/T7388.hs | 2 ++ testsuite/tests/ghci/scripts/T7388.script | 3 +++ 2 files changed, 5 insertions(+) diff --git a/testsuite/tests/ghci/scripts/T7388.hs b/testsuite/tests/ghci/scripts/T7388.hs index 91ca1c7268f3..348f2c446fc3 100644 --- a/testsuite/tests/ghci/scripts/T7388.hs +++ b/testsuite/tests/ghci/scripts/T7388.hs @@ -2,5 +2,7 @@ module T7388 where import Foreign.C +import Foreign.Ptr foreign import capi "stdio.h printf" printfb :: CString -> CInt -> IO () +foreign import capi "stdio.h fflush" fflushb :: Ptr () -> IO () diff --git a/testsuite/tests/ghci/scripts/T7388.script b/testsuite/tests/ghci/scripts/T7388.script index 7f02d864539d..c9477ab761de 100644 --- a/testsuite/tests/ghci/scripts/T7388.script +++ b/testsuite/tests/ghci/scripts/T7388.script @@ -1,2 +1,5 @@ :l T7388 withCString "I am a working CApi FFI call\n" $ \str -> printfb str 0 +-- don't forget to flush, otherwise when ghc is statically linked, the C stdout +-- buffer is never flushed. +fflushb nullPtr From d1ccf0073accd27887f25a80137bfaab2a0fbe33 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Mon, 17 Feb 2025 15:47:23 +0100 Subject: [PATCH 126/175] Fix RTS report of host arch/os for the testsuite --- Build.hs | 12 ++++-------- rts/RtsUtils.c | 4 ---- testsuite/ghc-config/ghc-config.hs | 6 +++--- 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/Build.hs b/Build.hs index 52e99fd1ac51..a0343f0d920b 100755 --- a/Build.hs +++ b/Build.hs @@ -460,18 +460,14 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst [ "package rts" , def_string "ProjectVersion" (Text.unpack (gboVersionInt opts)) , def_string "RtsWay" "FIXME" - , def_string "HostPlatform" "FIXME" - , def_string "HostArch" "FIXME" - , def_string "HostOS" "FIXME" - , def_string "HostVendor" "FIXME" + , def_string "HostPlatform" "x86_64-unknown-linux" -- FIXME + , def_string "HostArch" "x86_64" -- FIXME: appropriate value required for the tests + , def_string "HostOS" "linux" -- FIXME: appropriate value required for the tests + , def_string "HostVendor" "unknown" , def_string "BuildPlatform" "FIXME" , def_string "BuildArch" "FIXME" , def_string "BuildOS" "FIXME" , def_string "BuildVendor" "FIXME" - , def_string "TargetPlatform" "FIXME" - , def_string "TargetArch" "FIXME" - , def_string "TargetOS" "FIXME" - , def_string "TargetVendor" "FIXME" , def_string "GhcUnregisterised" "FIXME" , def_string "TablesNextToCode" "FIXME" -- Set the namespace for the rts fs functions diff --git a/rts/RtsUtils.c b/rts/RtsUtils.c index f842c1cc3c43..7640d7cca817 100644 --- a/rts/RtsUtils.c +++ b/rts/RtsUtils.c @@ -372,10 +372,6 @@ void printRtsInfo(const RtsConfig rts_config) { mkRtsInfoPair("Host architecture", HostArch); mkRtsInfoPair("Host OS", HostOS); mkRtsInfoPair("Host vendor", HostVendor); - mkRtsInfoPair("Target platform", TargetPlatform); - mkRtsInfoPair("Target architecture", TargetArch); - mkRtsInfoPair("Target OS", TargetOS); - mkRtsInfoPair("Target vendor", TargetVendor); mkRtsInfoPair("Word size", TOSTRING(WORD_SIZE_IN_BITS)); // TODO(@Ericson2314) This is a joint property of the RTS and generated // code. The compiler will soon be multi-target so it doesn't make sense to diff --git a/testsuite/ghc-config/ghc-config.hs b/testsuite/ghc-config/ghc-config.hs index 95f58d06789a..6dd442542e74 100644 --- a/testsuite/ghc-config/ghc-config.hs +++ b/testsuite/ghc-config/ghc-config.hs @@ -10,9 +10,9 @@ main = do let fields = read info :: [(String,String)] getGhcFieldOrFail fields "HostOS" "Host OS" getGhcFieldOrFail fields "WORDSIZE" "Word size" - getGhcFieldOrFail fields "TARGETPLATFORM" "Target platform" - getGhcFieldOrFail fields "TargetOS_CPP" "Target OS" - getGhcFieldOrFail fields "TargetARCH_CPP" "Target architecture" + getGhcFieldOrFail fields "TARGETPLATFORM" "Host platform" + getGhcFieldOrFail fields "TargetOS_CPP" "Host OS" + getGhcFieldOrFail fields "TargetARCH_CPP" "Host architecture" getGhcFieldOrFail fields "RTSWay" "RTS way" info <- readProcess ghc ["--info"] "" From 2f4dc99bb539c6b9688602b9943d5122b52c17cf Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 18 Feb 2025 16:16:01 +0100 Subject: [PATCH 127/175] Properly skip more tests --- Build.hs | 4 +++- testsuite/driver/testlib.py | 7 +++++++ testsuite/tests/codeGen/should_run/T25374/all.T | 2 +- testsuite/tests/rts/T8308/all.T | 2 +- testsuite/tests/rts/all.T | 4 ++-- 5 files changed, 14 insertions(+), 5 deletions(-) diff --git a/Build.hs b/Build.hs index a0343f0d920b..696542d3853c 100755 --- a/Build.hs +++ b/Build.hs @@ -1028,5 +1028,7 @@ generateSettings ghc_toolchain Settings{..} dst = do -- fixup settings generated by ghc-toolchain kvs <- (Map.fromList . read) <$> readFile gen_settings_path :: IO (Map String String) - let kvs' = Map.insert "Relative Global Package DB" "../pkgs" kvs + let kvs' = Map.insert "Relative Global Package DB" "../pkgs" + $ Map.insert "Support SMP" "NO" -- FIXME: this depends on the different ways used to build the RTS! + $ kvs writeFile (dst "lib/settings") (show $ Map.toList kvs') diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py index 5295b010a707..70e778bbad53 100644 --- a/testsuite/driver/testlib.py +++ b/testsuite/driver/testlib.py @@ -356,6 +356,13 @@ def req_ghc_smp( name, opts ): if not config.ghc_has_smp: opts.skip = True +def req_target_debug_rts( name, opts ): + """ + Mark a test as requiring the debug rts (e.g. compile with -debug or -ticky) + """ + if not config.debug_rts: + opts.skip = True + def req_target_smp( name, opts ): """ Mark a test as requiring smp when run on the target. If the target does diff --git a/testsuite/tests/codeGen/should_run/T25374/all.T b/testsuite/tests/codeGen/should_run/T25374/all.T index 1e4c3e9860b0..0e02dc0d263d 100644 --- a/testsuite/tests/codeGen/should_run/T25374/all.T +++ b/testsuite/tests/codeGen/should_run/T25374/all.T @@ -1,3 +1,3 @@ # This shouldn't crash the disassembler -test('T25374', [extra_hc_opts('+RTS -Di -RTS'), ignore_stderr, unless(debug_rts(), skip)], ghci_script, ['']) +test('T25374', [extra_hc_opts('+RTS -Di -RTS'), ignore_stderr, req_target_debug_rts], ghci_script, ['']) diff --git a/testsuite/tests/rts/T8308/all.T b/testsuite/tests/rts/T8308/all.T index 74eeec3ebcb1..080f09743f3c 100644 --- a/testsuite/tests/rts/T8308/all.T +++ b/testsuite/tests/rts/T8308/all.T @@ -1 +1 @@ -test('T8308', js_broken(22261), makefile_test, ['T8308']) +test('T8308', req_target_debug_rts, makefile_test, ['T8308']) diff --git a/testsuite/tests/rts/all.T b/testsuite/tests/rts/all.T index 96b93c72e3db..9cdefe8b1747 100644 --- a/testsuite/tests/rts/all.T +++ b/testsuite/tests/rts/all.T @@ -410,7 +410,7 @@ test('T10904', [ extra_run_opts('20000'), req_c ], test('T10728', [extra_run_opts('+RTS -maxN3 -RTS'), only_ways(['threaded2'])], compile_and_run, ['']) -test('T9405', [when(opsys('mingw32'), fragile(21361)), js_broken(22261)], makefile_test, ['T9405']) +test('T9405', [when(opsys('mingw32'), fragile(21361)), req_target_debug_rts], makefile_test, ['T9405']) test('T11788', [ when(ghc_dynamic(), skip) , req_interp @@ -620,7 +620,7 @@ test('T23221', compile_and_run, ['-O -with-rtsopts -T']) -test('T23142', [unless(debug_rts(), skip), req_interp], makefile_test, ['T23142']) +test('T23142', [req_target_debug_rts, req_interp], makefile_test, ['T23142']) test('T23400', [], compile_and_run, ['-with-rtsopts -A8k']) From 3f6299a4b0d9fcbd9a01e360d9134cf7efd859d8 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 19 Feb 2025 14:45:35 +0100 Subject: [PATCH 128/175] Fix InternalCounters test --- testsuite/tests/rts/all.T | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/testsuite/tests/rts/all.T b/testsuite/tests/rts/all.T index 9cdefe8b1747..64022e31146e 100644 --- a/testsuite/tests/rts/all.T +++ b/testsuite/tests/rts/all.T @@ -466,9 +466,11 @@ test('T14900', test('InternalCounters', [ js_skip # JS backend doesn't support internal counters + # Require threaded RTS + , req_target_smp # The ways which build against the debug RTS are built with PROF_SPIN and # therefore differ in output - , omit_ways(['nonmoving_thr_sanity', 'threaded2_sanity', 'sanity']) + , when (debug_rts(), skip) ], makefile_test, ['InternalCounters']) test('alloccounter1', js_broken(22261), compile_and_run, [ From 6073c29a2e4836c658e7827fb9c14e8c5de49525 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 19 Feb 2025 15:13:39 +0100 Subject: [PATCH 129/175] Use C++ compiler to link code depending on system-cxx-std-lib --- compiler/GHC/Driver/Config/Linker.hs | 11 ++++++++--- compiler/GHC/Driver/Config/StgToJS.hs | 2 +- compiler/GHC/Linker/Dynamic.hs | 4 +++- compiler/GHC/Linker/Static.hs | 6 +++++- 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/compiler/GHC/Driver/Config/Linker.hs b/compiler/GHC/Driver/Config/Linker.hs index bf4cc95f2dbf..489f3ba5bdf7 100644 --- a/compiler/GHC/Driver/Config/Linker.hs +++ b/compiler/GHC/Driver/Config/Linker.hs @@ -20,8 +20,8 @@ initFrameworkOpts dflags = FrameworkOpts } -- | Initialize linker configuration from DynFlags -initLinkerConfig :: DynFlags -> LinkerConfig -initLinkerConfig dflags = +initLinkerConfig :: DynFlags -> Bool -> LinkerConfig +initLinkerConfig dflags require_cxx = let -- see Note [Solaris linker] ld_filter = case platformOS (targetPlatform dflags) of @@ -46,8 +46,13 @@ initLinkerConfig dflags = (p,pre_args) = pgm_l dflags post_args = map Option (getOpts dflags opt_l) + -- sneakily switch to C++ compiler when we need C++ standard lib + -- FIXME: ld flags may be totally inappropriate for the C++ compiler? + ld_prog = if require_cxx then pgm_cxx dflags else p + + in LinkerConfig - { linkerProgram = p + { linkerProgram = ld_prog , linkerOptionsPre = pre_args , linkerOptionsPost = post_args , linkerTempDir = tmpDir dflags diff --git a/compiler/GHC/Driver/Config/StgToJS.hs b/compiler/GHC/Driver/Config/StgToJS.hs index a737f9a242fd..c27c1378537f 100644 --- a/compiler/GHC/Driver/Config/StgToJS.hs +++ b/compiler/GHC/Driver/Config/StgToJS.hs @@ -34,7 +34,7 @@ initStgToJSConfig dflags = StgToJSConfig , csRuntimeAssert = False -- settings , csContext = initSDocContext dflags defaultDumpStyle - , csLinkerConfig = initLinkerConfig dflags + , csLinkerConfig = initLinkerConfig dflags False -- no C++ linking } -- | Default linker configuration diff --git a/compiler/GHC/Linker/Dynamic.hs b/compiler/GHC/Linker/Dynamic.hs index b8aedd90acf5..28f814ca305a 100644 --- a/compiler/GHC/Linker/Dynamic.hs +++ b/compiler/GHC/Linker/Dynamic.hs @@ -24,6 +24,7 @@ import GHC.Linker.Unit import GHC.Linker.External import GHC.Utils.Logger import GHC.Utils.TmpFs +import GHC.Data.FastString import Control.Monad (when) import System.FilePath @@ -105,7 +106,8 @@ linkDynLib logger tmpfs dflags0 unit_env o_files dep_packages pkg_framework_opts <- getUnitFrameworkOpts unit_env (map unitId pkgs) let framework_opts = getFrameworkOpts (initFrameworkOpts dflags) platform - let linker_config = initLinkerConfig dflags + let require_cxx = any ((==) (PackageName (fsLit "system-cxx-std-lib")) . unitPackageName) pkgs + let linker_config = initLinkerConfig dflags require_cxx case os of OSMinGW32 -> do diff --git a/compiler/GHC/Linker/Static.hs b/compiler/GHC/Linker/Static.hs index ef356d724350..851e34db6a4a 100644 --- a/compiler/GHC/Linker/Static.hs +++ b/compiler/GHC/Linker/Static.hs @@ -33,6 +33,8 @@ import GHC.Linker.Static.Utils import GHC.Driver.Config.Linker import GHC.Driver.Session +import GHC.Data.FastString + import System.FilePath import System.Directory import Control.Monad @@ -192,7 +194,9 @@ linkBinary' staticLink logger tmpfs dflags unit_env o_files dep_units = do OSMinGW32 | gopt Opt_GenManifest dflags -> maybeCreateManifest logger tmpfs dflags output_fn _ -> return [] - let linker_config = initLinkerConfig dflags + let require_cxx = any ((==) (PackageName (fsLit "system-cxx-std-lib")) . unitPackageName) pkgs + + let linker_config = initLinkerConfig dflags require_cxx let link dflags args = do runLink logger tmpfs linker_config args -- Make sure to honour -fno-use-rpaths if set on darwin as well; see #20004 From 0cc3338197eed4c8133fef4e9df192a564b77462 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 19 Feb 2025 15:45:36 +0100 Subject: [PATCH 130/175] Fix stack_misc_closures --- libraries/ghc-heap/tests/all.T | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libraries/ghc-heap/tests/all.T b/libraries/ghc-heap/tests/all.T index 5722182d5f65..5b8b755ae812 100644 --- a/libraries/ghc-heap/tests/all.T +++ b/libraries/ghc-heap/tests/all.T @@ -94,7 +94,8 @@ test('stack_misc_closures', [ extra_files(['stack_misc_closures_c.c', 'stack_misc_closures_prim.cmm', 'TestUtils.hs']), ignore_stdout, - ignore_stderr + ignore_stderr, + req_target_debug_rts # Debug RTS to use checkSTACK() ], multi_compile_and_run, ['stack_misc_closures', From b6a2d46159e3627dabe419516cac972cb7d7f3ad Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Fri, 21 Feb 2025 13:22:53 +0100 Subject: [PATCH 131/175] Update Cabal: don't build Setup with -threaded --- libraries/Cabal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/Cabal b/libraries/Cabal index e8538c6597cc..adfd46f70d40 160000 --- a/libraries/Cabal +++ b/libraries/Cabal @@ -1 +1 @@ -Subproject commit e8538c6597cc355eca705827845f5341d1d815f2 +Subproject commit adfd46f70d40fca9f14603a15a8ac15089debab8 From ab5698949b38ac819138fbc0b51d914f8a913ccd Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Fri, 21 Feb 2025 13:23:15 +0100 Subject: [PATCH 132/175] Ensure that we use vanilla rts Also ensure that GHC properly fails when using -threaded or -debug --- Build.hs | 6 ++++-- compiler/GHC/Unit/Info.hs | 9 ++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/Build.hs b/Build.hs index 696542d3853c..c9c81e26ba6f 100755 --- a/Build.hs +++ b/Build.hs @@ -244,7 +244,8 @@ buildGhcStage booting opts cabal ghc0 dst = do , " executable-static: True" , "" , "package ghc-bin" - , " flags: +internal-interpreter" + -- FIXME: we don't support the threaded rts way yet + , " flags: +internal-interpreter -threaded" , "" , "package hsc2hs" , " flags: +in-ghc-tree" -- allow finding template-hsc.h in GHC's /lib @@ -459,7 +460,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst let rts_options = [ "package rts" , def_string "ProjectVersion" (Text.unpack (gboVersionInt opts)) - , def_string "RtsWay" "FIXME" + , def_string "RtsWay" "v" -- FIXME , def_string "HostPlatform" "x86_64-unknown-linux" -- FIXME , def_string "HostArch" "x86_64" -- FIXME: appropriate value required for the tests , def_string "HostOS" "linux" -- FIXME: appropriate value required for the tests @@ -1030,5 +1031,6 @@ generateSettings ghc_toolchain Settings{..} dst = do kvs <- (Map.fromList . read) <$> readFile gen_settings_path :: IO (Map String String) let kvs' = Map.insert "Relative Global Package DB" "../pkgs" $ Map.insert "Support SMP" "NO" -- FIXME: this depends on the different ways used to build the RTS! + $ Map.insert "RTS ways" "v" -- FIXME: this depends on the different ways used to build the RTS! $ kvs writeFile (dst "lib/settings") (show $ Map.toList kvs') diff --git a/compiler/GHC/Unit/Info.hs b/compiler/GHC/Unit/Info.hs index 04ac54b5d310..ba653c2b0a50 100644 --- a/compiler/GHC/Unit/Info.hs +++ b/compiler/GHC/Unit/Info.hs @@ -235,9 +235,12 @@ unitHsLibs namever ways0 p = map (mkDynName . addSuffix . ST.unpack) (unitLibrar -- -- This change elevates the need to add custom hooks -- and handling specifically for the `rts` package. - addSuffix rts@"HSrts" = rts ++ (expandTag rts_tag) - addSuffix rts@"HSrts-1.0.3" = rts ++ (expandTag rts_tag) - addSuffix other_lib = other_lib ++ (expandTag tag) + is_rts x = (x == "HSrts" || "HSrts-" `isPrefixOf` x) + -- ensure we don't consider packages with names like "rts-foo" + && unitPackageName p == PackageName (fsLit "rts") + addSuffix x + | is_rts x = x ++ (expandTag rts_tag) + | otherwise = x ++ (expandTag tag) expandTag t | null t = "" | otherwise = '_':t From 7996993375ccbf4823d60e2998647c2745145ed8 Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Mon, 24 Feb 2025 10:04:54 +0900 Subject: [PATCH 133/175] Drop libffi; Stop shipping a copy of libffi GHC used to depend on an in-flight version of libffi. As libffis releases are more frequent now, there is no immediate need for this anymore. GHC should be able to rely on system provided libffi, and not try to become a system library package manager as well. --- .gitmodules | 4 ---- compiler/GHC/Driver/CodeOutput.hs | 7 +++---- libffi-tarballs | 1 - packages | 1 - rts/rts.cabal | 25 +++++++------------------ 5 files changed, 10 insertions(+), 28 deletions(-) delete mode 160000 libffi-tarballs diff --git a/.gitmodules b/.gitmodules index fc634597fb31..9c72ac9d8846 100644 --- a/.gitmodules +++ b/.gitmodules @@ -100,10 +100,6 @@ path = utils/hsc2hs url = https://gitlab.haskell.org/ghc/hsc2hs.git ignore = untracked -[submodule "libffi-tarballs"] - path = libffi-tarballs - url = https://gitlab.haskell.org/ghc/libffi-tarballs.git - ignore = untracked [submodule "gmp-tarballs"] path = libraries/ghc-internal/gmp/gmp-tarballs url = https://gitlab.haskell.org/ghc/gmp-tarballs.git diff --git a/compiler/GHC/Driver/CodeOutput.hs b/compiler/GHC/Driver/CodeOutput.hs index ff5a25c3bae0..023c4e1e365f 100644 --- a/compiler/GHC/Driver/CodeOutput.hs +++ b/compiler/GHC/Driver/CodeOutput.hs @@ -255,12 +255,11 @@ outputJS _ _ _ _ _ = pgmError $ "codeOutput: Hit JavaScript case. We should neve -} {- -Note [Packaging libffi headers] +Note [libffi headers] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The C code emitted by GHC for libffi adjustors must depend upon the ffi_arg type, -defined in . For this reason, we must ensure that is available -in binary distributions. To do so, we install these headers as part of the -`rts` package. +defined in . On systems where GHC uses the libffi adjustors, the libffi +library, and headers must be installed. -} outputForeignStubs diff --git a/libffi-tarballs b/libffi-tarballs deleted file mode 160000 index 89a9b01c5647..000000000000 --- a/libffi-tarballs +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 89a9b01c5647c8f0d3899435b99df690f582e9f1 diff --git a/packages b/packages index 4f02d0133c0b..d6bb0cd77e13 100644 --- a/packages +++ b/packages @@ -37,7 +37,6 @@ # localpath tag remotepath upstreamurl # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ghc-tarballs windows ghc-tarballs.git - -libffi-tarballs - - - utils/hsc2hs - - ssh://git@github.com/haskell/hsc2hs.git libraries/array - - - libraries/binary - - https://github.com/kolmodin/binary.git diff --git a/rts/rts.cabal b/rts/rts.cabal index 31ebdd4eb82a..249e814810ec 100644 --- a/rts/rts.cabal +++ b/rts/rts.cabal @@ -276,9 +276,6 @@ flag librt flag libdl default: False manual: True -flag use-system-libffi - default: False - manual: True flag libffi-adjustors default: False manual: True @@ -402,18 +399,6 @@ library stg/Types.h else - -- If we are using an in-tree libffi then we must declare it as a bundled - -- library to ensure that Cabal installs it. - if !flag(use-system-libffi) - if os(windows) - extra-bundled-libraries: Cffi-6 - else - extra-bundled-libraries: Cffi - - install-includes: ffi.h ffitarget.h - -- ^ see Note [Packaging libffi headers] in - -- GHC.Driver.CodeOutput. - -- Here we declare several flavours to be available when passing the -- suitable (combination of) flag(s) when configuring the RTS from hadrian, -- using Cabal. @@ -465,9 +450,6 @@ library extra-libraries: rt if flag(libdl) extra-libraries: dl - if flag(use-system-libffi) - extra-libraries: ffi - extra-libraries-static: ffi if os(windows) extra-libraries: -- for the linker @@ -634,6 +616,8 @@ library -- Adjustor stuff if flag(libffi-adjustors) c-sources: adjustor/LibffiAdjustor.c + extra-libraries: ffi + extra-libraries-static: ffi else -- Use GHC's native adjustors if arch(i386) @@ -646,6 +630,11 @@ library else asm-sources: adjustor/NativeAmd64Asm.S c-sources: adjustor/NativeAmd64.c + -- fall back to the LibffiAdjustor if neither i386, or x86_64 + if !arch(x86_64) && !arch(i386) + c-sources: adjustor/LibffiAdjustor.c + extra-libraries: ffi + extra-libraries-static: ffi -- Use assembler STG entrypoint on architectures where it is used if arch(ppc) || arch(ppc64) || arch(s390x) || arch(riscv64) || arch(loongarch64) From 1ec11989ce5d46f4fcaf5fea1f713e91e546f8b9 Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Mon, 24 Feb 2025 15:15:17 +0900 Subject: [PATCH 134/175] [build] drop ffi --- Build.hs | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/Build.hs b/Build.hs index c9c81e26ba6f..16b66c17b622 100755 --- a/Build.hs +++ b/Build.hs @@ -575,31 +575,6 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst writeFile (src "libraries/ghc-internal/src/GHC/Internal/Prim.hs") =<< readCreateProcess (runGenPrimop genprimop ["--make-haskell-source"]) primops writeFile (src "libraries/ghc-internal/src/GHC/Internal/PrimopWrappers.hs") =<< readCreateProcess (runGenPrimop genprimop ["--make-haskell-wrappers"]) primops - -- build libffi - msg " - Building libffi..." - src_libffi <- makeAbsolute (src "libffi") - dst_libffi <- makeAbsolute (dst "libffi") - let libffi_version = "3.4.6" - createDirectoryIfMissing True src_libffi - createDirectoryIfMissing True dst_libffi - void $ readCreateProcess (shell ("tar -xvf libffi-tarballs/libffi-" ++ libffi_version ++ ".tar.gz -C " ++ src_libffi)) "" - let build_libffi = mconcat - [ "cd " ++ src_libffi "libffi-" ++ libffi_version ++ "; " - -- FIXME: pass the appropriate toolchain (CC, LD...) - , "./configure --disable-docs --with-pic=yes --disable-multi-os-directory --prefix=" ++ dst_libffi - , " && make install -j" - ] - (libffi_exit_code, libffi_stdout, libffi_stderr) <- readCreateProcessWithExitCode (shell build_libffi) "" - case libffi_exit_code of - ExitSuccess -> pure () - ExitFailure r -> do - putStrLn $ "Failed to build libffi with error code " ++ show r - putStrLn libffi_stdout - putStrLn libffi_stderr - exitFailure - cp (dst_libffi "include" "*") (src_rts "include") - cp (dst_libffi "lib" "libffi.a") (takeDirectory ghcplatform_dir "libCffi.a") - -- build boot libraries: ghc-internal, base... let cabal_project_bootlibs_path = dst "cabal-project-boot-libs" makeCabalProject cabal_project_bootlibs_path $ From a57e5f0925c014e1d585dbe334b270bc868ce011 Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Mon, 24 Feb 2025 10:11:24 +0900 Subject: [PATCH 135/175] [UX] Add quotes around missing tools When a tool is empty, GHC will print the helpful message: "could not execute:" which leaves the user fairly confused. By adding `' marks, the error will now read: "could not execute: `'" which makes it clear that the command was empty. --- compiler/GHC/SysTools/Process.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/GHC/SysTools/Process.hs b/compiler/GHC/SysTools/Process.hs index bb4129cdd406..ab4c7d675f1b 100644 --- a/compiler/GHC/SysTools/Process.hs +++ b/compiler/GHC/SysTools/Process.hs @@ -228,7 +228,7 @@ handleProc pgm phase_name proc = do then does_not_exist else throwGhcExceptionIO (ProgramError $ show err) - does_not_exist = throwGhcExceptionIO (InstallationError ("could not execute: " ++ pgm)) + does_not_exist = throwGhcExceptionIO (InstallationError ("could not execute: `" ++ pgm ++ "'")) withPipe :: ((Handle, Handle) -> IO a) -> IO a withPipe = bracket createPipe $ \ (readEnd, writeEnd) -> do From 5990cca9f2a17639276f4e4ff90d0845f004c387 Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Mon, 24 Feb 2025 15:03:50 +0900 Subject: [PATCH 136/175] Update cabal 98242d4d8 Add pre and post build hooks 252f5cbc5 No more vowel dropping on macOS 053825f0c Fix static linking on MacOS by preventing '-optl-static' flag --- libraries/Cabal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/Cabal b/libraries/Cabal index adfd46f70d40..98242d4d81e3 160000 --- a/libraries/Cabal +++ b/libraries/Cabal @@ -1 +1 @@ -Subproject commit adfd46f70d40fca9f14603a15a8ac15089debab8 +Subproject commit 98242d4d81e38dd591e212f3a9df7f04215ad1c7 From 538f8278dd367e50a61b6b02052dd0e7366748cf Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Mon, 24 Feb 2025 15:05:35 +0900 Subject: [PATCH 137/175] Add otool, and install_name_tool to lib/settings --- Build.hs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Build.hs b/Build.hs index 16b66c17b622..5325cf139174 100755 --- a/Build.hs +++ b/Build.hs @@ -1007,5 +1007,7 @@ generateSettings ghc_toolchain Settings{..} dst = do let kvs' = Map.insert "Relative Global Package DB" "../pkgs" $ Map.insert "Support SMP" "NO" -- FIXME: this depends on the different ways used to build the RTS! $ Map.insert "RTS ways" "v" -- FIXME: this depends on the different ways used to build the RTS! + $ Map.insert "otool command" "otool" -- FIXME: this should just arguably be a default in the settings in GHC, and not require the settings file? + $ Map.insert "install_name_tool command" "install_name_tool" $ kvs writeFile (dst "lib/settings") (show $ Map.toList kvs') From a84ee3ee7e2fd2fb5fa648e3a8bd699f6c980765 Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Mon, 24 Feb 2025 15:14:49 +0900 Subject: [PATCH 138/175] aarch64-apple-darwin support --- Build.hs | 53 +++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 12 deletions(-) diff --git a/Build.hs b/Build.hs index 5325cf139174..37b43b522330 100755 --- a/Build.hs +++ b/Build.hs @@ -269,7 +269,7 @@ buildGhcStage booting opts cabal ghc0 dst = do , "unlit:unlit" , "hsc2hs:hsc2hs" ] - | otherwise = + | otherwise = [ "ghc-bin:ghc" , "ghc-pkg:ghc-pkg" , "genprimopcode:genprimopcode" @@ -357,7 +357,7 @@ buildGhcStage booting opts cabal ghc0 dst = do prepareGhcSources :: GhcBuildOptions -> FilePath -> IO () prepareGhcSources opts dst = do msg $ " - Preparing sources in " ++ dst ++ "..." - createDirectoryIfMissing True dst + createDirectoryIfMissing True dst createDirectoryIfMissing True (dst "libraries/ghc/MachRegs") cp "./libraries" dst @@ -451,6 +451,10 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst build_dir <- makeAbsolute (dst "cabal") ghcversionh <- makeAbsolute (src_rts "include/ghcversion.h") + -- FIXME: could we build a cross compiler, simply by not reading this from the boot compiler, but passing it in? + target_triple <- ghcTargetTriple ghc + let [arch,vendor,os] = words $ map (\c -> if c == '-' then ' ' else c) target_triple + let cabal_project_rts_path = dst "cabal.project-rts" -- cabal's code handling escaping is bonkers. We need to wrap the whole -- option into \" otherwise it does weird things (like keeping only the @@ -460,11 +464,11 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst let rts_options = [ "package rts" , def_string "ProjectVersion" (Text.unpack (gboVersionInt opts)) - , def_string "RtsWay" "v" -- FIXME - , def_string "HostPlatform" "x86_64-unknown-linux" -- FIXME - , def_string "HostArch" "x86_64" -- FIXME: appropriate value required for the tests - , def_string "HostOS" "linux" -- FIXME: appropriate value required for the tests - , def_string "HostVendor" "unknown" + , def_string "RtsWay" "v" + , def_string "HostPlatform" target_triple + , def_string "HostArch" arch + , def_string "HostOS" os + , def_string "HostVendor" vendor , def_string "BuildPlatform" "FIXME" , def_string "BuildArch" "FIXME" , def_string "BuildOS" "FIXME" @@ -473,7 +477,12 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , def_string "TablesNextToCode" "FIXME" -- Set the namespace for the rts fs functions , def "FS_NAMESPACE" "rts" - , " flags: +use-system-libffi +tables-next-to-code" + -- This is stupid, I can't seem to figure out how to set this in cabal + -- this needs to be fixed in cabal. + , if os == "darwin" + then " flags: +tables-next-to-code +leading-underscore" + else " flags: +tables-next-to-code" + -- FIXME: we should -- FIXME: deal with libffi (add package?) -- -- FIXME: we should make tables-next-to-code optional here and in the @@ -543,14 +552,20 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst -- deriving constants let derived_constants = src_rts "include/DerivedConstants.h" withSystemTempDirectory "derive-constants" $ \tmp_dir -> do + target <- getTarget ghc void $ readCreateProcess (runDeriveConstants derive_constants [ "--gen-header" , "-o", derived_constants - , "--target-os", "linux" -- FIXME + , "--target-os", target , "--tmpdir", tmp_dir - , "--gcc-program", "gcc" -- FIXME + , "--gcc-program", "cc" -- FIXME , "--nm-program", "nm" -- FIXME , "--objdump-program", "objdump" -- FIXME + -- pass `-fcommon` to force symbols into the common section. If they + -- end up in the ro data section `nm` won't list their size, and thus + -- derivedConstants will fail. Recent clang (e.g. 16) will by default + -- use `-fno-common`. + , "--gcc-flag", "-fcommon" , "--gcc-flag", "-I" ++ src_rts "include" , "--gcc-flag", "-I" ++ src_rts , "--gcc-flag", "-I" ++ ghcplatform_dir @@ -570,7 +585,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst -- libraries that aren't built yet. let primops_txt = src "libraries/ghc/GHC/Builtin/primops.txt" let primops_txt_pp = primops_txt <.> ".pp" - primops <- readCreateProcess (shell $ "gcc -E -undef -traditional -P -x c " ++ primops_txt_pp) "" + primops <- readCreateProcess (shell $ "$CC -E -undef -traditional -P -x c " ++ primops_txt_pp) "" writeFile primops_txt primops writeFile (src "libraries/ghc-internal/src/GHC/Internal/Prim.hs") =<< readCreateProcess (runGenPrimop genprimop ["--make-haskell-source"]) primops writeFile (src "libraries/ghc-internal/src/GHC/Internal/PrimopWrappers.hs") =<< readCreateProcess (runGenPrimop genprimop ["--make-haskell-wrappers"]) primops @@ -901,6 +916,13 @@ ghcTargetArchOS ghc = do let os = fromMaybe (error "Couldn't read 'target os' setting") (lookup "target os" is) pure (arch,os) +-- | Retrieve GHC's target as linux, or darwin +getTarget :: Ghc -> IO String +getTarget ghc = ghcTargetArchOS ghc >>= \case + (_,"OSDarwin") -> pure "darwin" + (_,"OSLinux") -> pure "linux" + _ -> error "Unsupported target" + -- | Retrieve GHC's target triple ghcTargetTriple :: Ghc -> IO String ghcTargetTriple ghc = do @@ -979,10 +1001,17 @@ generateSettings ghc_toolchain Settings{..} dst = do createDirectoryIfMissing True (dst "lib") let gen_settings_path = dst "lib/settings.generated" + + mbCC <- lookupEnv "CC" >>= \case + Just cc -> pure ["--cc", cc] + Nothing -> pure [] + mbCXX <- lookupEnv "CXX" >>= \case + Just cxx -> pure ["--cxx", cxx] + Nothing -> pure [] let common_args = [ "--output-settings" , "-o", gen_settings_path - ] + ] ++ mbCC ++ mbCXX let opt m f = fmap f m let args = mconcat (catMaybes From 316032c2110fa52680e499ad0b96fbf954783a18 Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Mon, 24 Feb 2025 16:45:15 +0900 Subject: [PATCH 139/175] [test] add THREADS --- Makefile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 0845d9e27805..af6037281d66 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,7 @@ -export CABAL := $(shell cabal list-bin -v0 --project-dir libraries/Cabal cabal-install:exe:cabal) +CPUS=$(shell mk/detect-cpu-count.sh) + +# Use CPU cores + 1 if not already set +THREADS=${THREADS:-$((CPUS + 1))} all: $(CABAL) GHC=ghc-9.8.4 ./Build.hs @@ -12,8 +15,9 @@ clean: rm -rf _build test: all + echo "using THREADS=${THREADS}" >&2 TEST_HC=`pwd`/_build/bindist/bin/ghc \ METRICS_FILE=`pwd`/_build/test-perf.csv \ SUMMARY_FILE=`pwd`/_build/test-summary.txt \ JUNIT_FILE=`pwd`/_build/test-junit.xml \ - make -C testsuite/tests test + make -C testsuite/tests test THREADS=${THREADS} From 6d7ff2718cd5b7fafff4d75c8b84ce985a82e09a Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Mon, 24 Feb 2025 16:45:35 +0900 Subject: [PATCH 140/175] Make sure cabal update was run before cabal was built. --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index af6037281d66..4761e9773728 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,5 @@ +export CABAL := $(shell cabal update 2>&1 >/dev/null && cabal list-bin -v0 --project-dir libraries/Cabal cabal-install:exe:cabal) + CPUS=$(shell mk/detect-cpu-count.sh) # Use CPU cores + 1 if not already set From e6ab8ddab27b0211d410f48202246d0598c50613 Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Mon, 24 Feb 2025 16:45:58 +0900 Subject: [PATCH 141/175] Logic to track boot --- Makefile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 4761e9773728..d07edfae6fe9 100644 --- a/Makefile +++ b/Makefile @@ -5,15 +5,20 @@ CPUS=$(shell mk/detect-cpu-count.sh) # Use CPU cores + 1 if not already set THREADS=${THREADS:-$((CPUS + 1))} -all: $(CABAL) +all: $(CABAL) ./booted GHC=ghc-9.8.4 ./Build.hs cabal: $(CABAL) - + $(CABAL): cabal build --project-dir libraries/Cabal cabal-install:exe:cabal +./booted: + ./boot + touch $@ + clean: + rm -f ./booted rm -rf _build test: all From e72d3572c5820869224b6c2f276a7ca2caef3f4e Mon Sep 17 00:00:00 2001 From: Moritz Angermann Date: Mon, 24 Feb 2025 17:48:50 +0900 Subject: [PATCH 142/175] Let's try to set CC and CXX --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 591dae769c80..0c8875257dc8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,6 +39,9 @@ jobs: run: make cabal - name: Build the bindist + env: + CC: gcc + CXX: g++ run: make - name: Upload artifacts From ee031be400ad98b4e5276673c71608d43f0d9674 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Mon, 24 Feb 2025 14:28:25 +0100 Subject: [PATCH 143/175] Add link to cabal issue --- Build.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/Build.hs b/Build.hs index 37b43b522330..af6d74c9f09f 100755 --- a/Build.hs +++ b/Build.hs @@ -496,6 +496,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst -- flags: +use-system-libffi -- flags: +tables-next-to-code -- Apparently it makes it ignore the first set of flags... + -- See https://github.com/haskell/cabal/issues/10767 ] makeCabalProject cabal_project_rts_path $ From 43d7f9b214a785284b73391580dd32694e9b8235 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 26 Feb 2025 09:42:22 +0100 Subject: [PATCH 144/175] Fix triple warning --- Build.hs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Build.hs b/Build.hs index af6d74c9f09f..0b3a0f49a995 100755 --- a/Build.hs +++ b/Build.hs @@ -453,7 +453,10 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst -- FIXME: could we build a cross compiler, simply by not reading this from the boot compiler, but passing it in? target_triple <- ghcTargetTriple ghc - let [arch,vendor,os] = words $ map (\c -> if c == '-' then ' ' else c) target_triple + let to_triple = \case + [arch,vendor,os] -> (arch,vendor,os) + t -> error $ "Triple expected but got: " ++ show t + let (arch,vendor,os) = to_triple $ words $ map (\c -> if c == '-' then ' ' else c) target_triple let cabal_project_rts_path = dst "cabal.project-rts" -- cabal's code handling escaping is bonkers. We need to wrap the whole From 420310e09a6d86b3c2a5c102736bb19213927399 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 26 Feb 2025 10:09:17 +0100 Subject: [PATCH 145/175] Fix libffi linking --- rts/rts.cabal | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rts/rts.cabal b/rts/rts.cabal index 249e814810ec..612eae6bb273 100644 --- a/rts/rts.cabal +++ b/rts/rts.cabal @@ -613,11 +613,13 @@ library Jumps_V32.cmm Jumps_V64.cmm + -- we always link against libffi, even without libffi-adjustors + extra-libraries: ffi + extra-libraries-static: ffi + -- Adjustor stuff if flag(libffi-adjustors) c-sources: adjustor/LibffiAdjustor.c - extra-libraries: ffi - extra-libraries-static: ffi else -- Use GHC's native adjustors if arch(i386) @@ -633,8 +635,6 @@ library -- fall back to the LibffiAdjustor if neither i386, or x86_64 if !arch(x86_64) && !arch(i386) c-sources: adjustor/LibffiAdjustor.c - extra-libraries: ffi - extra-libraries-static: ffi -- Use assembler STG entrypoint on architectures where it is used if arch(ppc) || arch(ppc64) || arch(s390x) || arch(riscv64) || arch(loongarch64) From 6c9548cc66fc036b28f3faf982d9d57fd8e3652b Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 26 Feb 2025 11:24:33 +0100 Subject: [PATCH 146/175] Display unit-id on module name mistmatch https://github.com/stable-haskell/ghc/issues/19 --- compiler/GHC/Iface/Errors/Ppr.hs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/compiler/GHC/Iface/Errors/Ppr.hs b/compiler/GHC/Iface/Errors/Ppr.hs index d9f07343d245..c9d842fb1644 100644 --- a/compiler/GHC/Iface/Errors/Ppr.hs +++ b/compiler/GHC/Iface/Errors/Ppr.hs @@ -336,8 +336,8 @@ hiModuleNameMismatchWarn requested_mod read_mod ] ] | otherwise = - -- ToDo: This will fail to have enough qualification when the package IDs - -- are the same + -- Display fully qualified unit names by enabling ppr-debug + updSDocContext (\ctx -> ctx { sdocPprDebug = True}) $ withPprStyle (mkUserStyle alwaysQualify AllTheWay) $ -- we want the Modules below to be qualified with package names, -- so reset the NamePprCtx setting. @@ -345,7 +345,6 @@ hiModuleNameMismatchWarn requested_mod read_mod , ppr requested_mod , text "differs from name found in the interface file" , ppr read_mod - , parens (text "if these names look the same, try again with -dppr-debug") ] dynamicHashMismatchError :: Module -> ModLocation -> SDoc From b0e693700f140cb7a04c15bb407b32f8cf471b5b Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 26 Feb 2025 11:32:02 +0100 Subject: [PATCH 147/175] Replace $CC with cc --- Build.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Build.hs b/Build.hs index 0b3a0f49a995..694f67abb01c 100755 --- a/Build.hs +++ b/Build.hs @@ -589,7 +589,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst -- libraries that aren't built yet. let primops_txt = src "libraries/ghc/GHC/Builtin/primops.txt" let primops_txt_pp = primops_txt <.> ".pp" - primops <- readCreateProcess (shell $ "$CC -E -undef -traditional -P -x c " ++ primops_txt_pp) "" + primops <- readCreateProcess (shell $ "cc -E -undef -traditional -P -x c " ++ primops_txt_pp) "" writeFile primops_txt primops writeFile (src "libraries/ghc-internal/src/GHC/Internal/Prim.hs") =<< readCreateProcess (runGenPrimop genprimop ["--make-haskell-source"]) primops writeFile (src "libraries/ghc-internal/src/GHC/Internal/PrimopWrappers.hs") =<< readCreateProcess (runGenPrimop genprimop ["--make-haskell-wrappers"]) primops From afb7aa14e433e3b635d18acc691772adafdb9990 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 26 Feb 2025 12:11:17 +0100 Subject: [PATCH 148/175] Mark T25382 as broken See https://github.com/stable-haskell/ghc/issues/28 --- testsuite/tests/driver/all.T | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testsuite/tests/driver/all.T b/testsuite/tests/driver/all.T index 9f794b99e51e..1b361c3424de 100644 --- a/testsuite/tests/driver/all.T +++ b/testsuite/tests/driver/all.T @@ -330,4 +330,4 @@ test('T23944', [unless(have_dynamic(), skip), extra_files(['T23944A.hs'])], mult test('T24286', [cxx_src, unless(have_profiling(), skip), extra_files(['T24286.cpp'])], compile, ['-prof -no-hs-main']) test('T24839', [unless(arch('x86_64') or arch('aarch64'), skip), extra_files(["t24839_sub.S"])], compile_and_run, ['t24839_sub.S']) test('t25150', [extra_files(["t25150"])], multimod_compile, ['Main.hs', '-v0 -working-dir t25150/dir a.c']) -test('T25382', normal, makefile_test, []) +test('T25382', expect_broken(28), makefile_test, []) From ee0edb6ba2d7c5fb5c23d6aaf0ce87bc70bfc23c Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 26 Feb 2025 14:05:15 +0100 Subject: [PATCH 149/175] Skip more tests by adding rts-way-related predicates --- libraries/base/tests/IO/T12010/test.T | 1 + libraries/base/tests/IO/all.T | 2 +- testsuite/driver/testlib.py | 7 +++++++ testsuite/tests/rts/all.T | 7 +++++-- testsuite/tests/rts/linker/all.T | 7 ++++++- 5 files changed, 20 insertions(+), 4 deletions(-) diff --git a/libraries/base/tests/IO/T12010/test.T b/libraries/base/tests/IO/T12010/test.T index e33e69036a8c..bb926dc72dd8 100644 --- a/libraries/base/tests/IO/T12010/test.T +++ b/libraries/base/tests/IO/T12010/test.T @@ -4,5 +4,6 @@ test('T12010', extra_ways(['threaded1']), when(wordsize(32), fragile(16572)), js_broken(22374), + req_target_debug_rts, cmd_prefix('WAY_FLAGS="' + ' '.join(config.way_flags['threaded1']) + '"')], makefile_test, []) diff --git a/libraries/base/tests/IO/all.T b/libraries/base/tests/IO/all.T index 5b28156c96bf..992b5dfbac42 100644 --- a/libraries/base/tests/IO/all.T +++ b/libraries/base/tests/IO/all.T @@ -114,7 +114,7 @@ test('countReaders001', js_broken(22261), compile_and_run, ['']) test('concio001', [normal, multi_cpu_race], makefile_test, ['test.concio001']) -test('concio001.thr', [extra_files(['concio001.hs']), multi_cpu_race], +test('concio001.thr', [extra_files(['concio001.hs']), multi_cpu_race, req_target_threaded_rts], makefile_test, ['test.concio001.thr']) test('T2122', [], compile_and_run, ['']) diff --git a/testsuite/driver/testlib.py b/testsuite/driver/testlib.py index 70e778bbad53..9fd18503df81 100644 --- a/testsuite/driver/testlib.py +++ b/testsuite/driver/testlib.py @@ -363,6 +363,13 @@ def req_target_debug_rts( name, opts ): if not config.debug_rts: opts.skip = True +def req_target_threaded_rts( name, opts ): + # FIXME: this is probably wrong: we should have a different flag for the + # compiler's rts and the target rts... + if not config.ghc_with_threaded_rts: + opts.skip = True + + def req_target_smp( name, opts ): """ Mark a test as requiring smp when run on the target. If the target does diff --git a/testsuite/tests/rts/all.T b/testsuite/tests/rts/all.T index 64022e31146e..23bc7e698f14 100644 --- a/testsuite/tests/rts/all.T +++ b/testsuite/tests/rts/all.T @@ -238,6 +238,7 @@ test('return_mem_to_os', normal, compile_and_run, ['']) test('T4850', [ when(opsys('mingw32'), expect_broken(4850)) , js_broken(22261) # FFI "dynamic" convention unsupported + , req_target_debug_rts ], makefile_test, ['T4850']) def config_T5250(name, opts): @@ -500,6 +501,7 @@ test('keep-cafs-fail', filter_stdout_lines('Evaluated a CAF|exit.*'), ignore_stderr, # on OS X the shell emits an "Abort trap" message to stderr req_rts_linker, + req_target_debug_rts ], makefile_test, ['KeepCafsFail']) @@ -509,7 +511,8 @@ test('keep-cafs', 'KeepCafs2.hs', 'KeepCafsMain.hs']), when(opsys('mingw32'), expect_broken (5987)), when(opsys('freebsd') or opsys('openbsd'), expect_broken(16035)), - req_rts_linker + req_rts_linker, + req_target_debug_rts ], makefile_test, ['KeepCafs']) @@ -519,7 +522,7 @@ test('T11829', [ req_c, check_errmsg("This is a test"), when(arch('wasm32'), fra ['T11829_c.cpp -package system-cxx-std-lib']) test('T16514', [req_c, omit_ghci], compile_and_run, ['T16514_c.c']) -test('test-zeroongc', extra_run_opts('-DZ'), compile_and_run, ['-debug']) +test('test-zeroongc', [extra_run_opts('-DZ'), req_target_debug_rts], compile_and_run, ['-debug']) test('T13676', [when(opsys('mingw32'), expect_broken(17447)), diff --git a/testsuite/tests/rts/linker/all.T b/testsuite/tests/rts/linker/all.T index e88594b1025c..a5e6e8ddac5c 100644 --- a/testsuite/tests/rts/linker/all.T +++ b/testsuite/tests/rts/linker/all.T @@ -123,14 +123,17 @@ test('linker_unload_native', ###################################### test('linker_error1', [extra_files(['linker_error.c']), js_skip, # dynamic linking not supported by the JS backend + req_target_debug_rts, ignore_stderr], makefile_test, ['linker_error1']) test('linker_error2', [extra_files(['linker_error.c']), js_skip, # dynamic linking not supported by the JS backend + req_target_debug_rts, ignore_stderr], makefile_test, ['linker_error2']) test('linker_error3', [extra_files(['linker_error.c']), js_skip, # dynamic linking not supported by the JS backend + req_target_debug_rts, ignore_stderr], makefile_test, ['linker_error3']) ###################################### @@ -149,7 +152,9 @@ test('rdynamic', [ unless(opsys('linux') or opsys('mingw32'), skip) test('T7072', [extra_files(['load-object.c', 'T7072.c']), unless(opsys('linux'), skip), - req_rts_linker], + req_rts_linker, + req_target_debug_rts + ], makefile_test, ['T7072']) test('T20494', [req_rts_linker, when(opsys('linux') and not ghc_dynamic(), expect_broken(20706))], From 7a5e21ebaeea800d1a8bf951006f8648c14a717b Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 26 Feb 2025 14:11:13 +0100 Subject: [PATCH 150/175] Fix flushing in more tests --- testsuite/tests/ffi/should_run/T1288_c.c | 1 + testsuite/tests/ffi/should_run/T1288_ghci_c.c | 1 + testsuite/tests/ffi/should_run/T2276_c.c | 1 + testsuite/tests/ffi/should_run/T2276_ghci_c.c | 1 + 4 files changed, 4 insertions(+) diff --git a/testsuite/tests/ffi/should_run/T1288_c.c b/testsuite/tests/ffi/should_run/T1288_c.c index 2fe90a989b53..ac5f41314b9e 100644 --- a/testsuite/tests/ffi/should_run/T1288_c.c +++ b/testsuite/tests/ffi/should_run/T1288_c.c @@ -3,4 +3,5 @@ void test(int arg) { printf("The argument passed was %i\n", arg ); + fflush(NULL); } diff --git a/testsuite/tests/ffi/should_run/T1288_ghci_c.c b/testsuite/tests/ffi/should_run/T1288_ghci_c.c index 2fe90a989b53..ac5f41314b9e 100644 --- a/testsuite/tests/ffi/should_run/T1288_ghci_c.c +++ b/testsuite/tests/ffi/should_run/T1288_ghci_c.c @@ -3,4 +3,5 @@ void test(int arg) { printf("The argument passed was %i\n", arg ); + fflush(NULL); } diff --git a/testsuite/tests/ffi/should_run/T2276_c.c b/testsuite/tests/ffi/should_run/T2276_c.c index 2fe90a989b53..ac5f41314b9e 100644 --- a/testsuite/tests/ffi/should_run/T2276_c.c +++ b/testsuite/tests/ffi/should_run/T2276_c.c @@ -3,4 +3,5 @@ void test(int arg) { printf("The argument passed was %i\n", arg ); + fflush(NULL); } diff --git a/testsuite/tests/ffi/should_run/T2276_ghci_c.c b/testsuite/tests/ffi/should_run/T2276_ghci_c.c index 2fe90a989b53..ac5f41314b9e 100644 --- a/testsuite/tests/ffi/should_run/T2276_ghci_c.c +++ b/testsuite/tests/ffi/should_run/T2276_ghci_c.c @@ -3,4 +3,5 @@ void test(int arg) { printf("The argument passed was %i\n", arg ); + fflush(NULL); } From 75f7d87953b4905586344ceabfc64ffdf9f5ffaa Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 26 Feb 2025 14:22:27 +0100 Subject: [PATCH 151/175] Indicate that T22012 is fragile due to GHC#23043 --- testsuite/tests/rts/all.T | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testsuite/tests/rts/all.T b/testsuite/tests/rts/all.T index 23bc7e698f14..aa7941e11e43 100644 --- a/testsuite/tests/rts/all.T +++ b/testsuite/tests/rts/all.T @@ -605,7 +605,7 @@ test('decodeMyStack_emptyListForMissingFlag', test('T20201a', [js_skip, exit_code(1)], compile_and_run, ['-with-rtsopts -AturtlesM']) test('T20201b', [js_skip, exit_code(1)], compile_and_run, ['-with-rtsopts -A64z']) -test('T22012', [js_skip, extra_ways(['ghci'])], compile_and_run, ['T22012_c.c']) +test('T22012', [js_skip, fragile(23043), extra_ways(['ghci'])], compile_and_run, ['T22012_c.c']) # Skip for JS platform as the JS RTS is always single threaded test('T22795a', [only_ways(['normal']), js_skip, req_ghc_with_threaded_rts], compile_and_run, ['-threaded']) From 5a0bf3001a3f13b6ca559e1907c7a8871a16cd34 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 26 Feb 2025 14:36:05 +0100 Subject: [PATCH 152/175] Testsuite: remove expect_broken for static config We basically revert https://gitlab.haskell.org/ghc/ghc/-/merge_requests/9418 It doesn't fail for our statically-built GHC, so it could be Alpine/musl specific. --- testsuite/tests/ghci/linking/all.T | 3 +-- testsuite/tests/ghci/linking/dyn/all.T | 4 +--- testsuite/tests/ghci/prog001/prog001.T | 3 +-- testsuite/tests/ghci/prog002/prog002.T | 3 +-- testsuite/tests/ghci/prog010/all.T | 3 +-- testsuite/tests/ghci/scripts/all.T | 3 +-- testsuite/tests/rts/all.T | 3 +-- 7 files changed, 7 insertions(+), 15 deletions(-) diff --git a/testsuite/tests/ghci/linking/all.T b/testsuite/tests/ghci/linking/all.T index a2b45e0e09f8..df46765d9afc 100644 --- a/testsuite/tests/ghci/linking/all.T +++ b/testsuite/tests/ghci/linking/all.T @@ -32,8 +32,7 @@ test('ghcilink005', when(unregisterised(), fragile(16085)), unless(doing_ghci, skip), req_dynamic_lib_support, - req_interp, - when(opsys('linux') and not ghc_dynamic(), expect_broken(20706))], + req_interp], makefile_test, ['ghcilink005']) test('ghcilink006', diff --git a/testsuite/tests/ghci/linking/dyn/all.T b/testsuite/tests/ghci/linking/dyn/all.T index b215f6b7202b..5d834099deb1 100644 --- a/testsuite/tests/ghci/linking/dyn/all.T +++ b/testsuite/tests/ghci/linking/dyn/all.T @@ -3,7 +3,6 @@ setTestOpts(req_dynamic_lib_support) test('load_short_name', [ extra_files(['A.c']) , unless(doing_ghci, skip) , req_c - , when(opsys('linux') and not ghc_dynamic(), expect_broken(20706)) ], makefile_test, ['load_short_name']) @@ -12,8 +11,7 @@ test('T1407', unless(doing_ghci, skip), pre_cmd('$MAKE -s --no-print-directory compile_libT1407'), extra_hc_opts('-L"$PWD/T1407dir"'), - js_broken(22359), - when(opsys('linux') and not ghc_dynamic(), expect_broken(20706))], + js_broken(22359)], makefile_test, []) test('T3242', diff --git a/testsuite/tests/ghci/prog001/prog001.T b/testsuite/tests/ghci/prog001/prog001.T index f00b0b6a9865..519ee2e38211 100644 --- a/testsuite/tests/ghci/prog001/prog001.T +++ b/testsuite/tests/ghci/prog001/prog001.T @@ -3,6 +3,5 @@ test('prog001', when(arch('arm'), fragile(17555)), cmd_prefix('ghciWayFlags=' + config.ghci_way_flags), req_interp, - unless(opsys('mingw32') or not config.have_RTS_linker, extra_ways(['ghci-ext'])), - when(opsys('linux') and not ghc_dynamic(), expect_broken(20706))], + unless(opsys('mingw32') or not config.have_RTS_linker, extra_ways(['ghci-ext']))], ghci_script, ['prog001.script']) diff --git a/testsuite/tests/ghci/prog002/prog002.T b/testsuite/tests/ghci/prog002/prog002.T index 83f8d0d92e65..3e25bb455b00 100644 --- a/testsuite/tests/ghci/prog002/prog002.T +++ b/testsuite/tests/ghci/prog002/prog002.T @@ -1,4 +1,3 @@ test('prog002', [extra_files(['../shell.hs', 'A1.hs', 'A2.hs', 'B.hs', 'C.hs', 'D.hs']), - cmd_prefix('ghciWayFlags=' + config.ghci_way_flags), - when(opsys('linux') and not ghc_dynamic(), expect_broken(20706))], + cmd_prefix('ghciWayFlags=' + config.ghci_way_flags)], ghci_script, ['prog002.script']) diff --git a/testsuite/tests/ghci/prog010/all.T b/testsuite/tests/ghci/prog010/all.T index 103ff8338196..d30de29400ae 100644 --- a/testsuite/tests/ghci/prog010/all.T +++ b/testsuite/tests/ghci/prog010/all.T @@ -1,5 +1,4 @@ test('ghci.prog010', [cmd_prefix('ghciWayFlags=' + config.ghci_way_flags), - extra_files(['../shell.hs', 'A.hs', 'B.hs']), - when(opsys('linux') and not ghc_dynamic(), expect_broken(20706))], + extra_files(['../shell.hs', 'A.hs', 'B.hs'])], ghci_script, ['ghci.prog010.script']) diff --git a/testsuite/tests/ghci/scripts/all.T b/testsuite/tests/ghci/scripts/all.T index da5685011d3d..b1f36e8d1b32 100755 --- a/testsuite/tests/ghci/scripts/all.T +++ b/testsuite/tests/ghci/scripts/all.T @@ -163,8 +163,7 @@ test('T6106', [extra_files(['../shell.hs']), test('T6105', normal, ghci_script, ['T6105.script']) test('T7117', normal, ghci_script, ['T7117.script']) test('ghci058', [extra_files(['../shell.hs']), - cmd_prefix('ghciWayFlags=' + config.ghci_way_flags), - when(opsys('linux') and not ghc_dynamic(), expect_broken(20706))], + cmd_prefix('ghciWayFlags=' + config.ghci_way_flags)], ghci_script, ['ghci058.script']) test('T7587', normal, ghci_script, ['T7587.script']) test('T7688', normal, ghci_script, ['T7688.script']) diff --git a/testsuite/tests/rts/all.T b/testsuite/tests/rts/all.T index aa7941e11e43..8579be3f9322 100644 --- a/testsuite/tests/rts/all.T +++ b/testsuite/tests/rts/all.T @@ -526,8 +526,7 @@ test('test-zeroongc', [extra_run_opts('-DZ'), req_target_debug_rts], compile_and test('T13676', [when(opsys('mingw32'), expect_broken(17447)), - extra_files(['T13676.hs']), - when(opsys('linux') and not ghc_dynamic(), expect_broken(20706))], + extra_files(['T13676.hs'])], ghci_script, ['T13676.script']) test('InitEventLogging', [ only_ways(['normal']) From 5359eccaafee466b49f18677d6e5e7527c76584f Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 26 Feb 2025 17:45:03 +0100 Subject: [PATCH 153/175] More unexpected passes fixed (see previous commit) --- testsuite/tests/plugins/all.T | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/testsuite/tests/plugins/all.T b/testsuite/tests/plugins/all.T index a7bad499350a..40d6439828d9 100644 --- a/testsuite/tests/plugins/all.T +++ b/testsuite/tests/plugins/all.T @@ -131,7 +131,7 @@ test('T10294a', pre_cmd('$MAKE -s --no-print-directory -C annotation-plugin package.T10294a TOP={top}')], makefile_test, []) -test('frontend01', [extra_files(['FrontendPlugin.hs']), when(opsys('linux') and not ghc_dynamic(), expect_broken(20706))], +test('frontend01', [extra_files(['FrontendPlugin.hs'])], makefile_test, []) test('T11244', @@ -360,9 +360,7 @@ test('plugins-external', test('test-phase-hooks-plugin', [extra_files(['hooks-plugin/']), - pre_cmd('$MAKE -s --no-print-directory -C hooks-plugin package.test-phase-hooks-plugin TOP={top}'), - - when(opsys('linux') and not ghc_dynamic(), expect_broken(20706))], + pre_cmd('$MAKE -s --no-print-directory -C hooks-plugin package.test-phase-hooks-plugin TOP={top}')], compile, ['-package-db hooks-plugin/pkg.test-phase-hooks-plugin/local.package.conf -fplugin Hooks.PhasePlugin -package hooks-plugin ' + config.plugin_way_flags]) From 169699308e274732865d4b4726fc48af16e0f73f Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 26 Feb 2025 17:44:27 +0100 Subject: [PATCH 154/175] Fix type name for plugin loading failure Especially wrong for FrontendPlugins --- compiler/GHC/Runtime/Loader.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/GHC/Runtime/Loader.hs b/compiler/GHC/Runtime/Loader.hs index 5368d01e34ed..06347d226d41 100644 --- a/compiler/GHC/Runtime/Loader.hs +++ b/compiler/GHC/Runtime/Loader.hs @@ -220,7 +220,7 @@ loadPlugin' occ_name plugin_name hsc_env mod_name [ text "The value", ppr name , text "with type", ppr actual_type , text "did not have the type" - , text "GHC.Plugins.Plugin" + , ppr (mkTyConTy plugin_tycon) , text "as required"]) Right (plugin, links, pkgs) -> return (plugin, mod_iface, links, pkgs) } } } } } From a6742b927cc86cb20383dc65550dc1e9f8809e85 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Thu, 27 Feb 2025 10:57:49 +0100 Subject: [PATCH 155/175] Fix test output --- testsuite/tests/plugins/plugins02.stderr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testsuite/tests/plugins/plugins02.stderr b/testsuite/tests/plugins/plugins02.stderr index 2ea9331d3ec2..3d035ef35ee2 100644 --- a/testsuite/tests/plugins/plugins02.stderr +++ b/testsuite/tests/plugins/plugins02.stderr @@ -1 +1 @@ -: The value Simple.BadlyTypedPlugin.plugin with type GHC.Internal.Types.Int did not have the type GHC.Plugins.Plugin as required +: The value Simple.BadlyTypedPlugin.plugin with type GHC.Internal.Types.Int did not have the type GHC.Driver.Plugins.Plugin as required From 691101686592af0a52a7bc4f63a55455985dc39e Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Thu, 27 Feb 2025 11:07:07 +0100 Subject: [PATCH 156/175] Add comment --- rts/rts.cabal | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rts/rts.cabal b/rts/rts.cabal index 612eae6bb273..193e8dd79440 100644 --- a/rts/rts.cabal +++ b/rts/rts.cabal @@ -613,7 +613,9 @@ library Jumps_V32.cmm Jumps_V64.cmm - -- we always link against libffi, even without libffi-adjustors + -- we always link against libffi, even without libffi-adjustors enabled. + -- libffi is used by the Interpreter and some of its symbols are declared + -- in RtsSymbols.c extra-libraries: ffi extra-libraries-static: ffi From db67fa30db0161c72d35e8c5167e8113ab71500e Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Mon, 7 Apr 2025 14:22:18 +0200 Subject: [PATCH 157/175] Add zig cc support It requires zig 0.14 at least to compile xxhash.h which uses evex512. Wrappers are needed because zig needs its first argument (`cc` or `c++`) always. Trying to pass it as CFLAGS fails with: ghc-toolchain, GHC's response files used to call the linker (this arg isn't supported by zig in the response file), configure scripts that call the C compiler without passing the specified CFLAGS. Use the wrappers like this: CC=/path/to/zig-cc CXX=/path/to/zig-c++ make --- Build.hs | 2 +- compiler/GHC/Driver/Session.hs | 4 +++- zig-c++ | 3 +++ zig-cc | 2 ++ 4 files changed, 9 insertions(+), 2 deletions(-) create mode 100755 zig-c++ create mode 100755 zig-cc diff --git a/Build.hs b/Build.hs index 694f67abb01c..7ce66d8c3aa3 100755 --- a/Build.hs +++ b/Build.hs @@ -549,7 +549,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst ghcplatform_h <- readCreateProcess (shell ("find " ++ build_dir ++ " -name ghcplatform.h")) "" case ghcplatform_h of "" -> do - putStrLn "Couldn't find ghcplatform.h" + putStrLn $ "Couldn't find ghcplatform.h. Look into " ++ dst ++ "rts-conf.{stdout,stderr}" exitFailure d -> pure (takeDirectory d) diff --git a/compiler/GHC/Driver/Session.hs b/compiler/GHC/Driver/Session.hs index bf1db99c3b37..9a34afc9a891 100644 --- a/compiler/GHC/Driver/Session.hs +++ b/compiler/GHC/Driver/Session.hs @@ -3371,7 +3371,9 @@ picCCOpts dflags = else []) -- gcc may be configured to have PIC on by default, let's be -- explicit here, see #15847 - | otherwise -> ["-fno-PIC"] + -- FIXME: actually no, because -fPIC may be required for ASLR too! + -- Zig cc doesn't support `-fno-pic` in this case + | otherwise -> [] -- ["-fno-PIC"] pieCCLDOpts :: DynFlags -> [String] pieCCLDOpts dflags diff --git a/zig-c++ b/zig-c++ new file mode 100755 index 000000000000..66701c9b1837 --- /dev/null +++ b/zig-c++ @@ -0,0 +1,3 @@ +#!/bin/sh +zig c++ $@ + diff --git a/zig-cc b/zig-cc new file mode 100755 index 000000000000..c2b79d642979 --- /dev/null +++ b/zig-cc @@ -0,0 +1,2 @@ +#!/bin/sh +zig cc $@ From 1f9479d7634b45c4b80bec9c4d62df7b3665eed4 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Thu, 10 Apr 2025 16:05:26 +0200 Subject: [PATCH 158/175] Fix build without polluting the global store --- Build.hs | 97 ++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 63 insertions(+), 34 deletions(-) diff --git a/Build.hs b/Build.hs index 7ce66d8c3aa3..1013423d3c85 100755 --- a/Build.hs +++ b/Build.hs @@ -41,7 +41,7 @@ main = do ghc_path <- fromMaybe "ghc" <$> lookupEnv "GHC" findExecutable ghc_path >>= \case Nothing -> error ("Couldn't find GHC: " ++ show ghc_path) - Just x -> pure (Ghc x) + Just x -> pure (Ghc x []) cabal <- do cabal_path <- fromMaybe "cabal" <$> lookupEnv "CABAL" @@ -67,16 +67,15 @@ main = do cp "_build/stage0/lib/template-hsc.h" "_build/stage1/lib/template-hsc.h" cp "_build/stage0/pkgs/*" "_build/stage1/pkgs/" - ghc1 <- Ghc <$> makeAbsolute "_build/stage1/bin/ghc" + ghc1 <- Ghc <$> makeAbsolute "_build/stage1/bin/ghc" <*> pure [] ghcPkg1 <- GhcPkg <$> makeAbsolute "_build/stage1/bin/ghc-pkg" deriveConstants <- DeriveConstants <$> makeAbsolute "_build/stage1/bin/deriveConstants" genapply <- GenApply <$> makeAbsolute "_build/stage1/bin/genapply" genprimop <- GenPrimop <$> makeAbsolute "_build/stage1/bin/genprimopcode" ghcToolchain <- GhcToolchain <$> makeAbsolute "_build/stage1/bin/ghc-toolchain" - -- generate settings based on stage1 compiler settings: stage1 should never be - -- a cross-compiler! Hence we reuse the same target platform as the bootstrap - -- compiler. + -- generate settings for the stage1 compiler: we want a non cross-compiler so + -- we reuse the target from stage0 (bootstrap compiler). stage0_target_triple <- ghcTargetTriple ghc0 let stage1_settings = emptySettings { settingsTriple = Just stage0_target_triple @@ -88,19 +87,36 @@ main = do msg "Building stage2 GHC program" createDirectoryIfMissing True "_build/stage2" - ghc1' <- Ghc <$> makeAbsolute "_build/stage1/bin/ghc" + ghc1' <- Ghc <$> makeAbsolute "_build/stage1/bin/ghc" <*> pure [] buildGhcStage2 defaultGhcBuildOptions cabal ghc1' "_build/stage2/" - -- Reuse stage1 settings for stage2 and copy stage1's built boot package for - -- stage2 to use. + -- We keep the packages and the settings used to build the stage2 compiler. + -- They can be used to build plugins to use with fplugin-library and they can + -- also be used with the internal interpreter createDirectoryIfMissing True "_build/stage2/lib/" cp "_build/stage1/pkgs/*" "_build/stage2/pkgs" cp "_build/stage1/lib/settings" "_build/stage2/lib/settings" - -- TODO: in the future we want to generate different settings for cross - -- targets and build boot libraries with stage2 using these settings. In any - -- case, we need non-cross boot packages to build plugins for use with - -- -fplugin-library. + -- Now we build extra targets. Ideally those should be built on demand... + createDirectoryIfMissing True "_build/stage2/targets/" + let targets = + [-- (,) "aarch64-linux" emptySettings + -- { settingsTriple = Just "aarch64-linux" + -- , settingsCc = ProgOpt (Just "zig-cc") (Just ["-target", "aarch64-linux-gnu"]) + -- , settingsCxx = ProgOpt (Just "zig-c++") (Just ["-target", "aarch64-linux-gnu"]) + -- } +-- , (,) "javascript" emptySettings +-- { settingsTriple = Just "javascript-unknown-ghcjs" +-- , settingsCc = ProgOpt (Just "emcc") Nothing +-- } + ] + forM_ targets $ \(target,settings) -> do + msg $ "Bootstrapping target: " <> target + target_dir <- makeAbsolute ("_build/stage2/targets" target) + createDirectoryIfMissing True target_dir + generateSettings ghcToolchain settings target_dir + ghc2 <- Ghc <$> makeAbsolute "_build/stage2/bin/ghc" <*> pure ["-B"++ target_dir "lib"] + buildBootLibraries cabal ghc2 ghcPkg1 deriveConstants genapply genprimop defaultGhcBuildOptions target_dir -- Finally create bindist directory @@ -108,9 +124,11 @@ main = do createDirectoryIfMissing True "_build/bindist/lib/" createDirectoryIfMissing True "_build/bindist/bin/" createDirectoryIfMissing True "_build/bindist/pkgs/" + createDirectoryIfMissing True "_build/bindist/targets/" cp "_build/stage2/bin/*" "_build/bindist/bin/" cp "_build/stage2/lib/*" "_build/bindist/lib/" cp "_build/stage2/pkgs/*" "_build/bindist/pkgs/" + cp "_build/stage2/targets/*" "_build/bindist/targets/" cp "driver/ghc-usage.txt" "_build/bindist/lib/" cp "driver/ghci-usage.txt" "_build/bindist/lib/" @@ -448,9 +466,13 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst -- Build the RTS src_rts <- makeAbsolute (src "libraries/rts") - build_dir <- makeAbsolute (dst "cabal") + build_dir <- makeAbsolute (dst "cabal" "build") + store_dir <- makeAbsolute (dst "cabal" "store") ghcversionh <- makeAbsolute (src_rts "include/ghcversion.h") + createDirectoryIfMissing True build_dir + createDirectoryIfMissing True store_dir + -- FIXME: could we build a cross compiler, simply by not reading this from the boot compiler, but passing it in? target_triple <- ghcTargetTriple ghc let to_triple = \case @@ -503,7 +525,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst ] makeCabalProject cabal_project_rts_path $ - [ "package-dbs: clear, global" + [ "package-dbs: clear, store" , "" , "packages:" , " " ++ src "libraries/rts" @@ -523,7 +545,8 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst ] ++ rts_options let build_rts_cmd = runCabal cabal - [ "build" + [ "--store-dir=" ++ store_dir + , "build" , "--project-file=" ++ cabal_project_rts_path , "rts" , "--with-compiler=" ++ ghcPath ghc @@ -597,9 +620,8 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst -- build boot libraries: ghc-internal, base... let cabal_project_bootlibs_path = dst "cabal-project-boot-libs" makeCabalProject cabal_project_bootlibs_path $ - [ "package-dbs: clear, global" - , "" - , "packages:" + [-- "package-dbs: clear, store" -- this makes cabal fail because it can't find a dubious database in a temp directory + "packages:" , " " ++ src "libraries/rts" , " " ++ src "libraries/ghc-prim" , " " ++ src "libraries/ghc-internal" @@ -688,7 +710,8 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst let boot_libs_env = dst "boot-libs.env" let build_boot_cmd = runCabal cabal - [ "install" + [ "--store-dir=" ++ store_dir + , "install" , "--lib" , "--package-env=" ++ boot_libs_env , "--force-reinstalls" @@ -761,18 +784,26 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst -- The libraries have been installed globally. boot_libs_env_lines <- lines <$> readFile boot_libs_env - -- FIXME: Sometimes the package environment contains the path to the global db, - -- sometimes not... I don't know why yet. - (global_db,pkg_ids) <- case drop 2 boot_libs_env_lines of + (global_db,pkg_ids) <- case drop 2 boot_libs_env_lines of -- drop "clear-package-db\nglobal-package-db" [] -> error "Unexpected empty package environment" (x:xs) + -- FIXME: Sometimes the package environment contains the path to the global db, + -- sometimes not... I don't know why yet. | not ("package-db" `List.isPrefixOf` x) -> do putStrLn "For some reason cabal-install didn't generate a valid package environment (package-db is missing)." putStrLn "It happens sometimes for unknown reasons... Rerun 'make' to workaround this..." exitFailure - | otherwise -> pure (drop 11 x, map (drop 11) xs) - putStrLn $ "We've built boot libraries in " ++ global_db ++ ":" + | otherwise -> do + let !package_id_len = length ("package-id ":: String) + let !package_db_len = length ("package-db ":: String) + let pkgs_ids = map (drop package_id_len) xs + -- cabal always adds the `base` global package to the environment files + -- as first entry, so we remove it because it's wrong in our case. + -- See cabal-install/src/Distribution/Client/CmdInstall.hs:{globalPackages,installLibraries} + let pkgs_ids_without_wired_base = drop 1 pkgs_ids + pure (drop package_db_len x, pkgs_ids_without_wired_base) + -- putStrLn $ "We've built boot libraries in " ++ global_db ++ ":" mapM_ (putStrLn . (" - " ++)) pkg_ids -- copy the libs in another db @@ -842,7 +873,7 @@ msg x = do putStrLn (stp ++ replicate (6 - length stp) ' ' ++ x) -- Avoid FilePath blindness by using type aliases for programs. -newtype Ghc = Ghc FilePath +data Ghc = Ghc FilePath [String] newtype GhcPkg = GhcPkg FilePath newtype GhcToolchain = GhcToolchain FilePath newtype Cabal = Cabal FilePath @@ -851,10 +882,10 @@ newtype GenApply = GenApply FilePath newtype GenPrimop = GenPrimop FilePath runGhc :: Ghc -> [String] -> CreateProcess -runGhc (Ghc f) = proc f +runGhc (Ghc f args) xs = proc f (args ++ xs) ghcPath :: Ghc -> FilePath -ghcPath (Ghc x) = x +ghcPath (Ghc x _) = x runGhcPkg :: GhcPkg -> [String] -> CreateProcess runGhcPkg (GhcPkg f) = proc f @@ -1006,20 +1037,18 @@ generateSettings ghc_toolchain Settings{..} dst = do let gen_settings_path = dst "lib/settings.generated" - mbCC <- lookupEnv "CC" >>= \case - Just cc -> pure ["--cc", cc] - Nothing -> pure [] - mbCXX <- lookupEnv "CXX" >>= \case - Just cxx -> pure ["--cxx", cxx] - Nothing -> pure [] let common_args = [ "--output-settings" , "-o", gen_settings_path - ] ++ mbCC ++ mbCXX + ] let opt m f = fmap f m let args = mconcat (catMaybes [ opt settingsTriple $ \x -> ["--triple", x] + , opt (poPath settingsCc) $ \x -> ["--cc", x] + , opt (poFlags settingsCc) $ \xs -> concat [["--cc-opt", x] | x <- xs] + , opt (poPath settingsCxx) $ \x -> ["--cxx", x] + , opt (poFlags settingsCxx) $ \xs -> concat [["--cxx-opt", x] | x <- xs] -- FIXME: add other options for ghc-toolchain from Settings ]) ++ common_args From c088e3081c8c831156d2bcd0382c0766965edf93 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 15 Apr 2025 15:27:45 +0200 Subject: [PATCH 159/175] Use zig to cross-compile to aarch64-linux We need to use wrappers, which is not pretty. We can use the resulting target with -Bpath/to/targets/aarch64_linux/lib but we can't link a program yet because we lack libffi for aarch64... In the past we built libffi with rts, we might do it again? --- Build.hs | 18 ++++++++++++------ aarch64-linux-zig-c++ | 3 +++ aarch64-linux-zig-cc | 2 ++ 3 files changed, 17 insertions(+), 6 deletions(-) create mode 100755 aarch64-linux-zig-c++ create mode 100755 aarch64-linux-zig-cc diff --git a/Build.hs b/Build.hs index 1013423d3c85..7174529bfd86 100755 --- a/Build.hs +++ b/Build.hs @@ -100,11 +100,13 @@ main = do -- Now we build extra targets. Ideally those should be built on demand... createDirectoryIfMissing True "_build/stage2/targets/" let targets = - [-- (,) "aarch64-linux" emptySettings - -- { settingsTriple = Just "aarch64-linux" - -- , settingsCc = ProgOpt (Just "zig-cc") (Just ["-target", "aarch64-linux-gnu"]) - -- , settingsCxx = ProgOpt (Just "zig-c++") (Just ["-target", "aarch64-linux-gnu"]) - -- } + [ (,) "aarch64-linux" emptySettings + { settingsTriple = Just "aarch64-linux" + , settingsCc = ProgOpt (Just "aarch64-linux-zig-cc") Nothing + , settingsCxx = ProgOpt (Just "aarch64-linux-zig-c++") Nothing + , settingsLd = ProgOpt (Just "aarch64-linux-zig-cc") Nothing + , settingsMergeObjs = ProgOpt (Just "aarch64-linux-zig-cc") Nothing + } -- , (,) "javascript" emptySettings -- { settingsTriple = Just "javascript-unknown-ghcjs" -- , settingsCc = ProgOpt (Just "emcc") Nothing @@ -801,7 +803,9 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst -- cabal always adds the `base` global package to the environment files -- as first entry, so we remove it because it's wrong in our case. -- See cabal-install/src/Distribution/Client/CmdInstall.hs:{globalPackages,installLibraries} - let pkgs_ids_without_wired_base = drop 1 pkgs_ids + -- + -- But apparently in Moritz' version of cabal, it's fixed. + let pkgs_ids_without_wired_base = drop 0 pkgs_ids pure (drop package_db_len x, pkgs_ids_without_wired_base) -- putStrLn $ "We've built boot libraries in " ++ global_db ++ ":" mapM_ (putStrLn . (" - " ++)) pkg_ids @@ -1049,6 +1053,8 @@ generateSettings ghc_toolchain Settings{..} dst = do , opt (poFlags settingsCc) $ \xs -> concat [["--cc-opt", x] | x <- xs] , opt (poPath settingsCxx) $ \x -> ["--cxx", x] , opt (poFlags settingsCxx) $ \xs -> concat [["--cxx-opt", x] | x <- xs] + , opt (poPath settingsLd) $ \x -> ["--ld", x] + , opt (poPath settingsMergeObjs) $ \x -> ["--merge-objs", x] -- FIXME: add other options for ghc-toolchain from Settings ]) ++ common_args diff --git a/aarch64-linux-zig-c++ b/aarch64-linux-zig-c++ new file mode 100755 index 000000000000..1ebc93d572f7 --- /dev/null +++ b/aarch64-linux-zig-c++ @@ -0,0 +1,3 @@ +#!/bin/sh +zig c++ --target=aarch64-linux $@ + diff --git a/aarch64-linux-zig-cc b/aarch64-linux-zig-cc new file mode 100755 index 000000000000..41722d3bbdde --- /dev/null +++ b/aarch64-linux-zig-cc @@ -0,0 +1,2 @@ +#!/bin/sh +zig cc --target=aarch64-linux $@ From cc44de7f39c8cd73818bf1cb5e18d845453acb78 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 15 Apr 2025 15:37:00 +0200 Subject: [PATCH 160/175] Revert "Fix libffi linking" This reverts commit 420310e09a6d86b3c2a5c102736bb19213927399. --- rts/rts.cabal | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/rts/rts.cabal b/rts/rts.cabal index 193e8dd79440..249e814810ec 100644 --- a/rts/rts.cabal +++ b/rts/rts.cabal @@ -613,15 +613,11 @@ library Jumps_V32.cmm Jumps_V64.cmm - -- we always link against libffi, even without libffi-adjustors enabled. - -- libffi is used by the Interpreter and some of its symbols are declared - -- in RtsSymbols.c - extra-libraries: ffi - extra-libraries-static: ffi - -- Adjustor stuff if flag(libffi-adjustors) c-sources: adjustor/LibffiAdjustor.c + extra-libraries: ffi + extra-libraries-static: ffi else -- Use GHC's native adjustors if arch(i386) @@ -637,6 +633,8 @@ library -- fall back to the LibffiAdjustor if neither i386, or x86_64 if !arch(x86_64) && !arch(i386) c-sources: adjustor/LibffiAdjustor.c + extra-libraries: ffi + extra-libraries-static: ffi -- Use assembler STG entrypoint on architectures where it is used if arch(ppc) || arch(ppc64) || arch(s390x) || arch(riscv64) || arch(loongarch64) From 109e4b9c3b811c2fe01f6aa65494ede8a4d182ef Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 15 Apr 2025 15:37:08 +0200 Subject: [PATCH 161/175] Revert "[build] drop ffi" This reverts commit 1ec11989ce5d46f4fcaf5fea1f713e91e546f8b9. --- Build.hs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/Build.hs b/Build.hs index 7174529bfd86..2b27e18b92de 100755 --- a/Build.hs +++ b/Build.hs @@ -619,6 +619,31 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst writeFile (src "libraries/ghc-internal/src/GHC/Internal/Prim.hs") =<< readCreateProcess (runGenPrimop genprimop ["--make-haskell-source"]) primops writeFile (src "libraries/ghc-internal/src/GHC/Internal/PrimopWrappers.hs") =<< readCreateProcess (runGenPrimop genprimop ["--make-haskell-wrappers"]) primops + -- build libffi + msg " - Building libffi..." + src_libffi <- makeAbsolute (src "libffi") + dst_libffi <- makeAbsolute (dst "libffi") + let libffi_version = "3.4.6" + createDirectoryIfMissing True src_libffi + createDirectoryIfMissing True dst_libffi + void $ readCreateProcess (shell ("tar -xvf libffi-tarballs/libffi-" ++ libffi_version ++ ".tar.gz -C " ++ src_libffi)) "" + let build_libffi = mconcat + [ "cd " ++ src_libffi "libffi-" ++ libffi_version ++ "; " + -- FIXME: pass the appropriate toolchain (CC, LD...) + , "./configure --disable-docs --with-pic=yes --disable-multi-os-directory --prefix=" ++ dst_libffi + , " && make install -j" + ] + (libffi_exit_code, libffi_stdout, libffi_stderr) <- readCreateProcessWithExitCode (shell build_libffi) "" + case libffi_exit_code of + ExitSuccess -> pure () + ExitFailure r -> do + putStrLn $ "Failed to build libffi with error code " ++ show r + putStrLn libffi_stdout + putStrLn libffi_stderr + exitFailure + cp (dst_libffi "include" "*") (src_rts "include") + cp (dst_libffi "lib" "libffi.a") (takeDirectory ghcplatform_dir "libCffi.a") + -- build boot libraries: ghc-internal, base... let cabal_project_bootlibs_path = dst "cabal-project-boot-libs" makeCabalProject cabal_project_bootlibs_path $ From 1fd397ec82db5239d138d0b09767117a8fc5c433 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 15 Apr 2025 15:37:20 +0200 Subject: [PATCH 162/175] Revert "Drop libffi; Stop shipping a copy of libffi" This reverts commit 7996993375ccbf4823d60e2998647c2745145ed8. --- .gitmodules | 4 ++++ compiler/GHC/Driver/CodeOutput.hs | 7 ++++--- libffi-tarballs | 1 + packages | 1 + rts/rts.cabal | 25 ++++++++++++++++++------- 5 files changed, 28 insertions(+), 10 deletions(-) create mode 160000 libffi-tarballs diff --git a/.gitmodules b/.gitmodules index 9c72ac9d8846..fc634597fb31 100644 --- a/.gitmodules +++ b/.gitmodules @@ -100,6 +100,10 @@ path = utils/hsc2hs url = https://gitlab.haskell.org/ghc/hsc2hs.git ignore = untracked +[submodule "libffi-tarballs"] + path = libffi-tarballs + url = https://gitlab.haskell.org/ghc/libffi-tarballs.git + ignore = untracked [submodule "gmp-tarballs"] path = libraries/ghc-internal/gmp/gmp-tarballs url = https://gitlab.haskell.org/ghc/gmp-tarballs.git diff --git a/compiler/GHC/Driver/CodeOutput.hs b/compiler/GHC/Driver/CodeOutput.hs index 023c4e1e365f..ff5a25c3bae0 100644 --- a/compiler/GHC/Driver/CodeOutput.hs +++ b/compiler/GHC/Driver/CodeOutput.hs @@ -255,11 +255,12 @@ outputJS _ _ _ _ _ = pgmError $ "codeOutput: Hit JavaScript case. We should neve -} {- -Note [libffi headers] +Note [Packaging libffi headers] ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The C code emitted by GHC for libffi adjustors must depend upon the ffi_arg type, -defined in . On systems where GHC uses the libffi adjustors, the libffi -library, and headers must be installed. +defined in . For this reason, we must ensure that is available +in binary distributions. To do so, we install these headers as part of the +`rts` package. -} outputForeignStubs diff --git a/libffi-tarballs b/libffi-tarballs new file mode 160000 index 000000000000..89a9b01c5647 --- /dev/null +++ b/libffi-tarballs @@ -0,0 +1 @@ +Subproject commit 89a9b01c5647c8f0d3899435b99df690f582e9f1 diff --git a/packages b/packages index d6bb0cd77e13..4f02d0133c0b 100644 --- a/packages +++ b/packages @@ -37,6 +37,7 @@ # localpath tag remotepath upstreamurl # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ghc-tarballs windows ghc-tarballs.git - +libffi-tarballs - - - utils/hsc2hs - - ssh://git@github.com/haskell/hsc2hs.git libraries/array - - - libraries/binary - - https://github.com/kolmodin/binary.git diff --git a/rts/rts.cabal b/rts/rts.cabal index 249e814810ec..31ebdd4eb82a 100644 --- a/rts/rts.cabal +++ b/rts/rts.cabal @@ -276,6 +276,9 @@ flag librt flag libdl default: False manual: True +flag use-system-libffi + default: False + manual: True flag libffi-adjustors default: False manual: True @@ -399,6 +402,18 @@ library stg/Types.h else + -- If we are using an in-tree libffi then we must declare it as a bundled + -- library to ensure that Cabal installs it. + if !flag(use-system-libffi) + if os(windows) + extra-bundled-libraries: Cffi-6 + else + extra-bundled-libraries: Cffi + + install-includes: ffi.h ffitarget.h + -- ^ see Note [Packaging libffi headers] in + -- GHC.Driver.CodeOutput. + -- Here we declare several flavours to be available when passing the -- suitable (combination of) flag(s) when configuring the RTS from hadrian, -- using Cabal. @@ -450,6 +465,9 @@ library extra-libraries: rt if flag(libdl) extra-libraries: dl + if flag(use-system-libffi) + extra-libraries: ffi + extra-libraries-static: ffi if os(windows) extra-libraries: -- for the linker @@ -616,8 +634,6 @@ library -- Adjustor stuff if flag(libffi-adjustors) c-sources: adjustor/LibffiAdjustor.c - extra-libraries: ffi - extra-libraries-static: ffi else -- Use GHC's native adjustors if arch(i386) @@ -630,11 +646,6 @@ library else asm-sources: adjustor/NativeAmd64Asm.S c-sources: adjustor/NativeAmd64.c - -- fall back to the LibffiAdjustor if neither i386, or x86_64 - if !arch(x86_64) && !arch(i386) - c-sources: adjustor/LibffiAdjustor.c - extra-libraries: ffi - extra-libraries-static: ffi -- Use assembler STG entrypoint on architectures where it is used if arch(ppc) || arch(ppc64) || arch(s390x) || arch(riscv64) || arch(loongarch64) From 4eed2ea4f943a445c6bd5984d6a26ef1f304177b Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 29 Apr 2025 17:15:16 +0200 Subject: [PATCH 163/175] Cross build libffi with zig --- Build.hs | 66 ++++++++++++++++++++++++++++++++++++++------------------ Makefile | 1 + 2 files changed, 46 insertions(+), 21 deletions(-) diff --git a/Build.hs b/Build.hs index 2b27e18b92de..e0129410a0ad 100755 --- a/Build.hs +++ b/Build.hs @@ -481,6 +481,9 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst [arch,vendor,os] -> (arch,vendor,os) t -> error $ "Triple expected but got: " ++ show t let (arch,vendor,os) = to_triple $ words $ map (\c -> if c == '-' then ' ' else c) target_triple + let fixed_triple = case vendor of + "unknown" -> arch ++ "-" ++ os + _ -> target_triple let cabal_project_rts_path = dst "cabal.project-rts" -- cabal's code handling escaping is bonkers. We need to wrap the whole @@ -507,11 +510,8 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst -- This is stupid, I can't seem to figure out how to set this in cabal -- this needs to be fixed in cabal. , if os == "darwin" - then " flags: +tables-next-to-code +leading-underscore" - else " flags: +tables-next-to-code" - -- FIXME: we should - -- FIXME: deal with libffi (add package?) - -- + then " flags: +tables-next-to-code +leading-underscore +use-system-libffi" + else " flags: +tables-next-to-code +use-system-libffi" -- FIXME: we should make tables-next-to-code optional here and in the -- compiler settings. Ideally, GHC should even look into the rts's -- ghcautoconf.h to check whether TABLES_NEXT_TO_CODE is defined or @@ -578,6 +578,8 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst exitFailure d -> pure (takeDirectory d) + cc <- ghcSetting ghc "C compiler command" + -- deriving constants let derived_constants = src_rts "include/DerivedConstants.h" withSystemTempDirectory "derive-constants" $ \tmp_dir -> do @@ -587,7 +589,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , "-o", derived_constants , "--target-os", target , "--tmpdir", tmp_dir - , "--gcc-program", "cc" -- FIXME + , "--gcc-program", cc , "--nm-program", "nm" -- FIXME , "--objdump-program", "objdump" -- FIXME -- pass `-fcommon` to force symbols into the common section. If they @@ -623,15 +625,18 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst msg " - Building libffi..." src_libffi <- makeAbsolute (src "libffi") dst_libffi <- makeAbsolute (dst "libffi") - let libffi_version = "3.4.6" - createDirectoryIfMissing True src_libffi createDirectoryIfMissing True dst_libffi - void $ readCreateProcess (shell ("tar -xvf libffi-tarballs/libffi-" ++ libffi_version ++ ".tar.gz -C " ++ src_libffi)) "" + + doesDirectoryExist src_libffi >>= \case + True -> pure () + False -> do + createDirectoryIfMissing True src_libffi + -- fetch libffi fork with zig build system + void $ readCreateProcess (shell ("git clone git@github.com:vezel-dev/libffi.git " ++ src_libffi)) "" + let build_libffi = mconcat - [ "cd " ++ src_libffi "libffi-" ++ libffi_version ++ "; " - -- FIXME: pass the appropriate toolchain (CC, LD...) - , "./configure --disable-docs --with-pic=yes --disable-multi-os-directory --prefix=" ++ dst_libffi - , " && make install -j" + [ "cd " ++ src_libffi ++ "; " + , "zig build install --prefix " ++ dst_libffi ++ " -Dtarget=" ++ fixed_triple ] (libffi_exit_code, libffi_stdout, libffi_stderr) <- readCreateProcessWithExitCode (shell build_libffi) "" case libffi_exit_code of @@ -828,9 +833,12 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst -- cabal always adds the `base` global package to the environment files -- as first entry, so we remove it because it's wrong in our case. -- See cabal-install/src/Distribution/Client/CmdInstall.hs:{globalPackages,installLibraries} - -- - -- But apparently in Moritz' version of cabal, it's fixed. - let pkgs_ids_without_wired_base = drop 0 pkgs_ids + let pkgs_ids_without_wired_base + | (fid:fids) <- pkgs_ids + , "base-" `List.isPrefixOf` fid = fids + -- apparently in Moritz' version of cabal, it's fixed. + | otherwise = pkgs_ids + pure (drop package_db_len x, pkgs_ids_without_wired_base) -- putStrLn $ "We've built boot libraries in " ++ global_db ++ ":" mapM_ (putStrLn . (" - " ++)) pkg_ids @@ -845,8 +853,19 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst -- NOTE: GHC assumes that pkgroot is just one directory above the directory -- containing the package db. In our case where everything is at the same -- level in "pkgs" we need to re-add "/pkgs" + let fix_pkgroot = Text.replace (Text.pack pkg_root) "${pkgroot}/pkgs" + -- Add libCffi library to the rts. We can't use RTS cabal flag -use-system-ffi + -- because the library needs to be installed during setup. + let fix_cffi_line l + | "hs-libraries:" `Text.isPrefixOf` l = l <> " Cffi" + | otherwise = l + let fix_cffi c + | not ("rts-" `List.isPrefixOf` pid) = c + | otherwise = Text.unlines (map fix_cffi_line (Text.lines c)) + + Text.writeFile (dst "pkgs" pid <.> "conf") - (Text.replace (Text.pack pkg_root) "${pkgroot}/pkgs" conf) + (fix_cffi (fix_pkgroot conf)) cp (pkg_root pid) (dst "pkgs") void $ readCreateProcess (runGhcPkg ghcpkg ["recache", "--package-db=" ++ (dst "pkgs")]) "" @@ -987,12 +1006,17 @@ getTarget ghc = ghcTargetArchOS ghc >>= \case (_,"OSLinux") -> pure "linux" _ -> error "Unsupported target" +ghcSettings :: Ghc -> IO [(String,String)] +ghcSettings ghc = read <$> readCreateProcess (runGhc ghc ["--info"]) "" + +ghcSetting :: Ghc -> String -> IO String +ghcSetting ghc s = do + is <- ghcSettings ghc + pure $ fromMaybe (error $ "Couldn't read '" ++ s ++ "' setting of " ++ ghcPath ghc) (lookup s is) + -- | Retrieve GHC's target triple ghcTargetTriple :: Ghc -> IO String -ghcTargetTriple ghc = do - is <- read <$> readCreateProcess (runGhc ghc ["--info"]) "" :: IO [(String,String)] - pure $ fromMaybe (error "Couldn't read 'Target platform setting") (lookup "Target platform" is) - +ghcTargetTriple ghc = ghcSetting ghc "Target platform" data Settings = Settings { settingsTriple :: Maybe String diff --git a/Makefile b/Makefile index d07edfae6fe9..6a9b4f688d0b 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ CPUS=$(shell mk/detect-cpu-count.sh) THREADS=${THREADS:-$((CPUS + 1))} all: $(CABAL) ./booted + PATH=`pwd`:${PATH} \ GHC=ghc-9.8.4 ./Build.hs cabal: $(CABAL) From 27abce3ca21dd779ed5748bbe380c3fa48b5e773 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 30 Apr 2025 12:04:03 +0200 Subject: [PATCH 164/175] Disable ubsan and properly install libCffi --- Build.hs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Build.hs b/Build.hs index e0129410a0ad..c68c17b00e56 100755 --- a/Build.hs +++ b/Build.hs @@ -106,6 +106,7 @@ main = do , settingsCxx = ProgOpt (Just "aarch64-linux-zig-c++") Nothing , settingsLd = ProgOpt (Just "aarch64-linux-zig-cc") Nothing , settingsMergeObjs = ProgOpt (Just "aarch64-linux-zig-cc") Nothing + , settingsCrossCompiling = True } -- , (,) "javascript" emptySettings -- { settingsTriple = Just "javascript-unknown-ghcjs" @@ -637,6 +638,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst let build_libffi = mconcat [ "cd " ++ src_libffi ++ "; " , "zig build install --prefix " ++ dst_libffi ++ " -Dtarget=" ++ fixed_triple + , " -Doptimize=ReleaseFast -Dlinkage=static" ] (libffi_exit_code, libffi_stdout, libffi_stderr) <- readCreateProcessWithExitCode (shell build_libffi) "" case libffi_exit_code of @@ -858,6 +860,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst -- because the library needs to be installed during setup. let fix_cffi_line l | "hs-libraries:" `Text.isPrefixOf` l = l <> " Cffi" + | "extra-libraries:" `Text.isPrefixOf` l = Text.replace "ffi" "" l | otherwise = l let fix_cffi c | not ("rts-" `List.isPrefixOf` pid) = c @@ -868,6 +871,10 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst (fix_cffi (fix_pkgroot conf)) cp (pkg_root pid) (dst "pkgs") + -- install libffi... + when ("rts-" `List.isPrefixOf` pid) $ do + cp (dst_libffi "lib" "libffi.a") (dst "pkgs" pid "lib" "libCffi.a") + void $ readCreateProcess (runGhcPkg ghcpkg ["recache", "--package-db=" ++ (dst "pkgs")]) "" @@ -1043,6 +1050,7 @@ data Settings = Settings , settingsTablesNextToCode :: Maybe Bool , settingsUseLibFFIForAdjustors :: Maybe Bool , settingsLdOverride :: Maybe Bool + , settingsCrossCompiling :: Bool } -- | Program specifier from the command-line. @@ -1080,6 +1088,7 @@ emptySettings = Settings , settingsTablesNextToCode = Nothing , settingsUseLibFFIForAdjustors = Nothing , settingsLdOverride = Nothing + , settingsCrossCompiling = False } where po0 = emptyProgOpt @@ -1126,5 +1135,6 @@ generateSettings ghc_toolchain Settings{..} dst = do $ Map.insert "RTS ways" "v" -- FIXME: this depends on the different ways used to build the RTS! $ Map.insert "otool command" "otool" -- FIXME: this should just arguably be a default in the settings in GHC, and not require the settings file? $ Map.insert "install_name_tool command" "install_name_tool" + $ Map.insert "cross compiling" (if settingsCrossCompiling then "YES" else "NO") $ kvs writeFile (dst "lib/settings") (show $ Map.toList kvs') From fae581915d5bd09d581346c16bb9e3589664c5f2 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 30 Apr 2025 14:06:20 +0200 Subject: [PATCH 165/175] Progress towards fixing the build of cross libraries --- Build.hs | 42 ++++++++++++++++++++++++++++-------------- rts/rts.cabal | 8 +++++--- 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/Build.hs b/Build.hs index c68c17b00e56..da76aacd6667 100755 --- a/Build.hs +++ b/Build.hs @@ -113,12 +113,21 @@ main = do -- , settingsCc = ProgOpt (Just "emcc") Nothing -- } ] + + ghc_stage2_abs <- makeAbsolute "_build/stage2/bin/ghc" forM_ targets $ \(target,settings) -> do msg $ "Bootstrapping target: " <> target target_dir <- makeAbsolute ("_build/stage2/targets" target) createDirectoryIfMissing True target_dir generateSettings ghcToolchain settings target_dir - ghc2 <- Ghc <$> makeAbsolute "_build/stage2/bin/ghc" <*> pure ["-B"++ target_dir "lib"] + -- compiler flags aren't passed consistently to configure, etc. + -- So we need to create a wrapper. Yes this is garbage. Why are we + -- infliciting this (autotools, etc.) to ourselves? + let ghc_wrapper = target_dir "ghc" + writeFile ghc_wrapper ("#!/bin/sh\n" <> ghc_stage2_abs <> " -B" <> (target_dir "lib") <> " $@") + _ <- readCreateProcess (shell $ "chmod +x " ++ ghc_wrapper) "" + let ghc2 = Ghc ghc_wrapper [] + -- ghc2 <- Ghc <$> makeAbsolute "_build/stage2/bin/ghc" <*> pure ["-B"++ target_dir "lib"] buildBootLibraries cabal ghc2 ghcPkg1 deriveConstants genapply genprimop defaultGhcBuildOptions target_dir @@ -467,16 +476,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst src <- makeAbsolute (dst "src") prepareGhcSources opts src - -- Build the RTS - src_rts <- makeAbsolute (src "libraries/rts") - build_dir <- makeAbsolute (dst "cabal" "build") - store_dir <- makeAbsolute (dst "cabal" "store") - ghcversionh <- makeAbsolute (src_rts "include/ghcversion.h") - - createDirectoryIfMissing True build_dir - createDirectoryIfMissing True store_dir - - -- FIXME: could we build a cross compiler, simply by not reading this from the boot compiler, but passing it in? + -- detect target (inferred from the ghc we use) target_triple <- ghcTargetTriple ghc let to_triple = \case [arch,vendor,os] -> (arch,vendor,os) @@ -486,6 +486,15 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst "unknown" -> arch ++ "-" ++ os _ -> target_triple + -- Build the RTS + src_rts <- makeAbsolute (src "libraries/rts") + build_dir <- makeAbsolute (dst "cabal" "build") + store_dir <- makeAbsolute (dst "cabal" "store") + ghcversionh <- makeAbsolute (src_rts "include/ghcversion.h") + + createDirectoryIfMissing True build_dir + createDirectoryIfMissing True store_dir + let cabal_project_rts_path = dst "cabal.project-rts" -- cabal's code handling escaping is bonkers. We need to wrap the whole -- option into \" otherwise it does weird things (like keeping only the @@ -573,11 +582,16 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst writeFile (dst "rts-conf.stderr") rts_conf_stderr ghcplatform_dir <- do ghcplatform_h <- readCreateProcess (shell ("find " ++ build_dir ++ " -name ghcplatform.h")) "" - case ghcplatform_h of - "" -> do + case lines ghcplatform_h of + [] -> do putStrLn $ "Couldn't find ghcplatform.h. Look into " ++ dst ++ "rts-conf.{stdout,stderr}" exitFailure - d -> pure (takeDirectory d) + [d] -> pure (takeDirectory d) + ds -> do + putStrLn $ "ghcplatform.h found in several paths:" + forM_ ds $ \d -> putStrLn (" - " ++ d) + putStrLn $ "Check the log in " ++ (dst "rts-conf.{stdout,stderr}") + exitFailure cc <- ghcSetting ghc "C compiler command" diff --git a/rts/rts.cabal b/rts/rts.cabal index 31ebdd4eb82a..f3b80c7d1c0f 100644 --- a/rts/rts.cabal +++ b/rts/rts.cabal @@ -465,9 +465,11 @@ library extra-libraries: rt if flag(libdl) extra-libraries: dl - if flag(use-system-libffi) - extra-libraries: ffi - extra-libraries-static: ffi + -- we patch the package ourselves later to avoid cabal warning about + -- libffi not being available + -- if flag(use-system-libffi) + -- extra-libraries: ffi + -- extra-libraries-static: ffi if os(windows) extra-libraries: -- for the linker From bc31fb9962625656eb09375d86cfb7162950dda5 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 30 Apr 2025 17:26:08 +0200 Subject: [PATCH 166/175] Finally build hackily install libffi --- Build.hs | 71 ++++++++++++++++++++++++++++----------------------- rts/rts.cabal | 8 +++--- 2 files changed, 42 insertions(+), 37 deletions(-) diff --git a/Build.hs b/Build.hs index da76aacd6667..91bded7828d2 100755 --- a/Build.hs +++ b/Build.hs @@ -475,6 +475,7 @@ buildBootLibraries :: Cabal -> Ghc -> GhcPkg -> DeriveConstants -> GenApply -> G buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst = do src <- makeAbsolute (dst "src") prepareGhcSources opts src + src_rts <- makeAbsolute (src "libraries/rts") -- detect target (inferred from the ghc we use) target_triple <- ghcTargetTriple ghc @@ -486,8 +487,36 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst "unknown" -> arch ++ "-" ++ os _ -> target_triple + -- build libffi + msg " - Building libffi..." + src_libffi <- makeAbsolute (src "libffi") + dst_libffi <- makeAbsolute (dst "libffi") + createDirectoryIfMissing True dst_libffi + + doesDirectoryExist src_libffi >>= \case + True -> pure () + False -> do + createDirectoryIfMissing True src_libffi + -- fetch libffi fork with zig build system + void $ readCreateProcess (shell ("git clone git@github.com:vezel-dev/libffi.git " ++ src_libffi)) "" + + let build_libffi = mconcat + [ "cd " ++ src_libffi ++ "; " + , "zig build install --prefix " ++ dst_libffi ++ " -Dtarget=" ++ fixed_triple + , " -Doptimize=ReleaseFast -Dlinkage=static" + ] + (libffi_exit_code, libffi_stdout, libffi_stderr) <- readCreateProcessWithExitCode (shell build_libffi) "" + case libffi_exit_code of + ExitSuccess -> pure () + ExitFailure r -> do + putStrLn $ "Failed to build libffi with error code " ++ show r + putStrLn libffi_stdout + putStrLn libffi_stderr + exitFailure + cp (dst_libffi "include" "*") (src_rts "include") + -- cp (dst_libffi "lib" "libffi.a") (takeDirectory ghcplatform_dir "libCffi.a") + -- Build the RTS - src_rts <- makeAbsolute (src "libraries/rts") build_dir <- makeAbsolute (dst "cabal" "build") store_dir <- makeAbsolute (dst "cabal" "store") ghcversionh <- makeAbsolute (src_rts "include/ghcversion.h") @@ -553,6 +582,8 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , " executable-profiling: False" , " executable-dynamic: False" , " executable-static: False" + , " extra-lib-dirs: " ++ dst_libffi "lib" + , " extra-include-dirs: " ++ dst_libffi "include" , "" ] ++ rts_options @@ -584,7 +615,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst ghcplatform_h <- readCreateProcess (shell ("find " ++ build_dir ++ " -name ghcplatform.h")) "" case lines ghcplatform_h of [] -> do - putStrLn $ "Couldn't find ghcplatform.h. Look into " ++ dst ++ "rts-conf.{stdout,stderr}" + putStrLn $ "Couldn't find ghcplatform.h. Look into " ++ (dst "rts-conf.{stdout,stderr}") exitFailure [d] -> pure (takeDirectory d) ds -> do @@ -636,35 +667,6 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst writeFile (src "libraries/ghc-internal/src/GHC/Internal/Prim.hs") =<< readCreateProcess (runGenPrimop genprimop ["--make-haskell-source"]) primops writeFile (src "libraries/ghc-internal/src/GHC/Internal/PrimopWrappers.hs") =<< readCreateProcess (runGenPrimop genprimop ["--make-haskell-wrappers"]) primops - -- build libffi - msg " - Building libffi..." - src_libffi <- makeAbsolute (src "libffi") - dst_libffi <- makeAbsolute (dst "libffi") - createDirectoryIfMissing True dst_libffi - - doesDirectoryExist src_libffi >>= \case - True -> pure () - False -> do - createDirectoryIfMissing True src_libffi - -- fetch libffi fork with zig build system - void $ readCreateProcess (shell ("git clone git@github.com:vezel-dev/libffi.git " ++ src_libffi)) "" - - let build_libffi = mconcat - [ "cd " ++ src_libffi ++ "; " - , "zig build install --prefix " ++ dst_libffi ++ " -Dtarget=" ++ fixed_triple - , " -Doptimize=ReleaseFast -Dlinkage=static" - ] - (libffi_exit_code, libffi_stdout, libffi_stderr) <- readCreateProcessWithExitCode (shell build_libffi) "" - case libffi_exit_code of - ExitSuccess -> pure () - ExitFailure r -> do - putStrLn $ "Failed to build libffi with error code " ++ show r - putStrLn libffi_stdout - putStrLn libffi_stderr - exitFailure - cp (dst_libffi "include" "*") (src_rts "include") - cp (dst_libffi "lib" "libffi.a") (takeDirectory ghcplatform_dir "libCffi.a") - -- build boot libraries: ghc-internal, base... let cabal_project_bootlibs_path = dst "cabal-project-boot-libs" makeCabalProject cabal_project_bootlibs_path $ @@ -734,6 +736,8 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , " executable-profiling: False" , " executable-dynamic: False" , " executable-static: True" + , " extra-lib-dirs: " ++ dst_libffi "lib" + , " extra-include-dirs: " ++ dst_libffi "include" , "" , "package ghc" -- build-tool-depends: require genprimopcode, etc. used by Setup.hs @@ -827,7 +831,7 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst putStrLn $ "Failed to build boot libraries with error code " ++ show r putStrLn boot_stdout putStrLn boot_stderr - putStrLn $ "Logs can be found in " ++ dst ++ "boot-libs.{stdout,stderr}" + putStrLn $ "Logs can be found in " ++ (dst "boot-libs.{stdout,stderr}") exitFailure -- The libraries have been installed globally. @@ -888,6 +892,8 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst -- install libffi... when ("rts-" `List.isPrefixOf` pid) $ do cp (dst_libffi "lib" "libffi.a") (dst "pkgs" pid "lib" "libCffi.a") + cp (dst_libffi "include" "ffi.h") (dst "pkgs" pid "lib" "include" "ffi.h") + cp (dst_libffi "include" "ffitarget.h") (dst "pkgs" pid "lib" "include" "ffitarget.h") void $ readCreateProcess (runGhcPkg ghcpkg ["recache", "--package-db=" ++ (dst "pkgs")]) "" @@ -1110,6 +1116,7 @@ emptySettings = Settings generateSettings :: GhcToolchain -> Settings -> FilePath -> IO () generateSettings ghc_toolchain Settings{..} dst = do createDirectoryIfMissing True (dst "lib") + createDirectoryIfMissing True (dst "pkgs") let gen_settings_path = dst "lib/settings.generated" diff --git a/rts/rts.cabal b/rts/rts.cabal index f3b80c7d1c0f..31ebdd4eb82a 100644 --- a/rts/rts.cabal +++ b/rts/rts.cabal @@ -465,11 +465,9 @@ library extra-libraries: rt if flag(libdl) extra-libraries: dl - -- we patch the package ourselves later to avoid cabal warning about - -- libffi not being available - -- if flag(use-system-libffi) - -- extra-libraries: ffi - -- extra-libraries-static: ffi + if flag(use-system-libffi) + extra-libraries: ffi + extra-libraries-static: ffi if os(windows) extra-libraries: -- for the linker From 8dfd49d260fe72171e2f2a43044ceca682cb4583 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 30 Apr 2025 17:41:52 +0200 Subject: [PATCH 167/175] Deal with unlit path --- Build.hs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Build.hs b/Build.hs index 91bded7828d2..4920d220a2a7 100755 --- a/Build.hs +++ b/Build.hs @@ -107,6 +107,7 @@ main = do , settingsLd = ProgOpt (Just "aarch64-linux-zig-cc") Nothing , settingsMergeObjs = ProgOpt (Just "aarch64-linux-zig-cc") Nothing , settingsCrossCompiling = True + , settingsUnlit = "$topdir/../../../bin/unlit" } -- , (,) "javascript" emptySettings -- { settingsTriple = Just "javascript-unknown-ghcjs" @@ -1063,6 +1064,7 @@ data Settings = Settings , settingsReadelf :: ProgOpt , settingsMergeObjs :: ProgOpt , settingsWindres :: ProgOpt + , settingsUnlit :: String -- Note we don't actually configure LD into anything but -- see #23857 and #22550 for the very unfortunate story. , settingsLd :: ProgOpt @@ -1103,6 +1105,7 @@ emptySettings = Settings , settingsReadelf = po0 , settingsMergeObjs = po0 , settingsWindres = po0 + , settingsUnlit = "$topdir/../bin/unlit" , settingsLd = po0 , settingsUnregisterised = Nothing , settingsTablesNextToCode = Nothing @@ -1157,5 +1160,6 @@ generateSettings ghc_toolchain Settings{..} dst = do $ Map.insert "otool command" "otool" -- FIXME: this should just arguably be a default in the settings in GHC, and not require the settings file? $ Map.insert "install_name_tool command" "install_name_tool" $ Map.insert "cross compiling" (if settingsCrossCompiling then "YES" else "NO") + $ Map.insert "unlit command" settingsUnlit $ kvs writeFile (dst "lib/settings") (show $ Map.toList kvs') From 680b9c53a0e8ccf8bd105b0fd8cc263097d301f2 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 30 Apr 2025 17:48:38 +0200 Subject: [PATCH 168/175] Use libffi adjustors on aarch64 --- rts/rts.cabal | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/rts/rts.cabal b/rts/rts.cabal index 31ebdd4eb82a..01d986df2cea 100644 --- a/rts/rts.cabal +++ b/rts/rts.cabal @@ -631,21 +631,24 @@ library Jumps_V32.cmm Jumps_V64.cmm - -- Adjustor stuff if flag(libffi-adjustors) + -- forced libffi adjustors c-sources: adjustor/LibffiAdjustor.c + elif arch(javascript) + -- no adjustors for javascript + elif arch(i386) + asm-sources: adjustor/Nativei386Asm.S + c-sources: adjustor/Nativei386.c + elif arch(x86_64) + if os(mingw32) + asm-sources: adjustor/NativeAmd64MingwAsm.S + c-sources: adjustor/NativeAmd64Mingw.c + else + asm-sources: adjustor/NativeAmd64Asm.S + c-sources: adjustor/NativeAmd64.c else - -- Use GHC's native adjustors - if arch(i386) - asm-sources: adjustor/Nativei386Asm.S - c-sources: adjustor/Nativei386.c - if arch(x86_64) - if os(mingw32) - asm-sources: adjustor/NativeAmd64MingwAsm.S - c-sources: adjustor/NativeAmd64Mingw.c - else - asm-sources: adjustor/NativeAmd64Asm.S - c-sources: adjustor/NativeAmd64.c + -- default to libffi adjustors + c-sources: adjustor/LibffiAdjustor.c -- Use assembler STG entrypoint on architectures where it is used if arch(ppc) || arch(ppc64) || arch(s390x) || arch(riscv64) || arch(loongarch64) From 1a56bf11c05a7074a69f8301ad6932b5f2d8e966 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Mon, 5 May 2025 17:03:16 +0200 Subject: [PATCH 169/175] Add multi-target support Use --target=foo to select a target installed in $topdir/targets/foo/ --- ghc/Main.hs | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/ghc/Main.hs b/ghc/Main.hs index 87dbef1d89ef..dc8185212be4 100644 --- a/ghc/Main.hs +++ b/ghc/Main.hs @@ -86,6 +86,8 @@ import GHC.Iface.Errors.Ppr -- Standard Haskell libraries import System.IO +import System.FilePath +import System.Directory import System.Environment import System.Exit import System.FilePath @@ -123,11 +125,27 @@ main = do -- 1. extract the -B flag from the args argv0 <- getArgs - let (minusB_args, argv1) = partition ("-B" `isPrefixOf`) argv0 + let (target_args, argv1) = partition ("--target=" `isPrefixOf`) argv0 + mbTarget | null target_args = Nothing + | otherwise = Just (drop 9 (last target_args)) + + + let (minusB_args, argv1') = partition ("-B" `isPrefixOf`) argv1 mbMinusB | null minusB_args = Nothing | otherwise = Just (drop 2 (last minusB_args)) - let argv2 = map (mkGeneralLocated "on the commandline") argv1 + -- find top directory for the given target. Or default to usual topdir. + targettopdir <- Just <$> do + topdir <- findTopDir mbMinusB + case mbTarget of + Nothing -> pure topdir + Just target -> do + let r = topdir "targets" target "lib" + doesDirectoryExist r >>= \case + True -> pure r + False -> throwGhcException (UsageError $ "Couldn't find specific target `" ++ target ++ "' in `" ++ r ++ "'") + + let argv2 = map (mkGeneralLocated "on the commandline") argv1' -- 2. Parse the "mode" flags (--make, --interactive etc.) (mode, units, argv3, flagWarnings) <- parseModeFlags argv2 @@ -143,13 +161,13 @@ main = do case mode of Left preStartupMode -> do case preStartupMode of - ShowSupportedExtensions -> showSupportedExtensions mbMinusB + ShowSupportedExtensions -> showSupportedExtensions targettopdir ShowVersion -> showVersion ShowNumVersion -> putStrLn cProjectVersion ShowOptions isInteractive -> showOptions isInteractive Right postStartupMode -> -- start our GHC session - GHC.runGhc mbMinusB $ do + GHC.runGhc targettopdir $ do dflags <- GHC.getSessionDynFlags From bb87a13298134e2d86e0238c11452ef65a8f1194 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 7 May 2025 10:53:42 +0200 Subject: [PATCH 170/175] Support listing targets --- ghc/Main.hs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/ghc/Main.hs b/ghc/Main.hs index dc8185212be4..fcc469e4bd71 100644 --- a/ghc/Main.hs +++ b/ghc/Main.hs @@ -134,18 +134,32 @@ main = do mbMinusB | null minusB_args = Nothing | otherwise = Just (drop 2 (last minusB_args)) + let (list_targets_args, argv1'') = partition (== "--list-targets") argv1' + list_targets = not (null list_targets_args) + -- find top directory for the given target. Or default to usual topdir. targettopdir <- Just <$> do topdir <- findTopDir mbMinusB + let targets_dir = topdir "targets" + -- list targets when asked + when list_targets $ do + putStrLn "Installed extra targets:" + doesDirectoryExist targets_dir >>= \case + True -> do + ds <- listDirectory targets_dir + forM_ ds (\d -> putStrLn $ " - " ++ d) + False -> pure () + exitSuccess + -- otherwise select the appropriate target case mbTarget of Nothing -> pure topdir Just target -> do - let r = topdir "targets" target "lib" + let r = targets_dir target "lib" doesDirectoryExist r >>= \case True -> pure r False -> throwGhcException (UsageError $ "Couldn't find specific target `" ++ target ++ "' in `" ++ r ++ "'") - let argv2 = map (mkGeneralLocated "on the commandline") argv1' + let argv2 = map (mkGeneralLocated "on the commandline") argv1'' -- 2. Parse the "mode" flags (--make, --interactive etc.) (mode, units, argv3, flagWarnings) <- parseModeFlags argv2 From 17203d9b3577aee24d0bc3076f40962be83b8792 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 13 May 2025 14:29:26 +0200 Subject: [PATCH 171/175] Don't forget to rebuild cabal when needed --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6a9b4f688d0b..8ed3abf8fbf0 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -export CABAL := $(shell cabal update 2>&1 >/dev/null && cabal list-bin -v0 --project-dir libraries/Cabal cabal-install:exe:cabal) +export CABAL := $(shell cabal update 2>&1 >/dev/null && cabal build cabal-install -v0 --disable-tests --project-dir libraries/Cabal && cabal list-bin -v0 --project-dir libraries/Cabal cabal-install:exe:cabal) CPUS=$(shell mk/detect-cpu-count.sh) From a5bfd6ab38c7a3ef35a27185e2877c10f5cfb45e Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 13 May 2025 14:30:07 +0200 Subject: [PATCH 172/175] Use newer Cabal with support for -W --- libraries/Cabal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/Cabal b/libraries/Cabal index 98242d4d81e3..7e50837ade18 160000 --- a/libraries/Cabal +++ b/libraries/Cabal @@ -1 +1 @@ -Subproject commit 98242d4d81e38dd591e212f3a9df7f04215ad1c7 +Subproject commit 7e50837ade188504d1401bad932a5b8b3769661e From b2dca9294bf5858370b05c23bc92c62edd0bfa37 Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 13 May 2025 14:30:28 +0200 Subject: [PATCH 173/175] Bump cabal-version to support extra-static-libraries --- rts/rts.cabal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rts/rts.cabal b/rts/rts.cabal index 01d986df2cea..65fd926e3541 100644 --- a/rts/rts.cabal +++ b/rts/rts.cabal @@ -1,4 +1,4 @@ -cabal-version: 3.4 +cabal-version: 3.8 name: rts version: 1.0.3 synopsis: The GHC runtime system From 273aff977945d7cd810d097721015b3a5e8fbfad Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Tue, 13 May 2025 17:16:16 +0200 Subject: [PATCH 174/175] Use two GHCs with cabal --- Build.hs | 97 +++++++++++++++++++++++++++++--------------------------- 1 file changed, 50 insertions(+), 47 deletions(-) diff --git a/Build.hs b/Build.hs index 4920d220a2a7..094a16132f8b 100755 --- a/Build.hs +++ b/Build.hs @@ -83,7 +83,7 @@ main = do generateSettings ghcToolchain stage1_settings "_build/stage1/" msg "Building boot libraries with stage1 compiler..." - buildBootLibraries cabal ghc1 ghcPkg1 deriveConstants genapply genprimop defaultGhcBuildOptions "_build/stage1/" + buildBootLibraries cabal ghc1 ghc1 ghcPkg1 deriveConstants genapply genprimop defaultGhcBuildOptions "_build/stage1/" msg "Building stage2 GHC program" createDirectoryIfMissing True "_build/stage2" @@ -127,9 +127,10 @@ main = do let ghc_wrapper = target_dir "ghc" writeFile ghc_wrapper ("#!/bin/sh\n" <> ghc_stage2_abs <> " -B" <> (target_dir "lib") <> " $@") _ <- readCreateProcess (shell $ "chmod +x " ++ ghc_wrapper) "" + let ghc2_host = Ghc ghc_stage2_abs [] let ghc2 = Ghc ghc_wrapper [] -- ghc2 <- Ghc <$> makeAbsolute "_build/stage2/bin/ghc" <*> pure ["-B"++ target_dir "lib"] - buildBootLibraries cabal ghc2 ghcPkg1 deriveConstants genapply genprimop defaultGhcBuildOptions target_dir + buildBootLibraries cabal ghc2_host ghc2 ghcPkg1 deriveConstants genapply genprimop defaultGhcBuildOptions target_dir -- Finally create bindist directory @@ -472,8 +473,8 @@ prepareGhcSources opts dst = do subst_in (dst "libraries/rts/include/ghcversion.h") common_substs -buildBootLibraries :: Cabal -> Ghc -> GhcPkg -> DeriveConstants -> GenApply -> GenPrimop -> GhcBuildOptions -> FilePath -> IO () -buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst = do +buildBootLibraries :: Cabal -> Ghc -> Ghc -> GhcPkg -> DeriveConstants -> GenApply -> GenPrimop -> GhcBuildOptions -> FilePath -> IO () +buildBootLibraries cabal ghc_host ghc ghcpkg derive_constants genapply genprimop opts dst = do src <- makeAbsolute (dst "src") prepareGhcSources opts src src_rts <- makeAbsolute (src "libraries/rts") @@ -592,8 +593,9 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst [ "--store-dir=" ++ store_dir , "build" , "--project-file=" ++ cabal_project_rts_path - , "rts" - , "--with-compiler=" ++ ghcPath ghc + , "rts:rts" + , "-w", ghcPath ghc + , "-W", ghcPath ghc_host , "--with-hc-pkg=" ++ ghcPkgPath ghcpkg , "--ghc-options=\"-ghcversion-file=" ++ ghcversionh ++ "\"" , "--builddir=" ++ build_dir @@ -769,57 +771,58 @@ buildBootLibraries cabal ghc ghcpkg derive_constants genapply genprimop opts dst , "--package-env=" ++ boot_libs_env , "--force-reinstalls" , "--project-file=" ++ cabal_project_bootlibs_path - , "--with-compiler=" ++ ghcPath ghc + , "-w", ghcPath ghc + , "-W", ghcPath ghc_host , "--with-hc-pkg=" ++ ghcPkgPath ghcpkg , "--ghc-options=\"-ghcversion-file=" ++ ghcversionh ++ "\"" , "--builddir=" ++ build_dir , "-j" -- targets - , "rts" - , "ghc-internal" - , "ghc-experimental" - , "ghc-compact" - , "base" - , "stm" - , "system-cxx-std-lib" + , "rts:rts" + , "ghc-internal:ghc-internal" + , "ghc-experimental:ghc-experimental" + , "ghc-compact:ghc-compact" + , "base:base" + , "stm:stm" + , "system-cxx-std-lib:system-cxx-std-lib" -- shallow compat packages over ghc-internal - , "ghc-prim" - , "ghc-bignum" - , "integer-gmp" - , "template-haskell" + , "ghc-prim:ghc-prim" + , "ghc-bignum:ghc-bignum" + , "integer-gmp:integer-gmp" + , "template-haskell:template-haskell" -- target dependencies - , "ghc-boot-th" -- dependency of template-haskell - , "pretty" -- dependency of ghc-boot-th + , "ghc-boot-th:ghc-boot-th" -- dependency of template-haskell + , "pretty:pretty" -- dependency of ghc-boot-th -- other boot libraries used by tests - , "array" - , "binary" - , "bytestring" - , "Cabal" - , "Cabal-syntax" - , "containers" - , "deepseq" - , "directory" - , "exceptions" - , "file-io" - , "filepath" - , "hpc" - , "mtl" - , "os-string" - , "parsec" - , "process" - , "semaphore-compat" - , "text" - , "time" - , "transformers" - , "unix" -- FIXME: we'd have to install Win32 for Windows target. Maybe --libs could install dependencies too.. + , "array:array" + , "binary:binary" + , "bytestring:bytestring" + , "Cabal:Cabal" + , "Cabal-syntax:Cabal-syntax" + , "containers:containers" + , "deepseq:deepseq" + , "directory:directory" + , "exceptions:exceptions" + , "file-io:file-io" + , "filepath:filepath" + , "hpc:hpc" + , "mtl:mtl" + , "os-string:os-string" + , "parsec:parsec" + , "process:process" + , "semaphore-compat:semaphore-compat" + , "text:text" + , "time:time" + , "transformers:transformers" + , "unix:unix" -- FIXME: we'd have to install Win32 for Windows target. Maybe --libs could install dependencies too.. -- ghc related - , "ghc-boot" - , "ghc-heap" - , "ghc-platform" - , "ghc-toolchain" -- some test requires this - , "ghci" - , "ghc" + , "ghc-boot:ghc-boot" + , "ghc-heap:ghc-heap" + , "ghc-platform:ghc-platform" + , "ghc-toolchain:ghc-toolchain" -- some test requires this + , "ghci:ghci" + , "ghc:ghc" ] msg " - Building boot libraries..." From 306d8dbcf17b421dcd3cd167cb7d84aa3b8d801c Mon Sep 17 00:00:00 2001 From: Sylvain Henry Date: Wed, 14 May 2025 16:58:13 +0200 Subject: [PATCH 175/175] Fix installation path for targets --- Build.hs | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/Build.hs b/Build.hs index 094a16132f8b..aa2e1aba0b21 100755 --- a/Build.hs +++ b/Build.hs @@ -98,17 +98,18 @@ main = do cp "_build/stage1/lib/settings" "_build/stage2/lib/settings" -- Now we build extra targets. Ideally those should be built on demand... - createDirectoryIfMissing True "_build/stage2/targets/" - let targets = - [ (,) "aarch64-linux" emptySettings - { settingsTriple = Just "aarch64-linux" - , settingsCc = ProgOpt (Just "aarch64-linux-zig-cc") Nothing - , settingsCxx = ProgOpt (Just "aarch64-linux-zig-c++") Nothing - , settingsLd = ProgOpt (Just "aarch64-linux-zig-cc") Nothing - , settingsMergeObjs = ProgOpt (Just "aarch64-linux-zig-cc") Nothing - , settingsCrossCompiling = True - , settingsUnlit = "$topdir/../../../bin/unlit" - } + targets_dir <- makeAbsolute "_build/stage2/lib/targets/" + createDirectoryIfMissing True targets_dir + let targets = [ +-- [ (,) "aarch64-linux" emptySettings +-- { settingsTriple = Just "aarch64-linux" +-- , settingsCc = ProgOpt (Just "aarch64-linux-zig-cc") Nothing +-- , settingsCxx = ProgOpt (Just "aarch64-linux-zig-c++") Nothing +-- , settingsLd = ProgOpt (Just "aarch64-linux-zig-cc") Nothing +-- , settingsMergeObjs = ProgOpt (Just "aarch64-linux-zig-cc") Nothing +-- , settingsCrossCompiling = True +-- , settingsUnlit = "$topdir/../../../bin/unlit" +-- } -- , (,) "javascript" emptySettings -- { settingsTriple = Just "javascript-unknown-ghcjs" -- , settingsCc = ProgOpt (Just "emcc") Nothing @@ -118,7 +119,7 @@ main = do ghc_stage2_abs <- makeAbsolute "_build/stage2/bin/ghc" forM_ targets $ \(target,settings) -> do msg $ "Bootstrapping target: " <> target - target_dir <- makeAbsolute ("_build/stage2/targets" target) + target_dir <- makeAbsolute (targets_dir target) createDirectoryIfMissing True target_dir generateSettings ghcToolchain settings target_dir -- compiler flags aren't passed consistently to configure, etc.