From 87b7b9d759ee7381f1744bc55348693a2de50040 Mon Sep 17 00:00:00 2001 From: clux Date: Wed, 5 Feb 2025 15:32:36 +0000 Subject: [PATCH 1/7] try updating deps Signed-off-by: clux --- Dockerfile.arm64 | 10 +++++----- Dockerfile.x86_64 | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Dockerfile.arm64 b/Dockerfile.arm64 index a644b55..dbe304a 100644 --- a/Dockerfile.arm64 +++ b/Dockerfile.arm64 @@ -54,12 +54,12 @@ RUN chmod a+X /root # Convenience list of versions and variables for compilation later on # This helps continuing manually if anything breaks. ENV SSL_VER="1.1.1w" \ - CURL_VER="8.7.1" \ + CURL_VER="8.11.1" \ ZLIB_VER="1.3.1" \ PQ_VER="11.12" \ - SQLITE_VER="3450300" \ - PROTOBUF_VER="25.3" \ - SCCACHE_VER="0.8.0" \ + SQLITE_VER="3480000" \ + PROTOBUF_VER="29.2" \ + SCCACHE_VER="0.9.1" \ CC=musl-gcc \ PREFIX=/musl \ PATH=/usr/local/bin:/root/.cargo/bin:$PATH \ @@ -128,7 +128,7 @@ RUN curl -sSL https://ftp.postgresql.org/pub/source/v$PQ_VER/postgresql-$PQ_VER. cd .. && rm -rf postgresql-$PQ_VER # Build libsqlite3 using same configuration as the alpine linux main/sqlite package -RUN curl -sSL https://www.sqlite.org/2024/sqlite-autoconf-$SQLITE_VER.tar.gz | tar xz && \ +RUN curl -sSL https://www.sqlite.org/2025/sqlite-autoconf-$SQLITE_VER.tar.gz | tar xz && \ cd sqlite-autoconf-$SQLITE_VER && \ CFLAGS="-DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_SECURE_DELETE -DSQLITE_ENABLE_UNLOCK_NOTIFY -DSQLITE_ENABLE_RTREE -DSQLITE_USE_URI -DSQLITE_ENABLE_DBSTAT_VTAB -DSQLITE_ENABLE_JSON1" \ CC="musl-gcc -fPIC -pie" \ diff --git a/Dockerfile.x86_64 b/Dockerfile.x86_64 index c1fd54b..a4f46fd 100644 --- a/Dockerfile.x86_64 +++ b/Dockerfile.x86_64 @@ -53,12 +53,12 @@ RUN chmod a+X /root # Convenience list of versions and variables for compilation later on # This helps continuing manually if anything breaks. ENV SSL_VER="1.1.1w" \ - CURL_VER="8.7.1" \ + CURL_VER="8.11.1" \ ZLIB_VER="1.3.1" \ PQ_VER="11.12" \ - SQLITE_VER="3450300" \ - PROTOBUF_VER="25.3" \ - SCCACHE_VER="0.8.0" \ + SQLITE_VER="3480000" \ + PROTOBUF_VER="29.2" \ + SCCACHE_VER="0.9.1" \ CC=musl-gcc \ PREFIX=/musl \ PATH=/usr/local/bin:/root/.cargo/bin:$PATH \ @@ -127,7 +127,7 @@ RUN curl -sSL https://ftp.postgresql.org/pub/source/v$PQ_VER/postgresql-$PQ_VER. cd .. && rm -rf postgresql-$PQ_VER # Build libsqlite3 using same configuration as the alpine linux main/sqlite package -RUN curl -sSL https://www.sqlite.org/2024/sqlite-autoconf-$SQLITE_VER.tar.gz | tar xz && \ +RUN curl -sSL https://www.sqlite.org/2025/sqlite-autoconf-$SQLITE_VER.tar.gz | tar xz && \ cd sqlite-autoconf-$SQLITE_VER && \ CFLAGS="-DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_SECURE_DELETE -DSQLITE_ENABLE_UNLOCK_NOTIFY -DSQLITE_ENABLE_RTREE -DSQLITE_USE_URI -DSQLITE_ENABLE_DBSTAT_VTAB -DSQLITE_ENABLE_JSON1" \ CC="musl-gcc -fPIC -pie" \ From e6b2b1ed2818bcd02a4f1efcc7b7f9ac8847561c Mon Sep 17 00:00:00 2001 From: clux Date: Mon, 24 Feb 2025 09:21:11 +0000 Subject: [PATCH 2/7] re-run updater Signed-off-by: clux --- Dockerfile.arm64 | 4 ++-- Dockerfile.x86_64 | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile.arm64 b/Dockerfile.arm64 index dbe304a..a02c8cb 100644 --- a/Dockerfile.arm64 +++ b/Dockerfile.arm64 @@ -54,10 +54,10 @@ RUN chmod a+X /root # Convenience list of versions and variables for compilation later on # This helps continuing manually if anything breaks. ENV SSL_VER="1.1.1w" \ - CURL_VER="8.11.1" \ + CURL_VER="8.12.1" \ ZLIB_VER="1.3.1" \ PQ_VER="11.12" \ - SQLITE_VER="3480000" \ + SQLITE_VER="3490100" \ PROTOBUF_VER="29.2" \ SCCACHE_VER="0.9.1" \ CC=musl-gcc \ diff --git a/Dockerfile.x86_64 b/Dockerfile.x86_64 index a4f46fd..f49f466 100644 --- a/Dockerfile.x86_64 +++ b/Dockerfile.x86_64 @@ -53,10 +53,10 @@ RUN chmod a+X /root # Convenience list of versions and variables for compilation later on # This helps continuing manually if anything breaks. ENV SSL_VER="1.1.1w" \ - CURL_VER="8.11.1" \ + CURL_VER="8.12.1" \ ZLIB_VER="1.3.1" \ PQ_VER="11.12" \ - SQLITE_VER="3480000" \ + SQLITE_VER="3490100" \ PROTOBUF_VER="29.2" \ SCCACHE_VER="0.9.1" \ CC=musl-gcc \ From 9de5493195cb1eff48c6ac94d30e3d11be30e2c0 Mon Sep 17 00:00:00 2001 From: clux Date: Mon, 24 Feb 2025 09:26:50 +0000 Subject: [PATCH 3/7] try bumping to latest lts (noble) also remove a (now-default) removed flag. Signed-off-by: clux --- Dockerfile.arm64 | 4 ++-- Dockerfile.x86_64 | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile.arm64 b/Dockerfile.arm64 index a02c8cb..4669bc0 100644 --- a/Dockerfile.arm64 +++ b/Dockerfile.arm64 @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1 -FROM ubuntu:jammy +FROM ubuntu:noble LABEL maintainer="Eirik Albrigtsen " # Required packages: @@ -132,7 +132,7 @@ RUN curl -sSL https://www.sqlite.org/2025/sqlite-autoconf-$SQLITE_VER.tar.gz | t cd sqlite-autoconf-$SQLITE_VER && \ CFLAGS="-DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_SECURE_DELETE -DSQLITE_ENABLE_UNLOCK_NOTIFY -DSQLITE_ENABLE_RTREE -DSQLITE_USE_URI -DSQLITE_ENABLE_DBSTAT_VTAB -DSQLITE_ENABLE_JSON1" \ CC="musl-gcc -fPIC -pie" \ - ./configure --prefix=$PREFIX --host=x86_64-unknown-linux-musl --enable-threadsafe --enable-dynamic-extensions --disable-shared && \ + ./configure --prefix=$PREFIX --host=x86_64-unknown-linux-musl --enable-threadsafe --disable-shared && \ make && make install && \ cd .. && rm -rf sqlite-autoconf-$SQLITE_VER diff --git a/Dockerfile.x86_64 b/Dockerfile.x86_64 index f49f466..65d374c 100644 --- a/Dockerfile.x86_64 +++ b/Dockerfile.x86_64 @@ -1,4 +1,4 @@ -FROM ubuntu:jammy +FROM ubuntu:noble LABEL maintainer="Eirik Albrigtsen " # Required packages: @@ -131,7 +131,7 @@ RUN curl -sSL https://www.sqlite.org/2025/sqlite-autoconf-$SQLITE_VER.tar.gz | t cd sqlite-autoconf-$SQLITE_VER && \ CFLAGS="-DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_FTS3_PARENTHESIS -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_COLUMN_METADATA -DSQLITE_SECURE_DELETE -DSQLITE_ENABLE_UNLOCK_NOTIFY -DSQLITE_ENABLE_RTREE -DSQLITE_USE_URI -DSQLITE_ENABLE_DBSTAT_VTAB -DSQLITE_ENABLE_JSON1" \ CC="musl-gcc -fPIC -pie" \ - ./configure --prefix=$PREFIX --host=x86_64-unknown-linux-musl --enable-threadsafe --enable-dynamic-extensions --disable-shared && \ + ./configure --prefix=$PREFIX --host=x86_64-unknown-linux-musl --enable-threadsafe --disable-shared && \ make && make install && \ cd .. && rm -rf sqlite-autoconf-$SQLITE_VER From 22f29add99f929e445ff8c9bc58bb6639fab1f16 Mon Sep 17 00:00:00 2001 From: clux Date: Mon, 24 Feb 2025 11:31:02 +0000 Subject: [PATCH 4/7] test runner should also noble Signed-off-by: clux --- Dockerfile.test-runner | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.test-runner b/Dockerfile.test-runner index e6ddffc..ec2f0b9 100644 --- a/Dockerfile.test-runner +++ b/Dockerfile.test-runner @@ -1,4 +1,4 @@ -FROM ubuntu:jammy +FROM ubuntu:noble RUN apt-get update && apt-get install -y \ ca-certificates \ From 79c37b8fb81a17dacebb60ddbd54238c34ff6d7a Mon Sep 17 00:00:00 2001 From: clux Date: Mon, 24 Feb 2025 11:43:18 +0000 Subject: [PATCH 5/7] poor man's local environment detection Signed-off-by: clux --- justfile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/justfile b/justfile index 6772f97..b51ce9d 100644 --- a/justfile +++ b/justfile @@ -1,4 +1,4 @@ -# See https://just.systems/man/ + [private] default: @@ -24,12 +24,13 @@ test-setup: # Test an individual crate against built container _t crate: - ./test.sh {{crate}} + just {{ ENVIRON_FN }} {{crate}} +# poor man's environment detection for a man with 2 machines +ENVIRON_FN := if os() == "macos" { "_t_arm" } else { "_t_amd" } # when running locally use one of these instead of _t _t_amd crate: #!/bin/bash - # TODO: make a variant for arm here, or do platform inference export PLATFORM="linux/amd64" export TARGET_DIR="x86_64-unknown-linux-musl" export AR="amd64" From fecdc17261000feb7a668122160136f8e74f0bbc Mon Sep 17 00:00:00 2001 From: clux Date: Mon, 24 Feb 2025 11:50:41 +0000 Subject: [PATCH 6/7] drop curl for initial release Signed-off-by: clux --- Dockerfile.arm64 | 16 ++++++++-------- Dockerfile.x86_64 | 16 ++++++++-------- README.md | 4 ++-- justfile | 3 ++- 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/Dockerfile.arm64 b/Dockerfile.arm64 index 4669bc0..7d65f4f 100644 --- a/Dockerfile.arm64 +++ b/Dockerfile.arm64 @@ -107,14 +107,14 @@ RUN curl -sSL https://www.openssl.org/source/openssl-$SSL_VER.tar.gz | tar xz && # Build curl (needs with-zlib and all this stuff to allow https) # curl_LDFLAGS needed on stretch to avoid fPIC errors - though not sure from what -RUN curl -sSL https://curl.se/download/curl-$CURL_VER.tar.gz | tar xz && \ - cd curl-$CURL_VER && \ - CC="musl-gcc -fPIC -pie" LDFLAGS="-L$PREFIX/lib" CFLAGS="-I$PREFIX/include" ./configure \ - --enable-shared=no --with-zlib --enable-static=ssl --enable-optimize --prefix=$PREFIX \ - --with-ca-path=/etc/ssl/certs/ --with-ca-bundle=/etc/ssl/certs/ca-certificates.crt --without-ca-fallback \ - --with-openssl --without-libpsl && \ - make -j$(nproc) curl_LDFLAGS="-all-static" && make install && \ - cd .. && rm -rf curl-$CURL_VER +# RUN curl -sSL https://curl.se/download/curl-$CURL_VER.tar.gz | tar xz && \ + # cd curl-$CURL_VER && \ + # CC="musl-gcc -fPIC -pie" LDFLAGS="-L$PREFIX/lib" CFLAGS="-I$PREFIX/include" ./configure \ + # --enable-shared=no --with-zlib --enable-static=ssl --enable-optimize --prefix=$PREFIX \ + # --with-ca-path=/etc/ssl/certs/ --with-ca-bundle=/etc/ssl/certs/ca-certificates.crt --without-ca-fallback \ + # --with-openssl --without-libpsl && \ + # make -j$(nproc) curl_LDFLAGS="-all-static" && make install && \ + # cd .. && rm -rf curl-$CURL_VER # Build libpq RUN curl -sSL https://ftp.postgresql.org/pub/source/v$PQ_VER/postgresql-$PQ_VER.tar.gz | tar xz && \ diff --git a/Dockerfile.x86_64 b/Dockerfile.x86_64 index 65d374c..ec4ccf8 100644 --- a/Dockerfile.x86_64 +++ b/Dockerfile.x86_64 @@ -106,14 +106,14 @@ RUN curl -sSL https://www.openssl.org/source/openssl-$SSL_VER.tar.gz | tar xz && # Build curl (needs with-zlib and all this stuff to allow https) # curl_LDFLAGS needed on stretch to avoid fPIC errors - though not sure from what -RUN curl -sSL https://curl.se/download/curl-$CURL_VER.tar.gz | tar xz && \ - cd curl-$CURL_VER && \ - CC="musl-gcc -fPIC -pie" LDFLAGS="-L$PREFIX/lib" CFLAGS="-I$PREFIX/include" ./configure \ - --enable-shared=no --with-zlib --enable-static=ssl --enable-optimize --prefix=$PREFIX \ - --with-ca-path=/etc/ssl/certs/ --with-ca-bundle=/etc/ssl/certs/ca-certificates.crt --without-ca-fallback \ - --with-openssl --without-libpsl && \ - make -j$(nproc) curl_LDFLAGS="-all-static" && make install && \ - cd .. && rm -rf curl-$CURL_VER +# RUN curl -sSL https://curl.se/download/curl-$CURL_VER.tar.gz | tar xz && \ + # cd curl-$CURL_VER && \ + # CC="musl-gcc -fPIC -pie" LDFLAGS="-L$PREFIX/lib" CFLAGS="-I$PREFIX/include" ./configure \ + # --enable-shared=no --with-zlib --enable-static=ssl --enable-optimize --prefix=$PREFIX \ + # --with-ca-path=/etc/ssl/certs/ --with-ca-bundle=/etc/ssl/certs/ca-certificates.crt --without-ca-fallback \ + # --with-openssl --without-libpsl && \ + # make -j$(nproc) curl_LDFLAGS="-all-static" && make install && \ + # cd .. && rm -rf curl-$CURL_VER # Build libpq RUN curl -sSL https://ftp.postgresql.org/pub/source/v$PQ_VER/postgresql-$PQ_VER.tar.gz | tar xz && \ diff --git a/README.md b/README.md index e1a632a..45f43ef 100644 --- a/README.md +++ b/README.md @@ -50,11 +50,11 @@ For pinned, or historical builds, see the [available tags on dockerhub](https:// The following system libraries are compiled against `musl-gcc`: -- [x] curl ([curl crate](https://github.com/carllerche/curl-rust)) - [x] openssl ([openssl crate](https://github.com/sfackler/rust-openssl)) - [x] pq ([pq-sys crate](https://github.com/sgrif/pq-sys) used by [diesel](https://github.com/diesel-rs/diesel)) - [x] sqlite3 ([libsqlite3-sys crate](https://github.com/jgallagher/rusqlite/tree/master/libsqlite3-sys) used by [diesel](https://github.com/diesel-rs/diesel)) - [x] zlib (used by pq and openssl) +- [~] curl ([curl crate](https://github.com/carllerche/curl-rust)) - dropped [#96](https://github.com/clux/muslrust/issues/96) We **[try](https://github.com/clux/muslrust/blob/main/update_libs.py)** to keep these up to date. @@ -75,11 +75,11 @@ Before we push a new version of muslrust we [test](https://github.com/clux/muslr - [x] [serde](https://crates.io/crates/serde) - [x] [diesel](https://crates.io/crates/diesel) - [x] [hyper](https://crates.io/crates/hyper) -- [x] [curl](https://crates.io/crates/curl) - [x] [rustls](https://crates.io/crates/rustls) - [x] [openssl](https://crates.io/crates/openssl) - [x] [flate2](https://crates.io/crates/flate2) - [x] [rand](https://crates.io/crates/rand) +- [~] [curl](https://crates.io/crates/curl) - dropped [#96](https://github.com/clux/muslrust/issues/96) ## Caching diff --git a/justfile b/justfile index b51ce9d..6fd05f9 100644 --- a/justfile +++ b/justfile @@ -44,7 +44,8 @@ _t_arm crate: # Test all crates against built container -test: (_t "plain") (_t "ssl") (_t "rustls") (_t "pq") (_t "serde") (_t "curl") (_t "zlib") (_t "hyper") (_t "dieselpg") (_t "dieselsqlite") +# NB: taken out curl for #96 and build issuse +test: (_t "plain") (_t "ssl") (_t "rustls") (_t "pq") (_t "serde") (_t "zlib") (_t "hyper") (_t "dieselpg") (_t "dieselsqlite") # Cleanup everything clean: clean-docker clean-tests From dd0b0713b7aff2d3b1c58e84b5658126934922e9 Mon Sep 17 00:00:00 2001 From: clux Date: Mon, 24 Feb 2025 16:35:22 +0000 Subject: [PATCH 7/7] fix ci invocation of test also add nicer separation of local dev and ci entrypoints Signed-off-by: clux --- .github/workflows/nightly.yml | 2 +- .github/workflows/stable.yml | 2 +- justfile | 24 ++++++++++++++---------- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 9954e85..9419787 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -78,7 +78,7 @@ jobs: shell: bash run: | docker buildx build --platform ${{ matrix.platform }} --output type=docker -t test-runner - < Dockerfile.test-runner - TARGET_DIR=${{ matrix.target_dir }} PLATFORM=${{ matrix.platform }} just test + TARGET_DIR=${{ matrix.target_dir }} PLATFORM=${{ matrix.platform }} just test-ci # The date/channel/version are expected to be the same on both architectures and are needed for the merge step. # We store them here since it makes the merge step a bit easier - it doesn't need to figure out which of the diff --git a/.github/workflows/stable.yml b/.github/workflows/stable.yml index ebb3ff4..c2eaa14 100644 --- a/.github/workflows/stable.yml +++ b/.github/workflows/stable.yml @@ -92,7 +92,7 @@ jobs: shell: bash run: | docker buildx build --platform ${{ matrix.platform }} --output type=docker -t test-runner - < Dockerfile.test-runner - TARGET_DIR=${{ matrix.target_dir }} PLATFORM=${{ matrix.platform }} just test + TARGET_DIR=${{ matrix.target_dir }} PLATFORM=${{ matrix.platform }} just test-ci # The date/channel/version are expected to be the same on both architectures and are needed for the merge step. # We store them here since it makes the merge step a bit easier - it doesn't need to figure out which of the diff --git a/justfile b/justfile index 6fd05f9..49cbf73 100644 --- a/justfile +++ b/justfile @@ -1,5 +1,4 @@ - [private] default: @just --list --unsorted --color=always @@ -19,33 +18,38 @@ build-nightly-arm: (_build "nightly" "arm64" "linux/arm64" "arm64") run: docker run -v $PWD/test:/volume -w /volume -it rustmusl-temp /bin/bash +# Build test runner test-setup: docker build -t test-runner . -f Dockerfile.test-runner # Test an individual crate against built container _t crate: - just {{ ENVIRON_FN }} {{crate}} + ./test.sh {{crate}} -# poor man's environment detection for a man with 2 machines -ENVIRON_FN := if os() == "macos" { "_t_arm" } else { "_t_amd" } -# when running locally use one of these instead of _t -_t_amd crate: +# Test an individual crate locally using env vars set by _t_amd or t_arm +_ti crate: + # poor man's environment multiplex + just _t_{{ os() }}_{{ arch() }} {{crate}} + +# when running locally we can use one of these instead of _t +_t_linux_amd64 crate: #!/bin/bash export PLATFORM="linux/amd64" export TARGET_DIR="x86_64-unknown-linux-musl" export AR="amd64" ./test.sh {{crate}} -_t_arm crate: +_t_macos_aarch64 crate: #!/bin/bash export PLATFORM="linux/arm64" export TARGET_DIR="aarch64-unknown-linux-musl" export AR="arm64" ./test.sh {{crate}} - -# Test all crates against built container +# Test all crates against built container locally +test: (_ti "plain") (_ti "ssl") (_ti "rustls") (_ti "pq") (_ti "serde") (_ti "zlib") (_ti "hyper") (_ti "dieselpg") (_ti "dieselsqlite") +# Test all crates against built container in ci (inheriting set PLATFORM/TARGET_DIR/AR vars) +test-ci: (_t "plain") (_t "ssl") (_t "rustls") (_t "pq") (_t "serde") (_t "zlib") (_t "hyper") (_t "dieselpg") (_t "dieselsqlite") # NB: taken out curl for #96 and build issuse -test: (_t "plain") (_t "ssl") (_t "rustls") (_t "pq") (_t "serde") (_t "zlib") (_t "hyper") (_t "dieselpg") (_t "dieselsqlite") # Cleanup everything clean: clean-docker clean-tests