Cross compilation for aarch64-unknown-linux-musl fails

[windsource]

Problem:

When building aws-lc in Picus for aarch64-unknown-linux-musl target, the build fails with:

   Compiling picus v0.7.1 (/workspaces/picus)
error: linking with `aarch64-linux-gnu-gcc` failed: exit status: 1
  |
  = note: LC_ALL="C" PATH="/usr/local/rustup/toolchains/1.85.0-aarch64-unknown-linux-gnu/lib/rustlib/aarch64-unknown-linux-gnu/bin:/usr/local/rustup/toolchains/1.85.0-aarch64-unknown-linux-gnu/lib/rustlib/aarch64-unknown-linux-gnu/bin/self-contained:/vscode/vscode-server/bin/linux-arm64/ddc367ed5c8936efe395cffeec279b04ffd7db78/bin/remote-cli:/usr/local/cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/vscode/.vscode-server/data/User/globalStorage/github.copilot-chat/debugCommand" VSLANG="1033" "aarch64-linux-gnu-gcc" "/usr/local/rustup/toolchains/1.85.0-aarch64-unknown-linux-gnu/lib/rustlib/aarch64-unknown-linux-musl/lib/self-contained/crt1.o" "/usr/local/rustup/toolchains/1.85.0-aarch64-unknown-linux-gnu/lib/rustlib/aarch64-unknown-linux-musl/lib/self-contained/crti.o" "/usr/local/rustup/toolchains/1.85.0-aarch64-unknown-linux-gnu/lib/rustlib/aarch64-unknown-linux-musl/lib/self-contained/crtbegin.o" "/tmp/rustc95502k/symbols.o" "<257 object files omitted>" "-Wl,--as-needed" "-Wl,-Bstatic" "/workspaces/picus/target/aarch64-unknown-linux-musl/debug/deps/{libhcloud-203d2c00a344eb59.rlib,libserde_with-beedae677938022e.rlib,libreqwest-0ba1f631406d9781.rlib,libwebpki_roots-b247f70770fed730.rlib,libencoding_rs-8946287d664bf690.rlib,libmime_guess-9a7983fd6221b829.rlib,libunicase-6db22d836901dc56.rlib,libmime-bc7159e9f48d829e.rlib,libsync_wrapper-ee30449b00b1c2eb.rlib,libhandlebars-d4028117e2e3d930.rlib,libpest-798f2beb2b510c92.rlib,libucd_trie-169ecd0dcc4fa04e.rlib,libthiserror-ec54cdb1e9f25dd9.rlib,libthiserror-437d3f78c39a582a.rlib,libaws_sdk_ec2-c128c998b547b874.rlib,libaws_config-818d586a2aba7c6b.rlib,libaws_sdk_ssooidc-390c57629374051b.rlib,libaws_sdk_sso-65a6b2f76fb9d0ba.rlib,libaws_sdk_sts-dc62946c18f3256f.rlib,libaws_smithy_query-57bd7a2f35ffc70d.rlib,liburlencoding-56b89a313307f56f.rlib,libaws_smithy_xml-f056c80a106afcca.rlib,libxmlparser-7b25497f334a5814.rlib,libregex_lite-82dba9a18d8fb673.rlib,libaws_runtime-f485d93ebb9e26ea.rlib,libuuid-ec364e812c1cac93.rlib,libaws_sigv4-b36b926927ca3f7e.rlib,libhex-7a1180749c055cce.rlib,libaws_smithy_http-a7564e8c996ce779.rlib,libsha2-c8394ebc31f713dc.rlib,libhmac-76bcdb6a19f6ead2.rlib,libdigest-f1f7ae532a1e7395.rlib,libblock_buffer-1710bdb6896f44aa.rlib,libcrypto_common-ea1315506ed332f1.rlib,libgeneric_array-d370256e7aef9ecc.rlib,libtypenum-c03edd6229f81168.rlib,libaws_smithy_json-907ee06102dea8c1.rlib,libaws_smithy_runtime-8e417a511f6b4fa7.rlib,libfastrand-902a912f5d699ead.rlib,libaws_smithy_http_client-cee7b86c403fa9c2.rlib,libhyper_rustls-59c7d938ef572876.rlib,librustls_native_certs-006d73801baf047a.rlib,librustls_pemfile-f014fab5fc0c1112.rlib,libbase64-43a21695f7d9c80f.rlib,libtokio_rustls-b70443e6161221fb.rlib,librustls-b153408b9c2f51c4.rlib,libsct-0c56b2f36c2ce5e0.rlib,libwebpki-1ad44fb9fa5a32b4.rlib,libhyper-dfef3cc35e060084.rlib,libh2-05b63bc16c5ff7a9.rlib,libhttpdate-366d07c9da9bd718.rlib,libaws_types-8891ce2256571a33.rlib,libaws_credential_types-7e1d4f0137150fa2.rlib,libaws_smithy_runtime_api-78fdd42a272743cd.rlib,libaws_smithy_async-e5d77218217febc0.rlib,libaws_smithy_types-bcb2a96be22476de.rlib,libtime-8d6c423565b7308d.rlib,libtime_core-f86fbea70bd02fa5.rlib,libnum_conv-7c30ea80232c59b5.rlib,libderanged-d811404ad0218cfa.rlib,libpowerfmt-38fbe12198078070.rlib,libnum_integer-ffc5ad6140774435.rlib,libnum_traits-d9f407d0bdec46f4.rlib,libbytes_utils-a2fbcd9e85be06a5.rlib,libeither-231beae80f5fa747.rlib,libhttp_body-277f3ee3e755ed59.rlib,libhttp-88f7acbf5535980a.rlib,libbase64_simd-62b4f98e48e0b93a.rlib,liboutref-6f85f90e67e501a3.rlib,libvsimd-f896f42f3c0eb05c.rlib,libreqwest-ea3f77ea727f1183.rlib,libserde_urlencoded-f9c13ddd28c5cfb7.rlib,libbase64-7b8e80d38b328719.rlib,librustls_pemfile-2033bd9f568a6028.rlib,libipnet-d4783874fc0cefde.rlib,libhyper_rustls-46776c1c737e43d8.rlib,libwebpki_roots-1bb5a724865e16d4.rlib,librustls_native_certs-6820b48a1251c2fa.rlib,libopenssl_probe-3d96246855dc051c.rlib,libtokio_rustls-06a1df9209422339.rlib,librustls-565fb91c58dff2ea.rlib,libsubtle-829c4b1583ed0461.rlib,libwebpki-351fdc18eabfecfe.rlib,libaws_lc_rs-703c4cfd73d00519.rlib,libaws_lc_sys-bb7a2ae84157ba0b.rlib,libring-1df967c5702efe56.rlib,libgetrandom-9f5d050b5248214f.rlib,libuntrusted-0fa81c741479aa79.rlib,libzeroize-70930341f2782a68.rlib,librustls_pki_types-379b05a36a4eedf1.rlib,libserde_json-b13e971a18991a03.rlib,libryu-0562386e6bddca49.rlib,libserde-5547f4c85382ca2c.rlib,libtower-809030cdcb5f87b3.rlib,libsync_wrapper-d342e4fc501a8387.rlib,libtower_layer-7499423d9feb29ee.rlib,libhyper_util-a5e64df852ccc31b.rlib,libtower_service-a6fd0f94a4a8f31f.rlib,libhyper-451fe9f88d44e1db.rlib,libwant-53f7bb69b2f0a2e8.rlib,libtry_lock-071ef9316f9b352d.rlib,libhttparse-b0ced4259b1bb010.rlib,libh2-6d14c3f4e40f60c1.rlib,libtracing-14b9e764e13fec72.rlib,libtracing_core-afafb3d9c3b84d19.rlib,libonce_cell-0a65b63d3b9ad533.rlib,libindexmap-5b5b6b12099873ae.rlib,libequivalent-6272db8f689cf282.rlib,libhashbrown-ff60ca5ab557fae7.rlib,libslab-98487f9f5ae1429e.rlib,libatomic_waker-6ea510a99b4c96f8.rlib,libtokio_util-1b5bdea13fab4569.rlib,libfutures_sink-6de7e0d469f530f2.rlib,libfutures_channel-bf603c061aac12af.rlib,libfutures_util-cc91bba2c2c993cb.rlib,libfutures_task-34658bee21b7d9c6.rlib,libpin_utils-4901902bf9599d9a.rlib,libtokio-daa75b3bc92825fa.rlib,libsignal_hook_registry-8229406c5af17757.rlib,libsocket2-91f57e332d02270e.rlib,libmio-dc70807a15ab7ea7.rlib,libparking_lot-badecc2848e2a4b8.rlib,libparking_lot_core-d5d079b4854686f6.rlib,liblibc-a53b5f2d11a87dad.rlib,libcfg_if-b0973cc216c7c166.rlib,liblock_api-60724ca31af5b96d.rlib,libscopeguard-a96b28d7e9979a11.rlib,libhttp_body_util-4f0908782625dd0d.rlib,libpin_project_lite-19e694cd1d9c4eb8.rlib,libfutures_core-d69dacf7c40e152e.rlib,libhttp_body-579be86cdf3e5afd.rlib,liburl-f219b27b2c85cddb.rlib,libidna-e26efa9b57984c02.rlib,libidna_adapter-33a423bcd2f9521d.rlib,libicu_normalizer-cf7091937a2fba99.rlib,libicu_normalizer_data-5171ae772c269628.rlib,libwrite16-29ea5c97b95f2e71.rlib,libutf8_iter-9fa9f7f89f7caeac.rlib,libutf16_iter-d946f1e01567e97a.rlib,libsmallvec-1dd4928e0676beec.rlib,libicu_properties-d7a0cfc3563624f4.rlib,libicu_properties_data-ac6e5aa3250361e9.rlib,libicu_locid_transform-4b2bbc44a6c30a53.rlib,libicu_locid_transform_data-9c6f8f926683b9b4.rlib,libicu_collections-1531138923068925.rlib,libicu_provider-ee750cbbfe79b0b6.rlib,libicu_locid-dffd0e4810aa9495.rlib,liblitemap-6c909f0e31711a2a.rlib,libtinystr-3c1f5c60221efe46.rlib,libzerovec-5108f7e6e4a5085f.rlib,libwriteable-72a2dac0d87166ec.rlib,libyoke-07c6a9676e0e910c.rlib,libzerofrom-0500c1087b58a8b0.rlib,libstable_deref_trait-1402f23ccf345fac.rlib,libform_urlencoded-ae8f29fbcf0a0479.rlib,libpercent_encoding-69705fe372a4b436.rlib,libhttp-a80a26a426429bdf.rlib,libitoa-840c8d9bd7ad092b.rlib,libbytes-15215f0470bfce56.rlib,libfnv-02623a90c484b066.rlib,libgo_parse_duration-60238e02cbcbfced.rlib,libenv_logger-a4cd1f9eb662d181.rlib,libanstream-2556149212f70187.rlib,libanstyle_query-08d1886ed10e596a.rlib,libis_terminal_polyfill-13ad6d78e5d6a8da.rlib,libcolorchoice-5f488bb413c7edc1.rlib,libanstyle_parse-f999836da6b01155.rlib,libutf8parse-aea1c288d1aa365c.rlib,libjiff-7f849399b03f84c2.rlib,libenv_filter-ca08f85d159e1cb0.rlib,libregex-ac79614dbd1e3b8c.rlib,libregex_automata-1bd2e4182ae5d983.rlib,libaho_corasick-505409ca83460218.rlib,libmemchr-9dffc81fcca0329f.rlib,libregex_syntax-d7831e61749da18d.rlib,libanstyle-692c49b83d0dffa6.rlib,liblog-9498b37752b8e128.rlib}" "/usr/local/rustup/toolchains/1.85.0-aarch64-unknown-linux-gnu/lib/rustlib/aarch64-unknown-linux-musl/lib/{libstd-3e2efd1a481c0ab2.rlib,libpanic_unwind-8a4ea46307616d22.rlib,libobject-cf4984c6d1a83209.rlib,libmemchr-ec46ff5ce84eb35e.rlib,libaddr2line-d8b3534036063e7c.rlib,libgimli-c0ff44efac1a631b.rlib,librustc_demangle-34a9f646db9afd95.rlib,libstd_detect-efd85d558fe39cac.rlib,libhashbrown-f0cded53f01b2d3f.rlib,librustc_std_workspace_alloc-5e02cc48ef2d9866.rlib,libminiz_oxide-9443a2f1302633f8.rlib,libadler-24094517a017e0ba.rlib,libunwind-e3d00cd3cd35ddfa.rlib}" "-lunwind" "/usr/local/rustup/toolchains/1.85.0-aarch64-unknown-linux-gnu/lib/rustlib/aarch64-unknown-linux-musl/lib/{libcfg_if-767547d90cf597ba.rlib,liblibc-1d6ac4480253a986.rlib}" "-lc" "/usr/local/rustup/toolchains/1.85.0-aarch64-unknown-linux-gnu/lib/rustlib/aarch64-unknown-linux-musl/lib/{liballoc-cf7c629899f267e8.rlib,librustc_std_workspace_core-a81575ff4b0cef42.rlib,libcore-8f76cdd875c0ef03.rlib,libcompiler_builtins-aeee3f40d11e1338.rlib}" "-Wl,-Bdynamic" "-Wl,--eh-frame-hdr" "-Wl,-z,noexecstack" "-nostartfiles" "-L" "/workspaces/picus/target/aarch64-unknown-linux-musl/debug/build/ring-f9e31b0e8e04e310/out" "-L" "/workspaces/picus/target/aarch64-unknown-linux-musl/debug/build/aws-lc-sys-5a21f670eecc2a40/out" "-L" "/usr/local/rustup/toolchains/1.85.0-aarch64-unknown-linux-gnu/lib/rustlib/aarch64-unknown-linux-musl/lib/self-contained" "-L" "/usr/local/rustup/toolchains/1.85.0-aarch64-unknown-linux-gnu/lib/rustlib/aarch64-unknown-linux-musl/lib" "-o" "/workspaces/picus/target/aarch64-unknown-linux-musl/debug/deps/picus-8a240f453cbc494a" "-Wl,--gc-sections" "-static" "-no-pie" "-Wl,-z,relro,-z,now" "-nodefaultlibs" "/usr/local/rustup/toolchains/1.85.0-aarch64-unknown-linux-gnu/lib/rustlib/aarch64-unknown-linux-musl/lib/self-contained/crtend.o" "/usr/local/rustup/toolchains/1.85.0-aarch64-unknown-linux-gnu/lib/rustlib/aarch64-unknown-linux-musl/lib/self-contained/crtn.o"
  = note: some arguments are omitted. use `--verbose` to show all linker arguments
  = note: /usr/bin/ld: /workspaces/picus/target/aarch64-unknown-linux-musl/debug/deps/libaws_lc_sys-bb7a2ae84157ba0b.rlib(f8e4fd781484bd36-bcm.o): in function `aws_lc_0_27_0_handle_cpu_env':
          /usr/local/cargo/registry/src/index.crates.io-1949cf8c6b5b557f/aws-lc-sys-0.27.0/aws-lc/crypto/fipsmodule/cpucap/cpu_aarch64.c:18:(.text.aws_lc_0_27_0_handle_cpu_env+0x100): undefined reference to `__isoc23_sscanf'
          /usr/bin/ld: /usr/local/cargo/registry/src/index.crates.io-1949cf8c6b5b557f/aws-lc-sys-0.27.0/aws-lc/crypto/fipsmodule/cpucap/cpu_aarch64.c:20:(.text.aws_lc_0_27_0_handle_cpu_env+0x12c): undefined reference to `__isoc23_sscanf'
          /usr/bin/ld: /workspaces/picus/target/aarch64-unknown-linux-musl/debug/deps/libaws_lc_sys-bb7a2ae84157ba0b.rlib(f8e4fd781484bd36-bcm.o): in function `pkey_rsa_ctrl_str':
          /usr/local/cargo/registry/src/index.crates.io-1949cf8c6b5b557f/aws-lc-sys-0.27.0/aws-lc/crypto/fipsmodule/evp/p_rsa.c:741:(.text.pkey_rsa_ctrl_str+0x1a8): undefined reference to `__isoc23_strtol'
          /usr/bin/ld: /usr/local/cargo/registry/src/index.crates.io-1949cf8c6b5b557f/aws-lc-sys-0.27.0/aws-lc/crypto/fipsmodule/evp/p_rsa.c:752:(.text.pkey_rsa_ctrl_str+0x23c): undefined reference to `__isoc23_strtol'
          collect2: error: ld returned 1 exit status
          
  = note: some `extern` functions couldn't be found; some native libraries may need to be installed or have their path specified
  = note: use the `-l` flag to specify native libraries to link
  = note: use the `cargo:rustc-link-lib` directive to specify the native libraries to link with Cargo (see https://doc.rust-lang.org/cargo/reference/build-scripts.html#rustc-link-lib)

error: could not compile `picus` (bin "picus") due to 1 previous error

The same problem also occurs for the x86_64-unknown-linux-musl target. It looks like musl does not support the C23 standard yet. The Dockerfile of the build container is available and also the image.

Relevant details

AWS-LC for Rust versions or commit:

• aws-lc-rs: 1.12.6
• aws-lc-sys: 0.27.0

System information:

• CPU architecture: aarch64
• CPU name: Apple M4
• OS: Ubuntu 24.10 (in a dev container)

Build log: see above

> rustc --version
rustc 1.85.0 (4d91de4e4 2025-02-17)

[justsmth]

Hello! Thanks for reporting this issue.

This is an unusual error. We typically attempt to build with C11 if we detect that the compiler supports it. I do have a couple of potential workarounds:

• Try setting export AWS_LC_SYS_CMAKE_BUILDER=1 in the build environment. This will perform the build using CMake (instead of using the cc crate).
• Try setting export AWS_LC_SYS_C_STD=11 in the build environment. This will force the build to specify the C11 standard.

The variables above can be both be set at the same time, or they can be set/unset individually.

Please let us know if either (or both) of the above workarounds address the issue. I'll try to take some time next week to investigate and reproduce the problem. Once the problem is remediated, those workarounds should not be needed. Thanks again!

[windsource]

Hi @justsmth, unfortunately the fix does not work. I have tried export AWS_LC_SYS_CMAKE_BUILDER=1 and it did not work and then combined it it with export AWS_LC_SYS_C_STD=11 but the problem remains the same. The linker still complains undefined reference to '__isoc23_sscanf' and others.

[windsource]

In order to reproduce the problem:

1. git clone https://github.com/windsource/picus.git
2. Open in VSCode and use the dev container. It supports aarch64 and x86_64 platforms.
3. cargo build --target aarch64-unknown-linux-musl

The problem is already on the main branch.

[justsmth]

I looked into this. I can also reproduce it by targeting x86_64-unknown-linux-musl. I don't have a solution, but I think I've identified the cause.

The symbol __isoc23_sscanf appears in GNU's libc:

❯ nm -j /usr/lib/x86_64-linux-gnu/libc.a 2>&1 | grep __isoc23_sscanf
__isoc23_sscanf
__isoc23_sscanf
__isoc23_sscanf
__isoc23_sscanf

But not in the libc provided for MUSL:

❯ nm -j  /home/justsmth/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/x86_64-unknown-linux-musl/lib/self-contained/libc.a| grep sscanf
sscanf
__isoc99_sscanf
sscanf
vsscanf
__isoc99_vsscanf
vsscanf

So the issue seems to be that the wrong system header files were used when building AWS-LC. I would need to dig more into this build environment to identify why the wrong header files are being picked up. I hope this helps!

[windsource]

Hi @justsmth, well I have identified the same topic. But the problem only occurs with with aws-lc. I thought the the header file configuration is all done when installing a target like

rustup target add x86_64-unknown-linux-musl

Do you have any idea how aws-lc searches for header files when being build by cargo build?

[justsmth]

Do you have any idea how aws-lc searches for header files when being build by cargo build?

For the x86_64-unknown-linux-musl target, we just build with the cc crate. We don't do anything to directly affect which system header files are being used.

Could there be an environment variable (like CFLAGS?) that's causing additional include paths to be used?

[justsmth]

I set CC_ENABLE_DEBUG_OUTPUT=ON in my environment to get more logging.

It looks like the MUSL system include files are not being passed in when building AWS-LC. In this case the compiler (x86_64-linux-gnu-gcc) would use the default (GNU) header files.

[justsmth]

FYI - I was able to get the build to succeed by setting CC="/usr/bin/x86_64-linux-musl-gcc" in the environment.

[windsource]

@justsmth Thanks a lot for your help!

[cpu]

I bumped into this issue today building for x86_64-unknown-linux-musl.

FYI - I was able to get the build to succeed by setting CC="/usr/bin/x86_64-linux-musl-gcc" in the environment.

This worked for me as well, thank you! 🙇

I see this issue was closed, but I'm fairly certain I had success cross-compiling for a musl target without this step ~a month ago. I'm open to the idea i'm remembering wrong, or otherwise did something different previously, but it would be nice if the friction here could be smoothed out.

Do you think that's something that needs to be pursued here, with cc, or somewhere else?

[justsmth]

I see this issue was closed, but I'm fairly certain I had success cross-compiling for a musl target without this step ~a month ago.

We did make a few changes to our build recently, but it would have mostly affected targets requiring CMake. Our build depends on cc for the selection of the compiler.

Do you think that's something that needs to be pursued here, with cc, or somewhere else?

Yeah, this might be a bug in cc? Looking at its logic for compiler selection, I suspect this line selecting a compiler prefix should change from:

"x86_64-unknown-linux-musl" => Some("musl"),

to:

"x86_64-unknown-linux-musl" => Some("x86_64-linux-musl"),

... so that when this value is combined with the compiler name it results in x86_64-linux-musl-gcc as we would expect?

  let prefix = self.prefix_for_target(&raw_target);
  match prefix {
      Some(prefix) => {
          let cc = if target.abi == "llvm" { clang } else { gnu };
          format!("{}-{}", prefix, cc)
      }
      None => default.to_string(),
  }

It seems that logic should also be capable of considering more than one prefix? There are a few changes to the compiler selection logic that should be considered.

[cpu]

We did make a few changes to our build recently, but it would have mostly affected targets requiring CMake. Our build depends on cc for the selection of the compiler.

I wonder if my cc version changed between attempts 🤔 . Unfortunately I was doing this for a quick one-off test of something and so wasn't very rigorous.

It seems that logic should also be capable of considering more than one prefix? There are a few changes to the compiler selection logic that should be considered.

That makes sense to me. Would you be willing to open an issue with cc with your findings? I can do it myself, but suspect I have less experience here and so it might be more expedient if you're willing :-)