11//! Run-time feature detection for RISC-V on Linux.
2+ //!
3+ //! On RISC-V, detection using auxv only supports single-letter extensions.
4+ //! So, we use riscv_hwprobe that supports multi-letter extensions if available.
5+ //! <https://www.kernel.org/doc/html/latest/arch/riscv/hwprobe.html>
6+
7+ use core:: ptr;
28
39use super :: auxvec;
410use crate :: detect:: { Feature , bit, cache} ;
511
6- /// Read list of supported features from the auxiliary vector.
12+ // See <https://github.com/torvalds/linux/blob/master/arch/riscv/include/uapi/asm/hwprobe.h>
13+ // for riscv_hwprobe struct and RISCV_HWPROBE_* constants.
14+
15+ #[ repr( C ) ]
16+ struct riscv_hwprobe {
17+ key : i64 ,
18+ value : u64 ,
19+ }
20+
21+ #[ allow( non_upper_case_globals) ]
22+ const __NR_riscv_hwprobe: libc:: c_long = 258 ;
23+
24+ const RISCV_HWPROBE_KEY_BASE_BEHAVIOR : i64 = 3 ;
25+ const RISCV_HWPROBE_BASE_BEHAVIOR_IMA : u64 = 1 << 0 ;
26+
27+ const RISCV_HWPROBE_KEY_IMA_EXT_0 : i64 = 4 ;
28+ const RISCV_HWPROBE_IMA_FD : u64 = 1 << 0 ;
29+ const RISCV_HWPROBE_IMA_C : u64 = 1 << 1 ;
30+ const RISCV_HWPROBE_IMA_V : u64 = 1 << 2 ;
31+ const RISCV_HWPROBE_EXT_ZBA : u64 = 1 << 3 ;
32+ const RISCV_HWPROBE_EXT_ZBB : u64 = 1 << 4 ;
33+ const RISCV_HWPROBE_EXT_ZBS : u64 = 1 << 5 ;
34+ // const RISCV_HWPROBE_EXT_ZICBOZ: u64 = 1 << 6;
35+ const RISCV_HWPROBE_EXT_ZBC : u64 = 1 << 7 ;
36+ const RISCV_HWPROBE_EXT_ZBKB : u64 = 1 << 8 ;
37+ const RISCV_HWPROBE_EXT_ZBKC : u64 = 1 << 9 ;
38+ const RISCV_HWPROBE_EXT_ZBKX : u64 = 1 << 10 ;
39+ const RISCV_HWPROBE_EXT_ZKND : u64 = 1 << 11 ;
40+ const RISCV_HWPROBE_EXT_ZKNE : u64 = 1 << 12 ;
41+ const RISCV_HWPROBE_EXT_ZKNH : u64 = 1 << 13 ;
42+ const RISCV_HWPROBE_EXT_ZKSED : u64 = 1 << 14 ;
43+ const RISCV_HWPROBE_EXT_ZKSH : u64 = 1 << 15 ;
44+ const RISCV_HWPROBE_EXT_ZKT : u64 = 1 << 16 ;
45+ const RISCV_HWPROBE_EXT_ZVBB : u64 = 1 << 17 ;
46+ const RISCV_HWPROBE_EXT_ZVBC : u64 = 1 << 18 ;
47+ const RISCV_HWPROBE_EXT_ZVKB : u64 = 1 << 19 ;
48+ const RISCV_HWPROBE_EXT_ZVKG : u64 = 1 << 20 ;
49+ const RISCV_HWPROBE_EXT_ZVKNED : u64 = 1 << 21 ;
50+ const RISCV_HWPROBE_EXT_ZVKNHA : u64 = 1 << 22 ;
51+ const RISCV_HWPROBE_EXT_ZVKNHB : u64 = 1 << 23 ;
52+ const RISCV_HWPROBE_EXT_ZVKSED : u64 = 1 << 24 ;
53+ const RISCV_HWPROBE_EXT_ZVKSH : u64 = 1 << 25 ;
54+ const RISCV_HWPROBE_EXT_ZVKT : u64 = 1 << 26 ;
55+ const RISCV_HWPROBE_EXT_ZFH : u64 = 1 << 27 ;
56+ const RISCV_HWPROBE_EXT_ZFHMIN : u64 = 1 << 28 ;
57+ // const RISCV_HWPROBE_EXT_ZIHINTNTL: u64 = 1 << 29;
58+ const RISCV_HWPROBE_EXT_ZVFH : u64 = 1 << 30 ;
59+ const RISCV_HWPROBE_EXT_ZVFHMIN : u64 = 1 << 31 ;
60+ // const RISCV_HWPROBE_EXT_ZFA: u64 = 1 << 32;
61+ const RISCV_HWPROBE_EXT_ZTSO : u64 = 1 << 33 ;
62+ const RISCV_HWPROBE_EXT_ZACAS : u64 = 1 << 34 ;
63+ // const RISCV_HWPROBE_EXT_ZICOND: u64 = 1 << 35;
64+ const RISCV_HWPROBE_EXT_ZIHINTPAUSE : u64 = 1 << 36 ;
65+ const RISCV_HWPROBE_EXT_ZVE32X : u64 = 1 << 37 ;
66+ const RISCV_HWPROBE_EXT_ZVE32F : u64 = 1 << 38 ;
67+ const RISCV_HWPROBE_EXT_ZVE64X : u64 = 1 << 39 ;
68+ const RISCV_HWPROBE_EXT_ZVE64F : u64 = 1 << 40 ;
69+ const RISCV_HWPROBE_EXT_ZVE64D : u64 = 1 << 41 ;
70+ // const RISCV_HWPROBE_EXT_ZIMOP: u64 = 1 << 42;
71+ // const RISCV_HWPROBE_EXT_ZCA: u64 = 1 << 43;
72+ // const RISCV_HWPROBE_EXT_ZCB: u64 = 1 << 44;
73+ // const RISCV_HWPROBE_EXT_ZCD: u64 = 1 << 45;
74+ // const RISCV_HWPROBE_EXT_ZCF: u64 = 1 << 46;
75+ // const RISCV_HWPROBE_EXT_ZCMOP: u64 = 1 << 47;
76+ const RISCV_HWPROBE_EXT_ZAWRS : u64 = 1 << 48 ;
77+ // const RISCV_HWPROBE_EXT_SUPM: u64 = 1 << 49;
78+
79+ const RISCV_HWPROBE_KEY_CPUPERF_0 : i64 = 5 ;
80+ const RISCV_HWPROBE_MISALIGNED_FAST : u64 = 3 ;
81+ const RISCV_HWPROBE_MISALIGNED_MASK : u64 = 7 ;
82+
83+ const RISCV_HWPROBE_KEY_MISALIGNED_VECTOR_PERF : i64 = 10 ;
84+ const RISCV_HWPROBE_MISALIGNED_VECTOR_FAST : u64 = 3 ;
85+
86+ // syscall returns an unsupported error if riscv_hwprobe is not supported,
87+ // so we can safely use this function on older versions of Linux.
88+ fn _riscv_hwprobe ( out : & mut [ riscv_hwprobe ] ) -> bool {
89+ unsafe fn __riscv_hwprobe (
90+ pairs : * mut riscv_hwprobe ,
91+ pair_count : libc:: size_t ,
92+ cpu_set_size : libc:: size_t ,
93+ cpus : * mut libc:: c_ulong ,
94+ flags : libc:: c_uint ,
95+ ) -> libc:: c_long {
96+ unsafe {
97+ libc:: syscall (
98+ __NR_riscv_hwprobe,
99+ pairs,
100+ pair_count,
101+ cpu_set_size,
102+ cpus,
103+ flags,
104+ )
105+ }
106+ }
107+
108+ let len = out. len ( ) ;
109+ unsafe { __riscv_hwprobe ( out. as_mut_ptr ( ) , len, 0 , ptr:: null_mut ( ) , 0 ) == 0 }
110+ }
111+
112+ /// Read list of supported features from riscv_hwprobe or the auxiliary vector.
7113pub ( crate ) fn detect_features ( ) -> cache:: Initializer {
8114 let mut value = cache:: Initializer :: default ( ) ;
115+
116+ let mut out = [
117+ riscv_hwprobe {
118+ key : RISCV_HWPROBE_KEY_BASE_BEHAVIOR ,
119+ value : 0 ,
120+ } ,
121+ riscv_hwprobe {
122+ key : RISCV_HWPROBE_KEY_IMA_EXT_0 ,
123+ value : 0 ,
124+ } ,
125+ riscv_hwprobe {
126+ key : RISCV_HWPROBE_KEY_CPUPERF_0 ,
127+ value : 0 ,
128+ } ,
129+ riscv_hwprobe {
130+ key : RISCV_HWPROBE_KEY_MISALIGNED_VECTOR_PERF ,
131+ value : 0 ,
132+ } ,
133+ ] ;
134+ if _riscv_hwprobe ( & mut out) {
135+ let mut enable_feature = |feature, enable| {
136+ if enable {
137+ value. set ( feature as u32 ) ;
138+ }
139+ } ;
140+ let mut ima = false ;
141+ if out[ 0 ] . key != -1 {
142+ let base_behavior = out[ 0 ] . value ;
143+ ima = base_behavior & RISCV_HWPROBE_BASE_BEHAVIOR_IMA != 0 ;
144+ #[ cfg( target_arch = "riscv32" ) ]
145+ enable_feature ( Feature :: rv32i, ima) ;
146+ #[ cfg( target_arch = "riscv64" ) ]
147+ enable_feature ( Feature :: rv64i, ima) ;
148+ enable_feature ( Feature :: m, ima) ;
149+ enable_feature ( Feature :: a, ima) ;
150+ }
151+ if ima && out[ 1 ] . key != -1 {
152+ let ima_ext_0 = out[ 1 ] . value ;
153+ let fd = ima_ext_0 & RISCV_HWPROBE_IMA_FD != 0 ;
154+ enable_feature ( Feature :: f, fd) ;
155+ enable_feature ( Feature :: d, fd) ;
156+ enable_feature ( Feature :: zicsr, fd) ; // implied by f
157+ enable_feature ( Feature :: c, ima_ext_0 & RISCV_HWPROBE_IMA_C != 0 ) ;
158+ // enable_feature(Feature::zicboz, ima_ext_0 & RISCV_HWPROBE_EXT_ZICBOZ != 0);
159+ enable_feature ( Feature :: zfh, ima_ext_0 & RISCV_HWPROBE_EXT_ZFH != 0 ) ;
160+ enable_feature ( Feature :: zfhmin, ima_ext_0 & RISCV_HWPROBE_EXT_ZFHMIN != 0 ) ;
161+ // enable_feature(Feature::zihintntl, ima_ext_0 & RISCV_HWPROBE_EXT_ZIHINTNTL != 0);
162+ // enable_feature(Feature::zfa, ima_ext_0 & RISCV_HWPROBE_EXT_ZFA != 0);
163+ enable_feature ( Feature :: ztso, ima_ext_0 & RISCV_HWPROBE_EXT_ZTSO != 0 ) ;
164+ enable_feature ( Feature :: zacas, ima_ext_0 & RISCV_HWPROBE_EXT_ZACAS != 0 ) ;
165+ // enable_feature(Feature::zicond, ima_ext_0 & RISCV_HWPROBE_EXT_ZICOND != 0);
166+ enable_feature (
167+ Feature :: zihintpause,
168+ ima_ext_0 & RISCV_HWPROBE_EXT_ZIHINTPAUSE != 0 ,
169+ ) ;
170+ // enable_feature(Feature::zimop, ima_ext_0 & RISCV_HWPROBE_EXT_ZIMOP != 0);
171+ // enable_feature(Feature::zca, ima_ext_0 & RISCV_HWPROBE_EXT_ZCA != 0);
172+ // enable_feature(Feature::zcb, ima_ext_0 & RISCV_HWPROBE_EXT_ZCB != 0);
173+ // enable_feature(Feature::zcd, ima_ext_0 & RISCV_HWPROBE_EXT_ZCD != 0);
174+ // enable_feature(Feature::zcf, ima_ext_0 & RISCV_HWPROBE_EXT_ZCF != 0);
175+ // enable_feature(Feature::zcmop, ima_ext_0 & RISCV_HWPROBE_EXT_ZCMOP != 0);
176+ enable_feature ( Feature :: zawrs, ima_ext_0 & RISCV_HWPROBE_EXT_ZAWRS != 0 ) ;
177+ // enable_feature(Feature::supm, ima_ext_0 & RISCV_HWPROBE_EXT_SUPM != 0);
178+ // Bit-Manipulation ISA extensions
179+ enable_feature ( Feature :: zba, ima_ext_0 & RISCV_HWPROBE_EXT_ZBA != 0 ) ;
180+ enable_feature ( Feature :: zbb, ima_ext_0 & RISCV_HWPROBE_EXT_ZBB != 0 ) ;
181+ enable_feature ( Feature :: zbs, ima_ext_0 & RISCV_HWPROBE_EXT_ZBS != 0 ) ;
182+ enable_feature ( Feature :: zbc, ima_ext_0 & RISCV_HWPROBE_EXT_ZBC != 0 ) ;
183+ // Scalar Crypto ISA extensions
184+ let zbkb = ima_ext_0 & RISCV_HWPROBE_EXT_ZBKB != 0 ;
185+ enable_feature ( Feature :: zbkb, zbkb) ;
186+ let zbkc = ima_ext_0 & RISCV_HWPROBE_EXT_ZBKC != 0 ;
187+ enable_feature ( Feature :: zbkc, zbkc) ;
188+ let zbkx = ima_ext_0 & RISCV_HWPROBE_EXT_ZBKX != 0 ;
189+ enable_feature ( Feature :: zbkx, zbkx) ;
190+ let zknd = ima_ext_0 & RISCV_HWPROBE_EXT_ZKND != 0 ;
191+ enable_feature ( Feature :: zknd, zknd) ;
192+ let zkne = ima_ext_0 & RISCV_HWPROBE_EXT_ZKNE != 0 ;
193+ enable_feature ( Feature :: zkne, zkne) ;
194+ let zknh = ima_ext_0 & RISCV_HWPROBE_EXT_ZKNH != 0 ;
195+ enable_feature ( Feature :: zknh, zknh) ;
196+ let zksed = ima_ext_0 & RISCV_HWPROBE_EXT_ZKSED != 0 ;
197+ enable_feature ( Feature :: zksed, zksed) ;
198+ let zksh = ima_ext_0 & RISCV_HWPROBE_EXT_ZKSH != 0 ;
199+ enable_feature ( Feature :: zksh, zksh) ;
200+ let zkt = ima_ext_0 & RISCV_HWPROBE_EXT_ZKT != 0 ;
201+ enable_feature ( Feature :: zkt, zkt) ;
202+ let zkn = zbkb & zbkc & zbkx & zkne & zknd & zknh;
203+ enable_feature ( Feature :: zkn, zkn) ;
204+ // enable_feature(Feature::zk, zkn & zkr & zkt);
205+ enable_feature ( Feature :: zks, zbkb & zbkc & zbkx & zksed & zksh) ;
206+ // Standard Vector Extensions
207+ enable_feature ( Feature :: v, ima_ext_0 & RISCV_HWPROBE_IMA_V != 0 ) ;
208+ enable_feature ( Feature :: zvfh, ima_ext_0 & RISCV_HWPROBE_EXT_ZVFH != 0 ) ;
209+ enable_feature ( Feature :: zvfhmin, ima_ext_0 & RISCV_HWPROBE_EXT_ZVFHMIN != 0 ) ;
210+ enable_feature ( Feature :: zve32x, ima_ext_0 & RISCV_HWPROBE_EXT_ZVE32X != 0 ) ;
211+ enable_feature ( Feature :: zve32f, ima_ext_0 & RISCV_HWPROBE_EXT_ZVE32F != 0 ) ;
212+ enable_feature ( Feature :: zve64x, ima_ext_0 & RISCV_HWPROBE_EXT_ZVE64X != 0 ) ;
213+ enable_feature ( Feature :: zve64f, ima_ext_0 & RISCV_HWPROBE_EXT_ZVE64F != 0 ) ;
214+ enable_feature ( Feature :: zve64d, ima_ext_0 & RISCV_HWPROBE_EXT_ZVE64D != 0 ) ;
215+ // Vector Cryptography and Bit-manipulation Extensions
216+ let zvbb = ima_ext_0 & RISCV_HWPROBE_EXT_ZVBB != 0 ;
217+ enable_feature ( Feature :: zvbb, zvbb) ;
218+ let zvbc = ima_ext_0 & RISCV_HWPROBE_EXT_ZVBC != 0 ;
219+ enable_feature ( Feature :: zvbc, zvbc) ;
220+ let zvkb = zvbb || ima_ext_0 & RISCV_HWPROBE_EXT_ZVKB != 0 ;
221+ enable_feature ( Feature :: zvkb, zvkb) ;
222+ let zvkg = ima_ext_0 & RISCV_HWPROBE_EXT_ZVKG != 0 ;
223+ enable_feature ( Feature :: zvkg, zvkg) ;
224+ let zvkned = ima_ext_0 & RISCV_HWPROBE_EXT_ZVKNED != 0 ;
225+ enable_feature ( Feature :: zvkned, zvkned) ;
226+ enable_feature ( Feature :: zvknha, ima_ext_0 & RISCV_HWPROBE_EXT_ZVKNHA != 0 ) ;
227+ let zvknhb = ima_ext_0 & RISCV_HWPROBE_EXT_ZVKNHB != 0 ;
228+ enable_feature ( Feature :: zvknhb, zvknhb) ;
229+ let zvksed = ima_ext_0 & RISCV_HWPROBE_EXT_ZVKSED != 0 ;
230+ enable_feature ( Feature :: zvksed, zvksed) ;
231+ let zvksh = ima_ext_0 & RISCV_HWPROBE_EXT_ZVKSH != 0 ;
232+ enable_feature ( Feature :: zvksh, zvksh) ;
233+ let zvkt = ima_ext_0 & RISCV_HWPROBE_EXT_ZVKT != 0 ;
234+ enable_feature ( Feature :: zvkt, zvkt) ;
235+ let zvkn = zvkned & zvknhb & zvkb & zvkt;
236+ enable_feature ( Feature :: zvkn, zvkn) ;
237+ enable_feature ( Feature :: zvknc, zvkn & zvbc) ;
238+ enable_feature ( Feature :: zvkng, zvkn & zvkg) ;
239+ let zvks = zvksed & zvksh & zvkb & zvkt;
240+ enable_feature ( Feature :: zvks, zvks) ;
241+ enable_feature ( Feature :: zvksc, zvks & zvbc) ;
242+ enable_feature ( Feature :: zvksg, zvks & zvkg) ;
243+ }
244+ if out[ 2 ] . key != -1 {
245+ enable_feature (
246+ Feature :: unaligned_scalar_mem,
247+ out[ 2 ] . value & RISCV_HWPROBE_MISALIGNED_MASK == RISCV_HWPROBE_MISALIGNED_FAST ,
248+ ) ;
249+ }
250+ if out[ 3 ] . key != -1 {
251+ enable_feature (
252+ Feature :: unaligned_vector_mem,
253+ out[ 3 ] . value == RISCV_HWPROBE_MISALIGNED_VECTOR_FAST ,
254+ ) ;
255+ }
256+ // FIXME: should be enough with hwprobe only, but our code below checks h and e
257+ // unavailable in neither uapi/asm/hwprobe.h nor uapi/asm/hwcap.h.
258+ // https://github.com/torvalds/linux/blob/master/arch/riscv/include/uapi/asm/hwcap.h
259+ // return value;
260+ }
261+
262+ // FIXME: As said in the above FIXME, we currently alway checks auxv too.
263+ // // riscv_hwprobe requires Linux 6.4, so we fallback to auxv-based detection on
264+ // // old Linux kernel.
265+
9266 let enable_feature = |value : & mut cache:: Initializer , feature, enable| {
10267 if enable {
11268 value. set ( feature as u32 ) ;
@@ -22,6 +279,11 @@ pub(crate) fn detect_features() -> cache::Initializer {
22279 // The values are part of the platform-specific [asm/hwcap.h][hwcap]
23280 //
24281 // [hwcap]: https://github.com/torvalds/linux/blob/master/arch/riscv/include/asm/hwcap.h
282+ //
283+ // Note that there is no need to check b'v' - b'a' here for the case where riscv_hwprobe is unsupported,
284+ // since both RISCV_HWPROBE_IMA_V and COMPAT_HWCAP_ISA_V are only supported on Linux 6.5+.
285+ // https://github.com/torvalds/linux/commit/162e4df137c1fea6557fda3e4cdf5dc6ca6d5510
286+ // https://github.com/torvalds/linux/commit/dc6667a4e7e36f283bcd0264a0be55adae4d6f86
25287 let auxv = auxvec:: auxv ( ) . expect ( "read auxvec" ) ; // should not fail on RISC-V platform
26288 #[ allow( clippy:: eq_op) ]
27289 enable_feature (
@@ -46,16 +308,20 @@ pub(crate) fn detect_features() -> cache::Initializer {
46308 ) ;
47309 let has_i = bit:: test ( auxv. hwcap , ( b'i' - b'a' ) . into ( ) ) ;
48310 // If future RV128I is supported, implement with `enable_feature` here
49- #[ cfg( target_pointer_width = "64" ) ]
311+ // Checking target_pointer_width instead of target_arch is incorrect since
312+ // there are RV64ILP32* ABIs.
313+ #[ cfg( target_arch = "riscv64" ) ]
50314 enable_feature ( & mut value, Feature :: rv64i, has_i) ;
51- #[ cfg( target_pointer_width = "32 " ) ]
315+ #[ cfg( target_arch = "riscv32 " ) ]
52316 enable_feature ( & mut value, Feature :: rv32i, has_i) ;
53- #[ cfg( target_pointer_width = "32" ) ]
317+ // FIXME: e is not exposed in any of asm/hwcap.h, uapi/asm/hwcap.h, uapi/asm/hwprobe.h
318+ #[ cfg( target_arch = "riscv32" ) ]
54319 enable_feature (
55320 & mut value,
56321 Feature :: rv32e,
57322 bit:: test ( auxv. hwcap , ( b'e' - b'a' ) . into ( ) ) ,
58323 ) ;
324+ // FIXME: h is not exposed in uapi/asm/hwcap.h and uapi/asm/hwprobe.h
59325 enable_feature (
60326 & mut value,
61327 Feature :: h,
@@ -66,9 +332,9 @@ pub(crate) fn detect_features() -> cache::Initializer {
66332 Feature :: m,
67333 bit:: test ( auxv. hwcap , ( b'm' - b'a' ) . into ( ) ) ,
68334 ) ;
69- // FIXME: Auxvec does not show supervisor feature support, but this mode may be useful
70- // to detect when Rust is used to write Linux kernel modules .
71- // These should be more than Auxvec way to detect supervisor features .
335+
336+ // Neither hwprobe nor auxv supports detection of supervisor feature .
337+ // Since target_os = "linux" is for user mode, their detection is not useful .
72338
73339 value
74340}
0 commit comments