|
| 1 | +use crate::ty::{ |
| 2 | + layout::{LayoutCx, TyAndLayout}, |
| 3 | + TyCtxt, |
| 4 | +}; |
| 5 | +use rustc_target::abi::*; |
| 6 | + |
| 7 | +use std::cmp; |
| 8 | + |
| 9 | +/// Enforce some basic invariants on layouts. |
| 10 | +pub(super) fn sanity_check_layout<'tcx>( |
| 11 | + cx: &LayoutCx<'tcx, TyCtxt<'tcx>>, |
| 12 | + layout: &TyAndLayout<'tcx>, |
| 13 | +) { |
| 14 | + // Type-level uninhabitedness should always imply ABI uninhabitedness. |
| 15 | + if cx.tcx.conservative_is_privately_uninhabited(cx.param_env.and(layout.ty)) { |
| 16 | + assert!(layout.abi.is_uninhabited()); |
| 17 | + } |
| 18 | + |
| 19 | + if layout.size.bytes() % layout.align.abi.bytes() != 0 { |
| 20 | + bug!("size is not a multiple of align, in the following layout:\n{layout:#?}"); |
| 21 | + } |
| 22 | + |
| 23 | + if cfg!(debug_assertions) { |
| 24 | + /// Yields non-ZST fields of the type |
| 25 | + fn non_zst_fields<'tcx, 'a>( |
| 26 | + cx: &'a LayoutCx<'tcx, TyCtxt<'tcx>>, |
| 27 | + layout: &'a TyAndLayout<'tcx>, |
| 28 | + ) -> impl Iterator<Item = (Size, TyAndLayout<'tcx>)> + 'a { |
| 29 | + (0..layout.layout.fields().count()).filter_map(|i| { |
| 30 | + let field = layout.field(cx, i); |
| 31 | + // Also checking `align == 1` here leads to test failures in |
| 32 | + // `layout/zero-sized-array-union.rs`, where a type has a zero-size field with |
| 33 | + // alignment 4 that still gets ignored during layout computation (which is okay |
| 34 | + // since other fields already force alignment 4). |
| 35 | + let zst = field.is_zst(); |
| 36 | + (!zst).then(|| (layout.fields.offset(i), field)) |
| 37 | + }) |
| 38 | + } |
| 39 | + |
| 40 | + fn skip_newtypes<'tcx>( |
| 41 | + cx: &LayoutCx<'tcx, TyCtxt<'tcx>>, |
| 42 | + layout: &TyAndLayout<'tcx>, |
| 43 | + ) -> TyAndLayout<'tcx> { |
| 44 | + if matches!(layout.layout.variants(), Variants::Multiple { .. }) { |
| 45 | + // Definitely not a newtype of anything. |
| 46 | + return *layout; |
| 47 | + } |
| 48 | + let mut fields = non_zst_fields(cx, layout); |
| 49 | + let Some(first) = fields.next() else { |
| 50 | + // No fields here, so this could be a primitive or enum -- either way it's not a newtype around a thing |
| 51 | + return *layout |
| 52 | + }; |
| 53 | + if fields.next().is_none() { |
| 54 | + let (offset, first) = first; |
| 55 | + if offset == Size::ZERO && first.layout.size() == layout.size { |
| 56 | + // This is a newtype, so keep recursing. |
| 57 | + // FIXME(RalfJung): I don't think it would be correct to do any checks for |
| 58 | + // alignment here, so we don't. Is that correct? |
| 59 | + return skip_newtypes(cx, &first); |
| 60 | + } |
| 61 | + } |
| 62 | + // No more newtypes here. |
| 63 | + *layout |
| 64 | + } |
| 65 | + |
| 66 | + fn check_layout_abi<'tcx>(cx: &LayoutCx<'tcx, TyCtxt<'tcx>>, layout: &TyAndLayout<'tcx>) { |
| 67 | + match layout.layout.abi() { |
| 68 | + Abi::Scalar(scalar) => { |
| 69 | + // No padding in scalars. |
| 70 | + let size = scalar.size(cx); |
| 71 | + let align = scalar.align(cx).abi; |
| 72 | + assert_eq!( |
| 73 | + layout.layout.size(), |
| 74 | + size, |
| 75 | + "size mismatch between ABI and layout in {layout:#?}" |
| 76 | + ); |
| 77 | + assert_eq!( |
| 78 | + layout.layout.align().abi, |
| 79 | + align, |
| 80 | + "alignment mismatch between ABI and layout in {layout:#?}" |
| 81 | + ); |
| 82 | + // Check that this matches the underlying field. |
| 83 | + let inner = skip_newtypes(cx, layout); |
| 84 | + assert!( |
| 85 | + matches!(inner.layout.abi(), Abi::Scalar(_)), |
| 86 | + "`Scalar` type {} is newtype around non-`Scalar` type {}", |
| 87 | + layout.ty, |
| 88 | + inner.ty |
| 89 | + ); |
| 90 | + match inner.layout.fields() { |
| 91 | + FieldsShape::Primitive => { |
| 92 | + // Fine. |
| 93 | + } |
| 94 | + FieldsShape::Union(..) => { |
| 95 | + // FIXME: I guess we could also check something here? Like, look at all fields? |
| 96 | + return; |
| 97 | + } |
| 98 | + FieldsShape::Arbitrary { .. } => { |
| 99 | + // Should be an enum, the only field is the discriminant. |
| 100 | + assert!( |
| 101 | + inner.ty.is_enum(), |
| 102 | + "`Scalar` layout for non-primitive non-enum type {}", |
| 103 | + inner.ty |
| 104 | + ); |
| 105 | + assert_eq!( |
| 106 | + inner.layout.fields().count(), |
| 107 | + 1, |
| 108 | + "`Scalar` layout for multiple-field type in {inner:#?}", |
| 109 | + ); |
| 110 | + let offset = inner.layout.fields().offset(0); |
| 111 | + let field = inner.field(cx, 0); |
| 112 | + // The field should be at the right offset, and match the `scalar` layout. |
| 113 | + assert_eq!( |
| 114 | + offset, |
| 115 | + Size::ZERO, |
| 116 | + "`Scalar` field at non-0 offset in {inner:#?}", |
| 117 | + ); |
| 118 | + assert_eq!( |
| 119 | + field.size, size, |
| 120 | + "`Scalar` field with bad size in {inner:#?}", |
| 121 | + ); |
| 122 | + assert_eq!( |
| 123 | + field.align.abi, align, |
| 124 | + "`Scalar` field with bad align in {inner:#?}", |
| 125 | + ); |
| 126 | + assert!( |
| 127 | + matches!(field.abi, Abi::Scalar(_)), |
| 128 | + "`Scalar` field with bad ABI in {inner:#?}", |
| 129 | + ); |
| 130 | + } |
| 131 | + _ => { |
| 132 | + panic!("`Scalar` layout for non-primitive non-enum type {}", inner.ty); |
| 133 | + } |
| 134 | + } |
| 135 | + } |
| 136 | + Abi::ScalarPair(scalar1, scalar2) => { |
| 137 | + // Sanity-check scalar pairs. These are a bit more flexible and support |
| 138 | + // padding, but we can at least ensure both fields actually fit into the layout |
| 139 | + // and the alignment requirement has not been weakened. |
| 140 | + let size1 = scalar1.size(cx); |
| 141 | + let align1 = scalar1.align(cx).abi; |
| 142 | + let size2 = scalar2.size(cx); |
| 143 | + let align2 = scalar2.align(cx).abi; |
| 144 | + assert!( |
| 145 | + layout.layout.align().abi >= cmp::max(align1, align2), |
| 146 | + "alignment mismatch between ABI and layout in {layout:#?}", |
| 147 | + ); |
| 148 | + let field2_offset = size1.align_to(align2); |
| 149 | + assert!( |
| 150 | + layout.layout.size() >= field2_offset + size2, |
| 151 | + "size mismatch between ABI and layout in {layout:#?}" |
| 152 | + ); |
| 153 | + // Check that the underlying pair of fields matches. |
| 154 | + let inner = skip_newtypes(cx, layout); |
| 155 | + assert!( |
| 156 | + matches!(inner.layout.abi(), Abi::ScalarPair(..)), |
| 157 | + "`ScalarPair` type {} is newtype around non-`ScalarPair` type {}", |
| 158 | + layout.ty, |
| 159 | + inner.ty |
| 160 | + ); |
| 161 | + if matches!(inner.layout.variants(), Variants::Multiple { .. }) { |
| 162 | + // FIXME: ScalarPair for enums is enormously complicated and it is very hard |
| 163 | + // to check anything about them. |
| 164 | + return; |
| 165 | + } |
| 166 | + match inner.layout.fields() { |
| 167 | + FieldsShape::Arbitrary { .. } => { |
| 168 | + // Checked below. |
| 169 | + } |
| 170 | + FieldsShape::Union(..) => { |
| 171 | + // FIXME: I guess we could also check something here? Like, look at all fields? |
| 172 | + return; |
| 173 | + } |
| 174 | + _ => { |
| 175 | + panic!("`ScalarPair` layout with unexpected field shape in {inner:#?}"); |
| 176 | + } |
| 177 | + } |
| 178 | + let mut fields = non_zst_fields(cx, &inner); |
| 179 | + let (offset1, field1) = fields.next().unwrap_or_else(|| { |
| 180 | + panic!("`ScalarPair` layout for type with not even one non-ZST field: {inner:#?}") |
| 181 | + }); |
| 182 | + let (offset2, field2) = fields.next().unwrap_or_else(|| { |
| 183 | + panic!("`ScalarPair` layout for type with less than two non-ZST fields: {inner:#?}") |
| 184 | + }); |
| 185 | + assert!( |
| 186 | + fields.next().is_none(), |
| 187 | + "`ScalarPair` layout for type with at least three non-ZST fields: {inner:#?}" |
| 188 | + ); |
| 189 | + // The fields might be in opposite order. |
| 190 | + let (offset1, field1, offset2, field2) = if offset1 <= offset2 { |
| 191 | + (offset1, field1, offset2, field2) |
| 192 | + } else { |
| 193 | + (offset2, field2, offset1, field1) |
| 194 | + }; |
| 195 | + // The fields should be at the right offset, and match the `scalar` layout. |
| 196 | + assert_eq!( |
| 197 | + offset1, |
| 198 | + Size::ZERO, |
| 199 | + "`ScalarPair` first field at non-0 offset in {inner:#?}", |
| 200 | + ); |
| 201 | + assert_eq!( |
| 202 | + field1.size, size1, |
| 203 | + "`ScalarPair` first field with bad size in {inner:#?}", |
| 204 | + ); |
| 205 | + assert_eq!( |
| 206 | + field1.align.abi, align1, |
| 207 | + "`ScalarPair` first field with bad align in {inner:#?}", |
| 208 | + ); |
| 209 | + assert!( |
| 210 | + matches!(field1.abi, Abi::Scalar(_)), |
| 211 | + "`ScalarPair` first field with bad ABI in {inner:#?}", |
| 212 | + ); |
| 213 | + assert_eq!( |
| 214 | + offset2, field2_offset, |
| 215 | + "`ScalarPair` second field at bad offset in {inner:#?}", |
| 216 | + ); |
| 217 | + assert_eq!( |
| 218 | + field2.size, size2, |
| 219 | + "`ScalarPair` second field with bad size in {inner:#?}", |
| 220 | + ); |
| 221 | + assert_eq!( |
| 222 | + field2.align.abi, align2, |
| 223 | + "`ScalarPair` second field with bad align in {inner:#?}", |
| 224 | + ); |
| 225 | + assert!( |
| 226 | + matches!(field2.abi, Abi::Scalar(_)), |
| 227 | + "`ScalarPair` second field with bad ABI in {inner:#?}", |
| 228 | + ); |
| 229 | + } |
| 230 | + Abi::Vector { count, element } => { |
| 231 | + // No padding in vectors. Alignment can be strengthened, though. |
| 232 | + assert!( |
| 233 | + layout.layout.align().abi >= element.align(cx).abi, |
| 234 | + "alignment mismatch between ABI and layout in {layout:#?}" |
| 235 | + ); |
| 236 | + let size = element.size(cx) * count; |
| 237 | + assert_eq!( |
| 238 | + layout.layout.size(), |
| 239 | + size.align_to(cx.data_layout().vector_align(size).abi), |
| 240 | + "size mismatch between ABI and layout in {layout:#?}" |
| 241 | + ); |
| 242 | + } |
| 243 | + Abi::Uninhabited | Abi::Aggregate { .. } => {} // Nothing to check. |
| 244 | + } |
| 245 | + } |
| 246 | + |
| 247 | + check_layout_abi(cx, layout); |
| 248 | + |
| 249 | + if let Variants::Multiple { variants, .. } = &layout.variants { |
| 250 | + for variant in variants.iter() { |
| 251 | + // No nested "multiple". |
| 252 | + assert!(matches!(variant.variants(), Variants::Single { .. })); |
| 253 | + // Variants should have the same or a smaller size as the full thing, |
| 254 | + // and same for alignment. |
| 255 | + if variant.size() > layout.size { |
| 256 | + bug!( |
| 257 | + "Type with size {} bytes has variant with size {} bytes: {layout:#?}", |
| 258 | + layout.size.bytes(), |
| 259 | + variant.size().bytes(), |
| 260 | + ) |
| 261 | + } |
| 262 | + if variant.align().abi > layout.align.abi { |
| 263 | + bug!( |
| 264 | + "Type with alignment {} bytes has variant with alignment {} bytes: {layout:#?}", |
| 265 | + layout.align.abi.bytes(), |
| 266 | + variant.align().abi.bytes(), |
| 267 | + ) |
| 268 | + } |
| 269 | + // Skip empty variants. |
| 270 | + if variant.size() == Size::ZERO |
| 271 | + || variant.fields().count() == 0 |
| 272 | + || variant.abi().is_uninhabited() |
| 273 | + { |
| 274 | + // These are never actually accessed anyway, so we can skip the coherence check |
| 275 | + // for them. They also fail that check, since they have |
| 276 | + // `Aggregate`/`Uninhbaited` ABI even when the main type is |
| 277 | + // `Scalar`/`ScalarPair`. (Note that sometimes, variants with fields have size |
| 278 | + // 0, and sometimes, variants without fields have non-0 size.) |
| 279 | + continue; |
| 280 | + } |
| 281 | + // The top-level ABI and the ABI of the variants should be coherent. |
| 282 | + let scalar_coherent = |s1: Scalar, s2: Scalar| { |
| 283 | + s1.size(cx) == s2.size(cx) && s1.align(cx) == s2.align(cx) |
| 284 | + }; |
| 285 | + let abi_coherent = match (layout.abi, variant.abi()) { |
| 286 | + (Abi::Scalar(s1), Abi::Scalar(s2)) => scalar_coherent(s1, s2), |
| 287 | + (Abi::ScalarPair(a1, b1), Abi::ScalarPair(a2, b2)) => { |
| 288 | + scalar_coherent(a1, a2) && scalar_coherent(b1, b2) |
| 289 | + } |
| 290 | + (Abi::Uninhabited, _) => true, |
| 291 | + (Abi::Aggregate { .. }, _) => true, |
| 292 | + _ => false, |
| 293 | + }; |
| 294 | + if !abi_coherent { |
| 295 | + bug!( |
| 296 | + "Variant ABI is incompatible with top-level ABI:\nvariant={:#?}\nTop-level: {layout:#?}", |
| 297 | + variant |
| 298 | + ); |
| 299 | + } |
| 300 | + } |
| 301 | + } |
| 302 | + } |
| 303 | +} |
0 commit comments