Rollup of 9 pull requests

compiler/rustc_borrowck/src/region_infer/mod.rs

@@ -1,3 +1,4 @@
+use std::cell::OnceCell;
 use std::collections::VecDeque;
 use std::rc::Rc;
 
@@ -197,8 +198,8 @@ pub struct RegionInferenceContext<'tcx> {
 
     /// Reverse of the SCC constraint graph --  i.e., an edge `A -> B` exists if
     /// `B: A`. This is used to compute the universal regions that are required
-    /// to outlive a given SCC. Computed lazily.
-    rev_scc_graph: Option<ReverseSccGraph>,
+    /// to outlive a given SCC.
+    rev_scc_graph: OnceCell<ReverseSccGraph>,
 
     /// The "R0 member of [R1..Rn]" constraints, indexed by SCC.
     member_constraints: Rc<MemberConstraintSet<'tcx, ConstraintSccIndex>>,
@@ -502,7 +503,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
             constraint_graph,
             constraint_sccs,
             scc_annotations,
-            rev_scc_graph: None,
+            rev_scc_graph: OnceCell::new(),
             member_constraints,
             member_constraints_applied: Vec::new(),
             universe_causes,
@@ -809,9 +810,6 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         member_constraint_index: NllMemberConstraintIndex,
         choice_regions: &[ty::RegionVid],
     ) {
-        // Lazily compute the reverse graph, we'll need it later.
-        self.compute_reverse_scc_graph();
-
         // Create a mutable vector of the options. We'll try to winnow
         // them down.
         let mut choice_regions: Vec<ty::RegionVid> = choice_regions.to_vec();
@@ -849,7 +847,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
         // R0`). Therefore, we need only keep an option `O` if `UB: O`
         // for all UB.
         let universal_region_relations = &self.universal_region_relations;
-        for ub in self.rev_scc_graph.as_ref().unwrap().upper_bounds(scc) {
+        for ub in self.reverse_scc_graph().upper_bounds(scc) {
             debug!(?ub);
             choice_regions.retain(|&o_r| universal_region_relations.outlives(ub, o_r));
         }

compiler/rustc_borrowck/src/region_infer/opaque_types.rs

@@ -215,9 +215,7 @@ impl<'tcx> RegionInferenceContext<'tcx> {
                 // FIXME: We could probably compute the LUB if there is one.
                 let scc = self.constraint_sccs.scc(vid);
                 let upper_bounds: Vec<_> = self
-                    .rev_scc_graph
-                    .as_ref()
-                    .unwrap()
+                    .reverse_scc_graph()
                     .upper_bounds(scc)
                     .filter_map(|vid| self.definitions[vid].external_name)
                     .filter(|r| !r.is_static())

compiler/rustc_borrowck/src/region_infer/reverse_sccs.rs

@@ -59,13 +59,10 @@ impl ReverseSccGraph {
 }
 
 impl RegionInferenceContext<'_> {
-    /// Compute the reverse SCC-based constraint graph (lazily).
-    pub(super) fn compute_reverse_scc_graph(&mut self) {
-        if self.rev_scc_graph.is_some() {
-            return;
-        }
-
-        self.rev_scc_graph =
-            Some(ReverseSccGraph::compute(&self.constraint_sccs, self.universal_regions()));
+    /// Return the reverse graph of the region SCCs, initialising it if needed.
+    pub(super) fn reverse_scc_graph(&self) -> &ReverseSccGraph {
+        self.rev_scc_graph.get_or_init(|| {
+            ReverseSccGraph::compute(&self.constraint_sccs, self.universal_regions())
+        })
     }
 }

compiler/rustc_codegen_llvm/src/va_arg.rs

@@ -40,6 +40,7 @@ fn emit_direct_ptr_va_arg<'ll, 'tcx>(
     align: Align,
     slot_size: Align,
     allow_higher_align: bool,
+    force_right_adjust: bool,
 ) -> (&'ll Value, Align) {
     let va_list_ty = bx.type_ptr();
     let va_list_addr = list.immediate();
@@ -57,7 +58,10 @@ fn emit_direct_ptr_va_arg<'ll, 'tcx>(
     let next = bx.inbounds_ptradd(addr, full_direct_size);
     bx.store(next, va_list_addr, bx.tcx().data_layout.pointer_align.abi);
 
-    if size.bytes() < slot_size.bytes() && bx.tcx().sess.target.endian == Endian::Big {
+    if size.bytes() < slot_size.bytes()
+        && bx.tcx().sess.target.endian == Endian::Big
+        && force_right_adjust
+    {
         let adjusted_size = bx.cx().const_i32((slot_size.bytes() - size.bytes()) as i32);
         let adjusted = bx.inbounds_ptradd(addr, adjusted_size);
         (adjusted, addr_align)
@@ -81,16 +85,23 @@ enum AllowHigherAlign {
     Yes,
 }
 
+enum ForceRightAdjust {
+    No,
+    Yes,
+}
+
 fn emit_ptr_va_arg<'ll, 'tcx>(
     bx: &mut Builder<'_, 'll, 'tcx>,
     list: OperandRef<'tcx, &'ll Value>,
     target_ty: Ty<'tcx>,
     pass_mode: PassMode,
     slot_size: SlotSize,
     allow_higher_align: AllowHigherAlign,
+    force_right_adjust: ForceRightAdjust,
 ) -> &'ll Value {
     let indirect = matches!(pass_mode, PassMode::Indirect);
     let allow_higher_align = matches!(allow_higher_align, AllowHigherAlign::Yes);
+    let force_right_adjust = matches!(force_right_adjust, ForceRightAdjust::Yes);
     let slot_size = Align::from_bytes(slot_size as u64).unwrap();
 
     let layout = bx.cx.layout_of(target_ty);
@@ -103,8 +114,15 @@ fn emit_ptr_va_arg<'ll, 'tcx>(
     } else {
         (layout.llvm_type(bx.cx), layout.size, layout.align)
     };
-    let (addr, addr_align) =
-        emit_direct_ptr_va_arg(bx, list, size, align.abi, slot_size, allow_higher_align);
+    let (addr, addr_align) = emit_direct_ptr_va_arg(
+        bx,
+        list,
+        size,
+        align.abi,
+        slot_size,
+        allow_higher_align,
+        force_right_adjust,
+    );
     if indirect {
         let tmp_ret = bx.load(llty, addr, addr_align);
         bx.load(bx.cx.layout_of(target_ty).llvm_type(bx.cx), tmp_ret, align.abi)
@@ -208,6 +226,7 @@ fn emit_aapcs_va_arg<'ll, 'tcx>(
         PassMode::Direct,
         SlotSize::Bytes8,
         AllowHigherAlign::Yes,
+        ForceRightAdjust::No,
     );
     bx.br(end);
 
@@ -218,6 +237,150 @@ fn emit_aapcs_va_arg<'ll, 'tcx>(
     val
 }
 
+fn emit_powerpc_va_arg<'ll, 'tcx>(
+    bx: &mut Builder<'_, 'll, 'tcx>,
+    list: OperandRef<'tcx, &'ll Value>,
+    target_ty: Ty<'tcx>,
+) -> &'ll Value {
+    let dl = bx.cx.data_layout();
+
+    // struct __va_list_tag {
+    //   unsigned char gpr;
+    //   unsigned char fpr;
+    //   unsigned short reserved;
+    //   void *overflow_arg_area;
+    //   void *reg_save_area;
+    // };
+    let va_list_addr = list.immediate();
+
+    // Peel off any newtype wrappers.
+    let layout = {
+        let mut layout = bx.cx.layout_of(target_ty);
+
+        while let Some((_, inner)) = layout.non_1zst_field(bx.cx) {
+            layout = inner;
+        }
+
+        layout
+    };
+
+    // Rust does not currently support any powerpc softfloat targets.
+    let target = &bx.cx.tcx.sess.target;
+    let is_soft_float_abi = target.abi == "softfloat";
+    assert!(!is_soft_float_abi);
+
+    // All instances of VaArgSafe are passed directly.
+    let is_indirect = false;
+
+    let (is_i64, is_int, is_f64) = match layout.layout.backend_repr() {
+        BackendRepr::Scalar(scalar) => match scalar.primitive() {
+            rustc_abi::Primitive::Int(integer, _) => (integer.size().bits() == 64, true, false),
+            rustc_abi::Primitive::Float(float) => (false, false, float.size().bits() == 64),
+            rustc_abi::Primitive::Pointer(_) => (false, true, false),
+        },
+        _ => unreachable!("all instances of VaArgSafe are represented as scalars"),
+    };
+
+    let num_regs_addr = if is_int || is_soft_float_abi {
+        va_list_addr // gpr
+    } else {
+        bx.inbounds_ptradd(va_list_addr, bx.const_usize(1)) // fpr
+    };
+
+    let mut num_regs = bx.load(bx.type_i8(), num_regs_addr, dl.i8_align.abi);
+
+    // "Align" the register count when the type is passed as `i64`.
+    if is_i64 || (is_f64 && is_soft_float_abi) {
+        num_regs = bx.add(num_regs, bx.const_u8(1));
+        num_regs = bx.and(num_regs, bx.const_u8(0b1111_1110));
+    }
+
+    let max_regs = 8u8;
+    let use_regs = bx.icmp(IntPredicate::IntULT, num_regs, bx.const_u8(max_regs));
+
+    let in_reg = bx.append_sibling_block("va_arg.in_reg");
+    let in_mem = bx.append_sibling_block("va_arg.in_mem");
+    let end = bx.append_sibling_block("va_arg.end");
+
+    bx.cond_br(use_regs, in_reg, in_mem);
+
+    let reg_addr = {
+        bx.switch_to_block(in_reg);
+
+        let reg_safe_area_ptr = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(1 + 1 + 2 + 4));
+        let mut reg_addr = bx.load(bx.type_ptr(), reg_safe_area_ptr, dl.pointer_align.abi);
+
+        // Floating-point registers start after the general-purpose registers.
+        if !is_int && !is_soft_float_abi {
+            reg_addr = bx.inbounds_ptradd(reg_addr, bx.cx.const_usize(32))
+        }
+
+        // Get the address of the saved value by scaling the number of
+        // registers we've used by the number of.
+        let reg_size = if is_int || is_soft_float_abi { 4 } else { 8 };
+        let reg_offset = bx.mul(num_regs, bx.cx().const_u8(reg_size));
+        let reg_addr = bx.inbounds_ptradd(reg_addr, reg_offset);
+
+        // Increase the used-register count.
+        let reg_incr = if is_i64 || (is_f64 && is_soft_float_abi) { 2 } else { 1 };
+        let new_num_regs = bx.add(num_regs, bx.cx.const_u8(reg_incr));
+        bx.store(new_num_regs, num_regs_addr, dl.i8_align.abi);
+
+        bx.br(end);
+
+        reg_addr
+    };
+
+    let mem_addr = {
+        bx.switch_to_block(in_mem);
+
+        bx.store(bx.const_u8(max_regs), num_regs_addr, dl.i8_align.abi);
+
+        // Everything in the overflow area is rounded up to a size of at least 4.
+        let overflow_area_align = Align::from_bytes(4).unwrap();
+
+        let size = if !is_indirect {
+            layout.layout.size.align_to(overflow_area_align)
+        } else {
+            dl.pointer_size
+        };
+
+        let overflow_area_ptr = bx.inbounds_ptradd(va_list_addr, bx.cx.const_usize(1 + 1 + 2));
+        let mut overflow_area = bx.load(bx.type_ptr(), overflow_area_ptr, dl.pointer_align.abi);
+
+        // Round up address of argument to alignment
+        if layout.layout.align.abi > overflow_area_align {
+            overflow_area = round_pointer_up_to_alignment(
+                bx,
+                overflow_area,
+                layout.layout.align.abi,
+                bx.type_ptr(),
+            );
+        }
+
+        let mem_addr = overflow_area;
+
+        // Increase the overflow area.
+        overflow_area = bx.inbounds_ptradd(overflow_area, bx.const_usize(size.bytes()));
+        bx.store(overflow_area, overflow_area_ptr, dl.pointer_align.abi);
+
+        bx.br(end);
+
+        mem_addr
+    };
+
+    // Return the appropriate result.
+    bx.switch_to_block(end);
+    let val_addr = bx.phi(bx.type_ptr(), &[reg_addr, mem_addr], &[in_reg, in_mem]);
+    let val_type = layout.llvm_type(bx);
+    let val_addr = if is_indirect {
+        bx.load(bx.cx.type_ptr(), val_addr, dl.pointer_align.abi)
+    } else {
+        val_addr
+    };
+    bx.load(val_type, val_addr, layout.align.abi)
+}
+
 fn emit_s390x_va_arg<'ll, 'tcx>(
     bx: &mut Builder<'_, 'll, 'tcx>,
     list: OperandRef<'tcx, &'ll Value>,
@@ -728,6 +891,7 @@ pub(super) fn emit_va_arg<'ll, 'tcx>(
             PassMode::Direct,
             SlotSize::Bytes4,
             if target.is_like_windows { AllowHigherAlign::No } else { AllowHigherAlign::Yes },
+            ForceRightAdjust::No,
         ),
         "aarch64" | "arm64ec" if target.is_like_windows || target.is_like_darwin => {
             emit_ptr_va_arg(
@@ -737,10 +901,24 @@ pub(super) fn emit_va_arg<'ll, 'tcx>(
                 PassMode::Direct,
                 SlotSize::Bytes8,
                 if target.is_like_windows { AllowHigherAlign::No } else { AllowHigherAlign::Yes },
+                ForceRightAdjust::No,
             )
         }
         "aarch64" => emit_aapcs_va_arg(bx, addr, target_ty),
         "s390x" => emit_s390x_va_arg(bx, addr, target_ty),
+        "powerpc" => emit_powerpc_va_arg(bx, addr, target_ty),
+        "powerpc64" | "powerpc64le" => emit_ptr_va_arg(
+            bx,
+            addr,
+            target_ty,
+            PassMode::Direct,
+            SlotSize::Bytes8,
+            AllowHigherAlign::Yes,
+            match &*target.arch {
+                "powerpc64" => ForceRightAdjust::Yes,
+                _ => ForceRightAdjust::No,
+            },
+        ),
         // Windows x86_64
         "x86_64" if target.is_like_windows => {
             let target_ty_size = bx.cx.size_of(target_ty).bytes();
@@ -755,6 +933,7 @@ pub(super) fn emit_va_arg<'ll, 'tcx>(
                 },
                 SlotSize::Bytes8,
                 AllowHigherAlign::No,
+                ForceRightAdjust::No,
             )
         }
         // This includes `target.is_like_darwin`, which on x86_64 targets is like sysv64.