borrowck typeck children together with their root

compiler/rustc_borrowck/src/consumers.rs

@@ -15,6 +15,7 @@ pub use super::polonius::legacy::{
     RichLocation, RustcFacts,
 };
 pub use super::region_infer::RegionInferenceContext;
+use crate::{BorrowCheckRootCtxt, do_mir_borrowck};
 
 /// Options determining the output behavior of [`get_body_with_borrowck_facts`].
 ///
@@ -97,8 +98,9 @@ pub struct BodyWithBorrowckFacts<'tcx> {
 /// *   Polonius is highly unstable, so expect regular changes in its signature or other details.
 pub fn get_body_with_borrowck_facts(
     tcx: TyCtxt<'_>,
-    def: LocalDefId,
+    def_id: LocalDefId,
     options: ConsumerOptions,
 ) -> BodyWithBorrowckFacts<'_> {
-    *super::do_mir_borrowck(tcx, def, Some(options)).1.unwrap()
+    let mut root_cx = BorrowCheckRootCtxt::new(tcx, def_id);
+    *do_mir_borrowck(&mut root_cx, def_id, Some(options)).1.unwrap()
 }

compiler/rustc_borrowck/src/lib.rs

@@ -9,6 +9,7 @@
 #![feature(file_buffered)]
 #![feature(if_let_guard)]
 #![feature(let_chains)]
+#![feature(negative_impls)]
 #![feature(never_type)]
 #![feature(rustc_attrs)]
 #![feature(rustdoc_internals)]
@@ -21,6 +22,7 @@ use std::cell::RefCell;
 use std::marker::PhantomData;
 use std::ops::{ControlFlow, Deref};
 
+use root_cx::BorrowCheckRootCtxt;
 use rustc_abi::FieldIdx;
 use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
 use rustc_data_structures::graph::dominators::Dominators;
@@ -35,7 +37,9 @@ use rustc_infer::infer::{
 };
 use rustc_middle::mir::*;
 use rustc_middle::query::Providers;
-use rustc_middle::ty::{self, ParamEnv, RegionVid, TyCtxt, TypingMode};
+use rustc_middle::ty::{
+    self, ParamEnv, RegionVid, Ty, TyCtxt, TypeFoldable, TypeVisitable, TypingMode, fold_regions,
+};
 use rustc_middle::{bug, span_bug};
 use rustc_mir_dataflow::impls::{
     EverInitializedPlaces, MaybeInitializedPlaces, MaybeUninitializedPlaces,
@@ -45,7 +49,7 @@ use rustc_mir_dataflow::move_paths::{
 };
 use rustc_mir_dataflow::{Analysis, EntryStates, Results, ResultsVisitor, visit_results};
 use rustc_session::lint::builtin::{TAIL_EXPR_DROP_ORDER, UNUSED_MUT};
-use rustc_span::{Span, Symbol};
+use rustc_span::{ErrorGuaranteed, Span, Symbol};
 use smallvec::SmallVec;
 use tracing::{debug, instrument};
 
@@ -73,14 +77,14 @@ mod def_use;
 mod diagnostics;
 mod member_constraints;
 mod nll;
-mod opaque_types;
 mod path_utils;
 mod place_ext;
 mod places_conflict;
 mod polonius;
 mod prefixes;
 mod region_infer;
 mod renumber;
+mod root_cx;
 mod session_diagnostics;
 mod type_check;
 mod universal_regions;
@@ -102,44 +106,202 @@ pub fn provide(providers: &mut Providers) {
     *providers = Providers { mir_borrowck, ..*providers };
 }
 
-fn mir_borrowck(tcx: TyCtxt<'_>, def: LocalDefId) -> &BorrowCheckResult<'_> {
+/// Provider for `query mir_borrowck`. Similar to `typeck`, this must
+/// only be called for typeck roots which will then borrowck all
+/// nested bodies as well.
+fn mir_borrowck(
+    tcx: TyCtxt<'_>,
+    def: LocalDefId,
+) -> Result<&ConcreteOpaqueTypes<'_>, ErrorGuaranteed> {
+    assert!(!tcx.is_typeck_child(def.to_def_id()));
     let (input_body, _) = tcx.mir_promoted(def);
+    debug!("run query mir_borrowck: {}", tcx.def_path_str(def));
+
     let input_body: &Body<'_> = &input_body.borrow();
-    if input_body.should_skip() || input_body.tainted_by_errors.is_some() {
-        debug!("Skipping borrowck because of injected body or tainted body");
-        // Let's make up a borrowck result! Fun times!
-        let result = BorrowCheckResult {
-            concrete_opaque_types: FxIndexMap::default(),
-            closure_requirements: None,
-            used_mut_upvars: SmallVec::new(),
-            tainted_by_errors: input_body.tainted_by_errors,
-        };
-        return tcx.arena.alloc(result);
+    if let Some(guar) = input_body.tainted_by_errors {
+        debug!("Skipping borrowck because of tainted body");
+        Err(guar)
+    } else if input_body.should_skip() {
+        debug!("Skipping borrowck because of injected body");
+        let opaque_types = ConcreteOpaqueTypes(Default::default());
+        Ok(tcx.arena.alloc(opaque_types))
+    } else {
+        let mut root_cx = BorrowCheckRootCtxt::new(tcx, def);
+        let PropagatedBorrowCheckResults { closure_requirements, used_mut_upvars } =
+            do_mir_borrowck(&mut root_cx, def, None).0;
+        debug_assert!(closure_requirements.is_none());
+        debug_assert!(used_mut_upvars.is_empty());
+        root_cx.finalize()
     }
+}
+
+/// Data propagated to the typeck parent by nested items.
+/// This should always be empty for the typeck root.
+#[derive(Debug)]
+struct PropagatedBorrowCheckResults<'tcx> {
+    closure_requirements: Option<ClosureRegionRequirements<'tcx>>,
+    used_mut_upvars: SmallVec<[FieldIdx; 8]>,
+}
+
+/// After we borrow check a closure, we are left with various
+/// requirements that we have inferred between the free regions that
+/// appear in the closure's signature or on its field types. These
+/// requirements are then verified and proved by the closure's
+/// creating function. This struct encodes those requirements.
+///
+/// The requirements are listed as being between various `RegionVid`. The 0th
+/// region refers to `'static`; subsequent region vids refer to the free
+/// regions that appear in the closure (or coroutine's) type, in order of
+/// appearance. (This numbering is actually defined by the `UniversalRegions`
+/// struct in the NLL region checker. See for example
+/// `UniversalRegions::closure_mapping`.) Note the free regions in the
+/// closure's signature and captures are erased.
+///
+/// Example: If type check produces a closure with the closure args:
+///
+/// ```text
+/// ClosureArgs = [
+///     'a,                                         // From the parent.
+///     'b,
+///     i8,                                         // the "closure kind"
+///     for<'x> fn(&'<erased> &'x u32) -> &'x u32,  // the "closure signature"
+///     &'<erased> String,                          // some upvar
+/// ]
+/// ```
+///
+/// We would "renumber" each free region to a unique vid, as follows:
+///
+/// ```text
+/// ClosureArgs = [
+///     '1,                                         // From the parent.
+///     '2,
+///     i8,                                         // the "closure kind"
+///     for<'x> fn(&'3 &'x u32) -> &'x u32,         // the "closure signature"
+///     &'4 String,                                 // some upvar
+/// ]
+/// ```
+///
+/// Now the code might impose a requirement like `'1: '2`. When an
+/// instance of the closure is created, the corresponding free regions
+/// can be extracted from its type and constrained to have the given
+/// outlives relationship.
+#[derive(Clone, Debug)]
+pub struct ClosureRegionRequirements<'tcx> {
+    /// The number of external regions defined on the closure. In our
+    /// example above, it would be 3 -- one for `'static`, then `'1`
+    /// and `'2`. This is just used for a sanity check later on, to
+    /// make sure that the number of regions we see at the callsite
+    /// matches.
+    pub num_external_vids: usize,
+
+    /// Requirements between the various free regions defined in
+    /// indices.
+    pub outlives_requirements: Vec<ClosureOutlivesRequirement<'tcx>>,
+}
 
-    let borrowck_result = do_mir_borrowck(tcx, def, None).0;
-    debug!("mir_borrowck done");
+/// Indicates an outlives-constraint between a type or between two
+/// free regions declared on the closure.
+#[derive(Copy, Clone, Debug)]
+pub struct ClosureOutlivesRequirement<'tcx> {
+    // This region or type ...
+    pub subject: ClosureOutlivesSubject<'tcx>,
 
-    tcx.arena.alloc(borrowck_result)
+    // ... must outlive this one.
+    pub outlived_free_region: ty::RegionVid,
+
+    // If not, report an error here ...
+    pub blame_span: Span,
+
+    // ... due to this reason.
+    pub category: ConstraintCategory<'tcx>,
+}
+
+// Make sure this enum doesn't unintentionally grow
+#[cfg(target_pointer_width = "64")]
+rustc_data_structures::static_assert_size!(ConstraintCategory<'_>, 16);
+
+/// The subject of a `ClosureOutlivesRequirement` -- that is, the thing
+/// that must outlive some region.
+#[derive(Copy, Clone, Debug)]
+pub enum ClosureOutlivesSubject<'tcx> {
+    /// Subject is a type, typically a type parameter, but could also
+    /// be a projection. Indicates a requirement like `T: 'a` being
+    /// passed to the caller, where the type here is `T`.
+    Ty(ClosureOutlivesSubjectTy<'tcx>),
+
+    /// Subject is a free region from the closure. Indicates a requirement
+    /// like `'a: 'b` being passed to the caller; the region here is `'a`.
+    Region(ty::RegionVid),
+}
+
+/// Represents a `ty::Ty` for use in [`ClosureOutlivesSubject`].
+///
+/// This abstraction is necessary because the type may include `ReVar` regions,
+/// which is what we use internally within NLL code, and they can't be used in
+/// a query response.
+#[derive(Copy, Clone, Debug)]
+pub struct ClosureOutlivesSubjectTy<'tcx> {
+    inner: Ty<'tcx>,
+}
+// DO NOT implement `TypeVisitable` or `TypeFoldable` traits, because this
+// type is not recognized as a binder for late-bound region.
+impl<'tcx, I> !TypeVisitable<I> for ClosureOutlivesSubjectTy<'tcx> {}
+impl<'tcx, I> !TypeFoldable<I> for ClosureOutlivesSubjectTy<'tcx> {}
+
+impl<'tcx> ClosureOutlivesSubjectTy<'tcx> {
+    /// All regions of `ty` must be of kind `ReVar` and must represent
+    /// universal regions *external* to the closure.
+    pub fn bind(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Self {
+        let inner = fold_regions(tcx, ty, |r, depth| match r.kind() {
+            ty::ReVar(vid) => {
+                let br = ty::BoundRegion {
+                    var: ty::BoundVar::from_usize(vid.index()),
+                    kind: ty::BoundRegionKind::Anon,
+                };
+                ty::Region::new_bound(tcx, depth, br)
+            }
+            _ => bug!("unexpected region in ClosureOutlivesSubjectTy: {r:?}"),
+        });
+
+        Self { inner }
+    }
+
+    pub fn instantiate(
+        self,
+        tcx: TyCtxt<'tcx>,
+        mut map: impl FnMut(ty::RegionVid) -> ty::Region<'tcx>,
+    ) -> Ty<'tcx> {
+        fold_regions(tcx, self.inner, |r, depth| match r.kind() {
+            ty::ReBound(debruijn, br) => {
+                debug_assert_eq!(debruijn, depth);
+                map(ty::RegionVid::from_usize(br.var.index()))
+            }
+            _ => bug!("unexpected region {r:?}"),
+        })
+    }
 }
 
 /// Perform the actual borrow checking.
 ///
 /// Use `consumer_options: None` for the default behavior of returning
-/// [`BorrowCheckResult`] only. Otherwise, return [`BodyWithBorrowckFacts`] according
-/// to the given [`ConsumerOptions`].
-#[instrument(skip(tcx), level = "debug")]
+/// [`PropagatedBorrowCheckResults`] only. Otherwise, return [`BodyWithBorrowckFacts`]
+/// according to the given [`ConsumerOptions`].
+///
+/// For nested bodies this should only be called through `root_cx.get_or_insert_nested`.
+#[instrument(skip(root_cx), level = "debug")]
 fn do_mir_borrowck<'tcx>(
-    tcx: TyCtxt<'tcx>,
+    root_cx: &mut BorrowCheckRootCtxt<'tcx>,
     def: LocalDefId,
     consumer_options: Option<ConsumerOptions>,
-) -> (BorrowCheckResult<'tcx>, Option<Box<BodyWithBorrowckFacts<'tcx>>>) {
+) -> (PropagatedBorrowCheckResults<'tcx>, Option<Box<BodyWithBorrowckFacts<'tcx>>>) {
+    let tcx = root_cx.tcx;
     let infcx = BorrowckInferCtxt::new(tcx, def);
     let (input_body, promoted) = tcx.mir_promoted(def);
     let input_body: &Body<'_> = &input_body.borrow();
     let input_promoted: &IndexSlice<_, _> = &promoted.borrow();
     if let Some(e) = input_body.tainted_by_errors {
         infcx.set_tainted_by_errors(e);
+        root_cx.set_tainted_by_errors(e);
     }
 
     let mut local_names = IndexVec::from_elem(None, &input_body.local_decls);
@@ -185,13 +347,13 @@ fn do_mir_borrowck<'tcx>(
     // Compute non-lexical lifetimes.
     let nll::NllOutput {
         regioncx,
-        concrete_opaque_types,
         polonius_input,
         polonius_output,
         opt_closure_req,
         nll_errors,
         polonius_diagnostics,
     } = nll::compute_regions(
+        root_cx,
         &infcx,
         free_regions,
         body,
@@ -210,26 +372,19 @@ fn do_mir_borrowck<'tcx>(
     // We also have a `#[rustc_regions]` annotation that causes us to dump
     // information.
     let diags_buffer = &mut BorrowckDiagnosticsBuffer::default();
-    nll::dump_annotation(
-        &infcx,
-        body,
-        &regioncx,
-        &opt_closure_req,
-        &concrete_opaque_types,
-        diags_buffer,
-    );
+    nll::dump_annotation(&infcx, body, &regioncx, &opt_closure_req, diags_buffer);
 
     let movable_coroutine =
-        // The first argument is the coroutine type passed by value
-        if let Some(local) = body.local_decls.raw.get(1)
-        // Get the interior types and args which typeck computed
-        && let ty::Coroutine(def_id, _) = *local.ty.kind()
-        && tcx.coroutine_movability(def_id) == hir::Movability::Movable
-    {
-        true
-    } else {
-        false
-    };
+    // The first argument is the coroutine type passed by value
+    if let Some(local) = body.local_decls.raw.get(1)
+    // Get the interior types and args which typeck computed
+    && let ty::Coroutine(def_id, _) = *local.ty.kind()
+    && tcx.coroutine_movability(def_id) == hir::Movability::Movable
+{
+    true
+} else {
+    false
+};
 
     // While promoteds should mostly be correct by construction, we need to check them for
     // invalid moves to detect moving out of arrays:`struct S; fn main() { &([S][0]); }`.
@@ -240,6 +395,7 @@ fn do_mir_borrowck<'tcx>(
         // this check out of `MirBorrowckCtxt`, actually doing so is far from trivial.
         let move_data = MoveData::gather_moves(promoted_body, tcx, |_| true);
         let mut promoted_mbcx = MirBorrowckCtxt {
+            root_cx,
             infcx: &infcx,
             body: promoted_body,
             move_data: &move_data,
@@ -280,6 +436,7 @@ fn do_mir_borrowck<'tcx>(
     }
 
     let mut mbcx = MirBorrowckCtxt {
+        root_cx,
         infcx: &infcx,
         body,
         move_data: &move_data,
@@ -347,13 +504,13 @@ fn do_mir_borrowck<'tcx>(
 
     debug!("mbcx.used_mut: {:?}", mbcx.used_mut);
     mbcx.lint_unused_mut();
-    let tainted_by_errors = mbcx.emit_errors();
+    if let Some(guar) = mbcx.emit_errors() {
+        mbcx.root_cx.set_tainted_by_errors(guar);
+    }
 
-    let result = BorrowCheckResult {
-        concrete_opaque_types: concrete_opaque_types.into_inner(),
+    let result = PropagatedBorrowCheckResults {
         closure_requirements: opt_closure_req,
         used_mut_upvars: mbcx.used_mut_upvars,
-        tainted_by_errors,
     };
 
     let body_with_facts = if consumer_options.is_some() {
@@ -488,6 +645,7 @@ impl<'tcx> Deref for BorrowckInferCtxt<'tcx> {
 }
 
 struct MirBorrowckCtxt<'a, 'infcx, 'tcx> {
+    root_cx: &'a mut BorrowCheckRootCtxt<'tcx>,
     infcx: &'infcx BorrowckInferCtxt<'tcx>,
     body: &'a Body<'tcx>,
     move_data: &'a MoveData<'tcx>,
@@ -1361,11 +1519,13 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> {
                     | AggregateKind::CoroutineClosure(def_id, _)
                     | AggregateKind::Coroutine(def_id, _) => {
                         let def_id = def_id.expect_local();
-                        let BorrowCheckResult { used_mut_upvars, .. } =
-                            self.infcx.tcx.mir_borrowck(def_id);
+                        let used_mut_upvars = self.root_cx.used_mut_upvars(def_id);
                         debug!("{:?} used_mut_upvars={:?}", def_id, used_mut_upvars);
-                        for field in used_mut_upvars {
-                            self.propagate_closure_used_mut_upvar(&operands[*field]);
+                        // FIXME: We're cloning the `SmallVec` here to avoid borrowing `root_cx`
+                        // when calling `propagate_closure_used_mut_upvar`. This should ideally
+                        // be unnecessary.
+                        for field in used_mut_upvars.clone() {
+                            self.propagate_closure_used_mut_upvar(&operands[field]);
                         }
                     }
                     AggregateKind::Adt(..)

compiler/rustc_borrowck/src/nll.rs

@@ -8,10 +8,7 @@ use std::str::FromStr;
 use polonius_engine::{Algorithm, Output};
 use rustc_index::IndexSlice;
 use rustc_middle::mir::pretty::{PrettyPrintMirOptions, dump_mir_with_options};
-use rustc_middle::mir::{
-    Body, ClosureOutlivesSubject, ClosureRegionRequirements, PassWhere, Promoted, create_dump_file,
-    dump_enabled, dump_mir,
-};
+use rustc_middle::mir::{Body, PassWhere, Promoted, create_dump_file, dump_enabled, dump_mir};
 use rustc_middle::ty::print::with_no_trimmed_paths;
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_mir_dataflow::ResultsCursor;
@@ -25,21 +22,22 @@ use tracing::{debug, instrument};
 use crate::borrow_set::BorrowSet;
 use crate::consumers::ConsumerOptions;
 use crate::diagnostics::{BorrowckDiagnosticsBuffer, RegionErrors};
-use crate::opaque_types::ConcreteOpaqueTypes;
 use crate::polonius::PoloniusDiagnosticsContext;
 use crate::polonius::legacy::{
     PoloniusFacts, PoloniusFactsExt, PoloniusLocationTable, PoloniusOutput,
 };
 use crate::region_infer::RegionInferenceContext;
 use crate::type_check::{self, MirTypeckResults};
 use crate::universal_regions::UniversalRegions;
-use crate::{BorrowckInferCtxt, polonius, renumber};
+use crate::{
+    BorrowCheckRootCtxt, BorrowckInferCtxt, ClosureOutlivesSubject, ClosureRegionRequirements,
+    polonius, renumber,
+};
 
 /// The output of `nll::compute_regions`. This includes the computed `RegionInferenceContext`, any
 /// closure requirements to propagate, and any generated errors.
 pub(crate) struct NllOutput<'tcx> {
     pub regioncx: RegionInferenceContext<'tcx>,
-    pub concrete_opaque_types: ConcreteOpaqueTypes<'tcx>,
     pub polonius_input: Option<Box<PoloniusFacts>>,
     pub polonius_output: Option<Box<PoloniusOutput>>,
     pub opt_closure_req: Option<ClosureRegionRequirements<'tcx>>,
@@ -78,6 +76,7 @@ pub(crate) fn replace_regions_in_mir<'tcx>(
 ///
 /// This may result in errors being reported.
 pub(crate) fn compute_regions<'a, 'tcx>(
+    root_cx: &mut BorrowCheckRootCtxt<'tcx>,
     infcx: &BorrowckInferCtxt<'tcx>,
     universal_regions: UniversalRegions<'tcx>,
     body: &Body<'tcx>,
@@ -98,15 +97,14 @@ pub(crate) fn compute_regions<'a, 'tcx>(
 
     let location_map = Rc::new(DenseLocationMap::new(body));
 
-    let mut concrete_opaque_types = ConcreteOpaqueTypes::default();
-
     // Run the MIR type-checker.
     let MirTypeckResults {
         constraints,
         universal_region_relations,
         opaque_type_values,
         polonius_context,
     } = type_check::type_check(
+        root_cx,
         infcx,
         body,
         promoted,
@@ -117,7 +115,6 @@ pub(crate) fn compute_regions<'a, 'tcx>(
         flow_inits,
         move_data,
         Rc::clone(&location_map),
-        &mut concrete_opaque_types,
     );
 
     // Create the region inference context, taking ownership of the
@@ -181,11 +178,10 @@ pub(crate) fn compute_regions<'a, 'tcx>(
         infcx.set_tainted_by_errors(guar);
     }
 
-    regioncx.infer_opaque_types(infcx, opaque_type_values, &mut concrete_opaque_types);
+    regioncx.infer_opaque_types(root_cx, infcx, opaque_type_values);
 
     NllOutput {
         regioncx,
-        concrete_opaque_types,
         polonius_input: polonius_facts.map(Box::new),
         polonius_output,
         opt_closure_req: closure_region_requirements,
@@ -301,7 +297,6 @@ pub(super) fn dump_annotation<'tcx, 'infcx>(
     body: &Body<'tcx>,
     regioncx: &RegionInferenceContext<'tcx>,
     closure_region_requirements: &Option<ClosureRegionRequirements<'tcx>>,
-    concrete_opaque_types: &ConcreteOpaqueTypes<'tcx>,
     diagnostics_buffer: &mut BorrowckDiagnosticsBuffer<'infcx, 'tcx>,
 ) {
     let tcx = infcx.tcx;
@@ -318,7 +313,7 @@ pub(super) fn dump_annotation<'tcx, 'infcx>(
     // better.
 
     let def_span = tcx.def_span(body.source.def_id());
-    let mut err = if let Some(closure_region_requirements) = closure_region_requirements {
+    let err = if let Some(closure_region_requirements) = closure_region_requirements {
         let mut err = infcx.dcx().struct_span_note(def_span, "external requirements");
 
         regioncx.annotate(tcx, &mut err);
@@ -344,9 +339,7 @@ pub(super) fn dump_annotation<'tcx, 'infcx>(
         err
     };
 
-    if !concrete_opaque_types.is_empty() {
-        err.note(format!("Inferred opaque type values:\n{concrete_opaque_types:#?}"));
-    }
+    // FIXME(@lcnr): We currently don't dump the inferred hidden types here.
 
     diagnostics_buffer.buffer_non_error(err);
 }

compiler/rustc_borrowck/src/polonius/dump.rs

@@ -5,7 +5,7 @@ use rustc_index::IndexVec;
 use rustc_middle::mir::pretty::{
     PassWhere, PrettyPrintMirOptions, create_dump_file, dump_enabled, dump_mir_to_writer,
 };
-use rustc_middle::mir::{Body, ClosureRegionRequirements, Location};
+use rustc_middle::mir::{Body, Location};
 use rustc_middle::ty::{RegionVid, TyCtxt};
 use rustc_mir_dataflow::points::PointIndex;
 use rustc_session::config::MirIncludeSpans;
@@ -17,7 +17,7 @@ use crate::polonius::{
 };
 use crate::region_infer::values::LivenessValues;
 use crate::type_check::Locations;
-use crate::{BorrowckInferCtxt, RegionInferenceContext};
+use crate::{BorrowckInferCtxt, ClosureRegionRequirements, RegionInferenceContext};
 
 /// `-Zdump-mir=polonius` dumps MIR annotated with NLL and polonius specific information.
 pub(crate) fn dump_polonius_mir<'tcx>(

compiler/rustc_borrowck/src/region_infer/mod.rs

@@ -13,9 +13,8 @@ use rustc_infer::infer::region_constraints::{GenericKind, VarInfos, VerifyBound,
 use rustc_infer::infer::{InferCtxt, NllRegionVariableOrigin, RegionVariableOrigin};
 use rustc_middle::bug;
 use rustc_middle::mir::{
-    AnnotationSource, BasicBlock, Body, ClosureOutlivesRequirement, ClosureOutlivesSubject,
-    ClosureOutlivesSubjectTy, ClosureRegionRequirements, ConstraintCategory, Local, Location,
-    ReturnConstraint, TerminatorKind,
+    AnnotationSource, BasicBlock, Body, ConstraintCategory, Local, Location, ReturnConstraint,
+    TerminatorKind,
 };
 use rustc_middle::traits::{ObligationCause, ObligationCauseCode};
 use rustc_middle::ty::{self, RegionVid, Ty, TyCtxt, TypeFoldable, UniverseIndex, fold_regions};
@@ -24,7 +23,6 @@ use rustc_span::hygiene::DesugaringKind;
 use rustc_span::{DUMMY_SP, Span};
 use tracing::{Level, debug, enabled, instrument, trace};
 
-use crate::BorrowckInferCtxt;
 use crate::constraints::graph::{self, NormalConstraintGraph, RegionGraph};
 use crate::constraints::{ConstraintSccIndex, OutlivesConstraint, OutlivesConstraintSet};
 use crate::dataflow::BorrowIndex;
@@ -37,6 +35,10 @@ use crate::region_infer::values::{LivenessValues, RegionElement, RegionValues, T
 use crate::type_check::free_region_relations::UniversalRegionRelations;
 use crate::type_check::{Locations, MirTypeckRegionConstraints};
 use crate::universal_regions::UniversalRegions;
+use crate::{
+    BorrowckInferCtxt, ClosureOutlivesRequirement, ClosureOutlivesSubject,
+    ClosureOutlivesSubjectTy, ClosureRegionRequirements,
+};
 
 mod dump_mir;
 mod graphviz;

compiler/rustc_borrowck/src/region_infer/opaque_types.rs

@@ -10,7 +10,7 @@ use rustc_trait_selection::opaque_types::check_opaque_type_parameter_valid;
 use tracing::{debug, instrument};
 
 use super::RegionInferenceContext;
-use crate::opaque_types::ConcreteOpaqueTypes;
+use crate::BorrowCheckRootCtxt;
 use crate::session_diagnostics::LifetimeMismatchOpaqueParam;
 use crate::universal_regions::RegionClassification;
 
@@ -58,12 +58,12 @@ impl<'tcx> RegionInferenceContext<'tcx> {
     ///
     /// [rustc-dev-guide chapter]:
     /// https://rustc-dev-guide.rust-lang.org/opaque-types-region-infer-restrictions.html
-    #[instrument(level = "debug", skip(self, infcx), ret)]
+    #[instrument(level = "debug", skip(self, root_cx, infcx), ret)]
     pub(crate) fn infer_opaque_types(
         &self,
+        root_cx: &mut BorrowCheckRootCtxt<'tcx>,
         infcx: &InferCtxt<'tcx>,
         opaque_ty_decls: FxIndexMap<OpaqueTypeKey<'tcx>, OpaqueHiddenType<'tcx>>,
-        concrete_opaque_types: &mut ConcreteOpaqueTypes<'tcx>,
     ) {
         let mut decls_modulo_regions: FxIndexMap<OpaqueTypeKey<'tcx>, (OpaqueTypeKey<'tcx>, Span)> =
             FxIndexMap::default();
@@ -140,11 +140,11 @@ impl<'tcx> RegionInferenceContext<'tcx> {
                 }
             }
 
-            concrete_opaque_types.insert(
-                infcx.tcx,
+            root_cx.add_concrete_opaque_type(
                 opaque_type_key.def_id,
-                OpaqueHiddenType { ty, span: concrete_type.span },
+                OpaqueHiddenType { span: concrete_type.span, ty },
             );
+
             // Check that all opaque types have the same region parameters if they have the same
             // non-region parameters. This is necessary because within the new solver we perform
             // various query operations modulo regions, and thus could unsoundly select some impls

compiler/rustc_borrowck/src/root_cx.rs

@@ -0,0 +1,101 @@
+use rustc_abi::FieldIdx;
+use rustc_data_structures::fx::FxHashMap;
+use rustc_hir::def_id::LocalDefId;
+use rustc_middle::bug;
+use rustc_middle::ty::{OpaqueHiddenType, Ty, TyCtxt, TypeVisitableExt};
+use rustc_span::ErrorGuaranteed;
+use smallvec::SmallVec;
+
+use crate::{ClosureRegionRequirements, ConcreteOpaqueTypes, PropagatedBorrowCheckResults};
+
+/// The shared context used by both the root as well as all its nested
+/// items.
+pub(super) struct BorrowCheckRootCtxt<'tcx> {
+    pub tcx: TyCtxt<'tcx>,
+    root_def_id: LocalDefId,
+    concrete_opaque_types: ConcreteOpaqueTypes<'tcx>,
+    nested_bodies: FxHashMap<LocalDefId, PropagatedBorrowCheckResults<'tcx>>,
+    tainted_by_errors: Option<ErrorGuaranteed>,
+}
+
+impl<'tcx> BorrowCheckRootCtxt<'tcx> {
+    pub(super) fn new(tcx: TyCtxt<'tcx>, root_def_id: LocalDefId) -> BorrowCheckRootCtxt<'tcx> {
+        BorrowCheckRootCtxt {
+            tcx,
+            root_def_id,
+            concrete_opaque_types: Default::default(),
+            nested_bodies: Default::default(),
+            tainted_by_errors: None,
+        }
+    }
+
+    /// Collect all defining uses of opaque types inside of this typeck root. This
+    /// expects the hidden type to be mapped to the definition parameters of the opaque
+    /// and errors if we end up with distinct hidden types.
+    pub(super) fn add_concrete_opaque_type(
+        &mut self,
+        def_id: LocalDefId,
+        hidden_ty: OpaqueHiddenType<'tcx>,
+    ) {
+        // Sometimes two opaque types are the same only after we remap the generic parameters
+        // back to the opaque type definition. E.g. we may have `OpaqueType<X, Y>` mapped to
+        // `(X, Y)` and `OpaqueType<Y, X>` mapped to `(Y, X)`, and those are the same, but we
+        // only know that once we convert the generic parameters to those of the opaque type.
+        if let Some(prev) = self.concrete_opaque_types.0.get_mut(&def_id) {
+            if prev.ty != hidden_ty.ty {
+                let guar = hidden_ty.ty.error_reported().err().unwrap_or_else(|| {
+                    let (Ok(e) | Err(e)) =
+                        prev.build_mismatch_error(&hidden_ty, self.tcx).map(|d| d.emit());
+                    e
+                });
+                prev.ty = Ty::new_error(self.tcx, guar);
+            }
+            // Pick a better span if there is one.
+            // FIXME(oli-obk): collect multiple spans for better diagnostics down the road.
+            prev.span = prev.span.substitute_dummy(hidden_ty.span);
+        } else {
+            self.concrete_opaque_types.0.insert(def_id, hidden_ty);
+        }
+    }
+
+    pub(super) fn set_tainted_by_errors(&mut self, guar: ErrorGuaranteed) {
+        self.tainted_by_errors = Some(guar);
+    }
+
+    fn get_or_insert_nested(&mut self, def_id: LocalDefId) -> &PropagatedBorrowCheckResults<'tcx> {
+        debug_assert_eq!(
+            self.tcx.typeck_root_def_id(def_id.to_def_id()),
+            self.root_def_id.to_def_id()
+        );
+        if !self.nested_bodies.contains_key(&def_id) {
+            let result = super::do_mir_borrowck(self, def_id, None).0;
+            if let Some(prev) = self.nested_bodies.insert(def_id, result) {
+                bug!("unexpected previous nested body: {prev:?}");
+            }
+        }
+
+        self.nested_bodies.get(&def_id).unwrap()
+    }
+
+    pub(super) fn closure_requirements(
+        &mut self,
+        nested_body_def_id: LocalDefId,
+    ) -> &Option<ClosureRegionRequirements<'tcx>> {
+        &self.get_or_insert_nested(nested_body_def_id).closure_requirements
+    }
+
+    pub(super) fn used_mut_upvars(
+        &mut self,
+        nested_body_def_id: LocalDefId,
+    ) -> &SmallVec<[FieldIdx; 8]> {
+        &self.get_or_insert_nested(nested_body_def_id).used_mut_upvars
+    }
+
+    pub(super) fn finalize(self) -> Result<&'tcx ConcreteOpaqueTypes<'tcx>, ErrorGuaranteed> {
+        if let Some(guar) = self.tainted_by_errors {
+            Err(guar)
+        } else {
+            Ok(self.tcx.arena.alloc(self.concrete_opaque_types))
+        }
+    }
+}

compiler/rustc_borrowck/src/type_check/constraint_conversion.rs

@@ -6,7 +6,6 @@ use rustc_infer::infer::region_constraints::{GenericKind, VerifyBound};
 use rustc_infer::infer::{self, InferCtxt, SubregionOrigin};
 use rustc_infer::traits::query::type_op::DeeplyNormalize;
 use rustc_middle::bug;
-use rustc_middle::mir::{ClosureOutlivesSubject, ClosureRegionRequirements, ConstraintCategory};
 use rustc_middle::ty::{
     self, GenericArgKind, Ty, TyCtxt, TypeFoldable, TypeVisitableExt, fold_regions,
 };
@@ -18,6 +17,7 @@ use crate::constraints::OutlivesConstraint;
 use crate::region_infer::TypeTest;
 use crate::type_check::{Locations, MirTypeckRegionConstraints};
 use crate::universal_regions::UniversalRegions;
+use crate::{ClosureOutlivesSubject, ClosureRegionRequirements, ConstraintCategory};
 
 pub(crate) struct ConstraintConversion<'a, 'tcx> {
     infcx: &'a InferCtxt<'tcx>,

compiler/rustc_borrowck/src/type_check/mod.rs

@@ -45,15 +45,14 @@ use crate::borrow_set::BorrowSet;
 use crate::constraints::{OutlivesConstraint, OutlivesConstraintSet};
 use crate::diagnostics::UniverseInfo;
 use crate::member_constraints::MemberConstraintSet;
-use crate::opaque_types::ConcreteOpaqueTypes;
 use crate::polonius::legacy::{PoloniusFacts, PoloniusLocationTable};
 use crate::polonius::{PoloniusContext, PoloniusLivenessContext};
 use crate::region_infer::TypeTest;
 use crate::region_infer::values::{LivenessValues, PlaceholderIndex, PlaceholderIndices};
 use crate::session_diagnostics::{MoveUnsized, SimdIntrinsicArgConst};
 use crate::type_check::free_region_relations::{CreateResult, UniversalRegionRelations};
 use crate::universal_regions::{DefiningTy, UniversalRegions};
-use crate::{BorrowckInferCtxt, path_utils};
+use crate::{BorrowCheckRootCtxt, BorrowckInferCtxt, path_utils};
 
 macro_rules! span_mirbug {
     ($context:expr, $elem:expr, $($message:tt)*) => ({
@@ -102,6 +101,7 @@ mod relate_tys;
 /// - `move_data` -- move-data constructed when performing the maybe-init dataflow analysis
 /// - `location_map` -- map between MIR `Location` and `PointIndex`
 pub(crate) fn type_check<'a, 'tcx>(
+    root_cx: &mut BorrowCheckRootCtxt<'tcx>,
     infcx: &BorrowckInferCtxt<'tcx>,
     body: &Body<'tcx>,
     promoted: &IndexSlice<Promoted, Body<'tcx>>,
@@ -112,7 +112,6 @@ pub(crate) fn type_check<'a, 'tcx>(
     flow_inits: ResultsCursor<'a, 'tcx, MaybeInitializedPlaces<'a, 'tcx>>,
     move_data: &MoveData<'tcx>,
     location_map: Rc<DenseLocationMap>,
-    concrete_opaque_types: &mut ConcreteOpaqueTypes<'tcx>,
 ) -> MirTypeckResults<'tcx> {
     let implicit_region_bound = ty::Region::new_var(infcx.tcx, universal_regions.fr_fn_body);
     let mut constraints = MirTypeckRegionConstraints {
@@ -153,6 +152,7 @@ pub(crate) fn type_check<'a, 'tcx>(
     };
 
     let mut typeck = TypeChecker {
+        root_cx,
         infcx,
         last_span: body.span,
         body,
@@ -167,7 +167,6 @@ pub(crate) fn type_check<'a, 'tcx>(
         polonius_facts,
         borrow_set,
         constraints: &mut constraints,
-        concrete_opaque_types,
         polonius_liveness,
     };
 
@@ -215,6 +214,7 @@ enum FieldAccessError {
 /// way, it accrues region constraints -- these can later be used by
 /// NLL region checking.
 struct TypeChecker<'a, 'tcx> {
+    root_cx: &'a mut BorrowCheckRootCtxt<'tcx>,
     infcx: &'a BorrowckInferCtxt<'tcx>,
     last_span: Span,
     body: &'a Body<'tcx>,
@@ -233,7 +233,6 @@ struct TypeChecker<'a, 'tcx> {
     polonius_facts: &'a mut Option<PoloniusFacts>,
     borrow_set: &'a BorrowSet<'tcx>,
     constraints: &'a mut MirTypeckRegionConstraints<'tcx>,
-    concrete_opaque_types: &'a mut ConcreteOpaqueTypes<'tcx>,
     /// When using `-Zpolonius=next`, the liveness helper data used to create polonius constraints.
     polonius_liveness: Option<PoloniusLivenessContext>,
 }
@@ -2503,11 +2502,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
         args: GenericArgsRef<'tcx>,
         locations: Locations,
     ) -> ty::InstantiatedPredicates<'tcx> {
-        let closure_borrowck_results = tcx.mir_borrowck(def_id);
-        self.concrete_opaque_types
-            .extend_from_nested_body(tcx, &closure_borrowck_results.concrete_opaque_types);
-
-        if let Some(closure_requirements) = &closure_borrowck_results.closure_requirements {
+        if let Some(closure_requirements) = &self.root_cx.closure_requirements(def_id) {
             constraint_conversion::ConstraintConversion::new(
                 self.infcx,
                 self.universal_regions,

compiler/rustc_hir_analysis/src/check/check.rs

@@ -397,8 +397,11 @@ fn best_definition_site_of_opaque<'tcx>(
                 return ControlFlow::Continue(());
             }
 
-            if let Some(hidden_ty) =
-                self.tcx.mir_borrowck(item_def_id).concrete_opaque_types.get(&self.opaque_def_id)
+            if let Some(hidden_ty) = self
+                .tcx
+                .mir_borrowck(item_def_id)
+                .ok()
+                .and_then(|opaque_types| opaque_types.0.get(&self.opaque_def_id))
             {
                 ControlFlow::Break((hidden_ty.span, item_def_id))
             } else {
@@ -413,9 +416,6 @@ fn best_definition_site_of_opaque<'tcx>(
             self.tcx
         }
         fn visit_expr(&mut self, ex: &'tcx hir::Expr<'tcx>) -> Self::Result {
-            if let hir::ExprKind::Closure(closure) = ex.kind {
-                self.check(closure.def_id)?;
-            }
             intravisit::walk_expr(self, ex)
         }
         fn visit_item(&mut self, it: &'tcx hir::Item<'tcx>) -> Self::Result {

compiler/rustc_hir_analysis/src/collect/type_of/opaque.rs

@@ -183,25 +183,23 @@ impl<'tcx> TaitConstraintLocator<'tcx> {
                     self.non_defining_use_in_defining_scope(item_def_id);
                 }
             }
-            DefiningScopeKind::MirBorrowck => {
-                let borrowck_result = tcx.mir_borrowck(item_def_id);
-                if let Some(guar) = borrowck_result.tainted_by_errors {
-                    self.insert_found(ty::OpaqueHiddenType::new_error(tcx, guar));
-                } else if let Some(&hidden_type) =
-                    borrowck_result.concrete_opaque_types.get(&self.def_id)
-                {
-                    debug!(?hidden_type, "found constraint");
-                    self.insert_found(hidden_type);
-                } else if let Err(guar) = tcx
-                    .type_of_opaque_hir_typeck(self.def_id)
-                    .instantiate_identity()
-                    .error_reported()
-                {
-                    self.insert_found(ty::OpaqueHiddenType::new_error(tcx, guar));
-                } else {
-                    self.non_defining_use_in_defining_scope(item_def_id);
+            DefiningScopeKind::MirBorrowck => match tcx.mir_borrowck(item_def_id) {
+                Err(guar) => self.insert_found(ty::OpaqueHiddenType::new_error(tcx, guar)),
+                Ok(concrete_opaque_types) => {
+                    if let Some(&hidden_type) = concrete_opaque_types.0.get(&self.def_id) {
+                        debug!(?hidden_type, "found constraint");
+                        self.insert_found(hidden_type);
+                    } else if let Err(guar) = tcx
+                        .type_of_opaque_hir_typeck(self.def_id)
+                        .instantiate_identity()
+                        .error_reported()
+                    {
+                        self.insert_found(ty::OpaqueHiddenType::new_error(tcx, guar));
+                    } else {
+                        self.non_defining_use_in_defining_scope(item_def_id);
+                    }
                 }
-            }
+            },
         }
     }
 }
@@ -264,20 +262,20 @@ pub(super) fn find_opaque_ty_constraints_for_rpit<'tcx>(
                 Ty::new_diverging_default(tcx)
             }
         }
-        DefiningScopeKind::MirBorrowck => {
-            let borrowck_result = tcx.mir_borrowck(owner_def_id);
-            if let Some(guar) = borrowck_result.tainted_by_errors {
-                Ty::new_error(tcx, guar)
-            } else if let Some(hidden_ty) = borrowck_result.concrete_opaque_types.get(&def_id) {
-                hidden_ty.ty
-            } else {
-                let hir_ty = tcx.type_of_opaque_hir_typeck(def_id).instantiate_identity();
-                if let Err(guar) = hir_ty.error_reported() {
-                    Ty::new_error(tcx, guar)
+        DefiningScopeKind::MirBorrowck => match tcx.mir_borrowck(owner_def_id) {
+            Ok(concrete_opaque_types) => {
+                if let Some(hidden_ty) = concrete_opaque_types.0.get(&def_id) {
+                    hidden_ty.ty
                 } else {
-                    hir_ty
+                    let hir_ty = tcx.type_of_opaque_hir_typeck(def_id).instantiate_identity();
+                    if let Err(guar) = hir_ty.error_reported() {
+                        Ty::new_error(tcx, guar)
+                    } else {
+                        hir_ty
+                    }
                 }
             }
-        }
+            Err(guar) => Ty::new_error(tcx, guar),
+        },
     }
 }

compiler/rustc_interface/src/passes.rs

@@ -955,7 +955,9 @@ fn run_required_analyses(tcx: TyCtxt<'_>) {
             // Run unsafety check because it's responsible for stealing and
             // deallocating THIR.
             tcx.ensure_ok().check_unsafety(def_id);
-            tcx.ensure_ok().mir_borrowck(def_id)
+            if !tcx.is_typeck_child(def_id.to_def_id()) {
+                tcx.ensure_ok().mir_borrowck(def_id)
+            }
         });
     });
     sess.time("MIR_effect_checking", || {

compiler/rustc_middle/src/arena.rs

@@ -28,7 +28,7 @@ macro_rules! arena_types {
                     rustc_middle::mir::Body<'tcx>
                 >,
             [decode] typeck_results: rustc_middle::ty::TypeckResults<'tcx>,
-            [decode] borrowck_result: rustc_middle::mir::BorrowCheckResult<'tcx>,
+            [decode] borrowck_result: rustc_middle::mir::ConcreteOpaqueTypes<'tcx>,
             [] resolver: rustc_data_structures::steal::Steal<(
                 rustc_middle::ty::ResolverAstLowering,
                 std::sync::Arc<rustc_ast::Crate>,

compiler/rustc_middle/src/mir/query.rs

@@ -6,14 +6,13 @@ use rustc_abi::{FieldIdx, VariantIdx};
 use rustc_data_structures::fx::FxIndexMap;
 use rustc_errors::ErrorGuaranteed;
 use rustc_hir::def_id::LocalDefId;
+use rustc_index::IndexVec;
 use rustc_index::bit_set::BitMatrix;
-use rustc_index::{Idx, IndexVec};
 use rustc_macros::{HashStable, TyDecodable, TyEncodable, TypeFoldable, TypeVisitable};
 use rustc_span::{Span, Symbol};
-use smallvec::SmallVec;
 
 use super::{ConstValue, SourceInfo};
-use crate::ty::{self, CoroutineArgsExt, OpaqueHiddenType, Ty, TyCtxt, fold_regions};
+use crate::ty::{self, CoroutineArgsExt, OpaqueHiddenType, Ty};
 
 rustc_index::newtype_index! {
     #[derive(HashStable)]
@@ -85,16 +84,11 @@ impl Debug for CoroutineLayout<'_> {
     }
 }
 
-#[derive(Debug, TyEncodable, TyDecodable, HashStable)]
-pub struct BorrowCheckResult<'tcx> {
-    /// All the opaque types that are restricted to concrete types
-    /// by this function. Unlike the value in `TypeckResults`, this has
-    /// unerased regions.
-    pub concrete_opaque_types: FxIndexMap<LocalDefId, OpaqueHiddenType<'tcx>>,
-    pub closure_requirements: Option<ClosureRegionRequirements<'tcx>>,
-    pub used_mut_upvars: SmallVec<[FieldIdx; 8]>,
-    pub tainted_by_errors: Option<ErrorGuaranteed>,
-}
+/// All the opaque types that are restricted to concrete types
+/// by this function. Unlike the value in `TypeckResults`, this has
+/// unerased regions.
+#[derive(Default, Debug, TyEncodable, TyDecodable, HashStable)]
+pub struct ConcreteOpaqueTypes<'tcx>(pub FxIndexMap<LocalDefId, OpaqueHiddenType<'tcx>>);
 
 /// The result of the `mir_const_qualif` query.
 ///
@@ -108,84 +102,6 @@ pub struct ConstQualifs {
     pub needs_non_const_drop: bool,
     pub tainted_by_errors: Option<ErrorGuaranteed>,
 }
-
-/// After we borrow check a closure, we are left with various
-/// requirements that we have inferred between the free regions that
-/// appear in the closure's signature or on its field types. These
-/// requirements are then verified and proved by the closure's
-/// creating function. This struct encodes those requirements.
-///
-/// The requirements are listed as being between various `RegionVid`. The 0th
-/// region refers to `'static`; subsequent region vids refer to the free
-/// regions that appear in the closure (or coroutine's) type, in order of
-/// appearance. (This numbering is actually defined by the `UniversalRegions`
-/// struct in the NLL region checker. See for example
-/// `UniversalRegions::closure_mapping`.) Note the free regions in the
-/// closure's signature and captures are erased.
-///
-/// Example: If type check produces a closure with the closure args:
-///
-/// ```text
-/// ClosureArgs = [
-///     'a,                                         // From the parent.
-///     'b,
-///     i8,                                         // the "closure kind"
-///     for<'x> fn(&'<erased> &'x u32) -> &'x u32,  // the "closure signature"
-///     &'<erased> String,                          // some upvar
-/// ]
-/// ```
-///
-/// We would "renumber" each free region to a unique vid, as follows:
-///
-/// ```text
-/// ClosureArgs = [
-///     '1,                                         // From the parent.
-///     '2,
-///     i8,                                         // the "closure kind"
-///     for<'x> fn(&'3 &'x u32) -> &'x u32,         // the "closure signature"
-///     &'4 String,                                 // some upvar
-/// ]
-/// ```
-///
-/// Now the code might impose a requirement like `'1: '2`. When an
-/// instance of the closure is created, the corresponding free regions
-/// can be extracted from its type and constrained to have the given
-/// outlives relationship.
-#[derive(Clone, Debug, TyEncodable, TyDecodable, HashStable)]
-pub struct ClosureRegionRequirements<'tcx> {
-    /// The number of external regions defined on the closure. In our
-    /// example above, it would be 3 -- one for `'static`, then `'1`
-    /// and `'2`. This is just used for a sanity check later on, to
-    /// make sure that the number of regions we see at the callsite
-    /// matches.
-    pub num_external_vids: usize,
-
-    /// Requirements between the various free regions defined in
-    /// indices.
-    pub outlives_requirements: Vec<ClosureOutlivesRequirement<'tcx>>,
-}
-
-/// Indicates an outlives-constraint between a type or between two
-/// free regions declared on the closure.
-#[derive(Copy, Clone, Debug, TyEncodable, TyDecodable, HashStable)]
-pub struct ClosureOutlivesRequirement<'tcx> {
-    // This region or type ...
-    pub subject: ClosureOutlivesSubject<'tcx>,
-
-    // ... must outlive this one.
-    pub outlived_free_region: ty::RegionVid,
-
-    // If not, report an error here ...
-    pub blame_span: Span,
-
-    // ... due to this reason.
-    pub category: ConstraintCategory<'tcx>,
-}
-
-// Make sure this enum doesn't unintentionally grow
-#[cfg(target_pointer_width = "64")]
-rustc_data_structures::static_assert_size!(ConstraintCategory<'_>, 16);
-
 /// Outlives-constraints can be categorized to determine whether and why they
 /// are interesting (for error reporting). Order of variants indicates sort
 /// order of the category, thereby influencing diagnostic output.
@@ -253,66 +169,6 @@ pub enum AnnotationSource {
     GenericArg,
 }
 
-/// The subject of a `ClosureOutlivesRequirement` -- that is, the thing
-/// that must outlive some region.
-#[derive(Copy, Clone, Debug, TyEncodable, TyDecodable, HashStable)]
-pub enum ClosureOutlivesSubject<'tcx> {
-    /// Subject is a type, typically a type parameter, but could also
-    /// be a projection. Indicates a requirement like `T: 'a` being
-    /// passed to the caller, where the type here is `T`.
-    Ty(ClosureOutlivesSubjectTy<'tcx>),
-
-    /// Subject is a free region from the closure. Indicates a requirement
-    /// like `'a: 'b` being passed to the caller; the region here is `'a`.
-    Region(ty::RegionVid),
-}
-
-/// Represents a `ty::Ty` for use in [`ClosureOutlivesSubject`].
-///
-/// This abstraction is necessary because the type may include `ReVar` regions,
-/// which is what we use internally within NLL code, and they can't be used in
-/// a query response.
-///
-/// DO NOT implement `TypeVisitable` or `TypeFoldable` traits, because this
-/// type is not recognized as a binder for late-bound region.
-#[derive(Copy, Clone, Debug, TyEncodable, TyDecodable, HashStable)]
-pub struct ClosureOutlivesSubjectTy<'tcx> {
-    inner: Ty<'tcx>,
-}
-
-impl<'tcx> ClosureOutlivesSubjectTy<'tcx> {
-    /// All regions of `ty` must be of kind `ReVar` and must represent
-    /// universal regions *external* to the closure.
-    pub fn bind(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Self {
-        let inner = fold_regions(tcx, ty, |r, depth| match r.kind() {
-            ty::ReVar(vid) => {
-                let br = ty::BoundRegion {
-                    var: ty::BoundVar::new(vid.index()),
-                    kind: ty::BoundRegionKind::Anon,
-                };
-                ty::Region::new_bound(tcx, depth, br)
-            }
-            _ => bug!("unexpected region in ClosureOutlivesSubjectTy: {r:?}"),
-        });
-
-        Self { inner }
-    }
-
-    pub fn instantiate(
-        self,
-        tcx: TyCtxt<'tcx>,
-        mut map: impl FnMut(ty::RegionVid) -> ty::Region<'tcx>,
-    ) -> Ty<'tcx> {
-        fold_regions(tcx, self.inner, |r, depth| match r.kind() {
-            ty::ReBound(debruijn, br) => {
-                debug_assert_eq!(debruijn, depth);
-                map(ty::RegionVid::new(br.var.index()))
-            }
-            _ => bug!("unexpected region {r:?}"),
-        })
-    }
-}
-
 /// The constituent parts of a mir constant of kind ADT or array.
 #[derive(Copy, Clone, Debug, HashStable)]
 pub struct DestructuredConstant<'tcx> {

compiler/rustc_middle/src/query/mod.rs

@@ -1153,11 +1153,10 @@ rustc_queries! {
         return_result_from_ensure_ok
     }
 
-    /// Borrow-checks the function body. If this is a closure, returns
-    /// additional requirements that the closure's creator must verify.
-    query mir_borrowck(key: LocalDefId) -> &'tcx mir::BorrowCheckResult<'tcx> {
+    /// Borrow-checks the given typeck root, e.g. functions, const/static items,
+    /// and its children, e.g. closures, inline consts.
+    query mir_borrowck(key: LocalDefId) -> Result<&'tcx mir::ConcreteOpaqueTypes<'tcx>, ErrorGuaranteed> {
         desc { |tcx| "borrow-checking `{}`", tcx.def_path_str(key) }
-        cache_on_disk_if(tcx) { tcx.is_typeck_child(key.to_def_id()) }
     }
 
     /// Gets a complete map from all types to their inherent impls.

compiler/rustc_middle/src/ty/codec.rs

@@ -501,7 +501,7 @@ impl_decodable_via_ref! {
     &'tcx ty::List<ty::PolyExistentialPredicate<'tcx>>,
     &'tcx traits::ImplSource<'tcx, ()>,
     &'tcx mir::Body<'tcx>,
-    &'tcx mir::BorrowCheckResult<'tcx>,
+    &'tcx mir::ConcreteOpaqueTypes<'tcx>,
     &'tcx ty::List<ty::BoundVariableKind>,
     &'tcx ty::ListWithCachedTypeInfo<ty::Clause<'tcx>>,
     &'tcx ty::List<FieldIdx>,

compiler/rustc_mir_transform/src/lib.rs

@@ -498,8 +498,11 @@ fn mir_drops_elaborated_and_const_checked(tcx: TyCtxt<'_>, def: LocalDefId) -> &
     }
 
     // We only need to borrowck non-synthetic MIR.
-    let tainted_by_errors =
-        if !tcx.is_synthetic_mir(def) { tcx.mir_borrowck(def).tainted_by_errors } else { None };
+    let tainted_by_errors = if !tcx.is_synthetic_mir(def) {
+        tcx.mir_borrowck(tcx.typeck_root_def_id(def.to_def_id()).expect_local()).err()
+    } else {
+        None
+    };
 
     let is_fn_like = tcx.def_kind(def).is_fn_like();
     if is_fn_like {
@@ -795,7 +798,7 @@ fn promoted_mir(tcx: TyCtxt<'_>, def: LocalDefId) -> &IndexVec<Promoted, Body<'_
     }
 
     if !tcx.is_synthetic_mir(def) {
-        tcx.ensure_done().mir_borrowck(def);
+        tcx.ensure_done().mir_borrowck(tcx.typeck_root_def_id(def.to_def_id()).expect_local());
     }
     let mut promoted = tcx.mir_promoted(def).1.steal();
 

tests/ui/pattern/non-structural-match-types-cycle-err.rs

@@ -0,0 +1,24 @@
+//@ edition:2021
+
+struct AnyOption<T>(T);
+impl<T> AnyOption<T> {
+    const NONE: Option<T> = None;
+}
+
+// This is an unfortunate side-effect of borrowchecking nested items
+// together with their parent. Evaluating the `AnyOption::<_>::NONE`
+// pattern for exhaustiveness checking relies on the layout of the
+// async block. This layout relies on `optimized_mir` of the nested
+// item which is now borrowck'd together with its parent. As
+// borrowck of the parent requires us to have already lowered the match,
+// this is a query cycle.
+
+fn uwu() {}
+fn defines() {
+    match Some(async {}) {
+        AnyOption::<_>::NONE => {}
+        //~^ ERROR cycle detected when building THIR for `defines`
+        _ => {}
+    }
+}
+fn main() {}

tests/ui/pattern/non-structural-match-types-cycle-err.stderr

@@ -0,0 +1,64 @@
+error[E0391]: cycle detected when building THIR for `defines`
+  --> $DIR/non-structural-match-types-cycle-err.rs:19:9
+   |
+LL |         AnyOption::<_>::NONE => {}
+   |         ^^^^^^^^^^^^^^^^^^^^
+   |
+note: ...which requires evaluating type-level constant...
+  --> $DIR/non-structural-match-types-cycle-err.rs:5:5
+   |
+LL |     const NONE: Option<T> = None;
+   |     ^^^^^^^^^^^^^^^^^^^^^
+note: ...which requires const-evaluating + checking `<impl at $DIR/non-structural-match-types-cycle-err.rs:4:1: 4:21>::NONE`...
+  --> $DIR/non-structural-match-types-cycle-err.rs:5:5
+   |
+LL |     const NONE: Option<T> = None;
+   |     ^^^^^^^^^^^^^^^^^^^^^
+   = note: ...which requires computing layout of `core::option::Option<{async block@$DIR/non-structural-match-types-cycle-err.rs:18:16: 18:21}>`...
+   = note: ...which requires computing layout of `{async block@$DIR/non-structural-match-types-cycle-err.rs:18:16: 18:21}`...
+note: ...which requires optimizing MIR for `defines::{closure#0}`...
+  --> $DIR/non-structural-match-types-cycle-err.rs:18:16
+   |
+LL |     match Some(async {}) {
+   |                ^^^^^
+note: ...which requires elaborating drops for `defines::{closure#0}`...
+  --> $DIR/non-structural-match-types-cycle-err.rs:18:16
+   |
+LL |     match Some(async {}) {
+   |                ^^^^^
+note: ...which requires borrow-checking `defines`...
+  --> $DIR/non-structural-match-types-cycle-err.rs:17:1
+   |
+LL | fn defines() {
+   | ^^^^^^^^^^^^
+note: ...which requires promoting constants in MIR for `defines`...
+  --> $DIR/non-structural-match-types-cycle-err.rs:17:1
+   |
+LL | fn defines() {
+   | ^^^^^^^^^^^^
+note: ...which requires checking if `defines` contains FFI-unwind calls...
+  --> $DIR/non-structural-match-types-cycle-err.rs:17:1
+   |
+LL | fn defines() {
+   | ^^^^^^^^^^^^
+note: ...which requires building MIR for `defines`...
+  --> $DIR/non-structural-match-types-cycle-err.rs:17:1
+   |
+LL | fn defines() {
+   | ^^^^^^^^^^^^
+note: ...which requires match-checking `defines`...
+  --> $DIR/non-structural-match-types-cycle-err.rs:17:1
+   |
+LL | fn defines() {
+   | ^^^^^^^^^^^^
+   = note: ...which again requires building THIR for `defines`, completing the cycle
+note: cycle used when unsafety-checking `defines`
+  --> $DIR/non-structural-match-types-cycle-err.rs:17:1
+   |
+LL | fn defines() {
+   | ^^^^^^^^^^^^
+   = note: see https://rustc-dev-guide.rust-lang.org/overview.html#queries and https://rustc-dev-guide.rust-lang.org/query.html for more information
+
+error: aborting due to 1 previous error
+
+For more information about this error, try `rustc --explain E0391`.

tests/ui/pattern/non-structural-match-types.rs

@@ -1,6 +1,4 @@
 //@ edition:2021
-#![feature(const_async_blocks)]
-
 struct AnyOption<T>(T);
 impl<T> AnyOption<T> {
     const NONE: Option<T> = None;
@@ -19,11 +17,5 @@ fn defines() {
         //~^ ERROR constant of non-structural type
         _ => {}
     }
-
-    match Some(async {}) {
-        AnyOption::<_>::NONE => {}
-        //~^ ERROR constant of non-structural type
-        _ => {}
-    }
 }
 fn main() {}

tests/ui/pattern/non-structural-match-types.stderr

@@ -1,5 +1,5 @@
 error: constant of non-structural type `Option<fn() {uwu}>` in a pattern
-  --> $DIR/non-structural-match-types.rs:12:9
+  --> $DIR/non-structural-match-types.rs:10:9
    |
 LL | impl<T> AnyOption<T> {
    | --------------------
@@ -11,8 +11,8 @@ LL |         AnyOption::<_>::NONE => {}
    |
    = note: see https://doc.rust-lang.org/stable/std/marker/trait.StructuralPartialEq.html for details
 
-error: constant of non-structural type `Option<{closure@$DIR/non-structural-match-types.rs:17:16: 17:18}>` in a pattern
-  --> $DIR/non-structural-match-types.rs:18:9
+error: constant of non-structural type `Option<{closure@$DIR/non-structural-match-types.rs:15:16: 15:18}>` in a pattern
+  --> $DIR/non-structural-match-types.rs:16:9
    |
 LL | impl<T> AnyOption<T> {
    | --------------------
@@ -24,19 +24,5 @@ LL |         AnyOption::<_>::NONE => {}
    |
    = note: see https://doc.rust-lang.org/stable/std/marker/trait.StructuralPartialEq.html for details
 
-error: constant of non-structural type `Option<{async block@$DIR/non-structural-match-types.rs:23:16: 23:21}>` in a pattern
-  --> $DIR/non-structural-match-types.rs:24:9
-   |
-LL | impl<T> AnyOption<T> {
-   | --------------------
-LL |     const NONE: Option<T> = None;
-   |     --------------------- constant defined here
-...
-LL |         AnyOption::<_>::NONE => {}
-   |         ^^^^^^^^^^^^^^^^^^^^ constant of non-structural type
-   |
-   = note: see https://doc.rust-lang.org/stable/std/marker/trait.StructuralPartialEq.html for details
-   = note: `ResumeTy` must be annotated with `#[derive(PartialEq)]` to be usable in patterns
-
-error: aborting due to 3 previous errors
+error: aborting due to 2 previous errors