Rollup of 6 pull requests

.mailmap

@@ -286,7 +286,7 @@ Xuefeng Wu <[email protected]> Xuefeng Wu <[email protected]>
 Xuefeng Wu <[email protected]> XuefengWu <[email protected]>
 York Xiang <[email protected]>
 Youngsoo Son <[email protected]> <[email protected]>
-Yuki Okushi <[email protected]>
+Yuki Okushi <[email protected]> <[email protected]>
 Zach Pomerantz <[email protected]>
 Zack Corr <[email protected]> <[email protected]>
 Zack Slayton <[email protected]>

compiler/rustc_codegen_llvm/src/consts.rs

@@ -8,22 +8,47 @@ use crate::value::Value;
 use cstr::cstr;
 use libc::c_uint;
 use rustc_codegen_ssa::traits::*;
+use rustc_data_structures::captures::Captures;
 use rustc_hir::def_id::DefId;
 use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs};
 use rustc_middle::mir::interpret::{
-    read_target_uint, Allocation, ErrorHandled, GlobalAlloc, Pointer,
+    read_target_uint, Allocation, ErrorHandled, GlobalAlloc, InitChunk, Pointer,
 };
 use rustc_middle::mir::mono::MonoItem;
 use rustc_middle::ty::{self, Instance, Ty};
 use rustc_middle::{bug, span_bug};
 use rustc_target::abi::{AddressSpace, Align, HasDataLayout, LayoutOf, Primitive, Scalar, Size};
+use std::ops::Range;
 use tracing::debug;
 
 pub fn const_alloc_to_llvm(cx: &CodegenCx<'ll, '_>, alloc: &Allocation) -> &'ll Value {
     let mut llvals = Vec::with_capacity(alloc.relocations().len() + 1);
     let dl = cx.data_layout();
     let pointer_size = dl.pointer_size.bytes() as usize;
 
+    // Note: this function may call `inspect_with_uninit_and_ptr_outside_interpreter`,
+    // so `range` must be within the bounds of `alloc` and not within a relocation.
+    fn chunks_of_init_and_uninit_bytes<'ll, 'a, 'b>(
+        cx: &'a CodegenCx<'ll, 'b>,
+        alloc: &'a Allocation,
+        range: Range<usize>,
+    ) -> impl Iterator<Item = &'ll Value> + Captures<'a> + Captures<'b> {
+        alloc
+            .init_mask()
+            .range_as_init_chunks(Size::from_bytes(range.start), Size::from_bytes(range.end))
+            .map(move |chunk| match chunk {
+                InitChunk::Init(range) => {
+                    let range = (range.start.bytes() as usize)..(range.end.bytes() as usize);
+                    let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(range);
+                    cx.const_bytes(bytes)
+                }
+                InitChunk::Uninit(range) => {
+                    let len = range.end.bytes() - range.start.bytes();
+                    cx.const_undef(cx.type_array(cx.type_i8(), len))
+                }
+            })
+    }
+
     let mut next_offset = 0;
     for &(offset, ((), alloc_id)) in alloc.relocations().iter() {
         let offset = offset.bytes();
@@ -32,12 +57,8 @@ pub fn const_alloc_to_llvm(cx: &CodegenCx<'ll, '_>, alloc: &Allocation) -> &'ll
         if offset > next_offset {
             // This `inspect` is okay since we have checked that it is not within a relocation, it
             // is within the bounds of the allocation, and it doesn't affect interpreter execution
-            // (we inspect the result after interpreter execution). Any undef byte is replaced with
-            // some arbitrary byte value.
-            //
-            // FIXME: relay undef bytes to codegen as undef const bytes
-            let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(next_offset..offset);
-            llvals.push(cx.const_bytes(bytes));
+            // (we inspect the result after interpreter execution).
+            llvals.extend(chunks_of_init_and_uninit_bytes(cx, alloc, next_offset..offset));
         }
         let ptr_offset = read_target_uint(
             dl.endian,
@@ -65,12 +86,8 @@ pub fn const_alloc_to_llvm(cx: &CodegenCx<'ll, '_>, alloc: &Allocation) -> &'ll
         let range = next_offset..alloc.len();
         // This `inspect` is okay since we have check that it is after all relocations, it is
         // within the bounds of the allocation, and it doesn't affect interpreter execution (we
-        // inspect the result after interpreter execution). Any undef byte is replaced with some
-        // arbitrary byte value.
-        //
-        // FIXME: relay undef bytes to codegen as undef const bytes
-        let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(range);
-        llvals.push(cx.const_bytes(bytes));
+        // inspect the result after interpreter execution).
+        llvals.extend(chunks_of_init_and_uninit_bytes(cx, alloc, range));
     }
 
     cx.const_struct(&llvals, true)

compiler/rustc_middle/src/mir/interpret/allocation.rs

@@ -761,20 +761,24 @@ impl InitMask {
         }
 
         // FIXME(oli-obk): optimize this for allocations larger than a block.
-        let idx = (start.bytes()..end.bytes()).map(Size::from_bytes).find(|&i| !self.get(i));
+        let idx = (start..end).find(|&i| !self.get(i));
 
         match idx {
             Some(idx) => {
-                let uninit_end = (idx.bytes()..end.bytes())
-                    .map(Size::from_bytes)
-                    .find(|&i| self.get(i))
-                    .unwrap_or(end);
+                let uninit_end = (idx..end).find(|&i| self.get(i)).unwrap_or(end);
                 Err(idx..uninit_end)
             }
             None => Ok(()),
         }
     }
 
+    /// Returns an iterator, yielding a range of byte indexes for each contiguous region
+    /// of initialized or uninitialized bytes inside the range `start..end` (end-exclusive).
+    #[inline]
+    pub fn range_as_init_chunks(&self, start: Size, end: Size) -> InitChunkIter<'_> {
+        InitChunkIter::new(self, start, end)
+    }
+
     pub fn set_range(&mut self, start: Size, end: Size, new_state: bool) {
         let len = self.len;
         if end > len {
@@ -867,6 +871,49 @@ impl InitMask {
     }
 }
 
+/// Yields [`InitChunk`]s. See [`InitMask::range_as_init_chunks`].
+pub struct InitChunkIter<'a> {
+    init_mask: &'a InitMask,
+    /// The current byte index into `init_mask`.
+    start: Size,
+    /// The end byte index into `init_mask`.
+    end: Size,
+}
+
+/// A contiguous chunk of initialized or uninitialized memory.
+pub enum InitChunk {
+    Init(Range<Size>),
+    Uninit(Range<Size>),
+}
+
+impl<'a> InitChunkIter<'a> {
+    fn new(init_mask: &'a InitMask, start: Size, end: Size) -> Self {
+        assert!(start <= end);
+        assert!(end <= init_mask.len);
+        Self { init_mask, start, end }
+    }
+}
+
+impl<'a> Iterator for InitChunkIter<'a> {
+    type Item = InitChunk;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.start >= self.end {
+            return None;
+        }
+
+        let is_init = self.init_mask.get(self.start);
+        // FIXME(oli-obk): optimize this for allocations larger than a block.
+        let end_of_chunk =
+            (self.start..self.end).find(|&i| self.init_mask.get(i) != is_init).unwrap_or(self.end);
+        let range = self.start..end_of_chunk;
+
+        self.start = end_of_chunk;
+
+        Some(if is_init { InitChunk::Init(range) } else { InitChunk::Uninit(range) })
+    }
+}
+
 #[inline]
 fn bit_index(bits: Size) -> (usize, usize) {
     let bits = bits.bytes();

compiler/rustc_middle/src/mir/interpret/mod.rs

@@ -125,7 +125,9 @@ pub use self::error::{
 
 pub use self::value::{get_slice_bytes, ConstAlloc, ConstValue, Scalar, ScalarMaybeUninit};
 
-pub use self::allocation::{Allocation, AllocationExtra, InitMask, Relocations};
+pub use self::allocation::{
+    Allocation, AllocationExtra, InitChunk, InitChunkIter, InitMask, Relocations,
+};
 
 pub use self::pointer::{Pointer, PointerArithmetic};
 

compiler/rustc_middle/src/mir/mod.rs

@@ -1482,7 +1482,7 @@ pub enum StatementKind<'tcx> {
     ///
     /// Note that this also is emitted for regular `let` bindings to ensure that locals that are
     /// never accessed still get some sanity checks for, e.g., `let x: ! = ..;`
-    FakeRead(FakeReadCause, Box<Place<'tcx>>),
+    FakeRead(Box<(FakeReadCause, Place<'tcx>)>),
 
     /// Write the discriminant for a variant to the enum Place.
     SetDiscriminant { place: Box<Place<'tcx>>, variant_index: VariantIdx },
@@ -1575,7 +1575,12 @@ pub enum FakeReadCause {
 
     /// `let x: !; match x {}` doesn't generate any read of x so we need to
     /// generate a read of x to check that it is initialized and safe.
-    ForMatchedPlace,
+    ///
+    /// If a closure pattern matches a Place starting with an Upvar, then we introduce a
+    /// FakeRead for that Place outside the closure, in such a case this option would be
+    /// Some(closure_def_id).
+    /// Otherwise, the value of the optional DefId will be None.
+    ForMatchedPlace(Option<DefId>),
 
     /// A fake read of the RefWithinGuard version of a bind-by-value variable
     /// in a match guard to ensure that it's value hasn't change by the time
@@ -1594,7 +1599,12 @@ pub enum FakeReadCause {
     /// but in some cases it can affect the borrow checker, as in #53695.
     /// Therefore, we insert a "fake read" here to ensure that we get
     /// appropriate errors.
-    ForLet,
+    ///
+    /// If a closure pattern matches a Place starting with an Upvar, then we introduce a
+    /// FakeRead for that Place outside the closure, in such a case this option would be
+    /// Some(closure_def_id).
+    /// Otherwise, the value of the optional DefId will be None.
+    ForLet(Option<DefId>),
 
     /// If we have an index expression like
     ///
@@ -1618,7 +1628,9 @@ impl Debug for Statement<'_> {
         use self::StatementKind::*;
         match self.kind {
             Assign(box (ref place, ref rv)) => write!(fmt, "{:?} = {:?}", place, rv),
-            FakeRead(ref cause, ref place) => write!(fmt, "FakeRead({:?}, {:?})", cause, place),
+            FakeRead(box (ref cause, ref place)) => {
+                write!(fmt, "FakeRead({:?}, {:?})", cause, place)
+            }
             Retag(ref kind, ref place) => write!(
                 fmt,
                 "Retag({}{:?})",

compiler/rustc_middle/src/mir/visit.rs

@@ -380,7 +380,7 @@ macro_rules! make_mir_visitor {
                     ) => {
                         self.visit_assign(place, rvalue, location);
                     }
-                    StatementKind::FakeRead(_, place) => {
+                    StatementKind::FakeRead(box (_, place)) => {
                         self.visit_place(
                             place,
                             PlaceContext::NonMutatingUse(NonMutatingUseContext::Inspect),

compiler/rustc_mir/src/borrow_check/diagnostics/conflict_errors.rs

@@ -1728,7 +1728,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
         impl<'tcx> Visitor<'tcx> for FakeReadCauseFinder<'tcx> {
             fn visit_statement(&mut self, statement: &Statement<'tcx>, _: Location) {
                 match statement {
-                    Statement { kind: StatementKind::FakeRead(cause, box place), .. }
+                    Statement { kind: StatementKind::FakeRead(box (cause, place)), .. }
                         if *place == self.place =>
                     {
                         self.cause = Some(*cause);

compiler/rustc_mir/src/borrow_check/diagnostics/explain_borrow.rs

@@ -515,7 +515,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
                 let block = &self.body.basic_blocks()[location.block];
 
                 let kind = if let Some(&Statement {
-                    kind: StatementKind::FakeRead(FakeReadCause::ForLet, _),
+                    kind: StatementKind::FakeRead(box (FakeReadCause::ForLet(_), _)),
                     ..
                 }) = block.statements.get(location.statement_index)
                 {

compiler/rustc_mir/src/borrow_check/diagnostics/mod.rs

@@ -7,8 +7,8 @@ use rustc_hir::def_id::DefId;
 use rustc_hir::lang_items::LangItemGroup;
 use rustc_hir::GeneratorKind;
 use rustc_middle::mir::{
-    AggregateKind, Constant, Field, Local, LocalInfo, LocalKind, Location, Operand, Place,
-    PlaceRef, ProjectionElem, Rvalue, Statement, StatementKind, Terminator, TerminatorKind,
+    AggregateKind, Constant, FakeReadCause, Field, Local, LocalInfo, LocalKind, Location, Operand,
+    Place, PlaceRef, ProjectionElem, Rvalue, Statement, StatementKind, Terminator, TerminatorKind,
 };
 use rustc_middle::ty::print::Print;
 use rustc_middle::ty::{self, DefIdTree, Instance, Ty, TyCtxt};
@@ -795,6 +795,24 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
             }
         }
 
+        // StatementKind::FakeRead only contains a def_id if they are introduced as a result
+        // of pattern matching within a closure.
+        if let StatementKind::FakeRead(box (cause, ref place)) = stmt.kind {
+            match cause {
+                FakeReadCause::ForMatchedPlace(Some(closure_def_id))
+                | FakeReadCause::ForLet(Some(closure_def_id)) => {
+                    debug!("move_spans: def_id={:?} place={:?}", closure_def_id, place);
+                    let places = &[Operand::Move(*place)];
+                    if let Some((args_span, generator_kind, var_span)) =
+                        self.closure_span(closure_def_id, moved_place, places)
+                    {
+                        return ClosureUse { generator_kind, args_span, var_span };
+                    }
+                }
+                _ => {}
+            }
+        }
+
         let normal_ret =
             if moved_place.projection.iter().any(|p| matches!(p, ProjectionElem::Downcast(..))) {
                 PatUse(stmt.source_info.span)

compiler/rustc_mir/src/borrow_check/invalidation.rs

@@ -63,7 +63,7 @@ impl<'cx, 'tcx> Visitor<'tcx> for InvalidationGenerator<'cx, 'tcx> {
 
                 self.mutate_place(location, *lhs, Shallow(None), JustWrite);
             }
-            StatementKind::FakeRead(_, _) => {
+            StatementKind::FakeRead(box (_, _)) => {
                 // Only relevant for initialized/liveness/safety checks.
             }
             StatementKind::SetDiscriminant { place, variant_index: _ } => {

compiler/rustc_mir/src/borrow_check/mod.rs

@@ -574,7 +574,7 @@ impl<'cx, 'tcx> dataflow::ResultsVisitor<'cx, 'tcx> for MirBorrowckCtxt<'cx, 'tc
 
                 self.mutate_place(location, (*lhs, span), Shallow(None), JustWrite, flow_state);
             }
-            StatementKind::FakeRead(_, box ref place) => {
+            StatementKind::FakeRead(box (_, ref place)) => {
                 // Read for match doesn't access any memory and is used to
                 // assert that a place is safe and live. So we don't have to
                 // do any checks here.

compiler/rustc_mir/src/dataflow/move_paths/builder.rs

@@ -293,8 +293,8 @@ impl<'b, 'a, 'tcx> Gatherer<'b, 'a, 'tcx> {
                 }
                 self.gather_rvalue(rval);
             }
-            StatementKind::FakeRead(_, place) => {
-                self.create_move_path(**place);
+            StatementKind::FakeRead(box (_, place)) => {
+                self.create_move_path(*place);
             }
             StatementKind::LlvmInlineAsm(ref asm) => {
                 for (output, kind) in iter::zip(&*asm.outputs, &asm.asm.outputs) {

compiler/rustc_mir/src/transform/coverage/spans.rs

@@ -683,10 +683,10 @@ pub(super) fn filtered_statement_span(
         // and `_1` is the `Place` for `somenum`.
         //
         // If and when the Issue is resolved, remove this special case match pattern:
-        StatementKind::FakeRead(cause, _) if cause == FakeReadCause::ForGuardBinding => None,
+        StatementKind::FakeRead(box (cause, _)) if cause == FakeReadCause::ForGuardBinding => None,
 
         // Retain spans from all other statements
-        StatementKind::FakeRead(_, _) // Not including `ForGuardBinding`
+        StatementKind::FakeRead(box (_, _)) // Not including `ForGuardBinding`
         | StatementKind::CopyNonOverlapping(..)
         | StatementKind::Assign(_)
         | StatementKind::SetDiscriminant { .. }

compiler/rustc_mir_build/src/build/cfg.rs

@@ -80,7 +80,7 @@ impl<'tcx> CFG<'tcx> {
         cause: FakeReadCause,
         place: Place<'tcx>,
     ) {
-        let kind = StatementKind::FakeRead(cause, box place);
+        let kind = StatementKind::FakeRead(box (cause, place));
         let stmt = Statement { source_info, kind };
         self.push(block, stmt);
     }

compiler/rustc_mir_build/src/build/expr/as_rvalue.rs

@@ -179,24 +179,20 @@ impl<'a, 'tcx> Builder<'a, 'tcx> {
                 //     match x { _ => () } // fake read of `x`
                 // };
                 // ```
-                // FIXME(RFC2229): Remove feature gate once diagnostics are improved
-                if this.tcx.features().capture_disjoint_fields {
-                    for (thir_place, cause, hir_id) in fake_reads.into_iter() {
-                        let place_builder =
-                            unpack!(block = this.as_place_builder(block, thir_place));
-
-                        if let Ok(place_builder_resolved) =
-                            place_builder.try_upvars_resolved(this.tcx, this.typeck_results)
-                        {
-                            let mir_place =
-                                place_builder_resolved.into_place(this.tcx, this.typeck_results);
-                            this.cfg.push_fake_read(
-                                block,
-                                this.source_info(this.tcx.hir().span(*hir_id)),
-                                *cause,
-                                mir_place,
-                            );
-                        }
+                for (thir_place, cause, hir_id) in fake_reads.into_iter() {
+                    let place_builder = unpack!(block = this.as_place_builder(block, thir_place));
+
+                    if let Ok(place_builder_resolved) =
+                        place_builder.try_upvars_resolved(this.tcx, this.typeck_results)
+                    {
+                        let mir_place =
+                            place_builder_resolved.into_place(this.tcx, this.typeck_results);
+                        this.cfg.push_fake_read(
+                            block,
+                            this.source_info(this.tcx.hir().span(*hir_id)),
+                            *cause,
+                            mir_place,
+                        );
                     }
                 }