diff --git a/src/stacked_borrows/mod.rs b/src/stacked_borrows/mod.rs
index 66fdd685de..5cdcaecc17 100644
--- a/src/stacked_borrows/mod.rs
+++ b/src/stacked_borrows/mod.rs
@@ -973,6 +973,14 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx
             }
 
             fn visit_value(&mut self, place: &PlaceTy<'tcx, Provenance>) -> InterpResult<'tcx> {
+                // If this place is smaller than a pointer, we know that it can't contain any
+                // pointers we need to retag, so we can stop recursion early.
+                // This optimization is crucial for ZSTs, because they can contain way more fields
+                // than we can ever visit.
+                if !place.layout.is_unsized() && place.layout.size < self.ecx.pointer_size() {
+                    return Ok(());
+                }
+
                 if let Some((ref_kind, protector)) = qualify(place.layout.ty, self.kind) {
                     self.retag_place(place, ref_kind, self.retag_cause, protector)?;
                 } else if matches!(place.layout.ty.kind(), ty::RawPtr(..)) {
diff --git a/tests/pass/stacked-borrows/zst-field-retagging-terminates.rs b/tests/pass/stacked-borrows/zst-field-retagging-terminates.rs
new file mode 100644
index 0000000000..ce3c8b7d5f
--- /dev/null
+++ b/tests/pass/stacked-borrows/zst-field-retagging-terminates.rs
@@ -0,0 +1,6 @@
+//@compile-flags: -Zmiri-retag-fields
+// Checks that the test does not run forever (which relies on a fast path).
+fn main() {
+    let array = [(); usize::MAX];
+    drop(array); // Pass the array to a function, retagging its fields
+}