resolve: Populate external modules in more automatic and lazy way

The modules are now populated implicitly on the first access

Author: petrochenkov

src/librustc_resolve/build_reduced_graph.rs

@@ -365,7 +365,6 @@ impl<'a> Resolver<'a> {
                     self.get_module(DefId { krate: crate_id, index: CRATE_DEF_INDEX })
                 };
 
-                self.populate_module_if_necessary(module);
                 if let Some(name) = self.session.parse_sess.injected_crate_name.try_get() {
                     if name.as_str() == ident.name.as_str() {
                         self.injected_crate = Some(module);
@@ -632,7 +631,7 @@ impl<'a> Resolver<'a> {
     }
 
     /// Builds the reduced graph for a single item in an external crate.
-    fn build_reduced_graph_for_external_crate_res(
+    crate fn build_reduced_graph_for_external_crate_res(
         &mut self,
         parent: Module<'a>,
         child: Export<ast::NodeId>,
@@ -686,6 +685,7 @@ impl<'a> Resolver<'a> {
                                              span);
                 self.define(parent, ident, TypeNS, (module, vis, DUMMY_SP, expansion));
 
+                module.populate_on_access.set(false);
                 for child in self.cstore.item_children_untracked(def_id, self.session) {
                     let res = child.res.map_id(|_| panic!("unexpected id"));
                     let ns = if let Res::Def(DefKind::AssocTy, _) = res {
@@ -699,7 +699,6 @@ impl<'a> Resolver<'a> {
                         self.has_self.insert(res.def_id());
                     }
                 }
-                module.populated.set(true);
             }
             Res::Def(DefKind::Struct, def_id) | Res::Def(DefKind::Union, def_id) => {
                 self.define(parent, ident, TypeNS, (res, vis, DUMMY_SP, expansion));
@@ -780,18 +779,6 @@ impl<'a> Resolver<'a> {
         Some(ext)
     }
 
-    /// Ensures that the reduced graph rooted at the given external module
-    /// is built, building it if it is not.
-    pub fn populate_module_if_necessary(&mut self, module: Module<'a>) {
-        if module.populated.get() { return }
-        let def_id = module.def_id().unwrap();
-        for child in self.cstore.item_children_untracked(def_id, self.session) {
-            let child = child.map_id(|_| panic!("unexpected id"));
-            self.build_reduced_graph_for_external_crate_res(module, child);
-        }
-        module.populated.set(true)
-    }
-
     fn legacy_import_macro(&mut self,
                            name: Name,
                            binding: &'a NameBinding<'a>,
@@ -863,9 +850,9 @@ impl<'a> Resolver<'a> {
         if let Some(span) = import_all {
             let directive = macro_use_directive(span);
             self.potentially_unused_imports.push(directive);
-            module.for_each_child(|ident, ns, binding| if ns == MacroNS {
-                let imported_binding = self.import(binding, directive);
-                self.legacy_import_macro(ident.name, imported_binding, span, allow_shadowing);
+            module.for_each_child(self, |this, ident, ns, binding| if ns == MacroNS {
+                let imported_binding = this.import(binding, directive);
+                this.legacy_import_macro(ident.name, imported_binding, span, allow_shadowing);
             });
         } else {
             for ident in single_imports.iter().cloned() {

src/librustc_resolve/diagnostics.rs

@@ -65,10 +65,13 @@ fn add_typo_suggestion(
     false
 }
 
-fn add_module_candidates(
-    module: Module<'_>, names: &mut Vec<TypoSuggestion>, filter_fn: &impl Fn(Res) -> bool
+fn add_module_candidates<'a>(
+    resolver: &mut Resolver<'a>,
+    module: Module<'a>,
+    names: &mut Vec<TypoSuggestion>,
+    filter_fn: &impl Fn(Res) -> bool,
 ) {
-    for (&(ident, _), resolution) in module.resolutions.borrow().iter() {
+    for (&(ident, _), resolution) in resolver.resolutions(module).borrow().iter() {
         if let Some(binding) = resolution.borrow().binding {
             let res = binding.res();
             if filter_fn(res) {
@@ -593,10 +596,10 @@ impl<'a> Resolver<'a> {
                 Scope::CrateRoot => {
                     let root_ident = Ident::new(kw::PathRoot, ident.span);
                     let root_module = this.resolve_crate_root(root_ident);
-                    add_module_candidates(root_module, &mut suggestions, filter_fn);
+                    add_module_candidates(this, root_module, &mut suggestions, filter_fn);
                 }
                 Scope::Module(module) => {
-                    add_module_candidates(module, &mut suggestions, filter_fn);
+                    add_module_candidates(this, module, &mut suggestions, filter_fn);
                 }
                 Scope::MacroUsePrelude => {
                     suggestions.extend(this.macro_use_prelude.iter().filter_map(|(name, binding)| {
@@ -644,7 +647,7 @@ impl<'a> Resolver<'a> {
                 Scope::StdLibPrelude => {
                     if let Some(prelude) = this.prelude {
                         let mut tmp_suggestions = Vec::new();
-                        add_module_candidates(prelude, &mut tmp_suggestions, filter_fn);
+                        add_module_candidates(this, prelude, &mut tmp_suggestions, filter_fn);
                         suggestions.extend(tmp_suggestions.into_iter().filter(|s| {
                             use_prelude || this.is_builtin_macro(s.res.opt_def_id())
                         }));
@@ -694,7 +697,9 @@ impl<'a> Resolver<'a> {
         if path.len() == 1 {
             // Search in lexical scope.
             // Walk backwards up the ribs in scope and collect candidates.
-            for rib in self.ribs[ns].iter().rev() {
+            // Ribs have to be cloned to avoid borrowing the resolver.
+            let ribs = self.ribs[ns].clone();
+            for rib in ribs.iter().rev() {
                 // Locals and type parameters
                 for (ident, &res) in &rib.bindings {
                     if filter_fn(res) {
@@ -704,7 +709,7 @@ impl<'a> Resolver<'a> {
                 // Items in scope
                 if let RibKind::ModuleRibKind(module) = rib.kind {
                     // Items from this module
-                    add_module_candidates(module, &mut names, &filter_fn);
+                    add_module_candidates(self, module, &mut names, &filter_fn);
 
                     if let ModuleKind::Block(..) = module.kind {
                         // We can see through blocks
@@ -732,7 +737,7 @@ impl<'a> Resolver<'a> {
                             }));
 
                             if let Some(prelude) = self.prelude {
-                                add_module_candidates(prelude, &mut names, &filter_fn);
+                                add_module_candidates(self, prelude, &mut names, &filter_fn);
                             }
                         }
                         break;
@@ -754,7 +759,7 @@ impl<'a> Resolver<'a> {
                 mod_path, Some(TypeNS), false, span, CrateLint::No
             ) {
                 if let ModuleOrUniformRoot::Module(module) = module {
-                    add_module_candidates(module, &mut names, &filter_fn);
+                    add_module_candidates(self, module, &mut names, &filter_fn);
                 }
             }
         }
@@ -792,11 +797,9 @@ impl<'a> Resolver<'a> {
         while let Some((in_module,
                         path_segments,
                         in_module_is_extern)) = worklist.pop() {
-            self.populate_module_if_necessary(in_module);
-
             // We have to visit module children in deterministic order to avoid
             // instabilities in reported imports (#43552).
-            in_module.for_each_child_stable(|ident, ns, name_binding| {
+            in_module.for_each_child_stable(self, |this, ident, ns, name_binding| {
                 // avoid imports entirely
                 if name_binding.is_import() && !name_binding.is_extern_crate() { return; }
                 // avoid non-importable candidates as well
@@ -830,7 +833,7 @@ impl<'a> Resolver<'a> {
                         // outside crate private modules => no need to check this)
                         if !in_module_is_extern || name_binding.vis == ty::Visibility::Public {
                             let did = match res {
-                                Res::Def(DefKind::Ctor(..), did) => self.parent(did),
+                                Res::Def(DefKind::Ctor(..), did) => this.parent(did),
                                 _ => res.opt_def_id(),
                             };
                             candidates.push(ImportSuggestion { did, path });
@@ -890,8 +893,6 @@ impl<'a> Resolver<'a> {
                         krate: crate_id,
                         index: CRATE_DEF_INDEX,
                     });
-                    self.populate_module_if_necessary(&crate_root);
-
                     suggestions.extend(self.lookup_import_candidates_from_module(
                         lookup_ident, namespace, crate_root, ident, &filter_fn));
                 }
@@ -910,9 +911,7 @@ impl<'a> Resolver<'a> {
             // abort if the module is already found
             if result.is_some() { break; }
 
-            self.populate_module_if_necessary(in_module);
-
-            in_module.for_each_child_stable(|ident, _, name_binding| {
+            in_module.for_each_child_stable(self, |_, ident, _, name_binding| {
                 // abort if the module is already found or if name_binding is private external
                 if result.is_some() || !name_binding.vis.is_visible_locally() {
                     return
@@ -943,10 +942,8 @@ impl<'a> Resolver<'a> {
 
     fn collect_enum_variants(&mut self, def_id: DefId) -> Option<Vec<Path>> {
         self.find_module(def_id).map(|(enum_module, enum_import_suggestion)| {
-            self.populate_module_if_necessary(enum_module);
-
             let mut variants = Vec::new();
-            enum_module.for_each_child_stable(|ident, _, name_binding| {
+            enum_module.for_each_child_stable(self, |_, ident, _, name_binding| {
                 if let Res::Def(DefKind::Variant, _) = name_binding.res() {
                     let mut segms = enum_import_suggestion.path.segments.clone();
                     segms.push(ast::PathSegment::from_ident(ident));
@@ -1147,7 +1144,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
     ///            at the root of the crate instead of the module where it is defined
     /// ```
     pub(crate) fn check_for_module_export_macro(
-        &self,
+        &mut self,
         directive: &'b ImportDirective<'b>,
         module: ModuleOrUniformRoot<'b>,
         ident: Ident,
@@ -1168,7 +1165,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
             return None;
         }
 
-        let resolutions = crate_module.resolutions.borrow();
+        let resolutions = self.resolutions(crate_module).borrow();
         let resolution = resolutions.get(&(ident, MacroNS))?;
         let binding = resolution.borrow().binding()?;
         if let Res::Def(DefKind::Macro(MacroKind::Bang), _) = binding.res() {

src/librustc_resolve/lib.rs

@@ -1035,7 +1035,7 @@ enum RibKind<'a> {
 ///
 /// The resolution keeps a separate stack of ribs as it traverses the AST for each namespace. When
 /// resolving, the name is looked up from inside out.
-#[derive(Debug)]
+#[derive(Clone, Debug)]
 struct Rib<'a, R = Res> {
     bindings: FxHashMap<Ident, R>,
     kind: RibKind<'a>,
@@ -1156,6 +1156,8 @@ impl ModuleKind {
     }
 }
 
+type Resolutions<'a> = RefCell<FxHashMap<(Ident, Namespace), &'a RefCell<NameResolution<'a>>>>;
+
 /// One node in the tree of modules.
 pub struct ModuleData<'a> {
     parent: Option<Module<'a>>,
@@ -1164,7 +1166,11 @@ pub struct ModuleData<'a> {
     // The def id of the closest normal module (`mod`) ancestor (including this module).
     normal_ancestor_id: DefId,
 
-    resolutions: RefCell<FxHashMap<(Ident, Namespace), &'a RefCell<NameResolution<'a>>>>,
+    // Mapping between names and their (possibly in-progress) resolutions in this module.
+    // Resolutions in modules from other crates are not populated until accessed.
+    lazy_resolutions: Resolutions<'a>,
+    // True if this is a module from other crate that needs to be populated on access.
+    populate_on_access: Cell<bool>,
     single_segment_macro_resolutions: RefCell<Vec<(Ident, MacroKind, ParentScope<'a>,
                                                    Option<&'a NameBinding<'a>>)>>,
     multi_segment_macro_resolutions: RefCell<Vec<(Vec<Segment>, Span, MacroKind, ParentScope<'a>,
@@ -1182,11 +1188,6 @@ pub struct ModuleData<'a> {
     // Used to memoize the traits in this module for faster searches through all traits in scope.
     traits: RefCell<Option<Box<[(Ident, &'a NameBinding<'a>)]>>>,
 
-    // Whether this module is populated. If not populated, any attempt to
-    // access the children must be preceded with a
-    // `populate_module_if_necessary` call.
-    populated: Cell<bool>,
-
     /// Span of the module itself. Used for error reporting.
     span: Span,
 
@@ -1205,7 +1206,8 @@ impl<'a> ModuleData<'a> {
             parent,
             kind,
             normal_ancestor_id,
-            resolutions: Default::default(),
+            lazy_resolutions: Default::default(),
+            populate_on_access: Cell::new(!normal_ancestor_id.is_local()),
             single_segment_macro_resolutions: RefCell::new(Vec::new()),
             multi_segment_macro_resolutions: RefCell::new(Vec::new()),
             builtin_attrs: RefCell::new(Vec::new()),
@@ -1214,24 +1216,27 @@ impl<'a> ModuleData<'a> {
             glob_importers: RefCell::new(Vec::new()),
             globs: RefCell::new(Vec::new()),
             traits: RefCell::new(None),
-            populated: Cell::new(normal_ancestor_id.is_local()),
             span,
             expansion,
         }
     }
 
-    fn for_each_child<F: FnMut(Ident, Namespace, &'a NameBinding<'a>)>(&self, mut f: F) {
-        for (&(ident, ns), name_resolution) in self.resolutions.borrow().iter() {
-            name_resolution.borrow().binding.map(|binding| f(ident, ns, binding));
+    fn for_each_child<F>(&'a self, resolver: &mut Resolver<'a>, mut f: F)
+        where F: FnMut(&mut Resolver<'a>, Ident, Namespace, &'a NameBinding<'a>)
+    {
+        for (&(ident, ns), name_resolution) in resolver.resolutions(self).borrow().iter() {
+            name_resolution.borrow().binding.map(|binding| f(resolver, ident, ns, binding));
         }
     }
 
-    fn for_each_child_stable<F: FnMut(Ident, Namespace, &'a NameBinding<'a>)>(&self, mut f: F) {
-        let resolutions = self.resolutions.borrow();
+    fn for_each_child_stable<F>(&'a self, resolver: &mut Resolver<'a>, mut f: F)
+        where F: FnMut(&mut Resolver<'a>, Ident, Namespace, &'a NameBinding<'a>)
+    {
+        let resolutions = resolver.resolutions(self).borrow();
         let mut resolutions = resolutions.iter().collect::<Vec<_>>();
         resolutions.sort_by_cached_key(|&(&(ident, ns), _)| (ident.as_str(), ns));
         for &(&(ident, ns), &resolution) in resolutions.iter() {
-            resolution.borrow().binding.map(|binding| f(ident, ns, binding));
+            resolution.borrow().binding.map(|binding| f(resolver, ident, ns, binding));
         }
     }
 
@@ -4526,7 +4531,7 @@ impl<'a> Resolver<'a> {
         let mut traits = module.traits.borrow_mut();
         if traits.is_none() {
             let mut collected_traits = Vec::new();
-            module.for_each_child(|name, ns, binding| {
+            module.for_each_child(self, |_, name, ns, binding| {
                 if ns != TypeNS { return }
                 match binding.res() {
                     Res::Def(DefKind::Trait, _) |
@@ -5080,7 +5085,6 @@ impl<'a> Resolver<'a> {
                     return None;
                 };
                 let crate_root = self.get_module(DefId { krate: crate_id, index: CRATE_DEF_INDEX });
-                self.populate_module_if_necessary(&crate_root);
                 Some((crate_root, ty::Visibility::Public, DUMMY_SP, ExpnId::root())
                     .to_name_binding(self.arenas))
             }

src/librustc_resolve/resolve_imports.rs

@@ -5,7 +5,7 @@ use crate::{CrateLint, Module, ModuleOrUniformRoot, PerNS, ScopeSet, ParentScope
 use crate::Determinacy::{self, *};
 use crate::Namespace::{self, TypeNS, MacroNS};
 use crate::{NameBinding, NameBindingKind, ToNameBinding, PathResult, PrivacyError};
-use crate::{Resolver, Segment};
+use crate::{Resolutions, Resolver, Segment};
 use crate::{names_to_string, module_to_string};
 use crate::{resolve_error, ResolutionError};
 use crate::ModuleKind;
@@ -156,9 +156,21 @@ impl<'a> NameResolution<'a> {
 }
 
 impl<'a> Resolver<'a> {
-    fn resolution(&self, module: Module<'a>, ident: Ident, ns: Namespace)
+    crate fn resolutions(&mut self, module: Module<'a>) -> &'a Resolutions<'a> {
+        if module.populate_on_access.get() {
+            module.populate_on_access.set(false);
+            let def_id = module.def_id().expect("unpopulated module without a def-id");
+            for child in self.cstore.item_children_untracked(def_id, self.session) {
+                let child = child.map_id(|_| panic!("unexpected id"));
+                self.build_reduced_graph_for_external_crate_res(module, child);
+            }
+        }
+        &module.lazy_resolutions
+    }
+
+    fn resolution(&mut self, module: Module<'a>, ident: Ident, ns: Namespace)
                   -> &'a RefCell<NameResolution<'a>> {
-        *module.resolutions.borrow_mut().entry((ident.modern(), ns))
+        *self.resolutions(module).borrow_mut().entry((ident.modern(), ns))
                .or_insert_with(|| self.arenas.alloc_name_resolution())
     }
 
@@ -239,8 +251,6 @@ impl<'a> Resolver<'a> {
             }
         };
 
-        self.populate_module_if_necessary(module);
-
         let resolution = self.resolution(module, ident, ns)
             .try_borrow_mut()
             .map_err(|_| (Determined, Weak::No))?; // This happens when there is a cycle of imports.
@@ -1079,7 +1089,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
 
             return if all_ns_failed {
                 let resolutions = match module {
-                    ModuleOrUniformRoot::Module(module) => Some(module.resolutions.borrow()),
+                    ModuleOrUniformRoot::Module(module) => Some(self.resolutions(module).borrow()),
                     _ => None,
                 };
                 let resolutions = resolutions.as_ref().into_iter().flat_map(|r| r.iter());
@@ -1317,8 +1327,6 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
             }
         };
 
-        self.populate_module_if_necessary(module);
-
         if module.is_trait() {
             self.session.span_err(directive.span, "items in traits are not importable.");
             return;
@@ -1334,7 +1342,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
 
         // Ensure that `resolutions` isn't borrowed during `try_define`,
         // since it might get updated via a glob cycle.
-        let bindings = module.resolutions.borrow().iter().filter_map(|(&ident, resolution)| {
+        let bindings = self.resolutions(module).borrow().iter().filter_map(|(&ident, resolution)| {
             resolution.borrow().binding().map(|binding| (ident, binding))
         }).collect::<Vec<_>>();
         for ((mut ident, ns), binding) in bindings {
@@ -1361,7 +1369,7 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
 
         let mut reexports = Vec::new();
 
-        for (&(ident, ns), resolution) in module.resolutions.borrow().iter() {
+        for (&(ident, ns), resolution) in self.resolutions(module).borrow().iter() {
             let resolution = &mut *resolution.borrow_mut();
             let binding = match resolution.binding {
                 Some(binding) => binding,
@@ -1420,8 +1428,8 @@ impl<'a, 'b> ImportResolver<'a, 'b> {
                             Some(ModuleOrUniformRoot::Module(module)) => module,
                             _ => bug!("module should exist"),
                         };
-                        let resolutions = imported_module.parent.expect("parent should exist")
-                            .resolutions.borrow();
+                        let parent_module = imported_module.parent.expect("parent should exist");
+                        let resolutions = self.resolutions(parent_module).borrow();
                         let enum_path_segment_index = directive.module_path.len() - 1;
                         let enum_ident = directive.module_path[enum_path_segment_index].ident;