implement AddImport

src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs

@@ -57,6 +57,12 @@ public override bool Walk(ClassDefinition cd) {
 
             if (!string.IsNullOrEmpty(cd.NameExpression?.Name)) {
                 var classInfo = CreateClass(cd);
+                if (classInfo == null) {
+                    // we can't create class info for this node.
+                    // don't walk down
+                    return false;
+                }
+
                 // The variable is transient (non-user declared) hence it does not have location.
                 // Class type is tracking locations for references and renaming.
                 _eval.DeclareVariable(cd.Name, classInfo, VariableSource.Declaration);
@@ -68,7 +74,9 @@ public override bool Walk(ClassDefinition cd) {
         }
 
         public override void PostWalk(ClassDefinition cd) {
-            if (!IsDeprecated(cd) && !string.IsNullOrEmpty(cd.NameExpression?.Name)) {
+            if (!IsDeprecated(cd) &&
+                !string.IsNullOrEmpty(cd.NameExpression?.Name) &&
+                _typeMap.ContainsKey(cd)) {
                 _scopes.Pop().Dispose();
             }
             base.PostWalk(cd);
@@ -95,9 +103,14 @@ public override void PostWalk(FunctionDefinition fd) {
 
         private PythonClassType CreateClass(ClassDefinition cd) {
             PythonType declaringType = null;
-            if(!(cd.Parent is PythonAst)) {
-                Debug.Assert(_typeMap.ContainsKey(cd.Parent));
-                _typeMap.TryGetValue(cd.Parent, out declaringType);
+            if (!(cd.Parent is PythonAst)) {
+                if (!_typeMap.TryGetValue(cd.Parent, out declaringType)) {
+                    // we can get into this situation if parent is defined twice and we preserve
+                    // only one of them.
+                    // for example, code has function definition with exact same signature
+                    // and class is defined under one of that function
+                    return null;
+                }
             }
             var cls = new PythonClassType(cd, declaringType, _eval.GetLocationOfName(cd),
                 _eval.SuppressBuiltinLookup ? BuiltinTypeId.Unknown : BuiltinTypeId.Type);
@@ -120,6 +133,10 @@ private void AddFunction(FunctionDefinition fd, PythonType declaringType) {
                 f = new PythonFunctionType(fd, declaringType, _eval.GetLocationOfName(fd));
                 // The variable is transient (non-user declared) hence it does not have location.
                 // Function type is tracking locations for references and renaming.
+
+                // if there are multiple functions with same name exist, only the very first one will be
+                // maintained in the scope. we should improve this if possible.
+                // https://github.com/microsoft/python-language-server/issues/1693
                 _eval.DeclareVariable(fd.Name, f, VariableSource.Declaration);
                 _typeMap[fd] = f;
                 declaringType?.AddMember(f.Name, f, overwrite: true);

src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs

@@ -23,6 +23,9 @@
 
 namespace Microsoft.Python.Analysis.Dependencies {
     internal sealed class DependencyResolver<TKey, TValue> : IDependencyResolver<TKey, TValue> {
+        // optimization to only analyze one that is reachable from root
+        private readonly bool _checkVertexReachability = true;
+
         private readonly Dictionary<TKey, int> _keys = new Dictionary<TKey, int>();
         private readonly List<DependencyVertex<TKey, TValue>> _vertices = new List<DependencyVertex<TKey, TValue>>();
         private readonly object _syncObj = new object();
@@ -287,7 +290,7 @@ private bool TryCreateWalkingGraph(in ImmutableArray<DependencyVertex<TKey, TVal
 
             for (var index = 0; index < vertices.Count; index++) {
                 var vertex = vertices[index];
-                if (vertex == null || vertex.IsWalked || depths[index] == -1) {
+                if (vertex == null || vertex.IsWalked || !ReachableFromRoot(depths, index)) {
                     continue;
                 }
 
@@ -311,7 +314,7 @@ private bool TryCreateWalkingGraph(in ImmutableArray<DependencyVertex<TKey, TVal
                 foreach (var outgoingIndex in node.DependencyVertex.Outgoing) {
                     if (!nodesByVertexIndex.TryGetValue(outgoingIndex, out var outgoingNode)) {
                         var vertex = vertices[outgoingIndex];
-                        if (vertex == null || depths[vertex.Index] == -1) {
+                        if (vertex == null || !ReachableFromRoot(depths, vertex.Index)) {
                             continue;
                         }
 
@@ -326,6 +329,14 @@ private bool TryCreateWalkingGraph(in ImmutableArray<DependencyVertex<TKey, TVal
 
             analysisGraph = ImmutableArray<WalkingVertex<TKey, TValue>>.Create(nodesByVertexIndex.Values);
             return true;
+
+            bool ReachableFromRoot(ImmutableArray<int> reachable, int index) {
+                const int inaccessibleFromRoot = -1;
+
+                // one of usage case for this optimization is not analyzing module that is not reachable
+                // from user code
+                return _checkVertexReachability && reachable[index] != inaccessibleFromRoot;
+            }
         }
 
         private static ImmutableArray<int> CalculateDepths(in ImmutableArray<DependencyVertex<TKey, TValue>> vertices) {

src/Analysis/Ast/Impl/Modules/Definitions/IModuleManagement.cs

@@ -14,6 +14,8 @@
 // permissions and limitations under the License.
 
 using System;
+using System.Collections.Generic;
+using System.Threading;
 using Microsoft.Python.Analysis.Caching;
 using Microsoft.Python.Analysis.Core.Interpreter;
 using Microsoft.Python.Analysis.Types;
@@ -86,5 +88,7 @@ public interface IModuleManagement : IModuleResolution {
         ImmutableArray<PythonLibraryPath> LibraryPaths { get; }
 
         bool SetUserConfiguredPaths(ImmutableArray<string> paths);
+
+        IEnumerable<IPythonModule> GetImportedModules(CancellationToken cancellationToken);
     }
 }

src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs

@@ -55,6 +55,20 @@ public MainModuleResolution(string root, IServiceContainer services, ImmutableAr
 
         public IBuiltinsPythonModule BuiltinsModule { get; private set; }
 
+        public IEnumerable<IPythonModule> GetImportedModules(CancellationToken cancellationToken) {
+            foreach (var module in _specialized.Values) {
+                cancellationToken.ThrowIfCancellationRequested();
+                yield return module;
+            }
+
+            foreach (var moduleRef in Modules.Values) {
+                cancellationToken.ThrowIfCancellationRequested();
+                if (moduleRef.Value != null) {
+                    yield return moduleRef.Value;
+                }
+            }
+        }
+
         protected override IPythonModule CreateModule(string name) {
             var moduleImport = CurrentPathResolver.GetModuleImportFromModuleName(name);
             if (moduleImport == null) {

src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs

@@ -81,11 +81,6 @@ public IPythonModule GetOrLoadModule(string name) {
                 return module;
             }
 
-            module = Interpreter.ModuleResolution.GetSpecializedModule(name);
-            if (module != null) {
-                return module;
-            }
-
             // Now try regular case.
             if (Modules.TryGetValue(name, out var moduleRef)) {
                 return moduleRef.GetOrCreate(name, this);

src/Analysis/Core/Impl/DependencyResolution/PathResolverSnapshot.cs

@@ -82,15 +82,27 @@ private PathResolverSnapshot(PythonLanguageVersion pythonLanguageVersion, string
         }
 
         public ImmutableArray<string> GetAllImportableModuleNames(bool includeImplicitPackages = true) {
+            return GetAllImportableModuleInfo(n => !string.IsNullOrEmpty(n.FullModuleName), n => n.FullModuleName, includeImplicitPackages);
+        }
+
+        public ImmutableArray<string> GetAllImportableModulesByName(string name, bool includeImplicitPackages = true) {
+            return GetAllImportableModuleInfo(n => string.Equals(n.Name, name), n => n.FullModuleName, includeImplicitPackages);
+        }
+
+        public ImmutableArray<string> GetAllImportableModuleFilePaths(bool includeImplicitPackages = true) {
+            return GetAllImportableModuleInfo(n => !string.IsNullOrEmpty(n.ModulePath), n => n.ModulePath, includeImplicitPackages);
+        }
+
+        private ImmutableArray<T> GetAllImportableModuleInfo<T>(Func<Node, bool> predicate, Func<Node, T> valueGetter, bool includeImplicitPackages = true) {
             var roots = _roots.Prepend(_nonRooted);
             var items = new Queue<Node>(roots);
-            var names = ImmutableArray<string>.Empty;
+            var stringValues = ImmutableArray<T>.Empty;
 
             while (items.Count > 0) {
                 var item = items.Dequeue();
                 if (item != null) {
-                    if (!string.IsNullOrEmpty(item.FullModuleName) && (item.IsModule || includeImplicitPackages)) {
-                        names = names.Add(item.FullModuleName);
+                    if (predicate(item) && (item.IsModule || includeImplicitPackages)) {
+                        stringValues = stringValues.Add(valueGetter(item));
                     }
 
                     foreach (var child in item.Children.ExcludeDefault()) {
@@ -99,13 +111,17 @@ public ImmutableArray<string> GetAllImportableModuleNames(bool includeImplicitPa
                 }
             }
 
-            return names.AddRange(
+            return stringValues.AddRange(
                 _builtins.Children
-                    .Where(b => !string.IsNullOrEmpty(b.FullModuleName))
-                    .Select(b => b.FullModuleName)
+                    .Where(b => predicate(b))
+                    .Select(b => valueGetter(b))
             );
         }
 
+        public string GetModuleNameByPath(string modulePath) {
+            return TryFindModule(modulePath, out var edge, out _) ? edge.End.FullModuleName : null;
+        }
+
         public ModuleImport GetModuleImportFromModuleName(in string fullModuleName) {
             for (var rootIndex = 0; rootIndex < _roots.Count; rootIndex++) {
                 if (TryFindModuleByName(rootIndex, fullModuleName, out var lastEdge) && TryCreateModuleImport(lastEdge, out var moduleImports)) {

src/Core/Impl/Extensions/ArrayExtensions.cs

@@ -14,6 +14,7 @@
 // permissions and limitations under the License.
 
 using System;
+using System.Collections.Generic;
 
 namespace Microsoft.Python.Core {
     public static class ArrayExtensions {
@@ -36,5 +37,26 @@ public static int IndexOf<T, TValue>(this T[] array, TValue value, Func<T, TValu
 
             return -1;
         }
+
+        public static TCollection AddIfNotNull<TCollection, TItem>(this TCollection list, TItem item)
+            where TCollection : ICollection<TItem>
+            where TItem : class {
+            if (item == null) {
+                return list;
+            }
+
+            list.Add(item);
+            return list;
+        }
+
+        public static TCollection AddIfNotNull<TCollection, TItem>(this TCollection list, params TItem[] items)
+            where TCollection : ICollection<TItem>
+            where TItem : class {
+            foreach (var item in items) {
+                list.AddIfNotNull(item);
+            }
+
+            return list;
+        }
     }
 }

src/Core/Impl/Text/Position.cs

@@ -18,7 +18,7 @@
 
 namespace Microsoft.Python.Core.Text {
     [Serializable]
-    public struct Position {
+    public struct Position : IEquatable<Position> {
         /// <summary>
         /// Line position in a document (zero-based).
         /// </summary>
@@ -39,7 +39,14 @@ public struct Position {
 
         public static bool operator >(Position p1, Position p2) => p1.line > p2.line || p1.line == p2.line && p1.character > p2.character;
         public static bool operator <(Position p1, Position p2) => p1.line < p2.line || p1.line == p2.line && p1.character < p2.character;
+        public static bool operator ==(Position p1, Position p2) => p1.Equals(p2);
+        public static bool operator !=(Position p1, Position p2) => !p1.Equals(p2);
 
+        public bool Equals(Position other) => line == other.line && character == other.character;
+
+        public override bool Equals(object obj) => obj is Position other ? Equals(other) : false;
+
+        public override int GetHashCode() => 0;
         public override string ToString() => $"({line}, {character})";
     }
 }