- Address CR comments

- Fixed namespace package resolution in completions (microsoft#12)
- Fixed support for sys.modules (microsoft#363)
- Added several tests (microsoft#365)
- Fixed support for implicit relative module names (microsoft#366)
- Added support for multiple matching rules in PathResolver (microsoft#367)

Not yet implemented:
- Fix imports of binary modules (microsoft#362)
- Fix stubs handling
- AstAnalysisWalker
- More tests
- Code cleanup

Author: AlexanderSher

src/Analysis/Engine/Impl/Analyzer/DDG.cs

@@ -334,46 +334,38 @@ private void SetVariableForImportedMember(IModule module, NameExpression importN
         }
 
         public override bool Walk(FromImportStatement node) {
-            var rootNames = node.Root.Names.Select(n => n.Name);
-            var availableImports = node.Root is RelativeModuleName relativeName
-                ? _pathResolver.GetAvailableImportsFromRelativePath(_filePath, relativeName.DotCount, rootNames)
-                : _pathResolver.GetAvailableImportsFromAbsolutePath(_filePath, rootNames);
-
-            switch (availableImports) {
-                case AvailableModuleImports moduleImports:
-                    ImportMembersFromModule(node, moduleImports);
+            var importSearchResult = _pathResolver.FindImports(_filePath, node);
+
+            switch (importSearchResult) {
+                case ModuleImport moduleImports when TryGetModule(node.Root, moduleImports, out var module):
+                case PossibleModuleImport possibleModuleImport when TryGetModule(node.Root, possibleModuleImport, out module):
+                    ImportMembersFromModule(node, module);
                     return false;
-                case AvailablePackageImports packageImports:
+                case PackageImport packageImports:
                     ImportModulesFromPackage(node, packageImports);
                     return false;
+                case ImportNotFound notFound:
+                    MakeUnresolvedImport(notFound.FullName, node.Root);
+                    return false;
                 default:
-                    _unit.DeclaringModule.AddUnresolvedModule(node.Root.MakeString(), node.ForceAbsolute);
                     return false;
             }
         }
 
-        private void ImportMembersFromModule(FromImportStatement node, AvailableModuleImports moduleImports) {
-            if (!ProjectState.Modules.TryImport(moduleImports.FullName, out var moduleReference)) {
-                return;
-            }
-
-            _unit.DeclaringModule.AddModuleReference(moduleImports.FullName, moduleReference);
-            var module = moduleReference.Module;
-            module.Imported(_unit);
-
-            if (node.Names.Count == 1 && node.Names[0].Name == "*") {
+        private void ImportMembersFromModule(FromImportStatement node, IModule module) {
+            var names = node.Names;
+            if (names.Count == 1 && names[0].Name == "*") {
                 // import all module public members
-                var publicMembers = moduleReference.Module
+                var publicMembers = module
                     .GetModuleMemberNames(GlobalScope.InterpreterContext)
                     .Where(n => !n.StartsWithOrdinal("_"));
                 foreach (var member in publicMembers) {
                     // Don't add references to "*" node
-                    SetVariableForImportedMember(module, node.Names[0], member, null, member, false);
+                    SetVariableForImportedMember(module, names[0], member, null, member, false);
                 }
                 return;
             }
 
-            var names = node.Names;
             var asNames = node.AsNames;
 
             var len = Math.Min(names.Count, asNames.Count);
@@ -390,23 +382,26 @@ private void ImportMembersFromModule(FromImportStatement node, AvailableModuleIm
             }
         }
 
-        private void ImportModulesFromPackage(FromImportStatement node, AvailablePackageImports packageImports) {
+        private void ImportModulesFromPackage(FromImportStatement node, PackageImport packageImport) {
             var names = node.Names;
             var asNames = node.AsNames;
 
             var importNames = names.Select(n => n.Name).ToArray();
             if (importNames.Length == 1 && importNames[0] == "*") {
                 // TODO: Need tracking of previous imports to determine possible imports for namespace package
+                MakeUnresolvedImport(packageImport.Name, node.Root);
                 return;
             }
 
-            foreach (var module in packageImports.Modules) {
+            foreach (var module in packageImport.Modules) {
                 var index = importNames.IndexOf(module.Name, StringExtensions.EqualsOrdinal);
                 if (index == -1) {
+                    MakeUnresolvedImport(module.FullName, node.Root);
                     continue;
                 }
 
                 if (!ProjectState.Modules.TryImport(module.FullName, out var moduleReference)) {
+                    MakeUnresolvedImport(module.FullName, node.Root);
                     continue;
                 }
 
@@ -498,48 +493,108 @@ public override bool Walk(IfStatement node) {
         public override bool Walk(ImportStatement node) {
             var len = Math.Min(node.Names.Count, node.AsNames.Count);
             for (var i = 0; i < len; i++) {
-                var curName = node.Names[i];
-                var asName = node.AsNames[i];
-
-                var imports = _pathResolver.GetAvailableImportsFromAbsolutePath(_filePath, curName.Names.Select(n => n.Name));
-                if (!(imports is AvailableModuleImports moduleImports)) {
-                    continue;
+                var moduleImportExpression = node.Names[i];
+                var asNameExpression = node.AsNames[i];
+
+                var imports = _pathResolver.GetImportsFromAbsoluteName(_filePath, moduleImportExpression.Names.Select(n => n.Name), node.ForceAbsolute);
+                switch (imports) {
+                    case ModuleImport moduleImports when TryGetModule(moduleImportExpression, moduleImports, out var module):
+                    case PossibleModuleImport possibleModuleImport when TryGetModule(moduleImportExpression, possibleModuleImport, out module):
+                        ImportModule(node, module, moduleImportExpression, asNameExpression);
+                        break;
+                    default:
+                        MakeUnresolvedImport(moduleImportExpression.MakeString(), moduleImportExpression);
+                        break;
                 }
-
-                ImportModule(node, moduleImports, curName, asName);
             }
             return true;
         }
 
-        private void ImportModule(ImportStatement node, AvailableModuleImports moduleImports, DottedName moduleImportExpression, NameExpression asNameExpression) {
-            if (!ProjectState.Modules.TryImport(moduleImports.FullName, out var moduleReference)) {
-                return;
-            }
-
-            _unit.DeclaringModule.AddModuleReference(moduleImports.FullName, moduleReference);
-            var module = moduleReference.Module;
-            module.Imported(_unit);
+        private void MakeUnresolvedImport(string name, Node spanNode) {
+            _unit.DeclaringModule.AddUnresolvedModule(name);
+            ProjectState.AddDiagnostic(spanNode, _unit, ErrorMessages.UnresolvedImport(name), DiagnosticSeverity.Warning, ErrorMessages.UnresolvedImportCode);
+        }
 
+        private void ImportModule(in ImportStatement node, in IModule module, in DottedName moduleImportExpression, in NameExpression asNameExpression) {
             // "import fob.oar as baz" is handled as
             // baz = import_module('fob.oar')
             if (asNameExpression != null) {
-                AssignImportedModuleOrMember(asNameExpression.Name, moduleReference.AnalysisModule, true, node.Names.LastOrDefault(), asNameExpression);
+                AssignImportedModuleOrMember(asNameExpression.Name, module, true, node.Names.LastOrDefault(), asNameExpression);
                 return;
             }
 
             // "import fob.oar" is handled as
             // import_module('fob.oar')
             // fob = import_module('fob')
-            var firstImportNameExpression = moduleImportExpression.Names[0];
+            var importNames = moduleImportExpression.Names;
+
+            var existingDepth = 0;
+            PythonPackage pythonPackage = null;
+            if (Scope.TryGetVariable(importNames[0].Name, out var importVariable)) {
+                var childPackage = importVariable.Types.OfType<PythonPackage>().FirstOrDefault();
+                while (childPackage != null && existingDepth < importNames.Count - 1) {
+                    existingDepth++;
+                    pythonPackage = childPackage;
+                    childPackage = pythonPackage.GetChildPackage(null, importNames[existingDepth].Name) as PythonPackage;
+                }
+            }
 
-            // Should be able to just get the module, as we only just imported it
-            if (!ProjectState.Modules.TryGetImportedModule(firstImportNameExpression.Name, out moduleReference)) {
-                Debug.Fail($"Failed to get module {firstImportNameExpression.Name} we just imported");
-                Scope.CreateEphemeralVariable(firstImportNameExpression, _unit, firstImportNameExpression.Name, true);
-                return;
+            var child = module;
+            for (var i = importNames.Count - 2; i >= existingDepth; i--) {
+                var childName = importNames[i + 1].Name;
+                var parentName = importNames[i].Name;
+                var parent = new PythonPackage(parentName, ProjectState);
+                parent.AddChildModule(childName, child);
+                child = parent;
             }
 
-            AssignImportedModuleOrMember(firstImportNameExpression.Name, moduleReference.AnalysisModule, true, firstImportNameExpression, null);
+            if (pythonPackage == null) {
+                AssignImportedModuleOrMember(importNames[0].Name, child, true, importNames[0], null);
+            } else {
+                pythonPackage.AddChildModule(importNames[existingDepth].Name, child);
+            }
+        }
+
+        private bool TryGetModule(Node importNode, ModuleImport moduleImport, out IModule module) {
+            var fullName = moduleImport.FullName;
+            if (!ProjectState.Modules.TryImport(fullName, out var moduleReference)) {
+                MakeUnresolvedImport(fullName, importNode);
+                module = default;
+                return false;
+            }
+
+            _unit.DeclaringModule.AddModuleReference(fullName, moduleReference);
+            module = moduleReference.Module;
+            module.Imported(_unit);
+            return true;
+        }
+
+        private bool TryGetModule(Node importNode, PossibleModuleImport possibleModuleImport, out IModule module) {
+            var fullName = possibleModuleImport.PrecedingModuleFullName;
+            if (!ProjectState.Modules.TryImport(fullName, out var moduleReference)) {
+                MakeUnresolvedImport(fullName, importNode);
+                module = default;
+                return false;
+            }
+
+            _unit.DeclaringModule.AddModuleReference(fullName, moduleReference);
+            module = moduleReference.Module;
+            module.Imported(_unit);
+
+            var nameParts = possibleModuleImport.RemainingNameParts;
+            for (var i = 0; i < nameParts.Count; i++) {
+                var namePart = nameParts[i];
+                if (!module.Scope.TryGetVariable(namePart, out var variable) || !variable.Types.OfType<IModule>().Any()) {
+                    var unresolvedModuleName = string.Join(".", nameParts.Take(i + 1).Prepend(fullName));
+                    MakeUnresolvedImport(unresolvedModuleName, importNode);
+                    return false;
+                }
+
+                module = variable.Types.OfType<IModule>().First();
+                module.Imported(_unit);
+            }
+
+            return true;
         }
 
         public override bool Walk(ReturnStatement node) {

src/Analysis/Engine/Impl/Analyzer/ImportStatementWalker.cs

@@ -18,7 +18,6 @@
 
 namespace Microsoft.PythonTools.Analysis.Analyzer {
     sealed class ModuleScope : InterpreterScope, IModuleScope {
-
         public ModuleScope(ModuleInfo moduleInfo)
             : base(moduleInfo, null) {
         }

src/Analysis/Engine/Impl/Analyzer/ModuleScope.cs

@@ -67,25 +67,6 @@ public interface IPythonAnalyzer: IGroupableAnalysisProject {
 
         AnalysisValue GetAnalysisValueFromObjects(object attr);
 
-        /// <summary>
-        /// Returns true if a module has been imported.
-        /// </summary>
-        /// <param name="importFrom">
-        /// The entry of the module doing the import. If null, the module name
-        /// is resolved as an absolute name.
-        /// </param>
-        /// <param name="relativeModuleName">
-        /// The absolute or relative name of the module. If a relative name is 
-        /// passed here, <paramref name="importFrom"/> must be provided.
-        /// </param>
-        /// <param name="absoluteImports">
-        /// True if Python 2.6/3.x style imports should be used.
-        /// </param>
-        /// <returns>
-        /// True if the module was imported during analysis; otherwise, false.
-        /// </returns>
-        bool IsModuleResolved(IPythonProjectEntry importFrom, string relativeModuleName, bool absoluteImports);
-
         /// <summary>
         /// Gets a top-level list of all the available modules as a list of MemberResults.
         /// </summary>

src/Analysis/Engine/Impl/Definitions/IPythonAnalyzer.cs

@@ -14,16 +14,16 @@
 // See the Apache Version 2.0 License for specific language governing
 // permissions and limitations under the License.
 
-using System;
+using System.Linq;
 using Microsoft.PythonTools.Parsing.Ast;
 
-namespace Microsoft.PythonTools.Analysis.Analyzer {
-    public interface IImportSource {
-        bool IsResolved { get; }
-    }
-
-    public interface IImportDefinition {
-        IImportSource Source { get; }
-        IProjectEntry ProjectEntry { get; }
+namespace Microsoft.PythonTools.Analysis.DependencyResolution {
+    internal static class AstUtilities {
+        public static IImportSearchResult FindImports(this PathResolverSnapshot pathResolver, string modulePath, FromImportStatement fromImportStatement) {
+            var rootNames = fromImportStatement.Root.Names.Select(n => n.Name);
+            return fromImportStatement.Root is RelativeModuleName relativeName
+                ? pathResolver.GetImportsFromRelativePath(modulePath, relativeName.DotCount, rootNames)
+                : pathResolver.GetImportsFromAbsoluteName(modulePath, rootNames, fromImportStatement.ForceAbsolute);
+        }
     }
 }