Lazy types in caching

src/Analysis/Ast/Impl/Analyzer/AnalysisModuleKey.cs

@@ -22,7 +22,7 @@
 
 namespace Microsoft.Python.Analysis.Analyzer {
     [DebuggerDisplay("{Name} : {FilePath}")]
-    internal readonly struct AnalysisModuleKey : IEquatable<AnalysisModuleKey> {
+    public readonly struct AnalysisModuleKey : IEquatable<AnalysisModuleKey> {
         public string Name { get; }
         public string FilePath { get; }
         public bool IsTypeshed { get; }
@@ -34,7 +34,7 @@ public AnalysisModuleKey(IPythonModule module) : this(
             module.IsTypeshed,
             IsNonUserAsDocumentModule(module)) { }
 
-        public AnalysisModuleKey(string name, string filePath, bool isTypeshed)
+        public AnalysisModuleKey(string name, string filePath, bool isTypeshed = false)
             : this(name, filePath, isTypeshed, false) { }
 
         private AnalysisModuleKey(string name, string filePath, bool isTypeshed, bool isNonUserAsDocument) {

src/Analysis/Ast/Impl/Analyzer/CachedAnalysis.cs

@@ -0,0 +1,80 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.Python.Analysis.Analyzer.Evaluation;
+using Microsoft.Python.Analysis.Diagnostics;
+using Microsoft.Python.Analysis.Documents;
+using Microsoft.Python.Analysis.Utilities;
+using Microsoft.Python.Analysis.Values;
+using Microsoft.Python.Core;
+using Microsoft.Python.Core.Diagnostics;
+using Microsoft.Python.Parsing.Ast;
+
+namespace Microsoft.Python.Analysis.Analyzer {
+    /// <summary>
+    /// Analysis of a module restored from database.
+    /// </summary>
+    internal sealed class CachedAnalysis : IDocumentAnalysis {
+        public CachedAnalysis(IDocument document, IServiceContainer services) {
+            Check.ArgumentNotNull(nameof(document), document);
+            Document = document;
+            ExpressionEvaluator = new ExpressionEval(services, document, AstUtilities.MakeEmptyAst(document.Uri));
+        }
+
+        #region IDocumentAnalysis
+        /// <summary>
+        /// Analyzed document.
+        /// </summary>
+        public IDocument Document { get; }
+
+        /// <summary>
+        /// Version of the analysis. Usually matches document version,
+        /// but can be lower when document or its dependencies were
+        /// updated since.
+        /// </summary>
+        public int Version => 0;
+
+        /// <summary>
+        /// Empty AST.
+        /// </summary>
+        public PythonAst Ast => ExpressionEvaluator.Ast;
+
+        /// <summary>
+        /// Document/module global scope.
+        /// </summary>
+        public IGlobalScope GlobalScope => Document.GlobalScope;
+
+        /// <summary>
+        /// Expression evaluator used in the analysis.
+        /// Only supports scope operation since there is no AST
+        /// when library analysis is complete.
+        /// </summary>
+        public IExpressionEvaluator ExpressionEvaluator { get; }
+
+        /// <summary>
+        /// Members of the module which are transferred during a star import. null means __all__ was not defined.
+        /// </summary>
+        public IReadOnlyList<string> StarImportMemberNames => Array.Empty<string>();
+
+        /// <summary>
+        /// Analysis diagnostics.
+        /// </summary>
+        public IEnumerable<DiagnosticsEntry> Diagnostics => Enumerable.Empty<DiagnosticsEntry>();
+        #endregion
+    }
+}

src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs

@@ -21,11 +21,6 @@ namespace Microsoft.Python.Analysis.Analyzer {
     /// Represents document that can be analyzed asynchronously.
     /// </summary>
     internal interface IAnalyzable {
-        /// <summary>
-        /// Returns object that can calculate dependencies of this entry.
-        /// </summary>
-        IDependencyProvider DependencyProvider { get; }
-
         /// <summary>
         /// Notifies document that analysis is about to begin.
         /// </summary>

src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs

@@ -71,12 +71,5 @@ public interface IPythonAnalyzer {
         /// Fires when analysis is complete.
         /// </summary>
         event EventHandler<AnalysisCompleteEventArgs> AnalysisComplete;
-
-        /// <summary>
-        /// Attempts to restore modules analysis from database.
-        /// </summary>
-        /// <param name="module"></param>
-        /// <returns></returns>
-        IDocumentAnalysis TryRestoreCachedAnalysis(IPythonModule module);
     }
 }

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Operators.cs

@@ -89,10 +89,13 @@ private IMember GetValueFromBinaryOp(Expression expr, LookupOptions lookupOption
                 if (op.IsComparison()) {
                     return Interpreter.GetBuiltinType(BuiltinTypeId.Bool);
                 }
-
                 return UnknownType;
             }
 
+            var leftValue = left is IVariable v1 ? v1.Value : left;
+            var rightValue = right is IVariable v2 ? v2.Value : right;
+            var isInstance = leftValue is IPythonInstance || rightValue is IPythonInstance;
+
             var leftType = left.GetPythonType();
             var rightType = right.GetPythonType();
 
@@ -121,7 +124,7 @@ private IMember GetValueFromBinaryOp(Expression expr, LookupOptions lookupOption
 
             if (leftIsSupported && rightIsSupported) {
                 if (TryGetValueFromBuiltinBinaryOp(op, leftTypeId, rightTypeId, Interpreter.LanguageVersion.Is3x(), out var member)) {
-                    return member;
+                    return isInstance ? new PythonInstance(member.GetPythonType()) : member;
                 }
             }
 
@@ -136,11 +139,10 @@ private IMember GetValueFromBinaryOp(Expression expr, LookupOptions lookupOption
                     ret = CallOperator(op, left, leftType, right, rightType, expr, tryLeft: false);
                 }
 
-                if (!ret.IsUnknown()) {
-                    return ret;
+                if (ret.IsUnknown()) {
+                    ret = op.IsComparison() ? Interpreter.GetBuiltinType(BuiltinTypeId.Bool) : left;
                 }
-
-                return op.IsComparison() ? Interpreter.GetBuiltinType(BuiltinTypeId.Bool) : left;
+                return isInstance ? new PythonInstance(ret.GetPythonType()) : ret;
             }
 
             if (rightIsSupported) {

src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs

@@ -163,7 +163,7 @@ public IDisposable OpenScope(IPythonModule module, ScopeStatement node, out Scop
                     // node points to global scope, it is not a function or a class.
                     scope = gs;
                 } else {
-                    scope = outerScope.Children.OfType<Scope>().FirstOrDefault(s => s.Node == node);
+                    scope = outerScope.GetChildScope(node) as Scope;
                     if (scope == null) {
                         scope = new Scope(node, outerScope, Module);
                         outerScope.AddChildScope(scope);

src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs

@@ -15,6 +15,7 @@
 
 using System.Diagnostics;
 using System.Linq;
+using Microsoft.Python.Analysis.Specializations.Typing;
 using Microsoft.Python.Analysis.Types;
 using Microsoft.Python.Analysis.Values;
 using Microsoft.Python.Parsing.Ast;
@@ -29,14 +30,22 @@ public void HandleAssignment(AssignmentStatement node, LookupOptions lookupOptio
             }
 
             // Filter out parenthesis expression in assignment because it makes no difference.
-            var lhs = node.Left.Select(s => s.RemoveParenthesis());
+            var lhs = node.Left.Select(s => s.RemoveParenthesis()).ToArray();
 
             // Note that this is handling assignments of the same value to multiple variables,
             // i.e. with "x = y = z = value", x/y/z are the items in lhs. If an expression looks
             // like "x, y, z = value", then "x, y, z" is a *single* lhs value and its unpacking
             // will be handled by AssignToExpr.
             var value = ExtractRhs(node.Right, lhs.FirstOrDefault(), lookupOptions);
             if (value != null) {
+                // Named tuple may get assigned to variable that has name different from the tuple itself.
+                // Then the name may end up conflicting with other types in module when stub merges with
+                // module types. For example, 'tokenize' stub declares _TokenInfo = NamedTuple('TokenInfo', ...)
+                // but there is also 'class TokenInfo(_TokenInfo)' so we have to use the variable name
+                // in order to avoid type naming conflicts.
+                if (value is ITypingNamedTupleType nt && lhs.Length == 1 && lhs[0] is NameExpression nex) {
+                    nt.SetName(nex.Name);
+                }
                 foreach (var expr in lhs) {
                     AssignToExpr(expr, value);
                 }

src/Analysis/Ast/Impl/Analyzer/Handlers/LoopImportedVariableHandler.cs

@@ -93,7 +93,7 @@ public void EnsureModule(in PythonVariableModule variableModule) {
             EnsureModule(module);
         }
 
-        public void EnsureModule(in IPythonModule module) {
+        private void EnsureModule(in IPythonModule module) {
             if (module == null || _isCanceled()) {
                 return;
             }
@@ -104,15 +104,9 @@ public void EnsureModule(in IPythonModule module) {
         }
 
         public ModuleWalker WalkModule(IPythonModule module, PythonAst ast) {
-            var analyzer = _services.GetService<IPythonAnalyzer>();
-            var analysis = analyzer.TryRestoreCachedAnalysis(module);
-            if (analysis != null) {
-                return null;
-            }
-
             // If module has stub, make sure it is processed too.
-            if (module.Stub?.Analysis is EmptyAnalysis) {
-                WalkModule(module.Stub, module.GetAst());
+            if (module.Stub?.Analysis is EmptyAnalysis && module.Stub.GetAst() != null) {
+                WalkModule(module.Stub, module.Stub.GetAst());
             }
 
             var eval = new ExpressionEval(_services, module, ast);