diff --git a/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs b/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs index d3802e6d7..60cf40216 100644 --- a/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs @@ -69,7 +69,7 @@ public override bool Walk(ExpressionStatement node) { return true; case CallExpression callex when callex.Target is MemberExpression mex && !string.IsNullOrEmpty(mex.Name): var t = Eval.GetValueFromExpression(mex.Target)?.GetPythonType(); - t?.GetMember(mex.Name).AddReference(Eval.GetLocationOfName(mex)); + t?.GetMember(mex.Name)?.AddReference(Eval.GetLocationOfName(mex)); return true; default: return base.Walk(node); diff --git a/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs b/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs index 8d4b9751d..bb1665b8d 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs @@ -26,7 +26,6 @@ internal interface IAnalyzable { /// /// Notifies document that its analysis is now complete. /// - /// Document analysis - void NotifyAnalysisComplete(IDocumentAnalysis analysis); + void NotifyAnalysisComplete(int version, ModuleWalker walker, bool isFinalPass); } } diff --git a/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs index 970c4bb3f..122992414 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs @@ -20,7 +20,6 @@ using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core.Collections; -using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer { public interface IPythonAnalyzer { @@ -28,7 +27,7 @@ public interface IPythonAnalyzer { /// /// Schedules module for analysis. Module will be scheduled if version of AST is greater than the one used to get previous analysis /// - void EnqueueDocumentForAnalysis(IPythonModule module, PythonAst ast, int version); + void EnqueueDocumentForAnalysis(IPythonModule module, int version); /// /// Schedules module for analysis for its existing AST, but with new dependencies. diff --git a/src/Analysis/Ast/Impl/Analyzer/DocumentAnalysis.cs b/src/Analysis/Ast/Impl/Analyzer/DocumentAnalysis.cs index 2d6abf826..51e0cd378 100644 --- a/src/Analysis/Ast/Impl/Analyzer/DocumentAnalysis.cs +++ b/src/Analysis/Ast/Impl/Analyzer/DocumentAnalysis.cs @@ -13,17 +13,11 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.Collections.Generic; -using System.IO; -using System.Linq; -using Microsoft.Python.Analysis.Analyzer.Evaluation; using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Values; -using Microsoft.Python.Core; using Microsoft.Python.Core.Diagnostics; -using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer { @@ -77,24 +71,4 @@ public DocumentAnalysis(IDocument document, int version, IGlobalScope globalScop public IEnumerable Diagnostics => ExpressionEvaluator.Diagnostics; #endregion } - - public sealed class EmptyAnalysis : IDocumentAnalysis { - private static PythonAst _emptyAst; - - public EmptyAnalysis(IServiceContainer services, IDocument document) { - Document = document ?? throw new ArgumentNullException(nameof(document)); - GlobalScope = new EmptyGlobalScope(document); - - _emptyAst = _emptyAst ?? (_emptyAst = Parser.CreateParser(new StringReader(string.Empty), PythonLanguageVersion.None).ParseFile(document.Uri)); - ExpressionEvaluator = new ExpressionEval(services, document, Ast); - } - - public IDocument Document { get; } - public int Version { get; } = -1; - public IGlobalScope GlobalScope { get; } - public PythonAst Ast => _emptyAst; - public IExpressionEvaluator ExpressionEvaluator { get; } - public IReadOnlyList StarImportMemberNames => null; - public IEnumerable Diagnostics => Enumerable.Empty(); - } } diff --git a/src/Analysis/Ast/Impl/Analyzer/EmptyAnalysis.cs b/src/Analysis/Ast/Impl/Analyzer/EmptyAnalysis.cs new file mode 100644 index 000000000..b073f20b2 --- /dev/null +++ b/src/Analysis/Ast/Impl/Analyzer/EmptyAnalysis.cs @@ -0,0 +1,44 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Analysis.Analyzer.Evaluation; +using Microsoft.Python.Analysis.Diagnostics; +using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Analysis.Utilities; +using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Analyzer { + public sealed class EmptyAnalysis : IDocumentAnalysis { + public EmptyAnalysis(IServiceContainer services, IDocument document) { + Document = document ?? throw new ArgumentNullException(nameof(document)); + GlobalScope = new EmptyGlobalScope(document); + Ast = AstUtilities.MakeEmptyAst(document.Uri); + ExpressionEvaluator = new ExpressionEval(services, document); + } + + public IDocument Document { get; } + public int Version { get; } = -1; + public IGlobalScope GlobalScope { get; } + public PythonAst Ast { get; } + public IExpressionEvaluator ExpressionEvaluator { get; } + public IReadOnlyList StarImportMemberNames => null; + public IEnumerable Diagnostics => Enumerable.Empty(); + } +} diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Annotations.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Annotations.cs new file mode 100644 index 000000000..a5973ab5c --- /dev/null +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Annotations.cs @@ -0,0 +1,48 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Analyzer.Evaluation { + internal sealed partial class ExpressionEval { + public IPythonType GetTypeFromAnnotation(Expression expr, LookupOptions options = LookupOptions.Global | LookupOptions.Builtins) + => GetTypeFromAnnotation(expr, out _, options); + + public IPythonType GetTypeFromAnnotation(Expression expr, out bool isGeneric, LookupOptions options = LookupOptions.Global | LookupOptions.Builtins) { + isGeneric = false; + switch (expr) { + case null: + return null; + case CallExpression callExpr: + // x: NamedTuple(...) + return GetValueFromCallable(callExpr)?.GetPythonType() ?? UnknownType; + case IndexExpression indexExpr: + // Try generics + var target = GetValueFromExpression(indexExpr.Target); + var result = GetValueFromGeneric(target, indexExpr); + if (result != null) { + isGeneric = true; + return result.GetPythonType(); + } + break; + } + + // Look at specialization and typing first + var ann = new TypeAnnotation(Ast.LanguageVersion, expr); + return ann.GetValue(new TypeAnnotationConverter(this, options)); + } + } +} diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs index f2e79b8a5..38346a6e0 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs @@ -17,6 +17,7 @@ using System.Diagnostics; using System.Linq; using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Analysis.Extensions; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; @@ -77,9 +78,11 @@ public IMember GetValueFromLambda(LambdaExpression expr) { return null; } - var location = GetLocationOfName(expr.Function); - var ft = new PythonFunctionType(expr.Function, null, location); - var overload = new PythonFunctionOverload(expr.Function, ft, location); + var fd = expr.Function; + var location = GetLocationOfName(fd); + var ft = new PythonFunctionType(fd, null, location); + var overload = new PythonFunctionOverload(ft, fd, location, expr.Function.ReturnAnnotation?.ToCodeString(Ast)); + overload.SetParameters(CreateFunctionParameters(null, ft, fd, false)); ft.AddOverload(overload); return ft; } @@ -101,13 +104,13 @@ public IMember GetValueFromClassCtor(IPythonClassType cls, CallExpression expr) return cls.CreateInstance(cls.Name, args); } - public IMember GetValueFromBound(IPythonBoundType t, CallExpression expr) { + private IMember GetValueFromBound(IPythonBoundType t, CallExpression expr) { switch (t.Type) { case IPythonFunctionType fn: return GetValueFromFunctionType(fn, t.Self, expr); case IPythonPropertyType p: return GetValueFromProperty(p, t.Self); - case IPythonIteratorType it when t.Self is IPythonCollection seq: + case IPythonIteratorType _ when t.Self is IPythonCollection seq: return seq.GetIterator(); } return UnknownType; @@ -173,7 +176,7 @@ public IMember GetValueFromFunctionType(IPythonFunctionType fn, IPythonInstance if (instanceType == null || fn.DeclaringType == null || fn.IsSpecialized || instanceType.IsSpecialized || fn.DeclaringType.IsSpecialized || instanceType.Equals(fn.DeclaringType) || - fn.IsStub || !string.IsNullOrEmpty(fn.Overloads[args.OverloadIndex].GetReturnDocumentation(null))) { + fn.IsStub || !string.IsNullOrEmpty(fn.Overloads[args.OverloadIndex].GetReturnDocumentation())) { LoadFunctionDependencyModules(fn); @@ -192,14 +195,14 @@ public IMember GetValueFromFunctionType(IPythonFunctionType fn, IPythonInstance // def func(a, b): return a + b // from working in libraries, but this is small sacrifice for significant performance // increase in library analysis. - if (fn.DeclaringModule is IDocument doc && fd?.Ast == doc.GetAnyAst() && EvaluateFunctionBody(fn)) { + if (fn.DeclaringModule is IDocument && EvaluateFunctionBody(fn)) { // Stubs are coming from another module. return TryEvaluateWithArguments(fn, args); } return UnknownType; } - public IMember GetValueFromProperty(IPythonPropertyType p, IPythonInstance instance) { + private IMember GetValueFromProperty(IPythonPropertyType p, IPythonInstance instance) { // Function may not have been walked yet. Do it now. SymbolTable.Evaluate(p.FunctionDefinition); return instance.Call(p.Name, ArgumentSet.Empty); @@ -311,5 +314,69 @@ private void LoadFunctionDependencyModules(IPythonFunctionType fn) { Services.GetService().EnqueueDocumentForAnalysis(Module, dependencies); } } + + public IReadOnlyList CreateFunctionParameters(IPythonClassType self, IPythonClassMember function, FunctionDefinition fd, bool declareVariables) { + // For class method no need to add extra parameters, but first parameter type should be the class. + // For static and unbound methods do not add or set anything. + // For regular bound methods add first parameter and set it to the class. + + var parameters = new List(); + var skip = 0; + if (self != null && function.HasClassFirstArgument()) { + var p0 = fd.Parameters.FirstOrDefault(); + if (p0 != null && !string.IsNullOrEmpty(p0.Name)) { + // Actual parameter type will be determined when method is invoked. + // The reason is that if method might be called on a derived class. + // Declare self or cls in this scope. + if (declareVariables) { + DeclareVariable(p0.Name, new PythonInstance(self), VariableSource.Declaration, p0.NameExpression); + } + // Set parameter info. + var pi = new ParameterInfo(Ast, p0, self, null, false); + parameters.Add(pi); + skip++; + } + } + + // Declare parameters in scope + for (var i = skip; i < fd.Parameters.Length; i++) { + var p = fd.Parameters[i]; + if (!string.IsNullOrEmpty(p.Name)) { + var defaultValue = GetValueFromExpression(p.DefaultValue); + var paramType = GetTypeFromAnnotation(p.Annotation, out var isGeneric) ?? UnknownType; + if (paramType.IsUnknown()) { + // If parameter has default value, look for the annotation locally first + // since outer type may be getting redefined. Consider 's = None; def f(s: s = 123): ... + paramType = GetTypeFromAnnotation(p.Annotation, out isGeneric, LookupOptions.Local | LookupOptions.Builtins); + // Default value of None does not mean the parameter is None, just says it can be missing. + defaultValue = defaultValue.IsUnknown() || defaultValue.IsOfType(BuiltinTypeId.NoneType) ? null : defaultValue; + if (paramType == null && defaultValue != null) { + paramType = defaultValue.GetPythonType(); + } + } + // If all else fails, look up globally. + var pi = new ParameterInfo(Ast, p, paramType, defaultValue, isGeneric | paramType.IsGeneric()); + if (declareVariables) { + DeclareParameter(p, pi); + } + parameters.Add(pi); + } else if (p.IsList || p.IsDictionary) { + parameters.Add(new ParameterInfo(Ast, p, null, null, false)); + } + } + return parameters; + } + + private void DeclareParameter(Parameter p, ParameterInfo pi) { + IPythonType paramType; + // If type is known from annotation, use it. + if (pi != null && !pi.Type.IsUnknown() && !pi.Type.IsGenericParameter()) { + // TODO: technically generics may have constraints. Should we consider them? + paramType = pi.Type; + } else { + paramType = pi?.DefaultValue?.GetPythonType() ?? UnknownType; + } + DeclareVariable(p.Name, new PythonInstance(paramType), VariableSource.Declaration, p.NameExpression); + } } } diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs index a2060a301..c5baaa8ec 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs @@ -117,33 +117,6 @@ public IMember LookupNameInScopes(string name, out IScope scope, out IVariable v return value; } - public IPythonType GetTypeFromAnnotation(Expression expr, LookupOptions options = LookupOptions.Global | LookupOptions.Builtins) - => GetTypeFromAnnotation(expr, out _, options); - - public IPythonType GetTypeFromAnnotation(Expression expr, out bool isGeneric, LookupOptions options = LookupOptions.Global | LookupOptions.Builtins) { - isGeneric = false; - switch (expr) { - case null: - return null; - case CallExpression callExpr: - // x: NamedTuple(...) - return GetValueFromCallable(callExpr)?.GetPythonType() ?? UnknownType; - case IndexExpression indexExpr: - // Try generics - var target = GetValueFromExpression(indexExpr.Target); - var result = GetValueFromGeneric(target, indexExpr); - if (result != null) { - isGeneric = true; - return result.GetPythonType(); - } - break; - } - - // Look at specialization and typing first - var ann = new TypeAnnotation(Ast.LanguageVersion, expr); - return ann.GetValue(new TypeAnnotationConverter(this, options)); - } - /// /// Locates and opens existing scope for a node or creates a new scope /// as a child of the specified scope. Scope is pushed on the stack diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs index 0a637412c..e094e18e5 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs @@ -20,6 +20,7 @@ using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Utilities; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; using Microsoft.Python.Core.Disposables; @@ -37,12 +38,15 @@ internal sealed partial class ExpressionEval : IExpressionEvaluator { private readonly object _lock = new object(); private readonly List _diagnostics = new List(); - public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAst ast) { + public ExpressionEval(IServiceContainer services, IPythonModule module) + : this(services, module, new GlobalScope(module)) { } + + public ExpressionEval(IServiceContainer services, IPythonModule module, GlobalScope gs) { Services = services ?? throw new ArgumentNullException(nameof(services)); Module = module ?? throw new ArgumentNullException(nameof(module)); - Ast = ast ?? throw new ArgumentNullException(nameof(ast)); + Ast = module.GetAst(); - GlobalScope = new GlobalScope(module); + GlobalScope = gs; CurrentScope = GlobalScope; DefaultLocation = new Location(module); //Log = services.GetService(); @@ -59,14 +63,15 @@ public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAs public LocationInfo GetLocationInfo(Node node) => node?.GetLocation(Module) ?? LocationInfo.Empty; public Location GetLocationOfName(Node node) { - if (node == null || (Module.ModuleType != ModuleType.User && Module.ModuleType != ModuleType.Library)) { + node = node ?? throw new ArgumentNullException(nameof(node)); + if (Module.ModuleType != ModuleType.User && Module.ModuleType != ModuleType.Library) { return DefaultLocation; } IndexSpan indexSpan; switch (node) { case MemberExpression mex: - indexSpan = mex.GetNameSpan().ToIndexSpan(mex.Ast); + indexSpan = mex.GetNameSpan(Ast).ToIndexSpan(Ast); break; case ClassDefinition cd: indexSpan = cd.NameExpression.IndexSpan; @@ -86,7 +91,7 @@ public Location GetLocationOfName(Node node) { // turns into span at line 1 and very large column. This DOES can // produce false positives occasionally. #if DEBUG - var sourceSpan = indexSpan.ToSourceSpan(node.Ast); + var sourceSpan = indexSpan.ToSourceSpan(Ast); Debug.Assert(sourceSpan.Start.Line > 1 || sourceSpan.Start.Column < 1000); #endif return new Location(Module, indexSpan); diff --git a/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs b/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs new file mode 100644 index 000000000..42caa54da --- /dev/null +++ b/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs @@ -0,0 +1,92 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Analysis.Analyzer.Evaluation; +using Microsoft.Python.Analysis.Diagnostics; +using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Analysis.Modules; +using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core; +using Microsoft.Python.Core.Diagnostics; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Analyzer { + /// + /// Analysis of a library code. + /// + internal sealed class LibraryAnalysis : IDocumentAnalysis { + public LibraryAnalysis(IDocument document, int version, IServiceContainer services, GlobalScope globalScope, IReadOnlyList starImportMemberNames) { + Check.ArgumentNotNull(nameof(document), document); + Check.ArgumentNotNull(nameof(globalScope), globalScope); + + Document = document; + Version = version; + GlobalScope = globalScope; + + var ast = Document.GetAst(); + ast.Reduce(x => x is ImportStatement || x is FromImportStatement); + var c = (IAstNodeContainer)Document; + c.ClearContent(); + c.ClearAst(); + c.AddAstNode(document, ast); + + ExpressionEvaluator = new ExpressionEval(services, document, globalScope); + StarImportMemberNames = starImportMemberNames; + } + + #region IDocumentAnalysis + /// + /// Analyzed document. + /// + public IDocument Document { get; } + + /// + /// Version of the analysis. Usually matches document version, + /// but can be lower when document or its dependencies were + /// updated since. + /// + public int Version { get; } + + /// + /// Empty AST. + /// + public PythonAst Ast => ExpressionEvaluator.Ast; + + /// + /// Document/module global scope. + /// + public IGlobalScope GlobalScope { get; } + + /// + /// Expression evaluator used in the analysis. + /// Only supports scope operation since there is no AST + /// when library analysis is complete. + /// + public IExpressionEvaluator ExpressionEvaluator { get; } + + /// + /// Members of the module which are transferred during a star import. null means __all__ was not defined. + /// + public IReadOnlyList StarImportMemberNames { get; } + + /// + /// Analysis diagnostics. + /// + public IEnumerable Diagnostics => Enumerable.Empty(); + #endregion + } +} diff --git a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs index d3808c35a..2463dfc03 100644 --- a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs @@ -37,8 +37,8 @@ internal class ModuleWalker : AnalysisWalker { private int _allReferencesCount; private bool _allIsUsable = true; - public ModuleWalker(IServiceContainer services, IPythonModule module, PythonAst ast) - : base(new ExpressionEval(services, module, ast)) { + public ModuleWalker(IServiceContainer services, IPythonModule module) + : base(new ExpressionEval(services, module)) { _stubAnalysis = Module.Stub is IDocument doc ? doc.GetAnyAnalysis() : null; } @@ -207,7 +207,7 @@ public void Complete() { Eval.ClearCache(); } - public IGlobalScope GlobalScope => Eval.GlobalScope; + public GlobalScope GlobalScope => Eval.GlobalScope; public IReadOnlyList StarImportMemberNames { get; private set; } /// @@ -259,7 +259,7 @@ private void MergeStub() { var memberType = member?.GetPythonType(); var stubMemberType = stubMember.GetPythonType(); if (!IsStubBetterType(memberType, stubMemberType)) { - continue; ; + continue; } // Get documentation from the current type, if any, since stubs @@ -271,11 +271,11 @@ private void MergeStub() { // Re-declare variable with the data from the stub unless member is a module. // Modules members that are modules should remain as they are, i.e. os.path // should remain library with its own stub attached. - if (!stubType.IsUnknown() && !(stubType is IPythonModule)) { + if (!(stubType is IPythonModule)) { sourceType.TransferDocumentationAndLocation(stubType); // TODO: choose best type between the scrape and the stub. Stub probably should always win. var source = Eval.CurrentScope.Variables[v.Name]?.Source ?? VariableSource.Declaration; - Eval.DeclareVariable(v.Name, v.Value, source, Module); + Eval.DeclareVariable(v.Name, v.Value, source); } } } diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index f869680d7..0f3c037fc 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -31,7 +31,6 @@ using Microsoft.Python.Core.Disposables; using Microsoft.Python.Core.Logging; using Microsoft.Python.Core.Services; -using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer { public sealed class PythonAnalyzer : IPythonAnalyzer, IDisposable { @@ -65,9 +64,8 @@ public void Dispose() { _disposeToken.TryMarkDisposed(); } - public Task WaitForCompleteAnalysisAsync(CancellationToken cancellationToken = default) { - return _analysisCompleteEvent.WaitAsync(cancellationToken); - } + public Task WaitForCompleteAnalysisAsync(CancellationToken cancellationToken = default) + => _analysisCompleteEvent.WaitAsync(cancellationToken); public async Task GetAnalysisAsync(IPythonModule module, int waitTime, CancellationToken cancellationToken) { var key = new AnalysisModuleKey(module); @@ -146,7 +144,7 @@ public void EnqueueDocumentForAnalysis(IPythonModule module, ImmutableArray _analysisTcs; private IPythonModule _module; private bool _isUserModule; - private PythonAst _ast; private IDocumentAnalysis _previousAnalysis; private HashSet _parserDependencies; private HashSet _analysisDependencies; @@ -102,7 +101,7 @@ public Task GetAnalysisAsync(CancellationToken cancellationTo public bool IsValidVersion(int version, out IPythonModule module, out PythonAst ast) { lock (_syncObj) { module = _module; - ast = _ast; + ast = module.GetAst(); if (ast == null || module == null) { return false; } @@ -210,19 +209,18 @@ public bool Invalidate(ImmutableArray analysisDependencies, int a } } - public bool Invalidate(IPythonModule module, PythonAst ast, int bufferVersion, int analysisVersion, out ImmutableArray dependencies) { + public bool Invalidate(IPythonModule module, int bufferVersion, int analysisVersion, out ImmutableArray dependencies) { dependencies = ImmutableArray.Empty; if (_bufferVersion >= bufferVersion) { return false; } - var dependenciesHashSet = FindDependencies(module, ast, bufferVersion); + var dependenciesHashSet = FindDependencies(module, bufferVersion); lock (_syncObj) { if (_analysisVersion >= analysisVersion && _bufferVersion >= bufferVersion) { return false; } - _ast = ast; _module = module; _isUserModule = module.ModuleType == ModuleType.User; _parserDependencies = dependenciesHashSet; @@ -236,13 +234,13 @@ public bool Invalidate(IPythonModule module, PythonAst ast, int bufferVersion, i } } - private HashSet FindDependencies(IPythonModule module, PythonAst ast, int bufferVersion) { + private HashSet FindDependencies(IPythonModule module, int bufferVersion) { if (_bufferVersion > bufferVersion) { return new HashSet(); } - var walker = new DependencyWalker(this, module); - ast.Walk(walker); + var walker = new DependencyWalker(module); + module.GetAst().Walk(walker); var dependencies = walker.Dependencies; dependencies.Remove(new AnalysisModuleKey(module)); return dependencies; @@ -264,7 +262,6 @@ private void UpdateAnalysisTcs(int analysisVersion) { } private class DependencyWalker : PythonWalker { - private readonly PythonAnalyzerEntry _entry; private readonly IPythonModule _module; private readonly bool _isTypeshed; private readonly IModuleManagement _moduleResolution; @@ -272,8 +269,7 @@ private class DependencyWalker : PythonWalker { public HashSet Dependencies { get; } - public DependencyWalker(PythonAnalyzerEntry entry, IPythonModule module) { - _entry = entry; + public DependencyWalker(IPythonModule module) { _module = module; _isTypeshed = module is StubPythonModule stub && stub.IsTypeshed; _moduleResolution = module.Interpreter.ModuleResolution; diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index 8ea38f6ea..c0e9215f0 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -21,13 +21,11 @@ using System.Threading.Tasks; using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Diagnostics; -using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core; using Microsoft.Python.Core.Logging; using Microsoft.Python.Core.Services; -using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer { internal sealed class PythonAnalyzerSession { @@ -147,7 +145,7 @@ private async Task StartAsync() { if (!isCanceled) { _progress.ReportRemaining(remaining); - if(isFinal) { + if (isFinal) { ActivityTracker.EndTracking(); (_analyzer as PythonAnalyzer)?.RaiseAnalysisComplete(ActivityTracker.ModuleCount, ActivityTracker.MillisecondsElapsed); _log?.Log(TraceEventType.Verbose, $"Analysis complete: {ActivityTracker.ModuleCount} modules in { ActivityTracker.MillisecondsElapsed} ms."); @@ -172,7 +170,7 @@ private static void LogResults(ILogger logger, double elapsed, int originalRemai if (logger == null) { return; } - + if (remaining == 0) { logger.Log(TraceEventType.Verbose, $"Analysis version {version} of {originalRemaining} entries has been completed in {elapsed} ms."); } else if (remaining < originalRemaining) { @@ -237,11 +235,11 @@ private Task StartAnalysis(IDependencyChainNode node, Async /// of dependencies, it is intended for the single file analysis. /// private void Analyze(IDependencyChainNode node, AsyncCountdownEvent ace, Stopwatch stopWatch) { - IPythonModule module; try { ace?.AddOne(); var entry = node.Value; - if (!entry.IsValidVersion(_walker.Version, out module, out var ast)) { + + if (!entry.IsValidVersion(_walker.Version, out var module, out var ast)) { if (ast == null) { // Entry doesn't have ast yet. There should be at least one more session. Cancel(); @@ -251,8 +249,9 @@ private void Analyze(IDependencyChainNode node, AsyncCountd node.Skip(); return; } + var startTime = stopWatch.Elapsed; - AnalyzeEntry(entry, module, ast, _walker.Version); + AnalyzeEntry(entry, module, _walker.Version, node.IsComplete); node.Commit(); ActivityTracker.OnModuleAnalysisComplete(node.Value.Module.FilePath); @@ -288,13 +287,13 @@ private void AnalyzeEntry() { // Entry doesn't have ast yet. There should be at least one more session. Cancel(); } - _log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled."); return; } var startTime = stopWatch?.Elapsed ?? TimeSpan.Zero; - AnalyzeEntry(_entry, module, ast, Version); + + AnalyzeEntry(_entry, module, Version, true); LogCompleted(module, stopWatch, startTime); } catch (OperationCanceledException oce) { @@ -309,22 +308,26 @@ private void AnalyzeEntry() { } } - private void AnalyzeEntry(PythonAnalyzerEntry entry, IPythonModule module, PythonAst ast, int version) { + private void AnalyzeEntry(PythonAnalyzerEntry entry, IPythonModule module, int version, bool isFinalPass) { + if (entry.PreviousAnalysis is LibraryAnalysis) { + _log?.Log(TraceEventType.Verbose, $"Request to re-analyze finalized {module.Name}."); + } + // Now run the analysis. var analyzable = module as IAnalyzable; analyzable?.NotifyAnalysisBegins(); - var walker = new ModuleWalker(_services, module, ast); + var ast = module.GetAst(); + var walker = new ModuleWalker(_services, module); ast.Walk(walker); _analyzerCancellationToken.ThrowIfCancellationRequested(); walker.Complete(); _analyzerCancellationToken.ThrowIfCancellationRequested(); - var analysis = new DocumentAnalysis((IDocument)module, version, walker.GlobalScope, walker.Eval, walker.StarImportMemberNames); - analyzable?.NotifyAnalysisComplete(analysis); - entry.TrySetAnalysis(analysis, version); + analyzable?.NotifyAnalysisComplete(version, walker, isFinalPass); + entry.TrySetAnalysis(module.Analysis, version); if (module.ModuleType == ModuleType.User) { var linterDiagnostics = _analyzer.LintModule(module); diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs index cd3ecc4ae..847bbc000 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs @@ -43,7 +43,7 @@ public FunctionEvaluator(ExpressionEval eval, PythonFunctionOverload overload) FunctionDefinition = overload.FunctionDefinition; } - public FunctionDefinition FunctionDefinition { get; } + private FunctionDefinition FunctionDefinition { get; } public override void Evaluate() { var stub = SymbolTable.ReplacedByStubs.Contains(Target) @@ -52,7 +52,9 @@ public override void Evaluate() { using (Eval.OpenScope(_function.DeclaringModule, FunctionDefinition, out _)) { var returnType = TryDetermineReturnValue(); - DeclareParameters(!stub); + + var parameters = Eval.CreateFunctionParameters(_self, _function, FunctionDefinition, !stub); + _overload.SetParameters(parameters); // Do process body of constructors since they may be declaring // variables that are later used to determine return type of other @@ -71,6 +73,30 @@ public override void Evaluate() { Result = _function; } + private IPythonType TryDetermineReturnValue() { + var annotationType = Eval.GetTypeFromAnnotation(FunctionDefinition.ReturnAnnotation); + if (!annotationType.IsUnknown()) { + // Annotations are typically types while actually functions return + // instances unless specifically annotated to a type such as Type[T]. + var t = annotationType.CreateInstance(annotationType.Name, ArgumentSet.Empty); + // If instance could not be created, such as when return type is List[T] and + // type of T is not yet known, just use the type. + var instance = t.IsUnknown() ? annotationType : t; + _overload.SetReturnValue(instance, true); _overload.SetReturnValue(instance, true); + } else { + // Check if function is a generator + var suite = FunctionDefinition.Body as SuiteStatement; + var yieldExpr = suite?.Statements.OfType().Select(s => s.Expression as YieldExpression).ExcludeDefault().FirstOrDefault(); + if (yieldExpr != null) { + // Function return is an iterator + var yieldValue = Eval.GetValueFromExpression(yieldExpr.Expression) ?? Eval.UnknownType; + var returnValue = new PythonGenerator(Eval.Interpreter, yieldValue); + _overload.SetReturnValue(returnValue, true); + } + } + return annotationType; + } + public override bool Walk(AssignmentStatement node) { var value = Eval.GetValueFromExpression(node.Right) ?? Eval.UnknownType; foreach (var lhs in node.Left) { @@ -107,93 +133,5 @@ public override bool Walk(FunctionDefinition node) { } return false; } - - private IPythonType TryDetermineReturnValue() { - var annotationType = Eval.GetTypeFromAnnotation(FunctionDefinition.ReturnAnnotation); - if (!annotationType.IsUnknown()) { - // Annotations are typically types while actually functions return - // instances unless specifically annotated to a type such as Type[T]. - var t = annotationType.CreateInstance(annotationType.Name, ArgumentSet.Empty); - // If instance could not be created, such as when return type is List[T] and - // type of T is not yet known, just use the type. - var instance = t.IsUnknown() ? annotationType : t; - _overload.SetReturnValue(instance, true); - } else { - // Check if function is a generator - var suite = FunctionDefinition.Body as SuiteStatement; - var yieldExpr = suite?.Statements.OfType().Select(s => s.Expression as YieldExpression).ExcludeDefault().FirstOrDefault(); - if (yieldExpr != null) { - // Function return is an iterator - var yieldValue = Eval.GetValueFromExpression(yieldExpr.Expression) ?? Eval.UnknownType; - var returnValue = new PythonGenerator(Eval.Interpreter, yieldValue); - _overload.SetReturnValue(returnValue, true); - } - } - return annotationType; - } - - private void DeclareParameters(bool declareVariables) { - // For class method no need to add extra parameters, but first parameter type should be the class. - // For static and unbound methods do not add or set anything. - // For regular bound methods add first parameter and set it to the class. - - var parameters = new List(); - var skip = 0; - if (_self != null && _function.HasClassFirstArgument()) { - var p0 = FunctionDefinition.Parameters.FirstOrDefault(); - if (p0 != null && !string.IsNullOrEmpty(p0.Name)) { - // Actual parameter type will be determined when method is invoked. - // The reason is that if method might be called on a derived class. - // Declare self or cls in this scope. - if (declareVariables) { - Eval.DeclareVariable(p0.Name, new PythonInstance(_self), VariableSource.Declaration, p0.NameExpression); - } - // Set parameter info. - var pi = new ParameterInfo(Ast, p0, _self, null, false); - parameters.Add(pi); - skip++; - } - } - - // Declare parameters in scope - IMember defaultValue = null; - for (var i = skip; i < FunctionDefinition.Parameters.Length; i++) { - var p = FunctionDefinition.Parameters[i]; - if (!string.IsNullOrEmpty(p.Name)) { - var paramType = Eval.GetTypeFromAnnotation(p.Annotation, out var isGeneric); - if (paramType.IsUnknown() && p.DefaultValue != null) { - defaultValue = Eval.GetValueFromExpression(p.DefaultValue); - // If parameter has default value, look for the annotation locally first - // since outer type may be getting redefined. Consider 's = None; def f(s: s = 123): ... - paramType = Eval.GetTypeFromAnnotation(p.Annotation, out isGeneric, LookupOptions.Local | LookupOptions.Builtins); - // Default value of None does not mean the parameter is None, just says it can be missing. - defaultValue = defaultValue.IsUnknown() || defaultValue.IsOfType(BuiltinTypeId.NoneType) ? null : defaultValue; - if (paramType == null && defaultValue != null) { - paramType = defaultValue.GetPythonType(); - } - } - // If all else fails, look up globally. - paramType = paramType ?? Eval.GetTypeFromAnnotation(p.Annotation, out isGeneric) ?? Eval.UnknownType; - var pi = new ParameterInfo(Ast, p, paramType, defaultValue, isGeneric | paramType.IsGeneric()); - if (declareVariables) { - DeclareParameter(p, pi); - } - parameters.Add(pi); - } - } - _overload.SetParameters(parameters); - } - - private void DeclareParameter(Parameter p, ParameterInfo pi) { - IPythonType paramType; - // If type is known from annotation, use it. - if (pi != null && !pi.Type.IsUnknown() && !pi.Type.IsGenericParameter()) { - // TODO: technically generics may have constraints. Should we consider them? - paramType = pi.Type; - } else { - paramType = pi?.DefaultValue?.GetPythonType() ?? Eval.UnknownType; - } - Eval.DeclareVariable(p.Name, new PythonInstance(paramType), VariableSource.Declaration, p.NameExpression); - } } } diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/ModuleSymbolTable.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/ModuleSymbolTable.cs index ef4848991..31a3b4212 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/ModuleSymbolTable.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/ModuleSymbolTable.cs @@ -34,7 +34,6 @@ internal sealed class ModuleSymbolTable { public IEnumerable> Evaluators => _evaluators.ToArray(); public void Add(MemberEvaluator e) => _evaluators[e.Target] = e; - public MemberEvaluator Get(ScopeStatement target) => _evaluators.TryGetValue(target, out var w) ? w : null; public bool Contains(ScopeStatement node) => _evaluators.ContainsKey(node) || _processed.Contains(node); public void Build(ExpressionEval eval) diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs index f09974a28..fe45a8fb9 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs @@ -73,7 +73,7 @@ public override bool Walk(FunctionDefinition fd) { if (IsDeprecated(fd)) { return false; } - if (!string.IsNullOrEmpty(fd.NameExpression?.Name)) { + if (!string.IsNullOrEmpty(fd.Name)) { AddFunctionOrProperty(fd); // Open function scope _scopes.Push(_eval.OpenScope(_eval.Module, fd, out _)); @@ -82,7 +82,7 @@ public override bool Walk(FunctionDefinition fd) { } public override void PostWalk(FunctionDefinition fd) { - if (!IsDeprecated(fd) && !string.IsNullOrEmpty(fd.NameExpression?.Name)) { + if (!IsDeprecated(fd) && !string.IsNullOrEmpty(fd.Name)) { _scopes.Pop().Dispose(); } base.PostWalk(fd); @@ -119,8 +119,9 @@ private void AddOverload(FunctionDefinition fd, IPythonClassMember function, Act if (!_table.ReplacedByStubs.Contains(fd)) { var stubOverload = GetOverloadFromStub(fd); if (stubOverload != null) { - if (!string.IsNullOrEmpty(fd.GetDocumentation())) { - stubOverload.SetDocumentationProvider(_ => fd.GetDocumentation()); + var documentation = fd.GetDocumentation(); + if (!string.IsNullOrEmpty(documentation)) { + stubOverload.SetDocumentation(documentation); } addOverload(stubOverload); _table.ReplacedByStubs.Add(fd); @@ -131,7 +132,7 @@ private void AddOverload(FunctionDefinition fd, IPythonClassMember function, Act if (!_table.Contains(fd)) { // Do not evaluate parameter types just yet. During light-weight top-level information // collection types cannot be determined as imports haven't been processed. - var overload = new PythonFunctionOverload(fd, function, _eval.GetLocationOfName(fd)); + var overload = new PythonFunctionOverload(function, fd, _eval.GetLocationOfName(fd), fd.ReturnAnnotation?.ToCodeString(_eval.Ast)); addOverload(overload); _table.Add(new FunctionEvaluator(_eval, overload)); } @@ -164,14 +165,14 @@ private bool TryAddProperty(FunctionDefinition node, IPythonType declaringType) return false; } - private void AddProperty(FunctionDefinition node, IPythonType declaringType, bool isAbstract) { - if (!(_eval.LookupNameInScopes(node.Name, LookupOptions.Local) is PythonPropertyType existing)) { - existing = new PythonPropertyType(node, _eval.GetLocationOfName(node), declaringType, isAbstract); + private void AddProperty(FunctionDefinition fd, IPythonType declaringType, bool isAbstract) { + if (!(_eval.LookupNameInScopes(fd.Name, LookupOptions.Local) is PythonPropertyType existing)) { + existing = new PythonPropertyType(fd, _eval.GetLocationOfName(fd), declaringType, isAbstract); // The variable is transient (non-user declared) hence it does not have location. // Property type is tracking locations for references and renaming. - _eval.DeclareVariable(node.Name, existing, VariableSource.Declaration); + _eval.DeclareVariable(fd.Name, existing, VariableSource.Declaration); } - AddOverload(node, existing, o => existing.AddOverload(o)); + AddOverload(fd, existing, o => existing.AddOverload(o)); } private IMember GetMemberFromStub(string name) { diff --git a/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs b/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs index b05802463..1d2e76c6b 100644 --- a/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs +++ b/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs @@ -681,6 +681,7 @@ private sealed class DependencyChainNode : IDependencyChainNode { public TValue Value => _vertex.DependencyVertex.Value; public bool HasMissingDependencies => _vertex.HasMissingDependencies; public int VertexDepth { get; } + public bool IsComplete => _vertex.SecondPass == null && !HasMissingDependencies; public DependencyChainNode(DependencyChainWalker walker, WalkingVertex vertex, int depth) { _walker = walker; diff --git a/src/Analysis/Ast/Impl/Dependencies/IDependencyChainNode.cs b/src/Analysis/Ast/Impl/Dependencies/IDependencyChainNode.cs index 5261ee866..299235cc6 100644 --- a/src/Analysis/Ast/Impl/Dependencies/IDependencyChainNode.cs +++ b/src/Analysis/Ast/Impl/Dependencies/IDependencyChainNode.cs @@ -24,5 +24,6 @@ internal interface IDependencyChainNode { TValue Value { get; } void Commit(); void Skip(); + bool IsComplete { get; } } } diff --git a/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs b/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs index 8e53962f2..1a2bf48bc 100644 --- a/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs @@ -53,7 +53,7 @@ private static PythonFunctionType GetOrCreateFunction(this IDocumentAnalysis ana // We DO want to replace class by function. Consider type() in builtins. // 'type()' in code is a function call, not a type class instantiation. if (!(analysis.GlobalScope.Variables[name]?.Value is PythonFunctionType f)) { - f = PythonFunctionType.ForSpecialization(name, analysis.Document); + f = PythonFunctionType.Specialize(name, analysis.Document, string.Empty); f.AddOverload(new PythonFunctionOverload(name, new Location(analysis.Document))); analysis.GlobalScope.DeclareVariable(name, f, VariableSource.Declaration); } diff --git a/src/Analysis/Ast/Impl/Extensions/AstExtensions.cs b/src/Analysis/Ast/Impl/Extensions/AstExtensions.cs index c4a220000..a1d2b5d8f 100644 --- a/src/Analysis/Ast/Impl/Extensions/AstExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/AstExtensions.cs @@ -32,7 +32,7 @@ public static Expression FindExpression(this PythonAst ast, SourceLocation locat public static string GetDocumentation(this ScopeStatement node) { var docExpr = (node?.Body as SuiteStatement)?.Statements?.FirstOrDefault() as ExpressionStatement; var ce = docExpr?.Expression as ConstantExpression; - return ce?.Value as string; + return ce?.GetStringValue(); } public static bool IsInsideComment(this PythonAst ast, SourceLocation location) { diff --git a/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs b/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs index 4610d32ec..e5ccb7bfb 100644 --- a/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs @@ -23,8 +23,8 @@ public static LocationInfo GetLocation(this Node node, IPythonModule module) { return LocationInfo.Empty; } - var start = node.GetStart(); - var end = node.GetEnd(); + var start = node.GetStart(module.GetAst()); + var end = node.GetEnd(module.GetAst()); return new LocationInfo(module.FilePath, module.Uri, start.Line, start.Column, end.Line, end.Column); } diff --git a/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs b/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs new file mode 100644 index 000000000..c33b4620f --- /dev/null +++ b/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs @@ -0,0 +1,30 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using Microsoft.Python.Analysis.Modules; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis { + public static class PythonModuleExtensions { + internal static PythonAst GetAst(this IPythonModule module) + => ((IAstNodeContainer)module).GetAstNode(module); + + internal static T GetAstNode(this IPythonModule module, object o) where T : Node + => ((IAstNodeContainer)module).GetAstNode(o); + internal static void AddAstNode(this IPythonModule module, object o, Node n) + => ((IAstNodeContainer)module).AddAstNode(o, n); + } +} diff --git a/src/Analysis/Ast/Impl/Extensions/ScopeExtensions.cs b/src/Analysis/Ast/Impl/Extensions/ScopeExtensions.cs index 7dc6fdcdb..6ba17ce62 100644 --- a/src/Analysis/Ast/Impl/Extensions/ScopeExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/ScopeExtensions.cs @@ -21,14 +21,14 @@ namespace Microsoft.Python.Analysis.Analyzer { public static class ScopeExtensions { - public static int GetBodyStartIndex(this IScope scope, PythonAst ast) { + public static int GetBodyStartIndex(this IScope scope) { switch (scope.Node) { case ClassDefinition cd: return cd.HeaderIndex; case FunctionDefinition fd: return fd.HeaderIndex; default: - return ast.LocationToIndex(scope.Node.GetStart()); + return scope.Node.StartIndex; } } @@ -47,7 +47,7 @@ public static IScope FindScope(this IScope parent, IDocument document, SourceLoc return children[i]; } - var start = children[i].GetBodyStartIndex(ast); + var start = children[i].GetBodyStartIndex(); if (start > index) { // We've gone past index completely so our last candidate is // the best one. @@ -114,10 +114,10 @@ private static int GetParentScopeIndent(IScope scope, PythonAst ast) { switch (scope.Node) { case ClassDefinition cd: // Return column of "class" statement - return cd.GetStart().Column; + return cd.GetStart(ast).Column; case FunctionDefinition fd when !fd.IsLambda: // Return column of "def" statement - return fd.GetStart().Column; + return fd.GetStart(ast).Column; default: return -1; } diff --git a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs index 597c1f64e..f792b565f 100644 --- a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs @@ -121,13 +121,14 @@ private void SpecializeTypes() { _hiddenNames.Add("__builtin_module_names__"); + var location = new Location(this, default); if (_boolType != null) { - Analysis.GlobalScope.DeclareVariable("True", _boolType, VariableSource.Builtin, new Location(this, default)); - Analysis.GlobalScope.DeclareVariable("False", _boolType, VariableSource.Builtin, new Location(this, default)); + Analysis.GlobalScope.DeclareVariable("True", _boolType, VariableSource.Builtin, location); + Analysis.GlobalScope.DeclareVariable("False", _boolType, VariableSource.Builtin, location); } if (noneType != null) { - Analysis.GlobalScope.DeclareVariable("None", noneType, VariableSource.Builtin, new Location(this, default)); + Analysis.GlobalScope.DeclareVariable("None", noneType, VariableSource.Builtin, location); } foreach (var n in GetMemberNames()) { diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IHasQualifiedName.cs b/src/Analysis/Ast/Impl/Modules/Definitions/IAstNodeContainer.cs similarity index 50% rename from src/Analysis/Ast/Impl/Types/Definitions/IHasQualifiedName.cs rename to src/Analysis/Ast/Impl/Modules/Definitions/IAstNodeContainer.cs index 2b9f9a48b..0ed36cb97 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/IHasQualifiedName.cs +++ b/src/Analysis/Ast/Impl/Modules/Definitions/IAstNodeContainer.cs @@ -13,26 +13,29 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; -using System.Collections.Generic; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Parsing.Ast; -namespace Microsoft.Python.Analysis.Types { - public interface IHasQualifiedName { +namespace Microsoft.Python.Analysis.Modules { + internal interface IAstNodeContainer { /// - /// Gets the fully qualified, dot-separated name of the value. - /// This is typically used for displaying to users. + /// Provides access to AST nodes associated with objects such as + /// for . + /// Nodes are not available for library modules as AST is not retained + /// in libraries in order to conserve memory. /// - string FullyQualifiedName { get; } + T GetAstNode(object o) where T : Node; /// - /// Gets the import and eval names of the value. The first part - /// should be importable, and the second is a name that can be - /// resolved with getattr(). - /// These are often seen separated with a colon. + /// Associated AST node with the object. /// - /// - /// The value cannot be resolved (for example, a nested function). - /// - KeyValuePair FullyQualifiedNamePair { get; } + void AddAstNode(object o, Node n); + + /// + /// Removes all AST node associations. + /// + void ClearAst(); + + void ClearContent(); } } diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index 16ba277a5..4907c28f2 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -42,7 +42,7 @@ namespace Microsoft.Python.Analysis.Modules { /// to AST and the module analysis. /// [DebuggerDisplay("{Name} : {ModuleType}")] - internal class PythonModule : LocatedMember, IDocument, IAnalyzable, IEquatable { + internal class PythonModule : LocatedMember, IDocument, IAnalyzable, IEquatable, IAstNodeContainer { private enum State { None, Loading, @@ -56,13 +56,13 @@ private enum State { private readonly DocumentBuffer _buffer = new DocumentBuffer(); private readonly DisposeToken _disposeToken = DisposeToken.Create(); private IReadOnlyList _parseErrors = Array.Empty(); + private readonly Dictionary _astMap = new Dictionary(); private readonly IDiagnosticsService _diagnosticsService; private string _documentation; // Must be null initially. private CancellationTokenSource _parseCts; private CancellationTokenSource _linkedParseCts; // combined with 'dispose' cts private Task _parsingTask; - private PythonAst _ast; private bool _updated; protected ILogger Log { get; } @@ -129,7 +129,7 @@ internal PythonModule(ModuleCreationOptions creationOptions, IServiceContainer s public virtual string Documentation { get { - _documentation = _documentation ?? _ast?.Documentation; + _documentation = _documentation ?? this.GetAst()?.Documentation; if (_documentation == null) { var m = GetMember("__doc__"); _documentation = m.TryGetConstant(out var s) ? s : string.Empty; @@ -199,40 +199,7 @@ public virtual IEnumerable GetMemberNames() { /// Typically used in code navigation scenarios when user /// wants to see library code and not a stub. /// - public IPythonModule PrimaryModule { get; internal set; } - - protected virtual string LoadContent() { - if (ContentState < State.Loading) { - ContentState = State.Loading; - try { - var code = FileSystem.ReadTextWithRetry(FilePath); - ContentState = State.Loaded; - return code; - } catch (IOException) { } catch (UnauthorizedAccessException) { } - } - return null; // Keep content as null so module can be loaded later. - } - - private void InitializeContent(string content, int version) { - lock (AnalysisLock) { - LoadContent(content, version); - - var startParse = ContentState < State.Parsing && (_parsingTask == null || version > 0); - if (startParse) { - Parse(); - } - } - } - - private void LoadContent(string content, int version) { - if (ContentState < State.Loading) { - try { - content = content ?? LoadContent(); - _buffer.Reset(version, content); - ContentState = State.Loaded; - } catch (IOException) { } catch (UnauthorizedAccessException) { } - } - } + public IPythonModule PrimaryModule { get; private set; } #endregion #region IDisposable @@ -293,10 +260,10 @@ public async Task GetAstAsync(CancellationToken cancellationToken = d } } cancellationToken.ThrowIfCancellationRequested(); - return _ast; + return this.GetAst(); } - public PythonAst GetAnyAst() => _ast; + public PythonAst GetAnyAst() => GetAstNode(this); /// /// Provides collection of parsing errors, if any. @@ -370,7 +337,11 @@ private void Parse(CancellationToken cancellationToken) { if (version != _buffer.Version) { throw new OperationCanceledException(); } - _ast = ast; + + // Stored nodes are no longer valid. + _astMap.Clear(); + _astMap[this] = ast; + _parseErrors = sink?.Diagnostics ?? Array.Empty(); // Do not report issues with libraries or stubs @@ -379,6 +350,7 @@ private void Parse(CancellationToken cancellationToken) { } ContentState = State.Parsed; + Analysis = new EmptyAnalysis(Services, this); } NewAst?.Invoke(this, EventArgs.Empty); @@ -387,7 +359,7 @@ private void Parse(CancellationToken cancellationToken) { ContentState = State.Analyzing; var analyzer = Services.GetService(); - analyzer.EnqueueDocumentForAnalysis(this, ast, version); + analyzer.EnqueueDocumentForAnalysis(this, version); } lock (AnalysisLock) { @@ -408,6 +380,15 @@ public override void Add(string message, SourceSpan span, int errorCode, Severit public void NotifyAnalysisBegins() { lock (AnalysisLock) { + if (Analysis is LibraryAnalysis) { + var sw = Log != null ? Stopwatch.StartNew() : null; + lock (AnalysisLock) { + _astMap[this] = RecreateAst(); + } + sw?.Stop(); + Log?.Log(TraceEventType.Verbose, $"Reloaded AST of {Name} in {sw?.Elapsed.TotalMilliseconds} ms"); + } + if (_updated) { _updated = false; // In all variables find those imported, then traverse imported modules @@ -434,14 +415,14 @@ public void NotifyAnalysisBegins() { } } - public void NotifyAnalysisComplete(IDocumentAnalysis analysis) { + public void NotifyAnalysisComplete(int version, ModuleWalker walker, bool isFinalPass) { lock (AnalysisLock) { - if (analysis.Version < Analysis.Version) { + if (version < Analysis.Version) { return; } - Analysis = analysis; - GlobalScope = analysis.GlobalScope; + Analysis = CreateAnalysis(version, walker, isFinalPass); + GlobalScope = Analysis.GlobalScope; // Derived classes can override OnAnalysisComplete if they want // to perform additional actions on the completed analysis such @@ -456,7 +437,7 @@ public void NotifyAnalysisComplete(IDocumentAnalysis analysis) { // Do not report issues with libraries or stubs if (ModuleType == ModuleType.User) { - _diagnosticsService?.Replace(Uri, analysis.Diagnostics, DiagnosticSource.Analysis); + _diagnosticsService?.Replace(Uri, Analysis.Diagnostics, DiagnosticSource.Analysis); } NewAnalysis?.Invoke(this, EventArgs.Empty); @@ -465,6 +446,40 @@ public void NotifyAnalysisComplete(IDocumentAnalysis analysis) { protected virtual void OnAnalysisComplete() { } #endregion + #region IEquatable + public bool Equals(IPythonModule other) => Name.Equals(other?.Name) && FilePath.Equals(other?.FilePath); + #endregion + + #region IAstNodeContainer + public T GetAstNode(object o) where T : Node { + lock (AnalysisLock) { + return _astMap.TryGetValue(o, out var n) ? (T)n : null; + } + } + + public void AddAstNode(object o, Node n) { + lock (AnalysisLock) { + Debug.Assert(!_astMap.ContainsKey(o) || _astMap[o] == n); + _astMap[o] = n; + } + } + + public void ClearAst() { + lock (AnalysisLock) { + if (ModuleType != ModuleType.User) { + _astMap.Clear(); + } + } + } + public void ClearContent() { + lock (AnalysisLock) { + if (ModuleType != ModuleType.User) { + _buffer.Reset(_buffer.Version, string.Empty); + } + } + } + #endregion + #region Analysis public IDocumentAnalysis GetAnyAnalysis() => Analysis; @@ -473,14 +488,42 @@ public Task GetAnalysisAsync(int waitTime = 200, Cancellation #endregion - private void RemoveReferencesInModule(IPythonModule module) { - if (module.GlobalScope?.Variables != null) { - foreach (var v in module.GlobalScope.Variables) { - v.RemoveReferences(this); + #region Content management + protected virtual string LoadContent() { + if (ContentState < State.Loading) { + ContentState = State.Loading; + try { + var code = FileSystem.ReadTextWithRetry(FilePath); + ContentState = State.Loaded; + return code; + } catch (IOException) { } catch (UnauthorizedAccessException) { } + } + return null; // Keep content as null so module can be loaded later. + } + + private void InitializeContent(string content, int version) { + lock (AnalysisLock) { + LoadContent(content, version); + + var startParse = ContentState < State.Parsing && (_parsingTask == null || version > 0); + if (startParse) { + Parse(); } } } + private void LoadContent(string content, int version) { + if (ContentState < State.Loading) { + try { + content = content ?? LoadContent(); + _buffer.Reset(version, content); + ContentState = State.Loaded; + } catch (IOException) { } catch (UnauthorizedAccessException) { } + } + } + #endregion + + #region Documentation private string TryGetDocFromModuleInitFile() { if (string.IsNullOrEmpty(FilePath) || !FileSystem.FileExists(FilePath)) { return string.Empty; @@ -528,7 +571,30 @@ private string TryGetDocFromModuleInitFile() { } catch (IOException) { } catch (UnauthorizedAccessException) { } return string.Empty; } + #endregion - public bool Equals(IPythonModule other) => Name.Equals(other?.Name) && FilePath.Equals(other?.FilePath); + private void RemoveReferencesInModule(IPythonModule module) { + if (module.GlobalScope?.Variables != null) { + foreach (var v in module.GlobalScope.Variables) { + v.RemoveReferences(this); + } + } + } + + private IDocumentAnalysis CreateAnalysis(int version, ModuleWalker walker, bool isFinalPass) + => ModuleType == ModuleType.Library && isFinalPass + ? new LibraryAnalysis(this, version, walker.Eval.Services, walker.GlobalScope, walker.StarImportMemberNames) + : (IDocumentAnalysis)new DocumentAnalysis(this, version, walker.GlobalScope, walker.Eval, walker.StarImportMemberNames); + + private PythonAst RecreateAst() { + lock (AnalysisLock) { + ContentState = State.None; + LoadContent(); + var parser = Parser.CreateParser(new StringReader(_buffer.Text), Interpreter.LanguageVersion, ParserOptions.Default); + var ast = parser.ParseFile(Uri); + ContentState = State.Parsed; + return ast; + } + } } } diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs index fe0e3657a..1ddc7d3ed 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs @@ -100,7 +100,7 @@ public IPythonModule GetOrLoadModule(string name) { return moduleRef.GetOrCreate(name, this); } - public bool TryAddModulePath(in string path, in bool allowNonRooted, out string fullModuleName) + public bool TryAddModulePath(in string path, in bool allowNonRooted, out string fullModuleName) => PathResolver.TryAddModulePath(path, allowNonRooted, out fullModuleName); public ModulePath FindModule(string filePath) { @@ -125,13 +125,12 @@ protected void ReloadModulePaths(in IEnumerable rootPaths) { protected class ModuleRef { private readonly object _syncObj = new object(); private IPythonModule _module; - private bool _creating; public ModuleRef(IPythonModule module) { _module = module; } - public ModuleRef() {} + public ModuleRef() { } public IPythonModule Value { get { @@ -142,25 +141,12 @@ public IPythonModule Value { } public IPythonModule GetOrCreate(string name, ModuleResolutionBase mrb) { - var create = false; lock (_syncObj) { if (_module != null) { return _module; } - if (!_creating) { - create = true; - _creating = true; - } - } - - if (!create) { - return null; - } - - var module = mrb.CreateModule(name); - lock (_syncObj) { - _creating = false; + var module = mrb.CreateModule(name); _module = module; return module; } diff --git a/src/Analysis/Ast/Impl/Specializations/Specialized.cs b/src/Analysis/Ast/Impl/Specializations/Specialized.cs index 4da22b82c..e71222bc4 100644 --- a/src/Analysis/Ast/Impl/Specializations/Specialized.cs +++ b/src/Analysis/Ast/Impl/Specializations/Specialized.cs @@ -26,9 +26,9 @@ public static IPythonPropertyType Property(string name, IPythonModule declaringM return prop; } - public static IPythonFunctionType Function(string name, IPythonModule declaringModule, IPythonType declaringType, string documentation, IMember returnValue) { + public static IPythonFunctionType Function(string name, IPythonModule declaringModule, string documentation, IMember returnValue) { var location = new Location(declaringModule); - var prop = new PythonFunctionType(name, location, declaringType, documentation); + var prop = PythonFunctionType.Specialize(name, declaringModule, documentation); var o = new PythonFunctionOverload(prop.Name, location); o.AddReturnValue(returnValue); prop.AddOverload(o); diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs b/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs index d91afe25c..8e280f25b 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs @@ -46,7 +46,7 @@ private void SpecializeMembers() { var location = new Location(this, default); // TypeVar - var fn = new PythonFunctionType("TypeVar", location, null, GetMemberDocumentation); + var fn = PythonFunctionType.Specialize("TypeVar", this, GetMemberDocumentation("TypeVar")); var o = new PythonFunctionOverload(fn.Name, location); // When called, create generic parameter type. For documentation // use original TypeVar declaration so it appear as a tooltip. @@ -57,7 +57,7 @@ private void SpecializeMembers() { _members["TypeVar"] = fn; // NewType - fn = new PythonFunctionType("NewType", location, null, GetMemberDocumentation); + fn = PythonFunctionType.Specialize("NewType", this, GetMemberDocumentation("NewType")); o = new PythonFunctionOverload(fn.Name, location); // When called, create generic parameter type. For documentation // use original TypeVar declaration so it appear as a tooltip. @@ -65,8 +65,8 @@ private void SpecializeMembers() { fn.AddOverload(o); _members["NewType"] = fn; - // NewType - fn = new PythonFunctionType("Type", location, null, GetMemberDocumentation); + // Type + fn = PythonFunctionType.Specialize("Type", this, GetMemberDocumentation("Type")); o = new PythonFunctionOverload(fn.Name, location); // When called, create generic parameter type. For documentation // use original TypeVar declaration so it appear as a tooltip. @@ -115,7 +115,7 @@ private void SpecializeMembers() { _members["Union"] = new GenericType("Union", CreateUnion, this); - _members["Counter"] = Specialized.Function("Counter", this, null, "Counter", + _members["Counter"] = Specialized.Function("Counter", this, GetMemberDocumentation("Counter"), new PythonInstance(Interpreter.GetBuiltinType(BuiltinTypeId.Int))); _members["SupportsInt"] = Interpreter.GetBuiltinType(BuiltinTypeId.Int); @@ -124,7 +124,7 @@ private void SpecializeMembers() { _members["SupportsBytes"] = Interpreter.GetBuiltinType(BuiltinTypeId.Bytes); _members["ByteString"] = Interpreter.GetBuiltinType(BuiltinTypeId.Bytes); - fn = new PythonFunctionType("NamedTuple", location, null, GetMemberDocumentation); + fn = PythonFunctionType.Specialize("NamedTuple", this, GetMemberDocumentation("NamedTuple")); o = new PythonFunctionOverload(fn.Name, location); o.SetReturnValueProvider((interpreter, overload, args) => CreateNamedTuple(args.Values())); fn.AddOverload(o); diff --git a/src/Analysis/Ast/Impl/Types/ArgumentSet.cs b/src/Analysis/Ast/Impl/Types/ArgumentSet.cs index 344f47dbb..6aa2b5038 100644 --- a/src/Analysis/Ast/Impl/Types/ArgumentSet.cs +++ b/src/Analysis/Ast/Impl/Types/ArgumentSet.cs @@ -15,6 +15,7 @@ // permissions and limitations under the License. using System.Collections.Generic; +using System.IO; using System.Linq; using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Analyzer.Evaluation; @@ -81,10 +82,17 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in OverloadIndex = overloadIndex; DeclaringModule = fn.DeclaringModule; + if (callExpr == null) { + // Typically invoked by specialization code without call expression in the code. + // Caller usually does not care about arguments. + _evaluated = true; + return; + } + var overload = fn.Overloads[overloadIndex]; var fd = overload.FunctionDefinition; - if (fd == null || fn.IsSpecialized) { + if (fn.IsSpecialized) { // Typically specialized function, like TypeVar() that does not actually have AST definition. // Make the arguments from the call expression. If argument does not have name, // try using name from the function definition based on the argument position. @@ -92,21 +100,14 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in for (var i = 0; i < callExpr.Args.Count; i++) { var name = callExpr.Args[i].Name; if (string.IsNullOrEmpty(name)) { - name = fd != null && i < fd.Parameters.Length ? fd.Parameters[i].Name : null; + name = i < overload.Parameters.Count ? overload.Parameters[i].Name : $"arg{i}"; } - name = name ?? $"arg{i}"; - var parameter = fd != null && i < fd.Parameters.Length ? fd.Parameters[i] : null; - _arguments.Add(new Argument(name, ParameterKind.Normal, callExpr.Args[i].Expression, null, parameter)); + var node = fd != null && i < fd.Parameters.Length ? fd.Parameters[i] : null; + _arguments.Add(new Argument(name, ParameterKind.Normal, callExpr.Args[i].Expression, null, node)); } return; } - if (callExpr == null) { - // Typically invoked by specialization code without call expression in the code. - // Caller usually does not care about arguments. - _evaluated = true; - return; - } var callLocation = callExpr.GetLocation(module); // https://www.python.org/dev/peps/pep-3102/#id5 @@ -115,7 +116,12 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in // had values assigned to them are marked as 'filled'.Slots which have // no value assigned to them yet are considered 'empty'. - var slots = fd.Parameters.Select(p => new Argument(p, p)).ToArray(); + var slots = new Argument[overload.Parameters.Count]; + for (var i = 0; i < overload.Parameters.Count; i++) { + var node = fd != null && i < fd.Parameters.Length ? fd.Parameters[i] : null; + slots[i] = new Argument(overload.Parameters[i], node); + } + // Locate sequence argument, if any var sa = slots.Where(s => s.Kind == ParameterKind.List).ToArray(); if (sa.Length > 1) { @@ -150,7 +156,7 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in break; } - if (formalParamIndex >= fd.Parameters.Length) { + if (formalParamIndex >= overload.Parameters.Count) { // We ran out of formal parameters and yet haven't seen // any sequence or dictionary ones. This looks like an error. _errors.Add(new DiagnosticsEntry(Resources.Analysis_TooManyFunctionArguments, arg.GetLocation(module).Span, @@ -158,8 +164,8 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in return; } - var formalParam = fd.Parameters[formalParamIndex]; - if (formalParam.IsList) { + var formalParam = overload.Parameters[formalParamIndex]; + if (formalParam.Kind == ParameterKind.List) { if (string.IsNullOrEmpty(formalParam.Name)) { // If the next unfilled slot is a vararg slot, and it does not have a name, then it is an error. _errors.Add(new DiagnosticsEntry(Resources.Analysis_TooManyPositionalArgumentBeforeStar, arg.GetLocation(module).Span, @@ -188,7 +194,7 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in break; // Sequence or dictionary parameter found. Done here. } - if (formalParam.IsDictionary) { + if (formalParam.Kind == ParameterKind.Dictionary) { // Next slot is a dictionary slot, but we have positional arguments still. _errors.Add(new DiagnosticsEntry(Resources.Analysis_TooManyPositionalArgumentBeforeStar, arg.GetLocation(module).Span, ErrorCodes.TooManyPositionalArgumentsBeforeStar, Severity.Warning, DiagnosticSource.Analysis)); @@ -248,7 +254,7 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in // then it is an error. foreach (var slot in slots.Where(s => s.Kind != ParameterKind.List && s.Kind != ParameterKind.Dictionary && s.Value == null)) { if (slot.ValueExpression == null) { - var parameter = fd.Parameters.First(p => p.Name == slot.Name); + var parameter = overload.Parameters.First(p => p.Name == slot.Name); if (parameter.DefaultValue == null) { // TODO: parameter is not assigned and has no default value. _errors.Add(new DiagnosticsEntry(Resources.Analysis_ParameterMissing.FormatUI(slot.Name), callLocation.Span, @@ -256,7 +262,8 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in } // Note that parameter default value expression is from the function definition AST // while actual argument values are from the calling file AST. - slot.ValueExpression = parameter.DefaultValue; + slot.ValueExpression = CreateExpression(parameter.Name, parameter.DefaultValueString); + slot.Value = parameter.DefaultValue; slot.ValueIsDefault = true; } } @@ -273,7 +280,6 @@ public ArgumentSet Evaluate() { foreach (var a in _arguments.Where(x => x.Value == null)) { a.Value = GetArgumentValue(a); - a.Type = Eval.GetValueFromExpression(a.TypeExpression) as IPythonType; } if (_listArgument != null) { @@ -295,6 +301,9 @@ public ArgumentSet Evaluate() { } private IMember GetArgumentValue(Argument arg) { + if (arg.Value is IMember m) { + return m; + } // Evaluates expression in the specific module context. Typically used to evaluate // expressions representing default values of function arguments since they are // are defined in the function declaring module rather than in the caller context. @@ -310,6 +319,20 @@ private IMember GetArgumentValue(Argument arg) { return Eval.GetValueFromExpression(arg.ValueExpression) ?? Eval.UnknownType; } + private Expression CreateExpression(string paramName, string defaultValue) { + if (string.IsNullOrEmpty(defaultValue)) { + return null; + } + using (var sr = new StringReader($"{paramName}={defaultValue}")) { + var parser = Parser.CreateParser(sr, Eval.Interpreter.LanguageVersion,ParserOptions.Default); + var ast = parser.ParseFile(); + if (ast.Body is SuiteStatement ste && ste.Statements.Count > 0 && ste.Statements[0] is AssignmentStatement a) { + return a.Right; + } + return null; + } + } + private sealed class Argument : IArgument { /// /// Argument name. @@ -342,26 +365,21 @@ private sealed class Argument : IArgument { /// /// Type of the argument, if annotated. /// - public IPythonType Type { get; internal set; } - - /// - /// Type annotation expression. - /// - public Expression TypeExpression { get; } + public IPythonType Type { get; } /// /// AST node that defines the argument. /// public Node Location { get; } - public Argument(Parameter p, Node location) : - this(p.Name, p.Kind, null, p.Annotation, location) { } + public Argument(IParameterInfo p, Node location) : + this(p.Name, p.Kind, null, p.Type, location) { } - public Argument(string name, ParameterKind kind, Expression valueValueExpression, Expression typeExpression, Node location) { + public Argument(string name, ParameterKind kind, Expression valueValueExpression, IPythonType type, Node location) { Name = name; Kind = kind; + Type = type; ValueExpression = valueValueExpression; - TypeExpression = typeExpression; Location = location; } diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IArgumentSet.cs b/src/Analysis/Ast/Impl/Types/Definitions/IArgumentSet.cs index e4b952b6c..bc1caa61b 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/IArgumentSet.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/IArgumentSet.cs @@ -44,11 +44,6 @@ public interface IArgument { /// IPythonType Type { get; } - /// - /// Annotation expression. - /// - Expression TypeExpression { get; } - /// /// Parameter location in the AST. /// diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IParameterInfo.cs b/src/Analysis/Ast/Impl/Types/Definitions/IParameterInfo.cs index d0875c2b7..8c79fea2f 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/IParameterInfo.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/IParameterInfo.cs @@ -14,6 +14,7 @@ // permissions and limitations under the License. using Microsoft.Python.Core.Text; +using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Types { /// @@ -36,16 +37,6 @@ public interface IParameterInfo { /// string Documentation { get; } - /// - /// True if the parameter is a *args parameter. - /// - bool IsParamArray { get; } - - /// - /// True if the parameter is a **args parameter. - /// - bool IsKeywordDict { get; } - /// /// Parameter type is generic and specific type will be /// determined at the time of the function call. @@ -62,5 +53,10 @@ public interface IParameterInfo { /// Default value. /// IMember DefaultValue { get; } + + /// + /// Parameter kind. + /// + ParameterKind Kind { get; } } } diff --git a/src/Analysis/Ast/Impl/Types/Location.cs b/src/Analysis/Ast/Impl/Types/Location.cs index 8cc2540bf..fd2275707 100644 --- a/src/Analysis/Ast/Impl/Types/Location.cs +++ b/src/Analysis/Ast/Impl/Types/Location.cs @@ -30,7 +30,7 @@ public Location(IPythonModule module, IndexSpan indexSpan) { public LocationInfo LocationInfo { get { - var ast = Module?.Analysis.Ast; + var ast = Module?.GetAst(); if (ast != null && !string.IsNullOrEmpty(Module?.FilePath) && Module?.Uri != null) { return new LocationInfo(Module.FilePath, Module.Uri, IndexSpan.ToSourceSpan(ast)); } @@ -38,7 +38,7 @@ public LocationInfo LocationInfo { } } - public bool IsValid => Module != null; + public bool IsValid => Module != null && IndexSpan != default; public override bool Equals(object obj) => obj is Location other && other.Module == Module && other.IndexSpan == IndexSpan; diff --git a/src/Analysis/Ast/Impl/Types/ParameterInfo.cs b/src/Analysis/Ast/Impl/Types/ParameterInfo.cs index c6a67058b..07f5a5e5b 100644 --- a/src/Analysis/Ast/Impl/Types/ParameterInfo.cs +++ b/src/Analysis/Ast/Impl/Types/ParameterInfo.cs @@ -14,7 +14,6 @@ // permissions and limitations under the License. using System; -using Microsoft.Python.Core.Text; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Types { @@ -34,17 +33,15 @@ public ParameterInfo(string name, IPythonType type, ParameterKind? kind, IMember Documentation = string.Empty; DefaultValue = defaultValue; Type = type; - IsParamArray = kind == ParameterKind.List; - IsKeywordDict = kind == ParameterKind.Dictionary; + Kind = kind ?? ParameterKind.Normal; } public string Name { get; } public string Documentation { get; } - public bool IsParamArray { get; } - public bool IsKeywordDict { get; } public bool IsGeneric { get; } public IPythonType Type { get; } public string DefaultValueString { get; } public IMember DefaultValue { get; } + public ParameterKind Kind { get; } } } diff --git a/src/Analysis/Ast/Impl/Types/PythonClassType.cs b/src/Analysis/Ast/Impl/Types/PythonClassType.cs index bcf546d59..850cf4893 100644 --- a/src/Analysis/Ast/Impl/Types/PythonClassType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonClassType.cs @@ -50,7 +50,7 @@ public PythonClassType( BuiltinTypeId builtinTypeId = BuiltinTypeId.Type ) : base(classDefinition.Name, location, classDefinition.GetDocumentation(), builtinTypeId) { Check.ArgumentNotNull(nameof(location), location.Module); - ClassDefinition = classDefinition; + location.Module.AddAstNode(this, classDefinition); } #region IPythonType @@ -163,7 +163,7 @@ public override IMember Index(IPythonInstance instance, object index) { #endregion #region IPythonClass - public ClassDefinition ClassDefinition { get; } + public ClassDefinition ClassDefinition => DeclaringModule.GetAstNode(this); public IReadOnlyList Bases => _bases; public IReadOnlyList Mro { diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs index 00402f0cb..0e5f90a51 100644 --- a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs +++ b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs @@ -18,7 +18,6 @@ using System.Diagnostics; using System.Linq; using Microsoft.Python.Analysis.Analyzer.Evaluation; -using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; @@ -48,15 +47,14 @@ internal sealed class PythonFunctionOverload : LocatedMember, IPythonFunctionOve // Return value can be an instance or a type info. Consider type(C()) returning // type info of C vs. return C() that returns an instance of C. - private Func _documentationProvider; private bool _fromAnnotation; - public PythonFunctionOverload(FunctionDefinition fd, IPythonClassMember classMember, Location location) - : this(fd.Name, location) { - FunctionDefinition = fd; - ClassMember = classMember; - var ast = (location.Module as IDocument)?.Analysis.Ast; - _returnDocumentation = ast != null ? fd.ReturnAnnotation?.ToCodeString(ast) : null; + public PythonFunctionOverload(IPythonClassMember cm, FunctionDefinition fd, Location location, string returnDocumentation) + : this(cm.Name, location) { + ClassMember = cm; + Documentation = fd.GetDocumentation(); + cm.DeclaringModule.AddAstNode(this, fd); + _returnDocumentation = returnDocumentation; } public PythonFunctionOverload(string name, Location location) : base(location) { @@ -69,9 +67,6 @@ public PythonFunctionOverload(string name, Location location) : base(location) { internal void SetParameters(IReadOnlyList parameters) => Parameters = parameters; - internal void SetDocumentationProvider(Func documentationProvider) - => _documentationProvider = _documentationProvider ?? documentationProvider; - internal void AddReturnValue(IMember value) { if (value.IsUnknown()) { return; // Don't add useless values. @@ -97,24 +92,14 @@ internal void SetReturnValue(IMember value, bool fromAnnotation) { _fromAnnotation = fromAnnotation; } - internal void SetReturnValueProvider(ReturnValueProvider provider) - => _returnValueProvider = provider; + internal void SetReturnValueProvider(ReturnValueProvider provider) => _returnValueProvider = provider; + internal void SetDocumentation(string documentation) => Documentation = documentation; #region IPythonFunctionOverload - - public FunctionDefinition FunctionDefinition { get; } + public FunctionDefinition FunctionDefinition => ClassMember?.DeclaringModule?.GetAstNode(this); public IPythonClassMember ClassMember { get; } public string Name { get; } - - public string Documentation { - get { - var s = _documentationProvider?.Invoke(Name); - if (string.IsNullOrEmpty(s)) { - s = FunctionDefinition.GetDocumentation(); - } - return s ?? string.Empty; - } - } + public string Documentation { get; private set; } public string GetReturnDocumentation(IPythonType self = null) { if (self != null) { @@ -157,8 +142,10 @@ private IMember GetSpecificReturnType(IPythonClassType selfClassType, IArgumentS case IGenericType gt when args != null: // -> CLASS[T] on standalone function (i.e. -> List[T]). var typeArgs = ExpressionEval.GetTypeArgumentsFromParameters(this, args); - Debug.Assert(typeArgs != null); - return gt.CreateSpecificType(typeArgs); + if (typeArgs != null) { + return gt.CreateSpecificType(typeArgs); + } + break; } return StaticReturnValue; diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs index 6b14f81bc..82c1def63 100644 --- a/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs @@ -13,7 +13,6 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.Collections.Generic; using System.Diagnostics; using System.Linq; @@ -25,20 +24,19 @@ namespace Microsoft.Python.Analysis.Types { [DebuggerDisplay("Function {Name} ({TypeId})")] - internal class PythonFunctionType : PythonType, IPythonFunctionType { + internal sealed class PythonFunctionType : PythonType, IPythonFunctionType { private ImmutableArray _overloads = ImmutableArray.Empty; - private readonly string _documentation; private bool _isAbstract; private bool _isSpecialized; /// /// Creates function for specializations /// - public static PythonFunctionType ForSpecialization(string name, IPythonModule declaringModule) - => new PythonFunctionType(name, new Location(declaringModule, default), true); + public static PythonFunctionType Specialize(string name, IPythonModule declaringModule, string documentation) + => new PythonFunctionType(name, new Location(declaringModule, default), documentation, true); - private PythonFunctionType(string name, Location location, bool isSpecialized = false) : - base(name, location, string.Empty, BuiltinTypeId.Function) { + private PythonFunctionType(string name, Location location, string documentation, bool isSpecialized = false) : + base(name, location, documentation ?? string.Empty, BuiltinTypeId.Function) { Check.ArgumentNotNull(nameof(location), location.Module); _isSpecialized = isSpecialized; } @@ -53,21 +51,7 @@ public PythonFunctionType( Location location, IPythonType declaringType, string documentation - ) : this(name, location, declaringType, _ => documentation) { - Check.ArgumentNotNull(nameof(location), location.Module); - } - - /// - /// Creates function type to use in special cases when function is dynamically - /// created, such as in specializations and custom iterators, without the actual - /// function definition in the AST. - /// - public PythonFunctionType( - string name, - Location location, - IPythonType declaringType, - Func documentationProvider - ) : base(name, location, documentationProvider, declaringType != null ? BuiltinTypeId.Method : BuiltinTypeId.Function) { + ) : base(name, location, documentation, declaringType != null ? BuiltinTypeId.Method : BuiltinTypeId.Function) { DeclaringType = declaringType; } @@ -75,24 +59,16 @@ public PythonFunctionType( FunctionDefinition fd, IPythonType declaringType, Location location - ) : base(fd.Name, location, fd.Documentation, - declaringType != null ? BuiltinTypeId.Method : BuiltinTypeId.Function) { - - FunctionDefinition = fd; + ) : base(fd.Name, location, + fd.Name == "__init__" ? (declaringType?.Documentation ?? fd.GetDocumentation()) : fd.GetDocumentation(), + declaringType != null ? BuiltinTypeId.Method : BuiltinTypeId.Function) { DeclaringType = declaringType; - // For __init__ documentation may either come from the function node of the the declaring - // type. Note that if there is no documentation on the class node, the class will try and - // get documentation from its __init__ function, delegating down to this type. So we need - // to set documentation statically for __init__ here or we may end up/ with stack overflows. - if (fd.Name == "__init__") { - _documentation = declaringType?.Documentation ?? fd.Documentation; - } + location.Module.AddAstNode(this, fd); ProcessDecorators(fd); } #region IPythonType - public override PythonMemberType MemberType => TypeId == BuiltinTypeId.Function ? PythonMemberType.Function : PythonMemberType.Method; @@ -102,22 +78,21 @@ public override IMember Call(IPythonInstance instance, string memberName, IArgum return overload?.Call(args, instance?.GetPythonType() ?? DeclaringType); } - internal override void SetDocumentationProvider(Func provider) { + internal override void SetDocumentation(string documentation) { foreach (var o in Overloads) { - (o as PythonFunctionOverload)?.SetDocumentationProvider(provider); + (o as PythonFunctionOverload)?.SetDocumentation(documentation); } - - base.SetDocumentationProvider(provider); + base.SetDocumentation(documentation); } #endregion #region IPythonFunction - public FunctionDefinition FunctionDefinition { get; } + public FunctionDefinition FunctionDefinition => DeclaringModule.GetAstNode(this); public IPythonType DeclaringType { get; } - public override string Documentation => _documentation ?? base.Documentation ?? (_overloads.Count > 0 ? _overloads[0].Documentation : default); - public virtual bool IsClassMethod { get; private set; } - public virtual bool IsStatic { get; private set; } + public override string Documentation => (_overloads.Count > 0 ? _overloads[0].Documentation : default) ?? base.Documentation; + public bool IsClassMethod { get; private set; } + public bool IsStatic { get; private set; } public override bool IsAbstract => _isAbstract; public override bool IsSpecialized => _isSpecialized; @@ -128,12 +103,6 @@ internal override void SetDocumentationProvider(Func provider) { public IReadOnlyList Overloads => _overloads; #endregion - #region IHasQualifiedName - public override string FullyQualifiedName => FullyQualifiedNamePair.CombineNames(); - public override KeyValuePair FullyQualifiedNamePair => - new KeyValuePair((DeclaringType as IHasQualifiedName)?.FullyQualifiedName ?? DeclaringType?.Name ?? DeclaringModule?.Name, Name); - #endregion - internal void Specialize(string[] dependencies) { _isSpecialized = true; Dependencies = dependencies != null diff --git a/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs b/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs index 87ee5f713..f10078945 100644 --- a/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs @@ -23,7 +23,7 @@ class PythonPropertyType : PythonType, IPythonPropertyType { public PythonPropertyType(FunctionDefinition fd, Location location, IPythonType declaringType, bool isAbstract) : this(fd.Name, location, declaringType, isAbstract) { - FunctionDefinition = fd; + declaringType.DeclaringModule.AddAstNode(this, fd); } public PythonPropertyType(string name, Location location, IPythonType declaringType, bool isAbstract) @@ -37,7 +37,7 @@ public PythonPropertyType(string name, Location location, IPythonType declaringT #endregion #region IPythonPropertyType - public FunctionDefinition FunctionDefinition { get; } + public FunctionDefinition FunctionDefinition => DeclaringModule.GetAstNode(this); public override bool IsAbstract { get; } public bool IsReadOnly => true; public IPythonType DeclaringType { get; } diff --git a/src/Analysis/Ast/Impl/Types/PythonType.cs b/src/Analysis/Ast/Impl/Types/PythonType.cs index 41e9bda1c..27c8ffaff 100644 --- a/src/Analysis/Ast/Impl/Types/PythonType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonType.cs @@ -22,11 +22,9 @@ namespace Microsoft.Python.Analysis.Types { [DebuggerDisplay("{Name}")] - internal class PythonType : LocatedMember, IPythonType, IHasQualifiedName, IEquatable { + internal class PythonType : LocatedMember, IPythonType {//, IEquatable { private readonly object _lock = new object(); private readonly string _name; - private Func _documentationProvider; - private string _documentation; private Dictionary _members; private BuiltinTypeId _typeId; private bool _readonly; @@ -42,16 +40,7 @@ public PythonType( string documentation, BuiltinTypeId typeId = BuiltinTypeId.Unknown ) : this(name, location, typeId) { - _documentation = documentation; - } - - public PythonType( - string name, - Location location, - Func documentationProvider, - BuiltinTypeId typeId = BuiltinTypeId.Unknown - ) : this(name, location, typeId) { - _documentationProvider = documentationProvider; + Documentation = documentation; } private PythonType(string name, Location location, BuiltinTypeId typeId) : base(location) { @@ -67,7 +56,7 @@ private PythonType(string name, Location location, BuiltinTypeId typeId) : base( #region IPythonType public virtual string Name => TypeId == BuiltinTypeId.Ellipsis ? "..." : _name; - public virtual string Documentation => _documentationProvider != null ? _documentationProvider.Invoke(Name) : _documentation; + public virtual string Documentation { get; private set; } public virtual BuiltinTypeId TypeId => _typeId; public bool IsBuiltin => DeclaringModule == null || DeclaringModule is IBuiltinsPythonModule; public virtual bool IsAbstract => false; @@ -98,12 +87,6 @@ public virtual IMember Call(IPythonInstance instance, string memberName, IArgume public virtual IMember Index(IPythonInstance instance, object index) => instance?.Index(index) ?? UnknownType; #endregion - #region IHasQualifiedName - public virtual string FullyQualifiedName => FullyQualifiedNamePair.CombineNames(); - public virtual KeyValuePair FullyQualifiedNamePair - => new KeyValuePair(DeclaringModule?.Name ?? string.Empty, Name); - #endregion - #region IMemberContainer public virtual IMember GetMember(string name) => Members.TryGetValue(name, out var member) ? member : null; public virtual IEnumerable GetMemberNames() => Members.Keys; @@ -117,8 +100,7 @@ internal bool TrySetTypeId(BuiltinTypeId typeId) { return true; } - internal virtual void SetDocumentationProvider(Func provider) => _documentationProvider = provider; - internal virtual void SetDocumentation(string documentation) => _documentation = documentation; + internal virtual void SetDocumentation(string documentation) => Documentation = documentation; internal void AddMembers(IEnumerable variables, bool overwrite) { lock (_lock) { @@ -167,10 +149,10 @@ internal IMember AddMember(string name, IMember member, bool overwrite) { protected bool ContainsMember(string name) => Members.ContainsKey(name); protected IMember UnknownType => DeclaringModule.Interpreter.UnknownType; - public bool Equals(IPythonType other) => PythonTypeComparer.Instance.Equals(this, other); + //public bool Equals(IPythonType other) => PythonTypeComparer.Instance.Equals(this, other); - public override bool Equals(object obj) - => obj is IPythonType pt && PythonTypeComparer.Instance.Equals(this, pt); - public override int GetHashCode() => 0; + //public override bool Equals(object obj) + // => obj is IPythonType pt && PythonTypeComparer.Instance.Equals(this, pt); + //public override int GetHashCode() => 0; } } diff --git a/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs b/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs index 5201297b1..8058317cc 100644 --- a/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs +++ b/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs @@ -22,7 +22,7 @@ namespace Microsoft.Python.Analysis.Types { /// /// Delegates most of the methods to the wrapped/inner class. /// - internal class PythonTypeWrapper : IPythonType, IHasQualifiedName { + internal class PythonTypeWrapper : IPythonType { private readonly BuiltinTypeId _builtinTypeId; private IPythonType _innerType; @@ -86,11 +86,6 @@ public virtual IMember Index(IPythonInstance instance, object index) public virtual IEnumerable GetMemberNames() => InnerType.GetMemberNames(); #endregion - #region IHasQualifiedName - public virtual string FullyQualifiedName => (InnerType as IHasQualifiedName)?.FullyQualifiedName; - public virtual KeyValuePair FullyQualifiedNamePair => (InnerType as IHasQualifiedName)?.FullyQualifiedNamePair ?? default; - #endregion - protected IMember UnknownType => DeclaringModule.Interpreter.UnknownType; public override bool Equals(object obj) diff --git a/src/Analysis/Ast/Impl/Utilities/AstUtilities.cs b/src/Analysis/Ast/Impl/Utilities/AstUtilities.cs new file mode 100644 index 000000000..816b0c304 --- /dev/null +++ b/src/Analysis/Ast/Impl/Utilities/AstUtilities.cs @@ -0,0 +1,29 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.IO; +using Microsoft.Python.Parsing; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Utilities { + public static class AstUtilities { + public static PythonAst MakeEmptyAst(Uri documentUri) { + using (var sr = new StringReader(string.Empty)) { + return Parser.CreateParser(sr, PythonLanguageVersion.None).ParseFile(documentUri); + } + } + } +} diff --git a/src/Analysis/Ast/Impl/Values/GlobalScope.cs b/src/Analysis/Ast/Impl/Values/GlobalScope.cs index 42e42e41d..699809d20 100644 --- a/src/Analysis/Ast/Impl/Values/GlobalScope.cs +++ b/src/Analysis/Ast/Impl/Values/GlobalScope.cs @@ -23,25 +23,27 @@ public GlobalScope(IPythonModule module): base(null, null, module) { DeclareBuiltinVariables(); } - public override ScopeStatement Node => Module.Analysis?.Ast; + public override ScopeStatement Node => Module.GetAst(); private void DeclareBuiltinVariables() { if (Module.ModuleType != ModuleType.User) { return; } + var location = new Location(Module, default); + var boolType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Bool); var strType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Str); var listType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.List); var dictType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Dict); - DeclareVariable("__debug__", boolType, VariableSource.Builtin); - DeclareVariable("__doc__", strType, VariableSource.Builtin); - DeclareVariable("__file__", strType, VariableSource.Builtin); - DeclareVariable("__name__", strType, VariableSource.Builtin); - DeclareVariable("__package__", strType, VariableSource.Builtin); - DeclareVariable("__path__", listType, VariableSource.Builtin); - DeclareVariable("__dict__", dictType, VariableSource.Builtin); + DeclareVariable("__debug__", boolType, VariableSource.Builtin, location); + DeclareVariable("__doc__", strType, VariableSource.Builtin, location); + DeclareVariable("__file__", strType, VariableSource.Builtin, location); + DeclareVariable("__name__", strType, VariableSource.Builtin, location); + DeclareVariable("__package__", strType, VariableSource.Builtin, location); + DeclareVariable("__path__", listType, VariableSource.Builtin, location); + DeclareVariable("__dict__", dictType, VariableSource.Builtin, location); } } } diff --git a/src/Analysis/Ast/Impl/Values/Scope.cs b/src/Analysis/Ast/Impl/Values/Scope.cs index c62e93d58..d517c4457 100644 --- a/src/Analysis/Ast/Impl/Values/Scope.cs +++ b/src/Analysis/Ast/Impl/Values/Scope.cs @@ -32,16 +32,17 @@ internal class Scope : IScope { private List _childScopes; public Scope(ScopeStatement node, IScope outerScope, IPythonModule module) { - Node = node; OuterScope = outerScope; Module = module; + if (node != null) { + Module.AddAstNode(this, node); + } DeclareBuiltinVariables(); } #region IScope - public string Name => Node?.Name ?? ""; - public virtual ScopeStatement Node { get; } + public virtual ScopeStatement Node => Module.GetAstNode(this); public IScope OuterScope { get; } public IPythonModule Module { get; } @@ -95,24 +96,25 @@ private void DeclareBuiltinVariables() { return; } + var location = new Location(Module, default); var strType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Str); var objType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Object); - VariableCollection.DeclareVariable("__name__", strType, VariableSource.Builtin); + VariableCollection.DeclareVariable("__name__", strType, VariableSource.Builtin, location); if (Node is FunctionDefinition) { var dictType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Dict); var tupleType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Tuple); - VariableCollection.DeclareVariable("__closure__", tupleType, VariableSource.Builtin); - VariableCollection.DeclareVariable("__code__", objType, VariableSource.Builtin); - VariableCollection.DeclareVariable("__defaults__", tupleType, VariableSource.Builtin); - VariableCollection.DeclareVariable("__dict__", dictType, VariableSource.Builtin); - VariableCollection.DeclareVariable("__doc__", strType, VariableSource.Builtin); - VariableCollection.DeclareVariable("__func__", objType, VariableSource.Builtin); - VariableCollection.DeclareVariable("__globals__", dictType, VariableSource.Builtin); + VariableCollection.DeclareVariable("__closure__", tupleType, VariableSource.Builtin, location); + VariableCollection.DeclareVariable("__code__", objType, VariableSource.Builtin, location); + VariableCollection.DeclareVariable("__defaults__", tupleType, VariableSource.Builtin, location); + VariableCollection.DeclareVariable("__dict__", dictType, VariableSource.Builtin, location); + VariableCollection.DeclareVariable("__doc__", strType, VariableSource.Builtin, location); + VariableCollection.DeclareVariable("__func__", objType, VariableSource.Builtin, location); + VariableCollection.DeclareVariable("__globals__", dictType, VariableSource.Builtin, location); } else if (Node is ClassDefinition) { - VariableCollection.DeclareVariable("__self__", objType, VariableSource.Builtin); + VariableCollection.DeclareVariable("__self__", objType, VariableSource.Builtin, location); } } } diff --git a/src/Analysis/Ast/Impl/Values/VariableCollection.cs b/src/Analysis/Ast/Impl/Values/VariableCollection.cs index 817456eca..878182211 100644 --- a/src/Analysis/Ast/Impl/Values/VariableCollection.cs +++ b/src/Analysis/Ast/Impl/Values/VariableCollection.cs @@ -98,7 +98,7 @@ public IEnumerable GetMemberNames() { #endregion - internal void DeclareVariable(string name, IMember value, VariableSource source, Location location = default) { + internal void DeclareVariable(string name, IMember value, VariableSource source, Location location) { name = !string.IsNullOrWhiteSpace(name) ? name : throw new ArgumentException(nameof(name)); lock (_syncObj) { if (_variables.TryGetValue(name, out var existing)) { diff --git a/src/Analysis/Ast/Test/ArgumentSetTests.cs b/src/Analysis/Ast/Test/ArgumentSetTests.cs index fc613182a..2f4bee636 100644 --- a/src/Analysis/Ast/Test/ArgumentSetTests.cs +++ b/src/Analysis/Ast/Test/ArgumentSetTests.cs @@ -347,9 +347,14 @@ from builtins import pow [TestMethod, Priority(0)] public async Task DefaultArgumentAnotherFile() { const string code = @" -from DefaultArgument import func +from .module2 import func func() "; + const string code2 = @" +class A: ... +def func(a = A()): ... +"; + await TestData.CreateTestSpecificFileAsync("module2.py", code2); var argSet = await GetArgSetAsync(code, "func"); argSet.Arguments.Count.Should().Be(1); argSet.Evaluate(); @@ -361,7 +366,6 @@ from DefaultArgument import func t.MemberType.Should().Be(PythonMemberType.Class); } - private async Task GetArgSetAsync(string code, string funcName = "f") { var analysis = await GetAnalysisAsync(code); var f = analysis.Should().HaveFunction(funcName).Which; diff --git a/src/Analysis/Ast/Test/FluentAssertions/AssertionsUtilities.cs b/src/Analysis/Ast/Test/FluentAssertions/AssertionsUtilities.cs index a30be146a..9be29ed15 100644 --- a/src/Analysis/Ast/Test/FluentAssertions/AssertionsUtilities.cs +++ b/src/Analysis/Ast/Test/FluentAssertions/AssertionsUtilities.cs @@ -31,9 +31,9 @@ public static bool Is3X(IScope scope) => scope.GlobalScope.Module.Interpreter.LanguageVersion.Is3x(); public static void AssertTypeIds( - IEnumerable actualTypeIds, - IEnumerable typeIds, - string name, bool languageVersionIs3X, string because, + IEnumerable actualTypeIds, + IEnumerable typeIds, + string name, bool languageVersionIs3X, string because, object[] reasonArgs, string itemNameSingle = "type", string itemNamePlural = "types") { var expected = typeIds.Select(t => { switch (t) { @@ -163,22 +163,12 @@ private static StringBuilder AppendQuotedName(this StringBuilder stringBuilder, } public static string GetQuotedName(object value) { - string name; - switch (value) { - case IHasQualifiedName _: - case IPythonModule _: - name = GetName(value); - return string.IsNullOrEmpty(name) ? string.Empty : $"'{name}'"; - default: - name = GetName(value); - return string.IsNullOrEmpty(name) ? string.Empty : $"'{name}'"; - } + var name = GetName(value); + return string.IsNullOrEmpty(name) ? string.Empty : $"'{name}'"; } public static string GetName(object value) { switch (value) { - case IHasQualifiedName qualifiedName: - return qualifiedName.FullyQualifiedName; case IPythonModule pythonModule: return pythonModule.Name; case IScope scope: diff --git a/src/Analysis/Ast/Test/FunctionTests.cs b/src/Analysis/Ast/Test/FunctionTests.cs index d707be2eb..7ebefdfdb 100644 --- a/src/Analysis/Ast/Test/FunctionTests.cs +++ b/src/Analysis/Ast/Test/FunctionTests.cs @@ -356,7 +356,6 @@ def e(cls): pass [TestMethod, Priority(0)] public async Task OverloadsParamTypeMatch() { const string code = @" - def f(a: bool) -> None: ... def f(a: int) -> float: ... def f(a: str) -> bytes: ... diff --git a/src/Analysis/Ast/Test/PepHintTests.cs b/src/Analysis/Ast/Test/PepHintTests.cs index 91db0df13..8964ddc7f 100644 --- a/src/Analysis/Ast/Test/PepHintTests.cs +++ b/src/Analysis/Ast/Test/PepHintTests.cs @@ -13,14 +13,12 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System.Linq; using System.Threading.Tasks; using FluentAssertions; using Microsoft.Python.Analysis.Tests.FluentAssertions; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Parsing.Tests; -using Microsoft.Python.Tests.Utilities.FluentAssertions; using Microsoft.VisualStudio.TestTools.UnitTesting; using TestUtilities; diff --git a/src/Analysis/Ast/Test/ValuesTests.cs b/src/Analysis/Ast/Test/ValuesTests.cs index d200604a3..007c76b25 100644 --- a/src/Analysis/Ast/Test/ValuesTests.cs +++ b/src/Analysis/Ast/Test/ValuesTests.cs @@ -15,7 +15,6 @@ using System.IO; using System.Threading.Tasks; -using FluentAssertions; using Microsoft.Python.Analysis.Tests.FluentAssertions; using Microsoft.Python.Analysis.Types; using Microsoft.VisualStudio.TestTools.UnitTesting; diff --git a/src/LanguageServer/Impl/Completion/ExpressionLocator.cs b/src/LanguageServer/Impl/Completion/ExpressionLocator.cs index 9ce9cd0f6..f0835cb87 100644 --- a/src/LanguageServer/Impl/Completion/ExpressionLocator.cs +++ b/src/LanguageServer/Impl/Completion/ExpressionLocator.cs @@ -44,17 +44,17 @@ public static void FindExpression(PythonAst ast, SourceLocation position, FindEx expression = expression ?? (statement as ExpressionStatement)?.Expression; } - private static bool CanBackUp(PythonAst tree, Node node, Node statement, ScopeStatement scope, int column) { + private static bool CanBackUp(PythonAst ast, Node node, Node statement, ScopeStatement scope, int column) { if (node != null || !((statement as ExpressionStatement)?.Expression is ErrorExpression)) { return false; } var top = 1; if (scope != null) { - var scopeStart = scope.GetStart(); + var scopeStart = scope.GetStart(ast); if (scope.Body != null) { - top = scope.Body.GetEnd().Line == scopeStart.Line - ? scope.Body.GetStart().Column + top = scope.Body.GetEnd(ast).Line == scopeStart.Line + ? scope.Body.GetStart(ast).Column : scopeStart.Column; } else { top = scopeStart.Column; diff --git a/src/LanguageServer/Impl/Completion/FunctionDefinitionCompletion.cs b/src/LanguageServer/Impl/Completion/FunctionDefinitionCompletion.cs index 844cf3dda..2c56cbc20 100644 --- a/src/LanguageServer/Impl/Completion/FunctionDefinitionCompletion.cs +++ b/src/LanguageServer/Impl/Completion/FunctionDefinitionCompletion.cs @@ -33,7 +33,7 @@ public static bool TryGetCompletionsForOverride(FunctionDefinition function, Com } if (function.Parent is ClassDefinition cd && function.NameExpression != null && context.Position > function.NameExpression.StartIndex) { - var loc = function.GetStart(); + var loc = function.GetStart(context.Ast); var overrideable = GetOverrideable(context, location).ToArray(); overrideable = !string.IsNullOrEmpty(function.Name) ? overrideable.Where(o => o.Name.StartsWithOrdinal(function.Name)).ToArray() @@ -56,10 +56,10 @@ private static CompletionItem ToOverrideCompletionItem(IPythonFunctionOverload o } private static string MakeOverrideParameter(IParameterInfo paramInfo, string defaultValue) { - if (paramInfo.IsParamArray) { + if (paramInfo.Kind == ParameterKind.List) { return $"*{paramInfo.Name}"; } - if (paramInfo.IsKeywordDict) { + if (paramInfo.Kind == ParameterKind.Dictionary) { return $"**{paramInfo.Name}"; } return !string.IsNullOrEmpty(paramInfo.DefaultValueString) ? $"{paramInfo.Name}={defaultValue}" : paramInfo.Name; diff --git a/src/LanguageServer/Impl/Completion/ImportCompletion.cs b/src/LanguageServer/Impl/Completion/ImportCompletion.cs index 3a156912b..9fb15e009 100644 --- a/src/LanguageServer/Impl/Completion/ImportCompletion.cs +++ b/src/LanguageServer/Impl/Completion/ImportCompletion.cs @@ -79,7 +79,7 @@ public static CompletionResult GetCompletionsInFromImport(FromImportStatement fr } if (context.Position >= name.StartIndex) { - var applicableSpan = name.GetSpan(); + var applicableSpan = name.GetSpan(context.Ast); var importSearchResult = mres.CurrentPathResolver.FindImports(document.FilePath, fromImport); return GetResultFromImportSearch(importSearchResult, context, false, applicableSpan); } diff --git a/src/LanguageServer/Impl/Documentation/DocstringConverter.cs b/src/LanguageServer/Impl/Documentation/DocstringConverter.cs index 466668890..aab46c877 100644 --- a/src/LanguageServer/Impl/Documentation/DocstringConverter.cs +++ b/src/LanguageServer/Impl/Documentation/DocstringConverter.cs @@ -77,7 +77,7 @@ private int NextBlockIndent private string CurrentLineWithinBlock => CurrentLine.Substring(_blockIndent); private DocstringConverter(string input) { - _builder = new StringBuilder(input.Length); + _builder = new StringBuilder(input?.Length ?? 0); _state = ParseText; _lines = SplitDocstring(input); } diff --git a/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs b/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs index 0ebe6dda1..ddd1565df 100644 --- a/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs +++ b/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs @@ -41,8 +41,8 @@ public override bool Walk(ClassDefinition node) { _stack.AddSymbol(new HierarchicalSymbol( node.Name, SymbolKind.Class, - node.GetSpan(), - node.NameExpression.GetSpan(), + node.GetSpan(_ast), + node.NameExpression.GetSpan(_ast), children, FunctionKind.Class )); @@ -58,13 +58,13 @@ public override bool Walk(FunctionDefinition node) { node.Body?.Walk(this); var children = _stack.Exit(); - var span = node.GetSpan(); + var span = node.GetSpan(_ast); var ds = new HierarchicalSymbol( node.Name, SymbolKind.Function, span, - node.IsLambda ? span : node.NameExpression.GetSpan(), + node.IsLambda ? span : node.NameExpression.GetSpan(_ast), children, FunctionKind.Function ); @@ -102,7 +102,7 @@ public override bool Walk(FunctionDefinition node) { public override bool Walk(ImportStatement node) { foreach (var (nameNode, nameString) in node.Names.Zip(node.AsNames, (name, asName) => asName != null ? (asName, asName.Name) : ((Node)name, name.MakeString()))) { - var span = nameNode.GetSpan(); + var span = nameNode.GetSpan(_ast); _stack.AddSymbol(new HierarchicalSymbol(nameString, SymbolKind.Module, span)); } @@ -115,7 +115,7 @@ public override bool Walk(FromImportStatement node) { } foreach (var name in node.Names.Zip(node.AsNames, (name, asName) => asName ?? name)) { - var span = name.GetSpan(); + var span = name.GetSpan(_ast); _stack.AddSymbol(new HierarchicalSymbol(name.Name, SymbolKind.Module, span)); } @@ -202,7 +202,7 @@ public override bool Walk(GeneratorExpression node) { private void ExitComprehension(Comprehension node) { var children = _stack.Exit(); - var span = node.GetSpan(); + var span = node.GetSpan(_ast); _stack.AddSymbol(new HierarchicalSymbol( $"<{node.NodeName}>", @@ -229,7 +229,7 @@ private void AddVarSymbol(NameExpression node) { break; } - var span = node.GetSpan(); + var span = node.GetSpan(_ast); _stack.AddSymbol(new HierarchicalSymbol(node.Name, kind, span)); } diff --git a/src/LanguageServer/Impl/Sources/HoverSource.cs b/src/LanguageServer/Impl/Sources/HoverSource.cs index 729b69c97..d4a2f4537 100644 --- a/src/LanguageServer/Impl/Sources/HoverSource.cs +++ b/src/LanguageServer/Impl/Sources/HoverSource.cs @@ -45,8 +45,8 @@ public Hover GetHover(IDocumentAnalysis analysis, SourceLocation location) { } var range = new Range { - start = expr.GetStart(), - end = expr.GetEnd() + start = expr.GetStart(analysis.Ast), + end = expr.GetEnd(analysis.Ast) }; var eval = analysis.ExpressionEvaluator; @@ -91,7 +91,7 @@ public Hover GetHover(IDocumentAnalysis analysis, SourceLocation location) { if (expr is MemberExpression mex) { name = mex.Name; range = new Range { - start = mex.Target.GetEnd(), + start = mex.Target.GetEnd(analysis.Ast), end = range.end }; diff --git a/src/LanguageServer/Impl/Sources/ReferenceSource.cs b/src/LanguageServer/Impl/Sources/ReferenceSource.cs index c302d91f3..009c7894f 100644 --- a/src/LanguageServer/Impl/Sources/ReferenceSource.cs +++ b/src/LanguageServer/Impl/Sources/ReferenceSource.cs @@ -20,6 +20,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis; +using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; @@ -129,7 +130,7 @@ private List ScanClosedFiles(string name, CancellationToken cancellationTok return files; } - private static async Task AnalyzeFiles(IModuleManagement moduleManagement, IEnumerable files, CancellationToken cancellationToken) { + private async Task AnalyzeFiles(IModuleManagement moduleManagement, IEnumerable files, CancellationToken cancellationToken) { var analysisTasks = new List(); foreach (var f in files) { if (moduleManagement.TryAddModulePath(f.ToAbsolutePath(), false, out var fullName)) { diff --git a/src/LanguageServer/Test/GoToDefinitionTests.cs b/src/LanguageServer/Test/GoToDefinitionTests.cs index 0e23cc904..1e2f53924 100644 --- a/src/LanguageServer/Test/GoToDefinitionTests.cs +++ b/src/LanguageServer/Test/GoToDefinitionTests.cs @@ -230,7 +230,9 @@ public async Task GotoRelativeImportInExplicitPackage() { rdt.OpenDocument(subpkgPath, string.Empty); var submod = rdt.OpenDocument(submodPath, "from .. import mod"); - var analysis = await submod.GetAnalysisAsync(-1); + await Services.GetService().WaitForCompleteAnalysisAsync(); + var analysis = await submod.GetAnalysisAsync(Timeout.Infinite); + var ds = new DefinitionSource(Services); var reference = ds.FindDefinition(analysis, new SourceLocation(1, 18), out _); reference.Should().NotBeNull(); @@ -250,7 +252,8 @@ public async Task GotoModuleSourceFromRelativeImport() { z = 1 def X(): ... "); - var analysis = await modA.GetAnalysisAsync(-1); + await Services.GetService().WaitForCompleteAnalysisAsync(); + var analysis = await modA.GetAnalysisAsync(Timeout.Infinite); var ds = new DefinitionSource(Services); var reference = ds.FindDefinition(analysis, new SourceLocation(1, 7), out _); diff --git a/src/LanguageServer/Test/ReferencesTests.cs b/src/LanguageServer/Test/ReferencesTests.cs index 2d2b23c2e..cd53d92f4 100644 --- a/src/LanguageServer/Test/ReferencesTests.cs +++ b/src/LanguageServer/Test/ReferencesTests.cs @@ -17,6 +17,7 @@ using System.IO; using System.Threading.Tasks; using FluentAssertions; +using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Core.Text; using Microsoft.Python.LanguageServer.Sources; @@ -368,6 +369,7 @@ import logging var rdt = Services.GetService(); var doc = rdt.OpenDocument(uri, code); + await Services.GetService().WaitForCompleteAnalysisAsync(); var analysis = await GetDocumentAnalysisAsync(doc); var rs = new ReferenceSource(Services); diff --git a/src/Parsing/Impl/Ast/BinaryExpression.cs b/src/Parsing/Impl/Ast/BinaryExpression.cs index 3b0497b30..ca7c7536c 100644 --- a/src/Parsing/Impl/Ast/BinaryExpression.cs +++ b/src/Parsing/Impl/Ast/BinaryExpression.cs @@ -21,7 +21,7 @@ using System.Threading.Tasks; namespace Microsoft.Python.Parsing.Ast { - public partial class BinaryExpression : Expression { + public class BinaryExpression : Expression { public BinaryExpression(PythonOperator op, Expression left, Expression right, int operatorIndex) { if (op == PythonOperator.None) { throw new ArgumentException("bad operator"); @@ -88,8 +88,6 @@ public override async Task WalkAsync(PythonWalkerAsync walker, CancellationToken } internal override void AppendCodeString(StringBuilder res, PythonAst ast, CodeFormattingOptions format) { - var left = Left; - var right = Right; string op1, op2; if (Operator == PythonOperator.NotIn) { diff --git a/src/Parsing/Impl/Ast/ClassDefinition.cs b/src/Parsing/Impl/Ast/ClassDefinition.cs index c1db95498..9ec8289bf 100644 --- a/src/Parsing/Impl/Ast/ClassDefinition.cs +++ b/src/Parsing/Impl/Ast/ClassDefinition.cs @@ -85,8 +85,6 @@ internal override bool HasLateBoundVariableSets { set => base.HasLateBoundVariableSets = value; } - public override bool NeedsLocalContext => true; - internal override bool ExposesLocalVariable(PythonVariable variable) => true; internal override bool TryBindOuter(ScopeStatement from, string name, bool allowGlobals, out PythonVariable variable) { diff --git a/src/Parsing/Impl/Ast/DictionaryExpression.cs b/src/Parsing/Impl/Ast/DictionaryExpression.cs index 0feb297be..1ed01406d 100644 --- a/src/Parsing/Impl/Ast/DictionaryExpression.cs +++ b/src/Parsing/Impl/Ast/DictionaryExpression.cs @@ -17,7 +17,6 @@ using System.Text; using System.Threading; using System.Threading.Tasks; -using Microsoft.Python.Core; using Microsoft.Python.Core.Collections; namespace Microsoft.Python.Parsing.Ast { diff --git a/src/Parsing/Impl/Ast/DottedName.cs b/src/Parsing/Impl/Ast/DottedName.cs index f0e6e4b96..b169aabfa 100644 --- a/src/Parsing/Impl/Ast/DottedName.cs +++ b/src/Parsing/Impl/Ast/DottedName.cs @@ -14,11 +14,9 @@ // permissions and limitations under the License. using System.Collections.Generic; -using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; -using Microsoft.Python.Core; using Microsoft.Python.Core.Collections; namespace Microsoft.Python.Parsing.Ast { @@ -44,16 +42,6 @@ public virtual string MakeString() { public override IEnumerable GetChildNodes() => Names; - public override PythonAst Ast { - get => base.Ast; - internal set { - base.Ast = value; - foreach (var n in Names) { - n.Ast = value; - } - } - } - public override void Walk(PythonWalker walker) { if (walker.Walk(this)) { } diff --git a/src/Parsing/Impl/Ast/FromImportStatement.cs b/src/Parsing/Impl/Ast/FromImportStatement.cs index d4bbdb97d..794f904f3 100644 --- a/src/Parsing/Impl/Ast/FromImportStatement.cs +++ b/src/Parsing/Impl/Ast/FromImportStatement.cs @@ -49,25 +49,9 @@ public FromImportStatement(ModuleName/*!*/ root, ImmutableArray Justification = "breaking change")] public PythonVariable[] Variables { get; set; } - public PythonReference[] GetReferences(PythonAst ast) => GetVariableReferences(this, ast); - // TODO: return names and aliases when they are united into one node public override IEnumerable GetChildNodes() => Enumerable.Empty(); - public override PythonAst Ast { - get => base.Ast; - internal set { - foreach (var n in Names.ExcludeDefault()) { - n.Ast = value; - } - foreach (var n in AsNames.ExcludeDefault()) { - n.Ast = value; - } - - Root.Ast = value; - base.Ast = value; - } - } public override void Walk(PythonWalker walker) { if (walker.Walk(this)) { } diff --git a/src/Parsing/Impl/Ast/FunctionDefinition.cs b/src/Parsing/Impl/Ast/FunctionDefinition.cs index 1b70c2c44..46efe1e7b 100644 --- a/src/Parsing/Impl/Ast/FunctionDefinition.cs +++ b/src/Parsing/Impl/Ast/FunctionDefinition.cs @@ -52,8 +52,6 @@ public FunctionDefinition(NameExpression name, Parameter[] parameters, Statement public Parameter[] Parameters { get; } - public override int ArgCount => Parameters.Length; - internal void SetKeywordEndIndex(int index) => _keywordEndIndex = index; public override int KeywordEndIndex => _keywordEndIndex ?? (DefIndex + (IsCoroutine ? 9 : 3)); public override int KeywordLength => KeywordEndIndex - StartIndex; diff --git a/src/Parsing/Impl/Ast/ImportStatement.cs b/src/Parsing/Impl/Ast/ImportStatement.cs index a7126ba49..5c1a79fc3 100644 --- a/src/Parsing/Impl/Ast/ImportStatement.cs +++ b/src/Parsing/Impl/Ast/ImportStatement.cs @@ -13,14 +13,12 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; -using Microsoft.Python.Core; using Microsoft.Python.Core.Collections; namespace Microsoft.Python.Parsing.Ast { @@ -44,19 +42,6 @@ public ImportStatement(ImmutableArray names, ImmutableArray GetChildNodes() => Enumerable.Empty(); - public override PythonAst Ast { - get => base.Ast; - internal set { - foreach (var n in Names.ExcludeDefault()) { - n.Ast = value; - } - foreach (var n in AsNames.ExcludeDefault()) { - n.Ast = value; - } - base.Ast = value; - } - } - public override void Walk(PythonWalker walker) { if (walker.Walk(this)) { } diff --git a/src/Parsing/Impl/Ast/MemberExpression.cs b/src/Parsing/Impl/Ast/MemberExpression.cs index b11db83c2..d49d32c2e 100644 --- a/src/Parsing/Impl/Ast/MemberExpression.cs +++ b/src/Parsing/Impl/Ast/MemberExpression.cs @@ -95,6 +95,6 @@ public override void SetLeadingWhiteSpace(PythonAst ast, string whiteSpace) /// /// Returns the span of the name component of the expression /// - public SourceSpan GetNameSpan() => new SourceSpan(Ast.IndexToLocation(NameHeader), GetEnd()); + public SourceSpan GetNameSpan(PythonAst ast) => new SourceSpan(ast.IndexToLocation(NameHeader), GetEnd(ast)); } } diff --git a/src/Parsing/Impl/Ast/NamedExpression.cs b/src/Parsing/Impl/Ast/NamedExpression.cs index 8a55cd3da..3f5693ab2 100644 --- a/src/Parsing/Impl/Ast/NamedExpression.cs +++ b/src/Parsing/Impl/Ast/NamedExpression.cs @@ -1,5 +1,4 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; using System.Text; using System.Threading; using System.Threading.Tasks; diff --git a/src/Parsing/Impl/Ast/Node.cs b/src/Parsing/Impl/Ast/Node.cs index 82bc79c42..973ce6e70 100644 --- a/src/Parsing/Impl/Ast/Node.cs +++ b/src/Parsing/Impl/Ast/Node.cs @@ -23,8 +23,6 @@ namespace Microsoft.Python.Parsing.Ast { public abstract class Node { #region Public API - public virtual PythonAst Ast { get; internal set; } - public int EndIndex { get => IndexSpan.End; set => IndexSpan = new IndexSpan(IndexSpan.Start, value - IndexSpan.Start); @@ -50,11 +48,11 @@ public string ToCodeString(PythonAst ast, CodeFormattingOptions format) { return string.IsInterned(result) ?? result; } - public SourceLocation GetStart() => Ast.IndexToLocation(StartIndex); + public SourceLocation GetStart(PythonAst ast) => ast.IndexToLocation(StartIndex); - public SourceLocation GetEnd() => Ast.IndexToLocation(EndIndex); + public SourceLocation GetEnd(PythonAst ast) => ast.IndexToLocation(EndIndex); - public SourceSpan GetSpan() => new SourceSpan(GetStart(), GetEnd()); + public SourceSpan GetSpan(PythonAst ast) => new SourceSpan(GetStart(ast), GetEnd(ast)); /// /// Returns the proceeding whitespace (newlines and comments) that diff --git a/src/Parsing/Impl/Ast/PythonAst.cs b/src/Parsing/Impl/Ast/PythonAst.cs index 17cc32571..70a669cf9 100644 --- a/src/Parsing/Impl/Ast/PythonAst.cs +++ b/src/Parsing/Impl/Ast/PythonAst.cs @@ -22,11 +22,11 @@ using Microsoft.Python.Core.Text; namespace Microsoft.Python.Parsing.Ast { - /// - /// Top-level ast for all Python code. Holds onto the body and the line mapping information. + /// Top-level ast for all Python code. Holds onto the body and the line mapping information. /// public sealed class PythonAst : ScopeStatement { + private readonly object _lock = new object(); private readonly Statement _body; private readonly Dictionary> _attributes = new Dictionary>(); @@ -36,7 +36,6 @@ public PythonAst(Uri module, Statement body, NewLineLocation[] lineLocations, Py LanguageVersion = langVersion; NewLineLocations = lineLocations; CommentLocations = commentLocations; - } public PythonAst(IEnumerable existingAst) { @@ -60,9 +59,8 @@ public PythonAst(IEnumerable existingAst) { } public Uri Module { get; } - public NewLineLocation[] NewLineLocations { get; } - public SourceLocation[] CommentLocations { get; } - + public NewLineLocation[] NewLineLocations { get; private set; } + public SourceLocation[] CommentLocations { get; private set; } public override string Name => ""; /// @@ -76,7 +74,7 @@ public PythonAst(IEnumerable existingAst) { /// public bool HasVerbatim { get; internal set; } - public override IEnumerable GetChildNodes() => new[] {_body}; + public override IEnumerable GetChildNodes() => new[] { _body }; public override void Walk(PythonWalker walker) { if (walker.Walk(this)) { @@ -93,62 +91,58 @@ public override async Task WalkAsync(PythonWalkerAsync walker, CancellationToken } public override Statement Body => _body; - public PythonLanguageVersion LanguageVersion { get; } - public bool TryGetAttribute(Node node, object key, out object value) { - if (_attributes.TryGetValue(node, out var nodeAttrs)) { - return nodeAttrs.TryGetValue(key, out value); + public void Reduce(Func filter) { + lock (_lock) { + (Body as SuiteStatement)?.FilterStatements(filter); + _attributes?.Clear(); + Variables?.Clear(); + CommentLocations = Array.Empty(); + // DO keep NewLineLocations as they are required + // to calculate node positions for navigation; + base.Clear(); } - value = null; - return false; } - public void SetAttribute(Node node, object key, object value) { - if (!_attributes.TryGetValue(node, out var nodeAttrs)) { - nodeAttrs = _attributes[node] = new Dictionary(); + public bool TryGetAttribute(Node node, object key, out object value) { + lock (_lock) { + if (_attributes.TryGetValue(node, out var nodeAttrs)) { + return nodeAttrs.TryGetValue(key, out value); + } + + value = null; + return false; } - nodeAttrs[key] = value; } - /// - /// Copies attributes that apply to one node and makes them available for the other node. - /// - /// - /// - public void CopyAttributes(Node from, Node to) { - if (_attributes.TryGetValue(from, out var fromAttrs)) { - var toAttrs = new Dictionary(fromAttrs.Count); - foreach (var nodeAttr in fromAttrs) { - toAttrs[nodeAttr.Key] = nodeAttr.Value; + public void SetAttribute(Node node, object key, object value) { + lock (_lock) { + if (!_attributes.TryGetValue(node, out var nodeAttrs)) { + nodeAttrs = _attributes[node] = new Dictionary(); } - _attributes[to] = toAttrs; + nodeAttrs[key] = value; } } internal void SetAttributes(Dictionary> attributes) { - foreach (var nodeAttributes in attributes) { - var node = nodeAttributes.Key; - if (!_attributes.TryGetValue(node, out var existingNodeAttributes)) { - existingNodeAttributes = _attributes[node] = new Dictionary(nodeAttributes.Value.Count); - } - - foreach (var nodeAttr in nodeAttributes.Value) { - existingNodeAttributes[nodeAttr.Key] = nodeAttr.Value; + lock (_lock) { + foreach (var nodeAttributes in attributes) { + var node = nodeAttributes.Key; + if (!_attributes.TryGetValue(node, out var existingNodeAttributes)) { + existingNodeAttributes = _attributes[node] = new Dictionary(nodeAttributes.Value.Count); + } + + foreach (var nodeAttr in nodeAttributes.Value) { + existingNodeAttributes[nodeAttr.Key] = nodeAttr.Value; + } } } } public SourceLocation IndexToLocation(int index) => NewLineLocation.IndexToLocation(NewLineLocations, index); - public int LocationToIndex(SourceLocation location) => NewLineLocation.LocationToIndex(NewLineLocations, location, EndIndex); - /// - /// Length of the span (number of characters inside the span). - /// - public int GetSpanLength(SourceSpan span) => LocationToIndex(span.End) - LocationToIndex(span.Start); - - internal int GetLineEndFromPosition(int index) { var loc = IndexToLocation(index); if (loc.Line >= NewLineLocations.Length) { diff --git a/src/Parsing/Impl/Ast/ScopeStatement.cs b/src/Parsing/Impl/Ast/ScopeStatement.cs index 45900967d..d6181be4e 100644 --- a/src/Parsing/Impl/Ast/ScopeStatement.cs +++ b/src/Parsing/Impl/Ast/ScopeStatement.cs @@ -22,7 +22,6 @@ namespace Microsoft.Python.Parsing.Ast { public abstract class ScopeStatement : Statement { // due to "exec" or call to dir, locals, eval, vars... - private ClosureInfo[] _closureVariables; // closed over variables, bool indicates if we accessed it in this scope. private List _freeVars; // list of variables accessed from outer scopes private List _globalVars; // global variables accessed from this scope private List _cellVars; // variables accessed from nested scopes @@ -75,12 +74,10 @@ public abstract Statement Body { /// /// Gets the variables for this scope. /// - public ICollection ScopeVariables + public ICollection ScopeVariables => Variables?.Values ?? Array.Empty() as ICollection; public virtual bool IsGlobal => false; - public virtual bool NeedsLocalContext => NeedsLocalsDictionary || ContainsNestedFreeVariables; - public virtual int ArgCount => 0; public PythonAst GlobalParent { get { @@ -93,20 +90,23 @@ public PythonAst GlobalParent { } } - internal void AddFreeVariable(PythonVariable variable, bool accessedInScope) { - if (_freeVars == null) { - _freeVars = new List(); - } + protected void Clear() { + _references?.Clear(); + _cellVars?.Clear(); + _freeVars?.Clear(); + _globalVars?.Clear(); + _nonLocalVars?.Clear(); + } + internal void AddFreeVariable(PythonVariable variable, bool accessedInScope) { + _freeVars = _freeVars ?? new List(); if (!_freeVars.Contains(variable)) { _freeVars.Add(variable); } } internal string AddReferencedGlobal(string name) { - if (_globalVars == null) { - _globalVars = new List(); - } + _globalVars = _globalVars ?? new List(); if (!_globalVars.Contains(name)) { _globalVars.Add(name); } @@ -114,17 +114,12 @@ internal string AddReferencedGlobal(string name) { } internal void AddNonLocalVariable(NameExpression name) { - if (_nonLocalVars == null) { - _nonLocalVars = new List(); - } + _nonLocalVars = _nonLocalVars ?? new List(); _nonLocalVars.Add(name); } internal void AddCellVariable(PythonVariable variable) { - if (_cellVars == null) { - _cellVars = new List(); - } - + _cellVars = _cellVars ?? new List(); if (!_cellVars.Contains(variable.Name)) { _cellVars.Add(variable.Name); } @@ -135,17 +130,6 @@ internal void AddCellVariable(PythonVariable variable) { /// public IReadOnlyList FreeVariables => _freeVars; - /// - /// Variables that are bound to the global scope - /// - public IReadOnlyList GlobalVariables => _globalVars; - - /// - /// Variables that are referred to from a nested scope and need to be - /// promoted to cells. - /// - public IReadOnlyList CellVariables => _cellVars; - internal abstract bool ExposesLocalVariable(PythonVariable variable); public bool TryGetVariable(string name, out PythonVariable variable) { @@ -231,10 +215,6 @@ internal virtual void FinishBind(PythonNameBinder binder) { } } - if (closureVariables != null) { - _closureVariables = closureVariables.ToArray(); - } - // no longer needed _references = null; } diff --git a/src/Parsing/Impl/Ast/SuiteStatement.cs b/src/Parsing/Impl/Ast/SuiteStatement.cs index ddb4c87ac..920a1d8b6 100644 --- a/src/Parsing/Impl/Ast/SuiteStatement.cs +++ b/src/Parsing/Impl/Ast/SuiteStatement.cs @@ -14,7 +14,9 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; using System.Collections.Generic; +using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -22,16 +24,18 @@ namespace Microsoft.Python.Parsing.Ast { public sealed class SuiteStatement : Statement { - private readonly Statement[] _statements; + private Statement[] _statements; public SuiteStatement(Statement[] statements) { _statements = statements; } public IList Statements => _statements; - public override IEnumerable GetChildNodes() => _statements.WhereNotNull(); + public void FilterStatements(Func filter) + => _statements = _statements.Where(filter).ToArray(); + public override void Walk(PythonWalker walker) { if (walker.Walk(this)) { foreach (var s in _statements.MaybeEnumerate()) { diff --git a/src/Parsing/Impl/Parser.cs b/src/Parsing/Impl/Parser.cs index 2ef7dcd0a..e6a1b177f 100644 --- a/src/Parsing/Impl/Parser.cs +++ b/src/Parsing/Impl/Parser.cs @@ -247,10 +247,6 @@ private PythonAst CreateAst(Uri module, Statement ret) { ast.SetAttributes(_attributes); PythonNameBinder.BindAst(_langVersion, ast, _errors, _bindReferences); - foreach (var n in ((Node)ast).TraverseDepthFirst(c => c.GetChildNodes())) { - n.Ast = ast; - } - return ast; } @@ -3294,8 +3290,8 @@ private Expression buildFStringExpr(IEnumerable readTokens) { var openQuotes = readTokens.Where(t => t.Token.Kind == TokenKind.FString) .Select(t => ((FStringToken)t.Token).OpenQuotes).DefaultIfEmpty("'").First(); - List fStringChildren = new List(); - StringBuilder unparsedFStringBuilder = new StringBuilder(); + var fStringChildren = new List(); + var unparsedFStringBuilder = new StringBuilder(); foreach (var tokenWithSpan in readTokens) { if (tokenWithSpan.Token.Kind == TokenKind.FString) { diff --git a/src/Parsing/Impl/Token.cs b/src/Parsing/Impl/Token.cs index db7d8d1c5..703deb591 100644 --- a/src/Parsing/Impl/Token.cs +++ b/src/Parsing/Impl/Token.cs @@ -93,8 +93,6 @@ public ConstantValueToken(object value) Value = value; } - public object Constant => Value; - public override object Value { get; } public override string Image => Value == null ? "None" : Value.ToString(); @@ -140,11 +138,7 @@ public FStringToken(string value, string openQuote, bool isTriple, bool isRaw) public string Text => (string)Value; - public override string Image { - get { - return Value == null ? "None" : $"f{OpenQuotes}{Value.ToString()}{OpenQuotes}"; - } - } + public override string Image => Value == null ? "None" : $"f{OpenQuotes}{Value.ToString()}{OpenQuotes}"; } public sealed class VerbatimFStringToken : FStringToken {