diff --git a/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs b/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs
index d3802e6d7..60cf40216 100644
--- a/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs
+++ b/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs
@@ -69,7 +69,7 @@ public override bool Walk(ExpressionStatement node) {
return true;
case CallExpression callex when callex.Target is MemberExpression mex && !string.IsNullOrEmpty(mex.Name):
var t = Eval.GetValueFromExpression(mex.Target)?.GetPythonType();
- t?.GetMember(mex.Name).AddReference(Eval.GetLocationOfName(mex));
+ t?.GetMember(mex.Name)?.AddReference(Eval.GetLocationOfName(mex));
return true;
default:
return base.Walk(node);
diff --git a/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs b/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs
index 8d4b9751d..bb1665b8d 100644
--- a/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs
+++ b/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs
@@ -26,7 +26,6 @@ internal interface IAnalyzable {
///
/// Notifies document that its analysis is now complete.
///
- /// Document analysis
- void NotifyAnalysisComplete(IDocumentAnalysis analysis);
+ void NotifyAnalysisComplete(int version, ModuleWalker walker, bool isFinalPass);
}
}
diff --git a/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs
index 970c4bb3f..122992414 100644
--- a/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs
+++ b/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs
@@ -20,7 +20,6 @@
using Microsoft.Python.Analysis.Diagnostics;
using Microsoft.Python.Analysis.Types;
using Microsoft.Python.Core.Collections;
-using Microsoft.Python.Parsing.Ast;
namespace Microsoft.Python.Analysis.Analyzer {
public interface IPythonAnalyzer {
@@ -28,7 +27,7 @@ public interface IPythonAnalyzer {
///
/// Schedules module for analysis. Module will be scheduled if version of AST is greater than the one used to get previous analysis
///
- void EnqueueDocumentForAnalysis(IPythonModule module, PythonAst ast, int version);
+ void EnqueueDocumentForAnalysis(IPythonModule module, int version);
///
/// Schedules module for analysis for its existing AST, but with new dependencies.
diff --git a/src/Analysis/Ast/Impl/Analyzer/DocumentAnalysis.cs b/src/Analysis/Ast/Impl/Analyzer/DocumentAnalysis.cs
index 2d6abf826..51e0cd378 100644
--- a/src/Analysis/Ast/Impl/Analyzer/DocumentAnalysis.cs
+++ b/src/Analysis/Ast/Impl/Analyzer/DocumentAnalysis.cs
@@ -13,17 +13,11 @@
// See the Apache Version 2.0 License for specific language governing
// permissions and limitations under the License.
-using System;
using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using Microsoft.Python.Analysis.Analyzer.Evaluation;
using Microsoft.Python.Analysis.Diagnostics;
using Microsoft.Python.Analysis.Documents;
using Microsoft.Python.Analysis.Values;
-using Microsoft.Python.Core;
using Microsoft.Python.Core.Diagnostics;
-using Microsoft.Python.Parsing;
using Microsoft.Python.Parsing.Ast;
namespace Microsoft.Python.Analysis.Analyzer {
@@ -77,24 +71,4 @@ public DocumentAnalysis(IDocument document, int version, IGlobalScope globalScop
public IEnumerable Diagnostics => ExpressionEvaluator.Diagnostics;
#endregion
}
-
- public sealed class EmptyAnalysis : IDocumentAnalysis {
- private static PythonAst _emptyAst;
-
- public EmptyAnalysis(IServiceContainer services, IDocument document) {
- Document = document ?? throw new ArgumentNullException(nameof(document));
- GlobalScope = new EmptyGlobalScope(document);
-
- _emptyAst = _emptyAst ?? (_emptyAst = Parser.CreateParser(new StringReader(string.Empty), PythonLanguageVersion.None).ParseFile(document.Uri));
- ExpressionEvaluator = new ExpressionEval(services, document, Ast);
- }
-
- public IDocument Document { get; }
- public int Version { get; } = -1;
- public IGlobalScope GlobalScope { get; }
- public PythonAst Ast => _emptyAst;
- public IExpressionEvaluator ExpressionEvaluator { get; }
- public IReadOnlyList StarImportMemberNames => null;
- public IEnumerable Diagnostics => Enumerable.Empty();
- }
}
diff --git a/src/Analysis/Ast/Impl/Analyzer/EmptyAnalysis.cs b/src/Analysis/Ast/Impl/Analyzer/EmptyAnalysis.cs
new file mode 100644
index 000000000..b073f20b2
--- /dev/null
+++ b/src/Analysis/Ast/Impl/Analyzer/EmptyAnalysis.cs
@@ -0,0 +1,44 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.Python.Analysis.Analyzer.Evaluation;
+using Microsoft.Python.Analysis.Diagnostics;
+using Microsoft.Python.Analysis.Documents;
+using Microsoft.Python.Analysis.Utilities;
+using Microsoft.Python.Analysis.Values;
+using Microsoft.Python.Core;
+using Microsoft.Python.Parsing.Ast;
+
+namespace Microsoft.Python.Analysis.Analyzer {
+ public sealed class EmptyAnalysis : IDocumentAnalysis {
+ public EmptyAnalysis(IServiceContainer services, IDocument document) {
+ Document = document ?? throw new ArgumentNullException(nameof(document));
+ GlobalScope = new EmptyGlobalScope(document);
+ Ast = AstUtilities.MakeEmptyAst(document.Uri);
+ ExpressionEvaluator = new ExpressionEval(services, document);
+ }
+
+ public IDocument Document { get; }
+ public int Version { get; } = -1;
+ public IGlobalScope GlobalScope { get; }
+ public PythonAst Ast { get; }
+ public IExpressionEvaluator ExpressionEvaluator { get; }
+ public IReadOnlyList StarImportMemberNames => null;
+ public IEnumerable Diagnostics => Enumerable.Empty();
+ }
+}
diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Annotations.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Annotations.cs
new file mode 100644
index 000000000..a5973ab5c
--- /dev/null
+++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Annotations.cs
@@ -0,0 +1,48 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using Microsoft.Python.Analysis.Types;
+using Microsoft.Python.Parsing.Ast;
+
+namespace Microsoft.Python.Analysis.Analyzer.Evaluation {
+ internal sealed partial class ExpressionEval {
+ public IPythonType GetTypeFromAnnotation(Expression expr, LookupOptions options = LookupOptions.Global | LookupOptions.Builtins)
+ => GetTypeFromAnnotation(expr, out _, options);
+
+ public IPythonType GetTypeFromAnnotation(Expression expr, out bool isGeneric, LookupOptions options = LookupOptions.Global | LookupOptions.Builtins) {
+ isGeneric = false;
+ switch (expr) {
+ case null:
+ return null;
+ case CallExpression callExpr:
+ // x: NamedTuple(...)
+ return GetValueFromCallable(callExpr)?.GetPythonType() ?? UnknownType;
+ case IndexExpression indexExpr:
+ // Try generics
+ var target = GetValueFromExpression(indexExpr.Target);
+ var result = GetValueFromGeneric(target, indexExpr);
+ if (result != null) {
+ isGeneric = true;
+ return result.GetPythonType();
+ }
+ break;
+ }
+
+ // Look at specialization and typing first
+ var ann = new TypeAnnotation(Ast.LanguageVersion, expr);
+ return ann.GetValue(new TypeAnnotationConverter(this, options));
+ }
+ }
+}
diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs
index f2e79b8a5..38346a6e0 100644
--- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs
+++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs
@@ -17,6 +17,7 @@
using System.Diagnostics;
using System.Linq;
using Microsoft.Python.Analysis.Documents;
+using Microsoft.Python.Analysis.Extensions;
using Microsoft.Python.Analysis.Modules;
using Microsoft.Python.Analysis.Types;
using Microsoft.Python.Analysis.Values;
@@ -77,9 +78,11 @@ public IMember GetValueFromLambda(LambdaExpression expr) {
return null;
}
- var location = GetLocationOfName(expr.Function);
- var ft = new PythonFunctionType(expr.Function, null, location);
- var overload = new PythonFunctionOverload(expr.Function, ft, location);
+ var fd = expr.Function;
+ var location = GetLocationOfName(fd);
+ var ft = new PythonFunctionType(fd, null, location);
+ var overload = new PythonFunctionOverload(ft, fd, location, expr.Function.ReturnAnnotation?.ToCodeString(Ast));
+ overload.SetParameters(CreateFunctionParameters(null, ft, fd, false));
ft.AddOverload(overload);
return ft;
}
@@ -101,13 +104,13 @@ public IMember GetValueFromClassCtor(IPythonClassType cls, CallExpression expr)
return cls.CreateInstance(cls.Name, args);
}
- public IMember GetValueFromBound(IPythonBoundType t, CallExpression expr) {
+ private IMember GetValueFromBound(IPythonBoundType t, CallExpression expr) {
switch (t.Type) {
case IPythonFunctionType fn:
return GetValueFromFunctionType(fn, t.Self, expr);
case IPythonPropertyType p:
return GetValueFromProperty(p, t.Self);
- case IPythonIteratorType it when t.Self is IPythonCollection seq:
+ case IPythonIteratorType _ when t.Self is IPythonCollection seq:
return seq.GetIterator();
}
return UnknownType;
@@ -173,7 +176,7 @@ public IMember GetValueFromFunctionType(IPythonFunctionType fn, IPythonInstance
if (instanceType == null || fn.DeclaringType == null || fn.IsSpecialized ||
instanceType.IsSpecialized || fn.DeclaringType.IsSpecialized ||
instanceType.Equals(fn.DeclaringType) ||
- fn.IsStub || !string.IsNullOrEmpty(fn.Overloads[args.OverloadIndex].GetReturnDocumentation(null))) {
+ fn.IsStub || !string.IsNullOrEmpty(fn.Overloads[args.OverloadIndex].GetReturnDocumentation())) {
LoadFunctionDependencyModules(fn);
@@ -192,14 +195,14 @@ public IMember GetValueFromFunctionType(IPythonFunctionType fn, IPythonInstance
// def func(a, b): return a + b
// from working in libraries, but this is small sacrifice for significant performance
// increase in library analysis.
- if (fn.DeclaringModule is IDocument doc && fd?.Ast == doc.GetAnyAst() && EvaluateFunctionBody(fn)) {
+ if (fn.DeclaringModule is IDocument && EvaluateFunctionBody(fn)) {
// Stubs are coming from another module.
return TryEvaluateWithArguments(fn, args);
}
return UnknownType;
}
- public IMember GetValueFromProperty(IPythonPropertyType p, IPythonInstance instance) {
+ private IMember GetValueFromProperty(IPythonPropertyType p, IPythonInstance instance) {
// Function may not have been walked yet. Do it now.
SymbolTable.Evaluate(p.FunctionDefinition);
return instance.Call(p.Name, ArgumentSet.Empty);
@@ -311,5 +314,69 @@ private void LoadFunctionDependencyModules(IPythonFunctionType fn) {
Services.GetService().EnqueueDocumentForAnalysis(Module, dependencies);
}
}
+
+ public IReadOnlyList CreateFunctionParameters(IPythonClassType self, IPythonClassMember function, FunctionDefinition fd, bool declareVariables) {
+ // For class method no need to add extra parameters, but first parameter type should be the class.
+ // For static and unbound methods do not add or set anything.
+ // For regular bound methods add first parameter and set it to the class.
+
+ var parameters = new List();
+ var skip = 0;
+ if (self != null && function.HasClassFirstArgument()) {
+ var p0 = fd.Parameters.FirstOrDefault();
+ if (p0 != null && !string.IsNullOrEmpty(p0.Name)) {
+ // Actual parameter type will be determined when method is invoked.
+ // The reason is that if method might be called on a derived class.
+ // Declare self or cls in this scope.
+ if (declareVariables) {
+ DeclareVariable(p0.Name, new PythonInstance(self), VariableSource.Declaration, p0.NameExpression);
+ }
+ // Set parameter info.
+ var pi = new ParameterInfo(Ast, p0, self, null, false);
+ parameters.Add(pi);
+ skip++;
+ }
+ }
+
+ // Declare parameters in scope
+ for (var i = skip; i < fd.Parameters.Length; i++) {
+ var p = fd.Parameters[i];
+ if (!string.IsNullOrEmpty(p.Name)) {
+ var defaultValue = GetValueFromExpression(p.DefaultValue);
+ var paramType = GetTypeFromAnnotation(p.Annotation, out var isGeneric) ?? UnknownType;
+ if (paramType.IsUnknown()) {
+ // If parameter has default value, look for the annotation locally first
+ // since outer type may be getting redefined. Consider 's = None; def f(s: s = 123): ...
+ paramType = GetTypeFromAnnotation(p.Annotation, out isGeneric, LookupOptions.Local | LookupOptions.Builtins);
+ // Default value of None does not mean the parameter is None, just says it can be missing.
+ defaultValue = defaultValue.IsUnknown() || defaultValue.IsOfType(BuiltinTypeId.NoneType) ? null : defaultValue;
+ if (paramType == null && defaultValue != null) {
+ paramType = defaultValue.GetPythonType();
+ }
+ }
+ // If all else fails, look up globally.
+ var pi = new ParameterInfo(Ast, p, paramType, defaultValue, isGeneric | paramType.IsGeneric());
+ if (declareVariables) {
+ DeclareParameter(p, pi);
+ }
+ parameters.Add(pi);
+ } else if (p.IsList || p.IsDictionary) {
+ parameters.Add(new ParameterInfo(Ast, p, null, null, false));
+ }
+ }
+ return parameters;
+ }
+
+ private void DeclareParameter(Parameter p, ParameterInfo pi) {
+ IPythonType paramType;
+ // If type is known from annotation, use it.
+ if (pi != null && !pi.Type.IsUnknown() && !pi.Type.IsGenericParameter()) {
+ // TODO: technically generics may have constraints. Should we consider them?
+ paramType = pi.Type;
+ } else {
+ paramType = pi?.DefaultValue?.GetPythonType() ?? UnknownType;
+ }
+ DeclareVariable(p.Name, new PythonInstance(paramType), VariableSource.Declaration, p.NameExpression);
+ }
}
}
diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs
index a2060a301..c5baaa8ec 100644
--- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs
+++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs
@@ -117,33 +117,6 @@ public IMember LookupNameInScopes(string name, out IScope scope, out IVariable v
return value;
}
- public IPythonType GetTypeFromAnnotation(Expression expr, LookupOptions options = LookupOptions.Global | LookupOptions.Builtins)
- => GetTypeFromAnnotation(expr, out _, options);
-
- public IPythonType GetTypeFromAnnotation(Expression expr, out bool isGeneric, LookupOptions options = LookupOptions.Global | LookupOptions.Builtins) {
- isGeneric = false;
- switch (expr) {
- case null:
- return null;
- case CallExpression callExpr:
- // x: NamedTuple(...)
- return GetValueFromCallable(callExpr)?.GetPythonType() ?? UnknownType;
- case IndexExpression indexExpr:
- // Try generics
- var target = GetValueFromExpression(indexExpr.Target);
- var result = GetValueFromGeneric(target, indexExpr);
- if (result != null) {
- isGeneric = true;
- return result.GetPythonType();
- }
- break;
- }
-
- // Look at specialization and typing first
- var ann = new TypeAnnotation(Ast.LanguageVersion, expr);
- return ann.GetValue(new TypeAnnotationConverter(this, options));
- }
-
///
/// Locates and opens existing scope for a node or creates a new scope
/// as a child of the specified scope. Scope is pushed on the stack
diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs
index 0a637412c..e094e18e5 100644
--- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs
+++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs
@@ -20,6 +20,7 @@
using Microsoft.Python.Analysis.Diagnostics;
using Microsoft.Python.Analysis.Modules;
using Microsoft.Python.Analysis.Types;
+using Microsoft.Python.Analysis.Utilities;
using Microsoft.Python.Analysis.Values;
using Microsoft.Python.Core;
using Microsoft.Python.Core.Disposables;
@@ -37,12 +38,15 @@ internal sealed partial class ExpressionEval : IExpressionEvaluator {
private readonly object _lock = new object();
private readonly List _diagnostics = new List();
- public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAst ast) {
+ public ExpressionEval(IServiceContainer services, IPythonModule module)
+ : this(services, module, new GlobalScope(module)) { }
+
+ public ExpressionEval(IServiceContainer services, IPythonModule module, GlobalScope gs) {
Services = services ?? throw new ArgumentNullException(nameof(services));
Module = module ?? throw new ArgumentNullException(nameof(module));
- Ast = ast ?? throw new ArgumentNullException(nameof(ast));
+ Ast = module.GetAst();
- GlobalScope = new GlobalScope(module);
+ GlobalScope = gs;
CurrentScope = GlobalScope;
DefaultLocation = new Location(module);
//Log = services.GetService();
@@ -59,14 +63,15 @@ public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAs
public LocationInfo GetLocationInfo(Node node) => node?.GetLocation(Module) ?? LocationInfo.Empty;
public Location GetLocationOfName(Node node) {
- if (node == null || (Module.ModuleType != ModuleType.User && Module.ModuleType != ModuleType.Library)) {
+ node = node ?? throw new ArgumentNullException(nameof(node));
+ if (Module.ModuleType != ModuleType.User && Module.ModuleType != ModuleType.Library) {
return DefaultLocation;
}
IndexSpan indexSpan;
switch (node) {
case MemberExpression mex:
- indexSpan = mex.GetNameSpan().ToIndexSpan(mex.Ast);
+ indexSpan = mex.GetNameSpan(Ast).ToIndexSpan(Ast);
break;
case ClassDefinition cd:
indexSpan = cd.NameExpression.IndexSpan;
@@ -86,7 +91,7 @@ public Location GetLocationOfName(Node node) {
// turns into span at line 1 and very large column. This DOES can
// produce false positives occasionally.
#if DEBUG
- var sourceSpan = indexSpan.ToSourceSpan(node.Ast);
+ var sourceSpan = indexSpan.ToSourceSpan(Ast);
Debug.Assert(sourceSpan.Start.Line > 1 || sourceSpan.Start.Column < 1000);
#endif
return new Location(Module, indexSpan);
diff --git a/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs b/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs
new file mode 100644
index 000000000..42caa54da
--- /dev/null
+++ b/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs
@@ -0,0 +1,92 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.Python.Analysis.Analyzer.Evaluation;
+using Microsoft.Python.Analysis.Diagnostics;
+using Microsoft.Python.Analysis.Documents;
+using Microsoft.Python.Analysis.Modules;
+using Microsoft.Python.Analysis.Values;
+using Microsoft.Python.Core;
+using Microsoft.Python.Core.Diagnostics;
+using Microsoft.Python.Parsing.Ast;
+
+namespace Microsoft.Python.Analysis.Analyzer {
+ ///
+ /// Analysis of a library code.
+ ///
+ internal sealed class LibraryAnalysis : IDocumentAnalysis {
+ public LibraryAnalysis(IDocument document, int version, IServiceContainer services, GlobalScope globalScope, IReadOnlyList starImportMemberNames) {
+ Check.ArgumentNotNull(nameof(document), document);
+ Check.ArgumentNotNull(nameof(globalScope), globalScope);
+
+ Document = document;
+ Version = version;
+ GlobalScope = globalScope;
+
+ var ast = Document.GetAst();
+ ast.Reduce(x => x is ImportStatement || x is FromImportStatement);
+ var c = (IAstNodeContainer)Document;
+ c.ClearContent();
+ c.ClearAst();
+ c.AddAstNode(document, ast);
+
+ ExpressionEvaluator = new ExpressionEval(services, document, globalScope);
+ StarImportMemberNames = starImportMemberNames;
+ }
+
+ #region IDocumentAnalysis
+ ///
+ /// Analyzed document.
+ ///
+ public IDocument Document { get; }
+
+ ///
+ /// Version of the analysis. Usually matches document version,
+ /// but can be lower when document or its dependencies were
+ /// updated since.
+ ///
+ public int Version { get; }
+
+ ///
+ /// Empty AST.
+ ///
+ public PythonAst Ast => ExpressionEvaluator.Ast;
+
+ ///
+ /// Document/module global scope.
+ ///
+ public IGlobalScope GlobalScope { get; }
+
+ ///
+ /// Expression evaluator used in the analysis.
+ /// Only supports scope operation since there is no AST
+ /// when library analysis is complete.
+ ///
+ public IExpressionEvaluator ExpressionEvaluator { get; }
+
+ ///
+ /// Members of the module which are transferred during a star import. null means __all__ was not defined.
+ ///
+ public IReadOnlyList StarImportMemberNames { get; }
+
+ ///
+ /// Analysis diagnostics.
+ ///
+ public IEnumerable Diagnostics => Enumerable.Empty();
+ #endregion
+ }
+}
diff --git a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs
index d3808c35a..2463dfc03 100644
--- a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs
+++ b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs
@@ -37,8 +37,8 @@ internal class ModuleWalker : AnalysisWalker {
private int _allReferencesCount;
private bool _allIsUsable = true;
- public ModuleWalker(IServiceContainer services, IPythonModule module, PythonAst ast)
- : base(new ExpressionEval(services, module, ast)) {
+ public ModuleWalker(IServiceContainer services, IPythonModule module)
+ : base(new ExpressionEval(services, module)) {
_stubAnalysis = Module.Stub is IDocument doc ? doc.GetAnyAnalysis() : null;
}
@@ -207,7 +207,7 @@ public void Complete() {
Eval.ClearCache();
}
- public IGlobalScope GlobalScope => Eval.GlobalScope;
+ public GlobalScope GlobalScope => Eval.GlobalScope;
public IReadOnlyList StarImportMemberNames { get; private set; }
///
@@ -259,7 +259,7 @@ private void MergeStub() {
var memberType = member?.GetPythonType();
var stubMemberType = stubMember.GetPythonType();
if (!IsStubBetterType(memberType, stubMemberType)) {
- continue; ;
+ continue;
}
// Get documentation from the current type, if any, since stubs
@@ -271,11 +271,11 @@ private void MergeStub() {
// Re-declare variable with the data from the stub unless member is a module.
// Modules members that are modules should remain as they are, i.e. os.path
// should remain library with its own stub attached.
- if (!stubType.IsUnknown() && !(stubType is IPythonModule)) {
+ if (!(stubType is IPythonModule)) {
sourceType.TransferDocumentationAndLocation(stubType);
// TODO: choose best type between the scrape and the stub. Stub probably should always win.
var source = Eval.CurrentScope.Variables[v.Name]?.Source ?? VariableSource.Declaration;
- Eval.DeclareVariable(v.Name, v.Value, source, Module);
+ Eval.DeclareVariable(v.Name, v.Value, source);
}
}
}
diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs
index f869680d7..0f3c037fc 100644
--- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs
+++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs
@@ -31,7 +31,6 @@
using Microsoft.Python.Core.Disposables;
using Microsoft.Python.Core.Logging;
using Microsoft.Python.Core.Services;
-using Microsoft.Python.Parsing.Ast;
namespace Microsoft.Python.Analysis.Analyzer {
public sealed class PythonAnalyzer : IPythonAnalyzer, IDisposable {
@@ -65,9 +64,8 @@ public void Dispose() {
_disposeToken.TryMarkDisposed();
}
- public Task WaitForCompleteAnalysisAsync(CancellationToken cancellationToken = default) {
- return _analysisCompleteEvent.WaitAsync(cancellationToken);
- }
+ public Task WaitForCompleteAnalysisAsync(CancellationToken cancellationToken = default)
+ => _analysisCompleteEvent.WaitAsync(cancellationToken);
public async Task GetAnalysisAsync(IPythonModule module, int waitTime, CancellationToken cancellationToken) {
var key = new AnalysisModuleKey(module);
@@ -146,7 +144,7 @@ public void EnqueueDocumentForAnalysis(IPythonModule module, ImmutableArray _analysisTcs;
private IPythonModule _module;
private bool _isUserModule;
- private PythonAst _ast;
private IDocumentAnalysis _previousAnalysis;
private HashSet _parserDependencies;
private HashSet _analysisDependencies;
@@ -102,7 +101,7 @@ public Task GetAnalysisAsync(CancellationToken cancellationTo
public bool IsValidVersion(int version, out IPythonModule module, out PythonAst ast) {
lock (_syncObj) {
module = _module;
- ast = _ast;
+ ast = module.GetAst();
if (ast == null || module == null) {
return false;
}
@@ -210,19 +209,18 @@ public bool Invalidate(ImmutableArray analysisDependencies, int a
}
}
- public bool Invalidate(IPythonModule module, PythonAst ast, int bufferVersion, int analysisVersion, out ImmutableArray dependencies) {
+ public bool Invalidate(IPythonModule module, int bufferVersion, int analysisVersion, out ImmutableArray dependencies) {
dependencies = ImmutableArray.Empty;
if (_bufferVersion >= bufferVersion) {
return false;
}
- var dependenciesHashSet = FindDependencies(module, ast, bufferVersion);
+ var dependenciesHashSet = FindDependencies(module, bufferVersion);
lock (_syncObj) {
if (_analysisVersion >= analysisVersion && _bufferVersion >= bufferVersion) {
return false;
}
- _ast = ast;
_module = module;
_isUserModule = module.ModuleType == ModuleType.User;
_parserDependencies = dependenciesHashSet;
@@ -236,13 +234,13 @@ public bool Invalidate(IPythonModule module, PythonAst ast, int bufferVersion, i
}
}
- private HashSet FindDependencies(IPythonModule module, PythonAst ast, int bufferVersion) {
+ private HashSet FindDependencies(IPythonModule module, int bufferVersion) {
if (_bufferVersion > bufferVersion) {
return new HashSet();
}
- var walker = new DependencyWalker(this, module);
- ast.Walk(walker);
+ var walker = new DependencyWalker(module);
+ module.GetAst().Walk(walker);
var dependencies = walker.Dependencies;
dependencies.Remove(new AnalysisModuleKey(module));
return dependencies;
@@ -264,7 +262,6 @@ private void UpdateAnalysisTcs(int analysisVersion) {
}
private class DependencyWalker : PythonWalker {
- private readonly PythonAnalyzerEntry _entry;
private readonly IPythonModule _module;
private readonly bool _isTypeshed;
private readonly IModuleManagement _moduleResolution;
@@ -272,8 +269,7 @@ private class DependencyWalker : PythonWalker {
public HashSet Dependencies { get; }
- public DependencyWalker(PythonAnalyzerEntry entry, IPythonModule module) {
- _entry = entry;
+ public DependencyWalker(IPythonModule module) {
_module = module;
_isTypeshed = module is StubPythonModule stub && stub.IsTypeshed;
_moduleResolution = module.Interpreter.ModuleResolution;
diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs
index 8ea38f6ea..c0e9215f0 100644
--- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs
+++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs
@@ -21,13 +21,11 @@
using System.Threading.Tasks;
using Microsoft.Python.Analysis.Dependencies;
using Microsoft.Python.Analysis.Diagnostics;
-using Microsoft.Python.Analysis.Documents;
using Microsoft.Python.Analysis.Modules;
using Microsoft.Python.Analysis.Types;
using Microsoft.Python.Core;
using Microsoft.Python.Core.Logging;
using Microsoft.Python.Core.Services;
-using Microsoft.Python.Parsing.Ast;
namespace Microsoft.Python.Analysis.Analyzer {
internal sealed class PythonAnalyzerSession {
@@ -147,7 +145,7 @@ private async Task StartAsync() {
if (!isCanceled) {
_progress.ReportRemaining(remaining);
- if(isFinal) {
+ if (isFinal) {
ActivityTracker.EndTracking();
(_analyzer as PythonAnalyzer)?.RaiseAnalysisComplete(ActivityTracker.ModuleCount, ActivityTracker.MillisecondsElapsed);
_log?.Log(TraceEventType.Verbose, $"Analysis complete: {ActivityTracker.ModuleCount} modules in { ActivityTracker.MillisecondsElapsed} ms.");
@@ -172,7 +170,7 @@ private static void LogResults(ILogger logger, double elapsed, int originalRemai
if (logger == null) {
return;
}
-
+
if (remaining == 0) {
logger.Log(TraceEventType.Verbose, $"Analysis version {version} of {originalRemaining} entries has been completed in {elapsed} ms.");
} else if (remaining < originalRemaining) {
@@ -237,11 +235,11 @@ private Task StartAnalysis(IDependencyChainNode node, Async
/// of dependencies, it is intended for the single file analysis.
///
private void Analyze(IDependencyChainNode node, AsyncCountdownEvent ace, Stopwatch stopWatch) {
- IPythonModule module;
try {
ace?.AddOne();
var entry = node.Value;
- if (!entry.IsValidVersion(_walker.Version, out module, out var ast)) {
+
+ if (!entry.IsValidVersion(_walker.Version, out var module, out var ast)) {
if (ast == null) {
// Entry doesn't have ast yet. There should be at least one more session.
Cancel();
@@ -251,8 +249,9 @@ private void Analyze(IDependencyChainNode node, AsyncCountd
node.Skip();
return;
}
+
var startTime = stopWatch.Elapsed;
- AnalyzeEntry(entry, module, ast, _walker.Version);
+ AnalyzeEntry(entry, module, _walker.Version, node.IsComplete);
node.Commit();
ActivityTracker.OnModuleAnalysisComplete(node.Value.Module.FilePath);
@@ -288,13 +287,13 @@ private void AnalyzeEntry() {
// Entry doesn't have ast yet. There should be at least one more session.
Cancel();
}
-
_log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled.");
return;
}
var startTime = stopWatch?.Elapsed ?? TimeSpan.Zero;
- AnalyzeEntry(_entry, module, ast, Version);
+
+ AnalyzeEntry(_entry, module, Version, true);
LogCompleted(module, stopWatch, startTime);
} catch (OperationCanceledException oce) {
@@ -309,22 +308,26 @@ private void AnalyzeEntry() {
}
}
- private void AnalyzeEntry(PythonAnalyzerEntry entry, IPythonModule module, PythonAst ast, int version) {
+ private void AnalyzeEntry(PythonAnalyzerEntry entry, IPythonModule module, int version, bool isFinalPass) {
+ if (entry.PreviousAnalysis is LibraryAnalysis) {
+ _log?.Log(TraceEventType.Verbose, $"Request to re-analyze finalized {module.Name}.");
+ }
+
// Now run the analysis.
var analyzable = module as IAnalyzable;
analyzable?.NotifyAnalysisBegins();
- var walker = new ModuleWalker(_services, module, ast);
+ var ast = module.GetAst();
+ var walker = new ModuleWalker(_services, module);
ast.Walk(walker);
_analyzerCancellationToken.ThrowIfCancellationRequested();
walker.Complete();
_analyzerCancellationToken.ThrowIfCancellationRequested();
- var analysis = new DocumentAnalysis((IDocument)module, version, walker.GlobalScope, walker.Eval, walker.StarImportMemberNames);
- analyzable?.NotifyAnalysisComplete(analysis);
- entry.TrySetAnalysis(analysis, version);
+ analyzable?.NotifyAnalysisComplete(version, walker, isFinalPass);
+ entry.TrySetAnalysis(module.Analysis, version);
if (module.ModuleType == ModuleType.User) {
var linterDiagnostics = _analyzer.LintModule(module);
diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs
index cd3ecc4ae..847bbc000 100644
--- a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs
+++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs
@@ -43,7 +43,7 @@ public FunctionEvaluator(ExpressionEval eval, PythonFunctionOverload overload)
FunctionDefinition = overload.FunctionDefinition;
}
- public FunctionDefinition FunctionDefinition { get; }
+ private FunctionDefinition FunctionDefinition { get; }
public override void Evaluate() {
var stub = SymbolTable.ReplacedByStubs.Contains(Target)
@@ -52,7 +52,9 @@ public override void Evaluate() {
using (Eval.OpenScope(_function.DeclaringModule, FunctionDefinition, out _)) {
var returnType = TryDetermineReturnValue();
- DeclareParameters(!stub);
+
+ var parameters = Eval.CreateFunctionParameters(_self, _function, FunctionDefinition, !stub);
+ _overload.SetParameters(parameters);
// Do process body of constructors since they may be declaring
// variables that are later used to determine return type of other
@@ -71,6 +73,30 @@ public override void Evaluate() {
Result = _function;
}
+ private IPythonType TryDetermineReturnValue() {
+ var annotationType = Eval.GetTypeFromAnnotation(FunctionDefinition.ReturnAnnotation);
+ if (!annotationType.IsUnknown()) {
+ // Annotations are typically types while actually functions return
+ // instances unless specifically annotated to a type such as Type[T].
+ var t = annotationType.CreateInstance(annotationType.Name, ArgumentSet.Empty);
+ // If instance could not be created, such as when return type is List[T] and
+ // type of T is not yet known, just use the type.
+ var instance = t.IsUnknown() ? annotationType : t;
+ _overload.SetReturnValue(instance, true); _overload.SetReturnValue(instance, true);
+ } else {
+ // Check if function is a generator
+ var suite = FunctionDefinition.Body as SuiteStatement;
+ var yieldExpr = suite?.Statements.OfType().Select(s => s.Expression as YieldExpression).ExcludeDefault().FirstOrDefault();
+ if (yieldExpr != null) {
+ // Function return is an iterator
+ var yieldValue = Eval.GetValueFromExpression(yieldExpr.Expression) ?? Eval.UnknownType;
+ var returnValue = new PythonGenerator(Eval.Interpreter, yieldValue);
+ _overload.SetReturnValue(returnValue, true);
+ }
+ }
+ return annotationType;
+ }
+
public override bool Walk(AssignmentStatement node) {
var value = Eval.GetValueFromExpression(node.Right) ?? Eval.UnknownType;
foreach (var lhs in node.Left) {
@@ -107,93 +133,5 @@ public override bool Walk(FunctionDefinition node) {
}
return false;
}
-
- private IPythonType TryDetermineReturnValue() {
- var annotationType = Eval.GetTypeFromAnnotation(FunctionDefinition.ReturnAnnotation);
- if (!annotationType.IsUnknown()) {
- // Annotations are typically types while actually functions return
- // instances unless specifically annotated to a type such as Type[T].
- var t = annotationType.CreateInstance(annotationType.Name, ArgumentSet.Empty);
- // If instance could not be created, such as when return type is List[T] and
- // type of T is not yet known, just use the type.
- var instance = t.IsUnknown() ? annotationType : t;
- _overload.SetReturnValue(instance, true);
- } else {
- // Check if function is a generator
- var suite = FunctionDefinition.Body as SuiteStatement;
- var yieldExpr = suite?.Statements.OfType().Select(s => s.Expression as YieldExpression).ExcludeDefault().FirstOrDefault();
- if (yieldExpr != null) {
- // Function return is an iterator
- var yieldValue = Eval.GetValueFromExpression(yieldExpr.Expression) ?? Eval.UnknownType;
- var returnValue = new PythonGenerator(Eval.Interpreter, yieldValue);
- _overload.SetReturnValue(returnValue, true);
- }
- }
- return annotationType;
- }
-
- private void DeclareParameters(bool declareVariables) {
- // For class method no need to add extra parameters, but first parameter type should be the class.
- // For static and unbound methods do not add or set anything.
- // For regular bound methods add first parameter and set it to the class.
-
- var parameters = new List();
- var skip = 0;
- if (_self != null && _function.HasClassFirstArgument()) {
- var p0 = FunctionDefinition.Parameters.FirstOrDefault();
- if (p0 != null && !string.IsNullOrEmpty(p0.Name)) {
- // Actual parameter type will be determined when method is invoked.
- // The reason is that if method might be called on a derived class.
- // Declare self or cls in this scope.
- if (declareVariables) {
- Eval.DeclareVariable(p0.Name, new PythonInstance(_self), VariableSource.Declaration, p0.NameExpression);
- }
- // Set parameter info.
- var pi = new ParameterInfo(Ast, p0, _self, null, false);
- parameters.Add(pi);
- skip++;
- }
- }
-
- // Declare parameters in scope
- IMember defaultValue = null;
- for (var i = skip; i < FunctionDefinition.Parameters.Length; i++) {
- var p = FunctionDefinition.Parameters[i];
- if (!string.IsNullOrEmpty(p.Name)) {
- var paramType = Eval.GetTypeFromAnnotation(p.Annotation, out var isGeneric);
- if (paramType.IsUnknown() && p.DefaultValue != null) {
- defaultValue = Eval.GetValueFromExpression(p.DefaultValue);
- // If parameter has default value, look for the annotation locally first
- // since outer type may be getting redefined. Consider 's = None; def f(s: s = 123): ...
- paramType = Eval.GetTypeFromAnnotation(p.Annotation, out isGeneric, LookupOptions.Local | LookupOptions.Builtins);
- // Default value of None does not mean the parameter is None, just says it can be missing.
- defaultValue = defaultValue.IsUnknown() || defaultValue.IsOfType(BuiltinTypeId.NoneType) ? null : defaultValue;
- if (paramType == null && defaultValue != null) {
- paramType = defaultValue.GetPythonType();
- }
- }
- // If all else fails, look up globally.
- paramType = paramType ?? Eval.GetTypeFromAnnotation(p.Annotation, out isGeneric) ?? Eval.UnknownType;
- var pi = new ParameterInfo(Ast, p, paramType, defaultValue, isGeneric | paramType.IsGeneric());
- if (declareVariables) {
- DeclareParameter(p, pi);
- }
- parameters.Add(pi);
- }
- }
- _overload.SetParameters(parameters);
- }
-
- private void DeclareParameter(Parameter p, ParameterInfo pi) {
- IPythonType paramType;
- // If type is known from annotation, use it.
- if (pi != null && !pi.Type.IsUnknown() && !pi.Type.IsGenericParameter()) {
- // TODO: technically generics may have constraints. Should we consider them?
- paramType = pi.Type;
- } else {
- paramType = pi?.DefaultValue?.GetPythonType() ?? Eval.UnknownType;
- }
- Eval.DeclareVariable(p.Name, new PythonInstance(paramType), VariableSource.Declaration, p.NameExpression);
- }
}
}
diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/ModuleSymbolTable.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/ModuleSymbolTable.cs
index ef4848991..31a3b4212 100644
--- a/src/Analysis/Ast/Impl/Analyzer/Symbols/ModuleSymbolTable.cs
+++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/ModuleSymbolTable.cs
@@ -34,7 +34,6 @@ internal sealed class ModuleSymbolTable {
public IEnumerable> Evaluators => _evaluators.ToArray();
public void Add(MemberEvaluator e) => _evaluators[e.Target] = e;
- public MemberEvaluator Get(ScopeStatement target) => _evaluators.TryGetValue(target, out var w) ? w : null;
public bool Contains(ScopeStatement node) => _evaluators.ContainsKey(node) || _processed.Contains(node);
public void Build(ExpressionEval eval)
diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs
index f09974a28..fe45a8fb9 100644
--- a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs
+++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs
@@ -73,7 +73,7 @@ public override bool Walk(FunctionDefinition fd) {
if (IsDeprecated(fd)) {
return false;
}
- if (!string.IsNullOrEmpty(fd.NameExpression?.Name)) {
+ if (!string.IsNullOrEmpty(fd.Name)) {
AddFunctionOrProperty(fd);
// Open function scope
_scopes.Push(_eval.OpenScope(_eval.Module, fd, out _));
@@ -82,7 +82,7 @@ public override bool Walk(FunctionDefinition fd) {
}
public override void PostWalk(FunctionDefinition fd) {
- if (!IsDeprecated(fd) && !string.IsNullOrEmpty(fd.NameExpression?.Name)) {
+ if (!IsDeprecated(fd) && !string.IsNullOrEmpty(fd.Name)) {
_scopes.Pop().Dispose();
}
base.PostWalk(fd);
@@ -119,8 +119,9 @@ private void AddOverload(FunctionDefinition fd, IPythonClassMember function, Act
if (!_table.ReplacedByStubs.Contains(fd)) {
var stubOverload = GetOverloadFromStub(fd);
if (stubOverload != null) {
- if (!string.IsNullOrEmpty(fd.GetDocumentation())) {
- stubOverload.SetDocumentationProvider(_ => fd.GetDocumentation());
+ var documentation = fd.GetDocumentation();
+ if (!string.IsNullOrEmpty(documentation)) {
+ stubOverload.SetDocumentation(documentation);
}
addOverload(stubOverload);
_table.ReplacedByStubs.Add(fd);
@@ -131,7 +132,7 @@ private void AddOverload(FunctionDefinition fd, IPythonClassMember function, Act
if (!_table.Contains(fd)) {
// Do not evaluate parameter types just yet. During light-weight top-level information
// collection types cannot be determined as imports haven't been processed.
- var overload = new PythonFunctionOverload(fd, function, _eval.GetLocationOfName(fd));
+ var overload = new PythonFunctionOverload(function, fd, _eval.GetLocationOfName(fd), fd.ReturnAnnotation?.ToCodeString(_eval.Ast));
addOverload(overload);
_table.Add(new FunctionEvaluator(_eval, overload));
}
@@ -164,14 +165,14 @@ private bool TryAddProperty(FunctionDefinition node, IPythonType declaringType)
return false;
}
- private void AddProperty(FunctionDefinition node, IPythonType declaringType, bool isAbstract) {
- if (!(_eval.LookupNameInScopes(node.Name, LookupOptions.Local) is PythonPropertyType existing)) {
- existing = new PythonPropertyType(node, _eval.GetLocationOfName(node), declaringType, isAbstract);
+ private void AddProperty(FunctionDefinition fd, IPythonType declaringType, bool isAbstract) {
+ if (!(_eval.LookupNameInScopes(fd.Name, LookupOptions.Local) is PythonPropertyType existing)) {
+ existing = new PythonPropertyType(fd, _eval.GetLocationOfName(fd), declaringType, isAbstract);
// The variable is transient (non-user declared) hence it does not have location.
// Property type is tracking locations for references and renaming.
- _eval.DeclareVariable(node.Name, existing, VariableSource.Declaration);
+ _eval.DeclareVariable(fd.Name, existing, VariableSource.Declaration);
}
- AddOverload(node, existing, o => existing.AddOverload(o));
+ AddOverload(fd, existing, o => existing.AddOverload(o));
}
private IMember GetMemberFromStub(string name) {
diff --git a/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs b/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs
index b05802463..1d2e76c6b 100644
--- a/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs
+++ b/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs
@@ -681,6 +681,7 @@ private sealed class DependencyChainNode : IDependencyChainNode {
public TValue Value => _vertex.DependencyVertex.Value;
public bool HasMissingDependencies => _vertex.HasMissingDependencies;
public int VertexDepth { get; }
+ public bool IsComplete => _vertex.SecondPass == null && !HasMissingDependencies;
public DependencyChainNode(DependencyChainWalker walker, WalkingVertex vertex, int depth) {
_walker = walker;
diff --git a/src/Analysis/Ast/Impl/Dependencies/IDependencyChainNode.cs b/src/Analysis/Ast/Impl/Dependencies/IDependencyChainNode.cs
index 5261ee866..299235cc6 100644
--- a/src/Analysis/Ast/Impl/Dependencies/IDependencyChainNode.cs
+++ b/src/Analysis/Ast/Impl/Dependencies/IDependencyChainNode.cs
@@ -24,5 +24,6 @@ internal interface IDependencyChainNode {
TValue Value { get; }
void Commit();
void Skip();
+ bool IsComplete { get; }
}
}
diff --git a/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs b/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs
index 8e53962f2..1a2bf48bc 100644
--- a/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs
+++ b/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs
@@ -53,7 +53,7 @@ private static PythonFunctionType GetOrCreateFunction(this IDocumentAnalysis ana
// We DO want to replace class by function. Consider type() in builtins.
// 'type()' in code is a function call, not a type class instantiation.
if (!(analysis.GlobalScope.Variables[name]?.Value is PythonFunctionType f)) {
- f = PythonFunctionType.ForSpecialization(name, analysis.Document);
+ f = PythonFunctionType.Specialize(name, analysis.Document, string.Empty);
f.AddOverload(new PythonFunctionOverload(name, new Location(analysis.Document)));
analysis.GlobalScope.DeclareVariable(name, f, VariableSource.Declaration);
}
diff --git a/src/Analysis/Ast/Impl/Extensions/AstExtensions.cs b/src/Analysis/Ast/Impl/Extensions/AstExtensions.cs
index c4a220000..a1d2b5d8f 100644
--- a/src/Analysis/Ast/Impl/Extensions/AstExtensions.cs
+++ b/src/Analysis/Ast/Impl/Extensions/AstExtensions.cs
@@ -32,7 +32,7 @@ public static Expression FindExpression(this PythonAst ast, SourceLocation locat
public static string GetDocumentation(this ScopeStatement node) {
var docExpr = (node?.Body as SuiteStatement)?.Statements?.FirstOrDefault() as ExpressionStatement;
var ce = docExpr?.Expression as ConstantExpression;
- return ce?.Value as string;
+ return ce?.GetStringValue();
}
public static bool IsInsideComment(this PythonAst ast, SourceLocation location) {
diff --git a/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs b/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs
index 4610d32ec..e5ccb7bfb 100644
--- a/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs
+++ b/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs
@@ -23,8 +23,8 @@ public static LocationInfo GetLocation(this Node node, IPythonModule module) {
return LocationInfo.Empty;
}
- var start = node.GetStart();
- var end = node.GetEnd();
+ var start = node.GetStart(module.GetAst());
+ var end = node.GetEnd(module.GetAst());
return new LocationInfo(module.FilePath, module.Uri, start.Line, start.Column, end.Line, end.Column);
}
diff --git a/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs b/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs
new file mode 100644
index 000000000..c33b4620f
--- /dev/null
+++ b/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs
@@ -0,0 +1,30 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using Microsoft.Python.Analysis.Modules;
+using Microsoft.Python.Analysis.Types;
+using Microsoft.Python.Parsing.Ast;
+
+namespace Microsoft.Python.Analysis {
+ public static class PythonModuleExtensions {
+ internal static PythonAst GetAst(this IPythonModule module)
+ => ((IAstNodeContainer)module).GetAstNode(module);
+
+ internal static T GetAstNode(this IPythonModule module, object o) where T : Node
+ => ((IAstNodeContainer)module).GetAstNode(o);
+ internal static void AddAstNode(this IPythonModule module, object o, Node n)
+ => ((IAstNodeContainer)module).AddAstNode(o, n);
+ }
+}
diff --git a/src/Analysis/Ast/Impl/Extensions/ScopeExtensions.cs b/src/Analysis/Ast/Impl/Extensions/ScopeExtensions.cs
index 7dc6fdcdb..6ba17ce62 100644
--- a/src/Analysis/Ast/Impl/Extensions/ScopeExtensions.cs
+++ b/src/Analysis/Ast/Impl/Extensions/ScopeExtensions.cs
@@ -21,14 +21,14 @@
namespace Microsoft.Python.Analysis.Analyzer {
public static class ScopeExtensions {
- public static int GetBodyStartIndex(this IScope scope, PythonAst ast) {
+ public static int GetBodyStartIndex(this IScope scope) {
switch (scope.Node) {
case ClassDefinition cd:
return cd.HeaderIndex;
case FunctionDefinition fd:
return fd.HeaderIndex;
default:
- return ast.LocationToIndex(scope.Node.GetStart());
+ return scope.Node.StartIndex;
}
}
@@ -47,7 +47,7 @@ public static IScope FindScope(this IScope parent, IDocument document, SourceLoc
return children[i];
}
- var start = children[i].GetBodyStartIndex(ast);
+ var start = children[i].GetBodyStartIndex();
if (start > index) {
// We've gone past index completely so our last candidate is
// the best one.
@@ -114,10 +114,10 @@ private static int GetParentScopeIndent(IScope scope, PythonAst ast) {
switch (scope.Node) {
case ClassDefinition cd:
// Return column of "class" statement
- return cd.GetStart().Column;
+ return cd.GetStart(ast).Column;
case FunctionDefinition fd when !fd.IsLambda:
// Return column of "def" statement
- return fd.GetStart().Column;
+ return fd.GetStart(ast).Column;
default:
return -1;
}
diff --git a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs
index 597c1f64e..f792b565f 100644
--- a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs
+++ b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs
@@ -121,13 +121,14 @@ private void SpecializeTypes() {
_hiddenNames.Add("__builtin_module_names__");
+ var location = new Location(this, default);
if (_boolType != null) {
- Analysis.GlobalScope.DeclareVariable("True", _boolType, VariableSource.Builtin, new Location(this, default));
- Analysis.GlobalScope.DeclareVariable("False", _boolType, VariableSource.Builtin, new Location(this, default));
+ Analysis.GlobalScope.DeclareVariable("True", _boolType, VariableSource.Builtin, location);
+ Analysis.GlobalScope.DeclareVariable("False", _boolType, VariableSource.Builtin, location);
}
if (noneType != null) {
- Analysis.GlobalScope.DeclareVariable("None", noneType, VariableSource.Builtin, new Location(this, default));
+ Analysis.GlobalScope.DeclareVariable("None", noneType, VariableSource.Builtin, location);
}
foreach (var n in GetMemberNames()) {
diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IHasQualifiedName.cs b/src/Analysis/Ast/Impl/Modules/Definitions/IAstNodeContainer.cs
similarity index 50%
rename from src/Analysis/Ast/Impl/Types/Definitions/IHasQualifiedName.cs
rename to src/Analysis/Ast/Impl/Modules/Definitions/IAstNodeContainer.cs
index 2b9f9a48b..0ed36cb97 100644
--- a/src/Analysis/Ast/Impl/Types/Definitions/IHasQualifiedName.cs
+++ b/src/Analysis/Ast/Impl/Modules/Definitions/IAstNodeContainer.cs
@@ -13,26 +13,29 @@
// See the Apache Version 2.0 License for specific language governing
// permissions and limitations under the License.
-using System;
-using System.Collections.Generic;
+using Microsoft.Python.Analysis.Types;
+using Microsoft.Python.Parsing.Ast;
-namespace Microsoft.Python.Analysis.Types {
- public interface IHasQualifiedName {
+namespace Microsoft.Python.Analysis.Modules {
+ internal interface IAstNodeContainer {
///
- /// Gets the fully qualified, dot-separated name of the value.
- /// This is typically used for displaying to users.
+ /// Provides access to AST nodes associated with objects such as
+ /// for .
+ /// Nodes are not available for library modules as AST is not retained
+ /// in libraries in order to conserve memory.
///
- string FullyQualifiedName { get; }
+ T GetAstNode(object o) where T : Node;
///
- /// Gets the import and eval names of the value. The first part
- /// should be importable, and the second is a name that can be
- /// resolved with getattr().
- /// These are often seen separated with a colon.
+ /// Associated AST node with the object.
///
- ///
- /// The value cannot be resolved (for example, a nested function).
- ///
- KeyValuePair FullyQualifiedNamePair { get; }
+ void AddAstNode(object o, Node n);
+
+ ///
+ /// Removes all AST node associations.
+ ///
+ void ClearAst();
+
+ void ClearContent();
}
}
diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs
index 16ba277a5..4907c28f2 100644
--- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs
+++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs
@@ -42,7 +42,7 @@ namespace Microsoft.Python.Analysis.Modules {
/// to AST and the module analysis.
///
[DebuggerDisplay("{Name} : {ModuleType}")]
- internal class PythonModule : LocatedMember, IDocument, IAnalyzable, IEquatable {
+ internal class PythonModule : LocatedMember, IDocument, IAnalyzable, IEquatable, IAstNodeContainer {
private enum State {
None,
Loading,
@@ -56,13 +56,13 @@ private enum State {
private readonly DocumentBuffer _buffer = new DocumentBuffer();
private readonly DisposeToken _disposeToken = DisposeToken.Create();
private IReadOnlyList _parseErrors = Array.Empty();
+ private readonly Dictionary
- public IPythonModule PrimaryModule { get; internal set; }
-
- protected virtual string LoadContent() {
- if (ContentState < State.Loading) {
- ContentState = State.Loading;
- try {
- var code = FileSystem.ReadTextWithRetry(FilePath);
- ContentState = State.Loaded;
- return code;
- } catch (IOException) { } catch (UnauthorizedAccessException) { }
- }
- return null; // Keep content as null so module can be loaded later.
- }
-
- private void InitializeContent(string content, int version) {
- lock (AnalysisLock) {
- LoadContent(content, version);
-
- var startParse = ContentState < State.Parsing && (_parsingTask == null || version > 0);
- if (startParse) {
- Parse();
- }
- }
- }
-
- private void LoadContent(string content, int version) {
- if (ContentState < State.Loading) {
- try {
- content = content ?? LoadContent();
- _buffer.Reset(version, content);
- ContentState = State.Loaded;
- } catch (IOException) { } catch (UnauthorizedAccessException) { }
- }
- }
+ public IPythonModule PrimaryModule { get; private set; }
#endregion
#region IDisposable
@@ -293,10 +260,10 @@ public async Task GetAstAsync(CancellationToken cancellationToken = d
}
}
cancellationToken.ThrowIfCancellationRequested();
- return _ast;
+ return this.GetAst();
}
- public PythonAst GetAnyAst() => _ast;
+ public PythonAst GetAnyAst() => GetAstNode(this);
///
/// Provides collection of parsing errors, if any.
@@ -370,7 +337,11 @@ private void Parse(CancellationToken cancellationToken) {
if (version != _buffer.Version) {
throw new OperationCanceledException();
}
- _ast = ast;
+
+ // Stored nodes are no longer valid.
+ _astMap.Clear();
+ _astMap[this] = ast;
+
_parseErrors = sink?.Diagnostics ?? Array.Empty();
// Do not report issues with libraries or stubs
@@ -379,6 +350,7 @@ private void Parse(CancellationToken cancellationToken) {
}
ContentState = State.Parsed;
+ Analysis = new EmptyAnalysis(Services, this);
}
NewAst?.Invoke(this, EventArgs.Empty);
@@ -387,7 +359,7 @@ private void Parse(CancellationToken cancellationToken) {
ContentState = State.Analyzing;
var analyzer = Services.GetService();
- analyzer.EnqueueDocumentForAnalysis(this, ast, version);
+ analyzer.EnqueueDocumentForAnalysis(this, version);
}
lock (AnalysisLock) {
@@ -408,6 +380,15 @@ public override void Add(string message, SourceSpan span, int errorCode, Severit
public void NotifyAnalysisBegins() {
lock (AnalysisLock) {
+ if (Analysis is LibraryAnalysis) {
+ var sw = Log != null ? Stopwatch.StartNew() : null;
+ lock (AnalysisLock) {
+ _astMap[this] = RecreateAst();
+ }
+ sw?.Stop();
+ Log?.Log(TraceEventType.Verbose, $"Reloaded AST of {Name} in {sw?.Elapsed.TotalMilliseconds} ms");
+ }
+
if (_updated) {
_updated = false;
// In all variables find those imported, then traverse imported modules
@@ -434,14 +415,14 @@ public void NotifyAnalysisBegins() {
}
}
- public void NotifyAnalysisComplete(IDocumentAnalysis analysis) {
+ public void NotifyAnalysisComplete(int version, ModuleWalker walker, bool isFinalPass) {
lock (AnalysisLock) {
- if (analysis.Version < Analysis.Version) {
+ if (version < Analysis.Version) {
return;
}
- Analysis = analysis;
- GlobalScope = analysis.GlobalScope;
+ Analysis = CreateAnalysis(version, walker, isFinalPass);
+ GlobalScope = Analysis.GlobalScope;
// Derived classes can override OnAnalysisComplete if they want
// to perform additional actions on the completed analysis such
@@ -456,7 +437,7 @@ public void NotifyAnalysisComplete(IDocumentAnalysis analysis) {
// Do not report issues with libraries or stubs
if (ModuleType == ModuleType.User) {
- _diagnosticsService?.Replace(Uri, analysis.Diagnostics, DiagnosticSource.Analysis);
+ _diagnosticsService?.Replace(Uri, Analysis.Diagnostics, DiagnosticSource.Analysis);
}
NewAnalysis?.Invoke(this, EventArgs.Empty);
@@ -465,6 +446,40 @@ public void NotifyAnalysisComplete(IDocumentAnalysis analysis) {
protected virtual void OnAnalysisComplete() { }
#endregion
+ #region IEquatable
+ public bool Equals(IPythonModule other) => Name.Equals(other?.Name) && FilePath.Equals(other?.FilePath);
+ #endregion
+
+ #region IAstNodeContainer
+ public T GetAstNode(object o) where T : Node {
+ lock (AnalysisLock) {
+ return _astMap.TryGetValue(o, out var n) ? (T)n : null;
+ }
+ }
+
+ public void AddAstNode(object o, Node n) {
+ lock (AnalysisLock) {
+ Debug.Assert(!_astMap.ContainsKey(o) || _astMap[o] == n);
+ _astMap[o] = n;
+ }
+ }
+
+ public void ClearAst() {
+ lock (AnalysisLock) {
+ if (ModuleType != ModuleType.User) {
+ _astMap.Clear();
+ }
+ }
+ }
+ public void ClearContent() {
+ lock (AnalysisLock) {
+ if (ModuleType != ModuleType.User) {
+ _buffer.Reset(_buffer.Version, string.Empty);
+ }
+ }
+ }
+ #endregion
+
#region Analysis
public IDocumentAnalysis GetAnyAnalysis() => Analysis;
@@ -473,14 +488,42 @@ public Task GetAnalysisAsync(int waitTime = 200, Cancellation
#endregion
- private void RemoveReferencesInModule(IPythonModule module) {
- if (module.GlobalScope?.Variables != null) {
- foreach (var v in module.GlobalScope.Variables) {
- v.RemoveReferences(this);
+ #region Content management
+ protected virtual string LoadContent() {
+ if (ContentState < State.Loading) {
+ ContentState = State.Loading;
+ try {
+ var code = FileSystem.ReadTextWithRetry(FilePath);
+ ContentState = State.Loaded;
+ return code;
+ } catch (IOException) { } catch (UnauthorizedAccessException) { }
+ }
+ return null; // Keep content as null so module can be loaded later.
+ }
+
+ private void InitializeContent(string content, int version) {
+ lock (AnalysisLock) {
+ LoadContent(content, version);
+
+ var startParse = ContentState < State.Parsing && (_parsingTask == null || version > 0);
+ if (startParse) {
+ Parse();
}
}
}
+ private void LoadContent(string content, int version) {
+ if (ContentState < State.Loading) {
+ try {
+ content = content ?? LoadContent();
+ _buffer.Reset(version, content);
+ ContentState = State.Loaded;
+ } catch (IOException) { } catch (UnauthorizedAccessException) { }
+ }
+ }
+ #endregion
+
+ #region Documentation
private string TryGetDocFromModuleInitFile() {
if (string.IsNullOrEmpty(FilePath) || !FileSystem.FileExists(FilePath)) {
return string.Empty;
@@ -528,7 +571,30 @@ private string TryGetDocFromModuleInitFile() {
} catch (IOException) { } catch (UnauthorizedAccessException) { }
return string.Empty;
}
+ #endregion
- public bool Equals(IPythonModule other) => Name.Equals(other?.Name) && FilePath.Equals(other?.FilePath);
+ private void RemoveReferencesInModule(IPythonModule module) {
+ if (module.GlobalScope?.Variables != null) {
+ foreach (var v in module.GlobalScope.Variables) {
+ v.RemoveReferences(this);
+ }
+ }
+ }
+
+ private IDocumentAnalysis CreateAnalysis(int version, ModuleWalker walker, bool isFinalPass)
+ => ModuleType == ModuleType.Library && isFinalPass
+ ? new LibraryAnalysis(this, version, walker.Eval.Services, walker.GlobalScope, walker.StarImportMemberNames)
+ : (IDocumentAnalysis)new DocumentAnalysis(this, version, walker.GlobalScope, walker.Eval, walker.StarImportMemberNames);
+
+ private PythonAst RecreateAst() {
+ lock (AnalysisLock) {
+ ContentState = State.None;
+ LoadContent();
+ var parser = Parser.CreateParser(new StringReader(_buffer.Text), Interpreter.LanguageVersion, ParserOptions.Default);
+ var ast = parser.ParseFile(Uri);
+ ContentState = State.Parsed;
+ return ast;
+ }
+ }
}
}
diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs
index fe0e3657a..1ddc7d3ed 100644
--- a/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs
+++ b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs
@@ -100,7 +100,7 @@ public IPythonModule GetOrLoadModule(string name) {
return moduleRef.GetOrCreate(name, this);
}
- public bool TryAddModulePath(in string path, in bool allowNonRooted, out string fullModuleName)
+ public bool TryAddModulePath(in string path, in bool allowNonRooted, out string fullModuleName)
=> PathResolver.TryAddModulePath(path, allowNonRooted, out fullModuleName);
public ModulePath FindModule(string filePath) {
@@ -125,13 +125,12 @@ protected void ReloadModulePaths(in IEnumerable rootPaths) {
protected class ModuleRef {
private readonly object _syncObj = new object();
private IPythonModule _module;
- private bool _creating;
public ModuleRef(IPythonModule module) {
_module = module;
}
- public ModuleRef() {}
+ public ModuleRef() { }
public IPythonModule Value {
get {
@@ -142,25 +141,12 @@ public IPythonModule Value {
}
public IPythonModule GetOrCreate(string name, ModuleResolutionBase mrb) {
- var create = false;
lock (_syncObj) {
if (_module != null) {
return _module;
}
- if (!_creating) {
- create = true;
- _creating = true;
- }
- }
-
- if (!create) {
- return null;
- }
-
- var module = mrb.CreateModule(name);
- lock (_syncObj) {
- _creating = false;
+ var module = mrb.CreateModule(name);
_module = module;
return module;
}
diff --git a/src/Analysis/Ast/Impl/Specializations/Specialized.cs b/src/Analysis/Ast/Impl/Specializations/Specialized.cs
index 4da22b82c..e71222bc4 100644
--- a/src/Analysis/Ast/Impl/Specializations/Specialized.cs
+++ b/src/Analysis/Ast/Impl/Specializations/Specialized.cs
@@ -26,9 +26,9 @@ public static IPythonPropertyType Property(string name, IPythonModule declaringM
return prop;
}
- public static IPythonFunctionType Function(string name, IPythonModule declaringModule, IPythonType declaringType, string documentation, IMember returnValue) {
+ public static IPythonFunctionType Function(string name, IPythonModule declaringModule, string documentation, IMember returnValue) {
var location = new Location(declaringModule);
- var prop = new PythonFunctionType(name, location, declaringType, documentation);
+ var prop = PythonFunctionType.Specialize(name, declaringModule, documentation);
var o = new PythonFunctionOverload(prop.Name, location);
o.AddReturnValue(returnValue);
prop.AddOverload(o);
diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs b/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs
index d91afe25c..8e280f25b 100644
--- a/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs
+++ b/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs
@@ -46,7 +46,7 @@ private void SpecializeMembers() {
var location = new Location(this, default);
// TypeVar
- var fn = new PythonFunctionType("TypeVar", location, null, GetMemberDocumentation);
+ var fn = PythonFunctionType.Specialize("TypeVar", this, GetMemberDocumentation("TypeVar"));
var o = new PythonFunctionOverload(fn.Name, location);
// When called, create generic parameter type. For documentation
// use original TypeVar declaration so it appear as a tooltip.
@@ -57,7 +57,7 @@ private void SpecializeMembers() {
_members["TypeVar"] = fn;
// NewType
- fn = new PythonFunctionType("NewType", location, null, GetMemberDocumentation);
+ fn = PythonFunctionType.Specialize("NewType", this, GetMemberDocumentation("NewType"));
o = new PythonFunctionOverload(fn.Name, location);
// When called, create generic parameter type. For documentation
// use original TypeVar declaration so it appear as a tooltip.
@@ -65,8 +65,8 @@ private void SpecializeMembers() {
fn.AddOverload(o);
_members["NewType"] = fn;
- // NewType
- fn = new PythonFunctionType("Type", location, null, GetMemberDocumentation);
+ // Type
+ fn = PythonFunctionType.Specialize("Type", this, GetMemberDocumentation("Type"));
o = new PythonFunctionOverload(fn.Name, location);
// When called, create generic parameter type. For documentation
// use original TypeVar declaration so it appear as a tooltip.
@@ -115,7 +115,7 @@ private void SpecializeMembers() {
_members["Union"] = new GenericType("Union", CreateUnion, this);
- _members["Counter"] = Specialized.Function("Counter", this, null, "Counter",
+ _members["Counter"] = Specialized.Function("Counter", this, GetMemberDocumentation("Counter"),
new PythonInstance(Interpreter.GetBuiltinType(BuiltinTypeId.Int)));
_members["SupportsInt"] = Interpreter.GetBuiltinType(BuiltinTypeId.Int);
@@ -124,7 +124,7 @@ private void SpecializeMembers() {
_members["SupportsBytes"] = Interpreter.GetBuiltinType(BuiltinTypeId.Bytes);
_members["ByteString"] = Interpreter.GetBuiltinType(BuiltinTypeId.Bytes);
- fn = new PythonFunctionType("NamedTuple", location, null, GetMemberDocumentation);
+ fn = PythonFunctionType.Specialize("NamedTuple", this, GetMemberDocumentation("NamedTuple"));
o = new PythonFunctionOverload(fn.Name, location);
o.SetReturnValueProvider((interpreter, overload, args) => CreateNamedTuple(args.Values()));
fn.AddOverload(o);
diff --git a/src/Analysis/Ast/Impl/Types/ArgumentSet.cs b/src/Analysis/Ast/Impl/Types/ArgumentSet.cs
index 344f47dbb..6aa2b5038 100644
--- a/src/Analysis/Ast/Impl/Types/ArgumentSet.cs
+++ b/src/Analysis/Ast/Impl/Types/ArgumentSet.cs
@@ -15,6 +15,7 @@
// permissions and limitations under the License.
using System.Collections.Generic;
+using System.IO;
using System.Linq;
using Microsoft.Python.Analysis.Analyzer;
using Microsoft.Python.Analysis.Analyzer.Evaluation;
@@ -81,10 +82,17 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in
OverloadIndex = overloadIndex;
DeclaringModule = fn.DeclaringModule;
+ if (callExpr == null) {
+ // Typically invoked by specialization code without call expression in the code.
+ // Caller usually does not care about arguments.
+ _evaluated = true;
+ return;
+ }
+
var overload = fn.Overloads[overloadIndex];
var fd = overload.FunctionDefinition;
- if (fd == null || fn.IsSpecialized) {
+ if (fn.IsSpecialized) {
// Typically specialized function, like TypeVar() that does not actually have AST definition.
// Make the arguments from the call expression. If argument does not have name,
// try using name from the function definition based on the argument position.
@@ -92,21 +100,14 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in
for (var i = 0; i < callExpr.Args.Count; i++) {
var name = callExpr.Args[i].Name;
if (string.IsNullOrEmpty(name)) {
- name = fd != null && i < fd.Parameters.Length ? fd.Parameters[i].Name : null;
+ name = i < overload.Parameters.Count ? overload.Parameters[i].Name : $"arg{i}";
}
- name = name ?? $"arg{i}";
- var parameter = fd != null && i < fd.Parameters.Length ? fd.Parameters[i] : null;
- _arguments.Add(new Argument(name, ParameterKind.Normal, callExpr.Args[i].Expression, null, parameter));
+ var node = fd != null && i < fd.Parameters.Length ? fd.Parameters[i] : null;
+ _arguments.Add(new Argument(name, ParameterKind.Normal, callExpr.Args[i].Expression, null, node));
}
return;
}
- if (callExpr == null) {
- // Typically invoked by specialization code without call expression in the code.
- // Caller usually does not care about arguments.
- _evaluated = true;
- return;
- }
var callLocation = callExpr.GetLocation(module);
// https://www.python.org/dev/peps/pep-3102/#id5
@@ -115,7 +116,12 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in
// had values assigned to them are marked as 'filled'.Slots which have
// no value assigned to them yet are considered 'empty'.
- var slots = fd.Parameters.Select(p => new Argument(p, p)).ToArray();
+ var slots = new Argument[overload.Parameters.Count];
+ for (var i = 0; i < overload.Parameters.Count; i++) {
+ var node = fd != null && i < fd.Parameters.Length ? fd.Parameters[i] : null;
+ slots[i] = new Argument(overload.Parameters[i], node);
+ }
+
// Locate sequence argument, if any
var sa = slots.Where(s => s.Kind == ParameterKind.List).ToArray();
if (sa.Length > 1) {
@@ -150,7 +156,7 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in
break;
}
- if (formalParamIndex >= fd.Parameters.Length) {
+ if (formalParamIndex >= overload.Parameters.Count) {
// We ran out of formal parameters and yet haven't seen
// any sequence or dictionary ones. This looks like an error.
_errors.Add(new DiagnosticsEntry(Resources.Analysis_TooManyFunctionArguments, arg.GetLocation(module).Span,
@@ -158,8 +164,8 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in
return;
}
- var formalParam = fd.Parameters[formalParamIndex];
- if (formalParam.IsList) {
+ var formalParam = overload.Parameters[formalParamIndex];
+ if (formalParam.Kind == ParameterKind.List) {
if (string.IsNullOrEmpty(formalParam.Name)) {
// If the next unfilled slot is a vararg slot, and it does not have a name, then it is an error.
_errors.Add(new DiagnosticsEntry(Resources.Analysis_TooManyPositionalArgumentBeforeStar, arg.GetLocation(module).Span,
@@ -188,7 +194,7 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in
break; // Sequence or dictionary parameter found. Done here.
}
- if (formalParam.IsDictionary) {
+ if (formalParam.Kind == ParameterKind.Dictionary) {
// Next slot is a dictionary slot, but we have positional arguments still.
_errors.Add(new DiagnosticsEntry(Resources.Analysis_TooManyPositionalArgumentBeforeStar, arg.GetLocation(module).Span,
ErrorCodes.TooManyPositionalArgumentsBeforeStar, Severity.Warning, DiagnosticSource.Analysis));
@@ -248,7 +254,7 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in
// then it is an error.
foreach (var slot in slots.Where(s => s.Kind != ParameterKind.List && s.Kind != ParameterKind.Dictionary && s.Value == null)) {
if (slot.ValueExpression == null) {
- var parameter = fd.Parameters.First(p => p.Name == slot.Name);
+ var parameter = overload.Parameters.First(p => p.Name == slot.Name);
if (parameter.DefaultValue == null) {
// TODO: parameter is not assigned and has no default value.
_errors.Add(new DiagnosticsEntry(Resources.Analysis_ParameterMissing.FormatUI(slot.Name), callLocation.Span,
@@ -256,7 +262,8 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in
}
// Note that parameter default value expression is from the function definition AST
// while actual argument values are from the calling file AST.
- slot.ValueExpression = parameter.DefaultValue;
+ slot.ValueExpression = CreateExpression(parameter.Name, parameter.DefaultValueString);
+ slot.Value = parameter.DefaultValue;
slot.ValueIsDefault = true;
}
}
@@ -273,7 +280,6 @@ public ArgumentSet Evaluate() {
foreach (var a in _arguments.Where(x => x.Value == null)) {
a.Value = GetArgumentValue(a);
- a.Type = Eval.GetValueFromExpression(a.TypeExpression) as IPythonType;
}
if (_listArgument != null) {
@@ -295,6 +301,9 @@ public ArgumentSet Evaluate() {
}
private IMember GetArgumentValue(Argument arg) {
+ if (arg.Value is IMember m) {
+ return m;
+ }
// Evaluates expression in the specific module context. Typically used to evaluate
// expressions representing default values of function arguments since they are
// are defined in the function declaring module rather than in the caller context.
@@ -310,6 +319,20 @@ private IMember GetArgumentValue(Argument arg) {
return Eval.GetValueFromExpression(arg.ValueExpression) ?? Eval.UnknownType;
}
+ private Expression CreateExpression(string paramName, string defaultValue) {
+ if (string.IsNullOrEmpty(defaultValue)) {
+ return null;
+ }
+ using (var sr = new StringReader($"{paramName}={defaultValue}")) {
+ var parser = Parser.CreateParser(sr, Eval.Interpreter.LanguageVersion,ParserOptions.Default);
+ var ast = parser.ParseFile();
+ if (ast.Body is SuiteStatement ste && ste.Statements.Count > 0 && ste.Statements[0] is AssignmentStatement a) {
+ return a.Right;
+ }
+ return null;
+ }
+ }
+
private sealed class Argument : IArgument {
///
/// Argument name.
@@ -342,26 +365,21 @@ private sealed class Argument : IArgument {
///
/// Type of the argument, if annotated.
///
- public IPythonType Type { get; internal set; }
-
- ///
- /// Type annotation expression.
- ///
- public Expression TypeExpression { get; }
+ public IPythonType Type { get; }
///
/// AST node that defines the argument.
///
public Node Location { get; }
- public Argument(Parameter p, Node location) :
- this(p.Name, p.Kind, null, p.Annotation, location) { }
+ public Argument(IParameterInfo p, Node location) :
+ this(p.Name, p.Kind, null, p.Type, location) { }
- public Argument(string name, ParameterKind kind, Expression valueValueExpression, Expression typeExpression, Node location) {
+ public Argument(string name, ParameterKind kind, Expression valueValueExpression, IPythonType type, Node location) {
Name = name;
Kind = kind;
+ Type = type;
ValueExpression = valueValueExpression;
- TypeExpression = typeExpression;
Location = location;
}
diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IArgumentSet.cs b/src/Analysis/Ast/Impl/Types/Definitions/IArgumentSet.cs
index e4b952b6c..bc1caa61b 100644
--- a/src/Analysis/Ast/Impl/Types/Definitions/IArgumentSet.cs
+++ b/src/Analysis/Ast/Impl/Types/Definitions/IArgumentSet.cs
@@ -44,11 +44,6 @@ public interface IArgument {
///
IPythonType Type { get; }
- ///
- /// Annotation expression.
- ///
- Expression TypeExpression { get; }
-
///
/// Parameter location in the AST.
///
diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IParameterInfo.cs b/src/Analysis/Ast/Impl/Types/Definitions/IParameterInfo.cs
index d0875c2b7..8c79fea2f 100644
--- a/src/Analysis/Ast/Impl/Types/Definitions/IParameterInfo.cs
+++ b/src/Analysis/Ast/Impl/Types/Definitions/IParameterInfo.cs
@@ -14,6 +14,7 @@
// permissions and limitations under the License.
using Microsoft.Python.Core.Text;
+using Microsoft.Python.Parsing.Ast;
namespace Microsoft.Python.Analysis.Types {
///
@@ -36,16 +37,6 @@ public interface IParameterInfo {
///
string Documentation { get; }
- ///
- /// True if the parameter is a *args parameter.
- ///
- bool IsParamArray { get; }
-
- ///
- /// True if the parameter is a **args parameter.
- ///
- bool IsKeywordDict { get; }
-
///
/// Parameter type is generic and specific type will be
/// determined at the time of the function call.
@@ -62,5 +53,10 @@ public interface IParameterInfo {
/// Default value.
///
IMember DefaultValue { get; }
+
+ ///
+ /// Parameter kind.
+ ///
+ ParameterKind Kind { get; }
}
}
diff --git a/src/Analysis/Ast/Impl/Types/Location.cs b/src/Analysis/Ast/Impl/Types/Location.cs
index 8cc2540bf..fd2275707 100644
--- a/src/Analysis/Ast/Impl/Types/Location.cs
+++ b/src/Analysis/Ast/Impl/Types/Location.cs
@@ -30,7 +30,7 @@ public Location(IPythonModule module, IndexSpan indexSpan) {
public LocationInfo LocationInfo {
get {
- var ast = Module?.Analysis.Ast;
+ var ast = Module?.GetAst();
if (ast != null && !string.IsNullOrEmpty(Module?.FilePath) && Module?.Uri != null) {
return new LocationInfo(Module.FilePath, Module.Uri, IndexSpan.ToSourceSpan(ast));
}
@@ -38,7 +38,7 @@ public LocationInfo LocationInfo {
}
}
- public bool IsValid => Module != null;
+ public bool IsValid => Module != null && IndexSpan != default;
public override bool Equals(object obj)
=> obj is Location other && other.Module == Module && other.IndexSpan == IndexSpan;
diff --git a/src/Analysis/Ast/Impl/Types/ParameterInfo.cs b/src/Analysis/Ast/Impl/Types/ParameterInfo.cs
index c6a67058b..07f5a5e5b 100644
--- a/src/Analysis/Ast/Impl/Types/ParameterInfo.cs
+++ b/src/Analysis/Ast/Impl/Types/ParameterInfo.cs
@@ -14,7 +14,6 @@
// permissions and limitations under the License.
using System;
-using Microsoft.Python.Core.Text;
using Microsoft.Python.Parsing.Ast;
namespace Microsoft.Python.Analysis.Types {
@@ -34,17 +33,15 @@ public ParameterInfo(string name, IPythonType type, ParameterKind? kind, IMember
Documentation = string.Empty;
DefaultValue = defaultValue;
Type = type;
- IsParamArray = kind == ParameterKind.List;
- IsKeywordDict = kind == ParameterKind.Dictionary;
+ Kind = kind ?? ParameterKind.Normal;
}
public string Name { get; }
public string Documentation { get; }
- public bool IsParamArray { get; }
- public bool IsKeywordDict { get; }
public bool IsGeneric { get; }
public IPythonType Type { get; }
public string DefaultValueString { get; }
public IMember DefaultValue { get; }
+ public ParameterKind Kind { get; }
}
}
diff --git a/src/Analysis/Ast/Impl/Types/PythonClassType.cs b/src/Analysis/Ast/Impl/Types/PythonClassType.cs
index bcf546d59..850cf4893 100644
--- a/src/Analysis/Ast/Impl/Types/PythonClassType.cs
+++ b/src/Analysis/Ast/Impl/Types/PythonClassType.cs
@@ -50,7 +50,7 @@ public PythonClassType(
BuiltinTypeId builtinTypeId = BuiltinTypeId.Type
) : base(classDefinition.Name, location, classDefinition.GetDocumentation(), builtinTypeId) {
Check.ArgumentNotNull(nameof(location), location.Module);
- ClassDefinition = classDefinition;
+ location.Module.AddAstNode(this, classDefinition);
}
#region IPythonType
@@ -163,7 +163,7 @@ public override IMember Index(IPythonInstance instance, object index) {
#endregion
#region IPythonClass
- public ClassDefinition ClassDefinition { get; }
+ public ClassDefinition ClassDefinition => DeclaringModule.GetAstNode(this);
public IReadOnlyList Bases => _bases;
public IReadOnlyList Mro {
diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs
index 00402f0cb..0e5f90a51 100644
--- a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs
+++ b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs
@@ -18,7 +18,6 @@
using System.Diagnostics;
using System.Linq;
using Microsoft.Python.Analysis.Analyzer.Evaluation;
-using Microsoft.Python.Analysis.Documents;
using Microsoft.Python.Analysis.Specializations.Typing;
using Microsoft.Python.Analysis.Values;
using Microsoft.Python.Core;
@@ -48,15 +47,14 @@ internal sealed class PythonFunctionOverload : LocatedMember, IPythonFunctionOve
// Return value can be an instance or a type info. Consider type(C()) returning
// type info of C vs. return C() that returns an instance of C.
- private Func _documentationProvider;
private bool _fromAnnotation;
- public PythonFunctionOverload(FunctionDefinition fd, IPythonClassMember classMember, Location location)
- : this(fd.Name, location) {
- FunctionDefinition = fd;
- ClassMember = classMember;
- var ast = (location.Module as IDocument)?.Analysis.Ast;
- _returnDocumentation = ast != null ? fd.ReturnAnnotation?.ToCodeString(ast) : null;
+ public PythonFunctionOverload(IPythonClassMember cm, FunctionDefinition fd, Location location, string returnDocumentation)
+ : this(cm.Name, location) {
+ ClassMember = cm;
+ Documentation = fd.GetDocumentation();
+ cm.DeclaringModule.AddAstNode(this, fd);
+ _returnDocumentation = returnDocumentation;
}
public PythonFunctionOverload(string name, Location location) : base(location) {
@@ -69,9 +67,6 @@ public PythonFunctionOverload(string name, Location location) : base(location) {
internal void SetParameters(IReadOnlyList parameters) => Parameters = parameters;
- internal void SetDocumentationProvider(Func documentationProvider)
- => _documentationProvider = _documentationProvider ?? documentationProvider;
-
internal void AddReturnValue(IMember value) {
if (value.IsUnknown()) {
return; // Don't add useless values.
@@ -97,24 +92,14 @@ internal void SetReturnValue(IMember value, bool fromAnnotation) {
_fromAnnotation = fromAnnotation;
}
- internal void SetReturnValueProvider(ReturnValueProvider provider)
- => _returnValueProvider = provider;
+ internal void SetReturnValueProvider(ReturnValueProvider provider) => _returnValueProvider = provider;
+ internal void SetDocumentation(string documentation) => Documentation = documentation;
#region IPythonFunctionOverload
-
- public FunctionDefinition FunctionDefinition { get; }
+ public FunctionDefinition FunctionDefinition => ClassMember?.DeclaringModule?.GetAstNode(this);
public IPythonClassMember ClassMember { get; }
public string Name { get; }
-
- public string Documentation {
- get {
- var s = _documentationProvider?.Invoke(Name);
- if (string.IsNullOrEmpty(s)) {
- s = FunctionDefinition.GetDocumentation();
- }
- return s ?? string.Empty;
- }
- }
+ public string Documentation { get; private set; }
public string GetReturnDocumentation(IPythonType self = null) {
if (self != null) {
@@ -157,8 +142,10 @@ private IMember GetSpecificReturnType(IPythonClassType selfClassType, IArgumentS
case IGenericType gt when args != null: // -> CLASS[T] on standalone function (i.e. -> List[T]).
var typeArgs = ExpressionEval.GetTypeArgumentsFromParameters(this, args);
- Debug.Assert(typeArgs != null);
- return gt.CreateSpecificType(typeArgs);
+ if (typeArgs != null) {
+ return gt.CreateSpecificType(typeArgs);
+ }
+ break;
}
return StaticReturnValue;
diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs
index 6b14f81bc..82c1def63 100644
--- a/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs
+++ b/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs
@@ -13,7 +13,6 @@
// See the Apache Version 2.0 License for specific language governing
// permissions and limitations under the License.
-using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
@@ -25,20 +24,19 @@
namespace Microsoft.Python.Analysis.Types {
[DebuggerDisplay("Function {Name} ({TypeId})")]
- internal class PythonFunctionType : PythonType, IPythonFunctionType {
+ internal sealed class PythonFunctionType : PythonType, IPythonFunctionType {
private ImmutableArray _overloads = ImmutableArray.Empty;
- private readonly string _documentation;
private bool _isAbstract;
private bool _isSpecialized;
///
/// Creates function for specializations
///
- public static PythonFunctionType ForSpecialization(string name, IPythonModule declaringModule)
- => new PythonFunctionType(name, new Location(declaringModule, default), true);
+ public static PythonFunctionType Specialize(string name, IPythonModule declaringModule, string documentation)
+ => new PythonFunctionType(name, new Location(declaringModule, default), documentation, true);
- private PythonFunctionType(string name, Location location, bool isSpecialized = false) :
- base(name, location, string.Empty, BuiltinTypeId.Function) {
+ private PythonFunctionType(string name, Location location, string documentation, bool isSpecialized = false) :
+ base(name, location, documentation ?? string.Empty, BuiltinTypeId.Function) {
Check.ArgumentNotNull(nameof(location), location.Module);
_isSpecialized = isSpecialized;
}
@@ -53,21 +51,7 @@ public PythonFunctionType(
Location location,
IPythonType declaringType,
string documentation
- ) : this(name, location, declaringType, _ => documentation) {
- Check.ArgumentNotNull(nameof(location), location.Module);
- }
-
- ///
- /// Creates function type to use in special cases when function is dynamically
- /// created, such as in specializations and custom iterators, without the actual
- /// function definition in the AST.
- ///
- public PythonFunctionType(
- string name,
- Location location,
- IPythonType declaringType,
- Func documentationProvider
- ) : base(name, location, documentationProvider, declaringType != null ? BuiltinTypeId.Method : BuiltinTypeId.Function) {
+ ) : base(name, location, documentation, declaringType != null ? BuiltinTypeId.Method : BuiltinTypeId.Function) {
DeclaringType = declaringType;
}
@@ -75,24 +59,16 @@ public PythonFunctionType(
FunctionDefinition fd,
IPythonType declaringType,
Location location
- ) : base(fd.Name, location, fd.Documentation,
- declaringType != null ? BuiltinTypeId.Method : BuiltinTypeId.Function) {
-
- FunctionDefinition = fd;
+ ) : base(fd.Name, location,
+ fd.Name == "__init__" ? (declaringType?.Documentation ?? fd.GetDocumentation()) : fd.GetDocumentation(),
+ declaringType != null ? BuiltinTypeId.Method : BuiltinTypeId.Function) {
DeclaringType = declaringType;
- // For __init__ documentation may either come from the function node of the the declaring
- // type. Note that if there is no documentation on the class node, the class will try and
- // get documentation from its __init__ function, delegating down to this type. So we need
- // to set documentation statically for __init__ here or we may end up/ with stack overflows.
- if (fd.Name == "__init__") {
- _documentation = declaringType?.Documentation ?? fd.Documentation;
- }
+ location.Module.AddAstNode(this, fd);
ProcessDecorators(fd);
}
#region IPythonType
-
public override PythonMemberType MemberType
=> TypeId == BuiltinTypeId.Function ? PythonMemberType.Function : PythonMemberType.Method;
@@ -102,22 +78,21 @@ public override IMember Call(IPythonInstance instance, string memberName, IArgum
return overload?.Call(args, instance?.GetPythonType() ?? DeclaringType);
}
- internal override void SetDocumentationProvider(Func provider) {
+ internal override void SetDocumentation(string documentation) {
foreach (var o in Overloads) {
- (o as PythonFunctionOverload)?.SetDocumentationProvider(provider);
+ (o as PythonFunctionOverload)?.SetDocumentation(documentation);
}
-
- base.SetDocumentationProvider(provider);
+ base.SetDocumentation(documentation);
}
#endregion
#region IPythonFunction
- public FunctionDefinition FunctionDefinition { get; }
+ public FunctionDefinition FunctionDefinition => DeclaringModule.GetAstNode(this);
public IPythonType DeclaringType { get; }
- public override string Documentation => _documentation ?? base.Documentation ?? (_overloads.Count > 0 ? _overloads[0].Documentation : default);
- public virtual bool IsClassMethod { get; private set; }
- public virtual bool IsStatic { get; private set; }
+ public override string Documentation => (_overloads.Count > 0 ? _overloads[0].Documentation : default) ?? base.Documentation;
+ public bool IsClassMethod { get; private set; }
+ public bool IsStatic { get; private set; }
public override bool IsAbstract => _isAbstract;
public override bool IsSpecialized => _isSpecialized;
@@ -128,12 +103,6 @@ internal override void SetDocumentationProvider(Func provider) {
public IReadOnlyList Overloads => _overloads;
#endregion
- #region IHasQualifiedName
- public override string FullyQualifiedName => FullyQualifiedNamePair.CombineNames();
- public override KeyValuePair FullyQualifiedNamePair =>
- new KeyValuePair((DeclaringType as IHasQualifiedName)?.FullyQualifiedName ?? DeclaringType?.Name ?? DeclaringModule?.Name, Name);
- #endregion
-
internal void Specialize(string[] dependencies) {
_isSpecialized = true;
Dependencies = dependencies != null
diff --git a/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs b/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs
index 87ee5f713..f10078945 100644
--- a/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs
+++ b/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs
@@ -23,7 +23,7 @@ class PythonPropertyType : PythonType, IPythonPropertyType {
public PythonPropertyType(FunctionDefinition fd, Location location, IPythonType declaringType, bool isAbstract)
: this(fd.Name, location, declaringType, isAbstract) {
- FunctionDefinition = fd;
+ declaringType.DeclaringModule.AddAstNode(this, fd);
}
public PythonPropertyType(string name, Location location, IPythonType declaringType, bool isAbstract)
@@ -37,7 +37,7 @@ public PythonPropertyType(string name, Location location, IPythonType declaringT
#endregion
#region IPythonPropertyType
- public FunctionDefinition FunctionDefinition { get; }
+ public FunctionDefinition FunctionDefinition => DeclaringModule.GetAstNode(this);
public override bool IsAbstract { get; }
public bool IsReadOnly => true;
public IPythonType DeclaringType { get; }
diff --git a/src/Analysis/Ast/Impl/Types/PythonType.cs b/src/Analysis/Ast/Impl/Types/PythonType.cs
index 41e9bda1c..27c8ffaff 100644
--- a/src/Analysis/Ast/Impl/Types/PythonType.cs
+++ b/src/Analysis/Ast/Impl/Types/PythonType.cs
@@ -22,11 +22,9 @@
namespace Microsoft.Python.Analysis.Types {
[DebuggerDisplay("{Name}")]
- internal class PythonType : LocatedMember, IPythonType, IHasQualifiedName, IEquatable {
+ internal class PythonType : LocatedMember, IPythonType {//, IEquatable {
private readonly object _lock = new object();
private readonly string _name;
- private Func _documentationProvider;
- private string _documentation;
private Dictionary _members;
private BuiltinTypeId _typeId;
private bool _readonly;
@@ -42,16 +40,7 @@ public PythonType(
string documentation,
BuiltinTypeId typeId = BuiltinTypeId.Unknown
) : this(name, location, typeId) {
- _documentation = documentation;
- }
-
- public PythonType(
- string name,
- Location location,
- Func documentationProvider,
- BuiltinTypeId typeId = BuiltinTypeId.Unknown
- ) : this(name, location, typeId) {
- _documentationProvider = documentationProvider;
+ Documentation = documentation;
}
private PythonType(string name, Location location, BuiltinTypeId typeId) : base(location) {
@@ -67,7 +56,7 @@ private PythonType(string name, Location location, BuiltinTypeId typeId) : base(
#region IPythonType
public virtual string Name => TypeId == BuiltinTypeId.Ellipsis ? "..." : _name;
- public virtual string Documentation => _documentationProvider != null ? _documentationProvider.Invoke(Name) : _documentation;
+ public virtual string Documentation { get; private set; }
public virtual BuiltinTypeId TypeId => _typeId;
public bool IsBuiltin => DeclaringModule == null || DeclaringModule is IBuiltinsPythonModule;
public virtual bool IsAbstract => false;
@@ -98,12 +87,6 @@ public virtual IMember Call(IPythonInstance instance, string memberName, IArgume
public virtual IMember Index(IPythonInstance instance, object index) => instance?.Index(index) ?? UnknownType;
#endregion
- #region IHasQualifiedName
- public virtual string FullyQualifiedName => FullyQualifiedNamePair.CombineNames();
- public virtual KeyValuePair FullyQualifiedNamePair
- => new KeyValuePair(DeclaringModule?.Name ?? string.Empty, Name);
- #endregion
-
#region IMemberContainer
public virtual IMember GetMember(string name) => Members.TryGetValue(name, out var member) ? member : null;
public virtual IEnumerable GetMemberNames() => Members.Keys;
@@ -117,8 +100,7 @@ internal bool TrySetTypeId(BuiltinTypeId typeId) {
return true;
}
- internal virtual void SetDocumentationProvider(Func provider) => _documentationProvider = provider;
- internal virtual void SetDocumentation(string documentation) => _documentation = documentation;
+ internal virtual void SetDocumentation(string documentation) => Documentation = documentation;
internal void AddMembers(IEnumerable variables, bool overwrite) {
lock (_lock) {
@@ -167,10 +149,10 @@ internal IMember AddMember(string name, IMember member, bool overwrite) {
protected bool ContainsMember(string name) => Members.ContainsKey(name);
protected IMember UnknownType => DeclaringModule.Interpreter.UnknownType;
- public bool Equals(IPythonType other) => PythonTypeComparer.Instance.Equals(this, other);
+ //public bool Equals(IPythonType other) => PythonTypeComparer.Instance.Equals(this, other);
- public override bool Equals(object obj)
- => obj is IPythonType pt && PythonTypeComparer.Instance.Equals(this, pt);
- public override int GetHashCode() => 0;
+ //public override bool Equals(object obj)
+ // => obj is IPythonType pt && PythonTypeComparer.Instance.Equals(this, pt);
+ //public override int GetHashCode() => 0;
}
}
diff --git a/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs b/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs
index 5201297b1..8058317cc 100644
--- a/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs
+++ b/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs
@@ -22,7 +22,7 @@ namespace Microsoft.Python.Analysis.Types {
///
/// Delegates most of the methods to the wrapped/inner class.
///
- internal class PythonTypeWrapper : IPythonType, IHasQualifiedName {
+ internal class PythonTypeWrapper : IPythonType {
private readonly BuiltinTypeId _builtinTypeId;
private IPythonType _innerType;
@@ -86,11 +86,6 @@ public virtual IMember Index(IPythonInstance instance, object index)
public virtual IEnumerable GetMemberNames() => InnerType.GetMemberNames();
#endregion
- #region IHasQualifiedName
- public virtual string FullyQualifiedName => (InnerType as IHasQualifiedName)?.FullyQualifiedName;
- public virtual KeyValuePair FullyQualifiedNamePair => (InnerType as IHasQualifiedName)?.FullyQualifiedNamePair ?? default;
- #endregion
-
protected IMember UnknownType => DeclaringModule.Interpreter.UnknownType;
public override bool Equals(object obj)
diff --git a/src/Analysis/Ast/Impl/Utilities/AstUtilities.cs b/src/Analysis/Ast/Impl/Utilities/AstUtilities.cs
new file mode 100644
index 000000000..816b0c304
--- /dev/null
+++ b/src/Analysis/Ast/Impl/Utilities/AstUtilities.cs
@@ -0,0 +1,29 @@
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+using System;
+using System.IO;
+using Microsoft.Python.Parsing;
+using Microsoft.Python.Parsing.Ast;
+
+namespace Microsoft.Python.Analysis.Utilities {
+ public static class AstUtilities {
+ public static PythonAst MakeEmptyAst(Uri documentUri) {
+ using (var sr = new StringReader(string.Empty)) {
+ return Parser.CreateParser(sr, PythonLanguageVersion.None).ParseFile(documentUri);
+ }
+ }
+ }
+}
diff --git a/src/Analysis/Ast/Impl/Values/GlobalScope.cs b/src/Analysis/Ast/Impl/Values/GlobalScope.cs
index 42e42e41d..699809d20 100644
--- a/src/Analysis/Ast/Impl/Values/GlobalScope.cs
+++ b/src/Analysis/Ast/Impl/Values/GlobalScope.cs
@@ -23,25 +23,27 @@ public GlobalScope(IPythonModule module): base(null, null, module) {
DeclareBuiltinVariables();
}
- public override ScopeStatement Node => Module.Analysis?.Ast;
+ public override ScopeStatement Node => Module.GetAst();
private void DeclareBuiltinVariables() {
if (Module.ModuleType != ModuleType.User) {
return;
}
+ var location = new Location(Module, default);
+
var boolType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Bool);
var strType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Str);
var listType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.List);
var dictType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Dict);
- DeclareVariable("__debug__", boolType, VariableSource.Builtin);
- DeclareVariable("__doc__", strType, VariableSource.Builtin);
- DeclareVariable("__file__", strType, VariableSource.Builtin);
- DeclareVariable("__name__", strType, VariableSource.Builtin);
- DeclareVariable("__package__", strType, VariableSource.Builtin);
- DeclareVariable("__path__", listType, VariableSource.Builtin);
- DeclareVariable("__dict__", dictType, VariableSource.Builtin);
+ DeclareVariable("__debug__", boolType, VariableSource.Builtin, location);
+ DeclareVariable("__doc__", strType, VariableSource.Builtin, location);
+ DeclareVariable("__file__", strType, VariableSource.Builtin, location);
+ DeclareVariable("__name__", strType, VariableSource.Builtin, location);
+ DeclareVariable("__package__", strType, VariableSource.Builtin, location);
+ DeclareVariable("__path__", listType, VariableSource.Builtin, location);
+ DeclareVariable("__dict__", dictType, VariableSource.Builtin, location);
}
}
}
diff --git a/src/Analysis/Ast/Impl/Values/Scope.cs b/src/Analysis/Ast/Impl/Values/Scope.cs
index c62e93d58..d517c4457 100644
--- a/src/Analysis/Ast/Impl/Values/Scope.cs
+++ b/src/Analysis/Ast/Impl/Values/Scope.cs
@@ -32,16 +32,17 @@ internal class Scope : IScope {
private List _childScopes;
public Scope(ScopeStatement node, IScope outerScope, IPythonModule module) {
- Node = node;
OuterScope = outerScope;
Module = module;
+ if (node != null) {
+ Module.AddAstNode(this, node);
+ }
DeclareBuiltinVariables();
}
#region IScope
-
public string Name => Node?.Name ?? "";
- public virtual ScopeStatement Node { get; }
+ public virtual ScopeStatement Node => Module.GetAstNode(this);
public IScope OuterScope { get; }
public IPythonModule Module { get; }
@@ -95,24 +96,25 @@ private void DeclareBuiltinVariables() {
return;
}
+ var location = new Location(Module, default);
var strType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Str);
var objType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Object);
- VariableCollection.DeclareVariable("__name__", strType, VariableSource.Builtin);
+ VariableCollection.DeclareVariable("__name__", strType, VariableSource.Builtin, location);
if (Node is FunctionDefinition) {
var dictType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Dict);
var tupleType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Tuple);
- VariableCollection.DeclareVariable("__closure__", tupleType, VariableSource.Builtin);
- VariableCollection.DeclareVariable("__code__", objType, VariableSource.Builtin);
- VariableCollection.DeclareVariable("__defaults__", tupleType, VariableSource.Builtin);
- VariableCollection.DeclareVariable("__dict__", dictType, VariableSource.Builtin);
- VariableCollection.DeclareVariable("__doc__", strType, VariableSource.Builtin);
- VariableCollection.DeclareVariable("__func__", objType, VariableSource.Builtin);
- VariableCollection.DeclareVariable("__globals__", dictType, VariableSource.Builtin);
+ VariableCollection.DeclareVariable("__closure__", tupleType, VariableSource.Builtin, location);
+ VariableCollection.DeclareVariable("__code__", objType, VariableSource.Builtin, location);
+ VariableCollection.DeclareVariable("__defaults__", tupleType, VariableSource.Builtin, location);
+ VariableCollection.DeclareVariable("__dict__", dictType, VariableSource.Builtin, location);
+ VariableCollection.DeclareVariable("__doc__", strType, VariableSource.Builtin, location);
+ VariableCollection.DeclareVariable("__func__", objType, VariableSource.Builtin, location);
+ VariableCollection.DeclareVariable("__globals__", dictType, VariableSource.Builtin, location);
} else if (Node is ClassDefinition) {
- VariableCollection.DeclareVariable("__self__", objType, VariableSource.Builtin);
+ VariableCollection.DeclareVariable("__self__", objType, VariableSource.Builtin, location);
}
}
}
diff --git a/src/Analysis/Ast/Impl/Values/VariableCollection.cs b/src/Analysis/Ast/Impl/Values/VariableCollection.cs
index 817456eca..878182211 100644
--- a/src/Analysis/Ast/Impl/Values/VariableCollection.cs
+++ b/src/Analysis/Ast/Impl/Values/VariableCollection.cs
@@ -98,7 +98,7 @@ public IEnumerable GetMemberNames() {
#endregion
- internal void DeclareVariable(string name, IMember value, VariableSource source, Location location = default) {
+ internal void DeclareVariable(string name, IMember value, VariableSource source, Location location) {
name = !string.IsNullOrWhiteSpace(name) ? name : throw new ArgumentException(nameof(name));
lock (_syncObj) {
if (_variables.TryGetValue(name, out var existing)) {
diff --git a/src/Analysis/Ast/Test/ArgumentSetTests.cs b/src/Analysis/Ast/Test/ArgumentSetTests.cs
index fc613182a..2f4bee636 100644
--- a/src/Analysis/Ast/Test/ArgumentSetTests.cs
+++ b/src/Analysis/Ast/Test/ArgumentSetTests.cs
@@ -347,9 +347,14 @@ from builtins import pow
[TestMethod, Priority(0)]
public async Task DefaultArgumentAnotherFile() {
const string code = @"
-from DefaultArgument import func
+from .module2 import func
func()
";
+ const string code2 = @"
+class A: ...
+def func(a = A()): ...
+";
+ await TestData.CreateTestSpecificFileAsync("module2.py", code2);
var argSet = await GetArgSetAsync(code, "func");
argSet.Arguments.Count.Should().Be(1);
argSet.Evaluate();
@@ -361,7 +366,6 @@ from DefaultArgument import func
t.MemberType.Should().Be(PythonMemberType.Class);
}
-
private async Task GetArgSetAsync(string code, string funcName = "f") {
var analysis = await GetAnalysisAsync(code);
var f = analysis.Should().HaveFunction(funcName).Which;
diff --git a/src/Analysis/Ast/Test/FluentAssertions/AssertionsUtilities.cs b/src/Analysis/Ast/Test/FluentAssertions/AssertionsUtilities.cs
index a30be146a..9be29ed15 100644
--- a/src/Analysis/Ast/Test/FluentAssertions/AssertionsUtilities.cs
+++ b/src/Analysis/Ast/Test/FluentAssertions/AssertionsUtilities.cs
@@ -31,9 +31,9 @@ public static bool Is3X(IScope scope)
=> scope.GlobalScope.Module.Interpreter.LanguageVersion.Is3x();
public static void AssertTypeIds(
- IEnumerable actualTypeIds,
- IEnumerable typeIds,
- string name, bool languageVersionIs3X, string because,
+ IEnumerable actualTypeIds,
+ IEnumerable typeIds,
+ string name, bool languageVersionIs3X, string because,
object[] reasonArgs, string itemNameSingle = "type", string itemNamePlural = "types") {
var expected = typeIds.Select(t => {
switch (t) {
@@ -163,22 +163,12 @@ private static StringBuilder AppendQuotedName(this StringBuilder stringBuilder,
}
public static string GetQuotedName(object value) {
- string name;
- switch (value) {
- case IHasQualifiedName _:
- case IPythonModule _:
- name = GetName(value);
- return string.IsNullOrEmpty(name) ? string.Empty : $"'{name}'";
- default:
- name = GetName(value);
- return string.IsNullOrEmpty(name) ? string.Empty : $"'{name}'";
- }
+ var name = GetName(value);
+ return string.IsNullOrEmpty(name) ? string.Empty : $"'{name}'";
}
public static string GetName(object value) {
switch (value) {
- case IHasQualifiedName qualifiedName:
- return qualifiedName.FullyQualifiedName;
case IPythonModule pythonModule:
return pythonModule.Name;
case IScope scope:
diff --git a/src/Analysis/Ast/Test/FunctionTests.cs b/src/Analysis/Ast/Test/FunctionTests.cs
index d707be2eb..7ebefdfdb 100644
--- a/src/Analysis/Ast/Test/FunctionTests.cs
+++ b/src/Analysis/Ast/Test/FunctionTests.cs
@@ -356,7 +356,6 @@ def e(cls): pass
[TestMethod, Priority(0)]
public async Task OverloadsParamTypeMatch() {
const string code = @"
-
def f(a: bool) -> None: ...
def f(a: int) -> float: ...
def f(a: str) -> bytes: ...
diff --git a/src/Analysis/Ast/Test/PepHintTests.cs b/src/Analysis/Ast/Test/PepHintTests.cs
index 91db0df13..8964ddc7f 100644
--- a/src/Analysis/Ast/Test/PepHintTests.cs
+++ b/src/Analysis/Ast/Test/PepHintTests.cs
@@ -13,14 +13,12 @@
// See the Apache Version 2.0 License for specific language governing
// permissions and limitations under the License.
-using System.Linq;
using System.Threading.Tasks;
using FluentAssertions;
using Microsoft.Python.Analysis.Tests.FluentAssertions;
using Microsoft.Python.Analysis.Types;
using Microsoft.Python.Analysis.Values;
using Microsoft.Python.Parsing.Tests;
-using Microsoft.Python.Tests.Utilities.FluentAssertions;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using TestUtilities;
diff --git a/src/Analysis/Ast/Test/ValuesTests.cs b/src/Analysis/Ast/Test/ValuesTests.cs
index d200604a3..007c76b25 100644
--- a/src/Analysis/Ast/Test/ValuesTests.cs
+++ b/src/Analysis/Ast/Test/ValuesTests.cs
@@ -15,7 +15,6 @@
using System.IO;
using System.Threading.Tasks;
-using FluentAssertions;
using Microsoft.Python.Analysis.Tests.FluentAssertions;
using Microsoft.Python.Analysis.Types;
using Microsoft.VisualStudio.TestTools.UnitTesting;
diff --git a/src/LanguageServer/Impl/Completion/ExpressionLocator.cs b/src/LanguageServer/Impl/Completion/ExpressionLocator.cs
index 9ce9cd0f6..f0835cb87 100644
--- a/src/LanguageServer/Impl/Completion/ExpressionLocator.cs
+++ b/src/LanguageServer/Impl/Completion/ExpressionLocator.cs
@@ -44,17 +44,17 @@ public static void FindExpression(PythonAst ast, SourceLocation position, FindEx
expression = expression ?? (statement as ExpressionStatement)?.Expression;
}
- private static bool CanBackUp(PythonAst tree, Node node, Node statement, ScopeStatement scope, int column) {
+ private static bool CanBackUp(PythonAst ast, Node node, Node statement, ScopeStatement scope, int column) {
if (node != null || !((statement as ExpressionStatement)?.Expression is ErrorExpression)) {
return false;
}
var top = 1;
if (scope != null) {
- var scopeStart = scope.GetStart();
+ var scopeStart = scope.GetStart(ast);
if (scope.Body != null) {
- top = scope.Body.GetEnd().Line == scopeStart.Line
- ? scope.Body.GetStart().Column
+ top = scope.Body.GetEnd(ast).Line == scopeStart.Line
+ ? scope.Body.GetStart(ast).Column
: scopeStart.Column;
} else {
top = scopeStart.Column;
diff --git a/src/LanguageServer/Impl/Completion/FunctionDefinitionCompletion.cs b/src/LanguageServer/Impl/Completion/FunctionDefinitionCompletion.cs
index 844cf3dda..2c56cbc20 100644
--- a/src/LanguageServer/Impl/Completion/FunctionDefinitionCompletion.cs
+++ b/src/LanguageServer/Impl/Completion/FunctionDefinitionCompletion.cs
@@ -33,7 +33,7 @@ public static bool TryGetCompletionsForOverride(FunctionDefinition function, Com
}
if (function.Parent is ClassDefinition cd && function.NameExpression != null && context.Position > function.NameExpression.StartIndex) {
- var loc = function.GetStart();
+ var loc = function.GetStart(context.Ast);
var overrideable = GetOverrideable(context, location).ToArray();
overrideable = !string.IsNullOrEmpty(function.Name)
? overrideable.Where(o => o.Name.StartsWithOrdinal(function.Name)).ToArray()
@@ -56,10 +56,10 @@ private static CompletionItem ToOverrideCompletionItem(IPythonFunctionOverload o
}
private static string MakeOverrideParameter(IParameterInfo paramInfo, string defaultValue) {
- if (paramInfo.IsParamArray) {
+ if (paramInfo.Kind == ParameterKind.List) {
return $"*{paramInfo.Name}";
}
- if (paramInfo.IsKeywordDict) {
+ if (paramInfo.Kind == ParameterKind.Dictionary) {
return $"**{paramInfo.Name}";
}
return !string.IsNullOrEmpty(paramInfo.DefaultValueString) ? $"{paramInfo.Name}={defaultValue}" : paramInfo.Name;
diff --git a/src/LanguageServer/Impl/Completion/ImportCompletion.cs b/src/LanguageServer/Impl/Completion/ImportCompletion.cs
index 3a156912b..9fb15e009 100644
--- a/src/LanguageServer/Impl/Completion/ImportCompletion.cs
+++ b/src/LanguageServer/Impl/Completion/ImportCompletion.cs
@@ -79,7 +79,7 @@ public static CompletionResult GetCompletionsInFromImport(FromImportStatement fr
}
if (context.Position >= name.StartIndex) {
- var applicableSpan = name.GetSpan();
+ var applicableSpan = name.GetSpan(context.Ast);
var importSearchResult = mres.CurrentPathResolver.FindImports(document.FilePath, fromImport);
return GetResultFromImportSearch(importSearchResult, context, false, applicableSpan);
}
diff --git a/src/LanguageServer/Impl/Documentation/DocstringConverter.cs b/src/LanguageServer/Impl/Documentation/DocstringConverter.cs
index 466668890..aab46c877 100644
--- a/src/LanguageServer/Impl/Documentation/DocstringConverter.cs
+++ b/src/LanguageServer/Impl/Documentation/DocstringConverter.cs
@@ -77,7 +77,7 @@ private int NextBlockIndent
private string CurrentLineWithinBlock => CurrentLine.Substring(_blockIndent);
private DocstringConverter(string input) {
- _builder = new StringBuilder(input.Length);
+ _builder = new StringBuilder(input?.Length ?? 0);
_state = ParseText;
_lines = SplitDocstring(input);
}
diff --git a/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs b/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs
index 0ebe6dda1..ddd1565df 100644
--- a/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs
+++ b/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs
@@ -41,8 +41,8 @@ public override bool Walk(ClassDefinition node) {
_stack.AddSymbol(new HierarchicalSymbol(
node.Name,
SymbolKind.Class,
- node.GetSpan(),
- node.NameExpression.GetSpan(),
+ node.GetSpan(_ast),
+ node.NameExpression.GetSpan(_ast),
children,
FunctionKind.Class
));
@@ -58,13 +58,13 @@ public override bool Walk(FunctionDefinition node) {
node.Body?.Walk(this);
var children = _stack.Exit();
- var span = node.GetSpan();
+ var span = node.GetSpan(_ast);
var ds = new HierarchicalSymbol(
node.Name,
SymbolKind.Function,
span,
- node.IsLambda ? span : node.NameExpression.GetSpan(),
+ node.IsLambda ? span : node.NameExpression.GetSpan(_ast),
children,
FunctionKind.Function
);
@@ -102,7 +102,7 @@ public override bool Walk(FunctionDefinition node) {
public override bool Walk(ImportStatement node) {
foreach (var (nameNode, nameString) in node.Names.Zip(node.AsNames, (name, asName) => asName != null ? (asName, asName.Name) : ((Node)name, name.MakeString()))) {
- var span = nameNode.GetSpan();
+ var span = nameNode.GetSpan(_ast);
_stack.AddSymbol(new HierarchicalSymbol(nameString, SymbolKind.Module, span));
}
@@ -115,7 +115,7 @@ public override bool Walk(FromImportStatement node) {
}
foreach (var name in node.Names.Zip(node.AsNames, (name, asName) => asName ?? name)) {
- var span = name.GetSpan();
+ var span = name.GetSpan(_ast);
_stack.AddSymbol(new HierarchicalSymbol(name.Name, SymbolKind.Module, span));
}
@@ -202,7 +202,7 @@ public override bool Walk(GeneratorExpression node) {
private void ExitComprehension(Comprehension node) {
var children = _stack.Exit();
- var span = node.GetSpan();
+ var span = node.GetSpan(_ast);
_stack.AddSymbol(new HierarchicalSymbol(
$"<{node.NodeName}>",
@@ -229,7 +229,7 @@ private void AddVarSymbol(NameExpression node) {
break;
}
- var span = node.GetSpan();
+ var span = node.GetSpan(_ast);
_stack.AddSymbol(new HierarchicalSymbol(node.Name, kind, span));
}
diff --git a/src/LanguageServer/Impl/Sources/HoverSource.cs b/src/LanguageServer/Impl/Sources/HoverSource.cs
index 729b69c97..d4a2f4537 100644
--- a/src/LanguageServer/Impl/Sources/HoverSource.cs
+++ b/src/LanguageServer/Impl/Sources/HoverSource.cs
@@ -45,8 +45,8 @@ public Hover GetHover(IDocumentAnalysis analysis, SourceLocation location) {
}
var range = new Range {
- start = expr.GetStart(),
- end = expr.GetEnd()
+ start = expr.GetStart(analysis.Ast),
+ end = expr.GetEnd(analysis.Ast)
};
var eval = analysis.ExpressionEvaluator;
@@ -91,7 +91,7 @@ public Hover GetHover(IDocumentAnalysis analysis, SourceLocation location) {
if (expr is MemberExpression mex) {
name = mex.Name;
range = new Range {
- start = mex.Target.GetEnd(),
+ start = mex.Target.GetEnd(analysis.Ast),
end = range.end
};
diff --git a/src/LanguageServer/Impl/Sources/ReferenceSource.cs b/src/LanguageServer/Impl/Sources/ReferenceSource.cs
index c302d91f3..009c7894f 100644
--- a/src/LanguageServer/Impl/Sources/ReferenceSource.cs
+++ b/src/LanguageServer/Impl/Sources/ReferenceSource.cs
@@ -20,6 +20,7 @@
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Python.Analysis;
+using Microsoft.Python.Analysis.Analyzer;
using Microsoft.Python.Analysis.Documents;
using Microsoft.Python.Analysis.Modules;
using Microsoft.Python.Analysis.Types;
@@ -129,7 +130,7 @@ private List ScanClosedFiles(string name, CancellationToken cancellationTok
return files;
}
- private static async Task AnalyzeFiles(IModuleManagement moduleManagement, IEnumerable files, CancellationToken cancellationToken) {
+ private async Task AnalyzeFiles(IModuleManagement moduleManagement, IEnumerable files, CancellationToken cancellationToken) {
var analysisTasks = new List();
foreach (var f in files) {
if (moduleManagement.TryAddModulePath(f.ToAbsolutePath(), false, out var fullName)) {
diff --git a/src/LanguageServer/Test/GoToDefinitionTests.cs b/src/LanguageServer/Test/GoToDefinitionTests.cs
index 0e23cc904..1e2f53924 100644
--- a/src/LanguageServer/Test/GoToDefinitionTests.cs
+++ b/src/LanguageServer/Test/GoToDefinitionTests.cs
@@ -230,7 +230,9 @@ public async Task GotoRelativeImportInExplicitPackage() {
rdt.OpenDocument(subpkgPath, string.Empty);
var submod = rdt.OpenDocument(submodPath, "from .. import mod");
- var analysis = await submod.GetAnalysisAsync(-1);
+ await Services.GetService().WaitForCompleteAnalysisAsync();
+ var analysis = await submod.GetAnalysisAsync(Timeout.Infinite);
+
var ds = new DefinitionSource(Services);
var reference = ds.FindDefinition(analysis, new SourceLocation(1, 18), out _);
reference.Should().NotBeNull();
@@ -250,7 +252,8 @@ public async Task GotoModuleSourceFromRelativeImport() {
z = 1
def X(): ...
");
- var analysis = await modA.GetAnalysisAsync(-1);
+ await Services.GetService().WaitForCompleteAnalysisAsync();
+ var analysis = await modA.GetAnalysisAsync(Timeout.Infinite);
var ds = new DefinitionSource(Services);
var reference = ds.FindDefinition(analysis, new SourceLocation(1, 7), out _);
diff --git a/src/LanguageServer/Test/ReferencesTests.cs b/src/LanguageServer/Test/ReferencesTests.cs
index 2d2b23c2e..cd53d92f4 100644
--- a/src/LanguageServer/Test/ReferencesTests.cs
+++ b/src/LanguageServer/Test/ReferencesTests.cs
@@ -17,6 +17,7 @@
using System.IO;
using System.Threading.Tasks;
using FluentAssertions;
+using Microsoft.Python.Analysis.Analyzer;
using Microsoft.Python.Analysis.Documents;
using Microsoft.Python.Core.Text;
using Microsoft.Python.LanguageServer.Sources;
@@ -368,6 +369,7 @@ import logging
var rdt = Services.GetService();
var doc = rdt.OpenDocument(uri, code);
+ await Services.GetService().WaitForCompleteAnalysisAsync();
var analysis = await GetDocumentAnalysisAsync(doc);
var rs = new ReferenceSource(Services);
diff --git a/src/Parsing/Impl/Ast/BinaryExpression.cs b/src/Parsing/Impl/Ast/BinaryExpression.cs
index 3b0497b30..ca7c7536c 100644
--- a/src/Parsing/Impl/Ast/BinaryExpression.cs
+++ b/src/Parsing/Impl/Ast/BinaryExpression.cs
@@ -21,7 +21,7 @@
using System.Threading.Tasks;
namespace Microsoft.Python.Parsing.Ast {
- public partial class BinaryExpression : Expression {
+ public class BinaryExpression : Expression {
public BinaryExpression(PythonOperator op, Expression left, Expression right, int operatorIndex) {
if (op == PythonOperator.None) {
throw new ArgumentException("bad operator");
@@ -88,8 +88,6 @@ public override async Task WalkAsync(PythonWalkerAsync walker, CancellationToken
}
internal override void AppendCodeString(StringBuilder res, PythonAst ast, CodeFormattingOptions format) {
- var left = Left;
- var right = Right;
string op1, op2;
if (Operator == PythonOperator.NotIn) {
diff --git a/src/Parsing/Impl/Ast/ClassDefinition.cs b/src/Parsing/Impl/Ast/ClassDefinition.cs
index c1db95498..9ec8289bf 100644
--- a/src/Parsing/Impl/Ast/ClassDefinition.cs
+++ b/src/Parsing/Impl/Ast/ClassDefinition.cs
@@ -85,8 +85,6 @@ internal override bool HasLateBoundVariableSets {
set => base.HasLateBoundVariableSets = value;
}
- public override bool NeedsLocalContext => true;
-
internal override bool ExposesLocalVariable(PythonVariable variable) => true;
internal override bool TryBindOuter(ScopeStatement from, string name, bool allowGlobals, out PythonVariable variable) {
diff --git a/src/Parsing/Impl/Ast/DictionaryExpression.cs b/src/Parsing/Impl/Ast/DictionaryExpression.cs
index 0feb297be..1ed01406d 100644
--- a/src/Parsing/Impl/Ast/DictionaryExpression.cs
+++ b/src/Parsing/Impl/Ast/DictionaryExpression.cs
@@ -17,7 +17,6 @@
using System.Text;
using System.Threading;
using System.Threading.Tasks;
-using Microsoft.Python.Core;
using Microsoft.Python.Core.Collections;
namespace Microsoft.Python.Parsing.Ast {
diff --git a/src/Parsing/Impl/Ast/DottedName.cs b/src/Parsing/Impl/Ast/DottedName.cs
index f0e6e4b96..b169aabfa 100644
--- a/src/Parsing/Impl/Ast/DottedName.cs
+++ b/src/Parsing/Impl/Ast/DottedName.cs
@@ -14,11 +14,9 @@
// permissions and limitations under the License.
using System.Collections.Generic;
-using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
-using Microsoft.Python.Core;
using Microsoft.Python.Core.Collections;
namespace Microsoft.Python.Parsing.Ast {
@@ -44,16 +42,6 @@ public virtual string MakeString() {
public override IEnumerable GetChildNodes() => Names;
- public override PythonAst Ast {
- get => base.Ast;
- internal set {
- base.Ast = value;
- foreach (var n in Names) {
- n.Ast = value;
- }
- }
- }
-
public override void Walk(PythonWalker walker) {
if (walker.Walk(this)) {
}
diff --git a/src/Parsing/Impl/Ast/FromImportStatement.cs b/src/Parsing/Impl/Ast/FromImportStatement.cs
index d4bbdb97d..794f904f3 100644
--- a/src/Parsing/Impl/Ast/FromImportStatement.cs
+++ b/src/Parsing/Impl/Ast/FromImportStatement.cs
@@ -49,25 +49,9 @@ public FromImportStatement(ModuleName/*!*/ root, ImmutableArray
Justification = "breaking change")]
public PythonVariable[] Variables { get; set; }
- public PythonReference[] GetReferences(PythonAst ast) => GetVariableReferences(this, ast);
-
// TODO: return names and aliases when they are united into one node
public override IEnumerable GetChildNodes() => Enumerable.Empty();
- public override PythonAst Ast {
- get => base.Ast;
- internal set {
- foreach (var n in Names.ExcludeDefault()) {
- n.Ast = value;
- }
- foreach (var n in AsNames.ExcludeDefault()) {
- n.Ast = value;
- }
-
- Root.Ast = value;
- base.Ast = value;
- }
- }
public override void Walk(PythonWalker walker) {
if (walker.Walk(this)) {
}
diff --git a/src/Parsing/Impl/Ast/FunctionDefinition.cs b/src/Parsing/Impl/Ast/FunctionDefinition.cs
index 1b70c2c44..46efe1e7b 100644
--- a/src/Parsing/Impl/Ast/FunctionDefinition.cs
+++ b/src/Parsing/Impl/Ast/FunctionDefinition.cs
@@ -52,8 +52,6 @@ public FunctionDefinition(NameExpression name, Parameter[] parameters, Statement
public Parameter[] Parameters { get; }
- public override int ArgCount => Parameters.Length;
-
internal void SetKeywordEndIndex(int index) => _keywordEndIndex = index;
public override int KeywordEndIndex => _keywordEndIndex ?? (DefIndex + (IsCoroutine ? 9 : 3));
public override int KeywordLength => KeywordEndIndex - StartIndex;
diff --git a/src/Parsing/Impl/Ast/ImportStatement.cs b/src/Parsing/Impl/Ast/ImportStatement.cs
index a7126ba49..5c1a79fc3 100644
--- a/src/Parsing/Impl/Ast/ImportStatement.cs
+++ b/src/Parsing/Impl/Ast/ImportStatement.cs
@@ -13,14 +13,12 @@
// See the Apache Version 2.0 License for specific language governing
// permissions and limitations under the License.
-using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
-using Microsoft.Python.Core;
using Microsoft.Python.Core.Collections;
namespace Microsoft.Python.Parsing.Ast {
@@ -44,19 +42,6 @@ public ImportStatement(ImmutableArray names, ImmutableArray GetChildNodes() => Enumerable.Empty();
- public override PythonAst Ast {
- get => base.Ast;
- internal set {
- foreach (var n in Names.ExcludeDefault()) {
- n.Ast = value;
- }
- foreach (var n in AsNames.ExcludeDefault()) {
- n.Ast = value;
- }
- base.Ast = value;
- }
- }
-
public override void Walk(PythonWalker walker) {
if (walker.Walk(this)) {
}
diff --git a/src/Parsing/Impl/Ast/MemberExpression.cs b/src/Parsing/Impl/Ast/MemberExpression.cs
index b11db83c2..d49d32c2e 100644
--- a/src/Parsing/Impl/Ast/MemberExpression.cs
+++ b/src/Parsing/Impl/Ast/MemberExpression.cs
@@ -95,6 +95,6 @@ public override void SetLeadingWhiteSpace(PythonAst ast, string whiteSpace)
///
/// Returns the span of the name component of the expression
///
- public SourceSpan GetNameSpan() => new SourceSpan(Ast.IndexToLocation(NameHeader), GetEnd());
+ public SourceSpan GetNameSpan(PythonAst ast) => new SourceSpan(ast.IndexToLocation(NameHeader), GetEnd(ast));
}
}
diff --git a/src/Parsing/Impl/Ast/NamedExpression.cs b/src/Parsing/Impl/Ast/NamedExpression.cs
index 8a55cd3da..3f5693ab2 100644
--- a/src/Parsing/Impl/Ast/NamedExpression.cs
+++ b/src/Parsing/Impl/Ast/NamedExpression.cs
@@ -1,5 +1,4 @@
-using System;
-using System.Collections.Generic;
+using System.Collections.Generic;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
diff --git a/src/Parsing/Impl/Ast/Node.cs b/src/Parsing/Impl/Ast/Node.cs
index 82bc79c42..973ce6e70 100644
--- a/src/Parsing/Impl/Ast/Node.cs
+++ b/src/Parsing/Impl/Ast/Node.cs
@@ -23,8 +23,6 @@ namespace Microsoft.Python.Parsing.Ast {
public abstract class Node {
#region Public API
- public virtual PythonAst Ast { get; internal set; }
-
public int EndIndex {
get => IndexSpan.End;
set => IndexSpan = new IndexSpan(IndexSpan.Start, value - IndexSpan.Start);
@@ -50,11 +48,11 @@ public string ToCodeString(PythonAst ast, CodeFormattingOptions format) {
return string.IsInterned(result) ?? result;
}
- public SourceLocation GetStart() => Ast.IndexToLocation(StartIndex);
+ public SourceLocation GetStart(PythonAst ast) => ast.IndexToLocation(StartIndex);
- public SourceLocation GetEnd() => Ast.IndexToLocation(EndIndex);
+ public SourceLocation GetEnd(PythonAst ast) => ast.IndexToLocation(EndIndex);
- public SourceSpan GetSpan() => new SourceSpan(GetStart(), GetEnd());
+ public SourceSpan GetSpan(PythonAst ast) => new SourceSpan(GetStart(ast), GetEnd(ast));
///
/// Returns the proceeding whitespace (newlines and comments) that
diff --git a/src/Parsing/Impl/Ast/PythonAst.cs b/src/Parsing/Impl/Ast/PythonAst.cs
index 17cc32571..70a669cf9 100644
--- a/src/Parsing/Impl/Ast/PythonAst.cs
+++ b/src/Parsing/Impl/Ast/PythonAst.cs
@@ -22,11 +22,11 @@
using Microsoft.Python.Core.Text;
namespace Microsoft.Python.Parsing.Ast {
-
///
- /// Top-level ast for all Python code. Holds onto the body and the line mapping information.
+ /// Top-level ast for all Python code. Holds onto the body and the line mapping information.
///
public sealed class PythonAst : ScopeStatement {
+ private readonly object _lock = new object();
private readonly Statement _body;
private readonly Dictionary> _attributes = new Dictionary>();
@@ -36,7 +36,6 @@ public PythonAst(Uri module, Statement body, NewLineLocation[] lineLocations, Py
LanguageVersion = langVersion;
NewLineLocations = lineLocations;
CommentLocations = commentLocations;
-
}
public PythonAst(IEnumerable existingAst) {
@@ -60,9 +59,8 @@ public PythonAst(IEnumerable existingAst) {
}
public Uri Module { get; }
- public NewLineLocation[] NewLineLocations { get; }
- public SourceLocation[] CommentLocations { get; }
-
+ public NewLineLocation[] NewLineLocations { get; private set; }
+ public SourceLocation[] CommentLocations { get; private set; }
public override string Name => "";
///
@@ -76,7 +74,7 @@ public PythonAst(IEnumerable existingAst) {
///
public bool HasVerbatim { get; internal set; }
- public override IEnumerable GetChildNodes() => new[] {_body};
+ public override IEnumerable GetChildNodes() => new[] { _body };
public override void Walk(PythonWalker walker) {
if (walker.Walk(this)) {
@@ -93,62 +91,58 @@ public override async Task WalkAsync(PythonWalkerAsync walker, CancellationToken
}
public override Statement Body => _body;
-
public PythonLanguageVersion LanguageVersion { get; }
- public bool TryGetAttribute(Node node, object key, out object value) {
- if (_attributes.TryGetValue(node, out var nodeAttrs)) {
- return nodeAttrs.TryGetValue(key, out value);
+ public void Reduce(Func filter) {
+ lock (_lock) {
+ (Body as SuiteStatement)?.FilterStatements(filter);
+ _attributes?.Clear();
+ Variables?.Clear();
+ CommentLocations = Array.Empty();
+ // DO keep NewLineLocations as they are required
+ // to calculate node positions for navigation;
+ base.Clear();
}
- value = null;
- return false;
}
- public void SetAttribute(Node node, object key, object value) {
- if (!_attributes.TryGetValue(node, out var nodeAttrs)) {
- nodeAttrs = _attributes[node] = new Dictionary();
+ public bool TryGetAttribute(Node node, object key, out object value) {
+ lock (_lock) {
+ if (_attributes.TryGetValue(node, out var nodeAttrs)) {
+ return nodeAttrs.TryGetValue(key, out value);
+ }
+
+ value = null;
+ return false;
}
- nodeAttrs[key] = value;
}
- ///
- /// Copies attributes that apply to one node and makes them available for the other node.
- ///
- ///
- ///
- public void CopyAttributes(Node from, Node to) {
- if (_attributes.TryGetValue(from, out var fromAttrs)) {
- var toAttrs = new Dictionary(fromAttrs.Count);
- foreach (var nodeAttr in fromAttrs) {
- toAttrs[nodeAttr.Key] = nodeAttr.Value;
+ public void SetAttribute(Node node, object key, object value) {
+ lock (_lock) {
+ if (!_attributes.TryGetValue(node, out var nodeAttrs)) {
+ nodeAttrs = _attributes[node] = new Dictionary();
}
- _attributes[to] = toAttrs;
+ nodeAttrs[key] = value;
}
}
internal void SetAttributes(Dictionary> attributes) {
- foreach (var nodeAttributes in attributes) {
- var node = nodeAttributes.Key;
- if (!_attributes.TryGetValue(node, out var existingNodeAttributes)) {
- existingNodeAttributes = _attributes[node] = new Dictionary(nodeAttributes.Value.Count);
- }
-
- foreach (var nodeAttr in nodeAttributes.Value) {
- existingNodeAttributes[nodeAttr.Key] = nodeAttr.Value;
+ lock (_lock) {
+ foreach (var nodeAttributes in attributes) {
+ var node = nodeAttributes.Key;
+ if (!_attributes.TryGetValue(node, out var existingNodeAttributes)) {
+ existingNodeAttributes = _attributes[node] = new Dictionary(nodeAttributes.Value.Count);
+ }
+
+ foreach (var nodeAttr in nodeAttributes.Value) {
+ existingNodeAttributes[nodeAttr.Key] = nodeAttr.Value;
+ }
}
}
}
public SourceLocation IndexToLocation(int index) => NewLineLocation.IndexToLocation(NewLineLocations, index);
-
public int LocationToIndex(SourceLocation location) => NewLineLocation.LocationToIndex(NewLineLocations, location, EndIndex);
- ///
- /// Length of the span (number of characters inside the span).
- ///
- public int GetSpanLength(SourceSpan span) => LocationToIndex(span.End) - LocationToIndex(span.Start);
-
-
internal int GetLineEndFromPosition(int index) {
var loc = IndexToLocation(index);
if (loc.Line >= NewLineLocations.Length) {
diff --git a/src/Parsing/Impl/Ast/ScopeStatement.cs b/src/Parsing/Impl/Ast/ScopeStatement.cs
index 45900967d..d6181be4e 100644
--- a/src/Parsing/Impl/Ast/ScopeStatement.cs
+++ b/src/Parsing/Impl/Ast/ScopeStatement.cs
@@ -22,7 +22,6 @@ namespace Microsoft.Python.Parsing.Ast {
public abstract class ScopeStatement : Statement {
// due to "exec" or call to dir, locals, eval, vars...
- private ClosureInfo[] _closureVariables; // closed over variables, bool indicates if we accessed it in this scope.
private List _freeVars; // list of variables accessed from outer scopes
private List _globalVars; // global variables accessed from this scope
private List _cellVars; // variables accessed from nested scopes
@@ -75,12 +74,10 @@ public abstract Statement Body {
///
/// Gets the variables for this scope.
///
- public ICollection ScopeVariables
+ public ICollection ScopeVariables
=> Variables?.Values ?? Array.Empty() as ICollection;
public virtual bool IsGlobal => false;
- public virtual bool NeedsLocalContext => NeedsLocalsDictionary || ContainsNestedFreeVariables;
- public virtual int ArgCount => 0;
public PythonAst GlobalParent {
get {
@@ -93,20 +90,23 @@ public PythonAst GlobalParent {
}
}
- internal void AddFreeVariable(PythonVariable variable, bool accessedInScope) {
- if (_freeVars == null) {
- _freeVars = new List();
- }
+ protected void Clear() {
+ _references?.Clear();
+ _cellVars?.Clear();
+ _freeVars?.Clear();
+ _globalVars?.Clear();
+ _nonLocalVars?.Clear();
+ }
+ internal void AddFreeVariable(PythonVariable variable, bool accessedInScope) {
+ _freeVars = _freeVars ?? new List();
if (!_freeVars.Contains(variable)) {
_freeVars.Add(variable);
}
}
internal string AddReferencedGlobal(string name) {
- if (_globalVars == null) {
- _globalVars = new List();
- }
+ _globalVars = _globalVars ?? new List();
if (!_globalVars.Contains(name)) {
_globalVars.Add(name);
}
@@ -114,17 +114,12 @@ internal string AddReferencedGlobal(string name) {
}
internal void AddNonLocalVariable(NameExpression name) {
- if (_nonLocalVars == null) {
- _nonLocalVars = new List();
- }
+ _nonLocalVars = _nonLocalVars ?? new List();
_nonLocalVars.Add(name);
}
internal void AddCellVariable(PythonVariable variable) {
- if (_cellVars == null) {
- _cellVars = new List();
- }
-
+ _cellVars = _cellVars ?? new List();
if (!_cellVars.Contains(variable.Name)) {
_cellVars.Add(variable.Name);
}
@@ -135,17 +130,6 @@ internal void AddCellVariable(PythonVariable variable) {
///
public IReadOnlyList FreeVariables => _freeVars;
- ///
- /// Variables that are bound to the global scope
- ///
- public IReadOnlyList GlobalVariables => _globalVars;
-
- ///
- /// Variables that are referred to from a nested scope and need to be
- /// promoted to cells.
- ///
- public IReadOnlyList CellVariables => _cellVars;
-
internal abstract bool ExposesLocalVariable(PythonVariable variable);
public bool TryGetVariable(string name, out PythonVariable variable) {
@@ -231,10 +215,6 @@ internal virtual void FinishBind(PythonNameBinder binder) {
}
}
- if (closureVariables != null) {
- _closureVariables = closureVariables.ToArray();
- }
-
// no longer needed
_references = null;
}
diff --git a/src/Parsing/Impl/Ast/SuiteStatement.cs b/src/Parsing/Impl/Ast/SuiteStatement.cs
index ddb4c87ac..920a1d8b6 100644
--- a/src/Parsing/Impl/Ast/SuiteStatement.cs
+++ b/src/Parsing/Impl/Ast/SuiteStatement.cs
@@ -14,7 +14,9 @@
// See the Apache Version 2.0 License for specific language governing
// permissions and limitations under the License.
+using System;
using System.Collections.Generic;
+using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
@@ -22,16 +24,18 @@
namespace Microsoft.Python.Parsing.Ast {
public sealed class SuiteStatement : Statement {
- private readonly Statement[] _statements;
+ private Statement[] _statements;
public SuiteStatement(Statement[] statements) {
_statements = statements;
}
public IList Statements => _statements;
-
public override IEnumerable GetChildNodes() => _statements.WhereNotNull();
+ public void FilterStatements(Func filter)
+ => _statements = _statements.Where(filter).ToArray();
+
public override void Walk(PythonWalker walker) {
if (walker.Walk(this)) {
foreach (var s in _statements.MaybeEnumerate()) {
diff --git a/src/Parsing/Impl/Parser.cs b/src/Parsing/Impl/Parser.cs
index 2ef7dcd0a..e6a1b177f 100644
--- a/src/Parsing/Impl/Parser.cs
+++ b/src/Parsing/Impl/Parser.cs
@@ -247,10 +247,6 @@ private PythonAst CreateAst(Uri module, Statement ret) {
ast.SetAttributes(_attributes);
PythonNameBinder.BindAst(_langVersion, ast, _errors, _bindReferences);
- foreach (var n in ((Node)ast).TraverseDepthFirst(c => c.GetChildNodes())) {
- n.Ast = ast;
- }
-
return ast;
}
@@ -3294,8 +3290,8 @@ private Expression buildFStringExpr(IEnumerable readTokens) {
var openQuotes = readTokens.Where(t => t.Token.Kind == TokenKind.FString)
.Select(t => ((FStringToken)t.Token).OpenQuotes).DefaultIfEmpty("'").First();
- List fStringChildren = new List();
- StringBuilder unparsedFStringBuilder = new StringBuilder();
+ var fStringChildren = new List();
+ var unparsedFStringBuilder = new StringBuilder();
foreach (var tokenWithSpan in readTokens) {
if (tokenWithSpan.Token.Kind == TokenKind.FString) {
diff --git a/src/Parsing/Impl/Token.cs b/src/Parsing/Impl/Token.cs
index db7d8d1c5..703deb591 100644
--- a/src/Parsing/Impl/Token.cs
+++ b/src/Parsing/Impl/Token.cs
@@ -93,8 +93,6 @@ public ConstantValueToken(object value)
Value = value;
}
- public object Constant => Value;
-
public override object Value { get; }
public override string Image => Value == null ? "None" : Value.ToString();
@@ -140,11 +138,7 @@ public FStringToken(string value, string openQuote, bool isTriple, bool isRaw)
public string Text => (string)Value;
- public override string Image {
- get {
- return Value == null ? "None" : $"f{OpenQuotes}{Value.ToString()}{OpenQuotes}";
- }
- }
+ public override string Image => Value == null ? "None" : $"f{OpenQuotes}{Value.ToString()}{OpenQuotes}";
}
public sealed class VerbatimFStringToken : FStringToken {