From b605b76da408c072d6234f20437f6dd3ecfd1003 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Fri, 17 May 2019 09:20:28 -0700 Subject: [PATCH 01/39] Remove old qualified name --- .../Types/Definitions/IHasQualifiedName.cs | 38 ------------------- .../Ast/Impl/Types/PythonFunctionType.cs | 6 --- src/Analysis/Ast/Impl/Types/PythonType.cs | 8 +--- .../Ast/Impl/Types/PythonTypeWrapper.cs | 8 +--- .../FluentAssertions/AssertionsUtilities.cs | 20 +++------- 5 files changed, 7 insertions(+), 73 deletions(-) delete mode 100644 src/Analysis/Ast/Impl/Types/Definitions/IHasQualifiedName.cs diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IHasQualifiedName.cs b/src/Analysis/Ast/Impl/Types/Definitions/IHasQualifiedName.cs deleted file mode 100644 index 2b9f9a48b..000000000 --- a/src/Analysis/Ast/Impl/Types/Definitions/IHasQualifiedName.cs +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABILITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; -using System.Collections.Generic; - -namespace Microsoft.Python.Analysis.Types { - public interface IHasQualifiedName { - /// - /// Gets the fully qualified, dot-separated name of the value. - /// This is typically used for displaying to users. - /// - string FullyQualifiedName { get; } - - /// - /// Gets the import and eval names of the value. The first part - /// should be importable, and the second is a name that can be - /// resolved with getattr(). - /// These are often seen separated with a colon. - /// - /// - /// The value cannot be resolved (for example, a nested function). - /// - KeyValuePair FullyQualifiedNamePair { get; } - } -} diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs index 2008f30fa..7e95cc557 100644 --- a/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs @@ -124,12 +124,6 @@ internal override void SetDocumentationProvider(Func provider) { public IReadOnlyList Overloads => _overloads; #endregion - #region IHasQualifiedName - public override string FullyQualifiedName => FullyQualifiedNamePair.CombineNames(); - public override KeyValuePair FullyQualifiedNamePair => - new KeyValuePair((DeclaringType as IHasQualifiedName)?.FullyQualifiedName ?? DeclaringType?.Name ?? DeclaringModule?.Name, Name); - #endregion - internal void Specialize(string[] dependencies) { _isSpecialized = true; Dependencies = dependencies != null diff --git a/src/Analysis/Ast/Impl/Types/PythonType.cs b/src/Analysis/Ast/Impl/Types/PythonType.cs index 41e9bda1c..b6c0798c6 100644 --- a/src/Analysis/Ast/Impl/Types/PythonType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonType.cs @@ -22,7 +22,7 @@ namespace Microsoft.Python.Analysis.Types { [DebuggerDisplay("{Name}")] - internal class PythonType : LocatedMember, IPythonType, IHasQualifiedName, IEquatable { + internal class PythonType : LocatedMember, IPythonType, IEquatable { private readonly object _lock = new object(); private readonly string _name; private Func _documentationProvider; @@ -98,12 +98,6 @@ public virtual IMember Call(IPythonInstance instance, string memberName, IArgume public virtual IMember Index(IPythonInstance instance, object index) => instance?.Index(index) ?? UnknownType; #endregion - #region IHasQualifiedName - public virtual string FullyQualifiedName => FullyQualifiedNamePair.CombineNames(); - public virtual KeyValuePair FullyQualifiedNamePair - => new KeyValuePair(DeclaringModule?.Name ?? string.Empty, Name); - #endregion - #region IMemberContainer public virtual IMember GetMember(string name) => Members.TryGetValue(name, out var member) ? member : null; public virtual IEnumerable GetMemberNames() => Members.Keys; diff --git a/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs b/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs index 5201297b1..edd879a58 100644 --- a/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs +++ b/src/Analysis/Ast/Impl/Types/PythonTypeWrapper.cs @@ -16,13 +16,12 @@ using System; using System.Collections.Generic; using Microsoft.Python.Analysis.Values; -using Microsoft.Python.Core.Text; namespace Microsoft.Python.Analysis.Types { /// /// Delegates most of the methods to the wrapped/inner class. /// - internal class PythonTypeWrapper : IPythonType, IHasQualifiedName { + internal class PythonTypeWrapper : IPythonType { private readonly BuiltinTypeId _builtinTypeId; private IPythonType _innerType; @@ -86,11 +85,6 @@ public virtual IMember Index(IPythonInstance instance, object index) public virtual IEnumerable GetMemberNames() => InnerType.GetMemberNames(); #endregion - #region IHasQualifiedName - public virtual string FullyQualifiedName => (InnerType as IHasQualifiedName)?.FullyQualifiedName; - public virtual KeyValuePair FullyQualifiedNamePair => (InnerType as IHasQualifiedName)?.FullyQualifiedNamePair ?? default; - #endregion - protected IMember UnknownType => DeclaringModule.Interpreter.UnknownType; public override bool Equals(object obj) diff --git a/src/Analysis/Ast/Test/FluentAssertions/AssertionsUtilities.cs b/src/Analysis/Ast/Test/FluentAssertions/AssertionsUtilities.cs index a30be146a..9be29ed15 100644 --- a/src/Analysis/Ast/Test/FluentAssertions/AssertionsUtilities.cs +++ b/src/Analysis/Ast/Test/FluentAssertions/AssertionsUtilities.cs @@ -31,9 +31,9 @@ public static bool Is3X(IScope scope) => scope.GlobalScope.Module.Interpreter.LanguageVersion.Is3x(); public static void AssertTypeIds( - IEnumerable actualTypeIds, - IEnumerable typeIds, - string name, bool languageVersionIs3X, string because, + IEnumerable actualTypeIds, + IEnumerable typeIds, + string name, bool languageVersionIs3X, string because, object[] reasonArgs, string itemNameSingle = "type", string itemNamePlural = "types") { var expected = typeIds.Select(t => { switch (t) { @@ -163,22 +163,12 @@ private static StringBuilder AppendQuotedName(this StringBuilder stringBuilder, } public static string GetQuotedName(object value) { - string name; - switch (value) { - case IHasQualifiedName _: - case IPythonModule _: - name = GetName(value); - return string.IsNullOrEmpty(name) ? string.Empty : $"'{name}'"; - default: - name = GetName(value); - return string.IsNullOrEmpty(name) ? string.Empty : $"'{name}'"; - } + var name = GetName(value); + return string.IsNullOrEmpty(name) ? string.Empty : $"'{name}'"; } public static string GetName(object value) { switch (value) { - case IHasQualifiedName qualifiedName: - return qualifiedName.FullyQualifiedName; case IPythonModule pythonModule: return pythonModule.Name; case IScope scope: From 5471ce4dbc43f2e8821bd2ee40c4f7362a465d90 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Fri, 17 May 2019 10:12:16 -0700 Subject: [PATCH 02/39] Node storage --- .../Evaluation/ExpressionEval.Callables.cs | 2 +- .../Impl/Analyzer/Symbols/SymbolCollector.cs | 7 +- .../Impl/Extensions/PythonModuleExtensions.cs | 27 +++++ .../Modules/Definitions/IAstNodeContainer.cs | 39 +++++++ src/Analysis/Ast/Impl/Modules/PythonModule.cs | 105 ++++++++++-------- .../Specializations/Typing/TypingModule.cs | 8 +- .../Ast/Impl/Types/PythonFunctionOverload.cs | 43 ++++--- .../Ast/Impl/Types/PythonFunctionType.cs | 31 ++---- .../Ast/Impl/Types/PythonPropertyType.cs | 4 +- src/Analysis/Ast/Impl/Types/PythonType.cs | 18 +-- 10 files changed, 168 insertions(+), 116 deletions(-) create mode 100644 src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs create mode 100644 src/Analysis/Ast/Impl/Modules/Definitions/IAstNodeContainer.cs diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs index f2e79b8a5..87535666c 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs @@ -79,7 +79,7 @@ public IMember GetValueFromLambda(LambdaExpression expr) { var location = GetLocationOfName(expr.Function); var ft = new PythonFunctionType(expr.Function, null, location); - var overload = new PythonFunctionOverload(expr.Function, ft, location); + var overload = new PythonFunctionOverload(ft, location); ft.AddOverload(overload); return ft; } diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs index f09974a28..ef71bf50f 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs @@ -119,8 +119,9 @@ private void AddOverload(FunctionDefinition fd, IPythonClassMember function, Act if (!_table.ReplacedByStubs.Contains(fd)) { var stubOverload = GetOverloadFromStub(fd); if (stubOverload != null) { - if (!string.IsNullOrEmpty(fd.GetDocumentation())) { - stubOverload.SetDocumentationProvider(_ => fd.GetDocumentation()); + var documentation = fd.GetDocumentation(); + if (!string.IsNullOrEmpty(documentation)) { + stubOverload.SetDocumentation(documentation); } addOverload(stubOverload); _table.ReplacedByStubs.Add(fd); @@ -131,7 +132,7 @@ private void AddOverload(FunctionDefinition fd, IPythonClassMember function, Act if (!_table.Contains(fd)) { // Do not evaluate parameter types just yet. During light-weight top-level information // collection types cannot be determined as imports haven't been processed. - var overload = new PythonFunctionOverload(fd, function, _eval.GetLocationOfName(fd)); + var overload = new PythonFunctionOverload(function, _eval.GetLocationOfName(fd)); addOverload(overload); _table.Add(new FunctionEvaluator(_eval, overload)); } diff --git a/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs b/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs new file mode 100644 index 000000000..36c55099e --- /dev/null +++ b/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs @@ -0,0 +1,27 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using Microsoft.Python.Analysis.Modules; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis { + public static class PythonModuleExtensions { + internal static T GetAstNode(this IPythonModule module, IPythonType t) where T : Node + => ((IAstNodeContainer)module).GetAstNode(t); + internal static void AddAstNode(this IPythonModule module, IPythonType t, Node n) + => ((IAstNodeContainer)module).AddAstNode(t, n); + } +} diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/IAstNodeContainer.cs b/src/Analysis/Ast/Impl/Modules/Definitions/IAstNodeContainer.cs new file mode 100644 index 000000000..f4f04a8d4 --- /dev/null +++ b/src/Analysis/Ast/Impl/Modules/Definitions/IAstNodeContainer.cs @@ -0,0 +1,39 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Modules { + internal interface IAstNodeContainer { + /// + /// Provides access to AST nodes associated with types such as + /// for . + /// Nodes are not available for library modules as AST is not retained + /// in libraries in order to conserve memory. + /// + T GetAstNode(IPythonType t) where T : Node; + + /// + /// Associated AST node with the type. + /// + void AddAstNode(IPythonType t, Node n); + + /// + /// Removes all AST node associations. + /// + void Clear(); + } +} diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index 3aa880aca..c033fc1e0 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -42,7 +42,7 @@ namespace Microsoft.Python.Analysis.Modules { /// to AST and the module analysis. /// [DebuggerDisplay("{Name} : {ModuleType}")] - internal class PythonModule : LocatedMember, IDocument, IAnalyzable, IEquatable { + internal class PythonModule : LocatedMember, IDocument, IAnalyzable, IEquatable, IAstNodeContainer { private enum State { None, Loading, @@ -56,13 +56,13 @@ private enum State { private readonly DocumentBuffer _buffer = new DocumentBuffer(); private readonly DisposeToken _disposeToken = DisposeToken.Create(); private IReadOnlyList _parseErrors = Array.Empty(); + private readonly Dictionary _astMap = new Dictionary(); private readonly IDiagnosticsService _diagnosticsService; private string _documentation; // Must be null initially. private CancellationTokenSource _parseCts; private CancellationTokenSource _linkedParseCts; // combined with 'dispose' cts private Task _parsingTask; - private PythonAst _ast; private bool _updated; protected ILogger Log { get; } @@ -129,7 +129,7 @@ internal PythonModule(ModuleCreationOptions creationOptions, IServiceContainer s public virtual string Documentation { get { - _documentation = _documentation ?? _ast?.Documentation; + _documentation = _documentation ?? GetAstNode(this)?.Documentation; if (_documentation == null) { var m = GetMember("__doc__"); _documentation = m.TryGetConstant(out var s) ? s : string.Empty; @@ -189,39 +189,6 @@ public virtual IEnumerable GetMemberNames() { /// wants to see library code and not a stub. /// public IPythonModule PrimaryModule { get; internal set; } - - protected virtual string LoadContent() { - if (ContentState < State.Loading) { - ContentState = State.Loading; - try { - var code = FileSystem.ReadTextWithRetry(FilePath); - ContentState = State.Loaded; - return code; - } catch (IOException) { } catch (UnauthorizedAccessException) { } - } - return null; // Keep content as null so module can be loaded later. - } - - private void InitializeContent(string content, int version) { - lock (AnalysisLock) { - LoadContent(content, version); - - var startParse = ContentState < State.Parsing && (_parsingTask == null || version > 0); - if (startParse) { - Parse(); - } - } - } - - private void LoadContent(string content, int version) { - if (ContentState < State.Loading) { - try { - content = content ?? LoadContent(); - _buffer.Reset(version, content); - ContentState = State.Loaded; - } catch (IOException) { } catch (UnauthorizedAccessException) { } - } - } #endregion #region IDisposable @@ -282,10 +249,10 @@ public async Task GetAstAsync(CancellationToken cancellationToken = d } } cancellationToken.ThrowIfCancellationRequested(); - return _ast; + return GetAstNode(this); } - public PythonAst GetAnyAst() => _ast; + public PythonAst GetAnyAst() => GetAstNode(this); /// /// Provides collection of parsing errors, if any. @@ -359,7 +326,7 @@ private void Parse(CancellationToken cancellationToken) { if (version != _buffer.Version) { throw new OperationCanceledException(); } - _ast = ast; + _astMap[this] = ast; _parseErrors = sink?.Diagnostics ?? Array.Empty(); // Do not report issues with libraries or stubs @@ -450,6 +417,19 @@ public void NotifyAnalysisComplete(IDocumentAnalysis analysis) { protected virtual void OnAnalysisComplete() { } #endregion + #region IEquatable + public bool Equals(IPythonModule other) => Name.Equals(other?.Name) && FilePath.Equals(other?.FilePath); + #endregion + + #region IAstNodeContainer + public T GetAstNode(IPythonType t) where T : Node => _astMap.TryGetValue(t, out var n) ? (T)n : null; + public void AddAstNode(IPythonType t, Node n) { + Debug.Assert(!_astMap.ContainsKey(t)); + _astMap[t] = n; + } + public void Clear() => _astMap.Clear(); + #endregion + #region Analysis public IDocumentAnalysis GetAnyAnalysis() => Analysis; @@ -458,14 +438,42 @@ public Task GetAnalysisAsync(int waitTime = 200, Cancellation #endregion - private void RemoveReferencesInModule(IPythonModule module) { - if (module.GlobalScope?.Variables != null) { - foreach (var v in module.GlobalScope.Variables) { - v.RemoveReferences(this); + #region Content management + protected virtual string LoadContent() { + if (ContentState < State.Loading) { + ContentState = State.Loading; + try { + var code = FileSystem.ReadTextWithRetry(FilePath); + ContentState = State.Loaded; + return code; + } catch (IOException) { } catch (UnauthorizedAccessException) { } + } + return null; // Keep content as null so module can be loaded later. + } + + private void InitializeContent(string content, int version) { + lock (AnalysisLock) { + LoadContent(content, version); + + var startParse = ContentState < State.Parsing && (_parsingTask == null || version > 0); + if (startParse) { + Parse(); } } } + private void LoadContent(string content, int version) { + if (ContentState < State.Loading) { + try { + content = content ?? LoadContent(); + _buffer.Reset(version, content); + ContentState = State.Loaded; + } catch (IOException) { } catch (UnauthorizedAccessException) { } + } + } + #endregion + + #region Documentation private string TryGetDocFromModuleInitFile() { if (string.IsNullOrEmpty(FilePath) || !FileSystem.FileExists(FilePath)) { return string.Empty; @@ -496,7 +504,7 @@ private string TryGetDocFromModuleInitFile() { // Also, handle quadruple+ quotes. line = line.Trim(); line = line.All(c => c == quote[0]) ? quote : line; - if (line.EndsWithOrdinal(quote) && line.IndexOf(quote, StringComparison.Ordinal) < line.LastIndexOf(quote, StringComparison.Ordinal)) { + if (line.EndsWithOrdinal(quote) && line.IndexOf(quote, StringComparison.Ordinal) < line.LastIndexOf(quote, StringComparison.Ordinal)) { return line.Substring(quote.Length, line.Length - 2 * quote.Length).Trim(); } var sb = new StringBuilder(); @@ -513,7 +521,14 @@ private string TryGetDocFromModuleInitFile() { } catch (IOException) { } catch (UnauthorizedAccessException) { } return string.Empty; } + #endregion - public bool Equals(IPythonModule other) => Name.Equals(other?.Name) && FilePath.Equals(other?.FilePath); + private void RemoveReferencesInModule(IPythonModule module) { + if (module.GlobalScope?.Variables != null) { + foreach (var v in module.GlobalScope.Variables) { + v.RemoveReferences(this); + } + } + } } } diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs b/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs index d91afe25c..cf9be594d 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs @@ -46,7 +46,7 @@ private void SpecializeMembers() { var location = new Location(this, default); // TypeVar - var fn = new PythonFunctionType("TypeVar", location, null, GetMemberDocumentation); + var fn = new PythonFunctionType("TypeVar", location, null, GetMemberDocumentation("TypeVar")); var o = new PythonFunctionOverload(fn.Name, location); // When called, create generic parameter type. For documentation // use original TypeVar declaration so it appear as a tooltip. @@ -57,7 +57,7 @@ private void SpecializeMembers() { _members["TypeVar"] = fn; // NewType - fn = new PythonFunctionType("NewType", location, null, GetMemberDocumentation); + fn = new PythonFunctionType("NewType", location, null, GetMemberDocumentation("NewType")); o = new PythonFunctionOverload(fn.Name, location); // When called, create generic parameter type. For documentation // use original TypeVar declaration so it appear as a tooltip. @@ -66,7 +66,7 @@ private void SpecializeMembers() { _members["NewType"] = fn; // NewType - fn = new PythonFunctionType("Type", location, null, GetMemberDocumentation); + fn = new PythonFunctionType("Type", location, null, GetMemberDocumentation("Type")); o = new PythonFunctionOverload(fn.Name, location); // When called, create generic parameter type. For documentation // use original TypeVar declaration so it appear as a tooltip. @@ -124,7 +124,7 @@ private void SpecializeMembers() { _members["SupportsBytes"] = Interpreter.GetBuiltinType(BuiltinTypeId.Bytes); _members["ByteString"] = Interpreter.GetBuiltinType(BuiltinTypeId.Bytes); - fn = new PythonFunctionType("NamedTuple", location, null, GetMemberDocumentation); + fn = new PythonFunctionType("NamedTuple", location, null, GetMemberDocumentation("NamedTuple")); o = new PythonFunctionOverload(fn.Name, location); o.SetReturnValueProvider((interpreter, overload, args) => CreateNamedTuple(args.Values())); fn.AddOverload(o); diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs index 6215512ad..33f47eb04 100644 --- a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs +++ b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs @@ -47,15 +47,26 @@ internal sealed class PythonFunctionOverload : LocatedMember, IPythonFunctionOve // Return value can be an instance or a type info. Consider type(C()) returning // type info of C vs. return C() that returns an instance of C. - private Func _documentationProvider; private bool _fromAnnotation; + private string _documentation; - public PythonFunctionOverload(FunctionDefinition fd, IPythonClassMember classMember, Location location) - : this(fd.Name, location) { - FunctionDefinition = fd; - ClassMember = classMember; + public PythonFunctionOverload(IPythonClassMember cm, Location location) + : this(cm.Name, location) { + ClassMember = cm; var ast = (location.Module as IDocument)?.Analysis.Ast; - _returnDocumentation = ast != null ? fd.ReturnAnnotation?.ToCodeString(ast) : null; + + FunctionDefinition fd = null; + switch(cm) { + case IPythonFunctionType ft: + fd = ft.FunctionDefinition; + break; + case IPythonPropertyType prop: + fd = prop.FunctionDefinition; + break; + } + + _documentation = fd?.Documentation; + _returnDocumentation = ast != null ? fd?.ReturnAnnotation?.ToCodeString(ast) : null; } public PythonFunctionOverload(string name, Location location) : base(location) { @@ -68,9 +79,6 @@ public PythonFunctionOverload(string name, Location location) : base(location) { internal void SetParameters(IReadOnlyList parameters) => Parameters = parameters; - internal void SetDocumentationProvider(Func documentationProvider) - => _documentationProvider = _documentationProvider ?? documentationProvider; - internal void AddReturnValue(IMember value) { if (value.IsUnknown()) { return; // Don't add useless values. @@ -94,23 +102,14 @@ internal void SetReturnValue(IMember value, bool fromAnnotation) { _fromAnnotation = fromAnnotation; } - internal void SetReturnValueProvider(ReturnValueProvider provider) - => _returnValueProvider = provider; + internal void SetReturnValueProvider(ReturnValueProvider provider) => _returnValueProvider = provider; + internal void SetDocumentation(string documentation) => _documentation = documentation; #region IPythonFunctionOverload - public FunctionDefinition FunctionDefinition { get; } + public FunctionDefinition FunctionDefinition => ClassMember.DeclaringModule.GetAstNode(ClassMember); public IPythonClassMember ClassMember { get; } public string Name { get; } - - public string Documentation { - get { - var s = _documentationProvider?.Invoke(Name); - if (string.IsNullOrEmpty(s)) { - s = FunctionDefinition.GetDocumentation(); - } - return s ?? string.Empty; - } - } + public string Documentation => _documentation ?? ClassMember.Documentation; public string GetReturnDocumentation(IPythonType self = null) { if (self == null) { diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs index 7e95cc557..6708076e0 100644 --- a/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs @@ -53,21 +53,7 @@ public PythonFunctionType( Location location, IPythonType declaringType, string documentation - ) : this(name, location, declaringType, _ => documentation) { - Check.ArgumentNotNull(nameof(location), location.Module); - } - - /// - /// Creates function type to use in special cases when function is dynamically - /// created, such as in specializations and custom iterators, without the actual - /// function definition in the AST. - /// - public PythonFunctionType( - string name, - Location location, - IPythonType declaringType, - Func documentationProvider - ) : base(name, location, documentationProvider, declaringType != null ? BuiltinTypeId.Method : BuiltinTypeId.Function) { + ) : base(name, location, documentation, declaringType != null ? BuiltinTypeId.Method : BuiltinTypeId.Function) { DeclaringType = declaringType; } @@ -75,10 +61,9 @@ public PythonFunctionType( FunctionDefinition fd, IPythonType declaringType, Location location - ) : base(fd.Name, location, fd.Documentation, - declaringType != null ? BuiltinTypeId.Method : BuiltinTypeId.Function) { + ) : base(fd.Name, location, fd.Documentation, declaringType != null ? BuiltinTypeId.Method : BuiltinTypeId.Function) { - FunctionDefinition = fd; + location.Module.AddAstNode(this, fd); DeclaringType = declaringType; if (fd.Name == "__init__") { @@ -88,7 +73,6 @@ Location location } #region IPythonType - public override PythonMemberType MemberType => TypeId == BuiltinTypeId.Function ? PythonMemberType.Function : PythonMemberType.Method; @@ -98,18 +82,17 @@ public override IMember Call(IPythonInstance instance, string memberName, IArgum return overload?.Call(args, instance?.GetPythonType() ?? DeclaringType); } - internal override void SetDocumentationProvider(Func provider) { + internal override void SetDocumentation(string documentation) { foreach (var o in Overloads) { - (o as PythonFunctionOverload)?.SetDocumentationProvider(provider); + (o as PythonFunctionOverload)?.SetDocumentation(documentation); } - - base.SetDocumentationProvider(provider); + base.SetDocumentation(documentation); } #endregion #region IPythonFunction - public FunctionDefinition FunctionDefinition { get; } + public FunctionDefinition FunctionDefinition => DeclaringModule.GetAstNode(this); public IPythonType DeclaringType { get; } public override string Documentation => _documentation ?? base.Documentation ?? (_overloads.Count > 0 ? _overloads[0].Documentation : default); public virtual bool IsClassMethod { get; private set; } diff --git a/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs b/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs index 87ee5f713..f10078945 100644 --- a/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonPropertyType.cs @@ -23,7 +23,7 @@ class PythonPropertyType : PythonType, IPythonPropertyType { public PythonPropertyType(FunctionDefinition fd, Location location, IPythonType declaringType, bool isAbstract) : this(fd.Name, location, declaringType, isAbstract) { - FunctionDefinition = fd; + declaringType.DeclaringModule.AddAstNode(this, fd); } public PythonPropertyType(string name, Location location, IPythonType declaringType, bool isAbstract) @@ -37,7 +37,7 @@ public PythonPropertyType(string name, Location location, IPythonType declaringT #endregion #region IPythonPropertyType - public FunctionDefinition FunctionDefinition { get; } + public FunctionDefinition FunctionDefinition => DeclaringModule.GetAstNode(this); public override bool IsAbstract { get; } public bool IsReadOnly => true; public IPythonType DeclaringType { get; } diff --git a/src/Analysis/Ast/Impl/Types/PythonType.cs b/src/Analysis/Ast/Impl/Types/PythonType.cs index b6c0798c6..0c123906c 100644 --- a/src/Analysis/Ast/Impl/Types/PythonType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonType.cs @@ -25,8 +25,6 @@ namespace Microsoft.Python.Analysis.Types { internal class PythonType : LocatedMember, IPythonType, IEquatable { private readonly object _lock = new object(); private readonly string _name; - private Func _documentationProvider; - private string _documentation; private Dictionary _members; private BuiltinTypeId _typeId; private bool _readonly; @@ -42,16 +40,7 @@ public PythonType( string documentation, BuiltinTypeId typeId = BuiltinTypeId.Unknown ) : this(name, location, typeId) { - _documentation = documentation; - } - - public PythonType( - string name, - Location location, - Func documentationProvider, - BuiltinTypeId typeId = BuiltinTypeId.Unknown - ) : this(name, location, typeId) { - _documentationProvider = documentationProvider; + Documentation = documentation; } private PythonType(string name, Location location, BuiltinTypeId typeId) : base(location) { @@ -67,7 +56,7 @@ private PythonType(string name, Location location, BuiltinTypeId typeId) : base( #region IPythonType public virtual string Name => TypeId == BuiltinTypeId.Ellipsis ? "..." : _name; - public virtual string Documentation => _documentationProvider != null ? _documentationProvider.Invoke(Name) : _documentation; + public virtual string Documentation { get; private set; } public virtual BuiltinTypeId TypeId => _typeId; public bool IsBuiltin => DeclaringModule == null || DeclaringModule is IBuiltinsPythonModule; public virtual bool IsAbstract => false; @@ -111,8 +100,7 @@ internal bool TrySetTypeId(BuiltinTypeId typeId) { return true; } - internal virtual void SetDocumentationProvider(Func provider) => _documentationProvider = provider; - internal virtual void SetDocumentation(string documentation) => _documentation = documentation; + internal virtual void SetDocumentation(string documentation) => Documentation = documentation; internal void AddMembers(IEnumerable variables, bool overwrite) { lock (_lock) { From 8a8d847705a5e47ac06a1574ea5874c850d522ee Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Fri, 17 May 2019 10:20:01 -0700 Subject: [PATCH 03/39] Class and scope to use AST map --- .../Ast/Impl/Extensions/PythonModuleExtensions.cs | 8 ++++---- .../Ast/Impl/Modules/Definitions/IAstNodeContainer.cs | 8 ++++---- src/Analysis/Ast/Impl/Modules/PythonModule.cs | 10 +++++----- src/Analysis/Ast/Impl/Types/PythonClassType.cs | 4 ++-- src/Analysis/Ast/Impl/Values/Scope.cs | 5 ++--- 5 files changed, 17 insertions(+), 18 deletions(-) diff --git a/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs b/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs index 36c55099e..6b5b5f22d 100644 --- a/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs @@ -19,9 +19,9 @@ namespace Microsoft.Python.Analysis { public static class PythonModuleExtensions { - internal static T GetAstNode(this IPythonModule module, IPythonType t) where T : Node - => ((IAstNodeContainer)module).GetAstNode(t); - internal static void AddAstNode(this IPythonModule module, IPythonType t, Node n) - => ((IAstNodeContainer)module).AddAstNode(t, n); + internal static T GetAstNode(this IPythonModule module, object o) where T : Node + => ((IAstNodeContainer)module).GetAstNode(o); + internal static void AddAstNode(this IPythonModule module, object o, Node n) + => ((IAstNodeContainer)module).AddAstNode(o, n); } } diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/IAstNodeContainer.cs b/src/Analysis/Ast/Impl/Modules/Definitions/IAstNodeContainer.cs index f4f04a8d4..c6fb7ce74 100644 --- a/src/Analysis/Ast/Impl/Modules/Definitions/IAstNodeContainer.cs +++ b/src/Analysis/Ast/Impl/Modules/Definitions/IAstNodeContainer.cs @@ -19,17 +19,17 @@ namespace Microsoft.Python.Analysis.Modules { internal interface IAstNodeContainer { /// - /// Provides access to AST nodes associated with types such as + /// Provides access to AST nodes associated with objects such as /// for . /// Nodes are not available for library modules as AST is not retained /// in libraries in order to conserve memory. /// - T GetAstNode(IPythonType t) where T : Node; + T GetAstNode(object o) where T : Node; /// - /// Associated AST node with the type. + /// Associated AST node with the object. /// - void AddAstNode(IPythonType t, Node n); + void AddAstNode(object o, Node n); /// /// Removes all AST node associations. diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index c033fc1e0..d7cd58814 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -56,7 +56,7 @@ private enum State { private readonly DocumentBuffer _buffer = new DocumentBuffer(); private readonly DisposeToken _disposeToken = DisposeToken.Create(); private IReadOnlyList _parseErrors = Array.Empty(); - private readonly Dictionary _astMap = new Dictionary(); + private readonly Dictionary _astMap = new Dictionary(); private readonly IDiagnosticsService _diagnosticsService; private string _documentation; // Must be null initially. @@ -422,10 +422,10 @@ protected virtual void OnAnalysisComplete() { } #endregion #region IAstNodeContainer - public T GetAstNode(IPythonType t) where T : Node => _astMap.TryGetValue(t, out var n) ? (T)n : null; - public void AddAstNode(IPythonType t, Node n) { - Debug.Assert(!_astMap.ContainsKey(t)); - _astMap[t] = n; + public T GetAstNode(object o) where T : Node => _astMap.TryGetValue(o, out var n) ? (T)n : null; + public void AddAstNode(object o, Node n) { + Debug.Assert(!_astMap.ContainsKey(o)); + _astMap[o] = n; } public void Clear() => _astMap.Clear(); #endregion diff --git a/src/Analysis/Ast/Impl/Types/PythonClassType.cs b/src/Analysis/Ast/Impl/Types/PythonClassType.cs index dded0f17d..d189e6ad3 100644 --- a/src/Analysis/Ast/Impl/Types/PythonClassType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonClassType.cs @@ -51,7 +51,7 @@ public PythonClassType( BuiltinTypeId builtinTypeId = BuiltinTypeId.Type ) : base(classDefinition.Name, location, classDefinition.GetDocumentation(), builtinTypeId) { Check.ArgumentNotNull(nameof(location), location.Module); - ClassDefinition = classDefinition; + DeclaringModule.AddAstNode(this, classDefinition); } #region IPythonType @@ -153,7 +153,7 @@ public override IMember Index(IPythonInstance instance, object index) { #endregion #region IPythonClass - public ClassDefinition ClassDefinition { get; } + public ClassDefinition ClassDefinition => DeclaringModule.GetAstNode(this); public IReadOnlyList Bases => _bases; public IReadOnlyList Mro { diff --git a/src/Analysis/Ast/Impl/Values/Scope.cs b/src/Analysis/Ast/Impl/Values/Scope.cs index c62e93d58..53e6fc0bb 100644 --- a/src/Analysis/Ast/Impl/Values/Scope.cs +++ b/src/Analysis/Ast/Impl/Values/Scope.cs @@ -32,16 +32,15 @@ internal class Scope : IScope { private List _childScopes; public Scope(ScopeStatement node, IScope outerScope, IPythonModule module) { - Node = node; OuterScope = outerScope; Module = module; + Module.AddAstNode(this, node); DeclareBuiltinVariables(); } #region IScope - public string Name => Node?.Name ?? ""; - public virtual ScopeStatement Node { get; } + public virtual ScopeStatement Node => Module.GetAstNode(this); public IScope OuterScope { get; } public IPythonModule Module { get; } From 89ce2ca1af0463ab53df92f51f80493a4570098a Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Fri, 17 May 2019 10:31:58 -0700 Subject: [PATCH 04/39] Library analysis --- .../Ast/Impl/Analyzer/DocumentAnalysis.cs | 26 ------ .../Ast/Impl/Analyzer/EmptyAnalysis.cs | 48 +++++++++++ .../Evaluation/ExpressionEval.Annotations.cs | 48 +++++++++++ .../Evaluation/ExpressionEval.Scopes.cs | 28 ------- .../Analyzer/Evaluation/ExpressionEval.cs | 11 ++- .../Ast/Impl/Analyzer/LibraryAnalysis.cs | 84 +++++++++++++++++++ .../Ast/Impl/Analyzer/ModuleWalker.cs | 2 +- .../Impl/Analyzer/PythonAnalyzerSession.cs | 7 +- .../Ast/Impl/Utilities/AstUtilities.cs | 29 +++++++ 9 files changed, 225 insertions(+), 58 deletions(-) create mode 100644 src/Analysis/Ast/Impl/Analyzer/EmptyAnalysis.cs create mode 100644 src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Annotations.cs create mode 100644 src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs create mode 100644 src/Analysis/Ast/Impl/Utilities/AstUtilities.cs diff --git a/src/Analysis/Ast/Impl/Analyzer/DocumentAnalysis.cs b/src/Analysis/Ast/Impl/Analyzer/DocumentAnalysis.cs index 2d6abf826..51e0cd378 100644 --- a/src/Analysis/Ast/Impl/Analyzer/DocumentAnalysis.cs +++ b/src/Analysis/Ast/Impl/Analyzer/DocumentAnalysis.cs @@ -13,17 +13,11 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.Collections.Generic; -using System.IO; -using System.Linq; -using Microsoft.Python.Analysis.Analyzer.Evaluation; using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Values; -using Microsoft.Python.Core; using Microsoft.Python.Core.Diagnostics; -using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer { @@ -77,24 +71,4 @@ public DocumentAnalysis(IDocument document, int version, IGlobalScope globalScop public IEnumerable Diagnostics => ExpressionEvaluator.Diagnostics; #endregion } - - public sealed class EmptyAnalysis : IDocumentAnalysis { - private static PythonAst _emptyAst; - - public EmptyAnalysis(IServiceContainer services, IDocument document) { - Document = document ?? throw new ArgumentNullException(nameof(document)); - GlobalScope = new EmptyGlobalScope(document); - - _emptyAst = _emptyAst ?? (_emptyAst = Parser.CreateParser(new StringReader(string.Empty), PythonLanguageVersion.None).ParseFile(document.Uri)); - ExpressionEvaluator = new ExpressionEval(services, document, Ast); - } - - public IDocument Document { get; } - public int Version { get; } = -1; - public IGlobalScope GlobalScope { get; } - public PythonAst Ast => _emptyAst; - public IExpressionEvaluator ExpressionEvaluator { get; } - public IReadOnlyList StarImportMemberNames => null; - public IEnumerable Diagnostics => Enumerable.Empty(); - } } diff --git a/src/Analysis/Ast/Impl/Analyzer/EmptyAnalysis.cs b/src/Analysis/Ast/Impl/Analyzer/EmptyAnalysis.cs new file mode 100644 index 000000000..5f62e83ad --- /dev/null +++ b/src/Analysis/Ast/Impl/Analyzer/EmptyAnalysis.cs @@ -0,0 +1,48 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Microsoft.Python.Analysis.Analyzer.Evaluation; +using Microsoft.Python.Analysis.Diagnostics; +using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core; +using Microsoft.Python.Parsing; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Analyzer { + public sealed class EmptyAnalysis : IDocumentAnalysis { + private static PythonAst _emptyAst; + + public EmptyAnalysis(IServiceContainer services, IDocument document) { + Document = document ?? throw new ArgumentNullException(nameof(document)); + GlobalScope = new EmptyGlobalScope(document); + + _emptyAst = _emptyAst ?? (_emptyAst = Parser.CreateParser(new StringReader(string.Empty), PythonLanguageVersion.None).ParseFile(document.Uri)); + ExpressionEvaluator = new ExpressionEval(services, document, Ast); + } + + public IDocument Document { get; } + public int Version { get; } = -1; + public IGlobalScope GlobalScope { get; } + public PythonAst Ast => _emptyAst; + public IExpressionEvaluator ExpressionEvaluator { get; } + public IReadOnlyList StarImportMemberNames => null; + public IEnumerable Diagnostics => Enumerable.Empty(); + } +} diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Annotations.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Annotations.cs new file mode 100644 index 000000000..a5973ab5c --- /dev/null +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Annotations.cs @@ -0,0 +1,48 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Analyzer.Evaluation { + internal sealed partial class ExpressionEval { + public IPythonType GetTypeFromAnnotation(Expression expr, LookupOptions options = LookupOptions.Global | LookupOptions.Builtins) + => GetTypeFromAnnotation(expr, out _, options); + + public IPythonType GetTypeFromAnnotation(Expression expr, out bool isGeneric, LookupOptions options = LookupOptions.Global | LookupOptions.Builtins) { + isGeneric = false; + switch (expr) { + case null: + return null; + case CallExpression callExpr: + // x: NamedTuple(...) + return GetValueFromCallable(callExpr)?.GetPythonType() ?? UnknownType; + case IndexExpression indexExpr: + // Try generics + var target = GetValueFromExpression(indexExpr.Target); + var result = GetValueFromGeneric(target, indexExpr); + if (result != null) { + isGeneric = true; + return result.GetPythonType(); + } + break; + } + + // Look at specialization and typing first + var ann = new TypeAnnotation(Ast.LanguageVersion, expr); + return ann.GetValue(new TypeAnnotationConverter(this, options)); + } + } +} diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs index a2060a301..f332513d9 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs @@ -21,7 +21,6 @@ using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; using Microsoft.Python.Core.Disposables; -using Microsoft.Python.Core.Text; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer.Evaluation { @@ -117,33 +116,6 @@ public IMember LookupNameInScopes(string name, out IScope scope, out IVariable v return value; } - public IPythonType GetTypeFromAnnotation(Expression expr, LookupOptions options = LookupOptions.Global | LookupOptions.Builtins) - => GetTypeFromAnnotation(expr, out _, options); - - public IPythonType GetTypeFromAnnotation(Expression expr, out bool isGeneric, LookupOptions options = LookupOptions.Global | LookupOptions.Builtins) { - isGeneric = false; - switch (expr) { - case null: - return null; - case CallExpression callExpr: - // x: NamedTuple(...) - return GetValueFromCallable(callExpr)?.GetPythonType() ?? UnknownType; - case IndexExpression indexExpr: - // Try generics - var target = GetValueFromExpression(indexExpr.Target); - var result = GetValueFromGeneric(target, indexExpr); - if (result != null) { - isGeneric = true; - return result.GetPythonType(); - } - break; - } - - // Look at specialization and typing first - var ann = new TypeAnnotation(Ast.LanguageVersion, expr); - return ann.GetValue(new TypeAnnotationConverter(this, options)); - } - /// /// Locates and opens existing scope for a node or creates a new scope /// as a child of the specified scope. Scope is pushed on the stack diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs index 0a637412c..8bf17895b 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs @@ -20,6 +20,7 @@ using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Analysis.Utilities; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; using Microsoft.Python.Core.Disposables; @@ -37,12 +38,18 @@ internal sealed partial class ExpressionEval : IExpressionEvaluator { private readonly object _lock = new object(); private readonly List _diagnostics = new List(); - public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAst ast) { + public ExpressionEval(IServiceContainer services, IPythonModule module, GlobalScope gs) + : this(services, module, AstUtilities.MakeEmptyAst(module.Uri), gs) { } + + public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAst ast) + : this(services, module, ast, new GlobalScope(module)) { } + + public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAst ast, GlobalScope gs) { Services = services ?? throw new ArgumentNullException(nameof(services)); Module = module ?? throw new ArgumentNullException(nameof(module)); Ast = ast ?? throw new ArgumentNullException(nameof(ast)); - GlobalScope = new GlobalScope(module); + GlobalScope = gs; CurrentScope = GlobalScope; DefaultLocation = new Location(module); //Log = services.GetService(); diff --git a/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs b/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs new file mode 100644 index 000000000..3c8a76575 --- /dev/null +++ b/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs @@ -0,0 +1,84 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Analysis.Analyzer.Evaluation; +using Microsoft.Python.Analysis.Diagnostics; +using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Analysis.Modules; +using Microsoft.Python.Analysis.Values; +using Microsoft.Python.Core; +using Microsoft.Python.Core.Diagnostics; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Analyzer { + /// + /// Analysis of a library code. + /// + internal sealed class LibraryAnalysis : IDocumentAnalysis { + public LibraryAnalysis(IDocument document, int version, IServiceContainer services, GlobalScope globalScope, IReadOnlyList starImportMemberNames) { + Check.ArgumentNotNull(nameof(document), document); + Check.ArgumentNotNull(nameof(globalScope), globalScope); + Document = document; + Version = version; + GlobalScope = globalScope; + ExpressionEvaluator = new ExpressionEval(services, document, globalScope); + StarImportMemberNames = starImportMemberNames; + ((IAstNodeContainer)Document).Clear(); + } + + #region IDocumentAnalysis + /// + /// Analyzed document. + /// + public IDocument Document { get; } + + /// + /// Version of the analysis. Usually matches document version, + /// but can be lower when document or its dependencies were + /// updated since. + /// + public int Version { get; } + + /// + /// Empty AST. + /// + public PythonAst Ast => ExpressionEvaluator.Ast; + + /// + /// Document/module global scope. + /// + public IGlobalScope GlobalScope { get; } + + /// + /// Expression evaluator used in the analysis. + /// Only supports scope operation since there is no AST + /// when library analysis is complete. + /// + public IExpressionEvaluator ExpressionEvaluator { get; } + + /// + /// Members of the module which are transferred during a star import. null means __all__ was not defined. + /// + public IReadOnlyList StarImportMemberNames { get; } + + /// + /// Analysis diagnostics. + /// + public IEnumerable Diagnostics => Enumerable.Empty(); + #endregion + } +} diff --git a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs index 1ad20860d..19784204f 100644 --- a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs @@ -206,7 +206,7 @@ public void Complete() { Eval.ClearCache(); } - public IGlobalScope GlobalScope => Eval.GlobalScope; + public GlobalScope GlobalScope => Eval.GlobalScope; public IReadOnlyList StarImportMemberNames { get; private set; } /// diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index ca2a0a7df..fbe7d8ac4 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -349,7 +349,7 @@ private void AnalyzeEntry(PythonAnalyzerEntry entry, IPythonModule module, Pytho walker.Complete(); _analyzerCancellationToken.ThrowIfCancellationRequested(); - var analysis = new DocumentAnalysis((IDocument)module, version, walker.GlobalScope, walker.Eval, walker.StarImportMemberNames); + var analysis = CreateAnalysis((IDocument)module, version, walker); analyzable?.NotifyAnalysisComplete(analysis); entry.TrySetAnalysis(analysis, version); @@ -365,5 +365,10 @@ private enum State { Started = 1, Completed = 2 } + + private IDocumentAnalysis CreateAnalysis(IDocument document, int version, ModuleWalker walker) + => document.ModuleType == ModuleType.User + ? (IDocumentAnalysis)new DocumentAnalysis(document, version, walker.GlobalScope, walker.Eval, walker.StarImportMemberNames) + : new LibraryAnalysis(document, version, walker.Eval.Services, walker.GlobalScope, walker.StarImportMemberNames); } } diff --git a/src/Analysis/Ast/Impl/Utilities/AstUtilities.cs b/src/Analysis/Ast/Impl/Utilities/AstUtilities.cs new file mode 100644 index 000000000..816b0c304 --- /dev/null +++ b/src/Analysis/Ast/Impl/Utilities/AstUtilities.cs @@ -0,0 +1,29 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.IO; +using Microsoft.Python.Parsing; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Utilities { + public static class AstUtilities { + public static PythonAst MakeEmptyAst(Uri documentUri) { + using (var sr = new StringReader(string.Empty)) { + return Parser.CreateParser(sr, PythonLanguageVersion.None).ParseFile(documentUri); + } + } + } +} From 7a93bf7a85d65e3081e14bf99e7f25fc6ab33734 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Sat, 18 May 2019 11:04:33 -0700 Subject: [PATCH 05/39] Fix SO --- .../Analyzer/Definitions/IPythonAnalyzer.cs | 2 +- .../Ast/Impl/Analyzer/PythonAnalyzer.cs | 10 ++++------ .../Ast/Impl/Analyzer/PythonAnalyzerEntry.cs | 18 +++++++----------- .../Impl/Extensions/PythonModuleExtensions.cs | 3 +++ src/Analysis/Ast/Impl/Modules/PythonModule.cs | 6 +++--- .../Ast/Impl/Types/PythonFunctionOverload.cs | 6 ++---- .../Ast/Impl/Types/PythonFunctionType.cs | 11 ++++------- src/Analysis/Ast/Impl/Values/Scope.cs | 6 ++++-- src/LanguageServer/Impl/Program.cs | 2 +- 9 files changed, 29 insertions(+), 35 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs index 1922eb08e..94a298a26 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs @@ -27,7 +27,7 @@ public interface IPythonAnalyzer { /// /// Schedules module for analysis. Module will be scheduled if version of AST is greater than the one used to get previous analysis /// - void EnqueueDocumentForAnalysis(IPythonModule module, PythonAst ast, int version); + void EnqueueDocumentForAnalysis(IPythonModule module, int version); /// /// Schedules module for analysis for its existing AST, but with new dependencies. diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index 888a6a945..c6d88f9e2 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -30,7 +30,6 @@ using Microsoft.Python.Core.Disposables; using Microsoft.Python.Core.Logging; using Microsoft.Python.Core.Services; -using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer { public sealed class PythonAnalyzer : IPythonAnalyzer, IDisposable { @@ -63,9 +62,8 @@ public void Dispose() { _disposeToken.TryMarkDisposed(); } - public Task WaitForCompleteAnalysisAsync(CancellationToken cancellationToken = default) { - return _analysisCompleteEvent.WaitAsync(cancellationToken); - } + public Task WaitForCompleteAnalysisAsync(CancellationToken cancellationToken = default) + => _analysisCompleteEvent.WaitAsync(cancellationToken); public async Task GetAnalysisAsync(IPythonModule module, int waitTime, CancellationToken cancellationToken) { var key = new AnalysisModuleKey(module); @@ -144,7 +142,7 @@ public void EnqueueDocumentForAnalysis(IPythonModule module, ImmutableArray _analysisTcs; private IPythonModule _module; private bool _isUserModule; - private PythonAst _ast; private IDocumentAnalysis _previousAnalysis; private HashSet _parserDependencies; private HashSet _analysisDependencies; @@ -102,7 +101,7 @@ public Task GetAnalysisAsync(CancellationToken cancellationTo public bool IsValidVersion(int version, out IPythonModule module, out PythonAst ast) { lock (_syncObj) { module = _module; - ast = _ast; + ast = module.GetAst(); if (ast == null || module == null) { return false; } @@ -210,19 +209,18 @@ public bool Invalidate(ImmutableArray analysisDependencies, int a } } - public bool Invalidate(IPythonModule module, PythonAst ast, int bufferVersion, int analysisVersion, out ImmutableArray dependencies) { + public bool Invalidate(IPythonModule module, int bufferVersion, int analysisVersion, out ImmutableArray dependencies) { dependencies = ImmutableArray.Empty; if (_bufferVersion >= bufferVersion) { return false; } - var dependenciesHashSet = FindDependencies(module, ast, bufferVersion); + var dependenciesHashSet = FindDependencies(module, bufferVersion); lock (_syncObj) { if (_analysisVersion >= analysisVersion && _bufferVersion >= bufferVersion) { return false; } - _ast = ast; _module = module; _isUserModule = module.ModuleType == ModuleType.User; _parserDependencies = dependenciesHashSet; @@ -236,13 +234,13 @@ public bool Invalidate(IPythonModule module, PythonAst ast, int bufferVersion, i } } - private HashSet FindDependencies(IPythonModule module, PythonAst ast, int bufferVersion) { + private HashSet FindDependencies(IPythonModule module, int bufferVersion) { if (_bufferVersion > bufferVersion) { return new HashSet(); } - var walker = new DependencyWalker(this, module); - ast.Walk(walker); + var walker = new DependencyWalker(module); + module.GetAst().Walk(walker); var dependencies = walker.Dependencies; dependencies.Remove(new AnalysisModuleKey(module)); return dependencies; @@ -264,7 +262,6 @@ private void UpdateAnalysisTcs(int analysisVersion) { } private class DependencyWalker : PythonWalker { - private readonly PythonAnalyzerEntry _entry; private readonly IPythonModule _module; private readonly bool _isTypeshed; private readonly IModuleManagement _moduleResolution; @@ -272,8 +269,7 @@ private class DependencyWalker : PythonWalker { public HashSet Dependencies { get; } - public DependencyWalker(PythonAnalyzerEntry entry, IPythonModule module) { - _entry = entry; + public DependencyWalker(IPythonModule module) { _module = module; _isTypeshed = module is StubPythonModule stub && stub.IsTypeshed; _moduleResolution = module.Interpreter.ModuleResolution; diff --git a/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs b/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs index 6b5b5f22d..c33b4620f 100644 --- a/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/PythonModuleExtensions.cs @@ -19,6 +19,9 @@ namespace Microsoft.Python.Analysis { public static class PythonModuleExtensions { + internal static PythonAst GetAst(this IPythonModule module) + => ((IAstNodeContainer)module).GetAstNode(module); + internal static T GetAstNode(this IPythonModule module, object o) where T : Node => ((IAstNodeContainer)module).GetAstNode(o); internal static void AddAstNode(this IPythonModule module, object o, Node n) diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index d7cd58814..9a05b8d99 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -343,7 +343,7 @@ private void Parse(CancellationToken cancellationToken) { ContentState = State.Analyzing; var analyzer = Services.GetService(); - analyzer.EnqueueDocumentForAnalysis(this, ast, version); + analyzer.EnqueueDocumentForAnalysis(this, version); } lock (AnalysisLock) { @@ -424,10 +424,10 @@ protected virtual void OnAnalysisComplete() { } #region IAstNodeContainer public T GetAstNode(object o) where T : Node => _astMap.TryGetValue(o, out var n) ? (T)n : null; public void AddAstNode(object o, Node n) { - Debug.Assert(!_astMap.ContainsKey(o)); + Debug.Assert(!_astMap.ContainsKey(o) || _astMap[o] == n); _astMap[o] = n; } - public void Clear() => _astMap.Clear(); + public void Clear() =>_astMap.Clear(); #endregion #region Analysis diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs index 33f47eb04..51e28f638 100644 --- a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs +++ b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs @@ -48,7 +48,6 @@ internal sealed class PythonFunctionOverload : LocatedMember, IPythonFunctionOve // Return value can be an instance or a type info. Consider type(C()) returning // type info of C vs. return C() that returns an instance of C. private bool _fromAnnotation; - private string _documentation; public PythonFunctionOverload(IPythonClassMember cm, Location location) : this(cm.Name, location) { @@ -65,7 +64,6 @@ public PythonFunctionOverload(IPythonClassMember cm, Location location) break; } - _documentation = fd?.Documentation; _returnDocumentation = ast != null ? fd?.ReturnAnnotation?.ToCodeString(ast) : null; } @@ -103,13 +101,13 @@ internal void SetReturnValue(IMember value, bool fromAnnotation) { } internal void SetReturnValueProvider(ReturnValueProvider provider) => _returnValueProvider = provider; - internal void SetDocumentation(string documentation) => _documentation = documentation; + internal void SetDocumentation(string documentation) => Documentation = documentation; #region IPythonFunctionOverload public FunctionDefinition FunctionDefinition => ClassMember.DeclaringModule.GetAstNode(ClassMember); public IPythonClassMember ClassMember { get; } public string Name { get; } - public string Documentation => _documentation ?? ClassMember.Documentation; + public string Documentation { get; private set; } public string GetReturnDocumentation(IPythonType self = null) { if (self == null) { diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs index 6708076e0..c05db492e 100644 --- a/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs @@ -13,7 +13,6 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.Collections.Generic; using System.Diagnostics; using System.Linq; @@ -27,7 +26,6 @@ namespace Microsoft.Python.Analysis.Types { [DebuggerDisplay("Function {Name} ({TypeId})")] internal class PythonFunctionType : PythonType, IPythonFunctionType { private ImmutableArray _overloads = ImmutableArray.Empty; - private readonly string _documentation; private bool _isAbstract; private bool _isSpecialized; @@ -61,14 +59,13 @@ public PythonFunctionType( FunctionDefinition fd, IPythonType declaringType, Location location - ) : base(fd.Name, location, fd.Documentation, declaringType != null ? BuiltinTypeId.Method : BuiltinTypeId.Function) { + ) : base(fd.Name, location, + fd.Name == "__init__" ? declaringType?.Documentation : fd.Documentation, + declaringType != null ? BuiltinTypeId.Method : BuiltinTypeId.Function) { location.Module.AddAstNode(this, fd); DeclaringType = declaringType; - if (fd.Name == "__init__") { - _documentation = declaringType?.Documentation; - } ProcessDecorators(fd); } @@ -94,7 +91,7 @@ internal override void SetDocumentation(string documentation) { #region IPythonFunction public FunctionDefinition FunctionDefinition => DeclaringModule.GetAstNode(this); public IPythonType DeclaringType { get; } - public override string Documentation => _documentation ?? base.Documentation ?? (_overloads.Count > 0 ? _overloads[0].Documentation : default); + public override string Documentation => (_overloads.Count > 0 ? _overloads[0].Documentation : default) ?? base.Documentation; public virtual bool IsClassMethod { get; private set; } public virtual bool IsStatic { get; private set; } public override bool IsAbstract => _isAbstract; diff --git a/src/Analysis/Ast/Impl/Values/Scope.cs b/src/Analysis/Ast/Impl/Values/Scope.cs index 53e6fc0bb..3ce44ab06 100644 --- a/src/Analysis/Ast/Impl/Values/Scope.cs +++ b/src/Analysis/Ast/Impl/Values/Scope.cs @@ -34,13 +34,15 @@ internal class Scope : IScope { public Scope(ScopeStatement node, IScope outerScope, IPythonModule module) { OuterScope = outerScope; Module = module; - Module.AddAstNode(this, node); + if (node != null) { + Module.AddAstNode(this, node); + } DeclareBuiltinVariables(); } #region IScope public string Name => Node?.Name ?? ""; - public virtual ScopeStatement Node => Module.GetAstNode(this); + public virtual ScopeStatement Node => Module.GetAstNode(this); public IScope OuterScope { get; } public IPythonModule Module { get; } diff --git a/src/LanguageServer/Impl/Program.cs b/src/LanguageServer/Impl/Program.cs index 024f9af18..24adfa924 100644 --- a/src/LanguageServer/Impl/Program.cs +++ b/src/LanguageServer/Impl/Program.cs @@ -13,7 +13,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -// #define WAIT_FOR_DEBUGGER +#define WAIT_FOR_DEBUGGER using System; using System.Diagnostics; From 2a2bdf86d6beaa1d698479f7db8663e15dd4b85f Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Sat, 18 May 2019 14:02:29 -0700 Subject: [PATCH 06/39] Keep small AST with imports --- src/Analysis/Ast/Impl/Analyzer/EmptyAnalysis.cs | 12 ++++-------- .../Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs | 11 ++++------- src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs | 8 +++++++- src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs | 4 ++-- .../Ast/Impl/Analyzer/PythonAnalyzerSession.cs | 10 +++++----- src/Analysis/Ast/Impl/Modules/PythonModule.cs | 4 ++++ src/Analysis/Ast/Impl/Types/ArgumentSet.cs | 13 +++++++------ .../Ast/Impl/Types/PythonFunctionOverload.cs | 2 +- src/Analysis/Ast/Impl/Types/PythonFunctionType.cs | 3 +-- src/Analysis/Ast/Impl/Types/PythonType.cs | 10 +++++----- src/Parsing/Impl/Ast/SuiteStatement.cs | 8 ++++++-- 11 files changed, 46 insertions(+), 39 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/EmptyAnalysis.cs b/src/Analysis/Ast/Impl/Analyzer/EmptyAnalysis.cs index 5f62e83ad..b073f20b2 100644 --- a/src/Analysis/Ast/Impl/Analyzer/EmptyAnalysis.cs +++ b/src/Analysis/Ast/Impl/Analyzer/EmptyAnalysis.cs @@ -15,32 +15,28 @@ using System; using System.Collections.Generic; -using System.IO; using System.Linq; using Microsoft.Python.Analysis.Analyzer.Evaluation; using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Analysis.Utilities; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; -using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer { public sealed class EmptyAnalysis : IDocumentAnalysis { - private static PythonAst _emptyAst; - public EmptyAnalysis(IServiceContainer services, IDocument document) { Document = document ?? throw new ArgumentNullException(nameof(document)); GlobalScope = new EmptyGlobalScope(document); - - _emptyAst = _emptyAst ?? (_emptyAst = Parser.CreateParser(new StringReader(string.Empty), PythonLanguageVersion.None).ParseFile(document.Uri)); - ExpressionEvaluator = new ExpressionEval(services, document, Ast); + Ast = AstUtilities.MakeEmptyAst(document.Uri); + ExpressionEvaluator = new ExpressionEval(services, document); } public IDocument Document { get; } public int Version { get; } = -1; public IGlobalScope GlobalScope { get; } - public PythonAst Ast => _emptyAst; + public PythonAst Ast { get; } public IExpressionEvaluator ExpressionEvaluator { get; } public IReadOnlyList StarImportMemberNames => null; public IEnumerable Diagnostics => Enumerable.Empty(); diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs index 8bf17895b..24c8515dc 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs @@ -38,16 +38,13 @@ internal sealed partial class ExpressionEval : IExpressionEvaluator { private readonly object _lock = new object(); private readonly List _diagnostics = new List(); - public ExpressionEval(IServiceContainer services, IPythonModule module, GlobalScope gs) - : this(services, module, AstUtilities.MakeEmptyAst(module.Uri), gs) { } + public ExpressionEval(IServiceContainer services, IPythonModule module) + : this(services, module, new GlobalScope(module)) { } - public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAst ast) - : this(services, module, ast, new GlobalScope(module)) { } - - public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAst ast, GlobalScope gs) { + public ExpressionEval(IServiceContainer services, IPythonModule module, GlobalScope gs) { Services = services ?? throw new ArgumentNullException(nameof(services)); Module = module ?? throw new ArgumentNullException(nameof(module)); - Ast = ast ?? throw new ArgumentNullException(nameof(ast)); + Ast = module.GetAst(); GlobalScope = gs; CurrentScope = GlobalScope; diff --git a/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs b/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs index 3c8a76575..240abfd5a 100644 --- a/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs +++ b/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs @@ -35,9 +35,15 @@ public LibraryAnalysis(IDocument document, int version, IServiceContainer servic Document = document; Version = version; GlobalScope = globalScope; + + var ast = Document.GetAst(); + (ast.Body as SuiteStatement)?.RemoveStatements(x => !(x is ImportStatement || x is FromImportStatement)); + var c = (IAstNodeContainer)Document; + c.Clear(); + c.AddAstNode(document, ast); + ExpressionEvaluator = new ExpressionEval(services, document, globalScope); StarImportMemberNames = starImportMemberNames; - ((IAstNodeContainer)Document).Clear(); } #region IDocumentAnalysis diff --git a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs index 19784204f..5d8fc92f3 100644 --- a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs @@ -36,8 +36,8 @@ internal class ModuleWalker : AnalysisWalker { private int _allReferencesCount; private bool _allIsUsable = true; - public ModuleWalker(IServiceContainer services, IPythonModule module, PythonAst ast) - : base(new ExpressionEval(services, module, ast)) { + public ModuleWalker(IServiceContainer services, IPythonModule module) + : base(new ExpressionEval(services, module)) { _stubAnalysis = Module.Stub is IDocument doc ? doc.GetAnyAnalysis() : null; } diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index fbe7d8ac4..c9badc025 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -277,7 +277,7 @@ private void Analyze(IDependencyChainNode node, AsyncCountd return; } var startTime = stopWatch.Elapsed; - AnalyzeEntry(entry, module, ast, _walker.Version); + AnalyzeEntry(entry, module, _walker.Version); node.Commit(); _log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) completed in {(stopWatch.Elapsed - startTime).TotalMilliseconds} ms."); @@ -314,13 +314,12 @@ private void AnalyzeEntry() { // Entry doesn't have ast yet. There should be at least one more session. Cancel(); } - _log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled."); return; } var startTime = stopWatch.Elapsed; - AnalyzeEntry(_entry, module, ast, Version); + AnalyzeEntry(_entry, module, Version); _log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) completed in {(stopWatch.Elapsed - startTime).TotalMilliseconds} ms."); } catch (OperationCanceledException oce) { @@ -337,12 +336,13 @@ private void AnalyzeEntry() { } } - private void AnalyzeEntry(PythonAnalyzerEntry entry, IPythonModule module, PythonAst ast, int version) { + private void AnalyzeEntry(PythonAnalyzerEntry entry, IPythonModule module, int version) { // Now run the analysis. var analyzable = module as IAnalyzable; analyzable?.NotifyAnalysisBegins(); - var walker = new ModuleWalker(_services, module, ast); + var ast = module.GetAst(); + var walker = new ModuleWalker(_services, module); ast.Walk(walker); _analyzerCancellationToken.ThrowIfCancellationRequested(); diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index 9a05b8d99..26204290a 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -326,7 +326,11 @@ private void Parse(CancellationToken cancellationToken) { if (version != _buffer.Version) { throw new OperationCanceledException(); } + + // Stored nodes are no longer valid. + _astMap.Clear(); _astMap[this] = ast; + _parseErrors = sink?.Diagnostics ?? Array.Empty(); // Do not report issues with libraries or stubs diff --git a/src/Analysis/Ast/Impl/Types/ArgumentSet.cs b/src/Analysis/Ast/Impl/Types/ArgumentSet.cs index 344f47dbb..4d0b741a8 100644 --- a/src/Analysis/Ast/Impl/Types/ArgumentSet.cs +++ b/src/Analysis/Ast/Impl/Types/ArgumentSet.cs @@ -81,6 +81,13 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in OverloadIndex = overloadIndex; DeclaringModule = fn.DeclaringModule; + if (callExpr == null) { + // Typically invoked by specialization code without call expression in the code. + // Caller usually does not care about arguments. + _evaluated = true; + return; + } + var overload = fn.Overloads[overloadIndex]; var fd = overload.FunctionDefinition; @@ -101,12 +108,6 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in return; } - if (callExpr == null) { - // Typically invoked by specialization code without call expression in the code. - // Caller usually does not care about arguments. - _evaluated = true; - return; - } var callLocation = callExpr.GetLocation(module); // https://www.python.org/dev/peps/pep-3102/#id5 diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs index 51e28f638..4269fc5bf 100644 --- a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs +++ b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs @@ -104,7 +104,7 @@ internal void SetReturnValue(IMember value, bool fromAnnotation) { internal void SetDocumentation(string documentation) => Documentation = documentation; #region IPythonFunctionOverload - public FunctionDefinition FunctionDefinition => ClassMember.DeclaringModule.GetAstNode(ClassMember); + public FunctionDefinition FunctionDefinition => ClassMember?.DeclaringModule?.GetAstNode(ClassMember); public IPythonClassMember ClassMember { get; } public string Name { get; } public string Documentation { get; private set; } diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs index c05db492e..041f91eaf 100644 --- a/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs @@ -62,10 +62,9 @@ Location location ) : base(fd.Name, location, fd.Name == "__init__" ? declaringType?.Documentation : fd.Documentation, declaringType != null ? BuiltinTypeId.Method : BuiltinTypeId.Function) { - - location.Module.AddAstNode(this, fd); DeclaringType = declaringType; + location.Module.AddAstNode(this, fd); ProcessDecorators(fd); } diff --git a/src/Analysis/Ast/Impl/Types/PythonType.cs b/src/Analysis/Ast/Impl/Types/PythonType.cs index 0c123906c..27c8ffaff 100644 --- a/src/Analysis/Ast/Impl/Types/PythonType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonType.cs @@ -22,7 +22,7 @@ namespace Microsoft.Python.Analysis.Types { [DebuggerDisplay("{Name}")] - internal class PythonType : LocatedMember, IPythonType, IEquatable { + internal class PythonType : LocatedMember, IPythonType {//, IEquatable { private readonly object _lock = new object(); private readonly string _name; private Dictionary _members; @@ -149,10 +149,10 @@ internal IMember AddMember(string name, IMember member, bool overwrite) { protected bool ContainsMember(string name) => Members.ContainsKey(name); protected IMember UnknownType => DeclaringModule.Interpreter.UnknownType; - public bool Equals(IPythonType other) => PythonTypeComparer.Instance.Equals(this, other); + //public bool Equals(IPythonType other) => PythonTypeComparer.Instance.Equals(this, other); - public override bool Equals(object obj) - => obj is IPythonType pt && PythonTypeComparer.Instance.Equals(this, pt); - public override int GetHashCode() => 0; + //public override bool Equals(object obj) + // => obj is IPythonType pt && PythonTypeComparer.Instance.Equals(this, pt); + //public override int GetHashCode() => 0; } } diff --git a/src/Parsing/Impl/Ast/SuiteStatement.cs b/src/Parsing/Impl/Ast/SuiteStatement.cs index ddb4c87ac..a47b197f4 100644 --- a/src/Parsing/Impl/Ast/SuiteStatement.cs +++ b/src/Parsing/Impl/Ast/SuiteStatement.cs @@ -14,7 +14,9 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; using System.Collections.Generic; +using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -22,16 +24,18 @@ namespace Microsoft.Python.Parsing.Ast { public sealed class SuiteStatement : Statement { - private readonly Statement[] _statements; + private Statement[] _statements; public SuiteStatement(Statement[] statements) { _statements = statements; } public IList Statements => _statements; - public override IEnumerable GetChildNodes() => _statements.WhereNotNull(); + public void RemoveStatements(Func filter) + => _statements = _statements.Where(filter).ToArray(); + public override void Walk(PythonWalker walker) { if (walker.Walk(this)) { foreach (var s in _statements.MaybeEnumerate()) { From 3edaa73be40b5907759c5a807679efa6cc058ec9 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Sat, 18 May 2019 14:21:24 -0700 Subject: [PATCH 07/39] AST reduction --- src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs | 3 ++- src/Parsing/Impl/Ast/PythonAst.cs | 9 +++++---- src/Parsing/Impl/Ast/SuiteStatement.cs | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs b/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs index 240abfd5a..fb76b2fd4 100644 --- a/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs +++ b/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs @@ -37,7 +37,8 @@ public LibraryAnalysis(IDocument document, int version, IServiceContainer servic GlobalScope = globalScope; var ast = Document.GetAst(); - (ast.Body as SuiteStatement)?.RemoveStatements(x => !(x is ImportStatement || x is FromImportStatement)); + (ast.Body as SuiteStatement)?.FilterStatements(x => x is ImportStatement || x is FromImportStatement); + ast.Reduce(); var c = (IAstNodeContainer)Document; c.Clear(); c.AddAstNode(document, ast); diff --git a/src/Parsing/Impl/Ast/PythonAst.cs b/src/Parsing/Impl/Ast/PythonAst.cs index 8ce4a97de..c91ecfda5 100644 --- a/src/Parsing/Impl/Ast/PythonAst.cs +++ b/src/Parsing/Impl/Ast/PythonAst.cs @@ -36,7 +36,6 @@ public PythonAst(Uri module, Statement body, NewLineLocation[] lineLocations, Py LanguageVersion = langVersion; NewLineLocations = lineLocations; CommentLocations = commentLocations; - } public PythonAst(IEnumerable existingAst) { @@ -62,7 +61,6 @@ public PythonAst(IEnumerable existingAst) { public Uri Module { get; } public NewLineLocation[] NewLineLocations { get; } public SourceLocation[] CommentLocations { get; } - public override string Name => ""; /// @@ -76,7 +74,7 @@ public PythonAst(IEnumerable existingAst) { /// public bool HasVerbatim { get; internal set; } - public override IEnumerable GetChildNodes() => new[] {_body}; + public override IEnumerable GetChildNodes() => new[] { _body }; public override void Walk(PythonWalker walker) { if (walker.Walk(this)) { @@ -93,9 +91,12 @@ public override async Task WalkAsync(PythonWalkerAsync walker, CancellationToken } public override Statement Body => _body; - public PythonLanguageVersion LanguageVersion { get; } + public void Reduce() { + _attributes.Clear(); + } + internal bool TryGetAttribute(Node node, object key, out object value) { if (_attributes.TryGetValue(node, out var nodeAttrs)) { return nodeAttrs.TryGetValue(key, out value); diff --git a/src/Parsing/Impl/Ast/SuiteStatement.cs b/src/Parsing/Impl/Ast/SuiteStatement.cs index a47b197f4..920a1d8b6 100644 --- a/src/Parsing/Impl/Ast/SuiteStatement.cs +++ b/src/Parsing/Impl/Ast/SuiteStatement.cs @@ -33,7 +33,7 @@ public SuiteStatement(Statement[] statements) { public IList Statements => _statements; public override IEnumerable GetChildNodes() => _statements.WhereNotNull(); - public void RemoveStatements(Func filter) + public void FilterStatements(Func filter) => _statements = _statements.Where(filter).ToArray(); public override void Walk(PythonWalker walker) { From 73371e13a0e60c7de7a577c4d68cf6df3c750541 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Sat, 18 May 2019 14:53:14 -0700 Subject: [PATCH 08/39] Final field --- .../Ast/Impl/Analyzer/LibraryAnalysis.cs | 3 +- .../Impl/Analyzer/PythonAnalyzerSession.cs | 13 +++--- .../Impl/Dependencies/DependencyResolver.cs | 1 + .../Impl/Dependencies/IDependencyChainNode.cs | 1 + src/Parsing/Impl/Ast/ClassDefinition.cs | 2 - src/Parsing/Impl/Ast/FunctionDefinition.cs | 2 - src/Parsing/Impl/Ast/PythonAst.cs | 13 ++++-- src/Parsing/Impl/Ast/ScopeStatement.cs | 46 ++++++------------- 8 files changed, 32 insertions(+), 49 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs b/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs index fb76b2fd4..f90d57985 100644 --- a/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs +++ b/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs @@ -37,8 +37,7 @@ public LibraryAnalysis(IDocument document, int version, IServiceContainer servic GlobalScope = globalScope; var ast = Document.GetAst(); - (ast.Body as SuiteStatement)?.FilterStatements(x => x is ImportStatement || x is FromImportStatement); - ast.Reduce(); + ast.Reduce(x => x is ImportStatement || x is FromImportStatement); var c = (IAstNodeContainer)Document; c.Clear(); c.AddAstNode(document, ast); diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index c9badc025..d0f251ac2 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -277,7 +277,7 @@ private void Analyze(IDependencyChainNode node, AsyncCountd return; } var startTime = stopWatch.Elapsed; - AnalyzeEntry(entry, module, _walker.Version); + AnalyzeEntry(entry, module, _walker.Version, node.IsComplete); node.Commit(); _log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) completed in {(stopWatch.Elapsed - startTime).TotalMilliseconds} ms."); @@ -319,7 +319,7 @@ private void AnalyzeEntry() { } var startTime = stopWatch.Elapsed; - AnalyzeEntry(_entry, module, Version); + AnalyzeEntry(_entry, module, Version, true); _log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) completed in {(stopWatch.Elapsed - startTime).TotalMilliseconds} ms."); } catch (OperationCanceledException oce) { @@ -336,7 +336,7 @@ private void AnalyzeEntry() { } } - private void AnalyzeEntry(PythonAnalyzerEntry entry, IPythonModule module, int version) { + private void AnalyzeEntry(PythonAnalyzerEntry entry, IPythonModule module, int version, bool isFinalPass) { // Now run the analysis. var analyzable = module as IAnalyzable; analyzable?.NotifyAnalysisBegins(); @@ -349,7 +349,8 @@ private void AnalyzeEntry(PythonAnalyzerEntry entry, IPythonModule module, int v walker.Complete(); _analyzerCancellationToken.ThrowIfCancellationRequested(); - var analysis = CreateAnalysis((IDocument)module, version, walker); + + var analysis = CreateAnalysis((IDocument)module, version, walker, isFinalPass); analyzable?.NotifyAnalysisComplete(analysis); entry.TrySetAnalysis(analysis, version); @@ -366,8 +367,8 @@ private enum State { Completed = 2 } - private IDocumentAnalysis CreateAnalysis(IDocument document, int version, ModuleWalker walker) - => document.ModuleType == ModuleType.User + private IDocumentAnalysis CreateAnalysis(IDocument document, int version, ModuleWalker walker, bool isFinalPass) + => document.ModuleType == ModuleType.User || !isFinalPass ? (IDocumentAnalysis)new DocumentAnalysis(document, version, walker.GlobalScope, walker.Eval, walker.StarImportMemberNames) : new LibraryAnalysis(document, version, walker.Eval.Services, walker.GlobalScope, walker.StarImportMemberNames); } diff --git a/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs b/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs index 5e806d7a8..df05c9ee6 100644 --- a/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs +++ b/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs @@ -604,6 +604,7 @@ private sealed class DependencyChainNode : IDependencyChainNode { private DependencyChainWalker _walker; public TValue Value => _vertex.DependencyVertex.Value; public int VertexDepth { get; } + public bool IsComplete => _vertex.SecondPass == null; public DependencyChainNode(DependencyChainWalker walker, WalkingVertex vertex, int depth) { _walker = walker; diff --git a/src/Analysis/Ast/Impl/Dependencies/IDependencyChainNode.cs b/src/Analysis/Ast/Impl/Dependencies/IDependencyChainNode.cs index db637ad98..1e9ceb8be 100644 --- a/src/Analysis/Ast/Impl/Dependencies/IDependencyChainNode.cs +++ b/src/Analysis/Ast/Impl/Dependencies/IDependencyChainNode.cs @@ -19,5 +19,6 @@ internal interface IDependencyChainNode { TValue Value { get; } void Commit(); void Skip(); + bool IsComplete { get; } } } diff --git a/src/Parsing/Impl/Ast/ClassDefinition.cs b/src/Parsing/Impl/Ast/ClassDefinition.cs index c1db95498..9ec8289bf 100644 --- a/src/Parsing/Impl/Ast/ClassDefinition.cs +++ b/src/Parsing/Impl/Ast/ClassDefinition.cs @@ -85,8 +85,6 @@ internal override bool HasLateBoundVariableSets { set => base.HasLateBoundVariableSets = value; } - public override bool NeedsLocalContext => true; - internal override bool ExposesLocalVariable(PythonVariable variable) => true; internal override bool TryBindOuter(ScopeStatement from, string name, bool allowGlobals, out PythonVariable variable) { diff --git a/src/Parsing/Impl/Ast/FunctionDefinition.cs b/src/Parsing/Impl/Ast/FunctionDefinition.cs index 1b70c2c44..46efe1e7b 100644 --- a/src/Parsing/Impl/Ast/FunctionDefinition.cs +++ b/src/Parsing/Impl/Ast/FunctionDefinition.cs @@ -52,8 +52,6 @@ public FunctionDefinition(NameExpression name, Parameter[] parameters, Statement public Parameter[] Parameters { get; } - public override int ArgCount => Parameters.Length; - internal void SetKeywordEndIndex(int index) => _keywordEndIndex = index; public override int KeywordEndIndex => _keywordEndIndex ?? (DefIndex + (IsCoroutine ? 9 : 3)); public override int KeywordLength => KeywordEndIndex - StartIndex; diff --git a/src/Parsing/Impl/Ast/PythonAst.cs b/src/Parsing/Impl/Ast/PythonAst.cs index c91ecfda5..fc6d8661c 100644 --- a/src/Parsing/Impl/Ast/PythonAst.cs +++ b/src/Parsing/Impl/Ast/PythonAst.cs @@ -59,8 +59,8 @@ public PythonAst(IEnumerable existingAst) { } public Uri Module { get; } - public NewLineLocation[] NewLineLocations { get; } - public SourceLocation[] CommentLocations { get; } + public NewLineLocation[] NewLineLocations { get; private set; } + public SourceLocation[] CommentLocations { get; private set; } public override string Name => ""; /// @@ -93,8 +93,13 @@ public override async Task WalkAsync(PythonWalkerAsync walker, CancellationToken public override Statement Body => _body; public PythonLanguageVersion LanguageVersion { get; } - public void Reduce() { - _attributes.Clear(); + public void Reduce(Func filter) { + (Body as SuiteStatement)?.FilterStatements(filter); + _attributes?.Clear(); + Variables?.Clear(); + NewLineLocations = Array.Empty(); + CommentLocations = Array.Empty(); + base.Clear(); } internal bool TryGetAttribute(Node node, object key, out object value) { diff --git a/src/Parsing/Impl/Ast/ScopeStatement.cs b/src/Parsing/Impl/Ast/ScopeStatement.cs index 45900967d..d6181be4e 100644 --- a/src/Parsing/Impl/Ast/ScopeStatement.cs +++ b/src/Parsing/Impl/Ast/ScopeStatement.cs @@ -22,7 +22,6 @@ namespace Microsoft.Python.Parsing.Ast { public abstract class ScopeStatement : Statement { // due to "exec" or call to dir, locals, eval, vars... - private ClosureInfo[] _closureVariables; // closed over variables, bool indicates if we accessed it in this scope. private List _freeVars; // list of variables accessed from outer scopes private List _globalVars; // global variables accessed from this scope private List _cellVars; // variables accessed from nested scopes @@ -75,12 +74,10 @@ public abstract Statement Body { /// /// Gets the variables for this scope. /// - public ICollection ScopeVariables + public ICollection ScopeVariables => Variables?.Values ?? Array.Empty() as ICollection; public virtual bool IsGlobal => false; - public virtual bool NeedsLocalContext => NeedsLocalsDictionary || ContainsNestedFreeVariables; - public virtual int ArgCount => 0; public PythonAst GlobalParent { get { @@ -93,20 +90,23 @@ public PythonAst GlobalParent { } } - internal void AddFreeVariable(PythonVariable variable, bool accessedInScope) { - if (_freeVars == null) { - _freeVars = new List(); - } + protected void Clear() { + _references?.Clear(); + _cellVars?.Clear(); + _freeVars?.Clear(); + _globalVars?.Clear(); + _nonLocalVars?.Clear(); + } + internal void AddFreeVariable(PythonVariable variable, bool accessedInScope) { + _freeVars = _freeVars ?? new List(); if (!_freeVars.Contains(variable)) { _freeVars.Add(variable); } } internal string AddReferencedGlobal(string name) { - if (_globalVars == null) { - _globalVars = new List(); - } + _globalVars = _globalVars ?? new List(); if (!_globalVars.Contains(name)) { _globalVars.Add(name); } @@ -114,17 +114,12 @@ internal string AddReferencedGlobal(string name) { } internal void AddNonLocalVariable(NameExpression name) { - if (_nonLocalVars == null) { - _nonLocalVars = new List(); - } + _nonLocalVars = _nonLocalVars ?? new List(); _nonLocalVars.Add(name); } internal void AddCellVariable(PythonVariable variable) { - if (_cellVars == null) { - _cellVars = new List(); - } - + _cellVars = _cellVars ?? new List(); if (!_cellVars.Contains(variable.Name)) { _cellVars.Add(variable.Name); } @@ -135,17 +130,6 @@ internal void AddCellVariable(PythonVariable variable) { /// public IReadOnlyList FreeVariables => _freeVars; - /// - /// Variables that are bound to the global scope - /// - public IReadOnlyList GlobalVariables => _globalVars; - - /// - /// Variables that are referred to from a nested scope and need to be - /// promoted to cells. - /// - public IReadOnlyList CellVariables => _cellVars; - internal abstract bool ExposesLocalVariable(PythonVariable variable); public bool TryGetVariable(string name, out PythonVariable variable) { @@ -231,10 +215,6 @@ internal virtual void FinishBind(PythonNameBinder binder) { } } - if (closureVariables != null) { - _closureVariables = closureVariables.ToArray(); - } - // no longer needed _references = null; } From 02475f446afefaeae0e8367df9910d8975fc97cf Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Tue, 21 May 2019 18:10:48 -0700 Subject: [PATCH 09/39] Reload --- src/Analysis/Ast/Impl/Modules/PythonModule.cs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index 26204290a..d6cd41b20 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -339,6 +339,7 @@ private void Parse(CancellationToken cancellationToken) { } ContentState = State.Parsed; + Analysis = new EmptyAnalysis(Services, this); } NewAst?.Invoke(this, EventArgs.Empty); @@ -368,7 +369,7 @@ public override void Add(string message, SourceSpan span, int errorCode, Severit public void NotifyAnalysisBegins() { lock (AnalysisLock) { - if (_updated) { + if (_updated || Analysis is LibraryAnalysis) { _updated = false; // In all variables find those imported, then traverse imported modules // and remove references to this module. If variable refers to a module, @@ -378,6 +379,22 @@ public void NotifyAnalysisBegins() { return; } + if (Analysis is LibraryAnalysis) { + var sw = new Stopwatch(); + sw.Start(); + ContentState = State.None; + LoadContent(); + using (var sr = new StringReader(_buffer.Text)) { + var parser = Parser.CreateParser(sr, Interpreter.LanguageVersion, ParserOptions.Default); + var ast = parser.ParseFile(Uri); + // Stored nodes are no longer valid. + _astMap.Clear(); + _astMap[this] = ast; + } + sw.Stop(); + Log?.Log(TraceEventType.Verbose, $"Reloaded {Name} in {sw.ElapsedMilliseconds}ms because new analysis was requested."); + } + // TODO: Figure out where the nulls below are coming from. var importedVariables = ((IScope)GlobalScope) .TraverseDepthFirst(c => c?.Children ?? Enumerable.Empty()) From 6b6e928acabdb5d31434835b8721d449f8535322 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Sat, 25 May 2019 11:22:39 -0700 Subject: [PATCH 10/39] Ignore post-final requests --- .../Impl/Analyzer/PythonAnalyzerSession.cs | 34 +++++++++---------- .../Impl/Dependencies/DependencyResolver.cs | 2 +- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index 359f098b1..e95570605 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -262,11 +262,11 @@ private Task StartAnalysis(IDependencyChainNode node, Async /// of dependencies, it is intended for the single file analysis. /// private void Analyze(IDependencyChainNode node, AsyncCountdownEvent ace, Stopwatch stopWatch) { - IPythonModule module; try { ace?.AddOne(); var entry = node.Value; - if (!entry.IsValidVersion(_walker.Version, out module, out var ast)) { + + if (!entry.IsValidVersion(_walker.Version, out var module, out var ast)) { if (ast == null) { // Entry doesn't have ast yet. There should be at least one more session. Cancel(); @@ -276,6 +276,7 @@ private void Analyze(IDependencyChainNode node, AsyncCountd node.Skip(); return; } + var startTime = stopWatch.Elapsed; AnalyzeEntry(entry, module, _walker.Version, node.IsComplete); node.Commit(); @@ -317,7 +318,8 @@ private void AnalyzeEntry() { } var startTime = stopWatch?.Elapsed ?? TimeSpan.Zero; - AnalyzeEntry(_entry, module, ast, Version); + + AnalyzeEntry(_entry, module, Version, true); LogCompleted(module, stopWatch, startTime); } catch (OperationCanceledException oce) { @@ -333,6 +335,11 @@ private void AnalyzeEntry() { } private void AnalyzeEntry(PythonAnalyzerEntry entry, IPythonModule module, int version, bool isFinalPass) { + _log?.Log(TraceEventType.Verbose, $"Request to re-analyze finalized {module.Name} ignored."); + if (entry.PreviousAnalysis is LibraryAnalysis) { + return; + } + // Now run the analysis. var analyzable = module as IAnalyzable; analyzable?.NotifyAnalysisBegins(); @@ -357,23 +364,14 @@ private void AnalyzeEntry(PythonAnalyzerEntry entry, IPythonModule module, int v } } - private void LogCompleted(IPythonModule module, Stopwatch stopWatch, TimeSpan startTime) { - if (_log != null) { - _log.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) completed in {(stopWatch.Elapsed - startTime).TotalMilliseconds} ms."); - } - } + private void LogCompleted(IPythonModule module, Stopwatch stopWatch, TimeSpan startTime) + => _log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) completed in {(stopWatch.Elapsed - startTime).TotalMilliseconds} ms."); - private void LogCanceled(IPythonModule module) { - if (_log != null) { - _log.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled."); - } - } + private void LogCanceled(IPythonModule module) + => _log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled."); - private void LogException(IPythonModule module, Exception exception) { - if (_log != null) { - _log.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) failed. Exception message: {exception.Message}."); - } - } + private void LogException(IPythonModule module, Exception exception) + => _log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) failed. Exception message: {exception.Message}."); private enum State { NotStarted = 0, diff --git a/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs b/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs index 4d62eb1e2..1d2e76c6b 100644 --- a/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs +++ b/src/Analysis/Ast/Impl/Dependencies/DependencyResolver.cs @@ -681,7 +681,7 @@ private sealed class DependencyChainNode : IDependencyChainNode { public TValue Value => _vertex.DependencyVertex.Value; public bool HasMissingDependencies => _vertex.HasMissingDependencies; public int VertexDepth { get; } - public bool IsComplete => _vertex.SecondPass == null; + public bool IsComplete => _vertex.SecondPass == null && !HasMissingDependencies; public DependencyChainNode(DependencyChainWalker walker, WalkingVertex vertex, int depth) { _walker = walker; From 9692397637ce2127084a348917276f774b197a3e Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Sat, 25 May 2019 14:36:26 -0700 Subject: [PATCH 11/39] Drop AST --- .../Analyzer/Definitions/IPythonAnalyzer.cs | 1 - .../Evaluation/ExpressionEval.Callables.cs | 10 +++--- .../Analyzer/Evaluation/ExpressionEval.cs | 5 +-- .../Ast/Impl/Analyzer/LibraryAnalysis.cs | 4 ++- .../Ast/Impl/Analyzer/PythonAnalyzer.cs | 2 ++ .../Impl/Analyzer/PythonAnalyzerSession.cs | 4 +-- .../Ast/Impl/Extensions/NodeExtensions.cs | 4 +-- .../Ast/Impl/Extensions/ScopeExtensions.cs | 10 +++--- .../Modules/Definitions/IAstNodeContainer.cs | 4 ++- src/Analysis/Ast/Impl/Modules/PythonModule.cs | 34 +++++++++---------- src/Analysis/Ast/Test/PepHintTests.cs | 2 -- src/Analysis/Ast/Test/ValuesTests.cs | 1 - .../Impl/Completion/ExpressionLocator.cs | 8 ++--- .../FunctionDefinitionCompletion.cs | 2 +- .../Impl/Completion/ImportCompletion.cs | 2 +- .../Impl/Documentation/DocstringConverter.cs | 2 +- .../Impl/Indexing/SymbolIndexWalker.cs | 16 ++++----- .../Impl/Sources/HoverSource.cs | 6 ++-- src/Parsing/Impl/Ast/BinaryExpression.cs | 4 +-- src/Parsing/Impl/Ast/DictionaryExpression.cs | 1 - src/Parsing/Impl/Ast/DottedName.cs | 12 ------- src/Parsing/Impl/Ast/FromImportStatement.cs | 16 --------- src/Parsing/Impl/Ast/ImportStatement.cs | 15 -------- src/Parsing/Impl/Ast/MemberExpression.cs | 2 +- src/Parsing/Impl/Ast/NamedExpression.cs | 3 +- src/Parsing/Impl/Ast/Node.cs | 8 ++--- src/Parsing/Impl/Parser.cs | 8 ++--- src/Parsing/Impl/Token.cs | 8 +---- 28 files changed, 68 insertions(+), 126 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs index 94a298a26..ed870cbe9 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Definitions/IPythonAnalyzer.cs @@ -19,7 +19,6 @@ using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core.Collections; -using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer { public interface IPythonAnalyzer { diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs index 87535666c..e76d46dc6 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs @@ -101,13 +101,13 @@ public IMember GetValueFromClassCtor(IPythonClassType cls, CallExpression expr) return cls.CreateInstance(cls.Name, args); } - public IMember GetValueFromBound(IPythonBoundType t, CallExpression expr) { + private IMember GetValueFromBound(IPythonBoundType t, CallExpression expr) { switch (t.Type) { case IPythonFunctionType fn: return GetValueFromFunctionType(fn, t.Self, expr); case IPythonPropertyType p: return GetValueFromProperty(p, t.Self); - case IPythonIteratorType it when t.Self is IPythonCollection seq: + case IPythonIteratorType _ when t.Self is IPythonCollection seq: return seq.GetIterator(); } return UnknownType; @@ -173,7 +173,7 @@ public IMember GetValueFromFunctionType(IPythonFunctionType fn, IPythonInstance if (instanceType == null || fn.DeclaringType == null || fn.IsSpecialized || instanceType.IsSpecialized || fn.DeclaringType.IsSpecialized || instanceType.Equals(fn.DeclaringType) || - fn.IsStub || !string.IsNullOrEmpty(fn.Overloads[args.OverloadIndex].GetReturnDocumentation(null))) { + fn.IsStub || !string.IsNullOrEmpty(fn.Overloads[args.OverloadIndex].GetReturnDocumentation())) { LoadFunctionDependencyModules(fn); @@ -192,14 +192,14 @@ public IMember GetValueFromFunctionType(IPythonFunctionType fn, IPythonInstance // def func(a, b): return a + b // from working in libraries, but this is small sacrifice for significant performance // increase in library analysis. - if (fn.DeclaringModule is IDocument doc && fd?.Ast == doc.GetAnyAst() && EvaluateFunctionBody(fn)) { + if (fn.DeclaringModule is IDocument && EvaluateFunctionBody(fn)) { // Stubs are coming from another module. return TryEvaluateWithArguments(fn, args); } return UnknownType; } - public IMember GetValueFromProperty(IPythonPropertyType p, IPythonInstance instance) { + private IMember GetValueFromProperty(IPythonPropertyType p, IPythonInstance instance) { // Function may not have been walked yet. Do it now. SymbolTable.Evaluate(p.FunctionDefinition); return instance.Call(p.Name, ArgumentSet.Empty); diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs index 24c8515dc..fda123b1e 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs @@ -68,9 +68,10 @@ public Location GetLocationOfName(Node node) { } IndexSpan indexSpan; + var ast = Module.GetAst(); switch (node) { case MemberExpression mex: - indexSpan = mex.GetNameSpan().ToIndexSpan(mex.Ast); + indexSpan = mex.GetNameSpan(ast).ToIndexSpan(ast); break; case ClassDefinition cd: indexSpan = cd.NameExpression.IndexSpan; @@ -90,7 +91,7 @@ public Location GetLocationOfName(Node node) { // turns into span at line 1 and very large column. This DOES can // produce false positives occasionally. #if DEBUG - var sourceSpan = indexSpan.ToSourceSpan(node.Ast); + var sourceSpan = indexSpan.ToSourceSpan(ast); Debug.Assert(sourceSpan.Start.Line > 1 || sourceSpan.Start.Column < 1000); #endif return new Location(Module, indexSpan); diff --git a/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs b/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs index f90d57985..42caa54da 100644 --- a/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs +++ b/src/Analysis/Ast/Impl/Analyzer/LibraryAnalysis.cs @@ -32,6 +32,7 @@ internal sealed class LibraryAnalysis : IDocumentAnalysis { public LibraryAnalysis(IDocument document, int version, IServiceContainer services, GlobalScope globalScope, IReadOnlyList starImportMemberNames) { Check.ArgumentNotNull(nameof(document), document); Check.ArgumentNotNull(nameof(globalScope), globalScope); + Document = document; Version = version; GlobalScope = globalScope; @@ -39,7 +40,8 @@ public LibraryAnalysis(IDocument document, int version, IServiceContainer servic var ast = Document.GetAst(); ast.Reduce(x => x is ImportStatement || x is FromImportStatement); var c = (IAstNodeContainer)Document; - c.Clear(); + c.ClearContent(); + c.ClearAst(); c.AddAstNode(document, ast); ExpressionEvaluator = new ExpressionEval(services, document, globalScope); diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index fcc298f50..d6b52c46b 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -193,6 +193,8 @@ public void ResetAnalyzer() { public IReadOnlyList LoadedModules => _analysisEntries.Values.ExcludeDefault().Select(v => v.Module).ExcludeDefault().ToArray(); + internal bool IsFinalSession => _nextSession == null; + private void AnalyzeDocument(AnalysisModuleKey key, PythonAnalyzerEntry entry, ImmutableArray dependencies) { _analysisCompleteEvent.Reset(); _log?.Log(TraceEventType.Verbose, $"Analysis of {entry.Module.Name}({entry.Module.ModuleType}) queued"); diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index e95570605..931cebbd5 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -14,6 +14,7 @@ // permissions and limitations under the License. using System; +using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Runtime; @@ -27,7 +28,6 @@ using Microsoft.Python.Core; using Microsoft.Python.Core.Logging; using Microsoft.Python.Core.Services; -using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer { internal sealed class PythonAnalyzerSession { @@ -335,8 +335,8 @@ private void AnalyzeEntry() { } private void AnalyzeEntry(PythonAnalyzerEntry entry, IPythonModule module, int version, bool isFinalPass) { - _log?.Log(TraceEventType.Verbose, $"Request to re-analyze finalized {module.Name} ignored."); if (entry.PreviousAnalysis is LibraryAnalysis) { + _log?.Log(TraceEventType.Verbose, $"Request to re-analyze finalized {module.Name} ignored."); return; } diff --git a/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs b/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs index 4610d32ec..e5ccb7bfb 100644 --- a/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs @@ -23,8 +23,8 @@ public static LocationInfo GetLocation(this Node node, IPythonModule module) { return LocationInfo.Empty; } - var start = node.GetStart(); - var end = node.GetEnd(); + var start = node.GetStart(module.GetAst()); + var end = node.GetEnd(module.GetAst()); return new LocationInfo(module.FilePath, module.Uri, start.Line, start.Column, end.Line, end.Column); } diff --git a/src/Analysis/Ast/Impl/Extensions/ScopeExtensions.cs b/src/Analysis/Ast/Impl/Extensions/ScopeExtensions.cs index 7dc6fdcdb..6ba17ce62 100644 --- a/src/Analysis/Ast/Impl/Extensions/ScopeExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/ScopeExtensions.cs @@ -21,14 +21,14 @@ namespace Microsoft.Python.Analysis.Analyzer { public static class ScopeExtensions { - public static int GetBodyStartIndex(this IScope scope, PythonAst ast) { + public static int GetBodyStartIndex(this IScope scope) { switch (scope.Node) { case ClassDefinition cd: return cd.HeaderIndex; case FunctionDefinition fd: return fd.HeaderIndex; default: - return ast.LocationToIndex(scope.Node.GetStart()); + return scope.Node.StartIndex; } } @@ -47,7 +47,7 @@ public static IScope FindScope(this IScope parent, IDocument document, SourceLoc return children[i]; } - var start = children[i].GetBodyStartIndex(ast); + var start = children[i].GetBodyStartIndex(); if (start > index) { // We've gone past index completely so our last candidate is // the best one. @@ -114,10 +114,10 @@ private static int GetParentScopeIndent(IScope scope, PythonAst ast) { switch (scope.Node) { case ClassDefinition cd: // Return column of "class" statement - return cd.GetStart().Column; + return cd.GetStart(ast).Column; case FunctionDefinition fd when !fd.IsLambda: // Return column of "def" statement - return fd.GetStart().Column; + return fd.GetStart(ast).Column; default: return -1; } diff --git a/src/Analysis/Ast/Impl/Modules/Definitions/IAstNodeContainer.cs b/src/Analysis/Ast/Impl/Modules/Definitions/IAstNodeContainer.cs index c6fb7ce74..0ed36cb97 100644 --- a/src/Analysis/Ast/Impl/Modules/Definitions/IAstNodeContainer.cs +++ b/src/Analysis/Ast/Impl/Modules/Definitions/IAstNodeContainer.cs @@ -34,6 +34,8 @@ internal interface IAstNodeContainer { /// /// Removes all AST node associations. /// - void Clear(); + void ClearAst(); + + void ClearContent(); } } diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index 39bcccf50..344443356 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -370,7 +370,7 @@ public override void Add(string message, SourceSpan span, int errorCode, Severit public void NotifyAnalysisBegins() { lock (AnalysisLock) { - if (_updated || Analysis is LibraryAnalysis) { + if (_updated) { _updated = false; // In all variables find those imported, then traverse imported modules // and remove references to this module. If variable refers to a module, @@ -380,22 +380,6 @@ public void NotifyAnalysisBegins() { return; } - if (Analysis is LibraryAnalysis) { - var sw = new Stopwatch(); - sw.Start(); - ContentState = State.None; - LoadContent(); - using (var sr = new StringReader(_buffer.Text)) { - var parser = Parser.CreateParser(sr, Interpreter.LanguageVersion, ParserOptions.Default); - var ast = parser.ParseFile(Uri); - // Stored nodes are no longer valid. - _astMap.Clear(); - _astMap[this] = ast; - } - sw.Stop(); - Log?.Log(TraceEventType.Verbose, $"Reloaded {Name} in {sw.ElapsedMilliseconds}ms because new analysis was requested."); - } - // TODO: Figure out where the nulls below are coming from. var importedVariables = ((IScope)GlobalScope) .TraverseDepthFirst(c => c?.Children ?? Enumerable.Empty()) @@ -426,6 +410,9 @@ public void NotifyAnalysisComplete(IDocumentAnalysis analysis) { // as declare additional variables, etc. OnAnalysisComplete(); ContentState = State.Analyzed; + + // For non-user code drop content to conserve memory + ClearContent(); } // Do not report issues with libraries or stubs @@ -449,7 +436,18 @@ public void AddAstNode(object o, Node n) { Debug.Assert(!_astMap.ContainsKey(o) || _astMap[o] == n); _astMap[o] = n; } - public void Clear() =>_astMap.Clear(); + + public void ClearAst() { + if (ModuleType != ModuleType.User) { + _astMap.Clear(); + } + } + + public void ClearContent() { + if (ModuleType != ModuleType.User) { + _buffer.Reset(_buffer.Version, string.Empty); + } + } #endregion #region Analysis diff --git a/src/Analysis/Ast/Test/PepHintTests.cs b/src/Analysis/Ast/Test/PepHintTests.cs index 91db0df13..8964ddc7f 100644 --- a/src/Analysis/Ast/Test/PepHintTests.cs +++ b/src/Analysis/Ast/Test/PepHintTests.cs @@ -13,14 +13,12 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System.Linq; using System.Threading.Tasks; using FluentAssertions; using Microsoft.Python.Analysis.Tests.FluentAssertions; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Parsing.Tests; -using Microsoft.Python.Tests.Utilities.FluentAssertions; using Microsoft.VisualStudio.TestTools.UnitTesting; using TestUtilities; diff --git a/src/Analysis/Ast/Test/ValuesTests.cs b/src/Analysis/Ast/Test/ValuesTests.cs index d200604a3..007c76b25 100644 --- a/src/Analysis/Ast/Test/ValuesTests.cs +++ b/src/Analysis/Ast/Test/ValuesTests.cs @@ -15,7 +15,6 @@ using System.IO; using System.Threading.Tasks; -using FluentAssertions; using Microsoft.Python.Analysis.Tests.FluentAssertions; using Microsoft.Python.Analysis.Types; using Microsoft.VisualStudio.TestTools.UnitTesting; diff --git a/src/LanguageServer/Impl/Completion/ExpressionLocator.cs b/src/LanguageServer/Impl/Completion/ExpressionLocator.cs index 9ce9cd0f6..f0835cb87 100644 --- a/src/LanguageServer/Impl/Completion/ExpressionLocator.cs +++ b/src/LanguageServer/Impl/Completion/ExpressionLocator.cs @@ -44,17 +44,17 @@ public static void FindExpression(PythonAst ast, SourceLocation position, FindEx expression = expression ?? (statement as ExpressionStatement)?.Expression; } - private static bool CanBackUp(PythonAst tree, Node node, Node statement, ScopeStatement scope, int column) { + private static bool CanBackUp(PythonAst ast, Node node, Node statement, ScopeStatement scope, int column) { if (node != null || !((statement as ExpressionStatement)?.Expression is ErrorExpression)) { return false; } var top = 1; if (scope != null) { - var scopeStart = scope.GetStart(); + var scopeStart = scope.GetStart(ast); if (scope.Body != null) { - top = scope.Body.GetEnd().Line == scopeStart.Line - ? scope.Body.GetStart().Column + top = scope.Body.GetEnd(ast).Line == scopeStart.Line + ? scope.Body.GetStart(ast).Column : scopeStart.Column; } else { top = scopeStart.Column; diff --git a/src/LanguageServer/Impl/Completion/FunctionDefinitionCompletion.cs b/src/LanguageServer/Impl/Completion/FunctionDefinitionCompletion.cs index 844cf3dda..629705baa 100644 --- a/src/LanguageServer/Impl/Completion/FunctionDefinitionCompletion.cs +++ b/src/LanguageServer/Impl/Completion/FunctionDefinitionCompletion.cs @@ -33,7 +33,7 @@ public static bool TryGetCompletionsForOverride(FunctionDefinition function, Com } if (function.Parent is ClassDefinition cd && function.NameExpression != null && context.Position > function.NameExpression.StartIndex) { - var loc = function.GetStart(); + var loc = function.GetStart(context.Ast); var overrideable = GetOverrideable(context, location).ToArray(); overrideable = !string.IsNullOrEmpty(function.Name) ? overrideable.Where(o => o.Name.StartsWithOrdinal(function.Name)).ToArray() diff --git a/src/LanguageServer/Impl/Completion/ImportCompletion.cs b/src/LanguageServer/Impl/Completion/ImportCompletion.cs index 3a156912b..9fb15e009 100644 --- a/src/LanguageServer/Impl/Completion/ImportCompletion.cs +++ b/src/LanguageServer/Impl/Completion/ImportCompletion.cs @@ -79,7 +79,7 @@ public static CompletionResult GetCompletionsInFromImport(FromImportStatement fr } if (context.Position >= name.StartIndex) { - var applicableSpan = name.GetSpan(); + var applicableSpan = name.GetSpan(context.Ast); var importSearchResult = mres.CurrentPathResolver.FindImports(document.FilePath, fromImport); return GetResultFromImportSearch(importSearchResult, context, false, applicableSpan); } diff --git a/src/LanguageServer/Impl/Documentation/DocstringConverter.cs b/src/LanguageServer/Impl/Documentation/DocstringConverter.cs index 466668890..aab46c877 100644 --- a/src/LanguageServer/Impl/Documentation/DocstringConverter.cs +++ b/src/LanguageServer/Impl/Documentation/DocstringConverter.cs @@ -77,7 +77,7 @@ private int NextBlockIndent private string CurrentLineWithinBlock => CurrentLine.Substring(_blockIndent); private DocstringConverter(string input) { - _builder = new StringBuilder(input.Length); + _builder = new StringBuilder(input?.Length ?? 0); _state = ParseText; _lines = SplitDocstring(input); } diff --git a/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs b/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs index 0ebe6dda1..ddd1565df 100644 --- a/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs +++ b/src/LanguageServer/Impl/Indexing/SymbolIndexWalker.cs @@ -41,8 +41,8 @@ public override bool Walk(ClassDefinition node) { _stack.AddSymbol(new HierarchicalSymbol( node.Name, SymbolKind.Class, - node.GetSpan(), - node.NameExpression.GetSpan(), + node.GetSpan(_ast), + node.NameExpression.GetSpan(_ast), children, FunctionKind.Class )); @@ -58,13 +58,13 @@ public override bool Walk(FunctionDefinition node) { node.Body?.Walk(this); var children = _stack.Exit(); - var span = node.GetSpan(); + var span = node.GetSpan(_ast); var ds = new HierarchicalSymbol( node.Name, SymbolKind.Function, span, - node.IsLambda ? span : node.NameExpression.GetSpan(), + node.IsLambda ? span : node.NameExpression.GetSpan(_ast), children, FunctionKind.Function ); @@ -102,7 +102,7 @@ public override bool Walk(FunctionDefinition node) { public override bool Walk(ImportStatement node) { foreach (var (nameNode, nameString) in node.Names.Zip(node.AsNames, (name, asName) => asName != null ? (asName, asName.Name) : ((Node)name, name.MakeString()))) { - var span = nameNode.GetSpan(); + var span = nameNode.GetSpan(_ast); _stack.AddSymbol(new HierarchicalSymbol(nameString, SymbolKind.Module, span)); } @@ -115,7 +115,7 @@ public override bool Walk(FromImportStatement node) { } foreach (var name in node.Names.Zip(node.AsNames, (name, asName) => asName ?? name)) { - var span = name.GetSpan(); + var span = name.GetSpan(_ast); _stack.AddSymbol(new HierarchicalSymbol(name.Name, SymbolKind.Module, span)); } @@ -202,7 +202,7 @@ public override bool Walk(GeneratorExpression node) { private void ExitComprehension(Comprehension node) { var children = _stack.Exit(); - var span = node.GetSpan(); + var span = node.GetSpan(_ast); _stack.AddSymbol(new HierarchicalSymbol( $"<{node.NodeName}>", @@ -229,7 +229,7 @@ private void AddVarSymbol(NameExpression node) { break; } - var span = node.GetSpan(); + var span = node.GetSpan(_ast); _stack.AddSymbol(new HierarchicalSymbol(node.Name, kind, span)); } diff --git a/src/LanguageServer/Impl/Sources/HoverSource.cs b/src/LanguageServer/Impl/Sources/HoverSource.cs index 729b69c97..d4a2f4537 100644 --- a/src/LanguageServer/Impl/Sources/HoverSource.cs +++ b/src/LanguageServer/Impl/Sources/HoverSource.cs @@ -45,8 +45,8 @@ public Hover GetHover(IDocumentAnalysis analysis, SourceLocation location) { } var range = new Range { - start = expr.GetStart(), - end = expr.GetEnd() + start = expr.GetStart(analysis.Ast), + end = expr.GetEnd(analysis.Ast) }; var eval = analysis.ExpressionEvaluator; @@ -91,7 +91,7 @@ public Hover GetHover(IDocumentAnalysis analysis, SourceLocation location) { if (expr is MemberExpression mex) { name = mex.Name; range = new Range { - start = mex.Target.GetEnd(), + start = mex.Target.GetEnd(analysis.Ast), end = range.end }; diff --git a/src/Parsing/Impl/Ast/BinaryExpression.cs b/src/Parsing/Impl/Ast/BinaryExpression.cs index 3b0497b30..ca7c7536c 100644 --- a/src/Parsing/Impl/Ast/BinaryExpression.cs +++ b/src/Parsing/Impl/Ast/BinaryExpression.cs @@ -21,7 +21,7 @@ using System.Threading.Tasks; namespace Microsoft.Python.Parsing.Ast { - public partial class BinaryExpression : Expression { + public class BinaryExpression : Expression { public BinaryExpression(PythonOperator op, Expression left, Expression right, int operatorIndex) { if (op == PythonOperator.None) { throw new ArgumentException("bad operator"); @@ -88,8 +88,6 @@ public override async Task WalkAsync(PythonWalkerAsync walker, CancellationToken } internal override void AppendCodeString(StringBuilder res, PythonAst ast, CodeFormattingOptions format) { - var left = Left; - var right = Right; string op1, op2; if (Operator == PythonOperator.NotIn) { diff --git a/src/Parsing/Impl/Ast/DictionaryExpression.cs b/src/Parsing/Impl/Ast/DictionaryExpression.cs index 0feb297be..1ed01406d 100644 --- a/src/Parsing/Impl/Ast/DictionaryExpression.cs +++ b/src/Parsing/Impl/Ast/DictionaryExpression.cs @@ -17,7 +17,6 @@ using System.Text; using System.Threading; using System.Threading.Tasks; -using Microsoft.Python.Core; using Microsoft.Python.Core.Collections; namespace Microsoft.Python.Parsing.Ast { diff --git a/src/Parsing/Impl/Ast/DottedName.cs b/src/Parsing/Impl/Ast/DottedName.cs index f0e6e4b96..b169aabfa 100644 --- a/src/Parsing/Impl/Ast/DottedName.cs +++ b/src/Parsing/Impl/Ast/DottedName.cs @@ -14,11 +14,9 @@ // permissions and limitations under the License. using System.Collections.Generic; -using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; -using Microsoft.Python.Core; using Microsoft.Python.Core.Collections; namespace Microsoft.Python.Parsing.Ast { @@ -44,16 +42,6 @@ public virtual string MakeString() { public override IEnumerable GetChildNodes() => Names; - public override PythonAst Ast { - get => base.Ast; - internal set { - base.Ast = value; - foreach (var n in Names) { - n.Ast = value; - } - } - } - public override void Walk(PythonWalker walker) { if (walker.Walk(this)) { } diff --git a/src/Parsing/Impl/Ast/FromImportStatement.cs b/src/Parsing/Impl/Ast/FromImportStatement.cs index d4bbdb97d..794f904f3 100644 --- a/src/Parsing/Impl/Ast/FromImportStatement.cs +++ b/src/Parsing/Impl/Ast/FromImportStatement.cs @@ -49,25 +49,9 @@ public FromImportStatement(ModuleName/*!*/ root, ImmutableArray Justification = "breaking change")] public PythonVariable[] Variables { get; set; } - public PythonReference[] GetReferences(PythonAst ast) => GetVariableReferences(this, ast); - // TODO: return names and aliases when they are united into one node public override IEnumerable GetChildNodes() => Enumerable.Empty(); - public override PythonAst Ast { - get => base.Ast; - internal set { - foreach (var n in Names.ExcludeDefault()) { - n.Ast = value; - } - foreach (var n in AsNames.ExcludeDefault()) { - n.Ast = value; - } - - Root.Ast = value; - base.Ast = value; - } - } public override void Walk(PythonWalker walker) { if (walker.Walk(this)) { } diff --git a/src/Parsing/Impl/Ast/ImportStatement.cs b/src/Parsing/Impl/Ast/ImportStatement.cs index a7126ba49..5c1a79fc3 100644 --- a/src/Parsing/Impl/Ast/ImportStatement.cs +++ b/src/Parsing/Impl/Ast/ImportStatement.cs @@ -13,14 +13,12 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Text; using System.Threading; using System.Threading.Tasks; -using Microsoft.Python.Core; using Microsoft.Python.Core.Collections; namespace Microsoft.Python.Parsing.Ast { @@ -44,19 +42,6 @@ public ImportStatement(ImmutableArray names, ImmutableArray GetChildNodes() => Enumerable.Empty(); - public override PythonAst Ast { - get => base.Ast; - internal set { - foreach (var n in Names.ExcludeDefault()) { - n.Ast = value; - } - foreach (var n in AsNames.ExcludeDefault()) { - n.Ast = value; - } - base.Ast = value; - } - } - public override void Walk(PythonWalker walker) { if (walker.Walk(this)) { } diff --git a/src/Parsing/Impl/Ast/MemberExpression.cs b/src/Parsing/Impl/Ast/MemberExpression.cs index b11db83c2..d49d32c2e 100644 --- a/src/Parsing/Impl/Ast/MemberExpression.cs +++ b/src/Parsing/Impl/Ast/MemberExpression.cs @@ -95,6 +95,6 @@ public override void SetLeadingWhiteSpace(PythonAst ast, string whiteSpace) /// /// Returns the span of the name component of the expression /// - public SourceSpan GetNameSpan() => new SourceSpan(Ast.IndexToLocation(NameHeader), GetEnd()); + public SourceSpan GetNameSpan(PythonAst ast) => new SourceSpan(ast.IndexToLocation(NameHeader), GetEnd(ast)); } } diff --git a/src/Parsing/Impl/Ast/NamedExpression.cs b/src/Parsing/Impl/Ast/NamedExpression.cs index 8a55cd3da..3f5693ab2 100644 --- a/src/Parsing/Impl/Ast/NamedExpression.cs +++ b/src/Parsing/Impl/Ast/NamedExpression.cs @@ -1,5 +1,4 @@ -using System; -using System.Collections.Generic; +using System.Collections.Generic; using System.Text; using System.Threading; using System.Threading.Tasks; diff --git a/src/Parsing/Impl/Ast/Node.cs b/src/Parsing/Impl/Ast/Node.cs index 82bc79c42..973ce6e70 100644 --- a/src/Parsing/Impl/Ast/Node.cs +++ b/src/Parsing/Impl/Ast/Node.cs @@ -23,8 +23,6 @@ namespace Microsoft.Python.Parsing.Ast { public abstract class Node { #region Public API - public virtual PythonAst Ast { get; internal set; } - public int EndIndex { get => IndexSpan.End; set => IndexSpan = new IndexSpan(IndexSpan.Start, value - IndexSpan.Start); @@ -50,11 +48,11 @@ public string ToCodeString(PythonAst ast, CodeFormattingOptions format) { return string.IsInterned(result) ?? result; } - public SourceLocation GetStart() => Ast.IndexToLocation(StartIndex); + public SourceLocation GetStart(PythonAst ast) => ast.IndexToLocation(StartIndex); - public SourceLocation GetEnd() => Ast.IndexToLocation(EndIndex); + public SourceLocation GetEnd(PythonAst ast) => ast.IndexToLocation(EndIndex); - public SourceSpan GetSpan() => new SourceSpan(GetStart(), GetEnd()); + public SourceSpan GetSpan(PythonAst ast) => new SourceSpan(GetStart(ast), GetEnd(ast)); /// /// Returns the proceeding whitespace (newlines and comments) that diff --git a/src/Parsing/Impl/Parser.cs b/src/Parsing/Impl/Parser.cs index 36f13fc28..0813780c0 100644 --- a/src/Parsing/Impl/Parser.cs +++ b/src/Parsing/Impl/Parser.cs @@ -197,10 +197,6 @@ private PythonAst CreateAst(Uri module, Statement ret) { ast.SetAttributes(_attributes); PythonNameBinder.BindAst(_langVersion, ast, _errors, _bindReferences); - foreach (var n in ((Node)ast).TraverseDepthFirst(c => c.GetChildNodes())) { - n.Ast = ast; - } - return ast; } @@ -3244,8 +3240,8 @@ private Expression buildFStringExpr(IEnumerable readTokens) { var openQuotes = readTokens.Where(t => t.Token.Kind == TokenKind.FString) .Select(t => ((FStringToken)t.Token).OpenQuotes).DefaultIfEmpty("'").First(); - List fStringChildren = new List(); - StringBuilder unparsedFStringBuilder = new StringBuilder(); + var fStringChildren = new List(); + var unparsedFStringBuilder = new StringBuilder(); foreach (var tokenWithSpan in readTokens) { if (tokenWithSpan.Token.Kind == TokenKind.FString) { diff --git a/src/Parsing/Impl/Token.cs b/src/Parsing/Impl/Token.cs index db7d8d1c5..703deb591 100644 --- a/src/Parsing/Impl/Token.cs +++ b/src/Parsing/Impl/Token.cs @@ -93,8 +93,6 @@ public ConstantValueToken(object value) Value = value; } - public object Constant => Value; - public override object Value { get; } public override string Image => Value == null ? "None" : Value.ToString(); @@ -140,11 +138,7 @@ public FStringToken(string value, string openQuote, bool isTriple, bool isRaw) public string Text => (string)Value; - public override string Image { - get { - return Value == null ? "None" : $"f{OpenQuotes}{Value.ToString()}{OpenQuotes}"; - } - } + public override string Image => Value == null ? "None" : $"f{OpenQuotes}{Value.ToString()}{OpenQuotes}"; } public sealed class VerbatimFStringToken : FStringToken { From 080beabab18df498c753ae2f2294d7b5f4d4a6f1 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Sat, 25 May 2019 15:10:53 -0700 Subject: [PATCH 12/39] Remove local variables --- src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs | 4 ++++ src/Analysis/Ast/Impl/Values/VariableCollection.cs | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs index 6221d6af7..6684bd412 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs @@ -79,6 +79,10 @@ public override void Evaluate() { if (ctor || annotationType.IsUnknown() || Module.ModuleType == ModuleType.User) { // Return type from the annotation is sufficient for libraries and stubs, no need to walk the body. FunctionDefinition.Body?.Walk(this); + // For libraries remove declared local function variables to free up some memory. + if (Module.ModuleType != ModuleType.User) { + ((VariableCollection)Eval.CurrentScope.Variables).Clear(); + } } } Result = _function; diff --git a/src/Analysis/Ast/Impl/Values/VariableCollection.cs b/src/Analysis/Ast/Impl/Values/VariableCollection.cs index 893358d73..817456eca 100644 --- a/src/Analysis/Ast/Impl/Values/VariableCollection.cs +++ b/src/Analysis/Ast/Impl/Values/VariableCollection.cs @@ -122,5 +122,11 @@ internal void RemoveVariable(string name) { _variables.Remove(name); } } + + internal void Clear() { + lock (_syncObj) { + _variables.Clear(); + } + } } } From 93a691579eae83b3dc648490f831dc4b41b889db Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Sat, 25 May 2019 19:11:27 -0700 Subject: [PATCH 13/39] Test fixes --- .../Evaluation/ExpressionEval.Callables.cs | 2 +- .../Ast/Impl/Analyzer/Symbols/SymbolCollector.cs | 2 +- src/Analysis/Ast/Impl/Types/PythonClassType.cs | 2 +- .../Ast/Impl/Types/PythonFunctionOverload.cs | 16 ++-------------- .../Ast/Impl/Types/PythonFunctionType.cs | 2 +- src/Analysis/Ast/Test/ArgumentSetTests.cs | 8 ++++++-- 6 files changed, 12 insertions(+), 20 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs index e76d46dc6..ec4eb668d 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs @@ -79,7 +79,7 @@ public IMember GetValueFromLambda(LambdaExpression expr) { var location = GetLocationOfName(expr.Function); var ft = new PythonFunctionType(expr.Function, null, location); - var overload = new PythonFunctionOverload(ft, location); + var overload = new PythonFunctionOverload(ft, location, expr.Function.ReturnAnnotation?.ToCodeString(Ast)); ft.AddOverload(overload); return ft; } diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs index ef71bf50f..ce3a1de22 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs @@ -132,7 +132,7 @@ private void AddOverload(FunctionDefinition fd, IPythonClassMember function, Act if (!_table.Contains(fd)) { // Do not evaluate parameter types just yet. During light-weight top-level information // collection types cannot be determined as imports haven't been processed. - var overload = new PythonFunctionOverload(function, _eval.GetLocationOfName(fd)); + var overload = new PythonFunctionOverload(function, _eval.GetLocationOfName(fd), fd.ReturnAnnotation?.ToCodeString(_eval.Ast)); addOverload(overload); _table.Add(new FunctionEvaluator(_eval, overload)); } diff --git a/src/Analysis/Ast/Impl/Types/PythonClassType.cs b/src/Analysis/Ast/Impl/Types/PythonClassType.cs index 16361a990..850cf4893 100644 --- a/src/Analysis/Ast/Impl/Types/PythonClassType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonClassType.cs @@ -50,7 +50,7 @@ public PythonClassType( BuiltinTypeId builtinTypeId = BuiltinTypeId.Type ) : base(classDefinition.Name, location, classDefinition.GetDocumentation(), builtinTypeId) { Check.ArgumentNotNull(nameof(location), location.Module); - DeclaringModule.AddAstNode(this, classDefinition); + location.Module.AddAstNode(this, classDefinition); } #region IPythonType diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs index 4269fc5bf..9aac1a537 100644 --- a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs +++ b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs @@ -49,22 +49,10 @@ internal sealed class PythonFunctionOverload : LocatedMember, IPythonFunctionOve // type info of C vs. return C() that returns an instance of C. private bool _fromAnnotation; - public PythonFunctionOverload(IPythonClassMember cm, Location location) + public PythonFunctionOverload(IPythonClassMember cm, Location location, string returnDocumentation) : this(cm.Name, location) { ClassMember = cm; - var ast = (location.Module as IDocument)?.Analysis.Ast; - - FunctionDefinition fd = null; - switch(cm) { - case IPythonFunctionType ft: - fd = ft.FunctionDefinition; - break; - case IPythonPropertyType prop: - fd = prop.FunctionDefinition; - break; - } - - _returnDocumentation = ast != null ? fd?.ReturnAnnotation?.ToCodeString(ast) : null; + _returnDocumentation = returnDocumentation; } public PythonFunctionOverload(string name, Location location) : base(location) { diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs index a2bbe4d09..31cf4092a 100644 --- a/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs @@ -60,7 +60,7 @@ public PythonFunctionType( IPythonType declaringType, Location location ) : base(fd.Name, location, - fd.Name == "__init__" ? declaringType?.Documentation : fd.Documentation, + fd.Name == "__init__" ? (declaringType?.Documentation ?? fd.GetDocumentation()) : fd.GetDocumentation(), declaringType != null ? BuiltinTypeId.Method : BuiltinTypeId.Function) { DeclaringType = declaringType; diff --git a/src/Analysis/Ast/Test/ArgumentSetTests.cs b/src/Analysis/Ast/Test/ArgumentSetTests.cs index fc613182a..2f4bee636 100644 --- a/src/Analysis/Ast/Test/ArgumentSetTests.cs +++ b/src/Analysis/Ast/Test/ArgumentSetTests.cs @@ -347,9 +347,14 @@ from builtins import pow [TestMethod, Priority(0)] public async Task DefaultArgumentAnotherFile() { const string code = @" -from DefaultArgument import func +from .module2 import func func() "; + const string code2 = @" +class A: ... +def func(a = A()): ... +"; + await TestData.CreateTestSpecificFileAsync("module2.py", code2); var argSet = await GetArgSetAsync(code, "func"); argSet.Arguments.Count.Should().Be(1); argSet.Evaluate(); @@ -361,7 +366,6 @@ from DefaultArgument import func t.MemberType.Should().Be(PythonMemberType.Class); } - private async Task GetArgSetAsync(string code, string funcName = "f") { var analysis = await GetAnalysisAsync(code); var f = analysis.Should().HaveFunction(funcName).Which; From cadd7ceeefa109c425614af4288ace83a6bab44c Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Sat, 25 May 2019 20:00:39 -0700 Subject: [PATCH 14/39] Fix overload match --- .../Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs | 7 ++++--- .../Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs | 3 ++- .../Ast/Impl/Analyzer/Symbols/ModuleSymbolTable.cs | 1 - src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs | 6 +++--- src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs | 5 +++-- src/Analysis/Ast/Test/FunctionTests.cs | 1 - 6 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs index ec4eb668d..55b4a8d02 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs @@ -77,9 +77,10 @@ public IMember GetValueFromLambda(LambdaExpression expr) { return null; } - var location = GetLocationOfName(expr.Function); - var ft = new PythonFunctionType(expr.Function, null, location); - var overload = new PythonFunctionOverload(ft, location, expr.Function.ReturnAnnotation?.ToCodeString(Ast)); + var fd = expr.Function; + var location = GetLocationOfName(fd); + var ft = new PythonFunctionType(fd, null, location); + var overload = new PythonFunctionOverload(ft, fd, location, expr.Function.ReturnAnnotation?.ToCodeString(Ast)); ft.AddOverload(overload); return ft; } diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs index 6684bd412..8b7d70b73 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs @@ -43,7 +43,7 @@ public FunctionEvaluator(ExpressionEval eval, PythonFunctionOverload overload) FunctionDefinition = overload.FunctionDefinition; } - public FunctionDefinition FunctionDefinition { get; } + private FunctionDefinition FunctionDefinition { get; } public override void Evaluate() { var stub = SymbolTable.ReplacedByStubs.Contains(Target) @@ -79,6 +79,7 @@ public override void Evaluate() { if (ctor || annotationType.IsUnknown() || Module.ModuleType == ModuleType.User) { // Return type from the annotation is sufficient for libraries and stubs, no need to walk the body. FunctionDefinition.Body?.Walk(this); + // For libraries remove declared local function variables to free up some memory. if (Module.ModuleType != ModuleType.User) { ((VariableCollection)Eval.CurrentScope.Variables).Clear(); diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/ModuleSymbolTable.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/ModuleSymbolTable.cs index ef4848991..31a3b4212 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/ModuleSymbolTable.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/ModuleSymbolTable.cs @@ -34,7 +34,6 @@ internal sealed class ModuleSymbolTable { public IEnumerable> Evaluators => _evaluators.ToArray(); public void Add(MemberEvaluator e) => _evaluators[e.Target] = e; - public MemberEvaluator Get(ScopeStatement target) => _evaluators.TryGetValue(target, out var w) ? w : null; public bool Contains(ScopeStatement node) => _evaluators.ContainsKey(node) || _processed.Contains(node); public void Build(ExpressionEval eval) diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs index ce3a1de22..55b322520 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs @@ -73,7 +73,7 @@ public override bool Walk(FunctionDefinition fd) { if (IsDeprecated(fd)) { return false; } - if (!string.IsNullOrEmpty(fd.NameExpression?.Name)) { + if (!string.IsNullOrEmpty(fd.Name)) { AddFunctionOrProperty(fd); // Open function scope _scopes.Push(_eval.OpenScope(_eval.Module, fd, out _)); @@ -82,7 +82,7 @@ public override bool Walk(FunctionDefinition fd) { } public override void PostWalk(FunctionDefinition fd) { - if (!IsDeprecated(fd) && !string.IsNullOrEmpty(fd.NameExpression?.Name)) { + if (!IsDeprecated(fd) && !string.IsNullOrEmpty(fd.Name)) { _scopes.Pop().Dispose(); } base.PostWalk(fd); @@ -132,7 +132,7 @@ private void AddOverload(FunctionDefinition fd, IPythonClassMember function, Act if (!_table.Contains(fd)) { // Do not evaluate parameter types just yet. During light-weight top-level information // collection types cannot be determined as imports haven't been processed. - var overload = new PythonFunctionOverload(function, _eval.GetLocationOfName(fd), fd.ReturnAnnotation?.ToCodeString(_eval.Ast)); + var overload = new PythonFunctionOverload(function, fd, _eval.GetLocationOfName(fd), fd.ReturnAnnotation?.ToCodeString(_eval.Ast)); addOverload(overload); _table.Add(new FunctionEvaluator(_eval, overload)); } diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs index 9aac1a537..43363a6d2 100644 --- a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs +++ b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs @@ -49,9 +49,10 @@ internal sealed class PythonFunctionOverload : LocatedMember, IPythonFunctionOve // type info of C vs. return C() that returns an instance of C. private bool _fromAnnotation; - public PythonFunctionOverload(IPythonClassMember cm, Location location, string returnDocumentation) + public PythonFunctionOverload(IPythonClassMember cm, FunctionDefinition fd, Location location, string returnDocumentation) : this(cm.Name, location) { ClassMember = cm; + cm.DeclaringModule.AddAstNode(this, fd); _returnDocumentation = returnDocumentation; } @@ -92,7 +93,7 @@ internal void SetReturnValue(IMember value, bool fromAnnotation) { internal void SetDocumentation(string documentation) => Documentation = documentation; #region IPythonFunctionOverload - public FunctionDefinition FunctionDefinition => ClassMember?.DeclaringModule?.GetAstNode(ClassMember); + public FunctionDefinition FunctionDefinition => ClassMember?.DeclaringModule?.GetAstNode(this); public IPythonClassMember ClassMember { get; } public string Name { get; } public string Documentation { get; private set; } diff --git a/src/Analysis/Ast/Test/FunctionTests.cs b/src/Analysis/Ast/Test/FunctionTests.cs index a1159af57..83693bf62 100644 --- a/src/Analysis/Ast/Test/FunctionTests.cs +++ b/src/Analysis/Ast/Test/FunctionTests.cs @@ -357,7 +357,6 @@ def e(cls): pass [TestMethod, Priority(0)] public async Task OverloadsParamTypeMatch() { const string code = @" - def f(a: bool) -> None: ... def f(a: int) -> float: ... def f(a: str) -> bytes: ... From f61b1a762ac7518c35107805eb6aad3e95c80565 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Sat, 25 May 2019 22:25:04 -0700 Subject: [PATCH 15/39] Tests --- .../Analyzer/Evaluation/ExpressionEval.cs | 5 +- src/Analysis/Ast/Impl/Types/ArgumentSet.cs | 47 ++++++++++--------- .../Impl/Types/Definitions/IArgumentSet.cs | 5 -- .../Impl/Types/Definitions/IParameterInfo.cs | 16 +++---- src/Analysis/Ast/Impl/Types/Location.cs | 2 +- src/Analysis/Ast/Impl/Types/ParameterInfo.cs | 6 +-- .../FunctionDefinitionCompletion.cs | 4 +- src/Parsing/Impl/Ast/PythonAst.cs | 3 +- 8 files changed, 39 insertions(+), 49 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs index fda123b1e..392dc1abe 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs @@ -68,10 +68,9 @@ public Location GetLocationOfName(Node node) { } IndexSpan indexSpan; - var ast = Module.GetAst(); switch (node) { case MemberExpression mex: - indexSpan = mex.GetNameSpan(ast).ToIndexSpan(ast); + indexSpan = mex.GetNameSpan(Ast).ToIndexSpan(Ast); break; case ClassDefinition cd: indexSpan = cd.NameExpression.IndexSpan; @@ -91,7 +90,7 @@ public Location GetLocationOfName(Node node) { // turns into span at line 1 and very large column. This DOES can // produce false positives occasionally. #if DEBUG - var sourceSpan = indexSpan.ToSourceSpan(ast); + var sourceSpan = indexSpan.ToSourceSpan(Ast); Debug.Assert(sourceSpan.Start.Line > 1 || sourceSpan.Start.Column < 1000); #endif return new Location(Module, indexSpan); diff --git a/src/Analysis/Ast/Impl/Types/ArgumentSet.cs b/src/Analysis/Ast/Impl/Types/ArgumentSet.cs index 4d0b741a8..dbcd33293 100644 --- a/src/Analysis/Ast/Impl/Types/ArgumentSet.cs +++ b/src/Analysis/Ast/Impl/Types/ArgumentSet.cs @@ -91,7 +91,7 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in var overload = fn.Overloads[overloadIndex]; var fd = overload.FunctionDefinition; - if (fd == null || fn.IsSpecialized) { + if (fn.IsSpecialized) { // Typically specialized function, like TypeVar() that does not actually have AST definition. // Make the arguments from the call expression. If argument does not have name, // try using name from the function definition based on the argument position. @@ -99,11 +99,10 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in for (var i = 0; i < callExpr.Args.Count; i++) { var name = callExpr.Args[i].Name; if (string.IsNullOrEmpty(name)) { - name = fd != null && i < fd.Parameters.Length ? fd.Parameters[i].Name : null; + name = i < overload.Parameters.Count ? overload.Parameters[i].Name : $"arg{i}"; } - name = name ?? $"arg{i}"; - var parameter = fd != null && i < fd.Parameters.Length ? fd.Parameters[i] : null; - _arguments.Add(new Argument(name, ParameterKind.Normal, callExpr.Args[i].Expression, null, parameter)); + var node = fd != null && i < fd.Parameters.Length ? fd.Parameters[i] : null; + _arguments.Add(new Argument(name, ParameterKind.Normal, callExpr.Args[i].Expression, null, node)); } return; } @@ -116,7 +115,12 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in // had values assigned to them are marked as 'filled'.Slots which have // no value assigned to them yet are considered 'empty'. - var slots = fd.Parameters.Select(p => new Argument(p, p)).ToArray(); + var slots = new Argument[overload.Parameters.Count]; + for (var i = 0; i < overload.Parameters.Count; i++) { + var node = fd != null && i < fd.Parameters.Length ? fd.Parameters[i] : null; + slots[i] = new Argument(overload.Parameters[i], node); + } + // Locate sequence argument, if any var sa = slots.Where(s => s.Kind == ParameterKind.List).ToArray(); if (sa.Length > 1) { @@ -151,7 +155,7 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in break; } - if (formalParamIndex >= fd.Parameters.Length) { + if (formalParamIndex >= overload.Parameters.Count) { // We ran out of formal parameters and yet haven't seen // any sequence or dictionary ones. This looks like an error. _errors.Add(new DiagnosticsEntry(Resources.Analysis_TooManyFunctionArguments, arg.GetLocation(module).Span, @@ -159,8 +163,8 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in return; } - var formalParam = fd.Parameters[formalParamIndex]; - if (formalParam.IsList) { + var formalParam = overload.Parameters[formalParamIndex]; + if (formalParam.Kind == ParameterKind.List) { if (string.IsNullOrEmpty(formalParam.Name)) { // If the next unfilled slot is a vararg slot, and it does not have a name, then it is an error. _errors.Add(new DiagnosticsEntry(Resources.Analysis_TooManyPositionalArgumentBeforeStar, arg.GetLocation(module).Span, @@ -189,7 +193,7 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in break; // Sequence or dictionary parameter found. Done here. } - if (formalParam.IsDictionary) { + if (formalParam.Kind == ParameterKind.Dictionary) { // Next slot is a dictionary slot, but we have positional arguments still. _errors.Add(new DiagnosticsEntry(Resources.Analysis_TooManyPositionalArgumentBeforeStar, arg.GetLocation(module).Span, ErrorCodes.TooManyPositionalArgumentsBeforeStar, Severity.Warning, DiagnosticSource.Analysis)); @@ -248,8 +252,8 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in // then fill the slot with the default value. If there is no default value, // then it is an error. foreach (var slot in slots.Where(s => s.Kind != ParameterKind.List && s.Kind != ParameterKind.Dictionary && s.Value == null)) { - if (slot.ValueExpression == null) { - var parameter = fd.Parameters.First(p => p.Name == slot.Name); + if (slot.Value == null) { + var parameter = overload.Parameters.First(p => p.Name == slot.Name); if (parameter.DefaultValue == null) { // TODO: parameter is not assigned and has no default value. _errors.Add(new DiagnosticsEntry(Resources.Analysis_ParameterMissing.FormatUI(slot.Name), callLocation.Span, @@ -257,7 +261,7 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in } // Note that parameter default value expression is from the function definition AST // while actual argument values are from the calling file AST. - slot.ValueExpression = parameter.DefaultValue; + slot.Value = parameter.DefaultValue; slot.ValueIsDefault = true; } } @@ -274,7 +278,6 @@ public ArgumentSet Evaluate() { foreach (var a in _arguments.Where(x => x.Value == null)) { a.Value = GetArgumentValue(a); - a.Type = Eval.GetValueFromExpression(a.TypeExpression) as IPythonType; } if (_listArgument != null) { @@ -296,6 +299,9 @@ public ArgumentSet Evaluate() { } private IMember GetArgumentValue(Argument arg) { + if (arg.Value is IMember m) { + return m; + } // Evaluates expression in the specific module context. Typically used to evaluate // expressions representing default values of function arguments since they are // are defined in the function declaring module rather than in the caller context. @@ -345,24 +351,19 @@ private sealed class Argument : IArgument { /// public IPythonType Type { get; internal set; } - /// - /// Type annotation expression. - /// - public Expression TypeExpression { get; } - /// /// AST node that defines the argument. /// public Node Location { get; } - public Argument(Parameter p, Node location) : - this(p.Name, p.Kind, null, p.Annotation, location) { } + public Argument(IParameterInfo p, Node location) : + this(p.Name, p.Kind, null, p.Type, location) { } - public Argument(string name, ParameterKind kind, Expression valueValueExpression, Expression typeExpression, Node location) { + public Argument(string name, ParameterKind kind, Expression valueValueExpression, IPythonType type, Node location) { Name = name; Kind = kind; + Type = type; ValueExpression = valueValueExpression; - TypeExpression = typeExpression; Location = location; } diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IArgumentSet.cs b/src/Analysis/Ast/Impl/Types/Definitions/IArgumentSet.cs index e4b952b6c..bc1caa61b 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/IArgumentSet.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/IArgumentSet.cs @@ -44,11 +44,6 @@ public interface IArgument { /// IPythonType Type { get; } - /// - /// Annotation expression. - /// - Expression TypeExpression { get; } - /// /// Parameter location in the AST. /// diff --git a/src/Analysis/Ast/Impl/Types/Definitions/IParameterInfo.cs b/src/Analysis/Ast/Impl/Types/Definitions/IParameterInfo.cs index d0875c2b7..8c79fea2f 100644 --- a/src/Analysis/Ast/Impl/Types/Definitions/IParameterInfo.cs +++ b/src/Analysis/Ast/Impl/Types/Definitions/IParameterInfo.cs @@ -14,6 +14,7 @@ // permissions and limitations under the License. using Microsoft.Python.Core.Text; +using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Types { /// @@ -36,16 +37,6 @@ public interface IParameterInfo { /// string Documentation { get; } - /// - /// True if the parameter is a *args parameter. - /// - bool IsParamArray { get; } - - /// - /// True if the parameter is a **args parameter. - /// - bool IsKeywordDict { get; } - /// /// Parameter type is generic and specific type will be /// determined at the time of the function call. @@ -62,5 +53,10 @@ public interface IParameterInfo { /// Default value. /// IMember DefaultValue { get; } + + /// + /// Parameter kind. + /// + ParameterKind Kind { get; } } } diff --git a/src/Analysis/Ast/Impl/Types/Location.cs b/src/Analysis/Ast/Impl/Types/Location.cs index 8cc2540bf..a7edb604d 100644 --- a/src/Analysis/Ast/Impl/Types/Location.cs +++ b/src/Analysis/Ast/Impl/Types/Location.cs @@ -30,7 +30,7 @@ public Location(IPythonModule module, IndexSpan indexSpan) { public LocationInfo LocationInfo { get { - var ast = Module?.Analysis.Ast; + var ast = Module.GetAst(); if (ast != null && !string.IsNullOrEmpty(Module?.FilePath) && Module?.Uri != null) { return new LocationInfo(Module.FilePath, Module.Uri, IndexSpan.ToSourceSpan(ast)); } diff --git a/src/Analysis/Ast/Impl/Types/ParameterInfo.cs b/src/Analysis/Ast/Impl/Types/ParameterInfo.cs index c6a67058b..15b0a04c0 100644 --- a/src/Analysis/Ast/Impl/Types/ParameterInfo.cs +++ b/src/Analysis/Ast/Impl/Types/ParameterInfo.cs @@ -34,17 +34,15 @@ public ParameterInfo(string name, IPythonType type, ParameterKind? kind, IMember Documentation = string.Empty; DefaultValue = defaultValue; Type = type; - IsParamArray = kind == ParameterKind.List; - IsKeywordDict = kind == ParameterKind.Dictionary; + Kind = kind ?? ParameterKind.Normal; } public string Name { get; } public string Documentation { get; } - public bool IsParamArray { get; } - public bool IsKeywordDict { get; } public bool IsGeneric { get; } public IPythonType Type { get; } public string DefaultValueString { get; } public IMember DefaultValue { get; } + public ParameterKind Kind { get; } } } diff --git a/src/LanguageServer/Impl/Completion/FunctionDefinitionCompletion.cs b/src/LanguageServer/Impl/Completion/FunctionDefinitionCompletion.cs index 629705baa..2c56cbc20 100644 --- a/src/LanguageServer/Impl/Completion/FunctionDefinitionCompletion.cs +++ b/src/LanguageServer/Impl/Completion/FunctionDefinitionCompletion.cs @@ -56,10 +56,10 @@ private static CompletionItem ToOverrideCompletionItem(IPythonFunctionOverload o } private static string MakeOverrideParameter(IParameterInfo paramInfo, string defaultValue) { - if (paramInfo.IsParamArray) { + if (paramInfo.Kind == ParameterKind.List) { return $"*{paramInfo.Name}"; } - if (paramInfo.IsKeywordDict) { + if (paramInfo.Kind == ParameterKind.Dictionary) { return $"**{paramInfo.Name}"; } return !string.IsNullOrEmpty(paramInfo.DefaultValueString) ? $"{paramInfo.Name}={defaultValue}" : paramInfo.Name; diff --git a/src/Parsing/Impl/Ast/PythonAst.cs b/src/Parsing/Impl/Ast/PythonAst.cs index fc6d8661c..8626600ef 100644 --- a/src/Parsing/Impl/Ast/PythonAst.cs +++ b/src/Parsing/Impl/Ast/PythonAst.cs @@ -97,8 +97,9 @@ public void Reduce(Func filter) { (Body as SuiteStatement)?.FilterStatements(filter); _attributes?.Clear(); Variables?.Clear(); - NewLineLocations = Array.Empty(); CommentLocations = Array.Empty(); + // DO keep NewLineLocations as they are required + // to calculate node positions for navigation; base.Clear(); } From 1b203267a2470867252703ce40cba0e6fdad0dea Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 27 May 2019 15:53:31 -0700 Subject: [PATCH 16/39] Add locks --- .../Ast/Impl/Analyzer/PythonAnalyzer.cs | 11 +++++--- src/Analysis/Ast/Impl/Modules/PythonModule.cs | 26 +++++++++++++------ 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index d6b52c46b..a8b9977ae 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -62,7 +62,7 @@ public void Dispose() { _disposeToken.TryMarkDisposed(); } - public Task WaitForCompleteAnalysisAsync(CancellationToken cancellationToken = default) + public Task WaitForCompleteAnalysisAsync(CancellationToken cancellationToken = default) => _analysisCompleteEvent.WaitAsync(cancellationToken); public async Task GetAnalysisAsync(IPythonModule module, int waitTime, CancellationToken cancellationToken) { @@ -190,8 +190,13 @@ public void ResetAnalyzer() { } } - public IReadOnlyList LoadedModules - => _analysisEntries.Values.ExcludeDefault().Select(v => v.Module).ExcludeDefault().ToArray(); + public IReadOnlyList LoadedModules { + get { + lock (_syncObj) { + return _analysisEntries.Values.ExcludeDefault().Select(v => v.Module).ExcludeDefault().ToArray(); + } + } + } internal bool IsFinalSession => _nextSession == null; diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index 344443356..cbbd1b379 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -431,21 +431,31 @@ protected virtual void OnAnalysisComplete() { } #endregion #region IAstNodeContainer - public T GetAstNode(object o) where T : Node => _astMap.TryGetValue(o, out var n) ? (T)n : null; + public T GetAstNode(object o) where T : Node { + lock (AnalysisLock) { + return _astMap.TryGetValue(o, out var n) ? (T)n : null; + } + } + public void AddAstNode(object o, Node n) { - Debug.Assert(!_astMap.ContainsKey(o) || _astMap[o] == n); - _astMap[o] = n; + lock (AnalysisLock) { + Debug.Assert(!_astMap.ContainsKey(o) || _astMap[o] == n); + _astMap[o] = n; + } } public void ClearAst() { - if (ModuleType != ModuleType.User) { - _astMap.Clear(); + lock (AnalysisLock) { + if (ModuleType != ModuleType.User) { + _astMap.Clear(); + } } } - public void ClearContent() { - if (ModuleType != ModuleType.User) { - _buffer.Reset(_buffer.Version, string.Empty); + lock (AnalysisLock) { + if (ModuleType != ModuleType.User) { + _buffer.Reset(_buffer.Version, string.Empty); + } } } #endregion From d7efdac4256324c49e9c39420c4559338c93fa4d Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Sat, 25 May 2019 15:10:53 -0700 Subject: [PATCH 17/39] Remove local variables --- src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs | 4 ++++ src/Analysis/Ast/Impl/Values/VariableCollection.cs | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs index 6221d6af7..6684bd412 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs @@ -79,6 +79,10 @@ public override void Evaluate() { if (ctor || annotationType.IsUnknown() || Module.ModuleType == ModuleType.User) { // Return type from the annotation is sufficient for libraries and stubs, no need to walk the body. FunctionDefinition.Body?.Walk(this); + // For libraries remove declared local function variables to free up some memory. + if (Module.ModuleType != ModuleType.User) { + ((VariableCollection)Eval.CurrentScope.Variables).Clear(); + } } } Result = _function; diff --git a/src/Analysis/Ast/Impl/Values/VariableCollection.cs b/src/Analysis/Ast/Impl/Values/VariableCollection.cs index 893358d73..817456eca 100644 --- a/src/Analysis/Ast/Impl/Values/VariableCollection.cs +++ b/src/Analysis/Ast/Impl/Values/VariableCollection.cs @@ -122,5 +122,11 @@ internal void RemoveVariable(string name) { _variables.Remove(name); } } + + internal void Clear() { + lock (_syncObj) { + _variables.Clear(); + } + } } } From 98934d4a372e604530a84f11639ad4b299e3f74c Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 27 May 2019 16:48:44 -0700 Subject: [PATCH 18/39] Drop file content to save memory --- src/Analysis/Ast/Impl/Modules/PythonModule.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index 18677e4b3..58ec9379d 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -438,6 +438,10 @@ public void NotifyAnalysisComplete(IDocumentAnalysis analysis) { // as declare additional variables, etc. OnAnalysisComplete(); ContentState = State.Analyzed; + + if (ModuleType != ModuleType.User) { + _buffer.Reset(_buffer.Version, string.Empty); + } } // Do not report issues with libraries or stubs From 417ae03eeb7ab655394505c5fe0732dbe8e1a68d Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 27 May 2019 17:01:15 -0700 Subject: [PATCH 19/39] Cache PEP hints --- .../Impl/Analyzer/Evaluation/ExpressionEval.Hints.cs | 11 ++++++++++- src/Parsing/Impl/Ast/PythonAst.cs | 4 ++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Hints.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Hints.cs index b12e08072..557ad5a2e 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Hints.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Hints.cs @@ -26,8 +26,15 @@ namespace Microsoft.Python.Analysis.Analyzer.Evaluation { /// and types in a chain of scopes during analysis. /// internal sealed partial class ExpressionEval { + private const string _pepHintKey = "PEP Hint"; + public IPythonType GetTypeFromPepHint(Node node) { + if (Ast.TryGetAttribute(node, _pepHintKey, out var typeStringObject) && typeStringObject is string typeString) { + return GetTypeFromString(typeString); + } + var location = GetLocationInfo(node); + var content = (Module as IDocument)?.Content; if (string.IsNullOrEmpty(content) || !location.EndLine.HasValue) { return null; @@ -79,7 +86,9 @@ public IPythonType GetTypeFromPepHint(Node node) { } // Type alone is not a valid syntax, so we need to simulate the annotation. - var typeString = content.Substring(hintStart, i - hintStart).Trim(); + typeString = content.Substring(hintStart, i - hintStart).Trim(); + Ast.SetAttribute(node, _pepHintKey, typeString); + return GetTypeFromString(typeString); } diff --git a/src/Parsing/Impl/Ast/PythonAst.cs b/src/Parsing/Impl/Ast/PythonAst.cs index 8ce4a97de..17cc32571 100644 --- a/src/Parsing/Impl/Ast/PythonAst.cs +++ b/src/Parsing/Impl/Ast/PythonAst.cs @@ -96,7 +96,7 @@ public override async Task WalkAsync(PythonWalkerAsync walker, CancellationToken public PythonLanguageVersion LanguageVersion { get; } - internal bool TryGetAttribute(Node node, object key, out object value) { + public bool TryGetAttribute(Node node, object key, out object value) { if (_attributes.TryGetValue(node, out var nodeAttrs)) { return nodeAttrs.TryGetValue(key, out value); } @@ -104,7 +104,7 @@ internal bool TryGetAttribute(Node node, object key, out object value) { return false; } - internal void SetAttribute(Node node, object key, object value) { + public void SetAttribute(Node node, object key, object value) { if (!_attributes.TryGetValue(node, out var nodeAttrs)) { nodeAttrs = _attributes[node] = new Dictionary(); } From ec5605f8ba6ebdf778fa90277a1243c41065fe50 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 27 May 2019 17:52:35 -0700 Subject: [PATCH 20/39] Recreate AST --- src/Analysis/Ast/Impl/Modules/PythonModule.cs | 20 ++++++++++++++++++- src/LanguageServer/Impl/Program.cs | 2 +- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index cbbd1b379..56e67d3d6 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -189,7 +189,7 @@ public virtual IEnumerable GetMemberNames() { /// Typically used in code navigation scenarios when user /// wants to see library code and not a stub. /// - public IPythonModule PrimaryModule { get; internal set; } + public IPythonModule PrimaryModule { get; private set; } #endregion #region IDisposable @@ -370,6 +370,13 @@ public override void Add(string message, SourceSpan span, int errorCode, Severit public void NotifyAnalysisBegins() { lock (AnalysisLock) { + if (Analysis is LibraryAnalysis) { + var sw = Log != null ? Stopwatch.StartNew() : null; + _astMap[this] = RecreateAst(); + sw?.Stop(); + Log?.Log(TraceEventType.Verbose, $"Reloaded AST of {Name} in {sw?.Elapsed.TotalMilliseconds} ms"); + } + if (_updated) { _updated = false; // In all variables find those imported, then traverse imported modules @@ -560,5 +567,16 @@ private void RemoveReferencesInModule(IPythonModule module) { } } } + + private PythonAst RecreateAst() { + lock (AnalysisLock) { + ContentState = State.None; + LoadContent(); + var parser = Parser.CreateParser(new StringReader(_buffer.Text), Interpreter.LanguageVersion, ParserOptions.Default); + var ast = parser.ParseFile(Uri); + ContentState = State.Parsed; + return ast; + } + } } } diff --git a/src/LanguageServer/Impl/Program.cs b/src/LanguageServer/Impl/Program.cs index 24adfa924..024f9af18 100644 --- a/src/LanguageServer/Impl/Program.cs +++ b/src/LanguageServer/Impl/Program.cs @@ -13,7 +13,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -#define WAIT_FOR_DEBUGGER +// #define WAIT_FOR_DEBUGGER using System; using System.Diagnostics; From 713d87f52222469f442f278616dd6e7411a94e47 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 27 May 2019 20:52:46 -0700 Subject: [PATCH 21/39] Fix specialization --- .../Ast/Impl/Analyzer/PythonAnalyzerSession.cs | 6 +++--- .../Ast/Impl/Extensions/AnalysisExtensions.cs | 2 +- .../Ast/Impl/Specializations/Specialized.cs | 4 ++-- .../Impl/Specializations/Typing/TypingModule.cs | 12 ++++++------ src/Analysis/Ast/Impl/Types/ArgumentSet.cs | 2 +- src/Analysis/Ast/Impl/Types/PythonFunctionType.cs | 14 +++++++------- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index 931cebbd5..831e9d9b6 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -380,8 +380,8 @@ private enum State { } private IDocumentAnalysis CreateAnalysis(IDocument document, int version, ModuleWalker walker, bool isFinalPass) - => document.ModuleType == ModuleType.User || !isFinalPass - ? (IDocumentAnalysis)new DocumentAnalysis(document, version, walker.GlobalScope, walker.Eval, walker.StarImportMemberNames) - : new LibraryAnalysis(document, version, walker.Eval.Services, walker.GlobalScope, walker.StarImportMemberNames); + => document.ModuleType == ModuleType.Library && isFinalPass + ? new LibraryAnalysis(document, version, walker.Eval.Services, walker.GlobalScope, walker.StarImportMemberNames) + : (IDocumentAnalysis)new DocumentAnalysis(document, version, walker.GlobalScope, walker.Eval, walker.StarImportMemberNames); } } diff --git a/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs b/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs index 8e53962f2..1a2bf48bc 100644 --- a/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs @@ -53,7 +53,7 @@ private static PythonFunctionType GetOrCreateFunction(this IDocumentAnalysis ana // We DO want to replace class by function. Consider type() in builtins. // 'type()' in code is a function call, not a type class instantiation. if (!(analysis.GlobalScope.Variables[name]?.Value is PythonFunctionType f)) { - f = PythonFunctionType.ForSpecialization(name, analysis.Document); + f = PythonFunctionType.Specialize(name, analysis.Document, string.Empty); f.AddOverload(new PythonFunctionOverload(name, new Location(analysis.Document))); analysis.GlobalScope.DeclareVariable(name, f, VariableSource.Declaration); } diff --git a/src/Analysis/Ast/Impl/Specializations/Specialized.cs b/src/Analysis/Ast/Impl/Specializations/Specialized.cs index 4da22b82c..e71222bc4 100644 --- a/src/Analysis/Ast/Impl/Specializations/Specialized.cs +++ b/src/Analysis/Ast/Impl/Specializations/Specialized.cs @@ -26,9 +26,9 @@ public static IPythonPropertyType Property(string name, IPythonModule declaringM return prop; } - public static IPythonFunctionType Function(string name, IPythonModule declaringModule, IPythonType declaringType, string documentation, IMember returnValue) { + public static IPythonFunctionType Function(string name, IPythonModule declaringModule, string documentation, IMember returnValue) { var location = new Location(declaringModule); - var prop = new PythonFunctionType(name, location, declaringType, documentation); + var prop = PythonFunctionType.Specialize(name, declaringModule, documentation); var o = new PythonFunctionOverload(prop.Name, location); o.AddReturnValue(returnValue); prop.AddOverload(o); diff --git a/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs b/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs index cf9be594d..8e280f25b 100644 --- a/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs +++ b/src/Analysis/Ast/Impl/Specializations/Typing/TypingModule.cs @@ -46,7 +46,7 @@ private void SpecializeMembers() { var location = new Location(this, default); // TypeVar - var fn = new PythonFunctionType("TypeVar", location, null, GetMemberDocumentation("TypeVar")); + var fn = PythonFunctionType.Specialize("TypeVar", this, GetMemberDocumentation("TypeVar")); var o = new PythonFunctionOverload(fn.Name, location); // When called, create generic parameter type. For documentation // use original TypeVar declaration so it appear as a tooltip. @@ -57,7 +57,7 @@ private void SpecializeMembers() { _members["TypeVar"] = fn; // NewType - fn = new PythonFunctionType("NewType", location, null, GetMemberDocumentation("NewType")); + fn = PythonFunctionType.Specialize("NewType", this, GetMemberDocumentation("NewType")); o = new PythonFunctionOverload(fn.Name, location); // When called, create generic parameter type. For documentation // use original TypeVar declaration so it appear as a tooltip. @@ -65,8 +65,8 @@ private void SpecializeMembers() { fn.AddOverload(o); _members["NewType"] = fn; - // NewType - fn = new PythonFunctionType("Type", location, null, GetMemberDocumentation("Type")); + // Type + fn = PythonFunctionType.Specialize("Type", this, GetMemberDocumentation("Type")); o = new PythonFunctionOverload(fn.Name, location); // When called, create generic parameter type. For documentation // use original TypeVar declaration so it appear as a tooltip. @@ -115,7 +115,7 @@ private void SpecializeMembers() { _members["Union"] = new GenericType("Union", CreateUnion, this); - _members["Counter"] = Specialized.Function("Counter", this, null, "Counter", + _members["Counter"] = Specialized.Function("Counter", this, GetMemberDocumentation("Counter"), new PythonInstance(Interpreter.GetBuiltinType(BuiltinTypeId.Int))); _members["SupportsInt"] = Interpreter.GetBuiltinType(BuiltinTypeId.Int); @@ -124,7 +124,7 @@ private void SpecializeMembers() { _members["SupportsBytes"] = Interpreter.GetBuiltinType(BuiltinTypeId.Bytes); _members["ByteString"] = Interpreter.GetBuiltinType(BuiltinTypeId.Bytes); - fn = new PythonFunctionType("NamedTuple", location, null, GetMemberDocumentation("NamedTuple")); + fn = PythonFunctionType.Specialize("NamedTuple", this, GetMemberDocumentation("NamedTuple")); o = new PythonFunctionOverload(fn.Name, location); o.SetReturnValueProvider((interpreter, overload, args) => CreateNamedTuple(args.Values())); fn.AddOverload(o); diff --git a/src/Analysis/Ast/Impl/Types/ArgumentSet.cs b/src/Analysis/Ast/Impl/Types/ArgumentSet.cs index dbcd33293..f014073da 100644 --- a/src/Analysis/Ast/Impl/Types/ArgumentSet.cs +++ b/src/Analysis/Ast/Impl/Types/ArgumentSet.cs @@ -252,7 +252,7 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in // then fill the slot with the default value. If there is no default value, // then it is an error. foreach (var slot in slots.Where(s => s.Kind != ParameterKind.List && s.Kind != ParameterKind.Dictionary && s.Value == null)) { - if (slot.Value == null) { + if (slot.ValueExpression == null) { var parameter = overload.Parameters.First(p => p.Name == slot.Name); if (parameter.DefaultValue == null) { // TODO: parameter is not assigned and has no default value. diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs index 31cf4092a..82c1def63 100644 --- a/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs +++ b/src/Analysis/Ast/Impl/Types/PythonFunctionType.cs @@ -24,7 +24,7 @@ namespace Microsoft.Python.Analysis.Types { [DebuggerDisplay("Function {Name} ({TypeId})")] - internal class PythonFunctionType : PythonType, IPythonFunctionType { + internal sealed class PythonFunctionType : PythonType, IPythonFunctionType { private ImmutableArray _overloads = ImmutableArray.Empty; private bool _isAbstract; private bool _isSpecialized; @@ -32,11 +32,11 @@ internal class PythonFunctionType : PythonType, IPythonFunctionType { /// /// Creates function for specializations /// - public static PythonFunctionType ForSpecialization(string name, IPythonModule declaringModule) - => new PythonFunctionType(name, new Location(declaringModule, default), true); + public static PythonFunctionType Specialize(string name, IPythonModule declaringModule, string documentation) + => new PythonFunctionType(name, new Location(declaringModule, default), documentation, true); - private PythonFunctionType(string name, Location location, bool isSpecialized = false) : - base(name, location, string.Empty, BuiltinTypeId.Function) { + private PythonFunctionType(string name, Location location, string documentation, bool isSpecialized = false) : + base(name, location, documentation ?? string.Empty, BuiltinTypeId.Function) { Check.ArgumentNotNull(nameof(location), location.Module); _isSpecialized = isSpecialized; } @@ -91,8 +91,8 @@ internal override void SetDocumentation(string documentation) { public FunctionDefinition FunctionDefinition => DeclaringModule.GetAstNode(this); public IPythonType DeclaringType { get; } public override string Documentation => (_overloads.Count > 0 ? _overloads[0].Documentation : default) ?? base.Documentation; - public virtual bool IsClassMethod { get; private set; } - public virtual bool IsStatic { get; private set; } + public bool IsClassMethod { get; private set; } + public bool IsStatic { get; private set; } public override bool IsAbstract => _isAbstract; public override bool IsSpecialized => _isSpecialized; From 50e63e67401e5910ddd3605be586fbfa4c3bce12 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 27 May 2019 21:06:57 -0700 Subject: [PATCH 22/39] Fix locations --- .../Ast/Impl/Modules/BuiltinsPythonModule.cs | 7 ++++--- src/Analysis/Ast/Impl/Values/GlobalScope.cs | 18 ++++++++++-------- src/Analysis/Ast/Impl/Values/Scope.cs | 19 ++++++++++--------- .../Ast/Impl/Values/VariableCollection.cs | 2 +- 4 files changed, 25 insertions(+), 21 deletions(-) diff --git a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs index 60cc720b7..3ce776620 100644 --- a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs @@ -121,13 +121,14 @@ private void SpecializeTypes() { _hiddenNames.Add("__builtin_module_names__"); + var location = new Location(this, default); if (_boolType != null) { - Analysis.GlobalScope.DeclareVariable("True", _boolType, VariableSource.Builtin, new Location(this, default)); - Analysis.GlobalScope.DeclareVariable("False", _boolType, VariableSource.Builtin, new Location(this, default)); + Analysis.GlobalScope.DeclareVariable("True", _boolType, VariableSource.Builtin, location); + Analysis.GlobalScope.DeclareVariable("False", _boolType, VariableSource.Builtin, location); } if (noneType != null) { - Analysis.GlobalScope.DeclareVariable("None", noneType, VariableSource.Builtin, new Location(this, default)); + Analysis.GlobalScope.DeclareVariable("None", noneType, VariableSource.Builtin, location); } foreach (var n in GetMemberNames()) { diff --git a/src/Analysis/Ast/Impl/Values/GlobalScope.cs b/src/Analysis/Ast/Impl/Values/GlobalScope.cs index 42e42e41d..fbdddd602 100644 --- a/src/Analysis/Ast/Impl/Values/GlobalScope.cs +++ b/src/Analysis/Ast/Impl/Values/GlobalScope.cs @@ -23,25 +23,27 @@ public GlobalScope(IPythonModule module): base(null, null, module) { DeclareBuiltinVariables(); } - public override ScopeStatement Node => Module.Analysis?.Ast; + public override ScopeStatement Node => Module.GetAstNode(this); private void DeclareBuiltinVariables() { if (Module.ModuleType != ModuleType.User) { return; } + var location = new Location(Module, default); + var boolType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Bool); var strType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Str); var listType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.List); var dictType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Dict); - DeclareVariable("__debug__", boolType, VariableSource.Builtin); - DeclareVariable("__doc__", strType, VariableSource.Builtin); - DeclareVariable("__file__", strType, VariableSource.Builtin); - DeclareVariable("__name__", strType, VariableSource.Builtin); - DeclareVariable("__package__", strType, VariableSource.Builtin); - DeclareVariable("__path__", listType, VariableSource.Builtin); - DeclareVariable("__dict__", dictType, VariableSource.Builtin); + DeclareVariable("__debug__", boolType, VariableSource.Builtin, location); + DeclareVariable("__doc__", strType, VariableSource.Builtin, location); + DeclareVariable("__file__", strType, VariableSource.Builtin, location); + DeclareVariable("__name__", strType, VariableSource.Builtin, location); + DeclareVariable("__package__", strType, VariableSource.Builtin, location); + DeclareVariable("__path__", listType, VariableSource.Builtin, location); + DeclareVariable("__dict__", dictType, VariableSource.Builtin, location); } } } diff --git a/src/Analysis/Ast/Impl/Values/Scope.cs b/src/Analysis/Ast/Impl/Values/Scope.cs index 3ce44ab06..d517c4457 100644 --- a/src/Analysis/Ast/Impl/Values/Scope.cs +++ b/src/Analysis/Ast/Impl/Values/Scope.cs @@ -96,24 +96,25 @@ private void DeclareBuiltinVariables() { return; } + var location = new Location(Module, default); var strType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Str); var objType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Object); - VariableCollection.DeclareVariable("__name__", strType, VariableSource.Builtin); + VariableCollection.DeclareVariable("__name__", strType, VariableSource.Builtin, location); if (Node is FunctionDefinition) { var dictType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Dict); var tupleType = Module.Interpreter.GetBuiltinType(BuiltinTypeId.Tuple); - VariableCollection.DeclareVariable("__closure__", tupleType, VariableSource.Builtin); - VariableCollection.DeclareVariable("__code__", objType, VariableSource.Builtin); - VariableCollection.DeclareVariable("__defaults__", tupleType, VariableSource.Builtin); - VariableCollection.DeclareVariable("__dict__", dictType, VariableSource.Builtin); - VariableCollection.DeclareVariable("__doc__", strType, VariableSource.Builtin); - VariableCollection.DeclareVariable("__func__", objType, VariableSource.Builtin); - VariableCollection.DeclareVariable("__globals__", dictType, VariableSource.Builtin); + VariableCollection.DeclareVariable("__closure__", tupleType, VariableSource.Builtin, location); + VariableCollection.DeclareVariable("__code__", objType, VariableSource.Builtin, location); + VariableCollection.DeclareVariable("__defaults__", tupleType, VariableSource.Builtin, location); + VariableCollection.DeclareVariable("__dict__", dictType, VariableSource.Builtin, location); + VariableCollection.DeclareVariable("__doc__", strType, VariableSource.Builtin, location); + VariableCollection.DeclareVariable("__func__", objType, VariableSource.Builtin, location); + VariableCollection.DeclareVariable("__globals__", dictType, VariableSource.Builtin, location); } else if (Node is ClassDefinition) { - VariableCollection.DeclareVariable("__self__", objType, VariableSource.Builtin); + VariableCollection.DeclareVariable("__self__", objType, VariableSource.Builtin, location); } } } diff --git a/src/Analysis/Ast/Impl/Values/VariableCollection.cs b/src/Analysis/Ast/Impl/Values/VariableCollection.cs index 817456eca..878182211 100644 --- a/src/Analysis/Ast/Impl/Values/VariableCollection.cs +++ b/src/Analysis/Ast/Impl/Values/VariableCollection.cs @@ -98,7 +98,7 @@ public IEnumerable GetMemberNames() { #endregion - internal void DeclareVariable(string name, IMember value, VariableSource source, Location location = default) { + internal void DeclareVariable(string name, IMember value, VariableSource source, Location location) { name = !string.IsNullOrWhiteSpace(name) ? name : throw new ArgumentException(nameof(name)); lock (_syncObj) { if (_variables.TryGetValue(name, out var existing)) { From fcd0c06f34153014baa5ad3a0dedd904da70c03a Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 28 May 2019 09:38:58 -0700 Subject: [PATCH 23/39] usings --- src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs index 43363a6d2..7b136035e 100644 --- a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs +++ b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs @@ -17,7 +17,6 @@ using System.Collections.Generic; using System.Linq; using Microsoft.Python.Analysis.Analyzer.Evaluation; -using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; From f6a992bbd23a3d515575987299079035f4743fa6 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 28 May 2019 11:00:54 -0700 Subject: [PATCH 24/39] Test fixes --- src/LanguageServer/Impl/Sources/DocumentationSource.cs | 7 ++++--- src/LanguageServer/Test/GoToDefinitionTests.cs | 3 +++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/LanguageServer/Impl/Sources/DocumentationSource.cs b/src/LanguageServer/Impl/Sources/DocumentationSource.cs index 17ea23396..bcebf6a71 100644 --- a/src/LanguageServer/Impl/Sources/DocumentationSource.cs +++ b/src/LanguageServer/Impl/Sources/DocumentationSource.cs @@ -13,7 +13,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System.Collections.Generic; +using System; using System.Linq; using Microsoft.Python.Analysis; using Microsoft.Python.Analysis.Types; @@ -50,8 +50,9 @@ private string[] GetFunctionParameters(IPythonFunctionType ft, out int[] paramet var o = ft.Overloads[overloadIndex]; // TODO: display all? var skip = ft.IsStatic || ft.IsUnbound ? 0 : 1; - var parameters = new string[o.Parameters.Count - skip]; - parameterNameLengths = new int[o.Parameters.Count - skip]; + var count = Math.Max(0, o.Parameters.Count - skip); + var parameters = new string[count]; + parameterNameLengths = new int[count]; for (var i = skip; i < o.Parameters.Count; i++) { string paramString; var p = o.Parameters[i]; diff --git a/src/LanguageServer/Test/GoToDefinitionTests.cs b/src/LanguageServer/Test/GoToDefinitionTests.cs index f42f9a696..0e23cc904 100644 --- a/src/LanguageServer/Test/GoToDefinitionTests.cs +++ b/src/LanguageServer/Test/GoToDefinitionTests.cs @@ -332,6 +332,9 @@ def foo(self): var mainPath = TestData.GetTestSpecificUri("main.py"); var doc = rdt.OpenDocument(mainPath, code); + var analyzer = Services.GetService(); + await analyzer.WaitForCompleteAnalysisAsync(); + var analysis = await doc.GetAnalysisAsync(Timeout.Infinite); var ds = new DefinitionSource(Services); From acad202d5049cdb1e700740d482821892ac4f6b5 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 28 May 2019 12:27:13 -0700 Subject: [PATCH 25/39] Add options to keep data in memory --- .../Impl/Analyzer/Symbols/FunctionEvaluator.cs | 3 ++- .../Ast/Impl/Definitions/AnalysisOptions.cs | 13 +++++++++++++ .../Impl/Definitions/ServerSettings.cs | 16 ++++++++++++++++ src/LanguageServer/Impl/Implementation/Server.cs | 5 +++++ .../Impl/LanguageServer.Configuration.cs | 5 +++++ 5 files changed, 41 insertions(+), 1 deletion(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs index 6684bd412..00657ad83 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs @@ -80,7 +80,8 @@ public override void Evaluate() { // Return type from the annotation is sufficient for libraries and stubs, no need to walk the body. FunctionDefinition.Body?.Walk(this); // For libraries remove declared local function variables to free up some memory. - if (Module.ModuleType != ModuleType.User) { + var optionsProvider = Eval.Services.GetService(); + if (Module.ModuleType != ModuleType.User && optionsProvider?.Options.KeepLibraryLocalVariables != true) { ((VariableCollection)Eval.CurrentScope.Variables).Clear(); } } diff --git a/src/Analysis/Ast/Impl/Definitions/AnalysisOptions.cs b/src/Analysis/Ast/Impl/Definitions/AnalysisOptions.cs index d34c88998..36c4eb180 100644 --- a/src/Analysis/Ast/Impl/Definitions/AnalysisOptions.cs +++ b/src/Analysis/Ast/Impl/Definitions/AnalysisOptions.cs @@ -16,5 +16,18 @@ namespace Microsoft.Python.Analysis { public class AnalysisOptions { public bool LintingEnabled { get; set; } + + /// + /// Keep in memory information on local variables declared in + /// functions in libraries. Provides ability to navigate to + /// symbols used in function bodies in packages and libraries. + /// + public bool KeepLibraryLocalVariables { get; set; } + + /// + /// Keep in memory AST of library source code. May somewhat + /// improve performance when library code has to be re-analyzed. + /// + public bool KeepLibraryAst { get; set; } } } diff --git a/src/LanguageServer/Impl/Definitions/ServerSettings.cs b/src/LanguageServer/Impl/Definitions/ServerSettings.cs index 442538191..2f1c38d22 100644 --- a/src/LanguageServer/Impl/Definitions/ServerSettings.cs +++ b/src/LanguageServer/Impl/Definitions/ServerSettings.cs @@ -29,6 +29,22 @@ public class PythonAnalysisOptions { public string[] warnings { get; private set; } = Array.Empty(); public string[] information { get; private set; } = Array.Empty(); public string[] disabled { get; private set; } = Array.Empty(); + + public class AnalysisMemoryOptions { + /// + /// Keep in memory information on local variables declared in + /// functions in libraries. Provides ability to navigate to + /// symbols used in function bodies in packages and libraries. + /// + public bool keepLibraryLocalVariables; + + /// + /// Keep in memory AST of library source code. May somewhat + /// improve performance when library code has to be re-analyzed. + /// + public bool keepLibraryAst; + } + public AnalysisMemoryOptions memory; } public readonly PythonAnalysisOptions analysis = new PythonAnalysisOptions(); diff --git a/src/LanguageServer/Impl/Implementation/Server.cs b/src/LanguageServer/Impl/Implementation/Server.cs index a21c1c1ea..a69d25b87 100644 --- a/src/LanguageServer/Impl/Implementation/Server.cs +++ b/src/LanguageServer/Impl/Implementation/Server.cs @@ -212,6 +212,11 @@ private bool HandleConfigurationChanges(ServerSettings newSettings) { return true; } + if(newSettings.analysis?.memory?.keepLibraryAst != oldSettings.analysis?.memory?.keepLibraryAst || + newSettings.analysis?.memory?.keepLibraryLocalVariables != oldSettings.analysis?.memory?.keepLibraryLocalVariables) { + return true; + } + return false; } diff --git a/src/LanguageServer/Impl/LanguageServer.Configuration.cs b/src/LanguageServer/Impl/LanguageServer.Configuration.cs index 9f66b4f7b..180a8fb21 100644 --- a/src/LanguageServer/Impl/LanguageServer.Configuration.cs +++ b/src/LanguageServer/Impl/LanguageServer.Configuration.cs @@ -73,6 +73,11 @@ private void HandleDiagnosticsChanges(JToken pythonSection, LanguageServerSettin var linting = pythonSection["linting"]; HandleLintingOnOff(_services, GetSetting(linting, "enabled", true)); + + var memory = analysis["memory"]; + var optionsProvider = _services.GetService(); + optionsProvider.Options.KeepLibraryLocalVariables = GetSetting(memory, "keepLibraryLocalVariables", false); + optionsProvider.Options.KeepLibraryAst = GetSetting(memory, "keepLibraryAst", false); } internal static void HandleLintingOnOff(IServiceContainer services, bool linterEnabled) { From 86e36a634da7ca3861a71f9a3fe6627ad1c1ccaf Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 28 May 2019 14:59:43 -0700 Subject: [PATCH 26/39] Fix test --- src/Analysis/Ast/Test/ArgumentSetTests.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Analysis/Ast/Test/ArgumentSetTests.cs b/src/Analysis/Ast/Test/ArgumentSetTests.cs index 2f4bee636..3f242eea5 100644 --- a/src/Analysis/Ast/Test/ArgumentSetTests.cs +++ b/src/Analysis/Ast/Test/ArgumentSetTests.cs @@ -77,7 +77,9 @@ def f(a, b, c='str'): ... argSet.Arguments[1].Name.Should().Be("b"); argSet.Arguments[1].ValueExpression.Should().BeOfType().Which.Value.Should().Be(1); argSet.Arguments[2].Name.Should().Be("c"); - argSet.Arguments[2].ValueExpression.Should().BeOfType().Which.Value.Should().Be("str"); + argSet.Arguments[2].ValueExpression.Should().BeNull(); + argSet.Arguments[2].Value.Should().NotBeNull(); + argSet.Arguments[2].Value.Should().BeAssignableTo().Which.Should().HaveType(BuiltinTypeId.Str); argSet.ListArgument.Should().BeNull(); argSet.DictionaryArgument.Should().BeNull(); } From ba97e9f3bb2b7cca6aca3d3e527ba155020930a9 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 28 May 2019 15:41:06 -0700 Subject: [PATCH 27/39] Fix lambda parameters --- .../Evaluation/ExpressionEval.Callables.cs | 67 ++++++++++++++++++ .../Analyzer/Symbols/FunctionEvaluator.cs | 68 +------------------ 2 files changed, 69 insertions(+), 66 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs index 55b4a8d02..2ea8aa437 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs @@ -17,6 +17,7 @@ using System.Diagnostics; using System.Linq; using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Analysis.Extensions; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; @@ -81,6 +82,7 @@ public IMember GetValueFromLambda(LambdaExpression expr) { var location = GetLocationOfName(fd); var ft = new PythonFunctionType(fd, null, location); var overload = new PythonFunctionOverload(ft, fd, location, expr.Function.ReturnAnnotation?.ToCodeString(Ast)); + overload.SetParameters(CreateFunctionParameters(null, ft, fd, false)); ft.AddOverload(overload); return ft; } @@ -312,5 +314,70 @@ private void LoadFunctionDependencyModules(IPythonFunctionType fn) { Services.GetService().EnqueueDocumentForAnalysis(Module, dependencies); } } + + public IReadOnlyList CreateFunctionParameters(IPythonClassType self, IPythonClassMember function, FunctionDefinition fd, bool declareVariables) { + // For class method no need to add extra parameters, but first parameter type should be the class. + // For static and unbound methods do not add or set anything. + // For regular bound methods add first parameter and set it to the class. + + var parameters = new List(); + var skip = 0; + if (self != null && function.HasClassFirstArgument()) { + var p0 = fd.Parameters.FirstOrDefault(); + if (p0 != null && !string.IsNullOrEmpty(p0.Name)) { + // Actual parameter type will be determined when method is invoked. + // The reason is that if method might be called on a derived class. + // Declare self or cls in this scope. + if (declareVariables) { + DeclareVariable(p0.Name, new PythonInstance(self), VariableSource.Declaration, p0.NameExpression); + } + // Set parameter info. + var pi = new ParameterInfo(Ast, p0, self, null, false); + parameters.Add(pi); + skip++; + } + } + + // Declare parameters in scope + IMember defaultValue = null; + for (var i = skip; i < fd.Parameters.Length; i++) { + var isGeneric = false; + var p = fd.Parameters[i]; + if (!string.IsNullOrEmpty(p.Name)) { + IPythonType paramType = null; + if (p.DefaultValue != null) { + defaultValue = GetValueFromExpression(p.DefaultValue); + // If parameter has default value, look for the annotation locally first + // since outer type may be getting redefined. Consider 's = None; def f(s: s = 123): ... + paramType = GetTypeFromAnnotation(p.Annotation, out isGeneric, LookupOptions.Local | LookupOptions.Builtins); + // Default value of None does not mean the parameter is None, just says it can be missing. + defaultValue = defaultValue.IsUnknown() || defaultValue.IsOfType(BuiltinTypeId.NoneType) ? null : defaultValue; + if (paramType == null && defaultValue != null) { + paramType = defaultValue.GetPythonType(); + } + } + // If all else fails, look up globally. + paramType = paramType ?? GetTypeFromAnnotation(p.Annotation, out isGeneric) ?? UnknownType; + var pi = new ParameterInfo(Ast, p, paramType, defaultValue, isGeneric); + if (declareVariables) { + DeclareParameter(p, pi); + } + parameters.Add(pi); + } + } + return parameters; + } + + private void DeclareParameter(Parameter p, ParameterInfo pi) { + IPythonType paramType; + // If type is known from annotation, use it. + if (pi != null && !pi.Type.IsUnknown() && !pi.Type.IsGenericParameter()) { + // TODO: technically generics may have constraints. Should we consider them? + paramType = pi.Type; + } else { + paramType = pi?.DefaultValue?.GetPythonType() ?? UnknownType; + } + DeclareVariable(p.Name, new PythonInstance(paramType), VariableSource.Declaration, p.NameExpression); + } } } diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs index 96d8cc46e..73882535b 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs @@ -70,7 +70,8 @@ public override void Evaluate() { } } - DeclareParameters(!stub); + var parameters = Eval.CreateFunctionParameters(_self, _function, FunctionDefinition, !stub); + _overload.SetParameters(parameters); // Do process body of constructors since they may be declaring // variables that are later used to determine return type of other @@ -125,70 +126,5 @@ public override bool Walk(FunctionDefinition node) { } return false; } - - private void DeclareParameters(bool declareVariables) { - // For class method no need to add extra parameters, but first parameter type should be the class. - // For static and unbound methods do not add or set anything. - // For regular bound methods add first parameter and set it to the class. - - var parameters = new List(); - var skip = 0; - if (_self != null && _function.HasClassFirstArgument()) { - var p0 = FunctionDefinition.Parameters.FirstOrDefault(); - if (p0 != null && !string.IsNullOrEmpty(p0.Name)) { - // Actual parameter type will be determined when method is invoked. - // The reason is that if method might be called on a derived class. - // Declare self or cls in this scope. - if (declareVariables) { - Eval.DeclareVariable(p0.Name, new PythonInstance(_self), VariableSource.Declaration, p0.NameExpression); - } - // Set parameter info. - var pi = new ParameterInfo(Ast, p0, _self, null, false); - parameters.Add(pi); - skip++; - } - } - - // Declare parameters in scope - IMember defaultValue = null; - for (var i = skip; i < FunctionDefinition.Parameters.Length; i++) { - var isGeneric = false; - var p = FunctionDefinition.Parameters[i]; - if (!string.IsNullOrEmpty(p.Name)) { - IPythonType paramType = null; - if (p.DefaultValue != null) { - defaultValue = Eval.GetValueFromExpression(p.DefaultValue); - // If parameter has default value, look for the annotation locally first - // since outer type may be getting redefined. Consider 's = None; def f(s: s = 123): ... - paramType = Eval.GetTypeFromAnnotation(p.Annotation, out isGeneric, LookupOptions.Local | LookupOptions.Builtins); - // Default value of None does not mean the parameter is None, just says it can be missing. - defaultValue = defaultValue.IsUnknown() || defaultValue.IsOfType(BuiltinTypeId.NoneType) ? null : defaultValue; - if (paramType == null && defaultValue != null) { - paramType = defaultValue.GetPythonType(); - } - } - // If all else fails, look up globally. - paramType = paramType ?? Eval.GetTypeFromAnnotation(p.Annotation, out isGeneric) ?? Eval.UnknownType; - var pi = new ParameterInfo(Ast, p, paramType, defaultValue, isGeneric); - if (declareVariables) { - DeclareParameter(p, pi); - } - parameters.Add(pi); - } - } - _overload.SetParameters(parameters); - } - - private void DeclareParameter(Parameter p, ParameterInfo pi) { - IPythonType paramType; - // If type is known from annotation, use it. - if (pi != null && !pi.Type.IsUnknown() && !pi.Type.IsGenericParameter()) { - // TODO: technically generics may have constraints. Should we consider them? - paramType = pi.Type; - } else { - paramType = pi?.DefaultValue?.GetPythonType() ?? Eval.UnknownType; - } - Eval.DeclareVariable(p.Name, new PythonInstance(paramType), VariableSource.Declaration, p.NameExpression); - } } } From ffed87e7af7c609e9a37c7f9f5858578a1556a68 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Tue, 28 May 2019 20:33:54 -0700 Subject: [PATCH 28/39] Fix argument set Fix global scope node --- .../Evaluation/ExpressionEval.Callables.cs | 2 ++ src/Analysis/Ast/Impl/Types/ArgumentSet.cs | 18 +++++++++++++++++- src/Analysis/Ast/Impl/Values/GlobalScope.cs | 2 +- src/Analysis/Ast/Test/ArgumentSetTests.cs | 4 +--- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs index 2ea8aa437..35e0df1be 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs @@ -363,6 +363,8 @@ public IReadOnlyList CreateFunctionParameters(IPythonClassType s DeclareParameter(p, pi); } parameters.Add(pi); + } else if (p.IsList || p.IsDictionary) { + parameters.Add(new ParameterInfo(Ast, p, null, null, false)); } } return parameters; diff --git a/src/Analysis/Ast/Impl/Types/ArgumentSet.cs b/src/Analysis/Ast/Impl/Types/ArgumentSet.cs index f014073da..6aa2b5038 100644 --- a/src/Analysis/Ast/Impl/Types/ArgumentSet.cs +++ b/src/Analysis/Ast/Impl/Types/ArgumentSet.cs @@ -15,6 +15,7 @@ // permissions and limitations under the License. using System.Collections.Generic; +using System.IO; using System.Linq; using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Analyzer.Evaluation; @@ -261,6 +262,7 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in } // Note that parameter default value expression is from the function definition AST // while actual argument values are from the calling file AST. + slot.ValueExpression = CreateExpression(parameter.Name, parameter.DefaultValueString); slot.Value = parameter.DefaultValue; slot.ValueIsDefault = true; } @@ -317,6 +319,20 @@ private IMember GetArgumentValue(Argument arg) { return Eval.GetValueFromExpression(arg.ValueExpression) ?? Eval.UnknownType; } + private Expression CreateExpression(string paramName, string defaultValue) { + if (string.IsNullOrEmpty(defaultValue)) { + return null; + } + using (var sr = new StringReader($"{paramName}={defaultValue}")) { + var parser = Parser.CreateParser(sr, Eval.Interpreter.LanguageVersion,ParserOptions.Default); + var ast = parser.ParseFile(); + if (ast.Body is SuiteStatement ste && ste.Statements.Count > 0 && ste.Statements[0] is AssignmentStatement a) { + return a.Right; + } + return null; + } + } + private sealed class Argument : IArgument { /// /// Argument name. @@ -349,7 +365,7 @@ private sealed class Argument : IArgument { /// /// Type of the argument, if annotated. /// - public IPythonType Type { get; internal set; } + public IPythonType Type { get; } /// /// AST node that defines the argument. diff --git a/src/Analysis/Ast/Impl/Values/GlobalScope.cs b/src/Analysis/Ast/Impl/Values/GlobalScope.cs index fbdddd602..699809d20 100644 --- a/src/Analysis/Ast/Impl/Values/GlobalScope.cs +++ b/src/Analysis/Ast/Impl/Values/GlobalScope.cs @@ -23,7 +23,7 @@ public GlobalScope(IPythonModule module): base(null, null, module) { DeclareBuiltinVariables(); } - public override ScopeStatement Node => Module.GetAstNode(this); + public override ScopeStatement Node => Module.GetAst(); private void DeclareBuiltinVariables() { if (Module.ModuleType != ModuleType.User) { diff --git a/src/Analysis/Ast/Test/ArgumentSetTests.cs b/src/Analysis/Ast/Test/ArgumentSetTests.cs index 3f242eea5..2f4bee636 100644 --- a/src/Analysis/Ast/Test/ArgumentSetTests.cs +++ b/src/Analysis/Ast/Test/ArgumentSetTests.cs @@ -77,9 +77,7 @@ def f(a, b, c='str'): ... argSet.Arguments[1].Name.Should().Be("b"); argSet.Arguments[1].ValueExpression.Should().BeOfType().Which.Value.Should().Be(1); argSet.Arguments[2].Name.Should().Be("c"); - argSet.Arguments[2].ValueExpression.Should().BeNull(); - argSet.Arguments[2].Value.Should().NotBeNull(); - argSet.Arguments[2].Value.Should().BeAssignableTo().Which.Should().HaveType(BuiltinTypeId.Str); + argSet.Arguments[2].ValueExpression.Should().BeOfType().Which.Value.Should().Be("str"); argSet.ListArgument.Should().BeNull(); argSet.DictionaryArgument.Should().BeNull(); } From 186a9c6ac1de0fde576e7da0c3df34f2f919b2e5 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Tue, 28 May 2019 21:05:04 -0700 Subject: [PATCH 29/39] Fix overload doc --- src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs index 7b136035e..a0cb983da 100644 --- a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs +++ b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs @@ -51,6 +51,7 @@ internal sealed class PythonFunctionOverload : LocatedMember, IPythonFunctionOve public PythonFunctionOverload(IPythonClassMember cm, FunctionDefinition fd, Location location, string returnDocumentation) : this(cm.Name, location) { ClassMember = cm; + Documentation = fd.GetDocumentation(); cm.DeclaringModule.AddAstNode(this, fd); _returnDocumentation = returnDocumentation; } From 419669950e0f3f440776ea718ddaa7ce4a087f8e Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Tue, 28 May 2019 21:51:18 -0700 Subject: [PATCH 30/39] Fix stub merge errors --- src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs index 5d8fc92f3..7fdfbe3f6 100644 --- a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs @@ -233,7 +233,7 @@ private void MergeStub() { // from the library source of from the scraped module. foreach (var v in _stubAnalysis.GlobalScope.Variables) { var stubType = v.Value.GetPythonType(); - if (stubType.IsUnknown()) { + if (stubType.IsUnknown() || !_stubAnalysis.Document.Equals(stubType?.DeclaringModule)) { continue; } @@ -249,7 +249,11 @@ private void MergeStub() { foreach (var name in stubType.GetMemberNames()) { var stubMember = stubType.GetMember(name); var member = cls.GetMember(name); - + // Do not pick members from base classes since otherwise we may end up + // reassigning members of a different module or even of built-in types. + if (member is IPythonClassMember clsm && !cls.Equals(clsm.DeclaringType)) { + continue; + } // Get documentation from the current type, if any, since stubs // typically do not contain documentation while scraped code does. member?.GetPythonType()?.TransferDocumentationAndLocation(stubMember.GetPythonType()); From 93249d40e12e7973140d43035b3d749441d67943 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Wed, 29 May 2019 12:41:23 -0700 Subject: [PATCH 31/39] Fix async issues --- .../Impl/Analyzer/Definitions/IAnalyzable.cs | 3 +- .../Evaluation/ExpressionEval.Scopes.cs | 3 - .../Analyzer/Handlers/FromImportHandler.cs | 3 +- .../Ast/Impl/Analyzer/ModuleWalker.cs | 4 +- .../Impl/Analyzer/PythonAnalyzerSession.cs | 14 +--- .../Impl/Analyzer/Symbols/ClassEvaluator.cs | 2 +- .../Impl/Analyzer/Symbols/SymbolCollector.cs | 15 ++-- .../Ast/Impl/Extensions/AnalysisExtensions.cs | 2 +- .../Ast/Impl/Extensions/AstExtensions.cs | 2 +- src/Analysis/Ast/Impl/Modules/PythonModule.cs | 19 +++-- src/Analysis/Ast/Impl/Types/LocatedMember.cs | 2 + .../Ast/Impl/Values/Definitions/IScope.cs | 2 +- src/Analysis/Ast/Impl/Values/Scope.cs | 2 +- src/Parsing/Impl/Ast/PythonAst.cs | 77 ++++++++----------- 14 files changed, 68 insertions(+), 82 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs b/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs index 8d4b9751d..bb1665b8d 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs @@ -26,7 +26,6 @@ internal interface IAnalyzable { /// /// Notifies document that its analysis is now complete. /// - /// Document analysis - void NotifyAnalysisComplete(IDocumentAnalysis analysis); + void NotifyAnalysisComplete(int version, ModuleWalker walker, bool isFinalPass); } } diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs index f332513d9..85cdcff5d 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs @@ -36,9 +36,6 @@ public T GetInScope(string name, IScope scope) where T : class, IMember public IMember GetInScope(string name) => GetInScope(name, CurrentScope); public T GetInScope(string name) where T : class, IMember => GetInScope(name, CurrentScope); - public void DeclareVariable(string name, IMember value, VariableSource source) - => DeclareVariable(name, value, source, default(Location)); - public void DeclareVariable(string name, IMember value, VariableSource source, IPythonModule module) => DeclareVariable(name, value, source, new Location(module, default)); diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs index cba333e73..71c389b97 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs @@ -83,6 +83,7 @@ private void HandleModuleImportStar(PythonVariableModule variableModule) { // If __all__ is present, take it, otherwise declare all members from the module that do not begin with an underscore. var memberNames = variableModule.Analysis.StarImportMemberNames ?? variableModule.GetMemberNames().Where(s => !s.StartsWithOrdinal("_")); + var location = new Location(Module); foreach (var memberName in memberNames) { var member = variableModule.GetMember(memberName); @@ -98,7 +99,7 @@ private void HandleModuleImportStar(PythonVariableModule variableModule) { } var variable = variableModule.Analysis?.GlobalScope?.Variables[memberName]; - Eval.DeclareVariable(memberName, variable ?? member, VariableSource.Import); + Eval.DeclareVariable(memberName, variable ?? member, VariableSource.Import, location); } } diff --git a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs index 7fdfbe3f6..4b376fe91 100644 --- a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs @@ -233,7 +233,7 @@ private void MergeStub() { // from the library source of from the scraped module. foreach (var v in _stubAnalysis.GlobalScope.Variables) { var stubType = v.Value.GetPythonType(); - if (stubType.IsUnknown() || !_stubAnalysis.Document.Equals(stubType?.DeclaringModule)) { + if (stubType.IsUnknown()) { continue; } @@ -263,7 +263,7 @@ private void MergeStub() { // Re-declare variable with the data from the stub unless member is a module. // Modules members that are modules should remain as they are, i.e. os.path // should remain library with its own stub attached. - if (!stubType.IsUnknown() && !(stubType is IPythonModule)) { + if (!(stubType is IPythonModule)) { sourceType.TransferDocumentationAndLocation(stubType); // TODO: choose best type between the scrape and the stub. Stub probably should always win. var source = Eval.CurrentScope.Variables[v.Name]?.Source ?? VariableSource.Declaration; diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index 831e9d9b6..5abdce573 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -336,8 +336,7 @@ private void AnalyzeEntry() { private void AnalyzeEntry(PythonAnalyzerEntry entry, IPythonModule module, int version, bool isFinalPass) { if (entry.PreviousAnalysis is LibraryAnalysis) { - _log?.Log(TraceEventType.Verbose, $"Request to re-analyze finalized {module.Name} ignored."); - return; + _log?.Log(TraceEventType.Verbose, $"Request to re-analyze finalized {module.Name}."); } // Now run the analysis. @@ -353,10 +352,8 @@ private void AnalyzeEntry(PythonAnalyzerEntry entry, IPythonModule module, int v walker.Complete(); _analyzerCancellationToken.ThrowIfCancellationRequested(); - var analysis = CreateAnalysis((IDocument)module, version, walker, isFinalPass); - - analyzable?.NotifyAnalysisComplete(analysis); - entry.TrySetAnalysis(analysis, version); + analyzable?.NotifyAnalysisComplete(version, walker, isFinalPass); + entry.TrySetAnalysis(module.Analysis, version); if (module.ModuleType == ModuleType.User) { var linterDiagnostics = _analyzer.LintModule(module); @@ -378,10 +375,5 @@ private enum State { Started = 1, Completed = 2 } - - private IDocumentAnalysis CreateAnalysis(IDocument document, int version, ModuleWalker walker, bool isFinalPass) - => document.ModuleType == ModuleType.Library && isFinalPass - ? new LibraryAnalysis(document, version, walker.Eval.Services, walker.GlobalScope, walker.StarImportMemberNames) - : (IDocumentAnalysis)new DocumentAnalysis(document, version, walker.GlobalScope, walker.Eval, walker.StarImportMemberNames); } } diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs index 1e1e6a767..08a0d740d 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs @@ -64,7 +64,7 @@ public void EvaluateClass() { _class.SetBases(bases); // Declare __class__ variable in the scope. - Eval.DeclareVariable("__class__", _class, VariableSource.Declaration); + Eval.DeclareVariable("__class__", _class, VariableSource.Declaration, Eval.DefaultLocation); ProcessClassBody(); } diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs index 55b322520..a5fafa899 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs @@ -54,7 +54,7 @@ public override bool Walk(ClassDefinition cd) { var classInfo = CreateClass(cd); // The variable is transient (non-user declared) hence it does not have location. // Class type is tracking locations for references and renaming. - _eval.DeclareVariable(cd.Name, classInfo, VariableSource.Declaration); + _eval.DeclareVariable(cd.Name, classInfo, VariableSource.Declaration, _eval.GetLocationOfName(cd)); _table.Add(new ClassEvaluator(_eval, cd)); // Open class scope _scopes.Push(_eval.OpenScope(_eval.Module, cd, out _)); @@ -108,7 +108,7 @@ private void AddFunction(FunctionDefinition fd, IPythonType declaringType) { existing = new PythonFunctionType(fd, declaringType, _eval.GetLocationOfName(fd)); // The variable is transient (non-user declared) hence it does not have location. // Function type is tracking locations for references and renaming. - _eval.DeclareVariable(fd.Name, existing, VariableSource.Declaration); + _eval.DeclareVariable(fd.Name, existing, VariableSource.Declaration, _eval.GetLocationOfName(fd)); } AddOverload(fd, existing, o => existing.AddOverload(o)); } @@ -165,14 +165,15 @@ private bool TryAddProperty(FunctionDefinition node, IPythonType declaringType) return false; } - private void AddProperty(FunctionDefinition node, IPythonType declaringType, bool isAbstract) { - if (!(_eval.LookupNameInScopes(node.Name, LookupOptions.Local) is PythonPropertyType existing)) { - existing = new PythonPropertyType(node, _eval.GetLocationOfName(node), declaringType, isAbstract); + private void AddProperty(FunctionDefinition fd, IPythonType declaringType, bool isAbstract) { + if (!(_eval.LookupNameInScopes(fd.Name, LookupOptions.Local) is PythonPropertyType existing)) { + var location = _eval.GetLocationOfName(fd); + existing = new PythonPropertyType(fd, location, declaringType, isAbstract); // The variable is transient (non-user declared) hence it does not have location. // Property type is tracking locations for references and renaming. - _eval.DeclareVariable(node.Name, existing, VariableSource.Declaration); + _eval.DeclareVariable(fd.Name, existing, VariableSource.Declaration, location); } - AddOverload(node, existing, o => existing.AddOverload(o)); + AddOverload(fd, existing, o => existing.AddOverload(o)); } private IMember GetMemberFromStub(string name) { diff --git a/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs b/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs index 1a2bf48bc..8be299581 100644 --- a/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs @@ -55,7 +55,7 @@ private static PythonFunctionType GetOrCreateFunction(this IDocumentAnalysis ana if (!(analysis.GlobalScope.Variables[name]?.Value is PythonFunctionType f)) { f = PythonFunctionType.Specialize(name, analysis.Document, string.Empty); f.AddOverload(new PythonFunctionOverload(name, new Location(analysis.Document))); - analysis.GlobalScope.DeclareVariable(name, f, VariableSource.Declaration); + analysis.GlobalScope.DeclareVariable(name, f, VariableSource.Declaration, new Location(analysis.Document)); } return f; } diff --git a/src/Analysis/Ast/Impl/Extensions/AstExtensions.cs b/src/Analysis/Ast/Impl/Extensions/AstExtensions.cs index c4a220000..a1d2b5d8f 100644 --- a/src/Analysis/Ast/Impl/Extensions/AstExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/AstExtensions.cs @@ -32,7 +32,7 @@ public static Expression FindExpression(this PythonAst ast, SourceLocation locat public static string GetDocumentation(this ScopeStatement node) { var docExpr = (node?.Body as SuiteStatement)?.Statements?.FirstOrDefault() as ExpressionStatement; var ce = docExpr?.Expression as ConstantExpression; - return ce?.Value as string; + return ce?.GetStringValue(); } public static bool IsInsideComment(this PythonAst ast, SourceLocation location) { diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index 49d2b3525..f9f0901e8 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -372,7 +372,9 @@ public void NotifyAnalysisBegins() { lock (AnalysisLock) { if (Analysis is LibraryAnalysis) { var sw = Log != null ? Stopwatch.StartNew() : null; - _astMap[this] = RecreateAst(); + lock (AnalysisLock) { + _astMap[this] = RecreateAst(); + } sw?.Stop(); Log?.Log(TraceEventType.Verbose, $"Reloaded AST of {Name} in {sw?.Elapsed.TotalMilliseconds} ms"); } @@ -403,14 +405,14 @@ public void NotifyAnalysisBegins() { } } - public void NotifyAnalysisComplete(IDocumentAnalysis analysis) { + public void NotifyAnalysisComplete(int version, ModuleWalker walker, bool isFinalPass) { lock (AnalysisLock) { - if (analysis.Version < Analysis.Version) { + if (version < Analysis.Version) { return; } - Analysis = analysis; - GlobalScope = analysis.GlobalScope; + Analysis = CreateAnalysis(version, walker, isFinalPass); + GlobalScope = Analysis.GlobalScope; // Derived classes can override OnAnalysisComplete if they want // to perform additional actions on the completed analysis such @@ -425,7 +427,7 @@ public void NotifyAnalysisComplete(IDocumentAnalysis analysis) { // Do not report issues with libraries or stubs if (ModuleType == ModuleType.User) { - _diagnosticsService?.Replace(Uri, analysis.Diagnostics, DiagnosticSource.Analysis); + _diagnosticsService?.Replace(Uri, Analysis.Diagnostics, DiagnosticSource.Analysis); } NewAnalysis?.Invoke(this, EventArgs.Empty); @@ -569,6 +571,11 @@ private void RemoveReferencesInModule(IPythonModule module) { } } + private IDocumentAnalysis CreateAnalysis(int version, ModuleWalker walker, bool isFinalPass) + => ModuleType == ModuleType.Library && isFinalPass + ? new LibraryAnalysis(this, version, walker.Eval.Services, walker.GlobalScope, walker.StarImportMemberNames) + : (IDocumentAnalysis)new DocumentAnalysis(this, version, walker.GlobalScope, walker.Eval, walker.StarImportMemberNames); + private PythonAst RecreateAst() { lock (AnalysisLock) { ContentState = State.None; diff --git a/src/Analysis/Ast/Impl/Types/LocatedMember.cs b/src/Analysis/Ast/Impl/Types/LocatedMember.cs index d752e3f4f..8faef76c0 100644 --- a/src/Analysis/Ast/Impl/Types/LocatedMember.cs +++ b/src/Analysis/Ast/Impl/Types/LocatedMember.cs @@ -18,6 +18,7 @@ using System.Linq; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Core; +using Microsoft.Python.Core.Diagnostics; namespace Microsoft.Python.Analysis.Types { internal abstract class LocatedMember : ILocatedMember { @@ -26,6 +27,7 @@ internal abstract class LocatedMember : ILocatedMember { protected LocatedMember(IPythonModule module) : this(new Location(module, default)) { } protected LocatedMember(Location location) { + Check.InvalidOperation(() => this is IPythonModule || location.Module != null, "Declaring module can only be null for modules."); Location = location; } diff --git a/src/Analysis/Ast/Impl/Values/Definitions/IScope.cs b/src/Analysis/Ast/Impl/Values/Definitions/IScope.cs index 08f9ec8f3..66233242c 100644 --- a/src/Analysis/Ast/Impl/Values/Definitions/IScope.cs +++ b/src/Analysis/Ast/Impl/Values/Definitions/IScope.cs @@ -85,7 +85,7 @@ public interface IScope { /// Variable value. /// Variable source. /// Variable name node location. - void DeclareVariable(string name, IMember value, VariableSource source, Location location = default); + void DeclareVariable(string name, IMember value, VariableSource source, Location location); /// /// Links variable from another module such as when it is imported. diff --git a/src/Analysis/Ast/Impl/Values/Scope.cs b/src/Analysis/Ast/Impl/Values/Scope.cs index d517c4457..5712f0235 100644 --- a/src/Analysis/Ast/Impl/Values/Scope.cs +++ b/src/Analysis/Ast/Impl/Values/Scope.cs @@ -73,7 +73,7 @@ public IEnumerable EnumerateTowardsGlobal { public IEnumerable EnumerateFromGlobal => EnumerateTowardsGlobal.Reverse(); - public void DeclareVariable(string name, IMember value, VariableSource source, Location location = default) + public void DeclareVariable(string name, IMember value, VariableSource source, Location location) => VariableCollection.DeclareVariable(name, value, source, location); public void LinkVariable(string name, IVariable v, Location location) diff --git a/src/Parsing/Impl/Ast/PythonAst.cs b/src/Parsing/Impl/Ast/PythonAst.cs index 8a01ebac5..70a669cf9 100644 --- a/src/Parsing/Impl/Ast/PythonAst.cs +++ b/src/Parsing/Impl/Ast/PythonAst.cs @@ -22,11 +22,11 @@ using Microsoft.Python.Core.Text; namespace Microsoft.Python.Parsing.Ast { - /// - /// Top-level ast for all Python code. Holds onto the body and the line mapping information. + /// Top-level ast for all Python code. Holds onto the body and the line mapping information. /// public sealed class PythonAst : ScopeStatement { + private readonly object _lock = new object(); private readonly Statement _body; private readonly Dictionary> _attributes = new Dictionary>(); @@ -94,68 +94,55 @@ public override async Task WalkAsync(PythonWalkerAsync walker, CancellationToken public PythonLanguageVersion LanguageVersion { get; } public void Reduce(Func filter) { - (Body as SuiteStatement)?.FilterStatements(filter); - _attributes?.Clear(); - Variables?.Clear(); - CommentLocations = Array.Empty(); - // DO keep NewLineLocations as they are required - // to calculate node positions for navigation; - base.Clear(); + lock (_lock) { + (Body as SuiteStatement)?.FilterStatements(filter); + _attributes?.Clear(); + Variables?.Clear(); + CommentLocations = Array.Empty(); + // DO keep NewLineLocations as they are required + // to calculate node positions for navigation; + base.Clear(); + } } public bool TryGetAttribute(Node node, object key, out object value) { - if (_attributes.TryGetValue(node, out var nodeAttrs)) { - return nodeAttrs.TryGetValue(key, out value); - } - value = null; - return false; - } + lock (_lock) { + if (_attributes.TryGetValue(node, out var nodeAttrs)) { + return nodeAttrs.TryGetValue(key, out value); + } - public void SetAttribute(Node node, object key, object value) { - if (!_attributes.TryGetValue(node, out var nodeAttrs)) { - nodeAttrs = _attributes[node] = new Dictionary(); + value = null; + return false; } - nodeAttrs[key] = value; } - /// - /// Copies attributes that apply to one node and makes them available for the other node. - /// - /// - /// - public void CopyAttributes(Node from, Node to) { - if (_attributes.TryGetValue(from, out var fromAttrs)) { - var toAttrs = new Dictionary(fromAttrs.Count); - foreach (var nodeAttr in fromAttrs) { - toAttrs[nodeAttr.Key] = nodeAttr.Value; + public void SetAttribute(Node node, object key, object value) { + lock (_lock) { + if (!_attributes.TryGetValue(node, out var nodeAttrs)) { + nodeAttrs = _attributes[node] = new Dictionary(); } - _attributes[to] = toAttrs; + nodeAttrs[key] = value; } } internal void SetAttributes(Dictionary> attributes) { - foreach (var nodeAttributes in attributes) { - var node = nodeAttributes.Key; - if (!_attributes.TryGetValue(node, out var existingNodeAttributes)) { - existingNodeAttributes = _attributes[node] = new Dictionary(nodeAttributes.Value.Count); - } - - foreach (var nodeAttr in nodeAttributes.Value) { - existingNodeAttributes[nodeAttr.Key] = nodeAttr.Value; + lock (_lock) { + foreach (var nodeAttributes in attributes) { + var node = nodeAttributes.Key; + if (!_attributes.TryGetValue(node, out var existingNodeAttributes)) { + existingNodeAttributes = _attributes[node] = new Dictionary(nodeAttributes.Value.Count); + } + + foreach (var nodeAttr in nodeAttributes.Value) { + existingNodeAttributes[nodeAttr.Key] = nodeAttr.Value; + } } } } public SourceLocation IndexToLocation(int index) => NewLineLocation.IndexToLocation(NewLineLocations, index); - public int LocationToIndex(SourceLocation location) => NewLineLocation.LocationToIndex(NewLineLocations, location, EndIndex); - /// - /// Length of the span (number of characters inside the span). - /// - public int GetSpanLength(SourceSpan span) => LocationToIndex(span.End) - LocationToIndex(span.Start); - - internal int GetLineEndFromPosition(int index) { var loc = IndexToLocation(index); if (loc.Line >= NewLineLocations.Length) { From 5e61392eed24cc8ad947d0c42eb5c833b1b97ae1 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Wed, 29 May 2019 16:12:43 -0700 Subject: [PATCH 32/39] Undo some changes --- src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs | 2 +- .../Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs | 3 +++ .../Ast/Impl/Analyzer/Handlers/FromImportHandler.cs | 3 +-- src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs | 4 ++-- src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs | 2 +- .../Ast/Impl/Analyzer/Symbols/SymbolCollector.cs | 9 ++++----- src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs | 2 +- src/Analysis/Ast/Impl/Types/LocatedMember.cs | 2 -- src/Analysis/Ast/Impl/Types/Location.cs | 2 +- src/Analysis/Ast/Impl/Values/Definitions/IScope.cs | 2 +- src/Analysis/Ast/Impl/Values/Scope.cs | 2 +- src/LanguageServer/Test/GoToDefinitionTests.cs | 7 +++++-- src/LanguageServer/Test/ReferencesTests.cs | 2 ++ 13 files changed, 23 insertions(+), 19 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs b/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs index d3802e6d7..60cf40216 100644 --- a/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs @@ -69,7 +69,7 @@ public override bool Walk(ExpressionStatement node) { return true; case CallExpression callex when callex.Target is MemberExpression mex && !string.IsNullOrEmpty(mex.Name): var t = Eval.GetValueFromExpression(mex.Target)?.GetPythonType(); - t?.GetMember(mex.Name).AddReference(Eval.GetLocationOfName(mex)); + t?.GetMember(mex.Name)?.AddReference(Eval.GetLocationOfName(mex)); return true; default: return base.Walk(node); diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs index 85cdcff5d..f332513d9 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs @@ -36,6 +36,9 @@ public T GetInScope(string name, IScope scope) where T : class, IMember public IMember GetInScope(string name) => GetInScope(name, CurrentScope); public T GetInScope(string name) where T : class, IMember => GetInScope(name, CurrentScope); + public void DeclareVariable(string name, IMember value, VariableSource source) + => DeclareVariable(name, value, source, default(Location)); + public void DeclareVariable(string name, IMember value, VariableSource source, IPythonModule module) => DeclareVariable(name, value, source, new Location(module, default)); diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs index 71c389b97..cba333e73 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/FromImportHandler.cs @@ -83,7 +83,6 @@ private void HandleModuleImportStar(PythonVariableModule variableModule) { // If __all__ is present, take it, otherwise declare all members from the module that do not begin with an underscore. var memberNames = variableModule.Analysis.StarImportMemberNames ?? variableModule.GetMemberNames().Where(s => !s.StartsWithOrdinal("_")); - var location = new Location(Module); foreach (var memberName in memberNames) { var member = variableModule.GetMember(memberName); @@ -99,7 +98,7 @@ private void HandleModuleImportStar(PythonVariableModule variableModule) { } var variable = variableModule.Analysis?.GlobalScope?.Variables[memberName]; - Eval.DeclareVariable(memberName, variable ?? member, VariableSource.Import, location); + Eval.DeclareVariable(memberName, variable ?? member, VariableSource.Import); } } diff --git a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs index 4b376fe91..d51328645 100644 --- a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs @@ -251,7 +251,7 @@ private void MergeStub() { var member = cls.GetMember(name); // Do not pick members from base classes since otherwise we may end up // reassigning members of a different module or even of built-in types. - if (member is IPythonClassMember clsm && !cls.Equals(clsm.DeclaringType)) { + if (member is IPythonClassMember clsm && !cls.Name.EqualsOrdinal(clsm.DeclaringType?.Name)) { continue; } // Get documentation from the current type, if any, since stubs @@ -267,7 +267,7 @@ private void MergeStub() { sourceType.TransferDocumentationAndLocation(stubType); // TODO: choose best type between the scrape and the stub. Stub probably should always win. var source = Eval.CurrentScope.Variables[v.Name]?.Source ?? VariableSource.Declaration; - Eval.DeclareVariable(v.Name, v.Value, source, Module); + Eval.DeclareVariable(v.Name, v.Value, source); } } } diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs index 08a0d740d..1e1e6a767 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/ClassEvaluator.cs @@ -64,7 +64,7 @@ public void EvaluateClass() { _class.SetBases(bases); // Declare __class__ variable in the scope. - Eval.DeclareVariable("__class__", _class, VariableSource.Declaration, Eval.DefaultLocation); + Eval.DeclareVariable("__class__", _class, VariableSource.Declaration); ProcessClassBody(); } diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs index a5fafa899..fe45a8fb9 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/SymbolCollector.cs @@ -54,7 +54,7 @@ public override bool Walk(ClassDefinition cd) { var classInfo = CreateClass(cd); // The variable is transient (non-user declared) hence it does not have location. // Class type is tracking locations for references and renaming. - _eval.DeclareVariable(cd.Name, classInfo, VariableSource.Declaration, _eval.GetLocationOfName(cd)); + _eval.DeclareVariable(cd.Name, classInfo, VariableSource.Declaration); _table.Add(new ClassEvaluator(_eval, cd)); // Open class scope _scopes.Push(_eval.OpenScope(_eval.Module, cd, out _)); @@ -108,7 +108,7 @@ private void AddFunction(FunctionDefinition fd, IPythonType declaringType) { existing = new PythonFunctionType(fd, declaringType, _eval.GetLocationOfName(fd)); // The variable is transient (non-user declared) hence it does not have location. // Function type is tracking locations for references and renaming. - _eval.DeclareVariable(fd.Name, existing, VariableSource.Declaration, _eval.GetLocationOfName(fd)); + _eval.DeclareVariable(fd.Name, existing, VariableSource.Declaration); } AddOverload(fd, existing, o => existing.AddOverload(o)); } @@ -167,11 +167,10 @@ private bool TryAddProperty(FunctionDefinition node, IPythonType declaringType) private void AddProperty(FunctionDefinition fd, IPythonType declaringType, bool isAbstract) { if (!(_eval.LookupNameInScopes(fd.Name, LookupOptions.Local) is PythonPropertyType existing)) { - var location = _eval.GetLocationOfName(fd); - existing = new PythonPropertyType(fd, location, declaringType, isAbstract); + existing = new PythonPropertyType(fd, _eval.GetLocationOfName(fd), declaringType, isAbstract); // The variable is transient (non-user declared) hence it does not have location. // Property type is tracking locations for references and renaming. - _eval.DeclareVariable(fd.Name, existing, VariableSource.Declaration, location); + _eval.DeclareVariable(fd.Name, existing, VariableSource.Declaration); } AddOverload(fd, existing, o => existing.AddOverload(o)); } diff --git a/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs b/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs index 8be299581..1a2bf48bc 100644 --- a/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/AnalysisExtensions.cs @@ -55,7 +55,7 @@ private static PythonFunctionType GetOrCreateFunction(this IDocumentAnalysis ana if (!(analysis.GlobalScope.Variables[name]?.Value is PythonFunctionType f)) { f = PythonFunctionType.Specialize(name, analysis.Document, string.Empty); f.AddOverload(new PythonFunctionOverload(name, new Location(analysis.Document))); - analysis.GlobalScope.DeclareVariable(name, f, VariableSource.Declaration, new Location(analysis.Document)); + analysis.GlobalScope.DeclareVariable(name, f, VariableSource.Declaration); } return f; } diff --git a/src/Analysis/Ast/Impl/Types/LocatedMember.cs b/src/Analysis/Ast/Impl/Types/LocatedMember.cs index 8faef76c0..d752e3f4f 100644 --- a/src/Analysis/Ast/Impl/Types/LocatedMember.cs +++ b/src/Analysis/Ast/Impl/Types/LocatedMember.cs @@ -18,7 +18,6 @@ using System.Linq; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Core; -using Microsoft.Python.Core.Diagnostics; namespace Microsoft.Python.Analysis.Types { internal abstract class LocatedMember : ILocatedMember { @@ -27,7 +26,6 @@ internal abstract class LocatedMember : ILocatedMember { protected LocatedMember(IPythonModule module) : this(new Location(module, default)) { } protected LocatedMember(Location location) { - Check.InvalidOperation(() => this is IPythonModule || location.Module != null, "Declaring module can only be null for modules."); Location = location; } diff --git a/src/Analysis/Ast/Impl/Types/Location.cs b/src/Analysis/Ast/Impl/Types/Location.cs index a7edb604d..dadb1d106 100644 --- a/src/Analysis/Ast/Impl/Types/Location.cs +++ b/src/Analysis/Ast/Impl/Types/Location.cs @@ -38,7 +38,7 @@ public LocationInfo LocationInfo { } } - public bool IsValid => Module != null; + public bool IsValid => Module != null && IndexSpan != default; public override bool Equals(object obj) => obj is Location other && other.Module == Module && other.IndexSpan == IndexSpan; diff --git a/src/Analysis/Ast/Impl/Values/Definitions/IScope.cs b/src/Analysis/Ast/Impl/Values/Definitions/IScope.cs index 66233242c..08f9ec8f3 100644 --- a/src/Analysis/Ast/Impl/Values/Definitions/IScope.cs +++ b/src/Analysis/Ast/Impl/Values/Definitions/IScope.cs @@ -85,7 +85,7 @@ public interface IScope { /// Variable value. /// Variable source. /// Variable name node location. - void DeclareVariable(string name, IMember value, VariableSource source, Location location); + void DeclareVariable(string name, IMember value, VariableSource source, Location location = default); /// /// Links variable from another module such as when it is imported. diff --git a/src/Analysis/Ast/Impl/Values/Scope.cs b/src/Analysis/Ast/Impl/Values/Scope.cs index 5712f0235..d517c4457 100644 --- a/src/Analysis/Ast/Impl/Values/Scope.cs +++ b/src/Analysis/Ast/Impl/Values/Scope.cs @@ -73,7 +73,7 @@ public IEnumerable EnumerateTowardsGlobal { public IEnumerable EnumerateFromGlobal => EnumerateTowardsGlobal.Reverse(); - public void DeclareVariable(string name, IMember value, VariableSource source, Location location) + public void DeclareVariable(string name, IMember value, VariableSource source, Location location = default) => VariableCollection.DeclareVariable(name, value, source, location); public void LinkVariable(string name, IVariable v, Location location) diff --git a/src/LanguageServer/Test/GoToDefinitionTests.cs b/src/LanguageServer/Test/GoToDefinitionTests.cs index 0e23cc904..1e2f53924 100644 --- a/src/LanguageServer/Test/GoToDefinitionTests.cs +++ b/src/LanguageServer/Test/GoToDefinitionTests.cs @@ -230,7 +230,9 @@ public async Task GotoRelativeImportInExplicitPackage() { rdt.OpenDocument(subpkgPath, string.Empty); var submod = rdt.OpenDocument(submodPath, "from .. import mod"); - var analysis = await submod.GetAnalysisAsync(-1); + await Services.GetService().WaitForCompleteAnalysisAsync(); + var analysis = await submod.GetAnalysisAsync(Timeout.Infinite); + var ds = new DefinitionSource(Services); var reference = ds.FindDefinition(analysis, new SourceLocation(1, 18), out _); reference.Should().NotBeNull(); @@ -250,7 +252,8 @@ public async Task GotoModuleSourceFromRelativeImport() { z = 1 def X(): ... "); - var analysis = await modA.GetAnalysisAsync(-1); + await Services.GetService().WaitForCompleteAnalysisAsync(); + var analysis = await modA.GetAnalysisAsync(Timeout.Infinite); var ds = new DefinitionSource(Services); var reference = ds.FindDefinition(analysis, new SourceLocation(1, 7), out _); diff --git a/src/LanguageServer/Test/ReferencesTests.cs b/src/LanguageServer/Test/ReferencesTests.cs index 2d2b23c2e..cd53d92f4 100644 --- a/src/LanguageServer/Test/ReferencesTests.cs +++ b/src/LanguageServer/Test/ReferencesTests.cs @@ -17,6 +17,7 @@ using System.IO; using System.Threading.Tasks; using FluentAssertions; +using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Core.Text; using Microsoft.Python.LanguageServer.Sources; @@ -368,6 +369,7 @@ import logging var rdt = Services.GetService(); var doc = rdt.OpenDocument(uri, code); + await Services.GetService().WaitForCompleteAnalysisAsync(); var analysis = await GetDocumentAnalysisAsync(doc); var rs = new ReferenceSource(Services); From 130b95dd48b18c534505e39e7cb1e10baf9c93e5 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Wed, 29 May 2019 16:32:22 -0700 Subject: [PATCH 33/39] Fix test --- src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs index d51328645..accdd872b 100644 --- a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs @@ -249,11 +249,6 @@ private void MergeStub() { foreach (var name in stubType.GetMemberNames()) { var stubMember = stubType.GetMember(name); var member = cls.GetMember(name); - // Do not pick members from base classes since otherwise we may end up - // reassigning members of a different module or even of built-in types. - if (member is IPythonClassMember clsm && !cls.Name.EqualsOrdinal(clsm.DeclaringType?.Name)) { - continue; - } // Get documentation from the current type, if any, since stubs // typically do not contain documentation while scraped code does. member?.GetPythonType()?.TransferDocumentationAndLocation(stubMember.GetPythonType()); From a6676f191cb5d1ec33967bc67261392acf58e9e6 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Thu, 30 May 2019 14:34:14 -0700 Subject: [PATCH 34/39] Fix race condition --- .../Analyzer/Evaluation/ExpressionEval.cs | 3 ++- src/Analysis/Ast/Impl/Modules/PythonModule.cs | 4 ++-- .../Resolution/ModuleResolutionBase.cs | 22 ++++--------------- src/Analysis/Ast/Impl/Types/Location.cs | 2 +- .../Impl/Sources/ReferenceSource.cs | 3 ++- 5 files changed, 11 insertions(+), 23 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs index 392dc1abe..e094e18e5 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs @@ -63,7 +63,8 @@ public ExpressionEval(IServiceContainer services, IPythonModule module, GlobalSc public LocationInfo GetLocationInfo(Node node) => node?.GetLocation(Module) ?? LocationInfo.Empty; public Location GetLocationOfName(Node node) { - if (node == null || (Module.ModuleType != ModuleType.User && Module.ModuleType != ModuleType.Library)) { + node = node ?? throw new ArgumentNullException(nameof(node)); + if (Module.ModuleType != ModuleType.User && Module.ModuleType != ModuleType.Library) { return DefaultLocation; } diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index f9f0901e8..6b0d7f3a8 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -129,7 +129,7 @@ internal PythonModule(ModuleCreationOptions creationOptions, IServiceContainer s public virtual string Documentation { get { - _documentation = _documentation ?? GetAstNode(this)?.Documentation; + _documentation = _documentation ?? this.GetAst()?.Documentation; if (_documentation == null) { var m = GetMember("__doc__"); _documentation = m.TryGetConstant(out var s) ? s : string.Empty; @@ -250,7 +250,7 @@ public async Task GetAstAsync(CancellationToken cancellationToken = d } } cancellationToken.ThrowIfCancellationRequested(); - return GetAstNode(this); + return this.GetAst(); } public PythonAst GetAnyAst() => GetAstNode(this); diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs index 2e7ac764d..1e952a1a9 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs @@ -80,7 +80,7 @@ public IReadOnlyCollection GetPackagesFromDirectory(string searchPath, C ).Select(mp => mp.ModuleName).Where(n => !string.IsNullOrEmpty(n)).TakeWhile(_ => !cancellationToken.IsCancellationRequested).ToList(); } - public IPythonModule GetImportedModule(string name) + public IPythonModule GetImportedModule(string name) => Modules.TryGetValue(name, out var moduleRef) ? moduleRef.Value : _interpreter.ModuleResolution.GetSpecializedModule(name); public IPythonModule GetOrLoadModule(string name) { @@ -97,7 +97,7 @@ public IPythonModule GetOrLoadModule(string name) { return moduleRef.GetOrCreate(name, this); } - public bool TryAddModulePath(in string path, in bool allowNonRooted, out string fullModuleName) + public bool TryAddModulePath(in string path, in bool allowNonRooted, out string fullModuleName) => PathResolver.TryAddModulePath(path, allowNonRooted, out fullModuleName); public ModulePath FindModule(string filePath) { @@ -122,13 +122,12 @@ protected void ReloadModulePaths(in IEnumerable rootPaths) { protected class ModuleRef { private readonly object _syncObj = new object(); private IPythonModule _module; - private bool _creating; public ModuleRef(IPythonModule module) { _module = module; } - public ModuleRef() {} + public ModuleRef() { } public IPythonModule Value { get { @@ -139,25 +138,12 @@ public IPythonModule Value { } public IPythonModule GetOrCreate(string name, ModuleResolutionBase mrb) { - var create = false; lock (_syncObj) { if (_module != null) { return _module; } - if (!_creating) { - create = true; - _creating = true; - } - } - - if (!create) { - return null; - } - - var module = mrb.CreateModule(name); - lock (_syncObj) { - _creating = false; + var module = mrb.CreateModule(name); _module = module; return module; } diff --git a/src/Analysis/Ast/Impl/Types/Location.cs b/src/Analysis/Ast/Impl/Types/Location.cs index dadb1d106..fd2275707 100644 --- a/src/Analysis/Ast/Impl/Types/Location.cs +++ b/src/Analysis/Ast/Impl/Types/Location.cs @@ -30,7 +30,7 @@ public Location(IPythonModule module, IndexSpan indexSpan) { public LocationInfo LocationInfo { get { - var ast = Module.GetAst(); + var ast = Module?.GetAst(); if (ast != null && !string.IsNullOrEmpty(Module?.FilePath) && Module?.Uri != null) { return new LocationInfo(Module.FilePath, Module.Uri, IndexSpan.ToSourceSpan(ast)); } diff --git a/src/LanguageServer/Impl/Sources/ReferenceSource.cs b/src/LanguageServer/Impl/Sources/ReferenceSource.cs index c302d91f3..009c7894f 100644 --- a/src/LanguageServer/Impl/Sources/ReferenceSource.cs +++ b/src/LanguageServer/Impl/Sources/ReferenceSource.cs @@ -20,6 +20,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis; +using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; @@ -129,7 +130,7 @@ private List ScanClosedFiles(string name, CancellationToken cancellationTok return files; } - private static async Task AnalyzeFiles(IModuleManagement moduleManagement, IEnumerable files, CancellationToken cancellationToken) { + private async Task AnalyzeFiles(IModuleManagement moduleManagement, IEnumerable files, CancellationToken cancellationToken) { var analysisTasks = new List(); foreach (var f in files) { if (moduleManagement.TryAddModulePath(f.ToAbsolutePath(), false, out var fullName)) { From 90318e91c8270e1725eb9490bb847dee1dc937ff Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 4 Jun 2019 16:15:31 -0700 Subject: [PATCH 35/39] Restore log null checks --- .../Impl/Analyzer/PythonAnalyzerSession.cs | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index 5d9290f6c..c0e9215f0 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -14,7 +14,6 @@ // permissions and limitations under the License. using System; -using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Runtime; @@ -22,7 +21,6 @@ using System.Threading.Tasks; using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Diagnostics; -using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core; @@ -147,7 +145,7 @@ private async Task StartAsync() { if (!isCanceled) { _progress.ReportRemaining(remaining); - if(isFinal) { + if (isFinal) { ActivityTracker.EndTracking(); (_analyzer as PythonAnalyzer)?.RaiseAnalysisComplete(ActivityTracker.ModuleCount, ActivityTracker.MillisecondsElapsed); _log?.Log(TraceEventType.Verbose, $"Analysis complete: {ActivityTracker.ModuleCount} modules in { ActivityTracker.MillisecondsElapsed} ms."); @@ -172,7 +170,7 @@ private static void LogResults(ILogger logger, double elapsed, int originalRemai if (logger == null) { return; } - + if (remaining == 0) { logger.Log(TraceEventType.Verbose, $"Analysis version {version} of {originalRemaining} entries has been completed in {elapsed} ms."); } else if (remaining < originalRemaining) { @@ -337,14 +335,23 @@ private void AnalyzeEntry(PythonAnalyzerEntry entry, IPythonModule module, int v } } - private void LogCompleted(IPythonModule module, Stopwatch stopWatch, TimeSpan startTime) - => _log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) completed in {(stopWatch.Elapsed - startTime).TotalMilliseconds} ms."); + private void LogCompleted(IPythonModule module, Stopwatch stopWatch, TimeSpan startTime) { + if (_log != null) { + _log.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) completed in {(stopWatch.Elapsed - startTime).TotalMilliseconds} ms."); + } + } - private void LogCanceled(IPythonModule module) - => _log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled."); + private void LogCanceled(IPythonModule module) { + if (_log != null) { + _log.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) canceled."); + } + } - private void LogException(IPythonModule module, Exception exception) - => _log?.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) failed. Exception message: {exception.Message}."); + private void LogException(IPythonModule module, Exception exception) { + if (_log != null) { + _log.Log(TraceEventType.Verbose, $"Analysis of {module.Name}({module.ModuleType}) failed. Exception message: {exception.Message}."); + } + } private enum State { NotStarted = 0, From a02c6f32a4f504fce8625f353289c80c3c431ca9 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Tue, 4 Jun 2019 16:45:07 -0700 Subject: [PATCH 36/39] Fix merge conflict --- .../Analyzer/Symbols/FunctionEvaluator.cs | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs index 164a36612..847bbc000 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Symbols/FunctionEvaluator.cs @@ -52,7 +52,9 @@ public override void Evaluate() { using (Eval.OpenScope(_function.DeclaringModule, FunctionDefinition, out _)) { var returnType = TryDetermineReturnValue(); - DeclareParameters(!stub); + + var parameters = Eval.CreateFunctionParameters(_self, _function, FunctionDefinition, !stub); + _overload.SetParameters(parameters); // Do process body of constructors since they may be declaring // variables that are later used to determine return type of other @@ -71,6 +73,30 @@ public override void Evaluate() { Result = _function; } + private IPythonType TryDetermineReturnValue() { + var annotationType = Eval.GetTypeFromAnnotation(FunctionDefinition.ReturnAnnotation); + if (!annotationType.IsUnknown()) { + // Annotations are typically types while actually functions return + // instances unless specifically annotated to a type such as Type[T]. + var t = annotationType.CreateInstance(annotationType.Name, ArgumentSet.Empty); + // If instance could not be created, such as when return type is List[T] and + // type of T is not yet known, just use the type. + var instance = t.IsUnknown() ? annotationType : t; + _overload.SetReturnValue(instance, true); _overload.SetReturnValue(instance, true); + } else { + // Check if function is a generator + var suite = FunctionDefinition.Body as SuiteStatement; + var yieldExpr = suite?.Statements.OfType().Select(s => s.Expression as YieldExpression).ExcludeDefault().FirstOrDefault(); + if (yieldExpr != null) { + // Function return is an iterator + var yieldValue = Eval.GetValueFromExpression(yieldExpr.Expression) ?? Eval.UnknownType; + var returnValue = new PythonGenerator(Eval.Interpreter, yieldValue); + _overload.SetReturnValue(returnValue, true); + } + } + return annotationType; + } + public override bool Walk(AssignmentStatement node) { var value = Eval.GetValueFromExpression(node.Right) ?? Eval.UnknownType; foreach (var lhs in node.Left) { From 57358ebb5320bcd8e160ae1d107323e1929f6baf Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Wed, 5 Jun 2019 08:48:00 -0700 Subject: [PATCH 37/39] Fix merge issue --- .../Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs index 35e0df1be..adf17e852 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs @@ -341,11 +341,10 @@ public IReadOnlyList CreateFunctionParameters(IPythonClassType s // Declare parameters in scope IMember defaultValue = null; for (var i = skip; i < fd.Parameters.Length; i++) { - var isGeneric = false; var p = fd.Parameters[i]; if (!string.IsNullOrEmpty(p.Name)) { - IPythonType paramType = null; - if (p.DefaultValue != null) { + var paramType = GetTypeFromAnnotation(p.Annotation, out var isGeneric) ?? UnknownType; + if (paramType.IsUnknown() && p.DefaultValue != null) { defaultValue = GetValueFromExpression(p.DefaultValue); // If parameter has default value, look for the annotation locally first // since outer type may be getting redefined. Consider 's = None; def f(s: s = 123): ... @@ -357,7 +356,6 @@ public IReadOnlyList CreateFunctionParameters(IPythonClassType s } } // If all else fails, look up globally. - paramType = paramType ?? GetTypeFromAnnotation(p.Annotation, out isGeneric) ?? UnknownType; var pi = new ParameterInfo(Ast, p, paramType, defaultValue, isGeneric); if (declareVariables) { DeclareParameter(p, pi); From 6ceb9b817fb9c8dcf64af228bd3151eec4b18b99 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Thu, 6 Jun 2019 11:40:02 -0700 Subject: [PATCH 38/39] Null check --- src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs index 39287f71a..0e5f90a51 100644 --- a/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs +++ b/src/Analysis/Ast/Impl/Types/PythonFunctionOverload.cs @@ -142,8 +142,10 @@ private IMember GetSpecificReturnType(IPythonClassType selfClassType, IArgumentS case IGenericType gt when args != null: // -> CLASS[T] on standalone function (i.e. -> List[T]). var typeArgs = ExpressionEval.GetTypeArgumentsFromParameters(this, args); - Debug.Assert(typeArgs != null); - return gt.CreateSpecificType(typeArgs); + if (typeArgs != null) { + return gt.CreateSpecificType(typeArgs); + } + break; } return StaticReturnValue; From 2164ad5b56a823dd7a11ebad27c3d9538420fbf1 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Thu, 6 Jun 2019 15:49:27 -0700 Subject: [PATCH 39/39] Fix test --- .../Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs | 7 +++---- src/Analysis/Ast/Impl/Types/ParameterInfo.cs | 1 - 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs index adf17e852..38346a6e0 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs @@ -339,13 +339,12 @@ public IReadOnlyList CreateFunctionParameters(IPythonClassType s } // Declare parameters in scope - IMember defaultValue = null; for (var i = skip; i < fd.Parameters.Length; i++) { var p = fd.Parameters[i]; if (!string.IsNullOrEmpty(p.Name)) { + var defaultValue = GetValueFromExpression(p.DefaultValue); var paramType = GetTypeFromAnnotation(p.Annotation, out var isGeneric) ?? UnknownType; - if (paramType.IsUnknown() && p.DefaultValue != null) { - defaultValue = GetValueFromExpression(p.DefaultValue); + if (paramType.IsUnknown()) { // If parameter has default value, look for the annotation locally first // since outer type may be getting redefined. Consider 's = None; def f(s: s = 123): ... paramType = GetTypeFromAnnotation(p.Annotation, out isGeneric, LookupOptions.Local | LookupOptions.Builtins); @@ -356,7 +355,7 @@ public IReadOnlyList CreateFunctionParameters(IPythonClassType s } } // If all else fails, look up globally. - var pi = new ParameterInfo(Ast, p, paramType, defaultValue, isGeneric); + var pi = new ParameterInfo(Ast, p, paramType, defaultValue, isGeneric | paramType.IsGeneric()); if (declareVariables) { DeclareParameter(p, pi); } diff --git a/src/Analysis/Ast/Impl/Types/ParameterInfo.cs b/src/Analysis/Ast/Impl/Types/ParameterInfo.cs index 15b0a04c0..07f5a5e5b 100644 --- a/src/Analysis/Ast/Impl/Types/ParameterInfo.cs +++ b/src/Analysis/Ast/Impl/Types/ParameterInfo.cs @@ -14,7 +14,6 @@ // permissions and limitations under the License. using System; -using Microsoft.Python.Core.Text; using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Types {