From 8798e27075eb46a3496a09db6c19dce91836b916 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Thu, 4 Oct 2018 14:09:39 -0700 Subject: [PATCH 1/7] Make tuples with different arguments different types --- src/Analysis/Engine/Impl/Values/Protocols.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Analysis/Engine/Impl/Values/Protocols.cs b/src/Analysis/Engine/Impl/Values/Protocols.cs index 8a095c638..ba4e6ca01 100644 --- a/src/Analysis/Engine/Impl/Values/Protocols.cs +++ b/src/Analysis/Engine/Impl/Values/Protocols.cs @@ -415,7 +415,7 @@ public override IAnalysisSet GetIndex(Node node, AnalysisUnit unit, IAnalysisSet return AnalysisSet.UnionAll(constants.Select(GetItem)); } - public override string Name => "tuple"; + public override string Name => "tuple[{0}]".FormatInvariant(string.Join(", ", _values.Select(v => v.GetShortDescriptions()))); public override IEnumerable> GetRichDescription() { if (_values.Any()) { From 6d4956fefbab0786af442a0bd5c2881654e90af9 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Thu, 4 Oct 2018 14:58:31 -0700 Subject: [PATCH 2/7] Cache name --- src/Analysis/Engine/Impl/Values/Protocols.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Analysis/Engine/Impl/Values/Protocols.cs b/src/Analysis/Engine/Impl/Values/Protocols.cs index ba4e6ca01..43ca5e3e9 100644 --- a/src/Analysis/Engine/Impl/Values/Protocols.cs +++ b/src/Analysis/Engine/Impl/Values/Protocols.cs @@ -385,10 +385,11 @@ protected override Protocol UnionMergeTypes(Protocol p) { } class TupleProtocol : IterableProtocol { - internal readonly IAnalysisSet[] _values; + private readonly IAnalysisSet[] _values; public TupleProtocol(ProtocolInfo self, IEnumerable values) : base(self, AnalysisSet.UnionAll(values)) { _values = values.Select(s => s.AsUnion(1)).ToArray(); + Name = "tuple[{0}]".FormatInvariant(string.Join(", ", _values.Select(v => v.GetShortDescriptions()))); } protected override void EnsureMembers(IDictionary members) { @@ -415,7 +416,7 @@ public override IAnalysisSet GetIndex(Node node, AnalysisUnit unit, IAnalysisSet return AnalysisSet.UnionAll(constants.Select(GetItem)); } - public override string Name => "tuple[{0}]".FormatInvariant(string.Join(", ", _values.Select(v => v.GetShortDescriptions()))); + public override string Name { get; } public override IEnumerable> GetRichDescription() { if (_values.Any()) { From 0b62fa5ea91741559eca7f3f2934791e1cebf8f2 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Thu, 4 Oct 2018 15:42:37 -0700 Subject: [PATCH 3/7] Symbols limit --- .../Extensions/EnumerableExtensions.cs | 19 ++++++++ .../Implementation/Server.WorkspaceSymbols.cs | 45 +++++++++++-------- .../Impl/Implementation/Server.cs | 2 +- src/LanguageServer/Impl/LanguageServer.cs | 2 +- .../Impl/LanguageServerSettings.cs | 2 +- 5 files changed, 49 insertions(+), 21 deletions(-) diff --git a/src/Analysis/Engine/Impl/Infrastructure/Extensions/EnumerableExtensions.cs b/src/Analysis/Engine/Impl/Infrastructure/Extensions/EnumerableExtensions.cs index 33b5855f0..b683a100a 100644 --- a/src/Analysis/Engine/Impl/Infrastructure/Extensions/EnumerableExtensions.cs +++ b/src/Analysis/Engine/Impl/Infrastructure/Extensions/EnumerableExtensions.cs @@ -59,5 +59,24 @@ public static bool SetEquals(this IEnumerable source, IEnumerable other public static IEnumerable Keys(this IEnumerable> source) => source.Select(GetKey); public static IEnumerable ExcludeDefault(this IEnumerable source) => source.Where(i => !Equals(i, default(T))); + + + public static IEnumerable TraverseBreadthFirst(this T root, Func> selectChildren) { + var items = new Queue(); + items.Enqueue(root); + while (items.Count > 0) { + var item = items.Dequeue(); + yield return item; + + var childen = selectChildren(item); + if (childen == null) { + continue; + } + + foreach (var child in childen) { + items.Enqueue(child); + } + } + } } } diff --git a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs b/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs index 23df611ed..d01357de8 100644 --- a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs +++ b/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs @@ -14,6 +14,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; using System.Collections.Generic; using System.Linq; using System.Threading; @@ -26,7 +27,7 @@ namespace Microsoft.Python.LanguageServer.Implementation { public sealed partial class Server { - private static int _symbolHierarchyDepthLimit = 1; + private static int _symbolHierarchyMaxSymbols = 1000; public override async Task WorkspaceSymbols(WorkspaceSymbolParams @params, CancellationToken cancellationToken) { await WaitForCompleteAnalysisAsync(cancellationToken); @@ -82,16 +83,16 @@ private static IEnumerable GetModuleVariables(ProjectEntry entry, } return false; }) - .Concat(GetChildScopesVariables(analysis, analysis.Scope, opts, 0)); + .Take(_symbolHierarchyMaxSymbols) + .Concat(GetChildScopesVariables(analysis, analysis.Scope, opts)) + .Take(_symbolHierarchyMaxSymbols); } - private static IEnumerable GetChildScopesVariables(IModuleAnalysis analysis, IScope scope, GetMemberOptions opts, int currentDepth) - => currentDepth < _symbolHierarchyDepthLimit - ? scope.Children.SelectMany(c => GetScopeVariables(analysis, c, opts, currentDepth)) - : Enumerable.Empty(); + private static IEnumerable GetChildScopesVariables(IModuleAnalysis analysis, IScope scope, GetMemberOptions opts) + => scope.TraverseBreadthFirst(s => s.Children).SelectMany(c => GetScopeVariables(analysis, c, opts)); - private static IEnumerable GetScopeVariables(IModuleAnalysis analysis, IScope scope, GetMemberOptions opts, int currentDepth) - => analysis.GetAllAvailableMembersFromScope(scope, opts).Concat(GetChildScopesVariables(analysis, scope, opts, currentDepth + 1)); + private static IEnumerable GetScopeVariables(IModuleAnalysis analysis, IScope scope, GetMemberOptions opts) + => analysis.GetAllAvailableMembersFromScope(scope, opts).Concat(GetChildScopesVariables(analysis, scope, opts)); private SymbolInformation ToSymbolInformation(IMemberResult m) { var res = new SymbolInformation { @@ -115,8 +116,9 @@ private SymbolInformation ToSymbolInformation(IMemberResult m) { } private DocumentSymbol[] ToDocumentSymbols(List members) { - var childMap = new Dictionary>(); var topLevel = new List(); + var childMap = new Dictionary>(); + var totalCount = 0; foreach (var m in members) { var parent = members.FirstOrDefault(x => x.Scope?.Node == m.Scope?.OuterScope?.Node && x.Name == m.Scope?.Name); @@ -128,18 +130,23 @@ private DocumentSymbol[] ToDocumentSymbols(List members) { } else { topLevel.Add(m); } + if (++totalCount >= _symbolHierarchyMaxSymbols) { + break; + } } - var symbols = topLevel - .GroupBy(mr => mr.Name) - .Select(g => g.First()) - .Select(m => ToDocumentSymbol(m, childMap, 0)) - .ToArray(); + var symbols = topLevel.SelectMany(t => t + .TraverseBreadthFirst(c => childMap.ContainsKey(c) ? childMap[c] : Enumerable.Empty()) + .Take(_symbolHierarchyMaxSymbols) + .GroupBy(mr => mr.Name) + .Select(g => g.First()) + .Select(m => ToDocumentSymbol(m, childMap))) + .ToArray(); return symbols; } - private DocumentSymbol ToDocumentSymbol(IMemberResult m, Dictionary> childMap, int currentDepth) { + private DocumentSymbol ToDocumentSymbol(IMemberResult m, Dictionary> childMap) { var res = new DocumentSymbol { name = m.Name, detail = m.Name, @@ -148,10 +155,12 @@ private DocumentSymbol ToDocumentSymbol(IMemberResult m, Dictionary ToDocumentSymbol(x, childMap, currentDepth + 1)).ToArray(); + if (childMap.TryGetValue(m, out var children)) { + res.children = children + .Select(x => ToDocumentSymbol(x, childMap)) + .ToArray(); } else { - res.children = new DocumentSymbol[0]; + res.children = Array.Empty(); } var loc = m.Locations.FirstOrDefault(l => !string.IsNullOrEmpty(l.FilePath)); diff --git a/src/LanguageServer/Impl/Implementation/Server.cs b/src/LanguageServer/Impl/Implementation/Server.cs index fe041a71c..7d4197be7 100644 --- a/src/LanguageServer/Impl/Implementation/Server.cs +++ b/src/LanguageServer/Impl/Implementation/Server.cs @@ -684,7 +684,7 @@ private bool HandleConfigurationChanges(ServerSettings newSettings) { var oldSettings = Settings; Settings = newSettings; - _symbolHierarchyDepthLimit = Settings.analysis.symbolsHierarchyDepthLimit; + _symbolHierarchyMaxSymbols = Settings.analysis.symbolsHierarchyMaxSymbols; if (oldSettings == null) { return true; diff --git a/src/LanguageServer/Impl/LanguageServer.cs b/src/LanguageServer/Impl/LanguageServer.cs index 425475bba..5ed979f88 100644 --- a/src/LanguageServer/Impl/LanguageServer.cs +++ b/src/LanguageServer/Impl/LanguageServer.cs @@ -142,7 +142,7 @@ public async Task DidChangeConfiguration(JToken token, CancellationToken cancell var analysis = pythonSection["analysis"]; settings.analysis.openFilesOnly = GetSetting(analysis, "openFilesOnly", false); settings.diagnosticPublishDelay = GetSetting(analysis, "diagnosticPublishDelay", 1000); - settings.symbolsHierarchyDepthLimit = GetSetting(analysis, "symbolsHierarchyDepthLimit", 10); + settings.symbolsHierarchyMaxSymbols = GetSetting(analysis, "symbolsHierarchyDepthLimit", 1000); _ui.SetLogLevel(GetLogLevel(analysis)); diff --git a/src/LanguageServer/Impl/LanguageServerSettings.cs b/src/LanguageServer/Impl/LanguageServerSettings.cs index d965ec9eb..44966fdbe 100644 --- a/src/LanguageServer/Impl/LanguageServerSettings.cs +++ b/src/LanguageServer/Impl/LanguageServerSettings.cs @@ -17,6 +17,6 @@ namespace Microsoft.Python.LanguageServer.Implementation { public sealed class LanguageServerSettings: ServerSettings { public int diagnosticPublishDelay = 1000; - public int symbolsHierarchyDepthLimit = 10; + public int symbolsHierarchyMaxSymbols = 1000; } } From 7a7a3ba772e39bcfb4d6b76001748f76e2221380 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Thu, 4 Oct 2018 15:42:49 -0700 Subject: [PATCH 4/7] Symbols limit --- src/LanguageServer/Impl/Definitions/ServerSettings.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageServer/Impl/Definitions/ServerSettings.cs b/src/LanguageServer/Impl/Definitions/ServerSettings.cs index 88fc757a9..209f81cc2 100644 --- a/src/LanguageServer/Impl/Definitions/ServerSettings.cs +++ b/src/LanguageServer/Impl/Definitions/ServerSettings.cs @@ -24,7 +24,7 @@ public class PythonAnalysisOptions { private Dictionary _map = new Dictionary(); public bool openFilesOnly; - public int symbolsHierarchyDepthLimit = 10; + public int symbolsHierarchyMaxSymbols = 1000; public string[] errors { get; } = Array.Empty(); public string[] warnings { get; } = Array.Empty(); From 3889bc480a2b14a200b8e5264c0c3e04300fd5ae Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Thu, 4 Oct 2018 15:45:54 -0700 Subject: [PATCH 5/7] Symbols limit --- .../Impl/Implementation/Server.WorkspaceSymbols.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs b/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs index d01357de8..b159cb6ae 100644 --- a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs +++ b/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs @@ -120,7 +120,7 @@ private DocumentSymbol[] ToDocumentSymbols(List members) { var childMap = new Dictionary>(); var totalCount = 0; - foreach (var m in members) { + foreach (var m in members.Take(_symbolHierarchyMaxSymbols)) { var parent = members.FirstOrDefault(x => x.Scope?.Node == m.Scope?.OuterScope?.Node && x.Name == m.Scope?.Name); if (parent != null) { if (!childMap.TryGetValue(parent, out var children)) { @@ -130,9 +130,6 @@ private DocumentSymbol[] ToDocumentSymbols(List members) { } else { topLevel.Add(m); } - if (++totalCount >= _symbolHierarchyMaxSymbols) { - break; - } } var symbols = topLevel.SelectMany(t => t From f5cc09709fc79f8641ffaeec4f6d030ce190c381 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Thu, 4 Oct 2018 16:26:34 -0700 Subject: [PATCH 6/7] Limit depth and count --- .../Impl/Definitions/ServerSettings.cs | 1 + .../Implementation/Server.WorkspaceSymbols.cs | 30 +++++++------------ .../Impl/Implementation/Server.cs | 1 + src/LanguageServer/Impl/LanguageServer.cs | 3 +- .../Impl/LanguageServerSettings.cs | 1 + 5 files changed, 16 insertions(+), 20 deletions(-) diff --git a/src/LanguageServer/Impl/Definitions/ServerSettings.cs b/src/LanguageServer/Impl/Definitions/ServerSettings.cs index 209f81cc2..0b519fc97 100644 --- a/src/LanguageServer/Impl/Definitions/ServerSettings.cs +++ b/src/LanguageServer/Impl/Definitions/ServerSettings.cs @@ -24,6 +24,7 @@ public class PythonAnalysisOptions { private Dictionary _map = new Dictionary(); public bool openFilesOnly; + public int symbolsHierarchyDepthLimit = 10; public int symbolsHierarchyMaxSymbols = 1000; public string[] errors { get; } = Array.Empty(); diff --git a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs b/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs index b159cb6ae..a23156b14 100644 --- a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs +++ b/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs @@ -27,6 +27,7 @@ namespace Microsoft.Python.LanguageServer.Implementation { public sealed partial class Server { + private static int _symbolHierarchyDepthLimit = 10; private static int _symbolHierarchyMaxSymbols = 1000; public override async Task WorkspaceSymbols(WorkspaceSymbolParams @params, CancellationToken cancellationToken) { @@ -73,8 +74,9 @@ private static async Task> GetModuleVariablesAsync(ProjectEn } private static IEnumerable GetModuleVariables(ProjectEntry entry, GetMemberOptions opts, string prefix, IModuleAnalysis analysis) { - var all = analysis.GetAllMembers(SourceLocation.None, opts); - return all + var breadthFirst = analysis.Scope.TraverseBreadthFirst(s => s.Children); + var all = breadthFirst.SelectMany(c => analysis.GetAllAvailableMembersFromScope(c, opts)); + var result = all .Where(m => { if (m.Values.Any(v => v.DeclaringModule == entry || v.Locations.Any(l => l.DocumentUri == entry.DocumentUri))) { if (string.IsNullOrEmpty(prefix) || m.Name.StartsWithOrdinal(prefix, ignoreCase: true)) { @@ -83,17 +85,10 @@ private static IEnumerable GetModuleVariables(ProjectEntry entry, } return false; }) - .Take(_symbolHierarchyMaxSymbols) - .Concat(GetChildScopesVariables(analysis, analysis.Scope, opts)) .Take(_symbolHierarchyMaxSymbols); + return result; } - private static IEnumerable GetChildScopesVariables(IModuleAnalysis analysis, IScope scope, GetMemberOptions opts) - => scope.TraverseBreadthFirst(s => s.Children).SelectMany(c => GetScopeVariables(analysis, c, opts)); - - private static IEnumerable GetScopeVariables(IModuleAnalysis analysis, IScope scope, GetMemberOptions opts) - => analysis.GetAllAvailableMembersFromScope(scope, opts).Concat(GetChildScopesVariables(analysis, scope, opts)); - private SymbolInformation ToSymbolInformation(IMemberResult m) { var res = new SymbolInformation { name = m.Name, @@ -118,9 +113,8 @@ private SymbolInformation ToSymbolInformation(IMemberResult m) { private DocumentSymbol[] ToDocumentSymbols(List members) { var topLevel = new List(); var childMap = new Dictionary>(); - var totalCount = 0; - foreach (var m in members.Take(_symbolHierarchyMaxSymbols)) { + foreach (var m in members) { var parent = members.FirstOrDefault(x => x.Scope?.Node == m.Scope?.OuterScope?.Node && x.Name == m.Scope?.Name); if (parent != null) { if (!childMap.TryGetValue(parent, out var children)) { @@ -132,18 +126,16 @@ private DocumentSymbol[] ToDocumentSymbols(List members) { } } - var symbols = topLevel.SelectMany(t => t - .TraverseBreadthFirst(c => childMap.ContainsKey(c) ? childMap[c] : Enumerable.Empty()) - .Take(_symbolHierarchyMaxSymbols) + var symbols = topLevel .GroupBy(mr => mr.Name) .Select(g => g.First()) - .Select(m => ToDocumentSymbol(m, childMap))) + .Select(m => ToDocumentSymbol(m, childMap, 0)) .ToArray(); return symbols; } - private DocumentSymbol ToDocumentSymbol(IMemberResult m, Dictionary> childMap) { + private DocumentSymbol ToDocumentSymbol(IMemberResult m, Dictionary> childMap, int currentDepth) { var res = new DocumentSymbol { name = m.Name, detail = m.Name, @@ -152,9 +144,9 @@ private DocumentSymbol ToDocumentSymbol(IMemberResult m, Dictionary ToDocumentSymbol(x, childMap)) + .Select(x => ToDocumentSymbol(x, childMap, currentDepth + 1)) .ToArray(); } else { res.children = Array.Empty(); diff --git a/src/LanguageServer/Impl/Implementation/Server.cs b/src/LanguageServer/Impl/Implementation/Server.cs index 7d4197be7..b48c232dd 100644 --- a/src/LanguageServer/Impl/Implementation/Server.cs +++ b/src/LanguageServer/Impl/Implementation/Server.cs @@ -684,6 +684,7 @@ private bool HandleConfigurationChanges(ServerSettings newSettings) { var oldSettings = Settings; Settings = newSettings; + _symbolHierarchyDepthLimit = Settings.analysis.symbolsHierarchyDepthLimit; _symbolHierarchyMaxSymbols = Settings.analysis.symbolsHierarchyMaxSymbols; if (oldSettings == null) { diff --git a/src/LanguageServer/Impl/LanguageServer.cs b/src/LanguageServer/Impl/LanguageServer.cs index 5ed979f88..21ce4110f 100644 --- a/src/LanguageServer/Impl/LanguageServer.cs +++ b/src/LanguageServer/Impl/LanguageServer.cs @@ -142,7 +142,8 @@ public async Task DidChangeConfiguration(JToken token, CancellationToken cancell var analysis = pythonSection["analysis"]; settings.analysis.openFilesOnly = GetSetting(analysis, "openFilesOnly", false); settings.diagnosticPublishDelay = GetSetting(analysis, "diagnosticPublishDelay", 1000); - settings.symbolsHierarchyMaxSymbols = GetSetting(analysis, "symbolsHierarchyDepthLimit", 1000); + settings.symbolsHierarchyDepthLimit = GetSetting(analysis, "symbolsHierarchyDepthLimit", 10); + settings.symbolsHierarchyMaxSymbols = GetSetting(analysis, "symbolsHierarchyMaxSymbols", 1000); _ui.SetLogLevel(GetLogLevel(analysis)); diff --git a/src/LanguageServer/Impl/LanguageServerSettings.cs b/src/LanguageServer/Impl/LanguageServerSettings.cs index 44966fdbe..0cc6917ed 100644 --- a/src/LanguageServer/Impl/LanguageServerSettings.cs +++ b/src/LanguageServer/Impl/LanguageServerSettings.cs @@ -17,6 +17,7 @@ namespace Microsoft.Python.LanguageServer.Implementation { public sealed class LanguageServerSettings: ServerSettings { public int diagnosticPublishDelay = 1000; + public int symbolsHierarchyDepthLimit = 10; public int symbolsHierarchyMaxSymbols = 1000; } } From 565e23e1db6f2eb8376abd1494ce6b4a695eef57 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Thu, 4 Oct 2018 16:41:41 -0700 Subject: [PATCH 7/7] Use qualified names --- src/Analysis/Engine/Impl/Values/Protocols.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Analysis/Engine/Impl/Values/Protocols.cs b/src/Analysis/Engine/Impl/Values/Protocols.cs index 43ca5e3e9..def71b7d7 100644 --- a/src/Analysis/Engine/Impl/Values/Protocols.cs +++ b/src/Analysis/Engine/Impl/Values/Protocols.cs @@ -389,7 +389,9 @@ class TupleProtocol : IterableProtocol { public TupleProtocol(ProtocolInfo self, IEnumerable values) : base(self, AnalysisSet.UnionAll(values)) { _values = values.Select(s => s.AsUnion(1)).ToArray(); - Name = "tuple[{0}]".FormatInvariant(string.Join(", ", _values.Select(v => v.GetShortDescriptions()))); + + var argTypes = _values.SelectMany(e => e.Select(v => v is IHasQualifiedName qn ? qn.FullyQualifiedName : v.ShortDescription)); + Name = "tuple[{0}]".FormatInvariant(string.Join(", ", argTypes)); } protected override void EnsureMembers(IDictionary members) {