diff --git a/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs b/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs index 5c6ce1867..c99d4cbc7 100644 --- a/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/AnalysisWalker.cs @@ -62,23 +62,24 @@ public override bool Walk(AssignmentStatement node) { } public override bool Walk(ExpressionStatement node) { - AssignmentHandler.HandleAnnotatedExpression(node.Expression as ExpressionWithAnnotation, null); + switch (node.Expression) { + case ExpressionWithAnnotation ea: + AssignmentHandler.HandleAnnotatedExpression(ea, null); + break; + case Comprehension comp: + Eval.ProcessComprehension(comp); + break; + } return false; } - public override bool Walk(ForStatement node) { - LoopHandler.HandleFor(node); - return base.Walk(node); - } - + public override bool Walk(ForStatement node) => LoopHandler.HandleFor(node); public override bool Walk(FromImportStatement node) => ImportHandler.HandleFromImport(node); public override bool Walk(GlobalStatement node) => NonLocalHandler.HandleGlobal(node); public override bool Walk(IfStatement node) => ConditionalHandler.HandleIf(node); public override bool Walk(ImportStatement node) => ImportHandler.HandleImport(node); - - public override bool Walk(NonlocalStatement node) - => NonLocalHandler.HandleNonLocal(node); + public override bool Walk(NonlocalStatement node) => NonLocalHandler.HandleNonLocal(node); public override bool Walk(TryStatement node) { TryExceptHandler.HandleTryExcept(node); diff --git a/src/Analysis/Ast/Impl/Analyzer/Definitions/IExpressionEvaluator.cs b/src/Analysis/Ast/Impl/Analyzer/Definitions/IExpressionEvaluator.cs index b674b7770..285d9d156 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Definitions/IExpressionEvaluator.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Definitions/IExpressionEvaluator.cs @@ -15,6 +15,7 @@ using System; using System.Collections.Generic; +using Microsoft.Python.Analysis.Analyzer.Evaluation; using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; @@ -57,7 +58,7 @@ public interface IExpressionEvaluator { /// IMember GetValueFromExpression(Expression expr); - IMember LookupNameInScopes(string name, out IScope scope); + IMember LookupNameInScopes(string name, out IScope scope, LookupOptions options = LookupOptions.Normal); IPythonType GetTypeFromString(string typeString); diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/LookupOptions.cs b/src/Analysis/Ast/Impl/Analyzer/Definitions/LookupOptions.cs similarity index 90% rename from src/Analysis/Ast/Impl/Analyzer/Evaluation/LookupOptions.cs rename to src/Analysis/Ast/Impl/Analyzer/Definitions/LookupOptions.cs index 7a5f9a5b0..32956751e 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/LookupOptions.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Definitions/LookupOptions.cs @@ -15,9 +15,9 @@ using System; -namespace Microsoft.Python.Analysis.Analyzer.Evaluation { +namespace Microsoft.Python.Analysis.Analyzer { [Flags] - internal enum LookupOptions { + public enum LookupOptions { None = 0, Local, Nonlocal, diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs index 3373a64ec..77e3eb020 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Callables.cs @@ -69,6 +69,18 @@ public IMember GetValueFromCallable(CallExpression expr) { return value; } + public IMember GetValueFromLambda(LambdaExpression expr) { + if (expr == null) { + return null; + } + + var loc = GetLoc(expr); + var ft = new PythonFunctionType(expr.Function, Module, null, loc); + var overload = new PythonFunctionOverload(expr.Function, ft, Module, GetLoc(expr)); + ft.AddOverload(overload); + return ft; + } + public IMember GetValueFromClassCtor(IPythonClassType cls, CallExpression expr) { SymbolTable.Evaluate(cls.ClassDefinition); // Determine argument types diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Collections.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Collections.cs index 859e91e2b..c549c0dfa 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Collections.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Collections.cs @@ -98,5 +98,65 @@ public IMember GetValueFromGenerator(GeneratorExpression expression) { } return UnknownType; } + + public IMember GetValueFromComprehension(Comprehension node) { + var oldVariables = CurrentScope.Variables.OfType().ToDictionary(k => k.Name, v => v); + try { + ProcessComprehension(node); + switch (node) { + case ListComprehension lc: + var v1 = GetValueFromExpression(lc.Item) ?? UnknownType; + return PythonCollectionType.CreateList(Interpreter, GetLoc(lc), new[] { v1 }); + case SetComprehension sc: + var v2 = GetValueFromExpression(sc.Item) ?? UnknownType; + return PythonCollectionType.CreateSet(Interpreter, GetLoc(sc), new[] { v2 }); + case DictionaryComprehension dc: + var k = GetValueFromExpression(dc.Key) ?? UnknownType; + var v = GetValueFromExpression(dc.Value) ?? UnknownType; + return new PythonDictionary(new PythonDictionaryType(Interpreter), GetLoc(dc), new Dictionary { { k, v } }); + } + + return UnknownType; + } finally { + // Remove temporary variables since this is assignment and the right hand + // side comprehension does not leak internal variables into the scope. + var newVariables = CurrentScope.Variables.ToDictionary(k => k.Name, v => v); + var variables = (VariableCollection)CurrentScope.Variables; + foreach (var kvp in newVariables) { + if (!oldVariables.ContainsKey(kvp.Key)) { + variables.RemoveVariable(kvp.Key); + } else { + variables.DeclareVariable(oldVariables[kvp.Key]); + } + } + } + } + + internal void ProcessComprehension(Comprehension node) { + foreach (var cfor in node.Iterators.OfType().Where(c => c.Left != null)) { + var value = GetValueFromExpression(cfor.List); + if (value != null) { + switch (cfor.Left) { + case NameExpression nex when value is IPythonCollection coll: + DeclareVariable(nex.Name, coll.GetIterator().Next, VariableSource.Declaration, GetLoc(nex)); + break; + case NameExpression nex: + DeclareVariable(nex.Name, UnknownType, VariableSource.Declaration, GetLoc(nex)); + break; + case TupleExpression tex when value is IPythonDictionary dict && tex.Items.Count > 0: + if (tex.Items[0] is NameExpression nx0 && !string.IsNullOrEmpty(nx0.Name)) { + DeclareVariable(nx0.Name, dict.Keys.FirstOrDefault() ?? UnknownType, VariableSource.Declaration, GetLoc(nx0)); + } + if (tex.Items.Count > 1 && tex.Items[1] is NameExpression nx1 && !string.IsNullOrEmpty(nx1.Name)) { + DeclareVariable(nx1.Name, dict.Values.FirstOrDefault() ?? UnknownType, VariableSource.Declaration, GetLoc(nx1)); + } + foreach (var item in tex.Items.Skip(2).OfType().Where(x => !string.IsNullOrEmpty(x.Name))) { + DeclareVariable(item.Name, UnknownType, VariableSource.Declaration, GetLoc(item)); + } + break; + } + } + } + } } } diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs index a2877e8a9..4dc334008 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.Scopes.cs @@ -141,8 +141,8 @@ public IPythonType GetTypeFromAnnotation(Expression expr, out bool isGeneric, Lo /// as a child of the specified scope. Scope is pushed on the stack /// and will be removed when returned the disposable is disposed. /// - public IDisposable OpenScope(IPythonModule module, ScopeStatement node, out Scope fromScope) { - fromScope = null; + public IDisposable OpenScope(IPythonModule module, ScopeStatement node, out Scope outerScope) { + outerScope = null; if (node == null) { return Disposable.Empty; } @@ -156,25 +156,25 @@ public IDisposable OpenScope(IPythonModule module, ScopeStatement node, out Scop } if (node.Parent != null) { - if (!_scopeLookupCache.TryGetValue(node.Parent, out fromScope)) { - fromScope = gs + if (!_scopeLookupCache.TryGetValue(node.Parent, out outerScope)) { + outerScope = gs .TraverseDepthFirst(s => s.Children.OfType()) .FirstOrDefault(s => s.Node == node.Parent); - _scopeLookupCache[node.Parent] = fromScope; + _scopeLookupCache[node.Parent] = outerScope; } } - fromScope = fromScope ?? gs; - if (fromScope != null) { + outerScope = outerScope ?? gs; + if (outerScope != null) { Scope scope; if (node is PythonAst) { // node points to global scope, it is not a function or a class. scope = gs; } else { - scope = fromScope.Children.OfType().FirstOrDefault(s => s.Node == node); + scope = outerScope.Children.OfType().FirstOrDefault(s => s.Node == node); if (scope == null) { - scope = new Scope(node, fromScope, true); - fromScope.AddChildScope(scope); + scope = new Scope(node, outerScope, true); + outerScope.AddChildScope(scope); _scopeLookupCache[node] = scope; } } diff --git a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs index 14465c9f5..8de5d64d0 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Evaluation/ExpressionEval.cs @@ -69,6 +69,15 @@ public ExpressionEval(IServiceContainer services, IPythonModule module, PythonAs public LocationInfo GetLocation(Node node) => node?.GetLocation(Module, Ast) ?? LocationInfo.Empty; public IEnumerable Diagnostics => _diagnostics; + public void ReportDiagnostics(Uri documentUri, DiagnosticsEntry entry) { + // Do not add if module is library, etc. Only handle user code. + if (Module.ModuleType == ModuleType.User) { + lock (_lock) { + _diagnostics.Add(entry); + } + } + } + public IMember GetValueFromExpression(Expression expr) => GetValueFromExpression(expr, DefaultLookupOptions); @@ -89,9 +98,7 @@ public IMember GetValueFromExpression(Expression expr, LookupOptions options) { return null; } - while (expr is ParenthesisExpression parExpr) { - expr = parExpr.Expression; - } + expr = expr.RemoveParenthesis(); IMember m; switch (expr) { @@ -131,6 +138,12 @@ public IMember GetValueFromExpression(Expression expr, LookupOptions options) { case GeneratorExpression genex: m = GetValueFromGenerator(genex); break; + case Comprehension comp: + m = GetValueFromComprehension(comp); + break; + case LambdaExpression lambda: + m = GetValueFromLambda(lambda); + break; default: m = GetValueFromBinaryOp(expr) ?? GetConstantFromLiteral(expr, options); break; @@ -228,14 +241,5 @@ private IMember GetValueFromConditional(ConditionalExpression expr) { return trueValue ?? falseValue ?? UnknownType; } - - public void ReportDiagnostics(Uri documentUri, DiagnosticsEntry entry) { - // Do not add if module is library, etc. Only handle user code. - if (Module.ModuleType == ModuleType.User) { - lock (_lock) { - _diagnostics.Add(entry); - } - } - } } } diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs index 27b3de368..15068b59f 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/AssignmentHandler.cs @@ -15,7 +15,6 @@ using System.Diagnostics; using System.Linq; -using Microsoft.Python.Analysis.Analyzer.Evaluation; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; @@ -62,7 +61,7 @@ public void HandleAssignment(AssignmentStatement node) { if (Eval.CurrentScope.NonLocals[ne.Name] != null) { Eval.LookupNameInScopes(ne.Name, out var scope, LookupOptions.Nonlocal); if (scope != null) { - scope.Variables[ne.Name].Value = value; + scope.Variables[ne.Name].Assign(value, Eval.GetLoc(ne)); } else { // TODO: report variable is not declared in outer scopes. } @@ -72,7 +71,7 @@ public void HandleAssignment(AssignmentStatement node) { if (Eval.CurrentScope.Globals[ne.Name] != null) { Eval.LookupNameInScopes(ne.Name, out var scope, LookupOptions.Global); if (scope != null) { - scope.Variables[ne.Name].Value = value; + scope.Variables[ne.Name].Assign(value, Eval.GetLoc(ne)); } else { // TODO: report variable is not declared in global scope. } diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/ConditionalHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/ConditionalHandler.cs index b104acb94..8720bc8e3 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/ConditionalHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/ConditionalHandler.cs @@ -15,8 +15,6 @@ using System; using System.Linq; -using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core.OS; using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Ast; diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/LoopHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/LoopHandler.cs index f4539ef16..2e359fd4b 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/LoopHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/LoopHandler.cs @@ -21,35 +21,27 @@ namespace Microsoft.Python.Analysis.Analyzer.Handlers { internal sealed class LoopHandler : StatementHandler { public LoopHandler(AnalysisWalker walker) : base(walker) { } - public void HandleFor(ForStatement node) { + public bool HandleFor(ForStatement node) { var iterable = Eval.GetValueFromExpression(node.List); var iterator = (iterable as IPythonIterable)?.GetIterator(); - if (iterator == null) { - // TODO: report that expression does not evaluate to iterable. - } var value = iterator?.Next ?? Eval.UnknownType; - switch (node.Left) { case NameExpression nex: // for x in y: - Eval.DeclareVariable(nex.Name, value, VariableSource.Declaration, Eval.GetLoc(node.Left)); + if (!string.IsNullOrEmpty(nex.Name)) { + Eval.DeclareVariable(nex.Name, value, VariableSource.Declaration, Eval.GetLoc(nex)); + } break; case TupleExpression tex: // x = [('abc', 42, True), ('abc', 23, False)] - // for some_str, some_int, some_bool in x: - var names = tex.Items.OfType().Select(x => x.Name).ToArray(); - if (value is IPythonIterable valueIterable) { - var valueIterator = valueIterable.GetIterator(); - foreach (var n in names) { - Eval.DeclareVariable(n, valueIterator?.Next ?? Eval.UnknownType, VariableSource.Declaration, Eval.GetLoc(node.Left)); - } - } else { - // TODO: report that expression yields value that does not evaluate to iterable. - } + // for some_str, (some_int, some_bool) in x: + var h = new TupleExpressionHandler(Walker); + h.HandleTupleAssignment(tex, node.List, value); break; } node.Body?.Walk(Walker); + return false; } public void HandleWhile(WhileStatement node) { diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/StatementHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/StatementHandler.cs index 3104db964..843fb41ed 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/StatementHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/StatementHandler.cs @@ -14,10 +14,8 @@ // permissions and limitations under the License. using Microsoft.Python.Analysis.Analyzer.Evaluation; -using Microsoft.Python.Analysis.Analyzer.Symbols; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core.Logging; using Microsoft.Python.Parsing.Ast; diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/TupleExpressionHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/TupleExpressionHandler.cs index ffb839918..124cac1bb 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/TupleExpressionHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/TupleExpressionHandler.cs @@ -13,8 +13,8 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.Linq; +using Microsoft.Python.Analysis.Analyzer.Evaluation; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Parsing.Ast; @@ -24,29 +24,76 @@ internal sealed class TupleExpressionHandler : StatementHandler { public TupleExpressionHandler(AnalysisWalker walker) : base(walker) { } public void HandleTupleAssignment(TupleExpression lhs, Expression rhs, IMember value) { - if (rhs is TupleExpression tex) { - var returnedExpressions = tex.Items.ToArray(); - var names = lhs.Items.OfType().Select(x => x.Name).ToArray(); - for (var i = 0; i < Math.Min(names.Length, returnedExpressions.Length); i++) { - if (returnedExpressions[i] != null && !string.IsNullOrEmpty(names[i])) { - var v = Eval.GetValueFromExpression(returnedExpressions[i]); - Eval.DeclareVariable(names[i], v ?? Eval.UnknownType, VariableSource.Declaration, returnedExpressions[i]); - } + AssignTuple(lhs, tex, Eval); + } else { + AssignTuple(lhs, value, Eval); + } + } + + internal static void AssignTuple(TupleExpression lhs, TupleExpression rhs, ExpressionEval eval) { + var returnedExpressions = rhs.Items.ToArray(); + var names = lhs.Items.OfType().Select(x => x.Name).ToArray(); + for (var i = 0; i < names.Length; i++) { + Expression e = null; + if (returnedExpressions.Length > 0) { + e = i < returnedExpressions.Length ? returnedExpressions[i] : returnedExpressions[returnedExpressions.Length - 1]; + } + + if (e != null && !string.IsNullOrEmpty(names[i])) { + var v = eval.GetValueFromExpression(e); + eval.DeclareVariable(names[i], v ?? eval.UnknownType, VariableSource.Declaration, e); } - return; } + } + internal static void AssignTuple(TupleExpression lhs, IMember value, ExpressionEval eval) { // Tuple = 'tuple value' (such as from callable). Transfer values. + IMember[] values; if (value is IPythonCollection seq) { - var types = seq.Contents.Select(c => c.GetPythonType()).ToArray(); - var expressions = lhs.Items.OfType().ToArray(); - var names = expressions.Select(x => x.Name).ToArray(); - for (var i = 0; i < Math.Min(names.Length, types.Length); i++) { - if (names[i] != null && types[i] != null) { - var instance = types[i].CreateInstance(null, Eval.GetLoc(expressions[i]), ArgumentSet.Empty); - Eval.DeclareVariable(names[i], instance, VariableSource.Declaration, expressions[i]); + values = seq.Contents.ToArray(); + } else { + values = new[] { value }; + } + + var typeEnum = new ValueEnumerator(values, eval.UnknownType); + AssignTuple(lhs, typeEnum, eval); + } + + private static void AssignTuple(TupleExpression tex, ValueEnumerator valueEnum, ExpressionEval eval) { + foreach (var item in tex.Items) { + switch (item) { + case NameExpression nex when !string.IsNullOrEmpty(nex.Name): + eval.DeclareVariable(nex.Name, valueEnum.Next, VariableSource.Declaration, nex); + break; + case TupleExpression te: + AssignTuple(te, valueEnum, eval); + break; + } + } + } + + private class ValueEnumerator { + private readonly IMember[] _values; + private readonly IMember _filler; + private int _index; + + public ValueEnumerator(IMember[] values, IMember filler) { + _values = values; + _filler = filler; + } + + public IMember Next { + get { + IMember t; + if (_values.Length > 0) { + t = _index < _values.Length ? _values[_index] : _values[_values.Length - 1]; + } else { + t = _filler; } + + _index++; + return t; } } } diff --git a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs index 3ba82e9aa..1769d5291 100644 --- a/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs +++ b/src/Analysis/Ast/Impl/Analyzer/ModuleWalker.cs @@ -15,6 +15,7 @@ using System.Diagnostics; using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Analysis.Linting; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index 9929221af..330a5d940 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -22,6 +22,7 @@ using Microsoft.Python.Analysis.Core.DependencyResolution; using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Documents; +using Microsoft.Python.Analysis.Linting; using Microsoft.Python.Analysis.Modules; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Core; @@ -358,12 +359,18 @@ private void AnalyzeEntry(PythonAnalyzerEntry entry, IPythonModule module, Pytho ast.Walk(walker); cancellationToken.ThrowIfCancellationRequested(); - // Note that we do not set the new analysis here and rather let - // Python analyzer to call NotifyAnalysisComplete. walker.Complete(); cancellationToken.ThrowIfCancellationRequested(); var analysis = new DocumentAnalysis((IDocument)module, version, walker.GlobalScope, walker.Eval); + if (module.ModuleType == ModuleType.User) { + var optionsProvider = _services.GetService(); + if (optionsProvider?.Options?.LintingEnabled != false) { + var linter = new LinterAggregator(); + linter.Lint(analysis, _services); + } + } + (module as IAnalyzable)?.NotifyAnalysisComplete(analysis); entry.TrySetAnalysis(analysis, version); } diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs b/src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs index dd387a924..4864de8ec 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonInterpreter.cs @@ -47,7 +47,6 @@ private async Task LoadBuiltinTypesAsync(string root, IServiceManager sm, Cancel sm.AddService(this); _moduleResolution = new MainModuleResolution(root, sm); - lock (_lock) { var builtinModule = _moduleResolution.CreateBuiltinsModule(); _builtinTypes[BuiltinTypeId.NoneType] diff --git a/src/Analysis/Ast/Impl/Definitions/AnalysisOptions.cs b/src/Analysis/Ast/Impl/Definitions/AnalysisOptions.cs new file mode 100644 index 000000000..d34c88998 --- /dev/null +++ b/src/Analysis/Ast/Impl/Definitions/AnalysisOptions.cs @@ -0,0 +1,20 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +namespace Microsoft.Python.Analysis { + public class AnalysisOptions { + public bool LintingEnabled { get; set; } + } +} diff --git a/src/Analysis/Ast/Impl/Definitions/IAnalysisOptionsProvider.cs b/src/Analysis/Ast/Impl/Definitions/IAnalysisOptionsProvider.cs new file mode 100644 index 000000000..26a9c7659 --- /dev/null +++ b/src/Analysis/Ast/Impl/Definitions/IAnalysisOptionsProvider.cs @@ -0,0 +1,20 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +namespace Microsoft.Python.Analysis { + public interface IAnalysisOptionsProvider { + AnalysisOptions Options { get; } + } +} diff --git a/src/Analysis/Ast/Impl/Diagnostics/ErrorCodes.cs b/src/Analysis/Ast/Impl/Diagnostics/ErrorCodes.cs index 6f7289338..dcebb7356 100644 --- a/src/Analysis/Ast/Impl/Diagnostics/ErrorCodes.cs +++ b/src/Analysis/Ast/Impl/Diagnostics/ErrorCodes.cs @@ -22,5 +22,6 @@ public static class ErrorCodes { public const string ParameterAlreadySpecified = "parameter-already-specified"; public const string ParameterMissing = "parameter-missing"; public const string UnresolvedImport = "unresolved-import"; + public const string UndefinedVariable = "undefined-variable"; } } diff --git a/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs b/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs index 82ea2f5b0..281c38744 100644 --- a/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs +++ b/src/Analysis/Ast/Impl/Extensions/NodeExtensions.cs @@ -61,5 +61,12 @@ public static bool IsInAst(this ScopeStatement node, PythonAst ast) { } return ast == node; } + + public static Expression RemoveParenthesis(this Expression e) { + while (e is ParenthesisExpression parExpr) { + e = parExpr.Expression; + } + return e; + } } } diff --git a/src/Analysis/Ast/Impl/Extensions/SourceSpanExtensions.cs b/src/Analysis/Ast/Impl/Extensions/SourceSpanExtensions.cs new file mode 100644 index 000000000..2bdd9f6b8 --- /dev/null +++ b/src/Analysis/Ast/Impl/Extensions/SourceSpanExtensions.cs @@ -0,0 +1,28 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using Microsoft.Python.Core.Text; + +namespace Microsoft.Python.Analysis { + public static class SourceSpanExtensions { + public static bool IsAfter(this SourceSpan span, SourceSpan other) { + if (!span.IsValid || !other.IsValid) { + return false; + } + return span.Start > other.Start; + } + } +} diff --git a/src/Analysis/Ast/Impl/Linting/ILinter.cs b/src/Analysis/Ast/Impl/Linting/ILinter.cs new file mode 100644 index 000000000..947386dce --- /dev/null +++ b/src/Analysis/Ast/Impl/Linting/ILinter.cs @@ -0,0 +1,22 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using Microsoft.Python.Core; + +namespace Microsoft.Python.Analysis.Linting { + public interface ILinter { + void Lint(IDocumentAnalysis analysis, IServiceContainer services); + } +} diff --git a/src/Analysis/Ast/Impl/Linting/LinterAggregator.cs b/src/Analysis/Ast/Impl/Linting/LinterAggregator.cs new file mode 100644 index 000000000..e20266ce1 --- /dev/null +++ b/src/Analysis/Ast/Impl/Linting/LinterAggregator.cs @@ -0,0 +1,34 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using Microsoft.Python.Analysis.Linting.UndefinedVariables; +using Microsoft.Python.Core; + +namespace Microsoft.Python.Analysis.Linting { + internal sealed class LinterAggregator { + private readonly List _linters = new List(); + + public LinterAggregator() { + // TODO: develop mechanism for dynamic and external linter discovery. + _linters.Add(new UndefinedVariablesLinter()); + } + public void Lint(IDocumentAnalysis analysis, IServiceContainer services) { + foreach (var l in _linters) { + l.Lint(analysis, services); + } + } + } +} diff --git a/src/Analysis/Ast/Impl/Linting/LinterWalker.cs b/src/Analysis/Ast/Impl/Linting/LinterWalker.cs new file mode 100644 index 000000000..5fe727f2e --- /dev/null +++ b/src/Analysis/Ast/Impl/Linting/LinterWalker.cs @@ -0,0 +1,47 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Core; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Linting { + internal abstract class LinterWalker: PythonWalker { + private readonly Stack _scopeStack = new Stack(); + + protected IDocumentAnalysis Analysis { get; } + protected IExpressionEvaluator Eval => Analysis.ExpressionEvaluator; + protected IServiceContainer Services { get; } + + protected LinterWalker(IDocumentAnalysis analysis, IServiceContainer services) { + Analysis = analysis; + Services = services; + } + + public override bool Walk(ClassDefinition cd) { + _scopeStack.Push(Eval.OpenScope(Analysis.Document, cd)); + return true; + } + public override void PostWalk(ClassDefinition cd) => _scopeStack.Pop().Dispose(); + + public override bool Walk(FunctionDefinition fd) { + _scopeStack.Push(Eval.OpenScope(Analysis.Document, fd)); + return true; + } + public override void PostWalk(FunctionDefinition cd) => _scopeStack.Pop().Dispose(); + } +} diff --git a/src/Analysis/Ast/Impl/Linting/UndefinedVariables/AnalysisExtensions.cs b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/AnalysisExtensions.cs new file mode 100644 index 000000000..23450802b --- /dev/null +++ b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/AnalysisExtensions.cs @@ -0,0 +1,31 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using Microsoft.Python.Analysis.Diagnostics; +using Microsoft.Python.Core; +using Microsoft.Python.Parsing; +using Microsoft.Python.Parsing.Ast; +using ErrorCodes = Microsoft.Python.Analysis.Diagnostics.ErrorCodes; + +namespace Microsoft.Python.Analysis.Linting.UndefinedVariables { + internal static class AnalysisExtensions { + public static void ReportUndefinedVariable(this IDocumentAnalysis analysis, NameExpression node) { + var eval = analysis.ExpressionEvaluator; + eval.ReportDiagnostics(analysis.Document.Uri, new DiagnosticsEntry( + Resources.UndefinedVariable.FormatInvariant(node.Name), + eval.GetLocation(node).Span, ErrorCodes.UndefinedVariable, Severity.Warning)); + } + } +} diff --git a/src/Analysis/Ast/Impl/Linting/UndefinedVariables/ComprehensionWalker.cs b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/ComprehensionWalker.cs new file mode 100644 index 000000000..9f6ca794a --- /dev/null +++ b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/ComprehensionWalker.cs @@ -0,0 +1,78 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Linting.UndefinedVariables { + internal sealed class ComprehensionWalker : PythonWalker { + private readonly IDocumentAnalysis _analysis; + private readonly HashSet _localNames = new HashSet(); + private readonly HashSet _localNameNodes = new HashSet(); + + public ComprehensionWalker(IDocumentAnalysis analysis) { + _analysis = analysis; + } + + public override bool Walk(GeneratorExpression node) { + ProcessComprehension(node, node.Item, node.Iterators); + return false; + } + + public override bool Walk(ListComprehension node) { + ProcessComprehension(node, node.Item, node.Iterators); + return false; + } + + public override bool Walk(SetComprehension node) { + ProcessComprehension(node, node.Item, node.Iterators); + return false; + } + + public override bool Walk(ForStatement node) { + var nc = new NameCollectorWalker(_localNames, _localNameNodes); + node.Left?.Walk(nc); + return true; + } + + public override bool Walk(DictionaryComprehension node) { + CollectNames(node); + var ew = new ExpressionWalker(_analysis, _localNames, _localNameNodes); + node.Key?.Walk(ew); + node.Value?.Walk(ew); + foreach (var iter in node.Iterators) { + iter?.Walk(ew); + } + return true; + } + + private void CollectNames(Comprehension c) { + var nc = new NameCollectorWalker(_localNames, _localNameNodes); + foreach (var cfor in c.Iterators.OfType()) { + cfor.Left?.Walk(nc); + } + } + + private void ProcessComprehension(Comprehension c, Node item, IEnumerable iterators) { + CollectNames(c); + var ew = new ExpressionWalker(_analysis, _localNames, _localNameNodes); + item?.Walk(ew); + foreach (var iter in iterators) { + iter.Walk(ew); + } + } + } +} diff --git a/src/Analysis/Ast/Impl/Linting/UndefinedVariables/ExpressionWalker.cs b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/ExpressionWalker.cs new file mode 100644 index 000000000..c8e663625 --- /dev/null +++ b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/ExpressionWalker.cs @@ -0,0 +1,115 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Core; +using Microsoft.Python.Core.Text; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Linting.UndefinedVariables { + internal sealed class ExpressionWalker : PythonWalker { + private readonly IDocumentAnalysis _analysis; + private readonly HashSet _localNames; + private readonly HashSet _localNameNodes; + + public ExpressionWalker(IDocumentAnalysis analysis) + : this(analysis, null, null) { } + + /// + /// Creates walker for detection of undefined variables. + /// + /// Document analysis. + /// Locally defined names, such as variables in a comprehension. + /// Name nodes for local names. + public ExpressionWalker(IDocumentAnalysis analysis, HashSet localNames, HashSet localNameNodes) { + _analysis = analysis; + _localNames = localNames; + _localNameNodes = localNameNodes; + } + + public override bool Walk(CallExpression node) { + foreach (var arg in node.Args) { + arg?.Expression?.Walk(this); + } + return false; + } + + public override bool Walk(LambdaExpression node) { + node.Walk(new LambdaWalker(_analysis)); + return false; + } + + public override bool Walk(ListComprehension node) { + node.Walk(new ComprehensionWalker(_analysis)); + return false; + } + + public override bool Walk(SetComprehension node) { + node.Walk(new ComprehensionWalker(_analysis)); + return false; + } + public override bool Walk(DictionaryComprehension node) { + node.Walk(new ComprehensionWalker(_analysis)); + return false; + } + + public override bool Walk(GeneratorExpression node) { + node.Walk(new ComprehensionWalker(_analysis)); + return false; + } + + public override bool Walk(NameExpression node) { + if (_localNames?.Contains(node.Name) == true) { + return false; + } + if (_localNameNodes?.Contains(node) == true) { + return false; + } + var m = _analysis.ExpressionEvaluator.LookupNameInScopes(node.Name, out var scope); + if (m == null) { + _analysis.ReportUndefinedVariable(node); + } + // Take into account where variable is defined so we do detect + // undefined x in + // y = x + // x = 1 + var v = scope?.Variables[node.Name]; + if (v != null && v.Location.DocumentUri == _analysis.Document.Uri) { + // Do not complain about functions and classes that appear later in the file + if (!(v.Value is IPythonFunctionType || v.Value is IPythonClassType)) { + var span = v.Locations.First().Span; + var nodeLoc = node.GetLocation(_analysis.Document); + // Exclude same-name variables declared within the same statement + // like 'e' that appears before its declaration in '[e in for e in {}]' + if (span.IsAfter(nodeLoc.Span) && !IsSpanInComprehension(nodeLoc.Span)) { + _analysis.ReportUndefinedVariable(node); + } + } + } + return false; + } + + private bool IsSpanInComprehension(SourceSpan span) { + var start = span.Start.ToIndex(_analysis.Ast); + var end = span.End.ToIndex(_analysis.Ast); + return ((Node)_analysis.ExpressionEvaluator.CurrentScope.Node) + .TraverseDepthFirst(n => n.GetChildNodes()) + .OfType() + .Any(n => n.StartIndex <= start && end < n.EndIndex); + } + } +} diff --git a/src/Analysis/Ast/Impl/Linting/UndefinedVariables/LambdaWalker.cs b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/LambdaWalker.cs new file mode 100644 index 000000000..38c404ea5 --- /dev/null +++ b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/LambdaWalker.cs @@ -0,0 +1,44 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.Python.Core; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Linting.UndefinedVariables { + internal sealed class LambdaWalker : PythonWalker { + private readonly IDocumentAnalysis _analysis; + private readonly HashSet _names = new HashSet(); + private readonly HashSet _additionalNameNodes = new HashSet(); + + public LambdaWalker(IDocumentAnalysis analysis) { + _analysis = analysis; + } + + public override bool Walk(FunctionDefinition node) { + CollectNames(node); + node.Body?.Walk(new ExpressionWalker(_analysis, _names, _additionalNameNodes)); + return false; + } + + private void CollectNames(FunctionDefinition fd) { + var nc = new NameCollectorWalker(_names, _additionalNameNodes); + foreach (var nex in fd.Parameters.Select(p => p.NameExpression).ExcludeDefault()) { + nex.Walk(nc); + } + } + } +} diff --git a/src/Analysis/Ast/Impl/Linting/UndefinedVariables/NameCollectorWalker.cs b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/NameCollectorWalker.cs new file mode 100644 index 000000000..01fc065c8 --- /dev/null +++ b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/NameCollectorWalker.cs @@ -0,0 +1,37 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Linting.UndefinedVariables { + internal sealed class NameCollectorWalker : PythonWalker { + private readonly HashSet _names; + private readonly HashSet _additionalNameNodes; + + public NameCollectorWalker(HashSet names, HashSet additionalNameNodes) { + _names = names; + _additionalNameNodes = additionalNameNodes; + } + + public override bool Walk(NameExpression nex) { + if (!string.IsNullOrEmpty(nex.Name)) { + _names.Add(nex.Name); + _additionalNameNodes.Add(nex); + } + return false; + } + } +} diff --git a/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesLinter.cs b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesLinter.cs new file mode 100644 index 000000000..ff02b2d34 --- /dev/null +++ b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesLinter.cs @@ -0,0 +1,25 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using Microsoft.Python.Core; + +namespace Microsoft.Python.Analysis.Linting.UndefinedVariables { + internal sealed class UndefinedVariablesLinter : ILinter { + public void Lint(IDocumentAnalysis analysis, IServiceContainer services) { + var w = new UndefinedVariablesWalker(analysis, services); + analysis.Ast.Walk(w); + } + } +} diff --git a/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesWalker.cs b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesWalker.cs new file mode 100644 index 000000000..c57ad7b56 --- /dev/null +++ b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesWalker.cs @@ -0,0 +1,68 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Core; +using Microsoft.Python.Parsing.Ast; + +namespace Microsoft.Python.Analysis.Linting.UndefinedVariables { + internal sealed class UndefinedVariablesWalker : LinterWalker { + public UndefinedVariablesWalker(IDocumentAnalysis analysis, IServiceContainer services) + : base(analysis, services) { } + + public override bool Walk(AssignmentStatement node) { + if (node.Right is ErrorExpression) { + return false; + } + node.Right?.Walk(new ExpressionWalker(Analysis)); + return false; + } + + public override bool Walk(CallExpression node) { + node.Target?.Walk(new ExpressionWalker(Analysis)); + foreach (var arg in node.Args) { + arg?.Expression?.Walk(new ExpressionWalker(Analysis)); + } + return false; + } + + public override bool Walk(IfStatement node) { + foreach (var test in node.Tests) { + test.Test.Walk(new ExpressionWalker(Analysis)); + } + return true; + } + + public override bool Walk(GlobalStatement node) { + foreach (var nex in node.Names) { + var m = Eval.LookupNameInScopes(nex.Name, out _, LookupOptions.Global); + if (m == null) { + Analysis.ReportUndefinedVariable(nex); + } + } + return false; + } + + public override bool Walk(NonlocalStatement node) { + foreach (var nex in node.Names) { + var m = Eval.LookupNameInScopes(nex.Name, out _, LookupOptions.Nonlocal); + if (m == null) { + Analysis.ReportUndefinedVariable(nex); + } + } + return false; + } + } +} diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs index e13d613d6..29e91c87f 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs @@ -189,7 +189,12 @@ public async Task ReloadAsync(CancellationToken cancellationToken = default) { foreach (var m in Modules) { GetRdt()?.UnlockDocument(m.Value.Value.Uri); } + + // Preserve builtins, they don't need to be reloaded since interpreter does not change. + var builtins = Modules[BuiltinModuleName]; Modules.Clear(); + Modules[BuiltinModuleName] = builtins; + PathResolver = new PathResolver(_interpreter.LanguageVersion); var addedRoots = new HashSet(); diff --git a/src/Analysis/Ast/Impl/Resources.Designer.cs b/src/Analysis/Ast/Impl/Resources.Designer.cs index 895e001d2..cc3e88c40 100644 --- a/src/Analysis/Ast/Impl/Resources.Designer.cs +++ b/src/Analysis/Ast/Impl/Resources.Designer.cs @@ -663,6 +663,15 @@ internal static string SpaceWithinIndexBracketsShort { } } + /// + /// Looks up a localized string similar to Undefined variable: '{0}'. + /// + internal static string UndefinedVariable { + get { + return ResourceManager.GetString("UndefinedVariable", resourceCulture); + } + } + /// /// Looks up a localized string similar to If checked, comments are wrapped to the specified width. If unchecked, comments are not modified.. /// diff --git a/src/Analysis/Ast/Impl/Resources.resx b/src/Analysis/Ast/Impl/Resources.resx index e2ed86638..74c7db316 100644 --- a/src/Analysis/Ast/Impl/Resources.resx +++ b/src/Analysis/Ast/Impl/Resources.resx @@ -345,4 +345,7 @@ Analyzing in background, {0} items left... + + Undefined variable: '{0}' + \ No newline at end of file diff --git a/src/Analysis/Ast/Impl/Types/ArgumentSet.cs b/src/Analysis/Ast/Impl/Types/ArgumentSet.cs index 65599495f..8efcd414c 100644 --- a/src/Analysis/Ast/Impl/Types/ArgumentSet.cs +++ b/src/Analysis/Ast/Impl/Types/ArgumentSet.cs @@ -66,7 +66,7 @@ public ArgumentSet(IPythonFunctionType fn, int overloadIndex, IPythonInstance in /// /// Creates set of arguments for a function call based on the call expression /// and the function signature. The result contains expressions - /// for arguments, but not actual values. on how to + /// for arguments, but not actual values. on how to /// get values for actual parameters. /// /// Function type. diff --git a/src/Analysis/Ast/Impl/Values/Definitions/IVariable.cs b/src/Analysis/Ast/Impl/Values/Definitions/IVariable.cs index b8e4976c8..b47d7f61a 100644 --- a/src/Analysis/Ast/Impl/Values/Definitions/IVariable.cs +++ b/src/Analysis/Ast/Impl/Values/Definitions/IVariable.cs @@ -13,6 +13,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System.Collections.Generic; using Microsoft.Python.Analysis.Types; namespace Microsoft.Python.Analysis.Values { @@ -24,13 +25,25 @@ public interface IVariable: ILocatedMember { /// Variable name. /// string Name { get; } + /// /// Variable source. /// VariableSource Source { get; } + /// /// Variable value. /// - IMember Value { get; set; } + IMember Value { get; } + + /// + /// Assigns value to the variable. + /// + void Assign(IMember value, LocationInfo location); + + /// + /// Provides list of all known assignment locations along the path of analysis. + /// + IReadOnlyList Locations { get; } } } diff --git a/src/Analysis/Ast/Impl/Values/GlobalScope.cs b/src/Analysis/Ast/Impl/Values/GlobalScope.cs index 80d55ec47..a3a746f04 100644 --- a/src/Analysis/Ast/Impl/Values/GlobalScope.cs +++ b/src/Analysis/Ast/Impl/Values/GlobalScope.cs @@ -14,6 +14,7 @@ // permissions and limitations under the License. using Microsoft.Python.Analysis.Types; +using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Values { internal sealed class GlobalScope: Scope, IGlobalScope { @@ -23,5 +24,6 @@ public GlobalScope(IPythonModule module): } public IPythonModule Module { get; } + public override ScopeStatement Node => Module.Analysis?.Ast; } } diff --git a/src/Analysis/Ast/Impl/Values/Scope.cs b/src/Analysis/Ast/Impl/Values/Scope.cs index b494ce642..7885421e3 100644 --- a/src/Analysis/Ast/Impl/Values/Scope.cs +++ b/src/Analysis/Ast/Impl/Values/Scope.cs @@ -38,7 +38,7 @@ public Scope(ScopeStatement node, IScope outerScope, bool visibleToChildren = tr #region IScope public string Name => Node?.Name ?? ""; - public ScopeStatement Node { get; } + public virtual ScopeStatement Node { get; } public IScope OuterScope { get; } public bool VisibleToChildren { get; } diff --git a/src/Analysis/Ast/Impl/Values/Variable.cs b/src/Analysis/Ast/Impl/Values/Variable.cs index e18beb614..1780fc9b7 100644 --- a/src/Analysis/Ast/Impl/Values/Variable.cs +++ b/src/Analysis/Ast/Impl/Values/Variable.cs @@ -13,12 +13,16 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System; +using System.Collections.Generic; using System.Diagnostics; using Microsoft.Python.Analysis.Types; namespace Microsoft.Python.Analysis.Values { [DebuggerDisplay("{DebuggerDisplay}")] internal sealed class Variable : IVariable { + private List _locations; + public Variable(string name, IMember value, VariableSource source, LocationInfo location = null) { Name = name; Value = value; @@ -28,9 +32,22 @@ public Variable(string name, IMember value, VariableSource source, LocationInfo public string Name { get; } public VariableSource Source { get; } - public IMember Value { get; set; } + public IMember Value { get; private set; } public LocationInfo Location { get; } public PythonMemberType MemberType => PythonMemberType.Variable; + public IReadOnlyList Locations => _locations as IReadOnlyList ?? new[] { Location }; + + public void Assign(IMember value, LocationInfo location) { + if (Value == null || Value.GetPythonType().IsUnknown() || value?.GetPythonType().IsUnknown() == false) { + Value = value; + } + AddLocation(location); + } + + private void AddLocation(LocationInfo location) { + _locations = _locations ?? new List { Location }; + _locations.Add(location); + } private string DebuggerDisplay { get { diff --git a/src/Analysis/Ast/Impl/Values/VariableCollection.cs b/src/Analysis/Ast/Impl/Values/VariableCollection.cs index 7c1b48a88..73dea055f 100644 --- a/src/Analysis/Ast/Impl/Values/VariableCollection.cs +++ b/src/Analysis/Ast/Impl/Values/VariableCollection.cs @@ -20,13 +20,12 @@ using System.Diagnostics; using System.Linq; using Microsoft.Python.Analysis.Types; -using Microsoft.Python.Core.Diagnostics; namespace Microsoft.Python.Analysis.Values { [DebuggerDisplay("Count: {Count}")] internal sealed class VariableCollection : IVariableCollection { public static readonly IVariableCollection Empty = new VariableCollection(); - private readonly ConcurrentDictionary _variables = new ConcurrentDictionary(); + private readonly ConcurrentDictionary _variables = new ConcurrentDictionary(); #region ICollection public int Count => _variables.Count; @@ -37,8 +36,16 @@ internal sealed class VariableCollection : IVariableCollection { #region IVariableCollection public IVariable this[string name] => _variables.TryGetValue(name, out var v) ? v : null; public bool Contains(string name) => _variables.ContainsKey(name); - public bool TryGetVariable(string key, out IVariable value) => _variables.TryGetValue(key, out value); public IReadOnlyList Names => _variables.Keys.ToArray(); + + public bool TryGetVariable(string key, out IVariable value) { + value = null; + if (_variables.TryGetValue(key, out var v)) { + value = v; + return true; + } + return false; + } #endregion #region IMemberContainer @@ -48,7 +55,14 @@ internal sealed class VariableCollection : IVariableCollection { internal void DeclareVariable(string name, IMember value, VariableSource source, LocationInfo location) { name = !string.IsNullOrWhiteSpace(name) ? name : throw new ArgumentException(nameof(name)); - _variables[name] = new Variable(name, value, source, location); + if (_variables.TryGetValue(name, out var existing)) { + existing.Assign(value, location); + } else { + _variables[name] = new Variable(name, value, source, location); + } } + + internal void DeclareVariable(Variable variable) =>_variables[variable.Name] = variable; + internal void RemoveVariable(string name) => _variables.TryRemove(name, out _); } } diff --git a/src/Analysis/Ast/Test/AnalysisTestBase.cs b/src/Analysis/Ast/Test/AnalysisTestBase.cs index 22e055093..261e2744a 100644 --- a/src/Analysis/Ast/Test/AnalysisTestBase.cs +++ b/src/Analysis/Ast/Test/AnalysisTestBase.cs @@ -14,16 +14,13 @@ // permissions and limitations under the License. using System; -using System.Collections.Generic; using System.Diagnostics; using System.IO; -using System.Linq; using System.Threading; using System.Threading.Tasks; using FluentAssertions; using Microsoft.Python.Analysis.Analyzer; using Microsoft.Python.Analysis.Core.Interpreter; -using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Modules; @@ -32,7 +29,6 @@ using Microsoft.Python.Core.IO; using Microsoft.Python.Core.OS; using Microsoft.Python.Core.Services; -using Microsoft.Python.Core.Shell; using Microsoft.Python.Core.Tests; using Microsoft.Python.Parsing; using Microsoft.Python.Parsing.Tests; diff --git a/src/Analysis/Ast/Test/AssignmentTests.cs b/src/Analysis/Ast/Test/AssignmentTests.cs index 14d7b6153..439d6dea5 100644 --- a/src/Analysis/Ast/Test/AssignmentTests.cs +++ b/src/Analysis/Ast/Test/AssignmentTests.cs @@ -155,6 +155,15 @@ public async Task Tuple() { .And.HaveVariable("z").OfType(BuiltinTypeId.Float); } + [TestMethod, Priority(0)] + public async Task TupleUnknownReturn() { + const string code = @" +x, y, z = func() +"; + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X); + analysis.Should().HaveVariable("x").And.HaveVariable("y").And.HaveVariable("z"); + } + [TestMethod, Priority(0)] public async Task AnnotatedAssign() { const string code = @" @@ -205,7 +214,6 @@ def __init__(self): } [TestMethod, Priority(0)] - [Ignore] public async Task LambdaExpression1() { const string code = @" x = lambda a: a @@ -216,7 +224,6 @@ public async Task LambdaExpression1() { } [TestMethod, Priority(0)] - [Ignore] public async Task LambdaExpression2() { const string code = @" def f(a): @@ -280,5 +287,15 @@ public async Task StrIndex() { var analysis = await GetAnalysisAsync(code); analysis.Should().HaveVariable("x").OfType(BuiltinTypeId.Str); } + + [TestMethod, Priority(0)] + public async Task IncompleteTuple() { + const string code = @" +a, b = 1 +"; + var analysis = await GetAnalysisAsync(code); + analysis.Should().HaveVariable("a").OfType(BuiltinTypeId.Int) + .And.HaveVariable("b").OfType(BuiltinTypeId.Int); + } } } diff --git a/src/Analysis/Ast/Test/CollectionsTests.cs b/src/Analysis/Ast/Test/CollectionsTests.cs index 8857819f6..be9f31a1c 100644 --- a/src/Analysis/Ast/Test/CollectionsTests.cs +++ b/src/Analysis/Ast/Test/CollectionsTests.cs @@ -317,6 +317,29 @@ print some_bool .And.HaveVariable("some_bool").OfType(BuiltinTypeId.Bool); } + [TestMethod, Priority(0)] + public async Task ForSequenceWithList1() { + const string code = @" +x = [('abc', 42, True), ('abc', 23, False)] +for a, (b, c) in x: + pass +"; + var analysis = await GetAnalysisAsync(code); + analysis.Should().HaveVariable("a").OfType(BuiltinTypeId.Str) + .And.HaveVariable("b").OfType(BuiltinTypeId.Int) + .And.HaveVariable("c").OfType(BuiltinTypeId.Bool); + } + + [TestMethod, Priority(0)] + public async Task ForSequenceWithList2() { + const string code = @" +for a, (b, c) in x: + pass +"; + var analysis = await GetAnalysisAsync(code); + analysis.Should().HaveVariable("a").And.HaveVariable("b").And.HaveVariable("c"); + } + [TestMethod, Priority(0)] public async Task Generator2X() { const string code = @" @@ -500,5 +523,19 @@ def g(a, b): analysis.Should().HaveVariable("y1").OfType(BuiltinTypeId.Int) .And.HaveVariable("y2").OfType(BuiltinTypeId.Tuple); } + + [TestMethod, Priority(0)] + [Ignore] + public async Task NamedTuple() { + const string code = @" +from collections import namedtuple +nt = namedtuple('Point', ['x', 'y']) +pt = nt(1, 2) +"; + var analysis = await GetAnalysisAsync(code); + var nt = analysis.Should().HaveVariable("nt").Which; + nt.Should().HaveType(BuiltinTypeId.Tuple); + nt.Should().HaveMembers("x", "y"); + } } } diff --git a/src/Analysis/Ast/Test/ComprehensionTests.cs b/src/Analysis/Ast/Test/ComprehensionTests.cs new file mode 100644 index 000000000..bab03e13f --- /dev/null +++ b/src/Analysis/Ast/Test/ComprehensionTests.cs @@ -0,0 +1,82 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Python.Analysis.Tests.FluentAssertions; +using Microsoft.Python.Analysis.Types; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; + +namespace Microsoft.Python.Analysis.Tests { + [TestClass] + public class ComprehensionTests : AnalysisTestBase { + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() + => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + + [TestCleanup] + public void Cleanup() => TestEnvironmentImpl.TestCleanup(); + + [TestMethod, Priority(0)] + public async Task ListComprehension() { + const string code = @" +x = [e for e in {1, 2, 3}] +y = x[0] +"; + var analysis = await GetAnalysisAsync(code); + analysis.Should().HaveVariable("x").OfType(BuiltinTypeId.List) + .And.HaveVariable("y").OfType(BuiltinTypeId.Int) + .And.NotHaveVariable("e"); + } + + [TestMethod, Priority(0)] + public async Task ListComprehensionExpression() { + const string code = @" +x = [e > 0 for e in {1, 2, 3}] +y = x[0] +"; + var analysis = await GetAnalysisAsync(code); + analysis.Should().HaveVariable("y").OfType(BuiltinTypeId.Bool) + .And.NotHaveVariable("e"); + } + + [TestMethod, Priority(0)] + public async Task DictionaryComprehension() { + const string code = @" +x = {str(k): e > 0 for e in {1, 2, 3}} + +keys = x.keys() +k = keys[0] +values = x.values() +v = values[0] +"; + var analysis = await GetAnalysisAsync(code); + analysis.Should().HaveVariable("x").OfType(BuiltinTypeId.Dict) + .And.HaveVariable("k").OfType(BuiltinTypeId.Str) + .And.HaveVariable("v").OfType(BuiltinTypeId.Bool) + .And.NotHaveVariable("e"); + } + + [TestMethod, Priority(0)] + public async Task ListComprehensionStatement() { + const string code = @"[e > 0 for e in {1, 2, 3}]"; + var analysis = await GetAnalysisAsync(code); + analysis.Should().HaveVariable("e").OfType(BuiltinTypeId.Int); + } + } +} diff --git a/src/Analysis/Ast/Test/LintUndefinedVarsTests.cs b/src/Analysis/Ast/Test/LintUndefinedVarsTests.cs new file mode 100644 index 000000000..09377f449 --- /dev/null +++ b/src/Analysis/Ast/Test/LintUndefinedVarsTests.cs @@ -0,0 +1,403 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Linq; +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Python.Analysis.Tests.FluentAssertions; +using Microsoft.Python.Parsing.Tests; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; +using ErrorCodes = Microsoft.Python.Analysis.Diagnostics.ErrorCodes; + +namespace Microsoft.Python.Analysis.Tests { + [TestClass] + public class LintUndefinedVarsTests : AnalysisTestBase { + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() + => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + + [TestCleanup] + public void Cleanup() => TestEnvironmentImpl.TestCleanup(); + + [TestMethod, Priority(0)] + public async Task BasicVariables() { + const string code = @" +y = x + +class A: + x1: int = 0 + y1: int +"; + var analysis = await GetAnalysisAsync(code); + var d = analysis.Diagnostics.ToArray(); + d.Should().HaveCount(1); + d[0].ErrorCode.Should().Be(ErrorCodes.UndefinedVariable); + d[0].SourceSpan.Should().Be(2, 5, 2, 6); + } + + [TestMethod, Priority(0)] + public async Task ClassVariables() { + const string code = @" +class A: + x1: int = 0 + y1: int +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public async Task Conditionals() { + const string code = @" +z = 3 +if x > 2 and y == 3 or z < 2: + pass +"; + var analysis = await GetAnalysisAsync(code); + var d = analysis.Diagnostics.ToArray(); + d.Should().HaveCount(2); + d[0].ErrorCode.Should().Be(ErrorCodes.UndefinedVariable); + d[0].SourceSpan.Should().Be(3, 4, 3, 5); + d[1].ErrorCode.Should().Be(ErrorCodes.UndefinedVariable); + d[1].SourceSpan.Should().Be(3, 14, 3, 15); + } + + [TestMethod, Priority(0)] + public async Task Calls() { + const string code = @" +z = 3 +func(x, 1, y+1, z) +"; + var analysis = await GetAnalysisAsync(code); + var d = analysis.Diagnostics.ToArray(); + d.Should().HaveCount(3); + d[0].ErrorCode.Should().Be(ErrorCodes.UndefinedVariable); + d[0].SourceSpan.Should().Be(3, 1, 3, 5); + d[1].ErrorCode.Should().Be(ErrorCodes.UndefinedVariable); + d[1].SourceSpan.Should().Be(3, 6, 3, 7); + d[2].ErrorCode.Should().Be(ErrorCodes.UndefinedVariable); + d[2].SourceSpan.Should().Be(3, 12, 3, 13); + } + + [TestMethod, Priority(0)] + public async Task TupleAssignment() { + const string code = @" +a, *b, c = range(5) +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public async Task TupleUsage() { + const string code = @" +a, b = 1 +x = a +y = b +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public async Task FunctionNoneArgument() { + const string code = @" +def func(a=None, b=True): + x = a + y = b +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public async Task FunctionReference() { + const string code = @" +def func1(): + func2() + return + +def func2(a=None, b=True): ... +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public async Task ClassReference() { + const string code = @" +class A(B): + def __init__(self): + x = B() + return + +class B(): ... +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public async Task ForVariables() { + const string code = @" +c = {} +for i in c: + x = i +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public async Task ForVariablesCondition() { + const string code = @" +c = {} +for a, b in c if a < 0: + x = b +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public async Task ForVariablesList() { + const string code = @" +for i, (j, k) in {}: + x = j + y = k +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public async Task ForExpression() { + const string code = @" +def func1(a): + return a + +def func2(a, b): + return a + b + +func1(func2(a) for a, b in {} if a < 0) +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().BeEmpty(); + } + + + [TestMethod, Priority(0)] + public async Task ListComprehension() { + const string code = @" +NAME = ' '.join(str(x) for x in {z, 2, 3}) + +class C: + EVENTS = ['x'] + x = [(e, e) for e in EVENTS] + y = EVENTS +"; + var analysis = await GetAnalysisAsync(code); + var d = analysis.Diagnostics.ToArray(); + d.Should().HaveCount(1); + d[0].ErrorCode.Should().Be(ErrorCodes.UndefinedVariable); + d[0].SourceSpan.Should().Be(2, 34, 2, 35); + } + + [TestMethod, Priority(0)] + public async Task SelfAssignment() { + const string code = @" +def foo(m): + m = m +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().BeEmpty(); + } + + + [TestMethod, Priority(0)] + public async Task AssignmentBefore() { + const string code = @" +x = 1 +y = x +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public async Task AssignmentBeforeAndAfter() { + const string code = @" +x = 1 +y = x +x = 's' +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public async Task AssignmentAfter() { + const string code = @" +y = x +x = 1 +"; + var analysis = await GetAnalysisAsync(code); + var d = analysis.Diagnostics.ToArray(); + d.Should().HaveCount(1); + d[0].ErrorCode.Should().Be(ErrorCodes.UndefinedVariable); + d[0].SourceSpan.Should().Be(2, 5, 2, 6); + } + + [TestMethod, Priority(0)] + public async Task FunctionArguments() { + const string code = @" +def z(x): + return x + +def func(a, b, c): + a = b + x = c + z(c * 3) +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public async Task NonLocal() { + const string code = @" +class A: + x: int + def func(): + nonlocal x, y + y = 2 +"; + var analysis = await GetAnalysisAsync(code); + var d = analysis.Diagnostics.ToArray(); + d.Should().HaveCount(1); + d[0].ErrorCode.Should().Be(ErrorCodes.UndefinedVariable); + d[0].SourceSpan.Should().Be(5, 21, 5, 22); + } + + [TestMethod, Priority(0)] + public async Task Global() { + const string code = @" +x = 1 + +class A: + def func(): + global x, y + y = 2 +"; + var analysis = await GetAnalysisAsync(code); + var d = analysis.Diagnostics.ToArray(); + d.Should().HaveCount(1); + d[0].ErrorCode.Should().Be(ErrorCodes.UndefinedVariable); + d[0].SourceSpan.Should().Be(6, 19, 6, 20); + } + + [DataRow(false)] + [DataRow(true)] + [DataTestMethod, Priority(0)] + public async Task OptionsSwitch(bool enabled) { + const string code = @"x = y"; + + var sm = CreateServiceManager(); + var op = new AnalysisOptionsProvider(); + sm.AddService(op); + + op.Options.LintingEnabled = enabled; + var analysis = await GetAnalysisAsync(code, PythonVersions.LatestAvailable3X, sm); + analysis.Diagnostics.Should().HaveCount(enabled ? 1 : 0); + } + + [TestMethod, Priority(0)] + public async Task Builtins() { + const string code = @" +print(1) +abs(3) +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public async Task Enumeration() { + const string code = @" +x = {} +for a, b in enumerate(x): + if a: + pass + if b: + pass +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public async Task ReassignInLoop() { + const string code = @" +x = {} +for a, b in enumerate(x): + y = a + a = b + b = 1 +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public async Task ListComprehensionStatement() { + const string code = @" +[a == 1 for a in {}] +x = a +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public async Task DictionaryComprehension() { + const string code = @" +b = {str(a): a == 1 for a in {}} +x = a +"; + var analysis = await GetAnalysisAsync(code); + var d = analysis.Diagnostics.ToArray(); + d.Should().HaveCount(1); + d[0].ErrorCode.Should().Be(ErrorCodes.UndefinedVariable); + d[0].SourceSpan.Should().Be(3, 5, 3, 6); + } + + [TestMethod, Priority(0)] + public async Task Lambda() { + const string code = @" +x = lambda a: a +x(1) +"; + var analysis = await GetAnalysisAsync(code); + analysis.Diagnostics.Should().BeEmpty(); + } + + private class AnalysisOptionsProvider : IAnalysisOptionsProvider { + public AnalysisOptions Options { get; } = new AnalysisOptions(); + } + } +} diff --git a/src/Analysis/Ast/Test/LocationTests.cs b/src/Analysis/Ast/Test/LocationTests.cs new file mode 100644 index 000000000..46f8d496c --- /dev/null +++ b/src/Analysis/Ast/Test/LocationTests.cs @@ -0,0 +1,109 @@ +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Threading.Tasks; +using FluentAssertions; +using Microsoft.Python.Analysis.Tests.FluentAssertions; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; + +namespace Microsoft.Python.Analysis.Tests { + [TestClass] + public class LocationTests : AnalysisTestBase { + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() + => TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + + [TestCleanup] + public void Cleanup() => TestEnvironmentImpl.TestCleanup(); + + [TestMethod, Priority(0)] + public async Task Assignments() { + const string code = @" +a = 1 +a = 2 + +for x in y: + a = 3 + +if True: + a = 4 +elif False: + a = 5 +else: + a = 6 + +def func(): + a = 0 + +def func(a): + a = 0 + +def func(): + global a + a = 7 + +class A: + a: int +"; + var analysis = await GetAnalysisAsync(code); + var a = analysis.Should().HaveVariable("a").Which; + a.Locations.Should().HaveCount(7); + a.Locations[0].Span.Should().Be(2, 1, 2, 2); + a.Locations[1].Span.Should().Be(3, 1, 3, 2); + a.Locations[2].Span.Should().Be(6, 5, 6, 6); + a.Locations[3].Span.Should().Be(9, 5, 9, 6); + a.Locations[4].Span.Should().Be(11, 5, 11, 6); + a.Locations[5].Span.Should().Be(13, 5, 13, 6); + a.Locations[6].Span.Should().Be(23, 5, 23, 6); + } + + [TestMethod, Priority(0)] + public async Task NonLocal() { + const string code = @" +def outer(): + b = 1 + def inner(): + nonlocal b + b = 2 +"; + var analysis = await GetAnalysisAsync(code); + var outer = analysis.Should().HaveFunction("outer").Which; + var b = outer.Should().HaveVariable("b").Which; + b.Locations.Should().HaveCount(2); + b.Locations[0].Span.Should().Be(3, 5, 3, 6); + b.Locations[1].Span.Should().Be(6, 9, 6, 10); + } + + [TestMethod, Priority(0)] + public async Task FunctionParameter() { + const string code = @" +def func(a): + a = 1 + if True: + a = 2 +"; + var analysis = await GetAnalysisAsync(code); + var outer = analysis.Should().HaveFunction("func").Which; + var a = outer.Should().HaveVariable("a").Which; + a.Locations.Should().HaveCount(3); + a.Locations[0].Span.Should().Be(2, 10, 2, 11); + a.Locations[1].Span.Should().Be(3, 5, 3, 6); + a.Locations[2].Span.Should().Be(5, 9, 5, 10); + } + } +} diff --git a/src/Analysis/Ast/Test/ScrapeTests.cs b/src/Analysis/Ast/Test/ScrapeTests.cs index a0f32342a..1bc648709 100644 --- a/src/Analysis/Ast/Test/ScrapeTests.cs +++ b/src/Analysis/Ast/Test/ScrapeTests.cs @@ -217,8 +217,9 @@ public async Task FullStdLibV27() { await FullStdLibTest(v); } - [TestMethod, TestCategory("60s"), Priority(1)] - [Timeout(10 * 60 * 1000)] + [TestMethod, Priority(1)] + [Timeout(10 * 180 * 1000)] + [Ignore] public async Task FullStdLibAnaconda3() { var v = PythonVersions.Anaconda36_x64 ?? PythonVersions.Anaconda36; await FullStdLibTest(v, @@ -229,8 +230,9 @@ await FullStdLibTest(v, ); } - [TestMethod, TestCategory("60s"), Priority(1)] - [Timeout(10 * 60 * 1000)] + [TestMethod, Priority(1)] + [Timeout(10 * 180 * 1000)] + [Ignore] public async Task FullStdLibAnaconda2() { var v = PythonVersions.Anaconda27_x64 ?? PythonVersions.Anaconda27; await FullStdLibTest(v, diff --git a/src/LanguageServer/Impl/Definitions/ServerSettings.cs b/src/LanguageServer/Impl/Definitions/ServerSettings.cs index 7bf28c3ac..32428d0bf 100644 --- a/src/LanguageServer/Impl/Definitions/ServerSettings.cs +++ b/src/LanguageServer/Impl/Definitions/ServerSettings.cs @@ -37,7 +37,6 @@ public class PythonCompletionOptions { public bool showAdvancedMembers = true; public bool addBrackets = false; } - public readonly PythonCompletionOptions completion = new PythonCompletionOptions(); } } diff --git a/src/LanguageServer/Impl/LanguageServer.cs b/src/LanguageServer/Impl/LanguageServer.cs index a1d38bccd..2bdf287f2 100644 --- a/src/LanguageServer/Impl/LanguageServer.cs +++ b/src/LanguageServer/Impl/LanguageServer.cs @@ -45,6 +45,7 @@ public sealed partial class LanguageServer : IDisposable { private readonly CancellationTokenSource _sessionTokenSource = new CancellationTokenSource(); private readonly Prioritizer _prioritizer = new Prioritizer(); private readonly CancellationTokenSource _shutdownCts = new CancellationTokenSource(); + private readonly AnalysisOptionsProvider _optionsProvider = new AnalysisOptionsProvider(); private IServiceContainer _services; private Server _server; @@ -78,6 +79,7 @@ public CancellationToken Start(IServiceManager services, JsonRpc rpc) { .Add(() => _pathsWatcher?.Dispose()) .Add(() => _rpc.TraceSource.Listeners.Remove(rpcTraceListener)); + services.AddService(_optionsProvider); return _sessionTokenSource.Token; } @@ -110,6 +112,9 @@ public async Task DidChangeConfiguration(JToken token, CancellationToken cancell settings.symbolsHierarchyDepthLimit = GetSetting(analysis, "symbolsHierarchyDepthLimit", 10); settings.symbolsHierarchyMaxSymbols = GetSetting(analysis, "symbolsHierarchyMaxSymbols", 1000); + var linting = pythonSection["linting"]; + _optionsProvider.Options.LintingEnabled = GetSetting(linting, "enabled", true); + _logger.LogLevel = GetLogLevel(analysis).ToTraceEventType(); HandlePathWatchChange(token, cancellationToken); @@ -377,8 +382,6 @@ private void HandlePathWatchChange(JToken section, CancellationToken cancellatio if (!_watchSearchPaths || (_watchSearchPaths && _searchPaths.SetEquals(_initParams.initializationOptions.searchPaths))) { // Were not watching OR were watching but paths have changed. Recreate the watcher. _pathsWatcher?.Dispose(); - var interpreter = _services.GetService(); - var logger = _services.GetService(); _pathsWatcher = new PathsWatcher( _initParams.initializationOptions.searchPaths, () =>_server.NotifyPackagesChanged(cancellationToken), @@ -468,5 +471,9 @@ public PrioritizerDisposable(CancellationToken cancellationToken) { public void Dispose() => _ppc.Dispose(); } + + private class AnalysisOptionsProvider : IAnalysisOptionsProvider { + public AnalysisOptions Options { get; } = new AnalysisOptions(); + } } } diff --git a/src/LanguageServer/Test/CompletionTests.cs b/src/LanguageServer/Test/CompletionTests.cs index f0c0770d6..841b48d80 100644 --- a/src/LanguageServer/Test/CompletionTests.cs +++ b/src/LanguageServer/Test/CompletionTests.cs @@ -1073,5 +1073,16 @@ def main(req: func.HttpRequest) -> func.HttpResponse: result.Should().HaveLabels("get"); result.Completions.First(x => x.label == "get").Should().HaveDocumentation("dict.get*"); } + + [TestMethod, Priority(0)] + public async Task InForEnumeration() { + var analysis = await GetAnalysisAsync(@" +for a, b in x: + +"); + var cs = new CompletionSource(new PlainTextDocumentationSource(), ServerSettings.completion); + var result = cs.GetCompletions(analysis, new SourceLocation(3, 4)); + result.Should().HaveLabels("a", "b"); + } } } diff --git a/src/LanguageServer/Test/GoToDefinitionTests.cs b/src/LanguageServer/Test/GoToDefinitionTests.cs index 299ad4651..25f02508c 100644 --- a/src/LanguageServer/Test/GoToDefinitionTests.cs +++ b/src/LanguageServer/Test/GoToDefinitionTests.cs @@ -82,7 +82,7 @@ def func(a, b): reference.range.Should().Be(11, 0, 14, 12); reference = ds.FindDefinition(analysis, new SourceLocation(18, 1)); - reference.range.Should().Be(17, 0, 17, 1); // TODO: store all locations + reference.range.Should().Be(3, 0, 3, 1); // TODO: store all locations reference = ds.FindDefinition(analysis, new SourceLocation(19, 5)); reference.range.Should().Be(5, 6, 9, 18);