Cherry pick fixes from Release branch into PTVS branch (#578)

* Fix #470 (#478)

* Fix various common exceptions, log on a failed directory load (#498)

* rethrow exception after telemetry instead of disposing

* log an error instead of crashing when hitting some exceptions loading files, fix nullref when shutting down without an idle tracker

* fix short circuiting of question mark checks to prevent null from being returned

* print name of exception instead of relying on ToString

* don't use MaybeEnumerate

* upgrade message to Show

* Mitigate some of #495 via MRO member selection and analysis set optimization (#517)

This isn't a complete fix, but does seem to improve #495 in some cases. Adds back the early MRO return removed in #277, so now large class hierarchies won't over-propagate types (where some of the trouble with fastai happens do to the Transform class). I also optimized AnalysisHashSet a little to do length checks when possible.

There are still times when things get caught up comparing unions for a while, but it seems to be nondeterministic.

* Two path resolution bug fixes

- Fix #509: PLS doesn't flag error in a relative import if it is found in another root directory (#519)
- Fix #510: PLS doesn't resolve relative paths correctly

* Catch exceptions when importing from search paths (#525)

* catch exceptions when importing from search paths

* retry instead of not found

Author: AlexanderSher

src/Analysis/Engine/Impl/AnalysisHashSet.cs

@@ -275,7 +275,19 @@ public bool SetEquals(IAnalysisSet other) {
                 return true;
             }
 
+            // Only access Count once to prevent wasted time in bucket locks.
+            var lCount = Count;
+            var rCount = other.Count;
+
+            if (lCount == 0 && rCount == 0) {
+                return true;
+            }
+
             if (Comparer == other.Comparer) {
+                if (lCount != rCount) {
+                    return false;
+                }
+
                 // Quick check for any unmatched hashcodes.
                 // This can conclusively prove the sets are not equal, but cannot
                 // prove equality.
@@ -290,17 +302,16 @@ public bool SetEquals(IAnalysisSet other) {
                 }
             }
 
-            var otherHc = new HashSet<AnalysisValue>(other, _comparer);
-            foreach (var key in this) {
-                if (!otherHc.Remove(key)) {
-                    return false;
-                }
-            }
-            if (otherHc.Any()) {
+            if (lCount == 0 || rCount == 0) {
                 return false;
             }
 
-            return true;
+            if (lCount == 1 && rCount == 1) {
+                return _comparer.Equals(this.First(), other.First());
+            }
+
+            var hs = new HashSet<AnalysisValue>(other, _comparer);
+            return hs.SetEquals(this);
         }
 
         /// <summary>

src/Analysis/Engine/Impl/AnalysisSet.cs

@@ -534,6 +534,8 @@ public static bool ContainsAny(this IAnalysisSet set, IAnalysisSet values) {
     sealed class ObjectComparer : IEqualityComparer<AnalysisValue>, IEqualityComparer<IAnalysisSet> {
         public static readonly ObjectComparer Instance = new ObjectComparer();
 
+        private ObjectComparer() { }
+
         public bool Equals(AnalysisValue x, AnalysisValue y) {
 #if FULL_VALIDATION
             if (x != null && y != null) {
@@ -574,7 +576,7 @@ sealed class UnionComparer : IEqualityComparer<AnalysisValue>, IEqualityComparer
 
         public readonly int Strength;
 
-        public UnionComparer(int strength = 0) {
+        private UnionComparer(int strength = 0) {
             Strength = strength;
         }
 

src/Analysis/Engine/Impl/Analyzer/DDG.cs

@@ -296,6 +296,9 @@ public override bool Walk(FromImportStatement node) {
                 case ImportNotFound notFound:
                     MakeUnresolvedImport(notFound.FullName, node.Root);
                     return false;
+                case NoKnownParentPackage _:
+                    MakeNoKnownParentPackageImport(node.Root);
+                    return false;
                 default:
                     return false;
             }
@@ -464,6 +467,10 @@ private void MakeUnresolvedImport(string name, Node spanNode) {
             ProjectState.AddDiagnostic(spanNode, _unit, ErrorMessages.UnresolvedImport(name), DiagnosticSeverity.Warning, ErrorMessages.UnresolvedImportCode);
         }
 
+        private void MakeNoKnownParentPackageImport(Node spanNode) {
+            ProjectState.AddDiagnostic(spanNode, _unit, Resources.ErrorRelativeImportNoPackage, DiagnosticSeverity.Warning, ErrorMessages.UnresolvedImportCode);
+        }
+
         private void ImportModule(in ImportStatement node, in IModule module, in DottedName moduleImportExpression, in NameExpression asNameExpression) {
             // "import fob.oar as baz" is handled as
             // baz = import_module('fob.oar')

src/Analysis/Engine/Impl/DependencyResolution/NoKnownParentPackage.cs

@@ -0,0 +1,19 @@
+// Python Tools for Visual Studio
+// Copyright(c) Microsoft Corporation
+// All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the License); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+//
+// THIS CODE IS PROVIDED ON AN  *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
+// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
+// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
+// MERCHANTABILITY OR NON-INFRINGEMENT.
+//
+// See the Apache Version 2.0 License for specific language governing
+// permissions and limitations under the License.
+
+namespace Microsoft.PythonTools.Analysis.DependencyResolution {
+    internal class NoKnownParentPackage : IImportSearchResult { }
+}

src/Analysis/Engine/Impl/DependencyResolution/PathResolverSnapshot.cs

@@ -187,16 +187,11 @@ public IImportSearchResult GetImportsFromRelativePath(in string modulePath, in i
                 return default;
             }
 
-            if (parentCount > lastEdge.PathLength) {
-                // Can't get outside of the root
-                return default;
-            }
-
             var fullNameList = relativePath.ToList();
             if (lastEdge.IsNonRooted) {
                 // Handle relative imports only for modules in the same folder
                 if (parentCount > 1) {
-                    return default;
+                    return new NoKnownParentPackage();
                 }
 
                 if (parentCount == 1 && fullNameList.Count == 1 && lastEdge.Start.TryGetChild(fullNameList[0], out var nameNode)) {
@@ -209,12 +204,27 @@ public IImportSearchResult GetImportsFromRelativePath(in string modulePath, in i
                     .ToString());
             }
 
+            var relativeInWorkDirectory = false;
+            if (parentCount > lastEdge.PathLength - 2) {
+                relativeInWorkDirectory = _workDirectory.EqualsOrdinal(lastEdge.FirstEdge.End.Name, IgnoreCaseInPaths);
+
+                // Relative path must be only inside package
+                // Exception for working directory cause it can be a root directory of the package
+                if (!relativeInWorkDirectory) {
+                    return new NoKnownParentPackage();
+                }
+            }
+
             var relativeParentEdge = lastEdge.GetPrevious(parentCount);
 
             var rootEdges = new List<Edge>();
-            for (var i = 0; i < _roots.Count; i++) {
-                if (RootContains(i, relativeParentEdge, out var rootEdge)) {
-                    rootEdges.Add(rootEdge);
+            if (relativeInWorkDirectory) {
+                rootEdges.Add(lastEdge.FirstEdge);
+            } else {
+                for (var i = 0; i < _roots.Count; i++) {
+                    if (RootContains(i, relativeParentEdge, out var rootEdge)) {
+                        rootEdges.Add(rootEdge);
+                    }
                 }
             }
 
@@ -226,7 +236,13 @@ public IImportSearchResult GetImportsFromRelativePath(in string modulePath, in i
                 return default;
             }
 
-            var fullName = GetFullModuleNameBuilder(relativeParentEdge).Append(".", fullNameList).ToString();
+            var fullNameBuilder = GetFullModuleNameBuilder(relativeParentEdge);
+            if (!relativeParentEdge.IsFirst) {
+                AppendName(fullNameBuilder, relativeParentEdge.End.Name);
+                fullNameBuilder.Append(".");
+            }
+            var fullName = fullNameBuilder.Append(".", fullNameList).ToString();
+
             return new ImportNotFound(fullName);
         }
 
@@ -250,7 +266,7 @@ private static bool TryCreateModuleImport(Edge lastEdge, out ModuleImport module
             => TryCreateModuleImport(lastEdge.FirstEdge.End, lastEdge.End, out moduleImport);
 
         private static bool TryCreateModuleImport(Node rootNode, Node moduleNode, out ModuleImport moduleImport) {
-            if (moduleNode.TryGetChild("__init__", out var initPyNode) && initPyNode.IsModule) {
+            if (IsPackageWithInitPy(moduleNode, out var initPyNode)) {
                 moduleImport = new ModuleImport(moduleNode.Name, initPyNode.FullModuleName, rootNode.Name, initPyNode.ModulePath, false);
                 return true;
             }
@@ -517,7 +533,7 @@ private static bool TryFindImport(IEnumerable<Edge> rootEdges, List<string> full
         private static bool TryFindName(in Edge edge, in IEnumerable<string> nameParts, out Edge lastEdge) {
             lastEdge = edge;
             foreach (var name in nameParts) {
-                if (lastEdge.End.IsModule) {
+                if (lastEdge.End.IsModule && !IsPackageWithInitPy(lastEdge.End, out _)) {
                     return false;
                 }
                 var index = lastEdge.End.GetChildIndex(name);
@@ -653,7 +669,7 @@ private static StringBuilder GetFullModuleNameBuilder(in Edge lastEdge) {
             while (edge != lastEdge) {
                 AppendName(sb, edge.End.Name);
                 edge = edge.Next;
-            };
+            }
 
             return sb;
         }
@@ -744,6 +760,9 @@ private static int GetModuleNameStart(string rootedModulePath)
         private static int GetModuleNameEnd(string rootedModulePath) 
             => IsPythonCompiled(rootedModulePath) ? rootedModulePath.IndexOf('.', GetModuleNameStart(rootedModulePath)) : rootedModulePath.LastIndexOf('.');
 
+        private static bool IsPackageWithInitPy(Node node, out Node initPyNode)
+            => node.TryGetChild("__init__", out initPyNode) && initPyNode.IsModule;
+
         private static bool IsNotInitPy(string name)
             => !name.EqualsOrdinal("__init__");
 

src/Analysis/Engine/Impl/Infrastructure/Extensions/StringExtensions.cs

@@ -195,6 +195,9 @@ public static bool EqualsIgnoreCase(this string s, string other)
         public static bool EqualsOrdinal(this string s, string other)
             => string.Equals(s, other, StringComparison.Ordinal);
 
+        public static bool EqualsOrdinal(this string s, string other, bool ignoreCase)
+            => string.Equals(s, other, ignoreCase ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal);
+
         public static bool EqualsOrdinal(this string s, int index, string other, int otherIndex, int length, bool ignoreCase = false)
             => string.Compare(s, index, other, otherIndex, length, ignoreCase ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal) == 0;
 

src/Analysis/Engine/Impl/Intellisense/AnalysisQueue.cs

@@ -67,7 +67,7 @@ private async Task ConsumerLoop() {
                     return;
                 } catch (Exception ex) when (!ex.IsCriticalException()) {
                     UnhandledException?.Invoke(this, new UnhandledExceptionEventArgs(ex, false));
-                    Dispose();
+                    throw;
                 }
             }
             RaiseEventOnThreadPool(AnalysisComplete);
@@ -208,7 +208,7 @@ public async Task Handler(CancellationToken cancellationToken) {
         }
 
         private sealed class QueueItemComparer : IEqualityComparer<QueueItem> {
-            public static IEqualityComparer<QueueItem> Instance { get; } = new QueueItemComparer();
+            public static readonly IEqualityComparer<QueueItem> Instance = new QueueItemComparer();
 
             private QueueItemComparer() { }
             public bool Equals(QueueItem x, QueueItem y) => Equals(x.Key, y.Key);

src/Analysis/Engine/Impl/Interpreter/Ast/AstModuleResolution.cs

@@ -205,6 +205,12 @@ public async Task<TryImportModuleResult> TryImportModuleAsync(string name, PathR
                 } catch (OperationCanceledException) {
                     _log?.Log(TraceLevel.Error, "ImportTimeout", name, "ImportFromSearchPaths");
                     return TryImportModuleResult.Timeout;
+                } catch (Exception ex) when (
+                    ex is IOException // FileNotFoundException, DirectoryNotFoundException, PathTooLongException, etc 
+                    || ex is UnauthorizedAccessException
+                ) {
+                    _log?.Log(TraceLevel.Error, "ImportException", name, "ImportFromSearchPaths", ex.GetType().Name, ex.Message);
+                    return TryImportModuleResult.NeedRetry;
                 }
             }
 

src/Analysis/Engine/Impl/LocationInfo.cs

@@ -83,9 +83,12 @@ public bool Equals(ILocationInfo other) {
         /// Provides an IEqualityComparer that compares line, column and project entries.  By
         /// default locations are equaitable based upon only line/project entry.
         /// </summary>
-        public static IEqualityComparer<ILocationInfo> FullComparer { get; } = new FullLocationComparer();
+        public static IEqualityComparer<ILocationInfo> FullComparer => FullLocationComparer.Instance;
 
         sealed class FullLocationComparer : IEqualityComparer<ILocationInfo>, IEqualityComparer<LocationInfo> {
+            public static readonly FullLocationComparer Instance = new FullLocationComparer();
+
+            private FullLocationComparer() { }
             public bool Equals(LocationInfo x, LocationInfo y) => EqualsImpl(x, y);
             public bool Equals(ILocationInfo x, ILocationInfo y) => EqualsImpl(x, y);
 

src/Analysis/Engine/Impl/OverloadResult.cs

@@ -411,8 +411,8 @@ internal ParameterResult GetParameterResultFromParameterInfo(IParameterInfo para
     }
 
     class OverloadResultComparer : EqualityComparer<OverloadResult> {
-        public static IEqualityComparer<OverloadResult> Instance = new OverloadResultComparer(false);
-        public static IEqualityComparer<OverloadResult> WeakInstance = new OverloadResultComparer(true);
+        public static readonly IEqualityComparer<OverloadResult> Instance = new OverloadResultComparer(false);
+        public static readonly IEqualityComparer<OverloadResult> WeakInstance = new OverloadResultComparer(true);
 
         private readonly bool _weak;