Skip to content
This repository was archived by the owner on Nov 4, 2024. It is now read-only.

Commit 12d3928

Browse files
jakebaileyjakebailey
authored
Rework search path resolution (microsoft#1289)
* first working path classfier, breaks existing path code for now * inherit Comparer instead of implementing both * don't do path depth comparison, preserve user ordering as before * switch to using new path classification code in main module resolution * fix typo * clean up comment about paths in Server.cs * use String.Split instead of regex when parsing script output * test ordering of user provided paths * add new normalize and trim helper, check preconditions as debug asserts rather than commenting on them * use Split extension, don't MaybeEnumerate paths
1 parent 4310371 commit 12d3928

File tree

7 files changed

+389
-79
lines changed

7 files changed

+389
-79
lines changed

‎src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs

Lines changed: 9 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@ namespace Microsoft.Python.Analysis.Modules.Resolution {
3737
internal sealed class MainModuleResolution : ModuleResolutionBase, IModuleManagement {
3838
private readonly ConcurrentDictionary<string, IPythonModule> _specialized = new ConcurrentDictionary<string, IPythonModule>();
3939
private IRunningDocumentTable _rdt;
40-
private IReadOnlyList<string> _searchPaths;
4140

4241
public MainModuleResolution(string root, IServiceContainer services)
4342
: base(root, services) { }
@@ -62,29 +61,6 @@ internal async Task InitializeAsync(CancellationToken cancellationToken = defaul
6261
cancellationToken.ThrowIfCancellationRequested();
6362
}
6463

65-
public async Task<IReadOnlyList<string>> GetSearchPathsAsync(CancellationToken cancellationToken = default) {
66-
if (_searchPaths != null) {
67-
return _searchPaths;
68-
}
69-
70-
_searchPaths = await GetInterpreterSearchPathsAsync(cancellationToken);
71-
Debug.Assert(_searchPaths != null, "Should have search paths");
72-
_searchPaths = _searchPaths ?? Array.Empty<string>();
73-
74-
_log?.Log(TraceEventType.Verbose, "Python search paths:");
75-
foreach (var s in _searchPaths) {
76-
_log?.Log(TraceEventType.Verbose, $" {s}");
77-
}
78-
79-
var configurationSearchPaths = Configuration.SearchPaths ?? Array.Empty<string>();
80-
81-
_log?.Log(TraceEventType.Verbose, "Configuration search paths:");
82-
foreach (var s in configurationSearchPaths) {
83-
_log?.Log(TraceEventType.Verbose, $" {s}");
84-
}
85-
return _searchPaths;
86-
}
87-
8864
protected override IPythonModule CreateModule(string name) {
8965
var moduleImport = CurrentPathResolver.GetModuleImportFromModuleName(name);
9066
if (moduleImport == null) {
@@ -135,11 +111,11 @@ protected override IPythonModule CreateModule(string name) {
135111
return GetRdt().AddModule(mco);
136112
}
137113

138-
private async Task<IReadOnlyList<string>> GetInterpreterSearchPathsAsync(CancellationToken cancellationToken = default) {
114+
private async Task<IReadOnlyList<PythonLibraryPath>> GetInterpreterSearchPathsAsync(CancellationToken cancellationToken = default) {
139115
if (!_fs.FileExists(Configuration.InterpreterPath)) {
140116
_log?.Log(TraceEventType.Warning, "Interpreter does not exist:", Configuration.InterpreterPath);
141117
_ui?.ShowMessageAsync(Resources.InterpreterNotFound, TraceEventType.Error);
142-
return Array.Empty<string>();
118+
return Array.Empty<PythonLibraryPath>();
143119
}
144120

145121
_log?.Log(TraceEventType.Information, "GetCurrentSearchPaths", Configuration.InterpreterPath);
@@ -148,11 +124,11 @@ private async Task<IReadOnlyList<string>> GetInterpreterSearchPathsAsync(Cancell
148124
var ps = _services.GetService<IProcessServices>();
149125
var paths = await PythonLibraryPath.GetSearchPathsAsync(Configuration, fs, ps, cancellationToken);
150126
cancellationToken.ThrowIfCancellationRequested();
151-
return paths.MaybeEnumerate().Select(p => p.Path).ToArray();
127+
return paths.ToArray();
152128
} catch (InvalidOperationException ex) {
153129
_log?.Log(TraceEventType.Warning, "Exception getting search paths", ex);
154130
_ui?.ShowMessageAsync(Resources.ExceptionGettingSearchPaths, TraceEventType.Error);
155-
return Array.Empty<string>();
131+
return Array.Empty<PythonLibraryPath>();
156132
}
157133
}
158134

@@ -211,16 +187,13 @@ public async Task ReloadAsync(CancellationToken cancellationToken = default) {
211187
var addedRoots = new HashSet<string>();
212188
addedRoots.UnionWith(PathResolver.SetRoot(Root));
213189

214-
InterpreterPaths = await GetSearchPathsAsync(cancellationToken);
190+
var ps = _services.GetService<IProcessServices>();
215191

216-
IEnumerable<string> userSearchPaths = Configuration.SearchPaths;
217-
InterpreterPaths = InterpreterPaths.Except(userSearchPaths, StringExtensions.PathsStringComparer).Where(p => !p.PathEquals(Root));
192+
var paths = await GetInterpreterSearchPathsAsync(cancellationToken);
193+
var (interpreterPaths, userPaths) = PythonLibraryPath.ClassifyPaths(Root, _fs, paths, Configuration.SearchPaths);
218194

219-
if (Root != null) {
220-
var underRoot = userSearchPaths.ToLookup(p => _fs.IsPathUnderRoot(Root, p));
221-
userSearchPaths = underRoot[true];
222-
InterpreterPaths = underRoot[false].Concat(InterpreterPaths);
223-
}
195+
InterpreterPaths = interpreterPaths.Select(p => p.Path);
196+
var userSearchPaths = userPaths.Select(p => p.Path);
224197

225198
_log?.Log(TraceEventType.Information, "Interpreter search paths:");
226199
foreach (var s in InterpreterPaths) {

‎src/Analysis/Ast/Impl/get_search_paths.py

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,11 @@ def clean(path):
4545
BEFORE_SITE = set(clean(p) for p in BEFORE_SITE)
4646
AFTER_SITE = set(clean(p) for p in AFTER_SITE)
4747

48+
try:
49+
SITE_PKGS = set(clean(p) for p in site.getsitepackages())
50+
except AttributeError:
51+
SITE_PKGS = set()
52+
4853
for prefix in [
4954
sys.prefix,
5055
sys.exec_prefix,
@@ -69,4 +74,7 @@ def clean(path):
6974
if p in BEFORE_SITE:
7075
print("%s|stdlib|" % p)
7176
elif p in AFTER_SITE:
72-
print("%s||" % p)
77+
if p in SITE_PKGS:
78+
print("%s|site|" % p)
79+
else:
80+
print("%s|pth|" % p)

‎src/Analysis/Ast/Test/PathClassificationTests.cs

Lines changed: 218 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,218 @@
1+
// Copyright(c) Microsoft Corporation
2+
// All rights reserved.
3+
//
4+
// Licensed under the Apache License, Version 2.0 (the License); you may not use
5+
// this file except in compliance with the License. You may obtain a copy of the
6+
// License at http://www.apache.org/licenses/LICENSE-2.0
7+
//
8+
// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
9+
// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
10+
// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
11+
// MERCHANTABILITY OR NON-INFRINGEMENT.
12+
//
13+
// See the Apache Version 2.0 License for specific language governing
14+
// permissions and limitations under the License.
15+
16+
using System;
17+
using System.IO;
18+
using FluentAssertions;
19+
using Microsoft.Python.Analysis.Core.Interpreter;
20+
using Microsoft.Python.Core.IO;
21+
using Microsoft.Python.Core.OS;
22+
using Microsoft.Python.Tests.Utilities.FluentAssertions;
23+
using Microsoft.VisualStudio.TestTools.UnitTesting;
24+
using TestUtilities;
25+
26+
namespace Microsoft.Python.Analysis.Tests {
27+
[TestClass]
28+
public class PathClassificationTests {
29+
private readonly FileSystem _fs = new FileSystem(new OSPlatform());
30+
31+
public TestContext TestContext { get; set; }
32+
33+
[TestInitialize]
34+
public void TestInitialize()
35+
=> TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}");
36+
37+
[TestCleanup]
38+
public void Cleanup() => TestEnvironmentImpl.TestCleanup();
39+
40+
[TestMethod]
41+
public void Plain() {
42+
var appPath = TestData.GetTestSpecificPath("app.py");
43+
var root = Path.GetDirectoryName(appPath);
44+
45+
var venv = Path.Combine(root, "venv");
46+
var venvLib = Path.Combine(venv, "Lib");
47+
var venvSitePackages = Path.Combine(venvLib, "site-packages");
48+
49+
var fromInterpreter = new[] {
50+
new PythonLibraryPath(venvLib, PythonLibraryPathType.StdLib),
51+
new PythonLibraryPath(venv, PythonLibraryPathType.StdLib),
52+
new PythonLibraryPath(venvSitePackages, PythonLibraryPathType.Site),
53+
};
54+
55+
var (interpreterPaths, userPaths) = PythonLibraryPath.ClassifyPaths(root, _fs, fromInterpreter, Array.Empty<string>());
56+
57+
interpreterPaths.Should().BeEquivalentToWithStrictOrdering(new[] {
58+
new PythonLibraryPath(venvLib, PythonLibraryPathType.StdLib),
59+
new PythonLibraryPath(venv, PythonLibraryPathType.StdLib),
60+
new PythonLibraryPath(venvSitePackages, PythonLibraryPathType.Site),
61+
});
62+
63+
userPaths.Should().BeEmpty();
64+
}
65+
66+
[TestMethod]
67+
public void WithSrcDir() {
68+
var appPath = TestData.GetTestSpecificPath("app.py");
69+
var root = Path.GetDirectoryName(appPath);
70+
71+
var venv = Path.Combine(root, "venv");
72+
var venvLib = Path.Combine(venv, "Lib");
73+
var venvSitePackages = Path.Combine(venvLib, "site-packages");
74+
75+
var src = Path.Combine(root, "src");
76+
77+
var fromInterpreter = new[] {
78+
new PythonLibraryPath(venvLib, PythonLibraryPathType.StdLib),
79+
new PythonLibraryPath(venv, PythonLibraryPathType.StdLib),
80+
new PythonLibraryPath(venvSitePackages, PythonLibraryPathType.Site),
81+
};
82+
83+
var fromUser = new[] {
84+
"./src",
85+
};
86+
87+
var (interpreterPaths, userPaths) = PythonLibraryPath.ClassifyPaths(root, _fs, fromInterpreter, fromUser);
88+
89+
interpreterPaths.Should().BeEquivalentToWithStrictOrdering(new[] {
90+
new PythonLibraryPath(venvLib, PythonLibraryPathType.StdLib),
91+
new PythonLibraryPath(venv, PythonLibraryPathType.StdLib),
92+
new PythonLibraryPath(venvSitePackages, PythonLibraryPathType.Site),
93+
});
94+
95+
userPaths.Should().BeEquivalentToWithStrictOrdering(new[] {
96+
new PythonLibraryPath(src, PythonLibraryPathType.Unspecified),
97+
});
98+
}
99+
100+
[TestMethod]
101+
public void NormalizeUser() {
102+
var appPath = TestData.GetTestSpecificPath("app.py");
103+
var root = Path.GetDirectoryName(appPath);
104+
105+
var src = Path.Combine(root, "src");
106+
107+
var fromUser = new[] {
108+
"./src/",
109+
};
110+
111+
var (interpreterPaths, userPaths) = PythonLibraryPath.ClassifyPaths(root, _fs, Array.Empty<PythonLibraryPath>(), fromUser);
112+
113+
interpreterPaths.Should().BeEmpty();
114+
115+
userPaths.Should().BeEquivalentToWithStrictOrdering(new[] {
116+
new PythonLibraryPath(src, PythonLibraryPathType.Unspecified),
117+
});
118+
}
119+
120+
[TestMethod]
121+
public void NestedUser() {
122+
var appPath = TestData.GetTestSpecificPath("app.py");
123+
var root = Path.GetDirectoryName(appPath);
124+
125+
var src = Path.Combine(root, "src");
126+
var srcSomething = Path.Combine(src, "something");
127+
var srcFoo = Path.Combine(src, "foo");
128+
var srcFooBar = Path.Combine(srcFoo, "bar");
129+
130+
var fromUser = new[] {
131+
"./src",
132+
"./src/something",
133+
"./src/foo/bar",
134+
"./src/foo",
135+
};
136+
137+
var (interpreterPaths, userPaths) = PythonLibraryPath.ClassifyPaths(root, _fs, Array.Empty<PythonLibraryPath>(), fromUser);
138+
139+
interpreterPaths.Should().BeEmpty();
140+
141+
userPaths.Should().BeEquivalentToWithStrictOrdering(new[] {
142+
new PythonLibraryPath(src, PythonLibraryPathType.Unspecified),
143+
new PythonLibraryPath(srcSomething, PythonLibraryPathType.Unspecified),
144+
new PythonLibraryPath(srcFooBar, PythonLibraryPathType.Unspecified),
145+
new PythonLibraryPath(srcFoo, PythonLibraryPathType.Unspecified),
146+
});
147+
}
148+
149+
[TestMethod]
150+
public void NestedUserOrdering() {
151+
var appPath = TestData.GetTestSpecificPath("app.py");
152+
var root = Path.GetDirectoryName(appPath);
153+
154+
var src = Path.Combine(root, "src");
155+
var srcSomething = Path.Combine(src, "something");
156+
var srcFoo = Path.Combine(src, "foo");
157+
var srcFooBar = Path.Combine(srcFoo, "bar");
158+
159+
var fromUser = new[] {
160+
"./src/foo",
161+
"./src/foo/bar",
162+
"./src",
163+
"./src/something",
164+
};
165+
166+
var (interpreterPaths, userPaths) = PythonLibraryPath.ClassifyPaths(root, _fs, Array.Empty<PythonLibraryPath>(), fromUser);
167+
168+
interpreterPaths.Should().BeEmpty();
169+
170+
userPaths.Should().BeEquivalentToWithStrictOrdering(new[] {
171+
new PythonLibraryPath(srcFoo, PythonLibraryPathType.Unspecified),
172+
new PythonLibraryPath(srcFooBar, PythonLibraryPathType.Unspecified),
173+
new PythonLibraryPath(src, PythonLibraryPathType.Unspecified),
174+
new PythonLibraryPath(srcSomething, PythonLibraryPathType.Unspecified),
175+
});
176+
}
177+
178+
[TestMethod]
179+
public void InsideStdLib() {
180+
var appPath = TestData.GetTestSpecificPath("app.py");
181+
var root = Path.GetDirectoryName(appPath);
182+
183+
var venv = Path.Combine(root, "venv");
184+
var venvLib = Path.Combine(venv, "Lib");
185+
var venvSitePackages = Path.Combine(venvLib, "site-packages");
186+
var inside = Path.Combine(venvSitePackages, "inside");
187+
var what = Path.Combine(venvSitePackages, "what");
188+
189+
var src = Path.Combine(root, "src");
190+
191+
var fromInterpreter = new[] {
192+
new PythonLibraryPath(venvLib, PythonLibraryPathType.StdLib),
193+
new PythonLibraryPath(venv, PythonLibraryPathType.StdLib),
194+
new PythonLibraryPath(venvSitePackages, PythonLibraryPathType.Site),
195+
new PythonLibraryPath(inside, PythonLibraryPathType.Pth),
196+
};
197+
198+
var fromUser = new[] {
199+
"./src",
200+
"./venv/Lib/site-packages/what",
201+
};
202+
203+
var (interpreterPaths, userPaths) = PythonLibraryPath.ClassifyPaths(root, _fs, fromInterpreter, fromUser);
204+
205+
interpreterPaths.Should().BeEquivalentToWithStrictOrdering(new[] {
206+
new PythonLibraryPath(venvLib, PythonLibraryPathType.StdLib),
207+
new PythonLibraryPath(venv, PythonLibraryPathType.StdLib),
208+
new PythonLibraryPath(what, PythonLibraryPathType.Unspecified),
209+
new PythonLibraryPath(venvSitePackages, PythonLibraryPathType.Site),
210+
new PythonLibraryPath(inside, PythonLibraryPathType.Pth),
211+
});
212+
213+
userPaths.Should().BeEquivalentToWithStrictOrdering(new[] {
214+
new PythonLibraryPath(src, PythonLibraryPathType.Unspecified),
215+
});
216+
}
217+
}
218+
}

0 commit comments

Comments
 (0)