Skip to content
This repository was archived by the owner on Apr 14, 2022. It is now read-only.

Reenable file path watcher, watching interpreter paths #1306

Merged
merged 20 commits into from
Jul 15, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
ead5c7b
first working path classfier, breaks existing path code for now
jakebailey Jun 28, 2019
10f42be
inherit Comparer instead of implementing both
jakebailey Jun 28, 2019
5567c37
don't do path depth comparison, preserve user ordering as before
jakebailey Jul 1, 2019
98cc56f
switch to using new path classification code in main module resolution
jakebailey Jul 2, 2019
dc9259d
fix typo
jakebailey Jul 8, 2019
01339ce
clean up comment about paths in Server.cs
jakebailey Jul 8, 2019
b7f010a
use String.Split instead of regex when parsing script output
jakebailey Jul 9, 2019
78010cc
test ordering of user provided paths
jakebailey Jul 9, 2019
524ae5b
add new normalize and trim helper, check preconditions as debug asser…
jakebailey Jul 9, 2019
51b1fec
Bring back search path watcher for interpreter paths
jakebailey Jul 9, 2019
6d183be
Merge branch 'master' into fix-path-watcher
jakebailey Jul 10, 2019
8fcd0d4
make watchSearchPaths change code more obvious, remove stray semicolon
jakebailey Jul 10, 2019
e2343e5
perform check only once
jakebailey Jul 10, 2019
a437527
filter files to python-ish ones, up buffer size to maximum
jakebailey Jul 11, 2019
af303d6
avoid reanalysis of non-existent files on reload by removing all unop…
jakebailey Jul 11, 2019
f83a42b
Merge branch 'master' into fix-path-watcher
jakebailey Jul 12, 2019
70a149f
fix build
jakebailey Jul 12, 2019
6b1fc91
check parsed path instead of unparsed line
jakebailey Jul 12, 2019
1baca6a
Merge branch 'master' into fix-path-watcher
jakebailey Jul 12, 2019
b4697c5
move RDT reloading into RDT
jakebailey Jul 15, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,12 @@ public interface IRunningDocumentTable {
/// <returns>New lock count or -1 if document was not found.</returns>
int UnlockDocument(Uri uri);

/// <summary>
/// Reloads the table by removing all unopened files (which would have been loaded from disk),
/// and resetting the content of all other files to trigger reanalysis.
/// </summary>
void ReloadAll();

/// <summary>
/// Fires when document is opened.
/// </summary>
Expand Down
24 changes: 24 additions & 0 deletions src/Analysis/Ast/Impl/Documents/RunningDocumentTable.cs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
using Microsoft.Python.Analysis.Analyzer;
using Microsoft.Python.Analysis.Modules;
using Microsoft.Python.Core;
using Microsoft.Python.Core.Collections;
using Microsoft.Python.Core.Logging;

namespace Microsoft.Python.Analysis.Documents {
Expand Down Expand Up @@ -189,6 +190,29 @@ public void CloseDocument(Uri documentUri) {
}
}

public void ReloadAll() {
ImmutableArray<KeyValuePair<Uri, DocumentEntry>> opened;
ImmutableArray<KeyValuePair<Uri, DocumentEntry>> closed;

lock (_lock) {
_documentsByUri.Split(kvp => kvp.Value.Document.IsOpen, out opened, out closed);

foreach (var (uri, entry) in closed) {
_documentsByUri.Remove(uri);
entry.Document.Dispose();
}
}

foreach (var (_, entry) in closed) {
Closed?.Invoke(this, new DocumentEventArgs(entry.Document));
Removed?.Invoke(this, new DocumentEventArgs(entry.Document));
}

foreach (var (_, entry) in opened) {
entry.Document.Reset(null);
}
}

public void Dispose() {
lock (_lock) {
foreach (var d in _documentsByUri.Values.OfType<IDisposable>()) {
Expand Down
46 changes: 26 additions & 20 deletions src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ internal sealed class MainModuleResolution : ModuleResolutionBase, IModuleManage
private readonly ConcurrentDictionary<string, IPythonModule> _specialized = new ConcurrentDictionary<string, IPythonModule>();
private IRunningDocumentTable _rdt;

private IEnumerable<string> _userPaths = Enumerable.Empty<string>();

public MainModuleResolution(string root, IServiceContainer services)
: base(root, services) { }

Expand Down Expand Up @@ -75,7 +77,7 @@ protected override IPythonModule CreateModule(string name) {
return module;
}
}

// If there is a stub, make sure it is loaded and attached
// First check stub next to the module.
if (!TryCreateModuleStub(name, moduleImport.ModulePath, out var stub)) {
Expand Down Expand Up @@ -169,14 +171,34 @@ internal async Task LoadBuiltinTypesAsync(CancellationToken cancellationToken =
}
}

internal async Task ReloadSearchPaths(CancellationToken cancellationToken = default) {
var ps = _services.GetService<IProcessServices>();

var paths = await GetInterpreterSearchPathsAsync(cancellationToken);
var (interpreterPaths, userPaths) = PythonLibraryPath.ClassifyPaths(Root, _fs, paths, Configuration.SearchPaths);

InterpreterPaths = interpreterPaths.Select(p => p.Path);
_userPaths = userPaths.Select(p => p.Path);

_log?.Log(TraceEventType.Information, "Interpreter search paths:");
foreach (var s in InterpreterPaths) {
_log?.Log(TraceEventType.Information, $" {s}");
}

_log?.Log(TraceEventType.Information, "User search paths:");
foreach (var s in _userPaths) {
_log?.Log(TraceEventType.Information, $" {s}");
}
}

public async Task ReloadAsync(CancellationToken cancellationToken = default) {
foreach (var uri in Modules
.Where(m => m.Value.Value?.Name != BuiltinModuleName)
.Select(m => m.Value.Value?.Uri)
.ExcludeDefault()) {
GetRdt()?.UnlockDocument(uri);
}

// Preserve builtins, they don't need to be reloaded since interpreter does not change.
var builtins = Modules[BuiltinModuleName];
Modules.Clear();
Expand All @@ -187,26 +209,10 @@ public async Task ReloadAsync(CancellationToken cancellationToken = default) {
var addedRoots = new HashSet<string>();
addedRoots.UnionWith(PathResolver.SetRoot(Root));

var ps = _services.GetService<IProcessServices>();

var paths = await GetInterpreterSearchPathsAsync(cancellationToken);
var (interpreterPaths, userPaths) = PythonLibraryPath.ClassifyPaths(Root, _fs, paths, Configuration.SearchPaths);

InterpreterPaths = interpreterPaths.Select(p => p.Path);
var userSearchPaths = userPaths.Select(p => p.Path);

_log?.Log(TraceEventType.Information, "Interpreter search paths:");
foreach (var s in InterpreterPaths) {
_log?.Log(TraceEventType.Information, $" {s}");
}

_log?.Log(TraceEventType.Information, "User search paths:");
foreach (var s in userSearchPaths) {
_log?.Log(TraceEventType.Information, $" {s}");
}
await ReloadSearchPaths(cancellationToken);

addedRoots.UnionWith(PathResolver.SetInterpreterSearchPaths(InterpreterPaths));
addedRoots.UnionWith(PathResolver.SetUserSearchPaths(userSearchPaths));
addedRoots.UnionWith(PathResolver.SetUserSearchPaths(_userPaths));
ReloadModulePaths(addedRoots);
}

Expand Down
53 changes: 44 additions & 9 deletions src/LanguageServer/Impl/Implementation/Server.cs
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,10 @@ public sealed partial class Server : IDisposable {
private IIndexManager _indexManager;
private string _rootDir;

private bool _watchSearchPaths;
private PathsWatcher _pathsWatcher;
private string[] _searchPaths;

public Server(IServiceManager services) {
_services = services;

Expand All @@ -58,6 +62,7 @@ public Server(IServiceManager services) {
ext.Dispose();
}
})
.Add(() => _pathsWatcher?.Dispose())
.Add(() => _shutdownCts.Cancel());
}

Expand Down Expand Up @@ -173,11 +178,11 @@ public void DidChangeConfiguration(DidChangeConfigurationParams @params, Cancell
_disposableBag.ThrowIfDisposed();
switch (@params.settings) {
case ServerSettings settings: {
if (HandleConfigurationChanges(settings)) {
RestartAnalysis();
if (HandleConfigurationChanges(settings)) {
RestartAnalysis();
}
break;
}
break;
}
default:
_log?.Log(TraceEventType.Error, "change configuration notification sent unsupported settings");
break;
Expand Down Expand Up @@ -233,7 +238,34 @@ private IDocumentationSource ChooseDocumentationSource(string[] kinds) {
}
#endregion

public void NotifyPackagesChanged(CancellationToken cancellationToken) {
public void HandleWatchPathsChange(bool watchSearchPaths) {
if (watchSearchPaths == _watchSearchPaths) {
return;
}

_watchSearchPaths = watchSearchPaths;

if (!_watchSearchPaths) {
_searchPaths = null;
_pathsWatcher?.Dispose();
_pathsWatcher = null;
return;
}

ResetPathWatcher();
}

private void ResetPathWatcher() {
var paths = _interpreter.ModuleResolution.InterpreterPaths.ToArray();

if (_searchPaths == null || !_searchPaths.SequenceEqual(paths)) {
_searchPaths = paths;
_pathsWatcher?.Dispose();
_pathsWatcher = new PathsWatcher(_searchPaths, () => NotifyPackagesChanged(), _log);
}
}

public void NotifyPackagesChanged(CancellationToken cancellationToken = default) {
var interpreter = _services.GetService<IPythonInterpreter>();
_log?.Log(TraceEventType.Information, Resources.ReloadingModules);
// No need to reload typeshed resolution since it is a static storage.
Expand All @@ -242,17 +274,20 @@ public void NotifyPackagesChanged(CancellationToken cancellationToken) {
interpreter.ModuleResolution.ReloadAsync(cancellationToken).ContinueWith(t => {
_log?.Log(TraceEventType.Information, Resources.Done);
_log?.Log(TraceEventType.Information, Resources.AnalysisRestarted);

RestartAnalysis();

if (_watchSearchPaths) {
ResetPathWatcher();
}
}, cancellationToken).DoNotWait();

}

private void RestartAnalysis() {
var analyzer = Services.GetService<IPythonAnalyzer>();;
var analyzer = Services.GetService<IPythonAnalyzer>();
analyzer.ResetAnalyzer();
foreach (var doc in _rdt.GetDocuments()) {
doc.Reset(null);
}
_rdt.ReloadAll();
}
}
}
2 changes: 1 addition & 1 deletion src/LanguageServer/Impl/LanguageServer.Configuration.cs
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ public async Task DidChangeConfiguration(JToken token, CancellationToken cancell
settings.symbolsHierarchyMaxSymbols = GetSetting(analysis, "symbolsHierarchyMaxSymbols", 1000);

_logger.LogLevel = GetLogLevel(analysis).ToTraceEventType();
HandlePathWatchChanges(token, cancellationToken);
HandlePathWatchChanges(token);
HandleDiagnosticsChanges(pythonSection, settings);

_server.DidChangeConfiguration(new DidChangeConfigurationParams { settings = settings }, cancellationToken);
Expand Down
31 changes: 2 additions & 29 deletions src/LanguageServer/Impl/LanguageServer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -55,12 +55,8 @@ public sealed partial class LanguageServer : IDisposable {

private JsonRpc _rpc;
private JsonSerializer _jsonSerializer;
private PathsWatcher _pathsWatcher;
private IIdleTimeTracker _idleTimeTracker;

private bool _watchSearchPaths;
private string[] _searchPaths = Array.Empty<string>();

public CancellationToken Start(IServiceManager services, JsonRpc rpc) {
_server = new Server(services);
_services = services;
Expand All @@ -78,7 +74,6 @@ public CancellationToken Start(IServiceManager services, JsonRpc rpc) {
_disposables
.Add(() => _shutdownCts.Cancel())
.Add(_prioritizer)
.Add(() => _pathsWatcher?.Dispose())
.Add(() => _rpc.TraceSource.Listeners.Remove(rpcTraceListener));

services.AddService(_optionsProvider);
Expand Down Expand Up @@ -357,30 +352,8 @@ private MessageType GetLogLevel(JToken analysisKey) {
return MessageType.Error;
}

private void HandlePathWatchChanges(JToken section, CancellationToken cancellationToken) {
var watchSearchPaths = GetSetting(section, "watchSearchPaths", true);
if (!watchSearchPaths) {
// No longer watching.
_pathsWatcher?.Dispose();
_searchPaths = Array.Empty<string>();
_watchSearchPaths = false;
return;
}

// Now watching.
if (!_watchSearchPaths || (_watchSearchPaths && _searchPaths.SetEquals(_initParams.initializationOptions.searchPaths))) {
// Were not watching OR were watching but paths have changed. Recreate the watcher.
_pathsWatcher?.Dispose();
_pathsWatcher = new PathsWatcher(
_initParams.initializationOptions.searchPaths,
() =>_server.NotifyPackagesChanged(cancellationToken),
_services.GetService<ILogger>()
);

_watchSearchPaths = true;
_searchPaths = _initParams.initializationOptions.searchPaths;
}
}
private void HandlePathWatchChanges(JToken section)
=> _server.HandleWatchPathsChange(GetSetting(section, "watchSearchPaths", true));

private static CancellationToken GetToken(CancellationToken original)
=> Debugger.IsAttached ? CancellationToken.None : original;
Expand Down
12 changes: 11 additions & 1 deletion src/LanguageServer/Impl/PathsWatcher.cs
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ public PathsWatcher(string[] paths, Action onChanged, ILogger log) {
_onChanged = onChanged;

var reduced = ReduceToCommonRoots(paths);

foreach (var p in reduced) {
try {
if (!Directory.Exists(p)) {
Expand All @@ -53,20 +54,29 @@ public PathsWatcher(string[] paths, Action onChanged, ILogger log) {
continue;
}

_log.Log(TraceEventType.Verbose, $"Watching {p}");

try {
var fsw = new System.IO.FileSystemWatcher(p) {
IncludeSubdirectories = true,
EnableRaisingEvents = true,
NotifyFilter = NotifyFilters.FileName | NotifyFilters.DirectoryName
NotifyFilter = NotifyFilters.FileName | NotifyFilters.DirectoryName | NotifyFilters.LastWrite,
InternalBufferSize = 1 << 16, // Max buffer size of 64 KB
};

fsw.Changed += OnChanged;
fsw.Created += OnChanged;
fsw.Deleted += OnChanged;
fsw.Renamed += OnChanged;

fsw.Filter = "*.p*"; // .py, .pyc, .pth - TODO: Use Filters in .NET Core 3.0.

_disposableBag
.Add(() => _throttleTimer?.Dispose())
.Add(() => fsw.Changed -= OnChanged)
.Add(() => fsw.Created -= OnChanged)
.Add(() => fsw.Deleted -= OnChanged)
.Add(() => fsw.Renamed -= OnChanged)
.Add(() => fsw.EnableRaisingEvents = false)
.Add(fsw);
} catch (ArgumentException ex) {
Expand Down