Skip to content
This repository was archived by the owner on Apr 14, 2022. It is now read-only.

Adding support for egg and zip files #1552

Merged
merged 82 commits into from
Sep 16, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
82 commits
Select commit Hold shift + click to select a range
61d60c4
Persistent analysis, part I (#1224)
Jun 20, 2019
b2c9afb
Merge master into database branch (#1312)
Jul 11, 2019
ab8ecaf
Merge master
Jul 15, 2019
4a2fd79
Treat persistent module as regular specialized (#1334)
Jul 23, 2019
da3eb70
Store location of module members in the database (#1339)
Jul 26, 2019
a502cb5
Merge master
Jul 30, 2019
2143719
Multiple fixes and additional tests for persistence (#1372)
Aug 8, 2019
ff46089
Merge master
Aug 10, 2019
78b1bcf
Merge master
Aug 13, 2019
f267b80
Add persistence tests for 60+ system modules (#1429)
Aug 15, 2019
3c53083
Merge master
Aug 16, 2019
69806e0
Test update (we no longer reassign classes)
Aug 16, 2019
356ae4b
Fix typing tests
Aug 16, 2019
36a1d72
Update baseline
Aug 16, 2019
41c94e7
Fix missing keys issue (#1451)
AlexanderSher Aug 16, 2019
6284174
Typevar fix
Aug 17, 2019
28ec49b
Merge master
Aug 20, 2019
8b4c00a
Merge mast
Aug 21, 2019
86c8426
Persistence of generics and named tuples (#1459)
Aug 21, 2019
a82c356
- Fix #1455: AF: Library module of type Stub has been analyzed alread…
AlexanderSher Aug 22, 2019
ab93845
Merge master
Aug 23, 2019
79685eb
Update baselines since __spec__ member was added
Aug 26, 2019
809dc5c
Merge master (partial)
Aug 26, 2019
0f7f7ca
Undo some of changes to match master.
Aug 26, 2019
252d303
Fix null ref
Aug 26, 2019
a013b3a
Merge branch 'master' of https://github.com/Microsoft/python-language…
Aug 27, 2019
4de0812
Merge master
Aug 28, 2019
0913ea3
Merge master
Aug 30, 2019
8adb6cc
Merge master
Aug 30, 2019
c1883b5
Merge master
Sep 3, 2019
b923349
Resolve conflicts
Sep 3, 2019
28ce8e3
Fix null ref on bases due to new AnyStr behavior
Sep 3, 2019
e7f29d7
Handling dependencies in module persistence (#1471)
Sep 5, 2019
8cf831f
Upgrade to 3.0
Sep 5, 2019
958617b
Merge branch 'net3' of https://github.com/MikhailArkhipov/python-lang…
Sep 5, 2019
54d67cb
Undo change + .NET 3
Sep 5, 2019
26ea552
Get back to 2.0
Sep 5, 2019
fff4e0c
Merge branch 'net3' of https://github.com/MikhailArkhipov/python-lang…
Sep 5, 2019
227726c
Update components
Sep 5, 2019
2dd8fcf
Display caching level
Sep 5, 2019
c0f16e6
Fix null ref due to conflicting changes
Sep 5, 2019
987a837
Null ref fix
Sep 5, 2019
6d79b49
Simplify
Sep 5, 2019
28a4ce5
Pass AST whish is required during analysis when module does not have …
Sep 5, 2019
e24a4fd
tensorflow test
Sep 6, 2019
c4404cf
Fix from import model
Sep 6, 2019
c20485b
Undo debug
Sep 6, 2019
119e1b2
Improve goto def in import and fix class base processing (#1529)
Sep 9, 2019
2313d44
Usings
Sep 10, 2019
af2d7c7
Test fix
Sep 10, 2019
8e3289c
Remove CompletionItemKind.None
Sep 10, 2019
af70958
Add info to asserts
Sep 10, 2019
83ddd00
Merge master
Sep 10, 2019
967e741
PR feedback
Sep 10, 2019
653fd80
Undo test change, enable class reassignment
Sep 10, 2019
655818e
Prevent merge of unrelated types
Sep 11, 2019
48561b8
more tests
Sep 11, 2019
61014c1
Fix variable update
Sep 11, 2019
ab29574
Null check
Sep 11, 2019
81e36ce
Baselines
Sep 11, 2019
8323cd2
Move analysis complete event
Sep 11, 2019
f44c051
More stable enumeration
Sep 11, 2019
87093f3
Improve restoring imports in tests
Sep 11, 2019
a81874e
Make extension for tests
Sep 11, 2019
6a6fbb2
Add URI null check
Sep 12, 2019
8547a9b
Fix completion typo
Sep 12, 2019
8aa373c
Add assert on unresolved modules
Sep 12, 2019
820fdd4
Add more details to assert
Sep 12, 2019
db002c4
Using
Sep 12, 2019
56b8d88
To 3.7
Sep 12, 2019
68d38e7
Implement check for big/little endian
Sep 12, 2019
8dcd2bd
Merge master
Sep 12, 2019
93be7b1
Sync
Sep 12, 2019
2bdc314
Synchronization
Sep 13, 2019
3ee351c
Add hard assert
Sep 13, 2019
55b85e4
Fix exception
Sep 14, 2019
97f280c
Merge branch 'master' of https://github.com/Microsoft/python-language…
Sep 16, 2019
4108948
Fix typo
Sep 16, 2019
d6260fe
Add lock
Sep 16, 2019
c08dc38
Adding support for egg and zip files (#1477)
CTrando Sep 16, 2019
fbfafb8
Merge branch 'master' of https://github.com/Microsoft/python-language…
Sep 16, 2019
80fcca7
Merge branch 'db' of https://github.com/Microsoft/python-language-ser…
Sep 16, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 17 additions & 4 deletions src/Analysis/Ast/Impl/Modules/Resolution/ModuleResolutionBase.cs
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ public IPythonModule GetOrLoadModule(string name) {
moduleRef = Modules.GetOrAdd(name, new ModuleRef());
return moduleRef.GetOrCreate(name, this);
}

public ModulePath FindModule(string filePath) {
var bestLibraryPath = string.Empty;

Expand All @@ -102,11 +102,24 @@ public ModulePath FindModule(string filePath) {
}

protected void ReloadModulePaths(in IEnumerable<string> rootPaths) {
foreach (var moduleFile in rootPaths.Where(Directory.Exists).SelectMany(p => PathUtils.EnumerateFiles(FileSystem, p))) {
PathResolver.TryAddModulePath(moduleFile.FullName, moduleFile.Length, false, out _);
foreach (var root in rootPaths) {
foreach (var moduleFile in PathUtils.EnumerateFiles(FileSystem, root)) {
PathResolver.TryAddModulePath(moduleFile.FullName, moduleFile.Length, false, out _);
}

if (PathUtils.TryGetZipFilePath(root, out var zipFilePath, out var _) && File.Exists(zipFilePath)) {
foreach (var moduleFile in PathUtils.EnumerateZip(zipFilePath)) {
if (!PathUtils.PathStartsWith(moduleFile.FullName, "EGG-INFO")) {
PathResolver.TryAddModulePath(
Path.Combine(zipFilePath,
PathUtils.NormalizePath(moduleFile.FullName)),
moduleFile.Length, false, out _
);
}
}
}
}
}

protected class ModuleRef {
private readonly object _syncObj = new object();
private IPythonModule _module;
Expand Down
21 changes: 13 additions & 8 deletions src/Analysis/Ast/Impl/get_search_paths.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,13 +68,18 @@ def clean(path):
BEFORE_SITE.discard(None)
AFTER_SITE.discard(None)

import zipfile

for p in sys.path:
p = clean(p)
if os.path.isdir(p):
if p in BEFORE_SITE:
print("%s|stdlib|" % p)
elif p in AFTER_SITE:
if p in SITE_PKGS:
print("%s|site|" % p)
else:
print("%s|pth|" % p)

if not os.path.isdir(p) and not (os.path.isfile(p) and zipfile.is_zipfile(p)):
continue

if p in BEFORE_SITE:
print("%s|stdlib|" % p)
elif p in AFTER_SITE:
if p in SITE_PKGS:
print("%s|site|" % p)
else:
print("%s|pth|" % p)
2 changes: 0 additions & 2 deletions src/Analysis/Ast/Test/ImportTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
// See the Apache Version 2.0 License for specific language governing
// permissions and limitations under the License.

using System.IO;
using System.Linq;
using System.Threading.Tasks;
using FluentAssertions;
Expand All @@ -23,7 +22,6 @@
using Microsoft.Python.Analysis.Types;
using Microsoft.Python.Core;
using Microsoft.Python.Parsing.Tests;
using Microsoft.Python.Tests.Utilities.FluentAssertions;
using Microsoft.VisualStudio.TestTools.UnitTesting;
using TestUtilities;

Expand Down
8 changes: 7 additions & 1 deletion src/Core/Impl/IO/FileSystem.cs
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,13 @@ public long FileSize(string path) {
return fileInfo.Length;
}

public string ReadAllText(string path) => File.ReadAllText(path);
public string ReadAllText(string filePath) {
if (PathUtils.TryGetZipFilePath(filePath, out var zipPath, out var relativeZipPath)) {
return PathUtils.GetZipContent(zipPath, relativeZipPath);
}
return File.ReadAllText(filePath);
}

public void WriteAllText(string path, string content) => File.WriteAllText(path, content);
public IEnumerable<string> FileReadAllLines(string path) => File.ReadLines(path);
public void FileWriteAllLines(string path, IEnumerable<string> contents) => File.WriteAllLines(path, contents);
Expand Down
90 changes: 88 additions & 2 deletions src/Core/Impl/IO/PathUtils.cs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Runtime.InteropServices;
using System.Threading;
Expand Down Expand Up @@ -47,7 +48,6 @@ public static bool IsValidFileNameCharacter(char character)
public static bool HasEndSeparator(string path)
=> !string.IsNullOrEmpty(path) && IsDirectorySeparator(path[path.Length - 1]);


public static bool IsDirectorySeparator(char c) => Array.IndexOf(DirectorySeparators, c) != -1;

public static bool PathStartsWith(string s, string prefix)
Expand Down Expand Up @@ -117,7 +117,7 @@ public static string FindFile(IFileSystem fileSystem,
int depthLimit = 2,
IEnumerable<string> firstCheck = null
) {
if (!Directory.Exists(root)) {
if (!fileSystem.DirectoryExists(root)) {
return null;
}

Expand Down Expand Up @@ -185,12 +185,17 @@ public static IEnumerable<string> EnumerateDirectories(IFileSystem fileSystem, s
var path = queue.Dequeue();
path = EnsureEndSeparator(path);

if (!fileSystem.DirectoryExists(path)) {
continue;
}

IEnumerable<string> dirs = null;
try {
dirs = fileSystem.GetDirectories(path);
} catch (UnauthorizedAccessException) {
} catch (IOException) {
}

if (dirs == null) {
continue;
}
Expand Down Expand Up @@ -308,6 +313,87 @@ public static IEnumerable<IFileInfo> EnumerateFiles(IFileSystem fileSystem, stri
}
}

public static bool TryGetZipFilePath(string filePath, out string zipPath, out string relativeZipPath) {
zipPath = string.Empty;
relativeZipPath = string.Empty;
if (string.IsNullOrEmpty(filePath)) {
return false;
}

var workingPath = filePath;
// Filepath doesn't have zip or egg in it, bail
if (!filePath.Contains(".zip") && !filePath.Contains(".egg")) {
return false;
}

while (!string.IsNullOrEmpty(workingPath)) {
if (IsZipFile(workingPath, out zipPath)) {
// File path is '..\\test\\test.zip\\test\\a.py'
// Working path is '..\\test\\test.zip'
// Relative path in zip file becomes 'test/a.py'
relativeZipPath = filePath.Substring(workingPath.Length);

// According to https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT, zip files must have forward slashes
foreach (var separator in DirectorySeparators) {
relativeZipPath = relativeZipPath.Replace(separator, '/');
}
return true;
}
// \\test\\test.zip => \\test\\
workingPath = GetParent(workingPath);
}

// Filepath had .zip or .egg in it but no zip or egg files
// e.g /tmp/tmp.zip.txt
return false;
}

/// <summary>
/// Returns whether the given file path is a path to a zip (or egg) file
/// The path can be of the form ..\\test.zip or ..\\test.zip\\
/// </summary>
public static bool IsZipFile(string rawZipPath, out string zipPath) {
var path = NormalizePathAndTrim(rawZipPath);
var extension = Path.GetExtension(path);
switch (extension) {
case ".zip":
case ".egg":
zipPath = path;
return true;
default:
zipPath = string.Empty;
return false;
}
}

/// <summary>
/// Given the path to the zip file and the relative path to a file inside the zip,
/// returns the contents of the zip entry
/// e.g
/// test.zip
/// a.py
/// b.py
/// Can get the contents of a.py by passing in "test.zip" and "a.py"
/// </summary>
public static string GetZipContent(string zipPath, string relativeZipPath) {
using (var zip = ZipFile.OpenRead(zipPath)) {
var zipFile = zip.GetEntry(relativeZipPath);
// Could not open zip, bail
if (zipFile == null) {
return null;
}
using (var reader = new StreamReader(zipFile.Open())) {
return reader.ReadToEnd();
}
}
}

public static IEnumerable<ZipArchiveEntry> EnumerateZip(string root) {
using (var zip = ZipFile.OpenRead(root)) {
return zip.Entries.ToList();
}
}

/// <summary>
/// Deletes a file, making multiple attempts and suppressing any
/// IO-related errors.
Expand Down
53 changes: 53 additions & 0 deletions src/Core/Test/PathUtilsTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
// Copyright(c) Microsoft Corporation
// All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the License); you may not use
// this file except in compliance with the License. You may obtain a copy of the
// License at http://www.apache.org/licenses/LICENSE-2.0
//
// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS
// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY
// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
// MERCHANTABILITY OR NON-INFRINGEMENT.
//
// See the Apache Version 2.0 License for specific language governing
// permissions and limitations under the License.

using FluentAssertions;
using Microsoft.Python.Core.IO;
using Microsoft.VisualStudio.TestTools.UnitTesting;

namespace Microsoft.Python.Core.Tests {
[TestClass]
public class PathUtilsTests {
[TestMethod, Priority(0)]
public void ZipFileUNCPath() {
PathUtils.TryGetZipFilePath(@"\\server\home\share\test.zip", out var zipPath, out var relativeZipPath);
zipPath.Should().Be(@"\\server\home\share\test.zip");
relativeZipPath.Should().BeEmpty();

PathUtils.TryGetZipFilePath(@"\\server\home\share\test.zip\test\a.py", out zipPath, out relativeZipPath);
zipPath.Should().Be(@"\\server\home\share\test.zip");
relativeZipPath.Should().Be("test/a.py");

PathUtils.TryGetZipFilePath("\\path\\foo\\baz\\test.zip\\test\\a.py", out zipPath, out relativeZipPath);
zipPath.Should().Be("\\path\\foo\\baz\\test.zip");
relativeZipPath.Should().Be("test/a.py");
}

[TestMethod, Priority(0)]
public void ZipFilePath() {
PathUtils.TryGetZipFilePath("\\path\\foo\\baz\\test.zip", out var zipPath, out var relativeZipPath);
zipPath.Should().Be("\\path\\foo\\baz\\test.zip");
relativeZipPath.Should().BeEmpty();

PathUtils.TryGetZipFilePath("\\path\\foo\\baz\\test.zip\\test\\a.py", out zipPath, out relativeZipPath);
zipPath.Should().Be("\\path\\foo\\baz\\test.zip");
relativeZipPath.Should().Be("test/a.py");

PathUtils.TryGetZipFilePath("\\path\\foo\\baz\\test.zip\\test\\foo\\baz.py", out zipPath, out relativeZipPath);
zipPath.Should().Be("\\path\\foo\\baz\\test.zip");
relativeZipPath.Should().Be("test/foo/baz.py");
}
}
}
98 changes: 97 additions & 1 deletion src/LanguageServer/Test/ImportsTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,10 @@
using System;
using System.IO;
using System.Threading.Tasks;
using Microsoft.Python.Analysis;
using Microsoft.Python.Analysis.Analyzer;
using Microsoft.Python.Analysis.Documents;
using Microsoft.Python.Analysis.Tests.FluentAssertions;
using Microsoft.Python.Analysis.Types;
using Microsoft.Python.Core.Text;
using Microsoft.Python.LanguageServer.Completion;
using Microsoft.Python.LanguageServer.Sources;
Expand Down Expand Up @@ -830,5 +831,100 @@ import module2
comps = cs.GetCompletions(analysis, new SourceLocation(4, 9));
comps.Should().HaveLabels("Y");
}

[DataRow("Basic.egg")]
[DataRow("Basic.zip")]
[DataTestMethod, Priority(0)]
public async Task BasicEggZip(string eggZipFilePath) {
var root = Path.Combine(GetAnalysisTestDataFilesPath(), "EggZip");
await CreateServicesAsync(root, PythonVersions.LatestAvailable3X, searchPaths: new[] { root, Path.Combine(root, eggZipFilePath) });
var rdt = Services.GetService<IRunningDocumentTable>();
var analyzer = Services.GetService<IPythonAnalyzer>();

var uriPath = Path.Combine(root, "BasicEggZip.py");
var code = await File.ReadAllTextAsync(uriPath);
var moduleUri = TestData.GetTestSpecificUri(uriPath);
var module = rdt.OpenDocument(moduleUri, code);

await analyzer.WaitForCompleteAnalysisAsync();
var analysis = await module.GetAnalysisAsync(-1);
analysis.Should().HaveVariable("i").OfType(BuiltinTypeId.Int);
}

[DataRow("ZipImports.zip")]
[DataRow("EggImports.egg")]
[DataTestMethod, Priority(0)]
public async Task EggZipImports(string eggZipFilePath) {
var root = Path.Combine(GetAnalysisTestDataFilesPath(), "EggZip");
await CreateServicesAsync(root, PythonVersions.LatestAvailable3X, searchPaths: new[] { root, Path.Combine(root, eggZipFilePath, "test") });
var rdt = Services.GetService<IRunningDocumentTable>();
var analyzer = Services.GetService<IPythonAnalyzer>();

var uriPath = Path.Combine(root, "EggZipImports.py");
var code = await File.ReadAllTextAsync(uriPath);
var moduleUri = TestData.GetTestSpecificUri(uriPath);
var module = rdt.OpenDocument(moduleUri, code);

await analyzer.WaitForCompleteAnalysisAsync();
var analysis = await module.GetAnalysisAsync(-1);
analysis.Should().HaveVariable("h").OfType("X");
analysis.Should().HaveVariable("y").OfType(BuiltinTypeId.Int);
analysis.Should().HaveVariable("b").OfType("A");
analysis.Should().HaveVariable("i").OfType(BuiltinTypeId.Int);
}

[DataRow("ZipRelativeImports.zip")]
[DataRow("EggRelativeImports.egg")]
[DataTestMethod, Priority(0)]
public async Task EggZipRelativeImports(string eggZipFilePath) {
var root = Path.Combine(GetAnalysisTestDataFilesPath(), "EggZip");
await CreateServicesAsync(root, PythonVersions.LatestAvailable3X, searchPaths: new[] { root, Path.Combine(root, eggZipFilePath, "test") });
var rdt = Services.GetService<IRunningDocumentTable>();
var analyzer = Services.GetService<IPythonAnalyzer>();

var uriPath = Path.Combine(root, "EggZipRelativeImports.py");
var code = await File.ReadAllTextAsync(uriPath);
var moduleUri = TestData.GetTestSpecificUri(uriPath);
var module = rdt.OpenDocument(moduleUri, code);

await analyzer.WaitForCompleteAnalysisAsync();
var analysis = await module.GetAnalysisAsync(-1);
analysis.Should().HaveVariable("h").OfType(BuiltinTypeId.Float);
analysis.Should().HaveVariable("i").OfType(BuiltinTypeId.Int);
analysis.Should().HaveVariable("s").OfType(BuiltinTypeId.Str);
}

[DataRow("simplejson.egg")]
[DataRow("simplejson.zip")]
[DataTestMethod, Priority(0)]
public async Task SimpleJsonEggZip(string eggZipFilePath) {
var root = Path.Combine(GetAnalysisTestDataFilesPath(), "EggZip");
await CreateServicesAsync(root, PythonVersions.LatestAvailable3X, searchPaths: new[] { root, Path.Combine(root, eggZipFilePath) });
var rdt = Services.GetService<IRunningDocumentTable>();
var analyzer = Services.GetService<IPythonAnalyzer>();

const string code = "import simplejson";
var uriPath = Path.Combine(root, "test.py");
var moduleUri = TestData.GetTestSpecificUri(uriPath);
var module = rdt.OpenDocument(moduleUri, code);

await analyzer.WaitForCompleteAnalysisAsync();
var analysis = await module.GetAnalysisAsync(-1);
analysis.Should().HaveVariable("simplejson").Which.Should().HaveMembers(
"Decimal",
"JSONDecodeError",
"JSONDecoder",
"JSONEncoder",
"JSONEncoderForHTML",
"OrderedDict",
"RawJSON",
"dump",
"dumps",
"load",
"loads",
"simple_first"
);
}

}
}
Binary file not shown.
Binary file not shown.
5 changes: 5 additions & 0 deletions src/UnitTests/TestData/AstAnalysis/EggZip/BasicEggZip.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
import sys
import test.a

a = test.a.A()
i = a.test()
Binary file not shown.
Binary file not shown.
Loading