Skip to content

gh-117349: Micro-optimize a few os.path functions #117350

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 50 commits into from
Apr 2, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
50 commits
Select commit Hold shift + click to select a range
c90a883
Speed up `posixpath.ismount`
nineteendo Mar 28, 2024
5984959
Speed up `posixpath.expanduser`
nineteendo Mar 28, 2024
833ddc9
Speed up `posixpath.normpath`
nineteendo Mar 28, 2024
a4d9fcb
Refactor `posixpath.expandvars` & `ntpath.commonpath`
nineteendo Mar 28, 2024
b853d4d
Remove start- & endswith
nineteendo Mar 29, 2024
a8984dc
Remove `startswith`
nineteendo Mar 29, 2024
78929b0
Remove `isabs` calls
nineteendo Mar 29, 2024
afc7bbc
Rename result of `splitroot`
nineteendo Mar 29, 2024
f5bbaf6
Refactor `os.path`
nineteendo Mar 29, 2024
e740f10
fix unbound variable
nineteendo Mar 29, 2024
d02c726
hardcode constants like documented
nineteendo Mar 29, 2024
b794897
📜🤖 Added by blurb_it.
blurb-it[bot] Mar 29, 2024
c2c04bf
Fix typo
nineteendo Mar 29, 2024
7c9dcae
exclude stylistic-only changes
nineteendo Mar 29, 2024
d35783c
Revert renaming
nineteendo Mar 29, 2024
eb24723
Revert unnesting
nineteendo Mar 29, 2024
5ca9ea3
Revert further changes
nineteendo Mar 29, 2024
3e4d0e3
Revert renaming
nineteendo Mar 29, 2024
72babad
Remove newline
nineteendo Mar 29, 2024
abfe46c
Remove slice
nineteendo Mar 30, 2024
2a055ba
Speedup `posixpath.ismount`
nineteendo Mar 30, 2024
759b189
Update Lib/posixpath.py
nineteendo Mar 30, 2024
cca16ba
Revert unnesting
nineteendo Mar 30, 2024
7e21192
Merge branch 'speedup-os.path' of https://github.com/nineteendo/cpyth…
nineteendo Mar 30, 2024
581862e
Remove line breaks
nineteendo Mar 30, 2024
d2987fd
Revert `ntpath.expandvars`
nineteendo Mar 30, 2024
1866544
Unexpose `posixpath._normpath_fallback`
nineteendo Mar 30, 2024
4a01123
Update Lib/posixpath.py
nineteendo Mar 30, 2024
67f0620
Indent for git blame
nineteendo Mar 30, 2024
9905730
Merge branch 'speedup-os.path' of https://github.com/nineteendo/cpyth…
nineteendo Mar 30, 2024
9d18f8c
Move back in except
nineteendo Mar 30, 2024
9520cc6
Revert changing comment
nineteendo Mar 30, 2024
de4b09e
Split try except blocks
nineteendo Mar 30, 2024
6f8e7d7
Update Misc/NEWS.d/next/Core and Builtins/2024-03-29-15-04-13.gh-issu…
nineteendo Mar 30, 2024
47a4b57
Update ACKS
nineteendo Mar 30, 2024
48f2971
Remove `_get_sep()` call
nineteendo Mar 30, 2024
33259d9
Revert unnecessary change
nineteendo Mar 30, 2024
4568cd1
Revert insignificant changes
nineteendo Mar 30, 2024
1382f49
Fix incorrect change
nineteendo Mar 30, 2024
a81dc6e
Revert unnesting
nineteendo Mar 30, 2024
e6cb4a3
Move to new pull request
nineteendo Mar 30, 2024
67ec9a6
assign `devnull` first
nineteendo Mar 30, 2024
87d43b8
speedup `posixpath.realpath`
nineteendo Mar 31, 2024
6c8901a
Speedup `ntpath.isreserved()`
nineteendo Apr 1, 2024
7940a64
Revert starts- & endswith
nineteendo Apr 1, 2024
e637698
Revert `len()` call
nineteendo Apr 1, 2024
b5fdd27
;(
nineteendo Apr 1, 2024
c923e49
Merge branch 'main' into speedup-os.path
nineteendo Apr 2, 2024
887f3c1
Replace definition of colon
nineteendo Apr 2, 2024
e3ace2f
Merge branch 'speedup-os.path' of https://github.com/nineteendo/cpyth…
nineteendo Apr 2, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 13 additions & 16 deletions Lib/ntpath.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,11 +102,11 @@ def join(path, *paths):
if isinstance(path, bytes):
sep = b'\\'
seps = b'\\/'
colon = b':'
colon_seps = b':\\/'
else:
sep = '\\'
seps = '\\/'
colon = ':'
colon_seps = ':\\/'
try:
if not paths:
path[:0] + sep #23780: Ensure compatible data type even if p is null.
Expand Down Expand Up @@ -135,7 +135,7 @@ def join(path, *paths):
result_path = result_path + p_path
## add separator between UNC and non-absolute path
if (result_path and not result_root and
result_drive and result_drive[-1:] not in colon + seps):
result_drive and result_drive[-1] not in colon_seps):
return result_drive + sep + result_path
return result_drive + result_root + result_path
except (TypeError, AttributeError, BytesWarning):
Expand Down Expand Up @@ -279,7 +279,7 @@ def isjunction(path):
st = os.lstat(path)
except (OSError, ValueError, AttributeError):
return False
return bool(st.st_reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT)
return st.st_reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT
else:
# Use genericpath.isjunction as imported above
pass
Expand Down Expand Up @@ -340,8 +340,8 @@ def isreserved(path):
def _isreservedname(name):
"""Return true if the filename is reserved by the system."""
# Trailing dots and spaces are reserved.
if name.endswith(('.', ' ')) and name not in ('.', '..'):
return True
if name[-1:] in ('.', ' '):
return name not in ('.', '..')
# Wildcards, separators, colon, and pipe (*?"<>/\:|) are reserved.
# ASCII control characters (0-31) are reserved.
# Colon is reserved for file streams (e.g. "name:stream[:type]").
Expand All @@ -350,9 +350,7 @@ def _isreservedname(name):
# DOS device names are reserved (e.g. "nul" or "nul .txt"). The rules
# are complex and vary across Windows versions. On the side of
# caution, return True for names that may not be reserved.
if name.partition('.')[0].rstrip(' ').upper() in _reserved_names:
return True
return False
return name.partition('.')[0].rstrip(' ').upper() in _reserved_names


# Expand paths beginning with '~' or '~user'.
Expand Down Expand Up @@ -381,13 +379,10 @@ def expanduser(path):

if 'USERPROFILE' in os.environ:
userhome = os.environ['USERPROFILE']
elif not 'HOMEPATH' in os.environ:
elif 'HOMEPATH' not in os.environ:
return path
else:
try:
drive = os.environ['HOMEDRIVE']
except KeyError:
drive = ''
drive = os.environ.get('HOMEDRIVE', '')
userhome = join(drive, os.environ['HOMEPATH'])

if i != 1: #~user
Expand Down Expand Up @@ -727,15 +722,17 @@ def realpath(path, *, strict=False):
new_unc_prefix = b'\\\\'
cwd = os.getcwdb()
# bpo-38081: Special case for realpath(b'nul')
if normcase(path) == normcase(os.fsencode(devnull)):
devnull = b'nul'
if normcase(path) == devnull:
return b'\\\\.\\NUL'
else:
prefix = '\\\\?\\'
unc_prefix = '\\\\?\\UNC\\'
new_unc_prefix = '\\\\'
cwd = os.getcwd()
# bpo-38081: Special case for realpath('nul')
if normcase(path) == normcase(devnull):
devnull = 'nul'
if normcase(path) == devnull:
return '\\\\.\\NUL'
had_prefix = path.startswith(prefix)
if not had_prefix and not isabs(path):
Expand Down
43 changes: 20 additions & 23 deletions Lib/posixpath.py
Original file line number Diff line number Diff line change
Expand Up @@ -213,15 +213,8 @@ def ismount(path):
except (OSError, ValueError):
return False

dev1 = s1.st_dev
dev2 = s2.st_dev
if dev1 != dev2:
return True # path/.. on a different device as path
ino1 = s1.st_ino
ino2 = s2.st_ino
if ino1 == ino2:
return True # path/.. is the same i-node as path
return False
# path/.. on a different device as path or the same i-node as path
return s1.st_dev != s2.st_dev or s1.st_ino == s2.st_ino


# Expand paths beginning with '~' or '~user'.
Expand Down Expand Up @@ -270,7 +263,7 @@ def expanduser(path):
return path
name = path[1:i]
if isinstance(name, bytes):
name = str(name, 'ASCII')
name = name.decode('ascii')
try:
pwent = pwd.getpwnam(name)
except KeyError:
Expand Down Expand Up @@ -359,21 +352,19 @@ def normpath(path):
path = os.fspath(path)
if isinstance(path, bytes):
sep = b'/'
empty = b''
dot = b'.'
dotdot = b'..'
else:
sep = '/'
empty = ''
dot = '.'
dotdot = '..'
if path == empty:
if not path:
return dot
_, initial_slashes, path = splitroot(path)
comps = path.split(sep)
new_comps = []
for comp in comps:
if comp in (empty, dot):
if not comp or comp == dot:
continue
if (comp != dotdot or (not initial_slashes and not new_comps) or
(new_comps and new_comps[-1] == dotdot)):
Expand All @@ -396,12 +387,12 @@ def normpath(path):
def abspath(path):
"""Return an absolute path."""
path = os.fspath(path)
if not isabs(path):
if isinstance(path, bytes):
cwd = os.getcwdb()
else:
cwd = os.getcwd()
path = join(cwd, path)
if isinstance(path, bytes):
if not path.startswith(b'/'):
path = join(os.getcwdb(), path)
else:
if not path.startswith('/'):
path = join(os.getcwd(), path)
return normpath(path)


Expand All @@ -417,6 +408,7 @@ def realpath(filename, *, strict=False):

# Join two paths, normalizing and eliminating any symbolic links
# encountered in the second path.
# Two leading slashes are replaced by a single slash.
def _joinrealpath(path, rest, strict, seen):
if isinstance(path, bytes):
sep = b'/'
Expand All @@ -427,7 +419,7 @@ def _joinrealpath(path, rest, strict, seen):
curdir = '.'
pardir = '..'

if isabs(rest):
if rest.startswith(sep):
rest = rest[1:]
path = sep

Expand All @@ -439,10 +431,15 @@ def _joinrealpath(path, rest, strict, seen):
if name == pardir:
# parent dir
if path:
path, name = split(path)
parent, name = split(path)
if name == pardir:
path = join(path, pardir, pardir)
# ../..
path = join(path, pardir)
else:
# foo/bar/.. -> foo
path = parent
else:
# ..
path = pardir
continue
newpath = join(path, name)
Expand Down
1 change: 1 addition & 0 deletions Misc/ACKS
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,7 @@ Finn Bock
Paul Boddie
Matthew Boedicker
Robin Boerdijk
Wannes Boeykens
Andra Bogildea
Matt Bogosian
Nikolay Bogoychev
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Optimise several functions in :mod:`os.path`.
Loading