Skip to content

Use locale.getencoding instead of getpreferredencoding #958

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Mar 3, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/testsuite.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ jobs:

tests:
runs-on: ${{ matrix.os }}
env:
PYTHONWARNDEFAULTENCODING: true
strategy:
fail-fast: false
matrix:
Expand Down
1 change: 1 addition & 0 deletions CHANGES.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ The released versions correspond to PyPI releases.
* Fixed a specific problem on reloading a pandas-related module (see [#947](../../issues/947)),
added possibility for unload hooks for specific modules
* Use this also to reload django views (see [#932](../../issues/932))
* Fixed `EncodingWarning` for Python >= 3.11 (see [#957](../../issues/957))

## [Version 5.3.5](https://pypi.python.org/pypi/pyfakefs/5.3.5) (2024-01-30)
Fixes a regression.
Expand Down
8 changes: 4 additions & 4 deletions pyfakefs/fake_file.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
"""
import errno
import io
import locale
import os
import sys
from stat import (
Expand Down Expand Up @@ -52,6 +51,7 @@
real_encoding,
AnyPath,
AnyString,
get_locale_encoding,
)

if TYPE_CHECKING:
Expand Down Expand Up @@ -190,7 +190,7 @@ def contents(self) -> Optional[str]:
"""Return the contents as string with the original encoding."""
if isinstance(self.byte_contents, bytes):
return self.byte_contents.decode(
self.encoding or locale.getpreferredencoding(False),
self.encoding or get_locale_encoding(),
errors=self.errors,
)
return None
Expand Down Expand Up @@ -263,7 +263,7 @@ def _encode_contents(self, contents: Union[str, bytes, None]) -> Optional[bytes]
if is_unicode_string(contents):
contents = bytes(
cast(str, contents),
self.encoding or locale.getpreferredencoding(False),
self.encoding or get_locale_encoding(),
self.errors,
)
return cast(bytes, contents)
Expand Down Expand Up @@ -745,7 +745,7 @@ def __init__(
self._use_line_buffer = not binary and buffering == 1

contents = file_object.byte_contents
self._encoding = encoding or locale.getpreferredencoding(False)
self._encoding = encoding or get_locale_encoding()
errors = errors or "strict"
self._io: Union[BinaryBufferIO, TextBufferIO] = (
BinaryBufferIO(contents)
Expand Down
12 changes: 9 additions & 3 deletions pyfakefs/helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,12 @@ def is_unicode_string(val: Any) -> bool:
return hasattr(val, "encode")


def get_locale_encoding():
if sys.version_info >= (3, 11):
return locale.getencoding()
return locale.getpreferredencoding(False)


@overload
def make_string_path(dir_name: AnyStr) -> AnyStr:
...
Expand All @@ -127,15 +133,15 @@ def to_string(path: Union[AnyStr, Union[str, bytes]]) -> str:
"""Return the string representation of a byte string using the preferred
encoding, or the string itself if path is a str."""
if isinstance(path, bytes):
return path.decode(locale.getpreferredencoding(False))
return path.decode(get_locale_encoding())
return path


def to_bytes(path: Union[AnyStr, Union[str, bytes]]) -> bytes:
"""Return the bytes representation of a string using the preferred
encoding, or the byte string itself if path is a byte string."""
if isinstance(path, str):
return bytes(path, locale.getpreferredencoding(False))
return bytes(path, get_locale_encoding())
return path


Expand Down Expand Up @@ -181,7 +187,7 @@ def matching_string( # type: ignore[misc]
if string is None:
return string
if isinstance(matched, bytes) and isinstance(string, str):
return string.encode(locale.getpreferredencoding(False))
return string.encode(get_locale_encoding())
return string # pytype: disable=bad-return-type


Expand Down
13 changes: 5 additions & 8 deletions pyfakefs/tests/fake_open_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,14 @@

import errno
import io
import locale
import os
import stat
import sys
import time
import unittest

from pyfakefs import fake_filesystem, helpers
from pyfakefs.helpers import is_root, IS_PYPY
from pyfakefs.helpers import is_root, IS_PYPY, get_locale_encoding
from pyfakefs.fake_io import FakeIoModule
from pyfakefs.fake_filesystem_unittest import PatchMode
from pyfakefs.tests.test_utils import RealFsTestCase
Expand Down Expand Up @@ -119,14 +118,12 @@ def test_write_str_read_bytes(self):
try:
with self.open(file_path, "w") as f:
f.write(str_contents)
except UnicodeEncodeError:
with self.open(file_path, "rb") as f:
contents = f.read()
self.assertEqual(str_contents, contents.decode(get_locale_encoding()))
except UnicodeError:
# see https://github.com/pytest-dev/pyfakefs/issues/623
self.skipTest("This test does not work with an ASCII locale")
with self.open(file_path, "rb") as f:
contents = f.read()
self.assertEqual(
str_contents, contents.decode(locale.getpreferredencoding(False))
)

def test_open_valid_file(self):
contents = [
Expand Down