From 526445fea9a33ac685eca3864ef5ccfff3777a6b Mon Sep 17 00:00:00 2001 From: Emma Harper Smith Date: Tue, 6 May 2025 19:15:42 -0700 Subject: [PATCH 1/8] Format compression.zstd to match PEP 8 Mostly just make both files consistently use single vs double quotes. Also add a missing space in a multi-line string. --- Lib/compression/zstd/__init__.py | 32 ++++++++++++++++--------------- Lib/compression/zstd/_zstdfile.py | 10 +++++----- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/Lib/compression/zstd/__init__.py b/Lib/compression/zstd/__init__.py index 4f734eb07b00e3..b01613fb82543c 100644 --- a/Lib/compression/zstd/__init__.py +++ b/Lib/compression/zstd/__init__.py @@ -37,15 +37,16 @@ class FrameInfo: """Information about a Zstandard frame.""" - __slots__ = 'decompressed_size', 'dictionary_id' + + __slots__ = "decompressed_size", "dictionary_id" def __init__(self, decompressed_size, dictionary_id): - super().__setattr__('decompressed_size', decompressed_size) - super().__setattr__('dictionary_id', dictionary_id) + super().__setattr__("decompressed_size", decompressed_size) + super().__setattr__("dictionary_id", dictionary_id) def __repr__(self): - return (f'FrameInfo(decompressed_size={self.decompressed_size}, ' - f'dictionary_id={self.dictionary_id})') + return (f"FrameInfo(decompressed_size={self.decompressed_size}, " + f"dictionary_id={self.dictionary_id})") def __setattr__(self, name, _): raise AttributeError(f"can't set attribute {name!r}") @@ -78,10 +79,10 @@ def train_dict(samples, dict_size): """ if not isinstance(dict_size, int): ds_cls = type(dict_size).__qualname__ - raise TypeError(f'dict_size must be an int object, not {ds_cls!r}.') + raise TypeError(f"dict_size must be an int object, not {ds_cls!r}.") samples = tuple(samples) - chunks = b''.join(samples) + chunks = b"".join(samples) chunk_sizes = tuple(_nbytes(sample) for sample in samples) if not chunks: raise ValueError("samples contained no data; can't train dictionary.") @@ -109,23 +110,23 @@ def finalize_dict(zstd_dict, /, samples, dict_size, level): """ if not isinstance(zstd_dict, ZstdDict): - raise TypeError('zstd_dict argument should be a ZstdDict object.') + raise TypeError("zstd_dict argument should be a ZstdDict object.") if not isinstance(dict_size, int): - raise TypeError('dict_size argument should be an int object.') + raise TypeError("dict_size argument should be an int object.") if not isinstance(level, int): - raise TypeError('level argument should be an int object.') + raise TypeError("level argument should be an int object.") samples = tuple(samples) - chunks = b''.join(samples) + chunks = b"".join(samples) chunk_sizes = tuple(_nbytes(sample) for sample in samples) if not chunks: - raise ValueError("The samples are empty content, can't finalize the" + raise ValueError("The samples are empty content, can't finalize the " "dictionary.") - dict_content = _zstd._finalize_dict(zstd_dict.dict_content, - chunks, chunk_sizes, - dict_size, level) + dict_content = _zstd._finalize_dict(zstd_dict.dict_content, chunks, + chunk_sizes, dict_size, level) return ZstdDict(dict_content) + def compress(data, level=None, options=None, zstd_dict=None): """Return Zstandard compressed *data* as bytes. @@ -141,6 +142,7 @@ def compress(data, level=None, options=None, zstd_dict=None): comp = ZstdCompressor(level=level, options=options, zstd_dict=zstd_dict) return comp.compress(data, mode=ZstdCompressor.FLUSH_FRAME) + def decompress(data, zstd_dict=None, options=None): """Decompress one or more frames of Zstandard compressed *data*. diff --git a/Lib/compression/zstd/_zstdfile.py b/Lib/compression/zstd/_zstdfile.py index fbc9e02a733626..1f45161625b7d9 100644 --- a/Lib/compression/zstd/_zstdfile.py +++ b/Lib/compression/zstd/_zstdfile.py @@ -78,10 +78,10 @@ def __init__(self, file, /, mode="r", *, raise ValueError(f"Invalid mode: {mode!r}") if isinstance(file, (str, bytes, PathLike)): - self._fp = io.open(file, f'{mode}b') + self._fp = io.open(file, f"{mode}b") self._close_fp = True - elif ((mode == 'r' and hasattr(file, "read")) - or (mode != 'r' and hasattr(file, "write"))): + elif ((mode == "r" and hasattr(file, "read")) + or (mode != "r" and hasattr(file, "write"))): self._fp = file else: raise TypeError("file must be a file-like object " @@ -107,7 +107,7 @@ def close(self): return try: if self._mode == _MODE_READ: - if getattr(self, '_buffer', None): + if getattr(self, "_buffer", None): self._buffer.close() self._buffer = None elif self._mode == _MODE_WRITE: @@ -274,7 +274,7 @@ def name(self): @property def mode(self): - return 'wb' if self._mode == _MODE_WRITE else 'rb' + return "wb" if self._mode == _MODE_WRITE else "rb" @property def closed(self): From a5c20a882ed144e6f25f9f373a3b919aa2cc7725 Mon Sep 17 00:00:00 2001 From: Emma Harper Smith Date: Tue, 6 May 2025 19:42:28 -0700 Subject: [PATCH 2/8] Support finding libzstd not in pkg-config GH-133479 removed the logic to check for libzstd outside of pkg-config. This commit adds that logic back with a check for the version so that users can provide their own libzstd. This is to bring parity with lzma, bz2, and zlib detection. --- configure | 264 +++++++++++++++++++++++++++++++++++++++++++++++++- configure.ac | 21 +++- pyconfig.h.in | 6 ++ 3 files changed, 288 insertions(+), 3 deletions(-) diff --git a/configure b/configure index c51192f12c8223..a3f14fb785b3e9 100755 --- a/configure +++ b/configure @@ -22451,11 +22451,271 @@ fi # Put the nasty error message in config.log where it belongs echo "$LIBZSTD_PKG_ERRORS" >&5 - have_libzstd=no + + save_CFLAGS=$CFLAGS +save_CPPFLAGS=$CPPFLAGS +save_LDFLAGS=$LDFLAGS +save_LIBS=$LIBS + + + CPPFLAGS="$CPPFLAGS $LIBZSTD_CFLAGS" + LIBS="$LIBS $LIBZSTD_LIBS" + for ac_header in zstd.h zdict.h +do : + as_ac_Header=`printf "%s\n" "ac_cv_header_$ac_header" | sed "$as_sed_sh"` +ac_fn_c_check_header_compile "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default" +if eval test \"x\$"$as_ac_Header"\" = x"yes" +then : + cat >>confdefs.h <<_ACEOF +#define `printf "%s\n" "HAVE_$ac_header" | sed "$as_sed_cpp"` 1 +_ACEOF + + py_check_lib_save_LIBS=$LIBS +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ZDICT_finalizeDictionary in -lzstd" >&5 +printf %s "checking for ZDICT_finalizeDictionary in -lzstd... " >&6; } +if test ${ac_cv_lib_zstd_ZDICT_finalizeDictionary+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS +LIBS="-lzstd $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char ZDICT_finalizeDictionary (void); +int +main (void) +{ +return ZDICT_finalizeDictionary (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO" +then : + ac_cv_lib_zstd_ZDICT_finalizeDictionary=yes +else case e in #( + e) ac_cv_lib_zstd_ZDICT_finalizeDictionary=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_zstd_ZDICT_finalizeDictionary" >&5 +printf "%s\n" "$ac_cv_lib_zstd_ZDICT_finalizeDictionary" >&6; } +if test "x$ac_cv_lib_zstd_ZDICT_finalizeDictionary" = xyes +then : + + if test "$cross_compiling" = yes +then : + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "cannot run test program while cross compiling +See 'config.log' for more details" "$LINENO" 5; } +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include "zstd.h" +int +main (void) +{ + + #if ZSTD_VERSION_NUMBER < 10405 + exit(1); + #endif + + ; + return 0; +} + +_ACEOF +if ac_fn_c_try_run "$LINENO" +then : + have_libzstd=yes +else case e in #( + e) have_libzstd=no ;; +esac +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac +fi + + +else case e in #( + e) have_libzstd=no ;; +esac +fi + +LIBS=$py_check_lib_save_LIBS + + +else case e in #( + e) have_libzstd=no ;; +esac +fi + +done + if test "x$have_libzstd" = xyes +then : + + LIBZSTD_CFLAGS=${LIBZSTD_CFLAGS-""} + LIBZSTD_LIBS=${LIBZSTD_LIBS-"-lzstd"} + +fi + +CFLAGS=$save_CFLAGS +CPPFLAGS=$save_CPPFLAGS +LDFLAGS=$save_LDFLAGS +LIBS=$save_LIBS + + + elif test $pkg_failed = untried; then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } - have_libzstd=no + + save_CFLAGS=$CFLAGS +save_CPPFLAGS=$CPPFLAGS +save_LDFLAGS=$LDFLAGS +save_LIBS=$LIBS + + + CPPFLAGS="$CPPFLAGS $LIBZSTD_CFLAGS" + LIBS="$LIBS $LIBZSTD_LIBS" + for ac_header in zstd.h zdict.h +do : + as_ac_Header=`printf "%s\n" "ac_cv_header_$ac_header" | sed "$as_sed_sh"` +ac_fn_c_check_header_compile "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default" +if eval test \"x\$"$as_ac_Header"\" = x"yes" +then : + cat >>confdefs.h <<_ACEOF +#define `printf "%s\n" "HAVE_$ac_header" | sed "$as_sed_cpp"` 1 +_ACEOF + + py_check_lib_save_LIBS=$LIBS +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ZDICT_finalizeDictionary in -lzstd" >&5 +printf %s "checking for ZDICT_finalizeDictionary in -lzstd... " >&6; } +if test ${ac_cv_lib_zstd_ZDICT_finalizeDictionary+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) ac_check_lib_save_LIBS=$LIBS +LIBS="-lzstd $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char ZDICT_finalizeDictionary (void); +int +main (void) +{ +return ZDICT_finalizeDictionary (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO" +then : + ac_cv_lib_zstd_ZDICT_finalizeDictionary=yes +else case e in #( + e) ac_cv_lib_zstd_ZDICT_finalizeDictionary=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_zstd_ZDICT_finalizeDictionary" >&5 +printf "%s\n" "$ac_cv_lib_zstd_ZDICT_finalizeDictionary" >&6; } +if test "x$ac_cv_lib_zstd_ZDICT_finalizeDictionary" = xyes +then : + + if test "$cross_compiling" = yes +then : + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} +as_fn_error $? "cannot run test program while cross compiling +See 'config.log' for more details" "$LINENO" 5; } +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include "zstd.h" +int +main (void) +{ + + #if ZSTD_VERSION_NUMBER < 10405 + exit(1); + #endif + + ; + return 0; +} + +_ACEOF +if ac_fn_c_try_run "$LINENO" +then : + have_libzstd=yes +else case e in #( + e) have_libzstd=no ;; +esac +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac +fi + + +else case e in #( + e) have_libzstd=no ;; +esac +fi + +LIBS=$py_check_lib_save_LIBS + + +else case e in #( + e) have_libzstd=no ;; +esac +fi + +done + if test "x$have_libzstd" = xyes +then : + + LIBZSTD_CFLAGS=${LIBZSTD_CFLAGS-""} + LIBZSTD_LIBS=${LIBZSTD_LIBS-"-lzstd"} + +fi + +CFLAGS=$save_CFLAGS +CPPFLAGS=$save_CPPFLAGS +LDFLAGS=$save_LDFLAGS +LIBS=$save_LIBS + + + else LIBZSTD_CFLAGS=$pkg_cv_LIBZSTD_CFLAGS LIBZSTD_LIBS=$pkg_cv_LIBZSTD_LIBS diff --git a/configure.ac b/configure.ac index a7b2f62579b0e9..33acec12aed6e9 100644 --- a/configure.ac +++ b/configure.ac @@ -5387,7 +5387,26 @@ PKG_CHECK_MODULES([LIBLZMA], [liblzma], [have_liblzma=yes], [ ]) dnl zstd 1.4.5 stabilised ZDICT_finalizeDictionary -PKG_CHECK_MODULES([LIBZSTD], [libzstd >= 1.4.5], [have_libzstd=yes], [have_libzstd=no]) +PKG_CHECK_MODULES([LIBZSTD], [libzstd >= 1.4.5], [have_libzstd=yes], [ + WITH_SAVE_ENV([ + CPPFLAGS="$CPPFLAGS $LIBZSTD_CFLAGS" + LIBS="$LIBS $LIBZSTD_LIBS" + AC_CHECK_HEADERS([zstd.h zdict.h], [ + PY_CHECK_LIB([zstd], [ZDICT_finalizeDictionary], [ + AC_RUN_IFELSE([AC_LANG_PROGRAM([@%:@include "zstd.h"], [ + #if ZSTD_VERSION_NUMBER < 10405 + exit(1); + #endif + ]) + ], [have_libzstd=yes], [have_libzstd=no]) + ], [have_libzstd=no]) + ], [have_libzstd=no]) + AS_VAR_IF([have_libzstd], [yes], [ + LIBZSTD_CFLAGS=${LIBZSTD_CFLAGS-""} + LIBZSTD_LIBS=${LIBZSTD_LIBS-"-lzstd"} + ]) + ]) +]) dnl PY_CHECK_NETDB_FUNC(FUNCTION) AC_DEFUN([PY_CHECK_NETDB_FUNC], [PY_CHECK_FUNC([$1], [@%:@include ])]) diff --git a/pyconfig.h.in b/pyconfig.h.in index 7586ad3f266705..c91facbedf94e5 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1630,12 +1630,18 @@ /* Define to 1 if you have the 'writev' function. */ #undef HAVE_WRITEV +/* Define to 1 if you have the header file. */ +#undef HAVE_ZDICT_H + /* Define if the zlib library has inflateCopy */ #undef HAVE_ZLIB_COPY /* Define to 1 if you have the header file. */ #undef HAVE_ZLIB_H +/* Define to 1 if you have the header file. */ +#undef HAVE_ZSTD_H + /* Define to 1 if you have the '_getpty' function. */ #undef HAVE__GETPTY From f7fd640cf7982becf6746270fbc2afd52e706b35 Mon Sep 17 00:00:00 2001 From: Emma Harper Smith Date: Sun, 11 May 2025 18:46:52 -0700 Subject: [PATCH 3/8] Fix configure from bad merge --- configure | 156 ------------------------------------------------------ 1 file changed, 156 deletions(-) diff --git a/configure b/configure index 1d23b771d326ff..2649a800f91a48 100755 --- a/configure +++ b/configure @@ -22459,28 +22459,6 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $LIBZSTD_CFLAGS" -<<<<<<< HEAD - LIBS="$LIBS $LIBZSTD_LIBS" - for ac_header in zstd.h zdict.h -do : - as_ac_Header=`printf "%s\n" "ac_cv_header_$ac_header" | sed "$as_sed_sh"` -ac_fn_c_check_header_compile "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default" -if eval test \"x\$"$as_ac_Header"\" = x"yes" -then : - cat >>confdefs.h <<_ACEOF -#define `printf "%s\n" "HAVE_$ac_header" | sed "$as_sed_cpp"` 1 -_ACEOF - - py_check_lib_save_LIBS=$LIBS -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ZDICT_finalizeDictionary in -lzstd" >&5 -printf %s "checking for ZDICT_finalizeDictionary in -lzstd... " >&6; } -if test ${ac_cv_lib_zstd_ZDICT_finalizeDictionary+y} -then : - printf %s "(cached) " >&6 -else case e in #( - e) ac_check_lib_save_LIBS=$LIBS -LIBS="-lzstd $LIBS" -======= CFLAGS="$CFLAGS $LIBZSTD_CFLAGS" LIBS="$LIBS $LIBZSTD_LIBS" { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for library containing ZDICT_finalizeDictionary" >&5 @@ -22490,7 +22468,6 @@ then : printf %s "(cached) " >&6 else case e in #( e) ac_func_search_save_LIBS=$LIBS ->>>>>>> 0eb448cae5e9008f815204d8b46bfd7cd641a152 cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -22512,35 +22489,6 @@ return ZDICT_finalizeDictionary (); return 0; } _ACEOF -<<<<<<< HEAD -if ac_fn_c_try_link "$LINENO" -then : - ac_cv_lib_zstd_ZDICT_finalizeDictionary=yes -else case e in #( - e) ac_cv_lib_zstd_ZDICT_finalizeDictionary=no ;; -esac -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS ;; -esac -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_zstd_ZDICT_finalizeDictionary" >&5 -printf "%s\n" "$ac_cv_lib_zstd_ZDICT_finalizeDictionary" >&6; } -if test "x$ac_cv_lib_zstd_ZDICT_finalizeDictionary" = xyes -then : - - if test "$cross_compiling" = yes -then : - { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in '$ac_pwd':" >&5 -printf "%s\n" "$as_me: error: in '$ac_pwd':" >&2;} -as_fn_error $? "cannot run test program while cross compiling -See 'config.log' for more details" "$LINENO" 5; } -else case e in #( - e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include "zstd.h" -======= for ac_lib in '' zstd do if test -z "$ac_lib"; then @@ -22584,17 +22532,12 @@ printf %s "checking ZSTD_VERSION_NUMBER >= 1.4.5... " >&6; } /* end confdefs.h. */ #include "zstd.h" ->>>>>>> 0eb448cae5e9008f815204d8b46bfd7cd641a152 int main (void) { #if ZSTD_VERSION_NUMBER < 10405 -<<<<<<< HEAD - exit(1); -======= # error "zstd version is too old" ->>>>>>> 0eb448cae5e9008f815204d8b46bfd7cd641a152 #endif ; @@ -22602,29 +22545,6 @@ main (void) } _ACEOF -<<<<<<< HEAD -if ac_fn_c_try_run "$LINENO" -then : - have_libzstd=yes -else case e in #( - e) have_libzstd=no ;; -esac -fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext ;; -esac -fi - - -else case e in #( - e) have_libzstd=no ;; -esac -fi - -LIBS=$py_check_lib_save_LIBS - - -======= if ac_fn_c_try_compile "$LINENO" then : @@ -22640,33 +22560,12 @@ then : #define `printf "%s\n" "HAVE_$ac_header" | sed "$as_sed_cpp"` 1 _ACEOF have_libzstd=yes ->>>>>>> 0eb448cae5e9008f815204d8b46bfd7cd641a152 else case e in #( e) have_libzstd=no ;; esac fi done -<<<<<<< HEAD - if test "x$have_libzstd" = xyes -then : - - LIBZSTD_CFLAGS=${LIBZSTD_CFLAGS-""} - LIBZSTD_LIBS=${LIBZSTD_LIBS-"-lzstd"} - -fi - -CFLAGS=$save_CFLAGS -CPPFLAGS=$save_CPPFLAGS -LDFLAGS=$save_LDFLAGS -LIBS=$save_LIBS - - - -elif test $pkg_failed = untried; then - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 -printf "%s\n" "no" >&6; } -======= else case e in #( e) @@ -22701,7 +22600,6 @@ LIBS=$save_LIBS elif test $pkg_failed = untried; then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } ->>>>>>> 0eb448cae5e9008f815204d8b46bfd7cd641a152 save_CFLAGS=$CFLAGS save_CPPFLAGS=$CPPFLAGS @@ -22710,28 +22608,6 @@ save_LIBS=$LIBS CPPFLAGS="$CPPFLAGS $LIBZSTD_CFLAGS" -<<<<<<< HEAD - LIBS="$LIBS $LIBZSTD_LIBS" - for ac_header in zstd.h zdict.h -do : - as_ac_Header=`printf "%s\n" "ac_cv_header_$ac_header" | sed "$as_sed_sh"` -ac_fn_c_check_header_compile "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default" -if eval test \"x\$"$as_ac_Header"\" = x"yes" -then : - cat >>confdefs.h <<_ACEOF -#define `printf "%s\n" "HAVE_$ac_header" | sed "$as_sed_cpp"` 1 -_ACEOF - - py_check_lib_save_LIBS=$LIBS -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ZDICT_finalizeDictionary in -lzstd" >&5 -printf %s "checking for ZDICT_finalizeDictionary in -lzstd... " >&6; } -if test ${ac_cv_lib_zstd_ZDICT_finalizeDictionary+y} -then : - printf %s "(cached) " >&6 -else case e in #( - e) ac_check_lib_save_LIBS=$LIBS -LIBS="-lzstd $LIBS" -======= CFLAGS="$CFLAGS $LIBZSTD_CFLAGS" LIBS="$LIBS $LIBZSTD_LIBS" { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for library containing ZDICT_finalizeDictionary" >&5 @@ -22741,7 +22617,6 @@ then : printf %s "(cached) " >&6 else case e in #( e) ac_func_search_save_LIBS=$LIBS ->>>>>>> 0eb448cae5e9008f815204d8b46bfd7cd641a152 cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -22811,11 +22686,7 @@ main (void) { #if ZSTD_VERSION_NUMBER < 10405 -<<<<<<< HEAD - exit(1); -======= # error "zstd version is too old" ->>>>>>> 0eb448cae5e9008f815204d8b46bfd7cd641a152 #endif ; @@ -22823,29 +22694,6 @@ main (void) } _ACEOF -<<<<<<< HEAD -if ac_fn_c_try_run "$LINENO" -then : - have_libzstd=yes -else case e in #( - e) have_libzstd=no ;; -esac -fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext ;; -esac -fi - - -else case e in #( - e) have_libzstd=no ;; -esac -fi - -LIBS=$py_check_lib_save_LIBS - - -======= if ac_fn_c_try_compile "$LINENO" then : @@ -22861,15 +22709,12 @@ then : #define `printf "%s\n" "HAVE_$ac_header" | sed "$as_sed_cpp"` 1 _ACEOF have_libzstd=yes ->>>>>>> 0eb448cae5e9008f815204d8b46bfd7cd641a152 else case e in #( e) have_libzstd=no ;; esac fi done -<<<<<<< HEAD -======= else case e in #( e) @@ -22886,7 +22731,6 @@ else case e in #( esac fi ->>>>>>> 0eb448cae5e9008f815204d8b46bfd7cd641a152 if test "x$have_libzstd" = xyes then : From d81422e7d1836bf1a22281333da0879dd61f4186 Mon Sep 17 00:00:00 2001 From: Emma Harper Smith Date: Sun, 11 May 2025 18:54:32 -0700 Subject: [PATCH 4/8] Fix function renamed in C --- Lib/compression/zstd/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/compression/zstd/__init__.py b/Lib/compression/zstd/__init__.py index ee153ea6bb6f36..b3060607a75f27 100644 --- a/Lib/compression/zstd/__init__.py +++ b/Lib/compression/zstd/__init__.py @@ -128,8 +128,8 @@ def finalize_dict(zstd_dict, /, samples, dict_size, level): if not chunks: raise ValueError("The samples are empty content, can't finalize the " "dictionary.") - dict_content = _zstd._finalize_dict(zstd_dict.dict_content, chunks, - chunk_sizes, dict_size, level) + dict_content = _zstd.finalize_dict(zstd_dict.dict_content, chunks, + chunk_sizes, dict_size, level) return ZstdDict(dict_content) From 9f8162a460c028a3eab3d36d1cdc87b36fbb5580 Mon Sep 17 00:00:00 2001 From: Emma Harper Smith Date: Tue, 13 May 2025 15:41:40 -0700 Subject: [PATCH 5/8] Move compression packages to modules --- Lib/compression/{bz2/__init__.py => bz2.py} | 0 Lib/compression/{gzip/__init__.py => gzip.py} | 0 Lib/compression/{lzma/__init__.py => lzma.py} | 0 Lib/compression/{zlib/__init__.py => zlib.py} | 0 Makefile.pre.in | 3 +-- 5 files changed, 1 insertion(+), 2 deletions(-) rename Lib/compression/{bz2/__init__.py => bz2.py} (100%) rename Lib/compression/{gzip/__init__.py => gzip.py} (100%) rename Lib/compression/{lzma/__init__.py => lzma.py} (100%) rename Lib/compression/{zlib/__init__.py => zlib.py} (100%) diff --git a/Lib/compression/bz2/__init__.py b/Lib/compression/bz2.py similarity index 100% rename from Lib/compression/bz2/__init__.py rename to Lib/compression/bz2.py diff --git a/Lib/compression/gzip/__init__.py b/Lib/compression/gzip.py similarity index 100% rename from Lib/compression/gzip/__init__.py rename to Lib/compression/gzip.py diff --git a/Lib/compression/lzma/__init__.py b/Lib/compression/lzma.py similarity index 100% rename from Lib/compression/lzma/__init__.py rename to Lib/compression/lzma.py diff --git a/Lib/compression/zlib/__init__.py b/Lib/compression/zlib.py similarity index 100% rename from Lib/compression/zlib/__init__.py rename to Lib/compression/zlib.py diff --git a/Makefile.pre.in b/Makefile.pre.in index 452c4ad35ac96c..a5c35fb1c4ccd5 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2512,8 +2512,7 @@ maninstall: altmaninstall XMLLIBSUBDIRS= xml xml/dom xml/etree xml/parsers xml/sax LIBSUBDIRS= asyncio \ collections \ - compression compression/bz2 compression/gzip compression/zstd \ - compression/lzma compression/zlib compression/_common \ + compression compression/zstd compression/_common \ concurrent concurrent/futures \ csv \ ctypes ctypes/macholib \ From 3f39ea4263a0f0ff2352c26d593c2e1b5fc5f13c Mon Sep 17 00:00:00 2001 From: Emma Harper Smith Date: Tue, 13 May 2025 15:43:16 -0700 Subject: [PATCH 6/8] Use single quotes over double quotes This better matches CPython's style --- Lib/compression/zstd/__init__.py | 64 +++++++++++++-------------- Lib/compression/zstd/_zstdfile.py | 72 +++++++++++++++---------------- 2 files changed, 68 insertions(+), 68 deletions(-) diff --git a/Lib/compression/zstd/__init__.py b/Lib/compression/zstd/__init__.py index b3060607a75f27..84b25914b0aa93 100644 --- a/Lib/compression/zstd/__init__.py +++ b/Lib/compression/zstd/__init__.py @@ -2,28 +2,28 @@ __all__ = ( # compression.zstd - "COMPRESSION_LEVEL_DEFAULT", - "compress", - "CompressionParameter", - "decompress", - "DecompressionParameter", - "finalize_dict", - "get_frame_info", - "Strategy", - "train_dict", + 'COMPRESSION_LEVEL_DEFAULT', + 'compress', + 'CompressionParameter', + 'decompress', + 'DecompressionParameter', + 'finalize_dict', + 'get_frame_info', + 'Strategy', + 'train_dict', # compression.zstd._zstdfile - "open", - "ZstdFile", + 'open', + 'ZstdFile', # _zstd - "get_frame_size", - "zstd_version", - "zstd_version_info", - "ZstdCompressor", - "ZstdDecompressor", - "ZstdDict", - "ZstdError", + 'get_frame_size', + 'zstd_version', + 'zstd_version_info', + 'ZstdCompressor', + 'ZstdDecompressor', + 'ZstdDict', + 'ZstdError', ) import _zstd @@ -44,15 +44,15 @@ class FrameInfo: """Information about a Zstandard frame.""" - __slots__ = "decompressed_size", "dictionary_id" + __slots__ = 'decompressed_size', 'dictionary_id' def __init__(self, decompressed_size, dictionary_id): - super().__setattr__("decompressed_size", decompressed_size) - super().__setattr__("dictionary_id", dictionary_id) + super().__setattr__('decompressed_size', decompressed_size) + super().__setattr__('dictionary_id', dictionary_id) def __repr__(self): - return (f"FrameInfo(decompressed_size={self.decompressed_size}, " - f"dictionary_id={self.dictionary_id})") + return (f'FrameInfo(decompressed_size={self.decompressed_size}, ' + f'dictionary_id={self.dictionary_id})') def __setattr__(self, name, _): raise AttributeError(f"can't set attribute {name!r}") @@ -85,10 +85,10 @@ def train_dict(samples, dict_size): """ if not isinstance(dict_size, int): ds_cls = type(dict_size).__qualname__ - raise TypeError(f"dict_size must be an int object, not {ds_cls!r}.") + raise TypeError(f'dict_size must be an int object, not {ds_cls!r}.') samples = tuple(samples) - chunks = b"".join(samples) + chunks = b''.join(samples) chunk_sizes = tuple(_nbytes(sample) for sample in samples) if not chunks: raise ValueError("samples contained no data; can't train dictionary.") @@ -116,14 +116,14 @@ def finalize_dict(zstd_dict, /, samples, dict_size, level): """ if not isinstance(zstd_dict, ZstdDict): - raise TypeError("zstd_dict argument should be a ZstdDict object.") + raise TypeError('zstd_dict argument should be a ZstdDict object.') if not isinstance(dict_size, int): - raise TypeError("dict_size argument should be an int object.") + raise TypeError('dict_size argument should be an int object.') if not isinstance(level, int): - raise TypeError("level argument should be an int object.") + raise TypeError('level argument should be an int object.') samples = tuple(samples) - chunks = b"".join(samples) + chunks = b''.join(samples) chunk_sizes = tuple(_nbytes(sample) for sample in samples) if not chunks: raise ValueError("The samples are empty content, can't finalize the " @@ -164,12 +164,12 @@ def decompress(data, zstd_dict=None, options=None): decomp = ZstdDecompressor(options=options, zstd_dict=zstd_dict) results.append(decomp.decompress(data)) if not decomp.eof: - raise ZstdError("Compressed data ended before the " - "end-of-stream marker was reached") + raise ZstdError('Compressed data ended before the ' + 'end-of-stream marker was reached') data = decomp.unused_data if not data: break - return b"".join(results) + return b''.join(results) class CompressionParameter(enum.IntEnum): diff --git a/Lib/compression/zstd/_zstdfile.py b/Lib/compression/zstd/_zstdfile.py index 3b4938a328de65..8770e576f509f4 100644 --- a/Lib/compression/zstd/_zstdfile.py +++ b/Lib/compression/zstd/_zstdfile.py @@ -4,7 +4,7 @@ ZSTD_DStreamOutSize) from compression._common import _streams -__all__ = ("ZstdFile", "open") +__all__ = ('ZstdFile', 'open') _MODE_CLOSED = 0 _MODE_READ = 1 @@ -31,15 +31,15 @@ class ZstdFile(_streams.BaseStream): FLUSH_BLOCK = ZstdCompressor.FLUSH_BLOCK FLUSH_FRAME = ZstdCompressor.FLUSH_FRAME - def __init__(self, file, /, mode="r", *, + def __init__(self, file, /, mode='r', *, level=None, options=None, zstd_dict=None): """Open a Zstandard compressed file in binary mode. *file* can be either an file-like object, or a file name to open. - *mode* can be "r" for reading (default), "w" for (over)writing, "x" for - creating exclusively, or "a" for appending. These can equivalently be - given as "rb", "wb", "xb" and "ab" respectively. + *mode* can be 'r' for reading (default), 'w' for (over)writing, 'x' for + creating exclusively, or 'a' for appending. These can equivalently be + given as 'rb', 'wb', 'xb' and 'ab' respectively. *level* is an optional int specifying the compression level to use, or COMPRESSION_LEVEL_DEFAULT if not given. @@ -57,33 +57,33 @@ def __init__(self, file, /, mode="r", *, self._buffer = None if not isinstance(mode, str): - raise ValueError("mode must be a str") + raise ValueError('mode must be a str') if options is not None and not isinstance(options, dict): - raise TypeError("options must be a dict or None") - mode = mode.removesuffix("b") # handle rb, wb, xb, ab - if mode == "r": + raise TypeError('options must be a dict or None') + mode = mode.removesuffix('b') # handle rb, wb, xb, ab + if mode == 'r': if level is not None: - raise TypeError("level is illegal in read mode") + raise TypeError('level is illegal in read mode') self._mode = _MODE_READ - elif mode in {"w", "a", "x"}: + elif mode in {'w', 'a', 'x'}: if level is not None and not isinstance(level, int): - raise TypeError("level must be int or None") + raise TypeError('level must be int or None') self._mode = _MODE_WRITE self._compressor = ZstdCompressor(level=level, options=options, zstd_dict=zstd_dict) self._pos = 0 else: - raise ValueError(f"Invalid mode: {mode!r}") + raise ValueError(f'Invalid mode: {mode!r}') if isinstance(file, (str, bytes, PathLike)): - self._fp = io.open(file, f"{mode}b") + self._fp = io.open(file, f'{mode}b') self._close_fp = True - elif ((mode == "r" and hasattr(file, "read")) - or (mode != "r" and hasattr(file, "write"))): + elif ((mode == 'r' and hasattr(file, 'read')) + or (mode != 'r' and hasattr(file, 'write'))): self._fp = file else: - raise TypeError("file must be a file-like object " - "or a str, bytes, or PathLike object") + raise TypeError('file must be a file-like object ' + 'or a str, bytes, or PathLike object') if self._mode == _MODE_READ: raw = _streams.DecompressReader( @@ -104,7 +104,7 @@ def close(self): return try: if self._mode == _MODE_READ: - if getattr(self, "_buffer", None): + if getattr(self, '_buffer', None): self._buffer.close() self._buffer = None elif self._mode == _MODE_WRITE: @@ -151,22 +151,22 @@ def flush(self, mode=FLUSH_BLOCK): return self._check_not_closed() if mode not in {self.FLUSH_BLOCK, self.FLUSH_FRAME}: - raise ValueError("Invalid mode argument, expected either " - "ZstdFile.FLUSH_FRAME or " - "ZstdFile.FLUSH_BLOCK") + raise ValueError('Invalid mode argument, expected either ' + 'ZstdFile.FLUSH_FRAME or ' + 'ZstdFile.FLUSH_BLOCK') if self._compressor.last_mode == mode: return # Flush zstd block/frame, and write. data = self._compressor.flush(mode) self._fp.write(data) - if hasattr(self._fp, "flush"): + if hasattr(self._fp, 'flush'): self._fp.flush() def read(self, size=-1): """Read up to size uncompressed bytes from the file. If size is negative or omitted, read until EOF is reached. - Returns b"" if the file is already at EOF. + Returns b'' if the file is already at EOF. """ if size is None: size = -1 @@ -178,7 +178,7 @@ def read1(self, size=-1): making multiple reads from the underlying stream. Reads up to a buffer's worth of data if size is negative. - Returns b"" if the file is at EOF. + Returns b'' if the file is at EOF. """ self._check_can_read() if size < 0: @@ -271,7 +271,7 @@ def name(self): @property def mode(self): - return "wb" if self._mode == _MODE_WRITE else "rb" + return 'wb' if self._mode == _MODE_WRITE else 'rb' @property def closed(self): @@ -293,7 +293,7 @@ def writable(self): return self._mode == _MODE_WRITE -def open(file, /, mode="rb", *, level=None, options=None, zstd_dict=None, +def open(file, /, mode='rb', *, level=None, options=None, zstd_dict=None, encoding=None, errors=None, newline=None): """Open a Zstandard compressed file in binary or text mode. @@ -301,8 +301,8 @@ def open(file, /, mode="rb", *, level=None, options=None, zstd_dict=None, in which case the named file is opened, or it can be an existing file object to read from or write to. - The mode parameter can be "r", "rb" (default), "w", "wb", "x", "xb", "a", - "ab" for binary mode, or "rt", "wt", "xt", "at" for text mode. + The mode parameter can be 'r', 'rb' (default), 'w', 'wb', 'x', 'xb', 'a', + 'ab' for binary mode, or 'rt', 'wt', 'xt', 'at' for text mode. The level, options, and zstd_dict parameters specify the settings the same as ZstdFile. @@ -323,19 +323,19 @@ def open(file, /, mode="rb", *, level=None, options=None, zstd_dict=None, behavior, and line ending(s). """ - text_mode = "t" in mode - mode = mode.replace("t", "") + text_mode = 't' in mode + mode = mode.replace('t', '') if text_mode: - if "b" in mode: - raise ValueError(f"Invalid mode: {mode!r}") + if 'b' in mode: + raise ValueError(f'Invalid mode: {mode!r}') else: if encoding is not None: - raise ValueError("Argument 'encoding' not supported in binary mode") + raise ValueError('Argument "encoding" not supported in binary mode') if errors is not None: - raise ValueError("Argument 'errors' not supported in binary mode") + raise ValueError('Argument "errors" not supported in binary mode') if newline is not None: - raise ValueError("Argument 'newline' not supported in binary mode") + raise ValueError('Argument "newline" not supported in binary mode') binary_file = ZstdFile(file, mode, level=level, options=options, zstd_dict=zstd_dict) From 393e47b9b5154af1a5dd3ebf81cc86a9cf943af5 Mon Sep 17 00:00:00 2001 From: Emma Harper Smith Date: Tue, 13 May 2025 15:44:36 -0700 Subject: [PATCH 7/8] Add trailing comma to tarfile module This was missed when adding Zstandard support. --- Lib/tarfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/tarfile.py b/Lib/tarfile.py index c0f5a609b9f42f..13889d768021b1 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -2065,7 +2065,7 @@ def zstopen(cls, name, mode="r", fileobj=None, level=None, options=None, "gz": "gzopen", # gzip compressed tar "bz2": "bz2open", # bzip2 compressed tar "xz": "xzopen", # lzma compressed tar - "zst": "zstopen" # zstd compressed tar + "zst": "zstopen", # zstd compressed tar } #-------------------------------------------------------------------------- From 491421249fa33f5fcac76bcad84c3da766326d10 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Date: Wed, 14 May 2025 11:49:00 +0100 Subject: [PATCH 8/8] Update Makefile.pre.in --- Makefile.pre.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index a5c35fb1c4ccd5..3ab7c3d6c48ad9 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2512,7 +2512,7 @@ maninstall: altmaninstall XMLLIBSUBDIRS= xml xml/dom xml/etree xml/parsers xml/sax LIBSUBDIRS= asyncio \ collections \ - compression compression/zstd compression/_common \ + compression compression/_common compression/zstd \ concurrent concurrent/futures \ csv \ ctypes ctypes/macholib \