support plain tar or no compression #110

Open

opened

on May 8, 2021

Unless I'm misunderstanding something, currently one must use gz or xz as compression formats. Because of this, during the building of rust itself, the fabricate process uses 100% CPU (aka 1 core only) and thus wastes a lot of time trying to compress to one or both of the two formats (specified in config.toml)

On systems where the filesystem is btrfs and using zstd compression already, some may wish to use no compression at all, thus just .tar (instead of .tar.gz or .tar.xz would be preferred). Maybe such people don't care about any extra used space(presumably filesystem compression if poorer than .gz or .xz ?), or they do care about how much time rust compilation takes (it would take less time this way with avoiding doing any compression especially since it's using only one CPU core).

Either way, I'd like to suggest supporting no compression at all - which would probably mean simply just .tar

In the past I used to hack a patch to do this (.tar instead of .tar.gz) and it worked just fine, but since v1.48.0 rust's changed that code a bit in 1.52.0 and I'm unable to understand how to change the patch to apply it, so far.

acheechee

Author

EDIT: a 10 minute save with this patch (down to 45mins from 56mins) to emerge 'rust' on Gentoo.

I've decided that it's unlikely this will be implemented soon enough so I went back to re-make my hacky patch(that avoids compression and keeps the files as .tar) to work for 1.52.0 rust version, and this is it:

Index: /var/tmp/portage/dev-lang/rust-1.52.0/work/rustc-1.52.0-src/src/tools/rust-installer/src/tarballer.rs
===================================================================
--- rustc-1.52.0-src/src/tools/rust-installer/src/tarballer.rs
+++ rustc-1.52.0-src/src/tools/rust-installer/src/tarballer.rs
@@ -6,7 +6,7 @@ use tar::{Builder, Header};
 use walkdir::WalkDir;
 
 use crate::{
-    compression::{CombinedEncoder, CompressionFormats},
+    //compression::{CombinedEncoder, CompressionFormats},
     util::*,
 };
 
@@ -22,8 +22,8 @@ actor! {
         /// The folder in which the input is to be found.
         work_dir: String = "./workdir",
 
-        /// The formats used to compress the tarball.
-        compression_formats: CompressionFormats = CompressionFormats::default(),
+        ///// The formats used to compress the tarball.
+        //compression_formats: CompressionFormats = CompressionFormats::default(),
     }
 }
 
@@ -31,12 +31,13 @@ impl Tarballer {
     /// Generates the actual tarballs
     pub fn run(self) -> Result<()> {
         let tarball_name = self.output.clone() + ".tar";
-        let encoder = CombinedEncoder::new(
-            self.compression_formats
-                .iter()
-                .map(|f| f.encode(&tarball_name))
-                .collect::<Result<Vec<_>>>()?,
-        );
+        let encoder = ensure_new_file(tarball_name)?;
+        //let encoder = CombinedEncoder::new(
+        //    self.compression_formats
+        //        .iter()
+        //        .map(|f| f.encode(&tarball_name))
+        //        .collect::<Result<Vec<_>>>()?,
+        //);
 
         // Sort files by their suffix, to group files with the same name from
         // different locations (likely identical) and files with the same
#@@ -51,7 +52,7 @@ impl Tarballer {
#         let mut builder = Builder::new(buf);
# 
#         let pool = rayon::ThreadPoolBuilder::new()
#-            .num_threads(2)
#+            .num_threads(12)
#             .build()
#             .unwrap();
#         pool.install(move || {
@@ -71,8 +72,8 @@ impl Tarballer {
                 .context("failed to finish writing .tar stream")?
                 .into_inner()
                 .ok()
-                .unwrap()
-                .finish()?;
+                .unwrap();
+                //.finish()?;
 
             Ok(())
         })
Index: /var/tmp/portage/dev-lang/rust-1.52.0/work/rustc-1.52.0-src/src/tools/rust-installer/src/combiner.rs
===================================================================
--- rustc-1.52.0-src/src/tools/rust-installer/src/combiner.rs
+++ rustc-1.52.0-src/src/tools/rust-installer/src/combiner.rs
@@ -1,7 +1,7 @@
 use super::Scripter;
 use super::Tarballer;
 use crate::{
-    compression::{CompressionFormat, CompressionFormats},
+    //compression::{CompressionFormat, CompressionFormats},
     util::*,
 };
 use anyhow::{bail, Context, Result};
@@ -39,8 +39,8 @@ actor! {
         /// The location to put the final image and tarball.
         output_dir: String = "./dist",
 
-        /// The formats used to compress the tarball
-        compression_formats: CompressionFormats = CompressionFormats::default(),
+        // /// The formats used to compress the tarball
+        //compression_formats: CompressionFormats = CompressionFormats::default(),
     }
 }
 
@@ -64,11 +64,11 @@ impl Combiner {
             .filter(|s| !s.is_empty())
         {
             // Extract the input tarballs
-            let compression =
-                CompressionFormat::detect_from_path(input_tarball).ok_or_else(|| {
-                    anyhow::anyhow!("couldn't figure out the format of {}", input_tarball)
-                })?;
-            Archive::new(compression.decode(input_tarball)?)
+            //let compression =
+            //    CompressionFormat::detect_from_path(input_tarball).ok_or_else(|| {
+            //        anyhow::anyhow!("couldn't figure out the format of {}", input_tarball)
+            /*    })?;*/ let input_tarball=input_tarball.trim_end_matches(".gz").trim_end_matches(".xz");
+            Archive::new(open_file(&input_tarball)?)//compression.decode(input_tarball)?)
                 .unpack(&self.work_dir)
                 .with_context(|| {
                     format!(
#@@ -68,6 +68,7 @@ impl Combiner {
#             //    CompressionFormat::detect_from_path(input_tarball).ok_or_else(|| {
#             //        anyhow::anyhow!("couldn't figure out the format of {}", input_tarball)
#             //    })?;
#+            let input_tarball=input_tarball.trim_end_matches(".gz").trim_end_matches(".xz");
#             Archive::new(open_file(&input_tarball)?)//compression.decode(input_tarball)?)
#                 .unpack(&self.work_dir)
#                 .with_context(|| {
@@ -78,7 +78,8 @@ impl Combiner {
                 })?;
 
             let pkg_name =
-                input_tarball.trim_end_matches(&format!(".tar.{}", compression.extension()));
+                input_tarball.trim_end_matches(".tar");
+                //input_tarball.trim_end_matches(&format!(".tar.{}", compression.extension()));
             let pkg_name = Path::new(pkg_name).file_name().unwrap();
             let pkg_dir = Path::new(&self.work_dir).join(&pkg_name);
 
@@ -142,8 +143,8 @@ impl Combiner {
         tarballer
             .work_dir(self.work_dir)
             .input(self.package_name)
-            .output(path_to_str(&output)?.into())
-            .compression_formats(self.compression_formats.clone());
+            .output(path_to_str(&output)?.into());
+            //.compression_formats(self.compression_formats.clone());
         tarballer.run()?;
 
         Ok(())
Index: /var/tmp/portage/dev-lang/rust-1.52.0/work/rustc-1.52.0-src/src/tools/rust-installer/src/generator.rs
===================================================================
--- rustc-1.52.0-src/src/tools/rust-installer/src/generator.rs
+++ rustc-1.52.0-src/src/tools/rust-installer/src/generator.rs
@@ -42,8 +42,8 @@ actor! {
         /// The location to put the final image and tarball
         output_dir: String = "./dist",
 
-        /// The formats used to compress the tarball
-        compression_formats: CompressionFormats = CompressionFormats::default(),
+        // /// The formats used to compress the tarball
+        // compression_formats: CompressionFormats = CompressionFormats::default(),
     }
 }
 
@@ -99,8 +99,8 @@ impl Generator {
         tarballer
             .work_dir(self.work_dir)
             .input(self.package_name)
-            .output(path_to_str(&output)?.into())
-            .compression_formats(self.compression_formats.clone());
+            .output(path_to_str(&output)?.into());
+            //.compression_formats(self.compression_formats.clone());
         tarballer.run()?;
 
         Ok(())
Index: /var/tmp/portage/dev-lang/rust-1.52.0/work/rustc-1.52.0-src/src/tools/rust-installer/src/main.rs
===================================================================
--- rustc-1.52.0-src/src/tools/rust-installer/src/main.rs
+++ rustc-1.52.0-src/src/tools/rust-installer/src/main.rs
@@ -41,7 +41,7 @@ fn combine(matches: &ArgMatches<'_>) ->
         "non-installed-overlay" => non_installed_overlay,
         "work-dir" => work_dir,
         "output-dir" => output_dir,
-        "compression-formats" => compression_formats,
+        //"compression-formats" => compression_formats,
     });
 
     combiner.run().context("failed to combine installers")?;
@@ -61,7 +61,7 @@ fn generate(matches: &ArgMatches<'_>) ->
         "image-dir" => image_dir,
         "work-dir" => work_dir,
         "output-dir" => output_dir,
-        "compression-formats" => compression_formats,
+        //"compression-formats" => compression_formats,
     });
 
     generator.run().context("failed to generate installer")?;
@@ -88,7 +88,7 @@ fn tarball(matches: &ArgMatches<'_>) ->
         "input" => input,
         "output" => output,
         "work-dir" => work_dir,
-        "compression-formats" => compression_formats,
+        //"compression-formats" => compression_formats,
     });
 
     tarballer.run().context("failed to generate tarballs")?;
Index: /var/tmp/portage/dev-lang/rust-1.52.0/work/rustc-1.52.0-src/src/tools/rust-installer/src/util.rs
===================================================================
--- rustc-1.52.0-src/src/tools/rust-installer/src/util.rs
+++ rustc-1.52.0-src/src/tools/rust-installer/src/util.rs
@@ -73,6 +73,16 @@ pub fn create_new_file<P: AsRef<Path>>(p
     Ok(file)
 }
 
+/// Ensure a new file or truncates existing one
+pub fn ensure_new_file<P: AsRef<Path>>(path: P) -> Result<fs::File> {
+    let file = fs::OpenOptions::new()
+        .write(true)
+        .create(true).truncate(true)
+        .open(&path)
+        .with_context(|| format!("failed to truncate existing or create new  file '{}'", path.as_ref().display()))?;
+    Ok(file)
+}
+
 /// Wraps `fs::File::open()` with a nicer error message.
 pub fn open_file<P: AsRef<Path>>(path: P) -> Result<fs::File> {
     let file = fs::File::open(&path)

I'm using this on Gentoo if it matters... and with this ebuild:

# Copyright 1999-2021 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2

EAPI=7

PYTHON_COMPAT=( python3_{7..9} )

inherit bash-completion-r1 check-reqs estack flag-o-matic llvm multiprocessing \
	multilib-build python-any-r1 rust-toolchain toolchain-funcs verify-sig

if [[ ${PV} = *beta* ]]; then
	betaver=${PV//*beta}
	BETA_SNAPSHOT="${betaver:0:4}-${betaver:4:2}-${betaver:6:2}"
	MY_P="rustc-beta"
	SLOT="beta/${PV}"
	SRC="${BETA_SNAPSHOT}/rustc-beta-src.tar.xz -> rustc-${PV}-src.tar.xz"
else
	ABI_VER="$(ver_cut 1-2)"
	SLOT="stable/${ABI_VER}"
	MY_P="rustc-${PV}"
	SRC="${MY_P}-src.tar.xz"
	KEYWORDS="~amd64 ~arm ~arm64 ~ppc64 ~x86"
fi

RUST_STAGE0_VERSION="1.$(($(ver_cut 2) - 1)).0"

DESCRIPTION="Systems programming language from Mozilla"
HOMEPAGE="https://www.rust-lang.org/"

SRC_URI="
	https://static.rust-lang.org/dist/${SRC}
	verify-sig? ( https://static.rust-lang.org/dist/${SRC}.asc )
	!system-bootstrap? ( $(rust_all_arch_uris rust-${RUST_STAGE0_VERSION}) )
"

# keep in sync with llvm ebuild of the same version as bundled one.
ALL_LLVM_TARGETS=( AArch64 AMDGPU ARM AVR BPF Hexagon Lanai Mips MSP430
	NVPTX PowerPC RISCV Sparc SystemZ WebAssembly X86 XCore )
ALL_LLVM_TARGETS=( "${ALL_LLVM_TARGETS[@]/#/llvm_targets_}" )
LLVM_TARGET_USEDEPS=${ALL_LLVM_TARGETS[@]/%/(-)?}

LICENSE="|| ( MIT Apache-2.0 ) BSD-1 BSD-2 BSD-4 UoI-NCSA"

IUSE="clippy cpu_flags_x86_sse2 debug doc miri nightly parallel-compiler rls rustfmt system-bootstrap system-llvm test wasm ${ALL_LLVM_TARGETS[*]}"

# Please keep the LLVM dependency block separate. Since LLVM is slotted,
# we need to *really* make sure we're not pulling more than one slot
# simultaneously.

# How to use it:
# List all the working slots in LLVM_VALID_SLOTS, newest first.
LLVM_VALID_SLOTS=( 12 )
LLVM_MAX_SLOT="${LLVM_VALID_SLOTS[0]}"

# splitting usedeps needed to avoid CI/pkgcheck's UncheckableDep limitation
# (-) usedep needed because we may build with older llvm without that target
LLVM_DEPEND="|| ( "
for _s in ${LLVM_VALID_SLOTS[@]}; do
	LLVM_DEPEND+=" ( "
	for _x in ${ALL_LLVM_TARGETS[@]}; do
		LLVM_DEPEND+="
			${_x}? ( sys-devel/llvm:${_s}[${_x}(-)] )"
	done
	LLVM_DEPEND+=" )"
done
unset _s _x
LLVM_DEPEND+=" )
	<sys-devel/llvm-$(( LLVM_MAX_SLOT + 1 )):=
	wasm? ( sys-devel/lld )
"

# to bootstrap we need at least exactly previous version, or same.
# most of the time previous versions fail to bootstrap with newer
# for example 1.47.x, requires at least 1.46.x, 1.47.x is ok,
# but it fails to bootstrap with 1.48.x
# https://github.com/rust-lang/rust/blob/${PV}/src/stage0.txt
BOOTSTRAP_DEPEND="||
	(
		=dev-lang/rust-$(ver_cut 1).$(($(ver_cut 2) - 1))*
		=dev-lang/rust-bin-$(ver_cut 1).$(($(ver_cut 2) - 1))*
		=dev-lang/rust-$(ver_cut 1).$(ver_cut 2)*
		=dev-lang/rust-bin-$(ver_cut 1).$(ver_cut 2)*
	)
"

BDEPEND="${PYTHON_DEPS}
	app-eselect/eselect-rust
	|| (
		>=sys-devel/gcc-4.7
		>=sys-devel/clang-3.5
	)
	system-bootstrap? ( ${BOOTSTRAP_DEPEND} )
	!system-llvm? (
		dev-util/cmake
		dev-util/ninja
	)
	test? ( sys-devel/gdb )
	verify-sig? ( app-crypt/openpgp-keys-rust )
"

DEPEND="
	>=app-arch/xz-utils-5.2
	net-misc/curl:=[http2,ssl]
	sys-libs/zlib:=
	dev-libs/openssl:0=
	elibc_musl? ( sys-libs/libunwind:= )
	system-llvm? ( ${LLVM_DEPEND} )
"

# we need to block older versions due to layout changes.
RDEPEND="${DEPEND}
	app-eselect/eselect-rust
	!<dev-lang/rust-1.47.0-r1
	!<dev-lang/rust-bin-1.47.0-r1
"

REQUIRED_USE="|| ( ${ALL_LLVM_TARGETS[*]} )
	miri? ( nightly )
	parallel-compiler? ( nightly )
	test? ( ${ALL_LLVM_TARGETS[*]} )
	wasm? ( llvm_targets_WebAssembly )
	x86? ( cpu_flags_x86_sse2 )
"

# we don't use cmake.eclass, but can get a warnings
CMAKE_WARN_UNUSED_CLI=no

QA_FLAGS_IGNORED="
	usr/lib/${PN}/${PV}/bin/.*
	usr/lib/${PN}/${PV}/libexec/.*
	usr/lib/${PN}/${PV}/lib/lib.*.so
	usr/lib/${PN}/${PV}/lib/rustlib/.*/bin/.*
	usr/lib/${PN}/${PV}/lib/rustlib/.*/lib/lib.*.so
"

QA_SONAME="
	usr/lib/${PN}/${PV}/lib/lib.*.so.*
	usr/lib/${PN}/${PV}/lib/rustlib/.*/lib/lib.*.so
"

# causes double bootstrap
RESTRICT="test"

VERIFY_SIG_OPENPGP_KEY_PATH="/usr/share/openpgp-keys/rust.asc"

PATCHES=(
	"${FILESDIR}"/1.47.0-ignore-broken-and-non-applicable-tests.patch
	"${FILESDIR}"/1.49.0-gentoo-musl-target-specs.patch
	"${FILESDIR}"/1.51.0-slow-doc-install.patch
)

S="${WORKDIR}/${MY_P}-src"

toml_usex() {
	usex "${1}" true false
}

boostrap_rust_version_check() {
	# never call from pkg_pretend. eselect-rust may be not installed yet.
	[[ ${MERGE_TYPE} == binary ]] && return
	local rustc_wanted="$(ver_cut 1).$(($(ver_cut 2) - 1))"
	local rustc_toonew="$(ver_cut 1).$(($(ver_cut 2) + 1))"
	local rustc_version=( $(eselect --brief rust show 2>/dev/null) )
	rustc_version=${rustc_version[0]#rust-bin-}
	rustc_version=${rustc_version#rust-}

	[[ -z "${rustc_version}" ]] && die "Failed to determine rust version, check 'eselect rust' output"

	if ver_test "${rustc_version}" -lt "${rustc_wanted}" ; then
		eerror "Rust >=${rustc_wanted} is required"
		eerror "please run 'eselect rust' and set correct rust version"
		die "selected rust version is too old"
	elif ver_test "${rustc_version}" -ge "${rustc_toonew}" ; then
		eerror "Rust <${rustc_toonew} is required"
		eerror "please run 'eselect rust' and set correct rust version"
		die "selected rust version is too new"
	else
		einfo "Using rust ${rustc_version} to build"
	fi
}

pre_build_checks() {
	local M=8192
	# multiply requirements by 1.5 if we are doing x86-multilib
	if use amd64; then
		M=$(( $(usex abi_x86_32 15 10) * ${M} / 10 ))
	fi
	M=$(( $(usex clippy 128 0) + ${M} ))
	M=$(( $(usex miri 128 0) + ${M} ))
	M=$(( $(usex rls 512 0) + ${M} ))
	M=$(( $(usex rustfmt 256 0) + ${M} ))
	# add 2G if we compile llvm and 256M per llvm_target
	if ! use system-llvm; then
		M=$(( 2048 + ${M} ))
		local ltarget
		for ltarget in ${ALL_LLVM_TARGETS[@]}; do
			M=$(( $(usex ${ltarget} 256 0) + ${M} ))
		done
	fi
	M=$(( $(usex wasm 256 0) + ${M} ))
	M=$(( $(usex debug 2 1) * ${M} ))
	eshopts_push -s extglob
	if is-flagq '-g?(gdb)?([1-9])'; then
		M=$(( 15 * ${M} / 10 ))
	fi
	eshopts_pop
	M=$(( $(usex system-bootstrap 0 1024) + ${M} ))
	M=$(( $(usex doc 256 0) + ${M} ))
	M=$(( 61000 + ${M} )) #takes 61G so to be sure add wtw else they think it would take, for crazy safety.
	#current ebuild (29nov2020) took 60mins to compile (mitigations=2) and 60G in /var/tmp/portage (oddly 610M left even tho it was a 64G ext4 zram
	CHECKREQS_DISK_BUILD=${M}M check-reqs_pkg_${EBUILD_PHASE}
}

llvm_check_deps() {
	has_version -r "sys-devel/llvm:${LLVM_SLOT}[${LLVM_TARGET_USEDEPS// /,}]"
}

pkg_pretend() {
	pre_build_checks
}

pkg_setup() {
	pre_build_checks
	python-any-r1_pkg_setup

	export LIBGIT2_NO_PKG_CONFIG=1 #749381

	use system-bootstrap && boostrap_rust_version_check

	if use system-llvm; then
		llvm_pkg_setup

		local llvm_config="$(get_llvm_prefix "${LLVM_MAX_SLOT}")/bin/llvm-config"
		export LLVM_LINK_SHARED=1
		export RUSTFLAGS="${RUSTFLAGS} -Lnative=$("${llvm_config}" --libdir)"
	fi
}

src_prepare() {
	if ! use system-bootstrap; then
		local rust_stage0_root="${WORKDIR}"/rust-stage0
		local rust_stage0="rust-${RUST_STAGE0_VERSION}-$(rust_abi)"

		"${WORKDIR}/${rust_stage0}"/install.sh --disable-ldconfig \
			--without=rust-docs --destdir="${rust_stage0_root}" --prefix=/ || die
	fi

	default
}

src_configure() {
	local rust_target="" rust_targets="" arch_cflags

	# Collect rust target names to compile standard libs for all ABIs.
	for v in $(multilib_get_enabled_abi_pairs); do
		rust_targets="${rust_targets},\"$(rust_abi $(get_abi_CHOST ${v##*.}))\""
	done
	if use wasm; then
		rust_targets="${rust_targets},\"wasm32-unknown-unknown\""
		if use system-llvm; then
			# un-hardcode rust-lld linker for this target
			# https://bugs.gentoo.org/715348
			sed -i '/linker:/ s/rust-lld/wasm-ld/' compiler/rustc_target/src/spec/wasm32_base.rs || die
		fi
	fi
	rust_targets="${rust_targets#,}"

	local tools="\"cargo\","
	if use clippy; then
		tools="\"clippy\",$tools"
	fi
	if use miri; then
		tools="\"miri\",$tools"
	fi
	if use rls; then
		tools="\"rls\",\"analysis\",\"src\",$tools"
	fi
	if use rustfmt; then
		tools="\"rustfmt\",$tools"
	fi

	local rust_stage0_root
	if use system-bootstrap; then
		rust_stage0_root="$(rustc --print sysroot)"
	else
		rust_stage0_root="${WORKDIR}"/rust-stage0
	fi

	rust_target="$(rust_abi)"

	cat <<- _EOF_ > "${S}"/config.toml
		[llvm]
		download-ci-llvm = false
		optimize = $(toml_usex !debug)
		release-debuginfo = $(toml_usex debug)
		assertions = $(toml_usex debug)
		ccache = "/usr/bin/ccache"
		ninja = true
		targets = "${LLVM_TARGETS// /;}"
		experimental-targets = ""
		link-shared = $(toml_usex system-llvm)
		[build]
		build = "${rust_target}"
		host = ["${rust_target}"]
		target = [${rust_targets}]
		cargo = "${rust_stage0_root}/bin/cargo"
		rustc = "${rust_stage0_root}/bin/rustc"
		rustfmt = "${rust_stage0_root}/bin/rustfmt"
		docs = $(toml_usex doc)
		compiler-docs = false
		submodules = false
		python = "${EPYTHON}"
		locked-deps = true
		vendor = true
		extended = true
		tools = [${tools}]
		# Verbosity level: 0 == not verbose, 1 == verbose, 2 == very verbose
		verbose = 2
		sanitizers = false
		profiler = false
		cargo-native-static = false
		low-priority = true
		print-step-timings = true
		[install]
		prefix = "${EPREFIX}/usr/lib/${PN}/${PV}"
		sysconfdir = "etc"
		docdir = "share/doc/rust"
		bindir = "bin"
		libdir = "lib"
		mandir = "share/man"
		[rust]
		# https://github.com/rust-lang/rust/issues/54872
		codegen-units-std = 1
		optimize = true
		debug = $(toml_usex debug)
		debug-assertions = $(toml_usex debug)
		debug-assertions-std = $(toml_usex debug)
		debuginfo-level = 2
		debuginfo-level-rustc = 2
		debuginfo-level-std = 2
		debuginfo-level-tools = 2
		#debuginfo-level = $(usex debug 2 0)
		#debuginfo-level-rustc = $(usex debug 2 0)
		#debuginfo-level-std = $(usex debug 2 0)
		#debuginfo-level-tools = $(usex debug 2 0)
		debuginfo-level-tests = 0
		backtrace = true
		incremental = false
		default-linker = "$(tc-getCC)"
		parallel-compiler = $(toml_usex parallel-compiler)
		channel = "$(usex nightly nightly stable)"
		description = "gentoo"
		rpath = false
		verbose-tests = true
		optimize-tests = $(toml_usex !debug)
		codegen-tests = true
		dist-src = false
		remap-debuginfo = true
		lld = $(usex system-llvm false $(toml_usex wasm))
		# only deny warnings if doc+wasm are NOT requested, documenting stage0 wasm std fails without it
		# https://github.com/rust-lang/rust/issues/74976
		# https://github.com/rust-lang/rust/issues/76526
		deny-warnings = $(usex wasm $(usex doc false true) true)
		backtrace-on-ice = true
		jemalloc = false
		[dist]
		src-tarball = false
		compression-formats = ["gz"]
		#TODO: try compression-formats 'cat'(not valid!) or just make sure it doesn't use any! However list must not be empty! as per https://github.com/rust-lang/rust/blob/abf3ec5b3353be973b18269fcdda76a59743f235/config.toml.example#L696 see: https://github.com/rust-lang/rust-installer/issues/110
		#missing-tools = false
		#^ this won't work (either with =false or =true), it fails to compile miri even though USE=-miri due to /var/db/repos/gentoo/profiles/base/package.use.mask:30 so it has to be =true
	_EOF_

	for v in $(multilib_get_enabled_abi_pairs); do
		rust_target=$(rust_abi $(get_abi_CHOST ${v##*.}))
		arch_cflags="$(get_abi_CFLAGS ${v##*.})"

		cat <<- _EOF_ >> "${S}"/config.env
			CFLAGS_${rust_target}=${arch_cflags}
		_EOF_

		cat <<- _EOF_ >> "${S}"/config.toml
			[target.${rust_target}]
			cc = "$(tc-getBUILD_CC)"
			cxx = "$(tc-getBUILD_CXX)"
			linker = "$(tc-getCC)"
			ar = "$(tc-getAR)"
		_EOF_
		# librustc_target/spec/linux_musl_base.rs sets base.crt_static_default = true;
		if use elibc_musl; then
			cat <<- _EOF_ >> "${S}"/config.toml
				crt-static = false
			_EOF_
		fi
		if use system-llvm; then
			cat <<- _EOF_ >> "${S}"/config.toml
				llvm-config = "$(get_llvm_prefix "${LLVM_MAX_SLOT}")/bin/llvm-config"
			_EOF_
		fi
	done
	if use wasm; then
		cat <<- _EOF_ >> "${S}"/config.toml
			[target.wasm32-unknown-unknown]
			linker = "$(usex system-llvm lld rust-lld)"
		_EOF_
	fi

	if [[ -n ${I_KNOW_WHAT_I_AM_DOING_CROSS} ]]; then # whitespace intentionally shifted below
	# experimental cross support
	# discussion: https://bugs.gentoo.org/679878
	# TODO: c*flags, clang, system-llvm, cargo.eclass target support
	# it would be much better if we could split out stdlib
	# complilation to separate ebuild and abuse CATEGORY to
	# just install to /usr/lib/rustlib/<target>

	# extra targets defined as a bash array
	# spec format:  <LLVM target>:<rust-target>:<CTARGET>
	# best place would be /etc/portage/env/dev-lang/rust
	# Example:
	# RUST_CROSS_TARGETS=(
	#	"AArch64:aarch64-unknown-linux-gnu:aarch64-unknown-linux-gnu"
	# )
	# no extra hand holding is done, no target transformations, all
	# values are passed as-is with just basic checks, so it's up to user to supply correct values
	# valid rust targets can be obtained with
	# 	rustc --print target-list
	# matching cross toolchain has to be installed
	# matching LLVM_TARGET has to be enabled for both rust and llvm (if using system one)
	# only gcc toolchains installed with crossdev are checked for now.

	# BUG: we can't pass host flags to cross compiler, so just filter for now
	# BUG: this should be more fine-grained.
	filter-flags '-mcpu=*' '-march=*' '-mtune=*'

	local cross_target_spec
	for cross_target_spec in "${RUST_CROSS_TARGETS[@]}";do
		# extracts first element form <LLVM target>:<rust-target>:<CTARGET>
		local cross_llvm_target="${cross_target_spec%%:*}"
		# extracts toolchain triples, <rust-target>:<CTARGET>
		local cross_triples="${cross_target_spec#*:}"
		# extracts first element after before : separator
		local cross_rust_target="${cross_triples%%:*}"
		# extracts last element after : separator
		local cross_toolchain="${cross_triples##*:}"
		use llvm_targets_${cross_llvm_target} || die "need llvm_targets_${cross_llvm_target} target enabled"
		command -v ${cross_toolchain}-gcc > /dev/null 2>&1 || die "need ${cross_toolchain} cross toolchain"

		cat <<- _EOF_ >> "${S}"/config.toml
			[target.${cross_rust_target}]
			cc = "${cross_toolchain}-gcc"
			cxx = "${cross_toolchain}-g++"
			linker = "${cross_toolchain}-gcc"
			ar = "${cross_toolchain}-ar"
		_EOF_
		if use system-llvm; then
			cat <<- _EOF_ >> "${S}"/config.toml
				llvm-config = "$(get_llvm_prefix "${LLVM_MAX_SLOT}")/bin/llvm-config"
			_EOF_
		fi

		# append cross target to "normal" target list
		# example 'target = ["powerpc64le-unknown-linux-gnu"]'
		# becomes 'target = ["powerpc64le-unknown-linux-gnu","aarch64-unknown-linux-gnu"]'

		rust_targets="${rust_targets},\"${cross_rust_target}\""
		sed -i "/^target = \[/ s#\[.*\]#\[${rust_targets}\]#" config.toml || die

		ewarn
		ewarn "Enabled ${cross_rust_target} rust target"
		ewarn "Using ${cross_toolchain} cross toolchain"
		ewarn
		if ! has_version -b 'sys-devel/binutils[multitarget]' ; then
			ewarn "'sys-devel/binutils[multitarget]' is not installed"
			ewarn "'strip' will be unable to strip cross libraries"
			ewarn "cross targets will be installed with full debug information"
			ewarn "enable 'multitarget' USE flag for binutils to be able to strip object files"
			ewarn
			ewarn "Alternatively llvm-strip can be used, it supports stripping any target"
			ewarn "define STRIP=\"llvm-strip\" to use it (experimental)"
			ewarn
		fi
	done
	fi # I_KNOW_WHAT_I_AM_DOING_CROSS

	einfo "Rust configured with the following flags:"
	echo
	echo "RUSTFLAGS=\"${RUSTFLAGS:-}\""
	echo "RUSTFLAGS_BOOTSTRAP=\"${RUSTFLAGS_BOOTSTRAP:-}\""
	echo "RUSTFLAGS_NOT_BOOTSTRAP=\"${RUSTFLAGS_NOT_BOOTSTRAP:-}\""
	cat "${S}"/config.env || die
	echo
	einfo "config.toml contents:"
	cat "${S}"/config.toml || die
	echo
}

src_compile() {
	# we need \n IFS to have config.env with spaces loaded properly. #734018
	(
	IFS=$'\n'
	env $(cat "${S}"/config.env) RUST_BACKTRACE=1\
		"${EPYTHON}" ./x.py dist -vv --config="${S}"/config.toml -j$(makeopts_jobs) || die
	)
}

src_test() {
	# https://rustc-dev-guide.rust-lang.org/tests/intro.html

	# those are basic and codegen tests.
	local tests=(
		codegen
		codegen-units
		compile-fail
		incremental
		mir-opt
		pretty
		run-make
	)

	# fails if llvm is not built with ALL targets.
	# and known to fail with system llvm sometimes.
	use system-llvm || tests+=( assembly )

	# fragile/expensive/less important tests
	# or tests that require extra builds
	# TODO: instead of skipping, just make some nonfatal.
	if [[ ${ERUST_RUN_EXTRA_TESTS:-no} != no ]]; then
		tests+=(
			rustdoc
			rustdoc-js
			rustdoc-js-std
			rustdoc-ui
			run-make-fulldeps
			ui
			ui-fulldeps
		)
	fi

	local i failed=()
	einfo "rust_src_test: enabled tests ${tests[@]/#/src/test/}"
	for i in "${tests[@]}"; do
		local t="src/test/${i}"
		einfo "rust_src_test: running ${t}"
		if ! (
				IFS=$'\n'
				env $(cat "${S}"/config.env) RUST_BACKTRACE=1 \
				"${EPYTHON}" ./x.py test -vv --config="${S}"/config.toml \
				-j$(makeopts_jobs) --no-doc --no-fail-fast "${t}"
			)
		then
				failed+=( "${t}" )
				eerror "rust_src_test: ${t} failed"
		fi
	done

	if [[ ${#failed[@]} -ne 0 ]]; then
		eerror "rust_src_test: failure summary: ${failed[@]}"
		die "aborting due to test failures"
	fi
}

src_install() {
	(
	IFS=$'\n'
	env $(cat "${S}"/config.env) DESTDIR="${D}" \
		"${EPYTHON}" ./x.py install -vv --config="${S}"/config.toml -j$(makeopts_jobs) || die
	)

	# bug #689562, #689160
	rm -v "${ED}/usr/lib/${PN}/${PV}/etc/bash_completion.d/cargo" || die
	rmdir -v "${ED}/usr/lib/${PN}/${PV}"/etc{/bash_completion.d,} || die
	newbashcomp src/tools/cargo/src/etc/cargo.bashcomp.sh cargo

	local symlinks=(
		cargo
		rustc
		rustdoc
		rust-gdb
		rust-gdbgui
		rust-lldb
	)

	use clippy && symlinks+=( clippy-driver cargo-clippy )
	use miri && symlinks+=( miri cargo-miri )
	use rls && symlinks+=( rls )
	use rustfmt && symlinks+=( rustfmt cargo-fmt )

	einfo "installing eselect-rust symlinks and paths: ${symlinks[@]}"
	local i
	for i in "${symlinks[@]}"; do
		# we need realpath on /usr/bin/* symlink return version-appended binary path.
		# so /usr/bin/rustc should point to /usr/lib/rust/<ver>/bin/rustc-<ver>
		# need to fix eselect-rust to remove this hack.
		local ver_i="${i}-${PV}"
		if [[ -f "${ED}/usr/lib/${PN}/${PV}/bin/${i}" ]]; then
			einfo "Installing ${i} symlink"
			ln -v "${ED}/usr/lib/${PN}/${PV}/bin/${i}" "${ED}/usr/lib/${PN}/${PV}/bin/${ver_i}" || die
		else
			ewarn "${i} symlink requested, but source file not found"
			ewarn "please report this"
		fi
		dosym "../lib/${PN}/${PV}/bin/${ver_i}" "/usr/bin/${ver_i}"
	done

	# symlinks to switch components to active rust in eselect
	dosym "${PV}/lib" "/usr/lib/${PN}/lib-${PV}"
	dosym "${PV}/libexec" "/usr/lib/${PN}/libexec-${PV}"
	dosym "${PV}/share/man" "/usr/lib/${PN}/man-${PV}"
	dosym "rust/${PV}/lib/rustlib" "/usr/lib/rustlib-${PV}"
	dosym "../../lib/${PN}/${PV}/share/doc/rust" "/usr/share/doc/${P}"

	newenvd - "50${P}" <<-_EOF_
		LDPATH="${EPREFIX}/usr/lib/rust/lib"
		MANPATH="${EPREFIX}/usr/lib/rust/man"
		$(use amd64 && usex elibc_musl 'CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_RUSTFLAGS="-C target-feature=-crt-static"' '')
		$(use arm64 && usex elibc_musl 'CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS="-C target-feature=-crt-static"' '')
	_EOF_

	rm -rf "${ED}/usr/lib/${PN}/${PV}"/*.old || die
	rm -rf "${ED}/usr/lib/${PN}/${PV}/doc"/*.old || die

	# note: eselect-rust adds EROOT to all paths below
	cat <<-_EOF_ > "${T}/provider-${P}"
		/usr/bin/cargo
		/usr/bin/rustdoc
		/usr/bin/rust-gdb
		/usr/bin/rust-gdbgui
		/usr/bin/rust-lldb
		/usr/lib/rustlib
		/usr/lib/rust/lib
		/usr/lib/rust/libexec
		/usr/lib/rust/man
		/usr/share/doc/rust
	_EOF_

	if use clippy; then
		echo /usr/bin/clippy-driver >> "${T}/provider-${P}"
		echo /usr/bin/cargo-clippy >> "${T}/provider-${P}"
	fi
	if use miri; then
		echo /usr/bin/miri >> "${T}/provider-${P}"
		echo /usr/bin/cargo-miri >> "${T}/provider-${P}"
	fi
	if use rls; then
		echo /usr/bin/rls >> "${T}/provider-${P}"
	fi
	if use rustfmt; then
		echo /usr/bin/rustfmt >> "${T}/provider-${P}"
		echo /usr/bin/cargo-fmt >> "${T}/provider-${P}"
	fi

	insinto /etc/env.d/rust
	doins "${T}/provider-${P}"
}

pkg_postinst() {
	eselect rust update

	if has_version sys-devel/gdb || has_version dev-util/lldb; then
		elog "Rust installs a helper script for calling GDB and LLDB,"
		elog "for your convenience it is installed under /usr/bin/rust-{gdb,lldb}-${PV}."
	fi

	if has_version app-editors/emacs; then
		elog "install app-emacs/rust-mode to get emacs support for rust."
	fi

	if has_version app-editors/gvim || has_version app-editors/vim; then
		elog "install app-vim/rust-vim to get vim support for rust."
	fi
}

pkg_postrm() {
	eselect rust cleanup
}

Without patch it used to emerge rust in 56 mins (ran it twice and got the same time), but now, with this patch, it does it in ... about 43 mins (second run was: 45mins because I was also watching a movie meanwhile

real    44m56.608s
user    353m59.402s
sys     5m25.139s

)

In the 'compile' phase(./x.py dist)(this takes like 33 mins to complete), some .tar (instead of .tar.gz) files get generated in dir: /var/tmp/portage/dev-lang/rust-1.52.0/work/rustc-1.52.0-src/build/dist

total 10628612
drwxr-xr-x 6 portage portage       4096 May 11 23:48 ..
-rw-r--r-- 1 portage portage 1450638336 May 12 00:38 rustc-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage  178041344 May 12 00:38 rust-std-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage 4334052864 May 12 00:38 rustc-dev-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage   38921216 May 12 00:38 rust-analysis-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage   40742400 May 12 00:38 rust-src-nightly.tar
-rw-r--r-- 1 portage portage  202902016 May 12 00:38 cargo-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage  196488704 May 12 00:38 rustfmt-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage  538494464 May 12 00:38 rls-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage  436925952 May 12 00:39 rust-analyzer-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage  134438912 May 12 00:39 clippy-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage   77839360 May 12 00:39 miri-nightly-x86_64-unknown-linux-gnu.tar
drwxr-xr-x 2 portage portage       4096 May 12 00:39 .
-rw-r--r-- 1 portage portage 3254147072 May 12 00:39 rust-nightly-x86_64-unknown-linux-gnu.tar

In the 'install' phase(./x.py install)(this takes like 10 mins to complete), some of the above files get regenerated(those from timestamp 0:41am inclusive):

real    9m53.212s
user    65m38.853s
sys     1m54.637s

total 10628596
drwxr-xr-x 6 portage portage       4096 May 11 23:48 ..
-rw-r--r-- 1 portage portage 4334052864 May 12 00:38 rustc-dev-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage  436925952 May 12 00:39 rust-analyzer-nightly-x86_64-unknown-linux-gnu.tar
drwxr-xr-x 2 portage portage       4096 May 12 00:39 .
-rw-r--r-- 1 portage portage 3254147072 May 12 00:39 rust-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage  178041344 May 12 00:41 rust-std-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage  202902016 May 12 00:43 cargo-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage  538494464 May 12 00:47 rls-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage  196488704 May 12 00:48 rustfmt-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage  134438912 May 12 00:48 clippy-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage   77839360 May 12 00:49 miri-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage   38921216 May 12 00:49 rust-analysis-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 portage portage   40742400 May 12 00:49 rust-src-nightly.tar
-rw-r--r-- 1 portage portage 1450638336 May 12 00:50 rustc-nightly-x86_64-unknown-linux-gnu.tar

Actual config.toml used while building (based on the above ebuild):

[llvm]
download-ci-llvm = false
optimize = true
release-debuginfo = false
assertions = false
ccache = "/usr/bin/ccache"
ninja = true
targets = "X86"
experimental-targets = ""
link-shared = true
[build]
build = "x86_64-unknown-linux-gnu"
host = ["x86_64-unknown-linux-gnu"]
target = ["x86_64-unknown-linux-gnu"]
cargo = "/usr/lib/rust/1.52.0/bin/cargo"
rustc = "/usr/lib/rust/1.52.0/bin/rustc"
rustfmt = "/usr/lib/rust/1.52.0/bin/rustfmt"
docs = false
compiler-docs = false
submodules = false
python = "python3.9"
locked-deps = true
vendor = true
extended = true
tools = ["rustfmt","rls","analysis","src","miri","clippy","cargo",]
# Verbosity level: 0 == not verbose, 1 == verbose, 2 == very verbose
verbose = 2
sanitizers = false
profiler = false
cargo-native-static = false
low-priority = true
print-step-timings = true
[install]
prefix = "/usr/lib/rust/1.52.0"
sysconfdir = "etc"
docdir = "share/doc/rust"
bindir = "bin"
libdir = "lib"
mandir = "share/man"
[rust]
# https://github.com/rust-lang/rust/issues/54872
codegen-units-std = 1
optimize = true
debug = false
debug-assertions = false
debug-assertions-std = false
debuginfo-level = 2
debuginfo-level-rustc = 2
debuginfo-level-std = 2
debuginfo-level-tools = 2
#debuginfo-level = 0
#debuginfo-level-rustc = 0
#debuginfo-level-std = 0
#debuginfo-level-tools = 0
debuginfo-level-tests = 0
backtrace = true
incremental = false
default-linker = "x86_64-pc-linux-gnu-gcc"
parallel-compiler = true
channel = "nightly"
description = "gentoo"
rpath = false
verbose-tests = true
optimize-tests = true
codegen-tests = true
dist-src = false
remap-debuginfo = true
lld = false
# only deny warnings if doc+wasm are NOT requested, documenting stage0 wasm std fails without it
# https://github.com/rust-lang/rust/issues/74976
# https://github.com/rust-lang/rust/issues/76526
deny-warnings = true
backtrace-on-ice = true
jemalloc = false
[dist]
src-tarball = false
compression-formats = ["gz"]
#nvm patch not made! //keep 'gz' here so that 0600_fabricate_neutering_try2_just_tar_not_gz_or_xz_for_rust_1_52_0.patch applies neatly
#TODO: try compression-formats 'cat'(not valid!) or just make sure it doesn't use any! However list must not be empty! as per https://github.com/rust-lang/rust/blob/abf3ec5b3353be973b18269fcdda76a59743f235/config.toml.example#L696 see: https://github.com/rust-lang/rust-installer/issues/110
#missing-tools = false
#^ this won't work (either with =false or =true), it fails to compile miri even though USE=-miri due to /var/db/repos/gentoo/profiles/base/package.use.mask:30 so it has to be =true
[target.x86_64-unknown-linux-gnu]
cc = "x86_64-pc-linux-gnu-gcc"
cxx = "x86_64-pc-linux-gnu-g++"
linker = "x86_64-pc-linux-gnu-gcc"
ar = "x86_64-pc-linux-gnu-ar"
llvm-config = "/usr/lib/llvm/12/bin/llvm-config"

aswild

This has been on my wishlist for a while, compressing tarballs only to immediately decompress them is a big bottleneck for local installs. Here's a proof of concept that adds a proper no-compression mode. If there's maintainer interest, I'd be happy to turn this into a PR.

The speedup is significant, since the vast majority of the time during dist or install is spent on compression. Tested on my PC with an i7-8086K and an NVMe drive. I configured an extended build (cargo, clippy, rustfmt) and did ./x.py build --stage 2 so all the times here are just dist/install and no recompiling.

Command \ Compression	gz	xz	none
`./x.py install`	42s	51s	16s
`./x.py dist`	2m 12s	2m 4s	3s
size of all tarballs in `build/dist/`	515 MiB	333 MiB	1766 MiB

rust-installer patch:

diff --git a/src/combiner.rs b/src/combiner.rs
index 006a40c..549b753 100644
--- a/src/combiner.rs
+++ b/src/combiner.rs
@@ -78,7 +78,7 @@ impl Combiner {
                 })?;
 
             let pkg_name =
-                input_tarball.trim_end_matches(&format!(".tar.{}", compression.extension()));
+                input_tarball.trim_end_matches(&format!(".tar{}", compression.extension()));
             let pkg_name = Path::new(pkg_name).file_name().unwrap();
             let pkg_dir = Path::new(&self.work_dir).join(&pkg_name);
 
diff --git a/src/compression.rs b/src/compression.rs
index b3010cb..91c7ae6 100644
--- a/src/compression.rs
+++ b/src/compression.rs
@@ -6,6 +6,7 @@ use xz2::{read::XzDecoder, write::XzEncoder};
 
 #[derive(Debug, Copy, Clone)]
 pub enum CompressionFormat {
+    None,
     Gz,
     Xz,
 }
@@ -13,6 +14,7 @@ pub enum CompressionFormat {
 impl CompressionFormat {
     pub(crate) fn detect_from_path(path: impl AsRef<Path>) -> Option<Self> {
         match path.as_ref().extension().and_then(|e| e.to_str()) {
+            Some("tar") => Some(CompressionFormat::None),
             Some("gz") => Some(CompressionFormat::Gz),
             Some("xz") => Some(CompressionFormat::Xz),
             _ => None,
@@ -21,14 +23,15 @@ impl CompressionFormat {
 
     pub(crate) fn extension(&self) -> &'static str {
         match self {
-            CompressionFormat::Gz => "gz",
-            CompressionFormat::Xz => "xz",
+            CompressionFormat::None => "",
+            CompressionFormat::Gz => ".gz",
+            CompressionFormat::Xz => ".xz",
         }
     }
 
     pub(crate) fn encode(&self, path: impl AsRef<Path>) -> Result<Box<dyn Encoder>, Error> {
         let mut os = path.as_ref().as_os_str().to_os_string();
-        os.push(format!(".{}", self.extension()));
+        os.push(format!("{}", self.extension()));
         let path = Path::new(&os);
 
         if path.exists() {
@@ -37,6 +40,7 @@ impl CompressionFormat {
         let file = crate::util::create_new_file(path)?;
 
         Ok(match self {
+            CompressionFormat::None => Box::new(NopEncoder::new(file)),
             CompressionFormat::Gz => Box::new(GzEncoder::new(file, flate2::Compression::best())),
             CompressionFormat::Xz => {
                 // Note that preset 6 takes about 173MB of memory per thread, so we limit the number of
@@ -54,6 +58,7 @@ impl CompressionFormat {
     pub(crate) fn decode(&self, path: impl AsRef<Path>) -> Result<Box<dyn Read>, Error> {
         let file = crate::util::open_file(path.as_ref())?;
         Ok(match self {
+            CompressionFormat::None => Box::new(file),
             CompressionFormat::Gz => Box::new(GzDecoder::new(file)),
             CompressionFormat::Xz => Box::new(XzDecoder::new(file)),
         })
@@ -71,6 +76,7 @@ impl TryFrom<&'_ str> for CompressionFormats {
         let mut parsed = Vec::new();
         for format in value.split(',') {
             match format.trim() {
+                "none" => parsed.push(CompressionFormat::None),
                 "gz" => parsed.push(CompressionFormat::Gz),
                 "xz" => parsed.push(CompressionFormat::Xz),
                 other => anyhow::bail!("unknown compression format: {}", other),
@@ -96,6 +102,31 @@ pub(crate) trait Encoder: Send + Write {
     fn finish(self: Box<Self>) -> Result<(), Error>;
 }
 
+#[derive(Debug)]
+struct NopEncoder<W>(W);
+
+impl<W> NopEncoder<W> {
+    fn new(writer: W) -> Self {
+        Self(writer)
+    }
+}
+
+impl<W: Send + Write> Write for NopEncoder<W> {
+    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        self.0.write(buf)
+    }
+
+    fn flush(&mut self) -> std::io::Result<()> {
+        self.0.flush()
+    }
+}
+
+impl<W: Send + Write> Encoder for NopEncoder<W> {
+    fn finish(self: Box<Self>) -> Result<(), Error> {
+        Ok(())
+    }
+}
+
 impl<W: Send + Write> Encoder for GzEncoder<W> {
     fn finish(self: Box<Self>) -> Result<(), Error> {
         GzEncoder::finish(*self).context("failed to finish .gz file")?;

rust (bootstrap) patch:

diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/tarball.rs
index 9ff5c2327e0..bca82e0489e 100644
--- a/src/bootstrap/tarball.rs
+++ b/src/bootstrap/tarball.rs
@@ -322,11 +322,14 @@ fn run(self, build_cli: impl FnOnce(&Tarball<'a>, &mut Command)) -> GeneratedTar
             .dist_compression_formats
             .as_ref()
             .and_then(|formats| formats.get(0))
-            .map(|s| s.as_str())
-            .unwrap_or("gz");
+            .map(|format| match format.as_str() {
+                "none" => "".to_string(),
+                s => format!(".{}", s),
+            })
+            .unwrap_or_else(|| ".gz".to_string());
 
         GeneratedTarball {
-            path: crate::dist::distdir(self.builder).join(format!("{}.tar.{}", package_name, ext)),
+            path: crate::dist::distdir(self.builder).join(format!("{}.tar{}", package_name, ext)),
             decompressed_output: self.temp_dir.join(package_name),
             work: self.temp_dir,
         }

correabuscar

Thanks aswild! I'm using your patch on Gentoo for dev-lang/rust-1.64.0-r1 and the install phase takes (at least) this long:

real    2m17.875s
user    1m12.461s
sys     1m37.807s

and yield the following in /var/tmp/portage/dev-lang/rust-1.64.0-r1/work/rustc-1.64.0-src/build/dist:

total 3618000
drwxr-xr-x 2 root    root          4096 Sep 28 09:21 .
drwxr-xr-x 6 portage portage       4096 Sep 28 08:59 ..
-rw-r--r-- 1 root    root     285510656 Sep 28 09:20 cargo-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 root    root     187013120 Sep 28 09:20 clippy-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 root    root     605404672 Sep 28 09:20 rls-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 root    root      51220992 Sep 28 09:20 rust-analysis-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 root    root    2114523136 Sep 28 09:21 rustc-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 root    root     254292480 Sep 28 09:20 rustfmt-nightly-x86_64-unknown-linux-gnu.tar
-rw-r--r-- 1 root    root      25233408 Sep 28 09:20 rust-src-nightly.tar
-rw-r--r-- 1 root    root     181593600 Sep 28 09:20 rust-std-nightly-x86_64-unknown-linux-gnu.tar

(while the compile phase takes this long:

real    20m43.942s
user    184m16.076s
sys     3m38.705s

)

One thing to note is that user can override compression format in config.toml(via a modified .ebuild) like: compression-formats = ["gz"](while the default is the overridden 'xz' in Gentoo) and wonder why the patch isn't working.
That (gz) install phase would take, at least:

real    5m51.131s
user    4m15.991s
sys     2m13.200s

and yield:

total 680092
drwxr-xr-x 2 root    root         4096 Sep 28 09:15 .
drwxr-xr-x 6 portage portage      4096 Sep 28 08:59 ..
-rw-r--r-- 1 root    root     52474477 Sep 28 09:12 cargo-nightly-x86_64-unknown-linux-gnu.tar.gz
-rw-r--r-- 1 root    root     35523955 Sep 28 09:13 clippy-nightly-x86_64-unknown-linux-gnu.tar.gz
-rw-r--r-- 1 root    root    111652340 Sep 28 09:12 rls-nightly-x86_64-unknown-linux-gnu.tar.gz
-rw-r--r-- 1 root    root      5110512 Sep 28 09:13 rust-analysis-nightly-x86_64-unknown-linux-gnu.tar.gz
-rw-r--r-- 1 root    root    386615670 Sep 28 09:16 rustc-nightly-x86_64-unknown-linux-gnu.tar.gz
-rw-r--r-- 1 root    root     49449083 Sep 28 09:12 rustfmt-nightly-x86_64-unknown-linux-gnu.tar.gz
-rw-r--r-- 1 root    root      3437413 Sep 28 09:13 rust-src-nightly.tar.gz
-rw-r--r-- 1 root    root     52129123 Sep 28 09:11 rust-std-nightly-x86_64-unknown-linux-gnu.tar.gz

The exact patch that I used is inside this(ie. click me to expand):

Index: /var/tmp/portage/dev-lang/rust-1.64.0-r1/work/rustc-1.64.0-src/src/bootstrap/tarball.rs
===================================================================
--- rustc-1.64.0-src/src/bootstrap/tarball.rs
+++ rustc-1.64.0-src/src/bootstrap/tarball.rs
@@ -341,11 +341,14 @@ impl<'a> Tarball<'a> {
             .dist_compression_formats
             .as_ref()
             .and_then(|formats| formats.get(0))
-            .map(|s| s.as_str())
-            .unwrap_or("gz");
+            .map(|format| match format.as_str() {
+                "none" => "".to_string(),
+                s => format!(".{}", s),
+                })
+            .unwrap_or_else(|| ".gz".to_string());
 
         GeneratedTarball {
-            path: crate::dist::distdir(self.builder).join(format!("{}.tar.{}", package_name, ext)),
+            path: crate::dist::distdir(self.builder).join(format!("{}.tar{}", package_name, ext)),
             decompressed_output,
             work: self.temp_dir,
         }
Index: /var/tmp/portage/dev-lang/rust-1.64.0-r1/work/rustc-1.64.0-src/src/tools/rust-installer/src/combiner.rs
===================================================================
--- rustc-1.64.0-src/src/tools/rust-installer/src/combiner.rs
+++ rustc-1.64.0-src/src/tools/rust-installer/src/combiner.rs
@@ -88,7 +88,7 @@ impl Combiner {
                 })?;
 
             let pkg_name =
-                input_tarball.trim_end_matches(&format!(".tar.{}", compression.extension()));
+                input_tarball.trim_end_matches(&format!(".tar{}", compression.extension()));
             let pkg_name = Path::new(pkg_name).file_name().unwrap();
             let pkg_dir = Path::new(&self.work_dir).join(&pkg_name);
 
Index: /var/tmp/portage/dev-lang/rust-1.64.0-r1/work/rustc-1.64.0-src/src/tools/rust-installer/src/compression.rs
===================================================================
--- rustc-1.64.0-src/src/tools/rust-installer/src/compression.rs
+++ rustc-1.64.0-src/src/tools/rust-installer/src/compression.rs
@@ -6,6 +6,7 @@ use xz2::{read::XzDecoder, write::XzEnco
 
 #[derive(Debug, Copy, Clone)]
 pub enum CompressionFormat {
+    None,
     Gz,
     Xz,
 }
@@ -13,6 +14,7 @@ pub enum CompressionFormat {
 impl CompressionFormat {
     pub(crate) fn detect_from_path(path: impl AsRef<Path>) -> Option<Self> {
         match path.as_ref().extension().and_then(|e| e.to_str()) {
+            Some("tar") => Some(CompressionFormat::None),
             Some("gz") => Some(CompressionFormat::Gz),
             Some("xz") => Some(CompressionFormat::Xz),
             _ => None,
@@ -21,14 +23,15 @@ impl CompressionFormat {
 
     pub(crate) fn extension(&self) -> &'static str {
         match self {
-            CompressionFormat::Gz => "gz",
-            CompressionFormat::Xz => "xz",
+            CompressionFormat::None => "",
+            CompressionFormat::Gz => ".gz",
+            CompressionFormat::Xz => ".xz",
         }
     }
 
     pub(crate) fn encode(&self, path: impl AsRef<Path>) -> Result<Box<dyn Encoder>, Error> {
         let mut os = path.as_ref().as_os_str().to_os_string();
-        os.push(format!(".{}", self.extension()));
+        os.push(format!("{}", self.extension()));
         let path = Path::new(&os);
 
         if path.exists() {
@@ -37,6 +40,7 @@ impl CompressionFormat {
         let file = crate::util::create_new_file(path)?;
 
         Ok(match self {
+            CompressionFormat::None => Box::new(NopEncoder::new(file)),
             CompressionFormat::Gz => Box::new(GzEncoder::new(file, flate2::Compression::best())),
             CompressionFormat::Xz => {
                 // Note that preset 6 takes about 173MB of memory per thread, so we limit the number of
@@ -54,6 +58,7 @@ impl CompressionFormat {
     pub(crate) fn decode(&self, path: impl AsRef<Path>) -> Result<Box<dyn Read>, Error> {
         let file = crate::util::open_file(path.as_ref())?;
         Ok(match self {
+            CompressionFormat::None => Box::new(file),
             CompressionFormat::Gz => Box::new(GzDecoder::new(file)),
             CompressionFormat::Xz => Box::new(XzDecoder::new(file)),
         })
@@ -71,6 +76,7 @@ impl TryFrom<&'_ str> for CompressionFor
         let mut parsed = Vec::new();
         for format in value.split(',') {
             match format.trim() {
+                "none" => parsed.push(CompressionFormat::None),
                 "gz" => parsed.push(CompressionFormat::Gz),
                 "xz" => parsed.push(CompressionFormat::Xz),
                 other => anyhow::bail!("unknown compression format: {}", other),
@@ -97,6 +103,7 @@ impl fmt::Display for CompressionFormats
             fmt::Display::fmt(match format {
                 CompressionFormat::Xz => "xz",
                 CompressionFormat::Gz => "gz",
+                CompressionFormat::None => "none",
             }, f)?;
         }
         Ok(())
@@ -105,7 +112,7 @@ impl fmt::Display for CompressionFormats
 
 impl Default for CompressionFormats {
     fn default() -> Self {
-        Self(vec![CompressionFormat::Gz, CompressionFormat::Xz])
+        Self(vec![CompressionFormat::None, CompressionFormat::Gz, CompressionFormat::Xz])
     }
 }
 
@@ -119,6 +125,31 @@ pub(crate) trait Encoder: Send + Write {
     fn finish(self: Box<Self>) -> Result<(), Error>;
 }
 
+#[derive(Debug)]
+struct NopEncoder<W>(W);
+
+impl<W> NopEncoder<W> {
+    fn new(writer: W) -> Self {
+        Self(writer)
+    }
+}
+
+impl<W: Send + Write> Write for NopEncoder<W> {
+    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        self.0.write(buf)
+    }
+
+    fn flush(&mut self) -> std::io::Result<()> {
+        self.0.flush()
+    }
+}
+
+impl<W: Send + Write> Encoder for NopEncoder<W> {
+    fn finish(self: Box<Self>) -> Result<(), Error> {
+        Ok(())
+    }
+}
+
 impl<W: Send + Write> Encoder for GzEncoder<W> {
     fn finish(self: Box<Self>) -> Result<(), Error> {
         GzEncoder::finish(*self).context("failed to finish .gz file")?;

onur-ozkan

Member

This should be no longer needed as rust-lang/rust#118724 PR is merged.

to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Assignees

No one assigned

Labels

No labels

Type

No type

Projects

No projects

Milestone

No milestone

Relationships

None yet

Development

No branches or pull requests

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

support plain tar or no compression #110

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

Participants

support plain tar or no compression #110

Description

Activity

acheechee commented on May 11, 2021

aswild commented on Aug 30, 2021

correabuscar commented on Sep 28, 2022

onur-ozkan commented on Feb 25, 2024

Metadata

Metadata

Assignees

Labels

Type

Projects

Milestone

Relationships

Development

Participants

Issue actions