Merge pull request #402 from ojeda/alloc

Custom `alloc`, using stable Rust compilers, and more

Author: ojeda

.github/workflows/ci.yaml

@@ -13,7 +13,7 @@ jobs:
         arch: [arm, arm64, ppc64le, riscv64, x86_64]
         toolchain: [gcc, clang, llvm]
         config: [debug, release]
-        rustc: [2021-05-29]
+        rustc: [2021-06-23]
         output: [src] # [src, build]
         install: [rustup] # [rustup, standalone]
         sysroot: [common] # [common, custom]
@@ -30,30 +30,30 @@ jobs:
 
         # A few independent combinations to avoid exploding the matrix:
         #   - The other option for `output`.
-        #   - Different nightlies for `rustc`.
+        #   - Different releases for `rustc`.
         #   - The other three (`install`, `sysroot`) combinations
         #     (they are interrelated, so the cross-product needs to be tested)
         include:
           - arch: arm64
             toolchain: gcc
             config: debug
-            rustc: 2021-05-29
+            rustc: 2021-06-23
             output: build
             install: rustup
             sysroot: custom
 
           - arch: ppc64le
             toolchain: clang
             config: release
-            rustc: 2021-05-29
+            rustc: 2021-06-23
             output: build
             install: standalone
             sysroot: common
 
           - arch: x86_64
             toolchain: llvm
             config: debug
-            rustc: 2021-05-29
+            rustc: 2021-06-23
             output: build
             install: standalone
             sysroot: custom
@@ -178,12 +178,14 @@ jobs:
       # Setup: rustc
       - if: matrix.install == 'rustup'
         run: |
-          rustup default nightly-${{ matrix.rustc }}
+          rustup default beta-${{ matrix.rustc }}
           rustup component add rustfmt
       - if: matrix.install == 'standalone'
         run: |
-          curl https://static.rust-lang.org/dist/${{ matrix.rustc }}/rust-nightly-x86_64-unknown-linux-gnu.tar.gz | tar xz
-          rust-nightly-x86_64-unknown-linux-gnu/install.sh --without=rust-docs --prefix=$HOME/rustc
+          # FIXME: there is no beta equivalent for https://static.rust-lang.org/dist/channel-rust-x.yy.z.toml,
+          # so just hardcode the URL/date for the moment, since we will moving to a stable release soon anyway.
+          curl https://static.rust-lang.org/dist/${{ matrix.rustc }}/rust-beta-x86_64-unknown-linux-gnu.tar.gz | tar xz
+          rust-beta-x86_64-unknown-linux-gnu/install.sh --without=rust-docs --prefix=$HOME/rustc
           echo $HOME/rustc/bin >> $GITHUB_PATH
 
       # Setup: rustc native libs
@@ -402,7 +404,7 @@ jobs:
       - run: make ${{ env.MAKE_ARCH }} ${{ env.MAKE_CROSS_COMPILE }} ${{ env.MAKE_TOOLCHAIN }} ${{ env.MAKE_OUTPUT }} ${{ env.MAKE_SYSROOT }} -j3 rusttest
 
       # Formatting
-      - run: make rustfmtcheck
+      - run: make ${{ env.MAKE_ARCH }} ${{ env.MAKE_CROSS_COMPILE }} ${{ env.MAKE_TOOLCHAIN }} ${{ env.MAKE_OUTPUT }} ${{ env.MAKE_SYSROOT }} -j3 rustfmtcheck
 
       # View changes to ccache
       - run: ccache -s

.rustfmt.toml

@@ -1,11 +1,11 @@
 edition = "2018"
-format_code_in_doc_comments = true
 newline_style = "Unix"
 
 # Unstable options that help catching some mistakes in formatting and that we may want to enable
 # when they become stable.
 #
 # They are kept here since they are useful to run from time to time.
+#format_code_in_doc_comments = true
 #reorder_impl_items = true
 #comment_width = 100
 #wrap_comments = true

Documentation/process/changes.rst

@@ -26,12 +26,17 @@ running a Linux kernel.  Also, not all tools are necessary on all
 systems; obviously, if you don't have any PC Card hardware, for example,
 you probably needn't concern yourself with pcmciautils.
 
+Furthermore, note that newer versions of the Rust toolchain may or may not work
+because, for the moment, we depend on some unstable features. Thus, unless you
+know what you are doing, use the exact version listed here. Please see
+:ref:`Documentation/rust/quick-start.rst <rust_quick_start>` for details.
+
 ====================== ===============  ========================================
         Program        Minimal version       Command to check the version
 ====================== ===============  ========================================
 GNU C                  4.9              gcc --version
 Clang/LLVM (optional)  10.0.1           clang --version
-rustc (optional)       nightly          rustc --version
+rustc (optional)       1.54.0-beta.1    rustc --version
 bindgen (optional)     0.56.0           bindgen --version
 GNU make               3.81             make --version
 binutils               2.23             ld -v
@@ -57,7 +62,6 @@ iptables               1.4.2            iptables -V
 openssl & libcrypto    1.0.0            openssl version
 bc                     1.06.95          bc --version
 Sphinx\ [#f1]_	       1.3		sphinx-build --version
-rustdoc (optional)     nightly          rustdoc --version
 ====================== ===============  ========================================
 
 .. [#f1] Sphinx is needed only to build the Kernel documentation

Documentation/rust/quick-start.rst

@@ -23,13 +23,13 @@ at the time of writing, they are likely to not be recent enough.
 rustc
 *****
 
-A recent *nightly* Rust toolchain (with, at least, ``rustc``) is required,
-e.g. ``nightly-2021-05-29``. Our goal is to use a stable toolchain as soon
-as possible, but for the moment we depend on a handful of nightly features.
+A particular version (`1.54.0-beta.1`) of the Rust compiler is required.
+Newer versions may or may not work because, for the moment, we depend on
+some unstable Rust features.
 
 If you are using ``rustup``, run::
 
-    rustup default nightly-2021-05-29
+    rustup default beta-2021-06-23
 
 Otherwise, fetch a standalone installer or install ``rustup`` from:
 
@@ -125,16 +125,30 @@ the component manually::
 The standalone installers also come with ``clippy``.
 
 
+cargo
+*****
+
+``cargo`` is the Rust native build system. It is currently required to run
+the tests (``rusttest`` target) since we use it to build a custom standard
+library that contains the facilities provided by our custom ``alloc``.
+
+If you are using ``rustup``, all the profiles already install the tool,
+so you should be good to go. The standalone installers also include ``cargo``.
+
+
 rustdoc
 *******
 
-If you install the ``rustdoc`` tool, then you will be able to generate pretty
-HTML documentation for Rust code, including for the libraries (crates) inside
-``rust/`` that are used by the rest of the kernel (for details, please see
-:ref:`Documentation/rust/docs.rst <rust_docs>`).
+``rustdoc`` is the documentation tool for Rust. It generates pretty HTML
+documentation for Rust code (for details, please see
+:ref:`Documentation/rust/docs.rst <rust_docs>`.
+
+``rustdoc`` is also able to test the examples provided in documented Rust code
+(called doctests or documentation tests). We use this feature, thus ``rustdoc``
+is required to run the tests (``rusttest`` target).
 
 If you are using ``rustup``, all the profiles already install the tool,
-so you should be good to go. The standalone installers also come with ``rustdoc``.
+so you should be good to go. The standalone installers also include ``rustdoc``.
 
 
 rust-analyzer

Makefile

@@ -460,9 +460,12 @@ READELF		= $(CROSS_COMPILE)readelf
 STRIP		= $(CROSS_COMPILE)strip
 endif
 RUSTC		= rustc
+RUSTC_BOOTSTRAP = 1
+RUSTDOC		= rustdoc
 RUSTFMT		= rustfmt
 CLIPPY_DRIVER	= clippy-driver
 BINDGEN		= bindgen
+CARGO		= cargo
 PAHOLE		= pahole
 RESOLVE_BTFIDS	= $(objtree)/tools/bpf/resolve_btfids/resolve_btfids
 LEX		= flex
@@ -524,7 +527,7 @@ KBUILD_RUSTCFLAGS := --emit=dep-info,obj,metadata --edition=2018 \
 		     -Cpanic=abort -Cembed-bitcode=n -Clto=n -Crpath=n \
 		     -Cforce-unwind-tables=n -Ccodegen-units=1 \
 		     -Zbinary_dep_depinfo=y -Zsymbol-mangling-version=v0 \
-		     -D unsafe_op_in_unsafe_fn
+		     -Dunsafe_op_in_unsafe_fn -Wmissing_docs
 KBUILD_AFLAGS_KERNEL :=
 KBUILD_CFLAGS_KERNEL :=
 KBUILD_RUSTCFLAGS_KERNEL :=
@@ -542,13 +545,13 @@ else
 	RUSTC_OR_CLIPPY_QUIET := RUSTC
 	RUSTC_OR_CLIPPY = $(RUSTC)
 endif
-export RUSTC_OR_CLIPPY_QUIET RUSTC_OR_CLIPPY
 
 ifdef RUST_LIB_SRC
 	export RUST_LIB_SRC
 endif
 
-export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE LD CC RUSTC BINDGEN
+export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE LD CC
+export RUSTC RUSTC_BOOTSTRAP RUSTDOC RUSTFMT RUSTC_OR_CLIPPY_QUIET RUSTC_OR_CLIPPY BINDGEN CARGO
 export CPP AR NM STRIP OBJCOPY OBJDUMP READELF PAHOLE RESOLVE_BTFIDS LEX YACC AWK INSTALLKERNEL
 export PERL PYTHON3 CHECK CHECKFLAGS MAKE UTS_MACHINE HOSTCXX
 export KGZIP KBZIP2 KLZOP LZMA LZ4 XZ ZSTD
@@ -1596,7 +1599,7 @@ endif # CONFIG_MODULES
 # Directories & files removed with 'make clean'
 CLEAN_FILES += include/ksym vmlinux.symvers modules-only.symvers \
 	       modules.builtin modules.builtin.modinfo modules.nsdeps \
-	       compile_commands.json .thinlto-cache
+	       compile_commands.json .thinlto-cache rust/test rust/doc
 
 # Directories & files removed with 'make mrproper'
 MRPROPER_FILES += include/config include/generated          \
@@ -1726,7 +1729,7 @@ help:
 	@echo  '  rustdoc	  - Generate Rust documentation'
 	@echo  '		    (requires kernel .config)'
 	@echo  '  rusttest        - Runs the Rust tests'
-	@echo  '                    (requires kernel .config)'
+	@echo  '                    (requires kernel .config; downloads external repos)'
 	@echo  '  rust-analyzer	  - Generate rust-project.json rust-analyzer support file'
 	@echo  '		    (requires kernel .config)'
 	@echo  ''
@@ -1820,11 +1823,21 @@ rusttest: prepare0
 # Formatting targets
 PHONY += rustfmt rustfmtcheck
 
+# We skip `rust/alloc` since we want to minimize the diff w.r.t. upstream.
+#
+# We match using absolute paths since `find` does not resolve them
+# when matching, which is a problem when e.g. `srctree` is `..`.
+# We `grep` afterwards in order to remove the directory entry itself.
 rustfmt:
-	find $(srctree) -type f -name '*.rs' | xargs $(RUSTFMT)
-
-rustfmtcheck:
-	find $(srctree) -type f -name '*.rs' | xargs $(RUSTFMT) --check
+	$(Q)find $(abs_srctree) -type f -name '*.rs' \
+		-o -path $(abs_srctree)/rust/alloc -prune \
+		-o -path $(abs_objtree)/rust/test -prune \
+		| grep -Fv $(abs_srctree)/rust/alloc \
+		| grep -Fv $(abs_objtree)/rust/test \
+		| xargs $(RUSTFMT) $(rustfmt_flags)
+
+rustfmtcheck: rustfmt_flags = --check
+rustfmtcheck: rustfmt
 
 # IDE support targets
 PHONY += rust-analyzer

drivers/android/process.rs

@@ -614,12 +614,11 @@ impl Process {
         for _ in 0..page_count {
             let page = Pages::<0>::new()?;
             page.insert_page(vma, address)?;
-            pages.push(page);
+            pages.try_push(page)?;
             address += 1 << bindings::PAGE_SHIFT;
         }
 
-        // TODO: This allocates memory.
-        let arc = Arc::from(pages);
+        let arc = Arc::try_from_vec(pages)?;
 
         // Save pages for later.
         let mut inner = self.inner.lock();