From fa6b05feaf32de4b255e41b4710d70fcc0863841 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Wed, 5 May 2021 10:05:54 -0400 Subject: [PATCH 01/10] Delete rustfmt submodule --- .gitmodules | 3 --- src/tools/rustfmt | 1 - 2 files changed, 4 deletions(-) delete mode 160000 src/tools/rustfmt diff --git a/.gitmodules b/.gitmodules index d4a80efc277e3..aee038aa4c8e5 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,9 +16,6 @@ [submodule "src/tools/rls"] path = src/tools/rls url = https://github.com/rust-lang/rls.git -[submodule "src/tools/rustfmt"] - path = src/tools/rustfmt - url = https://github.com/rust-lang/rustfmt.git [submodule "src/tools/miri"] path = src/tools/miri url = https://github.com/rust-lang/miri.git diff --git a/src/tools/rustfmt b/src/tools/rustfmt deleted file mode 160000 index 2a3635d5d1218..0000000000000 --- a/src/tools/rustfmt +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2a3635d5d1218c726ff58af4bc35418836143f69 From 9c1c134e2a541015559842f2ed2dc3094fd0e6f9 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Tue, 16 Feb 2021 21:59:28 -0500 Subject: [PATCH 02/10] Don't show CRLF tests as modified in the working directory This was being overriden by `lf=eof` in `.gitattributes` in the root directory. Override it again to avoid warnings like this: ``` $ git diff src/tools/rustfmt/tests/target/issue-3494/crlf.rs warning: CRLF will be replaced by LF in src/tools/rustfmt/tests/target/issue-3494/crlf.rs. The file will have its original line endings in your working directory ``` --- src/tools/rustfmt/.gitattributes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tools/rustfmt/.gitattributes b/src/tools/rustfmt/.gitattributes index bce42ce79cc71..7cb7459e0bd33 100644 --- a/src/tools/rustfmt/.gitattributes +++ b/src/tools/rustfmt/.gitattributes @@ -1,2 +1,2 @@ # Stop git from showing CRLF tests as modified -* eol=Unset +* text=auto eol=Unset From eb0a62411e1c6c7e3855453382284931b35076b0 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Tue, 16 Feb 2021 22:37:17 -0500 Subject: [PATCH 03/10] Update bootstrap for in-tree rustfmt - Add rustfmt to `x.py check` - Update Cargo.lock - Remove rustfmt from the toolstate list - Make rustfmt an in-tree tool - Give an error on `x.py test rustfmt` if rustfmt fails to build or if tests fail - Don't call `save_toolstate` when testing rustfmt --- src/bootstrap/builder.rs | 1 + src/bootstrap/check.rs | 3 ++- src/bootstrap/test.rs | 16 ++++------------ src/bootstrap/tool.rs | 4 ++-- src/bootstrap/toolstate.rs | 8 +++----- .../host-x86_64/x86_64-gnu-tools/checktools.sh | 2 +- 6 files changed, 13 insertions(+), 21 deletions(-) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 62a3a87eeb850..fb1bba175b480 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -377,6 +377,7 @@ impl<'a> Builder<'a> { check::Rustdoc, check::CodegenBackend, check::Clippy, + check::Rustfmt, check::Bootstrap ), Kind::Test => describe!( diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs index 9b76c8b9a2d26..15f553b69ad2d 100644 --- a/src/bootstrap/check.rs +++ b/src/bootstrap/check.rs @@ -362,11 +362,12 @@ macro_rules! tool_check_step { } tool_check_step!(Rustdoc, "src/tools/rustdoc", "src/librustdoc", SourceType::InTree); -// Clippy is a hybrid. It is an external tool, but uses a git subtree instead +// Clippy and Rustfmt are hybrids. They are external tools, but use a git subtree instead // of a submodule. Since the SourceType only drives the deny-warnings // behavior, treat it as in-tree so that any new warnings in clippy will be // rejected. tool_check_step!(Clippy, "src/tools/clippy", SourceType::InTree); +tool_check_step!(Rustfmt, "src/tools/rustfmt", SourceType::InTree); tool_check_step!(Bootstrap, "src/bootstrap", SourceType::InTree); diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index de9c1882c7d48..92bed962121b0 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -318,15 +318,9 @@ impl Step for Rustfmt { let host = self.host; let compiler = builder.compiler(stage, host); - let build_result = builder.ensure(tool::Rustfmt { - compiler, - target: self.host, - extra_features: Vec::new(), - }); - if build_result.is_none() { - eprintln!("failed to test rustfmt: could not build"); - return; - } + builder + .ensure(tool::Rustfmt { compiler, target: self.host, extra_features: Vec::new() }) + .expect("in-tree tool"); let mut cargo = tool::prepare_tool_cargo( builder, @@ -345,9 +339,7 @@ impl Step for Rustfmt { cargo.add_rustc_lib_path(builder, compiler); - if try_run(builder, &mut cargo.into()) { - builder.save_toolstate("rustfmt", ToolState::TestPass); - } + builder.run(&mut cargo.into()); } } diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs index 4f2426648fd8a..12571c1b97db6 100644 --- a/src/bootstrap/tool.rs +++ b/src/bootstrap/tool.rs @@ -726,7 +726,7 @@ macro_rules! tool_extended { // Note: tools need to be also added to `Builder::get_step_descriptions` in `builder.rs` // to make `./x.py build ` work. tool_extended!((self, builder), - Cargofmt, rustfmt, "src/tools/rustfmt", "cargo-fmt", stable=true, {}; + Cargofmt, rustfmt, "src/tools/rustfmt", "cargo-fmt", stable=true, in_tree=true, {}; CargoClippy, clippy, "src/tools/clippy", "cargo-clippy", stable=true, in_tree=true, {}; Clippy, clippy, "src/tools/clippy", "clippy-driver", stable=true, in_tree=true, {}; Miri, miri, "src/tools/miri", "miri", stable=false, {}; @@ -740,7 +740,7 @@ tool_extended!((self, builder), self.extra_features.push("clippy".to_owned()); }; RustDemangler, rust_demangler, "src/tools/rust-demangler", "rust-demangler", stable=false, in_tree=true, {}; - Rustfmt, rustfmt, "src/tools/rustfmt", "rustfmt", stable=true, {}; + Rustfmt, rustfmt, "src/tools/rustfmt", "rustfmt", stable=true, in_tree=true, {}; RustAnalyzer, rust_analyzer, "src/tools/rust-analyzer/crates/rust-analyzer", "rust-analyzer", stable=false, {}; ); diff --git a/src/bootstrap/toolstate.rs b/src/bootstrap/toolstate.rs index 205524ad84fb7..2394c5e020d2b 100644 --- a/src/bootstrap/toolstate.rs +++ b/src/bootstrap/toolstate.rs @@ -77,7 +77,6 @@ static STABLE_TOOLS: &[(&str, &str)] = &[ ("rust-by-example", "src/doc/rust-by-example"), ("edition-guide", "src/doc/edition-guide"), ("rls", "src/tools/rls"), - ("rustfmt", "src/tools/rustfmt"), ]; // These tools are permitted to not build on the beta/stable channels. @@ -278,10 +277,9 @@ impl Builder<'_> { if self.config.dry_run { return; } - // Toolstate isn't tracked for clippy, but since most tools do, we avoid - // checking in all the places we could save toolstate and just do so - // here. - if tool == "clippy-driver" { + // Toolstate isn't tracked for clippy or rustfmt, but since most tools do, we avoid checking + // in all the places we could save toolstate and just do so here. + if tool == "clippy-driver" || tool == "rustfmt" { return; } if let Some(ref path) = self.config.save_toolstates { diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh b/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh index 49a8e5e88a039..412efe5c4480d 100755 --- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh +++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh @@ -15,7 +15,6 @@ python3 "$X_PY" test --stage 2 --no-fail-fast \ src/doc/embedded-book \ src/doc/edition-guide \ src/tools/rls \ - src/tools/rustfmt \ src/tools/miri \ set -e @@ -24,3 +23,4 @@ set -e cat /tmp/toolstate/toolstates.json python3 "$X_PY" test --stage 2 check-tools python3 "$X_PY" test --stage 2 src/tools/clippy +python3 "$X_PY" test --stage 2 src/tools/rustfmt From f3a47cc6ac68467fef0f4cba175960261234bf4b Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Sun, 2 May 2021 14:27:57 -0400 Subject: [PATCH 04/10] Don't copy tool dependencies to the sysroot This fixes the following error: ``` error: found crates (`serde_derive` and `serde_derive`) with colliding StableCrateId values. --> /home/joshua/.local/lib/cargo/registry/src/github.com-1ecc6299db9ec823/cargo_metadata-0.8.2/src/lib.rs:162:1 | 162 | extern crate serde_derive; | ^^^^^^^^^^^^^^^^^^^^^^^^^^ ``` This is a bug in resolve (https://github.com/rust-lang/rust/issues/56935) but it will be difficult to fix in the near future. This works around it in the meantime by not copying serde_derive and other dependencies to the sysroot when they're built for other tools. This rebuilds the dependencies slightly more often than necessary, but avoids the crate conflicts. This can be reverted once #56935 is fixed. --- src/bootstrap/check.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs index 15f553b69ad2d..2948a143c99d3 100644 --- a/src/bootstrap/check.rs +++ b/src/bootstrap/check.rs @@ -341,10 +341,6 @@ macro_rules! tool_check_step { true, ); - let libdir = builder.sysroot_libdir(compiler, target); - let hostdir = builder.sysroot_libdir(compiler, compiler.host); - add_to_sysroot(&builder, &libdir, &hostdir, &stamp(builder, compiler, target)); - /// Cargo's output path in a given stage, compiled by a particular /// compiler for the specified target. fn stamp( From 619dd29bcff7e2b0e9816d7f561aec69a018b6de Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Sun, 2 May 2021 16:56:25 -0400 Subject: [PATCH 05/10] Update log to 0.4.14 This avoids the following warning: ``` warning: trailing semicolon in macro used in expression position --> /home/joshua/.local/lib/cargo/registry/src/github.com-1ecc6299db9ec823/log-0.4.11/src/macros.rs:152:45 | 147 | / macro_rules! debug { 148 | | (target: $target:expr, $($arg:tt)+) => ( 149 | | log!(target: $target, $crate::Level::Debug, $($arg)+); 150 | | ); 151 | | ($($arg:tt)+) => ( 152 | | log!($crate::Level::Debug, $($arg)+); | | ^ 153 | | ) 154 | | } | |_- in this expansion of `debug!` | ::: src/tools/rustfmt/src/modules/visitor.rs:36:23 | 36 | Err(e) => debug!("{}", e), | --------------- in this macro invocation | = note: requested on the command line with `-W semicolon-in-expressions-from-macros` = warning: this was previously accepted by the compiler but is being phased out; it will become a hard error in a future release! = note: for more information, see issue #79813 ``` --- Cargo.lock | 6 +++--- src/tools/rustfmt/Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0939f19cdfe20..ec7991928abb1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1974,11 +1974,11 @@ dependencies = [ [[package]] name = "log" -version = "0.4.11" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", ] [[package]] diff --git a/src/tools/rustfmt/Cargo.toml b/src/tools/rustfmt/Cargo.toml index 24b3b79343b05..3a04fb28f7cbf 100644 --- a/src/tools/rustfmt/Cargo.toml +++ b/src/tools/rustfmt/Cargo.toml @@ -42,7 +42,7 @@ unicode-segmentation = "1.0.0" regex = "1.0" term = "0.6" diff = "0.1" -log = "0.4" +log = "0.4.14" env_logger = "0.6" getopts = "0.2" derive-new = "0.5" From 0312d86b71c5972871a1536ac4544ee6edbc7de9 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Wed, 5 May 2021 19:13:38 -0400 Subject: [PATCH 06/10] Run toolstate jobs when src/tools/rustfmt is modified. Previously, this would be caught by a change for modified submodules; now that rustfmt is no longer a submodule, the check needs to be explicit. --- src/ci/scripts/should-skip-this.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ci/scripts/should-skip-this.sh b/src/ci/scripts/should-skip-this.sh index 36bf4368990c9..03fad62d331fd 100755 --- a/src/ci/scripts/should-skip-this.sh +++ b/src/ci/scripts/should-skip-this.sh @@ -14,10 +14,10 @@ elif git diff HEAD^ | grep --quiet "^index .* 160000"; then # Submodules pseudo-files inside git have the 160000 permissions, so when # those files are present in the diff a submodule was updated. echo "Executing the job since submodules are updated" -elif git diff --name-only HEAD^ | grep --quiet src/tools/clippy; then +elif git diff --name-only HEAD^ | grep --quiet src/tools/'\(clippy\|rustfmt\)'; then # There is not an easy blanket search for subtrees. For now, manually list # clippy. - echo "Executing the job since clippy subtree was updated" + echo "Executing the job since clippy or rustfmt subtree was updated" else echo "Not executing this job since no submodules were updated" ciCommandSetEnv SKIP_JOB 1 From 444164235192a9c3d36b51ed01d223b998b2db74 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Wed, 5 May 2021 19:21:42 -0400 Subject: [PATCH 07/10] should-skip-this: Check for changes between the master branch, not the previous commit. The previous commit could be part of the current PR. --- src/ci/scripts/should-skip-this.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/ci/scripts/should-skip-this.sh b/src/ci/scripts/should-skip-this.sh index 03fad62d331fd..ebe4a31dd89b4 100755 --- a/src/ci/scripts/should-skip-this.sh +++ b/src/ci/scripts/should-skip-this.sh @@ -6,15 +6,19 @@ set -euo pipefail IFS=$'\n\t' +git fetch "https://github.com/$GITHUB_REPOSITORY" "$GITHUB_BASE_REF" +BASE_COMMIT="$(git merge-base FETCH_HEAD HEAD)" + +echo "Searching for toolstate changes between $BASE_COMMIT and $(git rev-parse HEAD)" source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" if [[ -z "${CI_ONLY_WHEN_SUBMODULES_CHANGED+x}" ]]; then echo "Executing the job since there is no skip rule in effect" -elif git diff HEAD^ | grep --quiet "^index .* 160000"; then +elif git diff "$BASE_COMMIT" | grep --quiet "^index .* 160000"; then # Submodules pseudo-files inside git have the 160000 permissions, so when # those files are present in the diff a submodule was updated. echo "Executing the job since submodules are updated" -elif git diff --name-only HEAD^ | grep --quiet src/tools/'\(clippy\|rustfmt\)'; then +elif git diff --name-only "$BASE_COMMIT" | grep --quiet src/tools/'\(clippy\|rustfmt\)'; then # There is not an easy blanket search for subtrees. For now, manually list # clippy. echo "Executing the job since clippy or rustfmt subtree was updated" From ea6ec114bfeb5b5b8afe9d2112f86df62151a336 Mon Sep 17 00:00:00 2001 From: Caleb Cartwright Date: Mon, 10 May 2021 21:29:48 -0500 Subject: [PATCH 08/10] ci(should-skip-this): only check commits when skip rule enabled --- src/ci/scripts/should-skip-this.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/ci/scripts/should-skip-this.sh b/src/ci/scripts/should-skip-this.sh index ebe4a31dd89b4..dde240b17a8b9 100755 --- a/src/ci/scripts/should-skip-this.sh +++ b/src/ci/scripts/should-skip-this.sh @@ -6,23 +6,27 @@ set -euo pipefail IFS=$'\n\t' +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +if [[ -z "${CI_ONLY_WHEN_SUBMODULES_CHANGED+x}" ]]; then + echo "Executing the job since there is no skip rule in effect" + exit 0 +fi + git fetch "https://github.com/$GITHUB_REPOSITORY" "$GITHUB_BASE_REF" BASE_COMMIT="$(git merge-base FETCH_HEAD HEAD)" echo "Searching for toolstate changes between $BASE_COMMIT and $(git rev-parse HEAD)" -source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" -if [[ -z "${CI_ONLY_WHEN_SUBMODULES_CHANGED+x}" ]]; then - echo "Executing the job since there is no skip rule in effect" -elif git diff "$BASE_COMMIT" | grep --quiet "^index .* 160000"; then +if git diff "$BASE_COMMIT" | grep --quiet "^index .* 160000"; then # Submodules pseudo-files inside git have the 160000 permissions, so when # those files are present in the diff a submodule was updated. echo "Executing the job since submodules are updated" elif git diff --name-only "$BASE_COMMIT" | grep --quiet src/tools/'\(clippy\|rustfmt\)'; then # There is not an easy blanket search for subtrees. For now, manually list - # clippy. + # the subtrees. echo "Executing the job since clippy or rustfmt subtree was updated" else - echo "Not executing this job since no submodules were updated" + echo "Not executing this job since no submodules nor subtrees were updated" ciCommandSetEnv SKIP_JOB 1 fi From 67a48e2d6822068c742ff9547fd0cf5d1fc465d5 Mon Sep 17 00:00:00 2001 From: Eric Huss Date: Mon, 10 May 2021 20:32:39 -0700 Subject: [PATCH 09/10] Eric test --- .github/workflows/ci.yml | 607 +---------------------------- src/ci/scripts/should-skip-this.sh | 5 + 2 files changed, 6 insertions(+), 606 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ffaa2b03df9e7..1fc1b61578faf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,16 +40,10 @@ jobs: strategy: matrix: include: - - name: mingw-check - os: ubuntu-latest-xl - env: {} - - name: x86_64-gnu-llvm-10 - os: ubuntu-latest-xl - env: {} - name: x86_64-gnu-tools env: CI_ONLY_WHEN_SUBMODULES_CHANGED: 1 - os: ubuntu-latest-xl + os: ubuntu-latest timeout-minutes: 600 runs-on: "${{ matrix.os }}" steps: @@ -72,602 +66,3 @@ jobs: - name: decide whether to skip this job run: src/ci/scripts/should-skip-this.sh if: success() && !env.SKIP_JOB - - name: configure GitHub Actions to kill the build when outdated - uses: rust-lang/simpleinfra/github-actions/cancel-outdated-builds@master - with: - github_token: "${{ secrets.github_token }}" - if: "success() && !env.SKIP_JOB && github.ref != 'refs/heads/try'" - - name: collect CPU statistics - run: src/ci/scripts/collect-cpu-stats.sh - if: success() && !env.SKIP_JOB - - name: show the current environment - run: src/ci/scripts/dump-environment.sh - if: success() && !env.SKIP_JOB - - name: install awscli - run: src/ci/scripts/install-awscli.sh - if: success() && !env.SKIP_JOB - - name: install sccache - run: src/ci/scripts/install-sccache.sh - if: success() && !env.SKIP_JOB - - name: select Xcode - run: src/ci/scripts/select-xcode.sh - if: success() && !env.SKIP_JOB - - name: install clang - run: src/ci/scripts/install-clang.sh - if: success() && !env.SKIP_JOB - - name: install WIX - run: src/ci/scripts/install-wix.sh - if: success() && !env.SKIP_JOB - - name: ensure the build happens on a partition with enough space - run: src/ci/scripts/symlink-build-dir.sh - if: success() && !env.SKIP_JOB - - name: disable git crlf conversion - run: src/ci/scripts/disable-git-crlf-conversion.sh - if: success() && !env.SKIP_JOB - - name: install MSYS2 - run: src/ci/scripts/install-msys2.sh - if: success() && !env.SKIP_JOB - - name: install MinGW - run: src/ci/scripts/install-mingw.sh - if: success() && !env.SKIP_JOB - - name: install ninja - run: src/ci/scripts/install-ninja.sh - if: success() && !env.SKIP_JOB - - name: enable ipv6 on Docker - run: src/ci/scripts/enable-docker-ipv6.sh - if: success() && !env.SKIP_JOB - - name: disable git crlf conversion - run: src/ci/scripts/disable-git-crlf-conversion.sh - if: success() && !env.SKIP_JOB - - name: checkout submodules - run: src/ci/scripts/checkout-submodules.sh - if: success() && !env.SKIP_JOB - - name: ensure line endings are correct - run: src/ci/scripts/verify-line-endings.sh - if: success() && !env.SKIP_JOB - - name: run the build - run: src/ci/scripts/run-build-from-ci.sh - env: - AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}" - AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}" - TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}" - if: success() && !env.SKIP_JOB - - name: upload artifacts to S3 - run: src/ci/scripts/upload-artifacts.sh - env: - AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}" - AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}" - if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')" - auto: - name: auto - env: - CI_JOB_NAME: "${{ matrix.name }}" - SCCACHE_BUCKET: rust-lang-ci-sccache2 - DEPLOY_BUCKET: rust-lang-ci2 - TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate" - TOOLSTATE_ISSUES_API_URL: "https://api.github.com/repos/rust-lang/rust/issues" - TOOLSTATE_PUBLISH: 1 - CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL - ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55 - CACHE_DOMAIN: ci-caches.rust-lang.org - if: "github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'" - strategy: - matrix: - include: - - name: aarch64-gnu - os: - - self-hosted - - ARM64 - - linux - - name: arm-android - os: ubuntu-latest-xl - env: {} - - name: armhf-gnu - os: ubuntu-latest-xl - env: {} - - name: dist-aarch64-linux - os: ubuntu-latest-xl - env: {} - - name: dist-android - os: ubuntu-latest-xl - env: {} - - name: dist-arm-linux - os: ubuntu-latest-xl - env: {} - - name: dist-armhf-linux - os: ubuntu-latest-xl - env: {} - - name: dist-armv7-linux - os: ubuntu-latest-xl - env: {} - - name: dist-i586-gnu-i586-i686-musl - os: ubuntu-latest-xl - env: {} - - name: dist-i686-linux - os: ubuntu-latest-xl - env: {} - - name: dist-mips-linux - os: ubuntu-latest-xl - env: {} - - name: dist-mips64-linux - os: ubuntu-latest-xl - env: {} - - name: dist-mips64el-linux - os: ubuntu-latest-xl - env: {} - - name: dist-mipsel-linux - os: ubuntu-latest-xl - env: {} - - name: dist-powerpc-linux - os: ubuntu-latest-xl - env: {} - - name: dist-powerpc64-linux - os: ubuntu-latest-xl - env: {} - - name: dist-powerpc64le-linux - os: ubuntu-latest-xl - env: {} - - name: dist-riscv64-linux - os: ubuntu-latest-xl - env: {} - - name: dist-s390x-linux - os: ubuntu-latest-xl - env: {} - - name: dist-various-1 - os: ubuntu-latest-xl - env: {} - - name: dist-various-2 - os: ubuntu-latest-xl - env: {} - - name: dist-x86_64-freebsd - os: ubuntu-latest-xl - env: {} - - name: dist-x86_64-illumos - os: ubuntu-latest-xl - env: {} - - name: dist-x86_64-linux - os: ubuntu-latest-xl - env: {} - - name: dist-x86_64-linux-alt - env: - IMAGE: dist-x86_64-linux - os: ubuntu-latest-xl - - name: dist-x86_64-musl - os: ubuntu-latest-xl - env: {} - - name: dist-x86_64-netbsd - os: ubuntu-latest-xl - env: {} - - name: i686-gnu - os: ubuntu-latest-xl - env: {} - - name: i686-gnu-nopt - os: ubuntu-latest-xl - env: {} - - name: mingw-check - os: ubuntu-latest-xl - env: {} - - name: test-various - os: ubuntu-latest-xl - env: {} - - name: wasm32 - os: ubuntu-latest-xl - env: {} - - name: x86_64-gnu - os: ubuntu-latest-xl - env: {} - - name: x86_64-gnu-aux - os: ubuntu-latest-xl - env: {} - - name: x86_64-gnu-debug - os: ubuntu-latest-xl - env: {} - - name: x86_64-gnu-distcheck - os: ubuntu-latest-xl - env: {} - - name: x86_64-gnu-llvm-10 - env: - RUST_BACKTRACE: 1 - os: ubuntu-latest-xl - - name: x86_64-gnu-nopt - os: ubuntu-latest-xl - env: {} - - name: x86_64-gnu-tools - env: - DEPLOY_TOOLSTATES_JSON: toolstates-linux.json - os: ubuntu-latest-xl - - name: dist-x86_64-apple - env: - SCRIPT: "./x.py dist" - RUST_CONFIGURE_ARGS: "--host=x86_64-apple-darwin --target=x86_64-apple-darwin,aarch64-apple-ios,x86_64-apple-ios --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false" - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - DIST_REQUIRE_ALL_TOOLS: 1 - os: macos-latest - - name: dist-x86_64-apple-alt - env: - SCRIPT: "./x.py dist" - RUST_CONFIGURE_ARGS: "--enable-extended --enable-profiler --set rust.jemalloc --set llvm.ninja=false" - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - os: macos-latest - - name: x86_64-apple - env: - SCRIPT: "./x.py --stage 2 test" - RUST_CONFIGURE_ARGS: "--build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false" - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - MACOSX_DEPLOYMENT_TARGET: 10.8 - MACOSX_STD_DEPLOYMENT_TARGET: 10.7 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - os: macos-latest - - name: dist-aarch64-apple - env: - SCRIPT: "./x.py dist --stage 2" - RUST_CONFIGURE_ARGS: "--build=x86_64-apple-darwin --host=aarch64-apple-darwin --target=aarch64-apple-darwin --enable-full-tools --enable-sanitizers --enable-profiler --set rust.jemalloc --set llvm.ninja=false" - RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 - SELECT_XCODE: /Applications/Xcode_12.2.app - USE_XCODE_CLANG: 1 - MACOSX_DEPLOYMENT_TARGET: 11.0 - MACOSX_STD_DEPLOYMENT_TARGET: 11.0 - NO_LLVM_ASSERTIONS: 1 - NO_DEBUG_ASSERTIONS: 1 - DIST_REQUIRE_ALL_TOOLS: 1 - JEMALLOC_SYS_WITH_LG_PAGE: 14 - os: macos-latest - - name: x86_64-msvc-1 - env: - RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-profiler" - SCRIPT: make ci-subset-1 - os: windows-latest-xl - - name: x86_64-msvc-2 - env: - RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-profiler" - SCRIPT: make ci-subset-2 - os: windows-latest-xl - - name: i686-msvc-1 - env: - RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-msvc" - SCRIPT: make ci-subset-1 - os: windows-latest-xl - - name: i686-msvc-2 - env: - RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-msvc" - SCRIPT: make ci-subset-2 - os: windows-latest-xl - - name: x86_64-msvc-cargo - env: - SCRIPT: python x.py --stage 2 test src/tools/cargotest src/tools/cargo - RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-lld" - VCVARS_BAT: vcvars64.bat - os: windows-latest-xl - - name: x86_64-msvc-tools - env: - SCRIPT: src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows - RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstate/toolstates.json" - DEPLOY_TOOLSTATES_JSON: toolstates-windows.json - os: windows-latest-xl - - name: i686-mingw-1 - env: - RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu" - SCRIPT: make ci-mingw-subset-1 - CUSTOM_MINGW: 1 - os: windows-latest-xl - - name: i686-mingw-2 - env: - RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu" - SCRIPT: make ci-mingw-subset-2 - CUSTOM_MINGW: 1 - os: windows-latest-xl - - name: x86_64-mingw-1 - env: - SCRIPT: make ci-mingw-subset-1 - RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-profiler" - CUSTOM_MINGW: 1 - os: windows-latest-xl - - name: x86_64-mingw-2 - env: - SCRIPT: make ci-mingw-subset-2 - RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-profiler" - CUSTOM_MINGW: 1 - os: windows-latest-xl - - name: dist-x86_64-msvc - env: - RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=x86_64-pc-windows-msvc --target=x86_64-pc-windows-msvc --enable-full-tools --enable-profiler" - SCRIPT: python x.py dist - DIST_REQUIRE_ALL_TOOLS: 1 - os: windows-latest-xl - - name: dist-i686-msvc - env: - RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-msvc --host=i686-pc-windows-msvc --target=i686-pc-windows-msvc,i586-pc-windows-msvc --enable-full-tools --enable-profiler" - SCRIPT: python x.py dist - DIST_REQUIRE_ALL_TOOLS: 1 - os: windows-latest-xl - - name: dist-aarch64-msvc - env: - RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --host=aarch64-pc-windows-msvc --enable-full-tools --enable-profiler" - SCRIPT: python x.py dist - DIST_REQUIRE_ALL_TOOLS: 0 - os: windows-latest-xl - - name: dist-i686-mingw - env: - RUST_CONFIGURE_ARGS: "--build=i686-pc-windows-gnu --enable-full-tools --enable-profiler" - SCRIPT: python x.py dist - CUSTOM_MINGW: 1 - DIST_REQUIRE_ALL_TOOLS: 1 - os: windows-latest-xl - - name: dist-x86_64-mingw - env: - SCRIPT: python x.py dist - RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-gnu --enable-full-tools --enable-profiler" - CUSTOM_MINGW: 1 - DIST_REQUIRE_ALL_TOOLS: 1 - os: windows-latest-xl - - name: dist-x86_64-msvc-alt - env: - RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-extended --enable-profiler" - SCRIPT: python x.py dist - os: windows-latest-xl - timeout-minutes: 600 - runs-on: "${{ matrix.os }}" - steps: - - name: disable git crlf conversion - run: git config --global core.autocrlf false - - name: checkout the source code - uses: actions/checkout@v2 - with: - fetch-depth: 2 - - name: configure the PR in which the error message will be posted - run: "echo \"[CI_PR_NUMBER=$num]\"" - env: - num: "${{ github.event.number }}" - if: "success() && !env.SKIP_JOB && github.event_name == 'pull_request'" - - name: add extra environment variables - run: src/ci/scripts/setup-environment.sh - env: - EXTRA_VARIABLES: "${{ toJson(matrix.env) }}" - if: success() && !env.SKIP_JOB - - name: decide whether to skip this job - run: src/ci/scripts/should-skip-this.sh - if: success() && !env.SKIP_JOB - - name: configure GitHub Actions to kill the build when outdated - uses: rust-lang/simpleinfra/github-actions/cancel-outdated-builds@master - with: - github_token: "${{ secrets.github_token }}" - if: "success() && !env.SKIP_JOB && github.ref != 'refs/heads/try'" - - name: collect CPU statistics - run: src/ci/scripts/collect-cpu-stats.sh - if: success() && !env.SKIP_JOB - - name: show the current environment - run: src/ci/scripts/dump-environment.sh - if: success() && !env.SKIP_JOB - - name: install awscli - run: src/ci/scripts/install-awscli.sh - if: success() && !env.SKIP_JOB - - name: install sccache - run: src/ci/scripts/install-sccache.sh - if: success() && !env.SKIP_JOB - - name: select Xcode - run: src/ci/scripts/select-xcode.sh - if: success() && !env.SKIP_JOB - - name: install clang - run: src/ci/scripts/install-clang.sh - if: success() && !env.SKIP_JOB - - name: install WIX - run: src/ci/scripts/install-wix.sh - if: success() && !env.SKIP_JOB - - name: ensure the build happens on a partition with enough space - run: src/ci/scripts/symlink-build-dir.sh - if: success() && !env.SKIP_JOB - - name: disable git crlf conversion - run: src/ci/scripts/disable-git-crlf-conversion.sh - if: success() && !env.SKIP_JOB - - name: install MSYS2 - run: src/ci/scripts/install-msys2.sh - if: success() && !env.SKIP_JOB - - name: install MinGW - run: src/ci/scripts/install-mingw.sh - if: success() && !env.SKIP_JOB - - name: install ninja - run: src/ci/scripts/install-ninja.sh - if: success() && !env.SKIP_JOB - - name: enable ipv6 on Docker - run: src/ci/scripts/enable-docker-ipv6.sh - if: success() && !env.SKIP_JOB - - name: disable git crlf conversion - run: src/ci/scripts/disable-git-crlf-conversion.sh - if: success() && !env.SKIP_JOB - - name: checkout submodules - run: src/ci/scripts/checkout-submodules.sh - if: success() && !env.SKIP_JOB - - name: ensure line endings are correct - run: src/ci/scripts/verify-line-endings.sh - if: success() && !env.SKIP_JOB - - name: run the build - run: src/ci/scripts/run-build-from-ci.sh - env: - AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}" - AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}" - TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}" - if: success() && !env.SKIP_JOB - - name: upload artifacts to S3 - run: src/ci/scripts/upload-artifacts.sh - env: - AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}" - AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}" - if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')" - try: - name: try - env: - CI_JOB_NAME: "${{ matrix.name }}" - SCCACHE_BUCKET: rust-lang-ci-sccache2 - DEPLOY_BUCKET: rust-lang-ci2 - TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate" - TOOLSTATE_ISSUES_API_URL: "https://api.github.com/repos/rust-lang/rust/issues" - TOOLSTATE_PUBLISH: 1 - CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL - ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55 - CACHE_DOMAIN: ci-caches.rust-lang.org - if: "github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'" - strategy: - matrix: - include: - - name: dist-x86_64-linux - os: ubuntu-latest-xl - env: {} - timeout-minutes: 600 - runs-on: "${{ matrix.os }}" - steps: - - name: disable git crlf conversion - run: git config --global core.autocrlf false - - name: checkout the source code - uses: actions/checkout@v2 - with: - fetch-depth: 2 - - name: configure the PR in which the error message will be posted - run: "echo \"[CI_PR_NUMBER=$num]\"" - env: - num: "${{ github.event.number }}" - if: "success() && !env.SKIP_JOB && github.event_name == 'pull_request'" - - name: add extra environment variables - run: src/ci/scripts/setup-environment.sh - env: - EXTRA_VARIABLES: "${{ toJson(matrix.env) }}" - if: success() && !env.SKIP_JOB - - name: decide whether to skip this job - run: src/ci/scripts/should-skip-this.sh - if: success() && !env.SKIP_JOB - - name: configure GitHub Actions to kill the build when outdated - uses: rust-lang/simpleinfra/github-actions/cancel-outdated-builds@master - with: - github_token: "${{ secrets.github_token }}" - if: "success() && !env.SKIP_JOB && github.ref != 'refs/heads/try'" - - name: collect CPU statistics - run: src/ci/scripts/collect-cpu-stats.sh - if: success() && !env.SKIP_JOB - - name: show the current environment - run: src/ci/scripts/dump-environment.sh - if: success() && !env.SKIP_JOB - - name: install awscli - run: src/ci/scripts/install-awscli.sh - if: success() && !env.SKIP_JOB - - name: install sccache - run: src/ci/scripts/install-sccache.sh - if: success() && !env.SKIP_JOB - - name: select Xcode - run: src/ci/scripts/select-xcode.sh - if: success() && !env.SKIP_JOB - - name: install clang - run: src/ci/scripts/install-clang.sh - if: success() && !env.SKIP_JOB - - name: install WIX - run: src/ci/scripts/install-wix.sh - if: success() && !env.SKIP_JOB - - name: ensure the build happens on a partition with enough space - run: src/ci/scripts/symlink-build-dir.sh - if: success() && !env.SKIP_JOB - - name: disable git crlf conversion - run: src/ci/scripts/disable-git-crlf-conversion.sh - if: success() && !env.SKIP_JOB - - name: install MSYS2 - run: src/ci/scripts/install-msys2.sh - if: success() && !env.SKIP_JOB - - name: install MinGW - run: src/ci/scripts/install-mingw.sh - if: success() && !env.SKIP_JOB - - name: install ninja - run: src/ci/scripts/install-ninja.sh - if: success() && !env.SKIP_JOB - - name: enable ipv6 on Docker - run: src/ci/scripts/enable-docker-ipv6.sh - if: success() && !env.SKIP_JOB - - name: disable git crlf conversion - run: src/ci/scripts/disable-git-crlf-conversion.sh - if: success() && !env.SKIP_JOB - - name: checkout submodules - run: src/ci/scripts/checkout-submodules.sh - if: success() && !env.SKIP_JOB - - name: ensure line endings are correct - run: src/ci/scripts/verify-line-endings.sh - if: success() && !env.SKIP_JOB - - name: run the build - run: src/ci/scripts/run-build-from-ci.sh - env: - AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}" - AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}" - TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}" - if: success() && !env.SKIP_JOB - - name: upload artifacts to S3 - run: src/ci/scripts/upload-artifacts.sh - env: - AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}" - AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}" - if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')" - master: - name: master - runs-on: ubuntu-latest - env: - SCCACHE_BUCKET: rust-lang-ci-sccache2 - DEPLOY_BUCKET: rust-lang-ci2 - TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate" - TOOLSTATE_ISSUES_API_URL: "https://api.github.com/repos/rust-lang/rust/issues" - TOOLSTATE_PUBLISH: 1 - CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL - ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55 - CACHE_DOMAIN: ci-caches.rust-lang.org - if: "github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'rust-lang-ci/rust'" - steps: - - name: checkout the source code - uses: actions/checkout@v2 - with: - fetch-depth: 2 - - name: publish toolstate - run: src/ci/publish_toolstate.sh - shell: bash - env: - TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}" - if: success() && !env.SKIP_JOB - try-success: - needs: - - try - if: "success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'" - steps: - - name: mark the job as a success - run: exit 0 - shell: bash - name: bors build finished - runs-on: ubuntu-latest - try-failure: - needs: - - try - if: "!success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'" - steps: - - name: mark the job as a failure - run: exit 1 - shell: bash - name: bors build finished - runs-on: ubuntu-latest - auto-success: - needs: - - auto - if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'" - steps: - - name: mark the job as a success - run: exit 0 - shell: bash - name: bors build finished - runs-on: ubuntu-latest - auto-failure: - needs: - - auto - if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'" - steps: - - name: mark the job as a failure - run: exit 1 - shell: bash - name: bors build finished - runs-on: ubuntu-latest diff --git a/src/ci/scripts/should-skip-this.sh b/src/ci/scripts/should-skip-this.sh index dde240b17a8b9..505d9d3589b37 100755 --- a/src/ci/scripts/should-skip-this.sh +++ b/src/ci/scripts/should-skip-this.sh @@ -13,11 +13,16 @@ if [[ -z "${CI_ONLY_WHEN_SUBMODULES_CHANGED+x}" ]]; then exit 0 fi +echo $GITHUB_REPOSITORY +echo base_ref=$GITHUB_BASE_REF + git fetch "https://github.com/$GITHUB_REPOSITORY" "$GITHUB_BASE_REF" BASE_COMMIT="$(git merge-base FETCH_HEAD HEAD)" echo "Searching for toolstate changes between $BASE_COMMIT and $(git rev-parse HEAD)" +git diff --name-only "$BASE_COMMIT" + if git diff "$BASE_COMMIT" | grep --quiet "^index .* 160000"; then # Submodules pseudo-files inside git have the 160000 permissions, so when # those files are present in the diff a submodule was updated. From 6bb9dffadcd3dcdd9c6b85c4a6b36454188515aa Mon Sep 17 00:00:00 2001 From: Eric Huss Date: Mon, 10 May 2021 20:37:44 -0700 Subject: [PATCH 10/10] Test --- src/ci/scripts/should-skip-this.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/ci/scripts/should-skip-this.sh b/src/ci/scripts/should-skip-this.sh index 505d9d3589b37..7ce3ec18dfb8e 100755 --- a/src/ci/scripts/should-skip-this.sh +++ b/src/ci/scripts/should-skip-this.sh @@ -23,6 +23,16 @@ echo "Searching for toolstate changes between $BASE_COMMIT and $(git rev-parse H git diff --name-only "$BASE_COMMIT" +echo +echo ======================================= +echo + +git diff --name-only "$BASE_COMMIT" | grep --quiet src/tools/'\(clippy\|rustfmt\)' + +echo +echo ======================================= +echo + if git diff "$BASE_COMMIT" | grep --quiet "^index .* 160000"; then # Submodules pseudo-files inside git have the 160000 permissions, so when # those files are present in the diff a submodule was updated.