From 4bbf13481ac43ba1391a326f9a2da15cf565c201 Mon Sep 17 00:00:00 2001 From: Chase Wilson Date: Sat, 4 Jul 2020 13:49:14 -0600 Subject: [PATCH 1/4] Reworked CI --- .github/workflows/ci.yml | 140 ++++++++++++++++++++++++----------- .github/workflows/deps.yml | 59 +++++++++++++++ .github/workflows/docker.yml | 23 ++++++ docker-compose.yml | 32 ++++++++ 4 files changed, 211 insertions(+), 43 deletions(-) create mode 100644 .github/workflows/deps.yml create mode 100644 .github/workflows/docker.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 49ebbd9ee..b065302f7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,7 @@ ---- -on: [push, pull_request] name: CI +on: [push, pull_request] + env: RUST_BACKTRACE: 1 @@ -10,38 +10,43 @@ jobs: name: Test runs-on: ubuntu-latest - strategy: - fail-fast: false - steps: - uses: actions/checkout@master + - id: install + uses: actions-rs/toolchain@v1 with: - fetch-depth: 2 - - - name: Install stable Rust - run: rustup update stable && rustup default stable - - - name: Set rustc version - run: echo "::set-env name=CURRENT_RUSTC_VERSION::$(rustc -V)" + toolchain: stable + override: true + - uses: actions-rs/cargo@v1 + with: + command: install + args: cargo-sweep - # Caching - - name: Cache Cargo directories - uses: actions/cache@v2 + - name: Cache directories + uses: actions/cache@v1 with: path: | + ~/.cargo/registry ~/.cargo/bin ~/.cargo/git - ~/.cargo/registry - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - name: Cache Cargo Build - uses: actions/cache@v2 + key: cargo-dirs-${{ hashFiles('**/Cargo.lock') }} + restore-keys: cargo-dirs- + - name: Cache build + uses: actions/cache@v1 with: path: target - key: ${{ runner.os }}-cargo-build-target-${{ env.CURRENT_RUSTC_VERSION }}-${{ hashFiles('**/Cargo.lock') }} + key: cargo-build-${{ steps.install.outputs.rustc_hash }}-${{ hashFiles('**/Cargo.lock') }} restore-keys: | - ${{ runner.os }}-cargo-build-target-${{ env.CURRENT_RUSTC_VERSION }}- + cargo-build-${{ steps.install.outputs.rustc_hash }}- + cargo-build- - - name: Launch the postgres and min.io images + - name: Register artifacts + uses: actions-rs/cargo@v1 + with: + command: sweep + args: --stamp + + - name: Launch postgres and min.io run: | cp .env.sample .env . .env @@ -52,38 +57,87 @@ jobs: # Make sure the database is actually working psql "${CRATESFYI_DATABASE_URL}" - - name: Run rustfmt - run: cargo fmt -- --check - - - name: Run clippy - run: cargo clippy --locked -- -D warnings - - - name: Build docs.rs + - name: Build run: cargo build --locked - - name: Test docs.rs + - name: Test run: cargo test --locked -- --test-threads=1 - name: Clean up the database run: docker-compose down --volumes - docker: - name: Docker + - name: Clean unused artifacts + uses: actions-rs/cargo@v1 + with: + command: sweep + args: --file + + fmt: + name: Rustfmt runs-on: ubuntu-latest + steps: - uses: actions/checkout@master + - id: install + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + components: rustfmt + + - uses: actions-rs/cargo@v1 with: - fetch-depth: 2 + command: fmt + args: -- --check - - name: Build the Docker image - run: docker build -t docs-rs -f dockerfiles/Dockerfile . + clippy: + name: Clippy + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@master + - id: install + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + components: clippy + - uses: actions-rs/cargo@v1 + with: + command: install + args: cargo-sweep + + - name: Cache directories + uses: actions/cache@v1 + with: + path: | + ~/.cargo/registry + ~/.cargo/bin + ~/.cargo/git + key: cargo-dirs-${{ hashFiles('**/Cargo.lock') }} + restore-keys: cargo-dirs- + - name: Cache build + uses: actions/cache@v1 + with: + path: target + key: cargo-build-${{ steps.install.outputs.rustc_hash }}-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + cargo-build-${{ steps.install.outputs.rustc_hash }}- + cargo-build- + + - name: Register artifacts + uses: actions-rs/cargo@v1 + with: + command: sweep + args: --stamp + + - uses: actions-rs/cargo@v1 + with: + command: clippy + args: --locked -- -D warnings - - name: Upload the Docker image to ECR - uses: rust-lang/simpleinfra/github-actions/upload-docker-image@master + - name: Clean unused artifacts + uses: actions-rs/cargo@v1 with: - image: docs-rs - repository: staging-docs-rs - region: us-west-1 - aws_access_key_id: "${{ secrets.aws_access_key_id }}" - aws_secret_access_key: "${{ secrets.aws_secret_access_key }}" - if: github.ref == 'refs/heads/master' + command: sweep + args: --file diff --git a/.github/workflows/deps.yml b/.github/workflows/deps.yml new file mode 100644 index 000000000..be91f20bf --- /dev/null +++ b/.github/workflows/deps.yml @@ -0,0 +1,59 @@ +name: Dependencies + +on: + push: + pull_request: + paths: + - "**/Cargo.toml" + - "**/Cargo.lock" + +jobs: + cargo_bloat: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@master + - id: install + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + - uses: actions-rs/cargo@v1 + with: + command: install + args: cargo-sweep + + - name: Cache directories + uses: actions/cache@v1 + with: + path: | + ~/.cargo/registry + ~/.cargo/bin + ~/.cargo/git + key: cargo-dirs-${{ hashFiles('**/Cargo.lock') }} + restore-keys: cargo-dirs- + - name: Cache build + uses: actions/cache@v1 + with: + path: target + key: cargo-build-${{ steps.install.outputs.rustc_hash }}-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + cargo-build-${{ steps.install.outputs.rustc_hash }}- + cargo-build- + + - name: Register artifacts + uses: actions-rs/cargo@v1 + with: + command: sweep + args: --stamp + + - name: Run cargo bloat + uses: orf/cargo-bloat-action@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Clean unused artifacts + uses: actions-rs/cargo@v1 + with: + command: sweep + args: --file diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 000000000..f1ccd3fef --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,23 @@ +name: Docker + +on: [push, pull_request] + +jobs: + docker: + name: Build and upload docker image + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + + - name: Build the Docker image + run: docker build -t docs-rs -f dockerfiles/Dockerfile . + + - name: Upload the Docker image to ECR + uses: rust-lang/simpleinfra/github-actions/upload-docker-image@master + with: + image: docs-rs + repository: staging-docs-rs + region: us-west-1 + aws_access_key_id: "${{ secrets.aws_access_key_id }}" + aws_secret_access_key: "${{ secrets.aws_secret_access_key }}" + if: github.ref == 'refs/heads/master' diff --git a/docker-compose.yml b/docker-compose.yml index a88738109..658dd81b4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,6 +23,12 @@ services: AWS_SECRET_ACCESS_KEY: secret_key env_file: - .env + healthcheck: + test: ["CMD", "curl", "--silent", "--fail", "localhost:3000"] + interval: 10s + timeout: 5s + retries: 10 + db: build: context: ./dockerfiles @@ -35,6 +41,12 @@ services: ports: # Use a non-standard port on the host to avoid conflicting with existing postgres servers - "15432:5432" + healthcheck: + test: ["CMD", "pg_isready"] + interval: 10s + timeout: 5s + retries: 10 + s3: image: minio/minio entrypoint: > @@ -49,12 +61,32 @@ services: environment: MINIO_ACCESS_KEY: cratesfyi MINIO_SECRET_KEY: secret_key + healthcheck: + test: + [ + "CMD", + "curl", + "--silent", + "--fail", + "localhost:9000/minio/health/ready", + ] + interval: 10s + timeout: 5s + retries: 10 + prometheus: build: context: ./dockerfiles dockerfile: ./Dockerfile-prometheus ports: - "9090:9090" + healthcheck: + test: + ["CMD", "curl", "--silent", "--fail", "localhost:9090/-/ready"] + interval: 10s + timeout: 5s + retries: 10 + volumes: postgres-data: {} minio-data: {} From 80e60927d5df87dc5ebbb2a07d1f29051d9f43ed Mon Sep 17 00:00:00 2001 From: Chase Wilson Date: Sun, 5 Jul 2020 08:32:19 -0600 Subject: [PATCH 2/4] Cache v2, disambiguate caches and install sweep after caching --- .github/workflows/ci.yml | 44 +++++++++++++++++++------------------- .github/workflows/deps.yml | 16 +++++++------- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b065302f7..bd600cc94 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,28 +17,28 @@ jobs: with: toolchain: stable override: true - - uses: actions-rs/cargo@v1 - with: - command: install - args: cargo-sweep - name: Cache directories - uses: actions/cache@v1 + uses: actions/cache@v2 with: path: | ~/.cargo/registry ~/.cargo/bin ~/.cargo/git - key: cargo-dirs-${{ hashFiles('**/Cargo.lock') }} - restore-keys: cargo-dirs- + key: cargo-test-dirs-${{ hashFiles('**/Cargo.lock') }} + restore-keys: cargo-test-dirs- - name: Cache build - uses: actions/cache@v1 + uses: actions/cache@v2 with: path: target - key: cargo-build-${{ steps.install.outputs.rustc_hash }}-${{ hashFiles('**/Cargo.lock') }} + key: cargo-test-build-${{ steps.install.outputs.rustc_hash }}-${{ hashFiles('**/Cargo.lock') }} restore-keys: | - cargo-build-${{ steps.install.outputs.rustc_hash }}- - cargo-build- + cargo-test-build-${{ steps.install.outputs.rustc_hash }}- + cargo-test-build- + - uses: actions-rs/cargo@v1 + with: + command: install + args: cargo-sweep - name: Register artifacts uses: actions-rs/cargo@v1 @@ -102,28 +102,28 @@ jobs: toolchain: stable override: true components: clippy - - uses: actions-rs/cargo@v1 - with: - command: install - args: cargo-sweep - name: Cache directories - uses: actions/cache@v1 + uses: actions/cache@v2 with: path: | ~/.cargo/registry ~/.cargo/bin ~/.cargo/git - key: cargo-dirs-${{ hashFiles('**/Cargo.lock') }} - restore-keys: cargo-dirs- + key: cargo-clippy-dirs-${{ hashFiles('**/Cargo.lock') }} + restore-keys: cargo-clippy-dirs- - name: Cache build - uses: actions/cache@v1 + uses: actions/cache@v2 with: path: target - key: cargo-build-${{ steps.install.outputs.rustc_hash }}-${{ hashFiles('**/Cargo.lock') }} + key: cargo-clippy-${{ steps.install.outputs.rustc_hash }}-${{ hashFiles('**/Cargo.lock') }} restore-keys: | - cargo-build-${{ steps.install.outputs.rustc_hash }}- - cargo-build- + cargo-clippy-${{ steps.install.outputs.rustc_hash }}- + cargo-clippy- + - uses: actions-rs/cargo@v1 + with: + command: install + args: cargo-sweep - name: Register artifacts uses: actions-rs/cargo@v1 diff --git a/.github/workflows/deps.yml b/.github/workflows/deps.yml index be91f20bf..6bc4509bd 100644 --- a/.github/workflows/deps.yml +++ b/.github/workflows/deps.yml @@ -18,10 +18,6 @@ jobs: with: toolchain: stable override: true - - uses: actions-rs/cargo@v1 - with: - command: install - args: cargo-sweep - name: Cache directories uses: actions/cache@v1 @@ -30,16 +26,20 @@ jobs: ~/.cargo/registry ~/.cargo/bin ~/.cargo/git - key: cargo-dirs-${{ hashFiles('**/Cargo.lock') }} - restore-keys: cargo-dirs- + key: cargo-deps-dirs-${{ hashFiles('**/Cargo.lock') }} + restore-keys: cargo-deps-dirs- - name: Cache build uses: actions/cache@v1 with: path: target key: cargo-build-${{ steps.install.outputs.rustc_hash }}-${{ hashFiles('**/Cargo.lock') }} restore-keys: | - cargo-build-${{ steps.install.outputs.rustc_hash }}- - cargo-build- + cargo-deps-build-${{ steps.install.outputs.rustc_hash }}- + cargo-deps-build- + - uses: actions-rs/cargo@v1 + with: + command: install + args: cargo-sweep - name: Register artifacts uses: actions-rs/cargo@v1 From 8e3477c6e9f18bc22e589eac73b5e7e1bc04359e Mon Sep 17 00:00:00 2001 From: Chase Wilson Date: Sun, 5 Jul 2020 08:47:20 -0600 Subject: [PATCH 3/4] Made audit run on PRs that touch Cargo.toml & Cargo.lock --- .github/workflows/audit.yml | 1 + .github/workflows/ci.yml | 16 ++++++++-------- .github/workflows/deps.yml | 8 ++++---- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index 9b15640a5..8f8646c71 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -2,6 +2,7 @@ name: Audit on: push: + pull_request: paths: - "**/Cargo.toml" - "**/Cargo.lock" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bd600cc94..d0c2ce804 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,10 @@ jobs: with: toolchain: stable override: true + - uses: actions-rs/cargo@v1 + with: + command: install + args: cargo-sweep - name: Cache directories uses: actions/cache@v2 @@ -35,10 +39,6 @@ jobs: restore-keys: | cargo-test-build-${{ steps.install.outputs.rustc_hash }}- cargo-test-build- - - uses: actions-rs/cargo@v1 - with: - command: install - args: cargo-sweep - name: Register artifacts uses: actions-rs/cargo@v1 @@ -102,6 +102,10 @@ jobs: toolchain: stable override: true components: clippy + - uses: actions-rs/cargo@v1 + with: + command: install + args: cargo-sweep - name: Cache directories uses: actions/cache@v2 @@ -120,10 +124,6 @@ jobs: restore-keys: | cargo-clippy-${{ steps.install.outputs.rustc_hash }}- cargo-clippy- - - uses: actions-rs/cargo@v1 - with: - command: install - args: cargo-sweep - name: Register artifacts uses: actions-rs/cargo@v1 diff --git a/.github/workflows/deps.yml b/.github/workflows/deps.yml index 6bc4509bd..490bebad0 100644 --- a/.github/workflows/deps.yml +++ b/.github/workflows/deps.yml @@ -18,6 +18,10 @@ jobs: with: toolchain: stable override: true + - uses: actions-rs/cargo@v1 + with: + command: install + args: cargo-sweep - name: Cache directories uses: actions/cache@v1 @@ -36,10 +40,6 @@ jobs: restore-keys: | cargo-deps-build-${{ steps.install.outputs.rustc_hash }}- cargo-deps-build- - - uses: actions-rs/cargo@v1 - with: - command: install - args: cargo-sweep - name: Register artifacts uses: actions-rs/cargo@v1 From 227241412a807e6eca17fa1e8c35eb02fce482c3 Mon Sep 17 00:00:00 2001 From: Chase Wilson Date: Sat, 25 Jul 2020 14:03:23 -0500 Subject: [PATCH 4/4] Removed depedendency flow :( --- .github/workflows/deps.yml | 59 -------------------------------------- 1 file changed, 59 deletions(-) delete mode 100644 .github/workflows/deps.yml diff --git a/.github/workflows/deps.yml b/.github/workflows/deps.yml deleted file mode 100644 index 490bebad0..000000000 --- a/.github/workflows/deps.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Dependencies - -on: - push: - pull_request: - paths: - - "**/Cargo.toml" - - "**/Cargo.lock" - -jobs: - cargo_bloat: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@master - - id: install - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - - uses: actions-rs/cargo@v1 - with: - command: install - args: cargo-sweep - - - name: Cache directories - uses: actions/cache@v1 - with: - path: | - ~/.cargo/registry - ~/.cargo/bin - ~/.cargo/git - key: cargo-deps-dirs-${{ hashFiles('**/Cargo.lock') }} - restore-keys: cargo-deps-dirs- - - name: Cache build - uses: actions/cache@v1 - with: - path: target - key: cargo-build-${{ steps.install.outputs.rustc_hash }}-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - cargo-deps-build-${{ steps.install.outputs.rustc_hash }}- - cargo-deps-build- - - - name: Register artifacts - uses: actions-rs/cargo@v1 - with: - command: sweep - args: --stamp - - - name: Run cargo bloat - uses: orf/cargo-bloat-action@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Clean unused artifacts - uses: actions-rs/cargo@v1 - with: - command: sweep - args: --file