diff --git a/.github/labeler.yml b/.github/labeler.yml index 146f7e2d1098..ae138e56b08e 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -42,3 +42,4 @@ documentation: "QL-for-QL": - ql/**/* + - .github/workflows/ql-for-ql* diff --git a/.github/workflows/ql-for-ql-build.yml b/.github/workflows/ql-for-ql-build.yml index da0666c1a1e8..99370014505f 100644 --- a/.github/workflows/ql-for-ql-build.yml +++ b/.github/workflows/ql-for-ql-build.yml @@ -27,31 +27,37 @@ jobs: shell: bash env: CODEQL: ${{ steps.find-codeql.outputs.codeql-path }} + - name: Cache entire pack + id: cache-pack + uses: actions/cache@v3 + with: + path: ${{ runner.temp }}/pack + key: ${{ runner.os }}-pack-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }}-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/**/qlpack.yml') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}--${{ hashFiles('.github/workflows/ql-for-ql-build.yml') }} - name: Cache queries + if: steps.cache-pack.outputs.cache-hit != 'true' id: cache-queries uses: actions/cache@v3 with: - path: ${{ runner.temp }}/query-pack.zip + path: ${{ runner.temp }}/queries key: queries-${{ hashFiles('ql/**/*.ql*') }}-${{ hashFiles('ql/**/qlpack.yml') }}-${{ hashFiles('ql/ql/src/ql.dbscheme*') }}-${{ steps.get-codeql-version.outputs.version }}--${{ hashFiles('.github/workflows/ql-for-ql-build.yml') }} - name: Build query pack - if: steps.cache-queries.outputs.cache-hit != 'true' + if: steps.cache-queries.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true' run: | cd ql/ql/src "${CODEQL}" pack create -j 16 - cd .codeql/pack/codeql/ql/0.0.0 - zip "${PACKZIP}" -r . - rm -rf * + mv .codeql/pack/codeql/ql/0.0.0 ${{ runner.temp }}/queries env: CODEQL: ${{ steps.find-codeql.outputs.codeql-path }} - PACKZIP: ${{ runner.temp }}/query-pack.zip - - name: Upload query pack - uses: actions/upload-artifact@v3 - with: - name: query-pack-zip - path: ${{ runner.temp }}/query-pack.zip - + - name: Move cache queries to pack + if: steps.cache-pack.outputs.cache-hit != 'true' + run: | + cp -r ${{ runner.temp }}/queries ${{ runner.temp }}/pack + env: + CODEQL: ${{ steps.find-codeql.outputs.codeql-path }} + ### Build the extractor ### - name: Cache entire extractor + if: steps.cache-pack.outputs.cache-hit != 'true' id: cache-extractor uses: actions/cache@v3 with: @@ -62,7 +68,7 @@ jobs: ql/target/release/ql-extractor.exe key: ${{ runner.os }}-extractor-${{ hashFiles('ql/**/Cargo.lock') }}-${{ hashFiles('ql/**/*.rs') }} - name: Cache cargo - if: steps.cache-extractor.outputs.cache-hit != 'true' + if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true' uses: actions/cache@v3 with: path: | @@ -71,73 +77,35 @@ jobs: ql/target key: ${{ runner.os }}-rust-cargo-${{ hashFiles('ql/**/Cargo.lock') }} - name: Check formatting - if: steps.cache-extractor.outputs.cache-hit != 'true' + if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true' run: cd ql; cargo fmt --all -- --check - name: Build - if: steps.cache-extractor.outputs.cache-hit != 'true' + if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true' run: cd ql; cargo build --verbose - name: Run tests - if: steps.cache-extractor.outputs.cache-hit != 'true' + if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true' run: cd ql; cargo test --verbose - name: Release build - if: steps.cache-extractor.outputs.cache-hit != 'true' + if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true' run: cd ql; cargo build --release - name: Generate dbscheme - if: steps.cache-extractor.outputs.cache-hit != 'true' + if: steps.cache-extractor.outputs.cache-hit != 'true' && steps.cache-pack.outputs.cache-hit != 'true' run: ql/target/release/ql-generator --dbscheme ql/ql/src/ql.dbscheme --library ql/ql/src/codeql_ql/ast/internal/TreeSitter.qll - - uses: actions/upload-artifact@v3 - with: - name: extractor-ubuntu-latest - path: | - ql/target/release/ql-autobuilder - ql/target/release/ql-autobuilder.exe - ql/target/release/ql-extractor - ql/target/release/ql-extractor.exe - retention-days: 1 ### Package the queries and extractor ### - - uses: actions/download-artifact@v3 - with: - name: query-pack-zip - path: query-pack-zip - - uses: actions/download-artifact@v3 - with: - name: extractor-ubuntu-latest - path: linux64 - - run: | - unzip query-pack-zip/*.zip -d pack - cp -r ql/codeql-extractor.yml ql/tools ql/ql/src/ql.dbscheme.stats pack/ - mkdir -p pack/tools/linux64 - if [[ -f linux64/ql-autobuilder ]]; then - cp linux64/ql-autobuilder pack/tools/linux64/autobuilder - chmod +x pack/tools/linux64/autobuilder - fi - if [[ -f linux64/ql-extractor ]]; then - cp linux64/ql-extractor pack/tools/linux64/extractor - chmod +x pack/tools/linux64/extractor - fi - cd pack - zip -rq ../codeql-ql.zip . - rm -rf * - - uses: actions/upload-artifact@v3 - with: - name: codeql-ql-pack - path: codeql-ql.zip - retention-days: 1 - - ### Run the analysis ### - - name: Download pack - uses: actions/download-artifact@v3 - with: - name: codeql-ql-pack - path: ${{ runner.temp }}/codeql-ql-pack-artifact - - - name: Prepare pack + - name: Package pack + if: steps.cache-pack.outputs.cache-hit != 'true' run: | - unzip "${PACK_ARTIFACT}/*.zip" -d "${PACK}" + cp -r ql/codeql-extractor.yml ql/tools ql/ql/src/ql.dbscheme.stats ${PACK}/ + mkdir -p ${PACK}/tools/linux64 + cp ql/target/release/ql-autobuilder ${PACK}/tools/linux64/autobuilder + cp ql/target/release/ql-extractor ${PACK}/tools/linux64/extractor + chmod +x ${PACK}/tools/linux64/autobuilder + chmod +x ${PACK}/tools/linux64/extractor env: - PACK_ARTIFACT: ${{ runner.temp }}/codeql-ql-pack-artifact PACK: ${{ runner.temp }}/pack + + ### Run the analysis ### - name: Hack codeql-action options run: | JSON=$(jq -nc --arg pack "${PACK}" '.database."run-queries"=["--search-path", $pack] | .resolve.queries=["--search-path", $pack] | .resolve.extractor=["--search-path", $pack] | .database.init=["--search-path", $pack]')