diff --git a/.github/actions/reasonable-memory/action.yml b/.github/actions/reasonable-memory/action.yml new file mode 100644 index 000000000000..5ec7877e194c --- /dev/null +++ b/.github/actions/reasonable-memory/action.yml @@ -0,0 +1,84 @@ +name: Make more memory available +description: Stops services and adds swap + +inputs: + max: + description: Maximum memory to suggest using + default: 50000 + +outputs: + available: + description: Available memory + value: ${{ steps.memory.outputs.available }} + suggested: + description: Memory suggested for use (constrained by max and available) + value: ${{ steps.memory.outputs.suggested }} + +runs: + using: composite + steps: + - name: Stop services + if: env.RUNNER_OS == 'Linux' + shell: bash + run: | + sudo systemctl disable php8.1-fpm mono-xsp4 walinuxagent multipathd walinuxagent chrony cron getty@tty1 networkd-dispatcher rsyslog serial-getty@ttyS0 snapd multipathd.socket snapd.socket + sudo systemctl stop php8.1-fpm mono-xsp4 walinuxagent multipathd walinuxagent chrony cron getty@tty1 networkd-dispatcher rsyslog serial-getty@ttyS0 snapd + sudo killall mono + + - name: enable swap + if: env.RUNNER_OS == 'Linux' + shell: bash + run: | + sudo fallocate -l 10G /mnt/big-swapfile + sudo chmod 600 /mnt/big-swapfile + sudo mkswap /mnt/big-swapfile + sudo swapon /mnt/big-swapfile + + - name: Report Available Memory (Linux) + id: memory-linux + if: runner.os == 'Linux' + shell: bash + run: | + free | + perl -ne ' + next unless /Mem:.*\s(\d+)$/; + my $m = int($1 / 102400)*100; + print "available=$m\n"; + ' >> "$GITHUB_OUTPUT" + + - name: Report Available Memory (macOS) + id: memory-mac + if: runner.os == 'macOS' + shell: bash + run: | + sysctl -a | + perl -e ' + my $m; + while (<>) { + next unless /.*memsize.*:\s*(\d+)/; + my $m0 = int($1 >> 20); + $m = $m0 unless ($m && $m > $m0); + } + print "available=$m\n"; + ' >> "$GITHUB_OUTPUT" + + - name: Report Available Memory (Windows) + id: memory-windows + if: runner.os == 'Windows' + shell: bash + run: | + echo "available=10240" >> "$GITHUB_OUTPUT" + + - name: Report Memory + id: memory + shell: bash + env: + max: ${{ inputs.max }} + available: ${{ steps.memory-linux.outputs.available || steps.memory-mac.outputs.available || steps.memory-windows.outputs.available }} + run: | + perl -e ' + my $available = $ENV{available}; + my $max = $ENV{max} || $available; + my $s = ($m < $max ? $available : $max); + print "suggested=$s\n"."available=$available\n"; + ' >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/csharp-qltest.yml b/.github/workflows/csharp-qltest.yml index f9e1db3f4c7f..bea668485895 100644 --- a/.github/workflows/csharp-qltest.yml +++ b/.github/workflows/csharp-qltest.yml @@ -5,7 +5,10 @@ on: paths: - "csharp/**" - "shared/**" + - .github/workflows/csharp-qltest.yml - .github/actions/fetch-codeql/action.yml + - .github/actions/cache-query-compilation/action.yml + - .github/actions/reasonable-memory/action.yml - codeql-workspace.yml branches: - main @@ -16,6 +19,8 @@ on: - "shared/**" - .github/workflows/csharp-qltest.yml - .github/actions/fetch-codeql/action.yml + - .github/actions/cache-query-compilation/action.yml + - .github/actions/reasonable-memory/action.yml - codeql-workspace.yml branches: - main @@ -53,6 +58,9 @@ jobs: slice: ["1/2", "2/2"] steps: - uses: actions/checkout@v4 + - name: Allocate memory + id: memory + uses: ./.github/actions/reasonable-memory - uses: ./csharp/actions/create-extractor-pack - name: Cache compilation cache id: query-cache @@ -61,9 +69,10 @@ jobs: key: csharp-qltest-${{ matrix.slice }} - name: Run QL tests run: | - codeql test run --threads=0 --ram 50000 --slice ${{ matrix.slice }} --search-path extractor-pack --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" + codeql test run --threads=0 --ram $memory --slice ${{ matrix.slice }} --search-path extractor-pack --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" env: GITHUB_TOKEN: ${{ github.token }} + memory: ${{ steps.memory.outputs.suggested }} unit-tests: strategy: matrix: diff --git a/.github/workflows/ruby-qltest.yml b/.github/workflows/ruby-qltest.yml index 19d5325091fd..09e6c37c5a4b 100644 --- a/.github/workflows/ruby-qltest.yml +++ b/.github/workflows/ruby-qltest.yml @@ -5,8 +5,11 @@ on: paths: - "ruby/**" - "shared/**" - - .github/workflows/ruby-build.yml + - .github/workflows/ruby-qltest.yml - .github/actions/fetch-codeql/action.yml + - .github/actions/cache-query-compilation/action.yml + - .github/actions/reasonable-memory/action.yml + - csharp/actions/create-extractor-pack/action.yml - codeql-workspace.yml branches: - main @@ -17,10 +20,14 @@ on: - "shared/**" - .github/workflows/ruby-qltest.yml - .github/actions/fetch-codeql/action.yml + - .github/actions/cache-query-compilation/action.yml + - .github/actions/reasonable-memory/action.yml + - csharp/actions/create-extractor-pack/action.yml - codeql-workspace.yml branches: - main - "rc/*" + workflow_dispatch: env: CARGO_TERM_COLOR: always @@ -55,6 +62,9 @@ jobs: fail-fast: false steps: - uses: actions/checkout@v4 + - name: Allocate memory + id: memory + uses: ./.github/actions/reasonable-memory - uses: ./.github/actions/fetch-codeql - uses: ./ruby/actions/create-extractor-pack - name: Cache compilation cache @@ -64,6 +74,7 @@ jobs: key: ruby-qltest - name: Run QL tests run: | - codeql test run --threads=0 --ram 50000 --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" + codeql test run --threads=0 --ram $memory --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" env: GITHUB_TOKEN: ${{ github.token }} + memory: ${{ steps.memory.outputs.suggested }} diff --git a/.github/workflows/swift.yml b/.github/workflows/swift.yml index ff9cd29e238d..5f6bc00da2f3 100644 --- a/.github/workflows/swift.yml +++ b/.github/workflows/swift.yml @@ -32,6 +32,7 @@ on: - main - rc/* - codeql-cli-* + workflow_dispatch: jobs: # not using a matrix as you cannot depend on a specific job in a matrix, and we want to start linux checks diff --git a/swift/actions/run-ql-tests/action.yml b/swift/actions/run-ql-tests/action.yml index da10997df448..56df710f95c5 100644 --- a/swift/actions/run-ql-tests/action.yml +++ b/swift/actions/run-ql-tests/action.yml @@ -8,6 +8,9 @@ inputs: runs: using: composite steps: + - name: Allocate memory + id: memory + uses: ./.github/actions/reasonable-memory - uses: ./swift/actions/share-extractor-pack - uses: ./.github/actions/fetch-codeql - id: query-cache @@ -19,7 +22,7 @@ runs: run: | codeql test run \ --threads=0 \ - --ram 50000 \ + --ram $memory \ --search-path "${{ github.workspace }}/swift/extractor-pack" \ --check-databases \ --check-unused-labels \ @@ -32,3 +35,4 @@ runs: swift/ql/test env: GITHUB_TOKEN: ${{ github.token }} + memory: ${{ steps.memory.outputs.suggested }}