diff --git a/java/justfile b/java/justfile new file mode 100644 index 000000000000..a6618b527848 --- /dev/null +++ b/java/justfile @@ -0,0 +1,3 @@ +import '../lib.just' + +build: (_build_dist "java") diff --git a/java/ql/justfile b/java/ql/justfile new file mode 100644 index 000000000000..7a46396fb856 --- /dev/null +++ b/java/ql/justfile @@ -0,0 +1,6 @@ +import "../../lib.just" + +[no-cd] +format *ARGS=".": (_ql_format ARGS) + +consistency_queries := source_dir() / "consistency-queries" diff --git a/java/ql/test/justfile b/java/ql/test/justfile new file mode 100644 index 000000000000..f31d7c3c2139 --- /dev/null +++ b/java/ql/test/justfile @@ -0,0 +1,16 @@ +import "../justfile" + +base_flags := """\ + CODEQL_EXTRACTOR_KOTLIN_DIAGNOSTIC_LIMIT="\\ " \ +""" + +all_checks := default_db_checks + """\ + --check-undefined-labels \ + --check-repeated-labels \ + --check-redefined-labels \ + --check-use-before-definition \ + --consistency-queries=""" + consistency_queries + +[no-cd] +test *ARGS=".": (_codeql_test "java" base_flags all_checks ARGS) + diff --git a/justfile b/justfile new file mode 100644 index 000000000000..a25d51fd9a5d --- /dev/null +++ b/justfile @@ -0,0 +1,2 @@ +import 'misc/just/lib.just' +import 'misc/just/forward.just' diff --git a/lib.just b/lib.just new file mode 100644 index 000000000000..0ddd926bcda5 --- /dev/null +++ b/lib.just @@ -0,0 +1 @@ +import "misc/just/lib.just" diff --git a/misc/bazel/justfile b/misc/bazel/justfile new file mode 100644 index 000000000000..55b2eb9dbd8e --- /dev/null +++ b/misc/bazel/justfile @@ -0,0 +1,5 @@ +import '../just/lib.just' + +[no-cd, positional-arguments, no-exit-message] +hello +ARGS: + @echo "hello from bzl" "$@" diff --git a/misc/codegen/justfile b/misc/codegen/justfile new file mode 100644 index 000000000000..9dd2019b5fac --- /dev/null +++ b/misc/codegen/justfile @@ -0,0 +1,5 @@ +import "../just/lib.just" + +test *ARGS="": (_bazel "test" "@codeql//misc/codegen/...") + +format *ARGS=".": (_black ARGS) \ No newline at end of file diff --git a/misc/just/codeql-test-run.ts b/misc/just/codeql-test-run.ts new file mode 100644 index 000000000000..b1c39aa4bbc6 --- /dev/null +++ b/misc/just/codeql-test-run.ts @@ -0,0 +1,142 @@ +import * as child_process from "child_process"; +import * as path from "path"; +import * as os from "os"; +import * as fs from "fs"; + +function invoke( + invocation: string[], + options: { cwd?: string; log_prefix?: string } = {}, +): number { + const log_prefix = + options.log_prefix && options.log_prefix !== "" + ? `${options.log_prefix} ` + : ""; + console.log( + `${process.env["CMD_BEGIN"] || ""}${log_prefix}${invocation.join(" ")}${process.env["CMD_END"] || ""}`, + ); + try { + child_process.execFileSync(invocation[0], invocation.slice(1), { + stdio: "inherit", + cwd: options.cwd, + }); + } catch (error) { + return 1; + } + return 0; +} + +type Args = { + tests: string[]; + flags: string[]; + env: string[]; + codeql: string; + all: boolean; +}; + +function parseArgs(args: Args, argv: string) { + argv.split(/(? arg.replace("\\ ", " ")) + .forEach((arg) => { + if (arg.startsWith("--codeql=")) { + args.codeql = arg.split("=")[1]; + } else if (arg === "+" || arg === "--all-checks") { + args.all = true; + } else if (arg.startsWith("-")) { + args.flags.push(arg); + } else if (/^[A-Z_][A-Z_0-9]*=.*$/.test(arg)) { + args.env.push(arg); + } else if (arg !== "") { + args.tests.push(arg); + } + }); +} + +function codeqlTestRun(argv: string[]): number { + const semmle_code = process.env["SEMMLE_CODE"]; + const [language, base_args, all_args, extra_args] = argv; + const ram_per_thread = process.platform === "linux" ? 3000 : 2048; + const cpus = os.cpus().length; + let args: Args = { + tests: [], + flags: [`--ram=${ram_per_thread * cpus}`, `-j${cpus}`], + env: [], + codeql: semmle_code ? "build" : "host", + all: false, + }; + parseArgs(args, base_args); + parseArgs(args, extra_args); + if (args.all) { + parseArgs(args, all_args); + } + if (!semmle_code && (args.codeql === "build" || args.codeql === "built")) { + console.error( + "Using `--codeql=build` or `--codeql=built` requires working with the internal repository", + ); + return 1; + } + if (args.tests.length === 0) { + args.tests.push("."); + } + if (args.codeql === "build") { + if ( + invoke([process.env["JUST_EXECUTABLE"] || "just", language, "build"], { + cwd: semmle_code, + }) !== 0 + ) { + return 1; + } + } + if (args.codeql !== "host") { + // disable the default implicit config file, but keep an explicit one + // this is the same behavior wrt to `--codeql` as the integration test runner + process.env["CODEQL_CONFIG_FILE"] ||= "."; + } + // Set and unset environment variables + args.env.forEach((envVar) => { + const [key, value] = envVar.split("=", 2); + if (key) { + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } + } else { + console.error(`Invalid environment variable assignment: ${envVar}`); + process.exit(1); + } + }); + let codeql; + if (args.codeql === "built" || args.codeql === "build") { + codeql = path.join( + semmle_code!, + "target", + "intree", + `codeql-${language}`, + "codeql", + ); + if (!fs.existsSync(codeql)) { + console.error(`CodeQL executable not found: ${codeql}`); + return 1; + } + } else if (args.codeql === "host") { + codeql = "codeql"; + } else { + codeql = args.codeql; + if (fs.lstatSync(codeql).isDirectory()) { + codeql = path.join(codeql, "codeql"); + if (process.platform === "win32") { + codeql += ".exe"; + } + } + if (!fs.existsSync(codeql)) { + console.error(`CodeQL executable not found: ${codeql}`); + return 1; + } + } + + return invoke([codeql, "test", "run", ...args.flags, "--", ...args.tests], { + log_prefix: args.env.join(" "), + }); +} + +process.exit(codeqlTestRun(process.argv.slice(2))); diff --git a/misc/just/forward-command.ts b/misc/just/forward-command.ts new file mode 100644 index 000000000000..ed01bf3c4c56 --- /dev/null +++ b/misc/just/forward-command.ts @@ -0,0 +1,80 @@ +import * as child_process from "child_process"; +import * as path from "path"; +import * as fs from "fs"; + +function commonJustfile(paths: string[]): string { + if (paths.length === 0) return ""; + const splitPaths = paths.map((p) => p.split(path.sep)); + let justfile: string | undefined = undefined; + for (let i = 0; i <= splitPaths[0].length; i++) { + let candidate = path.join(splitPaths[0].slice(0, i).join(path.sep), "justfile"); + if (fs.existsSync(candidate)) { + justfile = candidate; + } + if (!splitPaths.every((parts) => parts[i] === splitPaths[0][i])) { + break; + } + } + if (justfile === undefined) { + throw new Error("No common justfile found"); + } + return justfile; +} + +function forwardCommand(args: string[]): number { + // Avoid infinite recursion + if (args.length == 0) { + console.error("No command provided"); + return 1; + } + const cmd = args[0]; + const envVariable = `__JUST_FORWARD_${cmd}`; + if (process.env[envVariable]) { + console.error(`No common ${cmd} handler found`); + return 1; + } + process.env[envVariable] = "true"; + const cmdArgs = args.slice(1); + // non-positional arguments are flags, + (used by language tests) or environment variable settings + const is_non_positional = /^(-.*|\+|[A-Z_][A-Z_0-9]*=.*)$/; + const flags = cmdArgs.filter((arg) => is_non_positional.test(arg)); + const positionalArgs = cmdArgs.filter( + (arg) => !is_non_positional.test(arg), + ); + + const justfile = commonJustfile(positionalArgs.length !== 0 ? positionalArgs : ["."]); + let cwd: string | undefined; + let invocation = [process.env["JUST_EXECUTABLE"] || "just"]; + if (positionalArgs.length === 1 && justfile == path.join(positionalArgs[0], "justfile")) { + // If there's only one positional argument and it matches the justfile path, suppress arguments + // so for example `just build ql/rust` becomes `just build` in the `ql/rust` directory + cwd = positionalArgs[0]; + invocation.push( + cmd, + ...flags, + ); + console.log(`-> cd ${cwd}; just ${invocation.slice(1).join(" ")}`); + } else { + cwd = undefined; + invocation.push( + "--justfile", + justfile, + cmd, + ...flags, + ...positionalArgs, + ); + console.log(`-> just ${invocation.slice(1).join(" ")}`); + } + + try { + child_process.execFileSync(invocation[0], invocation.slice(1), { + stdio: "inherit", + cwd, + }); + } catch (error) { + return 1; + } + return 0; +} + +process.exit(forwardCommand(process.argv.slice(2))); diff --git a/misc/just/forward.just b/misc/just/forward.just new file mode 100644 index 000000000000..5d3664da221d --- /dev/null +++ b/misc/just/forward.just @@ -0,0 +1,34 @@ +import "lib.just" + +# copy&paste necessary for each command until proper forwarding of multiple args is implemented +# see https://github.com/casey/just/issues/1988 + +_forward := tsx + ' "' + source_dir() + '/forward-command.ts"' + +alias t := test +alias b := build +alias g := generate +alias gen := generate +alias f := format +alias l := lint + +[no-cd, positional-arguments, no-exit-message] +@test *ARGS: + {{ _forward }} test "$@" + + +[no-cd, positional-arguments, no-exit-message] +@build *ARGS: + {{ _forward }} build "$@" + +[no-cd, positional-arguments, no-exit-message] +@generate *ARGS: + {{ _forward }} generate "$@" + +[no-cd, positional-arguments, no-exit-message] +@lint *ARGS: + {{ _forward }} lint "$@" + +[no-cd, positional-arguments, no-exit-message] +@format *ARGS: + {{ _forward }} format "$@" diff --git a/misc/just/justfile b/misc/just/justfile new file mode 100644 index 000000000000..bfa7bed4db2e --- /dev/null +++ b/misc/just/justfile @@ -0,0 +1,2 @@ +format *ARGS=".": + npx prettier --write {{ ARGS }} diff --git a/misc/just/language-tests.ts b/misc/just/language-tests.ts new file mode 100644 index 000000000000..c3626109e03d --- /dev/null +++ b/misc/just/language-tests.ts @@ -0,0 +1,31 @@ +import * as path from "path"; +import * as process from "process" +import * as child_process from "child_process"; + +function languageTests(argv: string[]): number { + const [extra_args, dir, ...relativeRoots] = argv; + const semmle_code = process.env["SEMMLE_CODE"]!; + let roots = relativeRoots.map((root) => path.relative(semmle_code, path.join(dir, root))); + const invocation = [ + process.env["JUST_EXECUTABLE"] || "just", + "--justfile", + path.join(roots[0], "justfile"), + "test", + "--all-checks", + "--codeql=built", + ...extra_args.split(" "), + ...roots, + ]; + console.log(`-> just ${invocation.slice(1).join(" ")}`); + try { + child_process.execFileSync(invocation[0], invocation.slice(1), { + stdio: "inherit", + cwd: semmle_code, + }); + } catch (error) { + return 1; + } + return 0; +} + +process.exit(languageTests(process.argv.slice(2))); diff --git a/misc/just/lib.just b/misc/just/lib.just new file mode 100644 index 000000000000..94ee77efaa50 --- /dev/null +++ b/misc/just/lib.just @@ -0,0 +1,87 @@ +set fallback +set allow-duplicate-recipes +set allow-duplicate-variables +set unstable + +export PATH_SEP := if os() == "windows" { ";" } else { ":" } +export JUST_EXECUTABLE := just_executable() + +error := style("error") + "error" + NORMAL + ": " +cmd_sep := "\n#--------------------------------------------------------\n" +export CMD_BEGIN := style("command") + cmd_sep +export CMD_END := cmd_sep + NORMAL + +tsx := "npx tsx@4.19.0" + +import? '../../../semmle-code.just' # internal repo just file, if present +import 'semmle-code-stub.just' + + +[no-exit-message] +@_require_semmle_code: + {{ if SEMMLE_CODE == "" { ''' + echo "''' + error + ''' running this recipe requires doing so from an internal repository checkout" >&2 + exit 1 + ''' } else { "" } }} + +_build_dist LANGUAGE: _require_semmle_code (_maybe_build_dist LANGUAGE) + +[no-exit-message] +_maybe_build_dist LANGUAGE: + {{ cmd_sep }}{{ if SEMMLE_CODE == "" { '# using codeql from PATH, if any' } else { 'cd "$SEMMLE_CODE"; ./build target/intree/codeql-' + LANGUAGE } }}{{ cmd_sep }} + + +default_db_checks := """\ + --check-databases \ + --check-diff-informed \ + --fail-on-trap-errors \ +""" + +[no-cd, positional-arguments, no-exit-message] +@_codeql_test LANGUAGE BASE_FLAGS ALL_CHECKS_FLAGS EXTRA_ARGS: + {{ tsx }} "{{ source_dir() }}/codeql-test-run.ts" "$@" + +[no-cd, positional-arguments, no-exit-message] +@_language_tests EXTRA_ARGS SOURCE_DIR +ROOTS: _require_semmle_code + {{ tsx }} "{{ source_dir() }}/language-tests.ts" "$@" + +[no-cd, no-exit-message] +_ql_format +ARGS: (_maybe_build_dist "nolang") + {{ cmd_sep }}{{ if SEMMLE_CODE != "" { '"$SEMMLE_CODE/target/intree/codeql-nolang/codeql"' } else { 'codeql' } }} query format --in-place $(find {{ ARGS }} -type f -name '*.ql' -or -name '*.qll'){{ cmd_sep }} + + +[no-cd, no-exit-message] +_bazel COMMAND *ARGS: + {{ cmd_sep }}{{ if SEMMLE_CODE != "" { 'cd "$SEMMLE_CODE"; tools/bazel' } else { 'bazel' } }} {{ COMMAND }} {{ ARGS }}{{ cmd_sep }} + +[no-cd, no-exit-message] +_sembuild *ARGS: (_run_in_semmle_code "./build" ARGS) + +[no-cd, no-exit-message] +_integration_test *ARGS: _require_semmle_code (_run "$SEMMLE_CODE/tools/pytest" ARGS) + +[no-cd] +_run +ARGS: + {{ cmd_sep }}{{ ARGS }}{{ cmd_sep }} + +[no-cd] +_run_in DIR +ARGS: + {{ cmd_sep }}cd "{{ DIR }}"; {{ ARGS }}{{ cmd_sep }} + +[no-cd] +_run_in_semmle_code +ARGS: _require_semmle_code (_run_in "$SEMMLE_CODE" ARGS) + +[no-cd, positional-arguments, no-exit-message] +@_just +ARGS: + echo "-> just $@" + "{{ JUST_EXECUTABLE }}" "$@" + +[no-cd, positional-arguments] +@_if_not_on_ci_just +ARGS: + if [ "${GITHUB_ACTIONS:-}" != "true" ]; then \ + echo "-> just $@"; \ + "$JUST_EXECUTABLE" "$@"; \ + fi + +[no-cd] +_black *ARGS=".": (_run "uv" "run" "black" ARGS) diff --git a/misc/just/semmle-code-stub.just b/misc/just/semmle-code-stub.just new file mode 100644 index 000000000000..14733ffb648e --- /dev/null +++ b/misc/just/semmle-code-stub.just @@ -0,0 +1 @@ +export SEMMLE_CODE := "" diff --git a/rust/codegen/codegen.sh b/rust/codegen/codegen.sh index 2d415009aed8..726ff138db78 100755 --- a/rust/codegen/codegen.sh +++ b/rust/codegen/codegen.sh @@ -2,7 +2,7 @@ set -eu -source misc/bazel/runfiles.sh 2>/dev/null || source external/ql+/misc/bazel/runfiles.sh +source misc/bazel/runfiles.sh 2>/dev/null || source ../ql+/misc/bazel/runfiles.sh ast_generator="$(rlocation "$1")" grammar_file="$(rlocation "$2")" diff --git a/rust/justfile b/rust/justfile new file mode 100644 index 000000000000..a09cabd1d168 --- /dev/null +++ b/rust/justfile @@ -0,0 +1,14 @@ +import '../lib.just' + +install: (_bazel "run" "@codeql//rust:install") + +build: (_if_not_on_ci_just "generate" source_dir()) (_build_dist "rust") + +generate: (_bazel "run" "@codeql//rust/codegen") + +lint: (_run_in source_dir() "python3" "lint.py") + +format: (_run_in source_dir() "python3" "lint.py" "--format-only") + +[group('test')] +language-tests *EXTRA_ARGS: (_language_tests EXTRA_ARGS source_dir() 'ql/test') diff --git a/rust/lint.py b/rust/lint.py index 600a888649e9..e3a078635fa5 100755 --- a/rust/lint.py +++ b/rust/lint.py @@ -4,6 +4,14 @@ import pathlib import shutil import sys +import argparse + + +def options(): + parser = argparse.ArgumentParser(description="lint rust language pack code") + parser.add_argument("--format-only", action="store_true", help="Only apply formatting") + return parser.parse_args() + def tool(name): @@ -12,27 +20,35 @@ def tool(name): return ret -this_dir = pathlib.Path(__file__).resolve().parent +def main(): + args = options() + this_dir = pathlib.Path(__file__).resolve().parent + + + cargo = tool("cargo") + bazel = tool("bazel") + + runs = [] -cargo = tool("cargo") -bazel = tool("bazel") -runs = [] + def run(tool, args, *, cwd=this_dir): + print("+", tool, args) + runs.append(subprocess.run([tool] + args.split(), cwd=cwd)) -def run(tool, args, *, cwd=this_dir): - print("+", tool, args) - runs.append(subprocess.run([tool] + args.split(), cwd=cwd)) + # make sure bazel-provided sources are put in tree for `cargo` to work with them + run(bazel, "run ast-generator:inject-sources") + run(cargo, "fmt --all --quiet") + if not args.format_only: + for manifest in this_dir.rglob("Cargo.toml"): + if not manifest.is_relative_to(this_dir / "ql") and not manifest.is_relative_to(this_dir / "integration-tests"): + run(cargo, + "clippy --fix --allow-dirty --allow-staged --quiet -- -D warnings", + cwd=manifest.parent) -# make sure bazel-provided sources are put in tree for `cargo` to work with them -run(bazel, "run ast-generator:inject-sources") -run(cargo, "fmt --all --quiet") + return max(r.returncode for r in runs) -for manifest in this_dir.rglob("Cargo.toml"): - if not manifest.is_relative_to(this_dir / "ql") and not manifest.is_relative_to(this_dir / "integration-tests"): - run(cargo, - "clippy --fix --allow-dirty --allow-staged --quiet -- -D warnings", - cwd=manifest.parent) -sys.exit(max(r.returncode for r in runs)) +if __name__ == "__main__": + sys.exit(main()) diff --git a/rust/ql/integration-tests/justfile b/rust/ql/integration-tests/justfile new file mode 100644 index 000000000000..7f5aab6c508c --- /dev/null +++ b/rust/ql/integration-tests/justfile @@ -0,0 +1,10 @@ + +import "../../../lib.just" + + +[no-cd] +test *ARGS=".": (_if_not_on_ci_just "generate" source_dir()) (_integration_test ARGS) + +# TODO in separate PR +# [no-cd] +# format *ARGS=".": (_ql_format ARGS) (_black ARGS) diff --git a/rust/ql/justfile b/rust/ql/justfile new file mode 100644 index 000000000000..7a46396fb856 --- /dev/null +++ b/rust/ql/justfile @@ -0,0 +1,6 @@ +import "../../lib.just" + +[no-cd] +format *ARGS=".": (_ql_format ARGS) + +consistency_queries := source_dir() / "consistency-queries" diff --git a/rust/ql/test/justfile b/rust/ql/test/justfile new file mode 100644 index 000000000000..0b1b075fbbb0 --- /dev/null +++ b/rust/ql/test/justfile @@ -0,0 +1,7 @@ +import "../justfile" + +all_checks := default_db_checks + """\ + --consistency-queries=""" + consistency_queries + +[no-cd] +test *ARGS=".": (_codeql_test "rust" "" all_checks ARGS) diff --git a/swift/justfile b/swift/justfile new file mode 100644 index 000000000000..f09fea32e525 --- /dev/null +++ b/swift/justfile @@ -0,0 +1,22 @@ +import '../lib.just' + +install: (_bazel "run" "@codeql//swift:install") + +[group('build')] +build: (_build_dist "swift") + +generate: (_bazel "run" "@codeql//swift/codegen") + +@_check_clang_format: + if ! which clang-format > /dev/null; then \ + "{{ JUST_EXECUTABLE }}" _run_in_semmle_code "tools/bazel" "run" "//c/clang-format:install"; \ + fi + +format ARGS=".": _check_clang_format (_run "clang-format" "-i" ("$(find " + ARGS + " -type f -name '*.h' -or -name '*.cpp')")) +import "../../ql/swift/ql/justfile" + +[group('test')] +language-tests *EXTRA_ARGS: (_language_tests EXTRA_ARGS source_dir() 'ql/test') + +[group('test')] +extra-tests: (_sembuild "target/test/check-queries-swift") (_sembuild "target/test/check-db-upgrades-swift") (_sembuild "target/test/check-db-downgrades-swift") diff --git a/swift/ql/integration-tests/justfile b/swift/ql/integration-tests/justfile new file mode 100644 index 000000000000..370f7ef87794 --- /dev/null +++ b/swift/ql/integration-tests/justfile @@ -0,0 +1,9 @@ +import "../../../lib.just" + + +[no-cd] +test *ARGS=".": (_just "generate") (_integration_test ARGS) + +# TODO in separate PR +# [no-cd] +# format *ARGS=".": (_ql_format ARGS) (_black ARGS) diff --git a/swift/ql/justfile b/swift/ql/justfile new file mode 100644 index 000000000000..7a46396fb856 --- /dev/null +++ b/swift/ql/justfile @@ -0,0 +1,6 @@ +import "../../lib.just" + +[no-cd] +format *ARGS=".": (_ql_format ARGS) + +consistency_queries := source_dir() / "consistency-queries" diff --git a/swift/ql/test/justfile b/swift/ql/test/justfile new file mode 100644 index 000000000000..b27ef6f52ed7 --- /dev/null +++ b/swift/ql/test/justfile @@ -0,0 +1,11 @@ +import "../justfile" + +all_checks := default_db_checks + """\ + --check-repeated-labels \ + --check-redefined-labels \ + --check-use-before-definition \ + --consistency-queries=""" + consistency_queries + + +[no-cd] +test *ARGS=".": (_codeql_test "swift" "" all_checks ARGS)