Just: introduce common "verbs"

justfile

@@ -0,0 +1,2 @@
+import 'misc/just/lib.just'
+import 'misc/just/forward.just'

misc/bazel/justfile

@@ -0,0 +1,5 @@
+import '../just/lib.just'
+
+[no-cd, positional-arguments, no-exit-message]
+hello +ARGS:
+    @echo "hello from bzl" "$@"

misc/codegen/justfile

@@ -0,0 +1,5 @@
+import "../just/lib.just"
+
+test *ARGS="": (_bazel "test" "@codeql//misc/codegen/...")
+
+format *ARGS=".": (_black ARGS)

misc/just/codeql-test-run.ts

@@ -0,0 +1,97 @@
+import * as child_process from "child_process";
+import * as path from "path";
+import * as os from "os";
+
+
+
+function invoke(invocation: string[], options: {cwd?: string, log_prefix?: string} = {}) : number {
+    const log_prefix = options.log_prefix && options.log_prefix !== "" ? `${options.log_prefix} ` : "";
+    console.log(`${process.env["CMD_BEGIN"] || ""}${log_prefix}${invocation.join(" ")}${process.env["CMD_END"] || ""}`);
+    try {
+        child_process.execFileSync(invocation[0], invocation.slice(1), { stdio: "inherit", cwd: options.cwd });
+    } catch (error) {
+        return 1;
+    }
+    return 0;   
+}
+
+type Args = {
+    tests: string[];
+    flags: string[];
+    env: string[];
+    build: boolean;
+    testing_level: number;
+};
+
+function parseArgs(args: Args, argv: string) {
+    argv
+        .split(/(?<!\\) /)
+        .forEach((arg) => {
+            if (arg === "--no-build") {
+                args.build = false;
+            } else if (arg.startsWith("-")) {
+                args.flags.push(arg);
+            } else if (/^[A-Z_][A-Z_0-9]*=.*$/.test(arg)) {
+                args.env.push(arg);
+            } else if (/^\++$/.test(arg)) {
+                args.testing_level = Math.max(args.testing_level, arg.length);
+            } else if (arg !== "") {
+                args.tests.push(arg);
+            }
+        });
+}
+
+
+function codeqlTestRun(argv: string[]): number {
+    const [language, extra_args, ...plus] = argv;
+    let codeql =
+    process.env["SEMMLE_CODE"] ?
+        path.join(process.env["SEMMLE_CODE"], "target", "intree", `codeql-${language}`, "codeql")
+    :
+        "codeql"
+    ;
+    const ram_per_thread = process.platform === "linux" ? 3000 : 2048;
+    const cpus = os.cpus().length;
+    let args: Args = {
+        tests: [],
+        flags: [
+            `--ram=${ram_per_thread}`,
+            `-j${cpus}`,
+        ],
+        env: [],
+        build: true,
+        testing_level: 0
+    };
+    parseArgs(args, extra_args);
+    for (let i = 0; i < Math.min(plus.length, args.testing_level); i++) {
+        parseArgs(args, plus[i]);
+    }
+    if (args.tests.length === 0) {
+        args.tests.push(".");
+    }
+    if (args.build && process.env["SEMMLE_CODE"]) {
+        // If SEMMLE_CODE is set, we are in the semmle-code repo, so we build the codeql binary.
+        // Otherwise, we use codeql from PATH.
+        if (invoke(["python3", "build", `target/intree/codeql-${language}`], {cwd: process.env["SEMMLE_CODE"]}) !== 0) {
+            return 1;
+        }
+    }
+    process.env["CODEQL_CONFIG_FILE"] ||= "."  // disable the default implicit config file, but keep an explicit one
+    // Set and unset environment variables
+    args.env.forEach((envVar) => {
+        const [key, value] = envVar.split("=", 2);
+        if (key) {
+            if (value === undefined) {
+                delete process.env[key];
+            } else {
+                process.env[key] = value;
+            }
+        } else {
+            console.error(`Invalid environment variable assignment: ${envVar}`);
+            process.exit(1);
+        }
+    });
+    return invoke([codeql, "test", "run", ...args.flags, "--", ...args.tests], {log_prefix: args.env.join(" ")});
+}
+
+process.exit(codeqlTestRun(process.argv.slice(2)));

misc/just/forward-command.ts

@@ -0,0 +1,65 @@
+import * as child_process from "child_process";
+import * as path from "path";
+import * as fs from "fs";
+
+function commonDir(paths: string[]): string {
+  if (paths.length === 0) return "";
+  const splitPaths = paths.map(p => p.split(path.sep));
+  let i;
+  for (i = 0; i < splitPaths[0].length; i++) {
+    if (!splitPaths.every(parts => parts[i] === splitPaths[0][i])) {
+      break;
+    }
+    }
+    const commonParts = splitPaths[0].slice(0, i);
+  let ret = commonParts.join(path.sep);
+  if (!fs.existsSync(ret)) {
+    throw new Error(`Common directory does not exist: ${ret}`);
+  }
+  if (!fs.lstatSync(ret).isDirectory()) {
+    ret = path.dirname(ret);
+  }
+  return ret;
+}
+
+function forwardCommand(args: string[]): number {
+  // Avoid infinite recursion
+  if (args.length == 0) {
+    console.error("No command provided");
+    return 1;
+  }
+  const cmd = args[0];
+  const envVariable = `__JUST_FORWARD_${cmd}`;
+  if (process.env[envVariable]) {
+    console.error(`No common ${cmd} handler found`);
+    return 1;
+  }
+  process.env[envVariable] = "true";
+  const cmdArgs = args.slice(1);
+  // non-positional arguments are flags, repeated + (used by language tests) or environment variable settings
+  const is_non_positional = /^(-.*|\++|[A-Z_][A-Z_0-9]*=.*)$/;
+  const flags = cmdArgs.filter(arg => is_non_positional.test(arg));
+  const positionalArgs = cmdArgs.filter(arg => !is_non_positional.test(arg));
+
+  if (positionalArgs.length === 0) {
+    console.error("No positional arguments provided");
+    return 1;
+  }
+
+  const commonPath = commonDir(positionalArgs);
+  let relativeArgs = positionalArgs.map(arg => path.relative(commonPath, arg) || ".");
+  if (relativeArgs.length === 1 && relativeArgs[0] === ".") {
+    relativeArgs = [];
+  }
+
+  const invocation = [process.env["JUST_EXECUTABLE"] || "just", cmd, ...flags, ...relativeArgs];
+  console.log(`-> ${commonPath}: just ${invocation.slice(1).join(" ")}`);
+  try {
+    child_process.execFileSync(invocation[0], invocation.slice(1), { stdio: "inherit", cwd: commonPath });
+  } catch (error) {
+    return 1;
+  }
+  return 0;
+}
+
+process.exit(forwardCommand(process.argv.slice(2)));

misc/just/forward.just

@@ -0,0 +1,28 @@
+import "lib.just"
+
+# copy&paste necessary for each command until proper forwarding of multiple args is implemented
+# see https://github.com/casey/just/issues/1988
+
+_forward := tsx + ' "' + source_dir() + '/forward-command.ts"'
+
+
+[no-cd, positional-arguments, no-exit-message]
+@test *ARGS=".":
+    {{ _forward }} test "$@"
+
+
+[no-cd, positional-arguments, no-exit-message]
+@build *ARGS=".":
+    {{ _forward }} build "$@"
+
+[no-cd, positional-arguments, no-exit-message]
+@generate *ARGS=".":
+    {{ _forward }} generate "$@"
+
+[no-cd, positional-arguments, no-exit-message]
+@lint *ARGS=".":
+    {{ _forward }} lint "$@"
+
+[no-cd, positional-arguments, no-exit-message]
+@format *ARGS=".":
+    {{ _forward }} format "$@"

misc/just/lib.just

@@ -0,0 +1,74 @@
+set fallback
+set allow-duplicate-recipes
+set allow-duplicate-variables
+set unstable
+
+export PATH_SEP := if os() == "windows" { ";" } else { ":" }
+export JUST_EXECUTABLE := just_executable()
+
+error := style("error") + "error" + NORMAL + ": "
+cmd_sep := "\n#--------------------------------------------------------\n"
+export CMD_BEGIN := style("command") + cmd_sep
+export CMD_END := cmd_sep + NORMAL
+
+tsx := "npx [email protected]"
+
+import? '../../../semmle-code.just'  # internal repo just file, if present
+import 'semmle-code-stub.just'
+
+
+[no-exit-message]
+@_require_semmle_code:
+    {{ if SEMMLE_CODE == "" { '''
+        echo "''' + error + ''' running this recipe requires doing so from an internal repository checkout" >&2
+        exit 1
+    ''' } else { "" } }}
+
+_build LANGUAGE: _require_semmle_code (_maybe_build LANGUAGE)
+
+[no-exit-message]
+_maybe_build LANGUAGE:
+    {{ cmd_sep }}{{ if SEMMLE_CODE == "" { '# using codeql from PATH, if any' } else { 'cd "$SEMMLE_CODE"; ./build target/intree/codeql-' + LANGUAGE } }}{{ cmd_sep }}
+
+
+default_db_checks := """\
+    --check-databases \
+    --check-diff-informed \
+    --fail-on-trap-errors \
+"""
+
+[no-cd, positional-arguments, no-exit-message]
+@_codeql_test LANGUAGE +ARGS:
+    {{ tsx }} "{{ source_dir() }}/codeql-test-run.ts" "$@"
+
+
+[no-cd, no-exit-message]
+_ql_format +ARGS: (_maybe_build "nolang")
+    {{ cmd_sep }}{{ if SEMMLE_CODE != "" { '"$SEMMLE_CODE/target/intree/codeql-nolang/codeql"' } else { 'codeql' } }} query format --in-place $(find {{ ARGS }} -type f -name '*.ql' -or -name '*.qll'){{ cmd_sep }}
+
+
+[no-cd, no-exit-message]
+_bazel COMMAND *ARGS:
+    {{ cmd_sep }}{{ if SEMMLE_CODE != "" { '"$SEMMLE_CODE/tools/bazel"' } else { 'bazel' } }} {{ COMMAND }} {{ ARGS }}{{ cmd_sep }}
+
+[no-cd, no-exit-message]
+_sembuild *ARGS: (_run_in_semmle_code "./build" ARGS)
+
+[no-cd, no-exit-message]
+_integration_test *ARGS: _require_semmle_code (_run "$SEMMLE_CODE/tools/pytest" ARGS)
+
+[no-cd]
+_run +ARGS:
+    {{ cmd_sep }}{{ ARGS }}{{ cmd_sep }}
+
+[no-cd]
+_run_in_semmle_code +ARGS: _require_semmle_code
+    {{ cmd_sep }}cd "$SEMMLE_CODE"; {{ ARGS }}{{ cmd_sep }}
+
+
+[no-cd, positional-arguments, no-exit-message]
+_just +ARGS:
+    "{{ JUST_EXECUTABLE }}" "$@"
+
+[no-cd]
+_black *ARGS=".": (_run "uv" "run" "black" ARGS)

misc/just/semmle-code-stub.just

@@ -0,0 +1 @@
+export SEMMLE_CODE := ""

rust/justfile

@@ -0,0 +1,11 @@
+import '../misc/just/lib.just'
+
+install: (_bazel "run" "@codeql//rust:install")
+
+build: generate (_build "rust")
+
+generate: (_bazel "run" "@codeql//rust/codegen")
+
+lint: (_run "python3" "lint.py")
+
+format: (_run "python3" "lint.py" "--format-only")

rust/lint.py

@@ -4,6 +4,14 @@
 import pathlib
 import shutil
 import sys
+import argparse
+
+
+def options():
+    parser = argparse.ArgumentParser(description="lint rust language pack code")
+    parser.add_argument("--format-only", action="store_true", help="Only apply formatting")
+    return parser.parse_args()
+
 
 
 def tool(name):
@@ -12,27 +20,35 @@ def tool(name):
     return ret
 
 
-this_dir = pathlib.Path(__file__).resolve().parent
+def main():
+    args = options()
+    this_dir = pathlib.Path(__file__).resolve().parent
+
+
+    cargo = tool("cargo")
+    bazel = tool("bazel")
+
+    runs = []
 
-cargo = tool("cargo")
-bazel = tool("bazel")
 
-runs = []
+    def run(tool, args, *, cwd=this_dir):
+        print("+", tool, args)
+        runs.append(subprocess.run([tool] + args.split(), cwd=cwd))
 
 
-def run(tool, args, *, cwd=this_dir):
-    print("+", tool, args)
-    runs.append(subprocess.run([tool] + args.split(), cwd=cwd))
+    # make sure bazel-provided sources are put in tree for `cargo` to work with them
+    run(bazel, "run ast-generator:inject-sources")
+    run(cargo, "fmt --all --quiet")
 
+    if not args.format_only:
+        for manifest in this_dir.rglob("Cargo.toml"):
+            if not manifest.is_relative_to(this_dir / "ql") and not manifest.is_relative_to(this_dir / "integration-tests"):
+                run(cargo,
+                    "clippy --fix --allow-dirty --allow-staged --quiet -- -D warnings",
+                    cwd=manifest.parent)
 
-# make sure bazel-provided sources are put in tree for `cargo` to work with them
-run(bazel, "run ast-generator:inject-sources")
-run(cargo, "fmt --all --quiet")
+    return max(r.returncode for r in runs)
 
-for manifest in this_dir.rglob("Cargo.toml"):
-    if not manifest.is_relative_to(this_dir / "ql") and not manifest.is_relative_to(this_dir / "integration-tests"):
-        run(cargo,
-            "clippy --fix --allow-dirty --allow-staged --quiet -- -D warnings",
-            cwd=manifest.parent)
 
-sys.exit(max(r.returncode for r in runs))
+if __name__ == "__main__":
+    sys.exit(main())

rust/ql/integration-tests/justfile

@@ -0,0 +1,9 @@
+import "../../../misc/just/lib.just"
+
+
+[no-cd]
+test *ARGS=".": (_just "generate") (_integration_test ARGS)
+
+# TODO in separate PR
+# [no-cd]
+# format *ARGS=".": (_ql_format ARGS) (_black ARGS)

rust/ql/justfile

@@ -0,0 +1,11 @@
+import "../../misc/just/lib.just"
+
+[no-cd]
+format *ARGS=".": (_ql_format ARGS)
+
+db_checks := default_db_checks
+
+consistency := "--consistency-queries=" + source_dir() / "consistency-queries"
+
+[no-cd]
+test *ARGS=".": (_codeql_test "rust" ARGS db_checks consistency)