From da96b2a6c4f478cface7e20a5ff01f6b4aeb545e Mon Sep 17 00:00:00 2001 From: Chuan-kai Lin Date: Mon, 15 Sep 2025 12:57:04 -0700 Subject: [PATCH 1/6] Add src/resolved-languages.json --- src/resolved-languages.json | 291 ++++++++++++++++++++++++++++++++++++ 1 file changed, 291 insertions(+) create mode 100644 src/resolved-languages.json diff --git a/src/resolved-languages.json b/src/resolved-languages.json new file mode 100644 index 0000000000..71a3151ee2 --- /dev/null +++ b/src/resolved-languages.json @@ -0,0 +1,291 @@ +{ + "aliases" : { + "c" : "cpp", + "c++" : "cpp", + "c-c++" : "cpp", + "c-cpp" : "cpp", + "c#" : "csharp", + "java-kotlin" : "java", + "kotlin" : "java", + "javascript-typescript" : "javascript", + "typescript" : "javascript" + }, + "extractors" : { + "xml" : [ + { + "extractor_root" : "/opt/hostedtoolcache/CodeQL/2.23.0/x64/codeql/xml" + } + ], + "python" : [ + { + "extractor_root" : "/opt/hostedtoolcache/CodeQL/2.23.0/x64/codeql/python", + "extractor_options" : { + "logging" : { + "title" : "Options pertaining to logging.", + "description" : "Options pertaining to logging.", + "type" : "object", + "properties" : { + "verbosity" : { + "title" : "Python extractor logging verbosity level.", + "description" : "Controls the level of verbosity of the CodeQL Python extractor.\nThe supported levels are (in order of increasing verbosity):\n\n - off\n - errors\n - warnings\n - info or progress\n - debug or progress+\n - trace or progress++\n - progress+++\n", + "type" : "string", + "pattern" : "^(off|errors|warnings|(info|progress)|(debug|progress\\+)|(trace|progress\\+\\+)|progress\\+\\+\\+)$" + } + } + }, + "python_executable_name" : { + "title" : "Controls the name of the Python executable used by the Python extractor.", + "description" : "The Python extractor uses platform-dependent heuristics to determine the name of the Python executable to use. Specifying a value for this option overrides the name of the Python executable used by the extractor. Accepted values are py, python and python3. Use this setting with caution, the Python extractor requires Python 3 to run.\n", + "type" : "string", + "pattern" : "^(py|python|python3)$" + } + } + } + ], + "properties" : [ + { + "extractor_root" : "/opt/hostedtoolcache/CodeQL/2.23.0/x64/codeql/properties" + } + ], + "javascript" : [ + { + "extractor_root" : "/opt/hostedtoolcache/CodeQL/2.23.0/x64/codeql/javascript", + "extractor_options" : { + "skip_types" : { + "title" : "Skip type extraction for TypeScript", + "description" : "Whether to skip the extraction of types in a TypeScript application", + "type" : "string", + "pattern" : "^(false|true)$" + } + } + } + ], + "csharp" : [ + { + "extractor_root" : "/opt/hostedtoolcache/CodeQL/2.23.0/x64/codeql/csharp", + "extractor_options" : { + "trap" : { + "title" : "Options pertaining to TRAP.", + "description" : "Options pertaining to TRAP.", + "type" : "object", + "properties" : { + "compression" : { + "title" : "Controls compression for the TRAP files written by the extractor.", + "description" : "This option is only intended for use in debugging the extractor. Accepted values are 'brotli' (the default, to write brotli-compressed TRAP), 'gzip', and 'none' (to write uncompressed TRAP).\n", + "type" : "string", + "pattern" : "^(none|gzip|brotli)$" + } + } + }, + "buildless" : { + "title" : "DEPRECATED - Whether to use buildless (standalone) extraction.", + "description" : "DEPRECATED: Use `--build-mode none` instead.\nA value indicating, which type of extraction the autobuilder should perform. If 'true', then the standalone extractor will be used, otherwise tracing extraction will be performed. The default is 'false'. Note that buildless extraction will generally yield less accurate analysis results, and should only be used in cases where it is not possible to build the code (for example if it uses inaccessible dependencies).\n", + "type" : "string", + "pattern" : "^(false|true)$" + }, + "logging" : { + "title" : "Options pertaining to logging.", + "description" : "Options pertaining to logging.", + "type" : "object", + "properties" : { + "verbosity" : { + "title" : "Extractor logging verbosity level.", + "description" : "Controls the level of verbosity of the extractor. The supported levels are (in order of increasing verbosity):\n - off\n - errors\n - warnings\n - info or progress\n - debug or progress+\n - trace or progress++\n - progress+++\n", + "type" : "string", + "pattern" : "^(off|errors|warnings|(info|progress)|(debug|progress\\+)|(trace|progress\\+\\+)|progress\\+\\+\\+)$" + } + } + }, + "binlog" : { + "title" : "Binlog", + "description" : "[EXPERIMENTAL] The value is a path to the MsBuild binary log file that should be extracted. This option only works when `--build-mode none` is also specified.\n", + "type" : "array" + } + } + } + ], + "rust" : [ + { + "extractor_root" : "/opt/hostedtoolcache/CodeQL/2.23.0/x64/codeql/rust", + "extractor_options" : { + "trap" : { + "title" : "Options pertaining to TRAP.", + "description" : "Options pertaining to TRAP.", + "type" : "object", + "properties" : { + "compression" : { + "title" : "Controls compression for the TRAP files written by the extractor.", + "description" : "This option is only intended for use in debugging the extractor. Accepted values are 'gzip' (to write gzip-compressed TRAP) 'zstd' (to write Zstandard-compressed TRAP) and 'none' (the default, to write uncompressed TRAP).\n", + "type" : "string", + "pattern" : "^(none|gzip|zstd)$" + } + } + }, + "cargo_target_dir" : { + "title" : "Directory to use for cargo output files.", + "description" : "This value is an optional path to use as `CARGO_TARGET_DIR` for the internal cargo commands the extractor uses. Pointing it to a persistent directory may reduce execution time of consecutive extractor runs. By default, a new scratch directory is used for each run.\n", + "type" : "string" + }, + "cargo_target" : { + "title" : "Target architecture", + "description" : "Target architecture to use for analysis, analogous to `cargo --target`. By default the host architecture is used.\n", + "type" : "string" + }, + "cargo_features" : { + "title" : "Cargo features to turn on", + "description" : "Comma-separated list of features to turn on. By default all features are enabled. If any features are specified, then only those features are enabled. The `default` feature must be explicitly specified if only default features are desired. Can be repeated.\n", + "type" : "array" + }, + "cargo_cfg_overrides" : { + "title" : "Cargo cfg overrides", + "description" : "Comma-separated list of cfg settings to enable, or disable if prefixed with `-`. Can be repeated.\n", + "type" : "array" + }, + "logging" : { + "title" : "Options pertaining to logging.", + "description" : "Options pertaining to logging.", + "type" : "object", + "properties" : { + "verbosity" : { + "title" : "Extractor logging verbosity level.", + "description" : "Controls the level of verbosity of the extractor. The supported levels are (in order of increasing verbosity):\n - off\n - errors\n - warnings\n - info or progress\n - debug or progress+\n - trace or progress++\n - progress+++\n", + "type" : "string", + "pattern" : "^(off|errors|warnings|(info|progress)|(debug|progress\\+)|(trace|progress\\+\\+)|progress\\+\\+\\+)$" + }, + "flamegraph" : { + "title" : "[Experimental] File path for write flame graph log", + "description" : "Collect flame graph data using the `tracing-flame` crate. To render a flame graph or chart, run the `inferno-flamegraph` command. See also: https://crates.io/crates/tracing-flame\n", + "type" : "string" + } + } + }, + "extract_dependencies_as_source" : { + "title" : "Extract dependencies as source code", + "description" : "Extract the full source code of dependencies instead of only extracting signatures.\n", + "type" : "string", + "pattern" : "^(false|true)$" + } + } + } + ], + "ruby" : [ + { + "extractor_root" : "/opt/hostedtoolcache/CodeQL/2.23.0/x64/codeql/ruby", + "extractor_options" : { + "trap" : { + "title" : "Options pertaining to TRAP.", + "description" : "Options pertaining to TRAP.", + "type" : "object", + "properties" : { + "compression" : { + "title" : "Controls compression for the TRAP files written by the extractor.", + "description" : "This option is only intended for use in debugging the extractor. Accepted values are 'gzip' (the default, to write gzip-compressed TRAP) 'zstd' (to write Zstandard-compressed TRAP) and 'none' (to write uncompressed TRAP).\n", + "type" : "string", + "pattern" : "^(none|gzip|zstd)$" + } + } + } + } + } + ], + "go" : [ + { + "extractor_root" : "/opt/hostedtoolcache/CodeQL/2.23.0/x64/codeql/go", + "extractor_options" : { + "extract_tests" : { + "title" : "Whether to include Go test files in the CodeQL database.", + "description" : "A value indicating whether Go test files should be included in the CodeQL database. The default is 'false'.\n", + "type" : "string", + "pattern" : "^(false|true)$" + }, + "extract_vendor_dirs" : { + "title" : "Whether to include Go vendor directories in the CodeQL database.", + "description" : "A value indicating whether Go vendor directories should be included in the CodeQL database. The default is 'false'.\n", + "type" : "string", + "pattern" : "^(false|true)$" + } + } + } + ], + "cpp" : [ + { + "extractor_root" : "/opt/hostedtoolcache/CodeQL/2.23.0/x64/codeql/cpp", + "extractor_options" : { + "scale_timeouts" : { + "title" : "Value to scale compiler introspection timeouts with", + "description" : "The extractor attempts to determine what compiler the source code being extracted is compiled with. To this end the extractor makes additional calls to the compiler, some of which are expected to return within a certain fixed time (either 10s or 15s). On some systems that are under high load this time might be too short, and can be scaled up using this option.\n", + "type" : "string", + "pattern" : "[0-9]+" + }, + "log_verbosity" : { + "title" : "Verbosity of the extractor logging", + "description" : "Set the verbosity of the extractor logging to 'quiet' (0), 'normal' (1), 'chatty' (2), or 'noisy' (3). The default is 'normal'.\n", + "type" : "string", + "pattern" : "[0-3]" + } + } + } + ], + "yaml" : [ + { + "extractor_root" : "/opt/hostedtoolcache/CodeQL/2.23.0/x64/codeql/yaml" + } + ], + "html" : [ + { + "extractor_root" : "/opt/hostedtoolcache/CodeQL/2.23.0/x64/codeql/html" + } + ], + "swift" : [ + { + "extractor_root" : "/opt/hostedtoolcache/CodeQL/2.23.0/x64/codeql/swift" + } + ], + "csv" : [ + { + "extractor_root" : "/opt/hostedtoolcache/CodeQL/2.23.0/x64/codeql/csv" + } + ], + "actions" : [ + { + "extractor_root" : "/opt/hostedtoolcache/CodeQL/2.23.0/x64/codeql/actions", + "extractor_options" : { } + } + ], + "java" : [ + { + "extractor_root" : "/opt/hostedtoolcache/CodeQL/2.23.0/x64/codeql/java", + "extractor_options" : { + "exclude" : { + "title" : "A glob excluding files from analysis.", + "description" : "A glob indicating what files to exclude from the analysis. This accepts glob patterns that are supported by Java's 'getPathMatcher' implementation.\n", + "type" : "string" + }, + "add_prefer_source" : { + "title" : "Whether to always prefer source files over class files.", + "description" : "A value indicating whether source files should be preferred over class files. If set to 'true', the extraction adds '-Xprefer:source' to the javac command line. If set to 'false', the extraction uses the default javac behavior ('-Xprefer:newer'). The default is 'true'.\n", + "type" : "string", + "pattern" : "^(false|true)$" + }, + "buildless" : { + "title" : "Whether to use buildless (standalone) extraction (experimental).", + "description" : "A value indicating, which type of extraction the autobuilder should perform. If 'true', then the standalone extractor will be used, otherwise tracing extraction will be performed. The default is 'false'. Note that buildless extraction will generally yield less accurate analysis results, and should only be used in cases where it is not possible to build the code (for example if it uses inaccessible dependencies).\n", + "type" : "string", + "pattern" : "^(false|true)$" + }, + "buildless_dependency_dir" : { + "title" : "The path where buildless (standalone) extraction should keep dependencies.", + "description" : "If set, the buildless (standalone) extractor will store dependencies in this directory.\n", + "type" : "string" + }, + "minimize_dependency_jars" : { + "title" : "Whether to rewrite and minimize downloaded JAR dependencies (experimental).", + "description" : "If 'true', JAR dependencies downloaded during extraction will be rewritten to remove unneeded data, such as method bodies. The default is 'false'.\n", + "type" : "string", + "pattern" : "^(false|true)$" + } + } + } + ] + } +} From dc74db4ce3e5f774d6145c4fa5a14867bb9f82b0 Mon Sep 17 00:00:00 2001 From: Chuan-kai Lin Date: Mon, 15 Sep 2025 11:27:49 -0700 Subject: [PATCH 2/6] Add validate-resolved-languages.yml This workflow checks that src/resolved-languages.json exists and is up-to-date with respect to the default (linked) CodeQL version. --- .../workflows/validate-resolved-languages.yml | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 .github/workflows/validate-resolved-languages.yml diff --git a/.github/workflows/validate-resolved-languages.yml b/.github/workflows/validate-resolved-languages.yml new file mode 100644 index 0000000000..f5db7bddf1 --- /dev/null +++ b/.github/workflows/validate-resolved-languages.yml @@ -0,0 +1,76 @@ +name: Validate Resolved Languages File +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GO111MODULE: auto +on: + push: + branches: + - main + - releases/v* + pull_request: + types: + - opened + - synchronize + - reopened + - ready_for_review + schedule: + - cron: '0 5 * * *' + workflow_dispatch: + inputs: {} + workflow_call: + inputs: {} +defaults: + run: + shell: bash +jobs: + validate-resolved-languages: + name: Validate Resolved Languages File + permissions: + contents: read + security-events: read + timeout-minutes: 10 + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v5 + - name: Install Node.js + uses: actions/setup-node@v4 + with: + node-version: 20.x + cache: npm + - name: Install dependencies + run: npm ci + - name: Prepare test + id: prepare-test + uses: ./.github/actions/prepare-test + with: + version: linked + use-all-platform-bundle: 'false' + setup-kotlin: 'false' + - name: Initialize CodeQL + id: init + uses: ./../action/init + with: + tools: ${{ steps.prepare-test.outputs.tools-url }} + - name: Verify that the resolved languages file is up to date + env: + CODEQL: ${{ steps.init.outputs.codeql-path }} + RESOLVED_LANGUAGES_FILE: ../action/src/resolved-languages.json + run: | + if [ ! -f "${RESOLVED_LANGUAGES_FILE}" ]; then + echo "::error file=src/resolved-languages.json::The resolved-languages.json file is missing." + exit 1 + fi + "${CODEQL}" resolve languages --format=betterjson --extractor-include-aliases > resolved-languages.json + if ! diff -u resolved-languages.json "${RESOLVED_LANGUAGES_FILE}"; then + echo "::error file=src/resolved-languages.json::The saved resolved-languages.json file is out of date." + exit 1 + fi + - name: Upload resolved languages file for debugging + if: ${{ failure() }} + uses: actions/upload-artifact@v4 + with: + name: resolved-languages + path: resolved-languages.json + env: + CODEQL_ACTION_TEST_MODE: true From 796b998c07ba1ad13167024e862e30c0951db296 Mon Sep 17 00:00:00 2001 From: Chuan-kai Lin Date: Mon, 15 Sep 2025 13:47:08 -0700 Subject: [PATCH 3/6] Fix BetterResolveLanguagesOutput This commit fixes the BetterResolveLanguagesOutput interface, which currently defines the `extractors` map values as tuples: [ { extractor_root: string; ... }, ] Instead of arrays (which are what the CodeQL CLI produces): Array<{ extractor_root: string; ... }> --- src/codeql.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/codeql.ts b/src/codeql.ts index 16b105a3df..77321a18a7 100644 --- a/src/codeql.ts +++ b/src/codeql.ts @@ -237,12 +237,10 @@ export interface BetterResolveLanguagesOutput { [alias: string]: string; }; extractors: { - [language: string]: [ - { - extractor_root: string; - extractor_options?: any; - }, - ]; + [language: string]: Array<{ + extractor_root: string; + extractor_options?: any; + }>; }; } From 76be57fa32e8148c75bb6246cd102257ac5529b3 Mon Sep 17 00:00:00 2001 From: Chuan-kai Lin Date: Mon, 15 Sep 2025 13:58:20 -0700 Subject: [PATCH 4/6] Add getStoredSupportedLanguageMap() --- .../workflows/validate-resolved-languages.yml | 4 +++ src/config-utils.ts | 27 ++++++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validate-resolved-languages.yml b/.github/workflows/validate-resolved-languages.yml index f5db7bddf1..a275e44446 100644 --- a/.github/workflows/validate-resolved-languages.yml +++ b/.github/workflows/validate-resolved-languages.yml @@ -61,6 +61,10 @@ jobs: echo "::error file=src/resolved-languages.json::The resolved-languages.json file is missing." exit 1 fi + # + # Please update getStoredSupportedLanguageMap() when you update the + # following call to use --filter-to-languages-with-queries. + # "${CODEQL}" resolve languages --format=betterjson --extractor-include-aliases > resolved-languages.json if ! diff -u resolved-languages.json "${RESOLVED_LANGUAGES_FILE}"; then echo "::error file=src/resolved-languages.json::The saved resolved-languages.json file is out of date." diff --git a/src/config-utils.ts b/src/config-utils.ts index 538c366e86..5b41d0cfbf 100644 --- a/src/config-utils.ts +++ b/src/config-utils.ts @@ -16,7 +16,7 @@ import { } from "./analyses"; import * as api from "./api-client"; import { CachingKind, getCachingKind } from "./caching-utils"; -import { type CodeQL } from "./codeql"; +import { type BetterResolveLanguagesOutput, type CodeQL } from "./codeql"; import { shouldPerformDiffInformedAnalysis } from "./diff-informed-analysis-utils"; import { Feature, FeatureEnablement } from "./feature-flags"; import { getGitRoot, isAnalyzingDefaultBranch } from "./git-utils"; @@ -27,6 +27,7 @@ import { OverlayDatabaseMode, } from "./overlay-database-utils"; import { RepositoryNwo } from "./repository"; +import * as resolvedLanguages from "./resolved-languages.json"; import { downloadTrapCaches } from "./trap-caching"; import { GitHubVersion, @@ -331,6 +332,30 @@ export async function getSupportedLanguageMap( `The CodeQL CLI supports the following languages: ${Object.keys(resolveResult.extractors).join(", ")}`, ); } + return buildSupportedLanguageMap( + resolveResult, + resolveSupportedLanguagesUsingCli, + ); +} + +// This function serves the same purpose as getSupportedLanguageMap(), but it +// uses the stored resolved-languages.json file instead of calling out to the +// CodeQL CLI. +export function getStoredSupportedLanguageMap(): Record { + return buildSupportedLanguageMap( + resolvedLanguages, + // resolveSupportedLanguagesUsingCli is false because we currently generate + // resolved-languages.json without the --filter-to-languages-with-queries + // flag (see .github/workflows/validate-resolved-languages.yml). Once that + // changes, we should set this to true. + false, + ); +} + +function buildSupportedLanguageMap( + resolveResult: BetterResolveLanguagesOutput, + resolveSupportedLanguagesUsingCli: boolean, +): Record { const supportedLanguages: Record = {}; // Populate canonical language names for (const extractor of Object.keys(resolveResult.extractors)) { From 2969f87fedafc3d8595593db7898e40594c71342 Mon Sep 17 00:00:00 2001 From: Chuan-kai Lin Date: Mon, 15 Sep 2025 13:43:14 -0700 Subject: [PATCH 5/6] Call getSupportedLanguageMap() outside getLanguages() --- src/config-utils.test.ts | 11 +++++++---- src/config-utils.ts | 9 +++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/config-utils.test.ts b/src/config-utils.test.ts index b5ef777170..9c4c5bd8ca 100644 --- a/src/config-utils.test.ts +++ b/src/config-utils.test.ts @@ -1163,14 +1163,18 @@ const mockRepositoryNwo = parseRepositoryNwo("owner/repo"); }), }); + const languageMap = await configUtils.getSupportedLanguageMap( + codeQL, + features, + mockLogger, + ); if (args.expectedLanguages) { // happy path const actualLanguages = await configUtils.getLanguages( - codeQL, + languageMap, args.languagesInput, mockRepositoryNwo, ".", - features, mockLogger, ); @@ -1180,11 +1184,10 @@ const mockRepositoryNwo = parseRepositoryNwo("owner/repo"); await t.throwsAsync( async () => await configUtils.getLanguages( - codeQL, + languageMap, args.languagesInput, mockRepositoryNwo, ".", - features, mockLogger, ), { message: args.expectedError }, diff --git a/src/config-utils.ts b/src/config-utils.ts index 5b41d0cfbf..c071de1d23 100644 --- a/src/config-utils.ts +++ b/src/config-utils.ts @@ -443,11 +443,10 @@ export async function getRawLanguagesInRepo( * then throw an error. */ export async function getLanguages( - codeql: CodeQL, + languageMap: Record, languagesInput: string | undefined, repository: RepositoryNwo, sourceRoot: string, - features: FeatureEnablement, logger: Logger, ): Promise { // Obtain languages without filtering them. @@ -457,8 +456,6 @@ export async function getLanguages( sourceRoot, logger, ); - - const languageMap = await getSupportedLanguageMap(codeql, features, logger); const languagesSet = new Set(); const unknownLanguages: string[] = []; @@ -600,12 +597,12 @@ export async function initActionState( analysisKinds.push(AnalysisKind.CodeQuality); } + const languageMap = await getSupportedLanguageMap(codeql, features, logger); const languages = await getLanguages( - codeql, + languageMap, languagesInput, repository, sourceRoot, - features, logger, ); From 376055c94fcc266eb5325c9b6be9a2e1125ec84b Mon Sep 17 00:00:00 2001 From: Chuan-kai Lin Date: Mon, 15 Sep 2025 13:59:45 -0700 Subject: [PATCH 6/6] build: refresh js files --- lib/init-action.js | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/init-action.js b/lib/init-action.js index 2f2fc6a350..fa94fc5908 100644 --- a/lib/init-action.js +++ b/lib/init-action.js @@ -87221,6 +87221,12 @@ async function getSupportedLanguageMap(codeql, features, logger) { `The CodeQL CLI supports the following languages: ${Object.keys(resolveResult.extractors).join(", ")}` ); } + return buildSupportedLanguageMap( + resolveResult, + resolveSupportedLanguagesUsingCli + ); +} +function buildSupportedLanguageMap(resolveResult, resolveSupportedLanguagesUsingCli) { const supportedLanguages = {}; for (const extractor of Object.keys(resolveResult.extractors)) { if (resolveSupportedLanguagesUsingCli || KnownLanguage[extractor] !== void 0) { @@ -87259,14 +87265,13 @@ async function getRawLanguagesInRepo(repository, sourceRoot, logger) { logger.debug(`Raw languages in repository: ${result.join(", ")}`); return result; } -async function getLanguages(codeql, languagesInput, repository, sourceRoot, features, logger) { +async function getLanguages(languageMap, languagesInput, repository, sourceRoot, logger) { const { rawLanguages, autodetected } = await getRawLanguages( languagesInput, repository, sourceRoot, logger ); - const languageMap = await getSupportedLanguageMap(codeql, features, logger); const languagesSet = /* @__PURE__ */ new Set(); const unknownLanguages = []; for (const language of rawLanguages) { @@ -87329,12 +87334,12 @@ async function initActionState({ if (!analysisKinds.includes("code-quality" /* CodeQuality */) && qualityQueriesInput !== void 0) { analysisKinds.push("code-quality" /* CodeQuality */); } + const languageMap = await getSupportedLanguageMap(codeql, features, logger); const languages = await getLanguages( - codeql, + languageMap, languagesInput, repository, sourceRoot, - features, logger ); const buildMode = await parseBuildModeInput(