From 4b011eba0de372973e245c7107d061625bd090d8 Mon Sep 17 00:00:00 2001 From: Suzanne Aitchison Date: Thu, 1 May 2025 10:56:18 +0100 Subject: [PATCH 01/10] feat: expect enhancedSecretScan flag --- packages/build/docs/flow.md | 3 ++- packages/build/src/core/build.ts | 8 ++++++++ packages/build/src/core/flags.js | 5 +++++ packages/build/src/log/messages/config.js | 1 + packages/build/src/plugins_core/secrets_scanning/index.ts | 4 ++++ packages/build/src/plugins_core/types.ts | 1 + packages/build/src/steps/core_step.ts | 2 ++ packages/build/src/steps/run_step.ts | 7 +++++++ packages/build/src/steps/run_steps.js | 2 ++ 9 files changed, 32 insertions(+), 1 deletion(-) diff --git a/packages/build/docs/flow.md b/packages/build/docs/flow.md index 97a0f28215..eed2be0c84 100644 --- a/packages/build/docs/flow.md +++ b/packages/build/docs/flow.md @@ -95,7 +95,8 @@ Current core steps are: [build command](https://github.com/netlify/build/blob/9b261a6182e1ba6853966bd8d0bde9064209af7d/packages/build/src/plugins_core/build_command.js#L11). - [Functions bundling](https://github.com/netlify/build/blob/9b261a6182e1ba6853966bd8d0bde9064209af7d/packages/build/src/plugins_core/functions/index.js#L142), which uses [`zip-it-and-ship-it`](https://github.com/netlify/zip-it-and-ship-it). -- Secrets Scanning only runs when the account passes non-empty explicitSecretKeys. +- Secrets Scanning only runs when the account passes non-empty explicitSecretKeys, or when the account has enhanced + secret scanning enabled (only applies to certain plan types). - [Site deploy](https://github.com/netlify/build/blob/9b261a6182e1ba6853966bd8d0bde9064209af7d/packages/build/src/plugins_core/deploy/index.js#L66): - This sends a network request to the buildbot to initiate site deploy. - When the buildbot diff --git a/packages/build/src/core/build.ts b/packages/build/src/core/build.ts index 63d1a4ad8f..3840a21b3c 100644 --- a/packages/build/src/core/build.ts +++ b/packages/build/src/core/build.ts @@ -85,6 +85,7 @@ const tExecBuild = async function ({ quiet, framework, explicitSecretKeys, + enhancedSecretScan, edgeFunctionsBootstrapURL, eventHandlers, }) { @@ -223,6 +224,7 @@ const tExecBuild = async function ({ quiet, integrations, explicitSecretKeys, + enhancedSecretScan, edgeFunctionsBootstrapURL, eventHandlers, }) @@ -284,6 +286,7 @@ export const runAndReportBuild = async function ({ quiet, integrations, explicitSecretKeys, + enhancedSecretScan, edgeFunctionsBootstrapURL, eventHandlers, }) { @@ -340,6 +343,7 @@ export const runAndReportBuild = async function ({ quiet, integrations, explicitSecretKeys, + enhancedSecretScan, edgeFunctionsBootstrapURL, eventHandlers, }) @@ -446,6 +450,7 @@ const initAndRunBuild = async function ({ quiet, integrations, explicitSecretKeys, + enhancedSecretScan, edgeFunctionsBootstrapURL, eventHandlers, }) { @@ -551,6 +556,7 @@ const initAndRunBuild = async function ({ devCommand, quiet, explicitSecretKeys, + enhancedSecretScan, edgeFunctionsBootstrapURL, eventHandlers, }) @@ -627,6 +633,7 @@ const runBuild = async function ({ devCommand, quiet, explicitSecretKeys, + enhancedSecretScan, edgeFunctionsBootstrapURL, eventHandlers, }) { @@ -694,6 +701,7 @@ const runBuild = async function ({ quiet, userNodeVersion, explicitSecretKeys, + enhancedSecretScan, edgeFunctionsBootstrapURL, }) diff --git a/packages/build/src/core/flags.js b/packages/build/src/core/flags.js index 661bd86ec8..f4676cb2e3 100644 --- a/packages/build/src/core/flags.js +++ b/packages/build/src/core/flags.js @@ -218,4 +218,9 @@ Default: false`, describe: 'Env var keys that are marked as secret explicitly.', hidden: true, }, + enhancedSecretScan: { + boolean: true, + hidden: true, + describe: 'Scan for potential secrets in all env vars', + }, } diff --git a/packages/build/src/log/messages/config.js b/packages/build/src/log/messages/config.js index a99361cfb7..32342369b5 100644 --- a/packages/build/src/log/messages/config.js +++ b/packages/build/src/log/messages/config.js @@ -56,6 +56,7 @@ const INTERNAL_FLAGS = [ 'systemLogFile', 'timeline', 'explicitSecretKeys', + 'enhancedSecretScan', 'edgeFunctionsBootstrapURL', 'eventHandlers', ] diff --git a/packages/build/src/plugins_core/secrets_scanning/index.ts b/packages/build/src/plugins_core/secrets_scanning/index.ts index 5ac3b8e3fb..d548e4f82e 100644 --- a/packages/build/src/plugins_core/secrets_scanning/index.ts +++ b/packages/build/src/plugins_core/secrets_scanning/index.ts @@ -27,6 +27,8 @@ const coreStep: CoreStepFunction = async function ({ logs, netlifyConfig, explicitSecretKeys, + // eslint-disable-next-line @typescript-eslint/no-unused-vars + enhancedSecretScan, systemLog, deployId, api, @@ -51,6 +53,8 @@ const coreStep: CoreStepFunction = async function ({ log(logs, `SECRETS_SCAN_OMIT_PATHS override option set to: ${envVars['SECRETS_SCAN_OMIT_PATHS']}\n`) } + // TODO: here detect if there are any other potential secrets we should pass to the keys to search for + const keysToSearchFor = getSecretKeysToScanFor(envVars, passedSecretKeys) if (keysToSearchFor.length === 0) { diff --git a/packages/build/src/plugins_core/types.ts b/packages/build/src/plugins_core/types.ts index 5d65b5599f..7d4c0d8446 100644 --- a/packages/build/src/plugins_core/types.ts +++ b/packages/build/src/plugins_core/types.ts @@ -29,6 +29,7 @@ export type CoreStepFunctionArgs = { netlifyConfig: NetlifyConfig explicitSecretKeys: $TSFixme + enhancedSecretScan: boolean buildbotServerSocket: $TSFixme api: DynamicMethods diff --git a/packages/build/src/steps/core_step.ts b/packages/build/src/steps/core_step.ts index 9ea1fa0cab..692e0133ca 100644 --- a/packages/build/src/steps/core_step.ts +++ b/packages/build/src/steps/core_step.ts @@ -37,6 +37,7 @@ export const fireCoreStep = async function ({ saveConfig, userNodeVersion, explicitSecretKeys, + enhancedSecretScan, edgeFunctionsBootstrapURL, deployId, outputFlusher, @@ -79,6 +80,7 @@ export const fireCoreStep = async function ({ saveConfig, userNodeVersion, explicitSecretKeys, + enhancedSecretScan, edgeFunctionsBootstrapURL, deployId, }) diff --git a/packages/build/src/steps/run_step.ts b/packages/build/src/steps/run_step.ts index 6ace3b0238..b15fd3fa2d 100644 --- a/packages/build/src/steps/run_step.ts +++ b/packages/build/src/steps/run_step.ts @@ -68,6 +68,7 @@ export const runStep = async function ({ quiet, userNodeVersion, explicitSecretKeys, + enhancedSecretScan, edgeFunctionsBootstrapURL, extensionMetadata, }) { @@ -108,6 +109,7 @@ export const runStep = async function ({ buildDir, saveConfig, explicitSecretKeys, + enhancedSecretScan, deployId, featureFlags, }) @@ -182,6 +184,7 @@ export const runStep = async function ({ featureFlags, userNodeVersion, explicitSecretKeys, + enhancedSecretScan, edgeFunctionsBootstrapURL, deployId, api, @@ -264,6 +267,7 @@ const shouldRunStep = async function ({ buildDir, saveConfig, explicitSecretKeys, + enhancedSecretScan, deployId, featureFlags = {}, }) { @@ -278,6 +282,7 @@ const shouldRunStep = async function ({ netlifyConfig, saveConfig, explicitSecretKeys, + enhancedSecretScan, deployId, featureFlags, }))) @@ -344,6 +349,7 @@ const tFireStep = function ({ featureFlags, userNodeVersion, explicitSecretKeys, + enhancedSecretScan, edgeFunctionsBootstrapURL, deployId, extensionMetadata, @@ -383,6 +389,7 @@ const tFireStep = function ({ saveConfig, userNodeVersion, explicitSecretKeys, + enhancedSecretScan, edgeFunctionsBootstrapURL, deployId, api, diff --git a/packages/build/src/steps/run_steps.js b/packages/build/src/steps/run_steps.js index 265dd001be..91812bdebf 100644 --- a/packages/build/src/steps/run_steps.js +++ b/packages/build/src/steps/run_steps.js @@ -45,6 +45,7 @@ export const runSteps = async function ({ quiet, userNodeVersion, explicitSecretKeys, + enhancedSecretScan, edgeFunctionsBootstrapURL, }) { const { @@ -153,6 +154,7 @@ export const runSteps = async function ({ quiet, userNodeVersion, explicitSecretKeys, + enhancedSecretScan, edgeFunctionsBootstrapURL, }) From 9b626fe382343be243ddf4174c3e2b21b7efde3d Mon Sep 17 00:00:00 2001 From: Suzanne Aitchison Date: Thu, 1 May 2025 14:01:14 +0100 Subject: [PATCH 02/10] feat: run enhanced secret scan --- packages/build/src/log/messages/core_steps.js | 40 ++-- .../plugins_core/secrets_scanning/index.ts | 57 ++++-- .../plugins_core/secrets_scanning/utils.ts | 173 +++++++++++++----- 3 files changed, 197 insertions(+), 73 deletions(-) diff --git a/packages/build/src/log/messages/core_steps.js b/packages/build/src/log/messages/core_steps.js index e5587e288a..af80b1830c 100644 --- a/packages/build/src/log/messages/core_steps.js +++ b/packages/build/src/log/messages/core_steps.js @@ -129,22 +129,40 @@ export const logSecretsScanSuccessMessage = function (logs, msg) { } export const logSecretsScanFailBuildMessage = function ({ logs, scanResults, groupedResults }) { + const { secretMatches, enhancedSecretMatches } = groupedResults + logErrorSubHeader( logs, `Scanning complete. ${scanResults.scannedFilesCount} file(s) scanned. Secrets scanning found ${scanResults.matches.length} instance(s) of secrets in build output or repo code.\n`, ) - Object.keys(groupedResults).forEach((key) => { - logError(logs, `Secret env var "${key}"'s value detected:`) - - groupedResults[key] - .sort((a, b) => { - return a.file > b.file ? 0 : 1 - }) - .forEach(({ lineNumber, file }) => { - logError(logs, `found value at line ${lineNumber} in ${file}`, { indent: true }) - }) - }) + if (Object.keys(secretMatches).length > 0) { + Object.keys(secretMatches).forEach((key) => { + logError(logs, `Secret env var "${key}"'s value detected:`) + + secretMatches[key] + .sort((a, b) => { + return a.file > b.file ? 0 : 1 + }) + .forEach(({ lineNumber, file }) => { + logError(logs, `found value at line ${lineNumber} in ${file}`, { indent: true }) + }) + }) + } + + if (Object.keys(enhancedSecretMatches).length > 0) { + Object.keys(enhancedSecretMatches).forEach((key) => { + logError(logs, `Env var "${key}"'s value detected as a likely secret value:`) + + enhancedSecretMatches[key] + .sort((a, b) => { + return a.file > b.file ? 0 : 1 + }) + .forEach(({ lineNumber, file }) => { + logError(logs, `found value at line ${lineNumber} in ${file}`, { indent: true }) + }) + }) + } logError( logs, diff --git a/packages/build/src/plugins_core/secrets_scanning/index.ts b/packages/build/src/plugins_core/secrets_scanning/index.ts index d548e4f82e..c01954ab4a 100644 --- a/packages/build/src/plugins_core/secrets_scanning/index.ts +++ b/packages/build/src/plugins_core/secrets_scanning/index.ts @@ -14,8 +14,9 @@ import { ScanResults, SecretScanResult, getFilePathsToScan, + getNonSecretKeysToScanFor, getSecretKeysToScanFor, - groupScanResultsByKey, + groupScanResultsByKeyAndScanType, isSecretsScanningEnabled, scanFilesForKeyValues, } from './utils.js' @@ -27,7 +28,6 @@ const coreStep: CoreStepFunction = async function ({ logs, netlifyConfig, explicitSecretKeys, - // eslint-disable-next-line @typescript-eslint/no-unused-vars enhancedSecretScan, systemLog, deployId, @@ -53,17 +53,19 @@ const coreStep: CoreStepFunction = async function ({ log(logs, `SECRETS_SCAN_OMIT_PATHS override option set to: ${envVars['SECRETS_SCAN_OMIT_PATHS']}\n`) } - // TODO: here detect if there are any other potential secrets we should pass to the keys to search for - - const keysToSearchFor = getSecretKeysToScanFor(envVars, passedSecretKeys) + const explicitSecretKeysToScanFor = getSecretKeysToScanFor(envVars, passedSecretKeys) + const otherKeysToScanFor = enhancedSecretScan ? getNonSecretKeysToScanFor(envVars, passedSecretKeys) : [] + const keysToSearchFor = explicitSecretKeysToScanFor.concat(otherKeysToScanFor) if (keysToSearchFor.length === 0) { - logSecretsScanSkipMessage( - logs, - 'Secrets scanning skipped because no env vars marked as secret are set to non-empty/non-trivial values or they are all omitted with SECRETS_SCAN_OMIT_KEYS env var setting.', - ) + const skippedTheEnhancedScan = enhancedSecretScan && otherKeysToScanFor.length > 0 + const msg = skippedTheEnhancedScan + ? 'Secrets scanning skipped because no env vars are set to non-empty/non-trivial values or they are all omitted with SECRETS_SCAN_OMIT_KEYS env var setting.' + : 'Secrets scanning skipped because no env vars marked as secret are set to non-empty/non-trivial values or they are all omitted with SECRETS_SCAN_OMIT_KEYS env var setting.' + logSecretsScanSkipMessage(logs, msg) return stepResults } + // TODO: separate arrays for the secret matches vs enhanced secret matches // buildDir is the repository root or the base folder // The scanning will look at builddir so that it can review both repo pulled files @@ -79,6 +81,8 @@ const coreStep: CoreStepFunction = async function ({ } let scanResults: ScanResults | undefined + let secretMatches: SecretScanResult['secretsScanMatches'] | undefined + let enhancedSecretMatches: SecretScanResult['enhancedSecretsScanMatches'] | undefined await tracer.startActiveSpan( 'scanning-files', @@ -91,9 +95,14 @@ const coreStep: CoreStepFunction = async function ({ filePaths, }) + secretMatches = scanResults.matches.filter((match) => match.key in explicitSecretKeysToScanFor) + enhancedSecretMatches = scanResults.matches.filter((match) => match.key in otherKeysToScanFor) + const attributesForLogsAndSpan = { - secretsScanFoundSecrets: scanResults.matches.length > 0, - secretsScanMatchesCount: scanResults.matches.length, + secretsScanFoundSecrets: secretMatches.length > 0, + enhancedSecretsScanFoundSecrets: enhancedSecretMatches.length > 0, + secretsScanMatchesCount: secretMatches.length, + enhancedSecretsScanMatchesCount: enhancedSecretMatches.length, secretsFilesCount: scanResults.scannedFilesCount, keysToSearchFor, } @@ -107,7 +116,8 @@ const coreStep: CoreStepFunction = async function ({ if (deployId !== '0') { const secretScanResult: SecretScanResult = { scannedFilesCount: scanResults?.scannedFilesCount ?? 0, - secretsScanMatches: scanResults?.matches ?? [], + secretsScanMatches: secretMatches ?? [], + enhancedSecretsScanMatches: enhancedSecretMatches ?? [], } reportValidations({ api, secretScanResult, deployId, systemLog }) } @@ -122,18 +132,27 @@ const coreStep: CoreStepFunction = async function ({ // at this point we have found matching secrets // Output the results and fail the build - - logSecretsScanFailBuildMessage({ logs, scanResults, groupedResults: groupScanResultsByKey(scanResults) }) + logSecretsScanFailBuildMessage({ + logs, + scanResults, + groupedResults: groupScanResultsByKeyAndScanType(scanResults, explicitSecretKeysToScanFor), + }) const error = new Error(`Secrets scanning found secrets in build.`) addErrorInfo(error, { type: 'secretScanningFoundSecrets' }) throw error } -// We run this core step if the build was run with explicit secret keys. This -// is passed from BB to build so only accounts that are allowed to have explicit -// secrets and actually have them will have them. -const hasExplicitSecretsKeys: CoreStepCondition = function ({ explicitSecretKeys }): boolean { +// We run this core step if the build was run with explicit secret keys or if enhanced secret scanning is enabled. +// This is passed from BB to build so only accounts that are allowed to have explicit +// secrets and actually have them / have enhanced secret scanning enabled will have them. +const hasExplicitSecretsKeysOrEnhancedScanningEnabled: CoreStepCondition = function ({ + explicitSecretKeys, + enhancedSecretScan, +}): boolean { + if (enhancedSecretScan) { + return true + } if (typeof explicitSecretKeys !== 'string') { return false } @@ -147,5 +166,5 @@ export const scanForSecrets: CoreStep = { coreStepId: 'secrets_scanning', coreStepName: 'Secrets scanning', coreStepDescription: () => 'Scanning for secrets in code and build output.', - condition: hasExplicitSecretsKeys, + condition: hasExplicitSecretsKeysOrEnhancedScanningEnabled, } diff --git a/packages/build/src/plugins_core/secrets_scanning/utils.ts b/packages/build/src/plugins_core/secrets_scanning/utils.ts index a5d1620ba3..ba85b8fdfe 100644 --- a/packages/build/src/plugins_core/secrets_scanning/utils.ts +++ b/packages/build/src/plugins_core/secrets_scanning/utils.ts @@ -26,6 +26,7 @@ interface MatchResult { export type SecretScanResult = { scannedFilesCount: number secretsScanMatches: MatchResult[] + enhancedSecretsScanMatches: MatchResult[] } /** @@ -40,6 +41,41 @@ export function isSecretsScanningEnabled(env: Record): boolean return true } +function filterOmittedKeys(env: Record, envKeys: string[] = []): string[] { + let omitKeys: string[] = [] + if (typeof env.SECRETS_SCAN_OMIT_KEYS === 'string') { + omitKeys = env.SECRETS_SCAN_OMIT_KEYS.split(',') + .map((s) => s.trim()) + .filter(Boolean) + } + + return envKeys.filter((key) => !omitKeys.includes(key)) +} + +/** + * non-trivial values are values that are: + * - >4 characters/digits + * - not booleans + */ +function isValueNonTrivial(val): boolean { + if (typeof val === 'string') { + // string forms of booleans + if (val === 'true' || val === 'false') { + return false + } + + // non-trivial/non-empty values only + return val.trim().length > 4 + } else if (typeof val === 'boolean') { + // booleans are trivial values + return false + } else if (typeof val === 'number' || typeof val === 'object') { + return JSON.stringify(val).length > 4 + } + + return !!val +} + /** * given the explicit secret keys and env vars, return the list of secret keys which have non-empty or non-trivial values. This * will also filter out keys passed in the SECRETS_SCAN_OMIT_KEYS env var. @@ -53,36 +89,66 @@ export function isSecretsScanningEnabled(env: Record): boolean * @returns string[] */ export function getSecretKeysToScanFor(env: Record, secretKeys: string[]): string[] { - let omitKeys: string[] = [] - if (typeof env.SECRETS_SCAN_OMIT_KEYS === 'string') { - omitKeys = env.SECRETS_SCAN_OMIT_KEYS.split(',') - .map((s) => s.trim()) - .filter(Boolean) - } - + const omitKeys = filterOmittedKeys(env, Object.keys(env)) return secretKeys.filter((key) => { if (omitKeys.includes(key)) { return false } const val = env[key] - if (typeof val === 'string') { - // string forms of booleans - if (val === 'true' || val === 'false') { - return false - } + return isValueNonTrivial(val) + }) +} + +/** + * given the explicit secret keys and env vars, return the list of non-secret keys which should be scanned in the enhanced secret scan + * (i.e. any that look very likely to be secrets). + * This will also filter out keys passed in the SECRETS_SCAN_OMIT_KEYS env var. + * + * @param env env vars list + * @param secretKeys + * @returns string[] + */ +export function getNonSecretKeysToScanFor(env: Record, secretKeys: string[]): string[] { + const nonSecretKeys = Object.keys(env).filter((key) => !secretKeys.includes(key)) + const omitKeys = filterOmittedKeys(env, nonSecretKeys) - // non-trivial/non-empty values only - return val.trim().length > 4 - } else if (typeof val === 'boolean') { - // booleans are trivial values + const nonSecretKeysToScanFor = nonSecretKeys.filter((key) => { + if (omitKeys.includes(key)) { return false - } else if (typeof val === 'number' || typeof val === 'object') { - return JSON.stringify(val).length > 4 } - - return !!val + const val = env[key] + if (!isValueNonTrivial(val)) { + return false + } + return isLikelySecretValue(val) }) + return nonSecretKeysToScanFor +} + +const AWS_PREFIXES = ['aws_', 'asia'] +const SLACK_PREFIXES = ['xoxb-', 'xwfp-', 'xoxb-', 'xoxp-', 'xapp-'] +const LIKELY_SECRET_PREFIXES = ['pk_', 'sk_', 'pat_', 'db_', 'github_pat_', ...AWS_PREFIXES, ...SLACK_PREFIXES] +const LIKELY_SECRET_MIN_LENGTH = 16 + +/** + * When the enhanced secret scan is run, we check any env vars _not_ marked as secret if they are highly likely to be secret values. + * For now, this means the value is a string of at least 16 chars and starts with one of the known prefixes. + * + * @param val env var value + * @returns boolean + */ +function isLikelySecretValue(val): boolean { + if (typeof val !== 'string') { + return false + } + if (val.length < LIKELY_SECRET_MIN_LENGTH) { + return false + } + if (LIKELY_SECRET_PREFIXES.some((prefix) => val.toLowerCase().startsWith(prefix))) { + return true + } + return false } /** @@ -335,43 +401,64 @@ const searchStream = (basePath: string, file: string, keyValues: Record { - if (!matchesByKeys[matchResult.key]) { - matchesByKeys[matchResult.key] = [] + const isEnhancedCheck = enhancedScanKeys.includes(matchResult.key) + if (isEnhancedCheck) { + if (!enhancedSecretMatchesByKeys[matchResult.key]) { + enhancedSecretMatchesByKeys[matchResult.key] = [] + } + enhancedSecretMatchesByKeys[matchResult.key].push(matchResult) + } else { + if (!secretMatchesByKeys[matchResult.key]) { + secretMatchesByKeys[matchResult.key] = [] + } + secretMatchesByKeys[matchResult.key].push(matchResult) } - matchesByKeys[matchResult.key].push(matchResult) }) // sort results to get a consistent output and logically ordered match results - Object.keys(matchesByKeys).forEach((key) => { - matchesByKeys[key].sort((a, b) => { - // sort by file name first - if (a.file > b.file) { - return 1 - } - - // sort by line number second - if (a.file === b.file) { - if (a.lineNumber > b.lineNumber) { + const sortMatches = (matchesByKeys: { [key: string]: MatchResult[] }) => { + Object.keys(matchesByKeys).forEach((key) => { + matchesByKeys[key].sort((a, b) => { + // sort by file name first + if (a.file > b.file) { return 1 } - if (a.lineNumber === b.lineNumber) { - return 0 + + // sort by line number second + if (a.file === b.file) { + if (a.lineNumber > b.lineNumber) { + return 1 + } + if (a.lineNumber === b.lineNumber) { + return 0 + } + return -1 } return -1 - } - return -1 + }) }) - }) - return matchesByKeys + } + + sortMatches(secretMatchesByKeys) + sortMatches(enhancedSecretMatchesByKeys) + + return { secretMatches: secretMatchesByKeys, enhancedSecretMatches: enhancedSecretMatchesByKeys } } function isMultiLineVal(v) { From c675501c1a8a34c2997a5ded721d28a36b9b2694 Mon Sep 17 00:00:00 2001 From: Suzanne Aitchison Date: Fri, 2 May 2025 14:33:00 +0100 Subject: [PATCH 03/10] feat: add tests for enhanced scan, fix logic with omit keys --- .../plugins_core/secrets_scanning/index.ts | 3 +- .../plugins_core/secrets_scanning/utils.ts | 18 +-- .../netlify.toml | 1 + .../netlify.toml | 3 + .../netlify.toml | 4 + .../netlify.toml | 3 + .../src_scanning_omit_all_keys/netlify.toml | 2 +- .../src_scanning_omit_all_paths/netlify.toml | 2 +- .../src_scanning_omit_glob_path/netlify.toml | 2 +- .../secrets_scanning/snapshots/tests.js.md | 40 +----- .../secrets_scanning/snapshots/tests.js.snap | Bin 1615 -> 1586 bytes .../build/tests/secrets_scanning/tests.js | 119 +++++++++++++++++- 12 files changed, 134 insertions(+), 63 deletions(-) create mode 100644 packages/build/tests/secrets_scanning/fixtures/src_scanning_likely_enhanced_scan_secrets/netlify.toml create mode 100644 packages/build/tests/secrets_scanning/fixtures/src_scanning_likely_enhanced_scan_secrets_disabled/netlify.toml create mode 100644 packages/build/tests/secrets_scanning/fixtures/src_scanning_no_likely_enhanced_scan_secrets/netlify.toml diff --git a/packages/build/src/plugins_core/secrets_scanning/index.ts b/packages/build/src/plugins_core/secrets_scanning/index.ts index c01954ab4a..4e657c3c11 100644 --- a/packages/build/src/plugins_core/secrets_scanning/index.ts +++ b/packages/build/src/plugins_core/secrets_scanning/index.ts @@ -65,7 +65,6 @@ const coreStep: CoreStepFunction = async function ({ logSecretsScanSkipMessage(logs, msg) return stepResults } - // TODO: separate arrays for the secret matches vs enhanced secret matches // buildDir is the repository root or the base folder // The scanning will look at builddir so that it can review both repo pulled files @@ -135,7 +134,7 @@ const coreStep: CoreStepFunction = async function ({ logSecretsScanFailBuildMessage({ logs, scanResults, - groupedResults: groupScanResultsByKeyAndScanType(scanResults, explicitSecretKeysToScanFor), + groupedResults: groupScanResultsByKeyAndScanType(scanResults, otherKeysToScanFor), }) const error = new Error(`Secrets scanning found secrets in build.`) diff --git a/packages/build/src/plugins_core/secrets_scanning/utils.ts b/packages/build/src/plugins_core/secrets_scanning/utils.ts index ba85b8fdfe..d1313544bc 100644 --- a/packages/build/src/plugins_core/secrets_scanning/utils.ts +++ b/packages/build/src/plugins_core/secrets_scanning/utils.ts @@ -89,15 +89,8 @@ function isValueNonTrivial(val): boolean { * @returns string[] */ export function getSecretKeysToScanFor(env: Record, secretKeys: string[]): string[] { - const omitKeys = filterOmittedKeys(env, Object.keys(env)) - return secretKeys.filter((key) => { - if (omitKeys.includes(key)) { - return false - } - - const val = env[key] - return isValueNonTrivial(val) - }) + const filteredSecretKeys = filterOmittedKeys(env, secretKeys) + return filteredSecretKeys.filter((key) => isValueNonTrivial(env[key])) } /** @@ -111,12 +104,9 @@ export function getSecretKeysToScanFor(env: Record, secretKeys: */ export function getNonSecretKeysToScanFor(env: Record, secretKeys: string[]): string[] { const nonSecretKeys = Object.keys(env).filter((key) => !secretKeys.includes(key)) - const omitKeys = filterOmittedKeys(env, nonSecretKeys) + const filteredNonSecretKeys = filterOmittedKeys(env, nonSecretKeys) - const nonSecretKeysToScanFor = nonSecretKeys.filter((key) => { - if (omitKeys.includes(key)) { - return false - } + const nonSecretKeysToScanFor = filteredNonSecretKeys.filter((key) => { const val = env[key] if (!isValueNonTrivial(val)) { return false diff --git a/packages/build/tests/secrets_scanning/fixtures/src_scanning_env_vars_no_matches/netlify.toml b/packages/build/tests/secrets_scanning/fixtures/src_scanning_env_vars_no_matches/netlify.toml index 7e1454a91c..1916320bae 100644 --- a/packages/build/tests/secrets_scanning/fixtures/src_scanning_env_vars_no_matches/netlify.toml +++ b/packages/build/tests/secrets_scanning/fixtures/src_scanning_env_vars_no_matches/netlify.toml @@ -5,6 +5,7 @@ [build.environment] ENV_VAR_1 = "some other val1" ENV_VAR_2 = "some other val2" + ENV_VAR_3 = "pk_potential_secret_key" ENV_VAR_NOT_SECRET = "not secret val" # using the toml to provide the values means we omit this file to result in not matches SECRETS_SCAN_OMIT_PATHS = "netlify.toml" diff --git a/packages/build/tests/secrets_scanning/fixtures/src_scanning_likely_enhanced_scan_secrets/netlify.toml b/packages/build/tests/secrets_scanning/fixtures/src_scanning_likely_enhanced_scan_secrets/netlify.toml new file mode 100644 index 0000000000..c0033e52b7 --- /dev/null +++ b/packages/build/tests/secrets_scanning/fixtures/src_scanning_likely_enhanced_scan_secrets/netlify.toml @@ -0,0 +1,3 @@ +[build.environment] + ENV_VAR_1 = "pk_12345678901234567890" + ENV_VAR_2 = "val2-val2-val2" diff --git a/packages/build/tests/secrets_scanning/fixtures/src_scanning_likely_enhanced_scan_secrets_disabled/netlify.toml b/packages/build/tests/secrets_scanning/fixtures/src_scanning_likely_enhanced_scan_secrets_disabled/netlify.toml new file mode 100644 index 0000000000..f662a236bf --- /dev/null +++ b/packages/build/tests/secrets_scanning/fixtures/src_scanning_likely_enhanced_scan_secrets_disabled/netlify.toml @@ -0,0 +1,4 @@ +[build.environment] + SECRETS_SCAN_ENABLED = "false" + ENV_VAR_1 = "pk_12345678901234567890" + ENV_VAR_2 = "val2-val2-val2" diff --git a/packages/build/tests/secrets_scanning/fixtures/src_scanning_no_likely_enhanced_scan_secrets/netlify.toml b/packages/build/tests/secrets_scanning/fixtures/src_scanning_no_likely_enhanced_scan_secrets/netlify.toml new file mode 100644 index 0000000000..2ab33aee04 --- /dev/null +++ b/packages/build/tests/secrets_scanning/fixtures/src_scanning_no_likely_enhanced_scan_secrets/netlify.toml @@ -0,0 +1,3 @@ +[build.environment] + ENV_VAR_1 = "val1-val1-val1" + ENV_VAR_2 = "val2-val2-val2" diff --git a/packages/build/tests/secrets_scanning/fixtures/src_scanning_omit_all_keys/netlify.toml b/packages/build/tests/secrets_scanning/fixtures/src_scanning_omit_all_keys/netlify.toml index 989a3b1d5d..f3dcc9c81a 100644 --- a/packages/build/tests/secrets_scanning/fixtures/src_scanning_omit_all_keys/netlify.toml +++ b/packages/build/tests/secrets_scanning/fixtures/src_scanning_omit_all_keys/netlify.toml @@ -1,4 +1,4 @@ [build.environment] SECRETS_SCAN_OMIT_KEYS = "ENV_VAR_2,ENV_VAR_1" - ENV_VAR_1 = "val1-val1-val1" + ENV_VAR_1 = "pk_12345678901234567890_val1" ENV_VAR_2 = "val2-val2-val2" diff --git a/packages/build/tests/secrets_scanning/fixtures/src_scanning_omit_all_paths/netlify.toml b/packages/build/tests/secrets_scanning/fixtures/src_scanning_omit_all_paths/netlify.toml index c4f5198b3a..c7a5f5a0ef 100644 --- a/packages/build/tests/secrets_scanning/fixtures/src_scanning_omit_all_paths/netlify.toml +++ b/packages/build/tests/secrets_scanning/fixtures/src_scanning_omit_all_paths/netlify.toml @@ -3,7 +3,7 @@ publish = "./dist" [build.environment] - ENV_VAR_1 = "val1-val1-val1" + ENV_VAR_1 = "pk_12345678901234567890" ENV_VAR_2 = "val2-val2-val2" # the / path here will skip all files SECRETS_SCAN_OMIT_PATHS = "/some-path-that-doesnt-exist,src/,dist,netlify.toml" diff --git a/packages/build/tests/secrets_scanning/fixtures/src_scanning_omit_glob_path/netlify.toml b/packages/build/tests/secrets_scanning/fixtures/src_scanning_omit_glob_path/netlify.toml index 5744c384c4..9efee863bf 100644 --- a/packages/build/tests/secrets_scanning/fixtures/src_scanning_omit_glob_path/netlify.toml +++ b/packages/build/tests/secrets_scanning/fixtures/src_scanning_omit_glob_path/netlify.toml @@ -3,6 +3,6 @@ publish = "./dist" [build.environment] - ENV_VAR_1 = "val1-val1-val1" + ENV_VAR_1 = "pk_12345678901234567890" ENV_VAR_2 = "val2-val2-val2" SECRETS_SCAN_OMIT_PATHS = "netlify.toml,**/safefile.js" diff --git a/packages/build/tests/secrets_scanning/snapshots/tests.js.md b/packages/build/tests/secrets_scanning/snapshots/tests.js.md index 9870e69a83..1741624b54 100644 --- a/packages/build/tests/secrets_scanning/snapshots/tests.js.md +++ b/packages/build/tests/secrets_scanning/snapshots/tests.js.md @@ -1,4 +1,4 @@ -# Snapshot report for `tests/secrets_scanning/tests.js` +# Snapshot report for `packages/build/tests/secrets_scanning/tests.js` The actual snapshot is saved in `tests.js.snap`. @@ -6,44 +6,6 @@ Generated by [AVA](https://avajs.dev). ## secrets scanning, don't run when secrets are provided/default -> Snapshot 1 - - `␊ - Netlify Build ␊ - ────────────────────────────────────────────────────────────────␊ - ␊ - > Version␊ - @netlify/build 1.0.0␊ - ␊ - > Flags␊ - debug: false␊ - ␊ - > Current directory␊ - packages/build/tests/secrets_scanning/fixtures/src_default␊ - ␊ - > Config file␊ - packages/build/tests/secrets_scanning/fixtures/src_default/netlify.toml␊ - ␊ - > Context␊ - production␊ - ␊ - > Loading plugins␊ - - ./plugin@1.0.0 from netlify.toml␊ - ␊ - ./plugin (onPreBuild event) ␊ - ────────────────────────────────────────────────────────────────␊ - ␊ - undefined␊ - ␊ - (./plugin onPreBuild completed in 1ms)␊ - ␊ - Netlify Build Complete ␊ - ────────────────────────────────────────────────────────────────␊ - ␊ - (Netlify Build completed in 1ms)` - -## secrets scanning, don't run when there are no secrets - > Snapshot 1 `␊ diff --git a/packages/build/tests/secrets_scanning/snapshots/tests.js.snap b/packages/build/tests/secrets_scanning/snapshots/tests.js.snap index d68335cad150e9ed895b6bcc4665ea9878e8c7f5..ed1420e4a0004d021229ec7799e016c9679c29e3 100644 GIT binary patch delta 1575 zcmV+?2H5$}46+PAK~_N^Q*L2!b7*gLAa*kf0|27d@!9I{l5u&Mrx@tcM!_MO(^g`6 zvfH@uJ*CXWM!+A72mk;800003?OM-~KOcWO&CtUpFyfO}Y9XLU$nBWe;GoGU1omwG2{W zJ>>vUJWYCS2uY;SwC^WUBG*8f#6Syq?f}FjI3qn&sSQt~(#p%c0k~iON$l z@H2N=i-L!&2O*2_qilKEPFyVpQD%Qw<47B$QqUyOX2;GB9121i??D_TJ;u!@VI5pA z{rSuy0HG8E$QvB@fEB@COH88&hsgEnmk~Azrx-Gh)NxiOV9a0Wd_zpw3tf@0spTc<(gP(CzHM=^S_c?*4!7k>5Gm z-8<|&e`*iu0tWmy$ABLi2D}aoIBxYRQ{*%SLn4*&PmvMt@*IC7`ga>fyatTepy|>DjB7 z2gm+Top;^wtj4}(yw~ViMUPh1v!840+0@lzHW%S#I&e4f~virv0SR4P9+1QN_U-NZSr% zv!(4sL&P{j%gldm-ZRj-bZ7j7xZ0kTMMEE(k-6hlHpP5Gm7ihRPBt#Flj%mp0PQ3t zi*J40wrh>tt_s)$uc{j<2e)J&^}Moq39oF`hO1Q{>2mJ|B_p3y_0d*c+*=iZui@&T z)MH!KJ+|?%HtauC1TUSUUhKVmdw6_scyQG5cP|lz+qHiKV)IGuq2*CX%SJw`8iv~! z$3)q_>WnN6Lt4>!d)1v+&LE{QmX0iA)s0p~2b9vlg=Zycl7MUjs$M?nAb#PiN>uf2@@(Q~M*Et9?8)g)aV(XqYFs1jB zLV9hsYltkL$<{&zTq4@*^EQTUhkAWfq9D=~sH8Koe;?q`B~oNn8OsQX0TfDbh3b z*OCNvJ{@(<*f*#|G(=Mq7>pfApP5q(kkt$A#Z_O$OpjNR^%!qYjeubdCIufL*Qqh{ z8r<~%C-W^Eg_g}i%T}S~QK98=p=Ep0a`fuhPovmB**z?DoYAC`XM2mE`BUSzL!6vO ZO!eEJq8Of_pG}fS|9^+H6>$eO000AT3!VS~ delta 1604 zcmV-K2D|yP49^TdK~_N^Q*L2!b7*gLAa*kf0|1uQBdTK)ckQliPb+kby^>)TuA^VNTy zTlc>H^R{z?ACraM_RZ-(SCd)?dfjf@7453GM(q<0%^6iAAI1K6D?N9xo!& zR}KKhlce8;kVFbi`)(p7at)M8478AE4nRzTQ_@G3+VC_gt-Q=je(pvuWMiF3RGyN7 zpSjCg6g*^o2w8+5Wy{M};%YIBGQ)ow$J!W`f+m4BD|WVEUl7W8AL1zKGj0|MtKfR+ zqOM<_am_`qdkn80yBWw~*F=QO6<19_Un86|t!#F~X6c~rjP%S&> zw%Tsi;S1VoOGN}q5aq1*=t9=h1C(evkBgiFTehp=f|l)CwrdbJE!$llv$}t5w^wF6 zH4sTeL7lP~-ZMRzWO*lvhQr?Oo8Hl(f4IAS;P(!;clLYFpW012hXMaBFyM!l0j~oC z=B++uikw7dS6u-YmaNuf#lvhIghEaNjd2JBMIf9)3XVv0oM`t3M#SN{{RcUKFk)ox zc7re?eNd=Di!?f1$5|@bX<2`<40c^s>|MZ$v$bzbltOCBkqbaRf)SC*_@~T>cX@#k z{kttAUIRvKbHdN<+bAb&rRKau7&40ucVR#hg$4c^5_yUgh{~X{^=>9QT<}#K#`?@N zzqDi{Mj{xIC_x26ngaRE_9syUV#u`dgH0t1diLt&-jV-P@7-ZOtFeEt8Sl4xR@0+( z_3Y=ydNy@+?&fNRyR%$SxbN_@ZjzlNANj_4zC!I=zg}(jU$)=Y0rjG!CFh&w z(YoaPYcq0Q;4R}(mHIX`z8=ou=%B#SM=eKRD~>iE^hp%?r+B8WmYxfyTV}2o8;dsD z2&H5cLBs~AjDEY&J2-##kGJ3WYb*J$PRs5$FDmM?`!83;?yt9xew?LuLkDyIcDy?- zkUMLSn64+e$3zc46}wy8HR;{X;|5|lG+bUfYP$K+@jV1FtV(bVn4Z!^vtZS3E^s}{ zly;n@B1rAoQW?dCe6>I&JsfM4oJ5`p)*1p?c}ypXUc+X+_pn1+yXs%CeL9v~F(_oly2i*t8~{%=?X_dT<}kR+%%b^ zRla)BQdB;-mAm@TIa_deUMpPaY}lY)Qp+j#A$6f7Euly_h4nRHTxr4s+g*gSFwUZ9 zR8C?nEVgbgloNjuyFV#1?>|s!9S9UP1{!JCfo!(4ooI*{N9dTj&3gvAknZFc#MO4M z%p3aHj4W)gswvJVRK*zPTVsu`pb#>$aZ zth&~z>3~WaxbWE)4NM=+IT8>{`z8^Xs$o{G+tk!a5WAg}YDi$HZ@pmnUep2v0MhG)zRYSVy4F{$@+|Ur$)fA29tshk?Yi$c@1v*|H=85wNlG^sb!WA8+rMI?iZP#l4;R_x!1G+aXR)BBlo2Pf-j{(9b3*qW?cCs{;J>IRF4# C6b$tM diff --git a/packages/build/tests/secrets_scanning/tests.js b/packages/build/tests/secrets_scanning/tests.js index a642c37117..8e3e7d2a9e 100644 --- a/packages/build/tests/secrets_scanning/tests.js +++ b/packages/build/tests/secrets_scanning/tests.js @@ -6,11 +6,34 @@ test("secrets scanning, don't run when secrets are provided/default", async (t) t.snapshot(normalizeOutput(output)) }) -test("secrets scanning, don't run when there are no secrets", async (t) => { - const output = await new Fixture('./fixtures/src_default') - .withFlags({ debug: false, explicitSecretKeys: '' }) - .runWithBuild() - t.snapshot(normalizeOutput(output)) +test("secrets scanning, don't run when there are no secrets and enhanced scan not enabled", async (t) => { + const { requests } = await new Fixture('./fixtures/src_default') + .withFlags({ debug: false, explicitSecretKeys: '', deployId: 'test', token: 'test' }) + .runBuildServer({ path: '/api/v1/deploys/test/validations_report' }) + + t.true(requests.length === 0) +}) + +test("secrets scanning, don't run when no explicit secrets, enhanced scan enabled but no likely secrets", async (t) => { + const { requests } = await new Fixture('./fixtures/src_scanning_no_likely_enhanced_scan_secrets') + .withFlags({ debug: false, explicitSecretKeys: '', enhancedSecretScan: true, deployId: 'test', token: 'test' }) + .runBuildServer({ path: '/api/v1/deploys/test/validations_report' }) + + t.true(requests.length === 0) +}) + +test('secrets scanning, run and report result to API when there are no secrets and enhanced scan is enabled with likely secrets', async (t) => { + const { requests } = await new Fixture('./fixtures/src_scanning_likely_enhanced_scan_secrets') + .withFlags({ debug: false, explicitSecretKeys: '', enhancedSecretScan: true, deployId: 'test', token: 'test' }) + .runBuildServer({ path: '/api/v1/deploys/test/validations_report' }) + + t.true(requests.length === 1) + const request = requests[0] + t.is(request.method, 'PATCH') + t.is(request.url, '/api/v1/deploys/test/validations_report') + t.truthy(request.body.secrets_scan.scannedFilesCount) + t.truthy(request.body.secrets_scan.secretsScanMatches) + t.truthy(request.body.secrets_scan.enhancedSecretsScanMatches) }) test('secrets scanning, should skip with secrets but SECRETS_SCAN_ENABLED=false', async (t) => { @@ -20,6 +43,14 @@ test('secrets scanning, should skip with secrets but SECRETS_SCAN_ENABLED=false' t.snapshot(normalizeOutput(output)) }) +test('secrets scanning, should skip with enhanced scan but SECRETS_SCAN_ENABLED=false', async (t) => { + const { requests } = await new Fixture('./fixtures/src_scanning_likely_enhanced_scan_secrets_disabled') + .withFlags({ debug: false, explicitSecretKeys: '', enhancedSecretScan: true, deployId: 'test', token: 'test' }) + .runBuildServer({ path: '/api/v1/deploys/test/validations_report' }) + + t.true(requests.length === 0) +}) + test('secrets scanning, should skip when secrets passed but no env vars set', async (t) => { const output = await new Fixture('./fixtures/src_default') .withFlags({ debug: false, explicitSecretKeys: 'abc,DEF' }) @@ -27,6 +58,14 @@ test('secrets scanning, should skip when secrets passed but no env vars set', as t.snapshot(normalizeOutput(output)) }) +test('secrets scanning, should skip when enhanced scan enabled but no env vars set', async (t) => { + const { requests } = await new Fixture('./fixtures/src_default') + .withFlags({ debug: false, explicitSecretKeys: '', enhancedSecretScan: true, deployId: 'test', token: 'test' }) + .runBuildServer({ path: '/api/v1/deploys/test/validations_report' }) + + t.true(requests.length === 0) +}) + test('secrets scanning, should skip when secrets passed but no non-empty/trivial env vars set', async (t) => { const output = await new Fixture('./fixtures/src_scanning_env_vars_set_empty') .withFlags({ debug: false, explicitSecretKeys: 'ENV_VAR_1,ENV_VAR_,2ENV_VAR_3,ENV_VAR_4,ENV_VAR_5' }) @@ -34,6 +73,14 @@ test('secrets scanning, should skip when secrets passed but no non-empty/trivial t.snapshot(normalizeOutput(output)) }) +test('secrets scanning, should skip when enhanced scan enabled but no non-empty/trivial env vars set', async (t) => { + const { requests } = await new Fixture('./fixtures/src_scanning_env_vars_set_non_empty') + .withFlags({ debug: false, explicitSecretKeys: '', enhancedSecretScan: true, deployId: 'test', token: 'test' }) + .runBuildServer({ path: '/api/v1/deploys/test/validations_report' }) + + t.true(requests.length === 0) +}) + test('secrets scanning, should skip when secrets passed but SECRETS_SCAN_OMIT_KEYS omits all of them', async (t) => { const output = await new Fixture('./fixtures/src_scanning_omit_all_keys') .withFlags({ debug: false, explicitSecretKeys: 'ENV_VAR_1,ENV_VAR_2' }) @@ -41,6 +88,14 @@ test('secrets scanning, should skip when secrets passed but SECRETS_SCAN_OMIT_KE t.snapshot(normalizeOutput(output)) }) +test('secrets scanning, should skip when enhanced scan and likely secrets passed but SECRETS_SCAN_OMIT_KEYS omits all of them', async (t) => { + const { requests } = await new Fixture('./fixtures/src_scanning_omit_all_keys') + .withFlags({ debug: false, explicitSecretKeys: '', enhancedSecretScan: true, deployId: 'test', token: 'test' }) + .runBuildServer({ path: '/api/v1/deploys/test/validations_report' }) + + t.true(requests.length === 0) +}) + test('secrets scanning, should skip when secrets passed but SECRETS_SCAN_OMIT_PATHS omits all files', async (t) => { const output = await new Fixture('./fixtures/src_scanning_omit_all_paths') .withFlags({ debug: false, explicitSecretKeys: 'ENV_VAR_1,ENV_VAR_2' }) @@ -48,6 +103,14 @@ test('secrets scanning, should skip when secrets passed but SECRETS_SCAN_OMIT_PA t.snapshot(normalizeOutput(output)) }) +test('secrets scanning, should skip when enhanced scan and likely secrets passed but SECRETS_SCAN_OMIT_PATHS omits all files', async (t) => { + const { requests } = await new Fixture('./fixtures/src_scanning_omit_all_paths') + .withFlags({ debug: false, explicitSecretKeys: '', enhancedSecretScan: true, deployId: 'test', token: 'test' }) + .runBuildServer({ path: '/api/v1/deploys/test/validations_report' }) + + t.true(requests.length === 0) +}) + test('secrets scanning, should skip when secrets passed but SECRETS_SCAN_OMIT_PATHS omits globbed files', async (t) => { const output = await new Fixture('./fixtures/src_scanning_omit_glob_path') .withFlags({ debug: false, explicitSecretKeys: 'ENV_VAR_1,ENV_VAR_2' }) @@ -71,6 +134,19 @@ test('secrets scanning, should fail build when it finds secrets in the src and b t.snapshot(normalizeOutput(output)) }) +test('secrets scanning, should fail build when enhanced scan finds likely secret in the src and build output', async (t) => { + const output = await new Fixture('./fixtures/src_scanning_likely_enhanced_scan_secrets') + .withFlags({ debug: false, explicitSecretKeys: '', enhancedSecretScan: true }) + .runWithBuild() + + t.assert(normalizeOutput(output).includes(`Env var "ENV_VAR_1"'s value detected as a likely secret value`)) + t.assert( + normalizeOutput(output).includes( + `the build will fail until these secret values are not found in build output or repo files`, + ), + ) +}) + test('secrets scanning should report success to API when no secrets are found', async (t) => { const { requests } = await new Fixture('./fixtures/src_scanning_env_vars_no_matches') .withFlags({ @@ -87,6 +163,26 @@ test('secrets scanning should report success to API when no secrets are found', t.is(request.url, '/api/v1/deploys/test/validations_report') t.truthy(request.body.secrets_scan.scannedFilesCount) t.truthy(request.body.secrets_scan.secretsScanMatches) + t.truthy(request.body.secrets_scan.enhancedSecretsScanMatches) +}) + +test('secrets scanning, should report success to API when enhanced scans finds no likely secrets', async (t) => { + const { requests } = await new Fixture('./fixtures/src_scanning_env_vars_no_matches') + .withFlags({ + debug: false, + enhancedSecretScan: true, + deployId: 'test', + token: 'test', + }) + .runBuildServer({ path: '/api/v1/deploys/test/validations_report' }) + + t.true(requests.length === 1) + const request = requests[0] + t.is(request.method, 'PATCH') + t.is(request.url, '/api/v1/deploys/test/validations_report') + t.truthy(request.body.secrets_scan.scannedFilesCount) + t.truthy(request.body.secrets_scan.secretsScanMatches) + t.truthy(request.body.secrets_scan.enhancedSecretsScanMatches) }) test('secrets scanning failure should produce an user error', async (t) => { @@ -101,6 +197,18 @@ test('secrets scanning failure should produce an user error', async (t) => { t.is(severityCode, 2) }) +test('secrets scanning, enhanced scanning failure should produce a user error', async (t) => { + const { severityCode } = await new Fixture('./fixtures/src_scanning_likely_enhanced_scan_secrets') + .withFlags({ + debug: false, + explicitSecretKeys: '', + enhancedSecretScan: true, + }) + .runBuildProgrammatic() + // Severity code of 2 is user error + t.is(severityCode, 2) +}) + test('secrets scanning should report failure to API when secrets are found', async (t) => { const { requests } = await new Fixture('./fixtures/src_scanning_env_vars_set_non_empty') .withFlags({ @@ -118,6 +226,7 @@ test('secrets scanning should report failure to API when secrets are found', asy t.is(request.url, '/api/v1/deploys/test/validations_report') t.truthy(request.body.secrets_scan.scannedFilesCount) t.truthy(request.body.secrets_scan.secretsScanMatches) + t.truthy(request.body.secrets_scan.enhancedSecretsScanMatches) }) test('secrets scan does not send report to API for local builds', async (t) => { From 9413228223ed5fb8c5b720221f8d29c5b5fede70 Mon Sep 17 00:00:00 2001 From: Suzanne Aitchison Date: Thu, 8 May 2025 10:50:19 +0100 Subject: [PATCH 04/10] feat: small tweaks to documentationand logging --- packages/build/docs/flow.md | 4 ++-- packages/build/src/log/messages/core_steps.js | 2 ++ packages/build/src/plugins_core/secrets_scanning/index.ts | 3 +-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/build/docs/flow.md b/packages/build/docs/flow.md index eed2be0c84..7ec5793fa5 100644 --- a/packages/build/docs/flow.md +++ b/packages/build/docs/flow.md @@ -95,8 +95,8 @@ Current core steps are: [build command](https://github.com/netlify/build/blob/9b261a6182e1ba6853966bd8d0bde9064209af7d/packages/build/src/plugins_core/build_command.js#L11). - [Functions bundling](https://github.com/netlify/build/blob/9b261a6182e1ba6853966bd8d0bde9064209af7d/packages/build/src/plugins_core/functions/index.js#L142), which uses [`zip-it-and-ship-it`](https://github.com/netlify/zip-it-and-ship-it). -- Secrets Scanning only runs when the account passes non-empty explicitSecretKeys, or when the account has enhanced - secret scanning enabled (only applies to certain plan types). +- Secrets Scanning only runs when the account passes non-empty explicitSecretKeys, or when the account has + enhancedSecretScan enabled (only applies to certain plan types). - [Site deploy](https://github.com/netlify/build/blob/9b261a6182e1ba6853966bd8d0bde9064209af7d/packages/build/src/plugins_core/deploy/index.js#L66): - This sends a network request to the buildbot to initiate site deploy. - When the buildbot diff --git a/packages/build/src/log/messages/core_steps.js b/packages/build/src/log/messages/core_steps.js index af80b1830c..08814046ed 100644 --- a/packages/build/src/log/messages/core_steps.js +++ b/packages/build/src/log/messages/core_steps.js @@ -136,6 +136,7 @@ export const logSecretsScanFailBuildMessage = function ({ logs, scanResults, gro `Scanning complete. ${scanResults.scannedFilesCount} file(s) scanned. Secrets scanning found ${scanResults.matches.length} instance(s) of secrets in build output or repo code.\n`, ) + // Explicit secret matches if (Object.keys(secretMatches).length > 0) { Object.keys(secretMatches).forEach((key) => { logError(logs, `Secret env var "${key}"'s value detected:`) @@ -150,6 +151,7 @@ export const logSecretsScanFailBuildMessage = function ({ logs, scanResults, gro }) } + // Likely secret matches from enhanced scan if (Object.keys(enhancedSecretMatches).length > 0) { Object.keys(enhancedSecretMatches).forEach((key) => { logError(logs, `Env var "${key}"'s value detected as a likely secret value:`) diff --git a/packages/build/src/plugins_core/secrets_scanning/index.ts b/packages/build/src/plugins_core/secrets_scanning/index.ts index 4e657c3c11..946791df2a 100644 --- a/packages/build/src/plugins_core/secrets_scanning/index.ts +++ b/packages/build/src/plugins_core/secrets_scanning/index.ts @@ -58,8 +58,7 @@ const coreStep: CoreStepFunction = async function ({ const keysToSearchFor = explicitSecretKeysToScanFor.concat(otherKeysToScanFor) if (keysToSearchFor.length === 0) { - const skippedTheEnhancedScan = enhancedSecretScan && otherKeysToScanFor.length > 0 - const msg = skippedTheEnhancedScan + const msg = enhancedSecretScan ? 'Secrets scanning skipped because no env vars are set to non-empty/non-trivial values or they are all omitted with SECRETS_SCAN_OMIT_KEYS env var setting.' : 'Secrets scanning skipped because no env vars marked as secret are set to non-empty/non-trivial values or they are all omitted with SECRETS_SCAN_OMIT_KEYS env var setting.' logSecretsScanSkipMessage(logs, msg) From e1f55830b5f33f366272ac24036f3c88ca92748d Mon Sep 17 00:00:00 2001 From: Suzanne Aitchison Date: Fri, 9 May 2025 10:28:18 +0100 Subject: [PATCH 05/10] feat: apply review suggestions --- packages/build/src/log/messages/core_steps.js | 48 +++++++++---------- .../plugins_core/secrets_scanning/utils.ts | 45 +++++++++-------- 2 files changed, 46 insertions(+), 47 deletions(-) diff --git a/packages/build/src/log/messages/core_steps.js b/packages/build/src/log/messages/core_steps.js index 08814046ed..3092c9f27e 100644 --- a/packages/build/src/log/messages/core_steps.js +++ b/packages/build/src/log/messages/core_steps.js @@ -137,34 +137,30 @@ export const logSecretsScanFailBuildMessage = function ({ logs, scanResults, gro ) // Explicit secret matches - if (Object.keys(secretMatches).length > 0) { - Object.keys(secretMatches).forEach((key) => { - logError(logs, `Secret env var "${key}"'s value detected:`) - - secretMatches[key] - .sort((a, b) => { - return a.file > b.file ? 0 : 1 - }) - .forEach(({ lineNumber, file }) => { - logError(logs, `found value at line ${lineNumber} in ${file}`, { indent: true }) - }) - }) - } + Object.keys(secretMatches).forEach((key) => { + logError(logs, `Secret env var "${key}"'s value detected:`) + + secretMatches[key] + .sort((a, b) => { + return a.file > b.file ? 0 : 1 + }) + .forEach(({ lineNumber, file }) => { + logError(logs, `found value at line ${lineNumber} in ${file}`, { indent: true }) + }) + }) // Likely secret matches from enhanced scan - if (Object.keys(enhancedSecretMatches).length > 0) { - Object.keys(enhancedSecretMatches).forEach((key) => { - logError(logs, `Env var "${key}"'s value detected as a likely secret value:`) - - enhancedSecretMatches[key] - .sort((a, b) => { - return a.file > b.file ? 0 : 1 - }) - .forEach(({ lineNumber, file }) => { - logError(logs, `found value at line ${lineNumber} in ${file}`, { indent: true }) - }) - }) - } + Object.keys(enhancedSecretMatches).forEach((key) => { + logError(logs, `Env var "${key}"'s value detected as a likely secret value:`) + + enhancedSecretMatches[key] + .sort((a, b) => { + return a.file > b.file ? 0 : 1 + }) + .forEach(({ lineNumber, file }) => { + logError(logs, `found value at line ${lineNumber} in ${file}`, { indent: true }) + }) + }) logError( logs, diff --git a/packages/build/src/plugins_core/secrets_scanning/utils.ts b/packages/build/src/plugins_core/secrets_scanning/utils.ts index d1313544bc..8c4d27650b 100644 --- a/packages/build/src/plugins_core/secrets_scanning/utils.ts +++ b/packages/build/src/plugins_core/secrets_scanning/utils.ts @@ -42,38 +42,41 @@ export function isSecretsScanningEnabled(env: Record): boolean } function filterOmittedKeys(env: Record, envKeys: string[] = []): string[] { - let omitKeys: string[] = [] - if (typeof env.SECRETS_SCAN_OMIT_KEYS === 'string') { - omitKeys = env.SECRETS_SCAN_OMIT_KEYS.split(',') - .map((s) => s.trim()) - .filter(Boolean) + if (typeof env.SECRETS_SCAN_OMIT_KEYS !== 'string') { + return envKeys } + const omitKeys = env.SECRETS_SCAN_OMIT_KEYS.split(',') + .map((s) => s.trim()) + .filter(Boolean) return envKeys.filter((key) => !omitKeys.includes(key)) } /** - * non-trivial values are values that are: - * - >4 characters/digits - * - not booleans + * Trivial values are values that are: + * - empty or short strings + * - string forms of booleans + * - booleans + * - numbers or objects with fewer than 4 chars */ -function isValueNonTrivial(val): boolean { +function isValueTrivial(val): boolean { if (typeof val === 'string') { // string forms of booleans if (val === 'true' || val === 'false') { - return false + return true } - - // non-trivial/non-empty values only - return val.trim().length > 4 - } else if (typeof val === 'boolean') { - // booleans are trivial values - return false - } else if (typeof val === 'number' || typeof val === 'object') { - return JSON.stringify(val).length > 4 + // trivial values are empty or short strings + return val.trim().length < 4 + } + if (typeof val === 'boolean') { + // booleans are always considered trivial + return true + } + if (typeof val === 'number' || typeof val === 'object') { + return JSON.stringify(val).length < 4 } - return !!val + return !val } /** @@ -90,7 +93,7 @@ function isValueNonTrivial(val): boolean { */ export function getSecretKeysToScanFor(env: Record, secretKeys: string[]): string[] { const filteredSecretKeys = filterOmittedKeys(env, secretKeys) - return filteredSecretKeys.filter((key) => isValueNonTrivial(env[key])) + return filteredSecretKeys.filter((key) => !isValueTrivial(env[key])) } /** @@ -108,7 +111,7 @@ export function getNonSecretKeysToScanFor(env: Record, secretKe const nonSecretKeysToScanFor = filteredNonSecretKeys.filter((key) => { const val = env[key] - if (!isValueNonTrivial(val)) { + if (isValueTrivial(val)) { return false } return isLikelySecretValue(val) From 981aff68ece5ba00b6be64f5dde718cd08b7dfdc Mon Sep 17 00:00:00 2001 From: Suzanne Aitchison Date: Fri, 9 May 2025 10:37:09 +0100 Subject: [PATCH 06/10] feat: rename --- packages/build/src/plugins_core/secrets_scanning/index.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/build/src/plugins_core/secrets_scanning/index.ts b/packages/build/src/plugins_core/secrets_scanning/index.ts index 946791df2a..08292f5cca 100644 --- a/packages/build/src/plugins_core/secrets_scanning/index.ts +++ b/packages/build/src/plugins_core/secrets_scanning/index.ts @@ -54,8 +54,8 @@ const coreStep: CoreStepFunction = async function ({ } const explicitSecretKeysToScanFor = getSecretKeysToScanFor(envVars, passedSecretKeys) - const otherKeysToScanFor = enhancedSecretScan ? getNonSecretKeysToScanFor(envVars, passedSecretKeys) : [] - const keysToSearchFor = explicitSecretKeysToScanFor.concat(otherKeysToScanFor) + const potentialSecretKeysToScanFor = enhancedSecretScan ? getNonSecretKeysToScanFor(envVars, passedSecretKeys) : [] + const keysToSearchFor = explicitSecretKeysToScanFor.concat(potentialSecretKeysToScanFor) if (keysToSearchFor.length === 0) { const msg = enhancedSecretScan @@ -94,7 +94,7 @@ const coreStep: CoreStepFunction = async function ({ }) secretMatches = scanResults.matches.filter((match) => match.key in explicitSecretKeysToScanFor) - enhancedSecretMatches = scanResults.matches.filter((match) => match.key in otherKeysToScanFor) + enhancedSecretMatches = scanResults.matches.filter((match) => match.key in potentialSecretKeysToScanFor) const attributesForLogsAndSpan = { secretsScanFoundSecrets: secretMatches.length > 0, @@ -133,7 +133,7 @@ const coreStep: CoreStepFunction = async function ({ logSecretsScanFailBuildMessage({ logs, scanResults, - groupedResults: groupScanResultsByKeyAndScanType(scanResults, otherKeysToScanFor), + groupedResults: groupScanResultsByKeyAndScanType(scanResults, potentialSecretKeysToScanFor), }) const error = new Error(`Secrets scanning found secrets in build.`) From 8f74b1068cb5f99130c45d512eb40ea36fff37d3 Mon Sep 17 00:00:00 2001 From: Suzanne Aitchison Date: Mon, 12 May 2025 11:56:07 +0100 Subject: [PATCH 07/10] NO MERGE: test version bump --- packages/build/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/build/package.json b/packages/build/package.json index 560863c4c7..e05c75af39 100644 --- a/packages/build/package.json +++ b/packages/build/package.json @@ -1,6 +1,6 @@ { "name": "@netlify/build", - "version": "32.1.2", + "version": "32.1.3-suzanne-test", "description": "Netlify build module", "type": "module", "exports": "./lib/index.js", From 7dced541abc40bc1e0659bd5d0465bb7628494dc Mon Sep 17 00:00:00 2001 From: Suzanne Aitchison Date: Tue, 13 May 2025 14:03:48 +0100 Subject: [PATCH 08/10] fix: report matches to api correctly --- packages/build/package.json | 2 +- .../plugins_core/secrets_scanning/index.ts | 4 +- .../secrets_scanning/snapshots/tests.js.md | 3 +- .../secrets_scanning/snapshots/tests.js.snap | Bin 1586 -> 1602 bytes .../build/tests/secrets_scanning/tests.js | 51 +++++++++--------- 5 files changed, 29 insertions(+), 31 deletions(-) diff --git a/packages/build/package.json b/packages/build/package.json index e05c75af39..560863c4c7 100644 --- a/packages/build/package.json +++ b/packages/build/package.json @@ -1,6 +1,6 @@ { "name": "@netlify/build", - "version": "32.1.3-suzanne-test", + "version": "32.1.2", "description": "Netlify build module", "type": "module", "exports": "./lib/index.js", diff --git a/packages/build/src/plugins_core/secrets_scanning/index.ts b/packages/build/src/plugins_core/secrets_scanning/index.ts index 08292f5cca..9341aafd04 100644 --- a/packages/build/src/plugins_core/secrets_scanning/index.ts +++ b/packages/build/src/plugins_core/secrets_scanning/index.ts @@ -93,8 +93,8 @@ const coreStep: CoreStepFunction = async function ({ filePaths, }) - secretMatches = scanResults.matches.filter((match) => match.key in explicitSecretKeysToScanFor) - enhancedSecretMatches = scanResults.matches.filter((match) => match.key in potentialSecretKeysToScanFor) + secretMatches = scanResults.matches.filter((match) => explicitSecretKeysToScanFor.includes(match.key)) + enhancedSecretMatches = scanResults.matches.filter((match) => potentialSecretKeysToScanFor.includes(match.key)) const attributesForLogsAndSpan = { secretsScanFoundSecrets: secretMatches.length > 0, diff --git a/packages/build/tests/secrets_scanning/snapshots/tests.js.md b/packages/build/tests/secrets_scanning/snapshots/tests.js.md index 1741624b54..ec9cf7f6f0 100644 --- a/packages/build/tests/secrets_scanning/snapshots/tests.js.md +++ b/packages/build/tests/secrets_scanning/snapshots/tests.js.md @@ -247,10 +247,11 @@ Generated by [AVA](https://avajs.dev). ────────────────────────────────────────────────────────────────␊ ␊ > Version␊ - @netlify/build 1.0.0␊ + @netlify/build 1.0.0-test␊ ␊ > Flags␊ debug: false␊ + deployId: test␊ ␊ > Current directory␊ packages/build/tests/secrets_scanning/fixtures/src_scanning_env_vars_set_non_empty␊ diff --git a/packages/build/tests/secrets_scanning/snapshots/tests.js.snap b/packages/build/tests/secrets_scanning/snapshots/tests.js.snap index ed1420e4a0004d021229ec7799e016c9679c29e3..dac3ff4ce6dacd78129521929c2efe1f7ce52143 100644 GIT binary patch delta 1600 zcmV-G2EX~T48jawK~_N^Q*L2!b7*gLAa*kf0|1wlKkA(I|GPyvOt#iz?%-s+12xUJ zQo38Z=?EUQS~VYw2mk;800003?OH)^+cp&Lwsr$_>vgAxVn7ohk?JIEIs+}5q%M{% zO^YUxUlxCU!G6u5uB(${z1)jgySa) z1BqIJ3JB+n_g8=leqV!3IJ_Sqhr9!k2(c6+Mv;0HLy|febQgYc?w4;6%*V6c*avSykm<- z$9>>FFg;#Gq^}$RipNR63n7UVn)aPUO5_?SlNe|rPaS}m1Sh1ADz)KhR9bnNm;Bs~ zUdYBek*GW+13z<@wJ3PV`Vg`RKgyPut;E%07-fbvjq#rI*hv0uV|ugrdQ`2P_HxT4EYKI6|(MKaa3UIK_~0q>i&R0b>S>Kn&vu zHBw+4IzzSWoY`tSS%=SPt1T4~C_$97-fo%o)IcN=1$Dw=c+d1;!h6SwhJ)VDo8IAp zf3UN)@Avk%w)c9^pV&=0hXMaBFyM!l0k3}p1LmzhWr`d}W>;MT7nZEnWW|GQ9E3tn z0*!G91Vtd6LJE#Zbew4S21dlenf(VjfG}cY?skJPB7IP(L5nmxT*p}|*=bp^40c^s z?484kv)N!wltOCBkqbaRf)SC*_@~T>U-JSZ`d_zVaR_h zHr#~)NfZ|NYe?h?QXndW%2vIZ=y1VTaTx1U&-~JojTnhwM4|*02nl+Cr?x+dA`nBS zjUQ|(Sz|2&3M1nvzi{Qt7ktq*0ZUrb2nEj+@0lu!hL(ERk&-Q zaFgsD`N%iU^A&2}`t@?N|FZqI4yb<@B`rDMG>_IL=UO{ zL&N2zqo$i59p6C^!>R<=faxhsGz(Vk<^tEFOlikiDuUFWEtOGR$X5$g(!;Sv$w}my zV67pLdVogNq?O9lJwi^25Z;^H=LlZEb8%h8d%IBaLc*eKCs^HM8bZdYn%yya zfS_bh+B9XQyZf%7bU)s0mF|B$rCT+6vQX~KWgLmqUD}1b(ais|lY zu-Lk}d`_h8-lX)rcVDH|AyCv9Xrx^Svf0vhq9I}&p=0JYZy)GF5|n>m5Les1GH>W3 zGqSL~s-`%bW)){xw34+8tYo?nF+@8_#lv?#ZrP1SZq@~Cf>+&zRD)Zwj%Hq2zkpXZ z8pG9Tj&!wmgNl)l>-uP;Dej#bz&CJpQ0cLa`W{<*&=~gbYl4?f(Jb~}zTG?A-P_&o z`CAu=!p+73vHrO6(CU9Eq*Wsy)(ykW^JAiFT}?(-h9Rx#yzTnVt7ec&7%N9svFci< zrUNQz;M`|hG%$TM=TJZ_?fX4|<5(zLA*ZRqiY=M4s*0I<*i=~w*DNx36y~SM*&%US zLhI~N7UdQ8AzWuS%xsuZIF7A*R=^bG3(D)21qaP5#`N6R8TEg3ZD|15qWp!BFccDj z@lePiv8M-OdR979T%tW4fV}#9E&|nsf!49=dLGx28=f*Ru;oP}aW%8Bnp>Fn4rUgJ zv*}lQ%s>-qI;6SqjY(VqdQuv|Q7O_l_1A&~^=vxonz3(CiD-nTCNLN~klr)97$B<` zy7Q}wnI5ks>oZv1of-kd8cYg4M6Od~=H2)=DkwrIw9S%fnL3qf*P}q-For yp`S*vf3&q%>NulG75BF1-}9%&ZHG8Hj+h#Buc8>9pr1`rME`#}fp8BUH2?q=9}M^a delta 1584 zcmV-02G9Az46+PgK~_N^Q*L2!b7*gLAa*kf0|27d@!9I{l5u&Mrx@tcM!_MO(^g`6 zvfH@uJ*CXWM!+A72mk;800003?OM-o+cp&LwzdOw>vgAxVn7ohk?JIEIs*-wq%IaL zO^Z5_UlxD)tOp^B@S|*b*-l(722o~M<47B$QqUyOX2;GB9121i??D_T zJ;r~{CSe_1Fa7zX#8V38xq`j?{5hCSc595r{z? zp+*XfLu;T`ow=j7pLO_xj@n8dfdWKh);lP&p6ZJvqM*)L4DXrlPk8S%(a`Pezv&!z z{qFwmk>5Gm-8<|&e`*iu0tWmy$ABLi2E2a`3^;D}DO2P$GNt2ty*3@lTNv@A4cY`ga>fyatTeKc5`{Vb8W4Gg6o|^8vQ=*;I$ZE|9K`zEGk#>)F)RV>g#8+?~~&!hLtAQMhZM zaFgsD`Or7c^A&2}`t@qF|Dye-4yb>G>=v#=U?lQ^Ac|vkE+zSq4D)#0Y?Wp zjy`TU`dV?c@t{wl$Unn#b+z=IGu<$Awb)p8(1s`_qX;54KxOpXhR)H6f3o|=-&h;} z)oR%N=0!zScK_w7*!}hH@sG3gZs)O|FnCaqMa?h$fIgz(-xzCiHOB{hmy&&5?0@9k2>3ki#|lVEj^X$TpoYWBqF zK7x`#;m{P7?%sY*>9+1QN_T&qlrDD_#0CH4*G-c-8s)1NEqUd08@a0vowEbod984v zwPk~PMXjdXht!3Fw1gtz6gD@2ais|lYWqIZ4MSSdd3)8JSI!`%FqV!iW7UmTMF*79z=dmDHZWb9b1Wd1 z_DvqZQ7n|Lh11kv%@#*l{lrY=YwD?lYZjR&3di@w*>!MQEbDA3%km1l2-i6XGaF_U zj$-ScH87?3l0tfI$(8b&F}?7`MLpeH8UVKZejy|bghXIG6mow+>^*^)-ipqYk!bG$ zU|i=t7lCR+U+Y-4J&)_i4bPbu*zzKgxSm;9A6r=Twq+KGv*}lQp+FO9I;6SqjY(Vq zI#L?IQ7O_h_1BUFbv_++&Db}nL^MQG6BvvgNS~Qg43O0e?Zs8aOpjNR^%!qYjeubd zCIufL*Qqh{8r(?q|0nY;8- { - const output = await new Fixture('./fixtures/src_scanning_env_vars_set_non_empty') +test('secrets scanning, should fail build and report to API when it finds secrets in the src and build output', async (t) => { + const { output, requests } = await new Fixture('./fixtures/src_scanning_env_vars_set_non_empty') .withFlags({ debug: false, explicitSecretKeys: 'ENV_VAR_MULTILINE_A,ENV_VAR_1,ENV_VAR_2,ENV_VAR_3,ENV_VAR_4,ENV_VAR_5,ENV_VAR_6,ENV_VAR_MULTILINE_B', + deployId: 'test', + token: 'test', }) - .runWithBuild() + .runBuildServer({ path: '/api/v1/deploys/test/validations_report' }) t.snapshot(normalizeOutput(output)) + + t.true(requests.length === 1) + const request = requests[0] + t.is(request.method, 'PATCH') + t.is(request.url, '/api/v1/deploys/test/validations_report') + t.truthy(request.body.secrets_scan.scannedFilesCount) + t.is(request.body.secrets_scan.secretsScanMatches.length, 32) + t.is(request.body.secrets_scan.enhancedSecretsScanMatches.length, 0) }) -test('secrets scanning, should fail build when enhanced scan finds likely secret in the src and build output', async (t) => { - const output = await new Fixture('./fixtures/src_scanning_likely_enhanced_scan_secrets') - .withFlags({ debug: false, explicitSecretKeys: '', enhancedSecretScan: true }) - .runWithBuild() +test('secrets scanning, should fail build and report to API when enhanced scan finds likely secret in the src and build output', async (t) => { + const { output, requests } = await new Fixture('./fixtures/src_scanning_likely_enhanced_scan_secrets') + .withFlags({ debug: false, explicitSecretKeys: '', enhancedSecretScan: true, deployId: 'test', token: 'test' }) + .runBuildServer({ path: '/api/v1/deploys/test/validations_report' }) t.assert(normalizeOutput(output).includes(`Env var "ENV_VAR_1"'s value detected as a likely secret value`)) t.assert( @@ -145,6 +155,13 @@ test('secrets scanning, should fail build when enhanced scan finds likely secret `the build will fail until these secret values are not found in build output or repo files`, ), ) + t.true(requests.length === 1) + const request = requests[0] + t.is(request.method, 'PATCH') + t.is(request.url, '/api/v1/deploys/test/validations_report') + t.truthy(request.body.secrets_scan.scannedFilesCount) + t.is(request.body.secrets_scan.secretsScanMatches.length, 0) + t.is(request.body.secrets_scan.enhancedSecretsScanMatches.length, 1) }) test('secrets scanning should report success to API when no secrets are found', async (t) => { @@ -209,26 +226,6 @@ test('secrets scanning, enhanced scanning failure should produce a user error', t.is(severityCode, 2) }) -test('secrets scanning should report failure to API when secrets are found', async (t) => { - const { requests } = await new Fixture('./fixtures/src_scanning_env_vars_set_non_empty') - .withFlags({ - debug: false, - explicitSecretKeys: - 'ENV_VAR_MULTILINE_A,ENV_VAR_1,ENV_VAR_2,ENV_VAR_3,ENV_VAR_4,ENV_VAR_5,ENV_VAR_6,ENV_VAR_MULTILINE_B', - deployId: 'test', - token: 'test', - }) - .runBuildServer({ path: '/api/v1/deploys/test/validations_report' }) - - t.true(requests.length === 1) - const request = requests[0] - t.is(request.method, 'PATCH') - t.is(request.url, '/api/v1/deploys/test/validations_report') - t.truthy(request.body.secrets_scan.scannedFilesCount) - t.truthy(request.body.secrets_scan.secretsScanMatches) - t.truthy(request.body.secrets_scan.enhancedSecretsScanMatches) -}) - test('secrets scan does not send report to API for local builds', async (t) => { const { requests } = await new Fixture('./fixtures/src_scanning_env_vars_set_non_empty') .withFlags({ From 9196b925c5f2ccdaa4dd6363dc7fbaf67929dc88 Mon Sep 17 00:00:00 2001 From: Suzanne Aitchison Date: Tue, 13 May 2025 14:40:35 +0100 Subject: [PATCH 09/10] fix: missing snapshot --- .../secrets_scanning/snapshots/tests.js.md | 4 ++-- .../secrets_scanning/snapshots/tests.js.snap | Bin 1602 -> 1608 bytes 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/build/tests/secrets_scanning/snapshots/tests.js.md b/packages/build/tests/secrets_scanning/snapshots/tests.js.md index ec9cf7f6f0..1b22fc3f62 100644 --- a/packages/build/tests/secrets_scanning/snapshots/tests.js.md +++ b/packages/build/tests/secrets_scanning/snapshots/tests.js.md @@ -238,7 +238,7 @@ Generated by [AVA](https://avajs.dev). ␊ (Netlify Build completed in 1ms)` -## secrets scanning, should fail build when it finds secrets in the src and build output +## secrets scanning, should fail build and report to API when it finds secrets in the src and build output > Snapshot 1 @@ -247,7 +247,7 @@ Generated by [AVA](https://avajs.dev). ────────────────────────────────────────────────────────────────␊ ␊ > Version␊ - @netlify/build 1.0.0-test␊ + @netlify/build 1.0.0␊ ␊ > Flags␊ debug: false␊ diff --git a/packages/build/tests/secrets_scanning/snapshots/tests.js.snap b/packages/build/tests/secrets_scanning/snapshots/tests.js.snap index dac3ff4ce6dacd78129521929c2efe1f7ce52143..071e52035e2202770b3a1d153d907d93b4b2dc91 100644 GIT binary patch delta 1601 zcmV-H2EO^i49E;WK~_N^Q*L2!b7*gLAa*kf0|2qypcZuv6|Lu!7N3pApdOCoyjIRl z`yoxcf9qk(nF=3^2mk;800003?OM-~KO28$EKQ5X%h1CnFyfOiby!|4DyOga#W5KA#)6sbosBuS*1;v`lBq1EpnC?%S_zx(%BcfNnQ zV?JNpyL11`KkqsRs3R7RVK-qBh0A`N|Ni>zlm0r+b2vh&nBWe;3!b9k9a}V7?j!e+ z>G3KeedPd9JWl#;2uY;SwC^QSBG*8f#6Sx2qilKEN?d;}hEZl%gHXt6pfL`C zpa_IhNWl?_juY+P!iYFLxBp-QAdDDUxZNO(NFNkx&?1cv*Kt+~b{bYJf?bsrI~TCx zd^Q*prI1>3MxQ4R>Kc z5`{Vb8WMSe6o|^8vQ=*;I$ZE|9L9R=nO|D65hD?dNR*%gAwdstZ2OZa0x@LT_`#-< zIX!#xy4&-A?7Tmm%xdgw#`}$)RrF|8J^Q(~p3Pi6adWxC-CNBm+_!)C8il(73OCKp zk&k@iJYS*qtzWM;`!CvW>VRrd(vb6Q^JrCa{TmVq@7x8=;hpB8b=kmCxRd``ew~5A%QYZskvnUSm~JMy$3zdVi`}j5iu7*haRV_N8ZIv! zHQoN`_zr>?)+M+GOiyW|S+H(57q}i}N;}R<9;D`MsfglIzFMM^9-e8GoJ5`p)(Qft z2WV7HTB%IkBjk(-;ndu|K=9H>Y80=Yi>oSLf2rbyghkm-FnfPOVl1T1YIfdsQzc{? zLdL0@-$QzUpkz?kMn&zr-^*!V=YFGowYAS&#*r9zX&a0RUHfSi7yQ~!q^5H;;#e(w z^5W<=idY>w=Q$jnmlGFSTQ;ay)M`qPNL?tXPbd;jVRHi*SDNs^b{FA1jPvLjAv9cS zv2`a}I@P%c(>j0j!9$gnia=3gppmv6$Yx91iH3-AgqE4xyo#W68PVi|xZ3WOMMIyM zk-6>WnN6Lt4>!yVadn&LE{QmX0iA)s0p~ z2b9vlh0nHZVESlIPe3f~%RYc-u~4>LPE&(5TP>x@!6`Gvu_?I{u32R6D4ZNAXD7yK zJ*~4xS(blS*oSbPZkX9Hqwp-Y?pXshwWR@IEA&@F!ca&A z#zP^8#2zJx*}>^trHS^a049asOA)9x4783_+w-`N+;Ggiz?K(@#P!U=`ozMb7c#Rz zoKL^fGY6VT(;>};Z%yI~(2>#rj!KcfslS#asONdpQP+%phe|{vG&O<2*n#w(*~I`^ zz0h7GjGbx{$e@bvQcQ+EVOJDS{@f#o)lWPr!5C> zdVU(k{?X2Uq2rt;mE7B1e9xa5w;kf-IAUthzK&veihedp9{v9T8R|u}Dm4HAo>>(! delta 1595 zcmV-B2E_Tu48jaQK~_N^Q*L2!b7*gLAa*kf0|1wlKkA(I|GPyvOt#iz?%-s+12xUJ zQo38Z=?EUQS~VYw2mk;800003?OH*RKO29RElrCi&CtUpFyfOTK)ckQliPb+kby`>zjXD z=JUm!TX(VaO~t+=T&2 z6c+evNaP7pAS#2(R=t_%aKTq`80%Bd{L+$*7>Qs+q68HP33`C1wm*p?5JRSoA8aaF z(6d)BcMtuadhZVMS&eW%S#H-u{t)wDrbcTgiWQT6VvA zQBjxOf4MAnf4z11!z_Qj8#+*Pj+n*;a%b%k)Ac0xnCQV(vAeZhliuw-ZXkw3!{w!; zrkfuf-$4+=ssz`7=_yS#3s&vs0@tHVX~$VAg4CWZl~G*CR|{0q!?8xmN#vPets#(l zfJW7%mCDpTLQaVg-kaO!2wwU~t>QIvab3lGyHN2$!lG;^SlxeP8bZdYn%yyafS_bh z+B9XQyZf%7bU)s0mF_&HTQz#JQ0~lS9EsCi+J(I1l^#HG!LR%fYBEQwhxKBpsE2MV zg7u+up2ERd1#zLXVS{=}EvJl#)P<7Zgd*V-*4Kb>r3nvgcM;CQIE$XqK8dlg*t)rV zPNePLr1ZRZU!{N5AyCv9Xrx^Svf0vhq9I}&p=0JYZy)GF5|m#MSKGZZZ|Ea4var3X zrZ}5s6=ztqlC=x0WV#SBL_0~v!*@Pz*^NeS)&*>WSKWnFgIlqVW?ospfLAse!_{ey zbhUSbijj}&`e>sm?wuOIH*j@O>9LLa9$S0R820aLf|q|z(Jb~}zTG?A-P_&o`CAu= z!p+73vHrO6(CR3pRU;qP4a3dzW1?zZO-5FRA+71W?fTBEW{^r4D@Rtb>RP9!11f3Y z+-F-fFnu)VP(Up0`#pf;SSVW|r>Vh;Et#^aikW)YR9OkvEHZZ#=BLQnA#qwl>+De$ zoeY+8Ue!^ObR|ku2W;?<+$m;DrZ~PN-gW9mW@)&!&1wmQp@I~W&hQopGL8N tw6$02IHO4w_qONX^QXpbhd4Qom>P7iq8Of_pG{Ik|9?7xa1R|d007B$4AcMs From e9fd6fc05c4f2922dce82477022b84317ac7fc64 Mon Sep 17 00:00:00 2001 From: Suzanne Aitchison Date: Tue, 13 May 2025 14:41:14 +0100 Subject: [PATCH 10/10] fix: undo formatting change --- packages/build/tests/secrets_scanning/snapshots/tests.js.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/build/tests/secrets_scanning/snapshots/tests.js.md b/packages/build/tests/secrets_scanning/snapshots/tests.js.md index 1b22fc3f62..a216559ebd 100644 --- a/packages/build/tests/secrets_scanning/snapshots/tests.js.md +++ b/packages/build/tests/secrets_scanning/snapshots/tests.js.md @@ -1,4 +1,4 @@ -# Snapshot report for `packages/build/tests/secrets_scanning/tests.js` +# Snapshot report for `tests/secrets_scanning/tests.js` The actual snapshot is saved in `tests.js.snap`.