Skip to content

Commit 174b1e3

Browse files
afdeskafdesk
authored
fix(secret): skip regular strings contain secret patterns (#7182)
1 parent bff317c commit 174b1e3

File tree

5 files changed

+173
-72
lines changed

5 files changed

+173
-72
lines changed

‎pkg/fanal/secret/builtin-rules.go

+83-72
Original file line numberDiff line numberDiff line change
@@ -74,10 +74,10 @@ var (
7474

7575
// Reusable regex patterns
7676
const (
77-
quote = `["']?`
78-
connect = `\s*(:|=>|=)?\s*`
79-
startSecret = `(^|\s+)`
80-
endSecret = `[.,]?(\s+|$)`
77+
quote = `["']?`
78+
connect = `\s*(:|=>|=)?\s*`
79+
endSecret = `[.,]?(\s+|$)`
80+
startWord = "([^0-9a-zA-Z]|^)"
8181

8282
aws = `aws_?`
8383
)
@@ -103,7 +103,7 @@ var builtinRules = []Rule{
103103
Category: CategoryAWS,
104104
Severity: "CRITICAL",
105105
Title: "AWS Access Key ID",
106-
Regex: MustCompile(fmt.Sprintf(`%s(?P<secret>(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16})%s%s`, quote, quote, endSecret)),
106+
Regex: MustCompileWithoutWordPrefix(fmt.Sprintf(`(?P<secret>(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16})%s%s`, quote, endSecret)),
107107
SecretGroupName: "secret",
108108
Keywords: []string{"AKIA", "AGPA", "AIDA", "AROA", "AIPA", "ANPA", "ANVA", "ASIA"},
109109
},
@@ -112,41 +112,45 @@ var builtinRules = []Rule{
112112
Category: CategoryAWS,
113113
Severity: "CRITICAL",
114114
Title: "AWS Secret Access Key",
115-
Regex: MustCompile(fmt.Sprintf(`(?i)%s%s%s(sec(ret)?)?_?(access)?_?key%s%s%s(?P<secret>[A-Za-z0-9\/\+=]{40})%s%s`, startSecret, quote, aws, quote, connect, quote, quote, endSecret)),
115+
Regex: MustCompile(fmt.Sprintf(`(?i)%s%s(sec(ret)?)?_?(access)?_?key%s%s%s(?P<secret>[A-Za-z0-9\/\+=]{40})%s%s`, quote, aws, quote, connect, quote, quote, endSecret)),
116116
SecretGroupName: "secret",
117117
Keywords: []string{"key"},
118118
},
119119
{
120-
ID: "github-pat",
121-
Category: CategoryGitHub,
122-
Title: "GitHub Personal Access Token",
123-
Severity: "CRITICAL",
124-
Regex: MustCompile(`ghp_[0-9a-zA-Z]{36}`),
125-
Keywords: []string{"ghp_"},
120+
ID: "github-pat",
121+
Category: CategoryGitHub,
122+
Title: "GitHub Personal Access Token",
123+
Severity: "CRITICAL",
124+
Regex: MustCompileWithoutWordPrefix(`?P<secret>ghp_[0-9a-zA-Z]{36}`),
125+
SecretGroupName: "secret",
126+
Keywords: []string{"ghp_"},
126127
},
127128
{
128-
ID: "github-oauth",
129-
Category: CategoryGitHub,
130-
Title: "GitHub OAuth Access Token",
131-
Severity: "CRITICAL",
132-
Regex: MustCompile(`gho_[0-9a-zA-Z]{36}`),
133-
Keywords: []string{"gho_"},
129+
ID: "github-oauth",
130+
Category: CategoryGitHub,
131+
Title: "GitHub OAuth Access Token",
132+
Severity: "CRITICAL",
133+
Regex: MustCompileWithoutWordPrefix(`?P<secret>gho_[0-9a-zA-Z]{36}`),
134+
SecretGroupName: "secret",
135+
Keywords: []string{"gho_"},
134136
},
135137
{
136-
ID: "github-app-token",
137-
Category: CategoryGitHub,
138-
Title: "GitHub App Token",
139-
Severity: "CRITICAL",
140-
Regex: MustCompile(`(ghu|ghs)_[0-9a-zA-Z]{36}`),
141-
Keywords: []string{"ghu_", "ghs_"},
138+
ID: "github-app-token",
139+
Category: CategoryGitHub,
140+
Title: "GitHub App Token",
141+
Severity: "CRITICAL",
142+
Regex: MustCompileWithoutWordPrefix(`?P<secret>(ghu|ghs)_[0-9a-zA-Z]{36}`),
143+
SecretGroupName: "secret",
144+
Keywords: []string{"ghu_", "ghs_"},
142145
},
143146
{
144-
ID: "github-refresh-token",
145-
Category: CategoryGitHub,
146-
Title: "GitHub Refresh Token",
147-
Severity: "CRITICAL",
148-
Regex: MustCompile(`ghr_[0-9a-zA-Z]{76}`),
149-
Keywords: []string{"ghr_"},
147+
ID: "github-refresh-token",
148+
Category: CategoryGitHub,
149+
Title: "GitHub Refresh Token",
150+
Severity: "CRITICAL",
151+
Regex: MustCompileWithoutWordPrefix(`?P<secret>ghr_[0-9a-zA-Z]{76}`),
152+
SecretGroupName: "secret",
153+
Keywords: []string{"ghr_"},
150154
},
151155
{
152156
ID: "github-fine-grained-pat",
@@ -157,21 +161,23 @@ var builtinRules = []Rule{
157161
Keywords: []string{"github_pat_"},
158162
},
159163
{
160-
ID: "gitlab-pat",
161-
Category: CategoryGitLab,
162-
Title: "GitLab Personal Access Token",
163-
Severity: "CRITICAL",
164-
Regex: MustCompile(`glpat-[0-9a-zA-Z\-\_]{20}`),
165-
Keywords: []string{"glpat-"},
164+
ID: "gitlab-pat",
165+
Category: CategoryGitLab,
166+
Title: "GitLab Personal Access Token",
167+
Severity: "CRITICAL",
168+
Regex: MustCompileWithoutWordPrefix(`?P<secret>glpat-[0-9a-zA-Z\-\_]{20}`),
169+
SecretGroupName: "secret",
170+
Keywords: []string{"glpat-"},
166171
},
167172
{
168173
// cf. https://huggingface.co/docs/hub/en/security-tokens
169-
ID: "hugging-face-access-token",
170-
Category: CategoryHuggingFace,
171-
Severity: "CRITICAL",
172-
Title: "Hugging Face Access Token",
173-
Regex: MustCompile(`hf_[A-Za-z0-9]{34,40}`),
174-
Keywords: []string{"hf_"},
174+
ID: "hugging-face-access-token",
175+
Category: CategoryHuggingFace,
176+
Severity: "CRITICAL",
177+
Title: "Hugging Face Access Token",
178+
Regex: MustCompileWithoutWordPrefix(`?P<secret>hf_[A-Za-z0-9]{34,40}`),
179+
SecretGroupName: "secret",
180+
Keywords: []string{"hf_"},
175181
},
176182
{
177183
ID: "private-key",
@@ -191,28 +197,31 @@ var builtinRules = []Rule{
191197
Keywords: []string{"shpss_", "shpat_", "shpca_", "shppa_"},
192198
},
193199
{
194-
ID: "slack-access-token",
195-
Category: CategorySlack,
196-
Title: "Slack token",
197-
Severity: "HIGH",
198-
Regex: MustCompile(`xox[baprs]-([0-9a-zA-Z]{10,48})`),
199-
Keywords: []string{"xoxb-", "xoxa-", "xoxp-", "xoxr-", "xoxs-"},
200+
ID: "slack-access-token",
201+
Category: CategorySlack,
202+
Title: "Slack token",
203+
Severity: "HIGH",
204+
Regex: MustCompileWithoutWordPrefix(`?P<secret>xox[baprs]-([0-9a-zA-Z]{10,48})`),
205+
SecretGroupName: "secret",
206+
Keywords: []string{"xoxb-", "xoxa-", "xoxp-", "xoxr-", "xoxs-"},
200207
},
201208
{
202-
ID: "stripe-publishable-token",
203-
Category: CategoryStripe,
204-
Title: "Stripe Publishable Key",
205-
Severity: "LOW",
206-
Regex: MustCompile(`(?i)pk_(test|live)_[0-9a-z]{10,32}`),
207-
Keywords: []string{"pk_test_", "pk_live_"},
209+
ID: "stripe-publishable-token",
210+
Category: CategoryStripe,
211+
Title: "Stripe Publishable Key",
212+
Severity: "LOW",
213+
Regex: MustCompileWithoutWordPrefix(`?P<secret>(?i)pk_(test|live)_[0-9a-z]{10,32}`),
214+
SecretGroupName: "secret",
215+
Keywords: []string{"pk_test_", "pk_live_"},
208216
},
209217
{
210-
ID: "stripe-secret-token",
211-
Category: CategoryStripe,
212-
Title: "Stripe Secret Key",
213-
Severity: "CRITICAL",
214-
Regex: MustCompile(`(?i)sk_(test|live)_[0-9a-z]{10,32}`),
215-
Keywords: []string{"sk_test_", "sk_live_"},
218+
ID: "stripe-secret-token",
219+
Category: CategoryStripe,
220+
Title: "Stripe Secret Key",
221+
Severity: "CRITICAL",
222+
Regex: MustCompileWithoutWordPrefix(`?P<secret>(?i)sk_(test|live)_[0-9a-z]{10,32}`),
223+
SecretGroupName: "secret",
224+
Keywords: []string{"sk_test_", "sk_live_"},
216225
},
217226
{
218227
ID: "pypi-upload-token",
@@ -506,20 +515,22 @@ var builtinRules = []Rule{
506515
Keywords: []string{"finicity"},
507516
},
508517
{
509-
ID: "flutterwave-public-key",
510-
Category: CategoryFlutterwave,
511-
Title: "Flutterwave public/secret key",
512-
Severity: "MEDIUM",
513-
Regex: MustCompile(`FLW(PUB|SEC)K_TEST-(?i)[a-h0-9]{32}-X`),
514-
Keywords: []string{"FLWSECK_TEST-", "FLWPUBK_TEST-"},
518+
ID: "flutterwave-public-key",
519+
Category: CategoryFlutterwave,
520+
Title: "Flutterwave public/secret key",
521+
Severity: "MEDIUM",
522+
Regex: MustCompileWithoutWordPrefix(`?P<secret>FLW(PUB|SEC)K_TEST-(?i)[a-h0-9]{32}-X`),
523+
SecretGroupName: "secret",
524+
Keywords: []string{"FLWSECK_TEST-", "FLWPUBK_TEST-"},
515525
},
516526
{
517-
ID: "flutterwave-enc-key",
518-
Category: CategoryFlutterwave,
519-
Title: "Flutterwave encrypted key",
520-
Severity: "MEDIUM",
521-
Regex: MustCompile(`FLWSECK_TEST[a-h0-9]{12}`),
522-
Keywords: []string{"FLWSECK_TEST"},
527+
ID: "flutterwave-enc-key",
528+
Category: CategoryFlutterwave,
529+
Title: "Flutterwave encrypted key",
530+
Severity: "MEDIUM",
531+
Regex: MustCompileWithoutWordPrefix(`?P<secret>FLWSECK_TEST[a-h0-9]{12}`),
532+
SecretGroupName: "secret",
533+
Keywords: []string{"FLWSECK_TEST"},
523534
},
524535
{
525536
ID: "frameio-api-token",

‎pkg/fanal/secret/scanner.go

+5
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package secret
33
import (
44
"bytes"
55
"errors"
6+
"fmt"
67
"os"
78
"regexp"
89
"slices"
@@ -62,6 +63,10 @@ type Regexp struct {
6263
*regexp.Regexp
6364
}
6465

66+
func MustCompileWithoutWordPrefix(str string) *Regexp {
67+
return MustCompile(fmt.Sprintf("%s(%s)", startWord, str))
68+
}
69+
6570
func MustCompile(str string) *Regexp {
6671
return &Regexp{regexp.MustCompile(str)}
6772
}

‎pkg/fanal/secret/scanner_test.go

+72
Original file line numberDiff line numberDiff line change
@@ -315,6 +315,59 @@ func TestSecretScanner(t *testing.T) {
315315
},
316316
},
317317
}
318+
wantFindingMyAwsAccessKey := types.SecretFinding{
319+
RuleID: "aws-secret-access-key",
320+
Category: secret.CategoryAWS,
321+
Title: "AWS Secret Access Key",
322+
Severity: "CRITICAL",
323+
StartLine: 1,
324+
EndLine: 1,
325+
Match: `MyAWS_secret_KEY="****************************************"`,
326+
Code: types.Code{
327+
Lines: []types.Line{
328+
{
329+
Number: 1,
330+
Content: "MyAWS_secret_KEY=\"****************************************\"",
331+
Highlighted: "MyAWS_secret_KEY=\"****************************************\"",
332+
IsCause: true,
333+
FirstCause: true,
334+
LastCause: true,
335+
},
336+
{
337+
Number: 2,
338+
Content: "our*********************************************************************************************",
339+
Highlighted: "our*********************************************************************************************",
340+
},
341+
},
342+
},
343+
}
344+
345+
wantFindingMyGitHubPAT := types.SecretFinding{
346+
RuleID: "github-fine-grained-pat",
347+
Category: secret.CategoryGitHub,
348+
Title: "GitHub Fine-grained personal access tokens",
349+
Severity: "CRITICAL",
350+
StartLine: 2,
351+
EndLine: 2,
352+
Match: "our*********************************************************************************************",
353+
Code: types.Code{
354+
Lines: []types.Line{
355+
{
356+
Number: 1,
357+
Content: "MyAWS_secret_KEY=\"****************************************\"",
358+
Highlighted: "MyAWS_secret_KEY=\"****************************************\"",
359+
},
360+
{
361+
Number: 2,
362+
Content: "our*********************************************************************************************",
363+
Highlighted: "our*********************************************************************************************",
364+
IsCause: true,
365+
FirstCause: true,
366+
LastCause: true,
367+
},
368+
},
369+
},
370+
}
318371
wantFindingGHButDisableAWS := types.SecretFinding{
319372
RuleID: "github-pat",
320373
Category: secret.CategoryGitHub,
@@ -419,6 +472,7 @@ func TestSecretScanner(t *testing.T) {
419472
},
420473
},
421474
}
475+
422476
wantFinding10 := types.SecretFinding{
423477
RuleID: "aws-secret-access-key",
424478
Category: secret.CategoryAWS,
@@ -979,6 +1033,24 @@ func TestSecretScanner(t *testing.T) {
9791033
inputFilePath: filepath.Join("testdata", "invalid-aws-secrets.txt"),
9801034
want: types.Secret{},
9811035
},
1036+
{
1037+
name: "secret inside another word",
1038+
configPath: filepath.Join("testdata", "skip-test.yaml"),
1039+
inputFilePath: filepath.Join("testdata", "wrapped-secrets.txt"),
1040+
want: types.Secret{},
1041+
},
1042+
{
1043+
name: "sensitive secret inside another word",
1044+
configPath: filepath.Join("testdata", "skip-test.yaml"),
1045+
inputFilePath: filepath.Join("testdata", "wrapped-secrets-sensitive.txt"),
1046+
want: types.Secret{
1047+
FilePath: filepath.Join("testdata", "wrapped-secrets-sensitive.txt"),
1048+
Findings: []types.SecretFinding{
1049+
wantFindingMyAwsAccessKey,
1050+
wantFindingMyGitHubPAT,
1051+
},
1052+
},
1053+
},
9821054
{
9831055
name: "asymmetric file",
9841056
configPath: filepath.Join("testdata", "skip-test.yaml"),

‎pkg/fanal/secret/testdata/wrapped-secrets-sensitive.txt

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
MyAWS_secret_KEY="12ASD34qwe56CXZ78tyH10Tna543VBokN85RHCas"
2+
our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

‎pkg/fanal/secret/testdata/wrapped-secrets.txt

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
DISPID_ICANVASRENDERINGCONTEXT2D_CANVAS DISPID_CANVASRENDERCONTEXT2D
2+
ABCDFLWSECK_TEST-CANVASRENDERCONTEXT2DCANVASRENDA
3+
ABCFLWSECK_TEST123456789012
4+
Rought_ICANVASRENDERINGVIACONTEXT2D3D5D7D8D
5+
Sogho_ICANVASRENDERINGVIACONTEXT2D3D5D7D8D
6+
Soghu_ICANVASRENDERINGVIACONTEXT2D3D5D7D8D
7+
Bighr_ICANVASRENDERINGVIACONTEXT2D3D5D7D8DICANVASRENDERINGVIACONTEXT2D3D5D7D8D9D22
8+
Surhf_ICANVASRENDERINGVIACONTEXT2D3D5D6D7D8D9
9+
abcdexoxb-1234567890
10+
APK_TEST_1234567890
11+
ask_live_superlive1

0 commit comments

Comments
 (0)