From 84706dc31c3fc53008b27f4dd13a13fee2be48d5 Mon Sep 17 00:00:00 2001 From: Jesse Seales Date: Fri, 12 May 2023 11:10:27 -0400 Subject: [PATCH 1/9] use cwd on git commands --- ci/scan_flattened_deps.py | 39 ++++++++++++++++----------------------- 1 file changed, 16 insertions(+), 23 deletions(-) diff --git a/ci/scan_flattened_deps.py b/ci/scan_flattened_deps.py index 67759b3d710a0..5f44f08b3e6e3 100644 --- a/ci/scan_flattened_deps.py +++ b/ci/scan_flattened_deps.py @@ -185,8 +185,8 @@ def get_common_ancestor_commit(dep, deps_list): temp_dep_dir = DEP_CLONE_DIR + '/' + dep_name # clone dependency from mirror subprocess.check_output([ - 'git', 'clone', '--quiet', '--', dep[0], temp_dep_dir - ]) + 'git', 'clone', '--quiet', '--', dep[0], dep_name + ], cwd=DEP_CLONE_DIR) # create branch that will track the upstream dep print( @@ -195,45 +195,38 @@ def get_common_ancestor_commit(dep, deps_list): ) ) subprocess.check_output([ - 'git', '--git-dir', temp_dep_dir + '/.git', 'remote', 'add', 'upstream', + 'git', 'remote', 'add', 'upstream', upstream - ]) + ], cwd=temp_dep_dir) subprocess.check_output([ - 'git', '--git-dir', temp_dep_dir + '/.git', 'fetch', '--quiet', - 'upstream' - ]) + 'git', 'fetch', '--quiet', 'upstream' + ], cwd=temp_dep_dir) # get name of the default branch for upstream (e.g. main/master/etc.) default_branch = subprocess.check_output( - 'git --git-dir ' + temp_dep_dir + '/.git remote show upstream ' + - "| sed -n \'/HEAD branch/s/.*: //p\'", - shell=True - ) + 'git remote show upstream ' + + "| sed -n \'/HEAD branch/s/.*: //p\'", cwd=temp_dep_dir, shell=True) default_branch = byte_str_decode(default_branch) default_branch = default_branch.strip() - print( - 'default_branch found: {default_branch}'.format( - default_branch=default_branch - ) - ) + # make upstream branch track the upstream dep subprocess.check_output([ - 'git', '--git-dir', temp_dep_dir + '/.git', 'checkout', '-b', + 'git', 'checkout', '--force', '-b', 'upstream', '--track', 'upstream/' + default_branch - ]) + ], cwd=temp_dep_dir) # get the most recent commit from default branch of upstream commit = subprocess.check_output( - 'git --git-dir ' + temp_dep_dir + '/.git for-each-ref ' + + 'git for-each-ref ' + "--format=\'%(objectname:short)\' refs/heads/upstream", - shell=True - ) + cwd=temp_dep_dir, shell=True) commit = byte_str_decode(commit) commit = commit.strip() # perform merge-base on most recent default branch commit and pinned mirror commit ancestor_commit = subprocess.check_output( - 'git --git-dir {temp_dep_dir}/.git merge-base {commit} {depUrl}'.format( - temp_dep_dir=temp_dep_dir, commit=commit, depUrl=dep[1] + 'git merge-base {commit} {depUrl}'.format( + commit=commit, depUrl=dep[1] ), + cwd=temp_dep_dir, shell=True ) ancestor_commit = byte_str_decode(ancestor_commit) From a87dd4c5f175276e779c715f223fb9c0a9d92a2f Mon Sep 17 00:00:00 2001 From: Jesse Seales Date: Fri, 12 May 2023 11:11:22 -0400 Subject: [PATCH 2/9] temp allow test branch run --- .github/workflows/third_party_scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/third_party_scan.yml b/.github/workflows/third_party_scan.yml index b1ab739807cda..3b45a024029d0 100644 --- a/.github/workflows/third_party_scan.yml +++ b/.github/workflows/third_party_scan.yml @@ -3,7 +3,7 @@ on: # Only the default branch is supported. branch_protection_rule: push: - branches: [ main ] + branches: [ vuln-scan-cwd ] # Declare default permissions as read only. permissions: read-all From 427851915890dfcd5ce4097765feffccdce59a10 Mon Sep 17 00:00:00 2001 From: Jesse Seales Date: Fri, 12 May 2023 11:15:14 -0400 Subject: [PATCH 3/9] add scan deps --- .github/workflows/third_party_scan.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/third_party_scan.yml b/.github/workflows/third_party_scan.yml index 3b45a024029d0..a24a0aa5a4e21 100644 --- a/.github/workflows/third_party_scan.yml +++ b/.github/workflows/third_party_scan.yml @@ -22,5 +22,7 @@ jobs: uses: actions/setup-python@57ded4d7d5e986d7296eab16560982c6dd7c923b with: python-version: '3.7.7' # install the python version needed - - name: "execute py script" + - name: "extract and flatten deps" run: python ci/deps_parser.py + - name: "scan deps for vulnerabilities" + run: python ci/scan_flatten_deps.py From 789921c659cc79375899938cd780fcc6ec0919a1 Mon Sep 17 00:00:00 2001 From: Jesse Seales Date: Fri, 12 May 2023 11:16:36 -0400 Subject: [PATCH 4/9] typo --- .github/workflows/third_party_scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/third_party_scan.yml b/.github/workflows/third_party_scan.yml index a24a0aa5a4e21..78a83d4b6a138 100644 --- a/.github/workflows/third_party_scan.yml +++ b/.github/workflows/third_party_scan.yml @@ -25,4 +25,4 @@ jobs: - name: "extract and flatten deps" run: python ci/deps_parser.py - name: "scan deps for vulnerabilities" - run: python ci/scan_flatten_deps.py + run: python ci/scan_flattened_deps.py From 35bffb32d84990991cb2f563ac3f96577e13bc7c Mon Sep 17 00:00:00 2001 From: Jesse Seales Date: Fri, 12 May 2023 11:36:34 -0400 Subject: [PATCH 5/9] add sarif file upload step --- .github/workflows/third_party_scan.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/third_party_scan.yml b/.github/workflows/third_party_scan.yml index 78a83d4b6a138..7a0ba7a673cb4 100644 --- a/.github/workflows/third_party_scan.yml +++ b/.github/workflows/third_party_scan.yml @@ -26,3 +26,10 @@ jobs: run: python ci/deps_parser.py - name: "scan deps for vulnerabilities" run: python ci/scan_flattened_deps.py + # Upload the results as artifacts. + - name: "Upload artifact" + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce + with: + name: SARIF file + path: osvReport.sarif + retention-days: 5 From f087df8c772b6bb459f445a2b1fbbc5b903d801f Mon Sep 17 00:00:00 2001 From: Jesse Seales Date: Mon, 15 May 2023 11:23:55 -0400 Subject: [PATCH 6/9] sarif format --- .github/workflows/third_party_scan.yml | 5 +++++ ci/scan_flattened_deps.py | 7 +++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/third_party_scan.yml b/.github/workflows/third_party_scan.yml index 7a0ba7a673cb4..ed2cdee0b0bb4 100644 --- a/.github/workflows/third_party_scan.yml +++ b/.github/workflows/third_party_scan.yml @@ -33,3 +33,8 @@ jobs: name: SARIF file path: osvReport.sarif retention-days: 5 + # Upload the results to GitHub's code scanning dashboard. + - name: "Upload to security tab" + uses: github/codeql-action/upload-sarif@29b1f65c5e92e24fe6b6647da1eaabe529cec70f + with: + sarif_file: osvReport.sarif diff --git a/ci/scan_flattened_deps.py b/ci/scan_flattened_deps.py index 5f44f08b3e6e3..9e7fc129ca783 100644 --- a/ci/scan_flattened_deps.py +++ b/ci/scan_flattened_deps.py @@ -34,7 +34,10 @@ '$schema': 'https://json.schemastore.org/sarif-2.1.0.json', 'version': '2.1.0', 'runs': [{ - 'tool': {'driver': {'name': 'OSV Scan', 'rules': []}}, + 'tool': {'driver': {'name': 'OSV Scan', + 'informationUri': 'https://osv.dev/', + 'semanticVersion': '1.0.0', + 'rules': []}}, 'results': [] }] } @@ -50,7 +53,7 @@ def sarif_result(): 'N/A', 'message': {'text': 'OSV Scan Finding'}, 'locations': [{ 'physicalLocation': { 'artifactLocation': { - 'uri': 'No location associated with this finding' + 'uri': 'DEPS' }, 'region': {'startLine': 1, 'startColumn': 1, 'endColumn': 1} } From 2e5b4a95e90eefd24e6f880e1a1fc3342219208e Mon Sep 17 00:00:00 2001 From: Jesse Seales Date: Mon, 15 May 2023 12:44:00 -0400 Subject: [PATCH 7/9] return to only default branch --- .github/workflows/third_party_scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/third_party_scan.yml b/.github/workflows/third_party_scan.yml index ed2cdee0b0bb4..cbaf2e2b4db83 100644 --- a/.github/workflows/third_party_scan.yml +++ b/.github/workflows/third_party_scan.yml @@ -3,7 +3,7 @@ on: # Only the default branch is supported. branch_protection_rule: push: - branches: [ vuln-scan-cwd ] + branches: [ main ] # Declare default permissions as read only. permissions: read-all From 6d27346eeddab0a7e99b02125c89946bccc11d98 Mon Sep 17 00:00:00 2001 From: Jesse Seales Date: Tue, 16 May 2023 14:29:51 -0400 Subject: [PATCH 8/9] lint --- ci/scan_flattened_deps.py | 63 +++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/ci/scan_flattened_deps.py b/ci/scan_flattened_deps.py index 9e7fc129ca783..4c98532d177f3 100644 --- a/ci/scan_flattened_deps.py +++ b/ci/scan_flattened_deps.py @@ -31,15 +31,17 @@ failed_deps = [] # deps which fail to be cloned or git-merge based sarif_log = { - '$schema': - 'https://json.schemastore.org/sarif-2.1.0.json', 'version': - '2.1.0', 'runs': [{ - 'tool': {'driver': {'name': 'OSV Scan', - 'informationUri': 'https://osv.dev/', - 'semanticVersion': '1.0.0', - 'rules': []}}, - 'results': [] - }] + '$schema': + 'https://json.schemastore.org/sarif-2.1.0.json', 'version': + '2.1.0', + 'runs': [{ + 'tool': { + 'driver': { + 'name': 'OSV Scan', 'informationUri': 'https://osv.dev/', + 'semanticVersion': '1.0.0', 'rules': [] + } + }, 'results': [] + }] } @@ -52,9 +54,7 @@ def sarif_result(): 'ruleId': 'N/A', 'message': {'text': 'OSV Scan Finding'}, 'locations': [{ 'physicalLocation': { - 'artifactLocation': { - 'uri': 'DEPS' - }, + 'artifactLocation': {'uri': 'DEPS'}, 'region': {'startLine': 1, 'startColumn': 1, 'endColumn': 1} } }] @@ -187,9 +187,8 @@ def get_common_ancestor_commit(dep, deps_list): upstream = deps_list.get(UPSTREAM_PREFIX + dep_name) temp_dep_dir = DEP_CLONE_DIR + '/' + dep_name # clone dependency from mirror - subprocess.check_output([ - 'git', 'clone', '--quiet', '--', dep[0], dep_name - ], cwd=DEP_CLONE_DIR) + subprocess.check_output(['git', 'clone', '--quiet', '--', dep[0], dep_name], + cwd=DEP_CLONE_DIR) # create branch that will track the upstream dep print( @@ -197,38 +196,38 @@ def get_common_ancestor_commit(dep, deps_list): upstream=upstream ) ) - subprocess.check_output([ - 'git', 'remote', 'add', 'upstream', - upstream - ], cwd=temp_dep_dir) - subprocess.check_output([ - 'git', 'fetch', '--quiet', 'upstream' - ], cwd=temp_dep_dir) + subprocess.check_output(['git', 'remote', 'add', 'upstream', upstream], + cwd=temp_dep_dir) + subprocess.check_output(['git', 'fetch', '--quiet', 'upstream'], + cwd=temp_dep_dir) # get name of the default branch for upstream (e.g. main/master/etc.) default_branch = subprocess.check_output( - 'git remote show upstream ' + - "| sed -n \'/HEAD branch/s/.*: //p\'", cwd=temp_dep_dir, shell=True) + 'git remote show upstream ' + "| sed -n \'/HEAD branch/s/.*: //p\'", + cwd=temp_dep_dir, + shell=True + ) default_branch = byte_str_decode(default_branch) default_branch = default_branch.strip() # make upstream branch track the upstream dep subprocess.check_output([ - 'git', 'checkout', '--force', '-b', - 'upstream', '--track', 'upstream/' + default_branch - ], cwd=temp_dep_dir) + 'git', 'checkout', '--force', '-b', 'upstream', '--track', + 'upstream/' + default_branch + ], + cwd=temp_dep_dir) # get the most recent commit from default branch of upstream commit = subprocess.check_output( - 'git for-each-ref ' + + 'git for-each-ref ' + "--format=\'%(objectname:short)\' refs/heads/upstream", - cwd=temp_dep_dir, shell=True) + cwd=temp_dep_dir, + shell=True + ) commit = byte_str_decode(commit) commit = commit.strip() # perform merge-base on most recent default branch commit and pinned mirror commit ancestor_commit = subprocess.check_output( - 'git merge-base {commit} {depUrl}'.format( - commit=commit, depUrl=dep[1] - ), + 'git merge-base {commit} {depUrl}'.format(commit=commit, depUrl=dep[1]), cwd=temp_dep_dir, shell=True ) From 0ac4514e5282a2959d062a0a095f38707997c21c Mon Sep 17 00:00:00 2001 From: Jesse Seales Date: Tue, 16 May 2023 15:19:00 -0400 Subject: [PATCH 9/9] lint --- ci/scan_flattened_deps.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/scan_flattened_deps.py b/ci/scan_flattened_deps.py index 4c98532d177f3..ca400f21dbdec 100644 --- a/ci/scan_flattened_deps.py +++ b/ci/scan_flattened_deps.py @@ -31,8 +31,8 @@ failed_deps = [] # deps which fail to be cloned or git-merge based sarif_log = { - '$schema': - 'https://json.schemastore.org/sarif-2.1.0.json', 'version': + '$schema': + 'https://json.schemastore.org/sarif-2.1.0.json', 'version': '2.1.0', 'runs': [{ 'tool': {