Merge branch 'main' into parser

Author: metabiswadeep

.github/workflows/linting.yml

@@ -12,7 +12,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        tool: ['isort', 'black', 'pyupgrade', 'flake8', 'bandit', 'gitlint']
+        tool: ['isort', 'black', 'pyupgrade', 'flake8', 'bandit', 'gitlint', 'mypy']
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4

.github/workflows/sbom.yml

@@ -1,6 +1,7 @@
 name: SBOM generation
 
 on:
+  workflow_dispatch:
   schedule:
     # Runs at 02:00 UTC every Monday
     - cron: '2 0 * * 1'
@@ -9,45 +10,51 @@ jobs:
   sbom_gen:
     name: Generate SBOM
     runs-on: ubuntu-latest
-    timeout-minutes: 10
-    continue-on-error: true
+    strategy:
+      matrix:
+        python: ['3.7', '3.8', '3.9', '3.10', '3.11']
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: '3.x'
+          python-version: ${{ matrix.python }}
           cache: 'pip'
           cache-dependency-path: '**/requirements.txt'
-      - name: Get date
-        id: get-date
-        run: |
-          echo "date=$(/bin/date -u "+%Y%m%d")" >> $GITHUB_OUTPUT
-      - name: Get cached database
-        uses: actions/cache@v3
-        with:
-          path: ~/.cache/cve-bin-tool
-          key: ${{ runner.os }}-cve-bin-tool-${{ steps.get-date.outputs.date }}
       - name: Install dependencies and cve-bin-tool
         run: |
           python -m pip install --upgrade pip
           python -m pip install --upgrade setuptools
           python -m pip install --upgrade wheel
-          python -m pip install --upgrade pytest
           python -m pip install --upgrade sbom4python
-          pip install . -r doc/requirements.txt
+          pip install . --upgrade --upgrade-strategy=eager
       - name: Generate SBOM for cve-bin-tool
         run: |
-          sbom4python --module cve-bin-tool --output sbom/cve-bin-tool.spdx
-          sbom4python --module cve-bin-tool --sbom cyclonedx --format json --output sbom/cve-bin-tool.json
-#      - name: Compare SBOM for cve-bin-tool
-#        # This would fail due to time/date of SBOM generation in SBOM header
-#        # Therefore ignore first 10 lines of file in comparison which is SBOM header
-#        run: |
-#          /bin/tail -n +10 sbom/cve-bin-tool.spdx > orig
-#          /bin/tail -n +10 cve-bin-tool.spdx > new
-#          /bin/diff -b orig new
-#      - name: Display generated SBOM if difference detected
-#        if: ${{ failure() }}
-#        run: |
-#          /bin/cat cve-bin-tool.spdx
-
+          sbom4python --module cve-bin-tool --output cve-bin-tool-py${{ matrix.python }}.spdx
+          sbom4python --module cve-bin-tool --sbom cyclonedx --format json --output cve-bin-tool-py${{ matrix.python }}.json
+      - name: Compare SBOM for cve-bin-tool
+        id: diff-sbom
+        # This would fail due to time/date of SBOM generation in SBOM header
+        # Therefore ignore first 10 lines of file in comparison which is SBOM header
+        run: |
+          /bin/tail -n +10 sbom/cve-bin-tool-py${{ matrix.python }}.spdx > orig
+          /bin/tail -n +10 cve-bin-tool-py${{ matrix.python }}.spdx > new
+          echo "changed=$(/bin/diff -q orig new)" >> $GITHUB_OUTPUT
+      - name: Display generated SBOM if difference detected
+        if: ${{ steps.diff-sbom.outputs.changed }}
+        run: |
+          /bin/cat cve-bin-tool-py${{ matrix.python }}.spdx
+      - name: Update existing SBOM if difference detected
+        if: ${{ steps.diff-sbom.outputs.changed }}
+        run: |
+          cp cve-bin-tool-py${{ matrix.python }}.spdx sbom/cve-bin-tool-py${{ matrix.python }}.spdx
+          cp cve-bin-tool-py${{ matrix.python }}.json sbom/cve-bin-tool-py${{ matrix.python }}.json
+      - name: Create Pull Request
+        if: ${{ steps.diff-sbom.outputs.changed }}
+        uses: peter-evans/create-pull-request@v4
+        with:
+          commit-message: "chore: update SBOM for Python ${{ matrix.python }}"
+          title: "chore: update SBOM for Python ${{ matrix.python }}"
+          branch: chore-sbom-py${{ matrix.python }}
+          delete-branch: true
+          author: GitHub <[email protected]>
+          add-paths: sbom

.github/workflows/testing.yml

@@ -47,7 +47,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python: ['3.7', '3.8', '3.9']
+        python: ['3.7', '3.8', '3.9', '3.11']
     timeout-minutes: 20
     steps:
       - uses: actions/checkout@v3
@@ -294,31 +294,6 @@ jobs:
           test/test_cvedb.py
       - name: Run HTML tests
         run: pytest -v -n auto test/test_html.py
-      - name: Cache conda
-        uses: actions/cache@v3
-        env:
-          # Increase to reset cache if requirements.txt file has not changed
-          CACHE_NUMBER: 0
-        with:
-          path: ~/conda_pkgs_dir
-          key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{
-            hashFiles('requirements.txt') }}
-      - uses: conda-incubator/setup-miniconda@v2
-        with:
-          auto-update-conda: true
-          activate-environment: pdftotext
-          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!
-      - name: Install pdftotext, reportlab and cve-bin-tool
-        run: |
-          conda install -c conda-forge python=3.10 poppler pdftotext
-          python -m pip install --upgrade pip
-          python -m pip install --upgrade setuptools
-          python -m pip install --upgrade wheel
-          python -m pip install --upgrade reportlab
-          python -m pip install --upgrade -r dev-requirements.txt
-          python -m pip install --upgrade .
-      - name: Test PDF generation on Windows
-        run: pytest test/test_output_engine.py -k test_output_pdf
 
   windows_long_tests:
     name: Windows long tests
@@ -401,4 +376,4 @@ jobs:
           files: ./coverage.xml
           flags: win-longtests
           name: codecov-umbrella
-          fail_ci_if_error: false
+          fail_ci_if_error: false

.github/workflows/update-js-dependencies.yml

@@ -22,8 +22,7 @@ jobs:
           python-version: '3.x'
 
       - name: Update JS dependencies
-        run: |
-          python .github/workflows/update_js_dependencies.py
+        run: python .github/workflows/update_js_dependencies.py
 
       - name: Get cached Python packages
         uses: actions/cache@v3
@@ -35,15 +34,13 @@ jobs:
 
       - name: Install dependencies
         run: |
-          sudo apt-get update
-          sudo apt-get install build-essential libpoppler-cpp-dev pkg-config python3-dev
           python -m pip install --upgrade pip
           python -m pip install --upgrade setuptools wheel
-          python -m pip install --upgrade -r requirements.txt pytest pdftotext
+          python -m pip install --upgrade . pytest-xdist pytest-playwright
+          python -m playwright install chromium --with-deps
 
-      - name: Test HTML report generation
-        run: |
-          pytest test/test_output_engine.py -k test_output_html
+      - name: Run HTML tests
+        run: python -m pytest -v -n auto test/test_html.py
 
       - name: Create Pull Request
         uses: peter-evans/create-pull-request@v4
@@ -62,11 +59,10 @@ jobs:
         run: |
           python -c 'from test.test_output_engine import TestOutputEngine; \
           from cve_bin_tool.output_engine.html import output_html; \
-          output_html(TestOutputEngine.MOCK_OUTPUT, "", "", "", 3, 3, 0, None, None, open("test.html", "w"))'
+          output_html(TestOutputEngine.MOCK_OUTPUT, None, "", "", "", 3, 3, 0, None, None, open("test.html", "w"))'
 
       - name: Upload mock report
         uses: actions/upload-artifact@v3
         with:
           name: HTML report
-          path: |
-            test.html
+          path: test.html

.pre-commit-config.yaml

@@ -31,3 +31,38 @@ repos:
     rev: v0.17.0
     hooks:
     - id: gitlint
+
+-   repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.991
+    hooks:
+    - id: mypy
+      additional_dependencies:
+        - types-beautifulsoup4
+        - types-jsonschema
+        - types-PyYAML
+        - types-requests
+        - types-setuptools
+        - types-toml
+      files: |
+        (?x)^(
+            cve_bin_tool/parsers/.*|
+            cve_bin_tool/__init__.py|
+            cve_bin_tool/async_utils.py|
+            cve_bin_tool/file.py|
+            cve_bin_tool/linkify.py|
+            cve_bin_tool/log.py|
+            cve_bin_tool/strings.py|
+            cve_bin_tool/theme.py|
+            cve_bin_tool/util.py|
+            cve_bin_tool/validator.py|
+            cve_bin_tool/version.py|
+            doc/.*|
+            test/test_data/.*|
+            test/__init__.py|
+            test/test_file.py|s
+            test/test_requirements.py|
+            test/test_strings.py|
+            test/test_triage.py|
+            test/test_version.py|
+            test/utils.py|
+        )$

cve_bin_tool/data_sources/gad_source.py

@@ -211,6 +211,9 @@ def format_data(self, all_cve_entries):
         for cve_item in all_cve_entries:
             cve_in_identifier = None
 
+            if not cve_item:
+                continue
+
             for cve in cve_item.get("identifiers"):
                 if "CVE" in cve:
                     cve_in_identifier = cve

cve_bin_tool/output_engine/html_reports/js/bootstrap.js

@@ -4,6 +4,7 @@ pre-commit==2.20.0
 flake8==5.0.4
 bandit==1.7.4
 gitlint== v0.17.0
+mypy==v0.991
 py>=1.10.0
 pytest
 pytest-xdist