From 17aff60091ca422401600c55537e4fe2ec9dd93a Mon Sep 17 00:00:00 2001 From: ambuj Date: Mon, 16 Jun 2025 13:24:26 +0530 Subject: [PATCH 1/3] Add functionality to handle poetryV2 - Changed utils.js - Created tests for the changes Signed-off-by: ambuj --- lib/helpers/utils.js | 32 +++++++++++++++++-- lib/helpers/utils.test.js | 30 ++++++++++++++++-- test/data/pyproject_poetryv2.toml | 52 +++++++++++++++++++++++++++++++ 3 files changed, 109 insertions(+), 5 deletions(-) create mode 100644 test/data/pyproject_poetryv2.toml diff --git a/lib/helpers/utils.js b/lib/helpers/utils.js index 21db971a37..0f2b95f26f 100644 --- a/lib/helpers/utils.js +++ b/lib/helpers/utils.js @@ -17,14 +17,14 @@ import { } from "node:fs"; import { homedir, platform, tmpdir } from "node:os"; import path, { - basename, delimiter as _delimiter, + sep as _sep, + basename, dirname, extname, join, - resolve, relative, - sep as _sep, + resolve, } from "node:path"; import process from "node:process"; import { URL, fileURLToPath } from "node:url"; @@ -4904,6 +4904,7 @@ export function parsePyProjectTomlFile(tomlFile) { } } + let isPoetryV2 = false; let poetryMode = false; let uvMode = false; let hatchMode = false; @@ -4926,6 +4927,16 @@ export function parsePyProjectTomlFile(tomlFile) { ) { poetryMode = true; } + const requires = tomlData?.["build-system"]?.["requires"]; + if (requires && Array.isArray(requires)) { + for (const req of requires) { + if (req.startsWith("poetry-core") && req.includes(">=2.0")) { + isPoetryV2 = true; + break; + } + } + } + if (tomlData?.tool?.uv) { uvMode = true; } @@ -5029,6 +5040,7 @@ export function parsePyProjectTomlFile(tomlFile) { return { parentComponent: pkg, poetryMode, + isPoetryV2, uvMode, hatchMode, workspacePaths, @@ -13517,6 +13529,15 @@ export function getPipFrozenTree( }); thoughtLog("Performing poetry install"); let poetryInstallArgs = ["-m", "poetry", "install", "-n", "--no-root"]; + const isPoetryV2 = parsePyProjectTomlFile( + join(basePath, "pyproject.toml"), + ).isPoetryV2; + // checking if poetryV2 is true or not + if (isPoetryV2) { + // Include all dependency groups and extras (Poetry v2+) + poetryInstallArgs.push("--all-groups", "--all-extras"); + } + // Attempt to perform poetry install result = safeSpawnSync(PYTHON_CMD, poetryInstallArgs, { cwd: basePath, @@ -13530,6 +13551,11 @@ export function getPipFrozenTree( "Hmm, poetry doesn't seem to be available as a module. Perhaps it was installed directly 🤔?", ); poetryInstallArgs = ["install", "-n", "--no-root"]; + + if (isPoetryV2) { + // Also include flags when calling poetry directly + poetryInstallArgs.push("--all-groups", "--all-extras"); + } // Attempt to perform poetry install result = safeSpawnSync("poetry", poetryInstallArgs, { cwd: basePath, diff --git a/lib/helpers/utils.test.js b/lib/helpers/utils.test.js index 89de314d5d..3333ab5211 100644 --- a/lib/helpers/utils.test.js +++ b/lib/helpers/utils.test.js @@ -11,12 +11,10 @@ import { findLicenseId, getCratesMetadata, getDartMetadata, - getGoPkgLicense, getLicenses, getMvnMetadata, getNugetMetadata, getPyMetadata, - getRepoLicense, guessPypiMatchingVersion, hasAnyProjectType, isPackageManagerAllowed, @@ -4975,6 +4973,34 @@ test("parse pyproject.toml", () => { }); }); +test("parse pyproject.toml with poetryv2 requirement", () => { + const retMap = parsePyProjectTomlFile("./test/data/pyproject_poetryv2.toml"); + // expect(retMap.parentComponent).toEqual({ + // name: "cpggen", + // version: "1.9.0", + // description: + // "Generate CPG for multiple languages for code and threat analysis", + // license: "Apache-2.0", + // author: "Team AppThreat ", + // homepage: { url: "https://github.com/AppThreat/cpggen" }, + // repository: { url: "https://github.com/AppThreat/cpggen" }, + // tags: [ + // "atom", + // "code analysis", + // "code property graph", + // "cpg", + // "joern", + // "static analysis", + // "threat analysis", + // ], + // type: "application", + // "bom-ref": "pkg:pypi/cpggen@1.9.0", + // purl: "pkg:pypi/cpggen@1.9.0", + // evidence: { identity: { field: "purl", confidence: 1, methods: [Array] } }, + // }); + expect(retMap.isPoetryV2).toBeTruthy(); +}); + test("parse pyproject.toml with custom poetry source", () => { const retMap = parsePyProjectTomlFile( "./test/data/pyproject_with_custom_poetry_source.toml", diff --git a/test/data/pyproject_poetryv2.toml b/test/data/pyproject_poetryv2.toml new file mode 100644 index 0000000000..1b3fdc52a0 --- /dev/null +++ b/test/data/pyproject_poetryv2.toml @@ -0,0 +1,52 @@ +[tool.poetry] +name = "cpggen" +version = "1.9.0" # 1.9.0 is not version 2.0.0 +description = "Generate CPG for multiple languages for code and threat analysis" +authors = ["Team AppThreat "] +license = "Apache-2.0" +readme = "README.md" +packages = [{include = "cpggen"}] +homepage = "https://github.com/AppThreat/cpggen" +repository = "https://github.com/AppThreat/cpggen" +keywords = ["joern", "code analysis", "static analysis", "cpg", "code property graph", "atom", "threat analysis"] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Topic :: Utilities", + "Topic :: Security", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Operating System :: OS Independent", +] +exclude = ["contrib", "tests"] +include = ["cpggen/atom/*"] + +[tool.poetry.scripts] +atomgen = 'cpggen.cli:main' +cpggen = 'cpggen.cli:main' +cpg = 'cpggen.cli:main' + +[tool.poetry.dependencies] +python = ">=3.8.1,<3.12" +rich = "^13.4.2" +gitpython = "^3.1.31" +quart = "^0.18.4" +psutil = "^5.9.5" +packageurl-python = "^0.11.1" +httpx = "^0.24.1" + +[tool.poetry.group.dev.dependencies] +pytest = "^7.4.0" +black = "^23.3.0" +flake8 = "^6.0.0" +pytest-cov = "^4.0.0" +pyinstaller = "^5.12.0" +bandit = "^1.7.5" +pylint = "^2.17.4" + +[build-system] +requires = ["poetry-core>=2.0.0"] +build-backend = "poetry.core.masonry.api" From ce26c4714f8bf862b49628ff25f2fb5d5da342ac Mon Sep 17 00:00:00 2001 From: ambuj Date: Mon, 16 Jun 2025 18:52:45 +0530 Subject: [PATCH 2/3] Modify utils.js and its test code Signed-off-by: ambuj --- lib/helpers/utils.js | 12 ++-- lib/helpers/utils.test.js | 50 ++++++++------- test/data/pyproject_poetryv2.toml | 103 +++++++++++++++++++----------- 3 files changed, 99 insertions(+), 66 deletions(-) diff --git a/lib/helpers/utils.js b/lib/helpers/utils.js index 0f2b95f26f..e8209da154 100644 --- a/lib/helpers/utils.js +++ b/lib/helpers/utils.js @@ -4927,9 +4927,9 @@ export function parsePyProjectTomlFile(tomlFile) { ) { poetryMode = true; } - const requires = tomlData?.["build-system"]?.["requires"]; - if (requires && Array.isArray(requires)) { - for (const req of requires) { + const buildRequires = tomlData?.["build-system"]?.["requires"]; + if (buildRequires && Array.isArray(buildRequires)) { + for (const req of buildRequires) { if (req.startsWith("poetry-core") && req.includes(">=2.0")) { isPoetryV2 = true; break; @@ -13529,9 +13529,9 @@ export function getPipFrozenTree( }); thoughtLog("Performing poetry install"); let poetryInstallArgs = ["-m", "poetry", "install", "-n", "--no-root"]; - const isPoetryV2 = parsePyProjectTomlFile( - join(basePath, "pyproject.toml"), - ).isPoetryV2; + + const pyprojectpath = safeExistsSync(join(basePath, "pyproject.toml")); + const isPoetryV2 = parsePyProjectTomlFile(pyprojectpath).isPoetryV2; // checking if poetryV2 is true or not if (isPoetryV2) { // Include all dependency groups and extras (Poetry v2+) diff --git a/lib/helpers/utils.test.js b/lib/helpers/utils.test.js index 3333ab5211..1d2a98dcee 100644 --- a/lib/helpers/utils.test.js +++ b/lib/helpers/utils.test.js @@ -4975,29 +4975,33 @@ test("parse pyproject.toml", () => { test("parse pyproject.toml with poetryv2 requirement", () => { const retMap = parsePyProjectTomlFile("./test/data/pyproject_poetryv2.toml"); - // expect(retMap.parentComponent).toEqual({ - // name: "cpggen", - // version: "1.9.0", - // description: - // "Generate CPG for multiple languages for code and threat analysis", - // license: "Apache-2.0", - // author: "Team AppThreat ", - // homepage: { url: "https://github.com/AppThreat/cpggen" }, - // repository: { url: "https://github.com/AppThreat/cpggen" }, - // tags: [ - // "atom", - // "code analysis", - // "code property graph", - // "cpg", - // "joern", - // "static analysis", - // "threat analysis", - // ], - // type: "application", - // "bom-ref": "pkg:pypi/cpggen@1.9.0", - // purl: "pkg:pypi/cpggen@1.9.0", - // evidence: { identity: { field: "purl", confidence: 1, methods: [Array] } }, - // }); + expect(retMap.parentComponent).toEqual({ + name: "blint", + version: "2.4.2", + description: "Linter and SBOM generator for binary files.", + license: "MIT", + authors: [{ name: "Team AppThreat", email: "cloud@appthreat.com" }], + homepage: { url: "https://github.com/owasp-dep-scan/blint" }, + repository: { url: "https://github.com/owasp-dep-scan/blint" }, + tags: ["binary", "linter", "sast", "security"], + properties: [{ name: "cdx:pypi:requiresPython", value: ">=3.10,<3.14" }], + type: "application", + "bom-ref": "pkg:pypi/blint@2.4.2", + purl: "pkg:pypi/blint@2.4.2", + evidence: { + identity: { + field: "purl", + confidence: 1, + methods: [ + { + technique: "manifest-analysis", + confidence: 1, + value: "./test/data/pyproject_poetryv2.toml", + }, + ], + }, + }, + }); expect(retMap.isPoetryV2).toBeTruthy(); }); diff --git a/test/data/pyproject_poetryv2.toml b/test/data/pyproject_poetryv2.toml index 1b3fdc52a0..10e1a505a0 100644 --- a/test/data/pyproject_poetryv2.toml +++ b/test/data/pyproject_poetryv2.toml @@ -1,52 +1,81 @@ -[tool.poetry] -name = "cpggen" -version = "1.9.0" # 1.9.0 is not version 2.0.0 -description = "Generate CPG for multiple languages for code and threat analysis" -authors = ["Team AppThreat "] -license = "Apache-2.0" +[project] +name = "blint" +version = "2.4.2" +description = "Linter and SBOM generator for binary files." +authors = [ + {name= "Team AppThreat", email = "cloud@appthreat.com"}, +] +dependencies = [ + "lief>=0.16.6", + "rich>=14.0.0", + "PyYAML>=6.0.2", + "defusedxml>=0.7.1", + "pydantic[email]>=2.11.3", + "orjson>=3.10.16", + "symbolic==10.2.1", + "ar>=1.0.0", + "custom-json-diff>=2.1.6", + "appdirs>=1.4.4", + "apsw>=3.49.1.0", + "packageurl-python>=0.16.0", + "oras>=0.2.28", +] +license = "MIT" readme = "README.md" -packages = [{include = "cpggen"}] -homepage = "https://github.com/AppThreat/cpggen" -repository = "https://github.com/AppThreat/cpggen" -keywords = ["joern", "code analysis", "static analysis", "cpg", "code property graph", "atom", "threat analysis"] +homepage = "https://github.com/owasp-dep-scan/blint" +repository = "https://github.com/owasp-dep-scan/blint" +keywords = ["linter", "binary", "security", "sast"] classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Intended Audience :: System Administrators", "Topic :: Utilities", "Topic :: Security", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Operating System :: OS Independent", ] -exclude = ["contrib", "tests"] -include = ["cpggen/atom/*"] - -[tool.poetry.scripts] -atomgen = 'cpggen.cli:main' -cpggen = 'cpggen.cli:main' -cpg = 'cpggen.cli:main' - -[tool.poetry.dependencies] -python = ">=3.8.1,<3.12" -rich = "^13.4.2" -gitpython = "^3.1.31" -quart = "^0.18.4" -psutil = "^5.9.5" -packageurl-python = "^0.11.1" -httpx = "^0.24.1" - -[tool.poetry.group.dev.dependencies] -pytest = "^7.4.0" -black = "^23.3.0" -flake8 = "^6.0.0" -pytest-cov = "^4.0.0" -pyinstaller = "^5.12.0" -bandit = "^1.7.5" -pylint = "^2.17.4" +requires-python = ">=3.10,<3.14" +include = ["blint/data/*.yml"] + +[project.urls] +"CI" = "https://github.com/owasp-dep-scan/blint/actions" + +[project.scripts] +blint = 'blint.cli:main' + +[project.optional-dependencies] +dev = [ + "pytest>=8.3.5", + "black>=25.1.0", + "flake8>=7.2.0", + "pylint>=3.3.6", + "pytest-cov>=6.1.1", + "pyinstaller>=6.12.0" +] + +[tool.black] +line-length = 99 [build-system] requires = ["poetry-core>=2.0.0"] build-backend = "poetry.core.masonry.api" + +[tool.pytest.ini_options] +addopts = "--verbose --cov-append --cov-report term --cov blint" + +[tool.pylint] +generated-members = ["lief", "orjson"] +ignore-paths = ["blint/cyclonedx/*", "tests/*"] +# Let's not fuss about long strings +ignore-long-lines = "[r|f]\"" +disable = ["missing-module-docstring", "logging-fstring-interpolation"] + +[tool.pylint.format] +max-line-length = 99 + +[tool.pylint.design] +max-args = 6 +max-nested-blocks = 6 \ No newline at end of file From 017cba3f41e8821554aaf23f68720e55cbc856c0 Mon Sep 17 00:00:00 2001 From: ambuj Date: Mon, 16 Jun 2025 19:28:57 +0530 Subject: [PATCH 3/3] Refactor utils.js and utils.test.js - Changed the isPoetryV2 variable name to poetryV2Mode - removed try catch block while assigning poetryV2Mode in pipFrozenTree function - Added few comments Signed-off-by: ambuj --- lib/helpers/utils.js | 23 ++++++++++++++++------- lib/helpers/utils.test.js | 2 +- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/lib/helpers/utils.js b/lib/helpers/utils.js index c522c56270..7a69ce04de 100644 --- a/lib/helpers/utils.js +++ b/lib/helpers/utils.js @@ -4902,7 +4902,7 @@ export function parsePyProjectTomlFile(tomlFile) { } } - let isPoetryV2 = false; + let poetryV2Mode = false; let poetryMode = false; let uvMode = false; let hatchMode = false; @@ -4925,11 +4925,12 @@ export function parsePyProjectTomlFile(tomlFile) { ) { poetryMode = true; } + const buildRequires = tomlData?.["build-system"]?.["requires"]; if (buildRequires && Array.isArray(buildRequires)) { for (const req of buildRequires) { if (req.startsWith("poetry-core") && req.includes(">=2.0")) { - isPoetryV2 = true; + poetryV2Mode = true; break; } } @@ -5038,7 +5039,7 @@ export function parsePyProjectTomlFile(tomlFile) { return { parentComponent: pkg, poetryMode, - isPoetryV2, + poetryV2Mode, uvMode, hatchMode, workspacePaths, @@ -13539,10 +13540,18 @@ export function getPipFrozenTree( thoughtLog("Performing poetry install"); let poetryInstallArgs = ["-m", "poetry", "install", "-n", "--no-root"]; - const pyprojectpath = safeExistsSync(join(basePath, "pyproject.toml")); - const isPoetryV2 = parsePyProjectTomlFile(pyprojectpath).isPoetryV2; + // Storing pyProjectpath after checking the existence of pyproject.toml file + const pyprojectpath = safeExistsSync(join(basePath, "pyproject.toml")) + ? join(basePath, "pyproject.toml") + : null; + let poetryV2Mode = false; + // Check if the pyproject.toml is not null + if (pyprojectpath) { + poetryV2Mode = + parsePyProjectTomlFile(pyprojectpath).poetryV2Mode || false; + } // checking if poetryV2 is true or not - if (isPoetryV2) { + if (poetryV2Mode) { // Include all dependency groups and extras (Poetry v2+) poetryInstallArgs.push("--all-groups", "--all-extras"); } @@ -13561,7 +13570,7 @@ export function getPipFrozenTree( ); poetryInstallArgs = ["install", "-n", "--no-root"]; - if (isPoetryV2) { + if (poetryV2Mode) { // Also include flags when calling poetry directly poetryInstallArgs.push("--all-groups", "--all-extras"); } diff --git a/lib/helpers/utils.test.js b/lib/helpers/utils.test.js index b8e38a45fa..5fb5560c48 100644 --- a/lib/helpers/utils.test.js +++ b/lib/helpers/utils.test.js @@ -5006,7 +5006,7 @@ test("parse pyproject.toml with poetryv2 requirement", () => { }, }, }); - expect(retMap.isPoetryV2).toBeTruthy(); + expect(retMap.poetryV2Mode).toBeTruthy(); }); test("parse pyproject.toml with custom poetry source", () => {