From 0ce7c4f95a4a95514b931911d2442ff4caa54412 Mon Sep 17 00:00:00 2001 From: Michael Ehab Mikhail Date: Thu, 21 Aug 2025 16:03:50 +0300 Subject: [PATCH 1/2] Add Postgres Live V2 Importer Pipeline #1980 * Add Postgres Live V2 Importer * Add tests for the Postgres Live V2 Importer * Tested functionally using the Live Evaluation API in #1969 Signed-off-by: Michael Ehab Mikhail --- vulnerabilities/importers/__init__.py | 9 ++ .../v2_importers/postgresql_live_importer.py | 102 ++++++++++++++++++ .../test_postgresql_live_importer_v2.py | 102 ++++++++++++++++++ 3 files changed, 213 insertions(+) create mode 100644 vulnerabilities/pipelines/v2_importers/postgresql_live_importer.py create mode 100644 vulnerabilities/tests/pipelines/v2_importers/test_postgresql_live_importer_v2.py diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index c0cf04ed7..8b5f623b8 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -72,6 +72,9 @@ from vulnerabilities.pipelines.v2_importers import ( project_kb_statements_importer as project_kb_statements_importer_v2, ) +from vulnerabilities.pipelines.v2_importers import ( + postgresql_live_importer as postgresql_live_importer_v2, +) from vulnerabilities.pipelines.v2_importers import pypa_importer as pypa_importer_v2 from vulnerabilities.pipelines.v2_importers import pysec_importer as pysec_importer_v2 from vulnerabilities.pipelines.v2_importers import redhat_importer as redhat_importer_v2 @@ -196,3 +199,9 @@ for key, value in IMPORTERS_REGISTRY.items() if issubclass(value, VulnerableCodeBaseImporterPipelineV2) and value.exclude_from_package_todo ] + +LIVE_IMPORTERS_REGISTRY = create_registry( + [ + postgresql_live_importer_v2.PostgreSQLLiveImporterPipeline, + ] +) diff --git a/vulnerabilities/pipelines/v2_importers/postgresql_live_importer.py b/vulnerabilities/pipelines/v2_importers/postgresql_live_importer.py new file mode 100644 index 000000000..d1436aac2 --- /dev/null +++ b/vulnerabilities/pipelines/v2_importers/postgresql_live_importer.py @@ -0,0 +1,102 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# + +import logging +from typing import Iterable + +from packageurl import PackageURL +from univers.versions import GenericVersion +from univers.versions import SemverVersion + +from vulnerabilities.importer import AdvisoryData +from vulnerabilities.pipelines.v2_importers.postgresql_importer import PostgreSQLImporterPipeline + +logger = logging.getLogger(__name__) + + +class PostgreSQLLiveImporterPipeline(PostgreSQLImporterPipeline): + """ + Live importer for PostgreSQL that filters the batch output to a single PURL. + """ + + pipeline_id = "postgresql_live_importer_v2" + supported_types = ["generic"] + + @classmethod + def steps(cls): + return ( + cls.get_purl_inputs, + cls.collect_and_store_advisories, + ) + + def get_purl_inputs(self): + purl = self.inputs.get("purl") + if not purl: + raise ValueError("PURL is required for PostgreSQLLiveImporterPipeline") + + if isinstance(purl, str): + purl = PackageURL.from_string(purl) + + if not isinstance(purl, PackageURL): + raise ValueError(f"Object of type {type(purl)} {purl!r} is not a PackageURL instance") + + if purl.type not in self.supported_types: + raise ValueError( + f"PURL: {purl!s} is not among the supported package types {self.supported_types!r}" + ) + + if purl.name != "postgresql": + raise ValueError(f"PURL: {purl!s} is expected to be for 'postgresql'") + + if not purl.version: + raise ValueError(f"PURL: {purl!s} is expected to have a version") + + self.purl = purl + + def collect_advisories(self) -> Iterable[AdvisoryData]: + for advisory in super().collect_advisories(): + if self._advisory_affects_purl(advisory): + yield advisory + + def _advisory_affects_purl(self, advisory: AdvisoryData) -> bool: + if not advisory.affected_packages: + return False + + try: + package_semver_version = SemverVersion(self.purl.version) + package_generic_version = GenericVersion(self.purl.version) + except Exception as e: + logger.debug(f"Invalid PURL version {self.purl.version!r}: {e}") + return False + + for ap in advisory.affected_packages: + if ap.package.type != "generic" or ap.package.name != "postgresql": + continue + + purl_q = self.purl.qualifiers or None + ap_q = ap.package.qualifiers or None + + if purl_q is None and ap_q is None: + qualifiers_match = True + else: + qualifiers_match = all(ap_q.get(k) == v for k, v in purl_q.items()) + + if not qualifiers_match: + continue + + try: + if getattr(ap, "affected_version_range", None): + if package_semver_version in ap.affected_version_range: + return True + elif getattr(ap, "fixed_version", None): + if package_generic_version < ap.fixed_version: + return True + except Exception as e: + logger.debug(f"Version comparison failed for {package_semver_version}: {e}") + continue + + return False diff --git a/vulnerabilities/tests/pipelines/v2_importers/test_postgresql_live_importer_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_postgresql_live_importer_v2.py new file mode 100644 index 000000000..6cbe3bd2f --- /dev/null +++ b/vulnerabilities/tests/pipelines/v2_importers/test_postgresql_live_importer_v2.py @@ -0,0 +1,102 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# + +import pytest +import requests +from packageurl import PackageURL + +from vulnerabilities.importer import AdvisoryData +from vulnerabilities.pipelines.v2_importers.postgresql_live_importer import ( + PostgreSQLLiveImporterPipeline, +) + +HTML_BASE = """ + + + + + + + + + + + + +
+ CVE-2022-1234
+ Announcement
+ 		{affected}{fixed}9.8{summary}
+ + +""" + + +class DummyResponse: + def __init__(self, content): + self.content = content.encode("utf-8") + + +def test_affected_version(monkeypatch): + html = HTML_BASE.format(affected="10.0, 10.1", fixed="10.2", summary="Issue affects all") + monkeypatch.setattr(requests, "get", lambda url: DummyResponse(html)) + + purl = PackageURL(type="generic", name="postgresql", version="10.1") + pipeline = PostgreSQLLiveImporterPipeline(purl=purl) + pipeline.get_purl_inputs() + advisories = list(pipeline.collect_advisories()) + + assert len(advisories) == 1 + adv = advisories[0] + assert isinstance(adv, AdvisoryData) + assert adv.advisory_id == "CVE-2022-1234" + + +def test_unaffected_version(monkeypatch): + html = HTML_BASE.format(affected="10.0, 10.1", fixed="10.2", summary="Issue affects all") + monkeypatch.setattr(requests, "get", lambda url: DummyResponse(html)) + + purl = PackageURL(type="generic", name="postgresql", version="10.2") + pipeline = PostgreSQLLiveImporterPipeline(purl=purl) + pipeline.get_purl_inputs() + advisories = list(pipeline.collect_advisories()) + + assert len(advisories) == 0 + + +def test_qualifier_filtering(monkeypatch): + html = HTML_BASE.format(affected="12.0, 12.1", fixed="12.2", summary="Windows-specific issue") + monkeypatch.setattr(requests, "get", lambda url: DummyResponse(html)) + + purl = PackageURL( + type="generic", name="postgresql", version="12.1", qualifiers={"os": "windows"} + ) + pipeline = PostgreSQLLiveImporterPipeline(purl=purl) + pipeline.get_purl_inputs() + advisories = list(pipeline.collect_advisories()) + assert len(advisories) == 1 + + purl = PackageURL(type="generic", name="postgresql", version="12.1", qualifiers={"os": "linux"}) + pipeline = PostgreSQLLiveImporterPipeline(purl=purl) + pipeline.get_purl_inputs() + advisories = list(pipeline.collect_advisories()) + assert len(advisories) == 0 + + +def test_invalid_purl(): + pipeline = PostgreSQLLiveImporterPipeline() + + pipeline.inputs = {"purl": "pkg:pypi/postgresql@10.1"} + with pytest.raises(ValueError): + pipeline.get_purl_inputs() + + pipeline.inputs = {"purl": "pkg:generic/notpostgresql@10.1"} + with pytest.raises(ValueError): + pipeline.get_purl_inputs() + + pipeline.inputs = {"purl": "pkg:generic/postgresql"} + with pytest.raises(ValueError): + pipeline.get_purl_inputs() From 6fcb0683e581743e7d5090e4ad5a1636747eca0b Mon Sep 17 00:00:00 2001 From: ziad hany Date: Mon, 11 May 2026 01:24:14 +0300 Subject: [PATCH 2/2] Refactor the code for postgresql_live importer Signed-off-by: ziad hany --- vulnerabilities/importers/__init__.py | 6 +- .../v2_importers/postgresql_live_importer.py | 47 +++-------- .../test_postgresql_live_importer_v2.py | 81 +++++++++++-------- 3 files changed, 61 insertions(+), 73 deletions(-) diff --git a/vulnerabilities/importers/__init__.py b/vulnerabilities/importers/__init__.py index 8b5f623b8..3668a7662 100644 --- a/vulnerabilities/importers/__init__.py +++ b/vulnerabilities/importers/__init__.py @@ -67,13 +67,13 @@ from vulnerabilities.pipelines.v2_importers import oss_fuzz as oss_fuzz_v2 from vulnerabilities.pipelines.v2_importers import postgresql_importer as postgresql_importer_v2 from vulnerabilities.pipelines.v2_importers import ( - project_kb_msr2019_importer as project_kb_msr2019_importer_v2, + postgresql_live_importer as postgresql_live_importer_v2, ) from vulnerabilities.pipelines.v2_importers import ( - project_kb_statements_importer as project_kb_statements_importer_v2, + project_kb_msr2019_importer as project_kb_msr2019_importer_v2, ) from vulnerabilities.pipelines.v2_importers import ( - postgresql_live_importer as postgresql_live_importer_v2, + project_kb_statements_importer as project_kb_statements_importer_v2, ) from vulnerabilities.pipelines.v2_importers import pypa_importer as pypa_importer_v2 from vulnerabilities.pipelines.v2_importers import pysec_importer as pysec_importer_v2 diff --git a/vulnerabilities/pipelines/v2_importers/postgresql_live_importer.py b/vulnerabilities/pipelines/v2_importers/postgresql_live_importer.py index d1436aac2..dab92163d 100644 --- a/vulnerabilities/pipelines/v2_importers/postgresql_live_importer.py +++ b/vulnerabilities/pipelines/v2_importers/postgresql_live_importer.py @@ -9,10 +9,10 @@ from typing import Iterable from packageurl import PackageURL -from univers.versions import GenericVersion +from univers.versions import InvalidVersion from univers.versions import SemverVersion -from vulnerabilities.importer import AdvisoryData +from vulnerabilities.importer import AdvisoryDataV2 from vulnerabilities.pipelines.v2_importers.postgresql_importer import PostgreSQLImporterPipeline logger = logging.getLogger(__name__) @@ -49,54 +49,29 @@ def get_purl_inputs(self): f"PURL: {purl!s} is not among the supported package types {self.supported_types!r}" ) - if purl.name != "postgresql": - raise ValueError(f"PURL: {purl!s} is expected to be for 'postgresql'") - if not purl.version: raise ValueError(f"PURL: {purl!s} is expected to have a version") - self.purl = purl - def collect_advisories(self) -> Iterable[AdvisoryData]: + def collect_advisories(self) -> Iterable[AdvisoryDataV2]: for advisory in super().collect_advisories(): - if self._advisory_affects_purl(advisory): + if self._advisory_related_purl(advisory): yield advisory - def _advisory_affects_purl(self, advisory: AdvisoryData) -> bool: + def _advisory_related_purl(self, advisory: AdvisoryDataV2) -> bool: if not advisory.affected_packages: return False try: - package_semver_version = SemverVersion(self.purl.version) - package_generic_version = GenericVersion(self.purl.version) - except Exception as e: + package_version = SemverVersion(self.purl.version) + except InvalidVersion as e: logger.debug(f"Invalid PURL version {self.purl.version!r}: {e}") return False for ap in advisory.affected_packages: - if ap.package.type != "generic" or ap.package.name != "postgresql": - continue - - purl_q = self.purl.qualifiers or None - ap_q = ap.package.qualifiers or None - - if purl_q is None and ap_q is None: - qualifiers_match = True - else: - qualifiers_match = all(ap_q.get(k) == v for k, v in purl_q.items()) - - if not qualifiers_match: - continue - - try: - if getattr(ap, "affected_version_range", None): - if package_semver_version in ap.affected_version_range: - return True - elif getattr(ap, "fixed_version", None): - if package_generic_version < ap.fixed_version: - return True - except Exception as e: - logger.debug(f"Version comparison failed for {package_semver_version}: {e}") - continue + if (ap.affected_version_range and package_version in ap.affected_version_range) or ( + ap.fixed_version_range and package_version in ap.fixed_version_range + ): + return True return False diff --git a/vulnerabilities/tests/pipelines/v2_importers/test_postgresql_live_importer_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_postgresql_live_importer_v2.py index 6cbe3bd2f..0fa111516 100644 --- a/vulnerabilities/tests/pipelines/v2_importers/test_postgresql_live_importer_v2.py +++ b/vulnerabilities/tests/pipelines/v2_importers/test_postgresql_live_importer_v2.py @@ -8,7 +8,6 @@ import requests from packageurl import PackageURL -from vulnerabilities.importer import AdvisoryData from vulnerabilities.pipelines.v2_importers.postgresql_live_importer import ( PostgreSQLLiveImporterPipeline, ) @@ -48,18 +47,59 @@ def test_affected_version(monkeypatch): pipeline = PostgreSQLLiveImporterPipeline(purl=purl) pipeline.get_purl_inputs() advisories = list(pipeline.collect_advisories()) - - assert len(advisories) == 1 - adv = advisories[0] - assert isinstance(adv, AdvisoryData) - assert adv.advisory_id == "CVE-2022-1234" + assert [adv.to_dict() for adv in advisories] == [ + { + "advisory_id": "CVE-2022-1234", + "affected_packages": [ + { + "affected_version_range": "vers:generic/10.0.0|10.1.0", + "fixed_by_commit_patches": [], + "fixed_version_range": "vers:generic/10.2.0", + "introduced_by_commit_patches": [], + "package": { + "name": "postgresql", + "namespace": "", + "qualifiers": "", + "subpath": "", + "type": "generic", + "version": "", + }, + } + ], + "aliases": [], + "date_published": None, + "patches": [], + "references": [ + { + "reference_id": "", + "reference_type": "", + "url": "https://www.postgresql.org/support/security/CVE-2022-1234/", + }, + { + "reference_id": "", + "reference_type": "", + "url": "https://www.postgresql.org/about/news/postgresql-175-169-1513-1418-and-1321-released-3072/", + }, + ], + "severities": [ + { + "scoring_elements": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", + "system": "cvssv3", + "value": "9.8", + } + ], + "summary": "Issue affects all", + "url": "https://www.postgresql.org/support/security/", + "weaknesses": [], + } + ] def test_unaffected_version(monkeypatch): html = HTML_BASE.format(affected="10.0, 10.1", fixed="10.2", summary="Issue affects all") monkeypatch.setattr(requests, "get", lambda url: DummyResponse(html)) - purl = PackageURL(type="generic", name="postgresql", version="10.2") + purl = PackageURL(type="generic", name="postgresql", version="14.3") pipeline = PostgreSQLLiveImporterPipeline(purl=purl) pipeline.get_purl_inputs() advisories = list(pipeline.collect_advisories()) @@ -67,36 +107,9 @@ def test_unaffected_version(monkeypatch): assert len(advisories) == 0 -def test_qualifier_filtering(monkeypatch): - html = HTML_BASE.format(affected="12.0, 12.1", fixed="12.2", summary="Windows-specific issue") - monkeypatch.setattr(requests, "get", lambda url: DummyResponse(html)) - - purl = PackageURL( - type="generic", name="postgresql", version="12.1", qualifiers={"os": "windows"} - ) - pipeline = PostgreSQLLiveImporterPipeline(purl=purl) - pipeline.get_purl_inputs() - advisories = list(pipeline.collect_advisories()) - assert len(advisories) == 1 - - purl = PackageURL(type="generic", name="postgresql", version="12.1", qualifiers={"os": "linux"}) - pipeline = PostgreSQLLiveImporterPipeline(purl=purl) - pipeline.get_purl_inputs() - advisories = list(pipeline.collect_advisories()) - assert len(advisories) == 0 - - def test_invalid_purl(): pipeline = PostgreSQLLiveImporterPipeline() pipeline.inputs = {"purl": "pkg:pypi/postgresql@10.1"} with pytest.raises(ValueError): pipeline.get_purl_inputs() - - pipeline.inputs = {"purl": "pkg:generic/notpostgresql@10.1"} - with pytest.raises(ValueError): - pipeline.get_purl_inputs() - - pipeline.inputs = {"purl": "pkg:generic/postgresql"} - with pytest.raises(ValueError): - pipeline.get_purl_inputs()