Cleanup requirement string to avoid packvers parsing issues

Extract clean name and version from requirements instead of using
dumps() output that includes hash options and other pip-specific
syntax that `packvers.requirements.Requirement` cannot parse.

Resolves: #243.

Signed-off-by: Marcel Bochtler <[email protected]>

Author: MarcelBochtler

src/_packagedcode/pypi.py

@@ -534,7 +534,7 @@ def parse(cls, location):
                     for metapath in path.iterdir():
                         if not metapath.name.endswith('METADATA'):
                             continue
-    
+
                         yield parse_metadata(
                             location=metapath,
                             datasource_id=cls.datasource_id,
@@ -907,6 +907,37 @@ def parse(cls, location):
 # TODO: enable nested load
 
 
+def build_pep508_requirement_string(req):
+    """
+    Build a PEP 508 compliant requirement string based on https://peps.python.org/pep-0508.
+    The format follows the PEP 508 grammar: name [extras] version_spec ; marker
+    This excludes pip-specific options that are not part of PEP 508 like --hash or --editable.
+    But preserves standard PEP 508 requirement components:
+    - Package name (required)
+    - Version specifiers (>=, ==, etc.)
+    - Extras [extra1,extra2]
+    - Environment markers ; python_version >= "3.8"
+    """
+    if not req.name:
+        return ""
+
+    parts = [req.name]
+
+    # Add extras: package[extra1,extra2]
+    if req.extras:
+        parts.append(f"[{','.join(sorted(req.extras))}]")
+
+    # Add version specifiers: >=1.0,<2.0
+    if req.specifier:
+        parts.append(req.dumps_specifier())
+
+    # Add environment markers: ; python_version >= "3.8"
+    if req.marker:
+        parts.append(f"; {req.marker}")
+
+    return "".join(parts)
+
+
 def get_requirements_txt_dependencies(location, include_nested=False):
     """
     Return a two-tuple of (list of deps, mapping of extra data) list of
@@ -945,7 +976,7 @@ def get_requirements_txt_dependencies(location, include_nested=False):
 
         purl = purl and purl.to_string() or None
 
-        requirement = req.dumps()
+        requirement = build_pep508_requirement_string(req)
 
         if location.endswith(
             (

tests/data/hash-and-environment-markers-requirements.txt

@@ -0,0 +1,17 @@
+addict==2.4.0 \
+    --hash=sha256:249bb56bbfd3cdc2a004ea0ff4c2b6ddc84d53bc2194761636eb314d5cfa5dfc \
+    --hash=sha256:b3b2210e0e067a281f5646c8c5db92e99b7231ea8b0eb5f74dbdf9e259d4e494
+
+# Package with environment marker and no hashes
+license-expression ; platform_system == "Windows"
+
+# Package with both hash and environment marker
+requests==2.25.1 ; python_version >= "3.6" \
+    --hash=sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e \
+    --hash=sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804
+
+# Package with complex specifiers and extras
+click[unicode]>=6.0,<7.0 ; python_version < "3.8"
+
+# Package with multiple extras and environment marker
+flask[async,dotenv]>=1.0 ; python_version >= "3.7" and platform_machine == "x86_64"

tests/data/hash-requirements.txt

@@ -0,0 +1,7 @@
+addict==2.4.0 \
+    --hash=sha256:249bb56bbfd3cdc2a004ea0ff4c2b6ddc84d53bc2194761636eb314d5cfa5dfc \
+    --hash=sha256:b3b2210e0e067a281f5646c8c5db92e99b7231ea8b0eb5f74dbdf9e259d4e494
+
+requests==2.25.1 \
+    --hash=sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e \
+    --hash=sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804

tests/data/hash-requirements.txt-expected.json

@@ -414,6 +414,17 @@ def test_cli_with_pinned_requirements_file():
     )
 
 
+@pytest.mark.online
+def test_cli_with_hash_requirements():
+    requirements_file = test_env.get_test_loc("hash-requirements.txt")
+    expected_file = test_env.get_test_loc("hash-requirements.txt-expected.json", must_exist=False)
+    check_requirements_resolution(
+        requirements_file=requirements_file,
+        expected_file=expected_file,
+        regen=REGEN_TEST_FIXTURES,
+    )
+
+
 @pytest.mark.online
 def test_cli_with_setup_py_failure():
     setup_py_file = setup_test_env.get_test_loc("simple-setup.py")

tests/test_cli.py

@@ -11,6 +11,9 @@
 
 from _packagedcode import models
 from _packagedcode.pypi import can_process_dependent_package
+from _packagedcode.pypi import build_pep508_requirement_string
+from pip_requirements_parser import InstallRequirement, RequirementLine
+from packaging.requirements import Requirement
 
 
 def test_can_process_dependent_package():
@@ -99,3 +102,35 @@ def test_can_not_process_dependent_package_with_any_flags_set():
     )
 
     assert not can_process_dependent_package(dependency)
+
+
+def create_requirement(requirement_string):
+    req_line = RequirementLine(requirement_string, 1, "test.txt")
+    requirement = Requirement(requirement_string)
+    return InstallRequirement(requirement, req_line)
+
+
+def test_build_pep508_requirement_string():
+    req = create_requirement("django")
+    result = build_pep508_requirement_string(req)
+    assert result == "django"
+
+    req = create_requirement("requests>=2.25.0")
+    result = build_pep508_requirement_string(req)
+    assert result == "requests>=2.25.0"
+
+    req = create_requirement("pytest[coverage,xdist]")
+    result = build_pep508_requirement_string(req)
+    assert (result == "pytest[coverage,xdist]")
+
+    req = create_requirement('pywin32; sys_platform == "win32"')
+    result = build_pep508_requirement_string(req)
+    assert result == 'pywin32; sys_platform == "win32"'
+
+    req = create_requirement('django[admin,auth]>=3.2,<4.0; python_version >= "3.8"')
+    result = build_pep508_requirement_string(req)
+    assert result == 'django[admin,auth]>=3.2,<4.0; python_version >= "3.8"'
+
+    req = create_requirement("mypackage")
+    result = build_pep508_requirement_string(req)
+    assert result == "mypackage"