Merge pull request #67 from nexB/metadata

Add metadata for packages

Author: TG1999

src/python_inspector/package_data.py

@@ -0,0 +1,155 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/python-inspector for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+from typing import List
+
+from packageurl import PackageURL
+
+from _packagedcode import models
+from _packagedcode.models import PackageData
+from python_inspector import utils_pypi
+from python_inspector.resolution import get_python_version_from_env_tag
+from python_inspector.utils_pypi import Environment
+from python_inspector.utils_pypi import PypiSimpleRepository
+
+
+def get_pypi_bugtracker_url(project_urls):
+    bug_tracking_url = project_urls.get("Tracker")
+    if not (bug_tracking_url):
+        bug_tracking_url = project_urls.get("Issue Tracker")
+    if not (bug_tracking_url):
+        bug_tracking_url = project_urls.get("Bug Tracker")
+    return bug_tracking_url
+
+
+def get_pypi_codeview_url(project_urls):
+    code_view_url = project_urls.get("Source")
+    if not (code_view_url):
+        code_view_url = project_urls.get("Code")
+    if not (code_view_url):
+        code_view_url = project_urls.get("Source Code")
+    return code_view_url
+
+
+def get_wheel_download_urls(
+    purl: PackageURL,
+    repos: List[PypiSimpleRepository],
+    environment: Environment,
+    python_version: str,
+) -> List[str]:
+    """
+    Return a list of download urls for the given purl.
+    """
+    for repo in repos:
+        for wheel in utils_pypi.get_supported_and_valid_wheels(
+            repo=repo,
+            name=purl.name,
+            version=purl.version,
+            environment=environment,
+            python_version=python_version,
+        ):
+            yield wheel.download_url
+
+
+def get_sdist_download_url(
+    purl: PackageURL, repos: List[PypiSimpleRepository], python_version: str
+) -> str:
+    """
+    Return a list of download urls for the given purl.
+    """
+    for repo in repos:
+        sdist = utils_pypi.get_valid_sdist(
+            repo=repo,
+            name=purl.name,
+            version=purl.version,
+            python_version=python_version,
+        )
+        if sdist:
+            return sdist.download_url
+
+
+def get_pypi_data_from_purl(
+    purl: str, environment: Environment, repos: List[PypiSimpleRepository]
+) -> PackageData:
+    """
+    Generate `Package` object from the `purl` string of npm type
+    """
+    purl = PackageURL.from_string(purl)
+    name = purl.name
+    version = purl.version
+    if not version:
+        raise Exception("Version is not specified in the purl")
+    base_path = "https://pypi.org/pypi"
+    api_url = f"{base_path}/{name}/{version}/json"
+    from python_inspector.resolution import get_response
+
+    response = get_response(api_url)
+    info = response.get("info") or {}
+    homepage_url = info.get("home_page")
+    license = info.get("license")
+    project_urls = info.get("project_urls") or {}
+    code_view_url = get_pypi_codeview_url(project_urls)
+    bug_tracking_url = get_pypi_bugtracker_url(project_urls)
+    python_version = get_python_version_from_env_tag(python_version=environment.python_version)
+    valid_distribution_urls = []
+    valid_distribution_urls.extend(
+        list(
+            get_wheel_download_urls(
+                purl=purl,
+                repos=repos,
+                environment=environment,
+                python_version=python_version,
+            )
+        )
+    )
+    valid_distribution_urls.append(
+        get_sdist_download_url(
+            purl=purl,
+            repos=repos,
+            python_version=python_version,
+        )
+    )
+    urls = response.get("urls") or []
+    for url in urls:
+        dist_url = url.get("url")
+        if dist_url not in valid_distribution_urls:
+            continue
+        digests = url.get("digests") or {}
+        yield PackageData(
+            primary_language="Python",
+            description=info.get("description"),
+            homepage_url=homepage_url,
+            api_data_url=api_url,
+            bug_tracking_url=bug_tracking_url,
+            code_view_url=code_view_url,
+            declared_license=license,
+            download_url=dist_url,
+            size=url.get("size"),
+            md5=digests.get("md5") or url.get("md5_digest"),
+            sha256=digests.get("sha256"),
+            release_date=url.get("upload_time"),
+            keywords=info.get("keywords") or [],
+            parties=[
+                models.Party(
+                    type=models.party_person,
+                    name=info.get("author"),
+                    role="author",
+                    email=info.get("author_email"),
+                ),
+                models.Party(
+                    type=models.party_person,
+                    name=info.get("maintainer"),
+                    role="maintainer",
+                    email=info.get("maintainer_email"),
+                ),
+            ],
+            **purl.to_dict(),
+        ).to_dict()

src/python_inspector/resolution.py

@@ -185,6 +185,10 @@ def fetch_and_extract_sdist(
     if not sdist:
         return
 
+    return get_sdist_file_path_from_filename(sdist)
+
+
+def get_sdist_file_path_from_filename(sdist):
     if sdist.endswith(".tar.gz"):
         sdist_file = sdist.rstrip(".tar.gz")
         with tarfile.open(os.path.join(utils_pypi.CACHE_THIRDPARTY_DIR, sdist)) as file:
@@ -625,8 +629,6 @@ def format_resolution(
             parent_children = dict(
                 package=str(parent_purl),
                 dependencies=dependencies,
-                wheel_urls=list(dict.fromkeys(wheel_urls)),
-                sdist_url=sdist_url,
             )
             as_parent_children.append(parent_children)
         as_parent_children.sort(key=lambda d: d["package"])
@@ -678,6 +680,31 @@ def format_pdt_tree(results):
     return dependencies
 
 
+def get_package_list(results):
+    """
+    Return a list of packages in the resolution.
+    """
+    mapping = results.mapping
+    graph = results.graph
+    parents = mapping.keys()
+    packages = set()
+    for parent in parents:
+        parent_purl = PackageURL(
+            type="pypi",
+            name=parent,
+            version=str(mapping[parent].version),
+        )
+        packages.add(str(parent_purl))
+        for dependency in graph.iter_children(parent):
+            dep_purl = PackageURL(
+                type="pypi",
+                name=dependency,
+                version=str(mapping[dependency].version),
+            )
+            packages.add(str(dep_purl))
+    return list(sorted(packages))
+
+
 def get_resolved_dependencies(
     requirements: List[Requirement],
     environment: Environment = None,
@@ -701,10 +728,14 @@ def get_resolved_dependencies(
             reporter=BaseReporter(),
         )
         resolver_results = resolver.resolve(requirements=requirements, max_rounds=max_rounds)
+        package_list = get_package_list(results=resolver_results)
         if pdt_output:
-            return format_pdt_tree(resolver_results)
-        return format_resolution(
-            resolver_results, as_tree=as_tree, environment=environment, repos=repos
+            return (format_pdt_tree(resolver_results), package_list)
+        return (
+            format_resolution(
+                resolver_results, as_tree=as_tree, environment=environment, repos=repos
+            ),
+            package_list,
         )
     except Exception as e:
         if verbose:

src/python_inspector/resolve_cli.py

@@ -25,6 +25,7 @@
 from python_inspector import utils
 from python_inspector import utils_pypi
 from python_inspector.cli_utils import FileOptionType
+from python_inspector.package_data import get_pypi_data_from_purl
 from python_inspector.resolution import get_environment_marker_from_environment
 from python_inspector.resolution import get_python_version_from_env_tag
 from python_inspector.resolution import get_resolved_dependencies
@@ -321,7 +322,7 @@ def resolve_dependencies(
             click.secho(f" {repo}")
 
     # resolve dependencies proper
-    requirements, resolved_dependencies = resolve(
+    requirements, resolved_dependencies, purls = resolve(
         direct_dependencies=direct_dependencies,
         environment=environment,
         repos=repos,
@@ -354,12 +355,20 @@ def resolve_dependencies(
         errors=[],
     )
 
+    packages = []
+
+    for package in purls:
+        packages.extend(
+            list(get_pypi_data_from_purl(package, repos=repos, environment=environment)),
+        )
+
     if json_output:
         write_output(
             headers=headers,
             requirements=requirements,
             resolved_dependencies=resolved_dependencies,
             json_output=json_output,
+            packages=packages,
         )
 
     else:
@@ -368,6 +377,7 @@ def resolve_dependencies(
             requirements=requirements,
             resolved_dependencies=resolved_dependencies,
             json_output=pdt_output,
+            packages=packages,
             pdt_output=True,
         )
 
@@ -400,7 +410,7 @@ def resolve(
         )
     )
 
-    resolved_dependencies = get_resolved_dependencies(
+    resolved_dependencies, packages = get_resolved_dependencies(
         requirements=requirements,
         environment=environment,
         repos=repos,
@@ -412,7 +422,7 @@ def resolve(
 
     initial_requirements = [d.to_dict() for d in direct_dependencies]
 
-    return initial_requirements, resolved_dependencies
+    return initial_requirements, resolved_dependencies, packages
 
 
 def get_requirements_from_direct_dependencies(
@@ -432,19 +442,26 @@ def get_requirements_from_direct_dependencies(
                 yield req
 
 
-def write_output(headers, requirements, resolved_dependencies, json_output, pdt_output=False):
+def write_output(
+    headers, requirements, resolved_dependencies, json_output, packages, pdt_output=False
+):
     """
     Write headers, requirements and resolved_dependencies as JSON to ``json_output``.
     Return the output data.
     """
+
     if not pdt_output:
         output = dict(
             headers=headers,
             requirements=requirements,
             resolved_dependencies=resolved_dependencies,
+            packages=packages,
         )
     else:
-        output = resolved_dependencies
+        output = dict(
+            resolved_dependencies=resolved_dependencies,
+            packages=packages,
+        )
 
     json.dump(output, json_output, indent=2)
     return output

tests/data/default-url-expected.json

@@ -29,9 +29,8 @@
   "resolved_dependencies": [
     {
       "package": "pkg:pypi/[email protected]",
-      "dependencies": [],
-      "wheel_urls": [],
-      "sdist_url": null
+      "dependencies": []
     }
-  ]
+  ],
+  "packages": []
 }