diff --git a/README.md b/README.md index 63f4976..dd5c536 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,7 @@ The list of all available options that can be set as environmental variables is - `GITHUB_REPORT_SCOPE`: The scope of the report to generate. Valid values are `repository` (default), `organization` or `enterprise`. - `SCOPE_NAME` or `GITHUB_REPOSITORY`: The name of the repository, organization or enterprise to generate the report for. If `SCOPE_NAME` is not set, the value of `GITHUB_REPOSITORY` is used if it is set. If neither is set, an error occurs. - `FEATURES`: A comma-separated list of features to include in the report. Valid values are `codescanning`, `secretscanning`, `dependabot` or simply `all`. Default value: `all`. +- `INCLUDE_REPO_METADATA`: Include extended repository metadata (teams, topics, custom properties) in the CSV output. Valid values are `true` or `false`. Default value: `false`. **Warning**: Enabling this feature will significantly increase API calls and execution time at organization or enterprise scope. The first two are only needed if you're running this in a GitHub Enterprise Server or GitHub AE environment. The last one is useful if you only want to get data on a specific feature. For example, if you only want to get data on secret scanning, you can set `FEATURES` to `secretscanning`. Here's just another example how you would configure this on a GitHub Enterprise Server: @@ -92,6 +93,20 @@ The first two are only needed if you're running this in a GitHub Enterprise Serv FEATURES: "secretscanning,codescanning" ``` +To include extended repository metadata (teams, topics, and custom properties) in the CSV output: + +```yaml + - name: CSV export with extended metadata + uses: advanced-security/ghas-to-csv@v3 + env: + GITHUB_PAT: ${{ secrets.PAT }} + GITHUB_REPORT_SCOPE: "organization" + SCOPE_NAME: "org-name-goes-here" + INCLUDE_REPO_METADATA: "true" +``` + +⚠️ **Note**: The `INCLUDE_REPO_METADATA` feature will make additional API calls for each repository to fetch teams, topics, and custom properties. This can significantly increase execution time and API usage when used at organization or enterprise scope. + ## Reporting | | GitHub Enterprise Cloud | GitHub Enterprise Server (3.5+) | GitHub AE (M2) | Notes | diff --git a/main.py b/main.py index 5033214..4f55292 100755 --- a/main.py +++ b/main.py @@ -34,6 +34,8 @@ report_scope = os.getenv("GITHUB_REPORT_SCOPE", "repository") scope_name = os.getenv("SCOPE_NAME", os.getenv("GITHUB_REPOSITORY")) requested_features = os.getenv("FEATURES") +# Flag to enable extended repository metadata (teams, topics, properties) +include_repo_metadata = os.getenv("INCLUDE_REPO_METADATA", "false").lower() == "true" if (requested_features is None) or (requested_features == "all"): features = FEATURES else: @@ -53,7 +55,7 @@ if "secretscanning" in features: try: secrets_list = secret_scanning.get_enterprise_ss_alerts(api_endpoint, github_pat, scope_name) - secret_scanning.write_enterprise_ss_list(secrets_list) + secret_scanning.write_enterprise_ss_list(secrets_list, include_repo_metadata, api_endpoint, github_pat) except Exception as e: if any(x in str(e).lower() for x in secret_scanning_disabled_strings): print("Skipping Secret Scanning as it is not enabled.") @@ -69,15 +71,15 @@ if version.startswith("3.5") or version.startswith("3.6"): repo_list = enterprise.get_repo_report(url, github_pat) cs_list = code_scanning.list_enterprise_server_cs_alerts(api_endpoint, github_pat, repo_list) - code_scanning.write_enterprise_server_cs_list(cs_list) + code_scanning.write_enterprise_server_cs_list(cs_list, include_repo_metadata, api_endpoint, github_pat) else: cs_list = code_scanning.list_enterprise_cloud_cs_alerts(api_endpoint, github_pat, scope_name) - code_scanning.write_enterprise_cloud_cs_list(cs_list) + code_scanning.write_enterprise_cloud_cs_list(cs_list, include_repo_metadata, api_endpoint, github_pat) # dependabot alerts if "dependabot" in features: try: dependabot_list = dependabot.list_enterprise_dependabot_alerts(api_endpoint, github_pat, scope_name) - dependabot.write_org_or_enterprise_dependabot_list(dependabot_list) + dependabot.write_org_or_enterprise_dependabot_list(dependabot_list, include_repo_metadata, api_endpoint, github_pat) except Exception as e: if any(x in str(e).lower() for x in dependabot_disabled_strings): print("Skipping Dependabot as it is not enabled.") @@ -89,12 +91,12 @@ # code scanning if "codescanning" in features: cs_list = code_scanning.list_org_cs_alerts(api_endpoint, github_pat, scope_name) - code_scanning.write_org_cs_list(cs_list) + code_scanning.write_org_cs_list(cs_list, include_repo_metadata, api_endpoint, github_pat) # dependabot alerts if "dependabot" in features: try: dependabot_list = dependabot.list_org_dependabot_alerts(api_endpoint, github_pat, scope_name) - dependabot.write_org_or_enterprise_dependabot_list(dependabot_list) + dependabot.write_org_or_enterprise_dependabot_list(dependabot_list, include_repo_metadata, api_endpoint, github_pat) except Exception as e: if any(x in str(e).lower() for x in dependabot_disabled_strings): print("Skipping Dependabot as it is not enabled.") @@ -105,7 +107,7 @@ if "secretscanning" in features: try: secrets_list = secret_scanning.get_org_ss_alerts(api_endpoint, github_pat, scope_name) - secret_scanning.write_org_ss_list(secrets_list) + secret_scanning.write_org_ss_list(secrets_list, include_repo_metadata, api_endpoint, github_pat) except Exception as e: if any(x in str(e).lower() for x in secret_scanning_disabled_strings): print("Skipping Secret Scanning as it is not enabled.") @@ -117,12 +119,12 @@ # code scanning if "codescanning" in features: cs_list = code_scanning.list_repo_cs_alerts(api_endpoint, github_pat, scope_name) - code_scanning.write_repo_cs_list(cs_list) + code_scanning.write_repo_cs_list(cs_list, include_repo_metadata, api_endpoint, github_pat, scope_name) # dependabot alerts if "dependabot" in features: try: dependabot_list = dependabot.list_repo_dependabot_alerts(api_endpoint, github_pat, scope_name) - dependabot.write_repo_dependabot_list(dependabot_list) + dependabot.write_repo_dependabot_list(dependabot_list, include_repo_metadata, api_endpoint, github_pat, scope_name) except Exception as e: if any(x in str(e).lower() for x in dependabot_disabled_strings): print("Skipping Dependabot as it is not enabled.") @@ -133,7 +135,7 @@ if "secretscanning" in features: try: secrets_list = secret_scanning.get_repo_ss_alerts(api_endpoint, github_pat, scope_name) - secret_scanning.write_repo_ss_list(secrets_list) + secret_scanning.write_repo_ss_list(secrets_list, include_repo_metadata, api_endpoint, github_pat, scope_name) except Exception as e: if any(x in str(e).lower() for x in secret_scanning_disabled_strings): print("Skipping Secret Scanning as it is not enabled.") diff --git a/src/api_helpers.py b/src/api_helpers.py index 0b2fa6a..1f701a1 100644 --- a/src/api_helpers.py +++ b/src/api_helpers.py @@ -13,4 +13,66 @@ def make_api_call(url, github_pat): while "next" in response.links.keys(): response = requests.get(response.links["next"]["url"], headers=headers) response_json.extend(response.json()) - return response_json \ No newline at end of file + return response_json + + +def get_repo_metadata(api_endpoint, github_pat, repo_name): + """ + Get extended repository metadata including teams, topics, and custom properties. + + Inputs: + - API endpoint (for GHES/GHAE compatibility) + - PAT of appropriate scope + - Repository name (owner/repo format) + + Outputs: + - Dictionary with teams, topics, and custom_properties + """ + metadata = { + "teams": [], + "topics": [], + "custom_properties": {} + } + + try: + # Get repository teams + teams_url = f"{api_endpoint}/repos/{repo_name}/teams?per_page=100&page=1" + teams = make_api_call(teams_url, github_pat) + metadata["teams"] = [team["name"] for team in teams] + except Exception as e: + print(f"Warning: Could not fetch teams for {repo_name}: {e}") + metadata["teams"] = [] + + try: + # Get repository details (includes topics) + repo_url = f"{api_endpoint}/repos/{repo_name}" + repo_data = make_single_api_call(repo_url, github_pat) + metadata["topics"] = repo_data.get("topics", []) + except Exception as e: + print(f"Warning: Could not fetch repository details for {repo_name}: {e}") + metadata["topics"] = [] + + try: + # Get custom properties + properties_url = f"{api_endpoint}/repos/{repo_name}/properties" + properties = make_single_api_call(properties_url, github_pat) + metadata["custom_properties"] = properties + except Exception as e: + print(f"Warning: Could not fetch custom properties for {repo_name}: {e}") + metadata["custom_properties"] = {} + + return metadata + + +def make_single_api_call(url, github_pat): + """ + Make a single API call without pagination. + """ + headers = { + "Authorization": "token {}".format(github_pat), + "Accept": "application/vnd.github.v3+json", + } + response = requests.get(url, headers=headers) + if not response.ok: + raise Exception(response.status_code, response.text) + return response.json() \ No newline at end of file diff --git a/src/code_scanning.py b/src/code_scanning.py index 788a452..e92f08f 100644 --- a/src/code_scanning.py +++ b/src/code_scanning.py @@ -23,12 +23,16 @@ def list_repo_cs_alerts(api_endpoint, github_pat, repo_name): return code_scanning_alerts -def write_repo_cs_list(cs_list): +def write_repo_cs_list(cs_list, include_repo_metadata=False, api_endpoint=None, github_pat=None, repo_name=None): """ Write a list of code scanning alerts to a csv file. Inputs: - List of code scanning alerts + - include_repo_metadata: Whether to include extended repo metadata + - api_endpoint: API endpoint for metadata calls + - github_pat: GitHub PAT for metadata calls + - repo_name: Repository name for metadata calls Outputs: - CSV file of code scanning alerts @@ -36,59 +40,86 @@ def write_repo_cs_list(cs_list): with open("cs_list.csv", "w") as f: writer = csv.writer(f) - writer.writerow( - [ - "number", - "created_at", - "html_url", - "state", - "fixed_at", - "dismissed_by", - "dismissed_at", - "dismissed_reason", - "rule_id", - "rule_severity", - "security_severity_level", - "rule_tags", - "rule_description", - "rule_name", - "tool_name", - "tool_version", - "most_recent_instance_ref", - "most_recent_instance_state", - "most_recent_instance_sha", - "instances_url", - "most_recent_instance_category", - "most_recent_instance_location_path", - ] - ) + + # Base headers + headers = [ + "number", + "created_at", + "html_url", + "state", + "fixed_at", + "dismissed_by", + "dismissed_at", + "dismissed_reason", + "rule_id", + "rule_severity", + "security_severity_level", + "rule_tags", + "rule_description", + "rule_name", + "tool_name", + "tool_version", + "most_recent_instance_ref", + "most_recent_instance_state", + "most_recent_instance_sha", + "instances_url", + "most_recent_instance_category", + "most_recent_instance_location_path", + ] + + # Add extended metadata headers if enabled + if include_repo_metadata: + headers.extend([ + "repo_teams", + "repo_topics", + "repo_custom_properties" + ]) + + writer.writerow(headers) for cs in cs_list: - writer.writerow( - [ - cs["number"], - cs["created_at"], - cs["html_url"], - cs["state"], - cs["fixed_at"], - cs["dismissed_at"], - cs["dismissed_by"], - cs["dismissed_reason"], - cs["rule"]["id"], - cs["rule"]["severity"], - cs["rule"].get("security_severity_level", ""), - cs["rule"]["tags"], - cs["rule"]["description"], - cs["rule"]["name"], - cs["tool"]["name"], - cs["tool"]["version"], - cs["most_recent_instance"]["ref"], - cs["most_recent_instance"]["state"], - cs["most_recent_instance"]["commit_sha"], - cs["instances_url"], - cs["most_recent_instance"]["category"], - cs["most_recent_instance"]["location"]["path"], - ] - ) + # Base row data + row_data = [ + cs["number"], + cs["created_at"], + cs["html_url"], + cs["state"], + cs["fixed_at"], + cs["dismissed_at"], + cs["dismissed_by"], + cs["dismissed_reason"], + cs["rule"]["id"], + cs["rule"]["severity"], + cs["rule"].get("security_severity_level", ""), + cs["rule"]["tags"], + cs["rule"]["description"], + cs["rule"]["name"], + cs["tool"]["name"], + cs["tool"]["version"], + cs["most_recent_instance"]["ref"], + cs["most_recent_instance"]["state"], + cs["most_recent_instance"]["commit_sha"], + cs["instances_url"], + cs["most_recent_instance"]["category"], + cs["most_recent_instance"]["location"]["path"], + ] + + # Add extended metadata if enabled + if include_repo_metadata and api_endpoint and github_pat and repo_name: + try: + metadata = api_helpers.get_repo_metadata(api_endpoint, github_pat, repo_name) + row_data.extend([ + ",".join(metadata["teams"]), + ",".join(metadata["topics"]), + str(metadata["custom_properties"]) + ]) + except Exception as e: + print(f"Warning: Failed to get metadata for {repo_name}: {e}") + row_data.extend(["", "", ""]) + elif include_repo_metadata: + # If metadata is requested but details not provided + row_data.extend(["", "", ""]) + + writer.writerow(row_data) def list_org_cs_alerts(api_endpoint, github_pat, org_name): @@ -110,12 +141,15 @@ def list_org_cs_alerts(api_endpoint, github_pat, org_name): return code_scanning_alerts -def write_org_cs_list(cs_list): +def write_org_cs_list(cs_list, include_repo_metadata=False, api_endpoint=None, github_pat=None): """ Write a list of code scanning alerts to a csv file. Inputs: - List of code scanning alerts + - include_repo_metadata: Whether to include extended repo metadata + - api_endpoint: API endpoint for metadata calls + - github_pat: GitHub PAT for metadata calls Outputs: - CSV file of code scanning alerts @@ -124,69 +158,96 @@ def write_org_cs_list(cs_list): # Write code scanning alerts to csv file with open("cs_list.csv", "w") as f: writer = csv.writer(f) - writer.writerow( - [ - "number", - "created_at", - "html_url", - "state", - "fixed_at", - "dismissed_by", - "dismissed_at", - "dismissed_reason", - "rule_id", - "rule_severity", - "security_severity_level", - "rule_tags", - "rule_description", - "rule_name", - "tool_name", - "tool_version", - "most_recent_instance_ref", - "most_recent_instance_state", - "most_recent_instance_sha", - "instances_url", - "repo_name", - "repo_owner", - "repo_owner_type", - "repo_owner_isadmin", - "repo_url", - "repo_isfork", - "repo_isprivate", - ] - ) + + # Base headers + headers = [ + "number", + "created_at", + "html_url", + "state", + "fixed_at", + "dismissed_by", + "dismissed_at", + "dismissed_reason", + "rule_id", + "rule_severity", + "security_severity_level", + "rule_tags", + "rule_description", + "rule_name", + "tool_name", + "tool_version", + "most_recent_instance_ref", + "most_recent_instance_state", + "most_recent_instance_sha", + "instances_url", + "repo_name", + "repo_owner", + "repo_owner_type", + "repo_owner_isadmin", + "repo_url", + "repo_isfork", + "repo_isprivate", + ] + + # Add extended metadata headers if enabled + if include_repo_metadata: + headers.extend([ + "repo_teams", + "repo_topics", + "repo_custom_properties" + ]) + + writer.writerow(headers) for cs in cs_list: - writer.writerow( - [ - cs["number"], - cs["created_at"], - cs["html_url"], - cs["state"], - cs.get("fixed_at", ""), - cs.get("dismissed_by", ""), - cs.get("dismissed_at", ""), - cs.get("dismissed_reason", ""), - cs["rule"]["id"], - cs["rule"]["severity"], - cs["rule"].get("security_severity_level", ""), - cs["rule"]["tags"], - cs["rule"]["description"], - cs["rule"]["name"], - cs["tool"]["name"], - cs["tool"]["version"], - cs["most_recent_instance"]["ref"], - cs["most_recent_instance"]["state"], - cs["most_recent_instance"]["commit_sha"], - cs["instances_url"], - cs["repository"]["full_name"], - cs["repository"]["owner"]["login"], - cs["repository"]["owner"]["type"], - cs["repository"]["owner"]["site_admin"], - cs["repository"]["html_url"], - str(cs["repository"]["fork"]), - str(cs["repository"]["private"]), - ] - ) + # Base row data + row_data = [ + cs["number"], + cs["created_at"], + cs["html_url"], + cs["state"], + cs.get("fixed_at", ""), + cs.get("dismissed_by", ""), + cs.get("dismissed_at", ""), + cs.get("dismissed_reason", ""), + cs["rule"]["id"], + cs["rule"]["severity"], + cs["rule"].get("security_severity_level", ""), + cs["rule"]["tags"], + cs["rule"]["description"], + cs["rule"]["name"], + cs["tool"]["name"], + cs["tool"]["version"], + cs["most_recent_instance"]["ref"], + cs["most_recent_instance"]["state"], + cs["most_recent_instance"]["commit_sha"], + cs["instances_url"], + cs["repository"]["full_name"], + cs["repository"]["owner"]["login"], + cs["repository"]["owner"]["type"], + cs["repository"]["owner"]["site_admin"], + cs["repository"]["html_url"], + str(cs["repository"]["fork"]), + str(cs["repository"]["private"]), + ] + + # Add extended metadata if enabled + if include_repo_metadata and api_endpoint and github_pat: + try: + metadata = api_helpers.get_repo_metadata(api_endpoint, github_pat, cs["repository"]["full_name"]) + row_data.extend([ + ",".join(metadata["teams"]), + ",".join(metadata["topics"]), + str(metadata["custom_properties"]) + ]) + except Exception as e: + print(f"Warning: Failed to get metadata for {cs['repository']['full_name']}: {e}") + row_data.extend(["", "", ""]) + elif include_repo_metadata: + # If metadata is requested but API details not provided + row_data.extend(["", "", ""]) + + writer.writerow(row_data) def list_enterprise_server_cs_alerts(api_endpoint, github_pat, repo_list): @@ -218,13 +279,16 @@ def list_enterprise_server_cs_alerts(api_endpoint, github_pat, repo_list): return alerts -def write_enterprise_server_cs_list(cs_list): +def write_enterprise_server_cs_list(cs_list, include_repo_metadata=False, api_endpoint=None, github_pat=None): """ Write a list of code scanning alerts to a csv file. Inputs: - List from list_enterprise_code_scanning_alerts function, which contains strings and lists of dictionaries for the alerts. + - include_repo_metadata: Whether to include extended repo metadata + - api_endpoint: API endpoint for metadata calls + - github_pat: GitHub PAT for metadata calls Outputs: - CSV file of code scanning alerts @@ -312,13 +376,16 @@ def list_enterprise_cloud_cs_alerts(api_endpoint, github_pat, enterprise_slug): return code_scanning_alerts -def write_enterprise_cloud_cs_list(cs_list): +def write_enterprise_cloud_cs_list(cs_list, include_repo_metadata=False, api_endpoint=None, github_pat=None): """ Write a list of code scanning alerts to a csv file. Inputs: - List from list_enterprise_code_scanning_alerts function, which contains strings and lists of dictionaries for the alerts. + - include_repo_metadata: Whether to include extended repo metadata + - api_endpoint: API endpoint for metadata calls + - github_pat: GitHub PAT for metadata calls Outputs: - CSV file of code scanning alerts @@ -327,56 +394,83 @@ def write_enterprise_cloud_cs_list(cs_list): with open("cs_list.csv", "a") as f: writer = csv.writer(f) - writer.writerow( - [ - "repository", - "repo_id", - "number", - "created_at", - "html_url", - "state", - "fixed_at", - "dismissed_by", - "dismissed_at", - "dismissed_reason", - "rule_id", - "rule_severity", - "security_severity_level", - "rule_tags", - "rule_description", - "rule_name", - "tool_name", - "tool_version", - "most_recent_instance_ref", - "most_recent_instance_state", - "most_recent_instance_sha", - "instances_url", - ] - ) + + # Base headers + headers = [ + "repository", + "repo_id", + "number", + "created_at", + "html_url", + "state", + "fixed_at", + "dismissed_by", + "dismissed_at", + "dismissed_reason", + "rule_id", + "rule_severity", + "security_severity_level", + "rule_tags", + "rule_description", + "rule_name", + "tool_name", + "tool_version", + "most_recent_instance_ref", + "most_recent_instance_state", + "most_recent_instance_sha", + "instances_url", + ] + + # Add extended metadata headers if enabled + if include_repo_metadata: + headers.extend([ + "repo_teams", + "repo_topics", + "repo_custom_properties" + ]) + + writer.writerow(headers) for cs in cs_list: # loop through each alert in the list - writer.writerow( - [ - cs["repository"]["full_name"], - cs["repository"]["id"], - cs["number"], - cs["created_at"], - cs["html_url"], - cs["state"], - cs.get("fixed_at", ""), - cs.get("dismissed_by", ""), - cs.get("dismissed_at", ""), - cs.get("dismissed_reason", ""), - cs["rule"]["id"], - cs["rule"]["severity"], - cs["rule"].get("security_severity_level", "N/A"), - cs["rule"]["tags"], - cs["rule"]["description"], - cs["rule"]["name"], - cs["tool"]["name"], - cs["tool"]["version"], - cs["most_recent_instance"]["ref"], - cs["most_recent_instance"]["state"], - cs["most_recent_instance"]["commit_sha"], - cs["instances_url"], - ] - ) + # Base row data + row_data = [ + cs["repository"]["full_name"], + cs["repository"]["id"], + cs["number"], + cs["created_at"], + cs["html_url"], + cs["state"], + cs.get("fixed_at", ""), + cs.get("dismissed_by", ""), + cs.get("dismissed_at", ""), + cs.get("dismissed_reason", ""), + cs["rule"]["id"], + cs["rule"]["severity"], + cs["rule"].get("security_severity_level", "N/A"), + cs["rule"]["tags"], + cs["rule"]["description"], + cs["rule"]["name"], + cs["tool"]["name"], + cs["tool"]["version"], + cs["most_recent_instance"]["ref"], + cs["most_recent_instance"]["state"], + cs["most_recent_instance"]["commit_sha"], + cs["instances_url"], + ] + + # Add extended metadata if enabled + if include_repo_metadata and api_endpoint and github_pat: + try: + metadata = api_helpers.get_repo_metadata(api_endpoint, github_pat, cs["repository"]["full_name"]) + row_data.extend([ + ",".join(metadata["teams"]), + ",".join(metadata["topics"]), + str(metadata["custom_properties"]) + ]) + except Exception as e: + print(f"Warning: Failed to get metadata for {cs['repository']['full_name']}: {e}") + row_data.extend(["", "", ""]) + elif include_repo_metadata: + # If metadata is requested but API details not provided + row_data.extend(["", "", ""]) + + writer.writerow(row_data) diff --git a/src/dependabot.py b/src/dependabot.py index 63e228b..4847814 100644 --- a/src/dependabot.py +++ b/src/dependabot.py @@ -23,59 +23,90 @@ def list_repo_dependabot_alerts(api_endpoint, github_pat, repo_name): return dependabot_alerts -def write_repo_dependabot_list(dependabot_list): +def write_repo_dependabot_list(dependabot_list, include_repo_metadata=False, api_endpoint=None, github_pat=None, repo_name=None): """ Write the list of dependabot alerts to a CSV file. Inputs: - List of dependabot alerts + - include_repo_metadata: Whether to include extended repo metadata + - api_endpoint: API endpoint for metadata calls + - github_pat: GitHub PAT for metadata calls + - repo_name: Repository name for metadata calls Outputs: - CSV file of dependabot alerts """ with open("dependabot_list.csv", "w") as f: writer = csv.writer(f) - writer.writerow( - [ - "number", - "state", - "created_at", - "updated_at", - "fixed_at", - "dismissed_at", - "dismissed_by", - "dismissed_reason", - "html_url", - "dependency_manifest", - "dependency_ecosystem", - "dependency_name", - "severity", - "ghsa_id", - "cve_id", - "cvss_score", - ] - ) + + # Base headers + headers = [ + "number", + "state", + "created_at", + "updated_at", + "fixed_at", + "dismissed_at", + "dismissed_by", + "dismissed_reason", + "html_url", + "dependency_manifest", + "dependency_ecosystem", + "dependency_name", + "severity", + "ghsa_id", + "cve_id", + "cvss_score", + ] + + # Add extended metadata headers if enabled + if include_repo_metadata: + headers.extend([ + "repo_teams", + "repo_topics", + "repo_custom_properties" + ]) + + writer.writerow(headers) for alert in dependabot_list: - writer.writerow( - [ - alert["number"], - alert["state"], - alert["created_at"], - alert["updated_at"], - alert["fixed_at"], - alert["dismissed_at"], - alert["dismissed_by"], - alert["dismissed_reason"], - alert["html_url"], - alert["dependency"]["manifest_path"], - alert["dependency"]["package"]["ecosystem"], - alert["dependency"]["package"]["name"], - alert["security_vulnerability"]["severity"], - alert["security_advisory"]["ghsa_id"], - alert["security_advisory"]["cve_id"], - alert["security_advisory"]["cvss"]["score"], - ] - ) + # Base row data + row_data = [ + alert["number"], + alert["state"], + alert["created_at"], + alert["updated_at"], + alert["fixed_at"], + alert["dismissed_at"], + alert["dismissed_by"], + alert["dismissed_reason"], + alert["html_url"], + alert["dependency"]["manifest_path"], + alert["dependency"]["package"]["ecosystem"], + alert["dependency"]["package"]["name"], + alert["security_vulnerability"]["severity"], + alert["security_advisory"]["ghsa_id"], + alert["security_advisory"]["cve_id"], + alert["security_advisory"]["cvss"]["score"], + ] + + # Add extended metadata if enabled + if include_repo_metadata and api_endpoint and github_pat and repo_name: + try: + metadata = api_helpers.get_repo_metadata(api_endpoint, github_pat, repo_name) + row_data.extend([ + ",".join(metadata["teams"]), + ",".join(metadata["topics"]), + str(metadata["custom_properties"]) + ]) + except Exception as e: + print(f"Warning: Failed to get metadata for {repo_name}: {e}") + row_data.extend(["", "", ""]) + elif include_repo_metadata: + # If metadata is requested but details not provided + row_data.extend(["", "", ""]) + + writer.writerow(row_data) def list_org_dependabot_alerts(api_endpoint, github_pat, org_name): @@ -115,70 +146,100 @@ def list_enterprise_dependabot_alerts(api_endpoint, github_pat, enterprise_slug) return dependabot_alerts -def write_org_or_enterprise_dependabot_list(dependabot_list): +def write_org_or_enterprise_dependabot_list(dependabot_list, include_repo_metadata=False, api_endpoint=None, github_pat=None): """ Write the list of dependabot alerts to a CSV file. Inputs: - List of dependabot alerts + - include_repo_metadata: Whether to include extended repo metadata + - api_endpoint: API endpoint for metadata calls + - github_pat: GitHub PAT for metadata calls Outputs: - CSV file of dependabot alerts """ with open("dependabot_list.csv", "w") as f: writer = csv.writer(f) - writer.writerow( - [ - "number", - "state", - "created_at", - "updated_at", - "fixed_at", - "dismissed_at", - "dismissed_by", - "dismissed_reason", - "html_url", - "dependency_manifest", - "dependency_ecosystem", - "dependency_name", - "severity", - "ghsa_id", - "cve_id", - "cvss_score", - "repo_name", - "repo_owner", - "repo_owner_type", - "repo_owner_isadmin", - "repo_url", - "repo_isfork", - "repo_isprivate", - ] - ) + + # Base headers + headers = [ + "number", + "state", + "created_at", + "updated_at", + "fixed_at", + "dismissed_at", + "dismissed_by", + "dismissed_reason", + "html_url", + "dependency_manifest", + "dependency_ecosystem", + "dependency_name", + "severity", + "ghsa_id", + "cve_id", + "cvss_score", + "repo_name", + "repo_owner", + "repo_owner_type", + "repo_owner_isadmin", + "repo_url", + "repo_isfork", + "repo_isprivate", + ] + + # Add extended metadata headers if enabled + if include_repo_metadata: + headers.extend([ + "repo_teams", + "repo_topics", + "repo_custom_properties" + ]) + + writer.writerow(headers) for alert in dependabot_list: - writer.writerow( - [ - alert["number"], - alert["state"], - alert["created_at"], - alert["updated_at"], - alert["fixed_at"], - alert["dismissed_at"], - alert["dismissed_by"], - alert["dismissed_reason"], - alert["html_url"], - alert["dependency"]["manifest_path"], - alert["dependency"]["package"]["ecosystem"], - alert["dependency"]["package"]["name"], - alert["security_vulnerability"]["severity"], - alert["security_advisory"]["ghsa_id"], - alert["security_advisory"]["cve_id"], - alert["security_advisory"]["cvss"]["score"], - alert["repository"]["full_name"], - alert["repository"]["owner"]["login"], - alert["repository"]["owner"]["type"], - alert["repository"]["owner"]["site_admin"], - alert["repository"]["html_url"], - str(alert["repository"]["fork"]), - str(alert["repository"]["private"]), - ] - ) + # Base row data + row_data = [ + alert["number"], + alert["state"], + alert["created_at"], + alert["updated_at"], + alert["fixed_at"], + alert["dismissed_at"], + alert["dismissed_by"], + alert["dismissed_reason"], + alert["html_url"], + alert["dependency"]["manifest_path"], + alert["dependency"]["package"]["ecosystem"], + alert["dependency"]["package"]["name"], + alert["security_vulnerability"]["severity"], + alert["security_advisory"]["ghsa_id"], + alert["security_advisory"]["cve_id"], + alert["security_advisory"]["cvss"]["score"], + alert["repository"]["full_name"], + alert["repository"]["owner"]["login"], + alert["repository"]["owner"]["type"], + alert["repository"]["owner"]["site_admin"], + alert["repository"]["html_url"], + str(alert["repository"]["fork"]), + str(alert["repository"]["private"]), + ] + + # Add extended metadata if enabled + if include_repo_metadata and api_endpoint and github_pat: + try: + metadata = api_helpers.get_repo_metadata(api_endpoint, github_pat, alert["repository"]["full_name"]) + row_data.extend([ + ",".join(metadata["teams"]), + ",".join(metadata["topics"]), + str(metadata["custom_properties"]) + ]) + except Exception as e: + print(f"Warning: Failed to get metadata for {alert['repository']['full_name']}: {e}") + row_data.extend(["", "", ""]) + elif include_repo_metadata: + # If metadata is requested but API details not provided + row_data.extend(["", "", ""]) + + writer.writerow(row_data) diff --git a/src/secret_scanning.py b/src/secret_scanning.py index 08e499c..548cf23 100644 --- a/src/secret_scanning.py +++ b/src/secret_scanning.py @@ -56,12 +56,16 @@ def get_repo_ss_alerts(api_endpoint, github_pat, repo_name): return combined_alerts -def write_repo_ss_list(secrets_list): +def write_repo_ss_list(secrets_list, include_repo_metadata=False, api_endpoint=None, github_pat=None, repo_name=None): """ Write the list of repository secret scanning alerts to a csv file. Inputs: - List of secret scanning alerts + - include_repo_metadata: Whether to include extended repo metadata + - api_endpoint: API endpoint for metadata calls + - github_pat: GitHub PAT for metadata calls + - repo_name: Repository name for metadata calls Outputs: - CSV file of secret scanning alerts @@ -69,37 +73,64 @@ def write_repo_ss_list(secrets_list): # Write secret scanning alerts to csv file with open("secrets_list.csv", "w") as f: writer = csv.writer(f) - writer.writerow( - [ - "number", - "created_at", - "html_url", - "state", - "resolution", - "resolved_at", - "resolved_by_username", - "resolved_by_type", - "resolved_by_isadmin", - "secret_type", - "secret_type_display_name", - ] - ) + + # Base headers + headers = [ + "number", + "created_at", + "html_url", + "state", + "resolution", + "resolved_at", + "resolved_by_username", + "resolved_by_type", + "resolved_by_isadmin", + "secret_type", + "secret_type_display_name", + ] + + # Add extended metadata headers if enabled + if include_repo_metadata: + headers.extend([ + "repo_teams", + "repo_topics", + "repo_custom_properties" + ]) + + writer.writerow(headers) for alert in secrets_list: - writer.writerow( - [ - alert["number"], - alert["created_at"], - alert["html_url"], - alert["state"], - alert["resolution"], - alert["resolved_at"], - "" if alert["resolved_by"] is None else alert["resolved_by"]["login"], - "" if alert["resolved_by"] is None else alert["resolved_by"]["type"], - "" if alert["resolved_by"] is None else alert["resolved_by"]["site_admin"], - alert["secret_type"], - alert["secret_type_display_name"], - ] - ) + # Base row data + row_data = [ + alert["number"], + alert["created_at"], + alert["html_url"], + alert["state"], + alert["resolution"], + alert["resolved_at"], + "" if alert["resolved_by"] is None else alert["resolved_by"]["login"], + "" if alert["resolved_by"] is None else alert["resolved_by"]["type"], + "" if alert["resolved_by"] is None else alert["resolved_by"]["site_admin"], + alert["secret_type"], + alert["secret_type_display_name"], + ] + + # Add extended metadata if enabled + if include_repo_metadata and api_endpoint and github_pat and repo_name: + try: + metadata = api_helpers.get_repo_metadata(api_endpoint, github_pat, repo_name) + row_data.extend([ + ",".join(metadata["teams"]), + ",".join(metadata["topics"]), + str(metadata["custom_properties"]) + ]) + except Exception as e: + print(f"Warning: Failed to get metadata for {repo_name}: {e}") + row_data.extend(["", "", ""]) + elif include_repo_metadata: + # If metadata is requested but details not provided + row_data.extend(["", "", ""]) + + writer.writerow(row_data) def get_org_ss_alerts(api_endpoint, github_pat, org_name): @@ -155,12 +186,15 @@ def get_org_ss_alerts(api_endpoint, github_pat, org_name): return combined_alerts -def write_org_ss_list(secrets_list): +def write_org_ss_list(secrets_list, include_repo_metadata=False, api_endpoint=None, github_pat=None): """ Write the list of organization secret scanning alerts to a csv file. Inputs: - List of secret scanning alerts + - include_repo_metadata: Whether to include extended repo metadata + - api_endpoint: API endpoint for metadata calls + - github_pat: GitHub PAT for metadata calls Outputs: - CSV file of secret scanning alerts @@ -168,51 +202,78 @@ def write_org_ss_list(secrets_list): # Write secret scanning alerts to csv file with open("secrets_list.csv", "w") as f: writer = csv.writer(f) - writer.writerow( - [ - "number", - "created_at", - "html_url", - "state", - "resolution", - "resolved_at", - "resolved_by_username", - "resolved_by_type", - "resolved_by_isadmin", - "secret_type", - "secret_type_display_name", - "repo_name", - "repo_owner", - "repo_owner_type", - "repo_owner_isadmin", - "repo_url", - "repo_isfork", - "repo_isprivate", - ] - ) + + # Base headers + headers = [ + "number", + "created_at", + "html_url", + "state", + "resolution", + "resolved_at", + "resolved_by_username", + "resolved_by_type", + "resolved_by_isadmin", + "secret_type", + "secret_type_display_name", + "repo_name", + "repo_owner", + "repo_owner_type", + "repo_owner_isadmin", + "repo_url", + "repo_isfork", + "repo_isprivate", + ] + + # Add extended metadata headers if enabled + if include_repo_metadata: + headers.extend([ + "repo_teams", + "repo_topics", + "repo_custom_properties" + ]) + + writer.writerow(headers) for alert in secrets_list: - writer.writerow( - [ - alert["number"], - alert["created_at"], - alert["html_url"], - alert["state"], - alert["resolution"], - alert["resolved_at"], - "" if alert["resolved_by"] is None else alert["resolved_by"]["login"], - "" if alert["resolved_by"] is None else alert["resolved_by"]["type"], - "" if alert["resolved_by"] is None else alert["resolved_by"]["site_admin"], - alert["secret_type"], - alert["secret_type_display_name"], - alert["repository"]["full_name"], - alert["repository"]["owner"]["login"], - alert["repository"]["owner"]["type"], - alert["repository"]["owner"]["site_admin"], - alert["repository"]["html_url"], - str(alert["repository"]["fork"]), - str(alert["repository"]["private"]), - ] - ) + # Base row data + row_data = [ + alert["number"], + alert["created_at"], + alert["html_url"], + alert["state"], + alert["resolution"], + alert["resolved_at"], + "" if alert["resolved_by"] is None else alert["resolved_by"]["login"], + "" if alert["resolved_by"] is None else alert["resolved_by"]["type"], + "" if alert["resolved_by"] is None else alert["resolved_by"]["site_admin"], + alert["secret_type"], + alert["secret_type_display_name"], + alert["repository"]["full_name"], + alert["repository"]["owner"]["login"], + alert["repository"]["owner"]["type"], + alert["repository"]["owner"]["site_admin"], + alert["repository"]["html_url"], + str(alert["repository"]["fork"]), + str(alert["repository"]["private"]), + ] + + # Add extended metadata if enabled + if include_repo_metadata and api_endpoint and github_pat: + try: + metadata = api_helpers.get_repo_metadata(api_endpoint, github_pat, alert["repository"]["full_name"]) + row_data.extend([ + ",".join(metadata["teams"]), + ",".join(metadata["topics"]), + str(metadata["custom_properties"]) + ]) + except Exception as e: + print(f"Warning: Failed to get metadata for {alert['repository']['full_name']}: {e}") + row_data.extend(["", "", ""]) + elif include_repo_metadata: + # If metadata is requested but API details not provided + row_data.extend(["", "", ""]) + + writer.writerow(row_data) def get_enterprise_ss_alerts(api_endpoint, github_pat, enterprise_slug): @@ -267,12 +328,15 @@ def get_enterprise_ss_alerts(api_endpoint, github_pat, enterprise_slug): return combined_alerts -def write_enterprise_ss_list(secrets_list): +def write_enterprise_ss_list(secrets_list, include_repo_metadata=False, api_endpoint=None, github_pat=None): """ Write the list of enterprise secret scanning alerts to a csv file. Inputs: - List of secret scanning alerts + - include_repo_metadata: Whether to include extended repo metadata + - api_endpoint: API endpoint for metadata calls + - github_pat: GitHub PAT for metadata calls Outputs: - CSV file of secret scanning alerts @@ -280,48 +344,75 @@ def write_enterprise_ss_list(secrets_list): # Write secret scanning alerts to csv file with open("secrets_list.csv", "w") as f: writer = csv.writer(f) - writer.writerow( - [ - "number", - "created_at", - "html_url", - "state", - "resolution", - "resolved_at", - "resolved_by_username", - "resolved_by_type", - "resolved_by_isadmin", - "secret_type", - "secret_type_display_name", - "repo_name", - "repo_owner", - "repo_owner_type", - "repo_owner_isadmin", - "repo_url", - "repo_isfork", - "repo_isprivate", - ] - ) + + # Base headers + headers = [ + "number", + "created_at", + "html_url", + "state", + "resolution", + "resolved_at", + "resolved_by_username", + "resolved_by_type", + "resolved_by_isadmin", + "secret_type", + "secret_type_display_name", + "repo_name", + "repo_owner", + "repo_owner_type", + "repo_owner_isadmin", + "repo_url", + "repo_isfork", + "repo_isprivate", + ] + + # Add extended metadata headers if enabled + if include_repo_metadata: + headers.extend([ + "repo_teams", + "repo_topics", + "repo_custom_properties" + ]) + + writer.writerow(headers) for alert in secrets_list: - writer.writerow( - [ - alert["number"], - alert["created_at"], - alert["html_url"], - alert["state"], - alert["resolution"], - alert["resolved_at"], - "" if alert["resolved_by"] is None else alert["resolved_by"]["login"], - "" if alert["resolved_by"] is None else alert["resolved_by"]["type"], - "" if alert["resolved_by"] is None else alert["resolved_by"]["site_admin"], - alert["secret_type"], - alert["secret_type_display_name"], - alert["repository"]["full_name"], - alert["repository"]["owner"]["login"], - alert["repository"]["owner"]["type"], - alert["repository"]["owner"]["site_admin"], - alert["repository"]["html_url"], - str(alert["repository"]["fork"]), - str(alert["repository"]["private"]), - ] - ) + # Base row data + row_data = [ + alert["number"], + alert["created_at"], + alert["html_url"], + alert["state"], + alert["resolution"], + alert["resolved_at"], + "" if alert["resolved_by"] is None else alert["resolved_by"]["login"], + "" if alert["resolved_by"] is None else alert["resolved_by"]["type"], + "" if alert["resolved_by"] is None else alert["resolved_by"]["site_admin"], + alert["secret_type"], + alert["secret_type_display_name"], + alert["repository"]["full_name"], + alert["repository"]["owner"]["login"], + alert["repository"]["owner"]["type"], + alert["repository"]["owner"]["site_admin"], + alert["repository"]["html_url"], + str(alert["repository"]["fork"]), + str(alert["repository"]["private"]), + ] + + # Add extended metadata if enabled + if include_repo_metadata and api_endpoint and github_pat: + try: + metadata = api_helpers.get_repo_metadata(api_endpoint, github_pat, alert["repository"]["full_name"]) + row_data.extend([ + ",".join(metadata["teams"]), + ",".join(metadata["topics"]), + str(metadata["custom_properties"]) + ]) + except Exception as e: + print(f"Warning: Failed to get metadata for {alert['repository']['full_name']}: {e}") + row_data.extend(["", "", ""]) + elif include_repo_metadata: + # If metadata is requested but API details not provided + row_data.extend(["", "", ""]) + + writer.writerow(row_data)