From b20b5980e015096ea5fcf3912f6a85c06de5c2f2 Mon Sep 17 00:00:00 2001 From: Neil Mehta Date: Wed, 29 Apr 2026 12:40:34 -0400 Subject: [PATCH 01/16] feat(finserv): Add 64 FinServ GenAI risk checks (FS-01 to FS-69) Implements 64 standalone security checks plus 5 upstream extensions derived from the AWS guide for Financial Services risk management of the use of Generative AI (March 2026): https://d1.awsstatic.com/onedam/marketing-channels/website/public/global-FinServ-ComplianceGuide-GenAIRisks-public.pdf Covers 15 risk categories across FS-01 to FS-69: FS-01..06 Unbounded Consumption (WAF/Shield, rate limiting, token quotas, cost monitoring) FS-07..11 Excessive Agency (action boundaries, AgentCore Policy, transaction limits, rate alarms) FS-12..16 Supply Chain Vulnerabilities (SCPs, model inventory, onboarding governance, adversarial evaluation) FS-17..21 Training Data & Model Poisoning (Model Monitor, data drift, Model Registry, Feature Store rollback) FS-22..26 Vector & Embedding Weaknesses (KB least privilege, CloudTrail logging, metadata filtering, multi-tenancy) FS-27..30 Non-Compliant Output (Automated Reasoning, RAG controls, human-in-the-loop, compliance guardrails) FS-31..34 Misinformation (KB data quality, source attribution, integrity monitoring, sync cadence) FS-35..38 Abusive or Harmful Output (FMEval, user reporting, allowlists, AI Service Cards) FS-39..42 Biased Output (SageMaker Clarify, Bedrock Evaluations, bias datasets, AI Service Cards) FS-43..46 Sensitive Information Disclosure (CloudWatch log masking, Macie, PII pre-processing, data classification) FS-47..50 Hallucination (contextual grounding, Automated Reasoning, disclaimers, RAG) FS-51..54 Prompt Injection (input sanitization, parameterized queries, pen testing, SDK currency) FS-55..58 Improper Output Handling (output validation, sanitization, encoding, XSS prevention) FS-59..60 Off-Topic & Inappropriate Output (contextual grounding, topic allowlists) FS-61..63 Out-of-Date Training Data (KB sync cadence, data currency disclaimers, FM version updates) FS-64..69 Material Gap Checks (guardrail trace logging, KB data-source S3 event notifications, AgentCore end-user identity propagation, agent financial transaction value thresholds, API Gateway request body size limits, prompt input validation) 5 checks contributed as upstream extensions (not standalone entries): FS-17 -> SM-07 Model Monitor data quality baseline cadence FS-18 -> SM-23 Model drift low-entropy classification monitoring FS-19 -> SM-22 Model Registry PendingManualApproval default enforcement FS-23 -> BR-06 Knowledge Base data-plane CloudTrail event selectors FS-64 -> BR-04 Guardrail trace logging verification New files: functions/security/finserv_assessments/app.py (64 check functions) functions/security/finserv_assessments/schema.py (Finding schema) functions/security/finserv_assessments/requirements.txt functions/security/finserv_assessments/__init__.py docs/SECURITY_CHECKS_FINSERV_COMMON.md docs/SECURITY_CHECKS_FINSERV_PART1_INFRA_CONTROLS.md docs/SECURITY_CHECKS_FINSERV_PART2_GUARDRAILS_CONTENT_SAFETY.md docs/SECURITY_CHECKS_FINSERV_PART3_APP_LAYER_AND_GAPS.md docs/AIMLSecurityAssessment-MappingsTable.csv .ash/.ash.yaml (ASH v3.2.6 config with documented suppressions) Modified files: template.yaml + template-multi-account.yaml - Add FinServSecurityAssessmentFunction resource - Wire DefinitionSubstitutions and LambdaInvokePolicy statemachine/assessments.asl.json - Add FinServ branch to Run Security Assessments parallel state deployment/1-aiml-security-member-roles.yaml deployment/aiml-security-single-account.yaml - Add FinServ GenAI Risk Assessment Permissions statement functions/security/generate_consolidated_report/app.py - Load finserv_security_report_{execution_id}.csv into consolidated report functions/security/bedrock_assessments/app.py - Add FS-64 (BR-04) and FS-23 (BR-06) extension notes functions/security/sagemaker_assessments/app.py - Add FS-17 (SM-07), FS-18 (SM-23), FS-19 (SM-22) extension notes docs/SECURITY_CHECKS.md - Add FinServ section pointing to the four new reference files Compliance frameworks mapped (preliminary, for MRM/compliance team review): SR 11-7, FFIEC CAT, NYDFS 500.06, PCI-DSS 12.3.2, DORA Art.6, MAS TRM 9, ISO 27001 A.12, ECOA, OWASP LLM Top 10 Quality gates passed locally: ruff check + ruff format --check: clean cfn-lint (with .cfnlintrc): exit 0 on all 5 CFN templates sam validate --lint: valid on both SAM templates sam build: Build Succeeded ASH v3.2.6 local scan: detect-secrets PASSED, semgrep PASSED, npm-audit PASSED; remaining Checkov/cdk-nag findings are pre-existing upstream patterns (CI uses continue-on-error: true for ASH) Note: HTML report rendering for FinServ findings (fourth section in report_template.py) is a follow-up item. The finserv_security_report_*.csv is generated and available in S3 for downstream consumption. Closes #22 --- .ash/.ash.yaml | 235 + .ash/.gitignore | 2 + .../security/bedrock_assessments/app.py | 12 + .../security/finserv_assessments/__init__.py | 0 .../security/finserv_assessments/app.py | 4425 +++++++++++++++++ .../finserv_assessments/requirements.txt | 1 + .../security/finserv_assessments/schema.py | 80 + .../generate_consolidated_report/app.py | 20 +- .../security/sagemaker_assessments/app.py | 16 + .../statemachine/assessments.asl.json | 357 +- .../template-multi-account.yaml | 167 + aiml-security-assessment/template.yaml | 167 + deployment/1-aiml-security-member-roles.yaml | 47 + deployment/aiml-security-single-account.yaml | 47 + docs/AIMLSecurityAssessment-MappingsTable.csv | 16 + docs/SECURITY_CHECKS.md | 29 + docs/SECURITY_CHECKS_FINSERV_COMMON.md | 169 + ...ITY_CHECKS_FINSERV_PART1_INFRA_CONTROLS.md | 401 ++ ...FINSERV_PART2_GUARDRAILS_CONTENT_SAFETY.md | 318 ++ ...CHECKS_FINSERV_PART3_APP_LAYER_AND_GAPS.md | 368 ++ 20 files changed, 6710 insertions(+), 167 deletions(-) create mode 100644 .ash/.ash.yaml create mode 100644 .ash/.gitignore create mode 100644 aiml-security-assessment/functions/security/finserv_assessments/__init__.py create mode 100644 aiml-security-assessment/functions/security/finserv_assessments/app.py create mode 100644 aiml-security-assessment/functions/security/finserv_assessments/requirements.txt create mode 100644 aiml-security-assessment/functions/security/finserv_assessments/schema.py create mode 100644 docs/AIMLSecurityAssessment-MappingsTable.csv create mode 100644 docs/SECURITY_CHECKS_FINSERV_COMMON.md create mode 100644 docs/SECURITY_CHECKS_FINSERV_PART1_INFRA_CONTROLS.md create mode 100644 docs/SECURITY_CHECKS_FINSERV_PART2_GUARDRAILS_CONTENT_SAFETY.md create mode 100644 docs/SECURITY_CHECKS_FINSERV_PART3_APP_LAYER_AND_GAPS.md diff --git a/.ash/.ash.yaml b/.ash/.ash.yaml new file mode 100644 index 0000000..791bba4 --- /dev/null +++ b/.ash/.ash.yaml @@ -0,0 +1,235 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/awslabs/automated-security-helper/refs/heads/main/automated_security_helper/schemas/AshConfig.json +project_name: sample-aiml-security-assessment +global_settings: + severity_threshold: MEDIUM + ignore_paths: + # SAM build cache — transitively scanned pydantic/bs4/botocore deps + # produce thousands of non-actionable Bandit findings in third-party code. + # CI scans only changed files, so it never touches .aws-sam/ either. + - path: "aiml-security-assessment/.aws-sam/**" + reason: "SAM build cache; contains third-party deps, not project source" + - path: "**/.aws-sam/**" + reason: "SAM build cache at any depth (deps/, build/, etc.)" + # Pre-existing static HTML sample reports; the embedded base64 SVG logos + # are flagged as high-entropy strings by detect-secrets. + - path: "sample-reports/**" + reason: "Static HTML sample outputs with embedded base64 SVG icons (upstream)" + # ASH's own output directory. + - path: ".ash/ash_output/**" + reason: "ASH output directory; scanning it creates recursion" + suppressions: + # The 4 Checkov rules below flag every assessment Lambda in the upstream + # repo for: no VPC (CKV_AWS_117), no DLQ (CKV_AWS_116), no per-function + # concurrency cap (CKV_AWS_115), and no KMS-encrypted env vars + # (CKV_AWS_173). This matches the architecture choices made for all 6 + # pre-existing Lambdas (BedrockSecurity, SagemakerSecurity, + # AgentCoreSecurity, GenerateConsolidatedReport, IAMPermissionCaching, + # CleanupBucket) and applied consistently to the new + # FinServSecurityAssessmentFunction. The assessment framework runs on an + # internal schedule with short-lived invocations and reads config from a + # non-sensitive bucket-name env var; hardening these four items for every + # Lambda in the stack is a framework-wide change that should be proposed + # in a separate PR. + - rule_id: "CKV_AWS_117" + path: "aiml-security-assessment/template.yaml" + reason: "Upstream convention: assessment Lambdas are non-VPC (applies to all 7 Lambdas)" + - rule_id: "CKV_AWS_117" + path: "aiml-security-assessment/template-multi-account.yaml" + reason: "Upstream convention: assessment Lambdas are non-VPC (applies to all 7 Lambdas)" + - rule_id: "CKV_AWS_116" + path: "aiml-security-assessment/template.yaml" + reason: "Upstream convention: assessment Lambdas have no DLQ (applies to all 7 Lambdas)" + - rule_id: "CKV_AWS_116" + path: "aiml-security-assessment/template-multi-account.yaml" + reason: "Upstream convention: assessment Lambdas have no DLQ (applies to all 7 Lambdas)" + - rule_id: "CKV_AWS_115" + path: "aiml-security-assessment/template.yaml" + reason: "Upstream convention: no per-function concurrency cap (applies to all 7 Lambdas)" + - rule_id: "CKV_AWS_115" + path: "aiml-security-assessment/template-multi-account.yaml" + reason: "Upstream convention: no per-function concurrency cap (applies to all 7 Lambdas)" + - rule_id: "CKV_AWS_173" + path: "aiml-security-assessment/template.yaml" + reason: "Upstream convention: env var is a non-sensitive bucket-name reference" + - rule_id: "CKV_AWS_173" + path: "aiml-security-assessment/template-multi-account.yaml" + reason: "Upstream convention: env var is a non-sensitive bucket-name reference" + + # cdk-nag AwsSolutions-IAM5 flags any IAM statement with Resource: "*". + # The FinServ IAM statement I added to 1-aiml-security-member-roles.yaml + # uses the same pattern as every pre-existing Sid in that policy. The + # wildcard is required for list/describe operations that are not + # ARN-scopable (e.g., ListBuckets, ListTrails, ListRules, ListPolicies). + # If upstream tightens IAM across all assessment statements, the FinServ + # Sid should be tightened in the same PR. + - rule_id: "AwsSolutions-IAM5" + path: "deployment/1-aiml-security-member-roles.yaml" + reason: "Read-only list/describe actions that cannot be ARN-scoped (upstream convention)" + - rule_id: "AwsSolutions-IAM5" + path: "deployment/aiml-security-single-account.yaml" + reason: "Read-only list/describe actions that cannot be ARN-scoped (upstream convention)" + - rule_id: "CKV_AWS_173" + path: "aiml-security-assessment/template-multi-account.yaml" + reason: "Upstream convention: env var is a bucket name reference, not sensitive (same as BR/SM/AC)" +fail_on_findings: true +ash_plugin_modules: [] +external_reports_to_include: [] +converters: + archive: + enabled: true + options: {} + jupyter: + enabled: true + options: + tool_version: '>=7.16.0,<8.0.0' + install_timeout: 300 +scanners: + bandit: + enabled: true + options: + severity_threshold: null + config_file: null + confidence_level: all + ignore_nosec: false + excluded_paths: [] + additional_formats: [] + tool_version: '>=1.7.0,<2.0.0' + install_timeout: 300 + cdk-nag: + enabled: true + options: + severity_threshold: null + nag_packs: + AwsSolutionsChecks: true + HIPAASecurityChecks: false + NIST80053R4Checks: false + NIST80053R5Checks: false + PCIDSS321Checks: false + cfn-nag: + enabled: true + options: + severity_threshold: null + checkov: + enabled: true + options: + severity_threshold: null + config_file: null + skip_path: [] + additional_formats: + - cyclonedx_json + offline: false + frameworks: + - all + skip_frameworks: [] + tool_version: null + install_timeout: 300 + detect-secrets: + enabled: true + options: + severity_threshold: null + baseline_file: null + scan_settings: + version: null + generated_at: null + plugins_used: [] + filters_used: [] + results: {} + grype: + enabled: true + options: + severity_threshold: null + config_file: null + offline: false + npm-audit: + enabled: true + options: + severity_threshold: null + offline: false + opengrep: + enabled: true + options: + severity_threshold: null + config: auto + exclude: + - '*-converted.py' + - '*_report_result.txt' + exclude_rule: [] + severity: [] + metrics: auto + offline: false + patterns: [] + version: v1.15.1 + semgrep: + enabled: true + options: + severity_threshold: null + config: auto + exclude: + - '*-converted.py' + - '*_report_result.txt' + exclude_rule: [] + severity: [] + metrics: auto + offline: false + tool_version: null + install_timeout: 300 + syft: + enabled: true + options: + severity_threshold: null + config_file: null + exclude: [] + additional_outputs: + - syft-table +reporters: + csv: + enabled: true + options: {} + cyclonedx: + enabled: true + options: {} + html: + enabled: true + options: {} + flat-json: + enabled: true + options: + include_scanner_metrics: true + include_summary_metrics: true + include_metadata: true + gitlab-sast: + enabled: true + options: {} + junitxml: + enabled: true + options: + respect_severity_threshold: true + markdown: + enabled: true + options: + include_summary: true + include_findings_table: false + include_detailed_findings: true + max_detailed_findings: 20 + top_hotspots_limit: 10 + use_collapsible_details: true + ocsf: + enabled: true + options: {} + sarif: + enabled: true + options: {} + spdx: + enabled: false + options: {} + text: + enabled: true + options: + include_summary: true + include_findings_table: false + include_detailed_findings: false + max_detailed_findings: 20 + top_hotspots_limit: 20 + yaml: + enabled: false + options: {} diff --git a/.ash/.gitignore b/.ash/.gitignore new file mode 100644 index 0000000..d831134 --- /dev/null +++ b/.ash/.gitignore @@ -0,0 +1,2 @@ +# ASH default output directory (and variants) +ash_output* diff --git a/aiml-security-assessment/functions/security/bedrock_assessments/app.py b/aiml-security-assessment/functions/security/bedrock_assessments/app.py index 24b02bc..76e7ebb 100644 --- a/aiml-security-assessment/functions/security/bedrock_assessments/app.py +++ b/aiml-security-assessment/functions/security/bedrock_assessments/app.py @@ -810,6 +810,12 @@ def check_bedrock_logging_configuration() -> Dict[str, Any]: """ Check if model invocation logging is enabled for Amazon Bedrock """ + # FinServ extension (FS-64): In addition to verifying that invocation + # logging is enabled, the FinServ guide (PDF §1.2.1, §1.2.6, §1.2.7) + # expects the log output to include guardrailTrace with action, + # inputAssessments, and outputAssessments to support SR 11-7 audit trails + # and NYDFS 500.06 retention. See docs/SECURITY_CHECKS_FINSERV_PART3_APP_LAYER_AND_GAPS.md + # (FS-64 → BR-04 extension note) for the detection / remediation language. logger.debug("Starting check for Bedrock model invocation logging configuration") try: findings = { @@ -905,6 +911,12 @@ def check_bedrock_cloudtrail_logging() -> Dict[str, Any]: """ Check if CloudTrail is configured to log Amazon Bedrock API calls """ + # FinServ extension (FS-23): In addition to verifying CloudTrail is logging + # Bedrock API calls, the FinServ guide (PDF §1.2.15) expects an advanced + # event selector for AWS::Bedrock::KnowledgeBase so Retrieve and + # RetrieveAndGenerate data events are captured (NOT logged by default). + # See docs/SECURITY_CHECKS_FINSERV_PART1_INFRA_CONTROLS.md (FS-23 → BR-06 + # extension note) for the detection / remediation language. logger.debug("Starting check for Bedrock CloudTrail logging configuration") try: findings = { diff --git a/aiml-security-assessment/functions/security/finserv_assessments/__init__.py b/aiml-security-assessment/functions/security/finserv_assessments/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aiml-security-assessment/functions/security/finserv_assessments/app.py b/aiml-security-assessment/functions/security/finserv_assessments/app.py new file mode 100644 index 0000000..f036618 --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_assessments/app.py @@ -0,0 +1,4425 @@ +""" +AWS FinServ GenAI Risk Assessment Lambda +========================================= +Implements 64 standalone security checks derived from the AWS guide: +"Financial Services risk management of the use of Generative AI" +https://d1.awsstatic.com/onedam/marketing-channels/website/public/global-FinServ-ComplianceGuide-GenAIRisks-public.pdf + +Check ID namespace: FS-01 through FS-69 + FS-01 to FS-63 — original 63 checks across 15 risk categories + (FS-17, FS-18, FS-19 merged into upstream SM-07, SM-23, SM-22) + FS-64 to FS-69 — 6 material gap checks covering mitigations explicitly + called out in the Guide but absent from FS-01..63 and + the existing BR/SM/AC checks in the AIML Security Assessment. + (FS-64 merged into upstream BR-04) + +5 checks (FS-17, FS-18, FS-19, FS-23, FS-64) are contributed as upstream extensions +rather than standalone entries — see extension notes in the SECURITY_CHECKS_FINSERV +Part 1 and Part 3 markdown files. + +These checks complement the existing BR/SM/AC checks in the AIML Security Assessment. + +COMPLIANCE_PLACEHOLDER: Each check includes a comment listing the FinServ regulatory +frameworks it maps to. The prototype report owner should wire these into the compliance +standards column of the HTML report template. +Frameworks referenced: FFIEC CAT, SR 11-7, NYDFS 500.06, PCI-DSS 12.3.2, SOC 2 CC6, +ISO 27001 A.12, DORA Art.6, MAS TRM 9. + +Contribution workflow: + - Upstream repo: aws-samples/sample-aiml-security-assessment (OSPO-managed, so forks + are auto-approved by Amazon Code Defender). + - This Lambda is delivered via a personal fork + feature branch + PR. See + GIT_WORKFLOW.md for the full 9-step process (fork, branch, ASH scan, commit, push, + PR, GitHub Actions verification, reviewer assignment, optional Git Defender + exception ticket). + +Pre-commit quality gates (run every edit): + 1. ruff check + ruff format --check on this directory. + 2. sam local invoke FinServSecurityAssessmentFunction against a test event. + 3. cfn-lint / sam validate on the updated SAM templates. + 4. ash --source-dir --fail-on-findings --config-overrides + 'global_settings.severity_threshold=MEDIUM' — resolve every Critical and High + finding before opening the PR. + 5. git defender scan on the staged diff. +""" + +import boto3 +import csv +import json +import logging +import os +from datetime import datetime, timezone +from io import StringIO +from typing import Any, Dict, List, Optional + +from botocore.config import Config +from botocore.exceptions import ClientError + +from schema import create_finding + +# --------------------------------------------------------------------------- +# Boto3 config with adaptive retries +# --------------------------------------------------------------------------- +boto3_config = Config(retries=dict(max_attempts=10, mode="adaptive")) + +logger = logging.getLogger() +logger.setLevel(logging.ERROR) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def get_permissions_cache(execution_id: str) -> Optional[Dict[str, Any]]: + """Retrieve IAM permissions cache from S3 (same pattern as other assessments).""" + try: + s3_client = boto3.client("s3", config=boto3_config) + s3_key = f"permissions_cache_{execution_id}.json" + s3_bucket = os.environ.get("AIML_ASSESSMENT_BUCKET_NAME") + response = s3_client.get_object(Bucket=s3_bucket, Key=s3_key) + return json.loads(response["Body"].read().decode("utf-8")) + except ClientError as e: + logger.warning(f"Could not load permissions cache: {e}") + return None + except Exception as e: + logger.error(f"Unexpected error loading permissions cache: {e}", exc_info=True) + return None + + +def _empty_findings(check_name: str) -> Dict[str, Any]: + return {"check_name": check_name, "status": "PASS", "details": "", "csv_data": []} + + +def _error_findings(check_name: str, err: Exception) -> Dict[str, Any]: + return { + "check_name": check_name, + "status": "ERROR", + "details": str(err), + "csv_data": [], + } + + +# =========================================================================== +# CATEGORY 1: UNBOUNDED CONSUMPTION (FS-01 to FS-06) +# Risk: GenAI workloads can be exploited to exhaust compute/cost budgets +# COMPLIANCE_PLACEHOLDER: [FFIEC CAT, DORA Art.6, SR 11-7 Appendix A] +# =========================================================================== + + +def check_waf_shield_on_bedrock_endpoints() -> Dict[str, Any]: + """ + FS-01 — Verify AWS WAF is associated with API Gateway or ALB endpoints + that front Bedrock/GenAI workloads, and that AWS Shield Advanced is enabled. + COMPLIANCE_PLACEHOLDER: [FFIEC CAT Cyber Risk Management, DORA Art.6 ICT Risk] + """ + findings = _empty_findings("WAF and Shield Protection Check") + try: + wafv2 = boto3.client("wafv2", config=boto3_config) + shield = boto3.client("shield", config=boto3_config) + + # Check Shield Advanced subscription + shield_enabled = False + try: + shield.describe_subscription() + shield_enabled = True + except shield.exceptions.ResourceNotFoundException: + pass + except ClientError: + pass + + # Check WAF Web ACLs exist (regional, covering API GW / ALB) + acls = wafv2.list_web_acls(Scope="REGIONAL").get("WebACLs", []) + + if not shield_enabled: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-01", + finding_name="AWS Shield Advanced Not Enabled", + finding_details=( + "AWS Shield Advanced is not subscribed. GenAI API endpoints are " + "vulnerable to volumetric DDoS attacks that can exhaust token quotas " + "and inflate costs." + ), + resolution=( + "1. Subscribe to AWS Shield Advanced for DDoS protection.\n" + "2. Associate Shield Advanced with Bedrock-facing API Gateway stages, " + "ALBs, and CloudFront distributions.\n" + "3. Enable Shield Response Team (SRT) access." + ), + reference="https://docs.aws.amazon.com/waf/latest/developerguide/shield-chapter.html", + severity="High", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-01", + finding_name="AWS Shield Advanced Enabled", + finding_details="AWS Shield Advanced subscription is active.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/waf/latest/developerguide/shield-chapter.html", + severity="Informational", + status="Passed", + ) + ) + + if not acls: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-01", + finding_name="No Regional WAF Web ACLs Found", + finding_details=( + "No AWS WAF regional Web ACLs found. Without WAF, GenAI endpoints " + "lack rate-based rules to block abusive callers." + ), + resolution=( + "1. Create a WAF Web ACL with rate-based rules (e.g., 1000 req/5 min per IP).\n" + "2. Associate the ACL with API Gateway stages or ALBs fronting Bedrock.\n" + "3. Add AWS Managed Rules for known bad inputs." + ), + reference="https://docs.aws.amazon.com/waf/latest/developerguide/waf-chapter.html", + severity="High", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-01", + finding_name="Regional WAF Web ACLs Present", + finding_details=f"Found {len(acls)} regional WAF Web ACL(s).", + resolution="Verify ACLs are associated with Bedrock-facing endpoints.", + reference="https://docs.aws.amazon.com/waf/latest/developerguide/waf-chapter.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("WAF and Shield Protection Check", e) + return findings + + +def check_api_gateway_rate_limiting() -> Dict[str, Any]: + """ + FS-02 — Verify API Gateway usage plans enforce throttling on GenAI endpoints. + COMPLIANCE_PLACEHOLDER: [FFIEC CAT, DORA Art.6, PCI-DSS 12.3.2] + """ + findings = _empty_findings("API Gateway Rate Limiting Check") + try: + apigw = boto3.client("apigateway", config=boto3_config) + plans = apigw.get_usage_plans().get("items", []) + + plans_without_throttle = [ + p["name"] + for p in plans + if not p.get("throttle") or p["throttle"].get("rateLimit", 0) == 0 + ] + + if not plans: + findings["csv_data"].append( + create_finding( + check_id="FS-02", + finding_name="No API Gateway Usage Plans Found", + finding_details="No usage plans configured. GenAI API endpoints may have no rate limits.", + resolution=( + "Create API Gateway usage plans with throttle settings " + "(rateLimit and burstLimit) for all Bedrock-facing APIs." + ), + reference="https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-request-throttling.html", + severity="Medium", + status="N/A", + ) + ) + elif plans_without_throttle: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-02", + finding_name="API Gateway Usage Plans Missing Throttle", + finding_details=( + f"Usage plans without throttling: {', '.join(plans_without_throttle)}. " + "Unbounded API calls can exhaust Bedrock token quotas and inflate costs." + ), + resolution=( + "Set rateLimit and burstLimit on all usage plans associated with " + "GenAI API stages. Consider per-consumer API keys with individual quotas." + ), + reference="https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-request-throttling.html", + severity="High", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-02", + finding_name="API Gateway Rate Limiting Configured", + finding_details=f"All {len(plans)} usage plan(s) have throttle settings.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-request-throttling.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("API Gateway Rate Limiting Check", e) + return findings + + +def check_bedrock_token_quotas() -> Dict[str, Any]: + """ + FS-03 — Check whether Bedrock service quotas for tokens-per-minute (TPM) + and requests-per-minute (RPM) have been reviewed and set appropriately. + COMPLIANCE_PLACEHOLDER: [FFIEC CAT, SR 11-7] + """ + findings = _empty_findings("Bedrock Token Quota Review") + try: + sq = boto3.client("service-quotas", config=boto3_config) + quotas = sq.list_service_quotas(ServiceCode="bedrock").get("Quotas", []) + + tpm_quotas = [q for q in quotas if "token" in q.get("QuotaName", "").lower()] + rpm_quotas = [q for q in quotas if "request" in q.get("QuotaName", "").lower()] + + # Flag if no custom quota increases have been requested (still at default) + default_only = all(not q.get("Adjustable") for q in tpm_quotas + rpm_quotas) + + details = f"Found {len(tpm_quotas)} token-based and {len(rpm_quotas)} request-based Bedrock quotas." + findings["csv_data"].append( + create_finding( + check_id="FS-03", + finding_name="Bedrock Token and Request Quota Review", + finding_details=details, + resolution=( + "1. Review current Bedrock TPM/RPM quotas in Service Quotas console.\n" + "2. Request increases aligned with expected peak load.\n" + "3. Implement client-side token counting and pre-flight quota checks.\n" + "4. Use Bedrock cross-region inference profiles to distribute load." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/quotas.html", + severity="Medium", + status="Passed" if not default_only else "Failed", + ) + ) + except Exception as e: + return _error_findings("Bedrock Token Quota Review", e) + return findings + + +def check_cost_anomaly_detection() -> Dict[str, Any]: + """ + FS-04 — Verify AWS Cost Anomaly Detection monitors are configured for + Bedrock and SageMaker services. + COMPLIANCE_PLACEHOLDER: [FFIEC CAT, SR 11-7 Appendix A] + """ + findings = _empty_findings("Cost Anomaly Detection Check") + try: + ce = boto3.client("ce", config=boto3_config) + monitors = ce.get_anomaly_monitors().get("AnomalyMonitors", []) + + bedrock_monitors = [ + m + for m in monitors + if "bedrock" in json.dumps(m.get("MonitorSpecification", {})).lower() + or m.get("MonitorType") == "DIMENSIONAL" + ] + + if not monitors: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-04", + finding_name="No Cost Anomaly Detection Monitors", + finding_details=( + "No AWS Cost Anomaly Detection monitors found. Unexpected spikes in " + "Bedrock/SageMaker usage (e.g., from prompt injection loops) will go undetected." + ), + resolution=( + "1. Create a Cost Anomaly Detection monitor scoped to AWS/Bedrock and AWS/SageMaker.\n" + "2. Configure alert subscriptions (SNS/email) for anomalies above threshold.\n" + "3. Set daily spend budgets with AWS Budgets as a secondary control." + ), + reference="https://docs.aws.amazon.com/cost-management/latest/userguide/getting-started-ad.html", + severity="Medium", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-04", + finding_name="Cost Anomaly Detection Configured", + finding_details=f"Found {len(monitors)} anomaly monitor(s); {len(bedrock_monitors)} appear Bedrock-related.", + resolution="Verify monitors cover Bedrock and SageMaker service dimensions.", + reference="https://docs.aws.amazon.com/cost-management/latest/userguide/getting-started-ad.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Cost Anomaly Detection Check", e) + return findings + + +def check_cloudwatch_token_alarms() -> Dict[str, Any]: + """ + FS-05 — Check for CloudWatch alarms on Bedrock InvocationThrottles and + TokensProcessed metrics to detect runaway consumption. + COMPLIANCE_PLACEHOLDER: [FFIEC CAT, DORA Art.6] + """ + findings = _empty_findings("CloudWatch Token Usage Alarms Check") + try: + cw = boto3.client("cloudwatch", config=boto3_config) + paginator = cw.get_paginator("describe_alarms") + all_alarms = [] + for page in paginator.paginate(AlarmTypes=["MetricAlarm"]): + all_alarms.extend(page.get("MetricAlarms", [])) + + bedrock_alarms = [ + a + for a in all_alarms + if a.get("Namespace", "").startswith("AWS/Bedrock") + or "bedrock" in a.get("AlarmName", "").lower() + ] + + throttle_alarms = [ + a + for a in bedrock_alarms + if "throttl" in a.get("MetricName", "").lower() + or "throttl" in a.get("AlarmName", "").lower() + ] + + if not bedrock_alarms: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-05", + finding_name="No Bedrock CloudWatch Alarms Found", + finding_details=( + "No CloudWatch alarms found for Bedrock metrics. " + "Token exhaustion and throttling events will not trigger operational alerts." + ), + resolution=( + "Create CloudWatch alarms for:\n" + "- AWS/Bedrock InvocationThrottles (threshold > 0)\n" + "- AWS/Bedrock TokensProcessed (threshold based on quota)\n" + "- Custom application-level token counters via EMF" + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/monitoring-cw.html", + severity="Medium", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-05", + finding_name="Bedrock CloudWatch Alarms Present", + finding_details=( + f"Found {len(bedrock_alarms)} Bedrock-related alarm(s), " + f"{len(throttle_alarms)} covering throttling." + ), + resolution="Ensure alarms have SNS actions and are in OK state.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/monitoring-cw.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("CloudWatch Token Usage Alarms Check", e) + return findings + + +def check_aws_budgets_for_aiml() -> Dict[str, Any]: + """ + FS-06 — Verify AWS Budgets are configured with alerts for AI/ML service spend. + COMPLIANCE_PLACEHOLDER: [FFIEC CAT, SR 11-7] + """ + findings = _empty_findings("AWS Budgets AI/ML Spend Check") + try: + budgets_client = boto3.client("budgets", config=boto3_config) + sts = boto3.client("sts", config=boto3_config) + account_id = sts.get_caller_identity()["Account"] + + all_budgets = budgets_client.describe_budgets(AccountId=account_id).get( + "Budgets", [] + ) + aiml_budgets = [ + b + for b in all_budgets + if any( + svc in json.dumps(b.get("CostFilters", {})).lower() + for svc in ["bedrock", "sagemaker"] + ) + ] + + if not aiml_budgets: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-06", + finding_name="No AI/ML Service Budgets Configured", + finding_details=( + "No AWS Budgets found scoped to Bedrock or SageMaker. " + "Unbounded GenAI spend can go undetected until the monthly bill." + ), + resolution=( + "1. Create cost budgets for AWS Bedrock and SageMaker with 80%/100% alert thresholds.\n" + "2. Add SNS notifications to on-call channels.\n" + "3. Consider budget actions to apply IAM deny policies when thresholds are breached." + ), + reference="https://docs.aws.amazon.com/cost-management/latest/userguide/budgets-managing-costs.html", + severity="Medium", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-06", + finding_name="AI/ML Service Budgets Configured", + finding_details=f"Found {len(aiml_budgets)} budget(s) covering AI/ML services.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/cost-management/latest/userguide/budgets-managing-costs.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("AWS Budgets AI/ML Spend Check", e) + return findings + + +# =========================================================================== +# CATEGORY 2: EXCESSIVE AGENCY (FS-07 to FS-11) +# Risk: Agents take unintended real-world actions beyond their intended scope +# COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, DORA Art.6, MAS TRM 9] +# =========================================================================== + + +def check_bedrock_agent_action_boundaries(permission_cache) -> Dict[str, Any]: + """ + FS-07 — Verify Bedrock agent execution roles have narrow action boundaries + (no wildcard actions on sensitive services like s3:*, iam:*, ec2:*). + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT Cyber Risk Management] + """ + findings = _empty_findings("Agent Action Boundary Check") + try: + bedrock_agent = boto3.client("bedrock-agent", config=boto3_config) + agents = bedrock_agent.list_agents().get("agentSummaries", []) + + if not agents: + findings["csv_data"].append( + create_finding( + check_id="FS-07", + finding_name="Agent Action Boundary Check", + finding_details="No Bedrock agents found.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/agents-permissions.html", + severity="Informational", + status="N/A", + ) + ) + return findings + + SENSITIVE_WILDCARDS = ["iam:*", "s3:*", "ec2:*", "lambda:*", "*"] + agents_with_issues = [] + + for agent_summary in agents: + agent_id = agent_summary["agentId"] + agent_name = agent_summary["agentName"] + detail = bedrock_agent.get_agent(agentId=agent_id) + role_arn = detail.get("agent", {}).get("agentResourceRoleArn", "") + if not role_arn: + continue + role_name = role_arn.split("/")[-1] + role_perms = ( + (permission_cache or {}).get("role_permissions", {}).get(role_name, {}) + ) + for policy in role_perms.get("attached_policies", []) + role_perms.get( + "inline_policies", [] + ): + doc = policy.get("document", {}) + if isinstance(doc, str): + doc = json.loads(doc) + for stmt in doc.get("Statement", []): + if stmt.get("Effect") != "Allow": + continue + actions = stmt.get("Action", []) + if isinstance(actions, str): + actions = [actions] + for action in actions: + if action in SENSITIVE_WILDCARDS: + agents_with_issues.append( + f"Agent '{agent_name}' role '{role_name}' allows '{action}'" + ) + + if agents_with_issues: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-07", + finding_name="Bedrock Agent Overly Broad Action Permissions", + finding_details=( + "The following agents have execution roles with wildcard or overly broad actions:\n" + + "\n".join(f"- {i}" for i in agents_with_issues[:10]) + ), + resolution=( + "1. Replace wildcard actions with specific actions the agent needs.\n" + "2. Apply permission boundaries to agent execution roles.\n" + "3. Use resource-level conditions to restrict to specific ARNs.\n" + "4. Implement human-in-the-loop approval for high-impact actions." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/agents-permissions.html", + severity="High", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-07", + finding_name="Agent Action Boundaries Look Appropriate", + finding_details=f"Reviewed {len(agents)} agent(s); no wildcard sensitive actions found.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/agents-permissions.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Agent Action Boundary Check", e) + return findings + + +def check_agentcore_policy_engine() -> Dict[str, Any]: + """ + FS-08 — Check whether Bedrock AgentCore Policy Engine is configured to + enforce action-level authorization for agent tool calls. + COMPLIANCE_PLACEHOLDER: [SR 11-7, MAS TRM 9.1] + """ + findings = _empty_findings("AgentCore Policy Engine Check") + try: + # AgentCore policy engine is checked via bedrock-agentcore control plane + agentcore = boto3.client("bedrock-agentcore-control", config=boto3_config) + try: + # List policy stores (policy engine resources) + response = agentcore.list_agent_runtimes() + runtimes = response.get("agentRuntimes", []) + except ClientError as e: + if "AccessDenied" in str(e) or "UnrecognizedClientException" in str(e): + findings["csv_data"].append( + create_finding( + check_id="FS-08", + finding_name="AgentCore Policy Engine — Access Check", + finding_details="Unable to enumerate AgentCore runtimes (access denied or service unavailable in region).", + resolution="Ensure assessment role has bedrock-agentcore:ListAgentRuntimes permission.", + reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/security-authorization.html", + severity="Low", + status="N/A", + ) + ) + return findings + raise + + if not runtimes: + findings["csv_data"].append( + create_finding( + check_id="FS-08", + finding_name="No AgentCore Runtimes Found", + finding_details="No AgentCore runtimes found; policy engine check not applicable.", + resolution="If using AgentCore, configure the Policy Engine to authorize tool calls.", + reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/security-authorization.html", + severity="Informational", + status="N/A", + ) + ) + else: + # Check each runtime for policy engine association + runtimes_without_policy = [ + r["agentRuntimeName"] + for r in runtimes + if not r.get("authorizerConfiguration") + ] + if runtimes_without_policy: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-08", + finding_name="AgentCore Runtimes Missing Policy Engine", + finding_details=( + f"Runtimes without authorizer configuration: {', '.join(runtimes_without_policy)}. " + "Without a policy engine, agents can invoke any registered tool without authorization checks." + ), + resolution=( + "Configure an authorizer (Lambda or Cedar policy store) on each AgentCore runtime " + "to enforce fine-grained tool-call authorization." + ), + reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/security-authorization.html", + severity="High", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-08", + finding_name="AgentCore Policy Engine Configured", + finding_details=f"All {len(runtimes)} runtime(s) have authorizer configurations.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/security-authorization.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("AgentCore Policy Engine Check", e) + return findings + + +def check_agent_transaction_limits() -> Dict[str, Any]: + """ + FS-09 — Check for application-level transaction/action limits on agents + via Lambda concurrency limits or Step Functions execution limits. + COMPLIANCE_PLACEHOLDER: [FFIEC CAT, SR 11-7] + """ + findings = _empty_findings("Agent Transaction Limits Check") + try: + lambda_client = boto3.client("lambda", config=boto3_config) + functions = lambda_client.list_functions().get("Functions", []) + + # Look for agent-related Lambda functions without reserved concurrency + agent_lambdas = [ + f + for f in functions + if any( + kw in f["FunctionName"].lower() for kw in ["agent", "bedrock", "aiml"] + ) + ] + + lambdas_without_concurrency = [] + for fn in agent_lambdas: + try: + config = lambda_client.get_function_concurrency( + FunctionName=fn["FunctionName"] + ) + if not config.get("ReservedConcurrentExecutions"): + lambdas_without_concurrency.append(fn["FunctionName"]) + except ClientError: + lambdas_without_concurrency.append(fn["FunctionName"]) + + if lambdas_without_concurrency: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-09", + finding_name="Agent Lambda Functions Without Concurrency Limits", + finding_details=( + f"Agent-related Lambda functions without reserved concurrency: " + f"{', '.join(lambdas_without_concurrency[:10])}. " + "Unlimited concurrency allows runaway agent loops to exhaust account limits." + ), + resolution=( + "1. Set reserved concurrency on agent Lambda functions.\n" + "2. Implement maximum iteration counts in agent orchestration logic.\n" + "3. Use Step Functions with MaxConcurrency and timeout states.\n" + "4. Add circuit-breaker patterns to agent tool invocations." + ), + reference="https://docs.aws.amazon.com/lambda/latest/dg/configuration-concurrency.html", + severity="Medium", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-09", + finding_name="Agent Lambda Concurrency Limits Present", + finding_details=f"Reviewed {len(agent_lambdas)} agent Lambda(s); concurrency limits appear configured.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/lambda/latest/dg/configuration-concurrency.html", + severity="Informational", + status="Passed" if agent_lambdas else "N/A", + ) + ) + except Exception as e: + return _error_findings("Agent Transaction Limits Check", e) + return findings + + +def check_human_in_the_loop_for_high_risk_actions() -> Dict[str, Any]: + """ + FS-10 — Check for Step Functions or SNS-based human approval steps in + agent workflows that perform high-risk financial actions. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2] + """ + findings = _empty_findings("Human-in-the-Loop Approval Check") + try: + sfn = boto3.client("stepfunctions", config=boto3_config) + machines = sfn.list_state_machines().get("stateMachines", []) + + agent_machines = [ + m + for m in machines + if any( + kw in m["name"].lower() + for kw in ["agent", "approval", "human", "review"] + ) + ] + + machines_with_wait = [] + for machine in agent_machines: + defn = sfn.describe_state_machine( + stateMachineArn=machine["stateMachineArn"] + ).get("definition", "{}") + if '"waitForTaskToken"' in defn or '"TaskToken"' in defn: + machines_with_wait.append(machine["name"]) + + if not agent_machines: + findings["csv_data"].append( + create_finding( + check_id="FS-10", + finding_name="Human-in-the-Loop Check — No Agent Workflows Found", + finding_details=( + "No Step Functions state machines with agent/approval naming found. " + "Verify that high-risk agent actions (e.g., fund transfers, account changes) " + "have human approval gates." + ), + resolution=( + "Implement Step Functions .waitForTaskToken patterns for high-risk agent actions. " + "Route approval requests to human reviewers via SNS/SES/Slack." + ), + reference="https://docs.aws.amazon.com/step-functions/latest/dg/connect-to-resource.html#connect-wait-token", + severity="Medium", + status="N/A", + ) + ) + elif machines_with_wait: + findings["csv_data"].append( + create_finding( + check_id="FS-10", + finding_name="Human Approval Steps Found in Agent Workflows", + finding_details=f"State machines with waitForTaskToken (human approval): {', '.join(machines_with_wait)}.", + resolution="No action required. Verify approval routing reaches the correct reviewers.", + reference="https://docs.aws.amazon.com/step-functions/latest/dg/connect-to-resource.html#connect-wait-token", + severity="Informational", + status="Passed", + ) + ) + else: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-10", + finding_name="Agent Workflows Missing Human Approval Steps", + finding_details=( + f"Found {len(agent_machines)} agent-related state machine(s) but none use " + "waitForTaskToken for human approval. High-risk financial actions may execute autonomously." + ), + resolution=( + "Add .waitForTaskToken states before irreversible financial actions. " + "Define risk tiers and require human approval for Tier 1 actions." + ), + reference="https://docs.aws.amazon.com/step-functions/latest/dg/connect-to-resource.html#connect-wait-token", + severity="High", + status="Failed", + ) + ) + except Exception as e: + return _error_findings("Human-in-the-Loop Approval Check", e) + return findings + + +def check_agent_rate_alarms() -> Dict[str, Any]: + """ + FS-11 — Check for CloudWatch alarms on agent invocation rates to detect + runaway or looping agent behavior. + COMPLIANCE_PLACEHOLDER: [FFIEC CAT, DORA Art.6] + """ + findings = _empty_findings("Agent Rate Alarms Check") + try: + cw = boto3.client("cloudwatch", config=boto3_config) + paginator = cw.get_paginator("describe_alarms") + all_alarms = [] + for page in paginator.paginate(AlarmTypes=["MetricAlarm"]): + all_alarms.extend(page.get("MetricAlarms", [])) + + agent_alarms = [ + a + for a in all_alarms + if "agent" in a.get("AlarmName", "").lower() + or "agent" in a.get("Namespace", "").lower() + ] + + if not agent_alarms: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-11", + finding_name="No Agent Rate Alarms Found", + finding_details=( + "No CloudWatch alarms found for agent invocation rates. " + "Looping or runaway agents will not trigger operational alerts." + ), + resolution=( + "Create CloudWatch alarms on:\n" + "- Bedrock agent invocation counts (threshold based on expected max)\n" + "- Lambda invocation errors for agent functions\n" + "- Step Functions execution failures and timeouts" + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/monitoring-cw.html", + severity="Medium", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-11", + finding_name="Agent Rate Alarms Present", + finding_details=f"Found {len(agent_alarms)} agent-related CloudWatch alarm(s).", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/monitoring-cw.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Agent Rate Alarms Check", e) + return findings + + +# =========================================================================== +# CATEGORY 3: SUPPLY CHAIN VULNERABILITIES (FS-12 to FS-16) +# Risk: Third-party models, datasets, or plugins introduce malicious code/data +# COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, DORA Art.6, ISO 27001 A.15] +# =========================================================================== + + +def check_scp_model_access_restrictions() -> Dict[str, Any]: + """ + FS-12 — Verify SCPs restrict Bedrock model access to an approved model list, + preventing use of unapproved third-party models. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, ISO 27001 A.15.2] + """ + findings = _empty_findings("SCP Model Access Restriction Check") + try: + orgs = boto3.client("organizations", config=boto3_config) + try: + policies = orgs.list_policies(Filter="SERVICE_CONTROL_POLICY").get( + "Policies", [] + ) + except ClientError as e: + if "AccessDenied" in str(e) or "AWSOrganizationsNotInUseException" in str( + e + ): + findings["csv_data"].append( + create_finding( + check_id="FS-12", + finding_name="SCP Model Access Check — Not in Organization", + finding_details="Account is not part of an AWS Organization or lacks SCP read access.", + resolution="If using AWS Organizations, ensure SCPs restrict Bedrock model access to approved models.", + reference="https://docs.aws.amazon.com/organizations/latest/userguide/orgs_manage_policies_scps.html", + severity="Low", + status="N/A", + ) + ) + return findings + raise + + bedrock_scps = [] + for policy in policies: + doc_response = orgs.describe_policy(PolicyId=policy["Id"]) + doc = json.loads(doc_response["Policy"]["Content"]) + if "bedrock" in json.dumps(doc).lower(): + bedrock_scps.append(policy["Name"]) + + if not bedrock_scps: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-12", + finding_name="No Bedrock-Scoped SCPs Found", + finding_details=( + "No Service Control Policies reference Bedrock. " + "Without SCPs, any account in the organization can access any Bedrock model, " + "including unapproved third-party models." + ), + resolution=( + "1. Create an SCP that denies bedrock:InvokeModel for model IDs not on the approved list.\n" + "2. Use bedrock:ModelId condition key to allowlist approved models.\n" + "3. Maintain a model inventory and update the SCP when models are approved/retired." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/security_iam_id-based-policy-examples.html", + severity="High", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-12", + finding_name="Bedrock SCPs Found", + finding_details=f"SCPs referencing Bedrock: {', '.join(bedrock_scps)}.", + resolution="Verify SCPs use bedrock:ModelId conditions to allowlist approved models.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/security_iam_id-based-policy-examples.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("SCP Model Access Restriction Check", e) + return findings + + +def check_model_inventory_tagging() -> Dict[str, Any]: + """ + FS-13 — Check that custom Bedrock models and SageMaker models are tagged + with provenance metadata (source, version, approval-date). + COMPLIANCE_PLACEHOLDER: [SR 11-7, ISO 27001 A.12.5, FFIEC CAT] + """ + findings = _empty_findings("Model Inventory Tagging Check") + try: + bedrock = boto3.client("bedrock", config=boto3_config) + sm = boto3.client("sagemaker", config=boto3_config) + REQUIRED_TAGS = {"source", "version", "approval-date"} + + untagged_models = [] + + # Check Bedrock custom models + for model in bedrock.list_custom_models().get("modelSummaries", []): + tags_response = bedrock.list_tags_for_resource( + resourceARN=model["modelArn"] + ) + tag_keys = {t["key"].lower() for t in tags_response.get("tags", [])} + missing = REQUIRED_TAGS - tag_keys + if missing: + untagged_models.append( + f"Bedrock model '{model['modelName']}' missing tags: {missing}" + ) + + # Check SageMaker registered models + for model in sm.list_models().get("Models", []): + tags_response = sm.list_tags(ResourceArn=model["ModelArn"]) + tag_keys = {t["Key"].lower() for t in tags_response.get("Tags", [])} + missing = REQUIRED_TAGS - tag_keys + if missing: + untagged_models.append( + f"SageMaker model '{model['ModelName']}' missing tags: {missing}" + ) + + if untagged_models: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-13", + finding_name="Models Missing Provenance Tags", + finding_details=( + f"{len(untagged_models)} model(s) missing required provenance tags:\n" + + "\n".join(f"- {m}" for m in untagged_models[:10]) + ), + resolution=( + "Tag all models with: source (e.g., 'aws-marketplace', 'internal'), " + "version, and approval-date. " + "Enforce tagging via SCP or AWS Config rule." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/tagging.html", + severity="Medium", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-13", + finding_name="Model Provenance Tags Present", + finding_details="All reviewed models have required provenance tags.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/tagging.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Model Inventory Tagging Check", e) + return findings + + +def check_model_onboarding_governance() -> Dict[str, Any]: + """ + FS-14 — Check for AWS Config rules or Service Catalog constraints that + enforce model onboarding governance (approved sources only). + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, ISO 27001 A.15.1] + """ + findings = _empty_findings("Model Onboarding Governance Check") + try: + config = boto3.client("config", config=boto3_config) + rules = config.describe_config_rules().get("ConfigRules", []) + + bedrock_rules = [ + r + for r in rules + if "bedrock" in r.get("ConfigRuleName", "").lower() + or "model" in r.get("ConfigRuleName", "").lower() + ] + + if not bedrock_rules: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-14", + finding_name="No Model Governance Config Rules Found", + finding_details=( + "No AWS Config rules found for Bedrock model governance. " + "Unapproved models may be deployed without detection." + ), + resolution=( + "1. Create custom AWS Config rules to detect use of non-approved Bedrock models.\n" + "2. Use AWS Service Catalog to publish approved model configurations.\n" + "3. Implement a model risk management (MRM) process per SR 11-7." + ), + reference="https://docs.aws.amazon.com/config/latest/developerguide/evaluate-config.html", + severity="Medium", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-14", + finding_name="Model Governance Config Rules Present", + finding_details=f"Found {len(bedrock_rules)} model-related Config rule(s).", + resolution="No action required.", + reference="https://docs.aws.amazon.com/config/latest/developerguide/evaluate-config.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Model Onboarding Governance Check", e) + return findings + + +def check_bedrock_model_evaluation_adversarial() -> Dict[str, Any]: + """ + FS-15 — Check whether Bedrock Model Evaluation jobs include adversarial + test datasets (robustness/red-team evaluations). + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.3] + """ + findings = _empty_findings("Adversarial Model Evaluation Check") + try: + bedrock = boto3.client("bedrock", config=boto3_config) + evals = bedrock.list_evaluation_jobs().get("jobSummaries", []) + + if not evals: + findings["csv_data"].append( + create_finding( + check_id="FS-15", + finding_name="No Bedrock Evaluation Jobs Found", + finding_details=( + "No Bedrock Model Evaluation jobs found. " + "Models have not been evaluated for adversarial robustness." + ), + resolution=( + "1. Run Bedrock Model Evaluation with adversarial/red-team datasets.\n" + "2. Use FMEval library for automated robustness testing.\n" + "3. Schedule periodic re-evaluation after model updates." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", + severity="Medium", + status="N/A", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-15", + finding_name="Bedrock Evaluation Jobs Present", + finding_details=f"Found {len(evals)} evaluation job(s). Verify adversarial datasets are included.", + resolution="Ensure evaluation datasets include adversarial/red-team test cases.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Adversarial Model Evaluation Check", e) + return findings + + +def check_ecr_image_scanning() -> Dict[str, Any]: + """ + FS-16 — Verify ECR repositories used for custom model containers have + image scanning enabled (supply chain vulnerability detection). + COMPLIANCE_PLACEHOLDER: [ISO 27001 A.12.6, FFIEC CAT, DORA Art.6] + """ + findings = _empty_findings("ECR Image Scanning Check") + try: + ecr = boto3.client("ecr", config=boto3_config) + repos = ecr.describe_repositories().get("repositories", []) + + if not repos: + findings["csv_data"].append( + create_finding( + check_id="FS-16", + finding_name="No ECR Repositories Found", + finding_details="No ECR repositories found.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/AmazonECR/latest/userguide/image-scanning.html", + severity="Informational", + status="N/A", + ) + ) + return findings + + repos_without_scanning = [ + r["repositoryName"] + for r in repos + if not r.get("imageScanningConfiguration", {}).get("scanOnPush", False) + ] + + if repos_without_scanning: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-16", + finding_name="ECR Repositories Without Image Scanning", + finding_details=( + f"{len(repos_without_scanning)} ECR repo(s) without scan-on-push: " + f"{', '.join(repos_without_scanning[:10])}." + ), + resolution=( + "Enable scan-on-push for all ECR repositories containing model containers. " + "Consider enabling Enhanced Scanning (Inspector) for CVE detection." + ), + reference="https://docs.aws.amazon.com/AmazonECR/latest/userguide/image-scanning.html", + severity="High", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-16", + finding_name="ECR Image Scanning Enabled", + finding_details=f"All {len(repos)} ECR repo(s) have scan-on-push enabled.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/AmazonECR/latest/userguide/image-scanning.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("ECR Image Scanning Check", e) + return findings + + +# =========================================================================== +# CATEGORY 4: TRAINING DATA & MODEL POISONING (FS-17 to FS-21) +# Risk: Malicious data corrupts model behavior during training or fine-tuning +# COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.3, ISO 27001 A.12] +# +# NOTE: FS-17 (Model Monitor Data Quality → SM-07), FS-18 (Model Drift Detection → SM-23), +# and FS-19 (Model Registry Approval → SM-22) are merged into upstream checks. +# See extension notes in SECURITY_CHECKS_FINSERV_PART1_INFRA_CONTROLS.md. +# =========================================================================== + + +def check_feature_store_rollback_capability() -> Dict[str, Any]: + """ + FS-20 — Check SageMaker Feature Store for versioning/offline store + configuration that enables rollback of poisoned feature data. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT] + """ + findings = _empty_findings("Feature Store Rollback Check") + try: + sm = boto3.client("sagemaker", config=boto3_config) + groups = sm.list_feature_groups().get("FeatureGroupSummaries", []) + + if not groups: + findings["csv_data"].append( + create_finding( + check_id="FS-20", + finding_name="No SageMaker Feature Groups Found", + finding_details="No SageMaker Feature Store groups found.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/sagemaker/latest/dg/feature-store.html", + severity="Informational", + status="N/A", + ) + ) + return findings + + groups_without_offline = [ + g["FeatureGroupName"] + for g in groups + if g.get("OfflineStoreStatus", {}).get("Status") != "Active" + ] + + if groups_without_offline: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-20", + finding_name="Feature Groups Without Offline Store", + finding_details=( + f"{len(groups_without_offline)} feature group(s) lack an active offline store: " + f"{', '.join(groups_without_offline[:10])}. " + "Without offline store, historical feature data cannot be used for rollback." + ), + resolution=( + "1. Enable offline store (S3-backed) for all production feature groups.\n" + "2. Enable S3 versioning on the offline store bucket.\n" + "3. Document rollback procedures for poisoned feature data." + ), + reference="https://docs.aws.amazon.com/sagemaker/latest/dg/feature-store-offline.html", + severity="Medium", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-20", + finding_name="Feature Store Offline Store Active", + finding_details=f"All {len(groups)} feature group(s) have active offline stores.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/sagemaker/latest/dg/feature-store-offline.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Feature Store Rollback Check", e) + return findings + + +def check_training_data_s3_versioning() -> Dict[str, Any]: + """ + FS-21 — Verify S3 buckets used for training data have versioning enabled + to support rollback of poisoned datasets. + COMPLIANCE_PLACEHOLDER: [SR 11-7, ISO 27001 A.12.3, FFIEC CAT] + """ + findings = _empty_findings("Training Data S3 Versioning Check") + try: + s3 = boto3.client("s3", config=boto3_config) + buckets = s3.list_buckets().get("Buckets", []) + + training_buckets = [ + b + for b in buckets + if any( + kw in b["Name"].lower() + for kw in ["train", "dataset", "model", "sagemaker", "bedrock"] + ) + ] + + if not training_buckets: + findings["csv_data"].append( + create_finding( + check_id="FS-21", + finding_name="No Training Data Buckets Identified", + finding_details="No S3 buckets with training/model naming found.", + resolution="Tag training data buckets and enable versioning.", + reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/Versioning.html", + severity="Informational", + status="N/A", + ) + ) + return findings + + unversioned = [] + for bucket in training_buckets: + versioning = s3.get_bucket_versioning(Bucket=bucket["Name"]) + if versioning.get("Status") != "Enabled": + unversioned.append(bucket["Name"]) + + if unversioned: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-21", + finding_name="Training Data Buckets Without Versioning", + finding_details=( + f"{len(unversioned)} training data bucket(s) without versioning: " + f"{', '.join(unversioned[:10])}." + ), + resolution=( + "Enable S3 versioning on all training data buckets. " + "Consider enabling MFA Delete for additional protection against poisoning." + ), + reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/Versioning.html", + severity="High", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-21", + finding_name="Training Data Buckets Have Versioning", + finding_details=f"All {len(training_buckets)} training bucket(s) have versioning enabled.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/Versioning.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Training Data S3 Versioning Check", e) + return findings + + +# =========================================================================== +# CATEGORY 5: VECTOR & EMBEDDING WEAKNESSES (FS-22 to FS-26) +# Risk: Knowledge base / RAG vector stores are improperly secured +# COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, NYDFS 500.06, PCI-DSS 12.3.2] +# =========================================================================== + + +def check_knowledge_base_iam_least_privilege(permission_cache) -> Dict[str, Any]: + """ + FS-22 — Verify IAM roles accessing Bedrock Knowledge Bases follow + least privilege (no wildcard bedrock-agent:* permissions). + COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, FFIEC CAT, PCI-DSS 12.3.2] + """ + findings = _empty_findings("Knowledge Base IAM Least Privilege Check") + try: + issues = [] + for role_name, perms in ( + (permission_cache or {}).get("role_permissions", {}).items() + ): + for policy in perms.get("attached_policies", []) + perms.get( + "inline_policies", [] + ): + doc = policy.get("document", {}) + if isinstance(doc, str): + doc = json.loads(doc) + for stmt in doc.get("Statement", []): + if stmt.get("Effect") != "Allow": + continue + actions = stmt.get("Action", []) + if isinstance(actions, str): + actions = [actions] + for action in actions: + if action in ("bedrock-agent:*", "bedrock:*", "*"): + issues.append(f"Role '{role_name}' allows '{action}'") + + if issues: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-22", + finding_name="Overly Permissive Knowledge Base IAM Roles", + finding_details=( + f"{len(issues)} role(s) with wildcard KB permissions:\n" + + "\n".join(f"- {i}" for i in issues[:10]) + ), + resolution=( + "Replace wildcard bedrock-agent:* with specific actions: " + "bedrock:Retrieve, bedrock:RetrieveAndGenerate. " + "Scope resources to specific Knowledge Base ARNs." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/security-iam-awsmanpol.html", + severity="High", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-22", + finding_name="Knowledge Base IAM Permissions Look Appropriate", + finding_details="No wildcard KB permissions found in reviewed roles.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/security-iam-awsmanpol.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Knowledge Base IAM Least Privilege Check", e) + return findings + + +def check_knowledge_base_metadata_filtering() -> Dict[str, Any]: + """ + FS-24 — Check that Bedrock Knowledge Bases have metadata fields configured + to support tenant-level filtering (multi-tenancy isolation). + COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, FFIEC CAT, PCI-DSS 12.3.2] + """ + findings = _empty_findings("Knowledge Base Metadata Filtering Check") + try: + bedrock_agent = boto3.client("bedrock-agent", config=boto3_config) + paginator = bedrock_agent.get_paginator("list_knowledge_bases") + kbs = [] + for page in paginator.paginate(): + kbs.extend(page.get("knowledgeBaseSummaries", [])) + + if not kbs: + findings["csv_data"].append( + create_finding( + check_id="FS-24", + finding_name="No Knowledge Bases Found", + finding_details="No Bedrock Knowledge Bases found.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/kb-test-config.html", + severity="Informational", + status="N/A", + ) + ) + return findings + + # Advisory check — metadata filtering is a design pattern, not directly inspectable + findings["csv_data"].append( + create_finding( + check_id="FS-24", + finding_name="Knowledge Base Metadata Filtering — Manual Review Required", + finding_details=( + f"Found {len(kbs)} Knowledge Base(s). " + "Verify that metadata attributes (e.g., tenantId, classification) are indexed " + "and that Retrieve calls include RetrievalFilter conditions for tenant isolation." + ), + resolution=( + "1. Add metadata fields (tenantId, dataClassification) to KB data sources.\n" + "2. Pass RetrievalFilter in all Retrieve/RetrieveAndGenerate calls.\n" + "3. Validate filters in integration tests to prevent cross-tenant data leakage." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/kb-test-config.html", + severity="Medium", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Knowledge Base Metadata Filtering Check", e) + return findings + + +def check_opensearch_serverless_encryption() -> Dict[str, Any]: + """ + FS-25 — Verify OpenSearch Serverless collections (used as KB vector stores) + have encryption policies with customer-managed KMS keys. + COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, PCI-DSS 3.5, FFIEC CAT] + """ + findings = _empty_findings("OpenSearch Serverless Encryption Check") + try: + oss = boto3.client("opensearchserverless", config=boto3_config) + policies = oss.list_security_policies(type="encryption").get( + "securityPolicySummaries", [] + ) + + if not policies: + findings["csv_data"].append( + create_finding( + check_id="FS-25", + finding_name="No OpenSearch Serverless Encryption Policies", + finding_details=( + "No OpenSearch Serverless encryption policies found. " + "Vector embeddings may be stored without customer-managed encryption." + ), + resolution=( + "Create encryption security policies for OpenSearch Serverless collections " + "used as Bedrock KB vector stores, specifying a customer-managed KMS key." + ), + reference="https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-encryption.html", + severity="High", + status="N/A", + ) + ) + else: + # Check for CMK usage + cmk_policies = [] + for policy in policies: + doc = json.loads(policy.get("policy", "{}")) + if "AWSOwnedKey" not in json.dumps(doc): + cmk_policies.append(policy["name"]) + + findings["csv_data"].append( + create_finding( + check_id="FS-25", + finding_name="OpenSearch Serverless Encryption Policies Present", + finding_details=( + f"Found {len(policies)} encryption policy(ies); " + f"{len(cmk_policies)} appear to use CMK." + ), + resolution="Verify all vector store collections use customer-managed KMS keys.", + reference="https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-encryption.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("OpenSearch Serverless Encryption Check", e) + return findings + + +def check_knowledge_base_vpc_access() -> Dict[str, Any]: + """ + FS-26 — Verify OpenSearch Serverless collections have VPC access policies + restricting access to private network endpoints. + COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, FFIEC CAT, PCI-DSS 1.3] + """ + findings = _empty_findings("Knowledge Base VPC Access Check") + try: + oss = boto3.client("opensearchserverless", config=boto3_config) + network_policies = oss.list_security_policies(type="network").get( + "securityPolicySummaries", [] + ) + + if not network_policies: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-26", + finding_name="No OpenSearch Serverless Network Policies", + finding_details=( + "No OpenSearch Serverless network policies found. " + "Vector store collections may be publicly accessible." + ), + resolution=( + "Create network security policies for OpenSearch Serverless collections " + "restricting access to VPC endpoints only." + ), + reference="https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-network.html", + severity="High", + status="Failed", + ) + ) + else: + vpc_policies = [] + for policy in network_policies: + doc = json.loads(policy.get("policy", "{}")) + if "vpc" in json.dumps(doc).lower(): + vpc_policies.append(policy["name"]) + + if not vpc_policies: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-26", + finding_name="OpenSearch Serverless Collections Not VPC-Restricted", + finding_details=( + f"Found {len(network_policies)} network policy(ies) but none restrict to VPC. " + "Vector stores may be accessible from the public internet." + ), + resolution=( + "Update network policies to allow access only from VPC endpoints. " + "Create an OpenSearch Serverless VPC endpoint in your VPC." + ), + reference="https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-network.html", + severity="High", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-26", + finding_name="OpenSearch Serverless VPC Access Configured", + finding_details=f"{len(vpc_policies)} network policy(ies) restrict to VPC.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-network.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Knowledge Base VPC Access Check", e) + return findings + + +# =========================================================================== +# CATEGORY 6: NON-COMPLIANT OUTPUT (FS-27 to FS-30) +# Risk: GenAI outputs violate regulatory requirements (e.g., fair lending, disclosures) +# COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, NYDFS 500, MAS TRM 9.2] +# =========================================================================== + + +def check_automated_reasoning_checks() -> Dict[str, Any]: + """ + FS-27 — Check whether Bedrock Guardrails have Automated Reasoning checks + configured to validate factual accuracy of outputs. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2] + """ + findings = _empty_findings("Automated Reasoning Checks") + try: + bedrock = boto3.client("bedrock", config=boto3_config) + guardrails = bedrock.list_guardrails().get("guardrails", []) + + if not guardrails: + findings["csv_data"].append( + create_finding( + check_id="FS-27", + finding_name="No Guardrails — Automated Reasoning Not Applicable", + finding_details="No Bedrock Guardrails configured. Configure guardrails first (see BR-05).", + resolution="Configure Bedrock Guardrails with contextual grounding checks.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", + severity="Medium", + status="N/A", + ) + ) + return findings + + guardrails_with_grounding = [] + for g in guardrails: + detail = bedrock.get_guardrail( + guardrailIdentifier=g["id"], guardrailVersion="DRAFT" + ) + if detail.get("contextualGroundingPolicy"): + guardrails_with_grounding.append(g["name"]) + + if not guardrails_with_grounding: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-27", + finding_name="No Guardrails With Contextual Grounding", + finding_details=( + f"Found {len(guardrails)} guardrail(s) but none have contextual grounding enabled. " + "Non-compliant outputs (hallucinations, regulatory violations) will not be filtered." + ), + resolution=( + "Enable contextual grounding checks on Bedrock Guardrails with:\n" + "- Grounding threshold (0.7+ recommended for financial advice)\n" + "- Relevance threshold to filter off-topic responses" + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", + severity="High", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-27", + finding_name="Contextual Grounding Enabled on Guardrails", + finding_details=f"Guardrails with grounding: {', '.join(guardrails_with_grounding)}.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Automated Reasoning Checks", e) + return findings + + +def check_guardrail_denied_topics_financial() -> Dict[str, Any]: + """ + FS-28 — Verify Bedrock Guardrails have denied topics configured for + regulated financial advice categories (investment advice, credit decisions). + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, NYDFS 500, MAS TRM 9.2] + """ + findings = _empty_findings("Financial Denied Topics Check") + try: + bedrock = boto3.client("bedrock", config=boto3_config) + guardrails = bedrock.list_guardrails().get("guardrails", []) + + if not guardrails: + findings["csv_data"].append( + create_finding( + check_id="FS-28", + finding_name="No Guardrails — Denied Topics Not Applicable", + finding_details="No Bedrock Guardrails configured.", + resolution="Configure guardrails with denied topics for regulated financial content.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", + severity="Medium", + status="N/A", + ) + ) + return findings + + guardrails_with_topics = [] + for g in guardrails: + detail = bedrock.get_guardrail( + guardrailIdentifier=g["id"], guardrailVersion="DRAFT" + ) + if detail.get("topicPolicy", {}).get("topics"): + guardrails_with_topics.append(g["name"]) + + if not guardrails_with_topics: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-28", + finding_name="No Guardrails With Denied Financial Topics", + finding_details=( + "No guardrails have topic policies configured. " + "GenAI may provide regulated financial advice without controls." + ), + resolution=( + "Add denied topics to guardrails for:\n" + "- Specific investment advice (securities recommendations)\n" + "- Credit/lending decisions\n" + "- Insurance underwriting advice\n" + "- Tax advice beyond general information" + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", + severity="High", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-28", + finding_name="Guardrails With Topic Policies Found", + finding_details=f"Guardrails with topic policies: {', '.join(guardrails_with_topics)}.", + resolution="Verify topics cover regulated financial advice categories.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Financial Denied Topics Check", e) + return findings + + +def check_compliance_disclaimer_in_outputs() -> Dict[str, Any]: + """ + FS-29 — Advisory check: verify application-level disclaimers are added to + GenAI outputs for regulated financial content (not directly checkable via API). + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, NYDFS 500, MAS TRM 9.2] + """ + findings = _empty_findings("Compliance Disclaimer Check") + # This is an advisory/manual check — no AWS API can verify application-level disclaimers + findings["csv_data"].append( + create_finding( + check_id="FS-29", + finding_name="Compliance Disclaimer — Manual Review Required", + finding_details=( + "Application-level compliance disclaimers cannot be verified via AWS APIs. " + "Manual review required to confirm GenAI outputs include required regulatory disclosures." + ), + resolution=( + "1. Implement post-processing to append required disclaimers to GenAI outputs.\n" + "2. Use Bedrock Guardrails word filters to block outputs that omit required disclosures.\n" + "3. Document disclaimer requirements in the AI use case register.\n" + "4. Test disclaimer presence in QA/UAT before production deployment." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", + severity="Medium", + status="Passed", + ) + ) + return findings + + +def check_bedrock_evaluation_compliance_datasets() -> Dict[str, Any]: + """ + FS-30 — Check whether Bedrock Model Evaluation jobs use compliance-specific + datasets (e.g., fair lending, UDAP, ECOA test cases). + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, NYDFS 500] + """ + findings = _empty_findings("Compliance Evaluation Datasets Check") + try: + bedrock = boto3.client("bedrock", config=boto3_config) + evals = bedrock.list_evaluation_jobs().get("jobSummaries", []) + + if not evals: + findings["csv_data"].append( + create_finding( + check_id="FS-30", + finding_name="No Bedrock Evaluation Jobs — Compliance Datasets Not Verified", + finding_details="No Bedrock Model Evaluation jobs found.", + resolution=( + "Run Bedrock Model Evaluation with compliance-specific datasets:\n" + "- Fair lending test cases (ECOA, Fair Housing Act)\n" + "- UDAP/UDAAP unfair/deceptive practice scenarios\n" + "- AML/KYC edge cases" + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", + severity="Medium", + status="N/A", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-30", + finding_name="Bedrock Evaluation Jobs Present", + finding_details=f"Found {len(evals)} evaluation job(s). Verify compliance datasets are included.", + resolution="Ensure evaluation datasets include FinServ regulatory test cases.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Compliance Evaluation Datasets Check", e) + return findings + + +# =========================================================================== +# CATEGORY 7: MISINFORMATION (FS-31 to FS-34) +# Risk: GenAI outputs contain false or misleading financial information +# COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2, NYDFS 500] +# =========================================================================== + + +def check_knowledge_base_data_source_sync() -> Dict[str, Any]: + """ + FS-31 — Verify Bedrock Knowledge Base data sources have recent sync jobs + to ensure information currency. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT] + """ + findings = _empty_findings("Knowledge Base Data Source Sync Check") + try: + bedrock_agent = boto3.client("bedrock-agent", config=boto3_config) + paginator = bedrock_agent.get_paginator("list_knowledge_bases") + kbs = [] + for page in paginator.paginate(): + kbs.extend(page.get("knowledgeBaseSummaries", [])) + + if not kbs: + findings["csv_data"].append( + create_finding( + check_id="FS-31", + finding_name="No Knowledge Bases Found", + finding_details="No Bedrock Knowledge Bases found.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base-ingest.html", + severity="Informational", + status="N/A", + ) + ) + return findings + + stale_kbs = [] + now = datetime.now(timezone.utc) + for kb in kbs: + kb_id = kb["knowledgeBaseId"] + sources = bedrock_agent.list_data_sources(knowledgeBaseId=kb_id).get( + "dataSourceSummaries", [] + ) + for source in sources: + last_updated = source.get("updatedAt") + if last_updated: + age_days = (now - last_updated).days + if age_days > 7: + stale_kbs.append( + f"KB '{kb['name']}' source '{source['name']}' last synced {age_days} days ago" + ) + + if stale_kbs: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-31", + finding_name="Stale Knowledge Base Data Sources", + finding_details=( + f"{len(stale_kbs)} data source(s) not synced in >7 days:\n" + + "\n".join(f"- {s}" for s in stale_kbs[:10]) + ), + resolution=( + "1. Configure automated sync schedules for KB data sources.\n" + "2. Set CloudWatch alarms on sync job failures.\n" + "3. Define maximum acceptable data age per use case (e.g., 24h for market data)." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base-ingest.html", + severity="Medium", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-31", + finding_name="Knowledge Base Data Sources Recently Synced", + finding_details="All reviewed KB data sources synced within 7 days.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base-ingest.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Knowledge Base Data Source Sync Check", e) + return findings + + +def check_source_attribution_in_guardrails() -> Dict[str, Any]: + """ + FS-32 — Advisory check: verify application implements source attribution + (citations) in GenAI responses to enable fact-checking. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2] + """ + findings = _empty_findings("Source Attribution Check") + findings["csv_data"].append( + create_finding( + check_id="FS-32", + finding_name="Source Attribution — Manual Review Required", + finding_details=( + "Source attribution in GenAI responses cannot be verified via AWS APIs. " + "Manual review required to confirm responses include citations." + ), + resolution=( + "1. Use Bedrock RetrieveAndGenerate with citations enabled.\n" + "2. Include source document references in response post-processing.\n" + "3. Test citation accuracy in QA before production deployment.\n" + "4. Consider Bedrock Guardrails grounding checks to validate response accuracy." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/kb-test-config.html", + severity="Medium", + status="Passed", + ) + ) + return findings + + +def check_knowledge_base_integrity_monitoring() -> Dict[str, Any]: + """ + FS-33 — Check for S3 object integrity monitoring (checksums, versioning) + on Knowledge Base data source buckets. + COMPLIANCE_PLACEHOLDER: [SR 11-7, ISO 27001 A.12.3, FFIEC CAT] + """ + findings = _empty_findings("Knowledge Base Integrity Monitoring Check") + try: + bedrock_agent = boto3.client("bedrock-agent", config=boto3_config) + s3 = boto3.client("s3", config=boto3_config) + + paginator = bedrock_agent.get_paginator("list_knowledge_bases") + kbs = [] + for page in paginator.paginate(): + kbs.extend(page.get("knowledgeBaseSummaries", [])) + + if not kbs: + findings["csv_data"].append( + create_finding( + check_id="FS-33", + finding_name="No Knowledge Bases Found", + finding_details="No Bedrock Knowledge Bases found.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html", + severity="Informational", + status="N/A", + ) + ) + return findings + + buckets_without_versioning = [] + for kb in kbs[:10]: + sources = bedrock_agent.list_data_sources( + knowledgeBaseId=kb["knowledgeBaseId"] + ).get("dataSourceSummaries", []) + for source in sources: + source_detail = bedrock_agent.get_data_source( + knowledgeBaseId=kb["knowledgeBaseId"], + dataSourceId=source["dataSourceId"], + ) + s3_config = ( + source_detail.get("dataSource", {}) + .get("dataSourceConfiguration", {}) + .get("s3Configuration", {}) + ) + bucket = s3_config.get("bucketArn", "").split(":::")[-1] + if bucket: + versioning = s3.get_bucket_versioning(Bucket=bucket) + if versioning.get("Status") != "Enabled": + buckets_without_versioning.append(bucket) + + if buckets_without_versioning: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-33", + finding_name="KB Data Source Buckets Without Versioning", + finding_details=( + f"KB data source S3 buckets without versioning: " + f"{', '.join(buckets_without_versioning[:10])}." + ), + resolution=( + "Enable S3 versioning on all KB data source buckets. " + "Enable S3 Object Integrity (checksum) for tamper detection." + ), + reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html", + severity="Medium", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-33", + finding_name="KB Data Source Buckets Have Versioning", + finding_details="All reviewed KB data source buckets have versioning enabled.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Knowledge Base Integrity Monitoring Check", e) + return findings + + +def check_fm_version_currency() -> Dict[str, Any]: + """ + FS-34 — Advisory check: verify foundation model versions in use are current + and not deprecated (outdated models may have stale training data). + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT] + """ + findings = _empty_findings("Foundation Model Version Currency Check") + try: + bedrock = boto3.client("bedrock", config=boto3_config) + models = bedrock.list_foundation_models(byOutputModality="TEXT").get( + "modelSummaries", [] + ) + + deprecated = [ + m["modelId"] + for m in models + if m.get("modelLifecycle", {}).get("status") == "LEGACY" + ] + + if deprecated: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-34", + finding_name="Legacy Foundation Models in Use", + finding_details=( + f"Legacy/deprecated foundation models available: {', '.join(deprecated[:10])}. " + "These models have older training data cutoffs and may produce outdated information." + ), + resolution=( + "1. Migrate to current model versions.\n" + "2. Document training data cutoff dates for all models in use.\n" + "3. Add data currency disclaimers to outputs from models with old cutoffs." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle.html", + severity="Medium", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-34", + finding_name="Foundation Models Are Current", + finding_details="No legacy/deprecated foundation models detected.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Foundation Model Version Currency Check", e) + return findings + + +# =========================================================================== +# CATEGORY 8: ABUSIVE OR HARMFUL OUTPUT (FS-35 to FS-38) +# CATEGORY 9: BIASED OUTPUT (FS-39 to FS-42) +# COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2, NYDFS 500] +# =========================================================================== + + +def check_fmeval_harmful_content() -> Dict[str, Any]: + """ + FS-35 — Check for FMEval or Bedrock Evaluation jobs testing for harmful + content (toxicity, hate speech, violence). + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2] + """ + findings = _empty_findings("FMEval Harmful Content Check") + try: + bedrock = boto3.client("bedrock", config=boto3_config) + evals = bedrock.list_evaluation_jobs().get("jobSummaries", []) + + if not evals: + findings["csv_data"].append( + create_finding( + check_id="FS-35", + finding_name="No Evaluation Jobs — Harmful Content Testing Not Verified", + finding_details="No Bedrock Model Evaluation jobs found.", + resolution=( + "Run Bedrock Model Evaluation or FMEval with harmful content datasets:\n" + "- Toxicity detection\n" + "- Hate speech classification\n" + "- Violence/self-harm content" + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", + severity="Medium", + status="N/A", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-35", + finding_name="Evaluation Jobs Present", + finding_details=f"Found {len(evals)} evaluation job(s). Verify harmful content datasets are included.", + resolution="Ensure evaluation includes toxicity and harmful content test cases.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("FMEval Harmful Content Check", e) + return findings + + +def check_guardrail_content_filters() -> Dict[str, Any]: + """ + FS-36 — Verify Bedrock Guardrails have content filters configured for + hate speech, violence, and sexual content at appropriate thresholds. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2] + """ + findings = _empty_findings("Guardrail Content Filters Check") + try: + bedrock = boto3.client("bedrock", config=boto3_config) + guardrails = bedrock.list_guardrails().get("guardrails", []) + + if not guardrails: + findings["csv_data"].append( + create_finding( + check_id="FS-36", + finding_name="No Guardrails — Content Filters Not Applicable", + finding_details="No Bedrock Guardrails configured.", + resolution="Configure guardrails with content filters.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-filters.html", + severity="High", + status="N/A", + ) + ) + return findings + + guardrails_with_filters = [] + for g in guardrails: + detail = bedrock.get_guardrail( + guardrailIdentifier=g["id"], guardrailVersion="DRAFT" + ) + if detail.get("contentPolicy", {}).get("filters"): + guardrails_with_filters.append(g["name"]) + + if not guardrails_with_filters: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-36", + finding_name="No Guardrails With Content Filters", + finding_details=( + f"Found {len(guardrails)} guardrail(s) but none have content filters. " + "Harmful content (hate, violence, sexual) may pass through unfiltered." + ), + resolution=( + "Add content filters to guardrails for: HATE, INSULTS, SEXUAL, VIOLENCE. " + "Set filter strength to HIGH for financial services use cases." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-filters.html", + severity="High", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-36", + finding_name="Guardrail Content Filters Configured", + finding_details=f"Guardrails with content filters: {', '.join(guardrails_with_filters)}.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-filters.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Guardrail Content Filters Check", e) + return findings + + +def check_user_feedback_mechanism() -> Dict[str, Any]: + """ + FS-37 — Advisory check: verify application has a user feedback/reporting + mechanism for harmful GenAI outputs. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2] + """ + findings = _empty_findings("User Feedback Mechanism Check") + findings["csv_data"].append( + create_finding( + check_id="FS-37", + finding_name="User Feedback Mechanism — Manual Review Required", + finding_details=( + "User feedback mechanisms for harmful outputs cannot be verified via AWS APIs. " + "Manual review required." + ), + resolution=( + "1. Implement thumbs-up/down or flag-for-review UI in GenAI applications.\n" + "2. Route flagged outputs to human reviewers via SQS/SNS.\n" + "3. Log feedback to DynamoDB/S3 for model improvement.\n" + "4. Define SLAs for reviewing flagged content." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails.html", + severity="Medium", + status="Passed", + ) + ) + return findings + + +def check_guardrail_word_filters() -> Dict[str, Any]: + """ + FS-38 — Verify Bedrock Guardrails have word/phrase filters (allowlists/denylists) + configured for financial services context. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT] + """ + findings = _empty_findings("Guardrail Word Filters Check") + try: + bedrock = boto3.client("bedrock", config=boto3_config) + guardrails = bedrock.list_guardrails().get("guardrails", []) + + if not guardrails: + findings["csv_data"].append( + create_finding( + check_id="FS-38", + finding_name="No Guardrails — Word Filters Not Applicable", + finding_details="No Bedrock Guardrails configured.", + resolution="Configure guardrails with word filters.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", + severity="Medium", + status="N/A", + ) + ) + return findings + + guardrails_with_words = [] + for g in guardrails: + detail = bedrock.get_guardrail( + guardrailIdentifier=g["id"], guardrailVersion="DRAFT" + ) + if detail.get("wordPolicy", {}).get("words") or detail.get( + "wordPolicy", {} + ).get("managedWordLists"): + guardrails_with_words.append(g["name"]) + + if not guardrails_with_words: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-38", + finding_name="No Guardrails With Word Filters", + finding_details=( + f"Found {len(guardrails)} guardrail(s) but none have word/phrase filters. " + "Profanity and prohibited financial terms may appear in outputs." + ), + resolution=( + "Add word filters to guardrails:\n" + "- Enable AWS managed profanity list\n" + "- Add custom denylist for prohibited financial terms\n" + "- Add allowlist for required regulatory language" + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", + severity="Medium", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-38", + finding_name="Guardrail Word Filters Configured", + finding_details=f"Guardrails with word filters: {', '.join(guardrails_with_words)}.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Guardrail Word Filters Check", e) + return findings + + +def check_sagemaker_clarify_bias() -> Dict[str, Any]: + """ + FS-39 — Verify SageMaker Clarify bias detection jobs are configured for + production models making financial decisions. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, ECOA, Fair Housing Act] + """ + findings = _empty_findings("SageMaker Clarify Bias Check") + try: + sm = boto3.client("sagemaker", config=boto3_config) + schedules = sm.list_monitoring_schedules().get( + "MonitoringScheduleSummaries", [] + ) + + bias_schedules = [ + s for s in schedules if s.get("MonitoringType") == "ModelBias" + ] + + if not bias_schedules: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-39", + finding_name="No SageMaker Clarify Bias Monitoring", + finding_details=( + "No SageMaker Clarify model bias monitoring schedules found. " + "Models making financial decisions (credit, insurance) may exhibit " + "discriminatory bias without detection." + ), + resolution=( + "1. Configure SageMaker Clarify bias detection for all models making " + "credit, insurance, or employment decisions.\n" + "2. Define protected attributes (age, gender, race proxies).\n" + "3. Set bias metric thresholds and alert on violations.\n" + "4. Document bias testing results for regulatory examination." + ), + reference="https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-model-monitor-bias-drift.html", + severity="High", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-39", + finding_name="SageMaker Clarify Bias Monitoring Active", + finding_details=f"Found {len(bias_schedules)} model bias monitoring schedule(s).", + resolution="No action required.", + reference="https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-model-monitor-bias-drift.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("SageMaker Clarify Bias Check", e) + return findings + + +def check_bedrock_evaluation_bias_datasets() -> Dict[str, Any]: + """ + FS-40 — Check whether Bedrock Model Evaluation includes bias-specific + test datasets for GenAI models used in financial decisions. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, ECOA] + """ + findings = _empty_findings("Bedrock Bias Evaluation Datasets Check") + try: + bedrock = boto3.client("bedrock", config=boto3_config) + evals = bedrock.list_evaluation_jobs().get("jobSummaries", []) + + if not evals: + findings["csv_data"].append( + create_finding( + check_id="FS-40", + finding_name="No Evaluation Jobs — Bias Datasets Not Verified", + finding_details="No Bedrock Model Evaluation jobs found.", + resolution=( + "Run Bedrock Model Evaluation with bias test datasets:\n" + "- Demographic parity test cases\n" + "- Equal opportunity scenarios\n" + "- Counterfactual fairness tests" + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", + severity="Medium", + status="N/A", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-40", + finding_name="Evaluation Jobs Present", + finding_details=f"Found {len(evals)} evaluation job(s). Verify bias datasets are included.", + resolution="Ensure evaluation includes demographic fairness test cases.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Bedrock Bias Evaluation Datasets Check", e) + return findings + + +def check_sagemaker_clarify_explainability() -> Dict[str, Any]: + """ + FS-41 — Verify SageMaker Clarify explainability jobs are configured to + provide model decision explanations for adverse action notices. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, ECOA Adverse Action] + """ + findings = _empty_findings("SageMaker Clarify Explainability Check") + try: + sm = boto3.client("sagemaker", config=boto3_config) + schedules = sm.list_monitoring_schedules().get( + "MonitoringScheduleSummaries", [] + ) + + explainability_schedules = [ + s for s in schedules if s.get("MonitoringType") == "ModelExplainability" + ] + + if not explainability_schedules: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-41", + finding_name="No SageMaker Clarify Explainability Monitoring", + finding_details=( + "No SageMaker Clarify explainability monitoring found. " + "Models making adverse financial decisions may not provide " + "required explanations (ECOA adverse action notices)." + ), + resolution=( + "1. Configure SageMaker Clarify explainability for credit/lending models.\n" + "2. Generate SHAP values for feature importance.\n" + "3. Map top features to human-readable adverse action reason codes.\n" + "4. Store explanations for regulatory examination." + ), + reference="https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-model-explainability.html", + severity="High", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-41", + finding_name="SageMaker Clarify Explainability Active", + finding_details=f"Found {len(explainability_schedules)} explainability monitoring schedule(s).", + resolution="No action required.", + reference="https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-model-explainability.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("SageMaker Clarify Explainability Check", e) + return findings + + +def check_ai_service_cards_documentation() -> Dict[str, Any]: + """ + FS-42 — Advisory check: verify AI Service Cards / Model Cards are + documented for all production GenAI models. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.3] + """ + findings = _empty_findings("AI Service Cards Documentation Check") + try: + sm = boto3.client("sagemaker", config=boto3_config) + model_cards = sm.list_model_cards().get("ModelCardSummaryList", []) + + if not model_cards: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-42", + finding_name="No SageMaker Model Cards Found", + finding_details=( + "No SageMaker Model Cards found. " + "Production AI models lack documented intended use, limitations, and bias evaluations." + ), + resolution=( + "1. Create SageMaker Model Cards for all production models.\n" + "2. Document: intended use, out-of-scope uses, training data, bias evaluations.\n" + "3. Include regulatory compliance attestations.\n" + "4. Review and update cards at each model version release." + ), + reference="https://docs.aws.amazon.com/sagemaker/latest/dg/model-cards.html", + severity="Medium", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-42", + finding_name="SageMaker Model Cards Present", + finding_details=f"Found {len(model_cards)} model card(s).", + resolution="Verify cards are current and include bias/fairness evaluations.", + reference="https://docs.aws.amazon.com/sagemaker/latest/dg/model-cards.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("AI Service Cards Documentation Check", e) + return findings + + +# =========================================================================== +# CATEGORY 10: SENSITIVE INFORMATION DISCLOSURE (FS-43 to FS-46) +# COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, FFIEC CAT, PCI-DSS 3.4, GDPR Art.25] +# =========================================================================== + + +def check_cloudwatch_log_pii_masking() -> Dict[str, Any]: + """ + FS-43 — Check for CloudWatch Logs data protection policies that mask PII + in Bedrock invocation logs. + COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, GDPR Art.25, PCI-DSS 3.4] + """ + findings = _empty_findings("CloudWatch Log PII Masking Check") + try: + logs = boto3.client("logs", config=boto3_config) + # List data protection policies + try: + policies = logs.describe_account_policies( + policyType="DATA_PROTECTION_POLICY" + ).get("accountPolicies", []) + except ClientError: + policies = [] + + if not policies: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-43", + finding_name="No CloudWatch Logs Data Protection Policies", + finding_details=( + "No CloudWatch Logs data protection policies found. " + "PII (SSN, account numbers, credit card numbers) in Bedrock invocation logs " + "may be stored in plaintext." + ), + resolution=( + "1. Create CloudWatch Logs data protection policies to mask PII.\n" + "2. Enable masking for: SSN, credit card numbers, bank account numbers, email.\n" + "3. Apply policies to Bedrock invocation log groups.\n" + "4. Test masking with synthetic PII before production deployment." + ), + reference="https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/mask-sensitive-log-data.html", + severity="High", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-43", + finding_name="CloudWatch Logs Data Protection Policies Present", + finding_details=f"Found {len(policies)} data protection policy(ies).", + resolution="Verify policies cover Bedrock invocation log groups.", + reference="https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/mask-sensitive-log-data.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("CloudWatch Log PII Masking Check", e) + return findings + + +def check_macie_on_training_data_buckets() -> Dict[str, Any]: + """ + FS-44 — Verify Amazon Macie is enabled and scanning S3 buckets that + contain training data or KB data sources for PII. + COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, GDPR Art.25, PCI-DSS 3.4, FFIEC CAT] + """ + findings = _empty_findings("Amazon Macie PII Scanning Check") + try: + macie = boto3.client("macie2", config=boto3_config) + try: + status = macie.get_macie_session() + macie_enabled = status.get("status") == "ENABLED" + except ClientError: + macie_enabled = False + + if not macie_enabled: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-44", + finding_name="Amazon Macie Not Enabled", + finding_details=( + "Amazon Macie is not enabled. S3 buckets containing training data " + "and KB data sources are not being scanned for PII/sensitive data." + ), + resolution=( + "1. Enable Amazon Macie in all regions where AI/ML data is stored.\n" + "2. Create Macie classification jobs for training data and KB buckets.\n" + "3. Configure Macie findings to route to Security Hub and SNS.\n" + "4. Remediate PII findings before using data for model training." + ), + reference="https://docs.aws.amazon.com/macie/latest/user/what-is-macie.html", + severity="High", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-44", + finding_name="Amazon Macie Enabled", + finding_details="Amazon Macie is enabled and scanning S3 buckets.", + resolution="Verify Macie jobs cover training data and KB data source buckets.", + reference="https://docs.aws.amazon.com/macie/latest/user/what-is-macie.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Amazon Macie PII Scanning Check", e) + return findings + + +def check_guardrail_pii_filters() -> Dict[str, Any]: + """ + FS-45 — Verify Bedrock Guardrails have sensitive information (PII) filters + configured to block PII in prompts and responses. + COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, GDPR Art.25, PCI-DSS 3.4] + """ + findings = _empty_findings("Guardrail PII Filters Check") + try: + bedrock = boto3.client("bedrock", config=boto3_config) + guardrails = bedrock.list_guardrails().get("guardrails", []) + + if not guardrails: + findings["csv_data"].append( + create_finding( + check_id="FS-45", + finding_name="No Guardrails — PII Filters Not Applicable", + finding_details="No Bedrock Guardrails configured.", + resolution="Configure guardrails with PII/sensitive information filters.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-sensitive-filters.html", + severity="High", + status="N/A", + ) + ) + return findings + + guardrails_with_pii = [] + for g in guardrails: + detail = bedrock.get_guardrail( + guardrailIdentifier=g["id"], guardrailVersion="DRAFT" + ) + if detail.get("sensitiveInformationPolicy", {}).get("piiEntities"): + guardrails_with_pii.append(g["name"]) + + if not guardrails_with_pii: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-45", + finding_name="No Guardrails With PII Filters", + finding_details=( + f"Found {len(guardrails)} guardrail(s) but none have PII entity filters. " + "SSN, credit card numbers, and account numbers may appear in GenAI outputs." + ), + resolution=( + "Add PII entity filters to guardrails for:\n" + "- US_SOCIAL_SECURITY_NUMBER\n" + "- CREDIT_DEBIT_CARD_NUMBER\n" + "- BANK_ACCOUNT_NUMBER\n" + "- EMAIL, PHONE, NAME (as appropriate)\n" + "Set action to ANONYMIZE or BLOCK." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-sensitive-filters.html", + severity="High", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-45", + finding_name="Guardrail PII Filters Configured", + finding_details=f"Guardrails with PII filters: {', '.join(guardrails_with_pii)}.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-sensitive-filters.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Guardrail PII Filters Check", e) + return findings + + +def check_data_classification_tagging() -> Dict[str, Any]: + """ + FS-46 — Check that S3 buckets containing AI/ML data are tagged with + data classification labels (e.g., Confidential, PII, Public). + COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, FFIEC CAT, ISO 27001 A.8.2] + """ + findings = _empty_findings("Data Classification Tagging Check") + try: + s3 = boto3.client("s3", config=boto3_config) + buckets = s3.list_buckets().get("Buckets", []) + + aiml_buckets = [ + b + for b in buckets + if any( + kw in b["Name"].lower() + for kw in ["train", "model", "bedrock", "sagemaker", "kb", "knowledge"] + ) + ] + + if not aiml_buckets: + findings["csv_data"].append( + create_finding( + check_id="FS-46", + finding_name="No AI/ML Data Buckets Identified", + finding_details="No S3 buckets with AI/ML naming found.", + resolution="Tag AI/ML data buckets with data-classification labels.", + reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html", + severity="Informational", + status="N/A", + ) + ) + return findings + + unclassified = [] + for bucket in aiml_buckets: + try: + tags = s3.get_bucket_tagging(Bucket=bucket["Name"]).get("TagSet", []) + tag_keys = {t["Key"].lower() for t in tags} + if ( + "data-classification" not in tag_keys + and "classification" not in tag_keys + ): + unclassified.append(bucket["Name"]) + except ClientError: + unclassified.append(bucket["Name"]) + + if unclassified: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-46", + finding_name="AI/ML Buckets Without Data Classification Tags", + finding_details=( + f"{len(unclassified)} AI/ML bucket(s) without data-classification tags: " + f"{', '.join(unclassified[:10])}." + ), + resolution=( + "Tag all AI/ML data buckets with 'data-classification' key. " + "Values: Public, Internal, Confidential, Restricted. " + "Enforce via SCP or AWS Config rule." + ), + reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html", + severity="Medium", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-46", + finding_name="AI/ML Buckets Have Classification Tags", + finding_details=f"All {len(aiml_buckets)} AI/ML bucket(s) have classification tags.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Data Classification Tagging Check", e) + return findings + + +# =========================================================================== +# CATEGORY 11: HALLUCINATION (FS-47 to FS-50) +# CATEGORY 12: PROMPT INJECTION (FS-51 to FS-54) +# COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2, NYDFS 500] +# =========================================================================== + + +def check_guardrail_grounding_threshold() -> Dict[str, Any]: + """ + FS-47 — Verify Bedrock Guardrails contextual grounding thresholds are + set appropriately high for financial services use cases. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2] + """ + findings = _empty_findings("Guardrail Grounding Threshold Check") + try: + bedrock = boto3.client("bedrock", config=boto3_config) + guardrails = bedrock.list_guardrails().get("guardrails", []) + + if not guardrails: + findings["csv_data"].append( + create_finding( + check_id="FS-47", + finding_name="No Guardrails — Grounding Threshold Not Applicable", + finding_details="No Bedrock Guardrails configured.", + resolution="Configure guardrails with contextual grounding checks.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", + severity="High", + status="N/A", + ) + ) + return findings + + low_threshold_guardrails = [] + for g in guardrails: + detail = bedrock.get_guardrail( + guardrailIdentifier=g["id"], guardrailVersion="DRAFT" + ) + grounding = detail.get("contextualGroundingPolicy", {}) + for filter_item in grounding.get("filters", []): + if ( + filter_item.get("type") == "GROUNDING" + and filter_item.get("threshold", 1.0) < 0.7 + ): + low_threshold_guardrails.append( + f"{g['name']} (threshold={filter_item['threshold']})" + ) + + if low_threshold_guardrails: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-47", + finding_name="Guardrails With Low Grounding Thresholds", + finding_details=( + f"Guardrails with grounding threshold <0.7: {', '.join(low_threshold_guardrails)}. " + "Low thresholds allow hallucinated responses to pass through." + ), + resolution=( + "Set grounding threshold to 0.7 or higher for financial services use cases. " + "Test threshold impact on response quality before increasing." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", + severity="High", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-47", + finding_name="Guardrail Grounding Thresholds Appropriate", + finding_details="All guardrails with grounding have thresholds ≥0.7.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Guardrail Grounding Threshold Check", e) + return findings + + +def check_rag_knowledge_base_configured() -> Dict[str, Any]: + """ + FS-48 — Verify RAG (Retrieval Augmented Generation) is used via Bedrock + Knowledge Bases to ground responses in authoritative data. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT] + """ + findings = _empty_findings("RAG Knowledge Base Configuration Check") + try: + bedrock_agent = boto3.client("bedrock-agent", config=boto3_config) + paginator = bedrock_agent.get_paginator("list_knowledge_bases") + kbs = [] + for page in paginator.paginate(): + kbs.extend(page.get("knowledgeBaseSummaries", [])) + + active_kbs = [k for k in kbs if k.get("status") == "ACTIVE"] + + if not active_kbs: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-48", + finding_name="No Active Knowledge Bases for RAG", + finding_details=( + "No active Bedrock Knowledge Bases found. " + "GenAI responses are not grounded in authoritative data sources, " + "increasing hallucination risk." + ), + resolution=( + "1. Create Bedrock Knowledge Bases with authoritative financial data.\n" + "2. Use RetrieveAndGenerate API to ground responses.\n" + "3. Configure data sources with current regulatory and product information." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base.html", + severity="Medium", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-48", + finding_name="Active Knowledge Bases for RAG Present", + finding_details=f"Found {len(active_kbs)} active Knowledge Base(s) for RAG grounding.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("RAG Knowledge Base Configuration Check", e) + return findings + + +def check_hallucination_disclaimer_advisory() -> Dict[str, Any]: + """ + FS-49 — Advisory check: verify application adds hallucination disclaimers + to GenAI outputs in financial contexts. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2] + """ + findings = _empty_findings("Hallucination Disclaimer Advisory") + findings["csv_data"].append( + create_finding( + check_id="FS-49", + finding_name="Hallucination Disclaimer — Manual Review Required", + finding_details=( + "Application-level hallucination disclaimers cannot be verified via AWS APIs. " + "Manual review required." + ), + resolution=( + "1. Add disclaimers to GenAI outputs: 'AI-generated content may contain errors. " + "Verify with authoritative sources before acting.'\n" + "2. Implement post-processing to append disclaimers.\n" + "3. Test disclaimer presence in QA before production." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails.html", + severity="Medium", + status="Passed", + ) + ) + return findings + + +def check_automated_reasoning_checks_hallucination() -> Dict[str, Any]: + """ + FS-50 — Check for Bedrock Automated Reasoning checks (ARC) configured + to validate factual claims in GenAI outputs. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT] + """ + findings = _empty_findings("Automated Reasoning Checks for Hallucination") + try: + bedrock = boto3.client("bedrock", config=boto3_config) + # ARC is part of guardrails contextual grounding — check for RELEVANCE filter + guardrails = bedrock.list_guardrails().get("guardrails", []) + + arc_guardrails = [] + for g in guardrails: + detail = bedrock.get_guardrail( + guardrailIdentifier=g["id"], guardrailVersion="DRAFT" + ) + grounding = detail.get("contextualGroundingPolicy", {}) + for f in grounding.get("filters", []): + if f.get("type") == "RELEVANCE": + arc_guardrails.append(g["name"]) + + if not arc_guardrails: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-50", + finding_name="No Guardrails With Relevance Grounding", + finding_details=( + "No guardrails have relevance grounding filters. " + "Off-topic or hallucinated responses will not be filtered." + ), + resolution=( + "Enable relevance grounding filter in Bedrock Guardrails " + "with threshold ≥0.7 to filter responses not grounded in context." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", + severity="Medium", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-50", + finding_name="Relevance Grounding Filters Present", + finding_details=f"Guardrails with relevance grounding: {', '.join(arc_guardrails)}.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Automated Reasoning Checks for Hallucination", e) + return findings + + +def check_prompt_injection_input_validation() -> Dict[str, Any]: + """ + FS-51 — Check for Bedrock Guardrails prompt attack filters to detect + and block prompt injection attempts. + COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, FFIEC CAT, OWASP LLM01] + """ + findings = _empty_findings("Prompt Injection Input Validation Check") + try: + bedrock = boto3.client("bedrock", config=boto3_config) + guardrails = bedrock.list_guardrails().get("guardrails", []) + + if not guardrails: + findings["csv_data"].append( + create_finding( + check_id="FS-51", + finding_name="No Guardrails — Prompt Attack Filters Not Applicable", + finding_details="No Bedrock Guardrails configured.", + resolution="Configure guardrails with prompt attack filters.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", + severity="High", + status="N/A", + ) + ) + return findings + + guardrails_with_prompt_attack = [] + for g in guardrails: + detail = bedrock.get_guardrail( + guardrailIdentifier=g["id"], guardrailVersion="DRAFT" + ) + content_policy = detail.get("contentPolicy", {}) + for f in content_policy.get("filters", []): + if f.get("type") == "PROMPT_ATTACK": + guardrails_with_prompt_attack.append(g["name"]) + + if not guardrails_with_prompt_attack: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-51", + finding_name="No Guardrails With Prompt Attack Filters", + finding_details=( + f"Found {len(guardrails)} guardrail(s) but none have PROMPT_ATTACK filters. " + "Prompt injection attacks may bypass system prompts and access controls." + ), + resolution=( + "1. Enable PROMPT_ATTACK content filter in Bedrock Guardrails.\n" + "2. Set input filter strength to HIGH.\n" + "3. Implement application-level input sanitization as defense-in-depth.\n" + "4. Use parameterized prompts (never concatenate user input directly)." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", + severity="High", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-51", + finding_name="Prompt Attack Filters Configured", + finding_details=f"Guardrails with prompt attack filters: {', '.join(guardrails_with_prompt_attack)}.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Prompt Injection Input Validation Check", e) + return findings + + +def check_bedrock_sdk_version_currency() -> Dict[str, Any]: + """ + FS-52 — Advisory check: verify Bedrock SDK versions in Lambda functions + are current (outdated SDKs may lack prompt injection mitigations). + COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, FFIEC CAT, ISO 27001 A.12.6] + """ + findings = _empty_findings("Bedrock SDK Version Currency Check") + try: + lambda_client = boto3.client("lambda", config=boto3_config) + functions = lambda_client.list_functions().get("Functions", []) + + bedrock_functions = [ + f + for f in functions + if any( + kw in f["FunctionName"].lower() + for kw in ["bedrock", "agent", "aiml", "genai"] + ) + ] + + if not bedrock_functions: + findings["csv_data"].append( + create_finding( + check_id="FS-52", + finding_name="No Bedrock-Related Lambda Functions Found", + finding_details="No Lambda functions with Bedrock-related naming found.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/lambda/latest/dg/runtimes-update.html", + severity="Informational", + status="N/A", + ) + ) + return findings + + # Check for deprecated runtimes + deprecated_runtimes = {"python3.7", "python3.8", "nodejs14.x", "nodejs12.x"} + outdated_functions = [ + f["FunctionName"] + for f in bedrock_functions + if f.get("Runtime", "") in deprecated_runtimes + ] + + if outdated_functions: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-52", + finding_name="Bedrock Lambda Functions on Deprecated Runtimes", + finding_details=( + f"Functions on deprecated runtimes: {', '.join(outdated_functions[:10])}. " + "Deprecated runtimes may use outdated boto3/SDK versions lacking security patches." + ), + resolution=( + "1. Upgrade Lambda functions to Python 3.12+ or Node.js 20.x.\n" + "2. Update boto3 to latest version in Lambda layers.\n" + "3. Enable Lambda runtime management for automatic updates." + ), + reference="https://docs.aws.amazon.com/lambda/latest/dg/runtimes-update.html", + severity="Medium", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-52", + finding_name="Bedrock Lambda Functions on Current Runtimes", + finding_details=f"All {len(bedrock_functions)} Bedrock Lambda function(s) use current runtimes.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/lambda/latest/dg/runtimes-update.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Bedrock SDK Version Currency Check", e) + return findings + + +def check_waf_sql_injection_rules() -> Dict[str, Any]: + """ + FS-53 — Verify WAF Web ACLs include SQL injection and XSS managed rules + to protect GenAI API endpoints from injection attacks. + COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, FFIEC CAT, PCI-DSS 6.4.1, OWASP LLM01] + """ + findings = _empty_findings("WAF Injection Protection Rules Check") + try: + wafv2 = boto3.client("wafv2", config=boto3_config) + acls = wafv2.list_web_acls(Scope="REGIONAL").get("WebACLs", []) + + if not acls: + findings["csv_data"].append( + create_finding( + check_id="FS-53", + finding_name="No WAF Web ACLs — Injection Rules Not Applicable", + finding_details="No regional WAF Web ACLs found.", + resolution="Create WAF Web ACLs with injection protection rules (see FS-01).", + reference="https://docs.aws.amazon.com/waf/latest/developerguide/aws-managed-rule-groups-baseline.html", + severity="High", + status="N/A", + ) + ) + return findings + + INJECTION_RULE_GROUPS = { + "AWSManagedRulesSQLiRuleSet", + "AWSManagedRulesCommonRuleSet", + "AWSManagedRulesKnownBadInputsRuleSet", + } + + acls_without_injection_rules = [] + for acl_summary in acls: + acl = wafv2.get_web_acl( + Name=acl_summary["Name"], + Scope="REGIONAL", + Id=acl_summary["Id"], + ).get("WebACL", {}) + rule_names = { + r.get("Statement", {}) + .get("ManagedRuleGroupStatement", {}) + .get("Name", "") + for r in acl.get("Rules", []) + } + if not rule_names.intersection(INJECTION_RULE_GROUPS): + acls_without_injection_rules.append(acl_summary["Name"]) + + if acls_without_injection_rules: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-53", + finding_name="WAF ACLs Missing Injection Protection Rules", + finding_details=( + f"WAF ACLs without SQL injection/XSS rules: " + f"{', '.join(acls_without_injection_rules[:10])}." + ), + resolution=( + "Add AWS Managed Rule Groups to WAF ACLs:\n" + "- AWSManagedRulesSQLiRuleSet (SQL injection)\n" + "- AWSManagedRulesCommonRuleSet (XSS, LFI, RFI)\n" + "- AWSManagedRulesKnownBadInputsRuleSet (prompt injection patterns)" + ), + reference="https://docs.aws.amazon.com/waf/latest/developerguide/aws-managed-rule-groups-baseline.html", + severity="High", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-53", + finding_name="WAF Injection Protection Rules Present", + finding_details=f"All {len(acls)} WAF ACL(s) have injection protection rules.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/waf/latest/developerguide/aws-managed-rule-groups-baseline.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("WAF Injection Protection Rules Check", e) + return findings + + +def check_penetration_testing_evidence() -> Dict[str, Any]: + """ + FS-54 — Advisory check: verify penetration testing has been conducted + on GenAI applications (prompt injection, jailbreak testing). + COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, FFIEC CAT, PCI-DSS 11.4, DORA Art.26] + """ + findings = _empty_findings("Penetration Testing Evidence Check") + findings["csv_data"].append( + create_finding( + check_id="FS-54", + finding_name="Penetration Testing — Manual Review Required", + finding_details=( + "Penetration testing evidence cannot be verified via AWS APIs. " + "Manual review required to confirm GenAI applications have been tested." + ), + resolution=( + "1. Conduct annual penetration testing of GenAI applications.\n" + "2. Include prompt injection, jailbreak, and indirect injection test cases.\n" + "3. Use AWS Bedrock red-teaming capabilities.\n" + "4. Document findings and remediation for regulatory examination.\n" + "5. For DORA compliance, include GenAI in TLPT (Threat-Led Penetration Testing) scope." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/security.html", + severity="Medium", + status="Passed", + ) + ) + return findings + + +# =========================================================================== +# CATEGORY 13: IMPROPER OUTPUT HANDLING (FS-55 to FS-58) +# CATEGORY 14: OFF-TOPIC & INAPPROPRIATE OUTPUT (FS-59 to FS-60) +# CATEGORY 15: OUT-OF-DATE TRAINING DATA (FS-61 to FS-63) +# COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, NYDFS 500, OWASP LLM02] +# =========================================================================== + + +def check_output_validation_lambda() -> Dict[str, Any]: + """ + FS-55 — Check for Lambda functions implementing output validation/sanitization + in GenAI application pipelines. + COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, FFIEC CAT, OWASP LLM02] + """ + findings = _empty_findings("Output Validation Lambda Check") + try: + lambda_client = boto3.client("lambda", config=boto3_config) + functions = lambda_client.list_functions().get("Functions", []) + + validation_functions = [ + f + for f in functions + if any( + kw in f["FunctionName"].lower() + for kw in ["validate", "sanitize", "filter", "output"] + ) + ] + + if not validation_functions: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-55", + finding_name="No Output Validation Functions Found", + finding_details=( + "No Lambda functions with output validation/sanitization naming found. " + "GenAI outputs may be passed directly to downstream systems without validation." + ), + resolution=( + "1. Implement output validation Lambda functions in GenAI pipelines.\n" + "2. Validate output schema, length, and content before downstream use.\n" + "3. Sanitize outputs before rendering in web UIs (XSS prevention).\n" + "4. Encode outputs appropriately for the target context (HTML, SQL, JSON)." + ), + reference="https://owasp.org/www-project-top-10-for-large-language-model-applications/", + severity="Medium", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-55", + finding_name="Output Validation Functions Present", + finding_details=f"Found {len(validation_functions)} output validation/sanitization function(s).", + resolution="No action required.", + reference="https://owasp.org/www-project-top-10-for-large-language-model-applications/", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Output Validation Lambda Check", e) + return findings + + +def check_xss_prevention_waf() -> Dict[str, Any]: + """ + FS-56 — Verify WAF rules include XSS prevention for GenAI web application outputs. + COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, FFIEC CAT, PCI-DSS 6.4.1, OWASP LLM02] + """ + findings = _empty_findings("XSS Prevention WAF Check") + try: + wafv2 = boto3.client("wafv2", config=boto3_config) + acls = wafv2.list_web_acls(Scope="REGIONAL").get("WebACLs", []) + + if not acls: + findings["csv_data"].append( + create_finding( + check_id="FS-56", + finding_name="No WAF ACLs — XSS Prevention Not Applicable", + finding_details="No regional WAF Web ACLs found.", + resolution="Create WAF ACLs with XSS prevention rules.", + reference="https://docs.aws.amazon.com/waf/latest/developerguide/aws-managed-rule-groups-baseline.html", + severity="High", + status="N/A", + ) + ) + return findings + + # XSS is covered by AWSManagedRulesCommonRuleSet — reuse FS-53 logic + findings["csv_data"].append( + create_finding( + check_id="FS-56", + finding_name="XSS Prevention — Review WAF Common Rule Set", + finding_details=( + f"Found {len(acls)} WAF ACL(s). " + "Verify AWSManagedRulesCommonRuleSet is enabled for XSS prevention (see FS-53)." + ), + resolution=( + "Ensure AWSManagedRulesCommonRuleSet is enabled on all WAF ACLs " + "protecting GenAI web applications. " + "Additionally, implement Content Security Policy (CSP) headers." + ), + reference="https://docs.aws.amazon.com/waf/latest/developerguide/aws-managed-rule-groups-baseline.html", + severity="Medium", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("XSS Prevention WAF Check", e) + return findings + + +def check_output_encoding_advisory() -> Dict[str, Any]: + """ + FS-57 — Advisory check: verify application encodes GenAI outputs + appropriately for the rendering context. + COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, FFIEC CAT, OWASP LLM02] + """ + findings = _empty_findings("Output Encoding Advisory") + findings["csv_data"].append( + create_finding( + check_id="FS-57", + finding_name="Output Encoding — Manual Review Required", + finding_details=( + "Output encoding practices cannot be verified via AWS APIs. " + "Manual code review required." + ), + resolution=( + "1. HTML-encode GenAI outputs before rendering in web UIs.\n" + "2. Use parameterized queries when GenAI output is used in database operations.\n" + "3. JSON-encode outputs before embedding in JavaScript contexts.\n" + "4. Validate output length and format before passing to downstream APIs." + ), + reference="https://owasp.org/www-project-top-10-for-large-language-model-applications/", + severity="Medium", + status="Passed", + ) + ) + return findings + + +def check_output_schema_validation() -> Dict[str, Any]: + """ + FS-58 — Check for structured output validation using Bedrock response + schemas or application-level JSON schema validation. + COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, FFIEC CAT, OWASP LLM02] + """ + findings = _empty_findings("Output Schema Validation Check") + try: + # Check for EventBridge Pipes or Lambda destinations that could validate outputs + lambda_client = boto3.client("lambda", config=boto3_config) + functions = lambda_client.list_functions().get("Functions", []) + + schema_functions = [ + f + for f in functions + if any( + kw in f["FunctionName"].lower() + for kw in ["schema", "validate", "parse", "format"] + ) + ] + + findings["csv_data"].append( + create_finding( + check_id="FS-58", + finding_name="Output Schema Validation — Review Required", + finding_details=( + f"Found {len(schema_functions)} potential schema validation function(s). " + "Verify structured output validation is implemented for all GenAI responses." + ), + resolution=( + "1. Use Bedrock structured output (response schemas) where supported.\n" + "2. Implement JSON schema validation on Lambda output processors.\n" + "3. Reject malformed outputs and return safe error responses.\n" + "4. Log schema validation failures to CloudWatch for monitoring." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/inference-parameters.html", + severity="Medium", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Output Schema Validation Check", e) + return findings + + +def check_guardrail_topic_allowlist() -> Dict[str, Any]: + """ + FS-59 — Verify Bedrock Guardrails topic policies restrict GenAI to + on-topic financial services responses only. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2] + """ + findings = _empty_findings("Guardrail Topic Allowlist Check") + try: + bedrock = boto3.client("bedrock", config=boto3_config) + guardrails = bedrock.list_guardrails().get("guardrails", []) + + if not guardrails: + findings["csv_data"].append( + create_finding( + check_id="FS-59", + finding_name="No Guardrails — Topic Allowlist Not Applicable", + finding_details="No Bedrock Guardrails configured.", + resolution="Configure guardrails with topic policies to restrict off-topic responses.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", + severity="Medium", + status="N/A", + ) + ) + return findings + + guardrails_with_topics = [] + for g in guardrails: + detail = bedrock.get_guardrail( + guardrailIdentifier=g["id"], guardrailVersion="DRAFT" + ) + if detail.get("topicPolicy", {}).get("topics"): + guardrails_with_topics.append(g["name"]) + + if not guardrails_with_topics: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-59", + finding_name="No Guardrails With Topic Restrictions", + finding_details=( + f"Found {len(guardrails)} guardrail(s) but none have topic policies. " + "GenAI may respond to off-topic requests (e.g., medical advice, legal advice)." + ), + resolution=( + "Add denied topics to guardrails for off-topic categories:\n" + "- Medical/health advice\n" + "- Legal advice\n" + "- Political opinions\n" + "- Non-financial product recommendations" + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", + severity="Medium", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-59", + finding_name="Guardrail Topic Restrictions Configured", + finding_details=f"Guardrails with topic policies: {', '.join(guardrails_with_topics)}.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Guardrail Topic Allowlist Check", e) + return findings + + +def check_contextual_grounding_for_offtopic() -> Dict[str, Any]: + """ + FS-60 — Verify contextual grounding is used to keep GenAI responses + within the scope of the provided context/system prompt. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT] + """ + # This overlaps with FS-47/FS-48 but focuses on off-topic prevention + findings = _empty_findings("Contextual Grounding for Off-Topic Prevention") + findings["csv_data"].append( + create_finding( + check_id="FS-60", + finding_name="Contextual Grounding for Off-Topic Prevention", + finding_details=( + "Contextual grounding for off-topic prevention is covered by guardrail " + "grounding checks (FS-47) and RAG configuration (FS-48). " + "Additionally verify system prompts explicitly scope the assistant's role." + ), + resolution=( + "1. Include explicit scope instructions in system prompts.\n" + "2. Use Bedrock Guardrails relevance grounding filter.\n" + "3. Test with off-topic prompts in QA to verify rejection behavior." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", + severity="Low", + status="Passed", + ) + ) + return findings + + +def check_knowledge_base_sync_schedule() -> Dict[str, Any]: + """ + FS-61 — Verify Bedrock Knowledge Base data sources have automated sync + schedules to keep training/retrieval data current. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT] + """ + # Reuses logic from FS-31 but focuses on scheduled automation + findings = _empty_findings("Knowledge Base Sync Schedule Check") + try: + bedrock_agent = boto3.client("bedrock-agent", config=boto3_config) + events = boto3.client("events", config=boto3_config) + + paginator = bedrock_agent.get_paginator("list_knowledge_bases") + kbs = [] + for page in paginator.paginate(): + kbs.extend(page.get("knowledgeBaseSummaries", [])) + + if not kbs: + findings["csv_data"].append( + create_finding( + check_id="FS-61", + finding_name="No Knowledge Bases Found", + finding_details="No Bedrock Knowledge Bases found.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base-ingest.html", + severity="Informational", + status="N/A", + ) + ) + return findings + + # Check for EventBridge rules that trigger KB sync + rules = events.list_rules().get("Rules", []) + kb_sync_rules = [ + r + for r in rules + if "bedrock" in r.get("Name", "").lower() + or "knowledge" in r.get("Name", "").lower() + ] + + if not kb_sync_rules: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-61", + finding_name="No Automated KB Sync Schedules Found", + finding_details=( + f"Found {len(kbs)} Knowledge Base(s) but no EventBridge rules for automated sync. " + "KB data may become stale without manual intervention." + ), + resolution=( + "1. Create EventBridge scheduled rules to trigger KB data source sync.\n" + "2. Set sync frequency based on data currency requirements.\n" + "3. Configure SNS alerts on sync failures." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base-ingest.html", + severity="Medium", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-61", + finding_name="Automated KB Sync Schedules Present", + finding_details=f"Found {len(kb_sync_rules)} EventBridge rule(s) for KB sync.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base-ingest.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Knowledge Base Sync Schedule Check", e) + return findings + + +def check_data_currency_disclaimer_advisory() -> Dict[str, Any]: + """ + FS-62 — Advisory check: verify application adds data currency disclaimers + to GenAI outputs (e.g., 'Information current as of [date]'). + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2] + """ + findings = _empty_findings("Data Currency Disclaimer Advisory") + findings["csv_data"].append( + create_finding( + check_id="FS-62", + finding_name="Data Currency Disclaimer — Manual Review Required", + finding_details=( + "Data currency disclaimers cannot be verified via AWS APIs. " + "Manual review required." + ), + resolution=( + "1. Add data currency disclaimers to GenAI outputs: " + "'Information based on data current as of [KB last sync date].'\n" + "2. Expose KB last sync timestamp in application responses.\n" + "3. Alert users when KB data is older than defined threshold." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base-ingest.html", + severity="Low", + status="Passed", + ) + ) + return findings + + +def check_foundation_model_lifecycle_policy() -> Dict[str, Any]: + """ + FS-63 — Check for a documented process to update foundation models when + new versions with more recent training data are released. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, ISO 27001 A.12.5] + """ + findings = _empty_findings("Foundation Model Lifecycle Policy Check") + try: + bedrock = boto3.client("bedrock", config=boto3_config) + models = bedrock.list_foundation_models(byOutputModality="TEXT").get( + "modelSummaries", [] + ) + + legacy_models = [ + m["modelId"] + for m in models + if m.get("modelLifecycle", {}).get("status") == "LEGACY" + ] + + # Check for Config rules or SSM documents related to model lifecycle + config_client = boto3.client("config", config=boto3_config) + rules = config_client.describe_config_rules().get("ConfigRules", []) + lifecycle_rules = [ + r + for r in rules + if "lifecycle" in r.get("ConfigRuleName", "").lower() + or "model" in r.get("ConfigRuleName", "").lower() + ] + + if legacy_models and not lifecycle_rules: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-63", + finding_name="Legacy Models Without Lifecycle Management", + finding_details=( + f"Legacy foundation models available: {', '.join(legacy_models[:5])}. " + "No Config rules found for model lifecycle management." + ), + resolution=( + "1. Create a model lifecycle management process.\n" + "2. Subscribe to AWS Bedrock model deprecation notifications.\n" + "3. Test and migrate to new model versions before deprecation dates.\n" + "4. Document training data cutoff dates in model inventory." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle.html", + severity="Medium", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-63", + finding_name="Foundation Model Lifecycle Management", + finding_details=( + f"No legacy models detected. " + f"{len(lifecycle_rules)} lifecycle-related Config rule(s) found." + ), + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Foundation Model Lifecycle Policy Check", e) + return findings + + +# =========================================================================== +# MATERIAL GAP CHECKS (FS-65 to FS-69) +# Mitigations explicitly in the AWS FinServ Guide not covered by FS-01..63 +# or the existing BR/SM/AC checks. +# NOTE: FS-64 (Guardrail Trace Logging) is merged into upstream BR-04. +# See extension note in SECURITY_CHECKS_FINSERV_PART3_APP_LAYER_AND_GAPS.md. +# =========================================================================== + + +def check_kb_datasource_s3_event_notifications() -> Dict[str, Any]: + """ + FS-65 — Check that S3 event notifications (EventBridge or SNS/SQS) are + configured on Knowledge Base data-source buckets to detect unauthorized + document changes in real time. + COMPLIANCE_PLACEHOLDER: [SR 11-7, ISO 27001 A.12, FFIEC CAT] + """ + findings = _empty_findings("KB Data Source S3 Event Notifications Check") + try: + bedrock_agent = boto3.client("bedrock-agent", config=boto3_config) + s3_client = boto3.client("s3", config=boto3_config) + + kbs = bedrock_agent.list_knowledge_bases().get("knowledgeBaseSummaries", []) + if not kbs: + findings["csv_data"].append( + create_finding( + check_id="FS-65", + finding_name="No Knowledge Bases Found", + finding_details="No Bedrock Knowledge Bases found; S3 event notification check not applicable.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/NotificationHowTo.html", + severity="Informational", + status="N/A", + ) + ) + return findings + + buckets_without_notifications = [] + for kb in kbs: + kb_id = kb["knowledgeBaseId"] + data_sources = bedrock_agent.list_data_sources(knowledgeBaseId=kb_id).get( + "dataSourceSummaries", [] + ) + for ds in data_sources: + ds_detail = bedrock_agent.get_data_source( + knowledgeBaseId=kb_id, + dataSourceId=ds["dataSourceId"], + ) + s3_config = ( + ds_detail.get("dataSource", {}) + .get("dataSourceConfiguration", {}) + .get("s3Configuration", {}) + ) + bucket = s3_config.get("bucketArn", "").split(":::")[-1] + if not bucket: + continue + try: + notif = s3_client.get_bucket_notification_configuration( + Bucket=bucket + ) + has_notif = any( + [ + notif.get("TopicConfigurations"), + notif.get("QueueConfigurations"), + notif.get("LambdaFunctionConfigurations"), + notif.get("EventBridgeConfiguration"), + ] + ) + if not has_notif: + buckets_without_notifications.append(bucket) + except ClientError: + buckets_without_notifications.append(f"{bucket} (access error)") + + if buckets_without_notifications: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-65", + finding_name="KB Data Source Buckets Missing S3 Event Notifications", + finding_details=( + "The following KB data-source S3 buckets have no event notifications configured. " + "Unauthorized document modifications will not be detected in real time:\n" + + "\n".join( + f"- {b}" for b in buckets_without_notifications[:10] + ) + ), + resolution=( + "1. Enable Amazon EventBridge notifications on each KB data-source S3 bucket.\n" + "2. Create an EventBridge rule to route s3:ObjectCreated, s3:ObjectRemoved, " + "and s3:ObjectModified events to an SNS topic or Lambda for alerting.\n" + "3. Integrate alerts into your security incident response workflow." + ), + reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/EventBridge.html", + severity="Medium", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-65", + finding_name="KB Data Source S3 Event Notifications Configured", + finding_details="All KB data-source S3 buckets have event notifications configured.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/EventBridge.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("KB Data Source S3 Event Notifications Check", e) + return findings + + +def check_agentcore_end_user_identity_propagation() -> Dict[str, Any]: + """ + FS-66 — Verify AgentCore runtimes are configured to propagate end-user + identities to downstream tool services so tool calls are authorized by + the originating user, not solely by the agent execution role. + COMPLIANCE_PLACEHOLDER: [SR 11-7, NYDFS 500.06, MAS TRM 9.1] + """ + findings = _empty_findings("AgentCore End-User Identity Propagation Check") + try: + agentcore = boto3.client("bedrock-agentcore-control", config=boto3_config) + try: + runtimes = agentcore.list_agent_runtimes().get("agentRuntimes", []) + except ClientError as e: + if "AccessDenied" in str(e) or "UnrecognizedClientException" in str(e): + findings["csv_data"].append( + create_finding( + check_id="FS-66", + finding_name="AgentCore Identity Propagation — Access Check", + finding_details="Unable to enumerate AgentCore runtimes (access denied or service unavailable in region).", + resolution="Ensure assessment role has bedrock-agentcore:ListAgentRuntimes permission.", + reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/security-authorization.html", + severity="Low", + status="N/A", + ) + ) + return findings + raise + + if not runtimes: + findings["csv_data"].append( + create_finding( + check_id="FS-66", + finding_name="No AgentCore Runtimes Found", + finding_details="No AgentCore runtimes found; identity propagation check not applicable.", + resolution=( + "If using AgentCore, configure token propagation so end-user identities " + "are forwarded to tool services." + ), + reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/security-authorization.html", + severity="Informational", + status="N/A", + ) + ) + return findings + + runtimes_without_identity = [ + r["agentRuntimeName"] + for r in runtimes + if not r.get("authorizerConfiguration", {}).get("customJWTAuthorizer") + and not r.get("authorizerConfiguration", {}).get("iamAuthorizer") + ] + + if runtimes_without_identity: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-66", + finding_name="AgentCore Runtimes Missing End-User Identity Propagation", + finding_details=( + "The following runtimes have no JWT or IAM authorizer configured for " + "end-user identity propagation. Tool calls are authorized only by the " + "agent execution role, not the originating user:\n" + + "\n".join(f"- {r}" for r in runtimes_without_identity[:10]) + ), + resolution=( + "1. Configure a custom JWT authorizer or IAM authorizer on each AgentCore runtime.\n" + "2. Propagate the end-user's identity token to downstream tool services.\n" + "3. Ensure tool services validate the propagated identity before executing actions.\n" + "4. Do not expose propagated identity tokens to unauthorized third parties." + ), + reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/security-authorization.html", + severity="High", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-66", + finding_name="AgentCore End-User Identity Propagation Configured", + finding_details=f"All {len(runtimes)} runtime(s) have authorizer configurations supporting identity propagation.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/security-authorization.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("AgentCore End-User Identity Propagation Check", e) + return findings + + +def check_agent_financial_transaction_thresholds() -> Dict[str, Any]: + """ + FS-67 — Check AgentCore Policy Engine or action-group Lambda functions + enforce maximum transaction-value limits to prevent runaway or unauthorized + high-value financial transactions initiated by agents. + COMPLIANCE_PLACEHOLDER: [SR 11-7, MAS TRM 9.1, FFIEC CAT, PCI-DSS] + """ + findings = _empty_findings("Agent Financial Transaction Value Thresholds Check") + try: + lambda_client = boto3.client("lambda", config=boto3_config) + functions = lambda_client.list_functions().get("Functions", []) + + # Look for agent action-group Lambda functions + action_group_lambdas = [ + f + for f in functions + if any( + kw in f["FunctionName"].lower() + for kw in [ + "agent", + "action", + "tool", + "bedrock", + "finserv", + "transaction", + ] + ) + ] + + if not action_group_lambdas: + findings["csv_data"].append( + create_finding( + check_id="FS-67", + finding_name="No Agent Action-Group Lambda Functions Found", + finding_details=( + "No Lambda functions matching agent action-group naming patterns found. " + "If agents perform financial transactions, verify transaction-value limits " + "are enforced in the action-group implementation." + ), + resolution=( + "1. Implement transaction-value threshold checks in all agent action-group " + "Lambda functions that initiate financial operations.\n" + "2. Use AgentCore Policy Engine to enforce maximum transaction amounts as " + "a policy constraint on tool calls.\n" + "3. Reject or escalate to human review any transaction exceeding defined limits." + ), + reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/policy-engine.html", + severity="High", + status="N/A", + ) + ) + else: + # Advisory: check for environment variables indicating threshold configuration + lambdas_without_threshold_config = [ + f["FunctionName"] + for f in action_group_lambdas + if not any( + "threshold" in k.lower() + or "limit" in k.lower() + or "max" in k.lower() + for k in f.get("Environment", {}).get("Variables", {}).keys() + ) + ] + + if lambdas_without_threshold_config: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-67", + finding_name="Agent Action-Group Lambdas May Lack Transaction Thresholds", + finding_details=( + "The following agent action-group Lambda functions have no environment " + "variables indicating transaction-value threshold configuration. " + "Without explicit limits, agents could initiate unbounded financial transactions:\n" + + "\n".join( + f"- {n}" for n in lambdas_without_threshold_config[:10] + ) + ), + resolution=( + "1. Add transaction-value threshold environment variables (e.g., MAX_TRANSACTION_AMOUNT) " + "to each agent action-group Lambda.\n" + "2. Implement threshold enforcement logic in the Lambda handler.\n" + "3. Configure AgentCore Policy Engine rules to cap financial transaction amounts.\n" + "4. Route transactions exceeding thresholds to a human-in-the-loop approval step." + ), + reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/policy-engine.html", + severity="High", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-67", + finding_name="Agent Action-Group Lambdas Have Threshold Configuration", + finding_details=( + f"Found {len(action_group_lambdas)} agent action-group Lambda(s) with " + "threshold/limit environment variables present." + ), + resolution="Verify threshold values are appropriate for your financial risk tolerance.", + reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/policy-engine.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Agent Financial Transaction Value Thresholds Check", e) + return findings + + +def check_api_gateway_request_body_size_limits() -> Dict[str, Any]: + """ + FS-68 — Verify API Gateway REST/HTTP APIs fronting GenAI endpoints enforce + maximum input payload sizes via request validators or WAF body-size rules + to prevent token-exhaustion attacks via oversized prompts. + COMPLIANCE_PLACEHOLDER: [FFIEC CAT, DORA Art.6, PCI-DSS, OWASP LLM10] + """ + findings = _empty_findings("API Gateway Request Body Size Limits Check") + try: + apigw = boto3.client("apigateway", config=boto3_config) + wafv2 = boto3.client("wafv2", config=boto3_config) + + # Check REST APIs for request validators + rest_apis = apigw.get_rest_apis().get("items", []) + apis_without_validators = [] + for api in rest_apis: + validators = apigw.get_request_validators(restApiId=api["id"]).get( + "items", [] + ) + if not validators: + apis_without_validators.append(api.get("name", api["id"])) + + # Check WAF rules for body size constraints + acls = wafv2.list_web_acls(Scope="REGIONAL").get("WebACLs", []) + acls_with_size_rules = 0 + for acl in acls: + acl_detail = wafv2.get_web_acl( + Name=acl["Name"], Scope="REGIONAL", Id=acl["Id"] + ) + rules = acl_detail.get("WebACL", {}).get("Rules", []) + for rule in rules: + stmt = json.dumps(rule.get("Statement", {})) + if "SizeConstraintStatement" in stmt or "body" in stmt.lower(): + acls_with_size_rules += 1 + break + + issues = [] + if apis_without_validators: + issues.append( + f"REST APIs without request validators: {', '.join(apis_without_validators[:5])}" + ) + if acls and acls_with_size_rules == 0: + issues.append("No WAF Web ACLs have body-size constraint rules configured.") + + if issues: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-68", + finding_name="API Gateway Request Body Size Limits Not Enforced", + finding_details=( + "Input payload size limits are not fully enforced on GenAI API endpoints. " + "Oversized prompts can exhaust Bedrock token quotas and inflate costs:\n" + + "\n".join(f"- {i}" for i in issues) + ), + resolution=( + "1. Add API Gateway request validators to enforce maximum body size on " + "all Bedrock-facing REST API methods.\n" + "2. Add a WAF SizeConstraintStatement rule to block requests with body " + "size exceeding your maximum prompt length (e.g., 32 KB).\n" + "3. Set the max_tokens parameter in Bedrock API calls to cap output length.\n" + "4. Implement client-side token counting before submitting requests." + ), + reference="https://docs.aws.amazon.com/waf/latest/developerguide/waf-rule-statement-type-size-constraint.html", + severity="Medium", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-68", + finding_name="API Gateway Request Body Size Limits Configured", + finding_details=( + f"Found {len(rest_apis)} REST API(s) with validators and " + f"{acls_with_size_rules} WAF ACL(s) with body-size rules." + ), + resolution="No action required.", + reference="https://docs.aws.amazon.com/waf/latest/developerguide/waf-rule-statement-type-size-constraint.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("API Gateway Request Body Size Limits Check", e) + return findings + + +def check_prompt_input_validation_function() -> Dict[str, Any]: + """ + FS-69 — Check for a Lambda function or API Gateway request validator that + sanitizes user prompt input (strips special characters, enforces expected + format, rejects oversized inputs) before forwarding to Bedrock. + COMPLIANCE_PLACEHOLDER: [OWASP LLM01, FFIEC CAT, NYDFS 500.06] + """ + findings = _empty_findings("Prompt Input Validation Function Check") + try: + lambda_client = boto3.client("lambda", config=boto3_config) + functions = lambda_client.list_functions().get("Functions", []) + + # Look for Lambda functions with input validation / sanitization naming patterns + VALIDATION_KEYWORDS = [ + "sanitiz", + "validat", + "input", + "preprocess", + "pre-process", + "filter", + "clean", + "prompt-guard", + "promptguard", + ] + validation_lambdas = [ + f["FunctionName"] + for f in functions + if any(kw in f["FunctionName"].lower() for kw in VALIDATION_KEYWORDS) + ] + + if not validation_lambdas: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-69", + finding_name="No Prompt Input Validation Function Found", + finding_details=( + "No Lambda functions matching input validation or sanitization naming " + "patterns were found. Without explicit prompt input validation, malicious " + "inputs (special characters, oversized payloads, injection sequences) may " + "reach Bedrock unfiltered, bypassing WAF-level controls." + ), + resolution=( + "1. Implement a Lambda authorizer or pre-processing function that:\n" + " - Strips or escapes special characters from user input.\n" + " - Validates input against an expected format (e.g., regex allowlist).\n" + " - Rejects inputs exceeding maximum token/character limits.\n" + " - Logs rejected inputs for security monitoring.\n" + "2. Use parameterized prompt templates instead of string concatenation.\n" + "3. Apply Bedrock Guardrails PROMPT_ATTACK filter as a complementary control.\n" + "4. Reference: AWS Prompt Injection Security guidance." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-injection.html", + severity="Medium", + status="Failed", + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-69", + finding_name="Prompt Input Validation Functions Present", + finding_details=( + f"Found {len(validation_lambdas)} Lambda function(s) with input " + f"validation/sanitization naming patterns: " + f"{', '.join(validation_lambdas[:5])}." + ), + resolution=( + "Review these functions to confirm they cover: special-character stripping, " + "format validation, size limits, and injection-sequence detection." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-injection.html", + severity="Informational", + status="Passed", + ) + ) + except Exception as e: + return _error_findings("Prompt Input Validation Function Check", e) + return findings + + +# =========================================================================== +# REPORT GENERATION & LAMBDA HANDLER +# =========================================================================== + + +def generate_csv_report(findings: List[Dict[str, Any]]) -> str: + """Generate CSV report from all security check findings.""" + csv_buffer = StringIO() + fieldnames = [ + "Check_ID", + "Finding", + "Finding_Details", + "Resolution", + "Reference", + "Severity", + "Status", + ] + writer = csv.DictWriter(csv_buffer, fieldnames=fieldnames) + writer.writeheader() + for finding in findings: + for row in finding.get("csv_data", []): + writer.writerow(row) + return csv_buffer.getvalue() + + +def write_to_s3(execution_id: str, csv_content: str, bucket_name: str) -> str: + """Write CSV report to S3 bucket.""" + s3_client = boto3.client("s3", config=boto3_config) + file_name = f"finserv_security_report_{execution_id}.csv" + s3_client.put_object( + Bucket=bucket_name, Key=file_name, Body=csv_content, ContentType="text/csv" + ) + return f"https://{bucket_name}.s3.amazonaws.com/{file_name}" + + +def lambda_handler(event, context): + """Main Lambda handler — runs all 69 FinServ security checks.""" + logger.info("Starting FinServ GenAI security assessment") + all_findings = [] + + execution_id = event.get("Execution", {}).get("Name", "local-test") + permission_cache = get_permissions_cache(execution_id) or { + "role_permissions": {}, + "user_permissions": {}, + } + + # --- Category 1: Unbounded Consumption --- + all_findings.append(check_waf_shield_on_bedrock_endpoints()) + all_findings.append(check_api_gateway_rate_limiting()) + all_findings.append(check_bedrock_token_quotas()) + all_findings.append(check_cost_anomaly_detection()) + all_findings.append(check_cloudwatch_token_alarms()) + all_findings.append(check_aws_budgets_for_aiml()) + + # --- Category 2: Excessive Agency --- + all_findings.append(check_bedrock_agent_action_boundaries(permission_cache)) + all_findings.append(check_agentcore_policy_engine()) + all_findings.append(check_agent_transaction_limits()) + all_findings.append(check_human_in_the_loop_for_high_risk_actions()) + all_findings.append(check_agent_rate_alarms()) + + # --- Category 3: Supply Chain Vulnerabilities --- + all_findings.append(check_scp_model_access_restrictions()) + all_findings.append(check_model_inventory_tagging()) + all_findings.append(check_model_onboarding_governance()) + all_findings.append(check_bedrock_model_evaluation_adversarial()) + all_findings.append(check_ecr_image_scanning()) + + # --- Category 4: Training Data & Model Poisoning --- + # NOTE: FS-17 (check_sagemaker_model_monitor_data_quality), FS-18 (check_sagemaker_model_monitor_drift), + # and FS-19 (check_sagemaker_model_registry_approval) are merged into upstream SM-07, SM-23, SM-22 + # respectively — see extension notes in SECURITY_CHECKS_FINSERV_PART1_INFRA_CONTROLS.md. + all_findings.append(check_feature_store_rollback_capability()) + all_findings.append(check_training_data_s3_versioning()) + + # --- Category 5: Vector & Embedding Weaknesses --- + all_findings.append(check_knowledge_base_iam_least_privilege(permission_cache)) + # NOTE: FS-23 (check_knowledge_base_cloudtrail_logging) is merged into upstream BR-06 + # — see extension note in SECURITY_CHECKS_FINSERV_PART1_INFRA_CONTROLS.md. + all_findings.append(check_knowledge_base_metadata_filtering()) + all_findings.append(check_opensearch_serverless_encryption()) + all_findings.append(check_knowledge_base_vpc_access()) + + # --- Category 6: Non-Compliant Output --- + all_findings.append(check_automated_reasoning_checks()) + all_findings.append(check_guardrail_denied_topics_financial()) + all_findings.append(check_compliance_disclaimer_in_outputs()) + all_findings.append(check_bedrock_evaluation_compliance_datasets()) + + # --- Category 7: Misinformation --- + all_findings.append(check_knowledge_base_data_source_sync()) + all_findings.append(check_source_attribution_in_guardrails()) + all_findings.append(check_knowledge_base_integrity_monitoring()) + all_findings.append(check_fm_version_currency()) + + # --- Category 8: Abusive or Harmful Output --- + all_findings.append(check_fmeval_harmful_content()) + all_findings.append(check_guardrail_content_filters()) + all_findings.append(check_user_feedback_mechanism()) + all_findings.append(check_guardrail_word_filters()) + + # --- Category 9: Biased Output --- + all_findings.append(check_sagemaker_clarify_bias()) + all_findings.append(check_bedrock_evaluation_bias_datasets()) + all_findings.append(check_sagemaker_clarify_explainability()) + all_findings.append(check_ai_service_cards_documentation()) + + # --- Category 10: Sensitive Information Disclosure --- + all_findings.append(check_cloudwatch_log_pii_masking()) + all_findings.append(check_macie_on_training_data_buckets()) + all_findings.append(check_guardrail_pii_filters()) + all_findings.append(check_data_classification_tagging()) + + # --- Category 11: Hallucination --- + all_findings.append(check_guardrail_grounding_threshold()) + all_findings.append(check_rag_knowledge_base_configured()) + all_findings.append(check_hallucination_disclaimer_advisory()) + all_findings.append(check_automated_reasoning_checks_hallucination()) + + # --- Category 12: Prompt Injection --- + all_findings.append(check_prompt_injection_input_validation()) + all_findings.append(check_bedrock_sdk_version_currency()) + all_findings.append(check_waf_sql_injection_rules()) + all_findings.append(check_penetration_testing_evidence()) + + # --- Category 13: Improper Output Handling --- + all_findings.append(check_output_validation_lambda()) + all_findings.append(check_xss_prevention_waf()) + all_findings.append(check_output_encoding_advisory()) + all_findings.append(check_output_schema_validation()) + + # --- Category 14: Off-Topic & Inappropriate Output --- + all_findings.append(check_guardrail_topic_allowlist()) + all_findings.append(check_contextual_grounding_for_offtopic()) + + # --- Category 15: Out-of-Date Training Data --- + all_findings.append(check_knowledge_base_sync_schedule()) + all_findings.append(check_data_currency_disclaimer_advisory()) + all_findings.append(check_foundation_model_lifecycle_policy()) + + # --- Material Gap Checks (FS-65 to FS-69) --- + # NOTE: FS-64 (check_guardrail_trace_logging) is merged into upstream BR-04 + # — see extension note in SECURITY_CHECKS_FINSERV_PART3_APP_LAYER_AND_GAPS.md. + all_findings.append(check_kb_datasource_s3_event_notifications()) + all_findings.append(check_agentcore_end_user_identity_propagation()) + all_findings.append(check_agent_financial_transaction_thresholds()) + all_findings.append(check_api_gateway_request_body_size_limits()) + all_findings.append(check_prompt_input_validation_function()) + + # Generate and upload report + csv_content = generate_csv_report(all_findings) + bucket_name = os.environ.get("AIML_ASSESSMENT_BUCKET_NAME") + if not bucket_name: + raise ValueError("AIML_ASSESSMENT_BUCKET_NAME environment variable is not set") + + s3_url = write_to_s3(execution_id, csv_content, bucket_name) + + return { + "statusCode": 200, + "body": { + "message": "FinServ security assessment completed", + "findings": all_findings, + "report_url": s3_url, + }, + } diff --git a/aiml-security-assessment/functions/security/finserv_assessments/requirements.txt b/aiml-security-assessment/functions/security/finserv_assessments/requirements.txt new file mode 100644 index 0000000..572b352 --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_assessments/requirements.txt @@ -0,0 +1 @@ +pydantic diff --git a/aiml-security-assessment/functions/security/finserv_assessments/schema.py b/aiml-security-assessment/functions/security/finserv_assessments/schema.py new file mode 100644 index 0000000..cc7caa3 --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_assessments/schema.py @@ -0,0 +1,80 @@ +""" +Schema module for FinServ security findings. +Mirrors the schema used in bedrock_assessments/schema.py. +""" + +from enum import Enum +from typing import Dict, Any +from pydantic import BaseModel, Field, validator +import re + + +class SeverityEnum(str, Enum): + HIGH = "High" + MEDIUM = "Medium" + LOW = "Low" + INFORMATIONAL = "Informational" + + +class StatusEnum(str, Enum): + FAILED = "Failed" + PASSED = "Passed" + NA = "N/A" + + +class Finding(BaseModel): + """Represents a security finding with required fields and validations.""" + + Check_ID: str = Field( + ..., + min_length=1, + description="Unique check identifier (e.g., FS-01)", + ) + Finding: str = Field(..., min_length=1, description="The name/title of the finding") + Finding_Details: str = Field( + ..., min_length=1, description="Detailed description of the finding" + ) + Resolution: str = Field( + ..., min_length=0, description="Steps to resolve the finding" + ) + Reference: str = Field(..., description="Documentation reference URL") + Severity: SeverityEnum = Field(..., description="Severity level of the finding") + Status: StatusEnum = Field(..., description="Current status of the finding") + + @validator("Check_ID") + def validate_check_id(cls, v): + # Allow FS-NN pattern for FinServ checks + pattern = r"^[A-Z]{2,3}-\d{2}$" + if not re.match(pattern, v): + raise ValueError( + "Check_ID must follow pattern XX-NN (e.g., FS-01, BR-14, AC-05)" + ) + return v + + @validator("Reference") + def validate_reference_url(cls, v): + if not str(v).startswith("https://"): + raise ValueError("Reference URL must start with https://") + return v + + +def create_finding( + check_id: str, + finding_name: str, + finding_details: str, + resolution: str, + reference: str, + severity: SeverityEnum, + status: StatusEnum, +) -> Dict[str, Any]: + """Create a validated finding dict.""" + finding = Finding( + Check_ID=check_id, + Finding=finding_name, + Finding_Details=finding_details, + Resolution=resolution, + Reference=reference, + Severity=severity, + Status=status, + ) + return dict(finding.model_dump()) diff --git a/aiml-security-assessment/functions/security/generate_consolidated_report/app.py b/aiml-security-assessment/functions/security/generate_consolidated_report/app.py index 7626a42..3342372 100644 --- a/aiml-security-assessment/functions/security/generate_consolidated_report/app.py +++ b/aiml-security-assessment/functions/security/generate_consolidated_report/app.py @@ -44,7 +44,7 @@ def parse_csv_content(csv_content: str) -> List[Dict[str, str]]: def get_assessment_results(execution_id: str, account_id: str = None) -> Dict[str, Any]: """ - Download and parse Bedrock and SageMaker assessment CSV files for a given execution + Download and parse Bedrock, SageMaker, AgentCore, and FinServ assessment CSV files for a given execution Args: s3_bucket (str): Source S3 bucket name @@ -72,6 +72,11 @@ def get_assessment_results(execution_id: str, account_id: str = None) -> Dict[st Bucket=s3_bucket, Prefix=f"agentcore_security_report_{execution_id}" ) + # Also check for FinServ reports + finserv_response = s3_client.list_objects_v2( + Bucket=s3_bucket, Prefix=f"finserv_security_report_{execution_id}" + ) + # Combine all responses all_objects = [] if "Contents" in response: @@ -80,6 +85,8 @@ def get_assessment_results(execution_id: str, account_id: str = None) -> Dict[st all_objects.extend(sagemaker_response["Contents"]) if "Contents" in agentcore_response: all_objects.extend(agentcore_response["Contents"]) + if "Contents" in finserv_response: + all_objects.extend(finserv_response["Contents"]) if not all_objects: logger.warning(f"No assessment files found for execution {execution_id}") return {} @@ -91,6 +98,7 @@ def get_assessment_results(execution_id: str, account_id: str = None) -> Dict[st "bedrock": {}, "sagemaker": {}, "agentcore": {}, + "finserv": {}, } # Process each CSV file @@ -126,6 +134,8 @@ def get_assessment_results(execution_id: str, account_id: str = None) -> Dict[st category = "sagemaker" elif "agentcore" in s3_key.lower(): category = "agentcore" + elif "finserv" in s3_key.lower(): + category = "finserv" else: logger.warning(f"Unknown assessment type for file: {s3_key}") continue @@ -146,10 +156,11 @@ def get_assessment_results(execution_id: str, account_id: str = None) -> Dict[st assessment_results["summary"] = { "total_files_processed": len(assessment_results["bedrock"]) + len(assessment_results["sagemaker"]) - + len(assessment_results["agentcore"]), + + len(assessment_results["agentcore"]) + + len(assessment_results["finserv"]), "categories_found": [ cat - for cat in ["bedrock", "sagemaker", "agentcore"] + for cat in ["bedrock", "sagemaker", "agentcore", "finserv"] if assessment_results[cat] ], "rows": assessment_results["bedrock"], @@ -157,6 +168,7 @@ def get_assessment_results(execution_id: str, account_id: str = None) -> Dict[st "bedrock": list(assessment_results["bedrock"].keys()), "sagemaker": list(assessment_results["sagemaker"].keys()), "agentcore": list(assessment_results["agentcore"].keys()), + "finserv": list(assessment_results["finserv"].keys()), }, } @@ -185,7 +197,7 @@ def generate_html_report(assessment_results: Dict[str, Any]) -> str: expected by the shared report_template module. Args: - assessment_results: Dict containing bedrock, sagemaker, agentcore findings + assessment_results: Dict containing bedrock, sagemaker, agentcore, finserv findings Returns: HTML report string diff --git a/aiml-security-assessment/functions/security/sagemaker_assessments/app.py b/aiml-security-assessment/functions/security/sagemaker_assessments/app.py index c6e0cba..558893d 100644 --- a/aiml-security-assessment/functions/security/sagemaker_assessments/app.py +++ b/aiml-security-assessment/functions/security/sagemaker_assessments/app.py @@ -1107,6 +1107,12 @@ def check_sagemaker_model_monitor_usage(permission_cache) -> Dict[str, Any]: """ Check if SageMaker Model Monitor is configured and actively monitoring models """ + # FinServ extension (FS-17): The FinServ guide (PDF §1.2.14) asks for Model + # Monitor data-quality baselines to be refreshed on a regulator-aligned cadence + # (SR 11-7 ongoing monitoring) and for the baseline statistics to be emitted to + # CloudWatch under namespace /aws/sagemaker/Endpoints/data-metric with + # emit_metrics=Enabled. See docs/SECURITY_CHECKS_FINSERV_PART1_INFRA_CONTROLS.md + # (FS-17 → SM-07 extension note). logger.debug("Starting check for SageMaker Model Monitor usage") try: findings = {"csv_data": []} @@ -2692,6 +2698,11 @@ def check_model_approval_workflow() -> Dict[str, Any]: Check if Model Registry has proper approval workflows configured. Validates that models go through approval process before production deployment. """ + # FinServ extension (FS-19): The FinServ guide (PDF §1.2.14) expects model + # package groups to enforce ModelApprovalStatus=PendingManualApproval by default + # and to flag model packages that are auto-approved as their latest version. + # See docs/SECURITY_CHECKS_FINSERV_PART1_INFRA_CONTROLS.md (FS-19 → SM-22 + # extension note) for the detection refinement. logger.debug("Starting check for model approval workflow") try: findings = {"csv_data": []} @@ -2821,6 +2832,11 @@ def check_model_drift_detection() -> Dict[str, Any]: Check if Model Monitor is configured for drift detection with proper baselines. Validates that models have data quality and model quality monitoring configured. """ + # FinServ extension (FS-18): In addition to ModelQuality drift monitoring, the + # FinServ guide (PDF §1.2.14) calls out low-entropy classification monitoring + # as an early-warning indicator of training-data poisoning. See + # docs/SECURITY_CHECKS_FINSERV_PART1_INFRA_CONTROLS.md (FS-18 → SM-23 + # extension note) for the remediation step to add. logger.debug("Starting check for model drift detection") try: findings = {"csv_data": []} diff --git a/aiml-security-assessment/statemachine/assessments.asl.json b/aiml-security-assessment/statemachine/assessments.asl.json index 81a48d8..c95a99b 100644 --- a/aiml-security-assessment/statemachine/assessments.asl.json +++ b/aiml-security-assessment/statemachine/assessments.asl.json @@ -1,184 +1,215 @@ { - "Comment": "A state machine that performs AI/ML security assessments.", - "StartAt": "Cleanup S3 Bucket", - "States": { - "Cleanup S3 Bucket": { - "Type": "Task", - "Resource": "arn:aws:states:::lambda:invoke", - "Parameters": { - "FunctionName": "${CleanupBucketFunction}", + "Comment": "A state machine that performs AI/ML security assessments.", + "StartAt": "Cleanup S3 Bucket", + "States": { + "Cleanup S3 Bucket": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${CleanupBucketFunction}", + "Payload": { + "Execution.$": "$$.Execution", + "StateMachine.$": "$$.StateMachine" + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2, + "JitterStrategy": "FULL" + } + ], + "ResultPath": null, + "Next": "IAM Permission Caching" + }, + "IAM Permission Caching": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${IAMPermissionCachingFunction}", + "Payload": { + "Execution.$": "$$.Execution", + "StateMachine.$": "$$.StateMachine" + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2, + "JitterStrategy": "FULL" + } + ], + "ResultPath": null, + "Next": "Run Security Assessments" + }, + "Run Security Assessments": { + "Type": "Parallel", + "Branches": [ + { + "StartAt": "Bedrock Security Assessment", + "States": { + "Bedrock Security Assessment": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${BedrockSecurityAssessmentFunction}", "Payload": { - "Execution.$": "$$.Execution", - "StateMachine.$": "$$.StateMachine" + "Execution.$": "$$.Execution", + "StateMachine.$": "$$.StateMachine" } - }, - "Retry": [ + }, + "Retry": [ { - "ErrorEquals": [ - "Lambda.ServiceException", - "Lambda.AWSLambdaException", - "Lambda.SdkClientException", - "Lambda.TooManyRequestsException" - ], - "IntervalSeconds": 1, - "MaxAttempts": 3, - "BackoffRate": 2, - "JitterStrategy": "FULL" + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2, + "JitterStrategy": "FULL" } - ], - "ResultPath": null, - "Next": "IAM Permission Caching" + ], + "End": true + } + } }, - "IAM Permission Caching": { - "Type": "Task", - "Resource": "arn:aws:states:::lambda:invoke", - "Parameters": { - "FunctionName": "${IAMPermissionCachingFunction}", + { + "StartAt": "Sagemaker Security Assessment", + "States": { + "Sagemaker Security Assessment": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${SagemakerSecurityAssessmentFunction}", "Payload": { - "Execution.$": "$$.Execution", - "StateMachine.$": "$$.StateMachine" + "Execution.$": "$$.Execution", + "StateMachine.$": "$$.StateMachine" } - }, - "Retry": [ + }, + "Retry": [ { - "ErrorEquals": [ - "Lambda.ServiceException", - "Lambda.AWSLambdaException", - "Lambda.SdkClientException", - "Lambda.TooManyRequestsException" - ], - "IntervalSeconds": 1, - "MaxAttempts": 3, - "BackoffRate": 2, - "JitterStrategy": "FULL" + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2, + "JitterStrategy": "FULL" } - ], - "ResultPath": null, - "Next": "Run Security Assessments" + ], + "End": true + } + } }, - "Run Security Assessments": { - "Type": "Parallel", - "Branches": [ - { - "StartAt": "Bedrock Security Assessment", - "States": { - "Bedrock Security Assessment": { - "Type": "Task", - "Resource": "arn:aws:states:::lambda:invoke", - "Parameters": { - "FunctionName": "${BedrockSecurityAssessmentFunction}", - "Payload": { - "Execution.$": "$$.Execution", - "StateMachine.$": "$$.StateMachine" - } - }, - "Retry": [ - { - "ErrorEquals": [ - "Lambda.ServiceException", - "Lambda.AWSLambdaException", - "Lambda.SdkClientException", - "Lambda.TooManyRequestsException" - ], - "IntervalSeconds": 1, - "MaxAttempts": 3, - "BackoffRate": 2, - "JitterStrategy": "FULL" - } - ], - "End": true - } - } - }, - { - "StartAt": "Sagemaker Security Assessment", - "States": { - "Sagemaker Security Assessment": { - "Type": "Task", - "Resource": "arn:aws:states:::lambda:invoke", - "Parameters": { - "FunctionName": "${SagemakerSecurityAssessmentFunction}", - "Payload": { - "Execution.$": "$$.Execution", - "StateMachine.$": "$$.StateMachine" - } - }, - "Retry": [ - { - "ErrorEquals": [ - "Lambda.ServiceException", - "Lambda.AWSLambdaException", - "Lambda.SdkClientException", - "Lambda.TooManyRequestsException" - ], - "IntervalSeconds": 1, - "MaxAttempts": 3, - "BackoffRate": 2, - "JitterStrategy": "FULL" - } - ], - "End": true - } - } - }, + { + "StartAt": "AgentCore Security Assessment", + "States": { + "AgentCore Security Assessment": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${AgentCoreSecurityAssessmentFunction}", + "Payload": { + "Execution.$": "$$.Execution", + "StateMachine.$": "$$.StateMachine" + } + }, + "Retry": [ { - "StartAt": "AgentCore Security Assessment", - "States": { - "AgentCore Security Assessment": { - "Type": "Task", - "Resource": "arn:aws:states:::lambda:invoke", - "Parameters": { - "FunctionName": "${AgentCoreSecurityAssessmentFunction}", - "Payload": { - "Execution.$": "$$.Execution", - "StateMachine.$": "$$.StateMachine" - } - }, - "Retry": [ - { - "ErrorEquals": [ - "Lambda.ServiceException", - "Lambda.AWSLambdaException", - "Lambda.SdkClientException", - "Lambda.TooManyRequestsException" - ], - "IntervalSeconds": 1, - "MaxAttempts": 3, - "BackoffRate": 2, - "JitterStrategy": "FULL" - } - ], - "End": true - } - } + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2, + "JitterStrategy": "FULL" } - ], - "Next": "Generate Consolidated Report" + ], + "End": true + } + } }, - "Generate Consolidated Report": { - "Type": "Task", - "Resource": "arn:aws:states:::lambda:invoke", - "Parameters": { - "FunctionName": "${GenerateConsolidatedReportFunction}", + { + "StartAt": "FinServ Security Assessment", + "States": { + "FinServ Security Assessment": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${FinServSecurityAssessmentFunction}", "Payload": { - "Execution.$": "$$.Execution" + "Execution.$": "$$.Execution", + "StateMachine.$": "$$.StateMachine" } - }, - "Retry": [ + }, + "Retry": [ { - "ErrorEquals": [ - "Lambda.ServiceException", - "Lambda.AWSLambdaException", - "Lambda.SdkClientException", - "Lambda.TooManyRequestsException" - ], - "IntervalSeconds": 1, - "MaxAttempts": 3, - "BackoffRate": 2, - "JitterStrategy": "FULL" + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2, + "JitterStrategy": "FULL" } - ], - "End": true + ], + "End": true + } + } + } + ], + "Next": "Generate Consolidated Report" + }, + "Generate Consolidated Report": { + "Type": "Task", + "Resource": "arn:aws:states:::lambda:invoke", + "Parameters": { + "FunctionName": "${GenerateConsolidatedReportFunction}", + "Payload": { + "Execution.$": "$$.Execution" + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 1, + "MaxAttempts": 3, + "BackoffRate": 2, + "JitterStrategy": "FULL" } + ], + "End": true } + } } diff --git a/aiml-security-assessment/template-multi-account.yaml b/aiml-security-assessment/template-multi-account.yaml index a16af9d..01268a1 100644 --- a/aiml-security-assessment/template-multi-account.yaml +++ b/aiml-security-assessment/template-multi-account.yaml @@ -16,6 +16,7 @@ Resources: BedrockSecurityAssessmentFunction: !GetAtt BedrockSecurityAssessmentFunction.Arn SagemakerSecurityAssessmentFunction: !GetAtt SagemakerSecurityAssessmentFunction.Arn AgentCoreSecurityAssessmentFunction: !GetAtt AgentCoreSecurityAssessmentFunction.Arn + FinServSecurityAssessmentFunction: !GetAtt FinServSecurityAssessmentFunction.Arn GenerateConsolidatedReportFunction: !GetAtt GenerateConsolidatedReportFunction.Arn AIMLAssessmentBucketName: !Ref AIMLAssessmentBucket Policies: @@ -29,6 +30,8 @@ Resources: FunctionName: !Ref SagemakerSecurityAssessmentFunction - LambdaInvokePolicy: FunctionName: !Ref AgentCoreSecurityAssessmentFunction + - LambdaInvokePolicy: + FunctionName: !Ref FinServSecurityAssessmentFunction - LambdaInvokePolicy: FunctionName: !Ref GenerateConsolidatedReportFunction - S3CrudPolicy: @@ -382,6 +385,170 @@ Resources: - cloudwatch:PutMetricData Resource: '*' + + FinServSecurityAssessmentFunction: + Type: AWS::Serverless::Function + Properties: + FunctionName: !Sub 'aiml-security-${AWS::StackName}-FinServAssessment' + CodeUri: functions/security/finserv_assessments/ + Handler: app.lambda_handler + Runtime: python3.12 + Architectures: + - x86_64 + Timeout: 600 # 10 minutes + MemorySize: 1024 + Environment: + Variables: + AIML_ASSESSMENT_BUCKET_NAME: !Ref AIMLAssessmentBucket + Policies: + - S3CrudPolicy: + BucketName: !Ref AIMLAssessmentBucket + - Statement: + - Sid: IAMPermissions + Effect: Allow + Action: + - iam:ListRoles + - iam:ListUsers + - iam:ListAttachedRolePolicies + - iam:ListAttachedUserPolicies + - iam:ListRolePolicies + - iam:ListUserPolicies + - iam:GetRolePolicy + - iam:GetUserPolicy + - iam:GetPolicy + - iam:GetPolicyVersion + - iam:GetRole + - iam:GetUser + - sts:GetCallerIdentity + Resource: '*' + - Sid: WAFShieldPermissions + Effect: Allow + Action: + - wafv2:ListWebACLs + - wafv2:GetWebACL + - shield:DescribeSubscription + Resource: '*' + - Sid: APIGatewayPermissions + Effect: Allow + Action: + - apigateway:GET + Resource: '*' + - Sid: ServiceQuotasPermissions + Effect: Allow + Action: + - servicequotas:ListServiceQuotas + - servicequotas:ListAWSDefaultServiceQuotas + Resource: '*' + - Sid: CostBudgetPermissions + Effect: Allow + Action: + - ce:GetAnomalyMonitors + - budgets:ViewBudget + Resource: '*' + - Sid: CloudWatchPermissions + Effect: Allow + Action: + - cloudwatch:DescribeAlarms + - cloudwatch:ListMetrics + Resource: '*' + - Sid: LogsPermissions + Effect: Allow + Action: + - logs:DescribeAccountPolicies + - logs:DescribeLogGroups + Resource: '*' + - Sid: BedrockPermissions + Effect: Allow + Action: + - bedrock:ListGuardrails + - bedrock:GetGuardrail + - bedrock:ListFoundationModels + - bedrock:ListCustomModels + - bedrock:ListEvaluationJobs + - bedrock:GetModelInvocationLoggingConfiguration + Resource: '*' + - Sid: BedrockAgentPermissions + Effect: Allow + Action: + - bedrock:ListAgents + - bedrock:GetAgent + - bedrock:ListKnowledgeBases + - bedrock:GetKnowledgeBase + - bedrock:ListDataSources + - bedrock:GetDataSource + Resource: '*' + - Sid: BedrockAgentCorePermissions + Effect: Allow + Action: + - bedrock-agentcore:ListAgentRuntimes + - bedrock-agentcore:GetAgentRuntime + Resource: '*' + - Sid: SageMakerPermissions + Effect: Allow + Action: + - sagemaker:ListMonitoringSchedules + - sagemaker:ListModelPackageGroups + - sagemaker:ListModelPackages + - sagemaker:ListFeatureGroups + - sagemaker:ListModels + - sagemaker:ListModelCards + - sagemaker:ListTags + Resource: '*' + - Sid: LambdaStatesPermissions + Effect: Allow + Action: + - lambda:ListFunctions + - lambda:GetFunctionConcurrency + - states:ListStateMachines + - states:DescribeStateMachine + Resource: '*' + - Sid: OrganizationsPermissions + Effect: Allow + Action: + - organizations:ListPolicies + - organizations:DescribePolicy + Resource: '*' + - Sid: ECRPermissions + Effect: Allow + Action: + - ecr:DescribeRepositories + - ecr:DescribeImageScanFindings + Resource: '*' + - Sid: CloudTrailPermissions + Effect: Allow + Action: + - cloudtrail:ListTrails + - cloudtrail:GetTrail + - cloudtrail:GetTrailStatus + - cloudtrail:GetEventSelectors + Resource: '*' + - Sid: OpenSearchServerlessPermissions + Effect: Allow + Action: + - aoss:ListSecurityPolicies + Resource: '*' + - Sid: MaciePermissions + Effect: Allow + Action: + - macie2:GetMacieSession + Resource: '*' + - Sid: EventsConfigPermissions + Effect: Allow + Action: + - events:ListRules + - config:DescribeConfigRules + Resource: '*' + - Sid: S3BucketPermissions + Effect: Allow + Action: + - s3:ListAllMyBuckets + - s3:ListBucket + - s3:GetBucketVersioning + - s3:GetBucketTagging + - s3:GetBucketEncryption + - s3:GetBucketNotification + Resource: 'arn:aws:s3:::*' + # S3 bucket (always created for local storage) AIMLAssessmentBucket: Type: AWS::S3::Bucket diff --git a/aiml-security-assessment/template.yaml b/aiml-security-assessment/template.yaml index 0cb74e5..91b0306 100644 --- a/aiml-security-assessment/template.yaml +++ b/aiml-security-assessment/template.yaml @@ -16,6 +16,7 @@ Resources: BedrockSecurityAssessmentFunction: !GetAtt BedrockSecurityAssessmentFunction.Arn SagemakerSecurityAssessmentFunction: !GetAtt SagemakerSecurityAssessmentFunction.Arn AgentCoreSecurityAssessmentFunction: !GetAtt AgentCoreSecurityAssessmentFunction.Arn + FinServSecurityAssessmentFunction: !GetAtt FinServSecurityAssessmentFunction.Arn GenerateConsolidatedReportFunction: !GetAtt GenerateConsolidatedReportFunction.Arn AIMLAssessmentBucketName: !Ref AIMLAssessmentBucket Policies: @@ -30,6 +31,8 @@ Resources: FunctionName: !Ref SagemakerSecurityAssessmentFunction - LambdaInvokePolicy: FunctionName: !Ref AgentCoreSecurityAssessmentFunction + - LambdaInvokePolicy: + FunctionName: !Ref FinServSecurityAssessmentFunction - LambdaInvokePolicy: FunctionName: !Ref GenerateConsolidatedReportFunction #Add S3 Read Write policy @@ -387,6 +390,170 @@ Resources: - s3:HeadBucket Resource: 'arn:aws:s3:::*' + + FinServSecurityAssessmentFunction: + Type: AWS::Serverless::Function + Properties: + FunctionName: !Sub 'aiml-security-${AWS::StackName}-FinServAssessment' + CodeUri: functions/security/finserv_assessments/ + Handler: app.lambda_handler + Runtime: python3.12 + Architectures: + - x86_64 + Timeout: 600 # 10 minutes + MemorySize: 1024 + Environment: + Variables: + AIML_ASSESSMENT_BUCKET_NAME: !Ref AIMLAssessmentBucket + Policies: + - S3CrudPolicy: + BucketName: !Ref AIMLAssessmentBucket + - Statement: + - Sid: IAMPermissions + Effect: Allow + Action: + - iam:ListRoles + - iam:ListUsers + - iam:ListAttachedRolePolicies + - iam:ListAttachedUserPolicies + - iam:ListRolePolicies + - iam:ListUserPolicies + - iam:GetRolePolicy + - iam:GetUserPolicy + - iam:GetPolicy + - iam:GetPolicyVersion + - iam:GetRole + - iam:GetUser + - sts:GetCallerIdentity + Resource: '*' + - Sid: WAFShieldPermissions + Effect: Allow + Action: + - wafv2:ListWebACLs + - wafv2:GetWebACL + - shield:DescribeSubscription + Resource: '*' + - Sid: APIGatewayPermissions + Effect: Allow + Action: + - apigateway:GET + Resource: '*' + - Sid: ServiceQuotasPermissions + Effect: Allow + Action: + - servicequotas:ListServiceQuotas + - servicequotas:ListAWSDefaultServiceQuotas + Resource: '*' + - Sid: CostBudgetPermissions + Effect: Allow + Action: + - ce:GetAnomalyMonitors + - budgets:ViewBudget + Resource: '*' + - Sid: CloudWatchPermissions + Effect: Allow + Action: + - cloudwatch:DescribeAlarms + - cloudwatch:ListMetrics + Resource: '*' + - Sid: LogsPermissions + Effect: Allow + Action: + - logs:DescribeAccountPolicies + - logs:DescribeLogGroups + Resource: '*' + - Sid: BedrockPermissions + Effect: Allow + Action: + - bedrock:ListGuardrails + - bedrock:GetGuardrail + - bedrock:ListFoundationModels + - bedrock:ListCustomModels + - bedrock:ListEvaluationJobs + - bedrock:GetModelInvocationLoggingConfiguration + Resource: '*' + - Sid: BedrockAgentPermissions + Effect: Allow + Action: + - bedrock:ListAgents + - bedrock:GetAgent + - bedrock:ListKnowledgeBases + - bedrock:GetKnowledgeBase + - bedrock:ListDataSources + - bedrock:GetDataSource + Resource: '*' + - Sid: BedrockAgentCorePermissions + Effect: Allow + Action: + - bedrock-agentcore:ListAgentRuntimes + - bedrock-agentcore:GetAgentRuntime + Resource: '*' + - Sid: SageMakerPermissions + Effect: Allow + Action: + - sagemaker:ListMonitoringSchedules + - sagemaker:ListModelPackageGroups + - sagemaker:ListModelPackages + - sagemaker:ListFeatureGroups + - sagemaker:ListModels + - sagemaker:ListModelCards + - sagemaker:ListTags + Resource: '*' + - Sid: LambdaStatesPermissions + Effect: Allow + Action: + - lambda:ListFunctions + - lambda:GetFunctionConcurrency + - states:ListStateMachines + - states:DescribeStateMachine + Resource: '*' + - Sid: OrganizationsPermissions + Effect: Allow + Action: + - organizations:ListPolicies + - organizations:DescribePolicy + Resource: '*' + - Sid: ECRPermissions + Effect: Allow + Action: + - ecr:DescribeRepositories + - ecr:DescribeImageScanFindings + Resource: '*' + - Sid: CloudTrailPermissions + Effect: Allow + Action: + - cloudtrail:ListTrails + - cloudtrail:GetTrail + - cloudtrail:GetTrailStatus + - cloudtrail:GetEventSelectors + Resource: '*' + - Sid: OpenSearchServerlessPermissions + Effect: Allow + Action: + - aoss:ListSecurityPolicies + Resource: '*' + - Sid: MaciePermissions + Effect: Allow + Action: + - macie2:GetMacieSession + Resource: '*' + - Sid: EventsConfigPermissions + Effect: Allow + Action: + - events:ListRules + - config:DescribeConfigRules + Resource: '*' + - Sid: S3BucketPermissions + Effect: Allow + Action: + - s3:ListAllMyBuckets + - s3:ListBucket + - s3:GetBucketVersioning + - s3:GetBucketTagging + - s3:GetBucketEncryption + - s3:GetBucketNotification + Resource: 'arn:aws:s3:::*' + # Add a S3 bucket to store the AIML assessment results AIMLAssessmentBucket: Type: AWS::S3::Bucket diff --git a/deployment/1-aiml-security-member-roles.yaml b/deployment/1-aiml-security-member-roles.yaml index e4fe71b..f3d1956 100644 --- a/deployment/1-aiml-security-member-roles.yaml +++ b/deployment/1-aiml-security-member-roles.yaml @@ -318,6 +318,53 @@ Resources: - "arn:aws:s3:::aws-sam-cli-managed-default-*" - "arn:aws:s3:::aws-sam-cli-managed-default-*/*" + # FinServ GenAI Risk Assessment Permissions (FS-01 to FS-69) + - Effect: Allow + Action: + # WAF / Shield (FS-01, FS-51..54) + - wafv2:ListWebACLs + - wafv2:GetWebACL + - shield:DescribeSubscription + # API Gateway (FS-02, FS-68) + - apigateway:GET + # Service Quotas (FS-03) + - servicequotas:ListServiceQuotas + - servicequotas:ListAWSDefaultServiceQuotas + # Cost / Budgets (FS-04, FS-06) + - ce:GetAnomalyMonitors + - budgets:ViewBudget + # CloudWatch (FS-05, FS-11, FS-21, FS-33, FS-46) + - cloudwatch:DescribeAlarms + - cloudwatch:ListMetrics + # CloudWatch Logs (FS-43, FS-64) + - logs:DescribeAccountPolicies + # Bedrock — guardrail and evaluation (FS-27..30, FS-36..38, FS-43..45, FS-47..50, FS-59..60) + - bedrock:ListEvaluationJobs + - bedrock:GetEvaluationJob + - bedrock:ListFoundationModels + # Bedrock Agent — runtimes and data sources (FS-31, FS-65) + - bedrock-agent:ListDataSources + - bedrock-agent:GetDataSource + # SageMaker — additional inventory (FS-42, FS-61, FS-63) + - sagemaker:ListModelCards + - sagemaker:ListTags + # S3 — additional bucket metadata (FS-20, FS-46, FS-65) + - s3:ListAllMyBuckets + - s3:GetBucketNotification + # ECR — image-scan findings (FS-16) + - ecr:DescribeImageScanFindings + # OpenSearch Serverless (FS-25, FS-26) + - aoss:ListSecurityPolicies + # Macie — PII protection (FS-44) + - macie2:GetMacieSession + # EventBridge / Config (FS-32, FS-34) + - events:ListRules + - config:DescribeConfigRules + # Organizations — SCPs (FS-12) + - organizations:ListPolicies + - organizations:DescribePolicy + Resource: "*" + Outputs: MemberRoleArn: Description: ARN of the AIML Security Member Role diff --git a/deployment/aiml-security-single-account.yaml b/deployment/aiml-security-single-account.yaml index ad42582..39c23a1 100644 --- a/deployment/aiml-security-single-account.yaml +++ b/deployment/aiml-security-single-account.yaml @@ -229,6 +229,53 @@ Resources: - s3:HeadBucket Resource: "*" + # FinServ GenAI Risk Assessment Permissions (FS-01 to FS-69) + - Effect: Allow + Action: + # WAF / Shield (FS-01, FS-51..54) + - wafv2:ListWebACLs + - wafv2:GetWebACL + - shield:DescribeSubscription + # API Gateway (FS-02, FS-68) + - apigateway:GET + # Service Quotas (FS-03) + - servicequotas:ListServiceQuotas + - servicequotas:ListAWSDefaultServiceQuotas + # Cost / Budgets (FS-04, FS-06) + - ce:GetAnomalyMonitors + - budgets:ViewBudget + # CloudWatch (FS-05, FS-11, FS-21, FS-33, FS-46) + - cloudwatch:DescribeAlarms + - cloudwatch:ListMetrics + # CloudWatch Logs (FS-43, FS-64) + - logs:DescribeAccountPolicies + # Bedrock — guardrail and evaluation (FS-27..30, FS-36..38, FS-43..45, FS-47..50, FS-59..60) + - bedrock:ListEvaluationJobs + - bedrock:GetEvaluationJob + - bedrock:ListFoundationModels + # Bedrock Agent — runtimes and data sources (FS-31, FS-65) + - bedrock-agent:ListDataSources + - bedrock-agent:GetDataSource + # SageMaker — additional inventory (FS-42, FS-61, FS-63) + - sagemaker:ListModelCards + - sagemaker:ListTags + # S3 — additional bucket metadata (FS-20, FS-46, FS-65) + - s3:ListAllMyBuckets + - s3:GetBucketNotification + # ECR — image-scan findings (FS-16) + - ecr:DescribeImageScanFindings + # OpenSearch Serverless (FS-25, FS-26) + - aoss:ListSecurityPolicies + # Macie — PII protection (FS-44) + - macie2:GetMacieSession + # EventBridge / Config (FS-32, FS-34) + - events:ListRules + - config:DescribeConfigRules + # Organizations — SCPs (FS-12) + - organizations:ListPolicies + - organizations:DescribePolicy + Resource: "*" + # CodeBuild role for running assessments CodeBuildRole: Type: AWS::IAM::Role diff --git a/docs/AIMLSecurityAssessment-MappingsTable.csv b/docs/AIMLSecurityAssessment-MappingsTable.csv new file mode 100644 index 0000000..9f16695 --- /dev/null +++ b/docs/AIMLSecurityAssessment-MappingsTable.csv @@ -0,0 +1,16 @@ +Risk Category,Gap Count,Key Missing Focus Areas +Unbounded Consumption,7,"WAF/Shield, rate limiting, token quotas, cost monitoring, input body-size limits" +Excessive Agency,6,"Action boundaries, AgentCore Policy, transaction limits, rate alarms, financial transaction value thresholds, end-user identity propagation" +Supply Chain Vulnerabilities,5,"SCPs, model inventory, onboarding governance, adversarial evaluation, TPRM monitoring" +Training Data & Model Poisoning,5,"Model Monitor, data drift, Model Registry, Feature Store rollback, training data audit trail" +Vector & Embedding Weaknesses,6,"KB least privilege, CloudTrail logging, metadata filtering, multi-tenancy, S3 event notifications" +Non-Compliant Output,5,"Automated Reasoning, RAG controls, human-in-the-loop, compliance-driven guardrails, guardrail trace logging" +Misinformation,4,"KB data quality, source attribution, integrity monitoring with event notifications, sync cadence" +Abusive or Harmful Output,4,"FMEval, user reporting, allowlists, AI Service Cards" +Biased Output,4,"SageMaker Clarify, Bedrock Evaluations with cadence, bias datasets, AI Service Cards" +Sensitive Information Disclosure,5,"CloudWatch log masking, Macie, PII pre-processing pipeline, data classification, end-user identity propagation" +Hallucination,4,"Contextual grounding, Automated Reasoning, disclaimers, RAG" +Prompt Injection,5,"Input sanitization, prompt input validation function, parameterized queries, pen testing, SDK currency" +Improper Output Handling,4,"Output validation, sanitization, encoding, XSS prevention" +Off-Topic & Inappropriate Output,2,"Contextual grounding, topic allowlists" +Out-of-Date Training Data,3,"KB sync cadence, data currency disclaimers, FM version updates and TPRM" diff --git a/docs/SECURITY_CHECKS.md b/docs/SECURITY_CHECKS.md index af04b4f..88f6ee6 100644 --- a/docs/SECURITY_CHECKS.md +++ b/docs/SECURITY_CHECKS.md @@ -344,3 +344,32 @@ Each security check has a unique identifier with a service prefix: - [Amazon Bedrock Security](https://docs.aws.amazon.com/bedrock/latest/userguide/security.html) - [AWS Security Hub SageMaker Controls](https://docs.aws.amazon.com/securityhub/latest/userguide/sagemaker-controls.html) - [AWS Well-Architected Framework - Security Pillar](https://docs.aws.amazon.com/wellarchitected/latest/security-pillar/welcome.html) + +--- + +## Financial Services GenAI Risk Checks (64 additional, 5 upstream extensions) + +These 64 standalone checks (FS-XX) extend the framework with Financial Services +regulatory controls derived from the +[AWS guide for Financial Services risk management of the use of Generative AI (March 2026)](https://d1.awsstatic.com/onedam/marketing-channels/website/public/global-FinServ-ComplianceGuide-GenAIRisks-public.pdf). +An additional 5 FS checks are contributed as extensions to existing SM-07, +SM-22, SM-23, BR-04, and BR-06 (see in-file extension notes). + +The full catalog is split across three companion files for readability: + +- **[`SECURITY_CHECKS_FINSERV_COMMON.md`](./SECURITY_CHECKS_FINSERV_COMMON.md)** — shared + intro, severity rubric, validation note, upstream-overlap table, compliance framework + mapping. +- **[`SECURITY_CHECKS_FINSERV_PART1_INFRA_CONTROLS.md`](./SECURITY_CHECKS_FINSERV_PART1_INFRA_CONTROLS.md)** — FS-01 to FS-26 + (Unbounded Consumption, Excessive Agency, Supply Chain, Training Poisoning, Vector + Weaknesses). +- **[`SECURITY_CHECKS_FINSERV_PART2_GUARDRAILS_CONTENT_SAFETY.md`](./SECURITY_CHECKS_FINSERV_PART2_GUARDRAILS_CONTENT_SAFETY.md)** — FS-27 to FS-46 + (Non-Compliant Output, Misinformation, Abusive/Harmful Output, Biased Output, + Sensitive Information Disclosure). +- **[`SECURITY_CHECKS_FINSERV_PART3_APP_LAYER_AND_GAPS.md`](./SECURITY_CHECKS_FINSERV_PART3_APP_LAYER_AND_GAPS.md)** — FS-47 to FS-69 + (Hallucination, Prompt Injection, Improper Output Handling, Off-Topic Output, + Out-of-Date Training Data, and 6 cross-category material gap checks). + +Compliance framework mapping table is in `SECURITY_CHECKS_FINSERV_COMMON.md` +(SR 11-7, FFIEC CAT, NYDFS 500.06, PCI-DSS 12.3.2, DORA Art.6, MAS TRM 9, +ISO 27001 A.12, ECOA, OWASP LLM Top 10). diff --git a/docs/SECURITY_CHECKS_FINSERV_COMMON.md b/docs/SECURITY_CHECKS_FINSERV_COMMON.md new file mode 100644 index 0000000..23d7d93 --- /dev/null +++ b/docs/SECURITY_CHECKS_FINSERV_COMMON.md @@ -0,0 +1,169 @@ +# FinServ GenAI Risk Checks — Common Reference + +This file contains shared reference material used by all three parts of the FinServ GenAI +security checks. It is not a checks file itself — the 64 standalone checks are split across Parts 1-3 +(5 additional FS checks are merged into upstream SM/BR checks — see the consolidation table below): + +- `SECURITY_CHECKS_FINSERV_PART1_INFRA_CONTROLS.md` — FS-01 to FS-26 (Unbounded, Excessive Agency, Supply Chain, Training Poisoning, Vector Weaknesses); FS-17, FS-18, FS-19, FS-23 merged into upstream +- `SECURITY_CHECKS_FINSERV_PART2_GUARDRAILS_CONTENT_SAFETY.md` — FS-27 to FS-46 (Non-Compliant, Misinformation, Abusive, Biased, Sensitive Info) +- `SECURITY_CHECKS_FINSERV_PART3_APP_LAYER_AND_GAPS.md` — FS-47 to FS-69 (Hallucination, Prompt Injection, Output Handling, Off-Topic, Stale Data, Material Gaps); FS-64 merged into upstream + +Together, Parts 1-3 plus this common file replace the earlier single-file `SECURITY_CHECKS_FINSERV_ADDITION.md`. + +## About the source + +The 69 FS checks are derived from the [AWS guide for Financial Services risk management of the +use of Generative AI (March 2026)](https://d1.awsstatic.com/onedam/marketing-channels/website/public/global-FinServ-ComplianceGuide-GenAIRisks-public.pdf) +(referred to throughout as "the FinServ Guide"). + +Each check includes how it is **detected** (the AWS API calls or configuration inspected) +and how a failure is **remediated** (the specific AWS actions to take). + +## PDF traceability + +The FinServ Guide organises AI-specific risks into **15 categories** (§1.2.1 through §1.2.15 +in the PDF). Every check below is tagged with one of: + +- **[PDF §x.y.z]** — mitigation is explicitly listed in that PDF section's "Mitigations or controls" + table or "Practical guidance" callout. +- **[PDF §x.y.z, extension]** — mitigation is consistent with the PDF's risk description but is + not verbatim in the PDF; included because it is a widely-accepted AWS best practice for the + same risk. These are labelled so reviewers know the provenance. + +## Severity rubric + +| Severity | Criteria | +|---|---| +| **High** | Control whose absence can lead to direct regulatory breach, data exposure, large-scale financial loss, or full bypass of safety guardrails. | +| **Medium** | Control whose absence materially increases the likelihood or impact of a risk category but does not by itself produce a breach. | +| **Low** | Control that reduces residual risk or supports audit/observability but has alternative or compensating controls. | +| **Advisory** | Control the assessment tool cannot fully evaluate via AWS APIs; requires human verification (included for completeness). | + +## Validation note + +Detection and remediation guidance in this document was systematically validated against the +FinServ Guide (March 2026), current AWS documentation, API references, and AWS announcements as +of April 2026. IAM action names were verified against the AWS Service Authorization Reference +for [Amazon Bedrock](https://docs.aws.amazon.com/service-authorization/latest/reference/list_amazonbedrock.html), +[Amazon Bedrock AgentCore](https://docs.aws.amazon.com/service-authorization/latest/reference/list_amazonbedrockagentcore.html), +and [Amazon OpenSearch Serverless](https://docs.aws.amazon.com/service-authorization/latest/reference/list_amazonopensearchserverless.html) +(note: the OpenSearch Serverless IAM prefix is `aoss:`, not `opensearchserverless:` — the latter +is the boto3 client name). +CloudWatch metric namespaces were verified against the service-specific monitoring docs (Bedrock, +Bedrock Agents, Bedrock Guardrails, SageMaker Model Monitor, SageMaker Clarify). CloudTrail +event-type classification (management vs data) for Bedrock API operations was verified against the +[Bedrock CloudTrail integration guide](https://docs.aws.amazon.com/bedrock/latest/userguide/logging-using-cloudtrail.html). +Cost Anomaly Detection monitor-type values were verified against the +[AnomalyMonitor API reference](https://docs.aws.amazon.com/aws-cost-management/latest/APIReference/API_AnomalyMonitor.html). +Where AWS does not prescribe a specific value (e.g., grounding thresholds), this is explicitly +called out as an assessment recommendation rather than an AWS requirement. AWS regional +availability of new features (Automated Reasoning, AgentCore Policy, AWS Security Agent, +cross-account guardrails) evolves rapidly — region lists in Parts 1-3 reflect the state at the +cited announcement date and should be re-verified before audit reliance. + +Add this content to `docs/SECURITY_CHECKS.md` in the forked repository. + +## Contribution workflow + +The FS checks are contributed upstream as a single pull request via a personal GitHub fork +of `aws-samples/sample-aiml-security-assessment`. The full 9-step process — feature-request +GitHub issue, fork + feature branch, local ASH security scan, Conventional Commits, PR, +GitHub Actions verification, reviewer assignment, and (only if needed) Git Defender +exception — lives in [`GIT_WORKFLOW.md`](./GIT_WORKFLOW.md) at the repository root. A +condensed version is in the `Git Workflow` section of [`IMPLEMENTATION_PLAN.md`](./IMPLEMENTATION_PLAN.md). + +Key quality gates before opening the PR (see `GIT_WORKFLOW.md` Step 5): + +1. `ruff check` and `ruff format --check` pass on `functions/security/finserv_assessments/`. +2. `cfn-lint` and `sam validate --lint` pass on the SAM templates. +3. [ASH v3](https://awslabs.github.io/automated-security-helper/) scan + (`ash --source-dir . --fail-on-findings --config-overrides + 'global_settings.severity_threshold=MEDIUM'`) reports zero Critical / High findings, + or suppressions are documented in `.ash/.ash.yaml`. +4. Amazon Code Defender (`git defender scan`) reports no secrets in the staged diff. + +Because `aws-samples` is an OSPO-managed organization, pushes to your personal fork of +`aws-samples/*` are auto-allowed by Code Defender — a Git Defender exception ticket is +**not expected** for this contribution. + +## Relationship to upstream SM/BR/AC checks + +The upstream [sample-aiml-security-assessment](https://github.com/aws-samples/sample-aiml-security-assessment) +framework already provides 52 security checks (SM-01 to SM-25, BR-01 to BR-14, AC-01 to AC-13). +The 69 FS checks in this document are **additive**: they enhance the upstream with FinServ-specific +detection and remediation guidance drawn from the AWS FinServ GenAI Guide (March 2026). A few FS +checks overlap with upstream checks — in those cases, the FS check adds FinServ-specific depth +(e.g., protected-attribute facets, regulatory cadence requirements, denied-topic content for +financial advice). The table below surfaces each overlap with a systematic recommendation based +on five factors: (1) whether the detection target is the same AWS resource/configuration, (2) +whether the FS check adds FinServ-specific regulatory specificity, (3) severity differentiation, +(4) whether a customer would remediate them differently, and (5) PDF-traceability value. + +**Recommendation values:** + +- **Extend upstream** — merge FS detection/remediation detail into the upstream check; do not ship FS as a standalone entry in the final report. Best when both checks target the same resource and the FS content is an enhancement. +- **Keep separate** — ship as a standalone FS check alongside the upstream check. Best when the FS check targets a different AWS resource, has materially different severity, or encodes a FinServ-specific regulatory requirement that would be diluted by merging. + +| FS check | Upstream check | Overlap analysis | Recommendation | +|---|---|---|---| +| FS-17 (Model Monitor Data Quality) | SM-07 (Model Monitor) | Same resource (`sagemaker:ListMonitoringSchedules`); FS-17 adds training-data-drift-specific guidance, exact CloudWatch namespace (`/aws/sagemaker/Endpoints/data-metric`), and `emit_metrics` requirement. | **Extend SM-07** — add FS-17's detection detail (namespace, `emit_metrics`) as a refinement of the existing check | +| FS-18 (Model Drift Detection) | SM-23 (Model Drift Detection) | Same name, same resource, same detection logic (`MonitoringType=ModelQuality`). FS-18 adds PDF §1.2.14 low-entropy classification monitoring as an early-warning poisoning indicator. | **Extend SM-23** — add low-entropy monitoring as a new remediation step on SM-23; do not ship FS-18 separately | +| FS-19 (Model Registry Approval) | SM-08 (Model Registry) / SM-22 (Model Approval Workflow) | SM-22 is conceptually identical. FS-19 specifies exact `ModelApprovalStatus=PendingManualApproval` default and flags auto-approved latest versions. | **Extend SM-22** — add FS-19's detection specificity (flag auto-approved latest versions) to SM-22; do not ship FS-19 separately | +| FS-20 (Feature Store Rollback) | SM-15 (Feature Store Encryption) | Different security properties on the same resource: SM-15 checks encryption; FS-20 checks `OfflineStoreConfig` presence for point-in-time rollback. | **Keep separate** — different security property; no true overlap | +| FS-39 (SageMaker Clarify Bias) | SM-06 (Clarify Usage) | Same resource family but SM-06 is Severity Low and generic ("validates Clarify for bias detection"); FS-39 is Severity High with specific `MonitoringType=ModelBias`, protected-attribute facets (age/gender/race/geography), and specific bias metrics (DPL, DI, DPPL) for FinServ decision models. | **Keep separate** — severity, detection specificity, and FinServ regulatory context (ECOA/Fair Housing) warrant a standalone check | +| FS-41 (SageMaker Clarify Explainability) | SM-06 (Clarify Usage) | Same as FS-39 but for `MonitoringType=ModelExplainability`. FS-41 is Severity High with SHAP analysis for adverse-action-notice use cases. | **Keep separate** — severity and adverse-action-notice regulatory context justify a standalone check | +| FS-22 (KB IAM Least Privilege) | BR-01 (IAM Least Privilege) | BR-01 detects the managed policy `AmazonBedrockFullAccess` on any role. FS-22 inspects role policy documents for wildcard `bedrock:*` affecting KB actions and requires ARN-scoped resource restrictions. | **Keep separate** — different detection logic (managed-policy attachment vs policy-document statement analysis); FS-22 fills a detection gap BR-01 does not cover | +| FS-23 (KB CloudTrail Logging) | BR-06 (CloudTrail Logging) | BR-06 verifies CloudTrail is logging Bedrock API calls generally. FS-23 specifically requires an advanced event selector for `AWS::Bedrock::KnowledgeBase` to capture `Retrieve`/`RetrieveAndGenerate` data events (NOT logged by default). | **Extend BR-06** — add FS-23's data-event-selector requirement as a refinement of the same CloudTrail check | +| FS-25 (OpenSearch Serverless Encryption) | BR-09 (Knowledge Base Encryption) | Different AWS resources: BR-09 checks the Bedrock KB's `kmsKeyArn`; FS-25 checks the underlying AOSS collection's encryption policy (`aoss:ListSecurityPolicies(type=encryption)`). A KB can be CMK-encrypted while its vector store is not. | **Keep separate** — different AWS resources with independent encryption configurations; both needed for defense-in-depth | +| FS-26 (KB VPC Access) | BR-02 (VPC Endpoint Configuration) | BR-02 checks Bedrock VPC endpoints exist. FS-26 checks the AOSS collection's network policy for `AllowFromPublic=true` (whether the vector store itself is internet-reachable). | **Keep separate** — orthogonal controls: Bedrock VPC endpoint vs vector-store network policy | +| FS-27 (Automated Reasoning / Contextual Grounding) | BR-05 (Guardrail Configuration) | BR-05 verifies a guardrail exists and is enforced. FS-27 checks for `automatedReasoningPolicy` or `contextualGroundingPolicy` with specific threshold (≥ 0.7). | **Keep separate** — policy-level guardrail content BR-05 does not evaluate | +| FS-28 (Financial Denied Topics) | BR-05 | BR-05 is existence; FS-28 inspects `topicPolicy.topics` for FinServ-specific denied topics (investment advice, tax advice, guaranteed returns). | **Keep separate** — FinServ denied-topic content is a regulatory-specific requirement not representable as a generic extension | +| FS-36 (Guardrail Content Filters) | BR-05 | FS-36 inspects `contentPolicy.filters` for HATE/VIOLENCE/SEXUAL/INSULTS/MISCONDUCT/PROMPT_ATTACK with strength ≥ MEDIUM. | **Keep separate** — policy-level detection BR-05 does not cover | +| FS-38 (Word Filters and Allowlists) | BR-05 | FS-38 inspects `wordPolicy.words` and `managedWordLists` for FinServ business-term allowlist guidance. | **Keep separate** — advisory business-term allowlist has no upstream equivalent | +| FS-45 (Guardrail PII Filters) | BR-05 | FS-45 inspects `sensitiveInformationPolicy.piiEntities` for 12 specific PII types critical to FinServ (SSN, bank account, SWIFT code, etc.) with `inputAction=BLOCK`/`outputAction=ANONYMIZE`. | **Keep separate** — FinServ-specific PII entity list is a distinct regulatory requirement | +| FS-47 (Grounding Threshold) | BR-05 | FS-47 checks `contextualGroundingPolicy.filters` for `GROUNDING` filter with threshold ≥ 0.7. | **Keep separate** — threshold-value check BR-05 does not perform | +| FS-50 (Relevance Grounding Filters) | BR-05 | Same as FS-47 but for `RELEVANCE` filter type. | **Keep separate** — distinct filter type | +| FS-51 (Prompt Attack Filters) | BR-05 | FS-51 checks `PROMPT_ATTACK` filter in Standard tier with input-tagging requirement and `inputStrength=HIGH`. | **Keep separate** — Standard-tier cross-region-inference opt-in and input-tagging nuance warrant standalone guidance | +| FS-59 (Guardrail Topic Allowlist) | BR-05 | FS-59 checks `topicPolicy.topics` exist to block off-topic conversations (politics, entertainment, medical advice). | **Keep separate** — off-topic content restrictions are distinct from FS-28's regulated-advice restrictions; different PDF section (§1.2.2 vs §1.2.1) | +| FS-64 (Guardrail Trace Logging) | BR-04 (Model Invocation Logging) | BR-04 verifies invocation logging is enabled. FS-64 additionally verifies the log output captures `guardrailTrace` with `action`/`inputAssessments`/`outputAssessments` and adds NYDFS/SR 11-7 retention guidance. | **Extend BR-04** — add guardrail-trace verification as a refinement of the same invocation-logging check; retention guidance can be a remediation note | + +### Summary of consolidation recommendations + +- **Extend upstream (5 FS checks merged into 5 upstream checks):** FS-17 → SM-07; FS-18 → SM-23; FS-19 → SM-22; FS-23 → BR-06; FS-64 → BR-04. These checks are replaced by upstream-extension notes in Parts 1 and 3 and are removed from `finserv_assessments/app.py`. +- **Keep separate (64 FS checks):** All other FS checks ship as standalone entries. This includes FS-20, FS-22, FS-25, FS-26, FS-39, FS-41, all Guardrail-policy-level checks (FS-27, FS-28, FS-36, FS-38, FS-45, FS-47, FS-50, FS-51, FS-59), and all FS checks that have no upstream overlap at all. + +After consolidation the combined framework contains **52 upstream + 64 FS = 116 distinct checks** (down from 52 + 69 = 121 before merging). The consolidation reduces duplication without losing FinServ-specific regulatory depth. + +Add this content to `docs/SECURITY_CHECKS.md` in the forked repository. + + +--- + +## Compliance Framework Mapping + +> **Disclaimer:** The mappings below are **preliminary and illustrative**, provided by the +> authors of this assessment to help FSI teams start conversations with their MRM/compliance +> colleagues. They are **not** authoritative AWS compliance guidance and they have **not** been +> reviewed by AWS Security Assurance Services, external auditors, or the regulators whose +> frameworks are named. Each firm should have its own MRM, Legal, and Compliance teams +> validate these mappings against the firm's specific interpretation of each framework before +> relying on them as audit evidence. + +Each FS check maps to one or more FinServ regulatory frameworks (preliminary mapping): + +| Framework | Description | Relevant Checks | +|-----------|-------------|-----------------| +| SR 11-7 | Federal Reserve Model Risk Management Guidance | FS-07, FS-10, FS-12 to FS-16, FS-20, FS-21, FS-27 to FS-33, FS-34, FS-39 to FS-42, FS-66, FS-67 | +| FFIEC CAT | Cybersecurity Assessment Tool | All FS checks | +| NYDFS 500 | NY Cybersecurity Regulation | FS-22, FS-43 to FS-46, FS-51 to FS-54, FS-66 | +| PCI-DSS | Payment Card Industry Data Security Standard | FS-22, FS-25, FS-26, FS-43 to FS-46, FS-53, FS-56, FS-67, FS-68 | +| DORA | EU Digital Operational Resilience Act | FS-01 to FS-06, FS-08, FS-11, FS-54, FS-65, FS-68 | +| MAS TRM 9 | Monetary Authority of Singapore Technology Risk Management | FS-07 to FS-11, FS-15, FS-27 to FS-30, FS-32, FS-37, FS-39 to FS-42, FS-66, FS-67 | +| ISO 27001 | Information Security Management | FS-13, FS-14, FS-16, FS-21, FS-33, FS-46, FS-52, FS-63, FS-65 | +| ECOA/Fair Housing | Equal Credit Opportunity Act (US) | FS-39 to FS-42 (advisory — applicability depends on whether the model is used for ECOA-covered credit decisions; confirm with your compliance team) | +| OWASP LLM Top 10 | OWASP LLM Application Security | FS-51 to FS-58, FS-68, FS-69 | + +> **FS-34 note:** FS-34 (TPRM for FM Providers) is listed above under SR 11-7. Although the +> check appears in the Misinformation section of Part 2 for numbering continuity, its +> primary PDF source is §1.2.12 Supply Chain, which is the lens MRM and TPRM teams will +> evaluate it through. diff --git a/docs/SECURITY_CHECKS_FINSERV_PART1_INFRA_CONTROLS.md b/docs/SECURITY_CHECKS_FINSERV_PART1_INFRA_CONTROLS.md new file mode 100644 index 0000000..8e06b53 --- /dev/null +++ b/docs/SECURITY_CHECKS_FINSERV_PART1_INFRA_CONTROLS.md @@ -0,0 +1,401 @@ +# FinServ GenAI Risk Checks — Part 1: Infrastructure & Resource Controls (FS-01 to FS-26) + +This is **Part 1 of 3** of the FinServ GenAI security checks derived from the +[AWS guide for Financial Services risk management of the use of Generative AI (March 2026)](https://d1.awsstatic.com/onedam/marketing-channels/website/public/global-FinServ-ComplianceGuide-GenAIRisks-public.pdf) +(referred to throughout as "the FinServ Guide"). + +This part covers **22 standalone checks** across 5 PDF risk categories (FS-17, FS-18, FS-19, and FS-23 are merged into upstream checks — see extension notes in each section): + +- **Unbounded Consumption** (FS-01 to FS-06) — §1.2.11 +- **Excessive Agency** (FS-07 to FS-11) — §1.2.9 +- **Supply Chain Vulnerabilities** (FS-12 to FS-16) — §1.2.12 +- **Training Data & Model Poisoning** (FS-17 to FS-21) — §1.2.14 — *FS-17, FS-18, FS-19 merged into upstream* +- **Vector & Embedding Weaknesses** (FS-22 to FS-26) — §1.2.15 — *FS-23 merged into upstream* + +**Companion files:** + +- `SECURITY_CHECKS_FINSERV_PART2_GUARDRAILS_CONTENT_SAFETY.md` — FS-27 to FS-46 (Non-Compliant, Misinformation, Abusive, Biased, Sensitive Info) +- `SECURITY_CHECKS_FINSERV_PART3_APP_LAYER_AND_GAPS.md` — FS-47 to FS-69 (Hallucination, Prompt Injection, Output Handling, Off-Topic, Stale Data, Material Gaps) +- `SECURITY_CHECKS_FINSERV_COMMON.md` — shared intro, severity rubric, validation note, upstream-overlap table + +Each check includes how it is **detected** (the AWS API calls or configuration inspected) +and how a failure is **remediated** (the specific AWS actions to take). + +See `SECURITY_CHECKS_FINSERV_COMMON.md` for: + +- PDF traceability conventions (`[PDF §x.y.z]` vs `[PDF §x.y.z, extension]`) +- Severity rubric (High / Medium / Low / Advisory) +- Validation note and AWS service authorization references +- Relationship to upstream SM/BR/AC checks and consolidation recommendations + +--- + +## FinServ GenAI Risk Checks — Part 1 content + +### Unbounded Consumption (FS-01 to FS-06) + +> **PDF source:** §1.2.11 Unbounded consumption. PDF-listed mitigations: (a) AWS WAF and Shield +> Advanced for LLM APIs; (b) maximum input length limits; (c) rate limits/quotas on APIs +> accessing LLMs; (d) cost-and-usage tracking for generative AI. Practical guidance in the PDF +> also calls out `max_tokens` optimisation and CloudWatch metrics for token usage. + +#### FS-01 — WAF and Shield Protection + +| Field | Detail | +|-------|--------| +| Severity | High | +| PDF ref | [PDF §1.2.11] — "Protect your LLM APIs and Amazon Bedrock-hosted LLMs by using AWS WAF and AWS Shield Advanced." Also covers: "To protect your API endpoints, set maximum length limits for input requests when you use large language models (LLMs) directly or through Amazon Bedrock." | +| Description | Verifies AWS WAF Web ACLs and Shield Advanced protect GenAI API endpoints, and verifies the Web ACL enforces both rate-based limits and body-size (input-length) constraints. | +| Detection | Calls `shield:DescribeSubscription` to check Shield Advanced is active. Calls `wafv2:ListWebACLs(Scope=REGIONAL)` in each region where GenAI API endpoints run to verify at least one regional Web ACL exists (covers API Gateway, ALB, AppSync). **Additionally** calls `wafv2:ListWebACLs(Scope=CLOUDFRONT)` in `us-east-1` to detect Web ACLs protecting CloudFront distributions fronting GenAI workloads — CLOUDFRONT-scope Web ACLs must be created and queried in `us-east-1` per the [WAF resources documentation](https://docs.aws.amazon.com/waf/latest/developerguide/how-aws-waf-works-resources.html). For each Web ACL found, calls `wafv2:GetWebACL` and inspects the `Rules` array for: (a) at least one `RateBasedStatement` (rate limiting) and (b) at least one `SizeConstraintStatement` with `FieldToMatch=Body` or `FieldToMatch=JsonBody` (input-size limit — this implements PDF §1.2.11 mitigation "set maximum length limits for input requests when you use large language models (LLMs) directly or through Amazon Bedrock"). Flags accounts with no Web ACL in either scope, a Web ACL with no rate-based rule, a Web ACL with no body size-constraint rule, or where Shield Advanced is inactive. | +| Remediation | 1. Subscribe to AWS Shield Advanced via the Shield console. 2. Create a WAF Web ACL with both (a) a rate-based rule (e.g., 1 000 req / 5 min per IP) and (b) a `SizeConstraintStatement` that blocks requests where `FieldToMatch=Body` (or `JsonBody` for JSON APIs) exceeds your LLM's expected maximum input size — for example, `ComparisonOperator=GT, Size=100000` (100 KB) — use `Scope=REGIONAL` for API Gateway/ALB/AppSync resources, or `Scope=CLOUDFRONT` (created in `us-east-1`) for CloudFront distributions fronting Bedrock. The body size-constraint rule directly implements the PDF §1.2.11 mitigation "set maximum length limits for input requests when you use large language models (LLMs) directly or through Amazon Bedrock" and prevents large-prompt token-exhaustion attacks before they reach Bedrock. 3. Associate the ACL with the fronting resource (API Gateway stage, ALB, or CloudFront distribution). 4. Add AWS Managed Rules (e.g., `AWSManagedRulesCommonRuleSet`, which includes additional size checks). 5. For CloudFront-fronted workloads, register the distribution with Shield Advanced via `shield:CreateProtection` to unlock automatic application-layer DDoS mitigation. 6. For API Gateway REST APIs, also note the service's own payload-size quota: the default is 10 MB per request (see [API Gateway quotas](https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-execution-service-limits-table.html)); use a request validator or Lambda authorizer for sub-10 MB limits where WAF size constraints are unsuitable. | +| Reference | [Shield Advanced](https://docs.aws.amazon.com/waf/latest/developerguide/shield-chapter.html), [WAF](https://docs.aws.amazon.com/waf/latest/developerguide/waf-chapter.html), [WAF Size Constraint Rule](https://docs.aws.amazon.com/waf/latest/developerguide/waf-rule-statement-type-size-constraint-match.html), [API Gateway Quotas](https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-execution-service-limits-table.html) | + +#### FS-02 — API Gateway Rate Limiting + +| Field | Detail | +|-------|--------| +| Severity | High | +| PDF ref | [PDF §1.2.11] — "protect your API endpoints by implementing rate limits and quotas for APIs that access large language models (LLMs)". | +| Description | Checks API Gateway usage plans enforce throttling on GenAI endpoints. | +| Detection | Calls `apigateway:GetUsagePlans` and inspects each plan's `throttle.rateLimit` and `throttle.burstLimit`. Flags plans where either is zero or absent. | +| Remediation | 1. Create or update usage plans with `rateLimit` and `burstLimit` values appropriate for your traffic. 2. Associate plans with API stages serving Bedrock. 3. Issue per-consumer API keys with individual quotas. | +| Reference | [API Gateway Throttling](https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-request-throttling.html) | + +#### FS-03 — Bedrock Token Quota Review + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.11, extension] — PDF practical guidance notes "Bedrock has default quota on model inference based on token usage" and recommends optimising `max_tokens`. Quota review as an operational control is an extension aligned with this guidance. | +| Description | Verifies Bedrock TPM/RPM quotas have been reviewed and set appropriately. | +| Detection | Calls `service-quotas:ListServiceQuotas(ServiceCode=bedrock)` for applied quotas and `ListAWSDefaultServiceQuotas` for defaults, then compares each adjustable quota's `Value` against the default `Value`. Flags accounts where every quota equals the service default (indicating no quota review or increase has been requested). | +| Remediation | 1. Review current quotas in the Service Quotas console. 2. Request increases aligned with expected peak load via `service-quotas:RequestServiceQuotaIncrease`. 3. Implement client-side token counting and pre-flight quota checks. 4. Use Bedrock cross-region inference profiles to distribute load — note that cross-region inference routes requests across destination regions automatically with no additional cost, but requires the invoked model to be available in the destination regions defined in the inference profile. | +| Reference | [Bedrock Quotas](https://docs.aws.amazon.com/bedrock/latest/userguide/quotas.html) | + +#### FS-04 — Cost Anomaly Detection + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.11] — "Track, allocate, and manage your costs and usage for generative AI." | +| Description | Checks AWS Cost Anomaly Detection monitors cover Bedrock/SageMaker. | +| Detection | Calls `ce:GetAnomalyMonitors` and inspects each monitor. AWS Cost Anomaly Detection supports exactly two `MonitorType` values per the [AnomalyMonitor API](https://docs.aws.amazon.com/aws-cost-management/latest/APIReference/API_AnomalyMonitor.html): `DIMENSIONAL` (AWS-managed, where `MonitorDimension` is one of `SERVICE`, `LINKED_ACCOUNT`, `TAG`, or `COST_CATEGORY`) and `CUSTOM` (customer-managed, scoped via `MonitorSpecification` to specific values). For `DIMENSIONAL` monitors, checks `MonitorDimension=SERVICE` (the AWS-managed "AWS services" monitor that automatically covers all services including Bedrock and SageMaker — the recommended default). For `CUSTOM` monitors, inspects `MonitorSpecification` for references to Bedrock or SageMaker. Flags accounts with no monitors, or with only narrowly-scoped monitors that would not detect Bedrock cost anomalies (e.g., `DIMENSIONAL` with `MonitorDimension=LINKED_ACCOUNT` only). | +| Remediation | 1. Create an AWS-managed `DIMENSIONAL` monitor with `MonitorDimension=SERVICE` for comprehensive coverage across all AWS services (the recommended default — in the console this appears as "AWS services" under "Managed by AWS"). For narrower scope, add a `CUSTOM` monitor using `MonitorSpecification` with a `Dimensions` expression scoped to specific service values (e.g., `{"Dimensions": {"Key": "SERVICE", "Values": ["Amazon Bedrock", "Amazon SageMaker"]}}`) — note that for `CUSTOM` monitors you use `MonitorSpecification`, not `MonitorDimension`. 2. Configure alert subscriptions (SNS/email) for anomalies above threshold. 3. Set daily spend budgets with AWS Budgets as a secondary control. 4. Enable Bedrock IAM principal cost allocation: tag IAM users/roles with team or cost-center attributes, activate them as cost allocation tags in the Billing and Cost Management console, and include caller identity data in CUR 2.0 exports for per-user/per-team Bedrock spend attribution. | +| Reference | [Cost Anomaly Detection](https://docs.aws.amazon.com/cost-management/latest/userguide/getting-started-ad.html), [Bedrock IAM Cost Allocation](https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/iam-principal-cost-allocation.html) | + +#### FS-05 — CloudWatch Token Usage Alarms + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.11] — PDF practical guidance cites CloudWatch metrics for token usage; alarms operationalise that guidance. | +| Description | Verifies CloudWatch alarms exist for Bedrock throttling and token metrics. | +| Detection | Paginates `cloudwatch:DescribeAlarms(AlarmTypes=MetricAlarm)` and filters for alarms in the `AWS/Bedrock` namespace or with "bedrock" in the alarm name. Separately counts throttle-specific alarms. | +| Remediation | 1. Create alarms for `AWS/Bedrock InvocationThrottles` (threshold > 0). 2. Create alarms for `AWS/Bedrock EstimatedTPMQuotaUsage` to track approach to token quota limits, and separately on `InputTokenCount` + `OutputTokenCount` (sum via CloudWatch metric math) for absolute token consumption. Note: `TokensProcessed` is not a valid Bedrock metric — the correct runtime metrics are `InputTokenCount`, `OutputTokenCount`, `InvocationThrottles`, `EstimatedTPMQuotaUsage`, `Invocations`, `InvocationLatency`, `TimeToFirstToken`. 3. Publish custom application-level token counters via Embedded Metric Format (EMF) if you need per-tenant or per-feature attribution. 4. Attach SNS actions to all alarms. | +| Reference | [Bedrock CloudWatch Metrics](https://docs.aws.amazon.com/bedrock/latest/userguide/monitoring.html) | + +#### FS-06 — AWS Budgets AI/ML Spend + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.11] — "Track, allocate, and manage your costs and usage for generative AI." | +| Description | Checks AWS Budgets are configured with alerts for AI/ML service spend. | +| Detection | Calls `budgets:DescribeBudgets` and inspects each budget's `FilterExpression` (the current field) and `CostFilters` (deprecated but may still be populated on older budgets) for references to "bedrock" or "sagemaker". Note: `CostFilters` is marked deprecated in the AWS Budgets API — new budgets use `FilterExpression` with an `Expression` object; the detection should check both fields to cover both old and new budgets. | +| Remediation | 1. Create cost budgets for Bedrock and SageMaker with 80 %/100 % alert thresholds. 2. Add SNS notifications to on-call channels. 3. Consider budget actions to apply IAM deny policies when thresholds are breached. 4. Enable Bedrock IAM principal cost allocation to attribute inference costs to specific IAM users/roles via Cost Explorer and CUR 2.0 — tag IAM principals with team or cost-center attributes and activate them as cost allocation tags. | +| Reference | [AWS Budgets](https://docs.aws.amazon.com/cost-management/latest/userguide/budgets-managing-costs.html), [Bedrock IAM Cost Allocation](https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/iam-principal-cost-allocation.html) | + +### Excessive Agency (FS-07 to FS-11) + +> **PDF source:** §1.2.9 Excessive agency. PDF-listed mitigations: (a) Amazon Bedrock AgentCore +> for managing complex tasks; (b) least-privilege permissions on plugins; (c) human-in-the-loop +> output validation; (d) explicit action boundaries in agent configuration (AgentCore Policy); +> (e) audit logging of agent actions with reasoning chain (AgentCore Observability); +> (f) transaction-value thresholds on agent tool calls; (g) monitoring agent call rates with +> alarms (AgentCore Evaluations). Mitigation (e) is covered by the expanded FS-08 check, which +> now verifies both AgentCore Policy Engine and AgentCore Observability are configured. + +#### FS-07 — Agent Action Boundaries + +| Field | Detail | +|-------|--------| +| Severity | High | +| PDF ref | [PDF §1.2.9] — "grant only the minimum permissions required"; "Define and enforce explicit action boundaries in the agent configuration". | +| Description | Verifies Bedrock agent execution roles have no wildcard sensitive actions (iam:\*, s3:\*, ec2:\*, lambda:\*, \*). | +| Detection | Calls `ListAgents` and `GetAgent` (via the `bedrock-agent` boto3 client; IAM actions are `bedrock:ListAgents` and `bedrock:GetAgent`) to retrieve each agent's `agentResourceRoleArn`. Resolves the role name and inspects attached and inline policy documents from the permissions cache for wildcard Allow statements. | +| Remediation | 1. Replace wildcard actions with the specific actions the agent needs. 2. Apply IAM permission boundaries to agent execution roles. 3. Use resource-level conditions to restrict to specific ARNs. 4. Implement human-in-the-loop approval for high-impact actions. 5. For agents deployed in a VPC, use **AWS Network Firewall** with domain-based filtering to control which external domains agents can reach — this provides a network-layer boundary that limits agent tool access to approved endpoints regardless of IAM permissions. | +| Reference | [Bedrock Agent Permissions](https://docs.aws.amazon.com/bedrock/latest/userguide/agents-permissions.html), [Control Agent Domain Access](https://aws.amazon.com/blogs/machine-learning/control-which-domains-your-ai-agents-can-access/) | + +#### FS-08 — AgentCore Policy Engine and Observability + +| Field | Detail | +|-------|--------| +| Severity | High | +| PDF ref | [PDF §1.2.9] — "Use Amazon Bedrock AgentCore to manage complex tasks and connect securely"; "Define and enforce explicit action boundaries"; **"Implement audit logging of all actions taken by AI agents, including the reasoning chain that led to each action."** (The audit-logging mitigation's PDF reference is "Observe your agent applications on Amazon Bedrock AgentCore Observability.") | +| Description | Checks AgentCore Gateways have a Policy Engine attached to authorize agent-to-tool interactions, verifies AgentCore Runtimes have an inbound authorizer configured, and verifies AgentCore Observability is enabled so agent reasoning chains and tool calls are auditable. | +| Detection | (a) Calls `ListGateways` and `GetGateway` (via the `bedrock-agentcore-control` boto3 client; IAM actions are `bedrock-agentcore:ListGateways` and `bedrock-agentcore:GetGateway`); inspects `policyEngineConfiguration.arn` and `policyEngineConfiguration.mode` (must be `ENFORCE` for production). (b) Calls `ListAgentRuntimes` (IAM action `bedrock-agentcore:ListAgentRuntimes`) and inspects each runtime's `authorizerConfiguration.customJWTAuthorizer` for inbound auth. (c) Verifies **AgentCore Observability** is enabled by (i) checking that CloudWatch Transaction Search is on via `xray:GetTraceSegmentDestination` (destination should be `CloudWatchLogs`) and that the X-Ray → CloudWatch Logs resource policy is in place via `logs:GetResourcePolicy`, and (ii) calling `logs:DescribeDeliveries` / `logs:DescribeDeliverySources` for AgentCore resource sources (runtime, memory, gateway, built-in tools, identity) — flags runtimes/gateways with no log delivery configured. For memory resources, additionally checks that tracing was enabled at memory creation time. Flags gateways without a Policy Engine in `ENFORCE` mode, runtimes without an authorizer, or accounts where Transaction Search is not enabled or no delivery exists for AgentCore resources. | +| Remediation | 1. Configure a Policy Engine: create via `CreatePolicyEngine` (IAM action `bedrock-agentcore:CreatePolicyEngine`), then author Cedar policies using one of three methods: (a) write Cedar directly for fine-grained control via `CreatePolicy` (IAM action `bedrock-agentcore:CreatePolicy`), (b) use the form-based console UI, or (c) generate Cedar from natural language descriptions (natural-language-to-Cedar is a documented capability in the GA announcement; verify the exact IAM action name against the current [AgentCore Service Authorization Reference](https://docs.aws.amazon.com/service-authorization/latest/reference/list_amazonbedrockagentcore.html) before writing IAM policies for it). Policy in AgentCore went GA on March 3, 2026 in thirteen AWS regions (US East N. Virginia, US East Ohio, US West Oregon, Asia Pacific Mumbai/Seoul/Singapore/Sydney/Tokyo, Europe Frankfurt/Ireland/London/Paris/Stockholm) — verify current regional availability on the [launch announcement](https://aws.amazon.com/about-aws/whats-new/2026/03/policy-amazon-bedrock-agentcore-generally-available/) before audit reliance. 2. Attach the Policy Engine to each Gateway by specifying the Policy Engine ARN in the `policyEngineConfiguration` field during `CreateGateway`, or attach later via `UpdateGateway`. 3. Start in `LOG_ONLY` mode — the policy engine evaluates actions and logs whether they would be allowed or denied without enforcing the decision — then switch to `ENFORCE` mode once confident. 4. Configure a JWT inbound authorizer on each Runtime with discovery URL, allowed audiences, and allowed clients. 5. **Enable AgentCore Observability** so agent reasoning chains are captured (directly addresses the PDF §1.2.9 audit-logging mitigation): (a) one-time enable CloudWatch Transaction Search — console path **CloudWatch → Application Signals (APM) → Transaction search → Enable Transaction Search**, or CLI: `aws xray update-trace-segment-destination --destination CloudWatchLogs` plus a `logs:PutResourcePolicy` granting `xray.amazonaws.com` permission to `logs:PutLogEvents` on `aws/spans:*` and `/aws/application-signals/data:*`; (b) configure log delivery for AgentCore runtime, memory, gateway, built-in tools, and identity resources via `logs:PutDeliverySource` + `logs:PutDeliveryDestination` + `logs:CreateDelivery` (CloudWatch Logs / S3 / Firehose destinations supported; note the write APIs use `Put*` for source and destination but `Create*` for the delivery pairing); (c) enable tracing at memory creation. For traditional Bedrock Agents (non-AgentCore), set `enableTrace=true` on `InvokeAgent` calls to receive the reasoning-chain trace in the response. | +| Reference | [Policy in AgentCore](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/policy.html), [Inbound JWT Authorizer](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/inbound-jwt-authorizer.html), [AgentCore Observability Configuration](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/observability-configure.html), [Bedrock Agent Trace View](https://docs.aws.amazon.com/bedrock/latest/userguide/trace-view.html) | + +#### FS-09 — Agent Transaction Limits + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.9, extension] — Lambda reserved concurrency is not named in the PDF, but it directly implements the PDF mitigation "Monitor agent call rates and alarm upon exceeding defined thresholds" by capping execution parallelism. | +| Description | Verifies agent Lambda functions have reserved concurrency limits to cap execution parallelism. | +| Detection | Calls `lambda:ListFunctions` and filters for functions with agent-related naming patterns. For each, calls `lambda:GetFunctionConcurrency` and flags functions with no reserved concurrency set. | +| Remediation | 1. Set reserved concurrency on each agent action-group Lambda (e.g., 10–50 depending on expected load). 2. Add CloudWatch alarms for `Throttles` metric on these functions. 3. Consider Step Functions execution limits as an additional control. | +| Reference | [Lambda Reserved Concurrency](https://docs.aws.amazon.com/lambda/latest/dg/configuration-concurrency.html) | + +#### FS-10 — Human-in-the-Loop Approval + +| Field | Detail | +|-------|--------| +| Severity | High | +| PDF ref | [PDF §1.2.9, §1.2.1, §1.2.2, §1.2.3, §1.2.7, §1.2.10] — "For internal AI systems, validate outputs with human review before business use (human-in-the-loop)." HITL is referenced in six separate PDF risk sections. | +| Description | Checks Step Functions workflows have human approval steps for high-risk agent actions. | +| Detection | Calls `stepfunctions:ListStateMachines` and filters for agent/GenAI-related names. Retrieves each definition via `stepfunctions:DescribeStateMachine` and parses the ASL JSON for task states with `.waitForTaskToken` or callback patterns indicating human approval gates. | +| Remediation | 1. Add a callback-pattern task state in your Step Functions workflow before any high-risk action (financial transactions, data modifications, external communications). 2. Route the approval token to a human reviewer via SNS/SQS/Slack. 3. Set a `HeartbeatSeconds` timeout so stale approvals expire. 4. Enable **user confirmation on Bedrock Agent action groups** for inline approval — when configured, the agent returns a confirmation prompt in the `returnControl.invocationInputs` field of the `InvokeAgent` response (alongside `invocationType` and a unique `invocationId`); the client displays the prompt, collects confirm/deny, and returns the user's decision via `sessionState.returnControlInvocationResults` (with `confirmationState` on each `apiResult`/`functionResult`) in the next `InvokeAgent` request (there is no standalone `GetUserConfirmation` API). | +| Reference | [Step Functions Callback Pattern](https://docs.aws.amazon.com/step-functions/latest/dg/connect-to-resource.html#connect-wait-token), [Bedrock Agent User Confirmation](https://docs.aws.amazon.com/bedrock/latest/userguide/agents-userconfirmation.html) | + +#### FS-11 — Agent Rate Alarms + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.9] — "Monitor agent call rates and alarm upon exceeding defined thresholds." | +| Description | Verifies CloudWatch alarms exist for agent invocation rates. | +| Detection | Paginates `cloudwatch:DescribeAlarms` and filters for alarms referencing "agent" in the alarm name or targeting `AWS/Bedrock/Agents` agent-related metrics (such as `InvocationCount` or `InvocationThrottles` with the `Operation, AgentAliasArn, ModelId` dimension combination). | +| Remediation | 1. Create CloudWatch alarms on the `AWS/Bedrock/Agents` namespace for `InvocationCount` and `InvocationThrottles`. Per AWS docs, the available dimensions are: `Operation` alone; `Operation, ModelId`; or `Operation, AgentAliasArn, ModelId` — use the `Operation, AgentAliasArn, ModelId` combination to scope alarms to a specific agent alias. 2. Set thresholds based on expected peak agent call rates, established via CloudWatch metric math on historical `InvocationCount` data. 3. Attach SNS actions for on-call notification. 4. Use **AgentCore Evaluations** (GA March 2026, available in 9 AWS regions — verify current regional availability on the [GA announcement](https://aws.amazon.com/about-aws/whats-new/2026/03/agentcore-evaluations-generally-available/)) to monitor agent *quality* alongside rate-based alarms: online evaluation continuously scores production traffic against 13 built-in evaluators (response quality, safety, task completion, tool usage), and on-demand evaluation supports regression testing. | +| Reference | [Bedrock Agents CloudWatch Metrics](https://docs.aws.amazon.com/bedrock/latest/userguide/monitoring-agents-cw-metrics.html), [AgentCore Evaluation Types](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/evaluations-types.html) | + +### Supply Chain Vulnerabilities (FS-12 to FS-16) + +> **PDF source:** §1.2.12 Supply chain vulnerabilities. PDF-listed mitigations: +> (a) control access to serverless and marketplace models (IAM policies, SCPs); +> (b) model onboarding process — EULA review, procurement, security/compliance review, +> MRM assessment, documentation, stakeholder approvals; +> (c) update TPRM to continuously monitor model providers — vendor security advisories, +> deprecation notices, T&C changes; +> (d) maintain a model inventory recording provenance, version, license terms, and risk +> assessment status; +> (e) use Bedrock Evaluations against attack test cases (practical guidance); +> (f) allow-list approved models via SCP (practical guidance). + +#### FS-12 — SCP Model Access Restrictions + +| Field | Detail | +|-------|--------| +| Severity | High | +| PDF ref | [PDF §1.2.12 — Practical guidance] — "Implement an allow-list of models using a Service Control Policy (SCP) for your AWS organization." | +| Description | Checks SCPs restrict Bedrock model access to approved models only. | +| Detection | Calls `organizations:ListPolicies(Filter=SERVICE_CONTROL_POLICY)` and inspects each SCP document for Deny statements on `bedrock:InvokeModel*` with `StringNotEquals` conditions on `bedrock:ModelId`. Flags if no SCP restricts model access. | +| Remediation | 1. Create an SCP that denies `bedrock:InvokeModel*` except for an explicit allowlist of approved model ARNs. 2. Attach the SCP to the OU containing GenAI workload accounts. 3. For multi-account guardrail enforcement, use the Bedrock cross-account safeguards feature (GA April 3, 2026, available in all AWS commercial and GovCloud regions where Bedrock Guardrails is supported): enable the Amazon Bedrock policy type in AWS Organizations, create a guardrail in the management account, create a versioned guardrail, optionally attach a resource-based policy granting `bedrock:ApplyGuardrail` to member accounts for cross-account access, then create and attach an AWS Organizations Bedrock policy referencing the guardrail ARN and version to the target OUs or accounts. This automatically enforces content filters, denied topics, word filters, sensitive information filters, and contextual grounding checks across all member accounts for every model invocation — no application code changes required. **Important limitation:** Automated Reasoning checks are **not supported** with cross-account safeguards — omit Automated Reasoning policies from any guardrail used for org-level enforcement. If you rely on AR (see FS-27), you must configure AR guardrails separately at the application or account level. 4. Test with both allowed and denied model IDs. | +| Reference | [Managing Access in AWS Organizations](https://docs.aws.amazon.com/organizations/latest/userguide/orgs_manage_policies_scps.html), [Bedrock Cross-Account Guardrails](https://aws.amazon.com/blogs/aws/amazon-bedrock-guardrails-supports-cross-account-safeguards-with-centralized-control-and-management/) | + +#### FS-13 — Model Inventory Tagging + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.12] — "Maintain a model inventory that records the provenance, version, license terms, and risk assessment status of all models in use across the organization." | +| Description | Verifies models are tagged with provenance metadata (source, version, approval-date). | +| Detection | Calls `bedrock:ListFoundationModels` and `bedrock:ListCustomModels`. For custom models, calls `bedrock:ListTagsForResource` and checks for required tag keys: `model-source`, `model-version`, `approval-date`, `risk-tier`. | +| Remediation | 1. Define a mandatory tagging policy for all AI/ML models. 2. Tag each custom model with provenance metadata. 3. Create an AWS Config rule (`required-tags`) to enforce the tagging policy. 4. For foundation models, maintain an external inventory spreadsheet or CMDB entry. | +| Reference | [Bedrock Tagging](https://docs.aws.amazon.com/bedrock/latest/userguide/tagging.html) | + +#### FS-14 — Model Onboarding Governance + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.12] — "To onboard a model, follow these steps: Review EULA, Complete procurement, Follow security and compliance procedures, Assess MRM requirements, Document findings, Get necessary approvals from stakeholders." | +| Description | Checks AWS Config rules enforce model onboarding governance (EULA review, MRM assessment, stakeholder approval). | +| Detection | Calls `config:DescribeConfigRules` and searches for rules targeting `AWS::Bedrock::*` resources or custom rules with "model" or "onboarding" in the name. | +| Remediation | 1. Create a custom AWS Config rule that checks new Bedrock custom models have required tags (approval-date, risk-tier, eula-reviewed). 2. Document the model onboarding process: EULA review → procurement → security/compliance review → MRM assessment → stakeholder sign-off. 3. Store approval artifacts in a versioned S3 bucket. | +| Reference | [AWS Config Custom Rules](https://docs.aws.amazon.com/config/latest/developerguide/evaluate-config_develop-rules.html) | + +#### FS-15 — Adversarial Model Evaluation + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.12 — Practical guidance] — "Amazon Bedrock Evaluations can help to evaluate models against specific types of attacks by automating your test cases, scoring, reporting and to enable comparison of different models." | +| Description | Verifies Bedrock evaluation jobs include adversarial test datasets. | +| Detection | Calls `bedrock:ListEvaluationJobs` and inspects each job's configuration for evaluation datasets. Flags if no evaluation jobs exist or if none reference adversarial/red-team test data. | +| Remediation | 1. Create a Bedrock model evaluation job with adversarial prompt datasets (prompt injection attempts, jailbreak sequences, harmful content probes). 2. Include both automated metrics and human evaluation. 3. Run evaluations before production deployment and after model updates. 4. Store results for audit. | +| Reference | [Bedrock Model Evaluation](https://docs.aws.amazon.com/bedrock/latest/userguide/evaluation.html) | + +#### FS-16 — ECR Image Scanning + +| Field | Detail | +|-------|--------| +| Severity | High | +| PDF ref | [PDF §1.2.12, extension] — ECR image scanning is not named in the PDF, but directly mitigates the PDF's listed risk "Third-party package vulnerabilities" in LLM supply chains. Included for completeness of the supply-chain risk category. | +| Description | Checks ECR repositories have scan-on-push enabled for supply chain security of model containers. | +| Detection | Calls `ecr:DescribeRepositories` and for each repository checks `imageScanningConfiguration.scanOnPush`. Also checks whether Amazon Inspector ECR scanning is enabled via `inspector2:BatchGetAccountStatus`. Flags repositories relying solely on basic scanning or with no scanning configured. | +| Remediation | 1. Enable **enhanced scanning** via Amazon Inspector (the current best practice) — Inspector provides continuous vulnerability monitoring, re-scanning images automatically when new CVEs are published, and covers both OS and programming language package vulnerabilities. This requires two steps: (a) enable Inspector ECR scanning at the account level — `aws inspector2 enable --account-ids --resource-types ECR`; (b) set the ECR registry scanning configuration to enhanced mode — `aws ecr put-registry-scanning-configuration --scan-type ENHANCED --rules '[{"scanFrequency":"CONTINUOUS_SCAN","repositoryFilters":[{"filter":"*","filterType":"WILDCARD"}]}]'`. **Important limitations:** (i) When enhanced scanning is first enabled, Amazon Inspector only discovers images pushed within the **last 14 days** — older images receive `SCAN_ELIGIBILITY_EXPIRED` status and must be re-pushed to be scanned. (ii) After the initial scan, scan duration is controlled by the ECR re-scan duration setting in the Amazon Inspector console (defaults to `LIFETIME`); if you shorten this duration, images whose last scan exceeds the new window also move to `SCAN_ELIGIBILITY_EXPIRED`. (iii) Enhanced scanning incurs Amazon Inspector charges (no additional ECR cost). (iv) Repositories not matching a scan filter will have `Off` scan frequency and won't be scanned. 2. If enhanced scanning is not available in your region, enable basic scan-on-push as a fallback: `aws ecr put-image-scanning-configuration --repository-name --image-scanning-configuration scanOnPush=true`. 3. Create EventBridge rules to alert on CRITICAL/HIGH findings from Inspector. 4. Integrate findings into your vulnerability management workflow. | +| Reference | [ECR Enhanced Scanning](https://docs.aws.amazon.com/AmazonECR/latest/userguide/image-scanning-enhanced.html), [Amazon Inspector ECR Scanning](https://docs.aws.amazon.com/inspector/latest/user/scanning-ecr.html) | + +### Training Data & Model Poisoning (FS-17 to FS-21) + +> **PDF source:** §1.2.14 Training data and model poisoning. PDF-listed mitigations: +> (a) protect training datasets via data protection best practices; +> (b) use trusted data sources with audit controls tracking changes (who/when); +> (c) monitor training data for pattern/distribution changes (data drift); +> (d) compare retrained model performance against baseline before production; +> (e) rollback plan using versioned training data and models (Feature Store); +> (f) monitor low-entropy classification with thresholds and alerts; +> (g) AI Service Cards for evaluating third-party model testing procedures. + +#### FS-17 — Model Monitor Data Quality → *Merged into upstream SM-07* + +> **Upstream extension note (do not ship as a standalone check):** The detection and remediation +> content from FS-17 should be added as a refinement of the existing **SM-07 (Model Monitor)** +> check in the upstream `aws-samples/sample-aiml-security-assessment` repo. +> +> **What to add to SM-07:** +> +> - Filter `ListMonitoringSchedules` results for `MonitoringType=DataQuality` (not just any schedule). Note the format difference: `ListMonitoringSchedules`/`MonitoringScheduleSummary` returns `MonitoringType` in PascalCase (`DataQuality`, `ModelQuality`, `ModelBias`, `ModelExplainability`); `DescribeMonitoringSchedule` returns the same type in SCREAMING_SNAKE_CASE (`DATA_QUALITY`, `MODEL_QUALITY`, `MODEL_BIAS`, `MODEL_EXPLAINABILITY`) — the detection should normalise both forms. +> - Require `emit_metrics` to be enabled on the monitoring schedule. +> - Verify CloudWatch alarms exist on the `feature_baseline_drift_` metrics published +> to namespace `/aws/sagemaker/Endpoints/data-metric` (real-time endpoints, dimensions +> `EndpointName` + `ScheduleName`) or `/aws/sagemaker/ModelMonitoring/data-metric` (batch +> transform, dimension `MonitoringSchedule`). +> - PDF traceability: [PDF §1.2.14] — "Monitor your training data for pattern and distribution +> changes to detect data drift"; "Amazon SageMaker Model Monitor – Data quality." +> +> **Reference:** [SageMaker Model Monitor Data Quality](https://docs.aws.amazon.com/sagemaker/latest/dg/model-monitor-data-quality.html) + +#### FS-18 — Model Drift Detection → *Merged into upstream SM-23* + +> **Upstream extension note (do not ship as a standalone check):** The detection and remediation +> content from FS-18 should be added as a refinement of the existing **SM-23 (Model Drift +> Detection)** check in the upstream repo. +> +> **What to add to SM-23:** +> - Filter `ListMonitoringSchedules` results for `MonitoringType=ModelQuality`. +> - Add a new remediation step for **low-entropy classification monitoring** (PDF §1.2.14 +> mitigation): publish custom CloudWatch metrics tracking prediction confidence distributions, +> set threshold boundaries for unexpected low-confidence/high-confidence clusters, and alert +> when the retrained model produces unexpected classification patterns — this can indicate +> training data poisoning before accuracy metrics degrade. +> - PDF traceability: [PDF §1.2.14] — "Before deploying to production, compare your retrained +> model's performance against previous iterations using historical test data as a baseline." +> +> **Reference:** [SageMaker Model Monitor Model Quality](https://docs.aws.amazon.com/sagemaker/latest/dg/model-monitor-model-quality.html) + +#### FS-19 — Model Registry Approval → *Merged into upstream SM-22* + +> **Upstream extension note (do not ship as a standalone check):** The detection and remediation +> content from FS-19 should be added as a refinement of the existing **SM-22 (Model Approval +> Workflow)** check in the upstream repo. +> +> **What to add to SM-22:** +> - Explicitly check that `ModelApprovalStatus=PendingManualApproval` is the default for new +> model package versions (not `Approved`). +> - Flag any model package group where the latest version has `ModelApprovalStatus=Approved` +> without evidence of a manual approval step (i.e., auto-approved at creation time). +> - PDF traceability: [PDF §1.2.14] — cites "Amazon SageMaker AI – Model Registration and +> Deployment with Model Registry" as a reference for staged deployment with rollback. +> +> **Reference:** [SageMaker Model Registry](https://docs.aws.amazon.com/sagemaker/latest/dg/model-registry.html) + +#### FS-20 — Feature Store Rollback + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.14] — "Create a rollback plan by using versioned training data and models. This ensures that you can revert to a stable, working model if failures occur." References "Amazon SageMaker AI Feature Store". | +| Description | Checks SageMaker Feature Store has offline store for rollback capability. | +| Detection | Calls `sagemaker:ListFeatureGroups` to enumerate all groups, then `sagemaker:DescribeFeatureGroup` for each to inspect `OfflineStoreConfig`. Flags feature groups where `OfflineStoreConfig` is absent (online-only groups with no offline store for rollback). | +| Remediation | 1. Enable the offline store on each feature group: specify an S3 URI and data catalog in `OfflineStoreConfig`. 2. The offline store provides a versioned, immutable history of feature values for point-in-time rollback. 3. Test rollback by querying the offline store with a historical timestamp. | +| Reference | [SageMaker Feature Store](https://docs.aws.amazon.com/sagemaker/latest/dg/feature-store.html) | + +#### FS-21 — Training Data S3 Versioning and Audit Trail + +| Field | Detail | +|-------|--------| +| Severity | High | +| PDF ref | [PDF §1.2.14] — "Use trusted data sources for your training data. Implement audit controls that let you track and review changes, including who made them and when they occurred." | +| Description | Verifies S3 buckets with training data have versioning enabled and CloudTrail data-event logging active to record who modified training data and when. | +| Detection | Identifies training-data S3 buckets by tag (`data-classification=training` or `ml-purpose=training`) or by naming convention. Calls `s3:GetBucketVersioning` to verify `Status=Enabled`. Calls `cloudtrail:GetEventSelectors` on active trails to verify S3 data events are logged for these buckets. | +| Remediation | 1. Enable versioning: `aws s3api put-bucket-versioning --bucket --versioning-configuration Status=Enabled`. 2. Enable CloudTrail S3 data events for the training-data buckets to capture PutObject/DeleteObject with caller identity. 3. Enable MFA Delete for critical training datasets. 4. Apply S3 Object Lock for immutable baselines. | +| Reference | [S3 Versioning](https://docs.aws.amazon.com/AmazonS3/latest/userguide/Versioning.html), [CloudTrail Data Events](https://docs.aws.amazon.com/awscloudtrail/latest/userguide/logging-data-events-with-cloudtrail.html) | + +### Vector & Embedding Weaknesses (FS-22 to FS-26) + +> **PDF source:** §1.2.15 Vector and embedding weaknesses. PDF-listed mitigations: +> (a) apply least privilege to vector and embedding database access; +> (b) validate knowledge base data sources; +> (c) add data only from trusted sources to knowledge bases; +> (d) monitor and log all activities in knowledge base control plane (CloudTrail); +> (e) enable encryption at rest and in transit for vector and embedding databases; +> (f) implement document/record-level access controls via KB metadata filtering for +> multi-tenancy. + +#### FS-22 — Knowledge Base IAM Least Privilege + +| Field | Detail | +|-------|--------| +| Severity | High | +| PDF ref | [PDF §1.2.15] — "Apply the principle of least privilege to control access to your vector and embedding database. Only grant users and services the minimum permissions they need to perform their tasks." | +| Description | Checks IAM roles accessing Knowledge Bases have no wildcard `bedrock:*` permissions covering KB actions. | +| Detection | Inspects the permissions cache for all IAM roles. Flags any role with an Allow statement granting `bedrock:*` without resource-level restrictions, or broad `bedrock:` actions covering KB operations without a specific knowledge-base ARN. Note: Bedrock agent and KB operations use the single IAM service prefix `bedrock:` (not `bedrock-agent:`) — the `bedrock-agent` token refers to the boto3 SDK client name, not the IAM action prefix. | +| Remediation | 1. Replace wildcard `bedrock:*` with specific KB actions: `bedrock:Retrieve`, `bedrock:RetrieveAndGenerate`, `bedrock:GetKnowledgeBase` (these are the actual IAM action names — verify via the AWS Service Authorization Reference for Amazon Bedrock). 2. Scope the resource ARN to specific Knowledge Base IDs (e.g., `arn:aws:bedrock:::knowledge-base/`). 3. Apply IAM permission boundaries to limit blast radius. | +| Reference | [Bedrock Knowledge Base Permissions](https://docs.aws.amazon.com/bedrock/latest/userguide/kb-permissions.html) | + +#### FS-23 — Knowledge Base CloudTrail Logging → *Merged into upstream BR-06* + +> **Upstream extension note (do not ship as a standalone check):** The detection and remediation +> content from FS-23 should be added as a refinement of the existing **BR-06 (CloudTrail +> Logging)** check in the upstream repo. +> +> **What to add to BR-06:** +> - After verifying that a CloudTrail trail is active and logging Bedrock management events, +> additionally check for an **advanced event selector** with +> `resources.type = AWS::Bedrock::KnowledgeBase` to capture `Retrieve` and +> `RetrieveAndGenerate` data events (these are NOT logged by default — they require an +> explicit advanced event selector). +> - Note: `InvokeAgent` / `InvokeInlineAgent` are also data events requiring +> `resources.type = AWS::Bedrock::AgentAlias` or `AWS::Bedrock::InlineAgent` respectively. +> Data events incur additional CloudTrail charges and can produce high volumes under load. +> - PDF traceability: [PDF §1.2.15] — "Monitor and log all activities in knowledge base +> control plane" with reference "Monitor Amazon Bedrock API calls using CloudTrail." +> +> **Reference:** [CloudTrail Bedrock Logging](https://docs.aws.amazon.com/bedrock/latest/userguide/logging-using-cloudtrail.html) + +#### FS-24 — Knowledge Base Metadata Filtering + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.15] — "Implement access controls at the document or record level within knowledge bases where different users or applications should only have access to specific subsets of data. Use Amazon Bedrock Knowledge Bases metadata filtering to enforce data segmentation." | +| Description | Advisory: verifies KB metadata fields support tenant-level filtering for multi-tenancy. | +| Detection | Calls `ListKnowledgeBases` and `GetKnowledgeBase` (via the `bedrock-agent` boto3 client; IAM actions are `bedrock:ListKnowledgeBases` and `bedrock:GetKnowledgeBase`). Inspects the storage configuration for metadata field definitions. Flags KBs with no metadata fields defined (no tenant isolation possible). | +| Remediation | 1. Define metadata fields on your KB data sources (e.g., `tenant_id`, `department`, `classification`). 2. Populate metadata during document ingestion. 3. Use the `filter` parameter in Retrieve/RetrieveAndGenerate API calls to enforce tenant-scoped queries. 4. Test that cross-tenant data leakage is prevented. | +| Reference | [Bedrock KB Metadata Filtering](https://docs.aws.amazon.com/bedrock/latest/userguide/kb-test-config.html) | + +#### FS-25 — OpenSearch Serverless Encryption + +| Field | Detail | +|-------|--------| +| Severity | High | +| PDF ref | [PDF §1.2.15] — "Enable encryption at rest and in transit for vector and embedding databases." | +| Description | Checks OpenSearch Serverless collections used by KBs have CMK encryption policies. | +| Detection | Calls `opensearchserverless:ListCollections` (IAM action `aoss:ListCollections`) and for each calls `opensearchserverless:ListSecurityPolicies(type=encryption)` (IAM action `aoss:ListSecurityPolicies`). Inspects each encryption policy's document for `AWSOwnedKey=true` or missing `KmsARN`. Note: the encryption **policy JSON document** uses PascalCase field names — `AWSOwnedKey` and `KmsARN` — while the direct API `EncryptionConfig` struct uses camelCase (`aWSOwnedKey`, `kmsKeyArn`); detection should inspect the policy document form returned by `GetSecurityPolicy`/`ListSecurityPolicies`. Flags collections using AWS-owned keys instead of customer-managed KMS keys. Note: the boto3 client name is `opensearchserverless`, but IAM actions use the service prefix `aoss:` (not `opensearchserverless:`). Note also: encryption in transit is automatic (TLS 1.2, AES-256) for all OpenSearch Serverless traffic and is not configurable — this check focuses on encryption at rest. | +| Remediation | 1. Create an encryption security policy specifying a customer-managed KMS key: set `AWSOwnedKey=false` and provide `KmsARN` with the ARN of your KMS key. 2. Apply the policy to the collection by matching the collection name or prefix pattern in the policy `Rules`. 3. Ensure the KMS key policy grants the OpenSearch Serverless service principal `kms:Decrypt` and `kms:GenerateDataKey`. Note: if you provide a KMS key directly in the `CreateCollection` request, it takes precedence over any matching security policies. | +| Reference | [OpenSearch Serverless Encryption](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-encryption.html) | + +#### FS-26 — Knowledge Base VPC Access + +| Field | Detail | +|-------|--------| +| Severity | High | +| PDF ref | [PDF §1.2.15, extension] — network isolation is not verbatim in the PDF but directly implements "Apply the principle of least privilege to control access to your vector and embedding database" at the network layer. | +| Description | Verifies OpenSearch Serverless collections have VPC-only network policies (no public access). | +| Detection | Calls `opensearchserverless:ListSecurityPolicies(type=network)` (IAM action `aoss:ListSecurityPolicies` — the service prefix for OpenSearch Serverless is `aoss`, not `opensearchserverless`) and inspects each policy rule for `AllowFromPublic=true`. Flags collections accessible from the public internet. Note: a policy with `AllowFromPublic=false` may still grant private access to Bedrock via `SourceServices: ["bedrock.amazonaws.com"]` or to specific VPC endpoints via `SourceVPCEs` — these are the recommended private-access patterns and are not flagged. | +| Remediation | 1. Create a network security policy that restricts access to specific VPC endpoints only via `SourceVPCEs`, or grants private AWS service access (e.g., Bedrock) via `SourceServices: ["bedrock.amazonaws.com"]`. Per AWS docs, private access to AWS services applies only to the collection's OpenSearch endpoint, not to the OpenSearch Dashboards endpoint. 2. Create an OpenSearch Serverless VPC endpoint in your VPC if VPC-private access is required. 3. Remove any policy rules with `AllowFromPublic=true`. 4. Test connectivity from within the VPC. | +| Reference | [OpenSearch Serverless Network Access](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-network.html) | + diff --git a/docs/SECURITY_CHECKS_FINSERV_PART2_GUARDRAILS_CONTENT_SAFETY.md b/docs/SECURITY_CHECKS_FINSERV_PART2_GUARDRAILS_CONTENT_SAFETY.md new file mode 100644 index 0000000..3006cc6 --- /dev/null +++ b/docs/SECURITY_CHECKS_FINSERV_PART2_GUARDRAILS_CONTENT_SAFETY.md @@ -0,0 +1,318 @@ +# FinServ GenAI Risk Checks — Part 2: Guardrails & Content Safety (FS-27 to FS-46) + +This is **Part 2 of 3** of the FinServ GenAI security checks derived from the +[AWS guide for Financial Services risk management of the use of Generative AI (March 2026)](https://d1.awsstatic.com/onedam/marketing-channels/website/public/global-FinServ-ComplianceGuide-GenAIRisks-public.pdf) +(referred to throughout as "the FinServ Guide"). + +This part covers **20 checks** across 5 PDF risk categories: + +- **Non-Compliant Output** (FS-27 to FS-30) — §1.2.1 +- **Misinformation** (FS-31 to FS-34) — §1.2.3 (FS-34 sources from §1.2.12 — see note) +- **Abusive or Harmful Output** (FS-35 to FS-38) — §1.2.4 +- **Biased Output** (FS-39 to FS-42) — §1.2.5 +- **Sensitive Information Disclosure** (FS-43 to FS-46) — §1.2.6 + +**Companion files:** + +- `SECURITY_CHECKS_FINSERV_PART1_INFRA_CONTROLS.md` — FS-01 to FS-26 (Unbounded, Excessive Agency, Supply Chain, Training Poisoning, Vector Weaknesses) +- `SECURITY_CHECKS_FINSERV_PART3_APP_LAYER_AND_GAPS.md` — FS-47 to FS-69 (Hallucination, Prompt Injection, Output Handling, Off-Topic, Stale Data, Material Gaps) +- `SECURITY_CHECKS_FINSERV_COMMON.md` — shared intro, severity rubric, validation note, upstream-overlap table + +Each check includes how it is **detected** (the AWS API calls or configuration inspected) +and how a failure is **remediated** (the specific AWS actions to take). + +See `SECURITY_CHECKS_FINSERV_COMMON.md` for: + +- PDF traceability conventions (`[PDF §x.y.z]` vs `[PDF §x.y.z, extension]`) +- Severity rubric (High / Medium / Low / Advisory) +- Validation note and AWS service authorization references +- Relationship to upstream SM/BR/AC checks and consolidation recommendations + +--- + +## FinServ GenAI Risk Checks — Part 2 content + +### Non-Compliant Output (FS-27 to FS-30) + +> **PDF source:** §1.2.1 Non-compliant output. PDF-listed mitigations: +> (a) prompt engineering to guide the model and prevent unwanted responses; +> (b) content filters and denied topics in Bedrock Guardrails; +> (c) RAG with Bedrock Knowledge Bases; +> (d) Automated Reasoning checks in Bedrock Guardrails; +> (e) human-in-the-loop validation for internal AI systems; +> (f) audit logs of AI-generated outputs and guardrails applied for regulatory reporting. + +#### FS-27 — Automated Reasoning Checks + +| Field | Detail | +|-------|--------| +| Severity | High | +| PDF ref | [PDF §1.2.1, §1.2.7] — "Automated Reasoning checks in Amazon Bedrock Guardrails uses automated reasoning to verify that natural language content complies with your defined policies. This mathematical verification helps ensure that your content strictly follows your guardrails." | +| Description | Verifies Bedrock Guardrails have Automated Reasoning checks or contextual grounding enabled. | +| Detection | Calls `bedrock:ListGuardrails` and `bedrock:GetGuardrail` for each. Inspects the response fields `contextualGroundingPolicy` and `automatedReasoningPolicy`. Flags guardrails with neither enabled. | +| Remediation | 1. Enable contextual grounding filters (type `GROUNDING`) with a threshold ≥ 0.7 — these filters CAN block content that fails grounding checks. Note: valid threshold values are 0 to 0.99; a threshold of 1.0 is invalid and will block all content. **Important use-case limitation:** Contextual grounding checks support summarization, paraphrasing, and question answering use cases only — **Conversational QA / Chatbot use cases are not supported**. If your FinServ application is a conversational chatbot, contextual grounding cannot be used for hallucination detection; use Automated Reasoning checks or human-in-the-loop validation instead. 2. If available in your region, additionally enable Automated Reasoning checks by creating an Automated Reasoning policy and attaching it to the guardrail. **Cross-Region inference is REQUIRED for AR:** Guardrails that use Automated Reasoning checks require a cross-Region inference profile — set `crossRegionConfig.guardrailProfileIdentifier` to a profile matching your Region (for example, `us.guardrail.v1:0` for US Regions or `eu.guardrail.v1:0` for EU Regions). Omitting this parameter returns `ValidationException`. As of April 2026, AR is generally available in US East (N. Virginia), US East (Ohio), US West (Oregon), EU (Frankfurt), EU (Ireland), and EU (Paris) — verify current regional availability on the [AR documentation page](https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-automated-reasoning-checks.html) before audit reliance, as AWS regularly expands coverage. Attach the **versioned** policy ARN (for example, `...:1`) — the unversioned ARN returns an error. You can attach a maximum of 2 AR policies per guardrail. Important: Automated Reasoning operates in **detect mode only** — it returns findings and feedback but does NOT block content. AR finding types (per the user guide) are: `VALID` (response is consistent with policy), `INVALID` (response contradicts policy rules), `SATISFIABLE` (response could be true or false depending on unstated conditions), `IMPOSSIBLE` (premises are contradictory), `TRANSLATION_AMBIGUOUS` (natural language could not be reliably translated to formal logic), `TOO_COMPLEX` (policy complexity exceeded processing limits), and `NO_TRANSLATIONS` (some or all input was not translated into logic due to irrelevance or lack of matching policy variables). Note: in the `AutomatedReasoningCheckFinding` runtime response, these appear as a **union** with lowercase camelCase keys (`valid`, `invalid`, `satisfiable`, `impossible`, `translationAmbiguous`, `tooComplex`, `noTranslations`) — exactly one key is present per finding. Per AWS docs, AR also **does not protect against prompt injection attacks**, **cannot detect off-topic responses**, **does not support streaming APIs**, and **supports English (US) only** — use content filters, topic policies, and other guardrail components alongside AR. **Critical limitation for cross-account enforcement:** AR policies are NOT supported with Bedrock Guardrails cross-account safeguards (org-level or account-level enforcement) — including an AR policy in a guardrail used for enforcement will cause runtime failures. If you rely on AR, configure it at the application or account level separately. Your application must inspect the AR findings via the `ApplyGuardrail` (or `Converse` / `InvokeModel` / `InvokeAgent` / `RetrieveAndGenerate`) API response and decide whether to serve the response, rewrite it using AR feedback, ask the user for clarification, or fall back to a default behavior. 3. For `INVALID` responses, implement an iterative rewriting loop that feeds AR feedback (contradicting rules) back to the LLM to self-correct. 4. Build an audit trail of all AR validation iterations — log `supportingRules` and `claimsTrueScenario` for `VALID` findings as mathematically verifiable compliance evidence. | +| Reference | [Automated Reasoning in Bedrock Guardrails](https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-automated-reasoning-checks.html), [AR Checks Concepts (Validation Results Reference)](https://docs.aws.amazon.com/bedrock/latest/userguide/automated-reasoning-checks-concepts.html), [Integrate AR Checks in Your Application](https://docs.aws.amazon.com/bedrock/latest/userguide/integrate-automated-reasoning-checks.html), [Deploy Automated Reasoning Policy](https://docs.aws.amazon.com/bedrock/latest/userguide/deploy-automated-reasoning-policy.html) | + +#### FS-28 — Financial Denied Topics + +| Field | Detail | +|-------|--------| +| Severity | High | +| PDF ref | [PDF §1.2.1] — "Configure content filters and guardrails to restrict model responses to approved topics" with reference "Amazon Bedrock User Guide – Guardrails – Denied topics". | +| Description | Checks guardrails have denied topics for regulated financial advice. | +| Detection | Calls `bedrock:GetGuardrail` and inspects `topicPolicy.topics` for entries with `type=DENY`. Flags guardrails with no denied topics or with no topics related to financial advice, investment recommendations, or tax guidance. | +| Remediation | 1. Add denied topics to the guardrail following the AWS best-practice golden rules: (a) **Be crisp and precise** — e.g., "Investment advice is inquiries, guidance, or recommendations about the management or allocation of funds or assets with the goal of generating returns or achieving specific financial objectives" rather than vague "Investment advice". (b) **Define, don't instruct** — write "All content associated with specific investment recommendations" not "Block all investment advice". (c) **Stay positive** — never define topics negatively (e.g., avoid "All content except general financial education"). (d) **Focus on themes, not words** — denied topics capture subjects contextually; use word filters for specific names or entities. (e) **Provide sample phrases** — add up to 5 representative inputs per topic (each up to 100 characters). 2. **Quantity and character limits:** A guardrail can contain a maximum of **30 denied topics**. In Classic tier, topic definitions are limited to 200 characters; in Standard tier, up to 1,000 characters — use Standard tier for complex financial topic definitions. 3. Recommended denied topics for FinServ: "specific investment recommendations", "tax advice", "specific financial product recommendations", "guaranteed returns or performance claims". 4. For multi-account enforcement, use Bedrock cross-account safeguards to apply denied topics from a management-account guardrail across all member accounts automatically. When configuring account-level or org-level enforcement, set **both** `selectiveContentGuarding.messages` AND `selectiveContentGuarding.system` to `COMPREHENSIVE` to ensure guardrails evaluate all user messages AND system prompts regardless of input tags — use `SELECTIVE` only when you trust callers to correctly tag content. Setting only `messages` to COMPREHENSIVE leaves system prompts potentially unguarded. 5. Enforce guardrails via IAM policy conditions (`bedrock:GuardrailIdentifier`) to prevent any Bedrock inference call without a guardrail attached. 6. Test with prompts that attempt to elicit regulated financial advice. | +| Reference | [Bedrock Guardrails Denied Topics](https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-denied-topics.html), [Safeguard Tiers for Guardrails](https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-tiers.html), [Cross-Account Safeguards with Enforcements](https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-enforcements.html), [Guardrails Best Practices](https://aws.amazon.com/blogs/machine-learning/build-safe-generative-ai-applications-like-a-pro-best-practices-with-amazon-bedrock-guardrails/) | + +#### FS-29 — Compliance Disclaimer + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.1, extension] — disclaimers are not verbatim in §1.2.1 but the PDF references "Implement response disclaimers in customer-facing applications" under §1.2.7 Hallucination, which is conceptually the same control applied here for non-compliant financial-advice output. | +| Description | Advisory: verifies application adds required regulatory disclaimers to AI-generated outputs. | +| Detection | Advisory check — cannot be fully automated. Inspects application Lambda function environment variables or configuration for disclaimer-related settings (e.g., `DISCLAIMER_ENABLED`, `COMPLIANCE_FOOTER`). | +| Remediation | 1. Add a standard regulatory disclaimer to all customer-facing AI-generated responses (e.g., "This information is generated by AI and does not constitute financial advice. Please consult a qualified financial advisor."). 2. Make the disclaimer text configurable via environment variable or parameter store. 3. Ensure disclaimers are not removable by prompt manipulation. | +| Reference | [AWS Well-Architected GenAI Lens — Guardrails](https://docs.aws.amazon.com/wellarchitected/latest/generative-ai-lens/gensec02-bp01.html) | + +#### FS-30 — Compliance Evaluation Datasets + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.1, extension] — the PDF §1.2.12 practical guidance mentions "Amazon Bedrock Evaluations can help to evaluate models against specific types of attacks"; this check extends that concept to compliance-specific evaluation for FS-regulated outputs. | +| Description | Checks Bedrock evaluation jobs use compliance-specific test datasets. | +| Detection | Calls `bedrock:ListEvaluationJobs` to enumerate existing jobs, then calls `bedrock:GetEvaluationJob` for each to inspect the full `evaluationConfig` including dataset configuration. Flags if no evaluation jobs exist or if none reference compliance/regulatory test data. Note: `ListEvaluationJobs` returns only job summaries — dataset configuration details require `GetEvaluationJob`. | +| Remediation | 1. Create a compliance-specific evaluation dataset containing: prompts requesting regulated financial advice, prompts testing disclaimer presence, prompts testing denied-topic enforcement. 2. Run Bedrock evaluation jobs with this dataset before each production deployment. 3. Set pass/fail thresholds and gate deployments on results. | +| Reference | [Bedrock Model Evaluation](https://docs.aws.amazon.com/bedrock/latest/userguide/evaluation.html) | + +### Misinformation (FS-31 to FS-33) + +> **PDF source:** §1.2.3 Misinformation through inadvertent or malicious action. PDF-listed mitigations: +> (a) prompt engineering; +> (b) verify knowledge base data sources are up-to-date, accurate, reliable, and complete; +> (c) human-in-the-loop validation for internal AI systems; +> (d) source attribution in RAG responses for end users to verify provenance; +> (e) integrity monitoring on knowledge base data sources — e.g., S3 event notifications to +> track document changes. + +#### FS-31 — Knowledge Base Data Source Sync + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.3, §1.2.10] — "Verify that your knowledge base data sources are up-to-date, accurate, reliable, and complete"; "Sync your data with your Amazon Bedrock knowledge base". | +| Description | Verifies KB data sources have been synced within 7 days. | +| Detection | Calls `ListDataSources` then `ListIngestionJobs` for each data source (via the `bedrock-agent` boto3 client; IAM actions are `bedrock:ListDataSources` and `bedrock:ListIngestionJobs`). Checks the most recent successful ingestion job's `updatedAt` timestamp. Flags data sources not synced within 7 days. | +| Remediation | 1. Create an EventBridge scheduled rule to trigger KB data source sync at least weekly. 2. Use `StartIngestionJob` (IAM action `bedrock:StartIngestionJob`) as the rule target. 3. Add CloudWatch alarms for failed ingestion jobs. 4. For rapidly changing data, increase sync frequency. | +| Reference | [Bedrock KB Data Source Sync](https://docs.aws.amazon.com/bedrock/latest/userguide/kb-data-source-sync-ingest.html) | + +#### FS-32 — Source Attribution + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.3, §1.2.10] — "Use source attribution in RAG-based response for end users to verify provenance of information" (§1.2.3); "Use source attribution in RAG-based response for end users to verify currency of information" (§1.2.10). | +| Description | Advisory: verifies application implements source citations in RAG responses. | +| Detection | Advisory check — inspects application code or configuration for use of the `citations` field in `RetrieveAndGenerate` API responses. Checks Lambda environment variables for attribution-related settings. | +| Remediation | 1. Use the `RetrieveAndGenerate` API (IAM action `bedrock:RetrieveAndGenerate`) which returns `citations` with source document references. Each citation contains `retrievedReferences` — an array where each reference has a `content` object (the cited text), a `location` object (data source type and URI — for S3 sources, `location.type=S3` and `location.s3Location.uri` contains the S3 URI), and optional `metadata` (a string-to-JSON map with any custom metadata attributes stored on the chunk, which can hold document title and other fields). Note: there is no fixed `title` field in the API — if you need to display document titles to end users, store them as a metadata attribute during KB ingestion and retrieve them via `retrievedReferences[].metadata`. 2. Display source citations to end users alongside AI-generated responses. 3. Include the data source location (URI or other location identifier depending on source type: S3, Web, Confluence, SharePoint, Salesforce, Kendra, SQL, or Custom) and the cited text excerpt (from `content`). 4. If document titles are required, ensure they are populated in KB metadata and propagated to your UI. 5. Allow users to click through to the original source document where possible. | +| Reference | [Bedrock RetrieveAndGenerate API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_agent-runtime_RetrieveAndGenerate.html) | + +#### FS-33 — Knowledge Base Integrity Monitoring + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.3] — "Use integrity monitoring on knowledge base data sources to detect unauthorized modifications. Track changes to documents used in knowledge bases." References "For example on S3 data sources use Amazon S3 event notification to track changes to documents." | +| Description | Checks KB data source S3 buckets have versioning enabled and S3 event notifications (EventBridge or SNS) configured to detect unauthorized document modifications in real time. | +| Detection | Identifies KB data-source S3 buckets via `GetDataSource` (via the `bedrock-agent` boto3 client; IAM action `bedrock:GetDataSource`). Calls `s3:GetBucketVersioning` to verify `Status=Enabled`. Calls `s3:GetBucketNotificationConfiguration` and checks for `EventBridgeConfiguration`, `TopicConfigurations`, `QueueConfigurations`, or `LambdaFunctionConfigurations`. Flags buckets missing either control. | +| Remediation | 1. Enable versioning: `aws s3api put-bucket-versioning --bucket --versioning-configuration Status=Enabled`. 2. Enable EventBridge notifications on the bucket: `aws s3api put-bucket-notification-configuration --bucket --notification-configuration '{"EventBridgeConfiguration":{}}'`. Once enabled, S3 automatically sends **all** bucket events to EventBridge — you do not select specific event types at the bucket level. 3. Create an EventBridge rule that matches S3 events for this bucket — use the `detail-type` field values `Object Created` and `Object Deleted` (these are the EventBridge event type names; note: `s3:ObjectCreated:*` and `s3:ObjectRemoved:*` are the legacy SNS/SQS/Lambda notification event type names and are NOT used in EventBridge rules). Route matched events to an SNS topic or Lambda function for alerting. 4. Integrate alerts into your security incident response workflow. | +| Reference | [S3 EventBridge Integration](https://docs.aws.amazon.com/AmazonS3/latest/userguide/EventBridge.html) | + +> **Note:** FS-34 (Third-Party Risk Management for FM Providers) is kept adjacent to Misinformation +> in this file for continuity with the prior draft numbering, but its PDF source is §1.2.12 +> Supply Chain Vulnerabilities. Treat FS-34 as a Supply Chain check for compliance-framework +> mapping purposes. + +#### FS-34 — Third-Party Risk Management (TPRM) for Foundation Model Providers + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.12] — *"Update existing third-party risk management processes to continuously monitor model providers and third-party dependencies, including tracking vendor security advisories, model deprecation notices, and change to terms and conditions."* (Note: moved from the Misinformation section in the prior draft; the PDF places TPRM under Supply Chain.) | +| Description | Verifies a documented third-party risk management (TPRM) process exists to monitor FM providers for security advisories, model deprecation notices, and T&C changes; also flags legacy FMs currently in use. | +| Detection | Calls `bedrock:ListFoundationModels`, then `bedrock:GetFoundationModel` for each in-use model; inspects `modelLifecycle.status` and flags models with status `LEGACY`. Note: the `FoundationModelLifecycle.status` API field has only **two** valid values — `ACTIVE` and `LEGACY`. There is no `EOL` status value in the API; models that have passed their EOL date are removed from the service entirely and API calls referencing them will fail. The user-facing lifecycle page describes three conceptual states (Active, Legacy, EOL) but the API only exposes two. Advisory component checks for evidence of a TPRM process — e.g., an AWS Config rule or a tag on Bedrock resources indicating periodic review (`tprm-last-reviewed=`). | +| Remediation | 1. Establish a documented TPRM process: at least quarterly review of each in-use FM provider's security advisories, model lifecycle announcements, and T&C changes. 2. Assign an owner for the TPRM process and record review evidence in your MRM system. 3. Subscribe to AWS Bedrock model lifecycle notifications. 4. Migrate workloads from `LEGACY` models to active versions before their published EOL date — note that for models with EOL dates after February 1, 2026, there is a "public extended access" period where Legacy models remain usable but at higher pricing set by the model provider. 5. For third-party models procured via AWS Marketplace or consumed directly, evaluate the provider's own testing procedures — AWS AI Service Cards provide this transparency for Amazon-trained models. | +| Reference | [Bedrock Model Lifecycle](https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle.html), [Access Amazon Bedrock foundation models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-access.html) | + +### Abusive or Harmful Output (FS-35 to FS-38) + +> **PDF source:** §1.2.4 Model output is abusive or harmful. PDF-listed mitigations: +> (a) AWS AI Service Cards to understand how Amazon addresses toxicity per model; +> (b) Amazon Bedrock Guardrails to detect and filter harmful content; +> (c) FMEval to evaluate for inappropriate content (sexual, profanity, hate, aggression, +> insults, flirtation, identity attacks, threats); +> (d) user reporting mechanism so end users can flag abusive outputs, reviewed within a +> defined process; +> (e) Practical guidance: create allowlists for approved business terminology to reduce +> false positives on brand, product, industry, and technical vocabulary. + +#### FS-35 — FMEval Harmful Content + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.4] — "Foundation Model Evaluations (FMEval) evaluates your model to detect inappropriate content, including sexual references, profanity, hate speech, aggression, insults, flirtation, identity-based attacks, and threats." | +| Description | Checks Bedrock evaluation jobs test for harmful/toxic content. | +| Detection | Calls `bedrock:ListEvaluationJobs` to enumerate existing jobs, then calls `bedrock:GetEvaluationJob` for each to inspect the full `evaluationConfig`. The correct metric name depends on the evaluation job type: (a) For **automated model evaluation jobs** (pre-computed metrics), the toxicity metric is `"Builtin.Toxicity"` — the only valid harmful-content metric for this job type. (b) For **judge-based model evaluation jobs** (LLM-as-judge), the harmful content metrics are `"Builtin.Harmfulness"` and `"Builtin.Stereotyping"`. (c) For **knowledge base (RAG) evaluation jobs**, `"Builtin.Harmfulness"` and `"Builtin.Stereotyping"` are also valid. Flags if no evaluation jobs exist or none include a harmful-content metric (`Builtin.Toxicity` for automated, `Builtin.Harmfulness` for judge-based/RAG). Note: `ListEvaluationJobs` returns only job summaries — dataset configuration details require `GetEvaluationJob`. | +| Remediation | 1. For **automated model evaluation** (fastest, no judge model required): create a Bedrock evaluation job with `"Builtin.Toxicity"` in the `metricNames` array. Valid task types are `Summarization`, `Classification`, `QuestionAndAnswer`, `Generation`, and `Custom`. 2. For **judge-based model evaluation** (more nuanced, requires a judge model): create a Bedrock evaluation job with `"Builtin.Harmfulness"` and/or `"Builtin.Stereotyping"` in the `metricNames` array — these metrics are only valid for judge-based and RAG evaluation jobs, not automated model evaluation jobs. 3. Include test prompts designed to elicit harmful content. 4. Set pass/fail thresholds based on the scores returned. 5. Run evaluations before production deployment and after model updates. 6. For more granular toxicity scoring (the 7-category UnitaryAI Detoxify-unbiased scores: `toxicity`, `severe_toxicity`, `obscene`, `threat`, `insult`, `sexual_explicit`, `identity_attack` — or the Toxigen-roberta binary classifier), use SageMaker FMEval via SageMaker Studio or the `fmeval` Python library as a complementary evaluation path. | +| Reference | [Bedrock Model Evaluation Metrics](https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation-metrics.html), [SageMaker FMEval Toxicity](https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-toxicity-evaluation.html) | + +#### FS-36 — Guardrail Content Filters + +| Field | Detail | +|-------|--------| +| Severity | High | +| PDF ref | [PDF §1.2.4] — "Use Amazon Bedrock's guardrails to detect and filter harmful content." | +| Description | Verifies guardrails have content filters for hate, violence, sexual, and other harmful content. | +| Detection | Calls `bedrock:GetGuardrail` and inspects `contentPolicy.filters`. Flags guardrails missing filters for HATE, VIOLENCE, SEXUAL, INSULTS, or MISCONDUCT categories. Also checks that `inputStrength` and `outputStrength` are at least `MEDIUM`. | +| Remediation | 1. Update the guardrail to include content filters for all harmful categories: HATE, VIOLENCE, SEXUAL, INSULTS, MISCONDUCT. 2. Select the **Standard tier** (not Classic) for content filters — it offers better accuracy, broader language support (extensive multilingual support vs. English/French/Spanish only in Classic), prompt leakage detection, and extends protection to harmful content within code elements. Standard tier requires cross-Region inference to be enabled on the guardrail (configurable at creation or by modifying an existing guardrail). 3. Start with **HIGH** filter strength for customer-facing applications; evaluate false-positive rates on representative sample traffic and lower to MEDIUM only if necessary. 4. Apply filters to both INPUT and OUTPUT. 5. Before enabling blocking in production, use **detect mode** (`action=NONE`) to test guardrail behavior on live traffic — review trace output to validate decisions, then switch to `action=BLOCK` once confident. 6. Enforce guardrails organization-wide via IAM policy-based enforcement: add an IAM condition key (`bedrock:GuardrailIdentifier`) to deny any `InvokeModel`/`Converse` call that does not include a guardrail. For account-level or org-level enforcement configurations, set **both** `selectiveContentGuarding.messages` AND `selectiveContentGuarding.system` to `COMPREHENSIVE` to ensure guardrails evaluate all user messages AND system prompts regardless of input tags (use `SELECTIVE` only when you trust callers to correctly tag content). Setting only `messages` to COMPREHENSIVE leaves system prompts potentially unguarded. | +| Reference | [Bedrock Guardrails Content Filters](https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-content-filters.html), [Safeguard Tiers for Guardrails](https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-tiers.html), [Cross-Account Safeguards with Enforcements](https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-enforcements.html), [Guardrails Best Practices](https://aws.amazon.com/blogs/machine-learning/build-safe-generative-ai-applications-like-a-pro-best-practices-with-amazon-bedrock-guardrails/), [IAM Guardrail Enforcement](https://aws.amazon.com/blogs/machine-learning/amazon-bedrock-guardrails-announces-iam-policy-based-enforcement-to-deliver-safe-ai-interactions/) | + +#### FS-37 — User Feedback Mechanism + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.4] — "Implement a user reporting mechanism that allows end users to flag abusive or harmful outputs. Reported incidents [are] reviewed within a defined process to refine content filters." | +| Description | Advisory: verifies application has a user reporting mechanism for harmful outputs. | +| Detection | Advisory check — inspects application configuration for feedback-related settings (e.g., `FEEDBACK_ENABLED`, `REPORT_ABUSE_ENDPOINT`). Checks for Lambda functions with "feedback" or "report" in the name. | +| Remediation | 1. Implement a "Report this response" button in the application UI. 2. Route reported responses to an SQS queue or DynamoDB table for review. 3. Define an SLA for reviewing reported content (e.g., 24 hours). 4. Use reported incidents to refine guardrail content filters and word lists. 5. Log all reports with Bedrock invocation logging correlation IDs. | +| Reference | [Bedrock Model Invocation Logging](https://docs.aws.amazon.com/bedrock/latest/userguide/model-invocation-logging.html) | + +#### FS-38 — Guardrail Word Filters and Business Term Allowlists + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.4 — Practical guidance] — "Create allowlists for business terms that include approved terminology for: brand names, product names, industry terms, and technical vocabulary. Also test filter settings to verify that your content filters allow necessary business communications and generate accurate alerts. Monitor and adjust regularly your filtering system to reduce false positives." | +| Description | Checks guardrails have word/phrase block filters configured and that approved business terminology allowlists are defined to prevent false positives on legitimate financial services vocabulary. | +| Detection | Calls `bedrock:GetGuardrail` and inspects `wordPolicy`. Flags guardrails with no custom `words` array (blocked phrases). Also checks `managedWordLists` for the AWS-managed `PROFANITY` list. Note: a guardrail with only the profanity filter and no custom FinServ-specific blocked terms should still be flagged as incomplete for financial services use cases. | +| Remediation | 1. Add blocked words/phrases to the guardrail word filter (profanity, slurs, competitor names if applicable). Each custom word/phrase entry has a maximum length of **100 characters** per the API (`GuardrailWordConfig.text`); the console UI additionally limits entries to **up to three words** per phrase. You can add up to **10,000 items** to the custom word filter. 2. Enable the AWS-managed profanity filter (`managedWordListsConfig` with `type=PROFANITY`) as a baseline. 3. Create an allowlist of approved business terminology: brand names, product names, industry terms, technical vocabulary — document this separately as the guardrail word filter only blocks, it does not allowlist. Test filter settings to verify legitimate business communications are not blocked. 4. Monitor and adjust regularly to reduce false positives. | +| Reference | [Bedrock Guardrails Word Filters](https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-word-filters.html) | + +### Biased Output (FS-39 to FS-42) + +> **PDF source:** §1.2.5 Model output is biased. PDF-listed mitigations: +> (a) AWS AI Service Cards to understand how providers address fairness/bias per model; +> (b) prompt engineering; +> (c) Amazon Bedrock Guardrails; +> (d) Bedrock Evaluations to measure bias; +> (e) Amazon SageMaker Clarify for bias detection, transparency, and prediction explanation +> on fine-tuned and self-trained models; +> (f) develop and maintain a bias testing dataset with representative cases across +> demographic groups, geographic regions, and sensitive attributes — run periodically and +> after each model update. + +#### FS-39 — SageMaker Clarify Bias + +| Field | Detail | +|-------|--------| +| Severity | High | +| PDF ref | [PDF §1.2.5] — "Use Amazon SageMaker Clarify to detect bias, increase transparency, and explain predictions for your fine-tuned and self-trained AI models." | +| Description | Verifies Clarify model bias monitoring is configured for financial decision models. | +| Detection | Calls `sagemaker:ListMonitoringSchedules` with the `MonitoringTypeEquals=ModelBias` filter parameter (the `MonitoringType` field on the `MonitoringScheduleSummary` response has one of four values: `DataQuality`, `ModelQuality`, `ModelBias`, `ModelExplainability`). Flags if no bias monitoring schedules exist. Cross-references with endpoints tagged `use-case=financial-decision` or similar. Clarify bias monitoring publishes metrics to the `aws/sagemaker/Endpoints/bias-metrics` namespace for real-time endpoints (and `aws/sagemaker/ModelMonitoring/bias-metrics` for batch transform jobs) with `Endpoint`, `MonitoringSchedule`, `BiasStage`, `Label`, `LabelValue`, `Facet`, and `FacetValue` dimensions. | +| Remediation | 1. Create a SageMaker Clarify bias monitoring schedule for each financial decision model endpoint. 2. Specify facets (protected attributes: age, gender, race, geography) and bias metrics (DPL, DI, DPPL). 3. Provide a baseline bias report from training data. 4. Configure CloudWatch alarms on bias metric violations on the `aws/sagemaker/Endpoints/bias-metrics` namespace. Note: `publish_cloudwatch_metrics` is enabled by default — do NOT set it to `Disabled` in the model bias job definition's `Environment` map, as that would stop metrics from being published to CloudWatch. | +| Reference | [SageMaker Clarify Bias Detection](https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-detect-post-training-bias.html) | + +#### FS-40 — Bedrock Bias Evaluation Datasets and Cadence + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.5] — "Develop and maintain a bias testing dataset that includes representative test cases across demographic groups, geographic regions, and other sensitive attributes relevant to your use case. Run these test cases periodically and after model updates." | +| Description | Checks evaluation jobs include demographic fairness test cases across protected groups and verifies evaluations are run on a defined periodic schedule and after each model update. | +| Detection | Calls `bedrock:ListEvaluationJobs` to enumerate existing jobs, then calls `bedrock:GetEvaluationJob` for each to inspect the full `evaluationConfig` including dataset configuration for demographic diversity test cases. Checks the `creationTime` of the most recent evaluation job and flags if it is older than 90 days or if no evaluation was run after the most recent model deployment. Note: `ListEvaluationJobs` returns only job summaries — dataset configuration details require `GetEvaluationJob`. | +| Remediation | 1. Create a bias evaluation dataset with representative test cases across demographic groups, geographic regions, and other sensitive attributes. 2. Schedule evaluation jobs to run at least quarterly via EventBridge. 3. Trigger an evaluation job automatically after each model update in your CI/CD pipeline. 4. Store results for audit and trend analysis. | +| Reference | [Bedrock Model Evaluation](https://docs.aws.amazon.com/bedrock/latest/userguide/evaluation.html) | + +#### FS-41 — SageMaker Clarify Explainability + +| Field | Detail | +|-------|--------| +| Severity | High | +| PDF ref | [PDF §1.2.5, extension] — PDF §1.2.5 recommends "Amazon SageMaker Clarify to detect bias, increase transparency, and explain predictions". ECOA/Fair Housing adverse-action-notice use case is an FS-specific extension of Clarify explainability not named verbatim in the PDF. | +| Description | Verifies Clarify explainability monitoring for adverse action notices (commonly cited under ECOA for credit decisions; this is an FS industry-practice extension, not a PDF-prescribed control). | +| Detection | Calls `sagemaker:ListMonitoringSchedules` with the `MonitoringTypeEquals=ModelExplainability` filter parameter. Flags if no explainability monitoring schedules exist for financial decision model endpoints. Clarify explainability monitoring publishes metrics to the `aws/sagemaker/Endpoints/explainability-metrics` namespace for real-time endpoints (and `aws/sagemaker/ModelMonitoring/explainability-metrics` for batch transform jobs) with `Endpoint`, `MonitoringSchedule`, `ExplainabilityMethod` (value: `KernelShap`), `Label`, and `ValueType` (values: `GlobalShapValues` or `ExpectedValue`) dimensions. | +| Remediation | 1. Create a SageMaker Clarify explainability monitoring schedule using SHAP analysis. 2. Configure feature attribution baselines. 3. Use explainability outputs to generate adverse action notices (top contributing factors for negative decisions) where your firm's use case and regulatory interpretation require them. 4. Retain explainability reports for regulatory audit. | +| Reference | [SageMaker Clarify Explainability](https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-model-explainability.html) | + +#### FS-42 — AI Service Cards + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.4, §1.2.5, §1.2.14] — "Amazon provides AI Service Cards for models that are pre-trained for AWS services like Amazon Bedrock and Amazon Q. These cards help you understand how Amazon addresses toxicity in each model." Referenced in three separate PDF risk sections. | +| Description | Checks SageMaker Model Cards document intended use and bias evaluations. | +| Detection | Calls `sagemaker:ListModelCards`. For each card, calls `sagemaker:DescribeModelCard` and inspects the content JSON for `intended_uses`, `business_details`, and `evaluation_details` sections. Flags cards missing these sections. | +| Remediation | 1. Create a SageMaker Model Card for each production model. 2. Document: intended use cases, out-of-scope uses, training data description, bias evaluation results, performance metrics. 3. Review and update cards after each model retrain. 4. For Bedrock foundation models, reference the AWS AI Service Cards published by Amazon. | +| Reference | [SageMaker Model Cards](https://docs.aws.amazon.com/sagemaker/latest/dg/model-cards.html), [AWS AI Service Cards](https://aws.amazon.com/ai/responsible-ai/resources/) | + +### Sensitive Information Disclosure (FS-43 to FS-46) + +> **PDF source:** §1.2.6 Sensitive information disclosure. PDF-listed mitigations: +> (a) Bedrock Guardrails sensitive information filters for PII, PHI; +> (b) data classification scanning and access controls on AI data sources; +> (c) strict IAM access controls for Bedrock API; +> (d) mask sensitive information in CloudWatch Logs and custom application logging; +> (e) protect training and fine-tuning data via data protection best practices; +> (f) monitor PII in training/fine-tuning/RAG data with Amazon Macie; +> (g) remove, mask, or tokenize PII before use in training, fine-tuning, or RAG; +> (h) Practical guidance: least privilege for agent identities; user-authorized communications +> to tool services; propagate end-user identities so tool services can validate them without +> revealing them to unauthorized third parties. + +#### FS-43 — CloudWatch Log PII Masking + +| Field | Detail | +|-------|--------| +| Severity | High | +| PDF ref | [PDF §1.2.6] — "If you implement model invocation logging for the LLM or custom logging logic in your application, make sure to mask sensitive information in your log data." References "Amazon CloudWatch – Help protect sensitive log data with masking". | +| Description | Checks CloudWatch Logs data protection policies mask PII in Bedrock invocation logs. | +| Detection | Identifies CloudWatch log groups used by Bedrock invocation logging (from `bedrock:GetModelInvocationLoggingConfiguration`). Calls `logs:GetDataProtectionPolicy` for each log group. Flags log groups with no data protection policy or policies missing PII identifiers. Note: model invocation logging only captures calls made through the `bedrock-runtime` endpoint (`Converse`, `ConverseStream`, `InvokeModel`, `InvokeModelWithResponseStream`); calls through other endpoints such as the Responses API (`bedrock-mantle` endpoint) are not captured. | +| Remediation | 1. Create a CloudWatch Logs data protection policy on each Bedrock log group. 2. Include managed data identifiers using their exact ARN-based IDs — country-code suffixes are **required** in the ARN for most identifiers (the data-types table uses the short name such as `Ssn`, but the ARN must include the country code): `Ssn-US` (US Social Security Number; `Ssn-ES` for Spain — there is no bare `Ssn` ARN), `CreditCardNumber` (no suffix), `CreditCardSecurityCode` (no suffix), `EmailAddress` (no suffix), `Address` (no suffix), `PhoneNumber-US`, `BankAccountNumber-US`, `DriversLicense-US`, `PassportNumber-US`, `IndividualTaxIdentificationNumber-US`. 3. Add a `Deidentify` operation statement (no hyphen — this is the exact JSON key required in the policy document, even though AWS prose documentation uses "De-identify") to mask sensitive data, and a separate `Audit` statement to emit findings to CloudWatch. The `Deidentify` operation must contain an empty `"MaskConfig": {}` object. 4. **Retroactive masking scope:** A **log group-level** data protection policy only masks data ingested **after** the policy is applied — historical log events are not retroactively masked. However, an **account-level** data protection policy applies to both existing log groups and log groups created in the future. For maximum coverage, consider creating an account-level policy in addition to log group-level policies. Apply policies at log group creation time or as early as possible. 5. Test by sending a log entry containing sample PII and verifying it is masked in subsequent reads. | +| Reference | [CloudWatch Logs Data Protection](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/mask-sensitive-log-data.html), [PII Data Identifier ARNs](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/protect-sensitive-log-data-types-pii.html), [Financial Data Identifier ARNs](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/protect-sensitive-log-data-types-financial.html) | + +#### FS-44 — Amazon Macie PII Scanning and Pre-Processing + +| Field | Detail | +|-------|--------| +| Severity | High | +| PDF ref | [PDF §1.2.6] — "Monitor personally identifiable information (PII) in your data when you train models, fine-tune them, or use retrieval-augmented generation (RAG)" and "Remove, mask, or tokenize personally identifiable information (PII) or sensitive data before you use it for training, fine-tuning, or retrieval-augmented generation (RAG)." | +| Description | Verifies Macie is enabled and scanning AI/ML data buckets, and checks that a PII pre-processing step (tokenization, masking, or removal) exists in training and RAG ingestion pipelines before data reaches the model. | +| Detection | Calls `macie2:GetMacieSession` to verify Macie is enabled. Calls `macie2:GetAutomatedDiscoveryConfiguration` to check whether automated sensitive data discovery is enabled (preferred over manual classification jobs — automated discovery evaluates S3 buckets daily without explicit job creation). Also calls `macie2:ListClassificationJobs` to check for any additional targeted jobs covering S3 buckets tagged for AI/ML use. Additionally inspects SageMaker Processing jobs or Glue jobs for PII-related naming patterns indicating a pre-processing pipeline. | +| Remediation | 1. Enable Amazon Macie in the account. 2. **Preferred:** Enable Macie **Automated Sensitive Data Discovery** (via `macie2:UpdateAutomatedDiscoveryConfiguration` set to `ENABLED`) — this continuously evaluates ALL S3 buckets in the account or organization daily, selects representative objects, and produces sensitive-data findings without requiring manual job creation. 3. For higher-priority AI/ML buckets where you need full-depth scans, supplement with targeted classification jobs (`macie2:CreateClassificationJob`) scheduled at least weekly. 4. Implement a PII pre-processing step in your data pipeline (SageMaker Processing job, Glue job, or Lambda) that tokenizes, masks, or removes PII before data is used for training or RAG ingestion. 5. Use Amazon Comprehend `DetectPiiEntities` or Macie findings to identify PII locations and feed them into the pre-processing step. 6. Route Macie findings to EventBridge and then to your SIEM or ticketing system for timely investigation. | +| Reference | [Amazon Macie](https://docs.aws.amazon.com/macie/latest/user/what-is-macie.html), [Macie Automated Sensitive Data Discovery](https://docs.aws.amazon.com/macie/latest/user/discovery-asdd.html), [Amazon Comprehend PII Detection](https://docs.aws.amazon.com/comprehend/latest/dg/pii.html) | + +#### FS-45 — Guardrail PII Filters + +| Field | Detail | +|-------|--------| +| Severity | High | +| PDF ref | [PDF §1.2.6] — "Use Amazon Bedrock Guardrails to detect and filter structured sensitive information in model inputs and outputs, such as personally identifiable information (PII), protected health information (PHI)." | +| Description | Checks guardrails have PII entity filters for SSN, credit card, and account numbers. | +| Detection | Calls `bedrock:GetGuardrail` and inspects `sensitiveInformationPolicy.piiEntities`. Flags guardrails missing filters for critical PII types: `US_SOCIAL_SECURITY_NUMBER`, `CREDIT_DEBIT_CARD_NUMBER`, `CREDIT_DEBIT_CARD_CVV`, `CREDIT_DEBIT_CARD_EXPIRY`, `US_BANK_ACCOUNT_NUMBER`, `US_BANK_ROUTING_NUMBER`, `PIN`, `SWIFT_CODE`, `INTERNATIONAL_BANK_ACCOUNT_NUMBER`, `US_INDIVIDUAL_TAX_IDENTIFICATION_NUMBER`, `EMAIL`, `PHONE`. | +| Remediation | 1. Update the guardrail to add PII entity filters for all relevant types. 2. Configure separate input and output actions using the `inputAction` and `outputAction` fields: set `outputAction=ANONYMIZE` (replace with placeholder such as `{US_SOCIAL_SECURITY_NUMBER}`) so PII in model responses is masked before reaching the user; set `inputAction=BLOCK` for PII types that should never be submitted (e.g., SSN, credit card numbers). 3. Use `inputEnabled` and `outputEnabled` to selectively enable evaluation per direction — disable evaluation on a direction you don't need to reduce cost and latency. 4. **PHI coverage nuance:** The Bedrock Guardrails sensitive information filter has only limited built-in PHI entities — specifically `CA_HEALTH_NUMBER` (Canada) and `UK_NATIONAL_HEALTH_SERVICE_NUMBER` (UK). For US HIPAA PHI (for example, Medical Record Numbers, Health Plan Beneficiary Numbers, Medicare Beneficiary Identifiers), there is no built-in entity type — use `regexesConfig` (custom regex patterns) on the guardrail to detect these patterns, complemented by downstream CloudWatch Logs data protection policies (see FS-43) which have PHI identifiers under the HIPAA category. 5. **Critical limitation — tool_use outputs:** The sensitive information filter does NOT detect PII when models respond with `tool_use` (function call) output parameters via supported APIs. For FinServ agentic applications where models invoke tools and return structured function-call responses, implement application-layer PII scanning on tool outputs before they are processed or displayed. 6. **Critical limitation — invocation logs:** Guardrail PII masking applies only to content sent to and returned from the inference model. It does NOT apply to model invocation logs — the `input` field in CloudWatch Logs always contains the original, unmasked request regardless of guardrail intervention. Use CloudWatch Logs data protection policies (see FS-43) to mask PII in logs separately. Similarly, the `match` field in guardrail trace output contains the original PII value, not the masked output. 7. Test with sample inputs containing each PII type and verify both input blocking and output anonymization work as expected. | +| Reference | [Bedrock Guardrails Sensitive Information Filters](https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-sensitive-filters.html) | + +#### FS-46 — Data Classification Tagging + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.6] — "Implement data classification scanning and access controls on the data sources connected to your AI system to prevent disclosure of company-confidential or proprietary information." | +| Description | Verifies AI/ML S3 buckets are tagged with data classification labels. | +| Detection | Lists S3 buckets and filters for AI/ML-related names or tags. Calls `s3:GetBucketTagging` for each and checks for a `data-classification` tag with values like `public`, `internal`, `confidential`, `restricted`. Flags buckets missing the tag. | +| Remediation | 1. Define a data classification taxonomy (e.g., Public, Internal, Confidential, Restricted). 2. Tag all AI/ML S3 buckets with `data-classification=`. 3. **Detective enforcement:** Create an AWS Config managed rule (`required-tags`, checks up to six tag keys at a time) to identify buckets missing the tag and trigger remediation via a custom SSM automation document (note: the AWS-managed `AWS-SetRequiredTags` automation document does NOT work as a remediation with this rule — you must author a custom Systems Manager automation document). 4. **Preventive enforcement:** Use AWS Organizations **Tag Policies** to require the `data-classification` tag key with allowed values (Public, Internal, Confidential, Restricted) across accounts — Tag Policies are preventive and complement the detective Config rule. 5. Use tag-based IAM policies (via condition keys `aws:ResourceTag/data-classification`) to restrict S3 access based on classification level. 6. Pair with Macie classification jobs (see FS-44) so that buckets automatically classified as containing sensitive data are flagged if their `data-classification` tag is missing or inconsistent with the Macie findings. | +| Reference | [AWS Tagging Best Practices](https://docs.aws.amazon.com/tag-editor/latest/userguide/tagging.html), [AWS Config required-tags Rule](https://docs.aws.amazon.com/config/latest/developerguide/required-tags.html), [AWS Organizations Tag Policies](https://docs.aws.amazon.com/organizations/latest/userguide/orgs_manage_policies_tag-policies.html) | + diff --git a/docs/SECURITY_CHECKS_FINSERV_PART3_APP_LAYER_AND_GAPS.md b/docs/SECURITY_CHECKS_FINSERV_PART3_APP_LAYER_AND_GAPS.md new file mode 100644 index 0000000..12a4290 --- /dev/null +++ b/docs/SECURITY_CHECKS_FINSERV_PART3_APP_LAYER_AND_GAPS.md @@ -0,0 +1,368 @@ +# FinServ GenAI Risk Checks — Part 3: Application-Layer Controls & Material Gaps (FS-47 to FS-69) + +This is **Part 3 of 3** of the FinServ GenAI security checks derived from the +[AWS guide for Financial Services risk management of the use of Generative AI (March 2026)](https://d1.awsstatic.com/onedam/marketing-channels/website/public/global-FinServ-ComplianceGuide-GenAIRisks-public.pdf) +(referred to throughout as "the FinServ Guide"). + +This part covers **22 standalone checks** across 6 PDF risk categories (FS-64 is merged into upstream BR-04 — see extension note in the Material Gaps section): + +- **Hallucination** (FS-47 to FS-50) — §1.2.7 +- **Prompt Injection** (FS-51 to FS-54) — §1.2.8 +- **Improper Output Handling** (FS-55 to FS-58) — §1.2.13 +- **Off-Topic & Inappropriate Output** (FS-59 to FS-60) — §1.2.2 +- **Out-of-Date Training Data** (FS-61 to FS-63) — §1.2.10 +- **Additional Controls — Material Gaps** (FS-64 to FS-69) — cross-cutting checks addressing PDF-listed mitigations not covered elsewhere; *FS-64 merged into upstream* + +**Companion files:** + +- `SECURITY_CHECKS_FINSERV_PART1_INFRA_CONTROLS.md` — FS-01 to FS-26 (Unbounded, Excessive Agency, Supply Chain, Training Poisoning, Vector Weaknesses) +- `SECURITY_CHECKS_FINSERV_PART2_GUARDRAILS_CONTENT_SAFETY.md` — FS-27 to FS-46 (Non-Compliant, Misinformation, Abusive, Biased, Sensitive Info) +- `SECURITY_CHECKS_FINSERV_COMMON.md` — shared intro, severity rubric, validation note, upstream-overlap table + +Each check includes how it is **detected** (the AWS API calls or configuration inspected) +and how a failure is **remediated** (the specific AWS actions to take). + +See `SECURITY_CHECKS_FINSERV_COMMON.md` for: + +- PDF traceability conventions (`[PDF §x.y.z]` vs `[PDF §x.y.z, extension]`) +- Severity rubric (High / Medium / Low / Advisory) +- Validation note and AWS service authorization references +- Relationship to upstream SM/BR/AC checks and consolidation recommendations + +--- + +## FinServ GenAI Risk Checks — Part 3 content + +### Hallucination (FS-47 to FS-50) + +> **PDF source:** §1.2.7 Hallucination. PDF-listed mitigations: +> (a) prompt engineering; +> (b) RAG with Bedrock Knowledge Bases; +> (c) detect hallucinations in RAG and agent-based systems; +> (d) HITL validation for internal AI systems; +> (e) Automated Reasoning checks in Bedrock Guardrails; +> (f) Bedrock Guardrails contextual grounding checks with reference source and query; +> (g) response disclaimers in customer-facing applications informing users that AI responses +> should be verified for critical decisions. + +#### FS-47 — Guardrail Grounding Threshold + +| Field | Detail | +|-------|--------| +| Severity | High | +| PDF ref | [PDF §1.2.7] — "You can use Amazon Bedrock Guardrails to detect and filter hallucinations in model responses by performing contextual grounding checks when you provide a reference source and query." | +| Description | Verifies guardrail grounding thresholds are set appropriately for financial use cases (this assessment recommends ≥ 0.7; AWS does not prescribe a specific minimum, but the valid range is 0 to 0.99). Note: contextual grounding checks are not supported for conversational chatbot use cases — only for summarization, paraphrasing, and Q&A. | +| Detection | Calls `bedrock:GetGuardrail` and inspects `contextualGroundingPolicy.filters` for the `GROUNDING` filter type. Checks that the `threshold` value is ≥ 0.7. Flags guardrails with lower thresholds or no grounding filter. | +| Remediation | 1. Update the guardrail to set the grounding filter threshold to at least 0.7 (this assessment recommends 0.8 for financial services to reduce hallucination risk — note: AWS does not prescribe a specific minimum, but the valid range is **0 to 0.99**; a value of 1.0 is explicitly invalid and will block all content per AWS documentation). 2. Enable the grounding filter for both the `GROUNDING` and `RELEVANCE` types. 3. Test with prompts that should and should not be grounded in the reference source — tune the threshold based on your false-positive/false-negative tolerance. 4. Monitor grounding filter invocation rates via CloudWatch using the `AWS/Bedrock/Guardrails` namespace. **Important limitation:** Contextual grounding checks support only summarization, paraphrasing, and question-answering use cases — **Conversational QA / Chatbot use cases are explicitly not supported** per AWS documentation. For FinServ chatbot deployments, use denied topics and content filters (FS-28, FS-36, FS-59) as the primary hallucination-mitigation controls instead. | +| Reference | [Bedrock Guardrails Contextual Grounding](https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-contextual-grounding-check.html) | + +#### FS-48 — RAG Knowledge Base + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.1, §1.2.7, §1.2.10] — "Use Retrieval-Augmented Generation (RAG) to enhance your model responses with information from trusted knowledge bases." Referenced in three separate PDF risk sections. | +| Description | Checks active Knowledge Bases are configured for RAG grounding. | +| Detection | Calls `ListKnowledgeBases` (via the `bedrock-agent` boto3 client; IAM action `bedrock:ListKnowledgeBases`) and checks that at least one KB exists with `status=ACTIVE`. Flags accounts with no active KBs when Bedrock models are in use (indicating responses are ungrounded). | +| Remediation | 1. Create a Bedrock Knowledge Base with your authoritative data sources. 2. Configure the KB with an appropriate embedding model and vector store. 3. Use `RetrieveAndGenerate` API instead of direct `InvokeModel` for customer-facing use cases. 4. Sync data sources on a regular schedule. | +| Reference | [Bedrock Knowledge Bases](https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base.html) | + +#### FS-49 — Hallucination Disclaimer + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.7] — "Implement response disclaimers in customer-facing applications, to inform end users that AI-generated responses should be verified for critical decisions." References "AWS Well-Architected Framework Generative AI Lens - Implement guardrails to mitigate harmful or incorrect model responses". | +| Description | Advisory: verifies application adds hallucination disclaimers to AI-generated outputs. | +| Detection | Advisory check — inspects application Lambda environment variables for disclaimer-related settings. Checks for post-processing Lambda functions that append disclaimers. | +| Remediation | 1. Add a standard disclaimer to all AI-generated responses: "This response is generated by AI and may contain inaccuracies. Please verify critical information independently." 2. Make the disclaimer configurable and non-removable by prompt manipulation. 3. For financial decisions, add: "This does not constitute financial advice." | +| Reference | [AWS Well-Architected GenAI Lens](https://docs.aws.amazon.com/wellarchitected/latest/generative-ai-lens/gensec02-bp01.html) | + +#### FS-50 — Relevance Grounding Filters + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.2, §1.2.7] — "Use Amazon Bedrock Guardrails to detect and filter hallucinations in model responses by performing contextual grounding checks." Contextual grounding covers both `GROUNDING` and `RELEVANCE` filter sub-types. | +| Description | Checks guardrails have relevance grounding filters to prevent off-topic responses. | +| Detection | Calls `bedrock:GetGuardrail` and inspects `contextualGroundingPolicy.filters` for the `RELEVANCE` filter type. Flags guardrails with no relevance filter configured. | +| Remediation | 1. Update the guardrail to enable the `RELEVANCE` contextual grounding filter. 2. Set the threshold to at least 0.7 (valid range is **0 to 0.99**; a value of 1.0 is explicitly invalid per AWS documentation). 3. This ensures responses are relevant to the user's query and the provided reference source, filtering out off-topic hallucinations. **Important limitation:** Contextual grounding checks (both `GROUNDING` and `RELEVANCE`) support only summarization, paraphrasing, and question-answering use cases — **Conversational QA / Chatbot use cases are explicitly not supported** per AWS documentation. For FinServ chatbot deployments, use denied topics (FS-59) as the primary off-topic control. | +| Reference | [Bedrock Guardrails Contextual Grounding](https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-contextual-grounding-check.html) | + +### Prompt Injection (FS-51 to FS-54) + +> **PDF source:** §1.2.8 Prompt injection. PDF-listed mitigations: +> (a) prompt engineering best practices to avoid prompt injection; +> (b) input validation — sanitize user input, remove special characters or use escape sequences, +> match expected format; +> (c) secure coding practices — parameterized queries, avoid string concatenation, minimal +> privileges; +> (d) security testing — regular testing for prompt injection and vulnerabilities, pentest, +> static code analysis, DAST; +> (e) stay updated — keep Bedrock SDK, libraries, and dependencies current; +> (f) Bedrock Guardrails to detect and block user inputs attempting to override system +> instructions through prompt attacks. + +#### FS-51 — Prompt Attack Filters + +| Field | Detail | +|-------|--------| +| Severity | High | +| PDF ref | [PDF §1.2.8] — "Use Amazon Bedrock Guardrails to detect and block user inputs that attempt to override system instructions through prompt attacks." | +| Description | Verifies guardrails have PROMPT_ATTACK content filters enabled and are configured correctly for the Standard tier. | +| Detection | Calls `bedrock:GetGuardrail` and inspects `contentPolicy.filters` for a filter with `type=PROMPT_ATTACK`. Flags guardrails where this filter is absent, has `inputStrength` set to `NONE` or `LOW` (note: PROMPT_ATTACK only applies to inputs — there is no `outputStrength` for this filter type), or where `contentPolicy.tier.tierName=CLASSIC` (the PROMPT_ATTACK filter in Classic tier detects jailbreaks and prompt injection; in Standard tier it additionally detects **prompt leakage** — attempts to extract system prompts or developer instructions). | +| Remediation | 1. Ensure the guardrail is configured with the **Standard** content filters tier — prompt leakage detection (extracting system prompts/developer instructions) is available only in Standard tier; jailbreak and prompt injection detection are available in both tiers. Standard tier requires cross-Region inference to be enabled on the guardrail. You can configure Standard tier on a **new or existing guardrail**: for an existing guardrail, modify it via `UpdateGuardrail` (set `tierConfig.tierName=STANDARD` in `contentPolicyConfig` and add a `crossRegionConfig.guardrailProfileIdentifier`), or use the console by editing the guardrail and selecting Standard tier with cross-Region inference. 2. Add a `PROMPT_ATTACK` content filter with `inputStrength=HIGH`. 3. **Wrap user input in guardrail input tags when using `InvokeModel` or `InvokeModelResponseStream`** — for these APIs, PROMPT_ATTACK only evaluates content enclosed in input tags (e.g., `user text` — the reserved prefix is `amazon-bedrock-guardrails-guardContent` and the suffix should be a unique random string per request to prevent an attacker from closing the tag and appending malicious content). Untagged content is not evaluated for PROMPT_ATTACK when using these APIs. **Note:** When using the `Converse` API, use the `guardContent` field (`GuardrailConverseContentBlock`) in user messages to scope PROMPT_ATTACK evaluation to specific content — this is the Converse API equivalent of input tags. Without `guardContent`, the guardrail evaluates ALL message content (the entire messages array). Using `guardContent` in user messages ensures only user-provided content is evaluated for prompt attacks, while system prompts and conversation history are excluded. If no `guardContent` blocks are present in messages, the guardrail evaluates everything in the messages array. 4. Test with known prompt injection patterns (role-play attacks, instruction override, delimiter injection). 5. Monitor filter invocation rates via CloudWatch guardrail metrics (`InvocationsIntervened` in the `AWS/Bedrock/Guardrails` namespace, filtered by `GuardrailPolicyType=ContentPolicy`) for trending attack patterns. | +| Reference | [Bedrock Guardrails Prompt Attack](https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-prompt-attack.html), [Safeguard tiers for guardrails](https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-tiers.html), [Securing Amazon Bedrock Agents against indirect prompt injections](https://aws.amazon.com/blogs/machine-learning/securing-amazon-bedrock-agents-a-guide-to-safeguarding-against-indirect-prompt-injections/) | + +#### FS-52 — Bedrock SDK Version Currency + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.8] — "Stay Updated – Keep your Amazon Bedrock SDK, libraries, and dependencies current to receive the latest security patches and updates." | +| Description | Checks Bedrock Lambda functions use current (non-deprecated) runtimes and SDK versions. | +| Detection | Calls `lambda:ListFunctions` and filters for functions with Bedrock-related names or environment variables referencing Bedrock. Checks each function's `Runtime` against the list of deprecated Lambda runtimes. | +| Remediation | 1. Update Lambda functions to use a currently supported runtime — as of April 2026, recommended runtimes are `python3.13` or `python3.14` for Python (both deprecation date June 30, 2029; `python3.12` remains supported through Oct 31, 2028), and `nodejs22.x` or `nodejs24.x` for Node.js (`nodejs20.x` reaches deprecation on April 30, 2026 and should not be used for new deployments). 2. Update the Bedrock SDK (boto3/botocore) to the latest version in your requirements.txt or package.json. 3. Test after upgrading to verify no breaking changes. 4. Subscribe to AWS Lambda runtime deprecation notifications via EventBridge or SNS (Lambda also surfaces runtime deprecation notices via AWS Health Dashboard and Trusted Advisor). | +| Reference | [Lambda Runtime Deprecation Policy](https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html) | + +#### FS-53 — WAF Injection Protection Rules + +| Field | Detail | +|-------|--------| +| Severity | High | +| PDF ref | [PDF §1.2.8, extension] — WAF SQLi and known-bad-inputs rule groups are not named in the PDF, but implement the PDF mitigation "Secure Coding Practices – use parameterized queries, avoid string concatenation for input, grant minimal access privileges" at the network edge for web-facing GenAI endpoints. | +| Description | Verifies WAF ACLs include SQL injection (`AWSManagedRulesSQLiRuleSet`) and known-bad-inputs (`AWSManagedRulesKnownBadInputsRuleSet`) managed rule groups for GenAI endpoints. | +| Detection | Calls `wafv2:ListWebACLs(Scope=REGIONAL)` and for each calls `wafv2:GetWebACL`. Inspects the rules list for `AWSManagedRulesSQLiRuleSet` and `AWSManagedRulesKnownBadInputsRuleSet`. Flags ACLs missing either rule group. | +| Remediation | 1. Add `AWSManagedRulesSQLiRuleSet` to your WAF Web ACL (contains SQLi detection rules for body, URI path, cookie, and query-string components). 2. Add `AWSManagedRulesKnownBadInputsRuleSet` for known Remote Command Execution (RCE) and vulnerability-discovery patterns (e.g., Log4j, Spring Core deserialization, path traversal) — note this rule group does NOT cover XSS; XSS is in `AWSManagedRulesCommonRuleSet` (see FS-56). 3. Set both rule groups to COUNT mode initially, review logs for false positives, then switch to BLOCK. 4. Create custom rules for GenAI-specific injection patterns if needed. | +| Reference | [AWS WAF Managed Rules](https://docs.aws.amazon.com/waf/latest/developerguide/aws-managed-rule-groups-list.html) | + +#### FS-54 — Penetration Testing Evidence + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.8] — "Security Testing – Test your applications regularly for prompt injection and other security vulnerabilities. Use penetration testing, static code analysis, and dynamic application security testing (DAST)." | +| Description | Advisory: verifies GenAI applications have been penetration tested for prompt injection and other AI-specific vulnerabilities. | +| Detection | Advisory check — inspects resource tags for `last-pentest-date` or checks for a documented penetration testing schedule. Cannot be fully automated. | +| Remediation | 1. Conduct penetration testing of your GenAI application at least annually and before major releases. 2. Include AI-specific test cases: prompt injection, jailbreak attempts, data extraction, system prompt leakage. 3. Use tools like Garak, PyRIT, manual red-teaming, or the **AWS Security Agent**. As of the March 2026 GA announcement, Security Agent runs from 6 AWS regions (N. Virginia, Oregon, Ireland, Frankfurt, Sydney, Tokyo) but can test targets across AWS, Azure, GCP, and on-premises environments. For multi-account FinServ deployments, Security Agent supports penetration testing on VPC resources **shared across AWS accounts in the same AWS Organization** via AWS Resource Access Manager (RAM) — enable this by launching Security Agent from a central security account and sharing VPC resources from sub-accounts via RAM. **Verify current region coverage on the [AWS Security Agent page](https://aws.amazon.com/security-agent/) before citing**, as AWS has been expanding regional availability and feature set rapidly. 4. Document findings and track remediation. 5. Tag resources with `last-pentest-date` for audit trail. | +| Reference | [AWS Penetration Testing Policy](https://aws.amazon.com/security/penetration-testing/), [AWS Security Agent GA](https://aws.amazon.com/about-aws/whats-new/2026/03/aws-security-agent-ondemand-penetration/) | + +### Improper Output Handling (FS-55 to FS-58) + +> **PDF source:** §1.2.13 Improper output handling. PDF-listed mitigations: +> (a) implement output validation rules against expected response format (e.g., JSON schema, +> SQL schema); +> (b) apply context-specific output sanitization — HTML encoding for web apps, SQL +> parameterization for database queries, command escaping for system integrations; +> (c) Practical guidance: treat model output as untrusted user input; use Bedrock Agents +> action-group Lambda to implement output encoding so output text is non-executable by +> JavaScript or Markdown. + +#### FS-55 — Output Validation Lambda + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.13] — "Implement output validation rules specific to the expected response format. For example, if the AI system is expected to return structured data (JSON, SQL), validate the output against the expected schema before processing." | +| Description | Checks for Lambda functions implementing output validation/sanitization before AI responses reach downstream consumers. | +| Detection | Calls `lambda:ListFunctions` and searches for functions with naming patterns indicating output validation (e.g., "output-valid", "sanitiz", "post-process", "response-filter"). Flags if no such functions exist. | +| Remediation | 1. Implement a post-processing Lambda that validates AI model output before it reaches the end user or downstream system. 2. Validate output against expected schema (JSON schema validation for structured responses). 3. Strip or escape any executable content (HTML tags, JavaScript, SQL fragments). 4. Log rejected outputs for security monitoring. | +| Reference | [AWS Well-Architected Security Pillar — Application Security](https://docs.aws.amazon.com/wellarchitected/latest/security-pillar/application-security.html), [Bedrock Prompt Injection Security](https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-injection.html), [Well-Architected FSI Lens — FSISEC14 Monitor AI system outputs for security issues](https://docs.aws.amazon.com/wellarchitected/latest/financial-services-industry-lens/fsisec14.html) | + +#### FS-56 — XSS Prevention WAF + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.13, extension] — WAF XSS rule groups are not named in the PDF, but implement the PDF mitigation "Apply context-specific output sanitization ... apply HTML encoding for web applications" at the network edge. | +| Description | Verifies WAF ACLs include XSS prevention rules to protect against AI-generated outputs containing malicious scripts. | +| Detection | Calls `wafv2:GetWebACL` for each regional ACL and inspects rules for `AWSManagedRulesCommonRuleSet` (which includes the four `CrossSiteScripting_*` rules covering request body, query arguments, cookies, and URI path) or custom rules using `XssMatchStatement` on request components. Flags ACLs missing XSS protection. | +| Remediation | 1. Add `AWSManagedRulesCommonRuleSet` to your WAF Web ACL (includes `CrossSiteScripting_COOKIE`, `CrossSiteScripting_QUERYARGUMENTS`, `CrossSiteScripting_BODY`, and `CrossSiteScripting_URIPATH` rules — all four inspect **inbound request** components). 2. `XssMatchStatement` and the CRS XSS rules inspect **request** components only (body, query string, URI path, cookies, headers). WAF does NOT inspect arbitrary response bodies for XSS — response inspection (`ResponseInspection`) is available only in `AWSManagedRulesATPRuleSet`/`AWSManagedRulesACFPRuleSet` for CloudFront-protected ACLs and only scans for configured success/failure strings. 3. To protect against XSS in **AI-generated output**, enforce output encoding at the application layer (see FS-57) — rendering raw model output in a browser without encoding is the root cause that WAF cannot mitigate after the fact. 4. Apply output encoding in your application layer as defense-in-depth. | +| Reference | [AWS WAF XSS Protection](https://docs.aws.amazon.com/waf/latest/developerguide/aws-managed-rule-groups-baseline.html) | + +#### FS-57 — Output Encoding + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.13] — "Apply context-specific output sanitization based on the downstream consumer. For example, apply HTML encoding for web applications, SQL parameterization for database queries, and command escaping for system integrations." Practical guidance: "Use Amazon Bedrock Agents to securely integrate with AWS native and third-party services and implement output encoding in the action group Lambda function under an Amazon Bedrock Agent. Encoding all output text presented to end-users makes it automatically non-executable by JavaScript or Markdown." | +| Description | Advisory: verifies application encodes GenAI outputs appropriately for the rendering context (HTML, JSON, SQL). | +| Detection | Advisory check — inspects application Lambda functions for encoding libraries or patterns (e.g., `html.escape`, `json.dumps`, `markupsafe`). Checks environment variables for encoding-related configuration. | +| Remediation | 1. Treat all model output as untrusted user input. 2. Apply context-specific encoding: HTML encoding for web display, SQL parameterization for database queries, command escaping for system integrations. 3. Use Bedrock Agents action-group Lambda functions to implement output encoding — encoding all output text makes it non-executable by JavaScript or Markdown renderers. 4. Never render raw model output in a web page without encoding. | +| Reference | [OWASP Output Encoding](https://cheatsheetseries.owasp.org/cheatsheets/Cross_Site_Scripting_Prevention_Cheat_Sheet.html) | + +#### FS-58 — Output Schema Validation + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.13] — "Implement output validation rules specific to the expected response format. For example, if the AI system is expected to return structured data (JSON, SQL), validate the output against the expected schema before processing." | +| Description | Checks for structured output validation in GenAI pipelines (JSON schema, XML schema, or custom validators). | +| Detection | Inspects Step Functions state machine definitions for states that perform schema validation (e.g., `Choice` states with JSON path conditions, Lambda states with "schema" or "validate" in the name). Does not rely on API Gateway response models as a validation signal because those are used for SDK generation, not runtime validation. | +| Remediation | 1. Define a JSON schema for expected AI output format. 2. Add a validation step in your pipeline (Lambda function or Step Functions Choice state) that rejects non-conforming outputs **before** returning the response to clients — this is the runtime enforcement point. 3. Note: API Gateway *response models* in REST APIs are used for SDK generation (user-defined data types) and documentation — they do NOT perform runtime validation of response payloads. API Gateway *request validators* only validate inbound requests against request models. To validate AI output at runtime, implement the check in Lambda/Step Functions before the response reaches API Gateway. 4. Return a safe fallback response when validation fails. 5. Log rejected outputs (without leaking sensitive content) for security monitoring. | +| Reference | [API Gateway Request and Response Validation](https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-method-request-validation.html) | + +### Off-Topic & Inappropriate Output (FS-59 to FS-60) + +> **PDF source:** §1.2.2 Off-topic and inappropriate output. PDF-listed mitigations: +> (a) prompt engineering with an allowlist of approved topics aligned with business purpose; +> (b) content filters and denied topics in Bedrock Guardrails; +> (c) Bedrock Guardrails contextual grounding check with reference source and query; +> (d) HITL validation for internal AI systems. + +#### FS-59 — Guardrail Topic Allowlist + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.2] — "Configure content filters and guardrails to restrict model responses to approved topics." The check name uses "allowlist" loosely — implementation uses denied-topic lists to block out-of-scope content. | +| Description | Verifies guardrails restrict GenAI to on-topic financial services responses via denied topics. | +| Detection | Calls `bedrock:GetGuardrail` and inspects `topicPolicy.topics`. Checks that denied topics exist to block off-topic conversations (e.g., politics, entertainment, medical advice). Flags guardrails with no topic restrictions. | +| Remediation | 1. Define denied topics that are outside your business scope (e.g., "medical advice", "legal advice", "political opinions", "entertainment recommendations"). 2. Add these as denied topics in the guardrail with clear descriptions and sample phrases. 3. Test with off-topic prompts to verify they are blocked. 4. Use the system prompt to positively scope the assistant's role. | +| Reference | [Bedrock Guardrails Topic Policies](https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-denied-topics.html) | + +#### FS-60 — Contextual Grounding for Off-Topic + +| Field | Detail | +|-------|--------| +| Severity | Low | +| PDF ref | [PDF §1.2.2] — "Use prompt engineering techniques to guide the model toward appropriate topics and prevent unwanted responses. Include an allowlist of approved topics aligned with the business purpose." Use of Bedrock Prompt Management for system prompt versioning is an implementation choice. | +| Description | Advisory: verifies system prompts explicitly scope the assistant's role to prevent off-topic responses. | +| Detection | Advisory check — inspects Bedrock Prompt Management templates (via `ListPrompts` on the `bedrock-agent` boto3 client; IAM action `bedrock:ListPrompts`) for system prompt content that defines the assistant's role, scope, and boundaries. Flags if no prompt templates exist. | +| Remediation | 1. Define a clear system prompt that states: the assistant's role, allowed topics, prohibited topics, and response format. 2. Use Bedrock Prompt Management to version and manage system prompts. 3. Include explicit instructions like "You are a financial services assistant. Only answer questions related to [specific topics]. Decline all other requests politely." 4. Test with boundary-case prompts. | +| Reference | [Bedrock Prompt Management](https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-management.html) | + +### Out-of-Date Training Data (FS-61 to FS-63) + +> **PDF source:** §1.2.10 Out-of-date training data. PDF-listed mitigations: +> (a) RAG with Bedrock Knowledge Bases; +> (b) keep knowledge bases up to date (sync data sources); +> (c) HITL validation for internal AI systems; +> (d) data currency disclaimers in AI system responses; source attribution via +> RetrieveAndGenerate API for users to verify currency. + +#### FS-61 — Knowledge Base Sync Schedule + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.10] — "Keep your knowledge bases up to date." Automated scheduling via EventBridge operationalises this mitigation. | +| Description | Checks EventBridge Scheduler or EventBridge rules automate KB data source sync on a regular schedule. | +| Detection | Calls `events:ListRules` and searches for rules with targets that invoke `StartIngestionJob` (IAM action `bedrock:StartIngestionJob`) or Lambda functions that trigger KB sync. Also checks AWS Scheduler (`scheduler:ListSchedules`) for schedules targeting KB sync. Flags if no scheduled sync mechanism exists. | +| Remediation | 1. Use **EventBridge Scheduler** (the current recommended approach — EventBridge scheduled rules are a legacy feature) to create a recurring schedule that triggers KB data source sync: create a schedule with a rate expression (e.g., `rate(1 day)`) or cron expression (e.g., `cron(0 2 * * ? *)`) targeting a Lambda function. 2. The Lambda function calls `StartIngestionJob` (IAM action `bedrock:StartIngestionJob`) for each data source. 3. Add error handling and CloudWatch alarms for failed syncs. | +| Reference | [EventBridge Scheduler](https://docs.aws.amazon.com/scheduler/latest/UserGuide/what-is-scheduler.html), [EventBridge Scheduled Rules (legacy)](https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-create-rule-schedule.html) | + +#### FS-62 — Data Currency Disclaimer + +| Field | Detail | +|-------|--------| +| Severity | Low | +| PDF ref | [PDF §1.2.10] — "Include data currency disclaimers in AI system responses where appropriate. Use source attribution in RAG-based response for end users to verify currency of information." | +| Description | Advisory: verifies application adds data currency disclaimers to AI-generated outputs. | +| Detection | Advisory check — inspects application configuration for data-currency disclaimer settings. Checks system prompts for instructions to include data freshness information. | +| Remediation | 1. Add a data currency disclaimer to responses: "This information is based on data available as of [date]. It may not reflect the most recent changes." 2. Use the `RetrieveAndGenerate` API's source attribution to display document dates. 3. Configure the system prompt to instruct the model to caveat time-sensitive information. | +| Reference | [Bedrock RetrieveAndGenerate API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_agent-runtime_RetrieveAndGenerate.html) | + +#### FS-63 — Foundation Model Lifecycle Policy + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.10, extension] — FM currency is conceptually related to "out-of-date training data" but the specific Bedrock lifecycle-status check is not named in the PDF. The PDF's "1.1.6 Monitor and improve" general guidance says "Update your foundation models when new versions become available" — this FS check operationalises that guidance. See also FS-34 (TPRM) which the PDF places under §1.2.12. | +| Description | Checks for a model lifecycle management process and Config rules to ensure models are updated when new versions are available. | +| Detection | Calls `config:DescribeConfigRules` and searches for rules targeting Bedrock resources. Calls `bedrock:GetFoundationModel` for each model in use and inspects `modelLifecycle.status`. Flags models with status `LEGACY` (note: the Bedrock API exposes only two lifecycle status values — `ACTIVE` and `LEGACY`; models past their `endOfLifeTime` are removed from the service entirely and return a ResourceNotFound error, so any model still reachable via the API that is not `ACTIVE` will be `LEGACY`). | +| Remediation | 1. Create an AWS Config custom rule that flags Bedrock models with `modelLifecycle.status=LEGACY`. 2. Establish a model lifecycle policy: evaluate new model versions within 30 days of release, test in staging, migrate production within 90 days (and before the `endOfLifeTime` published in the Bedrock model lifecycle page). 3. Subscribe to AWS Bedrock model lifecycle notifications. 4. Document the policy and assign an owner. 5. **Budget planning for FinServ:** For models with EOL dates after February 1, 2026, after a minimum of 3 months in Legacy state a model enters a **public extended access period** during which the model provider may set higher pricing. The `publicExtendedAccessTime` timestamp in the `FoundationModelLifecycle` response indicates when this phase begins. Include this phase in contract-and-budget review so FinServ cost governance teams are aware of potential price changes before migrating off Legacy models. | +| Reference | [Bedrock Model Lifecycle](https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle.html) | + +### Additional Controls — Material Gaps (FS-64 to FS-69) + +These checks address mitigations explicitly called out in the AWS FinServ Guide that were +not covered by the original checks in the upstream AIML Security Assessment (BR/SM/AC). +FS-64 is merged into upstream BR-04 (see extension note below); FS-65 to FS-69 ship as +standalone checks. + +#### FS-64 — Guardrail Trace Logging → *Merged into upstream BR-04* + +> **Upstream extension note (do not ship as a standalone check):** The detection and remediation +> content from FS-64 should be added as a refinement of the existing **BR-04 (Model Invocation +> Logging)** check in the upstream repo. +> +> **What to add to BR-04:** +> - After verifying that `bedrock:GetModelInvocationLoggingConfiguration` shows logging is +> enabled, additionally verify the log output captures **guardrail trace data**: when +> guardrails are applied during inference, the invocation log contains a `guardrailTrace` +> object with `action` (values: `INTERVENED` or `NONE`), `inputAssessments`, and +> `outputAssessments` arrays detailing which policies were evaluated and their results. +> - **Important logging coverage gap:** Model invocation logging only captures calls made through the `bedrock-runtime` endpoint (`Converse`, `ConverseStream`, `InvokeModel`, `InvokeModelWithResponseStream`). Calls made through the `bedrock-mantle` endpoint (e.g., the Responses API) are **not currently captured** by invocation logging. If your application uses the Responses API, implement application-level logging as a compensating control. +> - Add a remediation note on **retention requirements**: NYDFS 23 NYCRR 500.06 explicitly +> requires cybersecurity records for ≥ 5 years; SR 11-7 does not prescribe a specific period +> but requires documentation be maintained for the duration of model use plus a reasonable +> period thereafter (commonly met with 5–7 year retention per firm policy). Consult your +> compliance and records-management team for exact requirements. +> - Suggest creating CloudWatch Metrics filters to track guardrail intervention rates (filter +> on `guardrailTrace.action = INTERVENED`) and applying CloudWatch Logs data protection +> policies to mask PII in traces. +> - PDF traceability: [PDF §1.2.1] — "Maintain audit logs of AI-generated outputs and the +> guardrails applied to support regulatory reporting and post-incident analysis." Also +> §1.2.9 — "Implement audit logging of all actions taken by AI agents." +> +> **Reference:** [Bedrock Model Invocation Logging](https://docs.aws.amazon.com/bedrock/latest/userguide/model-invocation-logging.html) + +#### FS-65 — KB Data Source S3 Event Notifications + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.3] — "Use integrity monitoring on knowledge base data sources to detect unauthorized modifications... For example on S3 data sources use Amazon S3 event notification to track changes to documents." **Note:** This check overlaps with FS-33; FS-33 verifies notifications are *enabled* on the bucket, while FS-65 verifies that notifications are *routed to an alerting destination* (SNS/Lambda/EventBridge rule with a target). In the final PR to aws-samples these two checks may be consolidated into a single check at the reviewer's discretion. | +| Description | Checks that S3 event notifications on KB data-source buckets are routed to an alerting destination (EventBridge rule with SNS/Lambda target, or direct SNS/SQS/Lambda notification) — not just enabled with no consumer. | +| Detection | Identifies KB data-source S3 buckets via `ListDataSources` and `GetDataSource` (via the `bedrock-agent` boto3 client; IAM actions `bedrock:ListDataSources` and `bedrock:GetDataSource`). For each bucket, calls `s3:GetBucketNotificationConfiguration` and checks for the presence of `EventBridgeConfiguration`, `TopicConfigurations`, `QueueConfigurations`, or `LambdaFunctionConfigurations`. Flags buckets with no notifications configured. | +| Remediation | 1. Enable EventBridge notifications on each KB data-source bucket: `aws s3api put-bucket-notification-configuration --bucket --notification-configuration '{"EventBridgeConfiguration":{}}'`. 2. Create an EventBridge rule matching S3 event detail types `"Object Created"` and `"Object Deleted"` for the bucket (note: when S3 sends events to **EventBridge**, the event detail types are `Object Created`/`Object Deleted`; the `s3:ObjectCreated:*` and `s3:ObjectRemoved:*` wildcard names are used only for **direct** SNS/SQS/Lambda notification configurations, not for EventBridge rule patterns). 3. Route events to an SNS topic or Lambda function for alerting. 4. Integrate alerts into your security incident response workflow. | +| Reference | [S3 EventBridge Integration](https://docs.aws.amazon.com/AmazonS3/latest/userguide/EventBridge.html) | + +#### FS-66 — AgentCore End-User Identity Propagation + +| Field | Detail | +|-------|--------| +| Severity | High | +| PDF ref | [PDF §1.2.6 — Practical guidance] — "1. Implement least privilege for identities associated with agents and tool services. 2. Where supported by the tool service ensure that communications to tool services or agents are authorized by the end user. 3. Customers building their own tool services should consider propagating end-user identities separately; ensuring these identities can be validated and are not revealed to unauthorized third parties." | +| Description | Verifies AgentCore runtimes are configured to propagate end-user identities to downstream tool services, ensuring tool calls are authorized by the originating user and not solely by the agent execution role. | +| Detection | Calls `ListAgentRuntimes` (via the `bedrock-agentcore-control` boto3 client; IAM action `bedrock-agentcore:ListAgentRuntimes`) and inspects each runtime's `authorizerConfiguration.customJWTAuthorizer` for a `discoveryUrl` and allowed audiences/clients/scopes. Flags runtimes with no JWT authorizer (meaning inbound calls carry no verifiable end-user identity), and advises configuring outbound OAuth for downstream tool services. | +| Remediation | 1. Configure a custom JWT inbound authorizer on each AgentCore runtime: specify `discoveryUrl`, `allowedAudience`, `allowedClients`, and optional required custom claims. 2. Propagate the end-user's identity via the `X-Amzn-Bedrock-AgentCore-Runtime-User-Id` header and JWT token in the `Authorization` header when calling downstream tool services. **Important:** Invoking `InvokeAgentRuntime` with the `X-Amzn-Bedrock-AgentCore-Runtime-User-Id` header requires the distinct IAM action `bedrock-agentcore:InvokeAgentRuntimeForUser` in addition to `bedrock-agentcore:InvokeAgentRuntime`. Only trusted principals should hold this permission — scope it to specific runtime resources with IAM resource conditions, never via wildcard. For runtimes that do not need user-id delegation, explicitly **deny** `bedrock-agentcore:InvokeAgentRuntimeForUser` to prevent the header from being accepted. Additionally, derive the user-id from the authenticated principal's context (IAM caller identity or JWT claims) rather than from arbitrary client-supplied values to prevent user impersonation, and log the relationship between the authenticated IAM principal (via CloudTrail's SigV4 context) and the `user-id` value passed. 3. Configure outbound OAuth 2.0 for agents accessing third-party resources on behalf of the user. 4. Ensure tool services validate the propagated JWT before executing actions. 5. Implement agent identity segregation: assign distinct identities to each sub-agent in multi-agent workflows so actions are separately attributable. 6. Apply a maker-checker pattern for critical financial actions — require a second agent or human to verify before execution. 7. Do not log or expose propagated identity tokens to unauthorized third parties. | +| Reference | [Configure Inbound JWT Authorizer](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/inbound-jwt-authorizer.html), [Inbound and Outbound Auth](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/runtime-oauth.html) | + +#### FS-67 — Agent Financial Transaction Value Thresholds + +| Field | Detail | +|-------|--------| +| Severity | High | +| PDF ref | [PDF §1.2.9] — "Enforce transaction value thresholds and action boundaries on agent tool calls (for example to cap financial transaction amounts)." | +| Description | Checks AgentCore Policy Engine (attached to Gateways) or action-group Lambda functions enforce maximum transaction-value limits (e.g., cap on financial amounts an agent can initiate) to prevent runaway or unauthorized high-value transactions. | +| Detection | (a) Calls `ListGateways` (via the `bedrock-agentcore-control` boto3 client; IAM action `bedrock-agentcore:ListGateways`) and for each inspects attached Policy Engine Cedar policies for transaction-value constraints (policies referencing amount, limit, or threshold context attributes). (b) Calls `lambda:ListFunctions` and filters for agent action-group Lambda functions. Inspects each function's environment variables for threshold-related keys (e.g., `MAX_TRANSACTION_AMOUNT`, `TRANSACTION_LIMIT`). Flags gateways and functions with no threshold configuration. | +| Remediation | 1. Add transaction-value threshold environment variables to each agent action-group Lambda (e.g., `MAX_TRANSACTION_AMOUNT=10000`). 2. Implement threshold enforcement logic in the Lambda handler that rejects or escalates transactions exceeding the limit. 3. Author Cedar policies in the AgentCore Policy Engine that evaluate tool-call context attributes (amount, currency, tool) and deny calls exceeding defined limits. 4. Route transactions exceeding thresholds to a human-in-the-loop approval step via Step Functions callback pattern. | +| Reference | [Policy in AgentCore](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/policy.html), [AgentCore Example Policies](https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/example-policies.html) | + +#### FS-68 — API Gateway Request Body Size Limits + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.11] — "To protect your API endpoints, set maximum length limits for input requests when you use large language models (LLMs) directly or through Amazon Bedrock." | +| Description | Verifies API Gateway REST/HTTP APIs fronting GenAI endpoints have WAF `SizeConstraintStatement` rules enforcing a maximum request body size, optionally paired with an API Gateway request-body JSON schema that bounds individual field lengths — to prevent token-exhaustion attacks via oversized prompts. | +| Detection | Calls `apigateway:GetRestApis` and for each calls `apigateway:GetRequestValidators` to check for validators (validators enforce parameter-existence and request-body JSON schema conformance — not total body size). Calls `wafv2:GetWebACL` for associated ACLs and inspects rules for `SizeConstraintStatement` targeting the request body. Flags APIs with no WAF `SizeConstraintStatement` on body, since that is the only AWS-native mechanism that enforces a custom maximum body size in front of API Gateway. | +| Remediation | 1. **Primary control — WAF `SizeConstraintStatement`:** Add a WAF `SizeConstraintStatement` rule on your regional Web ACL that blocks requests whose body size exceeds your maximum allowed prompt length (e.g., 32 KB). Verify that the Web ACL's `AssociationConfig.RequestBody.DefaultSizeInspectionLimit` is set high enough (16 KB default; can be increased to 32/48/64 KB) so WAF can actually inspect bodies at the size you are enforcing against — if the inspection limit is lower than the `SizeConstraintStatement` threshold, oversized requests fall through to oversize handling instead of the rule. This is the only AWS-native way to enforce a custom maximum body size before requests reach API Gateway. 2. **Secondary control — API Gateway request validation:** Add an API Gateway request validator with a request-body model (JSON schema). Request validators do **not** enforce total body size, but a JSON schema can constrain individual string fields with `maxLength` and arrays with `maxItems`, which indirectly bounds payload content. Note API Gateway REST APIs also enforce a service-level hard limit of 10 MB per request (6 MB when integrated with Lambda) that you cannot lower. 3. Set the `max_tokens` parameter in Bedrock API calls to cap output length. 4. Implement client-side token counting before submitting requests. | +| Reference | [WAF Size Constraint](https://docs.aws.amazon.com/waf/latest/developerguide/waf-rule-statement-type-size-constraint-match.html), [WAF Body Inspection Size Limit](https://docs.aws.amazon.com/waf/latest/developerguide/web-acl-setting-body-inspection-limit.html), [API Gateway Request Validation](https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-method-request-validation.html) | + +#### FS-69 — Prompt Input Validation Function + +| Field | Detail | +|-------|--------| +| Severity | Medium | +| PDF ref | [PDF §1.2.8] — "Input Validation – Before you send user input to Amazon Bedrock or the tokenizer, validate and sanitize it by removing special characters or using escape sequences. Make sure the input matches your expected format." | +| Description | Checks for a Lambda function or API Gateway request validator that sanitizes user prompt input (strips special characters, enforces expected format, rejects oversized inputs) before forwarding to Bedrock, complementing WAF-level controls. | +| Detection | Calls `lambda:ListFunctions` and searches for functions with input-validation naming patterns (e.g., "sanitiz", "validat", "input-filter", "prompt-guard", "preprocess"). Flags if no such functions exist. | +| Remediation | 1. Implement a Lambda authorizer or pre-processing function that: strips or escapes special characters from user input; validates input against an expected format (e.g., regex allowlist); rejects inputs exceeding maximum token/character limits; logs rejected inputs for security monitoring. 2. Use parameterized prompt templates (Bedrock Prompt Management) instead of string concatenation. 3. Apply Bedrock Guardrails PROMPT_ATTACK filter as a complementary control. 4. Integrate the validation function as an API Gateway Lambda authorizer or Step Functions pre-processing step. 5. Implement schema validation for all tool interactions — validate both inputs to and outputs from tools against defined JSON schemas per AWS Prescriptive Guidance for tool integration security. 6. Enforce TLS for all remote tool communications. | +| Reference | [Bedrock Prompt Injection Security](https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-injection.html), [Security Best Practices for Tool Integration](https://docs.aws.amazon.com/prescriptive-guidance/latest/agentic-ai-frameworks/security-best-practices-for-tool-integration.html) | + +--- + +*See `SECURITY_CHECKS_FINSERV_COMMON.md` for the Compliance Framework Mapping table that applies to all 69 FS checks.* From 85afca91932017188852e21952bf9be1bb6ba5d8 Mon Sep 17 00:00:00 2001 From: Neil Mehta Date: Fri, 1 May 2026 12:42:49 -0400 Subject: [PATCH 02/16] fix(finserv): Handle per-agent and per-bucket ClientErrors gracefully FS-07: Wrap get_agent() in try/except ClientError so agents encrypted with KMS keys the caller cannot decrypt are skipped rather than crashing the entire check with ERROR status. FS-33: Wrap get_bucket_versioning() in try/except ClientError so deleted or cross-account KB data-source buckets are recorded as '(access error)' and the check continues rather than returning ERROR. Both fixes follow the defensive pattern already used in FS-65's get_bucket_notification_configuration() call. Discovered and verified in Phase 2 live integration testing against account 469898429403 (us-east-1). sim: https://github.com/aws-samples/sample-aiml-security-assessment/issues/22 --- .../security/finserv_assessments/app.py | 249 ++++++------------ 1 file changed, 80 insertions(+), 169 deletions(-) diff --git a/aiml-security-assessment/functions/security/finserv_assessments/app.py b/aiml-security-assessment/functions/security/finserv_assessments/app.py index f036618..9665357 100644 --- a/aiml-security-assessment/functions/security/finserv_assessments/app.py +++ b/aiml-security-assessment/functions/security/finserv_assessments/app.py @@ -1,3 +1,4 @@ + """ AWS FinServ GenAI Risk Assessment Lambda ========================================= @@ -70,7 +71,6 @@ # Helpers # --------------------------------------------------------------------------- - def get_permissions_cache(execution_id: str) -> Optional[Dict[str, Any]]: """Retrieve IAM permissions cache from S3 (same pattern as other assessments).""" try: @@ -106,7 +106,6 @@ def _error_findings(check_name: str, err: Exception) -> Dict[str, Any]: # COMPLIANCE_PLACEHOLDER: [FFIEC CAT, DORA Art.6, SR 11-7 Appendix A] # =========================================================================== - def check_waf_shield_on_bedrock_endpoints() -> Dict[str, Any]: """ FS-01 — Verify AWS WAF is associated with API Gateway or ALB endpoints @@ -287,7 +286,9 @@ def check_bedrock_token_quotas() -> Dict[str, Any]: # Flag if no custom quota increases have been requested (still at default) default_only = all(not q.get("Adjustable") for q in tpm_quotas + rpm_quotas) - details = f"Found {len(tpm_quotas)} token-based and {len(rpm_quotas)} request-based Bedrock quotas." + details = ( + f"Found {len(tpm_quotas)} token-based and {len(rpm_quotas)} request-based Bedrock quotas." + ) findings["csv_data"].append( create_finding( check_id="FS-03", @@ -321,8 +322,7 @@ def check_cost_anomaly_detection() -> Dict[str, Any]: monitors = ce.get_anomaly_monitors().get("AnomalyMonitors", []) bedrock_monitors = [ - m - for m in monitors + m for m in monitors if "bedrock" in json.dumps(m.get("MonitorSpecification", {})).lower() or m.get("MonitorType") == "DIMENSIONAL" ] @@ -379,15 +379,13 @@ def check_cloudwatch_token_alarms() -> Dict[str, Any]: all_alarms.extend(page.get("MetricAlarms", [])) bedrock_alarms = [ - a - for a in all_alarms + a for a in all_alarms if a.get("Namespace", "").startswith("AWS/Bedrock") or "bedrock" in a.get("AlarmName", "").lower() ] throttle_alarms = [ - a - for a in bedrock_alarms + a for a in bedrock_alarms if "throttl" in a.get("MetricName", "").lower() or "throttl" in a.get("AlarmName", "").lower() ] @@ -448,8 +446,7 @@ def check_aws_budgets_for_aiml() -> Dict[str, Any]: "Budgets", [] ) aiml_budgets = [ - b - for b in all_budgets + b for b in all_budgets if any( svc in json.dumps(b.get("CostFilters", {})).lower() for svc in ["bedrock", "sagemaker"] @@ -499,7 +496,6 @@ def check_aws_budgets_for_aiml() -> Dict[str, Any]: # COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, DORA Art.6, MAS TRM 9] # =========================================================================== - def check_bedrock_agent_action_boundaries(permission_cache) -> Dict[str, Any]: """ FS-07 — Verify Bedrock agent execution roles have narrow action boundaries @@ -531,17 +527,17 @@ def check_bedrock_agent_action_boundaries(permission_cache) -> Dict[str, Any]: for agent_summary in agents: agent_id = agent_summary["agentId"] agent_name = agent_summary["agentName"] - detail = bedrock_agent.get_agent(agentId=agent_id) + try: + detail = bedrock_agent.get_agent(agentId=agent_id) + except ClientError as e: + logger.warning(f"Could not describe agent {agent_name}: {e}") + continue role_arn = detail.get("agent", {}).get("agentResourceRoleArn", "") if not role_arn: continue role_name = role_arn.split("/")[-1] - role_perms = ( - (permission_cache or {}).get("role_permissions", {}).get(role_name, {}) - ) - for policy in role_perms.get("attached_policies", []) + role_perms.get( - "inline_policies", [] - ): + role_perms = (permission_cache or {}).get("role_permissions", {}).get(role_name, {}) + for policy in role_perms.get("attached_policies", []) + role_perms.get("inline_policies", []): doc = policy.get("document", {}) if isinstance(doc, str): doc = json.loads(doc) @@ -693,11 +689,8 @@ def check_agent_transaction_limits() -> Dict[str, Any]: # Look for agent-related Lambda functions without reserved concurrency agent_lambdas = [ - f - for f in functions - if any( - kw in f["FunctionName"].lower() for kw in ["agent", "bedrock", "aiml"] - ) + f for f in functions + if any(kw in f["FunctionName"].lower() for kw in ["agent", "bedrock", "aiml"]) ] lambdas_without_concurrency = [] @@ -762,12 +755,8 @@ def check_human_in_the_loop_for_high_risk_actions() -> Dict[str, Any]: machines = sfn.list_state_machines().get("stateMachines", []) agent_machines = [ - m - for m in machines - if any( - kw in m["name"].lower() - for kw in ["agent", "approval", "human", "review"] - ) + m for m in machines + if any(kw in m["name"].lower() for kw in ["agent", "approval", "human", "review"]) ] machines_with_wait = [] @@ -848,8 +837,7 @@ def check_agent_rate_alarms() -> Dict[str, Any]: all_alarms.extend(page.get("MetricAlarms", [])) agent_alarms = [ - a - for a in all_alarms + a for a in all_alarms if "agent" in a.get("AlarmName", "").lower() or "agent" in a.get("Namespace", "").lower() ] @@ -898,7 +886,6 @@ def check_agent_rate_alarms() -> Dict[str, Any]: # COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, DORA Art.6, ISO 27001 A.15] # =========================================================================== - def check_scp_model_access_restrictions() -> Dict[str, Any]: """ FS-12 — Verify SCPs restrict Bedrock model access to an approved model list, @@ -913,9 +900,7 @@ def check_scp_model_access_restrictions() -> Dict[str, Any]: "Policies", [] ) except ClientError as e: - if "AccessDenied" in str(e) or "AWSOrganizationsNotInUseException" in str( - e - ): + if "AccessDenied" in str(e) or "AWSOrganizationsNotInUseException" in str(e): findings["csv_data"].append( create_finding( check_id="FS-12", @@ -991,9 +976,7 @@ def check_model_inventory_tagging() -> Dict[str, Any]: # Check Bedrock custom models for model in bedrock.list_custom_models().get("modelSummaries", []): - tags_response = bedrock.list_tags_for_resource( - resourceARN=model["modelArn"] - ) + tags_response = bedrock.list_tags_for_resource(resourceARN=model["modelArn"]) tag_keys = {t["key"].lower() for t in tags_response.get("tags", [])} missing = REQUIRED_TAGS - tag_keys if missing: @@ -1060,8 +1043,7 @@ def check_model_onboarding_governance() -> Dict[str, Any]: rules = config.describe_config_rules().get("ConfigRules", []) bedrock_rules = [ - r - for r in rules + r for r in rules if "bedrock" in r.get("ConfigRuleName", "").lower() or "model" in r.get("ConfigRuleName", "").lower() ] @@ -1309,12 +1291,8 @@ def check_training_data_s3_versioning() -> Dict[str, Any]: buckets = s3.list_buckets().get("Buckets", []) training_buckets = [ - b - for b in buckets - if any( - kw in b["Name"].lower() - for kw in ["train", "dataset", "model", "sagemaker", "bedrock"] - ) + b for b in buckets + if any(kw in b["Name"].lower() for kw in ["train", "dataset", "model", "sagemaker", "bedrock"]) ] if not training_buckets: @@ -1379,7 +1357,6 @@ def check_training_data_s3_versioning() -> Dict[str, Any]: # COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, NYDFS 500.06, PCI-DSS 12.3.2] # =========================================================================== - def check_knowledge_base_iam_least_privilege(permission_cache) -> Dict[str, Any]: """ FS-22 — Verify IAM roles accessing Bedrock Knowledge Bases follow @@ -1389,12 +1366,8 @@ def check_knowledge_base_iam_least_privilege(permission_cache) -> Dict[str, Any] findings = _empty_findings("Knowledge Base IAM Least Privilege Check") try: issues = [] - for role_name, perms in ( - (permission_cache or {}).get("role_permissions", {}).items() - ): - for policy in perms.get("attached_policies", []) + perms.get( - "inline_policies", [] - ): + for role_name, perms in (permission_cache or {}).get("role_permissions", {}).items(): + for policy in perms.get("attached_policies", []) + perms.get("inline_policies", []): doc = policy.get("document", {}) if isinstance(doc, str): doc = json.loads(doc) @@ -1637,7 +1610,6 @@ def check_knowledge_base_vpc_access() -> Dict[str, Any]: # COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, NYDFS 500, MAS TRM 9.2] # =========================================================================== - def check_automated_reasoning_checks() -> Dict[str, Any]: """ FS-27 — Check whether Bedrock Guardrails have Automated Reasoning checks @@ -1861,7 +1833,6 @@ def check_bedrock_evaluation_compliance_datasets() -> Dict[str, Any]: # COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2, NYDFS 500] # =========================================================================== - def check_knowledge_base_data_source_sync() -> Dict[str, Any]: """ FS-31 — Verify Bedrock Knowledge Base data sources have recent sync jobs @@ -2019,9 +1990,13 @@ def check_knowledge_base_integrity_monitoring() -> Dict[str, Any]: ) bucket = s3_config.get("bucketArn", "").split(":::")[-1] if bucket: - versioning = s3.get_bucket_versioning(Bucket=bucket) - if versioning.get("Status") != "Enabled": - buckets_without_versioning.append(bucket) + try: + versioning = s3.get_bucket_versioning(Bucket=bucket) + if versioning.get("Status") != "Enabled": + buckets_without_versioning.append(bucket) + except ClientError as e: + logger.warning(f"Could not check versioning for bucket {bucket}: {e}") + buckets_without_versioning.append(f"{bucket} (access error)") if buckets_without_versioning: findings["status"] = "WARN" @@ -2068,13 +2043,12 @@ def check_fm_version_currency() -> Dict[str, Any]: findings = _empty_findings("Foundation Model Version Currency Check") try: bedrock = boto3.client("bedrock", config=boto3_config) - models = bedrock.list_foundation_models(byOutputModality="TEXT").get( - "modelSummaries", [] - ) + models = bedrock.list_foundation_models( + byOutputModality="TEXT" + ).get("modelSummaries", []) deprecated = [ - m["modelId"] - for m in models + m["modelId"] for m in models if m.get("modelLifecycle", {}).get("status") == "LEGACY" ] @@ -2121,7 +2095,6 @@ def check_fm_version_currency() -> Dict[str, Any]: # COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2, NYDFS 500] # =========================================================================== - def check_fmeval_harmful_content() -> Dict[str, Any]: """ FS-35 — Check for FMEval or Bedrock Evaluation jobs testing for harmful @@ -2295,9 +2268,7 @@ def check_guardrail_word_filters() -> Dict[str, Any]: detail = bedrock.get_guardrail( guardrailIdentifier=g["id"], guardrailVersion="DRAFT" ) - if detail.get("wordPolicy", {}).get("words") or detail.get( - "wordPolicy", {} - ).get("managedWordLists"): + if detail.get("wordPolicy", {}).get("words") or detail.get("wordPolicy", {}).get("managedWordLists"): guardrails_with_words.append(g["name"]) if not guardrails_with_words: @@ -2347,12 +2318,11 @@ def check_sagemaker_clarify_bias() -> Dict[str, Any]: findings = _empty_findings("SageMaker Clarify Bias Check") try: sm = boto3.client("sagemaker", config=boto3_config) - schedules = sm.list_monitoring_schedules().get( - "MonitoringScheduleSummaries", [] - ) + schedules = sm.list_monitoring_schedules().get("MonitoringScheduleSummaries", []) bias_schedules = [ - s for s in schedules if s.get("MonitoringType") == "ModelBias" + s for s in schedules + if s.get("MonitoringType") == "ModelBias" ] if not bias_schedules: @@ -2449,12 +2419,11 @@ def check_sagemaker_clarify_explainability() -> Dict[str, Any]: findings = _empty_findings("SageMaker Clarify Explainability Check") try: sm = boto3.client("sagemaker", config=boto3_config) - schedules = sm.list_monitoring_schedules().get( - "MonitoringScheduleSummaries", [] - ) + schedules = sm.list_monitoring_schedules().get("MonitoringScheduleSummaries", []) explainability_schedules = [ - s for s in schedules if s.get("MonitoringType") == "ModelExplainability" + s for s in schedules + if s.get("MonitoringType") == "ModelExplainability" ] if not explainability_schedules: @@ -2550,7 +2519,6 @@ def check_ai_service_cards_documentation() -> Dict[str, Any]: # COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, FFIEC CAT, PCI-DSS 3.4, GDPR Art.25] # =========================================================================== - def check_cloudwatch_log_pii_masking() -> Dict[str, Any]: """ FS-43 — Check for CloudWatch Logs data protection policies that mask PII @@ -2745,12 +2713,8 @@ def check_data_classification_tagging() -> Dict[str, Any]: buckets = s3.list_buckets().get("Buckets", []) aiml_buckets = [ - b - for b in buckets - if any( - kw in b["Name"].lower() - for kw in ["train", "model", "bedrock", "sagemaker", "kb", "knowledge"] - ) + b for b in buckets + if any(kw in b["Name"].lower() for kw in ["train", "model", "bedrock", "sagemaker", "kb", "knowledge"]) ] if not aiml_buckets: @@ -2772,10 +2736,7 @@ def check_data_classification_tagging() -> Dict[str, Any]: try: tags = s3.get_bucket_tagging(Bucket=bucket["Name"]).get("TagSet", []) tag_keys = {t["Key"].lower() for t in tags} - if ( - "data-classification" not in tag_keys - and "classification" not in tag_keys - ): + if "data-classification" not in tag_keys and "classification" not in tag_keys: unclassified.append(bucket["Name"]) except ClientError: unclassified.append(bucket["Name"]) @@ -2823,7 +2784,6 @@ def check_data_classification_tagging() -> Dict[str, Any]: # COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2, NYDFS 500] # =========================================================================== - def check_guardrail_grounding_threshold() -> Dict[str, Any]: """ FS-47 — Verify Bedrock Guardrails contextual grounding thresholds are @@ -2856,10 +2816,7 @@ def check_guardrail_grounding_threshold() -> Dict[str, Any]: ) grounding = detail.get("contextualGroundingPolicy", {}) for filter_item in grounding.get("filters", []): - if ( - filter_item.get("type") == "GROUNDING" - and filter_item.get("threshold", 1.0) < 0.7 - ): + if filter_item.get("type") == "GROUNDING" and filter_item.get("threshold", 1.0) < 0.7: low_threshold_guardrails.append( f"{g['name']} (threshold={filter_item['threshold']})" ) @@ -3126,12 +3083,8 @@ def check_bedrock_sdk_version_currency() -> Dict[str, Any]: functions = lambda_client.list_functions().get("Functions", []) bedrock_functions = [ - f - for f in functions - if any( - kw in f["FunctionName"].lower() - for kw in ["bedrock", "agent", "aiml", "genai"] - ) + f for f in functions + if any(kw in f["FunctionName"].lower() for kw in ["bedrock", "agent", "aiml", "genai"]) ] if not bedrock_functions: @@ -3232,9 +3185,7 @@ def check_waf_sql_injection_rules() -> Dict[str, Any]: Id=acl_summary["Id"], ).get("WebACL", {}) rule_names = { - r.get("Statement", {}) - .get("ManagedRuleGroupStatement", {}) - .get("Name", "") + r.get("Statement", {}).get("ManagedRuleGroupStatement", {}).get("Name", "") for r in acl.get("Rules", []) } if not rule_names.intersection(INJECTION_RULE_GROUPS): @@ -3315,7 +3266,6 @@ def check_penetration_testing_evidence() -> Dict[str, Any]: # COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, NYDFS 500, OWASP LLM02] # =========================================================================== - def check_output_validation_lambda() -> Dict[str, Any]: """ FS-55 — Check for Lambda functions implementing output validation/sanitization @@ -3328,12 +3278,8 @@ def check_output_validation_lambda() -> Dict[str, Any]: functions = lambda_client.list_functions().get("Functions", []) validation_functions = [ - f - for f in functions - if any( - kw in f["FunctionName"].lower() - for kw in ["validate", "sanitize", "filter", "output"] - ) + f for f in functions + if any(kw in f["FunctionName"].lower() for kw in ["validate", "sanitize", "filter", "output"]) ] if not validation_functions: @@ -3464,12 +3410,8 @@ def check_output_schema_validation() -> Dict[str, Any]: functions = lambda_client.list_functions().get("Functions", []) schema_functions = [ - f - for f in functions - if any( - kw in f["FunctionName"].lower() - for kw in ["schema", "validate", "parse", "format"] - ) + f for f in functions + if any(kw in f["FunctionName"].lower() for kw in ["schema", "validate", "parse", "format"]) ] findings["csv_data"].append( @@ -3632,10 +3574,8 @@ def check_knowledge_base_sync_schedule() -> Dict[str, Any]: # Check for EventBridge rules that trigger KB sync rules = events.list_rules().get("Rules", []) kb_sync_rules = [ - r - for r in rules - if "bedrock" in r.get("Name", "").lower() - or "knowledge" in r.get("Name", "").lower() + r for r in rules + if "bedrock" in r.get("Name", "").lower() or "knowledge" in r.get("Name", "").lower() ] if not kb_sync_rules: @@ -3727,8 +3667,7 @@ def check_foundation_model_lifecycle_policy() -> Dict[str, Any]: config_client = boto3.client("config", config=boto3_config) rules = config_client.describe_config_rules().get("ConfigRules", []) lifecycle_rules = [ - r - for r in rules + r for r in rules if "lifecycle" in r.get("ConfigRuleName", "").lower() or "model" in r.get("ConfigRuleName", "").lower() ] @@ -3813,9 +3752,9 @@ def check_kb_datasource_s3_event_notifications() -> Dict[str, Any]: buckets_without_notifications = [] for kb in kbs: kb_id = kb["knowledgeBaseId"] - data_sources = bedrock_agent.list_data_sources(knowledgeBaseId=kb_id).get( - "dataSourceSummaries", [] - ) + data_sources = bedrock_agent.list_data_sources( + knowledgeBaseId=kb_id + ).get("dataSourceSummaries", []) for ds in data_sources: ds_detail = bedrock_agent.get_data_source( knowledgeBaseId=kb_id, @@ -3830,17 +3769,13 @@ def check_kb_datasource_s3_event_notifications() -> Dict[str, Any]: if not bucket: continue try: - notif = s3_client.get_bucket_notification_configuration( - Bucket=bucket - ) - has_notif = any( - [ - notif.get("TopicConfigurations"), - notif.get("QueueConfigurations"), - notif.get("LambdaFunctionConfigurations"), - notif.get("EventBridgeConfiguration"), - ] - ) + notif = s3_client.get_bucket_notification_configuration(Bucket=bucket) + has_notif = any([ + notif.get("TopicConfigurations"), + notif.get("QueueConfigurations"), + notif.get("LambdaFunctionConfigurations"), + notif.get("EventBridgeConfiguration"), + ]) if not has_notif: buckets_without_notifications.append(bucket) except ClientError: @@ -3855,9 +3790,7 @@ def check_kb_datasource_s3_event_notifications() -> Dict[str, Any]: finding_details=( "The following KB data-source S3 buckets have no event notifications configured. " "Unauthorized document modifications will not be detected in real time:\n" - + "\n".join( - f"- {b}" for b in buckets_without_notifications[:10] - ) + + "\n".join(f"- {b}" for b in buckets_without_notifications[:10]) ), resolution=( "1. Enable Amazon EventBridge notifications on each KB data-source S3 bucket.\n" @@ -3993,19 +3926,10 @@ def check_agent_financial_transaction_thresholds() -> Dict[str, Any]: # Look for agent action-group Lambda functions action_group_lambdas = [ - f - for f in functions - if any( - kw in f["FunctionName"].lower() - for kw in [ - "agent", - "action", - "tool", - "bedrock", - "finserv", - "transaction", - ] - ) + f for f in functions + if any(kw in f["FunctionName"].lower() for kw in [ + "agent", "action", "tool", "bedrock", "finserv", "transaction" + ]) ] if not action_group_lambdas: @@ -4036,9 +3960,7 @@ def check_agent_financial_transaction_thresholds() -> Dict[str, Any]: f["FunctionName"] for f in action_group_lambdas if not any( - "threshold" in k.lower() - or "limit" in k.lower() - or "max" in k.lower() + "threshold" in k.lower() or "limit" in k.lower() or "max" in k.lower() for k in f.get("Environment", {}).get("Variables", {}).keys() ) ] @@ -4053,9 +3975,7 @@ def check_agent_financial_transaction_thresholds() -> Dict[str, Any]: "The following agent action-group Lambda functions have no environment " "variables indicating transaction-value threshold configuration. " "Without explicit limits, agents could initiate unbounded financial transactions:\n" - + "\n".join( - f"- {n}" for n in lambdas_without_threshold_config[:10] - ) + + "\n".join(f"- {n}" for n in lambdas_without_threshold_config[:10]) ), resolution=( "1. Add transaction-value threshold environment variables (e.g., MAX_TRANSACTION_AMOUNT) " @@ -4099,15 +4019,14 @@ def check_api_gateway_request_body_size_limits() -> Dict[str, Any]: findings = _empty_findings("API Gateway Request Body Size Limits Check") try: apigw = boto3.client("apigateway", config=boto3_config) + apigwv2 = boto3.client("apigatewayv2", config=boto3_config) wafv2 = boto3.client("wafv2", config=boto3_config) # Check REST APIs for request validators rest_apis = apigw.get_rest_apis().get("items", []) apis_without_validators = [] for api in rest_apis: - validators = apigw.get_request_validators(restApiId=api["id"]).get( - "items", [] - ) + validators = apigw.get_request_validators(restApiId=api["id"]).get("items", []) if not validators: apis_without_validators.append(api.get("name", api["id"])) @@ -4191,15 +4110,8 @@ def check_prompt_input_validation_function() -> Dict[str, Any]: # Look for Lambda functions with input validation / sanitization naming patterns VALIDATION_KEYWORDS = [ - "sanitiz", - "validat", - "input", - "preprocess", - "pre-process", - "filter", - "clean", - "prompt-guard", - "promptguard", + "sanitiz", "validat", "input", "preprocess", "pre-process", + "filter", "clean", "prompt-guard", "promptguard", ] validation_lambdas = [ f["FunctionName"] @@ -4262,7 +4174,6 @@ def check_prompt_input_validation_function() -> Dict[str, Any]: # REPORT GENERATION & LAMBDA HANDLER # =========================================================================== - def generate_csv_report(findings: List[Dict[str, Any]]) -> str: """Generate CSV report from all security check findings.""" csv_buffer = StringIO() From 311eb89c2617beaa063844df976824826107f123 Mon Sep 17 00:00:00 2001 From: Neil Mehta Date: Fri, 1 May 2026 13:46:16 -0400 Subject: [PATCH 03/16] docs: Update README and SECURITY_CHECKS for FinServ addition - Bump check count from 52 to 116 (52 core + 64 FinServ) throughout README - Add Financial Services GenAI Risk to title, key features, services covered, How It Works module list, member role permissions, and report structure - Add finserv_security_report_{execution_id}.csv to individual account report file listing - Update SECURITY_CHECKS.md overview table, TOC, and check ID convention table to include FS-XX prefix and 64-check count - SECURITY_CHECKS.md already contains the full FinServ section (added in the original feat commit); this commit updates the summary tables only sim: https://github.com/aws-samples/sample-aiml-security-assessment/issues/22 --- README.md | 17 ++++++++++------- docs/SECURITY_CHECKS.md | 5 ++++- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 87a38d3..a497b24 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ -# AWS AI/ML Security Assessment — Amazon Bedrock, SageMaker & AgentCore. +# AWS AI/ML Security Assessment — Amazon Bedrock, SageMaker, AgentCore & Financial Services GenAI Risk. [![License: MIT-0](https://img.shields.io/badge/License-MIT--0-yellow.svg)](https://opensource.org/licenses/MIT-0) [![Python 3.12+](https://img.shields.io/badge/Python-3.12+-blue.svg)](https://www.python.org/downloads/) [![AWS SAM](https://img.shields.io/badge/AWS-SAM-orange.svg)](https://aws.amazon.com/serverless/sam/) [![Serverless](https://img.shields.io/badge/Architecture-Serverless-green.svg)](https://aws.amazon.com/serverless/) **Open-source automated security scanner for Amazon Bedrock, SageMaker AI, and Bedrock AgentCore** — Built on [AWS Well-Architected Framework (Generative AI Lens)](https://docs.aws.amazon.com/wellarchitected/latest/generative-ai-lens/generative-ai-lens.html) -Cloud security automation with **[52 security checks](docs/SECURITY_CHECKS.md)** for your generative AI and machine learning workloads. Identify IAM misconfigurations, encryption gaps, network isolation issues, and compliance violations with interactive HTML reports and actionable remediation guidance. +Cloud security automation with **[116 security checks](docs/SECURITY_CHECKS.md)** for your generative AI and machine learning workloads. Identify IAM misconfigurations, encryption gaps, network isolation issues, and compliance violations with interactive HTML reports and actionable remediation guidance. --- @@ -43,7 +43,7 @@ The framework generates professional, interactive security assessment reports wi - **Executive Summary** with severity counts and service breakdown - **Priority Recommendations** highlighting critical issues requiring immediate attention -- **[52 Security Checks](docs/SECURITY_CHECKS.md)** across Amazon Bedrock, SageMaker AI, and Bedrock AgentCore +- **[116 Security Checks](docs/SECURITY_CHECKS.md)** across Amazon Bedrock, SageMaker AI, Bedrock AgentCore, and Financial Services GenAI Risk - **Interactive Filtering** by account, service, severity, and status - **Light/Dark Mode Toggle** with persistent user preference - **Text Search** across all findings with real-time results @@ -85,7 +85,7 @@ Designed for workloads using [Amazon Bedrock](https://aws.amazon.com/bedrock/), | Challenge | How This Framework Helps | |-----------|-------------------------| | **Manual security audits are time-consuming** | Fully automated scanning with one-click CloudFormation deployment | -| **Inconsistent security checks across teams** | Standardized 52-check assessment based on AWS Well-Architected best practices | +| **Inconsistent security checks across teams** | Standardized 116-check assessment based on AWS Well-Architected best practices and AWS FinServ GenAI Risk guidance | | **Difficulty tracking AI/ML security posture** | Interactive HTML dashboards with severity breakdown and per-account visibility | | **Multi-account complexity** | Consolidated reporting across AWS Organizations with cross-account role assumption | | **Compliance and audit requirements** | Exportable reports with remediation guidance linked to AWS documentation | @@ -95,6 +95,7 @@ Designed for workloads using [Amazon Bedrock](https://aws.amazon.com/bedrock/), - **[Amazon Bedrock](docs/SECURITY_CHECKS.md#amazon-bedrock-security-checks-14)** (14 checks) - Guardrails, encryption, Amazon VPC endpoints, AWS IAM permissions, model invocation logging - **[Amazon SageMaker AI](docs/SECURITY_CHECKS.md#amazon-sagemaker-ai-security-checks-25)** (25 checks) - Security Hub controls (SageMaker.1-5), encryption, network isolation, AWS IAM, MLOps - **[Amazon Bedrock AgentCore](docs/SECURITY_CHECKS.md#amazon-bedrock-agentcore-security-checks-13)** (13 checks) - Amazon VPC configuration, encryption, observability, resource policies +- **[Financial Services GenAI Risk](docs/SECURITY_CHECKS.md#financial-services-genai-risk-checks-64-additional-5-upstream-extensions)** (64 checks) - Unbounded consumption, excessive agency, supply chain, training data poisoning, hallucination, prompt injection, PII disclosure, and 8 more FinServ-specific risk categories derived from the [AWS FinServ GenAI Risk Guide](https://d1.awsstatic.com/onedam/marketing-channels/website/public/global-FinServ-ComplianceGuide-GenAIRisks-public.pdf) **Deployment Options:** - **Single-Account**: Assess security in one AWS account @@ -272,6 +273,7 @@ Deploy [2-aiml-security-codebuild.yaml](deployment/2-aiml-security-codebuild.yam - Amazon Bedrock Assessment AWS Lambda - Amazon SageMaker Assessment AWS Lambda - Amazon Bedrock AgentCore Assessment AWS Lambda + - Financial Services GenAI Risk Assessment AWS Lambda - AWS IAM Permission Caching AWS Lambda - Consolidated Report Generation AWS Lambda 4. **Assessment Execution**: AWS Step Functions orchestrate parallel AWS Lambda execution @@ -292,7 +294,7 @@ Deploy [2-aiml-security-codebuild.yaml](deployment/2-aiml-security-codebuild.yam ### Member Account Role (`AIMLSecurityMemberRole`) -- Read-only access to AI/ML services (Amazon Bedrock, Amazon SageMaker AI, Amazon Bedrock AgentCore) +- Read-only access to AI/ML services (Amazon Bedrock, Amazon SageMaker AI, Amazon Bedrock AgentCore, and FinServ-specific services: AWS WAF, AWS Shield, Amazon Macie, AWS Organizations, Amazon OpenSearch Serverless) - AWS IAM read permissions for security assessment - AWS CloudTrail, Amazon GuardDuty, and AWS Lambda read permissions - Amazon VPC and Amazon EC2 read permissions @@ -335,7 +337,7 @@ You can check the AWS CodeBuild console to ensure that the assessment has comple - **File Format**: `multi_account_report_YYYYMMDD_HHMMSS.html` - **Features**: - Executive summary with metrics (Total, High, Medium, Low severity counts) - - Service breakdown (Amazon Bedrock, Amazon SageMaker, Amazon Bedrock AgentCore) + - Service breakdown (Amazon Bedrock, Amazon SageMaker, Amazon Bedrock AgentCore, Financial Services GenAI Risk) - Priority recommendations - Light/dark mode toggle (persists via localStorage) - Dropdown filters for Account ID, Severity, Status @@ -350,6 +352,7 @@ You can check the AWS CodeBuild console to ensure that the assessment has comple - `bedrock_security_report_{execution_id}.csv` - Amazon Bedrock security assessment results - `sagemaker_security_report_{execution_id}.csv` - Amazon SageMaker security assessment results - `agentcore_security_report_{execution_id}.csv` - Amazon Bedrock AgentCore security assessment results + - `finserv_security_report_{execution_id}.csv` - Financial Services GenAI risk assessment results (64 FS-XX checks) - `permissions_cache_{execution_id}.json` - IAM permissions cache - `security_assessment_{timestamp}_{execution_id}.html` - Consolidated HTML report (same features as multi-account report) @@ -491,7 +494,7 @@ For a clean removal, delete resources in this order: | Document | Description | |----------|-------------| -| [Security Checks Reference](docs/SECURITY_CHECKS.md) | Complete reference for all 52 security checks with severity levels | +| [Security Checks Reference](docs/SECURITY_CHECKS.md) | Complete reference for all 116 security checks with severity levels | | [Troubleshooting Guide](docs/TROUBLESHOOTING.md) | Common issues, debugging tips, and FAQ | | [Developer Guide](docs/DEVELOPER_GUIDE.md) | Architecture details, adding custom checks, and contributing | diff --git a/docs/SECURITY_CHECKS.md b/docs/SECURITY_CHECKS.md index 88f6ee6..61c40f0 100644 --- a/docs/SECURITY_CHECKS.md +++ b/docs/SECURITY_CHECKS.md @@ -1,6 +1,6 @@ # Security Checks Reference -This document provides a comprehensive reference for all 52 security checks performed by the AI/ML Security Assessment framework. +This document provides a comprehensive reference for all 116 security checks performed by the AI/ML Security Assessment framework (52 core checks across Bedrock, SageMaker, and AgentCore, plus 64 Financial Services GenAI Risk checks). ## Table of Contents @@ -11,6 +11,7 @@ This document provides a comprehensive reference for all 52 security checks perf - [Amazon SageMaker AI Security Checks (25)](#amazon-sagemaker-ai-security-checks-25) - [Amazon Bedrock Security Checks (14)](#amazon-bedrock-security-checks-14) - [Amazon Bedrock AgentCore Security Checks (13)](#amazon-bedrock-agentcore-security-checks-13) +- [Financial Services GenAI Risk Checks (64)](#financial-services-genai-risk-checks-64-additional-5-upstream-extensions) --- @@ -23,6 +24,7 @@ The framework evaluates your AI/ML workloads against AWS security best practices | Amazon SageMaker AI | 25 | Security Hub controls, encryption, network isolation, IAM, MLOps | | Amazon Bedrock | 14 | Guardrails, encryption, VPC endpoints, IAM permissions, logging | | Amazon Bedrock AgentCore | 13 | VPC configuration, encryption, observability, resource policies | +| Financial Services GenAI Risk | 64 | Unbounded consumption, excessive agency, supply chain, training data poisoning, vector weaknesses, non-compliant output, misinformation, harmful output, biased output, PII disclosure, hallucination, prompt injection, improper output handling, off-topic output, out-of-date training data | --- @@ -35,6 +37,7 @@ Each security check has a unique identifier with a service prefix: | **SM-XX** | Amazon SageMaker | SM-01, SM-25 | | **BR-XX** | Amazon Bedrock | BR-01, BR-14 | | **AC-XX** | Amazon Bedrock AgentCore | AC-01, AC-13 | +| **FS-XX** | Financial Services GenAI Risk | FS-01, FS-69 | --- From 4e4e5393f809f7ef5ff6dd52dece528dc0476278 Mon Sep 17 00:00:00 2001 From: Neil Mehta Date: Wed, 3 Jun 2026 11:45:10 -0400 Subject: [PATCH 04/16] =?UTF-8?q?docs:=20Address=20TASK-11=20=E2=80=94=20R?= =?UTF-8?q?EADME=20FinServ=20coverage=20and=20HTML=20limitation=20notice?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix tagline subtitle to include Financial Services GenAI Risk alongside Bedrock, SageMaker AI, and Bedrock AgentCore - Add prominent Known Limitation callout in Report Structure section documenting that FinServ findings appear in CSV only and are not yet rendered in the HTML report dashboard (follow-up enhancement) - Add all 5 FinServ documentation files to the Documentation table: SECURITY_CHECKS_FINSERV_COMMON.md, PART1/PART2/PART3 check references, and AIMLSecurityAssessment-MappingsTable.csv with compliance mappings --- README.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a497b24..c8da784 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![License: MIT-0](https://img.shields.io/badge/License-MIT--0-yellow.svg)](https://opensource.org/licenses/MIT-0) [![Python 3.12+](https://img.shields.io/badge/Python-3.12+-blue.svg)](https://www.python.org/downloads/) [![AWS SAM](https://img.shields.io/badge/AWS-SAM-orange.svg)](https://aws.amazon.com/serverless/sam/) [![Serverless](https://img.shields.io/badge/Architecture-Serverless-green.svg)](https://aws.amazon.com/serverless/) -**Open-source automated security scanner for Amazon Bedrock, SageMaker AI, and Bedrock AgentCore** — Built on [AWS Well-Architected Framework (Generative AI Lens)](https://docs.aws.amazon.com/wellarchitected/latest/generative-ai-lens/generative-ai-lens.html) +**Open-source automated security scanner for Amazon Bedrock, SageMaker AI, Bedrock AgentCore, and Financial Services GenAI Risk** — Built on [AWS Well-Architected Framework (Generative AI Lens)](https://docs.aws.amazon.com/wellarchitected/latest/generative-ai-lens/generative-ai-lens.html) Cloud security automation with **[116 security checks](docs/SECURITY_CHECKS.md)** for your generative AI and machine learning workloads. Identify IAM misconfigurations, encryption gaps, network isolation issues, and compliance violations with interactive HTML reports and actionable remediation guidance. @@ -353,6 +353,16 @@ You can check the AWS CodeBuild console to ensure that the assessment has comple - `sagemaker_security_report_{execution_id}.csv` - Amazon SageMaker security assessment results - `agentcore_security_report_{execution_id}.csv` - Amazon Bedrock AgentCore security assessment results - `finserv_security_report_{execution_id}.csv` - Financial Services GenAI risk assessment results (64 FS-XX checks) + +> **⚠️ Known Limitation — FinServ findings appear in CSV only, not in the HTML report** +> +> The HTML report (`security_assessment_*.html`) currently renders findings from Amazon Bedrock, +> Amazon SageMaker, and Amazon Bedrock AgentCore assessments only. The Financial Services GenAI +> risk checks write results to `finserv_security_report_{execution_id}.csv`, which is consolidated +> into the multi-account CSV, but **FinServ findings are not yet visible in the interactive HTML +> dashboard**. HTML rendering for the FinServ section is tracked as a follow-up enhancement. +> To review FinServ findings, open the CSV file directly from the Amazon S3 assessment bucket. + - `permissions_cache_{execution_id}.json` - IAM permissions cache - `security_assessment_{timestamp}_{execution_id}.html` - Consolidated HTML report (same features as multi-account report) @@ -495,6 +505,11 @@ For a clean removal, delete resources in this order: | Document | Description | |----------|-------------| | [Security Checks Reference](docs/SECURITY_CHECKS.md) | Complete reference for all 116 security checks with severity levels | +| [FinServ GenAI Risk Checks — Common](docs/SECURITY_CHECKS_FINSERV_COMMON.md) | Shared introduction, severity rubric, upstream-overlap table, and compliance framework mapping for FS-01..69 | +| [FinServ Part 1 — Infrastructure Controls](docs/SECURITY_CHECKS_FINSERV_PART1_INFRA_CONTROLS.md) | FS-01..26: Unbounded consumption, excessive agency, supply chain, training data poisoning, vector & embedding weaknesses | +| [FinServ Part 2 — Guardrails & Content Safety](docs/SECURITY_CHECKS_FINSERV_PART2_GUARDRAILS_CONTENT_SAFETY.md) | FS-27..46: Non-compliant output, misinformation, abusive/harmful output, biased output, PII disclosure | +| [FinServ Part 3 — App Layer & Gaps](docs/SECURITY_CHECKS_FINSERV_PART3_APP_LAYER_AND_GAPS.md) | FS-47..69: Hallucination, prompt injection, improper output handling, off-topic output, out-of-date training data, cross-category gap checks | +| [FinServ Compliance Mappings](docs/AIMLSecurityAssessment-MappingsTable.csv) | Machine-readable mapping of FS checks to SR 11-7, FFIEC CAT, NYDFS 500.06, PCI-DSS, DORA, MAS TRM, ISO 27001, OWASP LLM Top 10 | | [Troubleshooting Guide](docs/TROUBLESHOOTING.md) | Common issues, debugging tips, and FAQ | | [Developer Guide](docs/DEVELOPER_GUIDE.md) | Architecture details, adding custom checks, and contributing | From 6ae99a143c0c5b81d2b9c8da0d60efb40caa012b Mon Sep 17 00:00:00 2001 From: Neil Mehta Date: Wed, 3 Jun 2026 13:04:07 -0400 Subject: [PATCH 05/16] fix(iam): add FinServGenAIRiskAssessmentPermissions statement to deployment YAMLs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the FinServGenAIRiskAssessmentPermissions IAM statement to both deployment/1-aiml-security-member-roles.yaml and deployment/aiml-security-single-account.yaml. The FinServ checks (FS-01 to FS-69) require 51 read-only actions that were present in member_role_permissions_addition.yaml (the workspace snippet) and template_additions.yaml (the SAM template inline policy) but missing from the two CloudFormation deployment templates that govern actual role creation. Actions added (all read-only list/describe/get variants): WAF/Shield (FS-01): wafv2:ListWebACLs, wafv2:GetWebACL, shield:DescribeSubscription API Gateway (FS-02, FS-68, FS-69): apigateway:GET Cost/Budgets (FS-04, FS-06): ce:GetAnomalyMonitors, budgets:DescribeBudgets CloudWatch (FS-05, FS-11): cloudwatch:DescribeAlarms CloudWatch Logs (FS-43): logs:DescribeAccountPolicies Lambda (FS-09, FS-67, FS-69): lambda:GetFunctionConcurrency Step Functions (FS-10, FS-11): states:ListStateMachines, states:DescribeStateMachine Organizations (FS-12): organizations:ListPolicies, organizations:DescribePolicy SageMaker (FS-13, FS-42): sagemaker:ListModelCards, sagemaker:ListTags Bedrock (FS-15, FS-30, FS-34, FS-40): bedrock:ListEvaluationJobs, bedrock:ListFoundationModels, bedrock:ListModelPackageGroups Macie (FS-44): macie2:GetMacieSession OpenSearch Serverless (FS-25, FS-26): aoss:ListSecurityPolicies EventBridge (FS-33, FS-65): events:ListRules Config (FS-14): config:DescribeConfigRules S3 (FS-21, FS-44): s3:ListAllMyBuckets Resource: "*" — appropriate for inventory-style checks where ARNs are not known in advance; consistent with all other assessment statements. The bedrock-agentcore:ListAgentRuntimes/GetAgentRuntime/ListGateways/GetGateway actions were already present in both files from a prior commit. --- deployment/1-aiml-security-member-roles.yaml | 48 +++++++++++++++++++ deployment/aiml-security-single-account.yaml | 49 ++++++++++++++++++++ 2 files changed, 97 insertions(+) diff --git a/deployment/1-aiml-security-member-roles.yaml b/deployment/1-aiml-security-member-roles.yaml index f3d1956..ec2c907 100644 --- a/deployment/1-aiml-security-member-roles.yaml +++ b/deployment/1-aiml-security-member-roles.yaml @@ -203,6 +203,54 @@ Resources: - guardduty:GetRemainingFreeTrialDays - guardduty:GetUsageStatistics Resource: "*" + # FinServ GenAI Risk Assessment Permissions (FS-01 to FS-69) + # These read-only actions are required by the 64 FinServ security checks. + # All are list/describe/get variants; Resource: "*" is required for + # inventory-style checks where resource ARNs are not known in advance. + - Effect: Allow + Sid: FinServGenAIRiskAssessmentPermissions + Action: + # WAF / Shield (FS-01) + - wafv2:ListWebACLs + - wafv2:GetWebACL + - shield:DescribeSubscription + # API Gateway (FS-02, FS-68, FS-69) + - apigateway:GET + # Cost Explorer / Budgets (FS-04, FS-06) + - ce:GetAnomalyMonitors + - budgets:DescribeBudgets + # CloudWatch (FS-05, FS-11) + - cloudwatch:DescribeAlarms + # CloudWatch Logs — account-level data protection policies (FS-43) + - logs:DescribeAccountPolicies + # Lambda — concurrency limits for agent transaction checks (FS-09, FS-67, FS-69) + - lambda:GetFunctionConcurrency + # Step Functions — human-in-the-loop and agent rate checks (FS-10, FS-11) + - states:ListStateMachines + - states:DescribeStateMachine + # Organizations — SCP model access restrictions (FS-12) + - organizations:ListPolicies + - organizations:DescribePolicy + # SageMaker — model cards documentation (FS-42) + - sagemaker:ListModelCards + # Bedrock — evaluation jobs, foundation models (FS-15, FS-30, FS-34, FS-40) + - bedrock:ListEvaluationJobs + - bedrock:ListFoundationModels + # Bedrock — model package groups (supply chain / FS-13, FS-14) + - bedrock:ListModelPackageGroups + # Macie — PII detection on training data buckets (FS-44) + - macie2:GetMacieSession + # OpenSearch Serverless — KB vector store encryption/VPC (FS-25, FS-26) + - aoss:ListSecurityPolicies + # EventBridge — KB integrity monitoring (FS-33, FS-65) + - events:ListRules + # Config — model onboarding governance (FS-14) + - config:DescribeConfigRules + # S3 — list all buckets for training data versioning checks (FS-21, FS-44) + - s3:ListAllMyBuckets + # SageMaker — list tags for model inventory provenance checks (FS-13) + - sagemaker:ListTags + Resource: "*" # Allow access to central assessment bucket - Effect: Allow Action: diff --git a/deployment/aiml-security-single-account.yaml b/deployment/aiml-security-single-account.yaml index 39c23a1..cc205ce 100644 --- a/deployment/aiml-security-single-account.yaml +++ b/deployment/aiml-security-single-account.yaml @@ -276,6 +276,55 @@ Resources: - organizations:DescribePolicy Resource: "*" + # FinServ GenAI Risk Assessment Permissions (FS-01 to FS-69) + # These read-only actions are required by the 64 FinServ security checks. + # All are list/describe/get variants; Resource: "*" is required for + # inventory-style checks where resource ARNs are not known in advance. + - Effect: Allow + Sid: FinServGenAIRiskAssessmentPermissions + Action: + # WAF / Shield (FS-01) + - wafv2:ListWebACLs + - wafv2:GetWebACL + - shield:DescribeSubscription + # API Gateway (FS-02, FS-68, FS-69) + - apigateway:GET + # Cost Explorer / Budgets (FS-04, FS-06) + - ce:GetAnomalyMonitors + - budgets:DescribeBudgets + # CloudWatch (FS-05, FS-11) + - cloudwatch:DescribeAlarms + # CloudWatch Logs — account-level data protection policies (FS-43) + - logs:DescribeAccountPolicies + # Lambda — concurrency limits for agent transaction checks (FS-09, FS-67, FS-69) + - lambda:GetFunctionConcurrency + # Step Functions — human-in-the-loop and agent rate checks (FS-10, FS-11) + - states:ListStateMachines + - states:DescribeStateMachine + # Organizations — SCP model access restrictions (FS-12) + - organizations:ListPolicies + - organizations:DescribePolicy + # SageMaker — model cards documentation (FS-42) + - sagemaker:ListModelCards + # Bedrock — evaluation jobs, foundation models (FS-15, FS-30, FS-34, FS-40) + - bedrock:ListEvaluationJobs + - bedrock:ListFoundationModels + # Bedrock — model package groups (supply chain / FS-13, FS-14) + - bedrock:ListModelPackageGroups + # Macie — PII detection on training data buckets (FS-44) + - macie2:GetMacieSession + # OpenSearch Serverless — KB vector store encryption/VPC (FS-25, FS-26) + - aoss:ListSecurityPolicies + # EventBridge — KB integrity monitoring (FS-33, FS-65) + - events:ListRules + # Config — model onboarding governance (FS-14) + - config:DescribeConfigRules + # S3 — list all buckets for training data versioning checks (FS-21, FS-44) + - s3:ListAllMyBuckets + # SageMaker — list tags for model inventory provenance checks (FS-13) + - sagemaker:ListTags + Resource: "*" + # CodeBuild role for running assessments CodeBuildRole: Type: AWS::IAM::Role From 00fd8d78f5b81078ac510b9fb4c4bf6cdc946412 Mon Sep 17 00:00:00 2001 From: Neil Mehta Date: Wed, 3 Jun 2026 13:36:20 -0400 Subject: [PATCH 06/16] docs: remove stale copy-paste instructions from SECURITY_CHECKS_FINSERV_COMMON.md Remove two instances of the planning-artifact instruction: 'Add this content to docs/SECURITY_CHECKS.md in the forked repository.' These lines were carry-overs from the original planning document and were flagged by the PR reviewer as stale before merge. They appeared after the region-availability disclaimer paragraph and after the upstream-overlap consolidation summary, and have no relevance to the shipped document. --- docs/SECURITY_CHECKS_FINSERV_COMMON.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/SECURITY_CHECKS_FINSERV_COMMON.md b/docs/SECURITY_CHECKS_FINSERV_COMMON.md index 23d7d93..552039d 100644 --- a/docs/SECURITY_CHECKS_FINSERV_COMMON.md +++ b/docs/SECURITY_CHECKS_FINSERV_COMMON.md @@ -61,7 +61,6 @@ availability of new features (Automated Reasoning, AgentCore Policy, AWS Securit cross-account guardrails) evolves rapidly — region lists in Parts 1-3 reflect the state at the cited announcement date and should be re-verified before audit reliance. -Add this content to `docs/SECURITY_CHECKS.md` in the forked repository. ## Contribution workflow @@ -134,7 +133,6 @@ whether the FS check adds FinServ-specific regulatory specificity, (3) severity After consolidation the combined framework contains **52 upstream + 64 FS = 116 distinct checks** (down from 52 + 69 = 121 before merging). The consolidation reduces duplication without losing FinServ-specific regulatory depth. -Add this content to `docs/SECURITY_CHECKS.md` in the forked repository. --- From 02461b64dbcfb6e1849ec0015e3ad2b127a17e5b Mon Sep 17 00:00:00 2001 From: Neil Mehta Date: Wed, 3 Jun 2026 13:39:45 -0400 Subject: [PATCH 07/16] fix(finserv): apply all PR review fixes to finserv_assessments package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Syncs the fork's finserv_assessments/ package with the workspace staging area, applying all fixes from the PR review (TASK-1 through TASK-15): FS-03 (Bug 1 — TASK-1): Replace adjustability-based quota check with value-based comparison. Call list_aws_default_service_quotas(ServiceCode= 'bedrock') and paginate both applied and default quota lists. Compare applied Value > default Value for token-based quotas only. Remove rpm_quotas and default_only = all(not q.get('Adjustable') ...) which incorrectly evaluated whether quotas were adjustable (always True for Bedrock) rather than whether they had been customised. Add explicit branches for empty applied-quota list and unavailable defaults. FS-04 (Bug 2 — TASK-2): Fix monitor filter to require MonitorDimension= 'SERVICE' for DIMENSIONAL monitors. A LINKED_ACCOUNT monitor no longer counts as Bedrock coverage. FS-06 (Bug 3 — TASK-6): Add ShowFilterExpression=True and pagination to describe_budgets. Check both CostFilters and FilterExpression fields. Add ParamValidationError fallback for older SDK versions. Logger (TASK-7): Lower logger level from ERROR to WARNING so all logger.warning() calls emit to CloudWatch Logs. Advisory checks (TASK-8): Retag 8 advisory checks with Status=N/A, Severity=Informational, and 'ADVISORY: ' finding-name prefix. Checks that make real API calls (FS-24, FS-34, FS-42, FS-52) are unchanged. Dead variable (TASK-4): Remove unused apigwv2 variable from check_api_gateway_request_body_size_limits (F841 lint error). Check registry + could-not-assess (TASK-13): Add COULD_NOT_ASSESS_PREFIX constant, _could_not_assess_row() helper, and build_finserv_checks() registry of 64 (check_id, callable) tuples. Drive lambda_handler from the registry; synthesise a visible N/A/Medium row for any check that errors out with an empty csv_data list. Compliance_Frameworks column (TASK-15/D3): Add COMPLIANCE_MAP dict mapping all 64 FS IDs to pipe-separated regulatory framework strings. Add Compliance_Frameworks field to schema.py Finding model and create_finding() signature. Inject compliance_frameworks=COMPLIANCE_MAP ['FS-XX'] into all 153 create_finding() call sites. Add Compliance_Frameworks as 8th column in generate_csv_report fieldnames. requirements.txt (TASK-14): Bump boto3>=1.43.21, botocore>=1.43.21; add pydantic>=2.0.0 with version floor; add explanatory header. --- .../security/finserv_assessments/app.py | 791 ++++++++++++++---- .../finserv_assessments/requirements.txt | 14 +- .../security/finserv_assessments/schema.py | 22 +- 3 files changed, 663 insertions(+), 164 deletions(-) diff --git a/aiml-security-assessment/functions/security/finserv_assessments/app.py b/aiml-security-assessment/functions/security/finserv_assessments/app.py index 9665357..4bbec10 100644 --- a/aiml-security-assessment/functions/security/finserv_assessments/app.py +++ b/aiml-security-assessment/functions/security/finserv_assessments/app.py @@ -46,6 +46,7 @@ import boto3 import csv +import functools import json import logging import os @@ -54,7 +55,7 @@ from typing import Any, Dict, List, Optional from botocore.config import Config -from botocore.exceptions import ClientError +from botocore.exceptions import ClientError, ParamValidationError from schema import create_finding @@ -64,7 +65,7 @@ boto3_config = Config(retries=dict(max_attempts=10, mode="adaptive")) logger = logging.getLogger() -logger.setLevel(logging.ERROR) +logger.setLevel(logging.WARNING) # --------------------------------------------------------------------------- @@ -100,6 +101,142 @@ def _error_findings(check_name: str, err: Exception) -> Dict[str, Any]: } +# Findings whose name starts with this prefix were emitted because the check +# could not run (e.g., missing IAM permission). They are visible in the report +# (Status="N/A") so a failed/permission-denied check does not silently vanish. +COULD_NOT_ASSESS_PREFIX = "COULD NOT ASSESS: " + +# --------------------------------------------------------------------------- +# COMPLIANCE_MAP — per-check regulatory framework mappings +# +# Each value is a pipe-separated string of FinServ framework identifiers that +# travel with every finding row in the CSV report (Compliance_Frameworks column). +# This mirrors the ASFF Compliance.RelatedRequirements pattern used by AWS +# Security Hub: compliance metadata is embedded in the finding itself, not kept +# in a separate sidecar document. +# +# Disclaimer: mappings are PRELIMINARY and ILLUSTRATIVE. They have not been +# reviewed by AWS Security Assurance Services, external auditors, or the +# regulators named. Each firm should have its own MRM, Legal, and Compliance +# teams validate these mappings against their specific interpretation of each +# framework before relying on them as audit evidence. +# --------------------------------------------------------------------------- +COMPLIANCE_MAP: Dict[str, str] = { + # Category 1: Unbounded Consumption + "FS-01": "FFIEC CAT | DORA Art.6", + "FS-02": "FFIEC CAT | DORA Art.6 | PCI-DSS 12.3.2", + "FS-03": "FFIEC CAT | SR 11-7", + "FS-04": "FFIEC CAT | SR 11-7", + "FS-05": "FFIEC CAT | DORA Art.6", + "FS-06": "FFIEC CAT | SR 11-7", + # Category 2: Excessive Agency + "FS-07": "SR 11-7 | FFIEC CAT", + "FS-08": "SR 11-7 | MAS TRM 9.1", + "FS-09": "FFIEC CAT | SR 11-7", + "FS-10": "SR 11-7 | FFIEC CAT | MAS TRM 9.2", + "FS-11": "FFIEC CAT | DORA Art.6", + # Category 3: Supply Chain Vulnerabilities + "FS-12": "SR 11-7 | FFIEC CAT | ISO 27001 A.15.2", + "FS-13": "SR 11-7 | ISO 27001 A.12.5 | FFIEC CAT", + "FS-14": "SR 11-7 | FFIEC CAT | ISO 27001 A.15.1", + "FS-15": "SR 11-7 | FFIEC CAT | MAS TRM 9.3", + "FS-16": "ISO 27001 A.12.6 | FFIEC CAT | DORA Art.6", + # Category 4: Training Data & Model Poisoning + "FS-20": "SR 11-7 | FFIEC CAT", + "FS-21": "SR 11-7 | ISO 27001 A.12.3 | FFIEC CAT", + # Category 5: Vector & Embedding Weaknesses + "FS-22": "NYDFS 500 | FFIEC CAT | PCI-DSS 12.3.2", + "FS-24": "NYDFS 500 | FFIEC CAT | PCI-DSS 12.3.2", + "FS-25": "NYDFS 500 | PCI-DSS 3.5 | FFIEC CAT", + "FS-26": "NYDFS 500 | FFIEC CAT | PCI-DSS 1.3", + # Category 6: Non-Compliant Output + "FS-27": "SR 11-7 | FFIEC CAT | MAS TRM 9.2", + "FS-28": "SR 11-7 | FFIEC CAT | NYDFS 500 | MAS TRM 9.2", + "FS-29": "SR 11-7 | FFIEC CAT | NYDFS 500 | MAS TRM 9.2", + "FS-30": "SR 11-7 | FFIEC CAT | NYDFS 500", + # Category 7: Misinformation + "FS-31": "SR 11-7 | FFIEC CAT", + "FS-32": "SR 11-7 | FFIEC CAT | MAS TRM 9.2", + "FS-33": "SR 11-7 | FFIEC CAT | ISO 27001 A.12", + "FS-34": "SR 11-7 | FFIEC CAT", + # Category 8: Abusive or Harmful Output + "FS-35": "SR 11-7 | FFIEC CAT", + "FS-36": "SR 11-7 | FFIEC CAT", + "FS-37": "SR 11-7 | FFIEC CAT | MAS TRM 9.2", + "FS-38": "SR 11-7 | FFIEC CAT", + # Category 9: Biased Output + "FS-39": "SR 11-7 | FFIEC CAT | ECOA/Fair Housing", + "FS-40": "SR 11-7 | FFIEC CAT | ECOA/Fair Housing", + "FS-41": "SR 11-7 | FFIEC CAT", + "FS-42": "SR 11-7 | FFIEC CAT", + # Category 10: Sensitive Information Disclosure + "FS-43": "NYDFS 500 | FFIEC CAT | PCI-DSS", + "FS-44": "NYDFS 500 | FFIEC CAT | PCI-DSS", + "FS-45": "NYDFS 500 | FFIEC CAT | PCI-DSS", + "FS-46": "NYDFS 500 | FFIEC CAT | ISO 27001 A.12 | PCI-DSS", + # Category 11: Hallucination + "FS-47": "SR 11-7 | FFIEC CAT", + "FS-48": "SR 11-7 | FFIEC CAT", + "FS-49": "SR 11-7 | FFIEC CAT | MAS TRM 9.2", + "FS-50": "SR 11-7 | FFIEC CAT", + # Category 12: Prompt Injection + "FS-51": "NYDFS 500 | FFIEC CAT | OWASP LLM Top 10", + "FS-52": "NYDFS 500 | FFIEC CAT | ISO 27001 A.12 | OWASP LLM Top 10", + "FS-53": "NYDFS 500 | PCI-DSS | FFIEC CAT | OWASP LLM Top 10", + "FS-54": "NYDFS 500 | FFIEC CAT | OWASP LLM Top 10", + # Category 13: Improper Output Handling + "FS-55": "FFIEC CAT | OWASP LLM Top 10", + "FS-56": "NYDFS 500 | PCI-DSS | OWASP LLM Top 10", + "FS-57": "NYDFS 500.06 | FFIEC CAT | OWASP LLM Top 10", + "FS-58": "FFIEC CAT | OWASP LLM Top 10", + # Category 14: Off-Topic & Inappropriate Output + "FS-59": "SR 11-7 | FFIEC CAT", + "FS-60": "SR 11-7 | FFIEC CAT", + # Category 15: Out-of-Date Training Data + "FS-61": "SR 11-7 | FFIEC CAT", + "FS-62": "SR 11-7 | FFIEC CAT | MAS TRM 9.2", + "FS-63": "SR 11-7 | FFIEC CAT | ISO 27001 A.12", + # Material Gap Checks + "FS-65": "FFIEC CAT | DORA Art.6 | ISO 27001 A.12", + "FS-66": "NYDFS 500 | SR 11-7 | MAS TRM 9 | PCI-DSS", + "FS-67": "SR 11-7 | FFIEC CAT | MAS TRM 9 | PCI-DSS", + "FS-68": "DORA Art.6 | FFIEC CAT | PCI-DSS | OWASP LLM Top 10", + "FS-69": "NYDFS 500 | FFIEC CAT | OWASP LLM Top 10", +} + + +def _could_not_assess_row( + check_id: str, check_name: str, err: Any +) -> Dict[str, Any]: + """ + Synthesize one visible finding row for a check that errored out and produced + no rows. Uses the existing schema (Status="N/A", Severity="Medium") so the + gap surfaces in the report without inflating the Failed count. + """ + return create_finding( + check_id=check_id, + finding_name=f"{COULD_NOT_ASSESS_PREFIX}{check_name}", + finding_details=( + f"This check could not be completed (error: {err}). The most common cause " + "is a missing IAM permission for the assessment role; it may also indicate " + "an unsupported region or an outdated botocore. This control was NOT assessed " + "— verify the role's permissions and re-run, and assess this control manually " + "until resolved." + ), + resolution=( + "1. Confirm the assessment role grants the actions this check requires " + "(see the documented IAM permission set in the README).\n" + "2. Confirm the service/feature is supported in the assessed region.\n" + "3. Ensure botocore meets the version floor in requirements.txt.\n" + "4. Re-run the assessment; assess this control manually until it succeeds." + ), + reference="https://docs.aws.amazon.com/IAM/latest/UserGuide/troubleshoot_access-denied.html", + severity="Medium", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP.get(check_id, ""), + ) + + # =========================================================================== # CATEGORY 1: UNBOUNDED CONSUMPTION (FS-01 to FS-06) # Risk: GenAI workloads can be exploited to exhaust compute/cost budgets @@ -150,6 +287,7 @@ def check_waf_shield_on_bedrock_endpoints() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/waf/latest/developerguide/shield-chapter.html", severity="High", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-01"], ) ) else: @@ -162,6 +300,7 @@ def check_waf_shield_on_bedrock_endpoints() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/waf/latest/developerguide/shield-chapter.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-01"], ) ) @@ -183,6 +322,7 @@ def check_waf_shield_on_bedrock_endpoints() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/waf/latest/developerguide/waf-chapter.html", severity="High", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-01"], ) ) else: @@ -195,6 +335,7 @@ def check_waf_shield_on_bedrock_endpoints() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/waf/latest/developerguide/waf-chapter.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-01"], ) ) except Exception as e: @@ -231,6 +372,7 @@ def check_api_gateway_rate_limiting() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-request-throttling.html", severity="Medium", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-02"], ) ) elif plans_without_throttle: @@ -250,6 +392,7 @@ def check_api_gateway_rate_limiting() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-request-throttling.html", severity="High", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-02"], ) ) else: @@ -262,6 +405,7 @@ def check_api_gateway_rate_limiting() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-request-throttling.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-02"], ) ) except Exception as e: @@ -272,39 +416,150 @@ def check_api_gateway_rate_limiting() -> Dict[str, Any]: def check_bedrock_token_quotas() -> Dict[str, Any]: """ FS-03 — Check whether Bedrock service quotas for tokens-per-minute (TPM) - and requests-per-minute (RPM) have been reviewed and set appropriately. + have been reviewed and raised above AWS defaults. + + Token-based quotas are the only signal: Amazon Bedrock no longer enforces + requests-per-minute (RPM) quotas on the bedrock-runtime endpoint (throttling + is token-based only), so an absent RPM quota must never drive a verdict. + + Verdict logic (value-based, not adjustability-based): + - at least one applied token-quota Value > its AWS default → customized → PASS/Passed + - all applied token-quota Values == their defaults → at default → WARN/N-A (soft) + - no applied token quotas returned → WARN/Failed + explanation + - AWS default quotas could not be retrieved → WARN/Failed + "undetermined" COMPLIANCE_PLACEHOLDER: [FFIEC CAT, SR 11-7] """ findings = _empty_findings("Bedrock Token Quota Review") try: sq = boto3.client("service-quotas", config=boto3_config) - quotas = sq.list_service_quotas(ServiceCode="bedrock").get("Quotas", []) - tpm_quotas = [q for q in quotas if "token" in q.get("QuotaName", "").lower()] - rpm_quotas = [q for q in quotas if "request" in q.get("QuotaName", "").lower()] + # Applied quotas (paginated). Token-based quotas are the only signal. + applied = [] + for page in sq.get_paginator("list_service_quotas").paginate( + ServiceCode="bedrock" + ): + applied.extend(page.get("Quotas", [])) + token_quotas = [ + q for q in applied if "token" in q.get("QuotaName", "").lower() + ] - # Flag if no custom quota increases have been requested (still at default) - default_only = all(not q.get("Adjustable") for q in tpm_quotas + rpm_quotas) + # Empty-applied-list branch: no token quotas found at all. + if not token_quotas: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-03", + finding_name="No Bedrock Token Quotas Returned", + finding_details=( + "No Bedrock token-based service quotas were returned. This may indicate " + "a permissions issue (servicequotas:ListServiceQuotas), an unsupported " + "region, or that no Bedrock-specific quotas exist for this account. " + "Verify manually in the Service Quotas console." + ), + resolution=( + "1. Confirm the assessment role has servicequotas:ListServiceQuotas.\n" + "2. Verify Bedrock is available in the assessed region.\n" + "3. Review Bedrock token quotas in the Service Quotas console." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/quotas.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-03"], + ) + ) + return findings - details = ( - f"Found {len(tpm_quotas)} token-based and {len(rpm_quotas)} request-based Bedrock quotas." - ) - findings["csv_data"].append( - create_finding( - check_id="FS-03", - finding_name="Bedrock Token and Request Quota Review", - finding_details=details, - resolution=( - "1. Review current Bedrock TPM/RPM quotas in Service Quotas console.\n" - "2. Request increases aligned with expected peak load.\n" - "3. Implement client-side token counting and pre-flight quota checks.\n" - "4. Use Bedrock cross-region inference profiles to distribute load." - ), - reference="https://docs.aws.amazon.com/bedrock/latest/userguide/quotas.html", - severity="Medium", - status="Passed" if not default_only else "Failed", + # AWS default quotas for comparison (paginated). + default_values = {} + for page in sq.get_paginator("list_aws_default_service_quotas").paginate( + ServiceCode="bedrock" + ): + for q in page.get("Quotas", []): + if q.get("QuotaCode") is not None: + default_values[q["QuotaCode"]] = q.get("Value") + + # Default-lookup-fail branch: cannot compare without defaults. Do NOT + # silently compare a value against itself (which would always Fail). + if not default_values: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-03", + finding_name="Bedrock Default Quotas Unavailable — Customization Undetermined", + finding_details=( + "AWS default service quotas for Bedrock could not be retrieved " + "(list_aws_default_service_quotas returned nothing), so whether the " + "applied quotas have been customized cannot be determined. This is " + "commonly a permissions issue (servicequotas:ListAWSDefaultServiceQuotas) " + "or an unsupported region." + ), + resolution=( + "1. Confirm the assessment role has servicequotas:ListAWSDefaultServiceQuotas.\n" + "2. Re-run the assessment once defaults are retrievable.\n" + "3. Until then, verify Bedrock token quotas manually in the Service Quotas console." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/quotas.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-03"], + ) ) + return findings + + # Value-based comparison. An applied Value == default Value is the + # expected, non-customized state (ListServiceQuotas may return only + # default values for some quotas) — this is NOT an error. At-default is + # reported as a soft warning (WARN/N-A), not a failure, since it is a + # legitimate verified posture. + any_customized = any( + q.get("QuotaCode") in default_values + and default_values[q["QuotaCode"]] is not None + and q.get("Value", 0) > default_values[q["QuotaCode"]] + for q in token_quotas ) + + if any_customized: + findings["csv_data"].append( + create_finding( + check_id="FS-03", + finding_name="Bedrock Token Quotas Customized", + finding_details=( + f"Found {len(token_quotas)} Bedrock token-based quota(s); at least one " + "applied value exceeds the AWS default, indicating quotas have been " + "reviewed and raised." + ), + resolution="No action required. Periodically re-review quotas against expected peak load.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/quotas.html", + severity="Informational", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-03"], + ) + ) + else: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-03", + finding_name="Bedrock Token Quotas At Default", + finding_details=( + f"All {len(token_quotas)} Bedrock token-based quota(s) are at their AWS " + "default values — no quota increase has been applied. Running at default " + "is a legitimate posture, but it should be a reviewed decision aligned " + "with expected peak load rather than an oversight." + ), + resolution=( + "1. Review current Bedrock TPM/TPD quotas in the Service Quotas console.\n" + "2. Request increases aligned with expected peak load, or document a " + "deliberate decision to remain at default after review.\n" + "3. Implement client-side token counting and pre-flight quota checks.\n" + "4. Use Bedrock cross-region inference profiles to distribute load." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/quotas.html", + severity="Medium", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-03"], + ) + ) except Exception as e: return _error_findings("Bedrock Token Quota Review", e) return findings @@ -321,10 +576,19 @@ def check_cost_anomaly_detection() -> Dict[str, Any]: ce = boto3.client("ce", config=boto3_config) monitors = ce.get_anomaly_monitors().get("AnomalyMonitors", []) + # A monitor provides Bedrock/SageMaker service-level coverage if its + # spec mentions bedrock (rarely populated for DIMENSIONAL+SERVICE + # monitors, kept for completeness) OR it is a DIMENSIONAL monitor scoped + # to the SERVICE dimension. DIMENSIONAL+SERVICE is the operative signal; + # a DIMENSIONAL monitor on LINKED_ACCOUNT/TAG/COST_CATEGORY does NOT + # provide service-level Bedrock coverage and must not count. bedrock_monitors = [ m for m in monitors if "bedrock" in json.dumps(m.get("MonitorSpecification", {})).lower() - or m.get("MonitorType") == "DIMENSIONAL" + or ( + m.get("MonitorType") == "DIMENSIONAL" + and m.get("MonitorDimension") == "SERVICE" + ) ] if not monitors: @@ -345,6 +609,34 @@ def check_cost_anomaly_detection() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/cost-management/latest/userguide/getting-started-ad.html", severity="Medium", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-04"], + ) + ) + elif not bedrock_monitors: + # Monitors exist, but none provide Bedrock/SageMaker service-level + # coverage. This is the previously-masked false positive: the old + # code passed whenever ANY monitor existed. + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-04", + finding_name="Cost Anomaly Monitors Do Not Cover Bedrock/SageMaker", + finding_details=( + f"Found {len(monitors)} anomaly monitor(s), but none provide service-level " + "coverage for Bedrock/SageMaker (no DIMENSIONAL monitor scoped to the SERVICE " + "dimension, and no monitor specification referencing Bedrock). A generic or " + "linked-account monitor does not detect GenAI cost anomalies." + ), + resolution=( + "1. Create a DIMENSIONAL Cost Anomaly Detection monitor scoped to the SERVICE " + "dimension so AWS/Bedrock and AWS/SageMaker are covered.\n" + "2. Configure alert subscriptions (SNS/email) for anomalies above threshold.\n" + "3. Set daily spend budgets with AWS Budgets as a secondary control." + ), + reference="https://docs.aws.amazon.com/cost-management/latest/userguide/getting-started-ad.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-04"], ) ) else: @@ -352,11 +644,12 @@ def check_cost_anomaly_detection() -> Dict[str, Any]: create_finding( check_id="FS-04", finding_name="Cost Anomaly Detection Configured", - finding_details=f"Found {len(monitors)} anomaly monitor(s); {len(bedrock_monitors)} appear Bedrock-related.", + finding_details=f"Found {len(monitors)} anomaly monitor(s); {len(bedrock_monitors)} provide Bedrock/SageMaker service-level coverage.", resolution="Verify monitors cover Bedrock and SageMaker service dimensions.", reference="https://docs.aws.amazon.com/cost-management/latest/userguide/getting-started-ad.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-04"], ) ) except Exception as e: @@ -409,6 +702,7 @@ def check_cloudwatch_token_alarms() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/monitoring-cw.html", severity="Medium", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-05"], ) ) else: @@ -424,6 +718,7 @@ def check_cloudwatch_token_alarms() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/monitoring-cw.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-05"], ) ) except Exception as e: @@ -442,13 +737,38 @@ def check_aws_budgets_for_aiml() -> Dict[str, Any]: sts = boto3.client("sts", config=boto3_config) account_id = sts.get_caller_identity()["Account"] - all_budgets = budgets_client.describe_budgets(AccountId=account_id).get( - "Budgets", [] - ) + def _paginate_budgets(show_filter_expression: bool): + kwargs = {"AccountId": account_id} + if show_filter_expression: + kwargs["ShowFilterExpression"] = True + pages = [] + for page in budgets_client.get_paginator("describe_budgets").paginate( + **kwargs + ): + pages.extend(page.get("Budgets", [])) + return pages + + # FilterExpression is opt-in: DescribeBudgets only returns it when + # ShowFilterExpression=True. Modern budgets use the structured + # FilterExpression instead of the deprecated flat CostFilters map. + # On an old botocore that does not accept ShowFilterExpression, the call + # raises ParamValidationError (NOT a ClientError) — degrade gracefully + # to a CostFilters-only check rather than letting the check vanish. + try: + all_budgets = _paginate_budgets(show_filter_expression=True) + except ParamValidationError: + logger.warning( + "describe_budgets does not accept ShowFilterExpression on this " + "botocore; falling back to CostFilters-only budget detection. " + "Upgrade botocore to check new-style FilterExpression budgets." + ) + all_budgets = _paginate_budgets(show_filter_expression=False) + aiml_budgets = [ b for b in all_budgets if any( svc in json.dumps(b.get("CostFilters", {})).lower() + or svc in json.dumps(b.get("FilterExpression", {})).lower() for svc in ["bedrock", "sagemaker"] ) ] @@ -471,6 +791,7 @@ def check_aws_budgets_for_aiml() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/cost-management/latest/userguide/budgets-managing-costs.html", severity="Medium", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-06"], ) ) else: @@ -483,6 +804,7 @@ def check_aws_budgets_for_aiml() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/cost-management/latest/userguide/budgets-managing-costs.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-06"], ) ) except Exception as e: @@ -517,6 +839,7 @@ def check_bedrock_agent_action_boundaries(permission_cache) -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/agents-permissions.html", severity="Informational", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-07"], ) ) return findings @@ -572,6 +895,7 @@ def check_bedrock_agent_action_boundaries(permission_cache) -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/agents-permissions.html", severity="High", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-07"], ) ) else: @@ -584,6 +908,7 @@ def check_bedrock_agent_action_boundaries(permission_cache) -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/agents-permissions.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-07"], ) ) except Exception as e: @@ -616,6 +941,7 @@ def check_agentcore_policy_engine() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/security-authorization.html", severity="Low", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-08"], ) ) return findings @@ -631,6 +957,7 @@ def check_agentcore_policy_engine() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/security-authorization.html", severity="Informational", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-08"], ) ) else: @@ -657,6 +984,7 @@ def check_agentcore_policy_engine() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/security-authorization.html", severity="High", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-08"], ) ) else: @@ -669,6 +997,7 @@ def check_agentcore_policy_engine() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/security-authorization.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-08"], ) ) except Exception as e: @@ -724,6 +1053,7 @@ def check_agent_transaction_limits() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/lambda/latest/dg/configuration-concurrency.html", severity="Medium", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-09"], ) ) else: @@ -736,6 +1066,7 @@ def check_agent_transaction_limits() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/lambda/latest/dg/configuration-concurrency.html", severity="Informational", status="Passed" if agent_lambdas else "N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-09"], ) ) except Exception as e: @@ -784,6 +1115,7 @@ def check_human_in_the_loop_for_high_risk_actions() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/step-functions/latest/dg/connect-to-resource.html#connect-wait-token", severity="Medium", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-10"], ) ) elif machines_with_wait: @@ -796,6 +1128,7 @@ def check_human_in_the_loop_for_high_risk_actions() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/step-functions/latest/dg/connect-to-resource.html#connect-wait-token", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-10"], ) ) else: @@ -815,6 +1148,7 @@ def check_human_in_the_loop_for_high_risk_actions() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/step-functions/latest/dg/connect-to-resource.html#connect-wait-token", severity="High", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-10"], ) ) except Exception as e: @@ -861,6 +1195,7 @@ def check_agent_rate_alarms() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/monitoring-cw.html", severity="Medium", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-11"], ) ) else: @@ -873,6 +1208,7 @@ def check_agent_rate_alarms() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/monitoring-cw.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-11"], ) ) except Exception as e: @@ -910,6 +1246,7 @@ def check_scp_model_access_restrictions() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/organizations/latest/userguide/orgs_manage_policies_scps.html", severity="Low", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-12"], ) ) return findings @@ -941,6 +1278,7 @@ def check_scp_model_access_restrictions() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/security_iam_id-based-policy-examples.html", severity="High", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-12"], ) ) else: @@ -953,6 +1291,7 @@ def check_scp_model_access_restrictions() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/security_iam_id-based-policy-examples.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-12"], ) ) except Exception as e: @@ -1012,6 +1351,7 @@ def check_model_inventory_tagging() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/tagging.html", severity="Medium", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-13"], ) ) else: @@ -1024,6 +1364,7 @@ def check_model_inventory_tagging() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/tagging.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-13"], ) ) except Exception as e: @@ -1066,6 +1407,7 @@ def check_model_onboarding_governance() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/config/latest/developerguide/evaluate-config.html", severity="Medium", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-14"], ) ) else: @@ -1078,6 +1420,7 @@ def check_model_onboarding_governance() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/config/latest/developerguide/evaluate-config.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-14"], ) ) except Exception as e: @@ -1113,6 +1456,7 @@ def check_bedrock_model_evaluation_adversarial() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", severity="Medium", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-15"], ) ) else: @@ -1125,6 +1469,7 @@ def check_bedrock_model_evaluation_adversarial() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-15"], ) ) except Exception as e: @@ -1153,6 +1498,7 @@ def check_ecr_image_scanning() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/AmazonECR/latest/userguide/image-scanning.html", severity="Informational", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-16"], ) ) return findings @@ -1180,6 +1526,7 @@ def check_ecr_image_scanning() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/AmazonECR/latest/userguide/image-scanning.html", severity="High", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-16"], ) ) else: @@ -1192,6 +1539,7 @@ def check_ecr_image_scanning() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/AmazonECR/latest/userguide/image-scanning.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-16"], ) ) except Exception as e: @@ -1231,6 +1579,7 @@ def check_feature_store_rollback_capability() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/sagemaker/latest/dg/feature-store.html", severity="Informational", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-20"], ) ) return findings @@ -1260,6 +1609,7 @@ def check_feature_store_rollback_capability() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/sagemaker/latest/dg/feature-store-offline.html", severity="Medium", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-20"], ) ) else: @@ -1272,6 +1622,7 @@ def check_feature_store_rollback_capability() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/sagemaker/latest/dg/feature-store-offline.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-20"], ) ) except Exception as e: @@ -1305,6 +1656,7 @@ def check_training_data_s3_versioning() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/Versioning.html", severity="Informational", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-21"], ) ) return findings @@ -1332,6 +1684,7 @@ def check_training_data_s3_versioning() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/Versioning.html", severity="High", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-21"], ) ) else: @@ -1344,6 +1697,7 @@ def check_training_data_s3_versioning() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/Versioning.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-21"], ) ) except Exception as e: @@ -1399,6 +1753,7 @@ def check_knowledge_base_iam_least_privilege(permission_cache) -> Dict[str, Any] reference="https://docs.aws.amazon.com/bedrock/latest/userguide/security-iam-awsmanpol.html", severity="High", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-22"], ) ) else: @@ -1411,6 +1766,7 @@ def check_knowledge_base_iam_least_privilege(permission_cache) -> Dict[str, Any] reference="https://docs.aws.amazon.com/bedrock/latest/userguide/security-iam-awsmanpol.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-22"], ) ) except Exception as e: @@ -1442,6 +1798,7 @@ def check_knowledge_base_metadata_filtering() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/kb-test-config.html", severity="Informational", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-24"], ) ) return findings @@ -1464,6 +1821,7 @@ def check_knowledge_base_metadata_filtering() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/kb-test-config.html", severity="Medium", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-24"], ) ) except Exception as e: @@ -1500,6 +1858,7 @@ def check_opensearch_serverless_encryption() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-encryption.html", severity="High", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-25"], ) ) else: @@ -1522,6 +1881,7 @@ def check_opensearch_serverless_encryption() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-encryption.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-25"], ) ) except Exception as e: @@ -1559,6 +1919,7 @@ def check_knowledge_base_vpc_access() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-network.html", severity="High", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-26"], ) ) else: @@ -1585,6 +1946,7 @@ def check_knowledge_base_vpc_access() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-network.html", severity="High", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-26"], ) ) else: @@ -1597,6 +1959,7 @@ def check_knowledge_base_vpc_access() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-network.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-26"], ) ) except Exception as e: @@ -1631,6 +1994,7 @@ def check_automated_reasoning_checks() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", severity="Medium", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-27"], ) ) return findings @@ -1661,6 +2025,7 @@ def check_automated_reasoning_checks() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", severity="High", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-27"], ) ) else: @@ -1673,6 +2038,7 @@ def check_automated_reasoning_checks() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-27"], ) ) except Exception as e: @@ -1701,6 +2067,7 @@ def check_guardrail_denied_topics_financial() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", severity="Medium", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-28"], ) ) return findings @@ -1728,11 +2095,15 @@ def check_guardrail_denied_topics_financial() -> Dict[str, Any]: "- Specific investment advice (securities recommendations)\n" "- Credit/lending decisions\n" "- Insurance underwriting advice\n" - "- Tax advice beyond general information" + "- Tax advice beyond general information\n" + "When authoring denied-topic policies, use existing compliance materials " + "as the source: employee policies, training materials, procedure documents, " + "and incident reports (as recommended in PDF \u00a71.2.1 Practical guidance)." ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", severity="High", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-28"], ) ) else: @@ -1741,10 +2112,16 @@ def check_guardrail_denied_topics_financial() -> Dict[str, Any]: check_id="FS-28", finding_name="Guardrails With Topic Policies Found", finding_details=f"Guardrails with topic policies: {', '.join(guardrails_with_topics)}.", - resolution="Verify topics cover regulated financial advice categories.", + resolution=( + "Verify topics cover regulated financial advice categories. " + "When authoring or updating denied-topic policies, use existing compliance " + "materials as the source: employee policies, training materials, procedure " + "documents, and incident reports (as recommended in PDF \u00a71.2.1 Practical guidance)." + ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-28"], ) ) except Exception as e: @@ -1763,7 +2140,7 @@ def check_compliance_disclaimer_in_outputs() -> Dict[str, Any]: findings["csv_data"].append( create_finding( check_id="FS-29", - finding_name="Compliance Disclaimer — Manual Review Required", + finding_name="ADVISORY: Compliance Disclaimer — Manual Review Required", finding_details=( "Application-level compliance disclaimers cannot be verified via AWS APIs. " "Manual review required to confirm GenAI outputs include required regulatory disclosures." @@ -1775,8 +2152,9 @@ def check_compliance_disclaimer_in_outputs() -> Dict[str, Any]: "4. Test disclaimer presence in QA/UAT before production deployment." ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", - severity="Medium", - status="Passed", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-29"], ) ) return findings @@ -1808,6 +2186,7 @@ def check_bedrock_evaluation_compliance_datasets() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", severity="Medium", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-30"], ) ) else: @@ -1820,6 +2199,7 @@ def check_bedrock_evaluation_compliance_datasets() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-30"], ) ) except Exception as e: @@ -1857,6 +2237,7 @@ def check_knowledge_base_data_source_sync() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base-ingest.html", severity="Informational", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-31"], ) ) return findings @@ -1895,6 +2276,7 @@ def check_knowledge_base_data_source_sync() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base-ingest.html", severity="Medium", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-31"], ) ) else: @@ -1907,6 +2289,7 @@ def check_knowledge_base_data_source_sync() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base-ingest.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-31"], ) ) except Exception as e: @@ -1924,7 +2307,7 @@ def check_source_attribution_in_guardrails() -> Dict[str, Any]: findings["csv_data"].append( create_finding( check_id="FS-32", - finding_name="Source Attribution — Manual Review Required", + finding_name="ADVISORY: Source Attribution — Manual Review Required", finding_details=( "Source attribution in GenAI responses cannot be verified via AWS APIs. " "Manual review required to confirm responses include citations." @@ -1936,8 +2319,9 @@ def check_source_attribution_in_guardrails() -> Dict[str, Any]: "4. Consider Bedrock Guardrails grounding checks to validate response accuracy." ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/kb-test-config.html", - severity="Medium", - status="Passed", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-32"], ) ) return findings @@ -1969,6 +2353,7 @@ def check_knowledge_base_integrity_monitoring() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html", severity="Informational", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-33"], ) ) return findings @@ -2015,6 +2400,7 @@ def check_knowledge_base_integrity_monitoring() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html", severity="Medium", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-33"], ) ) else: @@ -2027,6 +2413,7 @@ def check_knowledge_base_integrity_monitoring() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-33"], ) ) except Exception as e: @@ -2070,6 +2457,7 @@ def check_fm_version_currency() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle.html", severity="Medium", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-34"], ) ) else: @@ -2082,6 +2470,7 @@ def check_fm_version_currency() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-34"], ) ) except Exception as e: @@ -2121,6 +2510,7 @@ def check_fmeval_harmful_content() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", severity="Medium", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-35"], ) ) else: @@ -2133,6 +2523,7 @@ def check_fmeval_harmful_content() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-35"], ) ) except Exception as e: @@ -2161,6 +2552,7 @@ def check_guardrail_content_filters() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-filters.html", severity="High", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-36"], ) ) return findings @@ -2190,6 +2582,7 @@ def check_guardrail_content_filters() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-filters.html", severity="High", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-36"], ) ) else: @@ -2202,6 +2595,7 @@ def check_guardrail_content_filters() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-filters.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-36"], ) ) except Exception as e: @@ -2219,7 +2613,7 @@ def check_user_feedback_mechanism() -> Dict[str, Any]: findings["csv_data"].append( create_finding( check_id="FS-37", - finding_name="User Feedback Mechanism — Manual Review Required", + finding_name="ADVISORY: User Feedback Mechanism — Manual Review Required", finding_details=( "User feedback mechanisms for harmful outputs cannot be verified via AWS APIs. " "Manual review required." @@ -2231,8 +2625,9 @@ def check_user_feedback_mechanism() -> Dict[str, Any]: "4. Define SLAs for reviewing flagged content." ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails.html", - severity="Medium", - status="Passed", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-37"], ) ) return findings @@ -2259,6 +2654,7 @@ def check_guardrail_word_filters() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", severity="Medium", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-38"], ) ) return findings @@ -2290,6 +2686,7 @@ def check_guardrail_word_filters() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", severity="Medium", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-38"], ) ) else: @@ -2302,6 +2699,7 @@ def check_guardrail_word_filters() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-38"], ) ) except Exception as e: @@ -2346,6 +2744,7 @@ def check_sagemaker_clarify_bias() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-model-monitor-bias-drift.html", severity="High", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-39"], ) ) else: @@ -2358,6 +2757,7 @@ def check_sagemaker_clarify_bias() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-model-monitor-bias-drift.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-39"], ) ) except Exception as e: @@ -2391,6 +2791,7 @@ def check_bedrock_evaluation_bias_datasets() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", severity="Medium", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-40"], ) ) else: @@ -2403,6 +2804,7 @@ def check_bedrock_evaluation_bias_datasets() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-40"], ) ) except Exception as e: @@ -2446,6 +2848,7 @@ def check_sagemaker_clarify_explainability() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-model-explainability.html", severity="High", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-41"], ) ) else: @@ -2458,6 +2861,7 @@ def check_sagemaker_clarify_explainability() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-model-explainability.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-41"], ) ) except Exception as e: @@ -2495,6 +2899,7 @@ def check_ai_service_cards_documentation() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/sagemaker/latest/dg/model-cards.html", severity="Medium", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-42"], ) ) else: @@ -2507,6 +2912,7 @@ def check_ai_service_cards_documentation() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/sagemaker/latest/dg/model-cards.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-42"], ) ) except Exception as e: @@ -2556,6 +2962,7 @@ def check_cloudwatch_log_pii_masking() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/mask-sensitive-log-data.html", severity="High", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-43"], ) ) else: @@ -2568,6 +2975,7 @@ def check_cloudwatch_log_pii_masking() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/mask-sensitive-log-data.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-43"], ) ) except Exception as e: @@ -2609,6 +3017,7 @@ def check_macie_on_training_data_buckets() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/macie/latest/user/what-is-macie.html", severity="High", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-44"], ) ) else: @@ -2621,6 +3030,7 @@ def check_macie_on_training_data_buckets() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/macie/latest/user/what-is-macie.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-44"], ) ) except Exception as e: @@ -2649,6 +3059,7 @@ def check_guardrail_pii_filters() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-sensitive-filters.html", severity="High", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-45"], ) ) return findings @@ -2682,6 +3093,7 @@ def check_guardrail_pii_filters() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-sensitive-filters.html", severity="High", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-45"], ) ) else: @@ -2694,6 +3106,7 @@ def check_guardrail_pii_filters() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-sensitive-filters.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-45"], ) ) except Exception as e: @@ -2727,6 +3140,7 @@ def check_data_classification_tagging() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html", severity="Informational", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-46"], ) ) return findings @@ -2759,6 +3173,7 @@ def check_data_classification_tagging() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html", severity="Medium", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-46"], ) ) else: @@ -2771,6 +3186,7 @@ def check_data_classification_tagging() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-46"], ) ) except Exception as e: @@ -2805,6 +3221,7 @@ def check_guardrail_grounding_threshold() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", severity="High", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-47"], ) ) return findings @@ -2838,6 +3255,7 @@ def check_guardrail_grounding_threshold() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", severity="High", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-47"], ) ) else: @@ -2850,6 +3268,7 @@ def check_guardrail_grounding_threshold() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-47"], ) ) except Exception as e: @@ -2892,6 +3311,7 @@ def check_rag_knowledge_base_configured() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base.html", severity="Medium", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-48"], ) ) else: @@ -2904,6 +3324,7 @@ def check_rag_knowledge_base_configured() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-48"], ) ) except Exception as e: @@ -2921,7 +3342,7 @@ def check_hallucination_disclaimer_advisory() -> Dict[str, Any]: findings["csv_data"].append( create_finding( check_id="FS-49", - finding_name="Hallucination Disclaimer — Manual Review Required", + finding_name="ADVISORY: Hallucination Disclaimer — Manual Review Required", finding_details=( "Application-level hallucination disclaimers cannot be verified via AWS APIs. " "Manual review required." @@ -2933,8 +3354,9 @@ def check_hallucination_disclaimer_advisory() -> Dict[str, Any]: "3. Test disclaimer presence in QA before production." ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails.html", - severity="Medium", - status="Passed", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-49"], ) ) return findings @@ -2979,6 +3401,7 @@ def check_automated_reasoning_checks_hallucination() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", severity="Medium", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-50"], ) ) else: @@ -2991,6 +3414,7 @@ def check_automated_reasoning_checks_hallucination() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-50"], ) ) except Exception as e: @@ -3019,6 +3443,7 @@ def check_prompt_injection_input_validation() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", severity="High", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-51"], ) ) return findings @@ -3052,6 +3477,7 @@ def check_prompt_injection_input_validation() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", severity="High", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-51"], ) ) else: @@ -3064,6 +3490,7 @@ def check_prompt_injection_input_validation() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-51"], ) ) except Exception as e: @@ -3097,6 +3524,7 @@ def check_bedrock_sdk_version_currency() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/lambda/latest/dg/runtimes-update.html", severity="Informational", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-52"], ) ) return findings @@ -3127,6 +3555,7 @@ def check_bedrock_sdk_version_currency() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/lambda/latest/dg/runtimes-update.html", severity="Medium", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-52"], ) ) else: @@ -3139,6 +3568,7 @@ def check_bedrock_sdk_version_currency() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/lambda/latest/dg/runtimes-update.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-52"], ) ) except Exception as e: @@ -3167,6 +3597,7 @@ def check_waf_sql_injection_rules() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/waf/latest/developerguide/aws-managed-rule-groups-baseline.html", severity="High", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-53"], ) ) return findings @@ -3210,6 +3641,7 @@ def check_waf_sql_injection_rules() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/waf/latest/developerguide/aws-managed-rule-groups-baseline.html", severity="High", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-53"], ) ) else: @@ -3222,6 +3654,7 @@ def check_waf_sql_injection_rules() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/waf/latest/developerguide/aws-managed-rule-groups-baseline.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-53"], ) ) except Exception as e: @@ -3239,7 +3672,7 @@ def check_penetration_testing_evidence() -> Dict[str, Any]: findings["csv_data"].append( create_finding( check_id="FS-54", - finding_name="Penetration Testing — Manual Review Required", + finding_name="ADVISORY: Penetration Testing — Manual Review Required", finding_details=( "Penetration testing evidence cannot be verified via AWS APIs. " "Manual review required to confirm GenAI applications have been tested." @@ -3252,8 +3685,9 @@ def check_penetration_testing_evidence() -> Dict[str, Any]: "5. For DORA compliance, include GenAI in TLPT (Threat-Led Penetration Testing) scope." ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/security.html", - severity="Medium", - status="Passed", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-54"], ) ) return findings @@ -3301,6 +3735,7 @@ def check_output_validation_lambda() -> Dict[str, Any]: reference="https://owasp.org/www-project-top-10-for-large-language-model-applications/", severity="Medium", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-55"], ) ) else: @@ -3313,6 +3748,7 @@ def check_output_validation_lambda() -> Dict[str, Any]: reference="https://owasp.org/www-project-top-10-for-large-language-model-applications/", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-55"], ) ) except Exception as e: @@ -3340,6 +3776,7 @@ def check_xss_prevention_waf() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/waf/latest/developerguide/aws-managed-rule-groups-baseline.html", severity="High", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-56"], ) ) return findings @@ -3361,6 +3798,7 @@ def check_xss_prevention_waf() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/waf/latest/developerguide/aws-managed-rule-groups-baseline.html", severity="Medium", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-56"], ) ) except Exception as e: @@ -3378,7 +3816,7 @@ def check_output_encoding_advisory() -> Dict[str, Any]: findings["csv_data"].append( create_finding( check_id="FS-57", - finding_name="Output Encoding — Manual Review Required", + finding_name="ADVISORY: Output Encoding — Manual Review Required", finding_details=( "Output encoding practices cannot be verified via AWS APIs. " "Manual code review required." @@ -3390,8 +3828,9 @@ def check_output_encoding_advisory() -> Dict[str, Any]: "4. Validate output length and format before passing to downstream APIs." ), reference="https://owasp.org/www-project-top-10-for-large-language-model-applications/", - severity="Medium", - status="Passed", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-57"], ) ) return findings @@ -3431,6 +3870,7 @@ def check_output_schema_validation() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/inference-parameters.html", severity="Medium", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-58"], ) ) except Exception as e: @@ -3459,6 +3899,7 @@ def check_guardrail_topic_allowlist() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", severity="Medium", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-59"], ) ) return findings @@ -3491,6 +3932,7 @@ def check_guardrail_topic_allowlist() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", severity="Medium", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-59"], ) ) else: @@ -3503,6 +3945,7 @@ def check_guardrail_topic_allowlist() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-59"], ) ) except Exception as e: @@ -3521,7 +3964,7 @@ def check_contextual_grounding_for_offtopic() -> Dict[str, Any]: findings["csv_data"].append( create_finding( check_id="FS-60", - finding_name="Contextual Grounding for Off-Topic Prevention", + finding_name="ADVISORY: Contextual Grounding for Off-Topic Prevention", finding_details=( "Contextual grounding for off-topic prevention is covered by guardrail " "grounding checks (FS-47) and RAG configuration (FS-48). " @@ -3533,8 +3976,9 @@ def check_contextual_grounding_for_offtopic() -> Dict[str, Any]: "3. Test with off-topic prompts in QA to verify rejection behavior." ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", - severity="Low", - status="Passed", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-60"], ) ) return findings @@ -3567,6 +4011,7 @@ def check_knowledge_base_sync_schedule() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base-ingest.html", severity="Informational", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-61"], ) ) return findings @@ -3596,6 +4041,7 @@ def check_knowledge_base_sync_schedule() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base-ingest.html", severity="Medium", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-61"], ) ) else: @@ -3608,6 +4054,7 @@ def check_knowledge_base_sync_schedule() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base-ingest.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-61"], ) ) except Exception as e: @@ -3625,7 +4072,7 @@ def check_data_currency_disclaimer_advisory() -> Dict[str, Any]: findings["csv_data"].append( create_finding( check_id="FS-62", - finding_name="Data Currency Disclaimer — Manual Review Required", + finding_name="ADVISORY: Data Currency Disclaimer — Manual Review Required", finding_details=( "Data currency disclaimers cannot be verified via AWS APIs. " "Manual review required." @@ -3637,8 +4084,9 @@ def check_data_currency_disclaimer_advisory() -> Dict[str, Any]: "3. Alert users when KB data is older than defined threshold." ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base-ingest.html", - severity="Low", - status="Passed", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-62"], ) ) return findings @@ -3691,6 +4139,7 @@ def check_foundation_model_lifecycle_policy() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle.html", severity="Medium", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-63"], ) ) else: @@ -3706,6 +4155,7 @@ def check_foundation_model_lifecycle_policy() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-63"], ) ) except Exception as e: @@ -3745,6 +4195,7 @@ def check_kb_datasource_s3_event_notifications() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/NotificationHowTo.html", severity="Informational", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-65"], ) ) return findings @@ -3801,6 +4252,7 @@ def check_kb_datasource_s3_event_notifications() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/EventBridge.html", severity="Medium", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-65"], ) ) else: @@ -3813,6 +4265,7 @@ def check_kb_datasource_s3_event_notifications() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/EventBridge.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-65"], ) ) except Exception as e: @@ -3843,6 +4296,7 @@ def check_agentcore_end_user_identity_propagation() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/security-authorization.html", severity="Low", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-66"], ) ) return findings @@ -3861,6 +4315,7 @@ def check_agentcore_end_user_identity_propagation() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/security-authorization.html", severity="Informational", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-66"], ) ) return findings @@ -3893,6 +4348,7 @@ def check_agentcore_end_user_identity_propagation() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/security-authorization.html", severity="High", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-66"], ) ) else: @@ -3905,6 +4361,7 @@ def check_agentcore_end_user_identity_propagation() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/security-authorization.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-66"], ) ) except Exception as e: @@ -3952,6 +4409,7 @@ def check_agent_financial_transaction_thresholds() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/policy-engine.html", severity="High", status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-67"], ) ) else: @@ -3987,6 +4445,7 @@ def check_agent_financial_transaction_thresholds() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/policy-engine.html", severity="High", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-67"], ) ) else: @@ -4002,6 +4461,7 @@ def check_agent_financial_transaction_thresholds() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/policy-engine.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-67"], ) ) except Exception as e: @@ -4019,7 +4479,6 @@ def check_api_gateway_request_body_size_limits() -> Dict[str, Any]: findings = _empty_findings("API Gateway Request Body Size Limits Check") try: apigw = boto3.client("apigateway", config=boto3_config) - apigwv2 = boto3.client("apigatewayv2", config=boto3_config) wafv2 = boto3.client("wafv2", config=boto3_config) # Check REST APIs for request validators @@ -4074,6 +4533,7 @@ def check_api_gateway_request_body_size_limits() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/waf/latest/developerguide/waf-rule-statement-type-size-constraint.html", severity="Medium", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-68"], ) ) else: @@ -4089,6 +4549,7 @@ def check_api_gateway_request_body_size_limits() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/waf/latest/developerguide/waf-rule-statement-type-size-constraint.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-68"], ) ) except Exception as e: @@ -4144,6 +4605,7 @@ def check_prompt_input_validation_function() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-injection.html", severity="Medium", status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-69"], ) ) else: @@ -4163,6 +4625,7 @@ def check_prompt_input_validation_function() -> Dict[str, Any]: reference="https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-injection.html", severity="Informational", status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-69"], ) ) except Exception as e: @@ -4185,6 +4648,7 @@ def generate_csv_report(findings: List[Dict[str, Any]]) -> str: "Reference", "Severity", "Status", + "Compliance_Frameworks", ] writer = csv.DictWriter(csv_buffer, fieldnames=fieldnames) writer.writeheader() @@ -4204,8 +4668,103 @@ def write_to_s3(execution_id: str, csv_content: str, bucket_name: str) -> str: return f"https://{bucket_name}.s3.amazonaws.com/{file_name}" +def build_finserv_checks(permission_cache): + """ + Single source of truth: ordered (check_id, zero-arg callable) registry of + all FinServ checks, in execution order (FS-01 → FS-69, skipping the ids + merged into upstream: FS-17/18/19/23/64). The two permission-cache checks + are bound with functools.partial so every entry is uniformly zero-arg. + + Driving the handler from this registry lets us attach the correct Check_ID + to a synthesized "could not assess" row when a check errors out, instead of + silently dropping the check from the report. + """ + return [ + # --- Category 1: Unbounded Consumption --- + ("FS-01", check_waf_shield_on_bedrock_endpoints), + ("FS-02", check_api_gateway_rate_limiting), + ("FS-03", check_bedrock_token_quotas), + ("FS-04", check_cost_anomaly_detection), + ("FS-05", check_cloudwatch_token_alarms), + ("FS-06", check_aws_budgets_for_aiml), + # --- Category 2: Excessive Agency --- + ("FS-07", functools.partial(check_bedrock_agent_action_boundaries, permission_cache)), + ("FS-08", check_agentcore_policy_engine), + ("FS-09", check_agent_transaction_limits), + ("FS-10", check_human_in_the_loop_for_high_risk_actions), + ("FS-11", check_agent_rate_alarms), + # --- Category 3: Supply Chain Vulnerabilities --- + ("FS-12", check_scp_model_access_restrictions), + ("FS-13", check_model_inventory_tagging), + ("FS-14", check_model_onboarding_governance), + ("FS-15", check_bedrock_model_evaluation_adversarial), + ("FS-16", check_ecr_image_scanning), + # --- Category 4: Training Data & Model Poisoning --- + ("FS-20", check_feature_store_rollback_capability), + ("FS-21", check_training_data_s3_versioning), + # --- Category 5: Vector & Embedding Weaknesses --- + ("FS-22", functools.partial(check_knowledge_base_iam_least_privilege, permission_cache)), + ("FS-24", check_knowledge_base_metadata_filtering), + ("FS-25", check_opensearch_serverless_encryption), + ("FS-26", check_knowledge_base_vpc_access), + # --- Category 6: Non-Compliant Output --- + ("FS-27", check_automated_reasoning_checks), + ("FS-28", check_guardrail_denied_topics_financial), + ("FS-29", check_compliance_disclaimer_in_outputs), + ("FS-30", check_bedrock_evaluation_compliance_datasets), + # --- Category 7: Misinformation --- + ("FS-31", check_knowledge_base_data_source_sync), + ("FS-32", check_source_attribution_in_guardrails), + ("FS-33", check_knowledge_base_integrity_monitoring), + ("FS-34", check_fm_version_currency), + # --- Category 8: Abusive or Harmful Output --- + ("FS-35", check_fmeval_harmful_content), + ("FS-36", check_guardrail_content_filters), + ("FS-37", check_user_feedback_mechanism), + ("FS-38", check_guardrail_word_filters), + # --- Category 9: Biased Output --- + ("FS-39", check_sagemaker_clarify_bias), + ("FS-40", check_bedrock_evaluation_bias_datasets), + ("FS-41", check_sagemaker_clarify_explainability), + ("FS-42", check_ai_service_cards_documentation), + # --- Category 10: Sensitive Information Disclosure --- + ("FS-43", check_cloudwatch_log_pii_masking), + ("FS-44", check_macie_on_training_data_buckets), + ("FS-45", check_guardrail_pii_filters), + ("FS-46", check_data_classification_tagging), + # --- Category 11: Hallucination --- + ("FS-47", check_guardrail_grounding_threshold), + ("FS-48", check_rag_knowledge_base_configured), + ("FS-49", check_hallucination_disclaimer_advisory), + ("FS-50", check_automated_reasoning_checks_hallucination), + # --- Category 12: Prompt Injection --- + ("FS-51", check_prompt_injection_input_validation), + ("FS-52", check_bedrock_sdk_version_currency), + ("FS-53", check_waf_sql_injection_rules), + ("FS-54", check_penetration_testing_evidence), + # --- Category 13: Improper Output Handling --- + ("FS-55", check_output_validation_lambda), + ("FS-56", check_xss_prevention_waf), + ("FS-57", check_output_encoding_advisory), + ("FS-58", check_output_schema_validation), + # --- Category 14: Off-Topic & Inappropriate Output --- + ("FS-59", check_guardrail_topic_allowlist), + ("FS-60", check_contextual_grounding_for_offtopic), + # --- Category 15: Out-of-Date Training Data --- + ("FS-61", check_knowledge_base_sync_schedule), + ("FS-62", check_data_currency_disclaimer_advisory), + ("FS-63", check_foundation_model_lifecycle_policy), + # --- Material Gap Checks (FS-65 to FS-69) --- + ("FS-65", check_kb_datasource_s3_event_notifications), + ("FS-66", check_agentcore_end_user_identity_propagation), + ("FS-67", check_agent_financial_transaction_thresholds), + ("FS-68", check_api_gateway_request_body_size_limits), + ("FS-69", check_prompt_input_validation_function), + ] + + def lambda_handler(event, context): - """Main Lambda handler — runs all 69 FinServ security checks.""" + """Main Lambda handler — runs all 64 FinServ security checks.""" logger.info("Starting FinServ GenAI security assessment") all_findings = [] @@ -4215,108 +4774,20 @@ def lambda_handler(event, context): "user_permissions": {}, } - # --- Category 1: Unbounded Consumption --- - all_findings.append(check_waf_shield_on_bedrock_endpoints()) - all_findings.append(check_api_gateway_rate_limiting()) - all_findings.append(check_bedrock_token_quotas()) - all_findings.append(check_cost_anomaly_detection()) - all_findings.append(check_cloudwatch_token_alarms()) - all_findings.append(check_aws_budgets_for_aiml()) - - # --- Category 2: Excessive Agency --- - all_findings.append(check_bedrock_agent_action_boundaries(permission_cache)) - all_findings.append(check_agentcore_policy_engine()) - all_findings.append(check_agent_transaction_limits()) - all_findings.append(check_human_in_the_loop_for_high_risk_actions()) - all_findings.append(check_agent_rate_alarms()) - - # --- Category 3: Supply Chain Vulnerabilities --- - all_findings.append(check_scp_model_access_restrictions()) - all_findings.append(check_model_inventory_tagging()) - all_findings.append(check_model_onboarding_governance()) - all_findings.append(check_bedrock_model_evaluation_adversarial()) - all_findings.append(check_ecr_image_scanning()) - - # --- Category 4: Training Data & Model Poisoning --- - # NOTE: FS-17 (check_sagemaker_model_monitor_data_quality), FS-18 (check_sagemaker_model_monitor_drift), - # and FS-19 (check_sagemaker_model_registry_approval) are merged into upstream SM-07, SM-23, SM-22 - # respectively — see extension notes in SECURITY_CHECKS_FINSERV_PART1_INFRA_CONTROLS.md. - all_findings.append(check_feature_store_rollback_capability()) - all_findings.append(check_training_data_s3_versioning()) - - # --- Category 5: Vector & Embedding Weaknesses --- - all_findings.append(check_knowledge_base_iam_least_privilege(permission_cache)) - # NOTE: FS-23 (check_knowledge_base_cloudtrail_logging) is merged into upstream BR-06 - # — see extension note in SECURITY_CHECKS_FINSERV_PART1_INFRA_CONTROLS.md. - all_findings.append(check_knowledge_base_metadata_filtering()) - all_findings.append(check_opensearch_serverless_encryption()) - all_findings.append(check_knowledge_base_vpc_access()) - - # --- Category 6: Non-Compliant Output --- - all_findings.append(check_automated_reasoning_checks()) - all_findings.append(check_guardrail_denied_topics_financial()) - all_findings.append(check_compliance_disclaimer_in_outputs()) - all_findings.append(check_bedrock_evaluation_compliance_datasets()) - - # --- Category 7: Misinformation --- - all_findings.append(check_knowledge_base_data_source_sync()) - all_findings.append(check_source_attribution_in_guardrails()) - all_findings.append(check_knowledge_base_integrity_monitoring()) - all_findings.append(check_fm_version_currency()) - - # --- Category 8: Abusive or Harmful Output --- - all_findings.append(check_fmeval_harmful_content()) - all_findings.append(check_guardrail_content_filters()) - all_findings.append(check_user_feedback_mechanism()) - all_findings.append(check_guardrail_word_filters()) - - # --- Category 9: Biased Output --- - all_findings.append(check_sagemaker_clarify_bias()) - all_findings.append(check_bedrock_evaluation_bias_datasets()) - all_findings.append(check_sagemaker_clarify_explainability()) - all_findings.append(check_ai_service_cards_documentation()) - - # --- Category 10: Sensitive Information Disclosure --- - all_findings.append(check_cloudwatch_log_pii_masking()) - all_findings.append(check_macie_on_training_data_buckets()) - all_findings.append(check_guardrail_pii_filters()) - all_findings.append(check_data_classification_tagging()) - - # --- Category 11: Hallucination --- - all_findings.append(check_guardrail_grounding_threshold()) - all_findings.append(check_rag_knowledge_base_configured()) - all_findings.append(check_hallucination_disclaimer_advisory()) - all_findings.append(check_automated_reasoning_checks_hallucination()) - - # --- Category 12: Prompt Injection --- - all_findings.append(check_prompt_injection_input_validation()) - all_findings.append(check_bedrock_sdk_version_currency()) - all_findings.append(check_waf_sql_injection_rules()) - all_findings.append(check_penetration_testing_evidence()) - - # --- Category 13: Improper Output Handling --- - all_findings.append(check_output_validation_lambda()) - all_findings.append(check_xss_prevention_waf()) - all_findings.append(check_output_encoding_advisory()) - all_findings.append(check_output_schema_validation()) - - # --- Category 14: Off-Topic & Inappropriate Output --- - all_findings.append(check_guardrail_topic_allowlist()) - all_findings.append(check_contextual_grounding_for_offtopic()) - - # --- Category 15: Out-of-Date Training Data --- - all_findings.append(check_knowledge_base_sync_schedule()) - all_findings.append(check_data_currency_disclaimer_advisory()) - all_findings.append(check_foundation_model_lifecycle_policy()) - - # --- Material Gap Checks (FS-65 to FS-69) --- - # NOTE: FS-64 (check_guardrail_trace_logging) is merged into upstream BR-04 - # — see extension note in SECURITY_CHECKS_FINSERV_PART3_APP_LAYER_AND_GAPS.md. - all_findings.append(check_kb_datasource_s3_event_notifications()) - all_findings.append(check_agentcore_end_user_identity_propagation()) - all_findings.append(check_agent_financial_transaction_thresholds()) - all_findings.append(check_api_gateway_request_body_size_limits()) - all_findings.append(check_prompt_input_validation_function()) + # Run every check from the registry. If a check errors out and produces no + # rows, synthesize a visible "could not assess" row (keyed by its Check_ID) + # so the gap surfaces in the report instead of the check silently vanishing. + for check_id, check_fn in build_finserv_checks(permission_cache): + result = check_fn() + if result.get("status") == "ERROR" and not result.get("csv_data"): + result["csv_data"].append( + _could_not_assess_row( + check_id, + result.get("check_name", check_id), + result.get("details", ""), + ) + ) + all_findings.append(result) # Generate and upload report csv_content = generate_csv_report(all_findings) diff --git a/aiml-security-assessment/functions/security/finserv_assessments/requirements.txt b/aiml-security-assessment/functions/security/finserv_assessments/requirements.txt index 572b352..5924120 100644 --- a/aiml-security-assessment/functions/security/finserv_assessments/requirements.txt +++ b/aiml-security-assessment/functions/security/finserv_assessments/requirements.txt @@ -1 +1,13 @@ -pydantic +# FinServ Security Assessment Lambda — runtime dependencies +# +# These pins are scanned by ASH (Automated Security Helper) via Grype + Syft +# during the pre-PR security scan. Keep versions current; if ASH reports a +# CVE against any pinned dep, bump to the patched version before opening the +# PR. See GIT_WORKFLOW.md Step 5 for the ASH workflow. +# +# Dev tooling (ruff, cfn-lint, ash, sam) is NOT pinned here — install those +# in your development environment, not into the Lambda image. + +boto3>=1.43.21 +botocore>=1.43.21 +pydantic>=2.0.0 diff --git a/aiml-security-assessment/functions/security/finserv_assessments/schema.py b/aiml-security-assessment/functions/security/finserv_assessments/schema.py index cc7caa3..c46126f 100644 --- a/aiml-security-assessment/functions/security/finserv_assessments/schema.py +++ b/aiml-security-assessment/functions/security/finserv_assessments/schema.py @@ -2,9 +2,8 @@ Schema module for FinServ security findings. Mirrors the schema used in bedrock_assessments/schema.py. """ - from enum import Enum -from typing import Dict, Any +from typing import Dict, Any, Optional from pydantic import BaseModel, Field, validator import re @@ -40,6 +39,14 @@ class Finding(BaseModel): Reference: str = Field(..., description="Documentation reference URL") Severity: SeverityEnum = Field(..., description="Severity level of the finding") Status: StatusEnum = Field(..., description="Current status of the finding") + Compliance_Frameworks: str = Field( + default="", + description=( + "Pipe-separated list of FinServ regulatory frameworks this control maps to " + "(e.g., 'FFIEC CAT | SR 11-7 | NYDFS 500'). Preliminary; validate with your " + "MRM/Legal/Compliance teams before using as audit evidence." + ), + ) @validator("Check_ID") def validate_check_id(cls, v): @@ -66,8 +73,16 @@ def create_finding( reference: str, severity: SeverityEnum, status: StatusEnum, + compliance_frameworks: Optional[str] = "", ) -> Dict[str, Any]: - """Create a validated finding dict.""" + """Create a validated finding dict. + + Args: + compliance_frameworks: Pipe-separated FinServ regulatory framework identifiers + (e.g., "FFIEC CAT | SR 11-7 | NYDFS 500"). Populated from COMPLIANCE_MAP + in app.py. Preliminary mappings — validate with MRM/Legal/Compliance before + using as audit evidence. + """ finding = Finding( Check_ID=check_id, Finding=finding_name, @@ -76,5 +91,6 @@ def create_finding( Reference=reference, Severity=severity, Status=status, + Compliance_Frameworks=compliance_frameworks or "", ) return dict(finding.model_dump()) From de487a6e9cb030789e6d13258b225d425b9d1400 Mon Sep 17 00:00:00 2001 From: Neil Mehta Date: Wed, 3 Jun 2026 17:26:45 -0400 Subject: [PATCH 08/16] fix(finserv): harden FinServ checks and reconcile IAM after deep audit Apply the second-round, solution-wide audit fixes to the FinServ package, deployment IAM, and docs. Checks (app.py): - Stop conflating IAM AccessDenied with real findings in FS-21/33/46/65 via a shared _is_access_error() helper (access errors now surface as could-not-assess instead of false non-compliant findings). - Remove the silent kbs[:10] cap in FS-33 so all knowledge bases are assessed. - Add a _paginate() helper and apply it to ~35 previously single-page reads (list_functions, list_guardrails, list_evaluation_jobs, list_data_sources, list_knowledge_bases, ECR/Config/Events/SageMaker/API Gateway/Organizations), handling NextToken/nextToken/Marker/position token conventions. - Fix FS-25 status logic: "no policies" is N/A; encryption policies with no customer-managed key now correctly WARN/Failed (was always Passed). - FS-34: report legacy models as "available in region" (availability != usage) and soften the verdict to N/A. - FS-67: disclose the env-var threshold check is a best-effort heuristic. - Harden bucket-ARN parsing via _bucket_name_from_arn(). Schema/deps: - Migrate schema.py to Pydantic v2 @field_validator; cap pydantic<3.0.0. IAM (deployment YAMLs): - Collapse two divergent, partial FinServ statements into one authoritative 45-action statement per file. - Replace invalid actions: bedrock-agent:* -> bedrock:*, budgets:DescribeBudgets -> budgets:ViewBudget; drop bedrock:ListModelPackageGroups. - Add servicequotas:ListAWSDefaultServiceQuotas, bedrock-agentcore Gateways, s3:GetBucketVersioning/Tagging/Notification; remove dead cloudtrail:* and sagemaker:ListModelPackage(Groups). Validated with cfn-lint (no unrecognized actions in the FinServ statement). Docs: - FS-21 detection text corrected (it verifies S3 versioning, not CloudTrail). - Remove stale blank lines in SECURITY_CHECKS_FINSERV_COMMON.md. Verification: ruff clean; 350 unit tests pass; app.py 99% / schema.py 100% coverage; live lambda_handler run against a real account returns 64 findings with zero silent drops. --- .../security/finserv_assessments/app.py | 351 +++++++++++++----- .../finserv_assessments/requirements.txt | 2 +- .../security/finserv_assessments/schema.py | 8 +- deployment/1-aiml-security-member-roles.yaml | 112 +++--- deployment/aiml-security-single-account.yaml | 111 +++--- docs/SECURITY_CHECKS_FINSERV_COMMON.md | 2 - ...ITY_CHECKS_FINSERV_PART1_INFRA_CONTROLS.md | 4 +- 7 files changed, 359 insertions(+), 231 deletions(-) diff --git a/aiml-security-assessment/functions/security/finserv_assessments/app.py b/aiml-security-assessment/functions/security/finserv_assessments/app.py index 4bbec10..9f21394 100644 --- a/aiml-security-assessment/functions/security/finserv_assessments/app.py +++ b/aiml-security-assessment/functions/security/finserv_assessments/app.py @@ -20,9 +20,9 @@ These checks complement the existing BR/SM/AC checks in the AIML Security Assessment. -COMPLIANCE_PLACEHOLDER: Each check includes a comment listing the FinServ regulatory -frameworks it maps to. The prototype report owner should wire these into the compliance -standards column of the HTML report template. +COMPLIANCE_PLACEHOLDER: Each check maps to FinServ regulatory frameworks via the +COMPLIANCE_MAP dict below. These mappings now travel with every finding row in the +CSV report (the Compliance_Frameworks column) — see COMPLIANCE_MAP and create_finding. Frameworks referenced: FFIEC CAT, SR 11-7, NYDFS 500.06, PCI-DSS 12.3.2, SOC 2 CC6, ISO 27001 A.12, DORA Art.6, MAS TRM 9. @@ -92,6 +92,63 @@ def _empty_findings(check_name: str) -> Dict[str, Any]: return {"check_name": check_name, "status": "PASS", "details": "", "csv_data": []} +def _bucket_name_from_arn(bucket_arn: str) -> str: + """Extract the bucket name from an S3 bucket ARN (arn:aws:s3:::name). + + Returns "" when the value is empty or is not a well-formed S3 bucket ARN, so + a malformed/unexpected value is skipped rather than mistakenly used as a + bucket name (which would then fail the subsequent S3 API call).""" + if not bucket_arn or ":::" not in bucket_arn: + return "" + return bucket_arn.split(":::", 1)[1] + + +def _paginate(client, operation_name: str, result_key: str, **kwargs) -> List[Dict[str, Any]]: + """Collect all items across pages for a paginated list/describe operation by + calling the operation directly and following its continuation token. + + Calling the bound method directly (rather than via get_paginator) keeps this + uniform across services and unit-test mocks. The AWS APIs this Lambda uses + employ three continuation-token conventions, all handled here: + - NextToken / NextToken (organizations, sagemaker, config, events, ...) + - nextToken / nextToken (bedrock, bedrock-agent, ecr, ...) + - Marker / NextMarker (lambda) + - position / position (apigateway) + + A single-page response (no continuation token) yields exactly one call, so + this is a safe drop-in for previously non-paginated reads and for mocks that + stub the operation with a single return value. + """ + # (output token field in response, input token kwarg on request) + token_conventions = [ + ("NextToken", "NextToken"), + ("nextToken", "nextToken"), + ("NextMarker", "Marker"), + ("position", "position"), + ] + method = getattr(client, operation_name) + items: List[Dict[str, Any]] = [] + call_kwargs = dict(kwargs) + seen_tokens = set() + while True: + resp = method(**call_kwargs) + items.extend(resp.get(result_key, []) or []) + next_token = None + input_field = None + for out_field, in_field in token_conventions: + if resp.get(out_field): + next_token = resp[out_field] + input_field = in_field + break + # Stop when there is no token, or if a token repeats (guards against a + # mock that returns the same token every call → infinite loop). + if not next_token or next_token in seen_tokens: + break + seen_tokens.add(next_token) + call_kwargs[input_field] = next_token + return items + + def _error_findings(check_name: str, err: Exception) -> Dict[str, Any]: return { "check_name": check_name, @@ -101,6 +158,31 @@ def _error_findings(check_name: str, err: Exception) -> Dict[str, Any]: } +# Error codes that mean "we were not allowed to read this resource" rather than +# "the control is absent." Treating an access error as a compliance failure +# produces a false non-compliant finding caused purely by a missing permission +# (the credibility problem a compliance tool must avoid). +_ACCESS_ERROR_CODES = frozenset( + { + "AccessDenied", + "AccessDeniedException", + "UnauthorizedOperation", + "AuthorizationError", + "Forbidden", + } +) + + +def _is_access_error(err: "ClientError") -> bool: + """True if a ClientError is a permission/authorization error (not a real + 'control absent' signal). Used so a missing IAM permission surfaces as a + could-not-assess condition instead of a false non-compliant finding.""" + try: + return err.response.get("Error", {}).get("Code", "") in _ACCESS_ERROR_CODES + except AttributeError: + return False + + # Findings whose name starts with this prefix were emitted because the check # could not run (e.g., missing IAM permission). They are visible in the report # (Status="N/A") so a failed/permission-denied check does not silently vanish. @@ -351,7 +433,7 @@ def check_api_gateway_rate_limiting() -> Dict[str, Any]: findings = _empty_findings("API Gateway Rate Limiting Check") try: apigw = boto3.client("apigateway", config=boto3_config) - plans = apigw.get_usage_plans().get("items", []) + plans = _paginate(apigw, "get_usage_plans", "items") plans_without_throttle = [ p["name"] @@ -574,7 +656,18 @@ def check_cost_anomaly_detection() -> Dict[str, Any]: findings = _empty_findings("Cost Anomaly Detection Check") try: ce = boto3.client("ce", config=boto3_config) - monitors = ce.get_anomaly_monitors().get("AnomalyMonitors", []) + # get_anomaly_monitors is paginated (NextPageToken). Read every page so a + # Bedrock-covering monitor beyond the first page is not missed (which would + # otherwise produce a false "no coverage" finding). + monitors = [] + next_token = None + while True: + kwargs = {"NextPageToken": next_token} if next_token else {} + resp = ce.get_anomaly_monitors(**kwargs) + monitors.extend(resp.get("AnomalyMonitors", [])) + next_token = resp.get("NextPageToken") + if not next_token: + break # A monitor provides Bedrock/SageMaker service-level coverage if its # spec mentions bedrock (rarely populated for DIMENSIONAL+SERVICE @@ -1014,7 +1107,7 @@ def check_agent_transaction_limits() -> Dict[str, Any]: findings = _empty_findings("Agent Transaction Limits Check") try: lambda_client = boto3.client("lambda", config=boto3_config) - functions = lambda_client.list_functions().get("Functions", []) + functions = _paginate(lambda_client, "list_functions", "Functions") # Look for agent-related Lambda functions without reserved concurrency agent_lambdas = [ @@ -1232,8 +1325,8 @@ def check_scp_model_access_restrictions() -> Dict[str, Any]: try: orgs = boto3.client("organizations", config=boto3_config) try: - policies = orgs.list_policies(Filter="SERVICE_CONTROL_POLICY").get( - "Policies", [] + policies = _paginate( + orgs, "list_policies", "Policies", Filter="SERVICE_CONTROL_POLICY" ) except ClientError as e: if "AccessDenied" in str(e) or "AWSOrganizationsNotInUseException" in str(e): @@ -1314,7 +1407,7 @@ def check_model_inventory_tagging() -> Dict[str, Any]: untagged_models = [] # Check Bedrock custom models - for model in bedrock.list_custom_models().get("modelSummaries", []): + for model in _paginate(bedrock, "list_custom_models", "modelSummaries"): tags_response = bedrock.list_tags_for_resource(resourceARN=model["modelArn"]) tag_keys = {t["key"].lower() for t in tags_response.get("tags", [])} missing = REQUIRED_TAGS - tag_keys @@ -1324,7 +1417,7 @@ def check_model_inventory_tagging() -> Dict[str, Any]: ) # Check SageMaker registered models - for model in sm.list_models().get("Models", []): + for model in _paginate(sm, "list_models", "Models"): tags_response = sm.list_tags(ResourceArn=model["ModelArn"]) tag_keys = {t["Key"].lower() for t in tags_response.get("Tags", [])} missing = REQUIRED_TAGS - tag_keys @@ -1381,7 +1474,7 @@ def check_model_onboarding_governance() -> Dict[str, Any]: findings = _empty_findings("Model Onboarding Governance Check") try: config = boto3.client("config", config=boto3_config) - rules = config.describe_config_rules().get("ConfigRules", []) + rules = _paginate(config, "describe_config_rules", "ConfigRules") bedrock_rules = [ r for r in rules @@ -1437,7 +1530,7 @@ def check_bedrock_model_evaluation_adversarial() -> Dict[str, Any]: findings = _empty_findings("Adversarial Model Evaluation Check") try: bedrock = boto3.client("bedrock", config=boto3_config) - evals = bedrock.list_evaluation_jobs().get("jobSummaries", []) + evals = _paginate(bedrock, "list_evaluation_jobs", "jobSummaries") if not evals: findings["csv_data"].append( @@ -1486,7 +1579,7 @@ def check_ecr_image_scanning() -> Dict[str, Any]: findings = _empty_findings("ECR Image Scanning Check") try: ecr = boto3.client("ecr", config=boto3_config) - repos = ecr.describe_repositories().get("repositories", []) + repos = _paginate(ecr, "describe_repositories", "repositories") if not repos: findings["csv_data"].append( @@ -1567,7 +1660,7 @@ def check_feature_store_rollback_capability() -> Dict[str, Any]: findings = _empty_findings("Feature Store Rollback Check") try: sm = boto3.client("sagemaker", config=boto3_config) - groups = sm.list_feature_groups().get("FeatureGroupSummaries", []) + groups = _paginate(sm, "list_feature_groups", "FeatureGroupSummaries") if not groups: findings["csv_data"].append( @@ -1663,7 +1756,19 @@ def check_training_data_s3_versioning() -> Dict[str, Any]: unversioned = [] for bucket in training_buckets: - versioning = s3.get_bucket_versioning(Bucket=bucket["Name"]) + try: + versioning = s3.get_bucket_versioning(Bucket=bucket["Name"]) + except ClientError as e: + # An access error means we could not read versioning; re-raise so + # it surfaces as could-not-assess rather than a false finding, and + # so one inaccessible bucket does not abort the whole check. + if _is_access_error(e): + raise + logger.warning( + f"Could not check versioning for bucket {bucket['Name']}: {e}" + ) + unversioned.append(f"{bucket['Name']} (error)") + continue if versioning.get("Status") != "Enabled": unversioned.append(bucket["Name"]) @@ -1848,42 +1953,71 @@ def check_opensearch_serverless_encryption() -> Dict[str, Any]: check_id="FS-25", finding_name="No OpenSearch Serverless Encryption Policies", finding_details=( - "No OpenSearch Serverless encryption policies found. " - "Vector embeddings may be stored without customer-managed encryption." + "No OpenSearch Serverless encryption policies found, which indicates " + "no OpenSearch Serverless vector-store collections exist in this region. " + "If Bedrock Knowledge Bases use a different vector store (e.g., Aurora, " + "Pinecone), verify its encryption separately." ), resolution=( - "Create encryption security policies for OpenSearch Serverless collections " - "used as Bedrock KB vector stores, specifying a customer-managed KMS key." + "If using OpenSearch Serverless as a Bedrock KB vector store, create an " + "encryption security policy specifying a customer-managed KMS key." ), reference="https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-encryption.html", - severity="High", + severity="Informational", status="N/A", compliance_frameworks=COMPLIANCE_MAP["FS-25"], ) ) else: - # Check for CMK usage + # Check for CMK usage. A policy that does not reference AWSOwnedKey is + # treated as customer-managed (CMK). cmk_policies = [] for policy in policies: doc = json.loads(policy.get("policy", "{}")) if "AWSOwnedKey" not in json.dumps(doc): cmk_policies.append(policy["name"]) - findings["csv_data"].append( - create_finding( - check_id="FS-25", - finding_name="OpenSearch Serverless Encryption Policies Present", - finding_details=( - f"Found {len(policies)} encryption policy(ies); " - f"{len(cmk_policies)} appear to use CMK." - ), - resolution="Verify all vector store collections use customer-managed KMS keys.", - reference="https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-encryption.html", - severity="Informational", - status="Passed", - compliance_frameworks=COMPLIANCE_MAP["FS-25"], + if not cmk_policies: + # Encryption policies exist but all use AWS-owned keys — the + # control (customer-managed encryption) the check verifies is + # absent, so this is a real finding, not a pass. + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-25", + finding_name="OpenSearch Serverless Encryption Not Using Customer-Managed Keys", + finding_details=( + f"Found {len(policies)} encryption policy(ies), but none use a " + "customer-managed KMS key (CMK) — all rely on AWS-owned keys. " + "FinServ data-protection controls typically require CMKs for " + "key lifecycle control and auditability." + ), + resolution=( + "Update OpenSearch Serverless encryption policies to specify a " + "customer-managed KMS key (KmsARN) instead of AWS-owned keys." + ), + reference="https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-encryption.html", + severity="High", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-25"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-25", + finding_name="OpenSearch Serverless Encryption Policies Present", + finding_details=( + f"Found {len(policies)} encryption policy(ies); " + f"{len(cmk_policies)} use a customer-managed KMS key." + ), + resolution="Verify all vector store collections use customer-managed KMS keys.", + reference="https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-encryption.html", + severity="Informational", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-25"], + ) ) - ) except Exception as e: return _error_findings("OpenSearch Serverless Encryption Check", e) return findings @@ -1982,7 +2116,7 @@ def check_automated_reasoning_checks() -> Dict[str, Any]: findings = _empty_findings("Automated Reasoning Checks") try: bedrock = boto3.client("bedrock", config=boto3_config) - guardrails = bedrock.list_guardrails().get("guardrails", []) + guardrails = _paginate(bedrock, "list_guardrails", "guardrails") if not guardrails: findings["csv_data"].append( @@ -2055,7 +2189,7 @@ def check_guardrail_denied_topics_financial() -> Dict[str, Any]: findings = _empty_findings("Financial Denied Topics Check") try: bedrock = boto3.client("bedrock", config=boto3_config) - guardrails = bedrock.list_guardrails().get("guardrails", []) + guardrails = _paginate(bedrock, "list_guardrails", "guardrails") if not guardrails: findings["csv_data"].append( @@ -2169,7 +2303,7 @@ def check_bedrock_evaluation_compliance_datasets() -> Dict[str, Any]: findings = _empty_findings("Compliance Evaluation Datasets Check") try: bedrock = boto3.client("bedrock", config=boto3_config) - evals = bedrock.list_evaluation_jobs().get("jobSummaries", []) + evals = _paginate(bedrock, "list_evaluation_jobs", "jobSummaries") if not evals: findings["csv_data"].append( @@ -2246,8 +2380,9 @@ def check_knowledge_base_data_source_sync() -> Dict[str, Any]: now = datetime.now(timezone.utc) for kb in kbs: kb_id = kb["knowledgeBaseId"] - sources = bedrock_agent.list_data_sources(knowledgeBaseId=kb_id).get( - "dataSourceSummaries", [] + sources = _paginate( + bedrock_agent, "list_data_sources", "dataSourceSummaries", + knowledgeBaseId=kb_id, ) for source in sources: last_updated = source.get("updatedAt") @@ -2359,10 +2494,11 @@ def check_knowledge_base_integrity_monitoring() -> Dict[str, Any]: return findings buckets_without_versioning = [] - for kb in kbs[:10]: - sources = bedrock_agent.list_data_sources( - knowledgeBaseId=kb["knowledgeBaseId"] - ).get("dataSourceSummaries", []) + for kb in kbs: + sources = _paginate( + bedrock_agent, "list_data_sources", "dataSourceSummaries", + knowledgeBaseId=kb["knowledgeBaseId"], + ) for source in sources: source_detail = bedrock_agent.get_data_source( knowledgeBaseId=kb["knowledgeBaseId"], @@ -2373,15 +2509,22 @@ def check_knowledge_base_integrity_monitoring() -> Dict[str, Any]: .get("dataSourceConfiguration", {}) .get("s3Configuration", {}) ) - bucket = s3_config.get("bucketArn", "").split(":::")[-1] + bucket = _bucket_name_from_arn(s3_config.get("bucketArn", "")) if bucket: try: versioning = s3.get_bucket_versioning(Bucket=bucket) if versioning.get("Status") != "Enabled": buckets_without_versioning.append(bucket) except ClientError as e: - logger.warning(f"Could not check versioning for bucket {bucket}: {e}") - buckets_without_versioning.append(f"{bucket} (access error)") + # An access error means we could not read versioning; do + # not mislabel the bucket as non-versioned. Re-raise so it + # surfaces as could-not-assess instead of a false finding. + if _is_access_error(e): + raise + logger.warning( + f"Could not check versioning for bucket {bucket}: {e}" + ) + buckets_without_versioning.append(f"{bucket} (error)") if buckets_without_versioning: findings["status"] = "WARN" @@ -2444,19 +2587,24 @@ def check_fm_version_currency() -> Dict[str, Any]: findings["csv_data"].append( create_finding( check_id="FS-34", - finding_name="Legacy Foundation Models in Use", + finding_name="Legacy Foundation Models Available in Region", finding_details=( - f"Legacy/deprecated foundation models available: {', '.join(deprecated[:10])}. " - "These models have older training data cutoffs and may produce outdated information." + f"Legacy/deprecated foundation models are available in this account/region: " + f"{', '.join(deprecated[:10])}. This API reports model *availability*, not " + "actual usage — it cannot determine which models your applications invoke. " + "Legacy models have older training-data cutoffs and may produce outdated " + "information if used. Review whether any are in active use." ), resolution=( - "1. Migrate to current model versions.\n" - "2. Document training data cutoff dates for all models in use.\n" - "3. Add data currency disclaimers to outputs from models with old cutoffs." + "1. Identify which (if any) of these legacy models your applications invoke " + "(e.g., via CloudTrail InvokeModel events or application config).\n" + "2. Migrate active usage to current model versions.\n" + "3. Document training-data cutoff dates for all models in use.\n" + "4. Add data-currency disclaimers to outputs from models with old cutoffs." ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle.html", severity="Medium", - status="Failed", + status="N/A", compliance_frameworks=COMPLIANCE_MAP["FS-34"], ) ) @@ -2493,7 +2641,7 @@ def check_fmeval_harmful_content() -> Dict[str, Any]: findings = _empty_findings("FMEval Harmful Content Check") try: bedrock = boto3.client("bedrock", config=boto3_config) - evals = bedrock.list_evaluation_jobs().get("jobSummaries", []) + evals = _paginate(bedrock, "list_evaluation_jobs", "jobSummaries") if not evals: findings["csv_data"].append( @@ -2540,7 +2688,7 @@ def check_guardrail_content_filters() -> Dict[str, Any]: findings = _empty_findings("Guardrail Content Filters Check") try: bedrock = boto3.client("bedrock", config=boto3_config) - guardrails = bedrock.list_guardrails().get("guardrails", []) + guardrails = _paginate(bedrock, "list_guardrails", "guardrails") if not guardrails: findings["csv_data"].append( @@ -2642,7 +2790,7 @@ def check_guardrail_word_filters() -> Dict[str, Any]: findings = _empty_findings("Guardrail Word Filters Check") try: bedrock = boto3.client("bedrock", config=boto3_config) - guardrails = bedrock.list_guardrails().get("guardrails", []) + guardrails = _paginate(bedrock, "list_guardrails", "guardrails") if not guardrails: findings["csv_data"].append( @@ -2716,7 +2864,7 @@ def check_sagemaker_clarify_bias() -> Dict[str, Any]: findings = _empty_findings("SageMaker Clarify Bias Check") try: sm = boto3.client("sagemaker", config=boto3_config) - schedules = sm.list_monitoring_schedules().get("MonitoringScheduleSummaries", []) + schedules = _paginate(sm, "list_monitoring_schedules", "MonitoringScheduleSummaries") bias_schedules = [ s for s in schedules @@ -2774,7 +2922,7 @@ def check_bedrock_evaluation_bias_datasets() -> Dict[str, Any]: findings = _empty_findings("Bedrock Bias Evaluation Datasets Check") try: bedrock = boto3.client("bedrock", config=boto3_config) - evals = bedrock.list_evaluation_jobs().get("jobSummaries", []) + evals = _paginate(bedrock, "list_evaluation_jobs", "jobSummaries") if not evals: findings["csv_data"].append( @@ -2821,7 +2969,7 @@ def check_sagemaker_clarify_explainability() -> Dict[str, Any]: findings = _empty_findings("SageMaker Clarify Explainability Check") try: sm = boto3.client("sagemaker", config=boto3_config) - schedules = sm.list_monitoring_schedules().get("MonitoringScheduleSummaries", []) + schedules = _paginate(sm, "list_monitoring_schedules", "MonitoringScheduleSummaries") explainability_schedules = [ s for s in schedules @@ -2878,7 +3026,7 @@ def check_ai_service_cards_documentation() -> Dict[str, Any]: findings = _empty_findings("AI Service Cards Documentation Check") try: sm = boto3.client("sagemaker", config=boto3_config) - model_cards = sm.list_model_cards().get("ModelCardSummaryList", []) + model_cards = _paginate(sm, "list_model_cards", "ModelCardSummaryList") if not model_cards: findings["status"] = "WARN" @@ -3047,7 +3195,7 @@ def check_guardrail_pii_filters() -> Dict[str, Any]: findings = _empty_findings("Guardrail PII Filters Check") try: bedrock = boto3.client("bedrock", config=boto3_config) - guardrails = bedrock.list_guardrails().get("guardrails", []) + guardrails = _paginate(bedrock, "list_guardrails", "guardrails") if not guardrails: findings["csv_data"].append( @@ -3152,7 +3300,13 @@ def check_data_classification_tagging() -> Dict[str, Any]: tag_keys = {t["Key"].lower() for t in tags} if "data-classification" not in tag_keys and "classification" not in tag_keys: unclassified.append(bucket["Name"]) - except ClientError: + except ClientError as e: + # A genuine "no tags" response (NoSuchTagSet) means the bucket is + # unclassified — a real finding. An access error means we could + # not read the tags; re-raise so it surfaces as could-not-assess + # rather than a false "unclassified" finding. + if _is_access_error(e): + raise unclassified.append(bucket["Name"]) if unclassified: @@ -3209,7 +3363,7 @@ def check_guardrail_grounding_threshold() -> Dict[str, Any]: findings = _empty_findings("Guardrail Grounding Threshold Check") try: bedrock = boto3.client("bedrock", config=boto3_config) - guardrails = bedrock.list_guardrails().get("guardrails", []) + guardrails = _paginate(bedrock, "list_guardrails", "guardrails") if not guardrails: findings["csv_data"].append( @@ -3372,7 +3526,7 @@ def check_automated_reasoning_checks_hallucination() -> Dict[str, Any]: try: bedrock = boto3.client("bedrock", config=boto3_config) # ARC is part of guardrails contextual grounding — check for RELEVANCE filter - guardrails = bedrock.list_guardrails().get("guardrails", []) + guardrails = _paginate(bedrock, "list_guardrails", "guardrails") arc_guardrails = [] for g in guardrails: @@ -3431,7 +3585,7 @@ def check_prompt_injection_input_validation() -> Dict[str, Any]: findings = _empty_findings("Prompt Injection Input Validation Check") try: bedrock = boto3.client("bedrock", config=boto3_config) - guardrails = bedrock.list_guardrails().get("guardrails", []) + guardrails = _paginate(bedrock, "list_guardrails", "guardrails") if not guardrails: findings["csv_data"].append( @@ -3507,7 +3661,7 @@ def check_bedrock_sdk_version_currency() -> Dict[str, Any]: findings = _empty_findings("Bedrock SDK Version Currency Check") try: lambda_client = boto3.client("lambda", config=boto3_config) - functions = lambda_client.list_functions().get("Functions", []) + functions = _paginate(lambda_client, "list_functions", "Functions") bedrock_functions = [ f for f in functions @@ -3709,7 +3863,7 @@ def check_output_validation_lambda() -> Dict[str, Any]: findings = _empty_findings("Output Validation Lambda Check") try: lambda_client = boto3.client("lambda", config=boto3_config) - functions = lambda_client.list_functions().get("Functions", []) + functions = _paginate(lambda_client, "list_functions", "Functions") validation_functions = [ f for f in functions @@ -3846,7 +4000,7 @@ def check_output_schema_validation() -> Dict[str, Any]: try: # Check for EventBridge Pipes or Lambda destinations that could validate outputs lambda_client = boto3.client("lambda", config=boto3_config) - functions = lambda_client.list_functions().get("Functions", []) + functions = _paginate(lambda_client, "list_functions", "Functions") schema_functions = [ f for f in functions @@ -3887,7 +4041,7 @@ def check_guardrail_topic_allowlist() -> Dict[str, Any]: findings = _empty_findings("Guardrail Topic Allowlist Check") try: bedrock = boto3.client("bedrock", config=boto3_config) - guardrails = bedrock.list_guardrails().get("guardrails", []) + guardrails = _paginate(bedrock, "list_guardrails", "guardrails") if not guardrails: findings["csv_data"].append( @@ -4017,7 +4171,7 @@ def check_knowledge_base_sync_schedule() -> Dict[str, Any]: return findings # Check for EventBridge rules that trigger KB sync - rules = events.list_rules().get("Rules", []) + rules = _paginate(events, "list_rules", "Rules") kb_sync_rules = [ r for r in rules if "bedrock" in r.get("Name", "").lower() or "knowledge" in r.get("Name", "").lower() @@ -4113,7 +4267,7 @@ def check_foundation_model_lifecycle_policy() -> Dict[str, Any]: # Check for Config rules or SSM documents related to model lifecycle config_client = boto3.client("config", config=boto3_config) - rules = config_client.describe_config_rules().get("ConfigRules", []) + rules = _paginate(config_client, "describe_config_rules", "ConfigRules") lifecycle_rules = [ r for r in rules if "lifecycle" in r.get("ConfigRuleName", "").lower() @@ -4184,7 +4338,10 @@ def check_kb_datasource_s3_event_notifications() -> Dict[str, Any]: bedrock_agent = boto3.client("bedrock-agent", config=boto3_config) s3_client = boto3.client("s3", config=boto3_config) - kbs = bedrock_agent.list_knowledge_bases().get("knowledgeBaseSummaries", []) + paginator = bedrock_agent.get_paginator("list_knowledge_bases") + kbs = [] + for page in paginator.paginate(): + kbs.extend(page.get("knowledgeBaseSummaries", [])) if not kbs: findings["csv_data"].append( create_finding( @@ -4203,9 +4360,10 @@ def check_kb_datasource_s3_event_notifications() -> Dict[str, Any]: buckets_without_notifications = [] for kb in kbs: kb_id = kb["knowledgeBaseId"] - data_sources = bedrock_agent.list_data_sources( - knowledgeBaseId=kb_id - ).get("dataSourceSummaries", []) + data_sources = _paginate( + bedrock_agent, "list_data_sources", "dataSourceSummaries", + knowledgeBaseId=kb_id, + ) for ds in data_sources: ds_detail = bedrock_agent.get_data_source( knowledgeBaseId=kb_id, @@ -4216,7 +4374,7 @@ def check_kb_datasource_s3_event_notifications() -> Dict[str, Any]: .get("dataSourceConfiguration", {}) .get("s3Configuration", {}) ) - bucket = s3_config.get("bucketArn", "").split(":::")[-1] + bucket = _bucket_name_from_arn(s3_config.get("bucketArn", "")) if not bucket: continue try: @@ -4229,8 +4387,13 @@ def check_kb_datasource_s3_event_notifications() -> Dict[str, Any]: ]) if not has_notif: buckets_without_notifications.append(bucket) - except ClientError: - buckets_without_notifications.append(f"{bucket} (access error)") + except ClientError as e: + # An access error means we could not read the notification + # config; re-raise so it surfaces as could-not-assess rather + # than a false "missing notifications" finding. + if _is_access_error(e): + raise + buckets_without_notifications.append(f"{bucket} (error)") if buckets_without_notifications: findings["status"] = "WARN" @@ -4379,7 +4542,7 @@ def check_agent_financial_transaction_thresholds() -> Dict[str, Any]: findings = _empty_findings("Agent Financial Transaction Value Thresholds Check") try: lambda_client = boto3.client("lambda", config=boto3_config) - functions = lambda_client.list_functions().get("Functions", []) + functions = _paginate(lambda_client, "list_functions", "Functions") # Look for agent action-group Lambda functions action_group_lambdas = [ @@ -4431,7 +4594,10 @@ def check_agent_financial_transaction_thresholds() -> Dict[str, Any]: finding_name="Agent Action-Group Lambdas May Lack Transaction Thresholds", finding_details=( "The following agent action-group Lambda functions have no environment " - "variables indicating transaction-value threshold configuration. " + "variables whose names suggest transaction-value threshold configuration " + "(this is a best-effort heuristic — a threshold enforced in code or in an " + "AgentCore Policy Engine rule would not be detected here, so treat this as " + "a prompt for manual verification rather than a definitive gap). " "Without explicit limits, agents could initiate unbounded financial transactions:\n" + "\n".join(f"- {n}" for n in lambdas_without_threshold_config[:10]) ), @@ -4482,7 +4648,7 @@ def check_api_gateway_request_body_size_limits() -> Dict[str, Any]: wafv2 = boto3.client("wafv2", config=boto3_config) # Check REST APIs for request validators - rest_apis = apigw.get_rest_apis().get("items", []) + rest_apis = _paginate(apigw, "get_rest_apis", "items") apis_without_validators = [] for api in rest_apis: validators = apigw.get_request_validators(restApiId=api["id"]).get("items", []) @@ -4567,7 +4733,7 @@ def check_prompt_input_validation_function() -> Dict[str, Any]: findings = _empty_findings("Prompt Input Validation Function Check") try: lambda_client = boto3.client("lambda", config=boto3_config) - functions = lambda_client.list_functions().get("Functions", []) + functions = _paginate(lambda_client, "list_functions", "Functions") # Look for Lambda functions with input validation / sanitization naming patterns VALIDATION_KEYWORDS = [ @@ -4774,17 +4940,24 @@ def lambda_handler(event, context): "user_permissions": {}, } - # Run every check from the registry. If a check errors out and produces no - # rows, synthesize a visible "could not assess" row (keyed by its Check_ID) - # so the gap surfaces in the report instead of the check silently vanishing. + # Run every check from the registry. If a check produces no rows for ANY + # reason (an ERROR envelope, or an unexpected empty non-error result), + # synthesize a visible "could not assess" row (keyed by its Check_ID) so the + # gap surfaces in the report instead of the check silently vanishing. The + # guard intentionally keys off empty csv_data (not just status=="ERROR") so + # the no-silent-drop invariant holds structurally, not by data coincidence. for check_id, check_fn in build_finserv_checks(permission_cache): result = check_fn() - if result.get("status") == "ERROR" and not result.get("csv_data"): - result["csv_data"].append( + if not result.get("csv_data"): + details = result.get("details", "") or ( + f"check returned status={result.get('status', 'UNKNOWN')!r} " + "with no findings" + ) + result.setdefault("csv_data", []).append( _could_not_assess_row( check_id, result.get("check_name", check_id), - result.get("details", ""), + details, ) ) all_findings.append(result) diff --git a/aiml-security-assessment/functions/security/finserv_assessments/requirements.txt b/aiml-security-assessment/functions/security/finserv_assessments/requirements.txt index 5924120..183ad4f 100644 --- a/aiml-security-assessment/functions/security/finserv_assessments/requirements.txt +++ b/aiml-security-assessment/functions/security/finserv_assessments/requirements.txt @@ -10,4 +10,4 @@ boto3>=1.43.21 botocore>=1.43.21 -pydantic>=2.0.0 +pydantic>=2.0.0,<3.0.0 diff --git a/aiml-security-assessment/functions/security/finserv_assessments/schema.py b/aiml-security-assessment/functions/security/finserv_assessments/schema.py index c46126f..d8f7b6a 100644 --- a/aiml-security-assessment/functions/security/finserv_assessments/schema.py +++ b/aiml-security-assessment/functions/security/finserv_assessments/schema.py @@ -4,7 +4,7 @@ """ from enum import Enum from typing import Dict, Any, Optional -from pydantic import BaseModel, Field, validator +from pydantic import BaseModel, Field, field_validator import re @@ -48,7 +48,8 @@ class Finding(BaseModel): ), ) - @validator("Check_ID") + @field_validator("Check_ID") + @classmethod def validate_check_id(cls, v): # Allow FS-NN pattern for FinServ checks pattern = r"^[A-Z]{2,3}-\d{2}$" @@ -58,7 +59,8 @@ def validate_check_id(cls, v): ) return v - @validator("Reference") + @field_validator("Reference") + @classmethod def validate_reference_url(cls, v): if not str(v).startswith("https://"): raise ValueError("Reference URL must start with https://") diff --git a/deployment/1-aiml-security-member-roles.yaml b/deployment/1-aiml-security-member-roles.yaml index ec2c907..2064c67 100644 --- a/deployment/1-aiml-security-member-roles.yaml +++ b/deployment/1-aiml-security-member-roles.yaml @@ -204,9 +204,10 @@ Resources: - guardduty:GetUsageStatistics Resource: "*" # FinServ GenAI Risk Assessment Permissions (FS-01 to FS-69) - # These read-only actions are required by the 64 FinServ security checks. - # All are list/describe/get variants; Resource: "*" is required for - # inventory-style checks where resource ARNs are not known in advance. + # Read-only list/describe/get actions required by the 64 FinServ + # security checks. Resource: "*" is required for inventory-style + # checks where resource ARNs are not known in advance. Validated + # with cfn-lint (no unrecognized actions). - Effect: Allow Sid: FinServGenAIRiskAssessmentPermissions Action: @@ -216,40 +217,63 @@ Resources: - shield:DescribeSubscription # API Gateway (FS-02, FS-68, FS-69) - apigateway:GET + # Service Quotas (FS-03) + - servicequotas:ListServiceQuotas + - servicequotas:ListAWSDefaultServiceQuotas # Cost Explorer / Budgets (FS-04, FS-06) - ce:GetAnomalyMonitors - - budgets:DescribeBudgets - # CloudWatch (FS-05, FS-11) + - budgets:ViewBudget + # CloudWatch / Logs (FS-05, FS-11, FS-43) - cloudwatch:DescribeAlarms - # CloudWatch Logs — account-level data protection policies (FS-43) - logs:DescribeAccountPolicies - # Lambda — concurrency limits for agent transaction checks (FS-09, FS-67, FS-69) + # Bedrock — guardrails, models, evaluation (FS-15, FS-27..50, FS-59..63) + - bedrock:ListGuardrails + - bedrock:GetGuardrail + - bedrock:ListFoundationModels + - bedrock:ListCustomModels + - bedrock:ListEvaluationJobs + # Bedrock — agents and knowledge bases (FS-07, FS-24, FS-31, FS-33, FS-48, FS-61, FS-65) + - bedrock:ListAgents + - bedrock:GetAgent + - bedrock:ListKnowledgeBases + - bedrock:GetKnowledgeBase + - bedrock:ListDataSources + - bedrock:GetDataSource + # Bedrock AgentCore (FS-08, FS-66) + - bedrock-agentcore:ListAgentRuntimes + - bedrock-agentcore:GetAgentRuntime + - bedrock-agentcore:ListGateways + - bedrock-agentcore:GetGateway + # Lambda — concurrency / agent transaction checks (FS-09, FS-67, FS-69) + - lambda:ListFunctions - lambda:GetFunctionConcurrency - # Step Functions — human-in-the-loop and agent rate checks (FS-10, FS-11) + # Step Functions — human-in-the-loop / rate checks (FS-10, FS-11) - states:ListStateMachines - states:DescribeStateMachine # Organizations — SCP model access restrictions (FS-12) - organizations:ListPolicies - organizations:DescribePolicy - # SageMaker — model cards documentation (FS-42) + # SageMaker — inventory / monitoring / bias / cards (FS-13, FS-20, FS-39..42, FS-61, FS-63) + - sagemaker:ListMonitoringSchedules + - sagemaker:ListFeatureGroups + - sagemaker:ListModels - sagemaker:ListModelCards - # Bedrock — evaluation jobs, foundation models (FS-15, FS-30, FS-34, FS-40) - - bedrock:ListEvaluationJobs - - bedrock:ListFoundationModels - # Bedrock — model package groups (supply chain / FS-13, FS-14) - - bedrock:ListModelPackageGroups - # Macie — PII detection on training data buckets (FS-44) - - macie2:GetMacieSession - # OpenSearch Serverless — KB vector store encryption/VPC (FS-25, FS-26) + - sagemaker:ListTags + # S3 — bucket metadata (FS-20, FS-21, FS-46, FS-65) + - s3:ListAllMyBuckets + - s3:GetBucketVersioning + - s3:GetBucketTagging + - s3:GetBucketNotification + # ECR — image scanning (FS-16) + - ecr:DescribeRepositories + # OpenSearch Serverless — KB vector store (FS-25, FS-26) - aoss:ListSecurityPolicies + # Macie — PII detection on training data (FS-44) + - macie2:GetMacieSession # EventBridge — KB integrity monitoring (FS-33, FS-65) - events:ListRules # Config — model onboarding governance (FS-14) - config:DescribeConfigRules - # S3 — list all buckets for training data versioning checks (FS-21, FS-44) - - s3:ListAllMyBuckets - # SageMaker — list tags for model inventory provenance checks (FS-13) - - sagemaker:ListTags Resource: "*" # Allow access to central assessment bucket - Effect: Allow @@ -366,52 +390,6 @@ Resources: - "arn:aws:s3:::aws-sam-cli-managed-default-*" - "arn:aws:s3:::aws-sam-cli-managed-default-*/*" - # FinServ GenAI Risk Assessment Permissions (FS-01 to FS-69) - - Effect: Allow - Action: - # WAF / Shield (FS-01, FS-51..54) - - wafv2:ListWebACLs - - wafv2:GetWebACL - - shield:DescribeSubscription - # API Gateway (FS-02, FS-68) - - apigateway:GET - # Service Quotas (FS-03) - - servicequotas:ListServiceQuotas - - servicequotas:ListAWSDefaultServiceQuotas - # Cost / Budgets (FS-04, FS-06) - - ce:GetAnomalyMonitors - - budgets:ViewBudget - # CloudWatch (FS-05, FS-11, FS-21, FS-33, FS-46) - - cloudwatch:DescribeAlarms - - cloudwatch:ListMetrics - # CloudWatch Logs (FS-43, FS-64) - - logs:DescribeAccountPolicies - # Bedrock — guardrail and evaluation (FS-27..30, FS-36..38, FS-43..45, FS-47..50, FS-59..60) - - bedrock:ListEvaluationJobs - - bedrock:GetEvaluationJob - - bedrock:ListFoundationModels - # Bedrock Agent — runtimes and data sources (FS-31, FS-65) - - bedrock-agent:ListDataSources - - bedrock-agent:GetDataSource - # SageMaker — additional inventory (FS-42, FS-61, FS-63) - - sagemaker:ListModelCards - - sagemaker:ListTags - # S3 — additional bucket metadata (FS-20, FS-46, FS-65) - - s3:ListAllMyBuckets - - s3:GetBucketNotification - # ECR — image-scan findings (FS-16) - - ecr:DescribeImageScanFindings - # OpenSearch Serverless (FS-25, FS-26) - - aoss:ListSecurityPolicies - # Macie — PII protection (FS-44) - - macie2:GetMacieSession - # EventBridge / Config (FS-32, FS-34) - - events:ListRules - - config:DescribeConfigRules - # Organizations — SCPs (FS-12) - - organizations:ListPolicies - - organizations:DescribePolicy - Resource: "*" Outputs: MemberRoleArn: diff --git a/deployment/aiml-security-single-account.yaml b/deployment/aiml-security-single-account.yaml index cc205ce..2b68f6c 100644 --- a/deployment/aiml-security-single-account.yaml +++ b/deployment/aiml-security-single-account.yaml @@ -230,99 +230,76 @@ Resources: Resource: "*" # FinServ GenAI Risk Assessment Permissions (FS-01 to FS-69) + # Read-only list/describe/get actions required by the 64 FinServ + # security checks. Resource: "*" is required for inventory-style + # checks where resource ARNs are not known in advance. Validated + # with cfn-lint (no unrecognized actions). - Effect: Allow + Sid: FinServGenAIRiskAssessmentPermissions Action: - # WAF / Shield (FS-01, FS-51..54) + # WAF / Shield (FS-01) - wafv2:ListWebACLs - wafv2:GetWebACL - shield:DescribeSubscription - # API Gateway (FS-02, FS-68) + # API Gateway (FS-02, FS-68, FS-69) - apigateway:GET # Service Quotas (FS-03) - servicequotas:ListServiceQuotas - servicequotas:ListAWSDefaultServiceQuotas - # Cost / Budgets (FS-04, FS-06) + # Cost Explorer / Budgets (FS-04, FS-06) - ce:GetAnomalyMonitors - budgets:ViewBudget - # CloudWatch (FS-05, FS-11, FS-21, FS-33, FS-46) + # CloudWatch / Logs (FS-05, FS-11, FS-43) - cloudwatch:DescribeAlarms - - cloudwatch:ListMetrics - # CloudWatch Logs (FS-43, FS-64) - logs:DescribeAccountPolicies - # Bedrock — guardrail and evaluation (FS-27..30, FS-36..38, FS-43..45, FS-47..50, FS-59..60) - - bedrock:ListEvaluationJobs - - bedrock:GetEvaluationJob + # Bedrock — guardrails, models, evaluation (FS-15, FS-27..50, FS-59..63) + - bedrock:ListGuardrails + - bedrock:GetGuardrail - bedrock:ListFoundationModels - # Bedrock Agent — runtimes and data sources (FS-31, FS-65) - - bedrock-agent:ListDataSources - - bedrock-agent:GetDataSource - # SageMaker — additional inventory (FS-42, FS-61, FS-63) - - sagemaker:ListModelCards - - sagemaker:ListTags - # S3 — additional bucket metadata (FS-20, FS-46, FS-65) - - s3:ListAllMyBuckets - - s3:GetBucketNotification - # ECR — image-scan findings (FS-16) - - ecr:DescribeImageScanFindings - # OpenSearch Serverless (FS-25, FS-26) - - aoss:ListSecurityPolicies - # Macie — PII protection (FS-44) - - macie2:GetMacieSession - # EventBridge / Config (FS-32, FS-34) - - events:ListRules - - config:DescribeConfigRules - # Organizations — SCPs (FS-12) - - organizations:ListPolicies - - organizations:DescribePolicy - Resource: "*" - - # FinServ GenAI Risk Assessment Permissions (FS-01 to FS-69) - # These read-only actions are required by the 64 FinServ security checks. - # All are list/describe/get variants; Resource: "*" is required for - # inventory-style checks where resource ARNs are not known in advance. - - Effect: Allow - Sid: FinServGenAIRiskAssessmentPermissions - Action: - # WAF / Shield (FS-01) - - wafv2:ListWebACLs - - wafv2:GetWebACL - - shield:DescribeSubscription - # API Gateway (FS-02, FS-68, FS-69) - - apigateway:GET - # Cost Explorer / Budgets (FS-04, FS-06) - - ce:GetAnomalyMonitors - - budgets:DescribeBudgets - # CloudWatch (FS-05, FS-11) - - cloudwatch:DescribeAlarms - # CloudWatch Logs — account-level data protection policies (FS-43) - - logs:DescribeAccountPolicies - # Lambda — concurrency limits for agent transaction checks (FS-09, FS-67, FS-69) + - bedrock:ListCustomModels + - bedrock:ListEvaluationJobs + # Bedrock — agents and knowledge bases (FS-07, FS-24, FS-31, FS-33, FS-48, FS-61, FS-65) + - bedrock:ListAgents + - bedrock:GetAgent + - bedrock:ListKnowledgeBases + - bedrock:GetKnowledgeBase + - bedrock:ListDataSources + - bedrock:GetDataSource + # Bedrock AgentCore (FS-08, FS-66) + - bedrock-agentcore:ListAgentRuntimes + - bedrock-agentcore:GetAgentRuntime + - bedrock-agentcore:ListGateways + - bedrock-agentcore:GetGateway + # Lambda — concurrency / agent transaction checks (FS-09, FS-67, FS-69) + - lambda:ListFunctions - lambda:GetFunctionConcurrency - # Step Functions — human-in-the-loop and agent rate checks (FS-10, FS-11) + # Step Functions — human-in-the-loop / rate checks (FS-10, FS-11) - states:ListStateMachines - states:DescribeStateMachine # Organizations — SCP model access restrictions (FS-12) - organizations:ListPolicies - organizations:DescribePolicy - # SageMaker — model cards documentation (FS-42) + # SageMaker — inventory / monitoring / bias / cards (FS-13, FS-20, FS-39..42, FS-61, FS-63) + - sagemaker:ListMonitoringSchedules + - sagemaker:ListFeatureGroups + - sagemaker:ListModels - sagemaker:ListModelCards - # Bedrock — evaluation jobs, foundation models (FS-15, FS-30, FS-34, FS-40) - - bedrock:ListEvaluationJobs - - bedrock:ListFoundationModels - # Bedrock — model package groups (supply chain / FS-13, FS-14) - - bedrock:ListModelPackageGroups - # Macie — PII detection on training data buckets (FS-44) - - macie2:GetMacieSession - # OpenSearch Serverless — KB vector store encryption/VPC (FS-25, FS-26) + - sagemaker:ListTags + # S3 — bucket metadata (FS-20, FS-21, FS-46, FS-65) + - s3:ListAllMyBuckets + - s3:GetBucketVersioning + - s3:GetBucketTagging + - s3:GetBucketNotification + # ECR — image scanning (FS-16) + - ecr:DescribeRepositories + # OpenSearch Serverless — KB vector store (FS-25, FS-26) - aoss:ListSecurityPolicies + # Macie — PII detection on training data (FS-44) + - macie2:GetMacieSession # EventBridge — KB integrity monitoring (FS-33, FS-65) - events:ListRules # Config — model onboarding governance (FS-14) - config:DescribeConfigRules - # S3 — list all buckets for training data versioning checks (FS-21, FS-44) - - s3:ListAllMyBuckets - # SageMaker — list tags for model inventory provenance checks (FS-13) - - sagemaker:ListTags Resource: "*" # CodeBuild role for running assessments diff --git a/docs/SECURITY_CHECKS_FINSERV_COMMON.md b/docs/SECURITY_CHECKS_FINSERV_COMMON.md index 552039d..1aa9740 100644 --- a/docs/SECURITY_CHECKS_FINSERV_COMMON.md +++ b/docs/SECURITY_CHECKS_FINSERV_COMMON.md @@ -61,7 +61,6 @@ availability of new features (Automated Reasoning, AgentCore Policy, AWS Securit cross-account guardrails) evolves rapidly — region lists in Parts 1-3 reflect the state at the cited announcement date and should be re-verified before audit reliance. - ## Contribution workflow The FS checks are contributed upstream as a single pull request via a personal GitHub fork @@ -134,7 +133,6 @@ whether the FS check adds FinServ-specific regulatory specificity, (3) severity After consolidation the combined framework contains **52 upstream + 64 FS = 116 distinct checks** (down from 52 + 69 = 121 before merging). The consolidation reduces duplication without losing FinServ-specific regulatory depth. - --- ## Compliance Framework Mapping diff --git a/docs/SECURITY_CHECKS_FINSERV_PART1_INFRA_CONTROLS.md b/docs/SECURITY_CHECKS_FINSERV_PART1_INFRA_CONTROLS.md index 8e06b53..5c22a2a 100644 --- a/docs/SECURITY_CHECKS_FINSERV_PART1_INFRA_CONTROLS.md +++ b/docs/SECURITY_CHECKS_FINSERV_PART1_INFRA_CONTROLS.md @@ -319,8 +319,8 @@ See `SECURITY_CHECKS_FINSERV_COMMON.md` for: |-------|--------| | Severity | High | | PDF ref | [PDF §1.2.14] — "Use trusted data sources for your training data. Implement audit controls that let you track and review changes, including who made them and when they occurred." | -| Description | Verifies S3 buckets with training data have versioning enabled and CloudTrail data-event logging active to record who modified training data and when. | -| Detection | Identifies training-data S3 buckets by tag (`data-classification=training` or `ml-purpose=training`) or by naming convention. Calls `s3:GetBucketVersioning` to verify `Status=Enabled`. Calls `cloudtrail:GetEventSelectors` on active trails to verify S3 data events are logged for these buckets. | +| Description | Verifies S3 buckets used for training data have versioning enabled so poisoned datasets can be rolled back. Recommends CloudTrail data-event logging as remediation to record who modified training data and when. | +| Detection | Identifies training-data S3 buckets by naming convention (`train`/`dataset`/`model`/`sagemaker`/`bedrock`). Calls `s3:GetBucketVersioning` to verify `Status=Enabled`. (CloudTrail data-event logging is recommended in remediation but is not asserted by this check — verifying it is covered by the upstream BR-06 CloudTrail control and the FS-23 extension.) | | Remediation | 1. Enable versioning: `aws s3api put-bucket-versioning --bucket --versioning-configuration Status=Enabled`. 2. Enable CloudTrail S3 data events for the training-data buckets to capture PutObject/DeleteObject with caller identity. 3. Enable MFA Delete for critical training datasets. 4. Apply S3 Object Lock for immutable baselines. | | Reference | [S3 Versioning](https://docs.aws.amazon.com/AmazonS3/latest/userguide/Versioning.html), [CloudTrail Data Events](https://docs.aws.amazon.com/awscloudtrail/latest/userguide/logging-data-events-with-cloudtrail.html) | From 3b05f7c37179d1b7b99071705ec7e49373567249 Mon Sep 17 00:00:00 2001 From: Neil Mehta Date: Thu, 4 Jun 2026 13:08:55 -0400 Subject: [PATCH 09/16] fix(finserv): Deep audit fixes, new ARC check, scheduler detection, and NoSuchBucket refinement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This commit incorporates three rounds of systematic solution-wide audit improvements to the FinServ GenAI risk assessment (TASK-17 through TASK-19). == Code quality and correctness (app.py) == FS-27 — Split into two checks sharing the FS-27 check_id: 1. check_guardrail_contextual_grounding() verifies contextualGroundingPolicy (threshold-based, per-inference). 2. check_automated_reasoning_policies() verifies Bedrock Automated Reasoning policies (formal verification, GA August 2025; bedrock:ListAutomatedReasoning Policies). Handles access-denied / unsupported-region gracefully. FS-28 / FS-59 — Denied-topics Standard tier advisory: both checks now surface a Low-severity advisory when denied topics exist but use the CLASSIC tier, consistent with the FS-36 content-filter tier advisory (June 2025 GA). FS-31 — Hardcoded 7-day staleness threshold promoted to a named STALE_AFTER_DAYS constant with a docstring note that it is a configurable review prompt, not an AWS-mandated limit. FS-34 / FS-63 — list_foundation_models() byOutputModality='TEXT' filter removed from both checks; now fetches all modalities (TEXT, EMBEDDING, IMAGE) so legacy embedding models used in FinServ RAG pipelines are not silently missed. FS-47 — False-pass fixed: guardrails that exist but have no GROUNDING filter at all (only RELEVANCE) now correctly WARN/Failed instead of silently Passing. Reference URL updated to guardrails-contextual-grounding-check.html. FS-50 — Renamed check_automated_reasoning_checks_hallucination() to check_guardrail_relevance_grounding() for accurate labeling. FS-52 — Deprecated-runtime check inverted from denylist to allowlist (SUPPORTED_RUNTIMES) sourced from the live AWS Lambda runtimes page (June 2026); catches python3.9, python3.10, nodejs18.x, nodejs20.x, and all other deprecated runtimes the old 4-entry denylist missed. FS-54 — Resolution text updated to reference AWS Security Agent (GA March 2026, 6 regions, cross-account shared-VPC via RAM) for on-demand GenAI pen testing. FS-61 — EventBridge Scheduler detection added alongside legacy EventBridge rules: customers following the AWS-recommended scheduler:ListSchedules approach no longer get a false WARN. Access-denied on Scheduler degrades gracefully. FS-63 — byOutputModality='TEXT' filter removed (same fix as FS-34). FS-33 / FS-65 — NoSuchBucket distinguished from other errors via a new _is_missing_bucket_error() helper. A KB data-source pointing to a deleted bucket now produces a distinct 'KB Data Source References a Deleted S3 Bucket' finding (Severity: High) instead of the misleading 'without versioning' / 'missing notifications' / '(error)' mislabel. Verified against a real dangling reference in the test account. == IAM permissions == Added to both deployment YAMLs (FinServGenAIRiskAssessmentPermissions statement): - scheduler:ListSchedules (FS-61 EventBridge Scheduler check) - bedrock:ListAutomatedReasoningPolicies (FS-27 ARC policies check) == Schema == Pydantic v2 migration already in prior commit; requirements.txt floor unchanged. == Docs == GIT_WORKFLOW.md: corrected '11 steps' to '9 steps'. scripts/*.py + convert_to_pdf.py: ruff format applied (cosmetic only — confirmed byte-identical build output before and after). == Verification == - ruff check + ruff format --check: all 16 real Python files pass. - Unit tests: 368 pass (up from 350; covers all new branches including scheduler fallback, FS-47 no-GROUNDING-filter, FS-28/FS-59 CLASSIC tier, FS-33/FS-65 deleted-bucket, _is_missing_bucket_error helper). - Live full handler run (account 469898429403): 65 checks, 0 ERROR, 0 EXCEPTION. New ARC-policies check and scheduler path execute cleanly. - Sample report regenerated; FS-33/FS-65 now show the High-severity deleted-bucket finding for the real dangling reference in the test account. --- .../security/finserv_assessments/app.py | 1052 ++++++++++++++--- .../security/finserv_assessments/schema.py | 1 + deployment/1-aiml-security-member-roles.yaml | 6 + deployment/aiml-security-single-account.yaml | 6 + .../finserv_security_report_phase3.csv | 190 +++ 5 files changed, 1080 insertions(+), 175 deletions(-) create mode 100644 sample-reports/finserv_security_report_phase3.csv diff --git a/aiml-security-assessment/functions/security/finserv_assessments/app.py b/aiml-security-assessment/functions/security/finserv_assessments/app.py index 9f21394..b875c0f 100644 --- a/aiml-security-assessment/functions/security/finserv_assessments/app.py +++ b/aiml-security-assessment/functions/security/finserv_assessments/app.py @@ -1,8 +1,8 @@ - """ AWS FinServ GenAI Risk Assessment Lambda ========================================= -Implements 64 standalone security checks derived from the AWS guide: +Implements 65 security checks (64 standalone + 1 shared FS-27 entry for the new +Automated Reasoning Policies check) derived from the AWS guide: "Financial Services risk management of the use of Generative AI" https://d1.awsstatic.com/onedam/marketing-channels/website/public/global-FinServ-ComplianceGuide-GenAIRisks-public.pdf @@ -18,6 +18,13 @@ rather than standalone entries — see extension notes in the SECURITY_CHECKS_FINSERV Part 1 and Part 3 markdown files. +FS-27 is split into two check functions sharing the same check_id: + 1. check_guardrail_contextual_grounding() — verifies contextualGroundingPolicy + on guardrails (threshold-based, per-inference filtering). + 2. check_automated_reasoning_policies() — verifies Bedrock Automated Reasoning + policies (formal verification, GA August 2025, limited regions). +Both use FS-27 in the CSV and both appear in the build_finserv_checks() registry. + These checks complement the existing BR/SM/AC checks in the AIML Security Assessment. COMPLIANCE_PLACEHOLDER: Each check maps to FinServ regulatory frameworks via the @@ -72,6 +79,7 @@ # Helpers # --------------------------------------------------------------------------- + def get_permissions_cache(execution_id: str) -> Optional[Dict[str, Any]]: """Retrieve IAM permissions cache from S3 (same pattern as other assessments).""" try: @@ -103,7 +111,9 @@ def _bucket_name_from_arn(bucket_arn: str) -> str: return bucket_arn.split(":::", 1)[1] -def _paginate(client, operation_name: str, result_key: str, **kwargs) -> List[Dict[str, Any]]: +def _paginate( + client, operation_name: str, result_key: str, **kwargs +) -> List[Dict[str, Any]]: """Collect all items across pages for a paginated list/describe operation by calling the operation directly and following its continuation token. @@ -183,6 +193,26 @@ def _is_access_error(err: "ClientError") -> bool: return False +# Error codes meaning the S3 bucket a data source points to no longer exists. +# This is a distinct, actionable condition (a dangling KB data-source reference +# to a deleted bucket) — NOT "versioning/notifications absent." Surfacing it +# separately avoids a misleading "bucket without versioning" label when the real +# problem is the bucket was deleted out from under the Knowledge Base. +_MISSING_BUCKET_ERROR_CODES = frozenset({"NoSuchBucket", "404", "NotFound"}) + + +def _is_missing_bucket_error(err: "ClientError") -> bool: + """True if a ClientError indicates the S3 bucket does not exist (deleted / + dangling data-source reference) rather than a missing control or a missing + permission.""" + try: + return ( + err.response.get("Error", {}).get("Code", "") in _MISSING_BUCKET_ERROR_CODES + ) + except AttributeError: + return False + + # Findings whose name starts with this prefix were emitted because the check # could not run (e.g., missing IAM permission). They are visible in the report # (Status="N/A") so a failed/permission-denied check does not silently vanish. @@ -287,9 +317,7 @@ def _is_access_error(err: "ClientError") -> bool: } -def _could_not_assess_row( - check_id: str, check_name: str, err: Any -) -> Dict[str, Any]: +def _could_not_assess_row(check_id: str, check_name: str, err: Any) -> Dict[str, Any]: """ Synthesize one visible finding row for a check that errored out and produced no rows. Uses the existing schema (Status="N/A", Severity="Medium") so the @@ -325,6 +353,7 @@ def _could_not_assess_row( # COMPLIANCE_PLACEHOLDER: [FFIEC CAT, DORA Art.6, SR 11-7 Appendix A] # =========================================================================== + def check_waf_shield_on_bedrock_endpoints() -> Dict[str, Any]: """ FS-01 — Verify AWS WAF is associated with API Gateway or ALB endpoints @@ -362,9 +391,14 @@ def check_waf_shield_on_bedrock_endpoints() -> Dict[str, Any]: ), resolution=( "1. Subscribe to AWS Shield Advanced for DDoS protection.\n" - "2. Associate Shield Advanced with Bedrock-facing API Gateway stages, " - "ALBs, and CloudFront distributions.\n" - "3. Enable Shield Response Team (SRT) access." + "2. After subscribing, explicitly add resource protections in the " + "Shield Advanced console for each Bedrock-facing resource " + "(API Gateway stages, ALBs, CloudFront distributions, Route 53 hosted zones). " + "Shield Advanced subscription alone does NOT automatically protect resources — " + "each resource must be individually added to receive protection.\n" + "3. Enable Shield Response Team (SRT) access and configure proactive engagement.\n" + "4. Alternatively, use AWS Firewall Manager with a Shield Advanced policy " + "to automate resource protection based on tags or resource types." ), reference="https://docs.aws.amazon.com/waf/latest/developerguide/shield-chapter.html", severity="High", @@ -500,9 +534,16 @@ def check_bedrock_token_quotas() -> Dict[str, Any]: FS-03 — Check whether Bedrock service quotas for tokens-per-minute (TPM) have been reviewed and raised above AWS defaults. - Token-based quotas are the only signal: Amazon Bedrock no longer enforces - requests-per-minute (RPM) quotas on the bedrock-runtime endpoint (throttling - is token-based only), so an absent RPM quota must never drive a verdict. + Token-based quotas are the primary signal. RPM (requests-per-minute) quotas + are model-specific on the bedrock-runtime endpoint: some models (e.g., Claude + Opus 4.7/4.8) are governed solely by TPM with no RPM quota; others have both. + Because RPM applicability varies by model, only TPM quotas drive this verdict + and an absent RPM quota must never trigger a failure. + + Note: The bedrock-mantle endpoint (OpenAI-compatible, GA May 2026) exposes + separate input-tokens-per-minute and output-tokens-per-minute quotas also + under ServiceCode "bedrock". This check focuses on bedrock-runtime on-demand + TPM quotas; bedrock-mantle quotas are not explicitly separated here. Verdict logic (value-based, not adjustability-based): - at least one applied token-quota Value > its AWS default → customized → PASS/Passed @@ -515,15 +556,14 @@ def check_bedrock_token_quotas() -> Dict[str, Any]: try: sq = boto3.client("service-quotas", config=boto3_config) - # Applied quotas (paginated). Token-based quotas are the only signal. + # Applied quotas (paginated). TPM quotas are the primary signal; RPM quotas + # are model-specific and their absence must not trigger a failure verdict. applied = [] for page in sq.get_paginator("list_service_quotas").paginate( ServiceCode="bedrock" ): applied.extend(page.get("Quotas", [])) - token_quotas = [ - q for q in applied if "token" in q.get("QuotaName", "").lower() - ] + token_quotas = [q for q in applied if "token" in q.get("QuotaName", "").lower()] # Empty-applied-list branch: no token quotas found at all. if not token_quotas: @@ -676,7 +716,8 @@ def check_cost_anomaly_detection() -> Dict[str, Any]: # a DIMENSIONAL monitor on LINKED_ACCOUNT/TAG/COST_CATEGORY does NOT # provide service-level Bedrock coverage and must not count. bedrock_monitors = [ - m for m in monitors + m + for m in monitors if "bedrock" in json.dumps(m.get("MonitorSpecification", {})).lower() or ( m.get("MonitorType") == "DIMENSIONAL" @@ -765,13 +806,15 @@ def check_cloudwatch_token_alarms() -> Dict[str, Any]: all_alarms.extend(page.get("MetricAlarms", [])) bedrock_alarms = [ - a for a in all_alarms + a + for a in all_alarms if a.get("Namespace", "").startswith("AWS/Bedrock") or "bedrock" in a.get("AlarmName", "").lower() ] throttle_alarms = [ - a for a in bedrock_alarms + a + for a in bedrock_alarms if "throttl" in a.get("MetricName", "").lower() or "throttl" in a.get("AlarmName", "").lower() ] @@ -858,7 +901,8 @@ def _paginate_budgets(show_filter_expression: bool): all_budgets = _paginate_budgets(show_filter_expression=False) aiml_budgets = [ - b for b in all_budgets + b + for b in all_budgets if any( svc in json.dumps(b.get("CostFilters", {})).lower() or svc in json.dumps(b.get("FilterExpression", {})).lower() @@ -911,6 +955,7 @@ def _paginate_budgets(show_filter_expression: bool): # COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, DORA Art.6, MAS TRM 9] # =========================================================================== + def check_bedrock_agent_action_boundaries(permission_cache) -> Dict[str, Any]: """ FS-07 — Verify Bedrock agent execution roles have narrow action boundaries @@ -952,8 +997,12 @@ def check_bedrock_agent_action_boundaries(permission_cache) -> Dict[str, Any]: if not role_arn: continue role_name = role_arn.split("/")[-1] - role_perms = (permission_cache or {}).get("role_permissions", {}).get(role_name, {}) - for policy in role_perms.get("attached_policies", []) + role_perms.get("inline_policies", []): + role_perms = ( + (permission_cache or {}).get("role_permissions", {}).get(role_name, {}) + ) + for policy in role_perms.get("attached_policies", []) + role_perms.get( + "inline_policies", [] + ): doc = policy.get("document", {}) if isinstance(doc, str): doc = json.loads(doc) @@ -1111,8 +1160,11 @@ def check_agent_transaction_limits() -> Dict[str, Any]: # Look for agent-related Lambda functions without reserved concurrency agent_lambdas = [ - f for f in functions - if any(kw in f["FunctionName"].lower() for kw in ["agent", "bedrock", "aiml"]) + f + for f in functions + if any( + kw in f["FunctionName"].lower() for kw in ["agent", "bedrock", "aiml"] + ) ] lambdas_without_concurrency = [] @@ -1179,8 +1231,12 @@ def check_human_in_the_loop_for_high_risk_actions() -> Dict[str, Any]: machines = sfn.list_state_machines().get("stateMachines", []) agent_machines = [ - m for m in machines - if any(kw in m["name"].lower() for kw in ["agent", "approval", "human", "review"]) + m + for m in machines + if any( + kw in m["name"].lower() + for kw in ["agent", "approval", "human", "review"] + ) ] machines_with_wait = [] @@ -1264,7 +1320,8 @@ def check_agent_rate_alarms() -> Dict[str, Any]: all_alarms.extend(page.get("MetricAlarms", [])) agent_alarms = [ - a for a in all_alarms + a + for a in all_alarms if "agent" in a.get("AlarmName", "").lower() or "agent" in a.get("Namespace", "").lower() ] @@ -1315,6 +1372,7 @@ def check_agent_rate_alarms() -> Dict[str, Any]: # COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, DORA Art.6, ISO 27001 A.15] # =========================================================================== + def check_scp_model_access_restrictions() -> Dict[str, Any]: """ FS-12 — Verify SCPs restrict Bedrock model access to an approved model list, @@ -1329,7 +1387,9 @@ def check_scp_model_access_restrictions() -> Dict[str, Any]: orgs, "list_policies", "Policies", Filter="SERVICE_CONTROL_POLICY" ) except ClientError as e: - if "AccessDenied" in str(e) or "AWSOrganizationsNotInUseException" in str(e): + if "AccessDenied" in str(e) or "AWSOrganizationsNotInUseException" in str( + e + ): findings["csv_data"].append( create_finding( check_id="FS-12", @@ -1408,7 +1468,9 @@ def check_model_inventory_tagging() -> Dict[str, Any]: # Check Bedrock custom models for model in _paginate(bedrock, "list_custom_models", "modelSummaries"): - tags_response = bedrock.list_tags_for_resource(resourceARN=model["modelArn"]) + tags_response = bedrock.list_tags_for_resource( + resourceARN=model["modelArn"] + ) tag_keys = {t["key"].lower() for t in tags_response.get("tags", [])} missing = REQUIRED_TAGS - tag_keys if missing: @@ -1477,7 +1539,8 @@ def check_model_onboarding_governance() -> Dict[str, Any]: rules = _paginate(config, "describe_config_rules", "ConfigRules") bedrock_rules = [ - r for r in rules + r + for r in rules if "bedrock" in r.get("ConfigRuleName", "").lower() or "model" in r.get("ConfigRuleName", "").lower() ] @@ -1735,8 +1798,12 @@ def check_training_data_s3_versioning() -> Dict[str, Any]: buckets = s3.list_buckets().get("Buckets", []) training_buckets = [ - b for b in buckets - if any(kw in b["Name"].lower() for kw in ["train", "dataset", "model", "sagemaker", "bedrock"]) + b + for b in buckets + if any( + kw in b["Name"].lower() + for kw in ["train", "dataset", "model", "sagemaker", "bedrock"] + ) ] if not training_buckets: @@ -1816,6 +1883,7 @@ def check_training_data_s3_versioning() -> Dict[str, Any]: # COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, NYDFS 500.06, PCI-DSS 12.3.2] # =========================================================================== + def check_knowledge_base_iam_least_privilege(permission_cache) -> Dict[str, Any]: """ FS-22 — Verify IAM roles accessing Bedrock Knowledge Bases follow @@ -1825,8 +1893,12 @@ def check_knowledge_base_iam_least_privilege(permission_cache) -> Dict[str, Any] findings = _empty_findings("Knowledge Base IAM Least Privilege Check") try: issues = [] - for role_name, perms in (permission_cache or {}).get("role_permissions", {}).items(): - for policy in perms.get("attached_policies", []) + perms.get("inline_policies", []): + for role_name, perms in ( + (permission_cache or {}).get("role_permissions", {}).items() + ): + for policy in perms.get("attached_policies", []) + perms.get( + "inline_policies", [] + ): doc = policy.get("document", {}) if isinstance(doc, str): doc = json.loads(doc) @@ -2107,13 +2179,26 @@ def check_knowledge_base_vpc_access() -> Dict[str, Any]: # COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, NYDFS 500, MAS TRM 9.2] # =========================================================================== -def check_automated_reasoning_checks() -> Dict[str, Any]: + +def check_guardrail_contextual_grounding() -> Dict[str, Any]: """ - FS-27 — Check whether Bedrock Guardrails have Automated Reasoning checks - configured to validate factual accuracy of outputs. + FS-27 — Check whether Bedrock Guardrails have contextual grounding checks + configured to validate that outputs are grounded in the provided context and + are relevant to the user query. + + NOTE: This check verifies *contextual grounding* (guardrails-grounding) — a + separate, independent feature from Automated Reasoning checks (ARC). True ARC + is assessed in check_automated_reasoning_policies() below. Both controls are + recommended for FinServ workloads; they complement each other: + - Contextual grounding: thresholded relevance/grounding filter applied per + inference call (no policy authoring required). + - Automated Reasoning: policy-based formal verification of factual claims + against a customer-authored business-rules document (GA August 2025; + limited to US/EU regions; requires bedrock:ListAutomatedReasoningPolicies). + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2] """ - findings = _empty_findings("Automated Reasoning Checks") + findings = _empty_findings("Guardrail Contextual Grounding Check") try: bedrock = boto3.client("bedrock", config=boto3_config) guardrails = _paginate(bedrock, "list_guardrails", "guardrails") @@ -2122,9 +2207,12 @@ def check_automated_reasoning_checks() -> Dict[str, Any]: findings["csv_data"].append( create_finding( check_id="FS-27", - finding_name="No Guardrails — Automated Reasoning Not Applicable", + finding_name="No Guardrails — Contextual Grounding Not Applicable", finding_details="No Bedrock Guardrails configured. Configure guardrails first (see BR-05).", - resolution="Configure Bedrock Guardrails with contextual grounding checks.", + resolution=( + "Configure Bedrock Guardrails with contextual grounding checks " + "(grounding threshold ≥0.7 and relevance threshold ≥0.7 for FinServ use cases)." + ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", severity="Medium", status="N/A", @@ -2148,13 +2236,17 @@ def check_automated_reasoning_checks() -> Dict[str, Any]: check_id="FS-27", finding_name="No Guardrails With Contextual Grounding", finding_details=( - f"Found {len(guardrails)} guardrail(s) but none have contextual grounding enabled. " - "Non-compliant outputs (hallucinations, regulatory violations) will not be filtered." + f"Found {len(guardrails)} guardrail(s) but none have contextual grounding " + "filters enabled. Without grounding checks, outputs that are not supported " + "by the source context (hallucinations, regulatory violations) will not be " + "filtered at inference time." ), resolution=( - "Enable contextual grounding checks on Bedrock Guardrails with:\n" - "- Grounding threshold (0.7+ recommended for financial advice)\n" - "- Relevance threshold to filter off-topic responses" + "Enable contextual grounding checks on Bedrock Guardrails:\n" + "- Set grounding threshold ≥0.7 (filters responses not supported by source context)\n" + "- Set relevance threshold ≥0.7 (filters off-topic responses)\n" + "Also consider enabling Automated Reasoning checks (bedrock:ListAutomatedReasoningPolicies) " + "for policy-based formal verification of factual claims — see FS-27 ARC check." ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", severity="High", @@ -2167,8 +2259,11 @@ def check_automated_reasoning_checks() -> Dict[str, Any]: create_finding( check_id="FS-27", finding_name="Contextual Grounding Enabled on Guardrails", - finding_details=f"Guardrails with grounding: {', '.join(guardrails_with_grounding)}.", - resolution="No action required.", + finding_details=f"Guardrails with contextual grounding: {', '.join(guardrails_with_grounding)}.", + resolution=( + "No action required for contextual grounding. " + "Also consider enabling Automated Reasoning checks for formal policy verification." + ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", severity="Informational", status="Passed", @@ -2176,7 +2271,119 @@ def check_automated_reasoning_checks() -> Dict[str, Any]: ) ) except Exception as e: - return _error_findings("Automated Reasoning Checks", e) + return _error_findings("Guardrail Contextual Grounding Check", e) + return findings + + +def check_automated_reasoning_policies() -> Dict[str, Any]: + """ + FS-27b — Check whether Bedrock Automated Reasoning policies have been + created to provide formal, policy-based verification of GenAI factual claims. + + Automated Reasoning checks (ARC) — GA August 2025 — use formal verification + to detect hallucinations and ensure outputs comply with authored business rules + (e.g., loan eligibility criteria, regulatory thresholds). Unlike contextual + grounding (a threshold applied per call), ARC requires authoring an Automated + Reasoning Policy document containing the rules to verify against. + + Regions supported (as of June 2026): us-east-1, us-east-2, us-west-2, + eu-central-1, eu-west-1, eu-west-3. + + IAM action required: bedrock:ListAutomatedReasoningPolicies + + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2] + """ + findings = _empty_findings("Automated Reasoning Policies Check") + try: + bedrock = boto3.client("bedrock", config=boto3_config) + + try: + policies = _paginate( + bedrock, + "list_automated_reasoning_policies", + "automatedReasoningPolicySummaries", + ) + except ClientError as e: + if _is_access_error(e): + findings["csv_data"].append( + create_finding( + check_id="FS-27", + finding_name="Automated Reasoning Policies — Access Check", + finding_details=( + "Unable to list Automated Reasoning policies (access denied or service " + "unavailable in this region). ARC is available in: us-east-1, us-east-2, " + "us-west-2, eu-central-1, eu-west-1, eu-west-3." + ), + resolution=( + "1. Ensure the assessment role grants bedrock:ListAutomatedReasoningPolicies.\n" + "2. Confirm the assessed region supports Automated Reasoning checks.\n" + "3. Add bedrock:ListAutomatedReasoningPolicies to the member role policy " + "in deployment/1-aiml-security-member-roles.yaml." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/automated-reasoning.html", + severity="Low", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-27"], + ) + ) + return findings + raise + + if not policies: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-27", + finding_name="No Automated Reasoning Policies Found", + finding_details=( + "No Bedrock Automated Reasoning policies have been created. " + "ARC (GA August 2025) uses formal verification to guarantee that GenAI " + "outputs comply with authored business rules — e.g., loan criteria, " + "regulatory thresholds, policy constraints. Without ARC policies, " + "factual accuracy of outputs is not formally verified, only heuristically " + "filtered by contextual grounding thresholds." + ), + resolution=( + "1. In the Amazon Bedrock console → Guardrails → Automated Reasoning, " + "create a policy document encoding your FinServ business rules " + "(e.g., eligibility criteria, rate limits, regulatory thresholds).\n" + "2. Associate the ARC policy with your guardrail " + "(automatedReasoningPolicy.policies field in CreateGuardrail/UpdateGuardrail).\n" + "3. Set confidenceThreshold on the policy to control strictness.\n" + "4. ARC requires cross-Region inference — ensure your guardrail has a " + "guardrailProfileArn configured (crossRegionDetails in GetGuardrail response).\n" + "5. Reference: AWS Announcement — Automated Reasoning checks GA (August 2025)." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/automated-reasoning.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-27"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-27", + finding_name="Automated Reasoning Policies Found", + finding_details=( + f"Found {len(policies)} Automated Reasoning policy(ies): " + f"{', '.join(p['name'] for p in policies[:5])}. " + "Verify policies are associated with active guardrails via the " + "automatedReasoningPolicy field in GetGuardrail." + ), + resolution=( + "Confirm each ARC policy is referenced in a guardrail's " + "automatedReasoningPolicy.policies list and that the " + "guardrail is applied to your Bedrock inference calls." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/automated-reasoning.html", + severity="Informational", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-27"], + ) + ) + except Exception as e: + return _error_findings("Automated Reasoning Policies Check", e) return findings @@ -2207,12 +2414,20 @@ def check_guardrail_denied_topics_financial() -> Dict[str, Any]: return findings guardrails_with_topics = [] + topics_classic_tier = [] for g in guardrails: detail = bedrock.get_guardrail( guardrailIdentifier=g["id"], guardrailVersion="DRAFT" ) - if detail.get("topicPolicy", {}).get("topics"): + topic_policy = detail.get("topicPolicy", {}) + if topic_policy.get("topics"): guardrails_with_topics.append(g["name"]) + # Denied topics also support tiers (GA June 2025). STANDARD tier + # adds broader language support and improved detection but requires + # cross-region inference. topicPolicy.tier.tierName in GetGuardrail. + tier = topic_policy.get("tier", {}).get("tierName", "CLASSIC") + if tier == "CLASSIC": + topics_classic_tier.append(g["name"]) if not guardrails_with_topics: findings["status"] = "WARN" @@ -2232,7 +2447,9 @@ def check_guardrail_denied_topics_financial() -> Dict[str, Any]: "- Tax advice beyond general information\n" "When authoring denied-topic policies, use existing compliance materials " "as the source: employee policies, training materials, procedure documents, " - "and incident reports (as recommended in PDF \u00a71.2.1 Practical guidance)." + "and incident reports (as recommended in PDF \u00a71.2.1 Practical guidance). " + "Consider the STANDARD tier (GA June 2025) for broader language support; " + "it requires cross-region inference on the guardrail." ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", severity="High", @@ -2240,6 +2457,33 @@ def check_guardrail_denied_topics_financial() -> Dict[str, Any]: compliance_frameworks=COMPLIANCE_MAP["FS-28"], ) ) + elif topics_classic_tier: + findings["csv_data"].append( + create_finding( + check_id="FS-28", + finding_name="Denied Topics Configured on CLASSIC Tier", + finding_details=( + f"Guardrails with topic policies: {', '.join(guardrails_with_topics)}. " + f"The following use the CLASSIC tier: {', '.join(topics_classic_tier)}. " + "CLASSIC tier supports English, French, and Spanish only. The STANDARD tier " + "(GA June 2025) provides broader language support and improved detection for " + "denied topics." + ), + resolution=( + "Verify topics cover regulated financial advice categories. For multilingual " + "FinServ deployments, consider upgrading denied topics to the STANDARD tier " + "(set topicsTierConfig.tierName=STANDARD via UpdateGuardrail; requires a " + "cross-region inference profile on the guardrail). When authoring denied-topic " + "policies, use existing compliance materials as the source: employee policies, " + "training materials, procedure documents, and incident reports " + "(PDF \u00a71.2.1 Practical guidance)." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", + severity="Low", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-28"], + ) + ) else: findings["csv_data"].append( create_finding( @@ -2347,12 +2591,22 @@ def check_bedrock_evaluation_compliance_datasets() -> Dict[str, Any]: # COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2, NYDFS 500] # =========================================================================== + def check_knowledge_base_data_source_sync() -> Dict[str, Any]: """ FS-31 — Verify Bedrock Knowledge Base data sources have recent sync jobs to ensure information currency. + + Staleness threshold: AWS does not prescribe a maximum data age for Knowledge + Bases — the appropriate cadence is workload-specific (intraday for market + data, weekly/monthly for slow-changing regulatory guidance). This check uses + a default of 7 days purely as a review prompt; treat a finding as "confirm + your data-currency requirement is met," not as an AWS-mandated failure. COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT] """ + # Default review threshold (days). Not an AWS requirement — a configurable + # heuristic. Firms with stricter or looser currency needs should adjust this. + STALE_AFTER_DAYS = 7 findings = _empty_findings("Knowledge Base Data Source Sync Check") try: bedrock_agent = boto3.client("bedrock-agent", config=boto3_config) @@ -2381,14 +2635,16 @@ def check_knowledge_base_data_source_sync() -> Dict[str, Any]: for kb in kbs: kb_id = kb["knowledgeBaseId"] sources = _paginate( - bedrock_agent, "list_data_sources", "dataSourceSummaries", + bedrock_agent, + "list_data_sources", + "dataSourceSummaries", knowledgeBaseId=kb_id, ) for source in sources: last_updated = source.get("updatedAt") if last_updated: age_days = (now - last_updated).days - if age_days > 7: + if age_days > STALE_AFTER_DAYS: stale_kbs.append( f"KB '{kb['name']}' source '{source['name']}' last synced {age_days} days ago" ) @@ -2398,15 +2654,21 @@ def check_knowledge_base_data_source_sync() -> Dict[str, Any]: findings["csv_data"].append( create_finding( check_id="FS-31", - finding_name="Stale Knowledge Base Data Sources", + finding_name="Knowledge Base Data Sources Past Review Threshold", finding_details=( - f"{len(stale_kbs)} data source(s) not synced in >7 days:\n" + f"{len(stale_kbs)} data source(s) not synced in >{STALE_AFTER_DAYS} days " + f"(a configurable review threshold, NOT an AWS-mandated limit):\n" + "\n".join(f"- {s}" for s in stale_kbs[:10]) + + "\nConfirm this age is acceptable for each data source's currency " + "requirement — slow-changing reference data may legitimately sync infrequently." ), resolution=( - "1. Configure automated sync schedules for KB data sources.\n" - "2. Set CloudWatch alarms on sync job failures.\n" - "3. Define maximum acceptable data age per use case (e.g., 24h for market data)." + "1. Define the maximum acceptable data age per use case (e.g., intraday for " + "market data, daily for product terms, weekly/monthly for regulatory guidance) " + "and adjust the review threshold to match.\n" + "2. Configure automated sync (EventBridge Scheduler → StartIngestionJob) at " + "that cadence — see FS-61.\n" + "3. Set CloudWatch alarms on sync job failures." ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base-ingest.html", severity="Medium", @@ -2419,7 +2681,10 @@ def check_knowledge_base_data_source_sync() -> Dict[str, Any]: create_finding( check_id="FS-31", finding_name="Knowledge Base Data Sources Recently Synced", - finding_details="All reviewed KB data sources synced within 7 days.", + finding_details=( + f"All reviewed KB data sources synced within {STALE_AFTER_DAYS} days " + "(the default review threshold)." + ), resolution="No action required.", reference="https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base-ingest.html", severity="Informational", @@ -2494,9 +2759,12 @@ def check_knowledge_base_integrity_monitoring() -> Dict[str, Any]: return findings buckets_without_versioning = [] + missing_buckets = [] for kb in kbs: sources = _paginate( - bedrock_agent, "list_data_sources", "dataSourceSummaries", + bedrock_agent, + "list_data_sources", + "dataSourceSummaries", knowledgeBaseId=kb["knowledgeBaseId"], ) for source in sources: @@ -2521,11 +2789,56 @@ def check_knowledge_base_integrity_monitoring() -> Dict[str, Any]: # surfaces as could-not-assess instead of a false finding. if _is_access_error(e): raise + # The data source points to a bucket that no longer exists + # (deleted out from under the KB). This is a distinct, + # actionable integrity problem — report it separately, not + # as "missing versioning." + if _is_missing_bucket_error(e): + logger.warning( + f"KB '{kb['name']}' data source '{source['name']}' " + f"references a deleted bucket: {bucket}" + ) + missing_buckets.append( + f"{bucket} (KB '{kb['name']}', source '{source['name']}')" + ) + continue logger.warning( f"Could not check versioning for bucket {bucket}: {e}" ) buckets_without_versioning.append(f"{bucket} (error)") + # A dangling data-source reference to a deleted bucket is a real integrity + # finding in its own right — emit it as a separate row so it is not + # conflated with "versioning not enabled." + if missing_buckets: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-33", + finding_name="KB Data Source References a Deleted S3 Bucket", + finding_details=( + "One or more Knowledge Base data sources point to S3 buckets that no " + "longer exist (NoSuchBucket). Retrieval will silently return no results " + "for these sources, and the integrity of the KB's grounding data cannot " + "be verified:\n" + + "\n".join(f"- {b}" for b in missing_buckets[:10]) + ), + resolution=( + "1. Investigate why the data-source bucket was deleted (accidental " + "deletion, environment teardown, or a stale KB configuration).\n" + "2. Recreate/restore the bucket with versioning enabled, or remove the " + "orphaned data source from the Knowledge Base.\n" + "3. Re-run a KB ingestion job after restoring the data source.\n" + "4. Enable S3 versioning and MFA Delete on KB data-source buckets to " + "reduce the risk of unrecoverable deletion." + ), + reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html", + severity="High", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-33"], + ) + ) + if buckets_without_versioning: findings["status"] = "WARN" findings["csv_data"].append( @@ -2546,7 +2859,7 @@ def check_knowledge_base_integrity_monitoring() -> Dict[str, Any]: compliance_frameworks=COMPLIANCE_MAP["FS-33"], ) ) - else: + elif not missing_buckets: findings["csv_data"].append( create_finding( check_id="FS-33", @@ -2573,12 +2886,16 @@ def check_fm_version_currency() -> Dict[str, Any]: findings = _empty_findings("Foundation Model Version Currency Check") try: bedrock = boto3.client("bedrock", config=boto3_config) - models = bedrock.list_foundation_models( - byOutputModality="TEXT" - ).get("modelSummaries", []) + # Do not filter by output modality: legacy/deprecated models exist across + # TEXT, EMBEDDING, and IMAGE modalities. Embedding models are widely used + # in FinServ RAG pipelines and a legacy embedding model produces stale + # embeddings — missing it would be a false-pass. Fetch all modalities and + # let the LEGACY lifecycle filter identify any deprecated model in use. + models = bedrock.list_foundation_models().get("modelSummaries", []) deprecated = [ - m["modelId"] for m in models + m["modelId"] + for m in models if m.get("modelLifecycle", {}).get("status") == "LEGACY" ] @@ -2632,6 +2949,7 @@ def check_fm_version_currency() -> Dict[str, Any]: # COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2, NYDFS 500] # =========================================================================== + def check_fmeval_harmful_content() -> Dict[str, Any]: """ FS-35 — Check for FMEval or Bedrock Evaluation jobs testing for harmful @@ -2706,12 +3024,22 @@ def check_guardrail_content_filters() -> Dict[str, Any]: return findings guardrails_with_filters = [] + guardrails_classic_tier = [] for g in guardrails: detail = bedrock.get_guardrail( guardrailIdentifier=g["id"], guardrailVersion="DRAFT" ) - if detail.get("contentPolicy", {}).get("filters"): + content_policy = detail.get("contentPolicy", {}) + if content_policy.get("filters"): guardrails_with_filters.append(g["name"]) + # Check tier: STANDARD offers better accuracy, multilingual support, + # and improved prompt-attack detection (GA June 2025). FinServ + # workloads benefit from STANDARD for its contextual understanding and + # typo-tolerant detection. STANDARD requires cross-region inference. + # The tier is nested at contentPolicy.tier.tierName in GetGuardrail response. + tier = content_policy.get("tier", {}).get("tierName", "CLASSIC") + if tier == "CLASSIC": + guardrails_classic_tier.append(g["name"]) if not guardrails_with_filters: findings["status"] = "WARN" @@ -2724,8 +3052,11 @@ def check_guardrail_content_filters() -> Dict[str, Any]: "Harmful content (hate, violence, sexual) may pass through unfiltered." ), resolution=( - "Add content filters to guardrails for: HATE, INSULTS, SEXUAL, VIOLENCE. " - "Set filter strength to HIGH for financial services use cases." + "1. Add content filters to guardrails for: HATE, INSULTS, SEXUAL, VIOLENCE.\n" + "2. Set filter strength to HIGH for financial services use cases.\n" + "3. Consider the STANDARD tier (GA June 2025) for improved accuracy, " + "typographical error detection, and 60+ language support. STANDARD tier " + "requires cross-region inference to be enabled on the guardrail." ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-filters.html", severity="High", @@ -2733,12 +3064,43 @@ def check_guardrail_content_filters() -> Dict[str, Any]: compliance_frameworks=COMPLIANCE_MAP["FS-36"], ) ) + elif guardrails_classic_tier: + # Content filters exist but using CLASSIC tier — emit advisory finding + findings["csv_data"].append( + create_finding( + check_id="FS-36", + finding_name="Guardrail Content Filters on CLASSIC Tier", + finding_details=( + f"Guardrails with content filters: {', '.join(guardrails_with_filters)}. " + f"The following use the CLASSIC tier: {', '.join(guardrails_classic_tier)}. " + "CLASSIC tier supports English, French, and Spanish only. The STANDARD tier " + "(GA June 2025) provides improved contextual understanding, typographical error " + "detection, 60+ language support, and better prompt-attack classification " + "(distinguishes jailbreaks from prompt injection)." + ), + resolution=( + "Consider upgrading to STANDARD tier content filters for FinServ workloads " + "that handle multiple languages or require higher detection accuracy. " + "STANDARD tier requires cross-region inference " + "(crossRegionDetails.guardrailProfileArn on the guardrail). " + "To upgrade: update the guardrail's contentPolicy.filtersConfig.contentFiltersTierConfig " + "with tierName=STANDARD and configure a guardrail cross-region profile." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-filters.html", + severity="Low", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-36"], + ) + ) else: findings["csv_data"].append( create_finding( check_id="FS-36", - finding_name="Guardrail Content Filters Configured", - finding_details=f"Guardrails with content filters: {', '.join(guardrails_with_filters)}.", + finding_name="Guardrail Content Filters Configured (STANDARD Tier)", + finding_details=( + f"All {len(guardrails_with_filters)} guardrail(s) with content filters " + "use the STANDARD tier, providing improved accuracy and multilingual support." + ), resolution="No action required.", reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-filters.html", severity="Informational", @@ -2812,7 +3174,9 @@ def check_guardrail_word_filters() -> Dict[str, Any]: detail = bedrock.get_guardrail( guardrailIdentifier=g["id"], guardrailVersion="DRAFT" ) - if detail.get("wordPolicy", {}).get("words") or detail.get("wordPolicy", {}).get("managedWordLists"): + if detail.get("wordPolicy", {}).get("words") or detail.get( + "wordPolicy", {} + ).get("managedWordLists"): guardrails_with_words.append(g["name"]) if not guardrails_with_words: @@ -2864,11 +3228,12 @@ def check_sagemaker_clarify_bias() -> Dict[str, Any]: findings = _empty_findings("SageMaker Clarify Bias Check") try: sm = boto3.client("sagemaker", config=boto3_config) - schedules = _paginate(sm, "list_monitoring_schedules", "MonitoringScheduleSummaries") + schedules = _paginate( + sm, "list_monitoring_schedules", "MonitoringScheduleSummaries" + ) bias_schedules = [ - s for s in schedules - if s.get("MonitoringType") == "ModelBias" + s for s in schedules if s.get("MonitoringType") == "ModelBias" ] if not bias_schedules: @@ -2969,11 +3334,12 @@ def check_sagemaker_clarify_explainability() -> Dict[str, Any]: findings = _empty_findings("SageMaker Clarify Explainability Check") try: sm = boto3.client("sagemaker", config=boto3_config) - schedules = _paginate(sm, "list_monitoring_schedules", "MonitoringScheduleSummaries") + schedules = _paginate( + sm, "list_monitoring_schedules", "MonitoringScheduleSummaries" + ) explainability_schedules = [ - s for s in schedules - if s.get("MonitoringType") == "ModelExplainability" + s for s in schedules if s.get("MonitoringType") == "ModelExplainability" ] if not explainability_schedules: @@ -3073,6 +3439,7 @@ def check_ai_service_cards_documentation() -> Dict[str, Any]: # COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, FFIEC CAT, PCI-DSS 3.4, GDPR Art.25] # =========================================================================== + def check_cloudwatch_log_pii_masking() -> Dict[str, Any]: """ FS-43 — Check for CloudWatch Logs data protection policies that mask PII @@ -3274,8 +3641,12 @@ def check_data_classification_tagging() -> Dict[str, Any]: buckets = s3.list_buckets().get("Buckets", []) aiml_buckets = [ - b for b in buckets - if any(kw in b["Name"].lower() for kw in ["train", "model", "bedrock", "sagemaker", "kb", "knowledge"]) + b + for b in buckets + if any( + kw in b["Name"].lower() + for kw in ["train", "model", "bedrock", "sagemaker", "kb", "knowledge"] + ) ] if not aiml_buckets: @@ -3298,7 +3669,10 @@ def check_data_classification_tagging() -> Dict[str, Any]: try: tags = s3.get_bucket_tagging(Bucket=bucket["Name"]).get("TagSet", []) tag_keys = {t["Key"].lower() for t in tags} - if "data-classification" not in tag_keys and "classification" not in tag_keys: + if ( + "data-classification" not in tag_keys + and "classification" not in tag_keys + ): unclassified.append(bucket["Name"]) except ClientError as e: # A genuine "no tags" response (NoSuchTagSet) means the bucket is @@ -3354,6 +3728,7 @@ def check_data_classification_tagging() -> Dict[str, Any]: # COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2, NYDFS 500] # =========================================================================== + def check_guardrail_grounding_threshold() -> Dict[str, Any]: """ FS-47 — Verify Bedrock Guardrails contextual grounding thresholds are @@ -3381,16 +3756,22 @@ def check_guardrail_grounding_threshold() -> Dict[str, Any]: return findings low_threshold_guardrails = [] + guardrails_with_grounding = [] for g in guardrails: detail = bedrock.get_guardrail( guardrailIdentifier=g["id"], guardrailVersion="DRAFT" ) grounding = detail.get("contextualGroundingPolicy", {}) + has_grounding_filter = False for filter_item in grounding.get("filters", []): - if filter_item.get("type") == "GROUNDING" and filter_item.get("threshold", 1.0) < 0.7: - low_threshold_guardrails.append( - f"{g['name']} (threshold={filter_item['threshold']})" - ) + if filter_item.get("type") == "GROUNDING": + has_grounding_filter = True + if filter_item.get("threshold", 1.0) < 0.7: + low_threshold_guardrails.append( + f"{g['name']} (threshold={filter_item['threshold']})" + ) + if has_grounding_filter: + guardrails_with_grounding.append(g["name"]) if low_threshold_guardrails: findings["status"] = "WARN" @@ -3403,10 +3784,40 @@ def check_guardrail_grounding_threshold() -> Dict[str, Any]: "Low thresholds allow hallucinated responses to pass through." ), resolution=( - "Set grounding threshold to 0.7 or higher for financial services use cases. " - "Test threshold impact on response quality before increasing." + "Set grounding threshold to 0.7 or higher for financial services use cases " + "(valid range is 0 to 0.99; 1.0 is invalid and blocks all content). " + "Test threshold impact on response quality before increasing. Note: contextual " + "grounding supports summarization, paraphrasing, and Q&A — not conversational " + "chatbot use cases; for chatbots use denied topics (FS-28/FS-59) instead." ), - reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-contextual-grounding-check.html", + severity="High", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-47"], + ) + ) + elif not guardrails_with_grounding: + # Guardrails exist but NONE has a GROUNDING filter at all. This is a + # genuine gap (not a pass): without a grounding filter, ungrounded / + # hallucinated responses are not detected. Previously this fell through + # to the "Passed" branch because low_threshold_guardrails was empty. + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-47", + finding_name="No Guardrails With a Grounding Filter", + finding_details=( + f"Found {len(guardrails)} guardrail(s) but none have a GROUNDING contextual " + "grounding filter configured. Ungrounded or hallucinated responses are not " + "detected for summarization/paraphrasing/Q&A use cases." + ), + resolution=( + "Add a GROUNDING contextual grounding filter (threshold ≥0.7; valid range " + "0 to 0.99) to each guardrail used for summarization, paraphrasing, or Q&A. " + "Note: contextual grounding is not supported for conversational chatbot use " + "cases — use denied topics (FS-28/FS-59) for those." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-contextual-grounding-check.html", severity="High", status="Failed", compliance_frameworks=COMPLIANCE_MAP["FS-47"], @@ -3417,9 +3828,12 @@ def check_guardrail_grounding_threshold() -> Dict[str, Any]: create_finding( check_id="FS-47", finding_name="Guardrail Grounding Thresholds Appropriate", - finding_details="All guardrails with grounding have thresholds ≥0.7.", + finding_details=( + f"All {len(guardrails_with_grounding)} guardrail(s) with a GROUNDING filter " + "have thresholds ≥0.7." + ), resolution="No action required.", - reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-contextual-grounding-check.html", severity="Informational", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-47"], @@ -3516,19 +3930,27 @@ def check_hallucination_disclaimer_advisory() -> Dict[str, Any]: return findings -def check_automated_reasoning_checks_hallucination() -> Dict[str, Any]: +def check_guardrail_relevance_grounding() -> Dict[str, Any]: """ - FS-50 — Check for Bedrock Automated Reasoning checks (ARC) configured - to validate factual claims in GenAI outputs. + FS-50 — Check for Bedrock Guardrails contextual grounding RELEVANCE filters + configured to detect and block responses that are not grounded in the context + retrieved by the RAG pipeline (hallucination prevention). + + NOTE: This check verifies the *RELEVANCE* filter within contextual grounding + (a different feature from GROUNDING-type filters). RELEVANCE filters block + responses that are off-topic relative to the user query. GROUNDING filters + block responses not supported by the source context. Both are important for + FinServ hallucination mitigation. For formal policy-based verification of + factual claims, see check_automated_reasoning_policies() (FS-27b). + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT] """ - findings = _empty_findings("Automated Reasoning Checks for Hallucination") + findings = _empty_findings("Guardrail Relevance Grounding Check") try: bedrock = boto3.client("bedrock", config=boto3_config) - # ARC is part of guardrails contextual grounding — check for RELEVANCE filter guardrails = _paginate(bedrock, "list_guardrails", "guardrails") - arc_guardrails = [] + guardrails_with_relevance = [] for g in guardrails: detail = bedrock.get_guardrail( guardrailIdentifier=g["id"], guardrailVersion="DRAFT" @@ -3536,21 +3958,25 @@ def check_automated_reasoning_checks_hallucination() -> Dict[str, Any]: grounding = detail.get("contextualGroundingPolicy", {}) for f in grounding.get("filters", []): if f.get("type") == "RELEVANCE": - arc_guardrails.append(g["name"]) + guardrails_with_relevance.append(g["name"]) - if not arc_guardrails: + if not guardrails_with_relevance: findings["status"] = "WARN" findings["csv_data"].append( create_finding( check_id="FS-50", - finding_name="No Guardrails With Relevance Grounding", + finding_name="No Guardrails With Relevance Grounding Filters", finding_details=( - "No guardrails have relevance grounding filters. " - "Off-topic or hallucinated responses will not be filtered." + "No guardrails have RELEVANCE contextual grounding filters. " + "Without relevance filters, responses that are off-topic or unrelated " + "to the user query will not be blocked, increasing hallucination risk " + "in RAG-based FinServ applications." ), resolution=( - "Enable relevance grounding filter in Bedrock Guardrails " - "with threshold ≥0.7 to filter responses not grounded in context." + "Enable the RELEVANCE contextual grounding filter in Bedrock Guardrails " + "with a threshold of ≥0.7 to block responses that are not relevant to " + "the user query. Also enable the GROUNDING filter (≥0.7) to block " + "responses not supported by the retrieved source context." ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", severity="Medium", @@ -3563,7 +3989,10 @@ def check_automated_reasoning_checks_hallucination() -> Dict[str, Any]: create_finding( check_id="FS-50", finding_name="Relevance Grounding Filters Present", - finding_details=f"Guardrails with relevance grounding: {', '.join(arc_guardrails)}.", + finding_details=( + f"Guardrails with RELEVANCE grounding filters: " + f"{', '.join(guardrails_with_relevance)}." + ), resolution="No action required.", reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", severity="Informational", @@ -3572,7 +4001,7 @@ def check_automated_reasoning_checks_hallucination() -> Dict[str, Any]: ) ) except Exception as e: - return _error_findings("Automated Reasoning Checks for Hallucination", e) + return _error_findings("Guardrail Relevance Grounding Check", e) return findings @@ -3603,6 +4032,7 @@ def check_prompt_injection_input_validation() -> Dict[str, Any]: return findings guardrails_with_prompt_attack = [] + guardrails_classic_tier_pa = [] for g in guardrails: detail = bedrock.get_guardrail( guardrailIdentifier=g["id"], guardrailVersion="DRAFT" @@ -3611,6 +4041,12 @@ def check_prompt_injection_input_validation() -> Dict[str, Any]: for f in content_policy.get("filters", []): if f.get("type") == "PROMPT_ATTACK": guardrails_with_prompt_attack.append(g["name"]) + # STANDARD tier (GA June 2025) improves PROMPT_ATTACK detection + # by distinguishing jailbreaks from prompt injection attacks. + tier = content_policy.get("tier", {}).get("tierName", "CLASSIC") + if tier == "CLASSIC": + guardrails_classic_tier_pa.append(g["name"]) + break if not guardrails_with_prompt_attack: findings["status"] = "WARN" @@ -3625,23 +4061,56 @@ def check_prompt_injection_input_validation() -> Dict[str, Any]: resolution=( "1. Enable PROMPT_ATTACK content filter in Bedrock Guardrails.\n" "2. Set input filter strength to HIGH.\n" - "3. Implement application-level input sanitization as defense-in-depth.\n" - "4. Use parameterized prompts (never concatenate user input directly)." + "3. Use input tags () to " + "differentiate user inputs from developer-provided prompts — required for " + "PROMPT_ATTACK filters to work correctly with InvokeModel/InvokeModelWithResponseStream.\n" + "4. Consider STANDARD tier (GA June 2025) for better jailbreak vs. injection " + "classification and broader language support.\n" + "5. Implement application-level input sanitization as defense-in-depth." ), - reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-prompt-attack.html", severity="High", status="Failed", compliance_frameworks=COMPLIANCE_MAP["FS-51"], ) ) + elif guardrails_classic_tier_pa: + findings["csv_data"].append( + create_finding( + check_id="FS-51", + finding_name="Prompt Attack Filters on CLASSIC Tier", + finding_details=( + f"Guardrails with PROMPT_ATTACK filters: {', '.join(guardrails_with_prompt_attack)}. " + f"Using CLASSIC tier: {', '.join(guardrails_classic_tier_pa)}. " + "STANDARD tier (GA June 2025) better distinguishes jailbreaks from " + "prompt injection and provides broader language support." + ), + resolution=( + "Consider upgrading to STANDARD tier for improved PROMPT_ATTACK detection. " + "Ensure input tags are used to scope user content for PROMPT_ATTACK evaluation. " + "STANDARD tier requires cross-region inference on the guardrail." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-prompt-attack.html", + severity="Low", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-51"], + ) + ) else: findings["csv_data"].append( create_finding( check_id="FS-51", - finding_name="Prompt Attack Filters Configured", - finding_details=f"Guardrails with prompt attack filters: {', '.join(guardrails_with_prompt_attack)}.", - resolution="No action required.", - reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", + finding_name="Prompt Attack Filters Configured (STANDARD Tier)", + finding_details=( + f"Guardrails with PROMPT_ATTACK filters (STANDARD tier): " + f"{', '.join(guardrails_with_prompt_attack)}." + ), + resolution=( + "Ensure input tags are used to scope user content when calling " + "InvokeModel/InvokeModelWithResponseStream — required for PROMPT_ATTACK " + "filters to evaluate user input separately from system prompts." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-prompt-attack.html", severity="Informational", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-51"], @@ -3664,8 +4133,12 @@ def check_bedrock_sdk_version_currency() -> Dict[str, Any]: functions = _paginate(lambda_client, "list_functions", "Functions") bedrock_functions = [ - f for f in functions - if any(kw in f["FunctionName"].lower() for kw in ["bedrock", "agent", "aiml", "genai"]) + f + for f in functions + if any( + kw in f["FunctionName"].lower() + for kw in ["bedrock", "agent", "aiml", "genai"] + ) ] if not bedrock_functions: @@ -3683,12 +4156,46 @@ def check_bedrock_sdk_version_currency() -> Dict[str, Any]: ) return findings - # Check for deprecated runtimes - deprecated_runtimes = {"python3.7", "python3.8", "nodejs14.x", "nodejs12.x"} + # Check for deprecated runtimes using the definitive allowlist of currently- + # supported Lambda managed runtimes (sourced from + # https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html, + # retrieved June 2026). Any runtime NOT in this set is considered deprecated + # or end-of-support. This allowlist approach is more reliable than maintaining + # a denylist (which silently misses newly-deprecated runtimes). + # + # Supported as of June 2026 (ordered newest-first per language): + SUPPORTED_RUNTIMES = { + # Python + "python3.14", + "python3.13", + "python3.12", + "python3.11", + "python3.10", + # Node.js + "nodejs24.x", + "nodejs22.x", + # Java + "java25", + "java21", + "java17", + "java11", + "java8.al2", + # .NET + "dotnet10", + "dotnet9", + "dotnet8", + # Ruby + "ruby4.0", + "ruby3.4", + "ruby3.3", + # OS-only / custom runtimes + "provided.al2023", + "provided.al2", + } outdated_functions = [ f["FunctionName"] for f in bedrock_functions - if f.get("Runtime", "") in deprecated_runtimes + if f.get("Runtime", "") and f["Runtime"] not in SUPPORTED_RUNTIMES ] if outdated_functions: @@ -3702,9 +4209,14 @@ def check_bedrock_sdk_version_currency() -> Dict[str, Any]: "Deprecated runtimes may use outdated boto3/SDK versions lacking security patches." ), resolution=( - "1. Upgrade Lambda functions to Python 3.12+ or Node.js 20.x.\n" - "2. Update boto3 to latest version in Lambda layers.\n" - "3. Enable Lambda runtime management for automatic updates." + "1. Upgrade Lambda functions to a supported runtime — Python 3.12+, " + "Node.js 22.x or 24.x, Java 21+, or .NET 8+.\n" + "2. Update boto3 to the latest version in Lambda layers (pin the version " + "in requirements.txt and redeploy).\n" + "3. Enable Lambda runtime management controls for automatic minor-version " + "updates (runtimeManagementConfig.updateRuntimeOn = 'Auto').\n" + "4. Refer to https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html " + "for the authoritative list of supported and deprecated runtimes." ), reference="https://docs.aws.amazon.com/lambda/latest/dg/runtimes-update.html", severity="Medium", @@ -3770,7 +4282,9 @@ def check_waf_sql_injection_rules() -> Dict[str, Any]: Id=acl_summary["Id"], ).get("WebACL", {}) rule_names = { - r.get("Statement", {}).get("ManagedRuleGroupStatement", {}).get("Name", "") + r.get("Statement", {}) + .get("ManagedRuleGroupStatement", {}) + .get("Name", "") for r in acl.get("Rules", []) } if not rule_names.intersection(INJECTION_RULE_GROUPS): @@ -3832,13 +4346,21 @@ def check_penetration_testing_evidence() -> Dict[str, Any]: "Manual review required to confirm GenAI applications have been tested." ), resolution=( - "1. Conduct annual penetration testing of GenAI applications.\n" - "2. Include prompt injection, jailbreak, and indirect injection test cases.\n" - "3. Use AWS Bedrock red-teaming capabilities.\n" - "4. Document findings and remediation for regulatory examination.\n" + "1. Conduct penetration testing of GenAI applications at least annually and " + "before major releases.\n" + "2. Include AI-specific test cases: prompt injection, jailbreak, indirect " + "(cross-domain) injection, system-prompt leakage, and data-extraction attempts.\n" + "3. Consider AWS Security Agent for on-demand, AI-driven penetration testing " + "(GA March 2026; available in US East N. Virginia, US West Oregon, Europe Ireland, " + "Europe Frankfurt, Asia Pacific Sydney, Asia Pacific Tokyo, with cross-account " + "shared-VPC testing via AWS RAM). Open-source tools such as Garak or PyRIT and " + "manual red-teaming are complementary options. Verify current regional availability " + "on the AWS Security Agent page before relying on it.\n" + "4. Document findings and remediation for regulatory examination, and tag tested " + "resources with a last-pentest-date for audit trail.\n" "5. For DORA compliance, include GenAI in TLPT (Threat-Led Penetration Testing) scope." ), - reference="https://docs.aws.amazon.com/bedrock/latest/userguide/security.html", + reference="https://aws.amazon.com/security/penetration-testing/", severity="Informational", status="N/A", compliance_frameworks=COMPLIANCE_MAP["FS-54"], @@ -3854,6 +4376,7 @@ def check_penetration_testing_evidence() -> Dict[str, Any]: # COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, NYDFS 500, OWASP LLM02] # =========================================================================== + def check_output_validation_lambda() -> Dict[str, Any]: """ FS-55 — Check for Lambda functions implementing output validation/sanitization @@ -3866,8 +4389,12 @@ def check_output_validation_lambda() -> Dict[str, Any]: functions = _paginate(lambda_client, "list_functions", "Functions") validation_functions = [ - f for f in functions - if any(kw in f["FunctionName"].lower() for kw in ["validate", "sanitize", "filter", "output"]) + f + for f in functions + if any( + kw in f["FunctionName"].lower() + for kw in ["validate", "sanitize", "filter", "output"] + ) ] if not validation_functions: @@ -3886,7 +4413,7 @@ def check_output_validation_lambda() -> Dict[str, Any]: "3. Sanitize outputs before rendering in web UIs (XSS prevention).\n" "4. Encode outputs appropriately for the target context (HTML, SQL, JSON)." ), - reference="https://owasp.org/www-project-top-10-for-large-language-model-applications/", + reference="https://genai.owasp.org/llm-top-10/", severity="Medium", status="Failed", compliance_frameworks=COMPLIANCE_MAP["FS-55"], @@ -3899,7 +4426,7 @@ def check_output_validation_lambda() -> Dict[str, Any]: finding_name="Output Validation Functions Present", finding_details=f"Found {len(validation_functions)} output validation/sanitization function(s).", resolution="No action required.", - reference="https://owasp.org/www-project-top-10-for-large-language-model-applications/", + reference="https://genai.owasp.org/llm-top-10/", severity="Informational", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-55"], @@ -3981,7 +4508,7 @@ def check_output_encoding_advisory() -> Dict[str, Any]: "3. JSON-encode outputs before embedding in JavaScript contexts.\n" "4. Validate output length and format before passing to downstream APIs." ), - reference="https://owasp.org/www-project-top-10-for-large-language-model-applications/", + reference="https://genai.owasp.org/llm-top-10/", severity="Informational", status="N/A", compliance_frameworks=COMPLIANCE_MAP["FS-57"], @@ -4003,8 +4530,12 @@ def check_output_schema_validation() -> Dict[str, Any]: functions = _paginate(lambda_client, "list_functions", "Functions") schema_functions = [ - f for f in functions - if any(kw in f["FunctionName"].lower() for kw in ["schema", "validate", "parse", "format"]) + f + for f in functions + if any( + kw in f["FunctionName"].lower() + for kw in ["schema", "validate", "parse", "format"] + ) ] findings["csv_data"].append( @@ -4059,12 +4590,17 @@ def check_guardrail_topic_allowlist() -> Dict[str, Any]: return findings guardrails_with_topics = [] + topics_classic_tier = [] for g in guardrails: detail = bedrock.get_guardrail( guardrailIdentifier=g["id"], guardrailVersion="DRAFT" ) - if detail.get("topicPolicy", {}).get("topics"): + topic_policy = detail.get("topicPolicy", {}) + if topic_policy.get("topics"): guardrails_with_topics.append(g["name"]) + tier = topic_policy.get("tier", {}).get("tierName", "CLASSIC") + if tier == "CLASSIC": + topics_classic_tier.append(g["name"]) if not guardrails_with_topics: findings["status"] = "WARN" @@ -4081,7 +4617,9 @@ def check_guardrail_topic_allowlist() -> Dict[str, Any]: "- Medical/health advice\n" "- Legal advice\n" "- Political opinions\n" - "- Non-financial product recommendations" + "- Non-financial product recommendations\n" + "Consider the STANDARD tier (GA June 2025) for broader language support; " + "it requires cross-region inference on the guardrail." ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", severity="Medium", @@ -4089,6 +4627,28 @@ def check_guardrail_topic_allowlist() -> Dict[str, Any]: compliance_frameworks=COMPLIANCE_MAP["FS-59"], ) ) + elif topics_classic_tier: + findings["csv_data"].append( + create_finding( + check_id="FS-59", + finding_name="Topic Restrictions Configured on CLASSIC Tier", + finding_details=( + f"Guardrails with topic policies: {', '.join(guardrails_with_topics)}. " + f"The following use the CLASSIC tier: {', '.join(topics_classic_tier)}. " + "CLASSIC tier supports English, French, and Spanish only; the STANDARD tier " + "(GA June 2025) adds broader language support for off-topic detection." + ), + resolution=( + "For multilingual FinServ deployments, consider upgrading denied topics to " + "the STANDARD tier (topicsTierConfig.tierName=STANDARD via UpdateGuardrail; " + "requires a cross-region inference profile)." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", + severity="Low", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-59"], + ) + ) else: findings["csv_data"].append( create_finding( @@ -4149,6 +4709,7 @@ def check_knowledge_base_sync_schedule() -> Dict[str, Any]: try: bedrock_agent = boto3.client("bedrock-agent", config=boto3_config) events = boto3.client("events", config=boto3_config) + scheduler = boto3.client("scheduler", config=boto3_config) paginator = bedrock_agent.get_paginator("list_knowledge_bases") kbs = [] @@ -4170,29 +4731,72 @@ def check_knowledge_base_sync_schedule() -> Dict[str, Any]: ) return findings - # Check for EventBridge rules that trigger KB sync + # Check for EventBridge rules (legacy) that trigger KB sync. rules = _paginate(events, "list_rules", "Rules") kb_sync_rules = [ - r for r in rules - if "bedrock" in r.get("Name", "").lower() or "knowledge" in r.get("Name", "").lower() + r + for r in rules + if "bedrock" in r.get("Name", "").lower() + or "knowledge" in r.get("Name", "").lower() ] - if not kb_sync_rules: + # Check for EventBridge Scheduler schedules (the AWS-recommended approach; + # classic EventBridge scheduled rules are a legacy feature). The + # ListSchedules name-prefix filter is server-side, so do a broad list and + # match on name/target heuristics here. ListSchedules is paginated via + # NextToken. Treat an access error as a soft signal (do not fail the whole + # check) since Scheduler may not be in use. + kb_sync_schedules = [] + try: + schedules = _paginate(scheduler, "list_schedules", "Schedules") + kb_sync_schedules = [ + s + for s in schedules + if "bedrock" in s.get("Name", "").lower() + or "knowledge" in s.get("Name", "").lower() + or "kb-sync" in s.get("Name", "").lower() + or "ingestion" in s.get("Name", "").lower() + or "bedrock" in s.get("Target", {}).get("Arn", "").lower() + ] + except ClientError as e: + if not _is_access_error(e): + raise + logger.warning( + "Could not list EventBridge Scheduler schedules (access denied); " + "FS-61 fell back to EventBridge rules only. Grant " + "scheduler:ListSchedules for full coverage." + ) + + total_sync_automation = len(kb_sync_rules) + len(kb_sync_schedules) + + if total_sync_automation == 0: findings["status"] = "WARN" findings["csv_data"].append( create_finding( check_id="FS-61", - finding_name="No Automated KB Sync Schedules Found", + finding_name="No Automated KB Sync Schedules Detected", finding_details=( - f"Found {len(kbs)} Knowledge Base(s) but no EventBridge rules for automated sync. " - "KB data may become stale without manual intervention." + f"Found {len(kbs)} Knowledge Base(s) but no EventBridge Scheduler " + "schedules or EventBridge rules with 'bedrock'/'knowledge' naming were " + "found. Note: this check uses a name/target heuristic — sync automation " + "with other naming conventions, AWS Step Functions-based orchestration, " + "or native Bedrock API-triggered syncs (StartIngestionJob called directly) " + "will not be detected. Verify sync automation manually if applicable." ), resolution=( - "1. Create EventBridge scheduled rules to trigger KB data source sync.\n" - "2. Set sync frequency based on data currency requirements.\n" - "3. Configure SNS alerts on sync failures." + "1. Use EventBridge Scheduler (the AWS-recommended approach) to create a " + "recurring schedule (e.g., rate(1 day) or a cron expression) that triggers a " + "Lambda function calling the Bedrock StartIngestionJob API for each data source. " + "Classic EventBridge scheduled rules also work but are a legacy feature.\n" + "2. As of December 2024, Bedrock Knowledge Bases supports custom connectors " + "and streaming data ingestion — use direct document ingestion " + "(KnowledgeBaseDocuments API) for real-time updates without a full S3 sync.\n" + "3. Set sync frequency based on data currency requirements " + "(e.g., hourly for market data, daily for regulatory guidance).\n" + "4. Configure CloudWatch alarms or SNS notifications on " + "IngestionJob FAILED status for sync failure alerting." ), - reference="https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base-ingest.html", + reference="https://docs.aws.amazon.com/scheduler/latest/UserGuide/what-is-scheduler.html", severity="Medium", status="Failed", compliance_frameworks=COMPLIANCE_MAP["FS-61"], @@ -4203,9 +4807,13 @@ def check_knowledge_base_sync_schedule() -> Dict[str, Any]: create_finding( check_id="FS-61", finding_name="Automated KB Sync Schedules Present", - finding_details=f"Found {len(kb_sync_rules)} EventBridge rule(s) for KB sync.", - resolution="No action required.", - reference="https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base-ingest.html", + finding_details=( + f"Found {len(kb_sync_schedules)} EventBridge Scheduler schedule(s) and " + f"{len(kb_sync_rules)} EventBridge rule(s) with KB-sync naming. " + "Verify each targets the Bedrock StartIngestionJob API for your KB data sources." + ), + resolution="Verify the schedule frequency matches your data-currency requirements.", + reference="https://docs.aws.amazon.com/scheduler/latest/UserGuide/what-is-scheduler.html", severity="Informational", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-61"], @@ -4255,9 +4863,11 @@ def check_foundation_model_lifecycle_policy() -> Dict[str, Any]: findings = _empty_findings("Foundation Model Lifecycle Policy Check") try: bedrock = boto3.client("bedrock", config=boto3_config) - models = bedrock.list_foundation_models(byOutputModality="TEXT").get( - "modelSummaries", [] - ) + # Do not filter by output modality: legacy/deprecated models exist across + # TEXT, EMBEDDING, and IMAGE modalities. Embedding models are widely used + # in FinServ RAG pipelines; a legacy embedding model silently serves stale + # vector representations. Fetch all modalities to surface any deprecated model. + models = bedrock.list_foundation_models().get("modelSummaries", []) legacy_models = [ m["modelId"] @@ -4269,7 +4879,8 @@ def check_foundation_model_lifecycle_policy() -> Dict[str, Any]: config_client = boto3.client("config", config=boto3_config) rules = _paginate(config_client, "describe_config_rules", "ConfigRules") lifecycle_rules = [ - r for r in rules + r + for r in rules if "lifecycle" in r.get("ConfigRuleName", "").lower() or "model" in r.get("ConfigRuleName", "").lower() ] @@ -4358,10 +4969,13 @@ def check_kb_datasource_s3_event_notifications() -> Dict[str, Any]: return findings buckets_without_notifications = [] + missing_buckets = [] for kb in kbs: kb_id = kb["knowledgeBaseId"] data_sources = _paginate( - bedrock_agent, "list_data_sources", "dataSourceSummaries", + bedrock_agent, + "list_data_sources", + "dataSourceSummaries", knowledgeBaseId=kb_id, ) for ds in data_sources: @@ -4378,13 +4992,17 @@ def check_kb_datasource_s3_event_notifications() -> Dict[str, Any]: if not bucket: continue try: - notif = s3_client.get_bucket_notification_configuration(Bucket=bucket) - has_notif = any([ - notif.get("TopicConfigurations"), - notif.get("QueueConfigurations"), - notif.get("LambdaFunctionConfigurations"), - notif.get("EventBridgeConfiguration"), - ]) + notif = s3_client.get_bucket_notification_configuration( + Bucket=bucket + ) + has_notif = any( + [ + notif.get("TopicConfigurations"), + notif.get("QueueConfigurations"), + notif.get("LambdaFunctionConfigurations"), + notif.get("EventBridgeConfiguration"), + ] + ) if not has_notif: buckets_without_notifications.append(bucket) except ClientError as e: @@ -4393,8 +5011,49 @@ def check_kb_datasource_s3_event_notifications() -> Dict[str, Any]: # than a false "missing notifications" finding. if _is_access_error(e): raise + # The data source points to a deleted bucket — a distinct + # integrity problem, not "notifications missing." + if _is_missing_bucket_error(e): + logger.warning( + f"KB '{kb.get('name', kb_id)}' data source " + f"'{ds.get('name', ds['dataSourceId'])}' references a " + f"deleted bucket: {bucket}" + ) + missing_buckets.append( + f"{bucket} (KB '{kb.get('name', kb_id)}', " + f"source '{ds.get('name', ds['dataSourceId'])}')" + ) + continue buckets_without_notifications.append(f"{bucket} (error)") + # A dangling data-source reference to a deleted bucket is a real integrity + # finding — emit it separately so it is not conflated with "no notifications." + if missing_buckets: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-65", + finding_name="KB Data Source References a Deleted S3 Bucket", + finding_details=( + "One or more Knowledge Base data sources point to S3 buckets that no " + "longer exist (NoSuchBucket). Document-change monitoring cannot be " + "assessed and KB grounding data is missing for these sources:\n" + + "\n".join(f"- {b}" for b in missing_buckets[:10]) + ), + resolution=( + "1. Investigate why the data-source bucket was deleted (accidental " + "deletion, environment teardown, or stale KB configuration).\n" + "2. Recreate/restore the bucket (with event notifications and versioning " + "enabled), or remove the orphaned data source from the Knowledge Base.\n" + "3. Re-run a KB ingestion job after restoring the data source." + ), + reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/EventBridge.html", + severity="High", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-65"], + ) + ) + if buckets_without_notifications: findings["status"] = "WARN" findings["csv_data"].append( @@ -4404,7 +5063,9 @@ def check_kb_datasource_s3_event_notifications() -> Dict[str, Any]: finding_details=( "The following KB data-source S3 buckets have no event notifications configured. " "Unauthorized document modifications will not be detected in real time:\n" - + "\n".join(f"- {b}" for b in buckets_without_notifications[:10]) + + "\n".join( + f"- {b}" for b in buckets_without_notifications[:10] + ) ), resolution=( "1. Enable Amazon EventBridge notifications on each KB data-source S3 bucket.\n" @@ -4418,7 +5079,7 @@ def check_kb_datasource_s3_event_notifications() -> Dict[str, Any]: compliance_frameworks=COMPLIANCE_MAP["FS-65"], ) ) - else: + elif not missing_buckets: findings["csv_data"].append( create_finding( check_id="FS-65", @@ -4546,10 +5207,19 @@ def check_agent_financial_transaction_thresholds() -> Dict[str, Any]: # Look for agent action-group Lambda functions action_group_lambdas = [ - f for f in functions - if any(kw in f["FunctionName"].lower() for kw in [ - "agent", "action", "tool", "bedrock", "finserv", "transaction" - ]) + f + for f in functions + if any( + kw in f["FunctionName"].lower() + for kw in [ + "agent", + "action", + "tool", + "bedrock", + "finserv", + "transaction", + ] + ) ] if not action_group_lambdas: @@ -4581,7 +5251,9 @@ def check_agent_financial_transaction_thresholds() -> Dict[str, Any]: f["FunctionName"] for f in action_group_lambdas if not any( - "threshold" in k.lower() or "limit" in k.lower() or "max" in k.lower() + "threshold" in k.lower() + or "limit" in k.lower() + or "max" in k.lower() for k in f.get("Environment", {}).get("Variables", {}).keys() ) ] @@ -4599,7 +5271,9 @@ def check_agent_financial_transaction_thresholds() -> Dict[str, Any]: "AgentCore Policy Engine rule would not be detected here, so treat this as " "a prompt for manual verification rather than a definitive gap). " "Without explicit limits, agents could initiate unbounded financial transactions:\n" - + "\n".join(f"- {n}" for n in lambdas_without_threshold_config[:10]) + + "\n".join( + f"- {n}" for n in lambdas_without_threshold_config[:10] + ) ), resolution=( "1. Add transaction-value threshold environment variables (e.g., MAX_TRANSACTION_AMOUNT) " @@ -4651,7 +5325,9 @@ def check_api_gateway_request_body_size_limits() -> Dict[str, Any]: rest_apis = _paginate(apigw, "get_rest_apis", "items") apis_without_validators = [] for api in rest_apis: - validators = apigw.get_request_validators(restApiId=api["id"]).get("items", []) + validators = apigw.get_request_validators(restApiId=api["id"]).get( + "items", [] + ) if not validators: apis_without_validators.append(api.get("name", api["id"])) @@ -4737,8 +5413,15 @@ def check_prompt_input_validation_function() -> Dict[str, Any]: # Look for Lambda functions with input validation / sanitization naming patterns VALIDATION_KEYWORDS = [ - "sanitiz", "validat", "input", "preprocess", "pre-process", - "filter", "clean", "prompt-guard", "promptguard", + "sanitiz", + "validat", + "input", + "preprocess", + "pre-process", + "filter", + "clean", + "prompt-guard", + "promptguard", ] validation_lambdas = [ f["FunctionName"] @@ -4803,6 +5486,7 @@ def check_prompt_input_validation_function() -> Dict[str, Any]: # REPORT GENERATION & LAMBDA HANDLER # =========================================================================== + def generate_csv_report(findings: List[Dict[str, Any]]) -> str: """Generate CSV report from all security check findings.""" csv_buffer = StringIO() @@ -4854,7 +5538,10 @@ def build_finserv_checks(permission_cache): ("FS-05", check_cloudwatch_token_alarms), ("FS-06", check_aws_budgets_for_aiml), # --- Category 2: Excessive Agency --- - ("FS-07", functools.partial(check_bedrock_agent_action_boundaries, permission_cache)), + ( + "FS-07", + functools.partial(check_bedrock_agent_action_boundaries, permission_cache), + ), ("FS-08", check_agentcore_policy_engine), ("FS-09", check_agent_transaction_limits), ("FS-10", check_human_in_the_loop_for_high_risk_actions), @@ -4869,12 +5556,21 @@ def build_finserv_checks(permission_cache): ("FS-20", check_feature_store_rollback_capability), ("FS-21", check_training_data_s3_versioning), # --- Category 5: Vector & Embedding Weaknesses --- - ("FS-22", functools.partial(check_knowledge_base_iam_least_privilege, permission_cache)), + ( + "FS-22", + functools.partial( + check_knowledge_base_iam_least_privilege, permission_cache + ), + ), ("FS-24", check_knowledge_base_metadata_filtering), ("FS-25", check_opensearch_serverless_encryption), ("FS-26", check_knowledge_base_vpc_access), # --- Category 6: Non-Compliant Output --- - ("FS-27", check_automated_reasoning_checks), + # FS-27 is split into two checks: contextual grounding (threshold-based) and + # Automated Reasoning policies (formal-verification, GA August 2025). Both + # use the FS-27 check_id so they appear together in the CSV report. + ("FS-27", check_guardrail_contextual_grounding), + ("FS-27", check_automated_reasoning_policies), ("FS-28", check_guardrail_denied_topics_financial), ("FS-29", check_compliance_disclaimer_in_outputs), ("FS-30", check_bedrock_evaluation_compliance_datasets), @@ -4902,7 +5598,7 @@ def build_finserv_checks(permission_cache): ("FS-47", check_guardrail_grounding_threshold), ("FS-48", check_rag_knowledge_base_configured), ("FS-49", check_hallucination_disclaimer_advisory), - ("FS-50", check_automated_reasoning_checks_hallucination), + ("FS-50", check_guardrail_relevance_grounding), # --- Category 12: Prompt Injection --- ("FS-51", check_prompt_injection_input_validation), ("FS-52", check_bedrock_sdk_version_currency), @@ -4930,7 +5626,13 @@ def build_finserv_checks(permission_cache): def lambda_handler(event, context): - """Main Lambda handler — runs all 64 FinServ security checks.""" + """Main Lambda handler — runs all FinServ security checks. + + The registry in build_finserv_checks() contains 65 entries (64 standalone + FS checks plus the new FS-27 Automated Reasoning Policies check). Two + entries share the FS-27 check_id (contextual grounding + ARC policies), + both contributing rows to the report under the same check namespace. + """ logger.info("Starting FinServ GenAI security assessment") all_findings = [] diff --git a/aiml-security-assessment/functions/security/finserv_assessments/schema.py b/aiml-security-assessment/functions/security/finserv_assessments/schema.py index d8f7b6a..657ad66 100644 --- a/aiml-security-assessment/functions/security/finserv_assessments/schema.py +++ b/aiml-security-assessment/functions/security/finserv_assessments/schema.py @@ -2,6 +2,7 @@ Schema module for FinServ security findings. Mirrors the schema used in bedrock_assessments/schema.py. """ + from enum import Enum from typing import Dict, Any, Optional from pydantic import BaseModel, Field, field_validator diff --git a/deployment/1-aiml-security-member-roles.yaml b/deployment/1-aiml-security-member-roles.yaml index 2064c67..bb61e4d 100644 --- a/deployment/1-aiml-security-member-roles.yaml +++ b/deployment/1-aiml-security-member-roles.yaml @@ -272,6 +272,12 @@ Resources: - macie2:GetMacieSession # EventBridge — KB integrity monitoring (FS-33, FS-65) - events:ListRules + # EventBridge Scheduler (KB sync schedules - FS-61). + # AWS-recommended scheduling approach; classic EventBridge rules are legacy. + - scheduler:ListSchedules + # Bedrock Automated Reasoning (GA August 2025). + # Required for check_automated_reasoning_policies() (FS-27). + - bedrock:ListAutomatedReasoningPolicies # Config — model onboarding governance (FS-14) - config:DescribeConfigRules Resource: "*" diff --git a/deployment/aiml-security-single-account.yaml b/deployment/aiml-security-single-account.yaml index 2b68f6c..5461862 100644 --- a/deployment/aiml-security-single-account.yaml +++ b/deployment/aiml-security-single-account.yaml @@ -298,6 +298,12 @@ Resources: - macie2:GetMacieSession # EventBridge — KB integrity monitoring (FS-33, FS-65) - events:ListRules + # EventBridge Scheduler (KB sync schedules - FS-61). + # AWS-recommended scheduling approach; classic EventBridge rules are legacy. + - scheduler:ListSchedules + # Bedrock Automated Reasoning (GA August 2025). + # Required for check_automated_reasoning_policies() (FS-27). + - bedrock:ListAutomatedReasoningPolicies # Config — model onboarding governance (FS-14) - config:DescribeConfigRules Resource: "*" diff --git a/sample-reports/finserv_security_report_phase3.csv b/sample-reports/finserv_security_report_phase3.csv new file mode 100644 index 0000000..bd877bd --- /dev/null +++ b/sample-reports/finserv_security_report_phase3.csv @@ -0,0 +1,190 @@ +Check_ID,Finding,Finding_Details,Resolution,Reference,Severity,Status,Compliance_Frameworks +FS-01,AWS Shield Advanced Not Enabled,AWS Shield Advanced is not subscribed. GenAI API endpoints are vulnerable to volumetric DDoS attacks that can exhaust token quotas and inflate costs.,"1. Subscribe to AWS Shield Advanced for DDoS protection. +2. After subscribing, explicitly add resource protections in the Shield Advanced console for each Bedrock-facing resource (API Gateway stages, ALBs, CloudFront distributions, Route 53 hosted zones). Shield Advanced subscription alone does NOT automatically protect resources — each resource must be individually added to receive protection. +3. Enable Shield Response Team (SRT) access and configure proactive engagement. +4. Alternatively, use AWS Firewall Manager with a Shield Advanced policy to automate resource protection based on tags or resource types.",https://docs.aws.amazon.com/waf/latest/developerguide/shield-chapter.html,High,Failed,FFIEC CAT | DORA Art.6 +FS-01,Regional WAF Web ACLs Present,Found 1 regional WAF Web ACL(s).,Verify ACLs are associated with Bedrock-facing endpoints.,https://docs.aws.amazon.com/waf/latest/developerguide/waf-chapter.html,Informational,Passed,FFIEC CAT | DORA Art.6 +FS-02,API Gateway Rate Limiting Configured,All 1 usage plan(s) have throttle settings.,No action required.,https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-request-throttling.html,Informational,Passed,FFIEC CAT | DORA Art.6 | PCI-DSS 12.3.2 +FS-03,Bedrock Token Quotas At Default,"All 224 Bedrock token-based quota(s) are at their AWS default values — no quota increase has been applied. Running at default is a legitimate posture, but it should be a reviewed decision aligned with expected peak load rather than an oversight.","1. Review current Bedrock TPM/TPD quotas in the Service Quotas console. +2. Request increases aligned with expected peak load, or document a deliberate decision to remain at default after review. +3. Implement client-side token counting and pre-flight quota checks. +4. Use Bedrock cross-region inference profiles to distribute load.",https://docs.aws.amazon.com/bedrock/latest/userguide/quotas.html,Medium,N/A,FFIEC CAT | SR 11-7 +FS-04,No Cost Anomaly Detection Monitors,"No AWS Cost Anomaly Detection monitors found. Unexpected spikes in Bedrock/SageMaker usage (e.g., from prompt injection loops) will go undetected.","1. Create a Cost Anomaly Detection monitor scoped to AWS/Bedrock and AWS/SageMaker. +2. Configure alert subscriptions (SNS/email) for anomalies above threshold. +3. Set daily spend budgets with AWS Budgets as a secondary control.",https://docs.aws.amazon.com/cost-management/latest/userguide/getting-started-ad.html,Medium,Failed,FFIEC CAT | SR 11-7 +FS-05,No Bedrock CloudWatch Alarms Found,No CloudWatch alarms found for Bedrock metrics. Token exhaustion and throttling events will not trigger operational alerts.,"Create CloudWatch alarms for: +- AWS/Bedrock InvocationThrottles (threshold > 0) +- AWS/Bedrock TokensProcessed (threshold based on quota) +- Custom application-level token counters via EMF",https://docs.aws.amazon.com/bedrock/latest/userguide/monitoring-cw.html,Medium,Failed,FFIEC CAT | DORA Art.6 +FS-06,No AI/ML Service Budgets Configured,No AWS Budgets found scoped to Bedrock or SageMaker. Unbounded GenAI spend can go undetected until the monthly bill.,"1. Create cost budgets for AWS Bedrock and SageMaker with 80%/100% alert thresholds. +2. Add SNS notifications to on-call channels. +3. Consider budget actions to apply IAM deny policies when thresholds are breached.",https://docs.aws.amazon.com/cost-management/latest/userguide/budgets-managing-costs.html,Medium,Failed,FFIEC CAT | SR 11-7 +FS-07,Agent Action Boundaries Look Appropriate,Reviewed 4 agent(s); no wildcard sensitive actions found.,No action required.,https://docs.aws.amazon.com/bedrock/latest/userguide/agents-permissions.html,Informational,Passed,SR 11-7 | FFIEC CAT +FS-08,AgentCore Runtimes Missing Policy Engine,"Runtimes without authorizer configuration: vendoriq_vendoriq, test_minimal, strands_claude_short_timeout, strands_claude_getting_started, proposal_drafter, harness_harness_ctuux, executive_summary_writer, document_generation, datetimemcp_Agent. Without a policy engine, agents can invoke any registered tool without authorization checks.",Configure an authorizer (Lambda or Cedar policy store) on each AgentCore runtime to enforce fine-grained tool-call authorization.,https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/security-authorization.html,High,Failed,SR 11-7 | MAS TRM 9.1 +FS-09,Agent Lambda Functions Without Concurrency Limits,"Agent-related Lambda functions without reserved concurrency: Dev-DashboardStack-DashboardApiLayerAgentSessionsF-EnQLRGDeLRkn, fedpoint-kms-poc-create-bedrock-index, Dev-DashboardStack-DashboardApiLayerAgentInvokeFn3-r3fQdrJuboTX. Unlimited concurrency allows runaway agent loops to exhaust account limits.","1. Set reserved concurrency on agent Lambda functions. +2. Implement maximum iteration counts in agent orchestration logic. +3. Use Step Functions with MaxConcurrency and timeout states. +4. Add circuit-breaker patterns to agent tool invocations.",https://docs.aws.amazon.com/lambda/latest/dg/configuration-concurrency.html,Medium,Failed,FFIEC CAT | SR 11-7 +FS-10,Human-in-the-Loop Check — No Agent Workflows Found,"No Step Functions state machines with agent/approval naming found. Verify that high-risk agent actions (e.g., fund transfers, account changes) have human approval gates.",Implement Step Functions .waitForTaskToken patterns for high-risk agent actions. Route approval requests to human reviewers via SNS/SES/Slack.,https://docs.aws.amazon.com/step-functions/latest/dg/connect-to-resource.html#connect-wait-token,Medium,N/A,SR 11-7 | FFIEC CAT | MAS TRM 9.2 +FS-11,No Agent Rate Alarms Found,No CloudWatch alarms found for agent invocation rates. Looping or runaway agents will not trigger operational alerts.,"Create CloudWatch alarms on: +- Bedrock agent invocation counts (threshold based on expected max) +- Lambda invocation errors for agent functions +- Step Functions execution failures and timeouts",https://docs.aws.amazon.com/bedrock/latest/userguide/monitoring-cw.html,Medium,Failed,FFIEC CAT | DORA Art.6 +FS-12,Bedrock SCPs Found,SCPs referencing Bedrock: InnovationSandboxAwsNukeSupportedServicesScp.,Verify SCPs use bedrock:ModelId conditions to allowlist approved models.,https://docs.aws.amazon.com/bedrock/latest/userguide/security_iam_id-based-policy-examples.html,Informational,Passed,SR 11-7 | FFIEC CAT | ISO 27001 A.15.2 +FS-13,Model Provenance Tags Present,All reviewed models have required provenance tags.,No action required.,https://docs.aws.amazon.com/bedrock/latest/userguide/tagging.html,Informational,Passed,SR 11-7 | ISO 27001 A.12.5 | FFIEC CAT +FS-14,No Model Governance Config Rules Found,No AWS Config rules found for Bedrock model governance. Unapproved models may be deployed without detection.,"1. Create custom AWS Config rules to detect use of non-approved Bedrock models. +2. Use AWS Service Catalog to publish approved model configurations. +3. Implement a model risk management (MRM) process per SR 11-7.",https://docs.aws.amazon.com/config/latest/developerguide/evaluate-config.html,Medium,Failed,SR 11-7 | FFIEC CAT | ISO 27001 A.15.1 +FS-15,No Bedrock Evaluation Jobs Found,No Bedrock Model Evaluation jobs found. Models have not been evaluated for adversarial robustness.,"1. Run Bedrock Model Evaluation with adversarial/red-team datasets. +2. Use FMEval library for automated robustness testing. +3. Schedule periodic re-evaluation after model updates.",https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html,Medium,N/A,SR 11-7 | FFIEC CAT | MAS TRM 9.3 +FS-16,ECR Repositories Without Image Scanning,"5 ECR repo(s) without scan-on-push: bedrock-agentcore-strands_claude_getting_started, bedrock-agentcore-strands_claude_short_timeout, cdk-hnb659fds-container-assets-469898429403-us-east-1, bracket-predictor, vendor-iq-dashboard.",Enable scan-on-push for all ECR repositories containing model containers. Consider enabling Enhanced Scanning (Inspector) for CVE detection.,https://docs.aws.amazon.com/AmazonECR/latest/userguide/image-scanning.html,High,Failed,ISO 27001 A.12.6 | FFIEC CAT | DORA Art.6 +FS-20,No SageMaker Feature Groups Found,No SageMaker Feature Store groups found.,No action required.,https://docs.aws.amazon.com/sagemaker/latest/dg/feature-store.html,Informational,N/A,SR 11-7 | FFIEC CAT +FS-21,Training Data Buckets Without Versioning,"4 training data bucket(s) without versioning: bedrock-agentcore-codebuild-sources-469898429403-us-east-1, bedrock-agentcore-runtime-469898429403-us-east-1-2h09526czr, sagemaker-studio-469898429403-vs7m9hvz7np, sagemaker-us-east-1-469898429403.",Enable S3 versioning on all training data buckets. Consider enabling MFA Delete for additional protection against poisoning.,https://docs.aws.amazon.com/AmazonS3/latest/userguide/Versioning.html,High,Failed,SR 11-7 | ISO 27001 A.12.3 | FFIEC CAT +FS-22,Knowledge Base IAM Permissions Look Appropriate,No wildcard KB permissions found in reviewed roles.,No action required.,https://docs.aws.amazon.com/bedrock/latest/userguide/security-iam-awsmanpol.html,Informational,Passed,NYDFS 500 | FFIEC CAT | PCI-DSS 12.3.2 +FS-24,Knowledge Base Metadata Filtering — Manual Review Required,"Found 2 Knowledge Base(s). Verify that metadata attributes (e.g., tenantId, classification) are indexed and that Retrieve calls include RetrievalFilter conditions for tenant isolation.","1. Add metadata fields (tenantId, dataClassification) to KB data sources. +2. Pass RetrievalFilter in all Retrieve/RetrieveAndGenerate calls. +3. Validate filters in integration tests to prevent cross-tenant data leakage.",https://docs.aws.amazon.com/bedrock/latest/userguide/kb-test-config.html,Medium,Passed,NYDFS 500 | FFIEC CAT | PCI-DSS 12.3.2 +FS-25,OpenSearch Serverless Encryption Policies Present,Found 5 encryption policy(ies); 5 use a customer-managed KMS key.,Verify all vector store collections use customer-managed KMS keys.,https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-encryption.html,Informational,Passed,NYDFS 500 | PCI-DSS 3.5 | FFIEC CAT +FS-26,OpenSearch Serverless Collections Not VPC-Restricted,Found 5 network policy(ies) but none restrict to VPC. Vector stores may be accessible from the public internet.,Update network policies to allow access only from VPC endpoints. Create an OpenSearch Serverless VPC endpoint in your VPC.,https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-network.html,High,Failed,NYDFS 500 | FFIEC CAT | PCI-DSS 1.3 +FS-27,No Guardrails — Contextual Grounding Not Applicable,No Bedrock Guardrails configured. Configure guardrails first (see BR-05).,Configure Bedrock Guardrails with contextual grounding checks (grounding threshold ≥0.7 and relevance threshold ≥0.7 for FinServ use cases).,https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html,Medium,N/A,SR 11-7 | FFIEC CAT | MAS TRM 9.2 +FS-27,No Automated Reasoning Policies Found,"No Bedrock Automated Reasoning policies have been created. ARC (GA August 2025) uses formal verification to guarantee that GenAI outputs comply with authored business rules — e.g., loan criteria, regulatory thresholds, policy constraints. Without ARC policies, factual accuracy of outputs is not formally verified, only heuristically filtered by contextual grounding thresholds.","1. In the Amazon Bedrock console → Guardrails → Automated Reasoning, create a policy document encoding your FinServ business rules (e.g., eligibility criteria, rate limits, regulatory thresholds). +2. Associate the ARC policy with your guardrail (automatedReasoningPolicy.policies field in CreateGuardrail/UpdateGuardrail). +3. Set confidenceThreshold on the policy to control strictness. +4. ARC requires cross-Region inference — ensure your guardrail has a guardrailProfileArn configured (crossRegionDetails in GetGuardrail response). +5. Reference: AWS Announcement — Automated Reasoning checks GA (August 2025).",https://docs.aws.amazon.com/bedrock/latest/userguide/automated-reasoning.html,Medium,Failed,SR 11-7 | FFIEC CAT | MAS TRM 9.2 +FS-28,No Guardrails — Denied Topics Not Applicable,No Bedrock Guardrails configured.,Configure guardrails with denied topics for regulated financial content.,https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html,Medium,N/A,SR 11-7 | FFIEC CAT | NYDFS 500 | MAS TRM 9.2 +FS-29,ADVISORY: Compliance Disclaimer — Manual Review Required,Application-level compliance disclaimers cannot be verified via AWS APIs. Manual review required to confirm GenAI outputs include required regulatory disclosures.,"1. Implement post-processing to append required disclaimers to GenAI outputs. +2. Use Bedrock Guardrails word filters to block outputs that omit required disclosures. +3. Document disclaimer requirements in the AI use case register. +4. Test disclaimer presence in QA/UAT before production deployment.",https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html,Informational,N/A,SR 11-7 | FFIEC CAT | NYDFS 500 | MAS TRM 9.2 +FS-30,No Bedrock Evaluation Jobs — Compliance Datasets Not Verified,No Bedrock Model Evaluation jobs found.,"Run Bedrock Model Evaluation with compliance-specific datasets: +- Fair lending test cases (ECOA, Fair Housing Act) +- UDAP/UDAAP unfair/deceptive practice scenarios +- AML/KYC edge cases",https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html,Medium,N/A,SR 11-7 | FFIEC CAT | NYDFS 500 +FS-31,Knowledge Base Data Sources Past Review Threshold,"1 data source(s) not synced in >7 days (a configurable review threshold, NOT an AWS-mandated limit): +- KB 'fedpoint-kms-poc-knowledge-base' source 'fedpoint-kms-poc-content-data-source' last synced 194 days ago +Confirm this age is acceptable for each data source's currency requirement — slow-changing reference data may legitimately sync infrequently.","1. Define the maximum acceptable data age per use case (e.g., intraday for market data, daily for product terms, weekly/monthly for regulatory guidance) and adjust the review threshold to match. +2. Configure automated sync (EventBridge Scheduler → StartIngestionJob) at that cadence — see FS-61. +3. Set CloudWatch alarms on sync job failures.",https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base-ingest.html,Medium,Failed,SR 11-7 | FFIEC CAT +FS-32,ADVISORY: Source Attribution — Manual Review Required,Source attribution in GenAI responses cannot be verified via AWS APIs. Manual review required to confirm responses include citations.,"1. Use Bedrock RetrieveAndGenerate with citations enabled. +2. Include source document references in response post-processing. +3. Test citation accuracy in QA before production deployment. +4. Consider Bedrock Guardrails grounding checks to validate response accuracy.",https://docs.aws.amazon.com/bedrock/latest/userguide/kb-test-config.html,Informational,N/A,SR 11-7 | FFIEC CAT | MAS TRM 9.2 +FS-33,KB Data Source References a Deleted S3 Bucket,"One or more Knowledge Base data sources point to S3 buckets that no longer exist (NoSuchBucket). Retrieval will silently return no results for these sources, and the integrity of the KB's grounding data cannot be verified: +- fedpoint-kms-poc-content-469898429403 (KB 'fedpoint-kms-poc-knowledge-base', source 'fedpoint-kms-poc-content-data-source')","1. Investigate why the data-source bucket was deleted (accidental deletion, environment teardown, or a stale KB configuration). +2. Recreate/restore the bucket with versioning enabled, or remove the orphaned data source from the Knowledge Base. +3. Re-run a KB ingestion job after restoring the data source. +4. Enable S3 versioning and MFA Delete on KB data-source buckets to reduce the risk of unrecoverable deletion.",https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html,High,Failed,SR 11-7 | FFIEC CAT | ISO 27001 A.12 +FS-34,Legacy Foundation Models Available in Region,"Legacy/deprecated foundation models are available in this account/region: anthropic.claude-sonnet-4-20250514-v1:0, amazon.titan-image-generator-v2:0, amazon.nova-premier-v1:0:8k, amazon.nova-premier-v1:0:20k, amazon.nova-premier-v1:0:1000k, amazon.nova-premier-v1:0:mm, amazon.nova-premier-v1:0, amazon.nova-canvas-v1:0, amazon.nova-reel-v1:0, amazon.nova-reel-v1:1. This API reports model *availability*, not actual usage — it cannot determine which models your applications invoke. Legacy models have older training-data cutoffs and may produce outdated information if used. Review whether any are in active use.","1. Identify which (if any) of these legacy models your applications invoke (e.g., via CloudTrail InvokeModel events or application config). +2. Migrate active usage to current model versions. +3. Document training-data cutoff dates for all models in use. +4. Add data-currency disclaimers to outputs from models with old cutoffs.",https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle.html,Medium,N/A,SR 11-7 | FFIEC CAT +FS-35,No Evaluation Jobs — Harmful Content Testing Not Verified,No Bedrock Model Evaluation jobs found.,"Run Bedrock Model Evaluation or FMEval with harmful content datasets: +- Toxicity detection +- Hate speech classification +- Violence/self-harm content",https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html,Medium,N/A,SR 11-7 | FFIEC CAT +FS-36,No Guardrails — Content Filters Not Applicable,No Bedrock Guardrails configured.,Configure guardrails with content filters.,https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-filters.html,High,N/A,SR 11-7 | FFIEC CAT +FS-37,ADVISORY: User Feedback Mechanism — Manual Review Required,User feedback mechanisms for harmful outputs cannot be verified via AWS APIs. Manual review required.,"1. Implement thumbs-up/down or flag-for-review UI in GenAI applications. +2. Route flagged outputs to human reviewers via SQS/SNS. +3. Log feedback to DynamoDB/S3 for model improvement. +4. Define SLAs for reviewing flagged content.",https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails.html,Informational,N/A,SR 11-7 | FFIEC CAT | MAS TRM 9.2 +FS-38,No Guardrails — Word Filters Not Applicable,No Bedrock Guardrails configured.,Configure guardrails with word filters.,https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html,Medium,N/A,SR 11-7 | FFIEC CAT +FS-39,No SageMaker Clarify Bias Monitoring,"No SageMaker Clarify model bias monitoring schedules found. Models making financial decisions (credit, insurance) may exhibit discriminatory bias without detection.","1. Configure SageMaker Clarify bias detection for all models making credit, insurance, or employment decisions. +2. Define protected attributes (age, gender, race proxies). +3. Set bias metric thresholds and alert on violations. +4. Document bias testing results for regulatory examination.",https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-model-monitor-bias-drift.html,High,Failed,SR 11-7 | FFIEC CAT | ECOA/Fair Housing +FS-40,No Evaluation Jobs — Bias Datasets Not Verified,No Bedrock Model Evaluation jobs found.,"Run Bedrock Model Evaluation with bias test datasets: +- Demographic parity test cases +- Equal opportunity scenarios +- Counterfactual fairness tests",https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html,Medium,N/A,SR 11-7 | FFIEC CAT | ECOA/Fair Housing +FS-41,No SageMaker Clarify Explainability Monitoring,No SageMaker Clarify explainability monitoring found. Models making adverse financial decisions may not provide required explanations (ECOA adverse action notices).,"1. Configure SageMaker Clarify explainability for credit/lending models. +2. Generate SHAP values for feature importance. +3. Map top features to human-readable adverse action reason codes. +4. Store explanations for regulatory examination.",https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-model-explainability.html,High,Failed,SR 11-7 | FFIEC CAT +FS-42,No SageMaker Model Cards Found,"No SageMaker Model Cards found. Production AI models lack documented intended use, limitations, and bias evaluations.","1. Create SageMaker Model Cards for all production models. +2. Document: intended use, out-of-scope uses, training data, bias evaluations. +3. Include regulatory compliance attestations. +4. Review and update cards at each model version release.",https://docs.aws.amazon.com/sagemaker/latest/dg/model-cards.html,Medium,Failed,SR 11-7 | FFIEC CAT +FS-43,No CloudWatch Logs Data Protection Policies,"No CloudWatch Logs data protection policies found. PII (SSN, account numbers, credit card numbers) in Bedrock invocation logs may be stored in plaintext.","1. Create CloudWatch Logs data protection policies to mask PII. +2. Enable masking for: SSN, credit card numbers, bank account numbers, email. +3. Apply policies to Bedrock invocation log groups. +4. Test masking with synthetic PII before production deployment.",https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/mask-sensitive-log-data.html,High,Failed,NYDFS 500 | FFIEC CAT | PCI-DSS +FS-44,Amazon Macie Not Enabled,Amazon Macie is not enabled. S3 buckets containing training data and KB data sources are not being scanned for PII/sensitive data.,"1. Enable Amazon Macie in all regions where AI/ML data is stored. +2. Create Macie classification jobs for training data and KB buckets. +3. Configure Macie findings to route to Security Hub and SNS. +4. Remediate PII findings before using data for model training.",https://docs.aws.amazon.com/macie/latest/user/what-is-macie.html,High,Failed,NYDFS 500 | FFIEC CAT | PCI-DSS +FS-45,No Guardrails — PII Filters Not Applicable,No Bedrock Guardrails configured.,Configure guardrails with PII/sensitive information filters.,https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-sensitive-filters.html,High,N/A,NYDFS 500 | FFIEC CAT | PCI-DSS +FS-46,AI/ML Buckets Without Data Classification Tags,"12 AI/ML bucket(s) without data-classification tags: bedrock-agentcore-codebuild-sources-469898429403-us-east-1, bedrock-agentcore-runtime-469898429403-us-east-1-2h09526czr, bedrock-usage-intel-failed-records-469898429403-us-east-1, bedrock-usage-intel-processed-data-469898429403-us-east-1, bedrock-usage-intel-raw-logs-469898429403-us-east-1, dev-vendorcommandcenterst-knowledgebasekbbucket1bf-bfoajiinsr4n, dev-vendorcommandcenterst-knowledgebasekbbucket1bf-dttwp5qrbqyq, dev-vendorcommandcenterst-knowledgebasekbbucket1bf-mure4ntcjmmf, dev-vendorcommandcenterst-knowledgebasekbbucket1bf-pivy5zfs0whp, fedpoint-quick-kb.","Tag all AI/ML data buckets with 'data-classification' key. Values: Public, Internal, Confidential, Restricted. Enforce via SCP or AWS Config rule.",https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html,Medium,Failed,NYDFS 500 | FFIEC CAT | ISO 27001 A.12 | PCI-DSS +FS-47,No Guardrails — Grounding Threshold Not Applicable,No Bedrock Guardrails configured.,Configure guardrails with contextual grounding checks.,https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html,High,N/A,SR 11-7 | FFIEC CAT +FS-48,Active Knowledge Bases for RAG Present,Found 2 active Knowledge Base(s) for RAG grounding.,No action required.,https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base.html,Informational,Passed,SR 11-7 | FFIEC CAT +FS-49,ADVISORY: Hallucination Disclaimer — Manual Review Required,Application-level hallucination disclaimers cannot be verified via AWS APIs. Manual review required.,"1. Add disclaimers to GenAI outputs: 'AI-generated content may contain errors. Verify with authoritative sources before acting.' +2. Implement post-processing to append disclaimers. +3. Test disclaimer presence in QA before production.",https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails.html,Informational,N/A,SR 11-7 | FFIEC CAT | MAS TRM 9.2 +FS-50,No Guardrails With Relevance Grounding Filters,"No guardrails have RELEVANCE contextual grounding filters. Without relevance filters, responses that are off-topic or unrelated to the user query will not be blocked, increasing hallucination risk in RAG-based FinServ applications.",Enable the RELEVANCE contextual grounding filter in Bedrock Guardrails with a threshold of ≥0.7 to block responses that are not relevant to the user query. Also enable the GROUNDING filter (≥0.7) to block responses not supported by the retrieved source context.,https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html,Medium,Failed,SR 11-7 | FFIEC CAT +FS-51,No Guardrails — Prompt Attack Filters Not Applicable,No Bedrock Guardrails configured.,Configure guardrails with prompt attack filters.,https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html,High,N/A,NYDFS 500 | FFIEC CAT | OWASP LLM Top 10 +FS-52,Bedrock Lambda Functions on Current Runtimes,All 3 Bedrock Lambda function(s) use current runtimes.,No action required.,https://docs.aws.amazon.com/lambda/latest/dg/runtimes-update.html,Informational,Passed,NYDFS 500 | FFIEC CAT | ISO 27001 A.12 | OWASP LLM Top 10 +FS-53,WAF Injection Protection Rules Present,All 1 WAF ACL(s) have injection protection rules.,No action required.,https://docs.aws.amazon.com/waf/latest/developerguide/aws-managed-rule-groups-baseline.html,Informational,Passed,NYDFS 500 | PCI-DSS | FFIEC CAT | OWASP LLM Top 10 +FS-54,ADVISORY: Penetration Testing — Manual Review Required,Penetration testing evidence cannot be verified via AWS APIs. Manual review required to confirm GenAI applications have been tested.,"1. Conduct penetration testing of GenAI applications at least annually and before major releases. +2. Include AI-specific test cases: prompt injection, jailbreak, indirect (cross-domain) injection, system-prompt leakage, and data-extraction attempts. +3. Consider AWS Security Agent for on-demand, AI-driven penetration testing (GA March 2026; available in US East N. Virginia, US West Oregon, Europe Ireland, Europe Frankfurt, Asia Pacific Sydney, Asia Pacific Tokyo, with cross-account shared-VPC testing via AWS RAM). Open-source tools such as Garak or PyRIT and manual red-teaming are complementary options. Verify current regional availability on the AWS Security Agent page before relying on it. +4. Document findings and remediation for regulatory examination, and tag tested resources with a last-pentest-date for audit trail. +5. For DORA compliance, include GenAI in TLPT (Threat-Led Penetration Testing) scope.",https://aws.amazon.com/security/penetration-testing/,Informational,N/A,NYDFS 500 | FFIEC CAT | OWASP LLM Top 10 +FS-55,No Output Validation Functions Found,No Lambda functions with output validation/sanitization naming found. GenAI outputs may be passed directly to downstream systems without validation.,"1. Implement output validation Lambda functions in GenAI pipelines. +2. Validate output schema, length, and content before downstream use. +3. Sanitize outputs before rendering in web UIs (XSS prevention). +4. Encode outputs appropriately for the target context (HTML, SQL, JSON).",https://genai.owasp.org/llm-top-10/,Medium,Failed,FFIEC CAT | OWASP LLM Top 10 +FS-56,XSS Prevention — Review WAF Common Rule Set,Found 1 WAF ACL(s). Verify AWSManagedRulesCommonRuleSet is enabled for XSS prevention (see FS-53).,"Ensure AWSManagedRulesCommonRuleSet is enabled on all WAF ACLs protecting GenAI web applications. Additionally, implement Content Security Policy (CSP) headers.",https://docs.aws.amazon.com/waf/latest/developerguide/aws-managed-rule-groups-baseline.html,Medium,Passed,NYDFS 500 | PCI-DSS | OWASP LLM Top 10 +FS-57,ADVISORY: Output Encoding — Manual Review Required,Output encoding practices cannot be verified via AWS APIs. Manual code review required.,"1. HTML-encode GenAI outputs before rendering in web UIs. +2. Use parameterized queries when GenAI output is used in database operations. +3. JSON-encode outputs before embedding in JavaScript contexts. +4. Validate output length and format before passing to downstream APIs.",https://genai.owasp.org/llm-top-10/,Informational,N/A,NYDFS 500.06 | FFIEC CAT | OWASP LLM Top 10 +FS-58,Output Schema Validation — Review Required,Found 1 potential schema validation function(s). Verify structured output validation is implemented for all GenAI responses.,"1. Use Bedrock structured output (response schemas) where supported. +2. Implement JSON schema validation on Lambda output processors. +3. Reject malformed outputs and return safe error responses. +4. Log schema validation failures to CloudWatch for monitoring.",https://docs.aws.amazon.com/bedrock/latest/userguide/inference-parameters.html,Medium,Passed,FFIEC CAT | OWASP LLM Top 10 +FS-59,No Guardrails — Topic Allowlist Not Applicable,No Bedrock Guardrails configured.,Configure guardrails with topic policies to restrict off-topic responses.,https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html,Medium,N/A,SR 11-7 | FFIEC CAT +FS-60,ADVISORY: Contextual Grounding for Off-Topic Prevention,Contextual grounding for off-topic prevention is covered by guardrail grounding checks (FS-47) and RAG configuration (FS-48). Additionally verify system prompts explicitly scope the assistant's role.,"1. Include explicit scope instructions in system prompts. +2. Use Bedrock Guardrails relevance grounding filter. +3. Test with off-topic prompts in QA to verify rejection behavior.",https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html,Informational,N/A,SR 11-7 | FFIEC CAT +FS-61,No Automated KB Sync Schedules Detected,"Found 2 Knowledge Base(s) but no EventBridge Scheduler schedules or EventBridge rules with 'bedrock'/'knowledge' naming were found. Note: this check uses a name/target heuristic — sync automation with other naming conventions, AWS Step Functions-based orchestration, or native Bedrock API-triggered syncs (StartIngestionJob called directly) will not be detected. Verify sync automation manually if applicable.","1. Use EventBridge Scheduler (the AWS-recommended approach) to create a recurring schedule (e.g., rate(1 day) or a cron expression) that triggers a Lambda function calling the Bedrock StartIngestionJob API for each data source. Classic EventBridge scheduled rules also work but are a legacy feature. +2. As of December 2024, Bedrock Knowledge Bases supports custom connectors and streaming data ingestion — use direct document ingestion (KnowledgeBaseDocuments API) for real-time updates without a full S3 sync. +3. Set sync frequency based on data currency requirements (e.g., hourly for market data, daily for regulatory guidance). +4. Configure CloudWatch alarms or SNS notifications on IngestionJob FAILED status for sync failure alerting.",https://docs.aws.amazon.com/scheduler/latest/UserGuide/what-is-scheduler.html,Medium,Failed,SR 11-7 | FFIEC CAT +FS-62,ADVISORY: Data Currency Disclaimer — Manual Review Required,Data currency disclaimers cannot be verified via AWS APIs. Manual review required.,"1. Add data currency disclaimers to GenAI outputs: 'Information based on data current as of [KB last sync date].' +2. Expose KB last sync timestamp in application responses. +3. Alert users when KB data is older than defined threshold.",https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base-ingest.html,Informational,N/A,SR 11-7 | FFIEC CAT | MAS TRM 9.2 +FS-63,Legacy Models Without Lifecycle Management,"Legacy foundation models available: anthropic.claude-sonnet-4-20250514-v1:0, amazon.titan-image-generator-v2:0, amazon.nova-premier-v1:0:8k, amazon.nova-premier-v1:0:20k, amazon.nova-premier-v1:0:1000k. No Config rules found for model lifecycle management.","1. Create a model lifecycle management process. +2. Subscribe to AWS Bedrock model deprecation notifications. +3. Test and migrate to new model versions before deprecation dates. +4. Document training data cutoff dates in model inventory.",https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle.html,Medium,Failed,SR 11-7 | FFIEC CAT | ISO 27001 A.12 +FS-65,KB Data Source References a Deleted S3 Bucket,"One or more Knowledge Base data sources point to S3 buckets that no longer exist (NoSuchBucket). Document-change monitoring cannot be assessed and KB grounding data is missing for these sources: +- fedpoint-kms-poc-content-469898429403 (KB 'fedpoint-kms-poc-knowledge-base', source 'fedpoint-kms-poc-content-data-source')","1. Investigate why the data-source bucket was deleted (accidental deletion, environment teardown, or stale KB configuration). +2. Recreate/restore the bucket (with event notifications and versioning enabled), or remove the orphaned data source from the Knowledge Base. +3. Re-run a KB ingestion job after restoring the data source.",https://docs.aws.amazon.com/AmazonS3/latest/userguide/EventBridge.html,High,Failed,FFIEC CAT | DORA Art.6 | ISO 27001 A.12 +FS-66,AgentCore Runtimes Missing End-User Identity Propagation,"The following runtimes have no JWT or IAM authorizer configured for end-user identity propagation. Tool calls are authorized only by the agent execution role, not the originating user: +- vendoriq_vendoriq +- test_minimal +- strands_claude_short_timeout +- strands_claude_getting_started +- proposal_drafter +- harness_harness_ctuux +- executive_summary_writer +- document_generation +- datetimemcp_Agent","1. Configure a custom JWT authorizer or IAM authorizer on each AgentCore runtime. +2. Propagate the end-user's identity token to downstream tool services. +3. Ensure tool services validate the propagated identity before executing actions. +4. Do not expose propagated identity tokens to unauthorized third parties.",https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/security-authorization.html,High,Failed,NYDFS 500 | SR 11-7 | MAS TRM 9 | PCI-DSS +FS-67,Agent Action-Group Lambdas May Lack Transaction Thresholds,"The following agent action-group Lambda functions have no environment variables whose names suggest transaction-value threshold configuration (this is a best-effort heuristic — a threshold enforced in code or in an AgentCore Policy Engine rule would not be detected here, so treat this as a prompt for manual verification rather than a definitive gap). Without explicit limits, agents could initiate unbounded financial transactions: +- Dev-DashboardStack-DashboardApiLayerAgentSessionsF-EnQLRGDeLRkn +- fedpoint-kms-poc-create-bedrock-index +- Dev-DashboardStack-DashboardApiLayerAgentInvokeFn3-r3fQdrJuboTX","1. Add transaction-value threshold environment variables (e.g., MAX_TRANSACTION_AMOUNT) to each agent action-group Lambda. +2. Implement threshold enforcement logic in the Lambda handler. +3. Configure AgentCore Policy Engine rules to cap financial transaction amounts. +4. Route transactions exceeding thresholds to a human-in-the-loop approval step.",https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/policy-engine.html,High,Failed,SR 11-7 | FFIEC CAT | MAS TRM 9 | PCI-DSS +FS-68,API Gateway Request Body Size Limits Not Enforced,"Input payload size limits are not fully enforced on GenAI API endpoints. Oversized prompts can exhaust Bedrock token quotas and inflate costs: +- REST APIs without request validators: VendorIQ MuleSoft API, VendorIQ Dashboard API, IsbRestApi","1. Add API Gateway request validators to enforce maximum body size on all Bedrock-facing REST API methods. +2. Add a WAF SizeConstraintStatement rule to block requests with body size exceeding your maximum prompt length (e.g., 32 KB). +3. Set the max_tokens parameter in Bedrock API calls to cap output length. +4. Implement client-side token counting before submitting requests.",https://docs.aws.amazon.com/waf/latest/developerguide/waf-rule-statement-type-size-constraint.html,Medium,Failed,DORA Art.6 | FFIEC CAT | PCI-DSS | OWASP LLM Top 10 +FS-69,Prompt Input Validation Functions Present,"Found 3 Lambda function(s) with input validation/sanitization naming patterns: ISB-InitializeCleanupLambda-myisb, fedpoint-kms-poc-user-api-clean, fedpoint-compliance-validation.","Review these functions to confirm they cover: special-character stripping, format validation, size limits, and injection-sequence detection.",https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-injection.html,Informational,Passed,NYDFS 500 | FFIEC CAT | OWASP LLM Top 10 From 5c17c6f59c0bccb6c4dbcaf282ae68aa3b5c6d3f Mon Sep 17 00:00:00 2001 From: Neil Mehta Date: Mon, 8 Jun 2026 09:59:19 -0400 Subject: [PATCH 10/16] fix(finserv): restore documented severity on all Passed findings (mirrors 1bfa472) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FinServ app.py independently introduced the same bug fixed upstream in commit 1bfa472 ("Fix 0% pass rate by retaining check severity on passed findings"). All ~60 Passed findings in finserv_assessments/app.py were hardcoded to severity="Informational", which the report template excludes from both the actionable-findings denominator and the passed-count numerator — producing artificially low pass rates in the generated reports. This commit applies the same fix pattern as 1bfa472: Passed findings now carry the check's documented severity (High/Medium/Low) instead of Informational. Changes: - 55 create_finding(status="Passed", severity="Informational") call sites updated to severity= across 53 unique FS check IDs. - N/A findings (Informational, "no resources found") — UNCHANGED (21 total). - Advisory checks (FS-29/32/37/49/54/57/60/62, ADVISORY: prefix, N/A+Info) — UNCHANGED (correct by design, D1 advisory convention from pr-review-fixes). - 5 already-correct Passed findings (FS-24/51/56/58/59) — UNCHANGED. Severity map derived from each check's own Failed/WARN branch (primary source) and cross-checked against docs/SECURITY_CHECKS_FINSERV_PART1-3.md. 6 checks with all-Informational non-Passed branches (FS-31/34/52/61/68/69) resolved to Medium via direct Failed-branch code inspection. Verification (post-fix): - Passed+Informational remaining: 0 (was 55) - Passed severity distribution: High=22, Medium=35, Low=3 - N/A+Informational unchanged: 21 - ruff check: All checks passed - ruff format --check: 1 file already formatted - sam build: Build Succeeded (all 7 Lambda functions) - sam validate --lint (both templates): valid SAM Template - cfn-lint (all 4 templates): exit 0 - pytest tests/ -q: 368 passed in 0.70s Fixes feedback from Vivek Mittal (2026-06-05): "Informational severity are coming as status Passed instead of N/A. Several existing checks which were earlier Medium/Low/High seems to be changed to Informational as well." --- .../security/finserv_assessments/app.py | 110 +++++++++--------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/aiml-security-assessment/functions/security/finserv_assessments/app.py b/aiml-security-assessment/functions/security/finserv_assessments/app.py index b875c0f..7e8e4df 100644 --- a/aiml-security-assessment/functions/security/finserv_assessments/app.py +++ b/aiml-security-assessment/functions/security/finserv_assessments/app.py @@ -414,7 +414,7 @@ def check_waf_shield_on_bedrock_endpoints() -> Dict[str, Any]: finding_details="AWS Shield Advanced subscription is active.", resolution="No action required.", reference="https://docs.aws.amazon.com/waf/latest/developerguide/shield-chapter.html", - severity="Informational", + severity="High", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-01"], ) @@ -449,7 +449,7 @@ def check_waf_shield_on_bedrock_endpoints() -> Dict[str, Any]: finding_details=f"Found {len(acls)} regional WAF Web ACL(s).", resolution="Verify ACLs are associated with Bedrock-facing endpoints.", reference="https://docs.aws.amazon.com/waf/latest/developerguide/waf-chapter.html", - severity="Informational", + severity="High", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-01"], ) @@ -519,7 +519,7 @@ def check_api_gateway_rate_limiting() -> Dict[str, Any]: finding_details=f"All {len(plans)} usage plan(s) have throttle settings.", resolution="No action required.", reference="https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-request-throttling.html", - severity="Informational", + severity="High", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-02"], ) @@ -652,7 +652,7 @@ def check_bedrock_token_quotas() -> Dict[str, Any]: ), resolution="No action required. Periodically re-review quotas against expected peak load.", reference="https://docs.aws.amazon.com/bedrock/latest/userguide/quotas.html", - severity="Informational", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-03"], ) @@ -781,7 +781,7 @@ def check_cost_anomaly_detection() -> Dict[str, Any]: finding_details=f"Found {len(monitors)} anomaly monitor(s); {len(bedrock_monitors)} provide Bedrock/SageMaker service-level coverage.", resolution="Verify monitors cover Bedrock and SageMaker service dimensions.", reference="https://docs.aws.amazon.com/cost-management/latest/userguide/getting-started-ad.html", - severity="Informational", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-04"], ) @@ -852,7 +852,7 @@ def check_cloudwatch_token_alarms() -> Dict[str, Any]: ), resolution="Ensure alarms have SNS actions and are in OK state.", reference="https://docs.aws.amazon.com/bedrock/latest/userguide/monitoring-cw.html", - severity="Informational", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-05"], ) @@ -939,7 +939,7 @@ def _paginate_budgets(show_filter_expression: bool): finding_details=f"Found {len(aiml_budgets)} budget(s) covering AI/ML services.", resolution="No action required.", reference="https://docs.aws.amazon.com/cost-management/latest/userguide/budgets-managing-costs.html", - severity="Informational", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-06"], ) @@ -1048,7 +1048,7 @@ def check_bedrock_agent_action_boundaries(permission_cache) -> Dict[str, Any]: finding_details=f"Reviewed {len(agents)} agent(s); no wildcard sensitive actions found.", resolution="No action required.", reference="https://docs.aws.amazon.com/bedrock/latest/userguide/agents-permissions.html", - severity="Informational", + severity="High", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-07"], ) @@ -1137,7 +1137,7 @@ def check_agentcore_policy_engine() -> Dict[str, Any]: finding_details=f"All {len(runtimes)} runtime(s) have authorizer configurations.", resolution="No action required.", reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/security-authorization.html", - severity="Informational", + severity="High", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-08"], ) @@ -1209,7 +1209,7 @@ def check_agent_transaction_limits() -> Dict[str, Any]: finding_details=f"Reviewed {len(agent_lambdas)} agent Lambda(s); concurrency limits appear configured.", resolution="No action required.", reference="https://docs.aws.amazon.com/lambda/latest/dg/configuration-concurrency.html", - severity="Informational", + severity="Medium", status="Passed" if agent_lambdas else "N/A", compliance_frameworks=COMPLIANCE_MAP["FS-09"], ) @@ -1275,7 +1275,7 @@ def check_human_in_the_loop_for_high_risk_actions() -> Dict[str, Any]: finding_details=f"State machines with waitForTaskToken (human approval): {', '.join(machines_with_wait)}.", resolution="No action required. Verify approval routing reaches the correct reviewers.", reference="https://docs.aws.amazon.com/step-functions/latest/dg/connect-to-resource.html#connect-wait-token", - severity="Informational", + severity="High", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-10"], ) @@ -1356,7 +1356,7 @@ def check_agent_rate_alarms() -> Dict[str, Any]: finding_details=f"Found {len(agent_alarms)} agent-related CloudWatch alarm(s).", resolution="No action required.", reference="https://docs.aws.amazon.com/bedrock/latest/userguide/monitoring-cw.html", - severity="Informational", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-11"], ) @@ -1442,7 +1442,7 @@ def check_scp_model_access_restrictions() -> Dict[str, Any]: finding_details=f"SCPs referencing Bedrock: {', '.join(bedrock_scps)}.", resolution="Verify SCPs use bedrock:ModelId conditions to allowlist approved models.", reference="https://docs.aws.amazon.com/bedrock/latest/userguide/security_iam_id-based-policy-examples.html", - severity="Informational", + severity="High", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-12"], ) @@ -1517,7 +1517,7 @@ def check_model_inventory_tagging() -> Dict[str, Any]: finding_details="All reviewed models have required provenance tags.", resolution="No action required.", reference="https://docs.aws.amazon.com/bedrock/latest/userguide/tagging.html", - severity="Informational", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-13"], ) @@ -1574,7 +1574,7 @@ def check_model_onboarding_governance() -> Dict[str, Any]: finding_details=f"Found {len(bedrock_rules)} model-related Config rule(s).", resolution="No action required.", reference="https://docs.aws.amazon.com/config/latest/developerguide/evaluate-config.html", - severity="Informational", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-14"], ) @@ -1623,7 +1623,7 @@ def check_bedrock_model_evaluation_adversarial() -> Dict[str, Any]: finding_details=f"Found {len(evals)} evaluation job(s). Verify adversarial datasets are included.", resolution="Ensure evaluation datasets include adversarial/red-team test cases.", reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", - severity="Informational", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-15"], ) @@ -1693,7 +1693,7 @@ def check_ecr_image_scanning() -> Dict[str, Any]: finding_details=f"All {len(repos)} ECR repo(s) have scan-on-push enabled.", resolution="No action required.", reference="https://docs.aws.amazon.com/AmazonECR/latest/userguide/image-scanning.html", - severity="Informational", + severity="High", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-16"], ) @@ -1776,7 +1776,7 @@ def check_feature_store_rollback_capability() -> Dict[str, Any]: finding_details=f"All {len(groups)} feature group(s) have active offline stores.", resolution="No action required.", reference="https://docs.aws.amazon.com/sagemaker/latest/dg/feature-store-offline.html", - severity="Informational", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-20"], ) @@ -1867,7 +1867,7 @@ def check_training_data_s3_versioning() -> Dict[str, Any]: finding_details=f"All {len(training_buckets)} training bucket(s) have versioning enabled.", resolution="No action required.", reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/Versioning.html", - severity="Informational", + severity="High", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-21"], ) @@ -1941,7 +1941,7 @@ def check_knowledge_base_iam_least_privilege(permission_cache) -> Dict[str, Any] finding_details="No wildcard KB permissions found in reviewed roles.", resolution="No action required.", reference="https://docs.aws.amazon.com/bedrock/latest/userguide/security-iam-awsmanpol.html", - severity="Informational", + severity="High", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-22"], ) @@ -2085,7 +2085,7 @@ def check_opensearch_serverless_encryption() -> Dict[str, Any]: ), resolution="Verify all vector store collections use customer-managed KMS keys.", reference="https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-encryption.html", - severity="Informational", + severity="High", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-25"], ) @@ -2163,7 +2163,7 @@ def check_knowledge_base_vpc_access() -> Dict[str, Any]: finding_details=f"{len(vpc_policies)} network policy(ies) restrict to VPC.", resolution="No action required.", reference="https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-network.html", - severity="Informational", + severity="High", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-26"], ) @@ -2265,7 +2265,7 @@ def check_guardrail_contextual_grounding() -> Dict[str, Any]: "Also consider enabling Automated Reasoning checks for formal policy verification." ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", - severity="Informational", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-27"], ) @@ -2377,7 +2377,7 @@ def check_automated_reasoning_policies() -> Dict[str, Any]: "guardrail is applied to your Bedrock inference calls." ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/automated-reasoning.html", - severity="Informational", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-27"], ) @@ -2497,7 +2497,7 @@ def check_guardrail_denied_topics_financial() -> Dict[str, Any]: "documents, and incident reports (as recommended in PDF \u00a71.2.1 Practical guidance)." ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", - severity="Informational", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-28"], ) @@ -2575,7 +2575,7 @@ def check_bedrock_evaluation_compliance_datasets() -> Dict[str, Any]: finding_details=f"Found {len(evals)} evaluation job(s). Verify compliance datasets are included.", resolution="Ensure evaluation datasets include FinServ regulatory test cases.", reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", - severity="Informational", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-30"], ) @@ -2687,7 +2687,7 @@ def check_knowledge_base_data_source_sync() -> Dict[str, Any]: ), resolution="No action required.", reference="https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base-ingest.html", - severity="Informational", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-31"], ) @@ -2867,7 +2867,7 @@ def check_knowledge_base_integrity_monitoring() -> Dict[str, Any]: finding_details="All reviewed KB data source buckets have versioning enabled.", resolution="No action required.", reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html", - severity="Informational", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-33"], ) @@ -2933,7 +2933,7 @@ def check_fm_version_currency() -> Dict[str, Any]: finding_details="No legacy/deprecated foundation models detected.", resolution="No action required.", reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle.html", - severity="Informational", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-34"], ) @@ -2987,7 +2987,7 @@ def check_fmeval_harmful_content() -> Dict[str, Any]: finding_details=f"Found {len(evals)} evaluation job(s). Verify harmful content datasets are included.", resolution="Ensure evaluation includes toxicity and harmful content test cases.", reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", - severity="Informational", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-35"], ) @@ -3103,7 +3103,7 @@ def check_guardrail_content_filters() -> Dict[str, Any]: ), resolution="No action required.", reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-filters.html", - severity="Informational", + severity="High", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-36"], ) @@ -3209,7 +3209,7 @@ def check_guardrail_word_filters() -> Dict[str, Any]: finding_details=f"Guardrails with word filters: {', '.join(guardrails_with_words)}.", resolution="No action required.", reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", - severity="Informational", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-38"], ) @@ -3268,7 +3268,7 @@ def check_sagemaker_clarify_bias() -> Dict[str, Any]: finding_details=f"Found {len(bias_schedules)} model bias monitoring schedule(s).", resolution="No action required.", reference="https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-model-monitor-bias-drift.html", - severity="Informational", + severity="High", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-39"], ) @@ -3315,7 +3315,7 @@ def check_bedrock_evaluation_bias_datasets() -> Dict[str, Any]: finding_details=f"Found {len(evals)} evaluation job(s). Verify bias datasets are included.", resolution="Ensure evaluation includes demographic fairness test cases.", reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", - severity="Informational", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-40"], ) @@ -3373,7 +3373,7 @@ def check_sagemaker_clarify_explainability() -> Dict[str, Any]: finding_details=f"Found {len(explainability_schedules)} explainability monitoring schedule(s).", resolution="No action required.", reference="https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-model-explainability.html", - severity="Informational", + severity="High", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-41"], ) @@ -3424,7 +3424,7 @@ def check_ai_service_cards_documentation() -> Dict[str, Any]: finding_details=f"Found {len(model_cards)} model card(s).", resolution="Verify cards are current and include bias/fairness evaluations.", reference="https://docs.aws.amazon.com/sagemaker/latest/dg/model-cards.html", - severity="Informational", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-42"], ) @@ -3488,7 +3488,7 @@ def check_cloudwatch_log_pii_masking() -> Dict[str, Any]: finding_details=f"Found {len(policies)} data protection policy(ies).", resolution="Verify policies cover Bedrock invocation log groups.", reference="https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/mask-sensitive-log-data.html", - severity="Informational", + severity="High", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-43"], ) @@ -3543,7 +3543,7 @@ def check_macie_on_training_data_buckets() -> Dict[str, Any]: finding_details="Amazon Macie is enabled and scanning S3 buckets.", resolution="Verify Macie jobs cover training data and KB data source buckets.", reference="https://docs.aws.amazon.com/macie/latest/user/what-is-macie.html", - severity="Informational", + severity="High", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-44"], ) @@ -3619,7 +3619,7 @@ def check_guardrail_pii_filters() -> Dict[str, Any]: finding_details=f"Guardrails with PII filters: {', '.join(guardrails_with_pii)}.", resolution="No action required.", reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-sensitive-filters.html", - severity="Informational", + severity="High", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-45"], ) @@ -3712,7 +3712,7 @@ def check_data_classification_tagging() -> Dict[str, Any]: finding_details=f"All {len(aiml_buckets)} AI/ML bucket(s) have classification tags.", resolution="No action required.", reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html", - severity="Informational", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-46"], ) @@ -3834,7 +3834,7 @@ def check_guardrail_grounding_threshold() -> Dict[str, Any]: ), resolution="No action required.", reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-contextual-grounding-check.html", - severity="Informational", + severity="High", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-47"], ) @@ -3890,7 +3890,7 @@ def check_rag_knowledge_base_configured() -> Dict[str, Any]: finding_details=f"Found {len(active_kbs)} active Knowledge Base(s) for RAG grounding.", resolution="No action required.", reference="https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base.html", - severity="Informational", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-48"], ) @@ -3995,7 +3995,7 @@ def check_guardrail_relevance_grounding() -> Dict[str, Any]: ), resolution="No action required.", reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", - severity="Informational", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-50"], ) @@ -4111,7 +4111,7 @@ def check_prompt_injection_input_validation() -> Dict[str, Any]: "filters to evaluate user input separately from system prompts." ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-prompt-attack.html", - severity="Informational", + severity="High", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-51"], ) @@ -4232,7 +4232,7 @@ def check_bedrock_sdk_version_currency() -> Dict[str, Any]: finding_details=f"All {len(bedrock_functions)} Bedrock Lambda function(s) use current runtimes.", resolution="No action required.", reference="https://docs.aws.amazon.com/lambda/latest/dg/runtimes-update.html", - severity="Informational", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-52"], ) @@ -4320,7 +4320,7 @@ def check_waf_sql_injection_rules() -> Dict[str, Any]: finding_details=f"All {len(acls)} WAF ACL(s) have injection protection rules.", resolution="No action required.", reference="https://docs.aws.amazon.com/waf/latest/developerguide/aws-managed-rule-groups-baseline.html", - severity="Informational", + severity="High", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-53"], ) @@ -4427,7 +4427,7 @@ def check_output_validation_lambda() -> Dict[str, Any]: finding_details=f"Found {len(validation_functions)} output validation/sanitization function(s).", resolution="No action required.", reference="https://genai.owasp.org/llm-top-10/", - severity="Informational", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-55"], ) @@ -4657,7 +4657,7 @@ def check_guardrail_topic_allowlist() -> Dict[str, Any]: finding_details=f"Guardrails with topic policies: {', '.join(guardrails_with_topics)}.", resolution="No action required.", reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", - severity="Informational", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-59"], ) @@ -4814,7 +4814,7 @@ def check_knowledge_base_sync_schedule() -> Dict[str, Any]: ), resolution="Verify the schedule frequency matches your data-currency requirements.", reference="https://docs.aws.amazon.com/scheduler/latest/UserGuide/what-is-scheduler.html", - severity="Informational", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-61"], ) @@ -4918,7 +4918,7 @@ def check_foundation_model_lifecycle_policy() -> Dict[str, Any]: ), resolution="No action required.", reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle.html", - severity="Informational", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-63"], ) @@ -5087,7 +5087,7 @@ def check_kb_datasource_s3_event_notifications() -> Dict[str, Any]: finding_details="All KB data-source S3 buckets have event notifications configured.", resolution="No action required.", reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/EventBridge.html", - severity="Informational", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-65"], ) @@ -5183,7 +5183,7 @@ def check_agentcore_end_user_identity_propagation() -> Dict[str, Any]: finding_details=f"All {len(runtimes)} runtime(s) have authorizer configurations supporting identity propagation.", resolution="No action required.", reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/security-authorization.html", - severity="Informational", + severity="Low", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-66"], ) @@ -5299,7 +5299,7 @@ def check_agent_financial_transaction_thresholds() -> Dict[str, Any]: ), resolution="Verify threshold values are appropriate for your financial risk tolerance.", reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/policy-engine.html", - severity="Informational", + severity="High", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-67"], ) @@ -5389,7 +5389,7 @@ def check_api_gateway_request_body_size_limits() -> Dict[str, Any]: ), resolution="No action required.", reference="https://docs.aws.amazon.com/waf/latest/developerguide/waf-rule-statement-type-size-constraint.html", - severity="Informational", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-68"], ) @@ -5472,7 +5472,7 @@ def check_prompt_input_validation_function() -> Dict[str, Any]: "format validation, size limits, and injection-sequence detection." ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-injection.html", - severity="Informational", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-69"], ) From 00c684ad88ea7290d968bf05e1f8124c1e8f0038 Mon Sep 17 00:00:00 2001 From: Neil Mehta Date: Thu, 11 Jun 2026 17:20:11 -0400 Subject: [PATCH 11/16] feat(finserv): round-3 correctness fixes, severity register, and shared-inventory refactor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses PR #23 round-3 review findings plus the pre-publish audit (REQ-2..REQ-15) in the FinServ assessment Lambda, and bundles the behavior-preserving shared-inventory refactor (the two live in the same regions of app.py, e.g. FS-68/FS-61, so they cannot be cleanly separated into independent commits). Correctness: - FS-58: retag to advisory (N/A / Informational) instead of hardcoded Passed. - FS-22: normalize single-statement-dict IAM policies (crash fix) and widen over-permissive detection (partial wildcards, Resource:"*", NotAction). - FS-68: evidence-based body-size disposition (WAF SizeConstraint that can fire, or a validator model with maxLength) — validator presence alone no longer Passes. - FS-61: AccessDenied on scheduler/events now yields COULD_NOT_ASSESS, not a false Failed. - FS-27: clearer ARC access-denied guidance (re-deploy role / SCP / region); region text adds AWS GovCloud (US) and points to current AWS docs. - FS-15 fails when no eval jobs exist; FS-30/35/40 become advisory; FS-56 gains a real XSS FAIL path; CLASSIC-tier guardrails stay Passed at control severity. Severity methodology (REQ-6): - SEVERITY_REGISTER keyed by finding-name; disposition rules (N/A->Informational, COULD_NOT_ASSESS->Low); FS-01 Shield->Low, WAF->Medium. - Drift-guard test asserts emitted severity == register and no Critical. Shared-inventory refactor (REQ-13 C3 / FU-3): - collect_resource_inventory() gathers buckets/functions/guardrails/KBs/web-ACLs once per run, injected like permission_cache; collapses the 4x/6x/9x duplicate sweeps. - Fixes latent WAFv2 (NextMarker) and S3 (ContinuationToken) pagination truncation — the only sanctioned finding-set change (accounts with >1 page now fully assessed). - Golden-equivalence baseline + at-most-once + resilience + large-estate tests. Tests: ported the full suite to functions/security/finserv_tests/ (CI-discoverable); 568 passing including the drift-guard. No new IAM actions introduced. sim: https://github.com/aws-samples/sample-aiml-security-assessment/pull/23 --- FOLLOWUPS.md | 196 + .../security/finserv_assessments/app.py | 1604 ++++++-- .../security/finserv_tests/__init__.py | 1 + .../security/finserv_tests/conftest.py | 135 + .../finserv_tests/test_at_most_once.py | 813 ++++ .../security/finserv_tests/test_checks.py | 3577 +++++++++++++++++ .../finserv_tests/test_checks_coverage.py | 1993 +++++++++ .../security/finserv_tests/test_collector.py | 685 ++++ .../test_inventory_equivalence.py | 1093 +++++ .../finserv_tests/test_inventory_model.py | 143 + .../finserv_tests/test_lambda_handler.py | 436 ++ .../finserv_tests/test_large_estate.py | 456 +++ .../security/finserv_tests/test_paginate.py | 436 ++ .../finserv_tests/test_phase2_live.py | 286 ++ .../security/finserv_tests/test_resilience.py | 765 ++++ .../security/finserv_tests/test_schema.py | 339 ++ .../finserv_tests/test_severity_register.py | 132 + ...ITY_CHECKS_FINSERV_SEVERITY_METHODOLOGY.md | 178 + ...CURITY_CHECKS_FINSERV_SEVERITY_REGISTER.md | 345 ++ 19 files changed, 13197 insertions(+), 416 deletions(-) create mode 100644 FOLLOWUPS.md create mode 100644 aiml-security-assessment/functions/security/finserv_tests/__init__.py create mode 100644 aiml-security-assessment/functions/security/finserv_tests/conftest.py create mode 100644 aiml-security-assessment/functions/security/finserv_tests/test_at_most_once.py create mode 100644 aiml-security-assessment/functions/security/finserv_tests/test_checks.py create mode 100644 aiml-security-assessment/functions/security/finserv_tests/test_checks_coverage.py create mode 100644 aiml-security-assessment/functions/security/finserv_tests/test_collector.py create mode 100644 aiml-security-assessment/functions/security/finserv_tests/test_inventory_equivalence.py create mode 100644 aiml-security-assessment/functions/security/finserv_tests/test_inventory_model.py create mode 100644 aiml-security-assessment/functions/security/finserv_tests/test_lambda_handler.py create mode 100644 aiml-security-assessment/functions/security/finserv_tests/test_large_estate.py create mode 100644 aiml-security-assessment/functions/security/finserv_tests/test_paginate.py create mode 100644 aiml-security-assessment/functions/security/finserv_tests/test_phase2_live.py create mode 100644 aiml-security-assessment/functions/security/finserv_tests/test_resilience.py create mode 100644 aiml-security-assessment/functions/security/finserv_tests/test_schema.py create mode 100644 aiml-security-assessment/functions/security/finserv_tests/test_severity_register.py create mode 100644 docs/SECURITY_CHECKS_FINSERV_SEVERITY_METHODOLOGY.md create mode 100644 docs/SECURITY_CHECKS_FINSERV_SEVERITY_REGISTER.md diff --git a/FOLLOWUPS.md b/FOLLOWUPS.md new file mode 100644 index 0000000..537ab51 --- /dev/null +++ b/FOLLOWUPS.md @@ -0,0 +1,196 @@ +# Follow-up items (deferred from PR #23 Round 3) + +> GitHub Issues are disabled on the fork `mehtadman87/sample-aiml-security-assessment`, so +> deferred work is tracked here in-repo (task T1b.7). Promote to an upstream issue/PR when ready. + +## FU-1 — Evaluate adopting a tool-wide `CRITICAL` severity tier + +**Status:** Deferred (intentionally out of scope for Round 3). + +**Background.** The FinServ severity methodology +([`docs/SECURITY_CHECKS_FINSERV_SEVERITY_METHODOLOGY.md`](./SECURITY_CHECKS_FINSERV_SEVERITY_METHODOLOGY.md)) +defines a Likelihood × Impact matrix mapped to the AWS Security Hub ASFF label set, which includes a +**CRITICAL** band (the Impact=High × Likelihood=High cell). Round 3 **capped that cell at High** and +did **not** introduce `Critical`, to keep the FinServ checks consistent with the upstream +Bedrock/SageMaker/AgentCore checks (which have no Critical tier). The drift-guard +(`tests/test_severity_register.py`) asserts no `Critical` is currently emitted. + +**Why it is a separate item.** Adopting `Critical` is a **tool-wide** change. Doing it for FinServ +alone would make FinServ inconsistent with the other three services. A tool-wide rollout touches: +- the shared `SeverityEnum` (each package's `schema.py`); +- all four assessment Lambdas (re-score the I=High × L=High controls); +- the report template severity filters/colors (`generate_consolidated_report/report_template.py` + and `consolidate_html_reports.py`); +- every service's unit tests plus the FinServ drift-guard. + +**Decision needed.** +1. Do we want a `CRITICAL` tier across the whole assessment? +2. If yes, which controls qualify (e.g., full guardrail bypass enabling a regulatory breach; + unauthorized high-value autonomous financial action)? + +**References:** methodology §2 (label scale) and §6 (the deferred decision); ASFF severity — +https://docs.aws.amazon.com/securityhub/1.0/APIReference/API_Severity.html + +--- + +## FU-2 — Report UI design for FinServ / OWASP (REQ-8) + +Broader report UI re-design (top-page FinServ summary box placement, OWASP grouping, Risk +Distribution treatment) is deferred to a separate PR per the reviewer's suggestion. Round 3 delivers +FinServ as a functionally first-class service (REQ-1, Wave 2); the UI-design discussion is tracked +here for a follow-up PR. + +--- +--- +--- + +## FU-3 — Shared-inventory refactor for the FinServ assessment Lambda (REQ-13 C3) + +**Status:** Deferred from Round 3.1 (Wave 5.5 task T5h.9). Classified **Should-Fix** (performance / +scalability optimization), not a correctness blocker. Tracked here for a focused follow-up PR. + +### Background + +The FinServ Lambda (`finserv_assessments/app.py`) runs all 64 checks **sequentially in a single +invocation** via `build_finserv_checks()`. There is no cross-check caching, so several checks +independently re-enumerate the **same** account-wide inventories, and many then issue an N+1 +per-resource call on top. The Round-3.1 pre-release audit (the per-check AWS API inventory) +identified the following duplicate full-account sweeps within one run: + +| Inventory (API) | Times enumerated | Checks performing the enumeration | Per-resource N+1 follow-up | +| --- | --- | --- | --- | +| `lambda:ListFunctions` | ~6× | FS-09, FS-52, FS-55, FS-58, FS-67, FS-69 | FS-09 `GetFunctionConcurrency` per function | +| `bedrock:ListGuardrails` | ~9× | FS-27a, FS-28, FS-36, FS-38, FS-45, FS-47, FS-50, FS-51, FS-59 | `GetGuardrail` per guardrail in each check | +| `bedrock:ListKnowledgeBases` | ~6× | FS-24, FS-31, FS-33, FS-48, FS-61, FS-65 | FS-31/33/65 `ListDataSources`/`GetDataSource` per KB | +| `s3:ListBuckets` (full account) | ~3–4× | FS-21, FS-46 (full sweep); FS-33, FS-65 (KB data-source buckets) | `GetBucketVersioning`/`GetBucketTagging`/`GetBucketNotification` per bucket | +| `wafv2:ListWebACLs` | 4× | FS-01, FS-53, FS-56, FS-68 | `GetWebACL` per ACL in each check | + +In small/empty accounts this is harmless (it is why the Wave-5 test account ran in ~3 minutes). In +**large enterprise accounts** — thousands of Lambda functions, hundreds of S3 buckets, many +guardrails/KBs/ACLs — the repeated full sweeps plus N+1 calls multiply API volume, drive adaptive +retries/throttling (`Config(retries=adaptive)`), and inflate wall-clock time. This is the most +likely driver of a slow run, and previously of a `States.Timeout`. + +### Why it was deferred (and why that is safe) + +1. **It is a large, regression-prone refactor.** It touches ~20 check functions and the way they + obtain AWS data, plus every corresponding mocked unit test (the per-check tests patch + `app.boto3.client`). Bundling that into a correctness-focused PR that "must be 100% sure" + (the reviewer's bar) trades correctness risk for a performance gain. +2. **The blast-radius risk it is associated with (audit C2) is already mitigated in Round 3.1.** + A `Catch [States.ALL] → "FinServ Assessment Incomplete"` was added to the FinServ task in + `statemachine/assessments.asl.json`, and the FinServ Lambda `Timeout` was raised 600 → 900 s. + So even if the FinServ Lambda is slow or times out in a very large account, the consolidated + report is still generated with the other services' findings and a visible incomplete marker — + the failure no longer sinks the whole assessment. +3. **It is purely an optimization** (Should-Fix). The disposition logic and severities are unchanged + by this work; nothing about the *correctness* of any finding depends on it. + +### Who is affected + +Customers running the FinServ assessment (`EnableFinServAssessment=true`) against accounts with large +resource estates — primarily large enterprises. Symptoms: long FinServ Lambda duration, CloudWatch +throttling/retry noise, and (pre-mitigation) the risk of a 900 s timeout. Small accounts are +unaffected. + +### Proposed design + +Collect each shared inventory **once** at the start of `lambda_handler`, into a read-only context +object, and pass it to the checks that need it — mirroring the existing `permission_cache` pattern +(which is already injected via `functools.partial` in `build_finserv_checks()`). + +1. **Add a `ResourceInventory` collected once per invocation**, e.g.: + + ```python + @dataclass + class ResourceInventory: + lambda_functions: list # lambda:ListFunctions (+ concurrency map) + guardrails: list # bedrock:ListGuardrails + GetGuardrail detail, keyed by id + knowledge_bases: list # bedrock:ListKnowledgeBases (+ data sources per KB) + buckets: list # s3:ListBuckets + web_acls: list # wafv2:ListWebACLs + GetWebACL detail (REGIONAL) + ``` + + Each field is populated by one paginated enumeration (reuse `_paginate`). Per-resource detail + (e.g., `GetGuardrail`, `GetWebACL`) is fetched once and stored alongside, eliminating the N+1 + repetition across checks. + +2. **Inject it like `permission_cache`.** In `build_finserv_checks(permission_cache, inventory)`, + bind the inventory to the relevant checks with `functools.partial`, so every registry entry + stays uniformly zero-arg and the handler loop is unchanged. + +3. **Make collection resilient.** A failure (e.g., `AccessDenied`) collecting one inventory must not + abort the others or the whole run. Per-inventory collection should be wrapped so a failed + inventory yields an explicit "unavailable" sentinel; checks that depend on an unavailable + inventory then emit `COULD_NOT_ASSESS` (consistent with `_is_access_error` handling today) rather + than a false `Failed`/`Passed`. Preserve the existing per-check `try/except` safety net. + +4. **Keep region/pagination semantics identical.** Same default-region clients, same `_paginate` + token handling; this is a call-site consolidation, not a behavior change. + +### Test strategy + +- Update the affected per-check unit tests to pass a constructed `ResourceInventory` (or a partial) + instead of patching `boto3.client` for the enumeration calls. Checks that still make non-inventory + calls keep their existing mocks. +- Add tests for the collector itself: one enumeration per inventory; pagination; and the + per-inventory failure path producing the "unavailable" sentinel (→ `COULD_NOT_ASSESS` downstream). +- **Behavior-preserving guarantee:** every existing disposition test (Passed/Failed/N/A per check) + and the severity drift-guard (`tests/test_severity_register.py`) must remain green with **no** + disposition or severity changes. That equivalence is the acceptance bar. +- Optionally add a counter/assertion (in a unit harness) proving each inventory API is called at + most once per handler invocation. + +### Acceptance criteria + +- Each shared inventory (`ListFunctions`, `ListGuardrails`+`GetGuardrail`, `ListKnowledgeBases`, + `ListBuckets`, `ListWebACLs`+`GetWebACL`) is enumerated **at most once** per FinServ run. +- No change to any finding's status or severity (all existing tests + the drift-guard pass + unchanged). +- A per-inventory collection failure degrades only the dependent checks (to `COULD_NOT_ASSESS`), not + the whole run. +- Workspace `finserv_assessments/` stays byte-identical to the fork copy after sync. + +### Risk / considerations + +- **Test isolation** if any memoization is module-level: prefer an explicit per-invocation object + passed as an argument over a module-global cache, to avoid state leaking across unit tests. +- **Partial-inventory correctness:** ensure a check that needs two inventories handles one being + unavailable independently. +- **Memory:** holding full inventories (e.g., all guardrail details) in memory is bounded and far + smaller than the permission cache already loaded; no concern at 1024 MB. + +### Effort estimate + +Roughly 1–2 focused days: ~0.5 day for the collector + injection, ~0.5–1 day updating the ~20 +affected checks and their tests, ~0.5 day validation (full suite + a large-account timing check). + +### References + +- Round-3.1 requirement: **REQ-13 (audit finding C3)** and design section "REQ-13 — Enterprise-scale + resilience & scope (audit C)" in `.kiro/specs/pr-review-round3-fixes/design.md`. +- Deferred task: **T5h.9** in `.kiro/specs/pr-review-round3-fixes/tasks.md`. +- Related mitigation already shipped in Round 3.1: **T5h.8** (ASL `Catch` on the FinServ task + + Lambda `Timeout` 600 → 900 s) addressing audit finding C2. +- Existing pattern to mirror: the `permission_cache` injection in `get_permissions_cache()` / + `build_finserv_checks()` in `finserv_assessments/app.py`. + +## FU-4 — Migrate upstream schemas from Pydantic V1 `@validator` to V2 `@field_validator` + +**Priority:** Low (tech-debt) — not a blocker. + +The upstream `schema.py` files for the Bedrock, SageMaker, AgentCore, consolidated-report, +and IAM-permission-caching Lambdas still use the deprecated Pydantic V1 `@validator` decorator, +which emits `PydanticDeprecatedSince20` warnings and will break when Pydantic V3 removes V1-style +validators. The FinServ `schema.py` already uses the V2 `@field_validator` form. + +**Why deferred:** this PR is scoped to `feature/finserv-risk-checks`. The affected files are +upstream/shared components; migrating them here would exceed the PR's scope and risk merge +conflicts with upstream. Best handled as a dedicated upstream change. + +**Scope:** swap `@validator("X")` → `@field_validator("X")` + `@classmethod`, adjust signatures, +and re-run each module's tests. + +### References + +- Identified in the pre-Wave-6 verification pass (`.kiro/specs/pr-review-round3-fixes/tasks.md`). diff --git a/aiml-security-assessment/functions/security/finserv_assessments/app.py b/aiml-security-assessment/functions/security/finserv_assessments/app.py index 7e8e4df..bdc5e1b 100644 --- a/aiml-security-assessment/functions/security/finserv_assessments/app.py +++ b/aiml-security-assessment/functions/security/finserv_assessments/app.py @@ -57,6 +57,7 @@ import json import logging import os +from dataclasses import dataclass from datetime import datetime, timezone from io import StringIO from typing import Any, Dict, List, Optional @@ -112,7 +113,12 @@ def _bucket_name_from_arn(bucket_arn: str) -> str: def _paginate( - client, operation_name: str, result_key: str, **kwargs + client, + operation_name: str, + result_key: str, + *, + token: "tuple[str, str] | None" = None, + **kwargs, ) -> List[Dict[str, Any]]: """Collect all items across pages for a paginated list/describe operation by calling the operation directly and following its continuation token. @@ -128,6 +134,19 @@ def _paginate( A single-page response (no continuation token) yields exactly one call, so this is a safe drop-in for previously non-paginated reads and for mocks that stub the operation with a single return value. + + Args: + token: Optional explicit ``(output_field, input_param)`` pair that + bypasses the convention table entirely. When set, ``token[0]`` is + read from each response to find the continuation value and + ``token[1]`` is the request kwarg used to send it on the next call. + Use this for operations whose token convention is not in the table, + or where the output and input names differ from every entry already + there (e.g. WAFv2 ``NextMarker``/``NextMarker`` vs Lambda's + ``NextMarker``/``Marker``). When ``None`` (the default) the + existing convention table is used unchanged — all current callers + pass no ``token`` argument and are therefore byte-for-byte identical + to before this parameter was added. """ # (output token field in response, input token kwarg on request) token_conventions = [ @@ -145,11 +164,18 @@ def _paginate( items.extend(resp.get(result_key, []) or []) next_token = None input_field = None - for out_field, in_field in token_conventions: + if token is not None: + # Explicit override: bypass the convention table. + out_field, in_field = token if resp.get(out_field): next_token = resp[out_field] input_field = in_field - break + else: + for out_field, in_field in token_conventions: + if resp.get(out_field): + next_token = resp[out_field] + input_field = in_field + break # Stop when there is no token, or if a token repeats (guards against a # mock that returns the same token every call → infinite loop). if not next_token or next_token in seen_tokens: @@ -218,6 +244,90 @@ def _is_missing_bucket_error(err: "ClientError") -> bool: # (Status="N/A") so a failed/permission-denied check does not silently vanish. COULD_NOT_ASSESS_PREFIX = "COULD NOT ASSESS: " +# --------------------------------------------------------------------------- +# ResourceInventory data model and helpers (REQ-3, REQ-4.1, REQ-6.4) +# --------------------------------------------------------------------------- + + +class _Unavailable: + """Sentinel marking an inventory field whose collection failed. Carries the + originating exception so dependent checks can reproduce the exact + COULD_NOT_ASSESS output that the check would have produced had it made the + call itself (design DD-3).""" + + __slots__ = ("error",) + + def __init__(self, error: Exception): + self.error = error + + +@dataclass(frozen=True) +class GuardrailInventory: + """Per-invocation guardrail enumeration. ``frozen=True`` prevents field + reassignment; the contained list/dict are read-only *by convention*.""" + + summaries: list # raw list_guardrails 'guardrails' entries + detail_by_id: dict # id -> get_guardrail(DRAFT) response (or _Unavailable) + + +@dataclass(frozen=True) +class KbInventory: + """Per-invocation Knowledge Base enumeration.""" + + summaries: list # knowledgeBaseSummaries + data_sources_by_kb: ( + dict # knowledgeBaseId -> list of dataSourceSummaries (or _Unavailable) + ) + data_source_detail: dict # (knowledgeBaseId, dataSourceId) -> get_data_source resp (or _Unavailable) + + +@dataclass(frozen=True) +class WebAclInventory: + """Per-invocation WAFv2 REGIONAL Web ACL enumeration.""" + + summaries: list # WebACLs list + detail_by_id: dict # Id -> get_web_acl response (or _Unavailable) + + +@dataclass(frozen=True) +class ResourceInventory: + """Per-invocation, read-only inventory of shared AWS resources. + + Each field is EITHER the collected data structure OR an ``_Unavailable`` + sentinel (carrying the original exception). Checks MUST NOT mutate the + lists/dicts stored here; the equivalence tests catch any such mutation. + + This is constructed once in ``lambda_handler`` and injected like + ``permission_cache`` (design DD-1, INV-6).""" + + lambda_functions: "list | _Unavailable" + guardrails: "GuardrailInventory | _Unavailable" + knowledge_bases: "KbInventory | _Unavailable" + buckets: "list | _Unavailable" + web_acls: "WebAclInventory | _Unavailable" + + +def inv_available(field) -> bool: + """Return True when *field* holds real data; False when it is an + ``_Unavailable`` sentinel.""" + return not isinstance(field, _Unavailable) + + +def require(inventory: "ResourceInventory | None", field_name: str): + """Return ``inventory.`` when available; raise the stored + exception when the field is ``_Unavailable``; raise ``RuntimeError`` when + ``inventory`` is ``None`` (the test-only default from design DD-2b). + + The caller's existing outer ``try/except`` turns the raised exception into + ``_error_findings`` → ``COULD_NOT_ASSESS``, matching today's behavior.""" + if inventory is None: + raise RuntimeError("resource inventory not provided") + value = getattr(inventory, field_name) + if isinstance(value, _Unavailable): + raise value.error + return value + + # --------------------------------------------------------------------------- # COMPLIANCE_MAP — per-check regulatory framework mappings # @@ -317,11 +427,289 @@ def _is_missing_bucket_error(err: "ClientError") -> bool: } +# --------------------------------------------------------------------------- +# SEVERITY METHODOLOGY (see docs/severity-methodology.md + severity-register.md) +# +# Severity = property of the CONTROL (the risk it mitigates), assigned once via a +# Likelihood (L) x Impact (I) matrix mapped to the AWS Security Hub ASFF label set, +# then applied to every Passed/Failed row of that control. The N/A family follows a +# fixed disposition rule so "nothing to assess" rows are consistent. +# +# Matrix (I x L -> label), 1=Low 2=Medium 3=High: +# I=3: L1->Medium L2->High L3->High +# I=2: L1->Low L2->Medium L3->High +# I=1: L1->Low L2->Low L3->Medium +# (Critical is intentionally NOT used in this round — see methodology §6.) +# +# Disposition -> severity: +# PASS / FAIL -> control severity (from the matrix) +# NOT_APPLICABLE -> Informational (ASFF: "no issue was found") +# ADVISORY -> Informational (no API can verify it) +# COULD_NOT_ASSESS -> Low (unknown state; re-run after fixing access) +# SOFT_WARNING -> control severity (intentional non-failing assessed state) +# --------------------------------------------------------------------------- + +_SEVERITY_MATRIX = { + (3, 1): "Medium", + (3, 2): "High", + (3, 3): "High", + (2, 1): "Low", + (2, 2): "Medium", + (2, 3): "High", + (1, 1): "Low", + (1, 2): "Low", + (1, 3): "Medium", +} + + +def _label_from_matrix(impact: int, likelihood: int) -> str: + """Map an (Impact, Likelihood) pair (each 1-3) to an ASFF severity label.""" + return _SEVERITY_MATRIX[(impact, likelihood)] + + +# Disposition tiers whose severity is fixed by the disposition, not by I x L. +_DISPOSITION_SEVERITY = { + "NOT_APPLICABLE": "Informational", + "ADVISORY": "Informational", + "COULD_NOT_ASSESS": "Low", +} + + +# Authoritative per-finding severity register (keyed by finding-name). +# Source of truth derived from docs/severity-register.md. The test suite +# (test_severity_register.py) asserts every emitted severity matches this map. +# Entries: finding_name -> (severity, disposition). I/L rationale lives in the doc. +SEVERITY_REGISTER: Dict[str, str] = { + # --- FS-01 (Shield = Low; WAF = Medium) --- + "AWS Shield Advanced Not Enabled": "Low", + "AWS Shield Advanced Enabled": "Low", + "No Regional WAF Web ACLs Found": "Medium", + "Regional WAF Web ACLs Present": "Medium", + # --- FS-02 (rate limiting = Medium) --- + "No API Gateway Usage Plans Found": "Informational", + "API Gateway Usage Plans Missing Throttle": "Medium", + "API Gateway Rate Limiting Configured": "Medium", + # --- FS-03 (token quotas = Medium) --- + "No Bedrock Token Quotas Returned": "Medium", + "Bedrock Default Quotas Unavailable — Customization Undetermined": "Medium", + "Bedrock Token Quotas Customized": "Medium", + "Bedrock Token Quotas At Default": "Medium", + # --- FS-04 --- + "No Cost Anomaly Detection Monitors": "Medium", + "Cost Anomaly Monitors Do Not Cover Bedrock/SageMaker": "Medium", + "Cost Anomaly Detection Configured": "Medium", + # --- FS-05 --- + "No Bedrock CloudWatch Alarms Found": "Medium", + "Bedrock CloudWatch Alarms Present": "Medium", + # --- FS-06 --- + "No AI/ML Service Budgets Configured": "Medium", + "AI/ML Service Budgets Configured": "Medium", + # --- FS-07 (agent action boundaries = High) --- + "Agent Action Boundary Check": "Informational", + "Bedrock Agent Overly Broad Action Permissions": "High", + "Agent Action Boundaries Look Appropriate": "High", + # --- FS-08 (AgentCore policy engine = High) --- + "AgentCore Policy Engine — Access Check": "Low", + "No AgentCore Runtimes Found": "Informational", + "AgentCore Runtimes Missing Policy Engine": "High", + "AgentCore Policy Engine Configured": "High", + # --- FS-09 --- + "Agent Lambda Functions Without Concurrency Limits": "Medium", + "Agent Lambda Concurrency Limits Present": "Medium", + # --- FS-10 (human-in-the-loop = High) --- + "Human-in-the-Loop Check — No Agent Workflows Found": "Informational", + "Human Approval Steps Found in Agent Workflows": "High", + "Agent Workflows Missing Human Approval Steps": "High", + # --- FS-11 --- + "No Agent Rate Alarms Found": "Medium", + "Agent Rate Alarms Present": "Medium", + # --- FS-12 (SCPs = High) --- + "SCP Model Access Check — Not in Organization": "Informational", + "No Bedrock-Scoped SCPs Found": "High", + "Bedrock SCPs Found": "High", + # --- FS-13 --- + "Models Missing Provenance Tags": "Medium", + "Model Provenance Tags Present": "Medium", + # --- FS-14 --- + "No Model Governance Config Rules Found": "Medium", + "Model Governance Config Rules Present": "Medium", + # --- FS-15 (eval jobs exist = Medium; absence is a real FAIL) --- + "No Bedrock Evaluation Jobs Found": "Medium", + "Bedrock Evaluation Jobs Present": "Medium", + # --- FS-16 (ECR scanning = High) --- + "No ECR Repositories Found": "Informational", + "ECR Repositories Without Image Scanning": "High", + "ECR Image Scanning Enabled": "High", + # --- FS-20 --- + "No SageMaker Feature Groups Found": "Informational", + "Feature Groups Without Offline Store": "Medium", + "Feature Store Offline Store Active": "Medium", + # --- FS-21 (training-data integrity = High) --- + "No Training Data Buckets Identified": "Informational", + "Training Data Buckets Without Versioning": "High", + "Training Data Buckets Have Versioning": "High", + # --- FS-22 (KB IAM least privilege = High) --- + "Overly Permissive Knowledge Base IAM Roles": "High", + "Knowledge Base IAM Permissions Look Appropriate": "High", + # --- FS-24 (advisory) --- + "No Knowledge Bases Found": "Informational", + "ADVISORY: Knowledge Base Metadata Filtering — Manual Review Required": "Informational", + # --- FS-25 (KB encryption = High) --- + "No OpenSearch Serverless Encryption Policies": "Informational", + "OpenSearch Serverless Encryption Not Using Customer-Managed Keys": "High", + "OpenSearch Serverless Encryption Policies Present": "High", + # --- FS-26 (VPC isolation = High) --- + "No OpenSearch Serverless Network Policies": "High", + "OpenSearch Serverless Collections Not VPC-Restricted": "High", + "OpenSearch Serverless VPC Access Configured": "High", + # --- FS-27 (contextual grounding = High; ARC = Medium) --- + "No Guardrails — Contextual Grounding Not Applicable": "Informational", + "No Guardrails With Contextual Grounding": "High", + "Contextual Grounding Enabled on Guardrails": "High", + "Automated Reasoning Policies — Access Check": "Low", + "No Automated Reasoning Policies Found": "Medium", + "Automated Reasoning Policies Found": "Medium", + # --- FS-28 (denied topics = High) --- + "No Guardrails — Denied Topics Not Applicable": "Informational", + "No Guardrails With Denied Financial Topics": "High", + "Denied Topics Configured on CLASSIC Tier": "High", + "Guardrails With Topic Policies Found": "High", + # --- FS-29 (advisory) --- + "ADVISORY: Compliance Disclaimer — Manual Review Required": "Informational", + # --- FS-30 (advisory — cannot inspect dataset content) --- + "ADVISORY: Compliance Dataset Coverage — Manual Review Required": "Informational", + # --- FS-31 --- + "Knowledge Base Data Sources Past Review Threshold": "Medium", + "Knowledge Base Data Sources Recently Synced": "Medium", + # --- FS-32 (advisory) --- + "ADVISORY: Source Attribution — Manual Review Required": "Informational", + # --- FS-33 (distinct risks: deleted bucket High, versioning Medium) --- + "KB Data Source References a Deleted S3 Bucket": "High", + "KB Data Source Buckets Without Versioning": "Medium", + "KB Data Source Buckets Have Versioning": "Medium", + # --- FS-34 --- + "Legacy Foundation Models Available in Region": "Informational", + "Foundation Models Are Current": "Medium", + # --- FS-35 (advisory) --- + "ADVISORY: Harmful-Content Test Coverage — Manual Review Required": "Informational", + # --- FS-36 (content filters = High) --- + "No Guardrails — Content Filters Not Applicable": "Informational", + "No Guardrails With Content Filters": "High", + "Guardrail Content Filters on CLASSIC Tier": "High", + "Guardrail Content Filters Configured (STANDARD Tier)": "High", + # --- FS-37 (advisory) --- + "ADVISORY: User Feedback Mechanism — Manual Review Required": "Informational", + # --- FS-38 (word filters = Medium) --- + "No Guardrails — Word Filters Not Applicable": "Informational", + "No Guardrails With Word Filters": "Medium", + "Guardrail Word Filters Configured": "Medium", + # --- FS-39 (bias = High) --- + "No SageMaker Clarify Bias Monitoring": "High", + "SageMaker Clarify Bias Monitoring Active": "High", + # --- FS-40 (advisory) --- + "ADVISORY: Bias Dataset Coverage — Manual Review Required": "Informational", + # --- FS-41 (explainability = High) --- + "No SageMaker Clarify Explainability Monitoring": "High", + "SageMaker Clarify Explainability Active": "High", + # --- FS-42 --- + "No SageMaker Model Cards Found": "Medium", + "SageMaker Model Cards Present": "Medium", + # --- FS-43 (log data protection = High) --- + "No CloudWatch Logs Data Protection Policies": "High", + "CloudWatch Logs Data Protection Policies Present": "High", + # --- FS-44 (Macie = High) --- + "Amazon Macie Not Enabled": "High", + "Amazon Macie Enabled": "High", + # --- FS-45 (PII filters = High) --- + "No Guardrails — PII Filters Not Applicable": "Informational", + "No Guardrails With PII Filters": "High", + "Guardrail PII Filters Configured": "High", + # --- FS-46 (classification = Medium) --- + "No AI/ML Data Buckets Identified": "Informational", + "AI/ML Buckets Without Data Classification Tags": "Medium", + "AI/ML Buckets Have Classification Tags": "Medium", + # --- FS-47 (grounding threshold = High) --- + "No Guardrails — Grounding Threshold Not Applicable": "Informational", + "Guardrails With Low Grounding Thresholds": "High", + "No Guardrails With a Grounding Filter": "High", + "Guardrail Grounding Thresholds Appropriate": "High", + # --- FS-48 --- + "No Active Knowledge Bases for RAG": "Medium", + "Active Knowledge Bases for RAG Present": "Medium", + # --- FS-49 (advisory) --- + "ADVISORY: Hallucination Disclaimer — Manual Review Required": "Informational", + # --- FS-50 (relevance grounding = Medium) --- + "No Guardrails With Relevance Grounding Filters": "Medium", + "Relevance Grounding Filters Present": "Medium", + # --- FS-51 (prompt attack = High) --- + "No Guardrails — Prompt Attack Filters Not Applicable": "Informational", + "No Guardrails With Prompt Attack Filters": "High", + "Prompt Attack Filters on CLASSIC Tier": "High", + "Prompt Attack Filters Configured (STANDARD Tier)": "High", + # --- FS-52 --- + "No Bedrock-Related Lambda Functions Found": "Informational", + "Bedrock Lambda Functions on Deprecated Runtimes": "Medium", + "Bedrock Lambda Functions on Current Runtimes": "Medium", + # --- FS-53 (injection rules = High) --- + "No WAF Web ACLs — Injection Rules Not Applicable": "Informational", + "WAF ACLs Missing Injection Protection Rules": "High", + "WAF Injection Protection Rules Present": "High", + # --- FS-54 (advisory) --- + "ADVISORY: Penetration Testing — Manual Review Required": "Informational", + # --- FS-55 --- + "No Output Validation Functions Found": "Medium", + "Output Validation Functions Present": "Medium", + # --- FS-56 (XSS = Medium; now has a FAIL path) --- + "No WAF ACLs — XSS Prevention Not Applicable": "Informational", + "WAF ACLs Missing Common Rule Set (XSS)": "Medium", + "XSS Prevention Common Rule Set Present": "Medium", + # --- FS-57 (advisory) --- + "ADVISORY: Output Encoding — Manual Review Required": "Informational", + # --- FS-58 (advisory) --- + "ADVISORY: Output Schema Validation — Manual Review Required": "Informational", + # --- FS-59 (topic allowlist = Medium) --- + "No Guardrails — Topic Allowlist Not Applicable": "Informational", + "No Guardrails With Topic Restrictions": "Medium", + "Topic Restrictions Configured on CLASSIC Tier": "Medium", + "Guardrail Topic Restrictions Configured": "Medium", + # --- FS-60 (advisory) --- + "ADVISORY: Contextual Grounding for Off-Topic Prevention": "Informational", + # --- FS-61 --- + "No Automated KB Sync Schedules Detected": "Medium", + "Automated KB Sync Schedules Present": "Medium", + # --- FS-62 (advisory) --- + "ADVISORY: Data Currency Disclaimer — Manual Review Required": "Informational", + # --- FS-63 --- + "Legacy Models Without Lifecycle Management": "Medium", + "Foundation Model Lifecycle Management": "Medium", + # --- FS-65 (distinct risks) --- + "KB Data Source Buckets Missing S3 Event Notifications": "Medium", + "KB Data Source S3 Event Notifications Configured": "Medium", + # --- FS-66 (identity propagation = High) --- + "AgentCore Identity Propagation — Access Check": "Low", + "AgentCore Runtimes Missing End-User Identity Propagation": "High", + "AgentCore End-User Identity Propagation Configured": "High", + # --- FS-67 (transaction thresholds = High) --- + "No Agent Action-Group Lambda Functions Found": "Informational", + "Agent Action-Group Lambdas May Lack Transaction Thresholds": "High", + "Agent Action-Group Lambdas Have Threshold Configuration": "High", + # --- FS-68 (body-size = Medium; now has N/A branch) --- + "API Gateway Request Body Size Limits Not Enforced": "Medium", + "API Gateway Request Body Size Limits — Not Applicable": "Informational", + "API Gateway Request Body Size Limits Configured": "Medium", + # --- FS-69 --- + "No Prompt Input Validation Function Found": "Medium", + "Prompt Input Validation Functions Present": "Medium", +} + + def _could_not_assess_row(check_id: str, check_name: str, err: Any) -> Dict[str, Any]: """ Synthesize one visible finding row for a check that errored out and produced - no rows. Uses the existing schema (Status="N/A", Severity="Medium") so the - gap surfaces in the report without inflating the Failed count. + no rows. Uses Status="N/A", Severity="Low" (the COULD_NOT_ASSESS disposition — + see severity-methodology.md §3.4) so the gap surfaces in the report as an + unknown/assessment-gap without inflating the Failed count or implying a + confirmed control failure. """ return create_finding( check_id=check_id, @@ -341,7 +729,7 @@ def _could_not_assess_row(check_id: str, check_name: str, err: Any) -> Dict[str, "4. Re-run the assessment; assess this control manually until it succeeds." ), reference="https://docs.aws.amazon.com/IAM/latest/UserGuide/troubleshoot_access-denied.html", - severity="Medium", + severity="Low", status="N/A", compliance_frameworks=COMPLIANCE_MAP.get(check_id, ""), ) @@ -354,7 +742,7 @@ def _could_not_assess_row(check_id: str, check_name: str, err: Any) -> Dict[str, # =========================================================================== -def check_waf_shield_on_bedrock_endpoints() -> Dict[str, Any]: +def check_waf_shield_on_bedrock_endpoints(inventory) -> Dict[str, Any]: """ FS-01 — Verify AWS WAF is associated with API Gateway or ALB endpoints that front Bedrock/GenAI workloads, and that AWS Shield Advanced is enabled. @@ -362,7 +750,6 @@ def check_waf_shield_on_bedrock_endpoints() -> Dict[str, Any]: """ findings = _empty_findings("WAF and Shield Protection Check") try: - wafv2 = boto3.client("wafv2", config=boto3_config) shield = boto3.client("shield", config=boto3_config) # Check Shield Advanced subscription @@ -376,7 +763,9 @@ def check_waf_shield_on_bedrock_endpoints() -> Dict[str, Any]: pass # Check WAF Web ACLs exist (regional, covering API GW / ALB) - acls = wafv2.list_web_acls(Scope="REGIONAL").get("WebACLs", []) + # require() raises if inventory is None or the field is _Unavailable, + # which propagates to the outer except and yields COULD_NOT_ASSESS. + acls = require(inventory, "web_acls").summaries if not shield_enabled: findings["status"] = "WARN" @@ -401,7 +790,7 @@ def check_waf_shield_on_bedrock_endpoints() -> Dict[str, Any]: "to automate resource protection based on tags or resource types." ), reference="https://docs.aws.amazon.com/waf/latest/developerguide/shield-chapter.html", - severity="High", + severity="Low", status="Failed", compliance_frameworks=COMPLIANCE_MAP["FS-01"], ) @@ -414,7 +803,7 @@ def check_waf_shield_on_bedrock_endpoints() -> Dict[str, Any]: finding_details="AWS Shield Advanced subscription is active.", resolution="No action required.", reference="https://docs.aws.amazon.com/waf/latest/developerguide/shield-chapter.html", - severity="High", + severity="Low", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-01"], ) @@ -436,7 +825,7 @@ def check_waf_shield_on_bedrock_endpoints() -> Dict[str, Any]: "3. Add AWS Managed Rules for known bad inputs." ), reference="https://docs.aws.amazon.com/waf/latest/developerguide/waf-chapter.html", - severity="High", + severity="Medium", status="Failed", compliance_frameworks=COMPLIANCE_MAP["FS-01"], ) @@ -449,7 +838,7 @@ def check_waf_shield_on_bedrock_endpoints() -> Dict[str, Any]: finding_details=f"Found {len(acls)} regional WAF Web ACL(s).", resolution="Verify ACLs are associated with Bedrock-facing endpoints.", reference="https://docs.aws.amazon.com/waf/latest/developerguide/waf-chapter.html", - severity="High", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-01"], ) @@ -486,7 +875,7 @@ def check_api_gateway_rate_limiting() -> Dict[str, Any]: "(rateLimit and burstLimit) for all Bedrock-facing APIs." ), reference="https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-request-throttling.html", - severity="Medium", + severity="Informational", status="N/A", compliance_frameworks=COMPLIANCE_MAP["FS-02"], ) @@ -506,7 +895,7 @@ def check_api_gateway_rate_limiting() -> Dict[str, Any]: "GenAI API stages. Consider per-consumer API keys with individual quotas." ), reference="https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-request-throttling.html", - severity="High", + severity="Medium", status="Failed", compliance_frameworks=COMPLIANCE_MAP["FS-02"], ) @@ -519,7 +908,7 @@ def check_api_gateway_rate_limiting() -> Dict[str, Any]: finding_details=f"All {len(plans)} usage plan(s) have throttle settings.", resolution="No action required.", reference="https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-request-throttling.html", - severity="High", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-02"], ) @@ -1147,7 +1536,7 @@ def check_agentcore_policy_engine() -> Dict[str, Any]: return findings -def check_agent_transaction_limits() -> Dict[str, Any]: +def check_agent_transaction_limits(inventory) -> Dict[str, Any]: """ FS-09 — Check for application-level transaction/action limits on agents via Lambda concurrency limits or Step Functions execution limits. @@ -1155,8 +1544,7 @@ def check_agent_transaction_limits() -> Dict[str, Any]: """ findings = _empty_findings("Agent Transaction Limits Check") try: - lambda_client = boto3.client("lambda", config=boto3_config) - functions = _paginate(lambda_client, "list_functions", "Functions") + functions = require(inventory, "lambda_functions") # Look for agent-related Lambda functions without reserved concurrency agent_lambdas = [ @@ -1167,6 +1555,7 @@ def check_agent_transaction_limits() -> Dict[str, Any]: ) ] + lambda_client = boto3.client("lambda", config=boto3_config) lambdas_without_concurrency = [] for fn in agent_lambdas: try: @@ -1262,7 +1651,7 @@ def check_human_in_the_loop_for_high_risk_actions() -> Dict[str, Any]: "Route approval requests to human reviewers via SNS/SES/Slack." ), reference="https://docs.aws.amazon.com/step-functions/latest/dg/connect-to-resource.html#connect-wait-token", - severity="Medium", + severity="Informational", status="N/A", compliance_frameworks=COMPLIANCE_MAP["FS-10"], ) @@ -1397,7 +1786,7 @@ def check_scp_model_access_restrictions() -> Dict[str, Any]: finding_details="Account is not part of an AWS Organization or lacks SCP read access.", resolution="If using AWS Organizations, ensure SCPs restrict Bedrock model access to approved models.", reference="https://docs.aws.amazon.com/organizations/latest/userguide/orgs_manage_policies_scps.html", - severity="Low", + severity="Informational", status="N/A", compliance_frameworks=COMPLIANCE_MAP["FS-12"], ) @@ -1596,13 +1985,15 @@ def check_bedrock_model_evaluation_adversarial() -> Dict[str, Any]: evals = _paginate(bedrock, "list_evaluation_jobs", "jobSummaries") if not evals: + findings["status"] = "WARN" findings["csv_data"].append( create_finding( check_id="FS-15", finding_name="No Bedrock Evaluation Jobs Found", finding_details=( - "No Bedrock Model Evaluation jobs found. " - "Models have not been evaluated for adversarial robustness." + "No Bedrock Model Evaluation jobs found. Models have not been evaluated " + "for adversarial robustness. FinServ model-risk management (SR 11-7) " + "expects documented model validation/evaluation." ), resolution=( "1. Run Bedrock Model Evaluation with adversarial/red-team datasets.\n" @@ -1611,7 +2002,7 @@ def check_bedrock_model_evaluation_adversarial() -> Dict[str, Any]: ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", severity="Medium", - status="N/A", + status="Failed", compliance_frameworks=COMPLIANCE_MAP["FS-15"], ) ) @@ -1786,7 +2177,7 @@ def check_feature_store_rollback_capability() -> Dict[str, Any]: return findings -def check_training_data_s3_versioning() -> Dict[str, Any]: +def check_training_data_s3_versioning(inventory) -> Dict[str, Any]: """ FS-21 — Verify S3 buckets used for training data have versioning enabled to support rollback of poisoned datasets. @@ -1794,8 +2185,8 @@ def check_training_data_s3_versioning() -> Dict[str, Any]: """ findings = _empty_findings("Training Data S3 Versioning Check") try: + buckets = require(inventory, "buckets") s3 = boto3.client("s3", config=boto3_config) - buckets = s3.list_buckets().get("Buckets", []) training_buckets = [ b @@ -1884,6 +2275,21 @@ def check_training_data_s3_versioning() -> Dict[str, Any]: # =========================================================================== +def _is_overbroad_kb_action(action: Any) -> bool: + """True if an IAM action grants overly broad Bedrock/Knowledge Base access: + the full wildcard, a service-wide bedrock(-agent) wildcard, or ANY partial + wildcard within those namespaces (e.g., 'bedrock-agent:Get*', 'bedrock:Invoke*'). + Round-3 fixed the crash; this widens detection beyond the three exact wildcards.""" + if not isinstance(action, str): + return False + a = action.lower() + if a in ("*", "bedrock:*", "bedrock-agent:*"): + return True + if a.endswith("*") and (a.startswith("bedrock:") or a.startswith("bedrock-agent:")): + return True + return False + + def check_knowledge_base_iam_least_privilege(permission_cache) -> Dict[str, Any]: """ FS-22 — Verify IAM roles accessing Bedrock Knowledge Bases follow @@ -1896,21 +2302,61 @@ def check_knowledge_base_iam_least_privilege(permission_cache) -> Dict[str, Any] for role_name, perms in ( (permission_cache or {}).get("role_permissions", {}).items() ): - for policy in perms.get("attached_policies", []) + perms.get( - "inline_policies", [] + if not isinstance(perms, dict): + continue + for policy in (perms.get("attached_policies", []) or []) + ( + perms.get("inline_policies", []) or [] ): + if not isinstance(policy, dict): + continue doc = policy.get("document", {}) if isinstance(doc, str): - doc = json.loads(doc) - for stmt in doc.get("Statement", []): + try: + doc = json.loads(doc) + except (ValueError, TypeError): + continue + if not isinstance(doc, dict): + continue + # IAM allows Statement to be a single object (dict) or a list. + # Normalize to a list so iterating never yields statement *keys* + # (the cause of the 'str' object has no attribute 'get' crash). + statements = doc.get("Statement", []) + if isinstance(statements, dict): + statements = [statements] + for stmt in statements: + if not isinstance(stmt, dict): + continue if stmt.get("Effect") != "Allow": continue + # NotAction Allow ("allow everything except …") is inherently + # broad and the antithesis of least privilege — flag it. + if "NotAction" in stmt: + issues.append( + f"Role '{role_name}' uses a NotAction Allow (overly broad — " + "grants all actions except those listed)" + ) + resources = stmt.get("Resource", []) + if isinstance(resources, str): + resources = [resources] + unscoped_resource = "*" in resources actions = stmt.get("Action", []) if isinstance(actions, str): actions = [actions] for action in actions: - if action in ("bedrock-agent:*", "bedrock:*", "*"): + if _is_overbroad_kb_action(action): issues.append(f"Role '{role_name}' allows '{action}'") + elif ( + unscoped_resource + and isinstance(action, str) + and ( + action.lower().startswith("bedrock:") + or action.lower().startswith("bedrock-agent:") + ) + ): + issues.append( + f"Role '{role_name}' allows '{action}' on Resource '*' " + "(no ARN scoping to specific Knowledge Bases)" + ) if issues: findings["status"] = "WARN" @@ -1951,7 +2397,7 @@ def check_knowledge_base_iam_least_privilege(permission_cache) -> Dict[str, Any] return findings -def check_knowledge_base_metadata_filtering() -> Dict[str, Any]: +def check_knowledge_base_metadata_filtering(inventory) -> Dict[str, Any]: """ FS-24 — Check that Bedrock Knowledge Bases have metadata fields configured to support tenant-level filtering (multi-tenancy isolation). @@ -1959,11 +2405,7 @@ def check_knowledge_base_metadata_filtering() -> Dict[str, Any]: """ findings = _empty_findings("Knowledge Base Metadata Filtering Check") try: - bedrock_agent = boto3.client("bedrock-agent", config=boto3_config) - paginator = bedrock_agent.get_paginator("list_knowledge_bases") - kbs = [] - for page in paginator.paginate(): - kbs.extend(page.get("knowledgeBaseSummaries", [])) + kbs = require(inventory, "knowledge_bases").summaries if not kbs: findings["csv_data"].append( @@ -1984,9 +2426,10 @@ def check_knowledge_base_metadata_filtering() -> Dict[str, Any]: findings["csv_data"].append( create_finding( check_id="FS-24", - finding_name="Knowledge Base Metadata Filtering — Manual Review Required", + finding_name="ADVISORY: Knowledge Base Metadata Filtering — Manual Review Required", finding_details=( - f"Found {len(kbs)} Knowledge Base(s). " + f"Found {len(kbs)} Knowledge Base(s). Tenant-isolation metadata filtering is a " + "design pattern that cannot be verified via API — manual review required. " "Verify that metadata attributes (e.g., tenantId, classification) are indexed " "and that Retrieve calls include RetrievalFilter conditions for tenant isolation." ), @@ -1996,8 +2439,8 @@ def check_knowledge_base_metadata_filtering() -> Dict[str, Any]: "3. Validate filters in integration tests to prevent cross-tenant data leakage." ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/kb-test-config.html", - severity="Medium", - status="Passed", + severity="Informational", + status="N/A", compliance_frameworks=COMPLIANCE_MAP["FS-24"], ) ) @@ -2180,7 +2623,7 @@ def check_knowledge_base_vpc_access() -> Dict[str, Any]: # =========================================================================== -def check_guardrail_contextual_grounding() -> Dict[str, Any]: +def check_guardrail_contextual_grounding(inventory) -> Dict[str, Any]: """ FS-27 — Check whether Bedrock Guardrails have contextual grounding checks configured to validate that outputs are grounded in the provided context and @@ -2200,8 +2643,8 @@ def check_guardrail_contextual_grounding() -> Dict[str, Any]: """ findings = _empty_findings("Guardrail Contextual Grounding Check") try: - bedrock = boto3.client("bedrock", config=boto3_config) - guardrails = _paginate(bedrock, "list_guardrails", "guardrails") + guardrail_inv = require(inventory, "guardrails") + guardrails = guardrail_inv.summaries if not guardrails: findings["csv_data"].append( @@ -2214,7 +2657,7 @@ def check_guardrail_contextual_grounding() -> Dict[str, Any]: "(grounding threshold ≥0.7 and relevance threshold ≥0.7 for FinServ use cases)." ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", - severity="Medium", + severity="Informational", status="N/A", compliance_frameworks=COMPLIANCE_MAP["FS-27"], ) @@ -2223,9 +2666,9 @@ def check_guardrail_contextual_grounding() -> Dict[str, Any]: guardrails_with_grounding = [] for g in guardrails: - detail = bedrock.get_guardrail( - guardrailIdentifier=g["id"], guardrailVersion="DRAFT" - ) + detail = guardrail_inv.detail_by_id[g["id"]] + if isinstance(detail, _Unavailable): + raise detail.error if detail.get("contextualGroundingPolicy"): guardrails_with_grounding.append(g["name"]) @@ -2265,7 +2708,7 @@ def check_guardrail_contextual_grounding() -> Dict[str, Any]: "Also consider enabling Automated Reasoning checks for formal policy verification." ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", - severity="Medium", + severity="High", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-27"], ) @@ -2286,8 +2729,11 @@ def check_automated_reasoning_policies() -> Dict[str, Any]: grounding (a threshold applied per call), ARC requires authoring an Automated Reasoning Policy document containing the rules to verify against. - Regions supported (as of June 2026): us-east-1, us-east-2, us-west-2, - eu-central-1, eu-west-1, eu-west-3. + Regions supported (verify against current AWS docs at run time — + https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-automated-reasoning-checks.html): + AWS GovCloud (US), us-east-1, us-east-2, us-west-2, eu-central-1, eu-west-1, + eu-west-3. The list expands over time; treat a region miss as "verify + availability", not a hard limitation. IAM action required: bedrock:ListAutomatedReasoningPolicies @@ -2310,17 +2756,27 @@ def check_automated_reasoning_policies() -> Dict[str, Any]: check_id="FS-27", finding_name="Automated Reasoning Policies — Access Check", finding_details=( - "Unable to list Automated Reasoning policies (access denied or service " - "unavailable in this region). ARC is available in: us-east-1, us-east-2, " - "us-west-2, eu-central-1, eu-west-1, eu-west-3." + "Access denied or service unavailable when listing Automated Reasoning " + "policies. The IAM action name (bedrock:ListAutomatedReasoningPolicies) " + "is correct, so the most likely causes are, in order: (1) the assessment " + "MEMBER ROLE in this account was deployed before this action was added " + "and has not been re-deployed; (2) an AWS Organizations SCP or permission " + "boundary denies this newer Bedrock action; (3) the region does not " + "support ARC. ARC is available in AWS GovCloud (US) and a growing set " + "of commercial regions (e.g., us-east-1, us-east-2, us-west-2, " + "eu-central-1, eu-west-1, eu-west-3) — verify the current list in the " + "AWS documentation." ), resolution=( - "1. Ensure the assessment role grants bedrock:ListAutomatedReasoningPolicies.\n" - "2. Confirm the assessed region supports Automated Reasoning checks.\n" - "3. Add bedrock:ListAutomatedReasoningPolicies to the member role policy " - "in deployment/1-aiml-security-member-roles.yaml." + "1. RE-DEPLOY the member-role CloudFormation stack so the role picks up " + "bedrock:ListAutomatedReasoningPolicies (templates may be current while " + "the *deployed* role is stale). See deployment/1-aiml-security-member-roles.yaml " + "and aiml-security-single-account.yaml.\n" + "2. Check for an Organizations SCP / permission boundary denying the action.\n" + "3. Confirm the assessed region supports Automated Reasoning checks.\n" + "4. Re-run the assessment after re-deploying." ), - reference="https://docs.aws.amazon.com/bedrock/latest/userguide/automated-reasoning.html", + reference="https://docs.aws.amazon.com/IAM/latest/UserGuide/troubleshoot_access-denied.html", severity="Low", status="N/A", compliance_frameworks=COMPLIANCE_MAP["FS-27"], @@ -2387,7 +2843,7 @@ def check_automated_reasoning_policies() -> Dict[str, Any]: return findings -def check_guardrail_denied_topics_financial() -> Dict[str, Any]: +def check_guardrail_denied_topics_financial(inventory) -> Dict[str, Any]: """ FS-28 — Verify Bedrock Guardrails have denied topics configured for regulated financial advice categories (investment advice, credit decisions). @@ -2395,8 +2851,8 @@ def check_guardrail_denied_topics_financial() -> Dict[str, Any]: """ findings = _empty_findings("Financial Denied Topics Check") try: - bedrock = boto3.client("bedrock", config=boto3_config) - guardrails = _paginate(bedrock, "list_guardrails", "guardrails") + guardrail_inv = require(inventory, "guardrails") + guardrails = guardrail_inv.summaries if not guardrails: findings["csv_data"].append( @@ -2406,7 +2862,7 @@ def check_guardrail_denied_topics_financial() -> Dict[str, Any]: finding_details="No Bedrock Guardrails configured.", resolution="Configure guardrails with denied topics for regulated financial content.", reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", - severity="Medium", + severity="Informational", status="N/A", compliance_frameworks=COMPLIANCE_MAP["FS-28"], ) @@ -2416,9 +2872,9 @@ def check_guardrail_denied_topics_financial() -> Dict[str, Any]: guardrails_with_topics = [] topics_classic_tier = [] for g in guardrails: - detail = bedrock.get_guardrail( - guardrailIdentifier=g["id"], guardrailVersion="DRAFT" - ) + detail = guardrail_inv.detail_by_id[g["id"]] + if isinstance(detail, _Unavailable): + raise detail.error topic_policy = detail.get("topicPolicy", {}) if topic_policy.get("topics"): guardrails_with_topics.append(g["name"]) @@ -2479,7 +2935,7 @@ def check_guardrail_denied_topics_financial() -> Dict[str, Any]: "(PDF \u00a71.2.1 Practical guidance)." ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", - severity="Low", + severity="High", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-28"], ) @@ -2497,7 +2953,7 @@ def check_guardrail_denied_topics_financial() -> Dict[str, Any]: "documents, and incident reports (as recommended in PDF \u00a71.2.1 Practical guidance)." ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", - severity="Medium", + severity="High", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-28"], ) @@ -2540,46 +2996,39 @@ def check_compliance_disclaimer_in_outputs() -> Dict[str, Any]: def check_bedrock_evaluation_compliance_datasets() -> Dict[str, Any]: """ - FS-30 — Check whether Bedrock Model Evaluation jobs use compliance-specific + FS-30 — Advisory: Bedrock Model Evaluation jobs should use compliance-specific datasets (e.g., fair lending, UDAP, ECOA test cases). + + The Bedrock evaluation-job APIs do not expose dataset *content*, so whether a + job actually includes compliance test cases cannot be verified programmatically. + This is therefore an advisory (manual-review) check. The existence of evaluation + jobs at all is the verifiable control and is assessed by FS-15. COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, NYDFS 500] """ findings = _empty_findings("Compliance Evaluation Datasets Check") try: - bedrock = boto3.client("bedrock", config=boto3_config) - evals = _paginate(bedrock, "list_evaluation_jobs", "jobSummaries") - - if not evals: - findings["csv_data"].append( - create_finding( - check_id="FS-30", - finding_name="No Bedrock Evaluation Jobs — Compliance Datasets Not Verified", - finding_details="No Bedrock Model Evaluation jobs found.", - resolution=( - "Run Bedrock Model Evaluation with compliance-specific datasets:\n" - "- Fair lending test cases (ECOA, Fair Housing Act)\n" - "- UDAP/UDAAP unfair/deceptive practice scenarios\n" - "- AML/KYC edge cases" - ), - reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", - severity="Medium", - status="N/A", - compliance_frameworks=COMPLIANCE_MAP["FS-30"], - ) - ) - else: - findings["csv_data"].append( - create_finding( - check_id="FS-30", - finding_name="Bedrock Evaluation Jobs Present", - finding_details=f"Found {len(evals)} evaluation job(s). Verify compliance datasets are included.", - resolution="Ensure evaluation datasets include FinServ regulatory test cases.", - reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", - severity="Medium", - status="Passed", - compliance_frameworks=COMPLIANCE_MAP["FS-30"], - ) + findings["csv_data"].append( + create_finding( + check_id="FS-30", + finding_name="ADVISORY: Compliance Dataset Coverage — Manual Review Required", + finding_details=( + "Bedrock model-evaluation dataset content cannot be inspected via API. " + "Manually verify your model-evaluation jobs include compliance-specific " + "datasets (fair lending/ECOA, Fair Housing Act, UDAP/UDAAP, AML/KYC edge cases). " + "Whether any evaluation jobs exist at all is assessed by FS-15." + ), + resolution=( + "Run Bedrock Model Evaluation with compliance-specific datasets:\n" + "- Fair lending test cases (ECOA, Fair Housing Act)\n" + "- UDAP/UDAAP unfair/deceptive practice scenarios\n" + "- AML/KYC edge cases" + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-30"], ) + ) except Exception as e: return _error_findings("Compliance Evaluation Datasets Check", e) return findings @@ -2592,7 +3041,7 @@ def check_bedrock_evaluation_compliance_datasets() -> Dict[str, Any]: # =========================================================================== -def check_knowledge_base_data_source_sync() -> Dict[str, Any]: +def check_knowledge_base_data_source_sync(inventory) -> Dict[str, Any]: """ FS-31 — Verify Bedrock Knowledge Base data sources have recent sync jobs to ensure information currency. @@ -2609,11 +3058,8 @@ def check_knowledge_base_data_source_sync() -> Dict[str, Any]: STALE_AFTER_DAYS = 7 findings = _empty_findings("Knowledge Base Data Source Sync Check") try: - bedrock_agent = boto3.client("bedrock-agent", config=boto3_config) - paginator = bedrock_agent.get_paginator("list_knowledge_bases") - kbs = [] - for page in paginator.paginate(): - kbs.extend(page.get("knowledgeBaseSummaries", [])) + kb_inv = require(inventory, "knowledge_bases") + kbs = kb_inv.summaries if not kbs: findings["csv_data"].append( @@ -2634,12 +3080,9 @@ def check_knowledge_base_data_source_sync() -> Dict[str, Any]: now = datetime.now(timezone.utc) for kb in kbs: kb_id = kb["knowledgeBaseId"] - sources = _paginate( - bedrock_agent, - "list_data_sources", - "dataSourceSummaries", - knowledgeBaseId=kb_id, - ) + sources = kb_inv.data_sources_by_kb.get(kb_id, []) + if isinstance(sources, _Unavailable): + raise sources.error for source in sources: last_updated = source.get("updatedAt") if last_updated: @@ -2727,7 +3170,7 @@ def check_source_attribution_in_guardrails() -> Dict[str, Any]: return findings -def check_knowledge_base_integrity_monitoring() -> Dict[str, Any]: +def check_knowledge_base_integrity_monitoring(inventory) -> Dict[str, Any]: """ FS-33 — Check for S3 object integrity monitoring (checksums, versioning) on Knowledge Base data source buckets. @@ -2735,14 +3178,10 @@ def check_knowledge_base_integrity_monitoring() -> Dict[str, Any]: """ findings = _empty_findings("Knowledge Base Integrity Monitoring Check") try: - bedrock_agent = boto3.client("bedrock-agent", config=boto3_config) + kb_inv = require(inventory, "knowledge_bases") + kbs = kb_inv.summaries s3 = boto3.client("s3", config=boto3_config) - paginator = bedrock_agent.get_paginator("list_knowledge_bases") - kbs = [] - for page in paginator.paginate(): - kbs.extend(page.get("knowledgeBaseSummaries", [])) - if not kbs: findings["csv_data"].append( create_finding( @@ -2761,17 +3200,17 @@ def check_knowledge_base_integrity_monitoring() -> Dict[str, Any]: buckets_without_versioning = [] missing_buckets = [] for kb in kbs: - sources = _paginate( - bedrock_agent, - "list_data_sources", - "dataSourceSummaries", - knowledgeBaseId=kb["knowledgeBaseId"], - ) + kb_id = kb["knowledgeBaseId"] + sources = kb_inv.data_sources_by_kb.get(kb_id, []) + if isinstance(sources, _Unavailable): + raise sources.error for source in sources: - source_detail = bedrock_agent.get_data_source( - knowledgeBaseId=kb["knowledgeBaseId"], - dataSourceId=source["dataSourceId"], - ) + ds_id = source["dataSourceId"] + source_detail = kb_inv.data_source_detail.get((kb_id, ds_id)) + if isinstance(source_detail, _Unavailable): + raise source_detail.error + if source_detail is None: + continue s3_config = ( source_detail.get("dataSource", {}) .get("dataSourceConfiguration", {}) @@ -2920,7 +3359,7 @@ def check_fm_version_currency() -> Dict[str, Any]: "4. Add data-currency disclaimers to outputs from models with old cutoffs." ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle.html", - severity="Medium", + severity="Informational", status="N/A", compliance_frameworks=COMPLIANCE_MAP["FS-34"], ) @@ -2958,46 +3397,34 @@ def check_fmeval_harmful_content() -> Dict[str, Any]: """ findings = _empty_findings("FMEval Harmful Content Check") try: - bedrock = boto3.client("bedrock", config=boto3_config) - evals = _paginate(bedrock, "list_evaluation_jobs", "jobSummaries") - - if not evals: - findings["csv_data"].append( - create_finding( - check_id="FS-35", - finding_name="No Evaluation Jobs — Harmful Content Testing Not Verified", - finding_details="No Bedrock Model Evaluation jobs found.", - resolution=( - "Run Bedrock Model Evaluation or FMEval with harmful content datasets:\n" - "- Toxicity detection\n" - "- Hate speech classification\n" - "- Violence/self-harm content" - ), - reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", - severity="Medium", - status="N/A", - compliance_frameworks=COMPLIANCE_MAP["FS-35"], - ) - ) - else: - findings["csv_data"].append( - create_finding( - check_id="FS-35", - finding_name="Evaluation Jobs Present", - finding_details=f"Found {len(evals)} evaluation job(s). Verify harmful content datasets are included.", - resolution="Ensure evaluation includes toxicity and harmful content test cases.", - reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", - severity="Medium", - status="Passed", - compliance_frameworks=COMPLIANCE_MAP["FS-35"], - ) + findings["csv_data"].append( + create_finding( + check_id="FS-35", + finding_name="ADVISORY: Harmful-Content Test Coverage — Manual Review Required", + finding_details=( + "Bedrock model-evaluation dataset content cannot be inspected via API. " + "Manually verify your model-evaluation/FMEval jobs include harmful-content " + "datasets (toxicity, hate speech, violence/self-harm). Whether any evaluation " + "jobs exist at all is assessed by FS-15." + ), + resolution=( + "Run Bedrock Model Evaluation or FMEval with harmful content datasets:\n" + "- Toxicity detection\n" + "- Hate speech classification\n" + "- Violence/self-harm content" + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-35"], ) + ) except Exception as e: return _error_findings("FMEval Harmful Content Check", e) return findings -def check_guardrail_content_filters() -> Dict[str, Any]: +def check_guardrail_content_filters(inventory) -> Dict[str, Any]: """ FS-36 — Verify Bedrock Guardrails have content filters configured for hate speech, violence, and sexual content at appropriate thresholds. @@ -3005,8 +3432,8 @@ def check_guardrail_content_filters() -> Dict[str, Any]: """ findings = _empty_findings("Guardrail Content Filters Check") try: - bedrock = boto3.client("bedrock", config=boto3_config) - guardrails = _paginate(bedrock, "list_guardrails", "guardrails") + guardrail_inv = require(inventory, "guardrails") + guardrails = guardrail_inv.summaries if not guardrails: findings["csv_data"].append( @@ -3016,7 +3443,7 @@ def check_guardrail_content_filters() -> Dict[str, Any]: finding_details="No Bedrock Guardrails configured.", resolution="Configure guardrails with content filters.", reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-filters.html", - severity="High", + severity="Informational", status="N/A", compliance_frameworks=COMPLIANCE_MAP["FS-36"], ) @@ -3026,9 +3453,9 @@ def check_guardrail_content_filters() -> Dict[str, Any]: guardrails_with_filters = [] guardrails_classic_tier = [] for g in guardrails: - detail = bedrock.get_guardrail( - guardrailIdentifier=g["id"], guardrailVersion="DRAFT" - ) + detail = guardrail_inv.detail_by_id[g["id"]] + if isinstance(detail, _Unavailable): + raise detail.error content_policy = detail.get("contentPolicy", {}) if content_policy.get("filters"): guardrails_with_filters.append(g["name"]) @@ -3087,7 +3514,7 @@ def check_guardrail_content_filters() -> Dict[str, Any]: "with tierName=STANDARD and configure a guardrail cross-region profile." ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-filters.html", - severity="Low", + severity="High", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-36"], ) @@ -3143,7 +3570,7 @@ def check_user_feedback_mechanism() -> Dict[str, Any]: return findings -def check_guardrail_word_filters() -> Dict[str, Any]: +def check_guardrail_word_filters(inventory) -> Dict[str, Any]: """ FS-38 — Verify Bedrock Guardrails have word/phrase filters (allowlists/denylists) configured for financial services context. @@ -3151,8 +3578,8 @@ def check_guardrail_word_filters() -> Dict[str, Any]: """ findings = _empty_findings("Guardrail Word Filters Check") try: - bedrock = boto3.client("bedrock", config=boto3_config) - guardrails = _paginate(bedrock, "list_guardrails", "guardrails") + guardrail_inv = require(inventory, "guardrails") + guardrails = guardrail_inv.summaries if not guardrails: findings["csv_data"].append( @@ -3162,7 +3589,7 @@ def check_guardrail_word_filters() -> Dict[str, Any]: finding_details="No Bedrock Guardrails configured.", resolution="Configure guardrails with word filters.", reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", - severity="Medium", + severity="Informational", status="N/A", compliance_frameworks=COMPLIANCE_MAP["FS-38"], ) @@ -3171,9 +3598,9 @@ def check_guardrail_word_filters() -> Dict[str, Any]: guardrails_with_words = [] for g in guardrails: - detail = bedrock.get_guardrail( - guardrailIdentifier=g["id"], guardrailVersion="DRAFT" - ) + detail = guardrail_inv.detail_by_id[g["id"]] + if isinstance(detail, _Unavailable): + raise detail.error if detail.get("wordPolicy", {}).get("words") or detail.get( "wordPolicy", {} ).get("managedWordLists"): @@ -3286,40 +3713,29 @@ def check_bedrock_evaluation_bias_datasets() -> Dict[str, Any]: """ findings = _empty_findings("Bedrock Bias Evaluation Datasets Check") try: - bedrock = boto3.client("bedrock", config=boto3_config) - evals = _paginate(bedrock, "list_evaluation_jobs", "jobSummaries") - - if not evals: - findings["csv_data"].append( - create_finding( - check_id="FS-40", - finding_name="No Evaluation Jobs — Bias Datasets Not Verified", - finding_details="No Bedrock Model Evaluation jobs found.", - resolution=( - "Run Bedrock Model Evaluation with bias test datasets:\n" - "- Demographic parity test cases\n" - "- Equal opportunity scenarios\n" - "- Counterfactual fairness tests" - ), - reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", - severity="Medium", - status="N/A", - compliance_frameworks=COMPLIANCE_MAP["FS-40"], - ) - ) - else: - findings["csv_data"].append( - create_finding( - check_id="FS-40", - finding_name="Evaluation Jobs Present", - finding_details=f"Found {len(evals)} evaluation job(s). Verify bias datasets are included.", - resolution="Ensure evaluation includes demographic fairness test cases.", - reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", - severity="Medium", - status="Passed", - compliance_frameworks=COMPLIANCE_MAP["FS-40"], - ) + findings["csv_data"].append( + create_finding( + check_id="FS-40", + finding_name="ADVISORY: Bias Dataset Coverage — Manual Review Required", + finding_details=( + "Bedrock model-evaluation dataset content cannot be inspected via API. " + "Manually verify your model-evaluation jobs include bias/fairness datasets " + "(demographic parity, equal-opportunity, counterfactual fairness) for any " + "GenAI models used in financial decisions (ECOA/Fair Housing). Whether any " + "evaluation jobs exist at all is assessed by FS-15." + ), + resolution=( + "Run Bedrock Model Evaluation with bias test datasets:\n" + "- Demographic parity test cases\n" + "- Equal opportunity scenarios\n" + "- Counterfactual fairness tests" + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-40"], ) + ) except Exception as e: return _error_findings("Bedrock Bias Evaluation Datasets Check", e) return findings @@ -3553,7 +3969,7 @@ def check_macie_on_training_data_buckets() -> Dict[str, Any]: return findings -def check_guardrail_pii_filters() -> Dict[str, Any]: +def check_guardrail_pii_filters(inventory) -> Dict[str, Any]: """ FS-45 — Verify Bedrock Guardrails have sensitive information (PII) filters configured to block PII in prompts and responses. @@ -3561,8 +3977,8 @@ def check_guardrail_pii_filters() -> Dict[str, Any]: """ findings = _empty_findings("Guardrail PII Filters Check") try: - bedrock = boto3.client("bedrock", config=boto3_config) - guardrails = _paginate(bedrock, "list_guardrails", "guardrails") + guardrail_inv = require(inventory, "guardrails") + guardrails = guardrail_inv.summaries if not guardrails: findings["csv_data"].append( @@ -3572,7 +3988,7 @@ def check_guardrail_pii_filters() -> Dict[str, Any]: finding_details="No Bedrock Guardrails configured.", resolution="Configure guardrails with PII/sensitive information filters.", reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-sensitive-filters.html", - severity="High", + severity="Informational", status="N/A", compliance_frameworks=COMPLIANCE_MAP["FS-45"], ) @@ -3581,9 +3997,9 @@ def check_guardrail_pii_filters() -> Dict[str, Any]: guardrails_with_pii = [] for g in guardrails: - detail = bedrock.get_guardrail( - guardrailIdentifier=g["id"], guardrailVersion="DRAFT" - ) + detail = guardrail_inv.detail_by_id[g["id"]] + if isinstance(detail, _Unavailable): + raise detail.error if detail.get("sensitiveInformationPolicy", {}).get("piiEntities"): guardrails_with_pii.append(g["name"]) @@ -3629,7 +4045,7 @@ def check_guardrail_pii_filters() -> Dict[str, Any]: return findings -def check_data_classification_tagging() -> Dict[str, Any]: +def check_data_classification_tagging(inventory) -> Dict[str, Any]: """ FS-46 — Check that S3 buckets containing AI/ML data are tagged with data classification labels (e.g., Confidential, PII, Public). @@ -3637,8 +4053,8 @@ def check_data_classification_tagging() -> Dict[str, Any]: """ findings = _empty_findings("Data Classification Tagging Check") try: + buckets = require(inventory, "buckets") s3 = boto3.client("s3", config=boto3_config) - buckets = s3.list_buckets().get("Buckets", []) aiml_buckets = [ b @@ -3729,7 +4145,7 @@ def check_data_classification_tagging() -> Dict[str, Any]: # =========================================================================== -def check_guardrail_grounding_threshold() -> Dict[str, Any]: +def check_guardrail_grounding_threshold(inventory) -> Dict[str, Any]: """ FS-47 — Verify Bedrock Guardrails contextual grounding thresholds are set appropriately high for financial services use cases. @@ -3737,8 +4153,8 @@ def check_guardrail_grounding_threshold() -> Dict[str, Any]: """ findings = _empty_findings("Guardrail Grounding Threshold Check") try: - bedrock = boto3.client("bedrock", config=boto3_config) - guardrails = _paginate(bedrock, "list_guardrails", "guardrails") + guardrail_inv = require(inventory, "guardrails") + guardrails = guardrail_inv.summaries if not guardrails: findings["csv_data"].append( @@ -3748,7 +4164,7 @@ def check_guardrail_grounding_threshold() -> Dict[str, Any]: finding_details="No Bedrock Guardrails configured.", resolution="Configure guardrails with contextual grounding checks.", reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", - severity="High", + severity="Informational", status="N/A", compliance_frameworks=COMPLIANCE_MAP["FS-47"], ) @@ -3758,9 +4174,9 @@ def check_guardrail_grounding_threshold() -> Dict[str, Any]: low_threshold_guardrails = [] guardrails_with_grounding = [] for g in guardrails: - detail = bedrock.get_guardrail( - guardrailIdentifier=g["id"], guardrailVersion="DRAFT" - ) + detail = guardrail_inv.detail_by_id[g["id"]] + if isinstance(detail, _Unavailable): + raise detail.error grounding = detail.get("contextualGroundingPolicy", {}) has_grounding_filter = False for filter_item in grounding.get("filters", []): @@ -3844,7 +4260,7 @@ def check_guardrail_grounding_threshold() -> Dict[str, Any]: return findings -def check_rag_knowledge_base_configured() -> Dict[str, Any]: +def check_rag_knowledge_base_configured(inventory) -> Dict[str, Any]: """ FS-48 — Verify RAG (Retrieval Augmented Generation) is used via Bedrock Knowledge Bases to ground responses in authoritative data. @@ -3852,11 +4268,7 @@ def check_rag_knowledge_base_configured() -> Dict[str, Any]: """ findings = _empty_findings("RAG Knowledge Base Configuration Check") try: - bedrock_agent = boto3.client("bedrock-agent", config=boto3_config) - paginator = bedrock_agent.get_paginator("list_knowledge_bases") - kbs = [] - for page in paginator.paginate(): - kbs.extend(page.get("knowledgeBaseSummaries", [])) + kbs = require(inventory, "knowledge_bases").summaries active_kbs = [k for k in kbs if k.get("status") == "ACTIVE"] @@ -3930,7 +4342,7 @@ def check_hallucination_disclaimer_advisory() -> Dict[str, Any]: return findings -def check_guardrail_relevance_grounding() -> Dict[str, Any]: +def check_guardrail_relevance_grounding(inventory) -> Dict[str, Any]: """ FS-50 — Check for Bedrock Guardrails contextual grounding RELEVANCE filters configured to detect and block responses that are not grounded in the context @@ -3947,14 +4359,14 @@ def check_guardrail_relevance_grounding() -> Dict[str, Any]: """ findings = _empty_findings("Guardrail Relevance Grounding Check") try: - bedrock = boto3.client("bedrock", config=boto3_config) - guardrails = _paginate(bedrock, "list_guardrails", "guardrails") + guardrail_inv = require(inventory, "guardrails") + guardrails = guardrail_inv.summaries guardrails_with_relevance = [] for g in guardrails: - detail = bedrock.get_guardrail( - guardrailIdentifier=g["id"], guardrailVersion="DRAFT" - ) + detail = guardrail_inv.detail_by_id[g["id"]] + if isinstance(detail, _Unavailable): + raise detail.error grounding = detail.get("contextualGroundingPolicy", {}) for f in grounding.get("filters", []): if f.get("type") == "RELEVANCE": @@ -4005,7 +4417,7 @@ def check_guardrail_relevance_grounding() -> Dict[str, Any]: return findings -def check_prompt_injection_input_validation() -> Dict[str, Any]: +def check_prompt_injection_input_validation(inventory) -> Dict[str, Any]: """ FS-51 — Check for Bedrock Guardrails prompt attack filters to detect and block prompt injection attempts. @@ -4013,8 +4425,8 @@ def check_prompt_injection_input_validation() -> Dict[str, Any]: """ findings = _empty_findings("Prompt Injection Input Validation Check") try: - bedrock = boto3.client("bedrock", config=boto3_config) - guardrails = _paginate(bedrock, "list_guardrails", "guardrails") + guardrail_inv = require(inventory, "guardrails") + guardrails = guardrail_inv.summaries if not guardrails: findings["csv_data"].append( @@ -4024,7 +4436,7 @@ def check_prompt_injection_input_validation() -> Dict[str, Any]: finding_details="No Bedrock Guardrails configured.", resolution="Configure guardrails with prompt attack filters.", reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", - severity="High", + severity="Informational", status="N/A", compliance_frameworks=COMPLIANCE_MAP["FS-51"], ) @@ -4034,9 +4446,9 @@ def check_prompt_injection_input_validation() -> Dict[str, Any]: guardrails_with_prompt_attack = [] guardrails_classic_tier_pa = [] for g in guardrails: - detail = bedrock.get_guardrail( - guardrailIdentifier=g["id"], guardrailVersion="DRAFT" - ) + detail = guardrail_inv.detail_by_id[g["id"]] + if isinstance(detail, _Unavailable): + raise detail.error content_policy = detail.get("contentPolicy", {}) for f in content_policy.get("filters", []): if f.get("type") == "PROMPT_ATTACK": @@ -4091,7 +4503,7 @@ def check_prompt_injection_input_validation() -> Dict[str, Any]: "STANDARD tier requires cross-region inference on the guardrail." ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-prompt-attack.html", - severity="Low", + severity="High", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-51"], ) @@ -4121,7 +4533,7 @@ def check_prompt_injection_input_validation() -> Dict[str, Any]: return findings -def check_bedrock_sdk_version_currency() -> Dict[str, Any]: +def check_bedrock_sdk_version_currency(inventory) -> Dict[str, Any]: """ FS-52 — Advisory check: verify Bedrock SDK versions in Lambda functions are current (outdated SDKs may lack prompt injection mitigations). @@ -4129,8 +4541,7 @@ def check_bedrock_sdk_version_currency() -> Dict[str, Any]: """ findings = _empty_findings("Bedrock SDK Version Currency Check") try: - lambda_client = boto3.client("lambda", config=boto3_config) - functions = _paginate(lambda_client, "list_functions", "Functions") + functions = require(inventory, "lambda_functions") bedrock_functions = [ f @@ -4242,7 +4653,7 @@ def check_bedrock_sdk_version_currency() -> Dict[str, Any]: return findings -def check_waf_sql_injection_rules() -> Dict[str, Any]: +def check_waf_sql_injection_rules(inventory) -> Dict[str, Any]: """ FS-53 — Verify WAF Web ACLs include SQL injection and XSS managed rules to protect GenAI API endpoints from injection attacks. @@ -4250,8 +4661,10 @@ def check_waf_sql_injection_rules() -> Dict[str, Any]: """ findings = _empty_findings("WAF Injection Protection Rules Check") try: - wafv2 = boto3.client("wafv2", config=boto3_config) - acls = wafv2.list_web_acls(Scope="REGIONAL").get("WebACLs", []) + # require() raises if inventory is None or web_acls is _Unavailable, + # propagating to the outer except which yields COULD_NOT_ASSESS. + web_acl_inv = require(inventory, "web_acls") + acls = web_acl_inv.summaries if not acls: findings["csv_data"].append( @@ -4261,7 +4674,7 @@ def check_waf_sql_injection_rules() -> Dict[str, Any]: finding_details="No regional WAF Web ACLs found.", resolution="Create WAF Web ACLs with injection protection rules (see FS-01).", reference="https://docs.aws.amazon.com/waf/latest/developerguide/aws-managed-rule-groups-baseline.html", - severity="High", + severity="Informational", status="N/A", compliance_frameworks=COMPLIANCE_MAP["FS-53"], ) @@ -4276,11 +4689,14 @@ def check_waf_sql_injection_rules() -> Dict[str, Any]: acls_without_injection_rules = [] for acl_summary in acls: - acl = wafv2.get_web_acl( - Name=acl_summary["Name"], - Scope="REGIONAL", - Id=acl_summary["Id"], - ).get("WebACL", {}) + # detail_by_id holds get_web_acl(...)['WebACL'] or _Unavailable. + # Accessing an _Unavailable entry re-raises its stored error, which + # propagates to the outer except and yields COULD_NOT_ASSESS — + # matching today's behaviour (no per-item try/except in this check). + detail = web_acl_inv.detail_by_id[acl_summary["Id"]] + if isinstance(detail, _Unavailable): + raise detail.error + acl = detail rule_names = { r.get("Statement", {}) .get("ManagedRuleGroupStatement", {}) @@ -4377,7 +4793,7 @@ def check_penetration_testing_evidence() -> Dict[str, Any]: # =========================================================================== -def check_output_validation_lambda() -> Dict[str, Any]: +def check_output_validation_lambda(inventory) -> Dict[str, Any]: """ FS-55 — Check for Lambda functions implementing output validation/sanitization in GenAI application pipelines. @@ -4385,8 +4801,7 @@ def check_output_validation_lambda() -> Dict[str, Any]: """ findings = _empty_findings("Output Validation Lambda Check") try: - lambda_client = boto3.client("lambda", config=boto3_config) - functions = _paginate(lambda_client, "list_functions", "Functions") + functions = require(inventory, "lambda_functions") validation_functions = [ f @@ -4437,15 +4852,16 @@ def check_output_validation_lambda() -> Dict[str, Any]: return findings -def check_xss_prevention_waf() -> Dict[str, Any]: +def check_xss_prevention_waf(inventory) -> Dict[str, Any]: """ FS-56 — Verify WAF rules include XSS prevention for GenAI web application outputs. COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, FFIEC CAT, PCI-DSS 6.4.1, OWASP LLM02] """ findings = _empty_findings("XSS Prevention WAF Check") try: - wafv2 = boto3.client("wafv2", config=boto3_config) - acls = wafv2.list_web_acls(Scope="REGIONAL").get("WebACLs", []) + # require() raises if inventory is None or web_acls is _Unavailable. + web_acl_inv = require(inventory, "web_acls") + acls = web_acl_inv.summaries if not acls: findings["csv_data"].append( @@ -4455,33 +4871,75 @@ def check_xss_prevention_waf() -> Dict[str, Any]: finding_details="No regional WAF Web ACLs found.", resolution="Create WAF ACLs with XSS prevention rules.", reference="https://docs.aws.amazon.com/waf/latest/developerguide/aws-managed-rule-groups-baseline.html", - severity="High", + severity="Informational", status="N/A", compliance_frameworks=COMPLIANCE_MAP["FS-56"], ) ) return findings - # XSS is covered by AWSManagedRulesCommonRuleSet — reuse FS-53 logic - findings["csv_data"].append( - create_finding( - check_id="FS-56", - finding_name="XSS Prevention — Review WAF Common Rule Set", - finding_details=( - f"Found {len(acls)} WAF ACL(s). " - "Verify AWSManagedRulesCommonRuleSet is enabled for XSS prevention (see FS-53)." - ), - resolution=( - "Ensure AWSManagedRulesCommonRuleSet is enabled on all WAF ACLs " - "protecting GenAI web applications. " - "Additionally, implement Content Security Policy (CSP) headers." - ), - reference="https://docs.aws.amazon.com/waf/latest/developerguide/aws-managed-rule-groups-baseline.html", - severity="Medium", - status="Passed", - compliance_frameworks=COMPLIANCE_MAP["FS-56"], + # XSS protections live in the AWS managed Common Rule Set. Inspect each ACL's + # managed-rule-group statements (mirrors FS-53) to actually verify coverage + # rather than emitting an unconditional "review required" pass. + acls_without_xss = [] + for acl_summary in acls: + # detail_by_id holds get_web_acl(...)['WebACL'] or _Unavailable. + # An _Unavailable entry re-raises its error → outer except → COULD_NOT_ASSESS. + detail = web_acl_inv.detail_by_id[acl_summary["Id"]] + if isinstance(detail, _Unavailable): + raise detail.error + acl = detail + rule_groups = { + r.get("Statement", {}) + .get("ManagedRuleGroupStatement", {}) + .get("Name", "") + for r in acl.get("Rules", []) + } + if "AWSManagedRulesCommonRuleSet" not in rule_groups: + acls_without_xss.append(acl_summary["Name"]) + + if acls_without_xss: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-56", + finding_name="WAF ACLs Missing Common Rule Set (XSS)", + finding_details=( + "The following WAF ACL(s) do not include AWSManagedRulesCommonRuleSet, " + "which provides cross-site-scripting (XSS) protections for GenAI web " + "application outputs:\n" + + "\n".join(f"- {a}" for a in acls_without_xss[:10]) + ), + resolution=( + "1. Add AWSManagedRulesCommonRuleSet to each WAF ACL protecting GenAI " + "web applications (it includes the CrossSiteScripting rules).\n" + "2. Additionally, implement Content Security Policy (CSP) response headers." + ), + reference="https://docs.aws.amazon.com/waf/latest/developerguide/aws-managed-rule-groups-baseline.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-56"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-56", + finding_name="XSS Prevention Common Rule Set Present", + finding_details=( + f"All {len(acls)} WAF ACL(s) include AWSManagedRulesCommonRuleSet " + "(XSS protections)." + ), + resolution=( + "No action required. Consider also implementing Content Security Policy " + "(CSP) response headers as defense in depth." + ), + reference="https://docs.aws.amazon.com/waf/latest/developerguide/aws-managed-rule-groups-baseline.html", + severity="Medium", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-56"], + ) ) - ) except Exception as e: return _error_findings("XSS Prevention WAF Check", e) return findings @@ -4517,7 +4975,7 @@ def check_output_encoding_advisory() -> Dict[str, Any]: return findings -def check_output_schema_validation() -> Dict[str, Any]: +def check_output_schema_validation(inventory) -> Dict[str, Any]: """ FS-58 — Check for structured output validation using Bedrock response schemas or application-level JSON schema validation. @@ -4526,8 +4984,7 @@ def check_output_schema_validation() -> Dict[str, Any]: findings = _empty_findings("Output Schema Validation Check") try: # Check for EventBridge Pipes or Lambda destinations that could validate outputs - lambda_client = boto3.client("lambda", config=boto3_config) - functions = _paginate(lambda_client, "list_functions", "Functions") + functions = require(inventory, "lambda_functions") schema_functions = [ f @@ -4541,10 +4998,12 @@ def check_output_schema_validation() -> Dict[str, Any]: findings["csv_data"].append( create_finding( check_id="FS-58", - finding_name="Output Schema Validation — Review Required", + finding_name="ADVISORY: Output Schema Validation — Manual Review Required", finding_details=( - f"Found {len(schema_functions)} potential schema validation function(s). " - "Verify structured output validation is implemented for all GenAI responses." + f"Found {len(schema_functions)} Lambda function(s) whose names suggest " + "schema/validation handling. Structured-output / JSON-schema validation of " + "GenAI responses is an application-layer control that cannot be verified " + "automatically — manual review required." ), resolution=( "1. Use Bedrock structured output (response schemas) where supported.\n" @@ -4553,8 +5012,8 @@ def check_output_schema_validation() -> Dict[str, Any]: "4. Log schema validation failures to CloudWatch for monitoring." ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/inference-parameters.html", - severity="Medium", - status="Passed", + severity="Informational", + status="N/A", compliance_frameworks=COMPLIANCE_MAP["FS-58"], ) ) @@ -4563,7 +5022,7 @@ def check_output_schema_validation() -> Dict[str, Any]: return findings -def check_guardrail_topic_allowlist() -> Dict[str, Any]: +def check_guardrail_topic_allowlist(inventory) -> Dict[str, Any]: """ FS-59 — Verify Bedrock Guardrails topic policies restrict GenAI to on-topic financial services responses only. @@ -4571,8 +5030,8 @@ def check_guardrail_topic_allowlist() -> Dict[str, Any]: """ findings = _empty_findings("Guardrail Topic Allowlist Check") try: - bedrock = boto3.client("bedrock", config=boto3_config) - guardrails = _paginate(bedrock, "list_guardrails", "guardrails") + guardrail_inv = require(inventory, "guardrails") + guardrails = guardrail_inv.summaries if not guardrails: findings["csv_data"].append( @@ -4582,7 +5041,7 @@ def check_guardrail_topic_allowlist() -> Dict[str, Any]: finding_details="No Bedrock Guardrails configured.", resolution="Configure guardrails with topic policies to restrict off-topic responses.", reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", - severity="Medium", + severity="Informational", status="N/A", compliance_frameworks=COMPLIANCE_MAP["FS-59"], ) @@ -4592,9 +5051,9 @@ def check_guardrail_topic_allowlist() -> Dict[str, Any]: guardrails_with_topics = [] topics_classic_tier = [] for g in guardrails: - detail = bedrock.get_guardrail( - guardrailIdentifier=g["id"], guardrailVersion="DRAFT" - ) + detail = guardrail_inv.detail_by_id[g["id"]] + if isinstance(detail, _Unavailable): + raise detail.error topic_policy = detail.get("topicPolicy", {}) if topic_policy.get("topics"): guardrails_with_topics.append(g["name"]) @@ -4644,7 +5103,7 @@ def check_guardrail_topic_allowlist() -> Dict[str, Any]: "requires a cross-region inference profile)." ), reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", - severity="Low", + severity="Medium", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-59"], ) @@ -4698,7 +5157,7 @@ def check_contextual_grounding_for_offtopic() -> Dict[str, Any]: return findings -def check_knowledge_base_sync_schedule() -> Dict[str, Any]: +def check_knowledge_base_sync_schedule(inventory) -> Dict[str, Any]: """ FS-61 — Verify Bedrock Knowledge Base data sources have automated sync schedules to keep training/retrieval data current. @@ -4707,15 +5166,10 @@ def check_knowledge_base_sync_schedule() -> Dict[str, Any]: # Reuses logic from FS-31 but focuses on scheduled automation findings = _empty_findings("Knowledge Base Sync Schedule Check") try: - bedrock_agent = boto3.client("bedrock-agent", config=boto3_config) + kbs = require(inventory, "knowledge_bases").summaries events = boto3.client("events", config=boto3_config) scheduler = boto3.client("scheduler", config=boto3_config) - paginator = bedrock_agent.get_paginator("list_knowledge_bases") - kbs = [] - for page in paginator.paginate(): - kbs.extend(page.get("knowledgeBaseSummaries", [])) - if not kbs: findings["csv_data"].append( create_finding( @@ -4747,6 +5201,7 @@ def check_knowledge_base_sync_schedule() -> Dict[str, Any]: # NextToken. Treat an access error as a soft signal (do not fail the whole # check) since Scheduler may not be in use. kb_sync_schedules = [] + scheduler_access_error = None try: schedules = _paginate(scheduler, "list_schedules", "Schedules") kb_sync_schedules = [ @@ -4761,14 +5216,25 @@ def check_knowledge_base_sync_schedule() -> Dict[str, Any]: except ClientError as e: if not _is_access_error(e): raise + # Remember the access error. If we ALSO find no EventBridge-rule + # automation, we cannot conclude the control is absent (a Scheduler + # schedule we were not allowed to read might exist) — so we surface + # COULD_NOT_ASSESS rather than a false Failed. If a rule IS found, the + # positive evidence stands and the access gap is immaterial. + scheduler_access_error = e logger.warning( "Could not list EventBridge Scheduler schedules (access denied); " - "FS-61 fell back to EventBridge rules only. Grant " - "scheduler:ListSchedules for full coverage." + "grant scheduler:ListSchedules for full FS-61 coverage." ) total_sync_automation = len(kb_sync_rules) + len(kb_sync_schedules) + # No positive evidence AND we were blocked from reading Scheduler → + # unknown state, not a confirmed failure. Re-raise so the handler emits a + # COULD NOT ASSESS row (Status="N/A", Severity="Low"). + if total_sync_automation == 0 and scheduler_access_error is not None: + raise scheduler_access_error + if total_sync_automation == 0: findings["status"] = "WARN" findings["csv_data"].append( @@ -4937,7 +5403,7 @@ def check_foundation_model_lifecycle_policy() -> Dict[str, Any]: # =========================================================================== -def check_kb_datasource_s3_event_notifications() -> Dict[str, Any]: +def check_kb_datasource_s3_event_notifications(inventory) -> Dict[str, Any]: """ FS-65 — Check that S3 event notifications (EventBridge or SNS/SQS) are configured on Knowledge Base data-source buckets to detect unauthorized @@ -4946,13 +5412,10 @@ def check_kb_datasource_s3_event_notifications() -> Dict[str, Any]: """ findings = _empty_findings("KB Data Source S3 Event Notifications Check") try: - bedrock_agent = boto3.client("bedrock-agent", config=boto3_config) + kb_inv = require(inventory, "knowledge_bases") + kbs = kb_inv.summaries s3_client = boto3.client("s3", config=boto3_config) - paginator = bedrock_agent.get_paginator("list_knowledge_bases") - kbs = [] - for page in paginator.paginate(): - kbs.extend(page.get("knowledgeBaseSummaries", [])) if not kbs: findings["csv_data"].append( create_finding( @@ -4972,17 +5435,16 @@ def check_kb_datasource_s3_event_notifications() -> Dict[str, Any]: missing_buckets = [] for kb in kbs: kb_id = kb["knowledgeBaseId"] - data_sources = _paginate( - bedrock_agent, - "list_data_sources", - "dataSourceSummaries", - knowledgeBaseId=kb_id, - ) + data_sources = kb_inv.data_sources_by_kb.get(kb_id, []) + if isinstance(data_sources, _Unavailable): + raise data_sources.error for ds in data_sources: - ds_detail = bedrock_agent.get_data_source( - knowledgeBaseId=kb_id, - dataSourceId=ds["dataSourceId"], - ) + ds_id = ds["dataSourceId"] + ds_detail = kb_inv.data_source_detail.get((kb_id, ds_id)) + if isinstance(ds_detail, _Unavailable): + raise ds_detail.error + if ds_detail is None: + continue s3_config = ( ds_detail.get("dataSource", {}) .get("dataSourceConfiguration", {}) @@ -5183,7 +5645,7 @@ def check_agentcore_end_user_identity_propagation() -> Dict[str, Any]: finding_details=f"All {len(runtimes)} runtime(s) have authorizer configurations supporting identity propagation.", resolution="No action required.", reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/security-authorization.html", - severity="Low", + severity="High", status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-66"], ) @@ -5193,7 +5655,7 @@ def check_agentcore_end_user_identity_propagation() -> Dict[str, Any]: return findings -def check_agent_financial_transaction_thresholds() -> Dict[str, Any]: +def check_agent_financial_transaction_thresholds(inventory) -> Dict[str, Any]: """ FS-67 — Check AgentCore Policy Engine or action-group Lambda functions enforce maximum transaction-value limits to prevent runaway or unauthorized @@ -5202,8 +5664,7 @@ def check_agent_financial_transaction_thresholds() -> Dict[str, Any]: """ findings = _empty_findings("Agent Financial Transaction Value Thresholds Check") try: - lambda_client = boto3.client("lambda", config=boto3_config) - functions = _paginate(lambda_client, "list_functions", "Functions") + functions = require(inventory, "lambda_functions") # Look for agent action-group Lambda functions action_group_lambdas = [ @@ -5240,7 +5701,7 @@ def check_agent_financial_transaction_thresholds() -> Dict[str, Any]: "3. Reject or escalate to human review any transaction exceeding defined limits." ), reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/policy-engine.html", - severity="High", + severity="Informational", status="N/A", compliance_frameworks=COMPLIANCE_MAP["FS-67"], ) @@ -5309,88 +5770,200 @@ def check_agent_financial_transaction_thresholds() -> Dict[str, Any]: return findings -def check_api_gateway_request_body_size_limits() -> Dict[str, Any]: +# Default body-inspection window AWS WAF applies before oversize handling kicks +# in. For CloudFront/API Gateway/Cognito/App Runner/Verified Access the default +# is 16 KB (raisable to 64 KB via the web ACL AssociationConfig); for ALB/AppSync +# it is a fixed 8 KB. A GT/GE SizeConstraint above this window can only ever fire +# if the rule's OversizeHandling is MATCH. See: +# https://docs.aws.amazon.com/waf/latest/developerguide/waf-oversize-request-components.html +_WAF_DEFAULT_BODY_INSPECTION_LIMIT = 16384 + +# JSON-Schema keywords that actually bound request-body SIZE. A request validator +# only enforces a size cap when its model schema carries one of these — merely +# enabling validateRequestBody does NOT cap payload size (it validates the schema +# / required params only; the REST hard limit is a fixed, non-configurable 10 MB). +_SCHEMA_SIZE_KEYWORDS = ('"maxLength"', '"maxItems"', '"maxProperties"') + + +def _waf_statement_has_firing_body_size_constraint( + stmt: Any, inspection_limit: int = _WAF_DEFAULT_BODY_INSPECTION_LIMIT +) -> bool: + """True if a WAF rule Statement contains a SizeConstraintStatement on the + request Body/JsonBody that can actually fire. + + Recurses into And/Or/Not combinators. A GT/GE size threshold above the body + inspection window cannot fire unless OversizeHandling is MATCH, so such a rule + is NOT credited as a working body-size control (the documented WAF limitation). """ - FS-68 — Verify API Gateway REST/HTTP APIs fronting GenAI endpoints enforce - maximum input payload sizes via request validators or WAF body-size rules - to prevent token-exhaustion attacks via oversized prompts. + if not isinstance(stmt, dict): + return False + sc = stmt.get("SizeConstraintStatement") + if isinstance(sc, dict): + ftm = sc.get("FieldToMatch", {}) or {} + body = ftm.get("Body") + json_body = ftm.get("JsonBody") + target = body if isinstance(body, dict) else json_body + if isinstance(target, dict): + comparison = sc.get("ComparisonOperator", "") + size = sc.get("Size", 0) or 0 + oversize = target.get("OversizeHandling") + # A "block if body > N" rule with N beyond the inspection window only + # fires when oversize content is treated as a match. + if ( + comparison in ("GT", "GE") + and size > inspection_limit + and oversize != "MATCH" + ): + return False + return True + for combinator in ("AndStatement", "OrStatement"): + sub = stmt.get(combinator) + if isinstance(sub, dict): + for inner in sub.get("Statements", []) or []: + if _waf_statement_has_firing_body_size_constraint( + inner, inspection_limit + ): + return True + not_stmt = stmt.get("NotStatement") + if isinstance(not_stmt, dict): + if _waf_statement_has_firing_body_size_constraint( + not_stmt.get("Statement"), inspection_limit + ): + return True + return False + + +def _api_has_body_size_validator(apigw, api_id: str) -> bool: + """True only if a REST API has a request validator that validates the body + AND at least one model whose schema bounds body size (maxLength/maxItems/ + maxProperties). Validator presence alone is NOT sufficient — it does not cap + payload size (the bug FS-68 previously had).""" + validators = apigw.get_request_validators(restApiId=api_id).get("items", []) + if not any(v.get("validateRequestBody") for v in validators): + return False + for model in _paginate(apigw, "get_models", "items", restApiId=api_id): + schema = model.get("schema", "") or "" + if any(tok in schema for tok in _SCHEMA_SIZE_KEYWORDS): + return True + return False + + +def check_api_gateway_request_body_size_limits(inventory) -> Dict[str, Any]: + """ + FS-68 — Verify API Gateway REST APIs fronting GenAI endpoints actually enforce + a maximum input-payload size (to blunt token-exhaustion via oversized prompts). + + Important correctness note: an API Gateway request validator does NOT cap body + size — it validates required params + a JSON-Schema model, and the REST payload + limit is a fixed, non-configurable 10 MB. A real size cap requires either (a) a + validator model with a maxLength/maxItems/maxProperties bound, or (b) a WAF + SizeConstraintStatement on the request Body that can actually fire within WAF's + body-inspection window (default 16 KB for API Gateway). This check credits only + those evidenced controls — validator presence alone is not a pass. COMPLIANCE_PLACEHOLDER: [FFIEC CAT, DORA Art.6, PCI-DSS, OWASP LLM10] """ findings = _empty_findings("API Gateway Request Body Size Limits Check") try: apigw = boto3.client("apigateway", config=boto3_config) - wafv2 = boto3.client("wafv2", config=boto3_config) - # Check REST APIs for request validators rest_apis = _paginate(apigw, "get_rest_apis", "items") - apis_without_validators = [] - for api in rest_apis: - validators = apigw.get_request_validators(restApiId=api["id"]).get( - "items", [] + # require() raises if inventory is None or web_acls is _Unavailable. + web_acl_inv = require(inventory, "web_acls") + acls = web_acl_inv.summaries + + # Nothing-to-assess branch: no REST APIs AND no WAF ACLs means there is no + # input-payload surface to evaluate in this region. + if not rest_apis and not acls: + findings["csv_data"].append( + create_finding( + check_id="FS-68", + finding_name="API Gateway Request Body Size Limits — Not Applicable", + finding_details=( + "No API Gateway REST APIs and no regional WAF Web ACLs were found in this " + "region. There is no input-payload surface to assess for body-size limits." + ), + resolution=( + "If GenAI endpoints are fronted by API Gateway or WAF in another region, " + "run the assessment there. Otherwise no action is required." + ), + reference="https://docs.aws.amazon.com/waf/latest/developerguide/waf-rule-statement-type-size-constraint.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-68"], + ) ) - if not validators: - apis_without_validators.append(api.get("name", api["id"])) + return findings + + # Evidence 1: REST APIs whose validator model actually bounds body size. + apis_with_size_control = [] + for api in rest_apis: + if _api_has_body_size_validator(apigw, api["id"]): + apis_with_size_control.append(api.get("name", api["id"])) - # Check WAF rules for body size constraints - acls = wafv2.list_web_acls(Scope="REGIONAL").get("WebACLs", []) + # Evidence 2: regional WAF ACLs with a body SizeConstraint that can fire. + # detail_by_id holds get_web_acl(...)['WebACL'] or _Unavailable. + # An _Unavailable entry re-raises its error → outer except → COULD_NOT_ASSESS. acls_with_size_rules = 0 for acl in acls: - acl_detail = wafv2.get_web_acl( - Name=acl["Name"], Scope="REGIONAL", Id=acl["Id"] - ) - rules = acl_detail.get("WebACL", {}).get("Rules", []) - for rule in rules: - stmt = json.dumps(rule.get("Statement", {})) - if "SizeConstraintStatement" in stmt or "body" in stmt.lower(): - acls_with_size_rules += 1 - break + detail = web_acl_inv.detail_by_id[acl["Id"]] + if isinstance(detail, _Unavailable): + raise detail.error + rules = detail.get("Rules", []) + if any( + _waf_statement_has_firing_body_size_constraint(r.get("Statement", {})) + for r in rules + ): + acls_with_size_rules += 1 - issues = [] - if apis_without_validators: - issues.append( - f"REST APIs without request validators: {', '.join(apis_without_validators[:5])}" - ) - if acls and acls_with_size_rules == 0: - issues.append("No WAF Web ACLs have body-size constraint rules configured.") + has_size_control = bool(apis_with_size_control) or acls_with_size_rules > 0 - if issues: - findings["status"] = "WARN" + if has_size_control: findings["csv_data"].append( create_finding( check_id="FS-68", - finding_name="API Gateway Request Body Size Limits Not Enforced", + finding_name="API Gateway Request Body Size Limits Configured", finding_details=( - "Input payload size limits are not fully enforced on GenAI API endpoints. " - "Oversized prompts can exhaust Bedrock token quotas and inflate costs:\n" - + "\n".join(f"- {i}" for i in issues) - ), - resolution=( - "1. Add API Gateway request validators to enforce maximum body size on " - "all Bedrock-facing REST API methods.\n" - "2. Add a WAF SizeConstraintStatement rule to block requests with body " - "size exceeding your maximum prompt length (e.g., 32 KB).\n" - "3. Set the max_tokens parameter in Bedrock API calls to cap output length.\n" - "4. Implement client-side token counting before submitting requests." + f"Found {len(rest_apis)} REST API(s) " + f"({len(apis_with_size_control)} with a body-size-bounding validator model) " + f"and {acls_with_size_rules} WAF ACL(s) with a firing body-size constraint. " + "Verify the WAF ACL(s) are associated with the GenAI-facing API stages, " + "since this check does not confirm resource association." ), + resolution="No action required (verify WAF/API association as noted).", reference="https://docs.aws.amazon.com/waf/latest/developerguide/waf-rule-statement-type-size-constraint.html", severity="Medium", - status="Failed", + status="Passed", compliance_frameworks=COMPLIANCE_MAP["FS-68"], ) ) else: + findings["status"] = "WARN" findings["csv_data"].append( create_finding( check_id="FS-68", - finding_name="API Gateway Request Body Size Limits Configured", + finding_name="API Gateway Request Body Size Limits Not Enforced", finding_details=( - f"Found {len(rest_apis)} REST API(s) with validators and " - f"{acls_with_size_rules} WAF ACL(s) with body-size rules." + f"Found {len(rest_apis)} REST API(s) and {len(acls)} regional WAF Web " + "ACL(s), but none enforce a maximum request-body size. Note: an API " + "Gateway request validator does NOT cap body size (it validates the schema " + "and required params; the REST limit is a fixed 10 MB), and a WAF body " + "SizeConstraint only inspects the first ~16 KB of the body by default. " + "Oversized prompts can exhaust Bedrock token quotas and inflate costs." ), - resolution="No action required.", - reference="https://docs.aws.amazon.com/waf/latest/developerguide/waf-rule-statement-type-size-constraint.html", + resolution=( + "1. Add a maxLength (or maxItems/maxProperties) bound to the request-body " + "JSON-Schema model used by your request validator, so oversized prompts are " + "rejected with a 400.\n" + "2. Add a WAF SizeConstraintStatement on the request Body sized within WAF's " + "body-inspection window (default 16 KB; raise via the web ACL " + "AssociationConfig, or set OversizeHandling=MATCH to block bodies beyond the " + "window), and associate the ACL with the API stage.\n" + "3. Set the max_tokens parameter in Bedrock API calls to cap output length.\n" + "4. Implement client-side token counting before submitting requests." + ), + reference="https://docs.aws.amazon.com/waf/latest/developerguide/waf-oversize-request-components.html", severity="Medium", - status="Passed", + status="Failed", compliance_frameworks=COMPLIANCE_MAP["FS-68"], ) ) @@ -5399,7 +5972,7 @@ def check_api_gateway_request_body_size_limits() -> Dict[str, Any]: return findings -def check_prompt_input_validation_function() -> Dict[str, Any]: +def check_prompt_input_validation_function(inventory) -> Dict[str, Any]: """ FS-69 — Check for a Lambda function or API Gateway request validator that sanitizes user prompt input (strips special characters, enforces expected @@ -5408,8 +5981,7 @@ def check_prompt_input_validation_function() -> Dict[str, Any]: """ findings = _empty_findings("Prompt Input Validation Function Check") try: - lambda_client = boto3.client("lambda", config=boto3_config) - functions = _paginate(lambda_client, "list_functions", "Functions") + functions = require(inventory, "lambda_functions") # Look for Lambda functions with input validation / sanitization naming patterns VALIDATION_KEYWORDS = [ @@ -5518,7 +6090,177 @@ def write_to_s3(execution_id: str, csv_content: str, bucket_name: str) -> str: return f"https://{bucket_name}.s3.amazonaws.com/{file_name}" -def build_finserv_checks(permission_cache): +# --------------------------------------------------------------------------- +# Inventory collector (REQ-1, REQ-2, REQ-3, REQ-4, REQ-7) +# --------------------------------------------------------------------------- + + +def _safe_collect_lambda_functions(): + """Collect all Lambda functions via list_functions (fully paginated). + Returns a list on success, or _Unavailable(exc) on any failure.""" + try: + client = boto3.client("lambda", config=boto3_config) + return _paginate(client, "list_functions", "Functions") + except Exception as e: + logger.warning( + "inventory:lambda_functions collection failed: %s", type(e).__name__ + ) + return _Unavailable(e) + + +def _safe_collect_guardrails(): + """Collect Bedrock guardrail summaries and per-guardrail DRAFT detail. + A single-guardrail detail failure is recorded as _Unavailable for that id + only — it does NOT abort the whole guardrail inventory.""" + try: + client = boto3.client("bedrock", config=boto3_config) + summaries = _paginate(client, "list_guardrails", "guardrails") + detail_by_id: dict = {} + for g in summaries: + gid = g["id"] + try: + detail_by_id[gid] = client.get_guardrail( + guardrailIdentifier=gid, guardrailVersion="DRAFT" + ) + except Exception as e: + logger.warning( + "inventory:guardrails detail for %s failed: %s", + gid, + type(e).__name__, + ) + detail_by_id[gid] = _Unavailable(e) + return GuardrailInventory(summaries=summaries, detail_by_id=detail_by_id) + except Exception as e: + logger.warning("inventory:guardrails collection failed: %s", type(e).__name__) + return _Unavailable(e) + + +def _safe_collect_knowledge_bases(): + """Collect Bedrock Knowledge Base summaries, per-KB data-source summaries, + and per-data-source detail. Per-KB and per-data-source failures are + recorded as _Unavailable without aborting the rest of the collection.""" + try: + client = boto3.client("bedrock-agent", config=boto3_config) + summaries = _paginate(client, "list_knowledge_bases", "knowledgeBaseSummaries") + data_sources_by_kb: dict = {} + data_source_detail: dict = {} + for kb in summaries: + kb_id = kb["knowledgeBaseId"] + try: + ds_summaries = _paginate( + client, + "list_data_sources", + "dataSourceSummaries", + knowledgeBaseId=kb_id, + ) + data_sources_by_kb[kb_id] = ds_summaries + for ds in ds_summaries: + ds_id = ds["dataSourceId"] + try: + data_source_detail[(kb_id, ds_id)] = client.get_data_source( + knowledgeBaseId=kb_id, dataSourceId=ds_id + ) + except Exception as e: + logger.warning( + "inventory:knowledge_bases data_source detail (%s, %s) failed: %s", + kb_id, + ds_id, + type(e).__name__, + ) + data_source_detail[(kb_id, ds_id)] = _Unavailable(e) + except Exception as e: + logger.warning( + "inventory:knowledge_bases data_sources for KB %s failed: %s", + kb_id, + type(e).__name__, + ) + data_sources_by_kb[kb_id] = _Unavailable(e) + return KbInventory( + summaries=summaries, + data_sources_by_kb=data_sources_by_kb, + data_source_detail=data_source_detail, + ) + except Exception as e: + logger.warning( + "inventory:knowledge_bases collection failed: %s", type(e).__name__ + ) + return _Unavailable(e) + + +def _safe_collect_buckets(): + """Collect the full account S3 bucket list with explicit ContinuationToken + pagination. MaxBuckets=1000 ensures pagination is always engaged so accounts + above the 10,000-bucket quota (where unpaginated requests are rejected) succeed.""" + try: + client = boto3.client("s3", config=boto3_config) + return _paginate( + client, + "list_buckets", + "Buckets", + token=("ContinuationToken", "ContinuationToken"), + MaxBuckets=1000, + ) + except Exception as e: + logger.warning("inventory:buckets collection failed: %s", type(e).__name__) + return _Unavailable(e) + + +def _safe_collect_web_acls(): + """Collect WAFv2 REGIONAL Web ACL summaries and per-ACL detail. Uses + explicit NextMarker/NextMarker pagination (WAFv2 input ≠ Lambda Marker). + Per-ACL detail failures are recorded as _Unavailable for that id only.""" + try: + client = boto3.client("wafv2", config=boto3_config) + summaries = _paginate( + client, + "list_web_acls", + "WebACLs", + token=("NextMarker", "NextMarker"), + Scope="REGIONAL", + ) + detail_by_id: dict = {} + for acl in summaries: + acl_id = acl["Id"] + try: + # Store the WebACL dict directly (response["WebACL"]), not the + # full envelope, so consuming checks can do detail.get("Rules") + # without an extra ["WebACL"] indirection (matches design §3). + resp = client.get_web_acl(Name=acl["Name"], Scope="REGIONAL", Id=acl_id) + detail_by_id[acl_id] = resp["WebACL"] + except Exception as e: + logger.warning( + "inventory:web_acls detail for %s failed: %s", + acl_id, + type(e).__name__, + ) + detail_by_id[acl_id] = _Unavailable(e) + return WebAclInventory(summaries=summaries, detail_by_id=detail_by_id) + except Exception as e: + logger.warning("inventory:web_acls collection failed: %s", type(e).__name__) + return _Unavailable(e) + + +def collect_resource_inventory() -> ResourceInventory: + """Collect each shared inventory at most once per invocation. + + Each inventory is isolated: a failure yields an ``_Unavailable`` sentinel + for that field without aborting the others (REQ-4, INV-5). All five + inventories are always collected because the current registry always + consumes all five (design DD-7). + + All clients use ``boto3.client(service, config=boto3_config)`` — no + ``region_name`` or ``endpoint_url`` — so the Lambda's default-Region + resolution is preserved (REQ-7, INV-4).""" + return ResourceInventory( + lambda_functions=_safe_collect_lambda_functions(), + guardrails=_safe_collect_guardrails(), + knowledge_bases=_safe_collect_knowledge_bases(), + buckets=_safe_collect_buckets(), + web_acls=_safe_collect_web_acls(), + ) + + +def build_finserv_checks(permission_cache, inventory=None): """ Single source of truth: ordered (check_id, zero-arg callable) registry of all FinServ checks, in execution order (FS-01 → FS-69, skipping the ids @@ -5528,10 +6270,18 @@ def build_finserv_checks(permission_cache): Driving the handler from this registry lets us attach the correct Check_ID to a synthesized "could not assess" row when a check errors out, instead of silently dropping the check from the report. + + ``inventory`` is optional (defaults to ``None``) so that existing one-arg + call sites — e.g. tests/test_severity_register.py — continue to work + without modification (DD-2b). ``lambda_handler`` always passes a real + ``ResourceInventory``; consuming checks will be bound to it in Wave 3. + FS-21 and FS-46 (S3-inventory consumers) are bound to ``inventory`` here. + Guardrail consumers (FS-27a, 28, 36, 38, 45, 47, 50, 51, 59) are now + bound to ``inventory`` as well (Task 7). """ return [ # --- Category 1: Unbounded Consumption --- - ("FS-01", check_waf_shield_on_bedrock_endpoints), + ("FS-01", functools.partial(check_waf_shield_on_bedrock_endpoints, inventory)), ("FS-02", check_api_gateway_rate_limiting), ("FS-03", check_bedrock_token_quotas), ("FS-04", check_cost_anomaly_detection), @@ -5543,7 +6293,7 @@ def build_finserv_checks(permission_cache): functools.partial(check_bedrock_agent_action_boundaries, permission_cache), ), ("FS-08", check_agentcore_policy_engine), - ("FS-09", check_agent_transaction_limits), + ("FS-09", functools.partial(check_agent_transaction_limits, inventory)), ("FS-10", check_human_in_the_loop_for_high_risk_actions), ("FS-11", check_agent_rate_alarms), # --- Category 3: Supply Chain Vulnerabilities --- @@ -5554,7 +6304,7 @@ def build_finserv_checks(permission_cache): ("FS-16", check_ecr_image_scanning), # --- Category 4: Training Data & Model Poisoning --- ("FS-20", check_feature_store_rollback_capability), - ("FS-21", check_training_data_s3_versioning), + ("FS-21", functools.partial(check_training_data_s3_versioning, inventory)), # --- Category 5: Vector & Embedding Weaknesses --- ( "FS-22", @@ -5562,28 +6312,37 @@ def build_finserv_checks(permission_cache): check_knowledge_base_iam_least_privilege, permission_cache ), ), - ("FS-24", check_knowledge_base_metadata_filtering), + ( + "FS-24", + functools.partial(check_knowledge_base_metadata_filtering, inventory), + ), ("FS-25", check_opensearch_serverless_encryption), ("FS-26", check_knowledge_base_vpc_access), # --- Category 6: Non-Compliant Output --- # FS-27 is split into two checks: contextual grounding (threshold-based) and # Automated Reasoning policies (formal-verification, GA August 2025). Both # use the FS-27 check_id so they appear together in the CSV report. - ("FS-27", check_guardrail_contextual_grounding), + ("FS-27", functools.partial(check_guardrail_contextual_grounding, inventory)), ("FS-27", check_automated_reasoning_policies), - ("FS-28", check_guardrail_denied_topics_financial), + ( + "FS-28", + functools.partial(check_guardrail_denied_topics_financial, inventory), + ), ("FS-29", check_compliance_disclaimer_in_outputs), ("FS-30", check_bedrock_evaluation_compliance_datasets), # --- Category 7: Misinformation --- - ("FS-31", check_knowledge_base_data_source_sync), + ("FS-31", functools.partial(check_knowledge_base_data_source_sync, inventory)), ("FS-32", check_source_attribution_in_guardrails), - ("FS-33", check_knowledge_base_integrity_monitoring), + ( + "FS-33", + functools.partial(check_knowledge_base_integrity_monitoring, inventory), + ), ("FS-34", check_fm_version_currency), # --- Category 8: Abusive or Harmful Output --- ("FS-35", check_fmeval_harmful_content), - ("FS-36", check_guardrail_content_filters), + ("FS-36", functools.partial(check_guardrail_content_filters, inventory)), ("FS-37", check_user_feedback_mechanism), - ("FS-38", check_guardrail_word_filters), + ("FS-38", functools.partial(check_guardrail_word_filters, inventory)), # --- Category 9: Biased Output --- ("FS-39", check_sagemaker_clarify_bias), ("FS-40", check_bedrock_evaluation_bias_datasets), @@ -5592,36 +6351,48 @@ def build_finserv_checks(permission_cache): # --- Category 10: Sensitive Information Disclosure --- ("FS-43", check_cloudwatch_log_pii_masking), ("FS-44", check_macie_on_training_data_buckets), - ("FS-45", check_guardrail_pii_filters), - ("FS-46", check_data_classification_tagging), + ("FS-45", functools.partial(check_guardrail_pii_filters, inventory)), + ("FS-46", functools.partial(check_data_classification_tagging, inventory)), # --- Category 11: Hallucination --- - ("FS-47", check_guardrail_grounding_threshold), - ("FS-48", check_rag_knowledge_base_configured), + ("FS-47", functools.partial(check_guardrail_grounding_threshold, inventory)), + ("FS-48", functools.partial(check_rag_knowledge_base_configured, inventory)), ("FS-49", check_hallucination_disclaimer_advisory), - ("FS-50", check_guardrail_relevance_grounding), + ("FS-50", functools.partial(check_guardrail_relevance_grounding, inventory)), # --- Category 12: Prompt Injection --- - ("FS-51", check_prompt_injection_input_validation), - ("FS-52", check_bedrock_sdk_version_currency), - ("FS-53", check_waf_sql_injection_rules), + ( + "FS-51", + functools.partial(check_prompt_injection_input_validation, inventory), + ), + ("FS-52", functools.partial(check_bedrock_sdk_version_currency, inventory)), + ("FS-53", functools.partial(check_waf_sql_injection_rules, inventory)), ("FS-54", check_penetration_testing_evidence), # --- Category 13: Improper Output Handling --- - ("FS-55", check_output_validation_lambda), - ("FS-56", check_xss_prevention_waf), + ("FS-55", functools.partial(check_output_validation_lambda, inventory)), + ("FS-56", functools.partial(check_xss_prevention_waf, inventory)), ("FS-57", check_output_encoding_advisory), - ("FS-58", check_output_schema_validation), + ("FS-58", functools.partial(check_output_schema_validation, inventory)), # --- Category 14: Off-Topic & Inappropriate Output --- - ("FS-59", check_guardrail_topic_allowlist), + ("FS-59", functools.partial(check_guardrail_topic_allowlist, inventory)), ("FS-60", check_contextual_grounding_for_offtopic), # --- Category 15: Out-of-Date Training Data --- - ("FS-61", check_knowledge_base_sync_schedule), + ("FS-61", functools.partial(check_knowledge_base_sync_schedule, inventory)), ("FS-62", check_data_currency_disclaimer_advisory), ("FS-63", check_foundation_model_lifecycle_policy), # --- Material Gap Checks (FS-65 to FS-69) --- - ("FS-65", check_kb_datasource_s3_event_notifications), + ( + "FS-65", + functools.partial(check_kb_datasource_s3_event_notifications, inventory), + ), ("FS-66", check_agentcore_end_user_identity_propagation), - ("FS-67", check_agent_financial_transaction_thresholds), - ("FS-68", check_api_gateway_request_body_size_limits), - ("FS-69", check_prompt_input_validation_function), + ( + "FS-67", + functools.partial(check_agent_financial_transaction_thresholds, inventory), + ), + ( + "FS-68", + functools.partial(check_api_gateway_request_body_size_limits, inventory), + ), + ("FS-69", functools.partial(check_prompt_input_validation_function, inventory)), ] @@ -5641,6 +6412,7 @@ def lambda_handler(event, context): "role_permissions": {}, "user_permissions": {}, } + inventory = collect_resource_inventory() # NEW: once per invocation # Run every check from the registry. If a check produces no rows for ANY # reason (an ERROR envelope, or an unexpected empty non-error result), @@ -5648,7 +6420,7 @@ def lambda_handler(event, context): # gap surfaces in the report instead of the check silently vanishing. The # guard intentionally keys off empty csv_data (not just status=="ERROR") so # the no-silent-drop invariant holds structurally, not by data coincidence. - for check_id, check_fn in build_finserv_checks(permission_cache): + for check_id, check_fn in build_finserv_checks(permission_cache, inventory): result = check_fn() if not result.get("csv_data"): details = result.get("details", "") or ( diff --git a/aiml-security-assessment/functions/security/finserv_tests/__init__.py b/aiml-security-assessment/functions/security/finserv_tests/__init__.py new file mode 100644 index 0000000..65140f2 --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_tests/__init__.py @@ -0,0 +1 @@ +# tests package diff --git a/aiml-security-assessment/functions/security/finserv_tests/conftest.py b/aiml-security-assessment/functions/security/finserv_tests/conftest.py new file mode 100644 index 0000000..7107ac3 --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_tests/conftest.py @@ -0,0 +1,135 @@ +""" +Shared fixtures and mock helpers for finserv_assessments tests. + +All boto3 clients are patched at the module level so check functions +never make real AWS API calls. +""" + +import os +import sys + +import pytest + +# --------------------------------------------------------------------------- +# Make finserv_assessments importable — the Lambda runtime adds the package +# root to sys.path, so we replicate that here. +# --------------------------------------------------------------------------- +FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") +if FINSERV_DIR not in sys.path: + sys.path.insert(0, FINSERV_DIR) + +import app # noqa: E402 (imported after sys.path manipulation) + + +# --------------------------------------------------------------------------- +# Environment variables expected by the Lambda +# --------------------------------------------------------------------------- +@pytest.fixture(autouse=True) +def _set_env(monkeypatch): + monkeypatch.setenv("AIML_ASSESSMENT_BUCKET_NAME", "test-bucket") + + +# --------------------------------------------------------------------------- +# A reusable permission_cache fixture +# --------------------------------------------------------------------------- +@pytest.fixture +def permission_cache_empty(): + return {"role_permissions": {}, "user_permissions": {}} + + +@pytest.fixture +def permission_cache_with_wildcard(): + """Permission cache where an agent role has iam:* — triggers FS-07 WARN.""" + return { + "role_permissions": { + "BedrockAgentRole": { + "attached_policies": [ + { + "document": { + "Statement": [ + { + "Effect": "Allow", + "Action": "iam:*", + "Resource": "*", + } + ] + } + } + ], + "inline_policies": [], + } + }, + "user_permissions": {}, + } + + +@pytest.fixture +def permission_cache_safe(): + """Permission cache where agent role has narrow permissions — triggers FS-07 PASS.""" + return { + "role_permissions": { + "BedrockAgentRole": { + "attached_policies": [ + { + "document": { + "Statement": [ + { + "Effect": "Allow", + "Action": "bedrock:InvokeModel", + "Resource": "arn:aws:bedrock:*:*:model/*", + } + ] + } + } + ], + "inline_policies": [], + } + }, + "user_permissions": {}, + } + + +# --------------------------------------------------------------------------- +# Synthetic Lambda event +# --------------------------------------------------------------------------- +@pytest.fixture +def lambda_event(): + return { + "Execution": {"Name": "unit-test-001"}, + "StateMachine": { + "Id": "arn:aws:states:us-east-1:123456789012:stateMachine:test" + }, + } + + +# --------------------------------------------------------------------------- +# ResourceInventory test builder (REQ-6.4, REQ-9.3) +# --------------------------------------------------------------------------- + + +def make_resource_inventory(**overrides) -> app.ResourceInventory: + """Build a fully-available ``ResourceInventory`` with sensible empty defaults. + + Any field can be replaced via keyword arguments. Pass an + ``app._Unavailable(exc)`` value to simulate a per-inventory collection + failure. + + Examples:: + + inv = make_resource_inventory() # fully available + inv = make_resource_inventory(lambda_functions=[...]) # real data + inv = make_resource_inventory( + guardrails=app._Unavailable(PermissionError("AccessDenied")) + ) # failed field + """ + defaults: dict = dict( + lambda_functions=[], + guardrails=app.GuardrailInventory(summaries=[], detail_by_id={}), + knowledge_bases=app.KbInventory( + summaries=[], data_sources_by_kb={}, data_source_detail={} + ), + buckets=[], + web_acls=app.WebAclInventory(summaries=[], detail_by_id={}), + ) + defaults.update(overrides) + return app.ResourceInventory(**defaults) diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_at_most_once.py b/aiml-security-assessment/functions/security/finserv_tests/test_at_most_once.py new file mode 100644 index 0000000..dd1a750 --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_tests/test_at_most_once.py @@ -0,0 +1,813 @@ +""" +"At most once" enforcement harness — Task 11 + +Handler-level tests with counting mocks that assert each shared listing API is +called ≤ 1 per run, each detail API ≤ 1 per resource per run, and +ListDataSources ≤ 1 per KB. The tests fail the build if any invariant is +exceeded. + +Requirements: REQ-9.1, REQ-9.4, REQ-9.6 + +Design reference: design.md §9.3 + list_functions ≤ 1 + list_guardrails ≤ 1 + list_knowledge_bases ≤ 1 + list_buckets ≤ 1 + list_web_acls ≤ 1 + list_data_sources ≤ 1 per KB + get_guardrail ≤ 1 per distinct guardrail id + get_web_acl ≤ 1 per distinct ACL id + get_data_source ≤ 1 per (kb_id, ds_id) pair +""" + +from __future__ import annotations + +import os +import sys +from collections import defaultdict +from datetime import datetime, timezone +from unittest.mock import MagicMock, patch + +import pytest + +# --------------------------------------------------------------------------- +# Make finserv_assessments importable +# --------------------------------------------------------------------------- +FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") +if FINSERV_DIR not in sys.path: + sys.path.insert(0, FINSERV_DIR) + +import app # noqa: E402 + + +# =========================================================================== +# Call-counting mock infrastructure +# =========================================================================== + + +class _CountingClient: + """Thin wrapper around a MagicMock that counts calls to specific methods. + + For listing APIs we track the total number of calls (must be ≤ 1). + For per-resource detail APIs we track calls per argument key (must be ≤ 1 + per distinct resource). + """ + + def __init__(self, service: str): + self.service = service + self.list_call_counts: dict[str, int] = defaultdict(int) + self.detail_call_counts: dict[str, dict] = defaultdict(lambda: defaultdict(int)) + self._mock = MagicMock() + + # ------------------------------------------------------------------ + # Wire counted list + detail methods and delegate rest to the mock + # ------------------------------------------------------------------ + + def __getattr__(self, name: str): + return getattr(self._mock, name) + + +# --------------------------------------------------------------------------- +# Reusable "full account state" data (same as test_inventory_equivalence.py +# but kept local so this module has no import dependency on that module). +# --------------------------------------------------------------------------- + +_NOW = datetime.now(timezone.utc) + +_LAMBDA_FUNCTIONS = [ + {"FunctionName": "my-bedrock-agent-handler"}, + {"FunctionName": "my-output-validate-fn"}, + {"FunctionName": "my-schema-validator-fn"}, +] + +_GUARDRAIL_SUMMARIES = [ + {"id": "gr-001", "name": "GuardrailA"}, + {"id": "gr-002", "name": "GuardrailB"}, +] + +_GUARDRAIL_DETAIL = { + "contextualGroundingPolicy": { + "filters": [ + {"type": "GROUNDING", "threshold": 0.8}, + {"type": "RELEVANCE", "threshold": 0.8}, + ] + }, + "topicPolicy": { + "topics": [{"name": "investment-advice", "type": "DENY"}], + "tier": {"tierName": "CLASSIC"}, + }, + "contentPolicy": { + "filters": [ + {"type": "HATE", "inputStrength": "HIGH", "outputStrength": "HIGH"}, + ] + }, + "wordPolicy": { + "words": [{"text": "fraud"}], + "managedWordLists": [{"type": "PROFANITY"}], + }, + "sensitiveInformationPolicy": { + "piiEntities": [{"type": "US_SOCIAL_SECURITY_NUMBER", "action": "BLOCK"}] + }, +} + +_KB_SUMMARIES = [ + {"knowledgeBaseId": "kb-001", "name": "KB_Alpha"}, + {"knowledgeBaseId": "kb-002", "name": "KB_Beta"}, +] + +_KB_DATA_SOURCE_SUMMARIES = { + "kb-001": [ + {"dataSourceId": "ds-001", "name": "DS_1", "updatedAt": _NOW}, + {"dataSourceId": "ds-002", "name": "DS_2", "updatedAt": _NOW}, + ], + "kb-002": [ + {"dataSourceId": "ds-003", "name": "DS_3", "updatedAt": _NOW}, + ], +} + +_S3_BUCKETS = [ + {"Name": "my-training-dataset-bucket"}, + {"Name": "my-bedrock-kb-bucket"}, + {"Name": "my-kb-datasource-bucket"}, +] + +_ACL_SUMMARIES = [ + {"Name": "ACL_One", "Id": "acl-id-001", "ARN": "arn:aws:wafv2:::acl-id-001"}, + {"Name": "ACL_Two", "Id": "acl-id-002", "ARN": "arn:aws:wafv2:::acl-id-002"}, +] + +_ACL_DETAIL_TEMPLATE = { + "Rules": [ + { + "Name": "SQLi", + "Statement": { + "ManagedRuleGroupStatement": {"Name": "AWSManagedRulesSQLiRuleSet"} + }, + }, + { + "Name": "Common", + "Statement": { + "ManagedRuleGroupStatement": {"Name": "AWSManagedRulesCommonRuleSet"} + }, + }, + { + "Name": "KnownBadInputs", + "Statement": { + "ManagedRuleGroupStatement": { + "Name": "AWSManagedRulesKnownBadInputsRuleSet" + } + }, + }, + { + "Name": "SizeConstraint", + "Statement": { + "SizeConstraintStatement": { + "FieldToMatch": {"Body": {}}, + "ComparisonOperator": "LE", + "Size": 8192, + "TextTransformations": [{"Priority": 0, "Type": "NONE"}], + } + }, + }, + ], +} + + +# =========================================================================== +# Counting mock client factory +# =========================================================================== + + +def _build_counting_client_factory(): + """Return a (side_effect_fn, call_tracker) pair. + + side_effect_fn: passed to mock_client.side_effect — returns a client mock + whose inventory-relevant methods are wrapped with call counters. + call_tracker: a dict containing all recorded call counts so tests can + assert at-most-once invariants after lambda_handler returns. + + Structure of call_tracker:: + + { + "list_functions": , # total calls + "list_guardrails": , + "list_knowledge_bases": , + "list_buckets": , + "list_web_acls": , + "list_data_sources": {kb_id: , ...}, # per-KB + "get_guardrail": {guardrail_id: , ...}, # per guardrail id + "get_web_acl": {acl_id: , ...}, # per ACL id + "get_data_source": {(kb_id, ds_id): , ...}, # per (kb, ds) + } + """ + tracker: dict = { + "list_functions": 0, + "list_guardrails": 0, + "list_knowledge_bases": 0, + "list_buckets": 0, + "list_web_acls": 0, + "list_data_sources": defaultdict(int), + "get_guardrail": defaultdict(int), + "get_web_acl": defaultdict(int), + "get_data_source": defaultdict(int), + } + + def side_effect(service, **kwargs): # noqa: C901 + c = MagicMock() + + # ------------------------------------------------------------------ # + # wafv2 — count list_web_acls and get_web_acl calls + # ------------------------------------------------------------------ # + if service == "wafv2": + + def _list_web_acls(Scope, **kw): + tracker["list_web_acls"] += 1 + return {"WebACLs": _ACL_SUMMARIES} + + c.list_web_acls.side_effect = _list_web_acls + + def _get_web_acl(Name, Scope, Id, **kw): + tracker["get_web_acl"][Id] += 1 + detail = dict(_ACL_DETAIL_TEMPLATE) + detail["Name"] = Name + detail["Id"] = Id + return {"WebACL": detail} + + c.get_web_acl.side_effect = _get_web_acl + return c + + # ------------------------------------------------------------------ # + # lambda — count list_functions + # ------------------------------------------------------------------ # + if service == "lambda": + + def _list_functions(**kw): + tracker["list_functions"] += 1 + return {"Functions": _LAMBDA_FUNCTIONS} + + c.list_functions.side_effect = _list_functions + + def _get_function_concurrency(FunctionName, **kw): + if "agent" in FunctionName.lower(): + return {"ReservedConcurrentExecutions": 5} + return {} + + c.get_function_concurrency.side_effect = _get_function_concurrency + return c + + # ------------------------------------------------------------------ # + # bedrock — count list_guardrails and get_guardrail per id + # ------------------------------------------------------------------ # + if service == "bedrock": + + def _list_guardrails(**kw): + tracker["list_guardrails"] += 1 + return {"guardrails": _GUARDRAIL_SUMMARIES} + + c.list_guardrails.side_effect = _list_guardrails + + def _get_guardrail(guardrailIdentifier, guardrailVersion="DRAFT", **kw): + tracker["get_guardrail"][guardrailIdentifier] += 1 + return _GUARDRAIL_DETAIL + + c.get_guardrail.side_effect = _get_guardrail + + # Non-inventory calls for other bedrock checks + c.list_foundation_models.return_value = { + "modelSummaries": [ + { + "modelId": "anthropic.claude-3-sonnet-20240229-v1:0", + "modelLifecycle": {"status": "ACTIVE"}, + } + ] + } + c.list_custom_models.return_value = {"modelSummaries": []} + c.list_model_cards.return_value = {"ModelCardSummaries": []} + c.list_evaluation_jobs.return_value = {"jobSummaries": []} + c.list_automated_reasoning_policies.return_value = { + "automatedReasoningPolicySummaries": [] + } + return c + + # ------------------------------------------------------------------ # + # bedrock-agent — count list_knowledge_bases, list_data_sources (per KB), + # get_data_source (per (kb, ds)) + # ------------------------------------------------------------------ # + if service == "bedrock-agent": + + def _list_knowledge_bases(**kw): + tracker["list_knowledge_bases"] += 1 + return {"knowledgeBaseSummaries": _KB_SUMMARIES} + + c.list_knowledge_bases.side_effect = _list_knowledge_bases + + def _list_data_sources(knowledgeBaseId, **kw): + tracker["list_data_sources"][knowledgeBaseId] += 1 + return { + "dataSourceSummaries": _KB_DATA_SOURCE_SUMMARIES.get( + knowledgeBaseId, [] + ) + } + + c.list_data_sources.side_effect = _list_data_sources + + def _get_data_source(knowledgeBaseId, dataSourceId, **kw): + tracker["get_data_source"][(knowledgeBaseId, dataSourceId)] += 1 + return { + "dataSource": { + "dataSourceConfiguration": { + "s3Configuration": { + "bucketArn": "arn:aws:s3:::my-kb-datasource-bucket" + } + } + } + } + + c.get_data_source.side_effect = _get_data_source + + # list_agents for FS-07 + c.list_agents.return_value = {"agentSummaries": []} + return c + + # ------------------------------------------------------------------ # + # s3 — count list_buckets + # ------------------------------------------------------------------ # + if service == "s3": + + def _list_buckets(**kw): + tracker["list_buckets"] += 1 + return {"Buckets": _S3_BUCKETS} + + c.list_buckets.side_effect = _list_buckets + + def _get_bucket_versioning(Bucket, **kw): + return {"Status": "Enabled"} + + c.get_bucket_versioning.side_effect = _get_bucket_versioning + + def _get_bucket_tagging(Bucket, **kw): + return { + "TagSet": [{"Key": "data-classification", "Value": "Confidential"}] + } + + c.get_bucket_tagging.side_effect = _get_bucket_tagging + + def _get_bucket_notification_configuration(Bucket, **kw): + return {"EventBridgeConfiguration": {}} + + c.get_bucket_notification_configuration.side_effect = ( + _get_bucket_notification_configuration + ) + return c + + # ------------------------------------------------------------------ # + # shield — needed for FS-01 + # ------------------------------------------------------------------ # + if service == "shield": + c.describe_subscription.return_value = {} + return c + + # ------------------------------------------------------------------ # + # Non-inventory services — return minimal responses so checks complete + # ------------------------------------------------------------------ # + if service == "apigateway": + c.get_usage_plans.return_value = { + "items": [ + { + "name": "default", + "throttle": {"rateLimit": 500, "burstLimit": 200}, + } + ] + } + c.get_rest_apis.return_value = {"items": []} + return c + + if service == "ce": + c.get_anomaly_monitors.return_value = { + "AnomalyMonitors": [ + { + "MonitorType": "DIMENSIONAL", + "MonitorDimension": "SERVICE", + "MonitorSpecification": {}, + } + ] + } + return c + + if service == "cloudwatch": + pag = MagicMock() + pag.paginate.return_value = [ + { + "MetricAlarms": [ + { + "AlarmName": "bedrock-throttle-alarm", + "Namespace": "AWS/Bedrock", + "MetricName": "InvocationThrottles", + } + ] + } + ] + c.get_paginator.return_value = pag + return c + + if service == "budgets": + pag = MagicMock() + pag.paginate.return_value = [ + { + "Budgets": [ + { + "BudgetName": "bedrock-spend", + "CostFilters": {"Service": ["Amazon Bedrock"]}, + } + ] + } + ] + c.get_paginator.return_value = pag + return c + + if service == "sts": + c.get_caller_identity.return_value = {"Account": "123456789012"} + return c + + if service == "service-quotas": + applied_pag = MagicMock() + applied_pag.paginate.return_value = [ + { + "Quotas": [ + { + "QuotaName": "On-demand InvokeModel tokens per minute for anthropic.claude", + "QuotaCode": "L-TPMTEST", + "Value": 200000, + } + ] + } + ] + defaults_pag = MagicMock() + defaults_pag.paginate.return_value = [ + {"Quotas": [{"QuotaCode": "L-TPMTEST", "Value": 100000}]} + ] + + def get_paginator(op): + if op == "list_service_quotas": + return applied_pag + if op == "list_aws_default_service_quotas": + return defaults_pag + p = MagicMock() + p.paginate.return_value = [{}] + return p + + c.get_paginator.side_effect = get_paginator + return c + + if service == "stepfunctions": + c.list_state_machines.return_value = {"stateMachines": []} + return c + + if service == "iam": + c.list_policies.return_value = {"Policies": []} + pag = MagicMock() + pag.paginate.return_value = [{}] + c.get_paginator.return_value = pag + return c + + if service == "config": + c.describe_config_rules.return_value = {"ConfigRules": []} + return c + + if service == "ecr": + c.describe_repositories.return_value = {"repositories": []} + return c + + if service == "sagemaker": + c.list_feature_groups.return_value = {"FeatureGroupSummaries": []} + c.list_processing_jobs.return_value = {"ProcessingJobSummaries": []} + c.list_models.return_value = {"Models": []} + pag = MagicMock() + pag.paginate.return_value = [{}] + c.get_paginator.return_value = pag + return c + + if service == "logs": + c.list_log_groups.return_value = {"logGroups": []} + pag = MagicMock() + pag.paginate.return_value = [{"logGroups": []}] + c.get_paginator.return_value = pag + return c + + if service == "macie2": + c.get_macie_session.side_effect = Exception("macie not enabled") + return c + + if service == "opensearchserverless": + c.list_security_policies.return_value = {"securityPolicySummaries": []} + return c + + if service == "bedrock-agent-runtime": + return c + + if service == "bedrock-runtime": + return c + + if service == "events": + c.list_rules.return_value = {"Rules": []} + return c + + if service == "scheduler": + c.list_schedules.return_value = {"Schedules": []} + return c + + if service == "agentcore": + c.list_agent_runtimes.return_value = {"agentRuntimes": []} + return c + + if service == "organizations": + c.list_policies.return_value = {"Policies": []} + return c + + # Catch-all: return generic mock for any other service + pag = MagicMock() + pag.paginate.return_value = [{}] + c.get_paginator.return_value = pag + return c + + return side_effect, tracker + + +# =========================================================================== +# Helper: run lambda_handler with counting mocks +# =========================================================================== + + +def _run_with_counting_mocks(event=None): + """Run lambda_handler end-to-end with counting mocks. + + Returns the (result, tracker) pair where tracker holds all call counts. + """ + if event is None: + event = {"Execution": {"Name": "at-most-once-test-001"}} + + side_effect, tracker = _build_counting_client_factory() + + with ( + patch("app.boto3.client") as mock_client, + patch("app.get_permissions_cache") as mock_cache, + patch("app.write_to_s3") as mock_s3, + ): + mock_client.side_effect = side_effect + mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} + mock_s3.return_value = "https://test-bucket.s3.amazonaws.com/finserv_security_report_at-most-once-test-001.csv" + + result = app.lambda_handler(event, None) + + return result, tracker + + +# =========================================================================== +# Test class +# =========================================================================== + + +class TestAtMostOnceInvariants: + """Handler-level counting-mock harness for the at-most-once invariants. + + Validates: Requirements REQ-9.1, REQ-9.4, REQ-9.6 + INV-2: Each shared listing API ≤ 1×/invocation; each shared detail API + ≤ 1× per resource/invocation. + """ + + @pytest.fixture(autouse=True) + def _run_handler(self): + """Run lambda_handler once and expose result + tracker to every test.""" + self.result, self.tracker = _run_with_counting_mocks() + + # ------------------------------------------------------------------ + # Sanity: handler completed successfully + # ------------------------------------------------------------------ + + def test_handler_completes_successfully(self): + """Counting mocks are sufficient for the handler to return 200.""" + assert self.result["statusCode"] == 200 + + # ------------------------------------------------------------------ + # Listing API invariants (≤ 1 per run) + # ------------------------------------------------------------------ + + def test_list_functions_at_most_once(self): + """list_functions SHALL be called ≤ 1 per run (REQ-9.1, INV-2). + + 6 checks consume Lambda functions; the collector must issue at most one + call regardless. + """ + count = self.tracker["list_functions"] + assert count <= 1, ( + f"list_functions was called {count} time(s); expected ≤ 1. " + "Multiple calls indicate the inventory was not consolidated." + ) + + def test_list_guardrails_at_most_once(self): + """list_guardrails SHALL be called ≤ 1 per run (REQ-9.1, INV-2). + + 9 checks consume guardrails; the collector must issue at most one call. + """ + count = self.tracker["list_guardrails"] + assert count <= 1, f"list_guardrails was called {count} time(s); expected ≤ 1." + + def test_list_knowledge_bases_at_most_once(self): + """list_knowledge_bases SHALL be called ≤ 1 per run (REQ-9.1, INV-2). + + 6 checks consume knowledge bases; the collector must issue at most one call. + """ + count = self.tracker["list_knowledge_bases"] + assert count <= 1, ( + f"list_knowledge_bases was called {count} time(s); expected ≤ 1." + ) + + def test_list_buckets_at_most_once(self): + """list_buckets SHALL be called ≤ 1 per run (REQ-9.1, INV-2). + + 2 checks consume S3 buckets; the collector must issue at most one call. + """ + count = self.tracker["list_buckets"] + assert count <= 1, f"list_buckets was called {count} time(s); expected ≤ 1." + + def test_list_web_acls_at_most_once(self): + """list_web_acls SHALL be called ≤ 1 per run (REQ-9.1, INV-2). + + 4 checks consume WAFv2 Web ACLs; the collector must issue at most one + call (previously each check called it independently). + """ + count = self.tracker["list_web_acls"] + assert count <= 1, f"list_web_acls was called {count} time(s); expected ≤ 1." + + def test_list_data_sources_at_most_once_per_kb(self): + """list_data_sources SHALL be called ≤ 1 per KB per run (REQ-9.1, REQ-3.5, INV-2). + + 3 checks consume data-source summaries (FS-31, FS-33, FS-65); each + KB's list_data_sources must be called at most once. + """ + for kb_id, count in self.tracker["list_data_sources"].items(): + assert count <= 1, ( + f"list_data_sources for KB '{kb_id}' was called {count} time(s); " + "expected ≤ 1." + ) + + # ------------------------------------------------------------------ + # Detail API invariants (≤ 1 per distinct resource per run) + # ------------------------------------------------------------------ + + def test_get_guardrail_at_most_once_per_id(self): + """get_guardrail SHALL be called ≤ 1 per distinct guardrail id per run. + + Requirements REQ-9.1, REQ-3.1, INV-2. + 9 checks inspect guardrail detail; the inventory must serve it from + cache without issuing further get_guardrail calls. + """ + for gid, count in self.tracker["get_guardrail"].items(): + assert count <= 1, ( + f"get_guardrail for guardrail '{gid}' was called {count} time(s); " + "expected ≤ 1." + ) + + def test_get_web_acl_at_most_once_per_id(self): + """get_web_acl SHALL be called ≤ 1 per distinct ACL id per run. + + Requirements REQ-9.1, REQ-3.2, INV-2. + 3 checks (FS-53, FS-56, FS-68) inspect ACL detail; the inventory must + serve it from cache. + """ + for acl_id, count in self.tracker["get_web_acl"].items(): + assert count <= 1, ( + f"get_web_acl for ACL '{acl_id}' was called {count} time(s); " + "expected ≤ 1." + ) + + def test_get_data_source_at_most_once_per_pair(self): + """get_data_source SHALL be called ≤ 1 per (kb_id, ds_id) pair per run. + + Requirements REQ-9.1, REQ-3.5, INV-2. + 2 checks (FS-33, FS-65) call get_data_source; the inventory must cache + the result and serve it to both. + """ + for (kb_id, ds_id), count in self.tracker["get_data_source"].items(): + assert count <= 1, ( + f"get_data_source for (kb='{kb_id}', ds='{ds_id}') was called " + f"{count} time(s); expected ≤ 1." + ) + + # ------------------------------------------------------------------ + # Positive coverage: calls were made (inventory was actually collected) + # ------------------------------------------------------------------ + + def test_listing_apis_were_called_at_least_once(self): + """Verify the counting mocks were exercised — all five listing APIs called.""" + assert self.tracker["list_functions"] >= 1, "list_functions was never called" + assert self.tracker["list_guardrails"] >= 1, "list_guardrails was never called" + assert self.tracker["list_knowledge_bases"] >= 1, ( + "list_knowledge_bases was never called" + ) + assert self.tracker["list_buckets"] >= 1, "list_buckets was never called" + assert self.tracker["list_web_acls"] >= 1, "list_web_acls was never called" + + def test_guardrail_detail_called_for_every_guardrail(self): + """get_guardrail was called for each guardrail in the summary list.""" + expected_ids = {g["id"] for g in _GUARDRAIL_SUMMARIES} + called_ids = set(self.tracker["get_guardrail"].keys()) + assert expected_ids == called_ids, ( + f"Expected get_guardrail calls for {expected_ids}; " + f"actually called for {called_ids}." + ) + + def test_web_acl_detail_called_for_every_acl(self): + """get_web_acl was called for each ACL in the summary list.""" + expected_ids = {acl["Id"] for acl in _ACL_SUMMARIES} + called_ids = set(self.tracker["get_web_acl"].keys()) + assert expected_ids == called_ids, ( + f"Expected get_web_acl calls for {expected_ids}; " + f"actually called for {called_ids}." + ) + + def test_data_source_detail_called_for_all_data_sources(self): + """get_data_source was called for each (kb_id, ds_id) pair.""" + expected_pairs = { + (kb_id, ds["dataSourceId"]) + for kb_id, ds_list in _KB_DATA_SOURCE_SUMMARIES.items() + for ds in ds_list + } + called_pairs = set(self.tracker["get_data_source"].keys()) + assert expected_pairs == called_pairs, ( + f"Expected get_data_source calls for {expected_pairs}; " + f"actually called for {called_pairs}." + ) + + def test_list_data_sources_called_for_all_kbs(self): + """list_data_sources was called for each KB in the summary list.""" + expected_kb_ids = {kb["knowledgeBaseId"] for kb in _KB_SUMMARIES} + called_kb_ids = set(self.tracker["list_data_sources"].keys()) + assert expected_kb_ids == called_kb_ids, ( + f"Expected list_data_sources calls for {expected_kb_ids}; " + f"actually called for {called_kb_ids}." + ) + + # ------------------------------------------------------------------ + # Exact counts (not just ≤ 1: assert exactly 1 for non-empty inventories) + # ------------------------------------------------------------------ + + def test_list_functions_called_exactly_once(self): + """list_functions is called exactly once — not zero, not more than one.""" + assert self.tracker["list_functions"] == 1 + + def test_list_guardrails_called_exactly_once(self): + """list_guardrails is called exactly once — not zero, not more than one.""" + assert self.tracker["list_guardrails"] == 1 + + def test_list_knowledge_bases_called_exactly_once(self): + """list_knowledge_bases is called exactly once.""" + assert self.tracker["list_knowledge_bases"] == 1 + + def test_list_buckets_called_exactly_once(self): + """list_buckets is called exactly once.""" + assert self.tracker["list_buckets"] == 1 + + def test_list_web_acls_called_exactly_once(self): + """list_web_acls is called exactly once.""" + assert self.tracker["list_web_acls"] == 1 + + def test_list_data_sources_called_exactly_once_per_kb(self): + """list_data_sources is called exactly once for each KB (not zero, not more).""" + for kb_id in {kb["knowledgeBaseId"] for kb in _KB_SUMMARIES}: + count = self.tracker["list_data_sources"][kb_id] + assert count == 1, ( + f"list_data_sources for KB '{kb_id}' was called {count} time(s); " + "expected exactly 1." + ) + + def test_get_guardrail_called_exactly_once_per_guardrail(self): + """get_guardrail is called exactly once per guardrail id.""" + for g in _GUARDRAIL_SUMMARIES: + gid = g["id"] + count = self.tracker["get_guardrail"][gid] + assert count == 1, ( + f"get_guardrail for '{gid}' was called {count} time(s); " + "expected exactly 1." + ) + + def test_get_web_acl_called_exactly_once_per_acl(self): + """get_web_acl is called exactly once per ACL id.""" + for acl in _ACL_SUMMARIES: + acl_id = acl["Id"] + count = self.tracker["get_web_acl"][acl_id] + assert count == 1, ( + f"get_web_acl for '{acl_id}' was called {count} time(s); " + "expected exactly 1." + ) + + def test_get_data_source_called_exactly_once_per_pair(self): + """get_data_source is called exactly once per (kb_id, ds_id) pair.""" + for kb_id, ds_list in _KB_DATA_SOURCE_SUMMARIES.items(): + for ds in ds_list: + ds_id = ds["dataSourceId"] + count = self.tracker["get_data_source"][(kb_id, ds_id)] + assert count == 1, ( + f"get_data_source for (kb='{kb_id}', ds='{ds_id}') was called " + f"{count} time(s); expected exactly 1." + ) diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_checks.py b/aiml-security-assessment/functions/security/finserv_tests/test_checks.py new file mode 100644 index 0000000..f456443 --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_tests/test_checks.py @@ -0,0 +1,3577 @@ +""" +Tests for all check functions in finserv_assessments/app.py + +Strategy: + - Every check function is tested for at least two scenarios: + 1. "PASS / N/A" path — mocked AWS responses indicate compliant state + 2. "WARN / FAIL" path — mocked AWS responses indicate non-compliant state + - Advisory-only checks (no AWS API calls) are tested for correct structure + - Functions that accept permission_cache are tested with both empty and + populated caches + - Every check is tested for graceful error handling (boto3 ClientError) + +All boto3 clients are patched via unittest.mock so no real AWS calls are made. +""" + +import json +import sys +import os +from unittest.mock import MagicMock, patch + +from botocore.exceptions import ClientError + +# Ensure finserv_assessments is importable +FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") +if FINSERV_DIR not in sys.path: + sys.path.insert(0, FINSERV_DIR) + +# Ensure tests/ directory is importable (for conftest helpers) +TESTS_DIR = os.path.dirname(__file__) +if TESTS_DIR not in sys.path: + sys.path.insert(0, TESTS_DIR) + +import app # noqa: E402 (import must follow sys.path setup above) +from conftest import make_resource_inventory # noqa: E402 + + +# ========================================================================= +# Helpers +# ========================================================================= + + +def _client_error(code="AccessDeniedException", message="Access Denied"): + """Build a botocore ClientError for mocking.""" + return ClientError( + {"Error": {"Code": code, "Message": message}}, + "TestOperation", + ) + + +def _assert_finding_structure(result): + """Assert the standard check-function return dict shape.""" + assert "check_name" in result + assert "status" in result + assert result["status"] in ("PASS", "WARN", "ERROR") + assert "csv_data" in result + assert isinstance(result["csv_data"], list) + for row in result["csv_data"]: + assert "Check_ID" in row + assert "Finding" in row + assert "Severity" in row + assert "Status" in row + # Compliance_Frameworks must be present in every row (D3 fix) + assert "Compliance_Frameworks" in row, ( + f"Missing Compliance_Frameworks in row for {row.get('Check_ID')}" + ) + + +def _assert_advisory_retag(result, check_id): + """ + REQ-6 Option B: an advisory check must emit a finding with Status="N/A", + Severity="Informational", and a Finding name prefixed "ADVISORY: ". + (The wrapper result["status"] is separate and may still be "PASS".) + """ + rows = [r for r in result["csv_data"] if r["Check_ID"] == check_id] + assert rows, f"no rows for {check_id}" + for r in rows: + assert r["Status"] == "N/A", f"{check_id} Status={r['Status']}" + assert r["Severity"] == "Informational", f"{check_id} Severity={r['Severity']}" + assert r["Finding"].startswith("ADVISORY: "), ( + f"{check_id} Finding={r['Finding']}" + ) + + +# ========================================================================= +# CATEGORY 1: UNBOUNDED CONSUMPTION (FS-01 to FS-06) +# ========================================================================= + + +class TestFS01WafShield: + """FS-01 — WAF and Shield Protection Check.""" + + def test_pass_shield_enabled_acls_present(self): + """Shield enabled + ACLs present via inventory → PASS.""" + inv = make_resource_inventory( + web_acls=app.WebAclInventory( + summaries=[{"Name": "acl1", "Id": "id1"}], + detail_by_id={}, + ) + ) + with patch("app.boto3.client") as mock_client: + shield_mock = MagicMock() + shield_mock.describe_subscription.return_value = {} + shield_mock.exceptions.ResourceNotFoundException = type( + "ResourceNotFoundException", (ClientError,), {} + ) + mock_client.return_value = shield_mock + + result = app.check_waf_shield_on_bedrock_endpoints(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + assert len(result["csv_data"]) == 2 # shield pass + waf pass + # REQ-6: Shield Advanced findings are Low; WAF findings are Medium. + shield_rows = [r for r in result["csv_data"] if "Shield" in r["Finding"]] + waf_rows = [r for r in result["csv_data"] if "WAF" in r["Finding"]] + assert shield_rows and all(r["Severity"] == "Low" for r in shield_rows) + assert waf_rows and all(r["Severity"] == "Medium" for r in waf_rows) + + def test_severity_shield_low_waf_medium_on_fail(self): + """REQ-6: Shield-absent = Low, WAF-absent = Medium (was both High).""" + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=[], detail_by_id={}) + ) + with patch("app.boto3.client") as mock_client: + shield_mock = MagicMock() + shield_mock.describe_subscription.side_effect = ClientError( + {"Error": {"Code": "ResourceNotFoundException", "Message": ""}}, + "DescribeSubscription", + ) + shield_mock.exceptions.ResourceNotFoundException = type( + "ResourceNotFoundException", (ClientError,), {} + ) + mock_client.return_value = shield_mock + + result = app.check_waf_shield_on_bedrock_endpoints(inv) + _assert_finding_structure(result) + assert any( + r["Finding"] == "AWS Shield Advanced Not Enabled" and r["Severity"] == "Low" + for r in result["csv_data"] + ) + assert any( + r["Finding"] == "No Regional WAF Web ACLs Found" + and r["Severity"] == "Medium" + for r in result["csv_data"] + ) + + def test_warn_no_shield_no_acls(self): + """No shield + no ACLs in inventory → WARN.""" + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=[], detail_by_id={}) + ) + with patch("app.boto3.client") as mock_client: + shield_mock = MagicMock() + shield_mock.describe_subscription.side_effect = ClientError( + {"Error": {"Code": "ResourceNotFoundException", "Message": ""}}, + "DescribeSubscription", + ) + shield_mock.exceptions.ResourceNotFoundException = type( + "ResourceNotFoundException", (ClientError,), {} + ) + mock_client.return_value = shield_mock + + result = app.check_waf_shield_on_bedrock_endpoints(inv) + _assert_finding_structure(result) + assert result["status"] == "WARN" + + def test_error_on_exception(self): + """Unavailable inventory → COULD_NOT_ASSESS (ERROR envelope).""" + inv = make_resource_inventory(web_acls=app._Unavailable(RuntimeError("boom"))) + with patch("app.boto3.client") as mock_client: + shield_mock = MagicMock() + shield_mock.describe_subscription.return_value = {} + shield_mock.exceptions.ResourceNotFoundException = type( + "ResourceNotFoundException", (ClientError,), {} + ) + mock_client.return_value = shield_mock + result = app.check_waf_shield_on_bedrock_endpoints(inv) + assert result["status"] == "ERROR" + assert "boom" in result["details"] + + def test_error_on_shield_exception(self): + """Shield client raises → overall ERROR.""" + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=[], detail_by_id={}) + ) + with patch("app.boto3.client") as mock_client: + mock_client.side_effect = RuntimeError("boom") + result = app.check_waf_shield_on_bedrock_endpoints(inv) + assert result["status"] == "ERROR" + assert "boom" in result["details"] + + # --- Pagination-correctness proof: >100 ACLs (Wave-3, task 8) --- + def test_pass_more_than_100_acls(self): + """Inventory with >100 ACLs (truncated in old code) → FS-01 PASS with correct count.""" + acls = [{"Name": f"acl{i}", "Id": f"id{i}"} for i in range(150)] + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=acls, detail_by_id={}) + ) + with patch("app.boto3.client") as mock_client: + shield_mock = MagicMock() + shield_mock.describe_subscription.return_value = {} + shield_mock.exceptions.ResourceNotFoundException = type( + "ResourceNotFoundException", (ClientError,), {} + ) + mock_client.return_value = shield_mock + result = app.check_waf_shield_on_bedrock_endpoints(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + waf_rows = [r for r in result["csv_data"] if "WAF" in r["Finding"]] + assert waf_rows + assert "150" in waf_rows[0]["Finding_Details"] + + # --- ≤1-page equivalence: 2-ACL case matches Wave-0 baseline --- + def test_two_acl_case_unchanged(self): + """The ≤1-page (2-ACL) case must produce the same finding as the pre-refactor baseline.""" + acls = [ + {"Name": "FinServACL1", "Id": "acl-id-001"}, + {"Name": "FinServACL2", "Id": "acl-id-002"}, + ] + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=acls, detail_by_id={}) + ) + with patch("app.boto3.client") as mock_client: + shield_mock = MagicMock() + shield_mock.describe_subscription.return_value = {} + shield_mock.exceptions.ResourceNotFoundException = type( + "ResourceNotFoundException", (ClientError,), {} + ) + mock_client.return_value = shield_mock + result = app.check_waf_shield_on_bedrock_endpoints(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + waf_rows = [r for r in result["csv_data"] if "WAF" in r["Finding"]] + assert waf_rows + assert waf_rows[0]["Finding"] == "Regional WAF Web ACLs Present" + + +class TestFS02ApiGatewayRateLimiting: + """FS-02 — API Gateway Rate Limiting Check.""" + + @patch("app.boto3.client") + def test_pass_all_plans_have_throttle(self, mock_client): + c = MagicMock() + c.get_usage_plans.return_value = { + "items": [ + {"name": "plan1", "throttle": {"rateLimit": 100, "burstLimit": 50}}, + ] + } + mock_client.return_value = c + result = app.check_api_gateway_rate_limiting() + _assert_finding_structure(result) + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_warn_plan_missing_throttle(self, mock_client): + c = MagicMock() + c.get_usage_plans.return_value = { + "items": [ + {"name": "no-throttle-plan", "throttle": {"rateLimit": 0}}, + ] + } + mock_client.return_value = c + result = app.check_api_gateway_rate_limiting() + _assert_finding_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_no_plans_returns_na(self, mock_client): + c = MagicMock() + c.get_usage_plans.return_value = {"items": []} + mock_client.return_value = c + result = app.check_api_gateway_rate_limiting() + _assert_finding_structure(result) + # No plans → advisory finding, status stays PASS + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("api error") + result = app.check_api_gateway_rate_limiting() + assert result["status"] == "ERROR" + + +class TestFS03BedrockTokenQuotas: + """FS-03 — Bedrock Token Quota Review (value-based, paginated).""" + + @staticmethod + def _sq_client(applied_quotas, default_quotas): + """Build a service-quotas client mock with paginated applied + default quotas.""" + c = MagicMock() + + def get_paginator(op_name): + paginator = MagicMock() + if op_name == "list_service_quotas": + paginator.paginate.return_value = [{"Quotas": applied_quotas}] + elif op_name == "list_aws_default_service_quotas": + paginator.paginate.return_value = [{"Quotas": default_quotas}] + else: + paginator.paginate.return_value = [{}] + return paginator + + c.get_paginator.side_effect = get_paginator + return c + + @patch("app.boto3.client") + def test_pass_customized_quota(self, mock_client): + # Applied value (200000) exceeds AWS default (100000) → customized → PASS/Passed + applied = [ + { + "QuotaName": "On-demand InvokeModel tokens per minute for anthropic.claude", + "QuotaCode": "L-1234ABCD", + "Value": 200000, + } + ] + defaults = [{"QuotaCode": "L-1234ABCD", "Value": 100000}] + mock_client.return_value = self._sq_client(applied, defaults) + result = app.check_bedrock_token_quotas() + _assert_finding_structure(result) + assert result["status"] == "PASS" + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_warn_default_quota(self, mock_client): + # Applied value == AWS default → still at default → WARN/N-A (soft, not a failure) + applied = [ + { + "QuotaName": "On-demand InvokeModel tokens per minute for anthropic.claude", + "QuotaCode": "L-1234ABCD", + "Value": 100000, + } + ] + defaults = [{"QuotaCode": "L-1234ABCD", "Value": 100000}] + mock_client.return_value = self._sq_client(applied, defaults) + result = app.check_bedrock_token_quotas() + _assert_finding_structure(result) + assert result["status"] == "WARN" + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + # At-default is NOT a failure. + assert not any(r["Status"] == "Failed" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_token_only_no_rpm(self, mock_client): + # Only token-based quotas present (no "request"/RPM quota). RPM is deprecated + # on bedrock-runtime; its absence must not drive a Failed verdict. + applied = [ + { + "QuotaName": "Model invocation max tokens per day for anthropic.claude", + "QuotaCode": "L-TPDAY01", + "Value": 5000000, + } + ] + defaults = [{"QuotaCode": "L-TPDAY01", "Value": 1000000}] + mock_client.return_value = self._sq_client(applied, defaults) + result = app.check_bedrock_token_quotas() + _assert_finding_structure(result) + # Customized token quota → PASS, regardless of any RPM quota existing. + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_empty_applied_quotas(self, mock_client): + # No token quotas returned at all → WARN/Failed + explanatory details. + mock_client.return_value = self._sq_client([], []) + result = app.check_bedrock_token_quotas() + _assert_finding_structure(result) + assert result["status"] == "WARN" + assert any(r["Status"] == "Failed" for r in result["csv_data"]) + assert any( + "No Bedrock token-based service quotas" in r["Finding_Details"] + for r in result["csv_data"] + ) + + @patch("app.boto3.client") + def test_default_lookup_fail(self, mock_client): + # Applied token quotas exist but defaults could not be retrieved → + # WARN/Failed + "undetermined" (NOT a silent value-vs-itself comparison). + applied = [ + { + "QuotaName": "On-demand InvokeModel tokens per minute for anthropic.claude", + "QuotaCode": "L-1234ABCD", + "Value": 100000, + } + ] + mock_client.return_value = self._sq_client(applied, []) + result = app.check_bedrock_token_quotas() + _assert_finding_structure(result) + assert result["status"] == "WARN" + assert any(r["Status"] == "Failed" for r in result["csv_data"]) + assert any( + "could not be retrieved" in r["Finding_Details"] + or "Undetermined" in r["Finding"] + for r in result["csv_data"] + ) + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("quota error") + result = app.check_bedrock_token_quotas() + assert result["status"] == "ERROR" + + +class TestFS04CostAnomalyDetection: + """FS-04 — Cost Anomaly Detection Check.""" + + @patch("app.boto3.client") + def test_pass_monitors_exist(self, mock_client): + c = MagicMock() + c.get_anomaly_monitors.return_value = { + "AnomalyMonitors": [ + { + "MonitorType": "DIMENSIONAL", + "MonitorDimension": "SERVICE", + "MonitorSpecification": {}, + } + ] + } + mock_client.return_value = c + result = app.check_cost_anomaly_detection() + _assert_finding_structure(result) + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_warn_monitors_without_bedrock_coverage(self, mock_client): + # A DIMENSIONAL monitor scoped to LINKED_ACCOUNT does NOT provide + # Bedrock service-level coverage → non-PASS (previously masked false positive). + c = MagicMock() + c.get_anomaly_monitors.return_value = { + "AnomalyMonitors": [ + { + "MonitorType": "DIMENSIONAL", + "MonitorDimension": "LINKED_ACCOUNT", + "MonitorSpecification": {}, + } + ] + } + mock_client.return_value = c + result = app.check_cost_anomaly_detection() + _assert_finding_structure(result) + assert result["status"] != "PASS" + assert any(r["Status"] == "Failed" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_warn_no_monitors(self, mock_client): + c = MagicMock() + c.get_anomaly_monitors.return_value = {"AnomalyMonitors": []} + mock_client.return_value = c + result = app.check_cost_anomaly_detection() + _assert_finding_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_pagination_finds_bedrock_monitor_on_second_page(self, mock_client): + # The Bedrock-covering monitor is on page 2. The check must paginate via + # NextPageToken and still find it (otherwise a false "no coverage" finding). + c = MagicMock() + c.get_anomaly_monitors.side_effect = [ + { + "AnomalyMonitors": [ + { + "MonitorType": "DIMENSIONAL", + "MonitorDimension": "LINKED_ACCOUNT", + "MonitorSpecification": {}, + } + ], + "NextPageToken": "page2", + }, + { + "AnomalyMonitors": [ + { + "MonitorType": "DIMENSIONAL", + "MonitorDimension": "SERVICE", + "MonitorSpecification": {}, + } + ] + }, + ] + mock_client.return_value = c + result = app.check_cost_anomaly_detection() + _assert_finding_structure(result) + assert result["status"] == "PASS" + # Verify it actually consumed both pages (passed the token on the 2nd call). + assert c.get_anomaly_monitors.call_count == 2 + c.get_anomaly_monitors.assert_any_call(NextPageToken="page2") + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("ce error") + result = app.check_cost_anomaly_detection() + assert result["status"] == "ERROR" + + +class TestFS05CloudWatchTokenAlarms: + """FS-05 — CloudWatch Token Usage Alarms Check.""" + + @patch("app.boto3.client") + def test_pass_bedrock_alarms_exist(self, mock_client): + c = MagicMock() + paginator = MagicMock() + paginator.paginate.return_value = [ + { + "MetricAlarms": [ + { + "AlarmName": "bedrock-throttle-alarm", + "Namespace": "AWS/Bedrock", + "MetricName": "InvocationThrottles", + } + ] + } + ] + c.get_paginator.return_value = paginator + mock_client.return_value = c + result = app.check_cloudwatch_token_alarms() + _assert_finding_structure(result) + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_warn_no_bedrock_alarms(self, mock_client): + c = MagicMock() + paginator = MagicMock() + paginator.paginate.return_value = [ + { + "MetricAlarms": [ + { + "AlarmName": "cpu-alarm", + "Namespace": "AWS/EC2", + "MetricName": "CPUUtilization", + } + ] + } + ] + c.get_paginator.return_value = paginator + mock_client.return_value = c + result = app.check_cloudwatch_token_alarms() + _assert_finding_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("cw error") + result = app.check_cloudwatch_token_alarms() + assert result["status"] == "ERROR" + + +class TestFS06AwsBudgets: + """FS-06 — AWS Budgets AI/ML Spend Check (ShowFilterExpression, paginated).""" + + @staticmethod + def _budgets_client(budgets, capture=None, raise_param_validation_on_show=False): + """ + Build a budgets client whose describe_budgets paginator returns `budgets`. + If raise_param_validation_on_show is True, paginate raises ParamValidationError + when called with ShowFilterExpression (simulating old botocore), and returns + budgets when called without it. + """ + from botocore.exceptions import ParamValidationError + + c = MagicMock() + paginator = MagicMock() + + def paginate(**kwargs): + if capture is not None: + capture.append(kwargs) + if raise_param_validation_on_show and "ShowFilterExpression" in kwargs: + raise ParamValidationError(report="ShowFilterExpression not accepted") + return [{"Budgets": budgets}] + + paginator.paginate.side_effect = paginate + c.get_paginator.return_value = paginator + return c + + def _client_factory(self, budgets, capture=None, raise_pv=False): + def side_effect(service, **kwargs): + if service == "budgets": + return self._budgets_client(budgets, capture, raise_pv) + if service == "sts": + c = MagicMock() + c.get_caller_identity.return_value = {"Account": "123456789012"} + return c + return MagicMock() + + return side_effect + + @patch("app.boto3.client") + def test_pass_aiml_budgets_exist(self, mock_client): + capture = [] + mock_client.side_effect = self._client_factory( + [{"BudgetName": "bedrock-budget", "CostFilters": {"Service": ["bedrock"]}}], + capture=capture, + ) + result = app.check_aws_budgets_for_aiml() + _assert_finding_structure(result) + assert result["status"] == "PASS" + # Regression guard: the call MUST pass ShowFilterExpression=True. + assert any(kw.get("ShowFilterExpression") is True for kw in capture) + + @patch("app.boto3.client") + def test_warn_no_aiml_budgets(self, mock_client): + mock_client.side_effect = self._client_factory( + [{"BudgetName": "general", "CostFilters": {"Service": ["ec2"]}}] + ) + result = app.check_aws_budgets_for_aiml() + _assert_finding_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_pass_filterexpression_budget(self, mock_client): + # New-style budget using only FilterExpression (no CostFilters) → detected. + mock_client.side_effect = self._client_factory( + [ + { + "BudgetName": "genai-budget", + "CostFilters": {}, + "FilterExpression": { + "Dimensions": {"Key": "SERVICE", "Values": ["Amazon Bedrock"]} + }, + } + ] + ) + result = app.check_aws_budgets_for_aiml() + _assert_finding_structure(result) + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_param_validation_fallback(self, mock_client): + # Old botocore: ShowFilterExpression rejected with ParamValidationError. + # FS-06 must degrade to CostFilters-only (non-ERROR) and still match. + capture = [] + mock_client.side_effect = self._client_factory( + [{"BudgetName": "bedrock-budget", "CostFilters": {"Service": ["bedrock"]}}], + capture=capture, + raise_pv=True, + ) + result = app.check_aws_budgets_for_aiml() + _assert_finding_structure(result) + assert result["status"] != "ERROR" + assert result["status"] == "PASS" + # It attempted with ShowFilterExpression, then retried without it. + assert any("ShowFilterExpression" in kw for kw in capture) + assert any("ShowFilterExpression" not in kw for kw in capture) + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("budgets error") + result = app.check_aws_budgets_for_aiml() + assert result["status"] == "ERROR" + + +# ========================================================================= +# CATEGORY 2: EXCESSIVE AGENCY (FS-07 to FS-11) +# ========================================================================= + + +class TestFS07AgentActionBoundaries: + """FS-07 — Agent Action Boundary Check (takes permission_cache).""" + + @patch("app.boto3.client") + def test_pass_no_agents(self, mock_client): + c = MagicMock() + c.list_agents.return_value = {"agentSummaries": []} + mock_client.return_value = c + result = app.check_bedrock_agent_action_boundaries({}) + _assert_finding_structure(result) + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_warn_wildcard_permissions( + self, mock_client, permission_cache_with_wildcard + ): + c = MagicMock() + c.list_agents.return_value = { + "agentSummaries": [{"agentId": "a1", "agentName": "TestAgent"}] + } + c.get_agent.return_value = { + "agent": {"agentResourceRoleArn": "arn:aws:iam::123:role/BedrockAgentRole"} + } + mock_client.return_value = c + result = app.check_bedrock_agent_action_boundaries( + permission_cache_with_wildcard + ) + _assert_finding_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_pass_narrow_permissions(self, mock_client, permission_cache_safe): + c = MagicMock() + c.list_agents.return_value = { + "agentSummaries": [{"agentId": "a1", "agentName": "TestAgent"}] + } + c.get_agent.return_value = { + "agent": {"agentResourceRoleArn": "arn:aws:iam::123:role/BedrockAgentRole"} + } + mock_client.return_value = c + result = app.check_bedrock_agent_action_boundaries(permission_cache_safe) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("agent error") + result = app.check_bedrock_agent_action_boundaries({}) + assert result["status"] == "ERROR" + + +class TestFS08AgentcorePolicyEngine: + """FS-08 — AgentCore Policy Engine Check.""" + + @patch("app.boto3.client") + def test_pass_runtimes_with_authorizer(self, mock_client): + c = MagicMock() + c.list_agent_runtimes.return_value = { + "agentRuntimes": [ + { + "agentRuntimeName": "rt1", + "authorizerConfiguration": {"customJWTAuthorizer": {}}, + } + ] + } + mock_client.return_value = c + result = app.check_agentcore_policy_engine() + _assert_finding_structure(result) + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_warn_runtimes_without_authorizer(self, mock_client): + c = MagicMock() + c.list_agent_runtimes.return_value = { + "agentRuntimes": [{"agentRuntimeName": "rt1"}] + } + mock_client.return_value = c + result = app.check_agentcore_policy_engine() + _assert_finding_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_na_no_runtimes(self, mock_client): + c = MagicMock() + c.list_agent_runtimes.return_value = {"agentRuntimes": []} + mock_client.return_value = c + result = app.check_agentcore_policy_engine() + _assert_finding_structure(result) + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_access_denied_returns_na(self, mock_client): + c = MagicMock() + c.list_agent_runtimes.side_effect = _client_error("AccessDeniedException") + mock_client.return_value = c + result = app.check_agentcore_policy_engine() + _assert_finding_structure(result) + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("agentcore error") + result = app.check_agentcore_policy_engine() + assert result["status"] == "ERROR" + + +class TestFS09AgentTransactionLimits: + """FS-09 — Agent Transaction Limits Check.""" + + @patch("app.boto3.client") + def test_pass_concurrency_set(self, mock_client): + c = MagicMock() + c.get_function_concurrency.return_value = {"ReservedConcurrentExecutions": 10} + mock_client.return_value = c + inv = make_resource_inventory( + lambda_functions=[{"FunctionName": "my-agent-handler"}] + ) + result = app.check_agent_transaction_limits(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_warn_no_concurrency(self, mock_client): + c = MagicMock() + c.get_function_concurrency.return_value = {} + mock_client.return_value = c + inv = make_resource_inventory( + lambda_functions=[{"FunctionName": "my-agent-handler"}] + ) + result = app.check_agent_transaction_limits(inv) + _assert_finding_structure(result) + assert result["status"] == "WARN" + + def test_error_on_unavailable_inventory(self): + inv = make_resource_inventory( + lambda_functions=app._Unavailable(RuntimeError("lambda error")) + ) + result = app.check_agent_transaction_limits(inv) + assert result["status"] == "ERROR" + + +class TestFS10HumanInTheLoop: + """FS-10 — Human-in-the-Loop Approval Check.""" + + @patch("app.boto3.client") + def test_pass_wait_for_task_token(self, mock_client): + c = MagicMock() + c.list_state_machines.return_value = { + "stateMachines": [ + { + "name": "agent-approval-flow", + "stateMachineArn": "arn:aws:states:us-east-1:123:sm:test", + } + ] + } + # The function checks for '"waitForTaskToken"' (with JSON quotes) in the + # definition string, so we embed it as a standalone JSON string value. + defn = json.dumps( + { + "States": { + "Approve": { + "Type": "Task", + "Resource": "arn:aws:states:::sqs:sendMessage", + "Integration": "waitForTaskToken", + } + } + } + ) + c.describe_state_machine.return_value = {"definition": defn} + mock_client.return_value = c + result = app.check_human_in_the_loop_for_high_risk_actions() + _assert_finding_structure(result) + # The function finds "waitForTaskToken" in the definition → PASS path + assert result["status"] == "PASS" + assert len(result["csv_data"]) >= 1 + + @patch("app.boto3.client") + def test_warn_no_wait_token(self, mock_client): + c = MagicMock() + c.list_state_machines.return_value = { + "stateMachines": [ + { + "name": "agent-workflow", + "stateMachineArn": "arn:aws:states:us-east-1:123:sm:test", + } + ] + } + c.describe_state_machine.return_value = { + "definition": json.dumps({"States": {"Run": {"Type": "Task"}}}) + } + mock_client.return_value = c + result = app.check_human_in_the_loop_for_high_risk_actions() + _assert_finding_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("sfn error") + result = app.check_human_in_the_loop_for_high_risk_actions() + assert result["status"] == "ERROR" + + +class TestFS11AgentRateAlarms: + """FS-11 — Agent Rate Alarms Check.""" + + @patch("app.boto3.client") + def test_pass_agent_alarms_exist(self, mock_client): + c = MagicMock() + paginator = MagicMock() + paginator.paginate.return_value = [ + { + "MetricAlarms": [ + { + "AlarmName": "agent-invocation-rate", + "Namespace": "AWS/Bedrock", + "MetricName": "AgentInvocations", + } + ] + } + ] + c.get_paginator.return_value = paginator + mock_client.return_value = c + result = app.check_agent_rate_alarms() + _assert_finding_structure(result) + # The function looks for agent-related alarms + assert result["status"] in ("PASS", "WARN") + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("cw error") + result = app.check_agent_rate_alarms() + assert result["status"] == "ERROR" + + +# ========================================================================= +# CATEGORY 3: SUPPLY CHAIN VULNERABILITIES (FS-12 to FS-16) +# ========================================================================= + + +class TestFS12ScpModelAccess: + """FS-12 — SCP Model Access Restrictions.""" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("org error") + result = app.check_scp_model_access_restrictions() + assert result["status"] == "ERROR" + + @patch("app.boto3.client") + def test_returns_valid_structure(self, mock_client): + c = MagicMock() + c.list_policies.return_value = {"Policies": []} + mock_client.return_value = c + result = app.check_scp_model_access_restrictions() + _assert_finding_structure(result) + + +class TestFS13ModelInventoryTagging: + """FS-13 — Model Inventory Tagging.""" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("tagging error") + result = app.check_model_inventory_tagging() + assert result["status"] == "ERROR" + + @patch("app.boto3.client") + def test_returns_valid_structure(self, mock_client): + c = MagicMock() + c.list_custom_models.return_value = {"modelSummaries": []} + c.list_models.return_value = {"Models": []} + mock_client.return_value = c + result = app.check_model_inventory_tagging() + _assert_finding_structure(result) + + +class TestFS14ModelOnboardingGovernance: + """FS-14 — Model Onboarding Governance.""" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("config error") + result = app.check_model_onboarding_governance() + assert result["status"] == "ERROR" + + @patch("app.boto3.client") + def test_returns_valid_structure(self, mock_client): + c = MagicMock() + c.describe_config_rules.return_value = {"ConfigRules": []} + mock_client.return_value = c + result = app.check_model_onboarding_governance() + _assert_finding_structure(result) + + +class TestFS15BedrockModelEvalAdversarial: + """FS-15 — Bedrock Model Evaluation Adversarial.""" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("eval error") + result = app.check_bedrock_model_evaluation_adversarial() + assert result["status"] == "ERROR" + + @patch("app.boto3.client") + def test_returns_valid_structure(self, mock_client): + c = MagicMock() + c.list_evaluation_jobs.return_value = {"jobSummaries": []} + mock_client.return_value = c + result = app.check_bedrock_model_evaluation_adversarial() + _assert_finding_structure(result) + + @patch("app.boto3.client") + def test_fail_no_eval_jobs(self, mock_client): + """REQ-10a: no Bedrock evaluation jobs → Failed/Medium (was N/A).""" + c = MagicMock() + c.list_evaluation_jobs.return_value = {"jobSummaries": []} + mock_client.return_value = c + result = app.check_bedrock_model_evaluation_adversarial() + _assert_finding_structure(result) + assert any( + r["Finding"] == "No Bedrock Evaluation Jobs Found" + and r["Status"] == "Failed" + and r["Severity"] == "Medium" + for r in result["csv_data"] + ) + + @patch("app.boto3.client") + def test_pass_eval_jobs_present(self, mock_client): + """Eval jobs present → Passed/Medium.""" + c = MagicMock() + c.list_evaluation_jobs.return_value = { + "jobSummaries": [{"jobName": "robustness-eval"}] + } + mock_client.return_value = c + result = app.check_bedrock_model_evaluation_adversarial() + _assert_finding_structure(result) + assert any( + r["Finding"] == "Bedrock Evaluation Jobs Present" + and r["Status"] == "Passed" + for r in result["csv_data"] + ) + + +class TestFS16EcrImageScanning: + """FS-16 — ECR Image Scanning.""" + + @patch("app.boto3.client") + def test_pass_scanning_enabled(self, mock_client): + c = MagicMock() + c.describe_repositories.return_value = { + "repositories": [ + { + "repositoryName": "ml-model", + "imageScanningConfiguration": {"scanOnPush": True}, + } + ] + } + mock_client.return_value = c + result = app.check_ecr_image_scanning() + _assert_finding_structure(result) + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("ecr error") + result = app.check_ecr_image_scanning() + assert result["status"] == "ERROR" + + +# ========================================================================= +# CATEGORY 4: TRAINING DATA & MODEL POISONING (FS-20, FS-21) +# ========================================================================= + + +class TestFS20FeatureStoreRollback: + """FS-20 — Feature Store Rollback Capability.""" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("fs error") + result = app.check_feature_store_rollback_capability() + assert result["status"] == "ERROR" + + @patch("app.boto3.client") + def test_returns_valid_structure(self, mock_client): + c = MagicMock() + c.list_feature_groups.return_value = {"FeatureGroupSummaries": []} + mock_client.return_value = c + result = app.check_feature_store_rollback_capability() + _assert_finding_structure(result) + + +class TestFS21TrainingDataS3Versioning: + """FS-21 — Training Data S3 Versioning.""" + + def test_error_on_unavailable_inventory(self): + """When the buckets inventory is _Unavailable, check must return ERROR.""" + inv = make_resource_inventory( + buckets=app._Unavailable(RuntimeError("s3 error")) + ) + result = app.check_training_data_s3_versioning(inv) + assert result["status"] == "ERROR" + + @patch("app.boto3.client") + def test_returns_valid_structure(self, mock_client): + """Empty bucket list → N/A finding (no training buckets identified).""" + inv = make_resource_inventory(buckets=[]) + mock_client.return_value = MagicMock() + result = app.check_training_data_s3_versioning(inv) + _assert_finding_structure(result) + + +# ========================================================================= +# CATEGORY 5: VECTOR & EMBEDDING WEAKNESSES (FS-22, FS-24, FS-25, FS-26) +# ========================================================================= + + +class TestFS22KnowledgeBaseIamLeastPrivilege: + """FS-22 — Knowledge Base IAM Least Privilege (takes permission_cache).""" + + def test_pass_empty_cache(self, permission_cache_empty): + # FS-22 reads only the permission cache (no boto3 calls). An empty cache + # means no roles to inspect → PASS with no wildcard findings. + result = app.check_knowledge_base_iam_least_privilege(permission_cache_empty) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + """FS-22 only reads permission_cache (no boto3 calls). To trigger + the error path, pass a cache that causes an exception during iteration.""" + # A non-dict value for role_permissions will cause .items() to fail + bad_cache = {"role_permissions": "not-a-dict"} + result = app.check_knowledge_base_iam_least_privilege(bad_cache) + assert result["status"] == "ERROR" + + def test_single_statement_dict_no_crash_wildcard(self): + """REQ-3: a policy whose Statement is a single dict (not a list) must not + crash ('str' object has no attribute 'get'); a wildcard is still flagged.""" + cache = { + "role_permissions": { + "AmazonBedrockExecutionRoleForKnowledgeBase_Test": { + "attached_policies": [], + "inline_policies": [ + { + "name": "KBInline", + "document": { + "Version": "2012-10-17", + "Statement": { + "Effect": "Allow", + "Action": "bedrock-agent:*", + "Resource": "*", + }, + }, + } + ], + } + } + } + result = app.check_knowledge_base_iam_least_privilege(cache) + _assert_finding_structure(result) + assert result["status"] != "ERROR" + assert any( + r["Finding"] == "Overly Permissive Knowledge Base IAM Roles" + and r["Status"] == "Failed" + and r["Severity"] == "High" + for r in result["csv_data"] + ) + + def test_single_statement_dict_no_wildcard(self): + """Single-statement-dict policy without a wildcard → Passed/High, no crash.""" + cache = { + "role_permissions": { + "KBRole": { + "attached_policies": [], + "inline_policies": [ + { + "name": "KBInline", + "document": { + "Statement": { + "Effect": "Allow", + "Action": "bedrock:Retrieve", + "Resource": "arn:aws:bedrock:*:*:knowledge-base/*", + } + }, + } + ], + } + } + } + result = app.check_knowledge_base_iam_least_privilege(cache) + _assert_finding_structure(result) + assert result["status"] != "ERROR" + assert any( + r["Finding"] == "Knowledge Base IAM Permissions Look Appropriate" + and r["Status"] == "Passed" + for r in result["csv_data"] + ) + + def test_partial_wildcard_flagged(self): + """REQ-14/D: a partial wildcard (e.g. 'bedrock-agent:Get*') is over-broad + and must be flagged, not just the three exact full wildcards.""" + cache = { + "role_permissions": { + "KBRole": { + "attached_policies": [], + "inline_policies": [ + { + "name": "KBInline", + "document": { + "Statement": [ + { + "Effect": "Allow", + "Action": "bedrock-agent:Get*", + "Resource": "arn:aws:bedrock:*:*:knowledge-base/kb-1", + } + ] + }, + } + ], + } + } + } + result = app.check_knowledge_base_iam_least_privilege(cache) + _assert_finding_structure(result) + assert result["status"] == "WARN" + assert any( + r["Finding"] == "Overly Permissive Knowledge Base IAM Roles" + and r["Status"] == "Failed" + for r in result["csv_data"] + ) + + def test_unscoped_resource_flagged(self): + """REQ-14/D: a scoped action on Resource '*' (no ARN scoping) is flagged.""" + cache = { + "role_permissions": { + "KBRole": { + "attached_policies": [], + "inline_policies": [ + { + "name": "KBInline", + "document": { + "Statement": [ + { + "Effect": "Allow", + "Action": "bedrock:Retrieve", + "Resource": "*", + } + ] + }, + } + ], + } + } + } + result = app.check_knowledge_base_iam_least_privilege(cache) + _assert_finding_structure(result) + assert result["status"] == "WARN" + assert any(r["Status"] == "Failed" for r in result["csv_data"]) + + def test_not_action_allow_flagged(self): + """REQ-14/D: a NotAction Allow grants everything except listed actions and + is inherently over-broad → flagged.""" + cache = { + "role_permissions": { + "KBRole": { + "attached_policies": [], + "inline_policies": [ + { + "name": "KBInline", + "document": { + "Statement": [ + { + "Effect": "Allow", + "NotAction": "s3:DeleteObject", + "Resource": "arn:aws:bedrock:*:*:knowledge-base/kb-1", + } + ] + }, + } + ], + } + } + } + result = app.check_knowledge_base_iam_least_privilege(cache) + _assert_finding_structure(result) + assert result["status"] == "WARN" + assert any(r["Status"] == "Failed" for r in result["csv_data"]) + + def test_scoped_specific_actions_pass(self): + """REQ-14/D: properly scoped specific actions on a specific KB ARN → Passed + (no false positive from the widened detection).""" + cache = { + "role_permissions": { + "KBRole": { + "attached_policies": [], + "inline_policies": [ + { + "name": "KBInline", + "document": { + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "bedrock:Retrieve", + "bedrock:RetrieveAndGenerate", + ], + "Resource": "arn:aws:bedrock:us-east-1:111122223333:knowledge-base/kb-1", + } + ] + }, + } + ], + } + } + } + result = app.check_knowledge_base_iam_least_privilege(cache) + _assert_finding_structure(result) + assert result["status"] == "PASS" + assert any( + r["Finding"] == "Knowledge Base IAM Permissions Look Appropriate" + and r["Status"] == "Passed" + for r in result["csv_data"] + ) + + +class TestFS24KnowledgeBaseMetadataFiltering: + """FS-24 — Knowledge Base Metadata Filtering.""" + + def test_error_on_unavailable_inventory(self): + inv = make_resource_inventory( + knowledge_bases=app._Unavailable(RuntimeError("kb error")) + ) + result = app.check_knowledge_base_metadata_filtering(inv) + assert result["status"] == "ERROR" + + def test_returns_valid_structure_no_kbs(self): + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[], data_sources_by_kb={}, data_source_detail={} + ) + ) + result = app.check_knowledge_base_metadata_filtering(inv) + _assert_finding_structure(result) + + def test_returns_advisory_with_kbs(self): + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1", "name": "rag-kb"}], + data_sources_by_kb={}, + data_source_detail={}, + ) + ) + result = app.check_knowledge_base_metadata_filtering(inv) + _assert_finding_structure(result) + assert any("ADVISORY" in r.get("Finding", "") for r in result["csv_data"]) + + +class TestFS25OpensearchServerlessEncryption: + """FS-25 — OpenSearch Serverless Encryption.""" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("oss error") + result = app.check_opensearch_serverless_encryption() + assert result["status"] == "ERROR" + + +class TestFS26KnowledgeBaseVpcAccess: + """FS-26 — Knowledge Base VPC Access.""" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("vpc error") + result = app.check_knowledge_base_vpc_access() + assert result["status"] == "ERROR" + + +# ========================================================================= +# CATEGORY 6: NON-COMPLIANT OUTPUT (FS-27 to FS-30) +# ========================================================================= + + +class TestFS27GuardrailContextualGrounding: + """FS-27 — Guardrail Contextual Grounding Check (renamed from check_automated_reasoning_checks).""" + + def test_pass_grounding_configured(self): + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "finserv-guard"}], + detail_by_id={ + "g1": { + "contextualGroundingPolicy": { + "filters": [{"type": "GROUNDING", "threshold": 0.7}] + } + } + }, + ) + ) + result = app.check_guardrail_contextual_grounding(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + def test_warn_no_grounding(self): + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "finserv-guard"}], + detail_by_id={"g1": {}}, + ) + ) + result = app.check_guardrail_contextual_grounding(inv) + _assert_finding_structure(result) + assert result["status"] == "WARN" + + def test_error_on_exception(self): + inv = make_resource_inventory( + guardrails=app._Unavailable(RuntimeError("arc error")) + ) + result = app.check_guardrail_contextual_grounding(inv) + assert result["status"] == "ERROR" + + +class TestFS27AutomatedReasoningPolicies: + """FS-27b — Automated Reasoning Policies Check (new, GA August 2025).""" + + @patch("app.boto3.client") + def test_pass_policies_exist(self, mock_client): + c = MagicMock() + c.list_automated_reasoning_policies.return_value = { + "automatedReasoningPolicySummaries": [ + {"name": "loan-eligibility-policy", "policyId": "pol-001"} + ] + } + mock_client.return_value = c + result = app.check_automated_reasoning_policies() + _assert_finding_structure(result) + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_warn_no_policies(self, mock_client): + c = MagicMock() + c.list_automated_reasoning_policies.return_value = { + "automatedReasoningPolicySummaries": [] + } + mock_client.return_value = c + result = app.check_automated_reasoning_policies() + _assert_finding_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_access_denied_returns_na(self, mock_client): + c = MagicMock() + c.list_automated_reasoning_policies.side_effect = _client_error( + "AccessDeniedException" + ) + mock_client.return_value = c + result = app.check_automated_reasoning_policies() + _assert_finding_structure(result) + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("arc error") + result = app.check_automated_reasoning_policies() + assert result["status"] == "ERROR" + + +class TestFS28GuardrailDeniedTopicsFinancial: + """FS-28 — Guardrail Denied Topics Financial.""" + + def test_pass_denied_topics_configured(self): + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "finserv-guard"}], + detail_by_id={ + "g1": { + "topicPolicy": { + "topics": [{"name": "financial-advice", "type": "DENY"}], + "tier": {"tierName": "STANDARD"}, + } + } + }, + ) + ) + result = app.check_guardrail_denied_topics_financial(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + def test_classic_tier_advisory(self): + # Topics exist but on CLASSIC tier → Low-severity advisory (still Passed wrapper). + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "finserv-guard"}], + detail_by_id={ + "g1": { + "topicPolicy": { + "topics": [{"name": "financial-advice", "type": "DENY"}], + "tier": {"tierName": "CLASSIC"}, + } + } + }, + ) + ) + result = app.check_guardrail_denied_topics_financial(inv) + _assert_finding_structure(result) + assert any( + r["Finding"] == "Denied Topics Configured on CLASSIC Tier" + and r["Severity"] == "High" + for r in result["csv_data"] + ) + + def test_error_on_exception(self): + inv = make_resource_inventory( + guardrails=app._Unavailable(RuntimeError("topic error")) + ) + result = app.check_guardrail_denied_topics_financial(inv) + assert result["status"] == "ERROR" + + +class TestFS29ComplianceDisclaimer: + """FS-29 — Advisory check (no AWS API calls).""" + + def test_returns_valid_structure(self): + result = app.check_compliance_disclaimer_in_outputs() + _assert_finding_structure(result) + assert result["status"] == "PASS" + _assert_advisory_retag(result, "FS-29") + assert len(result["csv_data"]) >= 1 + assert result["csv_data"][0]["Check_ID"] == "FS-29" + + +class TestFS30BedrockEvalComplianceDatasets: + """FS-30 — Advisory (cannot inspect eval-job dataset content; REQ-10a).""" + + def test_returns_advisory_structure(self): + result = app.check_bedrock_evaluation_compliance_datasets() + _assert_finding_structure(result) + _assert_advisory_retag(result, "FS-30") + assert result["csv_data"][0]["Check_ID"] == "FS-30" + + +# ========================================================================= +# CATEGORY 7: MISINFORMATION (FS-31 to FS-34) +# ========================================================================= + + +class TestFS31KnowledgeBaseDataSourceSync: + """FS-31 — Knowledge Base Data Source Sync.""" + + def test_error_on_unavailable_inventory(self): + inv = make_resource_inventory( + knowledge_bases=app._Unavailable(RuntimeError("sync error")) + ) + result = app.check_knowledge_base_data_source_sync(inv) + assert result["status"] == "ERROR" + + def test_na_no_kbs(self): + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[], data_sources_by_kb={}, data_source_detail={} + ) + ) + result = app.check_knowledge_base_data_source_sync(inv) + _assert_finding_structure(result) + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + + +class TestFS32SourceAttribution: + """FS-32 — Advisory check (no AWS API calls).""" + + def test_returns_valid_structure(self): + result = app.check_source_attribution_in_guardrails() + _assert_finding_structure(result) + assert result["status"] == "PASS" + _assert_advisory_retag(result, "FS-32") + assert result["csv_data"][0]["Check_ID"] == "FS-32" + + +class TestFS33KnowledgeBaseIntegrityMonitoring: + """FS-33 — Knowledge Base Integrity Monitoring.""" + + def test_error_on_unavailable_inventory(self): + inv = make_resource_inventory( + knowledge_bases=app._Unavailable(RuntimeError("integrity error")) + ) + result = app.check_knowledge_base_integrity_monitoring(inv) + assert result["status"] == "ERROR" + + def test_na_no_kbs(self): + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[], data_sources_by_kb={}, data_source_detail={} + ) + ) + result = app.check_knowledge_base_integrity_monitoring(inv) + _assert_finding_structure(result) + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + + +class TestFS34FmVersionCurrency: + """FS-34 — FM Version Currency Advisory.""" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("fm error") + result = app.check_fm_version_currency() + assert result["status"] == "ERROR" + + +# ========================================================================= +# CATEGORY 8: ABUSIVE OR HARMFUL OUTPUT (FS-35 to FS-38) +# ========================================================================= + + +class TestFS35FmevalHarmfulContent: + """FS-35 — Advisory (cannot inspect eval-job dataset content; REQ-10a).""" + + def test_returns_advisory_structure(self): + result = app.check_fmeval_harmful_content() + _assert_finding_structure(result) + _assert_advisory_retag(result, "FS-35") + assert result["csv_data"][0]["Check_ID"] == "FS-35" + + +class TestFS36GuardrailContentFilters: + """FS-36 — Guardrail Content Filters.""" + + def test_pass_content_filters_configured(self): + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": { + "contentPolicy": { + "filters": [ + { + "type": "SEXUAL", + "inputStrength": "HIGH", + "outputStrength": "HIGH", + }, + { + "type": "VIOLENCE", + "inputStrength": "HIGH", + "outputStrength": "HIGH", + }, + { + "type": "HATE", + "inputStrength": "HIGH", + "outputStrength": "HIGH", + }, + { + "type": "INSULTS", + "inputStrength": "HIGH", + "outputStrength": "HIGH", + }, + ] + } + } + }, + ) + ) + result = app.check_guardrail_content_filters(inv) + _assert_finding_structure(result) + + def test_error_on_exception(self): + inv = make_resource_inventory( + guardrails=app._Unavailable(RuntimeError("content error")) + ) + result = app.check_guardrail_content_filters(inv) + assert result["status"] == "ERROR" + + +class TestFS37UserFeedbackMechanism: + """FS-37 — Advisory check (no AWS API calls).""" + + def test_returns_valid_structure(self): + result = app.check_user_feedback_mechanism() + _assert_finding_structure(result) + assert result["status"] == "PASS" + _assert_advisory_retag(result, "FS-37") + assert result["csv_data"][0]["Check_ID"] == "FS-37" + + +class TestFS38GuardrailWordFilters: + """FS-38 — Guardrail Word Filters.""" + + def test_error_on_exception(self): + inv = make_resource_inventory( + guardrails=app._Unavailable(RuntimeError("word error")) + ) + result = app.check_guardrail_word_filters(inv) + assert result["status"] == "ERROR" + + +# ========================================================================= +# CATEGORY 9: BIASED OUTPUT (FS-39 to FS-42) +# ========================================================================= + + +class TestFS39SagemakerClarifyBias: + """FS-39 — SageMaker Clarify Bias.""" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("clarify error") + result = app.check_sagemaker_clarify_bias() + assert result["status"] == "ERROR" + + +class TestFS40BedrockEvalBiasDatasets: + """FS-40 — Advisory (cannot inspect eval-job dataset content; REQ-10a).""" + + def test_returns_advisory_structure(self): + result = app.check_bedrock_evaluation_bias_datasets() + _assert_finding_structure(result) + _assert_advisory_retag(result, "FS-40") + assert result["csv_data"][0]["Check_ID"] == "FS-40" + + +class TestFS41SagemakerClarifyExplainability: + """FS-41 — SageMaker Clarify Explainability.""" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("explain error") + result = app.check_sagemaker_clarify_explainability() + assert result["status"] == "ERROR" + + +class TestFS42AiServiceCards: + """FS-42 — AI Service Cards Documentation Advisory.""" + + @patch("app.boto3.client") + def test_returns_valid_structure(self, mock_client): + c = MagicMock() + c.list_model_cards.return_value = {"ModelCardSummaries": []} + mock_client.return_value = c + result = app.check_ai_service_cards_documentation() + _assert_finding_structure(result) + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("cards error") + result = app.check_ai_service_cards_documentation() + assert result["status"] == "ERROR" + + +# ========================================================================= +# CATEGORY 10: SENSITIVE INFORMATION DISCLOSURE (FS-43 to FS-46) +# ========================================================================= + + +class TestFS43CloudwatchLogPiiMasking: + """FS-43 — CloudWatch Log PII Masking.""" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("logs error") + result = app.check_cloudwatch_log_pii_masking() + assert result["status"] == "ERROR" + + +class TestFS44MacieOnTrainingDataBuckets: + """FS-44 — Macie on Training Data Buckets.""" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("macie error") + result = app.check_macie_on_training_data_buckets() + assert result["status"] == "ERROR" + + +class TestFS45GuardrailPiiFilters: + """FS-45 — Guardrail PII Filters.""" + + def test_pass_pii_filters_configured(self): + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": { + "sensitiveInformationPolicy": { + "piiEntities": [ + {"type": "SSN", "action": "BLOCK"}, + {"type": "CREDIT_DEBIT_CARD_NUMBER", "action": "BLOCK"}, + ] + } + } + }, + ) + ) + result = app.check_guardrail_pii_filters(inv) + _assert_finding_structure(result) + + def test_error_on_exception(self): + inv = make_resource_inventory( + guardrails=app._Unavailable(RuntimeError("pii error")) + ) + result = app.check_guardrail_pii_filters(inv) + assert result["status"] == "ERROR" + + +class TestFS46DataClassificationTagging: + """FS-46 — Data Classification Tagging.""" + + def test_error_on_unavailable_inventory(self): + """When the buckets inventory is _Unavailable, check must return ERROR.""" + inv = make_resource_inventory( + buckets=app._Unavailable(RuntimeError("tag error")) + ) + result = app.check_data_classification_tagging(inv) + assert result["status"] == "ERROR" + + +# ========================================================================= +# CATEGORY 11: HALLUCINATION (FS-47 to FS-50) +# ========================================================================= + + +class TestFS47GuardrailGroundingThreshold: + """FS-47 — Guardrail Grounding Threshold.""" + + def test_error_on_exception(self): + inv = make_resource_inventory( + guardrails=app._Unavailable(RuntimeError("grounding error")) + ) + result = app.check_guardrail_grounding_threshold(inv) + assert result["status"] == "ERROR" + + +class TestFS48RagKnowledgeBaseConfigured: + """FS-48 — RAG Knowledge Base Configured.""" + + def test_pass_kbs_exist(self): + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[ + {"knowledgeBaseId": "kb1", "name": "rag-kb", "status": "ACTIVE"} + ], + data_sources_by_kb={}, + data_source_detail={}, + ) + ) + result = app.check_rag_knowledge_base_configured(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + def test_error_on_unavailable_inventory(self): + inv = make_resource_inventory( + knowledge_bases=app._Unavailable(RuntimeError("rag error")) + ) + result = app.check_rag_knowledge_base_configured(inv) + assert result["status"] == "ERROR" + + +class TestFS49HallucinationDisclaimer: + """FS-49 — Advisory check (no AWS API calls).""" + + def test_returns_valid_structure(self): + result = app.check_hallucination_disclaimer_advisory() + _assert_finding_structure(result) + assert result["status"] == "PASS" + _assert_advisory_retag(result, "FS-49") + assert result["csv_data"][0]["Check_ID"] == "FS-49" + + +class TestFS50GuardrailRelevanceGrounding: + """FS-50 — Guardrail Relevance Grounding Check (renamed from check_automated_reasoning_checks_hallucination).""" + + def test_pass_relevance_filter_present(self): + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": { + "contextualGroundingPolicy": { + "filters": [{"type": "RELEVANCE", "threshold": 0.7}] + } + } + }, + ) + ) + result = app.check_guardrail_relevance_grounding(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + def test_warn_no_relevance_filter(self): + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": { + "contextualGroundingPolicy": { + "filters": [{"type": "GROUNDING", "threshold": 0.7}] + } + } + }, + ) + ) + result = app.check_guardrail_relevance_grounding(inv) + _assert_finding_structure(result) + assert result["status"] == "WARN" + + def test_error_on_exception(self): + inv = make_resource_inventory( + guardrails=app._Unavailable(RuntimeError("arc error")) + ) + result = app.check_guardrail_relevance_grounding(inv) + assert result["status"] == "ERROR" + + +# ========================================================================= +# CATEGORY 12: PROMPT INJECTION (FS-51 to FS-54) +# ========================================================================= + + +class TestFS51PromptInjectionInputValidation: + """FS-51 — Prompt Injection Input Validation.""" + + def test_pass_prompt_attack_filter(self): + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": { + "contentPolicy": { + "filters": [ + { + "type": "PROMPT_ATTACK", + "inputStrength": "HIGH", + "outputStrength": "NONE", + } + ] + } + } + }, + ) + ) + result = app.check_prompt_injection_input_validation(inv) + _assert_finding_structure(result) + + def test_error_on_exception(self): + inv = make_resource_inventory( + guardrails=app._Unavailable(RuntimeError("prompt error")) + ) + result = app.check_prompt_injection_input_validation(inv) + assert result["status"] == "ERROR" + + +class TestFS52BedrockSdkVersionCurrency: + """FS-52 — Bedrock SDK Version Currency Advisory.""" + + def test_error_on_unavailable_inventory(self): + inv = make_resource_inventory( + lambda_functions=app._Unavailable(RuntimeError("sdk error")) + ) + result = app.check_bedrock_sdk_version_currency(inv) + assert result["status"] == "ERROR" + + +class TestFS53WafSqlInjectionRules: + """FS-53 — WAF SQL Injection Rules.""" + + def test_pass_managed_rules_present(self): + """ACL with AWSManagedRulesSQLiRuleSet in inventory → PASS.""" + acl_detail = { + "Rules": [ + { + "Name": "AWS-AWSManagedRulesSQLiRuleSet", + "Statement": { + "ManagedRuleGroupStatement": { + "VendorName": "AWS", + "Name": "AWSManagedRulesSQLiRuleSet", + } + }, + } + ] + } + inv = make_resource_inventory( + web_acls=app.WebAclInventory( + summaries=[{"Name": "acl1", "Id": "id1"}], + detail_by_id={"id1": acl_detail}, + ) + ) + result = app.check_waf_sql_injection_rules(inv) + _assert_finding_structure(result) + + def test_na_no_acls(self): + """Empty summaries list → N/A (no ACLs found).""" + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=[], detail_by_id={}) + ) + result = app.check_waf_sql_injection_rules(inv) + _assert_finding_structure(result) + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + + def test_warn_acls_without_injection_rules(self): + """ACL with no injection rule groups → WARN.""" + acl_detail = { + "Rules": [ + { + "Name": "rate-limit", + "Statement": {"RateBasedStatement": {"Limit": 1000}}, + } + ] + } + inv = make_resource_inventory( + web_acls=app.WebAclInventory( + summaries=[{"Name": "acl1", "Id": "id1"}], + detail_by_id={"id1": acl_detail}, + ) + ) + result = app.check_waf_sql_injection_rules(inv) + _assert_finding_structure(result) + assert result["status"] == "WARN" + + def test_error_on_exception(self): + """Unavailable inventory → COULD_NOT_ASSESS (ERROR envelope).""" + inv = make_resource_inventory( + web_acls=app._Unavailable(RuntimeError("waf error")) + ) + result = app.check_waf_sql_injection_rules(inv) + assert result["status"] == "ERROR" + + # --- Pagination-correctness proof: >100 ACLs --- + def test_pass_more_than_100_acls_all_have_rules(self): + """Inventory with 150 ACLs all having injection rules → PASS (all pages visible).""" + acl_detail = { + "Rules": [ + { + "Statement": { + "ManagedRuleGroupStatement": { + "Name": "AWSManagedRulesSQLiRuleSet" + } + } + } + ] + } + summaries = [{"Name": f"acl{i}", "Id": f"id{i}"} for i in range(150)] + detail_by_id = {f"id{i}": acl_detail for i in range(150)} + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=summaries, detail_by_id=detail_by_id) + ) + result = app.check_waf_sql_injection_rules(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + assert any( + r["Finding"] == "WAF Injection Protection Rules Present" + and "150" in r["Finding_Details"] + for r in result["csv_data"] + ) + + def test_warn_more_than_100_acls_some_missing_rules(self): + """Inventory with 150 ACLs, last 50 missing rules → WARN (previously truncated acls now detected).""" + good_detail = { + "Rules": [ + { + "Statement": { + "ManagedRuleGroupStatement": { + "Name": "AWSManagedRulesSQLiRuleSet" + } + } + } + ] + } + bad_detail = {"Rules": []} + summaries = [{"Name": f"acl{i}", "Id": f"id{i}"} for i in range(150)] + detail_by_id = {} + for i in range(100): + detail_by_id[f"id{i}"] = good_detail + for i in range(100, 150): + detail_by_id[f"id{i}"] = bad_detail + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=summaries, detail_by_id=detail_by_id) + ) + result = app.check_waf_sql_injection_rules(inv) + _assert_finding_structure(result) + assert result["status"] == "WARN" + + # --- ≤1-page equivalence: 2-ACL case matches Wave-0 baseline --- + def test_two_acl_case_unchanged(self): + """The 2-ACL scenario produces the same PASS finding as the pre-refactor baseline.""" + acl_detail = { + "Rules": [ + { + "Statement": { + "ManagedRuleGroupStatement": { + "Name": "AWSManagedRulesSQLiRuleSet" + } + } + } + ] + } + inv = make_resource_inventory( + web_acls=app.WebAclInventory( + summaries=[ + {"Name": "FinServACL1", "Id": "acl-id-001"}, + {"Name": "FinServACL2", "Id": "acl-id-002"}, + ], + detail_by_id={ + "acl-id-001": acl_detail, + "acl-id-002": acl_detail, + }, + ) + ) + result = app.check_waf_sql_injection_rules(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + assert any( + r["Finding"] == "WAF Injection Protection Rules Present" + for r in result["csv_data"] + ) + + +class TestFS54PenetrationTestingEvidence: + """FS-54 — Advisory check (no AWS API calls).""" + + def test_returns_valid_structure(self): + result = app.check_penetration_testing_evidence() + _assert_finding_structure(result) + assert result["status"] == "PASS" + _assert_advisory_retag(result, "FS-54") + assert result["csv_data"][0]["Check_ID"] == "FS-54" + + +# ========================================================================= +# CATEGORY 13: IMPROPER OUTPUT HANDLING (FS-55 to FS-58) +# ========================================================================= + + +class TestFS55OutputValidationLambda: + """FS-55 — Output Validation Lambda Check.""" + + def test_pass_validation_functions_exist(self): + inv = make_resource_inventory( + lambda_functions=[{"FunctionName": "output-validate-handler"}] + ) + result = app.check_output_validation_lambda(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + def test_warn_no_validation_functions(self): + inv = make_resource_inventory( + lambda_functions=[{"FunctionName": "my-api-handler"}] + ) + result = app.check_output_validation_lambda(inv) + _assert_finding_structure(result) + assert result["status"] == "WARN" + + def test_error_on_unavailable_inventory(self): + inv = make_resource_inventory( + lambda_functions=app._Unavailable(RuntimeError("lambda error")) + ) + result = app.check_output_validation_lambda(inv) + assert result["status"] == "ERROR" + + +class TestFS56XssPreventionWaf: + """FS-56 — XSS Prevention WAF Check.""" + + def test_pass_acls_present(self): + """ACL with AWSManagedRulesCommonRuleSet in inventory → PASS.""" + acl_detail = { + "Rules": [ + { + "Statement": { + "ManagedRuleGroupStatement": { + "Name": "AWSManagedRulesCommonRuleSet" + } + } + } + ] + } + inv = make_resource_inventory( + web_acls=app.WebAclInventory( + summaries=[{"Name": "acl1", "Id": "id1"}], + detail_by_id={"id1": acl_detail}, + ) + ) + result = app.check_xss_prevention_waf(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + assert any( + r["Finding"] == "XSS Prevention Common Rule Set Present" + for r in result["csv_data"] + ) + + def test_fail_acls_without_common_rule_set(self): + """ACL without AWSManagedRulesCommonRuleSet → FAIL.""" + acl_detail = {"Rules": []} + inv = make_resource_inventory( + web_acls=app.WebAclInventory( + summaries=[{"Name": "acl1", "Id": "id1"}], + detail_by_id={"id1": acl_detail}, + ) + ) + result = app.check_xss_prevention_waf(inv) + _assert_finding_structure(result) + assert any( + r["Finding"] == "WAF ACLs Missing Common Rule Set (XSS)" + and r["Status"] == "Failed" + and r["Severity"] == "Medium" + for r in result["csv_data"] + ) + + def test_na_no_acls(self): + """Empty summaries → N/A.""" + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=[], detail_by_id={}) + ) + result = app.check_xss_prevention_waf(inv) + _assert_finding_structure(result) + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + + def test_error_on_exception(self): + """Unavailable inventory → COULD_NOT_ASSESS (ERROR envelope).""" + inv = make_resource_inventory( + web_acls=app._Unavailable(RuntimeError("xss error")) + ) + result = app.check_xss_prevention_waf(inv) + assert result["status"] == "ERROR" + + # --- Pagination-correctness proof: >100 ACLs --- + def test_pass_more_than_100_acls_all_have_common_rule_set(self): + """150 ACLs all with AWSManagedRulesCommonRuleSet → PASS (previously truncated ACLs visible).""" + acl_detail = { + "Rules": [ + { + "Statement": { + "ManagedRuleGroupStatement": { + "Name": "AWSManagedRulesCommonRuleSet" + } + } + } + ] + } + summaries = [{"Name": f"acl{i}", "Id": f"id{i}"} for i in range(150)] + detail_by_id = {f"id{i}": acl_detail for i in range(150)} + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=summaries, detail_by_id=detail_by_id) + ) + result = app.check_xss_prevention_waf(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + def test_warn_more_than_100_acls_some_missing_xss_rules(self): + """150 ACLs, last 50 missing AWSManagedRulesCommonRuleSet → WARN (pagination fix detects them).""" + good_detail = { + "Rules": [ + { + "Statement": { + "ManagedRuleGroupStatement": { + "Name": "AWSManagedRulesCommonRuleSet" + } + } + } + ] + } + bad_detail = {"Rules": []} + summaries = [{"Name": f"acl{i}", "Id": f"id{i}"} for i in range(150)] + detail_by_id = {} + for i in range(100): + detail_by_id[f"id{i}"] = good_detail + for i in range(100, 150): + detail_by_id[f"id{i}"] = bad_detail + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=summaries, detail_by_id=detail_by_id) + ) + result = app.check_xss_prevention_waf(inv) + _assert_finding_structure(result) + assert result["status"] == "WARN" + + # --- ≤1-page equivalence: 2-ACL case matches Wave-0 baseline --- + def test_two_acl_case_unchanged(self): + """The 2-ACL PASS scenario is unchanged vs the pre-refactor baseline.""" + acl_detail = { + "Rules": [ + { + "Statement": { + "ManagedRuleGroupStatement": { + "Name": "AWSManagedRulesCommonRuleSet" + } + } + } + ] + } + inv = make_resource_inventory( + web_acls=app.WebAclInventory( + summaries=[ + {"Name": "FinServACL1", "Id": "acl-id-001"}, + {"Name": "FinServACL2", "Id": "acl-id-002"}, + ], + detail_by_id={ + "acl-id-001": acl_detail, + "acl-id-002": acl_detail, + }, + ) + ) + result = app.check_xss_prevention_waf(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + assert any( + r["Finding"] == "XSS Prevention Common Rule Set Present" + for r in result["csv_data"] + ) + + +class TestFS57OutputEncodingAdvisory: + """FS-57 — Advisory check (no AWS API calls).""" + + def test_returns_valid_structure(self): + result = app.check_output_encoding_advisory() + _assert_finding_structure(result) + assert result["status"] == "PASS" + _assert_advisory_retag(result, "FS-57") + assert result["csv_data"][0]["Check_ID"] == "FS-57" + + +class TestFS58OutputSchemaValidation: + """FS-58 — Output Schema Validation Check.""" + + def test_returns_valid_structure(self): + inv = make_resource_inventory( + lambda_functions=[{"FunctionName": "schema-validate-fn"}] + ) + result = app.check_output_schema_validation(inv) + _assert_finding_structure(result) + # REQ-2: FS-58 is advisory — N/A + Informational + "ADVISORY: " prefix, never Passed. + _assert_advisory_retag(result, "FS-58") + assert not any(r["Status"] == "Passed" for r in result["csv_data"]) + + def test_error_on_unavailable_inventory(self): + inv = make_resource_inventory( + lambda_functions=app._Unavailable(RuntimeError("schema error")) + ) + result = app.check_output_schema_validation(inv) + assert result["status"] == "ERROR" + + +# ========================================================================= +# CATEGORY 14: OFF-TOPIC & INAPPROPRIATE OUTPUT (FS-59 to FS-60) +# ========================================================================= + + +class TestFS59GuardrailTopicAllowlist: + """FS-59 — Guardrail Topic Allowlist Check.""" + + def test_pass_topics_configured(self): + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": { + "topicPolicy": { + "topics": [{"name": "medical-advice", "type": "DENY"}], + "tier": {"tierName": "STANDARD"}, + } + } + }, + ) + ) + result = app.check_guardrail_topic_allowlist(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + def test_classic_tier_advisory(self): + # Topics on CLASSIC tier → Low advisory finding (wrapper stays PASS). + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": { + "topicPolicy": { + "topics": [{"name": "medical-advice", "type": "DENY"}], + "tier": {"tierName": "CLASSIC"}, + } + } + }, + ) + ) + result = app.check_guardrail_topic_allowlist(inv) + _assert_finding_structure(result) + assert any( + r["Finding"] == "Topic Restrictions Configured on CLASSIC Tier" + and r["Severity"] == "Medium" + for r in result["csv_data"] + ) + + def test_warn_no_topics(self): + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={"g1": {"topicPolicy": {"topics": []}}}, + ) + ) + result = app.check_guardrail_topic_allowlist(inv) + _assert_finding_structure(result) + assert result["status"] == "WARN" + + def test_na_no_guardrails(self): + inv = make_resource_inventory( + guardrails=app.GuardrailInventory(summaries=[], detail_by_id={}) + ) + result = app.check_guardrail_topic_allowlist(inv) + _assert_finding_structure(result) + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + + def test_error_on_exception(self): + inv = make_resource_inventory( + guardrails=app._Unavailable(RuntimeError("topic error")) + ) + result = app.check_guardrail_topic_allowlist(inv) + assert result["status"] == "ERROR" + + +class TestFS60ContextualGroundingForOfftopic: + """FS-60 — Advisory check (no AWS API calls).""" + + def test_returns_valid_structure(self): + result = app.check_contextual_grounding_for_offtopic() + _assert_finding_structure(result) + assert result["status"] == "PASS" + _assert_advisory_retag(result, "FS-60") + assert result["csv_data"][0]["Check_ID"] == "FS-60" + + +# ========================================================================= +# CATEGORY 15: OUT-OF-DATE TRAINING DATA (FS-61 to FS-63) +# ========================================================================= + + +class TestFS61KnowledgeBaseSyncSchedule: + """FS-61 — Knowledge Base Sync Schedule Check.""" + + @patch("app.boto3.client") + def test_pass_sync_rules_exist(self, mock_client): + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1"}], + data_sources_by_kb={}, + data_source_detail={}, + ) + ) + + def side_effect(service, **kwargs): + if service == "events": + c = MagicMock() + c.list_rules.return_value = { + "Rules": [{"Name": "bedrock-kb-sync-daily"}] + } + return c + if service == "scheduler": + c = MagicMock() + c.list_schedules.return_value = {"Schedules": []} + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_knowledge_base_sync_schedule(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_pass_scheduler_schedule_exists(self, mock_client): + # No legacy EventBridge rule, but an EventBridge Scheduler schedule targets + # KB sync — the AWS-recommended approach must be detected (no false WARN). + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1"}], + data_sources_by_kb={}, + data_source_detail={}, + ) + ) + + def side_effect(service, **kwargs): + if service == "events": + c = MagicMock() + c.list_rules.return_value = {"Rules": []} + return c + if service == "scheduler": + c = MagicMock() + c.list_schedules.return_value = { + "Schedules": [ + { + "Name": "bedrock-kb-ingestion-daily", + "Target": { + "Arn": "arn:aws:lambda:us-east-1:1:function:sync" + }, + } + ] + } + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_knowledge_base_sync_schedule(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_warn_no_sync_rules(self, mock_client): + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1"}], + data_sources_by_kb={}, + data_source_detail={}, + ) + ) + + def side_effect(service, **kwargs): + if service == "events": + c = MagicMock() + c.list_rules.return_value = {"Rules": [{"Name": "unrelated-rule"}]} + return c + if service == "scheduler": + c = MagicMock() + c.list_schedules.return_value = { + "Schedules": [ + {"Name": "unrelated-schedule", "Target": {"Arn": "x"}} + ] + } + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_knowledge_base_sync_schedule(inv) + _assert_finding_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_scheduler_access_denied_falls_back_to_rules(self, mock_client): + # scheduler:ListSchedules denied → fall back to EventBridge rules only, + # do NOT error the whole check. + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1"}], + data_sources_by_kb={}, + data_source_detail={}, + ) + ) + + def side_effect(service, **kwargs): + if service == "events": + c = MagicMock() + c.list_rules.return_value = { + "Rules": [{"Name": "bedrock-kb-sync-daily"}] + } + return c + if service == "scheduler": + c = MagicMock() + c.list_schedules.side_effect = _client_error("AccessDeniedException") + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_knowledge_base_sync_schedule(inv) + _assert_finding_structure(result) + # EventBridge rule still matched → PASS despite scheduler access denial. + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_scheduler_access_denied_no_rules_could_not_assess(self, mock_client): + """REQ-11/A3: scheduler:ListSchedules denied AND no matching EventBridge + rule → we cannot conclude absence → COULD_NOT_ASSESS (check returns ERROR + so the handler synthesizes the N/A row), NOT a false Failed.""" + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1"}], + data_sources_by_kb={}, + data_source_detail={}, + ) + ) + + def side_effect(service, **kwargs): + if service == "events": + c = MagicMock() + c.list_rules.return_value = {"Rules": []} + return c + if service == "scheduler": + c = MagicMock() + c.list_schedules.side_effect = _client_error("AccessDeniedException") + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_knowledge_base_sync_schedule(inv) + # Re-raised access error → ERROR envelope (handler → COULD NOT ASSESS row). + assert result["status"] == "ERROR" + assert not any(r.get("Status") == "Failed" for r in result.get("csv_data", [])) + + def test_error_on_unavailable_inventory(self): + inv = make_resource_inventory( + knowledge_bases=app._Unavailable(RuntimeError("sync error")) + ) + result = app.check_knowledge_base_sync_schedule(inv) + assert result["status"] == "ERROR" + + +class TestFS62DataCurrencyDisclaimer: + """FS-62 — Advisory check (no AWS API calls).""" + + def test_returns_valid_structure(self): + result = app.check_data_currency_disclaimer_advisory() + _assert_finding_structure(result) + assert result["status"] == "PASS" + _assert_advisory_retag(result, "FS-62") + assert result["csv_data"][0]["Check_ID"] == "FS-62" + + +class TestFS63FoundationModelLifecyclePolicy: + """FS-63 — Foundation Model Lifecycle Policy Check.""" + + @patch("app.boto3.client") + def test_pass_no_legacy_models(self, mock_client): + c = MagicMock() + c.list_foundation_models.return_value = { + "modelSummaries": [ + { + "modelId": "anthropic.claude-v2", + "modelLifecycle": {"status": "ACTIVE"}, + } + ] + } + c.describe_config_rules.return_value = {"ConfigRules": []} + mock_client.return_value = c + result = app.check_foundation_model_lifecycle_policy() + _assert_finding_structure(result) + + @patch("app.boto3.client") + def test_warn_legacy_models_no_rules(self, mock_client): + c = MagicMock() + c.list_foundation_models.return_value = { + "modelSummaries": [ + {"modelId": "old-model-v1", "modelLifecycle": {"status": "LEGACY"}} + ] + } + c.describe_config_rules.return_value = {"ConfigRules": []} + mock_client.return_value = c + result = app.check_foundation_model_lifecycle_policy() + _assert_finding_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("lifecycle error") + result = app.check_foundation_model_lifecycle_policy() + assert result["status"] == "ERROR" + + +# ========================================================================= +# MATERIAL GAP CHECKS (FS-65 to FS-69) +# ========================================================================= + + +class TestFS65KbDatasourceS3EventNotifications: + """FS-65 — KB Data Source S3 Event Notifications Check.""" + + def test_na_no_kbs(self): + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[], data_sources_by_kb={}, data_source_detail={} + ) + ) + result = app.check_kb_datasource_s3_event_notifications(inv) + _assert_finding_structure(result) + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_pass_notifications_configured(self, mock_client): + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1"}], + data_sources_by_kb={"kb1": [{"dataSourceId": "ds1", "name": "s3-src"}]}, + data_source_detail={ + ("kb1", "ds1"): { + "dataSource": { + "dataSourceConfiguration": { + "s3Configuration": { + "bucketArn": "arn:aws:s3:::my-kb-bucket" + } + } + } + } + }, + ) + ) + + def side_effect(service, **kwargs): + if service == "s3": + c = MagicMock() + c.get_bucket_notification_configuration.return_value = { + "EventBridgeConfiguration": {"EventBridgeEnabled": True} + } + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_kb_datasource_s3_event_notifications(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_warn_no_notifications(self, mock_client): + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1"}], + data_sources_by_kb={"kb1": [{"dataSourceId": "ds1", "name": "s3-src"}]}, + data_source_detail={ + ("kb1", "ds1"): { + "dataSource": { + "dataSourceConfiguration": { + "s3Configuration": { + "bucketArn": "arn:aws:s3:::my-kb-bucket" + } + } + } + } + }, + ) + ) + + def side_effect(service, **kwargs): + if service == "s3": + c = MagicMock() + c.get_bucket_notification_configuration.return_value = {} + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_kb_datasource_s3_event_notifications(inv) + _assert_finding_structure(result) + assert result["status"] == "WARN" + + def test_error_on_unavailable_inventory(self): + inv = make_resource_inventory( + knowledge_bases=app._Unavailable(RuntimeError("s3 event error")) + ) + result = app.check_kb_datasource_s3_event_notifications(inv) + assert result["status"] == "ERROR" + + +class TestFS66AgentcoreEndUserIdentityPropagation: + """FS-66 — AgentCore End-User Identity Propagation Check.""" + + @patch("app.boto3.client") + def test_pass_authorizer_configured(self, mock_client): + c = MagicMock() + c.list_agent_runtimes.return_value = { + "agentRuntimes": [ + { + "agentRuntimeName": "rt1", + "authorizerConfiguration": { + "customJWTAuthorizer": {"issuerUrl": "https://example.com"} + }, + } + ] + } + mock_client.return_value = c + result = app.check_agentcore_end_user_identity_propagation() + _assert_finding_structure(result) + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_warn_no_authorizer(self, mock_client): + c = MagicMock() + c.list_agent_runtimes.return_value = { + "agentRuntimes": [ + {"agentRuntimeName": "rt1", "authorizerConfiguration": {}} + ] + } + mock_client.return_value = c + result = app.check_agentcore_end_user_identity_propagation() + _assert_finding_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_access_denied_returns_na(self, mock_client): + c = MagicMock() + c.list_agent_runtimes.side_effect = _client_error("AccessDeniedException") + mock_client.return_value = c + result = app.check_agentcore_end_user_identity_propagation() + _assert_finding_structure(result) + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("identity error") + result = app.check_agentcore_end_user_identity_propagation() + assert result["status"] == "ERROR" + + +class TestFS67AgentFinancialTransactionThresholds: + """FS-67 — Agent Financial Transaction Value Thresholds Check.""" + + def test_pass_threshold_env_vars(self): + inv = make_resource_inventory( + lambda_functions=[ + { + "FunctionName": "agent-transaction-handler", + "Environment": {"Variables": {"MAX_TRANSACTION_AMOUNT": "10000"}}, + } + ] + ) + result = app.check_agent_financial_transaction_thresholds(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + def test_warn_no_threshold_env_vars(self): + inv = make_resource_inventory( + lambda_functions=[ + { + "FunctionName": "agent-transaction-handler", + "Environment": {"Variables": {"LOG_LEVEL": "INFO"}}, + } + ] + ) + result = app.check_agent_financial_transaction_thresholds(inv) + _assert_finding_structure(result) + assert result["status"] == "WARN" + + def test_na_no_agent_lambdas(self): + inv = make_resource_inventory( + lambda_functions=[{"FunctionName": "unrelated-function"}] + ) + result = app.check_agent_financial_transaction_thresholds(inv) + _assert_finding_structure(result) + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + + def test_error_on_unavailable_inventory(self): + inv = make_resource_inventory( + lambda_functions=app._Unavailable(RuntimeError("transaction error")) + ) + result = app.check_agent_financial_transaction_thresholds(inv) + assert result["status"] == "ERROR" + + +class TestFS68ApiGatewayRequestBodySizeLimits: + """FS-68 — API Gateway Request Body Size Limits Check.""" + + def _size_constraint_detail(self): + return { + "Rules": [ + { + "Name": "body-size-rule", + "Statement": { + "SizeConstraintStatement": { + "FieldToMatch": {"Body": {}}, + "ComparisonOperator": "LE", + "Size": 8192, + } + }, + } + ] + } + + def test_pass_validators_and_waf_rules(self): + """API with maxLength model + WAF ACL with SizeConstraint → PASS.""" + acl_detail = self._size_constraint_detail() + inv = make_resource_inventory( + web_acls=app.WebAclInventory( + summaries=[{"Name": "acl1", "Id": "id1"}], + detail_by_id={"id1": acl_detail}, + ) + ) + with patch("app.boto3.client") as mock_client: + + def side_effect(service, **kwargs): + if service == "apigateway": + c = MagicMock() + c.get_rest_apis.return_value = { + "items": [{"id": "api1", "name": "genai-api"}] + } + c.get_request_validators.return_value = { + "items": [{"id": "v1", "name": "body-validator"}] + } + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_api_gateway_request_body_size_limits(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + def test_na_no_rest_apis_no_waf(self): + """REQ-4: zero REST APIs AND zero WAF ACLs → N/A (not a false Passed).""" + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=[], detail_by_id={}) + ) + with patch("app.boto3.client") as mock_client: + + def side_effect(service, **kwargs): + if service == "apigateway": + c = MagicMock() + c.get_rest_apis.return_value = {"items": []} + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_api_gateway_request_body_size_limits(inv) + _assert_finding_structure(result) + assert any( + r["Finding"] == "API Gateway Request Body Size Limits — Not Applicable" + and r["Status"] == "N/A" + and r["Severity"] == "Informational" + for r in result["csv_data"] + ) + assert not any(r["Status"] == "Passed" for r in result["csv_data"]) + + def test_fail_rest_api_without_validator(self): + """REST API exists but has no request validator → Failed/Medium.""" + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=[], detail_by_id={}) + ) + with patch("app.boto3.client") as mock_client: + + def side_effect(service, **kwargs): + if service == "apigateway": + c = MagicMock() + c.get_rest_apis.return_value = { + "items": [{"id": "api1", "name": "genai-api"}] + } + c.get_request_validators.return_value = {"items": []} + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_api_gateway_request_body_size_limits(inv) + _assert_finding_structure(result) + assert any( + r["Finding"] == "API Gateway Request Body Size Limits Not Enforced" + and r["Status"] == "Failed" + and r["Severity"] == "Medium" + for r in result["csv_data"] + ) + + def test_validator_presence_without_size_model_not_passed(self): + """REQ-11/A1: validator without maxLength bound → Failed, not Passed.""" + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=[], detail_by_id={}) + ) + with patch("app.boto3.client") as mock_client: + + def side_effect(service, **kwargs): + if service == "apigateway": + c = MagicMock() + c.get_rest_apis.return_value = { + "items": [{"id": "api1", "name": "genai-api"}] + } + c.get_request_validators.return_value = { + "items": [ + { + "id": "v1", + "name": "body-validator", + "validateRequestBody": True, + } + ] + } + c.get_models.return_value = { + "items": [{"name": "Empty", "schema": '{"type":"object"}'}] + } + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_api_gateway_request_body_size_limits(inv) + _assert_finding_structure(result) + assert not any(r["Status"] == "Passed" for r in result["csv_data"]) + assert any( + r["Finding"] == "API Gateway Request Body Size Limits Not Enforced" + and r["Status"] == "Failed" + for r in result["csv_data"] + ) + + def test_validator_with_maxlength_model_passed(self): + """REQ-11/A1: validator with maxLength model IS a real size control → Passed.""" + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=[], detail_by_id={}) + ) + with patch("app.boto3.client") as mock_client: + + def side_effect(service, **kwargs): + if service == "apigateway": + c = MagicMock() + c.get_rest_apis.return_value = { + "items": [{"id": "api1", "name": "genai-api"}] + } + c.get_request_validators.return_value = { + "items": [{"id": "v1", "validateRequestBody": True}] + } + c.get_models.return_value = { + "items": [ + { + "name": "Prompt", + "schema": '{"type":"object","properties":{"prompt":{"type":"string","maxLength":4000}}}', + } + ] + } + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_api_gateway_request_body_size_limits(inv) + _assert_finding_structure(result) + assert any( + r["Finding"] == "API Gateway Request Body Size Limits Configured" + and r["Status"] == "Passed" + for r in result["csv_data"] + ) + + def test_waf_oversize_constraint_above_window_not_credited(self): + """REQ-11/A2: GT body SizeConstraint above 16 KB with CONTINUE oversize → not credited → Failed.""" + bad_detail = { + "Rules": [ + { + "Name": "too-big", + "Statement": { + "SizeConstraintStatement": { + "FieldToMatch": {"Body": {"OversizeHandling": "CONTINUE"}}, + "ComparisonOperator": "GT", + "Size": 32768, + } + }, + } + ] + } + inv = make_resource_inventory( + web_acls=app.WebAclInventory( + summaries=[{"Name": "acl1", "Id": "id1"}], + detail_by_id={"id1": bad_detail}, + ) + ) + with patch("app.boto3.client") as mock_client: + + def side_effect(service, **kwargs): + if service == "apigateway": + c = MagicMock() + c.get_rest_apis.return_value = { + "items": [{"id": "api1", "name": "genai-api"}] + } + c.get_request_validators.return_value = {"items": []} + c.get_models.return_value = {"items": []} + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_api_gateway_request_body_size_limits(inv) + _assert_finding_structure(result) + assert not any(r["Status"] == "Passed" for r in result["csv_data"]) + assert any(r["Status"] == "Failed" for r in result["csv_data"]) + + def test_waf_body_substring_rule_not_credited(self): + """REQ-11/A2: XSS match on body is NOT a SizeConstraint → not credited.""" + xss_detail = { + "Rules": [ + { + "Name": "xss-on-body", + "Statement": { + "XssMatchStatement": { + "FieldToMatch": {"Body": {}}, + "TextTransformations": [{"Priority": 0, "Type": "NONE"}], + } + }, + } + ] + } + inv = make_resource_inventory( + web_acls=app.WebAclInventory( + summaries=[{"Name": "acl1", "Id": "id1"}], + detail_by_id={"id1": xss_detail}, + ) + ) + with patch("app.boto3.client") as mock_client: + + def side_effect(service, **kwargs): + if service == "apigateway": + c = MagicMock() + c.get_rest_apis.return_value = { + "items": [{"id": "api1", "name": "genai-api"}] + } + c.get_request_validators.return_value = {"items": []} + c.get_models.return_value = {"items": []} + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_api_gateway_request_body_size_limits(inv) + _assert_finding_structure(result) + assert not any(r["Status"] == "Passed" for r in result["csv_data"]) + + def test_error_on_unavailable_inventory(self): + """Unavailable inventory → COULD_NOT_ASSESS (ERROR envelope).""" + inv = make_resource_inventory( + web_acls=app._Unavailable(RuntimeError("apigw error")) + ) + with patch("app.boto3.client") as mock_client: + + def side_effect(service, **kwargs): + if service == "apigateway": + c = MagicMock() + c.get_rest_apis.return_value = {"items": []} + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_api_gateway_request_body_size_limits(inv) + assert result["status"] == "ERROR" + + def test_error_on_apigw_exception(self): + """apigateway client raises → overall ERROR.""" + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=[], detail_by_id={}) + ) + with patch("app.boto3.client") as mock_client: + mock_client.side_effect = RuntimeError("apigw error") + result = app.check_api_gateway_request_body_size_limits(inv) + assert result["status"] == "ERROR" + + # --- Pagination-correctness proof: >100 ACLs --- + def test_pass_more_than_100_acls_with_size_constraints(self): + """150 ACLs all with SizeConstraint → PASS (previously truncated ACLs now assessed).""" + acl_detail = self._size_constraint_detail() + summaries = [{"Name": f"acl{i}", "Id": f"id{i}"} for i in range(150)] + detail_by_id = {f"id{i}": acl_detail for i in range(150)} + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=summaries, detail_by_id=detail_by_id) + ) + with patch("app.boto3.client") as mock_client: + + def side_effect(service, **kwargs): + if service == "apigateway": + c = MagicMock() + c.get_rest_apis.return_value = {"items": []} + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_api_gateway_request_body_size_limits(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + # --- ≤1-page equivalence: 2-ACL case matches Wave-0 baseline --- + def test_two_acl_case_unchanged(self): + """The 2-ACL PASS scenario is unchanged vs the pre-refactor baseline.""" + from test_inventory_equivalence import _acl_detail as _baseline_acl_detail + + inv = make_resource_inventory( + web_acls=app.WebAclInventory( + summaries=[ + {"Name": "FinServACL1", "Id": "acl-id-001"}, + {"Name": "FinServACL2", "Id": "acl-id-002"}, + ], + detail_by_id={ + "acl-id-001": _baseline_acl_detail("FinServACL1", "acl-id-001")[ + "WebACL" + ], + "acl-id-002": _baseline_acl_detail("FinServACL2", "acl-id-002")[ + "WebACL" + ], + }, + ) + ) + with patch("app.boto3.client") as mock_client: + + def side_effect(service, **kwargs): + if service == "apigateway": + c = MagicMock() + c.get_rest_apis.return_value = {"items": []} + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_api_gateway_request_body_size_limits(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + assert any( + r["Finding"] == "API Gateway Request Body Size Limits Configured" + for r in result["csv_data"] + ) + + +class TestFS69PromptInputValidationFunction: + """FS-69 — Prompt Input Validation Function Check.""" + + def test_pass_validation_lambda_exists(self): + inv = make_resource_inventory( + lambda_functions=[{"FunctionName": "prompt-input-sanitizer"}] + ) + result = app.check_prompt_input_validation_function(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + def test_warn_no_validation_lambda(self): + inv = make_resource_inventory( + lambda_functions=[{"FunctionName": "my-api-handler"}] + ) + result = app.check_prompt_input_validation_function(inv) + _assert_finding_structure(result) + assert result["status"] == "WARN" + + def test_error_on_unavailable_inventory(self): + inv = make_resource_inventory( + lambda_functions=app._Unavailable(RuntimeError("input error")) + ) + result = app.check_prompt_input_validation_function(inv) + assert result["status"] == "ERROR" + + +# ========================================================================= +# HELPERS +# ========================================================================= + + +class TestHelpers: + """Test _empty_findings and _error_findings helpers.""" + + def test_empty_findings_structure(self): + result = app._empty_findings("Test Check") + assert result["check_name"] == "Test Check" + assert result["status"] == "PASS" + assert result["csv_data"] == [] + + def test_error_findings_structure(self): + err = RuntimeError("something broke") + result = app._error_findings("Test Check", err) + assert result["check_name"] == "Test Check" + assert result["status"] == "ERROR" + assert "something broke" in result["details"] + # REQ-13: _error_findings contract is unchanged (csv_data stays empty); + # the visible row is synthesized by the handler, not here. + assert result["csv_data"] == [] + + def test_could_not_assess_row_shape(self): + # COULD_NOT_ASSESS disposition: synthesized row is N/A + Low (methodology §3.4). + row = app._could_not_assess_row( + "FS-44", "Amazon Macie PII Scanning Check", "AccessDenied" + ) + assert row["Check_ID"] == "FS-44" + assert row["Status"] == "N/A" + assert row["Severity"] == "Low" + assert row["Finding"].startswith(app.COULD_NOT_ASSESS_PREFIX) + assert "AccessDenied" in row["Finding_Details"] + + +# ========================================================================= +# REQ-13: CHECK REGISTRY + COULD-NOT-ASSESS HANDLING +# ========================================================================= + + +class TestFinservChecksRegistry: + """REQ-13 — registry-driven dispatch and could-not-assess rows.""" + + def test_registry_covers_all_checks_in_order(self): + registry = app.build_finserv_checks( + {"role_permissions": {}, "user_permissions": {}} + ) + # 65 entries: 64 standalone checks (FS-17/18/19/23/64 merged upstream) + # plus the new check_automated_reasoning_policies() which shares the FS-27 + # check_id with check_guardrail_contextual_grounding(). + assert len(registry) == 65 + ids = [cid for cid, _ in registry] + # FS-27 appears twice (contextual grounding + ARC policies); allow that. + unique_ids = set(ids) + assert len(unique_ids) == 64 + for cid, fn in registry: + assert callable(fn), f"{cid} is not callable" + # Order is non-decreasing by numeric FS id (execution order preserved). + nums = [int(cid.split("-")[1]) for cid in ids] + assert nums == sorted(nums) + # The two permission-cache checks are present. + assert "FS-07" in ids and "FS-22" in ids + + @patch("app.write_to_s3", return_value="https://example.com/report.csv") + @patch.dict(os.environ, {"AIML_ASSESSMENT_BUCKET_NAME": "test-bucket"}) + @patch("app.get_permissions_cache", return_value=None) + @patch("app.build_finserv_checks") + def test_errored_check_emits_could_not_assess_row( + self, mock_build, mock_cache, mock_write + ): + # One check raises (uncaught) → handler synthesizes one could-not-assess row. + def boom(): + return app._error_findings("Boom Check", RuntimeError("AccessDenied: nope")) + + def normal(): + return { + "check_name": "Normal Check", + "status": "PASS", + "csv_data": [ + { + "Check_ID": "FS-02", + "Finding": "Normal Finding", + "Finding_Details": "ok", + "Resolution": "none", + "Reference": "https://example.com", + "Severity": "Informational", + "Status": "Passed", + } + ], + } + + mock_build.return_value = [("FS-44", boom), ("FS-02", normal)] + + resp = app.lambda_handler({"Execution": {"Name": "exec-1"}}, None) + findings = resp["body"]["findings"] + # Errored check now contributes exactly one row, not zero (no silent drop). + boom_result = next(f for f in findings if f["check_name"] == "Boom Check") + assert len(boom_result["csv_data"]) == 1 + row = boom_result["csv_data"][0] + assert row["Check_ID"] == "FS-44" + assert row["Status"] == "N/A" + assert row["Severity"] == "Low" + assert row["Finding"].startswith(app.COULD_NOT_ASSESS_PREFIX) + # Normal check passes through unchanged (no spurious row added). + normal_result = next(f for f in findings if f["check_name"] == "Normal Check") + assert len(normal_result["csv_data"]) == 1 + assert normal_result["csv_data"][0]["Finding"] == "Normal Finding" + + @patch("app.write_to_s3", return_value="https://example.com/report.csv") + @patch.dict(os.environ, {"AIML_ASSESSMENT_BUCKET_NAME": "test-bucket"}) + @patch("app.get_permissions_cache", return_value=None) + @patch("app.build_finserv_checks") + def test_non_error_empty_result_still_emits_could_not_assess_row( + self, mock_build, mock_cache, mock_write + ): + # A check that returns a NON-error wrapper status but zero csv_data must + # NOT silently vanish — the handler synthesizes a could-not-assess row for + # any empty result, not only ERROR ones. This guards the no-silent-drop + # invariant structurally (Property 7) rather than by data coincidence. + def empty_pass(): + return {"check_name": "Empty Pass Check", "status": "PASS", "csv_data": []} + + mock_build.return_value = [("FS-13", empty_pass)] + + resp = app.lambda_handler({"Execution": {"Name": "exec-3"}}, None) + findings = resp["body"]["findings"] + result = next(f for f in findings if f["check_name"] == "Empty Pass Check") + assert len(result["csv_data"]) == 1 + row = result["csv_data"][0] + assert row["Check_ID"] == "FS-13" + assert row["Status"] == "N/A" + assert row["Severity"] == "Low" + assert row["Finding"].startswith(app.COULD_NOT_ASSESS_PREFIX) + + @patch("app.write_to_s3", return_value="https://example.com/report.csv") + @patch.dict(os.environ, {"AIML_ASSESSMENT_BUCKET_NAME": "test-bucket"}) + @patch("app.get_permissions_cache", return_value=None) + def test_no_check_contributes_zero_rows(self, mock_cache, mock_write): + # Full handler run with mocked AWS (all clients raise) — every check must + # still contribute at least one row (real rows or a could-not-assess row). + with patch("app.boto3.client", side_effect=RuntimeError("AccessDenied")): + resp = app.lambda_handler({"Execution": {"Name": "exec-2"}}, None) + findings = resp["body"]["findings"] + for f in findings: + assert len(f["csv_data"]) >= 1, f"{f['check_name']} contributed zero rows" + + +# ========================================================================= +# _paginate helper — multi-page collection across token conventions +# ========================================================================= + + +class TestPaginateHelper: + """app._paginate must collect all pages regardless of the API's token field.""" + + def test_single_page(self): + c = MagicMock() + c.list_things.return_value = {"items": [{"a": 1}, {"a": 2}]} + out = app._paginate(c, "list_things", "items") + assert out == [{"a": 1}, {"a": 2}] + assert c.list_things.call_count == 1 + + def test_lambda_marker_pagination(self): + # Lambda uses Marker (request) / NextMarker (response). + c = MagicMock() + c.list_functions.side_effect = [ + {"Functions": [{"FunctionName": "f1"}], "NextMarker": "m1"}, + {"Functions": [{"FunctionName": "f2"}]}, + ] + out = app._paginate(c, "list_functions", "Functions") + assert [f["FunctionName"] for f in out] == ["f1", "f2"] + assert c.list_functions.call_count == 2 + _, kwargs = c.list_functions.call_args + assert kwargs.get("Marker") == "m1" + + def test_next_token_lowercase_pagination(self): + # Bedrock uses nextToken. + c = MagicMock() + c.list_guardrails.side_effect = [ + {"guardrails": [{"id": "g1"}], "nextToken": "t1"}, + {"guardrails": [{"id": "g2"}]}, + ] + out = app._paginate(c, "list_guardrails", "guardrails") + assert [g["id"] for g in out] == ["g1", "g2"] + + def test_apigateway_position_pagination(self): + # API Gateway uses position (request and response). + c = MagicMock() + c.get_usage_plans.side_effect = [ + {"items": [{"id": "p1"}], "position": "pos1"}, + {"items": [{"id": "p2"}]}, + ] + out = app._paginate(c, "get_usage_plans", "items") + assert [p["id"] for p in out] == ["p1", "p2"] + + def test_missing_result_key_returns_empty(self): + c = MagicMock() + c.list_things.return_value = {} + assert app._paginate(c, "list_things", "items") == [] + + +class TestIsAccessError: + """app._is_access_error classifies permission errors vs other ClientErrors.""" + + def test_access_denied_is_access_error(self): + assert app._is_access_error(_client_error("AccessDenied")) is True + assert app._is_access_error(_client_error("AccessDeniedException")) is True + assert app._is_access_error(_client_error("UnauthorizedOperation")) is True + + def test_other_clienterror_is_not_access_error(self): + assert app._is_access_error(_client_error("NoSuchTagSet")) is False + assert app._is_access_error(_client_error("NoSuchBucket")) is False + + def test_non_clienterror_is_not_access_error(self): + # A plain exception without a .response attribute must not be misclassified. + assert app._is_access_error(RuntimeError("boom")) is False + + +class TestIsMissingBucketError: + """app._is_missing_bucket_error classifies deleted-bucket errors (NoSuchBucket / + 404 / NotFound) distinctly from access errors and other ClientErrors.""" + + def test_nosuchbucket_is_missing(self): + assert app._is_missing_bucket_error(_client_error("NoSuchBucket")) is True + assert app._is_missing_bucket_error(_client_error("404")) is True + assert app._is_missing_bucket_error(_client_error("NotFound")) is True + + def test_access_and_other_errors_are_not_missing(self): + assert app._is_missing_bucket_error(_client_error("AccessDenied")) is False + assert app._is_missing_bucket_error(_client_error("InvalidRequest")) is False + assert app._is_missing_bucket_error(_client_error("NoSuchTagSet")) is False + + def test_non_clienterror_is_not_missing(self): + assert app._is_missing_bucket_error(RuntimeError("boom")) is False + + +# ========================================================================= +# REQ-14: REQUIREMENTS.TXT VERSION FLOOR GUARD +# ========================================================================= + + +class TestRequirementVersionFloors: + """ + REQ-14 — Guard against future regression that lowers the botocore/boto3 floor + below the minimum needed for: + - FS-03: list_aws_default_service_quotas paginator + - FS-06: describe_budgets(ShowFilterExpression=True) + + Tests assert the floor in requirements.txt, not the installed version, so + they catch a source-file regression regardless of what pip has resolved in + the dev environment. The dev environment may be behind the floor + (e.g., 1.42.70 installed vs 1.43.21 required) — run + `pip install --upgrade boto3 botocore` to reach the floor. + + These tests do NOT hard-pin to 1.43.21; they assert a structural property + (floor >= minimum needed) so they stay valid as the floor is bumped over time. + """ + + # Minimum versions required for the features used in this Lambda. + # Chosen because 1.43.x is the first series that reliably supports both + # describe_budgets(ShowFilterExpression=True) and list_aws_default_service_quotas. + MIN_BOTO3 = (1, 43, 0) + MIN_BOTOCORE = (1, 43, 0) + + @staticmethod + def _parse_floor(line: str) -> tuple: + """ + Parse a 'pkg>=X.Y.Z' line from requirements.txt into (X, Y, Z). + Returns None if the line doesn't match the expected pattern. + """ + import re + + m = re.match(r"^\s*(boto[3core]*)\s*>=\s*(\d+)\.(\d+)\.(\d+)", line) + if m: + return (int(m.group(2)), int(m.group(3)), int(m.group(4))) + return None + + @staticmethod + def _load_requirements() -> str: + """Load finserv_assessments/requirements.txt relative to the tests/ dir.""" + req_path = os.path.join( + os.path.dirname(__file__), "..", "finserv_assessments", "requirements.txt" + ) + with open(req_path) as f: + return f.read() + + def test_boto3_floor_meets_minimum(self): + """requirements.txt must floor boto3 at >= 1.43.0.""" + content = self._load_requirements() + boto3_floor = None + for line in content.splitlines(): + if line.strip().startswith("boto3>="): + boto3_floor = self._parse_floor(line) + break + assert boto3_floor is not None, ( + "boto3 floor not found in requirements.txt — expected 'boto3>=X.Y.Z'" + ) + assert boto3_floor >= self.MIN_BOTO3, ( + f"boto3 floor {boto3_floor} is below minimum {self.MIN_BOTO3}; " + "FS-06 describe_budgets(ShowFilterExpression=True) requires boto3>=1.43.0. " + "Bump the floor in finserv_assessments/requirements.txt." + ) + + def test_botocore_floor_meets_minimum(self): + """requirements.txt must floor botocore at >= 1.43.0.""" + content = self._load_requirements() + botocore_floor = None + for line in content.splitlines(): + if line.strip().startswith("botocore>="): + botocore_floor = self._parse_floor(line) + break + assert botocore_floor is not None, ( + "botocore floor not found in requirements.txt — expected 'botocore>=X.Y.Z'" + ) + assert botocore_floor >= self.MIN_BOTOCORE, ( + f"botocore floor {botocore_floor} is below minimum {self.MIN_BOTOCORE}; " + "FS-03 list_aws_default_service_quotas and FS-06 ShowFilterExpression " + "require botocore>=1.43.0. Bump the floor in finserv_assessments/requirements.txt." + ) + + def test_pydantic_floor_present(self): + """pydantic>=2.0.0 must be present and unchanged (schema depends on Pydantic v2).""" + content = self._load_requirements() + assert "pydantic>=2.0.0" in content, ( + "pydantic>=2.0.0 not found in requirements.txt — Pydantic v2 is required " + "for the Finding model in schema.py." + ) + + def test_no_exact_pins_on_aws_sdk(self): + """ + boto3 and botocore must use >= floors, not exact pins (==). + Exact pins would block Lambda from resolving security patches. + """ + content = self._load_requirements() + import re + + exact_pins = re.findall(r"boto(?:3|core)==\S+", content) + assert not exact_pins, ( + f"Exact version pins found for AWS SDK: {exact_pins}. " + "Use >= floors instead so the Lambda can receive security patches." + ) + + +class TestGenerateCsvReport: + """Test CSV report generation.""" + + def test_empty_findings_produces_header_only(self): + csv_content = app.generate_csv_report([]) + lines = csv_content.strip().split("\n") + assert len(lines) == 1 # header only + assert "Check_ID" in lines[0] + + def test_findings_produce_csv_rows(self): + findings = [ + { + "check_name": "Test", + "status": "PASS", + "csv_data": [ + { + "Check_ID": "FS-01", + "Finding": "Test Finding", + "Finding_Details": "Details", + "Resolution": "Fix", + "Reference": "https://example.com", + "Severity": "High", + "Status": "Passed", + } + ], + } + ] + csv_content = app.generate_csv_report(findings) + lines = csv_content.strip().split("\n") + assert len(lines) == 2 # header + 1 data row + assert "FS-01" in lines[1] + + def test_multiple_findings_multiple_rows(self): + findings = [ + { + "check_name": "Check A", + "status": "WARN", + "csv_data": [ + { + "Check_ID": "FS-01", + "Finding": "A", + "Finding_Details": "D", + "Resolution": "R", + "Reference": "https://a.com", + "Severity": "High", + "Status": "Failed", + }, + { + "Check_ID": "FS-01", + "Finding": "B", + "Finding_Details": "D", + "Resolution": "R", + "Reference": "https://b.com", + "Severity": "Medium", + "Status": "Passed", + }, + ], + }, + { + "check_name": "Check B", + "status": "PASS", + "csv_data": [ + { + "Check_ID": "FS-02", + "Finding": "C", + "Finding_Details": "D", + "Resolution": "R", + "Reference": "https://c.com", + "Severity": "Low", + "Status": "Passed", + }, + ], + }, + ] + csv_content = app.generate_csv_report(findings) + lines = csv_content.strip().split("\n") + assert len(lines) == 4 # header + 3 data rows diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_checks_coverage.py b/aiml-security-assessment/functions/security/finserv_tests/test_checks_coverage.py new file mode 100644 index 0000000..d948cac --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_tests/test_checks_coverage.py @@ -0,0 +1,1993 @@ +""" +Additional tests targeting the uncovered branches in finserv_assessments/app.py. +These complement test_checks.py to push coverage from 83% → 90%+. + +Each class targets a specific uncovered branch identified from coverage.json. +""" + +import json +import os +import sys +from datetime import datetime, timezone, timedelta +from unittest.mock import MagicMock, patch +from botocore.exceptions import ClientError + + +FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") +if FINSERV_DIR not in sys.path: + sys.path.insert(0, FINSERV_DIR) + +# Ensure tests/ directory is importable (for conftest helpers) +TESTS_DIR = os.path.dirname(__file__) +if TESTS_DIR not in sys.path: + sys.path.insert(0, TESTS_DIR) + +import app # noqa: E402 (import must follow sys.path setup above) +from conftest import make_resource_inventory # noqa: E402 + + +def _client_error(code="AccessDeniedException", message="Access Denied"): + return ClientError({"Error": {"Code": code, "Message": message}}, "Op") + + +def _assert_structure(result): + assert "check_name" in result + assert result["status"] in ("PASS", "WARN", "ERROR") + assert isinstance(result["csv_data"], list) + + +# ========================================================================= +# FS-01 — line 126: shield ClientError path (not ResourceNotFoundException) +# ========================================================================= + + +class TestFS01ShieldClientError: + def test_shield_generic_client_error_treated_as_no_shield(self): + """Line 126-128: ClientError on describe_subscription → shield_enabled stays False.""" + inv = make_resource_inventory( + web_acls=app.WebAclInventory( + summaries=[{"Name": "acl1", "Id": "id1"}], + detail_by_id={}, + ) + ) + with patch("app.boto3.client") as mock_client: + + def side_effect(service, **kwargs): + if service == "shield": + c = MagicMock() + c.describe_subscription.side_effect = _client_error( + "ThrottlingException" + ) + c.exceptions.ResourceNotFoundException = type( + "ResourceNotFoundException", (ClientError,), {} + ) + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_waf_shield_on_bedrock_endpoints(inv) + _assert_structure(result) + # Shield not enabled → WARN, but WAF ACLs present → only 1 WARN finding + assert result["status"] == "WARN" + statuses = [r["Status"] for r in result["csv_data"]] + assert "Failed" in statuses # shield failed + assert "Passed" in statuses # waf passed + + +# ========================================================================= +# FS-07 — lines 532-534, 537, 543, 546: new per-agent error handling paths +# ========================================================================= + + +class TestFS07AgentBoundariesNewPaths: + @patch("app.boto3.client") + def test_get_agent_client_error_skips_agent(self, mock_client): + """Lines 532-534: get_agent raises ClientError → agent is skipped gracefully.""" + c = MagicMock() + c.list_agents.return_value = { + "agentSummaries": [{"agentId": "a1", "agentName": "EncryptedAgent"}] + } + c.get_agent.side_effect = _client_error("AccessDeniedException") + mock_client.return_value = c + result = app.check_bedrock_agent_action_boundaries({}) + _assert_structure(result) + # Should PASS (no issues found, agent was skipped) + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_agent_no_role_arn_skipped(self, mock_client): + """Line 537: agent with no agentResourceRoleArn → continue.""" + c = MagicMock() + c.list_agents.return_value = { + "agentSummaries": [{"agentId": "a1", "agentName": "NoRoleAgent"}] + } + c.get_agent.return_value = {"agent": {"agentResourceRoleArn": ""}} + mock_client.return_value = c + result = app.check_bedrock_agent_action_boundaries({}) + _assert_structure(result) + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_policy_doc_as_string_is_parsed(self, mock_client): + """Line 543: policy document stored as JSON string → json.loads branch.""" + c = MagicMock() + c.list_agents.return_value = { + "agentSummaries": [{"agentId": "a1", "agentName": "Agent1"}] + } + c.get_agent.return_value = { + "agent": {"agentResourceRoleArn": "arn:aws:iam::123:role/SafeRole"} + } + mock_client.return_value = c + cache = { + "role_permissions": { + "SafeRole": { + "attached_policies": [ + { + "document": json.dumps( + { + "Statement": [ + { + "Effect": "Allow", + "Action": "bedrock:InvokeModel", + "Resource": "*", + } + ] + } + ) + } + ], + "inline_policies": [], + } + } + } + result = app.check_bedrock_agent_action_boundaries(cache) + _assert_structure(result) + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_deny_effect_statement_skipped(self, mock_client): + """Line 546: Deny effect → continue (not counted as issue).""" + c = MagicMock() + c.list_agents.return_value = { + "agentSummaries": [{"agentId": "a1", "agentName": "Agent1"}] + } + c.get_agent.return_value = { + "agent": {"agentResourceRoleArn": "arn:aws:iam::123:role/DenyRole"} + } + mock_client.return_value = c + cache = { + "role_permissions": { + "DenyRole": { + "attached_policies": [ + { + "document": { + "Statement": [ + { + "Effect": "Deny", + "Action": "iam:*", + "Resource": "*", + } + ] + } + } + ], + "inline_policies": [], + } + } + } + result = app.check_bedrock_agent_action_boundaries(cache) + _assert_structure(result) + assert result["status"] == "PASS" + + +# ========================================================================= +# FS-08 — line 622: re-raise non-AccessDenied ClientError +# ========================================================================= + + +class TestFS08AgentcoreReraise: + @patch("app.boto3.client") + def test_non_access_denied_error_propagates_to_outer_except(self, mock_client): + """Line 622: ClientError that is NOT AccessDenied/Unrecognized → re-raised → ERROR.""" + c = MagicMock() + c.list_agent_runtimes.side_effect = _client_error("ServiceUnavailableException") + mock_client.return_value = c + result = app.check_agentcore_policy_engine() + _assert_structure(result) + assert result["status"] == "ERROR" + + +# ========================================================================= +# FS-09 — lines 704-705: get_function_concurrency ClientError path +# ========================================================================= + + +class TestFS09ConcurrencyClientError: + @patch("app.boto3.client") + def test_get_concurrency_client_error_adds_to_warn_list(self, mock_client): + """Lines 704-705: get_function_concurrency raises ClientError → appended to warn list.""" + c = MagicMock() + c.get_function_concurrency.side_effect = _client_error("AccessDeniedException") + mock_client.return_value = c + inv = make_resource_inventory( + lambda_functions=[{"FunctionName": "my-agent-fn"}] + ) + result = app.check_agent_transaction_limits(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + +# ========================================================================= +# FS-12 — lines 902-923, 947: SCP paths +# ========================================================================= + + +class TestFS12ScpPaths: + @patch("app.boto3.client") + def test_access_denied_returns_na(self, mock_client): + """Lines 902-915: AccessDeniedException → N/A finding.""" + c = MagicMock() + c.list_policies.side_effect = _client_error("AccessDeniedException") + mock_client.return_value = c + result = app.check_scp_model_access_restrictions() + _assert_structure(result) + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_orgs_not_in_use_returns_na(self, mock_client): + """Lines 902-915: AWSOrganizationsNotInUseException → N/A finding.""" + c = MagicMock() + c.list_policies.side_effect = _client_error("AWSOrganizationsNotInUseException") + mock_client.return_value = c + result = app.check_scp_model_access_restrictions() + _assert_structure(result) + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_non_access_denied_reraises(self, mock_client): + """Line 916: non-AccessDenied ClientError → re-raised → ERROR.""" + c = MagicMock() + c.list_policies.side_effect = _client_error("ServiceUnavailableException") + mock_client.return_value = c + result = app.check_scp_model_access_restrictions() + _assert_structure(result) + assert result["status"] == "ERROR" + + @patch("app.boto3.client") + def test_warn_no_bedrock_scps(self, mock_client): + """Lines 920-923, 925-945: policies exist but none reference bedrock → WARN.""" + c = MagicMock() + c.list_policies.return_value = { + "Policies": [{"Id": "p-001", "Name": "GeneralSCP"}] + } + c.describe_policy.return_value = { + "Policy": { + "Content": json.dumps( + {"Statement": [{"Effect": "Deny", "Action": "ec2:*"}]} + ) + } + } + mock_client.return_value = c + result = app.check_scp_model_access_restrictions() + _assert_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_pass_bedrock_scp_found(self, mock_client): + """Line 947: bedrock SCP found → Passed finding.""" + c = MagicMock() + c.list_policies.return_value = { + "Policies": [{"Id": "p-001", "Name": "BedrockModelSCP"}] + } + c.describe_policy.return_value = { + "Policy": { + "Content": json.dumps( + { + "Statement": [ + { + "Effect": "Deny", + "Action": "bedrock:InvokeModel", + "Condition": { + "StringNotEquals": { + "bedrock:ModelId": ["anthropic.claude-v2"] + } + }, + } + ] + } + ) + } + } + mock_client.return_value = c + result = app.check_scp_model_access_restrictions() + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-13 — lines 979-999: model tagging warn/pass paths +# ========================================================================= + + +class TestFS13ModelTaggingPaths: + @patch("app.boto3.client") + def test_warn_bedrock_model_missing_tags(self, mock_client): + """Lines 979-983, 998-999: Bedrock custom model missing required tags → WARN.""" + + def side_effect(service, **kwargs): + if service == "bedrock": + c = MagicMock() + c.list_custom_models.return_value = { + "modelSummaries": [ + { + "modelName": "my-model", + "modelArn": "arn:aws:bedrock:us-east-1:123:model/my-model", + } + ] + } + c.list_tags_for_resource.return_value = {"tags": []} + return c + if service == "sagemaker": + c = MagicMock() + c.list_models.return_value = {"Models": []} + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_model_inventory_tagging() + _assert_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_warn_sagemaker_model_missing_tags(self, mock_client): + """Lines 989-993: SageMaker model missing required tags → WARN.""" + + def side_effect(service, **kwargs): + if service == "bedrock": + c = MagicMock() + c.list_custom_models.return_value = {"modelSummaries": []} + return c + if service == "sagemaker": + c = MagicMock() + c.list_models.return_value = { + "Models": [ + { + "ModelName": "sm-model", + "ModelArn": "arn:aws:sagemaker:us-east-1:123:model/sm-model", + } + ] + } + c.list_tags.return_value = {"Tags": []} + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_model_inventory_tagging() + _assert_structure(result) + assert result["status"] == "WARN" + + +# ========================================================================= +# FS-14 — line 1072: pass path (config rules found) +# ========================================================================= + + +class TestFS14ModelGovernancePass: + @patch("app.boto3.client") + def test_pass_config_rules_found(self, mock_client): + """Line 1072: bedrock-related Config rules found → Passed.""" + c = MagicMock() + c.describe_config_rules.return_value = { + "ConfigRules": [{"ConfigRuleName": "bedrock-model-approval-rule"}] + } + mock_client.return_value = c + result = app.check_model_onboarding_governance() + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-15 — line 1119: pass path (eval jobs found) +# ========================================================================= + + +class TestFS15BedrockEvalPass: + @patch("app.boto3.client") + def test_pass_eval_jobs_found(self, mock_client): + """Line 1119: evaluation jobs exist → Passed finding.""" + c = MagicMock() + c.list_evaluation_jobs.return_value = { + "jobSummaries": [{"jobName": "adversarial-eval-2025"}] + } + mock_client.return_value = c + result = app.check_bedrock_model_evaluation_adversarial() + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-16 — lines 1167-1168: warn path (repos without scanning) +# ========================================================================= + + +class TestFS16EcrScanningWarn: + @patch("app.boto3.client") + def test_warn_repos_without_scanning(self, mock_client): + """Lines 1167-1168: repos exist but scan-on-push disabled → WARN.""" + c = MagicMock() + c.describe_repositories.return_value = { + "repositories": [ + { + "repositoryName": "ml-model-repo", + "imageScanningConfiguration": {"scanOnPush": False}, + } + ] + } + mock_client.return_value = c + result = app.check_ecr_image_scanning() + _assert_structure(result) + assert result["status"] == "WARN" + + +# ========================================================================= +# FS-20 — lines 1238-1266: feature store warn/pass paths +# ========================================================================= + + +class TestFS20FeatureStoreWarnPass: + @patch("app.boto3.client") + def test_warn_groups_without_offline_store(self, mock_client): + """Lines 1244-1246: feature groups without active offline store → WARN.""" + c = MagicMock() + c.list_feature_groups.return_value = { + "FeatureGroupSummaries": [ + { + "FeatureGroupName": "customer-features", + "OfflineStoreStatus": {"Status": "Disabled"}, + } + ] + } + mock_client.return_value = c + result = app.check_feature_store_rollback_capability() + _assert_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_pass_all_groups_have_offline_store(self, mock_client): + """Line 1266: all feature groups have active offline store → Passed.""" + c = MagicMock() + c.list_feature_groups.return_value = { + "FeatureGroupSummaries": [ + { + "FeatureGroupName": "customer-features", + "OfflineStoreStatus": {"Status": "Active"}, + } + ] + } + mock_client.return_value = c + result = app.check_feature_store_rollback_capability() + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-21 — lines 1312-1351: S3 versioning warn/pass paths +# ========================================================================= + + +class TestFS21TrainingDataVersioningPaths: + @patch("app.boto3.client") + def test_warn_unversioned_training_buckets(self, mock_client): + """Lines 1312-1316, 1318-1320: training buckets without versioning → WARN.""" + inv = make_resource_inventory(buckets=[{"Name": "training-data-bucket"}]) + c = MagicMock() + c.get_bucket_tagging.return_value = { + "TagSet": [{"Key": "Purpose", "Value": "training"}] + } + c.get_bucket_versioning.return_value = {} # no versioning + mock_client.return_value = c + + result = app.check_training_data_s3_versioning(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_pass_all_training_buckets_versioned(self, mock_client): + """Line 1338: all training buckets versioned → Passed.""" + inv = make_resource_inventory(buckets=[{"Name": "training-data-bucket"}]) + c = MagicMock() + c.get_bucket_tagging.return_value = { + "TagSet": [{"Key": "Purpose", "Value": "training"}] + } + c.get_bucket_versioning.return_value = {"Status": "Enabled"} + mock_client.return_value = c + + result = app.check_training_data_s3_versioning(inv) + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_access_error_surfaces_as_could_not_assess(self, mock_client): + """AccessDenied on get_bucket_versioning re-raises → ERROR (could-not-assess), + not a false 'no versioning' finding.""" + inv = make_resource_inventory(buckets=[{"Name": "training-data-bucket"}]) + c = MagicMock() + c.get_bucket_versioning.side_effect = _client_error("AccessDenied") + mock_client.return_value = c + + result = app.check_training_data_s3_versioning(inv) + _assert_structure(result) + assert result["status"] == "ERROR" + + @patch("app.boto3.client") + def test_nonaccess_error_flags_bucket(self, mock_client): + """A non-access ClientError on get_bucket_versioning flags the bucket + (WARN) without aborting the whole check.""" + inv = make_resource_inventory(buckets=[{"Name": "model-bucket"}]) + c = MagicMock() + c.get_bucket_versioning.side_effect = _client_error("NoSuchBucket") + mock_client.return_value = c + + result = app.check_training_data_s3_versioning(inv) + _assert_structure(result) + assert result["status"] == "WARN" + assert any( + "(error)" in r.get("Finding_Details", "") for r in result["csv_data"] + ) + + @patch("app.boto3.client") + def test_multi_page_buckets_completeness(self, mock_client): + """Pagination completeness: buckets from multiple pages are all checked. + + With the inventory approach, the collector already drains all pages via + _paginate with ContinuationToken. This test verifies that when 2+ pages + of buckets are provided in the inventory, the check assesses them all. + """ + # Simulate 2 "pages" of buckets — all training-named so they're all checked + page1 = [ + {"Name": "training-bucket-page1-001"}, + {"Name": "model-bucket-page1-002"}, + ] + page2 = [ + {"Name": "training-bucket-page2-001"}, + {"Name": "sagemaker-bucket-page2-002"}, + ] + all_buckets = page1 + page2 + + inv = make_resource_inventory(buckets=all_buckets) + c = MagicMock() + c.get_bucket_versioning.return_value = {"Status": "Enabled"} + mock_client.return_value = c + + result = app.check_training_data_s3_versioning(inv) + _assert_structure(result) + # All 4 buckets versioned → Passed + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + # Verify all 4 were checked (versioning call per training bucket) + assert c.get_bucket_versioning.call_count == 4 + + @patch("app.boto3.client") + def test_single_page_unchanged_vs_baseline(self, mock_client): + """Single-page case: result is identical to pre-migration behavior (baseline). + + With ≤1 page of buckets (no ContinuationToken), the inventory holds + all buckets and the check outcome is the same as before migration. + """ + inv = make_resource_inventory(buckets=[{"Name": "training-data-bucket"}]) + c = MagicMock() + c.get_bucket_versioning.return_value = {} # no versioning + mock_client.return_value = c + + result = app.check_training_data_s3_versioning(inv) + _assert_structure(result) + assert result["status"] == "WARN" + assert any( + r["Finding"] == "Training Data Buckets Without Versioning" + for r in result["csv_data"] + ) + + +# ========================================================================= + + +class TestFS22KbIamWarnPath: + def test_warn_wildcard_bedrock_agent_permission(self): + """Lines 1370-1386: role with bedrock-agent:* → WARN.""" + cache = { + "role_permissions": { + "KBAccessRole": { + "attached_policies": [ + { + "document": { + "Statement": [ + { + "Effect": "Allow", + "Action": "bedrock-agent:*", + "Resource": "*", + } + ] + } + } + ], + "inline_policies": [], + } + } + } + result = app.check_knowledge_base_iam_least_privilege(cache) + _assert_structure(result) + assert result["status"] == "WARN" + + def test_warn_wildcard_bedrock_permission(self): + """Lines 1370-1386: role with bedrock:* → WARN.""" + cache = { + "role_permissions": { + "KBRole": { + "attached_policies": [], + "inline_policies": [ + { + "document": { + "Statement": [ + { + "Effect": "Allow", + "Action": "bedrock:*", + "Resource": "*", + } + ] + } + } + ], + } + } + } + result = app.check_knowledge_base_iam_least_privilege(cache) + _assert_structure(result) + assert result["status"] == "WARN" + + def test_policy_doc_as_string_parsed(self): + """Line 1372-1373: policy document as JSON string → parsed correctly.""" + cache = { + "role_permissions": { + "KBRole": { + "attached_policies": [ + { + "document": json.dumps( + { + "Statement": [ + { + "Effect": "Allow", + "Action": "*", + "Resource": "*", + } + ] + } + ) + } + ], + "inline_policies": [], + } + } + } + result = app.check_knowledge_base_iam_least_privilege(cache) + _assert_structure(result) + assert result["status"] == "WARN" + + def test_deny_effect_not_flagged(self): + """Line 1375-1376: Deny effect → not counted as issue.""" + cache = { + "role_permissions": { + "KBRole": { + "attached_policies": [ + { + "document": { + "Statement": [ + { + "Effect": "Deny", + "Action": "bedrock-agent:*", + "Resource": "*", + } + ] + } + } + ], + "inline_policies": [], + } + } + } + result = app.check_knowledge_base_iam_least_privilege(cache) + _assert_structure(result) + assert result["status"] == "PASS" + + def test_action_as_string_not_list(self): + """Lines 1378-1379: Action as string (not list) → converted to list.""" + cache = { + "role_permissions": { + "KBRole": { + "attached_policies": [ + { + "document": { + "Statement": [ + { + "Effect": "Allow", + "Action": "bedrock:*", + "Resource": "*", + } + ] + } + } + ], + "inline_policies": [], + } + } + } + result = app.check_knowledge_base_iam_least_privilege(cache) + _assert_structure(result) + assert result["status"] == "WARN" + + +# ========================================================================= +# FS-24 — line 1450: KB metadata filtering pass path (KBs exist) +# ========================================================================= + + +class TestFS24MetadataFilteringPass: + def test_pass_kbs_exist(self): + """KBs found → advisory N/A finding (metadata filtering not API-verifiable).""" + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1", "name": "rag-kb"}], + data_sources_by_kb={}, + data_source_detail={}, + ) + ) + result = app.check_knowledge_base_metadata_filtering(inv) + _assert_structure(result) + assert any( + r["Status"] == "N/A" + and r["Severity"] == "Informational" + and r["Finding"].startswith("ADVISORY: ") + for r in result["csv_data"] + ) + + +# ========================================================================= +# FS-25 — lines 1509-1511: OSS encryption with CMK path +# ========================================================================= + + +class TestFS25OssEncryptionPaths: + @patch("app.boto3.client") + def test_pass_policies_with_cmk(self, mock_client): + """Lines 1509-1511: encryption policies exist with CMK → Passed.""" + c = MagicMock() + c.list_security_policies.return_value = { + "securityPolicySummaries": [ + { + "name": "kb-encryption", + "policy": json.dumps( + {"Rules": [{"KmsARN": "arn:aws:kms:us-east-1:123:key/abc"}]} + ), + } + ] + } + mock_client.return_value = c + result = app.check_opensearch_serverless_encryption() + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_pass_no_policies(self, mock_client): + """Line 1488: no encryption policies → N/A finding.""" + c = MagicMock() + c.list_security_policies.return_value = {"securityPolicySummaries": []} + mock_client.return_value = c + result = app.check_opensearch_serverless_encryption() + _assert_structure(result) + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_fail_policies_without_cmk(self, mock_client): + """Encryption policies exist but all use AWS-owned keys → WARN/Failed + (the customer-managed-key control is absent), not a false Pass.""" + c = MagicMock() + c.list_security_policies.return_value = { + "securityPolicySummaries": [ + { + "name": "aws-owned-enc", + "policy": json.dumps( + {"Rules": [{"ResourceType": "collection"}], "AWSOwnedKey": True} + ), + } + ] + } + mock_client.return_value = c + result = app.check_opensearch_serverless_encryption() + _assert_structure(result) + assert result["status"] == "WARN" + assert any(r["Status"] == "Failed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-26 — lines 1546-1591: VPC access warn/pass paths +# ========================================================================= + + +class TestFS26VpcAccessPaths: + @patch("app.boto3.client") + def test_warn_no_network_policies(self, mock_client): + """Lines 1546-1547: no network policies → WARN.""" + c = MagicMock() + c.list_security_policies.return_value = {"securityPolicySummaries": []} + mock_client.return_value = c + result = app.check_knowledge_base_vpc_access() + _assert_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_warn_policies_without_vpc(self, mock_client): + """Lines 1567-1569: network policies exist but no VPC restriction → WARN.""" + c = MagicMock() + c.list_security_policies.return_value = { + "securityPolicySummaries": [ + { + "name": "public-access", + "policy": json.dumps({"Rules": [{"AllowFromPublic": True}]}), + } + ] + } + mock_client.return_value = c + result = app.check_knowledge_base_vpc_access() + _assert_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_pass_vpc_restricted_policies(self, mock_client): + """Line 1591: network policies with VPC restriction → Passed.""" + c = MagicMock() + c.list_security_policies.return_value = { + "securityPolicySummaries": [ + { + "name": "vpc-only", + "policy": json.dumps({"Rules": [{"SourceVPCEs": ["vpce-abc123"]}]}), + } + ] + } + mock_client.return_value = c + result = app.check_knowledge_base_vpc_access() + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-27 — lines 1644, 1667: ARC guardrail paths +# ========================================================================= + + +class TestFS27AutomatedReasoningPaths: + def test_warn_guardrails_without_grounding(self): + """Guardrails exist but none have contextual grounding → WARN.""" + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={"g1": {}}, # no contextualGroundingPolicy + ) + ) + result = app.check_guardrail_contextual_grounding(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + def test_pass_guardrail_with_grounding(self): + """Guardrail with contextual grounding → Passed.""" + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": { + "contextualGroundingPolicy": { + "filters": [{"type": "GROUNDING", "threshold": 0.7}] + } + } + }, + ) + ) + result = app.check_guardrail_contextual_grounding(inv) + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-28 — lines 1717-1718: denied topics warn path +# ========================================================================= + + +class TestFS28DeniedTopicsWarn: + def test_warn_guardrails_without_financial_topics(self): + """Lines 1717-1718: guardrails exist but no topic policies → WARN.""" + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={"g1": {"topicPolicy": {"topics": []}}}, + ) + ) + result = app.check_guardrail_denied_topics_financial(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + +# ========================================================================= +# FS-30 — line 1814: eval jobs found pass path +# ========================================================================= + + +class TestFS30ComplianceEvalPass: + def test_advisory_finding(self): + """FS-30 is advisory (REQ-10a): always one N/A Informational ADVISORY row.""" + result = app.check_bedrock_evaluation_compliance_datasets() + _assert_structure(result) + assert any( + r["Status"] == "N/A" + and r["Severity"] == "Informational" + and r["Finding"].startswith("ADVISORY: ") + for r in result["csv_data"] + ) + + +# ========================================================================= +# FS-31 — lines 1864-1914: KB data source sync stale/fresh paths +# ========================================================================= + + +class TestFS31KbSyncPaths: + def test_warn_stale_data_sources(self): + """Lines 1864-1882: KB data sources not synced in >7 days → WARN.""" + stale_time = datetime.now(timezone.utc) - timedelta(days=10) + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1", "name": "my-kb"}], + data_sources_by_kb={ + "kb1": [ + { + "dataSourceId": "ds1", + "name": "s3-source", + "updatedAt": stale_time, + } + ] + }, + data_source_detail={}, + ) + ) + result = app.check_knowledge_base_data_source_sync(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + def test_pass_recently_synced(self): + """Line 1901: all data sources synced within 7 days → Passed.""" + fresh_time = datetime.now(timezone.utc) - timedelta(days=1) + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1", "name": "my-kb"}], + data_sources_by_kb={ + "kb1": [ + { + "dataSourceId": "ds1", + "name": "s3-source", + "updatedAt": fresh_time, + } + ] + }, + data_source_detail={}, + ) + ) + result = app.check_knowledge_base_data_source_sync(inv) + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-33 — lines 1976-2034: KB integrity monitoring paths +# ========================================================================= + + +class TestFS33KbIntegrityPaths: + @patch("app.boto3.client") + def test_warn_bucket_without_versioning(self, mock_client): + """Lines 1976-2003: KB data source bucket without versioning → WARN.""" + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1"}], + data_sources_by_kb={"kb1": [{"dataSourceId": "ds1"}]}, + data_source_detail={ + ("kb1", "ds1"): { + "dataSource": { + "dataSourceConfiguration": { + "s3Configuration": { + "bucketArn": "arn:aws:s3:::kb-data-bucket" + } + } + } + } + }, + ) + ) + + def side_effect(service, **kwargs): + if service == "s3": + c = MagicMock() + c.get_bucket_versioning.return_value = {} # not enabled + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_knowledge_base_integrity_monitoring(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_pass_bucket_with_versioning(self, mock_client): + """Line 2021: all KB buckets have versioning → Passed.""" + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1"}], + data_sources_by_kb={"kb1": [{"dataSourceId": "ds1"}]}, + data_source_detail={ + ("kb1", "ds1"): { + "dataSource": { + "dataSourceConfiguration": { + "s3Configuration": { + "bucketArn": "arn:aws:s3:::kb-data-bucket" + } + } + } + } + }, + ) + ) + + def side_effect(service, **kwargs): + if service == "s3": + c = MagicMock() + c.get_bucket_versioning.return_value = {"Status": "Enabled"} + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_knowledge_base_integrity_monitoring(inv) + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_deleted_bucket_reported_separately(self, mock_client): + """A NoSuchBucket on get_bucket_versioning → distinct 'deleted bucket' + finding (High), NOT conflated with 'without versioning' or labeled + '(error)'. Regression guard for the FS-33 NoSuchBucket refinement.""" + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1", "name": "kb-one"}], + data_sources_by_kb={"kb1": [{"dataSourceId": "ds1", "name": "ds-one"}]}, + data_source_detail={ + ("kb1", "ds1"): { + "dataSource": { + "dataSourceConfiguration": { + "s3Configuration": { + "bucketArn": "arn:aws:s3:::deleted-bucket" + } + } + } + } + }, + ) + ) + + def side_effect(service, **kwargs): + if service == "s3": + c = MagicMock() + c.get_bucket_versioning.side_effect = _client_error("NoSuchBucket") + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_knowledge_base_integrity_monitoring(inv) + _assert_structure(result) + assert result["status"] == "WARN" + # Distinct deleted-bucket finding present, High severity, not "(error)". + deleted = [ + r + for r in result["csv_data"] + if r["Finding"] == "KB Data Source References a Deleted S3 Bucket" + ] + assert deleted, "expected a distinct deleted-bucket finding" + assert deleted[0]["Severity"] == "High" + assert "deleted-bucket" in deleted[0]["Finding_Details"] + # No "(error)" mislabel and no "Without Versioning" finding for this bucket. + assert not any( + "(error)" in r.get("Finding_Details", "") for r in result["csv_data"] + ) + assert not any( + r["Finding"] == "KB Data Source Buckets Without Versioning" + for r in result["csv_data"] + ) + + @patch("app.boto3.client") + def test_bucket_nonaccess_nonmissing_error_treated_as_unversioned( + self, mock_client + ): + """A non-access, non-missing ClientError on get_bucket_versioning → + bucket flagged as '(error)' under the versioning finding (WARN), not + silently dropped and not treated as a deleted bucket.""" + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1"}], + data_sources_by_kb={"kb1": [{"dataSourceId": "ds1"}]}, + data_source_detail={ + ("kb1", "ds1"): { + "dataSource": { + "dataSourceConfiguration": { + "s3Configuration": { + "bucketArn": "arn:aws:s3:::weird-bucket" + } + } + } + } + }, + ) + ) + + def side_effect(service, **kwargs): + if service == "s3": + c = MagicMock() + c.get_bucket_versioning.side_effect = _client_error("InvalidRequest") + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_knowledge_base_integrity_monitoring(inv) + _assert_structure(result) + assert result["status"] == "WARN" + assert any( + "(error)" in r.get("Finding_Details", "") for r in result["csv_data"] + ) + + @patch("app.boto3.client") + def test_bucket_access_error_surfaces_as_could_not_assess(self, mock_client): + """An AccessDenied on get_bucket_versioning must re-raise → ERROR envelope + (could-not-assess), NOT a false 'no versioning' finding.""" + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1"}], + data_sources_by_kb={"kb1": [{"dataSourceId": "ds1"}]}, + data_source_detail={ + ("kb1", "ds1"): { + "dataSource": { + "dataSourceConfiguration": { + "s3Configuration": { + "bucketArn": "arn:aws:s3:::locked-bucket" + } + } + } + } + }, + ) + ) + + def side_effect(service, **kwargs): + if service == "s3": + c = MagicMock() + c.get_bucket_versioning.side_effect = _client_error("AccessDenied") + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_knowledge_base_integrity_monitoring(inv) + _assert_structure(result) + assert result["status"] == "ERROR" + + +# ========================================================================= +# FS-34 — lines 2056-2057: FM version currency pass path +# ========================================================================= + + +class TestFS34FmVersionPass: + @patch("app.boto3.client") + def test_pass_no_legacy_models_in_use(self, mock_client): + """Lines 2056-2057: no legacy models in use → Passed.""" + c = MagicMock() + c.list_foundation_models.return_value = { + "modelSummaries": [ + { + "modelId": "anthropic.claude-v2", + "modelLifecycle": {"status": "ACTIVE"}, + } + ] + } + c.list_custom_models.return_value = {"modelSummaries": []} + mock_client.return_value = c + result = app.check_fm_version_currency() + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-35 — line 2127: eval jobs found pass path +# ========================================================================= + + +class TestFS35FmevalPass: + def test_advisory_finding(self): + """FS-35 is advisory (REQ-10a): always one N/A Informational ADVISORY row.""" + result = app.check_fmeval_harmful_content() + _assert_structure(result) + assert any( + r["Status"] == "N/A" + and r["Severity"] == "Informational" + and r["Finding"].startswith("ADVISORY: ") + for r in result["csv_data"] + ) + + +# ========================================================================= +# FS-36 — lines 2177-2178: content filters warn path +# ========================================================================= + + +class TestFS36ContentFiltersWarn: + def test_warn_guardrails_without_content_filters(self): + """Lines 2177-2178: guardrails exist but no content filters → WARN.""" + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={"g1": {"contentPolicy": {"filters": []}}}, + ) + ) + result = app.check_guardrail_content_filters(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + +# ========================================================================= +# FS-38 — lines 2266-2309: word filters warn/pass paths +# ========================================================================= + + +class TestFS38WordFiltersPaths: + def test_warn_guardrails_without_word_filters(self): + """Lines 2266-2276: guardrails exist but no word filters → WARN.""" + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": {"wordPolicy": {"words": [], "managedWordLists": []}} + }, + ) + ) + result = app.check_guardrail_word_filters(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + def test_pass_word_filters_configured(self): + """Line 2296: guardrail with word filters → Passed.""" + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": { + "wordPolicy": { + "words": [{"text": "insider trading"}], + "managedWordLists": [{"type": "PROFANITY"}], + } + } + }, + ) + ) + result = app.check_guardrail_word_filters(inv) + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-39 — line 2352: Clarify bias pass path +# ========================================================================= + + +class TestFS39ClarifyBiasPass: + @patch("app.boto3.client") + def test_pass_bias_schedules_found(self, mock_client): + """Line 2352: bias monitoring schedules found → Passed.""" + c = MagicMock() + c.list_monitoring_schedules.return_value = { + "MonitoringScheduleSummaries": [ + { + "MonitoringScheduleName": "bias-monitor", + "MonitoringType": "ModelBias", + } + ] + } + mock_client.return_value = c + result = app.check_sagemaker_clarify_bias() + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-40 — line 2397: bias eval pass path +# ========================================================================= + + +class TestFS40BiasEvalPass: + def test_advisory_finding(self): + """FS-40 is advisory (REQ-10a): always one N/A Informational ADVISORY row.""" + result = app.check_bedrock_evaluation_bias_datasets() + _assert_structure(result) + assert any( + r["Status"] == "N/A" + and r["Severity"] == "Informational" + and r["Finding"].startswith("ADVISORY: ") + for r in result["csv_data"] + ) + + +# ========================================================================= +# FS-41 — line 2452: Clarify explainability pass path +# ========================================================================= + + +class TestFS41ClarifyExplainabilityPass: + @patch("app.boto3.client") + def test_pass_explainability_schedules_found(self, mock_client): + """Line 2452: explainability monitoring schedules found → Passed.""" + c = MagicMock() + c.list_monitoring_schedules.return_value = { + "MonitoringScheduleSummaries": [ + { + "MonitoringScheduleName": "explain-monitor", + "MonitoringType": "ModelExplainability", + } + ] + } + mock_client.return_value = c + result = app.check_sagemaker_clarify_explainability() + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-42 — line 2501: model cards pass path +# ========================================================================= + + +class TestFS42ModelCardsPass: + @patch("app.boto3.client") + def test_pass_model_cards_found(self, mock_client): + """Line 2501: model cards exist → Passed (key is ModelCardSummaryList).""" + c = MagicMock() + c.list_model_cards.return_value = { + "ModelCardSummaryList": [{"ModelCardName": "fraud-model-card"}] + } + mock_client.return_value = c + result = app.check_ai_service_cards_documentation() + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-43 — lines 2536-2541: CloudWatch PII masking paths +# ========================================================================= + + +class TestFS43CloudwatchPiiPaths: + @patch("app.boto3.client") + def test_warn_no_data_protection_policies(self, mock_client): + """Lines 2540-2541: no data protection policies → WARN.""" + c = MagicMock() + c.describe_account_policies.return_value = {"accountPolicies": []} + mock_client.return_value = c + result = app.check_cloudwatch_log_pii_masking() + _assert_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_client_error_on_describe_policies_treated_as_no_policies( + self, mock_client + ): + """Line 2536: ClientError on describe_account_policies → policies = [] → WARN.""" + c = MagicMock() + c.describe_account_policies.side_effect = _client_error("AccessDeniedException") + mock_client.return_value = c + result = app.check_cloudwatch_log_pii_masking() + _assert_structure(result) + assert result["status"] == "WARN" + + +# ========================================================================= +# FS-44 — lines 2589-2628: Macie paths +# ========================================================================= + + +class TestFS44MaciePaths: + @patch("app.boto3.client") + def test_warn_macie_not_enabled(self, mock_client): + """Lines 2593-2595: Macie session status not ENABLED → WARN.""" + c = MagicMock() + c.get_macie_session.return_value = {"status": "PAUSED"} + mock_client.return_value = c + result = app.check_macie_on_training_data_buckets() + _assert_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_warn_macie_client_error(self, mock_client): + """Lines 2590-2591: ClientError on get_macie_session → macie_enabled=False → WARN.""" + c = MagicMock() + c.get_macie_session.side_effect = _client_error("AccessDeniedException") + mock_client.return_value = c + result = app.check_macie_on_training_data_buckets() + _assert_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_pass_macie_enabled(self, mock_client): + """Line 2615: Macie enabled → Passed.""" + c = MagicMock() + c.get_macie_session.return_value = {"status": "ENABLED"} + mock_client.return_value = c + result = app.check_macie_on_training_data_buckets() + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-45 — lines 2665-2666: PII filters warn path +# ========================================================================= + + +class TestFS45PiiFiltersWarn: + def test_warn_guardrails_without_pii_filters(self): + """Lines 2665-2666: guardrails exist but no PII filters → WARN.""" + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": {"sensitiveInformationPolicy": {"piiEntities": []}} + }, + ) + ) + result = app.check_guardrail_pii_filters(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + +# ========================================================================= +# FS-46 — lines 2734-2778: data classification tagging paths +# ========================================================================= + + +class TestFS46DataClassificationPaths: + @patch("app.boto3.client") + def test_warn_buckets_without_classification_tags(self, mock_client): + """Lines 2734-2746: AI/ML buckets without classification tags → WARN.""" + inv = make_resource_inventory(buckets=[{"Name": "aiml-training-data"}]) + c = MagicMock() + c.get_bucket_tagging.return_value = { + "TagSet": [{"Key": "Environment", "Value": "prod"}] + } + mock_client.return_value = c + + result = app.check_data_classification_tagging(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_warn_tagging_client_error_treated_as_unclassified(self, mock_client): + """Line 2741-2742: ClientError on get_bucket_tagging → bucket added as unclassified.""" + inv = make_resource_inventory(buckets=[{"Name": "aiml-training-data"}]) + c = MagicMock() + c.get_bucket_tagging.side_effect = _client_error("NoSuchTagSet") + mock_client.return_value = c + + result = app.check_data_classification_tagging(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_tagging_access_error_surfaces_as_could_not_assess(self, mock_client): + """AccessDenied on get_bucket_tagging re-raises → ERROR (could-not-assess), + NOT a false 'unclassified' finding.""" + inv = make_resource_inventory(buckets=[{"Name": "aiml-training-data"}]) + c = MagicMock() + c.get_bucket_tagging.side_effect = _client_error("AccessDenied") + mock_client.return_value = c + + result = app.check_data_classification_tagging(inv) + _assert_structure(result) + assert result["status"] == "ERROR" + + @patch("app.boto3.client") + def test_pass_all_buckets_classified(self, mock_client): + """Line 2765: all AI/ML buckets have classification tags → Passed.""" + inv = make_resource_inventory(buckets=[{"Name": "aiml-training-data"}]) + c = MagicMock() + c.get_bucket_tagging.return_value = { + "TagSet": [{"Key": "data-classification", "Value": "Confidential"}] + } + mock_client.return_value = c + + result = app.check_data_classification_tagging(inv) + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_multi_page_buckets_completeness(self, mock_client): + """Pagination completeness: buckets from multiple pages are all assessed. + + The inventory (already fully paginated by the collector) contains buckets + from 2+ simulated pages. This test verifies the check inspects all of them. + """ + # Simulate 2 "pages" of buckets — all AI/ML-named so they're all filtered in + page1 = [{"Name": "train-data-page1-001"}, {"Name": "bedrock-page1-002"}] + page2 = [{"Name": "knowledge-page2-001"}, {"Name": "sagemaker-page2-002"}] + all_buckets = page1 + page2 + + inv = make_resource_inventory(buckets=all_buckets) + c = MagicMock() + c.get_bucket_tagging.return_value = { + "TagSet": [{"Key": "data-classification", "Value": "Internal"}] + } + mock_client.return_value = c + + result = app.check_data_classification_tagging(inv) + _assert_structure(result) + # All 4 buckets classified → Passed + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + # Verify all 4 were assessed (tagging call per AI/ML bucket) + assert c.get_bucket_tagging.call_count == 4 + + @patch("app.boto3.client") + def test_single_page_unchanged_vs_baseline(self, mock_client): + """Single-page case: result is identical to pre-migration behavior (baseline). + + With ≤1 page of buckets, the inventory holds all buckets and the check + outcome is the same as before migration. + """ + inv = make_resource_inventory(buckets=[{"Name": "aiml-training-data"}]) + c = MagicMock() + c.get_bucket_tagging.return_value = { + "TagSet": [{"Key": "Environment", "Value": "prod"}] + } + mock_client.return_value = c + + result = app.check_data_classification_tagging(inv) + _assert_structure(result) + assert result["status"] == "WARN" + assert any( + r["Finding"] == "AI/ML Buckets Without Data Classification Tags" + for r in result["csv_data"] + ) + + +# ========================================================================= +# FS-47 — lines 2812-2857: grounding threshold warn/pass paths +# ========================================================================= + + +class TestFS47GroundingThresholdPaths: + def test_warn_low_grounding_threshold(self): + """Lines 2812-2826: guardrail with grounding threshold < 0.7 → WARN.""" + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": { + "contextualGroundingPolicy": { + "filters": [{"type": "GROUNDING", "threshold": 0.5}] + } + } + }, + ) + ) + result = app.check_guardrail_grounding_threshold(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + def test_pass_adequate_grounding_threshold(self): + """Line 2844: guardrail with grounding threshold >= 0.7 → Passed.""" + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": { + "contextualGroundingPolicy": { + "filters": [{"type": "GROUNDING", "threshold": 0.8}] + } + } + }, + ) + ) + result = app.check_guardrail_grounding_threshold(inv) + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + def test_fail_no_grounding_filter_at_all(self): + """Guardrails exist but NONE has a GROUNDING filter (only RELEVANCE) → + Failed, not a false Pass. Regression guard for the FS-47 false-pass fix.""" + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": { + "contextualGroundingPolicy": { + "filters": [{"type": "RELEVANCE", "threshold": 0.9}] + } + } + }, + ) + ) + result = app.check_guardrail_grounding_threshold(inv) + _assert_structure(result) + assert result["status"] == "WARN" + assert any(r["Status"] == "Failed" for r in result["csv_data"]) + assert not any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-48 — line 2898: RAG KB pass path (active KBs) +# ========================================================================= + + +class TestFS48RagKbPass: + def test_pass_active_kbs_found(self): + """Line 2898: active KBs found → Passed.""" + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[ + {"knowledgeBaseId": "kb1", "name": "rag-kb", "status": "ACTIVE"} + ], + data_sources_by_kb={}, + data_source_detail={}, + ) + ) + result = app.check_rag_knowledge_base_configured(inv) + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-50 — lines 2957-2985: ARC relevance grounding paths +# ========================================================================= + + +class TestFS50ArcRelevancePaths: + def test_warn_no_relevance_filters(self): + """Lines 2957-2963: guardrails exist but no RELEVANCE filter → WARN.""" + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": { + "contextualGroundingPolicy": { + "filters": [{"type": "GROUNDING", "threshold": 0.8}] + } + } + }, + ) + ) + result = app.check_guardrail_relevance_grounding(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + def test_pass_relevance_filter_found(self): + """Guardrail with RELEVANCE filter → Passed.""" + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": { + "contextualGroundingPolicy": { + "filters": [{"type": "RELEVANCE", "threshold": 0.7}] + } + } + }, + ) + ) + result = app.check_guardrail_relevance_grounding(inv) + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-51 — lines 3037-3038: prompt injection warn path +# ========================================================================= + + +class TestFS51PromptInjectionWarn: + def test_warn_no_prompt_attack_filter(self): + """Lines 3037-3038: guardrails exist but no PROMPT_ATTACK filter → WARN.""" + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": { + "contentPolicy": { + "filters": [{"type": "HATE", "inputStrength": "HIGH"}] + } + } + }, + ) + ) + result = app.check_prompt_injection_input_validation(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + +# ========================================================================= +# FS-52 — lines 3105-3146: SDK version currency paths +# ========================================================================= + + +class TestFS52SdkVersionPaths: + def test_warn_deprecated_runtime(self): + """Lines 3105-3113: Bedrock Lambda on deprecated runtime → WARN.""" + inv = make_resource_inventory( + lambda_functions=[ + {"FunctionName": "bedrock-invoke-fn", "Runtime": "python3.8"} + ] + ) + result = app.check_bedrock_sdk_version_currency(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + def test_pass_current_runtime(self): + """Line 3133: Bedrock Lambda on current runtime → Passed.""" + inv = make_resource_inventory( + lambda_functions=[ + {"FunctionName": "bedrock-invoke-fn", "Runtime": "python3.12"} + ] + ) + result = app.check_bedrock_sdk_version_currency(inv) + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-53 — lines 3192-3196: WAF injection rules warn path +# ========================================================================= + + +class TestFS53WafInjectionWarn: + def test_warn_acls_without_injection_rules(self): + """Lines 3192-3196: WAF ACLs exist but no injection rule groups → WARN.""" + acl_detail = { + "Rules": [ + { + "Name": "rate-limit", + "Statement": {"RateBasedStatement": {"Limit": 1000}}, + } + ] + } + inv = make_resource_inventory( + web_acls=app.WebAclInventory( + summaries=[{"Name": "acl1", "Id": "id1"}], + detail_by_id={"id1": acl_detail}, + ) + ) + result = app.check_waf_sql_injection_rules(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + +# ========================================================================= +# FS-65 — lines 3770, 3781-3782: S3 event notification edge cases +# ========================================================================= + + +class TestFS65S3EventNotificationEdgeCases: + def test_skip_datasource_with_no_bucket(self): + """Line 3770: data source with no bucket ARN → continue (no bucket added).""" + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1"}], + data_sources_by_kb={"kb1": [{"dataSourceId": "ds1"}]}, + data_source_detail={ + ("kb1", "ds1"): { + "dataSource": { + "dataSourceConfiguration": { + "s3Configuration": {} # no bucketArn + } + } + } + }, + ) + ) + result = app.check_kb_datasource_s3_event_notifications(inv) + _assert_structure(result) + # No buckets to check → PASS + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_s3_notification_access_error_surfaces_as_could_not_assess( + self, mock_client + ): + """An AccessDenied on get_bucket_notification_configuration must re-raise → + ERROR envelope (could-not-assess), NOT a false 'missing notifications' finding.""" + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1"}], + data_sources_by_kb={"kb1": [{"dataSourceId": "ds1"}]}, + data_source_detail={ + ("kb1", "ds1"): { + "dataSource": { + "dataSourceConfiguration": { + "s3Configuration": { + "bucketArn": "arn:aws:s3:::kb-bucket" + } + } + } + } + }, + ) + ) + + def side_effect(service, **kwargs): + if service == "s3": + c = MagicMock() + c.get_bucket_notification_configuration.side_effect = _client_error( + "AccessDenied" + ) + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_kb_datasource_s3_event_notifications(inv) + _assert_structure(result) + assert result["status"] == "ERROR" + + @patch("app.boto3.client") + def test_deleted_bucket_reported_separately(self, mock_client): + """A NoSuchBucket on get_bucket_notification_configuration → distinct + 'deleted bucket' finding (High), not conflated with 'missing + notifications' or labeled '(error)'. Regression guard for the FS-65 + NoSuchBucket refinement.""" + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1", "name": "kb-one"}], + data_sources_by_kb={"kb1": [{"dataSourceId": "ds1", "name": "ds-one"}]}, + data_source_detail={ + ("kb1", "ds1"): { + "dataSource": { + "dataSourceConfiguration": { + "s3Configuration": { + "bucketArn": "arn:aws:s3:::deleted-kb-bucket" + } + } + } + } + }, + ) + ) + + def side_effect(service, **kwargs): + if service == "s3": + c = MagicMock() + c.get_bucket_notification_configuration.side_effect = _client_error( + "NoSuchBucket" + ) + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_kb_datasource_s3_event_notifications(inv) + _assert_structure(result) + assert result["status"] == "WARN" + deleted = [ + r + for r in result["csv_data"] + if r["Finding"] == "KB Data Source References a Deleted S3 Bucket" + ] + assert deleted, "expected a distinct deleted-bucket finding" + assert deleted[0]["Severity"] == "High" + assert "deleted-kb-bucket" in deleted[0]["Finding_Details"] + assert not any( + "(error)" in r.get("Finding_Details", "") for r in result["csv_data"] + ) + assert not any( + r["Finding"] == "KB Data Source Buckets Missing S3 Event Notifications" + for r in result["csv_data"] + ) + + +class TestFS66AgentcoreIdentityReraise: + @patch("app.boto3.client") + def test_non_access_denied_reraises(self, mock_client): + """Line 3849: non-AccessDenied ClientError → re-raised → ERROR.""" + c = MagicMock() + c.list_agent_runtimes.side_effect = _client_error("ServiceUnavailableException") + mock_client.return_value = c + result = app.check_agentcore_end_user_identity_propagation() + _assert_structure(result) + assert result["status"] == "ERROR" + + +# ========================================================================= +# FS-68 — lines 4031, 4049, 4053, 4056-4057: body size limit warn paths +# ========================================================================= + + +class TestFS68BodySizeLimitWarnPaths: + def test_warn_rest_api_without_validators(self): + """Lines 4031, 4049, 4056-4057: REST API without validators → WARN.""" + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=[], detail_by_id={}) + ) + with patch("app.boto3.client") as mock_client: + + def side_effect(service, **kwargs): + if service == "apigateway": + c = MagicMock() + c.get_rest_apis.return_value = { + "items": [{"id": "api1", "name": "genai-api"}] + } + c.get_request_validators.return_value = { + "items": [] + } # no validators + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_api_gateway_request_body_size_limits(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + def test_warn_waf_acls_without_size_rules(self): + """Lines 4053, 4056-4057: WAF ACLs exist but no size constraint rules → WARN.""" + acl_detail = { + "Rules": [ + { + "Name": "rate-limit", + "Statement": {"RateBasedStatement": {}}, + } + ] + } + inv = make_resource_inventory( + web_acls=app.WebAclInventory( + summaries=[{"Name": "acl1", "Id": "id1"}], + detail_by_id={"id1": acl_detail}, + ) + ) + with patch("app.boto3.client") as mock_client: + + def side_effect(service, **kwargs): + if service == "apigateway": + c = MagicMock() + c.get_rest_apis.return_value = {"items": []} + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_api_gateway_request_body_size_limits(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + +# ========================================================================= +# FS-34 — lines 2056-2057: legacy models warn path +# ========================================================================= + + +class TestFS34FmVersionWarn: + @patch("app.boto3.client") + def test_warn_legacy_models_available(self, mock_client): + """Legacy foundation models available in region → WARN wrapper with an N/A + finding (availability is not usage, so it is surfaced for review, not failed).""" + c = MagicMock() + c.list_foundation_models.return_value = { + "modelSummaries": [ + {"modelId": "old-model-v1", "modelLifecycle": {"status": "LEGACY"}} + ] + } + c.list_custom_models.return_value = {"modelSummaries": []} + mock_client.return_value = c + result = app.check_fm_version_currency() + _assert_structure(result) + assert result["status"] == "WARN" + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + assert any( + "availability" in r["Finding_Details"].lower() for r in result["csv_data"] + ) diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_collector.py b/aiml-security-assessment/functions/security/finserv_tests/test_collector.py new file mode 100644 index 0000000..ecf603b --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_tests/test_collector.py @@ -0,0 +1,685 @@ +""" +Unit tests for collect_resource_inventory() and the _safe_collect_* helpers. + +Validates: Requirements REQ-1, REQ-2, REQ-3.4, REQ-4, REQ-7.5, REQ-9.2 +""" + +import sys +import os +from unittest.mock import MagicMock, patch + +import pytest + +FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") +if FINSERV_DIR not in sys.path: + sys.path.insert(0, FINSERV_DIR) + +import app # noqa: E402 (import follows sys.path bootstrap above) + + +# --------------------------------------------------------------------------- +# Helpers to build mock boto3 clients +# --------------------------------------------------------------------------- + + +def _make_client(responses: dict) -> MagicMock: + """Return a MagicMock boto3 client where each key in *responses* is a method + name and the value is either a single dict (returned every call) or a list + of dicts (returned in sequence, raising StopIteration when exhausted).""" + client = MagicMock() + for method_name, return_values in responses.items(): + if isinstance(return_values, list): + method = getattr(client, method_name) + method.side_effect = return_values + else: + method = getattr(client, method_name) + method.return_value = return_values + return client + + +def _single_page(key, items, extra=None): + """Build a single-page response dict with no continuation token.""" + r = {key: items} + if extra: + r.update(extra) + return r + + +# --------------------------------------------------------------------------- +# _safe_collect_lambda_functions +# --------------------------------------------------------------------------- + + +class TestSafeCollectLambdaFunctions: + def test_single_page_returns_functions(self): + fn = {"FunctionName": "my-fn", "Runtime": "python3.12"} + client = _make_client({"list_functions": {"Functions": [fn]}}) + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_lambda_functions() + assert result == [fn] + + def test_single_listing_call(self): + client = _make_client({"list_functions": {"Functions": []}}) + with patch("app.boto3.client", return_value=client): + app._safe_collect_lambda_functions() + assert client.list_functions.call_count == 1 + + def test_multi_page_merges_all_functions(self): + """Multi-page result via NextMarker/Marker pagination is merged correctly.""" + fn1 = {"FunctionName": "fn-1"} + fn2 = {"FunctionName": "fn-2"} + client = _make_client( + { + "list_functions": [ + {"Functions": [fn1], "NextMarker": "page2"}, + {"Functions": [fn2]}, + ] + } + ) + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_lambda_functions() + assert result == [fn1, fn2] + assert client.list_functions.call_count == 2 + # Second call must use Marker= (Lambda convention) + _, kwargs = client.list_functions.call_args + assert "Marker" in kwargs + + def test_failure_returns_unavailable(self): + err = PermissionError("AccessDenied") + client = _make_client({}) + client.list_functions.side_effect = err + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_lambda_functions() + assert isinstance(result, app._Unavailable) + assert result.error is err + + def test_client_constructed_without_region_or_endpoint(self): + client = _make_client({"list_functions": {"Functions": []}}) + with patch("app.boto3.client", return_value=client) as mock_boto: + app._safe_collect_lambda_functions() + mock_boto.assert_called_once() + _, kwargs = mock_boto.call_args + assert "region_name" not in kwargs + assert "endpoint_url" not in kwargs + + +# --------------------------------------------------------------------------- +# _safe_collect_guardrails +# --------------------------------------------------------------------------- + + +class TestSafeCollectGuardrails: + def test_single_page_returns_guardrail_inventory(self): + g1 = {"id": "g-abc", "name": "my-guardrail"} + detail = {"guardrailId": "g-abc", "sensitiveInformationPolicy": {}} + client = _make_client( + { + "list_guardrails": {"guardrails": [g1]}, + "get_guardrail": detail, + } + ) + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_guardrails() + assert isinstance(result, app.GuardrailInventory) + assert result.summaries == [g1] + assert result.detail_by_id["g-abc"] is detail + + def test_single_listing_call(self): + client = _make_client( + { + "list_guardrails": {"guardrails": []}, + } + ) + with patch("app.boto3.client", return_value=client): + app._safe_collect_guardrails() + assert client.list_guardrails.call_count == 1 + + def test_multi_page_merges_guardrails(self): + g1 = {"id": "g1"} + g2 = {"id": "g2"} + client = _make_client( + { + "list_guardrails": [ + {"guardrails": [g1], "nextToken": "tok2"}, + {"guardrails": [g2]}, + ], + "get_guardrail": {"guardrailId": "gx"}, + } + ) + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_guardrails() + assert len(result.summaries) == 2 + assert client.list_guardrails.call_count == 2 + + def test_get_guardrail_called_with_draft(self): + """get_guardrail must always be called with guardrailVersion='DRAFT'.""" + g1 = {"id": "g-001"} + detail = {"guardrailId": "g-001"} + client = _make_client( + { + "list_guardrails": {"guardrails": [g1]}, + "get_guardrail": detail, + } + ) + with patch("app.boto3.client", return_value=client): + app._safe_collect_guardrails() + client.get_guardrail.assert_called_once_with( + guardrailIdentifier="g-001", guardrailVersion="DRAFT" + ) + + def test_whole_inventory_failure_returns_unavailable(self): + err = PermissionError("AccessDenied on list_guardrails") + client = _make_client({}) + client.list_guardrails.side_effect = err + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_guardrails() + assert isinstance(result, app._Unavailable) + assert result.error is err + + def test_single_detail_failure_isolates_to_that_id(self): + """A get_guardrail failure for one id stores _Unavailable only for that id.""" + g1 = {"id": "g-ok"} + g2 = {"id": "g-bad"} + ok_detail = {"guardrailId": "g-ok"} + err = PermissionError("denied") + + def detail_side_effect(**kwargs): + if kwargs["guardrailIdentifier"] == "g-bad": + raise err + return ok_detail + + client = _make_client({"list_guardrails": {"guardrails": [g1, g2]}}) + client.get_guardrail.side_effect = detail_side_effect + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_guardrails() + assert isinstance(result, app.GuardrailInventory) + assert result.detail_by_id["g-ok"] is ok_detail + assert isinstance(result.detail_by_id["g-bad"], app._Unavailable) + assert result.detail_by_id["g-bad"].error is err + + def test_client_constructed_without_region_or_endpoint(self): + client = _make_client({"list_guardrails": {"guardrails": []}}) + with patch("app.boto3.client", return_value=client) as mock_boto: + app._safe_collect_guardrails() + mock_boto.assert_called_once() + _, kwargs = mock_boto.call_args + assert "region_name" not in kwargs + assert "endpoint_url" not in kwargs + + +# --------------------------------------------------------------------------- +# _safe_collect_knowledge_bases +# --------------------------------------------------------------------------- + + +class TestSafeCollectKnowledgeBases: + def _setup_client(self, kb_list, ds_by_kb=None, ds_detail_by_pair=None): + """Build a mock bedrock-agent client.""" + client = MagicMock() + # list_knowledge_bases + client.list_knowledge_bases.return_value = {"knowledgeBaseSummaries": kb_list} + + # list_data_sources + def list_ds(**kwargs): + kb_id = kwargs["knowledgeBaseId"] + items = (ds_by_kb or {}).get(kb_id, []) + return {"dataSourceSummaries": items} + + client.list_data_sources.side_effect = list_ds + + # get_data_source + def get_ds(**kwargs): + pair = (kwargs["knowledgeBaseId"], kwargs["dataSourceId"]) + if ds_detail_by_pair and pair in ds_detail_by_pair: + v = ds_detail_by_pair[pair] + if isinstance(v, Exception): + raise v + return v + return {"dataSource": {"knowledgeBaseId": kwargs["knowledgeBaseId"]}} + + client.get_data_source.side_effect = get_ds + return client + + def test_single_kb_single_ds(self): + kb1 = {"knowledgeBaseId": "kb-1"} + ds1 = {"dataSourceId": "ds-1"} + detail = {"dataSource": {"dataSourceId": "ds-1"}} + client = self._setup_client([kb1], {"kb-1": [ds1]}, {("kb-1", "ds-1"): detail}) + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_knowledge_bases() + assert isinstance(result, app.KbInventory) + assert result.summaries == [kb1] + assert result.data_sources_by_kb["kb-1"] == [ds1] + assert result.data_source_detail[("kb-1", "ds-1")] is detail + + def test_single_listing_call(self): + client = self._setup_client([]) + with patch("app.boto3.client", return_value=client): + app._safe_collect_knowledge_bases() + assert client.list_knowledge_bases.call_count == 1 + + def test_multi_page_kbs(self): + kb1 = {"knowledgeBaseId": "kb-1"} + kb2 = {"knowledgeBaseId": "kb-2"} + client = MagicMock() + client.list_knowledge_bases.side_effect = [ + {"knowledgeBaseSummaries": [kb1], "nextToken": "tok2"}, + {"knowledgeBaseSummaries": [kb2]}, + ] + client.list_data_sources.return_value = {"dataSourceSummaries": []} + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_knowledge_bases() + assert len(result.summaries) == 2 + assert client.list_knowledge_bases.call_count == 2 + + def test_whole_inventory_failure_returns_unavailable(self): + err = PermissionError("denied") + client = MagicMock() + client.list_knowledge_bases.side_effect = err + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_knowledge_bases() + assert isinstance(result, app._Unavailable) + assert result.error is err + + def test_per_kb_data_source_failure_isolates(self): + """list_data_sources failure for one KB stores _Unavailable for that KB only.""" + kb1 = {"knowledgeBaseId": "kb-ok"} + kb2 = {"knowledgeBaseId": "kb-bad"} + err = PermissionError("denied for kb-bad") + + client = MagicMock() + client.list_knowledge_bases.return_value = { + "knowledgeBaseSummaries": [kb1, kb2] + } + + def list_ds(**kwargs): + if kwargs["knowledgeBaseId"] == "kb-bad": + raise err + return {"dataSourceSummaries": []} + + client.list_data_sources.side_effect = list_ds + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_knowledge_bases() + assert isinstance(result, app.KbInventory) + assert result.data_sources_by_kb["kb-ok"] == [] + assert isinstance(result.data_sources_by_kb["kb-bad"], app._Unavailable) + assert result.data_sources_by_kb["kb-bad"].error is err + + def test_per_data_source_detail_failure_isolates(self): + """get_data_source failure for one DS stores _Unavailable for that (kb, ds) only.""" + kb1 = {"knowledgeBaseId": "kb-1"} + ds_ok = {"dataSourceId": "ds-ok"} + ds_bad = {"dataSourceId": "ds-bad"} + ok_detail = {"dataSource": {"dataSourceId": "ds-ok"}} + err = PermissionError("denied") + + client = self._setup_client( + [kb1], + {"kb-1": [ds_ok, ds_bad]}, + {("kb-1", "ds-ok"): ok_detail, ("kb-1", "ds-bad"): err}, + ) + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_knowledge_bases() + assert result.data_source_detail[("kb-1", "ds-ok")] is ok_detail + assert isinstance( + result.data_source_detail[("kb-1", "ds-bad")], app._Unavailable + ) + + def test_client_constructed_without_region_or_endpoint(self): + client = self._setup_client([]) + with patch("app.boto3.client", return_value=client) as mock_boto: + app._safe_collect_knowledge_bases() + mock_boto.assert_called_once() + _, kwargs = mock_boto.call_args + assert "region_name" not in kwargs + assert "endpoint_url" not in kwargs + + +# --------------------------------------------------------------------------- +# _safe_collect_buckets +# --------------------------------------------------------------------------- + + +class TestSafeCollectBuckets: + def test_single_page_returns_buckets(self): + b1 = {"Name": "my-bucket"} + client = _make_client({"list_buckets": {"Buckets": [b1]}}) + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_buckets() + assert result == [b1] + + def test_single_listing_call(self): + client = _make_client({"list_buckets": {"Buckets": []}}) + with patch("app.boto3.client", return_value=client): + app._safe_collect_buckets() + assert client.list_buckets.call_count == 1 + + def test_multi_page_with_continuation_token(self): + """S3 ContinuationToken/ContinuationToken pagination is used (REQ-2.8).""" + b1 = {"Name": "bucket-1"} + b2 = {"Name": "bucket-2"} + client = _make_client( + { + "list_buckets": [ + {"Buckets": [b1], "ContinuationToken": "tok2"}, + {"Buckets": [b2]}, + ] + } + ) + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_buckets() + assert result == [b1, b2] + assert client.list_buckets.call_count == 2 + # Second call must use ContinuationToken= as input (not NextToken etc.) + _, kwargs = client.list_buckets.call_args + assert "ContinuationToken" in kwargs + + def test_max_buckets_parameter_sent_on_first_call(self): + """MaxBuckets must be included on the first call to engage pagination.""" + client = _make_client({"list_buckets": {"Buckets": []}}) + with patch("app.boto3.client", return_value=client): + app._safe_collect_buckets() + first_call_kwargs = client.list_buckets.call_args_list[0][1] + assert first_call_kwargs.get("MaxBuckets") == 1000 + + def test_failure_returns_unavailable(self): + err = PermissionError("AccessDenied") + client = MagicMock() + client.list_buckets.side_effect = err + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_buckets() + assert isinstance(result, app._Unavailable) + assert result.error is err + + def test_client_constructed_without_region_or_endpoint(self): + client = _make_client({"list_buckets": {"Buckets": []}}) + with patch("app.boto3.client", return_value=client) as mock_boto: + app._safe_collect_buckets() + mock_boto.assert_called_once() + _, kwargs = mock_boto.call_args + assert "region_name" not in kwargs + assert "endpoint_url" not in kwargs + + +# --------------------------------------------------------------------------- +# _safe_collect_web_acls +# --------------------------------------------------------------------------- + + +class TestSafeCollectWebAcls: + def test_single_page_returns_web_acl_inventory(self): + acl1 = {"Id": "acl-1", "Name": "my-acl", "ARN": "arn:aws:wafv2:::webacl/acl-1"} + detail = {"WebACL": {"Id": "acl-1"}} + client = _make_client( + { + "list_web_acls": {"WebACLs": [acl1]}, + "get_web_acl": detail, + } + ) + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_web_acls() + assert isinstance(result, app.WebAclInventory) + assert result.summaries == [acl1] + # The collector extracts response["WebACL"] — not the full envelope — + # so downstream checks can do detail.get("Rules") without extra indirection. + assert result.detail_by_id["acl-1"] == detail["WebACL"] + + def test_single_listing_call(self): + client = _make_client( + { + "list_web_acls": {"WebACLs": []}, + } + ) + with patch("app.boto3.client", return_value=client): + app._safe_collect_web_acls() + assert client.list_web_acls.call_count == 1 + + def test_list_web_acls_called_with_scope_regional(self): + """Scope='REGIONAL' must be passed on every list_web_acls call (REQ-7.2).""" + client = _make_client({"list_web_acls": {"WebACLs": []}}) + with patch("app.boto3.client", return_value=client): + app._safe_collect_web_acls() + client.list_web_acls.assert_called_once() + _, kwargs = client.list_web_acls.call_args + assert kwargs.get("Scope") == "REGIONAL" + + def test_multi_page_uses_next_marker_as_input(self): + """WAFv2 pagination uses NextMarker as BOTH output and input (REQ-2.2).""" + acl1 = {"Id": "acl-1", "Name": "acl-1"} + acl2 = {"Id": "acl-2", "Name": "acl-2"} + detail = {"WebACL": {}} + client = _make_client( + { + "list_web_acls": [ + {"WebACLs": [acl1], "NextMarker": "mark2"}, + {"WebACLs": [acl2]}, + ], + "get_web_acl": detail, + } + ) + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_web_acls() + assert len(result.summaries) == 2 + assert client.list_web_acls.call_count == 2 + # Second call must use NextMarker= (NOT Marker= which is Lambda's convention) + second_call_kwargs = client.list_web_acls.call_args_list[1][1] + assert "NextMarker" in second_call_kwargs + assert second_call_kwargs["NextMarker"] == "mark2" + assert "Marker" not in second_call_kwargs + + def test_get_web_acl_called_with_scope_regional(self): + """get_web_acl must always use Scope='REGIONAL' (REQ-7.2).""" + acl1 = {"Id": "acl-1", "Name": "acl-name"} + detail = {"WebACL": {"Id": "acl-1"}} + client = _make_client( + { + "list_web_acls": {"WebACLs": [acl1]}, + "get_web_acl": detail, + } + ) + with patch("app.boto3.client", return_value=client): + app._safe_collect_web_acls() + client.get_web_acl.assert_called_once_with( + Name="acl-name", Scope="REGIONAL", Id="acl-1" + ) + + def test_whole_inventory_failure_returns_unavailable(self): + err = PermissionError("AccessDenied") + client = MagicMock() + client.list_web_acls.side_effect = err + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_web_acls() + assert isinstance(result, app._Unavailable) + assert result.error is err + + def test_single_detail_failure_isolates_to_that_id(self): + acl_ok = {"Id": "acl-ok", "Name": "ok-acl"} + acl_bad = {"Id": "acl-bad", "Name": "bad-acl"} + ok_detail = {"WebACL": {"Id": "acl-ok"}} + err = PermissionError("denied") + + def get_web_acl(**kwargs): + if kwargs["Id"] == "acl-bad": + raise err + return ok_detail + + client = _make_client( + { + "list_web_acls": {"WebACLs": [acl_ok, acl_bad]}, + } + ) + client.get_web_acl.side_effect = get_web_acl + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_web_acls() + # Collector extracts ["WebACL"] — verify the stored dict matches the inner value + assert result.detail_by_id["acl-ok"] == ok_detail["WebACL"] + assert isinstance(result.detail_by_id["acl-bad"], app._Unavailable) + assert result.detail_by_id["acl-bad"].error is err + + def test_client_constructed_without_region_or_endpoint(self): + client = _make_client({"list_web_acls": {"WebACLs": []}}) + with patch("app.boto3.client", return_value=client) as mock_boto: + app._safe_collect_web_acls() + mock_boto.assert_called_once() + _, kwargs = mock_boto.call_args + assert "region_name" not in kwargs + assert "endpoint_url" not in kwargs + + +# --------------------------------------------------------------------------- +# collect_resource_inventory — integration +# --------------------------------------------------------------------------- + + +class TestCollectResourceInventory: + """Verify that collect_resource_inventory delegates to each _safe_collect_* + and assembles a ResourceInventory.""" + + def _patch_all_safe_fns(self): + """Context-manager that patches all five _safe_collect_* functions.""" + import contextlib + + @contextlib.contextmanager + def _ctx(): + lf = [{"FunctionName": "fn-1"}] + gi = app.GuardrailInventory(summaries=[{"id": "g-1"}], detail_by_id={}) + ki = app.KbInventory( + summaries=[], data_sources_by_kb={}, data_source_detail={} + ) + bk = [{"Name": "bkt-1"}] + wi = app.WebAclInventory(summaries=[{"Id": "acl-1"}], detail_by_id={}) + with ( + patch("app._safe_collect_lambda_functions", return_value=lf) as p1, + patch("app._safe_collect_guardrails", return_value=gi) as p2, + patch("app._safe_collect_knowledge_bases", return_value=ki) as p3, + patch("app._safe_collect_buckets", return_value=bk) as p4, + patch("app._safe_collect_web_acls", return_value=wi) as p5, + ): + yield p1, p2, p3, p4, p5 + + return _ctx() + + def test_returns_resource_inventory(self): + with self._patch_all_safe_fns(): + inv = app.collect_resource_inventory() + assert isinstance(inv, app.ResourceInventory) + + def test_each_safe_fn_called_exactly_once(self): + with self._patch_all_safe_fns() as (p1, p2, p3, p4, p5): + app.collect_resource_inventory() + for p in (p1, p2, p3, p4, p5): + assert p.call_count == 1 + + def test_inventory_fields_come_from_safe_fns(self): + with self._patch_all_safe_fns(): + inv = app.collect_resource_inventory() + # Spot-check a few field values match what the patched fns returned + assert inv.lambda_functions == [{"FunctionName": "fn-1"}] + assert inv.buckets == [{"Name": "bkt-1"}] + + def test_one_unavailable_does_not_prevent_others(self): + """An _Unavailable on one field doesn't affect the rest.""" + err = PermissionError("denied") + with ( + patch( + "app._safe_collect_lambda_functions", return_value=app._Unavailable(err) + ), + patch( + "app._safe_collect_guardrails", + return_value=app.GuardrailInventory([], {}), + ), + patch( + "app._safe_collect_knowledge_bases", + return_value=app.KbInventory([], {}, {}), + ), + patch("app._safe_collect_buckets", return_value=[]), + patch( + "app._safe_collect_web_acls", return_value=app.WebAclInventory([], {}) + ), + ): + inv = app.collect_resource_inventory() + assert isinstance(inv.lambda_functions, app._Unavailable) + assert isinstance(inv.guardrails, app.GuardrailInventory) + assert inv.buckets == [] + + def test_all_unavailable_still_returns_inventory(self): + """Even when every field fails, we get a ResourceInventory (not an exception).""" + err = RuntimeError("all down") + unav = app._Unavailable(err) + with ( + patch("app._safe_collect_lambda_functions", return_value=unav), + patch("app._safe_collect_guardrails", return_value=unav), + patch("app._safe_collect_knowledge_bases", return_value=unav), + patch("app._safe_collect_buckets", return_value=unav), + patch("app._safe_collect_web_acls", return_value=unav), + ): + inv = app.collect_resource_inventory() + assert isinstance(inv, app.ResourceInventory) + for field in ( + "lambda_functions", + "guardrails", + "knowledge_bases", + "buckets", + "web_acls", + ): + assert isinstance(getattr(inv, field), app._Unavailable) + + def test_unavailable_field_causes_could_not_assess_via_require(self): + """A consuming check using require() on an unavailable field gets the stored + error raised, which the outer try/except turns into _error_findings.""" + err = PermissionError("AccessDenied") + inv = app.ResourceInventory( + lambda_functions=app._Unavailable(err), + guardrails=app.GuardrailInventory([], {}), + knowledge_bases=app.KbInventory([], {}, {}), + buckets=[], + web_acls=app.WebAclInventory([], {}), + ) + with pytest.raises(PermissionError) as exc_info: + app.require(inv, "lambda_functions") + assert exc_info.value is err + + +# --------------------------------------------------------------------------- +# Ordering / listing-order preservation +# --------------------------------------------------------------------------- + + +class TestOrderPreservation: + def test_lambda_functions_order_preserved(self): + fns = [{"FunctionName": f"fn-{i}"} for i in range(5)] + client = _make_client({"list_functions": {"Functions": fns}}) + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_lambda_functions() + assert result == fns + + def test_web_acls_multi_page_order_preserved(self): + acls_p1 = [{"Id": f"acl-{i}", "Name": f"acl-{i}"} for i in range(3)] + acls_p2 = [{"Id": f"acl-{i}", "Name": f"acl-{i}"} for i in range(3, 6)] + client = _make_client( + { + "list_web_acls": [ + {"WebACLs": acls_p1, "NextMarker": "pg2"}, + {"WebACLs": acls_p2}, + ], + "get_web_acl": {"WebACL": {}}, + } + ) + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_web_acls() + assert result.summaries == acls_p1 + acls_p2 + + def test_buckets_multi_page_order_preserved(self): + bkts_p1 = [{"Name": f"bkt-{i}"} for i in range(3)] + bkts_p2 = [{"Name": f"bkt-{i}"} for i in range(3, 6)] + client = _make_client( + { + "list_buckets": [ + {"Buckets": bkts_p1, "ContinuationToken": "pg2"}, + {"Buckets": bkts_p2}, + ] + } + ) + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_buckets() + assert result == bkts_p1 + bkts_p2 diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_inventory_equivalence.py b/aiml-security-assessment/functions/security/finserv_tests/test_inventory_equivalence.py new file mode 100644 index 0000000..67e2c27 --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_tests/test_inventory_equivalence.py @@ -0,0 +1,1093 @@ +""" +Golden-equivalence baseline for the current FinServ lambda_handler. + +Task 1 — FinServ Shared-Inventory Refactor (FU-3) +Requirements: REQ-5.1, REQ-5.3, REQ-5.6 + +Strategy +-------- +Build a reusable single-page "account state" fixture that covers PASS, FAIL, +N/A (status="N/A"), and ADVISORY paths across the inventory-consuming checks: + + * Lambda functions (FS-09, 52, 55, 58, 67, 69) + * Guardrails + detail (FS-27a, 28, 36, 38, 45, 47, 50, 51, 59) + * Knowledge Bases + data sources + detail (FS-24, 31, 33, 48, 61, 65) + * S3 buckets (FS-21, 46) + * WAFv2 Web ACLs + detail (FS-01, 53, 56, 68) + +Constraints (REQ-5.3): ≤100 Web ACLs and ≤1 page of buckets so the fixture +remains single-page comparable after the pagination fix in Task 9. + +The frozen BASELINE literal below was generated by running lambda_handler +against this mock state BEFORE the refactor. After the refactor the handler +must produce the identical tuple sequence (Task 13 / REQ-5.3). +""" + +from __future__ import annotations + +import os +import sys +from datetime import datetime, timezone +from unittest.mock import MagicMock, patch + +import pytest + +# --------------------------------------------------------------------------- +# Make finserv_assessments importable (mirrors conftest.py) +# --------------------------------------------------------------------------- +FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") +if FINSERV_DIR not in sys.path: + sys.path.insert(0, FINSERV_DIR) + +import app # noqa: E402 + + +# =========================================================================== +# Account-state fixture helpers +# =========================================================================== + +# ---- Lambda functions ------------------------------------------------------- +# Two agent functions: one WITH reserved concurrency (→ PASS in FS-09), +# one validate/output function (→ PASS in FS-55, affects FS-58) +# A "schema-validator" function is added so FS-58 finds it (ADVISORY row with count) +_LAMBDA_FUNCTIONS = [ + {"FunctionName": "my-bedrock-agent-handler"}, + {"FunctionName": "my-output-validate-fn"}, + {"FunctionName": "my-schema-validator-fn"}, +] + +# ---- Guardrails ------------------------------------------------------------- +# One guardrail with full configuration (→ PASS paths across FS-27a/28/36/38/45/47/50/51/59) +_GUARDRAIL_SUMMARY = {"id": "gr-abc123", "name": "FinServGuardrail"} + +_GUARDRAIL_DETAIL = { + # topic policy with topics → FS-28 PASS (CLASSIC tier advisory), + # FS-59 PASS (CLASSIC tier advisory) + "topicPolicy": { + "topics": [{"name": "investment-advice", "type": "DENY"}], + "tier": {"tierName": "CLASSIC"}, + }, + # content filters → FS-36 PASS + "contentPolicy": { + "filters": [ + {"type": "HATE", "inputStrength": "HIGH", "outputStrength": "HIGH"}, + {"type": "VIOLENCE", "inputStrength": "MEDIUM", "outputStrength": "MEDIUM"}, + ] + }, + # word filters → FS-38 PASS + "wordPolicy": { + "words": [{"text": "fraud"}], + "managedWordLists": [{"type": "PROFANITY"}], + }, + # PII filters → FS-45 PASS + "sensitiveInformationPolicy": { + "piiEntities": [{"type": "US_SOCIAL_SECURITY_NUMBER", "action": "BLOCK"}] + }, + # contextual grounding → FS-27a PASS; grounding threshold ≥ 0.7 → FS-47 PASS; + # relevance-grounding → FS-50 PASS (single policy covers all three) + "contextualGroundingPolicy": { + "filters": [ + {"type": "GROUNDING", "threshold": 0.8}, + {"type": "RELEVANCE", "threshold": 0.8}, + ] + }, + # topic allowlist (FS-51 checks prompt injection input validation via guardrails) +} + +# ---- Knowledge Bases -------------------------------------------------------- +# One KB, one data source with recent updatedAt → FS-31 PASS +# Data source backed by a versioned S3 bucket → FS-33 PASS +_NOW = datetime.now(timezone.utc) +_KB_SUMMARY = {"knowledgeBaseId": "kb-00001", "name": "FinServKB"} + +_KB_DATA_SOURCE_SUMMARY = { + "dataSourceId": "ds-00001", + "name": "FinServDS", + "updatedAt": _NOW, # fresh → FS-31 PASS +} + +_KB_DATA_SOURCE_DETAIL = { + "dataSource": { + "dataSourceConfiguration": { + "s3Configuration": {"bucketArn": "arn:aws:s3:::my-kb-datasource-bucket"} + } + } +} + +# ---- S3 buckets ------------------------------------------------------------- +# Two buckets within one page (≤ 1 page constraint). +# "my-training-dataset-bucket": versioned → FS-21 PASS +# "my-bedrock-kb-bucket": tagged with data-classification → FS-46 PASS +# "my-kb-datasource-bucket": versioned (for FS-33 data-source check above) +_S3_BUCKETS = [ + {"Name": "my-training-dataset-bucket"}, + {"Name": "my-bedrock-kb-bucket"}, + {"Name": "my-kb-datasource-bucket"}, +] + +# ---- WAFv2 Web ACLs --------------------------------------------------------- +# Two ACLs, both with AWSManagedRulesSQLiRuleSet + AWSManagedRulesCommonRuleSet + +# body-size constraints → FS-53 PASS, FS-56 PASS, FS-68 PASS; FS-01 PASS. +# Constraint: ≤ 100 ACLs. +_ACL_SUMMARIES = [ + {"Name": "FinServACL1", "Id": "acl-id-001", "ARN": "arn:aws:wafv2:::acl-id-001"}, + {"Name": "FinServACL2", "Id": "acl-id-002", "ARN": "arn:aws:wafv2:::acl-id-002"}, +] + +_ACL_DETAIL_TEMPLATE = { + "Name": "FinServACL", + "Id": "acl-id-xxx", + "Rules": [ + { + "Name": "SQLi", + "Statement": { + "ManagedRuleGroupStatement": {"Name": "AWSManagedRulesSQLiRuleSet"} + }, + }, + { + "Name": "Common", + "Statement": { + "ManagedRuleGroupStatement": {"Name": "AWSManagedRulesCommonRuleSet"} + }, + }, + { + "Name": "KnownBadInputs", + "Statement": { + "ManagedRuleGroupStatement": { + "Name": "AWSManagedRulesKnownBadInputsRuleSet" + } + }, + }, + { + "Name": "SizeConstraint", + "Statement": { + "SizeConstraintStatement": { + "FieldToMatch": {"Body": {}}, + "ComparisonOperator": "LE", + "Size": 8192, + "TextTransformations": [{"Priority": 0, "Type": "NONE"}], + } + }, + }, + ], +} + + +def _acl_detail(name: str, acl_id: str) -> dict: + """Return a get_web_acl-style response for the given name/id.""" + detail = dict(_ACL_DETAIL_TEMPLATE) + detail["Name"] = name + detail["Id"] = acl_id + return {"WebACL": detail} + + +# =========================================================================== +# Mock builder +# =========================================================================== + + +def _build_mock_client( + lambda_functions=None, + guardrail_summaries=None, + guardrail_detail=None, + kb_summaries=None, + kb_data_source_summaries=None, + kb_data_source_detail=None, + s3_buckets=None, + acl_summaries=None, + acl_detail_fn=None, +): + """Return a boto3.client side-effect that satisfies every AWS call made + by lambda_handler for the supplied account state. + + Non-inventory calls (shield, sts, apigw, cloudwatch, etc.) return minimal + valid responses so those non-inventory checks produce deterministic output. + """ + if lambda_functions is None: + lambda_functions = _LAMBDA_FUNCTIONS + if guardrail_summaries is None: + guardrail_summaries = [_GUARDRAIL_SUMMARY] + if guardrail_detail is None: + guardrail_detail = _GUARDRAIL_DETAIL + if kb_summaries is None: + kb_summaries = [_KB_SUMMARY] + if kb_data_source_summaries is None: + kb_data_source_summaries = [_KB_DATA_SOURCE_SUMMARY] + if kb_data_source_detail is None: + kb_data_source_detail = _KB_DATA_SOURCE_DETAIL + if s3_buckets is None: + s3_buckets = _S3_BUCKETS + if acl_summaries is None: + acl_summaries = _ACL_SUMMARIES + if acl_detail_fn is None: + acl_detail_fn = lambda name, acl_id: _acl_detail(name, acl_id) # noqa: E731 + + def side_effect(service, **kwargs): # noqa: C901 (complexity OK for test helper) + c = MagicMock() + + # ------------------------------------------------------------------ # + # wafv2 + # ------------------------------------------------------------------ # + if service == "wafv2": + c.list_web_acls.return_value = {"WebACLs": acl_summaries} + + def get_web_acl(Name, Scope, Id, **kw): + return acl_detail_fn(Name, Id) + + c.get_web_acl.side_effect = get_web_acl + return c + + # ------------------------------------------------------------------ # + # shield + # ------------------------------------------------------------------ # + if service == "shield": + c.describe_subscription.return_value = {} # shield advanced active + return c + + # ------------------------------------------------------------------ # + # lambda + # ------------------------------------------------------------------ # + if service == "lambda": + c.list_functions.return_value = { + "Functions": lambda_functions + # No NextMarker → single page + } + + def get_function_concurrency(FunctionName, **kw): + # agent function has concurrency set + if "agent" in FunctionName.lower(): + return {"ReservedConcurrentExecutions": 5} + return {} + + c.get_function_concurrency.side_effect = get_function_concurrency + return c + + # ------------------------------------------------------------------ # + # bedrock (guardrails) + # ------------------------------------------------------------------ # + if service == "bedrock": + c.list_guardrails.return_value = { + "guardrails": guardrail_summaries + # No nextToken → single page + } + c.get_guardrail.return_value = guardrail_detail + + # Non-inventory calls for other bedrock checks + c.list_foundation_models.return_value = { + "modelSummaries": [ + { + "modelId": "anthropic.claude-3-sonnet-20240229-v1:0", + "modelLifecycle": {"status": "ACTIVE"}, + } + ] + } + c.list_custom_models.return_value = {"modelSummaries": []} + c.list_model_cards.return_value = {"ModelCardSummaries": []} + c.list_evaluation_jobs.return_value = {"jobSummaries": []} + c.get_macie_session.side_effect = Exception("macie not enabled") + c.list_automated_reasoning_policies.return_value = { + "automatedReasoningPolicySummaries": [] + } + return c + + # ------------------------------------------------------------------ # + # bedrock-agent (knowledge bases + data sources) + # ------------------------------------------------------------------ # + if service == "bedrock-agent": + # The collector calls list_knowledge_bases directly via _paginate + # (not via get_paginator). We set up both the direct-call return + # value AND the paginator so any remaining checks that still use + # get_paginator also work. + + kb_paginator = MagicMock() + kb_paginator.paginate.return_value = [ + {"knowledgeBaseSummaries": kb_summaries} + ] + + def get_paginator(op): + if op == "list_knowledge_bases": + return kb_paginator + pag = MagicMock() + pag.paginate.return_value = [{}] + return pag + + c.get_paginator.side_effect = get_paginator + + # Direct method call used by _paginate inside collect_resource_inventory + c.list_knowledge_bases.return_value = { + "knowledgeBaseSummaries": kb_summaries + # No nextToken → single page + } + + # list_data_sources called via _paginate (which uses the direct method) + c.list_data_sources.return_value = { + "dataSourceSummaries": kb_data_source_summaries + } + + # get_data_source for FS-33 / FS-65 + c.get_data_source.return_value = kb_data_source_detail + + # list_agents (for FS-07) + c.list_agents.return_value = {"agentSummaries": []} + return c + + # ------------------------------------------------------------------ # + # s3 + # ------------------------------------------------------------------ # + if service == "s3": + c.list_buckets.return_value = { + "Buckets": s3_buckets + # No ContinuationToken → single page + } + + def get_bucket_versioning(Bucket, **kw): + # All buckets have versioning enabled → FS-21 PASS, FS-33 PASS + return {"Status": "Enabled"} + + c.get_bucket_versioning.side_effect = get_bucket_versioning + + def get_bucket_tagging(Bucket, **kw): + # All buckets have classification tag → FS-46 PASS + return { + "TagSet": [{"Key": "data-classification", "Value": "Confidential"}] + } + + c.get_bucket_tagging.side_effect = get_bucket_tagging + + # Bucket notification config for FS-65 + def get_bucket_notification_configuration(Bucket, **kw): + return { + "EventBridgeConfiguration": {} # event notifications present + } + + c.get_bucket_notification_configuration.side_effect = ( + get_bucket_notification_configuration + ) + return c + + # ------------------------------------------------------------------ # + # Non-inventory services — return minimal responses so checks complete + # ------------------------------------------------------------------ # + + if service == "apigateway": + c.get_usage_plans.return_value = { + "items": [ + { + "name": "default", + "throttle": {"rateLimit": 500, "burstLimit": 200}, + } + ] + } + c.get_rest_apis.return_value = {"items": []} + return c + + if service == "ce": # cost explorer + c.get_anomaly_monitors.return_value = { + "AnomalyMonitors": [ + { + "MonitorType": "DIMENSIONAL", + "MonitorDimension": "SERVICE", + "MonitorSpecification": {}, + } + ] + } + return c + + if service == "cloudwatch": + pag = MagicMock() + pag.paginate.return_value = [ + { + "MetricAlarms": [ + { + "AlarmName": "bedrock-throttle-alarm", + "Namespace": "AWS/Bedrock", + "MetricName": "InvocationThrottles", + } + ] + } + ] + c.get_paginator.return_value = pag + return c + + if service == "budgets": + pag = MagicMock() + pag.paginate.return_value = [ + { + "Budgets": [ + { + "BudgetName": "bedrock-spend", + "CostFilters": {"Service": ["Amazon Bedrock"]}, + } + ] + } + ] + c.get_paginator.return_value = pag + return c + + if service == "sts": + c.get_caller_identity.return_value = {"Account": "123456789012"} + return c + + if service == "service-quotas": + applied_pag = MagicMock() + applied_pag.paginate.return_value = [ + { + "Quotas": [ + { + "QuotaName": "On-demand InvokeModel tokens per minute for anthropic.claude", + "QuotaCode": "L-TPMTEST", + "Value": 200000, + } + ] + } + ] + defaults_pag = MagicMock() + defaults_pag.paginate.return_value = [ + {"Quotas": [{"QuotaCode": "L-TPMTEST", "Value": 100000}]} + ] + + def get_paginator(op): + if op == "list_service_quotas": + return applied_pag + if op == "list_aws_default_service_quotas": + return defaults_pag + p = MagicMock() + p.paginate.return_value = [{}] + return p + + c.get_paginator.side_effect = get_paginator + return c + + if service == "stepfunctions": + c.list_state_machines.return_value = {"stateMachines": []} + return c + + if service == "iam": + c.list_policies.return_value = {"Policies": []} + pag = MagicMock() + pag.paginate.return_value = [{}] + c.get_paginator.return_value = pag + return c + + if service == "config": + c.describe_config_rules.return_value = {"ConfigRules": []} + return c + + if service == "ecr": + c.describe_repositories.return_value = {"repositories": []} + return c + + if service == "sagemaker": + c.list_feature_groups.return_value = {"FeatureGroupSummaries": []} + c.list_processing_jobs.return_value = {"ProcessingJobSummaries": []} + c.list_models.return_value = {"Models": []} + pag = MagicMock() + pag.paginate.return_value = [{}] + c.get_paginator.return_value = pag + return c + + if service == "logs": + c.list_log_groups.return_value = {"logGroups": []} + pag = MagicMock() + pag.paginate.return_value = [{"logGroups": []}] + c.get_paginator.return_value = pag + return c + + if service == "macie2": + c.get_macie_session.side_effect = Exception("macie not enabled") + return c + + if service == "opensearchserverless": + c.list_security_policies.return_value = {"securityPolicySummaries": []} + return c + + if service == "bedrock-agent-runtime": + return c + + if service == "bedrock-runtime": + return c + + if service == "events": + c.list_rules.return_value = {"Rules": []} + return c + + if service == "scheduler": + c.list_schedules.return_value = {"Schedules": []} + return c + + if service == "agentcore": + c.list_agent_runtimes.return_value = {"agentRuntimes": []} + return c + + if service == "organizations": + c.list_policies.return_value = {"Policies": []} + return c + + # Catch-all: return a generic mock for any other service + pag = MagicMock() + pag.paginate.return_value = [{}] + c.get_paginator.return_value = pag + return c + + return side_effect + + +# =========================================================================== +# Shared pytest fixture +# =========================================================================== + + +@pytest.fixture +def account_state_mock(): + """Full single-page account state mock covering all inventory-consuming checks.""" + return _build_mock_client() + + +@pytest.fixture +def lambda_event(): + return {"Execution": {"Name": "equivalence-test-001"}} + + +# =========================================================================== +# Helper: run lambda_handler and extract the frozen tuple sequence +# =========================================================================== + + +def _enum_val(v) -> str: + """Return the .value of an enum, or the value unchanged if it's already a str.""" + return v.value if hasattr(v, "value") else str(v) + + +def _run_handler_and_extract_tuples(mock_client_side_effect, event): + """Run lambda_handler with the given mock and return the ordered list of + (check_id, finding_name, status, severity, compliance_frameworks) tuples. + + Enum fields (Status, Severity) are coerced to their .value strings so the + frozen BASELINE literal uses plain str values, making it readable and + portable across Python versions. + """ + with ( + patch("app.boto3.client") as mock_client, + patch("app.get_permissions_cache") as mock_cache, + patch("app.write_to_s3") as mock_s3, + ): + mock_client.side_effect = mock_client_side_effect + mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} + mock_s3.return_value = "https://test-bucket.s3.amazonaws.com/finserv_security_report_equivalence-test-001.csv" + + result = app.lambda_handler(event, None) + + tuples = [] + for finding in result["body"]["findings"]: + for row in finding.get("csv_data", []): + tuples.append( + ( + row.get("Check_ID", ""), + row.get("Finding", ""), + _enum_val(row.get("Status", "")), + _enum_val(row.get("Severity", "")), + row.get("Compliance_Frameworks", ""), + ) + ) + return tuples + + +# =========================================================================== +# FROZEN BASELINE +# =========================================================================== +# Generated by running lambda_handler against the account-state fixture above +# on the PRE-REFACTOR code. After the refactor Task 13 asserts this literal +# is reproduced exactly (REQ-5.3). +# +# NOTE: This literal is intentionally left empty here and will be populated +# once by the _test_generate_baseline helper below (run once, paste output, +# then use BASELINE in the main assertion test). +# =========================================================================== + + +# fmt: off +BASELINE: list[tuple[str, str, str, str, str]] = [ + ('FS-01', 'AWS Shield Advanced Enabled', 'Passed', 'Low', 'FFIEC CAT | DORA Art.6'), + ('FS-01', 'Regional WAF Web ACLs Present', 'Passed', 'Medium', 'FFIEC CAT | DORA Art.6'), + ('FS-02', 'API Gateway Rate Limiting Configured', 'Passed', 'Medium', 'FFIEC CAT | DORA Art.6 | PCI-DSS 12.3.2'), + ('FS-03', 'Bedrock Token Quotas Customized', 'Passed', 'Medium', 'FFIEC CAT | SR 11-7'), + ('FS-04', 'Cost Anomaly Detection Configured', 'Passed', 'Medium', 'FFIEC CAT | SR 11-7'), + ('FS-05', 'Bedrock CloudWatch Alarms Present', 'Passed', 'Medium', 'FFIEC CAT | DORA Art.6'), + ('FS-06', 'AI/ML Service Budgets Configured', 'Passed', 'Medium', 'FFIEC CAT | SR 11-7'), + ('FS-07', 'Agent Action Boundary Check', 'N/A', 'Informational', 'SR 11-7 | FFIEC CAT'), + ('FS-08', 'AgentCore Policy Engine Configured', 'Passed', 'High', 'SR 11-7 | MAS TRM 9.1'), + ('FS-09', 'Agent Lambda Concurrency Limits Present', 'Passed', 'Medium', 'FFIEC CAT | SR 11-7'), + ('FS-10', 'Human-in-the-Loop Check \u2014 No Agent Workflows Found', 'N/A', 'Informational', 'SR 11-7 | FFIEC CAT | MAS TRM 9.2'), + ('FS-11', 'No Agent Rate Alarms Found', 'Failed', 'Medium', 'FFIEC CAT | DORA Art.6'), + ('FS-12', 'No Bedrock-Scoped SCPs Found', 'Failed', 'High', 'SR 11-7 | FFIEC CAT | ISO 27001 A.15.2'), + ('FS-13', 'Model Provenance Tags Present', 'Passed', 'Medium', 'SR 11-7 | ISO 27001 A.12.5 | FFIEC CAT'), + ('FS-14', 'No Model Governance Config Rules Found', 'Failed', 'Medium', 'SR 11-7 | FFIEC CAT | ISO 27001 A.15.1'), + ('FS-15', 'No Bedrock Evaluation Jobs Found', 'Failed', 'Medium', 'SR 11-7 | FFIEC CAT | MAS TRM 9.3'), + ('FS-16', 'No ECR Repositories Found', 'N/A', 'Informational', 'ISO 27001 A.12.6 | FFIEC CAT | DORA Art.6'), + ('FS-20', 'No SageMaker Feature Groups Found', 'N/A', 'Informational', 'SR 11-7 | FFIEC CAT'), + ('FS-21', 'Training Data Buckets Have Versioning', 'Passed', 'High', 'SR 11-7 | ISO 27001 A.12.3 | FFIEC CAT'), + ('FS-22', 'Knowledge Base IAM Permissions Look Appropriate', 'Passed', 'High', 'NYDFS 500 | FFIEC CAT | PCI-DSS 12.3.2'), + ('FS-24', 'ADVISORY: Knowledge Base Metadata Filtering \u2014 Manual Review Required', 'N/A', 'Informational', 'NYDFS 500 | FFIEC CAT | PCI-DSS 12.3.2'), + ('FS-25', 'No OpenSearch Serverless Encryption Policies', 'N/A', 'Informational', 'NYDFS 500 | PCI-DSS 3.5 | FFIEC CAT'), + ('FS-26', 'No OpenSearch Serverless Network Policies', 'Failed', 'High', 'NYDFS 500 | FFIEC CAT | PCI-DSS 1.3'), + ('FS-27', 'Contextual Grounding Enabled on Guardrails', 'Passed', 'High', 'SR 11-7 | FFIEC CAT | MAS TRM 9.2'), + ('FS-27', 'No Automated Reasoning Policies Found', 'Failed', 'Medium', 'SR 11-7 | FFIEC CAT | MAS TRM 9.2'), + ('FS-28', 'Denied Topics Configured on CLASSIC Tier', 'Passed', 'High', 'SR 11-7 | FFIEC CAT | NYDFS 500 | MAS TRM 9.2'), + ('FS-29', 'ADVISORY: Compliance Disclaimer \u2014 Manual Review Required', 'N/A', 'Informational', 'SR 11-7 | FFIEC CAT | NYDFS 500 | MAS TRM 9.2'), + ('FS-30', 'ADVISORY: Compliance Dataset Coverage \u2014 Manual Review Required', 'N/A', 'Informational', 'SR 11-7 | FFIEC CAT | NYDFS 500'), + ('FS-31', 'Knowledge Base Data Sources Recently Synced', 'Passed', 'Medium', 'SR 11-7 | FFIEC CAT'), + ('FS-32', 'ADVISORY: Source Attribution \u2014 Manual Review Required', 'N/A', 'Informational', 'SR 11-7 | FFIEC CAT | MAS TRM 9.2'), + ('FS-33', 'KB Data Source Buckets Have Versioning', 'Passed', 'Medium', 'SR 11-7 | FFIEC CAT | ISO 27001 A.12'), + ('FS-34', 'Foundation Models Are Current', 'Passed', 'Medium', 'SR 11-7 | FFIEC CAT'), + ('FS-35', 'ADVISORY: Harmful-Content Test Coverage \u2014 Manual Review Required', 'N/A', 'Informational', 'SR 11-7 | FFIEC CAT'), + ('FS-36', 'Guardrail Content Filters on CLASSIC Tier', 'Passed', 'High', 'SR 11-7 | FFIEC CAT'), + ('FS-37', 'ADVISORY: User Feedback Mechanism \u2014 Manual Review Required', 'N/A', 'Informational', 'SR 11-7 | FFIEC CAT | MAS TRM 9.2'), + ('FS-38', 'Guardrail Word Filters Configured', 'Passed', 'Medium', 'SR 11-7 | FFIEC CAT'), + ('FS-39', 'No SageMaker Clarify Bias Monitoring', 'Failed', 'High', 'SR 11-7 | FFIEC CAT | ECOA/Fair Housing'), + ('FS-40', 'ADVISORY: Bias Dataset Coverage \u2014 Manual Review Required', 'N/A', 'Informational', 'SR 11-7 | FFIEC CAT | ECOA/Fair Housing'), + ('FS-41', 'No SageMaker Clarify Explainability Monitoring', 'Failed', 'High', 'SR 11-7 | FFIEC CAT'), + ('FS-42', 'No SageMaker Model Cards Found', 'Failed', 'Medium', 'SR 11-7 | FFIEC CAT'), + ('FS-43', 'CloudWatch Logs Data Protection Policies Present', 'Passed', 'High', 'NYDFS 500 | FFIEC CAT | PCI-DSS'), + ('FS-44', 'COULD NOT ASSESS: Amazon Macie PII Scanning Check', 'N/A', 'Low', 'NYDFS 500 | FFIEC CAT | PCI-DSS'), + ('FS-45', 'Guardrail PII Filters Configured', 'Passed', 'High', 'NYDFS 500 | FFIEC CAT | PCI-DSS'), + ('FS-46', 'AI/ML Buckets Have Classification Tags', 'Passed', 'Medium', 'NYDFS 500 | FFIEC CAT | ISO 27001 A.12 | PCI-DSS'), + ('FS-47', 'Guardrail Grounding Thresholds Appropriate', 'Passed', 'High', 'SR 11-7 | FFIEC CAT'), + ('FS-48', 'No Active Knowledge Bases for RAG', 'Failed', 'Medium', 'SR 11-7 | FFIEC CAT'), + ('FS-49', 'ADVISORY: Hallucination Disclaimer \u2014 Manual Review Required', 'N/A', 'Informational', 'SR 11-7 | FFIEC CAT | MAS TRM 9.2'), + ('FS-50', 'Relevance Grounding Filters Present', 'Passed', 'Medium', 'SR 11-7 | FFIEC CAT'), + ('FS-51', 'No Guardrails With Prompt Attack Filters', 'Failed', 'High', 'NYDFS 500 | FFIEC CAT | OWASP LLM Top 10'), + ('FS-52', 'Bedrock Lambda Functions on Current Runtimes', 'Passed', 'Medium', 'NYDFS 500 | FFIEC CAT | ISO 27001 A.12 | OWASP LLM Top 10'), + ('FS-53', 'WAF Injection Protection Rules Present', 'Passed', 'High', 'NYDFS 500 | PCI-DSS | FFIEC CAT | OWASP LLM Top 10'), + ('FS-54', 'ADVISORY: Penetration Testing \u2014 Manual Review Required', 'N/A', 'Informational', 'NYDFS 500 | FFIEC CAT | OWASP LLM Top 10'), + ('FS-55', 'Output Validation Functions Present', 'Passed', 'Medium', 'FFIEC CAT | OWASP LLM Top 10'), + ('FS-56', 'XSS Prevention Common Rule Set Present', 'Passed', 'Medium', 'NYDFS 500 | PCI-DSS | OWASP LLM Top 10'), + ('FS-57', 'ADVISORY: Output Encoding \u2014 Manual Review Required', 'N/A', 'Informational', 'NYDFS 500.06 | FFIEC CAT | OWASP LLM Top 10'), + ('FS-58', 'ADVISORY: Output Schema Validation \u2014 Manual Review Required', 'N/A', 'Informational', 'FFIEC CAT | OWASP LLM Top 10'), + ('FS-59', 'Topic Restrictions Configured on CLASSIC Tier', 'Passed', 'Medium', 'SR 11-7 | FFIEC CAT'), + ('FS-60', 'ADVISORY: Contextual Grounding for Off-Topic Prevention', 'N/A', 'Informational', 'SR 11-7 | FFIEC CAT'), + ('FS-61', 'No Automated KB Sync Schedules Detected', 'Failed', 'Medium', 'SR 11-7 | FFIEC CAT'), + ('FS-62', 'ADVISORY: Data Currency Disclaimer \u2014 Manual Review Required', 'N/A', 'Informational', 'SR 11-7 | FFIEC CAT | MAS TRM 9.2'), + ('FS-63', 'Foundation Model Lifecycle Management', 'Passed', 'Medium', 'SR 11-7 | FFIEC CAT | ISO 27001 A.12'), + ('FS-65', 'KB Data Source Buckets Missing S3 Event Notifications', 'Failed', 'Medium', 'FFIEC CAT | DORA Art.6 | ISO 27001 A.12'), + ('FS-66', 'AgentCore End-User Identity Propagation Configured', 'Passed', 'High', 'NYDFS 500 | SR 11-7 | MAS TRM 9 | PCI-DSS'), + ('FS-67', 'Agent Action-Group Lambdas May Lack Transaction Thresholds', 'Failed', 'High', 'SR 11-7 | FFIEC CAT | MAS TRM 9 | PCI-DSS'), + ('FS-68', 'API Gateway Request Body Size Limits Configured', 'Passed', 'Medium', 'DORA Art.6 | FFIEC CAT | PCI-DSS | OWASP LLM Top 10'), + ('FS-69', 'Prompt Input Validation Functions Present', 'Passed', 'Medium', 'NYDFS 500 | FFIEC CAT | OWASP LLM Top 10'), +] +# fmt: on + + +# =========================================================================== +# REQ-5.4: Documented permitted deltas for multi-page account states +# =========================================================================== +# +# The BASELINE above was captured against a SINGLE-PAGE account state +# (≤100 WAFv2 REGIONAL Web ACLs, ≤1 page of S3 buckets). For that state +# the post-refactor handler output MUST be byte-for-byte identical to BASELINE +# (REQ-5.3 / test_handler_output_matches_frozen_baseline). +# +# For multi-page account states, REQ-5.4 permits EXACTLY TWO classes of +# finding-set changes — both attributable to pagination-correctness fixes: +# +# PERMITTED DELTA 1 — WAFv2 Web ACLs (REQ-2.2): +# Pre-refactor: wafv2.list_web_acls(Scope="REGIONAL") was called UNPAGINATED, +# silently truncating at one page (≤100 ACLs). Accounts with >100 REGIONAL +# Web ACLs would have their ACL set incorrectly truncated. +# Post-refactor: _paginate uses NextMarker/NextMarker token convention, so +# ALL ACLs are returned. Any finding rows attributable to ACLs beyond page 1 +# (e.g., FS-01, FS-53, FS-56, FS-68) that were previously invisible MAY now +# appear. Existing rows for ACLs within page 1 remain identical. +# +# PERMITTED DELTA 2 — S3 buckets (REQ-2.8): +# Pre-refactor: s3.list_buckets() was called UNPAGINATED, silently truncating +# at 1,000 buckets in standard accounts; outright FAILING (error → COULD_NOT_ASSESS) +# for accounts whose bucket quota is above 10,000. +# Post-refactor: _paginate uses ContinuationToken/ContinuationToken with +# MaxBuckets=1000, so ALL buckets are enumerated and 10k-quota accounts succeed. +# Finding rows for FS-21 and FS-46 may differ for buckets beyond page 1, and +# accounts that previously produced COULD_NOT_ASSESS may now produce PASS/FAIL. +# +# ALL OTHER finding changes are defects (REQ-5.5) and must be corrected. +# =========================================================================== + + +# =========================================================================== +# Tests +# =========================================================================== + + +class TestGoldenEquivalenceBaseline: + """Golden-equivalence oracle for the current (pre-refactor) lambda_handler. + + Requirements: REQ-5.1, REQ-5.3, REQ-5.6 + """ + + def test_handler_produces_65_findings(self, account_state_mock, lambda_event): + """Smoke test: all 65 registry entries produce at least one finding.""" + with ( + patch("app.boto3.client") as mock_client, + patch("app.get_permissions_cache") as mock_cache, + patch("app.write_to_s3") as mock_s3, + ): + mock_client.side_effect = account_state_mock + mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} + mock_s3.return_value = "https://test-bucket.s3.amazonaws.com/report.csv" + + result = app.lambda_handler(lambda_event, None) + + assert result["statusCode"] == 200 + findings = result["body"]["findings"] + assert len(findings) == 65, f"Expected 65 registry entries; got {len(findings)}" + + def test_all_findings_have_rows(self, account_state_mock, lambda_event): + """Every registry entry emits at least one CSV row (no silent drops).""" + with ( + patch("app.boto3.client") as mock_client, + patch("app.get_permissions_cache") as mock_cache, + patch("app.write_to_s3") as mock_s3, + ): + mock_client.side_effect = account_state_mock + mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} + mock_s3.return_value = "https://test-bucket.s3.amazonaws.com/report.csv" + + result = app.lambda_handler(lambda_event, None) + + for finding in result["body"]["findings"]: + assert finding.get("csv_data"), ( + f"Finding '{finding.get('check_name')}' produced no CSV rows" + ) + + def test_fixture_covers_pass_paths(self, account_state_mock, lambda_event): + """The fixture exercises PASS paths for the inventory-consuming checks.""" + with ( + patch("app.boto3.client") as mock_client, + patch("app.get_permissions_cache") as mock_cache, + patch("app.write_to_s3") as mock_s3, + ): + mock_client.side_effect = account_state_mock + mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} + mock_s3.return_value = "https://test-bucket.s3.amazonaws.com/report.csv" + + result = app.lambda_handler(lambda_event, None) + + all_rows = [ + row for f in result["body"]["findings"] for row in f.get("csv_data", []) + ] + passed_ids = {r["Check_ID"] for r in all_rows if r.get("Status") == "Passed"} + + # These checks should produce PASS rows with our fixture + expected_pass = {"FS-01", "FS-09", "FS-21", "FS-27", "FS-28", "FS-33", "FS-45"} + for check_id in expected_pass: + assert check_id in passed_ids, ( + f"Expected PASS for {check_id} but not found in {passed_ids}" + ) + + def test_fixture_covers_na_paths(self, account_state_mock, lambda_event): + """The fixture exercises N/A paths (advisory checks emit Status='N/A').""" + with ( + patch("app.boto3.client") as mock_client, + patch("app.get_permissions_cache") as mock_cache, + patch("app.write_to_s3") as mock_s3, + ): + mock_client.side_effect = account_state_mock + mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} + mock_s3.return_value = "https://test-bucket.s3.amazonaws.com/report.csv" + + result = app.lambda_handler(lambda_event, None) + + all_rows = [ + row for f in result["body"]["findings"] for row in f.get("csv_data", []) + ] + na_ids = {r["Check_ID"] for r in all_rows if r.get("Status") == "N/A"} + + # Advisory/N/A checks should appear + expected_na = {"FS-24", "FS-29", "FS-30", "FS-32", "FS-54", "FS-57", "FS-60"} + for check_id in expected_na: + assert check_id in na_ids, ( + f"Expected N/A for {check_id} but not found in {na_ids}" + ) + + def test_fixture_covers_fail_paths(self, lambda_event): + """The fixture exercises FAIL paths for select inventory-consuming checks.""" + # Use a variant fixture where guardrail has NO PII filters → FS-45 FAIL + guardrail_without_pii = dict(_GUARDRAIL_DETAIL) + guardrail_without_pii = { + k: v + for k, v in _GUARDRAIL_DETAIL.items() + if k != "sensitiveInformationPolicy" + } + guardrail_without_pii["sensitiveInformationPolicy"] = {} + + fail_mock = _build_mock_client(guardrail_detail=guardrail_without_pii) + + with ( + patch("app.boto3.client") as mock_client, + patch("app.get_permissions_cache") as mock_cache, + patch("app.write_to_s3") as mock_s3, + ): + mock_client.side_effect = fail_mock + mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} + mock_s3.return_value = "https://test-bucket.s3.amazonaws.com/report.csv" + + result = app.lambda_handler(lambda_event, None) + + all_rows = [ + row for f in result["body"]["findings"] for row in f.get("csv_data", []) + ] + failed_ids = {r["Check_ID"] for r in all_rows if r.get("Status") == "Failed"} + assert "FS-45" in failed_ids, ( + "Expected FS-45 FAIL when guardrail has no PII filters" + ) + + def test_baseline_tuple_count_is_stable(self, account_state_mock, lambda_event): + """Capture the pre-refactor tuples and assert the count is deterministic. + + The frozen BASELINE literal is populated by test_capture_and_freeze_baseline + below. This test asserts the count (and therefore structure) is stable + across runs, which is the prerequisite for freezing it as a literal. + """ + t1 = _run_handler_and_extract_tuples(account_state_mock, lambda_event) + t2 = _run_handler_and_extract_tuples( + _build_mock_client(), + lambda_event, # fresh instance, same state + ) + assert t1 == t2, ( + "lambda_handler is non-deterministic: two identical runs produced different tuples" + ) + assert len(t1) > 0, "No tuples extracted — fixture may be broken" + + def test_handler_output_matches_frozen_baseline( + self, account_state_mock, lambda_event + ): + """Golden-equivalence oracle: the handler output must equal BASELINE exactly. + + REQ-5.1, REQ-5.3, REQ-5.6 — This test is the live oracle that will remain + green after the refactor in Tasks 2–15. It asserts that: + 1. The (check_id, finding_name, status, severity, compliance_frameworks) + tuples are identical to the pre-refactor frozen literal. + 2. Row ordering is preserved (CSV row ordering per REQ-5.6). + 3. The BASELINE covers ≤100 Web ACLs and ≤1 page of S3 buckets so it + remains directly comparable after the pagination fix (REQ-5.3). + """ + actual = _run_handler_and_extract_tuples(account_state_mock, lambda_event) + assert actual == BASELINE, ( + f"Handler output differs from frozen BASELINE.\n" + f"Expected {len(BASELINE)} tuples, got {len(actual)}.\n" + f"First diff at index " + + str( + next( + (i for i, (a, b) in enumerate(zip(actual, BASELINE)) if a != b), + min(len(actual), len(BASELINE)), + ) + ) + + "." + ) + + +class TestPermittedMultiPageDeltas: + """Validates the two permitted finding-set changes for multi-page account states. + + REQ-5.4: findings may ONLY change for resources beyond page 1 of WAFv2 Web + ACLs (>100 ACLs) or S3 buckets (>1 page / >10k quota). All other rows + must remain identical to the single-page BASELINE. + + These tests use two-page fixtures so we can verify: + 1. Rows attributable to page-1 resources are identical to BASELINE. + 2. Rows attributable to page-2+ resources appear (new, post-refactor only). + """ + + def test_wafv2_multi_page_page1_rows_unchanged(self, lambda_event): + """REQ-5.4 WAFv2 delta: finding changes attributable ONLY to page-2+ ACLs are permitted. + + Scenario: page 1 = well-configured ACLs (same as BASELINE), page 2 = one ACL + MISSING injection-protection rules. Pre-refactor: that ACL was silently + truncated so checks passed (BASELINE). Post-refactor: the ACL is seen and + checks correctly fail. + + This is the ONLY class of WAFv2 finding change REQ-5.4 permits. + """ + # Extra ACL on page 2 — missing injection/XSS rules → FS-53, FS-56 should now FAIL + extra_acl = { + "Name": "BadACL", + "Id": "acl-id-999", + "ARN": "arn:aws:wafv2:::acl-id-999", + } + bad_acl_detail = { + "WebACL": { + "Name": "BadACL", + "Id": "acl-id-999", + "Rules": [], # No managed rule groups — missing injection protection + } + } + + def mock_with_two_page_wafv2(service, **kwargs): + c = _build_mock_client()(service, **kwargs) + if service == "wafv2": + c_waf = MagicMock() + call_count = {"n": 0} + + def list_web_acls_two_pages(**kw): + call_count["n"] += 1 + if call_count["n"] == 1: + return {"WebACLs": _ACL_SUMMARIES, "NextMarker": "page2token"} + return {"WebACLs": [extra_acl]} + + c_waf.list_web_acls.side_effect = list_web_acls_two_pages + + def get_web_acl(Name, Scope, Id, **kw): + if Id == "acl-id-999": + return bad_acl_detail + return _acl_detail(Name, Id) + + c_waf.get_web_acl.side_effect = get_web_acl + return c_waf + return c + + with ( + patch("app.boto3.client") as mock_client, + patch("app.get_permissions_cache") as mock_cache, + patch("app.write_to_s3"), + ): + mock_client.side_effect = mock_with_two_page_wafv2 + mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} + result = app.lambda_handler(lambda_event, None) + + actual = [] + for finding in result["body"]["findings"]: + for row in finding.get("csv_data", []): + actual.append( + ( + row.get("Check_ID", ""), + row.get("Finding", ""), + _enum_val(row.get("Status", "")), + _enum_val(row.get("Severity", "")), + row.get("Compliance_Frameworks", ""), + ) + ) + + actual_by_id = {t[0]: t for t in actual} + + # REQ-5.4: WAFv2 checks (FS-53, FS-56) MUST now fail because the page-2 ACL + # is missing injection/XSS rules. This is the PERMITTED delta. + assert actual_by_id.get("FS-53", ("", "", "", "", ""))[2] == "Failed", ( + "FS-53 must FAIL when a page-2 ACL has no injection-protection rules" + ) + assert actual_by_id.get("FS-56", ("", "", "", "", ""))[2] == "Failed", ( + "FS-56 must FAIL when a page-2 ACL has no XSS rules" + ) + + # REQ-5.4: All non-WAFv2 check rows are UNCHANGED (no collateral damage) + wafv2_check_ids = {"FS-01", "FS-53", "FS-56", "FS-68"} + for row in BASELINE: + check_id = row[0] + if check_id in wafv2_check_ids: + continue # WAFv2 rows are the permitted delta — skip + assert row in actual, ( + f"REQ-5.4 violation: non-WAFv2 BASELINE row {row!r} missing from output" + ) + + def test_s3_multi_page_page1_rows_unchanged(self, lambda_event): + """REQ-5.4 S3 delta: page-1 bucket rows must be identical to BASELINE. + + Uses a two-page S3 fixture (page 1 = the fixture buckets, page 2 = one + additional bucket). Asserts that the page-1 bucket findings are identical + to the single-page BASELINE. + """ + extra_bucket = {"Name": "extra-ml-bucket-page2"} + + def mock_with_two_page_s3(service, **kwargs): + c = _build_mock_client()(service, **kwargs) + if service == "s3": + c_s3 = MagicMock() + call_count = {"n": 0} + + def list_buckets_two_pages(**kw): + call_count["n"] += 1 + if call_count["n"] == 1: + return { + "Buckets": _S3_BUCKETS, + "ContinuationToken": "page2token", + } + return {"Buckets": [extra_bucket]} + + c_s3.list_buckets.side_effect = list_buckets_two_pages + + def get_bucket_versioning(Bucket, **kw): + return {"Status": "Enabled"} + + c_s3.get_bucket_versioning.side_effect = get_bucket_versioning + + def get_bucket_tagging(Bucket, **kw): + return { + "TagSet": [ + {"Key": "data-classification", "Value": "Confidential"} + ] + } + + c_s3.get_bucket_tagging.side_effect = get_bucket_tagging + + def get_bucket_notification_configuration(Bucket, **kw): + return {"EventBridgeConfiguration": {}} + + c_s3.get_bucket_notification_configuration.side_effect = ( + get_bucket_notification_configuration + ) + return c_s3 + return c + + with ( + patch("app.boto3.client") as mock_client, + patch("app.get_permissions_cache") as mock_cache, + patch("app.write_to_s3"), + ): + mock_client.side_effect = mock_with_two_page_s3 + mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} + result = app.lambda_handler(lambda_event, None) + + actual = [] + for finding in result["body"]["findings"]: + for row in finding.get("csv_data", []): + actual.append( + ( + row.get("Check_ID", ""), + row.get("Finding", ""), + _enum_val(row.get("Status", "")), + _enum_val(row.get("Severity", "")), + row.get("Compliance_Frameworks", ""), + ) + ) + + # All BASELINE rows must still appear — delta is ADDITIVE only for extra buckets + for row in BASELINE: + assert row in actual, ( + f"REQ-5.4 violation: BASELINE row {row!r} missing from multi-page S3 output" + ) + + # Must have at least as many rows as BASELINE (extra bucket may add rows) + assert len(actual) >= len(BASELINE) + + +class TestCaptureAndFreezeBaseline: + """Helper test that prints the frozen literal for copy/paste into BASELINE. + + Run once with: pytest tests/test_inventory_equivalence.py::TestCaptureAndFreezeBaseline -s + Then copy the printed literal into the BASELINE constant above and delete this class + (or keep it as a regeneration helper). + """ + + def test_print_baseline_literal(self, account_state_mock, lambda_event, capsys): + """Print the BASELINE literal so it can be pasted into the BASELINE constant.""" + tuples = _run_handler_and_extract_tuples(account_state_mock, lambda_event) + + with capsys.disabled(): + print("\n\n# ===== PASTE INTO BASELINE CONSTANT =====") + print("BASELINE: list[tuple[str, str, str, str, str]] = [") + for t in tuples: + # repr each element so special chars are safe + row_repr = ", ".join(repr(x) for x in t) + print(f" ({row_repr}),") + print("]") + print("# ===== END BASELINE =====\n") + + # The test itself passes as long as tuples are produced + assert len(tuples) > 0 diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_inventory_model.py b/aiml-security-assessment/functions/security/finserv_tests/test_inventory_model.py new file mode 100644 index 0000000..2370af2 --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_tests/test_inventory_model.py @@ -0,0 +1,143 @@ +""" +Unit tests for ResourceInventory data model and accessors. + +Validates: Requirements REQ-4.1, REQ-4.2, REQ-9.2 +""" + +import sys +import os + +import pytest + +# Ensure finserv_assessments is importable +FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") +if FINSERV_DIR not in sys.path: + sys.path.insert(0, FINSERV_DIR) + +import app # noqa: E402 (import follows sys.path bootstrap above) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_minimal_inventory(**overrides): + """Build a minimal ResourceInventory with all-available fields unless + overridden.""" + defaults = dict( + lambda_functions=[], + guardrails=app.GuardrailInventory(summaries=[], detail_by_id={}), + knowledge_bases=app.KbInventory( + summaries=[], data_sources_by_kb={}, data_source_detail={} + ), + buckets=[], + web_acls=app.WebAclInventory(summaries=[], detail_by_id={}), + ) + defaults.update(overrides) + return app.ResourceInventory(**defaults) + + +# --------------------------------------------------------------------------- +# Tests for `require` +# --------------------------------------------------------------------------- + + +class TestRequire: + def test_raises_runtime_error_when_inventory_is_none(self): + """require(None, ...) always raises RuntimeError (test-only default path).""" + with pytest.raises(RuntimeError, match="resource inventory not provided"): + app.require(None, "lambda_functions") + + def test_reraises_stored_error_when_field_is_unavailable(self): + """require re-raises the exact exception stored in _Unavailable.""" + original_err = PermissionError("AccessDenied: list_functions denied") + inv = _make_minimal_inventory(lambda_functions=app._Unavailable(original_err)) + with pytest.raises(PermissionError) as exc_info: + app.require(inv, "lambda_functions") + assert exc_info.value is original_err + + def test_reraises_stored_error_preserves_type(self): + """The re-raised error has the same type as the stored one.""" + err = ValueError("boom") + inv = _make_minimal_inventory(buckets=app._Unavailable(err)) + with pytest.raises(ValueError): + app.require(inv, "buckets") + + def test_returns_value_when_field_is_available(self): + """require returns the field's value when it is not an _Unavailable.""" + functions = [{"FunctionName": "my-fn"}] + inv = _make_minimal_inventory(lambda_functions=functions) + result = app.require(inv, "lambda_functions") + assert result is functions + + def test_returns_nested_inventory_when_available(self): + """require works for complex nested types like GuardrailInventory.""" + guardrail_inv = app.GuardrailInventory( + summaries=[{"id": "g1"}], detail_by_id={"g1": {"policy": {}}} + ) + inv = _make_minimal_inventory(guardrails=guardrail_inv) + result = app.require(inv, "guardrails") + assert result is guardrail_inv + + def test_raises_on_any_unavailable_field(self): + """require raises for each inventory field name when it holds _Unavailable.""" + err = RuntimeError("some error") + for field in ( + "lambda_functions", + "guardrails", + "knowledge_bases", + "buckets", + "web_acls", + ): + inv = _make_minimal_inventory(**{field: app._Unavailable(err)}) + with pytest.raises(RuntimeError): + app.require(inv, field) + + +# --------------------------------------------------------------------------- +# Tests for `inv_available` +# --------------------------------------------------------------------------- + + +class TestInvAvailable: + def test_returns_true_for_normal_value(self): + """inv_available returns True for a plain list.""" + assert app.inv_available([]) is True + + def test_returns_true_for_non_empty_list(self): + """inv_available returns True for a populated list.""" + assert app.inv_available([{"FunctionName": "fn"}]) is True + + def test_returns_true_for_nested_dataclass(self): + """inv_available returns True for a GuardrailInventory.""" + gi = app.GuardrailInventory(summaries=[], detail_by_id={}) + assert app.inv_available(gi) is True + + def test_returns_false_for_unavailable(self): + """inv_available returns False for an _Unavailable sentinel.""" + assert app.inv_available(app._Unavailable(Exception("x"))) is False + + def test_returns_false_regardless_of_stored_error(self): + """inv_available is False for _Unavailable regardless of the error type.""" + for err in (ValueError("v"), RuntimeError("r"), Exception("e")): + assert app.inv_available(app._Unavailable(err)) is False + + +# --------------------------------------------------------------------------- +# Tests for _Unavailable sentinel itself +# --------------------------------------------------------------------------- + + +class TestUnavailableSentinel: + def test_stores_error(self): + """_Unavailable stores the given exception on .error.""" + err = IOError("network error") + sentinel = app._Unavailable(err) + assert sentinel.error is err + + def test_is_not_list_or_dict(self): + """_Unavailable is distinguishable from the normal inventory types.""" + sentinel = app._Unavailable(Exception()) + assert not isinstance(sentinel, (list, dict)) + assert not isinstance(sentinel, app.GuardrailInventory) diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_lambda_handler.py b/aiml-security-assessment/functions/security/finserv_tests/test_lambda_handler.py new file mode 100644 index 0000000..af6be0d --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_tests/test_lambda_handler.py @@ -0,0 +1,436 @@ +""" +Integration tests for the lambda_handler in finserv_assessments/app.py + +These tests verify: + - lambda_handler runs end-to-end with all checks mocked + - Response structure (statusCode, body, findings, report_url) + - All 59 standalone check functions are called (5 merged upstream) + - CSV is written to S3 + - Error handling when AIML_ASSESSMENT_BUCKET_NAME is missing + - Error handling when S3 write fails +""" + +import json +import os +import sys +from unittest.mock import MagicMock, patch + +import pytest + +FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") +if FINSERV_DIR not in sys.path: + sys.path.insert(0, FINSERV_DIR) + +import app # noqa: E402 (import must follow sys.path setup above) + + +# ========================================================================= +# Full handler smoke test — all boto3 calls mocked +# ========================================================================= + + +class TestLambdaHandler: + """End-to-end handler tests with fully mocked AWS clients.""" + + @patch("app.write_to_s3") + @patch("app.get_permissions_cache") + @patch("app.boto3.client") + def test_handler_returns_200(self, mock_client, mock_cache, mock_s3, lambda_event): + """Smoke test: handler completes and returns 200.""" + # Return a generic mock for every boto3 client + generic = MagicMock() + # Make paginators return empty pages + paginator = MagicMock() + paginator.paginate.return_value = [{}] + generic.get_paginator.return_value = paginator + # Make list/describe calls return empty collections + generic.list_web_acls.return_value = {"WebACLs": []} + generic.describe_subscription.side_effect = Exception("no shield") + generic.get_usage_plans.return_value = {"items": []} + generic.list_service_quotas.return_value = {"Quotas": []} + generic.get_anomaly_monitors.return_value = {"AnomalyMonitors": []} + generic.describe_budgets.return_value = {"Budgets": []} + generic.get_caller_identity.return_value = {"Account": "123456789012"} + generic.list_agents.return_value = {"agentSummaries": []} + generic.list_agent_runtimes.return_value = {"agentRuntimes": []} + generic.list_functions.return_value = {"Functions": []} + generic.list_state_machines.return_value = {"stateMachines": []} + generic.list_policies.return_value = {"Policies": []} + generic.list_custom_models.return_value = {"modelSummaries": []} + generic.list_models.return_value = {"Models": []} + generic.describe_config_rules.return_value = {"ConfigRules": []} + generic.list_evaluation_jobs.return_value = {"jobSummaries": []} + generic.describe_repositories.return_value = {"repositories": []} + generic.list_feature_groups.return_value = {"FeatureGroupSummaries": []} + generic.list_buckets.return_value = {"Buckets": []} + generic.list_knowledge_bases.return_value = {"knowledgeBaseSummaries": []} + generic.list_guardrails.return_value = {"guardrails": []} + generic.list_log_groups.return_value = {"logGroups": []} + generic.get_macie_session.side_effect = Exception("not enabled") + generic.list_foundation_models.return_value = {"modelSummaries": []} + generic.list_model_cards.return_value = {"ModelCardSummaries": []} + generic.list_rules.return_value = {"Rules": []} + generic.list_schedules.return_value = {"Schedules": []} + generic.get_rest_apis.return_value = {"items": []} + generic.list_processing_jobs.return_value = {"ProcessingJobSummaries": []} + generic.list_automated_reasoning_policies.return_value = { + "automatedReasoningPolicySummaries": [] + } + + mock_client.return_value = generic + mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} + mock_s3.return_value = "https://test-bucket.s3.amazonaws.com/finserv_security_report_unit-test-001.csv" + + result = app.lambda_handler(lambda_event, None) + + assert result["statusCode"] == 200 + assert "findings" in result["body"] + assert "report_url" in result["body"] + assert isinstance(result["body"]["findings"], list) + # The handler runs 65 registry entries (64 standalone checks + the new + # FS-27 ARC policies check that shares the FS-27 check_id). + assert len(result["body"]["findings"]) == 65 + + @patch("app.write_to_s3") + @patch("app.get_permissions_cache") + @patch("app.boto3.client") + def test_handler_findings_all_have_check_name( + self, mock_client, mock_cache, mock_s3, lambda_event + ): + """Every finding dict should have check_name and status keys.""" + generic = MagicMock() + paginator = MagicMock() + paginator.paginate.return_value = [{}] + generic.get_paginator.return_value = paginator + generic.list_web_acls.return_value = {"WebACLs": []} + generic.describe_subscription.side_effect = Exception("no shield") + generic.get_usage_plans.return_value = {"items": []} + generic.list_service_quotas.return_value = {"Quotas": []} + generic.get_anomaly_monitors.return_value = {"AnomalyMonitors": []} + generic.describe_budgets.return_value = {"Budgets": []} + generic.get_caller_identity.return_value = {"Account": "123456789012"} + generic.list_agents.return_value = {"agentSummaries": []} + generic.list_agent_runtimes.return_value = {"agentRuntimes": []} + generic.list_functions.return_value = {"Functions": []} + generic.list_state_machines.return_value = {"stateMachines": []} + generic.list_policies.return_value = {"Policies": []} + generic.list_custom_models.return_value = {"modelSummaries": []} + generic.list_models.return_value = {"Models": []} + generic.describe_config_rules.return_value = {"ConfigRules": []} + generic.list_evaluation_jobs.return_value = {"jobSummaries": []} + generic.describe_repositories.return_value = {"repositories": []} + generic.list_feature_groups.return_value = {"FeatureGroupSummaries": []} + generic.list_buckets.return_value = {"Buckets": []} + generic.list_knowledge_bases.return_value = {"knowledgeBaseSummaries": []} + generic.list_guardrails.return_value = {"guardrails": []} + generic.list_log_groups.return_value = {"logGroups": []} + generic.get_macie_session.side_effect = Exception("not enabled") + generic.list_foundation_models.return_value = {"modelSummaries": []} + generic.list_model_cards.return_value = {"ModelCardSummaries": []} + generic.list_rules.return_value = {"Rules": []} + generic.list_schedules.return_value = {"Schedules": []} + generic.get_rest_apis.return_value = {"items": []} + generic.list_processing_jobs.return_value = {"ProcessingJobSummaries": []} + generic.list_automated_reasoning_policies.return_value = { + "automatedReasoningPolicySummaries": [] + } + + mock_client.return_value = generic + mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} + mock_s3.return_value = "https://test-bucket.s3.amazonaws.com/report.csv" + + result = app.lambda_handler(lambda_event, None) + + for finding in result["body"]["findings"]: + assert "check_name" in finding, f"Missing check_name in {finding}" + assert "status" in finding, f"Missing status in {finding}" + assert finding["status"] in ("PASS", "WARN", "ERROR"), ( + f"Unexpected status '{finding['status']}' in {finding['check_name']}" + ) + assert "csv_data" in finding + + def test_handler_raises_without_bucket_env(self, lambda_event, monkeypatch): + """Handler should raise ValueError when AIML_ASSESSMENT_BUCKET_NAME is unset.""" + monkeypatch.delenv("AIML_ASSESSMENT_BUCKET_NAME", raising=False) + + # We need to mock all boto3 calls so the checks themselves don't fail + with ( + patch("app.boto3.client") as mock_client, + patch("app.get_permissions_cache") as mock_cache, + ): + generic = MagicMock() + paginator = MagicMock() + paginator.paginate.return_value = [{}] + generic.get_paginator.return_value = paginator + # Set up enough mocks for checks to complete + for attr in [ + "list_web_acls", + "get_usage_plans", + "list_service_quotas", + "get_anomaly_monitors", + "describe_budgets", + "list_agents", + "list_agent_runtimes", + "list_functions", + "list_state_machines", + "list_policies", + "list_custom_models", + "list_models", + "describe_config_rules", + "list_evaluation_jobs", + "describe_repositories", + "list_feature_groups", + "list_buckets", + "list_knowledge_bases", + "list_guardrails", + "list_log_groups", + "list_foundation_models", + "list_model_cards", + "list_rules", + "get_rest_apis", + "list_processing_jobs", + ]: + getattr(generic, attr).return_value = ( + {"WebACLs": []} + if "acl" in attr.lower() + else {"items": []} + if "items" in attr.lower() or "rest_api" in attr.lower() + else {next(iter({})): []} + if False # fallback + else {} + ) + generic.describe_subscription.side_effect = Exception("no shield") + generic.get_caller_identity.return_value = {"Account": "123456789012"} + generic.get_macie_session.side_effect = Exception("not enabled") + + mock_client.return_value = generic + mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} + + with pytest.raises(ValueError, match="AIML_ASSESSMENT_BUCKET_NAME"): + app.lambda_handler(lambda_event, None) + + +class TestInventoryCollectedAndPassed: + """Task 5.1 — Assert lambda_handler always collects and passes a real inventory. + + Guards against the default-None footgun: even though build_finserv_checks + accepts inventory=None for backward-compat with the drift-guard, lambda_handler + must always call collect_resource_inventory() and pass its return value. + + Validates: REQ-6.5 + """ + + @patch("app.write_to_s3") + @patch("app.get_permissions_cache") + @patch("app.build_finserv_checks") + @patch("app.collect_resource_inventory") + def test_handler_calls_collect_inventory_exactly_once( + self, + mock_collect, + mock_build, + mock_cache, + mock_s3, + lambda_event, + ): + """lambda_handler calls collect_resource_inventory() exactly once per invocation.""" + fake_inventory = object() # any sentinel — not None + mock_collect.return_value = fake_inventory + mock_build.return_value = [] # no checks to run + mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} + mock_s3.return_value = "https://bucket.s3.amazonaws.com/report.csv" + + app.lambda_handler(lambda_event, None) + + mock_collect.assert_called_once_with() + + @patch("app.write_to_s3") + @patch("app.get_permissions_cache") + @patch("app.build_finserv_checks") + @patch("app.collect_resource_inventory") + def test_handler_passes_inventory_to_build_finserv_checks( + self, + mock_collect, + mock_build, + mock_cache, + mock_s3, + lambda_event, + ): + """The return value of collect_resource_inventory() is passed as inventory= + to build_finserv_checks, and it is never None.""" + fake_inventory = object() # distinct sentinel + mock_collect.return_value = fake_inventory + mock_build.return_value = [] + mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} + mock_s3.return_value = "https://bucket.s3.amazonaws.com/report.csv" + + app.lambda_handler(lambda_event, None) + + # build_finserv_checks must have been called with the real inventory + mock_build.assert_called_once() + call_args, call_kwargs = mock_build.call_args + # inventory can be passed positionally or as a keyword argument + passed_inventory = call_kwargs.get( + "inventory", call_args[1] if len(call_args) > 1 else None + ) + assert passed_inventory is fake_inventory, ( + "lambda_handler must pass the collect_resource_inventory() return value " + f"to build_finserv_checks; got {passed_inventory!r}" + ) + assert passed_inventory is not None, ( + "lambda_handler must never pass None as the inventory argument" + ) + + +class TestWriteToS3: + """Test the write_to_s3 helper.""" + + @patch("app.boto3.client") + def test_writes_csv_to_s3(self, mock_client): + s3 = MagicMock() + mock_client.return_value = s3 + + url = app.write_to_s3("exec-123", "col1,col2\nval1,val2", "my-bucket") + + s3.put_object.assert_called_once_with( + Bucket="my-bucket", + Key="finserv_security_report_exec-123.csv", + Body="col1,col2\nval1,val2", + ContentType="text/csv", + ) + assert "my-bucket" in url + assert "exec-123" in url + + @patch("app.boto3.client") + def test_s3_error_propagates(self, mock_client): + s3 = MagicMock() + s3.put_object.side_effect = RuntimeError("S3 write failed") + mock_client.return_value = s3 + + with pytest.raises(RuntimeError, match="S3 write failed"): + app.write_to_s3("exec-123", "data", "my-bucket") + + +class TestGetPermissionsCache: + """Test the get_permissions_cache helper.""" + + @patch("app.boto3.client") + def test_returns_parsed_json(self, mock_client): + s3 = MagicMock() + body = MagicMock() + body.read.return_value = json.dumps({"role_permissions": {"r1": {}}}).encode() + s3.get_object.return_value = {"Body": body} + mock_client.return_value = s3 + + result = app.get_permissions_cache("exec-123") + assert result == {"role_permissions": {"r1": {}}} + + @patch("app.boto3.client") + def test_returns_none_on_client_error(self, mock_client): + from botocore.exceptions import ClientError + + s3 = MagicMock() + s3.get_object.side_effect = ClientError( + {"Error": {"Code": "NoSuchKey", "Message": "Not found"}}, + "GetObject", + ) + mock_client.return_value = s3 + + result = app.get_permissions_cache("exec-123") + assert result is None + + @patch("app.boto3.client") + def test_returns_none_on_unexpected_error(self, mock_client): + s3 = MagicMock() + s3.get_object.side_effect = RuntimeError("unexpected") + mock_client.return_value = s3 + + result = app.get_permissions_cache("exec-123") + assert result is None + + +# ========================================================================= +# Importability smoke test — all 59 check functions are importable +# ========================================================================= + + +class TestAllCheckFunctionsImportable: + """Verify every check function referenced in lambda_handler is importable.""" + + EXPECTED_CHECK_FUNCTIONS = [ + "check_waf_shield_on_bedrock_endpoints", + "check_api_gateway_rate_limiting", + "check_bedrock_token_quotas", + "check_cost_anomaly_detection", + "check_cloudwatch_token_alarms", + "check_aws_budgets_for_aiml", + "check_bedrock_agent_action_boundaries", + "check_agentcore_policy_engine", + "check_agent_transaction_limits", + "check_human_in_the_loop_for_high_risk_actions", + "check_agent_rate_alarms", + "check_scp_model_access_restrictions", + "check_model_inventory_tagging", + "check_model_onboarding_governance", + "check_bedrock_model_evaluation_adversarial", + "check_ecr_image_scanning", + "check_feature_store_rollback_capability", + "check_training_data_s3_versioning", + "check_knowledge_base_iam_least_privilege", + "check_knowledge_base_metadata_filtering", + "check_opensearch_serverless_encryption", + "check_knowledge_base_vpc_access", + # FS-27 is now two separate functions: contextual grounding + ARC policies + "check_guardrail_contextual_grounding", + "check_automated_reasoning_policies", + "check_guardrail_denied_topics_financial", + "check_compliance_disclaimer_in_outputs", + "check_bedrock_evaluation_compliance_datasets", + "check_knowledge_base_data_source_sync", + "check_source_attribution_in_guardrails", + "check_knowledge_base_integrity_monitoring", + "check_fm_version_currency", + "check_fmeval_harmful_content", + "check_guardrail_content_filters", + "check_user_feedback_mechanism", + "check_guardrail_word_filters", + "check_sagemaker_clarify_bias", + "check_bedrock_evaluation_bias_datasets", + "check_sagemaker_clarify_explainability", + "check_ai_service_cards_documentation", + "check_cloudwatch_log_pii_masking", + "check_macie_on_training_data_buckets", + "check_guardrail_pii_filters", + "check_data_classification_tagging", + "check_guardrail_grounding_threshold", + "check_rag_knowledge_base_configured", + "check_hallucination_disclaimer_advisory", + # FS-50 renamed from check_automated_reasoning_checks_hallucination + "check_guardrail_relevance_grounding", + "check_prompt_injection_input_validation", + "check_bedrock_sdk_version_currency", + "check_waf_sql_injection_rules", + "check_penetration_testing_evidence", + "check_output_validation_lambda", + "check_xss_prevention_waf", + "check_output_encoding_advisory", + "check_output_schema_validation", + "check_guardrail_topic_allowlist", + "check_contextual_grounding_for_offtopic", + "check_knowledge_base_sync_schedule", + "check_data_currency_disclaimer_advisory", + "check_foundation_model_lifecycle_policy", + "check_kb_datasource_s3_event_notifications", + "check_agentcore_end_user_identity_propagation", + "check_agent_financial_transaction_thresholds", + "check_api_gateway_request_body_size_limits", + "check_prompt_input_validation_function", + ] + + @pytest.mark.parametrize("func_name", EXPECTED_CHECK_FUNCTIONS) + def test_function_exists(self, func_name): + assert hasattr(app, func_name), f"app.{func_name} not found" + assert callable(getattr(app, func_name)), f"app.{func_name} is not callable" + + def test_expected_count(self): + """Sanity check: 65 check functions (64 standalone + new ARC check sharing FS-27).""" + assert len(self.EXPECTED_CHECK_FUNCTIONS) == 65 diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_large_estate.py b/aiml-security-assessment/functions/security/finserv_tests/test_large_estate.py new file mode 100644 index 0000000..e92cfd2 --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_tests/test_large_estate.py @@ -0,0 +1,456 @@ +""" +Large-estate performance and memory guard — Task 15 (Wave 4). + +Validates: REQ-10.1, REQ-10.2, REQ-10.3, REQ-11.1, REQ-11.3 + +With ≥1,000 functions, ≥100 buckets, ≥50 guardrails/KBs/ACLs the handler must: + 1. Issue at most one listing call per inventory (enforced by patching collect_resource_inventory + to return a pre-built large inventory — the real collector never runs in this test). + 2. Issue ≤1 detail call per resource (get_guardrail per guardrail id, get_web_acl per ACL id). + 3. Complete well within the 900 s Lambda budget. + 4. Keep peak memory well under 1024 MB. + +The test builds the large inventory directly (bypassing the collector) and patches +app.collect_resource_inventory to return it, then runs lambda_handler end-to-end with +a generic MagicMock for every non-inventory boto3 client. +""" + +import os +import sys +import time +import tracemalloc +from datetime import datetime, timezone +from unittest.mock import MagicMock, patch + + +FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") +if FINSERV_DIR not in sys.path: + sys.path.insert(0, FINSERV_DIR) + +import app # noqa: E402 + + +# --------------------------------------------------------------------------- +# Inline inventory builder — mirrors conftest.make_resource_inventory so this +# module is self-contained while staying consistent with the shared fixture. +# --------------------------------------------------------------------------- + + +def make_resource_inventory(**overrides) -> app.ResourceInventory: + """Build a fully-available ResourceInventory with sensible empty defaults. + + Mirrors conftest.make_resource_inventory; defined here so the test module + is self-contained (conftest is only importable by pytest, not directly). + """ + defaults: dict = { + "lambda_functions": [], + "guardrails": app.GuardrailInventory(summaries=[], detail_by_id={}), + "knowledge_bases": app.KbInventory( + summaries=[], data_sources_by_kb={}, data_source_detail={} + ), + "buckets": [], + "web_acls": app.WebAclInventory(summaries=[], detail_by_id={}), + } + defaults.update(overrides) + return app.ResourceInventory(**defaults) + + +# --------------------------------------------------------------------------- +# Large-estate fixture helpers +# --------------------------------------------------------------------------- + +_N_FUNCTIONS = 1_000 +_N_BUCKETS = 100 +_N_GUARDRAILS = 50 +_N_KBS = 50 +_N_ACLS = 50 + + +def _build_large_inventory() -> app.ResourceInventory: + """Construct a fully-populated ResourceInventory at large-estate scale.""" + # Lambda functions — 1 000 entries + functions = [ + {"FunctionName": f"fn-{i}", "Runtime": "python3.12"} + for i in range(_N_FUNCTIONS) + ] + + # Guardrails — 50 summaries + 50 detail entries + guardrail_summaries = [ + {"id": f"g-{i}", "name": f"guardrail-{i}"} for i in range(_N_GUARDRAILS) + ] + detail_by_guardrail_id = { + g["id"]: { + "guardrailId": g["id"], + "name": g["name"], + "status": "READY", + "contentPolicy": { + "filters": [ + { + "type": "SEXUAL", + "inputStrength": "HIGH", + "outputStrength": "HIGH", + } + ] + }, + "topicPolicy": {"topics": []}, + "wordPolicy": {"words": [], "managedWordLists": []}, + "sensitiveInformationPolicy": {"piiEntities": [], "regexes": []}, + "contextualGroundingPolicy": { + "filters": [ + {"type": "GROUNDING", "threshold": 0.8}, + {"type": "RELEVANCE", "threshold": 0.8}, + ] + }, + } + for g in guardrail_summaries + } + + # Knowledge Bases — 50 summaries, 1 data source each, full detail + kb_summaries = [ + { + "knowledgeBaseId": f"kb-{i}", + "name": f"kb-{i}", + "status": "ACTIVE", + "updatedAt": datetime.now(timezone.utc), + } + for i in range(_N_KBS) + ] + data_sources_by_kb: dict = {} + data_source_detail: dict = {} + for kb in kb_summaries: + kb_id = kb["knowledgeBaseId"] + ds_id = f"ds-{kb_id}-0" + ds_summary = { + "dataSourceId": ds_id, + "name": "ds", + "status": "AVAILABLE", + "updatedAt": datetime.now(timezone.utc), + } + data_sources_by_kb[kb_id] = [ds_summary] + data_source_detail[(kb_id, ds_id)] = { + "dataSource": { + "dataSourceId": ds_id, + "knowledgeBaseId": kb_id, + "dataSourceConfiguration": { + "type": "S3", + "s3Configuration": { + "bucketArn": f"arn:aws:s3:::kb-bucket-{kb_id}", + }, + }, + } + } + + # S3 buckets — 100 entries + buckets = [{"Name": f"bucket-{i}"} for i in range(_N_BUCKETS)] + + # WAFv2 Web ACLs — 50 summaries + 50 detail entries + acl_summaries = [ + {"Id": f"acl-{i}", "Name": f"acl-{i}", "ARN": f"arn:aws:wafv2:::acl-{i}"} + for i in range(_N_ACLS) + ] + detail_by_acl_id = { + acl["Id"]: { + "Id": acl["Id"], + "Name": acl["Name"], + "ARN": acl["ARN"], + "Rules": [ + { + "Name": "SQLiRule", + "Statement": { + "ManagedRuleGroupStatement": { + "VendorName": "AWS", + "Name": "AWSManagedRulesSQLiRuleSet", + } + }, + } + ], + } + for acl in acl_summaries + } + + return make_resource_inventory( + lambda_functions=functions, + guardrails=app.GuardrailInventory( + summaries=guardrail_summaries, + detail_by_id=detail_by_guardrail_id, + ), + knowledge_bases=app.KbInventory( + summaries=kb_summaries, + data_sources_by_kb=data_sources_by_kb, + data_source_detail=data_source_detail, + ), + buckets=buckets, + web_acls=app.WebAclInventory( + summaries=acl_summaries, + detail_by_id=detail_by_acl_id, + ), + ) + + +def _make_generic_non_inventory_client() -> MagicMock: + """Return a MagicMock that satisfies every non-inventory boto3 call. + + The inventory itself is provided via the pre-built ResourceInventory; + this client handles everything else (shield, apigateway, cloudwatch, …). + """ + generic = MagicMock() + + # Paginator pattern used by some checks (cw.get_paginator, sfn, etc.) + paginator = MagicMock() + paginator.paginate.return_value = [{}] + generic.get_paginator.return_value = paginator + + # Inventory listing methods — these MUST NOT be called because + # collect_resource_inventory is patched to return the pre-built inventory. + # We leave them as MagicMock (auto-return) but we'll count their calls. + + # Non-inventory service methods — return empty / benign responses + generic.describe_subscription.side_effect = Exception("no shield subscription") + generic.get_usage_plans.return_value = {"items": []} + generic.list_service_quotas.return_value = {"Quotas": []} + generic.get_anomaly_monitors.return_value = {"AnomalyMonitors": []} + generic.describe_budgets.return_value = {"Budgets": []} + generic.get_caller_identity.return_value = {"Account": "123456789012"} + generic.list_agents.return_value = {"agentSummaries": []} + generic.list_agent_runtimes.return_value = {"agentRuntimes": []} + generic.list_state_machines.return_value = {"stateMachines": []} + generic.list_policies.return_value = {"Policies": []} + generic.list_custom_models.return_value = {"modelSummaries": []} + generic.list_models.return_value = {"Models": []} + generic.describe_config_rules.return_value = {"ConfigRules": []} + generic.list_evaluation_jobs.return_value = {"jobSummaries": []} + generic.describe_repositories.return_value = {"repositories": []} + generic.list_feature_groups.return_value = {"FeatureGroupSummaries": []} + generic.list_log_groups.return_value = {"logGroups": []} + generic.get_macie_session.side_effect = Exception("macie not enabled") + generic.list_foundation_models.return_value = {"modelSummaries": []} + generic.list_model_cards.return_value = {"ModelCardSummaryList": []} + generic.list_rules.return_value = {"Rules": []} + generic.list_schedules.return_value = {"Schedules": []} + generic.get_rest_apis.return_value = {"items": []} + generic.list_processing_jobs.return_value = {"ProcessingJobSummaries": []} + generic.list_automated_reasoning_policies.return_value = { + "automatedReasoningPolicySummaries": [] + } + generic.list_security_policies.return_value = {"securityPolicySummaries": []} + generic.list_monitoring_schedules.return_value = {"MonitoringScheduleSummaries": []} + + # S3 per-bucket detail calls (get_bucket_versioning, get_bucket_tagging, …) + generic.get_bucket_versioning.return_value = {"Status": "Enabled"} + generic.get_bucket_tagging.return_value = {"TagSet": []} + generic.get_bucket_notification_configuration.return_value = {} + + # S3 data-protection-policy for CloudWatch Logs PII check + generic.get_data_protection_policy.return_value = {} + + # per-function concurrency (FS-09 keeps its own concurrency loop) + generic.get_function_concurrency.return_value = { + "ReservedConcurrentExecutions": 100 + } + + return generic + + +# --------------------------------------------------------------------------- +# Test class +# --------------------------------------------------------------------------- + + +class TestLargeEstatePerformance: + """Large-estate performance and memory guard (Task 15, Wave 4). + + Validates: REQ-10.1, REQ-10.2, REQ-10.3, REQ-11.1, REQ-11.3 + """ + + @patch("app.write_to_s3") + @patch("app.get_permissions_cache") + @patch("app.boto3.client") + @patch("app.collect_resource_inventory") + def test_large_estate_single_listing_and_memory( + self, + mock_collect_inventory, + mock_boto3_client, + mock_get_perm_cache, + mock_write_s3, + ): + """With 1 000 functions, 100 buckets, 50 guardrails/KBs/ACLs: + - collect_resource_inventory is called exactly once (single enumeration). + - The five shared listing APIs are never called directly (≤0 for each + because the collector is patched out entirely). + - Detail APIs (get_guardrail, get_web_acl, get_function_concurrency) are + only called from within the checks themselves; each is called at most + once per resource since the checks read from the pre-built inventory. + - Handler completes well within 900 s. + - Peak memory well under 1024 MB. + """ + # --- Setup --- + large_inventory = _build_large_inventory() + mock_collect_inventory.return_value = large_inventory + + generic_client = _make_generic_non_inventory_client() + mock_boto3_client.return_value = generic_client + + mock_get_perm_cache.return_value = { + "role_permissions": {}, + "user_permissions": {}, + } + mock_write_s3.return_value = "https://test-bucket.s3.amazonaws.com/finserv_security_report_large-estate.csv" + + event = { + "Execution": {"Name": "large-estate-perf-test"}, + "StateMachine": { + "Id": "arn:aws:states:us-east-1:123456789012:stateMachine:test" + }, + } + + # --- Run under memory and time instrumentation --- + tracemalloc.start() + start = time.perf_counter() + + result = app.lambda_handler(event, None) + + elapsed = time.perf_counter() - start + _, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + + # --- Correctness: handler must succeed --- + assert result["statusCode"] == 200, ( + f"lambda_handler returned {result['statusCode']}" + ) + assert "findings" in result["body"] + assert len(result["body"]["findings"]) == 65, ( + f"Expected 65 findings, got {len(result['body']['findings'])}" + ) + + # --- REQ-10.1 / REQ-10.2: Single enumeration per inventory --- + # collect_resource_inventory is called exactly once per invocation. + mock_collect_inventory.assert_called_once_with() + + # The five shared listing APIs must NOT be called on the boto3 client, + # because collect_resource_inventory (which is patched out) would own + # those calls. Any direct call from a check would violate REQ-10.1. + assert generic_client.list_functions.call_count == 0, ( + f"list_functions called {generic_client.list_functions.call_count}× " + "— must be 0 when collect_resource_inventory is patched" + ) + assert generic_client.list_guardrails.call_count == 0, ( + f"list_guardrails called {generic_client.list_guardrails.call_count}×" + ) + assert generic_client.list_knowledge_bases.call_count == 0, ( + f"list_knowledge_bases called {generic_client.list_knowledge_bases.call_count}×" + ) + assert generic_client.list_buckets.call_count == 0, ( + f"list_buckets called {generic_client.list_buckets.call_count}×" + ) + assert generic_client.list_web_acls.call_count == 0, ( + f"list_web_acls called {generic_client.list_web_acls.call_count}×" + ) + + # --- REQ-10.2: ≤1 detail call per resource --- + # get_guardrail is NOT called from checks (the inventory pre-loads detail); + # verify no check bypasses the inventory and calls it directly. + assert generic_client.get_guardrail.call_count == 0, ( + f"get_guardrail called {generic_client.get_guardrail.call_count}× " + "— checks must read from inventory.guardrails.detail_by_id" + ) + # get_web_acl: same invariant. + assert generic_client.get_web_acl.call_count == 0, ( + f"get_web_acl called {generic_client.get_web_acl.call_count}× " + "— checks must read from inventory.web_acls.detail_by_id" + ) + # get_function_concurrency: FS-09 calls it per-function on its + # agent-name-filtered subset. With fn-0..fn-999 none match the + # agent/bedrock/aiml filter, so the count must be 0. + assert generic_client.get_function_concurrency.call_count == 0, ( + f"get_function_concurrency called " + f"{generic_client.get_function_concurrency.call_count}× " + "— none of fn-0..fn-999 match the agent/bedrock/aiml filter" + ) + + # --- REQ-10.3: Completion within the 900 s budget --- + assert elapsed < 900, ( + f"Handler took {elapsed:.2f}s — must complete within 900 s" + ) + + # --- REQ-11.1 / REQ-11.3: Peak memory well under 1024 MB --- + peak_mb = peak / (1024 * 1024) + assert peak_mb < 1024, ( + f"Peak memory {peak_mb:.1f} MB — must be well under 1024 MB" + ) + + @patch("app.write_to_s3") + @patch("app.get_permissions_cache") + @patch("app.boto3.client") + @patch("app.collect_resource_inventory") + def test_large_estate_timing_budget( + self, + mock_collect_inventory, + mock_boto3_client, + mock_get_perm_cache, + mock_write_s3, + ): + """Focused timing assertion: 65 checks over a large inventory must + complete well within the 900 s budget (target: under 60 s on any + reasonable CI host, giving 15× headroom). + + Validates: REQ-10.3 + """ + large_inventory = _build_large_inventory() + mock_collect_inventory.return_value = large_inventory + mock_boto3_client.return_value = _make_generic_non_inventory_client() + mock_get_perm_cache.return_value = { + "role_permissions": {}, + "user_permissions": {}, + } + mock_write_s3.return_value = "https://test-bucket.s3.amazonaws.com/report.csv" + + event = {"Execution": {"Name": "timing-test"}} + + start = time.perf_counter() + app.lambda_handler(event, None) + elapsed = time.perf_counter() - start + + # Strict budget: all mocked I/O, so well under 60 s even on slow hardware + assert elapsed < 60, ( + f"Handler took {elapsed:.2f}s with mocked I/O — target <60 s " + f"(900 s hard budget leaves {900 - elapsed:.0f}s headroom)" + ) + + @patch("app.write_to_s3") + @patch("app.get_permissions_cache") + @patch("app.boto3.client") + @patch("app.collect_resource_inventory") + def test_large_estate_memory_footprint( + self, + mock_collect_inventory, + mock_boto3_client, + mock_get_perm_cache, + mock_write_s3, + ): + """Peak memory footprint with a large inventory must stay well under + the 1024 MB Lambda limit. + + Validates: REQ-11.1, REQ-11.3 + """ + large_inventory = _build_large_inventory() + mock_collect_inventory.return_value = large_inventory + mock_boto3_client.return_value = _make_generic_non_inventory_client() + mock_get_perm_cache.return_value = { + "role_permissions": {}, + "user_permissions": {}, + } + mock_write_s3.return_value = "https://test-bucket.s3.amazonaws.com/report.csv" + + event = {"Execution": {"Name": "memory-test"}} + + tracemalloc.start() + app.lambda_handler(event, None) + _, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + + peak_mb = peak / (1024 * 1024) + # 50 MB is a generous ceiling for mocked I/O; the real ceiling is 1024 MB + assert peak_mb < 50, ( + f"Peak memory {peak_mb:.1f} MB with mocked I/O — expected well under 50 MB " + f"(hard limit is 1024 MB)" + ) diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_paginate.py b/aiml-security-assessment/functions/security/finserv_tests/test_paginate.py new file mode 100644 index 0000000..77c184b --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_tests/test_paginate.py @@ -0,0 +1,436 @@ +""" +Tests for the _paginate helper — Task 2.1 +FinServ Shared-Inventory Refactor (FU-3) + +Requirements: REQ-2.3, REQ-2.5, REQ-9.2 + +Coverage +-------- +1. WAFv2 token=("NextMarker", "NextMarker") — multi-page +2. S3 token=("ContinuationToken", "ContinuationToken") — multi-page +3. Regression: Lambda Marker convention (no token= passed) — multi-page +4. Regression: bedrock nextToken convention (no token= passed) — multi-page +5. Single-page = exactly one call (both with and without token=) +6. Repeated-token loop guard (infinite-loop protection) +7. token= override bypasses the convention table (WAFv2 NextMarker ≠ Lambda Marker) +""" + +from __future__ import annotations + +import os +import sys +from unittest.mock import MagicMock, call + + +# --------------------------------------------------------------------------- +# Make finserv_assessments importable (mirrors conftest.py / test_inventory_equivalence.py) +# --------------------------------------------------------------------------- +FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") +if FINSERV_DIR not in sys.path: + sys.path.insert(0, FINSERV_DIR) + +import app # noqa: E402 + + +# =========================================================================== +# Helpers +# =========================================================================== + + +def _make_client(pages: list[dict]) -> MagicMock: + """Return a mock boto3 client whose ``list_items`` method yields ``pages`` + sequentially. Each element in ``pages`` is the raw response dict.""" + client = MagicMock() + client.list_items.side_effect = pages + return client + + +# =========================================================================== +# 1. WAFv2 NextMarker / NextMarker (explicit token override, multi-page) +# =========================================================================== + + +class TestWafv2NextMarkerOverride: + """token=("NextMarker", "NextMarker") — both the output field and the + request parameter are "NextMarker", which collides with Lambda's + convention that maps "NextMarker" → "Marker". The explicit override + must bypass the table and use the correct input param. + + Validates: REQ-2.3 (correct per-operation convention), REQ-9.2-b + """ + + def test_multi_page_collects_all_items(self): + pages = [ + {"WebACLs": [{"Id": "acl-1"}, {"Id": "acl-2"}], "NextMarker": "tok-1"}, + {"WebACLs": [{"Id": "acl-3"}]}, + ] + client = _make_client(pages) + + result = app._paginate( + client, + "list_items", + "WebACLs", + token=("NextMarker", "NextMarker"), + Scope="REGIONAL", + ) + + assert result == [{"Id": "acl-1"}, {"Id": "acl-2"}, {"Id": "acl-3"}] + + def test_multi_page_request_uses_NextMarker_not_Marker(self): + """The second call must send NextMarker= (WAFv2), NOT Marker= (Lambda).""" + pages = [ + {"WebACLs": [{"Id": "acl-1"}], "NextMarker": "tok-1"}, + {"WebACLs": [{"Id": "acl-2"}]}, + ] + client = _make_client(pages) + + app._paginate( + client, + "list_items", + "WebACLs", + token=("NextMarker", "NextMarker"), + Scope="REGIONAL", + ) + + calls = client.list_items.call_args_list + assert len(calls) == 2 + # First call: no pagination token + assert calls[0] == call(Scope="REGIONAL") + # Second call: NextMarker= (not Marker=) + assert calls[1] == call(Scope="REGIONAL", NextMarker="tok-1") + assert "Marker" not in calls[1].kwargs + + def test_three_pages_correct_order(self): + pages = [ + {"WebACLs": [{"Id": "a"}], "NextMarker": "t1"}, + {"WebACLs": [{"Id": "b"}], "NextMarker": "t2"}, + {"WebACLs": [{"Id": "c"}]}, + ] + client = _make_client(pages) + + result = app._paginate( + client, "list_items", "WebACLs", token=("NextMarker", "NextMarker") + ) + + assert result == [{"Id": "a"}, {"Id": "b"}, {"Id": "c"}] + assert client.list_items.call_count == 3 + + +# =========================================================================== +# 2. S3 ContinuationToken / ContinuationToken (explicit token override, multi-page) +# =========================================================================== + + +class TestS3ContinuationTokenOverride: + """token=("ContinuationToken", "ContinuationToken") — ContinuationToken is + absent from the default convention table, so without the explicit override + _paginate would stop after the first page. + + Validates: REQ-2.3, REQ-2.8, REQ-9.2-b + """ + + def test_multi_page_collects_all_buckets(self): + pages = [ + { + "Buckets": [{"Name": "bucket-1"}, {"Name": "bucket-2"}], + "ContinuationToken": "ct-1", + }, + {"Buckets": [{"Name": "bucket-3"}]}, + ] + client = _make_client(pages) + + result = app._paginate( + client, + "list_items", + "Buckets", + token=("ContinuationToken", "ContinuationToken"), + MaxBuckets=1000, + ) + + assert result == [ + {"Name": "bucket-1"}, + {"Name": "bucket-2"}, + {"Name": "bucket-3"}, + ] + + def test_second_call_sends_continuation_token(self): + pages = [ + {"Buckets": [{"Name": "b1"}], "ContinuationToken": "ct-1"}, + {"Buckets": [{"Name": "b2"}]}, + ] + client = _make_client(pages) + + app._paginate( + client, + "list_items", + "Buckets", + token=("ContinuationToken", "ContinuationToken"), + MaxBuckets=1000, + ) + + calls = client.list_items.call_args_list + assert calls[0] == call(MaxBuckets=1000) + assert calls[1] == call(MaxBuckets=1000, ContinuationToken="ct-1") + + def test_three_pages_order_preserved(self): + pages = [ + {"Buckets": [{"Name": "b1"}], "ContinuationToken": "ct-1"}, + {"Buckets": [{"Name": "b2"}], "ContinuationToken": "ct-2"}, + {"Buckets": [{"Name": "b3"}]}, + ] + client = _make_client(pages) + + result = app._paginate( + client, + "list_items", + "Buckets", + token=("ContinuationToken", "ContinuationToken"), + ) + + assert [b["Name"] for b in result] == ["b1", "b2", "b3"] + assert client.list_items.call_count == 3 + + +# =========================================================================== +# 3. Regression: Lambda Marker convention unchanged (no token= passed) +# =========================================================================== + + +class TestLambdaMarkerRegression: + """Existing callers that use the Lambda convention (output: NextMarker, + input: Marker) must behave identically when no token= is passed. + + Validates: REQ-2.3 (no regression), REQ-9.2-b + """ + + def test_single_page_single_call(self): + client = _make_client([{"Functions": [{"FunctionName": "f1"}]}]) + + result = app._paginate(client, "list_items", "Functions") + + assert result == [{"FunctionName": "f1"}] + assert client.list_items.call_count == 1 + + def test_multi_page_uses_Marker_not_NextMarker(self): + pages = [ + {"Functions": [{"FunctionName": "f1"}], "NextMarker": "m1"}, + {"Functions": [{"FunctionName": "f2"}]}, + ] + client = _make_client(pages) + + result = app._paginate(client, "list_items", "Functions") + + assert result == [{"FunctionName": "f1"}, {"FunctionName": "f2"}] + calls = client.list_items.call_args_list + # Second call must use "Marker=" (Lambda convention), not "NextMarker=" + assert calls[1] == call(Marker="m1") + assert "NextMarker" not in calls[1].kwargs + + def test_multi_page_collects_all(self): + pages = [ + {"Functions": [{"FunctionName": "f1"}], "NextMarker": "m1"}, + {"Functions": [{"FunctionName": "f2"}], "NextMarker": "m2"}, + {"Functions": [{"FunctionName": "f3"}]}, + ] + client = _make_client(pages) + + result = app._paginate(client, "list_items", "Functions") + + assert [f["FunctionName"] for f in result] == ["f1", "f2", "f3"] + assert client.list_items.call_count == 3 + + +# =========================================================================== +# 4. Regression: bedrock nextToken convention unchanged (no token= passed) +# =========================================================================== + + +class TestBedrockNextTokenRegression: + """Existing callers that use the bedrock lower-camel nextToken convention + must behave identically when no token= is passed. + + Validates: REQ-2.3 (no regression), REQ-9.2-b + """ + + def test_single_page_single_call(self): + client = _make_client([{"guardrails": [{"id": "g1"}]}]) + + result = app._paginate(client, "list_items", "guardrails") + + assert result == [{"id": "g1"}] + assert client.list_items.call_count == 1 + + def test_multi_page_collects_all(self): + pages = [ + {"guardrails": [{"id": "g1"}], "nextToken": "nt1"}, + {"guardrails": [{"id": "g2"}], "nextToken": "nt2"}, + {"guardrails": [{"id": "g3"}]}, + ] + client = _make_client(pages) + + result = app._paginate(client, "list_items", "guardrails") + + assert [g["id"] for g in result] == ["g1", "g2", "g3"] + calls = client.list_items.call_args_list + assert calls[1] == call(nextToken="nt1") + assert calls[2] == call(nextToken="nt2") + assert client.list_items.call_count == 3 + + +# =========================================================================== +# 5. Single-page = exactly one call (both with and without token=) +# =========================================================================== + + +class TestSinglePageSingleCall: + """A single-page response (no continuation token in output) must yield + exactly one API call regardless of whether token= is supplied. + + Validates: REQ-2.5 + """ + + def test_no_token_override_single_call(self): + client = _make_client([{"Items": [{"id": "x"}]}]) + + result = app._paginate(client, "list_items", "Items") + + assert result == [{"id": "x"}] + assert client.list_items.call_count == 1 + + def test_wafv2_token_override_single_call(self): + client = _make_client([{"WebACLs": [{"Id": "a1"}]}]) + + result = app._paginate( + client, + "list_items", + "WebACLs", + token=("NextMarker", "NextMarker"), + Scope="REGIONAL", + ) + + assert result == [{"Id": "a1"}] + assert client.list_items.call_count == 1 + + def test_s3_token_override_single_call(self): + client = _make_client([{"Buckets": [{"Name": "b1"}]}]) + + result = app._paginate( + client, + "list_items", + "Buckets", + token=("ContinuationToken", "ContinuationToken"), + ) + + assert result == [{"Name": "b1"}] + assert client.list_items.call_count == 1 + + +# =========================================================================== +# 6. Repeated-token loop guard +# =========================================================================== + + +class TestRepeatedTokenLoopGuard: + """If a mock (or a misbehaving API) returns the same token every call, + _paginate must stop after collecting the first page and not loop forever. + + This guard must hold both with and without the token= override. + + Validates: REQ-9.2 (loop-guard regression) + """ + + def test_loop_guard_no_override(self): + """Lambda convention: repeated NextMarker stops after two calls.""" + repeated_page = {"Functions": [{"FunctionName": "f1"}], "NextMarker": "same"} + client = MagicMock() + client.list_items.return_value = repeated_page + + result = app._paginate(client, "list_items", "Functions") + + # First call collects items; token "same" is seen; second call also + # returns "same" which is in seen_tokens → stop. + assert result == [{"FunctionName": "f1"}, {"FunctionName": "f1"}] + assert client.list_items.call_count == 2 + + def test_loop_guard_with_wafv2_override(self): + """WAFv2 token= override: repeated NextMarker stops after two calls.""" + repeated_page = {"WebACLs": [{"Id": "a"}], "NextMarker": "same"} + client = MagicMock() + client.list_items.return_value = repeated_page + + result = app._paginate( + client, + "list_items", + "WebACLs", + token=("NextMarker", "NextMarker"), + ) + + assert result == [{"Id": "a"}, {"Id": "a"}] + assert client.list_items.call_count == 2 + + def test_loop_guard_with_s3_override(self): + """S3 token= override: repeated ContinuationToken stops after two calls.""" + repeated_page = {"Buckets": [{"Name": "b"}], "ContinuationToken": "same"} + client = MagicMock() + client.list_items.return_value = repeated_page + + result = app._paginate( + client, + "list_items", + "Buckets", + token=("ContinuationToken", "ContinuationToken"), + ) + + assert result == [{"Name": "b"}, {"Name": "b"}] + assert client.list_items.call_count == 2 + + +# =========================================================================== +# 7. token= override disambiguates WAFv2 NextMarker from Lambda NextMarker +# =========================================================================== + + +class TestWafv2VsLambdaDisambiguation: + """Prove that the explicit token= override is the only way to correctly + paginate WAFv2: without it, the convention table maps NextMarker → Marker + (Lambda convention), which would send the wrong input parameter. + + Validates: REQ-2.9 — the (output, input) pair must be supplied explicitly + for WAFv2 because output-field name alone is insufficient to infer the + correct input parameter. + """ + + def test_without_override_sends_Marker_for_NextMarker_output(self): + """Without token=, the convention table maps NextMarker → Marker. + This is the Lambda convention — correct for Lambda, wrong for WAFv2.""" + pages = [ + {"WebACLs": [{"Id": "a1"}], "NextMarker": "tok"}, + {"WebACLs": [{"Id": "a2"}]}, + ] + client = _make_client(pages) + + # Intentionally NOT passing token= to demonstrate the convention-table behavior + app._paginate(client, "list_items", "WebACLs") + + calls = client.list_items.call_args_list + # The convention table sends Marker= (Lambda convention) — wrong for WAFv2 + assert calls[1] == call(Marker="tok") + + def test_with_override_sends_NextMarker_for_NextMarker_output(self): + """With token=("NextMarker","NextMarker"), the second call sends NextMarker=.""" + pages = [ + {"WebACLs": [{"Id": "a1"}], "NextMarker": "tok"}, + {"WebACLs": [{"Id": "a2"}]}, + ] + client = _make_client(pages) + + app._paginate( + client, + "list_items", + "WebACLs", + token=("NextMarker", "NextMarker"), + ) + + calls = client.list_items.call_args_list + assert calls[1] == call(NextMarker="tok") + assert "Marker" not in calls[1].kwargs diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_phase2_live.py b/aiml-security-assessment/functions/security/finserv_tests/test_phase2_live.py new file mode 100644 index 0000000..bcdcd85 --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_tests/test_phase2_live.py @@ -0,0 +1,286 @@ +#!/usr/bin/env python3 +""" +Phase 2 — Live AWS integration test for finserv_assessments. + +Invokes each check function individually against a real AWS account, +captures results, and produces a triage report showing: + - PASS: check completed successfully, resources found compliant or N/A + - WARN: check completed, found non-compliant resources (expected in dev) + - ERROR: check failed — likely IAM permission issue or API incompatibility + +Usage: + # Run all checks (default): + python tests/test_phase2_live.py + + # Run a single check by name: + python tests/test_phase2_live.py check_waf_shield_on_bedrock_endpoints + + # Run with a specific S3 bucket for CSV output: + AIML_ASSESSMENT_BUCKET_NAME=my-bucket python tests/test_phase2_live.py + +Prerequisites: + - AWS credentials configured (aws configure or env vars) + - Read-only access to the target account + - No Docker required +""" + +import json +import os +import sys +import time +from datetime import datetime + +# Make finserv_assessments importable +FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") +if FINSERV_DIR not in sys.path: + sys.path.insert(0, FINSERV_DIR) + +# Set env var if not already set +if not os.environ.get("AIML_ASSESSMENT_BUCKET_NAME"): + os.environ["AIML_ASSESSMENT_BUCKET_NAME"] = "mehta-test-v55" + +import app # noqa: E402 (import must follow sys.path + env setup above) + + +# --------------------------------------------------------------------------- +# All check functions in execution order (matches lambda_handler). +# +# Each entry is a 3-tuple: (func_name, needs_cache, needs_inventory) +# needs_cache True → pass permission_cache as first argument +# needs_inventory True → pass inventory as first argument +# (both False) → call with no arguments +# --------------------------------------------------------------------------- +CHECK_FUNCTIONS = [ + # Category 1: Unbounded Consumption + ("check_waf_shield_on_bedrock_endpoints", False, True), + ("check_api_gateway_rate_limiting", False, False), + ("check_bedrock_token_quotas", False, False), + ("check_cost_anomaly_detection", False, False), + ("check_cloudwatch_token_alarms", False, False), + ("check_aws_budgets_for_aiml", False, False), + # Category 2: Excessive Agency + ("check_bedrock_agent_action_boundaries", True, False), + ("check_agentcore_policy_engine", False, False), + ("check_agent_transaction_limits", False, True), + ("check_human_in_the_loop_for_high_risk_actions", False, False), + ("check_agent_rate_alarms", False, False), + # Category 3: Supply Chain Vulnerabilities + ("check_scp_model_access_restrictions", False, False), + ("check_model_inventory_tagging", False, False), + ("check_model_onboarding_governance", False, False), + ("check_bedrock_model_evaluation_adversarial", False, False), + ("check_ecr_image_scanning", False, False), + # Category 4: Training Data & Model Poisoning + ("check_feature_store_rollback_capability", False, False), + ("check_training_data_s3_versioning", False, True), + # Category 5: Vector & Embedding Weaknesses + ("check_knowledge_base_iam_least_privilege", True, False), + ("check_knowledge_base_metadata_filtering", False, True), + ("check_opensearch_serverless_encryption", False, False), + ("check_knowledge_base_vpc_access", False, False), + # Category 6: Non-Compliant Output + ("check_guardrail_contextual_grounding", False, True), + ("check_automated_reasoning_policies", False, False), + ("check_guardrail_denied_topics_financial", False, True), + ("check_compliance_disclaimer_in_outputs", False, False), + ("check_bedrock_evaluation_compliance_datasets", False, False), + # Category 7: Misinformation + ("check_knowledge_base_data_source_sync", False, True), + ("check_source_attribution_in_guardrails", False, False), + ("check_knowledge_base_integrity_monitoring", False, True), + ("check_fm_version_currency", False, False), + # Category 8: Abusive or Harmful Output + ("check_fmeval_harmful_content", False, False), + ("check_guardrail_content_filters", False, True), + ("check_user_feedback_mechanism", False, False), + ("check_guardrail_word_filters", False, True), + # Category 9: Biased Output + ("check_sagemaker_clarify_bias", False, False), + ("check_bedrock_evaluation_bias_datasets", False, False), + ("check_sagemaker_clarify_explainability", False, False), + ("check_ai_service_cards_documentation", False, False), + # Category 10: Sensitive Information Disclosure + ("check_cloudwatch_log_pii_masking", False, False), + ("check_macie_on_training_data_buckets", False, False), + ("check_guardrail_pii_filters", False, True), + ("check_data_classification_tagging", False, True), + # Category 11: Hallucination + ("check_guardrail_grounding_threshold", False, True), + ("check_rag_knowledge_base_configured", False, True), + ("check_hallucination_disclaimer_advisory", False, False), + ("check_guardrail_relevance_grounding", False, True), + # Category 12: Prompt Injection + ("check_prompt_injection_input_validation", False, True), + ("check_bedrock_sdk_version_currency", False, True), + ("check_waf_sql_injection_rules", False, True), + ("check_penetration_testing_evidence", False, False), + # Category 13: Improper Output Handling + ("check_output_validation_lambda", False, True), + ("check_xss_prevention_waf", False, True), + ("check_output_encoding_advisory", False, False), + ("check_output_schema_validation", False, True), + # Category 14: Off-Topic & Inappropriate Output + ("check_guardrail_topic_allowlist", False, True), + ("check_contextual_grounding_for_offtopic", False, False), + # Category 15: Out-of-Date Training Data + ("check_knowledge_base_sync_schedule", False, True), + ("check_data_currency_disclaimer_advisory", False, False), + ("check_foundation_model_lifecycle_policy", False, False), + # Material Gap Checks + ("check_kb_datasource_s3_event_notifications", False, True), + ("check_agentcore_end_user_identity_propagation", False, False), + ("check_agent_financial_transaction_thresholds", False, True), + ("check_api_gateway_request_body_size_limits", False, True), + ("check_prompt_input_validation_function", False, True), +] + + +def run_single_check( + func_name, needs_cache, needs_inventory, permission_cache, inventory +): + """Run a single check function and return a result dict.""" + func = getattr(app, func_name) + start = time.time() + try: + if needs_cache: + result = func(permission_cache) + elif needs_inventory: + result = func(inventory) + else: + result = func() + elapsed = time.time() - start + status = result.get("status", "UNKNOWN") + detail = result.get("details", "") + csv_count = len(result.get("csv_data", [])) + + # Extract check IDs and statuses from csv_data for the report + csv_summary = [] + for row in result.get("csv_data", []): + csv_summary.append(f"{row.get('Check_ID', '?')}: {row.get('Status', '?')}") + + return { + "func": func_name, + "status": status, + "elapsed": round(elapsed, 2), + "csv_count": csv_count, + "csv_summary": csv_summary, + "error": detail if status == "ERROR" else "", + } + except Exception as e: + elapsed = time.time() - start + return { + "func": func_name, + "status": "EXCEPTION", + "elapsed": round(elapsed, 2), + "csv_count": 0, + "csv_summary": [], + "error": f"{type(e).__name__}: {e}", + } + + +def main(): + filter_name = sys.argv[1] if len(sys.argv) > 1 else None + + print("=" * 78) + print("Phase 2 — Live AWS Integration Test") + print( + f"Account: {os.popen('aws sts get-caller-identity --query Account --output text 2>/dev/null').read().strip()}" + ) + print( + f"Region: {os.popen('aws configure get region 2>/dev/null').read().strip() or 'us-east-1 (default)'}" + ) + print(f"Bucket: {os.environ.get('AIML_ASSESSMENT_BUCKET_NAME', 'NOT SET')}") + print(f"Time: {datetime.now().isoformat()}") + print("=" * 78) + + # Build a minimal permission cache (empty — no pre-cached IAM data) + permission_cache = {"role_permissions": {}, "user_permissions": {}} + + # Collect the shared resource inventory once — mirrors lambda_handler behaviour. + print("\nCollecting resource inventory ...", end=" ", flush=True) + inventory_start = time.time() + inventory = app.collect_resource_inventory() + inventory_elapsed = round(time.time() - inventory_start, 2) + print(f"done ({inventory_elapsed}s)") + + checks_to_run = CHECK_FUNCTIONS + if filter_name: + checks_to_run = [(n, c, i) for n, c, i in CHECK_FUNCTIONS if n == filter_name] + if not checks_to_run: + print(f"ERROR: No check function named '{filter_name}'") + sys.exit(1) + + results = [] + total = len(checks_to_run) + + for idx, (func_name, needs_cache, needs_inventory) in enumerate(checks_to_run, 1): + print(f"\n[{idx:2d}/{total}] {func_name} ...", end=" ", flush=True) + r = run_single_check( + func_name, needs_cache, needs_inventory, permission_cache, inventory + ) + results.append(r) + + # Color-coded status + status = r["status"] + if status == "PASS": + icon = "✅" + elif status == "WARN": + icon = "⚠️ " + elif status == "ERROR": + icon = "❌" + elif status == "EXCEPTION": + icon = "💥" + else: + icon = "❓" + + print(f"{icon} {status} ({r['elapsed']}s, {r['csv_count']} findings)") + if r["error"]: + # Truncate long error messages + err = r["error"][:200] + print(f" └─ {err}") + for cs in r["csv_summary"]: + print(f" └─ {cs}") + + # ----------------------------------------------------------------------- + # Summary + # ----------------------------------------------------------------------- + print("\n" + "=" * 78) + print("TRIAGE SUMMARY") + print("=" * 78) + + pass_count = sum(1 for r in results if r["status"] == "PASS") + warn_count = sum(1 for r in results if r["status"] == "WARN") + error_count = sum(1 for r in results if r["status"] == "ERROR") + exception_count = sum(1 for r in results if r["status"] == "EXCEPTION") + total_findings = sum(r["csv_count"] for r in results) + total_time = sum(r["elapsed"] for r in results) + + print(f" ✅ PASS: {pass_count:3d}") + print(f" ⚠️ WARN: {warn_count:3d}") + print(f" ❌ ERROR: {error_count:3d}") + print(f" 💥 EXCEPTION: {exception_count:3d}") + print(" ─────────────────") + print(f" Total checks: {len(results):3d}") + print(f" Total findings: {total_findings}") + print(f" Total time: {total_time:.1f}s") + + if error_count > 0 or exception_count > 0: + print("\n--- ERRORS TO TRIAGE ---") + for r in results: + if r["status"] in ("ERROR", "EXCEPTION"): + print(f" {r['func']}: {r['error'][:150]}") + + # ----------------------------------------------------------------------- + # Write JSON report for further analysis + # ----------------------------------------------------------------------- + report_path = os.path.join(os.path.dirname(__file__), "..", "phase2_results.json") + with open(report_path, "w") as f: + json.dump(results, f, indent=2) + print(f"\nDetailed results written to: {report_path}") + + # Exit code: 0 if no EXCEPTION, non-zero otherwise + sys.exit(1 if exception_count > 0 else 0) + + +if __name__ == "__main__": + main() diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_resilience.py b/aiml-security-assessment/functions/security/finserv_tests/test_resilience.py new file mode 100644 index 0000000..ffba205 --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_tests/test_resilience.py @@ -0,0 +1,765 @@ +""" +Resilience and partial-inventory tests for finserv_assessments/app.py. + +Verifies: + 1. Single-inventory failure → only dependent checks emit COULD_NOT_ASSESS + (status="ERROR", csv_data=[]), while all other checks produce normal dispositions. + 2. Multiple-inventory failure → each failure recorded independently, run completes. + 3. Multi-inventory independence (REQ-8): unavailability of one inventory does not + affect checks that depend on a different inventory. + +Validates: Requirements REQ-4.2, REQ-4.3, REQ-4.6, REQ-8 +""" + +import os +import sys +from unittest.mock import MagicMock, patch + + +FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") +if FINSERV_DIR not in sys.path: + sys.path.insert(0, FINSERV_DIR) + +TESTS_DIR = os.path.dirname(__file__) +if TESTS_DIR not in sys.path: + sys.path.insert(0, TESTS_DIR) + +import app # noqa: E402 +from conftest import make_resource_inventory # noqa: E402 + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _is_could_not_assess(result: dict) -> bool: + """Return True if the check result signals COULD_NOT_ASSESS: + status="ERROR" and csv_data is empty (the handler will synthesize the + COULD_NOT_ASSESS row from the empty csv_data — design DD-3).""" + return result["status"] == "ERROR" and result["csv_data"] == [] + + +def _is_normal_result(result: dict) -> bool: + """Return True if the check produced a real disposition (not ERROR).""" + return result["status"] in ("PASS", "WARN") or ( + result["status"] == "PASS" and isinstance(result["csv_data"], list) + ) + + +def _has_rows(result: dict) -> bool: + """Return True if the check emitted at least one CSV row.""" + return bool(result.get("csv_data")) + + +def _make_shield_mock_no_subscription(): + """Build a mock shield client whose describe_subscription raises the + ResourceNotFoundException subclass that FS-01 catches, so the check + proceeds past the shield block and tests the WAFv2 inventory path.""" + + class ResourceNotFoundException(Exception): + """Minimal stand-in for botocore's ResourceNotFoundException.""" + + class FakeExceptions: + pass + + shield = MagicMock() + FakeExceptions.ResourceNotFoundException = ResourceNotFoundException + shield.exceptions = FakeExceptions + shield.describe_subscription.side_effect = ResourceNotFoundException("no sub") + return shield + + +# --------------------------------------------------------------------------- +# 1. Single-inventory failure — Lambda inventory +# --------------------------------------------------------------------------- + + +class TestLambdaInventoryUnavailable: + """Lambda inventory unavailable → FS-09, FS-52, FS-55, FS-58, FS-67, FS-69 + become COULD_NOT_ASSESS; checks on other inventories are unaffected.""" + + _ACCESS_DENIED = PermissionError("AccessDenied: lambda:ListFunctions") + + def _make_inv(self): + return make_resource_inventory( + lambda_functions=app._Unavailable(self._ACCESS_DENIED) + ) + + # --- FS-09 (check_agent_transaction_limits) --- + def test_fs09_becomes_could_not_assess(self): + """Validates: Requirements REQ-4.2""" + result = app.check_agent_transaction_limits(self._make_inv()) + assert _is_could_not_assess(result), ( + f"Expected COULD_NOT_ASSESS but got status={result['status']!r}, " + f"csv_data={result['csv_data']!r}" + ) + + # --- FS-52 (check_bedrock_sdk_version_currency) --- + def test_fs52_becomes_could_not_assess(self): + """Validates: Requirements REQ-4.2""" + result = app.check_bedrock_sdk_version_currency(self._make_inv()) + assert _is_could_not_assess(result), ( + f"Expected COULD_NOT_ASSESS but got status={result['status']!r}" + ) + + # --- FS-55 (check_output_validation_lambda) --- + def test_fs55_becomes_could_not_assess(self): + """Validates: Requirements REQ-4.2""" + result = app.check_output_validation_lambda(self._make_inv()) + assert _is_could_not_assess(result), ( + f"Expected COULD_NOT_ASSESS but got status={result['status']!r}" + ) + + # --- FS-58 (check_output_schema_validation) --- + def test_fs58_becomes_could_not_assess(self): + """Validates: Requirements REQ-4.2""" + result = app.check_output_schema_validation(self._make_inv()) + assert _is_could_not_assess(result), ( + f"Expected COULD_NOT_ASSESS but got status={result['status']!r}" + ) + + # --- Guardrail check unaffected when lambda is unavailable --- + def test_guardrail_check_unaffected(self): + """REQ-4.3 / REQ-8: A guardrail check still produces a normal result + when only the lambda inventory is unavailable.""" + inv = self._make_inv() + result = app.check_guardrail_contextual_grounding(inv) + # Empty guardrails → "No Guardrails" informational row (normal disposition) + assert result["status"] != "ERROR", ( + "Guardrail check should not be affected by lambda inventory failure" + ) + assert _has_rows(result) + + # --- WAFv2 check unaffected when lambda is unavailable --- + def test_waf_check_unaffected(self): + """REQ-4.3 / REQ-8: WAFv2 check is unaffected by lambda unavailability.""" + inv = self._make_inv() + with patch( + "app.boto3.client", return_value=_make_shield_mock_no_subscription() + ): + result = app.check_waf_shield_on_bedrock_endpoints(inv) + assert result["status"] != "ERROR", ( + "WAFv2 check should not be affected by lambda inventory failure" + ) + + # --- S3 check unaffected when lambda is unavailable --- + def test_s3_check_unaffected(self): + """REQ-4.3 / REQ-8: S3 check is unaffected by lambda unavailability.""" + inv = self._make_inv() + with patch("app.boto3.client") as mock_boto: + mock_boto.return_value = MagicMock() + result = app.check_training_data_s3_versioning(inv) + assert result["status"] != "ERROR", ( + "S3 versioning check should not be affected by lambda inventory failure" + ) + + # --- KB check unaffected when lambda is unavailable --- + def test_kb_check_unaffected(self): + """REQ-4.3 / REQ-8: KB metadata check is unaffected by lambda unavailability.""" + inv = self._make_inv() + result = app.check_knowledge_base_metadata_filtering(inv) + assert result["status"] != "ERROR", ( + "KB metadata check should not be affected by lambda inventory failure" + ) + + +# --------------------------------------------------------------------------- +# 2. Single-inventory failure — Guardrail inventory +# --------------------------------------------------------------------------- + + +class TestGuardrailInventoryUnavailable: + """Guardrail inventory unavailable → guardrail-consuming checks become + COULD_NOT_ASSESS; lambda, S3, KB, WAFv2 checks are unaffected.""" + + _ACCESS_DENIED = PermissionError("AccessDenied: bedrock:ListGuardrails") + + def _make_inv(self): + return make_resource_inventory(guardrails=app._Unavailable(self._ACCESS_DENIED)) + + # --- FS-27 (check_guardrail_contextual_grounding) --- + def test_fs27_becomes_could_not_assess(self): + """Validates: Requirements REQ-4.2""" + result = app.check_guardrail_contextual_grounding(self._make_inv()) + assert _is_could_not_assess(result), ( + f"Expected COULD_NOT_ASSESS but got status={result['status']!r}" + ) + + # --- FS-28 (check_guardrail_denied_topics_financial) --- + def test_fs28_becomes_could_not_assess(self): + """Validates: Requirements REQ-4.2""" + result = app.check_guardrail_denied_topics_financial(self._make_inv()) + assert _is_could_not_assess(result), ( + f"Expected COULD_NOT_ASSESS but got status={result['status']!r}" + ) + + # --- FS-36 (check_guardrail_content_filters) --- + def test_fs36_becomes_could_not_assess(self): + """Validates: Requirements REQ-4.2""" + result = app.check_guardrail_content_filters(self._make_inv()) + assert _is_could_not_assess(result), ( + f"Expected COULD_NOT_ASSESS but got status={result['status']!r}" + ) + + # --- Lambda check unaffected when guardrails are unavailable --- + def test_lambda_check_unaffected(self): + """REQ-4.3: Lambda check is unaffected by guardrail inventory failure.""" + inv = self._make_inv() + with patch("app.boto3.client") as mock_boto: + mock_boto.return_value = MagicMock( + get_function_concurrency=MagicMock( + return_value={"ReservedConcurrentExecutions": 5} + ) + ) + result = app.check_agent_transaction_limits(inv) + assert result["status"] != "ERROR", ( + "Lambda check should not be affected by guardrail inventory failure" + ) + + # --- KB check unaffected when guardrails are unavailable --- + def test_kb_check_unaffected(self): + """REQ-4.3: KB check is unaffected by guardrail inventory failure.""" + inv = self._make_inv() + result = app.check_knowledge_base_metadata_filtering(inv) + assert result["status"] != "ERROR", ( + "KB check should not be affected by guardrail inventory failure" + ) + + # --- S3 check unaffected when guardrails are unavailable --- + def test_s3_check_unaffected(self): + """REQ-4.3: S3 check is unaffected by guardrail inventory failure.""" + inv = self._make_inv() + with patch("app.boto3.client") as mock_boto: + mock_boto.return_value = MagicMock() + result = app.check_training_data_s3_versioning(inv) + assert result["status"] != "ERROR", ( + "S3 check should not be affected by guardrail inventory failure" + ) + + +# --------------------------------------------------------------------------- +# 3. Single-inventory failure — S3 inventory +# --------------------------------------------------------------------------- + + +class TestS3InventoryUnavailable: + """S3 inventory unavailable → FS-21 and FS-46 become COULD_NOT_ASSESS; + other inventories' dependent checks are unaffected.""" + + _ACCESS_DENIED = PermissionError("AccessDenied: s3:ListBuckets") + + def _make_inv(self): + return make_resource_inventory(buckets=app._Unavailable(self._ACCESS_DENIED)) + + # --- FS-21 (check_training_data_s3_versioning) --- + def test_fs21_becomes_could_not_assess(self): + """Validates: Requirements REQ-4.2""" + result = app.check_training_data_s3_versioning(self._make_inv()) + assert _is_could_not_assess(result), ( + f"Expected COULD_NOT_ASSESS but got status={result['status']!r}" + ) + + # --- FS-46 (check_data_classification_tagging) --- + def test_fs46_becomes_could_not_assess(self): + """Validates: Requirements REQ-4.2""" + result = app.check_data_classification_tagging(self._make_inv()) + assert _is_could_not_assess(result), ( + f"Expected COULD_NOT_ASSESS but got status={result['status']!r}" + ) + + # --- Guardrail check unaffected when S3 is unavailable --- + def test_guardrail_check_unaffected(self): + """REQ-4.3 / REQ-8: Guardrail check is unaffected by S3 inventory failure.""" + inv = self._make_inv() + result = app.check_guardrail_contextual_grounding(inv) + assert result["status"] != "ERROR", ( + "Guardrail check should not be affected by S3 inventory failure" + ) + + # --- KB check unaffected when S3 is unavailable --- + def test_kb_check_unaffected(self): + """REQ-4.3 / REQ-8: KB check is unaffected by S3 inventory failure.""" + inv = self._make_inv() + result = app.check_knowledge_base_metadata_filtering(inv) + assert result["status"] != "ERROR", ( + "KB check should not be affected by S3 inventory failure" + ) + + +# --------------------------------------------------------------------------- +# 4. Single-inventory failure — WAFv2 inventory +# --------------------------------------------------------------------------- + + +class TestWafInventoryUnavailable: + """WAFv2 inventory unavailable → FS-01, FS-53, FS-56, FS-68 become + COULD_NOT_ASSESS; lambda, guardrail, S3, KB checks are unaffected.""" + + _ACCESS_DENIED = PermissionError("AccessDenied: wafv2:ListWebACLs") + + def _make_inv(self): + return make_resource_inventory(web_acls=app._Unavailable(self._ACCESS_DENIED)) + + # --- FS-01 (check_waf_shield_on_bedrock_endpoints) --- + def test_fs01_becomes_could_not_assess(self): + """Validates: Requirements REQ-4.2""" + with patch( + "app.boto3.client", return_value=_make_shield_mock_no_subscription() + ): + result = app.check_waf_shield_on_bedrock_endpoints(self._make_inv()) + assert _is_could_not_assess(result), ( + f"Expected COULD_NOT_ASSESS but got status={result['status']!r}" + ) + + # --- FS-53 (check_waf_sql_injection_rules) --- + def test_fs53_becomes_could_not_assess(self): + """Validates: Requirements REQ-4.2""" + result = app.check_waf_sql_injection_rules(self._make_inv()) + assert _is_could_not_assess(result), ( + f"Expected COULD_NOT_ASSESS but got status={result['status']!r}" + ) + + # --- FS-56 (check_xss_prevention_waf) --- + def test_fs56_becomes_could_not_assess(self): + """Validates: Requirements REQ-4.2""" + result = app.check_xss_prevention_waf(self._make_inv()) + assert _is_could_not_assess(result), ( + f"Expected COULD_NOT_ASSESS but got status={result['status']!r}" + ) + + # --- Lambda check unaffected when WAFv2 is unavailable --- + def test_lambda_check_unaffected(self): + """REQ-4.3 / REQ-8: Lambda check is unaffected by WAFv2 inventory failure.""" + inv = self._make_inv() + with patch("app.boto3.client") as mock_boto: + mock_boto.return_value = MagicMock( + get_function_concurrency=MagicMock( + return_value={"ReservedConcurrentExecutions": 5} + ) + ) + result = app.check_agent_transaction_limits(inv) + assert result["status"] != "ERROR", ( + "Lambda check should not be affected by WAFv2 inventory failure" + ) + + # --- Guardrail check unaffected when WAFv2 is unavailable --- + def test_guardrail_check_unaffected(self): + """REQ-4.3 / REQ-8: Guardrail check is unaffected by WAFv2 inventory failure.""" + inv = self._make_inv() + result = app.check_guardrail_contextual_grounding(inv) + assert result["status"] != "ERROR", ( + "Guardrail check should not be affected by WAFv2 inventory failure" + ) + + # --- S3 check unaffected when WAFv2 is unavailable --- + def test_s3_check_unaffected(self): + """REQ-4.3 / REQ-8: S3 check is unaffected by WAFv2 inventory failure.""" + inv = self._make_inv() + with patch("app.boto3.client") as mock_boto: + mock_boto.return_value = MagicMock() + result = app.check_training_data_s3_versioning(inv) + assert result["status"] != "ERROR", ( + "S3 check should not be affected by WAFv2 inventory failure" + ) + + +# --------------------------------------------------------------------------- +# 5. Single-inventory failure — Knowledge Base inventory +# --------------------------------------------------------------------------- + + +class TestKbInventoryUnavailable: + """KB inventory unavailable → FS-24, FS-31, FS-33, FS-48, FS-61, FS-65 + become COULD_NOT_ASSESS; other inventories' checks are unaffected.""" + + _ACCESS_DENIED = PermissionError("AccessDenied: bedrock-agent:ListKnowledgeBases") + + def _make_inv(self): + return make_resource_inventory( + knowledge_bases=app._Unavailable(self._ACCESS_DENIED) + ) + + # --- FS-24 (check_knowledge_base_metadata_filtering) --- + def test_fs24_becomes_could_not_assess(self): + """Validates: Requirements REQ-4.2""" + result = app.check_knowledge_base_metadata_filtering(self._make_inv()) + assert _is_could_not_assess(result), ( + f"Expected COULD_NOT_ASSESS but got status={result['status']!r}" + ) + + # --- FS-31 (check_knowledge_base_data_source_sync) --- + def test_fs31_becomes_could_not_assess(self): + """Validates: Requirements REQ-4.2""" + result = app.check_knowledge_base_data_source_sync(self._make_inv()) + assert _is_could_not_assess(result), ( + f"Expected COULD_NOT_ASSESS but got status={result['status']!r}" + ) + + # --- FS-48 (check_rag_knowledge_base_configured) --- + def test_fs48_becomes_could_not_assess(self): + """Validates: Requirements REQ-4.2""" + result = app.check_rag_knowledge_base_configured(self._make_inv()) + assert _is_could_not_assess(result), ( + f"Expected COULD_NOT_ASSESS but got status={result['status']!r}" + ) + + # --- Guardrail check unaffected when KB is unavailable --- + def test_guardrail_check_unaffected(self): + """REQ-4.3 / REQ-8: Guardrail check is unaffected by KB inventory failure.""" + inv = self._make_inv() + result = app.check_guardrail_contextual_grounding(inv) + assert result["status"] != "ERROR", ( + "Guardrail check should not be affected by KB inventory failure" + ) + + # --- Lambda check unaffected when KB is unavailable --- + def test_lambda_check_unaffected(self): + """REQ-4.3 / REQ-8: Lambda check is unaffected by KB inventory failure.""" + inv = self._make_inv() + with patch("app.boto3.client") as mock_boto: + mock_boto.return_value = MagicMock( + get_function_concurrency=MagicMock( + return_value={"ReservedConcurrentExecutions": 5} + ) + ) + result = app.check_agent_transaction_limits(inv) + assert result["status"] != "ERROR", ( + "Lambda check should not be affected by KB inventory failure" + ) + + # --- S3 check unaffected when KB is unavailable --- + def test_s3_check_unaffected(self): + """REQ-4.3 / REQ-8: S3 check is unaffected by KB inventory failure.""" + inv = self._make_inv() + with patch("app.boto3.client") as mock_boto: + mock_boto.return_value = MagicMock() + result = app.check_training_data_s3_versioning(inv) + assert result["status"] != "ERROR", ( + "S3 check should not be affected by KB inventory failure" + ) + + # --- WAFv2 check unaffected when KB is unavailable --- + def test_waf_check_unaffected(self): + """REQ-4.3 / REQ-8: WAFv2 check is unaffected by KB inventory failure.""" + inv = self._make_inv() + with patch( + "app.boto3.client", return_value=_make_shield_mock_no_subscription() + ): + result = app.check_waf_shield_on_bedrock_endpoints(inv) + assert result["status"] != "ERROR", ( + "WAFv2 check should not be affected by KB inventory failure" + ) + + +# --------------------------------------------------------------------------- +# 6. Multiple-inventory failure — independent sentinels, run completes +# --------------------------------------------------------------------------- + + +class TestMultipleInventoryFailure: + """REQ-4.6: multiple simultaneous inventory failures are each recorded + independently; the run completes and checks on available inventories + produce normal results.""" + + def _make_inv_guardrails_and_web_acls_unavailable(self): + return make_resource_inventory( + guardrails=app._Unavailable( + PermissionError("AccessDenied: bedrock:ListGuardrails") + ), + web_acls=app._Unavailable( + PermissionError("AccessDenied: wafv2:ListWebACLs") + ), + ) + + def _make_inv_lambda_and_s3_unavailable(self): + return make_resource_inventory( + lambda_functions=app._Unavailable( + PermissionError("AccessDenied: lambda:ListFunctions") + ), + buckets=app._Unavailable(PermissionError("AccessDenied: s3:ListBuckets")), + ) + + # --- Guardrails AND WAFv2 both unavailable --- + + def test_guardrail_check_could_not_assess_when_guardrails_unavailable(self): + """Validates: Requirements REQ-4.6 — guardrail failure independently recorded.""" + inv = self._make_inv_guardrails_and_web_acls_unavailable() + result = app.check_guardrail_contextual_grounding(inv) + assert _is_could_not_assess(result), ( + "FS-27 should be COULD_NOT_ASSESS when guardrails inventory is unavailable" + ) + + def test_waf_check_could_not_assess_when_web_acls_unavailable(self): + """Validates: Requirements REQ-4.6 — WAFv2 failure independently recorded.""" + inv = self._make_inv_guardrails_and_web_acls_unavailable() + result = app.check_waf_sql_injection_rules(inv) + assert _is_could_not_assess(result), ( + "FS-53 should be COULD_NOT_ASSESS when web_acls inventory is unavailable" + ) + + def test_kb_check_normal_when_guardrails_and_waf_unavailable(self): + """REQ-4.3: KB check produces a normal result despite guardrail+WAFv2 failures.""" + inv = self._make_inv_guardrails_and_web_acls_unavailable() + result = app.check_knowledge_base_metadata_filtering(inv) + assert result["status"] != "ERROR", ( + "KB check should not be affected when guardrails and WAFv2 are unavailable" + ) + + def test_s3_check_normal_when_guardrails_and_waf_unavailable(self): + """REQ-4.3: S3 check is unaffected by guardrail and WAFv2 failures.""" + inv = self._make_inv_guardrails_and_web_acls_unavailable() + with patch("app.boto3.client") as mock_boto: + mock_boto.return_value = MagicMock() + result = app.check_training_data_s3_versioning(inv) + assert result["status"] != "ERROR", ( + "S3 check should not be affected when guardrails and WAFv2 are unavailable" + ) + + # --- Lambda AND S3 both unavailable --- + + def test_lambda_check_could_not_assess_when_lambda_unavailable(self): + """Validates: Requirements REQ-4.6 — lambda failure independently recorded.""" + inv = self._make_inv_lambda_and_s3_unavailable() + result = app.check_agent_transaction_limits(inv) + assert _is_could_not_assess(result), ( + "FS-09 should be COULD_NOT_ASSESS when lambda_functions inventory is unavailable" + ) + + def test_s3_check_could_not_assess_when_s3_unavailable(self): + """Validates: Requirements REQ-4.6 — S3 failure independently recorded.""" + inv = self._make_inv_lambda_and_s3_unavailable() + result = app.check_training_data_s3_versioning(inv) + assert _is_could_not_assess(result), ( + "FS-21 should be COULD_NOT_ASSESS when buckets inventory is unavailable" + ) + + def test_guardrail_check_normal_when_lambda_and_s3_unavailable(self): + """REQ-4.3: Guardrail check is unaffected by lambda+S3 failures.""" + inv = self._make_inv_lambda_and_s3_unavailable() + result = app.check_guardrail_contextual_grounding(inv) + assert result["status"] != "ERROR", ( + "Guardrail check should not be affected when lambda and S3 are unavailable" + ) + + def test_kb_check_normal_when_lambda_and_s3_unavailable(self): + """REQ-4.3: KB check is unaffected by lambda+S3 failures.""" + inv = self._make_inv_lambda_and_s3_unavailable() + result = app.check_knowledge_base_metadata_filtering(inv) + assert result["status"] != "ERROR", ( + "KB check should not be affected when lambda and S3 are unavailable" + ) + + def test_waf_check_normal_when_lambda_and_s3_unavailable(self): + """REQ-4.3: WAFv2 check is unaffected by lambda+S3 failures.""" + inv = self._make_inv_lambda_and_s3_unavailable() + with patch( + "app.boto3.client", return_value=_make_shield_mock_no_subscription() + ): + result = app.check_waf_shield_on_bedrock_endpoints(inv) + assert result["status"] != "ERROR", ( + "WAFv2 check should not be affected when lambda and S3 are unavailable" + ) + + # --- All five inventories unavailable simultaneously --- + + def test_all_inventories_unavailable_run_still_completes(self): + """REQ-4.6: When all inventories fail, calling each dependent check + individually still returns a result (no unhandled exception propagates).""" + err = PermissionError("AccessDenied: all inventories") + inv = make_resource_inventory( + lambda_functions=app._Unavailable(err), + guardrails=app._Unavailable(err), + knowledge_bases=app._Unavailable(err), + buckets=app._Unavailable(err), + web_acls=app._Unavailable(err), + ) + + checks_and_kwargs = [ + (app.check_agent_transaction_limits, {"inventory": inv}), + (app.check_guardrail_contextual_grounding, {"inventory": inv}), + (app.check_knowledge_base_metadata_filtering, {"inventory": inv}), + (app.check_training_data_s3_versioning, {"inventory": inv}), + (app.check_waf_sql_injection_rules, {"inventory": inv}), + (app.check_guardrail_denied_topics_financial, {"inventory": inv}), + (app.check_guardrail_content_filters, {"inventory": inv}), + (app.check_knowledge_base_data_source_sync, {"inventory": inv}), + (app.check_rag_knowledge_base_configured, {"inventory": inv}), + (app.check_bedrock_sdk_version_currency, {"inventory": inv}), + (app.check_output_validation_lambda, {"inventory": inv}), + (app.check_waf_sql_injection_rules, {"inventory": inv}), + ] + + for check_fn, kwargs in checks_and_kwargs: + # Must not raise; must return a dict with status and csv_data + result = check_fn(**kwargs) + assert isinstance(result, dict), ( + f"{check_fn.__name__} raised instead of returning a result" + ) + assert "status" in result + assert "csv_data" in result + assert _is_could_not_assess(result), ( + f"{check_fn.__name__} should be COULD_NOT_ASSESS when all inventories unavailable; " + f"got status={result['status']!r}" + ) + + +# --------------------------------------------------------------------------- +# 7. Full handler run with partial-inventory failure +# --------------------------------------------------------------------------- + + +class TestHandlerWithPartialInventory: + """Verify that when collect_resource_inventory returns a partially-unavailable + ResourceInventory (patched in), the handler still completes (statusCode=200) + and synthesizes visible COULD_NOT_ASSESS rows for the affected checks while + other checks produce normal rows. + + Validates: Requirements REQ-4.2, REQ-4.3, REQ-4.6 + """ + + def _make_generic_mock_client(self): + generic = MagicMock() + paginator = MagicMock() + paginator.paginate.return_value = [{}] + generic.get_paginator.return_value = paginator + generic.list_web_acls.return_value = {"WebACLs": []} + + # FS-01 uses shield.exceptions.ResourceNotFoundException — set up a proper + # exception class so describe_subscription's side_effect is caught correctly. + class ResourceNotFoundException(Exception): + pass + + class FakeExceptions: + pass + + FakeExceptions.ResourceNotFoundException = ResourceNotFoundException + generic.exceptions = FakeExceptions + generic.describe_subscription.side_effect = ResourceNotFoundException("no sub") + + generic.get_usage_plans.return_value = {"items": []} + generic.list_service_quotas.return_value = {"Quotas": []} + generic.get_anomaly_monitors.return_value = {"AnomalyMonitors": []} + generic.describe_budgets.return_value = {"Budgets": []} + generic.get_caller_identity.return_value = {"Account": "123456789012"} + generic.list_agents.return_value = {"agentSummaries": []} + generic.list_agent_runtimes.return_value = {"agentRuntimes": []} + generic.list_functions.return_value = {"Functions": []} + generic.list_state_machines.return_value = {"stateMachines": []} + generic.list_policies.return_value = {"Policies": []} + generic.list_custom_models.return_value = {"modelSummaries": []} + generic.list_models.return_value = {"Models": []} + generic.describe_config_rules.return_value = {"ConfigRules": []} + generic.list_evaluation_jobs.return_value = {"jobSummaries": []} + generic.describe_repositories.return_value = {"repositories": []} + generic.list_feature_groups.return_value = {"FeatureGroupSummaries": []} + generic.list_buckets.return_value = {"Buckets": []} + generic.list_knowledge_bases.return_value = {"knowledgeBaseSummaries": []} + generic.list_guardrails.return_value = {"guardrails": []} + generic.list_log_groups.return_value = {"logGroups": []} + generic.get_macie_session.side_effect = Exception("not enabled") + generic.list_foundation_models.return_value = {"modelSummaries": []} + generic.list_model_cards.return_value = {"ModelCardSummaries": []} + generic.list_rules.return_value = {"Rules": []} + generic.list_schedules.return_value = {"Schedules": []} + generic.get_rest_apis.return_value = {"items": []} + generic.list_processing_jobs.return_value = {"ProcessingJobSummaries": []} + generic.list_automated_reasoning_policies.return_value = { + "automatedReasoningPolicySummaries": [] + } + return generic + + @patch("app.write_to_s3") + @patch("app.get_permissions_cache") + @patch("app.collect_resource_inventory") + @patch("app.boto3.client") + def test_handler_completes_with_lambda_inventory_unavailable( + self, + mock_boto_client, + mock_collect_inv, + mock_cache, + mock_s3, + lambda_event, + ): + """Handler returns 200 and all 65 findings are present (COULD_NOT_ASSESS + rows are synthesized for lambda-dependent checks) when lambda inventory fails. + + Validates: Requirements REQ-4.2, REQ-4.3, REQ-4.6 + """ + err = PermissionError("AccessDenied: lambda:ListFunctions") + partial_inv = make_resource_inventory(lambda_functions=app._Unavailable(err)) + mock_collect_inv.return_value = partial_inv + mock_boto_client.return_value = self._make_generic_mock_client() + mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} + mock_s3.return_value = "https://test-bucket.s3.amazonaws.com/report.csv" + + result = app.lambda_handler(lambda_event, None) + + assert result["statusCode"] == 200 + findings = result["body"]["findings"] + # All 65 registry entries must produce a result dict + assert len(findings) == 65 + + # Lambda-dependent checks should have a synthesized COULD_NOT_ASSESS row + # in their csv_data (the handler's guard calls _could_not_assess_row for + # checks that return empty csv_data — design DD-3). + lambda_dependent_check_names = { + "Agent Transaction Limits Check", # FS-09 + "Bedrock SDK Version Currency Check", # FS-52 + "Output Validation Lambda Check", # FS-55 + "Output Schema Validation Check", # FS-58 + } + + for finding in findings: + check_name = finding.get("check_name", "") + rows = finding.get("csv_data", []) + if check_name in lambda_dependent_check_names: + # The handler synthesizes a COULD_NOT_ASSESS row and appends it + # to csv_data before returning — so we expect exactly 1 row with + # the COULD NOT ASSESS prefix and Status="N/A". + assert len(rows) == 1, ( + f"{check_name} should have 1 synthesized COULD_NOT_ASSESS row, " + f"got {len(rows)}: {rows!r}" + ) + assert rows[0]["Finding"].startswith(app.COULD_NOT_ASSESS_PREFIX), ( + f"{check_name} row Finding should start with COULD NOT ASSESS prefix" + ) + # StatusEnum.NA == "N/A" because StatusEnum inherits from str + assert rows[0]["Status"] == "N/A", ( + f"{check_name} COULD_NOT_ASSESS row should have Status=N/A, " + f"got {rows[0]['Status']!r}" + ) + + @patch("app.write_to_s3") + @patch("app.get_permissions_cache") + @patch("app.collect_resource_inventory") + @patch("app.boto3.client") + def test_handler_completes_with_guardrails_and_waf_unavailable( + self, + mock_boto_client, + mock_collect_inv, + mock_cache, + mock_s3, + lambda_event, + ): + """Handler returns 200 with all 65 findings when guardrails and WAFv2 fail. + + Validates: Requirements REQ-4.6 — multiple independent failures + """ + partial_inv = make_resource_inventory( + guardrails=app._Unavailable( + PermissionError("AccessDenied: bedrock:ListGuardrails") + ), + web_acls=app._Unavailable( + PermissionError("AccessDenied: wafv2:ListWebACLs") + ), + ) + mock_collect_inv.return_value = partial_inv + mock_boto_client.return_value = self._make_generic_mock_client() + mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} + mock_s3.return_value = "https://test-bucket.s3.amazonaws.com/report.csv" + + result = app.lambda_handler(lambda_event, None) + + assert result["statusCode"] == 200 + assert len(result["body"]["findings"]) == 65 diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_schema.py b/aiml-security-assessment/functions/security/finserv_tests/test_schema.py new file mode 100644 index 0000000..5daead5 --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_tests/test_schema.py @@ -0,0 +1,339 @@ +""" +Tests for finserv_assessments/schema.py + +Covers: + - Valid finding creation for all severity/status combinations + - Check_ID pattern validation (FS-NN, BR-NN, AC-NN, SM-NN) + - Reference URL validation (must start with https://) + - Required field presence and min_length constraints + - Pydantic model_dump output structure +""" + +import pytest +from schema import Finding, SeverityEnum, StatusEnum, create_finding +from app import COMPLIANCE_MAP, build_finserv_checks + + +# ========================================================================= +# Enum completeness +# ========================================================================= + + +class TestEnums: + def test_severity_values(self): + assert set(SeverityEnum) == { + SeverityEnum.HIGH, + SeverityEnum.MEDIUM, + SeverityEnum.LOW, + SeverityEnum.INFORMATIONAL, + } + + def test_status_values(self): + assert set(StatusEnum) == { + StatusEnum.FAILED, + StatusEnum.PASSED, + StatusEnum.NA, + } + + def test_severity_string_values(self): + assert SeverityEnum.HIGH.value == "High" + assert SeverityEnum.MEDIUM.value == "Medium" + assert SeverityEnum.LOW.value == "Low" + assert SeverityEnum.INFORMATIONAL.value == "Informational" + + def test_status_string_values(self): + assert StatusEnum.FAILED.value == "Failed" + assert StatusEnum.PASSED.value == "Passed" + assert StatusEnum.NA.value == "N/A" + + +# ========================================================================= +# create_finding — happy paths +# ========================================================================= + + +class TestCreateFindingValid: + """Every severity × status combination should produce a valid dict.""" + + @pytest.mark.parametrize("severity", list(SeverityEnum)) + @pytest.mark.parametrize("status", list(StatusEnum)) + def test_all_severity_status_combos(self, severity, status): + result = create_finding( + check_id="FS-01", + finding_name="Test Finding", + finding_details="Some details here", + resolution="Fix it", + reference="https://docs.aws.amazon.com/example", + severity=severity.value, + status=status.value, + ) + assert isinstance(result, dict) + assert result["Check_ID"] == "FS-01" + assert result["Severity"] == severity.value + assert result["Status"] == status.value + + def test_output_has_all_csv_fields(self): + result = create_finding( + check_id="FS-42", + finding_name="Name", + finding_details="Details", + resolution="Resolution", + reference="https://example.com", + severity="High", + status="Failed", + ) + expected_keys = { + "Check_ID", + "Finding", + "Finding_Details", + "Resolution", + "Reference", + "Severity", + "Status", + "Compliance_Frameworks", + } + assert set(result.keys()) == expected_keys + + @pytest.mark.parametrize( + "check_id", + ["FS-01", "FS-69", "BR-14", "SM-07", "AC-05"], + ) + def test_valid_check_id_patterns(self, check_id): + result = create_finding( + check_id=check_id, + finding_name="Test", + finding_details="Details", + resolution="", + reference="https://example.com", + severity="Low", + status="Passed", + ) + assert result["Check_ID"] == check_id + + def test_empty_resolution_allowed(self): + """Resolution has min_length=0, so empty string is valid.""" + result = create_finding( + check_id="FS-01", + finding_name="Test", + finding_details="Details", + resolution="", + reference="https://example.com", + severity="Informational", + status="Passed", + ) + assert result["Resolution"] == "" + + +# ========================================================================= +# create_finding — validation errors +# ========================================================================= + + +class TestCreateFindingInvalid: + def test_invalid_check_id_pattern(self): + with pytest.raises(Exception): + create_finding( + check_id="INVALID", + finding_name="Test", + finding_details="Details", + resolution="Fix", + reference="https://example.com", + severity="High", + status="Failed", + ) + + def test_check_id_lowercase_rejected(self): + with pytest.raises(Exception): + create_finding( + check_id="fs-01", + finding_name="Test", + finding_details="Details", + resolution="Fix", + reference="https://example.com", + severity="High", + status="Failed", + ) + + def test_check_id_missing_dash(self): + with pytest.raises(Exception): + create_finding( + check_id="FS01", + finding_name="Test", + finding_details="Details", + resolution="Fix", + reference="https://example.com", + severity="High", + status="Failed", + ) + + def test_reference_not_https(self): + with pytest.raises(Exception): + create_finding( + check_id="FS-01", + finding_name="Test", + finding_details="Details", + resolution="Fix", + reference="http://example.com", + severity="High", + status="Failed", + ) + + def test_empty_finding_name_rejected(self): + with pytest.raises(Exception): + create_finding( + check_id="FS-01", + finding_name="", + finding_details="Details", + resolution="Fix", + reference="https://example.com", + severity="High", + status="Failed", + ) + + def test_empty_finding_details_rejected(self): + with pytest.raises(Exception): + create_finding( + check_id="FS-01", + finding_name="Test", + finding_details="", + resolution="Fix", + reference="https://example.com", + severity="High", + status="Failed", + ) + + def test_invalid_severity(self): + with pytest.raises(Exception): + create_finding( + check_id="FS-01", + finding_name="Test", + finding_details="Details", + resolution="Fix", + reference="https://example.com", + severity="Critical", + status="Failed", + ) + + def test_invalid_status(self): + with pytest.raises(Exception): + create_finding( + check_id="FS-01", + finding_name="Test", + finding_details="Details", + resolution="Fix", + reference="https://example.com", + severity="High", + status="Open", + ) + + +# ========================================================================= +# Finding model direct instantiation +# ========================================================================= + + +class TestFindingModel: + def test_model_dump_returns_dict(self): + f = Finding( + Check_ID="FS-01", + Finding="Test", + Finding_Details="Details", + Resolution="Fix", + Reference="https://example.com", + Severity=SeverityEnum.HIGH, + Status=StatusEnum.FAILED, + ) + d = f.model_dump() + assert isinstance(d, dict) + assert d["Check_ID"] == "FS-01" + + def test_check_id_three_letter_prefix(self): + """Three-letter prefixes like ACM-01 should be valid.""" + f = Finding( + Check_ID="ACM-01", + Finding="Test", + Finding_Details="Details", + Resolution="Fix", + Reference="https://example.com", + Severity=SeverityEnum.LOW, + Status=StatusEnum.PASSED, + ) + assert f.Check_ID == "ACM-01" + + +# ========================================================================= +# D3 fix — Compliance_Frameworks field in schema and COMPLIANCE_MAP coverage +# ========================================================================= + + +class TestComplianceFrameworksField: + def test_compliance_frameworks_defaults_to_empty_string(self): + """compliance_frameworks is optional and defaults to empty string.""" + result = create_finding( + check_id="FS-01", + finding_name="Test", + finding_details="Details", + resolution="Fix", + reference="https://example.com", + severity="High", + status="Failed", + ) + assert "Compliance_Frameworks" in result + assert result["Compliance_Frameworks"] == "" + + def test_compliance_frameworks_round_trips(self): + """A populated compliance_frameworks value is preserved verbatim.""" + fw = "FFIEC CAT | SR 11-7 | NYDFS 500" + result = create_finding( + check_id="FS-01", + finding_name="Test", + finding_details="Details", + resolution="Fix", + reference="https://example.com", + severity="High", + status="Failed", + compliance_frameworks=fw, + ) + assert result["Compliance_Frameworks"] == fw + + def test_compliance_map_covers_all_registry_checks(self): + """Every check in build_finserv_checks must have an entry in COMPLIANCE_MAP.""" + registry = build_finserv_checks({}) + registry_ids = {check_id for check_id, _ in registry} + missing = registry_ids - set(COMPLIANCE_MAP.keys()) + assert not missing, ( + f"These FS IDs are in the registry but missing from COMPLIANCE_MAP: {sorted(missing)}" + ) + + def test_compliance_map_all_values_non_empty(self): + """Every COMPLIANCE_MAP entry must have at least one framework.""" + empty = [k for k, v in COMPLIANCE_MAP.items() if not v.strip()] + assert not empty, f"Empty compliance_frameworks values: {empty}" + + def test_compliance_map_values_pipe_separated(self): + """All multi-framework values use ' | ' as separator (consistent with ASFF style).""" + for check_id, value in COMPLIANCE_MAP.items(): + if "|" in value: + # Each segment should be non-empty after strip + parts = [p.strip() for p in value.split("|")] + empty_parts = [p for p in parts if not p] + assert not empty_parts, f"{check_id}: empty segment in '{value}'" + + def test_compliance_frameworks_present_in_finding_model_dump(self): + """Finding.model_dump() includes Compliance_Frameworks — critical for CSV writer.""" + f = Finding( + Check_ID="FS-29", + Finding="ADVISORY: Test", + Finding_Details="Details", + Resolution="Fix", + Reference="https://example.com", + Severity=SeverityEnum.INFORMATIONAL, + Status=StatusEnum.NA, + Compliance_Frameworks="SR 11-7 | FFIEC CAT | NYDFS 500 | MAS TRM 9.2", + ) + d = f.model_dump() + assert "Compliance_Frameworks" in d + assert ( + d["Compliance_Frameworks"] + == "SR 11-7 | FFIEC CAT | NYDFS 500 | MAS TRM 9.2" + ) diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_severity_register.py b/aiml-security-assessment/functions/security/finserv_tests/test_severity_register.py new file mode 100644 index 0000000..64ce06a --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_tests/test_severity_register.py @@ -0,0 +1,132 @@ +""" +Drift-guard for the FinServ severity methodology (REQ-6 / severity-register.md). + +Asserts that: + 1. The Likelihood x Impact matrix helper matches the documented table (methodology §3.3). + 2. The disposition->severity rules match methodology §3.4. + 3. SEVERITY_REGISTER uses only the four allowed labels (no Critical this round). + 4. Every finding-name emitted by the check registry exists in SEVERITY_REGISTER, and the + emitted `severity=` equals the register value (prevents future drift across 64 checks). + +The boto3 calls in every check are mocked to raise, which forces each check down a path that +still emits its finding rows (or is caught), so we can collect the (finding_name, severity) +pairs the code actually produces without real AWS access. +""" + +import sys +import os +from unittest.mock import MagicMock, patch + +FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") +if FINSERV_DIR not in sys.path: + sys.path.insert(0, FINSERV_DIR) + +import app # noqa: E402 + +ALLOWED = {"High", "Medium", "Low", "Informational"} + + +# --------------------------------------------------------------------------- +# 1. Matrix helper matches the documented 3x3 table +# --------------------------------------------------------------------------- +def test_label_from_matrix_matches_methodology_table(): + expected = { + (3, 1): "Medium", + (3, 2): "High", + (3, 3): "High", + (2, 1): "Low", + (2, 2): "Medium", + (2, 3): "High", + (1, 1): "Low", + (1, 2): "Low", + (1, 3): "Medium", + } + for (i, ell), label in expected.items(): + assert app._label_from_matrix(i, ell) == label, f"matrix[{i},{ell}]" + + +# --------------------------------------------------------------------------- +# 2. Disposition -> severity rules (methodology §3.4) +# --------------------------------------------------------------------------- +def test_disposition_severity_rules(): + assert app._DISPOSITION_SEVERITY["NOT_APPLICABLE"] == "Informational" + assert app._DISPOSITION_SEVERITY["ADVISORY"] == "Informational" + assert app._DISPOSITION_SEVERITY["COULD_NOT_ASSESS"] == "Low" + + +# --------------------------------------------------------------------------- +# 3. Register uses only allowed labels (no Critical) +# --------------------------------------------------------------------------- +def test_register_labels_are_allowed_no_critical(): + assert app.SEVERITY_REGISTER, "register must not be empty" + bad = {v for v in app.SEVERITY_REGISTER.values() if v not in ALLOWED} + assert not bad, f"register has disallowed labels: {bad}" + + +# --------------------------------------------------------------------------- +# 4. Could-not-assess synthesized row is Low / N/A (COULD_NOT_ASSESS disposition) +# --------------------------------------------------------------------------- +def test_could_not_assess_row_is_low(): + row = app._could_not_assess_row("FS-01", "Some Check", "boom") + assert row["Severity"] == "Low" + assert row["Status"] == "N/A" + assert row["Finding"].startswith(app.COULD_NOT_ASSESS_PREFIX) + + +# --------------------------------------------------------------------------- +# 5. Every emitted finding's severity matches the register (the real drift-guard) +# --------------------------------------------------------------------------- +def _collect_emitted_rows(): + """Run every registered check with all boto3 calls raising, collecting CSV rows. + + A check whose body raises returns _error_findings (no rows) — that is fine; we only + assert on the rows that ARE emitted. Advisory checks (no boto3) emit their row directly. + """ + rows = [] + cache = {"role_permissions": {}, "user_permissions": {}} + + def boom_client(*_a, **_k): + m = MagicMock() + # Any attribute access returns a callable that raises, so checks that call + # boto3 fall into their except/_error_findings path deterministically. + m.side_effect = Exception("no aws in unit test") + + def _raise(*_aa, **_kk): + raise Exception("no aws in unit test") + + m.__getattr__ = lambda _name: _raise + return m + + with patch.object(app.boto3, "client", side_effect=boom_client): + for check_id, fn in app.build_finserv_checks(cache): + try: + result = fn() + except Exception: + continue + for row in result.get("csv_data", []): + rows.append((row["Finding"], str(row["Severity"]), str(row["Status"]))) + return rows + + +def test_emitted_severity_matches_register(): + """Advisory/static-path checks emit rows even with boto3 down; assert they match.""" + rows = _collect_emitted_rows() + # We only check rows whose finding-name is a static name in the register + # (could-not-assess rows use a dynamic name and are validated separately). + mismatches = [] + for finding, severity, _status in rows: + sev = severity.split(".")[-1].title() if "." in severity else severity + if finding in app.SEVERITY_REGISTER and app.SEVERITY_REGISTER[finding] != sev: + mismatches.append((finding, sev, app.SEVERITY_REGISTER[finding])) + assert not mismatches, f"severity drift: {mismatches}" + + +def test_advisory_rows_are_informational_na(): + """All ADVISORY-prefixed findings must be Informational / N/A.""" + rows = _collect_emitted_rows() + for finding, severity, status in rows: + if finding.startswith("ADVISORY: "): + sev = severity.split(".")[-1].title() if "." in severity else severity + st = status.split(".")[-1] if "." in status else status + assert sev == "Informational", f"{finding} severity={sev}" + assert st in ("NA", "N/A"), f"{finding} status={st}" diff --git a/docs/SECURITY_CHECKS_FINSERV_SEVERITY_METHODOLOGY.md b/docs/SECURITY_CHECKS_FINSERV_SEVERITY_METHODOLOGY.md new file mode 100644 index 0000000..c28fc8e --- /dev/null +++ b/docs/SECURITY_CHECKS_FINSERV_SEVERITY_METHODOLOGY.md @@ -0,0 +1,178 @@ +# FinServ GenAI Check Severity Methodology + +**Status:** Proposed (Round 3). This document is the authoritative reference for how every +FinServ (`FS-`) check is assigned a severity. It answers reviewer Finding 6 ("How are the priorities of the findings determined?") with a concrete reproducible, industry- and AWS-aligned formula — not per-check intuition. + +> Scope note: today the FinServ checks and the upstream Bedrock/SageMaker/AgentCore checks all use ad-hoc severities with **no documente methodology** (verified by inspection — the upstream `app.py` files hardcode `severity="High"|"Medium"|...` with no rationale and no rubric doc). This methodology is introduced for the FinServ checks first and is written so it can later be adopted tool-wide. + +--- + +## 1. Research basis (authoritative sources reviewed) + +| Standard | What it contributes | Why we did / didn't adopt it wholesale | +|---|---|---| +| **AWS Security Hub ASFF Severity** ([API_Severity](https://docs.aws.amazon.com/securityhub/1.0/APIReference/API_Severity.html)) | The AWS-native label set (`INFORMATIONAL/LOW/MEDIUM/HIGH/CRITICAL`) with precise semantics and normalized 0–100 ranges. | **Adopted as the target label vocabulary** so findings align with Security Hub, the service customers use to aggregate posture findings. | +| **AWS exposure-finding severity factors** ([doc](https://docs.aws.amazon.com/securityhub/latest/userguide/exposure-findings-severity.html)) | AWS's own model: *Awareness, Ease of discovery, Ease of exploit, Likelihood of exploit, Impact* — i.e. **Likelihood × Impact**. | **Adopted the Likelihood × Impact shape**; AWS itself uses it, so it is the most defensible AWS-aligned model. | +| **NIST SP 800-30 r1** ([CSRC](https://csrc.nist.gov/pubs/sp/800/30/final)) | Risk = Likelihood × Impact, a 5×5 qualitative matrix with a published lookup table. | **Adopted the matrix-lookup approach** (foundational US-government risk standard; FinServ regulators expect NIST-lineage rigor). Simplified to 3×3 for explainability. | +| **OWASP Risk Rating Methodology** ([OWASP](https://owasp.org/www-community/OWASP_Risk_Rating_Methodology)) | Likelihood (threat-agent + vulnerability) × Impact (technical + business), averaged and banded LOW/MEDIUM/HIGH. | **Adopted the factor-decomposition idea** (score sub-factors, then combine) and the business-impact dimension. | +| **CVSS v3.1** ([FIRST](https://www.first.org/cvss/)) qualitative bands (0 None / 0.1–3.9 Low / 4.0–6.9 Medium / 7.0–8.9 High / 9.0–10.0 Critical) | Standard numeric→qualitative banding. | **Referenced** for band shape; **not adopted wholesale** — CVSS scores *software vulnerabilities (CVEs)*, not *missing-control posture findings*. Using CVSS metrics (Attack Vector, etc.) on a "no WAF configured" finding is a category error. | +| **CISA SSVC** (Stakeholder-Specific Vulnerability Categorization) | Decision-tree (Exploitation/Automatable/Technical Impact/Mission) → Track/Attend/Act. | **Referenced**, not adopted — also CVE/vulnerability-centric and produces action labels, not the severity labels the report and Security Hub expect. | + +**Conclusion:** A control-gap posture tool should score **Likelihood × Impact** (per AWS's own +exposure model and NIST 800-30) and express the result in the **ASFF label set**. CVSS/SSVC are +for CVEs and are explicitly out of scope as the scoring engine, though the report remains +compatible with Security Hub's ASFF labels for customers who ingest it. + +--- + +## 2. The severity scale (ASFF-aligned) + +We use the ASFF labels with AWS's exact semantics. The tool's `SeverityEnum` today is +`High | Medium | Low | Informational` (no `Critical`) and is shared with the upstream services. + +| Label | ASFF meaning | ASFF normalized | Used by FinServ for | +|---|---|---|---| +| **Informational** | No issue / not action-bearing on its own | 0 | Advisory checks (no API to verify) and `N/A` (nothing to assess / could-not-assess) | +| **Low** | Does not require action on its own | 1–39 | Residual-risk / observability controls, or controls with strong compensating alternatives | +| **Medium** | Must be addressed, not urgently | 40–69 | Controls whose absence materially increases risk but is not itself a breach | +| **High** | Must be addressed as a priority | 70–89 | Controls whose absence can directly cause regulatory breach, data exposure, large loss, or full guardrail bypass | +| **Critical** | Remediate immediately | 90–100 | **Not currently used** (see §6 decision). Reserved. | + +--- + +## 3. The scoring model (Likelihood × Impact → label) + +Each **risk a control mitigates** is scored on two axes, each Low(1)/Medium(2)/High(3). Severity +is the inherent risk the control addresses, so the **same severity applies to that check's Passed, +Failed, and (where risk-bearing) N/A rows** — preserving the Round-2 invariant that Passed +findings keep their documented severity. + +### 3.1 Impact (I) — harm if the control is absent and the risk materializes + +| Score | Criteria (any one qualifies) | +|---|---| +| **3 — High** | Direct regulatory breach (e.g., fair-lending/ECOA, disclosure rules); sensitive-data/PII exposure; large-scale financial loss; full bypass of safety guardrails; unsafe automated financial action. | +| **2 — Medium** | Materially weakens oversight, model-risk governance, or assurance; increases blast radius of another failure; degraded auditability of a regulated decision — but not a breach by itself. | +| **1 — Low** | Reduces residual risk, supports observability/audit, or is fully covered by a compensating control; cost-optimization or defense-in-depth value. | + +### 3.2 Likelihood (L) — probability the adverse outcome occurs given the control is absent + +Blends AWS's *awareness / ease of discovery / ease of exploit* with the presence of compensating +controls. Applies to both attack-driven risks (prompt injection, cost exhaustion) and +governance-driven risks (an unreviewed model reaches production). + +| Score | Criteria | +|---|---| +| **3 — High** | Internet-reachable or default-on surface; common, automatable attack pattern; or near-certain to occur in normal operation; no compensating control. | +| **2 — Medium** | Reachable under common conditions; partial or adjacent compensating control exists; periodic rather than continuous exposure. | +| **1 — Low** | Requires unusual conditions or insider access; strong compensating controls substantially reduce exposure; rare in practice. | + +### 3.3 Lookup matrix (3×3 → ASFF label) + +| | **L = Low (1)** | **L = Medium (2)** | **L = High (3)** | +|--------------|-----------------|--------------------|------------------| +| **I = High (3)** | Medium | High | High *(Critical-eligible — see §6)* | +| **I = Medium (2)** | Low | Medium | High | +| **I = Low (1)** | Low | Low | Medium | + +Equivalent rule: `score = I × L`; `1–2 → Low`, `3–4 → Medium`, `6–9 → High` (with the +`I=3,L=3 → 9` cell Critical-eligible). Advisory/non-verifiable and `N/A` outcomes are handled by +the disposition rules in §3.4 (they are not risk-scored). + +### 3.4 Outcome disposition rules (CRITICAL — resolves the N/A inconsistency) + +**Severity is a property of the control (the risk), not the outcome.** A control is scored once +(§3.1–3.3) and that severity is applied to **every** Passed and Failed row of that control +(preserving the Round-2 invariant). The `N/A` family is where the current code is inconsistent +(audit found "nothing to assess" rows tagged High, Medium, AND Informational across checks). Each +row maps to exactly one **disposition**, and the disposition fixes the severity: + +| Disposition | When it applies | Severity | ASFF rationale | +|---|---|---|---| +| **FAIL** | control assessed, not satisfied | control severity (§3.3) | the asserted issue | +| **PASS** | control assessed, satisfied | control severity (§3.3) | Round-2 invariant: pass keeps documented severity | +| **NOT_APPLICABLE** | the control's resource type is absent (no KBs, no guardrails, no WAF, no REST APIs, not in an Org) | **Informational** | ASFF: *"INFORMATIONAL — No issue was found."* The "you should create guardrails/eval jobs" signal belongs to that resource's **own** existence check, not to every sub-check (avoids double-counting). | +| **ADVISORY** | no AWS API can verify the control (app-layer) | **Informational** | Option-B convention (Round 1); `"ADVISORY: "` name prefix | +| **COULD_NOT_ASSESS** | the check could not run (access-denied, unsupported region, SDK gap) | **Low** | not a confirmed issue (unknown state); the `"COULD NOT ASSESS: "` / access-check name keeps it visible; prompts a re-run. Unifies today's Low/Medium split. | +| **SOFT_WARNING** | control assessed; a legitimate-but-suboptimal non-failing state (the only instance is **FS-03 quotas-at-default**, an intentional Round-1 decision) | control severity | documented exception | + +This single table eliminates the audit-found inconsistency: every NOT_APPLICABLE row → Informational; +every COULD_NOT_ASSESS row → Low; every ADVISORY row → Informational. + +> **Check-logic items (now in scope as REQ-10):** a few checks used `N/A`/`Passed` in ways that +> understate or overstate risk. These are fixed in this round: FS-15 "no eval jobs" → `Failed`; +> FS-30/35/40 (cannot inspect dataset content) → advisory; FS-56 gains a real FAIL path. FS-28/36/ +> 51/59 CLASSIC-tier was investigated and intentionally kept `Passed` (CLASSIC provides real +> protection; not deprecated). See REQ-10 and `severity-register.md`. + +### 3.5 Control-family bands (ensures cross-check consistency) + +To guarantee similar controls get the same severity, every FS control is assigned to a family with +a default band. Per-control I×L may refine within ±1 with a documented reason. + +| Family | Risk on absence | Default | Example checks | +|---|---|---|---| +| **Safety-guardrail / content safety** | harmful output, guardrail bypass, PII leak | **High** | FS-36 content, FS-45 PII, FS-47 grounding threshold, FS-51 prompt-attack, FS-53 injection, FS-27 contextual grounding | +| **Sensitive-data exposure / integrity** | PII exposure or training-data tampering | **High** | FS-21 training-data versioning, FS-25 KB encryption, FS-43 log data-protection, FS-44 Macie | +| **Excessive agency / access control / isolation** | unauthorized action, over-broad permissions, regulated-decision breach | **High** | FS-07, FS-08, FS-10, FS-12, FS-22, FS-26, FS-39 bias, FS-41 explainability, FS-66, FS-67 | +| **Regulated-output controls** | non-compliant / off-regulatory output | **High** (denied-topics) / **Medium** (softer: word filters, topic allowlist, relevance) | FS-28 (High), FS-38/FS-59/FS-50 (Medium) | +| **Unbounded consumption / cost / rate-limiting** | cost exhaustion, DoS — compensating controls exist, no breach | **Medium** | FS-01 WAF, FS-02, FS-03, FS-05, FS-06, FS-09, FS-11, FS-68 | +| **Governance / model-risk / monitoring / currency** | weakened oversight/assurance, not a breach | **Medium** | FS-04, FS-13, FS-14, FS-15, FS-20, FS-30/34/35/40/42, FS-31/61, FS-46, FS-48, FS-52, FS-55, FS-63, FS-69 | +| **Premium-cost defense-in-depth** | residual DDoS risk; Shield Standard + WAF compensate; ~$3k/mo | **Low** | FS-01 Shield Advanced | +| **Emerging/advanced advisory control** | formal-verification gap; grounding compensates | **Medium→Low** | FS-27 ARC | +| **Non-verifiable advisory** | app-layer; no API | **Informational** | FS-24, FS-29, FS-32, FS-37, FS-49, FS-54, FS-57, FS-58, FS-60, FS-62 | + +The authoritative per-finding assignments are in +[`severity-register.md`](./severity-register.md). + + + +--- + +## 4. Worked examples (including the reviewer's case) + +| Check | Control | I | L | Rationale | Result | +|---|---|---|---|---|---| +| **FS-01 (Shield Advanced)** | Shield Advanced subscription | **1** | **2** | Impact Low: Shield *Standard* is always-on and free; WAF rate-limiting (FS-01 WAF / FS-02 usage plans) is a compensating control; absence is a premium-cost decision (~$3,000/mo), not a breach. Likelihood Medium: endpoints are discoverable but volumetric DDoS on a Bedrock-fronting endpoint is not the common case. | **Low** *(was High — fixes Finding 6)* | +| **FS-01 (Regional WAF)** | WAF Web ACL present | **2** | **2** | Impact Medium: no WAF → exposed to abusive callers / cost exhaustion, but API Gateway usage-plan throttling (FS-02) is a compensating control and there is no direct breach. Likelihood Medium: common but mitigated by throttling. | **Medium** | +| **FS-43-style (PII in logs / data exposure)** | Sensitive-data masking | **3** | **2** | Impact High: PII exposure = regulatory breach. Likelihood Medium: requires logging misconfig. | **High** | +| **FS-58 (output schema validation)** | App-layer validation | — | — | No AWS API can verify it → advisory. | **Informational** (advisory) | +| **FS-27 ARC (no policies)** | Automated Reasoning policy present | **2** | **1** | Impact Medium: ARC adds formal verification of factual claims; its absence leaves grounding-only assurance. Likelihood Low: ARC is an advanced, rarely-adopted control; contextual grounding (FS-27 grounding) compensates. | **Low/Medium** (confirm in register) | + +--- + +## 5. Application, governance, and drift-prevention + +1. **Per-finding severity register.** A machine-checkable register (`severity-register.csv` or a `SEVERITY_REGISTER` dict in `app.py`) lists every `FS-` finding-name with its `I`, `L`, resulting label, and a one-line justification. This is the single source of truth. + *(FS-01 emits four finding-names under one Check_ID; the register is keyed by finding-name so Shield=Low and WAF=Medium can coexist under FS-01.)* +2. **Code matches register.** Every `create_finding(... severity=...)` must equal the register's label for that finding. A unit test enforces this (prevents future drift) — a strong guard for a public tool. +3. **Docs match register.** The severity columns in `SECURITY_CHECKS_FINSERV_PART1/2/3.md` and the + `Severity rubric` section of `SECURITY_CHECKS_FINSERV_COMMON.md` are regenerated/checked against + the register. The "Advisory" tier in the existing rubric is reconciled (Advisory = the Informational disposition for non-verifiable controls). +4. **Methodology surfaced to users.** A condensed version of §2–§3 is added to the README and + linked from the FinServ report section so the methodology travels with the artifact (directly answering the reviewer). +5. **ASFF mapping documented.** README states the label↔ASFF-normalized mapping so customers who forward findings to Security Hub get correct severities. + +--- + +## 6. Decision (CONFIRMED 2026-06-10): keep four levels; do not introduce `Critical` yet + +The matrix has a Critical-eligible cell (I=High, L=High). Two paths: + +- **Path A — keep four levels (recommended for this PR).** Cap the I=3,L=3 cell at **High**. + Pro: stays consistent with the upstream Bedrock/SageMaker/AgentCore checks (which have no + Critical), no `SeverityEnum`/report/test changes, smaller reviewable PR. Con: a genuinely + critical FinServ risk is reported as High. +- **Path B — adopt `Critical` tool-wide (separate follow-up PR).** Add `CRITICAL` to + `SeverityEnum`, update the report template's severity filters/colors, and re-score the I=3,L=3 + FinServ checks. Pro: full ASFF alignment. Con: cross-cutting change touching all four services, + the schema, the report, and every service's tests — out of scope for a correctness-focused round. + +**Confirmed decision:** **Path A** for this round — keep `{High, Medium, Low, Informational}`. A follow-up issue will evaluate Path B tool-wide. The methodology already documents the `Critical` band (§2), so adopting it later is a labeling change, not a methodology change; until then the drift-guard test asserts no `Critical` is emitted. + +--- + +## 7. How this changes the report (expected, document for reviewers) + +Downgrading FS-01 Shield from High→Low (and any other audit-driven changes) **moves those findings into the Low band, which still counts toward the pass-rate denominator** (only `Informational`/`N/A` are excluded — Round-2 behavior). So pass rates and the High-severity count will shift; this is +intended and must be called out in the PR description with before/after numbers so reviewers do not mistake it for a new regression. diff --git a/docs/SECURITY_CHECKS_FINSERV_SEVERITY_REGISTER.md b/docs/SECURITY_CHECKS_FINSERV_SEVERITY_REGISTER.md new file mode 100644 index 0000000..c1d9374 --- /dev/null +++ b/docs/SECURITY_CHECKS_FINSERV_SEVERITY_REGISTER.md @@ -0,0 +1,345 @@ +# FinServ Severity Register (authoritative) + +This register is the **single source of truth** for the severity of every FinServ finding. It is +derived by applying [`severity-methodology.md`](./severity-methodology.md) (Likelihood × Impact → +ASFF label; §3.4 disposition rules; §3.5 family bands) to the **164 `create_finding` rows / 65 +check IDs** extracted from `finserv_assessments/app.py`. + +During implementation this becomes `SEVERITY_REGISTER` in `app.py` (keyed by finding-name) and the +`test_severity_register.py` drift-guard asserts each emitted `severity=` equals the value here. + +## How to read it + +- **Disposition:** FAIL / PASS / NA-NotApplicable / NA-CouldNotAssess / NA-Advisory / NA-SoftWarning. +- **I, L:** Impact and Likelihood (1–3) for the *control* (blank for advisory/NA rows whose + severity comes from the disposition rule, not I×L). +- **Sev (new):** the register-assigned severity. **Δ** marks a change from the current code. +- One severity per control across its PASS/FAIL rows (Round-2 invariant). + +## Validation outcome (what this register fixes) + +The audit of the live code found four defect classes, all resolved here: + +1. **Inconsistent NOT_APPLICABLE severity** — "nothing to assess" rows were tagged High (7), + Medium (10), and Informational (14). → **All NOT_APPLICABLE → Informational** (ASFF "no issue + found"); the resource-existence signal stays with that resource's own check. +2. **Split COULD_NOT_ASSESS severity** — inline access-checks were Low while `_could_not_assess_row` + was Medium. → **Unified to Low.** +3. **Pass/Fail severity mismatch for one control** — e.g., FS-66 Failed=High but Passed=Low; + FS-27 grounding Failed=High but Passed=Medium. → **One control severity applied to both.** +4. **Severity used to encode tier quality** — FS-28/36/51/59 tagged "CLASSIC tier" passes Low and + "STANDARD tier" High. → **Tier nuance moves to finding details/status; severity = control risk + (constant).** (Plus a flagged follow-up: should CLASSIC-tier be a `Failed`?) + +Plus the Round-3 decisions: FS-01 Shield→Low, WAF→Medium; the cost/rate-limiting family unified to +Medium (FS-02 was High); no `Critical` (capped at High). + +--- + +## Register + +### Category 1 — Unbounded Consumption (cost / rate-limiting family → Medium; Shield → Low) + +| Check | Finding name | Disposition | I | L | Sev (new) | Δ from current | +|---|---|---|---|---|---|---| +| FS-01 | AWS Shield Advanced Not Enabled | FAIL | 1 | 2 | **Low** | Δ High→Low | +| FS-01 | AWS Shield Advanced Enabled | PASS | 1 | 2 | **Low** | Δ High→Low | +| FS-01 | No Regional WAF Web ACLs Found | FAIL | 2 | 2 | **Medium** | Δ High→Medium | +| FS-01 | Regional WAF Web ACLs Present | PASS | 2 | 2 | **Medium** | Δ High→Medium | +| FS-02 | No API Gateway Usage Plans Found | NA-NotApplicable | – | – | **Informational** | Δ Medium→Info | +| FS-02 | API Gateway Usage Plans Missing Throttle | FAIL | 2 | 2 | **Medium** | Δ High→Medium | +| FS-02 | API Gateway Rate Limiting Configured | PASS | 2 | 2 | **Medium** | Δ High→Medium | +| FS-03 | No Bedrock Token Quotas Returned | FAIL | 2 | 2 | **Medium** | keep | +| FS-03 | Bedrock Default Quotas Unavailable — Customization Undetermined | FAIL | 2 | 2 | **Medium** | keep | +| FS-03 | Bedrock Token Quotas Customized | PASS | 2 | 2 | **Medium** | keep | +| FS-03 | Bedrock Token Quotas At Default | NA-SoftWarning | 2 | 2 | **Medium** | keep (documented exception) | +| FS-04 | No Cost Anomaly Detection Monitors | FAIL | 2 | 2 | **Medium** | keep | +| FS-04 | Cost Anomaly Monitors Do Not Cover Bedrock/SageMaker | FAIL | 2 | 2 | **Medium** | keep | +| FS-04 | Cost Anomaly Detection Configured | PASS | 2 | 2 | **Medium** | keep | +| FS-05 | No Bedrock CloudWatch Alarms Found | FAIL | 2 | 2 | **Medium** | keep | +| FS-05 | Bedrock CloudWatch Alarms Present | PASS | 2 | 2 | **Medium** | keep | +| FS-06 | No AI/ML Service Budgets Configured | FAIL | 2 | 2 | **Medium** | keep | +| FS-06 | AI/ML Service Budgets Configured | PASS | 2 | 2 | **Medium** | keep | + +### Category 2 — Excessive Agency (access/agency family → High; cost → Medium) + +| Check | Finding name | Disposition | I | L | Sev (new) | Δ from current | +|---|---|---|---|---|---|---| +| FS-07 | Agent Action Boundary Check | NA-NotApplicable | – | – | **Informational** | keep | +| FS-07 | Bedrock Agent Overly Broad Action Permissions | FAIL | 3 | 2 | **High** | keep | +| FS-07 | Agent Action Boundaries Look Appropriate | PASS | 3 | 2 | **High** | keep | +| FS-08 | AgentCore Policy Engine — Access Check | NA-CouldNotAssess | – | – | **Low** | keep | +| FS-08 | No AgentCore Runtimes Found | NA-NotApplicable | – | – | **Informational** | keep | +| FS-08 | AgentCore Runtimes Missing Policy Engine | FAIL | 3 | 2 | **High** | keep | +| FS-08 | AgentCore Policy Engine Configured | PASS | 3 | 2 | **High** | keep | +| FS-09 | Agent Lambda Functions Without Concurrency Limits | FAIL | 2 | 2 | **Medium** | keep | +| FS-09 | Agent Lambda Concurrency Limits Present | PASS (computed) | 2 | 2 | **Medium** | keep | +| FS-10 | Human-in-the-Loop Check — No Agent Workflows Found | NA-NotApplicable | – | – | **Informational** | Δ Medium→Info | +| FS-10 | Human Approval Steps Found in Agent Workflows | PASS | 3 | 2 | **High** | keep | +| FS-10 | Agent Workflows Missing Human Approval Steps | FAIL | 3 | 2 | **High** | keep | +| FS-11 | No Agent Rate Alarms Found | FAIL | 2 | 2 | **Medium** | keep | +| FS-11 | Agent Rate Alarms Present | PASS | 2 | 2 | **Medium** | keep | + +### Category 3 — Supply Chain (governance → Medium; SCP/scanning → High) + +| Check | Finding name | Disposition | I | L | Sev (new) | Δ from current | +|---|---|---|---|---|---|---| +| FS-12 | SCP Model Access Check — Not in Organization | NA-NotApplicable | – | – | **Informational** | Δ Low→Info | +| FS-12 | No Bedrock-Scoped SCPs Found | FAIL | 3 | 2 | **High** | keep | +| FS-12 | Bedrock SCPs Found | PASS | 3 | 2 | **High** | keep | +| FS-13 | Models Missing Provenance Tags | FAIL | 2 | 2 | **Medium** | keep | +| FS-13 | Model Provenance Tags Present | PASS | 2 | 2 | **Medium** | keep | +| FS-14 | No Model Governance Config Rules Found | FAIL | 2 | 2 | **Medium** | keep | +| FS-14 | Model Governance Config Rules Present | PASS | 2 | 2 | **Medium** | keep | +| FS-15 | No Bedrock Evaluation Jobs Found | FAIL | 2 | 2 | **Medium** | Δ N/A→FAIL, Info→Medium (REQ-10a) | +| FS-15 | Bedrock Evaluation Jobs Present | PASS | 2 | 2 | **Medium** | keep | +| FS-16 | No ECR Repositories Found | NA-NotApplicable | – | – | **Informational** | keep | +| FS-16 | ECR Repositories Without Image Scanning | FAIL | 3 | 2 | **High** | keep | +| FS-16 | ECR Image Scanning Enabled | PASS | 3 | 2 | **High** | keep | + +† **Resolved in REQ-10a:** FS-15 now treats "no eval jobs" as **Failed** (real control); FS-30/35/40 +are converted to **advisory** (they cannot inspect dataset content). See the REQ-10 section below. + +### Category 4 — Training Data & Model Poisoning (data integrity → High; governance → Medium) + +| Check | Finding name | Disposition | I | L | Sev (new) | Δ from current | +|---|---|---|---|---|---|---| +| FS-20 | No SageMaker Feature Groups Found | NA-NotApplicable | – | – | **Informational** | keep | +| FS-20 | Feature Groups Without Offline Store | FAIL | 2 | 2 | **Medium** | keep | +| FS-20 | Feature Store Offline Store Active | PASS | 2 | 2 | **Medium** | keep | +| FS-21 | No Training Data Buckets Identified | NA-NotApplicable | – | – | **Informational** | keep | +| FS-21 | Training Data Buckets Without Versioning | FAIL | 3 | 2 | **High** | keep (data-integrity/poisoning recovery) | +| FS-21 | Training Data Buckets Have Versioning | PASS | 3 | 2 | **High** | keep | + +### Category 5 — Vector & Embedding Weaknesses (access/encryption → High; governance → Medium) + +| Check | Finding name | Disposition | I | L | Sev (new) | Δ from current | +|---|---|---|---|---|---|---| +| FS-22 | Overly Permissive Knowledge Base IAM Roles | FAIL | 3 | 2 | **High** | keep | +| FS-22 | Knowledge Base IAM Permissions Look Appropriate | PASS | 3 | 2 | **High** | keep | +| FS-24 | No Knowledge Bases Found | NA-NotApplicable | – | – | **Informational** | keep | +| FS-24 | Knowledge Base Metadata Filtering — Manual Review Required | NA-Advisory | – | – | **Informational** | Δ Medium/Passed→Info/N/A; add `ADVISORY:` prefix | +| FS-25 | No OpenSearch Serverless Encryption Policies | NA-NotApplicable | – | – | **Informational** | keep | +| FS-25 | OpenSearch Serverless Encryption Not Using Customer-Managed Keys | FAIL | 3 | 2 | **High** | keep | +| FS-25 | OpenSearch Serverless Encryption Policies Present | PASS | 3 | 2 | **High** | keep | +| FS-26 | No OpenSearch Serverless Network Policies | FAIL | 3 | 2 | **High** | keep | +| FS-26 | OpenSearch Serverless Collections Not VPC-Restricted | FAIL | 3 | 2 | **High** | keep | +| FS-26 | OpenSearch Serverless VPC Access Configured | PASS | 3 | 2 | **High** | keep | + +### Category 6 — Non-Compliant Output (grounding → High; ARC → Medium) + +| Check | Finding name | Disposition | I | L | Sev (new) | Δ from current | +|---|---|---|---|---|---|---| +| FS-27 | No Guardrails — Contextual Grounding Not Applicable | NA-NotApplicable | – | – | **Informational** | Δ Medium→Info | +| FS-27 | No Guardrails With Contextual Grounding | FAIL | 3 | 2 | **High** | keep | +| FS-27 | Contextual Grounding Enabled on Guardrails | PASS | 3 | 2 | **High** | Δ Medium→High (match FAIL) | +| FS-27 | Automated Reasoning Policies — Access Check | NA-CouldNotAssess | – | – | **Low** | keep | +| FS-27 | No Automated Reasoning Policies Found | FAIL | 2 | 2 | **Medium** | keep | +| FS-27 | Automated Reasoning Policies Found | PASS | 2 | 2 | **Medium** | keep | +| FS-28 | No Guardrails — Denied Topics Not Applicable | NA-NotApplicable | – | – | **Informational** | Δ Medium→Info | +| FS-28 | No Guardrails With Denied Financial Topics | FAIL | 3 | 2 | **High** | keep | +| FS-28 | Denied Topics Configured on CLASSIC Tier | PASS | 3 | 2 | **High** | Δ Low→High; tier note → details ‡ | +| FS-28 | Guardrails With Topic Policies Found | PASS | 3 | 2 | **High** | Δ Medium→High | +| FS-29 | ADVISORY: Compliance Disclaimer — Manual Review Required | NA-Advisory | – | – | **Informational** | keep | +| FS-30 | ADVISORY: Compliance Dataset Coverage — Manual Review Required | NA-Advisory | – | – | **Informational** | Δ to advisory (REQ-10a): can't verify dataset content; replaces N/A+PASS | +| FS-31 | No Knowledge Bases Found | NA-NotApplicable | – | – | **Informational** | keep | +| FS-31 | Knowledge Base Data Sources Past Review Threshold | FAIL | 2 | 2 | **Medium** | keep | +| FS-31 | Knowledge Base Data Sources Recently Synced | PASS | 2 | 2 | **Medium** | keep | +| FS-32 | ADVISORY: Source Attribution — Manual Review Required | NA-Advisory | – | – | **Informational** | keep | +| FS-33 | No Knowledge Bases Found | NA-NotApplicable | – | – | **Informational** | keep | +| FS-33 | KB Data Source References a Deleted S3 Bucket | FAIL | 3 | 2 | **High** | keep (distinct risk: dangling/integrity) | +| FS-33 | KB Data Source Buckets Without Versioning | FAIL | 2 | 2 | **Medium** | keep (distinct risk) | +| FS-33 | KB Data Source Buckets Have Versioning | PASS | 2 | 2 | **Medium** | keep | +| FS-34 | Legacy Foundation Models Available in Region | NA-NotApplicable | – | – | **Informational** | Δ Medium→Info (availability ≠ usage) | +| FS-34 | Foundation Models Are Current | PASS | 2 | 2 | **Medium** | keep | + +‡ **Resolved in REQ-10b:** CLASSIC-tier guardrails are kept as **Passed** (they provide real +protection; CLASSIC is not deprecated/EOL — a hard FAIL would falsely fail adequate English-only +deployments). Severity = the control's inherent risk (constant); the STANDARD-upgrade +recommendation lives in the finding details. + +### Category 7 — Misinformation (eval governance → Medium) + +| Check | Finding name | Disposition | I | L | Sev (new) | Δ from current | +|---|---|---|---|---|---|---| +| FS-35 | ADVISORY: Harmful-Content Test Coverage — Manual Review Required | NA-Advisory | – | – | **Informational** | Δ to advisory (REQ-10a): can't verify dataset content; replaces N/A+PASS | + +### Category 8 — Abusive/Harmful Output (content safety → High; word filters → Medium) + +| Check | Finding name | Disposition | I | L | Sev (new) | Δ from current | +|---|---|---|---|---|---|---| +| FS-36 | No Guardrails — Content Filters Not Applicable | NA-NotApplicable | – | – | **Informational** | Δ High→Info | +| FS-36 | No Guardrails With Content Filters | FAIL | 3 | 2 | **High** | keep | +| FS-36 | Guardrail Content Filters on CLASSIC Tier | PASS | 3 | 2 | **High** | Δ Low→High; tier note → details ‡ | +| FS-36 | Guardrail Content Filters Configured (STANDARD Tier) | PASS | 3 | 2 | **High** | keep | +| FS-37 | ADVISORY: User Feedback Mechanism — Manual Review Required | NA-Advisory | – | – | **Informational** | keep | +| FS-38 | No Guardrails — Word Filters Not Applicable | NA-NotApplicable | – | – | **Informational** | Δ Medium→Info | +| FS-38 | No Guardrails With Word Filters | FAIL | 2 | 2 | **Medium** | keep | +| FS-38 | Guardrail Word Filters Configured | PASS | 2 | 2 | **Medium** | keep | + +### Category 9 — Biased Output (fair-lending/ECOA → High) + +| Check | Finding name | Disposition | I | L | Sev (new) | Δ from current | +|---|---|---|---|---|---|---| +| FS-39 | No SageMaker Clarify Bias Monitoring | FAIL | 3 | 2 | **High** | keep | +| FS-39 | SageMaker Clarify Bias Monitoring Active | PASS | 3 | 2 | **High** | keep | +| FS-40 | ADVISORY: Bias Dataset Coverage — Manual Review Required | NA-Advisory | – | – | **Informational** | Δ to advisory (REQ-10a): can't verify dataset content; replaces N/A+PASS | +| FS-41 | No SageMaker Clarify Explainability Monitoring | FAIL | 3 | 2 | **High** | keep | +| FS-41 | SageMaker Clarify Explainability Active | PASS | 3 | 2 | **High** | keep | +| FS-42 | No SageMaker Model Cards Found | FAIL | 2 | 2 | **Medium** | keep | +| FS-42 | SageMaker Model Cards Present | PASS | 2 | 2 | **Medium** | keep | + +### Category 10 — Sensitive Information Disclosure (data exposure → High; classification → Medium) + +| Check | Finding name | Disposition | I | L | Sev (new) | Δ from current | +|---|---|---|---|---|---|---| +| FS-43 | No CloudWatch Logs Data Protection Policies | FAIL | 3 | 2 | **High** | keep | +| FS-43 | CloudWatch Logs Data Protection Policies Present | PASS | 3 | 2 | **High** | keep | +| FS-44 | Amazon Macie Not Enabled | FAIL | 3 | 2 | **High** | keep | +| FS-44 | Amazon Macie Enabled | PASS | 3 | 2 | **High** | keep | +| FS-45 | No Guardrails — PII Filters Not Applicable | NA-NotApplicable | – | – | **Informational** | Δ High→Info | +| FS-45 | No Guardrails With PII Filters | FAIL | 3 | 2 | **High** | keep | +| FS-45 | Guardrail PII Filters Configured | PASS | 3 | 2 | **High** | keep | +| FS-46 | No AI/ML Data Buckets Identified | NA-NotApplicable | – | – | **Informational** | keep | +| FS-46 | AI/ML Buckets Without Data Classification Tags | FAIL | 2 | 2 | **Medium** | keep | +| FS-46 | AI/ML Buckets Have Classification Tags | PASS | 2 | 2 | **Medium** | keep | + +### Category 11 — Hallucination (grounding → High; relevance → Medium; advisory → Info) + +| Check | Finding name | Disposition | I | L | Sev (new) | Δ from current | +|---|---|---|---|---|---|---| +| FS-47 | No Guardrails — Grounding Threshold Not Applicable | NA-NotApplicable | – | – | **Informational** | Δ High→Info | +| FS-47 | Guardrails With Low Grounding Thresholds | FAIL | 3 | 2 | **High** | keep | +| FS-47 | No Guardrails With a Grounding Filter | FAIL | 3 | 2 | **High** | keep | +| FS-47 | Guardrail Grounding Thresholds Appropriate | PASS | 3 | 2 | **High** | keep | +| FS-48 | No Active Knowledge Bases for RAG | FAIL | 2 | 2 | **Medium** | keep | +| FS-48 | Active Knowledge Bases for RAG Present | PASS | 2 | 2 | **Medium** | keep | +| FS-49 | ADVISORY: Hallucination Disclaimer — Manual Review Required | NA-Advisory | – | – | **Informational** | keep | +| FS-50 | No Guardrails With Relevance Grounding Filters | FAIL | 2 | 2 | **Medium** | keep | +| FS-50 | Relevance Grounding Filters Present | PASS | 2 | 2 | **Medium** | keep | + +### Category 12 — Prompt Injection (prompt-attack → High; advisory → Info) + +| Check | Finding name | Disposition | I | L | Sev (new) | Δ from current | +|---|---|---|---|---|---|---| +| FS-51 | No Guardrails — Prompt Attack Filters Not Applicable | NA-NotApplicable | – | – | **Informational** | Δ High→Info | +| FS-51 | No Guardrails With Prompt Attack Filters | FAIL | 3 | 2 | **High** | keep | +| FS-51 | Prompt Attack Filters on CLASSIC Tier | PASS | 3 | 2 | **High** | Δ Low→High; tier note → details ‡ | +| FS-51 | Prompt Attack Filters Configured (STANDARD Tier) | PASS | 3 | 2 | **High** | keep | +| FS-52 | No Bedrock-Related Lambda Functions Found | NA-NotApplicable | – | – | **Informational** | keep | +| FS-52 | Bedrock Lambda Functions on Deprecated Runtimes | FAIL | 2 | 2 | **Medium** | keep | +| FS-52 | Bedrock Lambda Functions on Current Runtimes | PASS | 2 | 2 | **Medium** | keep | +| FS-54 | ADVISORY: Penetration Testing — Manual Review Required | NA-Advisory | – | – | **Informational** | keep | + +### Category 13 — Improper Output Handling (injection/XSS via WAF → see notes; advisory → Info) + +| Check | Finding name | Disposition | I | L | Sev (new) | Δ from current | +|---|---|---|---|---|---|---| +| FS-53 | No WAF Web ACLs — Injection Rules Not Applicable | NA-NotApplicable | – | – | **Informational** | Δ High→Info | +| FS-53 | WAF ACLs Missing Injection Protection Rules | FAIL | 3 | 2 | **High** | keep | +| FS-53 | WAF Injection Protection Rules Present | PASS | 3 | 2 | **High** | keep | +| FS-55 | No Output Validation Functions Found | FAIL | 2 | 2 | **Medium** | keep | +| FS-55 | Output Validation Functions Present | PASS | 2 | 2 | **Medium** | keep | +| FS-56 | No WAF ACLs — XSS Prevention Not Applicable | NA-NotApplicable | – | – | **Informational** | Δ High→Info | +| FS-56 | WAF ACLs Missing Common Rule Set (XSS) | FAIL | 2 | 2 | **Medium** | new FAIL path (REQ-10c) | +| FS-56 | XSS Prevention Common Rule Set Present | PASS | 2 | 2 | **Medium** | Δ replaces "Review" false-Passed (REQ-10c) | +| FS-57 | ADVISORY: Output Encoding — Manual Review Required | NA-Advisory | – | – | **Informational** | keep | +| FS-58 | ADVISORY: Output Schema Validation — Manual Review Required | NA-Advisory | – | – | **Informational** | Δ Medium/Passed→Info/N/A (REQ-2 retag) | + +### Category 14 — Off-Topic & Inappropriate Output (topic allowlist → Medium; advisory → Info) + +| Check | Finding name | Disposition | I | L | Sev (new) | Δ from current | +|---|---|---|---|---|---|---| +| FS-59 | No Guardrails — Topic Allowlist Not Applicable | NA-NotApplicable | – | – | **Informational** | Δ Medium→Info | +| FS-59 | No Guardrails With Topic Restrictions | FAIL | 2 | 2 | **Medium** | keep | +| FS-59 | Topic Restrictions Configured on CLASSIC Tier | PASS | 2 | 2 | **Medium** | Δ Low→Medium; tier note → details ‡ | +| FS-59 | Guardrail Topic Restrictions Configured | PASS | 2 | 2 | **Medium** | keep | +| FS-60 | ADVISORY: Contextual Grounding for Off-Topic Prevention | NA-Advisory | – | – | **Informational** | keep | + +### Category 15 — Out-of-Date Training Data (governance → Medium; advisory → Info) + +| Check | Finding name | Disposition | I | L | Sev (new) | Δ from current | +|---|---|---|---|---|---|---| +| FS-61 | No Knowledge Bases Found | NA-NotApplicable | – | – | **Informational** | keep | +| FS-61 | No Automated KB Sync Schedules Detected | FAIL | 2 | 2 | **Medium** | keep | +| FS-61 | Automated KB Sync Schedules Present | PASS | 2 | 2 | **Medium** | keep | +| FS-62 | ADVISORY: Data Currency Disclaimer — Manual Review Required | NA-Advisory | – | – | **Informational** | keep | +| FS-63 | Legacy Models Without Lifecycle Management | FAIL | 2 | 2 | **Medium** | keep | +| FS-63 | Foundation Model Lifecycle Management | PASS | 2 | 2 | **Medium** | keep | + +### Material Gap Checks (FS-65 to FS-69) + +| Check | Finding name | Disposition | I | L | Sev (new) | Δ from current | +|---|---|---|---|---|---|---| +| FS-65 | No Knowledge Bases Found | NA-NotApplicable | – | – | **Informational** | keep | +| FS-65 | KB Data Source References a Deleted S3 Bucket | FAIL | 3 | 2 | **High** | keep (distinct risk) | +| FS-65 | KB Data Source Buckets Missing S3 Event Notifications | FAIL | 2 | 2 | **Medium** | keep (distinct risk) | +| FS-65 | KB Data Source S3 Event Notifications Configured | PASS | 2 | 2 | **Medium** | keep | +| FS-66 | AgentCore Identity Propagation — Access Check | NA-CouldNotAssess | – | – | **Low** | keep | +| FS-66 | No AgentCore Runtimes Found | NA-NotApplicable | – | – | **Informational** | keep | +| FS-66 | AgentCore Runtimes Missing End-User Identity Propagation | FAIL | 3 | 2 | **High** | keep | +| FS-66 | AgentCore End-User Identity Propagation Configured | PASS | 3 | 2 | **High** | Δ Low→High (match FAIL) | +| FS-67 | No Agent Action-Group Lambda Functions Found | NA-NotApplicable | – | – | **Informational** | Δ High→Info | +| FS-67 | Agent Action-Group Lambdas May Lack Transaction Thresholds | FAIL | 3 | 2 | **High** | keep | +| FS-67 | Agent Action-Group Lambdas Have Threshold Configuration | PASS | 3 | 2 | **High** | keep | +| FS-68 | API Gateway Request Body Size Limits Not Enforced | FAIL | 2 | 2 | **Medium** | keep | +| FS-68 | API Gateway Request Body Size Limits — Not Applicable | NA-NotApplicable | – | – | **Informational** | new branch (REQ-4) | +| FS-68 | API Gateway Request Body Size Limits Configured | PASS | 2 | 2 | **Medium** | keep | +| FS-69 | No Prompt Input Validation Function Found | FAIL | 2 | 2 | **Medium** | keep | +| FS-69 | Prompt Input Validation Functions Present | PASS | 2 | 2 | **Medium** | keep | + +### Cross-cutting synthesized row + +| Source | Finding name | Disposition | Sev (new) | Δ from current | +|---|---|---|---|---| +| `_could_not_assess_row()` | `COULD NOT ASSESS: ` (any check that errors with no rows) | NA-CouldNotAssess | **Low** | Δ Medium→Low (unify with inline access-checks) | + +--- + +## Change summary + +| Change class | Count (approx) | Net effect | +|---|---|---| +| NOT_APPLICABLE N/A → Informational | ~21 rows (7 High, 10 Medium, ~4 Low/other) | removes misleading High/Medium **N/A** rows; no pass-rate impact (N/A excluded) | +| COULD_NOT_ASSESS → Low (unify) | 4 inline + generic row | consistent "unknown" signaling | +| FS-01 Shield High→Low | 2 rows | fixes reviewer Finding 6 | +| FS-01 WAF High→Medium | 2 rows | cost/rate-limiting family consistency | +| FS-02 High→Medium | 2 rows (PASS/FAIL) | cost/rate-limiting family consistency | +| Pass/Fail mismatch fixed (FS-27 grounding, FS-66) | ~3 rows | one severity per control | +| Tier-quality severity removed (FS-28/36/51/59) | 4 PASS rows | severity = control risk; tier nuance → details | +| FS-58 retag (REQ-2) | 1 row | advisory Informational | +| FS-24 advisory retag | 1 row | advisory Informational | +| **REQ-10a** FS-15 absence N/A→Failed | 1 row | now counted in pass rate (real model-validation gap) | +| **REQ-10a** FS-30/35/40 → advisory | 6 rows → 3 advisory | removes false Passed; honest manual-review | +| **REQ-10c** FS-56 gains FAIL path | +1 FAIL row | removes false "review" Passed | +| **REQ-10b** FS-28/36/51/59 CLASSIC kept Passed | 4 rows | severity normalized only (no FAIL) | + +**Expected report impact:** High-severity **counted** findings drop modestly (Shield, WAF, FS-02 +move out of High; several misleading High **N/A** rows become Informational). Pass-rate changes +only from the Passed-row severity moves (FS-01 Shield/WAF, FS-02), since N/A and Informational are +excluded from the rate. Capture before/after in the PR (task T1b.8). + +## Check-logic fixes pulled into this round (REQ-10) + +The three previously-flagged check-logic items are now **in scope** and resolved as follows (see +REQ-10 in requirements/design): + +1. **REQ-10a — Eval-job checks (FS-15/30/35/40).** FS-15 verifies a real, programmatically + checkable control (model-evaluation jobs exist) → "no eval jobs" becomes **FAIL/Medium** + (was N/A). FS-30/35/40 cannot inspect eval-job *dataset content* (they only re-checked job + existence, redundant with FS-15, and emitted a false **Passed**) → converted to **ADVISORY** + (Informational, `ADVISORY:` prefix, "manually verify {compliance|harmful-content|bias} + datasets") — the same honest treatment as FS-58. +2. **REQ-10b — CLASSIC-tier guardrails (FS-28/36/51/59).** **Investigated and intentionally NOT + converted to FAIL.** Reading the code: CLASSIC tier provides real protection (EN/FR/ES); + STANDARD adds multilingual + better prompt-attack classification but CLASSIC is **not** + deprecated/EOL. A hard FAIL would falsely fail adequate (e.g., English-only US) deployments. + Resolution: keep **PASS** at the control's severity (the §3.4 fix already removed the + severity-by-tier abuse), and strengthen the STANDARD-upgrade guidance in the finding details. +3. **REQ-10c — FS-56 XSS.** Add a real **FAIL** path mirroring FS-53: inspect each WAF ACL for + `AWSManagedRulesCommonRuleSet` (contains the XSS rules). Present → PASS; ACLs exist but missing + it → FAIL/Medium; no ACLs → N/A → Informational. Removes the false "review required" PASS. + +**Pass-rate impact of REQ-10:** FS-15-absent now counts as a Failed (was excluded N/A); FS-30/35/40 +lose three false Passes (now advisory/excluded); FS-56 can now Fail. Net effect lowers pass rate +slightly but accurately. Capture in T1b.8. From cb4a6b3ab3089b9a8282287ca3810281d738353b Mon Sep 17 00:00:00 2001 From: Neil Mehta Date: Thu, 11 Jun 2026 17:20:28 -0400 Subject: [PATCH 12/16] feat(report): render FinServ as a first-class service in single & multi-account reports REQ-1: FinServ findings were parsed but dropped before rendering (single-account) and mislabeled as Bedrock (multi-account). Make FinServ first-class end to end: - generate_consolidated_report/app.py: include "finserv" in service_stats, service_findings, and the service loop; thread FinServ kwargs into the template. - report_template.py: FinServ nav item, #finserv section with its own filters and table, Findings-by-Service card, and global Service-filter option; the section is omitted when finserv_total == 0 (supports the disabled/REQ-7 case). - consolidate_html_reports.py: categorize FS-* check IDs to "finserv" before the name fallback (fixes the multi-account Bedrock mislabeling). Make the account-files base path overridable via ACCOUNT_FILES_DIR (default /tmp/account-files unchanged) so the consolidator can be tested hermetically. - Tests: test_generate_report.py asserts FinServ renders/omits; new test_consolidate_finserv.py asserts FS-* categorize to finserv (hermetic temp dir). sim: https://github.com/aws-samples/sample-aiml-security-assessment/pull/23 --- .../generate_consolidated_report/app.py | 5 +- .../report_template.py | 81 ++++++++++- .../test_generate_report.py | 136 +++++++++++++----- consolidate_html_reports.py | 22 ++- test_consolidate_finserv.py | 90 ++++++++++++ 5 files changed, 296 insertions(+), 38 deletions(-) create mode 100644 test_consolidate_finserv.py diff --git a/aiml-security-assessment/functions/security/generate_consolidated_report/app.py b/aiml-security-assessment/functions/security/generate_consolidated_report/app.py index 3342372..ea707b9 100644 --- a/aiml-security-assessment/functions/security/generate_consolidated_report/app.py +++ b/aiml-security-assessment/functions/security/generate_consolidated_report/app.py @@ -208,10 +208,11 @@ def generate_html_report(assessment_results: Dict[str, Any]) -> str: "bedrock": {"passed": 0, "failed": 0, "na": 0}, "sagemaker": {"passed": 0, "failed": 0, "na": 0}, "agentcore": {"passed": 0, "failed": 0, "na": 0}, + "finserv": {"passed": 0, "failed": 0, "na": 0}, } - service_findings = {"bedrock": [], "sagemaker": [], "agentcore": []} + service_findings = {"bedrock": [], "sagemaker": [], "agentcore": [], "finserv": []} - for service in ["bedrock", "sagemaker", "agentcore"]: + for service in ["bedrock", "sagemaker", "agentcore", "finserv"]: if service in assessment_results: for report_type, findings in assessment_results[service].items(): for finding in findings: diff --git a/aiml-security-assessment/functions/security/generate_consolidated_report/report_template.py b/aiml-security-assessment/functions/security/generate_consolidated_report/report_template.py index 7f705df..7cb6d41 100644 --- a/aiml-security-assessment/functions/security/generate_consolidated_report/report_template.py +++ b/aiml-security-assessment/functions/security/generate_consolidated_report/report_template.py @@ -9,6 +9,21 @@ from datetime import datetime, timezone from typing import Dict, List, Optional +# FinServ service icon (no official AWS icon exists for "Financial Services"). +FINSERV_ICON = ( + '' + '' + '' +) +FINSERV_ICON_SMALL = ( + '' + '' + '' + '' +) + def generate_table_rows(findings: List[Dict], include_data_attrs: bool = True) -> str: """ @@ -281,6 +296,7 @@ def get_html_template() -> str: AgentCore {agentcore_total} + {finserv_nav}

Generated: {date_display}

@@ -326,7 +342,7 @@ def get_html_template() -> str:
{account_filter} -
+
@@ -348,6 +364,7 @@ def get_html_template() -> str:
Bedrock
{bedrock_total}
{bedrock_failed} Failed · {bedrock_passed} Passed
SageMaker
{sagemaker_total}
{sagemaker_failed} Failed · {sagemaker_passed} Passed
AgentCore
{agentcore_total}
{agentcore_failed} Failed · {agentcore_passed} Passed
+ {finserv_service_card}
@@ -383,6 +400,7 @@ def get_html_template() -> str:
{agentcore_rows}
Account IDCheck IDFindingDetailsResolutionReferenceSeverityStatus
+ {finserv_section}
Assessment Methodology

Severity Levels & Status Values

HighDirect security riskFailedRemediation needed
MediumDefense-in-depth gapPassedMeets requirements
LowBest practiceN/ANot applicable
InformationalNo action required
@@ -553,6 +571,7 @@ def get_html_template() -> str: createServiceFilter('bedrockTable', 'bedrockSearchInput', 'bedrockAccountFilter', 'bedrockSeverityFilter', 'bedrockStatusFilter', 'bedrockResetFilters'); createServiceFilter('sagemakerTable', 'sagemakerSearchInput', 'sagemakerAccountFilter', 'sagemakerSeverityFilter', 'sagemakerStatusFilter', 'sagemakerResetFilters'); createServiceFilter('agentcoreTable', 'agentcoreSearchInput', 'agentcoreAccountFilter', 'agentcoreSeverityFilter', 'agentcoreStatusFilter', 'agentcoreResetFilters'); + createServiceFilter('finservTable', 'finservSearchInput', 'finservAccountFilter', 'finservSeverityFilter', 'finservStatusFilter', 'finservResetFilters'); // Apply initial filters for main table applyFilters(); @@ -675,6 +694,8 @@ def generate_html_report( if f.get("_service") == "bedrock" else "SageMaker" if f.get("_service") == "sagemaker" + else "FinServ" + if f.get("_service") == "finserv" else "AgentCore" ) alerts_html += f"""
@@ -691,6 +712,8 @@ def generate_html_report( if f.get("_service") == "bedrock" else "SageMaker" if f.get("_service") == "sagemaker" + else "FinServ" + if f.get("_service") == "finserv" else "AgentCore" ) alerts_html += f"""
@@ -715,6 +738,9 @@ def generate_html_report( agentcore_rows = generate_table_rows( service_findings.get("agentcore", []), include_data_attrs=True ) + finserv_rows = generate_table_rows( + service_findings.get("finserv", []), include_data_attrs=True + ) # Mode-specific content num_accounts = len(account_ids) if account_ids else 1 @@ -734,6 +760,7 @@ def generate_html_report( bedrock_account_filter = f'
' sagemaker_account_filter = f'
' agentcore_account_filter = f'
' + finserv_account_filter = f'
' # Calculate per-account risk metrics account_metrics_html = "" @@ -788,8 +815,56 @@ def generate_html_report( bedrock_account_filter = "" sagemaker_account_filter = "" agentcore_account_filter = "" + finserv_account_filter = "" account_risk_section = "" + # FinServ (FS-*) — first-class service, rendered only when findings exist + # (so non-FinServ accounts and EnableFinServAssessment=false deploys stay clean). + finserv_total = ( + service_stats.get("finserv", {}).get("passed", 0) + + service_stats.get("finserv", {}).get("failed", 0) + + service_stats.get("finserv", {}).get("na", 0) + ) + finserv_failed = service_stats.get("finserv", {}).get("failed", 0) + finserv_passed = service_stats.get("finserv", {}).get("passed", 0) + if finserv_total > 0: + finserv_nav = ( + '' + + FINSERV_ICON + + " Financial Services" + + f'{finserv_total}' + ) + finserv_filter_option = '' + finserv_service_card = ( + '
' + + FINSERV_ICON_SMALL + + f' FinServ
{finserv_total}
' + + f'
{finserv_failed} Failed \u00b7 {finserv_passed} Passed
' + ) + finserv_section = ( + '
' + '
' + + FINSERV_ICON + + "Financial Services GenAI Risk Findings
" + '
Scope: this assessment evaluates the deployment Region only — run it per Region for multi-Region GenAI workloads. Severities follow a documented Likelihood × Impact methodology (see docs).
' + '
' + '
' + + finserv_account_filter + + '
' + '
' + '' + "
" + '
' + + finserv_rows + + "
Account IDCheck IDFindingDetailsResolutionReferenceSeverityStatus
" + "
" + ) + else: + finserv_nav = "" + finserv_filter_option = "" + finserv_service_card = "" + finserv_section = "" + # Fill template html_template = get_html_template() @@ -840,5 +915,9 @@ def generate_html_report( bedrock_account_filter=bedrock_account_filter, sagemaker_account_filter=sagemaker_account_filter, agentcore_account_filter=agentcore_account_filter, + finserv_nav=finserv_nav, + finserv_filter_option=finserv_filter_option, + finserv_service_card=finserv_service_card, + finserv_section=finserv_section, account_risk_section=account_risk_section, ) diff --git a/aiml-security-assessment/functions/security/generate_consolidated_report/test_generate_report.py b/aiml-security-assessment/functions/security/generate_consolidated_report/test_generate_report.py index a155817..01baaa1 100644 --- a/aiml-security-assessment/functions/security/generate_consolidated_report/test_generate_report.py +++ b/aiml-security-assessment/functions/security/generate_consolidated_report/test_generate_report.py @@ -1,10 +1,10 @@ # test_generate_report.py import unittest import os -import webbrowser from app import generate_html_report from report_template import generate_html_report as generate_report_direct + class TestHtmlReportGeneration(unittest.TestCase): def setUp(self): self.test_dir = "test_reports" @@ -24,7 +24,7 @@ def setUp(self): "Resolution": "Implement IAM policies to restrict access to specific principals and use resource-based policies for model invocations.", "Reference": "https://docs.aws.amazon.com/bedrock/latest/userguide/security_iam_id-based-policy-examples.html", "Severity": "High", - "Status": "Failed" + "Status": "Failed", }, { "Account_ID": "123456789012", @@ -34,7 +34,7 @@ def setUp(self): "Resolution": "Enable CloudTrail logging for Bedrock API actions and configure log retention policies.", "Reference": "https://docs.aws.amazon.com/bedrock/latest/userguide/logging-using-cloudtrail.html", "Severity": "Medium", - "Status": "Failed" + "Status": "Failed", }, { "Account_ID": "123456789012", @@ -44,8 +44,8 @@ def setUp(self): "Resolution": "No action required", "Reference": "https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails.html", "Severity": "Informational", - "Status": "Passed" - } + "Status": "Passed", + }, ] }, "sagemaker": { @@ -58,7 +58,7 @@ def setUp(self): "Resolution": "Enable AWS KMS encryption for SageMaker endpoints using customer managed keys.", "Reference": "https://docs.aws.amazon.com/sagemaker/latest/dg/encryption-at-rest.html", "Severity": "High", - "Status": "Failed" + "Status": "Failed", }, { "Account_ID": "123456789012", @@ -68,7 +68,7 @@ def setUp(self): "Resolution": "Enable network isolation for SageMaker training jobs and use VPC configurations.", "Reference": "https://docs.aws.amazon.com/sagemaker/latest/dg/mkt-algo-model-internet-free.html", "Severity": "Medium", - "Status": "Failed" + "Status": "Failed", }, { "Account_ID": "123456789012", @@ -78,8 +78,8 @@ def setUp(self): "Resolution": "Review and restrict IAM role permissions to only necessary actions and resources.", "Reference": "https://docs.aws.amazon.com/sagemaker/latest/dg/security_iam_id-based-policy-examples.html", "Severity": "High", - "Status": "Failed" - } + "Status": "Failed", + }, ] }, "agentcore": { @@ -92,10 +92,10 @@ def setUp(self): "Resolution": "No action required", "Reference": "https://docs.aws.amazon.com/singlesignon/latest/userguide/what-is.html", "Severity": "Informational", - "Status": "Passed" + "Status": "Passed", } ] - } + }, } def test_generate_viewable_report(self): @@ -117,7 +117,7 @@ def test_generate_viewable_report(self): self.assertTrue(os.path.getsize(report_path) > 0) # Basic content checks - with open(report_path, 'r') as f: + with open(report_path, "r") as f: content = f.read() # Bedrock findings self.assertIn("Bedrock Model Access Control", content) @@ -137,7 +137,7 @@ def test_generate_viewable_report(self): # New design elements self.assertIn("sidebar", content) - self.assertIn("service-badge", content) + self.assertIn("service-icon", content) self.assertIn("theme-toggle", content) # Verify new features from consolidation @@ -149,27 +149,57 @@ def test_generate_multi_account_report(self): """Test multi-account report generation using shared template directly""" # Create test data in multi-account format all_findings = [ - {'account_id': '111122223333', 'check_id': 'BR-01', 'finding': 'Test Finding 1', 'details': 'Details 1', 'resolution': 'Fix it', 'reference': 'https://example.com', 'severity': 'High', 'status': 'Failed', '_service': 'bedrock'}, - {'account_id': '444455556666', 'check_id': 'SM-01', 'finding': 'Test Finding 2', 'details': 'Details 2', 'resolution': 'Fix it', 'reference': 'https://example.com', 'severity': 'Medium', 'status': 'Failed', '_service': 'sagemaker'}, - {'account_id': '111122223333', 'check_id': 'AC-01', 'finding': 'Test Finding 3', 'details': 'Details 3', 'resolution': 'N/A', 'reference': 'https://example.com', 'severity': 'Low', 'status': 'Passed', '_service': 'agentcore'}, + { + "account_id": "111122223333", + "check_id": "BR-01", + "finding": "Test Finding 1", + "details": "Details 1", + "resolution": "Fix it", + "reference": "https://example.com", + "severity": "High", + "status": "Failed", + "_service": "bedrock", + }, + { + "account_id": "444455556666", + "check_id": "SM-01", + "finding": "Test Finding 2", + "details": "Details 2", + "resolution": "Fix it", + "reference": "https://example.com", + "severity": "Medium", + "status": "Failed", + "_service": "sagemaker", + }, + { + "account_id": "111122223333", + "check_id": "AC-01", + "finding": "Test Finding 3", + "details": "Details 3", + "resolution": "N/A", + "reference": "https://example.com", + "severity": "Low", + "status": "Passed", + "_service": "agentcore", + }, ] service_findings = { - 'bedrock': [all_findings[0]], - 'sagemaker': [all_findings[1]], - 'agentcore': [all_findings[2]] + "bedrock": [all_findings[0]], + "sagemaker": [all_findings[1]], + "agentcore": [all_findings[2]], } service_stats = { - 'bedrock': {'passed': 0, 'failed': 1}, - 'sagemaker': {'passed': 0, 'failed': 1}, - 'agentcore': {'passed': 1, 'failed': 0} + "bedrock": {"passed": 0, "failed": 1}, + "sagemaker": {"passed": 0, "failed": 1}, + "agentcore": {"passed": 1, "failed": 0}, } html_content = generate_report_direct( all_findings=all_findings, service_findings=service_findings, service_stats=service_stats, - mode='multi', - account_ids=['111122223333', '444455556666'] + mode="multi", + account_ids=["111122223333", "444455556666"], ) report_path = os.path.join(self.test_dir, "multi_account_report.html") @@ -180,7 +210,7 @@ def test_generate_multi_account_report(self): self.assertTrue(os.path.exists(report_path)) - with open(report_path, 'r') as f: + with open(report_path, "r") as f: content = f.read() # Multi-account specific self.assertIn("Multi-Account", content) @@ -194,13 +224,12 @@ def test_missing_data_fields(self): incomplete_data = { "account_id": "123456789012", "bedrock": { - "bedrock_report": [{ - "Finding": "Incomplete Bedrock Finding", - "Severity": "High" - }] + "bedrock_report": [ + {"Finding": "Incomplete Bedrock Finding", "Severity": "High"} + ] }, "sagemaker": {}, - "agentcore": {} + "agentcore": {}, } html_content = generate_html_report(incomplete_data) @@ -222,7 +251,7 @@ def test_empty_findings(self): "account_id": "123456789012", "bedrock": {}, "sagemaker": {}, - "agentcore": {} + "agentcore": {}, } html_content = generate_html_report(empty_data) @@ -233,9 +262,52 @@ def test_empty_findings(self): print(f"\nEmpty data report generated at: {os.path.abspath(report_path)}") self.assertTrue(os.path.exists(report_path)) + def test_finserv_renders_when_present(self): + """REQ-1: FinServ findings render as a first-class service in the HTML.""" + data = dict(self.test_assessment_results) + data["finserv"] = { + "finserv_security_report": [ + { + "Account_ID": "123456789012", + "Check_ID": "FS-01", + "Finding": "No Regional WAF Web ACLs Found", + "Finding_Details": "No WAF.", + "Resolution": "Add WAF.", + "Reference": "https://docs.aws.amazon.com/waf/latest/developerguide/waf-chapter.html", + "Severity": "Medium", + "Status": "Failed", + }, + { + "Account_ID": "123456789012", + "Check_ID": "FS-44", + "Finding": "Amazon Macie Enabled", + "Finding_Details": "Macie on.", + "Resolution": "None.", + "Reference": "https://docs.aws.amazon.com/macie/latest/user/what-is-macie.html", + "Severity": "High", + "Status": "Passed", + }, + ] + } + html = generate_html_report(data) + self.assertIn('id="finserv"', html) + self.assertIn('id="finservTable"', html) + self.assertIn('