diff --git a/.ash/.ash.yaml b/.ash/.ash.yaml new file mode 100644 index 0000000..791bba4 --- /dev/null +++ b/.ash/.ash.yaml @@ -0,0 +1,235 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/awslabs/automated-security-helper/refs/heads/main/automated_security_helper/schemas/AshConfig.json +project_name: sample-aiml-security-assessment +global_settings: + severity_threshold: MEDIUM + ignore_paths: + # SAM build cache — transitively scanned pydantic/bs4/botocore deps + # produce thousands of non-actionable Bandit findings in third-party code. + # CI scans only changed files, so it never touches .aws-sam/ either. + - path: "aiml-security-assessment/.aws-sam/**" + reason: "SAM build cache; contains third-party deps, not project source" + - path: "**/.aws-sam/**" + reason: "SAM build cache at any depth (deps/, build/, etc.)" + # Pre-existing static HTML sample reports; the embedded base64 SVG logos + # are flagged as high-entropy strings by detect-secrets. + - path: "sample-reports/**" + reason: "Static HTML sample outputs with embedded base64 SVG icons (upstream)" + # ASH's own output directory. + - path: ".ash/ash_output/**" + reason: "ASH output directory; scanning it creates recursion" + suppressions: + # The 4 Checkov rules below flag every assessment Lambda in the upstream + # repo for: no VPC (CKV_AWS_117), no DLQ (CKV_AWS_116), no per-function + # concurrency cap (CKV_AWS_115), and no KMS-encrypted env vars + # (CKV_AWS_173). This matches the architecture choices made for all 6 + # pre-existing Lambdas (BedrockSecurity, SagemakerSecurity, + # AgentCoreSecurity, GenerateConsolidatedReport, IAMPermissionCaching, + # CleanupBucket) and applied consistently to the new + # FinServSecurityAssessmentFunction. The assessment framework runs on an + # internal schedule with short-lived invocations and reads config from a + # non-sensitive bucket-name env var; hardening these four items for every + # Lambda in the stack is a framework-wide change that should be proposed + # in a separate PR. + - rule_id: "CKV_AWS_117" + path: "aiml-security-assessment/template.yaml" + reason: "Upstream convention: assessment Lambdas are non-VPC (applies to all 7 Lambdas)" + - rule_id: "CKV_AWS_117" + path: "aiml-security-assessment/template-multi-account.yaml" + reason: "Upstream convention: assessment Lambdas are non-VPC (applies to all 7 Lambdas)" + - rule_id: "CKV_AWS_116" + path: "aiml-security-assessment/template.yaml" + reason: "Upstream convention: assessment Lambdas have no DLQ (applies to all 7 Lambdas)" + - rule_id: "CKV_AWS_116" + path: "aiml-security-assessment/template-multi-account.yaml" + reason: "Upstream convention: assessment Lambdas have no DLQ (applies to all 7 Lambdas)" + - rule_id: "CKV_AWS_115" + path: "aiml-security-assessment/template.yaml" + reason: "Upstream convention: no per-function concurrency cap (applies to all 7 Lambdas)" + - rule_id: "CKV_AWS_115" + path: "aiml-security-assessment/template-multi-account.yaml" + reason: "Upstream convention: no per-function concurrency cap (applies to all 7 Lambdas)" + - rule_id: "CKV_AWS_173" + path: "aiml-security-assessment/template.yaml" + reason: "Upstream convention: env var is a non-sensitive bucket-name reference" + - rule_id: "CKV_AWS_173" + path: "aiml-security-assessment/template-multi-account.yaml" + reason: "Upstream convention: env var is a non-sensitive bucket-name reference" + + # cdk-nag AwsSolutions-IAM5 flags any IAM statement with Resource: "*". + # The FinServ IAM statement I added to 1-aiml-security-member-roles.yaml + # uses the same pattern as every pre-existing Sid in that policy. The + # wildcard is required for list/describe operations that are not + # ARN-scopable (e.g., ListBuckets, ListTrails, ListRules, ListPolicies). + # If upstream tightens IAM across all assessment statements, the FinServ + # Sid should be tightened in the same PR. + - rule_id: "AwsSolutions-IAM5" + path: "deployment/1-aiml-security-member-roles.yaml" + reason: "Read-only list/describe actions that cannot be ARN-scoped (upstream convention)" + - rule_id: "AwsSolutions-IAM5" + path: "deployment/aiml-security-single-account.yaml" + reason: "Read-only list/describe actions that cannot be ARN-scoped (upstream convention)" + - rule_id: "CKV_AWS_173" + path: "aiml-security-assessment/template-multi-account.yaml" + reason: "Upstream convention: env var is a bucket name reference, not sensitive (same as BR/SM/AC)" +fail_on_findings: true +ash_plugin_modules: [] +external_reports_to_include: [] +converters: + archive: + enabled: true + options: {} + jupyter: + enabled: true + options: + tool_version: '>=7.16.0,<8.0.0' + install_timeout: 300 +scanners: + bandit: + enabled: true + options: + severity_threshold: null + config_file: null + confidence_level: all + ignore_nosec: false + excluded_paths: [] + additional_formats: [] + tool_version: '>=1.7.0,<2.0.0' + install_timeout: 300 + cdk-nag: + enabled: true + options: + severity_threshold: null + nag_packs: + AwsSolutionsChecks: true + HIPAASecurityChecks: false + NIST80053R4Checks: false + NIST80053R5Checks: false + PCIDSS321Checks: false + cfn-nag: + enabled: true + options: + severity_threshold: null + checkov: + enabled: true + options: + severity_threshold: null + config_file: null + skip_path: [] + additional_formats: + - cyclonedx_json + offline: false + frameworks: + - all + skip_frameworks: [] + tool_version: null + install_timeout: 300 + detect-secrets: + enabled: true + options: + severity_threshold: null + baseline_file: null + scan_settings: + version: null + generated_at: null + plugins_used: [] + filters_used: [] + results: {} + grype: + enabled: true + options: + severity_threshold: null + config_file: null + offline: false + npm-audit: + enabled: true + options: + severity_threshold: null + offline: false + opengrep: + enabled: true + options: + severity_threshold: null + config: auto + exclude: + - '*-converted.py' + - '*_report_result.txt' + exclude_rule: [] + severity: [] + metrics: auto + offline: false + patterns: [] + version: v1.15.1 + semgrep: + enabled: true + options: + severity_threshold: null + config: auto + exclude: + - '*-converted.py' + - '*_report_result.txt' + exclude_rule: [] + severity: [] + metrics: auto + offline: false + tool_version: null + install_timeout: 300 + syft: + enabled: true + options: + severity_threshold: null + config_file: null + exclude: [] + additional_outputs: + - syft-table +reporters: + csv: + enabled: true + options: {} + cyclonedx: + enabled: true + options: {} + html: + enabled: true + options: {} + flat-json: + enabled: true + options: + include_scanner_metrics: true + include_summary_metrics: true + include_metadata: true + gitlab-sast: + enabled: true + options: {} + junitxml: + enabled: true + options: + respect_severity_threshold: true + markdown: + enabled: true + options: + include_summary: true + include_findings_table: false + include_detailed_findings: true + max_detailed_findings: 20 + top_hotspots_limit: 10 + use_collapsible_details: true + ocsf: + enabled: true + options: {} + sarif: + enabled: true + options: {} + spdx: + enabled: false + options: {} + text: + enabled: true + options: + include_summary: true + include_findings_table: false + include_detailed_findings: false + max_detailed_findings: 20 + top_hotspots_limit: 20 + yaml: + enabled: false + options: {} diff --git a/.ash/.gitignore b/.ash/.gitignore new file mode 100644 index 0000000..d831134 --- /dev/null +++ b/.ash/.gitignore @@ -0,0 +1,2 @@ +# ASH default output directory (and variants) +ash_output* diff --git a/.cfnlintrc b/.cfnlintrc index 4da6536..2e7e21f 100644 --- a/.cfnlintrc +++ b/.cfnlintrc @@ -1,12 +1,19 @@ configure_rules: W3037: - # cfn-lint's IAM action database doesn't include newer services yet. - # These are valid IAM actions used by the assessment framework. + # cfn-lint's IAM action database lags behind newer AWS services, so it + # cannot recognize a small set of *valid* actions used by the assessment. strict: false ignore_checks: - # W3037: IAM action not recognized - cfn-lint's action database lags behind - # newer AWS services (bedrock-agent, bedrock-agentcore, s3:GetBucketEncryption, s3:HeadBucket) + # W3037: IAM action not recognized. As of the last review the ONLY actions + # this suppresses are valid-but-newer ones cfn-lint's DB does not yet list: + # - bedrock-agentcore:* (Amazon Bedrock AgentCore control plane) + # - s3:GetBucketEncryption, s3:HeadBucket + # All Amazon Bedrock Knowledge Base / Data Source / Flow actions use the + # `bedrock:` IAM prefix (there is no `bedrock-agent:` IAM namespace — that is + # only the boto3 client name); the previously-present invalid `bedrock-agent:*` + # grants were removed. IAM-action correctness is now guarded positively by + # tests/test_iam_coverage.py rather than by this lint rule. - W3037 # W1030: Parameter default/constraint mismatch - CodeBuildTimeout has MinValue/MaxValue # constraints that cfn-lint incorrectly flags when Ref is used diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 1906b66..d0d65f4 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -6,12 +6,16 @@ on: paths: - "aiml-security-assessment/functions/security/**" - "tests/**" + - "consolidate_html_reports.py" + - "test_consolidate_finserv.py" - ".github/workflows/python-tests.yml" pull_request: branches: [main] paths: - "aiml-security-assessment/functions/security/**" - "tests/**" + - "consolidate_html_reports.py" + - "test_consolidate_finserv.py" - ".github/workflows/python-tests.yml" jobs: @@ -36,20 +40,50 @@ jobs: # Install pydantic for schema validation used by Lambda functions pip install pydantic>=2.0.0 - - name: Run tests with coverage + - name: Run upstream tests with coverage env: AIML_ASSESSMENT_BUCKET_NAME: test-assessment-bucket AWS_DEFAULT_REGION: us-east-1 AWS_ACCESS_KEY_ID: testing - AWS_SECRET_ACCESS_KEY: testing + AWS_SECRET_ACCESS_KEY: testing # pragma: allowlist secret run: | python -m pytest tests/ \ -v \ --tb=short \ --cov=aiml-security-assessment/functions/security \ + --cov-report=term-missing + + - name: Run FinServ assessment tests with coverage + # Run separately from tests/: the FinServ suite has its own conftest and + # imports the FinServ app as top-level `app`, which would collide in a + # single pytest session with the other assessment modules' app.py files. + env: + AIML_ASSESSMENT_BUCKET_NAME: test-assessment-bucket + AWS_DEFAULT_REGION: us-east-1 + AWS_ACCESS_KEY_ID: testing + AWS_SECRET_ACCESS_KEY: testing # pragma: allowlist secret + run: | + python -m pytest aiml-security-assessment/functions/security/finserv_tests/ \ + -v \ + --tb=short \ + --cov=aiml-security-assessment/functions/security \ + --cov-append \ --cov-report=term-missing \ - --cov-report=xml:coverage.xml \ - -x + --cov-report=xml:coverage.xml + + - name: Run report-pipeline tests + # The single- and multi-account HTML report generators (REQ-1) and the + # FinServ categorization test. The report generator package ships an + # __init__.py, so its tests run with that directory as CWD. + env: + AIML_ASSESSMENT_BUCKET_NAME: test-assessment-bucket + AWS_DEFAULT_REGION: us-east-1 + AWS_ACCESS_KEY_ID: testing + AWS_SECRET_ACCESS_KEY: testing # pragma: allowlist secret + run: | + python -m pytest test_consolidate_finserv.py -v --tb=short + cd aiml-security-assessment/functions/security/generate_consolidated_report + python -m pytest test_generate_report.py -v --tb=short - name: Upload coverage report if: always() diff --git a/.gitignore b/.gitignore index 219bbc1..d6df932 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ .kiro/ .vscode/ .claude/ +.codex/ .mcp.json # AWS SAM build artifacts @@ -12,6 +13,7 @@ __pycache__/ *.py[cod] *$py.class *.pyc +.pytest_cache/ # OS files .DS_Store @@ -39,4 +41,5 @@ samples/ # Test output **/test_reports/ -.ruff_cache/ \ No newline at end of file +.ruff_cache/ +.ash/ diff --git a/FOLLOWUPS.md b/FOLLOWUPS.md new file mode 100644 index 0000000..537ab51 --- /dev/null +++ b/FOLLOWUPS.md @@ -0,0 +1,196 @@ +# Follow-up items (deferred from PR #23 Round 3) + +> GitHub Issues are disabled on the fork `mehtadman87/sample-aiml-security-assessment`, so +> deferred work is tracked here in-repo (task T1b.7). Promote to an upstream issue/PR when ready. + +## FU-1 — Evaluate adopting a tool-wide `CRITICAL` severity tier + +**Status:** Deferred (intentionally out of scope for Round 3). + +**Background.** The FinServ severity methodology +([`docs/SECURITY_CHECKS_FINSERV_SEVERITY_METHODOLOGY.md`](./SECURITY_CHECKS_FINSERV_SEVERITY_METHODOLOGY.md)) +defines a Likelihood × Impact matrix mapped to the AWS Security Hub ASFF label set, which includes a +**CRITICAL** band (the Impact=High × Likelihood=High cell). Round 3 **capped that cell at High** and +did **not** introduce `Critical`, to keep the FinServ checks consistent with the upstream +Bedrock/SageMaker/AgentCore checks (which have no Critical tier). The drift-guard +(`tests/test_severity_register.py`) asserts no `Critical` is currently emitted. + +**Why it is a separate item.** Adopting `Critical` is a **tool-wide** change. Doing it for FinServ +alone would make FinServ inconsistent with the other three services. A tool-wide rollout touches: +- the shared `SeverityEnum` (each package's `schema.py`); +- all four assessment Lambdas (re-score the I=High × L=High controls); +- the report template severity filters/colors (`generate_consolidated_report/report_template.py` + and `consolidate_html_reports.py`); +- every service's unit tests plus the FinServ drift-guard. + +**Decision needed.** +1. Do we want a `CRITICAL` tier across the whole assessment? +2. If yes, which controls qualify (e.g., full guardrail bypass enabling a regulatory breach; + unauthorized high-value autonomous financial action)? + +**References:** methodology §2 (label scale) and §6 (the deferred decision); ASFF severity — +https://docs.aws.amazon.com/securityhub/1.0/APIReference/API_Severity.html + +--- + +## FU-2 — Report UI design for FinServ / OWASP (REQ-8) + +Broader report UI re-design (top-page FinServ summary box placement, OWASP grouping, Risk +Distribution treatment) is deferred to a separate PR per the reviewer's suggestion. Round 3 delivers +FinServ as a functionally first-class service (REQ-1, Wave 2); the UI-design discussion is tracked +here for a follow-up PR. + +--- +--- +--- + +## FU-3 — Shared-inventory refactor for the FinServ assessment Lambda (REQ-13 C3) + +**Status:** Deferred from Round 3.1 (Wave 5.5 task T5h.9). Classified **Should-Fix** (performance / +scalability optimization), not a correctness blocker. Tracked here for a focused follow-up PR. + +### Background + +The FinServ Lambda (`finserv_assessments/app.py`) runs all 64 checks **sequentially in a single +invocation** via `build_finserv_checks()`. There is no cross-check caching, so several checks +independently re-enumerate the **same** account-wide inventories, and many then issue an N+1 +per-resource call on top. The Round-3.1 pre-release audit (the per-check AWS API inventory) +identified the following duplicate full-account sweeps within one run: + +| Inventory (API) | Times enumerated | Checks performing the enumeration | Per-resource N+1 follow-up | +| --- | --- | --- | --- | +| `lambda:ListFunctions` | ~6× | FS-09, FS-52, FS-55, FS-58, FS-67, FS-69 | FS-09 `GetFunctionConcurrency` per function | +| `bedrock:ListGuardrails` | ~9× | FS-27a, FS-28, FS-36, FS-38, FS-45, FS-47, FS-50, FS-51, FS-59 | `GetGuardrail` per guardrail in each check | +| `bedrock:ListKnowledgeBases` | ~6× | FS-24, FS-31, FS-33, FS-48, FS-61, FS-65 | FS-31/33/65 `ListDataSources`/`GetDataSource` per KB | +| `s3:ListBuckets` (full account) | ~3–4× | FS-21, FS-46 (full sweep); FS-33, FS-65 (KB data-source buckets) | `GetBucketVersioning`/`GetBucketTagging`/`GetBucketNotification` per bucket | +| `wafv2:ListWebACLs` | 4× | FS-01, FS-53, FS-56, FS-68 | `GetWebACL` per ACL in each check | + +In small/empty accounts this is harmless (it is why the Wave-5 test account ran in ~3 minutes). In +**large enterprise accounts** — thousands of Lambda functions, hundreds of S3 buckets, many +guardrails/KBs/ACLs — the repeated full sweeps plus N+1 calls multiply API volume, drive adaptive +retries/throttling (`Config(retries=adaptive)`), and inflate wall-clock time. This is the most +likely driver of a slow run, and previously of a `States.Timeout`. + +### Why it was deferred (and why that is safe) + +1. **It is a large, regression-prone refactor.** It touches ~20 check functions and the way they + obtain AWS data, plus every corresponding mocked unit test (the per-check tests patch + `app.boto3.client`). Bundling that into a correctness-focused PR that "must be 100% sure" + (the reviewer's bar) trades correctness risk for a performance gain. +2. **The blast-radius risk it is associated with (audit C2) is already mitigated in Round 3.1.** + A `Catch [States.ALL] → "FinServ Assessment Incomplete"` was added to the FinServ task in + `statemachine/assessments.asl.json`, and the FinServ Lambda `Timeout` was raised 600 → 900 s. + So even if the FinServ Lambda is slow or times out in a very large account, the consolidated + report is still generated with the other services' findings and a visible incomplete marker — + the failure no longer sinks the whole assessment. +3. **It is purely an optimization** (Should-Fix). The disposition logic and severities are unchanged + by this work; nothing about the *correctness* of any finding depends on it. + +### Who is affected + +Customers running the FinServ assessment (`EnableFinServAssessment=true`) against accounts with large +resource estates — primarily large enterprises. Symptoms: long FinServ Lambda duration, CloudWatch +throttling/retry noise, and (pre-mitigation) the risk of a 900 s timeout. Small accounts are +unaffected. + +### Proposed design + +Collect each shared inventory **once** at the start of `lambda_handler`, into a read-only context +object, and pass it to the checks that need it — mirroring the existing `permission_cache` pattern +(which is already injected via `functools.partial` in `build_finserv_checks()`). + +1. **Add a `ResourceInventory` collected once per invocation**, e.g.: + + ```python + @dataclass + class ResourceInventory: + lambda_functions: list # lambda:ListFunctions (+ concurrency map) + guardrails: list # bedrock:ListGuardrails + GetGuardrail detail, keyed by id + knowledge_bases: list # bedrock:ListKnowledgeBases (+ data sources per KB) + buckets: list # s3:ListBuckets + web_acls: list # wafv2:ListWebACLs + GetWebACL detail (REGIONAL) + ``` + + Each field is populated by one paginated enumeration (reuse `_paginate`). Per-resource detail + (e.g., `GetGuardrail`, `GetWebACL`) is fetched once and stored alongside, eliminating the N+1 + repetition across checks. + +2. **Inject it like `permission_cache`.** In `build_finserv_checks(permission_cache, inventory)`, + bind the inventory to the relevant checks with `functools.partial`, so every registry entry + stays uniformly zero-arg and the handler loop is unchanged. + +3. **Make collection resilient.** A failure (e.g., `AccessDenied`) collecting one inventory must not + abort the others or the whole run. Per-inventory collection should be wrapped so a failed + inventory yields an explicit "unavailable" sentinel; checks that depend on an unavailable + inventory then emit `COULD_NOT_ASSESS` (consistent with `_is_access_error` handling today) rather + than a false `Failed`/`Passed`. Preserve the existing per-check `try/except` safety net. + +4. **Keep region/pagination semantics identical.** Same default-region clients, same `_paginate` + token handling; this is a call-site consolidation, not a behavior change. + +### Test strategy + +- Update the affected per-check unit tests to pass a constructed `ResourceInventory` (or a partial) + instead of patching `boto3.client` for the enumeration calls. Checks that still make non-inventory + calls keep their existing mocks. +- Add tests for the collector itself: one enumeration per inventory; pagination; and the + per-inventory failure path producing the "unavailable" sentinel (→ `COULD_NOT_ASSESS` downstream). +- **Behavior-preserving guarantee:** every existing disposition test (Passed/Failed/N/A per check) + and the severity drift-guard (`tests/test_severity_register.py`) must remain green with **no** + disposition or severity changes. That equivalence is the acceptance bar. +- Optionally add a counter/assertion (in a unit harness) proving each inventory API is called at + most once per handler invocation. + +### Acceptance criteria + +- Each shared inventory (`ListFunctions`, `ListGuardrails`+`GetGuardrail`, `ListKnowledgeBases`, + `ListBuckets`, `ListWebACLs`+`GetWebACL`) is enumerated **at most once** per FinServ run. +- No change to any finding's status or severity (all existing tests + the drift-guard pass + unchanged). +- A per-inventory collection failure degrades only the dependent checks (to `COULD_NOT_ASSESS`), not + the whole run. +- Workspace `finserv_assessments/` stays byte-identical to the fork copy after sync. + +### Risk / considerations + +- **Test isolation** if any memoization is module-level: prefer an explicit per-invocation object + passed as an argument over a module-global cache, to avoid state leaking across unit tests. +- **Partial-inventory correctness:** ensure a check that needs two inventories handles one being + unavailable independently. +- **Memory:** holding full inventories (e.g., all guardrail details) in memory is bounded and far + smaller than the permission cache already loaded; no concern at 1024 MB. + +### Effort estimate + +Roughly 1–2 focused days: ~0.5 day for the collector + injection, ~0.5–1 day updating the ~20 +affected checks and their tests, ~0.5 day validation (full suite + a large-account timing check). + +### References + +- Round-3.1 requirement: **REQ-13 (audit finding C3)** and design section "REQ-13 — Enterprise-scale + resilience & scope (audit C)" in `.kiro/specs/pr-review-round3-fixes/design.md`. +- Deferred task: **T5h.9** in `.kiro/specs/pr-review-round3-fixes/tasks.md`. +- Related mitigation already shipped in Round 3.1: **T5h.8** (ASL `Catch` on the FinServ task + + Lambda `Timeout` 600 → 900 s) addressing audit finding C2. +- Existing pattern to mirror: the `permission_cache` injection in `get_permissions_cache()` / + `build_finserv_checks()` in `finserv_assessments/app.py`. + +## FU-4 — Migrate upstream schemas from Pydantic V1 `@validator` to V2 `@field_validator` + +**Priority:** Low (tech-debt) — not a blocker. + +The upstream `schema.py` files for the Bedrock, SageMaker, AgentCore, consolidated-report, +and IAM-permission-caching Lambdas still use the deprecated Pydantic V1 `@validator` decorator, +which emits `PydanticDeprecatedSince20` warnings and will break when Pydantic V3 removes V1-style +validators. The FinServ `schema.py` already uses the V2 `@field_validator` form. + +**Why deferred:** this PR is scoped to `feature/finserv-risk-checks`. The affected files are +upstream/shared components; migrating them here would exceed the PR's scope and risk merge +conflicts with upstream. Best handled as a dedicated upstream change. + +**Scope:** swap `@validator("X")` → `@field_validator("X")` + `@classmethod`, adjust signatures, +and re-run each module's tests. + +### References + +- Identified in the pre-Wave-6 verification pass (`.kiro/specs/pr-review-round3-fixes/tasks.md`). diff --git a/README.md b/README.md index 78173da..7ff75a7 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ -# AWS AI/ML Security Assessment — Amazon Bedrock, Amazon SageMaker AI & Amazon Bedrock AgentCore +# AWS AI/ML Security Assessment — Amazon Bedrock, Amazon SageMaker AI, Amazon Bedrock AgentCore & Financial Services GenAI Risk [![License: MIT-0](https://img.shields.io/badge/License-MIT--0-yellow.svg)](https://opensource.org/licenses/MIT-0) [![Python 3.12+](https://img.shields.io/badge/Python-3.12+-blue.svg)](https://www.python.org/downloads/) [![AWS SAM](https://img.shields.io/badge/AWS-SAM-orange.svg)](https://aws.amazon.com/serverless/sam/) [![Serverless](https://img.shields.io/badge/Architecture-Serverless-green.svg)](https://aws.amazon.com/serverless/) -**Open-source automated security scanner for Amazon Bedrock, Amazon SageMaker AI, and Amazon Bedrock AgentCore** — Built on [AWS Well-Architected Framework (Generative AI Lens)](https://docs.aws.amazon.com/wellarchitected/latest/generative-ai-lens/generative-ai-lens.html) +**Open-source automated security scanner for Amazon Bedrock, Amazon SageMaker AI, Amazon Bedrock AgentCore, and Financial Services GenAI Risk** — Built on [AWS Well-Architected Framework (Generative AI Lens)](https://docs.aws.amazon.com/wellarchitected/latest/generative-ai-lens/generative-ai-lens.html) -Cloud security automation with **[52 security checks](docs/SECURITY_CHECKS.md)** for your generative AI and machine learning workloads. Identify IAM misconfigurations, encryption gaps, network isolation issues, and compliance violations with interactive HTML reports and actionable remediation guidance. +Cloud security automation with **[116 security checks](docs/SECURITY_CHECKS.md)** for your generative AI and machine learning workloads. Identify IAM misconfigurations, encryption gaps, network isolation issues, and compliance violations with interactive HTML reports and actionable remediation guidance. --- @@ -37,7 +37,7 @@ The framework generates professional, interactive security assessment reports wi - **Executive Summary** with severity counts and service breakdown - **Priority Recommendations** highlighting critical issues requiring immediate attention -- **[52 Security Checks](docs/SECURITY_CHECKS.md)** across Amazon Bedrock, Amazon SageMaker AI, and Amazon Bedrock AgentCore +- **[116 Security Checks](docs/SECURITY_CHECKS.md)** across Amazon Bedrock, Amazon SageMaker AI, Amazon Bedrock AgentCore, and Financial Services GenAI Risk - **Interactive Filtering** by account, service, severity, and status - **Light/Dark Mode Toggle** with persistent user preference - **Text Search** across all findings with real-time results @@ -80,7 +80,7 @@ Designed for workloads using [Amazon Bedrock](https://aws.amazon.com/bedrock/), | Challenge | How This Framework Helps | |-----------|-------------------------| | **Manual security audits are time-consuming** | Fully automated scanning with one-click CloudFormation deployment | -| **Inconsistent security checks across teams** | Standardized 52-check assessment based on AWS Well-Architected best practices | +| **Inconsistent security checks across teams** | Standardized 116-check assessment based on AWS Well-Architected best practices and AWS FinServ GenAI Risk guidance | | **Difficulty tracking AI/ML security posture** | Interactive HTML dashboards with severity breakdown and per-account visibility | | **Multi-account complexity** | Consolidated reporting across AWS Organizations with cross-account role assumption | | **Compliance and audit support** | Exportable reports to supplement your compliance program, with remediation guidance linked to AWS documentation | @@ -90,6 +90,7 @@ Designed for workloads using [Amazon Bedrock](https://aws.amazon.com/bedrock/), - **[Amazon Bedrock](docs/SECURITY_CHECKS.md#amazon-bedrock-security-checks-14)** (14 checks) - Guardrails, encryption, Amazon VPC endpoints, AWS IAM permissions, model invocation logging - **[Amazon SageMaker AI](docs/SECURITY_CHECKS.md#amazon-sagemaker-ai-security-checks-25)** (25 checks) - AWS Security Hub controls (SageMaker.1-5), encryption, network isolation, AWS IAM, MLOps - **[Amazon Bedrock AgentCore](docs/SECURITY_CHECKS.md#amazon-bedrock-agentcore-security-checks-13)** (13 checks) - Amazon VPC configuration, encryption, observability, resource policies +- **[Financial Services GenAI Risk](docs/SECURITY_CHECKS.md#financial-services-genai-risk-checks-64-additional-5-upstream-extensions)** (64 checks) - Unbounded consumption, excessive agency, supply chain, training data poisoning, hallucination, prompt injection, PII disclosure, and 8 more FinServ-specific risk categories derived from the [AWS FinServ GenAI Risk Guide](https://d1.awsstatic.com/onedam/marketing-channels/website/public/global-FinServ-ComplianceGuide-GenAIRisks-public.pdf) **Deployment Options:** - **Single-Account**: Assess security in one AWS account @@ -243,13 +244,32 @@ Deploy [2-aiml-security-codebuild.yaml](deployment/2-aiml-security-codebuild.yam 2. Select **Upload a template file** and upload the [2-aiml-security-codebuild.yaml](deployment/2-aiml-security-codebuild.yaml) file. 3. Set the `MultiAccountScan` parameter to `true`. 4. Optionally, provide your email address in the `EmailAddress` parameter for completion notifications. -5. Leave the remaining parameters at their default values. +5. Optionally, set `EnableFinServAssessment` to `true` to run the Financial Services GenAI risk checks (FS-01..FS-69). It defaults to `false`; enable it only if you must adhere to FinServ compliance, as it adds a dedicated FinServ section to the report. See [How finding severities are determined](#how-finding-severities-are-determined) and the [FinServ check references](docs/SECURITY_CHECKS_FINSERV_COMMON.md). +6. Leave the remaining parameters at their default values. 6. Navigate to the next page, read and acknowledge the notice, and click **Next**. 7. Review the information and click **Submit**. 8. Stack creation automatically triggers AWS CodeBuild, which deploys the assessment to each account and runs it. ## How It Works +### Optional: Financial Services GenAI Risk Checks (`EnableFinServAssessment`) + +The 64 Financial Services (FS-XX) GenAI risk checks are **opt-in** and default to `false`. Set the +`EnableFinServAssessment` deployment parameter to `true` only if you must adhere to FinServ +compliance. When enabled, the FinServ assessment Lambda runs and its findings appear in a dedicated +**Financial Services** section of the HTML report. When left `false`, no FinServ findings are +produced and the report omits the FinServ section entirely. The toggle is threaded into the Step +Functions execution input (`enableFinServ`); the FinServ Lambda is always deployed but is invoked +only when the flag is `true`. + +> **Deployment path note.** The `EnableFinServAssessment` parameter is wired through the CodeBuild-based deployment templates (`deployment/aiml-security-single-account.yaml` and `deployment/2-aiml-security-codebuild.yaml`), which thread it into every Step Functions `start-execution` call as `enableFinServ`. This is the supported install path. If you instead deploy `aiml-security-assessment/template.yaml` directly with `sam deploy` and start executions yourself, the state machine has no built-in trigger, so FinServ stays **off** unless you include `"enableFinServ": "true"` in the execution input you pass to `StartExecution`. + +#### Scope and limitations + +- **Single Region per run.** The assessment evaluates resources in the deployment Region only (the assessment Lambdas use their own Region). Region-scoped controls — WAF, API Gateway, Bedrock guardrails and Knowledge Bases, OpenSearch Serverless, Lambda, and SageMaker monitoring — are not evaluated in other Regions. For multi-Region GenAI workloads, deploy and run the assessment in each Region. +- **Heuristic and advisory checks.** Some controls cannot be verified through an API (application-layer controls, dataset contents, resource associations); these are reported as `ADVISORY`/`N/A` and require manual review. See [How finding severities are determined](#how-finding-severities-are-determined). +- **Permissions.** A check that lacks an IAM permission is reported as `COULD NOT ASSESS` (not a failure). Re-deploy the member role after any IAM template change so newer actions take effect. + ### Single-Account Mode (`MultiAccountScan=false`) - Creates a local `AIMLSecurityMemberRole` @@ -279,6 +299,7 @@ Deploy [2-aiml-security-codebuild.yaml](deployment/2-aiml-security-codebuild.yam - Amazon Bedrock Assessment AWS Lambda - Amazon SageMaker AI Assessment AWS Lambda - Amazon Bedrock AgentCore Assessment AWS Lambda + - Financial Services GenAI Risk Assessment AWS Lambda - AWS IAM Permission Caching AWS Lambda - Consolidated Report Generation AWS Lambda 4. **Assessment Execution**: AWS Step Functions orchestrate parallel AWS Lambda execution @@ -299,7 +320,7 @@ Deploy [2-aiml-security-codebuild.yaml](deployment/2-aiml-security-codebuild.yam ### Member Account Role (`AIMLSecurityMemberRole`) -- Read-only access to AI/ML services (Amazon Bedrock, Amazon SageMaker AI, Amazon Bedrock AgentCore) +- Read-only access to AI/ML services (Amazon Bedrock, Amazon SageMaker AI, Amazon Bedrock AgentCore, and FinServ-specific services: AWS WAF, AWS Shield, Amazon Macie, AWS Organizations, Amazon OpenSearch Serverless) - AWS IAM read permissions for security assessment - AWS CloudTrail, Amazon GuardDuty, and AWS Lambda read permissions - Amazon VPC and Amazon EC2 read permissions @@ -342,7 +363,7 @@ You can check the AWS CodeBuild console to confirm the assessment completed succ - **File Format**: `multi_account_report_YYYYMMDD_HHMMSS.html` - **Features**: - Executive summary with metrics (Total, High, Medium, Low severity counts) - - Service breakdown (Amazon Bedrock, Amazon SageMaker AI, Amazon Bedrock AgentCore) + - Service breakdown (Amazon Bedrock, Amazon SageMaker AI, Amazon Bedrock AgentCore, Financial Services GenAI Risk) - Priority recommendations - Light/dark mode toggle (persists through localStorage) - Dropdown filters for Account ID, Severity, Status @@ -357,6 +378,8 @@ You can check the AWS CodeBuild console to confirm the assessment completed succ - `bedrock_security_report_{execution_id}.csv` - Amazon Bedrock security assessment results - `sagemaker_security_report_{execution_id}.csv` - Amazon SageMaker AI security assessment results - `agentcore_security_report_{execution_id}.csv` - Amazon Bedrock AgentCore security assessment results + - `finserv_security_report_{execution_id}.csv` - Financial Services GenAI risk assessment results (64 FS-XX checks) + - `permissions_cache_{execution_id}.json` - IAM permissions cache - `security_assessment_{timestamp}_{execution_id}.html` - Consolidated HTML report (same features as multi-account report) @@ -376,6 +399,32 @@ You can check the AWS CodeBuild console to confirm the assessment completed succ | **Passed** | Checked resources met the assessed best practice at time of scan | | **N/A** | No resources exist to check (for example, no notebooks, no guardrails configured) | +### How finding severities are determined + +FinServ (`FS-`) check severities are assigned by a documented, reproducible methodology rather than +per-check intuition. Each control is scored on two axes — **Impact** (harm if the control is absent) +and **Likelihood** (probability the adverse outcome occurs given the control is absent) — and the +pair is mapped to a severity via a 3×3 matrix. The labels align with the **AWS Security Hub ASFF** +severity scale, so findings can be forwarded to Security Hub with consistent severities: + +| Label | ASFF normalized | Meaning | +|-------|-----------------|---------| +| Informational | 0 | No actionable issue (control not applicable, advisory/manual-review, or could-not-assess context) | +| Low | 1–39 | Does not require action on its own; compensating controls exist | +| Medium | 40–69 | Should be addressed, but not urgently | +| High | 70–89 | Should be addressed as a priority | + +Severity is a property of the **control** (its inherent risk), so a check's `Passed` and `Failed` +rows carry the same severity. The `N/A` family is fixed by disposition: *not-applicable* and +*advisory* findings are **Informational**; *could-not-assess* (access-denied / unsupported region) +findings are **Low**. `Critical` is reserved and not currently emitted. + +For the full methodology (matrix, factor definitions, disposition rules) and the authoritative +per-finding assignments, see +[FinServ Severity Methodology](docs/SECURITY_CHECKS_FINSERV_SEVERITY_METHODOLOGY.md) and the +[FinServ Severity Register](docs/SECURITY_CHECKS_FINSERV_SEVERITY_REGISTER.md). Mappings are +preliminary — validate with your MRM/Legal/Compliance teams before relying on them as audit evidence. + ## Customization ### Adding New Accounts @@ -498,7 +547,14 @@ For a clean removal, delete resources in this order: | Document | Description | |----------|-------------| -| [Security Checks Reference](docs/SECURITY_CHECKS.md) | Complete reference for all 52 security checks with severity levels | +| [Security Checks Reference](docs/SECURITY_CHECKS.md) | Complete reference for all 116 security checks with severity levels | +| [FinServ GenAI Risk Checks — Common](docs/SECURITY_CHECKS_FINSERV_COMMON.md) | Shared introduction, severity rubric, upstream-overlap table, and compliance framework mapping for FS-01..69 | +| [FinServ Part 1 — Infrastructure Controls](docs/SECURITY_CHECKS_FINSERV_PART1_INFRA_CONTROLS.md) | FS-01..26: Unbounded consumption, excessive agency, supply chain, training data poisoning, vector & embedding weaknesses | +| [FinServ Part 2 — Guardrails & Content Safety](docs/SECURITY_CHECKS_FINSERV_PART2_GUARDRAILS_CONTENT_SAFETY.md) | FS-27..46: Non-compliant output, misinformation, abusive/harmful output, biased output, PII disclosure | +| [FinServ Part 3 — App Layer & Gaps](docs/SECURITY_CHECKS_FINSERV_PART3_APP_LAYER_AND_GAPS.md) | FS-47..69: Hallucination, prompt injection, improper output handling, off-topic output, out-of-date training data, cross-category gap checks | +| [FinServ Severity Methodology](docs/SECURITY_CHECKS_FINSERV_SEVERITY_METHODOLOGY.md) | Likelihood × Impact → ASFF severity model, disposition rules, and research basis for FS check severities | +| [FinServ Severity Register](docs/SECURITY_CHECKS_FINSERV_SEVERITY_REGISTER.md) | Authoritative per-finding severity assignments (the single source of truth enforced by the drift-guard test) | +| [FinServ Compliance Mappings](docs/AIMLSecurityAssessment-MappingsTable.csv) | Machine-readable mapping of FS checks to SR 11-7, FFIEC CAT, NYDFS 500.06, PCI-DSS, DORA, MAS TRM, ISO 27001, OWASP LLM Top 10 | | [Troubleshooting Guide](docs/TROUBLESHOOTING.md) | Common issues, debugging tips, and FAQ | | [Developer Guide](docs/DEVELOPER_GUIDE.md) | Architecture details, adding custom checks, and contributing | diff --git a/aiml-security-assessment/functions/security/bedrock_assessments/app.py b/aiml-security-assessment/functions/security/bedrock_assessments/app.py index 2f62915..dc343f5 100644 --- a/aiml-security-assessment/functions/security/bedrock_assessments/app.py +++ b/aiml-security-assessment/functions/security/bedrock_assessments/app.py @@ -850,6 +850,12 @@ def check_bedrock_logging_configuration() -> Dict[str, Any]: """ Check if model invocation logging is enabled for Amazon Bedrock """ + # FinServ extension (FS-64): In addition to verifying that invocation + # logging is enabled, the FinServ guide (PDF §1.2.1, §1.2.6, §1.2.7) + # expects the log output to include guardrailTrace with action, + # inputAssessments, and outputAssessments to support SR 11-7 audit trails + # and NYDFS 500.06 retention. See docs/SECURITY_CHECKS_FINSERV_PART3_APP_LAYER_AND_GAPS.md + # (FS-64 → BR-04 extension note) for the detection / remediation language. logger.debug("Starting check for Bedrock model invocation logging configuration") try: findings = { @@ -955,6 +961,12 @@ def check_bedrock_cloudtrail_logging() -> Dict[str, Any]: """ Check if CloudTrail is configured to log Amazon Bedrock API calls """ + # FinServ extension (FS-23): In addition to verifying CloudTrail is logging + # Bedrock API calls, the FinServ guide (PDF §1.2.15) expects an advanced + # event selector for AWS::Bedrock::KnowledgeBase so Retrieve and + # RetrieveAndGenerate data events are captured (NOT logged by default). + # See docs/SECURITY_CHECKS_FINSERV_PART1_INFRA_CONTROLS.md (FS-23 → BR-06 + # extension note) for the detection / remediation language. logger.debug("Starting check for Bedrock CloudTrail logging configuration") try: findings = { diff --git a/aiml-security-assessment/functions/security/finserv_assessments/__init__.py b/aiml-security-assessment/functions/security/finserv_assessments/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/aiml-security-assessment/functions/security/finserv_assessments/app.py b/aiml-security-assessment/functions/security/finserv_assessments/app.py new file mode 100644 index 0000000..bdc5e1b --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_assessments/app.py @@ -0,0 +1,6454 @@ +""" +AWS FinServ GenAI Risk Assessment Lambda +========================================= +Implements 65 security checks (64 standalone + 1 shared FS-27 entry for the new +Automated Reasoning Policies check) derived from the AWS guide: +"Financial Services risk management of the use of Generative AI" +https://d1.awsstatic.com/onedam/marketing-channels/website/public/global-FinServ-ComplianceGuide-GenAIRisks-public.pdf + +Check ID namespace: FS-01 through FS-69 + FS-01 to FS-63 — original 63 checks across 15 risk categories + (FS-17, FS-18, FS-19 merged into upstream SM-07, SM-23, SM-22) + FS-64 to FS-69 — 6 material gap checks covering mitigations explicitly + called out in the Guide but absent from FS-01..63 and + the existing BR/SM/AC checks in the AIML Security Assessment. + (FS-64 merged into upstream BR-04) + +5 checks (FS-17, FS-18, FS-19, FS-23, FS-64) are contributed as upstream extensions +rather than standalone entries — see extension notes in the SECURITY_CHECKS_FINSERV +Part 1 and Part 3 markdown files. + +FS-27 is split into two check functions sharing the same check_id: + 1. check_guardrail_contextual_grounding() — verifies contextualGroundingPolicy + on guardrails (threshold-based, per-inference filtering). + 2. check_automated_reasoning_policies() — verifies Bedrock Automated Reasoning + policies (formal verification, GA August 2025, limited regions). +Both use FS-27 in the CSV and both appear in the build_finserv_checks() registry. + +These checks complement the existing BR/SM/AC checks in the AIML Security Assessment. + +COMPLIANCE_PLACEHOLDER: Each check maps to FinServ regulatory frameworks via the +COMPLIANCE_MAP dict below. These mappings now travel with every finding row in the +CSV report (the Compliance_Frameworks column) — see COMPLIANCE_MAP and create_finding. +Frameworks referenced: FFIEC CAT, SR 11-7, NYDFS 500.06, PCI-DSS 12.3.2, SOC 2 CC6, +ISO 27001 A.12, DORA Art.6, MAS TRM 9. + +Contribution workflow: + - Upstream repo: aws-samples/sample-aiml-security-assessment (OSPO-managed, so forks + are auto-approved by Amazon Code Defender). + - This Lambda is delivered via a personal fork + feature branch + PR. See + GIT_WORKFLOW.md for the full 9-step process (fork, branch, ASH scan, commit, push, + PR, GitHub Actions verification, reviewer assignment, optional Git Defender + exception ticket). + +Pre-commit quality gates (run every edit): + 1. ruff check + ruff format --check on this directory. + 2. sam local invoke FinServSecurityAssessmentFunction against a test event. + 3. cfn-lint / sam validate on the updated SAM templates. + 4. ash --source-dir --fail-on-findings --config-overrides + 'global_settings.severity_threshold=MEDIUM' — resolve every Critical and High + finding before opening the PR. + 5. git defender scan on the staged diff. +""" + +import boto3 +import csv +import functools +import json +import logging +import os +from dataclasses import dataclass +from datetime import datetime, timezone +from io import StringIO +from typing import Any, Dict, List, Optional + +from botocore.config import Config +from botocore.exceptions import ClientError, ParamValidationError + +from schema import create_finding + +# --------------------------------------------------------------------------- +# Boto3 config with adaptive retries +# --------------------------------------------------------------------------- +boto3_config = Config(retries=dict(max_attempts=10, mode="adaptive")) + +logger = logging.getLogger() +logger.setLevel(logging.WARNING) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def get_permissions_cache(execution_id: str) -> Optional[Dict[str, Any]]: + """Retrieve IAM permissions cache from S3 (same pattern as other assessments).""" + try: + s3_client = boto3.client("s3", config=boto3_config) + s3_key = f"permissions_cache_{execution_id}.json" + s3_bucket = os.environ.get("AIML_ASSESSMENT_BUCKET_NAME") + response = s3_client.get_object(Bucket=s3_bucket, Key=s3_key) + return json.loads(response["Body"].read().decode("utf-8")) + except ClientError as e: + logger.warning(f"Could not load permissions cache: {e}") + return None + except Exception as e: + logger.error(f"Unexpected error loading permissions cache: {e}", exc_info=True) + return None + + +def _empty_findings(check_name: str) -> Dict[str, Any]: + return {"check_name": check_name, "status": "PASS", "details": "", "csv_data": []} + + +def _bucket_name_from_arn(bucket_arn: str) -> str: + """Extract the bucket name from an S3 bucket ARN (arn:aws:s3:::name). + + Returns "" when the value is empty or is not a well-formed S3 bucket ARN, so + a malformed/unexpected value is skipped rather than mistakenly used as a + bucket name (which would then fail the subsequent S3 API call).""" + if not bucket_arn or ":::" not in bucket_arn: + return "" + return bucket_arn.split(":::", 1)[1] + + +def _paginate( + client, + operation_name: str, + result_key: str, + *, + token: "tuple[str, str] | None" = None, + **kwargs, +) -> List[Dict[str, Any]]: + """Collect all items across pages for a paginated list/describe operation by + calling the operation directly and following its continuation token. + + Calling the bound method directly (rather than via get_paginator) keeps this + uniform across services and unit-test mocks. The AWS APIs this Lambda uses + employ three continuation-token conventions, all handled here: + - NextToken / NextToken (organizations, sagemaker, config, events, ...) + - nextToken / nextToken (bedrock, bedrock-agent, ecr, ...) + - Marker / NextMarker (lambda) + - position / position (apigateway) + + A single-page response (no continuation token) yields exactly one call, so + this is a safe drop-in for previously non-paginated reads and for mocks that + stub the operation with a single return value. + + Args: + token: Optional explicit ``(output_field, input_param)`` pair that + bypasses the convention table entirely. When set, ``token[0]`` is + read from each response to find the continuation value and + ``token[1]`` is the request kwarg used to send it on the next call. + Use this for operations whose token convention is not in the table, + or where the output and input names differ from every entry already + there (e.g. WAFv2 ``NextMarker``/``NextMarker`` vs Lambda's + ``NextMarker``/``Marker``). When ``None`` (the default) the + existing convention table is used unchanged — all current callers + pass no ``token`` argument and are therefore byte-for-byte identical + to before this parameter was added. + """ + # (output token field in response, input token kwarg on request) + token_conventions = [ + ("NextToken", "NextToken"), + ("nextToken", "nextToken"), + ("NextMarker", "Marker"), + ("position", "position"), + ] + method = getattr(client, operation_name) + items: List[Dict[str, Any]] = [] + call_kwargs = dict(kwargs) + seen_tokens = set() + while True: + resp = method(**call_kwargs) + items.extend(resp.get(result_key, []) or []) + next_token = None + input_field = None + if token is not None: + # Explicit override: bypass the convention table. + out_field, in_field = token + if resp.get(out_field): + next_token = resp[out_field] + input_field = in_field + else: + for out_field, in_field in token_conventions: + if resp.get(out_field): + next_token = resp[out_field] + input_field = in_field + break + # Stop when there is no token, or if a token repeats (guards against a + # mock that returns the same token every call → infinite loop). + if not next_token or next_token in seen_tokens: + break + seen_tokens.add(next_token) + call_kwargs[input_field] = next_token + return items + + +def _error_findings(check_name: str, err: Exception) -> Dict[str, Any]: + return { + "check_name": check_name, + "status": "ERROR", + "details": str(err), + "csv_data": [], + } + + +# Error codes that mean "we were not allowed to read this resource" rather than +# "the control is absent." Treating an access error as a compliance failure +# produces a false non-compliant finding caused purely by a missing permission +# (the credibility problem a compliance tool must avoid). +_ACCESS_ERROR_CODES = frozenset( + { + "AccessDenied", + "AccessDeniedException", + "UnauthorizedOperation", + "AuthorizationError", + "Forbidden", + } +) + + +def _is_access_error(err: "ClientError") -> bool: + """True if a ClientError is a permission/authorization error (not a real + 'control absent' signal). Used so a missing IAM permission surfaces as a + could-not-assess condition instead of a false non-compliant finding.""" + try: + return err.response.get("Error", {}).get("Code", "") in _ACCESS_ERROR_CODES + except AttributeError: + return False + + +# Error codes meaning the S3 bucket a data source points to no longer exists. +# This is a distinct, actionable condition (a dangling KB data-source reference +# to a deleted bucket) — NOT "versioning/notifications absent." Surfacing it +# separately avoids a misleading "bucket without versioning" label when the real +# problem is the bucket was deleted out from under the Knowledge Base. +_MISSING_BUCKET_ERROR_CODES = frozenset({"NoSuchBucket", "404", "NotFound"}) + + +def _is_missing_bucket_error(err: "ClientError") -> bool: + """True if a ClientError indicates the S3 bucket does not exist (deleted / + dangling data-source reference) rather than a missing control or a missing + permission.""" + try: + return ( + err.response.get("Error", {}).get("Code", "") in _MISSING_BUCKET_ERROR_CODES + ) + except AttributeError: + return False + + +# Findings whose name starts with this prefix were emitted because the check +# could not run (e.g., missing IAM permission). They are visible in the report +# (Status="N/A") so a failed/permission-denied check does not silently vanish. +COULD_NOT_ASSESS_PREFIX = "COULD NOT ASSESS: " + +# --------------------------------------------------------------------------- +# ResourceInventory data model and helpers (REQ-3, REQ-4.1, REQ-6.4) +# --------------------------------------------------------------------------- + + +class _Unavailable: + """Sentinel marking an inventory field whose collection failed. Carries the + originating exception so dependent checks can reproduce the exact + COULD_NOT_ASSESS output that the check would have produced had it made the + call itself (design DD-3).""" + + __slots__ = ("error",) + + def __init__(self, error: Exception): + self.error = error + + +@dataclass(frozen=True) +class GuardrailInventory: + """Per-invocation guardrail enumeration. ``frozen=True`` prevents field + reassignment; the contained list/dict are read-only *by convention*.""" + + summaries: list # raw list_guardrails 'guardrails' entries + detail_by_id: dict # id -> get_guardrail(DRAFT) response (or _Unavailable) + + +@dataclass(frozen=True) +class KbInventory: + """Per-invocation Knowledge Base enumeration.""" + + summaries: list # knowledgeBaseSummaries + data_sources_by_kb: ( + dict # knowledgeBaseId -> list of dataSourceSummaries (or _Unavailable) + ) + data_source_detail: dict # (knowledgeBaseId, dataSourceId) -> get_data_source resp (or _Unavailable) + + +@dataclass(frozen=True) +class WebAclInventory: + """Per-invocation WAFv2 REGIONAL Web ACL enumeration.""" + + summaries: list # WebACLs list + detail_by_id: dict # Id -> get_web_acl response (or _Unavailable) + + +@dataclass(frozen=True) +class ResourceInventory: + """Per-invocation, read-only inventory of shared AWS resources. + + Each field is EITHER the collected data structure OR an ``_Unavailable`` + sentinel (carrying the original exception). Checks MUST NOT mutate the + lists/dicts stored here; the equivalence tests catch any such mutation. + + This is constructed once in ``lambda_handler`` and injected like + ``permission_cache`` (design DD-1, INV-6).""" + + lambda_functions: "list | _Unavailable" + guardrails: "GuardrailInventory | _Unavailable" + knowledge_bases: "KbInventory | _Unavailable" + buckets: "list | _Unavailable" + web_acls: "WebAclInventory | _Unavailable" + + +def inv_available(field) -> bool: + """Return True when *field* holds real data; False when it is an + ``_Unavailable`` sentinel.""" + return not isinstance(field, _Unavailable) + + +def require(inventory: "ResourceInventory | None", field_name: str): + """Return ``inventory.`` when available; raise the stored + exception when the field is ``_Unavailable``; raise ``RuntimeError`` when + ``inventory`` is ``None`` (the test-only default from design DD-2b). + + The caller's existing outer ``try/except`` turns the raised exception into + ``_error_findings`` → ``COULD_NOT_ASSESS``, matching today's behavior.""" + if inventory is None: + raise RuntimeError("resource inventory not provided") + value = getattr(inventory, field_name) + if isinstance(value, _Unavailable): + raise value.error + return value + + +# --------------------------------------------------------------------------- +# COMPLIANCE_MAP — per-check regulatory framework mappings +# +# Each value is a pipe-separated string of FinServ framework identifiers that +# travel with every finding row in the CSV report (Compliance_Frameworks column). +# This mirrors the ASFF Compliance.RelatedRequirements pattern used by AWS +# Security Hub: compliance metadata is embedded in the finding itself, not kept +# in a separate sidecar document. +# +# Disclaimer: mappings are PRELIMINARY and ILLUSTRATIVE. They have not been +# reviewed by AWS Security Assurance Services, external auditors, or the +# regulators named. Each firm should have its own MRM, Legal, and Compliance +# teams validate these mappings against their specific interpretation of each +# framework before relying on them as audit evidence. +# --------------------------------------------------------------------------- +COMPLIANCE_MAP: Dict[str, str] = { + # Category 1: Unbounded Consumption + "FS-01": "FFIEC CAT | DORA Art.6", + "FS-02": "FFIEC CAT | DORA Art.6 | PCI-DSS 12.3.2", + "FS-03": "FFIEC CAT | SR 11-7", + "FS-04": "FFIEC CAT | SR 11-7", + "FS-05": "FFIEC CAT | DORA Art.6", + "FS-06": "FFIEC CAT | SR 11-7", + # Category 2: Excessive Agency + "FS-07": "SR 11-7 | FFIEC CAT", + "FS-08": "SR 11-7 | MAS TRM 9.1", + "FS-09": "FFIEC CAT | SR 11-7", + "FS-10": "SR 11-7 | FFIEC CAT | MAS TRM 9.2", + "FS-11": "FFIEC CAT | DORA Art.6", + # Category 3: Supply Chain Vulnerabilities + "FS-12": "SR 11-7 | FFIEC CAT | ISO 27001 A.15.2", + "FS-13": "SR 11-7 | ISO 27001 A.12.5 | FFIEC CAT", + "FS-14": "SR 11-7 | FFIEC CAT | ISO 27001 A.15.1", + "FS-15": "SR 11-7 | FFIEC CAT | MAS TRM 9.3", + "FS-16": "ISO 27001 A.12.6 | FFIEC CAT | DORA Art.6", + # Category 4: Training Data & Model Poisoning + "FS-20": "SR 11-7 | FFIEC CAT", + "FS-21": "SR 11-7 | ISO 27001 A.12.3 | FFIEC CAT", + # Category 5: Vector & Embedding Weaknesses + "FS-22": "NYDFS 500 | FFIEC CAT | PCI-DSS 12.3.2", + "FS-24": "NYDFS 500 | FFIEC CAT | PCI-DSS 12.3.2", + "FS-25": "NYDFS 500 | PCI-DSS 3.5 | FFIEC CAT", + "FS-26": "NYDFS 500 | FFIEC CAT | PCI-DSS 1.3", + # Category 6: Non-Compliant Output + "FS-27": "SR 11-7 | FFIEC CAT | MAS TRM 9.2", + "FS-28": "SR 11-7 | FFIEC CAT | NYDFS 500 | MAS TRM 9.2", + "FS-29": "SR 11-7 | FFIEC CAT | NYDFS 500 | MAS TRM 9.2", + "FS-30": "SR 11-7 | FFIEC CAT | NYDFS 500", + # Category 7: Misinformation + "FS-31": "SR 11-7 | FFIEC CAT", + "FS-32": "SR 11-7 | FFIEC CAT | MAS TRM 9.2", + "FS-33": "SR 11-7 | FFIEC CAT | ISO 27001 A.12", + "FS-34": "SR 11-7 | FFIEC CAT", + # Category 8: Abusive or Harmful Output + "FS-35": "SR 11-7 | FFIEC CAT", + "FS-36": "SR 11-7 | FFIEC CAT", + "FS-37": "SR 11-7 | FFIEC CAT | MAS TRM 9.2", + "FS-38": "SR 11-7 | FFIEC CAT", + # Category 9: Biased Output + "FS-39": "SR 11-7 | FFIEC CAT | ECOA/Fair Housing", + "FS-40": "SR 11-7 | FFIEC CAT | ECOA/Fair Housing", + "FS-41": "SR 11-7 | FFIEC CAT", + "FS-42": "SR 11-7 | FFIEC CAT", + # Category 10: Sensitive Information Disclosure + "FS-43": "NYDFS 500 | FFIEC CAT | PCI-DSS", + "FS-44": "NYDFS 500 | FFIEC CAT | PCI-DSS", + "FS-45": "NYDFS 500 | FFIEC CAT | PCI-DSS", + "FS-46": "NYDFS 500 | FFIEC CAT | ISO 27001 A.12 | PCI-DSS", + # Category 11: Hallucination + "FS-47": "SR 11-7 | FFIEC CAT", + "FS-48": "SR 11-7 | FFIEC CAT", + "FS-49": "SR 11-7 | FFIEC CAT | MAS TRM 9.2", + "FS-50": "SR 11-7 | FFIEC CAT", + # Category 12: Prompt Injection + "FS-51": "NYDFS 500 | FFIEC CAT | OWASP LLM Top 10", + "FS-52": "NYDFS 500 | FFIEC CAT | ISO 27001 A.12 | OWASP LLM Top 10", + "FS-53": "NYDFS 500 | PCI-DSS | FFIEC CAT | OWASP LLM Top 10", + "FS-54": "NYDFS 500 | FFIEC CAT | OWASP LLM Top 10", + # Category 13: Improper Output Handling + "FS-55": "FFIEC CAT | OWASP LLM Top 10", + "FS-56": "NYDFS 500 | PCI-DSS | OWASP LLM Top 10", + "FS-57": "NYDFS 500.06 | FFIEC CAT | OWASP LLM Top 10", + "FS-58": "FFIEC CAT | OWASP LLM Top 10", + # Category 14: Off-Topic & Inappropriate Output + "FS-59": "SR 11-7 | FFIEC CAT", + "FS-60": "SR 11-7 | FFIEC CAT", + # Category 15: Out-of-Date Training Data + "FS-61": "SR 11-7 | FFIEC CAT", + "FS-62": "SR 11-7 | FFIEC CAT | MAS TRM 9.2", + "FS-63": "SR 11-7 | FFIEC CAT | ISO 27001 A.12", + # Material Gap Checks + "FS-65": "FFIEC CAT | DORA Art.6 | ISO 27001 A.12", + "FS-66": "NYDFS 500 | SR 11-7 | MAS TRM 9 | PCI-DSS", + "FS-67": "SR 11-7 | FFIEC CAT | MAS TRM 9 | PCI-DSS", + "FS-68": "DORA Art.6 | FFIEC CAT | PCI-DSS | OWASP LLM Top 10", + "FS-69": "NYDFS 500 | FFIEC CAT | OWASP LLM Top 10", +} + + +# --------------------------------------------------------------------------- +# SEVERITY METHODOLOGY (see docs/severity-methodology.md + severity-register.md) +# +# Severity = property of the CONTROL (the risk it mitigates), assigned once via a +# Likelihood (L) x Impact (I) matrix mapped to the AWS Security Hub ASFF label set, +# then applied to every Passed/Failed row of that control. The N/A family follows a +# fixed disposition rule so "nothing to assess" rows are consistent. +# +# Matrix (I x L -> label), 1=Low 2=Medium 3=High: +# I=3: L1->Medium L2->High L3->High +# I=2: L1->Low L2->Medium L3->High +# I=1: L1->Low L2->Low L3->Medium +# (Critical is intentionally NOT used in this round — see methodology §6.) +# +# Disposition -> severity: +# PASS / FAIL -> control severity (from the matrix) +# NOT_APPLICABLE -> Informational (ASFF: "no issue was found") +# ADVISORY -> Informational (no API can verify it) +# COULD_NOT_ASSESS -> Low (unknown state; re-run after fixing access) +# SOFT_WARNING -> control severity (intentional non-failing assessed state) +# --------------------------------------------------------------------------- + +_SEVERITY_MATRIX = { + (3, 1): "Medium", + (3, 2): "High", + (3, 3): "High", + (2, 1): "Low", + (2, 2): "Medium", + (2, 3): "High", + (1, 1): "Low", + (1, 2): "Low", + (1, 3): "Medium", +} + + +def _label_from_matrix(impact: int, likelihood: int) -> str: + """Map an (Impact, Likelihood) pair (each 1-3) to an ASFF severity label.""" + return _SEVERITY_MATRIX[(impact, likelihood)] + + +# Disposition tiers whose severity is fixed by the disposition, not by I x L. +_DISPOSITION_SEVERITY = { + "NOT_APPLICABLE": "Informational", + "ADVISORY": "Informational", + "COULD_NOT_ASSESS": "Low", +} + + +# Authoritative per-finding severity register (keyed by finding-name). +# Source of truth derived from docs/severity-register.md. The test suite +# (test_severity_register.py) asserts every emitted severity matches this map. +# Entries: finding_name -> (severity, disposition). I/L rationale lives in the doc. +SEVERITY_REGISTER: Dict[str, str] = { + # --- FS-01 (Shield = Low; WAF = Medium) --- + "AWS Shield Advanced Not Enabled": "Low", + "AWS Shield Advanced Enabled": "Low", + "No Regional WAF Web ACLs Found": "Medium", + "Regional WAF Web ACLs Present": "Medium", + # --- FS-02 (rate limiting = Medium) --- + "No API Gateway Usage Plans Found": "Informational", + "API Gateway Usage Plans Missing Throttle": "Medium", + "API Gateway Rate Limiting Configured": "Medium", + # --- FS-03 (token quotas = Medium) --- + "No Bedrock Token Quotas Returned": "Medium", + "Bedrock Default Quotas Unavailable — Customization Undetermined": "Medium", + "Bedrock Token Quotas Customized": "Medium", + "Bedrock Token Quotas At Default": "Medium", + # --- FS-04 --- + "No Cost Anomaly Detection Monitors": "Medium", + "Cost Anomaly Monitors Do Not Cover Bedrock/SageMaker": "Medium", + "Cost Anomaly Detection Configured": "Medium", + # --- FS-05 --- + "No Bedrock CloudWatch Alarms Found": "Medium", + "Bedrock CloudWatch Alarms Present": "Medium", + # --- FS-06 --- + "No AI/ML Service Budgets Configured": "Medium", + "AI/ML Service Budgets Configured": "Medium", + # --- FS-07 (agent action boundaries = High) --- + "Agent Action Boundary Check": "Informational", + "Bedrock Agent Overly Broad Action Permissions": "High", + "Agent Action Boundaries Look Appropriate": "High", + # --- FS-08 (AgentCore policy engine = High) --- + "AgentCore Policy Engine — Access Check": "Low", + "No AgentCore Runtimes Found": "Informational", + "AgentCore Runtimes Missing Policy Engine": "High", + "AgentCore Policy Engine Configured": "High", + # --- FS-09 --- + "Agent Lambda Functions Without Concurrency Limits": "Medium", + "Agent Lambda Concurrency Limits Present": "Medium", + # --- FS-10 (human-in-the-loop = High) --- + "Human-in-the-Loop Check — No Agent Workflows Found": "Informational", + "Human Approval Steps Found in Agent Workflows": "High", + "Agent Workflows Missing Human Approval Steps": "High", + # --- FS-11 --- + "No Agent Rate Alarms Found": "Medium", + "Agent Rate Alarms Present": "Medium", + # --- FS-12 (SCPs = High) --- + "SCP Model Access Check — Not in Organization": "Informational", + "No Bedrock-Scoped SCPs Found": "High", + "Bedrock SCPs Found": "High", + # --- FS-13 --- + "Models Missing Provenance Tags": "Medium", + "Model Provenance Tags Present": "Medium", + # --- FS-14 --- + "No Model Governance Config Rules Found": "Medium", + "Model Governance Config Rules Present": "Medium", + # --- FS-15 (eval jobs exist = Medium; absence is a real FAIL) --- + "No Bedrock Evaluation Jobs Found": "Medium", + "Bedrock Evaluation Jobs Present": "Medium", + # --- FS-16 (ECR scanning = High) --- + "No ECR Repositories Found": "Informational", + "ECR Repositories Without Image Scanning": "High", + "ECR Image Scanning Enabled": "High", + # --- FS-20 --- + "No SageMaker Feature Groups Found": "Informational", + "Feature Groups Without Offline Store": "Medium", + "Feature Store Offline Store Active": "Medium", + # --- FS-21 (training-data integrity = High) --- + "No Training Data Buckets Identified": "Informational", + "Training Data Buckets Without Versioning": "High", + "Training Data Buckets Have Versioning": "High", + # --- FS-22 (KB IAM least privilege = High) --- + "Overly Permissive Knowledge Base IAM Roles": "High", + "Knowledge Base IAM Permissions Look Appropriate": "High", + # --- FS-24 (advisory) --- + "No Knowledge Bases Found": "Informational", + "ADVISORY: Knowledge Base Metadata Filtering — Manual Review Required": "Informational", + # --- FS-25 (KB encryption = High) --- + "No OpenSearch Serverless Encryption Policies": "Informational", + "OpenSearch Serverless Encryption Not Using Customer-Managed Keys": "High", + "OpenSearch Serverless Encryption Policies Present": "High", + # --- FS-26 (VPC isolation = High) --- + "No OpenSearch Serverless Network Policies": "High", + "OpenSearch Serverless Collections Not VPC-Restricted": "High", + "OpenSearch Serverless VPC Access Configured": "High", + # --- FS-27 (contextual grounding = High; ARC = Medium) --- + "No Guardrails — Contextual Grounding Not Applicable": "Informational", + "No Guardrails With Contextual Grounding": "High", + "Contextual Grounding Enabled on Guardrails": "High", + "Automated Reasoning Policies — Access Check": "Low", + "No Automated Reasoning Policies Found": "Medium", + "Automated Reasoning Policies Found": "Medium", + # --- FS-28 (denied topics = High) --- + "No Guardrails — Denied Topics Not Applicable": "Informational", + "No Guardrails With Denied Financial Topics": "High", + "Denied Topics Configured on CLASSIC Tier": "High", + "Guardrails With Topic Policies Found": "High", + # --- FS-29 (advisory) --- + "ADVISORY: Compliance Disclaimer — Manual Review Required": "Informational", + # --- FS-30 (advisory — cannot inspect dataset content) --- + "ADVISORY: Compliance Dataset Coverage — Manual Review Required": "Informational", + # --- FS-31 --- + "Knowledge Base Data Sources Past Review Threshold": "Medium", + "Knowledge Base Data Sources Recently Synced": "Medium", + # --- FS-32 (advisory) --- + "ADVISORY: Source Attribution — Manual Review Required": "Informational", + # --- FS-33 (distinct risks: deleted bucket High, versioning Medium) --- + "KB Data Source References a Deleted S3 Bucket": "High", + "KB Data Source Buckets Without Versioning": "Medium", + "KB Data Source Buckets Have Versioning": "Medium", + # --- FS-34 --- + "Legacy Foundation Models Available in Region": "Informational", + "Foundation Models Are Current": "Medium", + # --- FS-35 (advisory) --- + "ADVISORY: Harmful-Content Test Coverage — Manual Review Required": "Informational", + # --- FS-36 (content filters = High) --- + "No Guardrails — Content Filters Not Applicable": "Informational", + "No Guardrails With Content Filters": "High", + "Guardrail Content Filters on CLASSIC Tier": "High", + "Guardrail Content Filters Configured (STANDARD Tier)": "High", + # --- FS-37 (advisory) --- + "ADVISORY: User Feedback Mechanism — Manual Review Required": "Informational", + # --- FS-38 (word filters = Medium) --- + "No Guardrails — Word Filters Not Applicable": "Informational", + "No Guardrails With Word Filters": "Medium", + "Guardrail Word Filters Configured": "Medium", + # --- FS-39 (bias = High) --- + "No SageMaker Clarify Bias Monitoring": "High", + "SageMaker Clarify Bias Monitoring Active": "High", + # --- FS-40 (advisory) --- + "ADVISORY: Bias Dataset Coverage — Manual Review Required": "Informational", + # --- FS-41 (explainability = High) --- + "No SageMaker Clarify Explainability Monitoring": "High", + "SageMaker Clarify Explainability Active": "High", + # --- FS-42 --- + "No SageMaker Model Cards Found": "Medium", + "SageMaker Model Cards Present": "Medium", + # --- FS-43 (log data protection = High) --- + "No CloudWatch Logs Data Protection Policies": "High", + "CloudWatch Logs Data Protection Policies Present": "High", + # --- FS-44 (Macie = High) --- + "Amazon Macie Not Enabled": "High", + "Amazon Macie Enabled": "High", + # --- FS-45 (PII filters = High) --- + "No Guardrails — PII Filters Not Applicable": "Informational", + "No Guardrails With PII Filters": "High", + "Guardrail PII Filters Configured": "High", + # --- FS-46 (classification = Medium) --- + "No AI/ML Data Buckets Identified": "Informational", + "AI/ML Buckets Without Data Classification Tags": "Medium", + "AI/ML Buckets Have Classification Tags": "Medium", + # --- FS-47 (grounding threshold = High) --- + "No Guardrails — Grounding Threshold Not Applicable": "Informational", + "Guardrails With Low Grounding Thresholds": "High", + "No Guardrails With a Grounding Filter": "High", + "Guardrail Grounding Thresholds Appropriate": "High", + # --- FS-48 --- + "No Active Knowledge Bases for RAG": "Medium", + "Active Knowledge Bases for RAG Present": "Medium", + # --- FS-49 (advisory) --- + "ADVISORY: Hallucination Disclaimer — Manual Review Required": "Informational", + # --- FS-50 (relevance grounding = Medium) --- + "No Guardrails With Relevance Grounding Filters": "Medium", + "Relevance Grounding Filters Present": "Medium", + # --- FS-51 (prompt attack = High) --- + "No Guardrails — Prompt Attack Filters Not Applicable": "Informational", + "No Guardrails With Prompt Attack Filters": "High", + "Prompt Attack Filters on CLASSIC Tier": "High", + "Prompt Attack Filters Configured (STANDARD Tier)": "High", + # --- FS-52 --- + "No Bedrock-Related Lambda Functions Found": "Informational", + "Bedrock Lambda Functions on Deprecated Runtimes": "Medium", + "Bedrock Lambda Functions on Current Runtimes": "Medium", + # --- FS-53 (injection rules = High) --- + "No WAF Web ACLs — Injection Rules Not Applicable": "Informational", + "WAF ACLs Missing Injection Protection Rules": "High", + "WAF Injection Protection Rules Present": "High", + # --- FS-54 (advisory) --- + "ADVISORY: Penetration Testing — Manual Review Required": "Informational", + # --- FS-55 --- + "No Output Validation Functions Found": "Medium", + "Output Validation Functions Present": "Medium", + # --- FS-56 (XSS = Medium; now has a FAIL path) --- + "No WAF ACLs — XSS Prevention Not Applicable": "Informational", + "WAF ACLs Missing Common Rule Set (XSS)": "Medium", + "XSS Prevention Common Rule Set Present": "Medium", + # --- FS-57 (advisory) --- + "ADVISORY: Output Encoding — Manual Review Required": "Informational", + # --- FS-58 (advisory) --- + "ADVISORY: Output Schema Validation — Manual Review Required": "Informational", + # --- FS-59 (topic allowlist = Medium) --- + "No Guardrails — Topic Allowlist Not Applicable": "Informational", + "No Guardrails With Topic Restrictions": "Medium", + "Topic Restrictions Configured on CLASSIC Tier": "Medium", + "Guardrail Topic Restrictions Configured": "Medium", + # --- FS-60 (advisory) --- + "ADVISORY: Contextual Grounding for Off-Topic Prevention": "Informational", + # --- FS-61 --- + "No Automated KB Sync Schedules Detected": "Medium", + "Automated KB Sync Schedules Present": "Medium", + # --- FS-62 (advisory) --- + "ADVISORY: Data Currency Disclaimer — Manual Review Required": "Informational", + # --- FS-63 --- + "Legacy Models Without Lifecycle Management": "Medium", + "Foundation Model Lifecycle Management": "Medium", + # --- FS-65 (distinct risks) --- + "KB Data Source Buckets Missing S3 Event Notifications": "Medium", + "KB Data Source S3 Event Notifications Configured": "Medium", + # --- FS-66 (identity propagation = High) --- + "AgentCore Identity Propagation — Access Check": "Low", + "AgentCore Runtimes Missing End-User Identity Propagation": "High", + "AgentCore End-User Identity Propagation Configured": "High", + # --- FS-67 (transaction thresholds = High) --- + "No Agent Action-Group Lambda Functions Found": "Informational", + "Agent Action-Group Lambdas May Lack Transaction Thresholds": "High", + "Agent Action-Group Lambdas Have Threshold Configuration": "High", + # --- FS-68 (body-size = Medium; now has N/A branch) --- + "API Gateway Request Body Size Limits Not Enforced": "Medium", + "API Gateway Request Body Size Limits — Not Applicable": "Informational", + "API Gateway Request Body Size Limits Configured": "Medium", + # --- FS-69 --- + "No Prompt Input Validation Function Found": "Medium", + "Prompt Input Validation Functions Present": "Medium", +} + + +def _could_not_assess_row(check_id: str, check_name: str, err: Any) -> Dict[str, Any]: + """ + Synthesize one visible finding row for a check that errored out and produced + no rows. Uses Status="N/A", Severity="Low" (the COULD_NOT_ASSESS disposition — + see severity-methodology.md §3.4) so the gap surfaces in the report as an + unknown/assessment-gap without inflating the Failed count or implying a + confirmed control failure. + """ + return create_finding( + check_id=check_id, + finding_name=f"{COULD_NOT_ASSESS_PREFIX}{check_name}", + finding_details=( + f"This check could not be completed (error: {err}). The most common cause " + "is a missing IAM permission for the assessment role; it may also indicate " + "an unsupported region or an outdated botocore. This control was NOT assessed " + "— verify the role's permissions and re-run, and assess this control manually " + "until resolved." + ), + resolution=( + "1. Confirm the assessment role grants the actions this check requires " + "(see the documented IAM permission set in the README).\n" + "2. Confirm the service/feature is supported in the assessed region.\n" + "3. Ensure botocore meets the version floor in requirements.txt.\n" + "4. Re-run the assessment; assess this control manually until it succeeds." + ), + reference="https://docs.aws.amazon.com/IAM/latest/UserGuide/troubleshoot_access-denied.html", + severity="Low", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP.get(check_id, ""), + ) + + +# =========================================================================== +# CATEGORY 1: UNBOUNDED CONSUMPTION (FS-01 to FS-06) +# Risk: GenAI workloads can be exploited to exhaust compute/cost budgets +# COMPLIANCE_PLACEHOLDER: [FFIEC CAT, DORA Art.6, SR 11-7 Appendix A] +# =========================================================================== + + +def check_waf_shield_on_bedrock_endpoints(inventory) -> Dict[str, Any]: + """ + FS-01 — Verify AWS WAF is associated with API Gateway or ALB endpoints + that front Bedrock/GenAI workloads, and that AWS Shield Advanced is enabled. + COMPLIANCE_PLACEHOLDER: [FFIEC CAT Cyber Risk Management, DORA Art.6 ICT Risk] + """ + findings = _empty_findings("WAF and Shield Protection Check") + try: + shield = boto3.client("shield", config=boto3_config) + + # Check Shield Advanced subscription + shield_enabled = False + try: + shield.describe_subscription() + shield_enabled = True + except shield.exceptions.ResourceNotFoundException: + pass + except ClientError: + pass + + # Check WAF Web ACLs exist (regional, covering API GW / ALB) + # require() raises if inventory is None or the field is _Unavailable, + # which propagates to the outer except and yields COULD_NOT_ASSESS. + acls = require(inventory, "web_acls").summaries + + if not shield_enabled: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-01", + finding_name="AWS Shield Advanced Not Enabled", + finding_details=( + "AWS Shield Advanced is not subscribed. GenAI API endpoints are " + "vulnerable to volumetric DDoS attacks that can exhaust token quotas " + "and inflate costs." + ), + resolution=( + "1. Subscribe to AWS Shield Advanced for DDoS protection.\n" + "2. After subscribing, explicitly add resource protections in the " + "Shield Advanced console for each Bedrock-facing resource " + "(API Gateway stages, ALBs, CloudFront distributions, Route 53 hosted zones). " + "Shield Advanced subscription alone does NOT automatically protect resources — " + "each resource must be individually added to receive protection.\n" + "3. Enable Shield Response Team (SRT) access and configure proactive engagement.\n" + "4. Alternatively, use AWS Firewall Manager with a Shield Advanced policy " + "to automate resource protection based on tags or resource types." + ), + reference="https://docs.aws.amazon.com/waf/latest/developerguide/shield-chapter.html", + severity="Low", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-01"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-01", + finding_name="AWS Shield Advanced Enabled", + finding_details="AWS Shield Advanced subscription is active.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/waf/latest/developerguide/shield-chapter.html", + severity="Low", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-01"], + ) + ) + + if not acls: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-01", + finding_name="No Regional WAF Web ACLs Found", + finding_details=( + "No AWS WAF regional Web ACLs found. Without WAF, GenAI endpoints " + "lack rate-based rules to block abusive callers." + ), + resolution=( + "1. Create a WAF Web ACL with rate-based rules (e.g., 1000 req/5 min per IP).\n" + "2. Associate the ACL with API Gateway stages or ALBs fronting Bedrock.\n" + "3. Add AWS Managed Rules for known bad inputs." + ), + reference="https://docs.aws.amazon.com/waf/latest/developerguide/waf-chapter.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-01"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-01", + finding_name="Regional WAF Web ACLs Present", + finding_details=f"Found {len(acls)} regional WAF Web ACL(s).", + resolution="Verify ACLs are associated with Bedrock-facing endpoints.", + reference="https://docs.aws.amazon.com/waf/latest/developerguide/waf-chapter.html", + severity="Medium", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-01"], + ) + ) + except Exception as e: + return _error_findings("WAF and Shield Protection Check", e) + return findings + + +def check_api_gateway_rate_limiting() -> Dict[str, Any]: + """ + FS-02 — Verify API Gateway usage plans enforce throttling on GenAI endpoints. + COMPLIANCE_PLACEHOLDER: [FFIEC CAT, DORA Art.6, PCI-DSS 12.3.2] + """ + findings = _empty_findings("API Gateway Rate Limiting Check") + try: + apigw = boto3.client("apigateway", config=boto3_config) + plans = _paginate(apigw, "get_usage_plans", "items") + + plans_without_throttle = [ + p["name"] + for p in plans + if not p.get("throttle") or p["throttle"].get("rateLimit", 0) == 0 + ] + + if not plans: + findings["csv_data"].append( + create_finding( + check_id="FS-02", + finding_name="No API Gateway Usage Plans Found", + finding_details="No usage plans configured. GenAI API endpoints may have no rate limits.", + resolution=( + "Create API Gateway usage plans with throttle settings " + "(rateLimit and burstLimit) for all Bedrock-facing APIs." + ), + reference="https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-request-throttling.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-02"], + ) + ) + elif plans_without_throttle: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-02", + finding_name="API Gateway Usage Plans Missing Throttle", + finding_details=( + f"Usage plans without throttling: {', '.join(plans_without_throttle)}. " + "Unbounded API calls can exhaust Bedrock token quotas and inflate costs." + ), + resolution=( + "Set rateLimit and burstLimit on all usage plans associated with " + "GenAI API stages. Consider per-consumer API keys with individual quotas." + ), + reference="https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-request-throttling.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-02"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-02", + finding_name="API Gateway Rate Limiting Configured", + finding_details=f"All {len(plans)} usage plan(s) have throttle settings.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/apigateway/latest/developerguide/api-gateway-request-throttling.html", + severity="Medium", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-02"], + ) + ) + except Exception as e: + return _error_findings("API Gateway Rate Limiting Check", e) + return findings + + +def check_bedrock_token_quotas() -> Dict[str, Any]: + """ + FS-03 — Check whether Bedrock service quotas for tokens-per-minute (TPM) + have been reviewed and raised above AWS defaults. + + Token-based quotas are the primary signal. RPM (requests-per-minute) quotas + are model-specific on the bedrock-runtime endpoint: some models (e.g., Claude + Opus 4.7/4.8) are governed solely by TPM with no RPM quota; others have both. + Because RPM applicability varies by model, only TPM quotas drive this verdict + and an absent RPM quota must never trigger a failure. + + Note: The bedrock-mantle endpoint (OpenAI-compatible, GA May 2026) exposes + separate input-tokens-per-minute and output-tokens-per-minute quotas also + under ServiceCode "bedrock". This check focuses on bedrock-runtime on-demand + TPM quotas; bedrock-mantle quotas are not explicitly separated here. + + Verdict logic (value-based, not adjustability-based): + - at least one applied token-quota Value > its AWS default → customized → PASS/Passed + - all applied token-quota Values == their defaults → at default → WARN/N-A (soft) + - no applied token quotas returned → WARN/Failed + explanation + - AWS default quotas could not be retrieved → WARN/Failed + "undetermined" + COMPLIANCE_PLACEHOLDER: [FFIEC CAT, SR 11-7] + """ + findings = _empty_findings("Bedrock Token Quota Review") + try: + sq = boto3.client("service-quotas", config=boto3_config) + + # Applied quotas (paginated). TPM quotas are the primary signal; RPM quotas + # are model-specific and their absence must not trigger a failure verdict. + applied = [] + for page in sq.get_paginator("list_service_quotas").paginate( + ServiceCode="bedrock" + ): + applied.extend(page.get("Quotas", [])) + token_quotas = [q for q in applied if "token" in q.get("QuotaName", "").lower()] + + # Empty-applied-list branch: no token quotas found at all. + if not token_quotas: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-03", + finding_name="No Bedrock Token Quotas Returned", + finding_details=( + "No Bedrock token-based service quotas were returned. This may indicate " + "a permissions issue (servicequotas:ListServiceQuotas), an unsupported " + "region, or that no Bedrock-specific quotas exist for this account. " + "Verify manually in the Service Quotas console." + ), + resolution=( + "1. Confirm the assessment role has servicequotas:ListServiceQuotas.\n" + "2. Verify Bedrock is available in the assessed region.\n" + "3. Review Bedrock token quotas in the Service Quotas console." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/quotas.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-03"], + ) + ) + return findings + + # AWS default quotas for comparison (paginated). + default_values = {} + for page in sq.get_paginator("list_aws_default_service_quotas").paginate( + ServiceCode="bedrock" + ): + for q in page.get("Quotas", []): + if q.get("QuotaCode") is not None: + default_values[q["QuotaCode"]] = q.get("Value") + + # Default-lookup-fail branch: cannot compare without defaults. Do NOT + # silently compare a value against itself (which would always Fail). + if not default_values: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-03", + finding_name="Bedrock Default Quotas Unavailable — Customization Undetermined", + finding_details=( + "AWS default service quotas for Bedrock could not be retrieved " + "(list_aws_default_service_quotas returned nothing), so whether the " + "applied quotas have been customized cannot be determined. This is " + "commonly a permissions issue (servicequotas:ListAWSDefaultServiceQuotas) " + "or an unsupported region." + ), + resolution=( + "1. Confirm the assessment role has servicequotas:ListAWSDefaultServiceQuotas.\n" + "2. Re-run the assessment once defaults are retrievable.\n" + "3. Until then, verify Bedrock token quotas manually in the Service Quotas console." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/quotas.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-03"], + ) + ) + return findings + + # Value-based comparison. An applied Value == default Value is the + # expected, non-customized state (ListServiceQuotas may return only + # default values for some quotas) — this is NOT an error. At-default is + # reported as a soft warning (WARN/N-A), not a failure, since it is a + # legitimate verified posture. + any_customized = any( + q.get("QuotaCode") in default_values + and default_values[q["QuotaCode"]] is not None + and q.get("Value", 0) > default_values[q["QuotaCode"]] + for q in token_quotas + ) + + if any_customized: + findings["csv_data"].append( + create_finding( + check_id="FS-03", + finding_name="Bedrock Token Quotas Customized", + finding_details=( + f"Found {len(token_quotas)} Bedrock token-based quota(s); at least one " + "applied value exceeds the AWS default, indicating quotas have been " + "reviewed and raised." + ), + resolution="No action required. Periodically re-review quotas against expected peak load.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/quotas.html", + severity="Medium", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-03"], + ) + ) + else: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-03", + finding_name="Bedrock Token Quotas At Default", + finding_details=( + f"All {len(token_quotas)} Bedrock token-based quota(s) are at their AWS " + "default values — no quota increase has been applied. Running at default " + "is a legitimate posture, but it should be a reviewed decision aligned " + "with expected peak load rather than an oversight." + ), + resolution=( + "1. Review current Bedrock TPM/TPD quotas in the Service Quotas console.\n" + "2. Request increases aligned with expected peak load, or document a " + "deliberate decision to remain at default after review.\n" + "3. Implement client-side token counting and pre-flight quota checks.\n" + "4. Use Bedrock cross-region inference profiles to distribute load." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/quotas.html", + severity="Medium", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-03"], + ) + ) + except Exception as e: + return _error_findings("Bedrock Token Quota Review", e) + return findings + + +def check_cost_anomaly_detection() -> Dict[str, Any]: + """ + FS-04 — Verify AWS Cost Anomaly Detection monitors are configured for + Bedrock and SageMaker services. + COMPLIANCE_PLACEHOLDER: [FFIEC CAT, SR 11-7 Appendix A] + """ + findings = _empty_findings("Cost Anomaly Detection Check") + try: + ce = boto3.client("ce", config=boto3_config) + # get_anomaly_monitors is paginated (NextPageToken). Read every page so a + # Bedrock-covering monitor beyond the first page is not missed (which would + # otherwise produce a false "no coverage" finding). + monitors = [] + next_token = None + while True: + kwargs = {"NextPageToken": next_token} if next_token else {} + resp = ce.get_anomaly_monitors(**kwargs) + monitors.extend(resp.get("AnomalyMonitors", [])) + next_token = resp.get("NextPageToken") + if not next_token: + break + + # A monitor provides Bedrock/SageMaker service-level coverage if its + # spec mentions bedrock (rarely populated for DIMENSIONAL+SERVICE + # monitors, kept for completeness) OR it is a DIMENSIONAL monitor scoped + # to the SERVICE dimension. DIMENSIONAL+SERVICE is the operative signal; + # a DIMENSIONAL monitor on LINKED_ACCOUNT/TAG/COST_CATEGORY does NOT + # provide service-level Bedrock coverage and must not count. + bedrock_monitors = [ + m + for m in monitors + if "bedrock" in json.dumps(m.get("MonitorSpecification", {})).lower() + or ( + m.get("MonitorType") == "DIMENSIONAL" + and m.get("MonitorDimension") == "SERVICE" + ) + ] + + if not monitors: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-04", + finding_name="No Cost Anomaly Detection Monitors", + finding_details=( + "No AWS Cost Anomaly Detection monitors found. Unexpected spikes in " + "Bedrock/SageMaker usage (e.g., from prompt injection loops) will go undetected." + ), + resolution=( + "1. Create a Cost Anomaly Detection monitor scoped to AWS/Bedrock and AWS/SageMaker.\n" + "2. Configure alert subscriptions (SNS/email) for anomalies above threshold.\n" + "3. Set daily spend budgets with AWS Budgets as a secondary control." + ), + reference="https://docs.aws.amazon.com/cost-management/latest/userguide/getting-started-ad.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-04"], + ) + ) + elif not bedrock_monitors: + # Monitors exist, but none provide Bedrock/SageMaker service-level + # coverage. This is the previously-masked false positive: the old + # code passed whenever ANY monitor existed. + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-04", + finding_name="Cost Anomaly Monitors Do Not Cover Bedrock/SageMaker", + finding_details=( + f"Found {len(monitors)} anomaly monitor(s), but none provide service-level " + "coverage for Bedrock/SageMaker (no DIMENSIONAL monitor scoped to the SERVICE " + "dimension, and no monitor specification referencing Bedrock). A generic or " + "linked-account monitor does not detect GenAI cost anomalies." + ), + resolution=( + "1. Create a DIMENSIONAL Cost Anomaly Detection monitor scoped to the SERVICE " + "dimension so AWS/Bedrock and AWS/SageMaker are covered.\n" + "2. Configure alert subscriptions (SNS/email) for anomalies above threshold.\n" + "3. Set daily spend budgets with AWS Budgets as a secondary control." + ), + reference="https://docs.aws.amazon.com/cost-management/latest/userguide/getting-started-ad.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-04"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-04", + finding_name="Cost Anomaly Detection Configured", + finding_details=f"Found {len(monitors)} anomaly monitor(s); {len(bedrock_monitors)} provide Bedrock/SageMaker service-level coverage.", + resolution="Verify monitors cover Bedrock and SageMaker service dimensions.", + reference="https://docs.aws.amazon.com/cost-management/latest/userguide/getting-started-ad.html", + severity="Medium", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-04"], + ) + ) + except Exception as e: + return _error_findings("Cost Anomaly Detection Check", e) + return findings + + +def check_cloudwatch_token_alarms() -> Dict[str, Any]: + """ + FS-05 — Check for CloudWatch alarms on Bedrock InvocationThrottles and + TokensProcessed metrics to detect runaway consumption. + COMPLIANCE_PLACEHOLDER: [FFIEC CAT, DORA Art.6] + """ + findings = _empty_findings("CloudWatch Token Usage Alarms Check") + try: + cw = boto3.client("cloudwatch", config=boto3_config) + paginator = cw.get_paginator("describe_alarms") + all_alarms = [] + for page in paginator.paginate(AlarmTypes=["MetricAlarm"]): + all_alarms.extend(page.get("MetricAlarms", [])) + + bedrock_alarms = [ + a + for a in all_alarms + if a.get("Namespace", "").startswith("AWS/Bedrock") + or "bedrock" in a.get("AlarmName", "").lower() + ] + + throttle_alarms = [ + a + for a in bedrock_alarms + if "throttl" in a.get("MetricName", "").lower() + or "throttl" in a.get("AlarmName", "").lower() + ] + + if not bedrock_alarms: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-05", + finding_name="No Bedrock CloudWatch Alarms Found", + finding_details=( + "No CloudWatch alarms found for Bedrock metrics. " + "Token exhaustion and throttling events will not trigger operational alerts." + ), + resolution=( + "Create CloudWatch alarms for:\n" + "- AWS/Bedrock InvocationThrottles (threshold > 0)\n" + "- AWS/Bedrock TokensProcessed (threshold based on quota)\n" + "- Custom application-level token counters via EMF" + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/monitoring-cw.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-05"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-05", + finding_name="Bedrock CloudWatch Alarms Present", + finding_details=( + f"Found {len(bedrock_alarms)} Bedrock-related alarm(s), " + f"{len(throttle_alarms)} covering throttling." + ), + resolution="Ensure alarms have SNS actions and are in OK state.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/monitoring-cw.html", + severity="Medium", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-05"], + ) + ) + except Exception as e: + return _error_findings("CloudWatch Token Usage Alarms Check", e) + return findings + + +def check_aws_budgets_for_aiml() -> Dict[str, Any]: + """ + FS-06 — Verify AWS Budgets are configured with alerts for AI/ML service spend. + COMPLIANCE_PLACEHOLDER: [FFIEC CAT, SR 11-7] + """ + findings = _empty_findings("AWS Budgets AI/ML Spend Check") + try: + budgets_client = boto3.client("budgets", config=boto3_config) + sts = boto3.client("sts", config=boto3_config) + account_id = sts.get_caller_identity()["Account"] + + def _paginate_budgets(show_filter_expression: bool): + kwargs = {"AccountId": account_id} + if show_filter_expression: + kwargs["ShowFilterExpression"] = True + pages = [] + for page in budgets_client.get_paginator("describe_budgets").paginate( + **kwargs + ): + pages.extend(page.get("Budgets", [])) + return pages + + # FilterExpression is opt-in: DescribeBudgets only returns it when + # ShowFilterExpression=True. Modern budgets use the structured + # FilterExpression instead of the deprecated flat CostFilters map. + # On an old botocore that does not accept ShowFilterExpression, the call + # raises ParamValidationError (NOT a ClientError) — degrade gracefully + # to a CostFilters-only check rather than letting the check vanish. + try: + all_budgets = _paginate_budgets(show_filter_expression=True) + except ParamValidationError: + logger.warning( + "describe_budgets does not accept ShowFilterExpression on this " + "botocore; falling back to CostFilters-only budget detection. " + "Upgrade botocore to check new-style FilterExpression budgets." + ) + all_budgets = _paginate_budgets(show_filter_expression=False) + + aiml_budgets = [ + b + for b in all_budgets + if any( + svc in json.dumps(b.get("CostFilters", {})).lower() + or svc in json.dumps(b.get("FilterExpression", {})).lower() + for svc in ["bedrock", "sagemaker"] + ) + ] + + if not aiml_budgets: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-06", + finding_name="No AI/ML Service Budgets Configured", + finding_details=( + "No AWS Budgets found scoped to Bedrock or SageMaker. " + "Unbounded GenAI spend can go undetected until the monthly bill." + ), + resolution=( + "1. Create cost budgets for AWS Bedrock and SageMaker with 80%/100% alert thresholds.\n" + "2. Add SNS notifications to on-call channels.\n" + "3. Consider budget actions to apply IAM deny policies when thresholds are breached." + ), + reference="https://docs.aws.amazon.com/cost-management/latest/userguide/budgets-managing-costs.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-06"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-06", + finding_name="AI/ML Service Budgets Configured", + finding_details=f"Found {len(aiml_budgets)} budget(s) covering AI/ML services.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/cost-management/latest/userguide/budgets-managing-costs.html", + severity="Medium", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-06"], + ) + ) + except Exception as e: + return _error_findings("AWS Budgets AI/ML Spend Check", e) + return findings + + +# =========================================================================== +# CATEGORY 2: EXCESSIVE AGENCY (FS-07 to FS-11) +# Risk: Agents take unintended real-world actions beyond their intended scope +# COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, DORA Art.6, MAS TRM 9] +# =========================================================================== + + +def check_bedrock_agent_action_boundaries(permission_cache) -> Dict[str, Any]: + """ + FS-07 — Verify Bedrock agent execution roles have narrow action boundaries + (no wildcard actions on sensitive services like s3:*, iam:*, ec2:*). + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT Cyber Risk Management] + """ + findings = _empty_findings("Agent Action Boundary Check") + try: + bedrock_agent = boto3.client("bedrock-agent", config=boto3_config) + agents = bedrock_agent.list_agents().get("agentSummaries", []) + + if not agents: + findings["csv_data"].append( + create_finding( + check_id="FS-07", + finding_name="Agent Action Boundary Check", + finding_details="No Bedrock agents found.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/agents-permissions.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-07"], + ) + ) + return findings + + SENSITIVE_WILDCARDS = ["iam:*", "s3:*", "ec2:*", "lambda:*", "*"] + agents_with_issues = [] + + for agent_summary in agents: + agent_id = agent_summary["agentId"] + agent_name = agent_summary["agentName"] + try: + detail = bedrock_agent.get_agent(agentId=agent_id) + except ClientError as e: + logger.warning(f"Could not describe agent {agent_name}: {e}") + continue + role_arn = detail.get("agent", {}).get("agentResourceRoleArn", "") + if not role_arn: + continue + role_name = role_arn.split("/")[-1] + role_perms = ( + (permission_cache or {}).get("role_permissions", {}).get(role_name, {}) + ) + for policy in role_perms.get("attached_policies", []) + role_perms.get( + "inline_policies", [] + ): + doc = policy.get("document", {}) + if isinstance(doc, str): + doc = json.loads(doc) + for stmt in doc.get("Statement", []): + if stmt.get("Effect") != "Allow": + continue + actions = stmt.get("Action", []) + if isinstance(actions, str): + actions = [actions] + for action in actions: + if action in SENSITIVE_WILDCARDS: + agents_with_issues.append( + f"Agent '{agent_name}' role '{role_name}' allows '{action}'" + ) + + if agents_with_issues: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-07", + finding_name="Bedrock Agent Overly Broad Action Permissions", + finding_details=( + "The following agents have execution roles with wildcard or overly broad actions:\n" + + "\n".join(f"- {i}" for i in agents_with_issues[:10]) + ), + resolution=( + "1. Replace wildcard actions with specific actions the agent needs.\n" + "2. Apply permission boundaries to agent execution roles.\n" + "3. Use resource-level conditions to restrict to specific ARNs.\n" + "4. Implement human-in-the-loop approval for high-impact actions." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/agents-permissions.html", + severity="High", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-07"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-07", + finding_name="Agent Action Boundaries Look Appropriate", + finding_details=f"Reviewed {len(agents)} agent(s); no wildcard sensitive actions found.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/agents-permissions.html", + severity="High", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-07"], + ) + ) + except Exception as e: + return _error_findings("Agent Action Boundary Check", e) + return findings + + +def check_agentcore_policy_engine() -> Dict[str, Any]: + """ + FS-08 — Check whether Bedrock AgentCore Policy Engine is configured to + enforce action-level authorization for agent tool calls. + COMPLIANCE_PLACEHOLDER: [SR 11-7, MAS TRM 9.1] + """ + findings = _empty_findings("AgentCore Policy Engine Check") + try: + # AgentCore policy engine is checked via bedrock-agentcore control plane + agentcore = boto3.client("bedrock-agentcore-control", config=boto3_config) + try: + # List policy stores (policy engine resources) + response = agentcore.list_agent_runtimes() + runtimes = response.get("agentRuntimes", []) + except ClientError as e: + if "AccessDenied" in str(e) or "UnrecognizedClientException" in str(e): + findings["csv_data"].append( + create_finding( + check_id="FS-08", + finding_name="AgentCore Policy Engine — Access Check", + finding_details="Unable to enumerate AgentCore runtimes (access denied or service unavailable in region).", + resolution="Ensure assessment role has bedrock-agentcore:ListAgentRuntimes permission.", + reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/security-authorization.html", + severity="Low", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-08"], + ) + ) + return findings + raise + + if not runtimes: + findings["csv_data"].append( + create_finding( + check_id="FS-08", + finding_name="No AgentCore Runtimes Found", + finding_details="No AgentCore runtimes found; policy engine check not applicable.", + resolution="If using AgentCore, configure the Policy Engine to authorize tool calls.", + reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/security-authorization.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-08"], + ) + ) + else: + # Check each runtime for policy engine association + runtimes_without_policy = [ + r["agentRuntimeName"] + for r in runtimes + if not r.get("authorizerConfiguration") + ] + if runtimes_without_policy: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-08", + finding_name="AgentCore Runtimes Missing Policy Engine", + finding_details=( + f"Runtimes without authorizer configuration: {', '.join(runtimes_without_policy)}. " + "Without a policy engine, agents can invoke any registered tool without authorization checks." + ), + resolution=( + "Configure an authorizer (Lambda or Cedar policy store) on each AgentCore runtime " + "to enforce fine-grained tool-call authorization." + ), + reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/security-authorization.html", + severity="High", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-08"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-08", + finding_name="AgentCore Policy Engine Configured", + finding_details=f"All {len(runtimes)} runtime(s) have authorizer configurations.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/security-authorization.html", + severity="High", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-08"], + ) + ) + except Exception as e: + return _error_findings("AgentCore Policy Engine Check", e) + return findings + + +def check_agent_transaction_limits(inventory) -> Dict[str, Any]: + """ + FS-09 — Check for application-level transaction/action limits on agents + via Lambda concurrency limits or Step Functions execution limits. + COMPLIANCE_PLACEHOLDER: [FFIEC CAT, SR 11-7] + """ + findings = _empty_findings("Agent Transaction Limits Check") + try: + functions = require(inventory, "lambda_functions") + + # Look for agent-related Lambda functions without reserved concurrency + agent_lambdas = [ + f + for f in functions + if any( + kw in f["FunctionName"].lower() for kw in ["agent", "bedrock", "aiml"] + ) + ] + + lambda_client = boto3.client("lambda", config=boto3_config) + lambdas_without_concurrency = [] + for fn in agent_lambdas: + try: + config = lambda_client.get_function_concurrency( + FunctionName=fn["FunctionName"] + ) + if not config.get("ReservedConcurrentExecutions"): + lambdas_without_concurrency.append(fn["FunctionName"]) + except ClientError: + lambdas_without_concurrency.append(fn["FunctionName"]) + + if lambdas_without_concurrency: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-09", + finding_name="Agent Lambda Functions Without Concurrency Limits", + finding_details=( + f"Agent-related Lambda functions without reserved concurrency: " + f"{', '.join(lambdas_without_concurrency[:10])}. " + "Unlimited concurrency allows runaway agent loops to exhaust account limits." + ), + resolution=( + "1. Set reserved concurrency on agent Lambda functions.\n" + "2. Implement maximum iteration counts in agent orchestration logic.\n" + "3. Use Step Functions with MaxConcurrency and timeout states.\n" + "4. Add circuit-breaker patterns to agent tool invocations." + ), + reference="https://docs.aws.amazon.com/lambda/latest/dg/configuration-concurrency.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-09"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-09", + finding_name="Agent Lambda Concurrency Limits Present", + finding_details=f"Reviewed {len(agent_lambdas)} agent Lambda(s); concurrency limits appear configured.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/lambda/latest/dg/configuration-concurrency.html", + severity="Medium", + status="Passed" if agent_lambdas else "N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-09"], + ) + ) + except Exception as e: + return _error_findings("Agent Transaction Limits Check", e) + return findings + + +def check_human_in_the_loop_for_high_risk_actions() -> Dict[str, Any]: + """ + FS-10 — Check for Step Functions or SNS-based human approval steps in + agent workflows that perform high-risk financial actions. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2] + """ + findings = _empty_findings("Human-in-the-Loop Approval Check") + try: + sfn = boto3.client("stepfunctions", config=boto3_config) + machines = sfn.list_state_machines().get("stateMachines", []) + + agent_machines = [ + m + for m in machines + if any( + kw in m["name"].lower() + for kw in ["agent", "approval", "human", "review"] + ) + ] + + machines_with_wait = [] + for machine in agent_machines: + defn = sfn.describe_state_machine( + stateMachineArn=machine["stateMachineArn"] + ).get("definition", "{}") + if '"waitForTaskToken"' in defn or '"TaskToken"' in defn: + machines_with_wait.append(machine["name"]) + + if not agent_machines: + findings["csv_data"].append( + create_finding( + check_id="FS-10", + finding_name="Human-in-the-Loop Check — No Agent Workflows Found", + finding_details=( + "No Step Functions state machines with agent/approval naming found. " + "Verify that high-risk agent actions (e.g., fund transfers, account changes) " + "have human approval gates." + ), + resolution=( + "Implement Step Functions .waitForTaskToken patterns for high-risk agent actions. " + "Route approval requests to human reviewers via SNS/SES/Slack." + ), + reference="https://docs.aws.amazon.com/step-functions/latest/dg/connect-to-resource.html#connect-wait-token", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-10"], + ) + ) + elif machines_with_wait: + findings["csv_data"].append( + create_finding( + check_id="FS-10", + finding_name="Human Approval Steps Found in Agent Workflows", + finding_details=f"State machines with waitForTaskToken (human approval): {', '.join(machines_with_wait)}.", + resolution="No action required. Verify approval routing reaches the correct reviewers.", + reference="https://docs.aws.amazon.com/step-functions/latest/dg/connect-to-resource.html#connect-wait-token", + severity="High", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-10"], + ) + ) + else: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-10", + finding_name="Agent Workflows Missing Human Approval Steps", + finding_details=( + f"Found {len(agent_machines)} agent-related state machine(s) but none use " + "waitForTaskToken for human approval. High-risk financial actions may execute autonomously." + ), + resolution=( + "Add .waitForTaskToken states before irreversible financial actions. " + "Define risk tiers and require human approval for Tier 1 actions." + ), + reference="https://docs.aws.amazon.com/step-functions/latest/dg/connect-to-resource.html#connect-wait-token", + severity="High", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-10"], + ) + ) + except Exception as e: + return _error_findings("Human-in-the-Loop Approval Check", e) + return findings + + +def check_agent_rate_alarms() -> Dict[str, Any]: + """ + FS-11 — Check for CloudWatch alarms on agent invocation rates to detect + runaway or looping agent behavior. + COMPLIANCE_PLACEHOLDER: [FFIEC CAT, DORA Art.6] + """ + findings = _empty_findings("Agent Rate Alarms Check") + try: + cw = boto3.client("cloudwatch", config=boto3_config) + paginator = cw.get_paginator("describe_alarms") + all_alarms = [] + for page in paginator.paginate(AlarmTypes=["MetricAlarm"]): + all_alarms.extend(page.get("MetricAlarms", [])) + + agent_alarms = [ + a + for a in all_alarms + if "agent" in a.get("AlarmName", "").lower() + or "agent" in a.get("Namespace", "").lower() + ] + + if not agent_alarms: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-11", + finding_name="No Agent Rate Alarms Found", + finding_details=( + "No CloudWatch alarms found for agent invocation rates. " + "Looping or runaway agents will not trigger operational alerts." + ), + resolution=( + "Create CloudWatch alarms on:\n" + "- Bedrock agent invocation counts (threshold based on expected max)\n" + "- Lambda invocation errors for agent functions\n" + "- Step Functions execution failures and timeouts" + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/monitoring-cw.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-11"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-11", + finding_name="Agent Rate Alarms Present", + finding_details=f"Found {len(agent_alarms)} agent-related CloudWatch alarm(s).", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/monitoring-cw.html", + severity="Medium", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-11"], + ) + ) + except Exception as e: + return _error_findings("Agent Rate Alarms Check", e) + return findings + + +# =========================================================================== +# CATEGORY 3: SUPPLY CHAIN VULNERABILITIES (FS-12 to FS-16) +# Risk: Third-party models, datasets, or plugins introduce malicious code/data +# COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, DORA Art.6, ISO 27001 A.15] +# =========================================================================== + + +def check_scp_model_access_restrictions() -> Dict[str, Any]: + """ + FS-12 — Verify SCPs restrict Bedrock model access to an approved model list, + preventing use of unapproved third-party models. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, ISO 27001 A.15.2] + """ + findings = _empty_findings("SCP Model Access Restriction Check") + try: + orgs = boto3.client("organizations", config=boto3_config) + try: + policies = _paginate( + orgs, "list_policies", "Policies", Filter="SERVICE_CONTROL_POLICY" + ) + except ClientError as e: + if "AccessDenied" in str(e) or "AWSOrganizationsNotInUseException" in str( + e + ): + findings["csv_data"].append( + create_finding( + check_id="FS-12", + finding_name="SCP Model Access Check — Not in Organization", + finding_details="Account is not part of an AWS Organization or lacks SCP read access.", + resolution="If using AWS Organizations, ensure SCPs restrict Bedrock model access to approved models.", + reference="https://docs.aws.amazon.com/organizations/latest/userguide/orgs_manage_policies_scps.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-12"], + ) + ) + return findings + raise + + bedrock_scps = [] + for policy in policies: + doc_response = orgs.describe_policy(PolicyId=policy["Id"]) + doc = json.loads(doc_response["Policy"]["Content"]) + if "bedrock" in json.dumps(doc).lower(): + bedrock_scps.append(policy["Name"]) + + if not bedrock_scps: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-12", + finding_name="No Bedrock-Scoped SCPs Found", + finding_details=( + "No Service Control Policies reference Bedrock. " + "Without SCPs, any account in the organization can access any Bedrock model, " + "including unapproved third-party models." + ), + resolution=( + "1. Create an SCP that denies bedrock:InvokeModel for model IDs not on the approved list.\n" + "2. Use bedrock:ModelId condition key to allowlist approved models.\n" + "3. Maintain a model inventory and update the SCP when models are approved/retired." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/security_iam_id-based-policy-examples.html", + severity="High", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-12"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-12", + finding_name="Bedrock SCPs Found", + finding_details=f"SCPs referencing Bedrock: {', '.join(bedrock_scps)}.", + resolution="Verify SCPs use bedrock:ModelId conditions to allowlist approved models.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/security_iam_id-based-policy-examples.html", + severity="High", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-12"], + ) + ) + except Exception as e: + return _error_findings("SCP Model Access Restriction Check", e) + return findings + + +def check_model_inventory_tagging() -> Dict[str, Any]: + """ + FS-13 — Check that custom Bedrock models and SageMaker models are tagged + with provenance metadata (source, version, approval-date). + COMPLIANCE_PLACEHOLDER: [SR 11-7, ISO 27001 A.12.5, FFIEC CAT] + """ + findings = _empty_findings("Model Inventory Tagging Check") + try: + bedrock = boto3.client("bedrock", config=boto3_config) + sm = boto3.client("sagemaker", config=boto3_config) + REQUIRED_TAGS = {"source", "version", "approval-date"} + + untagged_models = [] + + # Check Bedrock custom models + for model in _paginate(bedrock, "list_custom_models", "modelSummaries"): + tags_response = bedrock.list_tags_for_resource( + resourceARN=model["modelArn"] + ) + tag_keys = {t["key"].lower() for t in tags_response.get("tags", [])} + missing = REQUIRED_TAGS - tag_keys + if missing: + untagged_models.append( + f"Bedrock model '{model['modelName']}' missing tags: {missing}" + ) + + # Check SageMaker registered models + for model in _paginate(sm, "list_models", "Models"): + tags_response = sm.list_tags(ResourceArn=model["ModelArn"]) + tag_keys = {t["Key"].lower() for t in tags_response.get("Tags", [])} + missing = REQUIRED_TAGS - tag_keys + if missing: + untagged_models.append( + f"SageMaker model '{model['ModelName']}' missing tags: {missing}" + ) + + if untagged_models: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-13", + finding_name="Models Missing Provenance Tags", + finding_details=( + f"{len(untagged_models)} model(s) missing required provenance tags:\n" + + "\n".join(f"- {m}" for m in untagged_models[:10]) + ), + resolution=( + "Tag all models with: source (e.g., 'aws-marketplace', 'internal'), " + "version, and approval-date. " + "Enforce tagging via SCP or AWS Config rule." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/tagging.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-13"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-13", + finding_name="Model Provenance Tags Present", + finding_details="All reviewed models have required provenance tags.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/tagging.html", + severity="Medium", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-13"], + ) + ) + except Exception as e: + return _error_findings("Model Inventory Tagging Check", e) + return findings + + +def check_model_onboarding_governance() -> Dict[str, Any]: + """ + FS-14 — Check for AWS Config rules or Service Catalog constraints that + enforce model onboarding governance (approved sources only). + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, ISO 27001 A.15.1] + """ + findings = _empty_findings("Model Onboarding Governance Check") + try: + config = boto3.client("config", config=boto3_config) + rules = _paginate(config, "describe_config_rules", "ConfigRules") + + bedrock_rules = [ + r + for r in rules + if "bedrock" in r.get("ConfigRuleName", "").lower() + or "model" in r.get("ConfigRuleName", "").lower() + ] + + if not bedrock_rules: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-14", + finding_name="No Model Governance Config Rules Found", + finding_details=( + "No AWS Config rules found for Bedrock model governance. " + "Unapproved models may be deployed without detection." + ), + resolution=( + "1. Create custom AWS Config rules to detect use of non-approved Bedrock models.\n" + "2. Use AWS Service Catalog to publish approved model configurations.\n" + "3. Implement a model risk management (MRM) process per SR 11-7." + ), + reference="https://docs.aws.amazon.com/config/latest/developerguide/evaluate-config.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-14"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-14", + finding_name="Model Governance Config Rules Present", + finding_details=f"Found {len(bedrock_rules)} model-related Config rule(s).", + resolution="No action required.", + reference="https://docs.aws.amazon.com/config/latest/developerguide/evaluate-config.html", + severity="Medium", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-14"], + ) + ) + except Exception as e: + return _error_findings("Model Onboarding Governance Check", e) + return findings + + +def check_bedrock_model_evaluation_adversarial() -> Dict[str, Any]: + """ + FS-15 — Check whether Bedrock Model Evaluation jobs include adversarial + test datasets (robustness/red-team evaluations). + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.3] + """ + findings = _empty_findings("Adversarial Model Evaluation Check") + try: + bedrock = boto3.client("bedrock", config=boto3_config) + evals = _paginate(bedrock, "list_evaluation_jobs", "jobSummaries") + + if not evals: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-15", + finding_name="No Bedrock Evaluation Jobs Found", + finding_details=( + "No Bedrock Model Evaluation jobs found. Models have not been evaluated " + "for adversarial robustness. FinServ model-risk management (SR 11-7) " + "expects documented model validation/evaluation." + ), + resolution=( + "1. Run Bedrock Model Evaluation with adversarial/red-team datasets.\n" + "2. Use FMEval library for automated robustness testing.\n" + "3. Schedule periodic re-evaluation after model updates." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-15"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-15", + finding_name="Bedrock Evaluation Jobs Present", + finding_details=f"Found {len(evals)} evaluation job(s). Verify adversarial datasets are included.", + resolution="Ensure evaluation datasets include adversarial/red-team test cases.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", + severity="Medium", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-15"], + ) + ) + except Exception as e: + return _error_findings("Adversarial Model Evaluation Check", e) + return findings + + +def check_ecr_image_scanning() -> Dict[str, Any]: + """ + FS-16 — Verify ECR repositories used for custom model containers have + image scanning enabled (supply chain vulnerability detection). + COMPLIANCE_PLACEHOLDER: [ISO 27001 A.12.6, FFIEC CAT, DORA Art.6] + """ + findings = _empty_findings("ECR Image Scanning Check") + try: + ecr = boto3.client("ecr", config=boto3_config) + repos = _paginate(ecr, "describe_repositories", "repositories") + + if not repos: + findings["csv_data"].append( + create_finding( + check_id="FS-16", + finding_name="No ECR Repositories Found", + finding_details="No ECR repositories found.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/AmazonECR/latest/userguide/image-scanning.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-16"], + ) + ) + return findings + + repos_without_scanning = [ + r["repositoryName"] + for r in repos + if not r.get("imageScanningConfiguration", {}).get("scanOnPush", False) + ] + + if repos_without_scanning: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-16", + finding_name="ECR Repositories Without Image Scanning", + finding_details=( + f"{len(repos_without_scanning)} ECR repo(s) without scan-on-push: " + f"{', '.join(repos_without_scanning[:10])}." + ), + resolution=( + "Enable scan-on-push for all ECR repositories containing model containers. " + "Consider enabling Enhanced Scanning (Inspector) for CVE detection." + ), + reference="https://docs.aws.amazon.com/AmazonECR/latest/userguide/image-scanning.html", + severity="High", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-16"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-16", + finding_name="ECR Image Scanning Enabled", + finding_details=f"All {len(repos)} ECR repo(s) have scan-on-push enabled.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/AmazonECR/latest/userguide/image-scanning.html", + severity="High", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-16"], + ) + ) + except Exception as e: + return _error_findings("ECR Image Scanning Check", e) + return findings + + +# =========================================================================== +# CATEGORY 4: TRAINING DATA & MODEL POISONING (FS-17 to FS-21) +# Risk: Malicious data corrupts model behavior during training or fine-tuning +# COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.3, ISO 27001 A.12] +# +# NOTE: FS-17 (Model Monitor Data Quality → SM-07), FS-18 (Model Drift Detection → SM-23), +# and FS-19 (Model Registry Approval → SM-22) are merged into upstream checks. +# See extension notes in SECURITY_CHECKS_FINSERV_PART1_INFRA_CONTROLS.md. +# =========================================================================== + + +def check_feature_store_rollback_capability() -> Dict[str, Any]: + """ + FS-20 — Check SageMaker Feature Store for versioning/offline store + configuration that enables rollback of poisoned feature data. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT] + """ + findings = _empty_findings("Feature Store Rollback Check") + try: + sm = boto3.client("sagemaker", config=boto3_config) + groups = _paginate(sm, "list_feature_groups", "FeatureGroupSummaries") + + if not groups: + findings["csv_data"].append( + create_finding( + check_id="FS-20", + finding_name="No SageMaker Feature Groups Found", + finding_details="No SageMaker Feature Store groups found.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/sagemaker/latest/dg/feature-store.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-20"], + ) + ) + return findings + + groups_without_offline = [ + g["FeatureGroupName"] + for g in groups + if g.get("OfflineStoreStatus", {}).get("Status") != "Active" + ] + + if groups_without_offline: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-20", + finding_name="Feature Groups Without Offline Store", + finding_details=( + f"{len(groups_without_offline)} feature group(s) lack an active offline store: " + f"{', '.join(groups_without_offline[:10])}. " + "Without offline store, historical feature data cannot be used for rollback." + ), + resolution=( + "1. Enable offline store (S3-backed) for all production feature groups.\n" + "2. Enable S3 versioning on the offline store bucket.\n" + "3. Document rollback procedures for poisoned feature data." + ), + reference="https://docs.aws.amazon.com/sagemaker/latest/dg/feature-store-offline.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-20"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-20", + finding_name="Feature Store Offline Store Active", + finding_details=f"All {len(groups)} feature group(s) have active offline stores.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/sagemaker/latest/dg/feature-store-offline.html", + severity="Medium", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-20"], + ) + ) + except Exception as e: + return _error_findings("Feature Store Rollback Check", e) + return findings + + +def check_training_data_s3_versioning(inventory) -> Dict[str, Any]: + """ + FS-21 — Verify S3 buckets used for training data have versioning enabled + to support rollback of poisoned datasets. + COMPLIANCE_PLACEHOLDER: [SR 11-7, ISO 27001 A.12.3, FFIEC CAT] + """ + findings = _empty_findings("Training Data S3 Versioning Check") + try: + buckets = require(inventory, "buckets") + s3 = boto3.client("s3", config=boto3_config) + + training_buckets = [ + b + for b in buckets + if any( + kw in b["Name"].lower() + for kw in ["train", "dataset", "model", "sagemaker", "bedrock"] + ) + ] + + if not training_buckets: + findings["csv_data"].append( + create_finding( + check_id="FS-21", + finding_name="No Training Data Buckets Identified", + finding_details="No S3 buckets with training/model naming found.", + resolution="Tag training data buckets and enable versioning.", + reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/Versioning.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-21"], + ) + ) + return findings + + unversioned = [] + for bucket in training_buckets: + try: + versioning = s3.get_bucket_versioning(Bucket=bucket["Name"]) + except ClientError as e: + # An access error means we could not read versioning; re-raise so + # it surfaces as could-not-assess rather than a false finding, and + # so one inaccessible bucket does not abort the whole check. + if _is_access_error(e): + raise + logger.warning( + f"Could not check versioning for bucket {bucket['Name']}: {e}" + ) + unversioned.append(f"{bucket['Name']} (error)") + continue + if versioning.get("Status") != "Enabled": + unversioned.append(bucket["Name"]) + + if unversioned: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-21", + finding_name="Training Data Buckets Without Versioning", + finding_details=( + f"{len(unversioned)} training data bucket(s) without versioning: " + f"{', '.join(unversioned[:10])}." + ), + resolution=( + "Enable S3 versioning on all training data buckets. " + "Consider enabling MFA Delete for additional protection against poisoning." + ), + reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/Versioning.html", + severity="High", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-21"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-21", + finding_name="Training Data Buckets Have Versioning", + finding_details=f"All {len(training_buckets)} training bucket(s) have versioning enabled.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/Versioning.html", + severity="High", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-21"], + ) + ) + except Exception as e: + return _error_findings("Training Data S3 Versioning Check", e) + return findings + + +# =========================================================================== +# CATEGORY 5: VECTOR & EMBEDDING WEAKNESSES (FS-22 to FS-26) +# Risk: Knowledge base / RAG vector stores are improperly secured +# COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, NYDFS 500.06, PCI-DSS 12.3.2] +# =========================================================================== + + +def _is_overbroad_kb_action(action: Any) -> bool: + """True if an IAM action grants overly broad Bedrock/Knowledge Base access: + the full wildcard, a service-wide bedrock(-agent) wildcard, or ANY partial + wildcard within those namespaces (e.g., 'bedrock-agent:Get*', 'bedrock:Invoke*'). + Round-3 fixed the crash; this widens detection beyond the three exact wildcards.""" + if not isinstance(action, str): + return False + a = action.lower() + if a in ("*", "bedrock:*", "bedrock-agent:*"): + return True + if a.endswith("*") and (a.startswith("bedrock:") or a.startswith("bedrock-agent:")): + return True + return False + + +def check_knowledge_base_iam_least_privilege(permission_cache) -> Dict[str, Any]: + """ + FS-22 — Verify IAM roles accessing Bedrock Knowledge Bases follow + least privilege (no wildcard bedrock-agent:* permissions). + COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, FFIEC CAT, PCI-DSS 12.3.2] + """ + findings = _empty_findings("Knowledge Base IAM Least Privilege Check") + try: + issues = [] + for role_name, perms in ( + (permission_cache or {}).get("role_permissions", {}).items() + ): + if not isinstance(perms, dict): + continue + for policy in (perms.get("attached_policies", []) or []) + ( + perms.get("inline_policies", []) or [] + ): + if not isinstance(policy, dict): + continue + doc = policy.get("document", {}) + if isinstance(doc, str): + try: + doc = json.loads(doc) + except (ValueError, TypeError): + continue + if not isinstance(doc, dict): + continue + # IAM allows Statement to be a single object (dict) or a list. + # Normalize to a list so iterating never yields statement *keys* + # (the cause of the 'str' object has no attribute 'get' crash). + statements = doc.get("Statement", []) + if isinstance(statements, dict): + statements = [statements] + for stmt in statements: + if not isinstance(stmt, dict): + continue + if stmt.get("Effect") != "Allow": + continue + # NotAction Allow ("allow everything except …") is inherently + # broad and the antithesis of least privilege — flag it. + if "NotAction" in stmt: + issues.append( + f"Role '{role_name}' uses a NotAction Allow (overly broad — " + "grants all actions except those listed)" + ) + resources = stmt.get("Resource", []) + if isinstance(resources, str): + resources = [resources] + unscoped_resource = "*" in resources + actions = stmt.get("Action", []) + if isinstance(actions, str): + actions = [actions] + for action in actions: + if _is_overbroad_kb_action(action): + issues.append(f"Role '{role_name}' allows '{action}'") + elif ( + unscoped_resource + and isinstance(action, str) + and ( + action.lower().startswith("bedrock:") + or action.lower().startswith("bedrock-agent:") + ) + ): + issues.append( + f"Role '{role_name}' allows '{action}' on Resource '*' " + "(no ARN scoping to specific Knowledge Bases)" + ) + + if issues: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-22", + finding_name="Overly Permissive Knowledge Base IAM Roles", + finding_details=( + f"{len(issues)} role(s) with wildcard KB permissions:\n" + + "\n".join(f"- {i}" for i in issues[:10]) + ), + resolution=( + "Replace wildcard bedrock-agent:* with specific actions: " + "bedrock:Retrieve, bedrock:RetrieveAndGenerate. " + "Scope resources to specific Knowledge Base ARNs." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/security-iam-awsmanpol.html", + severity="High", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-22"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-22", + finding_name="Knowledge Base IAM Permissions Look Appropriate", + finding_details="No wildcard KB permissions found in reviewed roles.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/security-iam-awsmanpol.html", + severity="High", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-22"], + ) + ) + except Exception as e: + return _error_findings("Knowledge Base IAM Least Privilege Check", e) + return findings + + +def check_knowledge_base_metadata_filtering(inventory) -> Dict[str, Any]: + """ + FS-24 — Check that Bedrock Knowledge Bases have metadata fields configured + to support tenant-level filtering (multi-tenancy isolation). + COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, FFIEC CAT, PCI-DSS 12.3.2] + """ + findings = _empty_findings("Knowledge Base Metadata Filtering Check") + try: + kbs = require(inventory, "knowledge_bases").summaries + + if not kbs: + findings["csv_data"].append( + create_finding( + check_id="FS-24", + finding_name="No Knowledge Bases Found", + finding_details="No Bedrock Knowledge Bases found.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/kb-test-config.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-24"], + ) + ) + return findings + + # Advisory check — metadata filtering is a design pattern, not directly inspectable + findings["csv_data"].append( + create_finding( + check_id="FS-24", + finding_name="ADVISORY: Knowledge Base Metadata Filtering — Manual Review Required", + finding_details=( + f"Found {len(kbs)} Knowledge Base(s). Tenant-isolation metadata filtering is a " + "design pattern that cannot be verified via API — manual review required. " + "Verify that metadata attributes (e.g., tenantId, classification) are indexed " + "and that Retrieve calls include RetrievalFilter conditions for tenant isolation." + ), + resolution=( + "1. Add metadata fields (tenantId, dataClassification) to KB data sources.\n" + "2. Pass RetrievalFilter in all Retrieve/RetrieveAndGenerate calls.\n" + "3. Validate filters in integration tests to prevent cross-tenant data leakage." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/kb-test-config.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-24"], + ) + ) + except Exception as e: + return _error_findings("Knowledge Base Metadata Filtering Check", e) + return findings + + +def check_opensearch_serverless_encryption() -> Dict[str, Any]: + """ + FS-25 — Verify OpenSearch Serverless collections (used as KB vector stores) + have encryption policies with customer-managed KMS keys. + COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, PCI-DSS 3.5, FFIEC CAT] + """ + findings = _empty_findings("OpenSearch Serverless Encryption Check") + try: + oss = boto3.client("opensearchserverless", config=boto3_config) + policies = oss.list_security_policies(type="encryption").get( + "securityPolicySummaries", [] + ) + + if not policies: + findings["csv_data"].append( + create_finding( + check_id="FS-25", + finding_name="No OpenSearch Serverless Encryption Policies", + finding_details=( + "No OpenSearch Serverless encryption policies found, which indicates " + "no OpenSearch Serverless vector-store collections exist in this region. " + "If Bedrock Knowledge Bases use a different vector store (e.g., Aurora, " + "Pinecone), verify its encryption separately." + ), + resolution=( + "If using OpenSearch Serverless as a Bedrock KB vector store, create an " + "encryption security policy specifying a customer-managed KMS key." + ), + reference="https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-encryption.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-25"], + ) + ) + else: + # Check for CMK usage. A policy that does not reference AWSOwnedKey is + # treated as customer-managed (CMK). + cmk_policies = [] + for policy in policies: + doc = json.loads(policy.get("policy", "{}")) + if "AWSOwnedKey" not in json.dumps(doc): + cmk_policies.append(policy["name"]) + + if not cmk_policies: + # Encryption policies exist but all use AWS-owned keys — the + # control (customer-managed encryption) the check verifies is + # absent, so this is a real finding, not a pass. + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-25", + finding_name="OpenSearch Serverless Encryption Not Using Customer-Managed Keys", + finding_details=( + f"Found {len(policies)} encryption policy(ies), but none use a " + "customer-managed KMS key (CMK) — all rely on AWS-owned keys. " + "FinServ data-protection controls typically require CMKs for " + "key lifecycle control and auditability." + ), + resolution=( + "Update OpenSearch Serverless encryption policies to specify a " + "customer-managed KMS key (KmsARN) instead of AWS-owned keys." + ), + reference="https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-encryption.html", + severity="High", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-25"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-25", + finding_name="OpenSearch Serverless Encryption Policies Present", + finding_details=( + f"Found {len(policies)} encryption policy(ies); " + f"{len(cmk_policies)} use a customer-managed KMS key." + ), + resolution="Verify all vector store collections use customer-managed KMS keys.", + reference="https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-encryption.html", + severity="High", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-25"], + ) + ) + except Exception as e: + return _error_findings("OpenSearch Serverless Encryption Check", e) + return findings + + +def check_knowledge_base_vpc_access() -> Dict[str, Any]: + """ + FS-26 — Verify OpenSearch Serverless collections have VPC access policies + restricting access to private network endpoints. + COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, FFIEC CAT, PCI-DSS 1.3] + """ + findings = _empty_findings("Knowledge Base VPC Access Check") + try: + oss = boto3.client("opensearchserverless", config=boto3_config) + network_policies = oss.list_security_policies(type="network").get( + "securityPolicySummaries", [] + ) + + if not network_policies: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-26", + finding_name="No OpenSearch Serverless Network Policies", + finding_details=( + "No OpenSearch Serverless network policies found. " + "Vector store collections may be publicly accessible." + ), + resolution=( + "Create network security policies for OpenSearch Serverless collections " + "restricting access to VPC endpoints only." + ), + reference="https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-network.html", + severity="High", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-26"], + ) + ) + else: + vpc_policies = [] + for policy in network_policies: + doc = json.loads(policy.get("policy", "{}")) + if "vpc" in json.dumps(doc).lower(): + vpc_policies.append(policy["name"]) + + if not vpc_policies: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-26", + finding_name="OpenSearch Serverless Collections Not VPC-Restricted", + finding_details=( + f"Found {len(network_policies)} network policy(ies) but none restrict to VPC. " + "Vector stores may be accessible from the public internet." + ), + resolution=( + "Update network policies to allow access only from VPC endpoints. " + "Create an OpenSearch Serverless VPC endpoint in your VPC." + ), + reference="https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-network.html", + severity="High", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-26"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-26", + finding_name="OpenSearch Serverless VPC Access Configured", + finding_details=f"{len(vpc_policies)} network policy(ies) restrict to VPC.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless-network.html", + severity="High", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-26"], + ) + ) + except Exception as e: + return _error_findings("Knowledge Base VPC Access Check", e) + return findings + + +# =========================================================================== +# CATEGORY 6: NON-COMPLIANT OUTPUT (FS-27 to FS-30) +# Risk: GenAI outputs violate regulatory requirements (e.g., fair lending, disclosures) +# COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, NYDFS 500, MAS TRM 9.2] +# =========================================================================== + + +def check_guardrail_contextual_grounding(inventory) -> Dict[str, Any]: + """ + FS-27 — Check whether Bedrock Guardrails have contextual grounding checks + configured to validate that outputs are grounded in the provided context and + are relevant to the user query. + + NOTE: This check verifies *contextual grounding* (guardrails-grounding) — a + separate, independent feature from Automated Reasoning checks (ARC). True ARC + is assessed in check_automated_reasoning_policies() below. Both controls are + recommended for FinServ workloads; they complement each other: + - Contextual grounding: thresholded relevance/grounding filter applied per + inference call (no policy authoring required). + - Automated Reasoning: policy-based formal verification of factual claims + against a customer-authored business-rules document (GA August 2025; + limited to US/EU regions; requires bedrock:ListAutomatedReasoningPolicies). + + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2] + """ + findings = _empty_findings("Guardrail Contextual Grounding Check") + try: + guardrail_inv = require(inventory, "guardrails") + guardrails = guardrail_inv.summaries + + if not guardrails: + findings["csv_data"].append( + create_finding( + check_id="FS-27", + finding_name="No Guardrails — Contextual Grounding Not Applicable", + finding_details="No Bedrock Guardrails configured. Configure guardrails first (see BR-05).", + resolution=( + "Configure Bedrock Guardrails with contextual grounding checks " + "(grounding threshold ≥0.7 and relevance threshold ≥0.7 for FinServ use cases)." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-27"], + ) + ) + return findings + + guardrails_with_grounding = [] + for g in guardrails: + detail = guardrail_inv.detail_by_id[g["id"]] + if isinstance(detail, _Unavailable): + raise detail.error + if detail.get("contextualGroundingPolicy"): + guardrails_with_grounding.append(g["name"]) + + if not guardrails_with_grounding: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-27", + finding_name="No Guardrails With Contextual Grounding", + finding_details=( + f"Found {len(guardrails)} guardrail(s) but none have contextual grounding " + "filters enabled. Without grounding checks, outputs that are not supported " + "by the source context (hallucinations, regulatory violations) will not be " + "filtered at inference time." + ), + resolution=( + "Enable contextual grounding checks on Bedrock Guardrails:\n" + "- Set grounding threshold ≥0.7 (filters responses not supported by source context)\n" + "- Set relevance threshold ≥0.7 (filters off-topic responses)\n" + "Also consider enabling Automated Reasoning checks (bedrock:ListAutomatedReasoningPolicies) " + "for policy-based formal verification of factual claims — see FS-27 ARC check." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", + severity="High", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-27"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-27", + finding_name="Contextual Grounding Enabled on Guardrails", + finding_details=f"Guardrails with contextual grounding: {', '.join(guardrails_with_grounding)}.", + resolution=( + "No action required for contextual grounding. " + "Also consider enabling Automated Reasoning checks for formal policy verification." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", + severity="High", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-27"], + ) + ) + except Exception as e: + return _error_findings("Guardrail Contextual Grounding Check", e) + return findings + + +def check_automated_reasoning_policies() -> Dict[str, Any]: + """ + FS-27b — Check whether Bedrock Automated Reasoning policies have been + created to provide formal, policy-based verification of GenAI factual claims. + + Automated Reasoning checks (ARC) — GA August 2025 — use formal verification + to detect hallucinations and ensure outputs comply with authored business rules + (e.g., loan eligibility criteria, regulatory thresholds). Unlike contextual + grounding (a threshold applied per call), ARC requires authoring an Automated + Reasoning Policy document containing the rules to verify against. + + Regions supported (verify against current AWS docs at run time — + https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-automated-reasoning-checks.html): + AWS GovCloud (US), us-east-1, us-east-2, us-west-2, eu-central-1, eu-west-1, + eu-west-3. The list expands over time; treat a region miss as "verify + availability", not a hard limitation. + + IAM action required: bedrock:ListAutomatedReasoningPolicies + + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2] + """ + findings = _empty_findings("Automated Reasoning Policies Check") + try: + bedrock = boto3.client("bedrock", config=boto3_config) + + try: + policies = _paginate( + bedrock, + "list_automated_reasoning_policies", + "automatedReasoningPolicySummaries", + ) + except ClientError as e: + if _is_access_error(e): + findings["csv_data"].append( + create_finding( + check_id="FS-27", + finding_name="Automated Reasoning Policies — Access Check", + finding_details=( + "Access denied or service unavailable when listing Automated Reasoning " + "policies. The IAM action name (bedrock:ListAutomatedReasoningPolicies) " + "is correct, so the most likely causes are, in order: (1) the assessment " + "MEMBER ROLE in this account was deployed before this action was added " + "and has not been re-deployed; (2) an AWS Organizations SCP or permission " + "boundary denies this newer Bedrock action; (3) the region does not " + "support ARC. ARC is available in AWS GovCloud (US) and a growing set " + "of commercial regions (e.g., us-east-1, us-east-2, us-west-2, " + "eu-central-1, eu-west-1, eu-west-3) — verify the current list in the " + "AWS documentation." + ), + resolution=( + "1. RE-DEPLOY the member-role CloudFormation stack so the role picks up " + "bedrock:ListAutomatedReasoningPolicies (templates may be current while " + "the *deployed* role is stale). See deployment/1-aiml-security-member-roles.yaml " + "and aiml-security-single-account.yaml.\n" + "2. Check for an Organizations SCP / permission boundary denying the action.\n" + "3. Confirm the assessed region supports Automated Reasoning checks.\n" + "4. Re-run the assessment after re-deploying." + ), + reference="https://docs.aws.amazon.com/IAM/latest/UserGuide/troubleshoot_access-denied.html", + severity="Low", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-27"], + ) + ) + return findings + raise + + if not policies: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-27", + finding_name="No Automated Reasoning Policies Found", + finding_details=( + "No Bedrock Automated Reasoning policies have been created. " + "ARC (GA August 2025) uses formal verification to guarantee that GenAI " + "outputs comply with authored business rules — e.g., loan criteria, " + "regulatory thresholds, policy constraints. Without ARC policies, " + "factual accuracy of outputs is not formally verified, only heuristically " + "filtered by contextual grounding thresholds." + ), + resolution=( + "1. In the Amazon Bedrock console → Guardrails → Automated Reasoning, " + "create a policy document encoding your FinServ business rules " + "(e.g., eligibility criteria, rate limits, regulatory thresholds).\n" + "2. Associate the ARC policy with your guardrail " + "(automatedReasoningPolicy.policies field in CreateGuardrail/UpdateGuardrail).\n" + "3. Set confidenceThreshold on the policy to control strictness.\n" + "4. ARC requires cross-Region inference — ensure your guardrail has a " + "guardrailProfileArn configured (crossRegionDetails in GetGuardrail response).\n" + "5. Reference: AWS Announcement — Automated Reasoning checks GA (August 2025)." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/automated-reasoning.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-27"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-27", + finding_name="Automated Reasoning Policies Found", + finding_details=( + f"Found {len(policies)} Automated Reasoning policy(ies): " + f"{', '.join(p['name'] for p in policies[:5])}. " + "Verify policies are associated with active guardrails via the " + "automatedReasoningPolicy field in GetGuardrail." + ), + resolution=( + "Confirm each ARC policy is referenced in a guardrail's " + "automatedReasoningPolicy.policies list and that the " + "guardrail is applied to your Bedrock inference calls." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/automated-reasoning.html", + severity="Medium", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-27"], + ) + ) + except Exception as e: + return _error_findings("Automated Reasoning Policies Check", e) + return findings + + +def check_guardrail_denied_topics_financial(inventory) -> Dict[str, Any]: + """ + FS-28 — Verify Bedrock Guardrails have denied topics configured for + regulated financial advice categories (investment advice, credit decisions). + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, NYDFS 500, MAS TRM 9.2] + """ + findings = _empty_findings("Financial Denied Topics Check") + try: + guardrail_inv = require(inventory, "guardrails") + guardrails = guardrail_inv.summaries + + if not guardrails: + findings["csv_data"].append( + create_finding( + check_id="FS-28", + finding_name="No Guardrails — Denied Topics Not Applicable", + finding_details="No Bedrock Guardrails configured.", + resolution="Configure guardrails with denied topics for regulated financial content.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-28"], + ) + ) + return findings + + guardrails_with_topics = [] + topics_classic_tier = [] + for g in guardrails: + detail = guardrail_inv.detail_by_id[g["id"]] + if isinstance(detail, _Unavailable): + raise detail.error + topic_policy = detail.get("topicPolicy", {}) + if topic_policy.get("topics"): + guardrails_with_topics.append(g["name"]) + # Denied topics also support tiers (GA June 2025). STANDARD tier + # adds broader language support and improved detection but requires + # cross-region inference. topicPolicy.tier.tierName in GetGuardrail. + tier = topic_policy.get("tier", {}).get("tierName", "CLASSIC") + if tier == "CLASSIC": + topics_classic_tier.append(g["name"]) + + if not guardrails_with_topics: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-28", + finding_name="No Guardrails With Denied Financial Topics", + finding_details=( + "No guardrails have topic policies configured. " + "GenAI may provide regulated financial advice without controls." + ), + resolution=( + "Add denied topics to guardrails for:\n" + "- Specific investment advice (securities recommendations)\n" + "- Credit/lending decisions\n" + "- Insurance underwriting advice\n" + "- Tax advice beyond general information\n" + "When authoring denied-topic policies, use existing compliance materials " + "as the source: employee policies, training materials, procedure documents, " + "and incident reports (as recommended in PDF \u00a71.2.1 Practical guidance). " + "Consider the STANDARD tier (GA June 2025) for broader language support; " + "it requires cross-region inference on the guardrail." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", + severity="High", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-28"], + ) + ) + elif topics_classic_tier: + findings["csv_data"].append( + create_finding( + check_id="FS-28", + finding_name="Denied Topics Configured on CLASSIC Tier", + finding_details=( + f"Guardrails with topic policies: {', '.join(guardrails_with_topics)}. " + f"The following use the CLASSIC tier: {', '.join(topics_classic_tier)}. " + "CLASSIC tier supports English, French, and Spanish only. The STANDARD tier " + "(GA June 2025) provides broader language support and improved detection for " + "denied topics." + ), + resolution=( + "Verify topics cover regulated financial advice categories. For multilingual " + "FinServ deployments, consider upgrading denied topics to the STANDARD tier " + "(set topicsTierConfig.tierName=STANDARD via UpdateGuardrail; requires a " + "cross-region inference profile on the guardrail). When authoring denied-topic " + "policies, use existing compliance materials as the source: employee policies, " + "training materials, procedure documents, and incident reports " + "(PDF \u00a71.2.1 Practical guidance)." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", + severity="High", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-28"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-28", + finding_name="Guardrails With Topic Policies Found", + finding_details=f"Guardrails with topic policies: {', '.join(guardrails_with_topics)}.", + resolution=( + "Verify topics cover regulated financial advice categories. " + "When authoring or updating denied-topic policies, use existing compliance " + "materials as the source: employee policies, training materials, procedure " + "documents, and incident reports (as recommended in PDF \u00a71.2.1 Practical guidance)." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", + severity="High", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-28"], + ) + ) + except Exception as e: + return _error_findings("Financial Denied Topics Check", e) + return findings + + +def check_compliance_disclaimer_in_outputs() -> Dict[str, Any]: + """ + FS-29 — Advisory check: verify application-level disclaimers are added to + GenAI outputs for regulated financial content (not directly checkable via API). + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, NYDFS 500, MAS TRM 9.2] + """ + findings = _empty_findings("Compliance Disclaimer Check") + # This is an advisory/manual check — no AWS API can verify application-level disclaimers + findings["csv_data"].append( + create_finding( + check_id="FS-29", + finding_name="ADVISORY: Compliance Disclaimer — Manual Review Required", + finding_details=( + "Application-level compliance disclaimers cannot be verified via AWS APIs. " + "Manual review required to confirm GenAI outputs include required regulatory disclosures." + ), + resolution=( + "1. Implement post-processing to append required disclaimers to GenAI outputs.\n" + "2. Use Bedrock Guardrails word filters to block outputs that omit required disclosures.\n" + "3. Document disclaimer requirements in the AI use case register.\n" + "4. Test disclaimer presence in QA/UAT before production deployment." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-29"], + ) + ) + return findings + + +def check_bedrock_evaluation_compliance_datasets() -> Dict[str, Any]: + """ + FS-30 — Advisory: Bedrock Model Evaluation jobs should use compliance-specific + datasets (e.g., fair lending, UDAP, ECOA test cases). + + The Bedrock evaluation-job APIs do not expose dataset *content*, so whether a + job actually includes compliance test cases cannot be verified programmatically. + This is therefore an advisory (manual-review) check. The existence of evaluation + jobs at all is the verifiable control and is assessed by FS-15. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, NYDFS 500] + """ + findings = _empty_findings("Compliance Evaluation Datasets Check") + try: + findings["csv_data"].append( + create_finding( + check_id="FS-30", + finding_name="ADVISORY: Compliance Dataset Coverage — Manual Review Required", + finding_details=( + "Bedrock model-evaluation dataset content cannot be inspected via API. " + "Manually verify your model-evaluation jobs include compliance-specific " + "datasets (fair lending/ECOA, Fair Housing Act, UDAP/UDAAP, AML/KYC edge cases). " + "Whether any evaluation jobs exist at all is assessed by FS-15." + ), + resolution=( + "Run Bedrock Model Evaluation with compliance-specific datasets:\n" + "- Fair lending test cases (ECOA, Fair Housing Act)\n" + "- UDAP/UDAAP unfair/deceptive practice scenarios\n" + "- AML/KYC edge cases" + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-30"], + ) + ) + except Exception as e: + return _error_findings("Compliance Evaluation Datasets Check", e) + return findings + + +# =========================================================================== +# CATEGORY 7: MISINFORMATION (FS-31 to FS-34) +# Risk: GenAI outputs contain false or misleading financial information +# COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2, NYDFS 500] +# =========================================================================== + + +def check_knowledge_base_data_source_sync(inventory) -> Dict[str, Any]: + """ + FS-31 — Verify Bedrock Knowledge Base data sources have recent sync jobs + to ensure information currency. + + Staleness threshold: AWS does not prescribe a maximum data age for Knowledge + Bases — the appropriate cadence is workload-specific (intraday for market + data, weekly/monthly for slow-changing regulatory guidance). This check uses + a default of 7 days purely as a review prompt; treat a finding as "confirm + your data-currency requirement is met," not as an AWS-mandated failure. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT] + """ + # Default review threshold (days). Not an AWS requirement — a configurable + # heuristic. Firms with stricter or looser currency needs should adjust this. + STALE_AFTER_DAYS = 7 + findings = _empty_findings("Knowledge Base Data Source Sync Check") + try: + kb_inv = require(inventory, "knowledge_bases") + kbs = kb_inv.summaries + + if not kbs: + findings["csv_data"].append( + create_finding( + check_id="FS-31", + finding_name="No Knowledge Bases Found", + finding_details="No Bedrock Knowledge Bases found.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base-ingest.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-31"], + ) + ) + return findings + + stale_kbs = [] + now = datetime.now(timezone.utc) + for kb in kbs: + kb_id = kb["knowledgeBaseId"] + sources = kb_inv.data_sources_by_kb.get(kb_id, []) + if isinstance(sources, _Unavailable): + raise sources.error + for source in sources: + last_updated = source.get("updatedAt") + if last_updated: + age_days = (now - last_updated).days + if age_days > STALE_AFTER_DAYS: + stale_kbs.append( + f"KB '{kb['name']}' source '{source['name']}' last synced {age_days} days ago" + ) + + if stale_kbs: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-31", + finding_name="Knowledge Base Data Sources Past Review Threshold", + finding_details=( + f"{len(stale_kbs)} data source(s) not synced in >{STALE_AFTER_DAYS} days " + f"(a configurable review threshold, NOT an AWS-mandated limit):\n" + + "\n".join(f"- {s}" for s in stale_kbs[:10]) + + "\nConfirm this age is acceptable for each data source's currency " + "requirement — slow-changing reference data may legitimately sync infrequently." + ), + resolution=( + "1. Define the maximum acceptable data age per use case (e.g., intraday for " + "market data, daily for product terms, weekly/monthly for regulatory guidance) " + "and adjust the review threshold to match.\n" + "2. Configure automated sync (EventBridge Scheduler → StartIngestionJob) at " + "that cadence — see FS-61.\n" + "3. Set CloudWatch alarms on sync job failures." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base-ingest.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-31"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-31", + finding_name="Knowledge Base Data Sources Recently Synced", + finding_details=( + f"All reviewed KB data sources synced within {STALE_AFTER_DAYS} days " + "(the default review threshold)." + ), + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base-ingest.html", + severity="Medium", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-31"], + ) + ) + except Exception as e: + return _error_findings("Knowledge Base Data Source Sync Check", e) + return findings + + +def check_source_attribution_in_guardrails() -> Dict[str, Any]: + """ + FS-32 — Advisory check: verify application implements source attribution + (citations) in GenAI responses to enable fact-checking. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2] + """ + findings = _empty_findings("Source Attribution Check") + findings["csv_data"].append( + create_finding( + check_id="FS-32", + finding_name="ADVISORY: Source Attribution — Manual Review Required", + finding_details=( + "Source attribution in GenAI responses cannot be verified via AWS APIs. " + "Manual review required to confirm responses include citations." + ), + resolution=( + "1. Use Bedrock RetrieveAndGenerate with citations enabled.\n" + "2. Include source document references in response post-processing.\n" + "3. Test citation accuracy in QA before production deployment.\n" + "4. Consider Bedrock Guardrails grounding checks to validate response accuracy." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/kb-test-config.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-32"], + ) + ) + return findings + + +def check_knowledge_base_integrity_monitoring(inventory) -> Dict[str, Any]: + """ + FS-33 — Check for S3 object integrity monitoring (checksums, versioning) + on Knowledge Base data source buckets. + COMPLIANCE_PLACEHOLDER: [SR 11-7, ISO 27001 A.12.3, FFIEC CAT] + """ + findings = _empty_findings("Knowledge Base Integrity Monitoring Check") + try: + kb_inv = require(inventory, "knowledge_bases") + kbs = kb_inv.summaries + s3 = boto3.client("s3", config=boto3_config) + + if not kbs: + findings["csv_data"].append( + create_finding( + check_id="FS-33", + finding_name="No Knowledge Bases Found", + finding_details="No Bedrock Knowledge Bases found.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-33"], + ) + ) + return findings + + buckets_without_versioning = [] + missing_buckets = [] + for kb in kbs: + kb_id = kb["knowledgeBaseId"] + sources = kb_inv.data_sources_by_kb.get(kb_id, []) + if isinstance(sources, _Unavailable): + raise sources.error + for source in sources: + ds_id = source["dataSourceId"] + source_detail = kb_inv.data_source_detail.get((kb_id, ds_id)) + if isinstance(source_detail, _Unavailable): + raise source_detail.error + if source_detail is None: + continue + s3_config = ( + source_detail.get("dataSource", {}) + .get("dataSourceConfiguration", {}) + .get("s3Configuration", {}) + ) + bucket = _bucket_name_from_arn(s3_config.get("bucketArn", "")) + if bucket: + try: + versioning = s3.get_bucket_versioning(Bucket=bucket) + if versioning.get("Status") != "Enabled": + buckets_without_versioning.append(bucket) + except ClientError as e: + # An access error means we could not read versioning; do + # not mislabel the bucket as non-versioned. Re-raise so it + # surfaces as could-not-assess instead of a false finding. + if _is_access_error(e): + raise + # The data source points to a bucket that no longer exists + # (deleted out from under the KB). This is a distinct, + # actionable integrity problem — report it separately, not + # as "missing versioning." + if _is_missing_bucket_error(e): + logger.warning( + f"KB '{kb['name']}' data source '{source['name']}' " + f"references a deleted bucket: {bucket}" + ) + missing_buckets.append( + f"{bucket} (KB '{kb['name']}', source '{source['name']}')" + ) + continue + logger.warning( + f"Could not check versioning for bucket {bucket}: {e}" + ) + buckets_without_versioning.append(f"{bucket} (error)") + + # A dangling data-source reference to a deleted bucket is a real integrity + # finding in its own right — emit it as a separate row so it is not + # conflated with "versioning not enabled." + if missing_buckets: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-33", + finding_name="KB Data Source References a Deleted S3 Bucket", + finding_details=( + "One or more Knowledge Base data sources point to S3 buckets that no " + "longer exist (NoSuchBucket). Retrieval will silently return no results " + "for these sources, and the integrity of the KB's grounding data cannot " + "be verified:\n" + + "\n".join(f"- {b}" for b in missing_buckets[:10]) + ), + resolution=( + "1. Investigate why the data-source bucket was deleted (accidental " + "deletion, environment teardown, or a stale KB configuration).\n" + "2. Recreate/restore the bucket with versioning enabled, or remove the " + "orphaned data source from the Knowledge Base.\n" + "3. Re-run a KB ingestion job after restoring the data source.\n" + "4. Enable S3 versioning and MFA Delete on KB data-source buckets to " + "reduce the risk of unrecoverable deletion." + ), + reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html", + severity="High", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-33"], + ) + ) + + if buckets_without_versioning: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-33", + finding_name="KB Data Source Buckets Without Versioning", + finding_details=( + f"KB data source S3 buckets without versioning: " + f"{', '.join(buckets_without_versioning[:10])}." + ), + resolution=( + "Enable S3 versioning on all KB data source buckets. " + "Enable S3 Object Integrity (checksum) for tamper detection." + ), + reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-33"], + ) + ) + elif not missing_buckets: + findings["csv_data"].append( + create_finding( + check_id="FS-33", + finding_name="KB Data Source Buckets Have Versioning", + finding_details="All reviewed KB data source buckets have versioning enabled.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html", + severity="Medium", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-33"], + ) + ) + except Exception as e: + return _error_findings("Knowledge Base Integrity Monitoring Check", e) + return findings + + +def check_fm_version_currency() -> Dict[str, Any]: + """ + FS-34 — Advisory check: verify foundation model versions in use are current + and not deprecated (outdated models may have stale training data). + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT] + """ + findings = _empty_findings("Foundation Model Version Currency Check") + try: + bedrock = boto3.client("bedrock", config=boto3_config) + # Do not filter by output modality: legacy/deprecated models exist across + # TEXT, EMBEDDING, and IMAGE modalities. Embedding models are widely used + # in FinServ RAG pipelines and a legacy embedding model produces stale + # embeddings — missing it would be a false-pass. Fetch all modalities and + # let the LEGACY lifecycle filter identify any deprecated model in use. + models = bedrock.list_foundation_models().get("modelSummaries", []) + + deprecated = [ + m["modelId"] + for m in models + if m.get("modelLifecycle", {}).get("status") == "LEGACY" + ] + + if deprecated: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-34", + finding_name="Legacy Foundation Models Available in Region", + finding_details=( + f"Legacy/deprecated foundation models are available in this account/region: " + f"{', '.join(deprecated[:10])}. This API reports model *availability*, not " + "actual usage — it cannot determine which models your applications invoke. " + "Legacy models have older training-data cutoffs and may produce outdated " + "information if used. Review whether any are in active use." + ), + resolution=( + "1. Identify which (if any) of these legacy models your applications invoke " + "(e.g., via CloudTrail InvokeModel events or application config).\n" + "2. Migrate active usage to current model versions.\n" + "3. Document training-data cutoff dates for all models in use.\n" + "4. Add data-currency disclaimers to outputs from models with old cutoffs." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-34"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-34", + finding_name="Foundation Models Are Current", + finding_details="No legacy/deprecated foundation models detected.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle.html", + severity="Medium", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-34"], + ) + ) + except Exception as e: + return _error_findings("Foundation Model Version Currency Check", e) + return findings + + +# =========================================================================== +# CATEGORY 8: ABUSIVE OR HARMFUL OUTPUT (FS-35 to FS-38) +# CATEGORY 9: BIASED OUTPUT (FS-39 to FS-42) +# COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2, NYDFS 500] +# =========================================================================== + + +def check_fmeval_harmful_content() -> Dict[str, Any]: + """ + FS-35 — Check for FMEval or Bedrock Evaluation jobs testing for harmful + content (toxicity, hate speech, violence). + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2] + """ + findings = _empty_findings("FMEval Harmful Content Check") + try: + findings["csv_data"].append( + create_finding( + check_id="FS-35", + finding_name="ADVISORY: Harmful-Content Test Coverage — Manual Review Required", + finding_details=( + "Bedrock model-evaluation dataset content cannot be inspected via API. " + "Manually verify your model-evaluation/FMEval jobs include harmful-content " + "datasets (toxicity, hate speech, violence/self-harm). Whether any evaluation " + "jobs exist at all is assessed by FS-15." + ), + resolution=( + "Run Bedrock Model Evaluation or FMEval with harmful content datasets:\n" + "- Toxicity detection\n" + "- Hate speech classification\n" + "- Violence/self-harm content" + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-35"], + ) + ) + except Exception as e: + return _error_findings("FMEval Harmful Content Check", e) + return findings + + +def check_guardrail_content_filters(inventory) -> Dict[str, Any]: + """ + FS-36 — Verify Bedrock Guardrails have content filters configured for + hate speech, violence, and sexual content at appropriate thresholds. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2] + """ + findings = _empty_findings("Guardrail Content Filters Check") + try: + guardrail_inv = require(inventory, "guardrails") + guardrails = guardrail_inv.summaries + + if not guardrails: + findings["csv_data"].append( + create_finding( + check_id="FS-36", + finding_name="No Guardrails — Content Filters Not Applicable", + finding_details="No Bedrock Guardrails configured.", + resolution="Configure guardrails with content filters.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-filters.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-36"], + ) + ) + return findings + + guardrails_with_filters = [] + guardrails_classic_tier = [] + for g in guardrails: + detail = guardrail_inv.detail_by_id[g["id"]] + if isinstance(detail, _Unavailable): + raise detail.error + content_policy = detail.get("contentPolicy", {}) + if content_policy.get("filters"): + guardrails_with_filters.append(g["name"]) + # Check tier: STANDARD offers better accuracy, multilingual support, + # and improved prompt-attack detection (GA June 2025). FinServ + # workloads benefit from STANDARD for its contextual understanding and + # typo-tolerant detection. STANDARD requires cross-region inference. + # The tier is nested at contentPolicy.tier.tierName in GetGuardrail response. + tier = content_policy.get("tier", {}).get("tierName", "CLASSIC") + if tier == "CLASSIC": + guardrails_classic_tier.append(g["name"]) + + if not guardrails_with_filters: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-36", + finding_name="No Guardrails With Content Filters", + finding_details=( + f"Found {len(guardrails)} guardrail(s) but none have content filters. " + "Harmful content (hate, violence, sexual) may pass through unfiltered." + ), + resolution=( + "1. Add content filters to guardrails for: HATE, INSULTS, SEXUAL, VIOLENCE.\n" + "2. Set filter strength to HIGH for financial services use cases.\n" + "3. Consider the STANDARD tier (GA June 2025) for improved accuracy, " + "typographical error detection, and 60+ language support. STANDARD tier " + "requires cross-region inference to be enabled on the guardrail." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-filters.html", + severity="High", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-36"], + ) + ) + elif guardrails_classic_tier: + # Content filters exist but using CLASSIC tier — emit advisory finding + findings["csv_data"].append( + create_finding( + check_id="FS-36", + finding_name="Guardrail Content Filters on CLASSIC Tier", + finding_details=( + f"Guardrails with content filters: {', '.join(guardrails_with_filters)}. " + f"The following use the CLASSIC tier: {', '.join(guardrails_classic_tier)}. " + "CLASSIC tier supports English, French, and Spanish only. The STANDARD tier " + "(GA June 2025) provides improved contextual understanding, typographical error " + "detection, 60+ language support, and better prompt-attack classification " + "(distinguishes jailbreaks from prompt injection)." + ), + resolution=( + "Consider upgrading to STANDARD tier content filters for FinServ workloads " + "that handle multiple languages or require higher detection accuracy. " + "STANDARD tier requires cross-region inference " + "(crossRegionDetails.guardrailProfileArn on the guardrail). " + "To upgrade: update the guardrail's contentPolicy.filtersConfig.contentFiltersTierConfig " + "with tierName=STANDARD and configure a guardrail cross-region profile." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-filters.html", + severity="High", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-36"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-36", + finding_name="Guardrail Content Filters Configured (STANDARD Tier)", + finding_details=( + f"All {len(guardrails_with_filters)} guardrail(s) with content filters " + "use the STANDARD tier, providing improved accuracy and multilingual support." + ), + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-filters.html", + severity="High", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-36"], + ) + ) + except Exception as e: + return _error_findings("Guardrail Content Filters Check", e) + return findings + + +def check_user_feedback_mechanism() -> Dict[str, Any]: + """ + FS-37 — Advisory check: verify application has a user feedback/reporting + mechanism for harmful GenAI outputs. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2] + """ + findings = _empty_findings("User Feedback Mechanism Check") + findings["csv_data"].append( + create_finding( + check_id="FS-37", + finding_name="ADVISORY: User Feedback Mechanism — Manual Review Required", + finding_details=( + "User feedback mechanisms for harmful outputs cannot be verified via AWS APIs. " + "Manual review required." + ), + resolution=( + "1. Implement thumbs-up/down or flag-for-review UI in GenAI applications.\n" + "2. Route flagged outputs to human reviewers via SQS/SNS.\n" + "3. Log feedback to DynamoDB/S3 for model improvement.\n" + "4. Define SLAs for reviewing flagged content." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-37"], + ) + ) + return findings + + +def check_guardrail_word_filters(inventory) -> Dict[str, Any]: + """ + FS-38 — Verify Bedrock Guardrails have word/phrase filters (allowlists/denylists) + configured for financial services context. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT] + """ + findings = _empty_findings("Guardrail Word Filters Check") + try: + guardrail_inv = require(inventory, "guardrails") + guardrails = guardrail_inv.summaries + + if not guardrails: + findings["csv_data"].append( + create_finding( + check_id="FS-38", + finding_name="No Guardrails — Word Filters Not Applicable", + finding_details="No Bedrock Guardrails configured.", + resolution="Configure guardrails with word filters.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-38"], + ) + ) + return findings + + guardrails_with_words = [] + for g in guardrails: + detail = guardrail_inv.detail_by_id[g["id"]] + if isinstance(detail, _Unavailable): + raise detail.error + if detail.get("wordPolicy", {}).get("words") or detail.get( + "wordPolicy", {} + ).get("managedWordLists"): + guardrails_with_words.append(g["name"]) + + if not guardrails_with_words: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-38", + finding_name="No Guardrails With Word Filters", + finding_details=( + f"Found {len(guardrails)} guardrail(s) but none have word/phrase filters. " + "Profanity and prohibited financial terms may appear in outputs." + ), + resolution=( + "Add word filters to guardrails:\n" + "- Enable AWS managed profanity list\n" + "- Add custom denylist for prohibited financial terms\n" + "- Add allowlist for required regulatory language" + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-38"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-38", + finding_name="Guardrail Word Filters Configured", + finding_details=f"Guardrails with word filters: {', '.join(guardrails_with_words)}.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", + severity="Medium", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-38"], + ) + ) + except Exception as e: + return _error_findings("Guardrail Word Filters Check", e) + return findings + + +def check_sagemaker_clarify_bias() -> Dict[str, Any]: + """ + FS-39 — Verify SageMaker Clarify bias detection jobs are configured for + production models making financial decisions. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, ECOA, Fair Housing Act] + """ + findings = _empty_findings("SageMaker Clarify Bias Check") + try: + sm = boto3.client("sagemaker", config=boto3_config) + schedules = _paginate( + sm, "list_monitoring_schedules", "MonitoringScheduleSummaries" + ) + + bias_schedules = [ + s for s in schedules if s.get("MonitoringType") == "ModelBias" + ] + + if not bias_schedules: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-39", + finding_name="No SageMaker Clarify Bias Monitoring", + finding_details=( + "No SageMaker Clarify model bias monitoring schedules found. " + "Models making financial decisions (credit, insurance) may exhibit " + "discriminatory bias without detection." + ), + resolution=( + "1. Configure SageMaker Clarify bias detection for all models making " + "credit, insurance, or employment decisions.\n" + "2. Define protected attributes (age, gender, race proxies).\n" + "3. Set bias metric thresholds and alert on violations.\n" + "4. Document bias testing results for regulatory examination." + ), + reference="https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-model-monitor-bias-drift.html", + severity="High", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-39"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-39", + finding_name="SageMaker Clarify Bias Monitoring Active", + finding_details=f"Found {len(bias_schedules)} model bias monitoring schedule(s).", + resolution="No action required.", + reference="https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-model-monitor-bias-drift.html", + severity="High", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-39"], + ) + ) + except Exception as e: + return _error_findings("SageMaker Clarify Bias Check", e) + return findings + + +def check_bedrock_evaluation_bias_datasets() -> Dict[str, Any]: + """ + FS-40 — Check whether Bedrock Model Evaluation includes bias-specific + test datasets for GenAI models used in financial decisions. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, ECOA] + """ + findings = _empty_findings("Bedrock Bias Evaluation Datasets Check") + try: + findings["csv_data"].append( + create_finding( + check_id="FS-40", + finding_name="ADVISORY: Bias Dataset Coverage — Manual Review Required", + finding_details=( + "Bedrock model-evaluation dataset content cannot be inspected via API. " + "Manually verify your model-evaluation jobs include bias/fairness datasets " + "(demographic parity, equal-opportunity, counterfactual fairness) for any " + "GenAI models used in financial decisions (ECOA/Fair Housing). Whether any " + "evaluation jobs exist at all is assessed by FS-15." + ), + resolution=( + "Run Bedrock Model Evaluation with bias test datasets:\n" + "- Demographic parity test cases\n" + "- Equal opportunity scenarios\n" + "- Counterfactual fairness tests" + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-evaluation.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-40"], + ) + ) + except Exception as e: + return _error_findings("Bedrock Bias Evaluation Datasets Check", e) + return findings + + +def check_sagemaker_clarify_explainability() -> Dict[str, Any]: + """ + FS-41 — Verify SageMaker Clarify explainability jobs are configured to + provide model decision explanations for adverse action notices. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, ECOA Adverse Action] + """ + findings = _empty_findings("SageMaker Clarify Explainability Check") + try: + sm = boto3.client("sagemaker", config=boto3_config) + schedules = _paginate( + sm, "list_monitoring_schedules", "MonitoringScheduleSummaries" + ) + + explainability_schedules = [ + s for s in schedules if s.get("MonitoringType") == "ModelExplainability" + ] + + if not explainability_schedules: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-41", + finding_name="No SageMaker Clarify Explainability Monitoring", + finding_details=( + "No SageMaker Clarify explainability monitoring found. " + "Models making adverse financial decisions may not provide " + "required explanations (ECOA adverse action notices)." + ), + resolution=( + "1. Configure SageMaker Clarify explainability for credit/lending models.\n" + "2. Generate SHAP values for feature importance.\n" + "3. Map top features to human-readable adverse action reason codes.\n" + "4. Store explanations for regulatory examination." + ), + reference="https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-model-explainability.html", + severity="High", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-41"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-41", + finding_name="SageMaker Clarify Explainability Active", + finding_details=f"Found {len(explainability_schedules)} explainability monitoring schedule(s).", + resolution="No action required.", + reference="https://docs.aws.amazon.com/sagemaker/latest/dg/clarify-model-explainability.html", + severity="High", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-41"], + ) + ) + except Exception as e: + return _error_findings("SageMaker Clarify Explainability Check", e) + return findings + + +def check_ai_service_cards_documentation() -> Dict[str, Any]: + """ + FS-42 — Advisory check: verify AI Service Cards / Model Cards are + documented for all production GenAI models. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.3] + """ + findings = _empty_findings("AI Service Cards Documentation Check") + try: + sm = boto3.client("sagemaker", config=boto3_config) + model_cards = _paginate(sm, "list_model_cards", "ModelCardSummaryList") + + if not model_cards: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-42", + finding_name="No SageMaker Model Cards Found", + finding_details=( + "No SageMaker Model Cards found. " + "Production AI models lack documented intended use, limitations, and bias evaluations." + ), + resolution=( + "1. Create SageMaker Model Cards for all production models.\n" + "2. Document: intended use, out-of-scope uses, training data, bias evaluations.\n" + "3. Include regulatory compliance attestations.\n" + "4. Review and update cards at each model version release." + ), + reference="https://docs.aws.amazon.com/sagemaker/latest/dg/model-cards.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-42"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-42", + finding_name="SageMaker Model Cards Present", + finding_details=f"Found {len(model_cards)} model card(s).", + resolution="Verify cards are current and include bias/fairness evaluations.", + reference="https://docs.aws.amazon.com/sagemaker/latest/dg/model-cards.html", + severity="Medium", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-42"], + ) + ) + except Exception as e: + return _error_findings("AI Service Cards Documentation Check", e) + return findings + + +# =========================================================================== +# CATEGORY 10: SENSITIVE INFORMATION DISCLOSURE (FS-43 to FS-46) +# COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, FFIEC CAT, PCI-DSS 3.4, GDPR Art.25] +# =========================================================================== + + +def check_cloudwatch_log_pii_masking() -> Dict[str, Any]: + """ + FS-43 — Check for CloudWatch Logs data protection policies that mask PII + in Bedrock invocation logs. + COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, GDPR Art.25, PCI-DSS 3.4] + """ + findings = _empty_findings("CloudWatch Log PII Masking Check") + try: + logs = boto3.client("logs", config=boto3_config) + # List data protection policies + try: + policies = logs.describe_account_policies( + policyType="DATA_PROTECTION_POLICY" + ).get("accountPolicies", []) + except ClientError: + policies = [] + + if not policies: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-43", + finding_name="No CloudWatch Logs Data Protection Policies", + finding_details=( + "No CloudWatch Logs data protection policies found. " + "PII (SSN, account numbers, credit card numbers) in Bedrock invocation logs " + "may be stored in plaintext." + ), + resolution=( + "1. Create CloudWatch Logs data protection policies to mask PII.\n" + "2. Enable masking for: SSN, credit card numbers, bank account numbers, email.\n" + "3. Apply policies to Bedrock invocation log groups.\n" + "4. Test masking with synthetic PII before production deployment." + ), + reference="https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/mask-sensitive-log-data.html", + severity="High", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-43"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-43", + finding_name="CloudWatch Logs Data Protection Policies Present", + finding_details=f"Found {len(policies)} data protection policy(ies).", + resolution="Verify policies cover Bedrock invocation log groups.", + reference="https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/mask-sensitive-log-data.html", + severity="High", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-43"], + ) + ) + except Exception as e: + return _error_findings("CloudWatch Log PII Masking Check", e) + return findings + + +def check_macie_on_training_data_buckets() -> Dict[str, Any]: + """ + FS-44 — Verify Amazon Macie is enabled and scanning S3 buckets that + contain training data or KB data sources for PII. + COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, GDPR Art.25, PCI-DSS 3.4, FFIEC CAT] + """ + findings = _empty_findings("Amazon Macie PII Scanning Check") + try: + macie = boto3.client("macie2", config=boto3_config) + try: + status = macie.get_macie_session() + macie_enabled = status.get("status") == "ENABLED" + except ClientError: + macie_enabled = False + + if not macie_enabled: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-44", + finding_name="Amazon Macie Not Enabled", + finding_details=( + "Amazon Macie is not enabled. S3 buckets containing training data " + "and KB data sources are not being scanned for PII/sensitive data." + ), + resolution=( + "1. Enable Amazon Macie in all regions where AI/ML data is stored.\n" + "2. Create Macie classification jobs for training data and KB buckets.\n" + "3. Configure Macie findings to route to Security Hub and SNS.\n" + "4. Remediate PII findings before using data for model training." + ), + reference="https://docs.aws.amazon.com/macie/latest/user/what-is-macie.html", + severity="High", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-44"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-44", + finding_name="Amazon Macie Enabled", + finding_details="Amazon Macie is enabled and scanning S3 buckets.", + resolution="Verify Macie jobs cover training data and KB data source buckets.", + reference="https://docs.aws.amazon.com/macie/latest/user/what-is-macie.html", + severity="High", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-44"], + ) + ) + except Exception as e: + return _error_findings("Amazon Macie PII Scanning Check", e) + return findings + + +def check_guardrail_pii_filters(inventory) -> Dict[str, Any]: + """ + FS-45 — Verify Bedrock Guardrails have sensitive information (PII) filters + configured to block PII in prompts and responses. + COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, GDPR Art.25, PCI-DSS 3.4] + """ + findings = _empty_findings("Guardrail PII Filters Check") + try: + guardrail_inv = require(inventory, "guardrails") + guardrails = guardrail_inv.summaries + + if not guardrails: + findings["csv_data"].append( + create_finding( + check_id="FS-45", + finding_name="No Guardrails — PII Filters Not Applicable", + finding_details="No Bedrock Guardrails configured.", + resolution="Configure guardrails with PII/sensitive information filters.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-sensitive-filters.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-45"], + ) + ) + return findings + + guardrails_with_pii = [] + for g in guardrails: + detail = guardrail_inv.detail_by_id[g["id"]] + if isinstance(detail, _Unavailable): + raise detail.error + if detail.get("sensitiveInformationPolicy", {}).get("piiEntities"): + guardrails_with_pii.append(g["name"]) + + if not guardrails_with_pii: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-45", + finding_name="No Guardrails With PII Filters", + finding_details=( + f"Found {len(guardrails)} guardrail(s) but none have PII entity filters. " + "SSN, credit card numbers, and account numbers may appear in GenAI outputs." + ), + resolution=( + "Add PII entity filters to guardrails for:\n" + "- US_SOCIAL_SECURITY_NUMBER\n" + "- CREDIT_DEBIT_CARD_NUMBER\n" + "- BANK_ACCOUNT_NUMBER\n" + "- EMAIL, PHONE, NAME (as appropriate)\n" + "Set action to ANONYMIZE or BLOCK." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-sensitive-filters.html", + severity="High", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-45"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-45", + finding_name="Guardrail PII Filters Configured", + finding_details=f"Guardrails with PII filters: {', '.join(guardrails_with_pii)}.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-sensitive-filters.html", + severity="High", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-45"], + ) + ) + except Exception as e: + return _error_findings("Guardrail PII Filters Check", e) + return findings + + +def check_data_classification_tagging(inventory) -> Dict[str, Any]: + """ + FS-46 — Check that S3 buckets containing AI/ML data are tagged with + data classification labels (e.g., Confidential, PII, Public). + COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, FFIEC CAT, ISO 27001 A.8.2] + """ + findings = _empty_findings("Data Classification Tagging Check") + try: + buckets = require(inventory, "buckets") + s3 = boto3.client("s3", config=boto3_config) + + aiml_buckets = [ + b + for b in buckets + if any( + kw in b["Name"].lower() + for kw in ["train", "model", "bedrock", "sagemaker", "kb", "knowledge"] + ) + ] + + if not aiml_buckets: + findings["csv_data"].append( + create_finding( + check_id="FS-46", + finding_name="No AI/ML Data Buckets Identified", + finding_details="No S3 buckets with AI/ML naming found.", + resolution="Tag AI/ML data buckets with data-classification labels.", + reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-46"], + ) + ) + return findings + + unclassified = [] + for bucket in aiml_buckets: + try: + tags = s3.get_bucket_tagging(Bucket=bucket["Name"]).get("TagSet", []) + tag_keys = {t["Key"].lower() for t in tags} + if ( + "data-classification" not in tag_keys + and "classification" not in tag_keys + ): + unclassified.append(bucket["Name"]) + except ClientError as e: + # A genuine "no tags" response (NoSuchTagSet) means the bucket is + # unclassified — a real finding. An access error means we could + # not read the tags; re-raise so it surfaces as could-not-assess + # rather than a false "unclassified" finding. + if _is_access_error(e): + raise + unclassified.append(bucket["Name"]) + + if unclassified: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-46", + finding_name="AI/ML Buckets Without Data Classification Tags", + finding_details=( + f"{len(unclassified)} AI/ML bucket(s) without data-classification tags: " + f"{', '.join(unclassified[:10])}." + ), + resolution=( + "Tag all AI/ML data buckets with 'data-classification' key. " + "Values: Public, Internal, Confidential, Restricted. " + "Enforce via SCP or AWS Config rule." + ), + reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-46"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-46", + finding_name="AI/ML Buckets Have Classification Tags", + finding_details=f"All {len(aiml_buckets)} AI/ML bucket(s) have classification tags.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-tagging.html", + severity="Medium", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-46"], + ) + ) + except Exception as e: + return _error_findings("Data Classification Tagging Check", e) + return findings + + +# =========================================================================== +# CATEGORY 11: HALLUCINATION (FS-47 to FS-50) +# CATEGORY 12: PROMPT INJECTION (FS-51 to FS-54) +# COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2, NYDFS 500] +# =========================================================================== + + +def check_guardrail_grounding_threshold(inventory) -> Dict[str, Any]: + """ + FS-47 — Verify Bedrock Guardrails contextual grounding thresholds are + set appropriately high for financial services use cases. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2] + """ + findings = _empty_findings("Guardrail Grounding Threshold Check") + try: + guardrail_inv = require(inventory, "guardrails") + guardrails = guardrail_inv.summaries + + if not guardrails: + findings["csv_data"].append( + create_finding( + check_id="FS-47", + finding_name="No Guardrails — Grounding Threshold Not Applicable", + finding_details="No Bedrock Guardrails configured.", + resolution="Configure guardrails with contextual grounding checks.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-47"], + ) + ) + return findings + + low_threshold_guardrails = [] + guardrails_with_grounding = [] + for g in guardrails: + detail = guardrail_inv.detail_by_id[g["id"]] + if isinstance(detail, _Unavailable): + raise detail.error + grounding = detail.get("contextualGroundingPolicy", {}) + has_grounding_filter = False + for filter_item in grounding.get("filters", []): + if filter_item.get("type") == "GROUNDING": + has_grounding_filter = True + if filter_item.get("threshold", 1.0) < 0.7: + low_threshold_guardrails.append( + f"{g['name']} (threshold={filter_item['threshold']})" + ) + if has_grounding_filter: + guardrails_with_grounding.append(g["name"]) + + if low_threshold_guardrails: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-47", + finding_name="Guardrails With Low Grounding Thresholds", + finding_details=( + f"Guardrails with grounding threshold <0.7: {', '.join(low_threshold_guardrails)}. " + "Low thresholds allow hallucinated responses to pass through." + ), + resolution=( + "Set grounding threshold to 0.7 or higher for financial services use cases " + "(valid range is 0 to 0.99; 1.0 is invalid and blocks all content). " + "Test threshold impact on response quality before increasing. Note: contextual " + "grounding supports summarization, paraphrasing, and Q&A — not conversational " + "chatbot use cases; for chatbots use denied topics (FS-28/FS-59) instead." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-contextual-grounding-check.html", + severity="High", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-47"], + ) + ) + elif not guardrails_with_grounding: + # Guardrails exist but NONE has a GROUNDING filter at all. This is a + # genuine gap (not a pass): without a grounding filter, ungrounded / + # hallucinated responses are not detected. Previously this fell through + # to the "Passed" branch because low_threshold_guardrails was empty. + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-47", + finding_name="No Guardrails With a Grounding Filter", + finding_details=( + f"Found {len(guardrails)} guardrail(s) but none have a GROUNDING contextual " + "grounding filter configured. Ungrounded or hallucinated responses are not " + "detected for summarization/paraphrasing/Q&A use cases." + ), + resolution=( + "Add a GROUNDING contextual grounding filter (threshold ≥0.7; valid range " + "0 to 0.99) to each guardrail used for summarization, paraphrasing, or Q&A. " + "Note: contextual grounding is not supported for conversational chatbot use " + "cases — use denied topics (FS-28/FS-59) for those." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-contextual-grounding-check.html", + severity="High", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-47"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-47", + finding_name="Guardrail Grounding Thresholds Appropriate", + finding_details=( + f"All {len(guardrails_with_grounding)} guardrail(s) with a GROUNDING filter " + "have thresholds ≥0.7." + ), + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-contextual-grounding-check.html", + severity="High", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-47"], + ) + ) + except Exception as e: + return _error_findings("Guardrail Grounding Threshold Check", e) + return findings + + +def check_rag_knowledge_base_configured(inventory) -> Dict[str, Any]: + """ + FS-48 — Verify RAG (Retrieval Augmented Generation) is used via Bedrock + Knowledge Bases to ground responses in authoritative data. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT] + """ + findings = _empty_findings("RAG Knowledge Base Configuration Check") + try: + kbs = require(inventory, "knowledge_bases").summaries + + active_kbs = [k for k in kbs if k.get("status") == "ACTIVE"] + + if not active_kbs: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-48", + finding_name="No Active Knowledge Bases for RAG", + finding_details=( + "No active Bedrock Knowledge Bases found. " + "GenAI responses are not grounded in authoritative data sources, " + "increasing hallucination risk." + ), + resolution=( + "1. Create Bedrock Knowledge Bases with authoritative financial data.\n" + "2. Use RetrieveAndGenerate API to ground responses.\n" + "3. Configure data sources with current regulatory and product information." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-48"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-48", + finding_name="Active Knowledge Bases for RAG Present", + finding_details=f"Found {len(active_kbs)} active Knowledge Base(s) for RAG grounding.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base.html", + severity="Medium", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-48"], + ) + ) + except Exception as e: + return _error_findings("RAG Knowledge Base Configuration Check", e) + return findings + + +def check_hallucination_disclaimer_advisory() -> Dict[str, Any]: + """ + FS-49 — Advisory check: verify application adds hallucination disclaimers + to GenAI outputs in financial contexts. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2] + """ + findings = _empty_findings("Hallucination Disclaimer Advisory") + findings["csv_data"].append( + create_finding( + check_id="FS-49", + finding_name="ADVISORY: Hallucination Disclaimer — Manual Review Required", + finding_details=( + "Application-level hallucination disclaimers cannot be verified via AWS APIs. " + "Manual review required." + ), + resolution=( + "1. Add disclaimers to GenAI outputs: 'AI-generated content may contain errors. " + "Verify with authoritative sources before acting.'\n" + "2. Implement post-processing to append disclaimers.\n" + "3. Test disclaimer presence in QA before production." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-49"], + ) + ) + return findings + + +def check_guardrail_relevance_grounding(inventory) -> Dict[str, Any]: + """ + FS-50 — Check for Bedrock Guardrails contextual grounding RELEVANCE filters + configured to detect and block responses that are not grounded in the context + retrieved by the RAG pipeline (hallucination prevention). + + NOTE: This check verifies the *RELEVANCE* filter within contextual grounding + (a different feature from GROUNDING-type filters). RELEVANCE filters block + responses that are off-topic relative to the user query. GROUNDING filters + block responses not supported by the source context. Both are important for + FinServ hallucination mitigation. For formal policy-based verification of + factual claims, see check_automated_reasoning_policies() (FS-27b). + + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT] + """ + findings = _empty_findings("Guardrail Relevance Grounding Check") + try: + guardrail_inv = require(inventory, "guardrails") + guardrails = guardrail_inv.summaries + + guardrails_with_relevance = [] + for g in guardrails: + detail = guardrail_inv.detail_by_id[g["id"]] + if isinstance(detail, _Unavailable): + raise detail.error + grounding = detail.get("contextualGroundingPolicy", {}) + for f in grounding.get("filters", []): + if f.get("type") == "RELEVANCE": + guardrails_with_relevance.append(g["name"]) + + if not guardrails_with_relevance: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-50", + finding_name="No Guardrails With Relevance Grounding Filters", + finding_details=( + "No guardrails have RELEVANCE contextual grounding filters. " + "Without relevance filters, responses that are off-topic or unrelated " + "to the user query will not be blocked, increasing hallucination risk " + "in RAG-based FinServ applications." + ), + resolution=( + "Enable the RELEVANCE contextual grounding filter in Bedrock Guardrails " + "with a threshold of ≥0.7 to block responses that are not relevant to " + "the user query. Also enable the GROUNDING filter (≥0.7) to block " + "responses not supported by the retrieved source context." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-50"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-50", + finding_name="Relevance Grounding Filters Present", + finding_details=( + f"Guardrails with RELEVANCE grounding filters: " + f"{', '.join(guardrails_with_relevance)}." + ), + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", + severity="Medium", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-50"], + ) + ) + except Exception as e: + return _error_findings("Guardrail Relevance Grounding Check", e) + return findings + + +def check_prompt_injection_input_validation(inventory) -> Dict[str, Any]: + """ + FS-51 — Check for Bedrock Guardrails prompt attack filters to detect + and block prompt injection attempts. + COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, FFIEC CAT, OWASP LLM01] + """ + findings = _empty_findings("Prompt Injection Input Validation Check") + try: + guardrail_inv = require(inventory, "guardrails") + guardrails = guardrail_inv.summaries + + if not guardrails: + findings["csv_data"].append( + create_finding( + check_id="FS-51", + finding_name="No Guardrails — Prompt Attack Filters Not Applicable", + finding_details="No Bedrock Guardrails configured.", + resolution="Configure guardrails with prompt attack filters.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-51"], + ) + ) + return findings + + guardrails_with_prompt_attack = [] + guardrails_classic_tier_pa = [] + for g in guardrails: + detail = guardrail_inv.detail_by_id[g["id"]] + if isinstance(detail, _Unavailable): + raise detail.error + content_policy = detail.get("contentPolicy", {}) + for f in content_policy.get("filters", []): + if f.get("type") == "PROMPT_ATTACK": + guardrails_with_prompt_attack.append(g["name"]) + # STANDARD tier (GA June 2025) improves PROMPT_ATTACK detection + # by distinguishing jailbreaks from prompt injection attacks. + tier = content_policy.get("tier", {}).get("tierName", "CLASSIC") + if tier == "CLASSIC": + guardrails_classic_tier_pa.append(g["name"]) + break + + if not guardrails_with_prompt_attack: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-51", + finding_name="No Guardrails With Prompt Attack Filters", + finding_details=( + f"Found {len(guardrails)} guardrail(s) but none have PROMPT_ATTACK filters. " + "Prompt injection attacks may bypass system prompts and access controls." + ), + resolution=( + "1. Enable PROMPT_ATTACK content filter in Bedrock Guardrails.\n" + "2. Set input filter strength to HIGH.\n" + "3. Use input tags () to " + "differentiate user inputs from developer-provided prompts — required for " + "PROMPT_ATTACK filters to work correctly with InvokeModel/InvokeModelWithResponseStream.\n" + "4. Consider STANDARD tier (GA June 2025) for better jailbreak vs. injection " + "classification and broader language support.\n" + "5. Implement application-level input sanitization as defense-in-depth." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-prompt-attack.html", + severity="High", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-51"], + ) + ) + elif guardrails_classic_tier_pa: + findings["csv_data"].append( + create_finding( + check_id="FS-51", + finding_name="Prompt Attack Filters on CLASSIC Tier", + finding_details=( + f"Guardrails with PROMPT_ATTACK filters: {', '.join(guardrails_with_prompt_attack)}. " + f"Using CLASSIC tier: {', '.join(guardrails_classic_tier_pa)}. " + "STANDARD tier (GA June 2025) better distinguishes jailbreaks from " + "prompt injection and provides broader language support." + ), + resolution=( + "Consider upgrading to STANDARD tier for improved PROMPT_ATTACK detection. " + "Ensure input tags are used to scope user content for PROMPT_ATTACK evaluation. " + "STANDARD tier requires cross-region inference on the guardrail." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-prompt-attack.html", + severity="High", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-51"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-51", + finding_name="Prompt Attack Filters Configured (STANDARD Tier)", + finding_details=( + f"Guardrails with PROMPT_ATTACK filters (STANDARD tier): " + f"{', '.join(guardrails_with_prompt_attack)}." + ), + resolution=( + "Ensure input tags are used to scope user content when calling " + "InvokeModel/InvokeModelWithResponseStream — required for PROMPT_ATTACK " + "filters to evaluate user input separately from system prompts." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-prompt-attack.html", + severity="High", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-51"], + ) + ) + except Exception as e: + return _error_findings("Prompt Injection Input Validation Check", e) + return findings + + +def check_bedrock_sdk_version_currency(inventory) -> Dict[str, Any]: + """ + FS-52 — Advisory check: verify Bedrock SDK versions in Lambda functions + are current (outdated SDKs may lack prompt injection mitigations). + COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, FFIEC CAT, ISO 27001 A.12.6] + """ + findings = _empty_findings("Bedrock SDK Version Currency Check") + try: + functions = require(inventory, "lambda_functions") + + bedrock_functions = [ + f + for f in functions + if any( + kw in f["FunctionName"].lower() + for kw in ["bedrock", "agent", "aiml", "genai"] + ) + ] + + if not bedrock_functions: + findings["csv_data"].append( + create_finding( + check_id="FS-52", + finding_name="No Bedrock-Related Lambda Functions Found", + finding_details="No Lambda functions with Bedrock-related naming found.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/lambda/latest/dg/runtimes-update.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-52"], + ) + ) + return findings + + # Check for deprecated runtimes using the definitive allowlist of currently- + # supported Lambda managed runtimes (sourced from + # https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html, + # retrieved June 2026). Any runtime NOT in this set is considered deprecated + # or end-of-support. This allowlist approach is more reliable than maintaining + # a denylist (which silently misses newly-deprecated runtimes). + # + # Supported as of June 2026 (ordered newest-first per language): + SUPPORTED_RUNTIMES = { + # Python + "python3.14", + "python3.13", + "python3.12", + "python3.11", + "python3.10", + # Node.js + "nodejs24.x", + "nodejs22.x", + # Java + "java25", + "java21", + "java17", + "java11", + "java8.al2", + # .NET + "dotnet10", + "dotnet9", + "dotnet8", + # Ruby + "ruby4.0", + "ruby3.4", + "ruby3.3", + # OS-only / custom runtimes + "provided.al2023", + "provided.al2", + } + outdated_functions = [ + f["FunctionName"] + for f in bedrock_functions + if f.get("Runtime", "") and f["Runtime"] not in SUPPORTED_RUNTIMES + ] + + if outdated_functions: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-52", + finding_name="Bedrock Lambda Functions on Deprecated Runtimes", + finding_details=( + f"Functions on deprecated runtimes: {', '.join(outdated_functions[:10])}. " + "Deprecated runtimes may use outdated boto3/SDK versions lacking security patches." + ), + resolution=( + "1. Upgrade Lambda functions to a supported runtime — Python 3.12+, " + "Node.js 22.x or 24.x, Java 21+, or .NET 8+.\n" + "2. Update boto3 to the latest version in Lambda layers (pin the version " + "in requirements.txt and redeploy).\n" + "3. Enable Lambda runtime management controls for automatic minor-version " + "updates (runtimeManagementConfig.updateRuntimeOn = 'Auto').\n" + "4. Refer to https://docs.aws.amazon.com/lambda/latest/dg/lambda-runtimes.html " + "for the authoritative list of supported and deprecated runtimes." + ), + reference="https://docs.aws.amazon.com/lambda/latest/dg/runtimes-update.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-52"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-52", + finding_name="Bedrock Lambda Functions on Current Runtimes", + finding_details=f"All {len(bedrock_functions)} Bedrock Lambda function(s) use current runtimes.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/lambda/latest/dg/runtimes-update.html", + severity="Medium", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-52"], + ) + ) + except Exception as e: + return _error_findings("Bedrock SDK Version Currency Check", e) + return findings + + +def check_waf_sql_injection_rules(inventory) -> Dict[str, Any]: + """ + FS-53 — Verify WAF Web ACLs include SQL injection and XSS managed rules + to protect GenAI API endpoints from injection attacks. + COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, FFIEC CAT, PCI-DSS 6.4.1, OWASP LLM01] + """ + findings = _empty_findings("WAF Injection Protection Rules Check") + try: + # require() raises if inventory is None or web_acls is _Unavailable, + # propagating to the outer except which yields COULD_NOT_ASSESS. + web_acl_inv = require(inventory, "web_acls") + acls = web_acl_inv.summaries + + if not acls: + findings["csv_data"].append( + create_finding( + check_id="FS-53", + finding_name="No WAF Web ACLs — Injection Rules Not Applicable", + finding_details="No regional WAF Web ACLs found.", + resolution="Create WAF Web ACLs with injection protection rules (see FS-01).", + reference="https://docs.aws.amazon.com/waf/latest/developerguide/aws-managed-rule-groups-baseline.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-53"], + ) + ) + return findings + + INJECTION_RULE_GROUPS = { + "AWSManagedRulesSQLiRuleSet", + "AWSManagedRulesCommonRuleSet", + "AWSManagedRulesKnownBadInputsRuleSet", + } + + acls_without_injection_rules = [] + for acl_summary in acls: + # detail_by_id holds get_web_acl(...)['WebACL'] or _Unavailable. + # Accessing an _Unavailable entry re-raises its stored error, which + # propagates to the outer except and yields COULD_NOT_ASSESS — + # matching today's behaviour (no per-item try/except in this check). + detail = web_acl_inv.detail_by_id[acl_summary["Id"]] + if isinstance(detail, _Unavailable): + raise detail.error + acl = detail + rule_names = { + r.get("Statement", {}) + .get("ManagedRuleGroupStatement", {}) + .get("Name", "") + for r in acl.get("Rules", []) + } + if not rule_names.intersection(INJECTION_RULE_GROUPS): + acls_without_injection_rules.append(acl_summary["Name"]) + + if acls_without_injection_rules: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-53", + finding_name="WAF ACLs Missing Injection Protection Rules", + finding_details=( + f"WAF ACLs without SQL injection/XSS rules: " + f"{', '.join(acls_without_injection_rules[:10])}." + ), + resolution=( + "Add AWS Managed Rule Groups to WAF ACLs:\n" + "- AWSManagedRulesSQLiRuleSet (SQL injection)\n" + "- AWSManagedRulesCommonRuleSet (XSS, LFI, RFI)\n" + "- AWSManagedRulesKnownBadInputsRuleSet (prompt injection patterns)" + ), + reference="https://docs.aws.amazon.com/waf/latest/developerguide/aws-managed-rule-groups-baseline.html", + severity="High", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-53"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-53", + finding_name="WAF Injection Protection Rules Present", + finding_details=f"All {len(acls)} WAF ACL(s) have injection protection rules.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/waf/latest/developerguide/aws-managed-rule-groups-baseline.html", + severity="High", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-53"], + ) + ) + except Exception as e: + return _error_findings("WAF Injection Protection Rules Check", e) + return findings + + +def check_penetration_testing_evidence() -> Dict[str, Any]: + """ + FS-54 — Advisory check: verify penetration testing has been conducted + on GenAI applications (prompt injection, jailbreak testing). + COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, FFIEC CAT, PCI-DSS 11.4, DORA Art.26] + """ + findings = _empty_findings("Penetration Testing Evidence Check") + findings["csv_data"].append( + create_finding( + check_id="FS-54", + finding_name="ADVISORY: Penetration Testing — Manual Review Required", + finding_details=( + "Penetration testing evidence cannot be verified via AWS APIs. " + "Manual review required to confirm GenAI applications have been tested." + ), + resolution=( + "1. Conduct penetration testing of GenAI applications at least annually and " + "before major releases.\n" + "2. Include AI-specific test cases: prompt injection, jailbreak, indirect " + "(cross-domain) injection, system-prompt leakage, and data-extraction attempts.\n" + "3. Consider AWS Security Agent for on-demand, AI-driven penetration testing " + "(GA March 2026; available in US East N. Virginia, US West Oregon, Europe Ireland, " + "Europe Frankfurt, Asia Pacific Sydney, Asia Pacific Tokyo, with cross-account " + "shared-VPC testing via AWS RAM). Open-source tools such as Garak or PyRIT and " + "manual red-teaming are complementary options. Verify current regional availability " + "on the AWS Security Agent page before relying on it.\n" + "4. Document findings and remediation for regulatory examination, and tag tested " + "resources with a last-pentest-date for audit trail.\n" + "5. For DORA compliance, include GenAI in TLPT (Threat-Led Penetration Testing) scope." + ), + reference="https://aws.amazon.com/security/penetration-testing/", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-54"], + ) + ) + return findings + + +# =========================================================================== +# CATEGORY 13: IMPROPER OUTPUT HANDLING (FS-55 to FS-58) +# CATEGORY 14: OFF-TOPIC & INAPPROPRIATE OUTPUT (FS-59 to FS-60) +# CATEGORY 15: OUT-OF-DATE TRAINING DATA (FS-61 to FS-63) +# COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, NYDFS 500, OWASP LLM02] +# =========================================================================== + + +def check_output_validation_lambda(inventory) -> Dict[str, Any]: + """ + FS-55 — Check for Lambda functions implementing output validation/sanitization + in GenAI application pipelines. + COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, FFIEC CAT, OWASP LLM02] + """ + findings = _empty_findings("Output Validation Lambda Check") + try: + functions = require(inventory, "lambda_functions") + + validation_functions = [ + f + for f in functions + if any( + kw in f["FunctionName"].lower() + for kw in ["validate", "sanitize", "filter", "output"] + ) + ] + + if not validation_functions: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-55", + finding_name="No Output Validation Functions Found", + finding_details=( + "No Lambda functions with output validation/sanitization naming found. " + "GenAI outputs may be passed directly to downstream systems without validation." + ), + resolution=( + "1. Implement output validation Lambda functions in GenAI pipelines.\n" + "2. Validate output schema, length, and content before downstream use.\n" + "3. Sanitize outputs before rendering in web UIs (XSS prevention).\n" + "4. Encode outputs appropriately for the target context (HTML, SQL, JSON)." + ), + reference="https://genai.owasp.org/llm-top-10/", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-55"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-55", + finding_name="Output Validation Functions Present", + finding_details=f"Found {len(validation_functions)} output validation/sanitization function(s).", + resolution="No action required.", + reference="https://genai.owasp.org/llm-top-10/", + severity="Medium", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-55"], + ) + ) + except Exception as e: + return _error_findings("Output Validation Lambda Check", e) + return findings + + +def check_xss_prevention_waf(inventory) -> Dict[str, Any]: + """ + FS-56 — Verify WAF rules include XSS prevention for GenAI web application outputs. + COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, FFIEC CAT, PCI-DSS 6.4.1, OWASP LLM02] + """ + findings = _empty_findings("XSS Prevention WAF Check") + try: + # require() raises if inventory is None or web_acls is _Unavailable. + web_acl_inv = require(inventory, "web_acls") + acls = web_acl_inv.summaries + + if not acls: + findings["csv_data"].append( + create_finding( + check_id="FS-56", + finding_name="No WAF ACLs — XSS Prevention Not Applicable", + finding_details="No regional WAF Web ACLs found.", + resolution="Create WAF ACLs with XSS prevention rules.", + reference="https://docs.aws.amazon.com/waf/latest/developerguide/aws-managed-rule-groups-baseline.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-56"], + ) + ) + return findings + + # XSS protections live in the AWS managed Common Rule Set. Inspect each ACL's + # managed-rule-group statements (mirrors FS-53) to actually verify coverage + # rather than emitting an unconditional "review required" pass. + acls_without_xss = [] + for acl_summary in acls: + # detail_by_id holds get_web_acl(...)['WebACL'] or _Unavailable. + # An _Unavailable entry re-raises its error → outer except → COULD_NOT_ASSESS. + detail = web_acl_inv.detail_by_id[acl_summary["Id"]] + if isinstance(detail, _Unavailable): + raise detail.error + acl = detail + rule_groups = { + r.get("Statement", {}) + .get("ManagedRuleGroupStatement", {}) + .get("Name", "") + for r in acl.get("Rules", []) + } + if "AWSManagedRulesCommonRuleSet" not in rule_groups: + acls_without_xss.append(acl_summary["Name"]) + + if acls_without_xss: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-56", + finding_name="WAF ACLs Missing Common Rule Set (XSS)", + finding_details=( + "The following WAF ACL(s) do not include AWSManagedRulesCommonRuleSet, " + "which provides cross-site-scripting (XSS) protections for GenAI web " + "application outputs:\n" + + "\n".join(f"- {a}" for a in acls_without_xss[:10]) + ), + resolution=( + "1. Add AWSManagedRulesCommonRuleSet to each WAF ACL protecting GenAI " + "web applications (it includes the CrossSiteScripting rules).\n" + "2. Additionally, implement Content Security Policy (CSP) response headers." + ), + reference="https://docs.aws.amazon.com/waf/latest/developerguide/aws-managed-rule-groups-baseline.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-56"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-56", + finding_name="XSS Prevention Common Rule Set Present", + finding_details=( + f"All {len(acls)} WAF ACL(s) include AWSManagedRulesCommonRuleSet " + "(XSS protections)." + ), + resolution=( + "No action required. Consider also implementing Content Security Policy " + "(CSP) response headers as defense in depth." + ), + reference="https://docs.aws.amazon.com/waf/latest/developerguide/aws-managed-rule-groups-baseline.html", + severity="Medium", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-56"], + ) + ) + except Exception as e: + return _error_findings("XSS Prevention WAF Check", e) + return findings + + +def check_output_encoding_advisory() -> Dict[str, Any]: + """ + FS-57 — Advisory check: verify application encodes GenAI outputs + appropriately for the rendering context. + COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, FFIEC CAT, OWASP LLM02] + """ + findings = _empty_findings("Output Encoding Advisory") + findings["csv_data"].append( + create_finding( + check_id="FS-57", + finding_name="ADVISORY: Output Encoding — Manual Review Required", + finding_details=( + "Output encoding practices cannot be verified via AWS APIs. " + "Manual code review required." + ), + resolution=( + "1. HTML-encode GenAI outputs before rendering in web UIs.\n" + "2. Use parameterized queries when GenAI output is used in database operations.\n" + "3. JSON-encode outputs before embedding in JavaScript contexts.\n" + "4. Validate output length and format before passing to downstream APIs." + ), + reference="https://genai.owasp.org/llm-top-10/", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-57"], + ) + ) + return findings + + +def check_output_schema_validation(inventory) -> Dict[str, Any]: + """ + FS-58 — Check for structured output validation using Bedrock response + schemas or application-level JSON schema validation. + COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, FFIEC CAT, OWASP LLM02] + """ + findings = _empty_findings("Output Schema Validation Check") + try: + # Check for EventBridge Pipes or Lambda destinations that could validate outputs + functions = require(inventory, "lambda_functions") + + schema_functions = [ + f + for f in functions + if any( + kw in f["FunctionName"].lower() + for kw in ["schema", "validate", "parse", "format"] + ) + ] + + findings["csv_data"].append( + create_finding( + check_id="FS-58", + finding_name="ADVISORY: Output Schema Validation — Manual Review Required", + finding_details=( + f"Found {len(schema_functions)} Lambda function(s) whose names suggest " + "schema/validation handling. Structured-output / JSON-schema validation of " + "GenAI responses is an application-layer control that cannot be verified " + "automatically — manual review required." + ), + resolution=( + "1. Use Bedrock structured output (response schemas) where supported.\n" + "2. Implement JSON schema validation on Lambda output processors.\n" + "3. Reject malformed outputs and return safe error responses.\n" + "4. Log schema validation failures to CloudWatch for monitoring." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/inference-parameters.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-58"], + ) + ) + except Exception as e: + return _error_findings("Output Schema Validation Check", e) + return findings + + +def check_guardrail_topic_allowlist(inventory) -> Dict[str, Any]: + """ + FS-59 — Verify Bedrock Guardrails topic policies restrict GenAI to + on-topic financial services responses only. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2] + """ + findings = _empty_findings("Guardrail Topic Allowlist Check") + try: + guardrail_inv = require(inventory, "guardrails") + guardrails = guardrail_inv.summaries + + if not guardrails: + findings["csv_data"].append( + create_finding( + check_id="FS-59", + finding_name="No Guardrails — Topic Allowlist Not Applicable", + finding_details="No Bedrock Guardrails configured.", + resolution="Configure guardrails with topic policies to restrict off-topic responses.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-59"], + ) + ) + return findings + + guardrails_with_topics = [] + topics_classic_tier = [] + for g in guardrails: + detail = guardrail_inv.detail_by_id[g["id"]] + if isinstance(detail, _Unavailable): + raise detail.error + topic_policy = detail.get("topicPolicy", {}) + if topic_policy.get("topics"): + guardrails_with_topics.append(g["name"]) + tier = topic_policy.get("tier", {}).get("tierName", "CLASSIC") + if tier == "CLASSIC": + topics_classic_tier.append(g["name"]) + + if not guardrails_with_topics: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-59", + finding_name="No Guardrails With Topic Restrictions", + finding_details=( + f"Found {len(guardrails)} guardrail(s) but none have topic policies. " + "GenAI may respond to off-topic requests (e.g., medical advice, legal advice)." + ), + resolution=( + "Add denied topics to guardrails for off-topic categories:\n" + "- Medical/health advice\n" + "- Legal advice\n" + "- Political opinions\n" + "- Non-financial product recommendations\n" + "Consider the STANDARD tier (GA June 2025) for broader language support; " + "it requires cross-region inference on the guardrail." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-59"], + ) + ) + elif topics_classic_tier: + findings["csv_data"].append( + create_finding( + check_id="FS-59", + finding_name="Topic Restrictions Configured on CLASSIC Tier", + finding_details=( + f"Guardrails with topic policies: {', '.join(guardrails_with_topics)}. " + f"The following use the CLASSIC tier: {', '.join(topics_classic_tier)}. " + "CLASSIC tier supports English, French, and Spanish only; the STANDARD tier " + "(GA June 2025) adds broader language support for off-topic detection." + ), + resolution=( + "For multilingual FinServ deployments, consider upgrading denied topics to " + "the STANDARD tier (topicsTierConfig.tierName=STANDARD via UpdateGuardrail; " + "requires a cross-region inference profile)." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", + severity="Medium", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-59"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-59", + finding_name="Guardrail Topic Restrictions Configured", + finding_details=f"Guardrails with topic policies: {', '.join(guardrails_with_topics)}.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-components.html", + severity="Medium", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-59"], + ) + ) + except Exception as e: + return _error_findings("Guardrail Topic Allowlist Check", e) + return findings + + +def check_contextual_grounding_for_offtopic() -> Dict[str, Any]: + """ + FS-60 — Verify contextual grounding is used to keep GenAI responses + within the scope of the provided context/system prompt. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT] + """ + # This overlaps with FS-47/FS-48 but focuses on off-topic prevention + findings = _empty_findings("Contextual Grounding for Off-Topic Prevention") + findings["csv_data"].append( + create_finding( + check_id="FS-60", + finding_name="ADVISORY: Contextual Grounding for Off-Topic Prevention", + finding_details=( + "Contextual grounding for off-topic prevention is covered by guardrail " + "grounding checks (FS-47) and RAG configuration (FS-48). " + "Additionally verify system prompts explicitly scope the assistant's role." + ), + resolution=( + "1. Include explicit scope instructions in system prompts.\n" + "2. Use Bedrock Guardrails relevance grounding filter.\n" + "3. Test with off-topic prompts in QA to verify rejection behavior." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails-grounding.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-60"], + ) + ) + return findings + + +def check_knowledge_base_sync_schedule(inventory) -> Dict[str, Any]: + """ + FS-61 — Verify Bedrock Knowledge Base data sources have automated sync + schedules to keep training/retrieval data current. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT] + """ + # Reuses logic from FS-31 but focuses on scheduled automation + findings = _empty_findings("Knowledge Base Sync Schedule Check") + try: + kbs = require(inventory, "knowledge_bases").summaries + events = boto3.client("events", config=boto3_config) + scheduler = boto3.client("scheduler", config=boto3_config) + + if not kbs: + findings["csv_data"].append( + create_finding( + check_id="FS-61", + finding_name="No Knowledge Bases Found", + finding_details="No Bedrock Knowledge Bases found.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base-ingest.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-61"], + ) + ) + return findings + + # Check for EventBridge rules (legacy) that trigger KB sync. + rules = _paginate(events, "list_rules", "Rules") + kb_sync_rules = [ + r + for r in rules + if "bedrock" in r.get("Name", "").lower() + or "knowledge" in r.get("Name", "").lower() + ] + + # Check for EventBridge Scheduler schedules (the AWS-recommended approach; + # classic EventBridge scheduled rules are a legacy feature). The + # ListSchedules name-prefix filter is server-side, so do a broad list and + # match on name/target heuristics here. ListSchedules is paginated via + # NextToken. Treat an access error as a soft signal (do not fail the whole + # check) since Scheduler may not be in use. + kb_sync_schedules = [] + scheduler_access_error = None + try: + schedules = _paginate(scheduler, "list_schedules", "Schedules") + kb_sync_schedules = [ + s + for s in schedules + if "bedrock" in s.get("Name", "").lower() + or "knowledge" in s.get("Name", "").lower() + or "kb-sync" in s.get("Name", "").lower() + or "ingestion" in s.get("Name", "").lower() + or "bedrock" in s.get("Target", {}).get("Arn", "").lower() + ] + except ClientError as e: + if not _is_access_error(e): + raise + # Remember the access error. If we ALSO find no EventBridge-rule + # automation, we cannot conclude the control is absent (a Scheduler + # schedule we were not allowed to read might exist) — so we surface + # COULD_NOT_ASSESS rather than a false Failed. If a rule IS found, the + # positive evidence stands and the access gap is immaterial. + scheduler_access_error = e + logger.warning( + "Could not list EventBridge Scheduler schedules (access denied); " + "grant scheduler:ListSchedules for full FS-61 coverage." + ) + + total_sync_automation = len(kb_sync_rules) + len(kb_sync_schedules) + + # No positive evidence AND we were blocked from reading Scheduler → + # unknown state, not a confirmed failure. Re-raise so the handler emits a + # COULD NOT ASSESS row (Status="N/A", Severity="Low"). + if total_sync_automation == 0 and scheduler_access_error is not None: + raise scheduler_access_error + + if total_sync_automation == 0: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-61", + finding_name="No Automated KB Sync Schedules Detected", + finding_details=( + f"Found {len(kbs)} Knowledge Base(s) but no EventBridge Scheduler " + "schedules or EventBridge rules with 'bedrock'/'knowledge' naming were " + "found. Note: this check uses a name/target heuristic — sync automation " + "with other naming conventions, AWS Step Functions-based orchestration, " + "or native Bedrock API-triggered syncs (StartIngestionJob called directly) " + "will not be detected. Verify sync automation manually if applicable." + ), + resolution=( + "1. Use EventBridge Scheduler (the AWS-recommended approach) to create a " + "recurring schedule (e.g., rate(1 day) or a cron expression) that triggers a " + "Lambda function calling the Bedrock StartIngestionJob API for each data source. " + "Classic EventBridge scheduled rules also work but are a legacy feature.\n" + "2. As of December 2024, Bedrock Knowledge Bases supports custom connectors " + "and streaming data ingestion — use direct document ingestion " + "(KnowledgeBaseDocuments API) for real-time updates without a full S3 sync.\n" + "3. Set sync frequency based on data currency requirements " + "(e.g., hourly for market data, daily for regulatory guidance).\n" + "4. Configure CloudWatch alarms or SNS notifications on " + "IngestionJob FAILED status for sync failure alerting." + ), + reference="https://docs.aws.amazon.com/scheduler/latest/UserGuide/what-is-scheduler.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-61"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-61", + finding_name="Automated KB Sync Schedules Present", + finding_details=( + f"Found {len(kb_sync_schedules)} EventBridge Scheduler schedule(s) and " + f"{len(kb_sync_rules)} EventBridge rule(s) with KB-sync naming. " + "Verify each targets the Bedrock StartIngestionJob API for your KB data sources." + ), + resolution="Verify the schedule frequency matches your data-currency requirements.", + reference="https://docs.aws.amazon.com/scheduler/latest/UserGuide/what-is-scheduler.html", + severity="Medium", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-61"], + ) + ) + except Exception as e: + return _error_findings("Knowledge Base Sync Schedule Check", e) + return findings + + +def check_data_currency_disclaimer_advisory() -> Dict[str, Any]: + """ + FS-62 — Advisory check: verify application adds data currency disclaimers + to GenAI outputs (e.g., 'Information current as of [date]'). + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, MAS TRM 9.2] + """ + findings = _empty_findings("Data Currency Disclaimer Advisory") + findings["csv_data"].append( + create_finding( + check_id="FS-62", + finding_name="ADVISORY: Data Currency Disclaimer — Manual Review Required", + finding_details=( + "Data currency disclaimers cannot be verified via AWS APIs. " + "Manual review required." + ), + resolution=( + "1. Add data currency disclaimers to GenAI outputs: " + "'Information based on data current as of [KB last sync date].'\n" + "2. Expose KB last sync timestamp in application responses.\n" + "3. Alert users when KB data is older than defined threshold." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/knowledge-base-ingest.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-62"], + ) + ) + return findings + + +def check_foundation_model_lifecycle_policy() -> Dict[str, Any]: + """ + FS-63 — Check for a documented process to update foundation models when + new versions with more recent training data are released. + COMPLIANCE_PLACEHOLDER: [SR 11-7, FFIEC CAT, ISO 27001 A.12.5] + """ + findings = _empty_findings("Foundation Model Lifecycle Policy Check") + try: + bedrock = boto3.client("bedrock", config=boto3_config) + # Do not filter by output modality: legacy/deprecated models exist across + # TEXT, EMBEDDING, and IMAGE modalities. Embedding models are widely used + # in FinServ RAG pipelines; a legacy embedding model silently serves stale + # vector representations. Fetch all modalities to surface any deprecated model. + models = bedrock.list_foundation_models().get("modelSummaries", []) + + legacy_models = [ + m["modelId"] + for m in models + if m.get("modelLifecycle", {}).get("status") == "LEGACY" + ] + + # Check for Config rules or SSM documents related to model lifecycle + config_client = boto3.client("config", config=boto3_config) + rules = _paginate(config_client, "describe_config_rules", "ConfigRules") + lifecycle_rules = [ + r + for r in rules + if "lifecycle" in r.get("ConfigRuleName", "").lower() + or "model" in r.get("ConfigRuleName", "").lower() + ] + + if legacy_models and not lifecycle_rules: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-63", + finding_name="Legacy Models Without Lifecycle Management", + finding_details=( + f"Legacy foundation models available: {', '.join(legacy_models[:5])}. " + "No Config rules found for model lifecycle management." + ), + resolution=( + "1. Create a model lifecycle management process.\n" + "2. Subscribe to AWS Bedrock model deprecation notifications.\n" + "3. Test and migrate to new model versions before deprecation dates.\n" + "4. Document training data cutoff dates in model inventory." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-63"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-63", + finding_name="Foundation Model Lifecycle Management", + finding_details=( + f"No legacy models detected. " + f"{len(lifecycle_rules)} lifecycle-related Config rule(s) found." + ), + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-lifecycle.html", + severity="Medium", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-63"], + ) + ) + except Exception as e: + return _error_findings("Foundation Model Lifecycle Policy Check", e) + return findings + + +# =========================================================================== +# MATERIAL GAP CHECKS (FS-65 to FS-69) +# Mitigations explicitly in the AWS FinServ Guide not covered by FS-01..63 +# or the existing BR/SM/AC checks. +# NOTE: FS-64 (Guardrail Trace Logging) is merged into upstream BR-04. +# See extension note in SECURITY_CHECKS_FINSERV_PART3_APP_LAYER_AND_GAPS.md. +# =========================================================================== + + +def check_kb_datasource_s3_event_notifications(inventory) -> Dict[str, Any]: + """ + FS-65 — Check that S3 event notifications (EventBridge or SNS/SQS) are + configured on Knowledge Base data-source buckets to detect unauthorized + document changes in real time. + COMPLIANCE_PLACEHOLDER: [SR 11-7, ISO 27001 A.12, FFIEC CAT] + """ + findings = _empty_findings("KB Data Source S3 Event Notifications Check") + try: + kb_inv = require(inventory, "knowledge_bases") + kbs = kb_inv.summaries + s3_client = boto3.client("s3", config=boto3_config) + + if not kbs: + findings["csv_data"].append( + create_finding( + check_id="FS-65", + finding_name="No Knowledge Bases Found", + finding_details="No Bedrock Knowledge Bases found; S3 event notification check not applicable.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/NotificationHowTo.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-65"], + ) + ) + return findings + + buckets_without_notifications = [] + missing_buckets = [] + for kb in kbs: + kb_id = kb["knowledgeBaseId"] + data_sources = kb_inv.data_sources_by_kb.get(kb_id, []) + if isinstance(data_sources, _Unavailable): + raise data_sources.error + for ds in data_sources: + ds_id = ds["dataSourceId"] + ds_detail = kb_inv.data_source_detail.get((kb_id, ds_id)) + if isinstance(ds_detail, _Unavailable): + raise ds_detail.error + if ds_detail is None: + continue + s3_config = ( + ds_detail.get("dataSource", {}) + .get("dataSourceConfiguration", {}) + .get("s3Configuration", {}) + ) + bucket = _bucket_name_from_arn(s3_config.get("bucketArn", "")) + if not bucket: + continue + try: + notif = s3_client.get_bucket_notification_configuration( + Bucket=bucket + ) + has_notif = any( + [ + notif.get("TopicConfigurations"), + notif.get("QueueConfigurations"), + notif.get("LambdaFunctionConfigurations"), + notif.get("EventBridgeConfiguration"), + ] + ) + if not has_notif: + buckets_without_notifications.append(bucket) + except ClientError as e: + # An access error means we could not read the notification + # config; re-raise so it surfaces as could-not-assess rather + # than a false "missing notifications" finding. + if _is_access_error(e): + raise + # The data source points to a deleted bucket — a distinct + # integrity problem, not "notifications missing." + if _is_missing_bucket_error(e): + logger.warning( + f"KB '{kb.get('name', kb_id)}' data source " + f"'{ds.get('name', ds['dataSourceId'])}' references a " + f"deleted bucket: {bucket}" + ) + missing_buckets.append( + f"{bucket} (KB '{kb.get('name', kb_id)}', " + f"source '{ds.get('name', ds['dataSourceId'])}')" + ) + continue + buckets_without_notifications.append(f"{bucket} (error)") + + # A dangling data-source reference to a deleted bucket is a real integrity + # finding — emit it separately so it is not conflated with "no notifications." + if missing_buckets: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-65", + finding_name="KB Data Source References a Deleted S3 Bucket", + finding_details=( + "One or more Knowledge Base data sources point to S3 buckets that no " + "longer exist (NoSuchBucket). Document-change monitoring cannot be " + "assessed and KB grounding data is missing for these sources:\n" + + "\n".join(f"- {b}" for b in missing_buckets[:10]) + ), + resolution=( + "1. Investigate why the data-source bucket was deleted (accidental " + "deletion, environment teardown, or stale KB configuration).\n" + "2. Recreate/restore the bucket (with event notifications and versioning " + "enabled), or remove the orphaned data source from the Knowledge Base.\n" + "3. Re-run a KB ingestion job after restoring the data source." + ), + reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/EventBridge.html", + severity="High", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-65"], + ) + ) + + if buckets_without_notifications: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-65", + finding_name="KB Data Source Buckets Missing S3 Event Notifications", + finding_details=( + "The following KB data-source S3 buckets have no event notifications configured. " + "Unauthorized document modifications will not be detected in real time:\n" + + "\n".join( + f"- {b}" for b in buckets_without_notifications[:10] + ) + ), + resolution=( + "1. Enable Amazon EventBridge notifications on each KB data-source S3 bucket.\n" + "2. Create an EventBridge rule to route s3:ObjectCreated, s3:ObjectRemoved, " + "and s3:ObjectModified events to an SNS topic or Lambda for alerting.\n" + "3. Integrate alerts into your security incident response workflow." + ), + reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/EventBridge.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-65"], + ) + ) + elif not missing_buckets: + findings["csv_data"].append( + create_finding( + check_id="FS-65", + finding_name="KB Data Source S3 Event Notifications Configured", + finding_details="All KB data-source S3 buckets have event notifications configured.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/AmazonS3/latest/userguide/EventBridge.html", + severity="Medium", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-65"], + ) + ) + except Exception as e: + return _error_findings("KB Data Source S3 Event Notifications Check", e) + return findings + + +def check_agentcore_end_user_identity_propagation() -> Dict[str, Any]: + """ + FS-66 — Verify AgentCore runtimes are configured to propagate end-user + identities to downstream tool services so tool calls are authorized by + the originating user, not solely by the agent execution role. + COMPLIANCE_PLACEHOLDER: [SR 11-7, NYDFS 500.06, MAS TRM 9.1] + """ + findings = _empty_findings("AgentCore End-User Identity Propagation Check") + try: + agentcore = boto3.client("bedrock-agentcore-control", config=boto3_config) + try: + runtimes = agentcore.list_agent_runtimes().get("agentRuntimes", []) + except ClientError as e: + if "AccessDenied" in str(e) or "UnrecognizedClientException" in str(e): + findings["csv_data"].append( + create_finding( + check_id="FS-66", + finding_name="AgentCore Identity Propagation — Access Check", + finding_details="Unable to enumerate AgentCore runtimes (access denied or service unavailable in region).", + resolution="Ensure assessment role has bedrock-agentcore:ListAgentRuntimes permission.", + reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/security-authorization.html", + severity="Low", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-66"], + ) + ) + return findings + raise + + if not runtimes: + findings["csv_data"].append( + create_finding( + check_id="FS-66", + finding_name="No AgentCore Runtimes Found", + finding_details="No AgentCore runtimes found; identity propagation check not applicable.", + resolution=( + "If using AgentCore, configure token propagation so end-user identities " + "are forwarded to tool services." + ), + reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/security-authorization.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-66"], + ) + ) + return findings + + runtimes_without_identity = [ + r["agentRuntimeName"] + for r in runtimes + if not r.get("authorizerConfiguration", {}).get("customJWTAuthorizer") + and not r.get("authorizerConfiguration", {}).get("iamAuthorizer") + ] + + if runtimes_without_identity: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-66", + finding_name="AgentCore Runtimes Missing End-User Identity Propagation", + finding_details=( + "The following runtimes have no JWT or IAM authorizer configured for " + "end-user identity propagation. Tool calls are authorized only by the " + "agent execution role, not the originating user:\n" + + "\n".join(f"- {r}" for r in runtimes_without_identity[:10]) + ), + resolution=( + "1. Configure a custom JWT authorizer or IAM authorizer on each AgentCore runtime.\n" + "2. Propagate the end-user's identity token to downstream tool services.\n" + "3. Ensure tool services validate the propagated identity before executing actions.\n" + "4. Do not expose propagated identity tokens to unauthorized third parties." + ), + reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/security-authorization.html", + severity="High", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-66"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-66", + finding_name="AgentCore End-User Identity Propagation Configured", + finding_details=f"All {len(runtimes)} runtime(s) have authorizer configurations supporting identity propagation.", + resolution="No action required.", + reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/security-authorization.html", + severity="High", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-66"], + ) + ) + except Exception as e: + return _error_findings("AgentCore End-User Identity Propagation Check", e) + return findings + + +def check_agent_financial_transaction_thresholds(inventory) -> Dict[str, Any]: + """ + FS-67 — Check AgentCore Policy Engine or action-group Lambda functions + enforce maximum transaction-value limits to prevent runaway or unauthorized + high-value financial transactions initiated by agents. + COMPLIANCE_PLACEHOLDER: [SR 11-7, MAS TRM 9.1, FFIEC CAT, PCI-DSS] + """ + findings = _empty_findings("Agent Financial Transaction Value Thresholds Check") + try: + functions = require(inventory, "lambda_functions") + + # Look for agent action-group Lambda functions + action_group_lambdas = [ + f + for f in functions + if any( + kw in f["FunctionName"].lower() + for kw in [ + "agent", + "action", + "tool", + "bedrock", + "finserv", + "transaction", + ] + ) + ] + + if not action_group_lambdas: + findings["csv_data"].append( + create_finding( + check_id="FS-67", + finding_name="No Agent Action-Group Lambda Functions Found", + finding_details=( + "No Lambda functions matching agent action-group naming patterns found. " + "If agents perform financial transactions, verify transaction-value limits " + "are enforced in the action-group implementation." + ), + resolution=( + "1. Implement transaction-value threshold checks in all agent action-group " + "Lambda functions that initiate financial operations.\n" + "2. Use AgentCore Policy Engine to enforce maximum transaction amounts as " + "a policy constraint on tool calls.\n" + "3. Reject or escalate to human review any transaction exceeding defined limits." + ), + reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/policy-engine.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-67"], + ) + ) + else: + # Advisory: check for environment variables indicating threshold configuration + lambdas_without_threshold_config = [ + f["FunctionName"] + for f in action_group_lambdas + if not any( + "threshold" in k.lower() + or "limit" in k.lower() + or "max" in k.lower() + for k in f.get("Environment", {}).get("Variables", {}).keys() + ) + ] + + if lambdas_without_threshold_config: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-67", + finding_name="Agent Action-Group Lambdas May Lack Transaction Thresholds", + finding_details=( + "The following agent action-group Lambda functions have no environment " + "variables whose names suggest transaction-value threshold configuration " + "(this is a best-effort heuristic — a threshold enforced in code or in an " + "AgentCore Policy Engine rule would not be detected here, so treat this as " + "a prompt for manual verification rather than a definitive gap). " + "Without explicit limits, agents could initiate unbounded financial transactions:\n" + + "\n".join( + f"- {n}" for n in lambdas_without_threshold_config[:10] + ) + ), + resolution=( + "1. Add transaction-value threshold environment variables (e.g., MAX_TRANSACTION_AMOUNT) " + "to each agent action-group Lambda.\n" + "2. Implement threshold enforcement logic in the Lambda handler.\n" + "3. Configure AgentCore Policy Engine rules to cap financial transaction amounts.\n" + "4. Route transactions exceeding thresholds to a human-in-the-loop approval step." + ), + reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/policy-engine.html", + severity="High", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-67"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-67", + finding_name="Agent Action-Group Lambdas Have Threshold Configuration", + finding_details=( + f"Found {len(action_group_lambdas)} agent action-group Lambda(s) with " + "threshold/limit environment variables present." + ), + resolution="Verify threshold values are appropriate for your financial risk tolerance.", + reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/policy-engine.html", + severity="High", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-67"], + ) + ) + except Exception as e: + return _error_findings("Agent Financial Transaction Value Thresholds Check", e) + return findings + + +# Default body-inspection window AWS WAF applies before oversize handling kicks +# in. For CloudFront/API Gateway/Cognito/App Runner/Verified Access the default +# is 16 KB (raisable to 64 KB via the web ACL AssociationConfig); for ALB/AppSync +# it is a fixed 8 KB. A GT/GE SizeConstraint above this window can only ever fire +# if the rule's OversizeHandling is MATCH. See: +# https://docs.aws.amazon.com/waf/latest/developerguide/waf-oversize-request-components.html +_WAF_DEFAULT_BODY_INSPECTION_LIMIT = 16384 + +# JSON-Schema keywords that actually bound request-body SIZE. A request validator +# only enforces a size cap when its model schema carries one of these — merely +# enabling validateRequestBody does NOT cap payload size (it validates the schema +# / required params only; the REST hard limit is a fixed, non-configurable 10 MB). +_SCHEMA_SIZE_KEYWORDS = ('"maxLength"', '"maxItems"', '"maxProperties"') + + +def _waf_statement_has_firing_body_size_constraint( + stmt: Any, inspection_limit: int = _WAF_DEFAULT_BODY_INSPECTION_LIMIT +) -> bool: + """True if a WAF rule Statement contains a SizeConstraintStatement on the + request Body/JsonBody that can actually fire. + + Recurses into And/Or/Not combinators. A GT/GE size threshold above the body + inspection window cannot fire unless OversizeHandling is MATCH, so such a rule + is NOT credited as a working body-size control (the documented WAF limitation). + """ + if not isinstance(stmt, dict): + return False + sc = stmt.get("SizeConstraintStatement") + if isinstance(sc, dict): + ftm = sc.get("FieldToMatch", {}) or {} + body = ftm.get("Body") + json_body = ftm.get("JsonBody") + target = body if isinstance(body, dict) else json_body + if isinstance(target, dict): + comparison = sc.get("ComparisonOperator", "") + size = sc.get("Size", 0) or 0 + oversize = target.get("OversizeHandling") + # A "block if body > N" rule with N beyond the inspection window only + # fires when oversize content is treated as a match. + if ( + comparison in ("GT", "GE") + and size > inspection_limit + and oversize != "MATCH" + ): + return False + return True + for combinator in ("AndStatement", "OrStatement"): + sub = stmt.get(combinator) + if isinstance(sub, dict): + for inner in sub.get("Statements", []) or []: + if _waf_statement_has_firing_body_size_constraint( + inner, inspection_limit + ): + return True + not_stmt = stmt.get("NotStatement") + if isinstance(not_stmt, dict): + if _waf_statement_has_firing_body_size_constraint( + not_stmt.get("Statement"), inspection_limit + ): + return True + return False + + +def _api_has_body_size_validator(apigw, api_id: str) -> bool: + """True only if a REST API has a request validator that validates the body + AND at least one model whose schema bounds body size (maxLength/maxItems/ + maxProperties). Validator presence alone is NOT sufficient — it does not cap + payload size (the bug FS-68 previously had).""" + validators = apigw.get_request_validators(restApiId=api_id).get("items", []) + if not any(v.get("validateRequestBody") for v in validators): + return False + for model in _paginate(apigw, "get_models", "items", restApiId=api_id): + schema = model.get("schema", "") or "" + if any(tok in schema for tok in _SCHEMA_SIZE_KEYWORDS): + return True + return False + + +def check_api_gateway_request_body_size_limits(inventory) -> Dict[str, Any]: + """ + FS-68 — Verify API Gateway REST APIs fronting GenAI endpoints actually enforce + a maximum input-payload size (to blunt token-exhaustion via oversized prompts). + + Important correctness note: an API Gateway request validator does NOT cap body + size — it validates required params + a JSON-Schema model, and the REST payload + limit is a fixed, non-configurable 10 MB. A real size cap requires either (a) a + validator model with a maxLength/maxItems/maxProperties bound, or (b) a WAF + SizeConstraintStatement on the request Body that can actually fire within WAF's + body-inspection window (default 16 KB for API Gateway). This check credits only + those evidenced controls — validator presence alone is not a pass. + COMPLIANCE_PLACEHOLDER: [FFIEC CAT, DORA Art.6, PCI-DSS, OWASP LLM10] + """ + findings = _empty_findings("API Gateway Request Body Size Limits Check") + try: + apigw = boto3.client("apigateway", config=boto3_config) + + rest_apis = _paginate(apigw, "get_rest_apis", "items") + # require() raises if inventory is None or web_acls is _Unavailable. + web_acl_inv = require(inventory, "web_acls") + acls = web_acl_inv.summaries + + # Nothing-to-assess branch: no REST APIs AND no WAF ACLs means there is no + # input-payload surface to evaluate in this region. + if not rest_apis and not acls: + findings["csv_data"].append( + create_finding( + check_id="FS-68", + finding_name="API Gateway Request Body Size Limits — Not Applicable", + finding_details=( + "No API Gateway REST APIs and no regional WAF Web ACLs were found in this " + "region. There is no input-payload surface to assess for body-size limits." + ), + resolution=( + "If GenAI endpoints are fronted by API Gateway or WAF in another region, " + "run the assessment there. Otherwise no action is required." + ), + reference="https://docs.aws.amazon.com/waf/latest/developerguide/waf-rule-statement-type-size-constraint.html", + severity="Informational", + status="N/A", + compliance_frameworks=COMPLIANCE_MAP["FS-68"], + ) + ) + return findings + + # Evidence 1: REST APIs whose validator model actually bounds body size. + apis_with_size_control = [] + for api in rest_apis: + if _api_has_body_size_validator(apigw, api["id"]): + apis_with_size_control.append(api.get("name", api["id"])) + + # Evidence 2: regional WAF ACLs with a body SizeConstraint that can fire. + # detail_by_id holds get_web_acl(...)['WebACL'] or _Unavailable. + # An _Unavailable entry re-raises its error → outer except → COULD_NOT_ASSESS. + acls_with_size_rules = 0 + for acl in acls: + detail = web_acl_inv.detail_by_id[acl["Id"]] + if isinstance(detail, _Unavailable): + raise detail.error + rules = detail.get("Rules", []) + if any( + _waf_statement_has_firing_body_size_constraint(r.get("Statement", {})) + for r in rules + ): + acls_with_size_rules += 1 + + has_size_control = bool(apis_with_size_control) or acls_with_size_rules > 0 + + if has_size_control: + findings["csv_data"].append( + create_finding( + check_id="FS-68", + finding_name="API Gateway Request Body Size Limits Configured", + finding_details=( + f"Found {len(rest_apis)} REST API(s) " + f"({len(apis_with_size_control)} with a body-size-bounding validator model) " + f"and {acls_with_size_rules} WAF ACL(s) with a firing body-size constraint. " + "Verify the WAF ACL(s) are associated with the GenAI-facing API stages, " + "since this check does not confirm resource association." + ), + resolution="No action required (verify WAF/API association as noted).", + reference="https://docs.aws.amazon.com/waf/latest/developerguide/waf-rule-statement-type-size-constraint.html", + severity="Medium", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-68"], + ) + ) + else: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-68", + finding_name="API Gateway Request Body Size Limits Not Enforced", + finding_details=( + f"Found {len(rest_apis)} REST API(s) and {len(acls)} regional WAF Web " + "ACL(s), but none enforce a maximum request-body size. Note: an API " + "Gateway request validator does NOT cap body size (it validates the schema " + "and required params; the REST limit is a fixed 10 MB), and a WAF body " + "SizeConstraint only inspects the first ~16 KB of the body by default. " + "Oversized prompts can exhaust Bedrock token quotas and inflate costs." + ), + resolution=( + "1. Add a maxLength (or maxItems/maxProperties) bound to the request-body " + "JSON-Schema model used by your request validator, so oversized prompts are " + "rejected with a 400.\n" + "2. Add a WAF SizeConstraintStatement on the request Body sized within WAF's " + "body-inspection window (default 16 KB; raise via the web ACL " + "AssociationConfig, or set OversizeHandling=MATCH to block bodies beyond the " + "window), and associate the ACL with the API stage.\n" + "3. Set the max_tokens parameter in Bedrock API calls to cap output length.\n" + "4. Implement client-side token counting before submitting requests." + ), + reference="https://docs.aws.amazon.com/waf/latest/developerguide/waf-oversize-request-components.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-68"], + ) + ) + except Exception as e: + return _error_findings("API Gateway Request Body Size Limits Check", e) + return findings + + +def check_prompt_input_validation_function(inventory) -> Dict[str, Any]: + """ + FS-69 — Check for a Lambda function or API Gateway request validator that + sanitizes user prompt input (strips special characters, enforces expected + format, rejects oversized inputs) before forwarding to Bedrock. + COMPLIANCE_PLACEHOLDER: [OWASP LLM01, FFIEC CAT, NYDFS 500.06] + """ + findings = _empty_findings("Prompt Input Validation Function Check") + try: + functions = require(inventory, "lambda_functions") + + # Look for Lambda functions with input validation / sanitization naming patterns + VALIDATION_KEYWORDS = [ + "sanitiz", + "validat", + "input", + "preprocess", + "pre-process", + "filter", + "clean", + "prompt-guard", + "promptguard", + ] + validation_lambdas = [ + f["FunctionName"] + for f in functions + if any(kw in f["FunctionName"].lower() for kw in VALIDATION_KEYWORDS) + ] + + if not validation_lambdas: + findings["status"] = "WARN" + findings["csv_data"].append( + create_finding( + check_id="FS-69", + finding_name="No Prompt Input Validation Function Found", + finding_details=( + "No Lambda functions matching input validation or sanitization naming " + "patterns were found. Without explicit prompt input validation, malicious " + "inputs (special characters, oversized payloads, injection sequences) may " + "reach Bedrock unfiltered, bypassing WAF-level controls." + ), + resolution=( + "1. Implement a Lambda authorizer or pre-processing function that:\n" + " - Strips or escapes special characters from user input.\n" + " - Validates input against an expected format (e.g., regex allowlist).\n" + " - Rejects inputs exceeding maximum token/character limits.\n" + " - Logs rejected inputs for security monitoring.\n" + "2. Use parameterized prompt templates instead of string concatenation.\n" + "3. Apply Bedrock Guardrails PROMPT_ATTACK filter as a complementary control.\n" + "4. Reference: AWS Prompt Injection Security guidance." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-injection.html", + severity="Medium", + status="Failed", + compliance_frameworks=COMPLIANCE_MAP["FS-69"], + ) + ) + else: + findings["csv_data"].append( + create_finding( + check_id="FS-69", + finding_name="Prompt Input Validation Functions Present", + finding_details=( + f"Found {len(validation_lambdas)} Lambda function(s) with input " + f"validation/sanitization naming patterns: " + f"{', '.join(validation_lambdas[:5])}." + ), + resolution=( + "Review these functions to confirm they cover: special-character stripping, " + "format validation, size limits, and injection-sequence detection." + ), + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-injection.html", + severity="Medium", + status="Passed", + compliance_frameworks=COMPLIANCE_MAP["FS-69"], + ) + ) + except Exception as e: + return _error_findings("Prompt Input Validation Function Check", e) + return findings + + +# =========================================================================== +# REPORT GENERATION & LAMBDA HANDLER +# =========================================================================== + + +def generate_csv_report(findings: List[Dict[str, Any]]) -> str: + """Generate CSV report from all security check findings.""" + csv_buffer = StringIO() + fieldnames = [ + "Check_ID", + "Finding", + "Finding_Details", + "Resolution", + "Reference", + "Severity", + "Status", + "Compliance_Frameworks", + ] + writer = csv.DictWriter(csv_buffer, fieldnames=fieldnames) + writer.writeheader() + for finding in findings: + for row in finding.get("csv_data", []): + writer.writerow(row) + return csv_buffer.getvalue() + + +def write_to_s3(execution_id: str, csv_content: str, bucket_name: str) -> str: + """Write CSV report to S3 bucket.""" + s3_client = boto3.client("s3", config=boto3_config) + file_name = f"finserv_security_report_{execution_id}.csv" + s3_client.put_object( + Bucket=bucket_name, Key=file_name, Body=csv_content, ContentType="text/csv" + ) + return f"https://{bucket_name}.s3.amazonaws.com/{file_name}" + + +# --------------------------------------------------------------------------- +# Inventory collector (REQ-1, REQ-2, REQ-3, REQ-4, REQ-7) +# --------------------------------------------------------------------------- + + +def _safe_collect_lambda_functions(): + """Collect all Lambda functions via list_functions (fully paginated). + Returns a list on success, or _Unavailable(exc) on any failure.""" + try: + client = boto3.client("lambda", config=boto3_config) + return _paginate(client, "list_functions", "Functions") + except Exception as e: + logger.warning( + "inventory:lambda_functions collection failed: %s", type(e).__name__ + ) + return _Unavailable(e) + + +def _safe_collect_guardrails(): + """Collect Bedrock guardrail summaries and per-guardrail DRAFT detail. + A single-guardrail detail failure is recorded as _Unavailable for that id + only — it does NOT abort the whole guardrail inventory.""" + try: + client = boto3.client("bedrock", config=boto3_config) + summaries = _paginate(client, "list_guardrails", "guardrails") + detail_by_id: dict = {} + for g in summaries: + gid = g["id"] + try: + detail_by_id[gid] = client.get_guardrail( + guardrailIdentifier=gid, guardrailVersion="DRAFT" + ) + except Exception as e: + logger.warning( + "inventory:guardrails detail for %s failed: %s", + gid, + type(e).__name__, + ) + detail_by_id[gid] = _Unavailable(e) + return GuardrailInventory(summaries=summaries, detail_by_id=detail_by_id) + except Exception as e: + logger.warning("inventory:guardrails collection failed: %s", type(e).__name__) + return _Unavailable(e) + + +def _safe_collect_knowledge_bases(): + """Collect Bedrock Knowledge Base summaries, per-KB data-source summaries, + and per-data-source detail. Per-KB and per-data-source failures are + recorded as _Unavailable without aborting the rest of the collection.""" + try: + client = boto3.client("bedrock-agent", config=boto3_config) + summaries = _paginate(client, "list_knowledge_bases", "knowledgeBaseSummaries") + data_sources_by_kb: dict = {} + data_source_detail: dict = {} + for kb in summaries: + kb_id = kb["knowledgeBaseId"] + try: + ds_summaries = _paginate( + client, + "list_data_sources", + "dataSourceSummaries", + knowledgeBaseId=kb_id, + ) + data_sources_by_kb[kb_id] = ds_summaries + for ds in ds_summaries: + ds_id = ds["dataSourceId"] + try: + data_source_detail[(kb_id, ds_id)] = client.get_data_source( + knowledgeBaseId=kb_id, dataSourceId=ds_id + ) + except Exception as e: + logger.warning( + "inventory:knowledge_bases data_source detail (%s, %s) failed: %s", + kb_id, + ds_id, + type(e).__name__, + ) + data_source_detail[(kb_id, ds_id)] = _Unavailable(e) + except Exception as e: + logger.warning( + "inventory:knowledge_bases data_sources for KB %s failed: %s", + kb_id, + type(e).__name__, + ) + data_sources_by_kb[kb_id] = _Unavailable(e) + return KbInventory( + summaries=summaries, + data_sources_by_kb=data_sources_by_kb, + data_source_detail=data_source_detail, + ) + except Exception as e: + logger.warning( + "inventory:knowledge_bases collection failed: %s", type(e).__name__ + ) + return _Unavailable(e) + + +def _safe_collect_buckets(): + """Collect the full account S3 bucket list with explicit ContinuationToken + pagination. MaxBuckets=1000 ensures pagination is always engaged so accounts + above the 10,000-bucket quota (where unpaginated requests are rejected) succeed.""" + try: + client = boto3.client("s3", config=boto3_config) + return _paginate( + client, + "list_buckets", + "Buckets", + token=("ContinuationToken", "ContinuationToken"), + MaxBuckets=1000, + ) + except Exception as e: + logger.warning("inventory:buckets collection failed: %s", type(e).__name__) + return _Unavailable(e) + + +def _safe_collect_web_acls(): + """Collect WAFv2 REGIONAL Web ACL summaries and per-ACL detail. Uses + explicit NextMarker/NextMarker pagination (WAFv2 input ≠ Lambda Marker). + Per-ACL detail failures are recorded as _Unavailable for that id only.""" + try: + client = boto3.client("wafv2", config=boto3_config) + summaries = _paginate( + client, + "list_web_acls", + "WebACLs", + token=("NextMarker", "NextMarker"), + Scope="REGIONAL", + ) + detail_by_id: dict = {} + for acl in summaries: + acl_id = acl["Id"] + try: + # Store the WebACL dict directly (response["WebACL"]), not the + # full envelope, so consuming checks can do detail.get("Rules") + # without an extra ["WebACL"] indirection (matches design §3). + resp = client.get_web_acl(Name=acl["Name"], Scope="REGIONAL", Id=acl_id) + detail_by_id[acl_id] = resp["WebACL"] + except Exception as e: + logger.warning( + "inventory:web_acls detail for %s failed: %s", + acl_id, + type(e).__name__, + ) + detail_by_id[acl_id] = _Unavailable(e) + return WebAclInventory(summaries=summaries, detail_by_id=detail_by_id) + except Exception as e: + logger.warning("inventory:web_acls collection failed: %s", type(e).__name__) + return _Unavailable(e) + + +def collect_resource_inventory() -> ResourceInventory: + """Collect each shared inventory at most once per invocation. + + Each inventory is isolated: a failure yields an ``_Unavailable`` sentinel + for that field without aborting the others (REQ-4, INV-5). All five + inventories are always collected because the current registry always + consumes all five (design DD-7). + + All clients use ``boto3.client(service, config=boto3_config)`` — no + ``region_name`` or ``endpoint_url`` — so the Lambda's default-Region + resolution is preserved (REQ-7, INV-4).""" + return ResourceInventory( + lambda_functions=_safe_collect_lambda_functions(), + guardrails=_safe_collect_guardrails(), + knowledge_bases=_safe_collect_knowledge_bases(), + buckets=_safe_collect_buckets(), + web_acls=_safe_collect_web_acls(), + ) + + +def build_finserv_checks(permission_cache, inventory=None): + """ + Single source of truth: ordered (check_id, zero-arg callable) registry of + all FinServ checks, in execution order (FS-01 → FS-69, skipping the ids + merged into upstream: FS-17/18/19/23/64). The two permission-cache checks + are bound with functools.partial so every entry is uniformly zero-arg. + + Driving the handler from this registry lets us attach the correct Check_ID + to a synthesized "could not assess" row when a check errors out, instead of + silently dropping the check from the report. + + ``inventory`` is optional (defaults to ``None``) so that existing one-arg + call sites — e.g. tests/test_severity_register.py — continue to work + without modification (DD-2b). ``lambda_handler`` always passes a real + ``ResourceInventory``; consuming checks will be bound to it in Wave 3. + FS-21 and FS-46 (S3-inventory consumers) are bound to ``inventory`` here. + Guardrail consumers (FS-27a, 28, 36, 38, 45, 47, 50, 51, 59) are now + bound to ``inventory`` as well (Task 7). + """ + return [ + # --- Category 1: Unbounded Consumption --- + ("FS-01", functools.partial(check_waf_shield_on_bedrock_endpoints, inventory)), + ("FS-02", check_api_gateway_rate_limiting), + ("FS-03", check_bedrock_token_quotas), + ("FS-04", check_cost_anomaly_detection), + ("FS-05", check_cloudwatch_token_alarms), + ("FS-06", check_aws_budgets_for_aiml), + # --- Category 2: Excessive Agency --- + ( + "FS-07", + functools.partial(check_bedrock_agent_action_boundaries, permission_cache), + ), + ("FS-08", check_agentcore_policy_engine), + ("FS-09", functools.partial(check_agent_transaction_limits, inventory)), + ("FS-10", check_human_in_the_loop_for_high_risk_actions), + ("FS-11", check_agent_rate_alarms), + # --- Category 3: Supply Chain Vulnerabilities --- + ("FS-12", check_scp_model_access_restrictions), + ("FS-13", check_model_inventory_tagging), + ("FS-14", check_model_onboarding_governance), + ("FS-15", check_bedrock_model_evaluation_adversarial), + ("FS-16", check_ecr_image_scanning), + # --- Category 4: Training Data & Model Poisoning --- + ("FS-20", check_feature_store_rollback_capability), + ("FS-21", functools.partial(check_training_data_s3_versioning, inventory)), + # --- Category 5: Vector & Embedding Weaknesses --- + ( + "FS-22", + functools.partial( + check_knowledge_base_iam_least_privilege, permission_cache + ), + ), + ( + "FS-24", + functools.partial(check_knowledge_base_metadata_filtering, inventory), + ), + ("FS-25", check_opensearch_serverless_encryption), + ("FS-26", check_knowledge_base_vpc_access), + # --- Category 6: Non-Compliant Output --- + # FS-27 is split into two checks: contextual grounding (threshold-based) and + # Automated Reasoning policies (formal-verification, GA August 2025). Both + # use the FS-27 check_id so they appear together in the CSV report. + ("FS-27", functools.partial(check_guardrail_contextual_grounding, inventory)), + ("FS-27", check_automated_reasoning_policies), + ( + "FS-28", + functools.partial(check_guardrail_denied_topics_financial, inventory), + ), + ("FS-29", check_compliance_disclaimer_in_outputs), + ("FS-30", check_bedrock_evaluation_compliance_datasets), + # --- Category 7: Misinformation --- + ("FS-31", functools.partial(check_knowledge_base_data_source_sync, inventory)), + ("FS-32", check_source_attribution_in_guardrails), + ( + "FS-33", + functools.partial(check_knowledge_base_integrity_monitoring, inventory), + ), + ("FS-34", check_fm_version_currency), + # --- Category 8: Abusive or Harmful Output --- + ("FS-35", check_fmeval_harmful_content), + ("FS-36", functools.partial(check_guardrail_content_filters, inventory)), + ("FS-37", check_user_feedback_mechanism), + ("FS-38", functools.partial(check_guardrail_word_filters, inventory)), + # --- Category 9: Biased Output --- + ("FS-39", check_sagemaker_clarify_bias), + ("FS-40", check_bedrock_evaluation_bias_datasets), + ("FS-41", check_sagemaker_clarify_explainability), + ("FS-42", check_ai_service_cards_documentation), + # --- Category 10: Sensitive Information Disclosure --- + ("FS-43", check_cloudwatch_log_pii_masking), + ("FS-44", check_macie_on_training_data_buckets), + ("FS-45", functools.partial(check_guardrail_pii_filters, inventory)), + ("FS-46", functools.partial(check_data_classification_tagging, inventory)), + # --- Category 11: Hallucination --- + ("FS-47", functools.partial(check_guardrail_grounding_threshold, inventory)), + ("FS-48", functools.partial(check_rag_knowledge_base_configured, inventory)), + ("FS-49", check_hallucination_disclaimer_advisory), + ("FS-50", functools.partial(check_guardrail_relevance_grounding, inventory)), + # --- Category 12: Prompt Injection --- + ( + "FS-51", + functools.partial(check_prompt_injection_input_validation, inventory), + ), + ("FS-52", functools.partial(check_bedrock_sdk_version_currency, inventory)), + ("FS-53", functools.partial(check_waf_sql_injection_rules, inventory)), + ("FS-54", check_penetration_testing_evidence), + # --- Category 13: Improper Output Handling --- + ("FS-55", functools.partial(check_output_validation_lambda, inventory)), + ("FS-56", functools.partial(check_xss_prevention_waf, inventory)), + ("FS-57", check_output_encoding_advisory), + ("FS-58", functools.partial(check_output_schema_validation, inventory)), + # --- Category 14: Off-Topic & Inappropriate Output --- + ("FS-59", functools.partial(check_guardrail_topic_allowlist, inventory)), + ("FS-60", check_contextual_grounding_for_offtopic), + # --- Category 15: Out-of-Date Training Data --- + ("FS-61", functools.partial(check_knowledge_base_sync_schedule, inventory)), + ("FS-62", check_data_currency_disclaimer_advisory), + ("FS-63", check_foundation_model_lifecycle_policy), + # --- Material Gap Checks (FS-65 to FS-69) --- + ( + "FS-65", + functools.partial(check_kb_datasource_s3_event_notifications, inventory), + ), + ("FS-66", check_agentcore_end_user_identity_propagation), + ( + "FS-67", + functools.partial(check_agent_financial_transaction_thresholds, inventory), + ), + ( + "FS-68", + functools.partial(check_api_gateway_request_body_size_limits, inventory), + ), + ("FS-69", functools.partial(check_prompt_input_validation_function, inventory)), + ] + + +def lambda_handler(event, context): + """Main Lambda handler — runs all FinServ security checks. + + The registry in build_finserv_checks() contains 65 entries (64 standalone + FS checks plus the new FS-27 Automated Reasoning Policies check). Two + entries share the FS-27 check_id (contextual grounding + ARC policies), + both contributing rows to the report under the same check namespace. + """ + logger.info("Starting FinServ GenAI security assessment") + all_findings = [] + + execution_id = event.get("Execution", {}).get("Name", "local-test") + permission_cache = get_permissions_cache(execution_id) or { + "role_permissions": {}, + "user_permissions": {}, + } + inventory = collect_resource_inventory() # NEW: once per invocation + + # Run every check from the registry. If a check produces no rows for ANY + # reason (an ERROR envelope, or an unexpected empty non-error result), + # synthesize a visible "could not assess" row (keyed by its Check_ID) so the + # gap surfaces in the report instead of the check silently vanishing. The + # guard intentionally keys off empty csv_data (not just status=="ERROR") so + # the no-silent-drop invariant holds structurally, not by data coincidence. + for check_id, check_fn in build_finserv_checks(permission_cache, inventory): + result = check_fn() + if not result.get("csv_data"): + details = result.get("details", "") or ( + f"check returned status={result.get('status', 'UNKNOWN')!r} " + "with no findings" + ) + result.setdefault("csv_data", []).append( + _could_not_assess_row( + check_id, + result.get("check_name", check_id), + details, + ) + ) + all_findings.append(result) + + # Generate and upload report + csv_content = generate_csv_report(all_findings) + bucket_name = os.environ.get("AIML_ASSESSMENT_BUCKET_NAME") + if not bucket_name: + raise ValueError("AIML_ASSESSMENT_BUCKET_NAME environment variable is not set") + + s3_url = write_to_s3(execution_id, csv_content, bucket_name) + + return { + "statusCode": 200, + "body": { + "message": "FinServ security assessment completed", + "findings": all_findings, + "report_url": s3_url, + }, + } diff --git a/aiml-security-assessment/functions/security/finserv_assessments/requirements.txt b/aiml-security-assessment/functions/security/finserv_assessments/requirements.txt new file mode 100644 index 0000000..183ad4f --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_assessments/requirements.txt @@ -0,0 +1,13 @@ +# FinServ Security Assessment Lambda — runtime dependencies +# +# These pins are scanned by ASH (Automated Security Helper) via Grype + Syft +# during the pre-PR security scan. Keep versions current; if ASH reports a +# CVE against any pinned dep, bump to the patched version before opening the +# PR. See GIT_WORKFLOW.md Step 5 for the ASH workflow. +# +# Dev tooling (ruff, cfn-lint, ash, sam) is NOT pinned here — install those +# in your development environment, not into the Lambda image. + +boto3>=1.43.21 +botocore>=1.43.21 +pydantic>=2.0.0,<3.0.0 diff --git a/aiml-security-assessment/functions/security/finserv_assessments/schema.py b/aiml-security-assessment/functions/security/finserv_assessments/schema.py new file mode 100644 index 0000000..657ad66 --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_assessments/schema.py @@ -0,0 +1,99 @@ +""" +Schema module for FinServ security findings. +Mirrors the schema used in bedrock_assessments/schema.py. +""" + +from enum import Enum +from typing import Dict, Any, Optional +from pydantic import BaseModel, Field, field_validator +import re + + +class SeverityEnum(str, Enum): + HIGH = "High" + MEDIUM = "Medium" + LOW = "Low" + INFORMATIONAL = "Informational" + + +class StatusEnum(str, Enum): + FAILED = "Failed" + PASSED = "Passed" + NA = "N/A" + + +class Finding(BaseModel): + """Represents a security finding with required fields and validations.""" + + Check_ID: str = Field( + ..., + min_length=1, + description="Unique check identifier (e.g., FS-01)", + ) + Finding: str = Field(..., min_length=1, description="The name/title of the finding") + Finding_Details: str = Field( + ..., min_length=1, description="Detailed description of the finding" + ) + Resolution: str = Field( + ..., min_length=0, description="Steps to resolve the finding" + ) + Reference: str = Field(..., description="Documentation reference URL") + Severity: SeverityEnum = Field(..., description="Severity level of the finding") + Status: StatusEnum = Field(..., description="Current status of the finding") + Compliance_Frameworks: str = Field( + default="", + description=( + "Pipe-separated list of FinServ regulatory frameworks this control maps to " + "(e.g., 'FFIEC CAT | SR 11-7 | NYDFS 500'). Preliminary; validate with your " + "MRM/Legal/Compliance teams before using as audit evidence." + ), + ) + + @field_validator("Check_ID") + @classmethod + def validate_check_id(cls, v): + # Allow FS-NN pattern for FinServ checks + pattern = r"^[A-Z]{2,3}-\d{2}$" + if not re.match(pattern, v): + raise ValueError( + "Check_ID must follow pattern XX-NN (e.g., FS-01, BR-14, AC-05)" + ) + return v + + @field_validator("Reference") + @classmethod + def validate_reference_url(cls, v): + if not str(v).startswith("https://"): + raise ValueError("Reference URL must start with https://") + return v + + +def create_finding( + check_id: str, + finding_name: str, + finding_details: str, + resolution: str, + reference: str, + severity: SeverityEnum, + status: StatusEnum, + compliance_frameworks: Optional[str] = "", +) -> Dict[str, Any]: + """Create a validated finding dict. + + Args: + compliance_frameworks: Pipe-separated FinServ regulatory framework identifiers + (e.g., "FFIEC CAT | SR 11-7 | NYDFS 500"). Populated from COMPLIANCE_MAP + in app.py. Preliminary mappings — validate with MRM/Legal/Compliance before + using as audit evidence. + """ + finding = Finding( + Check_ID=check_id, + Finding=finding_name, + Finding_Details=finding_details, + Resolution=resolution, + Reference=reference, + Severity=severity, + Status=status, + Compliance_Frameworks=compliance_frameworks or "", + ) + return dict(finding.model_dump()) diff --git a/aiml-security-assessment/functions/security/finserv_tests/__init__.py b/aiml-security-assessment/functions/security/finserv_tests/__init__.py new file mode 100644 index 0000000..65140f2 --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_tests/__init__.py @@ -0,0 +1 @@ +# tests package diff --git a/aiml-security-assessment/functions/security/finserv_tests/conftest.py b/aiml-security-assessment/functions/security/finserv_tests/conftest.py new file mode 100644 index 0000000..7107ac3 --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_tests/conftest.py @@ -0,0 +1,135 @@ +""" +Shared fixtures and mock helpers for finserv_assessments tests. + +All boto3 clients are patched at the module level so check functions +never make real AWS API calls. +""" + +import os +import sys + +import pytest + +# --------------------------------------------------------------------------- +# Make finserv_assessments importable — the Lambda runtime adds the package +# root to sys.path, so we replicate that here. +# --------------------------------------------------------------------------- +FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") +if FINSERV_DIR not in sys.path: + sys.path.insert(0, FINSERV_DIR) + +import app # noqa: E402 (imported after sys.path manipulation) + + +# --------------------------------------------------------------------------- +# Environment variables expected by the Lambda +# --------------------------------------------------------------------------- +@pytest.fixture(autouse=True) +def _set_env(monkeypatch): + monkeypatch.setenv("AIML_ASSESSMENT_BUCKET_NAME", "test-bucket") + + +# --------------------------------------------------------------------------- +# A reusable permission_cache fixture +# --------------------------------------------------------------------------- +@pytest.fixture +def permission_cache_empty(): + return {"role_permissions": {}, "user_permissions": {}} + + +@pytest.fixture +def permission_cache_with_wildcard(): + """Permission cache where an agent role has iam:* — triggers FS-07 WARN.""" + return { + "role_permissions": { + "BedrockAgentRole": { + "attached_policies": [ + { + "document": { + "Statement": [ + { + "Effect": "Allow", + "Action": "iam:*", + "Resource": "*", + } + ] + } + } + ], + "inline_policies": [], + } + }, + "user_permissions": {}, + } + + +@pytest.fixture +def permission_cache_safe(): + """Permission cache where agent role has narrow permissions — triggers FS-07 PASS.""" + return { + "role_permissions": { + "BedrockAgentRole": { + "attached_policies": [ + { + "document": { + "Statement": [ + { + "Effect": "Allow", + "Action": "bedrock:InvokeModel", + "Resource": "arn:aws:bedrock:*:*:model/*", + } + ] + } + } + ], + "inline_policies": [], + } + }, + "user_permissions": {}, + } + + +# --------------------------------------------------------------------------- +# Synthetic Lambda event +# --------------------------------------------------------------------------- +@pytest.fixture +def lambda_event(): + return { + "Execution": {"Name": "unit-test-001"}, + "StateMachine": { + "Id": "arn:aws:states:us-east-1:123456789012:stateMachine:test" + }, + } + + +# --------------------------------------------------------------------------- +# ResourceInventory test builder (REQ-6.4, REQ-9.3) +# --------------------------------------------------------------------------- + + +def make_resource_inventory(**overrides) -> app.ResourceInventory: + """Build a fully-available ``ResourceInventory`` with sensible empty defaults. + + Any field can be replaced via keyword arguments. Pass an + ``app._Unavailable(exc)`` value to simulate a per-inventory collection + failure. + + Examples:: + + inv = make_resource_inventory() # fully available + inv = make_resource_inventory(lambda_functions=[...]) # real data + inv = make_resource_inventory( + guardrails=app._Unavailable(PermissionError("AccessDenied")) + ) # failed field + """ + defaults: dict = dict( + lambda_functions=[], + guardrails=app.GuardrailInventory(summaries=[], detail_by_id={}), + knowledge_bases=app.KbInventory( + summaries=[], data_sources_by_kb={}, data_source_detail={} + ), + buckets=[], + web_acls=app.WebAclInventory(summaries=[], detail_by_id={}), + ) + defaults.update(overrides) + return app.ResourceInventory(**defaults) diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_at_most_once.py b/aiml-security-assessment/functions/security/finserv_tests/test_at_most_once.py new file mode 100644 index 0000000..dd1a750 --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_tests/test_at_most_once.py @@ -0,0 +1,813 @@ +""" +"At most once" enforcement harness — Task 11 + +Handler-level tests with counting mocks that assert each shared listing API is +called ≤ 1 per run, each detail API ≤ 1 per resource per run, and +ListDataSources ≤ 1 per KB. The tests fail the build if any invariant is +exceeded. + +Requirements: REQ-9.1, REQ-9.4, REQ-9.6 + +Design reference: design.md §9.3 + list_functions ≤ 1 + list_guardrails ≤ 1 + list_knowledge_bases ≤ 1 + list_buckets ≤ 1 + list_web_acls ≤ 1 + list_data_sources ≤ 1 per KB + get_guardrail ≤ 1 per distinct guardrail id + get_web_acl ≤ 1 per distinct ACL id + get_data_source ≤ 1 per (kb_id, ds_id) pair +""" + +from __future__ import annotations + +import os +import sys +from collections import defaultdict +from datetime import datetime, timezone +from unittest.mock import MagicMock, patch + +import pytest + +# --------------------------------------------------------------------------- +# Make finserv_assessments importable +# --------------------------------------------------------------------------- +FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") +if FINSERV_DIR not in sys.path: + sys.path.insert(0, FINSERV_DIR) + +import app # noqa: E402 + + +# =========================================================================== +# Call-counting mock infrastructure +# =========================================================================== + + +class _CountingClient: + """Thin wrapper around a MagicMock that counts calls to specific methods. + + For listing APIs we track the total number of calls (must be ≤ 1). + For per-resource detail APIs we track calls per argument key (must be ≤ 1 + per distinct resource). + """ + + def __init__(self, service: str): + self.service = service + self.list_call_counts: dict[str, int] = defaultdict(int) + self.detail_call_counts: dict[str, dict] = defaultdict(lambda: defaultdict(int)) + self._mock = MagicMock() + + # ------------------------------------------------------------------ + # Wire counted list + detail methods and delegate rest to the mock + # ------------------------------------------------------------------ + + def __getattr__(self, name: str): + return getattr(self._mock, name) + + +# --------------------------------------------------------------------------- +# Reusable "full account state" data (same as test_inventory_equivalence.py +# but kept local so this module has no import dependency on that module). +# --------------------------------------------------------------------------- + +_NOW = datetime.now(timezone.utc) + +_LAMBDA_FUNCTIONS = [ + {"FunctionName": "my-bedrock-agent-handler"}, + {"FunctionName": "my-output-validate-fn"}, + {"FunctionName": "my-schema-validator-fn"}, +] + +_GUARDRAIL_SUMMARIES = [ + {"id": "gr-001", "name": "GuardrailA"}, + {"id": "gr-002", "name": "GuardrailB"}, +] + +_GUARDRAIL_DETAIL = { + "contextualGroundingPolicy": { + "filters": [ + {"type": "GROUNDING", "threshold": 0.8}, + {"type": "RELEVANCE", "threshold": 0.8}, + ] + }, + "topicPolicy": { + "topics": [{"name": "investment-advice", "type": "DENY"}], + "tier": {"tierName": "CLASSIC"}, + }, + "contentPolicy": { + "filters": [ + {"type": "HATE", "inputStrength": "HIGH", "outputStrength": "HIGH"}, + ] + }, + "wordPolicy": { + "words": [{"text": "fraud"}], + "managedWordLists": [{"type": "PROFANITY"}], + }, + "sensitiveInformationPolicy": { + "piiEntities": [{"type": "US_SOCIAL_SECURITY_NUMBER", "action": "BLOCK"}] + }, +} + +_KB_SUMMARIES = [ + {"knowledgeBaseId": "kb-001", "name": "KB_Alpha"}, + {"knowledgeBaseId": "kb-002", "name": "KB_Beta"}, +] + +_KB_DATA_SOURCE_SUMMARIES = { + "kb-001": [ + {"dataSourceId": "ds-001", "name": "DS_1", "updatedAt": _NOW}, + {"dataSourceId": "ds-002", "name": "DS_2", "updatedAt": _NOW}, + ], + "kb-002": [ + {"dataSourceId": "ds-003", "name": "DS_3", "updatedAt": _NOW}, + ], +} + +_S3_BUCKETS = [ + {"Name": "my-training-dataset-bucket"}, + {"Name": "my-bedrock-kb-bucket"}, + {"Name": "my-kb-datasource-bucket"}, +] + +_ACL_SUMMARIES = [ + {"Name": "ACL_One", "Id": "acl-id-001", "ARN": "arn:aws:wafv2:::acl-id-001"}, + {"Name": "ACL_Two", "Id": "acl-id-002", "ARN": "arn:aws:wafv2:::acl-id-002"}, +] + +_ACL_DETAIL_TEMPLATE = { + "Rules": [ + { + "Name": "SQLi", + "Statement": { + "ManagedRuleGroupStatement": {"Name": "AWSManagedRulesSQLiRuleSet"} + }, + }, + { + "Name": "Common", + "Statement": { + "ManagedRuleGroupStatement": {"Name": "AWSManagedRulesCommonRuleSet"} + }, + }, + { + "Name": "KnownBadInputs", + "Statement": { + "ManagedRuleGroupStatement": { + "Name": "AWSManagedRulesKnownBadInputsRuleSet" + } + }, + }, + { + "Name": "SizeConstraint", + "Statement": { + "SizeConstraintStatement": { + "FieldToMatch": {"Body": {}}, + "ComparisonOperator": "LE", + "Size": 8192, + "TextTransformations": [{"Priority": 0, "Type": "NONE"}], + } + }, + }, + ], +} + + +# =========================================================================== +# Counting mock client factory +# =========================================================================== + + +def _build_counting_client_factory(): + """Return a (side_effect_fn, call_tracker) pair. + + side_effect_fn: passed to mock_client.side_effect — returns a client mock + whose inventory-relevant methods are wrapped with call counters. + call_tracker: a dict containing all recorded call counts so tests can + assert at-most-once invariants after lambda_handler returns. + + Structure of call_tracker:: + + { + "list_functions": , # total calls + "list_guardrails": , + "list_knowledge_bases": , + "list_buckets": , + "list_web_acls": , + "list_data_sources": {kb_id: , ...}, # per-KB + "get_guardrail": {guardrail_id: , ...}, # per guardrail id + "get_web_acl": {acl_id: , ...}, # per ACL id + "get_data_source": {(kb_id, ds_id): , ...}, # per (kb, ds) + } + """ + tracker: dict = { + "list_functions": 0, + "list_guardrails": 0, + "list_knowledge_bases": 0, + "list_buckets": 0, + "list_web_acls": 0, + "list_data_sources": defaultdict(int), + "get_guardrail": defaultdict(int), + "get_web_acl": defaultdict(int), + "get_data_source": defaultdict(int), + } + + def side_effect(service, **kwargs): # noqa: C901 + c = MagicMock() + + # ------------------------------------------------------------------ # + # wafv2 — count list_web_acls and get_web_acl calls + # ------------------------------------------------------------------ # + if service == "wafv2": + + def _list_web_acls(Scope, **kw): + tracker["list_web_acls"] += 1 + return {"WebACLs": _ACL_SUMMARIES} + + c.list_web_acls.side_effect = _list_web_acls + + def _get_web_acl(Name, Scope, Id, **kw): + tracker["get_web_acl"][Id] += 1 + detail = dict(_ACL_DETAIL_TEMPLATE) + detail["Name"] = Name + detail["Id"] = Id + return {"WebACL": detail} + + c.get_web_acl.side_effect = _get_web_acl + return c + + # ------------------------------------------------------------------ # + # lambda — count list_functions + # ------------------------------------------------------------------ # + if service == "lambda": + + def _list_functions(**kw): + tracker["list_functions"] += 1 + return {"Functions": _LAMBDA_FUNCTIONS} + + c.list_functions.side_effect = _list_functions + + def _get_function_concurrency(FunctionName, **kw): + if "agent" in FunctionName.lower(): + return {"ReservedConcurrentExecutions": 5} + return {} + + c.get_function_concurrency.side_effect = _get_function_concurrency + return c + + # ------------------------------------------------------------------ # + # bedrock — count list_guardrails and get_guardrail per id + # ------------------------------------------------------------------ # + if service == "bedrock": + + def _list_guardrails(**kw): + tracker["list_guardrails"] += 1 + return {"guardrails": _GUARDRAIL_SUMMARIES} + + c.list_guardrails.side_effect = _list_guardrails + + def _get_guardrail(guardrailIdentifier, guardrailVersion="DRAFT", **kw): + tracker["get_guardrail"][guardrailIdentifier] += 1 + return _GUARDRAIL_DETAIL + + c.get_guardrail.side_effect = _get_guardrail + + # Non-inventory calls for other bedrock checks + c.list_foundation_models.return_value = { + "modelSummaries": [ + { + "modelId": "anthropic.claude-3-sonnet-20240229-v1:0", + "modelLifecycle": {"status": "ACTIVE"}, + } + ] + } + c.list_custom_models.return_value = {"modelSummaries": []} + c.list_model_cards.return_value = {"ModelCardSummaries": []} + c.list_evaluation_jobs.return_value = {"jobSummaries": []} + c.list_automated_reasoning_policies.return_value = { + "automatedReasoningPolicySummaries": [] + } + return c + + # ------------------------------------------------------------------ # + # bedrock-agent — count list_knowledge_bases, list_data_sources (per KB), + # get_data_source (per (kb, ds)) + # ------------------------------------------------------------------ # + if service == "bedrock-agent": + + def _list_knowledge_bases(**kw): + tracker["list_knowledge_bases"] += 1 + return {"knowledgeBaseSummaries": _KB_SUMMARIES} + + c.list_knowledge_bases.side_effect = _list_knowledge_bases + + def _list_data_sources(knowledgeBaseId, **kw): + tracker["list_data_sources"][knowledgeBaseId] += 1 + return { + "dataSourceSummaries": _KB_DATA_SOURCE_SUMMARIES.get( + knowledgeBaseId, [] + ) + } + + c.list_data_sources.side_effect = _list_data_sources + + def _get_data_source(knowledgeBaseId, dataSourceId, **kw): + tracker["get_data_source"][(knowledgeBaseId, dataSourceId)] += 1 + return { + "dataSource": { + "dataSourceConfiguration": { + "s3Configuration": { + "bucketArn": "arn:aws:s3:::my-kb-datasource-bucket" + } + } + } + } + + c.get_data_source.side_effect = _get_data_source + + # list_agents for FS-07 + c.list_agents.return_value = {"agentSummaries": []} + return c + + # ------------------------------------------------------------------ # + # s3 — count list_buckets + # ------------------------------------------------------------------ # + if service == "s3": + + def _list_buckets(**kw): + tracker["list_buckets"] += 1 + return {"Buckets": _S3_BUCKETS} + + c.list_buckets.side_effect = _list_buckets + + def _get_bucket_versioning(Bucket, **kw): + return {"Status": "Enabled"} + + c.get_bucket_versioning.side_effect = _get_bucket_versioning + + def _get_bucket_tagging(Bucket, **kw): + return { + "TagSet": [{"Key": "data-classification", "Value": "Confidential"}] + } + + c.get_bucket_tagging.side_effect = _get_bucket_tagging + + def _get_bucket_notification_configuration(Bucket, **kw): + return {"EventBridgeConfiguration": {}} + + c.get_bucket_notification_configuration.side_effect = ( + _get_bucket_notification_configuration + ) + return c + + # ------------------------------------------------------------------ # + # shield — needed for FS-01 + # ------------------------------------------------------------------ # + if service == "shield": + c.describe_subscription.return_value = {} + return c + + # ------------------------------------------------------------------ # + # Non-inventory services — return minimal responses so checks complete + # ------------------------------------------------------------------ # + if service == "apigateway": + c.get_usage_plans.return_value = { + "items": [ + { + "name": "default", + "throttle": {"rateLimit": 500, "burstLimit": 200}, + } + ] + } + c.get_rest_apis.return_value = {"items": []} + return c + + if service == "ce": + c.get_anomaly_monitors.return_value = { + "AnomalyMonitors": [ + { + "MonitorType": "DIMENSIONAL", + "MonitorDimension": "SERVICE", + "MonitorSpecification": {}, + } + ] + } + return c + + if service == "cloudwatch": + pag = MagicMock() + pag.paginate.return_value = [ + { + "MetricAlarms": [ + { + "AlarmName": "bedrock-throttle-alarm", + "Namespace": "AWS/Bedrock", + "MetricName": "InvocationThrottles", + } + ] + } + ] + c.get_paginator.return_value = pag + return c + + if service == "budgets": + pag = MagicMock() + pag.paginate.return_value = [ + { + "Budgets": [ + { + "BudgetName": "bedrock-spend", + "CostFilters": {"Service": ["Amazon Bedrock"]}, + } + ] + } + ] + c.get_paginator.return_value = pag + return c + + if service == "sts": + c.get_caller_identity.return_value = {"Account": "123456789012"} + return c + + if service == "service-quotas": + applied_pag = MagicMock() + applied_pag.paginate.return_value = [ + { + "Quotas": [ + { + "QuotaName": "On-demand InvokeModel tokens per minute for anthropic.claude", + "QuotaCode": "L-TPMTEST", + "Value": 200000, + } + ] + } + ] + defaults_pag = MagicMock() + defaults_pag.paginate.return_value = [ + {"Quotas": [{"QuotaCode": "L-TPMTEST", "Value": 100000}]} + ] + + def get_paginator(op): + if op == "list_service_quotas": + return applied_pag + if op == "list_aws_default_service_quotas": + return defaults_pag + p = MagicMock() + p.paginate.return_value = [{}] + return p + + c.get_paginator.side_effect = get_paginator + return c + + if service == "stepfunctions": + c.list_state_machines.return_value = {"stateMachines": []} + return c + + if service == "iam": + c.list_policies.return_value = {"Policies": []} + pag = MagicMock() + pag.paginate.return_value = [{}] + c.get_paginator.return_value = pag + return c + + if service == "config": + c.describe_config_rules.return_value = {"ConfigRules": []} + return c + + if service == "ecr": + c.describe_repositories.return_value = {"repositories": []} + return c + + if service == "sagemaker": + c.list_feature_groups.return_value = {"FeatureGroupSummaries": []} + c.list_processing_jobs.return_value = {"ProcessingJobSummaries": []} + c.list_models.return_value = {"Models": []} + pag = MagicMock() + pag.paginate.return_value = [{}] + c.get_paginator.return_value = pag + return c + + if service == "logs": + c.list_log_groups.return_value = {"logGroups": []} + pag = MagicMock() + pag.paginate.return_value = [{"logGroups": []}] + c.get_paginator.return_value = pag + return c + + if service == "macie2": + c.get_macie_session.side_effect = Exception("macie not enabled") + return c + + if service == "opensearchserverless": + c.list_security_policies.return_value = {"securityPolicySummaries": []} + return c + + if service == "bedrock-agent-runtime": + return c + + if service == "bedrock-runtime": + return c + + if service == "events": + c.list_rules.return_value = {"Rules": []} + return c + + if service == "scheduler": + c.list_schedules.return_value = {"Schedules": []} + return c + + if service == "agentcore": + c.list_agent_runtimes.return_value = {"agentRuntimes": []} + return c + + if service == "organizations": + c.list_policies.return_value = {"Policies": []} + return c + + # Catch-all: return generic mock for any other service + pag = MagicMock() + pag.paginate.return_value = [{}] + c.get_paginator.return_value = pag + return c + + return side_effect, tracker + + +# =========================================================================== +# Helper: run lambda_handler with counting mocks +# =========================================================================== + + +def _run_with_counting_mocks(event=None): + """Run lambda_handler end-to-end with counting mocks. + + Returns the (result, tracker) pair where tracker holds all call counts. + """ + if event is None: + event = {"Execution": {"Name": "at-most-once-test-001"}} + + side_effect, tracker = _build_counting_client_factory() + + with ( + patch("app.boto3.client") as mock_client, + patch("app.get_permissions_cache") as mock_cache, + patch("app.write_to_s3") as mock_s3, + ): + mock_client.side_effect = side_effect + mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} + mock_s3.return_value = "https://test-bucket.s3.amazonaws.com/finserv_security_report_at-most-once-test-001.csv" + + result = app.lambda_handler(event, None) + + return result, tracker + + +# =========================================================================== +# Test class +# =========================================================================== + + +class TestAtMostOnceInvariants: + """Handler-level counting-mock harness for the at-most-once invariants. + + Validates: Requirements REQ-9.1, REQ-9.4, REQ-9.6 + INV-2: Each shared listing API ≤ 1×/invocation; each shared detail API + ≤ 1× per resource/invocation. + """ + + @pytest.fixture(autouse=True) + def _run_handler(self): + """Run lambda_handler once and expose result + tracker to every test.""" + self.result, self.tracker = _run_with_counting_mocks() + + # ------------------------------------------------------------------ + # Sanity: handler completed successfully + # ------------------------------------------------------------------ + + def test_handler_completes_successfully(self): + """Counting mocks are sufficient for the handler to return 200.""" + assert self.result["statusCode"] == 200 + + # ------------------------------------------------------------------ + # Listing API invariants (≤ 1 per run) + # ------------------------------------------------------------------ + + def test_list_functions_at_most_once(self): + """list_functions SHALL be called ≤ 1 per run (REQ-9.1, INV-2). + + 6 checks consume Lambda functions; the collector must issue at most one + call regardless. + """ + count = self.tracker["list_functions"] + assert count <= 1, ( + f"list_functions was called {count} time(s); expected ≤ 1. " + "Multiple calls indicate the inventory was not consolidated." + ) + + def test_list_guardrails_at_most_once(self): + """list_guardrails SHALL be called ≤ 1 per run (REQ-9.1, INV-2). + + 9 checks consume guardrails; the collector must issue at most one call. + """ + count = self.tracker["list_guardrails"] + assert count <= 1, f"list_guardrails was called {count} time(s); expected ≤ 1." + + def test_list_knowledge_bases_at_most_once(self): + """list_knowledge_bases SHALL be called ≤ 1 per run (REQ-9.1, INV-2). + + 6 checks consume knowledge bases; the collector must issue at most one call. + """ + count = self.tracker["list_knowledge_bases"] + assert count <= 1, ( + f"list_knowledge_bases was called {count} time(s); expected ≤ 1." + ) + + def test_list_buckets_at_most_once(self): + """list_buckets SHALL be called ≤ 1 per run (REQ-9.1, INV-2). + + 2 checks consume S3 buckets; the collector must issue at most one call. + """ + count = self.tracker["list_buckets"] + assert count <= 1, f"list_buckets was called {count} time(s); expected ≤ 1." + + def test_list_web_acls_at_most_once(self): + """list_web_acls SHALL be called ≤ 1 per run (REQ-9.1, INV-2). + + 4 checks consume WAFv2 Web ACLs; the collector must issue at most one + call (previously each check called it independently). + """ + count = self.tracker["list_web_acls"] + assert count <= 1, f"list_web_acls was called {count} time(s); expected ≤ 1." + + def test_list_data_sources_at_most_once_per_kb(self): + """list_data_sources SHALL be called ≤ 1 per KB per run (REQ-9.1, REQ-3.5, INV-2). + + 3 checks consume data-source summaries (FS-31, FS-33, FS-65); each + KB's list_data_sources must be called at most once. + """ + for kb_id, count in self.tracker["list_data_sources"].items(): + assert count <= 1, ( + f"list_data_sources for KB '{kb_id}' was called {count} time(s); " + "expected ≤ 1." + ) + + # ------------------------------------------------------------------ + # Detail API invariants (≤ 1 per distinct resource per run) + # ------------------------------------------------------------------ + + def test_get_guardrail_at_most_once_per_id(self): + """get_guardrail SHALL be called ≤ 1 per distinct guardrail id per run. + + Requirements REQ-9.1, REQ-3.1, INV-2. + 9 checks inspect guardrail detail; the inventory must serve it from + cache without issuing further get_guardrail calls. + """ + for gid, count in self.tracker["get_guardrail"].items(): + assert count <= 1, ( + f"get_guardrail for guardrail '{gid}' was called {count} time(s); " + "expected ≤ 1." + ) + + def test_get_web_acl_at_most_once_per_id(self): + """get_web_acl SHALL be called ≤ 1 per distinct ACL id per run. + + Requirements REQ-9.1, REQ-3.2, INV-2. + 3 checks (FS-53, FS-56, FS-68) inspect ACL detail; the inventory must + serve it from cache. + """ + for acl_id, count in self.tracker["get_web_acl"].items(): + assert count <= 1, ( + f"get_web_acl for ACL '{acl_id}' was called {count} time(s); " + "expected ≤ 1." + ) + + def test_get_data_source_at_most_once_per_pair(self): + """get_data_source SHALL be called ≤ 1 per (kb_id, ds_id) pair per run. + + Requirements REQ-9.1, REQ-3.5, INV-2. + 2 checks (FS-33, FS-65) call get_data_source; the inventory must cache + the result and serve it to both. + """ + for (kb_id, ds_id), count in self.tracker["get_data_source"].items(): + assert count <= 1, ( + f"get_data_source for (kb='{kb_id}', ds='{ds_id}') was called " + f"{count} time(s); expected ≤ 1." + ) + + # ------------------------------------------------------------------ + # Positive coverage: calls were made (inventory was actually collected) + # ------------------------------------------------------------------ + + def test_listing_apis_were_called_at_least_once(self): + """Verify the counting mocks were exercised — all five listing APIs called.""" + assert self.tracker["list_functions"] >= 1, "list_functions was never called" + assert self.tracker["list_guardrails"] >= 1, "list_guardrails was never called" + assert self.tracker["list_knowledge_bases"] >= 1, ( + "list_knowledge_bases was never called" + ) + assert self.tracker["list_buckets"] >= 1, "list_buckets was never called" + assert self.tracker["list_web_acls"] >= 1, "list_web_acls was never called" + + def test_guardrail_detail_called_for_every_guardrail(self): + """get_guardrail was called for each guardrail in the summary list.""" + expected_ids = {g["id"] for g in _GUARDRAIL_SUMMARIES} + called_ids = set(self.tracker["get_guardrail"].keys()) + assert expected_ids == called_ids, ( + f"Expected get_guardrail calls for {expected_ids}; " + f"actually called for {called_ids}." + ) + + def test_web_acl_detail_called_for_every_acl(self): + """get_web_acl was called for each ACL in the summary list.""" + expected_ids = {acl["Id"] for acl in _ACL_SUMMARIES} + called_ids = set(self.tracker["get_web_acl"].keys()) + assert expected_ids == called_ids, ( + f"Expected get_web_acl calls for {expected_ids}; " + f"actually called for {called_ids}." + ) + + def test_data_source_detail_called_for_all_data_sources(self): + """get_data_source was called for each (kb_id, ds_id) pair.""" + expected_pairs = { + (kb_id, ds["dataSourceId"]) + for kb_id, ds_list in _KB_DATA_SOURCE_SUMMARIES.items() + for ds in ds_list + } + called_pairs = set(self.tracker["get_data_source"].keys()) + assert expected_pairs == called_pairs, ( + f"Expected get_data_source calls for {expected_pairs}; " + f"actually called for {called_pairs}." + ) + + def test_list_data_sources_called_for_all_kbs(self): + """list_data_sources was called for each KB in the summary list.""" + expected_kb_ids = {kb["knowledgeBaseId"] for kb in _KB_SUMMARIES} + called_kb_ids = set(self.tracker["list_data_sources"].keys()) + assert expected_kb_ids == called_kb_ids, ( + f"Expected list_data_sources calls for {expected_kb_ids}; " + f"actually called for {called_kb_ids}." + ) + + # ------------------------------------------------------------------ + # Exact counts (not just ≤ 1: assert exactly 1 for non-empty inventories) + # ------------------------------------------------------------------ + + def test_list_functions_called_exactly_once(self): + """list_functions is called exactly once — not zero, not more than one.""" + assert self.tracker["list_functions"] == 1 + + def test_list_guardrails_called_exactly_once(self): + """list_guardrails is called exactly once — not zero, not more than one.""" + assert self.tracker["list_guardrails"] == 1 + + def test_list_knowledge_bases_called_exactly_once(self): + """list_knowledge_bases is called exactly once.""" + assert self.tracker["list_knowledge_bases"] == 1 + + def test_list_buckets_called_exactly_once(self): + """list_buckets is called exactly once.""" + assert self.tracker["list_buckets"] == 1 + + def test_list_web_acls_called_exactly_once(self): + """list_web_acls is called exactly once.""" + assert self.tracker["list_web_acls"] == 1 + + def test_list_data_sources_called_exactly_once_per_kb(self): + """list_data_sources is called exactly once for each KB (not zero, not more).""" + for kb_id in {kb["knowledgeBaseId"] for kb in _KB_SUMMARIES}: + count = self.tracker["list_data_sources"][kb_id] + assert count == 1, ( + f"list_data_sources for KB '{kb_id}' was called {count} time(s); " + "expected exactly 1." + ) + + def test_get_guardrail_called_exactly_once_per_guardrail(self): + """get_guardrail is called exactly once per guardrail id.""" + for g in _GUARDRAIL_SUMMARIES: + gid = g["id"] + count = self.tracker["get_guardrail"][gid] + assert count == 1, ( + f"get_guardrail for '{gid}' was called {count} time(s); " + "expected exactly 1." + ) + + def test_get_web_acl_called_exactly_once_per_acl(self): + """get_web_acl is called exactly once per ACL id.""" + for acl in _ACL_SUMMARIES: + acl_id = acl["Id"] + count = self.tracker["get_web_acl"][acl_id] + assert count == 1, ( + f"get_web_acl for '{acl_id}' was called {count} time(s); " + "expected exactly 1." + ) + + def test_get_data_source_called_exactly_once_per_pair(self): + """get_data_source is called exactly once per (kb_id, ds_id) pair.""" + for kb_id, ds_list in _KB_DATA_SOURCE_SUMMARIES.items(): + for ds in ds_list: + ds_id = ds["dataSourceId"] + count = self.tracker["get_data_source"][(kb_id, ds_id)] + assert count == 1, ( + f"get_data_source for (kb='{kb_id}', ds='{ds_id}') was called " + f"{count} time(s); expected exactly 1." + ) diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_checks.py b/aiml-security-assessment/functions/security/finserv_tests/test_checks.py new file mode 100644 index 0000000..f456443 --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_tests/test_checks.py @@ -0,0 +1,3577 @@ +""" +Tests for all check functions in finserv_assessments/app.py + +Strategy: + - Every check function is tested for at least two scenarios: + 1. "PASS / N/A" path — mocked AWS responses indicate compliant state + 2. "WARN / FAIL" path — mocked AWS responses indicate non-compliant state + - Advisory-only checks (no AWS API calls) are tested for correct structure + - Functions that accept permission_cache are tested with both empty and + populated caches + - Every check is tested for graceful error handling (boto3 ClientError) + +All boto3 clients are patched via unittest.mock so no real AWS calls are made. +""" + +import json +import sys +import os +from unittest.mock import MagicMock, patch + +from botocore.exceptions import ClientError + +# Ensure finserv_assessments is importable +FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") +if FINSERV_DIR not in sys.path: + sys.path.insert(0, FINSERV_DIR) + +# Ensure tests/ directory is importable (for conftest helpers) +TESTS_DIR = os.path.dirname(__file__) +if TESTS_DIR not in sys.path: + sys.path.insert(0, TESTS_DIR) + +import app # noqa: E402 (import must follow sys.path setup above) +from conftest import make_resource_inventory # noqa: E402 + + +# ========================================================================= +# Helpers +# ========================================================================= + + +def _client_error(code="AccessDeniedException", message="Access Denied"): + """Build a botocore ClientError for mocking.""" + return ClientError( + {"Error": {"Code": code, "Message": message}}, + "TestOperation", + ) + + +def _assert_finding_structure(result): + """Assert the standard check-function return dict shape.""" + assert "check_name" in result + assert "status" in result + assert result["status"] in ("PASS", "WARN", "ERROR") + assert "csv_data" in result + assert isinstance(result["csv_data"], list) + for row in result["csv_data"]: + assert "Check_ID" in row + assert "Finding" in row + assert "Severity" in row + assert "Status" in row + # Compliance_Frameworks must be present in every row (D3 fix) + assert "Compliance_Frameworks" in row, ( + f"Missing Compliance_Frameworks in row for {row.get('Check_ID')}" + ) + + +def _assert_advisory_retag(result, check_id): + """ + REQ-6 Option B: an advisory check must emit a finding with Status="N/A", + Severity="Informational", and a Finding name prefixed "ADVISORY: ". + (The wrapper result["status"] is separate and may still be "PASS".) + """ + rows = [r for r in result["csv_data"] if r["Check_ID"] == check_id] + assert rows, f"no rows for {check_id}" + for r in rows: + assert r["Status"] == "N/A", f"{check_id} Status={r['Status']}" + assert r["Severity"] == "Informational", f"{check_id} Severity={r['Severity']}" + assert r["Finding"].startswith("ADVISORY: "), ( + f"{check_id} Finding={r['Finding']}" + ) + + +# ========================================================================= +# CATEGORY 1: UNBOUNDED CONSUMPTION (FS-01 to FS-06) +# ========================================================================= + + +class TestFS01WafShield: + """FS-01 — WAF and Shield Protection Check.""" + + def test_pass_shield_enabled_acls_present(self): + """Shield enabled + ACLs present via inventory → PASS.""" + inv = make_resource_inventory( + web_acls=app.WebAclInventory( + summaries=[{"Name": "acl1", "Id": "id1"}], + detail_by_id={}, + ) + ) + with patch("app.boto3.client") as mock_client: + shield_mock = MagicMock() + shield_mock.describe_subscription.return_value = {} + shield_mock.exceptions.ResourceNotFoundException = type( + "ResourceNotFoundException", (ClientError,), {} + ) + mock_client.return_value = shield_mock + + result = app.check_waf_shield_on_bedrock_endpoints(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + assert len(result["csv_data"]) == 2 # shield pass + waf pass + # REQ-6: Shield Advanced findings are Low; WAF findings are Medium. + shield_rows = [r for r in result["csv_data"] if "Shield" in r["Finding"]] + waf_rows = [r for r in result["csv_data"] if "WAF" in r["Finding"]] + assert shield_rows and all(r["Severity"] == "Low" for r in shield_rows) + assert waf_rows and all(r["Severity"] == "Medium" for r in waf_rows) + + def test_severity_shield_low_waf_medium_on_fail(self): + """REQ-6: Shield-absent = Low, WAF-absent = Medium (was both High).""" + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=[], detail_by_id={}) + ) + with patch("app.boto3.client") as mock_client: + shield_mock = MagicMock() + shield_mock.describe_subscription.side_effect = ClientError( + {"Error": {"Code": "ResourceNotFoundException", "Message": ""}}, + "DescribeSubscription", + ) + shield_mock.exceptions.ResourceNotFoundException = type( + "ResourceNotFoundException", (ClientError,), {} + ) + mock_client.return_value = shield_mock + + result = app.check_waf_shield_on_bedrock_endpoints(inv) + _assert_finding_structure(result) + assert any( + r["Finding"] == "AWS Shield Advanced Not Enabled" and r["Severity"] == "Low" + for r in result["csv_data"] + ) + assert any( + r["Finding"] == "No Regional WAF Web ACLs Found" + and r["Severity"] == "Medium" + for r in result["csv_data"] + ) + + def test_warn_no_shield_no_acls(self): + """No shield + no ACLs in inventory → WARN.""" + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=[], detail_by_id={}) + ) + with patch("app.boto3.client") as mock_client: + shield_mock = MagicMock() + shield_mock.describe_subscription.side_effect = ClientError( + {"Error": {"Code": "ResourceNotFoundException", "Message": ""}}, + "DescribeSubscription", + ) + shield_mock.exceptions.ResourceNotFoundException = type( + "ResourceNotFoundException", (ClientError,), {} + ) + mock_client.return_value = shield_mock + + result = app.check_waf_shield_on_bedrock_endpoints(inv) + _assert_finding_structure(result) + assert result["status"] == "WARN" + + def test_error_on_exception(self): + """Unavailable inventory → COULD_NOT_ASSESS (ERROR envelope).""" + inv = make_resource_inventory(web_acls=app._Unavailable(RuntimeError("boom"))) + with patch("app.boto3.client") as mock_client: + shield_mock = MagicMock() + shield_mock.describe_subscription.return_value = {} + shield_mock.exceptions.ResourceNotFoundException = type( + "ResourceNotFoundException", (ClientError,), {} + ) + mock_client.return_value = shield_mock + result = app.check_waf_shield_on_bedrock_endpoints(inv) + assert result["status"] == "ERROR" + assert "boom" in result["details"] + + def test_error_on_shield_exception(self): + """Shield client raises → overall ERROR.""" + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=[], detail_by_id={}) + ) + with patch("app.boto3.client") as mock_client: + mock_client.side_effect = RuntimeError("boom") + result = app.check_waf_shield_on_bedrock_endpoints(inv) + assert result["status"] == "ERROR" + assert "boom" in result["details"] + + # --- Pagination-correctness proof: >100 ACLs (Wave-3, task 8) --- + def test_pass_more_than_100_acls(self): + """Inventory with >100 ACLs (truncated in old code) → FS-01 PASS with correct count.""" + acls = [{"Name": f"acl{i}", "Id": f"id{i}"} for i in range(150)] + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=acls, detail_by_id={}) + ) + with patch("app.boto3.client") as mock_client: + shield_mock = MagicMock() + shield_mock.describe_subscription.return_value = {} + shield_mock.exceptions.ResourceNotFoundException = type( + "ResourceNotFoundException", (ClientError,), {} + ) + mock_client.return_value = shield_mock + result = app.check_waf_shield_on_bedrock_endpoints(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + waf_rows = [r for r in result["csv_data"] if "WAF" in r["Finding"]] + assert waf_rows + assert "150" in waf_rows[0]["Finding_Details"] + + # --- ≤1-page equivalence: 2-ACL case matches Wave-0 baseline --- + def test_two_acl_case_unchanged(self): + """The ≤1-page (2-ACL) case must produce the same finding as the pre-refactor baseline.""" + acls = [ + {"Name": "FinServACL1", "Id": "acl-id-001"}, + {"Name": "FinServACL2", "Id": "acl-id-002"}, + ] + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=acls, detail_by_id={}) + ) + with patch("app.boto3.client") as mock_client: + shield_mock = MagicMock() + shield_mock.describe_subscription.return_value = {} + shield_mock.exceptions.ResourceNotFoundException = type( + "ResourceNotFoundException", (ClientError,), {} + ) + mock_client.return_value = shield_mock + result = app.check_waf_shield_on_bedrock_endpoints(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + waf_rows = [r for r in result["csv_data"] if "WAF" in r["Finding"]] + assert waf_rows + assert waf_rows[0]["Finding"] == "Regional WAF Web ACLs Present" + + +class TestFS02ApiGatewayRateLimiting: + """FS-02 — API Gateway Rate Limiting Check.""" + + @patch("app.boto3.client") + def test_pass_all_plans_have_throttle(self, mock_client): + c = MagicMock() + c.get_usage_plans.return_value = { + "items": [ + {"name": "plan1", "throttle": {"rateLimit": 100, "burstLimit": 50}}, + ] + } + mock_client.return_value = c + result = app.check_api_gateway_rate_limiting() + _assert_finding_structure(result) + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_warn_plan_missing_throttle(self, mock_client): + c = MagicMock() + c.get_usage_plans.return_value = { + "items": [ + {"name": "no-throttle-plan", "throttle": {"rateLimit": 0}}, + ] + } + mock_client.return_value = c + result = app.check_api_gateway_rate_limiting() + _assert_finding_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_no_plans_returns_na(self, mock_client): + c = MagicMock() + c.get_usage_plans.return_value = {"items": []} + mock_client.return_value = c + result = app.check_api_gateway_rate_limiting() + _assert_finding_structure(result) + # No plans → advisory finding, status stays PASS + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("api error") + result = app.check_api_gateway_rate_limiting() + assert result["status"] == "ERROR" + + +class TestFS03BedrockTokenQuotas: + """FS-03 — Bedrock Token Quota Review (value-based, paginated).""" + + @staticmethod + def _sq_client(applied_quotas, default_quotas): + """Build a service-quotas client mock with paginated applied + default quotas.""" + c = MagicMock() + + def get_paginator(op_name): + paginator = MagicMock() + if op_name == "list_service_quotas": + paginator.paginate.return_value = [{"Quotas": applied_quotas}] + elif op_name == "list_aws_default_service_quotas": + paginator.paginate.return_value = [{"Quotas": default_quotas}] + else: + paginator.paginate.return_value = [{}] + return paginator + + c.get_paginator.side_effect = get_paginator + return c + + @patch("app.boto3.client") + def test_pass_customized_quota(self, mock_client): + # Applied value (200000) exceeds AWS default (100000) → customized → PASS/Passed + applied = [ + { + "QuotaName": "On-demand InvokeModel tokens per minute for anthropic.claude", + "QuotaCode": "L-1234ABCD", + "Value": 200000, + } + ] + defaults = [{"QuotaCode": "L-1234ABCD", "Value": 100000}] + mock_client.return_value = self._sq_client(applied, defaults) + result = app.check_bedrock_token_quotas() + _assert_finding_structure(result) + assert result["status"] == "PASS" + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_warn_default_quota(self, mock_client): + # Applied value == AWS default → still at default → WARN/N-A (soft, not a failure) + applied = [ + { + "QuotaName": "On-demand InvokeModel tokens per minute for anthropic.claude", + "QuotaCode": "L-1234ABCD", + "Value": 100000, + } + ] + defaults = [{"QuotaCode": "L-1234ABCD", "Value": 100000}] + mock_client.return_value = self._sq_client(applied, defaults) + result = app.check_bedrock_token_quotas() + _assert_finding_structure(result) + assert result["status"] == "WARN" + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + # At-default is NOT a failure. + assert not any(r["Status"] == "Failed" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_token_only_no_rpm(self, mock_client): + # Only token-based quotas present (no "request"/RPM quota). RPM is deprecated + # on bedrock-runtime; its absence must not drive a Failed verdict. + applied = [ + { + "QuotaName": "Model invocation max tokens per day for anthropic.claude", + "QuotaCode": "L-TPDAY01", + "Value": 5000000, + } + ] + defaults = [{"QuotaCode": "L-TPDAY01", "Value": 1000000}] + mock_client.return_value = self._sq_client(applied, defaults) + result = app.check_bedrock_token_quotas() + _assert_finding_structure(result) + # Customized token quota → PASS, regardless of any RPM quota existing. + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_empty_applied_quotas(self, mock_client): + # No token quotas returned at all → WARN/Failed + explanatory details. + mock_client.return_value = self._sq_client([], []) + result = app.check_bedrock_token_quotas() + _assert_finding_structure(result) + assert result["status"] == "WARN" + assert any(r["Status"] == "Failed" for r in result["csv_data"]) + assert any( + "No Bedrock token-based service quotas" in r["Finding_Details"] + for r in result["csv_data"] + ) + + @patch("app.boto3.client") + def test_default_lookup_fail(self, mock_client): + # Applied token quotas exist but defaults could not be retrieved → + # WARN/Failed + "undetermined" (NOT a silent value-vs-itself comparison). + applied = [ + { + "QuotaName": "On-demand InvokeModel tokens per minute for anthropic.claude", + "QuotaCode": "L-1234ABCD", + "Value": 100000, + } + ] + mock_client.return_value = self._sq_client(applied, []) + result = app.check_bedrock_token_quotas() + _assert_finding_structure(result) + assert result["status"] == "WARN" + assert any(r["Status"] == "Failed" for r in result["csv_data"]) + assert any( + "could not be retrieved" in r["Finding_Details"] + or "Undetermined" in r["Finding"] + for r in result["csv_data"] + ) + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("quota error") + result = app.check_bedrock_token_quotas() + assert result["status"] == "ERROR" + + +class TestFS04CostAnomalyDetection: + """FS-04 — Cost Anomaly Detection Check.""" + + @patch("app.boto3.client") + def test_pass_monitors_exist(self, mock_client): + c = MagicMock() + c.get_anomaly_monitors.return_value = { + "AnomalyMonitors": [ + { + "MonitorType": "DIMENSIONAL", + "MonitorDimension": "SERVICE", + "MonitorSpecification": {}, + } + ] + } + mock_client.return_value = c + result = app.check_cost_anomaly_detection() + _assert_finding_structure(result) + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_warn_monitors_without_bedrock_coverage(self, mock_client): + # A DIMENSIONAL monitor scoped to LINKED_ACCOUNT does NOT provide + # Bedrock service-level coverage → non-PASS (previously masked false positive). + c = MagicMock() + c.get_anomaly_monitors.return_value = { + "AnomalyMonitors": [ + { + "MonitorType": "DIMENSIONAL", + "MonitorDimension": "LINKED_ACCOUNT", + "MonitorSpecification": {}, + } + ] + } + mock_client.return_value = c + result = app.check_cost_anomaly_detection() + _assert_finding_structure(result) + assert result["status"] != "PASS" + assert any(r["Status"] == "Failed" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_warn_no_monitors(self, mock_client): + c = MagicMock() + c.get_anomaly_monitors.return_value = {"AnomalyMonitors": []} + mock_client.return_value = c + result = app.check_cost_anomaly_detection() + _assert_finding_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_pagination_finds_bedrock_monitor_on_second_page(self, mock_client): + # The Bedrock-covering monitor is on page 2. The check must paginate via + # NextPageToken and still find it (otherwise a false "no coverage" finding). + c = MagicMock() + c.get_anomaly_monitors.side_effect = [ + { + "AnomalyMonitors": [ + { + "MonitorType": "DIMENSIONAL", + "MonitorDimension": "LINKED_ACCOUNT", + "MonitorSpecification": {}, + } + ], + "NextPageToken": "page2", + }, + { + "AnomalyMonitors": [ + { + "MonitorType": "DIMENSIONAL", + "MonitorDimension": "SERVICE", + "MonitorSpecification": {}, + } + ] + }, + ] + mock_client.return_value = c + result = app.check_cost_anomaly_detection() + _assert_finding_structure(result) + assert result["status"] == "PASS" + # Verify it actually consumed both pages (passed the token on the 2nd call). + assert c.get_anomaly_monitors.call_count == 2 + c.get_anomaly_monitors.assert_any_call(NextPageToken="page2") + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("ce error") + result = app.check_cost_anomaly_detection() + assert result["status"] == "ERROR" + + +class TestFS05CloudWatchTokenAlarms: + """FS-05 — CloudWatch Token Usage Alarms Check.""" + + @patch("app.boto3.client") + def test_pass_bedrock_alarms_exist(self, mock_client): + c = MagicMock() + paginator = MagicMock() + paginator.paginate.return_value = [ + { + "MetricAlarms": [ + { + "AlarmName": "bedrock-throttle-alarm", + "Namespace": "AWS/Bedrock", + "MetricName": "InvocationThrottles", + } + ] + } + ] + c.get_paginator.return_value = paginator + mock_client.return_value = c + result = app.check_cloudwatch_token_alarms() + _assert_finding_structure(result) + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_warn_no_bedrock_alarms(self, mock_client): + c = MagicMock() + paginator = MagicMock() + paginator.paginate.return_value = [ + { + "MetricAlarms": [ + { + "AlarmName": "cpu-alarm", + "Namespace": "AWS/EC2", + "MetricName": "CPUUtilization", + } + ] + } + ] + c.get_paginator.return_value = paginator + mock_client.return_value = c + result = app.check_cloudwatch_token_alarms() + _assert_finding_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("cw error") + result = app.check_cloudwatch_token_alarms() + assert result["status"] == "ERROR" + + +class TestFS06AwsBudgets: + """FS-06 — AWS Budgets AI/ML Spend Check (ShowFilterExpression, paginated).""" + + @staticmethod + def _budgets_client(budgets, capture=None, raise_param_validation_on_show=False): + """ + Build a budgets client whose describe_budgets paginator returns `budgets`. + If raise_param_validation_on_show is True, paginate raises ParamValidationError + when called with ShowFilterExpression (simulating old botocore), and returns + budgets when called without it. + """ + from botocore.exceptions import ParamValidationError + + c = MagicMock() + paginator = MagicMock() + + def paginate(**kwargs): + if capture is not None: + capture.append(kwargs) + if raise_param_validation_on_show and "ShowFilterExpression" in kwargs: + raise ParamValidationError(report="ShowFilterExpression not accepted") + return [{"Budgets": budgets}] + + paginator.paginate.side_effect = paginate + c.get_paginator.return_value = paginator + return c + + def _client_factory(self, budgets, capture=None, raise_pv=False): + def side_effect(service, **kwargs): + if service == "budgets": + return self._budgets_client(budgets, capture, raise_pv) + if service == "sts": + c = MagicMock() + c.get_caller_identity.return_value = {"Account": "123456789012"} + return c + return MagicMock() + + return side_effect + + @patch("app.boto3.client") + def test_pass_aiml_budgets_exist(self, mock_client): + capture = [] + mock_client.side_effect = self._client_factory( + [{"BudgetName": "bedrock-budget", "CostFilters": {"Service": ["bedrock"]}}], + capture=capture, + ) + result = app.check_aws_budgets_for_aiml() + _assert_finding_structure(result) + assert result["status"] == "PASS" + # Regression guard: the call MUST pass ShowFilterExpression=True. + assert any(kw.get("ShowFilterExpression") is True for kw in capture) + + @patch("app.boto3.client") + def test_warn_no_aiml_budgets(self, mock_client): + mock_client.side_effect = self._client_factory( + [{"BudgetName": "general", "CostFilters": {"Service": ["ec2"]}}] + ) + result = app.check_aws_budgets_for_aiml() + _assert_finding_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_pass_filterexpression_budget(self, mock_client): + # New-style budget using only FilterExpression (no CostFilters) → detected. + mock_client.side_effect = self._client_factory( + [ + { + "BudgetName": "genai-budget", + "CostFilters": {}, + "FilterExpression": { + "Dimensions": {"Key": "SERVICE", "Values": ["Amazon Bedrock"]} + }, + } + ] + ) + result = app.check_aws_budgets_for_aiml() + _assert_finding_structure(result) + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_param_validation_fallback(self, mock_client): + # Old botocore: ShowFilterExpression rejected with ParamValidationError. + # FS-06 must degrade to CostFilters-only (non-ERROR) and still match. + capture = [] + mock_client.side_effect = self._client_factory( + [{"BudgetName": "bedrock-budget", "CostFilters": {"Service": ["bedrock"]}}], + capture=capture, + raise_pv=True, + ) + result = app.check_aws_budgets_for_aiml() + _assert_finding_structure(result) + assert result["status"] != "ERROR" + assert result["status"] == "PASS" + # It attempted with ShowFilterExpression, then retried without it. + assert any("ShowFilterExpression" in kw for kw in capture) + assert any("ShowFilterExpression" not in kw for kw in capture) + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("budgets error") + result = app.check_aws_budgets_for_aiml() + assert result["status"] == "ERROR" + + +# ========================================================================= +# CATEGORY 2: EXCESSIVE AGENCY (FS-07 to FS-11) +# ========================================================================= + + +class TestFS07AgentActionBoundaries: + """FS-07 — Agent Action Boundary Check (takes permission_cache).""" + + @patch("app.boto3.client") + def test_pass_no_agents(self, mock_client): + c = MagicMock() + c.list_agents.return_value = {"agentSummaries": []} + mock_client.return_value = c + result = app.check_bedrock_agent_action_boundaries({}) + _assert_finding_structure(result) + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_warn_wildcard_permissions( + self, mock_client, permission_cache_with_wildcard + ): + c = MagicMock() + c.list_agents.return_value = { + "agentSummaries": [{"agentId": "a1", "agentName": "TestAgent"}] + } + c.get_agent.return_value = { + "agent": {"agentResourceRoleArn": "arn:aws:iam::123:role/BedrockAgentRole"} + } + mock_client.return_value = c + result = app.check_bedrock_agent_action_boundaries( + permission_cache_with_wildcard + ) + _assert_finding_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_pass_narrow_permissions(self, mock_client, permission_cache_safe): + c = MagicMock() + c.list_agents.return_value = { + "agentSummaries": [{"agentId": "a1", "agentName": "TestAgent"}] + } + c.get_agent.return_value = { + "agent": {"agentResourceRoleArn": "arn:aws:iam::123:role/BedrockAgentRole"} + } + mock_client.return_value = c + result = app.check_bedrock_agent_action_boundaries(permission_cache_safe) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("agent error") + result = app.check_bedrock_agent_action_boundaries({}) + assert result["status"] == "ERROR" + + +class TestFS08AgentcorePolicyEngine: + """FS-08 — AgentCore Policy Engine Check.""" + + @patch("app.boto3.client") + def test_pass_runtimes_with_authorizer(self, mock_client): + c = MagicMock() + c.list_agent_runtimes.return_value = { + "agentRuntimes": [ + { + "agentRuntimeName": "rt1", + "authorizerConfiguration": {"customJWTAuthorizer": {}}, + } + ] + } + mock_client.return_value = c + result = app.check_agentcore_policy_engine() + _assert_finding_structure(result) + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_warn_runtimes_without_authorizer(self, mock_client): + c = MagicMock() + c.list_agent_runtimes.return_value = { + "agentRuntimes": [{"agentRuntimeName": "rt1"}] + } + mock_client.return_value = c + result = app.check_agentcore_policy_engine() + _assert_finding_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_na_no_runtimes(self, mock_client): + c = MagicMock() + c.list_agent_runtimes.return_value = {"agentRuntimes": []} + mock_client.return_value = c + result = app.check_agentcore_policy_engine() + _assert_finding_structure(result) + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_access_denied_returns_na(self, mock_client): + c = MagicMock() + c.list_agent_runtimes.side_effect = _client_error("AccessDeniedException") + mock_client.return_value = c + result = app.check_agentcore_policy_engine() + _assert_finding_structure(result) + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("agentcore error") + result = app.check_agentcore_policy_engine() + assert result["status"] == "ERROR" + + +class TestFS09AgentTransactionLimits: + """FS-09 — Agent Transaction Limits Check.""" + + @patch("app.boto3.client") + def test_pass_concurrency_set(self, mock_client): + c = MagicMock() + c.get_function_concurrency.return_value = {"ReservedConcurrentExecutions": 10} + mock_client.return_value = c + inv = make_resource_inventory( + lambda_functions=[{"FunctionName": "my-agent-handler"}] + ) + result = app.check_agent_transaction_limits(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_warn_no_concurrency(self, mock_client): + c = MagicMock() + c.get_function_concurrency.return_value = {} + mock_client.return_value = c + inv = make_resource_inventory( + lambda_functions=[{"FunctionName": "my-agent-handler"}] + ) + result = app.check_agent_transaction_limits(inv) + _assert_finding_structure(result) + assert result["status"] == "WARN" + + def test_error_on_unavailable_inventory(self): + inv = make_resource_inventory( + lambda_functions=app._Unavailable(RuntimeError("lambda error")) + ) + result = app.check_agent_transaction_limits(inv) + assert result["status"] == "ERROR" + + +class TestFS10HumanInTheLoop: + """FS-10 — Human-in-the-Loop Approval Check.""" + + @patch("app.boto3.client") + def test_pass_wait_for_task_token(self, mock_client): + c = MagicMock() + c.list_state_machines.return_value = { + "stateMachines": [ + { + "name": "agent-approval-flow", + "stateMachineArn": "arn:aws:states:us-east-1:123:sm:test", + } + ] + } + # The function checks for '"waitForTaskToken"' (with JSON quotes) in the + # definition string, so we embed it as a standalone JSON string value. + defn = json.dumps( + { + "States": { + "Approve": { + "Type": "Task", + "Resource": "arn:aws:states:::sqs:sendMessage", + "Integration": "waitForTaskToken", + } + } + } + ) + c.describe_state_machine.return_value = {"definition": defn} + mock_client.return_value = c + result = app.check_human_in_the_loop_for_high_risk_actions() + _assert_finding_structure(result) + # The function finds "waitForTaskToken" in the definition → PASS path + assert result["status"] == "PASS" + assert len(result["csv_data"]) >= 1 + + @patch("app.boto3.client") + def test_warn_no_wait_token(self, mock_client): + c = MagicMock() + c.list_state_machines.return_value = { + "stateMachines": [ + { + "name": "agent-workflow", + "stateMachineArn": "arn:aws:states:us-east-1:123:sm:test", + } + ] + } + c.describe_state_machine.return_value = { + "definition": json.dumps({"States": {"Run": {"Type": "Task"}}}) + } + mock_client.return_value = c + result = app.check_human_in_the_loop_for_high_risk_actions() + _assert_finding_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("sfn error") + result = app.check_human_in_the_loop_for_high_risk_actions() + assert result["status"] == "ERROR" + + +class TestFS11AgentRateAlarms: + """FS-11 — Agent Rate Alarms Check.""" + + @patch("app.boto3.client") + def test_pass_agent_alarms_exist(self, mock_client): + c = MagicMock() + paginator = MagicMock() + paginator.paginate.return_value = [ + { + "MetricAlarms": [ + { + "AlarmName": "agent-invocation-rate", + "Namespace": "AWS/Bedrock", + "MetricName": "AgentInvocations", + } + ] + } + ] + c.get_paginator.return_value = paginator + mock_client.return_value = c + result = app.check_agent_rate_alarms() + _assert_finding_structure(result) + # The function looks for agent-related alarms + assert result["status"] in ("PASS", "WARN") + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("cw error") + result = app.check_agent_rate_alarms() + assert result["status"] == "ERROR" + + +# ========================================================================= +# CATEGORY 3: SUPPLY CHAIN VULNERABILITIES (FS-12 to FS-16) +# ========================================================================= + + +class TestFS12ScpModelAccess: + """FS-12 — SCP Model Access Restrictions.""" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("org error") + result = app.check_scp_model_access_restrictions() + assert result["status"] == "ERROR" + + @patch("app.boto3.client") + def test_returns_valid_structure(self, mock_client): + c = MagicMock() + c.list_policies.return_value = {"Policies": []} + mock_client.return_value = c + result = app.check_scp_model_access_restrictions() + _assert_finding_structure(result) + + +class TestFS13ModelInventoryTagging: + """FS-13 — Model Inventory Tagging.""" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("tagging error") + result = app.check_model_inventory_tagging() + assert result["status"] == "ERROR" + + @patch("app.boto3.client") + def test_returns_valid_structure(self, mock_client): + c = MagicMock() + c.list_custom_models.return_value = {"modelSummaries": []} + c.list_models.return_value = {"Models": []} + mock_client.return_value = c + result = app.check_model_inventory_tagging() + _assert_finding_structure(result) + + +class TestFS14ModelOnboardingGovernance: + """FS-14 — Model Onboarding Governance.""" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("config error") + result = app.check_model_onboarding_governance() + assert result["status"] == "ERROR" + + @patch("app.boto3.client") + def test_returns_valid_structure(self, mock_client): + c = MagicMock() + c.describe_config_rules.return_value = {"ConfigRules": []} + mock_client.return_value = c + result = app.check_model_onboarding_governance() + _assert_finding_structure(result) + + +class TestFS15BedrockModelEvalAdversarial: + """FS-15 — Bedrock Model Evaluation Adversarial.""" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("eval error") + result = app.check_bedrock_model_evaluation_adversarial() + assert result["status"] == "ERROR" + + @patch("app.boto3.client") + def test_returns_valid_structure(self, mock_client): + c = MagicMock() + c.list_evaluation_jobs.return_value = {"jobSummaries": []} + mock_client.return_value = c + result = app.check_bedrock_model_evaluation_adversarial() + _assert_finding_structure(result) + + @patch("app.boto3.client") + def test_fail_no_eval_jobs(self, mock_client): + """REQ-10a: no Bedrock evaluation jobs → Failed/Medium (was N/A).""" + c = MagicMock() + c.list_evaluation_jobs.return_value = {"jobSummaries": []} + mock_client.return_value = c + result = app.check_bedrock_model_evaluation_adversarial() + _assert_finding_structure(result) + assert any( + r["Finding"] == "No Bedrock Evaluation Jobs Found" + and r["Status"] == "Failed" + and r["Severity"] == "Medium" + for r in result["csv_data"] + ) + + @patch("app.boto3.client") + def test_pass_eval_jobs_present(self, mock_client): + """Eval jobs present → Passed/Medium.""" + c = MagicMock() + c.list_evaluation_jobs.return_value = { + "jobSummaries": [{"jobName": "robustness-eval"}] + } + mock_client.return_value = c + result = app.check_bedrock_model_evaluation_adversarial() + _assert_finding_structure(result) + assert any( + r["Finding"] == "Bedrock Evaluation Jobs Present" + and r["Status"] == "Passed" + for r in result["csv_data"] + ) + + +class TestFS16EcrImageScanning: + """FS-16 — ECR Image Scanning.""" + + @patch("app.boto3.client") + def test_pass_scanning_enabled(self, mock_client): + c = MagicMock() + c.describe_repositories.return_value = { + "repositories": [ + { + "repositoryName": "ml-model", + "imageScanningConfiguration": {"scanOnPush": True}, + } + ] + } + mock_client.return_value = c + result = app.check_ecr_image_scanning() + _assert_finding_structure(result) + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("ecr error") + result = app.check_ecr_image_scanning() + assert result["status"] == "ERROR" + + +# ========================================================================= +# CATEGORY 4: TRAINING DATA & MODEL POISONING (FS-20, FS-21) +# ========================================================================= + + +class TestFS20FeatureStoreRollback: + """FS-20 — Feature Store Rollback Capability.""" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("fs error") + result = app.check_feature_store_rollback_capability() + assert result["status"] == "ERROR" + + @patch("app.boto3.client") + def test_returns_valid_structure(self, mock_client): + c = MagicMock() + c.list_feature_groups.return_value = {"FeatureGroupSummaries": []} + mock_client.return_value = c + result = app.check_feature_store_rollback_capability() + _assert_finding_structure(result) + + +class TestFS21TrainingDataS3Versioning: + """FS-21 — Training Data S3 Versioning.""" + + def test_error_on_unavailable_inventory(self): + """When the buckets inventory is _Unavailable, check must return ERROR.""" + inv = make_resource_inventory( + buckets=app._Unavailable(RuntimeError("s3 error")) + ) + result = app.check_training_data_s3_versioning(inv) + assert result["status"] == "ERROR" + + @patch("app.boto3.client") + def test_returns_valid_structure(self, mock_client): + """Empty bucket list → N/A finding (no training buckets identified).""" + inv = make_resource_inventory(buckets=[]) + mock_client.return_value = MagicMock() + result = app.check_training_data_s3_versioning(inv) + _assert_finding_structure(result) + + +# ========================================================================= +# CATEGORY 5: VECTOR & EMBEDDING WEAKNESSES (FS-22, FS-24, FS-25, FS-26) +# ========================================================================= + + +class TestFS22KnowledgeBaseIamLeastPrivilege: + """FS-22 — Knowledge Base IAM Least Privilege (takes permission_cache).""" + + def test_pass_empty_cache(self, permission_cache_empty): + # FS-22 reads only the permission cache (no boto3 calls). An empty cache + # means no roles to inspect → PASS with no wildcard findings. + result = app.check_knowledge_base_iam_least_privilege(permission_cache_empty) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + """FS-22 only reads permission_cache (no boto3 calls). To trigger + the error path, pass a cache that causes an exception during iteration.""" + # A non-dict value for role_permissions will cause .items() to fail + bad_cache = {"role_permissions": "not-a-dict"} + result = app.check_knowledge_base_iam_least_privilege(bad_cache) + assert result["status"] == "ERROR" + + def test_single_statement_dict_no_crash_wildcard(self): + """REQ-3: a policy whose Statement is a single dict (not a list) must not + crash ('str' object has no attribute 'get'); a wildcard is still flagged.""" + cache = { + "role_permissions": { + "AmazonBedrockExecutionRoleForKnowledgeBase_Test": { + "attached_policies": [], + "inline_policies": [ + { + "name": "KBInline", + "document": { + "Version": "2012-10-17", + "Statement": { + "Effect": "Allow", + "Action": "bedrock-agent:*", + "Resource": "*", + }, + }, + } + ], + } + } + } + result = app.check_knowledge_base_iam_least_privilege(cache) + _assert_finding_structure(result) + assert result["status"] != "ERROR" + assert any( + r["Finding"] == "Overly Permissive Knowledge Base IAM Roles" + and r["Status"] == "Failed" + and r["Severity"] == "High" + for r in result["csv_data"] + ) + + def test_single_statement_dict_no_wildcard(self): + """Single-statement-dict policy without a wildcard → Passed/High, no crash.""" + cache = { + "role_permissions": { + "KBRole": { + "attached_policies": [], + "inline_policies": [ + { + "name": "KBInline", + "document": { + "Statement": { + "Effect": "Allow", + "Action": "bedrock:Retrieve", + "Resource": "arn:aws:bedrock:*:*:knowledge-base/*", + } + }, + } + ], + } + } + } + result = app.check_knowledge_base_iam_least_privilege(cache) + _assert_finding_structure(result) + assert result["status"] != "ERROR" + assert any( + r["Finding"] == "Knowledge Base IAM Permissions Look Appropriate" + and r["Status"] == "Passed" + for r in result["csv_data"] + ) + + def test_partial_wildcard_flagged(self): + """REQ-14/D: a partial wildcard (e.g. 'bedrock-agent:Get*') is over-broad + and must be flagged, not just the three exact full wildcards.""" + cache = { + "role_permissions": { + "KBRole": { + "attached_policies": [], + "inline_policies": [ + { + "name": "KBInline", + "document": { + "Statement": [ + { + "Effect": "Allow", + "Action": "bedrock-agent:Get*", + "Resource": "arn:aws:bedrock:*:*:knowledge-base/kb-1", + } + ] + }, + } + ], + } + } + } + result = app.check_knowledge_base_iam_least_privilege(cache) + _assert_finding_structure(result) + assert result["status"] == "WARN" + assert any( + r["Finding"] == "Overly Permissive Knowledge Base IAM Roles" + and r["Status"] == "Failed" + for r in result["csv_data"] + ) + + def test_unscoped_resource_flagged(self): + """REQ-14/D: a scoped action on Resource '*' (no ARN scoping) is flagged.""" + cache = { + "role_permissions": { + "KBRole": { + "attached_policies": [], + "inline_policies": [ + { + "name": "KBInline", + "document": { + "Statement": [ + { + "Effect": "Allow", + "Action": "bedrock:Retrieve", + "Resource": "*", + } + ] + }, + } + ], + } + } + } + result = app.check_knowledge_base_iam_least_privilege(cache) + _assert_finding_structure(result) + assert result["status"] == "WARN" + assert any(r["Status"] == "Failed" for r in result["csv_data"]) + + def test_not_action_allow_flagged(self): + """REQ-14/D: a NotAction Allow grants everything except listed actions and + is inherently over-broad → flagged.""" + cache = { + "role_permissions": { + "KBRole": { + "attached_policies": [], + "inline_policies": [ + { + "name": "KBInline", + "document": { + "Statement": [ + { + "Effect": "Allow", + "NotAction": "s3:DeleteObject", + "Resource": "arn:aws:bedrock:*:*:knowledge-base/kb-1", + } + ] + }, + } + ], + } + } + } + result = app.check_knowledge_base_iam_least_privilege(cache) + _assert_finding_structure(result) + assert result["status"] == "WARN" + assert any(r["Status"] == "Failed" for r in result["csv_data"]) + + def test_scoped_specific_actions_pass(self): + """REQ-14/D: properly scoped specific actions on a specific KB ARN → Passed + (no false positive from the widened detection).""" + cache = { + "role_permissions": { + "KBRole": { + "attached_policies": [], + "inline_policies": [ + { + "name": "KBInline", + "document": { + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "bedrock:Retrieve", + "bedrock:RetrieveAndGenerate", + ], + "Resource": "arn:aws:bedrock:us-east-1:111122223333:knowledge-base/kb-1", + } + ] + }, + } + ], + } + } + } + result = app.check_knowledge_base_iam_least_privilege(cache) + _assert_finding_structure(result) + assert result["status"] == "PASS" + assert any( + r["Finding"] == "Knowledge Base IAM Permissions Look Appropriate" + and r["Status"] == "Passed" + for r in result["csv_data"] + ) + + +class TestFS24KnowledgeBaseMetadataFiltering: + """FS-24 — Knowledge Base Metadata Filtering.""" + + def test_error_on_unavailable_inventory(self): + inv = make_resource_inventory( + knowledge_bases=app._Unavailable(RuntimeError("kb error")) + ) + result = app.check_knowledge_base_metadata_filtering(inv) + assert result["status"] == "ERROR" + + def test_returns_valid_structure_no_kbs(self): + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[], data_sources_by_kb={}, data_source_detail={} + ) + ) + result = app.check_knowledge_base_metadata_filtering(inv) + _assert_finding_structure(result) + + def test_returns_advisory_with_kbs(self): + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1", "name": "rag-kb"}], + data_sources_by_kb={}, + data_source_detail={}, + ) + ) + result = app.check_knowledge_base_metadata_filtering(inv) + _assert_finding_structure(result) + assert any("ADVISORY" in r.get("Finding", "") for r in result["csv_data"]) + + +class TestFS25OpensearchServerlessEncryption: + """FS-25 — OpenSearch Serverless Encryption.""" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("oss error") + result = app.check_opensearch_serverless_encryption() + assert result["status"] == "ERROR" + + +class TestFS26KnowledgeBaseVpcAccess: + """FS-26 — Knowledge Base VPC Access.""" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("vpc error") + result = app.check_knowledge_base_vpc_access() + assert result["status"] == "ERROR" + + +# ========================================================================= +# CATEGORY 6: NON-COMPLIANT OUTPUT (FS-27 to FS-30) +# ========================================================================= + + +class TestFS27GuardrailContextualGrounding: + """FS-27 — Guardrail Contextual Grounding Check (renamed from check_automated_reasoning_checks).""" + + def test_pass_grounding_configured(self): + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "finserv-guard"}], + detail_by_id={ + "g1": { + "contextualGroundingPolicy": { + "filters": [{"type": "GROUNDING", "threshold": 0.7}] + } + } + }, + ) + ) + result = app.check_guardrail_contextual_grounding(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + def test_warn_no_grounding(self): + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "finserv-guard"}], + detail_by_id={"g1": {}}, + ) + ) + result = app.check_guardrail_contextual_grounding(inv) + _assert_finding_structure(result) + assert result["status"] == "WARN" + + def test_error_on_exception(self): + inv = make_resource_inventory( + guardrails=app._Unavailable(RuntimeError("arc error")) + ) + result = app.check_guardrail_contextual_grounding(inv) + assert result["status"] == "ERROR" + + +class TestFS27AutomatedReasoningPolicies: + """FS-27b — Automated Reasoning Policies Check (new, GA August 2025).""" + + @patch("app.boto3.client") + def test_pass_policies_exist(self, mock_client): + c = MagicMock() + c.list_automated_reasoning_policies.return_value = { + "automatedReasoningPolicySummaries": [ + {"name": "loan-eligibility-policy", "policyId": "pol-001"} + ] + } + mock_client.return_value = c + result = app.check_automated_reasoning_policies() + _assert_finding_structure(result) + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_warn_no_policies(self, mock_client): + c = MagicMock() + c.list_automated_reasoning_policies.return_value = { + "automatedReasoningPolicySummaries": [] + } + mock_client.return_value = c + result = app.check_automated_reasoning_policies() + _assert_finding_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_access_denied_returns_na(self, mock_client): + c = MagicMock() + c.list_automated_reasoning_policies.side_effect = _client_error( + "AccessDeniedException" + ) + mock_client.return_value = c + result = app.check_automated_reasoning_policies() + _assert_finding_structure(result) + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("arc error") + result = app.check_automated_reasoning_policies() + assert result["status"] == "ERROR" + + +class TestFS28GuardrailDeniedTopicsFinancial: + """FS-28 — Guardrail Denied Topics Financial.""" + + def test_pass_denied_topics_configured(self): + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "finserv-guard"}], + detail_by_id={ + "g1": { + "topicPolicy": { + "topics": [{"name": "financial-advice", "type": "DENY"}], + "tier": {"tierName": "STANDARD"}, + } + } + }, + ) + ) + result = app.check_guardrail_denied_topics_financial(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + def test_classic_tier_advisory(self): + # Topics exist but on CLASSIC tier → Low-severity advisory (still Passed wrapper). + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "finserv-guard"}], + detail_by_id={ + "g1": { + "topicPolicy": { + "topics": [{"name": "financial-advice", "type": "DENY"}], + "tier": {"tierName": "CLASSIC"}, + } + } + }, + ) + ) + result = app.check_guardrail_denied_topics_financial(inv) + _assert_finding_structure(result) + assert any( + r["Finding"] == "Denied Topics Configured on CLASSIC Tier" + and r["Severity"] == "High" + for r in result["csv_data"] + ) + + def test_error_on_exception(self): + inv = make_resource_inventory( + guardrails=app._Unavailable(RuntimeError("topic error")) + ) + result = app.check_guardrail_denied_topics_financial(inv) + assert result["status"] == "ERROR" + + +class TestFS29ComplianceDisclaimer: + """FS-29 — Advisory check (no AWS API calls).""" + + def test_returns_valid_structure(self): + result = app.check_compliance_disclaimer_in_outputs() + _assert_finding_structure(result) + assert result["status"] == "PASS" + _assert_advisory_retag(result, "FS-29") + assert len(result["csv_data"]) >= 1 + assert result["csv_data"][0]["Check_ID"] == "FS-29" + + +class TestFS30BedrockEvalComplianceDatasets: + """FS-30 — Advisory (cannot inspect eval-job dataset content; REQ-10a).""" + + def test_returns_advisory_structure(self): + result = app.check_bedrock_evaluation_compliance_datasets() + _assert_finding_structure(result) + _assert_advisory_retag(result, "FS-30") + assert result["csv_data"][0]["Check_ID"] == "FS-30" + + +# ========================================================================= +# CATEGORY 7: MISINFORMATION (FS-31 to FS-34) +# ========================================================================= + + +class TestFS31KnowledgeBaseDataSourceSync: + """FS-31 — Knowledge Base Data Source Sync.""" + + def test_error_on_unavailable_inventory(self): + inv = make_resource_inventory( + knowledge_bases=app._Unavailable(RuntimeError("sync error")) + ) + result = app.check_knowledge_base_data_source_sync(inv) + assert result["status"] == "ERROR" + + def test_na_no_kbs(self): + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[], data_sources_by_kb={}, data_source_detail={} + ) + ) + result = app.check_knowledge_base_data_source_sync(inv) + _assert_finding_structure(result) + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + + +class TestFS32SourceAttribution: + """FS-32 — Advisory check (no AWS API calls).""" + + def test_returns_valid_structure(self): + result = app.check_source_attribution_in_guardrails() + _assert_finding_structure(result) + assert result["status"] == "PASS" + _assert_advisory_retag(result, "FS-32") + assert result["csv_data"][0]["Check_ID"] == "FS-32" + + +class TestFS33KnowledgeBaseIntegrityMonitoring: + """FS-33 — Knowledge Base Integrity Monitoring.""" + + def test_error_on_unavailable_inventory(self): + inv = make_resource_inventory( + knowledge_bases=app._Unavailable(RuntimeError("integrity error")) + ) + result = app.check_knowledge_base_integrity_monitoring(inv) + assert result["status"] == "ERROR" + + def test_na_no_kbs(self): + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[], data_sources_by_kb={}, data_source_detail={} + ) + ) + result = app.check_knowledge_base_integrity_monitoring(inv) + _assert_finding_structure(result) + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + + +class TestFS34FmVersionCurrency: + """FS-34 — FM Version Currency Advisory.""" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("fm error") + result = app.check_fm_version_currency() + assert result["status"] == "ERROR" + + +# ========================================================================= +# CATEGORY 8: ABUSIVE OR HARMFUL OUTPUT (FS-35 to FS-38) +# ========================================================================= + + +class TestFS35FmevalHarmfulContent: + """FS-35 — Advisory (cannot inspect eval-job dataset content; REQ-10a).""" + + def test_returns_advisory_structure(self): + result = app.check_fmeval_harmful_content() + _assert_finding_structure(result) + _assert_advisory_retag(result, "FS-35") + assert result["csv_data"][0]["Check_ID"] == "FS-35" + + +class TestFS36GuardrailContentFilters: + """FS-36 — Guardrail Content Filters.""" + + def test_pass_content_filters_configured(self): + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": { + "contentPolicy": { + "filters": [ + { + "type": "SEXUAL", + "inputStrength": "HIGH", + "outputStrength": "HIGH", + }, + { + "type": "VIOLENCE", + "inputStrength": "HIGH", + "outputStrength": "HIGH", + }, + { + "type": "HATE", + "inputStrength": "HIGH", + "outputStrength": "HIGH", + }, + { + "type": "INSULTS", + "inputStrength": "HIGH", + "outputStrength": "HIGH", + }, + ] + } + } + }, + ) + ) + result = app.check_guardrail_content_filters(inv) + _assert_finding_structure(result) + + def test_error_on_exception(self): + inv = make_resource_inventory( + guardrails=app._Unavailable(RuntimeError("content error")) + ) + result = app.check_guardrail_content_filters(inv) + assert result["status"] == "ERROR" + + +class TestFS37UserFeedbackMechanism: + """FS-37 — Advisory check (no AWS API calls).""" + + def test_returns_valid_structure(self): + result = app.check_user_feedback_mechanism() + _assert_finding_structure(result) + assert result["status"] == "PASS" + _assert_advisory_retag(result, "FS-37") + assert result["csv_data"][0]["Check_ID"] == "FS-37" + + +class TestFS38GuardrailWordFilters: + """FS-38 — Guardrail Word Filters.""" + + def test_error_on_exception(self): + inv = make_resource_inventory( + guardrails=app._Unavailable(RuntimeError("word error")) + ) + result = app.check_guardrail_word_filters(inv) + assert result["status"] == "ERROR" + + +# ========================================================================= +# CATEGORY 9: BIASED OUTPUT (FS-39 to FS-42) +# ========================================================================= + + +class TestFS39SagemakerClarifyBias: + """FS-39 — SageMaker Clarify Bias.""" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("clarify error") + result = app.check_sagemaker_clarify_bias() + assert result["status"] == "ERROR" + + +class TestFS40BedrockEvalBiasDatasets: + """FS-40 — Advisory (cannot inspect eval-job dataset content; REQ-10a).""" + + def test_returns_advisory_structure(self): + result = app.check_bedrock_evaluation_bias_datasets() + _assert_finding_structure(result) + _assert_advisory_retag(result, "FS-40") + assert result["csv_data"][0]["Check_ID"] == "FS-40" + + +class TestFS41SagemakerClarifyExplainability: + """FS-41 — SageMaker Clarify Explainability.""" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("explain error") + result = app.check_sagemaker_clarify_explainability() + assert result["status"] == "ERROR" + + +class TestFS42AiServiceCards: + """FS-42 — AI Service Cards Documentation Advisory.""" + + @patch("app.boto3.client") + def test_returns_valid_structure(self, mock_client): + c = MagicMock() + c.list_model_cards.return_value = {"ModelCardSummaries": []} + mock_client.return_value = c + result = app.check_ai_service_cards_documentation() + _assert_finding_structure(result) + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("cards error") + result = app.check_ai_service_cards_documentation() + assert result["status"] == "ERROR" + + +# ========================================================================= +# CATEGORY 10: SENSITIVE INFORMATION DISCLOSURE (FS-43 to FS-46) +# ========================================================================= + + +class TestFS43CloudwatchLogPiiMasking: + """FS-43 — CloudWatch Log PII Masking.""" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("logs error") + result = app.check_cloudwatch_log_pii_masking() + assert result["status"] == "ERROR" + + +class TestFS44MacieOnTrainingDataBuckets: + """FS-44 — Macie on Training Data Buckets.""" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("macie error") + result = app.check_macie_on_training_data_buckets() + assert result["status"] == "ERROR" + + +class TestFS45GuardrailPiiFilters: + """FS-45 — Guardrail PII Filters.""" + + def test_pass_pii_filters_configured(self): + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": { + "sensitiveInformationPolicy": { + "piiEntities": [ + {"type": "SSN", "action": "BLOCK"}, + {"type": "CREDIT_DEBIT_CARD_NUMBER", "action": "BLOCK"}, + ] + } + } + }, + ) + ) + result = app.check_guardrail_pii_filters(inv) + _assert_finding_structure(result) + + def test_error_on_exception(self): + inv = make_resource_inventory( + guardrails=app._Unavailable(RuntimeError("pii error")) + ) + result = app.check_guardrail_pii_filters(inv) + assert result["status"] == "ERROR" + + +class TestFS46DataClassificationTagging: + """FS-46 — Data Classification Tagging.""" + + def test_error_on_unavailable_inventory(self): + """When the buckets inventory is _Unavailable, check must return ERROR.""" + inv = make_resource_inventory( + buckets=app._Unavailable(RuntimeError("tag error")) + ) + result = app.check_data_classification_tagging(inv) + assert result["status"] == "ERROR" + + +# ========================================================================= +# CATEGORY 11: HALLUCINATION (FS-47 to FS-50) +# ========================================================================= + + +class TestFS47GuardrailGroundingThreshold: + """FS-47 — Guardrail Grounding Threshold.""" + + def test_error_on_exception(self): + inv = make_resource_inventory( + guardrails=app._Unavailable(RuntimeError("grounding error")) + ) + result = app.check_guardrail_grounding_threshold(inv) + assert result["status"] == "ERROR" + + +class TestFS48RagKnowledgeBaseConfigured: + """FS-48 — RAG Knowledge Base Configured.""" + + def test_pass_kbs_exist(self): + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[ + {"knowledgeBaseId": "kb1", "name": "rag-kb", "status": "ACTIVE"} + ], + data_sources_by_kb={}, + data_source_detail={}, + ) + ) + result = app.check_rag_knowledge_base_configured(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + def test_error_on_unavailable_inventory(self): + inv = make_resource_inventory( + knowledge_bases=app._Unavailable(RuntimeError("rag error")) + ) + result = app.check_rag_knowledge_base_configured(inv) + assert result["status"] == "ERROR" + + +class TestFS49HallucinationDisclaimer: + """FS-49 — Advisory check (no AWS API calls).""" + + def test_returns_valid_structure(self): + result = app.check_hallucination_disclaimer_advisory() + _assert_finding_structure(result) + assert result["status"] == "PASS" + _assert_advisory_retag(result, "FS-49") + assert result["csv_data"][0]["Check_ID"] == "FS-49" + + +class TestFS50GuardrailRelevanceGrounding: + """FS-50 — Guardrail Relevance Grounding Check (renamed from check_automated_reasoning_checks_hallucination).""" + + def test_pass_relevance_filter_present(self): + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": { + "contextualGroundingPolicy": { + "filters": [{"type": "RELEVANCE", "threshold": 0.7}] + } + } + }, + ) + ) + result = app.check_guardrail_relevance_grounding(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + def test_warn_no_relevance_filter(self): + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": { + "contextualGroundingPolicy": { + "filters": [{"type": "GROUNDING", "threshold": 0.7}] + } + } + }, + ) + ) + result = app.check_guardrail_relevance_grounding(inv) + _assert_finding_structure(result) + assert result["status"] == "WARN" + + def test_error_on_exception(self): + inv = make_resource_inventory( + guardrails=app._Unavailable(RuntimeError("arc error")) + ) + result = app.check_guardrail_relevance_grounding(inv) + assert result["status"] == "ERROR" + + +# ========================================================================= +# CATEGORY 12: PROMPT INJECTION (FS-51 to FS-54) +# ========================================================================= + + +class TestFS51PromptInjectionInputValidation: + """FS-51 — Prompt Injection Input Validation.""" + + def test_pass_prompt_attack_filter(self): + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": { + "contentPolicy": { + "filters": [ + { + "type": "PROMPT_ATTACK", + "inputStrength": "HIGH", + "outputStrength": "NONE", + } + ] + } + } + }, + ) + ) + result = app.check_prompt_injection_input_validation(inv) + _assert_finding_structure(result) + + def test_error_on_exception(self): + inv = make_resource_inventory( + guardrails=app._Unavailable(RuntimeError("prompt error")) + ) + result = app.check_prompt_injection_input_validation(inv) + assert result["status"] == "ERROR" + + +class TestFS52BedrockSdkVersionCurrency: + """FS-52 — Bedrock SDK Version Currency Advisory.""" + + def test_error_on_unavailable_inventory(self): + inv = make_resource_inventory( + lambda_functions=app._Unavailable(RuntimeError("sdk error")) + ) + result = app.check_bedrock_sdk_version_currency(inv) + assert result["status"] == "ERROR" + + +class TestFS53WafSqlInjectionRules: + """FS-53 — WAF SQL Injection Rules.""" + + def test_pass_managed_rules_present(self): + """ACL with AWSManagedRulesSQLiRuleSet in inventory → PASS.""" + acl_detail = { + "Rules": [ + { + "Name": "AWS-AWSManagedRulesSQLiRuleSet", + "Statement": { + "ManagedRuleGroupStatement": { + "VendorName": "AWS", + "Name": "AWSManagedRulesSQLiRuleSet", + } + }, + } + ] + } + inv = make_resource_inventory( + web_acls=app.WebAclInventory( + summaries=[{"Name": "acl1", "Id": "id1"}], + detail_by_id={"id1": acl_detail}, + ) + ) + result = app.check_waf_sql_injection_rules(inv) + _assert_finding_structure(result) + + def test_na_no_acls(self): + """Empty summaries list → N/A (no ACLs found).""" + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=[], detail_by_id={}) + ) + result = app.check_waf_sql_injection_rules(inv) + _assert_finding_structure(result) + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + + def test_warn_acls_without_injection_rules(self): + """ACL with no injection rule groups → WARN.""" + acl_detail = { + "Rules": [ + { + "Name": "rate-limit", + "Statement": {"RateBasedStatement": {"Limit": 1000}}, + } + ] + } + inv = make_resource_inventory( + web_acls=app.WebAclInventory( + summaries=[{"Name": "acl1", "Id": "id1"}], + detail_by_id={"id1": acl_detail}, + ) + ) + result = app.check_waf_sql_injection_rules(inv) + _assert_finding_structure(result) + assert result["status"] == "WARN" + + def test_error_on_exception(self): + """Unavailable inventory → COULD_NOT_ASSESS (ERROR envelope).""" + inv = make_resource_inventory( + web_acls=app._Unavailable(RuntimeError("waf error")) + ) + result = app.check_waf_sql_injection_rules(inv) + assert result["status"] == "ERROR" + + # --- Pagination-correctness proof: >100 ACLs --- + def test_pass_more_than_100_acls_all_have_rules(self): + """Inventory with 150 ACLs all having injection rules → PASS (all pages visible).""" + acl_detail = { + "Rules": [ + { + "Statement": { + "ManagedRuleGroupStatement": { + "Name": "AWSManagedRulesSQLiRuleSet" + } + } + } + ] + } + summaries = [{"Name": f"acl{i}", "Id": f"id{i}"} for i in range(150)] + detail_by_id = {f"id{i}": acl_detail for i in range(150)} + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=summaries, detail_by_id=detail_by_id) + ) + result = app.check_waf_sql_injection_rules(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + assert any( + r["Finding"] == "WAF Injection Protection Rules Present" + and "150" in r["Finding_Details"] + for r in result["csv_data"] + ) + + def test_warn_more_than_100_acls_some_missing_rules(self): + """Inventory with 150 ACLs, last 50 missing rules → WARN (previously truncated acls now detected).""" + good_detail = { + "Rules": [ + { + "Statement": { + "ManagedRuleGroupStatement": { + "Name": "AWSManagedRulesSQLiRuleSet" + } + } + } + ] + } + bad_detail = {"Rules": []} + summaries = [{"Name": f"acl{i}", "Id": f"id{i}"} for i in range(150)] + detail_by_id = {} + for i in range(100): + detail_by_id[f"id{i}"] = good_detail + for i in range(100, 150): + detail_by_id[f"id{i}"] = bad_detail + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=summaries, detail_by_id=detail_by_id) + ) + result = app.check_waf_sql_injection_rules(inv) + _assert_finding_structure(result) + assert result["status"] == "WARN" + + # --- ≤1-page equivalence: 2-ACL case matches Wave-0 baseline --- + def test_two_acl_case_unchanged(self): + """The 2-ACL scenario produces the same PASS finding as the pre-refactor baseline.""" + acl_detail = { + "Rules": [ + { + "Statement": { + "ManagedRuleGroupStatement": { + "Name": "AWSManagedRulesSQLiRuleSet" + } + } + } + ] + } + inv = make_resource_inventory( + web_acls=app.WebAclInventory( + summaries=[ + {"Name": "FinServACL1", "Id": "acl-id-001"}, + {"Name": "FinServACL2", "Id": "acl-id-002"}, + ], + detail_by_id={ + "acl-id-001": acl_detail, + "acl-id-002": acl_detail, + }, + ) + ) + result = app.check_waf_sql_injection_rules(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + assert any( + r["Finding"] == "WAF Injection Protection Rules Present" + for r in result["csv_data"] + ) + + +class TestFS54PenetrationTestingEvidence: + """FS-54 — Advisory check (no AWS API calls).""" + + def test_returns_valid_structure(self): + result = app.check_penetration_testing_evidence() + _assert_finding_structure(result) + assert result["status"] == "PASS" + _assert_advisory_retag(result, "FS-54") + assert result["csv_data"][0]["Check_ID"] == "FS-54" + + +# ========================================================================= +# CATEGORY 13: IMPROPER OUTPUT HANDLING (FS-55 to FS-58) +# ========================================================================= + + +class TestFS55OutputValidationLambda: + """FS-55 — Output Validation Lambda Check.""" + + def test_pass_validation_functions_exist(self): + inv = make_resource_inventory( + lambda_functions=[{"FunctionName": "output-validate-handler"}] + ) + result = app.check_output_validation_lambda(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + def test_warn_no_validation_functions(self): + inv = make_resource_inventory( + lambda_functions=[{"FunctionName": "my-api-handler"}] + ) + result = app.check_output_validation_lambda(inv) + _assert_finding_structure(result) + assert result["status"] == "WARN" + + def test_error_on_unavailable_inventory(self): + inv = make_resource_inventory( + lambda_functions=app._Unavailable(RuntimeError("lambda error")) + ) + result = app.check_output_validation_lambda(inv) + assert result["status"] == "ERROR" + + +class TestFS56XssPreventionWaf: + """FS-56 — XSS Prevention WAF Check.""" + + def test_pass_acls_present(self): + """ACL with AWSManagedRulesCommonRuleSet in inventory → PASS.""" + acl_detail = { + "Rules": [ + { + "Statement": { + "ManagedRuleGroupStatement": { + "Name": "AWSManagedRulesCommonRuleSet" + } + } + } + ] + } + inv = make_resource_inventory( + web_acls=app.WebAclInventory( + summaries=[{"Name": "acl1", "Id": "id1"}], + detail_by_id={"id1": acl_detail}, + ) + ) + result = app.check_xss_prevention_waf(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + assert any( + r["Finding"] == "XSS Prevention Common Rule Set Present" + for r in result["csv_data"] + ) + + def test_fail_acls_without_common_rule_set(self): + """ACL without AWSManagedRulesCommonRuleSet → FAIL.""" + acl_detail = {"Rules": []} + inv = make_resource_inventory( + web_acls=app.WebAclInventory( + summaries=[{"Name": "acl1", "Id": "id1"}], + detail_by_id={"id1": acl_detail}, + ) + ) + result = app.check_xss_prevention_waf(inv) + _assert_finding_structure(result) + assert any( + r["Finding"] == "WAF ACLs Missing Common Rule Set (XSS)" + and r["Status"] == "Failed" + and r["Severity"] == "Medium" + for r in result["csv_data"] + ) + + def test_na_no_acls(self): + """Empty summaries → N/A.""" + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=[], detail_by_id={}) + ) + result = app.check_xss_prevention_waf(inv) + _assert_finding_structure(result) + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + + def test_error_on_exception(self): + """Unavailable inventory → COULD_NOT_ASSESS (ERROR envelope).""" + inv = make_resource_inventory( + web_acls=app._Unavailable(RuntimeError("xss error")) + ) + result = app.check_xss_prevention_waf(inv) + assert result["status"] == "ERROR" + + # --- Pagination-correctness proof: >100 ACLs --- + def test_pass_more_than_100_acls_all_have_common_rule_set(self): + """150 ACLs all with AWSManagedRulesCommonRuleSet → PASS (previously truncated ACLs visible).""" + acl_detail = { + "Rules": [ + { + "Statement": { + "ManagedRuleGroupStatement": { + "Name": "AWSManagedRulesCommonRuleSet" + } + } + } + ] + } + summaries = [{"Name": f"acl{i}", "Id": f"id{i}"} for i in range(150)] + detail_by_id = {f"id{i}": acl_detail for i in range(150)} + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=summaries, detail_by_id=detail_by_id) + ) + result = app.check_xss_prevention_waf(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + def test_warn_more_than_100_acls_some_missing_xss_rules(self): + """150 ACLs, last 50 missing AWSManagedRulesCommonRuleSet → WARN (pagination fix detects them).""" + good_detail = { + "Rules": [ + { + "Statement": { + "ManagedRuleGroupStatement": { + "Name": "AWSManagedRulesCommonRuleSet" + } + } + } + ] + } + bad_detail = {"Rules": []} + summaries = [{"Name": f"acl{i}", "Id": f"id{i}"} for i in range(150)] + detail_by_id = {} + for i in range(100): + detail_by_id[f"id{i}"] = good_detail + for i in range(100, 150): + detail_by_id[f"id{i}"] = bad_detail + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=summaries, detail_by_id=detail_by_id) + ) + result = app.check_xss_prevention_waf(inv) + _assert_finding_structure(result) + assert result["status"] == "WARN" + + # --- ≤1-page equivalence: 2-ACL case matches Wave-0 baseline --- + def test_two_acl_case_unchanged(self): + """The 2-ACL PASS scenario is unchanged vs the pre-refactor baseline.""" + acl_detail = { + "Rules": [ + { + "Statement": { + "ManagedRuleGroupStatement": { + "Name": "AWSManagedRulesCommonRuleSet" + } + } + } + ] + } + inv = make_resource_inventory( + web_acls=app.WebAclInventory( + summaries=[ + {"Name": "FinServACL1", "Id": "acl-id-001"}, + {"Name": "FinServACL2", "Id": "acl-id-002"}, + ], + detail_by_id={ + "acl-id-001": acl_detail, + "acl-id-002": acl_detail, + }, + ) + ) + result = app.check_xss_prevention_waf(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + assert any( + r["Finding"] == "XSS Prevention Common Rule Set Present" + for r in result["csv_data"] + ) + + +class TestFS57OutputEncodingAdvisory: + """FS-57 — Advisory check (no AWS API calls).""" + + def test_returns_valid_structure(self): + result = app.check_output_encoding_advisory() + _assert_finding_structure(result) + assert result["status"] == "PASS" + _assert_advisory_retag(result, "FS-57") + assert result["csv_data"][0]["Check_ID"] == "FS-57" + + +class TestFS58OutputSchemaValidation: + """FS-58 — Output Schema Validation Check.""" + + def test_returns_valid_structure(self): + inv = make_resource_inventory( + lambda_functions=[{"FunctionName": "schema-validate-fn"}] + ) + result = app.check_output_schema_validation(inv) + _assert_finding_structure(result) + # REQ-2: FS-58 is advisory — N/A + Informational + "ADVISORY: " prefix, never Passed. + _assert_advisory_retag(result, "FS-58") + assert not any(r["Status"] == "Passed" for r in result["csv_data"]) + + def test_error_on_unavailable_inventory(self): + inv = make_resource_inventory( + lambda_functions=app._Unavailable(RuntimeError("schema error")) + ) + result = app.check_output_schema_validation(inv) + assert result["status"] == "ERROR" + + +# ========================================================================= +# CATEGORY 14: OFF-TOPIC & INAPPROPRIATE OUTPUT (FS-59 to FS-60) +# ========================================================================= + + +class TestFS59GuardrailTopicAllowlist: + """FS-59 — Guardrail Topic Allowlist Check.""" + + def test_pass_topics_configured(self): + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": { + "topicPolicy": { + "topics": [{"name": "medical-advice", "type": "DENY"}], + "tier": {"tierName": "STANDARD"}, + } + } + }, + ) + ) + result = app.check_guardrail_topic_allowlist(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + def test_classic_tier_advisory(self): + # Topics on CLASSIC tier → Low advisory finding (wrapper stays PASS). + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": { + "topicPolicy": { + "topics": [{"name": "medical-advice", "type": "DENY"}], + "tier": {"tierName": "CLASSIC"}, + } + } + }, + ) + ) + result = app.check_guardrail_topic_allowlist(inv) + _assert_finding_structure(result) + assert any( + r["Finding"] == "Topic Restrictions Configured on CLASSIC Tier" + and r["Severity"] == "Medium" + for r in result["csv_data"] + ) + + def test_warn_no_topics(self): + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={"g1": {"topicPolicy": {"topics": []}}}, + ) + ) + result = app.check_guardrail_topic_allowlist(inv) + _assert_finding_structure(result) + assert result["status"] == "WARN" + + def test_na_no_guardrails(self): + inv = make_resource_inventory( + guardrails=app.GuardrailInventory(summaries=[], detail_by_id={}) + ) + result = app.check_guardrail_topic_allowlist(inv) + _assert_finding_structure(result) + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + + def test_error_on_exception(self): + inv = make_resource_inventory( + guardrails=app._Unavailable(RuntimeError("topic error")) + ) + result = app.check_guardrail_topic_allowlist(inv) + assert result["status"] == "ERROR" + + +class TestFS60ContextualGroundingForOfftopic: + """FS-60 — Advisory check (no AWS API calls).""" + + def test_returns_valid_structure(self): + result = app.check_contextual_grounding_for_offtopic() + _assert_finding_structure(result) + assert result["status"] == "PASS" + _assert_advisory_retag(result, "FS-60") + assert result["csv_data"][0]["Check_ID"] == "FS-60" + + +# ========================================================================= +# CATEGORY 15: OUT-OF-DATE TRAINING DATA (FS-61 to FS-63) +# ========================================================================= + + +class TestFS61KnowledgeBaseSyncSchedule: + """FS-61 — Knowledge Base Sync Schedule Check.""" + + @patch("app.boto3.client") + def test_pass_sync_rules_exist(self, mock_client): + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1"}], + data_sources_by_kb={}, + data_source_detail={}, + ) + ) + + def side_effect(service, **kwargs): + if service == "events": + c = MagicMock() + c.list_rules.return_value = { + "Rules": [{"Name": "bedrock-kb-sync-daily"}] + } + return c + if service == "scheduler": + c = MagicMock() + c.list_schedules.return_value = {"Schedules": []} + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_knowledge_base_sync_schedule(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_pass_scheduler_schedule_exists(self, mock_client): + # No legacy EventBridge rule, but an EventBridge Scheduler schedule targets + # KB sync — the AWS-recommended approach must be detected (no false WARN). + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1"}], + data_sources_by_kb={}, + data_source_detail={}, + ) + ) + + def side_effect(service, **kwargs): + if service == "events": + c = MagicMock() + c.list_rules.return_value = {"Rules": []} + return c + if service == "scheduler": + c = MagicMock() + c.list_schedules.return_value = { + "Schedules": [ + { + "Name": "bedrock-kb-ingestion-daily", + "Target": { + "Arn": "arn:aws:lambda:us-east-1:1:function:sync" + }, + } + ] + } + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_knowledge_base_sync_schedule(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_warn_no_sync_rules(self, mock_client): + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1"}], + data_sources_by_kb={}, + data_source_detail={}, + ) + ) + + def side_effect(service, **kwargs): + if service == "events": + c = MagicMock() + c.list_rules.return_value = {"Rules": [{"Name": "unrelated-rule"}]} + return c + if service == "scheduler": + c = MagicMock() + c.list_schedules.return_value = { + "Schedules": [ + {"Name": "unrelated-schedule", "Target": {"Arn": "x"}} + ] + } + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_knowledge_base_sync_schedule(inv) + _assert_finding_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_scheduler_access_denied_falls_back_to_rules(self, mock_client): + # scheduler:ListSchedules denied → fall back to EventBridge rules only, + # do NOT error the whole check. + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1"}], + data_sources_by_kb={}, + data_source_detail={}, + ) + ) + + def side_effect(service, **kwargs): + if service == "events": + c = MagicMock() + c.list_rules.return_value = { + "Rules": [{"Name": "bedrock-kb-sync-daily"}] + } + return c + if service == "scheduler": + c = MagicMock() + c.list_schedules.side_effect = _client_error("AccessDeniedException") + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_knowledge_base_sync_schedule(inv) + _assert_finding_structure(result) + # EventBridge rule still matched → PASS despite scheduler access denial. + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_scheduler_access_denied_no_rules_could_not_assess(self, mock_client): + """REQ-11/A3: scheduler:ListSchedules denied AND no matching EventBridge + rule → we cannot conclude absence → COULD_NOT_ASSESS (check returns ERROR + so the handler synthesizes the N/A row), NOT a false Failed.""" + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1"}], + data_sources_by_kb={}, + data_source_detail={}, + ) + ) + + def side_effect(service, **kwargs): + if service == "events": + c = MagicMock() + c.list_rules.return_value = {"Rules": []} + return c + if service == "scheduler": + c = MagicMock() + c.list_schedules.side_effect = _client_error("AccessDeniedException") + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_knowledge_base_sync_schedule(inv) + # Re-raised access error → ERROR envelope (handler → COULD NOT ASSESS row). + assert result["status"] == "ERROR" + assert not any(r.get("Status") == "Failed" for r in result.get("csv_data", [])) + + def test_error_on_unavailable_inventory(self): + inv = make_resource_inventory( + knowledge_bases=app._Unavailable(RuntimeError("sync error")) + ) + result = app.check_knowledge_base_sync_schedule(inv) + assert result["status"] == "ERROR" + + +class TestFS62DataCurrencyDisclaimer: + """FS-62 — Advisory check (no AWS API calls).""" + + def test_returns_valid_structure(self): + result = app.check_data_currency_disclaimer_advisory() + _assert_finding_structure(result) + assert result["status"] == "PASS" + _assert_advisory_retag(result, "FS-62") + assert result["csv_data"][0]["Check_ID"] == "FS-62" + + +class TestFS63FoundationModelLifecyclePolicy: + """FS-63 — Foundation Model Lifecycle Policy Check.""" + + @patch("app.boto3.client") + def test_pass_no_legacy_models(self, mock_client): + c = MagicMock() + c.list_foundation_models.return_value = { + "modelSummaries": [ + { + "modelId": "anthropic.claude-v2", + "modelLifecycle": {"status": "ACTIVE"}, + } + ] + } + c.describe_config_rules.return_value = {"ConfigRules": []} + mock_client.return_value = c + result = app.check_foundation_model_lifecycle_policy() + _assert_finding_structure(result) + + @patch("app.boto3.client") + def test_warn_legacy_models_no_rules(self, mock_client): + c = MagicMock() + c.list_foundation_models.return_value = { + "modelSummaries": [ + {"modelId": "old-model-v1", "modelLifecycle": {"status": "LEGACY"}} + ] + } + c.describe_config_rules.return_value = {"ConfigRules": []} + mock_client.return_value = c + result = app.check_foundation_model_lifecycle_policy() + _assert_finding_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("lifecycle error") + result = app.check_foundation_model_lifecycle_policy() + assert result["status"] == "ERROR" + + +# ========================================================================= +# MATERIAL GAP CHECKS (FS-65 to FS-69) +# ========================================================================= + + +class TestFS65KbDatasourceS3EventNotifications: + """FS-65 — KB Data Source S3 Event Notifications Check.""" + + def test_na_no_kbs(self): + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[], data_sources_by_kb={}, data_source_detail={} + ) + ) + result = app.check_kb_datasource_s3_event_notifications(inv) + _assert_finding_structure(result) + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_pass_notifications_configured(self, mock_client): + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1"}], + data_sources_by_kb={"kb1": [{"dataSourceId": "ds1", "name": "s3-src"}]}, + data_source_detail={ + ("kb1", "ds1"): { + "dataSource": { + "dataSourceConfiguration": { + "s3Configuration": { + "bucketArn": "arn:aws:s3:::my-kb-bucket" + } + } + } + } + }, + ) + ) + + def side_effect(service, **kwargs): + if service == "s3": + c = MagicMock() + c.get_bucket_notification_configuration.return_value = { + "EventBridgeConfiguration": {"EventBridgeEnabled": True} + } + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_kb_datasource_s3_event_notifications(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_warn_no_notifications(self, mock_client): + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1"}], + data_sources_by_kb={"kb1": [{"dataSourceId": "ds1", "name": "s3-src"}]}, + data_source_detail={ + ("kb1", "ds1"): { + "dataSource": { + "dataSourceConfiguration": { + "s3Configuration": { + "bucketArn": "arn:aws:s3:::my-kb-bucket" + } + } + } + } + }, + ) + ) + + def side_effect(service, **kwargs): + if service == "s3": + c = MagicMock() + c.get_bucket_notification_configuration.return_value = {} + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_kb_datasource_s3_event_notifications(inv) + _assert_finding_structure(result) + assert result["status"] == "WARN" + + def test_error_on_unavailable_inventory(self): + inv = make_resource_inventory( + knowledge_bases=app._Unavailable(RuntimeError("s3 event error")) + ) + result = app.check_kb_datasource_s3_event_notifications(inv) + assert result["status"] == "ERROR" + + +class TestFS66AgentcoreEndUserIdentityPropagation: + """FS-66 — AgentCore End-User Identity Propagation Check.""" + + @patch("app.boto3.client") + def test_pass_authorizer_configured(self, mock_client): + c = MagicMock() + c.list_agent_runtimes.return_value = { + "agentRuntimes": [ + { + "agentRuntimeName": "rt1", + "authorizerConfiguration": { + "customJWTAuthorizer": {"issuerUrl": "https://example.com"} + }, + } + ] + } + mock_client.return_value = c + result = app.check_agentcore_end_user_identity_propagation() + _assert_finding_structure(result) + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_warn_no_authorizer(self, mock_client): + c = MagicMock() + c.list_agent_runtimes.return_value = { + "agentRuntimes": [ + {"agentRuntimeName": "rt1", "authorizerConfiguration": {}} + ] + } + mock_client.return_value = c + result = app.check_agentcore_end_user_identity_propagation() + _assert_finding_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_access_denied_returns_na(self, mock_client): + c = MagicMock() + c.list_agent_runtimes.side_effect = _client_error("AccessDeniedException") + mock_client.return_value = c + result = app.check_agentcore_end_user_identity_propagation() + _assert_finding_structure(result) + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_error_on_exception(self, mock_client): + mock_client.side_effect = RuntimeError("identity error") + result = app.check_agentcore_end_user_identity_propagation() + assert result["status"] == "ERROR" + + +class TestFS67AgentFinancialTransactionThresholds: + """FS-67 — Agent Financial Transaction Value Thresholds Check.""" + + def test_pass_threshold_env_vars(self): + inv = make_resource_inventory( + lambda_functions=[ + { + "FunctionName": "agent-transaction-handler", + "Environment": {"Variables": {"MAX_TRANSACTION_AMOUNT": "10000"}}, + } + ] + ) + result = app.check_agent_financial_transaction_thresholds(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + def test_warn_no_threshold_env_vars(self): + inv = make_resource_inventory( + lambda_functions=[ + { + "FunctionName": "agent-transaction-handler", + "Environment": {"Variables": {"LOG_LEVEL": "INFO"}}, + } + ] + ) + result = app.check_agent_financial_transaction_thresholds(inv) + _assert_finding_structure(result) + assert result["status"] == "WARN" + + def test_na_no_agent_lambdas(self): + inv = make_resource_inventory( + lambda_functions=[{"FunctionName": "unrelated-function"}] + ) + result = app.check_agent_financial_transaction_thresholds(inv) + _assert_finding_structure(result) + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + + def test_error_on_unavailable_inventory(self): + inv = make_resource_inventory( + lambda_functions=app._Unavailable(RuntimeError("transaction error")) + ) + result = app.check_agent_financial_transaction_thresholds(inv) + assert result["status"] == "ERROR" + + +class TestFS68ApiGatewayRequestBodySizeLimits: + """FS-68 — API Gateway Request Body Size Limits Check.""" + + def _size_constraint_detail(self): + return { + "Rules": [ + { + "Name": "body-size-rule", + "Statement": { + "SizeConstraintStatement": { + "FieldToMatch": {"Body": {}}, + "ComparisonOperator": "LE", + "Size": 8192, + } + }, + } + ] + } + + def test_pass_validators_and_waf_rules(self): + """API with maxLength model + WAF ACL with SizeConstraint → PASS.""" + acl_detail = self._size_constraint_detail() + inv = make_resource_inventory( + web_acls=app.WebAclInventory( + summaries=[{"Name": "acl1", "Id": "id1"}], + detail_by_id={"id1": acl_detail}, + ) + ) + with patch("app.boto3.client") as mock_client: + + def side_effect(service, **kwargs): + if service == "apigateway": + c = MagicMock() + c.get_rest_apis.return_value = { + "items": [{"id": "api1", "name": "genai-api"}] + } + c.get_request_validators.return_value = { + "items": [{"id": "v1", "name": "body-validator"}] + } + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_api_gateway_request_body_size_limits(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + def test_na_no_rest_apis_no_waf(self): + """REQ-4: zero REST APIs AND zero WAF ACLs → N/A (not a false Passed).""" + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=[], detail_by_id={}) + ) + with patch("app.boto3.client") as mock_client: + + def side_effect(service, **kwargs): + if service == "apigateway": + c = MagicMock() + c.get_rest_apis.return_value = {"items": []} + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_api_gateway_request_body_size_limits(inv) + _assert_finding_structure(result) + assert any( + r["Finding"] == "API Gateway Request Body Size Limits — Not Applicable" + and r["Status"] == "N/A" + and r["Severity"] == "Informational" + for r in result["csv_data"] + ) + assert not any(r["Status"] == "Passed" for r in result["csv_data"]) + + def test_fail_rest_api_without_validator(self): + """REST API exists but has no request validator → Failed/Medium.""" + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=[], detail_by_id={}) + ) + with patch("app.boto3.client") as mock_client: + + def side_effect(service, **kwargs): + if service == "apigateway": + c = MagicMock() + c.get_rest_apis.return_value = { + "items": [{"id": "api1", "name": "genai-api"}] + } + c.get_request_validators.return_value = {"items": []} + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_api_gateway_request_body_size_limits(inv) + _assert_finding_structure(result) + assert any( + r["Finding"] == "API Gateway Request Body Size Limits Not Enforced" + and r["Status"] == "Failed" + and r["Severity"] == "Medium" + for r in result["csv_data"] + ) + + def test_validator_presence_without_size_model_not_passed(self): + """REQ-11/A1: validator without maxLength bound → Failed, not Passed.""" + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=[], detail_by_id={}) + ) + with patch("app.boto3.client") as mock_client: + + def side_effect(service, **kwargs): + if service == "apigateway": + c = MagicMock() + c.get_rest_apis.return_value = { + "items": [{"id": "api1", "name": "genai-api"}] + } + c.get_request_validators.return_value = { + "items": [ + { + "id": "v1", + "name": "body-validator", + "validateRequestBody": True, + } + ] + } + c.get_models.return_value = { + "items": [{"name": "Empty", "schema": '{"type":"object"}'}] + } + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_api_gateway_request_body_size_limits(inv) + _assert_finding_structure(result) + assert not any(r["Status"] == "Passed" for r in result["csv_data"]) + assert any( + r["Finding"] == "API Gateway Request Body Size Limits Not Enforced" + and r["Status"] == "Failed" + for r in result["csv_data"] + ) + + def test_validator_with_maxlength_model_passed(self): + """REQ-11/A1: validator with maxLength model IS a real size control → Passed.""" + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=[], detail_by_id={}) + ) + with patch("app.boto3.client") as mock_client: + + def side_effect(service, **kwargs): + if service == "apigateway": + c = MagicMock() + c.get_rest_apis.return_value = { + "items": [{"id": "api1", "name": "genai-api"}] + } + c.get_request_validators.return_value = { + "items": [{"id": "v1", "validateRequestBody": True}] + } + c.get_models.return_value = { + "items": [ + { + "name": "Prompt", + "schema": '{"type":"object","properties":{"prompt":{"type":"string","maxLength":4000}}}', + } + ] + } + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_api_gateway_request_body_size_limits(inv) + _assert_finding_structure(result) + assert any( + r["Finding"] == "API Gateway Request Body Size Limits Configured" + and r["Status"] == "Passed" + for r in result["csv_data"] + ) + + def test_waf_oversize_constraint_above_window_not_credited(self): + """REQ-11/A2: GT body SizeConstraint above 16 KB with CONTINUE oversize → not credited → Failed.""" + bad_detail = { + "Rules": [ + { + "Name": "too-big", + "Statement": { + "SizeConstraintStatement": { + "FieldToMatch": {"Body": {"OversizeHandling": "CONTINUE"}}, + "ComparisonOperator": "GT", + "Size": 32768, + } + }, + } + ] + } + inv = make_resource_inventory( + web_acls=app.WebAclInventory( + summaries=[{"Name": "acl1", "Id": "id1"}], + detail_by_id={"id1": bad_detail}, + ) + ) + with patch("app.boto3.client") as mock_client: + + def side_effect(service, **kwargs): + if service == "apigateway": + c = MagicMock() + c.get_rest_apis.return_value = { + "items": [{"id": "api1", "name": "genai-api"}] + } + c.get_request_validators.return_value = {"items": []} + c.get_models.return_value = {"items": []} + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_api_gateway_request_body_size_limits(inv) + _assert_finding_structure(result) + assert not any(r["Status"] == "Passed" for r in result["csv_data"]) + assert any(r["Status"] == "Failed" for r in result["csv_data"]) + + def test_waf_body_substring_rule_not_credited(self): + """REQ-11/A2: XSS match on body is NOT a SizeConstraint → not credited.""" + xss_detail = { + "Rules": [ + { + "Name": "xss-on-body", + "Statement": { + "XssMatchStatement": { + "FieldToMatch": {"Body": {}}, + "TextTransformations": [{"Priority": 0, "Type": "NONE"}], + } + }, + } + ] + } + inv = make_resource_inventory( + web_acls=app.WebAclInventory( + summaries=[{"Name": "acl1", "Id": "id1"}], + detail_by_id={"id1": xss_detail}, + ) + ) + with patch("app.boto3.client") as mock_client: + + def side_effect(service, **kwargs): + if service == "apigateway": + c = MagicMock() + c.get_rest_apis.return_value = { + "items": [{"id": "api1", "name": "genai-api"}] + } + c.get_request_validators.return_value = {"items": []} + c.get_models.return_value = {"items": []} + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_api_gateway_request_body_size_limits(inv) + _assert_finding_structure(result) + assert not any(r["Status"] == "Passed" for r in result["csv_data"]) + + def test_error_on_unavailable_inventory(self): + """Unavailable inventory → COULD_NOT_ASSESS (ERROR envelope).""" + inv = make_resource_inventory( + web_acls=app._Unavailable(RuntimeError("apigw error")) + ) + with patch("app.boto3.client") as mock_client: + + def side_effect(service, **kwargs): + if service == "apigateway": + c = MagicMock() + c.get_rest_apis.return_value = {"items": []} + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_api_gateway_request_body_size_limits(inv) + assert result["status"] == "ERROR" + + def test_error_on_apigw_exception(self): + """apigateway client raises → overall ERROR.""" + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=[], detail_by_id={}) + ) + with patch("app.boto3.client") as mock_client: + mock_client.side_effect = RuntimeError("apigw error") + result = app.check_api_gateway_request_body_size_limits(inv) + assert result["status"] == "ERROR" + + # --- Pagination-correctness proof: >100 ACLs --- + def test_pass_more_than_100_acls_with_size_constraints(self): + """150 ACLs all with SizeConstraint → PASS (previously truncated ACLs now assessed).""" + acl_detail = self._size_constraint_detail() + summaries = [{"Name": f"acl{i}", "Id": f"id{i}"} for i in range(150)] + detail_by_id = {f"id{i}": acl_detail for i in range(150)} + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=summaries, detail_by_id=detail_by_id) + ) + with patch("app.boto3.client") as mock_client: + + def side_effect(service, **kwargs): + if service == "apigateway": + c = MagicMock() + c.get_rest_apis.return_value = {"items": []} + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_api_gateway_request_body_size_limits(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + # --- ≤1-page equivalence: 2-ACL case matches Wave-0 baseline --- + def test_two_acl_case_unchanged(self): + """The 2-ACL PASS scenario is unchanged vs the pre-refactor baseline.""" + from test_inventory_equivalence import _acl_detail as _baseline_acl_detail + + inv = make_resource_inventory( + web_acls=app.WebAclInventory( + summaries=[ + {"Name": "FinServACL1", "Id": "acl-id-001"}, + {"Name": "FinServACL2", "Id": "acl-id-002"}, + ], + detail_by_id={ + "acl-id-001": _baseline_acl_detail("FinServACL1", "acl-id-001")[ + "WebACL" + ], + "acl-id-002": _baseline_acl_detail("FinServACL2", "acl-id-002")[ + "WebACL" + ], + }, + ) + ) + with patch("app.boto3.client") as mock_client: + + def side_effect(service, **kwargs): + if service == "apigateway": + c = MagicMock() + c.get_rest_apis.return_value = {"items": []} + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_api_gateway_request_body_size_limits(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + assert any( + r["Finding"] == "API Gateway Request Body Size Limits Configured" + for r in result["csv_data"] + ) + + +class TestFS69PromptInputValidationFunction: + """FS-69 — Prompt Input Validation Function Check.""" + + def test_pass_validation_lambda_exists(self): + inv = make_resource_inventory( + lambda_functions=[{"FunctionName": "prompt-input-sanitizer"}] + ) + result = app.check_prompt_input_validation_function(inv) + _assert_finding_structure(result) + assert result["status"] == "PASS" + + def test_warn_no_validation_lambda(self): + inv = make_resource_inventory( + lambda_functions=[{"FunctionName": "my-api-handler"}] + ) + result = app.check_prompt_input_validation_function(inv) + _assert_finding_structure(result) + assert result["status"] == "WARN" + + def test_error_on_unavailable_inventory(self): + inv = make_resource_inventory( + lambda_functions=app._Unavailable(RuntimeError("input error")) + ) + result = app.check_prompt_input_validation_function(inv) + assert result["status"] == "ERROR" + + +# ========================================================================= +# HELPERS +# ========================================================================= + + +class TestHelpers: + """Test _empty_findings and _error_findings helpers.""" + + def test_empty_findings_structure(self): + result = app._empty_findings("Test Check") + assert result["check_name"] == "Test Check" + assert result["status"] == "PASS" + assert result["csv_data"] == [] + + def test_error_findings_structure(self): + err = RuntimeError("something broke") + result = app._error_findings("Test Check", err) + assert result["check_name"] == "Test Check" + assert result["status"] == "ERROR" + assert "something broke" in result["details"] + # REQ-13: _error_findings contract is unchanged (csv_data stays empty); + # the visible row is synthesized by the handler, not here. + assert result["csv_data"] == [] + + def test_could_not_assess_row_shape(self): + # COULD_NOT_ASSESS disposition: synthesized row is N/A + Low (methodology §3.4). + row = app._could_not_assess_row( + "FS-44", "Amazon Macie PII Scanning Check", "AccessDenied" + ) + assert row["Check_ID"] == "FS-44" + assert row["Status"] == "N/A" + assert row["Severity"] == "Low" + assert row["Finding"].startswith(app.COULD_NOT_ASSESS_PREFIX) + assert "AccessDenied" in row["Finding_Details"] + + +# ========================================================================= +# REQ-13: CHECK REGISTRY + COULD-NOT-ASSESS HANDLING +# ========================================================================= + + +class TestFinservChecksRegistry: + """REQ-13 — registry-driven dispatch and could-not-assess rows.""" + + def test_registry_covers_all_checks_in_order(self): + registry = app.build_finserv_checks( + {"role_permissions": {}, "user_permissions": {}} + ) + # 65 entries: 64 standalone checks (FS-17/18/19/23/64 merged upstream) + # plus the new check_automated_reasoning_policies() which shares the FS-27 + # check_id with check_guardrail_contextual_grounding(). + assert len(registry) == 65 + ids = [cid for cid, _ in registry] + # FS-27 appears twice (contextual grounding + ARC policies); allow that. + unique_ids = set(ids) + assert len(unique_ids) == 64 + for cid, fn in registry: + assert callable(fn), f"{cid} is not callable" + # Order is non-decreasing by numeric FS id (execution order preserved). + nums = [int(cid.split("-")[1]) for cid in ids] + assert nums == sorted(nums) + # The two permission-cache checks are present. + assert "FS-07" in ids and "FS-22" in ids + + @patch("app.write_to_s3", return_value="https://example.com/report.csv") + @patch.dict(os.environ, {"AIML_ASSESSMENT_BUCKET_NAME": "test-bucket"}) + @patch("app.get_permissions_cache", return_value=None) + @patch("app.build_finserv_checks") + def test_errored_check_emits_could_not_assess_row( + self, mock_build, mock_cache, mock_write + ): + # One check raises (uncaught) → handler synthesizes one could-not-assess row. + def boom(): + return app._error_findings("Boom Check", RuntimeError("AccessDenied: nope")) + + def normal(): + return { + "check_name": "Normal Check", + "status": "PASS", + "csv_data": [ + { + "Check_ID": "FS-02", + "Finding": "Normal Finding", + "Finding_Details": "ok", + "Resolution": "none", + "Reference": "https://example.com", + "Severity": "Informational", + "Status": "Passed", + } + ], + } + + mock_build.return_value = [("FS-44", boom), ("FS-02", normal)] + + resp = app.lambda_handler({"Execution": {"Name": "exec-1"}}, None) + findings = resp["body"]["findings"] + # Errored check now contributes exactly one row, not zero (no silent drop). + boom_result = next(f for f in findings if f["check_name"] == "Boom Check") + assert len(boom_result["csv_data"]) == 1 + row = boom_result["csv_data"][0] + assert row["Check_ID"] == "FS-44" + assert row["Status"] == "N/A" + assert row["Severity"] == "Low" + assert row["Finding"].startswith(app.COULD_NOT_ASSESS_PREFIX) + # Normal check passes through unchanged (no spurious row added). + normal_result = next(f for f in findings if f["check_name"] == "Normal Check") + assert len(normal_result["csv_data"]) == 1 + assert normal_result["csv_data"][0]["Finding"] == "Normal Finding" + + @patch("app.write_to_s3", return_value="https://example.com/report.csv") + @patch.dict(os.environ, {"AIML_ASSESSMENT_BUCKET_NAME": "test-bucket"}) + @patch("app.get_permissions_cache", return_value=None) + @patch("app.build_finserv_checks") + def test_non_error_empty_result_still_emits_could_not_assess_row( + self, mock_build, mock_cache, mock_write + ): + # A check that returns a NON-error wrapper status but zero csv_data must + # NOT silently vanish — the handler synthesizes a could-not-assess row for + # any empty result, not only ERROR ones. This guards the no-silent-drop + # invariant structurally (Property 7) rather than by data coincidence. + def empty_pass(): + return {"check_name": "Empty Pass Check", "status": "PASS", "csv_data": []} + + mock_build.return_value = [("FS-13", empty_pass)] + + resp = app.lambda_handler({"Execution": {"Name": "exec-3"}}, None) + findings = resp["body"]["findings"] + result = next(f for f in findings if f["check_name"] == "Empty Pass Check") + assert len(result["csv_data"]) == 1 + row = result["csv_data"][0] + assert row["Check_ID"] == "FS-13" + assert row["Status"] == "N/A" + assert row["Severity"] == "Low" + assert row["Finding"].startswith(app.COULD_NOT_ASSESS_PREFIX) + + @patch("app.write_to_s3", return_value="https://example.com/report.csv") + @patch.dict(os.environ, {"AIML_ASSESSMENT_BUCKET_NAME": "test-bucket"}) + @patch("app.get_permissions_cache", return_value=None) + def test_no_check_contributes_zero_rows(self, mock_cache, mock_write): + # Full handler run with mocked AWS (all clients raise) — every check must + # still contribute at least one row (real rows or a could-not-assess row). + with patch("app.boto3.client", side_effect=RuntimeError("AccessDenied")): + resp = app.lambda_handler({"Execution": {"Name": "exec-2"}}, None) + findings = resp["body"]["findings"] + for f in findings: + assert len(f["csv_data"]) >= 1, f"{f['check_name']} contributed zero rows" + + +# ========================================================================= +# _paginate helper — multi-page collection across token conventions +# ========================================================================= + + +class TestPaginateHelper: + """app._paginate must collect all pages regardless of the API's token field.""" + + def test_single_page(self): + c = MagicMock() + c.list_things.return_value = {"items": [{"a": 1}, {"a": 2}]} + out = app._paginate(c, "list_things", "items") + assert out == [{"a": 1}, {"a": 2}] + assert c.list_things.call_count == 1 + + def test_lambda_marker_pagination(self): + # Lambda uses Marker (request) / NextMarker (response). + c = MagicMock() + c.list_functions.side_effect = [ + {"Functions": [{"FunctionName": "f1"}], "NextMarker": "m1"}, + {"Functions": [{"FunctionName": "f2"}]}, + ] + out = app._paginate(c, "list_functions", "Functions") + assert [f["FunctionName"] for f in out] == ["f1", "f2"] + assert c.list_functions.call_count == 2 + _, kwargs = c.list_functions.call_args + assert kwargs.get("Marker") == "m1" + + def test_next_token_lowercase_pagination(self): + # Bedrock uses nextToken. + c = MagicMock() + c.list_guardrails.side_effect = [ + {"guardrails": [{"id": "g1"}], "nextToken": "t1"}, + {"guardrails": [{"id": "g2"}]}, + ] + out = app._paginate(c, "list_guardrails", "guardrails") + assert [g["id"] for g in out] == ["g1", "g2"] + + def test_apigateway_position_pagination(self): + # API Gateway uses position (request and response). + c = MagicMock() + c.get_usage_plans.side_effect = [ + {"items": [{"id": "p1"}], "position": "pos1"}, + {"items": [{"id": "p2"}]}, + ] + out = app._paginate(c, "get_usage_plans", "items") + assert [p["id"] for p in out] == ["p1", "p2"] + + def test_missing_result_key_returns_empty(self): + c = MagicMock() + c.list_things.return_value = {} + assert app._paginate(c, "list_things", "items") == [] + + +class TestIsAccessError: + """app._is_access_error classifies permission errors vs other ClientErrors.""" + + def test_access_denied_is_access_error(self): + assert app._is_access_error(_client_error("AccessDenied")) is True + assert app._is_access_error(_client_error("AccessDeniedException")) is True + assert app._is_access_error(_client_error("UnauthorizedOperation")) is True + + def test_other_clienterror_is_not_access_error(self): + assert app._is_access_error(_client_error("NoSuchTagSet")) is False + assert app._is_access_error(_client_error("NoSuchBucket")) is False + + def test_non_clienterror_is_not_access_error(self): + # A plain exception without a .response attribute must not be misclassified. + assert app._is_access_error(RuntimeError("boom")) is False + + +class TestIsMissingBucketError: + """app._is_missing_bucket_error classifies deleted-bucket errors (NoSuchBucket / + 404 / NotFound) distinctly from access errors and other ClientErrors.""" + + def test_nosuchbucket_is_missing(self): + assert app._is_missing_bucket_error(_client_error("NoSuchBucket")) is True + assert app._is_missing_bucket_error(_client_error("404")) is True + assert app._is_missing_bucket_error(_client_error("NotFound")) is True + + def test_access_and_other_errors_are_not_missing(self): + assert app._is_missing_bucket_error(_client_error("AccessDenied")) is False + assert app._is_missing_bucket_error(_client_error("InvalidRequest")) is False + assert app._is_missing_bucket_error(_client_error("NoSuchTagSet")) is False + + def test_non_clienterror_is_not_missing(self): + assert app._is_missing_bucket_error(RuntimeError("boom")) is False + + +# ========================================================================= +# REQ-14: REQUIREMENTS.TXT VERSION FLOOR GUARD +# ========================================================================= + + +class TestRequirementVersionFloors: + """ + REQ-14 — Guard against future regression that lowers the botocore/boto3 floor + below the minimum needed for: + - FS-03: list_aws_default_service_quotas paginator + - FS-06: describe_budgets(ShowFilterExpression=True) + + Tests assert the floor in requirements.txt, not the installed version, so + they catch a source-file regression regardless of what pip has resolved in + the dev environment. The dev environment may be behind the floor + (e.g., 1.42.70 installed vs 1.43.21 required) — run + `pip install --upgrade boto3 botocore` to reach the floor. + + These tests do NOT hard-pin to 1.43.21; they assert a structural property + (floor >= minimum needed) so they stay valid as the floor is bumped over time. + """ + + # Minimum versions required for the features used in this Lambda. + # Chosen because 1.43.x is the first series that reliably supports both + # describe_budgets(ShowFilterExpression=True) and list_aws_default_service_quotas. + MIN_BOTO3 = (1, 43, 0) + MIN_BOTOCORE = (1, 43, 0) + + @staticmethod + def _parse_floor(line: str) -> tuple: + """ + Parse a 'pkg>=X.Y.Z' line from requirements.txt into (X, Y, Z). + Returns None if the line doesn't match the expected pattern. + """ + import re + + m = re.match(r"^\s*(boto[3core]*)\s*>=\s*(\d+)\.(\d+)\.(\d+)", line) + if m: + return (int(m.group(2)), int(m.group(3)), int(m.group(4))) + return None + + @staticmethod + def _load_requirements() -> str: + """Load finserv_assessments/requirements.txt relative to the tests/ dir.""" + req_path = os.path.join( + os.path.dirname(__file__), "..", "finserv_assessments", "requirements.txt" + ) + with open(req_path) as f: + return f.read() + + def test_boto3_floor_meets_minimum(self): + """requirements.txt must floor boto3 at >= 1.43.0.""" + content = self._load_requirements() + boto3_floor = None + for line in content.splitlines(): + if line.strip().startswith("boto3>="): + boto3_floor = self._parse_floor(line) + break + assert boto3_floor is not None, ( + "boto3 floor not found in requirements.txt — expected 'boto3>=X.Y.Z'" + ) + assert boto3_floor >= self.MIN_BOTO3, ( + f"boto3 floor {boto3_floor} is below minimum {self.MIN_BOTO3}; " + "FS-06 describe_budgets(ShowFilterExpression=True) requires boto3>=1.43.0. " + "Bump the floor in finserv_assessments/requirements.txt." + ) + + def test_botocore_floor_meets_minimum(self): + """requirements.txt must floor botocore at >= 1.43.0.""" + content = self._load_requirements() + botocore_floor = None + for line in content.splitlines(): + if line.strip().startswith("botocore>="): + botocore_floor = self._parse_floor(line) + break + assert botocore_floor is not None, ( + "botocore floor not found in requirements.txt — expected 'botocore>=X.Y.Z'" + ) + assert botocore_floor >= self.MIN_BOTOCORE, ( + f"botocore floor {botocore_floor} is below minimum {self.MIN_BOTOCORE}; " + "FS-03 list_aws_default_service_quotas and FS-06 ShowFilterExpression " + "require botocore>=1.43.0. Bump the floor in finserv_assessments/requirements.txt." + ) + + def test_pydantic_floor_present(self): + """pydantic>=2.0.0 must be present and unchanged (schema depends on Pydantic v2).""" + content = self._load_requirements() + assert "pydantic>=2.0.0" in content, ( + "pydantic>=2.0.0 not found in requirements.txt — Pydantic v2 is required " + "for the Finding model in schema.py." + ) + + def test_no_exact_pins_on_aws_sdk(self): + """ + boto3 and botocore must use >= floors, not exact pins (==). + Exact pins would block Lambda from resolving security patches. + """ + content = self._load_requirements() + import re + + exact_pins = re.findall(r"boto(?:3|core)==\S+", content) + assert not exact_pins, ( + f"Exact version pins found for AWS SDK: {exact_pins}. " + "Use >= floors instead so the Lambda can receive security patches." + ) + + +class TestGenerateCsvReport: + """Test CSV report generation.""" + + def test_empty_findings_produces_header_only(self): + csv_content = app.generate_csv_report([]) + lines = csv_content.strip().split("\n") + assert len(lines) == 1 # header only + assert "Check_ID" in lines[0] + + def test_findings_produce_csv_rows(self): + findings = [ + { + "check_name": "Test", + "status": "PASS", + "csv_data": [ + { + "Check_ID": "FS-01", + "Finding": "Test Finding", + "Finding_Details": "Details", + "Resolution": "Fix", + "Reference": "https://example.com", + "Severity": "High", + "Status": "Passed", + } + ], + } + ] + csv_content = app.generate_csv_report(findings) + lines = csv_content.strip().split("\n") + assert len(lines) == 2 # header + 1 data row + assert "FS-01" in lines[1] + + def test_multiple_findings_multiple_rows(self): + findings = [ + { + "check_name": "Check A", + "status": "WARN", + "csv_data": [ + { + "Check_ID": "FS-01", + "Finding": "A", + "Finding_Details": "D", + "Resolution": "R", + "Reference": "https://a.com", + "Severity": "High", + "Status": "Failed", + }, + { + "Check_ID": "FS-01", + "Finding": "B", + "Finding_Details": "D", + "Resolution": "R", + "Reference": "https://b.com", + "Severity": "Medium", + "Status": "Passed", + }, + ], + }, + { + "check_name": "Check B", + "status": "PASS", + "csv_data": [ + { + "Check_ID": "FS-02", + "Finding": "C", + "Finding_Details": "D", + "Resolution": "R", + "Reference": "https://c.com", + "Severity": "Low", + "Status": "Passed", + }, + ], + }, + ] + csv_content = app.generate_csv_report(findings) + lines = csv_content.strip().split("\n") + assert len(lines) == 4 # header + 3 data rows diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_checks_coverage.py b/aiml-security-assessment/functions/security/finserv_tests/test_checks_coverage.py new file mode 100644 index 0000000..d948cac --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_tests/test_checks_coverage.py @@ -0,0 +1,1993 @@ +""" +Additional tests targeting the uncovered branches in finserv_assessments/app.py. +These complement test_checks.py to push coverage from 83% → 90%+. + +Each class targets a specific uncovered branch identified from coverage.json. +""" + +import json +import os +import sys +from datetime import datetime, timezone, timedelta +from unittest.mock import MagicMock, patch +from botocore.exceptions import ClientError + + +FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") +if FINSERV_DIR not in sys.path: + sys.path.insert(0, FINSERV_DIR) + +# Ensure tests/ directory is importable (for conftest helpers) +TESTS_DIR = os.path.dirname(__file__) +if TESTS_DIR not in sys.path: + sys.path.insert(0, TESTS_DIR) + +import app # noqa: E402 (import must follow sys.path setup above) +from conftest import make_resource_inventory # noqa: E402 + + +def _client_error(code="AccessDeniedException", message="Access Denied"): + return ClientError({"Error": {"Code": code, "Message": message}}, "Op") + + +def _assert_structure(result): + assert "check_name" in result + assert result["status"] in ("PASS", "WARN", "ERROR") + assert isinstance(result["csv_data"], list) + + +# ========================================================================= +# FS-01 — line 126: shield ClientError path (not ResourceNotFoundException) +# ========================================================================= + + +class TestFS01ShieldClientError: + def test_shield_generic_client_error_treated_as_no_shield(self): + """Line 126-128: ClientError on describe_subscription → shield_enabled stays False.""" + inv = make_resource_inventory( + web_acls=app.WebAclInventory( + summaries=[{"Name": "acl1", "Id": "id1"}], + detail_by_id={}, + ) + ) + with patch("app.boto3.client") as mock_client: + + def side_effect(service, **kwargs): + if service == "shield": + c = MagicMock() + c.describe_subscription.side_effect = _client_error( + "ThrottlingException" + ) + c.exceptions.ResourceNotFoundException = type( + "ResourceNotFoundException", (ClientError,), {} + ) + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_waf_shield_on_bedrock_endpoints(inv) + _assert_structure(result) + # Shield not enabled → WARN, but WAF ACLs present → only 1 WARN finding + assert result["status"] == "WARN" + statuses = [r["Status"] for r in result["csv_data"]] + assert "Failed" in statuses # shield failed + assert "Passed" in statuses # waf passed + + +# ========================================================================= +# FS-07 — lines 532-534, 537, 543, 546: new per-agent error handling paths +# ========================================================================= + + +class TestFS07AgentBoundariesNewPaths: + @patch("app.boto3.client") + def test_get_agent_client_error_skips_agent(self, mock_client): + """Lines 532-534: get_agent raises ClientError → agent is skipped gracefully.""" + c = MagicMock() + c.list_agents.return_value = { + "agentSummaries": [{"agentId": "a1", "agentName": "EncryptedAgent"}] + } + c.get_agent.side_effect = _client_error("AccessDeniedException") + mock_client.return_value = c + result = app.check_bedrock_agent_action_boundaries({}) + _assert_structure(result) + # Should PASS (no issues found, agent was skipped) + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_agent_no_role_arn_skipped(self, mock_client): + """Line 537: agent with no agentResourceRoleArn → continue.""" + c = MagicMock() + c.list_agents.return_value = { + "agentSummaries": [{"agentId": "a1", "agentName": "NoRoleAgent"}] + } + c.get_agent.return_value = {"agent": {"agentResourceRoleArn": ""}} + mock_client.return_value = c + result = app.check_bedrock_agent_action_boundaries({}) + _assert_structure(result) + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_policy_doc_as_string_is_parsed(self, mock_client): + """Line 543: policy document stored as JSON string → json.loads branch.""" + c = MagicMock() + c.list_agents.return_value = { + "agentSummaries": [{"agentId": "a1", "agentName": "Agent1"}] + } + c.get_agent.return_value = { + "agent": {"agentResourceRoleArn": "arn:aws:iam::123:role/SafeRole"} + } + mock_client.return_value = c + cache = { + "role_permissions": { + "SafeRole": { + "attached_policies": [ + { + "document": json.dumps( + { + "Statement": [ + { + "Effect": "Allow", + "Action": "bedrock:InvokeModel", + "Resource": "*", + } + ] + } + ) + } + ], + "inline_policies": [], + } + } + } + result = app.check_bedrock_agent_action_boundaries(cache) + _assert_structure(result) + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_deny_effect_statement_skipped(self, mock_client): + """Line 546: Deny effect → continue (not counted as issue).""" + c = MagicMock() + c.list_agents.return_value = { + "agentSummaries": [{"agentId": "a1", "agentName": "Agent1"}] + } + c.get_agent.return_value = { + "agent": {"agentResourceRoleArn": "arn:aws:iam::123:role/DenyRole"} + } + mock_client.return_value = c + cache = { + "role_permissions": { + "DenyRole": { + "attached_policies": [ + { + "document": { + "Statement": [ + { + "Effect": "Deny", + "Action": "iam:*", + "Resource": "*", + } + ] + } + } + ], + "inline_policies": [], + } + } + } + result = app.check_bedrock_agent_action_boundaries(cache) + _assert_structure(result) + assert result["status"] == "PASS" + + +# ========================================================================= +# FS-08 — line 622: re-raise non-AccessDenied ClientError +# ========================================================================= + + +class TestFS08AgentcoreReraise: + @patch("app.boto3.client") + def test_non_access_denied_error_propagates_to_outer_except(self, mock_client): + """Line 622: ClientError that is NOT AccessDenied/Unrecognized → re-raised → ERROR.""" + c = MagicMock() + c.list_agent_runtimes.side_effect = _client_error("ServiceUnavailableException") + mock_client.return_value = c + result = app.check_agentcore_policy_engine() + _assert_structure(result) + assert result["status"] == "ERROR" + + +# ========================================================================= +# FS-09 — lines 704-705: get_function_concurrency ClientError path +# ========================================================================= + + +class TestFS09ConcurrencyClientError: + @patch("app.boto3.client") + def test_get_concurrency_client_error_adds_to_warn_list(self, mock_client): + """Lines 704-705: get_function_concurrency raises ClientError → appended to warn list.""" + c = MagicMock() + c.get_function_concurrency.side_effect = _client_error("AccessDeniedException") + mock_client.return_value = c + inv = make_resource_inventory( + lambda_functions=[{"FunctionName": "my-agent-fn"}] + ) + result = app.check_agent_transaction_limits(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + +# ========================================================================= +# FS-12 — lines 902-923, 947: SCP paths +# ========================================================================= + + +class TestFS12ScpPaths: + @patch("app.boto3.client") + def test_access_denied_returns_na(self, mock_client): + """Lines 902-915: AccessDeniedException → N/A finding.""" + c = MagicMock() + c.list_policies.side_effect = _client_error("AccessDeniedException") + mock_client.return_value = c + result = app.check_scp_model_access_restrictions() + _assert_structure(result) + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_orgs_not_in_use_returns_na(self, mock_client): + """Lines 902-915: AWSOrganizationsNotInUseException → N/A finding.""" + c = MagicMock() + c.list_policies.side_effect = _client_error("AWSOrganizationsNotInUseException") + mock_client.return_value = c + result = app.check_scp_model_access_restrictions() + _assert_structure(result) + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_non_access_denied_reraises(self, mock_client): + """Line 916: non-AccessDenied ClientError → re-raised → ERROR.""" + c = MagicMock() + c.list_policies.side_effect = _client_error("ServiceUnavailableException") + mock_client.return_value = c + result = app.check_scp_model_access_restrictions() + _assert_structure(result) + assert result["status"] == "ERROR" + + @patch("app.boto3.client") + def test_warn_no_bedrock_scps(self, mock_client): + """Lines 920-923, 925-945: policies exist but none reference bedrock → WARN.""" + c = MagicMock() + c.list_policies.return_value = { + "Policies": [{"Id": "p-001", "Name": "GeneralSCP"}] + } + c.describe_policy.return_value = { + "Policy": { + "Content": json.dumps( + {"Statement": [{"Effect": "Deny", "Action": "ec2:*"}]} + ) + } + } + mock_client.return_value = c + result = app.check_scp_model_access_restrictions() + _assert_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_pass_bedrock_scp_found(self, mock_client): + """Line 947: bedrock SCP found → Passed finding.""" + c = MagicMock() + c.list_policies.return_value = { + "Policies": [{"Id": "p-001", "Name": "BedrockModelSCP"}] + } + c.describe_policy.return_value = { + "Policy": { + "Content": json.dumps( + { + "Statement": [ + { + "Effect": "Deny", + "Action": "bedrock:InvokeModel", + "Condition": { + "StringNotEquals": { + "bedrock:ModelId": ["anthropic.claude-v2"] + } + }, + } + ] + } + ) + } + } + mock_client.return_value = c + result = app.check_scp_model_access_restrictions() + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-13 — lines 979-999: model tagging warn/pass paths +# ========================================================================= + + +class TestFS13ModelTaggingPaths: + @patch("app.boto3.client") + def test_warn_bedrock_model_missing_tags(self, mock_client): + """Lines 979-983, 998-999: Bedrock custom model missing required tags → WARN.""" + + def side_effect(service, **kwargs): + if service == "bedrock": + c = MagicMock() + c.list_custom_models.return_value = { + "modelSummaries": [ + { + "modelName": "my-model", + "modelArn": "arn:aws:bedrock:us-east-1:123:model/my-model", + } + ] + } + c.list_tags_for_resource.return_value = {"tags": []} + return c + if service == "sagemaker": + c = MagicMock() + c.list_models.return_value = {"Models": []} + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_model_inventory_tagging() + _assert_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_warn_sagemaker_model_missing_tags(self, mock_client): + """Lines 989-993: SageMaker model missing required tags → WARN.""" + + def side_effect(service, **kwargs): + if service == "bedrock": + c = MagicMock() + c.list_custom_models.return_value = {"modelSummaries": []} + return c + if service == "sagemaker": + c = MagicMock() + c.list_models.return_value = { + "Models": [ + { + "ModelName": "sm-model", + "ModelArn": "arn:aws:sagemaker:us-east-1:123:model/sm-model", + } + ] + } + c.list_tags.return_value = {"Tags": []} + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_model_inventory_tagging() + _assert_structure(result) + assert result["status"] == "WARN" + + +# ========================================================================= +# FS-14 — line 1072: pass path (config rules found) +# ========================================================================= + + +class TestFS14ModelGovernancePass: + @patch("app.boto3.client") + def test_pass_config_rules_found(self, mock_client): + """Line 1072: bedrock-related Config rules found → Passed.""" + c = MagicMock() + c.describe_config_rules.return_value = { + "ConfigRules": [{"ConfigRuleName": "bedrock-model-approval-rule"}] + } + mock_client.return_value = c + result = app.check_model_onboarding_governance() + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-15 — line 1119: pass path (eval jobs found) +# ========================================================================= + + +class TestFS15BedrockEvalPass: + @patch("app.boto3.client") + def test_pass_eval_jobs_found(self, mock_client): + """Line 1119: evaluation jobs exist → Passed finding.""" + c = MagicMock() + c.list_evaluation_jobs.return_value = { + "jobSummaries": [{"jobName": "adversarial-eval-2025"}] + } + mock_client.return_value = c + result = app.check_bedrock_model_evaluation_adversarial() + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-16 — lines 1167-1168: warn path (repos without scanning) +# ========================================================================= + + +class TestFS16EcrScanningWarn: + @patch("app.boto3.client") + def test_warn_repos_without_scanning(self, mock_client): + """Lines 1167-1168: repos exist but scan-on-push disabled → WARN.""" + c = MagicMock() + c.describe_repositories.return_value = { + "repositories": [ + { + "repositoryName": "ml-model-repo", + "imageScanningConfiguration": {"scanOnPush": False}, + } + ] + } + mock_client.return_value = c + result = app.check_ecr_image_scanning() + _assert_structure(result) + assert result["status"] == "WARN" + + +# ========================================================================= +# FS-20 — lines 1238-1266: feature store warn/pass paths +# ========================================================================= + + +class TestFS20FeatureStoreWarnPass: + @patch("app.boto3.client") + def test_warn_groups_without_offline_store(self, mock_client): + """Lines 1244-1246: feature groups without active offline store → WARN.""" + c = MagicMock() + c.list_feature_groups.return_value = { + "FeatureGroupSummaries": [ + { + "FeatureGroupName": "customer-features", + "OfflineStoreStatus": {"Status": "Disabled"}, + } + ] + } + mock_client.return_value = c + result = app.check_feature_store_rollback_capability() + _assert_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_pass_all_groups_have_offline_store(self, mock_client): + """Line 1266: all feature groups have active offline store → Passed.""" + c = MagicMock() + c.list_feature_groups.return_value = { + "FeatureGroupSummaries": [ + { + "FeatureGroupName": "customer-features", + "OfflineStoreStatus": {"Status": "Active"}, + } + ] + } + mock_client.return_value = c + result = app.check_feature_store_rollback_capability() + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-21 — lines 1312-1351: S3 versioning warn/pass paths +# ========================================================================= + + +class TestFS21TrainingDataVersioningPaths: + @patch("app.boto3.client") + def test_warn_unversioned_training_buckets(self, mock_client): + """Lines 1312-1316, 1318-1320: training buckets without versioning → WARN.""" + inv = make_resource_inventory(buckets=[{"Name": "training-data-bucket"}]) + c = MagicMock() + c.get_bucket_tagging.return_value = { + "TagSet": [{"Key": "Purpose", "Value": "training"}] + } + c.get_bucket_versioning.return_value = {} # no versioning + mock_client.return_value = c + + result = app.check_training_data_s3_versioning(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_pass_all_training_buckets_versioned(self, mock_client): + """Line 1338: all training buckets versioned → Passed.""" + inv = make_resource_inventory(buckets=[{"Name": "training-data-bucket"}]) + c = MagicMock() + c.get_bucket_tagging.return_value = { + "TagSet": [{"Key": "Purpose", "Value": "training"}] + } + c.get_bucket_versioning.return_value = {"Status": "Enabled"} + mock_client.return_value = c + + result = app.check_training_data_s3_versioning(inv) + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_access_error_surfaces_as_could_not_assess(self, mock_client): + """AccessDenied on get_bucket_versioning re-raises → ERROR (could-not-assess), + not a false 'no versioning' finding.""" + inv = make_resource_inventory(buckets=[{"Name": "training-data-bucket"}]) + c = MagicMock() + c.get_bucket_versioning.side_effect = _client_error("AccessDenied") + mock_client.return_value = c + + result = app.check_training_data_s3_versioning(inv) + _assert_structure(result) + assert result["status"] == "ERROR" + + @patch("app.boto3.client") + def test_nonaccess_error_flags_bucket(self, mock_client): + """A non-access ClientError on get_bucket_versioning flags the bucket + (WARN) without aborting the whole check.""" + inv = make_resource_inventory(buckets=[{"Name": "model-bucket"}]) + c = MagicMock() + c.get_bucket_versioning.side_effect = _client_error("NoSuchBucket") + mock_client.return_value = c + + result = app.check_training_data_s3_versioning(inv) + _assert_structure(result) + assert result["status"] == "WARN" + assert any( + "(error)" in r.get("Finding_Details", "") for r in result["csv_data"] + ) + + @patch("app.boto3.client") + def test_multi_page_buckets_completeness(self, mock_client): + """Pagination completeness: buckets from multiple pages are all checked. + + With the inventory approach, the collector already drains all pages via + _paginate with ContinuationToken. This test verifies that when 2+ pages + of buckets are provided in the inventory, the check assesses them all. + """ + # Simulate 2 "pages" of buckets — all training-named so they're all checked + page1 = [ + {"Name": "training-bucket-page1-001"}, + {"Name": "model-bucket-page1-002"}, + ] + page2 = [ + {"Name": "training-bucket-page2-001"}, + {"Name": "sagemaker-bucket-page2-002"}, + ] + all_buckets = page1 + page2 + + inv = make_resource_inventory(buckets=all_buckets) + c = MagicMock() + c.get_bucket_versioning.return_value = {"Status": "Enabled"} + mock_client.return_value = c + + result = app.check_training_data_s3_versioning(inv) + _assert_structure(result) + # All 4 buckets versioned → Passed + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + # Verify all 4 were checked (versioning call per training bucket) + assert c.get_bucket_versioning.call_count == 4 + + @patch("app.boto3.client") + def test_single_page_unchanged_vs_baseline(self, mock_client): + """Single-page case: result is identical to pre-migration behavior (baseline). + + With ≤1 page of buckets (no ContinuationToken), the inventory holds + all buckets and the check outcome is the same as before migration. + """ + inv = make_resource_inventory(buckets=[{"Name": "training-data-bucket"}]) + c = MagicMock() + c.get_bucket_versioning.return_value = {} # no versioning + mock_client.return_value = c + + result = app.check_training_data_s3_versioning(inv) + _assert_structure(result) + assert result["status"] == "WARN" + assert any( + r["Finding"] == "Training Data Buckets Without Versioning" + for r in result["csv_data"] + ) + + +# ========================================================================= + + +class TestFS22KbIamWarnPath: + def test_warn_wildcard_bedrock_agent_permission(self): + """Lines 1370-1386: role with bedrock-agent:* → WARN.""" + cache = { + "role_permissions": { + "KBAccessRole": { + "attached_policies": [ + { + "document": { + "Statement": [ + { + "Effect": "Allow", + "Action": "bedrock-agent:*", + "Resource": "*", + } + ] + } + } + ], + "inline_policies": [], + } + } + } + result = app.check_knowledge_base_iam_least_privilege(cache) + _assert_structure(result) + assert result["status"] == "WARN" + + def test_warn_wildcard_bedrock_permission(self): + """Lines 1370-1386: role with bedrock:* → WARN.""" + cache = { + "role_permissions": { + "KBRole": { + "attached_policies": [], + "inline_policies": [ + { + "document": { + "Statement": [ + { + "Effect": "Allow", + "Action": "bedrock:*", + "Resource": "*", + } + ] + } + } + ], + } + } + } + result = app.check_knowledge_base_iam_least_privilege(cache) + _assert_structure(result) + assert result["status"] == "WARN" + + def test_policy_doc_as_string_parsed(self): + """Line 1372-1373: policy document as JSON string → parsed correctly.""" + cache = { + "role_permissions": { + "KBRole": { + "attached_policies": [ + { + "document": json.dumps( + { + "Statement": [ + { + "Effect": "Allow", + "Action": "*", + "Resource": "*", + } + ] + } + ) + } + ], + "inline_policies": [], + } + } + } + result = app.check_knowledge_base_iam_least_privilege(cache) + _assert_structure(result) + assert result["status"] == "WARN" + + def test_deny_effect_not_flagged(self): + """Line 1375-1376: Deny effect → not counted as issue.""" + cache = { + "role_permissions": { + "KBRole": { + "attached_policies": [ + { + "document": { + "Statement": [ + { + "Effect": "Deny", + "Action": "bedrock-agent:*", + "Resource": "*", + } + ] + } + } + ], + "inline_policies": [], + } + } + } + result = app.check_knowledge_base_iam_least_privilege(cache) + _assert_structure(result) + assert result["status"] == "PASS" + + def test_action_as_string_not_list(self): + """Lines 1378-1379: Action as string (not list) → converted to list.""" + cache = { + "role_permissions": { + "KBRole": { + "attached_policies": [ + { + "document": { + "Statement": [ + { + "Effect": "Allow", + "Action": "bedrock:*", + "Resource": "*", + } + ] + } + } + ], + "inline_policies": [], + } + } + } + result = app.check_knowledge_base_iam_least_privilege(cache) + _assert_structure(result) + assert result["status"] == "WARN" + + +# ========================================================================= +# FS-24 — line 1450: KB metadata filtering pass path (KBs exist) +# ========================================================================= + + +class TestFS24MetadataFilteringPass: + def test_pass_kbs_exist(self): + """KBs found → advisory N/A finding (metadata filtering not API-verifiable).""" + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1", "name": "rag-kb"}], + data_sources_by_kb={}, + data_source_detail={}, + ) + ) + result = app.check_knowledge_base_metadata_filtering(inv) + _assert_structure(result) + assert any( + r["Status"] == "N/A" + and r["Severity"] == "Informational" + and r["Finding"].startswith("ADVISORY: ") + for r in result["csv_data"] + ) + + +# ========================================================================= +# FS-25 — lines 1509-1511: OSS encryption with CMK path +# ========================================================================= + + +class TestFS25OssEncryptionPaths: + @patch("app.boto3.client") + def test_pass_policies_with_cmk(self, mock_client): + """Lines 1509-1511: encryption policies exist with CMK → Passed.""" + c = MagicMock() + c.list_security_policies.return_value = { + "securityPolicySummaries": [ + { + "name": "kb-encryption", + "policy": json.dumps( + {"Rules": [{"KmsARN": "arn:aws:kms:us-east-1:123:key/abc"}]} + ), + } + ] + } + mock_client.return_value = c + result = app.check_opensearch_serverless_encryption() + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_pass_no_policies(self, mock_client): + """Line 1488: no encryption policies → N/A finding.""" + c = MagicMock() + c.list_security_policies.return_value = {"securityPolicySummaries": []} + mock_client.return_value = c + result = app.check_opensearch_serverless_encryption() + _assert_structure(result) + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_fail_policies_without_cmk(self, mock_client): + """Encryption policies exist but all use AWS-owned keys → WARN/Failed + (the customer-managed-key control is absent), not a false Pass.""" + c = MagicMock() + c.list_security_policies.return_value = { + "securityPolicySummaries": [ + { + "name": "aws-owned-enc", + "policy": json.dumps( + {"Rules": [{"ResourceType": "collection"}], "AWSOwnedKey": True} + ), + } + ] + } + mock_client.return_value = c + result = app.check_opensearch_serverless_encryption() + _assert_structure(result) + assert result["status"] == "WARN" + assert any(r["Status"] == "Failed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-26 — lines 1546-1591: VPC access warn/pass paths +# ========================================================================= + + +class TestFS26VpcAccessPaths: + @patch("app.boto3.client") + def test_warn_no_network_policies(self, mock_client): + """Lines 1546-1547: no network policies → WARN.""" + c = MagicMock() + c.list_security_policies.return_value = {"securityPolicySummaries": []} + mock_client.return_value = c + result = app.check_knowledge_base_vpc_access() + _assert_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_warn_policies_without_vpc(self, mock_client): + """Lines 1567-1569: network policies exist but no VPC restriction → WARN.""" + c = MagicMock() + c.list_security_policies.return_value = { + "securityPolicySummaries": [ + { + "name": "public-access", + "policy": json.dumps({"Rules": [{"AllowFromPublic": True}]}), + } + ] + } + mock_client.return_value = c + result = app.check_knowledge_base_vpc_access() + _assert_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_pass_vpc_restricted_policies(self, mock_client): + """Line 1591: network policies with VPC restriction → Passed.""" + c = MagicMock() + c.list_security_policies.return_value = { + "securityPolicySummaries": [ + { + "name": "vpc-only", + "policy": json.dumps({"Rules": [{"SourceVPCEs": ["vpce-abc123"]}]}), + } + ] + } + mock_client.return_value = c + result = app.check_knowledge_base_vpc_access() + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-27 — lines 1644, 1667: ARC guardrail paths +# ========================================================================= + + +class TestFS27AutomatedReasoningPaths: + def test_warn_guardrails_without_grounding(self): + """Guardrails exist but none have contextual grounding → WARN.""" + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={"g1": {}}, # no contextualGroundingPolicy + ) + ) + result = app.check_guardrail_contextual_grounding(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + def test_pass_guardrail_with_grounding(self): + """Guardrail with contextual grounding → Passed.""" + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": { + "contextualGroundingPolicy": { + "filters": [{"type": "GROUNDING", "threshold": 0.7}] + } + } + }, + ) + ) + result = app.check_guardrail_contextual_grounding(inv) + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-28 — lines 1717-1718: denied topics warn path +# ========================================================================= + + +class TestFS28DeniedTopicsWarn: + def test_warn_guardrails_without_financial_topics(self): + """Lines 1717-1718: guardrails exist but no topic policies → WARN.""" + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={"g1": {"topicPolicy": {"topics": []}}}, + ) + ) + result = app.check_guardrail_denied_topics_financial(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + +# ========================================================================= +# FS-30 — line 1814: eval jobs found pass path +# ========================================================================= + + +class TestFS30ComplianceEvalPass: + def test_advisory_finding(self): + """FS-30 is advisory (REQ-10a): always one N/A Informational ADVISORY row.""" + result = app.check_bedrock_evaluation_compliance_datasets() + _assert_structure(result) + assert any( + r["Status"] == "N/A" + and r["Severity"] == "Informational" + and r["Finding"].startswith("ADVISORY: ") + for r in result["csv_data"] + ) + + +# ========================================================================= +# FS-31 — lines 1864-1914: KB data source sync stale/fresh paths +# ========================================================================= + + +class TestFS31KbSyncPaths: + def test_warn_stale_data_sources(self): + """Lines 1864-1882: KB data sources not synced in >7 days → WARN.""" + stale_time = datetime.now(timezone.utc) - timedelta(days=10) + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1", "name": "my-kb"}], + data_sources_by_kb={ + "kb1": [ + { + "dataSourceId": "ds1", + "name": "s3-source", + "updatedAt": stale_time, + } + ] + }, + data_source_detail={}, + ) + ) + result = app.check_knowledge_base_data_source_sync(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + def test_pass_recently_synced(self): + """Line 1901: all data sources synced within 7 days → Passed.""" + fresh_time = datetime.now(timezone.utc) - timedelta(days=1) + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1", "name": "my-kb"}], + data_sources_by_kb={ + "kb1": [ + { + "dataSourceId": "ds1", + "name": "s3-source", + "updatedAt": fresh_time, + } + ] + }, + data_source_detail={}, + ) + ) + result = app.check_knowledge_base_data_source_sync(inv) + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-33 — lines 1976-2034: KB integrity monitoring paths +# ========================================================================= + + +class TestFS33KbIntegrityPaths: + @patch("app.boto3.client") + def test_warn_bucket_without_versioning(self, mock_client): + """Lines 1976-2003: KB data source bucket without versioning → WARN.""" + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1"}], + data_sources_by_kb={"kb1": [{"dataSourceId": "ds1"}]}, + data_source_detail={ + ("kb1", "ds1"): { + "dataSource": { + "dataSourceConfiguration": { + "s3Configuration": { + "bucketArn": "arn:aws:s3:::kb-data-bucket" + } + } + } + } + }, + ) + ) + + def side_effect(service, **kwargs): + if service == "s3": + c = MagicMock() + c.get_bucket_versioning.return_value = {} # not enabled + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_knowledge_base_integrity_monitoring(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_pass_bucket_with_versioning(self, mock_client): + """Line 2021: all KB buckets have versioning → Passed.""" + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1"}], + data_sources_by_kb={"kb1": [{"dataSourceId": "ds1"}]}, + data_source_detail={ + ("kb1", "ds1"): { + "dataSource": { + "dataSourceConfiguration": { + "s3Configuration": { + "bucketArn": "arn:aws:s3:::kb-data-bucket" + } + } + } + } + }, + ) + ) + + def side_effect(service, **kwargs): + if service == "s3": + c = MagicMock() + c.get_bucket_versioning.return_value = {"Status": "Enabled"} + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_knowledge_base_integrity_monitoring(inv) + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_deleted_bucket_reported_separately(self, mock_client): + """A NoSuchBucket on get_bucket_versioning → distinct 'deleted bucket' + finding (High), NOT conflated with 'without versioning' or labeled + '(error)'. Regression guard for the FS-33 NoSuchBucket refinement.""" + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1", "name": "kb-one"}], + data_sources_by_kb={"kb1": [{"dataSourceId": "ds1", "name": "ds-one"}]}, + data_source_detail={ + ("kb1", "ds1"): { + "dataSource": { + "dataSourceConfiguration": { + "s3Configuration": { + "bucketArn": "arn:aws:s3:::deleted-bucket" + } + } + } + } + }, + ) + ) + + def side_effect(service, **kwargs): + if service == "s3": + c = MagicMock() + c.get_bucket_versioning.side_effect = _client_error("NoSuchBucket") + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_knowledge_base_integrity_monitoring(inv) + _assert_structure(result) + assert result["status"] == "WARN" + # Distinct deleted-bucket finding present, High severity, not "(error)". + deleted = [ + r + for r in result["csv_data"] + if r["Finding"] == "KB Data Source References a Deleted S3 Bucket" + ] + assert deleted, "expected a distinct deleted-bucket finding" + assert deleted[0]["Severity"] == "High" + assert "deleted-bucket" in deleted[0]["Finding_Details"] + # No "(error)" mislabel and no "Without Versioning" finding for this bucket. + assert not any( + "(error)" in r.get("Finding_Details", "") for r in result["csv_data"] + ) + assert not any( + r["Finding"] == "KB Data Source Buckets Without Versioning" + for r in result["csv_data"] + ) + + @patch("app.boto3.client") + def test_bucket_nonaccess_nonmissing_error_treated_as_unversioned( + self, mock_client + ): + """A non-access, non-missing ClientError on get_bucket_versioning → + bucket flagged as '(error)' under the versioning finding (WARN), not + silently dropped and not treated as a deleted bucket.""" + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1"}], + data_sources_by_kb={"kb1": [{"dataSourceId": "ds1"}]}, + data_source_detail={ + ("kb1", "ds1"): { + "dataSource": { + "dataSourceConfiguration": { + "s3Configuration": { + "bucketArn": "arn:aws:s3:::weird-bucket" + } + } + } + } + }, + ) + ) + + def side_effect(service, **kwargs): + if service == "s3": + c = MagicMock() + c.get_bucket_versioning.side_effect = _client_error("InvalidRequest") + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_knowledge_base_integrity_monitoring(inv) + _assert_structure(result) + assert result["status"] == "WARN" + assert any( + "(error)" in r.get("Finding_Details", "") for r in result["csv_data"] + ) + + @patch("app.boto3.client") + def test_bucket_access_error_surfaces_as_could_not_assess(self, mock_client): + """An AccessDenied on get_bucket_versioning must re-raise → ERROR envelope + (could-not-assess), NOT a false 'no versioning' finding.""" + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1"}], + data_sources_by_kb={"kb1": [{"dataSourceId": "ds1"}]}, + data_source_detail={ + ("kb1", "ds1"): { + "dataSource": { + "dataSourceConfiguration": { + "s3Configuration": { + "bucketArn": "arn:aws:s3:::locked-bucket" + } + } + } + } + }, + ) + ) + + def side_effect(service, **kwargs): + if service == "s3": + c = MagicMock() + c.get_bucket_versioning.side_effect = _client_error("AccessDenied") + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_knowledge_base_integrity_monitoring(inv) + _assert_structure(result) + assert result["status"] == "ERROR" + + +# ========================================================================= +# FS-34 — lines 2056-2057: FM version currency pass path +# ========================================================================= + + +class TestFS34FmVersionPass: + @patch("app.boto3.client") + def test_pass_no_legacy_models_in_use(self, mock_client): + """Lines 2056-2057: no legacy models in use → Passed.""" + c = MagicMock() + c.list_foundation_models.return_value = { + "modelSummaries": [ + { + "modelId": "anthropic.claude-v2", + "modelLifecycle": {"status": "ACTIVE"}, + } + ] + } + c.list_custom_models.return_value = {"modelSummaries": []} + mock_client.return_value = c + result = app.check_fm_version_currency() + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-35 — line 2127: eval jobs found pass path +# ========================================================================= + + +class TestFS35FmevalPass: + def test_advisory_finding(self): + """FS-35 is advisory (REQ-10a): always one N/A Informational ADVISORY row.""" + result = app.check_fmeval_harmful_content() + _assert_structure(result) + assert any( + r["Status"] == "N/A" + and r["Severity"] == "Informational" + and r["Finding"].startswith("ADVISORY: ") + for r in result["csv_data"] + ) + + +# ========================================================================= +# FS-36 — lines 2177-2178: content filters warn path +# ========================================================================= + + +class TestFS36ContentFiltersWarn: + def test_warn_guardrails_without_content_filters(self): + """Lines 2177-2178: guardrails exist but no content filters → WARN.""" + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={"g1": {"contentPolicy": {"filters": []}}}, + ) + ) + result = app.check_guardrail_content_filters(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + +# ========================================================================= +# FS-38 — lines 2266-2309: word filters warn/pass paths +# ========================================================================= + + +class TestFS38WordFiltersPaths: + def test_warn_guardrails_without_word_filters(self): + """Lines 2266-2276: guardrails exist but no word filters → WARN.""" + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": {"wordPolicy": {"words": [], "managedWordLists": []}} + }, + ) + ) + result = app.check_guardrail_word_filters(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + def test_pass_word_filters_configured(self): + """Line 2296: guardrail with word filters → Passed.""" + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": { + "wordPolicy": { + "words": [{"text": "insider trading"}], + "managedWordLists": [{"type": "PROFANITY"}], + } + } + }, + ) + ) + result = app.check_guardrail_word_filters(inv) + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-39 — line 2352: Clarify bias pass path +# ========================================================================= + + +class TestFS39ClarifyBiasPass: + @patch("app.boto3.client") + def test_pass_bias_schedules_found(self, mock_client): + """Line 2352: bias monitoring schedules found → Passed.""" + c = MagicMock() + c.list_monitoring_schedules.return_value = { + "MonitoringScheduleSummaries": [ + { + "MonitoringScheduleName": "bias-monitor", + "MonitoringType": "ModelBias", + } + ] + } + mock_client.return_value = c + result = app.check_sagemaker_clarify_bias() + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-40 — line 2397: bias eval pass path +# ========================================================================= + + +class TestFS40BiasEvalPass: + def test_advisory_finding(self): + """FS-40 is advisory (REQ-10a): always one N/A Informational ADVISORY row.""" + result = app.check_bedrock_evaluation_bias_datasets() + _assert_structure(result) + assert any( + r["Status"] == "N/A" + and r["Severity"] == "Informational" + and r["Finding"].startswith("ADVISORY: ") + for r in result["csv_data"] + ) + + +# ========================================================================= +# FS-41 — line 2452: Clarify explainability pass path +# ========================================================================= + + +class TestFS41ClarifyExplainabilityPass: + @patch("app.boto3.client") + def test_pass_explainability_schedules_found(self, mock_client): + """Line 2452: explainability monitoring schedules found → Passed.""" + c = MagicMock() + c.list_monitoring_schedules.return_value = { + "MonitoringScheduleSummaries": [ + { + "MonitoringScheduleName": "explain-monitor", + "MonitoringType": "ModelExplainability", + } + ] + } + mock_client.return_value = c + result = app.check_sagemaker_clarify_explainability() + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-42 — line 2501: model cards pass path +# ========================================================================= + + +class TestFS42ModelCardsPass: + @patch("app.boto3.client") + def test_pass_model_cards_found(self, mock_client): + """Line 2501: model cards exist → Passed (key is ModelCardSummaryList).""" + c = MagicMock() + c.list_model_cards.return_value = { + "ModelCardSummaryList": [{"ModelCardName": "fraud-model-card"}] + } + mock_client.return_value = c + result = app.check_ai_service_cards_documentation() + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-43 — lines 2536-2541: CloudWatch PII masking paths +# ========================================================================= + + +class TestFS43CloudwatchPiiPaths: + @patch("app.boto3.client") + def test_warn_no_data_protection_policies(self, mock_client): + """Lines 2540-2541: no data protection policies → WARN.""" + c = MagicMock() + c.describe_account_policies.return_value = {"accountPolicies": []} + mock_client.return_value = c + result = app.check_cloudwatch_log_pii_masking() + _assert_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_client_error_on_describe_policies_treated_as_no_policies( + self, mock_client + ): + """Line 2536: ClientError on describe_account_policies → policies = [] → WARN.""" + c = MagicMock() + c.describe_account_policies.side_effect = _client_error("AccessDeniedException") + mock_client.return_value = c + result = app.check_cloudwatch_log_pii_masking() + _assert_structure(result) + assert result["status"] == "WARN" + + +# ========================================================================= +# FS-44 — lines 2589-2628: Macie paths +# ========================================================================= + + +class TestFS44MaciePaths: + @patch("app.boto3.client") + def test_warn_macie_not_enabled(self, mock_client): + """Lines 2593-2595: Macie session status not ENABLED → WARN.""" + c = MagicMock() + c.get_macie_session.return_value = {"status": "PAUSED"} + mock_client.return_value = c + result = app.check_macie_on_training_data_buckets() + _assert_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_warn_macie_client_error(self, mock_client): + """Lines 2590-2591: ClientError on get_macie_session → macie_enabled=False → WARN.""" + c = MagicMock() + c.get_macie_session.side_effect = _client_error("AccessDeniedException") + mock_client.return_value = c + result = app.check_macie_on_training_data_buckets() + _assert_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_pass_macie_enabled(self, mock_client): + """Line 2615: Macie enabled → Passed.""" + c = MagicMock() + c.get_macie_session.return_value = {"status": "ENABLED"} + mock_client.return_value = c + result = app.check_macie_on_training_data_buckets() + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-45 — lines 2665-2666: PII filters warn path +# ========================================================================= + + +class TestFS45PiiFiltersWarn: + def test_warn_guardrails_without_pii_filters(self): + """Lines 2665-2666: guardrails exist but no PII filters → WARN.""" + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": {"sensitiveInformationPolicy": {"piiEntities": []}} + }, + ) + ) + result = app.check_guardrail_pii_filters(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + +# ========================================================================= +# FS-46 — lines 2734-2778: data classification tagging paths +# ========================================================================= + + +class TestFS46DataClassificationPaths: + @patch("app.boto3.client") + def test_warn_buckets_without_classification_tags(self, mock_client): + """Lines 2734-2746: AI/ML buckets without classification tags → WARN.""" + inv = make_resource_inventory(buckets=[{"Name": "aiml-training-data"}]) + c = MagicMock() + c.get_bucket_tagging.return_value = { + "TagSet": [{"Key": "Environment", "Value": "prod"}] + } + mock_client.return_value = c + + result = app.check_data_classification_tagging(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_warn_tagging_client_error_treated_as_unclassified(self, mock_client): + """Line 2741-2742: ClientError on get_bucket_tagging → bucket added as unclassified.""" + inv = make_resource_inventory(buckets=[{"Name": "aiml-training-data"}]) + c = MagicMock() + c.get_bucket_tagging.side_effect = _client_error("NoSuchTagSet") + mock_client.return_value = c + + result = app.check_data_classification_tagging(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + @patch("app.boto3.client") + def test_tagging_access_error_surfaces_as_could_not_assess(self, mock_client): + """AccessDenied on get_bucket_tagging re-raises → ERROR (could-not-assess), + NOT a false 'unclassified' finding.""" + inv = make_resource_inventory(buckets=[{"Name": "aiml-training-data"}]) + c = MagicMock() + c.get_bucket_tagging.side_effect = _client_error("AccessDenied") + mock_client.return_value = c + + result = app.check_data_classification_tagging(inv) + _assert_structure(result) + assert result["status"] == "ERROR" + + @patch("app.boto3.client") + def test_pass_all_buckets_classified(self, mock_client): + """Line 2765: all AI/ML buckets have classification tags → Passed.""" + inv = make_resource_inventory(buckets=[{"Name": "aiml-training-data"}]) + c = MagicMock() + c.get_bucket_tagging.return_value = { + "TagSet": [{"Key": "data-classification", "Value": "Confidential"}] + } + mock_client.return_value = c + + result = app.check_data_classification_tagging(inv) + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + @patch("app.boto3.client") + def test_multi_page_buckets_completeness(self, mock_client): + """Pagination completeness: buckets from multiple pages are all assessed. + + The inventory (already fully paginated by the collector) contains buckets + from 2+ simulated pages. This test verifies the check inspects all of them. + """ + # Simulate 2 "pages" of buckets — all AI/ML-named so they're all filtered in + page1 = [{"Name": "train-data-page1-001"}, {"Name": "bedrock-page1-002"}] + page2 = [{"Name": "knowledge-page2-001"}, {"Name": "sagemaker-page2-002"}] + all_buckets = page1 + page2 + + inv = make_resource_inventory(buckets=all_buckets) + c = MagicMock() + c.get_bucket_tagging.return_value = { + "TagSet": [{"Key": "data-classification", "Value": "Internal"}] + } + mock_client.return_value = c + + result = app.check_data_classification_tagging(inv) + _assert_structure(result) + # All 4 buckets classified → Passed + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + # Verify all 4 were assessed (tagging call per AI/ML bucket) + assert c.get_bucket_tagging.call_count == 4 + + @patch("app.boto3.client") + def test_single_page_unchanged_vs_baseline(self, mock_client): + """Single-page case: result is identical to pre-migration behavior (baseline). + + With ≤1 page of buckets, the inventory holds all buckets and the check + outcome is the same as before migration. + """ + inv = make_resource_inventory(buckets=[{"Name": "aiml-training-data"}]) + c = MagicMock() + c.get_bucket_tagging.return_value = { + "TagSet": [{"Key": "Environment", "Value": "prod"}] + } + mock_client.return_value = c + + result = app.check_data_classification_tagging(inv) + _assert_structure(result) + assert result["status"] == "WARN" + assert any( + r["Finding"] == "AI/ML Buckets Without Data Classification Tags" + for r in result["csv_data"] + ) + + +# ========================================================================= +# FS-47 — lines 2812-2857: grounding threshold warn/pass paths +# ========================================================================= + + +class TestFS47GroundingThresholdPaths: + def test_warn_low_grounding_threshold(self): + """Lines 2812-2826: guardrail with grounding threshold < 0.7 → WARN.""" + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": { + "contextualGroundingPolicy": { + "filters": [{"type": "GROUNDING", "threshold": 0.5}] + } + } + }, + ) + ) + result = app.check_guardrail_grounding_threshold(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + def test_pass_adequate_grounding_threshold(self): + """Line 2844: guardrail with grounding threshold >= 0.7 → Passed.""" + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": { + "contextualGroundingPolicy": { + "filters": [{"type": "GROUNDING", "threshold": 0.8}] + } + } + }, + ) + ) + result = app.check_guardrail_grounding_threshold(inv) + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + def test_fail_no_grounding_filter_at_all(self): + """Guardrails exist but NONE has a GROUNDING filter (only RELEVANCE) → + Failed, not a false Pass. Regression guard for the FS-47 false-pass fix.""" + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": { + "contextualGroundingPolicy": { + "filters": [{"type": "RELEVANCE", "threshold": 0.9}] + } + } + }, + ) + ) + result = app.check_guardrail_grounding_threshold(inv) + _assert_structure(result) + assert result["status"] == "WARN" + assert any(r["Status"] == "Failed" for r in result["csv_data"]) + assert not any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-48 — line 2898: RAG KB pass path (active KBs) +# ========================================================================= + + +class TestFS48RagKbPass: + def test_pass_active_kbs_found(self): + """Line 2898: active KBs found → Passed.""" + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[ + {"knowledgeBaseId": "kb1", "name": "rag-kb", "status": "ACTIVE"} + ], + data_sources_by_kb={}, + data_source_detail={}, + ) + ) + result = app.check_rag_knowledge_base_configured(inv) + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-50 — lines 2957-2985: ARC relevance grounding paths +# ========================================================================= + + +class TestFS50ArcRelevancePaths: + def test_warn_no_relevance_filters(self): + """Lines 2957-2963: guardrails exist but no RELEVANCE filter → WARN.""" + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": { + "contextualGroundingPolicy": { + "filters": [{"type": "GROUNDING", "threshold": 0.8}] + } + } + }, + ) + ) + result = app.check_guardrail_relevance_grounding(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + def test_pass_relevance_filter_found(self): + """Guardrail with RELEVANCE filter → Passed.""" + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": { + "contextualGroundingPolicy": { + "filters": [{"type": "RELEVANCE", "threshold": 0.7}] + } + } + }, + ) + ) + result = app.check_guardrail_relevance_grounding(inv) + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-51 — lines 3037-3038: prompt injection warn path +# ========================================================================= + + +class TestFS51PromptInjectionWarn: + def test_warn_no_prompt_attack_filter(self): + """Lines 3037-3038: guardrails exist but no PROMPT_ATTACK filter → WARN.""" + inv = make_resource_inventory( + guardrails=app.GuardrailInventory( + summaries=[{"id": "g1", "name": "guard1"}], + detail_by_id={ + "g1": { + "contentPolicy": { + "filters": [{"type": "HATE", "inputStrength": "HIGH"}] + } + } + }, + ) + ) + result = app.check_prompt_injection_input_validation(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + +# ========================================================================= +# FS-52 — lines 3105-3146: SDK version currency paths +# ========================================================================= + + +class TestFS52SdkVersionPaths: + def test_warn_deprecated_runtime(self): + """Lines 3105-3113: Bedrock Lambda on deprecated runtime → WARN.""" + inv = make_resource_inventory( + lambda_functions=[ + {"FunctionName": "bedrock-invoke-fn", "Runtime": "python3.8"} + ] + ) + result = app.check_bedrock_sdk_version_currency(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + def test_pass_current_runtime(self): + """Line 3133: Bedrock Lambda on current runtime → Passed.""" + inv = make_resource_inventory( + lambda_functions=[ + {"FunctionName": "bedrock-invoke-fn", "Runtime": "python3.12"} + ] + ) + result = app.check_bedrock_sdk_version_currency(inv) + _assert_structure(result) + assert any(r["Status"] == "Passed" for r in result["csv_data"]) + + +# ========================================================================= +# FS-53 — lines 3192-3196: WAF injection rules warn path +# ========================================================================= + + +class TestFS53WafInjectionWarn: + def test_warn_acls_without_injection_rules(self): + """Lines 3192-3196: WAF ACLs exist but no injection rule groups → WARN.""" + acl_detail = { + "Rules": [ + { + "Name": "rate-limit", + "Statement": {"RateBasedStatement": {"Limit": 1000}}, + } + ] + } + inv = make_resource_inventory( + web_acls=app.WebAclInventory( + summaries=[{"Name": "acl1", "Id": "id1"}], + detail_by_id={"id1": acl_detail}, + ) + ) + result = app.check_waf_sql_injection_rules(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + +# ========================================================================= +# FS-65 — lines 3770, 3781-3782: S3 event notification edge cases +# ========================================================================= + + +class TestFS65S3EventNotificationEdgeCases: + def test_skip_datasource_with_no_bucket(self): + """Line 3770: data source with no bucket ARN → continue (no bucket added).""" + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1"}], + data_sources_by_kb={"kb1": [{"dataSourceId": "ds1"}]}, + data_source_detail={ + ("kb1", "ds1"): { + "dataSource": { + "dataSourceConfiguration": { + "s3Configuration": {} # no bucketArn + } + } + } + }, + ) + ) + result = app.check_kb_datasource_s3_event_notifications(inv) + _assert_structure(result) + # No buckets to check → PASS + assert result["status"] == "PASS" + + @patch("app.boto3.client") + def test_s3_notification_access_error_surfaces_as_could_not_assess( + self, mock_client + ): + """An AccessDenied on get_bucket_notification_configuration must re-raise → + ERROR envelope (could-not-assess), NOT a false 'missing notifications' finding.""" + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1"}], + data_sources_by_kb={"kb1": [{"dataSourceId": "ds1"}]}, + data_source_detail={ + ("kb1", "ds1"): { + "dataSource": { + "dataSourceConfiguration": { + "s3Configuration": { + "bucketArn": "arn:aws:s3:::kb-bucket" + } + } + } + } + }, + ) + ) + + def side_effect(service, **kwargs): + if service == "s3": + c = MagicMock() + c.get_bucket_notification_configuration.side_effect = _client_error( + "AccessDenied" + ) + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_kb_datasource_s3_event_notifications(inv) + _assert_structure(result) + assert result["status"] == "ERROR" + + @patch("app.boto3.client") + def test_deleted_bucket_reported_separately(self, mock_client): + """A NoSuchBucket on get_bucket_notification_configuration → distinct + 'deleted bucket' finding (High), not conflated with 'missing + notifications' or labeled '(error)'. Regression guard for the FS-65 + NoSuchBucket refinement.""" + inv = make_resource_inventory( + knowledge_bases=app.KbInventory( + summaries=[{"knowledgeBaseId": "kb1", "name": "kb-one"}], + data_sources_by_kb={"kb1": [{"dataSourceId": "ds1", "name": "ds-one"}]}, + data_source_detail={ + ("kb1", "ds1"): { + "dataSource": { + "dataSourceConfiguration": { + "s3Configuration": { + "bucketArn": "arn:aws:s3:::deleted-kb-bucket" + } + } + } + } + }, + ) + ) + + def side_effect(service, **kwargs): + if service == "s3": + c = MagicMock() + c.get_bucket_notification_configuration.side_effect = _client_error( + "NoSuchBucket" + ) + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_kb_datasource_s3_event_notifications(inv) + _assert_structure(result) + assert result["status"] == "WARN" + deleted = [ + r + for r in result["csv_data"] + if r["Finding"] == "KB Data Source References a Deleted S3 Bucket" + ] + assert deleted, "expected a distinct deleted-bucket finding" + assert deleted[0]["Severity"] == "High" + assert "deleted-kb-bucket" in deleted[0]["Finding_Details"] + assert not any( + "(error)" in r.get("Finding_Details", "") for r in result["csv_data"] + ) + assert not any( + r["Finding"] == "KB Data Source Buckets Missing S3 Event Notifications" + for r in result["csv_data"] + ) + + +class TestFS66AgentcoreIdentityReraise: + @patch("app.boto3.client") + def test_non_access_denied_reraises(self, mock_client): + """Line 3849: non-AccessDenied ClientError → re-raised → ERROR.""" + c = MagicMock() + c.list_agent_runtimes.side_effect = _client_error("ServiceUnavailableException") + mock_client.return_value = c + result = app.check_agentcore_end_user_identity_propagation() + _assert_structure(result) + assert result["status"] == "ERROR" + + +# ========================================================================= +# FS-68 — lines 4031, 4049, 4053, 4056-4057: body size limit warn paths +# ========================================================================= + + +class TestFS68BodySizeLimitWarnPaths: + def test_warn_rest_api_without_validators(self): + """Lines 4031, 4049, 4056-4057: REST API without validators → WARN.""" + inv = make_resource_inventory( + web_acls=app.WebAclInventory(summaries=[], detail_by_id={}) + ) + with patch("app.boto3.client") as mock_client: + + def side_effect(service, **kwargs): + if service == "apigateway": + c = MagicMock() + c.get_rest_apis.return_value = { + "items": [{"id": "api1", "name": "genai-api"}] + } + c.get_request_validators.return_value = { + "items": [] + } # no validators + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_api_gateway_request_body_size_limits(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + def test_warn_waf_acls_without_size_rules(self): + """Lines 4053, 4056-4057: WAF ACLs exist but no size constraint rules → WARN.""" + acl_detail = { + "Rules": [ + { + "Name": "rate-limit", + "Statement": {"RateBasedStatement": {}}, + } + ] + } + inv = make_resource_inventory( + web_acls=app.WebAclInventory( + summaries=[{"Name": "acl1", "Id": "id1"}], + detail_by_id={"id1": acl_detail}, + ) + ) + with patch("app.boto3.client") as mock_client: + + def side_effect(service, **kwargs): + if service == "apigateway": + c = MagicMock() + c.get_rest_apis.return_value = {"items": []} + return c + return MagicMock() + + mock_client.side_effect = side_effect + result = app.check_api_gateway_request_body_size_limits(inv) + _assert_structure(result) + assert result["status"] == "WARN" + + +# ========================================================================= +# FS-34 — lines 2056-2057: legacy models warn path +# ========================================================================= + + +class TestFS34FmVersionWarn: + @patch("app.boto3.client") + def test_warn_legacy_models_available(self, mock_client): + """Legacy foundation models available in region → WARN wrapper with an N/A + finding (availability is not usage, so it is surfaced for review, not failed).""" + c = MagicMock() + c.list_foundation_models.return_value = { + "modelSummaries": [ + {"modelId": "old-model-v1", "modelLifecycle": {"status": "LEGACY"}} + ] + } + c.list_custom_models.return_value = {"modelSummaries": []} + mock_client.return_value = c + result = app.check_fm_version_currency() + _assert_structure(result) + assert result["status"] == "WARN" + assert any(r["Status"] == "N/A" for r in result["csv_data"]) + assert any( + "availability" in r["Finding_Details"].lower() for r in result["csv_data"] + ) diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_collector.py b/aiml-security-assessment/functions/security/finserv_tests/test_collector.py new file mode 100644 index 0000000..ecf603b --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_tests/test_collector.py @@ -0,0 +1,685 @@ +""" +Unit tests for collect_resource_inventory() and the _safe_collect_* helpers. + +Validates: Requirements REQ-1, REQ-2, REQ-3.4, REQ-4, REQ-7.5, REQ-9.2 +""" + +import sys +import os +from unittest.mock import MagicMock, patch + +import pytest + +FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") +if FINSERV_DIR not in sys.path: + sys.path.insert(0, FINSERV_DIR) + +import app # noqa: E402 (import follows sys.path bootstrap above) + + +# --------------------------------------------------------------------------- +# Helpers to build mock boto3 clients +# --------------------------------------------------------------------------- + + +def _make_client(responses: dict) -> MagicMock: + """Return a MagicMock boto3 client where each key in *responses* is a method + name and the value is either a single dict (returned every call) or a list + of dicts (returned in sequence, raising StopIteration when exhausted).""" + client = MagicMock() + for method_name, return_values in responses.items(): + if isinstance(return_values, list): + method = getattr(client, method_name) + method.side_effect = return_values + else: + method = getattr(client, method_name) + method.return_value = return_values + return client + + +def _single_page(key, items, extra=None): + """Build a single-page response dict with no continuation token.""" + r = {key: items} + if extra: + r.update(extra) + return r + + +# --------------------------------------------------------------------------- +# _safe_collect_lambda_functions +# --------------------------------------------------------------------------- + + +class TestSafeCollectLambdaFunctions: + def test_single_page_returns_functions(self): + fn = {"FunctionName": "my-fn", "Runtime": "python3.12"} + client = _make_client({"list_functions": {"Functions": [fn]}}) + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_lambda_functions() + assert result == [fn] + + def test_single_listing_call(self): + client = _make_client({"list_functions": {"Functions": []}}) + with patch("app.boto3.client", return_value=client): + app._safe_collect_lambda_functions() + assert client.list_functions.call_count == 1 + + def test_multi_page_merges_all_functions(self): + """Multi-page result via NextMarker/Marker pagination is merged correctly.""" + fn1 = {"FunctionName": "fn-1"} + fn2 = {"FunctionName": "fn-2"} + client = _make_client( + { + "list_functions": [ + {"Functions": [fn1], "NextMarker": "page2"}, + {"Functions": [fn2]}, + ] + } + ) + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_lambda_functions() + assert result == [fn1, fn2] + assert client.list_functions.call_count == 2 + # Second call must use Marker= (Lambda convention) + _, kwargs = client.list_functions.call_args + assert "Marker" in kwargs + + def test_failure_returns_unavailable(self): + err = PermissionError("AccessDenied") + client = _make_client({}) + client.list_functions.side_effect = err + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_lambda_functions() + assert isinstance(result, app._Unavailable) + assert result.error is err + + def test_client_constructed_without_region_or_endpoint(self): + client = _make_client({"list_functions": {"Functions": []}}) + with patch("app.boto3.client", return_value=client) as mock_boto: + app._safe_collect_lambda_functions() + mock_boto.assert_called_once() + _, kwargs = mock_boto.call_args + assert "region_name" not in kwargs + assert "endpoint_url" not in kwargs + + +# --------------------------------------------------------------------------- +# _safe_collect_guardrails +# --------------------------------------------------------------------------- + + +class TestSafeCollectGuardrails: + def test_single_page_returns_guardrail_inventory(self): + g1 = {"id": "g-abc", "name": "my-guardrail"} + detail = {"guardrailId": "g-abc", "sensitiveInformationPolicy": {}} + client = _make_client( + { + "list_guardrails": {"guardrails": [g1]}, + "get_guardrail": detail, + } + ) + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_guardrails() + assert isinstance(result, app.GuardrailInventory) + assert result.summaries == [g1] + assert result.detail_by_id["g-abc"] is detail + + def test_single_listing_call(self): + client = _make_client( + { + "list_guardrails": {"guardrails": []}, + } + ) + with patch("app.boto3.client", return_value=client): + app._safe_collect_guardrails() + assert client.list_guardrails.call_count == 1 + + def test_multi_page_merges_guardrails(self): + g1 = {"id": "g1"} + g2 = {"id": "g2"} + client = _make_client( + { + "list_guardrails": [ + {"guardrails": [g1], "nextToken": "tok2"}, + {"guardrails": [g2]}, + ], + "get_guardrail": {"guardrailId": "gx"}, + } + ) + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_guardrails() + assert len(result.summaries) == 2 + assert client.list_guardrails.call_count == 2 + + def test_get_guardrail_called_with_draft(self): + """get_guardrail must always be called with guardrailVersion='DRAFT'.""" + g1 = {"id": "g-001"} + detail = {"guardrailId": "g-001"} + client = _make_client( + { + "list_guardrails": {"guardrails": [g1]}, + "get_guardrail": detail, + } + ) + with patch("app.boto3.client", return_value=client): + app._safe_collect_guardrails() + client.get_guardrail.assert_called_once_with( + guardrailIdentifier="g-001", guardrailVersion="DRAFT" + ) + + def test_whole_inventory_failure_returns_unavailable(self): + err = PermissionError("AccessDenied on list_guardrails") + client = _make_client({}) + client.list_guardrails.side_effect = err + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_guardrails() + assert isinstance(result, app._Unavailable) + assert result.error is err + + def test_single_detail_failure_isolates_to_that_id(self): + """A get_guardrail failure for one id stores _Unavailable only for that id.""" + g1 = {"id": "g-ok"} + g2 = {"id": "g-bad"} + ok_detail = {"guardrailId": "g-ok"} + err = PermissionError("denied") + + def detail_side_effect(**kwargs): + if kwargs["guardrailIdentifier"] == "g-bad": + raise err + return ok_detail + + client = _make_client({"list_guardrails": {"guardrails": [g1, g2]}}) + client.get_guardrail.side_effect = detail_side_effect + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_guardrails() + assert isinstance(result, app.GuardrailInventory) + assert result.detail_by_id["g-ok"] is ok_detail + assert isinstance(result.detail_by_id["g-bad"], app._Unavailable) + assert result.detail_by_id["g-bad"].error is err + + def test_client_constructed_without_region_or_endpoint(self): + client = _make_client({"list_guardrails": {"guardrails": []}}) + with patch("app.boto3.client", return_value=client) as mock_boto: + app._safe_collect_guardrails() + mock_boto.assert_called_once() + _, kwargs = mock_boto.call_args + assert "region_name" not in kwargs + assert "endpoint_url" not in kwargs + + +# --------------------------------------------------------------------------- +# _safe_collect_knowledge_bases +# --------------------------------------------------------------------------- + + +class TestSafeCollectKnowledgeBases: + def _setup_client(self, kb_list, ds_by_kb=None, ds_detail_by_pair=None): + """Build a mock bedrock-agent client.""" + client = MagicMock() + # list_knowledge_bases + client.list_knowledge_bases.return_value = {"knowledgeBaseSummaries": kb_list} + + # list_data_sources + def list_ds(**kwargs): + kb_id = kwargs["knowledgeBaseId"] + items = (ds_by_kb or {}).get(kb_id, []) + return {"dataSourceSummaries": items} + + client.list_data_sources.side_effect = list_ds + + # get_data_source + def get_ds(**kwargs): + pair = (kwargs["knowledgeBaseId"], kwargs["dataSourceId"]) + if ds_detail_by_pair and pair in ds_detail_by_pair: + v = ds_detail_by_pair[pair] + if isinstance(v, Exception): + raise v + return v + return {"dataSource": {"knowledgeBaseId": kwargs["knowledgeBaseId"]}} + + client.get_data_source.side_effect = get_ds + return client + + def test_single_kb_single_ds(self): + kb1 = {"knowledgeBaseId": "kb-1"} + ds1 = {"dataSourceId": "ds-1"} + detail = {"dataSource": {"dataSourceId": "ds-1"}} + client = self._setup_client([kb1], {"kb-1": [ds1]}, {("kb-1", "ds-1"): detail}) + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_knowledge_bases() + assert isinstance(result, app.KbInventory) + assert result.summaries == [kb1] + assert result.data_sources_by_kb["kb-1"] == [ds1] + assert result.data_source_detail[("kb-1", "ds-1")] is detail + + def test_single_listing_call(self): + client = self._setup_client([]) + with patch("app.boto3.client", return_value=client): + app._safe_collect_knowledge_bases() + assert client.list_knowledge_bases.call_count == 1 + + def test_multi_page_kbs(self): + kb1 = {"knowledgeBaseId": "kb-1"} + kb2 = {"knowledgeBaseId": "kb-2"} + client = MagicMock() + client.list_knowledge_bases.side_effect = [ + {"knowledgeBaseSummaries": [kb1], "nextToken": "tok2"}, + {"knowledgeBaseSummaries": [kb2]}, + ] + client.list_data_sources.return_value = {"dataSourceSummaries": []} + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_knowledge_bases() + assert len(result.summaries) == 2 + assert client.list_knowledge_bases.call_count == 2 + + def test_whole_inventory_failure_returns_unavailable(self): + err = PermissionError("denied") + client = MagicMock() + client.list_knowledge_bases.side_effect = err + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_knowledge_bases() + assert isinstance(result, app._Unavailable) + assert result.error is err + + def test_per_kb_data_source_failure_isolates(self): + """list_data_sources failure for one KB stores _Unavailable for that KB only.""" + kb1 = {"knowledgeBaseId": "kb-ok"} + kb2 = {"knowledgeBaseId": "kb-bad"} + err = PermissionError("denied for kb-bad") + + client = MagicMock() + client.list_knowledge_bases.return_value = { + "knowledgeBaseSummaries": [kb1, kb2] + } + + def list_ds(**kwargs): + if kwargs["knowledgeBaseId"] == "kb-bad": + raise err + return {"dataSourceSummaries": []} + + client.list_data_sources.side_effect = list_ds + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_knowledge_bases() + assert isinstance(result, app.KbInventory) + assert result.data_sources_by_kb["kb-ok"] == [] + assert isinstance(result.data_sources_by_kb["kb-bad"], app._Unavailable) + assert result.data_sources_by_kb["kb-bad"].error is err + + def test_per_data_source_detail_failure_isolates(self): + """get_data_source failure for one DS stores _Unavailable for that (kb, ds) only.""" + kb1 = {"knowledgeBaseId": "kb-1"} + ds_ok = {"dataSourceId": "ds-ok"} + ds_bad = {"dataSourceId": "ds-bad"} + ok_detail = {"dataSource": {"dataSourceId": "ds-ok"}} + err = PermissionError("denied") + + client = self._setup_client( + [kb1], + {"kb-1": [ds_ok, ds_bad]}, + {("kb-1", "ds-ok"): ok_detail, ("kb-1", "ds-bad"): err}, + ) + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_knowledge_bases() + assert result.data_source_detail[("kb-1", "ds-ok")] is ok_detail + assert isinstance( + result.data_source_detail[("kb-1", "ds-bad")], app._Unavailable + ) + + def test_client_constructed_without_region_or_endpoint(self): + client = self._setup_client([]) + with patch("app.boto3.client", return_value=client) as mock_boto: + app._safe_collect_knowledge_bases() + mock_boto.assert_called_once() + _, kwargs = mock_boto.call_args + assert "region_name" not in kwargs + assert "endpoint_url" not in kwargs + + +# --------------------------------------------------------------------------- +# _safe_collect_buckets +# --------------------------------------------------------------------------- + + +class TestSafeCollectBuckets: + def test_single_page_returns_buckets(self): + b1 = {"Name": "my-bucket"} + client = _make_client({"list_buckets": {"Buckets": [b1]}}) + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_buckets() + assert result == [b1] + + def test_single_listing_call(self): + client = _make_client({"list_buckets": {"Buckets": []}}) + with patch("app.boto3.client", return_value=client): + app._safe_collect_buckets() + assert client.list_buckets.call_count == 1 + + def test_multi_page_with_continuation_token(self): + """S3 ContinuationToken/ContinuationToken pagination is used (REQ-2.8).""" + b1 = {"Name": "bucket-1"} + b2 = {"Name": "bucket-2"} + client = _make_client( + { + "list_buckets": [ + {"Buckets": [b1], "ContinuationToken": "tok2"}, + {"Buckets": [b2]}, + ] + } + ) + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_buckets() + assert result == [b1, b2] + assert client.list_buckets.call_count == 2 + # Second call must use ContinuationToken= as input (not NextToken etc.) + _, kwargs = client.list_buckets.call_args + assert "ContinuationToken" in kwargs + + def test_max_buckets_parameter_sent_on_first_call(self): + """MaxBuckets must be included on the first call to engage pagination.""" + client = _make_client({"list_buckets": {"Buckets": []}}) + with patch("app.boto3.client", return_value=client): + app._safe_collect_buckets() + first_call_kwargs = client.list_buckets.call_args_list[0][1] + assert first_call_kwargs.get("MaxBuckets") == 1000 + + def test_failure_returns_unavailable(self): + err = PermissionError("AccessDenied") + client = MagicMock() + client.list_buckets.side_effect = err + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_buckets() + assert isinstance(result, app._Unavailable) + assert result.error is err + + def test_client_constructed_without_region_or_endpoint(self): + client = _make_client({"list_buckets": {"Buckets": []}}) + with patch("app.boto3.client", return_value=client) as mock_boto: + app._safe_collect_buckets() + mock_boto.assert_called_once() + _, kwargs = mock_boto.call_args + assert "region_name" not in kwargs + assert "endpoint_url" not in kwargs + + +# --------------------------------------------------------------------------- +# _safe_collect_web_acls +# --------------------------------------------------------------------------- + + +class TestSafeCollectWebAcls: + def test_single_page_returns_web_acl_inventory(self): + acl1 = {"Id": "acl-1", "Name": "my-acl", "ARN": "arn:aws:wafv2:::webacl/acl-1"} + detail = {"WebACL": {"Id": "acl-1"}} + client = _make_client( + { + "list_web_acls": {"WebACLs": [acl1]}, + "get_web_acl": detail, + } + ) + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_web_acls() + assert isinstance(result, app.WebAclInventory) + assert result.summaries == [acl1] + # The collector extracts response["WebACL"] — not the full envelope — + # so downstream checks can do detail.get("Rules") without extra indirection. + assert result.detail_by_id["acl-1"] == detail["WebACL"] + + def test_single_listing_call(self): + client = _make_client( + { + "list_web_acls": {"WebACLs": []}, + } + ) + with patch("app.boto3.client", return_value=client): + app._safe_collect_web_acls() + assert client.list_web_acls.call_count == 1 + + def test_list_web_acls_called_with_scope_regional(self): + """Scope='REGIONAL' must be passed on every list_web_acls call (REQ-7.2).""" + client = _make_client({"list_web_acls": {"WebACLs": []}}) + with patch("app.boto3.client", return_value=client): + app._safe_collect_web_acls() + client.list_web_acls.assert_called_once() + _, kwargs = client.list_web_acls.call_args + assert kwargs.get("Scope") == "REGIONAL" + + def test_multi_page_uses_next_marker_as_input(self): + """WAFv2 pagination uses NextMarker as BOTH output and input (REQ-2.2).""" + acl1 = {"Id": "acl-1", "Name": "acl-1"} + acl2 = {"Id": "acl-2", "Name": "acl-2"} + detail = {"WebACL": {}} + client = _make_client( + { + "list_web_acls": [ + {"WebACLs": [acl1], "NextMarker": "mark2"}, + {"WebACLs": [acl2]}, + ], + "get_web_acl": detail, + } + ) + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_web_acls() + assert len(result.summaries) == 2 + assert client.list_web_acls.call_count == 2 + # Second call must use NextMarker= (NOT Marker= which is Lambda's convention) + second_call_kwargs = client.list_web_acls.call_args_list[1][1] + assert "NextMarker" in second_call_kwargs + assert second_call_kwargs["NextMarker"] == "mark2" + assert "Marker" not in second_call_kwargs + + def test_get_web_acl_called_with_scope_regional(self): + """get_web_acl must always use Scope='REGIONAL' (REQ-7.2).""" + acl1 = {"Id": "acl-1", "Name": "acl-name"} + detail = {"WebACL": {"Id": "acl-1"}} + client = _make_client( + { + "list_web_acls": {"WebACLs": [acl1]}, + "get_web_acl": detail, + } + ) + with patch("app.boto3.client", return_value=client): + app._safe_collect_web_acls() + client.get_web_acl.assert_called_once_with( + Name="acl-name", Scope="REGIONAL", Id="acl-1" + ) + + def test_whole_inventory_failure_returns_unavailable(self): + err = PermissionError("AccessDenied") + client = MagicMock() + client.list_web_acls.side_effect = err + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_web_acls() + assert isinstance(result, app._Unavailable) + assert result.error is err + + def test_single_detail_failure_isolates_to_that_id(self): + acl_ok = {"Id": "acl-ok", "Name": "ok-acl"} + acl_bad = {"Id": "acl-bad", "Name": "bad-acl"} + ok_detail = {"WebACL": {"Id": "acl-ok"}} + err = PermissionError("denied") + + def get_web_acl(**kwargs): + if kwargs["Id"] == "acl-bad": + raise err + return ok_detail + + client = _make_client( + { + "list_web_acls": {"WebACLs": [acl_ok, acl_bad]}, + } + ) + client.get_web_acl.side_effect = get_web_acl + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_web_acls() + # Collector extracts ["WebACL"] — verify the stored dict matches the inner value + assert result.detail_by_id["acl-ok"] == ok_detail["WebACL"] + assert isinstance(result.detail_by_id["acl-bad"], app._Unavailable) + assert result.detail_by_id["acl-bad"].error is err + + def test_client_constructed_without_region_or_endpoint(self): + client = _make_client({"list_web_acls": {"WebACLs": []}}) + with patch("app.boto3.client", return_value=client) as mock_boto: + app._safe_collect_web_acls() + mock_boto.assert_called_once() + _, kwargs = mock_boto.call_args + assert "region_name" not in kwargs + assert "endpoint_url" not in kwargs + + +# --------------------------------------------------------------------------- +# collect_resource_inventory — integration +# --------------------------------------------------------------------------- + + +class TestCollectResourceInventory: + """Verify that collect_resource_inventory delegates to each _safe_collect_* + and assembles a ResourceInventory.""" + + def _patch_all_safe_fns(self): + """Context-manager that patches all five _safe_collect_* functions.""" + import contextlib + + @contextlib.contextmanager + def _ctx(): + lf = [{"FunctionName": "fn-1"}] + gi = app.GuardrailInventory(summaries=[{"id": "g-1"}], detail_by_id={}) + ki = app.KbInventory( + summaries=[], data_sources_by_kb={}, data_source_detail={} + ) + bk = [{"Name": "bkt-1"}] + wi = app.WebAclInventory(summaries=[{"Id": "acl-1"}], detail_by_id={}) + with ( + patch("app._safe_collect_lambda_functions", return_value=lf) as p1, + patch("app._safe_collect_guardrails", return_value=gi) as p2, + patch("app._safe_collect_knowledge_bases", return_value=ki) as p3, + patch("app._safe_collect_buckets", return_value=bk) as p4, + patch("app._safe_collect_web_acls", return_value=wi) as p5, + ): + yield p1, p2, p3, p4, p5 + + return _ctx() + + def test_returns_resource_inventory(self): + with self._patch_all_safe_fns(): + inv = app.collect_resource_inventory() + assert isinstance(inv, app.ResourceInventory) + + def test_each_safe_fn_called_exactly_once(self): + with self._patch_all_safe_fns() as (p1, p2, p3, p4, p5): + app.collect_resource_inventory() + for p in (p1, p2, p3, p4, p5): + assert p.call_count == 1 + + def test_inventory_fields_come_from_safe_fns(self): + with self._patch_all_safe_fns(): + inv = app.collect_resource_inventory() + # Spot-check a few field values match what the patched fns returned + assert inv.lambda_functions == [{"FunctionName": "fn-1"}] + assert inv.buckets == [{"Name": "bkt-1"}] + + def test_one_unavailable_does_not_prevent_others(self): + """An _Unavailable on one field doesn't affect the rest.""" + err = PermissionError("denied") + with ( + patch( + "app._safe_collect_lambda_functions", return_value=app._Unavailable(err) + ), + patch( + "app._safe_collect_guardrails", + return_value=app.GuardrailInventory([], {}), + ), + patch( + "app._safe_collect_knowledge_bases", + return_value=app.KbInventory([], {}, {}), + ), + patch("app._safe_collect_buckets", return_value=[]), + patch( + "app._safe_collect_web_acls", return_value=app.WebAclInventory([], {}) + ), + ): + inv = app.collect_resource_inventory() + assert isinstance(inv.lambda_functions, app._Unavailable) + assert isinstance(inv.guardrails, app.GuardrailInventory) + assert inv.buckets == [] + + def test_all_unavailable_still_returns_inventory(self): + """Even when every field fails, we get a ResourceInventory (not an exception).""" + err = RuntimeError("all down") + unav = app._Unavailable(err) + with ( + patch("app._safe_collect_lambda_functions", return_value=unav), + patch("app._safe_collect_guardrails", return_value=unav), + patch("app._safe_collect_knowledge_bases", return_value=unav), + patch("app._safe_collect_buckets", return_value=unav), + patch("app._safe_collect_web_acls", return_value=unav), + ): + inv = app.collect_resource_inventory() + assert isinstance(inv, app.ResourceInventory) + for field in ( + "lambda_functions", + "guardrails", + "knowledge_bases", + "buckets", + "web_acls", + ): + assert isinstance(getattr(inv, field), app._Unavailable) + + def test_unavailable_field_causes_could_not_assess_via_require(self): + """A consuming check using require() on an unavailable field gets the stored + error raised, which the outer try/except turns into _error_findings.""" + err = PermissionError("AccessDenied") + inv = app.ResourceInventory( + lambda_functions=app._Unavailable(err), + guardrails=app.GuardrailInventory([], {}), + knowledge_bases=app.KbInventory([], {}, {}), + buckets=[], + web_acls=app.WebAclInventory([], {}), + ) + with pytest.raises(PermissionError) as exc_info: + app.require(inv, "lambda_functions") + assert exc_info.value is err + + +# --------------------------------------------------------------------------- +# Ordering / listing-order preservation +# --------------------------------------------------------------------------- + + +class TestOrderPreservation: + def test_lambda_functions_order_preserved(self): + fns = [{"FunctionName": f"fn-{i}"} for i in range(5)] + client = _make_client({"list_functions": {"Functions": fns}}) + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_lambda_functions() + assert result == fns + + def test_web_acls_multi_page_order_preserved(self): + acls_p1 = [{"Id": f"acl-{i}", "Name": f"acl-{i}"} for i in range(3)] + acls_p2 = [{"Id": f"acl-{i}", "Name": f"acl-{i}"} for i in range(3, 6)] + client = _make_client( + { + "list_web_acls": [ + {"WebACLs": acls_p1, "NextMarker": "pg2"}, + {"WebACLs": acls_p2}, + ], + "get_web_acl": {"WebACL": {}}, + } + ) + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_web_acls() + assert result.summaries == acls_p1 + acls_p2 + + def test_buckets_multi_page_order_preserved(self): + bkts_p1 = [{"Name": f"bkt-{i}"} for i in range(3)] + bkts_p2 = [{"Name": f"bkt-{i}"} for i in range(3, 6)] + client = _make_client( + { + "list_buckets": [ + {"Buckets": bkts_p1, "ContinuationToken": "pg2"}, + {"Buckets": bkts_p2}, + ] + } + ) + with patch("app.boto3.client", return_value=client): + result = app._safe_collect_buckets() + assert result == bkts_p1 + bkts_p2 diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_inventory_equivalence.py b/aiml-security-assessment/functions/security/finserv_tests/test_inventory_equivalence.py new file mode 100644 index 0000000..67e2c27 --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_tests/test_inventory_equivalence.py @@ -0,0 +1,1093 @@ +""" +Golden-equivalence baseline for the current FinServ lambda_handler. + +Task 1 — FinServ Shared-Inventory Refactor (FU-3) +Requirements: REQ-5.1, REQ-5.3, REQ-5.6 + +Strategy +-------- +Build a reusable single-page "account state" fixture that covers PASS, FAIL, +N/A (status="N/A"), and ADVISORY paths across the inventory-consuming checks: + + * Lambda functions (FS-09, 52, 55, 58, 67, 69) + * Guardrails + detail (FS-27a, 28, 36, 38, 45, 47, 50, 51, 59) + * Knowledge Bases + data sources + detail (FS-24, 31, 33, 48, 61, 65) + * S3 buckets (FS-21, 46) + * WAFv2 Web ACLs + detail (FS-01, 53, 56, 68) + +Constraints (REQ-5.3): ≤100 Web ACLs and ≤1 page of buckets so the fixture +remains single-page comparable after the pagination fix in Task 9. + +The frozen BASELINE literal below was generated by running lambda_handler +against this mock state BEFORE the refactor. After the refactor the handler +must produce the identical tuple sequence (Task 13 / REQ-5.3). +""" + +from __future__ import annotations + +import os +import sys +from datetime import datetime, timezone +from unittest.mock import MagicMock, patch + +import pytest + +# --------------------------------------------------------------------------- +# Make finserv_assessments importable (mirrors conftest.py) +# --------------------------------------------------------------------------- +FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") +if FINSERV_DIR not in sys.path: + sys.path.insert(0, FINSERV_DIR) + +import app # noqa: E402 + + +# =========================================================================== +# Account-state fixture helpers +# =========================================================================== + +# ---- Lambda functions ------------------------------------------------------- +# Two agent functions: one WITH reserved concurrency (→ PASS in FS-09), +# one validate/output function (→ PASS in FS-55, affects FS-58) +# A "schema-validator" function is added so FS-58 finds it (ADVISORY row with count) +_LAMBDA_FUNCTIONS = [ + {"FunctionName": "my-bedrock-agent-handler"}, + {"FunctionName": "my-output-validate-fn"}, + {"FunctionName": "my-schema-validator-fn"}, +] + +# ---- Guardrails ------------------------------------------------------------- +# One guardrail with full configuration (→ PASS paths across FS-27a/28/36/38/45/47/50/51/59) +_GUARDRAIL_SUMMARY = {"id": "gr-abc123", "name": "FinServGuardrail"} + +_GUARDRAIL_DETAIL = { + # topic policy with topics → FS-28 PASS (CLASSIC tier advisory), + # FS-59 PASS (CLASSIC tier advisory) + "topicPolicy": { + "topics": [{"name": "investment-advice", "type": "DENY"}], + "tier": {"tierName": "CLASSIC"}, + }, + # content filters → FS-36 PASS + "contentPolicy": { + "filters": [ + {"type": "HATE", "inputStrength": "HIGH", "outputStrength": "HIGH"}, + {"type": "VIOLENCE", "inputStrength": "MEDIUM", "outputStrength": "MEDIUM"}, + ] + }, + # word filters → FS-38 PASS + "wordPolicy": { + "words": [{"text": "fraud"}], + "managedWordLists": [{"type": "PROFANITY"}], + }, + # PII filters → FS-45 PASS + "sensitiveInformationPolicy": { + "piiEntities": [{"type": "US_SOCIAL_SECURITY_NUMBER", "action": "BLOCK"}] + }, + # contextual grounding → FS-27a PASS; grounding threshold ≥ 0.7 → FS-47 PASS; + # relevance-grounding → FS-50 PASS (single policy covers all three) + "contextualGroundingPolicy": { + "filters": [ + {"type": "GROUNDING", "threshold": 0.8}, + {"type": "RELEVANCE", "threshold": 0.8}, + ] + }, + # topic allowlist (FS-51 checks prompt injection input validation via guardrails) +} + +# ---- Knowledge Bases -------------------------------------------------------- +# One KB, one data source with recent updatedAt → FS-31 PASS +# Data source backed by a versioned S3 bucket → FS-33 PASS +_NOW = datetime.now(timezone.utc) +_KB_SUMMARY = {"knowledgeBaseId": "kb-00001", "name": "FinServKB"} + +_KB_DATA_SOURCE_SUMMARY = { + "dataSourceId": "ds-00001", + "name": "FinServDS", + "updatedAt": _NOW, # fresh → FS-31 PASS +} + +_KB_DATA_SOURCE_DETAIL = { + "dataSource": { + "dataSourceConfiguration": { + "s3Configuration": {"bucketArn": "arn:aws:s3:::my-kb-datasource-bucket"} + } + } +} + +# ---- S3 buckets ------------------------------------------------------------- +# Two buckets within one page (≤ 1 page constraint). +# "my-training-dataset-bucket": versioned → FS-21 PASS +# "my-bedrock-kb-bucket": tagged with data-classification → FS-46 PASS +# "my-kb-datasource-bucket": versioned (for FS-33 data-source check above) +_S3_BUCKETS = [ + {"Name": "my-training-dataset-bucket"}, + {"Name": "my-bedrock-kb-bucket"}, + {"Name": "my-kb-datasource-bucket"}, +] + +# ---- WAFv2 Web ACLs --------------------------------------------------------- +# Two ACLs, both with AWSManagedRulesSQLiRuleSet + AWSManagedRulesCommonRuleSet + +# body-size constraints → FS-53 PASS, FS-56 PASS, FS-68 PASS; FS-01 PASS. +# Constraint: ≤ 100 ACLs. +_ACL_SUMMARIES = [ + {"Name": "FinServACL1", "Id": "acl-id-001", "ARN": "arn:aws:wafv2:::acl-id-001"}, + {"Name": "FinServACL2", "Id": "acl-id-002", "ARN": "arn:aws:wafv2:::acl-id-002"}, +] + +_ACL_DETAIL_TEMPLATE = { + "Name": "FinServACL", + "Id": "acl-id-xxx", + "Rules": [ + { + "Name": "SQLi", + "Statement": { + "ManagedRuleGroupStatement": {"Name": "AWSManagedRulesSQLiRuleSet"} + }, + }, + { + "Name": "Common", + "Statement": { + "ManagedRuleGroupStatement": {"Name": "AWSManagedRulesCommonRuleSet"} + }, + }, + { + "Name": "KnownBadInputs", + "Statement": { + "ManagedRuleGroupStatement": { + "Name": "AWSManagedRulesKnownBadInputsRuleSet" + } + }, + }, + { + "Name": "SizeConstraint", + "Statement": { + "SizeConstraintStatement": { + "FieldToMatch": {"Body": {}}, + "ComparisonOperator": "LE", + "Size": 8192, + "TextTransformations": [{"Priority": 0, "Type": "NONE"}], + } + }, + }, + ], +} + + +def _acl_detail(name: str, acl_id: str) -> dict: + """Return a get_web_acl-style response for the given name/id.""" + detail = dict(_ACL_DETAIL_TEMPLATE) + detail["Name"] = name + detail["Id"] = acl_id + return {"WebACL": detail} + + +# =========================================================================== +# Mock builder +# =========================================================================== + + +def _build_mock_client( + lambda_functions=None, + guardrail_summaries=None, + guardrail_detail=None, + kb_summaries=None, + kb_data_source_summaries=None, + kb_data_source_detail=None, + s3_buckets=None, + acl_summaries=None, + acl_detail_fn=None, +): + """Return a boto3.client side-effect that satisfies every AWS call made + by lambda_handler for the supplied account state. + + Non-inventory calls (shield, sts, apigw, cloudwatch, etc.) return minimal + valid responses so those non-inventory checks produce deterministic output. + """ + if lambda_functions is None: + lambda_functions = _LAMBDA_FUNCTIONS + if guardrail_summaries is None: + guardrail_summaries = [_GUARDRAIL_SUMMARY] + if guardrail_detail is None: + guardrail_detail = _GUARDRAIL_DETAIL + if kb_summaries is None: + kb_summaries = [_KB_SUMMARY] + if kb_data_source_summaries is None: + kb_data_source_summaries = [_KB_DATA_SOURCE_SUMMARY] + if kb_data_source_detail is None: + kb_data_source_detail = _KB_DATA_SOURCE_DETAIL + if s3_buckets is None: + s3_buckets = _S3_BUCKETS + if acl_summaries is None: + acl_summaries = _ACL_SUMMARIES + if acl_detail_fn is None: + acl_detail_fn = lambda name, acl_id: _acl_detail(name, acl_id) # noqa: E731 + + def side_effect(service, **kwargs): # noqa: C901 (complexity OK for test helper) + c = MagicMock() + + # ------------------------------------------------------------------ # + # wafv2 + # ------------------------------------------------------------------ # + if service == "wafv2": + c.list_web_acls.return_value = {"WebACLs": acl_summaries} + + def get_web_acl(Name, Scope, Id, **kw): + return acl_detail_fn(Name, Id) + + c.get_web_acl.side_effect = get_web_acl + return c + + # ------------------------------------------------------------------ # + # shield + # ------------------------------------------------------------------ # + if service == "shield": + c.describe_subscription.return_value = {} # shield advanced active + return c + + # ------------------------------------------------------------------ # + # lambda + # ------------------------------------------------------------------ # + if service == "lambda": + c.list_functions.return_value = { + "Functions": lambda_functions + # No NextMarker → single page + } + + def get_function_concurrency(FunctionName, **kw): + # agent function has concurrency set + if "agent" in FunctionName.lower(): + return {"ReservedConcurrentExecutions": 5} + return {} + + c.get_function_concurrency.side_effect = get_function_concurrency + return c + + # ------------------------------------------------------------------ # + # bedrock (guardrails) + # ------------------------------------------------------------------ # + if service == "bedrock": + c.list_guardrails.return_value = { + "guardrails": guardrail_summaries + # No nextToken → single page + } + c.get_guardrail.return_value = guardrail_detail + + # Non-inventory calls for other bedrock checks + c.list_foundation_models.return_value = { + "modelSummaries": [ + { + "modelId": "anthropic.claude-3-sonnet-20240229-v1:0", + "modelLifecycle": {"status": "ACTIVE"}, + } + ] + } + c.list_custom_models.return_value = {"modelSummaries": []} + c.list_model_cards.return_value = {"ModelCardSummaries": []} + c.list_evaluation_jobs.return_value = {"jobSummaries": []} + c.get_macie_session.side_effect = Exception("macie not enabled") + c.list_automated_reasoning_policies.return_value = { + "automatedReasoningPolicySummaries": [] + } + return c + + # ------------------------------------------------------------------ # + # bedrock-agent (knowledge bases + data sources) + # ------------------------------------------------------------------ # + if service == "bedrock-agent": + # The collector calls list_knowledge_bases directly via _paginate + # (not via get_paginator). We set up both the direct-call return + # value AND the paginator so any remaining checks that still use + # get_paginator also work. + + kb_paginator = MagicMock() + kb_paginator.paginate.return_value = [ + {"knowledgeBaseSummaries": kb_summaries} + ] + + def get_paginator(op): + if op == "list_knowledge_bases": + return kb_paginator + pag = MagicMock() + pag.paginate.return_value = [{}] + return pag + + c.get_paginator.side_effect = get_paginator + + # Direct method call used by _paginate inside collect_resource_inventory + c.list_knowledge_bases.return_value = { + "knowledgeBaseSummaries": kb_summaries + # No nextToken → single page + } + + # list_data_sources called via _paginate (which uses the direct method) + c.list_data_sources.return_value = { + "dataSourceSummaries": kb_data_source_summaries + } + + # get_data_source for FS-33 / FS-65 + c.get_data_source.return_value = kb_data_source_detail + + # list_agents (for FS-07) + c.list_agents.return_value = {"agentSummaries": []} + return c + + # ------------------------------------------------------------------ # + # s3 + # ------------------------------------------------------------------ # + if service == "s3": + c.list_buckets.return_value = { + "Buckets": s3_buckets + # No ContinuationToken → single page + } + + def get_bucket_versioning(Bucket, **kw): + # All buckets have versioning enabled → FS-21 PASS, FS-33 PASS + return {"Status": "Enabled"} + + c.get_bucket_versioning.side_effect = get_bucket_versioning + + def get_bucket_tagging(Bucket, **kw): + # All buckets have classification tag → FS-46 PASS + return { + "TagSet": [{"Key": "data-classification", "Value": "Confidential"}] + } + + c.get_bucket_tagging.side_effect = get_bucket_tagging + + # Bucket notification config for FS-65 + def get_bucket_notification_configuration(Bucket, **kw): + return { + "EventBridgeConfiguration": {} # event notifications present + } + + c.get_bucket_notification_configuration.side_effect = ( + get_bucket_notification_configuration + ) + return c + + # ------------------------------------------------------------------ # + # Non-inventory services — return minimal responses so checks complete + # ------------------------------------------------------------------ # + + if service == "apigateway": + c.get_usage_plans.return_value = { + "items": [ + { + "name": "default", + "throttle": {"rateLimit": 500, "burstLimit": 200}, + } + ] + } + c.get_rest_apis.return_value = {"items": []} + return c + + if service == "ce": # cost explorer + c.get_anomaly_monitors.return_value = { + "AnomalyMonitors": [ + { + "MonitorType": "DIMENSIONAL", + "MonitorDimension": "SERVICE", + "MonitorSpecification": {}, + } + ] + } + return c + + if service == "cloudwatch": + pag = MagicMock() + pag.paginate.return_value = [ + { + "MetricAlarms": [ + { + "AlarmName": "bedrock-throttle-alarm", + "Namespace": "AWS/Bedrock", + "MetricName": "InvocationThrottles", + } + ] + } + ] + c.get_paginator.return_value = pag + return c + + if service == "budgets": + pag = MagicMock() + pag.paginate.return_value = [ + { + "Budgets": [ + { + "BudgetName": "bedrock-spend", + "CostFilters": {"Service": ["Amazon Bedrock"]}, + } + ] + } + ] + c.get_paginator.return_value = pag + return c + + if service == "sts": + c.get_caller_identity.return_value = {"Account": "123456789012"} + return c + + if service == "service-quotas": + applied_pag = MagicMock() + applied_pag.paginate.return_value = [ + { + "Quotas": [ + { + "QuotaName": "On-demand InvokeModel tokens per minute for anthropic.claude", + "QuotaCode": "L-TPMTEST", + "Value": 200000, + } + ] + } + ] + defaults_pag = MagicMock() + defaults_pag.paginate.return_value = [ + {"Quotas": [{"QuotaCode": "L-TPMTEST", "Value": 100000}]} + ] + + def get_paginator(op): + if op == "list_service_quotas": + return applied_pag + if op == "list_aws_default_service_quotas": + return defaults_pag + p = MagicMock() + p.paginate.return_value = [{}] + return p + + c.get_paginator.side_effect = get_paginator + return c + + if service == "stepfunctions": + c.list_state_machines.return_value = {"stateMachines": []} + return c + + if service == "iam": + c.list_policies.return_value = {"Policies": []} + pag = MagicMock() + pag.paginate.return_value = [{}] + c.get_paginator.return_value = pag + return c + + if service == "config": + c.describe_config_rules.return_value = {"ConfigRules": []} + return c + + if service == "ecr": + c.describe_repositories.return_value = {"repositories": []} + return c + + if service == "sagemaker": + c.list_feature_groups.return_value = {"FeatureGroupSummaries": []} + c.list_processing_jobs.return_value = {"ProcessingJobSummaries": []} + c.list_models.return_value = {"Models": []} + pag = MagicMock() + pag.paginate.return_value = [{}] + c.get_paginator.return_value = pag + return c + + if service == "logs": + c.list_log_groups.return_value = {"logGroups": []} + pag = MagicMock() + pag.paginate.return_value = [{"logGroups": []}] + c.get_paginator.return_value = pag + return c + + if service == "macie2": + c.get_macie_session.side_effect = Exception("macie not enabled") + return c + + if service == "opensearchserverless": + c.list_security_policies.return_value = {"securityPolicySummaries": []} + return c + + if service == "bedrock-agent-runtime": + return c + + if service == "bedrock-runtime": + return c + + if service == "events": + c.list_rules.return_value = {"Rules": []} + return c + + if service == "scheduler": + c.list_schedules.return_value = {"Schedules": []} + return c + + if service == "agentcore": + c.list_agent_runtimes.return_value = {"agentRuntimes": []} + return c + + if service == "organizations": + c.list_policies.return_value = {"Policies": []} + return c + + # Catch-all: return a generic mock for any other service + pag = MagicMock() + pag.paginate.return_value = [{}] + c.get_paginator.return_value = pag + return c + + return side_effect + + +# =========================================================================== +# Shared pytest fixture +# =========================================================================== + + +@pytest.fixture +def account_state_mock(): + """Full single-page account state mock covering all inventory-consuming checks.""" + return _build_mock_client() + + +@pytest.fixture +def lambda_event(): + return {"Execution": {"Name": "equivalence-test-001"}} + + +# =========================================================================== +# Helper: run lambda_handler and extract the frozen tuple sequence +# =========================================================================== + + +def _enum_val(v) -> str: + """Return the .value of an enum, or the value unchanged if it's already a str.""" + return v.value if hasattr(v, "value") else str(v) + + +def _run_handler_and_extract_tuples(mock_client_side_effect, event): + """Run lambda_handler with the given mock and return the ordered list of + (check_id, finding_name, status, severity, compliance_frameworks) tuples. + + Enum fields (Status, Severity) are coerced to their .value strings so the + frozen BASELINE literal uses plain str values, making it readable and + portable across Python versions. + """ + with ( + patch("app.boto3.client") as mock_client, + patch("app.get_permissions_cache") as mock_cache, + patch("app.write_to_s3") as mock_s3, + ): + mock_client.side_effect = mock_client_side_effect + mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} + mock_s3.return_value = "https://test-bucket.s3.amazonaws.com/finserv_security_report_equivalence-test-001.csv" + + result = app.lambda_handler(event, None) + + tuples = [] + for finding in result["body"]["findings"]: + for row in finding.get("csv_data", []): + tuples.append( + ( + row.get("Check_ID", ""), + row.get("Finding", ""), + _enum_val(row.get("Status", "")), + _enum_val(row.get("Severity", "")), + row.get("Compliance_Frameworks", ""), + ) + ) + return tuples + + +# =========================================================================== +# FROZEN BASELINE +# =========================================================================== +# Generated by running lambda_handler against the account-state fixture above +# on the PRE-REFACTOR code. After the refactor Task 13 asserts this literal +# is reproduced exactly (REQ-5.3). +# +# NOTE: This literal is intentionally left empty here and will be populated +# once by the _test_generate_baseline helper below (run once, paste output, +# then use BASELINE in the main assertion test). +# =========================================================================== + + +# fmt: off +BASELINE: list[tuple[str, str, str, str, str]] = [ + ('FS-01', 'AWS Shield Advanced Enabled', 'Passed', 'Low', 'FFIEC CAT | DORA Art.6'), + ('FS-01', 'Regional WAF Web ACLs Present', 'Passed', 'Medium', 'FFIEC CAT | DORA Art.6'), + ('FS-02', 'API Gateway Rate Limiting Configured', 'Passed', 'Medium', 'FFIEC CAT | DORA Art.6 | PCI-DSS 12.3.2'), + ('FS-03', 'Bedrock Token Quotas Customized', 'Passed', 'Medium', 'FFIEC CAT | SR 11-7'), + ('FS-04', 'Cost Anomaly Detection Configured', 'Passed', 'Medium', 'FFIEC CAT | SR 11-7'), + ('FS-05', 'Bedrock CloudWatch Alarms Present', 'Passed', 'Medium', 'FFIEC CAT | DORA Art.6'), + ('FS-06', 'AI/ML Service Budgets Configured', 'Passed', 'Medium', 'FFIEC CAT | SR 11-7'), + ('FS-07', 'Agent Action Boundary Check', 'N/A', 'Informational', 'SR 11-7 | FFIEC CAT'), + ('FS-08', 'AgentCore Policy Engine Configured', 'Passed', 'High', 'SR 11-7 | MAS TRM 9.1'), + ('FS-09', 'Agent Lambda Concurrency Limits Present', 'Passed', 'Medium', 'FFIEC CAT | SR 11-7'), + ('FS-10', 'Human-in-the-Loop Check \u2014 No Agent Workflows Found', 'N/A', 'Informational', 'SR 11-7 | FFIEC CAT | MAS TRM 9.2'), + ('FS-11', 'No Agent Rate Alarms Found', 'Failed', 'Medium', 'FFIEC CAT | DORA Art.6'), + ('FS-12', 'No Bedrock-Scoped SCPs Found', 'Failed', 'High', 'SR 11-7 | FFIEC CAT | ISO 27001 A.15.2'), + ('FS-13', 'Model Provenance Tags Present', 'Passed', 'Medium', 'SR 11-7 | ISO 27001 A.12.5 | FFIEC CAT'), + ('FS-14', 'No Model Governance Config Rules Found', 'Failed', 'Medium', 'SR 11-7 | FFIEC CAT | ISO 27001 A.15.1'), + ('FS-15', 'No Bedrock Evaluation Jobs Found', 'Failed', 'Medium', 'SR 11-7 | FFIEC CAT | MAS TRM 9.3'), + ('FS-16', 'No ECR Repositories Found', 'N/A', 'Informational', 'ISO 27001 A.12.6 | FFIEC CAT | DORA Art.6'), + ('FS-20', 'No SageMaker Feature Groups Found', 'N/A', 'Informational', 'SR 11-7 | FFIEC CAT'), + ('FS-21', 'Training Data Buckets Have Versioning', 'Passed', 'High', 'SR 11-7 | ISO 27001 A.12.3 | FFIEC CAT'), + ('FS-22', 'Knowledge Base IAM Permissions Look Appropriate', 'Passed', 'High', 'NYDFS 500 | FFIEC CAT | PCI-DSS 12.3.2'), + ('FS-24', 'ADVISORY: Knowledge Base Metadata Filtering \u2014 Manual Review Required', 'N/A', 'Informational', 'NYDFS 500 | FFIEC CAT | PCI-DSS 12.3.2'), + ('FS-25', 'No OpenSearch Serverless Encryption Policies', 'N/A', 'Informational', 'NYDFS 500 | PCI-DSS 3.5 | FFIEC CAT'), + ('FS-26', 'No OpenSearch Serverless Network Policies', 'Failed', 'High', 'NYDFS 500 | FFIEC CAT | PCI-DSS 1.3'), + ('FS-27', 'Contextual Grounding Enabled on Guardrails', 'Passed', 'High', 'SR 11-7 | FFIEC CAT | MAS TRM 9.2'), + ('FS-27', 'No Automated Reasoning Policies Found', 'Failed', 'Medium', 'SR 11-7 | FFIEC CAT | MAS TRM 9.2'), + ('FS-28', 'Denied Topics Configured on CLASSIC Tier', 'Passed', 'High', 'SR 11-7 | FFIEC CAT | NYDFS 500 | MAS TRM 9.2'), + ('FS-29', 'ADVISORY: Compliance Disclaimer \u2014 Manual Review Required', 'N/A', 'Informational', 'SR 11-7 | FFIEC CAT | NYDFS 500 | MAS TRM 9.2'), + ('FS-30', 'ADVISORY: Compliance Dataset Coverage \u2014 Manual Review Required', 'N/A', 'Informational', 'SR 11-7 | FFIEC CAT | NYDFS 500'), + ('FS-31', 'Knowledge Base Data Sources Recently Synced', 'Passed', 'Medium', 'SR 11-7 | FFIEC CAT'), + ('FS-32', 'ADVISORY: Source Attribution \u2014 Manual Review Required', 'N/A', 'Informational', 'SR 11-7 | FFIEC CAT | MAS TRM 9.2'), + ('FS-33', 'KB Data Source Buckets Have Versioning', 'Passed', 'Medium', 'SR 11-7 | FFIEC CAT | ISO 27001 A.12'), + ('FS-34', 'Foundation Models Are Current', 'Passed', 'Medium', 'SR 11-7 | FFIEC CAT'), + ('FS-35', 'ADVISORY: Harmful-Content Test Coverage \u2014 Manual Review Required', 'N/A', 'Informational', 'SR 11-7 | FFIEC CAT'), + ('FS-36', 'Guardrail Content Filters on CLASSIC Tier', 'Passed', 'High', 'SR 11-7 | FFIEC CAT'), + ('FS-37', 'ADVISORY: User Feedback Mechanism \u2014 Manual Review Required', 'N/A', 'Informational', 'SR 11-7 | FFIEC CAT | MAS TRM 9.2'), + ('FS-38', 'Guardrail Word Filters Configured', 'Passed', 'Medium', 'SR 11-7 | FFIEC CAT'), + ('FS-39', 'No SageMaker Clarify Bias Monitoring', 'Failed', 'High', 'SR 11-7 | FFIEC CAT | ECOA/Fair Housing'), + ('FS-40', 'ADVISORY: Bias Dataset Coverage \u2014 Manual Review Required', 'N/A', 'Informational', 'SR 11-7 | FFIEC CAT | ECOA/Fair Housing'), + ('FS-41', 'No SageMaker Clarify Explainability Monitoring', 'Failed', 'High', 'SR 11-7 | FFIEC CAT'), + ('FS-42', 'No SageMaker Model Cards Found', 'Failed', 'Medium', 'SR 11-7 | FFIEC CAT'), + ('FS-43', 'CloudWatch Logs Data Protection Policies Present', 'Passed', 'High', 'NYDFS 500 | FFIEC CAT | PCI-DSS'), + ('FS-44', 'COULD NOT ASSESS: Amazon Macie PII Scanning Check', 'N/A', 'Low', 'NYDFS 500 | FFIEC CAT | PCI-DSS'), + ('FS-45', 'Guardrail PII Filters Configured', 'Passed', 'High', 'NYDFS 500 | FFIEC CAT | PCI-DSS'), + ('FS-46', 'AI/ML Buckets Have Classification Tags', 'Passed', 'Medium', 'NYDFS 500 | FFIEC CAT | ISO 27001 A.12 | PCI-DSS'), + ('FS-47', 'Guardrail Grounding Thresholds Appropriate', 'Passed', 'High', 'SR 11-7 | FFIEC CAT'), + ('FS-48', 'No Active Knowledge Bases for RAG', 'Failed', 'Medium', 'SR 11-7 | FFIEC CAT'), + ('FS-49', 'ADVISORY: Hallucination Disclaimer \u2014 Manual Review Required', 'N/A', 'Informational', 'SR 11-7 | FFIEC CAT | MAS TRM 9.2'), + ('FS-50', 'Relevance Grounding Filters Present', 'Passed', 'Medium', 'SR 11-7 | FFIEC CAT'), + ('FS-51', 'No Guardrails With Prompt Attack Filters', 'Failed', 'High', 'NYDFS 500 | FFIEC CAT | OWASP LLM Top 10'), + ('FS-52', 'Bedrock Lambda Functions on Current Runtimes', 'Passed', 'Medium', 'NYDFS 500 | FFIEC CAT | ISO 27001 A.12 | OWASP LLM Top 10'), + ('FS-53', 'WAF Injection Protection Rules Present', 'Passed', 'High', 'NYDFS 500 | PCI-DSS | FFIEC CAT | OWASP LLM Top 10'), + ('FS-54', 'ADVISORY: Penetration Testing \u2014 Manual Review Required', 'N/A', 'Informational', 'NYDFS 500 | FFIEC CAT | OWASP LLM Top 10'), + ('FS-55', 'Output Validation Functions Present', 'Passed', 'Medium', 'FFIEC CAT | OWASP LLM Top 10'), + ('FS-56', 'XSS Prevention Common Rule Set Present', 'Passed', 'Medium', 'NYDFS 500 | PCI-DSS | OWASP LLM Top 10'), + ('FS-57', 'ADVISORY: Output Encoding \u2014 Manual Review Required', 'N/A', 'Informational', 'NYDFS 500.06 | FFIEC CAT | OWASP LLM Top 10'), + ('FS-58', 'ADVISORY: Output Schema Validation \u2014 Manual Review Required', 'N/A', 'Informational', 'FFIEC CAT | OWASP LLM Top 10'), + ('FS-59', 'Topic Restrictions Configured on CLASSIC Tier', 'Passed', 'Medium', 'SR 11-7 | FFIEC CAT'), + ('FS-60', 'ADVISORY: Contextual Grounding for Off-Topic Prevention', 'N/A', 'Informational', 'SR 11-7 | FFIEC CAT'), + ('FS-61', 'No Automated KB Sync Schedules Detected', 'Failed', 'Medium', 'SR 11-7 | FFIEC CAT'), + ('FS-62', 'ADVISORY: Data Currency Disclaimer \u2014 Manual Review Required', 'N/A', 'Informational', 'SR 11-7 | FFIEC CAT | MAS TRM 9.2'), + ('FS-63', 'Foundation Model Lifecycle Management', 'Passed', 'Medium', 'SR 11-7 | FFIEC CAT | ISO 27001 A.12'), + ('FS-65', 'KB Data Source Buckets Missing S3 Event Notifications', 'Failed', 'Medium', 'FFIEC CAT | DORA Art.6 | ISO 27001 A.12'), + ('FS-66', 'AgentCore End-User Identity Propagation Configured', 'Passed', 'High', 'NYDFS 500 | SR 11-7 | MAS TRM 9 | PCI-DSS'), + ('FS-67', 'Agent Action-Group Lambdas May Lack Transaction Thresholds', 'Failed', 'High', 'SR 11-7 | FFIEC CAT | MAS TRM 9 | PCI-DSS'), + ('FS-68', 'API Gateway Request Body Size Limits Configured', 'Passed', 'Medium', 'DORA Art.6 | FFIEC CAT | PCI-DSS | OWASP LLM Top 10'), + ('FS-69', 'Prompt Input Validation Functions Present', 'Passed', 'Medium', 'NYDFS 500 | FFIEC CAT | OWASP LLM Top 10'), +] +# fmt: on + + +# =========================================================================== +# REQ-5.4: Documented permitted deltas for multi-page account states +# =========================================================================== +# +# The BASELINE above was captured against a SINGLE-PAGE account state +# (≤100 WAFv2 REGIONAL Web ACLs, ≤1 page of S3 buckets). For that state +# the post-refactor handler output MUST be byte-for-byte identical to BASELINE +# (REQ-5.3 / test_handler_output_matches_frozen_baseline). +# +# For multi-page account states, REQ-5.4 permits EXACTLY TWO classes of +# finding-set changes — both attributable to pagination-correctness fixes: +# +# PERMITTED DELTA 1 — WAFv2 Web ACLs (REQ-2.2): +# Pre-refactor: wafv2.list_web_acls(Scope="REGIONAL") was called UNPAGINATED, +# silently truncating at one page (≤100 ACLs). Accounts with >100 REGIONAL +# Web ACLs would have their ACL set incorrectly truncated. +# Post-refactor: _paginate uses NextMarker/NextMarker token convention, so +# ALL ACLs are returned. Any finding rows attributable to ACLs beyond page 1 +# (e.g., FS-01, FS-53, FS-56, FS-68) that were previously invisible MAY now +# appear. Existing rows for ACLs within page 1 remain identical. +# +# PERMITTED DELTA 2 — S3 buckets (REQ-2.8): +# Pre-refactor: s3.list_buckets() was called UNPAGINATED, silently truncating +# at 1,000 buckets in standard accounts; outright FAILING (error → COULD_NOT_ASSESS) +# for accounts whose bucket quota is above 10,000. +# Post-refactor: _paginate uses ContinuationToken/ContinuationToken with +# MaxBuckets=1000, so ALL buckets are enumerated and 10k-quota accounts succeed. +# Finding rows for FS-21 and FS-46 may differ for buckets beyond page 1, and +# accounts that previously produced COULD_NOT_ASSESS may now produce PASS/FAIL. +# +# ALL OTHER finding changes are defects (REQ-5.5) and must be corrected. +# =========================================================================== + + +# =========================================================================== +# Tests +# =========================================================================== + + +class TestGoldenEquivalenceBaseline: + """Golden-equivalence oracle for the current (pre-refactor) lambda_handler. + + Requirements: REQ-5.1, REQ-5.3, REQ-5.6 + """ + + def test_handler_produces_65_findings(self, account_state_mock, lambda_event): + """Smoke test: all 65 registry entries produce at least one finding.""" + with ( + patch("app.boto3.client") as mock_client, + patch("app.get_permissions_cache") as mock_cache, + patch("app.write_to_s3") as mock_s3, + ): + mock_client.side_effect = account_state_mock + mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} + mock_s3.return_value = "https://test-bucket.s3.amazonaws.com/report.csv" + + result = app.lambda_handler(lambda_event, None) + + assert result["statusCode"] == 200 + findings = result["body"]["findings"] + assert len(findings) == 65, f"Expected 65 registry entries; got {len(findings)}" + + def test_all_findings_have_rows(self, account_state_mock, lambda_event): + """Every registry entry emits at least one CSV row (no silent drops).""" + with ( + patch("app.boto3.client") as mock_client, + patch("app.get_permissions_cache") as mock_cache, + patch("app.write_to_s3") as mock_s3, + ): + mock_client.side_effect = account_state_mock + mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} + mock_s3.return_value = "https://test-bucket.s3.amazonaws.com/report.csv" + + result = app.lambda_handler(lambda_event, None) + + for finding in result["body"]["findings"]: + assert finding.get("csv_data"), ( + f"Finding '{finding.get('check_name')}' produced no CSV rows" + ) + + def test_fixture_covers_pass_paths(self, account_state_mock, lambda_event): + """The fixture exercises PASS paths for the inventory-consuming checks.""" + with ( + patch("app.boto3.client") as mock_client, + patch("app.get_permissions_cache") as mock_cache, + patch("app.write_to_s3") as mock_s3, + ): + mock_client.side_effect = account_state_mock + mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} + mock_s3.return_value = "https://test-bucket.s3.amazonaws.com/report.csv" + + result = app.lambda_handler(lambda_event, None) + + all_rows = [ + row for f in result["body"]["findings"] for row in f.get("csv_data", []) + ] + passed_ids = {r["Check_ID"] for r in all_rows if r.get("Status") == "Passed"} + + # These checks should produce PASS rows with our fixture + expected_pass = {"FS-01", "FS-09", "FS-21", "FS-27", "FS-28", "FS-33", "FS-45"} + for check_id in expected_pass: + assert check_id in passed_ids, ( + f"Expected PASS for {check_id} but not found in {passed_ids}" + ) + + def test_fixture_covers_na_paths(self, account_state_mock, lambda_event): + """The fixture exercises N/A paths (advisory checks emit Status='N/A').""" + with ( + patch("app.boto3.client") as mock_client, + patch("app.get_permissions_cache") as mock_cache, + patch("app.write_to_s3") as mock_s3, + ): + mock_client.side_effect = account_state_mock + mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} + mock_s3.return_value = "https://test-bucket.s3.amazonaws.com/report.csv" + + result = app.lambda_handler(lambda_event, None) + + all_rows = [ + row for f in result["body"]["findings"] for row in f.get("csv_data", []) + ] + na_ids = {r["Check_ID"] for r in all_rows if r.get("Status") == "N/A"} + + # Advisory/N/A checks should appear + expected_na = {"FS-24", "FS-29", "FS-30", "FS-32", "FS-54", "FS-57", "FS-60"} + for check_id in expected_na: + assert check_id in na_ids, ( + f"Expected N/A for {check_id} but not found in {na_ids}" + ) + + def test_fixture_covers_fail_paths(self, lambda_event): + """The fixture exercises FAIL paths for select inventory-consuming checks.""" + # Use a variant fixture where guardrail has NO PII filters → FS-45 FAIL + guardrail_without_pii = dict(_GUARDRAIL_DETAIL) + guardrail_without_pii = { + k: v + for k, v in _GUARDRAIL_DETAIL.items() + if k != "sensitiveInformationPolicy" + } + guardrail_without_pii["sensitiveInformationPolicy"] = {} + + fail_mock = _build_mock_client(guardrail_detail=guardrail_without_pii) + + with ( + patch("app.boto3.client") as mock_client, + patch("app.get_permissions_cache") as mock_cache, + patch("app.write_to_s3") as mock_s3, + ): + mock_client.side_effect = fail_mock + mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} + mock_s3.return_value = "https://test-bucket.s3.amazonaws.com/report.csv" + + result = app.lambda_handler(lambda_event, None) + + all_rows = [ + row for f in result["body"]["findings"] for row in f.get("csv_data", []) + ] + failed_ids = {r["Check_ID"] for r in all_rows if r.get("Status") == "Failed"} + assert "FS-45" in failed_ids, ( + "Expected FS-45 FAIL when guardrail has no PII filters" + ) + + def test_baseline_tuple_count_is_stable(self, account_state_mock, lambda_event): + """Capture the pre-refactor tuples and assert the count is deterministic. + + The frozen BASELINE literal is populated by test_capture_and_freeze_baseline + below. This test asserts the count (and therefore structure) is stable + across runs, which is the prerequisite for freezing it as a literal. + """ + t1 = _run_handler_and_extract_tuples(account_state_mock, lambda_event) + t2 = _run_handler_and_extract_tuples( + _build_mock_client(), + lambda_event, # fresh instance, same state + ) + assert t1 == t2, ( + "lambda_handler is non-deterministic: two identical runs produced different tuples" + ) + assert len(t1) > 0, "No tuples extracted — fixture may be broken" + + def test_handler_output_matches_frozen_baseline( + self, account_state_mock, lambda_event + ): + """Golden-equivalence oracle: the handler output must equal BASELINE exactly. + + REQ-5.1, REQ-5.3, REQ-5.6 — This test is the live oracle that will remain + green after the refactor in Tasks 2–15. It asserts that: + 1. The (check_id, finding_name, status, severity, compliance_frameworks) + tuples are identical to the pre-refactor frozen literal. + 2. Row ordering is preserved (CSV row ordering per REQ-5.6). + 3. The BASELINE covers ≤100 Web ACLs and ≤1 page of S3 buckets so it + remains directly comparable after the pagination fix (REQ-5.3). + """ + actual = _run_handler_and_extract_tuples(account_state_mock, lambda_event) + assert actual == BASELINE, ( + f"Handler output differs from frozen BASELINE.\n" + f"Expected {len(BASELINE)} tuples, got {len(actual)}.\n" + f"First diff at index " + + str( + next( + (i for i, (a, b) in enumerate(zip(actual, BASELINE)) if a != b), + min(len(actual), len(BASELINE)), + ) + ) + + "." + ) + + +class TestPermittedMultiPageDeltas: + """Validates the two permitted finding-set changes for multi-page account states. + + REQ-5.4: findings may ONLY change for resources beyond page 1 of WAFv2 Web + ACLs (>100 ACLs) or S3 buckets (>1 page / >10k quota). All other rows + must remain identical to the single-page BASELINE. + + These tests use two-page fixtures so we can verify: + 1. Rows attributable to page-1 resources are identical to BASELINE. + 2. Rows attributable to page-2+ resources appear (new, post-refactor only). + """ + + def test_wafv2_multi_page_page1_rows_unchanged(self, lambda_event): + """REQ-5.4 WAFv2 delta: finding changes attributable ONLY to page-2+ ACLs are permitted. + + Scenario: page 1 = well-configured ACLs (same as BASELINE), page 2 = one ACL + MISSING injection-protection rules. Pre-refactor: that ACL was silently + truncated so checks passed (BASELINE). Post-refactor: the ACL is seen and + checks correctly fail. + + This is the ONLY class of WAFv2 finding change REQ-5.4 permits. + """ + # Extra ACL on page 2 — missing injection/XSS rules → FS-53, FS-56 should now FAIL + extra_acl = { + "Name": "BadACL", + "Id": "acl-id-999", + "ARN": "arn:aws:wafv2:::acl-id-999", + } + bad_acl_detail = { + "WebACL": { + "Name": "BadACL", + "Id": "acl-id-999", + "Rules": [], # No managed rule groups — missing injection protection + } + } + + def mock_with_two_page_wafv2(service, **kwargs): + c = _build_mock_client()(service, **kwargs) + if service == "wafv2": + c_waf = MagicMock() + call_count = {"n": 0} + + def list_web_acls_two_pages(**kw): + call_count["n"] += 1 + if call_count["n"] == 1: + return {"WebACLs": _ACL_SUMMARIES, "NextMarker": "page2token"} + return {"WebACLs": [extra_acl]} + + c_waf.list_web_acls.side_effect = list_web_acls_two_pages + + def get_web_acl(Name, Scope, Id, **kw): + if Id == "acl-id-999": + return bad_acl_detail + return _acl_detail(Name, Id) + + c_waf.get_web_acl.side_effect = get_web_acl + return c_waf + return c + + with ( + patch("app.boto3.client") as mock_client, + patch("app.get_permissions_cache") as mock_cache, + patch("app.write_to_s3"), + ): + mock_client.side_effect = mock_with_two_page_wafv2 + mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} + result = app.lambda_handler(lambda_event, None) + + actual = [] + for finding in result["body"]["findings"]: + for row in finding.get("csv_data", []): + actual.append( + ( + row.get("Check_ID", ""), + row.get("Finding", ""), + _enum_val(row.get("Status", "")), + _enum_val(row.get("Severity", "")), + row.get("Compliance_Frameworks", ""), + ) + ) + + actual_by_id = {t[0]: t for t in actual} + + # REQ-5.4: WAFv2 checks (FS-53, FS-56) MUST now fail because the page-2 ACL + # is missing injection/XSS rules. This is the PERMITTED delta. + assert actual_by_id.get("FS-53", ("", "", "", "", ""))[2] == "Failed", ( + "FS-53 must FAIL when a page-2 ACL has no injection-protection rules" + ) + assert actual_by_id.get("FS-56", ("", "", "", "", ""))[2] == "Failed", ( + "FS-56 must FAIL when a page-2 ACL has no XSS rules" + ) + + # REQ-5.4: All non-WAFv2 check rows are UNCHANGED (no collateral damage) + wafv2_check_ids = {"FS-01", "FS-53", "FS-56", "FS-68"} + for row in BASELINE: + check_id = row[0] + if check_id in wafv2_check_ids: + continue # WAFv2 rows are the permitted delta — skip + assert row in actual, ( + f"REQ-5.4 violation: non-WAFv2 BASELINE row {row!r} missing from output" + ) + + def test_s3_multi_page_page1_rows_unchanged(self, lambda_event): + """REQ-5.4 S3 delta: page-1 bucket rows must be identical to BASELINE. + + Uses a two-page S3 fixture (page 1 = the fixture buckets, page 2 = one + additional bucket). Asserts that the page-1 bucket findings are identical + to the single-page BASELINE. + """ + extra_bucket = {"Name": "extra-ml-bucket-page2"} + + def mock_with_two_page_s3(service, **kwargs): + c = _build_mock_client()(service, **kwargs) + if service == "s3": + c_s3 = MagicMock() + call_count = {"n": 0} + + def list_buckets_two_pages(**kw): + call_count["n"] += 1 + if call_count["n"] == 1: + return { + "Buckets": _S3_BUCKETS, + "ContinuationToken": "page2token", + } + return {"Buckets": [extra_bucket]} + + c_s3.list_buckets.side_effect = list_buckets_two_pages + + def get_bucket_versioning(Bucket, **kw): + return {"Status": "Enabled"} + + c_s3.get_bucket_versioning.side_effect = get_bucket_versioning + + def get_bucket_tagging(Bucket, **kw): + return { + "TagSet": [ + {"Key": "data-classification", "Value": "Confidential"} + ] + } + + c_s3.get_bucket_tagging.side_effect = get_bucket_tagging + + def get_bucket_notification_configuration(Bucket, **kw): + return {"EventBridgeConfiguration": {}} + + c_s3.get_bucket_notification_configuration.side_effect = ( + get_bucket_notification_configuration + ) + return c_s3 + return c + + with ( + patch("app.boto3.client") as mock_client, + patch("app.get_permissions_cache") as mock_cache, + patch("app.write_to_s3"), + ): + mock_client.side_effect = mock_with_two_page_s3 + mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} + result = app.lambda_handler(lambda_event, None) + + actual = [] + for finding in result["body"]["findings"]: + for row in finding.get("csv_data", []): + actual.append( + ( + row.get("Check_ID", ""), + row.get("Finding", ""), + _enum_val(row.get("Status", "")), + _enum_val(row.get("Severity", "")), + row.get("Compliance_Frameworks", ""), + ) + ) + + # All BASELINE rows must still appear — delta is ADDITIVE only for extra buckets + for row in BASELINE: + assert row in actual, ( + f"REQ-5.4 violation: BASELINE row {row!r} missing from multi-page S3 output" + ) + + # Must have at least as many rows as BASELINE (extra bucket may add rows) + assert len(actual) >= len(BASELINE) + + +class TestCaptureAndFreezeBaseline: + """Helper test that prints the frozen literal for copy/paste into BASELINE. + + Run once with: pytest tests/test_inventory_equivalence.py::TestCaptureAndFreezeBaseline -s + Then copy the printed literal into the BASELINE constant above and delete this class + (or keep it as a regeneration helper). + """ + + def test_print_baseline_literal(self, account_state_mock, lambda_event, capsys): + """Print the BASELINE literal so it can be pasted into the BASELINE constant.""" + tuples = _run_handler_and_extract_tuples(account_state_mock, lambda_event) + + with capsys.disabled(): + print("\n\n# ===== PASTE INTO BASELINE CONSTANT =====") + print("BASELINE: list[tuple[str, str, str, str, str]] = [") + for t in tuples: + # repr each element so special chars are safe + row_repr = ", ".join(repr(x) for x in t) + print(f" ({row_repr}),") + print("]") + print("# ===== END BASELINE =====\n") + + # The test itself passes as long as tuples are produced + assert len(tuples) > 0 diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_inventory_model.py b/aiml-security-assessment/functions/security/finserv_tests/test_inventory_model.py new file mode 100644 index 0000000..2370af2 --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_tests/test_inventory_model.py @@ -0,0 +1,143 @@ +""" +Unit tests for ResourceInventory data model and accessors. + +Validates: Requirements REQ-4.1, REQ-4.2, REQ-9.2 +""" + +import sys +import os + +import pytest + +# Ensure finserv_assessments is importable +FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") +if FINSERV_DIR not in sys.path: + sys.path.insert(0, FINSERV_DIR) + +import app # noqa: E402 (import follows sys.path bootstrap above) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_minimal_inventory(**overrides): + """Build a minimal ResourceInventory with all-available fields unless + overridden.""" + defaults = dict( + lambda_functions=[], + guardrails=app.GuardrailInventory(summaries=[], detail_by_id={}), + knowledge_bases=app.KbInventory( + summaries=[], data_sources_by_kb={}, data_source_detail={} + ), + buckets=[], + web_acls=app.WebAclInventory(summaries=[], detail_by_id={}), + ) + defaults.update(overrides) + return app.ResourceInventory(**defaults) + + +# --------------------------------------------------------------------------- +# Tests for `require` +# --------------------------------------------------------------------------- + + +class TestRequire: + def test_raises_runtime_error_when_inventory_is_none(self): + """require(None, ...) always raises RuntimeError (test-only default path).""" + with pytest.raises(RuntimeError, match="resource inventory not provided"): + app.require(None, "lambda_functions") + + def test_reraises_stored_error_when_field_is_unavailable(self): + """require re-raises the exact exception stored in _Unavailable.""" + original_err = PermissionError("AccessDenied: list_functions denied") + inv = _make_minimal_inventory(lambda_functions=app._Unavailable(original_err)) + with pytest.raises(PermissionError) as exc_info: + app.require(inv, "lambda_functions") + assert exc_info.value is original_err + + def test_reraises_stored_error_preserves_type(self): + """The re-raised error has the same type as the stored one.""" + err = ValueError("boom") + inv = _make_minimal_inventory(buckets=app._Unavailable(err)) + with pytest.raises(ValueError): + app.require(inv, "buckets") + + def test_returns_value_when_field_is_available(self): + """require returns the field's value when it is not an _Unavailable.""" + functions = [{"FunctionName": "my-fn"}] + inv = _make_minimal_inventory(lambda_functions=functions) + result = app.require(inv, "lambda_functions") + assert result is functions + + def test_returns_nested_inventory_when_available(self): + """require works for complex nested types like GuardrailInventory.""" + guardrail_inv = app.GuardrailInventory( + summaries=[{"id": "g1"}], detail_by_id={"g1": {"policy": {}}} + ) + inv = _make_minimal_inventory(guardrails=guardrail_inv) + result = app.require(inv, "guardrails") + assert result is guardrail_inv + + def test_raises_on_any_unavailable_field(self): + """require raises for each inventory field name when it holds _Unavailable.""" + err = RuntimeError("some error") + for field in ( + "lambda_functions", + "guardrails", + "knowledge_bases", + "buckets", + "web_acls", + ): + inv = _make_minimal_inventory(**{field: app._Unavailable(err)}) + with pytest.raises(RuntimeError): + app.require(inv, field) + + +# --------------------------------------------------------------------------- +# Tests for `inv_available` +# --------------------------------------------------------------------------- + + +class TestInvAvailable: + def test_returns_true_for_normal_value(self): + """inv_available returns True for a plain list.""" + assert app.inv_available([]) is True + + def test_returns_true_for_non_empty_list(self): + """inv_available returns True for a populated list.""" + assert app.inv_available([{"FunctionName": "fn"}]) is True + + def test_returns_true_for_nested_dataclass(self): + """inv_available returns True for a GuardrailInventory.""" + gi = app.GuardrailInventory(summaries=[], detail_by_id={}) + assert app.inv_available(gi) is True + + def test_returns_false_for_unavailable(self): + """inv_available returns False for an _Unavailable sentinel.""" + assert app.inv_available(app._Unavailable(Exception("x"))) is False + + def test_returns_false_regardless_of_stored_error(self): + """inv_available is False for _Unavailable regardless of the error type.""" + for err in (ValueError("v"), RuntimeError("r"), Exception("e")): + assert app.inv_available(app._Unavailable(err)) is False + + +# --------------------------------------------------------------------------- +# Tests for _Unavailable sentinel itself +# --------------------------------------------------------------------------- + + +class TestUnavailableSentinel: + def test_stores_error(self): + """_Unavailable stores the given exception on .error.""" + err = IOError("network error") + sentinel = app._Unavailable(err) + assert sentinel.error is err + + def test_is_not_list_or_dict(self): + """_Unavailable is distinguishable from the normal inventory types.""" + sentinel = app._Unavailable(Exception()) + assert not isinstance(sentinel, (list, dict)) + assert not isinstance(sentinel, app.GuardrailInventory) diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_lambda_handler.py b/aiml-security-assessment/functions/security/finserv_tests/test_lambda_handler.py new file mode 100644 index 0000000..af6be0d --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_tests/test_lambda_handler.py @@ -0,0 +1,436 @@ +""" +Integration tests for the lambda_handler in finserv_assessments/app.py + +These tests verify: + - lambda_handler runs end-to-end with all checks mocked + - Response structure (statusCode, body, findings, report_url) + - All 59 standalone check functions are called (5 merged upstream) + - CSV is written to S3 + - Error handling when AIML_ASSESSMENT_BUCKET_NAME is missing + - Error handling when S3 write fails +""" + +import json +import os +import sys +from unittest.mock import MagicMock, patch + +import pytest + +FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") +if FINSERV_DIR not in sys.path: + sys.path.insert(0, FINSERV_DIR) + +import app # noqa: E402 (import must follow sys.path setup above) + + +# ========================================================================= +# Full handler smoke test — all boto3 calls mocked +# ========================================================================= + + +class TestLambdaHandler: + """End-to-end handler tests with fully mocked AWS clients.""" + + @patch("app.write_to_s3") + @patch("app.get_permissions_cache") + @patch("app.boto3.client") + def test_handler_returns_200(self, mock_client, mock_cache, mock_s3, lambda_event): + """Smoke test: handler completes and returns 200.""" + # Return a generic mock for every boto3 client + generic = MagicMock() + # Make paginators return empty pages + paginator = MagicMock() + paginator.paginate.return_value = [{}] + generic.get_paginator.return_value = paginator + # Make list/describe calls return empty collections + generic.list_web_acls.return_value = {"WebACLs": []} + generic.describe_subscription.side_effect = Exception("no shield") + generic.get_usage_plans.return_value = {"items": []} + generic.list_service_quotas.return_value = {"Quotas": []} + generic.get_anomaly_monitors.return_value = {"AnomalyMonitors": []} + generic.describe_budgets.return_value = {"Budgets": []} + generic.get_caller_identity.return_value = {"Account": "123456789012"} + generic.list_agents.return_value = {"agentSummaries": []} + generic.list_agent_runtimes.return_value = {"agentRuntimes": []} + generic.list_functions.return_value = {"Functions": []} + generic.list_state_machines.return_value = {"stateMachines": []} + generic.list_policies.return_value = {"Policies": []} + generic.list_custom_models.return_value = {"modelSummaries": []} + generic.list_models.return_value = {"Models": []} + generic.describe_config_rules.return_value = {"ConfigRules": []} + generic.list_evaluation_jobs.return_value = {"jobSummaries": []} + generic.describe_repositories.return_value = {"repositories": []} + generic.list_feature_groups.return_value = {"FeatureGroupSummaries": []} + generic.list_buckets.return_value = {"Buckets": []} + generic.list_knowledge_bases.return_value = {"knowledgeBaseSummaries": []} + generic.list_guardrails.return_value = {"guardrails": []} + generic.list_log_groups.return_value = {"logGroups": []} + generic.get_macie_session.side_effect = Exception("not enabled") + generic.list_foundation_models.return_value = {"modelSummaries": []} + generic.list_model_cards.return_value = {"ModelCardSummaries": []} + generic.list_rules.return_value = {"Rules": []} + generic.list_schedules.return_value = {"Schedules": []} + generic.get_rest_apis.return_value = {"items": []} + generic.list_processing_jobs.return_value = {"ProcessingJobSummaries": []} + generic.list_automated_reasoning_policies.return_value = { + "automatedReasoningPolicySummaries": [] + } + + mock_client.return_value = generic + mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} + mock_s3.return_value = "https://test-bucket.s3.amazonaws.com/finserv_security_report_unit-test-001.csv" + + result = app.lambda_handler(lambda_event, None) + + assert result["statusCode"] == 200 + assert "findings" in result["body"] + assert "report_url" in result["body"] + assert isinstance(result["body"]["findings"], list) + # The handler runs 65 registry entries (64 standalone checks + the new + # FS-27 ARC policies check that shares the FS-27 check_id). + assert len(result["body"]["findings"]) == 65 + + @patch("app.write_to_s3") + @patch("app.get_permissions_cache") + @patch("app.boto3.client") + def test_handler_findings_all_have_check_name( + self, mock_client, mock_cache, mock_s3, lambda_event + ): + """Every finding dict should have check_name and status keys.""" + generic = MagicMock() + paginator = MagicMock() + paginator.paginate.return_value = [{}] + generic.get_paginator.return_value = paginator + generic.list_web_acls.return_value = {"WebACLs": []} + generic.describe_subscription.side_effect = Exception("no shield") + generic.get_usage_plans.return_value = {"items": []} + generic.list_service_quotas.return_value = {"Quotas": []} + generic.get_anomaly_monitors.return_value = {"AnomalyMonitors": []} + generic.describe_budgets.return_value = {"Budgets": []} + generic.get_caller_identity.return_value = {"Account": "123456789012"} + generic.list_agents.return_value = {"agentSummaries": []} + generic.list_agent_runtimes.return_value = {"agentRuntimes": []} + generic.list_functions.return_value = {"Functions": []} + generic.list_state_machines.return_value = {"stateMachines": []} + generic.list_policies.return_value = {"Policies": []} + generic.list_custom_models.return_value = {"modelSummaries": []} + generic.list_models.return_value = {"Models": []} + generic.describe_config_rules.return_value = {"ConfigRules": []} + generic.list_evaluation_jobs.return_value = {"jobSummaries": []} + generic.describe_repositories.return_value = {"repositories": []} + generic.list_feature_groups.return_value = {"FeatureGroupSummaries": []} + generic.list_buckets.return_value = {"Buckets": []} + generic.list_knowledge_bases.return_value = {"knowledgeBaseSummaries": []} + generic.list_guardrails.return_value = {"guardrails": []} + generic.list_log_groups.return_value = {"logGroups": []} + generic.get_macie_session.side_effect = Exception("not enabled") + generic.list_foundation_models.return_value = {"modelSummaries": []} + generic.list_model_cards.return_value = {"ModelCardSummaries": []} + generic.list_rules.return_value = {"Rules": []} + generic.list_schedules.return_value = {"Schedules": []} + generic.get_rest_apis.return_value = {"items": []} + generic.list_processing_jobs.return_value = {"ProcessingJobSummaries": []} + generic.list_automated_reasoning_policies.return_value = { + "automatedReasoningPolicySummaries": [] + } + + mock_client.return_value = generic + mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} + mock_s3.return_value = "https://test-bucket.s3.amazonaws.com/report.csv" + + result = app.lambda_handler(lambda_event, None) + + for finding in result["body"]["findings"]: + assert "check_name" in finding, f"Missing check_name in {finding}" + assert "status" in finding, f"Missing status in {finding}" + assert finding["status"] in ("PASS", "WARN", "ERROR"), ( + f"Unexpected status '{finding['status']}' in {finding['check_name']}" + ) + assert "csv_data" in finding + + def test_handler_raises_without_bucket_env(self, lambda_event, monkeypatch): + """Handler should raise ValueError when AIML_ASSESSMENT_BUCKET_NAME is unset.""" + monkeypatch.delenv("AIML_ASSESSMENT_BUCKET_NAME", raising=False) + + # We need to mock all boto3 calls so the checks themselves don't fail + with ( + patch("app.boto3.client") as mock_client, + patch("app.get_permissions_cache") as mock_cache, + ): + generic = MagicMock() + paginator = MagicMock() + paginator.paginate.return_value = [{}] + generic.get_paginator.return_value = paginator + # Set up enough mocks for checks to complete + for attr in [ + "list_web_acls", + "get_usage_plans", + "list_service_quotas", + "get_anomaly_monitors", + "describe_budgets", + "list_agents", + "list_agent_runtimes", + "list_functions", + "list_state_machines", + "list_policies", + "list_custom_models", + "list_models", + "describe_config_rules", + "list_evaluation_jobs", + "describe_repositories", + "list_feature_groups", + "list_buckets", + "list_knowledge_bases", + "list_guardrails", + "list_log_groups", + "list_foundation_models", + "list_model_cards", + "list_rules", + "get_rest_apis", + "list_processing_jobs", + ]: + getattr(generic, attr).return_value = ( + {"WebACLs": []} + if "acl" in attr.lower() + else {"items": []} + if "items" in attr.lower() or "rest_api" in attr.lower() + else {next(iter({})): []} + if False # fallback + else {} + ) + generic.describe_subscription.side_effect = Exception("no shield") + generic.get_caller_identity.return_value = {"Account": "123456789012"} + generic.get_macie_session.side_effect = Exception("not enabled") + + mock_client.return_value = generic + mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} + + with pytest.raises(ValueError, match="AIML_ASSESSMENT_BUCKET_NAME"): + app.lambda_handler(lambda_event, None) + + +class TestInventoryCollectedAndPassed: + """Task 5.1 — Assert lambda_handler always collects and passes a real inventory. + + Guards against the default-None footgun: even though build_finserv_checks + accepts inventory=None for backward-compat with the drift-guard, lambda_handler + must always call collect_resource_inventory() and pass its return value. + + Validates: REQ-6.5 + """ + + @patch("app.write_to_s3") + @patch("app.get_permissions_cache") + @patch("app.build_finserv_checks") + @patch("app.collect_resource_inventory") + def test_handler_calls_collect_inventory_exactly_once( + self, + mock_collect, + mock_build, + mock_cache, + mock_s3, + lambda_event, + ): + """lambda_handler calls collect_resource_inventory() exactly once per invocation.""" + fake_inventory = object() # any sentinel — not None + mock_collect.return_value = fake_inventory + mock_build.return_value = [] # no checks to run + mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} + mock_s3.return_value = "https://bucket.s3.amazonaws.com/report.csv" + + app.lambda_handler(lambda_event, None) + + mock_collect.assert_called_once_with() + + @patch("app.write_to_s3") + @patch("app.get_permissions_cache") + @patch("app.build_finserv_checks") + @patch("app.collect_resource_inventory") + def test_handler_passes_inventory_to_build_finserv_checks( + self, + mock_collect, + mock_build, + mock_cache, + mock_s3, + lambda_event, + ): + """The return value of collect_resource_inventory() is passed as inventory= + to build_finserv_checks, and it is never None.""" + fake_inventory = object() # distinct sentinel + mock_collect.return_value = fake_inventory + mock_build.return_value = [] + mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} + mock_s3.return_value = "https://bucket.s3.amazonaws.com/report.csv" + + app.lambda_handler(lambda_event, None) + + # build_finserv_checks must have been called with the real inventory + mock_build.assert_called_once() + call_args, call_kwargs = mock_build.call_args + # inventory can be passed positionally or as a keyword argument + passed_inventory = call_kwargs.get( + "inventory", call_args[1] if len(call_args) > 1 else None + ) + assert passed_inventory is fake_inventory, ( + "lambda_handler must pass the collect_resource_inventory() return value " + f"to build_finserv_checks; got {passed_inventory!r}" + ) + assert passed_inventory is not None, ( + "lambda_handler must never pass None as the inventory argument" + ) + + +class TestWriteToS3: + """Test the write_to_s3 helper.""" + + @patch("app.boto3.client") + def test_writes_csv_to_s3(self, mock_client): + s3 = MagicMock() + mock_client.return_value = s3 + + url = app.write_to_s3("exec-123", "col1,col2\nval1,val2", "my-bucket") + + s3.put_object.assert_called_once_with( + Bucket="my-bucket", + Key="finserv_security_report_exec-123.csv", + Body="col1,col2\nval1,val2", + ContentType="text/csv", + ) + assert "my-bucket" in url + assert "exec-123" in url + + @patch("app.boto3.client") + def test_s3_error_propagates(self, mock_client): + s3 = MagicMock() + s3.put_object.side_effect = RuntimeError("S3 write failed") + mock_client.return_value = s3 + + with pytest.raises(RuntimeError, match="S3 write failed"): + app.write_to_s3("exec-123", "data", "my-bucket") + + +class TestGetPermissionsCache: + """Test the get_permissions_cache helper.""" + + @patch("app.boto3.client") + def test_returns_parsed_json(self, mock_client): + s3 = MagicMock() + body = MagicMock() + body.read.return_value = json.dumps({"role_permissions": {"r1": {}}}).encode() + s3.get_object.return_value = {"Body": body} + mock_client.return_value = s3 + + result = app.get_permissions_cache("exec-123") + assert result == {"role_permissions": {"r1": {}}} + + @patch("app.boto3.client") + def test_returns_none_on_client_error(self, mock_client): + from botocore.exceptions import ClientError + + s3 = MagicMock() + s3.get_object.side_effect = ClientError( + {"Error": {"Code": "NoSuchKey", "Message": "Not found"}}, + "GetObject", + ) + mock_client.return_value = s3 + + result = app.get_permissions_cache("exec-123") + assert result is None + + @patch("app.boto3.client") + def test_returns_none_on_unexpected_error(self, mock_client): + s3 = MagicMock() + s3.get_object.side_effect = RuntimeError("unexpected") + mock_client.return_value = s3 + + result = app.get_permissions_cache("exec-123") + assert result is None + + +# ========================================================================= +# Importability smoke test — all 59 check functions are importable +# ========================================================================= + + +class TestAllCheckFunctionsImportable: + """Verify every check function referenced in lambda_handler is importable.""" + + EXPECTED_CHECK_FUNCTIONS = [ + "check_waf_shield_on_bedrock_endpoints", + "check_api_gateway_rate_limiting", + "check_bedrock_token_quotas", + "check_cost_anomaly_detection", + "check_cloudwatch_token_alarms", + "check_aws_budgets_for_aiml", + "check_bedrock_agent_action_boundaries", + "check_agentcore_policy_engine", + "check_agent_transaction_limits", + "check_human_in_the_loop_for_high_risk_actions", + "check_agent_rate_alarms", + "check_scp_model_access_restrictions", + "check_model_inventory_tagging", + "check_model_onboarding_governance", + "check_bedrock_model_evaluation_adversarial", + "check_ecr_image_scanning", + "check_feature_store_rollback_capability", + "check_training_data_s3_versioning", + "check_knowledge_base_iam_least_privilege", + "check_knowledge_base_metadata_filtering", + "check_opensearch_serverless_encryption", + "check_knowledge_base_vpc_access", + # FS-27 is now two separate functions: contextual grounding + ARC policies + "check_guardrail_contextual_grounding", + "check_automated_reasoning_policies", + "check_guardrail_denied_topics_financial", + "check_compliance_disclaimer_in_outputs", + "check_bedrock_evaluation_compliance_datasets", + "check_knowledge_base_data_source_sync", + "check_source_attribution_in_guardrails", + "check_knowledge_base_integrity_monitoring", + "check_fm_version_currency", + "check_fmeval_harmful_content", + "check_guardrail_content_filters", + "check_user_feedback_mechanism", + "check_guardrail_word_filters", + "check_sagemaker_clarify_bias", + "check_bedrock_evaluation_bias_datasets", + "check_sagemaker_clarify_explainability", + "check_ai_service_cards_documentation", + "check_cloudwatch_log_pii_masking", + "check_macie_on_training_data_buckets", + "check_guardrail_pii_filters", + "check_data_classification_tagging", + "check_guardrail_grounding_threshold", + "check_rag_knowledge_base_configured", + "check_hallucination_disclaimer_advisory", + # FS-50 renamed from check_automated_reasoning_checks_hallucination + "check_guardrail_relevance_grounding", + "check_prompt_injection_input_validation", + "check_bedrock_sdk_version_currency", + "check_waf_sql_injection_rules", + "check_penetration_testing_evidence", + "check_output_validation_lambda", + "check_xss_prevention_waf", + "check_output_encoding_advisory", + "check_output_schema_validation", + "check_guardrail_topic_allowlist", + "check_contextual_grounding_for_offtopic", + "check_knowledge_base_sync_schedule", + "check_data_currency_disclaimer_advisory", + "check_foundation_model_lifecycle_policy", + "check_kb_datasource_s3_event_notifications", + "check_agentcore_end_user_identity_propagation", + "check_agent_financial_transaction_thresholds", + "check_api_gateway_request_body_size_limits", + "check_prompt_input_validation_function", + ] + + @pytest.mark.parametrize("func_name", EXPECTED_CHECK_FUNCTIONS) + def test_function_exists(self, func_name): + assert hasattr(app, func_name), f"app.{func_name} not found" + assert callable(getattr(app, func_name)), f"app.{func_name} is not callable" + + def test_expected_count(self): + """Sanity check: 65 check functions (64 standalone + new ARC check sharing FS-27).""" + assert len(self.EXPECTED_CHECK_FUNCTIONS) == 65 diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_large_estate.py b/aiml-security-assessment/functions/security/finserv_tests/test_large_estate.py new file mode 100644 index 0000000..e92cfd2 --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_tests/test_large_estate.py @@ -0,0 +1,456 @@ +""" +Large-estate performance and memory guard — Task 15 (Wave 4). + +Validates: REQ-10.1, REQ-10.2, REQ-10.3, REQ-11.1, REQ-11.3 + +With ≥1,000 functions, ≥100 buckets, ≥50 guardrails/KBs/ACLs the handler must: + 1. Issue at most one listing call per inventory (enforced by patching collect_resource_inventory + to return a pre-built large inventory — the real collector never runs in this test). + 2. Issue ≤1 detail call per resource (get_guardrail per guardrail id, get_web_acl per ACL id). + 3. Complete well within the 900 s Lambda budget. + 4. Keep peak memory well under 1024 MB. + +The test builds the large inventory directly (bypassing the collector) and patches +app.collect_resource_inventory to return it, then runs lambda_handler end-to-end with +a generic MagicMock for every non-inventory boto3 client. +""" + +import os +import sys +import time +import tracemalloc +from datetime import datetime, timezone +from unittest.mock import MagicMock, patch + + +FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") +if FINSERV_DIR not in sys.path: + sys.path.insert(0, FINSERV_DIR) + +import app # noqa: E402 + + +# --------------------------------------------------------------------------- +# Inline inventory builder — mirrors conftest.make_resource_inventory so this +# module is self-contained while staying consistent with the shared fixture. +# --------------------------------------------------------------------------- + + +def make_resource_inventory(**overrides) -> app.ResourceInventory: + """Build a fully-available ResourceInventory with sensible empty defaults. + + Mirrors conftest.make_resource_inventory; defined here so the test module + is self-contained (conftest is only importable by pytest, not directly). + """ + defaults: dict = { + "lambda_functions": [], + "guardrails": app.GuardrailInventory(summaries=[], detail_by_id={}), + "knowledge_bases": app.KbInventory( + summaries=[], data_sources_by_kb={}, data_source_detail={} + ), + "buckets": [], + "web_acls": app.WebAclInventory(summaries=[], detail_by_id={}), + } + defaults.update(overrides) + return app.ResourceInventory(**defaults) + + +# --------------------------------------------------------------------------- +# Large-estate fixture helpers +# --------------------------------------------------------------------------- + +_N_FUNCTIONS = 1_000 +_N_BUCKETS = 100 +_N_GUARDRAILS = 50 +_N_KBS = 50 +_N_ACLS = 50 + + +def _build_large_inventory() -> app.ResourceInventory: + """Construct a fully-populated ResourceInventory at large-estate scale.""" + # Lambda functions — 1 000 entries + functions = [ + {"FunctionName": f"fn-{i}", "Runtime": "python3.12"} + for i in range(_N_FUNCTIONS) + ] + + # Guardrails — 50 summaries + 50 detail entries + guardrail_summaries = [ + {"id": f"g-{i}", "name": f"guardrail-{i}"} for i in range(_N_GUARDRAILS) + ] + detail_by_guardrail_id = { + g["id"]: { + "guardrailId": g["id"], + "name": g["name"], + "status": "READY", + "contentPolicy": { + "filters": [ + { + "type": "SEXUAL", + "inputStrength": "HIGH", + "outputStrength": "HIGH", + } + ] + }, + "topicPolicy": {"topics": []}, + "wordPolicy": {"words": [], "managedWordLists": []}, + "sensitiveInformationPolicy": {"piiEntities": [], "regexes": []}, + "contextualGroundingPolicy": { + "filters": [ + {"type": "GROUNDING", "threshold": 0.8}, + {"type": "RELEVANCE", "threshold": 0.8}, + ] + }, + } + for g in guardrail_summaries + } + + # Knowledge Bases — 50 summaries, 1 data source each, full detail + kb_summaries = [ + { + "knowledgeBaseId": f"kb-{i}", + "name": f"kb-{i}", + "status": "ACTIVE", + "updatedAt": datetime.now(timezone.utc), + } + for i in range(_N_KBS) + ] + data_sources_by_kb: dict = {} + data_source_detail: dict = {} + for kb in kb_summaries: + kb_id = kb["knowledgeBaseId"] + ds_id = f"ds-{kb_id}-0" + ds_summary = { + "dataSourceId": ds_id, + "name": "ds", + "status": "AVAILABLE", + "updatedAt": datetime.now(timezone.utc), + } + data_sources_by_kb[kb_id] = [ds_summary] + data_source_detail[(kb_id, ds_id)] = { + "dataSource": { + "dataSourceId": ds_id, + "knowledgeBaseId": kb_id, + "dataSourceConfiguration": { + "type": "S3", + "s3Configuration": { + "bucketArn": f"arn:aws:s3:::kb-bucket-{kb_id}", + }, + }, + } + } + + # S3 buckets — 100 entries + buckets = [{"Name": f"bucket-{i}"} for i in range(_N_BUCKETS)] + + # WAFv2 Web ACLs — 50 summaries + 50 detail entries + acl_summaries = [ + {"Id": f"acl-{i}", "Name": f"acl-{i}", "ARN": f"arn:aws:wafv2:::acl-{i}"} + for i in range(_N_ACLS) + ] + detail_by_acl_id = { + acl["Id"]: { + "Id": acl["Id"], + "Name": acl["Name"], + "ARN": acl["ARN"], + "Rules": [ + { + "Name": "SQLiRule", + "Statement": { + "ManagedRuleGroupStatement": { + "VendorName": "AWS", + "Name": "AWSManagedRulesSQLiRuleSet", + } + }, + } + ], + } + for acl in acl_summaries + } + + return make_resource_inventory( + lambda_functions=functions, + guardrails=app.GuardrailInventory( + summaries=guardrail_summaries, + detail_by_id=detail_by_guardrail_id, + ), + knowledge_bases=app.KbInventory( + summaries=kb_summaries, + data_sources_by_kb=data_sources_by_kb, + data_source_detail=data_source_detail, + ), + buckets=buckets, + web_acls=app.WebAclInventory( + summaries=acl_summaries, + detail_by_id=detail_by_acl_id, + ), + ) + + +def _make_generic_non_inventory_client() -> MagicMock: + """Return a MagicMock that satisfies every non-inventory boto3 call. + + The inventory itself is provided via the pre-built ResourceInventory; + this client handles everything else (shield, apigateway, cloudwatch, …). + """ + generic = MagicMock() + + # Paginator pattern used by some checks (cw.get_paginator, sfn, etc.) + paginator = MagicMock() + paginator.paginate.return_value = [{}] + generic.get_paginator.return_value = paginator + + # Inventory listing methods — these MUST NOT be called because + # collect_resource_inventory is patched to return the pre-built inventory. + # We leave them as MagicMock (auto-return) but we'll count their calls. + + # Non-inventory service methods — return empty / benign responses + generic.describe_subscription.side_effect = Exception("no shield subscription") + generic.get_usage_plans.return_value = {"items": []} + generic.list_service_quotas.return_value = {"Quotas": []} + generic.get_anomaly_monitors.return_value = {"AnomalyMonitors": []} + generic.describe_budgets.return_value = {"Budgets": []} + generic.get_caller_identity.return_value = {"Account": "123456789012"} + generic.list_agents.return_value = {"agentSummaries": []} + generic.list_agent_runtimes.return_value = {"agentRuntimes": []} + generic.list_state_machines.return_value = {"stateMachines": []} + generic.list_policies.return_value = {"Policies": []} + generic.list_custom_models.return_value = {"modelSummaries": []} + generic.list_models.return_value = {"Models": []} + generic.describe_config_rules.return_value = {"ConfigRules": []} + generic.list_evaluation_jobs.return_value = {"jobSummaries": []} + generic.describe_repositories.return_value = {"repositories": []} + generic.list_feature_groups.return_value = {"FeatureGroupSummaries": []} + generic.list_log_groups.return_value = {"logGroups": []} + generic.get_macie_session.side_effect = Exception("macie not enabled") + generic.list_foundation_models.return_value = {"modelSummaries": []} + generic.list_model_cards.return_value = {"ModelCardSummaryList": []} + generic.list_rules.return_value = {"Rules": []} + generic.list_schedules.return_value = {"Schedules": []} + generic.get_rest_apis.return_value = {"items": []} + generic.list_processing_jobs.return_value = {"ProcessingJobSummaries": []} + generic.list_automated_reasoning_policies.return_value = { + "automatedReasoningPolicySummaries": [] + } + generic.list_security_policies.return_value = {"securityPolicySummaries": []} + generic.list_monitoring_schedules.return_value = {"MonitoringScheduleSummaries": []} + + # S3 per-bucket detail calls (get_bucket_versioning, get_bucket_tagging, …) + generic.get_bucket_versioning.return_value = {"Status": "Enabled"} + generic.get_bucket_tagging.return_value = {"TagSet": []} + generic.get_bucket_notification_configuration.return_value = {} + + # S3 data-protection-policy for CloudWatch Logs PII check + generic.get_data_protection_policy.return_value = {} + + # per-function concurrency (FS-09 keeps its own concurrency loop) + generic.get_function_concurrency.return_value = { + "ReservedConcurrentExecutions": 100 + } + + return generic + + +# --------------------------------------------------------------------------- +# Test class +# --------------------------------------------------------------------------- + + +class TestLargeEstatePerformance: + """Large-estate performance and memory guard (Task 15, Wave 4). + + Validates: REQ-10.1, REQ-10.2, REQ-10.3, REQ-11.1, REQ-11.3 + """ + + @patch("app.write_to_s3") + @patch("app.get_permissions_cache") + @patch("app.boto3.client") + @patch("app.collect_resource_inventory") + def test_large_estate_single_listing_and_memory( + self, + mock_collect_inventory, + mock_boto3_client, + mock_get_perm_cache, + mock_write_s3, + ): + """With 1 000 functions, 100 buckets, 50 guardrails/KBs/ACLs: + - collect_resource_inventory is called exactly once (single enumeration). + - The five shared listing APIs are never called directly (≤0 for each + because the collector is patched out entirely). + - Detail APIs (get_guardrail, get_web_acl, get_function_concurrency) are + only called from within the checks themselves; each is called at most + once per resource since the checks read from the pre-built inventory. + - Handler completes well within 900 s. + - Peak memory well under 1024 MB. + """ + # --- Setup --- + large_inventory = _build_large_inventory() + mock_collect_inventory.return_value = large_inventory + + generic_client = _make_generic_non_inventory_client() + mock_boto3_client.return_value = generic_client + + mock_get_perm_cache.return_value = { + "role_permissions": {}, + "user_permissions": {}, + } + mock_write_s3.return_value = "https://test-bucket.s3.amazonaws.com/finserv_security_report_large-estate.csv" + + event = { + "Execution": {"Name": "large-estate-perf-test"}, + "StateMachine": { + "Id": "arn:aws:states:us-east-1:123456789012:stateMachine:test" + }, + } + + # --- Run under memory and time instrumentation --- + tracemalloc.start() + start = time.perf_counter() + + result = app.lambda_handler(event, None) + + elapsed = time.perf_counter() - start + _, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + + # --- Correctness: handler must succeed --- + assert result["statusCode"] == 200, ( + f"lambda_handler returned {result['statusCode']}" + ) + assert "findings" in result["body"] + assert len(result["body"]["findings"]) == 65, ( + f"Expected 65 findings, got {len(result['body']['findings'])}" + ) + + # --- REQ-10.1 / REQ-10.2: Single enumeration per inventory --- + # collect_resource_inventory is called exactly once per invocation. + mock_collect_inventory.assert_called_once_with() + + # The five shared listing APIs must NOT be called on the boto3 client, + # because collect_resource_inventory (which is patched out) would own + # those calls. Any direct call from a check would violate REQ-10.1. + assert generic_client.list_functions.call_count == 0, ( + f"list_functions called {generic_client.list_functions.call_count}× " + "— must be 0 when collect_resource_inventory is patched" + ) + assert generic_client.list_guardrails.call_count == 0, ( + f"list_guardrails called {generic_client.list_guardrails.call_count}×" + ) + assert generic_client.list_knowledge_bases.call_count == 0, ( + f"list_knowledge_bases called {generic_client.list_knowledge_bases.call_count}×" + ) + assert generic_client.list_buckets.call_count == 0, ( + f"list_buckets called {generic_client.list_buckets.call_count}×" + ) + assert generic_client.list_web_acls.call_count == 0, ( + f"list_web_acls called {generic_client.list_web_acls.call_count}×" + ) + + # --- REQ-10.2: ≤1 detail call per resource --- + # get_guardrail is NOT called from checks (the inventory pre-loads detail); + # verify no check bypasses the inventory and calls it directly. + assert generic_client.get_guardrail.call_count == 0, ( + f"get_guardrail called {generic_client.get_guardrail.call_count}× " + "— checks must read from inventory.guardrails.detail_by_id" + ) + # get_web_acl: same invariant. + assert generic_client.get_web_acl.call_count == 0, ( + f"get_web_acl called {generic_client.get_web_acl.call_count}× " + "— checks must read from inventory.web_acls.detail_by_id" + ) + # get_function_concurrency: FS-09 calls it per-function on its + # agent-name-filtered subset. With fn-0..fn-999 none match the + # agent/bedrock/aiml filter, so the count must be 0. + assert generic_client.get_function_concurrency.call_count == 0, ( + f"get_function_concurrency called " + f"{generic_client.get_function_concurrency.call_count}× " + "— none of fn-0..fn-999 match the agent/bedrock/aiml filter" + ) + + # --- REQ-10.3: Completion within the 900 s budget --- + assert elapsed < 900, ( + f"Handler took {elapsed:.2f}s — must complete within 900 s" + ) + + # --- REQ-11.1 / REQ-11.3: Peak memory well under 1024 MB --- + peak_mb = peak / (1024 * 1024) + assert peak_mb < 1024, ( + f"Peak memory {peak_mb:.1f} MB — must be well under 1024 MB" + ) + + @patch("app.write_to_s3") + @patch("app.get_permissions_cache") + @patch("app.boto3.client") + @patch("app.collect_resource_inventory") + def test_large_estate_timing_budget( + self, + mock_collect_inventory, + mock_boto3_client, + mock_get_perm_cache, + mock_write_s3, + ): + """Focused timing assertion: 65 checks over a large inventory must + complete well within the 900 s budget (target: under 60 s on any + reasonable CI host, giving 15× headroom). + + Validates: REQ-10.3 + """ + large_inventory = _build_large_inventory() + mock_collect_inventory.return_value = large_inventory + mock_boto3_client.return_value = _make_generic_non_inventory_client() + mock_get_perm_cache.return_value = { + "role_permissions": {}, + "user_permissions": {}, + } + mock_write_s3.return_value = "https://test-bucket.s3.amazonaws.com/report.csv" + + event = {"Execution": {"Name": "timing-test"}} + + start = time.perf_counter() + app.lambda_handler(event, None) + elapsed = time.perf_counter() - start + + # Strict budget: all mocked I/O, so well under 60 s even on slow hardware + assert elapsed < 60, ( + f"Handler took {elapsed:.2f}s with mocked I/O — target <60 s " + f"(900 s hard budget leaves {900 - elapsed:.0f}s headroom)" + ) + + @patch("app.write_to_s3") + @patch("app.get_permissions_cache") + @patch("app.boto3.client") + @patch("app.collect_resource_inventory") + def test_large_estate_memory_footprint( + self, + mock_collect_inventory, + mock_boto3_client, + mock_get_perm_cache, + mock_write_s3, + ): + """Peak memory footprint with a large inventory must stay well under + the 1024 MB Lambda limit. + + Validates: REQ-11.1, REQ-11.3 + """ + large_inventory = _build_large_inventory() + mock_collect_inventory.return_value = large_inventory + mock_boto3_client.return_value = _make_generic_non_inventory_client() + mock_get_perm_cache.return_value = { + "role_permissions": {}, + "user_permissions": {}, + } + mock_write_s3.return_value = "https://test-bucket.s3.amazonaws.com/report.csv" + + event = {"Execution": {"Name": "memory-test"}} + + tracemalloc.start() + app.lambda_handler(event, None) + _, peak = tracemalloc.get_traced_memory() + tracemalloc.stop() + + peak_mb = peak / (1024 * 1024) + # 50 MB is a generous ceiling for mocked I/O; the real ceiling is 1024 MB + assert peak_mb < 50, ( + f"Peak memory {peak_mb:.1f} MB with mocked I/O — expected well under 50 MB " + f"(hard limit is 1024 MB)" + ) diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_paginate.py b/aiml-security-assessment/functions/security/finserv_tests/test_paginate.py new file mode 100644 index 0000000..77c184b --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_tests/test_paginate.py @@ -0,0 +1,436 @@ +""" +Tests for the _paginate helper — Task 2.1 +FinServ Shared-Inventory Refactor (FU-3) + +Requirements: REQ-2.3, REQ-2.5, REQ-9.2 + +Coverage +-------- +1. WAFv2 token=("NextMarker", "NextMarker") — multi-page +2. S3 token=("ContinuationToken", "ContinuationToken") — multi-page +3. Regression: Lambda Marker convention (no token= passed) — multi-page +4. Regression: bedrock nextToken convention (no token= passed) — multi-page +5. Single-page = exactly one call (both with and without token=) +6. Repeated-token loop guard (infinite-loop protection) +7. token= override bypasses the convention table (WAFv2 NextMarker ≠ Lambda Marker) +""" + +from __future__ import annotations + +import os +import sys +from unittest.mock import MagicMock, call + + +# --------------------------------------------------------------------------- +# Make finserv_assessments importable (mirrors conftest.py / test_inventory_equivalence.py) +# --------------------------------------------------------------------------- +FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") +if FINSERV_DIR not in sys.path: + sys.path.insert(0, FINSERV_DIR) + +import app # noqa: E402 + + +# =========================================================================== +# Helpers +# =========================================================================== + + +def _make_client(pages: list[dict]) -> MagicMock: + """Return a mock boto3 client whose ``list_items`` method yields ``pages`` + sequentially. Each element in ``pages`` is the raw response dict.""" + client = MagicMock() + client.list_items.side_effect = pages + return client + + +# =========================================================================== +# 1. WAFv2 NextMarker / NextMarker (explicit token override, multi-page) +# =========================================================================== + + +class TestWafv2NextMarkerOverride: + """token=("NextMarker", "NextMarker") — both the output field and the + request parameter are "NextMarker", which collides with Lambda's + convention that maps "NextMarker" → "Marker". The explicit override + must bypass the table and use the correct input param. + + Validates: REQ-2.3 (correct per-operation convention), REQ-9.2-b + """ + + def test_multi_page_collects_all_items(self): + pages = [ + {"WebACLs": [{"Id": "acl-1"}, {"Id": "acl-2"}], "NextMarker": "tok-1"}, + {"WebACLs": [{"Id": "acl-3"}]}, + ] + client = _make_client(pages) + + result = app._paginate( + client, + "list_items", + "WebACLs", + token=("NextMarker", "NextMarker"), + Scope="REGIONAL", + ) + + assert result == [{"Id": "acl-1"}, {"Id": "acl-2"}, {"Id": "acl-3"}] + + def test_multi_page_request_uses_NextMarker_not_Marker(self): + """The second call must send NextMarker= (WAFv2), NOT Marker= (Lambda).""" + pages = [ + {"WebACLs": [{"Id": "acl-1"}], "NextMarker": "tok-1"}, + {"WebACLs": [{"Id": "acl-2"}]}, + ] + client = _make_client(pages) + + app._paginate( + client, + "list_items", + "WebACLs", + token=("NextMarker", "NextMarker"), + Scope="REGIONAL", + ) + + calls = client.list_items.call_args_list + assert len(calls) == 2 + # First call: no pagination token + assert calls[0] == call(Scope="REGIONAL") + # Second call: NextMarker= (not Marker=) + assert calls[1] == call(Scope="REGIONAL", NextMarker="tok-1") + assert "Marker" not in calls[1].kwargs + + def test_three_pages_correct_order(self): + pages = [ + {"WebACLs": [{"Id": "a"}], "NextMarker": "t1"}, + {"WebACLs": [{"Id": "b"}], "NextMarker": "t2"}, + {"WebACLs": [{"Id": "c"}]}, + ] + client = _make_client(pages) + + result = app._paginate( + client, "list_items", "WebACLs", token=("NextMarker", "NextMarker") + ) + + assert result == [{"Id": "a"}, {"Id": "b"}, {"Id": "c"}] + assert client.list_items.call_count == 3 + + +# =========================================================================== +# 2. S3 ContinuationToken / ContinuationToken (explicit token override, multi-page) +# =========================================================================== + + +class TestS3ContinuationTokenOverride: + """token=("ContinuationToken", "ContinuationToken") — ContinuationToken is + absent from the default convention table, so without the explicit override + _paginate would stop after the first page. + + Validates: REQ-2.3, REQ-2.8, REQ-9.2-b + """ + + def test_multi_page_collects_all_buckets(self): + pages = [ + { + "Buckets": [{"Name": "bucket-1"}, {"Name": "bucket-2"}], + "ContinuationToken": "ct-1", + }, + {"Buckets": [{"Name": "bucket-3"}]}, + ] + client = _make_client(pages) + + result = app._paginate( + client, + "list_items", + "Buckets", + token=("ContinuationToken", "ContinuationToken"), + MaxBuckets=1000, + ) + + assert result == [ + {"Name": "bucket-1"}, + {"Name": "bucket-2"}, + {"Name": "bucket-3"}, + ] + + def test_second_call_sends_continuation_token(self): + pages = [ + {"Buckets": [{"Name": "b1"}], "ContinuationToken": "ct-1"}, + {"Buckets": [{"Name": "b2"}]}, + ] + client = _make_client(pages) + + app._paginate( + client, + "list_items", + "Buckets", + token=("ContinuationToken", "ContinuationToken"), + MaxBuckets=1000, + ) + + calls = client.list_items.call_args_list + assert calls[0] == call(MaxBuckets=1000) + assert calls[1] == call(MaxBuckets=1000, ContinuationToken="ct-1") + + def test_three_pages_order_preserved(self): + pages = [ + {"Buckets": [{"Name": "b1"}], "ContinuationToken": "ct-1"}, + {"Buckets": [{"Name": "b2"}], "ContinuationToken": "ct-2"}, + {"Buckets": [{"Name": "b3"}]}, + ] + client = _make_client(pages) + + result = app._paginate( + client, + "list_items", + "Buckets", + token=("ContinuationToken", "ContinuationToken"), + ) + + assert [b["Name"] for b in result] == ["b1", "b2", "b3"] + assert client.list_items.call_count == 3 + + +# =========================================================================== +# 3. Regression: Lambda Marker convention unchanged (no token= passed) +# =========================================================================== + + +class TestLambdaMarkerRegression: + """Existing callers that use the Lambda convention (output: NextMarker, + input: Marker) must behave identically when no token= is passed. + + Validates: REQ-2.3 (no regression), REQ-9.2-b + """ + + def test_single_page_single_call(self): + client = _make_client([{"Functions": [{"FunctionName": "f1"}]}]) + + result = app._paginate(client, "list_items", "Functions") + + assert result == [{"FunctionName": "f1"}] + assert client.list_items.call_count == 1 + + def test_multi_page_uses_Marker_not_NextMarker(self): + pages = [ + {"Functions": [{"FunctionName": "f1"}], "NextMarker": "m1"}, + {"Functions": [{"FunctionName": "f2"}]}, + ] + client = _make_client(pages) + + result = app._paginate(client, "list_items", "Functions") + + assert result == [{"FunctionName": "f1"}, {"FunctionName": "f2"}] + calls = client.list_items.call_args_list + # Second call must use "Marker=" (Lambda convention), not "NextMarker=" + assert calls[1] == call(Marker="m1") + assert "NextMarker" not in calls[1].kwargs + + def test_multi_page_collects_all(self): + pages = [ + {"Functions": [{"FunctionName": "f1"}], "NextMarker": "m1"}, + {"Functions": [{"FunctionName": "f2"}], "NextMarker": "m2"}, + {"Functions": [{"FunctionName": "f3"}]}, + ] + client = _make_client(pages) + + result = app._paginate(client, "list_items", "Functions") + + assert [f["FunctionName"] for f in result] == ["f1", "f2", "f3"] + assert client.list_items.call_count == 3 + + +# =========================================================================== +# 4. Regression: bedrock nextToken convention unchanged (no token= passed) +# =========================================================================== + + +class TestBedrockNextTokenRegression: + """Existing callers that use the bedrock lower-camel nextToken convention + must behave identically when no token= is passed. + + Validates: REQ-2.3 (no regression), REQ-9.2-b + """ + + def test_single_page_single_call(self): + client = _make_client([{"guardrails": [{"id": "g1"}]}]) + + result = app._paginate(client, "list_items", "guardrails") + + assert result == [{"id": "g1"}] + assert client.list_items.call_count == 1 + + def test_multi_page_collects_all(self): + pages = [ + {"guardrails": [{"id": "g1"}], "nextToken": "nt1"}, + {"guardrails": [{"id": "g2"}], "nextToken": "nt2"}, + {"guardrails": [{"id": "g3"}]}, + ] + client = _make_client(pages) + + result = app._paginate(client, "list_items", "guardrails") + + assert [g["id"] for g in result] == ["g1", "g2", "g3"] + calls = client.list_items.call_args_list + assert calls[1] == call(nextToken="nt1") + assert calls[2] == call(nextToken="nt2") + assert client.list_items.call_count == 3 + + +# =========================================================================== +# 5. Single-page = exactly one call (both with and without token=) +# =========================================================================== + + +class TestSinglePageSingleCall: + """A single-page response (no continuation token in output) must yield + exactly one API call regardless of whether token= is supplied. + + Validates: REQ-2.5 + """ + + def test_no_token_override_single_call(self): + client = _make_client([{"Items": [{"id": "x"}]}]) + + result = app._paginate(client, "list_items", "Items") + + assert result == [{"id": "x"}] + assert client.list_items.call_count == 1 + + def test_wafv2_token_override_single_call(self): + client = _make_client([{"WebACLs": [{"Id": "a1"}]}]) + + result = app._paginate( + client, + "list_items", + "WebACLs", + token=("NextMarker", "NextMarker"), + Scope="REGIONAL", + ) + + assert result == [{"Id": "a1"}] + assert client.list_items.call_count == 1 + + def test_s3_token_override_single_call(self): + client = _make_client([{"Buckets": [{"Name": "b1"}]}]) + + result = app._paginate( + client, + "list_items", + "Buckets", + token=("ContinuationToken", "ContinuationToken"), + ) + + assert result == [{"Name": "b1"}] + assert client.list_items.call_count == 1 + + +# =========================================================================== +# 6. Repeated-token loop guard +# =========================================================================== + + +class TestRepeatedTokenLoopGuard: + """If a mock (or a misbehaving API) returns the same token every call, + _paginate must stop after collecting the first page and not loop forever. + + This guard must hold both with and without the token= override. + + Validates: REQ-9.2 (loop-guard regression) + """ + + def test_loop_guard_no_override(self): + """Lambda convention: repeated NextMarker stops after two calls.""" + repeated_page = {"Functions": [{"FunctionName": "f1"}], "NextMarker": "same"} + client = MagicMock() + client.list_items.return_value = repeated_page + + result = app._paginate(client, "list_items", "Functions") + + # First call collects items; token "same" is seen; second call also + # returns "same" which is in seen_tokens → stop. + assert result == [{"FunctionName": "f1"}, {"FunctionName": "f1"}] + assert client.list_items.call_count == 2 + + def test_loop_guard_with_wafv2_override(self): + """WAFv2 token= override: repeated NextMarker stops after two calls.""" + repeated_page = {"WebACLs": [{"Id": "a"}], "NextMarker": "same"} + client = MagicMock() + client.list_items.return_value = repeated_page + + result = app._paginate( + client, + "list_items", + "WebACLs", + token=("NextMarker", "NextMarker"), + ) + + assert result == [{"Id": "a"}, {"Id": "a"}] + assert client.list_items.call_count == 2 + + def test_loop_guard_with_s3_override(self): + """S3 token= override: repeated ContinuationToken stops after two calls.""" + repeated_page = {"Buckets": [{"Name": "b"}], "ContinuationToken": "same"} + client = MagicMock() + client.list_items.return_value = repeated_page + + result = app._paginate( + client, + "list_items", + "Buckets", + token=("ContinuationToken", "ContinuationToken"), + ) + + assert result == [{"Name": "b"}, {"Name": "b"}] + assert client.list_items.call_count == 2 + + +# =========================================================================== +# 7. token= override disambiguates WAFv2 NextMarker from Lambda NextMarker +# =========================================================================== + + +class TestWafv2VsLambdaDisambiguation: + """Prove that the explicit token= override is the only way to correctly + paginate WAFv2: without it, the convention table maps NextMarker → Marker + (Lambda convention), which would send the wrong input parameter. + + Validates: REQ-2.9 — the (output, input) pair must be supplied explicitly + for WAFv2 because output-field name alone is insufficient to infer the + correct input parameter. + """ + + def test_without_override_sends_Marker_for_NextMarker_output(self): + """Without token=, the convention table maps NextMarker → Marker. + This is the Lambda convention — correct for Lambda, wrong for WAFv2.""" + pages = [ + {"WebACLs": [{"Id": "a1"}], "NextMarker": "tok"}, + {"WebACLs": [{"Id": "a2"}]}, + ] + client = _make_client(pages) + + # Intentionally NOT passing token= to demonstrate the convention-table behavior + app._paginate(client, "list_items", "WebACLs") + + calls = client.list_items.call_args_list + # The convention table sends Marker= (Lambda convention) — wrong for WAFv2 + assert calls[1] == call(Marker="tok") + + def test_with_override_sends_NextMarker_for_NextMarker_output(self): + """With token=("NextMarker","NextMarker"), the second call sends NextMarker=.""" + pages = [ + {"WebACLs": [{"Id": "a1"}], "NextMarker": "tok"}, + {"WebACLs": [{"Id": "a2"}]}, + ] + client = _make_client(pages) + + app._paginate( + client, + "list_items", + "WebACLs", + token=("NextMarker", "NextMarker"), + ) + + calls = client.list_items.call_args_list + assert calls[1] == call(NextMarker="tok") + assert "Marker" not in calls[1].kwargs diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_phase2_live.py b/aiml-security-assessment/functions/security/finserv_tests/test_phase2_live.py new file mode 100644 index 0000000..bcdcd85 --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_tests/test_phase2_live.py @@ -0,0 +1,286 @@ +#!/usr/bin/env python3 +""" +Phase 2 — Live AWS integration test for finserv_assessments. + +Invokes each check function individually against a real AWS account, +captures results, and produces a triage report showing: + - PASS: check completed successfully, resources found compliant or N/A + - WARN: check completed, found non-compliant resources (expected in dev) + - ERROR: check failed — likely IAM permission issue or API incompatibility + +Usage: + # Run all checks (default): + python tests/test_phase2_live.py + + # Run a single check by name: + python tests/test_phase2_live.py check_waf_shield_on_bedrock_endpoints + + # Run with a specific S3 bucket for CSV output: + AIML_ASSESSMENT_BUCKET_NAME=my-bucket python tests/test_phase2_live.py + +Prerequisites: + - AWS credentials configured (aws configure or env vars) + - Read-only access to the target account + - No Docker required +""" + +import json +import os +import sys +import time +from datetime import datetime + +# Make finserv_assessments importable +FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") +if FINSERV_DIR not in sys.path: + sys.path.insert(0, FINSERV_DIR) + +# Set env var if not already set +if not os.environ.get("AIML_ASSESSMENT_BUCKET_NAME"): + os.environ["AIML_ASSESSMENT_BUCKET_NAME"] = "mehta-test-v55" + +import app # noqa: E402 (import must follow sys.path + env setup above) + + +# --------------------------------------------------------------------------- +# All check functions in execution order (matches lambda_handler). +# +# Each entry is a 3-tuple: (func_name, needs_cache, needs_inventory) +# needs_cache True → pass permission_cache as first argument +# needs_inventory True → pass inventory as first argument +# (both False) → call with no arguments +# --------------------------------------------------------------------------- +CHECK_FUNCTIONS = [ + # Category 1: Unbounded Consumption + ("check_waf_shield_on_bedrock_endpoints", False, True), + ("check_api_gateway_rate_limiting", False, False), + ("check_bedrock_token_quotas", False, False), + ("check_cost_anomaly_detection", False, False), + ("check_cloudwatch_token_alarms", False, False), + ("check_aws_budgets_for_aiml", False, False), + # Category 2: Excessive Agency + ("check_bedrock_agent_action_boundaries", True, False), + ("check_agentcore_policy_engine", False, False), + ("check_agent_transaction_limits", False, True), + ("check_human_in_the_loop_for_high_risk_actions", False, False), + ("check_agent_rate_alarms", False, False), + # Category 3: Supply Chain Vulnerabilities + ("check_scp_model_access_restrictions", False, False), + ("check_model_inventory_tagging", False, False), + ("check_model_onboarding_governance", False, False), + ("check_bedrock_model_evaluation_adversarial", False, False), + ("check_ecr_image_scanning", False, False), + # Category 4: Training Data & Model Poisoning + ("check_feature_store_rollback_capability", False, False), + ("check_training_data_s3_versioning", False, True), + # Category 5: Vector & Embedding Weaknesses + ("check_knowledge_base_iam_least_privilege", True, False), + ("check_knowledge_base_metadata_filtering", False, True), + ("check_opensearch_serverless_encryption", False, False), + ("check_knowledge_base_vpc_access", False, False), + # Category 6: Non-Compliant Output + ("check_guardrail_contextual_grounding", False, True), + ("check_automated_reasoning_policies", False, False), + ("check_guardrail_denied_topics_financial", False, True), + ("check_compliance_disclaimer_in_outputs", False, False), + ("check_bedrock_evaluation_compliance_datasets", False, False), + # Category 7: Misinformation + ("check_knowledge_base_data_source_sync", False, True), + ("check_source_attribution_in_guardrails", False, False), + ("check_knowledge_base_integrity_monitoring", False, True), + ("check_fm_version_currency", False, False), + # Category 8: Abusive or Harmful Output + ("check_fmeval_harmful_content", False, False), + ("check_guardrail_content_filters", False, True), + ("check_user_feedback_mechanism", False, False), + ("check_guardrail_word_filters", False, True), + # Category 9: Biased Output + ("check_sagemaker_clarify_bias", False, False), + ("check_bedrock_evaluation_bias_datasets", False, False), + ("check_sagemaker_clarify_explainability", False, False), + ("check_ai_service_cards_documentation", False, False), + # Category 10: Sensitive Information Disclosure + ("check_cloudwatch_log_pii_masking", False, False), + ("check_macie_on_training_data_buckets", False, False), + ("check_guardrail_pii_filters", False, True), + ("check_data_classification_tagging", False, True), + # Category 11: Hallucination + ("check_guardrail_grounding_threshold", False, True), + ("check_rag_knowledge_base_configured", False, True), + ("check_hallucination_disclaimer_advisory", False, False), + ("check_guardrail_relevance_grounding", False, True), + # Category 12: Prompt Injection + ("check_prompt_injection_input_validation", False, True), + ("check_bedrock_sdk_version_currency", False, True), + ("check_waf_sql_injection_rules", False, True), + ("check_penetration_testing_evidence", False, False), + # Category 13: Improper Output Handling + ("check_output_validation_lambda", False, True), + ("check_xss_prevention_waf", False, True), + ("check_output_encoding_advisory", False, False), + ("check_output_schema_validation", False, True), + # Category 14: Off-Topic & Inappropriate Output + ("check_guardrail_topic_allowlist", False, True), + ("check_contextual_grounding_for_offtopic", False, False), + # Category 15: Out-of-Date Training Data + ("check_knowledge_base_sync_schedule", False, True), + ("check_data_currency_disclaimer_advisory", False, False), + ("check_foundation_model_lifecycle_policy", False, False), + # Material Gap Checks + ("check_kb_datasource_s3_event_notifications", False, True), + ("check_agentcore_end_user_identity_propagation", False, False), + ("check_agent_financial_transaction_thresholds", False, True), + ("check_api_gateway_request_body_size_limits", False, True), + ("check_prompt_input_validation_function", False, True), +] + + +def run_single_check( + func_name, needs_cache, needs_inventory, permission_cache, inventory +): + """Run a single check function and return a result dict.""" + func = getattr(app, func_name) + start = time.time() + try: + if needs_cache: + result = func(permission_cache) + elif needs_inventory: + result = func(inventory) + else: + result = func() + elapsed = time.time() - start + status = result.get("status", "UNKNOWN") + detail = result.get("details", "") + csv_count = len(result.get("csv_data", [])) + + # Extract check IDs and statuses from csv_data for the report + csv_summary = [] + for row in result.get("csv_data", []): + csv_summary.append(f"{row.get('Check_ID', '?')}: {row.get('Status', '?')}") + + return { + "func": func_name, + "status": status, + "elapsed": round(elapsed, 2), + "csv_count": csv_count, + "csv_summary": csv_summary, + "error": detail if status == "ERROR" else "", + } + except Exception as e: + elapsed = time.time() - start + return { + "func": func_name, + "status": "EXCEPTION", + "elapsed": round(elapsed, 2), + "csv_count": 0, + "csv_summary": [], + "error": f"{type(e).__name__}: {e}", + } + + +def main(): + filter_name = sys.argv[1] if len(sys.argv) > 1 else None + + print("=" * 78) + print("Phase 2 — Live AWS Integration Test") + print( + f"Account: {os.popen('aws sts get-caller-identity --query Account --output text 2>/dev/null').read().strip()}" + ) + print( + f"Region: {os.popen('aws configure get region 2>/dev/null').read().strip() or 'us-east-1 (default)'}" + ) + print(f"Bucket: {os.environ.get('AIML_ASSESSMENT_BUCKET_NAME', 'NOT SET')}") + print(f"Time: {datetime.now().isoformat()}") + print("=" * 78) + + # Build a minimal permission cache (empty — no pre-cached IAM data) + permission_cache = {"role_permissions": {}, "user_permissions": {}} + + # Collect the shared resource inventory once — mirrors lambda_handler behaviour. + print("\nCollecting resource inventory ...", end=" ", flush=True) + inventory_start = time.time() + inventory = app.collect_resource_inventory() + inventory_elapsed = round(time.time() - inventory_start, 2) + print(f"done ({inventory_elapsed}s)") + + checks_to_run = CHECK_FUNCTIONS + if filter_name: + checks_to_run = [(n, c, i) for n, c, i in CHECK_FUNCTIONS if n == filter_name] + if not checks_to_run: + print(f"ERROR: No check function named '{filter_name}'") + sys.exit(1) + + results = [] + total = len(checks_to_run) + + for idx, (func_name, needs_cache, needs_inventory) in enumerate(checks_to_run, 1): + print(f"\n[{idx:2d}/{total}] {func_name} ...", end=" ", flush=True) + r = run_single_check( + func_name, needs_cache, needs_inventory, permission_cache, inventory + ) + results.append(r) + + # Color-coded status + status = r["status"] + if status == "PASS": + icon = "✅" + elif status == "WARN": + icon = "⚠️ " + elif status == "ERROR": + icon = "❌" + elif status == "EXCEPTION": + icon = "💥" + else: + icon = "❓" + + print(f"{icon} {status} ({r['elapsed']}s, {r['csv_count']} findings)") + if r["error"]: + # Truncate long error messages + err = r["error"][:200] + print(f" └─ {err}") + for cs in r["csv_summary"]: + print(f" └─ {cs}") + + # ----------------------------------------------------------------------- + # Summary + # ----------------------------------------------------------------------- + print("\n" + "=" * 78) + print("TRIAGE SUMMARY") + print("=" * 78) + + pass_count = sum(1 for r in results if r["status"] == "PASS") + warn_count = sum(1 for r in results if r["status"] == "WARN") + error_count = sum(1 for r in results if r["status"] == "ERROR") + exception_count = sum(1 for r in results if r["status"] == "EXCEPTION") + total_findings = sum(r["csv_count"] for r in results) + total_time = sum(r["elapsed"] for r in results) + + print(f" ✅ PASS: {pass_count:3d}") + print(f" ⚠️ WARN: {warn_count:3d}") + print(f" ❌ ERROR: {error_count:3d}") + print(f" 💥 EXCEPTION: {exception_count:3d}") + print(" ─────────────────") + print(f" Total checks: {len(results):3d}") + print(f" Total findings: {total_findings}") + print(f" Total time: {total_time:.1f}s") + + if error_count > 0 or exception_count > 0: + print("\n--- ERRORS TO TRIAGE ---") + for r in results: + if r["status"] in ("ERROR", "EXCEPTION"): + print(f" {r['func']}: {r['error'][:150]}") + + # ----------------------------------------------------------------------- + # Write JSON report for further analysis + # ----------------------------------------------------------------------- + report_path = os.path.join(os.path.dirname(__file__), "..", "phase2_results.json") + with open(report_path, "w") as f: + json.dump(results, f, indent=2) + print(f"\nDetailed results written to: {report_path}") + + # Exit code: 0 if no EXCEPTION, non-zero otherwise + sys.exit(1 if exception_count > 0 else 0) + + +if __name__ == "__main__": + main() diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_resilience.py b/aiml-security-assessment/functions/security/finserv_tests/test_resilience.py new file mode 100644 index 0000000..ffba205 --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_tests/test_resilience.py @@ -0,0 +1,765 @@ +""" +Resilience and partial-inventory tests for finserv_assessments/app.py. + +Verifies: + 1. Single-inventory failure → only dependent checks emit COULD_NOT_ASSESS + (status="ERROR", csv_data=[]), while all other checks produce normal dispositions. + 2. Multiple-inventory failure → each failure recorded independently, run completes. + 3. Multi-inventory independence (REQ-8): unavailability of one inventory does not + affect checks that depend on a different inventory. + +Validates: Requirements REQ-4.2, REQ-4.3, REQ-4.6, REQ-8 +""" + +import os +import sys +from unittest.mock import MagicMock, patch + + +FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") +if FINSERV_DIR not in sys.path: + sys.path.insert(0, FINSERV_DIR) + +TESTS_DIR = os.path.dirname(__file__) +if TESTS_DIR not in sys.path: + sys.path.insert(0, TESTS_DIR) + +import app # noqa: E402 +from conftest import make_resource_inventory # noqa: E402 + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _is_could_not_assess(result: dict) -> bool: + """Return True if the check result signals COULD_NOT_ASSESS: + status="ERROR" and csv_data is empty (the handler will synthesize the + COULD_NOT_ASSESS row from the empty csv_data — design DD-3).""" + return result["status"] == "ERROR" and result["csv_data"] == [] + + +def _is_normal_result(result: dict) -> bool: + """Return True if the check produced a real disposition (not ERROR).""" + return result["status"] in ("PASS", "WARN") or ( + result["status"] == "PASS" and isinstance(result["csv_data"], list) + ) + + +def _has_rows(result: dict) -> bool: + """Return True if the check emitted at least one CSV row.""" + return bool(result.get("csv_data")) + + +def _make_shield_mock_no_subscription(): + """Build a mock shield client whose describe_subscription raises the + ResourceNotFoundException subclass that FS-01 catches, so the check + proceeds past the shield block and tests the WAFv2 inventory path.""" + + class ResourceNotFoundException(Exception): + """Minimal stand-in for botocore's ResourceNotFoundException.""" + + class FakeExceptions: + pass + + shield = MagicMock() + FakeExceptions.ResourceNotFoundException = ResourceNotFoundException + shield.exceptions = FakeExceptions + shield.describe_subscription.side_effect = ResourceNotFoundException("no sub") + return shield + + +# --------------------------------------------------------------------------- +# 1. Single-inventory failure — Lambda inventory +# --------------------------------------------------------------------------- + + +class TestLambdaInventoryUnavailable: + """Lambda inventory unavailable → FS-09, FS-52, FS-55, FS-58, FS-67, FS-69 + become COULD_NOT_ASSESS; checks on other inventories are unaffected.""" + + _ACCESS_DENIED = PermissionError("AccessDenied: lambda:ListFunctions") + + def _make_inv(self): + return make_resource_inventory( + lambda_functions=app._Unavailable(self._ACCESS_DENIED) + ) + + # --- FS-09 (check_agent_transaction_limits) --- + def test_fs09_becomes_could_not_assess(self): + """Validates: Requirements REQ-4.2""" + result = app.check_agent_transaction_limits(self._make_inv()) + assert _is_could_not_assess(result), ( + f"Expected COULD_NOT_ASSESS but got status={result['status']!r}, " + f"csv_data={result['csv_data']!r}" + ) + + # --- FS-52 (check_bedrock_sdk_version_currency) --- + def test_fs52_becomes_could_not_assess(self): + """Validates: Requirements REQ-4.2""" + result = app.check_bedrock_sdk_version_currency(self._make_inv()) + assert _is_could_not_assess(result), ( + f"Expected COULD_NOT_ASSESS but got status={result['status']!r}" + ) + + # --- FS-55 (check_output_validation_lambda) --- + def test_fs55_becomes_could_not_assess(self): + """Validates: Requirements REQ-4.2""" + result = app.check_output_validation_lambda(self._make_inv()) + assert _is_could_not_assess(result), ( + f"Expected COULD_NOT_ASSESS but got status={result['status']!r}" + ) + + # --- FS-58 (check_output_schema_validation) --- + def test_fs58_becomes_could_not_assess(self): + """Validates: Requirements REQ-4.2""" + result = app.check_output_schema_validation(self._make_inv()) + assert _is_could_not_assess(result), ( + f"Expected COULD_NOT_ASSESS but got status={result['status']!r}" + ) + + # --- Guardrail check unaffected when lambda is unavailable --- + def test_guardrail_check_unaffected(self): + """REQ-4.3 / REQ-8: A guardrail check still produces a normal result + when only the lambda inventory is unavailable.""" + inv = self._make_inv() + result = app.check_guardrail_contextual_grounding(inv) + # Empty guardrails → "No Guardrails" informational row (normal disposition) + assert result["status"] != "ERROR", ( + "Guardrail check should not be affected by lambda inventory failure" + ) + assert _has_rows(result) + + # --- WAFv2 check unaffected when lambda is unavailable --- + def test_waf_check_unaffected(self): + """REQ-4.3 / REQ-8: WAFv2 check is unaffected by lambda unavailability.""" + inv = self._make_inv() + with patch( + "app.boto3.client", return_value=_make_shield_mock_no_subscription() + ): + result = app.check_waf_shield_on_bedrock_endpoints(inv) + assert result["status"] != "ERROR", ( + "WAFv2 check should not be affected by lambda inventory failure" + ) + + # --- S3 check unaffected when lambda is unavailable --- + def test_s3_check_unaffected(self): + """REQ-4.3 / REQ-8: S3 check is unaffected by lambda unavailability.""" + inv = self._make_inv() + with patch("app.boto3.client") as mock_boto: + mock_boto.return_value = MagicMock() + result = app.check_training_data_s3_versioning(inv) + assert result["status"] != "ERROR", ( + "S3 versioning check should not be affected by lambda inventory failure" + ) + + # --- KB check unaffected when lambda is unavailable --- + def test_kb_check_unaffected(self): + """REQ-4.3 / REQ-8: KB metadata check is unaffected by lambda unavailability.""" + inv = self._make_inv() + result = app.check_knowledge_base_metadata_filtering(inv) + assert result["status"] != "ERROR", ( + "KB metadata check should not be affected by lambda inventory failure" + ) + + +# --------------------------------------------------------------------------- +# 2. Single-inventory failure — Guardrail inventory +# --------------------------------------------------------------------------- + + +class TestGuardrailInventoryUnavailable: + """Guardrail inventory unavailable → guardrail-consuming checks become + COULD_NOT_ASSESS; lambda, S3, KB, WAFv2 checks are unaffected.""" + + _ACCESS_DENIED = PermissionError("AccessDenied: bedrock:ListGuardrails") + + def _make_inv(self): + return make_resource_inventory(guardrails=app._Unavailable(self._ACCESS_DENIED)) + + # --- FS-27 (check_guardrail_contextual_grounding) --- + def test_fs27_becomes_could_not_assess(self): + """Validates: Requirements REQ-4.2""" + result = app.check_guardrail_contextual_grounding(self._make_inv()) + assert _is_could_not_assess(result), ( + f"Expected COULD_NOT_ASSESS but got status={result['status']!r}" + ) + + # --- FS-28 (check_guardrail_denied_topics_financial) --- + def test_fs28_becomes_could_not_assess(self): + """Validates: Requirements REQ-4.2""" + result = app.check_guardrail_denied_topics_financial(self._make_inv()) + assert _is_could_not_assess(result), ( + f"Expected COULD_NOT_ASSESS but got status={result['status']!r}" + ) + + # --- FS-36 (check_guardrail_content_filters) --- + def test_fs36_becomes_could_not_assess(self): + """Validates: Requirements REQ-4.2""" + result = app.check_guardrail_content_filters(self._make_inv()) + assert _is_could_not_assess(result), ( + f"Expected COULD_NOT_ASSESS but got status={result['status']!r}" + ) + + # --- Lambda check unaffected when guardrails are unavailable --- + def test_lambda_check_unaffected(self): + """REQ-4.3: Lambda check is unaffected by guardrail inventory failure.""" + inv = self._make_inv() + with patch("app.boto3.client") as mock_boto: + mock_boto.return_value = MagicMock( + get_function_concurrency=MagicMock( + return_value={"ReservedConcurrentExecutions": 5} + ) + ) + result = app.check_agent_transaction_limits(inv) + assert result["status"] != "ERROR", ( + "Lambda check should not be affected by guardrail inventory failure" + ) + + # --- KB check unaffected when guardrails are unavailable --- + def test_kb_check_unaffected(self): + """REQ-4.3: KB check is unaffected by guardrail inventory failure.""" + inv = self._make_inv() + result = app.check_knowledge_base_metadata_filtering(inv) + assert result["status"] != "ERROR", ( + "KB check should not be affected by guardrail inventory failure" + ) + + # --- S3 check unaffected when guardrails are unavailable --- + def test_s3_check_unaffected(self): + """REQ-4.3: S3 check is unaffected by guardrail inventory failure.""" + inv = self._make_inv() + with patch("app.boto3.client") as mock_boto: + mock_boto.return_value = MagicMock() + result = app.check_training_data_s3_versioning(inv) + assert result["status"] != "ERROR", ( + "S3 check should not be affected by guardrail inventory failure" + ) + + +# --------------------------------------------------------------------------- +# 3. Single-inventory failure — S3 inventory +# --------------------------------------------------------------------------- + + +class TestS3InventoryUnavailable: + """S3 inventory unavailable → FS-21 and FS-46 become COULD_NOT_ASSESS; + other inventories' dependent checks are unaffected.""" + + _ACCESS_DENIED = PermissionError("AccessDenied: s3:ListBuckets") + + def _make_inv(self): + return make_resource_inventory(buckets=app._Unavailable(self._ACCESS_DENIED)) + + # --- FS-21 (check_training_data_s3_versioning) --- + def test_fs21_becomes_could_not_assess(self): + """Validates: Requirements REQ-4.2""" + result = app.check_training_data_s3_versioning(self._make_inv()) + assert _is_could_not_assess(result), ( + f"Expected COULD_NOT_ASSESS but got status={result['status']!r}" + ) + + # --- FS-46 (check_data_classification_tagging) --- + def test_fs46_becomes_could_not_assess(self): + """Validates: Requirements REQ-4.2""" + result = app.check_data_classification_tagging(self._make_inv()) + assert _is_could_not_assess(result), ( + f"Expected COULD_NOT_ASSESS but got status={result['status']!r}" + ) + + # --- Guardrail check unaffected when S3 is unavailable --- + def test_guardrail_check_unaffected(self): + """REQ-4.3 / REQ-8: Guardrail check is unaffected by S3 inventory failure.""" + inv = self._make_inv() + result = app.check_guardrail_contextual_grounding(inv) + assert result["status"] != "ERROR", ( + "Guardrail check should not be affected by S3 inventory failure" + ) + + # --- KB check unaffected when S3 is unavailable --- + def test_kb_check_unaffected(self): + """REQ-4.3 / REQ-8: KB check is unaffected by S3 inventory failure.""" + inv = self._make_inv() + result = app.check_knowledge_base_metadata_filtering(inv) + assert result["status"] != "ERROR", ( + "KB check should not be affected by S3 inventory failure" + ) + + +# --------------------------------------------------------------------------- +# 4. Single-inventory failure — WAFv2 inventory +# --------------------------------------------------------------------------- + + +class TestWafInventoryUnavailable: + """WAFv2 inventory unavailable → FS-01, FS-53, FS-56, FS-68 become + COULD_NOT_ASSESS; lambda, guardrail, S3, KB checks are unaffected.""" + + _ACCESS_DENIED = PermissionError("AccessDenied: wafv2:ListWebACLs") + + def _make_inv(self): + return make_resource_inventory(web_acls=app._Unavailable(self._ACCESS_DENIED)) + + # --- FS-01 (check_waf_shield_on_bedrock_endpoints) --- + def test_fs01_becomes_could_not_assess(self): + """Validates: Requirements REQ-4.2""" + with patch( + "app.boto3.client", return_value=_make_shield_mock_no_subscription() + ): + result = app.check_waf_shield_on_bedrock_endpoints(self._make_inv()) + assert _is_could_not_assess(result), ( + f"Expected COULD_NOT_ASSESS but got status={result['status']!r}" + ) + + # --- FS-53 (check_waf_sql_injection_rules) --- + def test_fs53_becomes_could_not_assess(self): + """Validates: Requirements REQ-4.2""" + result = app.check_waf_sql_injection_rules(self._make_inv()) + assert _is_could_not_assess(result), ( + f"Expected COULD_NOT_ASSESS but got status={result['status']!r}" + ) + + # --- FS-56 (check_xss_prevention_waf) --- + def test_fs56_becomes_could_not_assess(self): + """Validates: Requirements REQ-4.2""" + result = app.check_xss_prevention_waf(self._make_inv()) + assert _is_could_not_assess(result), ( + f"Expected COULD_NOT_ASSESS but got status={result['status']!r}" + ) + + # --- Lambda check unaffected when WAFv2 is unavailable --- + def test_lambda_check_unaffected(self): + """REQ-4.3 / REQ-8: Lambda check is unaffected by WAFv2 inventory failure.""" + inv = self._make_inv() + with patch("app.boto3.client") as mock_boto: + mock_boto.return_value = MagicMock( + get_function_concurrency=MagicMock( + return_value={"ReservedConcurrentExecutions": 5} + ) + ) + result = app.check_agent_transaction_limits(inv) + assert result["status"] != "ERROR", ( + "Lambda check should not be affected by WAFv2 inventory failure" + ) + + # --- Guardrail check unaffected when WAFv2 is unavailable --- + def test_guardrail_check_unaffected(self): + """REQ-4.3 / REQ-8: Guardrail check is unaffected by WAFv2 inventory failure.""" + inv = self._make_inv() + result = app.check_guardrail_contextual_grounding(inv) + assert result["status"] != "ERROR", ( + "Guardrail check should not be affected by WAFv2 inventory failure" + ) + + # --- S3 check unaffected when WAFv2 is unavailable --- + def test_s3_check_unaffected(self): + """REQ-4.3 / REQ-8: S3 check is unaffected by WAFv2 inventory failure.""" + inv = self._make_inv() + with patch("app.boto3.client") as mock_boto: + mock_boto.return_value = MagicMock() + result = app.check_training_data_s3_versioning(inv) + assert result["status"] != "ERROR", ( + "S3 check should not be affected by WAFv2 inventory failure" + ) + + +# --------------------------------------------------------------------------- +# 5. Single-inventory failure — Knowledge Base inventory +# --------------------------------------------------------------------------- + + +class TestKbInventoryUnavailable: + """KB inventory unavailable → FS-24, FS-31, FS-33, FS-48, FS-61, FS-65 + become COULD_NOT_ASSESS; other inventories' checks are unaffected.""" + + _ACCESS_DENIED = PermissionError("AccessDenied: bedrock-agent:ListKnowledgeBases") + + def _make_inv(self): + return make_resource_inventory( + knowledge_bases=app._Unavailable(self._ACCESS_DENIED) + ) + + # --- FS-24 (check_knowledge_base_metadata_filtering) --- + def test_fs24_becomes_could_not_assess(self): + """Validates: Requirements REQ-4.2""" + result = app.check_knowledge_base_metadata_filtering(self._make_inv()) + assert _is_could_not_assess(result), ( + f"Expected COULD_NOT_ASSESS but got status={result['status']!r}" + ) + + # --- FS-31 (check_knowledge_base_data_source_sync) --- + def test_fs31_becomes_could_not_assess(self): + """Validates: Requirements REQ-4.2""" + result = app.check_knowledge_base_data_source_sync(self._make_inv()) + assert _is_could_not_assess(result), ( + f"Expected COULD_NOT_ASSESS but got status={result['status']!r}" + ) + + # --- FS-48 (check_rag_knowledge_base_configured) --- + def test_fs48_becomes_could_not_assess(self): + """Validates: Requirements REQ-4.2""" + result = app.check_rag_knowledge_base_configured(self._make_inv()) + assert _is_could_not_assess(result), ( + f"Expected COULD_NOT_ASSESS but got status={result['status']!r}" + ) + + # --- Guardrail check unaffected when KB is unavailable --- + def test_guardrail_check_unaffected(self): + """REQ-4.3 / REQ-8: Guardrail check is unaffected by KB inventory failure.""" + inv = self._make_inv() + result = app.check_guardrail_contextual_grounding(inv) + assert result["status"] != "ERROR", ( + "Guardrail check should not be affected by KB inventory failure" + ) + + # --- Lambda check unaffected when KB is unavailable --- + def test_lambda_check_unaffected(self): + """REQ-4.3 / REQ-8: Lambda check is unaffected by KB inventory failure.""" + inv = self._make_inv() + with patch("app.boto3.client") as mock_boto: + mock_boto.return_value = MagicMock( + get_function_concurrency=MagicMock( + return_value={"ReservedConcurrentExecutions": 5} + ) + ) + result = app.check_agent_transaction_limits(inv) + assert result["status"] != "ERROR", ( + "Lambda check should not be affected by KB inventory failure" + ) + + # --- S3 check unaffected when KB is unavailable --- + def test_s3_check_unaffected(self): + """REQ-4.3 / REQ-8: S3 check is unaffected by KB inventory failure.""" + inv = self._make_inv() + with patch("app.boto3.client") as mock_boto: + mock_boto.return_value = MagicMock() + result = app.check_training_data_s3_versioning(inv) + assert result["status"] != "ERROR", ( + "S3 check should not be affected by KB inventory failure" + ) + + # --- WAFv2 check unaffected when KB is unavailable --- + def test_waf_check_unaffected(self): + """REQ-4.3 / REQ-8: WAFv2 check is unaffected by KB inventory failure.""" + inv = self._make_inv() + with patch( + "app.boto3.client", return_value=_make_shield_mock_no_subscription() + ): + result = app.check_waf_shield_on_bedrock_endpoints(inv) + assert result["status"] != "ERROR", ( + "WAFv2 check should not be affected by KB inventory failure" + ) + + +# --------------------------------------------------------------------------- +# 6. Multiple-inventory failure — independent sentinels, run completes +# --------------------------------------------------------------------------- + + +class TestMultipleInventoryFailure: + """REQ-4.6: multiple simultaneous inventory failures are each recorded + independently; the run completes and checks on available inventories + produce normal results.""" + + def _make_inv_guardrails_and_web_acls_unavailable(self): + return make_resource_inventory( + guardrails=app._Unavailable( + PermissionError("AccessDenied: bedrock:ListGuardrails") + ), + web_acls=app._Unavailable( + PermissionError("AccessDenied: wafv2:ListWebACLs") + ), + ) + + def _make_inv_lambda_and_s3_unavailable(self): + return make_resource_inventory( + lambda_functions=app._Unavailable( + PermissionError("AccessDenied: lambda:ListFunctions") + ), + buckets=app._Unavailable(PermissionError("AccessDenied: s3:ListBuckets")), + ) + + # --- Guardrails AND WAFv2 both unavailable --- + + def test_guardrail_check_could_not_assess_when_guardrails_unavailable(self): + """Validates: Requirements REQ-4.6 — guardrail failure independently recorded.""" + inv = self._make_inv_guardrails_and_web_acls_unavailable() + result = app.check_guardrail_contextual_grounding(inv) + assert _is_could_not_assess(result), ( + "FS-27 should be COULD_NOT_ASSESS when guardrails inventory is unavailable" + ) + + def test_waf_check_could_not_assess_when_web_acls_unavailable(self): + """Validates: Requirements REQ-4.6 — WAFv2 failure independently recorded.""" + inv = self._make_inv_guardrails_and_web_acls_unavailable() + result = app.check_waf_sql_injection_rules(inv) + assert _is_could_not_assess(result), ( + "FS-53 should be COULD_NOT_ASSESS when web_acls inventory is unavailable" + ) + + def test_kb_check_normal_when_guardrails_and_waf_unavailable(self): + """REQ-4.3: KB check produces a normal result despite guardrail+WAFv2 failures.""" + inv = self._make_inv_guardrails_and_web_acls_unavailable() + result = app.check_knowledge_base_metadata_filtering(inv) + assert result["status"] != "ERROR", ( + "KB check should not be affected when guardrails and WAFv2 are unavailable" + ) + + def test_s3_check_normal_when_guardrails_and_waf_unavailable(self): + """REQ-4.3: S3 check is unaffected by guardrail and WAFv2 failures.""" + inv = self._make_inv_guardrails_and_web_acls_unavailable() + with patch("app.boto3.client") as mock_boto: + mock_boto.return_value = MagicMock() + result = app.check_training_data_s3_versioning(inv) + assert result["status"] != "ERROR", ( + "S3 check should not be affected when guardrails and WAFv2 are unavailable" + ) + + # --- Lambda AND S3 both unavailable --- + + def test_lambda_check_could_not_assess_when_lambda_unavailable(self): + """Validates: Requirements REQ-4.6 — lambda failure independently recorded.""" + inv = self._make_inv_lambda_and_s3_unavailable() + result = app.check_agent_transaction_limits(inv) + assert _is_could_not_assess(result), ( + "FS-09 should be COULD_NOT_ASSESS when lambda_functions inventory is unavailable" + ) + + def test_s3_check_could_not_assess_when_s3_unavailable(self): + """Validates: Requirements REQ-4.6 — S3 failure independently recorded.""" + inv = self._make_inv_lambda_and_s3_unavailable() + result = app.check_training_data_s3_versioning(inv) + assert _is_could_not_assess(result), ( + "FS-21 should be COULD_NOT_ASSESS when buckets inventory is unavailable" + ) + + def test_guardrail_check_normal_when_lambda_and_s3_unavailable(self): + """REQ-4.3: Guardrail check is unaffected by lambda+S3 failures.""" + inv = self._make_inv_lambda_and_s3_unavailable() + result = app.check_guardrail_contextual_grounding(inv) + assert result["status"] != "ERROR", ( + "Guardrail check should not be affected when lambda and S3 are unavailable" + ) + + def test_kb_check_normal_when_lambda_and_s3_unavailable(self): + """REQ-4.3: KB check is unaffected by lambda+S3 failures.""" + inv = self._make_inv_lambda_and_s3_unavailable() + result = app.check_knowledge_base_metadata_filtering(inv) + assert result["status"] != "ERROR", ( + "KB check should not be affected when lambda and S3 are unavailable" + ) + + def test_waf_check_normal_when_lambda_and_s3_unavailable(self): + """REQ-4.3: WAFv2 check is unaffected by lambda+S3 failures.""" + inv = self._make_inv_lambda_and_s3_unavailable() + with patch( + "app.boto3.client", return_value=_make_shield_mock_no_subscription() + ): + result = app.check_waf_shield_on_bedrock_endpoints(inv) + assert result["status"] != "ERROR", ( + "WAFv2 check should not be affected when lambda and S3 are unavailable" + ) + + # --- All five inventories unavailable simultaneously --- + + def test_all_inventories_unavailable_run_still_completes(self): + """REQ-4.6: When all inventories fail, calling each dependent check + individually still returns a result (no unhandled exception propagates).""" + err = PermissionError("AccessDenied: all inventories") + inv = make_resource_inventory( + lambda_functions=app._Unavailable(err), + guardrails=app._Unavailable(err), + knowledge_bases=app._Unavailable(err), + buckets=app._Unavailable(err), + web_acls=app._Unavailable(err), + ) + + checks_and_kwargs = [ + (app.check_agent_transaction_limits, {"inventory": inv}), + (app.check_guardrail_contextual_grounding, {"inventory": inv}), + (app.check_knowledge_base_metadata_filtering, {"inventory": inv}), + (app.check_training_data_s3_versioning, {"inventory": inv}), + (app.check_waf_sql_injection_rules, {"inventory": inv}), + (app.check_guardrail_denied_topics_financial, {"inventory": inv}), + (app.check_guardrail_content_filters, {"inventory": inv}), + (app.check_knowledge_base_data_source_sync, {"inventory": inv}), + (app.check_rag_knowledge_base_configured, {"inventory": inv}), + (app.check_bedrock_sdk_version_currency, {"inventory": inv}), + (app.check_output_validation_lambda, {"inventory": inv}), + (app.check_waf_sql_injection_rules, {"inventory": inv}), + ] + + for check_fn, kwargs in checks_and_kwargs: + # Must not raise; must return a dict with status and csv_data + result = check_fn(**kwargs) + assert isinstance(result, dict), ( + f"{check_fn.__name__} raised instead of returning a result" + ) + assert "status" in result + assert "csv_data" in result + assert _is_could_not_assess(result), ( + f"{check_fn.__name__} should be COULD_NOT_ASSESS when all inventories unavailable; " + f"got status={result['status']!r}" + ) + + +# --------------------------------------------------------------------------- +# 7. Full handler run with partial-inventory failure +# --------------------------------------------------------------------------- + + +class TestHandlerWithPartialInventory: + """Verify that when collect_resource_inventory returns a partially-unavailable + ResourceInventory (patched in), the handler still completes (statusCode=200) + and synthesizes visible COULD_NOT_ASSESS rows for the affected checks while + other checks produce normal rows. + + Validates: Requirements REQ-4.2, REQ-4.3, REQ-4.6 + """ + + def _make_generic_mock_client(self): + generic = MagicMock() + paginator = MagicMock() + paginator.paginate.return_value = [{}] + generic.get_paginator.return_value = paginator + generic.list_web_acls.return_value = {"WebACLs": []} + + # FS-01 uses shield.exceptions.ResourceNotFoundException — set up a proper + # exception class so describe_subscription's side_effect is caught correctly. + class ResourceNotFoundException(Exception): + pass + + class FakeExceptions: + pass + + FakeExceptions.ResourceNotFoundException = ResourceNotFoundException + generic.exceptions = FakeExceptions + generic.describe_subscription.side_effect = ResourceNotFoundException("no sub") + + generic.get_usage_plans.return_value = {"items": []} + generic.list_service_quotas.return_value = {"Quotas": []} + generic.get_anomaly_monitors.return_value = {"AnomalyMonitors": []} + generic.describe_budgets.return_value = {"Budgets": []} + generic.get_caller_identity.return_value = {"Account": "123456789012"} + generic.list_agents.return_value = {"agentSummaries": []} + generic.list_agent_runtimes.return_value = {"agentRuntimes": []} + generic.list_functions.return_value = {"Functions": []} + generic.list_state_machines.return_value = {"stateMachines": []} + generic.list_policies.return_value = {"Policies": []} + generic.list_custom_models.return_value = {"modelSummaries": []} + generic.list_models.return_value = {"Models": []} + generic.describe_config_rules.return_value = {"ConfigRules": []} + generic.list_evaluation_jobs.return_value = {"jobSummaries": []} + generic.describe_repositories.return_value = {"repositories": []} + generic.list_feature_groups.return_value = {"FeatureGroupSummaries": []} + generic.list_buckets.return_value = {"Buckets": []} + generic.list_knowledge_bases.return_value = {"knowledgeBaseSummaries": []} + generic.list_guardrails.return_value = {"guardrails": []} + generic.list_log_groups.return_value = {"logGroups": []} + generic.get_macie_session.side_effect = Exception("not enabled") + generic.list_foundation_models.return_value = {"modelSummaries": []} + generic.list_model_cards.return_value = {"ModelCardSummaries": []} + generic.list_rules.return_value = {"Rules": []} + generic.list_schedules.return_value = {"Schedules": []} + generic.get_rest_apis.return_value = {"items": []} + generic.list_processing_jobs.return_value = {"ProcessingJobSummaries": []} + generic.list_automated_reasoning_policies.return_value = { + "automatedReasoningPolicySummaries": [] + } + return generic + + @patch("app.write_to_s3") + @patch("app.get_permissions_cache") + @patch("app.collect_resource_inventory") + @patch("app.boto3.client") + def test_handler_completes_with_lambda_inventory_unavailable( + self, + mock_boto_client, + mock_collect_inv, + mock_cache, + mock_s3, + lambda_event, + ): + """Handler returns 200 and all 65 findings are present (COULD_NOT_ASSESS + rows are synthesized for lambda-dependent checks) when lambda inventory fails. + + Validates: Requirements REQ-4.2, REQ-4.3, REQ-4.6 + """ + err = PermissionError("AccessDenied: lambda:ListFunctions") + partial_inv = make_resource_inventory(lambda_functions=app._Unavailable(err)) + mock_collect_inv.return_value = partial_inv + mock_boto_client.return_value = self._make_generic_mock_client() + mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} + mock_s3.return_value = "https://test-bucket.s3.amazonaws.com/report.csv" + + result = app.lambda_handler(lambda_event, None) + + assert result["statusCode"] == 200 + findings = result["body"]["findings"] + # All 65 registry entries must produce a result dict + assert len(findings) == 65 + + # Lambda-dependent checks should have a synthesized COULD_NOT_ASSESS row + # in their csv_data (the handler's guard calls _could_not_assess_row for + # checks that return empty csv_data — design DD-3). + lambda_dependent_check_names = { + "Agent Transaction Limits Check", # FS-09 + "Bedrock SDK Version Currency Check", # FS-52 + "Output Validation Lambda Check", # FS-55 + "Output Schema Validation Check", # FS-58 + } + + for finding in findings: + check_name = finding.get("check_name", "") + rows = finding.get("csv_data", []) + if check_name in lambda_dependent_check_names: + # The handler synthesizes a COULD_NOT_ASSESS row and appends it + # to csv_data before returning — so we expect exactly 1 row with + # the COULD NOT ASSESS prefix and Status="N/A". + assert len(rows) == 1, ( + f"{check_name} should have 1 synthesized COULD_NOT_ASSESS row, " + f"got {len(rows)}: {rows!r}" + ) + assert rows[0]["Finding"].startswith(app.COULD_NOT_ASSESS_PREFIX), ( + f"{check_name} row Finding should start with COULD NOT ASSESS prefix" + ) + # StatusEnum.NA == "N/A" because StatusEnum inherits from str + assert rows[0]["Status"] == "N/A", ( + f"{check_name} COULD_NOT_ASSESS row should have Status=N/A, " + f"got {rows[0]['Status']!r}" + ) + + @patch("app.write_to_s3") + @patch("app.get_permissions_cache") + @patch("app.collect_resource_inventory") + @patch("app.boto3.client") + def test_handler_completes_with_guardrails_and_waf_unavailable( + self, + mock_boto_client, + mock_collect_inv, + mock_cache, + mock_s3, + lambda_event, + ): + """Handler returns 200 with all 65 findings when guardrails and WAFv2 fail. + + Validates: Requirements REQ-4.6 — multiple independent failures + """ + partial_inv = make_resource_inventory( + guardrails=app._Unavailable( + PermissionError("AccessDenied: bedrock:ListGuardrails") + ), + web_acls=app._Unavailable( + PermissionError("AccessDenied: wafv2:ListWebACLs") + ), + ) + mock_collect_inv.return_value = partial_inv + mock_boto_client.return_value = self._make_generic_mock_client() + mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} + mock_s3.return_value = "https://test-bucket.s3.amazonaws.com/report.csv" + + result = app.lambda_handler(lambda_event, None) + + assert result["statusCode"] == 200 + assert len(result["body"]["findings"]) == 65 diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_schema.py b/aiml-security-assessment/functions/security/finserv_tests/test_schema.py new file mode 100644 index 0000000..5daead5 --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_tests/test_schema.py @@ -0,0 +1,339 @@ +""" +Tests for finserv_assessments/schema.py + +Covers: + - Valid finding creation for all severity/status combinations + - Check_ID pattern validation (FS-NN, BR-NN, AC-NN, SM-NN) + - Reference URL validation (must start with https://) + - Required field presence and min_length constraints + - Pydantic model_dump output structure +""" + +import pytest +from schema import Finding, SeverityEnum, StatusEnum, create_finding +from app import COMPLIANCE_MAP, build_finserv_checks + + +# ========================================================================= +# Enum completeness +# ========================================================================= + + +class TestEnums: + def test_severity_values(self): + assert set(SeverityEnum) == { + SeverityEnum.HIGH, + SeverityEnum.MEDIUM, + SeverityEnum.LOW, + SeverityEnum.INFORMATIONAL, + } + + def test_status_values(self): + assert set(StatusEnum) == { + StatusEnum.FAILED, + StatusEnum.PASSED, + StatusEnum.NA, + } + + def test_severity_string_values(self): + assert SeverityEnum.HIGH.value == "High" + assert SeverityEnum.MEDIUM.value == "Medium" + assert SeverityEnum.LOW.value == "Low" + assert SeverityEnum.INFORMATIONAL.value == "Informational" + + def test_status_string_values(self): + assert StatusEnum.FAILED.value == "Failed" + assert StatusEnum.PASSED.value == "Passed" + assert StatusEnum.NA.value == "N/A" + + +# ========================================================================= +# create_finding — happy paths +# ========================================================================= + + +class TestCreateFindingValid: + """Every severity × status combination should produce a valid dict.""" + + @pytest.mark.parametrize("severity", list(SeverityEnum)) + @pytest.mark.parametrize("status", list(StatusEnum)) + def test_all_severity_status_combos(self, severity, status): + result = create_finding( + check_id="FS-01", + finding_name="Test Finding", + finding_details="Some details here", + resolution="Fix it", + reference="https://docs.aws.amazon.com/example", + severity=severity.value, + status=status.value, + ) + assert isinstance(result, dict) + assert result["Check_ID"] == "FS-01" + assert result["Severity"] == severity.value + assert result["Status"] == status.value + + def test_output_has_all_csv_fields(self): + result = create_finding( + check_id="FS-42", + finding_name="Name", + finding_details="Details", + resolution="Resolution", + reference="https://example.com", + severity="High", + status="Failed", + ) + expected_keys = { + "Check_ID", + "Finding", + "Finding_Details", + "Resolution", + "Reference", + "Severity", + "Status", + "Compliance_Frameworks", + } + assert set(result.keys()) == expected_keys + + @pytest.mark.parametrize( + "check_id", + ["FS-01", "FS-69", "BR-14", "SM-07", "AC-05"], + ) + def test_valid_check_id_patterns(self, check_id): + result = create_finding( + check_id=check_id, + finding_name="Test", + finding_details="Details", + resolution="", + reference="https://example.com", + severity="Low", + status="Passed", + ) + assert result["Check_ID"] == check_id + + def test_empty_resolution_allowed(self): + """Resolution has min_length=0, so empty string is valid.""" + result = create_finding( + check_id="FS-01", + finding_name="Test", + finding_details="Details", + resolution="", + reference="https://example.com", + severity="Informational", + status="Passed", + ) + assert result["Resolution"] == "" + + +# ========================================================================= +# create_finding — validation errors +# ========================================================================= + + +class TestCreateFindingInvalid: + def test_invalid_check_id_pattern(self): + with pytest.raises(Exception): + create_finding( + check_id="INVALID", + finding_name="Test", + finding_details="Details", + resolution="Fix", + reference="https://example.com", + severity="High", + status="Failed", + ) + + def test_check_id_lowercase_rejected(self): + with pytest.raises(Exception): + create_finding( + check_id="fs-01", + finding_name="Test", + finding_details="Details", + resolution="Fix", + reference="https://example.com", + severity="High", + status="Failed", + ) + + def test_check_id_missing_dash(self): + with pytest.raises(Exception): + create_finding( + check_id="FS01", + finding_name="Test", + finding_details="Details", + resolution="Fix", + reference="https://example.com", + severity="High", + status="Failed", + ) + + def test_reference_not_https(self): + with pytest.raises(Exception): + create_finding( + check_id="FS-01", + finding_name="Test", + finding_details="Details", + resolution="Fix", + reference="http://example.com", + severity="High", + status="Failed", + ) + + def test_empty_finding_name_rejected(self): + with pytest.raises(Exception): + create_finding( + check_id="FS-01", + finding_name="", + finding_details="Details", + resolution="Fix", + reference="https://example.com", + severity="High", + status="Failed", + ) + + def test_empty_finding_details_rejected(self): + with pytest.raises(Exception): + create_finding( + check_id="FS-01", + finding_name="Test", + finding_details="", + resolution="Fix", + reference="https://example.com", + severity="High", + status="Failed", + ) + + def test_invalid_severity(self): + with pytest.raises(Exception): + create_finding( + check_id="FS-01", + finding_name="Test", + finding_details="Details", + resolution="Fix", + reference="https://example.com", + severity="Critical", + status="Failed", + ) + + def test_invalid_status(self): + with pytest.raises(Exception): + create_finding( + check_id="FS-01", + finding_name="Test", + finding_details="Details", + resolution="Fix", + reference="https://example.com", + severity="High", + status="Open", + ) + + +# ========================================================================= +# Finding model direct instantiation +# ========================================================================= + + +class TestFindingModel: + def test_model_dump_returns_dict(self): + f = Finding( + Check_ID="FS-01", + Finding="Test", + Finding_Details="Details", + Resolution="Fix", + Reference="https://example.com", + Severity=SeverityEnum.HIGH, + Status=StatusEnum.FAILED, + ) + d = f.model_dump() + assert isinstance(d, dict) + assert d["Check_ID"] == "FS-01" + + def test_check_id_three_letter_prefix(self): + """Three-letter prefixes like ACM-01 should be valid.""" + f = Finding( + Check_ID="ACM-01", + Finding="Test", + Finding_Details="Details", + Resolution="Fix", + Reference="https://example.com", + Severity=SeverityEnum.LOW, + Status=StatusEnum.PASSED, + ) + assert f.Check_ID == "ACM-01" + + +# ========================================================================= +# D3 fix — Compliance_Frameworks field in schema and COMPLIANCE_MAP coverage +# ========================================================================= + + +class TestComplianceFrameworksField: + def test_compliance_frameworks_defaults_to_empty_string(self): + """compliance_frameworks is optional and defaults to empty string.""" + result = create_finding( + check_id="FS-01", + finding_name="Test", + finding_details="Details", + resolution="Fix", + reference="https://example.com", + severity="High", + status="Failed", + ) + assert "Compliance_Frameworks" in result + assert result["Compliance_Frameworks"] == "" + + def test_compliance_frameworks_round_trips(self): + """A populated compliance_frameworks value is preserved verbatim.""" + fw = "FFIEC CAT | SR 11-7 | NYDFS 500" + result = create_finding( + check_id="FS-01", + finding_name="Test", + finding_details="Details", + resolution="Fix", + reference="https://example.com", + severity="High", + status="Failed", + compliance_frameworks=fw, + ) + assert result["Compliance_Frameworks"] == fw + + def test_compliance_map_covers_all_registry_checks(self): + """Every check in build_finserv_checks must have an entry in COMPLIANCE_MAP.""" + registry = build_finserv_checks({}) + registry_ids = {check_id for check_id, _ in registry} + missing = registry_ids - set(COMPLIANCE_MAP.keys()) + assert not missing, ( + f"These FS IDs are in the registry but missing from COMPLIANCE_MAP: {sorted(missing)}" + ) + + def test_compliance_map_all_values_non_empty(self): + """Every COMPLIANCE_MAP entry must have at least one framework.""" + empty = [k for k, v in COMPLIANCE_MAP.items() if not v.strip()] + assert not empty, f"Empty compliance_frameworks values: {empty}" + + def test_compliance_map_values_pipe_separated(self): + """All multi-framework values use ' | ' as separator (consistent with ASFF style).""" + for check_id, value in COMPLIANCE_MAP.items(): + if "|" in value: + # Each segment should be non-empty after strip + parts = [p.strip() for p in value.split("|")] + empty_parts = [p for p in parts if not p] + assert not empty_parts, f"{check_id}: empty segment in '{value}'" + + def test_compliance_frameworks_present_in_finding_model_dump(self): + """Finding.model_dump() includes Compliance_Frameworks — critical for CSV writer.""" + f = Finding( + Check_ID="FS-29", + Finding="ADVISORY: Test", + Finding_Details="Details", + Resolution="Fix", + Reference="https://example.com", + Severity=SeverityEnum.INFORMATIONAL, + Status=StatusEnum.NA, + Compliance_Frameworks="SR 11-7 | FFIEC CAT | NYDFS 500 | MAS TRM 9.2", + ) + d = f.model_dump() + assert "Compliance_Frameworks" in d + assert ( + d["Compliance_Frameworks"] + == "SR 11-7 | FFIEC CAT | NYDFS 500 | MAS TRM 9.2" + ) diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_severity_register.py b/aiml-security-assessment/functions/security/finserv_tests/test_severity_register.py new file mode 100644 index 0000000..64ce06a --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_tests/test_severity_register.py @@ -0,0 +1,132 @@ +""" +Drift-guard for the FinServ severity methodology (REQ-6 / severity-register.md). + +Asserts that: + 1. The Likelihood x Impact matrix helper matches the documented table (methodology §3.3). + 2. The disposition->severity rules match methodology §3.4. + 3. SEVERITY_REGISTER uses only the four allowed labels (no Critical this round). + 4. Every finding-name emitted by the check registry exists in SEVERITY_REGISTER, and the + emitted `severity=` equals the register value (prevents future drift across 64 checks). + +The boto3 calls in every check are mocked to raise, which forces each check down a path that +still emits its finding rows (or is caught), so we can collect the (finding_name, severity) +pairs the code actually produces without real AWS access. +""" + +import sys +import os +from unittest.mock import MagicMock, patch + +FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") +if FINSERV_DIR not in sys.path: + sys.path.insert(0, FINSERV_DIR) + +import app # noqa: E402 + +ALLOWED = {"High", "Medium", "Low", "Informational"} + + +# --------------------------------------------------------------------------- +# 1. Matrix helper matches the documented 3x3 table +# --------------------------------------------------------------------------- +def test_label_from_matrix_matches_methodology_table(): + expected = { + (3, 1): "Medium", + (3, 2): "High", + (3, 3): "High", + (2, 1): "Low", + (2, 2): "Medium", + (2, 3): "High", + (1, 1): "Low", + (1, 2): "Low", + (1, 3): "Medium", + } + for (i, ell), label in expected.items(): + assert app._label_from_matrix(i, ell) == label, f"matrix[{i},{ell}]" + + +# --------------------------------------------------------------------------- +# 2. Disposition -> severity rules (methodology §3.4) +# --------------------------------------------------------------------------- +def test_disposition_severity_rules(): + assert app._DISPOSITION_SEVERITY["NOT_APPLICABLE"] == "Informational" + assert app._DISPOSITION_SEVERITY["ADVISORY"] == "Informational" + assert app._DISPOSITION_SEVERITY["COULD_NOT_ASSESS"] == "Low" + + +# --------------------------------------------------------------------------- +# 3. Register uses only allowed labels (no Critical) +# --------------------------------------------------------------------------- +def test_register_labels_are_allowed_no_critical(): + assert app.SEVERITY_REGISTER, "register must not be empty" + bad = {v for v in app.SEVERITY_REGISTER.values() if v not in ALLOWED} + assert not bad, f"register has disallowed labels: {bad}" + + +# --------------------------------------------------------------------------- +# 4. Could-not-assess synthesized row is Low / N/A (COULD_NOT_ASSESS disposition) +# --------------------------------------------------------------------------- +def test_could_not_assess_row_is_low(): + row = app._could_not_assess_row("FS-01", "Some Check", "boom") + assert row["Severity"] == "Low" + assert row["Status"] == "N/A" + assert row["Finding"].startswith(app.COULD_NOT_ASSESS_PREFIX) + + +# --------------------------------------------------------------------------- +# 5. Every emitted finding's severity matches the register (the real drift-guard) +# --------------------------------------------------------------------------- +def _collect_emitted_rows(): + """Run every registered check with all boto3 calls raising, collecting CSV rows. + + A check whose body raises returns _error_findings (no rows) — that is fine; we only + assert on the rows that ARE emitted. Advisory checks (no boto3) emit their row directly. + """ + rows = [] + cache = {"role_permissions": {}, "user_permissions": {}} + + def boom_client(*_a, **_k): + m = MagicMock() + # Any attribute access returns a callable that raises, so checks that call + # boto3 fall into their except/_error_findings path deterministically. + m.side_effect = Exception("no aws in unit test") + + def _raise(*_aa, **_kk): + raise Exception("no aws in unit test") + + m.__getattr__ = lambda _name: _raise + return m + + with patch.object(app.boto3, "client", side_effect=boom_client): + for check_id, fn in app.build_finserv_checks(cache): + try: + result = fn() + except Exception: + continue + for row in result.get("csv_data", []): + rows.append((row["Finding"], str(row["Severity"]), str(row["Status"]))) + return rows + + +def test_emitted_severity_matches_register(): + """Advisory/static-path checks emit rows even with boto3 down; assert they match.""" + rows = _collect_emitted_rows() + # We only check rows whose finding-name is a static name in the register + # (could-not-assess rows use a dynamic name and are validated separately). + mismatches = [] + for finding, severity, _status in rows: + sev = severity.split(".")[-1].title() if "." in severity else severity + if finding in app.SEVERITY_REGISTER and app.SEVERITY_REGISTER[finding] != sev: + mismatches.append((finding, sev, app.SEVERITY_REGISTER[finding])) + assert not mismatches, f"severity drift: {mismatches}" + + +def test_advisory_rows_are_informational_na(): + """All ADVISORY-prefixed findings must be Informational / N/A.""" + rows = _collect_emitted_rows() + for finding, severity, status in rows: + if finding.startswith("ADVISORY: "): + sev = severity.split(".")[-1].title() if "." in severity else severity + st = status.split(".")[-1] if "." in status else status + assert sev == "Informational", f"{finding} severity={sev}" + assert st in ("NA", "N/A"), f"{finding} status={st}" diff --git a/aiml-security-assessment/functions/security/generate_consolidated_report/app.py b/aiml-security-assessment/functions/security/generate_consolidated_report/app.py index 7626a42..ea707b9 100644 --- a/aiml-security-assessment/functions/security/generate_consolidated_report/app.py +++ b/aiml-security-assessment/functions/security/generate_consolidated_report/app.py @@ -44,7 +44,7 @@ def parse_csv_content(csv_content: str) -> List[Dict[str, str]]: def get_assessment_results(execution_id: str, account_id: str = None) -> Dict[str, Any]: """ - Download and parse Bedrock and SageMaker assessment CSV files for a given execution + Download and parse Bedrock, SageMaker, AgentCore, and FinServ assessment CSV files for a given execution Args: s3_bucket (str): Source S3 bucket name @@ -72,6 +72,11 @@ def get_assessment_results(execution_id: str, account_id: str = None) -> Dict[st Bucket=s3_bucket, Prefix=f"agentcore_security_report_{execution_id}" ) + # Also check for FinServ reports + finserv_response = s3_client.list_objects_v2( + Bucket=s3_bucket, Prefix=f"finserv_security_report_{execution_id}" + ) + # Combine all responses all_objects = [] if "Contents" in response: @@ -80,6 +85,8 @@ def get_assessment_results(execution_id: str, account_id: str = None) -> Dict[st all_objects.extend(sagemaker_response["Contents"]) if "Contents" in agentcore_response: all_objects.extend(agentcore_response["Contents"]) + if "Contents" in finserv_response: + all_objects.extend(finserv_response["Contents"]) if not all_objects: logger.warning(f"No assessment files found for execution {execution_id}") return {} @@ -91,6 +98,7 @@ def get_assessment_results(execution_id: str, account_id: str = None) -> Dict[st "bedrock": {}, "sagemaker": {}, "agentcore": {}, + "finserv": {}, } # Process each CSV file @@ -126,6 +134,8 @@ def get_assessment_results(execution_id: str, account_id: str = None) -> Dict[st category = "sagemaker" elif "agentcore" in s3_key.lower(): category = "agentcore" + elif "finserv" in s3_key.lower(): + category = "finserv" else: logger.warning(f"Unknown assessment type for file: {s3_key}") continue @@ -146,10 +156,11 @@ def get_assessment_results(execution_id: str, account_id: str = None) -> Dict[st assessment_results["summary"] = { "total_files_processed": len(assessment_results["bedrock"]) + len(assessment_results["sagemaker"]) - + len(assessment_results["agentcore"]), + + len(assessment_results["agentcore"]) + + len(assessment_results["finserv"]), "categories_found": [ cat - for cat in ["bedrock", "sagemaker", "agentcore"] + for cat in ["bedrock", "sagemaker", "agentcore", "finserv"] if assessment_results[cat] ], "rows": assessment_results["bedrock"], @@ -157,6 +168,7 @@ def get_assessment_results(execution_id: str, account_id: str = None) -> Dict[st "bedrock": list(assessment_results["bedrock"].keys()), "sagemaker": list(assessment_results["sagemaker"].keys()), "agentcore": list(assessment_results["agentcore"].keys()), + "finserv": list(assessment_results["finserv"].keys()), }, } @@ -185,7 +197,7 @@ def generate_html_report(assessment_results: Dict[str, Any]) -> str: expected by the shared report_template module. Args: - assessment_results: Dict containing bedrock, sagemaker, agentcore findings + assessment_results: Dict containing bedrock, sagemaker, agentcore, finserv findings Returns: HTML report string @@ -196,10 +208,11 @@ def generate_html_report(assessment_results: Dict[str, Any]) -> str: "bedrock": {"passed": 0, "failed": 0, "na": 0}, "sagemaker": {"passed": 0, "failed": 0, "na": 0}, "agentcore": {"passed": 0, "failed": 0, "na": 0}, + "finserv": {"passed": 0, "failed": 0, "na": 0}, } - service_findings = {"bedrock": [], "sagemaker": [], "agentcore": []} + service_findings = {"bedrock": [], "sagemaker": [], "agentcore": [], "finserv": []} - for service in ["bedrock", "sagemaker", "agentcore"]: + for service in ["bedrock", "sagemaker", "agentcore", "finserv"]: if service in assessment_results: for report_type, findings in assessment_results[service].items(): for finding in findings: diff --git a/aiml-security-assessment/functions/security/generate_consolidated_report/report_template.py b/aiml-security-assessment/functions/security/generate_consolidated_report/report_template.py index 7f705df..7cb6d41 100644 --- a/aiml-security-assessment/functions/security/generate_consolidated_report/report_template.py +++ b/aiml-security-assessment/functions/security/generate_consolidated_report/report_template.py @@ -9,6 +9,21 @@ from datetime import datetime, timezone from typing import Dict, List, Optional +# FinServ service icon (no official AWS icon exists for "Financial Services"). +FINSERV_ICON = ( + '' + '' + '' +) +FINSERV_ICON_SMALL = ( + '' + '' + '' + '' +) + def generate_table_rows(findings: List[Dict], include_data_attrs: bool = True) -> str: """ @@ -281,6 +296,7 @@ def get_html_template() -> str: AgentCore {agentcore_total} + {finserv_nav}

Generated: {date_display}

@@ -326,7 +342,7 @@ def get_html_template() -> str:
{account_filter} -
+
@@ -348,6 +364,7 @@ def get_html_template() -> str:
Bedrock
{bedrock_total}
{bedrock_failed} Failed · {bedrock_passed} Passed
SageMaker
{sagemaker_total}
{sagemaker_failed} Failed · {sagemaker_passed} Passed
AgentCore
{agentcore_total}
{agentcore_failed} Failed · {agentcore_passed} Passed
+ {finserv_service_card}
@@ -383,6 +400,7 @@ def get_html_template() -> str:
{agentcore_rows}
Account IDCheck IDFindingDetailsResolutionReferenceSeverityStatus
+ {finserv_section}
Assessment Methodology

Severity Levels & Status Values

HighDirect security riskFailedRemediation needed
MediumDefense-in-depth gapPassedMeets requirements
LowBest practiceN/ANot applicable
InformationalNo action required
@@ -553,6 +571,7 @@ def get_html_template() -> str: createServiceFilter('bedrockTable', 'bedrockSearchInput', 'bedrockAccountFilter', 'bedrockSeverityFilter', 'bedrockStatusFilter', 'bedrockResetFilters'); createServiceFilter('sagemakerTable', 'sagemakerSearchInput', 'sagemakerAccountFilter', 'sagemakerSeverityFilter', 'sagemakerStatusFilter', 'sagemakerResetFilters'); createServiceFilter('agentcoreTable', 'agentcoreSearchInput', 'agentcoreAccountFilter', 'agentcoreSeverityFilter', 'agentcoreStatusFilter', 'agentcoreResetFilters'); + createServiceFilter('finservTable', 'finservSearchInput', 'finservAccountFilter', 'finservSeverityFilter', 'finservStatusFilter', 'finservResetFilters'); // Apply initial filters for main table applyFilters(); @@ -675,6 +694,8 @@ def generate_html_report( if f.get("_service") == "bedrock" else "SageMaker" if f.get("_service") == "sagemaker" + else "FinServ" + if f.get("_service") == "finserv" else "AgentCore" ) alerts_html += f"""
@@ -691,6 +712,8 @@ def generate_html_report( if f.get("_service") == "bedrock" else "SageMaker" if f.get("_service") == "sagemaker" + else "FinServ" + if f.get("_service") == "finserv" else "AgentCore" ) alerts_html += f"""
@@ -715,6 +738,9 @@ def generate_html_report( agentcore_rows = generate_table_rows( service_findings.get("agentcore", []), include_data_attrs=True ) + finserv_rows = generate_table_rows( + service_findings.get("finserv", []), include_data_attrs=True + ) # Mode-specific content num_accounts = len(account_ids) if account_ids else 1 @@ -734,6 +760,7 @@ def generate_html_report( bedrock_account_filter = f'
' sagemaker_account_filter = f'
' agentcore_account_filter = f'
' + finserv_account_filter = f'
' # Calculate per-account risk metrics account_metrics_html = "" @@ -788,8 +815,56 @@ def generate_html_report( bedrock_account_filter = "" sagemaker_account_filter = "" agentcore_account_filter = "" + finserv_account_filter = "" account_risk_section = "" + # FinServ (FS-*) — first-class service, rendered only when findings exist + # (so non-FinServ accounts and EnableFinServAssessment=false deploys stay clean). + finserv_total = ( + service_stats.get("finserv", {}).get("passed", 0) + + service_stats.get("finserv", {}).get("failed", 0) + + service_stats.get("finserv", {}).get("na", 0) + ) + finserv_failed = service_stats.get("finserv", {}).get("failed", 0) + finserv_passed = service_stats.get("finserv", {}).get("passed", 0) + if finserv_total > 0: + finserv_nav = ( + '' + + FINSERV_ICON + + " Financial Services" + + f'{finserv_total}' + ) + finserv_filter_option = '' + finserv_service_card = ( + '
' + + FINSERV_ICON_SMALL + + f' FinServ
{finserv_total}
' + + f'
{finserv_failed} Failed \u00b7 {finserv_passed} Passed
' + ) + finserv_section = ( + '
' + '
' + + FINSERV_ICON + + "Financial Services GenAI Risk Findings
" + '
Scope: this assessment evaluates the deployment Region only — run it per Region for multi-Region GenAI workloads. Severities follow a documented Likelihood × Impact methodology (see docs).
' + '
' + '
' + + finserv_account_filter + + '
' + '
' + '' + "
" + '
' + + finserv_rows + + "
Account IDCheck IDFindingDetailsResolutionReferenceSeverityStatus
" + "
" + ) + else: + finserv_nav = "" + finserv_filter_option = "" + finserv_service_card = "" + finserv_section = "" + # Fill template html_template = get_html_template() @@ -840,5 +915,9 @@ def generate_html_report( bedrock_account_filter=bedrock_account_filter, sagemaker_account_filter=sagemaker_account_filter, agentcore_account_filter=agentcore_account_filter, + finserv_nav=finserv_nav, + finserv_filter_option=finserv_filter_option, + finserv_service_card=finserv_service_card, + finserv_section=finserv_section, account_risk_section=account_risk_section, ) diff --git a/aiml-security-assessment/functions/security/generate_consolidated_report/test_generate_report.py b/aiml-security-assessment/functions/security/generate_consolidated_report/test_generate_report.py index a155817..01baaa1 100644 --- a/aiml-security-assessment/functions/security/generate_consolidated_report/test_generate_report.py +++ b/aiml-security-assessment/functions/security/generate_consolidated_report/test_generate_report.py @@ -1,10 +1,10 @@ # test_generate_report.py import unittest import os -import webbrowser from app import generate_html_report from report_template import generate_html_report as generate_report_direct + class TestHtmlReportGeneration(unittest.TestCase): def setUp(self): self.test_dir = "test_reports" @@ -24,7 +24,7 @@ def setUp(self): "Resolution": "Implement IAM policies to restrict access to specific principals and use resource-based policies for model invocations.", "Reference": "https://docs.aws.amazon.com/bedrock/latest/userguide/security_iam_id-based-policy-examples.html", "Severity": "High", - "Status": "Failed" + "Status": "Failed", }, { "Account_ID": "123456789012", @@ -34,7 +34,7 @@ def setUp(self): "Resolution": "Enable CloudTrail logging for Bedrock API actions and configure log retention policies.", "Reference": "https://docs.aws.amazon.com/bedrock/latest/userguide/logging-using-cloudtrail.html", "Severity": "Medium", - "Status": "Failed" + "Status": "Failed", }, { "Account_ID": "123456789012", @@ -44,8 +44,8 @@ def setUp(self): "Resolution": "No action required", "Reference": "https://docs.aws.amazon.com/bedrock/latest/userguide/guardrails.html", "Severity": "Informational", - "Status": "Passed" - } + "Status": "Passed", + }, ] }, "sagemaker": { @@ -58,7 +58,7 @@ def setUp(self): "Resolution": "Enable AWS KMS encryption for SageMaker endpoints using customer managed keys.", "Reference": "https://docs.aws.amazon.com/sagemaker/latest/dg/encryption-at-rest.html", "Severity": "High", - "Status": "Failed" + "Status": "Failed", }, { "Account_ID": "123456789012", @@ -68,7 +68,7 @@ def setUp(self): "Resolution": "Enable network isolation for SageMaker training jobs and use VPC configurations.", "Reference": "https://docs.aws.amazon.com/sagemaker/latest/dg/mkt-algo-model-internet-free.html", "Severity": "Medium", - "Status": "Failed" + "Status": "Failed", }, { "Account_ID": "123456789012", @@ -78,8 +78,8 @@ def setUp(self): "Resolution": "Review and restrict IAM role permissions to only necessary actions and resources.", "Reference": "https://docs.aws.amazon.com/sagemaker/latest/dg/security_iam_id-based-policy-examples.html", "Severity": "High", - "Status": "Failed" - } + "Status": "Failed", + }, ] }, "agentcore": { @@ -92,10 +92,10 @@ def setUp(self): "Resolution": "No action required", "Reference": "https://docs.aws.amazon.com/singlesignon/latest/userguide/what-is.html", "Severity": "Informational", - "Status": "Passed" + "Status": "Passed", } ] - } + }, } def test_generate_viewable_report(self): @@ -117,7 +117,7 @@ def test_generate_viewable_report(self): self.assertTrue(os.path.getsize(report_path) > 0) # Basic content checks - with open(report_path, 'r') as f: + with open(report_path, "r") as f: content = f.read() # Bedrock findings self.assertIn("Bedrock Model Access Control", content) @@ -137,7 +137,7 @@ def test_generate_viewable_report(self): # New design elements self.assertIn("sidebar", content) - self.assertIn("service-badge", content) + self.assertIn("service-icon", content) self.assertIn("theme-toggle", content) # Verify new features from consolidation @@ -149,27 +149,57 @@ def test_generate_multi_account_report(self): """Test multi-account report generation using shared template directly""" # Create test data in multi-account format all_findings = [ - {'account_id': '111122223333', 'check_id': 'BR-01', 'finding': 'Test Finding 1', 'details': 'Details 1', 'resolution': 'Fix it', 'reference': 'https://example.com', 'severity': 'High', 'status': 'Failed', '_service': 'bedrock'}, - {'account_id': '444455556666', 'check_id': 'SM-01', 'finding': 'Test Finding 2', 'details': 'Details 2', 'resolution': 'Fix it', 'reference': 'https://example.com', 'severity': 'Medium', 'status': 'Failed', '_service': 'sagemaker'}, - {'account_id': '111122223333', 'check_id': 'AC-01', 'finding': 'Test Finding 3', 'details': 'Details 3', 'resolution': 'N/A', 'reference': 'https://example.com', 'severity': 'Low', 'status': 'Passed', '_service': 'agentcore'}, + { + "account_id": "111122223333", + "check_id": "BR-01", + "finding": "Test Finding 1", + "details": "Details 1", + "resolution": "Fix it", + "reference": "https://example.com", + "severity": "High", + "status": "Failed", + "_service": "bedrock", + }, + { + "account_id": "444455556666", + "check_id": "SM-01", + "finding": "Test Finding 2", + "details": "Details 2", + "resolution": "Fix it", + "reference": "https://example.com", + "severity": "Medium", + "status": "Failed", + "_service": "sagemaker", + }, + { + "account_id": "111122223333", + "check_id": "AC-01", + "finding": "Test Finding 3", + "details": "Details 3", + "resolution": "N/A", + "reference": "https://example.com", + "severity": "Low", + "status": "Passed", + "_service": "agentcore", + }, ] service_findings = { - 'bedrock': [all_findings[0]], - 'sagemaker': [all_findings[1]], - 'agentcore': [all_findings[2]] + "bedrock": [all_findings[0]], + "sagemaker": [all_findings[1]], + "agentcore": [all_findings[2]], } service_stats = { - 'bedrock': {'passed': 0, 'failed': 1}, - 'sagemaker': {'passed': 0, 'failed': 1}, - 'agentcore': {'passed': 1, 'failed': 0} + "bedrock": {"passed": 0, "failed": 1}, + "sagemaker": {"passed": 0, "failed": 1}, + "agentcore": {"passed": 1, "failed": 0}, } html_content = generate_report_direct( all_findings=all_findings, service_findings=service_findings, service_stats=service_stats, - mode='multi', - account_ids=['111122223333', '444455556666'] + mode="multi", + account_ids=["111122223333", "444455556666"], ) report_path = os.path.join(self.test_dir, "multi_account_report.html") @@ -180,7 +210,7 @@ def test_generate_multi_account_report(self): self.assertTrue(os.path.exists(report_path)) - with open(report_path, 'r') as f: + with open(report_path, "r") as f: content = f.read() # Multi-account specific self.assertIn("Multi-Account", content) @@ -194,13 +224,12 @@ def test_missing_data_fields(self): incomplete_data = { "account_id": "123456789012", "bedrock": { - "bedrock_report": [{ - "Finding": "Incomplete Bedrock Finding", - "Severity": "High" - }] + "bedrock_report": [ + {"Finding": "Incomplete Bedrock Finding", "Severity": "High"} + ] }, "sagemaker": {}, - "agentcore": {} + "agentcore": {}, } html_content = generate_html_report(incomplete_data) @@ -222,7 +251,7 @@ def test_empty_findings(self): "account_id": "123456789012", "bedrock": {}, "sagemaker": {}, - "agentcore": {} + "agentcore": {}, } html_content = generate_html_report(empty_data) @@ -233,9 +262,52 @@ def test_empty_findings(self): print(f"\nEmpty data report generated at: {os.path.abspath(report_path)}") self.assertTrue(os.path.exists(report_path)) + def test_finserv_renders_when_present(self): + """REQ-1: FinServ findings render as a first-class service in the HTML.""" + data = dict(self.test_assessment_results) + data["finserv"] = { + "finserv_security_report": [ + { + "Account_ID": "123456789012", + "Check_ID": "FS-01", + "Finding": "No Regional WAF Web ACLs Found", + "Finding_Details": "No WAF.", + "Resolution": "Add WAF.", + "Reference": "https://docs.aws.amazon.com/waf/latest/developerguide/waf-chapter.html", + "Severity": "Medium", + "Status": "Failed", + }, + { + "Account_ID": "123456789012", + "Check_ID": "FS-44", + "Finding": "Amazon Macie Enabled", + "Finding_Details": "Macie on.", + "Resolution": "None.", + "Reference": "https://docs.aws.amazon.com/macie/latest/user/what-is-macie.html", + "Severity": "High", + "Status": "Passed", + }, + ] + } + html = generate_html_report(data) + self.assertIn('id="finserv"', html) + self.assertIn('id="finservTable"', html) + self.assertIn('