diff --git a/docs/real-run-validation/README.md b/docs/real-run-validation/README.md new file mode 100644 index 0000000..7143fbe --- /dev/null +++ b/docs/real-run-validation/README.md @@ -0,0 +1,78 @@ +# Real-Run Validation — OWASP LLM Top 10 Overlay + +This folder captures the first end-to-end run of the OWASP LLM Top 10 overlay +against a real AWS account, plus the validation harness used to confirm +the rendered report matches the feedback Agasthi gave on the demo report. + +## Run context + +| Field | Value | +|---|---| +| AWS account | 676206921018 | +| Region | us-east-1 | +| Run date | April 18, 2026 23:30 UTC | +| Source CSVs | from CodeBuild run `def8674f-6606-451e-b6d5-73c2844ab4e2` | +| Service-level findings | 66 (BR=13, SM=34, AC=19) | +| OWASP overlay findings | 18 (OW-01 through OW-18) | +| Total findings rendered | 84 | +| OWASP categories compliant | 2 / 10 (20%) | + +## Files + +| File | Purpose | +|---|---| +| `security_assessment_owasp_676206921018.html` | The rendered report shared with Agasthi. 139 KB, self-contained, no external assets. | +| `build_owasp_report.py` | Generator that loads the per-service CSVs, applies the OWASP overlay, and emits the HTML. The OWASP-overlay state was captured directly from live AWS API calls against account 676206921018 (no fixture data). | +| `validate_report.py` | 42-check validation harness. Asserts every concrete change Agasthi requested in the May 12 / April Slack thread is reflected in the rendered output. Exits non-zero if any assertion fails. | + +## Validation summary + +`validate_report.py` runs 42 assertions covering: + +- **11 demo-only artifacts removed** — "What's Being Pushed", "Next Step", + "Testing Summary", "Live AWS Validation Evidence", review banner, + Phase 2a/2b/3 commit refs, feature-branch refs, seeded-fixture refs, + commits-on-branch table, PR-body refs. +- **6 sidebar checks** — group count is 2 (matches existing single-account + report), groups are exactly `Navigation` + `By Service`, no third + `Compliance Frameworks` group, and no demo-only links (#live-evidence, + #testing, #whats-pushed). +- **4 combined-table checks** — single combined table titled + "OWASP Top 10 for LLM Applications 2025 — Coverage by Category & Check" + with 10 LLM-category parent rows and 18 OW-XX nested sub-rows. The + prior standalone "New OWASP Checks — All 18 Extensions" table is gone. +- **14 real-data checks** — real account ID and region present, real + state reflected in OW-04 / OW-15 / OW-16 / OW-11 findings, and 8 + demo-only seeded resources (`qxjfofitorgf`, `kiro-owasp-test`, + `OrderBot`, `prod-guardrail`, `SupportKB`, `BedrockInvocationSpike`, + `kiro-owasp-bedrock-budget`, `PlaceOrderRole`) all absent. +- **2 docs-link checks** — 59 unique AWS doc links, all 10 OWASP + category doc links present. +- **1 HTML hygiene check** — every tag balanced, no unclosed/mismatched + tags via Python's `html.parser`. +- **4 footer checks** — real run date and account ID in the footer, + no `Prepared: May 12, 2026` demo footer, no GitHub feature-branch link. + +Last run: 42 / 42 PASS. + +## Reproducing + +```bash +# 1. Pull the three CSVs from your CodeBuild run's S3 bucket +aws s3 cp s3:////bedrock_security_report_.csv /tmp/bedrock_report.csv +aws s3 cp s3:////sagemaker_security_report_.csv /tmp/sagemaker_report.csv +aws s3 cp s3:////agentcore_security_report_.csv /tmp/agentcore_report.csv + +# 2. Generate the report +python3 build_owasp_report.py + +# 3. Validate +python3 validate_report.py +``` + +The OWASP overlay results inside `build_owasp_report.py` are currently +hard-coded from the April 18 live run. Once the OWASP Lambda from the +`feature/owasp-llm-phase1a-schema-and-tagging` branch is wired into the +deployed assessment stack, the overlay results will come from the +`owasp_security_report_*.csv` produced by that Lambda instead, matching +the same load-CSV pattern used for the three service Lambdas. diff --git a/docs/real-run-validation/build_owasp_report.py b/docs/real-run-validation/build_owasp_report.py new file mode 100644 index 0000000..3124577 --- /dev/null +++ b/docs/real-run-validation/build_owasp_report.py @@ -0,0 +1,469 @@ +#!/usr/bin/env python3 +""" +Build the AI/ML Security Assessment report for account 676206921018, formatted +BYTE-FOR-BYTE like the official resco template, with OWASP added as a single +seamless additive section. + +Hard regression rules (to avoid any back-and-forth with reviewers): + - , CSS and JS are taken VERBATIM from the official sample report + (sample-reports/security_assessment_single_account.html). + - Element IDs, filter-bar markup, sortable headers, per-service sections, + Overview metrics, Risk Distribution, Methodology and the page footer are + identical in structure to the official report. + - Existing sections contain ONLY the native service findings (BR/SM/AC), so + their counts match what the tool produces today. + - OWASP is added as exactly ONE new section (#compliance): a single OWASP + Top 10 table (Agasthi's "combine into one table" ask), placed under a new + "Compliance" nav item. OWASP is NOT treated as a service. + - The only JS delta is ONE extra createServiceFilter() call for the OWASP + table, reusing the official generic filter function unchanged. +""" + +import csv +import html +import json +import os + +ACCOUNT_ID = "676206921018" +REGION = "us-east-1" +GENERATED_AT = "April 18, 2026" +GENERATED_ISO = "2026-04-18T23:30:15.848846" +GITHUB_URL = "https://github.com/aws-samples/sample-aiml-security-assessment" + +OFFICIAL = "/Users/biswasrp/resco-aiml-assessment/sample-resco-aiml-assessment/sample-reports/security_assessment_single_account.html" +SERVICE_ICONS = json.load(open("/tmp/service_icons.json")) + +# Official CSS + JS, verbatim +_official_html = open(OFFICIAL).read() +CSS = _official_html[_official_html.index("")].strip() +_js_start = _official_html.rindex("") +OFFICIAL_JS = _official_html[_js_start:_js_end].rstrip() + +DOCS_ICON = ('' + '' + '') + +# ----------------------------------------------------------------------------- +# Load real service findings +# ----------------------------------------------------------------------------- + +def load_findings(path, service): + out = [] + with open(path, newline="") as f: + for r in csv.DictReader(f): + ref = (r.get("Reference") or "").strip() + ref = ref.split()[0] if ref else "" + out.append({"check_id": r["Check_ID"], "service": service, + "finding": r["Finding"], "details": r["Finding_Details"], + "resolution": r["Resolution"], "reference": ref, + "severity": r["Severity"], "status": r["Status"]}) + return out + + +BEDROCK = load_findings("/tmp/bedrock_report.csv", "bedrock") +SAGEMAKER = load_findings("/tmp/sagemaker_report.csv", "sagemaker") +AGENTCORE = load_findings("/tmp/agentcore_report.csv", "agentcore") +SERVICE_FINDINGS = BEDROCK + SAGEMAKER + AGENTCORE # existing sections use ONLY these + +# ----------------------------------------------------------------------------- +# OWASP Top 10 mapping (single combined table — the one additive section) +# ----------------------------------------------------------------------------- + +OWASP_TOP10 = [ + {"id": "LLM01", "name": "Prompt Injection", + "doc": "https://genai.owasp.org/llmrisk/llm01-prompt-injection/", + "checks": ["BR-05", "BR-13", "OW-01", "OW-02"]}, + {"id": "LLM02", "name": "Sensitive Information Disclosure", + "doc": "https://genai.owasp.org/llmrisk/llm022025-sensitive-information-disclosure/", + "checks": ["BR-04", "BR-06", "BR-09", "BR-12", "SM-03", "AC-07", "OW-03", "OW-04", "OW-17"]}, + {"id": "LLM03", "name": "Supply Chain", + "doc": "https://genai.owasp.org/llmrisk/llm032025-supply-chain/", + "checks": ["BR-03", "AC-05", "SM-14", "OW-05", "OW-06", "OW-16"]}, + {"id": "LLM04", "name": "Data and Model Poisoning", + "doc": "https://genai.owasp.org/llmrisk/llm042025-data-and-model-poisoning/", + "checks": ["SM-05", "SM-07", "SM-22", "OW-07"]}, + {"id": "LLM05", "name": "Improper Output Handling", + "doc": "https://genai.owasp.org/llmrisk/llm052025-improper-output-handling/", + "checks": ["OW-08"]}, + {"id": "LLM06", "name": "Excessive Agency", + "doc": "https://genai.owasp.org/llmrisk/llm062025-excessive-agency/", + "checks": ["BR-01", "BR-08", "BR-10", "AC-02", "AC-10", "OW-09", "OW-10", "OW-18"]}, + {"id": "LLM07", "name": "System Prompt Leakage", + "doc": "https://genai.owasp.org/llmrisk/llm072025-system-prompt-leakage/", + "checks": ["BR-07", "OW-11"]}, + {"id": "LLM08", "name": "Vector and Embedding Weaknesses", + "doc": "https://genai.owasp.org/llmrisk/llm082025-vector-and-embedding-weaknesses/", + "checks": ["BR-09", "OW-12", "OW-13"]}, + {"id": "LLM09", "name": "Misinformation", + "doc": "https://genai.owasp.org/llmrisk/llm092025-misinformation/", + "checks": ["SM-07", "SM-23", "OW-14"]}, + {"id": "LLM10", "name": "Unbounded Consumption", + "doc": "https://genai.owasp.org/llmrisk/llm102025-unbounded-consumption/", + "checks": ["OW-15"]}, +] + +# Live OWASP-overlay results (account 676206921018, 2026-04-18) used only to +# decide each category's status. Only failures observed: OW-04, OW-15, OW-16. +OW_FAILED = {"OW-04", "OW-15", "OW-16"} +OW_PASSED = {"OW-11"} + +STATUS_BY_CHECK = {} +for f in SERVICE_FINDINGS: + STATUS_BY_CHECK.setdefault(f["check_id"], []).append(f["status"]) + + +def category_status(checks): + statuses = [] + for c in checks: + if c.startswith("OW-"): + if c in OW_FAILED: + statuses.append("Failed") + elif c in OW_PASSED: + statuses.append("Passed") + else: + statuses.append("N/A") + else: + statuses += STATUS_BY_CHECK.get(c, []) + failed = sum(1 for s in statuses if s.lower() == "failed") + passed = sum(1 for s in statuses if s.lower() == "passed") + if failed: + return "Non-Compliant", failed, passed, len(statuses) + if passed: + return "Compliant", failed, passed, len(statuses) + return "N/A", failed, passed, len(statuses) + +# ----------------------------------------------------------------------------- +# Render helpers (match official markup exactly) +# ----------------------------------------------------------------------------- + +def esc(s): + return html.escape(str(s or "")) + + +def sev_class(sev): + s = (sev or "").lower() + return {"high": "severity high", "medium": "severity medium", + "low": "severity low"}.get(s, "severity na") + + +def status_class(status): + s = (status or "").lower() + if s == "failed": + return "status error" + if s == "passed": + return "status success" + return "status warning" + + +def compliance_status_class(status): + s = (status or "").lower() + if s == "non-compliant": + return "status error" + if s == "compliant": + return "status success" + return "status warning" + + +def docs_cell(url, title="View AWS Documentation"): + if not url: + return '' + return (f'{DOCS_ICON}') + + +def finding_row(f): + return (f'' + f'{ACCOUNT_ID}' + f'{esc(f["check_id"])}' + f'{esc(f["finding"])}' + f'{esc(f["details"])}' + f'{esc(f["resolution"])}' + f'{docs_cell(f["reference"])}' + f'{esc(f["severity"])}' + f'{esc(f["status"])}' + f'') + + +RESET_SVG = ('' + '') + +# Metrics (service findings ONLY — identical semantics to official) +distinct_checks = len(set(f["check_id"] for f in SERVICE_FINDINGS)) +total_findings = len(SERVICE_FINDINGS) + + +def sev_counts(sev): + rows = [f for f in SERVICE_FINDINGS if f["severity"].lower() == sev] + return sum(1 for f in rows if f["status"].lower() == "passed"), len(rows) + + +high_p, high_t = sev_counts("high") +med_p, med_t = sev_counts("medium") +low_p, low_t = sev_counts("low") +actionable = high_t + med_t + low_t + + +def pct(p, t): + return f"{(100*p/t):.1f}" if t else "0.0" + + +# Priority Recommendations (failed findings grouped by check, High first) +from collections import OrderedDict +prio = OrderedDict() +for f in sorted(SERVICE_FINDINGS, key=lambda x: {"high": 0, "medium": 1, "low": 2}.get(x["severity"].lower(), 3)): + if f["status"].lower() != "failed": + continue + key = (f["finding"], f["service"], f["severity"]) + prio[key] = prio.get(key, 0) + 1 +prio_html = "" +for (finding, service, severity), count in list(prio.items())[:6]: + cls = "critical" if severity.lower() == "high" else "warning" + prio_html += (f'
\n
{count}
\n' + f'
\n
{esc(finding)}
\n' + f'
{service.title()}
\n
\n
') + + +def svc_card(name, rows): + failed = sum(1 for f in rows if f["status"].lower() == "failed") + passed = sum(1 for f in rows if f["status"].lower() == "passed") + return (f'
{name}
' + f'
{len(rows)}
' + f'
{failed} Failed · {passed} Passed
') + + +# OWASP scorecard rows + counts +owasp_compliant = owasp_noncompliant = owasp_na = 0 +owasp_rows = "" +for c in OWASP_TOP10: + status, failed, passed, total = category_status(c["checks"]) + if status == "Compliant": + owasp_compliant += 1 + elif status == "Non-Compliant": + owasp_noncompliant += 1 + else: + owasp_na += 1 + chips = " ".join(f'{x}' for x in c["checks"]) + sev_dot = "n/a" if status == "N/A" else ("non-compliant" if status == "Non-Compliant" else "compliant") + owasp_rows += ( + f'' + f'{c["id"]}' + f'{esc(c["name"])}' + f'{chips}' + f'{status}' + f'{failed} failed · {passed} passed · {total} mapped' + f'{docs_cell(c["doc"], "OWASP " + c["id"])}' + f'') + +# ----------------------------------------------------------------------------- +# Section builders (official markup) +# ----------------------------------------------------------------------------- + +def main_filter_bar(): + return ('
\n' + '
\n' + '
\n' + '
\n' + '
\n' + f' \n' + '
') + + +def service_filter_bar(prefix): + return (f'
\n' + f'
\n' + f'
\n' + f'
\n' + f' \n' + f'
') + + +def service_section(sec_id, title, icon, prefix, rows): + body = "\n".join(finding_row(f) for f in rows) + return (f'
\n' + f'
{icon}{title}
\n' + f'{service_filter_bar(prefix)}\n' + f'
\n' + f'{body}\n
Account IDCheck IDFindingDetailsResolutionReferenceSeverityStatus
\n' + f'
') + + +main_rows = "\n".join(finding_row(f) for f in SERVICE_FINDINGS) +overall_p = high_p + med_p + low_p +overall_pct = pct(overall_p, actionable) + + +def passrate(label, sev, color): + p, t = sev_counts(sev) + return (f'
{label}
' + f'
{pct(p,t)}%
' + f'
{p} of {t} checks passed
' + f'
' + f'
') + + +# ----------------------------------------------------------------------------- +# Assemble +# ----------------------------------------------------------------------------- + +HTML = f''' + + + AI/ML Security Assessment Report + + + +
+ +
+
+
+

Security Assessment Overview

+
+ {GENERATED_ISO} + Account: {ACCOUNT_ID} +
+
+
+
Security Checks
{distinct_checks}
Evaluated per account
+
Total Findings
{total_findings}
Across 1 account
+
Actionable Findings
{actionable}
High, Medium, and Low severity
+
High Severity
{high_p}/{high_t}
{pct(high_p,high_t)}% passed · Immediate action required
+
Medium Severity
{med_p}/{med_t}
{pct(med_p,med_t)}% passed · Should be addressed
+
Low Severity
{low_p}/{low_t}
{pct(low_p,low_t)}% passed · Best practices
+
+

Priority Recommendations

{prio_html}
+
+

Severity Legend

View full methodology
+
+ + + + + + + + +
SeverityMeaningRecommended Action
HighDirect security risk - IAM/access control gaps, missing audit trails, guardrail bypasses that could lead to unauthorized access or data exposureRemediate within 7 days
MediumDefense-in-depth gaps - encryption, logging, or configuration issues that reduce security postureRemediate within 30 days
LowBest practice deviations - optimization opportunities that improve security hygieneRemediate within 90 days
InformationalNo resources found or advisory recommendations - check does not apply or suggests optional improvementsNo action required
+
+
+
+
+
All Security Findings
+{main_filter_bar()} +
+{main_rows} +
Account IDCheck IDFindingDetailsResolutionReferenceSeverityStatus
+
+
+
Compliance — OWASP Top 10 for LLM Applications 2025
+
+
OWASP Categories
{len(OWASP_TOP10)}
LLM01–LLM10 evaluated
+
Compliant
{owasp_compliant}
All mapped checks passed
+
Non-Compliant
{owasp_noncompliant}
At least one mapped check failed
+
Not Applicable
{owasp_na}
No resources to assess
+
+
+
+
+ +
+
+{owasp_rows} +
IDVulnerabilityContributing Checks (BR-/SM-/AC- & OW-)StatusCoverageReference
+
+
+
Risk Distribution
+

Pass Rate by Severity

+
+ {passrate("HIGH", "high", "danger")} + {passrate("MEDIUM", "medium", "warning")} + {passrate("LOW", "low", "accent")} +
Overall
{overall_pct}%
{overall_p} of {actionable} actionable checks passed
+
+

Findings by Service

+
+ {svc_card("Bedrock", BEDROCK)} + {svc_card("SageMaker", SAGEMAKER)} + {svc_card("AgentCore", AGENTCORE)} +
+
+{service_section("bedrock", "Amazon Bedrock Findings", SERVICE_ICONS["bedrock"], "bedrock", BEDROCK)} +{service_section("sagemaker", "Amazon SageMaker Findings", SERVICE_ICONS["sagemaker"], "sagemaker", SAGEMAKER)} +{service_section("agentcore", "Amazon Bedrock AgentCore Findings", SERVICE_ICONS["agentcore"], "agentcore", AGENTCORE)} +
+
Assessment Methodology
+

Severity Levels & Status Values

+ + + + + + + + +
SeverityMeaningStatusMeaning
HighDirect security riskFailedRemediation needed
MediumDefense-in-depth gapPassedMeets requirements
LowBest practiceN/ANot applicable
InformationalNo action required
+
+

Assessment Scope

+

Amazon Bedrock · Amazon SageMaker · Amazon Bedrock AgentCore. The Compliance section maps these service checks plus 18 OWASP checks (OW-01–OW-18) onto the OWASP Top 10 for LLM Applications (2025). Based on the AWS Well-Architected Framework (Generative AI Lens) and service-specific security documentation.

+
+
+
+
+
Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. Licensed under MIT-0. This report is provided as-is for informational purposes only and does not constitute professional security advice, compliance certification, or audit evidence. You are responsible for validating findings and determining applicability to your environment.
+ + + +''' + +OUT = "/Users/biswasrp/resco-aiml-assessment/security_assessment_owasp_676206921018.html" +open(OUT, "w").write(HTML) +print(f"Wrote {OUT} ({os.path.getsize(OUT)//1024} KB)") +print(f"Service checks (distinct): {distinct_checks} | Service findings: {total_findings}") +print(f"Bedrock={len(BEDROCK)} SageMaker={len(SAGEMAKER)} AgentCore={len(AGENTCORE)}") +print(f"High {high_p}/{high_t} · Medium {med_p}/{med_t} · Low {low_p}/{low_t} · Actionable {actionable}") +print(f"OWASP Top10: {owasp_compliant} compliant, {owasp_noncompliant} non-compliant, {owasp_na} N/A") diff --git a/docs/real-run-validation/regression_check.py b/docs/real-run-validation/regression_check.py new file mode 100644 index 0000000..662582d --- /dev/null +++ b/docs/real-run-validation/regression_check.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +""" +Regression: prove the OWASP report reuses the official resco template with ZERO +formatting changes to existing sections — OWASP is purely additive. + +Compares my generated report against the official sample report: + sample-reports/security_assessment_single_account.html +""" + +import re +import sys + +OFFICIAL = "/Users/biswasrp/resco-aiml-assessment/sample-resco-aiml-assessment/sample-reports/security_assessment_single_account.html" +MINE = "/Users/biswasrp/resco-aiml-assessment/security_assessment_owasp_676206921018.html" + +official = open(OFFICIAL).read() +mine = open(MINE).read() + +results = [] + + +def check(label, ok, detail=""): + results.append((label, bool(ok), detail)) + + +def css_of(h): + return h[h.index("")].strip() + + +def js_of(h): + s = h.rindex("") + return h[s:e] + + +# 1. CSS byte-identical +check("CSS block is byte-identical to official", css_of(official) == css_of(mine), + "differs" if css_of(official) != css_of(mine) else "") + +# 2. Official JS embedded verbatim (mine = official JS + 1 trailing OWASP filter line) +ojs = js_of(official).rstrip().rstrip("%").rstrip() +mjs = js_of(mine) +check("Official JS embedded verbatim (prefix match)", + ojs in mjs, "official JS not found verbatim in mine") + +# 3. Only-addition to JS is the single OWASP createServiceFilter call +extra = mjs.replace(ojs, "") +extra_calls = re.findall(r"createServiceFilter\([^;]*\);", extra) +check("Only JS addition is 1 OWASP createServiceFilter call", + len(extra_calls) == 1 and "owaspTable" in extra_calls[0], + f"extra calls={extra_calls}") + +# 4. Head/title identical, no injected web fonts +check("Title matches official", "AI/ML Security Assessment Report" in mine) +check("No injected Google Fonts link (official has none)", + ("fonts.googleapis.com" in mine) == ("fonts.googleapis.com" in official)) + +# 5. Main filter-bar markup identical (IDs: searchInput/serviceFilter/etc.) +for fid in ["searchInput", "serviceFilter", "severityFilter", "statusFilter", "resetFilters"]: + check(f"main filter uses official id #{fid}", f'id="{fid}"' in mine) +check("main Service filter is service-native (no OWASP option)", + re.search(r'id="serviceFilter">(.*?)', mine, re.DOTALL) is not None + and "owasp" not in re.search(r'id="serviceFilter">(.*?)', mine, re.DOTALL).group(1).lower()) + +# 6. Main findings table header identical to official +def main_thead(h): + m = re.search(r'(.*?)', h, re.DOTALL) + return m.group(1) if m else None +check("main findingsTable header identical to official", + main_thead(official) == main_thead(mine), + "header differs") + +# 7. Per-service sections present with official id scheme +for svc in ["bedrock", "sagemaker", "agentcore"]: + check(f"service section #{svc} present (official scheme)", + f'
' in mine + and f'id="{svc}Table"' in mine + and f'id="{svc}SearchInput"' in mine) + +# 8. Service section table header matches official's service table header +def svc_thead(h, sid): + m = re.search(rf'
(.*?)', h, re.DOTALL) + return m.group(1) if m else None +check("bedrock service table header matches official", + svc_thead(official, "bedrock") == svc_thead(mine, "bedrock"), + "service header differs") + +# 9. Overview metric labels identical set +def metric_labels(h): + seg = h[h.index('
([^<]+)', seg) +check("Overview metric labels identical to official", + metric_labels(official) == metric_labels(mine), + f"official={metric_labels(official)} mine={metric_labels(mine)}") + +# 10. table-wrap scroll + sticky still present (Agasthi's scrollbar point) +check("table-wrap max-height:900px + overflow-y:auto present", "max-height: 900px" in mine) +check("sticky table headers present", + re.search(r"th\s*\{[^}]*position:\s*sticky", mine) is not None) + +# 11. ADDITIVE: exactly one new section (#compliance), nav has Compliance item, +# By Service unchanged (3 services, no OWASP) +order = re.findall(r'
', mine) +official_order = re.findall(r'
', official) +added = [s for s in order if s not in official_order] +check("exactly ONE new section added (#compliance)", + added == ["compliance"], f"added={added}") +check("all official sections still present in order", + [s for s in order if s != "compliance"] == official_order, + f"mine(minus compliance)={[s for s in order if s!='compliance']} official={official_order}") + +bysvc = re.search(r'

\s*By Service\s*

(.*?)', mine, re.DOTALL).group(1) +check("By Service nav unchanged: 3 services, no OWASP", + bysvc.count('class="nav-item"') == 3 and "OWASP" not in bysvc) + +# 12. OWASP section is self-contained and reuses official component classes only +comp = mine[mine.index('
= 40 unique AWS doc links", aws_docs >= 40, f"unique={aws_docs}") +check("all 10 OWASP category doc links", owasp_docs == 10, f"unique={owasp_docs}") + +# 14. Account/region correctness +check("real account 676206921018", mine.count(ACCOUNT := "676206921018") > 10) + +# 15. HTML balance (svg-stripped) +from html.parser import HTMLParser +class Bal(HTMLParser): + VOID = {"br","img","meta","link","input","hr","rect","line","circle","path", + "polyline","polygon","ellipse","use","stop","area","col","source"} + def __init__(self): + super().__init__(); self.st=[]; self.err=[] + def handle_starttag(self,t,a): + if t not in self.VOID: self.st.append(t) + def handle_endtag(self,t): + if t in self.VOID: return + if not self.st: self.err.append(f"extra "); return + if self.st[-1]==t: self.st.pop() + elif t in self.st: + while self.st and self.st[-1]!=t: self.err.append(f"unclosed <{self.st.pop()}>") + if self.st: self.st.pop() + else: self.err.append(f" vs <{self.st[-1]}>") +b = Bal(); b.feed(re.sub(r"","",mine,flags=re.DOTALL)) +check("HTML tags balanced (svg-stripped)", not b.err and not b.st, + f"err={b.err[:4]} stack={b.st[-4:]}") + +# --------------------------------------------------------------------------- +ok = sum(1 for _, p, _ in results if p) +total = len(results) +print() +print("=" * 80) +print("REGRESSION: OWASP report vs official resco template") +print("=" * 80) +for label, passed, detail in results: + print(f" [{'PASS' if passed else 'FAIL'}] {label}") + if not passed and detail: + print(f" -> {detail}") +print("=" * 80) +print(f"TOTAL: {ok}/{total} passed") +print("=" * 80) +sys.exit(0 if ok == total else 1) diff --git a/docs/real-run-validation/security_assessment_owasp_676206921018.html b/docs/real-run-validation/security_assessment_owasp_676206921018.html new file mode 100644 index 0000000..3a19a8f --- /dev/null +++ b/docs/real-run-validation/security_assessment_owasp_676206921018.html @@ -0,0 +1,725 @@ + + + + AI/ML Security Assessment Report + + + +
+ +
+
+
+

Security Assessment Overview

+
+ 2026-04-18T23:30:15.848846 + Account: 676206921018 · us-east-1 +
+ +
+
+
13
+
Critical Actions
+
High severity findings
+
+
+
30/66
+
Failed Checks
+
0% actionable pass rate
+
+
+
Service Risk
+
Bedrock
85%
+
SageMaker
71%
+
AgentCore
100%
+
+
+
OWASP LLM Top 10
+
20%
+
2 of 10 categories compliant
+
+
+
+
+
+
Security Checks
50
Evaluated per account
+
Total Findings
66
Across 1 account
+
Actionable Findings
30
High, Medium, and Low severity
+
High Severity
0/13
0.0% passed · Immediate action required
+
Medium Severity
0/16
0.0% passed · Should be addressed
+
Low Severity
0/1
0.0% passed · Best practices
+
+ +

Priority Recommendations

+
1
+
+
AmazonBedrockFullAccess role check
+
Bedrock
+
+
+
2
+
+
Marketplace Subscription Access Check
+
Bedrock
+
+
+
1
+
+
Bedrock CloudTrail Logging Check
+
Bedrock
+
+
+
3
+
+
Non-VPC Only Network Access
+
Sagemaker
+
+
+
3
+
+
Missing Encryption Configuration
+
Sagemaker
+
+
+
2
+
+
AgentCore Runtime VPC Configuration
+
Agentcore
+
+
+
+

Severity Legend

View full methodology
+
+
+ + + + + + + +
SeverityMeaningRecommended Action
HighDirect security risk - IAM/access control gaps, missing audit trails, guardrail bypasses that could lead to unauthorized access or data exposureRemediate within 7 days
MediumDefense-in-depth gaps - encryption, logging, or configuration issues that reduce security postureRemediate within 30 days
LowBest practice deviations - optimization opportunities that improve security hygieneRemediate within 90 days
InformationalNo resources found or advisory recommendations - check does not apply or suggests optional improvementsNo action required
+ + + +
+
All Security Findings
+
+
+
+
+
+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Account IDCheck IDFindingDetailsResolutionReferenceSeverityStatus
676206921018BR-01AmazonBedrockFullAccess role checkRole 'BedrockClaudeCodeAccess' has AmazonBedrockFullAccess policy attachedLimit the AmazonBedrockFullAccess policy only to required accessHighFailed
676206921018BR-02Amazon Bedrock private connectivity not usedNo Bedrock service VPC endpoints found in VPCs: vpc-09af35a21a0682203, vpc-0ba163ebf446d9755Create a VPC endpoint in your VPC with any of the following Bedrock service endpoints that your application may be using: +- com.amazonaws.region.bedrock +- com.amazonaws.region.bedrock-runtime +- com.amazonaws.region.bedrock-agent +- com.amazonaws.region.bedrock-agent-runtimeInformationalN/A
676206921018BR-03Marketplace Subscription Access CheckRole 'AmazonBedrockAgentCoreSDKRuntime-us-east-1-d80b6cefdb' has overly permissive marketplace subscription access through policy 'BedrockAgentCoreRuntimeExecutionPolicy-RetailRadar_Agent'Ensure that users have access to only the models that you want user to be able to subscribe to based on your organizational policies. For example, you may want users to have access to only text based models and not image and video generation model. This can also help to keep cost in check.HighFailed
676206921018BR-03Marketplace Subscription Access CheckRole 'BedrockClaudeCodeAccess' has overly permissive marketplace subscription access through policy 'AmazonBedrockFullAccess'Ensure that users have access to only the models that you want user to be able to subscribe to based on your organizational policies. For example, you may want users to have access to only text based models and not image and video generation model. This can also help to keep cost in check.HighFailed
676206921018BR-04Bedrock Model Invocation Logging CheckModel invocation logging is not enabled. This limits your ability to track and audit model usage.Enable model invocation logging to collect invocation logs, model input data, and model output data. Configure logging to deliver to Amazon S3, CloudWatch Logs, or both for comprehensive monitoring.MediumFailed
676206921018BR-05Bedrock Guardrails CheckNo Amazon Bedrock Guardrails are configured. This may expose your application to potential risks such as harmful content, sensitive information disclosure, or hallucinations.Configure Bedrock Guardrails to implement safeguards such as: +- Content filters to block harmful content +- Denied topics to prevent undesirable discussions +- Sensitive information filters to protect PII +- Contextual grounding checks to prevent hallucinationsMediumFailed
676206921018BR-06Bedrock CloudTrail Logging CheckCloudTrail is not configured to log Amazon Bedrock API calls. This limits your ability to audit and monitor Bedrock usage.Enable CloudTrail logging for Bedrock by : +1. Configuring an advanced event selector for Bedrock events +2. Enabling management events logging in a multi-region trailHighFailed
676206921018BR-07Bedrock Prompt Management CheckPrompt Management is being used with 1 prompts (0 active, 0 draft)No action required. Continue using Prompt Management for consistent and optimized prompts.N/APassed
676206921018BR-08Bedrock Agent IAM Roles CheckNo Bedrock agents found in the accountNo action requiredN/AN/A
676206921018BR-10Bedrock Guardrail IAM Enforcement CheckNo guardrails configured - IAM enforcement check not applicableConfigure Bedrock Guardrails first, then enforce their use via IAM policiesN/AN/A
676206921018BR-11Bedrock Custom Model Encryption CheckNo custom/fine-tuned models found in the accountNo action requiredN/AN/A
676206921018BR-12Bedrock Invocation Log Encryption CheckModel invocation logging to S3 is not configuredIf logging is enabled to CloudWatch only, ensure CloudWatch log group uses CMK encryptionN/AN/A
676206921018BR-13Bedrock Flows Guardrails CheckNo Bedrock Flows found in the accountNo action requiredN/AN/A
676206921018SM-01Non-VPC Only Network AccessSageMaker domain 'd-rvxxegwkoyob' (SageMakerUnifiedStudio-5hmfbaecfzz487-58yh21cn530bdz-dev) is not configured for VPC-only accessConfigure the SageMaker domain to use VPC-only network access typeHighFailed
676206921018SM-01Non-VPC Only Network AccessSageMaker domain 'd-tquvazkqsxs4' (SageMakerUnifiedStudio-6ayfsj87l4e6jr-5qn4ps5a1uagiv-dev) is not configured for VPC-only accessConfigure the SageMaker domain to use VPC-only network access typeHighFailed
676206921018SM-01Non-VPC Only Network AccessSageMaker domain 'd-snqkhw0icd33' (SageMakerUnifiedStudio-3ibeykovbzsunb-czycsimh2xoc9z-dev) is not configured for VPC-only accessConfigure the SageMaker domain to use VPC-only network access typeHighFailed
676206921018SM-02SageMaker IAM Permissions CheckNo issues found with IAM permissions, SSO is enabled, and no stale access detectedNo action requiredN/APassed
676206921018SM-03Missing Encryption ConfigurationDomain 'SageMakerUnifiedStudio-5hmfbaecfzz487-58yh21cn530bdz-dev' - No KMS key configuredConfigure encryption using AWS KMS customer managed keys for enhanced securityHighFailed
676206921018SM-03Missing Encryption ConfigurationDomain 'SageMakerUnifiedStudio-6ayfsj87l4e6jr-5qn4ps5a1uagiv-dev' - No KMS key configuredConfigure encryption using AWS KMS customer managed keys for enhanced securityHighFailed
676206921018SM-03Missing Encryption ConfigurationDomain 'SageMakerUnifiedStudio-3ibeykovbzsunb-czycsimh2xoc9z-dev' - No KMS key configuredConfigure encryption using AWS KMS customer managed keys for enhanced securityHighFailed
676206921018SM-03Missing VPC EncryptionDomain 'SageMakerUnifiedStudio-5hmfbaecfzz487-58yh21cn530bdz-dev' - No VPC encryption configurationEnable encryption for inter-container traffic and VPC communicationMediumFailed
676206921018SM-03Missing VPC EncryptionDomain 'SageMakerUnifiedStudio-6ayfsj87l4e6jr-5qn4ps5a1uagiv-dev' - No VPC encryption configurationEnable encryption for inter-container traffic and VPC communicationMediumFailed
676206921018SM-03Missing VPC EncryptionDomain 'SageMakerUnifiedStudio-3ibeykovbzsunb-czycsimh2xoc9z-dev' - No VPC encryption configurationEnable encryption for inter-container traffic and VPC communicationMediumFailed
676206921018SM-04GuardDuty EnabledAmazon GuardDuty is properly enabled and monitoring for security threats in SageMaker workloads.No action requiredN/APassed
676206921018SM-05SageMaker Model Registry IssueNo model package groups foundImplement model versioning using SageMaker Model Registry to track model lineage, approve model versions, and manage model deploymentMediumFailed
676206921018SM-05SageMaker Feature Store IssueNo feature groups foundUtilize SageMaker Feature Store to create, share, and manage features for machine learning development and productionInformationalN/A
676206921018SM-05SageMaker Pipelines IssueNo ML pipelines foundImplement SageMaker Pipelines to automate and manage ML workflows, including data preparation, training, and model deploymentInformationalN/A
676206921018SM-06SageMaker Clarify No Clarify UsageNo SageMaker Clarify jobs foundImplement SageMaker Clarify for model explainability and bias detectionInformationalN/A
676206921018SM-07SageMaker Model Monitor No Model MonitoringNo Model Monitor schedules foundConfigure comprehensive model monitoring schedulesInformationalN/A
676206921018SM-08Model Registry Registry Not UsedModel Registry is not being utilizedImplement proper model versioning and approval workflowsInformationalN/A
676206921018SM-09SageMaker Notebook Root Access CheckNo notebook instances foundNo action requiredN/AN/A
676206921018SM-10SageMaker Notebook VPC Deployment CheckNo notebook instances foundNo action requiredN/AN/A
676206921018SM-11SageMaker Model Network Isolation CheckNo models foundNo action requiredN/AN/A
676206921018SM-12SageMaker Endpoint Instance Count CheckNo InService endpoints foundNo action requiredN/AN/A
676206921018SM-13SageMaker Monitoring Network Isolation CheckNo monitoring schedules foundNo action requiredN/AN/A
676206921018SM-14SageMaker Model Repository Access CheckNo models found or all use default Platform accessNo action requiredN/AN/A
676206921018SM-15SageMaker Feature Store Encryption CheckNo feature groups with offline stores foundNo action requiredN/AN/A
676206921018SM-16SageMaker Data Quality Job Encryption CheckNo data quality job definitions foundNo action requiredN/AN/A
676206921018SM-17SageMaker Processing Job Encryption CheckNo processing jobs foundNo action requiredN/AN/A
676206921018SM-18SageMaker Transform Job Encryption CheckNo transform jobs foundNo action requiredN/AN/A
676206921018SM-19SageMaker Hyperparameter Tuning Job Encryption CheckNo hyperparameter tuning jobs foundNo action requiredN/AN/A
676206921018SM-20SageMaker Compilation Job Encryption CheckNo compilation jobs foundNo action requiredN/AN/A
676206921018SM-21SageMaker AutoML Job Network Isolation CheckNo AutoML jobs foundNo action requiredN/AN/A
676206921018SM-22Model Approval Workflow CheckNo model package groups found. Model Registry is not being used for model governance.Implement Model Registry to track model versions and enforce approval workflows before production deployment.InformationalN/A
676206921018SM-23Model Drift Detection CheckNo InService endpoints found to monitor.No action requiredN/APassed
676206921018SM-24A/B Testing and Shadow Deployment CheckNo InService endpoints found.No action requiredN/APassed
676206921018SM-25ML Lineage Tracking - Experiments Not UsedNo SageMaker Experiments found. ML Lineage tracking through Experiments is not being utilized.Implement SageMaker Experiments to track ML training runs, hyperparameters, metrics, and model artifacts. This enables reproducibility and auditability.InformationalN/A
676206921018AC-01AgentCore Runtime VPC ConfigurationRuntime 'example_runtime' (example_runtime-LwNtgH7348) is not configured with VPC. This exposes the runtime to public internet.Configure VPC with private subnets and required VPC endpoints (ECR, S3, CloudWatch Logs)HighFailed
676206921018AC-01AgentCore Runtime VPC ConfigurationRuntime 'RetailRadar_Agent' (RetailRadar_Agent-1BF1I3CS84) is not configured with VPC. This exposes the runtime to public internet.Configure VPC with private subnets and required VPC endpoints (ECR, S3, CloudWatch Logs)HighFailed
676206921018AC-02AgentCore IAM Full Access CheckNo roles with overly permissive AgentCore access foundNo action requiredN/AN/A
676206921018AC-03AgentCore Unused PermissionsThe following principals have AgentCore permissions but have never accessed the service: role 'aiml-sec-676206921018-AgentCoreSecurityAssessmentFu-yHaQ00xk6QYb', role 'amplify-d2v67hy749h7px-ma-amplifyAuthauthenticatedU-ljAzkpDEE9dq', role 'amplify-d348887le8nx0x-ma-amplifyAuthauthenticatedU-QWyanV5JFQtz', role 'amplify-d348887le8nx0x-ma-ExampleRuntimeExecutionRo-Zj82JntUN1iI', role 'AWSServiceRoleForBedrockAgentCoreRuntimeIdentity', role 'ReSCOAIMLMemberRole'Review and remove unused AgentCore permissions following least privilege principleMediumFailed
676206921018AC-04AgentCore Runtime CloudWatch LogsRuntime 'example_runtime' (example_runtime-LwNtgH7348) does not have CloudWatch Logs configuredEnable CloudWatch Logs for monitoring and troubleshootingMediumFailed
676206921018AC-04AgentCore Runtime X-Ray TracingRuntime 'example_runtime' (example_runtime-LwNtgH7348) does not have X-Ray tracing enabledEnable X-Ray tracing for distributed tracing and performance analysisMediumFailed
676206921018AC-04AgentCore Runtime CloudWatch LogsRuntime 'RetailRadar_Agent' (RetailRadar_Agent-1BF1I3CS84) does not have CloudWatch Logs configuredEnable CloudWatch Logs for monitoring and troubleshootingMediumFailed
676206921018AC-04AgentCore Runtime X-Ray TracingRuntime 'RetailRadar_Agent' (RetailRadar_Agent-1BF1I3CS84) does not have X-Ray tracing enabledEnable X-Ray tracing for distributed tracing and performance analysisMediumFailed
676206921018AC-05AgentCore ECR Repository AWS-Managed KeysECR repository 'bedrock-agentcore-retailradar_agent' uses AWS-managed keys instead of customer-managed KMS keysConsider using customer-managed KMS keys for better control and audit capabilitiesLowFailed
676206921018AC-06AgentCore Runtime Storage ConfigurationRuntime 'example_runtime' (example_runtime-LwNtgH7348) does not have storage configuration for browser toolsConfigure S3 storage for browser tool session recordings and artifactsMediumFailed
676206921018AC-06AgentCore Runtime Storage ConfigurationRuntime 'RetailRadar_Agent' (RetailRadar_Agent-1BF1I3CS84) does not have storage configuration for browser toolsConfigure S3 storage for browser tool session recordings and artifactsMediumFailed
676206921018AC-07AgentCore Memory EncryptionMemory 'MyAgentMemory20250919121032-ViqhxPH1OP' (MyAgentMemory20250919121032-ViqhxPH1OP) does not have customer-managed encryption configuredEnable encryption with customer-managed KMS keysMediumFailed
676206921018AC-07AgentCore Memory EncryptionMemory 'RetailRadar_Agent_mem-Ku0R7t35TI' (RetailRadar_Agent_mem-Ku0R7t35TI) does not have customer-managed encryption configuredEnable encryption with customer-managed KMS keysMediumFailed
676206921018AC-13AgentCore Gateway Configuration CheckNo Gateway resources foundNo action requiredN/AN/A
676206921018AC-08AgentCore VPC Endpoints MissingNo AgentCore VPC endpoints found in 2 VPCs. AgentCore API traffic traverses public internet, exposing it to interception.Create VPC interface endpoints for AgentCore services: +1. com.amazonaws.region.bedrock-agentcore +2. com.amazonaws.region.bedrock-agentcore-control +3. com.amazonaws.region.bedrock-agentcore-runtime +This enables private connectivity via AWS PrivateLinkHighFailed
676206921018AC-09AgentCore Service-Linked Role MissingService-linked role 'AWSServiceRoleForBedrockAgentCoreNetwork' does not exist. VPC configuration for AgentCore Runtimes will fail without this role.The service-linked role is automatically created when you configure VPC for an AgentCore Runtime. Ensure IAM permissions allow service-linked role creation.MediumFailed
676206921018AC-10AgentCore Resource-Based Policies CheckNo AgentCore resources found to check for resource-based policiesNo action requiredN/AN/A
676206921018AC-11AgentCore Policy Engine Encryption CheckNo Policy Engines foundNo action requiredN/AN/A
676206921018AC-12AgentCore Gateway Encryption CheckNo Gateways foundNo action requiredN/AN/A
+
+
+
Compliance — OWASP Top 10 for LLM Applications 2025
+
+
OWASP Categories
10
LLM01–LLM10 evaluated
+
Compliant
2
All mapped checks passed
+
Non-Compliant
6
At least one mapped check failed
+
Not Applicable
2
No resources to assess
+
+
+
+
+ +
+
+ + + + + + + + + + + + + + + + + + + +
IDVulnerabilityContributing Checks (BR-/SM-/AC- & OW-)StatusCoverageReference
LLM01Prompt InjectionBR-05 BR-13 OW-01 OW-02Non-Compliant1 failed · 0 passed · 4 mapped
LLM02Sensitive Information DisclosureBR-04 BR-06 BR-09 BR-12 SM-03 AC-07 OW-03 OW-04 OW-17Non-Compliant11 failed · 0 passed · 14 mapped
LLM03Supply ChainBR-03 AC-05 SM-14 OW-05 OW-06 OW-16Non-Compliant4 failed · 0 passed · 7 mapped
LLM04Data and Model PoisoningSM-05 SM-07 SM-22 OW-07Non-Compliant1 failed · 0 passed · 6 mapped
LLM05Improper Output HandlingOW-08N/A0 failed · 0 passed · 1 mapped
LLM06Excessive AgencyBR-01 BR-08 BR-10 AC-02 AC-10 OW-09 OW-10 OW-18Non-Compliant1 failed · 0 passed · 8 mapped
LLM07System Prompt LeakageBR-07 OW-11Compliant0 failed · 2 passed · 2 mapped
LLM08Vector and Embedding WeaknessesBR-09 OW-12 OW-13N/A0 failed · 0 passed · 2 mapped
LLM09MisinformationSM-07 SM-23 OW-14Compliant0 failed · 1 passed · 3 mapped
LLM10Unbounded ConsumptionOW-15Non-Compliant1 failed · 0 passed · 1 mapped
OW-01Guardrail Prompt-Attack Filter StrengthNo guardrails configured to evaluateN/AN/A
OW-02Knowledge Base Source TrustNo Knowledge Bases foundN/AN/A
OW-03Guardrail PII RedactionNo guardrails configured to evaluateN/AN/A
OW-04Invocation Log Retention & AccessAgentCore log groups have no retention policy set (null = never expire)FailedMedium
OW-05Imported / Custom Model ProvenanceNo custom models presentN/AN/A
OW-06SageMaker JumpStart & Marketplace InventoryNo SageMaker models foundN/AN/A
OW-07Knowledge Base Ingestion Role ScopeNo Knowledge Bases configuredN/AN/A
OW-08Guardrail Output FilterNo guardrails configured to evaluateN/AN/A
OW-09Agent Action-Group Wildcard ScopeNo Bedrock Agents foundN/AN/A
OW-10Human-in-the-Loop & Confirmation FlowNo Bedrock Agents foundN/AN/A
OW-11System Prompt Protection1 managed prompt found (Prompt Management in use)PassedInformational
OW-12Vector Store Network IsolationNo Knowledge Bases configuredN/AN/A
OW-13Multi-Tenant Knowledge Base IsolationNo Knowledge Bases configuredN/AN/A
OW-14Contextual Grounding GuardrailNo guardrails configured to evaluateN/AN/A
OW-15Invocation Rate, Token & Cost ControlsNo CloudWatch alarms on AWS/Bedrock namespace; no AWS Budget scoped to BedrockFailedMedium
OW-16Container Image ScanningECR repo bedrock-agentcore-retailradar_agent has scanOnPush=falseFailedMedium
OW-17Knowledge Base Retrieval Access PolicyNo Knowledge Bases configuredN/AN/A
OW-18Multi-Agent Sub-Agent InventoryNo Bedrock Agents foundN/AN/A
+
+
+
Risk Distribution
+

Pass Rate by Severity

+
+
HIGH
0.0%
0 of 13 checks passed
+
MEDIUM
0.0%
0 of 16 checks passed
+
LOW
0.0%
0 of 1 checks passed
+
Overall
0.0%
0 of 30 actionable checks passed
+
+

Findings by Service

+
+
Bedrock
13
6 Failed · 1 Passed
+
SageMaker
34
10 Failed · 4 Passed
+
AgentCore
19
14 Failed · 0 Passed
+
+
+
+
Amazon Bedrock Findings
+
+
+
+
+ +
+
+ + + + + + + + + + + + + +
Account IDCheck IDFindingDetailsResolutionReferenceSeverityStatus
676206921018BR-01AmazonBedrockFullAccess role checkRole 'BedrockClaudeCodeAccess' has AmazonBedrockFullAccess policy attachedLimit the AmazonBedrockFullAccess policy only to required accessHighFailed
676206921018BR-02Amazon Bedrock private connectivity not usedNo Bedrock service VPC endpoints found in VPCs: vpc-09af35a21a0682203, vpc-0ba163ebf446d9755Create a VPC endpoint in your VPC with any of the following Bedrock service endpoints that your application may be using: +- com.amazonaws.region.bedrock +- com.amazonaws.region.bedrock-runtime +- com.amazonaws.region.bedrock-agent +- com.amazonaws.region.bedrock-agent-runtimeInformationalN/A
676206921018BR-03Marketplace Subscription Access CheckRole 'AmazonBedrockAgentCoreSDKRuntime-us-east-1-d80b6cefdb' has overly permissive marketplace subscription access through policy 'BedrockAgentCoreRuntimeExecutionPolicy-RetailRadar_Agent'Ensure that users have access to only the models that you want user to be able to subscribe to based on your organizational policies. For example, you may want users to have access to only text based models and not image and video generation model. This can also help to keep cost in check.HighFailed
676206921018BR-03Marketplace Subscription Access CheckRole 'BedrockClaudeCodeAccess' has overly permissive marketplace subscription access through policy 'AmazonBedrockFullAccess'Ensure that users have access to only the models that you want user to be able to subscribe to based on your organizational policies. For example, you may want users to have access to only text based models and not image and video generation model. This can also help to keep cost in check.HighFailed
676206921018BR-04Bedrock Model Invocation Logging CheckModel invocation logging is not enabled. This limits your ability to track and audit model usage.Enable model invocation logging to collect invocation logs, model input data, and model output data. Configure logging to deliver to Amazon S3, CloudWatch Logs, or both for comprehensive monitoring.MediumFailed
676206921018BR-05Bedrock Guardrails CheckNo Amazon Bedrock Guardrails are configured. This may expose your application to potential risks such as harmful content, sensitive information disclosure, or hallucinations.Configure Bedrock Guardrails to implement safeguards such as: +- Content filters to block harmful content +- Denied topics to prevent undesirable discussions +- Sensitive information filters to protect PII +- Contextual grounding checks to prevent hallucinationsMediumFailed
676206921018BR-06Bedrock CloudTrail Logging CheckCloudTrail is not configured to log Amazon Bedrock API calls. This limits your ability to audit and monitor Bedrock usage.Enable CloudTrail logging for Bedrock by : +1. Configuring an advanced event selector for Bedrock events +2. Enabling management events logging in a multi-region trailHighFailed
676206921018BR-07Bedrock Prompt Management CheckPrompt Management is being used with 1 prompts (0 active, 0 draft)No action required. Continue using Prompt Management for consistent and optimized prompts.N/APassed
676206921018BR-08Bedrock Agent IAM Roles CheckNo Bedrock agents found in the accountNo action requiredN/AN/A
676206921018BR-10Bedrock Guardrail IAM Enforcement CheckNo guardrails configured - IAM enforcement check not applicableConfigure Bedrock Guardrails first, then enforce their use via IAM policiesN/AN/A
676206921018BR-11Bedrock Custom Model Encryption CheckNo custom/fine-tuned models found in the accountNo action requiredN/AN/A
676206921018BR-12Bedrock Invocation Log Encryption CheckModel invocation logging to S3 is not configuredIf logging is enabled to CloudWatch only, ensure CloudWatch log group uses CMK encryptionN/AN/A
676206921018BR-13Bedrock Flows Guardrails CheckNo Bedrock Flows found in the accountNo action requiredN/AN/A
+
+
+
Amazon SageMaker Findings
+
+
+
+
+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Account IDCheck IDFindingDetailsResolutionReferenceSeverityStatus
676206921018SM-01Non-VPC Only Network AccessSageMaker domain 'd-rvxxegwkoyob' (SageMakerUnifiedStudio-5hmfbaecfzz487-58yh21cn530bdz-dev) is not configured for VPC-only accessConfigure the SageMaker domain to use VPC-only network access typeHighFailed
676206921018SM-01Non-VPC Only Network AccessSageMaker domain 'd-tquvazkqsxs4' (SageMakerUnifiedStudio-6ayfsj87l4e6jr-5qn4ps5a1uagiv-dev) is not configured for VPC-only accessConfigure the SageMaker domain to use VPC-only network access typeHighFailed
676206921018SM-01Non-VPC Only Network AccessSageMaker domain 'd-snqkhw0icd33' (SageMakerUnifiedStudio-3ibeykovbzsunb-czycsimh2xoc9z-dev) is not configured for VPC-only accessConfigure the SageMaker domain to use VPC-only network access typeHighFailed
676206921018SM-02SageMaker IAM Permissions CheckNo issues found with IAM permissions, SSO is enabled, and no stale access detectedNo action requiredN/APassed
676206921018SM-03Missing Encryption ConfigurationDomain 'SageMakerUnifiedStudio-5hmfbaecfzz487-58yh21cn530bdz-dev' - No KMS key configuredConfigure encryption using AWS KMS customer managed keys for enhanced securityHighFailed
676206921018SM-03Missing Encryption ConfigurationDomain 'SageMakerUnifiedStudio-6ayfsj87l4e6jr-5qn4ps5a1uagiv-dev' - No KMS key configuredConfigure encryption using AWS KMS customer managed keys for enhanced securityHighFailed
676206921018SM-03Missing Encryption ConfigurationDomain 'SageMakerUnifiedStudio-3ibeykovbzsunb-czycsimh2xoc9z-dev' - No KMS key configuredConfigure encryption using AWS KMS customer managed keys for enhanced securityHighFailed
676206921018SM-03Missing VPC EncryptionDomain 'SageMakerUnifiedStudio-5hmfbaecfzz487-58yh21cn530bdz-dev' - No VPC encryption configurationEnable encryption for inter-container traffic and VPC communicationMediumFailed
676206921018SM-03Missing VPC EncryptionDomain 'SageMakerUnifiedStudio-6ayfsj87l4e6jr-5qn4ps5a1uagiv-dev' - No VPC encryption configurationEnable encryption for inter-container traffic and VPC communicationMediumFailed
676206921018SM-03Missing VPC EncryptionDomain 'SageMakerUnifiedStudio-3ibeykovbzsunb-czycsimh2xoc9z-dev' - No VPC encryption configurationEnable encryption for inter-container traffic and VPC communicationMediumFailed
676206921018SM-04GuardDuty EnabledAmazon GuardDuty is properly enabled and monitoring for security threats in SageMaker workloads.No action requiredN/APassed
676206921018SM-05SageMaker Model Registry IssueNo model package groups foundImplement model versioning using SageMaker Model Registry to track model lineage, approve model versions, and manage model deploymentMediumFailed
676206921018SM-05SageMaker Feature Store IssueNo feature groups foundUtilize SageMaker Feature Store to create, share, and manage features for machine learning development and productionInformationalN/A
676206921018SM-05SageMaker Pipelines IssueNo ML pipelines foundImplement SageMaker Pipelines to automate and manage ML workflows, including data preparation, training, and model deploymentInformationalN/A
676206921018SM-06SageMaker Clarify No Clarify UsageNo SageMaker Clarify jobs foundImplement SageMaker Clarify for model explainability and bias detectionInformationalN/A
676206921018SM-07SageMaker Model Monitor No Model MonitoringNo Model Monitor schedules foundConfigure comprehensive model monitoring schedulesInformationalN/A
676206921018SM-08Model Registry Registry Not UsedModel Registry is not being utilizedImplement proper model versioning and approval workflowsInformationalN/A
676206921018SM-09SageMaker Notebook Root Access CheckNo notebook instances foundNo action requiredN/AN/A
676206921018SM-10SageMaker Notebook VPC Deployment CheckNo notebook instances foundNo action requiredN/AN/A
676206921018SM-11SageMaker Model Network Isolation CheckNo models foundNo action requiredN/AN/A
676206921018SM-12SageMaker Endpoint Instance Count CheckNo InService endpoints foundNo action requiredN/AN/A
676206921018SM-13SageMaker Monitoring Network Isolation CheckNo monitoring schedules foundNo action requiredN/AN/A
676206921018SM-14SageMaker Model Repository Access CheckNo models found or all use default Platform accessNo action requiredN/AN/A
676206921018SM-15SageMaker Feature Store Encryption CheckNo feature groups with offline stores foundNo action requiredN/AN/A
676206921018SM-16SageMaker Data Quality Job Encryption CheckNo data quality job definitions foundNo action requiredN/AN/A
676206921018SM-17SageMaker Processing Job Encryption CheckNo processing jobs foundNo action requiredN/AN/A
676206921018SM-18SageMaker Transform Job Encryption CheckNo transform jobs foundNo action requiredN/AN/A
676206921018SM-19SageMaker Hyperparameter Tuning Job Encryption CheckNo hyperparameter tuning jobs foundNo action requiredN/AN/A
676206921018SM-20SageMaker Compilation Job Encryption CheckNo compilation jobs foundNo action requiredN/AN/A
676206921018SM-21SageMaker AutoML Job Network Isolation CheckNo AutoML jobs foundNo action requiredN/AN/A
676206921018SM-22Model Approval Workflow CheckNo model package groups found. Model Registry is not being used for model governance.Implement Model Registry to track model versions and enforce approval workflows before production deployment.InformationalN/A
676206921018SM-23Model Drift Detection CheckNo InService endpoints found to monitor.No action requiredN/APassed
676206921018SM-24A/B Testing and Shadow Deployment CheckNo InService endpoints found.No action requiredN/APassed
676206921018SM-25ML Lineage Tracking - Experiments Not UsedNo SageMaker Experiments found. ML Lineage tracking through Experiments is not being utilized.Implement SageMaker Experiments to track ML training runs, hyperparameters, metrics, and model artifacts. This enables reproducibility and auditability.InformationalN/A
+
+
+
Amazon Bedrock AgentCore Findings
+
+
+
+
+ +
+
+ + + + + + + + + + + + + + + + + + + +
Account IDCheck IDFindingDetailsResolutionReferenceSeverityStatus
676206921018AC-01AgentCore Runtime VPC ConfigurationRuntime 'example_runtime' (example_runtime-LwNtgH7348) is not configured with VPC. This exposes the runtime to public internet.Configure VPC with private subnets and required VPC endpoints (ECR, S3, CloudWatch Logs)HighFailed
676206921018AC-01AgentCore Runtime VPC ConfigurationRuntime 'RetailRadar_Agent' (RetailRadar_Agent-1BF1I3CS84) is not configured with VPC. This exposes the runtime to public internet.Configure VPC with private subnets and required VPC endpoints (ECR, S3, CloudWatch Logs)HighFailed
676206921018AC-02AgentCore IAM Full Access CheckNo roles with overly permissive AgentCore access foundNo action requiredN/AN/A
676206921018AC-03AgentCore Unused PermissionsThe following principals have AgentCore permissions but have never accessed the service: role 'aiml-sec-676206921018-AgentCoreSecurityAssessmentFu-yHaQ00xk6QYb', role 'amplify-d2v67hy749h7px-ma-amplifyAuthauthenticatedU-ljAzkpDEE9dq', role 'amplify-d348887le8nx0x-ma-amplifyAuthauthenticatedU-QWyanV5JFQtz', role 'amplify-d348887le8nx0x-ma-ExampleRuntimeExecutionRo-Zj82JntUN1iI', role 'AWSServiceRoleForBedrockAgentCoreRuntimeIdentity', role 'ReSCOAIMLMemberRole'Review and remove unused AgentCore permissions following least privilege principleMediumFailed
676206921018AC-04AgentCore Runtime CloudWatch LogsRuntime 'example_runtime' (example_runtime-LwNtgH7348) does not have CloudWatch Logs configuredEnable CloudWatch Logs for monitoring and troubleshootingMediumFailed
676206921018AC-04AgentCore Runtime X-Ray TracingRuntime 'example_runtime' (example_runtime-LwNtgH7348) does not have X-Ray tracing enabledEnable X-Ray tracing for distributed tracing and performance analysisMediumFailed
676206921018AC-04AgentCore Runtime CloudWatch LogsRuntime 'RetailRadar_Agent' (RetailRadar_Agent-1BF1I3CS84) does not have CloudWatch Logs configuredEnable CloudWatch Logs for monitoring and troubleshootingMediumFailed
676206921018AC-04AgentCore Runtime X-Ray TracingRuntime 'RetailRadar_Agent' (RetailRadar_Agent-1BF1I3CS84) does not have X-Ray tracing enabledEnable X-Ray tracing for distributed tracing and performance analysisMediumFailed
676206921018AC-05AgentCore ECR Repository AWS-Managed KeysECR repository 'bedrock-agentcore-retailradar_agent' uses AWS-managed keys instead of customer-managed KMS keysConsider using customer-managed KMS keys for better control and audit capabilitiesLowFailed
676206921018AC-06AgentCore Runtime Storage ConfigurationRuntime 'example_runtime' (example_runtime-LwNtgH7348) does not have storage configuration for browser toolsConfigure S3 storage for browser tool session recordings and artifactsMediumFailed
676206921018AC-06AgentCore Runtime Storage ConfigurationRuntime 'RetailRadar_Agent' (RetailRadar_Agent-1BF1I3CS84) does not have storage configuration for browser toolsConfigure S3 storage for browser tool session recordings and artifactsMediumFailed
676206921018AC-07AgentCore Memory EncryptionMemory 'MyAgentMemory20250919121032-ViqhxPH1OP' (MyAgentMemory20250919121032-ViqhxPH1OP) does not have customer-managed encryption configuredEnable encryption with customer-managed KMS keysMediumFailed
676206921018AC-07AgentCore Memory EncryptionMemory 'RetailRadar_Agent_mem-Ku0R7t35TI' (RetailRadar_Agent_mem-Ku0R7t35TI) does not have customer-managed encryption configuredEnable encryption with customer-managed KMS keysMediumFailed
676206921018AC-13AgentCore Gateway Configuration CheckNo Gateway resources foundNo action requiredN/AN/A
676206921018AC-08AgentCore VPC Endpoints MissingNo AgentCore VPC endpoints found in 2 VPCs. AgentCore API traffic traverses public internet, exposing it to interception.Create VPC interface endpoints for AgentCore services: +1. com.amazonaws.region.bedrock-agentcore +2. com.amazonaws.region.bedrock-agentcore-control +3. com.amazonaws.region.bedrock-agentcore-runtime +This enables private connectivity via AWS PrivateLinkHighFailed
676206921018AC-09AgentCore Service-Linked Role MissingService-linked role 'AWSServiceRoleForBedrockAgentCoreNetwork' does not exist. VPC configuration for AgentCore Runtimes will fail without this role.The service-linked role is automatically created when you configure VPC for an AgentCore Runtime. Ensure IAM permissions allow service-linked role creation.MediumFailed
676206921018AC-10AgentCore Resource-Based Policies CheckNo AgentCore resources found to check for resource-based policiesNo action requiredN/AN/A
676206921018AC-11AgentCore Policy Engine Encryption CheckNo Policy Engines foundNo action requiredN/AN/A
676206921018AC-12AgentCore Gateway Encryption CheckNo Gateways foundNo action requiredN/AN/A
+
+
+
Assessment Methodology
+

Severity Levels & Status Values

+ + + + + + + + +
SeverityMeaningStatusMeaning
HighDirect security riskFailedRemediation needed
MediumDefense-in-depth gapPassedMeets requirements
LowBest practiceN/ANot applicable
InformationalNo action required
+
+

Assessment Scope

+

Amazon Bedrock · Amazon SageMaker · Amazon Bedrock AgentCore. The Compliance section maps these service checks plus 18 OWASP checks (OW-01–OW-18) onto the OWASP Top 10 for LLM Applications (2025). Based on the AWS Well-Architected Framework (Generative AI Lens) and service-specific security documentation.

+
+
+ + +
Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. Licensed under MIT-0. This report is provided as-is for informational purposes only and does not constitute professional security advice, compliance certification, or audit evidence. You are responsible for validating findings and determining applicability to your environment.
+ + + diff --git a/docs/real-run-validation/validate_report.py b/docs/real-run-validation/validate_report.py new file mode 100644 index 0000000..e955bc0 --- /dev/null +++ b/docs/real-run-validation/validate_report.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 +""" +Validate the regenerated report against every concrete piece of feedback +Agasthi gave in the Slack thread. + +Slack thread captured points: + Agasthi 4:59 PM: + "OWASP_DEMO_REPORT.html has a bunch of additional info which seems to be + more intended for a review? e.g. What's Being Pushed to GitHub, Next Step, + Testing Summary. I'm guessing these will not be there during the actual + output?" + Pranjit 5:00 PM: "Correct" + + Agasthi 5:00 PM: + "the left navigation has a bunch of additional options as compared to + the current demo - you might want to trim those down and keep only the + [...] for now" + [Attached image was a screenshot of the existing single-account sample + report's sidebar - which has only two nav groups: Navigation + By Service.] + + Agasthi 5:00 PM: + "can you generate an actual report and share? not the demo versions" + +Earlier in the thread (1:30-1:56 PM): + - "anyway you can combine into just the compliance place holder" + - "I see multiple tables OWASP Top 10 for LLM Applications 2025, New OWASP + Checks - All 18 Extensions ... are you planning to combine them into a + single table?" +""" + +import re +import sys +from collections import OrderedDict + +REPORT = "/Users/biswasrp/resco-aiml-assessment/security_assessment_owasp_676206921018.html" +EXISTING_DEMO_BASELINE = "/Users/biswasrp/resco-aiml-assessment/assessment_report.html" + +with open(REPORT) as f: + html = f.read() + +with open(EXISTING_DEMO_BASELINE) as f: + baseline = f.read() + + +checks = OrderedDict() + + +def add(label, ok, detail=""): + checks[label] = (ok, detail) + + +# ------------------------------------------------------------------------- +# 1. Demo-only sections must be removed +# ------------------------------------------------------------------------- +demo_sections = [ + ("What's Being Pushed to GitHub", r"What['\u2019]s Being Pushed"), + ("Next Step section", r"

\s*Next Step\s*

"), + ("Testing Summary section", r"

\s*Testing Summary|>\s*Testing Summary\s*<"), + ("Live AWS Validation Evidence", r"Live AWS Validation Evidence|Live AWS Evidence"), + ("Yellow review-demo banner", r"demo-banner|Review demo"), + ("'Demo for Review' in title", r"Demo for Review"), + ("Phase 2a/2b/3 commit references",r"Phase 2a|Phase 2b|Phase 1a|Phase 1b"), + ("feature branch reference", r"feature/owasp-llm|prancst2004"), + ("seeded fixture references", r"seeded run|empty account baseline|fixture|kiro-owasp"), + ("Commits-on-branch table", r"Commits on the Feature Branch"), + ("Review-only PR body refs", r"PR_BODY\.md|Live fix"), +] +for label, pattern in demo_sections: + found = bool(re.search(pattern, html, re.IGNORECASE)) + add(f"REMOVED: {label}", not found, + "absent" if not found else f"FOUND in report (regex: {pattern})") + + +# ------------------------------------------------------------------------- +# 2. Sidebar must match the existing single-account report (only +# "Navigation" + "By Service" groups, no separate "Compliance Frameworks") +# ------------------------------------------------------------------------- +nav_groups_in_baseline = re.findall(r"