diff --git a/aiml-security-assessment/functions/security/agentcore_assessments/app.py b/aiml-security-assessment/functions/security/agentcore_assessments/app.py index b06b88f..cd72a65 100644 --- a/aiml-security-assessment/functions/security/agentcore_assessments/app.py +++ b/aiml-security-assessment/functions/security/agentcore_assessments/app.py @@ -112,6 +112,70 @@ def check_timeout() -> bool: return elapsed < 540 # 9 minutes hard stop +def _agentcore_list_all( + list_method_name: str, result_keys: List[str] +) -> List[Dict[str, Any]]: + """Collect all items from an AgentCore list API, following nextToken.""" + if agentcore_client is None: + return [] + + items: List[Dict[str, Any]] = [] + next_token = None + list_method = getattr(agentcore_client, list_method_name) + + while True: + kwargs = {} + if next_token: + kwargs["nextToken"] = next_token + + response = list_method(**kwargs) + + for result_key in result_keys: + page_items = response.get(result_key) + if isinstance(page_items, list): + items.extend(page_items) + break + + next_token = response.get("nextToken") + if not next_token: + break + + return items + + +def _unwrap_agentcore_detail( + response: Dict[str, Any], wrapper_key: str +) -> Dict[str, Any]: + """Handle detail APIs that wrap the resource under a top-level key.""" + if not isinstance(response, dict): + return {} + + wrapped = response.get(wrapper_key) + if isinstance(wrapped, dict): + return wrapped + + return response + + +def _get_agentcore_resource_policy(resource_arn: str) -> str: + """Retrieve the generic AgentCore resource policy for a resource ARN.""" + response = agentcore_client.get_resource_policy(resourceArn=resource_arn) + return response.get("policy") or response.get("resourcePolicy") or "" + + +def _is_access_denied_client_error(error: ClientError) -> bool: + """Normalize access-denied checks across AgentCore control plane APIs.""" + if not isinstance(error, ClientError): + return False + + error_code = error.response.get("Error", {}).get("Code") + return error_code in { + "AccessDenied", + "AccessDeniedException", + "UnauthorizedOperation", + } + + def generate_csv_report(findings: List[Dict[str, Any]]) -> str: """ Generate CSV report from findings. @@ -237,8 +301,7 @@ def check_agentcore_vpc_configuration() -> List[Dict[str, Any]]: # Check Runtimes try: - runtimes_response = agentcore_client.list_agent_runtimes() - runtimes = runtimes_response.get("agentRuntimes", []) + runtimes = _agentcore_list_all("list_agent_runtimes", ["agentRuntimes"]) if not runtimes: logger.info("No AgentCore Runtimes found") @@ -955,8 +1018,7 @@ def check_agentcore_observability() -> List[Dict[str, Any]]: # Check Runtimes for logging and tracing try: - runtimes_response = agentcore_client.list_agent_runtimes() - runtimes = runtimes_response.get("agentRuntimes", []) + runtimes = _agentcore_list_all("list_agent_runtimes", ["agentRuntimes"]) if not runtimes: logger.info("No AgentCore Runtimes found") @@ -1237,8 +1299,7 @@ def check_browser_tool_recording() -> List[Dict[str, Any]]: # Browser Tools are part of Runtime configuration # Check if Runtimes have appropriate storage configured - runtimes_response = agentcore_client.list_agent_runtimes() - runtimes = runtimes_response.get("agentRuntimes", []) + runtimes = _agentcore_list_all("list_agent_runtimes", ["agentRuntimes"]) if not runtimes: logger.info("No AgentCore Runtimes found") @@ -1348,8 +1409,7 @@ def check_agentcore_memory_configuration() -> List[Dict[str, Any]]: try: logger.info("Checking AgentCore Memory configuration") - memories_response = agentcore_client.list_memories() - memories = memories_response.get("memories", []) + memories = _agentcore_list_all("list_memories", ["memories"]) if not memories: logger.info("No Memory resources found") @@ -1375,10 +1435,14 @@ def check_agentcore_memory_configuration() -> List[Dict[str, Any]]: ) try: - memory_details = agentcore_client.get_memory(memoryId=memory_id) + memory_details = _unwrap_agentcore_detail( + agentcore_client.get_memory(memoryId=memory_id), "memory" + ) # Check encryption configuration - encryption_key_arn = memory_details.get("encryptionKeyArn") + encryption_key_arn = memory_details.get( + "encryptionKeyArn" + ) or memory_details.get("kmsKeyArn") if not encryption_key_arn: findings.append( @@ -1652,7 +1716,6 @@ def check_agentcore_resource_based_policies() -> List[Dict[str, Any]]: Validates: - Agent Runtime resource policies - Gateway resource policies - - Memory resource policies Returns: List of findings @@ -1678,24 +1741,26 @@ def check_agentcore_resource_based_policies() -> List[Dict[str, Any]]: resources_without_rbp = [] resources_with_rbp = [] + policy_access_denied = [] + policy_check_errors = [] # Check Agent Runtimes try: - runtimes_response = agentcore_client.list_agent_runtimes() - runtimes = runtimes_response.get("agentRuntimes", []) + runtimes = _agentcore_list_all("list_agent_runtimes", ["agentRuntimes"]) for runtime in runtimes: runtime_id = runtime.get("agentRuntimeId", "unknown") runtime_name = runtime.get("agentRuntimeName", runtime_id) + runtime_arn = runtime.get("agentRuntimeArn") try: - # Try to get resource policy - policy_response = ( - agentcore_client.get_agent_runtime_resource_policy( - agentRuntimeId=runtime_id + if not runtime_arn: + resources_without_rbp.append( + {"type": "Runtime", "name": runtime_name, "id": runtime_id} ) - ) - policy = policy_response.get("resourcePolicy") + continue + + policy = _get_agentcore_resource_policy(runtime_arn) if policy: resources_with_rbp.append(f"Runtime: {runtime_name}") @@ -1709,14 +1774,24 @@ def check_agentcore_resource_based_policies() -> List[Dict[str, Any]]: resources_without_rbp.append( {"type": "Runtime", "name": runtime_name, "id": runtime_id} ) + elif _is_access_denied_client_error(e): + policy_access_denied.append( + {"type": "Runtime", "name": runtime_name, "id": runtime_id} + ) else: + policy_check_errors.append( + { + "type": "Runtime", + "name": runtime_name, + "id": runtime_id, + "error_code": e.response.get("Error", {}).get( + "Code", "Unknown" + ), + } + ) logger.warning( f"Error checking policy for runtime {runtime_id}: {e}" ) - except AttributeError: - # API method doesn't exist - logger.info("get_agent_runtime_resource_policy API not available") - break except ClientError as e: if e.response["Error"]["Code"] != "ResourceNotFoundException": @@ -1724,18 +1799,23 @@ def check_agentcore_resource_based_policies() -> List[Dict[str, Any]]: # Check Gateways try: - gateways_response = agentcore_client.list_gateways() - gateways = gateways_response.get("gateways", []) + gateways = _agentcore_list_all("list_gateways", ["items", "gateways"]) for gateway in gateways: gateway_id = gateway.get("gatewayId", "unknown") gateway_name = gateway.get("name", gateway_id) try: - policy_response = agentcore_client.get_gateway_resource_policy( - gatewayId=gateway_id - ) - policy = policy_response.get("resourcePolicy") + gateway_details = agentcore_client.get_gateway(gatewayId=gateway_id) + gateway_arn = gateway_details.get("gatewayArn") + + if not gateway_arn: + resources_without_rbp.append( + {"type": "Gateway", "name": gateway_name, "id": gateway_id} + ) + continue + + policy = _get_agentcore_resource_policy(gateway_arn) if policy: resources_with_rbp.append(f"Gateway: {gateway_name}") @@ -1749,9 +1829,24 @@ def check_agentcore_resource_based_policies() -> List[Dict[str, Any]]: resources_without_rbp.append( {"type": "Gateway", "name": gateway_name, "id": gateway_id} ) - except AttributeError: - logger.info("get_gateway_resource_policy API not available") - break + elif _is_access_denied_client_error(e): + policy_access_denied.append( + {"type": "Gateway", "name": gateway_name, "id": gateway_id} + ) + else: + policy_check_errors.append( + { + "type": "Gateway", + "name": gateway_name, + "id": gateway_id, + "error_code": e.response.get("Error", {}).get( + "Code", "Unknown" + ), + } + ) + logger.warning( + f"Error checking policy for gateway {gateway_id}: {e}" + ) except (ClientError, AttributeError) as e: logger.info(f"Gateway APIs not available: {e}") @@ -1780,6 +1875,57 @@ def check_agentcore_resource_based_policies() -> List[Dict[str, Any]]: ) ) + if policy_access_denied: + resource_list = ", ".join( + [f"{r['type']} '{r['name']}'" for r in policy_access_denied[:5]] + ) + if len(policy_access_denied) > 5: + resource_list += f" and {len(policy_access_denied) - 5} more" + + findings.append( + create_finding( + check_id="AC-10", + finding_name="AgentCore Resource-Based Policy Assessment Access Denied", + finding_details=( + f"Unable to assess resource-based policies for {resource_list} " + "because access to AgentCore resource policy metadata was denied." + ), + resolution=( + "Ensure the assessment role can call " + "bedrock-agentcore:GetResourcePolicy for AgentCore resources." + ), + reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/security_iam_service-with-iam.html", + severity=SeverityEnum.INFORMATIONAL, + status=StatusEnum.NA, + ) + ) + + if policy_check_errors: + resource_list = ", ".join( + [f"{r['type']} '{r['name']}'" for r in policy_check_errors[:5]] + ) + if len(policy_check_errors) > 5: + resource_list += f" and {len(policy_check_errors) - 5} more" + + error_codes = sorted({r["error_code"] for r in policy_check_errors}) + findings.append( + create_finding( + check_id="AC-10", + finding_name="AgentCore Resource-Based Policy Assessment Incomplete", + finding_details=( + f"Unable to fully assess resource-based policies for {resource_list} " + f"due to AgentCore API errors: {', '.join(error_codes)}." + ), + resolution=( + "Re-run the assessment. If the issue persists, review AgentCore " + "service health and the assessment role's control plane permissions." + ), + reference="https://docs.aws.amazon.com/bedrock-agentcore/latest/devguide/security_iam_service-with-iam.html", + severity=SeverityEnum.LOW, + status=StatusEnum.NA, + ) + ) + if not findings: if resources_with_rbp: findings.append( @@ -1854,8 +2000,9 @@ def check_agentcore_policy_engine_encryption() -> List[Dict[str, Any]]: try: # List policy engines - policy_engines_response = agentcore_client.list_policy_engines() - policy_engines = policy_engines_response.get("policyEngines", []) + policy_engines = _agentcore_list_all( + "list_policy_engines", ["policyEngines"] + ) if not policy_engines: findings.append( @@ -2000,8 +2147,7 @@ def check_agentcore_gateway_encryption() -> List[Dict[str, Any]]: logger.info("Checking AgentCore Gateway encryption") try: - gateways_response = agentcore_client.list_gateways() - gateways = gateways_response.get("gateways", []) + gateways = _agentcore_list_all("list_gateways", ["items", "gateways"]) if not gateways: findings.append( @@ -2148,8 +2294,7 @@ def check_agentcore_gateway_configuration() -> List[Dict[str, Any]]: # Try to list gateways - this API may not exist yet try: - gateways_response = agentcore_client.list_gateways() - gateways = gateways_response.get("gateways", []) + gateways = _agentcore_list_all("list_gateways", ["items", "gateways"]) if not gateways: logger.info("No Gateway resources found") diff --git a/aiml-security-assessment/functions/security/agentcore_assessments/schema.py b/aiml-security-assessment/functions/security/agentcore_assessments/schema.py index 229c163..6c5a9f5 100644 --- a/aiml-security-assessment/functions/security/agentcore_assessments/schema.py +++ b/aiml-security-assessment/functions/security/agentcore_assessments/schema.py @@ -1,59 +1,73 @@ from enum import Enum -from typing import Dict, List, Any -from pydantic import BaseModel, Field, HttpUrl, validator -from datetime import datetime +from typing import Dict, Any +from pydantic import BaseModel, Field, validator import re + class SeverityEnum(str, Enum): HIGH = "High" MEDIUM = "Medium" LOW = "Low" INFORMATIONAL = "Informational" + class StatusEnum(str, Enum): FAILED = "Failed" PASSED = "Passed" NA = "N/A" + class Finding(BaseModel): """Represents a security finding with required fields and validations""" - Check_ID: str = Field(..., min_length=1, description="Unique check identifier (e.g., SM-01, BR-01, AC-01)") + + Check_ID: str = Field( + ..., + min_length=1, + description="Unique check identifier (e.g., SM-01, BR-01, AC-01)", + ) Finding: str = Field(..., min_length=1, description="The name/title of the finding") - Finding_Details: str = Field(..., min_length=1, description="Detailed description of the finding") - Resolution: str = Field(..., min_length=0, description="Steps to resolve the finding") + Finding_Details: str = Field( + ..., min_length=1, description="Detailed description of the finding" + ) + Resolution: str = Field( + ..., min_length=0, description="Steps to resolve the finding" + ) Reference: str = Field(..., description="Documentation reference URL") Severity: SeverityEnum = Field(..., description="Severity level of the finding") Status: StatusEnum = Field(..., description="Current status of the finding") - @validator('Check_ID') + @validator("Check_ID") def validate_check_id(cls, v): """Validate that Check_ID follows the pattern XX-NN (e.g., SM-01, BR-14, AC-05)""" - pattern = r'^[A-Z]{2,3}-\d{2}$' + pattern = r"^[A-Z]{2,3}-\d{2}$" if not re.match(pattern, v): - raise ValueError('Check_ID must follow pattern XX-NN (e.g., SM-01, BR-14, AC-05)') + raise ValueError( + "Check_ID must follow pattern XX-NN (e.g., SM-01, BR-14, AC-05)" + ) return v - @validator('Reference') + @validator("Reference") def validate_reference_url(cls, v): """Validate that reference URL starts with https://""" - if not str(v).startswith('https://'): - raise ValueError('Reference URL must start with https://') + if not str(v).startswith("https://"): + raise ValueError("Reference URL must start with https://") return v - @validator('Severity') + @validator("Severity") def validate_severity(cls, v): """Validate that severity is one of the allowed values""" if v not in SeverityEnum.__members__.values(): - raise ValueError('Severity must be one of the allowed values') + raise ValueError("Severity must be one of the allowed values") return v - @validator('Status') + @validator("Status") def validate_status(cls, v): """Validate that status is one of the allowed values""" if v not in StatusEnum.__members__.values(): - raise ValueError('Status must be one of the allowed values') + raise ValueError("Status must be one of the allowed values") return v + def create_finding( check_id: str, finding_name: str, @@ -61,7 +75,7 @@ def create_finding( resolution: str, reference: str, severity: SeverityEnum, - status: StatusEnum + status: StatusEnum, ) -> Dict[str, Any]: """ Create a validated finding object @@ -88,6 +102,6 @@ def create_finding( Resolution=resolution, Reference=reference, Severity=severity, - Status=status + Status=status, ) return dict(finding.model_dump()) # Convert to regular dictionary diff --git a/aiml-security-assessment/functions/security/bedrock_assessments/app.py b/aiml-security-assessment/functions/security/bedrock_assessments/app.py index dc343f5..be4ecd5 100644 --- a/aiml-security-assessment/functions/security/bedrock_assessments/app.py +++ b/aiml-security-assessment/functions/security/bedrock_assessments/app.py @@ -26,6 +26,23 @@ logger.setLevel(logging.ERROR) +def _extract_s3_bucket_name(s3_config: Optional[Dict[str, Any]]) -> Optional[str]: + """Support both the documented field name and the legacy test fixture key.""" + if not s3_config: + return None + + return s3_config.get("bucketName") or s3_config.get("s3BucketName") + + +def _is_access_denied_client_error(error: Exception) -> bool: + """Normalize AccessDenied checks across Bedrock and S3 client errors.""" + if not isinstance(error, ClientError): + return False + + error_code = error.response.get("Error", {}).get("Code") + return error_code in {"AccessDenied", "AccessDeniedException"} + + def get_permissions_cache(execution_id: str) -> Optional[Dict[str, Any]]: """ Retrieve and parse the permissions cache JSON file from S3 @@ -876,7 +893,7 @@ def check_bedrock_logging_configuration() -> Dict[str, Any]: # Check S3 logging configuration s3_config = response.get("loggingConfig", {}).get("s3Config") - if s3_config and s3_config.get("s3BucketName"): + if _extract_s3_bucket_name(s3_config): logging_enabled = True enabled_destinations.append("Amazon S3") @@ -1123,22 +1140,18 @@ def check_bedrock_prompt_management() -> Dict[str, Any]: try: # List all prompts - response = bedrock_client.list_prompts() - prompts = response.get("promptSummaries", []) + paginator = bedrock_client.get_paginator("list_prompts") + prompts = [] + for page in paginator.paginate(): + prompts.extend(page.get("promptSummaries", [])) if prompts: - # Count prompts by status - active_prompts = [p for p in prompts if p.get("status") == "ACTIVE"] - draft_prompts = [p for p in prompts if p.get("status") == "DRAFT"] - - findings["details"] = ( - f"Found {len(prompts)} prompts ({len(active_prompts)} active, {len(draft_prompts)} draft)" - ) + findings["details"] = f"Found {len(prompts)} prompts" findings["csv_data"].append( create_finding( check_id="BR-07", finding_name="Bedrock Prompt Management Check", - finding_details=f"Prompt Management is being used with {len(prompts)} prompts ({len(active_prompts)} active, {len(draft_prompts)} draft)", + finding_details=f"Prompt Management is being used with {len(prompts)} prompts", resolution="No action required. Continue using Prompt Management for consistent and optimized prompts.", reference="https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-management.html", severity="Low", @@ -1149,15 +1162,24 @@ def check_bedrock_prompt_management() -> Dict[str, Any]: # Additional check for prompt variants prompts_without_variants = [] for prompt in prompts: + prompt_id = prompt.get("id") or prompt.get("promptId") + prompt_name = prompt.get("name") or prompt_id or "unknown" + if not prompt_id: + logger.warning( + "Skipping prompt without identifier in Prompt Management check" + ) + continue + try: prompt_details = bedrock_client.get_prompt( - promptId=prompt["promptId"] + promptIdentifier=prompt_id ) - if len(prompt_details.get("variants", [])) <= 1: - prompts_without_variants.append(prompt["name"]) + prompt_config = prompt_details.get("prompt", prompt_details) + if len(prompt_config.get("variants", [])) <= 1: + prompts_without_variants.append(prompt_name) except Exception as e: logger.warning( - f"Could not get details for prompt {prompt['name']}: {str(e)}" + f"Could not get details for prompt {prompt_name}: {str(e)}" ) if prompts_without_variants: @@ -1270,6 +1292,7 @@ def check_bedrock_knowledge_base_encryption() -> Dict[str, Any]: return findings kb_without_cmk = [] + kb_access_denied = [] for kb in knowledge_bases: kb_id = kb.get("knowledgeBaseId") @@ -1293,13 +1316,26 @@ def check_bedrock_knowledge_base_encryption() -> Dict[str, Any]: {"id": kb_id, "name": kb_name, "storage_type": storage_type} ) + except ClientError as e: + if _is_access_denied_client_error(e): + kb_access_denied.append({"id": kb_id, "name": kb_name}) + continue + + logger.warning(f"Error checking knowledge base {kb_id}: {str(e)}") except Exception as e: logger.warning(f"Error checking knowledge base {kb_id}: {str(e)}") - if kb_without_cmk: - findings["details"] = ( - f"Found {len(kb_without_cmk)} Knowledge Bases - encryption validated at storage layer" - ) + if kb_without_cmk or kb_access_denied: + detail_parts = [] + if kb_without_cmk: + detail_parts.append( + f"Found {len(kb_without_cmk)} Knowledge Bases - encryption validated at storage layer" + ) + if kb_access_denied: + detail_parts.append( + f"Could not assess {len(kb_access_denied)} Knowledge Bases due to access denied" + ) + findings["details"] = "; ".join(detail_parts) for kb in kb_without_cmk: findings["csv_data"].append( @@ -1313,6 +1349,19 @@ def check_bedrock_knowledge_base_encryption() -> Dict[str, Any]: status="N/A", ) ) + + for kb in kb_access_denied: + findings["csv_data"].append( + create_finding( + check_id="BR-09", + finding_name="Bedrock Knowledge Base Encryption Check", + finding_details=f"Unable to assess Knowledge Base '{kb['name']}' ({kb['id']}) because access to Knowledge Base metadata was denied.", + resolution="Ensure the assessment role can call bedrock:ListKnowledgeBases and bedrock:GetKnowledgeBase for the target account.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/encryption-kb.html", + severity="Informational", + status="N/A", + ) + ) else: findings["csv_data"].append( create_finding( @@ -1326,22 +1375,43 @@ def check_bedrock_knowledge_base_encryption() -> Dict[str, Any]: ) ) - except bedrock_agent_client.exceptions.ValidationException as e: - findings["status"] = "ERROR" - findings["details"] = ( - f"Error validating Knowledge Base configuration: {str(e)}" - ) - findings["csv_data"].append( - create_finding( - check_id="BR-09", - finding_name="Bedrock Knowledge Base Encryption Check", - finding_details=f"Error checking Knowledge Base encryption: {str(e)}", - resolution="Verify your AWS credentials and permissions to access Bedrock Knowledge Bases", - reference="https://docs.aws.amazon.com/bedrock/latest/userguide/encryption-kb.html", - severity="High", - status="Failed", + except ClientError as e: + if _is_access_denied_client_error(e): + findings["status"] = "WARN" + findings["details"] = ( + "Unable to assess Knowledge Base encryption because access was denied" ) - ) + findings["csv_data"].append( + create_finding( + check_id="BR-09", + finding_name="Bedrock Knowledge Base Encryption Check", + finding_details="Unable to assess Knowledge Base encryption because access to Knowledge Base metadata was denied.", + resolution="Ensure the assessment role can call bedrock:ListKnowledgeBases and bedrock:GetKnowledgeBase for the target account.", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/encryption-kb.html", + severity="Informational", + status="N/A", + ) + ) + else: + error_code = e.response.get("Error", {}).get("Code") + if error_code == "ValidationException": + findings["status"] = "ERROR" + findings["details"] = ( + f"Error validating Knowledge Base configuration: {str(e)}" + ) + findings["csv_data"].append( + create_finding( + check_id="BR-09", + finding_name="Bedrock Knowledge Base Encryption Check", + finding_details=f"Error checking Knowledge Base encryption: {str(e)}", + resolution="Verify your AWS credentials and permissions to access Bedrock Knowledge Bases", + reference="https://docs.aws.amazon.com/bedrock/latest/userguide/encryption-kb.html", + severity="High", + status="Failed", + ) + ) + else: + raise return findings @@ -1727,7 +1797,9 @@ def check_bedrock_invocation_log_encryption() -> Dict[str, Any]: s3_config = logging_config.get("s3Config") - if not s3_config or not s3_config.get("bucketName"): + bucket_name = _extract_s3_bucket_name(s3_config) + + if not bucket_name: findings["csv_data"].append( create_finding( check_id="BR-12", @@ -1741,8 +1813,6 @@ def check_bedrock_invocation_log_encryption() -> Dict[str, Any]: ) return findings - bucket_name = s3_config.get("bucketName") - # Check S3 bucket encryption try: encryption_response = s3_client.get_bucket_encryption( @@ -1813,16 +1883,20 @@ def check_bedrock_invocation_log_encryption() -> Dict[str, Any]: status="Failed", ) ) - elif e.response["Error"]["Code"] == "AccessDenied": + elif _is_access_denied_client_error(e): + findings["status"] = "WARN" + findings["details"] = ( + f"Unable to assess encryption for bucket '{bucket_name}' due to access denied" + ) findings["csv_data"].append( create_finding( check_id="BR-12", finding_name="Bedrock Invocation Log Encryption Check", - finding_details=f"Unable to check encryption for bucket '{bucket_name}' - access denied", - resolution="Ensure Lambda execution role has s3:GetEncryptionConfiguration permission", + finding_details=f"Unable to assess encryption for bucket '{bucket_name}' because access to the bucket encryption configuration was denied.", + resolution="Ensure the assessment role and bucket policy allow s3:GetEncryptionConfiguration for the logging bucket.", reference="https://docs.aws.amazon.com/bedrock/latest/userguide/model-invocation-logging.html", - severity="Medium", - status="Failed", + severity="Informational", + status="N/A", ) ) else: @@ -2067,8 +2141,10 @@ def check_bedrock_agent_roles(permission_cache) -> Dict[str, Any]: try: # Get all Bedrock agents - response = bedrock_client.list_agents() - agents = response.get("agents", []) + paginator = bedrock_client.get_paginator("list_agents") + agents = [] + for page in paginator.paginate(): + agents.extend(page.get("agentSummaries", page.get("agents", []))) if not agents: findings["details"] = "No Bedrock agents found" @@ -2089,12 +2165,19 @@ def check_bedrock_agent_roles(permission_cache) -> Dict[str, Any]: for agent in agents: agent_id = agent.get("agentId") - agent_name = agent.get("agentName") + agent_name = agent.get("agentName") or agent_id or "unknown" + if not agent_id: + logger.warning( + "Skipping Bedrock agent without agentId in IAM role check" + ) + continue # Get agent details including role ARN agent_details = bedrock_client.get_agent(agentId=agent_id) - role_arn = agent_details.get("agentResourceRoleArn") + role_arn = agent_details.get("agent", {}).get( + "agentResourceRoleArn" + ) or agent_details.get("agentResourceRoleArn") if not role_arn: continue diff --git a/aiml-security-assessment/functions/security/bedrock_assessments/schema.py b/aiml-security-assessment/functions/security/bedrock_assessments/schema.py index d2a713a..6c5a9f5 100644 --- a/aiml-security-assessment/functions/security/bedrock_assessments/schema.py +++ b/aiml-security-assessment/functions/security/bedrock_assessments/schema.py @@ -1,59 +1,73 @@ from enum import Enum -from typing import Dict, List, Any -from pydantic import BaseModel, Field, HttpUrl, validator -from datetime import datetime +from typing import Dict, Any +from pydantic import BaseModel, Field, validator import re + class SeverityEnum(str, Enum): HIGH = "High" MEDIUM = "Medium" LOW = "Low" INFORMATIONAL = "Informational" + class StatusEnum(str, Enum): FAILED = "Failed" PASSED = "Passed" NA = "N/A" + class Finding(BaseModel): """Represents a security finding with required fields and validations""" - Check_ID: str = Field(..., min_length=1, description="Unique check identifier (e.g., SM-01, BR-01, AC-01)") + + Check_ID: str = Field( + ..., + min_length=1, + description="Unique check identifier (e.g., SM-01, BR-01, AC-01)", + ) Finding: str = Field(..., min_length=1, description="The name/title of the finding") - Finding_Details: str = Field(..., min_length=1, description="Detailed description of the finding") - Resolution: str = Field(..., min_length=0, description="Steps to resolve the finding") + Finding_Details: str = Field( + ..., min_length=1, description="Detailed description of the finding" + ) + Resolution: str = Field( + ..., min_length=0, description="Steps to resolve the finding" + ) Reference: str = Field(..., description="Documentation reference URL") Severity: SeverityEnum = Field(..., description="Severity level of the finding") Status: StatusEnum = Field(..., description="Current status of the finding") - @validator('Check_ID') + @validator("Check_ID") def validate_check_id(cls, v): """Validate that Check_ID follows the pattern XX-NN (e.g., SM-01, BR-14, AC-05)""" - pattern = r'^[A-Z]{2,3}-\d{2}$' + pattern = r"^[A-Z]{2,3}-\d{2}$" if not re.match(pattern, v): - raise ValueError('Check_ID must follow pattern XX-NN (e.g., SM-01, BR-14, AC-05)') + raise ValueError( + "Check_ID must follow pattern XX-NN (e.g., SM-01, BR-14, AC-05)" + ) return v - @validator('Reference') + @validator("Reference") def validate_reference_url(cls, v): """Validate that reference URL starts with https://""" - if not str(v).startswith('https://'): - raise ValueError('Reference URL must start with https://') + if not str(v).startswith("https://"): + raise ValueError("Reference URL must start with https://") return v - @validator('Severity') + @validator("Severity") def validate_severity(cls, v): """Validate that severity is one of the allowed values""" if v not in SeverityEnum.__members__.values(): - raise ValueError('Severity must be one of the allowed values') + raise ValueError("Severity must be one of the allowed values") return v - @validator('Status') + @validator("Status") def validate_status(cls, v): """Validate that status is one of the allowed values""" if v not in StatusEnum.__members__.values(): - raise ValueError('Status must be one of the allowed values') + raise ValueError("Status must be one of the allowed values") return v + def create_finding( check_id: str, finding_name: str, @@ -61,7 +75,7 @@ def create_finding( resolution: str, reference: str, severity: SeverityEnum, - status: StatusEnum + status: StatusEnum, ) -> Dict[str, Any]: """ Create a validated finding object @@ -88,6 +102,6 @@ def create_finding( Resolution=resolution, Reference=reference, Severity=severity, - Status=status + Status=status, ) - return dict(finding.model_dump()) # Convert to regular dictionary \ No newline at end of file + return dict(finding.model_dump()) # Convert to regular dictionary diff --git a/aiml-security-assessment/functions/security/finserv_assessments/app.py b/aiml-security-assessment/functions/security/finserv_assessments/app.py index bdc5e1b..936795a 100644 --- a/aiml-security-assessment/functions/security/finserv_assessments/app.py +++ b/aiml-security-assessment/functions/security/finserv_assessments/app.py @@ -2277,15 +2277,14 @@ def check_training_data_s3_versioning(inventory) -> Dict[str, Any]: def _is_overbroad_kb_action(action: Any) -> bool: """True if an IAM action grants overly broad Bedrock/Knowledge Base access: - the full wildcard, a service-wide bedrock(-agent) wildcard, or ANY partial - wildcard within those namespaces (e.g., 'bedrock-agent:Get*', 'bedrock:Invoke*'). - Round-3 fixed the crash; this widens detection beyond the three exact wildcards.""" + the full wildcard, a service-wide Bedrock wildcard, or ANY partial wildcard + within the Bedrock IAM namespace (for example, 'bedrock:Invoke*').""" if not isinstance(action, str): return False a = action.lower() - if a in ("*", "bedrock:*", "bedrock-agent:*"): + if a in ("*", "bedrock:*"): return True - if a.endswith("*") and (a.startswith("bedrock:") or a.startswith("bedrock-agent:")): + if a.endswith("*") and a.startswith("bedrock:"): return True return False @@ -2293,7 +2292,7 @@ def _is_overbroad_kb_action(action: Any) -> bool: def check_knowledge_base_iam_least_privilege(permission_cache) -> Dict[str, Any]: """ FS-22 — Verify IAM roles accessing Bedrock Knowledge Bases follow - least privilege (no wildcard bedrock-agent:* permissions). + least privilege (no wildcard bedrock:* permissions). COMPLIANCE_PLACEHOLDER: [NYDFS 500.06, FFIEC CAT, PCI-DSS 12.3.2] """ findings = _empty_findings("Knowledge Base IAM Least Privilege Check") @@ -2348,10 +2347,7 @@ def check_knowledge_base_iam_least_privilege(permission_cache) -> Dict[str, Any] elif ( unscoped_resource and isinstance(action, str) - and ( - action.lower().startswith("bedrock:") - or action.lower().startswith("bedrock-agent:") - ) + and action.lower().startswith("bedrock:") ): issues.append( f"Role '{role_name}' allows '{action}' on Resource '*' " @@ -2369,7 +2365,7 @@ def check_knowledge_base_iam_least_privilege(permission_cache) -> Dict[str, Any] + "\n".join(f"- {i}" for i in issues[:10]) ), resolution=( - "Replace wildcard bedrock-agent:* with specific actions: " + "Replace wildcard bedrock:* with specific actions such as " "bedrock:Retrieve, bedrock:RetrieveAndGenerate. " "Scope resources to specific Knowledge Base ARNs." ), diff --git a/aiml-security-assessment/functions/security/finserv_tests/conftest.py b/aiml-security-assessment/functions/security/finserv_tests/conftest.py index 7107ac3..e47fb79 100644 --- a/aiml-security-assessment/functions/security/finserv_tests/conftest.py +++ b/aiml-security-assessment/functions/security/finserv_tests/conftest.py @@ -1,25 +1,7 @@ -""" -Shared fixtures and mock helpers for finserv_assessments tests. - -All boto3 clients are patched at the module level so check functions -never make real AWS API calls. -""" - -import os -import sys +"""Shared fixtures for FinServ assessment tests.""" import pytest -# --------------------------------------------------------------------------- -# Make finserv_assessments importable — the Lambda runtime adds the package -# root to sys.path, so we replicate that here. -# --------------------------------------------------------------------------- -FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") -if FINSERV_DIR not in sys.path: - sys.path.insert(0, FINSERV_DIR) - -import app # noqa: E402 (imported after sys.path manipulation) - # --------------------------------------------------------------------------- # Environment variables expected by the Lambda @@ -100,36 +82,3 @@ def lambda_event(): "Id": "arn:aws:states:us-east-1:123456789012:stateMachine:test" }, } - - -# --------------------------------------------------------------------------- -# ResourceInventory test builder (REQ-6.4, REQ-9.3) -# --------------------------------------------------------------------------- - - -def make_resource_inventory(**overrides) -> app.ResourceInventory: - """Build a fully-available ``ResourceInventory`` with sensible empty defaults. - - Any field can be replaced via keyword arguments. Pass an - ``app._Unavailable(exc)`` value to simulate a per-inventory collection - failure. - - Examples:: - - inv = make_resource_inventory() # fully available - inv = make_resource_inventory(lambda_functions=[...]) # real data - inv = make_resource_inventory( - guardrails=app._Unavailable(PermissionError("AccessDenied")) - ) # failed field - """ - defaults: dict = dict( - lambda_functions=[], - guardrails=app.GuardrailInventory(summaries=[], detail_by_id={}), - knowledge_bases=app.KbInventory( - summaries=[], data_sources_by_kb={}, data_source_detail={} - ), - buckets=[], - web_acls=app.WebAclInventory(summaries=[], detail_by_id={}), - ) - defaults.update(overrides) - return app.ResourceInventory(**defaults) diff --git a/aiml-security-assessment/functions/security/finserv_tests/support.py b/aiml-security-assessment/functions/security/finserv_tests/support.py new file mode 100644 index 0000000..3475c43 --- /dev/null +++ b/aiml-security-assessment/functions/security/finserv_tests/support.py @@ -0,0 +1,53 @@ +"""Stable imports and shared helpers for FinServ tests.""" + +from __future__ import annotations + +import importlib.util +import os +import sys + + +_THIS_DIR = os.path.dirname(__file__) +_FINSERV_DIR = os.path.abspath(os.path.join(_THIS_DIR, "..", "finserv_assessments")) +_APP_PATH = os.path.join(_FINSERV_DIR, "app.py") +_SCHEMA_PATH = os.path.join(_FINSERV_DIR, "schema.py") + + +def _load_module(module_name: str, path: str, add_path: str | None = None): + saved_sys_path = list(sys.path) + saved_schema = sys.modules.get("schema") + + try: + if add_path and add_path not in sys.path: + sys.path.insert(0, add_path) + + spec = importlib.util.spec_from_file_location(module_name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + return module + finally: + sys.path[:] = saved_sys_path + if saved_schema is None: + sys.modules.pop("schema", None) + else: + sys.modules["schema"] = saved_schema + + +finserv_schema = _load_module("finserv_schema", _SCHEMA_PATH) +finserv_app = _load_module("finserv_app", _APP_PATH, add_path=_FINSERV_DIR) + + +def make_resource_inventory(**overrides) -> finserv_app.ResourceInventory: + """Build a fully-available ResourceInventory with sensible empty defaults.""" + defaults: dict = dict( + lambda_functions=[], + guardrails=finserv_app.GuardrailInventory(summaries=[], detail_by_id={}), + knowledge_bases=finserv_app.KbInventory( + summaries=[], data_sources_by_kb={}, data_source_detail={} + ), + buckets=[], + web_acls=finserv_app.WebAclInventory(summaries=[], detail_by_id={}), + ) + defaults.update(overrides) + return finserv_app.ResourceInventory(**defaults) diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_at_most_once.py b/aiml-security-assessment/functions/security/finserv_tests/test_at_most_once.py index dd1a750..303a049 100644 --- a/aiml-security-assessment/functions/security/finserv_tests/test_at_most_once.py +++ b/aiml-security-assessment/functions/security/finserv_tests/test_at_most_once.py @@ -22,22 +22,13 @@ from __future__ import annotations -import os -import sys from collections import defaultdict from datetime import datetime, timezone from unittest.mock import MagicMock, patch import pytest -# --------------------------------------------------------------------------- -# Make finserv_assessments importable -# --------------------------------------------------------------------------- -FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") -if FINSERV_DIR not in sys.path: - sys.path.insert(0, FINSERV_DIR) - -import app # noqa: E402 +from .support import finserv_app as app # =========================================================================== @@ -549,9 +540,9 @@ def _run_with_counting_mocks(event=None): side_effect, tracker = _build_counting_client_factory() with ( - patch("app.boto3.client") as mock_client, - patch("app.get_permissions_cache") as mock_cache, - patch("app.write_to_s3") as mock_s3, + patch("finserv_app.boto3.client") as mock_client, + patch("finserv_app.get_permissions_cache") as mock_cache, + patch("finserv_app.write_to_s3") as mock_s3, ): mock_client.side_effect = side_effect mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_checks.py b/aiml-security-assessment/functions/security/finserv_tests/test_checks.py index f456443..a5a7f92 100644 --- a/aiml-security-assessment/functions/security/finserv_tests/test_checks.py +++ b/aiml-security-assessment/functions/security/finserv_tests/test_checks.py @@ -14,24 +14,13 @@ """ import json -import sys import os from unittest.mock import MagicMock, patch from botocore.exceptions import ClientError -# Ensure finserv_assessments is importable -FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") -if FINSERV_DIR not in sys.path: - sys.path.insert(0, FINSERV_DIR) - -# Ensure tests/ directory is importable (for conftest helpers) -TESTS_DIR = os.path.dirname(__file__) -if TESTS_DIR not in sys.path: - sys.path.insert(0, TESTS_DIR) - -import app # noqa: E402 (import must follow sys.path setup above) -from conftest import make_resource_inventory # noqa: E402 +from .support import finserv_app as app +from .support import make_resource_inventory # ========================================================================= @@ -97,7 +86,7 @@ def test_pass_shield_enabled_acls_present(self): detail_by_id={}, ) ) - with patch("app.boto3.client") as mock_client: + with patch("finserv_app.boto3.client") as mock_client: shield_mock = MagicMock() shield_mock.describe_subscription.return_value = {} shield_mock.exceptions.ResourceNotFoundException = type( @@ -120,7 +109,7 @@ def test_severity_shield_low_waf_medium_on_fail(self): inv = make_resource_inventory( web_acls=app.WebAclInventory(summaries=[], detail_by_id={}) ) - with patch("app.boto3.client") as mock_client: + with patch("finserv_app.boto3.client") as mock_client: shield_mock = MagicMock() shield_mock.describe_subscription.side_effect = ClientError( {"Error": {"Code": "ResourceNotFoundException", "Message": ""}}, @@ -148,7 +137,7 @@ def test_warn_no_shield_no_acls(self): inv = make_resource_inventory( web_acls=app.WebAclInventory(summaries=[], detail_by_id={}) ) - with patch("app.boto3.client") as mock_client: + with patch("finserv_app.boto3.client") as mock_client: shield_mock = MagicMock() shield_mock.describe_subscription.side_effect = ClientError( {"Error": {"Code": "ResourceNotFoundException", "Message": ""}}, @@ -166,7 +155,7 @@ def test_warn_no_shield_no_acls(self): def test_error_on_exception(self): """Unavailable inventory → COULD_NOT_ASSESS (ERROR envelope).""" inv = make_resource_inventory(web_acls=app._Unavailable(RuntimeError("boom"))) - with patch("app.boto3.client") as mock_client: + with patch("finserv_app.boto3.client") as mock_client: shield_mock = MagicMock() shield_mock.describe_subscription.return_value = {} shield_mock.exceptions.ResourceNotFoundException = type( @@ -182,7 +171,7 @@ def test_error_on_shield_exception(self): inv = make_resource_inventory( web_acls=app.WebAclInventory(summaries=[], detail_by_id={}) ) - with patch("app.boto3.client") as mock_client: + with patch("finserv_app.boto3.client") as mock_client: mock_client.side_effect = RuntimeError("boom") result = app.check_waf_shield_on_bedrock_endpoints(inv) assert result["status"] == "ERROR" @@ -195,7 +184,7 @@ def test_pass_more_than_100_acls(self): inv = make_resource_inventory( web_acls=app.WebAclInventory(summaries=acls, detail_by_id={}) ) - with patch("app.boto3.client") as mock_client: + with patch("finserv_app.boto3.client") as mock_client: shield_mock = MagicMock() shield_mock.describe_subscription.return_value = {} shield_mock.exceptions.ResourceNotFoundException = type( @@ -219,7 +208,7 @@ def test_two_acl_case_unchanged(self): inv = make_resource_inventory( web_acls=app.WebAclInventory(summaries=acls, detail_by_id={}) ) - with patch("app.boto3.client") as mock_client: + with patch("finserv_app.boto3.client") as mock_client: shield_mock = MagicMock() shield_mock.describe_subscription.return_value = {} shield_mock.exceptions.ResourceNotFoundException = type( @@ -237,7 +226,7 @@ def test_two_acl_case_unchanged(self): class TestFS02ApiGatewayRateLimiting: """FS-02 — API Gateway Rate Limiting Check.""" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_all_plans_have_throttle(self, mock_client): c = MagicMock() c.get_usage_plans.return_value = { @@ -250,7 +239,7 @@ def test_pass_all_plans_have_throttle(self, mock_client): _assert_finding_structure(result) assert result["status"] == "PASS" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_warn_plan_missing_throttle(self, mock_client): c = MagicMock() c.get_usage_plans.return_value = { @@ -263,7 +252,7 @@ def test_warn_plan_missing_throttle(self, mock_client): _assert_finding_structure(result) assert result["status"] == "WARN" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_no_plans_returns_na(self, mock_client): c = MagicMock() c.get_usage_plans.return_value = {"items": []} @@ -273,7 +262,7 @@ def test_no_plans_returns_na(self, mock_client): # No plans → advisory finding, status stays PASS assert any(r["Status"] == "N/A" for r in result["csv_data"]) - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_error_on_exception(self, mock_client): mock_client.side_effect = RuntimeError("api error") result = app.check_api_gateway_rate_limiting() @@ -301,7 +290,7 @@ def get_paginator(op_name): c.get_paginator.side_effect = get_paginator return c - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_customized_quota(self, mock_client): # Applied value (200000) exceeds AWS default (100000) → customized → PASS/Passed applied = [ @@ -318,7 +307,7 @@ def test_pass_customized_quota(self, mock_client): assert result["status"] == "PASS" assert any(r["Status"] == "Passed" for r in result["csv_data"]) - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_warn_default_quota(self, mock_client): # Applied value == AWS default → still at default → WARN/N-A (soft, not a failure) applied = [ @@ -337,7 +326,7 @@ def test_warn_default_quota(self, mock_client): # At-default is NOT a failure. assert not any(r["Status"] == "Failed" for r in result["csv_data"]) - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_token_only_no_rpm(self, mock_client): # Only token-based quotas present (no "request"/RPM quota). RPM is deprecated # on bedrock-runtime; its absence must not drive a Failed verdict. @@ -355,7 +344,7 @@ def test_token_only_no_rpm(self, mock_client): # Customized token quota → PASS, regardless of any RPM quota existing. assert result["status"] == "PASS" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_empty_applied_quotas(self, mock_client): # No token quotas returned at all → WARN/Failed + explanatory details. mock_client.return_value = self._sq_client([], []) @@ -368,7 +357,7 @@ def test_empty_applied_quotas(self, mock_client): for r in result["csv_data"] ) - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_default_lookup_fail(self, mock_client): # Applied token quotas exist but defaults could not be retrieved → # WARN/Failed + "undetermined" (NOT a silent value-vs-itself comparison). @@ -390,7 +379,7 @@ def test_default_lookup_fail(self, mock_client): for r in result["csv_data"] ) - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_error_on_exception(self, mock_client): mock_client.side_effect = RuntimeError("quota error") result = app.check_bedrock_token_quotas() @@ -400,7 +389,7 @@ def test_error_on_exception(self, mock_client): class TestFS04CostAnomalyDetection: """FS-04 — Cost Anomaly Detection Check.""" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_monitors_exist(self, mock_client): c = MagicMock() c.get_anomaly_monitors.return_value = { @@ -417,7 +406,7 @@ def test_pass_monitors_exist(self, mock_client): _assert_finding_structure(result) assert result["status"] == "PASS" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_warn_monitors_without_bedrock_coverage(self, mock_client): # A DIMENSIONAL monitor scoped to LINKED_ACCOUNT does NOT provide # Bedrock service-level coverage → non-PASS (previously masked false positive). @@ -437,7 +426,7 @@ def test_warn_monitors_without_bedrock_coverage(self, mock_client): assert result["status"] != "PASS" assert any(r["Status"] == "Failed" for r in result["csv_data"]) - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_warn_no_monitors(self, mock_client): c = MagicMock() c.get_anomaly_monitors.return_value = {"AnomalyMonitors": []} @@ -446,7 +435,7 @@ def test_warn_no_monitors(self, mock_client): _assert_finding_structure(result) assert result["status"] == "WARN" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pagination_finds_bedrock_monitor_on_second_page(self, mock_client): # The Bedrock-covering monitor is on page 2. The check must paginate via # NextPageToken and still find it (otherwise a false "no coverage" finding). @@ -480,7 +469,7 @@ def test_pagination_finds_bedrock_monitor_on_second_page(self, mock_client): assert c.get_anomaly_monitors.call_count == 2 c.get_anomaly_monitors.assert_any_call(NextPageToken="page2") - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_error_on_exception(self, mock_client): mock_client.side_effect = RuntimeError("ce error") result = app.check_cost_anomaly_detection() @@ -490,7 +479,7 @@ def test_error_on_exception(self, mock_client): class TestFS05CloudWatchTokenAlarms: """FS-05 — CloudWatch Token Usage Alarms Check.""" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_bedrock_alarms_exist(self, mock_client): c = MagicMock() paginator = MagicMock() @@ -511,7 +500,7 @@ def test_pass_bedrock_alarms_exist(self, mock_client): _assert_finding_structure(result) assert result["status"] == "PASS" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_warn_no_bedrock_alarms(self, mock_client): c = MagicMock() paginator = MagicMock() @@ -532,7 +521,7 @@ def test_warn_no_bedrock_alarms(self, mock_client): _assert_finding_structure(result) assert result["status"] == "WARN" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_error_on_exception(self, mock_client): mock_client.side_effect = RuntimeError("cw error") result = app.check_cloudwatch_token_alarms() @@ -578,7 +567,7 @@ def side_effect(service, **kwargs): return side_effect - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_aiml_budgets_exist(self, mock_client): capture = [] mock_client.side_effect = self._client_factory( @@ -591,7 +580,7 @@ def test_pass_aiml_budgets_exist(self, mock_client): # Regression guard: the call MUST pass ShowFilterExpression=True. assert any(kw.get("ShowFilterExpression") is True for kw in capture) - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_warn_no_aiml_budgets(self, mock_client): mock_client.side_effect = self._client_factory( [{"BudgetName": "general", "CostFilters": {"Service": ["ec2"]}}] @@ -600,7 +589,7 @@ def test_warn_no_aiml_budgets(self, mock_client): _assert_finding_structure(result) assert result["status"] == "WARN" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_filterexpression_budget(self, mock_client): # New-style budget using only FilterExpression (no CostFilters) → detected. mock_client.side_effect = self._client_factory( @@ -618,7 +607,7 @@ def test_pass_filterexpression_budget(self, mock_client): _assert_finding_structure(result) assert result["status"] == "PASS" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_param_validation_fallback(self, mock_client): # Old botocore: ShowFilterExpression rejected with ParamValidationError. # FS-06 must degrade to CostFilters-only (non-ERROR) and still match. @@ -636,7 +625,7 @@ def test_param_validation_fallback(self, mock_client): assert any("ShowFilterExpression" in kw for kw in capture) assert any("ShowFilterExpression" not in kw for kw in capture) - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_error_on_exception(self, mock_client): mock_client.side_effect = RuntimeError("budgets error") result = app.check_aws_budgets_for_aiml() @@ -651,7 +640,7 @@ def test_error_on_exception(self, mock_client): class TestFS07AgentActionBoundaries: """FS-07 — Agent Action Boundary Check (takes permission_cache).""" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_no_agents(self, mock_client): c = MagicMock() c.list_agents.return_value = {"agentSummaries": []} @@ -660,7 +649,7 @@ def test_pass_no_agents(self, mock_client): _assert_finding_structure(result) assert any(r["Status"] == "N/A" for r in result["csv_data"]) - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_warn_wildcard_permissions( self, mock_client, permission_cache_with_wildcard ): @@ -678,7 +667,7 @@ def test_warn_wildcard_permissions( _assert_finding_structure(result) assert result["status"] == "WARN" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_narrow_permissions(self, mock_client, permission_cache_safe): c = MagicMock() c.list_agents.return_value = { @@ -692,7 +681,7 @@ def test_pass_narrow_permissions(self, mock_client, permission_cache_safe): _assert_finding_structure(result) assert result["status"] == "PASS" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_error_on_exception(self, mock_client): mock_client.side_effect = RuntimeError("agent error") result = app.check_bedrock_agent_action_boundaries({}) @@ -702,7 +691,7 @@ def test_error_on_exception(self, mock_client): class TestFS08AgentcorePolicyEngine: """FS-08 — AgentCore Policy Engine Check.""" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_runtimes_with_authorizer(self, mock_client): c = MagicMock() c.list_agent_runtimes.return_value = { @@ -718,7 +707,7 @@ def test_pass_runtimes_with_authorizer(self, mock_client): _assert_finding_structure(result) assert result["status"] == "PASS" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_warn_runtimes_without_authorizer(self, mock_client): c = MagicMock() c.list_agent_runtimes.return_value = { @@ -729,7 +718,7 @@ def test_warn_runtimes_without_authorizer(self, mock_client): _assert_finding_structure(result) assert result["status"] == "WARN" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_na_no_runtimes(self, mock_client): c = MagicMock() c.list_agent_runtimes.return_value = {"agentRuntimes": []} @@ -738,7 +727,7 @@ def test_na_no_runtimes(self, mock_client): _assert_finding_structure(result) assert any(r["Status"] == "N/A" for r in result["csv_data"]) - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_access_denied_returns_na(self, mock_client): c = MagicMock() c.list_agent_runtimes.side_effect = _client_error("AccessDeniedException") @@ -747,7 +736,7 @@ def test_access_denied_returns_na(self, mock_client): _assert_finding_structure(result) assert any(r["Status"] == "N/A" for r in result["csv_data"]) - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_error_on_exception(self, mock_client): mock_client.side_effect = RuntimeError("agentcore error") result = app.check_agentcore_policy_engine() @@ -757,7 +746,7 @@ def test_error_on_exception(self, mock_client): class TestFS09AgentTransactionLimits: """FS-09 — Agent Transaction Limits Check.""" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_concurrency_set(self, mock_client): c = MagicMock() c.get_function_concurrency.return_value = {"ReservedConcurrentExecutions": 10} @@ -769,7 +758,7 @@ def test_pass_concurrency_set(self, mock_client): _assert_finding_structure(result) assert result["status"] == "PASS" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_warn_no_concurrency(self, mock_client): c = MagicMock() c.get_function_concurrency.return_value = {} @@ -792,7 +781,7 @@ def test_error_on_unavailable_inventory(self): class TestFS10HumanInTheLoop: """FS-10 — Human-in-the-Loop Approval Check.""" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_wait_for_task_token(self, mock_client): c = MagicMock() c.list_state_machines.return_value = { @@ -824,7 +813,7 @@ def test_pass_wait_for_task_token(self, mock_client): assert result["status"] == "PASS" assert len(result["csv_data"]) >= 1 - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_warn_no_wait_token(self, mock_client): c = MagicMock() c.list_state_machines.return_value = { @@ -843,7 +832,7 @@ def test_warn_no_wait_token(self, mock_client): _assert_finding_structure(result) assert result["status"] == "WARN" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_error_on_exception(self, mock_client): mock_client.side_effect = RuntimeError("sfn error") result = app.check_human_in_the_loop_for_high_risk_actions() @@ -853,7 +842,7 @@ def test_error_on_exception(self, mock_client): class TestFS11AgentRateAlarms: """FS-11 — Agent Rate Alarms Check.""" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_agent_alarms_exist(self, mock_client): c = MagicMock() paginator = MagicMock() @@ -875,7 +864,7 @@ def test_pass_agent_alarms_exist(self, mock_client): # The function looks for agent-related alarms assert result["status"] in ("PASS", "WARN") - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_error_on_exception(self, mock_client): mock_client.side_effect = RuntimeError("cw error") result = app.check_agent_rate_alarms() @@ -890,13 +879,13 @@ def test_error_on_exception(self, mock_client): class TestFS12ScpModelAccess: """FS-12 — SCP Model Access Restrictions.""" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_error_on_exception(self, mock_client): mock_client.side_effect = RuntimeError("org error") result = app.check_scp_model_access_restrictions() assert result["status"] == "ERROR" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_returns_valid_structure(self, mock_client): c = MagicMock() c.list_policies.return_value = {"Policies": []} @@ -908,13 +897,13 @@ def test_returns_valid_structure(self, mock_client): class TestFS13ModelInventoryTagging: """FS-13 — Model Inventory Tagging.""" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_error_on_exception(self, mock_client): mock_client.side_effect = RuntimeError("tagging error") result = app.check_model_inventory_tagging() assert result["status"] == "ERROR" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_returns_valid_structure(self, mock_client): c = MagicMock() c.list_custom_models.return_value = {"modelSummaries": []} @@ -927,13 +916,13 @@ def test_returns_valid_structure(self, mock_client): class TestFS14ModelOnboardingGovernance: """FS-14 — Model Onboarding Governance.""" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_error_on_exception(self, mock_client): mock_client.side_effect = RuntimeError("config error") result = app.check_model_onboarding_governance() assert result["status"] == "ERROR" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_returns_valid_structure(self, mock_client): c = MagicMock() c.describe_config_rules.return_value = {"ConfigRules": []} @@ -945,13 +934,13 @@ def test_returns_valid_structure(self, mock_client): class TestFS15BedrockModelEvalAdversarial: """FS-15 — Bedrock Model Evaluation Adversarial.""" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_error_on_exception(self, mock_client): mock_client.side_effect = RuntimeError("eval error") result = app.check_bedrock_model_evaluation_adversarial() assert result["status"] == "ERROR" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_returns_valid_structure(self, mock_client): c = MagicMock() c.list_evaluation_jobs.return_value = {"jobSummaries": []} @@ -959,7 +948,7 @@ def test_returns_valid_structure(self, mock_client): result = app.check_bedrock_model_evaluation_adversarial() _assert_finding_structure(result) - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_fail_no_eval_jobs(self, mock_client): """REQ-10a: no Bedrock evaluation jobs → Failed/Medium (was N/A).""" c = MagicMock() @@ -974,7 +963,7 @@ def test_fail_no_eval_jobs(self, mock_client): for r in result["csv_data"] ) - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_eval_jobs_present(self, mock_client): """Eval jobs present → Passed/Medium.""" c = MagicMock() @@ -994,7 +983,7 @@ def test_pass_eval_jobs_present(self, mock_client): class TestFS16EcrImageScanning: """FS-16 — ECR Image Scanning.""" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_scanning_enabled(self, mock_client): c = MagicMock() c.describe_repositories.return_value = { @@ -1009,7 +998,7 @@ def test_pass_scanning_enabled(self, mock_client): result = app.check_ecr_image_scanning() _assert_finding_structure(result) - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_error_on_exception(self, mock_client): mock_client.side_effect = RuntimeError("ecr error") result = app.check_ecr_image_scanning() @@ -1024,13 +1013,13 @@ def test_error_on_exception(self, mock_client): class TestFS20FeatureStoreRollback: """FS-20 — Feature Store Rollback Capability.""" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_error_on_exception(self, mock_client): mock_client.side_effect = RuntimeError("fs error") result = app.check_feature_store_rollback_capability() assert result["status"] == "ERROR" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_returns_valid_structure(self, mock_client): c = MagicMock() c.list_feature_groups.return_value = {"FeatureGroupSummaries": []} @@ -1050,7 +1039,7 @@ def test_error_on_unavailable_inventory(self): result = app.check_training_data_s3_versioning(inv) assert result["status"] == "ERROR" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_returns_valid_structure(self, mock_client): """Empty bucket list → N/A finding (no training buckets identified).""" inv = make_resource_inventory(buckets=[]) @@ -1074,7 +1063,7 @@ def test_pass_empty_cache(self, permission_cache_empty): _assert_finding_structure(result) assert result["status"] == "PASS" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_error_on_exception(self, mock_client): """FS-22 only reads permission_cache (no boto3 calls). To trigger the error path, pass a cache that causes an exception during iteration.""" @@ -1097,7 +1086,7 @@ def test_single_statement_dict_no_crash_wildcard(self): "Version": "2012-10-17", "Statement": { "Effect": "Allow", - "Action": "bedrock-agent:*", + "Action": "bedrock:*", "Resource": "*", }, }, @@ -1147,7 +1136,7 @@ def test_single_statement_dict_no_wildcard(self): ) def test_partial_wildcard_flagged(self): - """REQ-14/D: a partial wildcard (e.g. 'bedrock-agent:Get*') is over-broad + """REQ-14/D: a partial wildcard (e.g. 'bedrock:Get*') is over-broad and must be flagged, not just the three exact full wildcards.""" cache = { "role_permissions": { @@ -1160,7 +1149,7 @@ def test_partial_wildcard_flagged(self): "Statement": [ { "Effect": "Allow", - "Action": "bedrock-agent:Get*", + "Action": "bedrock:Get*", "Resource": "arn:aws:bedrock:*:*:knowledge-base/kb-1", } ] @@ -1308,7 +1297,7 @@ def test_returns_advisory_with_kbs(self): class TestFS25OpensearchServerlessEncryption: """FS-25 — OpenSearch Serverless Encryption.""" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_error_on_exception(self, mock_client): mock_client.side_effect = RuntimeError("oss error") result = app.check_opensearch_serverless_encryption() @@ -1318,7 +1307,7 @@ def test_error_on_exception(self, mock_client): class TestFS26KnowledgeBaseVpcAccess: """FS-26 — Knowledge Base VPC Access.""" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_error_on_exception(self, mock_client): mock_client.side_effect = RuntimeError("vpc error") result = app.check_knowledge_base_vpc_access() @@ -1372,7 +1361,7 @@ def test_error_on_exception(self): class TestFS27AutomatedReasoningPolicies: """FS-27b — Automated Reasoning Policies Check (new, GA August 2025).""" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_policies_exist(self, mock_client): c = MagicMock() c.list_automated_reasoning_policies.return_value = { @@ -1385,7 +1374,7 @@ def test_pass_policies_exist(self, mock_client): _assert_finding_structure(result) assert result["status"] == "PASS" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_warn_no_policies(self, mock_client): c = MagicMock() c.list_automated_reasoning_policies.return_value = { @@ -1396,7 +1385,7 @@ def test_warn_no_policies(self, mock_client): _assert_finding_structure(result) assert result["status"] == "WARN" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_access_denied_returns_na(self, mock_client): c = MagicMock() c.list_automated_reasoning_policies.side_effect = _client_error( @@ -1407,7 +1396,7 @@ def test_access_denied_returns_na(self, mock_client): _assert_finding_structure(result) assert any(r["Status"] == "N/A" for r in result["csv_data"]) - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_error_on_exception(self, mock_client): mock_client.side_effect = RuntimeError("arc error") result = app.check_automated_reasoning_policies() @@ -1549,7 +1538,7 @@ def test_na_no_kbs(self): class TestFS34FmVersionCurrency: """FS-34 — FM Version Currency Advisory.""" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_error_on_exception(self, mock_client): mock_client.side_effect = RuntimeError("fm error") result = app.check_fm_version_currency() @@ -1649,7 +1638,7 @@ def test_error_on_exception(self): class TestFS39SagemakerClarifyBias: """FS-39 — SageMaker Clarify Bias.""" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_error_on_exception(self, mock_client): mock_client.side_effect = RuntimeError("clarify error") result = app.check_sagemaker_clarify_bias() @@ -1669,7 +1658,7 @@ def test_returns_advisory_structure(self): class TestFS41SagemakerClarifyExplainability: """FS-41 — SageMaker Clarify Explainability.""" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_error_on_exception(self, mock_client): mock_client.side_effect = RuntimeError("explain error") result = app.check_sagemaker_clarify_explainability() @@ -1679,7 +1668,7 @@ def test_error_on_exception(self, mock_client): class TestFS42AiServiceCards: """FS-42 — AI Service Cards Documentation Advisory.""" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_returns_valid_structure(self, mock_client): c = MagicMock() c.list_model_cards.return_value = {"ModelCardSummaries": []} @@ -1687,7 +1676,7 @@ def test_returns_valid_structure(self, mock_client): result = app.check_ai_service_cards_documentation() _assert_finding_structure(result) - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_error_on_exception(self, mock_client): mock_client.side_effect = RuntimeError("cards error") result = app.check_ai_service_cards_documentation() @@ -1702,7 +1691,7 @@ def test_error_on_exception(self, mock_client): class TestFS43CloudwatchLogPiiMasking: """FS-43 — CloudWatch Log PII Masking.""" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_error_on_exception(self, mock_client): mock_client.side_effect = RuntimeError("logs error") result = app.check_cloudwatch_log_pii_masking() @@ -1712,7 +1701,7 @@ def test_error_on_exception(self, mock_client): class TestFS44MacieOnTrainingDataBuckets: """FS-44 — Macie on Training Data Buckets.""" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_error_on_exception(self, mock_client): mock_client.side_effect = RuntimeError("macie error") result = app.check_macie_on_training_data_buckets() @@ -2381,7 +2370,7 @@ def test_returns_valid_structure(self): class TestFS61KnowledgeBaseSyncSchedule: """FS-61 — Knowledge Base Sync Schedule Check.""" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_sync_rules_exist(self, mock_client): inv = make_resource_inventory( knowledge_bases=app.KbInventory( @@ -2409,7 +2398,7 @@ def side_effect(service, **kwargs): _assert_finding_structure(result) assert result["status"] == "PASS" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_scheduler_schedule_exists(self, mock_client): # No legacy EventBridge rule, but an EventBridge Scheduler schedule targets # KB sync — the AWS-recommended approach must be detected (no false WARN). @@ -2446,7 +2435,7 @@ def side_effect(service, **kwargs): _assert_finding_structure(result) assert result["status"] == "PASS" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_warn_no_sync_rules(self, mock_client): inv = make_resource_inventory( knowledge_bases=app.KbInventory( @@ -2476,7 +2465,7 @@ def side_effect(service, **kwargs): _assert_finding_structure(result) assert result["status"] == "WARN" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_scheduler_access_denied_falls_back_to_rules(self, mock_client): # scheduler:ListSchedules denied → fall back to EventBridge rules only, # do NOT error the whole check. @@ -2507,7 +2496,7 @@ def side_effect(service, **kwargs): # EventBridge rule still matched → PASS despite scheduler access denial. assert result["status"] == "PASS" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_scheduler_access_denied_no_rules_could_not_assess(self, mock_client): """REQ-11/A3: scheduler:ListSchedules denied AND no matching EventBridge rule → we cannot conclude absence → COULD_NOT_ASSESS (check returns ERROR @@ -2559,7 +2548,7 @@ def test_returns_valid_structure(self): class TestFS63FoundationModelLifecyclePolicy: """FS-63 — Foundation Model Lifecycle Policy Check.""" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_no_legacy_models(self, mock_client): c = MagicMock() c.list_foundation_models.return_value = { @@ -2575,7 +2564,7 @@ def test_pass_no_legacy_models(self, mock_client): result = app.check_foundation_model_lifecycle_policy() _assert_finding_structure(result) - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_warn_legacy_models_no_rules(self, mock_client): c = MagicMock() c.list_foundation_models.return_value = { @@ -2589,7 +2578,7 @@ def test_warn_legacy_models_no_rules(self, mock_client): _assert_finding_structure(result) assert result["status"] == "WARN" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_error_on_exception(self, mock_client): mock_client.side_effect = RuntimeError("lifecycle error") result = app.check_foundation_model_lifecycle_policy() @@ -2614,7 +2603,7 @@ def test_na_no_kbs(self): _assert_finding_structure(result) assert any(r["Status"] == "N/A" for r in result["csv_data"]) - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_notifications_configured(self, mock_client): inv = make_resource_inventory( knowledge_bases=app.KbInventory( @@ -2648,7 +2637,7 @@ def side_effect(service, **kwargs): _assert_finding_structure(result) assert result["status"] == "PASS" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_warn_no_notifications(self, mock_client): inv = make_resource_inventory( knowledge_bases=app.KbInventory( @@ -2691,7 +2680,7 @@ def test_error_on_unavailable_inventory(self): class TestFS66AgentcoreEndUserIdentityPropagation: """FS-66 — AgentCore End-User Identity Propagation Check.""" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_authorizer_configured(self, mock_client): c = MagicMock() c.list_agent_runtimes.return_value = { @@ -2709,7 +2698,7 @@ def test_pass_authorizer_configured(self, mock_client): _assert_finding_structure(result) assert result["status"] == "PASS" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_warn_no_authorizer(self, mock_client): c = MagicMock() c.list_agent_runtimes.return_value = { @@ -2722,7 +2711,7 @@ def test_warn_no_authorizer(self, mock_client): _assert_finding_structure(result) assert result["status"] == "WARN" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_access_denied_returns_na(self, mock_client): c = MagicMock() c.list_agent_runtimes.side_effect = _client_error("AccessDeniedException") @@ -2731,7 +2720,7 @@ def test_access_denied_returns_na(self, mock_client): _assert_finding_structure(result) assert any(r["Status"] == "N/A" for r in result["csv_data"]) - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_error_on_exception(self, mock_client): mock_client.side_effect = RuntimeError("identity error") result = app.check_agentcore_end_user_identity_propagation() @@ -2811,7 +2800,7 @@ def test_pass_validators_and_waf_rules(self): detail_by_id={"id1": acl_detail}, ) ) - with patch("app.boto3.client") as mock_client: + with patch("finserv_app.boto3.client") as mock_client: def side_effect(service, **kwargs): if service == "apigateway": @@ -2835,7 +2824,7 @@ def test_na_no_rest_apis_no_waf(self): inv = make_resource_inventory( web_acls=app.WebAclInventory(summaries=[], detail_by_id={}) ) - with patch("app.boto3.client") as mock_client: + with patch("finserv_app.boto3.client") as mock_client: def side_effect(service, **kwargs): if service == "apigateway": @@ -2860,7 +2849,7 @@ def test_fail_rest_api_without_validator(self): inv = make_resource_inventory( web_acls=app.WebAclInventory(summaries=[], detail_by_id={}) ) - with patch("app.boto3.client") as mock_client: + with patch("finserv_app.boto3.client") as mock_client: def side_effect(service, **kwargs): if service == "apigateway": @@ -2887,7 +2876,7 @@ def test_validator_presence_without_size_model_not_passed(self): inv = make_resource_inventory( web_acls=app.WebAclInventory(summaries=[], detail_by_id={}) ) - with patch("app.boto3.client") as mock_client: + with patch("finserv_app.boto3.client") as mock_client: def side_effect(service, **kwargs): if service == "apigateway": @@ -2925,7 +2914,7 @@ def test_validator_with_maxlength_model_passed(self): inv = make_resource_inventory( web_acls=app.WebAclInventory(summaries=[], detail_by_id={}) ) - with patch("app.boto3.client") as mock_client: + with patch("finserv_app.boto3.client") as mock_client: def side_effect(service, **kwargs): if service == "apigateway": @@ -2978,7 +2967,7 @@ def test_waf_oversize_constraint_above_window_not_credited(self): detail_by_id={"id1": bad_detail}, ) ) - with patch("app.boto3.client") as mock_client: + with patch("finserv_app.boto3.client") as mock_client: def side_effect(service, **kwargs): if service == "apigateway": @@ -3018,7 +3007,7 @@ def test_waf_body_substring_rule_not_credited(self): detail_by_id={"id1": xss_detail}, ) ) - with patch("app.boto3.client") as mock_client: + with patch("finserv_app.boto3.client") as mock_client: def side_effect(service, **kwargs): if service == "apigateway": @@ -3041,7 +3030,7 @@ def test_error_on_unavailable_inventory(self): inv = make_resource_inventory( web_acls=app._Unavailable(RuntimeError("apigw error")) ) - with patch("app.boto3.client") as mock_client: + with patch("finserv_app.boto3.client") as mock_client: def side_effect(service, **kwargs): if service == "apigateway": @@ -3059,7 +3048,7 @@ def test_error_on_apigw_exception(self): inv = make_resource_inventory( web_acls=app.WebAclInventory(summaries=[], detail_by_id={}) ) - with patch("app.boto3.client") as mock_client: + with patch("finserv_app.boto3.client") as mock_client: mock_client.side_effect = RuntimeError("apigw error") result = app.check_api_gateway_request_body_size_limits(inv) assert result["status"] == "ERROR" @@ -3073,7 +3062,7 @@ def test_pass_more_than_100_acls_with_size_constraints(self): inv = make_resource_inventory( web_acls=app.WebAclInventory(summaries=summaries, detail_by_id=detail_by_id) ) - with patch("app.boto3.client") as mock_client: + with patch("finserv_app.boto3.client") as mock_client: def side_effect(service, **kwargs): if service == "apigateway": @@ -3090,7 +3079,7 @@ def side_effect(service, **kwargs): # --- ≤1-page equivalence: 2-ACL case matches Wave-0 baseline --- def test_two_acl_case_unchanged(self): """The 2-ACL PASS scenario is unchanged vs the pre-refactor baseline.""" - from test_inventory_equivalence import _acl_detail as _baseline_acl_detail + from .test_inventory_equivalence import _acl_detail as _baseline_acl_detail inv = make_resource_inventory( web_acls=app.WebAclInventory( @@ -3108,7 +3097,7 @@ def test_two_acl_case_unchanged(self): }, ) ) - with patch("app.boto3.client") as mock_client: + with patch("finserv_app.boto3.client") as mock_client: def side_effect(service, **kwargs): if service == "apigateway": @@ -3218,14 +3207,17 @@ def test_registry_covers_all_checks_in_order(self): # The two permission-cache checks are present. assert "FS-07" in ids and "FS-22" in ids - @patch("app.write_to_s3", return_value="https://example.com/report.csv") + @patch("finserv_app.write_to_s3", return_value="https://example.com/report.csv") @patch.dict(os.environ, {"AIML_ASSESSMENT_BUCKET_NAME": "test-bucket"}) - @patch("app.get_permissions_cache", return_value=None) - @patch("app.build_finserv_checks") + @patch("finserv_app.get_permissions_cache", return_value=None) + @patch("finserv_app.collect_resource_inventory") + @patch("finserv_app.build_finserv_checks") def test_errored_check_emits_could_not_assess_row( - self, mock_build, mock_cache, mock_write + self, mock_build, mock_collect, mock_cache, mock_write ): # One check raises (uncaught) → handler synthesizes one could-not-assess row. + mock_collect.return_value = make_resource_inventory() + def boom(): return app._error_findings("Boom Check", RuntimeError("AccessDenied: nope")) @@ -3263,17 +3255,20 @@ def normal(): assert len(normal_result["csv_data"]) == 1 assert normal_result["csv_data"][0]["Finding"] == "Normal Finding" - @patch("app.write_to_s3", return_value="https://example.com/report.csv") + @patch("finserv_app.write_to_s3", return_value="https://example.com/report.csv") @patch.dict(os.environ, {"AIML_ASSESSMENT_BUCKET_NAME": "test-bucket"}) - @patch("app.get_permissions_cache", return_value=None) - @patch("app.build_finserv_checks") + @patch("finserv_app.get_permissions_cache", return_value=None) + @patch("finserv_app.collect_resource_inventory") + @patch("finserv_app.build_finserv_checks") def test_non_error_empty_result_still_emits_could_not_assess_row( - self, mock_build, mock_cache, mock_write + self, mock_build, mock_collect, mock_cache, mock_write ): # A check that returns a NON-error wrapper status but zero csv_data must # NOT silently vanish — the handler synthesizes a could-not-assess row for # any empty result, not only ERROR ones. This guards the no-silent-drop # invariant structurally (Property 7) rather than by data coincidence. + mock_collect.return_value = make_resource_inventory() + def empty_pass(): return {"check_name": "Empty Pass Check", "status": "PASS", "csv_data": []} @@ -3289,13 +3284,17 @@ def empty_pass(): assert row["Severity"] == "Low" assert row["Finding"].startswith(app.COULD_NOT_ASSESS_PREFIX) - @patch("app.write_to_s3", return_value="https://example.com/report.csv") + @patch("finserv_app.write_to_s3", return_value="https://example.com/report.csv") @patch.dict(os.environ, {"AIML_ASSESSMENT_BUCKET_NAME": "test-bucket"}) - @patch("app.get_permissions_cache", return_value=None) - def test_no_check_contributes_zero_rows(self, mock_cache, mock_write): + @patch("finserv_app.get_permissions_cache", return_value=None) + @patch("finserv_app.collect_resource_inventory") + def test_no_check_contributes_zero_rows(self, mock_collect, mock_cache, mock_write): # Full handler run with mocked AWS (all clients raise) — every check must # still contribute at least one row (real rows or a could-not-assess row). - with patch("app.boto3.client", side_effect=RuntimeError("AccessDenied")): + mock_collect.return_value = make_resource_inventory() + with patch( + "finserv_app.boto3.client", side_effect=RuntimeError("AccessDenied") + ): resp = app.lambda_handler({"Execution": {"Name": "exec-2"}}, None) findings = resp["body"]["findings"] for f in findings: diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_checks_coverage.py b/aiml-security-assessment/functions/security/finserv_tests/test_checks_coverage.py index d948cac..c45da7a 100644 --- a/aiml-security-assessment/functions/security/finserv_tests/test_checks_coverage.py +++ b/aiml-security-assessment/functions/security/finserv_tests/test_checks_coverage.py @@ -6,24 +6,12 @@ """ import json -import os -import sys from datetime import datetime, timezone, timedelta -from unittest.mock import MagicMock, patch from botocore.exceptions import ClientError +from unittest.mock import MagicMock, patch - -FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") -if FINSERV_DIR not in sys.path: - sys.path.insert(0, FINSERV_DIR) - -# Ensure tests/ directory is importable (for conftest helpers) -TESTS_DIR = os.path.dirname(__file__) -if TESTS_DIR not in sys.path: - sys.path.insert(0, TESTS_DIR) - -import app # noqa: E402 (import must follow sys.path setup above) -from conftest import make_resource_inventory # noqa: E402 +from .support import finserv_app as app +from .support import make_resource_inventory def _client_error(code="AccessDeniedException", message="Access Denied"): @@ -50,7 +38,7 @@ def test_shield_generic_client_error_treated_as_no_shield(self): detail_by_id={}, ) ) - with patch("app.boto3.client") as mock_client: + with patch("finserv_app.boto3.client") as mock_client: def side_effect(service, **kwargs): if service == "shield": @@ -80,7 +68,7 @@ def side_effect(service, **kwargs): class TestFS07AgentBoundariesNewPaths: - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_get_agent_client_error_skips_agent(self, mock_client): """Lines 532-534: get_agent raises ClientError → agent is skipped gracefully.""" c = MagicMock() @@ -94,7 +82,7 @@ def test_get_agent_client_error_skips_agent(self, mock_client): # Should PASS (no issues found, agent was skipped) assert result["status"] == "PASS" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_agent_no_role_arn_skipped(self, mock_client): """Line 537: agent with no agentResourceRoleArn → continue.""" c = MagicMock() @@ -107,7 +95,7 @@ def test_agent_no_role_arn_skipped(self, mock_client): _assert_structure(result) assert result["status"] == "PASS" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_policy_doc_as_string_is_parsed(self, mock_client): """Line 543: policy document stored as JSON string → json.loads branch.""" c = MagicMock() @@ -144,7 +132,7 @@ def test_policy_doc_as_string_is_parsed(self, mock_client): _assert_structure(result) assert result["status"] == "PASS" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_deny_effect_statement_skipped(self, mock_client): """Line 546: Deny effect → continue (not counted as issue).""" c = MagicMock() @@ -186,7 +174,7 @@ def test_deny_effect_statement_skipped(self, mock_client): class TestFS08AgentcoreReraise: - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_non_access_denied_error_propagates_to_outer_except(self, mock_client): """Line 622: ClientError that is NOT AccessDenied/Unrecognized → re-raised → ERROR.""" c = MagicMock() @@ -203,7 +191,7 @@ def test_non_access_denied_error_propagates_to_outer_except(self, mock_client): class TestFS09ConcurrencyClientError: - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_get_concurrency_client_error_adds_to_warn_list(self, mock_client): """Lines 704-705: get_function_concurrency raises ClientError → appended to warn list.""" c = MagicMock() @@ -223,7 +211,7 @@ def test_get_concurrency_client_error_adds_to_warn_list(self, mock_client): class TestFS12ScpPaths: - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_access_denied_returns_na(self, mock_client): """Lines 902-915: AccessDeniedException → N/A finding.""" c = MagicMock() @@ -233,7 +221,7 @@ def test_access_denied_returns_na(self, mock_client): _assert_structure(result) assert any(r["Status"] == "N/A" for r in result["csv_data"]) - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_orgs_not_in_use_returns_na(self, mock_client): """Lines 902-915: AWSOrganizationsNotInUseException → N/A finding.""" c = MagicMock() @@ -243,7 +231,7 @@ def test_orgs_not_in_use_returns_na(self, mock_client): _assert_structure(result) assert any(r["Status"] == "N/A" for r in result["csv_data"]) - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_non_access_denied_reraises(self, mock_client): """Line 916: non-AccessDenied ClientError → re-raised → ERROR.""" c = MagicMock() @@ -253,7 +241,7 @@ def test_non_access_denied_reraises(self, mock_client): _assert_structure(result) assert result["status"] == "ERROR" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_warn_no_bedrock_scps(self, mock_client): """Lines 920-923, 925-945: policies exist but none reference bedrock → WARN.""" c = MagicMock() @@ -272,7 +260,7 @@ def test_warn_no_bedrock_scps(self, mock_client): _assert_structure(result) assert result["status"] == "WARN" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_bedrock_scp_found(self, mock_client): """Line 947: bedrock SCP found → Passed finding.""" c = MagicMock() @@ -310,7 +298,7 @@ def test_pass_bedrock_scp_found(self, mock_client): class TestFS13ModelTaggingPaths: - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_warn_bedrock_model_missing_tags(self, mock_client): """Lines 979-983, 998-999: Bedrock custom model missing required tags → WARN.""" @@ -338,7 +326,7 @@ def side_effect(service, **kwargs): _assert_structure(result) assert result["status"] == "WARN" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_warn_sagemaker_model_missing_tags(self, mock_client): """Lines 989-993: SageMaker model missing required tags → WARN.""" @@ -373,7 +361,7 @@ def side_effect(service, **kwargs): class TestFS14ModelGovernancePass: - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_config_rules_found(self, mock_client): """Line 1072: bedrock-related Config rules found → Passed.""" c = MagicMock() @@ -392,7 +380,7 @@ def test_pass_config_rules_found(self, mock_client): class TestFS15BedrockEvalPass: - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_eval_jobs_found(self, mock_client): """Line 1119: evaluation jobs exist → Passed finding.""" c = MagicMock() @@ -411,7 +399,7 @@ def test_pass_eval_jobs_found(self, mock_client): class TestFS16EcrScanningWarn: - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_warn_repos_without_scanning(self, mock_client): """Lines 1167-1168: repos exist but scan-on-push disabled → WARN.""" c = MagicMock() @@ -435,7 +423,7 @@ def test_warn_repos_without_scanning(self, mock_client): class TestFS20FeatureStoreWarnPass: - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_warn_groups_without_offline_store(self, mock_client): """Lines 1244-1246: feature groups without active offline store → WARN.""" c = MagicMock() @@ -452,7 +440,7 @@ def test_warn_groups_without_offline_store(self, mock_client): _assert_structure(result) assert result["status"] == "WARN" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_all_groups_have_offline_store(self, mock_client): """Line 1266: all feature groups have active offline store → Passed.""" c = MagicMock() @@ -476,7 +464,7 @@ def test_pass_all_groups_have_offline_store(self, mock_client): class TestFS21TrainingDataVersioningPaths: - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_warn_unversioned_training_buckets(self, mock_client): """Lines 1312-1316, 1318-1320: training buckets without versioning → WARN.""" inv = make_resource_inventory(buckets=[{"Name": "training-data-bucket"}]) @@ -491,7 +479,7 @@ def test_warn_unversioned_training_buckets(self, mock_client): _assert_structure(result) assert result["status"] == "WARN" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_all_training_buckets_versioned(self, mock_client): """Line 1338: all training buckets versioned → Passed.""" inv = make_resource_inventory(buckets=[{"Name": "training-data-bucket"}]) @@ -506,7 +494,7 @@ def test_pass_all_training_buckets_versioned(self, mock_client): _assert_structure(result) assert any(r["Status"] == "Passed" for r in result["csv_data"]) - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_access_error_surfaces_as_could_not_assess(self, mock_client): """AccessDenied on get_bucket_versioning re-raises → ERROR (could-not-assess), not a false 'no versioning' finding.""" @@ -519,7 +507,7 @@ def test_access_error_surfaces_as_could_not_assess(self, mock_client): _assert_structure(result) assert result["status"] == "ERROR" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_nonaccess_error_flags_bucket(self, mock_client): """A non-access ClientError on get_bucket_versioning flags the bucket (WARN) without aborting the whole check.""" @@ -535,7 +523,7 @@ def test_nonaccess_error_flags_bucket(self, mock_client): "(error)" in r.get("Finding_Details", "") for r in result["csv_data"] ) - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_multi_page_buckets_completeness(self, mock_client): """Pagination completeness: buckets from multiple pages are all checked. @@ -566,7 +554,7 @@ def test_multi_page_buckets_completeness(self, mock_client): # Verify all 4 were checked (versioning call per training bucket) assert c.get_bucket_versioning.call_count == 4 - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_single_page_unchanged_vs_baseline(self, mock_client): """Single-page case: result is identical to pre-migration behavior (baseline). @@ -591,8 +579,8 @@ def test_single_page_unchanged_vs_baseline(self, mock_client): class TestFS22KbIamWarnPath: - def test_warn_wildcard_bedrock_agent_permission(self): - """Lines 1370-1386: role with bedrock-agent:* → WARN.""" + def test_invalid_bedrock_agent_namespace_not_treated_as_kb_access(self): + """bedrock-agent is a boto3 client name, not an IAM action namespace.""" cache = { "role_permissions": { "KBAccessRole": { @@ -615,7 +603,7 @@ def test_warn_wildcard_bedrock_agent_permission(self): } result = app.check_knowledge_base_iam_least_privilege(cache) _assert_structure(result) - assert result["status"] == "WARN" + assert result["status"] == "PASS" def test_warn_wildcard_bedrock_permission(self): """Lines 1370-1386: role with bedrock:* → WARN.""" @@ -682,7 +670,7 @@ def test_deny_effect_not_flagged(self): "Statement": [ { "Effect": "Deny", - "Action": "bedrock-agent:*", + "Action": "bedrock:*", "Resource": "*", } ] @@ -755,7 +743,7 @@ def test_pass_kbs_exist(self): class TestFS25OssEncryptionPaths: - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_policies_with_cmk(self, mock_client): """Lines 1509-1511: encryption policies exist with CMK → Passed.""" c = MagicMock() @@ -774,7 +762,7 @@ def test_pass_policies_with_cmk(self, mock_client): _assert_structure(result) assert any(r["Status"] == "Passed" for r in result["csv_data"]) - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_no_policies(self, mock_client): """Line 1488: no encryption policies → N/A finding.""" c = MagicMock() @@ -784,7 +772,7 @@ def test_pass_no_policies(self, mock_client): _assert_structure(result) assert any(r["Status"] == "N/A" for r in result["csv_data"]) - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_fail_policies_without_cmk(self, mock_client): """Encryption policies exist but all use AWS-owned keys → WARN/Failed (the customer-managed-key control is absent), not a false Pass.""" @@ -812,7 +800,7 @@ def test_fail_policies_without_cmk(self, mock_client): class TestFS26VpcAccessPaths: - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_warn_no_network_policies(self, mock_client): """Lines 1546-1547: no network policies → WARN.""" c = MagicMock() @@ -822,7 +810,7 @@ def test_warn_no_network_policies(self, mock_client): _assert_structure(result) assert result["status"] == "WARN" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_warn_policies_without_vpc(self, mock_client): """Lines 1567-1569: network policies exist but no VPC restriction → WARN.""" c = MagicMock() @@ -839,7 +827,7 @@ def test_warn_policies_without_vpc(self, mock_client): _assert_structure(result) assert result["status"] == "WARN" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_vpc_restricted_policies(self, mock_client): """Line 1591: network policies with VPC restriction → Passed.""" c = MagicMock() @@ -988,7 +976,7 @@ def test_pass_recently_synced(self): class TestFS33KbIntegrityPaths: - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_warn_bucket_without_versioning(self, mock_client): """Lines 1976-2003: KB data source bucket without versioning → WARN.""" inv = make_resource_inventory( @@ -1021,7 +1009,7 @@ def side_effect(service, **kwargs): _assert_structure(result) assert result["status"] == "WARN" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_bucket_with_versioning(self, mock_client): """Line 2021: all KB buckets have versioning → Passed.""" inv = make_resource_inventory( @@ -1054,7 +1042,7 @@ def side_effect(service, **kwargs): _assert_structure(result) assert any(r["Status"] == "Passed" for r in result["csv_data"]) - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_deleted_bucket_reported_separately(self, mock_client): """A NoSuchBucket on get_bucket_versioning → distinct 'deleted bucket' finding (High), NOT conflated with 'without versioning' or labeled @@ -1106,7 +1094,7 @@ def side_effect(service, **kwargs): for r in result["csv_data"] ) - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_bucket_nonaccess_nonmissing_error_treated_as_unversioned( self, mock_client ): @@ -1146,7 +1134,7 @@ def side_effect(service, **kwargs): "(error)" in r.get("Finding_Details", "") for r in result["csv_data"] ) - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_bucket_access_error_surfaces_as_could_not_assess(self, mock_client): """An AccessDenied on get_bucket_versioning must re-raise → ERROR envelope (could-not-assess), NOT a false 'no versioning' finding.""" @@ -1187,7 +1175,7 @@ def side_effect(service, **kwargs): class TestFS34FmVersionPass: - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_no_legacy_models_in_use(self, mock_client): """Lines 2056-2057: no legacy models in use → Passed.""" c = MagicMock() @@ -1289,7 +1277,7 @@ def test_pass_word_filters_configured(self): class TestFS39ClarifyBiasPass: - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_bias_schedules_found(self, mock_client): """Line 2352: bias monitoring schedules found → Passed.""" c = MagicMock() @@ -1331,7 +1319,7 @@ def test_advisory_finding(self): class TestFS41ClarifyExplainabilityPass: - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_explainability_schedules_found(self, mock_client): """Line 2452: explainability monitoring schedules found → Passed.""" c = MagicMock() @@ -1355,7 +1343,7 @@ def test_pass_explainability_schedules_found(self, mock_client): class TestFS42ModelCardsPass: - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_model_cards_found(self, mock_client): """Line 2501: model cards exist → Passed (key is ModelCardSummaryList).""" c = MagicMock() @@ -1374,7 +1362,7 @@ def test_pass_model_cards_found(self, mock_client): class TestFS43CloudwatchPiiPaths: - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_warn_no_data_protection_policies(self, mock_client): """Lines 2540-2541: no data protection policies → WARN.""" c = MagicMock() @@ -1384,7 +1372,7 @@ def test_warn_no_data_protection_policies(self, mock_client): _assert_structure(result) assert result["status"] == "WARN" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_client_error_on_describe_policies_treated_as_no_policies( self, mock_client ): @@ -1403,7 +1391,7 @@ def test_client_error_on_describe_policies_treated_as_no_policies( class TestFS44MaciePaths: - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_warn_macie_not_enabled(self, mock_client): """Lines 2593-2595: Macie session status not ENABLED → WARN.""" c = MagicMock() @@ -1413,7 +1401,7 @@ def test_warn_macie_not_enabled(self, mock_client): _assert_structure(result) assert result["status"] == "WARN" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_warn_macie_client_error(self, mock_client): """Lines 2590-2591: ClientError on get_macie_session → macie_enabled=False → WARN.""" c = MagicMock() @@ -1423,7 +1411,7 @@ def test_warn_macie_client_error(self, mock_client): _assert_structure(result) assert result["status"] == "WARN" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_macie_enabled(self, mock_client): """Line 2615: Macie enabled → Passed.""" c = MagicMock() @@ -1461,7 +1449,7 @@ def test_warn_guardrails_without_pii_filters(self): class TestFS46DataClassificationPaths: - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_warn_buckets_without_classification_tags(self, mock_client): """Lines 2734-2746: AI/ML buckets without classification tags → WARN.""" inv = make_resource_inventory(buckets=[{"Name": "aiml-training-data"}]) @@ -1475,7 +1463,7 @@ def test_warn_buckets_without_classification_tags(self, mock_client): _assert_structure(result) assert result["status"] == "WARN" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_warn_tagging_client_error_treated_as_unclassified(self, mock_client): """Line 2741-2742: ClientError on get_bucket_tagging → bucket added as unclassified.""" inv = make_resource_inventory(buckets=[{"Name": "aiml-training-data"}]) @@ -1487,7 +1475,7 @@ def test_warn_tagging_client_error_treated_as_unclassified(self, mock_client): _assert_structure(result) assert result["status"] == "WARN" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_tagging_access_error_surfaces_as_could_not_assess(self, mock_client): """AccessDenied on get_bucket_tagging re-raises → ERROR (could-not-assess), NOT a false 'unclassified' finding.""" @@ -1500,7 +1488,7 @@ def test_tagging_access_error_surfaces_as_could_not_assess(self, mock_client): _assert_structure(result) assert result["status"] == "ERROR" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_pass_all_buckets_classified(self, mock_client): """Line 2765: all AI/ML buckets have classification tags → Passed.""" inv = make_resource_inventory(buckets=[{"Name": "aiml-training-data"}]) @@ -1514,7 +1502,7 @@ def test_pass_all_buckets_classified(self, mock_client): _assert_structure(result) assert any(r["Status"] == "Passed" for r in result["csv_data"]) - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_multi_page_buckets_completeness(self, mock_client): """Pagination completeness: buckets from multiple pages are all assessed. @@ -1540,7 +1528,7 @@ def test_multi_page_buckets_completeness(self, mock_client): # Verify all 4 were assessed (tagging call per AI/ML bucket) assert c.get_bucket_tagging.call_count == 4 - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_single_page_unchanged_vs_baseline(self, mock_client): """Single-page case: result is identical to pre-migration behavior (baseline). @@ -1801,7 +1789,7 @@ def test_skip_datasource_with_no_bucket(self): # No buckets to check → PASS assert result["status"] == "PASS" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_s3_notification_access_error_surfaces_as_could_not_assess( self, mock_client ): @@ -1839,7 +1827,7 @@ def side_effect(service, **kwargs): _assert_structure(result) assert result["status"] == "ERROR" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_deleted_bucket_reported_separately(self, mock_client): """A NoSuchBucket on get_bucket_notification_configuration → distinct 'deleted bucket' finding (High), not conflated with 'missing @@ -1894,7 +1882,7 @@ def side_effect(service, **kwargs): class TestFS66AgentcoreIdentityReraise: - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_non_access_denied_reraises(self, mock_client): """Line 3849: non-AccessDenied ClientError → re-raised → ERROR.""" c = MagicMock() @@ -1916,7 +1904,7 @@ def test_warn_rest_api_without_validators(self): inv = make_resource_inventory( web_acls=app.WebAclInventory(summaries=[], detail_by_id={}) ) - with patch("app.boto3.client") as mock_client: + with patch("finserv_app.boto3.client") as mock_client: def side_effect(service, **kwargs): if service == "apigateway": @@ -1951,7 +1939,7 @@ def test_warn_waf_acls_without_size_rules(self): detail_by_id={"id1": acl_detail}, ) ) - with patch("app.boto3.client") as mock_client: + with patch("finserv_app.boto3.client") as mock_client: def side_effect(service, **kwargs): if service == "apigateway": @@ -1972,7 +1960,7 @@ def side_effect(service, **kwargs): class TestFS34FmVersionWarn: - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_warn_legacy_models_available(self, mock_client): """Legacy foundation models available in region → WARN wrapper with an N/A finding (availability is not usage, so it is surfaced for review, not failed).""" diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_collector.py b/aiml-security-assessment/functions/security/finserv_tests/test_collector.py index ecf603b..7af722a 100644 --- a/aiml-security-assessment/functions/security/finserv_tests/test_collector.py +++ b/aiml-security-assessment/functions/security/finserv_tests/test_collector.py @@ -4,17 +4,11 @@ Validates: Requirements REQ-1, REQ-2, REQ-3.4, REQ-4, REQ-7.5, REQ-9.2 """ -import sys -import os from unittest.mock import MagicMock, patch import pytest -FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") -if FINSERV_DIR not in sys.path: - sys.path.insert(0, FINSERV_DIR) - -import app # noqa: E402 (import follows sys.path bootstrap above) +from .support import finserv_app as app # --------------------------------------------------------------------------- @@ -54,13 +48,13 @@ class TestSafeCollectLambdaFunctions: def test_single_page_returns_functions(self): fn = {"FunctionName": "my-fn", "Runtime": "python3.12"} client = _make_client({"list_functions": {"Functions": [fn]}}) - with patch("app.boto3.client", return_value=client): + with patch("finserv_app.boto3.client", return_value=client): result = app._safe_collect_lambda_functions() assert result == [fn] def test_single_listing_call(self): client = _make_client({"list_functions": {"Functions": []}}) - with patch("app.boto3.client", return_value=client): + with patch("finserv_app.boto3.client", return_value=client): app._safe_collect_lambda_functions() assert client.list_functions.call_count == 1 @@ -76,7 +70,7 @@ def test_multi_page_merges_all_functions(self): ] } ) - with patch("app.boto3.client", return_value=client): + with patch("finserv_app.boto3.client", return_value=client): result = app._safe_collect_lambda_functions() assert result == [fn1, fn2] assert client.list_functions.call_count == 2 @@ -88,14 +82,14 @@ def test_failure_returns_unavailable(self): err = PermissionError("AccessDenied") client = _make_client({}) client.list_functions.side_effect = err - with patch("app.boto3.client", return_value=client): + with patch("finserv_app.boto3.client", return_value=client): result = app._safe_collect_lambda_functions() assert isinstance(result, app._Unavailable) assert result.error is err def test_client_constructed_without_region_or_endpoint(self): client = _make_client({"list_functions": {"Functions": []}}) - with patch("app.boto3.client", return_value=client) as mock_boto: + with patch("finserv_app.boto3.client", return_value=client) as mock_boto: app._safe_collect_lambda_functions() mock_boto.assert_called_once() _, kwargs = mock_boto.call_args @@ -118,7 +112,7 @@ def test_single_page_returns_guardrail_inventory(self): "get_guardrail": detail, } ) - with patch("app.boto3.client", return_value=client): + with patch("finserv_app.boto3.client", return_value=client): result = app._safe_collect_guardrails() assert isinstance(result, app.GuardrailInventory) assert result.summaries == [g1] @@ -130,7 +124,7 @@ def test_single_listing_call(self): "list_guardrails": {"guardrails": []}, } ) - with patch("app.boto3.client", return_value=client): + with patch("finserv_app.boto3.client", return_value=client): app._safe_collect_guardrails() assert client.list_guardrails.call_count == 1 @@ -146,7 +140,7 @@ def test_multi_page_merges_guardrails(self): "get_guardrail": {"guardrailId": "gx"}, } ) - with patch("app.boto3.client", return_value=client): + with patch("finserv_app.boto3.client", return_value=client): result = app._safe_collect_guardrails() assert len(result.summaries) == 2 assert client.list_guardrails.call_count == 2 @@ -161,7 +155,7 @@ def test_get_guardrail_called_with_draft(self): "get_guardrail": detail, } ) - with patch("app.boto3.client", return_value=client): + with patch("finserv_app.boto3.client", return_value=client): app._safe_collect_guardrails() client.get_guardrail.assert_called_once_with( guardrailIdentifier="g-001", guardrailVersion="DRAFT" @@ -171,7 +165,7 @@ def test_whole_inventory_failure_returns_unavailable(self): err = PermissionError("AccessDenied on list_guardrails") client = _make_client({}) client.list_guardrails.side_effect = err - with patch("app.boto3.client", return_value=client): + with patch("finserv_app.boto3.client", return_value=client): result = app._safe_collect_guardrails() assert isinstance(result, app._Unavailable) assert result.error is err @@ -190,7 +184,7 @@ def detail_side_effect(**kwargs): client = _make_client({"list_guardrails": {"guardrails": [g1, g2]}}) client.get_guardrail.side_effect = detail_side_effect - with patch("app.boto3.client", return_value=client): + with patch("finserv_app.boto3.client", return_value=client): result = app._safe_collect_guardrails() assert isinstance(result, app.GuardrailInventory) assert result.detail_by_id["g-ok"] is ok_detail @@ -199,7 +193,7 @@ def detail_side_effect(**kwargs): def test_client_constructed_without_region_or_endpoint(self): client = _make_client({"list_guardrails": {"guardrails": []}}) - with patch("app.boto3.client", return_value=client) as mock_boto: + with patch("finserv_app.boto3.client", return_value=client) as mock_boto: app._safe_collect_guardrails() mock_boto.assert_called_once() _, kwargs = mock_boto.call_args @@ -245,7 +239,7 @@ def test_single_kb_single_ds(self): ds1 = {"dataSourceId": "ds-1"} detail = {"dataSource": {"dataSourceId": "ds-1"}} client = self._setup_client([kb1], {"kb-1": [ds1]}, {("kb-1", "ds-1"): detail}) - with patch("app.boto3.client", return_value=client): + with patch("finserv_app.boto3.client", return_value=client): result = app._safe_collect_knowledge_bases() assert isinstance(result, app.KbInventory) assert result.summaries == [kb1] @@ -254,7 +248,7 @@ def test_single_kb_single_ds(self): def test_single_listing_call(self): client = self._setup_client([]) - with patch("app.boto3.client", return_value=client): + with patch("finserv_app.boto3.client", return_value=client): app._safe_collect_knowledge_bases() assert client.list_knowledge_bases.call_count == 1 @@ -267,7 +261,7 @@ def test_multi_page_kbs(self): {"knowledgeBaseSummaries": [kb2]}, ] client.list_data_sources.return_value = {"dataSourceSummaries": []} - with patch("app.boto3.client", return_value=client): + with patch("finserv_app.boto3.client", return_value=client): result = app._safe_collect_knowledge_bases() assert len(result.summaries) == 2 assert client.list_knowledge_bases.call_count == 2 @@ -276,7 +270,7 @@ def test_whole_inventory_failure_returns_unavailable(self): err = PermissionError("denied") client = MagicMock() client.list_knowledge_bases.side_effect = err - with patch("app.boto3.client", return_value=client): + with patch("finserv_app.boto3.client", return_value=client): result = app._safe_collect_knowledge_bases() assert isinstance(result, app._Unavailable) assert result.error is err @@ -298,7 +292,7 @@ def list_ds(**kwargs): return {"dataSourceSummaries": []} client.list_data_sources.side_effect = list_ds - with patch("app.boto3.client", return_value=client): + with patch("finserv_app.boto3.client", return_value=client): result = app._safe_collect_knowledge_bases() assert isinstance(result, app.KbInventory) assert result.data_sources_by_kb["kb-ok"] == [] @@ -318,7 +312,7 @@ def test_per_data_source_detail_failure_isolates(self): {"kb-1": [ds_ok, ds_bad]}, {("kb-1", "ds-ok"): ok_detail, ("kb-1", "ds-bad"): err}, ) - with patch("app.boto3.client", return_value=client): + with patch("finserv_app.boto3.client", return_value=client): result = app._safe_collect_knowledge_bases() assert result.data_source_detail[("kb-1", "ds-ok")] is ok_detail assert isinstance( @@ -327,7 +321,7 @@ def test_per_data_source_detail_failure_isolates(self): def test_client_constructed_without_region_or_endpoint(self): client = self._setup_client([]) - with patch("app.boto3.client", return_value=client) as mock_boto: + with patch("finserv_app.boto3.client", return_value=client) as mock_boto: app._safe_collect_knowledge_bases() mock_boto.assert_called_once() _, kwargs = mock_boto.call_args @@ -344,13 +338,13 @@ class TestSafeCollectBuckets: def test_single_page_returns_buckets(self): b1 = {"Name": "my-bucket"} client = _make_client({"list_buckets": {"Buckets": [b1]}}) - with patch("app.boto3.client", return_value=client): + with patch("finserv_app.boto3.client", return_value=client): result = app._safe_collect_buckets() assert result == [b1] def test_single_listing_call(self): client = _make_client({"list_buckets": {"Buckets": []}}) - with patch("app.boto3.client", return_value=client): + with patch("finserv_app.boto3.client", return_value=client): app._safe_collect_buckets() assert client.list_buckets.call_count == 1 @@ -366,7 +360,7 @@ def test_multi_page_with_continuation_token(self): ] } ) - with patch("app.boto3.client", return_value=client): + with patch("finserv_app.boto3.client", return_value=client): result = app._safe_collect_buckets() assert result == [b1, b2] assert client.list_buckets.call_count == 2 @@ -377,7 +371,7 @@ def test_multi_page_with_continuation_token(self): def test_max_buckets_parameter_sent_on_first_call(self): """MaxBuckets must be included on the first call to engage pagination.""" client = _make_client({"list_buckets": {"Buckets": []}}) - with patch("app.boto3.client", return_value=client): + with patch("finserv_app.boto3.client", return_value=client): app._safe_collect_buckets() first_call_kwargs = client.list_buckets.call_args_list[0][1] assert first_call_kwargs.get("MaxBuckets") == 1000 @@ -386,14 +380,14 @@ def test_failure_returns_unavailable(self): err = PermissionError("AccessDenied") client = MagicMock() client.list_buckets.side_effect = err - with patch("app.boto3.client", return_value=client): + with patch("finserv_app.boto3.client", return_value=client): result = app._safe_collect_buckets() assert isinstance(result, app._Unavailable) assert result.error is err def test_client_constructed_without_region_or_endpoint(self): client = _make_client({"list_buckets": {"Buckets": []}}) - with patch("app.boto3.client", return_value=client) as mock_boto: + with patch("finserv_app.boto3.client", return_value=client) as mock_boto: app._safe_collect_buckets() mock_boto.assert_called_once() _, kwargs = mock_boto.call_args @@ -416,7 +410,7 @@ def test_single_page_returns_web_acl_inventory(self): "get_web_acl": detail, } ) - with patch("app.boto3.client", return_value=client): + with patch("finserv_app.boto3.client", return_value=client): result = app._safe_collect_web_acls() assert isinstance(result, app.WebAclInventory) assert result.summaries == [acl1] @@ -430,14 +424,14 @@ def test_single_listing_call(self): "list_web_acls": {"WebACLs": []}, } ) - with patch("app.boto3.client", return_value=client): + with patch("finserv_app.boto3.client", return_value=client): app._safe_collect_web_acls() assert client.list_web_acls.call_count == 1 def test_list_web_acls_called_with_scope_regional(self): """Scope='REGIONAL' must be passed on every list_web_acls call (REQ-7.2).""" client = _make_client({"list_web_acls": {"WebACLs": []}}) - with patch("app.boto3.client", return_value=client): + with patch("finserv_app.boto3.client", return_value=client): app._safe_collect_web_acls() client.list_web_acls.assert_called_once() _, kwargs = client.list_web_acls.call_args @@ -457,7 +451,7 @@ def test_multi_page_uses_next_marker_as_input(self): "get_web_acl": detail, } ) - with patch("app.boto3.client", return_value=client): + with patch("finserv_app.boto3.client", return_value=client): result = app._safe_collect_web_acls() assert len(result.summaries) == 2 assert client.list_web_acls.call_count == 2 @@ -477,7 +471,7 @@ def test_get_web_acl_called_with_scope_regional(self): "get_web_acl": detail, } ) - with patch("app.boto3.client", return_value=client): + with patch("finserv_app.boto3.client", return_value=client): app._safe_collect_web_acls() client.get_web_acl.assert_called_once_with( Name="acl-name", Scope="REGIONAL", Id="acl-1" @@ -487,7 +481,7 @@ def test_whole_inventory_failure_returns_unavailable(self): err = PermissionError("AccessDenied") client = MagicMock() client.list_web_acls.side_effect = err - with patch("app.boto3.client", return_value=client): + with patch("finserv_app.boto3.client", return_value=client): result = app._safe_collect_web_acls() assert isinstance(result, app._Unavailable) assert result.error is err @@ -509,7 +503,7 @@ def get_web_acl(**kwargs): } ) client.get_web_acl.side_effect = get_web_acl - with patch("app.boto3.client", return_value=client): + with patch("finserv_app.boto3.client", return_value=client): result = app._safe_collect_web_acls() # Collector extracts ["WebACL"] — verify the stored dict matches the inner value assert result.detail_by_id["acl-ok"] == ok_detail["WebACL"] @@ -518,7 +512,7 @@ def get_web_acl(**kwargs): def test_client_constructed_without_region_or_endpoint(self): client = _make_client({"list_web_acls": {"WebACLs": []}}) - with patch("app.boto3.client", return_value=client) as mock_boto: + with patch("finserv_app.boto3.client", return_value=client) as mock_boto: app._safe_collect_web_acls() mock_boto.assert_called_once() _, kwargs = mock_boto.call_args @@ -549,11 +543,15 @@ def _ctx(): bk = [{"Name": "bkt-1"}] wi = app.WebAclInventory(summaries=[{"Id": "acl-1"}], detail_by_id={}) with ( - patch("app._safe_collect_lambda_functions", return_value=lf) as p1, - patch("app._safe_collect_guardrails", return_value=gi) as p2, - patch("app._safe_collect_knowledge_bases", return_value=ki) as p3, - patch("app._safe_collect_buckets", return_value=bk) as p4, - patch("app._safe_collect_web_acls", return_value=wi) as p5, + patch( + "finserv_app._safe_collect_lambda_functions", return_value=lf + ) as p1, + patch("finserv_app._safe_collect_guardrails", return_value=gi) as p2, + patch( + "finserv_app._safe_collect_knowledge_bases", return_value=ki + ) as p3, + patch("finserv_app._safe_collect_buckets", return_value=bk) as p4, + patch("finserv_app._safe_collect_web_acls", return_value=wi) as p5, ): yield p1, p2, p3, p4, p5 @@ -582,19 +580,21 @@ def test_one_unavailable_does_not_prevent_others(self): err = PermissionError("denied") with ( patch( - "app._safe_collect_lambda_functions", return_value=app._Unavailable(err) + "finserv_app._safe_collect_lambda_functions", + return_value=app._Unavailable(err), ), patch( - "app._safe_collect_guardrails", + "finserv_app._safe_collect_guardrails", return_value=app.GuardrailInventory([], {}), ), patch( - "app._safe_collect_knowledge_bases", + "finserv_app._safe_collect_knowledge_bases", return_value=app.KbInventory([], {}, {}), ), - patch("app._safe_collect_buckets", return_value=[]), + patch("finserv_app._safe_collect_buckets", return_value=[]), patch( - "app._safe_collect_web_acls", return_value=app.WebAclInventory([], {}) + "finserv_app._safe_collect_web_acls", + return_value=app.WebAclInventory([], {}), ), ): inv = app.collect_resource_inventory() @@ -607,11 +607,11 @@ def test_all_unavailable_still_returns_inventory(self): err = RuntimeError("all down") unav = app._Unavailable(err) with ( - patch("app._safe_collect_lambda_functions", return_value=unav), - patch("app._safe_collect_guardrails", return_value=unav), - patch("app._safe_collect_knowledge_bases", return_value=unav), - patch("app._safe_collect_buckets", return_value=unav), - patch("app._safe_collect_web_acls", return_value=unav), + patch("finserv_app._safe_collect_lambda_functions", return_value=unav), + patch("finserv_app._safe_collect_guardrails", return_value=unav), + patch("finserv_app._safe_collect_knowledge_bases", return_value=unav), + patch("finserv_app._safe_collect_buckets", return_value=unav), + patch("finserv_app._safe_collect_web_acls", return_value=unav), ): inv = app.collect_resource_inventory() assert isinstance(inv, app.ResourceInventory) @@ -649,7 +649,7 @@ class TestOrderPreservation: def test_lambda_functions_order_preserved(self): fns = [{"FunctionName": f"fn-{i}"} for i in range(5)] client = _make_client({"list_functions": {"Functions": fns}}) - with patch("app.boto3.client", return_value=client): + with patch("finserv_app.boto3.client", return_value=client): result = app._safe_collect_lambda_functions() assert result == fns @@ -665,7 +665,7 @@ def test_web_acls_multi_page_order_preserved(self): "get_web_acl": {"WebACL": {}}, } ) - with patch("app.boto3.client", return_value=client): + with patch("finserv_app.boto3.client", return_value=client): result = app._safe_collect_web_acls() assert result.summaries == acls_p1 + acls_p2 @@ -680,6 +680,6 @@ def test_buckets_multi_page_order_preserved(self): ] } ) - with patch("app.boto3.client", return_value=client): + with patch("finserv_app.boto3.client", return_value=client): result = app._safe_collect_buckets() assert result == bkts_p1 + bkts_p2 diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_inventory_equivalence.py b/aiml-security-assessment/functions/security/finserv_tests/test_inventory_equivalence.py index 67e2c27..8517e2b 100644 --- a/aiml-security-assessment/functions/security/finserv_tests/test_inventory_equivalence.py +++ b/aiml-security-assessment/functions/security/finserv_tests/test_inventory_equivalence.py @@ -25,21 +25,12 @@ from __future__ import annotations -import os -import sys from datetime import datetime, timezone from unittest.mock import MagicMock, patch import pytest -# --------------------------------------------------------------------------- -# Make finserv_assessments importable (mirrors conftest.py) -# --------------------------------------------------------------------------- -FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") -if FINSERV_DIR not in sys.path: - sys.path.insert(0, FINSERV_DIR) - -import app # noqa: E402 +from .support import finserv_app as app # =========================================================================== @@ -567,9 +558,9 @@ def _run_handler_and_extract_tuples(mock_client_side_effect, event): portable across Python versions. """ with ( - patch("app.boto3.client") as mock_client, - patch("app.get_permissions_cache") as mock_cache, - patch("app.write_to_s3") as mock_s3, + patch("finserv_app.boto3.client") as mock_client, + patch("finserv_app.get_permissions_cache") as mock_cache, + patch("finserv_app.write_to_s3") as mock_s3, ): mock_client.side_effect = mock_client_side_effect mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} @@ -725,9 +716,9 @@ class TestGoldenEquivalenceBaseline: def test_handler_produces_65_findings(self, account_state_mock, lambda_event): """Smoke test: all 65 registry entries produce at least one finding.""" with ( - patch("app.boto3.client") as mock_client, - patch("app.get_permissions_cache") as mock_cache, - patch("app.write_to_s3") as mock_s3, + patch("finserv_app.boto3.client") as mock_client, + patch("finserv_app.get_permissions_cache") as mock_cache, + patch("finserv_app.write_to_s3") as mock_s3, ): mock_client.side_effect = account_state_mock mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} @@ -742,9 +733,9 @@ def test_handler_produces_65_findings(self, account_state_mock, lambda_event): def test_all_findings_have_rows(self, account_state_mock, lambda_event): """Every registry entry emits at least one CSV row (no silent drops).""" with ( - patch("app.boto3.client") as mock_client, - patch("app.get_permissions_cache") as mock_cache, - patch("app.write_to_s3") as mock_s3, + patch("finserv_app.boto3.client") as mock_client, + patch("finserv_app.get_permissions_cache") as mock_cache, + patch("finserv_app.write_to_s3") as mock_s3, ): mock_client.side_effect = account_state_mock mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} @@ -760,9 +751,9 @@ def test_all_findings_have_rows(self, account_state_mock, lambda_event): def test_fixture_covers_pass_paths(self, account_state_mock, lambda_event): """The fixture exercises PASS paths for the inventory-consuming checks.""" with ( - patch("app.boto3.client") as mock_client, - patch("app.get_permissions_cache") as mock_cache, - patch("app.write_to_s3") as mock_s3, + patch("finserv_app.boto3.client") as mock_client, + patch("finserv_app.get_permissions_cache") as mock_cache, + patch("finserv_app.write_to_s3") as mock_s3, ): mock_client.side_effect = account_state_mock mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} @@ -785,9 +776,9 @@ def test_fixture_covers_pass_paths(self, account_state_mock, lambda_event): def test_fixture_covers_na_paths(self, account_state_mock, lambda_event): """The fixture exercises N/A paths (advisory checks emit Status='N/A').""" with ( - patch("app.boto3.client") as mock_client, - patch("app.get_permissions_cache") as mock_cache, - patch("app.write_to_s3") as mock_s3, + patch("finserv_app.boto3.client") as mock_client, + patch("finserv_app.get_permissions_cache") as mock_cache, + patch("finserv_app.write_to_s3") as mock_s3, ): mock_client.side_effect = account_state_mock mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} @@ -821,9 +812,9 @@ def test_fixture_covers_fail_paths(self, lambda_event): fail_mock = _build_mock_client(guardrail_detail=guardrail_without_pii) with ( - patch("app.boto3.client") as mock_client, - patch("app.get_permissions_cache") as mock_cache, - patch("app.write_to_s3") as mock_s3, + patch("finserv_app.boto3.client") as mock_client, + patch("finserv_app.get_permissions_cache") as mock_cache, + patch("finserv_app.write_to_s3") as mock_s3, ): mock_client.side_effect = fail_mock mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} @@ -944,9 +935,9 @@ def get_web_acl(Name, Scope, Id, **kw): return c with ( - patch("app.boto3.client") as mock_client, - patch("app.get_permissions_cache") as mock_cache, - patch("app.write_to_s3"), + patch("finserv_app.boto3.client") as mock_client, + patch("finserv_app.get_permissions_cache") as mock_cache, + patch("finserv_app.write_to_s3"), ): mock_client.side_effect = mock_with_two_page_wafv2 mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} @@ -1036,9 +1027,9 @@ def get_bucket_notification_configuration(Bucket, **kw): return c with ( - patch("app.boto3.client") as mock_client, - patch("app.get_permissions_cache") as mock_cache, - patch("app.write_to_s3"), + patch("finserv_app.boto3.client") as mock_client, + patch("finserv_app.get_permissions_cache") as mock_cache, + patch("finserv_app.write_to_s3"), ): mock_client.side_effect = mock_with_two_page_s3 mock_cache.return_value = {"role_permissions": {}, "user_permissions": {}} diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_inventory_model.py b/aiml-security-assessment/functions/security/finserv_tests/test_inventory_model.py index 2370af2..691e811 100644 --- a/aiml-security-assessment/functions/security/finserv_tests/test_inventory_model.py +++ b/aiml-security-assessment/functions/security/finserv_tests/test_inventory_model.py @@ -4,17 +4,9 @@ Validates: Requirements REQ-4.1, REQ-4.2, REQ-9.2 """ -import sys -import os - import pytest -# Ensure finserv_assessments is importable -FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") -if FINSERV_DIR not in sys.path: - sys.path.insert(0, FINSERV_DIR) - -import app # noqa: E402 (import follows sys.path bootstrap above) +from .support import finserv_app as app # --------------------------------------------------------------------------- diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_lambda_handler.py b/aiml-security-assessment/functions/security/finserv_tests/test_lambda_handler.py index af6be0d..bf4558a 100644 --- a/aiml-security-assessment/functions/security/finserv_tests/test_lambda_handler.py +++ b/aiml-security-assessment/functions/security/finserv_tests/test_lambda_handler.py @@ -11,17 +11,11 @@ """ import json -import os -import sys from unittest.mock import MagicMock, patch import pytest -FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") -if FINSERV_DIR not in sys.path: - sys.path.insert(0, FINSERV_DIR) - -import app # noqa: E402 (import must follow sys.path setup above) +from .support import finserv_app as app # ========================================================================= @@ -32,9 +26,9 @@ class TestLambdaHandler: """End-to-end handler tests with fully mocked AWS clients.""" - @patch("app.write_to_s3") - @patch("app.get_permissions_cache") - @patch("app.boto3.client") + @patch("finserv_app.write_to_s3") + @patch("finserv_app.get_permissions_cache") + @patch("finserv_app.boto3.client") def test_handler_returns_200(self, mock_client, mock_cache, mock_s3, lambda_event): """Smoke test: handler completes and returns 200.""" # Return a generic mock for every boto3 client @@ -91,9 +85,9 @@ def test_handler_returns_200(self, mock_client, mock_cache, mock_s3, lambda_even # FS-27 ARC policies check that shares the FS-27 check_id). assert len(result["body"]["findings"]) == 65 - @patch("app.write_to_s3") - @patch("app.get_permissions_cache") - @patch("app.boto3.client") + @patch("finserv_app.write_to_s3") + @patch("finserv_app.get_permissions_cache") + @patch("finserv_app.boto3.client") def test_handler_findings_all_have_check_name( self, mock_client, mock_cache, mock_s3, lambda_event ): @@ -155,8 +149,8 @@ def test_handler_raises_without_bucket_env(self, lambda_event, monkeypatch): # We need to mock all boto3 calls so the checks themselves don't fail with ( - patch("app.boto3.client") as mock_client, - patch("app.get_permissions_cache") as mock_cache, + patch("finserv_app.boto3.client") as mock_client, + patch("finserv_app.get_permissions_cache") as mock_cache, ): generic = MagicMock() paginator = MagicMock() @@ -220,10 +214,10 @@ class TestInventoryCollectedAndPassed: Validates: REQ-6.5 """ - @patch("app.write_to_s3") - @patch("app.get_permissions_cache") - @patch("app.build_finserv_checks") - @patch("app.collect_resource_inventory") + @patch("finserv_app.write_to_s3") + @patch("finserv_app.get_permissions_cache") + @patch("finserv_app.build_finserv_checks") + @patch("finserv_app.collect_resource_inventory") def test_handler_calls_collect_inventory_exactly_once( self, mock_collect, @@ -243,10 +237,10 @@ def test_handler_calls_collect_inventory_exactly_once( mock_collect.assert_called_once_with() - @patch("app.write_to_s3") - @patch("app.get_permissions_cache") - @patch("app.build_finserv_checks") - @patch("app.collect_resource_inventory") + @patch("finserv_app.write_to_s3") + @patch("finserv_app.get_permissions_cache") + @patch("finserv_app.build_finserv_checks") + @patch("finserv_app.collect_resource_inventory") def test_handler_passes_inventory_to_build_finserv_checks( self, mock_collect, @@ -284,7 +278,7 @@ def test_handler_passes_inventory_to_build_finserv_checks( class TestWriteToS3: """Test the write_to_s3 helper.""" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_writes_csv_to_s3(self, mock_client): s3 = MagicMock() mock_client.return_value = s3 @@ -300,7 +294,7 @@ def test_writes_csv_to_s3(self, mock_client): assert "my-bucket" in url assert "exec-123" in url - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_s3_error_propagates(self, mock_client): s3 = MagicMock() s3.put_object.side_effect = RuntimeError("S3 write failed") @@ -313,7 +307,7 @@ def test_s3_error_propagates(self, mock_client): class TestGetPermissionsCache: """Test the get_permissions_cache helper.""" - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_returns_parsed_json(self, mock_client): s3 = MagicMock() body = MagicMock() @@ -324,7 +318,7 @@ def test_returns_parsed_json(self, mock_client): result = app.get_permissions_cache("exec-123") assert result == {"role_permissions": {"r1": {}}} - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_returns_none_on_client_error(self, mock_client): from botocore.exceptions import ClientError @@ -338,7 +332,7 @@ def test_returns_none_on_client_error(self, mock_client): result = app.get_permissions_cache("exec-123") assert result is None - @patch("app.boto3.client") + @patch("finserv_app.boto3.client") def test_returns_none_on_unexpected_error(self, mock_client): s3 = MagicMock() s3.get_object.side_effect = RuntimeError("unexpected") diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_large_estate.py b/aiml-security-assessment/functions/security/finserv_tests/test_large_estate.py index e92cfd2..f8fe6d3 100644 --- a/aiml-security-assessment/functions/security/finserv_tests/test_large_estate.py +++ b/aiml-security-assessment/functions/security/finserv_tests/test_large_estate.py @@ -15,19 +15,13 @@ a generic MagicMock for every non-inventory boto3 client. """ -import os -import sys import time import tracemalloc from datetime import datetime, timezone from unittest.mock import MagicMock, patch -FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") -if FINSERV_DIR not in sys.path: - sys.path.insert(0, FINSERV_DIR) - -import app # noqa: E402 +from .support import finserv_app as app # --------------------------------------------------------------------------- @@ -262,10 +256,10 @@ class TestLargeEstatePerformance: Validates: REQ-10.1, REQ-10.2, REQ-10.3, REQ-11.1, REQ-11.3 """ - @patch("app.write_to_s3") - @patch("app.get_permissions_cache") - @patch("app.boto3.client") - @patch("app.collect_resource_inventory") + @patch("finserv_app.write_to_s3") + @patch("finserv_app.get_permissions_cache") + @patch("finserv_app.boto3.client") + @patch("finserv_app.collect_resource_inventory") def test_large_estate_single_listing_and_memory( self, mock_collect_inventory, @@ -378,10 +372,10 @@ def test_large_estate_single_listing_and_memory( f"Peak memory {peak_mb:.1f} MB — must be well under 1024 MB" ) - @patch("app.write_to_s3") - @patch("app.get_permissions_cache") - @patch("app.boto3.client") - @patch("app.collect_resource_inventory") + @patch("finserv_app.write_to_s3") + @patch("finserv_app.get_permissions_cache") + @patch("finserv_app.boto3.client") + @patch("finserv_app.collect_resource_inventory") def test_large_estate_timing_budget( self, mock_collect_inventory, @@ -416,10 +410,10 @@ def test_large_estate_timing_budget( f"(900 s hard budget leaves {900 - elapsed:.0f}s headroom)" ) - @patch("app.write_to_s3") - @patch("app.get_permissions_cache") - @patch("app.boto3.client") - @patch("app.collect_resource_inventory") + @patch("finserv_app.write_to_s3") + @patch("finserv_app.get_permissions_cache") + @patch("finserv_app.boto3.client") + @patch("finserv_app.collect_resource_inventory") def test_large_estate_memory_footprint( self, mock_collect_inventory, diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_paginate.py b/aiml-security-assessment/functions/security/finserv_tests/test_paginate.py index 77c184b..7947838 100644 --- a/aiml-security-assessment/functions/security/finserv_tests/test_paginate.py +++ b/aiml-security-assessment/functions/security/finserv_tests/test_paginate.py @@ -17,19 +17,9 @@ from __future__ import annotations -import os -import sys from unittest.mock import MagicMock, call - -# --------------------------------------------------------------------------- -# Make finserv_assessments importable (mirrors conftest.py / test_inventory_equivalence.py) -# --------------------------------------------------------------------------- -FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") -if FINSERV_DIR not in sys.path: - sys.path.insert(0, FINSERV_DIR) - -import app # noqa: E402 +from .support import finserv_app as app # =========================================================================== diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_phase2_live.py b/aiml-security-assessment/functions/security/finserv_tests/test_phase2_live.py index bcdcd85..634f486 100644 --- a/aiml-security-assessment/functions/security/finserv_tests/test_phase2_live.py +++ b/aiml-security-assessment/functions/security/finserv_tests/test_phase2_live.py @@ -30,16 +30,17 @@ import time from datetime import datetime -# Make finserv_assessments importable -FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") -if FINSERV_DIR not in sys.path: - sys.path.insert(0, FINSERV_DIR) - # Set env var if not already set if not os.environ.get("AIML_ASSESSMENT_BUCKET_NAME"): os.environ["AIML_ASSESSMENT_BUCKET_NAME"] = "mehta-test-v55" -import app # noqa: E402 (import must follow sys.path + env setup above) +try: + from .support import finserv_app as app +except ImportError: + TEST_DIR = os.path.dirname(__file__) + if TEST_DIR not in sys.path: + sys.path.insert(0, TEST_DIR) + from support import finserv_app as app # --------------------------------------------------------------------------- diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_resilience.py b/aiml-security-assessment/functions/security/finserv_tests/test_resilience.py index ffba205..b089372 100644 --- a/aiml-security-assessment/functions/security/finserv_tests/test_resilience.py +++ b/aiml-security-assessment/functions/security/finserv_tests/test_resilience.py @@ -11,21 +11,10 @@ Validates: Requirements REQ-4.2, REQ-4.3, REQ-4.6, REQ-8 """ -import os -import sys from unittest.mock import MagicMock, patch - -FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") -if FINSERV_DIR not in sys.path: - sys.path.insert(0, FINSERV_DIR) - -TESTS_DIR = os.path.dirname(__file__) -if TESTS_DIR not in sys.path: - sys.path.insert(0, TESTS_DIR) - -import app # noqa: E402 -from conftest import make_resource_inventory # noqa: E402 +from .support import finserv_app as app +from .support import make_resource_inventory # --------------------------------------------------------------------------- @@ -136,7 +125,8 @@ def test_waf_check_unaffected(self): """REQ-4.3 / REQ-8: WAFv2 check is unaffected by lambda unavailability.""" inv = self._make_inv() with patch( - "app.boto3.client", return_value=_make_shield_mock_no_subscription() + "finserv_app.boto3.client", + return_value=_make_shield_mock_no_subscription(), ): result = app.check_waf_shield_on_bedrock_endpoints(inv) assert result["status"] != "ERROR", ( @@ -147,7 +137,7 @@ def test_waf_check_unaffected(self): def test_s3_check_unaffected(self): """REQ-4.3 / REQ-8: S3 check is unaffected by lambda unavailability.""" inv = self._make_inv() - with patch("app.boto3.client") as mock_boto: + with patch("finserv_app.boto3.client") as mock_boto: mock_boto.return_value = MagicMock() result = app.check_training_data_s3_versioning(inv) assert result["status"] != "ERROR", ( @@ -206,7 +196,7 @@ def test_fs36_becomes_could_not_assess(self): def test_lambda_check_unaffected(self): """REQ-4.3: Lambda check is unaffected by guardrail inventory failure.""" inv = self._make_inv() - with patch("app.boto3.client") as mock_boto: + with patch("finserv_app.boto3.client") as mock_boto: mock_boto.return_value = MagicMock( get_function_concurrency=MagicMock( return_value={"ReservedConcurrentExecutions": 5} @@ -230,7 +220,7 @@ def test_kb_check_unaffected(self): def test_s3_check_unaffected(self): """REQ-4.3: S3 check is unaffected by guardrail inventory failure.""" inv = self._make_inv() - with patch("app.boto3.client") as mock_boto: + with patch("finserv_app.boto3.client") as mock_boto: mock_boto.return_value = MagicMock() result = app.check_training_data_s3_versioning(inv) assert result["status"] != "ERROR", ( @@ -305,7 +295,8 @@ def _make_inv(self): def test_fs01_becomes_could_not_assess(self): """Validates: Requirements REQ-4.2""" with patch( - "app.boto3.client", return_value=_make_shield_mock_no_subscription() + "finserv_app.boto3.client", + return_value=_make_shield_mock_no_subscription(), ): result = app.check_waf_shield_on_bedrock_endpoints(self._make_inv()) assert _is_could_not_assess(result), ( @@ -332,7 +323,7 @@ def test_fs56_becomes_could_not_assess(self): def test_lambda_check_unaffected(self): """REQ-4.3 / REQ-8: Lambda check is unaffected by WAFv2 inventory failure.""" inv = self._make_inv() - with patch("app.boto3.client") as mock_boto: + with patch("finserv_app.boto3.client") as mock_boto: mock_boto.return_value = MagicMock( get_function_concurrency=MagicMock( return_value={"ReservedConcurrentExecutions": 5} @@ -356,7 +347,7 @@ def test_guardrail_check_unaffected(self): def test_s3_check_unaffected(self): """REQ-4.3 / REQ-8: S3 check is unaffected by WAFv2 inventory failure.""" inv = self._make_inv() - with patch("app.boto3.client") as mock_boto: + with patch("finserv_app.boto3.client") as mock_boto: mock_boto.return_value = MagicMock() result = app.check_training_data_s3_versioning(inv) assert result["status"] != "ERROR", ( @@ -417,7 +408,7 @@ def test_guardrail_check_unaffected(self): def test_lambda_check_unaffected(self): """REQ-4.3 / REQ-8: Lambda check is unaffected by KB inventory failure.""" inv = self._make_inv() - with patch("app.boto3.client") as mock_boto: + with patch("finserv_app.boto3.client") as mock_boto: mock_boto.return_value = MagicMock( get_function_concurrency=MagicMock( return_value={"ReservedConcurrentExecutions": 5} @@ -432,7 +423,7 @@ def test_lambda_check_unaffected(self): def test_s3_check_unaffected(self): """REQ-4.3 / REQ-8: S3 check is unaffected by KB inventory failure.""" inv = self._make_inv() - with patch("app.boto3.client") as mock_boto: + with patch("finserv_app.boto3.client") as mock_boto: mock_boto.return_value = MagicMock() result = app.check_training_data_s3_versioning(inv) assert result["status"] != "ERROR", ( @@ -444,7 +435,8 @@ def test_waf_check_unaffected(self): """REQ-4.3 / REQ-8: WAFv2 check is unaffected by KB inventory failure.""" inv = self._make_inv() with patch( - "app.boto3.client", return_value=_make_shield_mock_no_subscription() + "finserv_app.boto3.client", + return_value=_make_shield_mock_no_subscription(), ): result = app.check_waf_shield_on_bedrock_endpoints(inv) assert result["status"] != "ERROR", ( @@ -509,7 +501,7 @@ def test_kb_check_normal_when_guardrails_and_waf_unavailable(self): def test_s3_check_normal_when_guardrails_and_waf_unavailable(self): """REQ-4.3: S3 check is unaffected by guardrail and WAFv2 failures.""" inv = self._make_inv_guardrails_and_web_acls_unavailable() - with patch("app.boto3.client") as mock_boto: + with patch("finserv_app.boto3.client") as mock_boto: mock_boto.return_value = MagicMock() result = app.check_training_data_s3_versioning(inv) assert result["status"] != "ERROR", ( @@ -554,7 +546,8 @@ def test_waf_check_normal_when_lambda_and_s3_unavailable(self): """REQ-4.3: WAFv2 check is unaffected by lambda+S3 failures.""" inv = self._make_inv_lambda_and_s3_unavailable() with patch( - "app.boto3.client", return_value=_make_shield_mock_no_subscription() + "finserv_app.boto3.client", + return_value=_make_shield_mock_no_subscription(), ): result = app.check_waf_shield_on_bedrock_endpoints(inv) assert result["status"] != "ERROR", ( @@ -669,10 +662,10 @@ class FakeExceptions: } return generic - @patch("app.write_to_s3") - @patch("app.get_permissions_cache") - @patch("app.collect_resource_inventory") - @patch("app.boto3.client") + @patch("finserv_app.write_to_s3") + @patch("finserv_app.get_permissions_cache") + @patch("finserv_app.collect_resource_inventory") + @patch("finserv_app.boto3.client") def test_handler_completes_with_lambda_inventory_unavailable( self, mock_boto_client, @@ -730,10 +723,10 @@ def test_handler_completes_with_lambda_inventory_unavailable( f"got {rows[0]['Status']!r}" ) - @patch("app.write_to_s3") - @patch("app.get_permissions_cache") - @patch("app.collect_resource_inventory") - @patch("app.boto3.client") + @patch("finserv_app.write_to_s3") + @patch("finserv_app.get_permissions_cache") + @patch("finserv_app.collect_resource_inventory") + @patch("finserv_app.boto3.client") def test_handler_completes_with_guardrails_and_waf_unavailable( self, mock_boto_client, diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_schema.py b/aiml-security-assessment/functions/security/finserv_tests/test_schema.py index 5daead5..7464790 100644 --- a/aiml-security-assessment/functions/security/finserv_tests/test_schema.py +++ b/aiml-security-assessment/functions/security/finserv_tests/test_schema.py @@ -10,8 +10,16 @@ """ import pytest -from schema import Finding, SeverityEnum, StatusEnum, create_finding -from app import COMPLIANCE_MAP, build_finserv_checks + +from .support import finserv_app, finserv_schema + + +Finding = finserv_schema.Finding +SeverityEnum = finserv_schema.SeverityEnum +StatusEnum = finserv_schema.StatusEnum +create_finding = finserv_schema.create_finding +COMPLIANCE_MAP = finserv_app.COMPLIANCE_MAP +build_finserv_checks = finserv_app.build_finserv_checks # ========================================================================= diff --git a/aiml-security-assessment/functions/security/finserv_tests/test_severity_register.py b/aiml-security-assessment/functions/security/finserv_tests/test_severity_register.py index 64ce06a..f8ef822 100644 --- a/aiml-security-assessment/functions/security/finserv_tests/test_severity_register.py +++ b/aiml-security-assessment/functions/security/finserv_tests/test_severity_register.py @@ -13,15 +13,9 @@ pairs the code actually produces without real AWS access. """ -import sys -import os from unittest.mock import MagicMock, patch -FINSERV_DIR = os.path.join(os.path.dirname(__file__), "..", "finserv_assessments") -if FINSERV_DIR not in sys.path: - sys.path.insert(0, FINSERV_DIR) - -import app # noqa: E402 +from .support import finserv_app as app ALLOWED = {"High", "Medium", "Low", "Informational"} diff --git a/aiml-security-assessment/functions/security/generate_consolidated_report/schema.py b/aiml-security-assessment/functions/security/generate_consolidated_report/schema.py index d745e06..ed2a321 100644 --- a/aiml-security-assessment/functions/security/generate_consolidated_report/schema.py +++ b/aiml-security-assessment/functions/security/generate_consolidated_report/schema.py @@ -1,7 +1,6 @@ from enum import Enum -from typing import Dict, List, Any -from pydantic import BaseModel, Field, HttpUrl, validator -from datetime import datetime +from pydantic import BaseModel, Field, validator + class SeverityEnum(str, Enum): HIGH = "High" @@ -9,39 +8,49 @@ class SeverityEnum(str, Enum): LOW = "Low" INFORMATIONAL = "Informational" + class StatusEnum(str, Enum): FAILED = "Failed" PASSED = "Passed" NA = "N/A" + class Finding(BaseModel): """Represents a security finding with required fields and validations""" + Finding: str = Field(..., min_length=1, description="The name/title of the finding") - Finding_Details: str = Field(..., min_length=1, description="Detailed description of the finding") - Resolution: str = Field(..., min_length=0, description="Steps to resolve the finding") + Finding_Details: str = Field( + ..., min_length=1, description="Detailed description of the finding" + ) + Resolution: str = Field( + ..., min_length=0, description="Steps to resolve the finding" + ) Reference: str = Field(..., description="Documentation reference URL") Severity: SeverityEnum = Field(..., description="Severity level of the finding") Status: StatusEnum = Field(..., description="Current status of the finding") - @validator('Reference') + @validator("Reference") def validate_reference_url(cls, v): """Validate that reference URL starts with https://""" - if not str(v).startswith('https://'): - raise ValueError('Reference URL must start with https://') + if not str(v).startswith("https://"): + raise ValueError("Reference URL must start with https://") return v - @validator('Severity') + + @validator("Severity") def validate_severity(cls, v): """Validate that severity is one of the allowed values""" if v not in SeverityEnum.__members__.values(): - raise ValueError('Severity must be one of the allowed values') + raise ValueError("Severity must be one of the allowed values") return v - @validator('Status') + + @validator("Status") def validate_status(cls, v): """Validate that status is one of the allowed values""" if v not in StatusEnum.__members__.values(): - raise ValueError('Status must be one of the allowed values') + raise ValueError("Status must be one of the allowed values") return v + # Example usage: def create_finding( finding_name: str, @@ -49,11 +58,11 @@ def create_finding( resolution: str, reference: str, severity: SeverityEnum, - status: StatusEnum + status: StatusEnum, ) -> Finding: """ Create a validated finding object - + Args: finding_name: Name of the finding finding_details: Detailed description @@ -61,10 +70,10 @@ def create_finding( reference: Documentation URL severity: Severity level status: Current status - + Returns: Finding: Validated finding object - + Raises: ValidationError: If any field fails validation """ @@ -74,6 +83,6 @@ def create_finding( Resolution=resolution, Reference=reference, Severity=severity, - Status=status + Status=status, ) - return dict(finding.model_dump()) # Convert to regular dictionary \ No newline at end of file + return dict(finding.model_dump()) # Convert to regular dictionary diff --git a/aiml-security-assessment/functions/security/generate_consolidated_report/test_generate_report.py b/aiml-security-assessment/functions/security/generate_consolidated_report/test_generate_report.py index 01baaa1..b92a6ea 100644 --- a/aiml-security-assessment/functions/security/generate_consolidated_report/test_generate_report.py +++ b/aiml-security-assessment/functions/security/generate_consolidated_report/test_generate_report.py @@ -1,8 +1,40 @@ -# test_generate_report.py import unittest import os -from app import generate_html_report -from report_template import generate_html_report as generate_report_direct +import sys +import importlib.util + + +_THIS_DIR = os.path.dirname(__file__) +_SAVED_SYS_PATH = list(sys.path) +_SAVED_REPORT_TEMPLATE = sys.modules.get("report_template") + +try: + if _THIS_DIR not in sys.path: + sys.path.insert(0, _THIS_DIR) + + _template_spec = importlib.util.spec_from_file_location( + "generate_report_template", os.path.join(_THIS_DIR, "report_template.py") + ) + generate_report_template = importlib.util.module_from_spec(_template_spec) + sys.modules["report_template"] = generate_report_template + _template_spec.loader.exec_module(generate_report_template) + + _app_spec = importlib.util.spec_from_file_location( + "generate_consolidated_report_app", os.path.join(_THIS_DIR, "app.py") + ) + generate_report_app = importlib.util.module_from_spec(_app_spec) + sys.modules["generate_consolidated_report_app"] = generate_report_app + _app_spec.loader.exec_module(generate_report_app) +finally: + sys.path[:] = _SAVED_SYS_PATH + if _SAVED_REPORT_TEMPLATE is None: + sys.modules.pop("report_template", None) + else: + sys.modules["report_template"] = _SAVED_REPORT_TEMPLATE + + +generate_html_report = generate_report_app.generate_html_report +generate_report_direct = generate_report_template.generate_html_report class TestHtmlReportGeneration(unittest.TestCase): diff --git a/aiml-security-assessment/functions/security/iam_permission_caching/app.py b/aiml-security-assessment/functions/security/iam_permission_caching/app.py index 742da9f..3359d75 100644 --- a/aiml-security-assessment/functions/security/iam_permission_caching/app.py +++ b/aiml-security-assessment/functions/security/iam_permission_caching/app.py @@ -1,18 +1,10 @@ import boto3 -import csv import os import logging -from datetime import datetime, timedelta, timezone -import time -from typing import Dict, List, Any, Optional -from io import StringIO -import asyncio +from datetime import datetime, timezone import json from botocore.config import Config -from botocore.exceptions import ClientError -import random -from datetime import datetime def get_current_utc_date(): return datetime.now(timezone.utc).strftime("%Y/%m/%d") @@ -20,9 +12,9 @@ def get_current_utc_date(): # Configure boto3 with retries boto3_config = Config( - retries = dict( - max_attempts = 10, # Maximum number of retries - mode = 'adaptive' # Exponential backoff with adaptive mode + retries=dict( + max_attempts=10, # Maximum number of retries + mode="adaptive", # Exponential backoff with adaptive mode ) ) @@ -30,54 +22,55 @@ def get_current_utc_date(): logger = logging.getLogger() logger.setLevel(logging.ERROR) + def write_permissions_to_s3(permission_cache, execution_id): """ Write the IAM permissions cache to S3 as a JSON file - + Args: permission_cache (IAMPermissionCache): The permission cache object s3_bucket (str): The name of the S3 bucket to write to """ try: # Create S3 client with the same retry configuration - s3_client = boto3.client('s3', config=boto3_config) - + s3_client = boto3.client("s3", config=boto3_config) + # Prepare the data to be written cache_data = { - 'role_permissions': permission_cache.role_permissions, - 'user_permissions': permission_cache.user_permissions, - 'generated_at': datetime.now().isoformat() + "role_permissions": permission_cache.role_permissions, + "user_permissions": permission_cache.user_permissions, + "generated_at": datetime.now().isoformat(), } - + # Convert to JSON string json_data = json.dumps(cache_data, default=str, indent=2) - + # Define the S3 key (filename) - write to bucket root - s3_key = f'permissions_cache_{execution_id}.json' - s3_bucket = os.environ.get('AIML_ASSESSMENT_BUCKET_NAME') + s3_key = f"permissions_cache_{execution_id}.json" + s3_bucket = os.environ.get("AIML_ASSESSMENT_BUCKET_NAME") # Upload to S3 s3_client.put_object( - Bucket=s3_bucket, - Key=s3_key, - Body=json_data, - ContentType='application/json' + Bucket=s3_bucket, Key=s3_key, Body=json_data, ContentType="application/json" + ) + + logger.info( + f"Successfully wrote permissions cache to s3://{s3_bucket}/{s3_key}" ) - - logger.info(f"Successfully wrote permissions cache to s3://{s3_bucket}/{s3_key}") return s3_key - + except Exception as e: logger.error(f"Error writing permissions cache to S3: {str(e)}", exc_info=True) raise + class IAMPermissionCache: def __init__(self, iam_client): self.iam_client = iam_client self.role_permissions = {} self.user_permissions = {} self.policy_cache = {} - + def initialize(self): """ Get all IAM permissions and cache them @@ -85,7 +78,7 @@ def initialize(self): logger.info("Initializing IAM permission cache") self._cache_role_permissions() self._cache_user_permissions() - + def _get_policy_document(self, policy_arn, version_id): """ Get policy document with caching @@ -94,12 +87,13 @@ def _get_policy_document(self, policy_arn, version_id): if cache_key not in self.policy_cache: try: response = self.iam_client.get_policy_version( - PolicyArn=policy_arn, - VersionId=version_id + PolicyArn=policy_arn, VersionId=version_id ) - self.policy_cache[cache_key] = response['PolicyVersion']['Document'] + self.policy_cache[cache_key] = response["PolicyVersion"]["Document"] except Exception as e: - logger.error(f"Error getting policy document for {policy_arn}: {str(e)}") + logger.error( + f"Error getting policy document for {policy_arn}: {str(e)}" + ) return None return self.policy_cache[cache_key] @@ -108,109 +102,142 @@ def _cache_role_permissions(self): Cache all role permissions """ logger.info("Caching role permissions") - paginator = self.iam_client.get_paginator('list_roles') + paginator = self.iam_client.get_paginator("list_roles") for page in paginator.paginate(): - for role in page['Roles']: - role_name = role['RoleName'] + for role in page["Roles"]: + role_name = role["RoleName"] self.role_permissions[role_name] = { - 'attached_policies': [], - 'inline_policies': [] + "attached_policies": [], + "inline_policies": [], } - + # Get attached policies try: - attached_policies = self.iam_client.list_attached_role_policies(RoleName=role_name) - for policy in attached_policies['AttachedPolicies']: - policy_arn = policy['PolicyArn'] + attached_policies = self.iam_client.list_attached_role_policies( + RoleName=role_name + ) + for policy in attached_policies["AttachedPolicies"]: + policy_arn = policy["PolicyArn"] try: - policy_info = self.iam_client.get_policy(PolicyArn=policy_arn)['Policy'] - policy_doc = self._get_policy_document(policy_arn, policy_info['DefaultVersionId']) + policy_info = self.iam_client.get_policy( + PolicyArn=policy_arn + )["Policy"] + policy_doc = self._get_policy_document( + policy_arn, policy_info["DefaultVersionId"] + ) if policy_doc: - self.role_permissions[role_name]['attached_policies'].append({ - 'name': policy['PolicyName'], - 'arn': policy_arn, - 'document': policy_doc - }) + self.role_permissions[role_name][ + "attached_policies" + ].append( + { + "name": policy["PolicyName"], + "arn": policy_arn, + "document": policy_doc, + } + ) except Exception as e: logger.error(f"Error getting policy {policy_arn}: {str(e)}") except Exception as e: - logger.error(f"Error getting attached policies for role {role_name}: {str(e)}") + logger.error( + f"Error getting attached policies for role {role_name}: {str(e)}" + ) # Get inline policies try: - inline_policies = self.iam_client.list_role_policies(RoleName=role_name) - for policy_name in inline_policies['PolicyNames']: + inline_policies = self.iam_client.list_role_policies( + RoleName=role_name + ) + for policy_name in inline_policies["PolicyNames"]: try: policy_doc = self.iam_client.get_role_policy( - RoleName=role_name, - PolicyName=policy_name - )['PolicyDocument'] - self.role_permissions[role_name]['inline_policies'].append({ - 'name': policy_name, - 'document': policy_doc - }) + RoleName=role_name, PolicyName=policy_name + )["PolicyDocument"] + self.role_permissions[role_name]["inline_policies"].append( + {"name": policy_name, "document": policy_doc} + ) except Exception as e: - logger.error(f"Error getting inline policy {policy_name}: {str(e)}") + logger.error( + f"Error getting inline policy {policy_name}: {str(e)}" + ) except Exception as e: - logger.error(f"Error getting inline policies for role {role_name}: {str(e)}") + logger.error( + f"Error getting inline policies for role {role_name}: {str(e)}" + ) def _cache_user_permissions(self): """ Cache all user permissions """ logger.info("Caching user permissions") - paginator = self.iam_client.get_paginator('list_users') + paginator = self.iam_client.get_paginator("list_users") for page in paginator.paginate(): - for user in page['Users']: - user_name = user['UserName'] + for user in page["Users"]: + user_name = user["UserName"] self.user_permissions[user_name] = { - 'attached_policies': [], - 'inline_policies': [] + "attached_policies": [], + "inline_policies": [], } - + # Get attached policies try: - attached_policies = self.iam_client.list_attached_user_policies(UserName=user_name) - for policy in attached_policies['AttachedPolicies']: - policy_arn = policy['PolicyArn'] + attached_policies = self.iam_client.list_attached_user_policies( + UserName=user_name + ) + for policy in attached_policies["AttachedPolicies"]: + policy_arn = policy["PolicyArn"] try: - policy_info = self.iam_client.get_policy(PolicyArn=policy_arn)['Policy'] - policy_doc = self._get_policy_document(policy_arn, policy_info['DefaultVersionId']) + policy_info = self.iam_client.get_policy( + PolicyArn=policy_arn + )["Policy"] + policy_doc = self._get_policy_document( + policy_arn, policy_info["DefaultVersionId"] + ) if policy_doc: - self.user_permissions[user_name]['attached_policies'].append({ - 'name': policy['PolicyName'], - 'arn': policy_arn, - 'document': policy_doc - }) + self.user_permissions[user_name][ + "attached_policies" + ].append( + { + "name": policy["PolicyName"], + "arn": policy_arn, + "document": policy_doc, + } + ) except Exception as e: logger.error(f"Error getting policy {policy_arn}: {str(e)}") except Exception as e: - logger.error(f"Error getting attached policies for user {user_name}: {str(e)}") + logger.error( + f"Error getting attached policies for user {user_name}: {str(e)}" + ) # Get inline policies try: - inline_policies = self.iam_client.list_user_policies(UserName=user_name) - for policy_name in inline_policies['PolicyNames']: + inline_policies = self.iam_client.list_user_policies( + UserName=user_name + ) + for policy_name in inline_policies["PolicyNames"]: try: policy_doc = self.iam_client.get_user_policy( - UserName=user_name, - PolicyName=policy_name - )['PolicyDocument'] - self.user_permissions[user_name]['inline_policies'].append({ - 'name': policy_name, - 'document': policy_doc - }) + UserName=user_name, PolicyName=policy_name + )["PolicyDocument"] + self.user_permissions[user_name]["inline_policies"].append( + {"name": policy_name, "document": policy_doc} + ) except Exception as e: - logger.error(f"Error getting inline policy {policy_name}: {str(e)}") + logger.error( + f"Error getting inline policy {policy_name}: {str(e)}" + ) except Exception as e: - logger.error(f"Error getting inline policies for user {user_name}: {str(e)}") - + logger.error( + f"Error getting inline policies for user {user_name}: {str(e)}" + ) + + def lambda_handler(event, context): """ Main Lambda handler """ logger.info("Starting Bedrock security assessment") - iam_client = boto3.client('iam', config=boto3_config) + iam_client = boto3.client("iam", config=boto3_config) logger.info(event, context) try: # Initialize permission cache @@ -221,13 +248,10 @@ def lambda_handler(event, context): s3_key = write_permissions_to_s3(permission_cache, execution_id) return { - 'statusCode': 200, - 'body': f'Successfully cached IAM permissions to {s3_key}' + "statusCode": 200, + "body": f"Successfully cached IAM permissions to {s3_key}", } except Exception as e: logger.error(f"Error in lambda_handler: {str(e)}", exc_info=True) - return { - 'statusCode': 500, - 'body': f'Error during security checks: {str(e)}' - } + return {"statusCode": 500, "body": f"Error during security checks: {str(e)}"} diff --git a/aiml-security-assessment/functions/security/iam_permission_caching/schema.py b/aiml-security-assessment/functions/security/iam_permission_caching/schema.py index d745e06..ed2a321 100644 --- a/aiml-security-assessment/functions/security/iam_permission_caching/schema.py +++ b/aiml-security-assessment/functions/security/iam_permission_caching/schema.py @@ -1,7 +1,6 @@ from enum import Enum -from typing import Dict, List, Any -from pydantic import BaseModel, Field, HttpUrl, validator -from datetime import datetime +from pydantic import BaseModel, Field, validator + class SeverityEnum(str, Enum): HIGH = "High" @@ -9,39 +8,49 @@ class SeverityEnum(str, Enum): LOW = "Low" INFORMATIONAL = "Informational" + class StatusEnum(str, Enum): FAILED = "Failed" PASSED = "Passed" NA = "N/A" + class Finding(BaseModel): """Represents a security finding with required fields and validations""" + Finding: str = Field(..., min_length=1, description="The name/title of the finding") - Finding_Details: str = Field(..., min_length=1, description="Detailed description of the finding") - Resolution: str = Field(..., min_length=0, description="Steps to resolve the finding") + Finding_Details: str = Field( + ..., min_length=1, description="Detailed description of the finding" + ) + Resolution: str = Field( + ..., min_length=0, description="Steps to resolve the finding" + ) Reference: str = Field(..., description="Documentation reference URL") Severity: SeverityEnum = Field(..., description="Severity level of the finding") Status: StatusEnum = Field(..., description="Current status of the finding") - @validator('Reference') + @validator("Reference") def validate_reference_url(cls, v): """Validate that reference URL starts with https://""" - if not str(v).startswith('https://'): - raise ValueError('Reference URL must start with https://') + if not str(v).startswith("https://"): + raise ValueError("Reference URL must start with https://") return v - @validator('Severity') + + @validator("Severity") def validate_severity(cls, v): """Validate that severity is one of the allowed values""" if v not in SeverityEnum.__members__.values(): - raise ValueError('Severity must be one of the allowed values') + raise ValueError("Severity must be one of the allowed values") return v - @validator('Status') + + @validator("Status") def validate_status(cls, v): """Validate that status is one of the allowed values""" if v not in StatusEnum.__members__.values(): - raise ValueError('Status must be one of the allowed values') + raise ValueError("Status must be one of the allowed values") return v + # Example usage: def create_finding( finding_name: str, @@ -49,11 +58,11 @@ def create_finding( resolution: str, reference: str, severity: SeverityEnum, - status: StatusEnum + status: StatusEnum, ) -> Finding: """ Create a validated finding object - + Args: finding_name: Name of the finding finding_details: Detailed description @@ -61,10 +70,10 @@ def create_finding( reference: Documentation URL severity: Severity level status: Current status - + Returns: Finding: Validated finding object - + Raises: ValidationError: If any field fails validation """ @@ -74,6 +83,6 @@ def create_finding( Resolution=resolution, Reference=reference, Severity=severity, - Status=status + Status=status, ) - return dict(finding.model_dump()) # Convert to regular dictionary \ No newline at end of file + return dict(finding.model_dump()) # Convert to regular dictionary diff --git a/aiml-security-assessment/functions/security/sagemaker_assessments/schema.py b/aiml-security-assessment/functions/security/sagemaker_assessments/schema.py index bb7a07f..9d4e097 100644 --- a/aiml-security-assessment/functions/security/sagemaker_assessments/schema.py +++ b/aiml-security-assessment/functions/security/sagemaker_assessments/schema.py @@ -1,62 +1,77 @@ from enum import Enum -from typing import Dict, List, Any -from pydantic import BaseModel, Field, HttpUrl, validator -from datetime import datetime +from typing import Dict, Any +from pydantic import BaseModel, Field, validator import re + class Config: strict = True # Enables strict type checking + class SeverityEnum(str, Enum): HIGH = "High" MEDIUM = "Medium" LOW = "Low" INFORMATIONAL = "Informational" + class StatusEnum(str, Enum): FAILED = "Failed" PASSED = "Passed" NA = "N/A" + class Finding(BaseModel): """Represents a security finding with required fields and validations""" - Check_ID: str = Field(..., min_length=1, description="Unique check identifier (e.g., SM-01, BR-01, AC-01)") + + Check_ID: str = Field( + ..., + min_length=1, + description="Unique check identifier (e.g., SM-01, BR-01, AC-01)", + ) Finding: str = Field(..., min_length=1, description="The name/title of the finding") - Finding_Details: str = Field(..., min_length=1, description="Detailed description of the finding") - Resolution: str = Field(..., min_length=0, description="Steps to resolve the finding") + Finding_Details: str = Field( + ..., min_length=1, description="Detailed description of the finding" + ) + Resolution: str = Field( + ..., min_length=0, description="Steps to resolve the finding" + ) Reference: str = Field(..., description="Documentation reference URL") Severity: SeverityEnum = Field(..., description="Severity level of the finding") Status: StatusEnum = Field(..., description="Current status of the finding") - @validator('Check_ID') + @validator("Check_ID") def validate_check_id(cls, v): """Validate that Check_ID follows the pattern XX-NN (e.g., SM-01, BR-14, AC-05)""" - pattern = r'^[A-Z]{2,3}-\d{2}$' + pattern = r"^[A-Z]{2,3}-\d{2}$" if not re.match(pattern, v): - raise ValueError('Check_ID must follow pattern XX-NN (e.g., SM-01, BR-14, AC-05)') + raise ValueError( + "Check_ID must follow pattern XX-NN (e.g., SM-01, BR-14, AC-05)" + ) return v - @validator('Reference') + @validator("Reference") def validate_reference_url(cls, v): """Validate that reference URL starts with https://""" - if not str(v).startswith('https://'): - raise ValueError('Reference URL must start with https://') + if not str(v).startswith("https://"): + raise ValueError("Reference URL must start with https://") return v - @validator('Severity') + @validator("Severity") def validate_severity(cls, v): """Validate that severity is one of the allowed values""" if v not in SeverityEnum.__members__.values(): - raise ValueError('Severity must be one of the allowed values') + raise ValueError("Severity must be one of the allowed values") return v - @validator('Status') + @validator("Status") def validate_status(cls, v): """Validate that status is one of the allowed values""" if v not in StatusEnum.__members__.values(): - raise ValueError('Status must be one of the allowed values') + raise ValueError("Status must be one of the allowed values") return v + def create_finding( check_id: str, finding_name: str, @@ -64,7 +79,7 @@ def create_finding( resolution: str, reference: str, severity: SeverityEnum, - status: StatusEnum + status: StatusEnum, ) -> Dict[str, Any]: """ Create a validated finding object @@ -91,6 +106,6 @@ def create_finding( Resolution=resolution, Reference=reference, Severity=severity, - Status=status + Status=status, ) - return dict(finding.model_dump()) # Convert to regular dictionary \ No newline at end of file + return dict(finding.model_dump()) # Convert to regular dictionary diff --git a/aiml-security-assessment/template-multi-account.yaml b/aiml-security-assessment/template-multi-account.yaml index 6331041..8ecdefc 100644 --- a/aiml-security-assessment/template-multi-account.yaml +++ b/aiml-security-assessment/template-multi-account.yaml @@ -149,7 +149,7 @@ Resources: Action: - bedrock:ListGuardrails - bedrock:GetGuardrail - - bedrock:ListModelInvocations + - bedrock:ListModelInvocationJobs - bedrock:GetModelInvocationLoggingConfiguration - bedrock:ListPrompts - bedrock:GetPrompt @@ -160,11 +160,14 @@ Resources: - bedrock:GetModelCustomizationJob - bedrock:ListFlows - bedrock:GetFlow + - bedrock:ListKnowledgeBases + - bedrock:GetKnowledgeBase Resource: '*' - Sid: S3BucketEncryptionPermissions Effect: Allow Action: - s3:GetBucketEncryption + - s3:GetEncryptionConfiguration Resource: 'arn:aws:s3:::*' - Sid: CloudTrailPermissions Effect: Allow @@ -261,6 +264,17 @@ Resources: - sagemaker:DescribeEndpoint - sagemaker:ListDataQualityJobDefinitions - sagemaker:DescribeDataQualityJobDefinition + - sagemaker:ListTransformJobs + - sagemaker:DescribeTransformJob + - sagemaker:ListHyperParameterTuningJobs + - sagemaker:DescribeHyperParameterTuningJob + - sagemaker:ListCompilationJobs + - sagemaker:DescribeCompilationJob + - sagemaker:ListAutoMLJobs + - sagemaker:DescribeAutoMLJob + - sagemaker:ListExperiments + - sagemaker:ListTrials + - sagemaker:ListAssociations Resource: '*' - Statement: - Sid: GuardDutyPermissions @@ -332,8 +346,7 @@ Resources: - bedrock-agentcore:GetGateway - bedrock-agentcore:ListPolicyEngines - bedrock-agentcore:GetPolicyEngine - - bedrock-agentcore:GetAgentRuntimeResourcePolicy - - bedrock-agentcore:GetGatewayResourcePolicy + - bedrock-agentcore:GetResourcePolicy Resource: '*' - Sid: IAMRolePermissions Effect: Allow @@ -458,6 +471,8 @@ Resources: - bedrock:ListCustomModels - bedrock:ListEvaluationJobs - bedrock:GetModelInvocationLoggingConfiguration + - bedrock:ListTagsForResource + - bedrock:ListAutomatedReasoningPolicies Resource: '*' - Sid: BedrockAgentPermissions Effect: Allow @@ -528,6 +543,7 @@ Resources: Effect: Allow Action: - events:ListRules + - scheduler:ListSchedules - config:DescribeConfigRules Resource: '*' - Sid: S3BucketPermissions diff --git a/aiml-security-assessment/template.yaml b/aiml-security-assessment/template.yaml index bb9f548..673d63d 100644 --- a/aiml-security-assessment/template.yaml +++ b/aiml-security-assessment/template.yaml @@ -160,11 +160,14 @@ Resources: - bedrock:GetModelCustomizationJob - bedrock:ListFlows - bedrock:GetFlow + - bedrock:ListKnowledgeBases + - bedrock:GetKnowledgeBase Resource: '*' - Sid: S3BucketEncryptionPermissions Effect: Allow Action: - s3:GetBucketEncryption + - s3:GetEncryptionConfiguration Resource: 'arn:aws:s3:::*' - Sid: CloudTrailPermissions Effect: Allow @@ -261,6 +264,17 @@ Resources: - sagemaker:DescribeEndpoint - sagemaker:ListDataQualityJobDefinitions - sagemaker:DescribeDataQualityJobDefinition + - sagemaker:ListTransformJobs + - sagemaker:DescribeTransformJob + - sagemaker:ListHyperParameterTuningJobs + - sagemaker:DescribeHyperParameterTuningJob + - sagemaker:ListCompilationJobs + - sagemaker:DescribeCompilationJob + - sagemaker:ListAutoMLJobs + - sagemaker:DescribeAutoMLJob + - sagemaker:ListExperiments + - sagemaker:ListTrials + - sagemaker:ListAssociations Resource: '*' - Statement: - Sid: GuardDutyPermissions @@ -332,8 +346,7 @@ Resources: - bedrock-agentcore:GetGateway - bedrock-agentcore:ListPolicyEngines - bedrock-agentcore:GetPolicyEngine - - bedrock-agentcore:GetAgentRuntimeResourcePolicy - - bedrock-agentcore:GetGatewayResourcePolicy + - bedrock-agentcore:GetResourcePolicy Resource: '*' - Sid: IAMRolePermissions Effect: Allow diff --git a/deployment/1-aiml-security-member-roles.yaml b/deployment/1-aiml-security-member-roles.yaml index 656a324..cfa70e1 100644 --- a/deployment/1-aiml-security-member-roles.yaml +++ b/deployment/1-aiml-security-member-roles.yaml @@ -68,6 +68,8 @@ Resources: - bedrock:GetPrompt - bedrock:ListAgents - bedrock:GetAgent + - bedrock:ListKnowledgeBases + - bedrock:GetKnowledgeBase Resource: "*" # Bedrock AgentCore Permissions - Effect: Allow @@ -80,8 +82,7 @@ Resources: - bedrock-agentcore:GetGateway - bedrock-agentcore:ListPolicyEngines - bedrock-agentcore:GetPolicyEngine - - bedrock-agentcore:GetAgentRuntimeResourcePolicy - - bedrock-agentcore:GetGatewayResourcePolicy + - bedrock-agentcore:GetResourcePolicy Resource: "*" # SageMaker Assessment Permissions - Effect: Allow @@ -170,6 +171,7 @@ Resources: - Effect: Allow Action: - s3:GetBucketEncryption + - s3:GetEncryptionConfiguration - s3:GetBucketVersioning - s3:GetBucketTagging - s3:HeadBucket diff --git a/deployment/2-aiml-security-codebuild.yaml b/deployment/2-aiml-security-codebuild.yaml index 8999a14..bf06f70 100644 --- a/deployment/2-aiml-security-codebuild.yaml +++ b/deployment/2-aiml-security-codebuild.yaml @@ -156,6 +156,8 @@ Resources: - bedrock:GetModelCustomizationJob - bedrock:ListFlows - bedrock:GetFlow + - bedrock:ListKnowledgeBases + - bedrock:GetKnowledgeBase # SageMaker Permissions - sagemaker:ListNotebookInstances - sagemaker:DescribeNotebookInstance @@ -229,8 +231,7 @@ Resources: - bedrock-agentcore:GetGateway - bedrock-agentcore:ListPolicyEngines - bedrock-agentcore:GetPolicyEngine - - bedrock-agentcore:GetAgentRuntimeResourcePolicy - - bedrock-agentcore:GetGatewayResourcePolicy + - bedrock-agentcore:GetResourcePolicy # ECR Permissions for AgentCore encryption checks - ecr:DescribeRepositories - ecr:GetRepositoryPolicy @@ -243,6 +244,7 @@ Resources: - ec2:DescribeNatGateways # S3 Permissions for encryption checks - s3:GetBucketEncryption + - s3:GetEncryptionConfiguration - s3:ListBucket - s3:GetBucketVersioning - s3:GetBucketTagging diff --git a/deployment/aiml-security-single-account.yaml b/deployment/aiml-security-single-account.yaml index e7b946b..0b962f8 100644 --- a/deployment/aiml-security-single-account.yaml +++ b/deployment/aiml-security-single-account.yaml @@ -138,6 +138,8 @@ Resources: - bedrock:ListFlows - bedrock:GetFlow # Bedrock Agent Permissions (Knowledge Bases, Flows) + - bedrock:ListKnowledgeBases + - bedrock:GetKnowledgeBase # SageMaker Permissions - sagemaker:ListNotebookInstances - sagemaker:DescribeNotebookInstance @@ -211,8 +213,7 @@ Resources: - bedrock-agentcore:GetGateway - bedrock-agentcore:ListPolicyEngines - bedrock-agentcore:GetPolicyEngine - - bedrock-agentcore:GetAgentRuntimeResourcePolicy - - bedrock-agentcore:GetGatewayResourcePolicy + - bedrock-agentcore:GetResourcePolicy # ECR Permissions for AgentCore encryption checks - ecr:DescribeRepositories - ecr:GetRepositoryPolicy @@ -225,6 +226,7 @@ Resources: - ec2:DescribeNatGateways # S3 Permissions for encryption checks - s3:GetBucketEncryption + - s3:GetEncryptionConfiguration - s3:ListBucket - s3:GetBucketVersioning - s3:GetBucketTagging diff --git a/sample-reports/scripts/capture_screenshots.py b/sample-reports/scripts/capture_screenshots.py index 1f2555b..bb66cea 100755 --- a/sample-reports/scripts/capture_screenshots.py +++ b/sample-reports/scripts/capture_screenshots.py @@ -18,7 +18,6 @@ python sample-reports/scripts/capture_screenshots.py """ -import os import sys from pathlib import Path from playwright.sync_api import sync_playwright @@ -92,23 +91,25 @@ def optimize_png(image_path: Path, max_size_kb: int = 300) -> None: img = Image.open(image_path) # Convert RGBA to RGB if needed (reduces size) - if img.mode == 'RGBA': - background = Image.new('RGB', img.size, (255, 255, 255)) + if img.mode == "RGBA": + background = Image.new("RGB", img.size, (255, 255, 255)) background.paste(img, mask=img.split()[3]) # Use alpha channel as mask img = background # Save with optimization - img.save(image_path, 'PNG', optimize=True) + img.save(image_path, "PNG", optimize=True) # Check file size file_size_kb = image_path.stat().st_size / 1024 # If still too large, reduce quality by converting to JPEG if file_size_kb > max_size_kb: - jpeg_path = image_path.with_suffix('.jpg') - img.save(jpeg_path, 'JPEG', quality=JPEG_QUALITY, optimize=True) + jpeg_path = image_path.with_suffix(".jpg") + img.save(jpeg_path, "JPEG", quality=JPEG_QUALITY, optimize=True) image_path.unlink() # Remove PNG - print(f" Converted to JPEG: {jpeg_path.name} ({jpeg_path.stat().st_size / 1024:.1f} KB)") + print( + f" Converted to JPEG: {jpeg_path.name} ({jpeg_path.stat().st_size / 1024:.1f} KB)" + ) return jpeg_path print(f" Optimized PNG: {image_path.name} ({file_size_kb:.1f} KB)") @@ -136,7 +137,9 @@ def capture_screenshot(browser, screenshot_config: dict) -> Path: print(f" Source: {screenshot_config['file']}") # Create a new page - page = browser.new_page(viewport={"width": VIEWPORT_WIDTH, "height": VIEWPORT_HEIGHT}) + page = browser.new_page( + viewport={"width": VIEWPORT_WIDTH, "height": VIEWPORT_HEIGHT} + ) # Navigate to the HTML file page.goto(f"file://{html_file.absolute()}") @@ -144,7 +147,9 @@ def capture_screenshot(browser, screenshot_config: dict) -> Path: # Execute actions for action in screenshot_config["actions"]: if action["type"] == "wait": - page.wait_for_selector(action["selector"], timeout=action.get("timeout", 5000)) + page.wait_for_selector( + action["selector"], timeout=action.get("timeout", 5000) + ) elif action["type"] == "click": page.click(action["selector"]) elif action["type"] == "scroll": @@ -225,7 +230,7 @@ def main(): print(" 3. Commit the screenshots to the repository") except ImportError as e: - print(f"\nERROR: Required library not installed") + print("\nERROR: Required library not installed") print(f" {e}") print("\nPlease install required dependencies:") print(" source .venv/bin/activate") diff --git a/tests/__init__.py b/tests/__init__.py index bc79c9a..65140f2 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1 +1 @@ -# AI/ML Security Assessment Test Suite +# tests package diff --git a/tests/conftest.py b/tests/conftest.py index b459b03..5b0a230 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -240,36 +240,3 @@ def permission_cache_agentcore_full_access(): }, "user_permissions": {}, } - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- -def extract_csv_data(result): - """Extract csv_data findings from a check function result. - - Works for both Bedrock/SageMaker style (dict with 'csv_data' key) - and AgentCore style (returns list directly). - """ - if isinstance(result, list): - return result - return result.get("csv_data", []) - - -def assert_finding_schema(finding): - """Assert that a single finding dict has all required schema fields.""" - required_keys = { - "Check_ID", - "Finding", - "Finding_Details", - "Resolution", - "Reference", - "Severity", - "Status", - } - assert required_keys.issubset(finding.keys()), ( - f"Missing keys: {required_keys - finding.keys()}" - ) - assert finding["Severity"] in ("High", "Medium", "Low", "Informational") - assert finding["Status"] in ("Failed", "Passed", "N/A") - assert finding["Reference"].startswith("https://") diff --git a/tests/test_agentcore_checks.py b/tests/test_agentcore_checks.py index 547588e..5e5fa19 100644 --- a/tests/test_agentcore_checks.py +++ b/tests/test_agentcore_checks.py @@ -21,9 +21,7 @@ from botocore.exceptions import ClientError sys.path.insert(0, "aiml-security-assessment/functions/security/agentcore_assessments") -sys.path.insert(0, os.path.join(os.path.dirname(__file__))) - -from conftest import extract_csv_data, assert_finding_schema +from tests.test_helpers import extract_csv_data, assert_finding_schema # Load agentcore app module directly to avoid name collisions with other app.py files _ac_dir = os.path.abspath( @@ -182,25 +180,41 @@ def test_ac02_schema_valid(self, mock_ac, empty_permission_cache): class TestAC03StaleAccess: """AC-03: Check stale AgentCore access.""" + @patch("agentcore_app.boto3.client") @patch("agentcore_app.agentcore_client", None) - def test_ac03_client_unavailable_returns_na(self, empty_permission_cache): + def test_ac03_client_unavailable_returns_na( + self, mock_boto_client, empty_permission_cache + ): + mock_boto_client.return_value.get_caller_identity.return_value = { + "Account": "123456789012" + } result = agentcore_app.check_stale_agentcore_access(empty_permission_cache) findings = extract_csv_data(result) assert len(findings) >= 1 assert findings[0]["Check_ID"] == "AC-03" + @patch("agentcore_app.boto3.client") @patch("agentcore_app.iam_client") @patch("agentcore_app.agentcore_client") def test_ac03_empty_cache_returns_findings( - self, mock_ac, mock_iam, empty_permission_cache + self, mock_ac, mock_iam, mock_boto_client, empty_permission_cache ): + mock_boto_client.return_value.get_caller_identity.return_value = { + "Account": "123456789012" + } result = agentcore_app.check_stale_agentcore_access(empty_permission_cache) findings = extract_csv_data(result) assert len(findings) >= 1 + @patch("agentcore_app.boto3.client") @patch("agentcore_app.iam_client") @patch("agentcore_app.agentcore_client") - def test_ac03_schema_valid(self, mock_ac, mock_iam, empty_permission_cache): + def test_ac03_schema_valid( + self, mock_ac, mock_iam, mock_boto_client, empty_permission_cache + ): + mock_boto_client.return_value.get_caller_identity.return_value = { + "Account": "123456789012" + } result = agentcore_app.check_stale_agentcore_access(empty_permission_cache) for f in extract_csv_data(result): assert_finding_schema(f) @@ -250,8 +264,10 @@ def test_ac04_schema_valid(self): class TestAC05Encryption: """AC-05: Check AgentCore ECR encryption.""" + @patch("agentcore_app.ecr_client") @patch("agentcore_app.agentcore_client", None) - def test_ac05_client_unavailable_returns_na(self): + def test_ac05_client_unavailable_returns_na(self, mock_ecr): + mock_ecr.describe_repositories.return_value = {"repositories": []} result = agentcore_app.check_agentcore_encryption() findings = extract_csv_data(result) assert len(findings) >= 1 @@ -275,8 +291,10 @@ def test_ac05_exception_returns_error_finding(self, mock_ac, mock_ecr): assert len(findings) >= 1 assert findings[0]["Status"] == "Failed" + @patch("agentcore_app.ecr_client") @patch("agentcore_app.agentcore_client", None) - def test_ac05_schema_valid(self): + def test_ac05_schema_valid(self, mock_ecr): + mock_ecr.describe_repositories.return_value = {"repositories": []} result = agentcore_app.check_agentcore_encryption() for f in extract_csv_data(result): assert_finding_schema(f) @@ -337,6 +355,22 @@ def test_ac07_no_memories_returns_na(self, mock_ac): findings = extract_csv_data(result) assert len(findings) >= 1 + @patch("agentcore_app.agentcore_client") + def test_ac07_memory_with_wrapped_kms_key_returns_passed(self, mock_ac): + mock_ac.list_memories.return_value = { + "memories": [{"id": "mem-123456789012", "name": "TestMemory"}] + } + mock_ac.get_memory.return_value = { + "memory": { + "id": "mem-123456789012", + "encryptionKeyArn": "arn:aws:kms:us-east-1:123:key/abc", + } + } + result = agentcore_app.check_agentcore_memory_configuration() + findings = extract_csv_data(result) + assert len(findings) >= 1 + assert findings[0]["Status"] == "Passed" + @patch("agentcore_app.agentcore_client") def test_ac07_exception_returns_error_finding(self, mock_ac): mock_ac.list_memories.side_effect = Exception("Memory error") @@ -358,8 +392,10 @@ def test_ac07_schema_valid(self): class TestAC08VPCEndpoints: """AC-08: Check VPC endpoints for AgentCore.""" + @patch("agentcore_app.ec2_client") @patch("agentcore_app.agentcore_client", None) - def test_ac08_client_unavailable_returns_na(self): + def test_ac08_client_unavailable_returns_na(self, mock_ec2): + mock_ec2.describe_vpcs.return_value = {"Vpcs": []} result = agentcore_app.check_agentcore_vpc_endpoints() findings = extract_csv_data(result) assert len(findings) >= 1 @@ -373,16 +409,19 @@ def test_ac08_no_runtimes_returns_na(self, mock_ac, mock_ec2): findings = extract_csv_data(result) assert len(findings) >= 1 + @patch("agentcore_app.ec2_client") @patch("agentcore_app.agentcore_client") - def test_ac08_exception_returns_error_finding(self, mock_ac): - mock_ac.list_agent_runtimes.side_effect = Exception("VPC endpoint error") + def test_ac08_exception_returns_error_finding(self, mock_ac, mock_ec2): + mock_ec2.describe_vpcs.side_effect = Exception("VPC endpoint error") result = agentcore_app.check_agentcore_vpc_endpoints() findings = extract_csv_data(result) assert len(findings) >= 1 assert findings[0]["Status"] == "Failed" + @patch("agentcore_app.ec2_client") @patch("agentcore_app.agentcore_client", None) - def test_ac08_schema_valid(self): + def test_ac08_schema_valid(self, mock_ec2): + mock_ec2.describe_vpcs.return_value = {"Vpcs": []} result = agentcore_app.check_agentcore_vpc_endpoints() for f in extract_csv_data(result): assert_finding_schema(f) @@ -394,8 +433,13 @@ def test_ac08_schema_valid(self): class TestAC09ServiceLinkedRole: """AC-09: Check AgentCore service-linked role.""" + @patch("agentcore_app.iam_client") @patch("agentcore_app.agentcore_client", None) - def test_ac09_client_unavailable_returns_na(self): + def test_ac09_client_unavailable_returns_na(self, mock_iam): + mock_iam.get_role.side_effect = _make_client_error( + "NoSuchEntity", "Role not found" + ) + mock_iam.exceptions.NoSuchEntityException = ClientError result = agentcore_app.check_agentcore_service_linked_role() findings = extract_csv_data(result) assert len(findings) >= 1 @@ -406,28 +450,22 @@ def test_ac09_client_unavailable_returns_na(self): def test_ac09_slr_exists_returns_passed(self, mock_ac, mock_iam): mock_iam.get_role.return_value = { "Role": { - "RoleName": "AWSServiceRoleForBedrockAgentCore", - "Arn": "arn:aws:iam::123:role/aws-service-role/agentcore.bedrock.amazonaws.com/AWSServiceRoleForBedrockAgentCore", - "Path": "/aws-service-role/agentcore.bedrock.amazonaws.com/", + "RoleName": "AWSServiceRoleForBedrockAgentCoreNetwork", + "Arn": "arn:aws:iam::123:role/aws-service-role/network.bedrock-agentcore.amazonaws.com/AWSServiceRoleForBedrockAgentCoreNetwork", + "Path": "/aws-service-role/network.bedrock-agentcore.amazonaws.com/", "AssumeRolePolicyDocument": { "Statement": [ { "Effect": "Allow", - "Principal": {"Service": "agentcore.bedrock.amazonaws.com"}, + "Principal": { + "Service": "network.bedrock-agentcore.amazonaws.com" + }, "Action": "sts:AssumeRole", } ] }, } } - mock_iam.list_attached_role_policies.return_value = { - "AttachedPolicies": [ - { - "PolicyName": "AWSBedrockAgentCoreServiceRolePolicy", - "PolicyArn": "arn:aws:iam::aws:policy/aws-service-role/AWSBedrockAgentCoreServiceRolePolicy", - } - ] - } result = agentcore_app.check_agentcore_service_linked_role() findings = extract_csv_data(result) assert len(findings) >= 1 @@ -454,8 +492,13 @@ def test_ac09_exception_returns_error_finding(self, mock_ac): assert len(findings) >= 1 assert findings[0]["Status"] == "Failed" + @patch("agentcore_app.iam_client") @patch("agentcore_app.agentcore_client", None) - def test_ac09_schema_valid(self): + def test_ac09_schema_valid(self, mock_iam): + mock_iam.get_role.side_effect = _make_client_error( + "NoSuchEntity", "Role not found" + ) + mock_iam.exceptions.NoSuchEntityException = ClientError result = agentcore_app.check_agentcore_service_linked_role() for f in extract_csv_data(result): assert_finding_schema(f) @@ -477,9 +520,87 @@ def test_ac10_client_unavailable_returns_na(self): @patch("agentcore_app.agentcore_client") def test_ac10_no_runtimes_returns_na(self, mock_ac): mock_ac.list_agent_runtimes.return_value = {"agentRuntimes": []} + mock_ac.list_gateways.return_value = {"items": []} + result = agentcore_app.check_agentcore_resource_based_policies() + findings = extract_csv_data(result) + assert len(findings) >= 1 + + @patch("agentcore_app.agentcore_client") + def test_ac10_uses_generic_resource_policy_api(self, mock_ac): + mock_ac.list_agent_runtimes.return_value = { + "agentRuntimes": [ + { + "agentRuntimeId": "rt-1", + "agentRuntimeName": "TestRuntime", + "agentRuntimeArn": "arn:aws:bedrock-agentcore:us-east-1:123456789012:runtime/rt-1", + } + ] + } + mock_ac.list_gateways.return_value = {"items": []} + mock_ac.get_resource_policy.return_value = { + "policy": '{"Version":"2012-10-17"}' + } + + result = agentcore_app.check_agentcore_resource_based_policies() + findings = extract_csv_data(result) + + assert len(findings) >= 1 + assert findings[0]["Status"] == "Passed" + mock_ac.get_resource_policy.assert_called_once_with( + resourceArn="arn:aws:bedrock-agentcore:us-east-1:123456789012:runtime/rt-1" + ) + + @patch("agentcore_app.agentcore_client") + def test_ac10_access_denied_policy_read_returns_na_finding(self, mock_ac): + mock_ac.list_agent_runtimes.return_value = { + "agentRuntimes": [ + { + "agentRuntimeId": "rt-1", + "agentRuntimeName": "TestRuntime", + "agentRuntimeArn": "arn:aws:bedrock-agentcore:us-east-1:123456789012:runtime/rt-1", + } + ] + } + mock_ac.list_gateways.return_value = {"items": []} + mock_ac.get_resource_policy.side_effect = _make_client_error( + "AccessDeniedException", "Denied" + ) + result = agentcore_app.check_agentcore_resource_based_policies() findings = extract_csv_data(result) + assert len(findings) >= 1 + assert any( + f["Finding"] == "AgentCore Resource-Based Policy Assessment Access Denied" + and f["Status"] == "N/A" + for f in findings + ) + + @patch("agentcore_app.agentcore_client") + def test_ac10_policy_read_throttling_returns_incomplete_finding(self, mock_ac): + mock_ac.list_agent_runtimes.return_value = { + "agentRuntimes": [ + { + "agentRuntimeId": "rt-1", + "agentRuntimeName": "TestRuntime", + "agentRuntimeArn": "arn:aws:bedrock-agentcore:us-east-1:123456789012:runtime/rt-1", + } + ] + } + mock_ac.list_gateways.return_value = {"items": []} + mock_ac.get_resource_policy.side_effect = _make_client_error( + "ThrottlingException", "Try again" + ) + + result = agentcore_app.check_agentcore_resource_based_policies() + findings = extract_csv_data(result) + + assert len(findings) >= 1 + assert any( + f["Finding"] == "AgentCore Resource-Based Policy Assessment Incomplete" + and f["Status"] == "N/A" + for f in findings + ) @patch("agentcore_app.agentcore_client") def test_ac10_exception_returns_error_finding(self, mock_ac): @@ -546,10 +667,25 @@ def test_ac12_client_unavailable_returns_na(self): @patch("agentcore_app.agentcore_client") def test_ac12_no_gateways_returns_na(self, mock_ac): - mock_ac.list_gateways.return_value = {"gateways": []} + mock_ac.list_gateways.return_value = {"items": []} + result = agentcore_app.check_agentcore_gateway_encryption() + findings = extract_csv_data(result) + assert len(findings) >= 1 + + @patch("agentcore_app.agentcore_client") + def test_ac12_gateway_with_kms_key_returns_passed(self, mock_ac): + mock_ac.list_gateways.return_value = { + "items": [{"gatewayId": "gw-1", "name": "TestGateway"}] + } + mock_ac.get_gateway.return_value = { + "gatewayId": "gw-1", + "name": "TestGateway", + "kmsKeyArn": "arn:aws:kms:us-east-1:123:key/abc", + } result = agentcore_app.check_agentcore_gateway_encryption() findings = extract_csv_data(result) assert len(findings) >= 1 + assert findings[0]["Status"] == "Passed" @patch("agentcore_app.agentcore_client") def test_ac12_exception_returns_error_finding(self, mock_ac): @@ -581,11 +717,21 @@ def test_ac13_client_unavailable_returns_na(self): @patch("agentcore_app.agentcore_client") def test_ac13_no_gateways_returns_na(self, mock_ac): - mock_ac.list_gateways.return_value = {"gateways": []} + mock_ac.list_gateways.return_value = {"items": []} result = agentcore_app.check_agentcore_gateway_configuration() findings = extract_csv_data(result) assert len(findings) >= 1 + @patch("agentcore_app.agentcore_client") + def test_ac13_items_gateway_shape_returns_passed(self, mock_ac): + mock_ac.list_gateways.return_value = { + "items": [{"gatewayId": "gw-1", "name": "TestGateway"}] + } + result = agentcore_app.check_agentcore_gateway_configuration() + findings = extract_csv_data(result) + assert len(findings) >= 1 + assert findings[0]["Status"] == "Passed" + @patch("agentcore_app.agentcore_client") def test_ac13_exception_returns_error_finding(self, mock_ac): mock_ac.list_gateways.side_effect = Exception("Gateway config error") diff --git a/tests/test_bedrock_checks.py b/tests/test_bedrock_checks.py index c201d22..8592c77 100644 --- a/tests/test_bedrock_checks.py +++ b/tests/test_bedrock_checks.py @@ -13,10 +13,9 @@ import os import importlib.util from unittest.mock import patch, MagicMock +from botocore.exceptions import ClientError -# Add tests dir so we can import helpers -sys.path.insert(0, os.path.join(os.path.dirname(__file__))) -from conftest import extract_csv_data, assert_finding_schema +from tests.test_helpers import extract_csv_data, assert_finding_schema # Load bedrock app module directly to avoid name collisions with other app.py files _bedrock_dir = os.path.abspath( @@ -203,7 +202,7 @@ def test_br04_logging_enabled_s3_returns_passed(self, mock_client): mock_client.return_value = mock_bedrock mock_bedrock.get_model_invocation_logging_configuration.return_value = { "loggingConfig": { - "s3Config": {"s3BucketName": "my-log-bucket"}, + "s3Config": {"bucketName": "my-log-bucket"}, "cloudWatchConfig": {}, } } @@ -243,7 +242,7 @@ def test_br04_schema_valid(self, mock_client): mock_client.return_value = mock_bedrock mock_bedrock.get_model_invocation_logging_configuration.return_value = { "loggingConfig": { - "s3Config": {"s3BucketName": "bucket"}, + "s3Config": {"bucketName": "bucket"}, "cloudWatchConfig": {}, } } @@ -251,6 +250,22 @@ def test_br04_schema_valid(self, mock_client): for f in extract_csv_data(result): assert_finding_schema(f) + @patch("boto3.client") + def test_br04_logging_enabled_s3_legacy_key_returns_passed(self, mock_client): + check = bedrock_app.check_bedrock_logging_configuration + mock_bedrock = MagicMock() + mock_client.return_value = mock_bedrock + mock_bedrock.get_model_invocation_logging_configuration.return_value = { + "loggingConfig": { + "s3Config": {"s3BucketName": "legacy-log-bucket"}, + "cloudWatchConfig": {}, + } + } + result = check() + findings = extract_csv_data(result) + assert len(findings) >= 1 + assert findings[0]["Status"] == "Passed" + # =================================================================== # BR-05: check_bedrock_guardrails @@ -398,25 +413,47 @@ class TestBR07PromptManagement: def test_br07_prompts_exist_returns_passed(self, mock_client): check = bedrock_app.check_bedrock_prompt_management mock_agent = MagicMock() + paginator = MagicMock() mock_client.return_value = mock_agent - mock_agent.list_prompts.return_value = { - "promptSummaries": [ - {"name": "prompt1", "promptId": "p1", "status": "ACTIVE"} - ] - } + mock_agent.get_paginator.return_value = paginator + paginator.paginate.return_value = [ + {"promptSummaries": [{"name": "prompt1", "id": "p1"}]} + ] mock_agent.get_prompt.return_value = {"variants": ["v1", "v2"]} result = check() findings = extract_csv_data(result) assert len(findings) >= 1 assert findings[0]["Status"] == "Passed" assert findings[0]["Check_ID"] == "BR-07" + mock_agent.get_prompt.assert_called_once_with(promptIdentifier="p1") + + @patch("boto3.client") + def test_br07_legacy_prompt_id_fallback_still_supported(self, mock_client): + check = bedrock_app.check_bedrock_prompt_management + mock_agent = MagicMock() + paginator = MagicMock() + mock_client.return_value = mock_agent + mock_agent.get_paginator.return_value = paginator + paginator.paginate.return_value = [ + {"promptSummaries": [{"name": "prompt1", "promptId": "p1"}]} + ] + mock_agent.get_prompt.return_value = {"variants": ["v1", "v2"]} + + result = check() + findings = extract_csv_data(result) + + assert len(findings) >= 1 + assert findings[0]["Status"] == "Passed" + mock_agent.get_prompt.assert_called_once_with(promptIdentifier="p1") @patch("boto3.client") def test_br07_no_prompts_returns_na(self, mock_client): check = bedrock_app.check_bedrock_prompt_management mock_agent = MagicMock() + paginator = MagicMock() mock_client.return_value = mock_agent - mock_agent.list_prompts.return_value = {"promptSummaries": []} + mock_agent.get_paginator.return_value = paginator + paginator.paginate.return_value = [{"promptSummaries": []}] result = check() findings = extract_csv_data(result) assert len(findings) >= 1 @@ -435,8 +472,10 @@ def test_br07_exception_returns_error_finding(self, mock_client): def test_br07_schema_valid(self, mock_client): check = bedrock_app.check_bedrock_prompt_management mock_agent = MagicMock() + paginator = MagicMock() mock_client.return_value = mock_agent - mock_agent.list_prompts.return_value = {"promptSummaries": []} + mock_agent.get_paginator.return_value = paginator + paginator.paginate.return_value = [{"promptSummaries": []}] result = check() for f in extract_csv_data(result): assert_finding_schema(f) @@ -452,8 +491,10 @@ class TestBR08AgentRoles: def test_br08_no_agents_returns_na(self, mock_client, empty_permission_cache): check = bedrock_app.check_bedrock_agent_roles mock_agent = MagicMock() + paginator = MagicMock() mock_client.return_value = mock_agent - mock_agent.list_agents.return_value = {"agents": []} + mock_agent.get_paginator.return_value = paginator + paginator.paginate.return_value = [{"agentSummaries": []}] result = check(empty_permission_cache) findings = extract_csv_data(result) assert len(findings) >= 1 @@ -466,17 +507,43 @@ def test_br08_agent_with_compliant_role_returns_passed( ): check = bedrock_app.check_bedrock_agent_roles mock_agent = MagicMock() + paginator = MagicMock() mock_client.return_value = mock_agent - mock_agent.list_agents.return_value = { - "agents": [{"agentId": "a1", "agentName": "TestAgent"}] + mock_agent.get_paginator.return_value = paginator + paginator.paginate.return_value = [ + {"agentSummaries": [{"agentId": "a1", "agentName": "TestAgent"}]} + ] + mock_agent.get_agent.return_value = { + "agent": { + "agentResourceRoleArn": "arn:aws:iam::123456789012:role/LeastPrivilegeRole" + } } + result = check(permission_cache_compliant) + findings = extract_csv_data(result) + assert len(findings) >= 1 + mock_agent.get_agent.assert_called_once_with(agentId="a1") + + @patch("boto3.client") + def test_br08_legacy_role_arn_shape_still_supported( + self, mock_client, permission_cache_compliant + ): + check = bedrock_app.check_bedrock_agent_roles + mock_agent = MagicMock() + paginator = MagicMock() + mock_client.return_value = mock_agent + mock_agent.get_paginator.return_value = paginator + paginator.paginate.return_value = [ + {"agentSummaries": [{"agentId": "a1", "agentName": "TestAgent"}]} + ] mock_agent.get_agent.return_value = { "agentResourceRoleArn": "arn:aws:iam::123456789012:role/LeastPrivilegeRole" } + result = check(permission_cache_compliant) findings = extract_csv_data(result) + assert len(findings) >= 1 - # With permission boundary and specific resources, should pass or have minimal issues + mock_agent.get_agent.assert_called_once_with(agentId="a1") @patch("boto3.client") def test_br08_exception_returns_error_finding( @@ -493,8 +560,10 @@ def test_br08_exception_returns_error_finding( def test_br08_schema_valid(self, mock_client, empty_permission_cache): check = bedrock_app.check_bedrock_agent_roles mock_agent = MagicMock() + paginator = MagicMock() mock_client.return_value = mock_agent - mock_agent.list_agents.return_value = {"agents": []} + mock_agent.get_paginator.return_value = paginator + paginator.paginate.return_value = [{"agentSummaries": []}] result = check(empty_permission_cache) for f in extract_csv_data(result): assert_finding_schema(f) @@ -547,6 +616,23 @@ def test_br09_exception_returns_error_finding(self, mock_client): assert len(findings) >= 1 assert findings[0]["Status"] == "Failed" + @patch("boto3.client") + def test_br09_access_denied_returns_na(self, mock_client): + check = bedrock_app.check_bedrock_knowledge_base_encryption + mock_agent = MagicMock() + mock_client.return_value = mock_agent + paginator = MagicMock() + mock_agent.get_paginator.return_value = paginator + paginator.paginate.side_effect = ClientError( + {"Error": {"Code": "AccessDeniedException", "Message": "denied"}}, + "ListKnowledgeBases", + ) + result = check() + findings = extract_csv_data(result) + assert len(findings) >= 1 + assert findings[0]["Status"] == "N/A" + assert findings[0]["Severity"] == "Informational" + @patch("boto3.client") def test_br09_schema_valid(self, mock_client): check = bedrock_app.check_bedrock_knowledge_base_encryption @@ -796,6 +882,31 @@ def test_br12_exception_returns_error_finding(self, mock_client): assert len(findings) >= 1 assert findings[0]["Status"] == "Failed" + @patch("boto3.client") + def test_br12_access_denied_returns_na(self, mock_client): + check = bedrock_app.check_bedrock_invocation_log_encryption + mock_bedrock = MagicMock() + mock_s3 = MagicMock() + + def client_factory(service, **kwargs): + if service == "bedrock": + return mock_bedrock + return mock_s3 + + mock_client.side_effect = client_factory + mock_bedrock.get_model_invocation_logging_configuration.return_value = { + "loggingConfig": {"s3Config": {"bucketName": "log-bucket"}} + } + mock_s3.get_bucket_encryption.side_effect = ClientError( + {"Error": {"Code": "AccessDenied", "Message": "denied"}}, + "GetBucketEncryption", + ) + result = check() + findings = extract_csv_data(result) + assert len(findings) >= 1 + assert findings[0]["Status"] == "N/A" + assert findings[0]["Severity"] == "Informational" + @patch("boto3.client") def test_br12_schema_valid(self, mock_client): check = bedrock_app.check_bedrock_invocation_log_encryption diff --git a/tests/test_core_iam_coverage.py b/tests/test_core_iam_coverage.py new file mode 100644 index 0000000..575cddc --- /dev/null +++ b/tests/test_core_iam_coverage.py @@ -0,0 +1,126 @@ +"""Guard the core Bedrock deployment roles against missing runtime IAM actions.""" + +import os + +import pytest + + +_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +_SECTION_CHECKS = [ + { + "path": os.path.join(_REPO_ROOT, "aiml-security-assessment", "template.yaml"), + "start": "- Sid: BedrockAssessmentPermissions", + "end": "- Sid: S3BucketEncryptionPermissions", + "required": { + "bedrock:GetModelInvocationLoggingConfiguration", + "bedrock:ListKnowledgeBases", + "bedrock:GetKnowledgeBase", + }, + }, + { + "path": os.path.join(_REPO_ROOT, "aiml-security-assessment", "template.yaml"), + "start": "- Sid: S3BucketEncryptionPermissions", + "end": "- Sid: CloudTrailPermissions", + "required": {"s3:GetEncryptionConfiguration"}, + }, + { + "path": os.path.join( + _REPO_ROOT, "aiml-security-assessment", "template-multi-account.yaml" + ), + "start": "- Sid: BedrockAssessmentPermissions", + "end": "- Sid: S3BucketEncryptionPermissions", + "required": { + "bedrock:GetModelInvocationLoggingConfiguration", + "bedrock:ListKnowledgeBases", + "bedrock:GetKnowledgeBase", + }, + }, + { + "path": os.path.join( + _REPO_ROOT, "aiml-security-assessment", "template-multi-account.yaml" + ), + "start": "- Sid: S3BucketEncryptionPermissions", + "end": "- Sid: CloudTrailPermissions", + "required": {"s3:GetEncryptionConfiguration"}, + }, + { + "path": os.path.join( + _REPO_ROOT, "deployment", "aiml-security-single-account.yaml" + ), + "start": "# Bedrock Permissions", + "end": "# SageMaker Permissions", + "required": { + "bedrock:GetModelInvocationLoggingConfiguration", + "bedrock:ListKnowledgeBases", + "bedrock:GetKnowledgeBase", + }, + }, + { + "path": os.path.join( + _REPO_ROOT, "deployment", "aiml-security-single-account.yaml" + ), + "start": "# S3 Permissions for encryption checks", + "end": 'Resource: "*"', + "required": {"s3:GetEncryptionConfiguration"}, + }, + { + "path": os.path.join( + _REPO_ROOT, "deployment", "2-aiml-security-codebuild.yaml" + ), + "start": "# Bedrock Permissions", + "end": "# SageMaker Permissions", + "required": { + "bedrock:GetModelInvocationLoggingConfiguration", + "bedrock:ListKnowledgeBases", + "bedrock:GetKnowledgeBase", + }, + }, + { + "path": os.path.join( + _REPO_ROOT, "deployment", "2-aiml-security-codebuild.yaml" + ), + "start": "# S3 Permissions for encryption checks", + "end": 'Resource: "*"', + "required": {"s3:GetEncryptionConfiguration"}, + }, + { + "path": os.path.join( + _REPO_ROOT, "deployment", "1-aiml-security-member-roles.yaml" + ), + "start": "# Bedrock Agent Permissions (Agents for Amazon Bedrock)", + "end": 'Resource: "*"', + "required": {"bedrock:ListKnowledgeBases", "bedrock:GetKnowledgeBase"}, + }, + { + "path": os.path.join( + _REPO_ROOT, "deployment", "1-aiml-security-member-roles.yaml" + ), + "start": "# S3 Bucket Permissions for encryption checks", + "end": 'Resource: "arn:aws:s3:::*"', + "required": {"s3:GetEncryptionConfiguration"}, + }, +] + + +def _load_section(path, start_marker, end_marker): + with open(path, encoding="utf-8") as fh: + text = fh.read() + + start = text.index(start_marker) + end = text.index(end_marker, start) + return text[start:end] + + +@pytest.mark.parametrize( + "check", + _SECTION_CHECKS, + ids=lambda c: f"{os.path.basename(c['path'])}:{c['start']}", +) +def test_required_core_bedrock_actions_are_granted(check): + section = _load_section(check["path"], check["start"], check["end"]) + missing = sorted(action for action in check["required"] if action not in section) + assert not missing, ( + f"{os.path.basename(check['path'])} section starting at " + f"'{check['start']}' is missing required IAM action(s): {missing}" + ) diff --git a/tests/test_helpers.py b/tests/test_helpers.py new file mode 100644 index 0000000..fdedaaa --- /dev/null +++ b/tests/test_helpers.py @@ -0,0 +1,27 @@ +"""Shared non-fixture helpers for the top-level test suite.""" + + +def extract_csv_data(result): + """Extract csv_data findings from a check function result.""" + if isinstance(result, list): + return result + return result.get("csv_data", []) + + +def assert_finding_schema(finding): + """Assert that a single finding dict has all required schema fields.""" + required_keys = { + "Check_ID", + "Finding", + "Finding_Details", + "Resolution", + "Reference", + "Severity", + "Status", + } + assert required_keys.issubset(finding.keys()), ( + f"Missing keys: {required_keys - finding.keys()}" + ) + assert finding["Severity"] in ("High", "Medium", "Low", "Informational") + assert finding["Status"] in ("Failed", "Passed", "N/A") + assert finding["Reference"].startswith("https://") diff --git a/tests/test_iam_coverage.py b/tests/test_iam_coverage.py index e64fbde..c7091dd 100644 --- a/tests/test_iam_coverage.py +++ b/tests/test_iam_coverage.py @@ -1,8 +1,9 @@ """IAM coverage guard (REQ-12 / Wave 5.5 T5h.6). -Asserts that every IAM action the FinServ checks require is granted in ALL three +Asserts that every IAM action the FinServ checks require is granted in all runtime grant sources: - aiml-security-assessment/template.yaml (SAM single-account roles) + - aiml-security-assessment/template-multi-account.yaml - deployment/1-aiml-security-member-roles.yaml (multi-account member role) - deployment/aiml-security-single-account.yaml (single-account CFN wrapper) @@ -21,10 +22,16 @@ _TEMPLATES = [ os.path.join(_REPO_ROOT, "aiml-security-assessment", "template.yaml"), + os.path.join(_REPO_ROOT, "aiml-security-assessment", "template-multi-account.yaml"), os.path.join(_REPO_ROOT, "deployment", "1-aiml-security-member-roles.yaml"), os.path.join(_REPO_ROOT, "deployment", "aiml-security-single-account.yaml"), ] +_AGENTCORE_PERMISSION_TEMPLATES = [ + *_TEMPLATES, + os.path.join(_REPO_ROOT, "deployment", "2-aiml-security-codebuild.yaml"), +] + # IAM actions the FinServ checks (FS-01..FS-69) require, by the check(s) that call # them. apigateway:GET covers get_rest_apis/get_request_validators/get_usage_plans/ # get_models. Keep this in sync with finserv_assessments/app.py. @@ -74,6 +81,56 @@ "bedrock:GetModelInvocationLoggingConfiguration", } +# IAM actions the standalone SageMaker assessment calls. Keep this in sync with +# sagemaker_assessments/app.py. +REQUIRED_SAGEMAKER_ACTIONS = { + "sagemaker:ListNotebookInstances", + "sagemaker:DescribeNotebookInstance", + "sagemaker:ListDomains", + "sagemaker:DescribeDomain", + "sagemaker:ListTrainingJobs", + "sagemaker:DescribeTrainingJob", + "sagemaker:ListModelPackageGroups", + "sagemaker:ListModelPackages", + "sagemaker:ListFeatureGroups", + "sagemaker:DescribeFeatureGroup", + "sagemaker:ListPipelines", + "sagemaker:ListPipelineExecutions", + "sagemaker:ListProcessingJobs", + "sagemaker:DescribeProcessingJob", + "sagemaker:ListMonitoringSchedules", + "sagemaker:DescribeMonitoringSchedule", + "sagemaker:ListModels", + "sagemaker:DescribeModel", + "sagemaker:ListEndpoints", + "sagemaker:DescribeEndpoint", + "sagemaker:ListDataQualityJobDefinitions", + "sagemaker:DescribeDataQualityJobDefinition", + "sagemaker:ListTransformJobs", + "sagemaker:DescribeTransformJob", + "sagemaker:ListHyperParameterTuningJobs", + "sagemaker:DescribeHyperParameterTuningJob", + "sagemaker:ListCompilationJobs", + "sagemaker:DescribeCompilationJob", + "sagemaker:ListAutoMLJobs", + "sagemaker:DescribeAutoMLJob", + "sagemaker:ListExperiments", + "sagemaker:ListTrials", + "sagemaker:ListAssociations", +} + +REQUIRED_AGENTCORE_ACTIONS = { + "bedrock-agentcore:ListAgentRuntimes", + "bedrock-agentcore:GetAgentRuntime", + "bedrock-agentcore:ListMemories", + "bedrock-agentcore:GetMemory", + "bedrock-agentcore:ListGateways", + "bedrock-agentcore:GetGateway", + "bedrock-agentcore:ListPolicyEngines", + "bedrock-agentcore:GetPolicyEngine", + "bedrock-agentcore:GetResourcePolicy", +} + _ACTION_RE = re.compile(r"-\s+([a-z0-9-]+:[A-Za-z0-9]+)") @@ -101,14 +158,49 @@ def test_guard_detects_a_removed_action(monkeypatch): assert "bedrock:ListTagsForResource" in missing +@pytest.mark.parametrize("template", _TEMPLATES, ids=lambda p: os.path.basename(p)) +def test_required_sagemaker_actions_are_granted(template): + assert os.path.exists(template), f"template not found: {template}" + granted = _granted_actions(template) + missing = sorted(a for a in REQUIRED_SAGEMAKER_ACTIONS if a not in granted) + assert not missing, ( + f"{os.path.basename(template)} is missing required SageMaker IAM action(s): " + f"{missing}. Add them or a SageMaker check will hit AccessDenied." + ) + + +@pytest.mark.parametrize( + "template", + _AGENTCORE_PERMISSION_TEMPLATES, + ids=lambda p: os.path.basename(p), +) +def test_required_agentcore_actions_are_granted(template): + assert os.path.exists(template), f"template not found: {template}" + granted = _granted_actions(template) + missing = sorted(a for a in REQUIRED_AGENTCORE_ACTIONS if a not in granted) + assert not missing, ( + f"{os.path.basename(template)} is missing required AgentCore IAM action(s): " + f"{missing}. Add them or an AgentCore check will hit AccessDenied." + ) + + # Known service prefixes used by this tool. `bedrock-agent:` is intentionally # absent: it is NOT a valid IAM namespace. Amazon Bedrock Knowledge Base / Data # Source / Flow / Agent actions all use the `bedrock:` prefix; AgentCore uses # `bedrock-agentcore:`. A `bedrock-agent:` grant silently authorizes nothing. -_INVALID_ACTION_PREFIXES = ("bedrock-agent:",) +_INVALID_ACTION_PREFIXES = ("bedrock-agent:", "bedrock-agentcore-control:") +_INVALID_ACTION_NAMES = { + "bedrock:ListModelInvocations", + "bedrock-agentcore:GetAgentRuntimeResourcePolicy", + "bedrock-agentcore:GetGatewayResourcePolicy", +} -@pytest.mark.parametrize("template", _TEMPLATES, ids=lambda p: os.path.basename(p)) +@pytest.mark.parametrize( + "template", + _AGENTCORE_PERMISSION_TEMPLATES, + ids=lambda p: os.path.basename(p), +) def test_no_invalid_iam_action_prefixes(template): """Guard against reintroducing the invalid `bedrock-agent:` IAM prefix. @@ -118,18 +210,37 @@ def test_no_invalid_iam_action_prefixes(template): """ granted = _granted_actions(template) bad = sorted( - a for a in granted if any(a.startswith(p) for p in _INVALID_ACTION_PREFIXES) + a + for a in granted + if any(a.startswith(p) for p in _INVALID_ACTION_PREFIXES) + or a in _INVALID_ACTION_NAMES ) assert not bad, ( - f"{os.path.basename(template)} uses invalid IAM action prefix(es): {bad}. " + f"{os.path.basename(template)} uses invalid IAM action(s): {bad}. " "Bedrock KB/DataSource/Flow/Agent actions use the 'bedrock:' prefix " - "(AgentCore uses 'bedrock-agentcore:'); 'bedrock-agent:' is not a real " - "IAM namespace and grants nothing." + "(AgentCore uses 'bedrock-agentcore:'); boto3 client names such as " + "'bedrock-agent' and 'bedrock-agentcore-control' are not IAM namespaces. " + "AgentCore resource policies use the generic bedrock-agentcore:GetResourcePolicy " + "action." ) def test_invalid_prefix_guard_detects_a_bad_action(): """Self-test: the invalid-prefix guard trips on a `bedrock-agent:` action.""" - sample = {"bedrock:ListKnowledgeBases", "bedrock-agent:ListKnowledgeBases"} - bad = [a for a in sample if any(a.startswith(p) for p in _INVALID_ACTION_PREFIXES)] - assert bad == ["bedrock-agent:ListKnowledgeBases"] + sample = { + "bedrock:ListKnowledgeBases", + "bedrock-agent:ListKnowledgeBases", + "bedrock-agentcore-control:GetResourcePolicy", + "bedrock-agentcore:GetGatewayResourcePolicy", + } + bad = sorted( + a + for a in sample + if any(a.startswith(p) for p in _INVALID_ACTION_PREFIXES) + or a in _INVALID_ACTION_NAMES + ) + assert bad == [ + "bedrock-agent:ListKnowledgeBases", + "bedrock-agentcore-control:GetResourcePolicy", + "bedrock-agentcore:GetGatewayResourcePolicy", + ] diff --git a/tests/test_sagemaker_checks.py b/tests/test_sagemaker_checks.py index f226273..aad9695 100644 --- a/tests/test_sagemaker_checks.py +++ b/tests/test_sagemaker_checks.py @@ -14,8 +14,7 @@ import importlib.util from unittest.mock import patch, MagicMock -sys.path.insert(0, os.path.join(os.path.dirname(__file__))) -from conftest import extract_csv_data, assert_finding_schema +from tests.test_helpers import extract_csv_data, assert_finding_schema # Load sagemaker app module directly to avoid name collisions _sm_dir = os.path.abspath(