FIX: Curate Scam scenario DEFAULT strategy (#2127)

Author: varunj-msft

doc/scanner/airt.ipynb

@@ -1131,7 +1131,9 @@
     "  --max-dataset-size 1\n",
     "```\n",
     "\n",
-    "**Available strategies:** ALL, SINGLE_TURN, MULTI_TURN, ContextCompliance, RolePlay, PersuasiveRedTeamingAttack"
+    "**Available strategies:** ALL, DEFAULT, SINGLE_TURN, MULTI_TURN, ContextCompliance, RolePlay,\n",
+    "PersuasiveRedTeamingAttack. DEFAULT runs the single-turn techniques (ContextCompliance, RolePlay)\n",
+    "and omits the slower multi-turn PersuasiveRedTeamingAttack; run it via ALL or MULTI_TURN."
    ]
   },
   {

doc/scanner/airt.py

@@ -242,7 +242,9 @@
 #   --max-dataset-size 1
 # ```
 #
-# **Available strategies:** ALL, SINGLE_TURN, MULTI_TURN, ContextCompliance, RolePlay, PersuasiveRedTeamingAttack
+# **Available strategies:** ALL, DEFAULT, SINGLE_TURN, MULTI_TURN, ContextCompliance, RolePlay,
+# PersuasiveRedTeamingAttack. DEFAULT runs the single-turn techniques (ContextCompliance, RolePlay)
+# and omits the slower multi-turn PersuasiveRedTeamingAttack; run it via ALL or MULTI_TURN.
 
 # %%
 from pyrit.scenario.airt import Scam, ScamStrategy

pyrit/scenario/scenarios/airt/scam.py

@@ -44,14 +44,24 @@ class ScamStrategy(ScenarioStrategy):
         to be used during training seminars.
     - PersuasiveRedTeamingAttack: This multi-turn attack uses a persuasive persona with the `RedTeamingAttack` to
         iteratively convince the target to comply with the scam objective over multiple turns.
+
+    Aggregate Values:
+    - ALL: Every technique (both single-turn and the multi-turn PersuasiveRedTeamingAttack).
+    - DEFAULT: The single-turn techniques (ContextCompliance, RolePlay). Excludes the multi-turn
+        PersuasiveRedTeamingAttack so the default run stays fast; opt into it via ALL or MULTI_TURN.
+    - SINGLE_TURN / MULTI_TURN: Group techniques by turn count. DEFAULT intentionally shares
+        membership with SINGLE_TURN today (both are the single-turn techniques); they are kept
+        distinct because DEFAULT is the recommended fast default while SINGLE_TURN is a turn-count
+        grouping, and their membership may diverge later.
     """
 
     ALL = ("all", {"all"})
+    DEFAULT = ("default", {"default"})
     SINGLE_TURN = ("single_turn", {"single_turn"})
     MULTI_TURN = ("multi_turn", {"multi_turn"})
 
-    ContextCompliance = ("context_compliance", {"single_turn"})
-    RolePlay = ("role_play", {"single_turn"})
+    ContextCompliance = ("context_compliance", {"single_turn", "default"})
+    RolePlay = ("role_play", {"single_turn", "default"})
     PersuasiveRedTeamingAttack = ("persuasive_rta", {"multi_turn"})
 
     @classmethod
@@ -63,7 +73,7 @@ def get_aggregate_tags(cls) -> set[str]:
             set[str]: Set of tags that are aggregate markers.
         """
         # Include base class aggregates ("all") and add scenario-specific ones
-        return super().get_aggregate_tags() | {"single_turn", "multi_turn"}
+        return super().get_aggregate_tags() | {"default", "single_turn", "multi_turn"}
 
 
 class Scam(Scenario):
@@ -72,7 +82,7 @@ class Scam(Scenario):
     (e.g., phishing emails, fraudulent messages) with primarily persuasion-oriented techniques.
     """
 
-    VERSION: int = 1
+    VERSION: int = 2
 
     @classmethod
     def _get_additional_scoring_questions(cls) -> list[Path]:
@@ -135,7 +145,7 @@ def __init__(
         super().__init__(
             version=self.VERSION,
             strategy_class=ScamStrategy,
-            default_strategy=ScamStrategy.ALL,
+            default_strategy=ScamStrategy.DEFAULT,
             default_dataset_config=DatasetAttackConfiguration(dataset_names=["airt_scams"], max_dataset_size=4),
             objective_scorer=objective_scorer,
             scenario_result_id=scenario_result_id,

tests/unit/scenario/airt/test_scam.py

@@ -115,6 +115,26 @@ def mock_adversarial_target() -> PromptTarget:
 FIXTURES = ["patch_central_database", "mock_runtime_env"]
 
 
+class TestScamStrategyEnum:
+    """Aggregate expansion for ScamStrategy (DEFAULT curation)."""
+
+    def test_default_expands_to_single_turn_only(self):
+        members = {m.value for m in ScamStrategy.expand({ScamStrategy.DEFAULT})}
+        assert members == {"context_compliance", "role_play"}
+
+    def test_default_excludes_persuasive_rta(self):
+        members = {m.value for m in ScamStrategy.expand({ScamStrategy.DEFAULT})}
+        assert "persuasive_rta" not in members
+
+    def test_all_includes_persuasive_rta(self):
+        members = {m.value for m in ScamStrategy.expand({ScamStrategy.ALL})}
+        assert members == {"context_compliance", "role_play", "persuasive_rta"}
+
+    def test_default_is_aggregate(self):
+        assert "default" in ScamStrategy.get_aggregate_tags()
+        assert ScamStrategy.DEFAULT in ScamStrategy.get_aggregate_strategies()
+
+
 @pytest.mark.usefixtures(*FIXTURES)
 class TestScamInitialization:
     """Tests for Scam initialization."""
@@ -134,7 +154,11 @@ def test_init_with_default_objectives(
             scenario = Scam(objective_scorer=mock_objective_scorer)
 
             assert scenario.name == "Scam"
-            assert scenario.VERSION == 1
+            assert scenario.VERSION == 2
+
+    def test_default_strategy_is_default(self, mock_objective_scorer) -> None:
+        scenario = Scam(objective_scorer=mock_objective_scorer)
+        assert scenario._default_strategy == ScamStrategy.DEFAULT
 
     def test_init_with_default_scorer(self, mock_memory_seed_groups) -> None:
         """Test initialization with default scorer."""
@@ -219,7 +243,7 @@ class TestScamAttackGeneration:
     async def test_attack_generation_for_all(
         self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups, mock_dataset_config
     ):
-        """Test that _get_atomic_attacks_async returns atomic attacks."""
+        """ALL runs every technique, including the multi-turn PersuasiveRedTeamingAttack."""
         with patch.object(
             Scam,
             "_resolve_seed_groups_by_dataset_async",
@@ -228,11 +252,41 @@ async def test_attack_generation_for_all(
         ):
             scenario = Scam(objective_scorer=mock_objective_scorer)
 
-            await scenario.initialize_async(objective_target=mock_objective_target, dataset_config=mock_dataset_config)
+            await scenario.initialize_async(
+                objective_target=mock_objective_target,
+                scenario_strategies=[ScamStrategy.ALL],
+                dataset_config=mock_dataset_config,
+                include_baseline=False,
+            )
+            atomic_attacks = scenario._atomic_attacks
+
+            assert len(atomic_attacks) == 3
+            attack_types = {type(run.attack_technique.attack) for run in atomic_attacks}
+            assert attack_types == {ContextComplianceAttack, RolePlayAttack, RedTeamingAttack}
+
+    async def test_default_run_yields_single_turn_only(
+        self, mock_objective_target, mock_objective_scorer, mock_memory_seed_groups, mock_dataset_config
+    ):
+        """No explicit strategies -> DEFAULT -> only the two single-turn techniques, no persuasive_rta."""
+        with patch.object(
+            Scam,
+            "_resolve_seed_groups_by_dataset_async",
+            new_callable=AsyncMock,
+            return_value={"memory": mock_memory_seed_groups},
+        ):
+            scenario = Scam(objective_scorer=mock_objective_scorer)
+
+            await scenario.initialize_async(
+                objective_target=mock_objective_target,
+                dataset_config=mock_dataset_config,
+                include_baseline=False,
+            )
             atomic_attacks = scenario._atomic_attacks
 
-            assert len(atomic_attacks) > 0
-            assert all(run.attack_technique is not None for run in atomic_attacks)
+            assert len(atomic_attacks) == 2
+            attack_types = {type(run.attack_technique.attack) for run in atomic_attacks}
+            assert attack_types == {ContextComplianceAttack, RolePlayAttack}
+            assert RedTeamingAttack not in attack_types
 
     async def test_attack_generation_for_singleturn_async(
         self,
@@ -429,7 +483,7 @@ def test_scenario_version_is_set(
             objective_scorer=mock_objective_scorer,
         )
 
-        assert scenario.VERSION == 1
+        assert scenario.VERSION == 2
 
     async def test_no_target_duplication_async(
         self,