Merge branch 'main' into ds-params

Author: behnam-o

.github/instructions/scenarios.instructions.md

@@ -163,6 +163,8 @@ The default implementation:
    `AttackTechniqueRegistry` singleton)
 2. Iterates over every (technique × dataset) pair from `self._dataset_config`
 3. Calls `factory.create()` with `objective_target` and conditional scorer override
+   (also forwards any per-technique converters from `self._strategy_converters`, populated
+   from the CLI `--strategies <technique>:converter.<name>` modifier, as `extra_request_converters`)
 4. Uses `self._build_display_group()` for user-facing grouping
 5. Builds `AtomicAttack` with unique `atomic_attack_name` = `"{technique}_{dataset}"`
 

doc/references.bib

@@ -99,9 +99,12 @@ @misc{odin2024
 @article{inie2025summon,
   title     = {Summon a Demon and Bind it: A Grounded Theory of {LLM} Red Teaming},
   author    = {Nanna Inie and Jonathan Stray and Leon Derczynski},
-  journal   = {PLoS ONE},
+  journal   = {PLOS ONE},
+  volume    = {20},
+  number    = {1},
+  pages     = {e0314658},
   year      = {2025},
-  url       = {https://arxiv.org/abs/2311.06237},
+  url       = {https://doi.org/10.1371/journal.pone.0314658},
 }
 
 @misc{vantaylor2024socialbias,
@@ -633,17 +636,19 @@ @article{rottger2025msts
   url       = {https://arxiv.org/abs/2501.10057},
 }
 
-@article{zong2024vlguard,
+@inproceedings{zong2024vlguard,
   title     = {Safety Fine-Tuning at (Almost) No Cost: A Baseline for Vision Large Language Models},
   author    = {Yongshuo Zong and Ondrej Bohdal and Tingyang Yu and Yongxin Yang and Timothy Hospedales},
-  journal   = {arXiv preprint arXiv:2402.02207},
+  booktitle = {Proceedings of the 41st International Conference on Machine Learning (ICML)},
+  pages     = {62867--62891},
   year      = {2024},
-  url       = {https://arxiv.org/abs/2402.02207},
+  publisher = {PMLR},
+  url       = {https://proceedings.mlr.press/v235/zong24a.html},
 }
 
 @article{lopez2024pyrit,
   title     = {{PyRIT}: A Framework for Security Risk Identification and Red Teaming in Generative {AI} Systems},
-  author    = {Gary D. Lopez Munoz and Amanda J. Minnich and Roman Lutz and Richard Lundeen and Raja Sekhar Rao Dheekonda and Nina Chikanov and Bolor-Erdene Jagdagdorj and Martin Pouliot and Shiven Chawla and Whitney Maxwell and Blake Bullwinkel and Katherine Pratt and Joris de Gruyter and Charlotte Siska and Pete Bryan and Tori Westerhoff and Chang Kawaguchi and Christian Seifert and Ram Shankar Siva Kumar and Yonatan Zunger},
+  author    = {Gary D. {Lopez Munoz} and Amanda J. Minnich and Roman Lutz and Richard Lundeen and Raja Sekhar Rao Dheekonda and Nina Chikanov and Bolor-Erdene Jagdagdorj and Martin Pouliot and Shiven Chawla and Whitney Maxwell and Blake Bullwinkel and Katherine Pratt and Joris de Gruyter and Charlotte Siska and Pete Bryan and Tori Westerhoff and Chang Kawaguchi and Christian Seifert and Ram Shankar Siva Kumar and Yonatan Zunger},
   journal   = {arXiv preprint arXiv:2410.02828},
   year      = {2024},
   url       = {https://arxiv.org/abs/2410.02828},
@@ -667,12 +672,13 @@ @inproceedings{wang2025siuo
   note      = {Introduces the {SIUO} (Safe Inputs but Unsafe Output) benchmark},
 }
 
-@misc{darkbench2025,
-  title     = {{DarkBench}: A Comprehensive Benchmark for Dark Design Patterns in Large Language Models},
-  author    = {{Apart Research}},
+@inproceedings{darkbench2025,
+  title     = {{DarkBench}: Benchmarking Dark Patterns in Large Language Models},
+  author    = {Esben Kran and Hieu Minh Nguyen and Akash Kundu and Sami Jawhar and Jinsuk Park and Mateusz Maria Jurewicz},
+  booktitle = {International Conference on Learning Representations (ICLR)},
   year      = {2025},
-  url       = {https://darkbench.ai/},
-  note      = {OpenReview: https://openreview.net/forum?id=odjMSBSWRt},
+  url       = {https://arxiv.org/abs/2503.10728},
+  note      = {Oral presentation at ICLR 2025},
 }
 
 @misc{embracethered2025sneakybits,

doc/scanner/1_pyrit_scan.ipynb

@@ -163,6 +163,36 @@
    "cell_type": "markdown",
    "id": "9",
    "metadata": {},
+   "source": [
+    "#### Attaching Converters to a Technique\n",
+    "\n",
+    "Strategies (techniques) can have a registered converter instance appended to them with the\n",
+    "`<technique>:converter.<name>` syntax. The converter is added to the request side of every attack\n",
+    "the technique produces, on top of any converters the technique already bakes in. This also works on\n",
+    "aggregate strategies (the converter is applied to every technique the aggregate expands to).\n",
+    "\n",
+    "First discover the registered converter instances with `--list-converters` (converters are\n",
+    "registered by initializers, so pass the same `--initializers`/`--initialization-scripts` you use to run):\n",
+    "\n",
+    "```shell\n",
+    "pyrit_scan --list-converters --initializers my_converters\n",
+    "```\n",
+    "\n",
+    "Then reference a converter by name in `--strategies`:\n",
+    "\n",
+    "```shell\n",
+    "# Add the registered \"translation_spanish\" converter to role_play only\n",
+    "pyrit_scan airt.rapid_response --target openai_chat --initializers load_default_datasets target my_converters --strategies role_play:converter.translation_spanish\n",
+    "\n",
+    "# Chain multiple converters (applied in order) and combine with plain strategies\n",
+    "pyrit_scan airt.rapid_response --target openai_chat --initializers load_default_datasets target my_converters --strategies role_play:converter.translation_spanish:converter.base64 many_shot\n",
+    "```"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "10",
+   "metadata": {},
    "source": [
     "#### Using Custom Scenarios\n",
     "\n",
@@ -172,7 +202,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "10",
+   "id": "11",
    "metadata": {
     "lines_to_next_cell": 2
    },
@@ -225,7 +255,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "11",
+   "id": "12",
    "metadata": {},
    "source": [
     "Then discover and run it:\n",

doc/scanner/1_pyrit_scan.py

@@ -114,6 +114,31 @@
 # pyrit_scan garak.encoding --initialization-scripts ./my_custom_config.py
 # ```
 
+# %% [markdown]
+# #### Attaching Converters to a Technique
+#
+# Strategies (techniques) can have a registered converter instance appended to them with the
+# `<technique>:converter.<name>` syntax. The converter is added to the request side of every attack
+# the technique produces, on top of any converters the technique already bakes in. This also works on
+# aggregate strategies (the converter is applied to every technique the aggregate expands to).
+#
+# First discover the registered converter instances with `--list-converters` (converters are
+# registered by initializers, so pass the same `--initializers`/`--initialization-scripts` you use to run):
+#
+# ```shell
+# pyrit_scan --list-converters --initializers my_converters
+# ```
+#
+# Then reference a converter by name in `--strategies`:
+#
+# ```shell
+# # Add the registered "translation_spanish" converter to role_play only
+# pyrit_scan airt.rapid_response --target openai_chat --initializers load_default_datasets target my_converters --strategies role_play:converter.translation_spanish
+#
+# # Chain multiple converters (applied in order) and combine with plain strategies
+# pyrit_scan airt.rapid_response --target openai_chat --initializers load_default_datasets target my_converters --strategies role_play:converter.translation_spanish:converter.base64 many_shot
+# ```
+
 # %% [markdown]
 # #### Using Custom Scenarios
 #

doc/scanner/2_pyrit_shell.md

@@ -40,6 +40,7 @@ Once starting the shell, you will see the list of commands you have access to. S
 | `list-scenarios` | List all available scenarios |
 | `list-initializers` | List all available initializers |
 | `list-targets` | List all available targets from the registry |
+| `list-converters` | List all registered converter instances |
 | `run <scenario> [options]` | Run a scenario with optional parameters |
 | `scenario-history` | List all previous scenario runs in this session |
 | `print-scenario [N]` | Print detailed results for scenario run(s) |
@@ -65,6 +66,21 @@ pyrit> run garak.encoding --target my_target --initializers target --strategies
 pyrit> run foundry.red_team_agent --target my_target --initializers target -s jailbreak crescendo
 ```
 
+### Attaching Converters to a Technique
+
+Append a registered converter instance to a single technique (or an aggregate strategy) with the
+`<technique>:converter.<name>` syntax. The converter is added to the request side of every attack
+the technique produces, on top of any converters the technique already bakes in. Use
+`list-converters` to discover the registered converter names:
+
+```bash
+# Add the registered "translation_spanish" converter to role_play only
+pyrit> run airt.rapid_response --target my_target --initializers target load_default_datasets -s role_play:converter.translation_spanish
+
+# Chain multiple converters (applied in order) and combine with plain strategies
+pyrit> run airt.rapid_response --target my_target --initializers target load_default_datasets -s role_play:converter.translation_spanish:converter.base64 many_shot
+```
+
 ### With Runtime Parameters
 
 ```bash

frontend/e2e/errors.spec.ts

@@ -264,6 +264,29 @@ test.describe("Error: backend 500 on send message", () => {
     // The failed text should be restored in the input for easy re-send
     await expect(input).toHaveValue("This should fail", { timeout: 5000 });
   });
+
+  test("should recover cleanly when the first send fails", async ({ page }) => {
+    await mockAllAPIs(page, async (route) => {
+      await route.fulfill({
+        status: 500,
+        contentType: "application/json",
+        body: JSON.stringify({ detail: "Internal server error" }),
+      });
+    });
+
+    await page.goto("/");
+    await activateMockTarget(page);
+
+    const input = page.getByRole("textbox");
+    await input.fill("First send fails");
+    await page.getByRole("button", { name: /send/i }).click();
+
+    await expect(page.getByText(/Internal server error/i)).toBeVisible({
+      timeout: 10000,
+    });
+    await expect(page.getByTestId("loading-state")).toHaveCount(0);
+    await expect(input).toHaveValue("First send fails", { timeout: 5000 });
+  });
 });
 
 // ---------------------------------------------------------------------------

frontend/src/components/Chat/ChatWindow.tsx

@@ -347,8 +347,21 @@ export default function ChatWindow({
         setLoadedConversationId(effectiveConvId!)
       }
     } catch (err) {
+      const viewedConversationId = viewedConvRef.current
+      const isViewingFailedConversation = viewedConversationId === sendConvId
+        || viewedConversationId === (activeConversationId ?? conversationId)
+        || (viewedConversationId == null && sendConvId !== '__pending__')
+
       // Only show error in UI if user is still on this conversation
-      if (viewedConvRef.current === sendConvId || viewedConvRef.current === (activeConversationId ?? conversationId)) {
+      if (isViewingFailedConversation) {
+        // Mark the viewed conversation as loaded so first-send failures do not
+        // get stuck behind the "Loading conversation..." placeholder.
+        if (viewedConversationId) {
+          setLoadedConversationId(viewedConversationId)
+        } else if (sendConvId !== '__pending__') {
+          setLoadedConversationId(sendConvId)
+        }
+
         const apiError = toApiError(err)
         let description: string
         if (apiError.isNetworkError) {

pyrit/backend/models/targets.py

@@ -9,11 +9,12 @@
 directly.
 """
 
-from typing import Any, Literal
+from typing import Literal
 
 from pydantic import BaseModel, Field
 
 from pyrit.backend.models.common import PaginationInfo
+from pyrit.models import JSONValue
 from pyrit.models.catalog.target import TargetInstance
 
 __all__ = [
@@ -33,7 +34,7 @@ class CreateTargetRequest(BaseModel):
     """Request to create a new target instance."""
 
     type: str = Field(..., description="Target type (e.g., 'OpenAIChatTarget')")
-    params: dict[str, Any] = Field(default_factory=dict, description="Target constructor parameters")
+    params: dict[str, JSONValue] = Field(default_factory=dict, description="Target constructor parameters")
     auth_mode: Literal["api_key", "entra"] = Field(
         "api_key",
         description=(