Add HTTP-first web fetch tool

README.md

@@ -78,6 +78,25 @@ Selectors remain supported, but ref-based interaction is the preferred 2.0 flow:
 2. act on `@eN` refs
 3. re-snapshot
 
+### Stateless Web Fetch
+
+```elixir
+{:ok, result} =
+  Jido.Browser.web_fetch(
+    "https://example.com/docs",
+    format: :markdown,
+    allowed_domains: ["example.com"],
+    focus_terms: ["API", "authentication"],
+    citations: true
+  )
+
+result.content
+result.passages
+result.metadata # present when extraction returns document metadata
+```
+
+`web_fetch/2` keeps HTML handling native for selector extraction and markdown conversion, and uses `extractous_ex` for fetched binary documents such as PDFs, Word, Excel, PowerPoint, OpenDocument, EPUB, and common email formats. Binary document responses may also include `result.metadata` when extraction returns document metadata.
+
 ### State Persistence
 
 ```elixir
@@ -143,6 +162,19 @@ config :jido_browser, :web,
   profile: "default"
 ```
 
+Optional web fetch settings:
+
+```elixir
+config :jido_browser, :web_fetch,
+  cache_ttl_ms: 300_000,
+  extractous: [
+    pdf: [extract_annotation_text: true],
+    office: [include_headers_and_footers: true]
+  ]
+```
+
+Configured `extractous` options are merged with any per-call `extractous:` keyword options passed to `Jido.Browser.web_fetch/2`.
+
 ## Backends
 
 ### AgentBrowser (Default)
@@ -173,6 +205,7 @@ Core operations:
 - `type/4`
 - `screenshot/2`
 - `extract_content/2`
+- `web_fetch/2`
 - `evaluate/3`
 
 Agent-browser-native operations:
@@ -252,6 +285,7 @@ Agent-browser-native operations:
 - `ReadPage`
 - `SnapshotUrl`
 - `SearchWeb`
+- `WebFetch`
 
 ## Using With Jido Agents
 

lib/jido_browser.ex

@@ -8,11 +8,13 @@ defmodule Jido.Browser do
 
   alias Jido.Browser.Error
   alias Jido.Browser.Session
+  alias Jido.Browser.WebFetch
 
   @default_adapter Jido.Browser.Adapters.AgentBrowser
   @default_timeout 30_000
   @supported_screenshot_formats [:png]
   @supported_extract_formats [:markdown, :html, :text]
+  @supported_web_fetch_formats [:markdown, :html, :text]
 
   @doc "Starts a browser session using the configured adapter or an explicit adapter override."
   @spec start_session(keyword()) :: {:ok, Session.t()} | {:error, term()}
@@ -107,6 +109,34 @@ defmodule Jido.Browser do
     end
   end
 
+  @doc """
+  Fetches a URL over HTTP(S) without starting a browser session.
+
+  HTML responses keep native selector extraction and format conversion, while
+  fetched binary documents such as PDFs and office files are extracted through
+  `ExtractousEx`.
+  """
+  @spec web_fetch(String.t(), keyword()) :: {:ok, map()} | {:error, term()}
+  def web_fetch(url, opts \\ [])
+
+  def web_fetch(url, _opts) when url in [nil, ""] do
+    {:error, Error.invalid_error("URL cannot be nil or empty", %{url: url})}
+  end
+
+  def web_fetch(url, opts) when is_binary(url) do
+    format = opts[:format] || :markdown
+
+    if format in @supported_web_fetch_formats do
+      WebFetch.fetch(url, normalize_timeout(opts))
+    else
+      {:error,
+       Error.invalid_error("Unsupported web fetch format: #{inspect(format)}", %{
+         format: format,
+         supported: @supported_web_fetch_formats
+       })}
+    end
+  end
+
   @doc "Evaluates JavaScript in the browser when the adapter supports it."
   @spec evaluate(Session.t(), String.t(), keyword()) ::
           {:ok, Session.t(), map()} | {:error, term()}

lib/jido_browser/actions/web_fetch.ex

@@ -0,0 +1,89 @@
+defmodule Jido.Browser.Actions.WebFetch do
+  @moduledoc """
+  Stateless HTTP-first document retrieval for agent workflows.
+
+  `WebFetch` is a lighter-weight alternative to browser navigation when the
+  target content can be retrieved over plain HTTP(S) without JavaScript
+  execution, including fetched PDFs and office-style documents.
+  """
+
+  use Jido.Action,
+    name: "web_fetch",
+    description:
+      "Fetch a URL over HTTP(S) with domain policy controls, Extractous-backed document extraction, " <>
+        "optional focused filtering, approximate token caps, and citation-ready passages.",
+    category: "Browser",
+    tags: ["browser", "web", "fetch", "http", "retrieval"],
+    vsn: "2.0.0",
+    schema: [
+      url: [type: :string, required: true, doc: "The URL to fetch"],
+      format: [type: {:in, [:markdown, :text, :html]}, default: :markdown, doc: "Output format"],
+      selector: [type: :string, doc: "Optional CSS selector for HTML pages"],
+      allowed_domains: [type: {:list, :string}, default: [], doc: "Allow-list of host or host/path rules"],
+      blocked_domains: [type: {:list, :string}, default: [], doc: "Block-list of host or host/path rules"],
+      focus_terms: [type: {:list, :string}, default: [], doc: "Terms used to filter the fetched document"],
+      focus_window: [type: :integer, default: 0, doc: "Paragraph window around each focus match"],
+      max_content_tokens: [type: :integer, doc: "Approximate token cap for returned content"],
+      citations: [type: :boolean, default: false, doc: "Include citation-ready passage offsets"],
+      cache: [type: :boolean, default: true, doc: "Reuse cached fetch results when available"],
+      timeout: [type: :integer, doc: "Receive timeout in milliseconds"],
+      require_known_url: [type: :boolean, default: false, doc: "Require the URL to already be present in tool context"],
+      known_urls: [type: {:list, :string}, default: [], doc: "Additional known URLs accepted for provenance checks"],
+      max_uses: [type: :integer, doc: "Maximum successful web fetch calls allowed in current skill state"]
+    ]
+
+  alias Jido.Browser.Error
+
+  @impl true
+  def run(params, context) do
+    with :ok <- validate_max_uses(params, context),
+         {:ok, result} <- Jido.Browser.web_fetch(params.url, build_opts(params, context)) do
+      {:ok, Map.put(result, :status, "success")}
+    else
+      {:error, error} ->
+        {:error, error}
+    end
+  end
+
+  defp build_opts(params, context) do
+    known_urls =
+      (Map.get(params, :known_urls, []) || [])
+      |> Kernel.++(get_in(context, [:skill_state, :seen_urls]) || [])
+      |> Enum.uniq()
+
+    []
+    |> maybe_put(:format, Map.get(params, :format, :markdown))
+    |> maybe_put(:selector, params[:selector])
+    |> maybe_put(:allowed_domains, Map.get(params, :allowed_domains, []))
+    |> maybe_put(:blocked_domains, Map.get(params, :blocked_domains, []))
+    |> maybe_put(:focus_terms, Map.get(params, :focus_terms, []))
+    |> maybe_put(:focus_window, Map.get(params, :focus_window, 0))
+    |> maybe_put(:max_content_tokens, params[:max_content_tokens])
+    |> maybe_put(:citations, Map.get(params, :citations, false))
+    |> maybe_put(:cache, Map.get(params, :cache, true))
+    |> maybe_put(:timeout, params[:timeout])
+    |> maybe_put(:require_known_url, Map.get(params, :require_known_url, false))
+    |> maybe_put(:known_urls, known_urls)
+  end
+
+  defp validate_max_uses(%{max_uses: max_uses}, context) when is_integer(max_uses) and max_uses >= 0 do
+    current_uses = get_in(context, [:skill_state, :web_fetch_uses]) || 0
+
+    if current_uses >= max_uses do
+      {:error,
+       Error.invalid_error("Web fetch max uses exceeded", %{
+         error_code: :max_uses_exceeded,
+         max_uses: max_uses,
+         current_uses: current_uses
+       })}
+    else
+      :ok
+    end
+  end
+
+  defp validate_max_uses(_params, _context), do: :ok
+
+  defp maybe_put(opts, _key, nil), do: opts
+  defp maybe_put(opts, _key, []), do: opts
+  defp maybe_put(opts, key, value), do: Keyword.put(opts, key, value)
+end

lib/jido_browser/plugin.ex

@@ -36,6 +36,7 @@ require Jido.Browser.Actions.WaitForSelector
 require Jido.Browser.Actions.ReadPage
 require Jido.Browser.Actions.SnapshotUrl
 require Jido.Browser.Actions.SearchWeb
+require Jido.Browser.Actions.WebFetch
 
 defmodule Jido.Browser.Plugin do
   @moduledoc """
@@ -119,7 +120,8 @@ defmodule Jido.Browser.Plugin do
       # Self-contained composite actions (manage own session)
       Jido.Browser.Actions.ReadPage,
       Jido.Browser.Actions.SnapshotUrl,
-      Jido.Browser.Actions.SearchWeb
+      Jido.Browser.Actions.SearchWeb,
+      Jido.Browser.Actions.WebFetch
     ],
     description: "Browser automation for web navigation, interaction, and content extraction",
     category: "browser",
@@ -136,7 +138,9 @@ defmodule Jido.Browser.Plugin do
       viewport: Map.get(config, :viewport, %{width: 1280, height: 720}),
       base_url: Map.get(config, :base_url),
       last_url: nil,
-      last_title: nil
+      last_title: nil,
+      seen_urls: [],
+      web_fetch_uses: 0
     }
 
     {:ok, initial_state}
@@ -151,7 +155,9 @@ defmodule Jido.Browser.Plugin do
       viewport: Zoi.any(description: "Browser viewport dimensions") |> Zoi.optional(),
       base_url: Zoi.string(description: "Base URL for relative navigation") |> Zoi.optional(),
       last_url: Zoi.string(description: "Last navigated URL") |> Zoi.optional(),
-      last_title: Zoi.string(description: "Last page title") |> Zoi.optional()
+      last_title: Zoi.string(description: "Last page title") |> Zoi.optional(),
+      seen_urls: Zoi.array(Zoi.string(description: "Known URLs discovered during tool use")) |> Zoi.default([]),
+      web_fetch_uses: Zoi.integer(description: "Successful web fetch calls in current skill state") |> Zoi.default(0)
     })
   end
 
@@ -204,7 +210,8 @@ defmodule Jido.Browser.Plugin do
       # Self-contained composite actions
       {"browser.read_page", Jido.Browser.Actions.ReadPage},
       {"browser.snapshot_url", Jido.Browser.Actions.SnapshotUrl},
-      {"browser.search_web", Jido.Browser.Actions.SearchWeb}
+      {"browser.search_web", Jido.Browser.Actions.SearchWeb},
+      {"browser.web_fetch", Jido.Browser.Actions.WebFetch}
     ]
   end
 
@@ -214,22 +221,17 @@ defmodule Jido.Browser.Plugin do
   end
 
   @impl Jido.Plugin
-  def transform_result(_action, {:ok, result}, _context) when is_map(result) do
-    case Map.get(result, :session) do
-      %Jido.Browser.Session{} = session ->
-        current_url = Map.get(result, :url) || Map.get(result, "url") || get_in(session, [:connection, :current_url])
-        current_title = Map.get(result, :title) || Map.get(result, "title") || get_in(session, [:connection, :title])
-
-        state_updates = %{
-          session: session,
-          last_url: current_url,
-          last_title: current_title
-        }
-
-        {:ok, result, state_updates}
+  def transform_result(action, {:ok, result}, context) when is_map(result) do
+    state_updates =
+      %{}
+      |> maybe_put_session_state(result)
+      |> maybe_put_seen_urls(result, context)
+      |> maybe_increment_web_fetch_uses(action, context)
 
-      _ ->
-        {:ok, result}
+    if map_size(state_updates) == 0 do
+      {:ok, result}
+    else
+      {:ok, result, state_updates}
     end
   end
 
@@ -260,6 +262,67 @@ defmodule Jido.Browser.Plugin do
     end
   end
 
+  defp maybe_put_session_state(acc, result) do
+    case Map.get(result, :session) do
+      %Jido.Browser.Session{} = session ->
+        current_url = Map.get(result, :url) || Map.get(result, "url") || get_in(session, [:connection, :current_url])
+        current_title = Map.get(result, :title) || Map.get(result, "title") || get_in(session, [:connection, :title])
+
+        Map.merge(acc, %{
+          session: session,
+          last_url: current_url,
+          last_title: current_title
+        })
+
+      _ ->
+        acc
+    end
+  end
+
+  defp maybe_put_seen_urls(acc, result, context) do
+    current_seen_urls = get_in(context, [:skill_state, :seen_urls]) || []
+
+    seen_urls =
+      current_seen_urls
+      |> Kernel.++(extract_urls(result))
+      |> Enum.reject(&nil_or_empty?/1)
+      |> Enum.uniq()
+
+    if seen_urls == [] or seen_urls == current_seen_urls do
+      acc
+    else
+      Map.put(acc, :seen_urls, seen_urls)
+    end
+  end
+
+  defp maybe_increment_web_fetch_uses(acc, Jido.Browser.Actions.WebFetch, context) do
+    current_uses = get_in(context, [:skill_state, :web_fetch_uses]) || 0
+    Map.put(acc, :web_fetch_uses, current_uses + 1)
+  end
+
+  defp maybe_increment_web_fetch_uses(acc, _action, _context), do: acc
+
+  defp extract_urls(result) do
+    direct_urls =
+      [Map.get(result, :url), Map.get(result, "url"), Map.get(result, :final_url), Map.get(result, "final_url")]
+      |> Enum.reject(&nil_or_empty?/1)
+
+    search_urls =
+      result
+      |> Map.get(:results, Map.get(result, "results", []))
+      |> List.wrap()
+      |> Enum.map(fn item ->
+        if is_map(item), do: Map.get(item, :url) || Map.get(item, "url")
+      end)
+      |> Enum.reject(&nil_or_empty?/1)
+
+    direct_urls ++ search_urls
+  end
+
+  defp nil_or_empty?(nil), do: true
+  defp nil_or_empty?(""), do: true
+  defp nil_or_empty?(_value), do: false
+
   def signal_patterns do
     [
       # Session lifecycle
@@ -308,7 +371,8 @@ defmodule Jido.Browser.Plugin do
       # Self-contained composite actions
       "browser.read_page",
       "browser.snapshot_url",
-      "browser.search_web"
+      "browser.search_web",
+      "browser.web_fetch"
     ]
   end
 end