add pagefind search

.gitignore

@@ -19,6 +19,7 @@ coverage
 dist
 tmp
 /out-tsc
+pagefind/
 
 # dependencies
 node_modules
@@ -70,4 +71,4 @@ gha-creds-*.json
 .vercel
 
 vite.config.*.timestamp*
-vitest.config.*.timestamp*
+vitest.config.*.timestamp*

examples/cosmo-cargo/zudoku.config.tsx

@@ -47,6 +47,9 @@ const config: ZudokuConfig = {
   defaults: {
     examplesLanguage: "js",
   },
+  search: {
+    type: "pagefind",
+  },
   apis: [
     {
       type: "file",

packages/zudoku/package.json

@@ -35,6 +35,7 @@
     "./plugins/redirect": "./src/lib/plugins/redirect/index.ts",
     "./plugins/custom-pages": "./src/lib/plugins/custom-pages/index.ts",
     "./plugins/search-inkeep": "./src/lib/plugins/search-inkeep/index.ts",
+    "./plugins/search-pagefind": "./src/lib/plugins/search-pagefind/index.ts",
     "./plugins/api-catalog": "./src/lib/plugins/api-catalog/index.ts",
     "./components": "./src/lib/components/index.ts",
     "./icons": "./src/lib/icons.ts",
@@ -107,6 +108,10 @@
         "import": "./lib/zudoku.plugin-search-inkeep.js",
         "types": "./dist/lib/plugins/search-inkeep/index.d.ts"
       },
+      "./plugins/search-pagefind": {
+        "import": "./lib/zudoku.plugin-search-pagefind.js",
+        "types": "./dist/lib/plugins/search-pagefind/index.d.ts"
+      },
       "./components": {
         "import": "./lib/zudoku.components.js",
         "types": "./dist/lib/components/index.d.ts"

packages/zudoku/src/config/validators/common.ts

@@ -203,14 +203,27 @@ const Redirect = z.object({
 });
 
 const SearchSchema = z
-  .object({
-    type: z.literal("inkeep"),
-    apiKey: z.string(),
-    integrationId: z.string(),
-    organizationId: z.string(),
-    primaryBrandColor: z.string(),
-    organizationDisplayName: z.string(),
-  })
+  .union([
+    z.object({
+      type: z.literal("inkeep"),
+      apiKey: z.string(),
+      integrationId: z.string(),
+      organizationId: z.string(),
+      primaryBrandColor: z.string(),
+      organizationDisplayName: z.string(),
+    }),
+    z.object({
+      type: z.literal("pagefind"),
+      ranking: z
+        .object({
+          termFrequency: z.number(),
+          pageLength: z.number(),
+          termSimilarity: z.number(),
+          termSaturation: z.number(),
+        })
+        .optional(),
+    }),
+  ])
   .optional();
 
 const AuthenticationSchema = z.union([

packages/zudoku/src/lib/components/Banner.tsx

@@ -34,6 +34,7 @@ export const Banner = () => {
         "relative text-primary-foreground text-sm font-medium px-4 py-2 flex gap-2 items-center",
         mappedColor,
       )}
+      data-pagefind-ignore
       style={style}
     >
       <div className="w-full">{page.banner.message}</div>

packages/zudoku/src/lib/components/Layout.tsx

@@ -89,6 +89,7 @@ export const Layout = ({ children }: { children?: ReactNode }) => {
                 </DrawerTrigger>
               </div>
               <main
+                data-pagefind-body
                 className={cn(
                   "h-full dark:border-white/10 translate-x-0",
                   "lg:overflow-visible",

packages/zudoku/src/lib/plugins/openapi/Sidecar.tsx

@@ -175,6 +175,7 @@ export const Sidecar = ({
     <aside
       ref={ref}
       className="flex flex-col overflow-hidden sticky top-[--scroll-padding] gap-4"
+      data-pagefind-ignore
     >
       <SidecarBox.Root>
         <SidecarBox.Head className="flex justify-between items-center flex-nowrap py-2.5 gap-2 text-xs">

packages/zudoku/src/lib/plugins/search-pagefind/PagefindSearch.tsx

@@ -0,0 +1,144 @@
+import { VisuallyHidden } from "@radix-ui/react-visually-hidden";
+import { useQuery } from "@tanstack/react-query";
+import { useState } from "react";
+import { useNavigate } from "react-router";
+import {
+  CommandDialog,
+  CommandEmpty,
+  CommandGroup,
+  CommandInput,
+  CommandItem,
+  CommandList,
+} from "zudoku/ui/Command.js";
+import { DialogTitle } from "zudoku/ui/Dialog.js";
+import type { PagefindOptions } from "./index.js";
+import { Pagefind, PagefindSubResult } from "./types.js";
+
+const DEFAULT_RANKING = {
+  // Slightly lower than default because API docs tend to have repetitive terms (parameter names, HTTP methods, etc.)
+  termFrequency: 0.8,
+  // Lower than default because API documentation pages tend to be longer due to comprehensive endpoint documentation
+  pageLength: 0.6,
+  // Slightly higher than default because in technical documentation, exact matches should be prioritized
+  termSimilarity: 1.2,
+  // Slightly lower than default because API docs might have legitimate repetition of terms
+  termSaturation: 1.2,
+};
+
+const usePagefind = (options: PagefindOptions) => {
+  const { data: pagefind, ...rest } = useQuery<Pagefind>({
+    queryKey: ["pagefind", options.ranking],
+    queryFn: async () => {
+      return await import(
+        /* @vite-ignore */ `${location.origin}/pagefind/pagefind.js`
+      ).then(async (pagefind: Pagefind) => {
+        await pagefind.init();
+        await pagefind.options({
+          ranking: {
+            termFrequency:
+              options.ranking?.termFrequency ?? DEFAULT_RANKING.termFrequency,
+            pageLength:
+              options.ranking?.pageLength ?? DEFAULT_RANKING.pageLength,
+            termSimilarity:
+              options.ranking?.termSimilarity ?? DEFAULT_RANKING.termSimilarity,
+            termSaturation:
+              options.ranking?.termSaturation ?? DEFAULT_RANKING.termSaturation,
+          },
+        });
+
+        return pagefind;
+      });
+    },
+    enabled: typeof window !== "undefined",
+  });
+
+  return { ...rest, pagefind };
+};
+
+const sortSubResults = (a: PagefindSubResult, b: PagefindSubResult) => {
+  const aScore = a.weighted_locations.reduce(
+    (sum, loc) => sum + loc.balanced_score,
+    0,
+  );
+  const bScore = b.weighted_locations.reduce(
+    (sum, loc) => sum + loc.balanced_score,
+    0,
+  );
+  return bScore - aScore;
+};
+
+export const PagefindSearch = ({
+  isOpen,
+  onClose,
+  options,
+}: {
+  isOpen: boolean;
+  onClose: () => void;
+  options: PagefindOptions;
+}) => {
+  const { pagefind } = usePagefind(options);
+  const [searchTerm, setSearchTerm] = useState("");
+  const navigate = useNavigate();
+
+  const { data: searchResults, isLoading } = useQuery({
+    queryKey: ["pagefind-search", searchTerm],
+    queryFn: async () => {
+      const search = await pagefind?.search(searchTerm);
+      return Promise.all(
+        search?.results.slice(0, 3).map((subResult) => subResult.data()) ?? [],
+      );
+    },
+    enabled: !!pagefind && !!searchTerm,
+  });
+
+  return (
+    <CommandDialog
+      command={{ shouldFilter: false }}
+      open={isOpen}
+      onOpenChange={onClose}
+    >
+      <VisuallyHidden>
+        <DialogTitle>Search</DialogTitle>
+      </VisuallyHidden>
+      <CommandInput
+        placeholder="Search..."
+        value={searchTerm}
+        onValueChange={(e) => setSearchTerm(e)}
+      />
+      <CommandEmpty>No results found.</CommandEmpty>
+      <CommandList>
+        {searchResults?.map((result, i) => (
+          <CommandGroup
+            heading={result.meta.title ?? "" + i}
+            key={[result.meta.title, result.meta.url, result.excerpt]
+              .filter(Boolean)
+              .join("-")}
+          >
+            {result.sub_results
+              .sort(sortSubResults)
+              .slice(0, 3)
+              .map((subResult) => (
+                <CommandItem
+                  key={result.meta.title + subResult.url + subResult.excerpt}
+                  className="flex flex-col items-start"
+                  onSelect={() => {
+                    void navigate(subResult.url.replace(".html", ""));
+                    onClose();
+                  }}
+                >
+                  <span className="font-bold">{subResult.title}</span>
+                  <span
+                    className="text-xs"
+                    dangerouslySetInnerHTML={{ __html: subResult.excerpt }}
+                  />
+                  <span className="text-xs text-muted-foreground">
+                    {subResult.url}
+                  </span>
+                </CommandItem>
+              ))}
+          </CommandGroup>
+        ))}
+      </CommandList>
+    </CommandDialog>
+  );
+};

packages/zudoku/src/lib/plugins/search-pagefind/index.tsx

@@ -0,0 +1,21 @@
+import type { ZudokuConfig } from "../../../config/validators/validate.js";
+import { ClientOnly } from "../../components/ClientOnly.js";
+import type { ZudokuPlugin } from "../../core/plugins.js";
+import { PagefindSearch } from "./PagefindSearch.js";
+
+export type PagefindOptions = Extract<
+  ZudokuConfig["search"],
+  { type: "pagefind" }
+>;
+
+export const pagefindSearchPlugin = (
+  options: PagefindOptions,
+): ZudokuPlugin => {
+  return {
+    renderSearch: ({ isOpen, onClose }) => (
+      <ClientOnly>
+        <PagefindSearch isOpen={isOpen} onClose={onClose} options={options} />
+      </ClientOnly>
+    ),
+  };
+};

packages/zudoku/src/lib/plugins/search-pagefind/types.ts

@@ -0,0 +1,118 @@
+interface PagefindIndexOptions {
+  basePath?: string;
+  baseUrl?: string;
+  excerptLength?: number;
+  indexWeight?: number;
+  mergeFilter?: Record<string, unknown>;
+  highlightParam?: string;
+  language?: string;
+  primary?: boolean;
+  ranking?: PagefindRankingWeights;
+}
+
+interface PagefindRankingWeights {
+  termSimilarity?: number;
+  pageLength?: number;
+  termSaturation?: number;
+  termFrequency?: number;
+}
+
+interface PagefindSearchOptions {
+  preload?: boolean;
+  verbose?: boolean;
+  filters?: Record<string, unknown>;
+  sort?: Record<string, unknown>;
+}
+
+type PagefindFilterCounts = Record<string, Record<string, number>> & {};
+
+interface PagefindSearchResults {
+  results: PagefindSearchResult[];
+  unfilteredResultCount: number;
+  filters: PagefindFilterCounts;
+  totalFilters: PagefindFilterCounts;
+  timings: {
+    preload: number;
+    search: number;
+    total: number;
+  };
+}
+
+interface PagefindSearchResult {
+  id: string;
+  score: number;
+  words: number[];
+  data: () => Promise<PagefindSearchFragment>;
+}
+
+interface PagefindSearchFragment {
+  url: string;
+  raw_url?: string;
+  content: string;
+  raw_content?: string;
+  excerpt: string;
+  sub_results: PagefindSubResult[];
+  word_count: number;
+  locations: number[];
+  weighted_locations: PagefindWordLocation[];
+  filters: Record<string, string[]>;
+  meta: Record<string, string>;
+  anchors: PagefindSearchAnchor[];
+}
+
+interface PagefindSubResult {
+  title: string;
+  url: string;
+  locations: number[];
+  weighted_locations: PagefindWordLocation[];
+  excerpt: string;
+  anchor?: PagefindSearchAnchor;
+}
+
+interface PagefindWordLocation {
+  weight: number;
+  balanced_score: number;
+  location: number;
+}
+
+interface PagefindSearchAnchor {
+  element: string;
+  id: string;
+  text?: string;
+  location: number;
+}
+
+interface Pagefind {
+  debouncedSearch: (
+    query: string,
+    options?: PagefindSearchOptions,
+    duration?: number,
+  ) => Promise<PagefindSearchResults>;
+  destroy: () => Promise<void>;
+  filters: () => Promise<PagefindFilterCounts>;
+  init: () => Promise<void>;
+  mergeIndex: (
+    indexPath: string,
+    options?: Record<string, unknown>,
+  ) => Promise<void>;
+  options: (options: PagefindIndexOptions) => Promise<void>;
+  preload: (term: string, options?: PagefindIndexOptions) => Promise<void>;
+  search: (
+    term: string,
+    options?: PagefindSearchOptions,
+  ) => Promise<PagefindSearchResults>;
+}
+
+export type {
+  Pagefind,
+  PagefindFilterCounts,
+  PagefindIndexOptions,
+  PagefindRankingWeights,
+  PagefindSearchAnchor,
+  PagefindSearchFragment,
+  PagefindSearchOptions,
+  PagefindSearchResult,
+  PagefindSearchResults,
+  PagefindSubResult,
+  PagefindWordLocation,
+};