Add a hast-producing render path to the unified pipeline (#61774)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: heiskr

src/content-render/index.ts

@@ -1,7 +1,8 @@
 import { renderLiquid } from './liquid/index'
-import { renderMarkdown, renderUnified } from './unified/index'
+import { renderMarkdown, renderUnified, renderUnifiedToHast } from './unified/index'
 import { engine } from './liquid/engine'
 import type { Context } from '@/types'
+import type { Root as HastRoot } from 'hast'
 import { createLogger } from '@/observability/logger'
 
 const logger = createLogger(import.meta.url)
@@ -66,4 +67,42 @@ function getDefaultCacheKey(template: string, context: Context): string {
   return `${template}:${context.currentVersion}:${context.currentLanguage}`
 }
 
+/**
+ * Like `renderContent`, but returns the hast (HTML AST) tree alongside the
+ * derived HTML string, both produced from a single unified pass. Used by the
+ * render-page boundary to thread a serializable AST to the React layer instead
+ * of an opaque HTML string, so React can render the body itself rather than
+ * injecting raw HTML (github/docs-engineering#6619).
+ *
+ * Does liquid first (same as renderContent), then runs the unified pipeline but
+ * stops before rehype-stringify. The `html` returned here is derived from the
+ * exact same tree, so it stays consistent with `hast`.
+ *
+ * Unlike `renderContent`, this does not support `context.markdownRequested`:
+ * the hast path always produces HTML. Callers that need markdown output should
+ * use `renderContent`.
+ */
+export async function renderContentToHast(
+  template = '',
+  context: Context = {} as Context,
+  options: Pick<RenderOptions, 'filename'> = {},
+): Promise<{ html: string; hast: HastRoot | null }> {
+  if (!template) return { html: template, hast: null }
+  if (context.markdownRequested) {
+    throw new Error(
+      'renderContentToHast does not support markdownRequested; use renderContent for markdown output',
+    )
+  }
+  try {
+    const liquidRendered = await renderLiquid(template, context)
+    const { hast, html } = await renderUnifiedToHast(liquidRendered, context)
+    return { html, hast }
+  } catch (error) {
+    if (options.filename) {
+      logger.error('renderContentToHast failed on file', { filename: options.filename })
+    }
+    throw error
+  }
+}
+
 export const liquid = engine

src/content-render/tests/render-to-hast.ts

@@ -0,0 +1,116 @@
+import { describe, expect, test } from 'vitest'
+
+import { renderContentToHast } from '@/content-render/index'
+import { renderUnified, renderUnifiedToHast } from '@/content-render/unified/index'
+import type { Context } from '@/types'
+
+// A corpus that exercises the parts of the pipeline most likely to differ
+// between "stringify the processed vfile" (today) and "stringify the hast tree
+// we stopped at" (the new hast path): headings (slug + anchor links), code
+// blocks (highlight + code-header), tables (several rewrite plugins), alerts,
+// raw inline HTML (rehype-raw), and images.
+const fixtures: Array<{ name: string; template: string }> = [
+  { name: 'paragraph', template: 'Hello **world**, this is a [link](https://github.com).' },
+  {
+    name: 'headings',
+    template: '# Title\n\n## Section one\n\nSome text.\n\n### Subsection\n\nMore text.',
+  },
+  {
+    name: 'fenced code block',
+    template: '```js\nconst x = 1\nconsole.log(x)\n```',
+  },
+  {
+    name: 'table',
+    template: '| Col A | Col B |\n| --- | --- |\n| one | two |\n| three | four |',
+  },
+  {
+    name: 'ordered list with nesting',
+    template: '1. First\n1. Second\n   - nested a\n   - nested b\n1. Third',
+  },
+  {
+    name: 'alert',
+    template: '> [!NOTE]\n> This is an alert body with a [link](/foo).',
+  },
+  {
+    name: 'inline raw html',
+    template: 'A paragraph with <kbd>Ctrl</kbd> and <em>emphasis</em> inline.',
+  },
+  {
+    name: 'blockquote and emphasis',
+    template: '> A quote\n>\n> spanning lines with _emphasis_ and `code`.',
+  },
+  {
+    name: 'asset image (png -> picture)',
+    template: '![Alternative text](/assets/images/help.png)',
+  },
+  {
+    name: 'external image',
+    template: '![A logo](https://example.com/logo.png)',
+  },
+]
+
+describe('renderUnifiedToHast', () => {
+  test.each(fixtures)('derived html matches renderUnified for: $name', async ({ template }) => {
+    const context = {} as Context
+    const stringOutput = await renderUnified(template, structuredClone(context))
+    const { html, hast } = await renderUnifiedToHast(template, structuredClone(context))
+
+    expect(html).toBe(stringOutput)
+    expect(hast).toBeTruthy()
+    expect(hast.type).toBe('root')
+    expect(Array.isArray(hast.children)).toBe(true)
+  })
+
+  test('returns a hast root with element children for a heading', async () => {
+    const { hast } = await renderUnifiedToHast('# Hello', {} as Context)
+    const firstElement = hast.children.find((node) => node.type === 'element')
+    expect(firstElement).toBeTruthy()
+    expect(firstElement && 'tagName' in firstElement && firstElement.tagName).toBe('h1')
+  })
+
+  test('strips position metadata from every node', async () => {
+    const { hast } = await renderUnifiedToHast(
+      '# Title\n\nA paragraph with a [link](/foo) and `code`.',
+      {} as Context,
+    )
+    const hasPosition = (node: unknown): boolean => {
+      if (!node || typeof node !== 'object') return false
+      if ('position' in node) return true
+      const children = (node as { children?: unknown[] }).children
+      return Array.isArray(children) && children.some(hasPosition)
+    }
+    expect(hasPosition(hast)).toBe(false)
+  })
+})
+
+describe('renderContentToHast', () => {
+  test('renders liquid before the unified pass', async () => {
+    const context = { page: { foo: 'bar' } } as unknown as Context
+    const { html, hast } = await renderContentToHast('A {{ page.foo }} value.', context)
+
+    expect(html).toContain('A bar value.')
+    expect(hast).toBeTruthy()
+    expect(hast?.type).toBe('root')
+  })
+
+  test('derived html matches the inner renderUnified path after liquid', async () => {
+    const context = { page: { foo: 'bar' } } as unknown as Context
+    const template = '# {{ page.foo }}\n\nWith a [link](/foo).'
+    const { html } = await renderContentToHast(template, structuredClone(context))
+    const expected = await renderUnified('# bar\n\nWith a [link](/foo).', {} as Context)
+
+    expect(html).toBe(expected)
+  })
+
+  test('returns null hast for an empty template', async () => {
+    const { html, hast } = await renderContentToHast('', {} as Context)
+    expect(html).toBe('')
+    expect(hast).toBeNull()
+  })
+
+  test('throws when markdownRequested is set', async () => {
+    await expect(
+      renderContentToHast('# Hello', { markdownRequested: true } as Context),
+    ).rejects.toThrow(/does not support markdownRequested/)
+  })
+})

src/content-render/unified/index.ts

@@ -1,3 +1,5 @@
+import type { Processor } from 'unified'
+import type { Nodes as HastNodes, Root as HastRoot } from 'hast'
 import { fastTextOnly } from '@/content-render/unified/text-only'
 import { createProcessor, createMarkdownOnlyProcessor } from '@/content-render/unified/processor'
 import type { Context } from '@/types'
@@ -22,6 +24,47 @@ export async function renderUnified(
   return html.trim()
 }
 
+/**
+ * Run the same unified pipeline as `renderUnified`, but stop before the terminal
+ * rehype-stringify step and return the hast (HTML AST) tree instead of a string.
+ *
+ * The 18 transform plugins all run during `processor.run()`; rehype-stringify is
+ * only invoked by `processor.stringify()`. So we get the fully transformed hast
+ * from a single pass, then derive the legacy HTML string from that exact same
+ * tree. This guarantees the string and hast can never drift, and avoids running
+ * the pipeline twice for consumers that still need the string (mini-TOC text,
+ * search indexing, validators).
+ */
+export async function renderUnifiedToHast(
+  template: string,
+  context: Context,
+): Promise<{ hast: HastRoot; html: string }> {
+  const processor = createProcessor(context) as unknown as Processor
+  const mdast = processor.parse(template)
+  const hast = (await processor.run(mdast)) as HastNodes
+  const html = processor.stringify(hast).toString()
+  // Strip `position` data (line/column offsets) the parser leaves on every node.
+  // It is useless to the client and meaningfully inflates the serialized tree we
+  // ship through the Next props boundary. Done after deriving `html` so the
+  // string output is byte-identical to the legacy path.
+  stripPositions(hast)
+  return { hast: hast as HastRoot, html: html.trim() }
+}
+
+/**
+ * Recursively delete `position` fields from a hast/unist tree in place.
+ * Avoids adding a dependency (unist-util-remove-position) for one field.
+ */
+function stripPositions(node: HastNodes): void {
+  if (node && typeof node === 'object') {
+    if ('position' in node) delete (node as { position?: unknown }).position
+    const children = (node as { children?: HastNodes[] }).children
+    if (Array.isArray(children)) {
+      for (const child of children) stripPositions(child)
+    }
+  }
+}
+
 export async function renderMarkdown(template: string, context: Context) {
   const processor = createMarkdownOnlyProcessor(context)
   const vFile = await processor.process(template)